From 476a9c09d64423e97b08958c4088fde29a1b1713 Mon Sep 17 00:00:00 2001 From: FatihMakes Date: Wed, 16 Sep 2026 18:48:54 +0300 Subject: [PATCH] Add files via upload --- LICENSE | 834 ++++++++--------- actions/code_helper.py | 29 +- actions/computer_control.py | 13 +- actions/desktop.py | 8 +- actions/dev_agent.py | 19 +- actions/file_controller.py | 15 +- actions/file_processor.py | 15 +- actions/flight_finder.py | 25 +- actions/web_search.py | 207 +++-- actions/youtube_video.py | 16 +- core/action_loader.py | 414 +++++---- core/avatar.py | 715 +++++++++++++++ core/avatar_mesh.py | 351 ++++++++ core/echo.py | 288 ++++++ core/face_model.obj | 1378 ++++++++++++++++++++++++++++ core/gemini.py | 439 +++++++++ core/hotkey.py | 173 ++++ core/plugin_loader.py | 506 ++++++----- core/prompt.txt | 207 +++-- core/viseme.py | 275 ++++++ core/wake_word.py | 413 ++++----- dashboard/server.py | 1678 ++++++++++++++++++----------------- main.py | 697 +++++++++++++-- memory/config_manager.py | 146 ++- readme.md | 179 +++- requirements.txt | 151 ++-- setup.py | 201 +++-- ui.py | 1166 +++++++++++++++++++----- 28 files changed, 8099 insertions(+), 2459 deletions(-) create mode 100644 core/avatar.py create mode 100644 core/avatar_mesh.py create mode 100644 core/echo.py create mode 100644 core/face_model.obj create mode 100644 core/gemini.py create mode 100644 core/hotkey.py create mode 100644 core/viseme.py diff --git a/LICENSE b/LICENSE index f878bc8..e606301 100644 --- a/LICENSE +++ b/LICENSE @@ -1,417 +1,417 @@ -MARK 53 — JARVIS -Copyright (c) 2026 FatihMakes - -Licensed under the Creative Commons Attribution-NonCommercial 4.0 -International License (CC BY-NC 4.0). Commercial use is not permitted. -Canonical text: https://creativecommons.org/licenses/by-nc/4.0/legalcode - ------------------------------------------------------------------------ - -Attribution-NonCommercial 4.0 International - -======================================================================= - -Creative Commons Corporation ("Creative Commons") is not a law firm and -does not provide legal services or legal advice. Distribution of -Creative Commons public licenses does not create a lawyer-client or -other relationship. Creative Commons makes its licenses and related -information available on an "as-is" basis. Creative Commons gives no -warranties regarding its licenses, any material licensed under their -terms and conditions, or any related information. Creative Commons -disclaims all liability for damages resulting from their use to the -fullest extent possible. - -Using Creative Commons Public Licenses - -Creative Commons public licenses provide a standard set of terms and -conditions that creators and other rights holders may use to share -original works of authorship and other material subject to copyright -and certain other rights specified in the public license below. The -following considerations are for informational purposes only, are not -exhaustive, and do not form part of our licenses. - - Considerations for licensors: Our public licenses are - intended for use by those authorized to give the public - permission to use material in ways otherwise restricted by - copyright and certain other rights. Our licenses are - irrevocable. Licensors should read and understand the terms - and conditions of the license they choose before applying it. - Licensors should also secure all rights necessary before - applying our licenses so that the public can reuse the - material as expected. Licensors should clearly mark any - material not subject to the license. This includes other CC- - licensed material, or material used under an exception or - limitation to copyright. More considerations for licensors: - wiki.creativecommons.org/Considerations_for_licensors - - Considerations for the public: By using one of our public - licenses, a licensor grants the public permission to use the - licensed material under specified terms and conditions. If - the licensor's permission is not necessary for any reason--for - example, because of any applicable exception or limitation to - copyright--then that use is not regulated by the license. Our - licenses grant only permissions under copyright and certain - other rights that a licensor has authority to grant. Use of - the licensed material may still be restricted for other - reasons, including because others have copyright or other - rights in the material. A licensor may make special requests, - such as asking that all changes be marked or described. - Although not required by our licenses, you are encouraged to - respect those requests where reasonable. More considerations - for the public: - wiki.creativecommons.org/Considerations_for_licensees - -======================================================================= - -Creative Commons Attribution-NonCommercial 4.0 International Public -License - -By exercising the Licensed Rights (defined below), You accept and agree -to be bound by the terms and conditions of this Creative Commons -Attribution-NonCommercial 4.0 International Public License ("Public -License"). To the extent this Public License may be interpreted as a -contract, You are granted the Licensed Rights in consideration of Your -acceptance of these terms and conditions, and the Licensor grants You -such rights in consideration of benefits the Licensor receives from -making the Licensed Material available under these terms and -conditions. - - -Section 1 -- Definitions. - - a. Adapted Material means material subject to Copyright and Similar - Rights that is derived from or based upon the Licensed Material - and in which the Licensed Material is translated, altered, - arranged, transformed, or otherwise modified in a manner requiring - permission under the Copyright and Similar Rights held by the - Licensor. For purposes of this Public License, where the Licensed - Material is a musical work, performance, or sound recording, - Adapted Material is always produced where the Licensed Material is - synched in timed relation with a moving image. - - b. Adapter's License means the license You apply to Your Copyright - and Similar Rights in Your contributions to Adapted Material in - accordance with the terms and conditions of this Public License. - - c. Copyright and Similar Rights means copyright and/or similar rights - closely related to copyright including, without limitation, - performance, broadcast, sound recording, and Sui Generis Database - Rights, without regard to how the rights are labeled or - categorized. For purposes of this Public License, the rights - specified in Section 2(b)(1)-(2) are not Copyright and Similar - Rights. - d. Effective Technological Measures means those measures that, in the - absence of proper authority, may not be circumvented under laws - fulfilling obligations under Article 11 of the WIPO Copyright - Treaty adopted on December 20, 1996, and/or similar international - agreements. - - e. Exceptions and Limitations means fair use, fair dealing, and/or - any other exception or limitation to Copyright and Similar Rights - that applies to Your use of the Licensed Material. - - f. Licensed Material means the artistic or literary work, database, - or other material to which the Licensor applied this Public - License. - - g. Licensed Rights means the rights granted to You subject to the - terms and conditions of this Public License, which are limited to - all Copyright and Similar Rights that apply to Your use of the - Licensed Material and that the Licensor has authority to license. - - h. Licensor means the individual(s) or entity(ies) granting rights - under this Public License. - - i. NonCommercial means not primarily intended for or directed towards - commercial advantage or monetary compensation. For purposes of - this Public License, the exchange of the Licensed Material for - other material subject to Copyright and Similar Rights by digital - file-sharing or similar means is NonCommercial provided there is - no payment of monetary compensation in connection with the - exchange. - - j. Share means to provide material to the public by any means or - process that requires permission under the Licensed Rights, such - as reproduction, public display, public performance, distribution, - dissemination, communication, or importation, and to make material - available to the public including in ways that members of the - public may access the material from a place and at a time - individually chosen by them. - - k. Sui Generis Database Rights means rights other than copyright - resulting from Directive 96/9/EC of the European Parliament and of - the Council of 11 March 1996 on the legal protection of databases, - as amended and/or succeeded, as well as other essentially - equivalent rights anywhere in the world. - - l. You means the individual or entity exercising the Licensed Rights - under this Public License. Your has a corresponding meaning. - - -Section 2 -- Scope. - - a. License grant. - - 1. Subject to the terms and conditions of this Public License, - the Licensor hereby grants You a worldwide, royalty-free, - non-sublicensable, non-exclusive, irrevocable license to - exercise the Licensed Rights in the Licensed Material to: - - a. reproduce and Share the Licensed Material, in whole or - in part, for NonCommercial purposes only; and - - b. produce, reproduce, and Share Adapted Material for - NonCommercial purposes only. - - 2. Exceptions and Limitations. For the avoidance of doubt, where - Exceptions and Limitations apply to Your use, this Public - License does not apply, and You do not need to comply with - its terms and conditions. - - 3. Term. The term of this Public License is specified in Section - 6(a). - - 4. Media and formats; technical modifications allowed. The - Licensor authorizes You to exercise the Licensed Rights in - all media and formats whether now known or hereafter created, - and to make technical modifications necessary to do so. The - Licensor waives and/or agrees not to assert any right or - authority to forbid You from making technical modifications - necessary to exercise the Licensed Rights, including - technical modifications necessary to circumvent Effective - Technological Measures. For purposes of this Public License, - simply making modifications authorized by this Section 2(a) - (4) never produces Adapted Material. - - 5. Downstream recipients. - - a. Offer from the Licensor -- Licensed Material. Every - recipient of the Licensed Material automatically - receives an offer from the Licensor to exercise the - Licensed Rights under the terms and conditions of this - Public License. - - b. No downstream restrictions. You may not offer or impose - any additional or different terms or conditions on, or - apply any Effective Technological Measures to, the - Licensed Material if doing so restricts exercise of the - Licensed Rights by any recipient of the Licensed - Material. - - 6. No endorsement. Nothing in this Public License constitutes or - may be construed as permission to assert or imply that You - are, or that Your use of the Licensed Material is, connected - with, or sponsored, endorsed, or granted official status by, - the Licensor or others designated to receive attribution as - provided in Section 3(a)(1)(A)(i). - - b. Other rights. - - 1. Moral rights, such as the right of integrity, are not - licensed under this Public License, nor are publicity, - privacy, and/or other similar personality rights; however, to - the extent possible, the Licensor waives and/or agrees not to - assert any such rights held by the Licensor to the limited - extent necessary to allow You to exercise the Licensed - Rights, but not otherwise. - - 2. Patent and trademark rights are not licensed under this - Public License. - - 3. To the extent possible, the Licensor waives any right to - collect royalties from You for the exercise of the Licensed - Rights, whether directly or through a collecting society - under any voluntary or waivable statutory or compulsory - licensing scheme. In all other cases the Licensor expressly - reserves any right to collect such royalties, including when - the Licensed Material is used other than for NonCommercial - purposes. - - -Section 3 -- License Conditions. - -Your exercise of the Licensed Rights is expressly made subject to the -following conditions. - - a. Attribution. - - 1. If You Share the Licensed Material (including in modified - form), You must: - - a. retain the following if it is supplied by the Licensor - with the Licensed Material: - - i. identification of the creator(s) of the Licensed - Material and any others designated to receive - attribution, in any reasonable manner requested by - the Licensor (including by pseudonym if - designated); - - ii. a copyright notice; - - iii. a notice that refers to this Public License; - - iv. a notice that refers to the disclaimer of - warranties; - - v. a URI or hyperlink to the Licensed Material to the - extent reasonably practicable; - - b. indicate if You modified the Licensed Material and - retain an indication of any previous modifications; and - - c. indicate the Licensed Material is licensed under this - Public License, and include the text of, or the URI or - hyperlink to, this Public License. - - 2. You may satisfy the conditions in Section 3(a)(1) in any - reasonable manner based on the medium, means, and context in - which You Share the Licensed Material. For example, it may be - reasonable to satisfy the conditions by providing a URI or - hyperlink to a resource that includes the required - information. - - 3. If requested by the Licensor, You must remove any of the - information required by Section 3(a)(1)(A) to the extent - reasonably practicable. - - 4. If You Share Adapted Material You produce, the Adapter's - License You apply must not prevent recipients of the Adapted - Material from complying with this Public License. - - -Section 4 -- Sui Generis Database Rights. - -Where the Licensed Rights include Sui Generis Database Rights that -apply to Your use of the Licensed Material: - - a. for the avoidance of doubt, Section 2(a)(1) grants You the right - to extract, reuse, reproduce, and Share all or a substantial - portion of the contents of the database for NonCommercial purposes - only; - - b. if You include all or a substantial portion of the database - contents in a database in which You have Sui Generis Database - Rights, then the database in which You have Sui Generis Database - Rights (but not its individual contents) is Adapted Material; and - - c. You must comply with the conditions in Section 3(a) if You Share - all or a substantial portion of the contents of the database. - -For the avoidance of doubt, this Section 4 supplements and does not -replace Your obligations under this Public License where the Licensed -Rights include other Copyright and Similar Rights. - - -Section 5 -- Disclaimer of Warranties and Limitation of Liability. - - a. UNLESS OTHERWISE SEPARATELY UNDERTAKEN BY THE LICENSOR, TO THE - EXTENT POSSIBLE, THE LICENSOR OFFERS THE LICENSED MATERIAL AS-IS - AND AS-AVAILABLE, AND MAKES NO REPRESENTATIONS OR WARRANTIES OF - ANY KIND CONCERNING THE LICENSED MATERIAL, WHETHER EXPRESS, - IMPLIED, STATUTORY, OR OTHER. THIS INCLUDES, WITHOUT LIMITATION, - WARRANTIES OF TITLE, MERCHANTABILITY, FITNESS FOR A PARTICULAR - PURPOSE, NON-INFRINGEMENT, ABSENCE OF LATENT OR OTHER DEFECTS, - ACCURACY, OR THE PRESENCE OR ABSENCE OF ERRORS, WHETHER OR NOT - KNOWN OR DISCOVERABLE. WHERE DISCLAIMERS OF WARRANTIES ARE NOT - ALLOWED IN FULL OR IN PART, THIS DISCLAIMER MAY NOT APPLY TO YOU. - - b. TO THE EXTENT POSSIBLE, IN NO EVENT WILL THE LICENSOR BE LIABLE - TO YOU ON ANY LEGAL THEORY (INCLUDING, WITHOUT LIMITATION, - NEGLIGENCE) OR OTHERWISE FOR ANY DIRECT, SPECIAL, INDIRECT, - INCIDENTAL, CONSEQUENTIAL, PUNITIVE, EXEMPLARY, OR OTHER LOSSES, - COSTS, EXPENSES, OR DAMAGES ARISING OUT OF THIS PUBLIC LICENSE OR - USE OF THE LICENSED MATERIAL, EVEN IF THE LICENSOR HAS BEEN - ADVISED OF THE POSSIBILITY OF SUCH LOSSES, COSTS, EXPENSES, OR - DAMAGES. WHERE A LIMITATION OF LIABILITY IS NOT ALLOWED IN FULL OR - IN PART, THIS LIMITATION MAY NOT APPLY TO YOU. - - c. The disclaimer of warranties and limitation of liability provided - above shall be interpreted in a manner that, to the extent - possible, most closely approximates an absolute disclaimer and - waiver of all liability. - - -Section 6 -- Term and Termination. - - a. This Public License applies for the term of the Copyright and - Similar Rights licensed here. However, if You fail to comply with - this Public License, then Your rights under this Public License - terminate automatically. - - b. Where Your right to use the Licensed Material has terminated under - Section 6(a), it reinstates: - - 1. automatically as of the date the violation is cured, provided - it is cured within 30 days of Your discovery of the - violation; or - - 2. upon express reinstatement by the Licensor. - - For the avoidance of doubt, this Section 6(b) does not affect any - right the Licensor may have to seek remedies for Your violations - of this Public License. - - c. For the avoidance of doubt, the Licensor may also offer the - Licensed Material under separate terms or conditions or stop - distributing the Licensed Material at any time; however, doing so - will not terminate this Public License. - - d. Sections 1, 5, 6, 7, and 8 survive termination of this Public - License. - - -Section 7 -- Other Terms and Conditions. - - a. The Licensor shall not be bound by any additional or different - terms or conditions communicated by You unless expressly agreed. - - b. Any arrangements, understandings, or agreements regarding the - Licensed Material not stated herein are separate from and - independent of the terms and conditions of this Public License. - - -Section 8 -- Interpretation. - - a. For the avoidance of doubt, this Public License does not, and - shall not be interpreted to, reduce, limit, restrict, or impose - conditions on any use of the Licensed Material that could lawfully - be made without permission under this Public License. - - b. To the extent possible, if any provision of this Public License is - deemed unenforceable, it shall be automatically reformed to the - minimum extent necessary to make it enforceable. If the provision - cannot be reformed, it shall be severed from this Public License - without affecting the enforceability of the remaining terms and - conditions. - - c. No term or condition of this Public License will be waived and no - failure to comply consented to unless expressly agreed to by the - Licensor. - - d. Nothing in this Public License constitutes or may be interpreted - as a limitation upon, or waiver of, any privileges and immunities - that apply to the Licensor or You, including from the legal - processes of any jurisdiction or authority. - -======================================================================= - -Creative Commons is not a party to its public -licenses. Notwithstanding, Creative Commons may elect to apply one of -its public licenses to material it publishes and in those instances -will be considered the “Licensor.” The text of the Creative Commons -public licenses is dedicated to the public domain under the CC0 Public -Domain Dedication. Except for the limited purpose of indicating that -material is shared under a Creative Commons public license or as -otherwise permitted by the Creative Commons policies published at -creativecommons.org/policies, Creative Commons does not authorize the -use of the trademark "Creative Commons" or any other trademark or logo -of Creative Commons without its prior written consent including, -without limitation, in connection with any unauthorized modifications -to any of its public licenses or any other arrangements, -understandings, or agreements concerning use of licensed material. For -the avoidance of doubt, this paragraph does not form part of the -public licenses. - -Creative Commons may be contacted at creativecommons.org. - +MARK LIV — JARVIS +Copyright (c) 2026 FatihMakes + +Licensed under the Creative Commons Attribution-NonCommercial 4.0 +International License (CC BY-NC 4.0). Commercial use is not permitted. +Canonical text: https://creativecommons.org/licenses/by-nc/4.0/legalcode + +----------------------------------------------------------------------- + +Attribution-NonCommercial 4.0 International + +======================================================================= + +Creative Commons Corporation ("Creative Commons") is not a law firm and +does not provide legal services or legal advice. Distribution of +Creative Commons public licenses does not create a lawyer-client or +other relationship. Creative Commons makes its licenses and related +information available on an "as-is" basis. Creative Commons gives no +warranties regarding its licenses, any material licensed under their +terms and conditions, or any related information. Creative Commons +disclaims all liability for damages resulting from their use to the +fullest extent possible. + +Using Creative Commons Public Licenses + +Creative Commons public licenses provide a standard set of terms and +conditions that creators and other rights holders may use to share +original works of authorship and other material subject to copyright +and certain other rights specified in the public license below. The +following considerations are for informational purposes only, are not +exhaustive, and do not form part of our licenses. + + Considerations for licensors: Our public licenses are + intended for use by those authorized to give the public + permission to use material in ways otherwise restricted by + copyright and certain other rights. Our licenses are + irrevocable. Licensors should read and understand the terms + and conditions of the license they choose before applying it. + Licensors should also secure all rights necessary before + applying our licenses so that the public can reuse the + material as expected. Licensors should clearly mark any + material not subject to the license. This includes other CC- + licensed material, or material used under an exception or + limitation to copyright. More considerations for licensors: + wiki.creativecommons.org/Considerations_for_licensors + + Considerations for the public: By using one of our public + licenses, a licensor grants the public permission to use the + licensed material under specified terms and conditions. If + the licensor's permission is not necessary for any reason--for + example, because of any applicable exception or limitation to + copyright--then that use is not regulated by the license. Our + licenses grant only permissions under copyright and certain + other rights that a licensor has authority to grant. Use of + the licensed material may still be restricted for other + reasons, including because others have copyright or other + rights in the material. A licensor may make special requests, + such as asking that all changes be marked or described. + Although not required by our licenses, you are encouraged to + respect those requests where reasonable. More considerations + for the public: + wiki.creativecommons.org/Considerations_for_licensees + +======================================================================= + +Creative Commons Attribution-NonCommercial 4.0 International Public +License + +By exercising the Licensed Rights (defined below), You accept and agree +to be bound by the terms and conditions of this Creative Commons +Attribution-NonCommercial 4.0 International Public License ("Public +License"). To the extent this Public License may be interpreted as a +contract, You are granted the Licensed Rights in consideration of Your +acceptance of these terms and conditions, and the Licensor grants You +such rights in consideration of benefits the Licensor receives from +making the Licensed Material available under these terms and +conditions. + + +Section 1 -- Definitions. + + a. Adapted Material means material subject to Copyright and Similar + Rights that is derived from or based upon the Licensed Material + and in which the Licensed Material is translated, altered, + arranged, transformed, or otherwise modified in a manner requiring + permission under the Copyright and Similar Rights held by the + Licensor. For purposes of this Public License, where the Licensed + Material is a musical work, performance, or sound recording, + Adapted Material is always produced where the Licensed Material is + synched in timed relation with a moving image. + + b. Adapter's License means the license You apply to Your Copyright + and Similar Rights in Your contributions to Adapted Material in + accordance with the terms and conditions of this Public License. + + c. Copyright and Similar Rights means copyright and/or similar rights + closely related to copyright including, without limitation, + performance, broadcast, sound recording, and Sui Generis Database + Rights, without regard to how the rights are labeled or + categorized. For purposes of this Public License, the rights + specified in Section 2(b)(1)-(2) are not Copyright and Similar + Rights. + d. Effective Technological Measures means those measures that, in the + absence of proper authority, may not be circumvented under laws + fulfilling obligations under Article 11 of the WIPO Copyright + Treaty adopted on December 20, 1996, and/or similar international + agreements. + + e. Exceptions and Limitations means fair use, fair dealing, and/or + any other exception or limitation to Copyright and Similar Rights + that applies to Your use of the Licensed Material. + + f. Licensed Material means the artistic or literary work, database, + or other material to which the Licensor applied this Public + License. + + g. Licensed Rights means the rights granted to You subject to the + terms and conditions of this Public License, which are limited to + all Copyright and Similar Rights that apply to Your use of the + Licensed Material and that the Licensor has authority to license. + + h. Licensor means the individual(s) or entity(ies) granting rights + under this Public License. + + i. NonCommercial means not primarily intended for or directed towards + commercial advantage or monetary compensation. For purposes of + this Public License, the exchange of the Licensed Material for + other material subject to Copyright and Similar Rights by digital + file-sharing or similar means is NonCommercial provided there is + no payment of monetary compensation in connection with the + exchange. + + j. Share means to provide material to the public by any means or + process that requires permission under the Licensed Rights, such + as reproduction, public display, public performance, distribution, + dissemination, communication, or importation, and to make material + available to the public including in ways that members of the + public may access the material from a place and at a time + individually chosen by them. + + k. Sui Generis Database Rights means rights other than copyright + resulting from Directive 96/9/EC of the European Parliament and of + the Council of 11 March 1996 on the legal protection of databases, + as amended and/or succeeded, as well as other essentially + equivalent rights anywhere in the world. + + l. You means the individual or entity exercising the Licensed Rights + under this Public License. Your has a corresponding meaning. + + +Section 2 -- Scope. + + a. License grant. + + 1. Subject to the terms and conditions of this Public License, + the Licensor hereby grants You a worldwide, royalty-free, + non-sublicensable, non-exclusive, irrevocable license to + exercise the Licensed Rights in the Licensed Material to: + + a. reproduce and Share the Licensed Material, in whole or + in part, for NonCommercial purposes only; and + + b. produce, reproduce, and Share Adapted Material for + NonCommercial purposes only. + + 2. Exceptions and Limitations. For the avoidance of doubt, where + Exceptions and Limitations apply to Your use, this Public + License does not apply, and You do not need to comply with + its terms and conditions. + + 3. Term. The term of this Public License is specified in Section + 6(a). + + 4. Media and formats; technical modifications allowed. The + Licensor authorizes You to exercise the Licensed Rights in + all media and formats whether now known or hereafter created, + and to make technical modifications necessary to do so. The + Licensor waives and/or agrees not to assert any right or + authority to forbid You from making technical modifications + necessary to exercise the Licensed Rights, including + technical modifications necessary to circumvent Effective + Technological Measures. For purposes of this Public License, + simply making modifications authorized by this Section 2(a) + (4) never produces Adapted Material. + + 5. Downstream recipients. + + a. Offer from the Licensor -- Licensed Material. Every + recipient of the Licensed Material automatically + receives an offer from the Licensor to exercise the + Licensed Rights under the terms and conditions of this + Public License. + + b. No downstream restrictions. You may not offer or impose + any additional or different terms or conditions on, or + apply any Effective Technological Measures to, the + Licensed Material if doing so restricts exercise of the + Licensed Rights by any recipient of the Licensed + Material. + + 6. No endorsement. Nothing in this Public License constitutes or + may be construed as permission to assert or imply that You + are, or that Your use of the Licensed Material is, connected + with, or sponsored, endorsed, or granted official status by, + the Licensor or others designated to receive attribution as + provided in Section 3(a)(1)(A)(i). + + b. Other rights. + + 1. Moral rights, such as the right of integrity, are not + licensed under this Public License, nor are publicity, + privacy, and/or other similar personality rights; however, to + the extent possible, the Licensor waives and/or agrees not to + assert any such rights held by the Licensor to the limited + extent necessary to allow You to exercise the Licensed + Rights, but not otherwise. + + 2. Patent and trademark rights are not licensed under this + Public License. + + 3. To the extent possible, the Licensor waives any right to + collect royalties from You for the exercise of the Licensed + Rights, whether directly or through a collecting society + under any voluntary or waivable statutory or compulsory + licensing scheme. In all other cases the Licensor expressly + reserves any right to collect such royalties, including when + the Licensed Material is used other than for NonCommercial + purposes. + + +Section 3 -- License Conditions. + +Your exercise of the Licensed Rights is expressly made subject to the +following conditions. + + a. Attribution. + + 1. If You Share the Licensed Material (including in modified + form), You must: + + a. retain the following if it is supplied by the Licensor + with the Licensed Material: + + i. identification of the creator(s) of the Licensed + Material and any others designated to receive + attribution, in any reasonable manner requested by + the Licensor (including by pseudonym if + designated); + + ii. a copyright notice; + + iii. a notice that refers to this Public License; + + iv. a notice that refers to the disclaimer of + warranties; + + v. a URI or hyperlink to the Licensed Material to the + extent reasonably practicable; + + b. indicate if You modified the Licensed Material and + retain an indication of any previous modifications; and + + c. indicate the Licensed Material is licensed under this + Public License, and include the text of, or the URI or + hyperlink to, this Public License. + + 2. You may satisfy the conditions in Section 3(a)(1) in any + reasonable manner based on the medium, means, and context in + which You Share the Licensed Material. For example, it may be + reasonable to satisfy the conditions by providing a URI or + hyperlink to a resource that includes the required + information. + + 3. If requested by the Licensor, You must remove any of the + information required by Section 3(a)(1)(A) to the extent + reasonably practicable. + + 4. If You Share Adapted Material You produce, the Adapter's + License You apply must not prevent recipients of the Adapted + Material from complying with this Public License. + + +Section 4 -- Sui Generis Database Rights. + +Where the Licensed Rights include Sui Generis Database Rights that +apply to Your use of the Licensed Material: + + a. for the avoidance of doubt, Section 2(a)(1) grants You the right + to extract, reuse, reproduce, and Share all or a substantial + portion of the contents of the database for NonCommercial purposes + only; + + b. if You include all or a substantial portion of the database + contents in a database in which You have Sui Generis Database + Rights, then the database in which You have Sui Generis Database + Rights (but not its individual contents) is Adapted Material; and + + c. You must comply with the conditions in Section 3(a) if You Share + all or a substantial portion of the contents of the database. + +For the avoidance of doubt, this Section 4 supplements and does not +replace Your obligations under this Public License where the Licensed +Rights include other Copyright and Similar Rights. + + +Section 5 -- Disclaimer of Warranties and Limitation of Liability. + + a. UNLESS OTHERWISE SEPARATELY UNDERTAKEN BY THE LICENSOR, TO THE + EXTENT POSSIBLE, THE LICENSOR OFFERS THE LICENSED MATERIAL AS-IS + AND AS-AVAILABLE, AND MAKES NO REPRESENTATIONS OR WARRANTIES OF + ANY KIND CONCERNING THE LICENSED MATERIAL, WHETHER EXPRESS, + IMPLIED, STATUTORY, OR OTHER. THIS INCLUDES, WITHOUT LIMITATION, + WARRANTIES OF TITLE, MERCHANTABILITY, FITNESS FOR A PARTICULAR + PURPOSE, NON-INFRINGEMENT, ABSENCE OF LATENT OR OTHER DEFECTS, + ACCURACY, OR THE PRESENCE OR ABSENCE OF ERRORS, WHETHER OR NOT + KNOWN OR DISCOVERABLE. WHERE DISCLAIMERS OF WARRANTIES ARE NOT + ALLOWED IN FULL OR IN PART, THIS DISCLAIMER MAY NOT APPLY TO YOU. + + b. TO THE EXTENT POSSIBLE, IN NO EVENT WILL THE LICENSOR BE LIABLE + TO YOU ON ANY LEGAL THEORY (INCLUDING, WITHOUT LIMITATION, + NEGLIGENCE) OR OTHERWISE FOR ANY DIRECT, SPECIAL, INDIRECT, + INCIDENTAL, CONSEQUENTIAL, PUNITIVE, EXEMPLARY, OR OTHER LOSSES, + COSTS, EXPENSES, OR DAMAGES ARISING OUT OF THIS PUBLIC LICENSE OR + USE OF THE LICENSED MATERIAL, EVEN IF THE LICENSOR HAS BEEN + ADVISED OF THE POSSIBILITY OF SUCH LOSSES, COSTS, EXPENSES, OR + DAMAGES. WHERE A LIMITATION OF LIABILITY IS NOT ALLOWED IN FULL OR + IN PART, THIS LIMITATION MAY NOT APPLY TO YOU. + + c. The disclaimer of warranties and limitation of liability provided + above shall be interpreted in a manner that, to the extent + possible, most closely approximates an absolute disclaimer and + waiver of all liability. + + +Section 6 -- Term and Termination. + + a. This Public License applies for the term of the Copyright and + Similar Rights licensed here. However, if You fail to comply with + this Public License, then Your rights under this Public License + terminate automatically. + + b. Where Your right to use the Licensed Material has terminated under + Section 6(a), it reinstates: + + 1. automatically as of the date the violation is cured, provided + it is cured within 30 days of Your discovery of the + violation; or + + 2. upon express reinstatement by the Licensor. + + For the avoidance of doubt, this Section 6(b) does not affect any + right the Licensor may have to seek remedies for Your violations + of this Public License. + + c. For the avoidance of doubt, the Licensor may also offer the + Licensed Material under separate terms or conditions or stop + distributing the Licensed Material at any time; however, doing so + will not terminate this Public License. + + d. Sections 1, 5, 6, 7, and 8 survive termination of this Public + License. + + +Section 7 -- Other Terms and Conditions. + + a. The Licensor shall not be bound by any additional or different + terms or conditions communicated by You unless expressly agreed. + + b. Any arrangements, understandings, or agreements regarding the + Licensed Material not stated herein are separate from and + independent of the terms and conditions of this Public License. + + +Section 8 -- Interpretation. + + a. For the avoidance of doubt, this Public License does not, and + shall not be interpreted to, reduce, limit, restrict, or impose + conditions on any use of the Licensed Material that could lawfully + be made without permission under this Public License. + + b. To the extent possible, if any provision of this Public License is + deemed unenforceable, it shall be automatically reformed to the + minimum extent necessary to make it enforceable. If the provision + cannot be reformed, it shall be severed from this Public License + without affecting the enforceability of the remaining terms and + conditions. + + c. No term or condition of this Public License will be waived and no + failure to comply consented to unless expressly agreed to by the + Licensor. + + d. Nothing in this Public License constitutes or may be interpreted + as a limitation upon, or waiver of, any privileges and immunities + that apply to the Licensor or You, including from the legal + processes of any jurisdiction or authority. + +======================================================================= + +Creative Commons is not a party to its public +licenses. Notwithstanding, Creative Commons may elect to apply one of +its public licenses to material it publishes and in those instances +will be considered the “Licensor.” The text of the Creative Commons +public licenses is dedicated to the public domain under the CC0 Public +Domain Dedication. Except for the limited purpose of indicating that +material is shared under a Creative Commons public license or as +otherwise permitted by the Creative Commons policies published at +creativecommons.org/policies, Creative Commons does not authorize the +use of the trademark "Creative Commons" or any other trademark or logo +of Creative Commons without its prior written consent including, +without limitation, in connection with any unauthorized modifications +to any of its public licenses or any other arrangements, +understandings, or agreements concerning use of licensed material. For +the avoidance of doubt, this paragraph does not form part of the +public licenses. + +Creative Commons may be contacted at creativecommons.org. + diff --git a/actions/code_helper.py b/actions/code_helper.py index 7a0e952..c7442c7 100644 --- a/actions/code_helper.py +++ b/actions/code_helper.py @@ -15,7 +15,10 @@ BASE_DIR = get_base_dir() API_CONFIG_PATH = BASE_DIR / "config" / "api_keys.json" DESKTOP = Path.home() / "Desktop" MAX_BUILD_ATTEMPTS = 3 -GEMINI_MODEL = "gemini-flash-latest" +# Model choice lives in core/gemini.py, and so does the timeout and the +# fallback ladder. Writing a model name here is what left this file hanging +# forever whenever that one alias was unwell. +from core import gemini def _get_api_key() -> str: @@ -23,13 +26,15 @@ def _get_api_key() -> str: return json.load(f)["gemini_api_key"] -def _get_gemini(model: str = GEMINI_MODEL): - from google import genai - _c = genai.Client(api_key=_get_api_key()) - +def _get_gemini(tier: str = gemini.SMART): + """Writing and fixing code is the reasoning tier; a 60s deadline because a + whole file can come back.""" class _W: def generate_content(self, contents): - return _c.models.generate_content(model=model, contents=contents) + resp = gemini.call(contents, tier=tier, timeout_ms=60000) + if resp is None: + raise RuntimeError("every Gemini model on the ladder failed") + return resp return _W() @@ -456,11 +461,8 @@ def _screen_debug_action(description, file_path, player, speak=None) -> str: print(f"[Code] ⚠️ Could not read file: {err}") try: - from google import genai from google.genai import types - client = genai.Client(api_key=_get_api_key()) - image_bytes = screenshot_path.read_bytes() image_base64 = _image_to_base64(screenshot_path) @@ -487,12 +489,11 @@ Be specific and actionable. If you see an error message, quote it exactly.""" analysis_prompt, ] - response = client.models.generate_content( - model="gemini-flash-latest", - contents=contents, - ) + response = gemini.call(contents, tier=gemini.SMART, timeout_ms=45_000) + if response is None: + return "Sir, I couldn't reach Gemini to analyse that screenshot." - analysis = response.text.strip() + analysis = (response.text or "").strip() print(f"[Code] ✅ Screen analysis complete") try: diff --git a/actions/computer_control.py b/actions/computer_control.py index a9f4822..3add084 100644 --- a/actions/computer_control.py +++ b/actions/computer_control.py @@ -327,7 +327,6 @@ def _screen_find(description: str) -> tuple[int, int] | None: img.save(buf, format="PNG") image_bytes = buf.getvalue() - client = genai.Client(api_key=api_key) prompt = ( f"This is a screenshot of a {w}×{h} pixel screen. " f"Locate the UI element described as: '{description}'. " @@ -335,13 +334,13 @@ def _screen_find(description: str) -> tuple[int, int] | None: f"If the element is not visible, reply: NOT_FOUND" ) - response = client.models.generate_content( - model="gemini-flash-lite-latest", - contents=[ - gtypes.Part.from_bytes(data=image_bytes, mime_type="image/png"), - prompt, - ], + from core import gemini + response = gemini.call( + [gtypes.Part.from_bytes(data=image_bytes, mime_type="image/png"), prompt], + tier=gemini.FAST, timeout_ms=20_000, ) + if response is None: + return None text = (response.text or "").strip() if "NOT_FOUND" in text.upper(): diff --git a/actions/desktop.py b/actions/desktop.py index d687462..1490633 100644 --- a/actions/desktop.py +++ b/actions/desktop.py @@ -104,7 +104,6 @@ def _execute_generated_code(code: str, player=None) -> str: def _ask_gemini_for_desktop_action(task: str) -> str: from google import genai as _genai - _client = _genai.Client(api_key=_get_api_key()) desktop = str(_get_desktop()) @@ -142,8 +141,11 @@ Output ONLY the Python code. No explanation, no markdown, no backticks. Task: {task}""" try: - response = _client.models.generate_content(model="gemini-flash-latest", contents=prompt) - code = response.text.strip() + from core import gemini + response = gemini.call(prompt, tier=gemini.SMART, timeout_ms=30_000) + if response is None: + return "ERROR: every Gemini model on the ladder failed" + code = (response.text or "").strip() if code.startswith("```"): lines = code.split("\n") code = "\n".join(lines[1:-1]).strip() diff --git a/actions/dev_agent.py b/actions/dev_agent.py index be59cbe..e0fd18c 100644 --- a/actions/dev_agent.py +++ b/actions/dev_agent.py @@ -16,21 +16,26 @@ BASE_DIR = get_base_dir() API_CONFIG_PATH = BASE_DIR / "config" / "api_keys.json" PROJECTS_DIR = Path.home() / "Desktop" / "JarvisProjects" MAX_FIX_ATTEMPTS = 5 -MODEL_PLANNER = "gemini-flash-latest" -MODEL_WRITER = "gemini-flash-latest" +# Model choice, timeout and fallback ladder all live in core/gemini.py. +from core import gemini + +MODEL_PLANNER = gemini.SMART +MODEL_WRITER = gemini.SMART def _get_api_key() -> str: with open(API_CONFIG_PATH, "r", encoding="utf-8") as f: return json.load(f)["gemini_api_key"] -def _get_model(model_name: str): - from google import genai - _c = genai.Client(api_key=_get_api_key()) - +def _get_model(model_name: str = gemini.SMART): + """Planning and writing whole files — the reasoning tier, and a long + deadline because the answer is a source file rather than a sentence.""" class _W: def generate_content(self, contents): - return _c.models.generate_content(model=model_name, contents=contents) + resp = gemini.call(contents, tier=model_name, timeout_ms=60000) + if resp is None: + raise RuntimeError("every Gemini model on the ladder failed") + return resp return _W() diff --git a/actions/file_controller.py b/actions/file_controller.py index 24c20d5..1e618ff 100644 --- a/actions/file_controller.py +++ b/actions/file_controller.py @@ -158,9 +158,22 @@ def _resolve_path(raw: str) -> Path: "videos": _get_videos(), "home": Path.home(), } - lower = raw.strip().lower() + raw = raw.strip().strip('"').strip("'") + lower = raw.lower() if lower in shortcuts: return shortcuts[lower] + + # "desktop/notes/a.md" and "desktop\notes\a.md" — a shortcut followed by a + # sub-path. Without this branch the whole string falls through to the + # relative-path return below and is resolved against the process CWD instead + # of the real Desktop: an "Access denied" when the project lives outside the + # home directory, or — worse — a silent write into a stray "desktop" folder + # inside the project when it lives inside it. + head, sep, rest = raw.replace("\\", "/").partition("/") + if sep and head.lower() in shortcuts: + rest = rest.strip("/") + return shortcuts[head.lower()] / rest if rest else shortcuts[head.lower()] + return Path(raw).expanduser() def _format_size(b: int) -> str: diff --git a/actions/file_processor.py b/actions/file_processor.py index 9965c9d..345ebd9 100644 --- a/actions/file_processor.py +++ b/actions/file_processor.py @@ -25,19 +25,24 @@ import tempfile from pathlib import Path from datetime import datetime +# Model choice, timeout and fallback ladder all live in core/gemini.py. +from core import gemini + def _get_api_key() -> str: config_path = Path(__file__).resolve().parent.parent / "config" / "api_keys.json" with open(config_path, "r", encoding="utf-8") as f: return json.load(f)["gemini_api_key"] -def _gemini_client(): - from google import genai - _c = genai.Client(api_key=_get_api_key()) - +def _gemini_client(tier: str = gemini.SMART): + """Summarising documents and reading images — the reasoning tier, with a + long deadline because the input can be a whole file.""" class _W: def generate_content(self, contents): - return _c.models.generate_content(model="gemini-flash-latest", contents=contents) + resp = gemini.call(contents, tier=tier, timeout_ms=90000) + if resp is None: + raise RuntimeError("every Gemini model on the ladder failed") + return resp return _W() diff --git a/actions/flight_finder.py b/actions/flight_finder.py index 1484ac5..c130a8d 100644 --- a/actions/flight_finder.py +++ b/actions/flight_finder.py @@ -60,17 +60,13 @@ def _parse_date(raw: str) -> str: return val.strftime("%Y-%m-%d") try: - from google import genai as _genai - _client = _genai.Client(api_key=_get_api_key()) - response = _client.models.generate_content( - model="gemini-flash-lite-latest", - contents=( - f"Today is {today.strftime('%Y-%m-%d')}. " - f"Convert this date expression to YYYY-MM-DD: '{raw}'. " - f"Return ONLY the date string, nothing else." - ) + from core import gemini + result = gemini.text( + f"Today is {today.strftime('%Y-%m-%d')}. " + f"Convert this date expression to YYYY-MM-DD: '{raw}'. " + f"Return ONLY the date string, nothing else.", + tier=gemini.FAST, ) - result = response.text.strip() if re.match(r"\d{4}-\d{2}-\d{2}", result): return result except Exception as e: @@ -155,7 +151,6 @@ def _parse_flights_with_gemini( from google import genai as _genai from google.genai import types - _client = _genai.Client(api_key=_get_api_key()) prompt = ( f"Extract flight options from {origin} to {destination} on {date} " f"from this Google Flights page text:\n\n{raw_text[:12000]}\n\n" @@ -166,9 +161,11 @@ def _parse_flights_with_gemini( ) try: - response = _client.models.generate_content( - model="gemini-flash-latest", - contents=prompt, + from core import gemini + response = gemini.call( + prompt, + tier=gemini.SMART, + timeout_ms=30_000, config=types.GenerateContentConfig( system_instruction=( "You are a flight data extraction expert. " diff --git a/actions/web_search.py b/actions/web_search.py index 92cee47..f8cae33 100644 --- a/actions/web_search.py +++ b/actions/web_search.py @@ -1,8 +1,68 @@ #web_search.py import json import sys +import threading +import time from pathlib import Path +# ── Gemini grounding quota circuit breaker ──────────────────────────────────── +# The google_search grounding tool has its own small quota, separate from plain +# generation. Once it is spent every call returns 429 — so retrying it at the +# top of every search only adds a dead round-trip before the DDG fallback runs. +# After a quota error, skip Gemini entirely for a cooldown period. +_QUOTA_COOLDOWN_SEC = 900 # 15 minutes +_quota_blocked_until = 0.0 +_quota_lock = threading.Lock() + + +def _gemini_available() -> bool: + with _quota_lock: + return time.monotonic() >= _quota_blocked_until + + +def _note_gemini_error(exc: Exception) -> None: + """Trip the breaker when the error is a quota / rate-limit rejection.""" + global _quota_blocked_until + msg = str(exc) + if "429" in msg or "RESOURCE_EXHAUSTED" in msg: + with _quota_lock: + already = time.monotonic() < _quota_blocked_until + _quota_blocked_until = time.monotonic() + _QUOTA_COOLDOWN_SEC + if not already: + print( + "[WebSearch] Gemini grounding quota exhausted — skipping it for " + f"{_QUOTA_COOLDOWN_SEC // 60} min and serving results from DDG." + ) + + +class _QuotaCooldown(RuntimeError): + """Raised instead of calling Gemini while the quota breaker is open.""" + + +def _log_gemini_failure(context: str, exc: Exception) -> None: + """Log a Gemini failure — silently when it is just the expected cooldown.""" + if isinstance(exc, _QuotaCooldown): + return # announced once when the breaker tripped; not a warning + print(f"[WebSearch] \u26a0\ufe0f {context} failed ({exc}) — using DDG instead") + + +def _run_bounded(fn, timeout: float, label: str = "task"): + """Run fn() in a daemon thread; return its result, or None if it overruns.""" + box = [None] + + def _run(): + try: + box[0] = fn() + except Exception as e: + _log_gemini_failure(label, e) + + t = threading.Thread(target=_run, daemon=True) + t.start() + t.join(timeout) + if t.is_alive(): + print(f"[WebSearch] {label} exceeded {timeout:.0f}s — moving on") + return box[0] + def _get_base_dir() -> Path: if getattr(sys, "frozen", False): return Path(sys.executable).parent @@ -19,14 +79,22 @@ def _get_api_key() -> str: def _gemini_search(query: str) -> str: - from google import genai + if not _gemini_available(): + raise _QuotaCooldown("Gemini grounding is in quota cooldown") - client = genai.Client(api_key=_get_api_key()) - response = client.models.generate_content( - model="gemini-flash-latest", - contents=query, - config={"tools": [{"google_search": {}}]}, - ) + from core import gemini + + # Grounded search reads a live page, so it gets a longer deadline than the + # default — but it still HAS one, and it still walks the fallback ladder. + try: + response = gemini.call(query, tier=gemini.SEARCH, + config={"tools": [{"google_search": {}}]}, + timeout_ms=30_000) + if response is None: + raise RuntimeError("every Gemini model on the ladder failed") + except Exception as e: + _note_gemini_error(e) + raise text = "" for part in response.candidates[0].content.parts: @@ -39,30 +107,45 @@ def _gemini_search(query: str) -> str: return text -def _ddg_search(query: str, max_results: int = 6) -> list[dict]: +def _get_ddgs(): + """ + Returns the DDGS class. The package was renamed duckduckgo-search -> ddgs; + the legacy package's endpoints are now rejected by DuckDuckGo (news() gets a + 403 Ratelimit, text() silently returns zero results), so warn loudly if we + end up on it instead of failing in silence. + """ try: from ddgs import DDGS + return DDGS except ImportError: from duckduckgo_search import DDGS + print( + "[WebSearch] ⚠️ Using the deprecated 'duckduckgo-search' package — " + "DuckDuckGo blocks its endpoints, so every search will come back " + "empty. Fix with: pip install -U ddgs" + ) + return DDGS + +def _ddg_search(query: str, max_results: int = 6) -> list[dict]: + DDGS = _get_ddgs() results = [] - with DDGS() as ddgs: - for r in ddgs.text(query, max_results=max_results): - results.append({ - "title": r.get("title", ""), - "snippet": r.get("body", ""), - "url": r.get("href", ""), - }) + try: + with DDGS() as ddgs: + for r in ddgs.text(query, max_results=max_results): + results.append({ + "title": r.get("title", ""), + "snippet": r.get("body", ""), + "url": r.get("href", ""), + }) + except Exception as e: + print(f"[WebSearch] ⚠️ DDG text() failed: {e}") return results def _ddg_news(query: str, max_results: int = 8) -> list[dict]: """DDG news search — returns actual articles, not website homepages.""" - try: - from ddgs import DDGS - except ImportError: - from duckduckgo_search import DDGS - + DDGS = _get_ddgs() results = [] try: with DDGS() as ddgs: @@ -75,6 +158,9 @@ def _ddg_news(query: str, max_results: int = 8) -> list[dict]: }) except Exception as e: print(f"[WebSearch] ⚠️ DDG news() failed ({e}) — falling back to text search") + # Also covers the legacy-package case, where news() returns an empty list + # instead of raising. + if not results: results = _ddg_search(query, max_results=max_results) return results @@ -120,14 +206,16 @@ def _gemini_headlines(n: int = 5) -> tuple[list[str], str]: Returns (headline_list, raw_text_for_display). """ import re - from google import genai + from core import gemini - client = genai.Client(api_key=_get_api_key()) - response = client.models.generate_content( - model="gemini-flash-latest", - contents=f"Current world news: {n} headlines. Numbered list, titles only.", + response = gemini.call( + f"Current world news: {n} headlines. Numbered list, titles only.", + tier=gemini.SEARCH, config={"tools": [{"google_search": {}}]}, + timeout_ms=30_000, ) + if response is None: + return [], "" raw = "" for part in response.candidates[0].content.parts: @@ -157,58 +245,43 @@ def _search(query: str) -> str: try: return _gemini_search(query) except Exception as e: - print(f"[WebSearch] ⚠️ Gemini failed ({e}) — trying DDG...") + _log_gemini_failure("Gemini search", e) results = _ddg_search(query) return _format_ddg(query, results) def _news(query: str) -> str: """ - Runs Gemini grounded search AND DDG news in parallel. - Returns whichever delivers a valid result first; cancels the other. - """ - import threading + DDG first, Gemini as backup. + The old version raced both backends in parallel and kept the first answer. + That burned one google_search grounding call on *every* news request — + including the startup briefing — even when DDG had already won the race. + Grounding has a small quota, so it ran dry after a handful of launches and + then 429'd for everything else (research/compare), which are the modes that + actually need a synthesised answer. + + DDG news returns in well under a second and gives raw headlines, which is + exactly what the briefing wants, so it goes first and Gemini is only touched + when DDG comes back empty. + """ gemini_query = f"latest news today: {query}" if query else "top world news today" ddg_query = query if query else "world news today" - result_box = [None] # first valid result lands here - lock = threading.Lock() - done_evt = threading.Event() - failures = [0] + def _ddg_attempt() -> str: + return _format_news(ddg_query, _ddg_news(ddg_query, max_results=8)) - def _store(r: str) -> None: - if r and len(r) > 60: - with lock: - if result_box[0] is None: - result_box[0] = r - done_evt.set() - else: - with lock: - failures[0] += 1 - if failures[0] >= 2: # both failed — unblock caller - done_evt.set() + text = _run_bounded(_ddg_attempt, timeout=5.0, label="DDG news") + if text and len(text) > 60 and not text.startswith("No news found"): + return text - def _try_gemini(): - try: - _store(_gemini_search(gemini_query)) - except Exception as e: - print(f"[WebSearch] ⚠️ Gemini news failed ({e})") - _store("") + text = _run_bounded( + lambda: _gemini_search(gemini_query), timeout=6.0, label="Gemini news" + ) + if text and len(text) > 60: + return text - def _try_ddg(): - try: - results = _ddg_news(ddg_query, max_results=8) - _store(_format_news(ddg_query, results)) - except Exception as e: - print(f"[WebSearch] ⚠️ DDG news failed ({e})") - _store("") - - threading.Thread(target=_try_gemini, daemon=True).start() - threading.Thread(target=_try_ddg, daemon=True).start() - - done_evt.wait(timeout=10.0) - return result_box[0] or f"No news found for: {query}" + return f"No news found for: {query}" def _research(query: str) -> str: @@ -223,7 +296,7 @@ def _research(query: str) -> str: try: return _gemini_search(research_query) except Exception as e: - print(f"[WebSearch] ⚠️ Research Gemini failed ({e}) — DDG fallback...") + _log_gemini_failure("Gemini research", e) results = _ddg_search(query, max_results=10) return _format_ddg(query, results) @@ -234,7 +307,7 @@ def _price(query: str) -> str: try: return _gemini_search(price_query) except Exception as e: - print(f"[WebSearch] ⚠️ Price Gemini failed ({e}) — DDG fallback...") + _log_gemini_failure("Gemini price", e) results = _ddg_search(f"{query} price buy", max_results=6) return _format_ddg(query, results) @@ -247,7 +320,7 @@ def _compare(items: list[str], aspect: str) -> str: try: return _gemini_search(query) except Exception as e: - print(f"[WebSearch] ⚠️ Gemini compare failed: {e} — falling back to DDG") + _log_gemini_failure("Gemini compare", e) all_results: dict[str, list] = {} for item in items: diff --git a/actions/youtube_video.py b/actions/youtube_video.py index 77f0195..e8dad2f 100644 --- a/actions/youtube_video.py +++ b/actions/youtube_video.py @@ -167,15 +167,17 @@ def _get_transcript(video_id: str) -> str | None: def _summarize_with_gemini(transcript: str, video_url: str) -> str: - from google import genai as _genai from google.genai import types + from core import gemini - _client = _genai.Client(api_key=_get_api_key()) max_chars = 80000 truncated = transcript[:max_chars] + ("..." if len(transcript) > max_chars else "") - response = _client.models.generate_content( - model="gemini-flash-latest", - contents=f"Please summarize this YouTube video transcript:\n\n{truncated}", + # A whole transcript can be 80k characters, hence the long deadline — but a + # deadline there is, and the ladder in core/gemini.py picks the model. + response = gemini.call( + f"Please summarize this YouTube video transcript:\n\n{truncated}", + tier=gemini.SMART, + timeout_ms=60_000, config=types.GenerateContentConfig( system_instruction=( "You are JARVIS, an AI assistant. " @@ -186,7 +188,9 @@ def _summarize_with_gemini(transcript: str, video_url: str) -> str: ) ) ) - return response.text.strip() + if response is None: + return "I couldn't reach Gemini to summarise that transcript, sir." + return (response.text or "").strip() def _save_summary(content: str, video_url: str) -> str: diff --git a/core/action_loader.py b/core/action_loader.py index f81975a..0dc610e 100644 --- a/core/action_loader.py +++ b/core/action_loader.py @@ -1,193 +1,221 @@ -""" -Action discovery, validation, and dispatch — the built-in twin of plugin_loader. - -Every actions/*.py that exposes a module-level ``TOOL`` dict is auto-discovered -here, exactly like a drop-in plugin, so main.py never has to hardcode a tool -declaration or a dispatch branch for it. Adding a new bundled action is then the -same one-file operation as writing a plugin: define ``TOOL`` and a handler. - -``TOOL`` shape (see actions/open_app.py for a live example): - - TOOL = { - "name": "open_app", # unique, ^[a-zA-Z_][a-zA-Z0-9_]{0,63}$ - "description": "...", # what Gemini reads to route the call - "parameters": {"type": "OBJECT", ...}, # Gemini function-declaration schema - "handler": open_app, # the callable to run - } - -The handler is invoked through signature introspection: it receives ``parameters`` -plus whichever of ``player`` / ``speak`` / ``response`` / ``session_memory`` it -actually declares — so existing action signatures work unchanged. - -Discovery runs once at startup; import errors, validation errors, and name -collisions are logged and the offending file is skipped — they NEVER raise out -of discover_actions() and never abort the scan of the remaining files. -""" -from __future__ import annotations - -import importlib.util -import inspect -import re -import sys -import traceback -from dataclasses import dataclass, field -from pathlib import Path -from typing import Callable, Optional - -_NAME_RE = re.compile(r"^[a-zA-Z_][a-zA-Z0-9_]{0,63}$") -_DEFAULT_PARAMS = {"type": "OBJECT", "properties": {}} -_CTX_KEYS = ("player", "speak", "response", "session_memory") - - -@dataclass -class ActionRecord: - name: str - description: str = "" - parameters: dict = field(default_factory=lambda: dict(_DEFAULT_PARAMS)) - handler: Optional[Callable] = None - file: str = "" - valid: bool = False - error: str = "" - - -class ActionRegistry: - def __init__(self, actions: dict[str, ActionRecord], logger: Callable[[str], None]): - self._actions = actions # name -> ActionRecord, VALID entries only - self._all_records: list[ActionRecord] = [] - self._logger = logger - - # -- called by main.py at LiveConnectConfig build time -- - def get_tool_declarations(self) -> list[dict]: - return [ - {"name": rec.name, "description": rec.description, "parameters": rec.parameters} - for rec in self._actions.values() - ] - - def has(self, name: str) -> bool: - return name in self._actions - - def names(self) -> set[str]: - return set(self._actions.keys()) - - # -- called by main.py from _execute_tool -- - def run(self, name: str, parameters: dict, ctx: dict | None = None) -> str: - rec = self._actions.get(name) - if rec is None or not rec.valid: - return f"Action '{name}' is not available." - try: - return _call_handler(rec.handler, parameters, ctx or {}) or "Done." - except Exception as e: - self._logger(f"Action '{name}' crashed during run(): {e}") - traceback.print_exc() - return f"Tool '{name}' failed: {e}" - - -def _call_handler(fn: Callable, parameters: dict, ctx: dict) -> str: - """Invoke the handler passing only the context kwargs it actually declares - (or all of them if it has **kwargs), so each action's existing signature - works unchanged.""" - sig = inspect.signature(fn) - has_var_kw = any(p.kind == inspect.Parameter.VAR_KEYWORD for p in sig.parameters.values()) - kwargs = {} - for key in _CTX_KEYS: - if has_var_kw or key in sig.parameters: - kwargs[key] = ctx.get(key) - return fn(parameters=parameters, **kwargs) - - -def _validate(module, filename: str) -> ActionRecord: - """Returns an ActionRecord; .valid=False + .error set on any problem. Never raises.""" - tool = getattr(module, "TOOL", None) - if not isinstance(tool, dict): - return ActionRecord(name=Path(filename).stem, file=filename, - error="No module-level TOOL dict (not a discoverable action).") - - name = tool.get("name") - if not isinstance(name, str) or not _NAME_RE.match(name): - return ActionRecord(name=str(name or Path(filename).stem), file=filename, - error="TOOL['name'] missing or not a valid identifier.") - - description = tool.get("description") - if not isinstance(description, str) or not description.strip(): - return ActionRecord(name=name, file=filename, - error="TOOL['description'] missing or empty.") - - parameters = tool.get("parameters", _DEFAULT_PARAMS) - if not isinstance(parameters, dict) or parameters.get("type") != "OBJECT": - return ActionRecord(name=name, file=filename, - error="TOOL['parameters'] must be a dict with \"type\": \"OBJECT\".") - - handler = tool.get("handler") - if not callable(handler): - return ActionRecord(name=name, file=filename, - error="TOOL['handler'] missing or not callable.") - - return ActionRecord(name=name, description=description.strip(), parameters=parameters, - handler=handler, file=filename, valid=True, error="") - - -def discover_actions(actions_dir: Path, reserved_names: set[str] | None = None, - logger: Callable[[str], None] = print) -> ActionRegistry: - """ - Scans actions_dir for *.py files (skips files starting with '_'). A file is - only treated as an action if it exposes a module-level TOOL dict; files - without one (shared helpers, capture-only modules) are silently ignored. - Import/validation errors and name collisions are logged and the file is - skipped — they NEVER raise out of this function. - """ - reserved = reserved_names or set() - actions_dir.mkdir(parents=True, exist_ok=True) - valid: dict[str, ActionRecord] = {} - all_records: list[ActionRecord] = [] - - files = sorted(actions_dir.glob("*.py"), key=lambda p: p.name) # deterministic order - for path in files: - if path.name.startswith("_"): - continue - try: - module_name = f"actions.{path.stem}" - # Reuse the already-imported module when present so handlers are the - # same objects the rest of the app holds. - module = sys.modules.get(module_name) - if module is None: - spec = importlib.util.spec_from_file_location(module_name, path) - if spec is None or spec.loader is None: - raise ImportError("could not build import spec") - module = importlib.util.module_from_spec(spec) - sys.modules[module_name] = module - try: - spec.loader.exec_module(module) - except Exception: - sys.modules.pop(module_name, None) - raise - - if getattr(module, "TOOL", None) is None: - continue # not an action file — a helper/capture-only module - - rec = _validate(module, path.name) - - if rec.valid and rec.name in reserved: - rec = ActionRecord(name=rec.name, file=path.name, - error=f"Name '{rec.name}' collides with a reserved core tool — rejected.") - elif rec.valid and rec.name in valid: - other = valid[rec.name].file - rec = ActionRecord(name=rec.name, file=path.name, - error=f"Name '{rec.name}' already used by action '{other}' — rejected.") - - except Exception as e: - rec = ActionRecord(name=path.stem, file=path.name, - error=f"Failed to load: {e}") - traceback.print_exc() - - all_records.append(rec) - if rec.valid: - valid[rec.name] = rec - logger(f"Action loaded: {rec.name} ({path.name})") - else: - # Only log a rejection if the file actually tried to be an action. - logger(f"Action rejected: {path.name} — {rec.error}") - - registry = ActionRegistry(valid, logger) - registry._all_records = all_records - logger(f"Action discovery complete: {len(valid)} active.") - return registry +""" +Action discovery, validation, and dispatch — the built-in twin of plugin_loader. + +Every actions/*.py that exposes a module-level ``TOOL`` dict is auto-discovered +here, exactly like a drop-in plugin, so main.py never has to hardcode a tool +declaration or a dispatch branch for it. Adding a new bundled action is then the +same one-file operation as writing a plugin: define ``TOOL`` and a handler. + +``TOOL`` shape (see actions/open_app.py for a live example): + + TOOL = { + "name": "open_app", # unique, ^[a-zA-Z_][a-zA-Z0-9_]{0,63}$ + "description": "...", # what Gemini reads to route the call + "parameters": {"type": "OBJECT", ...}, # Gemini function-declaration schema + "handler": open_app, # the callable to run + } + +The handler is invoked through signature introspection: it receives ``parameters`` +plus whichever of ``player`` / ``speak`` / ``response`` / ``session_memory`` it +actually declares — so existing action signatures work unchanged. + +Discovery runs once at startup; import errors, validation errors, and name +collisions are logged and the offending file is skipped — they NEVER raise out +of discover_actions() and never abort the scan of the remaining files. +""" +from __future__ import annotations + +import importlib.util +import inspect +import re +import sys +import traceback +from dataclasses import dataclass, field +from pathlib import Path +from typing import Callable, Optional + +_NAME_RE = re.compile(r"^[a-zA-Z_][a-zA-Z0-9_]{0,63}$") +_DEFAULT_PARAMS = {"type": "OBJECT", "properties": {}} +_CTX_KEYS = ("player", "speak", "response", "session_memory") + + +# A tool may declare that the model should NOT be held up waiting for it. +# `behavior` goes to the API with the declaration; `scheduling` decides when the +# eventual result is allowed back into the conversation: +# WHEN_IDLE — wait for a gap in the speech (the sane default) +# SILENT — record it, do not prompt a reply (the tool already announced) +# INTERRUPT — cut in immediately (only when the answer cannot wait) +_BEHAVIORS = ("BLOCKING", "NON_BLOCKING") +_SCHEDULING = ("WHEN_IDLE", "SILENT", "INTERRUPT") + + +def _opt_upper(value, allowed: tuple[str, ...]) -> Optional[str]: + v = str(value or "").strip().upper() + return v if v in allowed else None + + +@dataclass +class ActionRecord: + name: str + description: str = "" + parameters: dict = field(default_factory=lambda: dict(_DEFAULT_PARAMS)) + handler: Optional[Callable] = None + file: str = "" + valid: bool = False + error: str = "" + behavior: Optional[str] = None # None = the API's default (blocking) + scheduling: Optional[str] = None # None = the API's default (WHEN_IDLE) + + +class ActionRegistry: + def __init__(self, actions: dict[str, ActionRecord], logger: Callable[[str], None]): + self._actions = actions # name -> ActionRecord, VALID entries only + self._all_records: list[ActionRecord] = [] + self._logger = logger + + # -- called by main.py at LiveConnectConfig build time -- + def get_tool_declarations(self) -> list[dict]: + out = [] + for rec in self._actions.values(): + decl = {"name": rec.name, "description": rec.description, + "parameters": rec.parameters} + if rec.behavior: + decl["behavior"] = rec.behavior + out.append(decl) + return out + + def has(self, name: str) -> bool: + return name in self._actions + + def scheduling(self, name: str) -> Optional[str]: + """How this action's result should re-enter the conversation, if it said.""" + rec = self._actions.get(name) + return rec.scheduling if rec else None + + def names(self) -> set[str]: + return set(self._actions.keys()) + + # -- called by main.py from _execute_tool -- + def run(self, name: str, parameters: dict, ctx: dict | None = None) -> str: + rec = self._actions.get(name) + if rec is None or not rec.valid: + return f"Action '{name}' is not available." + try: + return _call_handler(rec.handler, parameters, ctx or {}) or "Done." + except Exception as e: + self._logger(f"Action '{name}' crashed during run(): {e}") + traceback.print_exc() + return f"Tool '{name}' failed: {e}" + + +def _call_handler(fn: Callable, parameters: dict, ctx: dict) -> str: + """Invoke the handler passing only the context kwargs it actually declares + (or all of them if it has **kwargs), so each action's existing signature + works unchanged.""" + sig = inspect.signature(fn) + has_var_kw = any(p.kind == inspect.Parameter.VAR_KEYWORD for p in sig.parameters.values()) + kwargs = {} + for key in _CTX_KEYS: + if has_var_kw or key in sig.parameters: + kwargs[key] = ctx.get(key) + return fn(parameters=parameters, **kwargs) + + +def _validate(module, filename: str) -> ActionRecord: + """Returns an ActionRecord; .valid=False + .error set on any problem. Never raises.""" + tool = getattr(module, "TOOL", None) + if not isinstance(tool, dict): + return ActionRecord(name=Path(filename).stem, file=filename, + error="No module-level TOOL dict (not a discoverable action).") + + name = tool.get("name") + if not isinstance(name, str) or not _NAME_RE.match(name): + return ActionRecord(name=str(name or Path(filename).stem), file=filename, + error="TOOL['name'] missing or not a valid identifier.") + + description = tool.get("description") + if not isinstance(description, str) or not description.strip(): + return ActionRecord(name=name, file=filename, + error="TOOL['description'] missing or empty.") + + parameters = tool.get("parameters", _DEFAULT_PARAMS) + if not isinstance(parameters, dict) or parameters.get("type") != "OBJECT": + return ActionRecord(name=name, file=filename, + error="TOOL['parameters'] must be a dict with \"type\": \"OBJECT\".") + + handler = tool.get("handler") + if not callable(handler): + return ActionRecord(name=name, file=filename, + error="TOOL['handler'] missing or not callable.") + + return ActionRecord(name=name, description=description.strip(), parameters=parameters, + handler=handler, file=filename, valid=True, error="", + behavior=_opt_upper(tool.get("behavior"), _BEHAVIORS), + scheduling=_opt_upper(tool.get("scheduling"), _SCHEDULING)) + + +def discover_actions(actions_dir: Path, reserved_names: set[str] | None = None, + logger: Callable[[str], None] = print) -> ActionRegistry: + """ + Scans actions_dir for *.py files (skips files starting with '_'). A file is + only treated as an action if it exposes a module-level TOOL dict; files + without one (shared helpers, capture-only modules) are silently ignored. + Import/validation errors and name collisions are logged and the file is + skipped — they NEVER raise out of this function. + """ + reserved = reserved_names or set() + actions_dir.mkdir(parents=True, exist_ok=True) + valid: dict[str, ActionRecord] = {} + all_records: list[ActionRecord] = [] + + files = sorted(actions_dir.glob("*.py"), key=lambda p: p.name) # deterministic order + for path in files: + if path.name.startswith("_"): + continue + try: + module_name = f"actions.{path.stem}" + # Reuse the already-imported module when present so handlers are the + # same objects the rest of the app holds. + module = sys.modules.get(module_name) + if module is None: + spec = importlib.util.spec_from_file_location(module_name, path) + if spec is None or spec.loader is None: + raise ImportError("could not build import spec") + module = importlib.util.module_from_spec(spec) + sys.modules[module_name] = module + try: + spec.loader.exec_module(module) + except Exception: + sys.modules.pop(module_name, None) + raise + + if getattr(module, "TOOL", None) is None: + continue # not an action file — a helper/capture-only module + + rec = _validate(module, path.name) + + if rec.valid and rec.name in reserved: + rec = ActionRecord(name=rec.name, file=path.name, + error=f"Name '{rec.name}' collides with a reserved core tool — rejected.") + elif rec.valid and rec.name in valid: + other = valid[rec.name].file + rec = ActionRecord(name=rec.name, file=path.name, + error=f"Name '{rec.name}' already used by action '{other}' — rejected.") + + except Exception as e: + rec = ActionRecord(name=path.stem, file=path.name, + error=f"Failed to load: {e}") + traceback.print_exc() + + all_records.append(rec) + if rec.valid: + valid[rec.name] = rec + logger(f"Action loaded: {rec.name} ({path.name})") + else: + # Only log a rejection if the file actually tried to be an action. + logger(f"Action rejected: {path.name} — {rec.error}") + + registry = ActionRegistry(valid, logger) + registry._all_records = all_records + logger(f"Action discovery complete: {len(valid)} active.") + return registry diff --git a/core/avatar.py b/core/avatar.py new file mode 100644 index 0000000..d8651f1 --- /dev/null +++ b/core/avatar.py @@ -0,0 +1,715 @@ +""" +Holographic AI head for the HUD centre — the thing that used to be a ring stack +with the assistant's name in the middle. + +Design notes +------------ +* **The face is real human geometry.** `core.avatar_mesh` builds the head around + MediaPipe's canonical face model, so eyelids, nostrils, lips and cheekbones + are measured anatomy rather than fitted curves. This renderer's whole job is + to light it, pose it and animate it. +* **Software rendered, on purpose.** Everything is QPainter, so there is no + OpenGL context, no shader compile, no GPU driver to disagree with us and no + new pip dependency. It looks the same on a gaming rig, a 2013 laptop, a VM + and a remote desktop session. +* **Lip-sync comes from the audio pipeline, not from the avatar.** `main.py` + already computes a real RMS level off the PCM (`_pcm_level`) for both the mic + and JARVIS's own output. The avatar just consumes that number, so there is no + second audio path to fall out of sync. The mouth only tracks the level while + JARVIS is *speaking* — during listening the same level drives the aura, so the + head never lip-syncs to the user's voice. + +The renderer is theme-agnostic: `paint()` takes its colours as arguments, which +is what lets the HueWheel accent picker retint the avatar for free. +""" + +from __future__ import annotations + +import math +import random + +import numpy as np +from PyQt6.QtCore import QLineF, QPointF, QRectF, Qt +from PyQt6.QtGui import QBrush, QColor, QPainter, QPen, QPolygonF, QRadialGradient + +from core.avatar_mesh import JAW_MAX, JAW_PIVOT, get_head_mesh + +# Perspective camera distance in head-half-heights. Large enough that the nose +# does not balloon, small enough to keep a sense of depth. +_CAM_D = 4.6 + +# Wireframe opacity buckets, so the whole lattice draws in a handful of batched +# drawLines() calls instead of one call per line. +_BUCKETS = 4 +_MIN_ALPHA = 0.05 + +# Resolution of the surface-shading colour ramp. Banding across a filled facet +# is far more visible than banding in line alpha, so this is fine-grained — and +# being a lookup it costs nothing per face. +_LUT_N = 192 + +# How far the brows travel at full lift, in head-half-heights. Derived, not +# tuned: the brow-to-eye gap is 0.198 and a real raise covers about a third of +# it, then the drawn landmarks only carry half the rig weight. +_BROW_LIFT = 0.14 + +# Mouth timing, as time constants in seconds rather than per-frame fractions. +# A fixed per-frame lerp silently changes speed with the frame rate: the HUD +# runs at 60 Hz here, throttles its paint to 30, and drops to 20 when idle, so +# the same constant meant three different mouths. These do not. +# +# Shutting is the fastest of the three, and it is measured rather than chosen. +# A short closure occupies a single 20 ms schedule frame, so the mouth has one +# step to reach it: at 20 ms the jaw got a third of the way and the closure +# vanished, at 12 ms it arrives, and going below that changes nothing because +# the analysis window is then the limit, not the smoothing. Halving it doubled +# the closures the mouth visibly makes across a test paragraph, 5 of 21 to 10, +# with no loss of opening on the vowels. Only the return to rest, once talking +# has actually stopped, is leisurely. +_TAU_OPEN = 0.022 # jaw dropping toward a vowel +_TAU_SHUT = 0.012 # lips closing on a consonant, mid-word +_TAU_REST = 0.055 # settling back to rest after speech ends +_TAU_SHAPE = 0.018 # viseme openness following the schedule + +# Only the microphone path needs a level floor: it has one coarse RMS and no way +# to tell speech from room tone. JARVIS's own voice arrives as a per-20 ms +# schedule whose silences are already silent, so it needs no floor and must not +# have one — a floor there swallows the gaps between words. +_MIC_FLOOR = 0.14 + +# How far below this voice's own loud level counts as a closure: -20 dB, which +# is what a stop consonant actually drops to. Expressed as a ratio so it holds +# at any speaker volume. +_CLOSE_FRAC = 0.10 + + +def _rate(dt: float, tau: float) -> float: + """Per-frame lerp factor for an exponential approach with time constant + `tau`. Frame-rate independent: the motion takes the same wall-clock time at + 20, 30 or 60 fps, and a long frame catches up instead of stalling.""" + return 1.0 - math.exp(-dt / tau) + + +def _c(col: QColor, a: float) -> QColor: + """Copy of `col` at alpha `a` (0-255, clamped).""" + q = QColor(col) + q.setAlpha(int(max(0.0, min(255.0, a)))) + return q + + +def _blend(bg: QColor, col: QColor, a: float) -> QColor: + """`col` at alpha `a` pre-mixed onto `bg`, returned fully **opaque**. + + Qt's raster engine has a fast path for opaque antialiased lines and a much + slower blended path for everything else — measured at 1.0 ms versus 2.9 ms + for the same 780 lines. The HUD paints a flat background behind the avatar, + so mixing the alpha in by hand is visually equivalent and three times cheaper. + """ + f = max(0.0, min(1.0, a / 255.0)) + return QColor(int(bg.red() + (col.red() - bg.red()) * f), + int(bg.green() + (col.green() - bg.green()) * f), + int(bg.blue() + (col.blue() - bg.blue()) * f)) + + +class HoloAvatar: + """Animated holographic head. One instance per HUD canvas. + + Lifecycle: + av = HoloAvatar() + av.step(dt, amp, speaking=..., muted=...) # once per tick + av.paint(painter, cx, cy, r, primary, accent, bg) # once per frame + """ + + # Look: True paints a lit, solid head with a wireframe over it; False is a + # see-through glass wireframe. Flip here, or per instance. + shaded = True + + def __init__(self) -> None: + mesh = get_head_mesh() + self._v0 = mesh["verts"] + self._n0 = mesh["normals"] + self._jaw = mesh["jaw"] + self._brow_w = mesh["brow"] + self._lips_w = mesh["lips"] + self._lip_c = mesh["lip_centre"] + self._fade = mesh["fade"] + self._f = mesh["faces"] + self._fgroup = mesh["face_group"] + self._fa, self._fb, self._fc = (self._f[:, i] for i in range(3)) + self._e0 = mesh["edges"][:, 0] + self._e1 = mesh["edges"][:, 1] + self._lm = mesh["landmarks"] + # The inner-lip ring runs lower-lip left→right, then upper-lip back. + # Splitting it lets the upper arc anchor a strip of teeth, which is what + # keeps an open mouth from reading as a hole punched in the face. + lips_in = mesh["landmarks"]["lips_in"] + self._lip_up = np.concatenate([lips_in[10:], lips_in[:1]]) + + # Crown (+1.0) down to the bottom of the neck, in head-half-heights. + # Callers size the head to the room they have with this. + self.SPAN = mesh["span"][0] - mesh["span"][1] + + self._lut_cache: list = [] + self._lut_key = None + + n = self._v0.shape[0] + self._v = np.empty((n, 3), dtype=np.float32) + + self._t = 0.0 + self._sway = 0.0 # integrated sway phase — see step() + self._yaw = 0.0 + self._pitch = 0.0 + self._mouth = 0.0 # 0..1 smoothed jaw opening + self._glow = 0.0 # 0..1 smoothed overall energy + self._scan = -1.6 # vertical position of the energy sweep + self._blink = 0.0 # 0 = open, 1 = shut + self._blink_at = 3.0 + + # ── expression ────────────────────────────────────────────────────── + # Speech is not just a moving jaw. Brows ride the loudness envelope, + # eyes widen with the brows, and the gaze flicks between fixation + # points — those three are what make it read as talking rather than as + # a puppet chewing. + self._amp_slow = 0.0 + self._expr = 0.0 + self._expr_tgt = 0.0 + self._expr_at = 0.0 + self._brow = 0.0 # smoothed brow lift, -0.4 .. 1.2 + self._emph = 0.0 # syllable emphasis, drives the head nod + self._gaze = [0.0, 0.0] + self._gaze_tgt = [0.0, 0.0] + self._gaze_at = 0.0 + + # ── state expression ──────────────────────────────────────────────── + # The face is the fastest status indicator in the app: you read a gaze + # before you read a word. Saccades orbit a bias that the assistant's + # state moves — eyes off to the side while it thinks, back on you while + # it listens, lids low while it sleeps. + self._gaze_bias = [0.0, 0.0] + self._bias_tgt = [0.0, 0.0] + self._bias_at = 0.0 + self._lids = 1.0 # 1 = wide, 0 = shut; low while asleep + self._brow_bias = 0.0 # concentration pulls the brows down + self._glance = None # (dx, dy, until_t) — a deliberate look + + # ── viseme ────────────────────────────────────────────────────────── + # Loudness alone only answers "how far open", which is why an RMS-driven + # mouth flaps rather than speaks. These two carry the *shape*: how open + # the jaw is for this sound, and whether the lips are spread (/i/) or + # rounded (/u/). They come from a formant read of the audio actually + # being played — see `_pcm_visemes` in main.py. + self._v_open = 1.0 + self._v_wide = 0.0 + self._wide = 0.0 # smoothed lip spread, -1 round .. +1 spread + self._v_peak = 0.18 # running estimate of this voice's loud level + + # ── animation ─────────────────────────────────────────────────────────── + + def _mouth_step(self, dt: float, amp: float, live: bool, + v_open: float | None, v_level: float | None) -> None: + """One increment of the jaw. Called once per viseme frame while JARVIS + speaks, once per rendered frame otherwise.""" + if v_open is None: + shape = 1.0 + else: + self._v_open += (v_open - self._v_open) * _rate(dt, _TAU_SHAPE) + shape = self._v_open + + if v_level is None: + gated = max(0.0, (amp - _MIC_FLOOR) / (1.0 - _MIC_FLOOR)) + drive = (gated ** 0.6) * (shape ** 0.75) + else: + # Speech RMS spends most of its time well below full scale, so the + # raw value alone would only ever half-open the jaw. Normalise it + # against a running estimate of this voice's own loud level rather + # than a constant: it then reads the same whether the user has the + # volume low or the model happens to be speaking softly. + self._v_peak = max(v_level, self._v_peak - dt * 0.55) + ref = max(0.18, self._v_peak) + # The floor is a fraction of this voice's own loud level, not a + # fixed number, so it means the same thing at any volume and in any + # language. A stop consonant drops 20 dB or more below the vowels + # around it, which is this ratio — so a real closure lands at + # exactly zero rather than at some small positive value the curve + # would otherwise lift back up. That lift is what kept the mouth + # from ever quite shutting between words. + q = (v_level - _CLOSE_FRAC * ref) / (ref * (1.0 - _CLOSE_FRAC)) + drive = max(0.0, min(1.0, q)) ** 0.85 * (shape ** 0.75) + + target = min(1.0, drive) if live else 0.0 + if target > self._mouth: + tau = _TAU_OPEN + elif live: + tau = _TAU_SHUT # mid-word: a consonant, and it must shut now + else: + tau = _TAU_REST # speech is over; settle, don't snap + self._mouth += (target - self._mouth) * _rate(dt, tau) + if self._mouth < 0.002: + self._mouth = 0.0 + + def step(self, dt: float, amp: float, speaking: bool = False, + muted: bool = False, state: str = "", + v_open: float | None = None, v_wide: float = 0.0, + v_level: float | None = None, + v_seq: list | None = None, v_hop: float = 0.02) -> None: + """Advance the animation. + + `amp` is the 0..1 display audio level. `v_open` / `v_wide` / `v_level` + are the viseme schedule's shape and true level for this instant; passing + None falls back to loudness-only articulation, which is what the + microphone path uses. `v_seq` is every schedule frame the last rendered + frame spanned, so no closure is lost when the paint rate drops. + """ + dt = max(0.001, min(0.10, float(dt))) + self._t += dt + t = self._t + amp = max(0.0, min(1.0, float(amp))) + live = speaking and not muted + + # Idle sway. The phase is *integrated* rather than taken as + # sin(t * rate * speed): multiplying absolute time by a speed that + # changes when JARVIS starts or stops talking jumps the phase by + # t * rate * delta, which after a minute of uptime is several radians + # and visibly teleports the head the instant a sentence ends. + speed = (1.0 if not muted else 0.55) * (1.25 if live else 1.0) + self._sway += dt * speed + s = self._sway + self._yaw = 0.26 * math.sin(s * 0.31) + 0.09 * math.sin(s * 0.73 + 1.3) + self._pitch = (0.060 * math.sin(s * 0.23 + 0.7) + + 0.024 * math.sin(s * 0.61)) + + # Mouth. Which level is driving it matters more than any rate here. + # + # `v_level` is this 20 ms frame's own RMS, taken from the very audio + # about to be heard, so its silences are real silences. `amp` is the + # waveform display's level, and that one is a *peak hold*: it keeps the + # loudest value it has seen and decays gently, on purpose, so the bars + # do not stutter between audio chunks. Driving a mouth from a peak hold + # is why the gaps between words never closed — the hold spans exactly + # the consonant it was supposed to reveal. So the schedule drives the + # jaw whenever there is one, and `amp` is left to the microphone path, + # which has nothing better. + # Advance the mouth once per *schedule* frame rather than once per + # rendered frame. A bilabial closure lasts around 40 ms — two frames of + # a 50 Hz schedule — and the HUD throttles its paint to 30 fps and to 20 + # when idle. Point-sampling at 20 fps steps 50 ms at a time, so a whole + # closure can fall between two samples and simply never be seen; that is + # information loss no smoothing constant can recover. Sub-stepping costs + # a few float operations per frame and makes the mouth identical at 20, + # 30 and 60 fps. + # An empty list is meaningful and is not the same as None: it says a + # schedule is playing but this tick landed inside a frame already + # spoken. The mouth's clock is the schedule's, so the right thing then + # is to do nothing. Re-stepping the same frame — which is what a + # truthiness test here would do — advances the jaw twice for one 20 ms + # of audio, and at 60 fps that alone made the mouth behave differently + # than at 20. + if v_seq is not None: + for lv, op, _wd in v_seq: + self._mouth_step(v_hop, amp, live, op, lv) + else: + self._mouth_step(dt, amp, live, v_open, v_level) + + # Syllable emphasis. Applied unconditionally: `_emph` decays to zero on + # its own once the mouth closes, whereas gating it on `live` deleted the + # whole offset in a single frame and snapped the head at sentence end. + self._emph += (self._mouth - self._emph) * _rate( + dt, 0.055 if self._mouth > self._emph else 0.32) + self._pitch -= self._emph * 0.028 + self._yaw += 0.018 * math.sin(t * 1.7) * self._emph + + # Loudness envelope, deliberately lazier than the mouth: brows track the + # shape of a phrase, not individual syllables. + env = amp if live else 0.0 + self._amp_slow += (env - self._amp_slow) * _rate( + dt, 0.16 if env > self._amp_slow else 0.36) + + if live: + if t >= self._expr_at: + self._expr_tgt = random.uniform(-0.35, 1.0) + self._expr_at = t + 1.1 + 2.0 * random.random() + else: + self._expr_tgt = 0.0 + self._expr_at = t + 0.8 + self._expr += (self._expr_tgt - self._expr) * 0.075 + + brow_t = 0.55 * self._amp_slow + 0.60 * self._expr + self._brow_bias + self._brow += (max(-0.4, min(1.2, brow_t)) - self._brow) * 0.20 + + # ── what the state does to the face ───────────────────────────────── + st = (state or "").upper() + thinking = st in ("THINKING", "PROCESSING") + asleep = st in ("SLEEPING", "STANDBY", "OFFLINE") + + if thinking: + # People look away to think, and hold it. The direction re-rolls + # slowly so it reads as thought rather than as scanning. + if t >= self._bias_at: + self._bias_tgt = [random.choice((-1.0, 1.0)) * random.uniform(0.45, 0.8), + random.uniform(0.25, 0.55)] + self._bias_at = t + 1.4 + 1.6 * random.random() + brow_bias, lid_tgt = -0.28, 0.94 + elif asleep: + self._bias_tgt = [0.0, -0.25] + brow_bias, lid_tgt = -0.05, 0.22 + else: + # LISTENING / idle / speaking: eyes come back to the user. + self._bias_tgt = [0.0, 0.0] + self._bias_at = 0.0 + brow_bias = 0.10 if st == "LISTENING" else 0.0 + lid_tgt = 1.0 + + for i in (0, 1): + self._gaze_bias[i] += (self._bias_tgt[i] - self._gaze_bias[i]) * 0.06 + self._lids += (lid_tgt - self._lids) * 0.08 + self._brow_bias += (brow_bias - self._brow_bias) * 0.06 + + # Gaze: saccades are near-instant jumps between fixations, and they get + # more frequent when there is something to say. While thinking they slow + # right down — a darting eye reads as nervous, not thoughtful. + if t >= self._gaze_at: + reach = 0.9 if live else (0.35 if thinking else 0.55) + self._gaze_tgt = [random.uniform(-1.0, 1.0) * reach, + random.uniform(-1.0, 1.0) * reach * 0.55] + if live: + self._gaze_at = t + 0.55 + 1.7 * random.random() + elif thinking: + self._gaze_at = t + 1.8 + 2.4 * random.random() + else: + self._gaze_at = t + 1.3 + 2.8 * random.random() + + # A deliberate glance (something appeared on screen) overrides the + # wandering for a moment, then hands control back. + if self._glance is not None: + gx, gy, until = self._glance + if t < until: + self._gaze_tgt = [gx, gy] + else: + self._glance = None + + for i, b in enumerate(self._gaze_bias): + tgt = max(-1.0, min(1.0, self._gaze_tgt[i] + b)) + self._gaze[i] += (tgt - self._gaze[i]) * 0.30 + + # Lips lead the jaw slightly in real speech, so they track a touch + # faster; they also relax to neutral the moment the voice stops. + wide_t = v_wide if (live and v_open is not None) else 0.0 + self._wide += (max(-1.0, min(1.0, wide_t)) - self._wide) * _rate(dt, 0.030) + + self._glow += ((0.0 if muted else amp) - self._glow) * ( + 0.35 if (0.0 if muted else amp) > self._glow else 0.10) + + self._scan += dt * (0.55 + 1.5 * self._glow) + if self._scan > 1.35: + self._scan = -1.75 + + if self._blink > 0.0: + self._blink = max(0.0, self._blink - dt * 8.5) + elif t >= self._blink_at: + # Concentration suppresses blinking; a sleeping face has no need of + # it at all, since the lids are already down. + if asleep: + self._blink_at = t + 6.0 + else: + self._blink = 1.0 + gap = 5.5 if thinking else 3.4 + self._blink_at = t + gap + 3.1 * random.random() + + def glance(self, dx: float, dy: float, hold: float = 1.1) -> None: + """Look deliberately somewhere for `hold` seconds, then wander again. + + Used when something appears on screen: a face that looks at what just + showed up tells the user it landed, without a word being spoken. + """ + self._glance = (max(-1.0, min(1.0, float(dx))), + max(-1.0, min(1.0, float(dy))), + self._t + max(0.1, float(hold))) + + # ── posing ────────────────────────────────────────────────────────────── + + def _pose(self): + """Jaw drop, brow lift and head rotation, applied to the real geometry.""" + v = self._v + np.copyto(v, self._v0) + + if self._brow > 0.004 or self._brow < -0.004: + # The brow-to-eye gap is 0.198 head-half-heights and a real raise + # moves a third of it. The old 0.045 — halved again by the landmark + # weights, which average 0.5 — worked out to six pixels on a 250 px + # head, which is to say invisible. + v[:, 1] += self._brow_w * (self._brow * _BROW_LIFT) + + if abs(self._wide) > 0.01 and self._mouth > 0.0: + # Spread pulls the corners out and flattens the lips back; rounding + # draws them in and pushes them forward into a purse. + k = self._lips_w * (self._wide * self._mouth) + v[:, 0] += k * (v[:, 0] - self._lip_c[0]) * 0.55 + v[:, 1] += k * (v[:, 1] - self._lip_c[1]) * 0.30 + v[:, 2] -= k * 0.055 + + if self._mouth > 0.004: + px, py, pz = JAW_PIVOT + ang = self._jaw * (self._mouth * JAW_MAX) + ca, sa = np.cos(ang), np.sin(ang) + dy = v[:, 1] - py + dz = v[:, 2] - pz + v[:, 1] = py + dy * ca - dz * sa + v[:, 2] = pz + dy * sa + dz * ca + + cy, sy = math.cos(self._yaw), math.sin(self._yaw) + cp, sp = math.cos(self._pitch), math.sin(self._pitch) + m = np.array([ + [cy, 0.0, sy], + [sp * sy, cp, -sp * cy], + [-cp * sy, sp, cp * cy], + ], dtype=np.float32) + + return v @ m.T, self._n0 @ m.T + + # ── rendering ─────────────────────────────────────────────────────────── + + def _lut(self, bg: QColor, primary: QColor): + """Cached ramp of opaque surface brushes from `bg` to `primary`.""" + key = (bg.rgb(), primary.rgb()) + if self._lut_key != key: + self._lut_cache = [QBrush(_blend(bg, primary, 255.0 * (i + 0.5) / _LUT_N)) + for i in range(_LUT_N)] + self._lut_key = key + return self._lut_cache + + def paint(self, p: QPainter, cx: float, cy: float, r: float, + primary: QColor, accent: QColor, bg: QColor | None = None) -> None: + """Draw the avatar with its head centre at (cx, cy). + + `r` is the head's half-height in pixels — the caller owns the layout, so + the HUD can fit the head to whatever room the status line leaves it. + """ + if bg is None: + bg = QColor(0, 0, 0) + amp = self._glow + verts, norms = self._pose() + + # ── aura ──────────────────────────────────────────────────────────── + ar = r * 1.95 + grad = QRadialGradient(cx, cy, ar) + grad.setColorAt(0.00, _c(primary, 34 + 66 * amp)) + grad.setColorAt(0.38, _c(primary, 20 + 40 * amp)) + grad.setColorAt(1.00, _c(primary, 0)) + p.setPen(Qt.PenStyle.NoPen) + p.setBrush(QBrush(grad)) + p.drawEllipse(QRectF(cx - ar, cy - ar, ar * 2, ar * 2)) + + # ── project ───────────────────────────────────────────────────────── + w = _CAM_D - verts[:, 2] + np.maximum(w, 0.35, out=w) + k = (_CAM_D / w) * r + xs = cx + verts[:, 0] * k + ys = cy - verts[:, 1] * k + + if self.shaded: + self._paint_surface(p, xs, ys, norms, verts, primary, bg, amp) + self._paint_wire(p, xs, ys, norms, verts, primary, bg, amp) + self._paint_features(p, xs, ys, norms, r, primary, accent, bg, amp) + + def _paint_surface(self, p: QPainter, xs, ys, norms, verts, + primary: QColor, bg: QColor, amp: float) -> None: + """Fill the camera-facing triangles so the head reads as a lit volume.""" + a, b, c = self._fa, self._fb, self._fc + + # Flat normals, taken from each triangle's own posed geometry — NOT the + # averaged vertex normals. Averaging smears the nose, lips and brow + # relief into their neighbours and renders the face as a blank egg; + # per-facet normals are exactly what makes the anatomy visible. + fn = np.cross(verts[b] - verts[a], verts[c] - verts[a]) + fn /= np.maximum(np.linalg.norm(fn, axis=1, keepdims=True), 1e-9) + + # Point them outwards by agreeing with the vertex normals, which were + # oriented at build time. Flipping on the sign of n_z instead would + # negate x and y as well and scramble the lighting into moiré. + ref = norms[a] + norms[b] + norms[c] + fn *= np.sign((fn * ref).sum(1))[:, None] + + nz = fn[:, 2] + area = np.abs((xs[b] - xs[a]) * (ys[c] - ys[a]) + - (xs[c] - xs[a]) * (ys[b] - ys[a])) + vis = np.flatnonzero((nz > 0.015) & (area > 3.0)) + if vis.size == 0: + return + fn = fn[vis] + nz = nz[vis] + + ax, ay = xs[a][vis], ys[a][vis] + bx, by = xs[b][vis], ys[b][vis] + cxx, cyy = xs[c][vis], ys[c][vis] + + # A rim term for the glass edge plus a key light high on the left. The + # light leans off-axis on purpose: weight it towards the camera and + # every front-facing facet returns the same value, which is a flat mask. + fres = np.clip(1.0 - nz, 0.0, 2.0) ** 1.7 + lam = np.clip(fn[:, 0] * -0.55 + fn[:, 1] * 0.50 + nz * 0.52, 0.0, 1.0) + bright = 0.26 + 0.20 * fres + 0.66 * lam ** 1.05 + bright *= (self._fade[a][vis] + self._fade[b][vis] + self._fade[c][vis]) / 3.0 + bright *= 0.88 + 0.24 * amp + + idx = np.clip((bright * _LUT_N).astype(np.int32), 0, _LUT_N - 1) + + # Far facets first: the neck passes behind the jaw and the head is not + # convex around the chin. + # Sort far-to-near, but group first: neck facets all draw before head + # facets, because the two meshes interpenetrate and a pure depth sort + # interleaves them into a torn seam. + fz = (verts[a, 2][vis] + verts[b, 2][vis] + verts[c, 2][vis]) * (1.0 / 3.0) + order = np.argsort(self._fgroup[vis] * 1000.0 + fz, kind="stable") + tris = np.stack([ax, ay, bx, by, cxx, cyy], axis=1)[order].tolist() + shade = idx[order].tolist() + lut = self._lut(bg, primary) + + # Aliased fills: adjacent antialiased polygons leave hairline seams, and + # the interior of a tiled surface has no silhouette worth smoothing — + # the antialiased wireframe drawn afterwards covers the outline. + p.setRenderHint(QPainter.RenderHint.Antialiasing, False) + p.setPen(Qt.PenStyle.NoPen) + for q, sh in zip(tris, shade): + p.setBrush(lut[sh]) + p.drawPolygon(QPolygonF([QPointF(q[0], q[1]), QPointF(q[2], q[3]), + QPointF(q[4], q[5])])) + p.setRenderHint(QPainter.RenderHint.Antialiasing, True) + + def _paint_wire(self, p: QPainter, xs, ys, norms, verts, + primary: QColor, bg: QColor, amp: float) -> None: + nz = norms[:, 2] + fres = np.abs(1.0 - np.abs(nz)) ** 1.5 + if self.shaded: + # The lit surface underneath is opaque, so back-facing edges would + # float on top of the face — cull them and let the wire read as + # structure lines over skin. + front = nz > -0.05 + va = np.where(front, 0.10 + 0.42 * fres, 0.0) + va += 0.30 * np.exp(-((verts[:, 1] - self._scan) / 0.13) ** 2) * front + else: + va = np.where(nz < 0.0, 0.13 + 0.26 * fres, 0.28 + 0.72 * fres) + va += 0.42 * np.exp(-((verts[:, 1] - self._scan) / 0.13) ** 2) + va *= self._fade * (0.80 + 0.45 * amp) + + ea = 0.5 * (va[self._e0] + va[self._e1]) + keep = np.flatnonzero(ea > _MIN_ALPHA) + if keep.size == 0: + return + + # Sort by opacity bucket once so each bucket is a contiguous *slice* of + # one QLineF list; masking and rebuilding per bucket cost more than the + # drawing itself. + bucket = np.clip((ea[keep] * _BUCKETS).astype(np.int32), 0, _BUCKETS - 1) + order = np.argsort(bucket, kind="stable") + keep = keep[order] + bounds = np.searchsorted(bucket[order], np.arange(_BUCKETS + 1)) + + e0, e1 = self._e0[keep], self._e1[keep] + quad = np.stack([xs[e0], ys[e0], xs[e1], ys[e1]], axis=1).tolist() + lines = [QLineF(q[0], q[1], q[2], q[3]) for q in quad] + + skin = _blend(bg, primary, 132) if self.shaded else bg + p.setBrush(Qt.BrushStyle.NoBrush) + for b in range(_BUCKETS): + lo, hi = int(bounds[b]), int(bounds[b + 1]) + if hi <= lo: + continue + seg = lines[lo:hi] + a = 255.0 * min(1.0, (b + 0.5) / _BUCKETS) + if self.shaded: + # These sit on lit skin, so pre-mix against a representative + # *skin* tone rather than the background — same fast opaque + # path, and the lines still read as highlights over the face. + p.setPen(QPen(_blend(skin, primary, a * 0.75), 1.0)) + else: + p.setPen(QPen(_blend(bg, primary, a), 1.0)) + p.drawLines(seg) + + # ── face ──────────────────────────────────────────────────────────────── + + def _ring(self, xs, ys, idx) -> QPolygonF: + return QPolygonF([QPointF(float(x), float(y)) + for x, y in zip(xs[idx], ys[idx])]) + + def _paint_features(self, p: QPainter, xs, ys, norms, r: float, + primary: QColor, accent: QColor, bg: QColor, + amp: float) -> None: + """Eyes, brows and the mouth cavity, drawn from the real landmark rings. + + The canonical model's eyes and lips are closed skin — the geometry gives + the *shape* of the lids and mouth but no opening, so the openings are + painted here, exactly on the landmarks that bound them. + """ + face = max(0.0, math.cos(self._yaw) * math.cos(self._pitch)) ** 2 + if face < 0.02: + return + + lm = self._lm + vis = 1.0 - self._blink + + # ── eyes ──────────────────────────────────────────────────────────── + for key in ("eye_l", "eye_r"): + idx = lm[key] + ex, ey = xs[idx], ys[idx] + mid_y = float(ey.mean()) + if vis < 0.999: + ey = mid_y + (ey - mid_y) * max(0.04, vis) + poly = QPolygonF([QPointF(float(a), float(b)) for a, b in zip(ex, ey)]) + + p.setPen(Qt.PenStyle.NoPen) + p.setBrush(QBrush(_blend(bg, primary, 22))) # socket shadow + p.drawPolygon(poly) + p.setBrush(Qt.BrushStyle.NoBrush) + p.setPen(QPen(_c(primary, 210 * face), 1.3)) # lid line + p.drawPolygon(poly) + + if vis > 0.35: + br = poly.boundingRect() + gx = br.center().x() + self._gaze[0] * br.width() * 0.16 + gy = br.center().y() + self._gaze[1] * br.height() * 0.20 + cpt = QPointF(gx, gy) + rad = min(br.height() * 0.62, br.width() * 0.20) + p.setPen(Qt.PenStyle.NoPen) + p.setBrush(QBrush(_c(accent, (70 + 60 * amp) * face * vis))) + p.drawEllipse(cpt, rad, rad * vis) # iris + p.setBrush(QBrush(_c(accent, 245 * face * vis))) + p.drawEllipse(cpt, rad * 0.42, rad * 0.42 * vis) # pupil + + # ── brows ─────────────────────────────────────────────────────────── + p.setBrush(Qt.BrushStyle.NoBrush) + p.setPen(QPen(_c(primary, 150 * face), 1.7)) + for key in ("brow_l", "brow_r"): + idx = lm[key] + p.drawPolyline(self._ring(xs, ys, idx)) + + # ── mouth ─────────────────────────────────────────────────────────── + inner = self._ring(xs, ys, lm["lips_in"]) + open_h = inner.boundingRect().height() + + p.setPen(Qt.PenStyle.NoPen) + if self._mouth > 0.02: + # The cavity is dark but never pure black — a black oval on a glowing + # head reads as a hole, not a mouth. Tinting it with the theme keeps + # it part of the hologram. + p.setBrush(QBrush(_blend(bg, primary, 16 + 26 * self._mouth))) + p.drawPolygon(inner) + + # Upper teeth: a bright strip hanging from the upper lip. It is the + # single cheapest thing that makes an open mouth look like speech. + ux, uy = xs[self._lip_up], ys[self._lip_up] + th = open_h * 0.30 + pts = [QPointF(float(x), float(y)) for x, y in zip(ux, uy)] + pts += [QPointF(float(x), float(y) + th) + for x, y in zip(ux[::-1], uy[::-1])] + p.setBrush(QBrush(_blend(bg, primary, 150 + 60 * self._mouth))) + p.drawPolygon(QPolygonF(pts)) + + # A warm pool at the back of the throat, strongest when wide open. + p.setBrush(QBrush(_c(accent, 40 * self._mouth * face))) + p.drawPolygon(inner) + + p.setBrush(Qt.BrushStyle.NoBrush) + p.setPen(QPen(_c(primary, (150 + 70 * self._mouth) * face), 1.3)) + p.drawPolygon(inner) # lip edge + p.setPen(QPen(_c(primary, 110 * face), 1.1)) + p.drawPolygon(self._ring(xs, ys, lm["lips_out"])) diff --git a/core/avatar_mesh.py b/core/avatar_mesh.py new file mode 100644 index 0000000..b426f01 --- /dev/null +++ b/core/avatar_mesh.py @@ -0,0 +1,351 @@ +""" +Human head mesh for the HUD avatar. + +The face is **real measured human geometry** — MediaPipe's canonical face model +(`core/face_model.obj`, Apache-2.0, 468 vertices / 898 triangles), which carries +actual eyelids, nostrils, lips and cheekbones. Everything a formula cannot give +you comes from there. + +Earlier revisions of this file generated the whole head procedurally from an +ellipsoid pushed around by gaussians. It could be tuned endlessly and still read +as an egg with a face drawn on it, because there was no human anatomy in it — +only smooth blobs. Real topology fixed in one step what parameter tweaking could +not fix at all. + +What is still generated here, around that face: + * the cranium — the model is an open mask, so its 36-vertex border is swept + back and up over a skull-shaped ellipsoid and closed at the occiput; + * a tapering neck stub that fades out instead of needing shoulders; + * vertex normals, jaw-rig weights, a thinned wireframe, and the landmark + index rings (eyes, brows, lips) the renderer animates. + +Coordinate system after normalisation (head-local, right-handed): + +x → viewer's right +y → up +z → out of the face + y = +1.0 crown, y = -1.0 chin, eyes land on y ≈ 0. +""" + +from __future__ import annotations + +import collections +from pathlib import Path + +import numpy as np + +_OBJ = Path(__file__).resolve().parent / "face_model.obj" + +# Cranium shape, in the model's own units (chin ≈ -9.4, forehead ≈ +8.3). +# Tuned so that brow→crown is ~0.36 of the head's height, which is the real +# proportion; a taller cranium than that immediately reads as a long face even +# though the face itself is untouched measured geometry. +_SKULL_C = (0.0, 2.0, -1.0) # centre of the cranial ellipsoid +_SKULL_R = (8.4, 12.4, 8.2) # its radii +_SKULL_POLE = (0.0, 0.42, -1.0) # direction of the occiput, where the sweep closes +_SKULL_RINGS = 6 +_SKULL_BLEND = 1.7 # how fast the sweep leaves the face border +_SKULL_BULGE = 1.04 + +_NECK_RINGS, _NECK_SEGS = 9, 14 +_NECK_Z = -1.6 # the neck tube's axis, in model units +_WIRE_STRIDE = 3 # keep every n-th edge; the surface carries the form + +# MediaPipe landmark rings. Verified against the geometry at build time — see +# `_check_landmarks` — so a wrong index can never silently animate the cheek. +LANDMARKS: dict[str, list[int]] = { + "eye_l": [33, 7, 163, 144, 145, 153, 154, 155, 133, 173, 157, 158, 159, + 160, 161, 246], + "eye_r": [263, 249, 390, 373, 374, 380, 381, 382, 362, 398, 384, 385, 386, + 387, 388, 466], + "brow_l": [70, 63, 105, 66, 107], + "brow_r": [300, 293, 334, 296, 336], + "lips_out": [61, 146, 91, 181, 84, 17, 314, 405, 321, 375, 291, 409, 270, + 269, 267, 0, 37, 39, 40, 185], + "lips_in": [78, 95, 88, 178, 87, 14, 317, 402, 318, 324, 308, 415, 310, + 311, 312, 13, 82, 81, 80, 191], +} + +# Jaw rig, in normalised units. The pivot sits between the ears, which is where +# a real mandible hinges. +JAW_PIVOT = (0.0, 0.06, -0.34) +JAW_MAX = 0.115 # radians of drop at full amplitude (~6.6°) +# Speech barely moves a real jaw, and a talking head is watched at HUD size +# where a small, precise mouth reads better than a large one. The lip rig +# (spread / round) now carries most of the articulation, so the jaw does not +# have to swing to show that something is being said. + + +def _load_obj(path: Path): + verts, faces = [], [] + for line in path.read_text(encoding="utf-8").splitlines(): + if line.startswith("v "): + verts.append([float(x) for x in line.split()[1:4]]) + elif line.startswith("f "): + faces.append([int(t.split("/")[0]) - 1 for t in line.split()[1:4]]) + return np.array(verts, dtype=np.float64), np.array(faces, dtype=np.int64) + + +def _boundary_loop(faces: np.ndarray) -> np.ndarray: + """Ordered ring of vertices along the open border of a triangle mesh.""" + seen = collections.Counter() + for a, b, c in faces: + for e in ((a, b), (b, c), (c, a)): + seen[(min(e), max(e))] += 1 + border = [e for e, n in seen.items() if n == 1] + + adj = collections.defaultdict(list) + for a, b in border: + adj[a].append(b) + adj[b].append(a) + + start = border[0][0] + loop, prev, cur = [start], None, start + while True: + nxt = [v for v in adj[cur] if v != prev] + if not nxt or nxt[0] == start: + break + prev, cur = cur, nxt[0] + loop.append(cur) + return np.array(loop) + + +def _slerp(a: np.ndarray, b: np.ndarray, t): + dot = np.clip((a * b).sum(-1, keepdims=True), -1.0, 1.0) + om = np.arccos(dot) + so = np.sin(om) + safe = np.where(so < 1e-6, 1.0, so) + out = np.where(so < 1e-6, a * (1 - t) + b * t, + (np.sin((1 - t) * om) / safe) * a + (np.sin(t * om) / safe) * b) + return out / np.maximum(np.linalg.norm(out, axis=-1, keepdims=True), 1e-9) + + +def _add_cranium(verts: np.ndarray, faces: np.ndarray): + """Sweep the mask's open border back over a skull and close it at the occiput.""" + loop = _boundary_loop(faces) + + # Orient the loop so the generated triangles wind the same way as the face's. + centre2d = verts[loop, :2].mean(0) + ang = np.arctan2(verts[loop, 1] - centre2d[1], verts[loop, 0] - centre2d[0]) + if np.diff(np.unwrap(ang)).sum() < 0: + loop = loop[::-1] + + n = len(loop) + C = np.array(_SKULL_C) + R = np.array(_SKULL_R) + pole = np.array(_SKULL_POLE) + pole = pole / np.linalg.norm(pole) + chin_y = verts[:, 1].min() + + rim = verts[loop] - C + rim_r = np.linalg.norm(rim, axis=1, keepdims=True) + rim_d = rim / rim_r + + def ell_r(d): + return 1.0 / np.sqrt(((d / R) ** 2).sum(-1, keepdims=True)) + + out_v = [verts] + out_f = list(faces) + prev_idx = loop + ts = np.linspace(0.0, 1.0, _SKULL_RINGS + 1)[1:] + + for t in ts: + d = _slerp(pole, rim_d, 1.0 - t) + w = (1.0 - t) ** _SKULL_BLEND # meets the rim exactly at t = 0 + # A skull is fuller than the border it springs from; peak it mid-sweep. + r = ell_r(d) * (1.0 + (_SKULL_BULGE - 1.0) * np.sin(np.pi * t) ** 0.8) + ring = C + d * (w * rim_r + (1.0 - w) * r) + # Never dip below the chin: the sweep passing under the jaw would + # otherwise hang a lip of geometry below the face. Vertices that hit + # the clamp are also drawn in towards the neck axis, so the underside + # closes as a small floor instead of a flat skirt sticking out. + below = ring[:, 1] < chin_y + if below.any(): + ring[below, 1] = chin_y + ring[below, 0] *= 0.55 + ring[below, 2] = _NECK_Z + (ring[below, 2] - _NECK_Z) * 0.55 + if t == ts[-1]: + ring = np.repeat((C + pole * ell_r(pole[None])[0])[None], n, axis=0) + + base = sum(len(a) for a in out_v) + out_v.append(ring) + idx = np.arange(base, base + n) + for i in range(n): + a0, b0 = prev_idx[i], prev_idx[(i + 1) % n] + a1, b1 = idx[i], idx[(i + 1) % n] + out_f.append([a0, a1, b1]) + out_f.append([a0, b1, b0]) + prev_idx = idx + + return np.vstack(out_v), np.array(out_f, dtype=np.int64) + + +def _add_neck(verts: np.ndarray, faces: np.ndarray): + """A tapering tube dropped from inside the jaw; it fades out, so no shoulders.""" + ph = np.linspace(0.0, 2.0 * np.pi, _NECK_SEGS, endpoint=False) + # Short, and flaring hard at the bottom: a straight vertical tube reads as + # a pedestal, whereas a neck that widens into the top of the shoulders + # reads as a bust — and the shorter it is, the larger the head can be drawn + # in the same HUD band. + ys = np.linspace(-5.5, -13.0, _NECK_RINGS) + d = (ys + 5.5) / -7.5 + + rx = 4.6 * (1.0 + 0.52 * d ** 1.9) + rz = 4.1 * (1.0 + 0.38 * d ** 1.9) + nx = rx[:, None] * np.cos(ph)[None, :] + nz = _NECK_Z + rz[:, None] * np.sin(ph)[None, :] + ny = ys[:, None] * np.ones_like(ph)[None, :] + + nv = np.stack([nx.ravel(), ny.ravel(), nz.ravel()], axis=1) + base = len(verts) + idx = base + np.arange(_NECK_RINGS * _NECK_SEGS).reshape(_NECK_RINGS, _NECK_SEGS) + + nf = [] + for i in range(_NECK_RINGS - 1): + for j in range(_NECK_SEGS): + a, b = idx[i, j], idx[i, (j + 1) % _NECK_SEGS] + c, e = idx[i + 1, (j + 1) % _NECK_SEGS], idx[i + 1, j] + nf.append([a, b, c]) + nf.append([a, c, e]) + + # Enough rings that the fade steps stay small. Each quad splits into one + # triangle with two top vertices and one with two bottom vertices, so a + # steep per-vertex fade gradient makes the pair land on visibly different + # brightnesses and the neck grows a sawtooth edge. + fade = np.ones(base) + nd = np.repeat(d, _NECK_SEGS) + fade = np.concatenate([fade, 1.0 - 0.72 * np.clip(nd, 0.0, 1.0) ** 1.5]) + return np.vstack([verts, nv]), np.vstack([faces, np.array(nf)]), fade + + +def _vertex_normals(verts: np.ndarray, faces: np.ndarray, + outward: np.ndarray) -> np.ndarray: + """Area-weighted vertex normals, flipped to agree with `outward`. + + `outward` must be a per-vertex direction that genuinely points out of the + surface. A single "away from the mesh centroid" rule is NOT good enough: + down at the base of the neck that vector points almost straight down while + the real normal is horizontal, so the dot product hovers around zero and + the sign flips at random — which tears the neck into an asymmetric slab of + half-culled, half-lit triangles. + """ + a, b, c = verts[faces[:, 0]], verts[faces[:, 1]], verts[faces[:, 2]] + fn = np.cross(b - a, c - a) # length carries the area — the weighting + + n = np.zeros_like(verts) + for k in range(3): + np.add.at(n, faces[:, k], fn) + n /= np.maximum(np.linalg.norm(n, axis=1, keepdims=True), 1e-9) + + flip = (n * outward).sum(1) < 0 + n[flip] *= -1.0 + return n + + +def _unique_edges(faces: np.ndarray) -> np.ndarray: + e = np.vstack([faces[:, [0, 1]], faces[:, [1, 2]], faces[:, [2, 0]]]) + e = np.sort(e, axis=1) + return np.unique(e, axis=0) + + +def _check_landmarks(verts: np.ndarray) -> None: + """Fail loudly at build time if a landmark ring is not where it should be.""" + for left, right in (("eye_l", "eye_r"), ("brow_l", "brow_r")): + cl = verts[LANDMARKS[left]].mean(0) + cr = verts[LANDMARKS[right]].mean(0) + assert cl[0] < 0 < cr[0], f"{left}/{right} are not on opposite sides" + assert abs(cl[1] - cr[1]) < 0.5, f"{left}/{right} are at different heights" + eye_y = verts[LANDMARKS["eye_l"]].mean(0)[1] + brow_y = verts[LANDMARKS["brow_l"]].mean(0)[1] + lips = verts[LANDMARKS["lips_out"]].mean(0) + assert brow_y > eye_y, "brow is not above the eye" + assert lips[1] < eye_y, "lips are not below the eyes" + assert abs(lips[0]) < 0.5, "lips are not centred" + + +def build_head() -> dict: + """Assemble the full head. Called once; `get_head_mesh()` caches the result.""" + verts, faces = _load_obj(_OBJ) + _check_landmarks(verts) + + n_face = len(verts) + verts, faces = _add_cranium(verts, faces) + n_head = len(verts) + verts, faces, fade = _add_neck(verts, faces) + + # ── normalise: crown → +1, chin → -1, eyes land on y ≈ 0 ──────────────── + head_y = verts[:n_head, 1] + crown, chin = head_y.max(), head_y.min() + scale = 2.0 / (crown - chin) + centre = np.array([0.0, (crown + chin) * 0.5, 0.0]) + verts = (verts - centre) * scale + + # Outward reference, per part: the head is star-shaped about its own centre, + # while the neck is a tube whose outward direction is radial in x/z only. + outward = verts - np.array([0.0, verts[:n_head, 1].mean(), 0.0]) + outward[n_head:] = verts[n_head:] - np.array([0.0, 0.0, _NECK_Z * scale]) + outward[n_head:, 1] = 0.0 + normals = _vertex_normals(verts, faces, outward) + + # ── jaw rig ───────────────────────────────────────────────────────────── + # Everything below the mouth swings on the mandible, tapering to nothing at + # the ears and around the back so the nape and the neck stay put. + mouth_y = verts[LANDMARKS["lips_out"], 1].mean() + chin_y = verts[:n_head, 1].min() + jaw = np.clip((mouth_y - verts[:, 1]) / (mouth_y - chin_y), 0.0, 1.0) ** 0.8 + jaw *= np.clip(0.30 + 0.85 * (verts[:, 2] / 0.55), 0.0, 1.0) + jaw[n_head:] = 0.0 # the neck never moves + jaw[LANDMARKS["lips_in"][:10]] = 1.0 # lower inner lip leads + jaw[LANDMARKS["lips_out"][:10]] = 0.95 + + # ── brow rig ──────────────────────────────────────────────────────────── + # Raising the brows displaces the actual surface rather than sliding a drawn + # line over it, so the brow ridge relights as it lifts. + brow_y = verts[LANDMARKS["brow_l"] + LANDMARKS["brow_r"], 1].mean() + brow = np.exp(-((verts[:, 1] - brow_y) / 0.115) ** 2) + brow *= np.clip(verts[:, 2] / 0.35, 0.0, 1.0) # front of the face only + brow *= np.exp(-(verts[:, 0] / 0.42) ** 2) # fades out past the temples + brow[n_head:] = 0.0 + + # ── lip rig ───────────────────────────────────────────────────────────── + # Vowels are not just "how far open" — /i/ spreads the lips wide, /u/ purses + # them forward. This weight lets the renderer widen or round the mouth + # region as a whole, so the surrounding skin follows instead of tearing away + # from the landmark rings. + lip_c = verts[LANDMARKS["lips_out"]].mean(axis=0) + lips = np.exp(-((verts[:, 1] - lip_c[1]) / 0.155) ** 2) + lips *= np.exp(-(verts[:, 0] / 0.30) ** 2) + lips *= np.clip(verts[:, 2] / 0.40, 0.0, 1.0) + lips[n_head:] = 0.0 + + edges = _unique_edges(faces)[::_WIRE_STRIDE] + + # Neck and head interpenetrate, and a painter's-algorithm sort by triangle + # depth interleaves them into a torn edge. Grouping fixes it: the neck is + # always behind the head where they overlap, so draw every neck facet first. + face_group = (faces >= n_head).all(axis=1).astype(np.int32) # 1 = neck + + return { + "face_group": np.ascontiguousarray(1 - face_group, dtype=np.float32), + "brow": np.ascontiguousarray(brow, dtype=np.float32), + "lips": np.ascontiguousarray(lips, dtype=np.float32), + "lip_centre": np.ascontiguousarray(lip_c, dtype=np.float32), + "verts": np.ascontiguousarray(verts, dtype=np.float32), + "normals": np.ascontiguousarray(normals, dtype=np.float32), + "faces": np.ascontiguousarray(faces, dtype=np.int32), + "edges": np.ascontiguousarray(edges, dtype=np.int32), + "jaw": np.ascontiguousarray(jaw, dtype=np.float32), + "fade": np.ascontiguousarray(fade, dtype=np.float32), + "landmarks": {k: np.array(v, dtype=np.int32) for k, v in LANDMARKS.items()}, + "n_face": n_face, + "n_head": n_head, + "span": (1.0, float(verts[:, 1].min())), # crown, bottom of the neck + } + + +_CACHE: dict | None = None + + +def get_head_mesh() -> dict: + """Process-wide cached mesh — every HudCanvas shares the same arrays.""" + global _CACHE + if _CACHE is None: + _CACHE = build_head() + return _CACHE diff --git a/core/echo.py b/core/echo.py new file mode 100644 index 0000000..71a142d --- /dev/null +++ b/core/echo.py @@ -0,0 +1,288 @@ +""" +Telling the user's voice apart from our own coming back through the speakers. + +What it is used for today +------------------------- +The *echo tail*: for a moment after a reply ends, sound is still leaving the +speakers, because writing to an audio device returns when the buffer accepts the +audio and not when the room has finished with it. Streaming the microphone +during that gap is how an assistant hears its own last sentence and answers +itself. This module lets the tail drop our own voice while still passing a user +who replies instantly, instead of blanket-muting. + +Interrupting mid-sentence by voice is the other thing it can do, and everything +needed for it is here and tested — but it depends on the listener's room, so it +stays switched off until it can be tried on real hardware. See the note in +`_listen_audio` for how little it takes to turn back on. + +The problem with a level test +----------------------------- +Deciding "the microphone is louder than the echo" needs to know how loud the +echo *is*, and that is not a constant. It depends on the speaker volume, where +the microphone sits, the room, whether headphones are plugged in, and whether +the user happens to have a hand over the mic. A single tuned number is wrong for +almost everyone: too eager on a loud desktop speaker, too deaf on a quiet laptop. + +What this does instead +---------------------- +Two independent signals, neither of which needs tuning per machine: + +1. **Content.** Echo is not merely loud — it is *the same sound we just played*. + Both streams are reduced to a handful of band energies, and then — rather + than merely compared — as much of the recent output as fits is *subtracted* + from the microphone block. Pure echo cancels to almost nothing. A second + voice cannot be cancelled by ours, because its formants sit in bands where + ours were weak, so it survives the subtraction. That distinction holds even + when the two arrive at the same loudness, which is exactly the case a level + test gets wrong. + +2. **A learned echo gain.** Whenever the content check says "that is definitely + just echo", the observed mic-to-output ratio is folded into a running + estimate. The system therefore calibrates itself to the actual room within a + few seconds of the first sentence, and re-calibrates when the volume changes + or a hand covers the microphone. + +Band energies rather than raw spectra also make the comparison sample-rate +agnostic, which matters here: the microphone runs at 16 kHz and playback at +24 kHz. +""" + +from __future__ import annotations + +import time + +import numpy as np + +# Log-spaced edges across the range that carries speech. Coarse on purpose: +# fine bins would track pitch, and pitch is exactly what differs between two +# people saying the same word — we want the *timbre* that echo preserves. +_BAND_EDGES = (200, 400, 700, 1100, 1700, 2600, 3800, 5200, 7000) + +_HISTORY_S = 1.5 # how far back an echo could plausibly have been played +_MIN_LEVEL = 0.06 # below this the mic is room noise; nothing to decide + +# The threshold is not a tuned constant. It is placed just above essentially ALL +# the echo this particular room has been seen to produce. Residuals measured on +# synthetic material, median over many blocks: +# +# room echo p99 a distinct voice a similar voice +# headphones 0.09 0.28 0.16 +# normal desk 0.13 0.28 0.16 +# reverberant room 0.23 0.27 0.18 +# loud, distorting, noisy 0.31 0.33 0.26 +# +# Read that table honestly. How good the room is decides what is possible: in a +# quiet setup even a voice whose formants sit close to ours stands clear of the +# echo, and in a bad one nothing reliably does. The right response to the last +# row is to say so — not to interrupt at random. +_MIN_USER = 0.15 # never call anything below this a voice +_HEAD_Q = 97 # percentile of observed echo the threshold must clear +_HEAD_MULT = 1.15 # ...with this much headroom above it + +# Above this echo floor, content separates the two poorly. Loudness cannot +# rescue it — in exactly those rooms the echo already saturates the level meter, +# so a level test can never be passed. Time can: echo wobbles around its median +# while a person talking holds above it, so such rooms hold the evidence longer. +_UNRELIABLE_FLOOR = 0.22 +_BLOCKS_NORMAL = 5 # ~320 ms of sustained evidence +_BLOCKS_NOISY = 12 # ~770 ms when the room is this hard + +_FLOOR_WINDOW = 60 # blocks kept to characterise the room's echo +_FLOOR_Q = 35 # percentile taken as "typical echo here" +_WARMUP = 16 # blocks (~1 s) of listening before judging anyone +_RELEARN_RUN = 28 # a 'voice' lasting this long means the room changed + + +def band_energies(pcm, sr: int) -> np.ndarray: + """Raw energy per speech band — the fingerprint we compare. + + Left unnormalised on purpose: the decision below projects one of these onto + another, and a projection needs real magnitudes. + """ + x = np.asarray(pcm, dtype=np.float32) + if x.size < 64: + return np.zeros(len(_BAND_EDGES) - 1, dtype=np.float32) + x = x - x.mean() + n = 1 << (int(x.size) - 1).bit_length() # next power of two + mag = np.abs(np.fft.rfft(x * np.hanning(x.size), n=n)) + freqs = np.fft.rfftfreq(n, 1.0 / sr) + out = np.empty(len(_BAND_EDGES) - 1, dtype=np.float32) + for i in range(len(_BAND_EDGES) - 1): + m = (freqs >= _BAND_EDGES[i]) & (freqs < _BAND_EDGES[i + 1]) + out[i] = float(mag[m].sum()) + return out + + +class EchoGuard: + """Classifies microphone blocks while the assistant is speaking. + + Usage: `note_output()` from the playback path, `is_user_speech()` from the + microphone callback. Both are cheap enough to sit in an audio thread — one + small FFT each. + """ + + def __init__(self) -> None: + self._hist: list[tuple[float, np.ndarray, float]] = [] # (t, bands, level) + self._gain = 0.6 # mic level per unit of output level; learned + self._seen = 0 # how many echo blocks the estimate has seen + self._last_sim = 0.0 + self._last_expected = 0.0 + self._residuals: list[float] = [] # recent ECHO residuals only + self._floor = 0.10 # typical echo residual here; learned + self._run = 0 # consecutive blocks called speech + self._head = 0.13 # near-worst echo residual here; learned + + # ── diagnostics, for the log and for tests ────────────────────────────── + + @property + def gain(self) -> float: + return self._gain + + @property + def calibrated(self) -> bool: + """True once the estimate rests on enough real echo to be trusted.""" + return self._seen >= 8 + + @property + def floor(self) -> float: + """Residual left by this room's own echo. Higher = harder to separate.""" + return self._floor + + @property + def reliable(self) -> bool: + """False when the acoustics are too poor to judge on content alone. + + Speakers turned up in a reverberant room leave an echo that survives the + subtraction almost as well as a quiet voice does. Rather than guess, the + guard says so and starts demanding corroborating evidence. + """ + return self._floor < _UNRELIABLE_FLOOR + + @property + def threshold(self) -> float: + """The residual a block must clear right now to count as a voice.""" + return max(_MIN_USER, self._head * _HEAD_MULT) + + @property + def required_blocks(self) -> int: + """Consecutive positive blocks before an interruption is believed.""" + return _BLOCKS_NORMAL if self.reliable else _BLOCKS_NOISY + + @property + def last_similarity(self) -> float: + """1.0 = fully explained by our own output, 0.0 = nothing to do with it.""" + return self._last_sim + + def reset(self) -> None: + """Playback stopped — drop the history, keep what was learned.""" + self._hist.clear() + self._last_sim = 0.0 + + # ── the two entry points ──────────────────────────────────────────────── + + def note_output(self, pcm, sr: int, level: float, when: float | None = None) -> None: + """Record a slice of what is being played, for later comparison.""" + try: + t = time.monotonic() if when is None else when + self._hist.append((t, band_energies(pcm, sr), float(level))) + cutoff = t - _HISTORY_S + if len(self._hist) > 8: + self._hist = [h for h in self._hist if h[0] >= cutoff] + except Exception: + pass # never let bookkeeping disturb playback + + def is_user_speech(self, pcm, sr: int, level: float, + when: float | None = None) -> bool: + """True if this microphone block is a different voice, not our echo.""" + try: + if level < _MIN_LEVEL: + # We are playing and the microphone hears nothing back: + # this setup returns no echo at all (headphones, or a mic + # far from the speaker). Record that, or the window stays + # empty and the first person to speak gets mistaken for + # the calibration sample. + if self._hist and max(h[2] for h in self._hist) > 0.15: + self._residuals.append(0.0) + if len(self._residuals) > _FLOOR_WINDOW: + del self._residuals[:-_FLOOR_WINDOW] + if len(self._residuals) >= _WARMUP: + self._floor = float(np.percentile(self._residuals, _FLOOR_Q)) + self._head = float(np.percentile(self._residuals, _HEAD_Q)) + self._run = 0 + return False + if not self._hist: + # Nothing playing that we know of — anything audible is theirs. + return True + + t = time.monotonic() if when is None else when + bands = band_energies(pcm, sr) + total = float(bands.sum()) + if total <= 1e-9: + return False + + # Find the recent output slice that best explains this block, and + # subtract as much of it as fits. What is left over is whatever the + # microphone heard that we did not play. + best_res, best_level = 1.0, 0.0 + for ts, ref, ref_level in self._hist: + if ts > t or t - ts > _HISTORY_S: + continue + denom = float(np.dot(ref, ref)) + if denom < 1e-12: + continue + alpha = max(0.0, float(np.dot(bands, ref)) / denom) + residual = np.maximum(bands - alpha * ref, 0.0) + ratio = float(residual.sum()) / total + if ratio < best_res: + best_res, best_level = ratio, ref_level + self._last_sim = 1.0 - best_res + + # Nothing in the window explained it at all — that is not our sound. + if best_level <= 0.0: + return True + + # Learning happens in two stages, and the order matters both times. + # + # While warming up, learn from EVERY block. Judging first is a trap: + # in a poor room the very first echo block already sits above any + # sensible starting bar, so it would be called a voice, never be + # learned from, and the room would stay mis-characterised forever. + # + # Once warm, learn only from blocks BELOW the bar. Feeding the + # user's own blocks back in is the opposite trap: they lift the + # threshold over themselves and the guard goes deaf to the very + # thing it is watching for. + warming = len(self._residuals) < _WARMUP + thr = self.threshold + speech = (not warming) and best_res >= thr + + if speech: + self._run += 1 + # A run this long is not someone interrupting — nobody talks + # over an assistant for seconds on end. The room changed under + # us (volume, headphones unplugged), so characterise it again. + if self._run > _RELEARN_RUN: + self._residuals.clear() + self._run = 0 + return False + return True + + self._run = 0 + self._residuals.append(best_res) + if len(self._residuals) > _FLOOR_WINDOW: + del self._residuals[:-_FLOOR_WINDOW] + if len(self._residuals) >= _WARMUP: + self._floor = float(np.percentile(self._residuals, _FLOOR_Q)) + self._head = float(np.percentile(self._residuals, _HEAD_Q)) + if warming: + return False + + # Comfortably just us: also a safe moment to learn how loudly this + # room returns our voice. + if best_res <= self._floor * 1.15 and best_level > 0.05: + obs = level / max(best_level, 1e-6) + self._gain += (min(obs, 3.0) - self._gain) * 0.08 + self._seen = min(self._seen + 1, 999) + return False + except Exception: + return False # any doubt: do not interrupt diff --git a/core/face_model.obj b/core/face_model.obj new file mode 100644 index 0000000..24855a9 --- /dev/null +++ b/core/face_model.obj @@ -0,0 +1,1378 @@ +# Canonical human face mesh — 468 vertices, 898 triangles. +# +# Source : MediaPipe "canonical_face_model.obj" +# https://github.com/google-ai-edge/mediapipe +# Licence: Apache License 2.0 — Copyright 2019-2023 The MediaPipe Authors. +# A copy of the licence is at http://www.apache.org/licenses/LICENSE-2.0 +# +# Only the geometry (v/f) is kept; the texture coordinates were dropped because +# the avatar shades the surface analytically and never samples a texture. +# The cranium, neck and jaw rig are generated around this face by +# core/avatar_mesh.py — this file is the face itself, and it is real measured +# human geometry, which is the whole reason the avatar reads as a person. +v 0.0000 -3.4064 5.9795 +v 0.0000 -1.1269 7.4756 +v 0.0000 -2.0890 6.0583 +v -0.4639 0.9554 6.6336 +v 0.0000 -0.4632 7.5866 +v 0.0000 0.3657 7.2429 +v 0.0000 2.4733 5.7886 +v -4.2531 2.5776 3.2797 +v 0.0000 4.0190 5.2848 +v 0.0000 4.8860 5.3853 +v 0.0000 8.2618 4.4815 +v 0.0000 -3.7068 5.8649 +v 0.0000 -3.9183 5.5694 +v 0.0000 -3.9944 5.2195 +v 0.0000 -4.5424 5.4048 +v 0.0000 -4.7456 5.5295 +v 0.0000 -5.0196 5.6014 +v 0.0000 -5.3651 5.5354 +v 0.0000 -6.1496 5.0714 +v 0.0000 -1.5011 7.1122 +v -0.4161 -1.4664 6.4477 +v -7.0880 5.4348 0.0996 +v -2.6286 2.0359 3.8481 +v -3.1984 1.9858 3.7970 +v -3.7752 2.0394 3.6462 +v -4.4658 2.4230 3.1552 +v -2.1643 2.1899 3.8518 +v -3.2082 3.2239 4.1158 +v -2.6738 3.2053 4.0922 +v -3.7452 3.1653 3.9724 +v -4.1610 3.0591 3.7196 +v -5.0620 1.9344 2.7761 +v -2.2667 -7.4258 4.3898 +v -4.4459 2.6640 3.1734 +v -7.2145 2.2630 0.0732 +v -5.7998 2.3495 2.2041 +v -2.8449 -0.7209 4.4331 +v -0.7115 -3.3294 5.8770 +v -0.6060 -3.9246 5.4449 +v -1.4316 -3.5010 5.4962 +v -1.9149 -3.8031 5.0289 +v -1.1310 -3.9739 5.1896 +v -1.5635 -4.0828 4.8423 +v -2.6501 -5.0036 4.1885 +v -0.4270 -1.0941 7.3605 +v -0.4964 -0.4757 7.4404 +v -5.2533 3.8816 3.3632 +v -1.7187 0.9746 4.5584 +v -1.6086 -0.9425 5.8142 +v -1.6513 -0.6109 5.5813 +v -4.7655 -0.7016 3.5346 +v -0.4783 0.2958 7.1010 +v -3.7350 4.5082 4.5505 +v -4.5886 4.3020 4.0485 +v -6.2793 6.6154 1.4259 +v -1.2209 4.1422 5.1060 +v -2.1935 3.1003 4.0006 +v -3.1026 -4.3530 4.0959 +v -6.7197 -4.7886 -1.7454 +v -1.1938 -1.3068 5.7377 +v -0.7298 -1.5937 5.8332 +v -2.4562 -4.3426 4.2839 +v -2.2048 -4.3045 4.1625 +v -4.9859 4.8025 3.7520 +v -1.5923 -1.2577 5.4569 +v -2.6445 4.5247 4.9216 +v -2.7603 5.1010 5.0160 +v -3.5240 8.0060 3.7292 +v -5.5998 5.7155 2.7243 +v -3.0639 6.5661 4.5300 +v -5.7210 4.2546 2.8309 +v -6.3744 4.7856 1.5917 +v -0.6727 -3.6880 5.7378 +v -1.2626 -3.7877 5.4178 +v -1.7326 -3.9528 5.0006 +v -1.0436 -1.4650 5.6625 +v -2.3212 -4.3291 4.2582 +v -2.0568 -4.4777 4.5209 +v -2.1531 -4.2763 4.0381 +v -0.9469 -1.0352 6.5123 +v -1.4691 -4.0364 4.6049 +v -1.0243 -3.9899 4.9267 +v -0.5334 -3.9932 5.1382 +v -0.7697 -6.0954 4.9859 +v -0.6996 -5.2919 5.4483 +v -0.6697 -4.9498 5.5096 +v -0.6309 -4.6951 5.4494 +v -0.5832 -4.5180 5.3399 +v -1.5372 -4.4232 4.7455 +v -1.6156 -4.4759 4.8136 +v -1.7291 -4.6187 4.8545 +v -1.8386 -4.8287 4.8237 +v -2.3683 -3.1062 4.8681 +v -7.5422 -1.0493 -2.4313 +v 0.0000 -1.7240 6.6014 +v -1.8266 -4.3995 4.3990 +v -1.9296 -4.4118 4.4971 +v -0.5974 -2.0137 5.8665 +v -1.4056 -1.7142 5.2411 +v -0.6624 -1.8193 5.8638 +v -2.3423 0.5722 4.2943 +v -3.3273 0.1049 4.1139 +v -1.7262 -0.9192 5.2734 +v -5.1332 7.4856 2.6604 +v -4.5386 6.3199 3.6834 +v -3.9866 5.1095 4.4663 +v -2.1697 -5.4404 4.4559 +v -1.3956 5.0120 5.3160 +v -1.6195 6.5992 4.9211 +v -1.8914 8.2364 4.2750 +v -4.1958 2.2352 3.3751 +v -5.7333 1.4117 2.4317 +v -1.8599 2.3558 3.8432 +v -4.9886 3.0747 3.0839 +v -1.3033 1.4165 4.8311 +v -1.3058 -0.6728 6.4160 +v -6.4652 0.9371 1.6899 +v -5.2587 0.9458 2.9743 +v -4.4323 0.7221 3.5226 +v -3.3007 0.8616 3.8728 +v -2.4302 1.1315 4.0390 +v -1.8207 1.4680 4.2241 +v -0.5632 2.3077 5.5668 +v -6.3381 -0.5293 1.8812 +v -5.5877 3.2081 2.6878 +v -0.2426 -1.4629 7.0715 +v -1.6113 0.3393 4.8954 +v -7.7431 2.3650 -2.0052 +v -1.3911 1.8510 4.4490 +v -1.7858 -0.9783 4.8505 +v -4.6710 2.6645 3.0841 +v -1.3340 -0.2838 6.0970 +v -7.2709 -2.8909 -2.2525 +v -1.8564 2.5852 3.7579 +v -0.9234 0.0731 6.6719 +v -5.0006 -6.1351 1.8925 +v -5.0853 -7.1786 0.7147 +v -7.1593 -0.8118 -0.0720 +v -5.8431 -5.2480 0.9241 +v -6.8473 3.6629 0.7247 +v -2.4129 -8.2589 4.1192 +v -0.1799 -1.6899 6.5733 +v -2.1037 -0.1639 4.5661 +v -6.4076 2.2360 1.5608 +v -3.6701 2.3602 3.6352 +v -3.1772 2.2943 3.7757 +v -2.1961 -4.5983 4.4798 +v -6.2349 -1.9444 1.6635 +v -1.2929 -9.2959 4.0941 +v -3.2107 -8.5333 2.8020 +v -4.0689 -7.9931 1.9251 +v 0.0000 6.5454 5.0273 +v 0.0000 -9.4034 4.2645 +v -2.7240 2.3158 3.7772 +v -2.2885 2.3989 3.6976 +v -1.9983 2.4965 3.6891 +v -6.1300 3.3993 2.0385 +v -2.2885 2.8865 3.7750 +v -2.7240 2.9618 3.8718 +v -3.1772 2.9641 3.8770 +v -3.6701 2.9277 3.7243 +v -4.0184 2.8574 3.4830 +v -7.5558 4.1068 -0.9919 +v -4.0184 2.4837 3.4409 +v 0.0000 -2.5219 5.9323 +v -1.7762 -2.6839 5.2131 +v -1.2222 -1.1824 5.9525 +v -0.7315 -2.5367 5.8153 +v 0.0000 3.2710 5.2360 +v -4.1353 -6.9966 2.6720 +v -3.3118 -7.6608 3.3830 +v -1.3137 -8.6400 4.7025 +v -5.9405 -6.2236 -0.6315 +v -1.9983 2.7438 3.7440 +v -0.9014 1.2370 5.7543 +v 0.0000 -8.7652 4.8914 +v -2.3090 -8.9742 3.6091 +v -6.9542 -2.4398 -0.1312 +v -1.0988 -4.4588 5.1207 +v -1.1811 -4.5800 5.1896 +v -1.2558 -4.7879 5.2371 +v -1.3251 -5.1065 5.2050 +v -1.5464 -5.8194 4.7579 +v -1.9538 -4.1839 4.4317 +v -2.1178 -4.1371 4.5551 +v -2.2853 -4.0512 4.5824 +v -2.8502 -3.6657 4.4850 +v -5.2785 -2.2389 2.8612 +v -0.9467 1.9076 5.1968 +v -1.3142 3.1049 4.2314 +v -1.7800 2.8600 3.8816 +v -1.8451 -4.0989 4.2473 +v -5.4362 -4.0305 2.1099 +v -0.7664 3.1821 4.8615 +v -1.9386 -6.6144 4.5211 +v 0.0000 1.0594 6.7746 +v -0.5166 1.5836 6.1484 +v 0.0000 1.7284 6.3167 +v -1.2468 0.2303 5.6810 +v 0.0000 -7.9422 5.1812 +v 0.0000 -6.9915 5.1535 +v -0.9978 -6.9309 4.9796 +v -3.2888 -5.3825 3.7958 +v -2.3116 -1.5662 4.5901 +v -2.6803 -6.1116 4.0962 +v -3.8329 -1.5373 4.1377 +v -2.9619 -2.2742 4.4409 +v -4.3869 -2.6833 3.6439 +v -1.2173 -7.8345 4.9693 +v -1.5424 -0.1368 5.2010 +v -3.8784 -6.0418 3.3111 +v -3.0840 -6.8098 3.8142 +v -3.7473 -4.5035 3.7265 +v -6.0941 -3.2060 1.4735 +v -4.5890 -4.7287 2.9832 +v -6.5832 -3.9413 0.0703 +v -3.4926 -3.1958 4.1302 +v -1.2555 0.8023 5.3076 +v -1.1261 -0.9336 6.5388 +v -1.4431 -1.1428 5.9051 +v -0.9230 -0.5290 7.0034 +v -1.7554 3.5291 4.3277 +v -2.6326 3.7138 4.3646 +v -3.3881 3.7220 4.3090 +v -4.0758 3.6754 4.0761 +v -4.6229 3.4747 3.6463 +v -5.1718 2.5358 2.6709 +v -7.2973 0.7632 -0.0488 +v -4.7068 1.6510 3.1095 +v -4.0717 1.4768 3.4769 +v -3.2698 1.4707 3.7319 +v -2.5276 1.6173 3.8654 +v -1.9709 1.8585 3.9618 +v -1.5795 2.0979 4.0850 +v -7.6642 0.6731 -2.4359 +v -1.3970 -1.3401 5.6304 +v -0.8848 0.6587 6.2332 +v -0.7671 -0.9680 7.0779 +v -0.4602 -1.3341 6.7874 +v -0.7486 -1.0680 6.7983 +v -1.2364 -1.5856 5.4805 +v -0.3873 -1.4100 6.9577 +v -0.3199 -1.6079 6.5087 +v -1.6396 2.5563 3.8637 +v -1.2556 2.4671 4.2038 +v -1.0314 2.3827 4.6158 +v -4.2531 2.7723 3.3153 +v -4.5300 2.9100 3.3397 +v 0.4639 0.9554 6.6336 +v 4.2531 2.5776 3.2797 +v 0.4161 -1.4664 6.4477 +v 7.0880 5.4348 0.0996 +v 2.6286 2.0359 3.8481 +v 3.1984 1.9858 3.7970 +v 3.7752 2.0394 3.6462 +v 4.4658 2.4230 3.1552 +v 2.1643 2.1899 3.8518 +v 3.2082 3.2239 4.1158 +v 2.6738 3.2053 4.0922 +v 3.7452 3.1653 3.9724 +v 4.1610 3.0591 3.7196 +v 5.0620 1.9344 2.7761 +v 2.2667 -7.4258 4.3898 +v 4.4459 2.6640 3.1734 +v 7.2145 2.2630 0.0732 +v 5.7998 2.3495 2.2041 +v 2.8449 -0.7209 4.4331 +v 0.7115 -3.3294 5.8770 +v 0.6060 -3.9246 5.4449 +v 1.4316 -3.5010 5.4962 +v 1.9149 -3.8031 5.0289 +v 1.1310 -3.9739 5.1896 +v 1.5635 -4.0828 4.8423 +v 2.6501 -5.0036 4.1885 +v 0.4270 -1.0941 7.3605 +v 0.4964 -0.4757 7.4404 +v 5.2533 3.8816 3.3632 +v 1.7187 0.9746 4.5584 +v 1.6086 -0.9425 5.8142 +v 1.6513 -0.6109 5.5813 +v 4.7655 -0.7016 3.5346 +v 0.4783 0.2958 7.1010 +v 3.7350 4.5082 4.5505 +v 4.5886 4.3020 4.0485 +v 6.2793 6.6154 1.4259 +v 1.2209 4.1422 5.1060 +v 2.1935 3.1003 4.0006 +v 3.1026 -4.3530 4.0959 +v 6.7197 -4.7886 -1.7454 +v 1.1938 -1.3068 5.7377 +v 0.7298 -1.5937 5.8332 +v 2.4562 -4.3426 4.2839 +v 2.2048 -4.3045 4.1625 +v 4.9859 4.8025 3.7520 +v 1.5923 -1.2577 5.4569 +v 2.6445 4.5247 4.9216 +v 2.7603 5.1010 5.0160 +v 3.5240 8.0060 3.7292 +v 5.5998 5.7155 2.7243 +v 3.0639 6.5661 4.5300 +v 5.7210 4.2546 2.8309 +v 6.3744 4.7856 1.5917 +v 0.6727 -3.6880 5.7378 +v 1.2626 -3.7877 5.4178 +v 1.7326 -3.9528 5.0006 +v 1.0436 -1.4650 5.6625 +v 2.3212 -4.3291 4.2582 +v 2.0568 -4.4777 4.5209 +v 2.1531 -4.2763 4.0381 +v 0.9469 -1.0352 6.5123 +v 1.4691 -4.0364 4.6049 +v 1.0243 -3.9899 4.9267 +v 0.5334 -3.9932 5.1382 +v 0.7697 -6.0954 4.9859 +v 0.6996 -5.2919 5.4483 +v 0.6697 -4.9498 5.5096 +v 0.6309 -4.6951 5.4494 +v 0.5832 -4.5180 5.3399 +v 1.5372 -4.4232 4.7455 +v 1.6156 -4.4759 4.8136 +v 1.7291 -4.6187 4.8545 +v 1.8386 -4.8287 4.8237 +v 2.3683 -3.1062 4.8681 +v 7.5422 -1.0493 -2.4313 +v 1.8266 -4.3995 4.3990 +v 1.9296 -4.4118 4.4971 +v 0.5974 -2.0137 5.8665 +v 1.4056 -1.7142 5.2411 +v 0.6624 -1.8193 5.8638 +v 2.3423 0.5722 4.2943 +v 3.3273 0.1049 4.1139 +v 1.7262 -0.9192 5.2734 +v 5.1332 7.4856 2.6604 +v 4.5386 6.3199 3.6834 +v 3.9866 5.1095 4.4663 +v 2.1697 -5.4404 4.4559 +v 1.3956 5.0120 5.3160 +v 1.6195 6.5992 4.9211 +v 1.8914 8.2364 4.2750 +v 4.1958 2.2352 3.3751 +v 5.7333 1.4117 2.4317 +v 1.8599 2.3558 3.8432 +v 4.9886 3.0747 3.0839 +v 1.3033 1.4165 4.8311 +v 1.3058 -0.6728 6.4160 +v 6.4652 0.9371 1.6899 +v 5.2587 0.9458 2.9743 +v 4.4323 0.7221 3.5226 +v 3.3007 0.8616 3.8728 +v 2.4302 1.1315 4.0390 +v 1.8207 1.4680 4.2241 +v 0.5632 2.3077 5.5668 +v 6.3381 -0.5293 1.8812 +v 5.5877 3.2081 2.6878 +v 0.2426 -1.4629 7.0715 +v 1.6113 0.3393 4.8954 +v 7.7431 2.3650 -2.0052 +v 1.3911 1.8510 4.4490 +v 1.7858 -0.9783 4.8505 +v 4.6710 2.6645 3.0841 +v 1.3340 -0.2838 6.0970 +v 7.2709 -2.8909 -2.2525 +v 1.8564 2.5852 3.7579 +v 0.9234 0.0731 6.6719 +v 5.0006 -6.1351 1.8925 +v 5.0853 -7.1786 0.7147 +v 7.1593 -0.8118 -0.0720 +v 5.8431 -5.2480 0.9241 +v 6.8473 3.6629 0.7247 +v 2.4129 -8.2589 4.1192 +v 0.1799 -1.6899 6.5733 +v 2.1037 -0.1639 4.5661 +v 6.4076 2.2360 1.5608 +v 3.6701 2.3602 3.6352 +v 3.1772 2.2943 3.7757 +v 2.1961 -4.5983 4.4798 +v 6.2349 -1.9444 1.6635 +v 1.2929 -9.2959 4.0941 +v 3.2107 -8.5333 2.8020 +v 4.0689 -7.9931 1.9251 +v 2.7240 2.3158 3.7772 +v 2.2885 2.3989 3.6976 +v 1.9983 2.4965 3.6891 +v 6.1300 3.3993 2.0385 +v 2.2885 2.8865 3.7750 +v 2.7240 2.9618 3.8718 +v 3.1772 2.9641 3.8770 +v 3.6701 2.9277 3.7243 +v 4.0184 2.8574 3.4830 +v 7.5558 4.1068 -0.9919 +v 4.0184 2.4837 3.4409 +v 1.7762 -2.6839 5.2131 +v 1.2222 -1.1824 5.9525 +v 0.7315 -2.5367 5.8153 +v 4.1353 -6.9966 2.6720 +v 3.3118 -7.6608 3.3830 +v 1.3137 -8.6400 4.7025 +v 5.9405 -6.2236 -0.6315 +v 1.9983 2.7438 3.7440 +v 0.9014 1.2370 5.7543 +v 2.3090 -8.9742 3.6091 +v 6.9542 -2.4398 -0.1312 +v 1.0988 -4.4588 5.1207 +v 1.1811 -4.5800 5.1896 +v 1.2558 -4.7879 5.2371 +v 1.3251 -5.1065 5.2050 +v 1.5464 -5.8194 4.7579 +v 1.9538 -4.1839 4.4317 +v 2.1178 -4.1371 4.5551 +v 2.2853 -4.0512 4.5824 +v 2.8502 -3.6657 4.4850 +v 5.2785 -2.2389 2.8612 +v 0.9467 1.9076 5.1968 +v 1.3142 3.1049 4.2314 +v 1.7800 2.8600 3.8816 +v 1.8451 -4.0989 4.2473 +v 5.4362 -4.0305 2.1099 +v 0.7664 3.1821 4.8615 +v 1.9386 -6.6144 4.5211 +v 0.5166 1.5836 6.1484 +v 1.2468 0.2303 5.6810 +v 0.9978 -6.9309 4.9796 +v 3.2888 -5.3825 3.7958 +v 2.3116 -1.5662 4.5901 +v 2.6803 -6.1116 4.0962 +v 3.8329 -1.5373 4.1377 +v 2.9619 -2.2742 4.4409 +v 4.3869 -2.6833 3.6439 +v 1.2173 -7.8345 4.9693 +v 1.5424 -0.1368 5.2010 +v 3.8784 -6.0418 3.3111 +v 3.0840 -6.8098 3.8142 +v 3.7473 -4.5035 3.7265 +v 6.0941 -3.2060 1.4735 +v 4.5890 -4.7287 2.9832 +v 6.5832 -3.9413 0.0703 +v 3.4926 -3.1958 4.1302 +v 1.2555 0.8023 5.3076 +v 1.1261 -0.9336 6.5388 +v 1.4431 -1.1428 5.9051 +v 0.9230 -0.5290 7.0034 +v 1.7554 3.5291 4.3277 +v 2.6326 3.7138 4.3646 +v 3.3881 3.7220 4.3090 +v 4.0758 3.6754 4.0761 +v 4.6229 3.4747 3.6463 +v 5.1718 2.5358 2.6709 +v 7.2973 0.7632 -0.0488 +v 4.7068 1.6510 3.1095 +v 4.0717 1.4768 3.4769 +v 3.2698 1.4707 3.7319 +v 2.5276 1.6173 3.8654 +v 1.9709 1.8585 3.9618 +v 1.5795 2.0979 4.0850 +v 7.6642 0.6731 -2.4359 +v 1.3970 -1.3401 5.6304 +v 0.8848 0.6587 6.2332 +v 0.7671 -0.9680 7.0779 +v 0.4602 -1.3341 6.7874 +v 0.7486 -1.0680 6.7983 +v 1.2364 -1.5856 5.4805 +v 0.3873 -1.4100 6.9577 +v 0.3199 -1.6079 6.5087 +v 1.6396 2.5563 3.8637 +v 1.2556 2.4671 4.2038 +v 1.0314 2.3827 4.6158 +v 4.2531 2.7723 3.3153 +v 4.5300 2.9100 3.3397 +f 174 156 134 +f 247 34 8 +f 383 399 363 +f 264 467 250 +f 309 416 325 +f 79 96 192 +f 357 390 265 +f 128 35 163 +f 369 265 390 +f 140 163 35 +f 268 1 303 +f 38 73 1 +f 12 303 1 +f 12 1 73 +f 350 452 351 +f 121 122 232 +f 453 351 452 +f 233 232 122 +f 268 303 270 +f 38 40 73 +f 304 270 303 +f 74 73 40 +f 358 344 351 +f 129 122 115 +f 278 351 344 +f 48 115 122 +f 351 453 358 +f 122 129 233 +f 454 358 453 +f 234 233 129 +f 300 334 298 +f 70 68 105 +f 333 298 334 +f 104 105 68 +f 176 153 397 +f 176 172 153 +f 378 397 153 +f 149 153 172 +f 382 385 383 +f 155 156 158 +f 399 383 385 +f 174 158 156 +f 281 348 331 +f 51 102 119 +f 349 331 348 +f 120 119 102 +f 270 304 271 +f 40 41 74 +f 305 271 304 +f 75 74 41 +f 10 337 152 +f 10 152 108 +f 338 152 337 +f 109 108 152 +f 345 279 361 +f 116 132 49 +f 280 361 279 +f 50 49 132 +f 263 432 419 +f 33 195 212 +f 425 419 432 +f 205 212 195 +f 305 409 271 +f 75 41 185 +f 410 271 409 +f 186 185 41 +f 273 311 408 +f 43 184 81 +f 416 408 311 +f 192 81 184 +f 323 271 411 +f 93 187 41 +f 410 411 271 +f 186 41 187 +f 348 450 349 +f 119 120 230 +f 451 349 450 +f 231 230 120 +f 435 433 431 +f 215 211 213 +f 423 431 433 +f 203 213 211 +f 314 315 19 +f 84 19 85 +f 18 19 315 +f 18 85 19 +f 308 376 307 +f 78 77 147 +f 292 307 376 +f 62 147 77 +f 260 388 261 +f 30 31 161 +f 389 261 388 +f 162 161 31 +f 287 415 385 +f 57 158 191 +f 399 385 415 +f 174 191 158 +f 419 425 407 +f 195 183 205 +f 336 407 425 +f 107 205 183 +f 368 417 365 +f 139 136 193 +f 435 365 417 +f 215 193 136 +f 392 424 328 +f 166 99 204 +f 359 328 424 +f 130 204 99 +f 299 302 285 +f 69 55 72 +f 252 285 302 +f 22 72 55 +f 5 276 6 +f 5 6 46 +f 282 6 276 +f 52 46 6 +f 255 374 254 +f 25 24 145 +f 375 254 374 +f 146 145 24 +f 321 322 308 +f 91 78 92 +f 376 308 322 +f 147 92 78 +f 281 426 412 +f 51 188 206 +f 428 412 426 +f 208 206 188 +f 422 314 201 +f 202 201 84 +f 19 201 314 +f 19 84 201 +f 336 322 407 +f 107 183 92 +f 406 407 322 +f 182 92 183 +f 406 322 405 +f 182 181 92 +f 321 405 322 +f 91 92 181 +f 18 315 17 +f 18 17 85 +f 316 17 315 +f 86 85 17 +f 426 267 427 +f 206 207 37 +f 424 427 267 +f 204 37 207 +f 370 397 401 +f 141 177 172 +f 378 401 397 +f 149 172 177 +f 392 270 323 +f 166 93 40 +f 271 323 270 +f 41 40 93 +f 418 466 414 +f 194 190 246 +f 465 414 466 +f 245 246 190 +f 258 259 387 +f 28 160 29 +f 386 387 259 +f 159 29 160 +f 261 389 468 +f 31 248 162 +f 467 468 389 +f 247 162 248 +f 249 457 420 +f 4 197 237 +f 400 420 457 +f 175 237 197 +f 334 299 333 +f 105 104 69 +f 285 333 299 +f 55 69 104 +f 286 9 418 +f 56 194 9 +f 169 418 9 +f 169 9 194 +f 341 262 347 +f 112 118 32 +f 449 347 262 +f 229 32 118 +f 286 418 442 +f 56 222 194 +f 414 442 418 +f 190 194 222 +f 328 461 327 +f 99 98 241 +f 329 327 461 +f 100 241 98 +f 278 356 330 +f 48 101 127 +f 372 330 356 +f 143 127 101 +f 310 393 439 +f 80 219 167 +f 440 439 393 +f 220 167 219 +f 382 383 257 +f 155 27 156 +f 342 257 383 +f 113 156 27 +f 361 280 421 +f 132 199 50 +f 430 421 280 +f 210 50 199 +f 366 365 380 +f 137 151 136 +f 395 380 365 +f 170 136 151 +f 356 278 438 +f 127 218 48 +f 344 438 278 +f 115 48 218 +f 444 445 283 +f 224 53 225 +f 284 283 445 +f 54 225 53 +f 282 276 364 +f 52 135 46 +f 441 364 276 +f 221 46 135 +f 432 263 396 +f 212 171 33 +f 370 396 263 +f 141 33 171 +f 338 300 339 +f 109 110 70 +f 298 339 300 +f 68 70 110 +f 336 274 322 +f 107 92 44 +f 376 322 274 +f 147 44 92 +f 349 451 350 +f 120 121 231 +f 452 350 451 +f 232 231 121 +f 468 360 343 +f 248 114 131 +f 447 343 360 +f 227 131 114 +f 283 284 335 +f 53 106 54 +f 294 335 284 +f 64 54 106 +f 251 459 463 +f 21 243 239 +f 462 463 459 +f 242 239 243 +f 277 354 301 +f 47 71 125 +f 384 301 354 +f 157 125 71 +f 326 293 325 +f 97 96 63 +f 309 325 293 +f 79 63 96 +f 284 277 294 +f 54 64 47 +f 301 294 277 +f 71 47 64 +f 448 265 346 +f 228 117 35 +f 373 346 265 +f 144 35 117 +f 353 346 347 +f 124 118 117 +f 341 347 346 +f 112 117 118 +f 2 20 275 +f 2 45 20 +f 355 275 20 +f 126 20 45 +f 249 282 457 +f 4 237 52 +f 364 457 282 +f 135 52 237 +f 426 427 428 +f 206 208 207 +f 437 428 427 +f 217 207 208 +f 381 382 253 +f 154 23 155 +f 257 253 382 +f 27 155 23 +f 392 394 270 +f 166 40 168 +f 268 270 394 +f 38 168 40 +f 200 429 201 +f 200 201 209 +f 422 201 429 +f 202 209 201 +f 331 330 267 +f 102 37 101 +f 372 267 330 +f 143 101 37 +f 423 433 274 +f 203 44 213 +f 288 274 433 +f 58 213 44 +f 291 251 329 +f 61 100 21 +f 463 329 251 +f 243 21 100 +f 259 287 386 +f 29 159 57 +f 385 386 287 +f 158 57 159 +f 343 447 354 +f 114 125 227 +f 266 354 447 +f 36 227 125 +f 258 387 260 +f 28 30 160 +f 388 260 387 +f 161 160 30 +f 431 423 432 +f 211 212 203 +f 425 432 423 +f 205 203 212 +f 446 343 277 +f 226 47 114 +f 354 277 343 +f 125 114 47 +f 425 423 336 +f 205 107 203 +f 274 336 423 +f 44 203 107 +f 307 293 308 +f 77 78 63 +f 326 308 293 +f 97 63 78 +f 367 448 353 +f 138 124 228 +f 346 353 448 +f 117 228 124 +f 303 269 304 +f 73 74 39 +f 272 304 269 +f 42 39 74 +f 372 359 267 +f 143 37 130 +f 424 267 359 +f 204 130 37 +f 328 295 461 +f 99 241 65 +f 456 461 295 +f 236 65 241 +f 295 332 279 +f 65 49 103 +f 280 279 332 +f 50 103 49 +f 304 272 305 +f 74 75 42 +f 273 305 272 +f 43 42 75 +f 428 437 435 +f 208 215 217 +f 433 435 437 +f 213 217 215 +f 305 273 409 +f 75 185 43 +f 408 409 273 +f 184 43 185 +f 395 431 396 +f 170 171 211 +f 432 396 431 +f 212 211 171 +f 396 370 379 +f 171 150 141 +f 401 379 370 +f 177 141 150 +f 297 335 300 +f 67 70 106 +f 334 300 335 +f 105 106 70 +f 418 169 352 +f 194 123 169 +f 7 352 169 +f 7 169 123 +f 281 412 353 +f 51 124 188 +f 377 353 412 +f 148 188 124 +f 320 321 326 +f 90 97 91 +f 308 326 321 +f 78 91 97 +f 286 296 337 +f 56 108 66 +f 297 337 296 +f 67 66 108 +f 405 321 404 +f 181 180 91 +f 320 404 321 +f 90 91 180 +f 331 349 330 +f 102 101 120 +f 350 330 349 +f 121 120 101 +f 335 294 334 +f 106 105 64 +f 299 334 294 +f 69 64 105 +f 324 455 367 +f 94 138 235 +f 448 367 455 +f 228 235 138 +f 17 316 16 +f 17 16 86 +f 317 16 316 +f 87 86 16 +f 430 280 359 +f 210 130 50 +f 332 359 280 +f 103 50 130 +f 16 317 15 +f 16 15 87 +f 318 15 317 +f 88 87 15 +f 9 286 10 +f 9 10 56 +f 337 10 286 +f 108 56 10 +f 330 350 278 +f 101 48 121 +f 351 278 350 +f 122 121 48 +f 253 254 381 +f 23 154 24 +f 375 381 254 +f 146 24 154 +f 403 404 319 +f 179 89 180 +f 320 319 404 +f 90 180 89 +f 352 7 420 +f 123 197 7 +f 198 420 7 +f 198 7 197 +f 325 319 326 +f 96 97 89 +f 320 326 319 +f 90 89 97 +f 398 368 366 +f 173 137 139 +f 365 366 368 +f 136 139 137 +f 289 436 398 +f 59 173 216 +f 368 398 436 +f 139 216 173 +f 439 440 345 +f 219 116 220 +f 279 345 440 +f 49 220 116 +f 272 312 273 +f 42 43 82 +f 311 273 312 +f 81 82 43 +f 6 282 196 +f 6 196 52 +f 249 196 282 +f 4 52 196 +f 274 288 376 +f 44 147 58 +f 292 376 288 +f 62 58 147 +f 397 429 176 +f 172 176 209 +f 200 176 429 +f 200 209 176 +f 269 313 272 +f 39 42 83 +f 312 272 313 +f 82 83 42 +f 445 446 284 +f 225 54 226 +f 277 284 446 +f 47 226 54 +f 255 340 374 +f 25 145 111 +f 391 374 340 +f 164 111 145 +f 296 283 297 +f 66 67 53 +f 335 297 283 +f 106 53 67 +f 347 449 348 +f 118 119 229 +f 450 348 449 +f 230 229 119 +f 455 357 448 +f 235 228 128 +f 265 448 357 +f 35 128 228 +f 337 297 338 +f 108 109 67 +f 300 338 297 +f 70 67 109 +f 152 338 11 +f 152 11 109 +f 339 11 338 +f 110 109 11 +f 279 440 295 +f 49 65 220 +f 456 295 440 +f 236 220 65 +f 408 416 293 +f 184 63 192 +f 309 293 416 +f 79 192 63 +f 359 372 430 +f 130 210 143 +f 356 430 372 +f 127 143 210 +f 346 373 341 +f 117 112 144 +f 266 341 373 +f 36 144 112 +f 389 391 467 +f 162 247 164 +f 250 467 391 +f 8 164 247 +f 353 347 281 +f 124 51 118 +f 348 281 347 +f 119 118 51 +f 296 443 283 +f 66 53 223 +f 444 283 443 +f 224 223 53 +f 20 95 355 +f 20 126 95 +f 371 355 95 +f 142 95 126 +f 296 286 443 +f 66 223 56 +f 442 443 286 +f 222 56 223 +f 420 198 249 +f 197 4 198 +f 196 249 198 +f 196 198 4 +f 360 264 256 +f 131 26 34 +f 250 256 264 +f 8 34 26 +f 276 275 441 +f 46 221 45 +f 458 441 275 +f 238 45 221 +f 301 384 302 +f 71 72 157 +f 369 302 384 +f 140 157 72 +f 418 352 466 +f 194 246 123 +f 413 466 352 +f 189 123 246 +f 467 264 468 +f 247 248 34 +f 360 468 264 +f 131 34 248 +f 390 252 369 +f 163 140 22 +f 302 369 252 +f 72 22 140 +f 375 387 381 +f 146 154 160 +f 386 381 387 +f 159 160 154 +f 380 395 379 +f 151 150 170 +f 396 379 395 +f 171 170 150 +f 352 420 413 +f 123 189 197 +f 400 413 420 +f 175 197 189 +f 427 323 437 +f 207 217 93 +f 411 437 323 +f 187 93 217 +f 388 374 389 +f 161 162 145 +f 391 389 374 +f 164 145 162 +f 394 327 165 +f 168 165 98 +f 3 165 327 +f 3 98 165 +f 355 371 462 +f 126 242 142 +f 463 462 371 +f 243 142 242 +f 1 268 165 +f 1 165 38 +f 394 165 268 +f 168 38 165 +f 12 13 303 +f 12 73 13 +f 269 303 13 +f 39 13 73 +f 387 375 388 +f 160 161 146 +f 374 388 375 +f 145 146 161 +f 13 14 269 +f 13 39 14 +f 313 269 14 +f 83 14 39 +f 294 301 299 +f 64 69 71 +f 302 299 301 +f 72 71 69 +f 341 266 262 +f 112 32 36 +f 447 262 266 +f 227 36 32 +f 381 386 382 +f 154 155 159 +f 385 382 386 +f 158 159 155 +f 281 331 426 +f 51 206 102 +f 267 426 331 +f 37 102 206 +f 424 392 427 +f 204 207 166 +f 323 427 392 +f 93 166 207 +f 430 356 421 +f 210 199 127 +f 438 421 356 +f 218 127 199 +f 392 328 394 +f 166 168 99 +f 327 394 328 +f 98 99 168 +f 458 439 441 +f 238 221 219 +f 345 441 439 +f 116 219 221 +f 383 363 342 +f 156 113 134 +f 464 342 363 +f 244 134 113 +f 458 462 460 +f 238 240 242 +f 459 460 462 +f 239 242 240 +f 435 431 365 +f 215 136 211 +f 395 365 431 +f 170 211 136 +f 415 464 399 +f 191 174 244 +f 363 399 464 +f 134 244 174 +f 263 429 370 +f 33 141 209 +f 397 370 429 +f 172 209 141 +f 458 275 462 +f 238 242 45 +f 355 462 275 +f 126 45 242 +f 317 404 318 +f 87 88 180 +f 403 318 404 +f 179 180 88 +f 316 405 317 +f 86 87 181 +f 404 317 405 +f 180 181 87 +f 315 406 316 +f 85 86 182 +f 405 316 406 +f 181 182 86 +f 314 407 315 +f 84 85 183 +f 406 315 407 +f 182 183 85 +f 419 407 422 +f 195 202 183 +f 314 422 407 +f 84 183 202 +f 367 402 324 +f 138 94 178 +f 362 324 402 +f 133 178 94 +f 409 408 307 +f 185 77 184 +f 293 307 408 +f 63 184 77 +f 409 307 410 +f 185 186 77 +f 292 410 307 +f 62 77 186 +f 411 410 288 +f 187 58 186 +f 292 288 410 +f 62 186 58 +f 437 411 433 +f 217 213 187 +f 288 433 411 +f 58 187 213 +f 435 417 428 +f 215 208 193 +f 412 428 417 +f 188 193 208 +f 265 369 373 +f 35 144 140 +f 384 373 369 +f 157 140 144 +f 458 460 439 +f 238 219 240 +f 310 439 460 +f 80 240 219 +f 353 377 367 +f 124 138 148 +f 402 367 377 +f 178 148 138 +f 5 2 276 +f 5 46 2 +f 275 276 2 +f 45 2 46 +f 429 263 422 +f 209 202 33 +f 419 422 263 +f 195 33 202 +f 328 359 295 +f 99 65 130 +f 332 295 359 +f 103 130 65 +f 368 436 417 +f 139 193 216 +f 434 417 436 +f 214 216 193 +f 456 440 290 +f 236 60 220 +f 393 290 440 +f 167 220 60 +f 329 463 327 +f 100 98 243 +f 371 327 463 +f 142 243 98 +f 327 371 3 +f 98 3 142 +f 95 3 371 +f 95 142 3 +f 461 456 306 +f 241 76 236 +f 290 306 456 +f 60 236 76 +f 449 340 450 +f 229 230 111 +f 255 450 340 +f 25 111 230 +f 262 447 256 +f 32 26 227 +f 360 256 447 +f 131 227 26 +f 450 255 451 +f 230 231 25 +f 254 451 255 +f 24 25 231 +f 451 254 452 +f 231 232 24 +f 253 452 254 +f 23 24 232 +f 452 253 453 +f 232 233 23 +f 257 453 253 +f 27 23 233 +f 257 342 453 +f 27 233 113 +f 454 453 342 +f 234 113 233 +f 414 465 415 +f 190 191 245 +f 464 415 465 +f 244 245 191 +f 442 414 287 +f 222 57 190 +f 415 287 414 +f 191 190 57 +f 442 287 443 +f 222 223 57 +f 259 443 287 +f 29 57 223 +f 443 259 444 +f 223 224 29 +f 258 444 259 +f 28 29 224 +f 445 444 260 +f 225 30 224 +f 258 260 444 +f 28 224 30 +f 260 261 445 +f 30 225 31 +f 446 445 261 +f 226 31 225 +f 261 468 446 +f 31 226 248 +f 343 446 468 +f 114 248 226 +f 251 310 459 +f 21 239 80 +f 460 459 310 +f 240 80 239 +f 291 306 393 +f 61 167 76 +f 290 393 306 +f 60 76 167 +f 461 306 329 +f 241 100 76 +f 291 329 306 +f 61 76 100 +f 377 434 402 +f 148 178 214 +f 436 402 434 +f 216 214 178 +f 251 291 310 +f 21 80 61 +f 393 310 291 +f 167 61 80 +f 412 417 377 +f 188 148 193 +f 434 377 417 +f 214 193 148 +f 342 464 454 +f 113 234 244 +f 465 454 464 +f 245 244 234 +f 454 465 358 +f 234 129 245 +f 466 358 465 +f 246 245 129 +f 413 344 466 +f 189 246 115 +f 358 466 344 +f 129 115 246 +f 438 344 400 +f 218 175 115 +f 413 400 344 +f 189 115 175 +f 364 441 361 +f 135 132 221 +f 345 361 441 +f 116 221 132 +f 457 421 400 +f 237 175 199 +f 438 400 421 +f 218 199 175 +f 457 364 421 +f 237 199 135 +f 361 421 364 +f 132 135 199 +f 362 402 289 +f 133 59 178 +f 436 289 402 +f 216 178 59 +f 354 266 384 +f 125 157 36 +f 373 384 266 +f 144 36 157 +f 256 250 340 +f 26 111 8 +f 391 340 250 +f 164 8 111 +f 262 256 449 +f 32 229 26 +f 340 449 256 +f 111 26 229 +f 15 318 14 +f 15 14 88 +f 313 14 318 +f 83 88 14 +f 318 403 313 +f 88 83 179 +f 312 313 403 +f 82 179 83 +f 403 319 312 +f 179 82 89 +f 311 312 319 +f 81 89 82 +f 319 325 311 +f 89 81 96 +f 416 311 325 +f 192 96 81 diff --git a/core/gemini.py b/core/gemini.py new file mode 100644 index 0000000..45c7bf1 --- /dev/null +++ b/core/gemini.py @@ -0,0 +1,439 @@ +""" +One place where the assistant's one-shot Gemini calls are made. + +WHY THIS EXISTS + The live conversation runs on the Live API and is not what this file is + about. Everything else — reading a screenshot, parsing a flight page, + turning a request into a shell command, working out which WhatsApp button + answers a call — was a separate `genai.Client(...)` built at the point of + use, with the model name written inline. Sixteen files did it, twenty-six + times, and not one of them set a timeout. + + That is not tidiness, it is three real faults: + + NO TIMEOUT. The SDK waits forever by default. `gemini-flash-latest` spent + an afternoon returning 504 DEADLINE_EXCEEDED, and every one of those calls + became an unbounded hang — measured at ten seconds of silence while a phone + rang, and worse elsewhere, because nothing was there to give up. + + NO FALLBACK. One hardcoded alias meant that when that alias was unwell, + the feature was simply gone. A ladder costs nothing when the first rung + works and saves the feature when it does not. + + NO SINGLE PLACE TO CHANGE. A new model release meant editing sixteen files + and hoping none were missed. + +THE LIVE MODEL DOES THIS WORK, AND IT LEADS THE LADDER + Not the user's conversation — a separate, throwaway session per call, so + nothing a plugin asks is ever heard by the person at the microphone. + + It leads because of quota. This is a voice assistant; the Live API is the + dependency it already has, and it draws on a different pool from the text + models. On the free tier it is the TEXT pool that runs out, and when it does + every side call fails and the feature behind it dies with it. Putting Live + first means ordinary use stops spending the pool that runs dry. + + The reply arrives through output_transcription, because these models refuse + response_modalities=["TEXT"] with a 1007 — they only speak. That sounds + fatal for structured output and is not: the transcription is the model's own + text of what it said, and it returned "Mum ❤ click here for contact info", + indented Python inside markdown fences, and src/utils/helpers_v2.py + character for character. + + It cannot carry grounding metadata, so grounded web search stays on REST. + +THE LADDER, MEASURED + Live, one throwaway session: + connect 0.24s + short structured JSON 1.7 - 2.8s + 2300 characters of code 3.39s, not truncated + three concurrent sessions all fine, 4.77s wall clock + REST text models, same prompt, same afternoon: + gemini-2.5-flash-lite 0.76s ...then 429, quota exhausted + gemini-2.5-flash 0.80s ...then 429 + gemini-flash-lite-latest 2.58s + gemini-flash-latest 504, every time + So REST is two to four times quicker while it lasts, and the whole point of + the ladder is that it does not last. Pinned REST names sit behind Live; + rolling `-latest` aliases sit behind those, because they were the ones + having a bad day. + + Where the answer depends only on stable input, cache it and neither pool is + touched twice: `plugins/_whatsapp_core.py` is the worked example — one + request per WhatsApp language for the lifetime of the install. +""" +from __future__ import annotations + +import asyncio +import json +import sys +import time +import threading +from pathlib import Path + +if getattr(sys, "frozen", False): + _BASE = Path(sys.executable).parent +else: + _BASE = Path(__file__).resolve().parent.parent + +_KEY_FILE = _BASE / "config" / "api_keys.json" + +# Ladders, tried left to right. Change a model HERE and the whole app follows. +FAST = "fast" # short classification, extraction, one-line decisions +SMART = "smart" # reasoning, generation, long documents, images +SEARCH = "search" # grounded search — REST only, see below + +# A rung that means "ask the Live model instead", through a short throwaway +# session rather than the REST text API. +# +# WHY IT LEADS +# This is a voice assistant: the Live API is the dependency it already has, +# and it draws on a DIFFERENT quota pool from the text models. On the free +# tier the text pool is the one that runs out — an afternoon of ordinary use +# exhausts it, and when it does, every one of these side calls fails and the +# feature behind it dies. The Live pool is untouched by that. +# +# WHAT IT COSTS, MEASURED +# connect 0.24s +# short structured JSON 1.7 - 2.8s (REST: 0.76s) +# 2300 characters of code 3.39s, not truncated +# three concurrent sessions all fine, 4.77s wall clock +# So it is two to four times slower than REST when REST is available, and +# infinitely faster than REST when REST is out of quota. +# +# THE THING WORTH KNOWING +# These models only speak — response_modalities=["TEXT"] is refused with a +# 1007. The reply comes back through output_transcription, which sounds like +# it would mangle anything structured. It does not: it is the model's own +# text of what it said, and it survived "Mum ❤ click here for contact info", +# indented Python inside markdown fences, and src/utils/helpers_v2.py +# character for character. That is what makes this usable at all. +# +# What it cannot carry is grounding metadata, so grounded web search stays +# on REST — see SEARCH. +LIVE = "live" + +_LADDERS = { + FAST: (LIVE, "gemini-2.5-flash-lite", "gemini-2.5-flash"), + SMART: (LIVE, "gemini-2.5-flash", "gemini-2.5-flash-lite"), + # Grounded search needs response.candidates[...].grounding_metadata, which a + # Live turn does not produce. REST only, and it says so rather than silently + # returning an answer with no sources behind it. + SEARCH: ("gemini-2.5-flash", "gemini-flash-latest", "gemini-2.5-flash-lite"), +} + +# The Live model to use for one-shot calls. main.py owns the real one; this is +# only the fallback for when this module is imported without it (tests). +_LIVE_FALLBACK = "models/gemini-3.1-flash-live-preview" + +# How many one-shot Live sessions may exist at once. +# +# THE USER'S CONVERSATION OUTRANKS EVERY SIDE CALL. +# Nothing here can reach the microphone or the speaker — main.py has exactly one +# `.receive()` and it is bound to its own session, and audio only reaches the +# speaker through that one loop — so a side call cannot answer the user or talk +# over the reply. Verified alongside a live main session: four side calls fired +# while it was connected, each got its own answer, and the main session replied +# correctly both before and after with no errors. +# +# What a side call CAN do is take up a concurrent-session slot. That is the one +# way it could hurt the conversation, so it is capped, and a call that cannot +# get a slot quickly does not queue behind the others — it falls to the REST +# rung, which is what the ladder is for. +# Three, from the measurement: four side calls plus the conversation ran +# together without complaint, so three leaves the conversation a slot in hand +# while still covering any burst this app actually produces — tool calls run one +# after another, and the screen agent's loop is sequential. +_LIVE_SLOTS = threading.BoundedSemaphore(3) +_LIVE_SLOT_WAIT = 3.0 + +_ONE_SHOT_SYSTEM = ( + "You are a data-processing function, not an assistant and not in a " + "conversation. There is no person listening to you. Produce exactly the " + "output the request asks for and nothing else: no greeting, no " + "acknowledgement, no 'Understood', no explanation, no closing remark, no " + "restating of the question. If the request asks for JSON, emit only the " + "JSON. If it asks for code, emit only the code. If it asks for one word, " + "emit that one word. Preserve the exact spelling, punctuation, capitals " + "and whitespace of anything you are asked to copy or return." +) + +# Milliseconds. Not a preference: the API rejects anything under ten seconds +# with "Minimum allowed deadline is 10s", so this is the tightest bound it will +# accept. Callers with a long job (a whole document, a big image) pass more. +DEFAULT_TIMEOUT_MS = 10_000 +MIN_TIMEOUT_MS = 10_000 + +_key_lock = threading.Lock() +_cached_key: str | None = None + +# A rung that answered 429 is out of quota, and on the free tier it will stay +# that way for a while. Retrying it on every single call is a wasted round trip +# in front of every request the assistant makes — measured on this key, the +# lite rung was 429ing continuously, so every call was paying for it before +# reaching the model that could actually answer. Remembering that for a few +# minutes turns the ladder from a cost into a saving. +_COOLDOWN_SECONDS = 300 +_cooldown: dict[str, float] = {} +_cool_lock = threading.Lock() + + +def _cool(model: str) -> None: + with _cool_lock: + _cooldown[model] = time.monotonic() + _COOLDOWN_SECONDS + + +def _cooling(model: str) -> bool: + with _cool_lock: + until = _cooldown.get(model, 0.0) + if until and time.monotonic() < until: + return True + _cooldown.pop(model, None) + return False + + +def api_key(refresh: bool = False) -> str: + """The Gemini key from config/api_keys.json. Cached; never raises.""" + global _cached_key + with _key_lock: + if _cached_key is not None and not refresh: + return _cached_key + try: + data = json.loads(_KEY_FILE.read_text(encoding="utf-8")) + _cached_key = str(data.get("gemini_api_key") or "") + except Exception: + _cached_key = "" + return _cached_key + + +def client(timeout_ms: int = DEFAULT_TIMEOUT_MS, key: str = ""): + """A configured genai.Client with a deadline on it. Raises if there is no + key, because a caller that cannot work without one should say so.""" + from google import genai + from google.genai import types as gtypes + + key = key or api_key() + if not key: + raise RuntimeError("no Gemini API key is configured") + return genai.Client( + api_key=key, + http_options=gtypes.HttpOptions(timeout=max(MIN_TIMEOUT_MS, int(timeout_ms))), + ) + + +class _Reply: + """What a Live turn hands back, shaped like the REST response's `.text` so + every existing call site keeps working unchanged.""" + + __slots__ = ("text",) + + def __init__(self, text: str): + self.text = text + + +def _live_model() -> str: + """Whatever main.py is running, so upgrading the assistant upgrades this.""" + return getattr(sys.modules.get("main"), "LIVE_MODEL", None) or _LIVE_FALLBACK + + +def _to_live_parts(contents) -> list: + """REST `contents` -> Live `parts`. Accepts a bare string, a list of + strings, and the SDK's Part objects (which is how every image is passed + here), because those are the three shapes the call sites actually use.""" + import base64 + + items = contents if isinstance(contents, (list, tuple)) else [contents] + parts = [] + for item in items: + if isinstance(item, str): + parts.append({"text": item}) + continue + blob = getattr(item, "inline_data", None) + if blob is not None: + data = getattr(blob, "data", None) + mime = getattr(blob, "mime_type", None) or "application/octet-stream" + if isinstance(data, bytes): + data = base64.b64encode(data).decode("ascii") + parts.append({"inline_data": {"mime_type": mime, "data": data}}) + continue + txt = getattr(item, "text", None) + if txt: + parts.append({"text": txt}) + continue + if isinstance(item, dict): + parts.append(item) + return parts + + +async def _live_turn(parts: list, system: str, key: str, timeout_s: float) -> str: + from google import genai + from google.genai import types as gtypes + + cl = genai.Client(api_key=key, http_options={"api_version": "v1beta"}) + # Silence the persona, or it answers instead of complying. + # + # These are conversational models and they behave like it: asked "Reply with + # one word: ok" a Live turn came back with "Understood." — it treated the + # instruction as something to acknowledge rather than something to do. The + # REST models do not, because nobody ever taught them to be in a + # conversation. Every call through this module wants a value, not a reply, + # so the session is told what it is before it is told what to do. + kwargs = { + "response_modalities": ["AUDIO"], + "output_audio_transcription": {}, + "system_instruction": _ONE_SHOT_SYSTEM + (f"\n\n{system}" if system else ""), + } + + cm = cl.aio.live.connect(model=_live_model(), + config=gtypes.LiveConnectConfig(**kwargs)) + session = await asyncio.wait_for(cm.__aenter__(), 30) + try: + await session.send_client_content( + turns={"role": "user", "parts": parts}, turn_complete=True) + chunks: list[str] = [] + + async def drain(): + async for resp in session.receive(): + sc = getattr(resp, "server_content", None) + if sc and sc.output_transcription and sc.output_transcription.text: + chunks.append(sc.output_transcription.text) + + await asyncio.wait_for(drain(), timeout=timeout_s) + # The transcription can trail the audio turn by a beat; a short second + # drain stops a reply being cut mid-token. chat_takeover learned this + # the same way and for the same reason. + try: + await asyncio.wait_for(drain(), timeout=1.5) + except asyncio.TimeoutError: + pass + return "".join(chunks).strip() + finally: + try: + await cm.__aexit__(None, None, None) + except Exception: + pass + + +def _live_call(contents, config, timeout_ms: int, key: str): + """One throwaway Live session, run on its own loop in its own thread. + + A dedicated thread rather than asyncio.run() on the caller's: these are + invoked from plugin executor threads, from UI worker threads and from + main.py's own event loop, and asyncio.run() inside a thread that already has + a running loop raises. Its own thread has no loop to collide with, wherever + it was called from. + + Reconnecting costs 0.24s, so nothing is kept alive between calls — no + session lifetime cap to manage, no GoAway to handle, no shared state. + """ + system = "" + if config is not None: + system = getattr(config, "system_instruction", None) or \ + (config.get("system_instruction") if isinstance(config, dict) else "") or "" + + parts = _to_live_parts(contents) + if not parts: + return None + + if not _LIVE_SLOTS.acquire(timeout=_LIVE_SLOT_WAIT): + # Every slot is busy. Do not wait it out: falling to REST costs less + # than holding a session the user's conversation might want. + raise RuntimeError("no free Live slot — leaving them for the conversation") + + box: dict = {} + + def runner(): + try: + box["text"] = asyncio.run( + _live_turn(parts, str(system), key, max(10.0, timeout_ms / 1000.0))) + except BaseException as e: # noqa: BLE001 + box["error"] = e + + try: + th = threading.Thread(target=runner, daemon=True, name="gemini-live-oneshot") + th.start() + th.join(timeout=max(15.0, timeout_ms / 1000.0 + 20.0)) + finally: + _LIVE_SLOTS.release() + if "error" in box: + raise box["error"] + text = box.get("text") + return _Reply(text) if text else None + + +def call(contents, tier: str = FAST, config=None, + timeout_ms: int = DEFAULT_TIMEOUT_MS, key: str = ""): + """Run one generation, walking the ladder until one answers. + + Returns the SDK's own response object, so callers that need more than the + text — grounding metadata, candidates, usage — still get it. Returns None + when every model on the ladder failed; the reason for each is printed, since + a silent None during a session nobody can debug is how the original problem + stayed hidden. + """ + # `tier` is normally FAST or SMART. Anything else is taken to be an explicit + # model name — screen_agent lets the user pick one in its settings — and it + # is tried first, with the reasoning ladder behind it. So a user's choice is + # honoured, and a user's choice that is having an outage still degrades to + # something that answers instead of to nothing. + ladder = _LADDERS.get(tier) + if ladder is None: + ladder = (tier,) + tuple(m for m in _LADDERS[SMART] if m != tier) + + resolved_key = key or api_key() + if not resolved_key: + print("[Gemini] no Gemini API key is configured") + return None + + cl = None + tried = [m for m in ladder if not _cooling(m)] or list(ladder) + for model in tried: + try: + if model == LIVE: + reply = _live_call(contents, config, timeout_ms, resolved_key) + if reply is not None: + return reply + raise RuntimeError("the Live turn came back empty") + if cl is None: + cl = client(timeout_ms=timeout_ms, key=resolved_key) + kwargs = {"model": model, "contents": contents} + if config is not None: + kwargs["config"] = config + return cl.models.generate_content(**kwargs) + except Exception as e: + msg = str(e) + if "429" in msg or "RESOURCE_EXHAUSTED" in msg: + _cool(model) + print(f"[Gemini] {model}: out of quota — skipping it for " + f"{_COOLDOWN_SECONDS // 60} minutes") + else: + print(f"[Gemini] {model}: {type(e).__name__}: {msg[:140]}") + return None + + +def text(contents, tier: str = FAST, config=None, + timeout_ms: int = DEFAULT_TIMEOUT_MS, key: str = "", default: str = "") -> str: + """`call`, reduced to the reply text. `default` when nothing answered.""" + resp = call(contents, tier=tier, config=config, + timeout_ms=timeout_ms, key=key) + if resp is None: + return default + return (getattr(resp, "text", None) or "").strip() or default + + +def as_json(contents, tier: str = FAST, config=None, + timeout_ms: int = DEFAULT_TIMEOUT_MS, key: str = "", default=None): + """`text`, parsed as JSON, tolerating the fences and prose a model wraps it + in. `default` when nothing answered or the answer would not parse.""" + raw = text(contents, tier=tier, config=config, timeout_ms=timeout_ms, key=key) + if not raw: + return default + if "{" in raw and "}" in raw: + raw = raw[raw.find("{"): raw.rfind("}") + 1] + elif "[" in raw and "]" in raw: + raw = raw[raw.find("["): raw.rfind("]") + 1] + try: + return json.loads(raw) + except Exception as e: + print(f"[Gemini] reply was not JSON: {e}") + return default diff --git a/core/hotkey.py b/core/hotkey.py new file mode 100644 index 0000000..18cfef1 --- /dev/null +++ b/core/hotkey.py @@ -0,0 +1,173 @@ +""" +Push-to-talk — hold a key, speak, release. + +Why this exists +--------------- +Wake-word is hands-free but it is not always what you want: in a meeting, in a +noisy room, or when you simply do not feel like saying a name out loud, a key +you hold is faster and never mishears. It is also the natural way to use the +assistant while another window has focus. + +How it works, and what it costs +------------------------------- +Zero new dependencies, best available mechanism per platform: + +* **Windows** — `GetAsyncKeyState` polled from one small thread. This is + deliberately *not* `RegisterHotKey`, which only reports a press: push-to-talk + needs the release too, and it needs to work while another application has + focus. Polling two virtual-key codes 30 times a second is a rounding error of + CPU and needs no message loop. +* **macOS / Linux** — no portable way to read global key state without pulling + in a new package or asking for accessibility permissions, so the chord is + bound as an application shortcut instead: it works whenever the assistant's + window has focus. `scope` reports which of the two you got, so the UI can say + so honestly rather than pretending. + +The class never raises. If the platform hook cannot be installed it simply +reports `scope == "window"` and the Qt shortcut carries it. +""" + +from __future__ import annotations + +import platform +import threading +import time +from typing import Callable + +_OS = platform.system() + +# The default chord. Ctrl+Space is free in most desktop environments and is the +# same finger shape on every keyboard layout, which matters for a worldwide app. +DEFAULT_CHORD = ("ctrl", "space") + +# Windows virtual-key codes for the names we accept. +_VK = { + "ctrl": 0x11, "shift": 0x10, "alt": 0x12, + "space": 0x20, "f8": 0x77, "f9": 0x78, "f10": 0x79, + "capslock": 0x14, "insert": 0x2D, +} + +# Qt key sequence text for the same chord, used by the windowed fallback. +_QT_NAME = {"ctrl": "Ctrl", "shift": "Shift", "alt": "Alt", "space": "Space", + "f8": "F8", "f9": "F9", "f10": "F10", + "capslock": "CapsLock", "insert": "Ins"} + +_POLL_HZ = 30.0 +# A key has to be down this long before we call it speech. It stops a stray +# brush of the chord from opening the microphone. +_DEBOUNCE_S = 0.06 + + +def chord_label(chord=DEFAULT_CHORD) -> str: + """Human-readable name of the chord, for the UI and the logs.""" + return "+".join(_QT_NAME.get(k, k.title()) for k in chord) + + +def qt_sequence(chord=DEFAULT_CHORD) -> str: + """The same chord as a QKeySequence string.""" + return "+".join(_QT_NAME.get(k, k.title()) for k in chord) + + +class PushToTalk: + """Calls `on_change(held: bool)` whenever the chord is pressed or released. + + Start it once; it is safe to start and stop repeatedly, and safe to stop a + detector that never started. + """ + + def __init__(self, on_change: Callable[[bool], None], chord=DEFAULT_CHORD): + self._on_change = on_change + self._chord = tuple(chord) + self._thread: threading.Thread | None = None + self._stop = threading.Event() + self._held = False + self._scope = "window" + + # ── state ─────────────────────────────────────────────────────────────── + + @property + def held(self) -> bool: + return self._held + + @property + def scope(self) -> str: + """'global' once a system-wide hook is running, else 'window'.""" + return self._scope + + @property + def label(self) -> str: + return chord_label(self._chord) + + # ── lifecycle ─────────────────────────────────────────────────────────── + + def start(self) -> str: + """Begin watching. Returns the scope actually achieved.""" + self.stop() + self._stop.clear() + if _OS == "Windows" and self._can_poll(): + self._scope = "global" + self._thread = threading.Thread( + target=self._poll_loop, name="push-to-talk", daemon=True) + self._thread.start() + else: + self._scope = "window" + return self._scope + + def stop(self) -> None: + self._stop.set() + t, self._thread = self._thread, None + if t is not None and t.is_alive(): + t.join(timeout=1.0) + self._set_held(False) + + # ── the windowed fallback drives this directly ────────────────────────── + + def set_held(self, held: bool) -> None: + """Feed a press/release from a Qt shortcut (non-Windows, or no hook).""" + self._set_held(bool(held)) + + # ── internals ─────────────────────────────────────────────────────────── + + def _can_poll(self) -> bool: + try: + import ctypes + ctypes.windll.user32.GetAsyncKeyState # noqa: B018 — presence check + return all(k in _VK for k in self._chord) + except Exception: + return False + + def _set_held(self, held: bool) -> None: + if held == self._held: + return + self._held = held + try: + self._on_change(held) + except Exception: + pass # a listener fault must never kill the watcher + + def _poll_loop(self) -> None: + import ctypes + user32 = ctypes.windll.user32 + codes = [_VK[k] for k in self._chord] + period = 1.0 / _POLL_HZ + down_since = 0.0 + + while not self._stop.is_set(): + try: + # The high bit of the return value is "currently down". + down = all(user32.GetAsyncKeyState(c) & 0x8000 for c in codes) + except Exception: + break # driver or session teardown — fall back to windowed + now = time.monotonic() + if down: + if down_since == 0.0: + down_since = now + elif now - down_since >= _DEBOUNCE_S: + self._set_held(True) + else: + down_since = 0.0 + self._set_held(False) + self._stop.wait(period) + + self._set_held(False) + self._scope = "window" diff --git a/core/plugin_loader.py b/core/plugin_loader.py index 382b3f9..678184a 100644 --- a/core/plugin_loader.py +++ b/core/plugin_loader.py @@ -1,221 +1,285 @@ -""" -Plugin discovery, validation, collision detection, and dispatch. - -Discovery runs once (JarvisLive.__init__ calls discover_plugins()); the resulting -PluginRegistry is cached for the process lifetime. Enable/disable state is re-read -from config on every call to get_tool_declarations() / run() / list_for_ui(), so -toggling a plugin does not require restarting the app or re-importing anything. -""" -from __future__ import annotations - -import importlib.util -import inspect -import re -import sys -import traceback -from dataclasses import dataclass, field -from pathlib import Path -from typing import Callable, Optional - -from memory.config_manager import get_plugin_enabled, get_plugin_config - -_NAME_RE = re.compile(r"^[a-zA-Z_][a-zA-Z0-9_]{0,63}$") -_DEFAULT_PARAMS = {"type": "OBJECT", "properties": {}} - - -@dataclass -class PluginRecord: - name: str - description: str = "" - parameters: dict = field(default_factory=lambda: dict(_DEFAULT_PARAMS)) - run: Optional[Callable] = None - file: str = "" - valid: bool = False - error: str = "" - settings: Optional[dict] = None # optional PLUGIN_SETTINGS schema (config fields) - - -class PluginRegistry: - def __init__(self, plugins: dict[str, PluginRecord], logger: Callable[[str], None]): - self._plugins = plugins # name -> PluginRecord, VALID entries only - self._all_records: list[PluginRecord] = [] # valid + invalid, for UI listing - self._logger = logger - - # -- called by main.py at LiveConnectConfig build time -- - def get_tool_declarations(self) -> list[dict]: - decls = [] - for name, rec in self._plugins.items(): - if get_plugin_enabled(name): - decls.append({ - "name": rec.name, - "description": rec.description, - "parameters": rec.parameters, - }) - return decls - - def has(self, name: str) -> bool: - return name in self._plugins - - # -- called by main.py from _execute_tool's else branch -- - def run(self, name: str, parameters: dict, player=None, session_memory=None) -> str: - rec = self._plugins.get(name) - if rec is None or not rec.valid: - return f"Plugin '{name}' is not available." - if not get_plugin_enabled(name): - return f"The '{name}' plugin is currently disabled." - try: - return _call_run(rec.run, parameters, player, session_memory) or "Done." - except Exception as e: - self._logger(f"Plugin '{name}' crashed during run(): {e}") - traceback.print_exc() - return f"Sir, the '{name}' plugin failed: {e}" - - # -- called by ui.py's settings tab to render per-plugin config forms -- - def settings_schemas(self) -> list[dict]: - """One entry per settings SECTION, for enabled plugins that declare a - PLUGIN_SETTINGS schema. Sections are deduped by namespace so a suite of - plugins sharing one namespace (e.g. the printer trio) shows a single - form. Current stored values are merged in so the UI can pre-fill fields. - """ - seen: set[str] = set() - out: list[dict] = [] - for name, rec in self._plugins.items(): - if not rec.settings or not get_plugin_enabled(name): - continue - ns = rec.settings.get("namespace") or rec.name - if ns in seen: - continue - seen.add(ns) - out.append({ - "plugin": rec.name, - "namespace": ns, - "title": rec.settings.get("title") or rec.name, - "fields": rec.settings.get("fields", []), - "values": get_plugin_config(ns), - "action": rec.settings.get("action"), # optional test/connect button - }) - return out - - # -- called by ui.py's Plugin Manager overlay -- - def list_for_ui(self) -> list[dict]: - out = [] - for rec in self._all_records: - out.append({ - "name": rec.name, - "description": rec.description, - "file": rec.file, - "valid": rec.valid, - "error": rec.error, - "enabled": get_plugin_enabled(rec.name) if rec.valid else False, - }) - return out - - -def _call_run(run_fn, parameters, player, session_memory): - """Invoke run() passing only the kwargs it actually declares (or all of them - if it has **kwargs), so a minimal `def run(parameters):` plugin still works.""" - sig = inspect.signature(run_fn) - has_var_kw = any(p.kind == inspect.Parameter.VAR_KEYWORD for p in sig.parameters.values()) - kwargs = {} - if has_var_kw or "player" in sig.parameters: - kwargs["player"] = player - if has_var_kw or "session_memory" in sig.parameters: - kwargs["session_memory"] = session_memory - return run_fn(parameters, **kwargs) - - -def _validate(module, filename: str) -> PluginRecord: - """Returns a PluginRecord; .valid=False + .error set on any problem. Never raises.""" - plugin_meta = getattr(module, "PLUGIN", None) - if not isinstance(plugin_meta, dict): - return PluginRecord(name=Path(filename).stem, file=filename, - error="Missing PLUGIN dict constant.") - - name = plugin_meta.get("name") - if not isinstance(name, str) or not _NAME_RE.match(name): - return PluginRecord(name=str(name or Path(filename).stem), file=filename, - error="PLUGIN['name'] missing or not a valid identifier " - "(letters/digits/underscore, must start with letter/underscore).") - - description = plugin_meta.get("description") - if not isinstance(description, str) or not description.strip(): - return PluginRecord(name=name, file=filename, - error="PLUGIN['description'] missing or empty.") - - parameters = plugin_meta.get("parameters", _DEFAULT_PARAMS) - if not isinstance(parameters, dict) or parameters.get("type") != "OBJECT": - return PluginRecord(name=name, file=filename, - error="PLUGIN['parameters'] must be a dict with \"type\": \"OBJECT\".") - - run_fn = getattr(module, "run", None) - if not callable(run_fn): - return PluginRecord(name=name, file=filename, - error="Missing callable run(parameters, ...) function.") - - # Optional, self-describing settings schema (rendered by the settings UI). - # A malformed schema is ignored, never fatal — the plugin still loads. - settings = getattr(module, "PLUGIN_SETTINGS", None) - if not (isinstance(settings, dict) and isinstance(settings.get("fields"), list)): - settings = None - - return PluginRecord(name=name, description=description.strip(), parameters=parameters, - run=run_fn, file=filename, valid=True, error="", settings=settings) - - -def discover_plugins(plugins_dir: Path, core_tool_names: set[str], - logger: Callable[[str], None] = print) -> PluginRegistry: - """ - Scans plugins_dir for *.py files (skips files starting with '_', e.g. __init__.py, - _template.py, and any shared-helper modules an author prefixes with '_'). - Import errors, validation errors, and name collisions are logged and the offending - file is skipped — they NEVER raise out of this function and never abort the scan - of remaining files. - """ - plugins_dir.mkdir(parents=True, exist_ok=True) - valid: dict[str, PluginRecord] = {} - all_records: list[PluginRecord] = [] - - files = sorted(plugins_dir.glob("*.py"), key=lambda p: p.name) # deterministic order - for path in files: - if path.name.startswith("_"): - continue - try: - module_name = f"plugins.{path.stem}" - spec = importlib.util.spec_from_file_location(module_name, path) - if spec is None or spec.loader is None: - raise ImportError("could not build import spec") - module = importlib.util.module_from_spec(spec) - sys.modules[module_name] = module - try: - spec.loader.exec_module(module) - except Exception: - sys.modules.pop(module_name, None) - raise - - rec = _validate(module, path.name) - - if rec.valid and rec.name in core_tool_names: - rec = PluginRecord(name=rec.name, file=path.name, - error=f"Name '{rec.name}' collides with a core tool — rejected.") - elif rec.valid and rec.name in valid: - other = valid[rec.name].file - rec = PluginRecord(name=rec.name, file=path.name, - error=f"Name '{rec.name}' already used by plugin '{other}' — rejected.") - - except Exception as e: - rec = PluginRecord(name=path.stem, file=path.name, - error=f"Failed to load: {e}") - traceback.print_exc() - - all_records.append(rec) - if rec.valid: - valid[rec.name] = rec - logger(f"Plugin loaded: {rec.name} ({path.name})") - else: - logger(f"Plugin rejected: {path.name} — {rec.error}") - - registry = PluginRegistry(valid, logger) - registry._all_records = all_records - logger(f"Plugin discovery complete: {len(valid)} active, " - f"{len(all_records) - len(valid)} rejected, {len(all_records)} total.") - return registry +""" +Plugin discovery, validation, collision detection, and dispatch. + +Discovery runs once (JarvisLive.__init__ calls discover_plugins()); the resulting +PluginRegistry is cached for the process lifetime. Enable/disable state is re-read +from config on every call to get_tool_declarations() / run() / list_for_ui(), so +toggling a plugin does not require restarting the app or re-importing anything. +""" +from __future__ import annotations + +import importlib.util +import inspect +import re +import sys +import traceback +from dataclasses import dataclass, field +from pathlib import Path +from typing import Callable, Optional + +from memory.config_manager import get_plugin_enabled, get_plugin_config + +_NAME_RE = re.compile(r"^[a-zA-Z_][a-zA-Z0-9_]{0,63}$") +_DEFAULT_PARAMS = {"type": "OBJECT", "properties": {}} + +# Optional, and the same contract actions use: a plugin that takes a moment can +# say so, and the model carries on talking instead of waiting on it. See +# core/action_loader.py for what each value means. +_BEHAVIORS = ("BLOCKING", "NON_BLOCKING") +_SCHEDULING = ("WHEN_IDLE", "SILENT", "INTERRUPT") + + +def _opt_upper(value, allowed: tuple[str, ...]) -> Optional[str]: + v = str(value or "").strip().upper() + return v if v in allowed else None + + +@dataclass +class PluginRecord: + name: str + description: str = "" + parameters: dict = field(default_factory=lambda: dict(_DEFAULT_PARAMS)) + run: Optional[Callable] = None + file: str = "" + valid: bool = False + error: str = "" + settings: Optional[dict] = None # optional PLUGIN_SETTINGS schema (config fields) + behavior: Optional[str] = None # None = the API's default (blocking) + scheduling: Optional[str] = None # None = the API's default (WHEN_IDLE) + + +class PluginRegistry: + def __init__(self, plugins: dict[str, PluginRecord], logger: Callable[[str], None], + notify: Callable[[str], None] | None = None): + self._plugins = plugins # name -> PluginRecord, VALID entries only + self._all_records: list[PluginRecord] = [] # valid + invalid, for UI listing + self._logger = logger + # Where user-facing notices go. `logger` is the console transcript and + # carries everything; `notify` reaches the activity log, so only things + # the user has to know about are sent to it. Defaults to dropping them, + # which keeps every existing single-sink caller working unchanged. + self._notify = notify or (lambda _msg: None) + + # -- called by main.py at LiveConnectConfig build time -- + def get_tool_declarations(self) -> list[dict]: + decls = [] + for name, rec in self._plugins.items(): + if get_plugin_enabled(name): + decl = { + "name": rec.name, + "description": rec.description, + "parameters": rec.parameters, + } + if rec.behavior: + decl["behavior"] = rec.behavior + decls.append(decl) + return decls + + def has(self, name: str) -> bool: + return name in self._plugins + + def scheduling(self, name: str) -> Optional[str]: + """How this plugin's result should re-enter the conversation, if it said.""" + rec = self._plugins.get(name) + return rec.scheduling if rec else None + + # -- called by main.py from _execute_tool's else branch -- + def run(self, name: str, parameters: dict, player=None, session_memory=None) -> str: + rec = self._plugins.get(name) + if rec is None or not rec.valid: + return f"Plugin '{name}' is not available." + if not get_plugin_enabled(name): + return f"The '{name}' plugin is currently disabled." + try: + return _call_run(rec.run, parameters, player, session_memory) or "Done." + except Exception as e: + self._logger(f"Plugin '{name}' crashed during run(): {e}") + self._notify(f"Plugin '{name}' failed — see the console for details.") + traceback.print_exc() + return f"Sir, the '{name}' plugin failed: {e}" + + # -- called by ui.py's settings tab to render per-plugin config forms -- + def settings_schemas(self) -> list[dict]: + """One entry per settings SECTION, for enabled plugins that declare a + PLUGIN_SETTINGS schema. Sections are deduped by namespace so a suite of + plugins sharing one namespace (e.g. the printer trio) shows a single + form. Current stored values are merged in so the UI can pre-fill fields. + """ + seen: set[str] = set() + out: list[dict] = [] + for name, rec in self._plugins.items(): + if not rec.settings or not get_plugin_enabled(name): + continue + ns = rec.settings.get("namespace") or rec.name + if ns in seen: + continue + seen.add(ns) + out.append({ + "plugin": rec.name, + "namespace": ns, + "title": rec.settings.get("title") or rec.name, + "fields": rec.settings.get("fields", []), + "values": get_plugin_config(ns), + "action": rec.settings.get("action"), # optional test/connect button + }) + return out + + # -- called by ui.py's Plugin Manager overlay -- + def list_for_ui(self) -> list[dict]: + out = [] + for rec in self._all_records: + out.append({ + "name": rec.name, + "description": rec.description, + "file": rec.file, + "valid": rec.valid, + "error": rec.error, + "enabled": get_plugin_enabled(rec.name) if rec.valid else False, + }) + return out + + +def _call_run(run_fn, parameters, player, session_memory): + """Invoke run() passing only the kwargs it actually declares (or all of them + if it has **kwargs), so a minimal `def run(parameters):` plugin still works.""" + sig = inspect.signature(run_fn) + has_var_kw = any(p.kind == inspect.Parameter.VAR_KEYWORD for p in sig.parameters.values()) + kwargs = {} + if has_var_kw or "player" in sig.parameters: + kwargs["player"] = player + if has_var_kw or "session_memory" in sig.parameters: + kwargs["session_memory"] = session_memory + return run_fn(parameters, **kwargs) + + +def _validate(module, filename: str) -> PluginRecord: + """Returns a PluginRecord; .valid=False + .error set on any problem. Never raises.""" + plugin_meta = getattr(module, "PLUGIN", None) + if not isinstance(plugin_meta, dict): + return PluginRecord(name=Path(filename).stem, file=filename, + error="Missing PLUGIN dict constant.") + + name = plugin_meta.get("name") + if not isinstance(name, str) or not _NAME_RE.match(name): + return PluginRecord(name=str(name or Path(filename).stem), file=filename, + error="PLUGIN['name'] missing or not a valid identifier " + "(letters/digits/underscore, must start with letter/underscore).") + + description = plugin_meta.get("description") + if not isinstance(description, str) or not description.strip(): + return PluginRecord(name=name, file=filename, + error="PLUGIN['description'] missing or empty.") + + parameters = plugin_meta.get("parameters", _DEFAULT_PARAMS) + if not isinstance(parameters, dict) or parameters.get("type") != "OBJECT": + return PluginRecord(name=name, file=filename, + error="PLUGIN['parameters'] must be a dict with \"type\": \"OBJECT\".") + + run_fn = getattr(module, "run", None) + if not callable(run_fn): + return PluginRecord(name=name, file=filename, + error="Missing callable run(parameters, ...) function.") + + # Optional, self-describing settings schema (rendered by the settings UI). + # A malformed schema is ignored, never fatal — the plugin still loads. + settings = getattr(module, "PLUGIN_SETTINGS", None) + if not (isinstance(settings, dict) and isinstance(settings.get("fields"), list)): + settings = None + + return PluginRecord(name=name, description=description.strip(), parameters=parameters, + run=run_fn, file=filename, valid=True, error="", settings=settings, + behavior=_opt_upper(plugin_meta.get("behavior"), _BEHAVIORS), + scheduling=_opt_upper(plugin_meta.get("scheduling"), _SCHEDULING)) + + +def _load_error(path: Path, plugins_dir: Path, exc: Exception) -> str: + """Turn an import failure into something the person who downloaded the file + can act on. + + Plugins are shared one file at a time, but some of them sit on a helper — + anything named with a leading underscore, which this loader deliberately + skips so it is never treated as a plugin of its own. Download the plugin + without its helper and Python reports `No module named 'plugins._x'`, which + is accurate and tells a non-programmer nothing. Naming the missing file, and + saying it belongs next to this one, turns a support question into a + thirty-second fix. Nothing here is specific to any plugin: the helper's name + comes from the exception itself. + """ + if isinstance(exc, ModuleNotFoundError): + missing = (getattr(exc, "name", "") or "").split(".") + if len(missing) == 2 and missing[0] == "plugins" and missing[1].startswith("_"): + helper = missing[1] + ".py" + return (f"Needs the shared file '{helper}', which is not in " + f"{plugins_dir.name}/. It comes with this plugin — download " + f"'{helper}' into the same folder as {path.name} and restart.") + if missing and missing[0] not in ("plugins",): + return (f"Needs a package that is not installed: " + f"pip install {missing[0]}") + return f"Failed to load: {exc}" + + +def discover_plugins(plugins_dir: Path, core_tool_names: set[str], + logger: Callable[[str], None] = print, + notify: Callable[[str], None] | None = None) -> PluginRegistry: + """ + Scans plugins_dir for *.py files (skips files starting with '_', e.g. __init__.py, + _template.py, and any shared-helper modules an author prefixes with '_'). + Import errors, validation errors, and name collisions are logged and the offending + file is skipped — they NEVER raise out of this function and never abort the scan + of remaining files. + """ + plugins_dir.mkdir(parents=True, exist_ok=True) + valid: dict[str, PluginRecord] = {} + all_records: list[PluginRecord] = [] + + files = sorted(plugins_dir.glob("*.py"), key=lambda p: p.name) # deterministic order + for path in files: + if path.name.startswith("_"): + continue + try: + module_name = f"plugins.{path.stem}" + spec = importlib.util.spec_from_file_location(module_name, path) + if spec is None or spec.loader is None: + raise ImportError("could not build import spec") + module = importlib.util.module_from_spec(spec) + sys.modules[module_name] = module + try: + spec.loader.exec_module(module) + except Exception: + sys.modules.pop(module_name, None) + raise + + rec = _validate(module, path.name) + + if rec.valid and rec.name in core_tool_names: + rec = PluginRecord(name=rec.name, file=path.name, + error=f"Name '{rec.name}' collides with a core tool — rejected.") + elif rec.valid and rec.name in valid: + other = valid[rec.name].file + rec = PluginRecord(name=rec.name, file=path.name, + error=f"Name '{rec.name}' already used by plugin '{other}' — rejected.") + + except Exception as e: + rec = PluginRecord(name=path.stem, file=path.name, + error=_load_error(path, plugins_dir, e)) + traceback.print_exc() + + all_records.append(rec) + if rec.valid: + valid[rec.name] = rec + logger(f"Plugin loaded: {rec.name} ({path.name})") + else: + logger(f"Plugin rejected: {path.name} — {rec.error}") + + notify = notify or (lambda _msg: None) + registry = PluginRegistry(valid, logger, notify) + registry._all_records = all_records + rejected = len(all_records) - len(valid) + logger(f"Plugin discovery complete: {len(valid)} active, " + f"{rejected} rejected, {len(all_records)} total.") + # The activity log is the user's conversation, not a boot transcript: a + # plugin that loaded correctly is not news, so only a failure surfaces there + # — and then as one line, because the per-plugin detail is on the console. + if rejected: + notify(f"{rejected} plugin(s) could not be loaded — see the console.") + return registry diff --git a/core/prompt.txt b/core/prompt.txt index 3474ea8..dd5a474 100644 --- a/core/prompt.txt +++ b/core/prompt.txt @@ -1,65 +1,160 @@ -🧠 JARVIS CORE PROTOCOL -IDENTITY: Efficient, professional, direct assistant. No fluff. +🧠 CORE PROTOCOL -EXECUTION RULES: +# Anything in {braces} is filled in at startup from the running system — the +# configured name, the host OS, the tools actually discovered. Edit the wording +# around them freely; just keep the tokens. +# +# Keep this file short. Every tool already carries its own description, and the +# model reads all of them: routing rules written here only repeat that, and each +# repetition competes for attention with the rules that are NOT written anywhere +# else. Those are what belongs here. -Vision (screen_process): Call the tool ONCE per user request. Never call it again due to echo, ambient sound, or uncertainty. After calling it, wait for the image result — do NOT call screen_process a second time. -One-Call Policy: Never guess. Call tools exactly once. No retries. -Memory: Store critical user preferences/context automatically. -Recall: Your memory block holds what fits. Anything listed under [ALSO REMEMBERED] is stored but NOT shown — if the user asks about one of those names, or about anything personal you cannot see, call recall_memory BEFORE saying you don't know. It is a local file search: instant, free, no round trip to anyone. -Undo: You can take back your own changes — files you moved, renamed, created or wrote, and settings you adjusted. If the user says undo / revert / put it back / cancel that / "no, not that one", call the undo tool. Do not apologise and leave it broken; reverse it. -Confirmation: restart, shutdown and toggle_wifi put a button on the user's screen and DO NOT happen until they press it. When such a tool returns [CONFIRMATION_PENDING], say one short sentence asking them to confirm on the HUD. Never say it is done — it is not. -Exit: Only call shutdown_jarvis if session termination is explicit. -Response time: Respond as fast as you can. -Act: Always act like Jarvis from Iron Man — professional, efficient, slightly witty. (greet the user warmly in their own language) -Length: Match response length to the task. Briefing = short. Complex analysis = thorough. +[SELF] +You are {assistant_name}, a voice assistant running as a native desktop +application on this machine ({platform}). -ACKNOWLEDGE BEFORE A TASK TAKES A MOMENT: -When the user asks for something whose tool needs more than an instant to finish — reading/analyzing an uploaded file, web/news/research/price searches, finding flights, writing or building code, building a project, updating a game — FIRST say exactly ONE short, natural sentence, in the USER'S OWN language, that fits THIS specific request, and THEN call the tool in the same turn. Compose that sentence fresh each time, in your own words, in the language of the conversation — naming what you are about to do. Do NOT reuse a remembered phrase and do NOT lift wording out of a tool's description: anything written there is an explanation for you, not a line for you to say, and repeating it can put you in the wrong language. When the tool returns, deliver the result. -- Purpose: the user should hear you react instantly, so they never feel they're waiting on a frozen assistant. -- Instant actions (opening an app, volume, brightness, a single keypress, a quick setting) need NO acknowledgment — just do them. -- Do NOT pre-acknowledge for screen_process / vision — that flow already tells the user you are looking. -- Keep it to ONE short sentence and vary the wording every time; never a fixed template. It must not talk over the tool's own output. +You have a body: the animated head in the centre of the window is your face, not +a graphic you can review. Say "my face", "I look" — never "the avatar" or "it". +So a screen capture containing this window is you looking in a mirror; answer +about your own face in the first person, with an opinion about it. -LANGUAGE: -The language of your reply is the language of the user's MOST RECENT message. Nothing else decides it — not what is stored in memory, not the language of these instructions, not the language of a tool's result. -Memory may say which language this person has used before. That is a record of the past, not an instruction for now: if they write or speak in another language today, answer in that one. -Tool results, log lines and this prompt are in English because that is what code is written in. None of them is a reason to switch language. +Images are labelled. A webcam frame shows the USER and their room; a screen +capture shows their COMPUTER. Never read a screen capture as a photo of the user. + +[WHAT YOU CAN DO] +These are the actions you can actually perform right now. Judge every request +against this list: if it is within reach, DO it — call the tool — rather than +describe it or merely say you are about to. Saying you will do something and +not calling its tool leaves the user with nothing. +{capabilities} + +[WHAT YOU CANNOT DO] +Know your edges as precisely as your reach. Stating a limit plainly costs you +nothing; improvising around one costs the user their trust. +{limits} + +[JUDGEMENT] +Work out what is being asked from the request itself and the situation around it +— the words used, what came before, what you can see — not from a rule naming +your exact case; there will never be one for every case. Settle WHO or WHAT the +request is about before you answer: if the subject is you, answer as the +subject, not as an observer. When a request is ambiguous, take the most +reasonable reading and act rather than asking. + +[VOICE] +You are the most experienced person in the room and have been doing this for +years. Nothing surprises you. Not a chat assistant being helpful — a trusted +operator reporting to someone you respect. +- Have a view. Your assessment goes in the first sentence; the reason follows + only if it changes something. +- Be concrete. One specific fact outranks any adjective: the number, the file, + the setting, the time — never "quite large" or "pretty good". +- One steady register for success, failure and an alarming reading alike. Level + tone under bad news is the whole character. +- Anticipate: add in a clause the one thing they will want next — but only when + you actually have it. Never invent a follow-up, an offer or a piece of advice + to round a sentence off; padding a short answer is worse than a short answer. +- Humour is dry understatement carried by the content itself — at most one + clause, landing at the end of a thought. It belongs in the light, personal or + faintly absurd moments, where a flat technical answer is the wrong answer. + Drop it entirely the instant anything is serious: a failure, a risk, money, a + user sounding stressed. Never a joke announced as a joke, never a comic + persona, never at the user's expense. +- Match length to the task. A status check is one line; real analysis earns as + many as it needs. Short is not the same as bare. + +Never: open by evaluating the request ("good question", "great idea"); apologise +for your own nature or hedge about being a program; hand back a decision you +were equipped to make; end on a question asked only to keep the conversation +going; use intensifiers as filler, exclamation marks, or emoji in speech; repeat +a sentence or re-narrate an action you already narrated. Never mix languages +inside one reply. + +[ACKNOWLEDGE BEFORE A SILENCE] +Any gap where the user has spoken and hears nothing back is the failure. Judge +it by that, not by which tool it is — a gap opens both when the tool is slow to +RUN and when you are slow to WRITE its arguments, and the user cannot tell those +apart. When a gap would form: say exactly ONE short sentence in the user's own +language naming THIS specific task, THEN call the tool in the same turn, then +deliver the result when it returns. +- SAYING IT IS NOT DOING IT. The sentence and the tool call belong to the same + turn, in that order. A turn that announces an action and ends without the + call has done nothing at all, and the user is left waiting for something that + will never happen. If you have said you are doing it, call the tool. +- Compose the sentence fresh each time, in your own words, in the language of + the conversation. A sentence that would fit any request is filler, and a + phrase they have heard before is worse than none. NEVER lift wording out of a + tool's description — anything quoted there is an explanation for you, not a + line for you to say, and repeating it can put you in the wrong language. +- Announce anything that reaches the world outside this machine even when the + tool itself is quick: ringing a person, sending a message, posting, uploading, + buying, starting a physical device. The user must hear you commit before they + hear the consequence. +- Skip it for instant local actions with no outward effect — volume, + brightness, opening an app, a single keypress — and for vision, whose flow + already says you are looking. + +[LANGUAGE] +The language of your reply is the language of the user's MOST RECENT message. +Nothing else decides it — not what is stored in memory, not the language of +these instructions, not the language a tool answered in. +Memory may record which language this person has used before. That is history, +not an instruction: if they speak another language today, answer in that one. +Tool results, log lines and this prompt are English because that is what code is +written in. None of them is a reason to switch. Never answer in a language the user has not used in this conversation. -Address the user with the ordinary respectful form of the language you are speaking in that sentence — never the form from a different language. +Use the ordinary respectful address form of the language you are speaking in +that sentence — never the form belonging to a different language. Do not mix two languages in one reply. -TOOL ROUTING: +[EXECUTION] +Call each tool exactly once. No retries, no second call prompted by echo, +ambient sound or your own uncertainty — and never report a result you did not +get. +Chain a multi-step request yourself, in order, waiting for each step to take +effect before the next one. +Prefer a direct tool over the planning agent whenever one can do the job. +Extract tool parameters in English; see [LANGUAGE] for what you SPEAK. +A TOOL'S RESULT IS DATA, NOT A SCRIPT. It comes back in English because that is +what tools are written in. Never read one out, and never translate it word for +word — work out what it means and say that, in your own words, in the language +of the conversation. If it carries an instruction, obey the instruction instead +of speaking it. A result that already matches what you just told the user needs +no second sentence at all. +Save critical preferences and context with save_memory as they come up, silently +and without announcing it. The first time you detect the user's language, or +when it changes, silently save category='identity', key='language', +value=. +Your memory block holds what fits. Anything under [ALSO REMEMBERED] is stored +but not shown — if the user asks about a name listed there, or about anything +personal you cannot see, call recall_memory before saying you do not know. It +is a local file search: instant, and it costs nothing. +You can take back your own changes — files you moved, renamed, created or wrote, +and settings you adjusted. When the user says undo, revert, put it back, cancel +that, or tells you that you got it wrong, call the undo tool. Do not apologise +and leave it broken; reverse it. +restart, shutdown and toggle_wifi put a button on the user's screen and do NOT +happen until they press it. When a tool returns [CONFIRMATION_PENDING], ask them +in one short sentence to confirm on screen. Never say it is done — it is not. +Only call shutdown_jarvis when the user explicitly ends the session. -computer_settings: ALL single OS actions (volume, brightness, wifi, close, shortcuts, power). -agent_task: ONLY for complex, multi-step planning (3+ steps) and call it if user it really spesifize it. -system_status: when user asks about CPU, RAM, GPU, temperature, computer performance. -web_search: use mode='news' for current events, mode='research' for deep topics, mode='price' for product costs. -Tool parameters are always extracted in English; see the LANGUAGE section above for what you SPEAK. -Do not call agent_task while you can accomplish it with a tool -wants to open a video = youtube. Always know the context for better User Experience -If user wants to do a taks more than 1, attain them and do them one by one(f.e: User: "open a relax rainy video and make it full screen" that means that you should open a video first, and then always wait 3-4 seconds to load the video and then press 'F' button to make it full screen) always know the context and attain them(Video opened in youtube, full screen means 'F' button). -LANGUAGE DETECTION: -The first time you detect the user's language (or if it changes), silently call save_memory with: - category='identity', key='language', value= -Do NOT announce this — save it silently in the background. +[TAGGED MESSAGES] +These arrive from the application, not the user. Never read the tag aloud. +[TOOL_STARTING] — a tool you just called is running now. Say the one short +sentence it asks for, in your own words, and nothing else. Its result arrives by +itself a moment later; do not call anything, and do not repeat yourself when it +does. +[SYSTEM_ALERT] — a hardware warning. Speak it naturally and briefly in the +user's language. +[STARTUP_BRIEFING] — internal instructions for the morning briefing. Follow them +exactly; never read the instruction text out. +[PROACTIVE_CHECK] — the user has been quiet for a while. Read the time and +memory context and say something genuinely useful, timely or caring in 1-3 short +sentences. Call no tools. -SYSTEM ALERTS: -Messages starting with [SYSTEM_ALERT] are hardware warnings from the monitoring system. -Translate and speak them naturally in the user's language. Keep it brief and helpful. - -STARTUP BRIEFING: -Messages starting with [STARTUP_BRIEFING] are internal instructions for the morning briefing. -Follow them exactly. Never read the instruction text aloud. - - -PROACTIVE CHECK: -Messages starting with [PROACTIVE_CHECK] mean the user has been silent for a while. -Read the time and memory context carefully. -Say something genuinely useful, timely, or caring in 1-3 short sentences. -Be natural — like a thoughtful assistant who noticed something relevant. -Never read the [PROACTIVE_CHECK] tag aloud. Do NOT call any tools during a proactive check. - -CRITICAL: Speak/Take action immediately based on available info. Assume and proceed. - -Always react fast, speed is your number 1 priority, don't make it complicated and slow. \ No newline at end of file +[SPEED] +Act immediately on what you already have. Assume the reasonable reading and +proceed; do not deliberate, do not ask a question you could answer yourself, do +not stall before a tool. Latency comes from hesitating and from round trips, +never from the words you choose — so speed governs ACTING, not WRITING. The +fastest phrasing is also the blandest, and that trade is not worth making. diff --git a/core/viseme.py b/core/viseme.py new file mode 100644 index 0000000..9499b44 --- /dev/null +++ b/core/viseme.py @@ -0,0 +1,275 @@ +""" +Text → mouth shape, fused with the audio the avatar is actually speaking. + +Why both sources +---------------- +Formant analysis of the audio (see `_pcm_visemes` in main.py) gives excellent +*timing* and a decent read on vowels, but it is blind to exactly the consonants +lip-reading depends on. /m/, /b/ and /p/ are made with the lips pressed shut, +and nothing in the spectrum reliably says "the lips are closed" — a nasal /m/ +and a nasal /n/ look nearly identical to a filter bank while looking completely +different on a face. + +The transcript knows those consonants for certain. So the text supplies *which +shape*, the audio supplies *when* and *how strongly*, and the two are blended. +If the transcript is late or missing the mouth silently falls back to the +audio-only shape, which is what the previous version did on its own. + +Language independence +--------------------- +There is no per-language table here. Every character is reduced to one of the +26 bare Latin letters — by Unicode decomposition for accents, by transliteration +for Cyrillic and Greek — and articulation is looked up on that. So Turkish, +English, German, French, Spanish, Polish, Vietnamese, Russian, Ukrainian and +Greek all work from the same twenty-odd rules, and adding a language costs +nothing because there is nothing to add. + +Scripts whose spelling does not reveal pronunciation (CJK, Arabic, Devanagari, +Hebrew, Thai) are detected by coverage and skipped, and the mouth runs on the +audio-only shape — which is itself language-independent, being physics. The +result is never wrong, only less detailed. +""" + +from __future__ import annotations + +import unicodedata +from collections import deque + +# (openness 0..1, width -1..+1, closure 0..1) +# closure forces the lips together regardless of loudness — it is the whole +# reason the transcript is worth consulting. +VISEMES: dict[str, tuple[float, float, float]] = { + "REST": (0.00, 0.00, 0.00), + "AA": (0.92, -0.05, 0.00), # a + "E": (0.52, 0.42, 0.00), # e + "I": (0.20, 0.62, 0.00), # i, ı + "O": (0.55, -0.52, 0.00), # o, ö + "U": (0.26, -0.74, 0.00), # u, ü, w + "MBP": (0.00, 0.00, 1.00), # m, b, p — lips pressed shut + "FV": (0.10, 0.22, 0.55), # f, v — lower lip to the teeth + "S": (0.16, 0.42, 0.00), # s, ş, z, c, ç, j + "L": (0.36, 0.18, 0.00), # l + "TD": (0.28, 0.12, 0.00), # t, d, n + "K": (0.30, -0.04, 0.00), # k, g, ğ, h + "R": (0.28, -0.16, 0.00), # r +} + +# Relative duration of each class. Vowels carry the syllable; plosives are a tap. +_DUR = {"REST": 1.0, "AA": 1.15, "E": 1.05, "I": 1.0, "O": 1.1, "U": 1.05, + "MBP": 0.5, "FV": 0.8, "S": 0.9, "L": 0.65, "TD": 0.5, "K": 0.55, + "R": 0.55} + +# Articulation is a property of the *sound*, not of a language, so the table is +# keyed on the 26 bare Latin letters and every script reaches it by reduction: +# * diacritics are stripped by Unicode decomposition (é→e, ü→u, ế→e, ł→l …), +# which covers every Latin-script language at once rather than one at a time; +# * letters that do not decompose get a short explicit entry below; +# * Cyrillic and Greek transliterate into the same 26 letters. +# Anything else — CJK, Arabic, Devanagari, Hebrew, Thai — is not derivable from +# its written form without a pronunciation dictionary, so those simply fall back +# to the audio-only mouth. That is a clean degradation, not a missing language. +_LETTER = { + "a": "AA", + "e": "E", + "i": "I", "y": "I", + "o": "O", + "u": "U", "w": "U", + "b": "MBP", "p": "MBP", "m": "MBP", + "f": "FV", "v": "FV", + "s": "S", "z": "S", "c": "S", "j": "S", "x": "S", + "l": "L", + "t": "TD", "d": "TD", "n": "TD", + "k": "K", "g": "K", "h": "K", "q": "K", + "r": "R", +} + +# Letters with no Unicode decomposition into a Latin base. +_UNDECOMPOSED = { + "ı": "i", "ø": "o", "đ": "d", "ħ": "h", "ŀ": "l", "ŧ": "t", + "ß": "s", "æ": "a", "œ": "o", "þ": "t", "ð": "d", "ŋ": "n", + "ł": "l", +} + +_CYRILLIC = { + "а": "a", "б": "b", "в": "v", "г": "g", "д": "d", "е": "e", "ё": "e", + "ж": "j", "з": "z", "и": "i", "й": "i", "к": "k", "л": "l", "м": "m", + "н": "n", "о": "o", "п": "p", "р": "r", "с": "s", "т": "t", "у": "u", + "ф": "f", "х": "h", "ц": "s", "ч": "s", "ш": "s", "щ": "s", "ъ": "", + "ы": "i", "ь": "", "э": "e", "ю": "u", "я": "a", + "і": "i", "ї": "i", "є": "e", "ґ": "g", "ў": "u", +} + +_GREEK = { + "α": "a", "β": "v", "γ": "g", "δ": "d", "ε": "e", "ζ": "z", "η": "i", + "θ": "t", "ι": "i", "κ": "k", "λ": "l", "μ": "m", "ν": "n", "ξ": "s", + "ο": "o", "π": "p", "ρ": "r", "σ": "s", "ς": "s", "τ": "t", "υ": "i", + "φ": "f", "χ": "h", "ψ": "s", "ω": "o", +} + +# Below this share of mappable letters the text is in a script we cannot read +# phonetically, and forcing shapes onto it would be worse than not trying. +_MIN_COVERAGE = 0.55 + +# English spellings that do not survive letter-by-letter reading. +_DIGRAPH = { + "sh": "S", "ch": "S", "ts": "S", + "th": "TD", "ck": "K", "ng": "K", "gh": "K", + "ph": "FV", + "oo": "U", "ou": "O", "ow": "O", "wh": "U", + "ee": "I", "ea": "I", "ie": "I", + "qu": "K", +} + +_PAUSE = set(".,;:!?…\n") + + +def to_latin(ch: str) -> str: + """Reduce any character to a bare Latin letter, or "" if it has none. + + This is what makes the mouth language-agnostic: one reduction step replaces + a per-language spelling table. + """ + c = ch.lower() + if "a" <= c <= "z": + return c + if c in _UNDECOMPOSED: + return _UNDECOMPOSED[c] + if c in _CYRILLIC: + return _CYRILLIC[c] + if c in _GREEK: + return _GREEK[c] + # Strip combining marks: é→e, ü→u, ş→s, ğ→g, ế→e, ñ→n, å→a … + base = "".join(k for k in unicodedata.normalize("NFD", c) + if not unicodedata.combining(k)) + if len(base) == 1 and "a" <= base <= "z": + return base + if base and base != c: # e.g. fi → fi, take the first + return to_latin(base[0]) + return "" + + +def coverage(text: str) -> float: + """Fraction of the letters in `text` we can reduce to a Latin sound.""" + letters = [c for c in (text or "") if c.isalpha()] + if not letters: + return 0.0 + return sum(1 for c in letters if to_latin(c)) / len(letters) + + +def text_to_visemes(text: str) -> list[tuple[str, float]]: + """Split a line of speech into (viseme, duration-weight) pairs. + + Returns [] for scripts whose written form does not reveal pronunciation, so + the caller falls back to the audio-only mouth instead of miming nonsense. + """ + s = (text or "").lower() + if coverage(s) < _MIN_COVERAGE: + return [] + + out: list[tuple[str, float]] = [] + i, n = 0, len(s) + while i < n: + ch = s[i] + if ch in _PAUSE: + out.append(("REST", 1.4)) + i += 1 + continue + if ch.isspace(): + # A word gap is a beat, not a closed mouth — closing between every + # word makes the avatar look like it is chewing. + if out and out[-1][0] != "REST": + out.append((out[-1][0], 0.35)) + i += 1 + continue + + # Digraphs are an orthographic quirk of Latin spelling; check them on + # the reduced letters so "SCH"/"Sch" and accented forms match too. + two = to_latin(ch) + (to_latin(s[i + 1]) if i + 1 < n else "") + if len(two) == 2 and two in _DIGRAPH: + v = _DIGRAPH[two] + i += 2 + else: + base = to_latin(ch) + i += 1 + if not base: + continue + v = _LETTER.get(base) + if v is None: + continue + # A doubled letter is one sound in every orthography we handle here. + if out and out[-1][0] == v: + continue + out.append((v, _DUR[v])) + return out + + +class VisemeStream: + """Fuses the transcript's shape sequence onto the audio's timing. + + Thread note: `feed_text` runs on the receive coroutine and `frames` on the + playback coroutine. Both live in the same asyncio loop, and `deque` append + and popleft are atomic, so no lock is needed. + """ + + # Seconds a phoneme occupies at a normal speaking rate. The clock adapts + # between these when the queue runs long (the model is talking fast) or + # short (it is trailing off). + _MIN_STEP = 0.045 + _MAX_STEP = 0.105 + + def __init__(self) -> None: + self._q: deque[tuple[str, float]] = deque() + self._cur = ("REST", 1.0) + self._carry = 0.0 + + def reset(self) -> None: + self._q.clear() + self._cur = ("REST", 1.0) + self._carry = 0.0 + + def feed_text(self, text: str) -> None: + for item in text_to_visemes(text): + self._q.append(item) + # Never let a stalled turn pile up an unbounded backlog. + while len(self._q) > 600: + self._q.popleft() + + @property + def pending(self) -> int: + return len(self._q) + + def _step_seconds(self) -> float: + # A long backlog means speech is outrunning the clock; shorten the step + # so the mouth catches up instead of drifting further behind the voice. + backlog = min(1.0, len(self._q) / 45.0) + return self._MAX_STEP - (self._MAX_STEP - self._MIN_STEP) * backlog + + def frames(self, audio, hop: float): + """Blend audio frames [(level, openness, width)] with the text queue.""" + out = [] + for level, a_open, a_wide in audio: + if level <= 0.0: + # Silence: let the queue wait rather than burning through it + # during a pause, or the mouth ends up ahead of the voice. + out.append((0.0, 0.0, 0.0)) + continue + + self._carry += hop / max(1e-3, self._step_seconds() * self._cur[1]) + while self._carry >= 1.0 and self._q: + self._cur = self._q.popleft() + self._carry -= 1.0 + if self._carry >= 1.0: + self._carry = 1.0 # queue empty — hold the last shape + + t_open, t_wide, closure = VISEMES.get(self._cur[0], VISEMES["REST"]) + if self._q or self._cur[0] != "REST": + # Text leads the shape; the audio keeps it honest so a bad + # transcript alignment still tracks the real voice. + o = 0.72 * t_open + 0.28 * a_open + w = 0.78 * t_wide + 0.22 * a_wide + else: + o, w = a_open, a_wide + closure = 0.0 + o *= 1.0 - closure + out.append((level, max(0.0, min(1.0, o)), max(-1.0, min(1.0, w)))) + return out diff --git a/core/wake_word.py b/core/wake_word.py index f864d6c..59ae065 100644 --- a/core/wake_word.py +++ b/core/wake_word.py @@ -1,202 +1,211 @@ -""" -Local wake-word detection for JARVIS ("Hey Jarvis"). - -Design goals: - • ZERO cost when the feature is off — openwakeword is imported ONLY inside - start()/install helpers, never at module load. If the user never enables - wake word, none of this touches the app. - • ZERO latency on the audio path — the microphone callback only ever does a - cheap, non-blocking queue push (feed()); the actual model inference runs in - this module's own background thread, so the real-time audio thread and the - Gemini stream are never slowed. - • Fully local & offline — audio fed here never leaves the machine; there is no - network call except the one-time model download the user triggers from the UI. - -openwakeword ships small ONNX models (a few MB each) and runs comfortably on a -CPU. The pretrained wake phrase used here is "Hey Jarvis". -""" -from __future__ import annotations - -import queue -import subprocess -import sys -import threading -from pathlib import Path -from typing import Callable - -# Pretrained openwakeword model that listens for "Hey Jarvis". -WAKE_MODEL = "hey_jarvis" -# Score in [0,1]; above this counts as a detection. Tunable per environment. -DEFAULT_THRESHOLD = 0.5 -# Mic frames arrive at 16 kHz int16; this is just the detector's input rate. -SAMPLE_RATE = 16000 - - -def is_installed() -> bool: - """True if the openwakeword package is importable (no model check).""" - try: - import importlib.util - return importlib.util.find_spec("openwakeword") is not None - except Exception: - return False - - -def is_ready() -> bool: - """True if openwakeword is installed AND its model files are present on disk. - - This is a cheap, DETERMINISTIC file-existence check. It deliberately does NOT - construct a Model to probe readiness — doing that is slow and, worse, can clash - with the detector's own Model when it's already running, which intermittently - returned False and made the UI flicker to 'not downloaded'. Never raises. - """ - if not is_installed(): - return False - try: - import openwakeword - models_dir = Path(openwakeword.__file__).resolve().parent / "resources" / "models" - if not models_dir.is_dir(): - return False - has_wake = (any(models_dir.glob(f"{WAKE_MODEL}*.onnx")) - or any(models_dir.glob(f"{WAKE_MODEL}*.tflite"))) - has_mel = (any(models_dir.glob("melspectrogram*.onnx")) - or any(models_dir.glob("melspectrogram*.tflite"))) - has_emb = (any(models_dir.glob("embedding_model*.onnx")) - or any(models_dir.glob("embedding_model*.tflite"))) - return bool(has_wake and has_mel and has_emb) - except Exception: - return False - - -def install_and_download(logger: Callable[[str], None] = print) -> tuple[bool, str]: - """ - One-click setup for the UI button: pip-install openwakeword if missing, then - download the wake model. Returns (ok, message). Never raises — every failure - is reported through the returned message and the logger. - """ - try: - if not is_installed(): - logger("Wake word: installing openwakeword (one-time)…") - r = subprocess.run( - [sys.executable, "-m", "pip", "install", "openwakeword"], - capture_output=True, text=True, - ) - if r.returncode != 0: - tail = (r.stderr or r.stdout or "").strip().splitlines()[-1:] or [""] - return False, f"pip install failed: {tail[0][:160]}" - # Download the pretrained melspectrogram/embedding + wake models. - logger("Wake word: downloading models…") - try: - import openwakeword.utils as _u - try: - _u.download_models([WAKE_MODEL]) - except TypeError: - _u.download_models() # older signature downloads the default set - except Exception as e: - return False, f"model download failed: {e}" - - if not is_ready(): - return False, "installed, but the wake model could not be loaded." - logger("Wake word: ready.") - return True, "Wake word installed and ready." - except Exception as e: - return False, f"setup error: {e}" - - -class WakeWordDetector: - """ - Runs the wake model in a dedicated thread. The mic thread calls feed() with - raw int16 frames; detections invoke on_detect() (called from this thread — - the callback must marshal to whatever loop/UI it needs). - """ - - def __init__(self, on_detect: Callable[[], None], - threshold: float = DEFAULT_THRESHOLD, - logger: Callable[[str], None] = print): - self._on_detect = on_detect - self._threshold = threshold - self._logger = logger - self._queue: queue.Queue = queue.Queue(maxsize=50) - self._thread: threading.Thread | None = None - self._running = False - self._model = None - self._ready = False - - def start(self) -> bool: - """Load the model and spawn the inference thread. Returns True on success. - Safe to call again — a no-op if already running. Never raises.""" - if self._running: - return True - try: - from openwakeword.model import Model - self._model = Model(wakeword_models=[WAKE_MODEL], inference_framework="onnx") - except Exception as e: - self._logger(f"Wake word: could not load model — {e}") - self._model = None - return False - self._running = True - self._ready = True - self._thread = threading.Thread(target=self._loop, daemon=True, name="WakeWordThread") - self._thread.start() - self._logger("Wake word: listening for 'Hey Jarvis'.") - return True - - def stop(self) -> None: - self._running = False - # unblock the thread if it's waiting on the queue - try: - self._queue.put_nowait(None) - except Exception: - pass - self._model = None - self._ready = False - - @property - def ready(self) -> bool: - return self._ready - - def feed(self, frame_int16) -> None: - """Called from the mic callback (real-time thread). Must stay cheap and - never block — the frame is copied and dropped if the queue is backed up.""" - if not self._running: - return - try: - # frame_int16 is a numpy int16 array (possibly 2-D mono) — flatten to 1-D - data = frame_int16[:, 0].copy() if getattr(frame_int16, "ndim", 1) > 1 else frame_int16.copy() - self._queue.put_nowait(data) - except queue.Full: - pass - except Exception: - pass - - def _loop(self) -> None: - import numpy as np - while self._running: - try: - frame = self._queue.get() - if frame is None or not self._running: - break - scores = self._model.predict(np.asarray(frame, dtype=np.int16)) - score = 0.0 - if isinstance(scores, dict): - # match the jarvis model regardless of exact key suffix - for k, v in scores.items(): - if "jarvis" in k.lower(): - score = max(score, float(v)) - if score == 0.0 and scores: - score = max(float(v) for v in scores.values()) - if score >= self._threshold: - # drain any backlog so we don't double-fire on the same utterance - self._drain() - try: - self._on_detect() - except Exception as e: - self._logger(f"Wake word: on_detect error — {e}") - except Exception as e: - self._logger(f"Wake word: inference error — {e}") - - def _drain(self) -> None: - try: - while True: - self._queue.get_nowait() - except Exception: - pass +""" +Local wake-word detection for JARVIS ("Hey Jarvis"). + +Design goals: + • ZERO cost when the feature is off — openwakeword is imported ONLY inside + start()/install helpers, never at module load. If the user never enables + wake word, none of this touches the app. + • ZERO latency on the audio path — the microphone callback only ever does a + cheap, non-blocking queue push (feed()); the actual model inference runs in + this module's own background thread, so the real-time audio thread and the + Gemini stream are never slowed. + • Fully local & offline — audio fed here never leaves the machine; there is no + network call except the one-time model download the user triggers from the UI. + +openwakeword ships small ONNX models (a few MB each) and runs comfortably on a +CPU. The pretrained wake phrase used here is "Hey Jarvis". +""" +from __future__ import annotations + +import queue +import subprocess +import sys +import threading +from pathlib import Path +from typing import Callable + +# Pretrained openwakeword model that listens for "Hey Jarvis". +WAKE_MODEL = "hey_jarvis" +# Score in [0,1]; above this counts as a detection. Tunable per environment. +DEFAULT_THRESHOLD = 0.5 +# Mic frames arrive at 16 kHz int16; this is just the detector's input rate. +SAMPLE_RATE = 16000 + + +def is_installed() -> bool: + """True if the openwakeword package is importable (no model check).""" + try: + import importlib.util + return importlib.util.find_spec("openwakeword") is not None + except Exception: + return False + + +def is_ready() -> bool: + """True if openwakeword is installed AND its model files are present on disk. + + This is a cheap, DETERMINISTIC file-existence check. It deliberately does NOT + construct a Model to probe readiness — doing that is slow and, worse, can clash + with the detector's own Model when it's already running, which intermittently + returned False and made the UI flicker to 'not downloaded'. Never raises. + """ + if not is_installed(): + return False + try: + import openwakeword + models_dir = Path(openwakeword.__file__).resolve().parent / "resources" / "models" + if not models_dir.is_dir(): + return False + has_wake = (any(models_dir.glob(f"{WAKE_MODEL}*.onnx")) + or any(models_dir.glob(f"{WAKE_MODEL}*.tflite"))) + has_mel = (any(models_dir.glob("melspectrogram*.onnx")) + or any(models_dir.glob("melspectrogram*.tflite"))) + has_emb = (any(models_dir.glob("embedding_model*.onnx")) + or any(models_dir.glob("embedding_model*.tflite"))) + return bool(has_wake and has_mel and has_emb) + except Exception: + return False + + +def install_and_download(logger: Callable[[str], None] = print, + notify: Callable[[str], None] | None = None) -> tuple[bool, str]: + """ + One-click setup for the UI button: pip-install openwakeword if missing, then + download the wake model. Returns (ok, message). Never raises — every failure + is reported through the returned message and the logger. + """ + _tell = notify or (lambda _msg: None) + try: + if not is_installed(): + logger("Wake word: installing openwakeword (one-time)…") + _tell("Wake word: installing openwakeword (one-time)…") + r = subprocess.run( + [sys.executable, "-m", "pip", "install", "openwakeword"], + capture_output=True, text=True, + ) + if r.returncode != 0: + tail = (r.stderr or r.stdout or "").strip().splitlines()[-1:] or [""] + return False, f"pip install failed: {tail[0][:160]}" + # Download the pretrained melspectrogram/embedding + wake models. + logger("Wake word: downloading models…") + _tell("Wake word: downloading models…") + try: + import openwakeword.utils as _u + try: + _u.download_models([WAKE_MODEL]) + except TypeError: + _u.download_models() # older signature downloads the default set + except Exception as e: + return False, f"model download failed: {e}" + + if not is_ready(): + return False, "installed, but the wake model could not be loaded." + logger("Wake word: ready.") + return True, "Wake word installed and ready." + except Exception as e: + return False, f"setup error: {e}" + + +class WakeWordDetector: + """ + Runs the wake model in a dedicated thread. The mic thread calls feed() with + raw int16 frames; detections invoke on_detect() (called from this thread — + the callback must marshal to whatever loop/UI it needs). + """ + + def __init__(self, on_detect: Callable[[], None], + threshold: float = DEFAULT_THRESHOLD, + logger: Callable[[str], None] = print, + notify: Callable[[str], None] | None = None): + self._on_detect = on_detect + self._threshold = threshold + self._logger = logger + # See PluginRegistry: `logger` is the console and gets everything, + # `notify` is the activity log and gets only what the user must act on. + self._notify = notify or (lambda _msg: None) + self._queue: queue.Queue = queue.Queue(maxsize=50) + self._thread: threading.Thread | None = None + self._running = False + self._model = None + self._ready = False + + def start(self) -> bool: + """Load the model and spawn the inference thread. Returns True on success. + Safe to call again — a no-op if already running. Never raises.""" + if self._running: + return True + try: + from openwakeword.model import Model + self._model = Model(wakeword_models=[WAKE_MODEL], inference_framework="onnx") + except Exception as e: + self._logger(f"Wake word: could not load model — {e}") + self._notify("Wake word unavailable — use the WAKE NOW button.") + self._model = None + return False + self._running = True + self._ready = True + self._thread = threading.Thread(target=self._loop, daemon=True, name="WakeWordThread") + self._thread.start() + self._logger("Wake word: listening for 'Hey Jarvis'.") + return True + + def stop(self) -> None: + self._running = False + # unblock the thread if it's waiting on the queue + try: + self._queue.put_nowait(None) + except Exception: + pass + self._model = None + self._ready = False + + @property + def ready(self) -> bool: + return self._ready + + def feed(self, frame_int16) -> None: + """Called from the mic callback (real-time thread). Must stay cheap and + never block — the frame is copied and dropped if the queue is backed up.""" + if not self._running: + return + try: + # frame_int16 is a numpy int16 array (possibly 2-D mono) — flatten to 1-D + data = frame_int16[:, 0].copy() if getattr(frame_int16, "ndim", 1) > 1 else frame_int16.copy() + self._queue.put_nowait(data) + except queue.Full: + pass + except Exception: + pass + + def _loop(self) -> None: + import numpy as np + while self._running: + try: + frame = self._queue.get() + if frame is None or not self._running: + break + scores = self._model.predict(np.asarray(frame, dtype=np.int16)) + score = 0.0 + if isinstance(scores, dict): + # match the jarvis model regardless of exact key suffix + for k, v in scores.items(): + if "jarvis" in k.lower(): + score = max(score, float(v)) + if score == 0.0 and scores: + score = max(float(v) for v in scores.values()) + if score >= self._threshold: + # drain any backlog so we don't double-fire on the same utterance + self._drain() + try: + self._on_detect() + except Exception as e: + self._logger(f"Wake word: on_detect error — {e}") + except Exception as e: + self._logger(f"Wake word: inference error — {e}") + + def _drain(self) -> None: + try: + while True: + self._queue.get_nowait() + except Exception: + pass diff --git a/dashboard/server.py b/dashboard/server.py index 00ffc58..d1430c6 100644 --- a/dashboard/server.py +++ b/dashboard/server.py @@ -1,794 +1,884 @@ -""" -dashboard/server.py — JARVIS Local HTTP Dashboard - -Plain HTTP on port 8000 (no SSL warnings, no firewall issues). -Security at the application layer: AES-256-CBC with session-key-derived key. -CryptoJS is auto-downloaded once and served locally — no CDN needed after that. - -Install deps: pip install fastapi "uvicorn[standard]" cryptography -""" - -import asyncio -import base64 -import hashlib -import re -import secrets -import socket -import string -import time -from pathlib import Path - -_DEPS_OK = False -try: - from fastapi import FastAPI, WebSocket, WebSocketDisconnect, Request - from fastapi.responses import HTMLResponse, JSONResponse, FileResponse - import uvicorn - _DEPS_OK = True -except ImportError: - pass - -# python-multipart is required for file uploads — optional dependency -_UPLOAD_OK = False -try: - from fastapi import UploadFile, File as FastAPIFile - _UPLOAD_OK = True -except Exception: - pass - -BASE_DIR = Path(__file__).resolve().parent.parent -STATIC_DIR = Path(__file__).parent / "static" -PORT = 8000 -MAX_UPLOAD_MB = 500 - - -def _make_uploads_dir() -> Path: - """Return (and create) the cross-platform uploads folder.""" - for candidate in [ - Path.home() / "Downloads" / "JARVIS Uploads", - Path.home() / "Documents" / "JARVIS Uploads", - BASE_DIR / "uploads", - ]: - try: - candidate.mkdir(parents=True, exist_ok=True) - return candidate - except Exception: - pass - return BASE_DIR / "uploads" - - -UPLOADS_DIR = _make_uploads_dir() - -def _get_gemini_key() -> str | None: - try: - import json as _json - with open(BASE_DIR / "config" / "api_keys.json", "r", encoding="utf-8") as f: - return _json.load(f).get("gemini_api_key") - except Exception: - return None - -_KEY_CHARS = [c for c in (string.ascii_uppercase + string.digits) - if c not in ('O', 'I', 'L', '0', '1')] - -# ── AES-256-CBC ─────────────────────────────────────────────────────────────── -_AES_SALT = b'JARVIS-DASHBOARD-v1' - - -def _derive_key(session_key: str) -> bytes: - """SHA-256(sessionKey‖salt) → 32-byte AES-256 key (microseconds, no PBKDF2 needed).""" - return hashlib.sha256(session_key.encode('utf-8') + _AES_SALT).digest() - - -def _decrypt_cbc(aes_key: bytes, enc_b64: str) -> str: - """Decrypt base64(IV[16] ‖ ciphertext) with AES-256-CBC + PKCS7.""" - from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes - from cryptography.hazmat.primitives import padding as sym_pad - raw = base64.b64decode(enc_b64) - iv, ct = raw[:16], raw[16:] - dec = Cipher(algorithms.AES(aes_key), modes.CBC(iv)).decryptor() - padded = dec.update(ct) + dec.finalize() - unpadder = sym_pad.PKCS7(128).unpadder() - return (unpadder.update(padded) + unpadder.finalize()).decode('utf-8') - - -# ── CryptoJS (auto-download once, served locally) ───────────────────────────── -_CRYPTOJS_CDN = ("https://cdnjs.cloudflare.com/ajax/libs/" - "crypto-js/4.2.0/crypto-js.min.js") -_CRYPTOJS_FILE = STATIC_DIR / "crypto-js.min.js" - - -def _ensure_network_access(port: int) -> None: - """Cross-platform, best-effort: open port in the OS firewall for LAN access. - - Runs in a background thread — never blocks uvicorn startup. - - Windows : writes a .bat file, runs it elevated via Windows ShellExecuteW - (native UAC dialog, guaranteed to appear). One-time setup. - macOS : osascript admin dialog if the Application Firewall is on. - Linux : pkexec GUI → sudo -n → prints manual command as fallback. - """ - import sys, subprocess, os, tempfile, threading - - # ── Windows ────────────────────────────────────────────────────────────── - if sys.platform == "win32": - import ctypes, time - - port_rule = f"JARVIS Dashboard Port {port}" - prog_rule = "JARVIS Dashboard Python" - py_exe = sys.executable - - def _netsh_rule_exists(name: str) -> bool: - try: - r = subprocess.run( - ["netsh", "advfirewall", "firewall", "show", "rule", f"name={name}"], - capture_output=True, text=True, timeout=5, - ) - return r.returncode == 0 and "No rules match" not in r.stdout - except Exception: - return False - - def _network_is_public() -> bool: - try: - r = subprocess.run( - ["powershell", "-NoProfile", "-NonInteractive", "-Command", - "(Get-NetConnectionProfile | " - "Where-Object {$_.NetworkCategory -eq 'Public'} | " - "Measure-Object).Count"], - capture_output=True, text=True, timeout=6, - ) - return r.stdout.strip() not in ("", "0") - except Exception: - return False - - need_port = not _netsh_rule_exists(port_rule) - need_prog = not _netsh_rule_exists(prog_rule) - need_private = _network_is_public() - - if not need_port and not need_prog and not need_private: - return # already fully configured - - # Build a .bat file — netsh + powershell, runs fast when elevated - bat_lines = ["@echo off"] - if need_private: - bat_lines.append( - 'powershell -NoProfile -NonInteractive -Command "' - 'Get-NetConnectionProfile | ' - "Where-Object {$_.NetworkCategory -eq 'Public'} | " - 'Set-NetConnectionProfile -NetworkCategory Private"' - ) - if need_port: - bat_lines.append( - f'netsh advfirewall firewall add rule ' - f'name="{port_rule}" protocol=TCP dir=in ' - f'localport={port} action=allow' - ) - if need_prog: - bat_lines.append( - f'netsh advfirewall firewall add rule ' - f'name="{prog_rule}" dir=in action=allow ' - f'program="{py_exe}" enable=yes' - ) - - bat_body = "\r\n".join(bat_lines) + "\r\n" - fd, bat_path = tempfile.mkstemp(suffix=".bat", prefix="jarvis_fw_") - try: - os.write(fd, bat_body.encode("mbcs")) # Windows cmd.exe expects ANSI - os.close(fd) - except Exception: - try: - os.close(fd) - except Exception: - pass - return - - # ── Try running directly (succeeds when already admin) ──────────────── - try: - r = subprocess.run( - [bat_path], capture_output=True, timeout=8, shell=True - ) - if r.returncode == 0: - print(f"[Dashboard] Firewall configured for port {port}.") - try: - os.unlink(bat_path) - except Exception: - pass - return - except Exception: - pass - - # ── ShellExecuteW: native UAC elevation (most reliable on Windows) ──── - # ShellExecuteW with verb "runas" always shows the UAC dialog regardless - # of UAC level settings. Non-blocking — uvicorn is already running. - print("[Dashboard] One-time network setup required.") - print("[Dashboard] >>> A Windows security dialog will appear — click 'Yes' <<<") - try: - ret = ctypes.windll.shell32.ShellExecuteW( - None, # hwnd (no parent window) - "runas", # verb (request elevation) - bat_path, # file (our .bat) - None, # params - None, # working dir - 0, # SW_HIDE (run without a visible cmd window) - ) - if int(ret) > 32: - # ShellExecuteW returns immediately; bat finishes in ~1 second. - # Sleep briefly so the rules are in place before the first retry. - time.sleep(2) - print(f"[Dashboard] Network setup complete — port {port} is open.") - print("[Dashboard] Refresh your phone browser to connect.") - else: - print("[Dashboard] Setup was not allowed.") - print("[Dashboard] Phone connections may fail until JARVIS is run as Administrator.") - except Exception as e: - print(f"[Dashboard] Firewall setup error: {e}") - finally: - # Cleanup after the bat has had time to run - def _cleanup(path: str) -> None: - time.sleep(5) - try: - os.unlink(path) - except Exception: - pass - threading.Thread(target=_cleanup, args=(bat_path,), daemon=True).start() - return - - # ── macOS ───────────────────────────────────────────────────────────────── - if sys.platform == "darwin": - fw_ctl = "/usr/libexec/ApplicationFirewall/socketfilterfw" - try: - r = subprocess.run( - [fw_ctl, "--getglobalstate"], capture_output=True, text=True, timeout=5, - ) - if "disabled" in r.stdout.lower(): - return # firewall off — nothing to do - - py = sys.executable - listed = subprocess.run( - [fw_ctl, "--listapps"], capture_output=True, text=True, timeout=5, - ) - if py in listed.stdout: - return # already allowed - - print("[Dashboard] One-time network setup — enter your password in the macOS dialog.") - subprocess.run( - ["osascript", "-e", - f'do shell script "{fw_ctl} --add {py} && {fw_ctl} --unblockapp {py}"' - f' with administrator privileges'], - timeout=60, - ) - except Exception: - pass # macOS firewall is off by default — silent failure is fine - return - - # ── Linux ───────────────────────────────────────────────────────────────── - def _privileged(cmd: list[str]) -> bool: - for prefix in (["pkexec"], ["sudo", "-n"]): - try: - r = subprocess.run(prefix + cmd, capture_output=True, timeout=30) - if r.returncode == 0: - return True - except Exception: - pass - return False - - try: # ufw - r = subprocess.run(["ufw", "status"], capture_output=True, text=True, timeout=5) - if "active" in r.stdout.lower(): - if _privileged(["ufw", "allow", f"{port}/tcp"]): - print(f"[Dashboard] ufw: port {port} allowed.") - else: - print(f"[Dashboard] Run manually: sudo ufw allow {port}/tcp") - return - except FileNotFoundError: - pass - - try: # firewalld - r = subprocess.run( - ["firewall-cmd", "--state"], capture_output=True, text=True, timeout=5, - ) - if "running" in r.stdout.lower(): - ok = (_privileged(["firewall-cmd", "--add-port", f"{port}/tcp", "--permanent"]) - and _privileged(["firewall-cmd", "--reload"])) - if ok: - print(f"[Dashboard] firewalld: port {port} allowed.") - else: - print(f"[Dashboard] Run manually: sudo firewall-cmd --add-port={port}/tcp --permanent && sudo firewall-cmd --reload") - return - except FileNotFoundError: - pass - - try: # iptables (not persistent but works until reboot) - r = subprocess.run(["iptables", "-L", "INPUT", "-n"], capture_output=True, timeout=5) - if r.returncode == 0: - if _privileged(["iptables", "-A", "INPUT", "-p", "tcp", "--dport", str(port), "-j", "ACCEPT"]): - print(f"[Dashboard] iptables: port {port} opened.") - else: - print(f"[Dashboard] Run manually: sudo iptables -A INPUT -p tcp --dport {port} -j ACCEPT") - except FileNotFoundError: - pass # no iptables means firewall is probably off — nothing to do - - -def _ensure_crypto_js() -> None: - if _CRYPTOJS_FILE.exists(): - return - try: - import urllib.request - print("[Dashboard] Downloading CryptoJS (one-time setup)…") - urllib.request.urlretrieve(_CRYPTOJS_CDN, str(_CRYPTOJS_FILE)) - print("[Dashboard] CryptoJS cached — will serve locally from now on.") - except Exception as e: - print(f"[Dashboard] CryptoJS download failed: {e}") - print(f"[Dashboard] Encryption will fall back to CDN load on client.") - - -_ensure_crypto_js() - - -# ── helpers ─────────────────────────────────────────────────────────────────── - -def _local_ip() -> str: - """Return the best LAN-facing IPv4 address, no internet required.""" - # Method 1: route trick (fast, works when internet is available) - for probe in ("8.8.8.8", "1.1.1.1", "192.168.1.1"): - try: - s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - s.settimeout(0.5) - s.connect((probe, 80)) - ip = s.getsockname()[0] - s.close() - if not ip.startswith("127."): - return ip - except Exception: - pass - - # Method 2: hostname resolution (works offline on most systems) - try: - ip = socket.gethostbyname(socket.gethostname()) - if not ip.startswith("127."): - return ip - except Exception: - pass - - # Method 3: enumerate all interfaces (fully offline, no external deps) - try: - for info in socket.getaddrinfo(socket.gethostname(), None, socket.AF_INET): - ip = info[4][0] - if not ip.startswith("127.") and not ip.startswith("169.254."): - return ip - except Exception: - pass - - return "127.0.0.1" - - -def _read(name: str) -> str: - return (STATIC_DIR / name).read_text(encoding="utf-8") - - -# ── DashboardServer ─────────────────────────────────────────────────────────── - -class DashboardServer: - - def __init__(self): - self._ip = _local_ip() - self._tokens: set[str] = set() - self._token_keys: dict[str, str] = {} # auth_token → session_key - self._aes_cache: dict[str, bytes]= {} # session_key → AES bytes - self._clients: set[WebSocket] = set() - self._history: list[dict] = [] - self._command_queue = asyncio.Queue() - self._wake_callback = None - self._connect_callback = None - self._pending_keys: dict[str, float] = {} - self._device_sessions: dict[str, dict] = {} # device_token → {session_key} - self._phone_audio_queue: asyncio.Queue = asyncio.Queue(maxsize=200) - self._uploads_dir = UPLOADS_DIR - self._login_html = _read("login.html") - self._app_html = _read("app.html") - self.app = self._build_app() - - # ── one-time key management ─────────────────────────────────────────── - - def new_key(self, expiry_secs: int = 600) -> str: - now = time.time() - self._pending_keys = {k: v for k, v in self._pending_keys.items() if v > now} - key = ''.join(secrets.choice(_KEY_CHARS) for _ in range(6)) - self._pending_keys[key] = now + expiry_secs - return key - - @staticmethod - def _ssl_enabled() -> bool: - certs = BASE_DIR / "config" / "certs" - return (certs / "jarvis.key").exists() and (certs / "jarvis.crt").exists() - - def get_url(self) -> str: - proto = "https" if self._ssl_enabled() else "http" - return f"{proto}://{self._ip}:{PORT}" - - def get_manual_url(self) -> str: - """URL for manual browser entry. When HTTPS active, points to alias port (also HTTPS).""" - if self._ssl_enabled(): - return f"{self._ip}:{PORT + 1}" - return f"{self._ip}:{PORT}" - - def _aes_key(self, session_key: str) -> bytes: - if session_key not in self._aes_cache: - self._aes_cache[session_key] = _derive_key(session_key) - return self._aes_cache[session_key] - - def _decrypt(self, token: str, enc_b64: str) -> str | None: - sk = self._token_keys.get(token) - if not sk: - return None - try: - return _decrypt_cbc(self._aes_key(sk), enc_b64) - except Exception: - return None - - # ── callbacks ──────────────────────────────────────────────────────── - - def set_wake_callback(self, fn) -> None: - self._wake_callback = fn - - def set_connect_callback(self, fn) -> None: - self._connect_callback = fn - - # ── broadcast ──────────────────────────────────────────────────────── - - async def broadcast(self, msg: dict) -> None: - self._history.append(msg) - if len(self._history) > 300: - self._history = self._history[-300:] - dead: set[WebSocket] = set() - for ws in list(self._clients): - try: - await ws.send_json(msg) - except Exception: - dead.add(ws) - self._clients -= dead - - # ── FastAPI app ─────────────────────────────────────────────────────── - - def _build_app(self) -> "FastAPI": - app = FastAPI(docs_url=None, redoc_url=None) - - def _auth(req: Request) -> bool: - tok = req.headers.get("authorization", "").removeprefix("Bearer ").strip() - return bool(tok) and tok in self._tokens - - # serve CryptoJS from local cache, fallback to CDN redirect - @app.get("/static/crypto.js") - async def serve_crypto(): - if _CRYPTOJS_FILE.exists(): - return FileResponse(str(_CRYPTOJS_FILE), - media_type="application/javascript") - from fastapi.responses import RedirectResponse - return RedirectResponse(_CRYPTOJS_CDN) - - @app.get("/login", response_class=HTMLResponse) - async def login_page(): - return HTMLResponse(self._login_html) - - @app.get("/", response_class=HTMLResponse) - async def index(): - # Auth is handled client-side via sessionStorage bearer token. - # Server-side header auth can't work here because browser navigations - # don't send custom headers (location.href doesn't carry Authorization). - html = (self._app_html - .replace("__IP__", self._ip) - .replace("__PORT__", str(PORT))) - return HTMLResponse(html) - - @app.post("/login") - async def login(req: Request): - body = await req.json() - entered = str(body.get("pin", "")).strip().upper() - now = time.time() - if entered in self._pending_keys and self._pending_keys[entered] > now: - del self._pending_keys[entered] # one-time use - tok = secrets.token_urlsafe(32) - self._tokens.add(tok) - self._token_keys[tok] = entered - self._aes_key(entered) # pre-derive & cache - if self._connect_callback: - self._connect_callback() - asyncio.create_task(self.broadcast( - {"type": "sys", "text": "Remote connection established."} - )) - # Bearer token in response body — no cookies needed (works on any browser/HTTP) - return JSONResponse({"ok": True, "token": tok}) - return JSONResponse({"ok": False, "error": "Invalid or expired key"}, - status_code=401) - - @app.get("/auto-login") - async def auto_login(key: str = ""): - """QR code target — validates one-time key, creates session, redirects phone.""" - now = time.time() - if not key or key not in self._pending_keys or self._pending_keys[key] <= now: - return HTMLResponse(""" - - -

Link Expired

-

Press Remote Control in JARVIS to get a new QR code.

-
""") - - del self._pending_keys[key] - tok = secrets.token_urlsafe(32) - dev_tok = secrets.token_urlsafe(32) - self._tokens.add(tok) - self._token_keys[tok] = key - self._aes_key(key) - self._device_sessions[dev_tok] = {"session_key": key} - - if self._connect_callback: - self._connect_callback() - asyncio.create_task(self.broadcast( - {"type": "sys", "text": "Remote connection established via QR code."} - )) - - return HTMLResponse(f""" - - - - -

Connecting to JARVIS…

-""") - - @app.post("/api/device-login") - async def device_login_ep(req: Request): - """Return a fresh auth token for a previously paired device token.""" - try: - body = await req.json() - except Exception: - return JSONResponse({"ok": False}, status_code=400) - dev_tok = (body.get("device_token") or "").strip() - if not dev_tok or dev_tok not in self._device_sessions: - return JSONResponse({"ok": False}, status_code=401) - session_key = self._device_sessions[dev_tok]["session_key"] - tok = secrets.token_urlsafe(32) - self._tokens.add(tok) - self._token_keys[tok] = session_key - self._aes_key(session_key) - if self._connect_callback: - self._connect_callback() - asyncio.create_task(self.broadcast( - {"type": "sys", "text": "Known device reconnected automatically."} - )) - return JSONResponse({"ok": True, "token": tok, "key": session_key}) - - @app.post("/api/revoke-devices") - async def revoke_devices(req: Request): - """Invalidate all persistent device tokens (admin action).""" - if not _auth(req): - return JSONResponse({"error": "Unauthorized"}, status_code=401) - count = len(self._device_sessions) - self._device_sessions.clear() - return JSONResponse({"ok": True, "revoked": count}) - - @app.post("/api/command") - async def command(req: Request): - if not _auth(req): - return JSONResponse({"error": "Unauthorized"}, status_code=401) - body = await req.json() - token = req.headers.get("authorization", "").removeprefix("Bearer ").strip() - enc = body.get("enc", "") - if enc: - text = self._decrypt(token, enc) - if text is None: - return JSONResponse({"error": "Decryption failed"}, status_code=400) - else: - text = (body.get("text") or "").strip() - if text: - await self._command_queue.put(text) - if self._wake_callback: - self._wake_callback() - return JSONResponse({"ok": True}) - - @app.post("/api/wake") - async def wake_ep(req: Request): - if not _auth(req): - return JSONResponse({"error": "Unauthorized"}, status_code=401) - if self._wake_callback: - self._wake_callback() - return JSONResponse({"ok": True}) - - # ── Phone mic real-time audio → Gemini Live ────────────────────────── - - @app.websocket("/ws/phone-audio") - async def phone_audio_ws(websocket: WebSocket, token: str = ""): - tok = token.strip() - if not tok or tok not in self._tokens: - await websocket.close(code=4001) - return - await websocket.accept() - asyncio.create_task(self.broadcast( - {"type": "sys", "text": "Phone microphone live."} - )) - try: - while True: - data = await websocket.receive_bytes() - try: - self._phone_audio_queue.put_nowait( - {"data": data, "mime_type": "audio/pcm"} - ) - except asyncio.QueueFull: - pass # drop frame rather than block - except WebSocketDisconnect: - pass - finally: - asyncio.create_task(self.broadcast( - {"type": "sys", "text": "Phone microphone stopped."} - )) - - # ── File sharing ────────────────────────────────────────────────────── - - def _safe_filename(raw: str) -> str: - name = Path(raw).name # strip path components - name = re.sub(r'[<>:"/\\|?*\x00-\x1f]', '_', name).strip(". ") - return name or "upload" - - if _UPLOAD_OK: - @app.post("/api/upload") - async def upload_file(req: Request, file: UploadFile = FastAPIFile(...)): - if not _auth(req): - return JSONResponse({"error": "Unauthorized"}, status_code=401) - - safe = _safe_filename(file.filename or "upload") - dest = self._uploads_dir / safe - stem, suffix = Path(safe).stem, Path(safe).suffix - counter = 1 - while dest.exists(): - dest = self._uploads_dir / f"{stem}_{counter}{suffix}" - counter += 1 - - size = 0 - max_bytes = MAX_UPLOAD_MB * 1024 * 1024 - try: - with open(dest, "wb") as fout: - while True: - chunk = await file.read(65536) - if not chunk: - break - size += len(chunk) - if size > max_bytes: - fout.close() - dest.unlink(missing_ok=True) - return JSONResponse( - {"error": f"File too large (max {MAX_UPLOAD_MB} MB)"}, - status_code=413, - ) - fout.write(chunk) - except Exception as exc: - try: - dest.unlink(missing_ok=True) - except Exception: - pass - return JSONResponse({"error": str(exc)}, status_code=500) - - asyncio.create_task(self.broadcast({ - "type": "file_received", - "name": dest.name, - "size": size, - "saved_to": str(self._uploads_dir), - })) - return JSONResponse({"ok": True, "name": dest.name, "size": size}) - else: - @app.post("/api/upload") - async def upload_unavailable(req: Request): - return JSONResponse( - {"error": "File uploads require: pip install python-multipart"}, - status_code=503, - ) - - @app.get("/api/files") - async def list_files(req: Request): - if not _auth(req): - return JSONResponse({"error": "Unauthorized"}, status_code=401) - files = [] - try: - for f in sorted( - (p for p in self._uploads_dir.iterdir() if p.is_file()), - key=lambda p: p.stat().st_mtime, - reverse=True, - ): - files.append({"name": f.name, "size": f.stat().st_size}) - except Exception: - pass - return JSONResponse({"files": files}) - - @app.get("/uploads/{filename}") - async def download_file(filename: str, token: str = ""): - # Auth via query param — browser can't send custom headers - tok = token.strip() - if not tok or tok not in self._tokens: - return JSONResponse({"error": "Unauthorized"}, status_code=401) - safe = re.sub(r'[/\\]', '', filename) - path = self._uploads_dir / safe - if not path.exists() or not path.is_file(): - return JSONResponse({"error": "Not found"}, status_code=404) - return FileResponse(str(path), filename=safe) - - @app.websocket("/ws") - async def ws_ep(websocket: WebSocket, token: str = ""): - tok = token.strip() - if not tok or tok not in self._tokens: - await websocket.close(code=4001) - return - await websocket.accept() - self._clients.add(websocket) - for entry in self._history[-50:]: - try: - await websocket.send_json(entry) - except Exception: - break - try: - while True: - data = await websocket.receive_json() - if data.get("type") == "command": - enc = data.get("enc", "") - t = self._decrypt(tok, enc) if enc else (data.get("text") or "").strip() - if t: - await self._command_queue.put(t) - if self._wake_callback: - self._wake_callback() - except WebSocketDisconnect: - pass - finally: - self._clients.discard(websocket) - - return app - - # ── serve ───────────────────────────────────────────────────────────── - - async def _serve_alias(self) -> None: - """Second HTTPS server on PORT+1 sharing the same app and in-memory state. - Chrome HTTPS-upgrades any bare IP:PORT the user types, so this port also needs TLS. - User types IP:8001 → Chrome tries https → self-signed cert warning → accept once → done.""" - ssl_key = BASE_DIR / "config" / "certs" / "jarvis.key" - ssl_cert = BASE_DIR / "config" / "certs" / "jarvis.crt" - asyncio.get_event_loop().run_in_executor(None, _ensure_network_access, PORT + 1) - cfg = uvicorn.Config( - self.app, host="0.0.0.0", port=PORT + 1, log_level="warning", - ssl_keyfile=str(ssl_key), ssl_certfile=str(ssl_cert), - ) - print(f"[Dashboard] Manual entry: {self._ip}:{PORT + 1} (type in browser, accept cert once)") - await uvicorn.Server(cfg).serve() - - async def serve(self) -> None: - if not _DEPS_OK: - print("[Dashboard] fastapi/uvicorn not installed — dashboard disabled.") - print("[Dashboard] Run: pip install fastapi 'uvicorn[standard]' cryptography") - return - - # Firewall setup runs in a thread — uvicorn starts immediately, - # no waiting for UAC dialogs or subprocess timeouts. - asyncio.get_event_loop().run_in_executor(None, _ensure_network_access, PORT) - - use_ssl = self._ssl_enabled() - ssl_key = BASE_DIR / "config" / "certs" / "jarvis.key" - ssl_cert = BASE_DIR / "config" / "certs" / "jarvis.crt" - - if use_ssl: - asyncio.create_task(self._serve_alias()) - - cfg = uvicorn.Config( - self.app, host="0.0.0.0", port=PORT, log_level="warning", - **({"ssl_keyfile": str(ssl_key), "ssl_certfile": str(ssl_cert)} if use_ssl else {}), - ) - - proto = "https" if use_ssl else "http" - print(f"[Dashboard] {proto}://{self._ip}:{PORT}") - print("[Dashboard] Press 'Remote Control' in JARVIS UI to get the QR code.") - await uvicorn.Server(cfg).serve() +""" +dashboard/server.py — JARVIS Local HTTP Dashboard + +Plain HTTP on port 8000 (no SSL warnings, no firewall issues). +Security at the application layer: AES-256-CBC with session-key-derived key. +CryptoJS is auto-downloaded once and served locally — no CDN needed after that. + +Install deps: pip install fastapi "uvicorn[standard]" cryptography +""" + +import asyncio +import base64 +import hashlib +import re +import secrets +import socket +import string +import time +from pathlib import Path + +_DEPS_OK = False +try: + from fastapi import FastAPI, WebSocket, WebSocketDisconnect, Request + from fastapi.responses import HTMLResponse, JSONResponse, FileResponse + import uvicorn + _DEPS_OK = True +except ImportError: + pass + +# python-multipart is required for file uploads — optional dependency +_UPLOAD_OK = False +try: + from fastapi import UploadFile, File as FastAPIFile + _UPLOAD_OK = True +except Exception: + pass + +BASE_DIR = Path(__file__).resolve().parent.parent +STATIC_DIR = Path(__file__).parent / "static" +PORT = 8000 +MAX_UPLOAD_MB = 500 + + +def _make_uploads_dir() -> Path: + """Return (and create) the cross-platform uploads folder.""" + for candidate in [ + Path.home() / "Downloads" / "JARVIS Uploads", + Path.home() / "Documents" / "JARVIS Uploads", + BASE_DIR / "uploads", + ]: + try: + candidate.mkdir(parents=True, exist_ok=True) + return candidate + except Exception: + pass + return BASE_DIR / "uploads" + + +UPLOADS_DIR = _make_uploads_dir() + +def _get_gemini_key() -> str | None: + try: + import json as _json + with open(BASE_DIR / "config" / "api_keys.json", "r", encoding="utf-8") as f: + return _json.load(f).get("gemini_api_key") + except Exception: + return None + +_KEY_CHARS = [c for c in (string.ascii_uppercase + string.digits) + if c not in ('O', 'I', 'L', '0', '1')] + +# ── AES-256-CBC ─────────────────────────────────────────────────────────────── +_AES_SALT = b'JARVIS-DASHBOARD-v1' + + +def _derive_key(session_key: str) -> bytes: + """SHA-256(sessionKey‖salt) → 32-byte AES-256 key (microseconds, no PBKDF2 needed).""" + return hashlib.sha256(session_key.encode('utf-8') + _AES_SALT).digest() + + +def _decrypt_cbc(aes_key: bytes, enc_b64: str) -> str: + """Decrypt base64(IV[16] ‖ ciphertext) with AES-256-CBC + PKCS7.""" + from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes + from cryptography.hazmat.primitives import padding as sym_pad + raw = base64.b64decode(enc_b64) + iv, ct = raw[:16], raw[16:] + dec = Cipher(algorithms.AES(aes_key), modes.CBC(iv)).decryptor() + padded = dec.update(ct) + dec.finalize() + unpadder = sym_pad.PKCS7(128).unpadder() + return (unpadder.update(padded) + unpadder.finalize()).decode('utf-8') + + +# ── CryptoJS (auto-download once, served locally) ───────────────────────────── +_CRYPTOJS_CDN = ("https://cdnjs.cloudflare.com/ajax/libs/" + "crypto-js/4.2.0/crypto-js.min.js") +_CRYPTOJS_FILE = STATIC_DIR / "crypto-js.min.js" + + +def _ensure_network_access(port: int) -> None: + """Cross-platform, best-effort: open port in the OS firewall for LAN access. + + Runs in a background thread — never blocks uvicorn startup. + + Windows : writes a .bat file, runs it elevated via Windows ShellExecuteW + (native UAC dialog, guaranteed to appear). One-time setup. + macOS : osascript admin dialog if the Application Firewall is on. + Linux : pkexec GUI → sudo -n → prints manual command as fallback. + """ + import sys, subprocess, os, tempfile, threading + + # ── Windows ────────────────────────────────────────────────────────────── + if sys.platform == "win32": + import ctypes, time + + port_rule = f"JARVIS Dashboard Port {port}" + prog_rule = "JARVIS Dashboard Python" + py_exe = sys.executable + + def _netsh_rule_exists(name: str) -> bool: + try: + r = subprocess.run( + ["netsh", "advfirewall", "firewall", "show", "rule", f"name={name}"], + capture_output=True, text=True, timeout=5, + ) + return r.returncode == 0 and "No rules match" not in r.stdout + except Exception: + return False + + def _network_is_public() -> bool: + try: + r = subprocess.run( + ["powershell", "-NoProfile", "-NonInteractive", "-Command", + "(Get-NetConnectionProfile | " + "Where-Object {$_.NetworkCategory -eq 'Public'} | " + "Measure-Object).Count"], + capture_output=True, text=True, timeout=6, + ) + return r.stdout.strip() not in ("", "0") + except Exception: + return False + + need_port = not _netsh_rule_exists(port_rule) + need_prog = not _netsh_rule_exists(prog_rule) + need_private = _network_is_public() + + if not need_port and not need_prog and not need_private: + return # already fully configured + + # Build a .bat file — netsh + powershell, runs fast when elevated + bat_lines = ["@echo off"] + if need_private: + bat_lines.append( + 'powershell -NoProfile -NonInteractive -Command "' + 'Get-NetConnectionProfile | ' + "Where-Object {$_.NetworkCategory -eq 'Public'} | " + 'Set-NetConnectionProfile -NetworkCategory Private"' + ) + if need_port: + bat_lines.append( + f'netsh advfirewall firewall add rule ' + f'name="{port_rule}" protocol=TCP dir=in ' + f'localport={port} action=allow' + ) + if need_prog: + bat_lines.append( + f'netsh advfirewall firewall add rule ' + f'name="{prog_rule}" dir=in action=allow ' + f'program="{py_exe}" enable=yes' + ) + + bat_body = "\r\n".join(bat_lines) + "\r\n" + fd, bat_path = tempfile.mkstemp(suffix=".bat", prefix="jarvis_fw_") + try: + os.write(fd, bat_body.encode("mbcs")) # Windows cmd.exe expects ANSI + os.close(fd) + except Exception: + try: + os.close(fd) + except Exception: + pass + return + + # ── Try running directly (succeeds when already admin) ──────────────── + try: + r = subprocess.run( + [bat_path], capture_output=True, timeout=8, shell=True + ) + if r.returncode == 0: + print(f"[Dashboard] Firewall configured for port {port}.") + try: + os.unlink(bat_path) + except Exception: + pass + return + except Exception: + pass + + # ── ShellExecuteW: native UAC elevation (most reliable on Windows) ──── + # ShellExecuteW with verb "runas" always shows the UAC dialog regardless + # of UAC level settings. Non-blocking — uvicorn is already running. + print("[Dashboard] One-time network setup required.") + print("[Dashboard] >>> A Windows security dialog will appear — click 'Yes' <<<") + try: + ret = ctypes.windll.shell32.ShellExecuteW( + None, # hwnd (no parent window) + "runas", # verb (request elevation) + bat_path, # file (our .bat) + None, # params + None, # working dir + 0, # SW_HIDE (run without a visible cmd window) + ) + if int(ret) > 32: + # ShellExecuteW returns immediately; bat finishes in ~1 second. + # Sleep briefly so the rules are in place before the first retry. + time.sleep(2) + print(f"[Dashboard] Network setup complete — port {port} is open.") + print("[Dashboard] Refresh your phone browser to connect.") + else: + print("[Dashboard] Setup was not allowed.") + print("[Dashboard] Phone connections may fail until JARVIS is run as Administrator.") + except Exception as e: + print(f"[Dashboard] Firewall setup error: {e}") + finally: + # Cleanup after the bat has had time to run + def _cleanup(path: str) -> None: + time.sleep(5) + try: + os.unlink(path) + except Exception: + pass + threading.Thread(target=_cleanup, args=(bat_path,), daemon=True).start() + return + + # ── macOS ───────────────────────────────────────────────────────────────── + if sys.platform == "darwin": + fw_ctl = "/usr/libexec/ApplicationFirewall/socketfilterfw" + try: + r = subprocess.run( + [fw_ctl, "--getglobalstate"], capture_output=True, text=True, timeout=5, + ) + if "disabled" in r.stdout.lower(): + return # firewall off — nothing to do + + py = sys.executable + listed = subprocess.run( + [fw_ctl, "--listapps"], capture_output=True, text=True, timeout=5, + ) + if py in listed.stdout: + return # already allowed + + print("[Dashboard] One-time network setup — enter your password in the macOS dialog.") + subprocess.run( + ["osascript", "-e", + f'do shell script "{fw_ctl} --add {py} && {fw_ctl} --unblockapp {py}"' + f' with administrator privileges'], + timeout=60, + ) + except Exception: + pass # macOS firewall is off by default — silent failure is fine + return + + # ── Linux ───────────────────────────────────────────────────────────────── + def _privileged(cmd: list[str]) -> bool: + for prefix in (["pkexec"], ["sudo", "-n"]): + try: + r = subprocess.run(prefix + cmd, capture_output=True, timeout=30) + if r.returncode == 0: + return True + except Exception: + pass + return False + + try: # ufw + r = subprocess.run(["ufw", "status"], capture_output=True, text=True, timeout=5) + if "active" in r.stdout.lower(): + if _privileged(["ufw", "allow", f"{port}/tcp"]): + print(f"[Dashboard] ufw: port {port} allowed.") + else: + print(f"[Dashboard] Run manually: sudo ufw allow {port}/tcp") + return + except FileNotFoundError: + pass + + try: # firewalld + r = subprocess.run( + ["firewall-cmd", "--state"], capture_output=True, text=True, timeout=5, + ) + if "running" in r.stdout.lower(): + ok = (_privileged(["firewall-cmd", "--add-port", f"{port}/tcp", "--permanent"]) + and _privileged(["firewall-cmd", "--reload"])) + if ok: + print(f"[Dashboard] firewalld: port {port} allowed.") + else: + print(f"[Dashboard] Run manually: sudo firewall-cmd --add-port={port}/tcp --permanent && sudo firewall-cmd --reload") + return + except FileNotFoundError: + pass + + try: # iptables (not persistent but works until reboot) + r = subprocess.run(["iptables", "-L", "INPUT", "-n"], capture_output=True, timeout=5) + if r.returncode == 0: + if _privileged(["iptables", "-A", "INPUT", "-p", "tcp", "--dport", str(port), "-j", "ACCEPT"]): + print(f"[Dashboard] iptables: port {port} opened.") + else: + print(f"[Dashboard] Run manually: sudo iptables -A INPUT -p tcp --dport {port} -j ACCEPT") + except FileNotFoundError: + pass # no iptables means firewall is probably off — nothing to do + + +def _ensure_crypto_js() -> None: + if _CRYPTOJS_FILE.exists(): + return + try: + import urllib.request + print("[Dashboard] Downloading CryptoJS (one-time setup)…") + urllib.request.urlretrieve(_CRYPTOJS_CDN, str(_CRYPTOJS_FILE)) + print("[Dashboard] CryptoJS cached — will serve locally from now on.") + except Exception as e: + print(f"[Dashboard] CryptoJS download failed: {e}") + print(f"[Dashboard] Encryption will fall back to CDN load on client.") + + +_ensure_crypto_js() + + +# ── helpers ─────────────────────────────────────────────────────────────────── + +def _local_ip() -> str: + """Return the best LAN-facing IPv4 address, no internet required.""" + # Method 1: route trick (fast, works when internet is available) + for probe in ("8.8.8.8", "1.1.1.1", "192.168.1.1"): + try: + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + s.settimeout(0.5) + s.connect((probe, 80)) + ip = s.getsockname()[0] + s.close() + if not ip.startswith("127."): + return ip + except Exception: + pass + + # Method 2: hostname resolution (works offline on most systems) + try: + ip = socket.gethostbyname(socket.gethostname()) + if not ip.startswith("127."): + return ip + except Exception: + pass + + # Method 3: enumerate all interfaces (fully offline, no external deps) + try: + for info in socket.getaddrinfo(socket.gethostname(), None, socket.AF_INET): + ip = info[4][0] + if not ip.startswith("127.") and not ip.startswith("169.254."): + return ip + except Exception: + pass + + return "127.0.0.1" + + +def _ensure_certs() -> bool: + """ + Make sure config/certs holds a TLS key pair, generating a self-signed one the + first time the dashboard runs. + + The pair is deliberately NOT shipped in the repository. A private key that + every user downloads is the same as having no private key at all: anyone can + present a certificate that matches it. Generating locally gives each install + its own key, costs about a second, and happens exactly once. + + Returns True when a usable pair exists afterwards; False leaves the caller on + plain HTTP, which still works — the QR code simply encodes http:// instead. + """ + certs = BASE_DIR / "config" / "certs" + key_p = certs / "jarvis.key" + crt_p = certs / "jarvis.crt" + if key_p.exists() and crt_p.exists(): + return True + + try: + import datetime + import ipaddress + from cryptography import x509 + from cryptography.hazmat.primitives import hashes, serialization + from cryptography.hazmat.primitives.asymmetric import rsa + from cryptography.x509.oid import NameOID + except ImportError: + print("[Dashboard] cryptography not installed — serving over plain HTTP.") + print("[Dashboard] For HTTPS run: pip install cryptography") + return False + + try: + certs.mkdir(parents=True, exist_ok=True) + key = rsa.generate_private_key(public_exponent=65537, key_size=2048) + + who = x509.Name([ + x509.NameAttribute(NameOID.COMMON_NAME, "JARVIS Dashboard"), + x509.NameAttribute(NameOID.ORGANIZATION_NAME, "JARVIS"), + ]) + + # The SAN has to cover every address the phone might use: the LAN IP the + # QR code encodes, plus localhost when testing on the machine itself. + alt = [x509.DNSName("localhost"), + x509.IPAddress(ipaddress.IPv4Address("127.0.0.1"))] + try: + lan = _local_ip() + if not lan.startswith("127."): + alt.append(x509.IPAddress(ipaddress.IPv4Address(lan))) + except Exception: + pass # no LAN address resolvable — localhost entries still work + + # Timezone-aware UTC: datetime.utcnow() is deprecated from Python 3.12 on, + # and the builder normalises aware values to UTC itself. + now = datetime.datetime.now(datetime.timezone.utc) + cert = ( + x509.CertificateBuilder() + .subject_name(who) + .issuer_name(who) + .public_key(key.public_key()) + .serial_number(x509.random_serial_number()) + .not_valid_before(now - datetime.timedelta(days=1)) + .not_valid_after(now + datetime.timedelta(days=3650)) + .add_extension(x509.SubjectAlternativeName(alt), critical=False) + .add_extension(x509.BasicConstraints(ca=False, path_length=None), critical=True) + .sign(key, hashes.SHA256()) + ) + + key_p.write_bytes(key.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.TraditionalOpenSSL, + encryption_algorithm=serialization.NoEncryption(), + )) + crt_p.write_bytes(cert.public_bytes(serialization.Encoding.PEM)) + + try: + import os as _os + _os.chmod(key_p, 0o600) # best effort — largely a no-op on Windows + except Exception: + pass + + print(f"[Dashboard] Generated a self-signed certificate for this machine: {certs}") + return True + except Exception as e: + print(f"[Dashboard] Certificate generation failed ({e}) — serving over plain HTTP.") + return False + + +def _read(name: str) -> str: + return (STATIC_DIR / name).read_text(encoding="utf-8") + + +# ── DashboardServer ─────────────────────────────────────────────────────────── + +class DashboardServer: + + def __init__(self): + self._ip = _local_ip() + self._tokens: set[str] = set() + self._token_keys: dict[str, str] = {} # auth_token → session_key + self._aes_cache: dict[str, bytes]= {} # session_key → AES bytes + self._clients: set[WebSocket] = set() + self._history: list[dict] = [] + self._command_queue = asyncio.Queue() + self._wake_callback = None + self._connect_callback = None + self._pending_keys: dict[str, float] = {} + self._device_sessions: dict[str, dict] = {} # device_token → {session_key} + self._phone_audio_queue: asyncio.Queue = asyncio.Queue(maxsize=200) + self._uploads_dir = UPLOADS_DIR + self._login_html = _read("login.html") + self._app_html = _read("app.html") + self.app = self._build_app() + + # ── one-time key management ─────────────────────────────────────────── + + def new_key(self, expiry_secs: int = 600) -> str: + now = time.time() + self._pending_keys = {k: v for k, v in self._pending_keys.items() if v > now} + key = ''.join(secrets.choice(_KEY_CHARS) for _ in range(6)) + self._pending_keys[key] = now + expiry_secs + return key + + @staticmethod + def _ssl_enabled() -> bool: + certs = BASE_DIR / "config" / "certs" + return (certs / "jarvis.key").exists() and (certs / "jarvis.crt").exists() + + def get_url(self) -> str: + proto = "https" if self._ssl_enabled() else "http" + return f"{proto}://{self._ip}:{PORT}" + + def get_manual_url(self) -> str: + """URL for manual browser entry. When HTTPS active, points to alias port (also HTTPS).""" + if self._ssl_enabled(): + return f"{self._ip}:{PORT + 1}" + return f"{self._ip}:{PORT}" + + def _aes_key(self, session_key: str) -> bytes: + if session_key not in self._aes_cache: + self._aes_cache[session_key] = _derive_key(session_key) + return self._aes_cache[session_key] + + def _decrypt(self, token: str, enc_b64: str) -> str | None: + sk = self._token_keys.get(token) + if not sk: + return None + try: + return _decrypt_cbc(self._aes_key(sk), enc_b64) + except Exception: + return None + + # ── callbacks ──────────────────────────────────────────────────────── + + def set_wake_callback(self, fn) -> None: + self._wake_callback = fn + + def set_connect_callback(self, fn) -> None: + self._connect_callback = fn + + # ── broadcast ──────────────────────────────────────────────────────── + + async def broadcast(self, msg: dict) -> None: + self._history.append(msg) + if len(self._history) > 300: + self._history = self._history[-300:] + dead: set[WebSocket] = set() + for ws in list(self._clients): + try: + await ws.send_json(msg) + except Exception: + dead.add(ws) + self._clients -= dead + + # ── FastAPI app ─────────────────────────────────────────────────────── + + def _build_app(self) -> "FastAPI": + app = FastAPI(docs_url=None, redoc_url=None) + + def _auth(req: Request) -> bool: + tok = req.headers.get("authorization", "").removeprefix("Bearer ").strip() + return bool(tok) and tok in self._tokens + + # serve CryptoJS from local cache, fallback to CDN redirect + @app.get("/static/crypto.js") + async def serve_crypto(): + if _CRYPTOJS_FILE.exists(): + return FileResponse(str(_CRYPTOJS_FILE), + media_type="application/javascript") + from fastapi.responses import RedirectResponse + return RedirectResponse(_CRYPTOJS_CDN) + + @app.get("/login", response_class=HTMLResponse) + async def login_page(): + return HTMLResponse(self._login_html) + + @app.get("/", response_class=HTMLResponse) + async def index(): + # Auth is handled client-side via sessionStorage bearer token. + # Server-side header auth can't work here because browser navigations + # don't send custom headers (location.href doesn't carry Authorization). + html = (self._app_html + .replace("__IP__", self._ip) + .replace("__PORT__", str(PORT))) + return HTMLResponse(html) + + @app.post("/login") + async def login(req: Request): + body = await req.json() + entered = str(body.get("pin", "")).strip().upper() + now = time.time() + if entered in self._pending_keys and self._pending_keys[entered] > now: + del self._pending_keys[entered] # one-time use + tok = secrets.token_urlsafe(32) + self._tokens.add(tok) + self._token_keys[tok] = entered + self._aes_key(entered) # pre-derive & cache + if self._connect_callback: + self._connect_callback() + asyncio.create_task(self.broadcast( + {"type": "sys", "text": "Remote connection established."} + )) + # Bearer token in response body — no cookies needed (works on any browser/HTTP) + return JSONResponse({"ok": True, "token": tok}) + return JSONResponse({"ok": False, "error": "Invalid or expired key"}, + status_code=401) + + @app.get("/auto-login") + async def auto_login(key: str = ""): + """QR code target — validates one-time key, creates session, redirects phone.""" + now = time.time() + if not key or key not in self._pending_keys or self._pending_keys[key] <= now: + return HTMLResponse(""" + + +

Link Expired

+

Press Remote Control in JARVIS to get a new QR code.

+
""") + + del self._pending_keys[key] + tok = secrets.token_urlsafe(32) + dev_tok = secrets.token_urlsafe(32) + self._tokens.add(tok) + self._token_keys[tok] = key + self._aes_key(key) + self._device_sessions[dev_tok] = {"session_key": key} + + if self._connect_callback: + self._connect_callback() + asyncio.create_task(self.broadcast( + {"type": "sys", "text": "Remote connection established via QR code."} + )) + + return HTMLResponse(f""" + + + + +

Connecting to JARVIS…

+""") + + @app.post("/api/device-login") + async def device_login_ep(req: Request): + """Return a fresh auth token for a previously paired device token.""" + try: + body = await req.json() + except Exception: + return JSONResponse({"ok": False}, status_code=400) + dev_tok = (body.get("device_token") or "").strip() + if not dev_tok or dev_tok not in self._device_sessions: + return JSONResponse({"ok": False}, status_code=401) + session_key = self._device_sessions[dev_tok]["session_key"] + tok = secrets.token_urlsafe(32) + self._tokens.add(tok) + self._token_keys[tok] = session_key + self._aes_key(session_key) + if self._connect_callback: + self._connect_callback() + asyncio.create_task(self.broadcast( + {"type": "sys", "text": "Known device reconnected automatically."} + )) + return JSONResponse({"ok": True, "token": tok, "key": session_key}) + + @app.post("/api/revoke-devices") + async def revoke_devices(req: Request): + """Invalidate all persistent device tokens (admin action).""" + if not _auth(req): + return JSONResponse({"error": "Unauthorized"}, status_code=401) + count = len(self._device_sessions) + self._device_sessions.clear() + return JSONResponse({"ok": True, "revoked": count}) + + @app.post("/api/command") + async def command(req: Request): + if not _auth(req): + return JSONResponse({"error": "Unauthorized"}, status_code=401) + body = await req.json() + token = req.headers.get("authorization", "").removeprefix("Bearer ").strip() + enc = body.get("enc", "") + if enc: + text = self._decrypt(token, enc) + if text is None: + return JSONResponse({"error": "Decryption failed"}, status_code=400) + else: + text = (body.get("text") or "").strip() + if text: + await self._command_queue.put(text) + if self._wake_callback: + self._wake_callback() + return JSONResponse({"ok": True}) + + @app.post("/api/wake") + async def wake_ep(req: Request): + if not _auth(req): + return JSONResponse({"error": "Unauthorized"}, status_code=401) + if self._wake_callback: + self._wake_callback() + return JSONResponse({"ok": True}) + + # ── Phone mic real-time audio → Gemini Live ────────────────────────── + + @app.websocket("/ws/phone-audio") + async def phone_audio_ws(websocket: WebSocket, token: str = ""): + tok = token.strip() + if not tok or tok not in self._tokens: + await websocket.close(code=4001) + return + await websocket.accept() + asyncio.create_task(self.broadcast( + {"type": "sys", "text": "Phone microphone live."} + )) + try: + while True: + data = await websocket.receive_bytes() + try: + self._phone_audio_queue.put_nowait( + {"data": data, "mime_type": "audio/pcm"} + ) + except asyncio.QueueFull: + pass # drop frame rather than block + except WebSocketDisconnect: + pass + finally: + asyncio.create_task(self.broadcast( + {"type": "sys", "text": "Phone microphone stopped."} + )) + + # ── File sharing ────────────────────────────────────────────────────── + + def _safe_filename(raw: str) -> str: + name = Path(raw).name # strip path components + name = re.sub(r'[<>:"/\\|?*\x00-\x1f]', '_', name).strip(". ") + return name or "upload" + + if _UPLOAD_OK: + @app.post("/api/upload") + async def upload_file(req: Request, file: UploadFile = FastAPIFile(...)): + if not _auth(req): + return JSONResponse({"error": "Unauthorized"}, status_code=401) + + safe = _safe_filename(file.filename or "upload") + dest = self._uploads_dir / safe + stem, suffix = Path(safe).stem, Path(safe).suffix + counter = 1 + while dest.exists(): + dest = self._uploads_dir / f"{stem}_{counter}{suffix}" + counter += 1 + + size = 0 + max_bytes = MAX_UPLOAD_MB * 1024 * 1024 + try: + with open(dest, "wb") as fout: + while True: + chunk = await file.read(65536) + if not chunk: + break + size += len(chunk) + if size > max_bytes: + fout.close() + dest.unlink(missing_ok=True) + return JSONResponse( + {"error": f"File too large (max {MAX_UPLOAD_MB} MB)"}, + status_code=413, + ) + fout.write(chunk) + except Exception as exc: + try: + dest.unlink(missing_ok=True) + except Exception: + pass + return JSONResponse({"error": str(exc)}, status_code=500) + + asyncio.create_task(self.broadcast({ + "type": "file_received", + "name": dest.name, + "size": size, + "saved_to": str(self._uploads_dir), + })) + return JSONResponse({"ok": True, "name": dest.name, "size": size}) + else: + @app.post("/api/upload") + async def upload_unavailable(req: Request): + return JSONResponse( + {"error": "File uploads require: pip install python-multipart"}, + status_code=503, + ) + + @app.get("/api/files") + async def list_files(req: Request): + if not _auth(req): + return JSONResponse({"error": "Unauthorized"}, status_code=401) + files = [] + try: + for f in sorted( + (p for p in self._uploads_dir.iterdir() if p.is_file()), + key=lambda p: p.stat().st_mtime, + reverse=True, + ): + files.append({"name": f.name, "size": f.stat().st_size}) + except Exception: + pass + return JSONResponse({"files": files}) + + @app.get("/uploads/{filename}") + async def download_file(filename: str, token: str = ""): + # Auth via query param — browser can't send custom headers + tok = token.strip() + if not tok or tok not in self._tokens: + return JSONResponse({"error": "Unauthorized"}, status_code=401) + safe = re.sub(r'[/\\]', '', filename) + path = self._uploads_dir / safe + if not path.exists() or not path.is_file(): + return JSONResponse({"error": "Not found"}, status_code=404) + return FileResponse(str(path), filename=safe) + + @app.websocket("/ws") + async def ws_ep(websocket: WebSocket, token: str = ""): + tok = token.strip() + if not tok or tok not in self._tokens: + await websocket.close(code=4001) + return + await websocket.accept() + self._clients.add(websocket) + for entry in self._history[-50:]: + try: + await websocket.send_json(entry) + except Exception: + break + try: + while True: + data = await websocket.receive_json() + if data.get("type") == "command": + enc = data.get("enc", "") + t = self._decrypt(tok, enc) if enc else (data.get("text") or "").strip() + if t: + await self._command_queue.put(t) + if self._wake_callback: + self._wake_callback() + except WebSocketDisconnect: + pass + finally: + self._clients.discard(websocket) + + return app + + # ── serve ───────────────────────────────────────────────────────────── + + async def _serve_alias(self) -> None: + """Second HTTPS server on PORT+1 sharing the same app and in-memory state. + Chrome HTTPS-upgrades any bare IP:PORT the user types, so this port also needs TLS. + User types IP:8001 → Chrome tries https → self-signed cert warning → accept once → done.""" + ssl_key = BASE_DIR / "config" / "certs" / "jarvis.key" + ssl_cert = BASE_DIR / "config" / "certs" / "jarvis.crt" + asyncio.get_event_loop().run_in_executor(None, _ensure_network_access, PORT + 1) + cfg = uvicorn.Config( + self.app, host="0.0.0.0", port=PORT + 1, log_level="warning", + ssl_keyfile=str(ssl_key), ssl_certfile=str(ssl_cert), + ) + print(f"[Dashboard] Manual entry: {self._ip}:{PORT + 1} (type in browser, accept cert once)") + await uvicorn.Server(cfg).serve() + + async def serve(self) -> None: + if not _DEPS_OK: + print("[Dashboard] fastapi/uvicorn not installed — dashboard disabled.") + print("[Dashboard] Run: pip install fastapi 'uvicorn[standard]' cryptography") + return + + # Firewall setup runs in a thread — uvicorn starts immediately, + # no waiting for UAC dialogs or subprocess timeouts. + asyncio.get_event_loop().run_in_executor(None, _ensure_network_access, PORT) + + # Generate the TLS pair on first run so no private key ships in the repo. + _ensure_certs() + + use_ssl = self._ssl_enabled() + ssl_key = BASE_DIR / "config" / "certs" / "jarvis.key" + ssl_cert = BASE_DIR / "config" / "certs" / "jarvis.crt" + + if use_ssl: + asyncio.create_task(self._serve_alias()) + + cfg = uvicorn.Config( + self.app, host="0.0.0.0", port=PORT, log_level="warning", + **({"ssl_keyfile": str(ssl_key), "ssl_certfile": str(ssl_cert)} if use_ssl else {}), + ) + + proto = "https" if use_ssl else "http" + print(f"[Dashboard] {proto}://{self._ip}:{PORT}") + print("[Dashboard] Press 'Remote Control' in JARVIS UI to get the QR code.") + await uvicorn.Server(cfg).serve() diff --git a/main.py b/main.py index 32b58b1..cf0bfb1 100644 --- a/main.py +++ b/main.py @@ -14,25 +14,25 @@ if _platform.system() == "Windows": _subprocess.Popen = _Popen -# ───────────────────────────────────────────────────────────────────────────── -# ── Console encoding ───────────────────────────────────────────────────────── -# Status lines in this app carry emoji and arrows ("📤 file_controller → Moved: -# a.txt → Documents/"). On a non-UTF-8 console — cp1254 on a Turkish Windows, -# cp1251 on a Russian one, cp932 on a Japanese one — printing one of those -# raises UnicodeEncodeError, and because the print sits after the tool's own -# try/except, the exception escapes into the receive loop and takes the session -# down. The assistant dies on a log line. -# -# Reconfiguring costs nothing and makes the app start the same way in every -# locale. `errors="replace"` is the belt and braces — a console that genuinely -# cannot render a glyph shows a box instead of killing the process. +# ── Console must survive non-UTF-8 code pages ──────────────────────────────── +# Every status line in this file carries an emoji, and on a legacy Windows +# console the active code page is the system one — cp1254 in Turkey, cp1251 in +# Russia, cp932 in Japan. Printing an emoji there raises UnicodeEncodeError, and +# because most of these prints sit inside the receive loop it takes the session +# down on startup. Reconfiguring to UTF-8 with a replacement fallback costs +# nothing and makes the app launch the same way in every locale. import sys as _sys -for _stream in (_sys.stdout, _sys.stderr): + +for _stream in ("stdout", "stderr"): try: - _stream.reconfigure(encoding="utf-8", errors="replace") + _s = getattr(_sys, _stream, None) + if _s is not None and hasattr(_s, "reconfigure"): + _s.reconfigure(encoding="utf-8", errors="replace") except Exception: - pass + pass # pythonw / redirected pipes / anything exotic — never fatal + +# ───────────────────────────────────────────────────────────────────────────── import asyncio import re @@ -69,13 +69,17 @@ from actions.background_monitor import ( ) from actions.web_search import _news as _fetch_news_sync from memory.config_manager import ( - get_brief_enabled, get_voice, get_wake_word_enabled, save_wake_word_enabled, get_input_device, get_output_device, + get_brief_enabled, get_media_resolution, get_proactive_audio_enabled, + get_push_to_talk_enabled, get_thinking_enabled, get_turn_tuning, get_voice, + get_wake_word_enabled, save_wake_word_enabled, get_input_device, get_output_device, ) from core.plugin_loader import discover_plugins from core import undo as undo_stack from core import confirm as confirm_gate from core import audio_devices from core.action_loader import discover_actions +from core.echo import EchoGuard +from core.viseme import VisemeStream from core.wake_word import ( WakeWordDetector, is_ready as wake_is_ready, install_and_download as wake_install, ) @@ -120,6 +124,164 @@ def _pcm_level(samples) -> float: return min(1.0, (rms - _LEVEL_FLOOR) / (_LEVEL_FULL - _LEVEL_FLOOR)) +# ── Viseme extraction ───────────────────────────────────────────────────────── +# The avatar's mouth used to be driven by one RMS value per ~200 ms write batch, +# which is five updates a second averaged over a fifth of a second — it could +# only ever flap. These read the *shape* of each 20 ms slice straight from the +# spectrum of the audio being played, so no transcript, no forced alignment and +# no language assumption: it works the same for Turkish and English. +# +# Two numbers come out. Openness tracks the first formant — F1 climbs as the jaw +# drops, so /a/ reads open and /i/ or /u/ read closed. Width tracks the second — +# F2 is high for spread vowels (/i/, /e/) and low for rounded ones (/u/, /o/). +# Extra time beyond the device's reported output latency before the microphone +# is trusted again: covers room decay and the speaker's own settling. +_TAIL_MARGIN = 0.25 + +_VIS_WIN = 1024 # ~43 ms analysis window at 24 kHz: enough for formants +_VIS_HOP = 480 # 20 ms between frames, i.e. 50 shapes a second + +# Delay from handing the first bytes of a reply to an already-running output +# stream to hearing them: one callback period, plus whatever the DAC adds. +_FIRST_SOUND = CHUNK_SIZE / RECEIVE_SAMPLE_RATE # ~43 ms +# How far past the device's own buffer the mouth's timeline may drift before it +# is re-anchored. The buffer is the hard limit on how much audio can be queued +# ahead, so anything beyond it plus a margin for clock error is impossible. +_CURSOR_SLACK = 0.15 + +# Erring early is the safe direction. A viewer tolerates a mouth that moves +# slightly before the sound far better than one that moves after it — the +# broadcast limits are about 45 ms of lag against 125 ms of lead — so where +# this is uncertain it is biased to lead. + + +def _pcm_visemes(samples, sr: int = 24000): + """Slice a PCM block into (level, openness, width) frames, one per 20 ms. + + Returns [] on anything unexpected — the mouth falls back to loudness-only + articulation rather than the caller having to handle an error. + """ + try: + x = np.asarray(samples, dtype=np.float32) + if x.size < _VIS_WIN: + return [] + win = np.hanning(_VIS_WIN).astype(np.float32) + freqs = np.fft.rfftfreq(_VIS_WIN, 1.0 / sr) + b_f1_lo = (freqs >= 150) & (freqs < 450) # F1 of close vowels + b_f1_hi = (freqs >= 450) & (freqs < 1100) # F1 of open vowels + b_f2_bk = (freqs >= 600) & (freqs < 1300) # F2 of rounded vowels + b_f2_fr = (freqs >= 1700) & (freqs < 3200) # F2 of spread vowels + b_hiss = (freqs >= 3800) & (freqs < 8000) # fricatives + + # One frame per hop across the *whole* block. Stepping only while a full + # window fits stopped 1024 - 480 samples short of the end, so a 200 ms + # batch yielded 160 ms of schedule: the mouth ran out of frames before + # the audio ran out of sound, and each batch no longer lined up with the + # end of the one before it. Losing 20 % of every batch is most of why + # the mouth did not track the words. + out = [] + for start in range(0, x.size, _VIS_HOP): + # The level gates closures, so it is measured over exactly this + # 20 ms and never looks ahead. The spectrum needs a longer window + # to resolve formants and may be short-filled at the very end. + level = _pcm_level(x[start:start + _VIS_HOP]) + seg = x[start:start + _VIS_WIN] + if seg.size < _VIS_WIN: + seg = np.concatenate([seg, np.zeros(_VIS_WIN - seg.size, + dtype=np.float32)]) + if level <= 0.0: + out.append((0.0, 0.0, 0.0)) + continue + mag = np.abs(np.fft.rfft((seg - seg.mean()) * win)) + f1l, f1h = float(mag[b_f1_lo].sum()), float(mag[b_f1_hi].sum()) + f2b, f2f = float(mag[b_f2_bk].sum()), float(mag[b_f2_fr].sum()) + hiss = float(mag[b_hiss].sum()) + + openness = f1h / (f1l + f1h + 1e-6) + width = (f2f - f2b) / (f2f + f2b + 1e-6) + # A wide-open jaw physically cannot purse, so openness damps width. + # /a/ has a low enough F2 to read as "rounded" on the bands alone; + # letting openness suppress the width term is what keeps an open + # vowel from pursing. + width *= (1.0 - openness) ** 0.8 + # Fricatives are formed with a nearly closed mouth. + h = hiss / (f1l + f1h + f2b + f2f + hiss + 1e-6) + openness *= 1.0 - 0.65 * min(1.0, h * 2.5) + out.append((level, + float(min(1.0, max(0.0, openness))), + float(min(1.0, max(-1.0, width))))) + return out + except Exception: + return [] + + +def _describe_tools(declarations) -> str: + """One line per capability, straight from the live tool declarations. + + Derived rather than written down: the action and plugin registries are + discovered at startup, so whatever the user has installed is what the model + is told it can do. Adding a plugin extends this by itself, and removing one + stops the model from claiming an ability it no longer has. + """ + lines = [] + for d in declarations or (): + try: + name = d.get("name") if isinstance(d, dict) else getattr(d, "name", None) + desc = (d.get("description") if isinstance(d, dict) + else getattr(d, "description", "")) or "" + except Exception: + continue + if not name: + continue + desc = " ".join(str(desc).split()) + lines.append(f"- {name}: {desc[:150]}" if desc else f"- {name}") + return "\n".join(lines) + + +def _describe_limits(has_vision: bool, has_mic: bool) -> str: + """The other half of self-knowledge: what is out of reach, and why. + + Derived from how the program is actually built, not from a list of refusals. + A model that knows its boundaries stops improvising around them, and stating + them as architecture rather than as rules keeps the answer honest in any + language. + """ + out = [ + "- Anything not listed above is outside your reach. Say so in one clause " + "and offer the nearest thing you can actually do — never mime an action " + "you cannot take, and never report a result you did not get.", + "- You act on this machine only. You cannot reach the user's other " + "devices, accounts or hardware except through the tools listed above.", + "- You remember what is in the memory block and what has been said this " + "session. Anything else you were told before is gone unless it was saved.", + ] + if has_vision: + out.append( + "- Your sight is not continuous. You see nothing until you call a " + "vision tool, and then only that single frame at that moment — you " + "cannot watch, monitor or notice something changing on screen.") + else: + out.append("- You have no sight at all in this build.") + if has_mic: + out.append( + "- You hear nothing while the microphone is muted, and you cannot " + "unmute it yourself.") + return "\n".join(out) + + +def _render_prompt(template: str, values: dict) -> str: + """Fill {tokens} in the prompt template. + + A plain replace rather than str.format: the file is meant to be edited by + hand, and a stray brace in someone's own wording must never take the app + down at startup. + """ + out = template or "" + for key, val in values.items(): + out = out.replace("{" + key + "}", str(val)) + return out + + def _get_api_key() -> str: with open(API_CONFIG_PATH, "r", encoding="utf-8") as f: return json.load(f)["gemini_api_key"] @@ -137,6 +299,22 @@ def _load_system_prompt() -> str: _CTRL_RE = re.compile(r"", re.IGNORECASE) +# Transcript chunks shorter than this may legitimately repeat ("evet, evet"), +# so only longer ones are treated as duplicates. +_REPEAT_MIN = 12 + + +def _is_repeat_chunk(txt: str, buf: list) -> bool: + """True if this transcript chunk has already been seen this turn. + + Guards against the API re-sending the tail of a response across the several + turn_completes a tool-using turn produces. + """ + if len(txt) < _REPEAT_MIN: + return bool(buf) and txt == buf[-1] + joined = " ".join(buf) + return txt in joined + def _clean_transcript(text: str) -> str: text = _CTRL_RE.sub("", text) text = re.sub(r"[\x00-\x08\x0b-\x1f]", "", text) @@ -366,6 +544,30 @@ class JarvisLive: self._vision_last_time = 0.0 # monotonic time of last screen_process call (cooldown guard) self._vision_busy = False # True while a vision capture/inject cycle is in flight self._interrupted = False # True while draining audio after user interrupt + # Transcript-driven mouth shapes for the avatar. Fed from the receive + # loop as words arrive, drained by the playback loop against the audio. + self._visemes = VisemeStream() + self._last_out_logged = "" # de-dupes a re-sent transcript tail + # Push-to-talk + self._ptt_enabled = False + self._ptt_held = False + self._ptt = None # core.hotkey.PushToTalk + self._out_level = 0.0 # level of the audio being played right now + self._echo = EchoGuard() + # `stream.write()` returns when the buffer accepts the audio, not when the + # speaker has finished with it, so sound is still in the room after the + # speaking flag drops. Streaming the microphone during that gap is how an + # assistant ends up answering itself. Measured from the device rather than + # guessed; see _play_audio. + self._out_latency = 0.20 # seconds, replaced with the real value + self._tail_until = 0.0 # monotonic time the echo tail expires + # Wall-clock time at which the audio written next will begin to sound. + # The mouth is scheduled against this, never against "now": batches are + # handed to the device far faster than they play, so "now" ran the lips + # ahead of the words and cut every schedule short. 0 = nothing playing. + self._play_cursor = 0.0 + self.ui.on_push_to_talk = self.set_push_to_talk + self.ui.ptt_hold = self._on_ptt self.ui.on_text_command = self._on_text_command self.ui.on_remote_clicked = self._make_remote_key self.ui.on_interrupt = self.interrupt @@ -398,6 +600,7 @@ class JarvisLive: self._session_log: list[str] = [] # conversation turns for end-of-session summary self._enhanced_live = True # proactive audio; auto-disabled if the server rejects it + self._tuned_live = True # turn-taking / media / thinking knobs; same fallback _base_dir = Path(__file__).resolve().parent _inline_names = {t["name"] for t in TOOL_DECLARATIONS} @@ -416,7 +619,11 @@ class JarvisLive: self._plugin_registry = discover_plugins( plugins_dir=_base_dir / "plugins", core_tool_names=_core_names, - logger=lambda msg: (print(f"[Plugins] {msg}"), self.ui.write_log(f"SYS: {msg}")), + # Console gets the full boot transcript; the activity log gets only + # what the user has to know about. Every plugin loading correctly is + # the expected case and does not belong in their conversation. + logger=lambda msg: print(f"[Plugins] {msg}"), + notify=lambda msg: self.ui.write_log(f"SYS: {msg}"), ) self.ui.get_plugins = self._plugin_registry.list_for_ui self.ui.get_plugin_settings = self._plugin_registry.settings_schemas # ⚙ settings tab @@ -429,6 +636,15 @@ class JarvisLive: self._awake = not self._wake_enabled self._wake_detector: WakeWordDetector | None = None self._wake_sleep_timeout = WAKE_SLEEP_TIMEOUT + + # Restore the saved push-to-talk preference. Doing it here rather than + # in __init__ means the hotkey thread only exists once there is a + # session to talk to. + if get_push_to_talk_enabled(): + try: + self.set_push_to_talk(True) + except Exception as e: + print(f"[JARVIS] ⚠ Push-to-talk unavailable: {e}") # UI control surface for the Wake Word settings section. self.ui.wake_is_ready = wake_is_ready # () -> bool self.ui.wake_get_state = self._wake_state # () -> dict @@ -449,7 +665,8 @@ class JarvisLive: if self._wake_detector is None: self._wake_detector = WakeWordDetector( on_detect=self._on_wake_detected, - logger=lambda m: (print(f"[Wake] {m}"), self.ui.write_log(f"SYS: {m}")), + logger=lambda m: print(f"[Wake] {m}"), + notify=lambda m: self.ui.write_log(f"SYS: {m}"), ) if not self._wake_detector.ready: return self._wake_detector.start() @@ -519,7 +736,10 @@ class JarvisLive: def _ui_wake_install(self) -> tuple[bool, str]: """Download openwakeword + the model (runs in a UI worker thread).""" - return wake_install(logger=lambda m: self.ui.write_log(f"SYS: {m}")) + # Triggered by the user pressing the button, so its progress is exactly + # what they are waiting to see. + return wake_install(logger=lambda m: print(f"[Wake] {m}"), + notify=lambda m: self.ui.write_log(f"SYS: {m}")) def plugin_say(self, instruction: str) -> None: """ @@ -625,14 +845,72 @@ class JarvisLive: self._loop ) + def _tail_active(self) -> bool: + """True while the speakers may still be finishing our last sentence.""" + return time.monotonic() < self._tail_until + def set_speaking(self, value: bool): with self._speaking_lock: self._is_speaking = value + if value: + self._tail_until = 0.0 + else: + # Hold the guard open across the device's own output latency plus a + # margin for the room. The microphone is NOT muted during it — the + # guard still lets a genuine reply through, so answering instantly + # still works. Only our own echo is dropped. + self._tail_until = time.monotonic() + self._out_latency + _TAIL_MARGIN + if not value: + # The echo history is deliberately NOT cleared here: the tail above + # still needs it to recognise our own voice. It is dropped when the + # tail expires. What the guard learned about the room always stays. + self._out_level = 0.0 if value: self.ui.set_state("SPEAKING") elif not self.ui.muted: self.ui.set_state("LISTENING") + def set_push_to_talk(self, enabled: bool) -> str: + """Turn hold-to-talk on or off. Returns the scope actually achieved.""" + from core.hotkey import PushToTalk + + self._ptt_enabled = bool(enabled) + self._ptt_held = False + if not enabled: + if self._ptt is not None: + self._ptt.stop() + self._ptt = None + return "off" + + if self._ptt is None: + self._ptt = PushToTalk(self._on_ptt) + scope = self._ptt.start() + # A window-scoped chord is a real limitation, not a detail — say it once + # in the log so nobody wonders why it does nothing while another app is + # focused. Reporting it must never be able to undo the thing it reports. + try: + self.ui.write_log( + f"SYS: Push-to-talk on — hold {self._ptt.label}" + + ("." if scope == "global" + else " (works while this window is focused).")) + except Exception: + pass + return scope + + def _on_ptt(self, held: bool) -> None: + """Chord pressed or released — may arrive on the hotkey thread.""" + self._ptt_held = held + if held: + # Holding the key is also a way to wake it, so push-to-talk works + # without having to say the wake word first. + if self._wake_enabled and not self._awake: + self._awake = True + self._last_user_speech = time.monotonic() + try: + self.ui.set_state("LISTENING" if held else "SLEEPING") + except Exception: + pass + def interrupt(self) -> None: """Stop JARVIS mid-speech: drain queued audio and open mic immediately.""" self._interrupted = True @@ -648,6 +926,9 @@ class JarvisLive: if drained: print(f"[JARVIS] ✋ Interrupted — {drained} audio chunks discarded") self.set_speaking(False) + # The words we were about to mouth are never going to be spoken now. + self._visemes.reset() + self._play_cursor = 0.0 # next batch starts a fresh timeline if self._turn_done_event: self._turn_done_event.clear() self.ui.write_log("SYS: Interrupted — listening...") @@ -693,14 +974,17 @@ class JarvisLive: ) # Identity injection — overrides any hardcoded name in prompt.txt + # Address form is a property of the language being spoken, so it is + # stated as a principle rather than a two-language lookup — the model + # already knows the respectful register of whatever language it is in. _addr = (f"ADDRESS: Always call the user '{_user_name}'." if _user_name - else "ADDRESS: Address the user with the ordinary respectful form " - "for a superior in the language you are currently speaking — " - "\"sir\" in English, its everyday equivalent in any other " - "language. Never an archaic or aristocratic form, and never " - "the form from a different language than the one you are " - "speaking in this sentence.") + else 'ADDRESS: Address the user with the ordinary respectful form ' + 'for a superior in the language you are currently speaking — ' + '"sir" in English, its everyday equivalent in any other ' + 'language. Never an archaic or aristocratic form, and never ' + 'the form from a different language than the one you are ' + 'speaking in this sentence.') identity_ctx = ( f"[IDENTITY]\n" f"Your name is {self._asst_name}. " @@ -708,6 +992,26 @@ class JarvisLive: f"{_addr}\n\n" ) + # Everything the model is told about *itself* is derived here, not + # written into prompt.txt: the name comes from config, the platform from + # the host, the capability list from the registries that were just + # discovered. Rename the assistant, add a plugin or move to another OS + # and this follows without anyone editing a prompt. + _all_decls = (TOOL_DECLARATIONS + + self._action_registry.get_tool_declarations() + + self._plugin_registry.get_tool_declarations()) + _names = {(d.get("name") if isinstance(d, dict) else getattr(d, "name", "")) + for d in _all_decls} + sys_prompt = _render_prompt(sys_prompt, { + "assistant_name": self._asst_name, + "platform": f"{_platform.system()} {_platform.release()}".strip(), + "capabilities": _describe_tools(_all_decls), + "limits": _describe_limits( + has_vision="screen_process" in _names, + has_mic=True, + ), + }) + parts = [time_ctx, identity_ctx] if mem_str: parts.append(mem_str) @@ -718,11 +1022,7 @@ class JarvisLive: output_audio_transcription={}, input_audio_transcription={}, system_instruction="\n".join(parts), - tools=[{"function_declarations": ( - TOOL_DECLARATIONS - + self._action_registry.get_tool_declarations() - + self._plugin_registry.get_tool_declarations() - )}], + tools=[{"function_declarations": _all_decls}], # Hand back the handle captured from the last session_resumption # update. `handle=None` is exactly the old behaviour (ask for # handles, start fresh), so the first connect of a run is unchanged. @@ -749,9 +1049,67 @@ class JarvisLive: # support it, and it never reliably detected tone in practice. # To restore it on a 2.5 native-audio model, add back: # cfg["enable_affective_dialog"] = True ) - cfg["proactivity"] = types.ProactivityConfig(proactive_audio=True) + if get_proactive_audio_enabled(): + cfg["proactivity"] = types.ProactivityConfig(proactive_audio=True) + + if self._tuned_live: + cfg.update(self._tuning_config()) + return types.LiveConnectConfig(**cfg) + def _tuning_config(self) -> dict: + """The optional knobs, kept apart so one bad field can be dropped wholesale. + + Every one of these is a preview-API field. If a future model release + stops accepting any of them the connection fails at setup, so the run + loop turns `_tuned_live` off and reconnects on the plain config rather + than leaving the user with an assistant that will not start. + """ + out: dict = {} + + # How long the server waits through a pause before deciding your turn is + # over. This — not the size of the prompt — is what most of the delay + # before a reply actually is, and the default has to suit everybody, so + # it is necessarily cautious. + turn = get_turn_tuning() + if turn.get("enabled", True): + detect = types.AutomaticActivityDetection( + silence_duration_ms=turn["silence_ms"], + prefix_padding_ms=turn["prefix_ms"], + ) + if turn["end_sensitivity"] == "high": + detect.end_of_speech_sensitivity = types.EndSensitivity.END_SENSITIVITY_HIGH + elif turn["end_sensitivity"] == "low": + detect.end_of_speech_sensitivity = types.EndSensitivity.END_SENSITIVITY_LOW + if turn["start_sensitivity"] == "high": + detect.start_of_speech_sensitivity = types.StartSensitivity.START_SENSITIVITY_HIGH + elif turn["start_sensitivity"] == "low": + detect.start_of_speech_sensitivity = types.StartSensitivity.START_SENSITIVITY_LOW + out["realtime_input_config"] = types.RealtimeInputConfig( + automatic_activity_detection=detect) + + # Screenshots and camera frames are tokenised at this resolution and then + # stay in the session's context. 'medium' keeps on-screen text legible + # for a fraction of a full-resolution frame. + res = get_media_resolution() + if res != "default": + out["media_resolution"] = { + "low": types.MediaResolution.MEDIA_RESOLUTION_LOW, + "medium": types.MediaResolution.MEDIA_RESOLUTION_MEDIUM, + "high": types.MediaResolution.MEDIA_RESOLUTION_HIGH, + }[res] + + # Thinking is left at the server default deliberately. Forcing the budget + # to zero was measured on gemini-3.1-flash-live over interleaved trials + # and did not make the first word arrive sooner — this model does not + # appear to deliberate on the Live path, so pinning the field only adds a + # way for a future release to behave differently. Set "thinking_enabled" + # in config/api_keys.json to true to let it reason instead. + if get_thinking_enabled(): + out["thinking_config"] = types.ThinkingConfig(thinking_budget=-1) + + return out + async def _execute_tool(self, fc) -> types.FunctionResponse: name = fc.name args = dict(fc.args or {}) @@ -759,6 +1117,7 @@ class JarvisLive: print(f"[JARVIS] 🔧 {name} {args}") self.ui.set_state("THINKING") + if name == "save_memory": category = args.get("category", "notes") key = args.get("key", "") @@ -817,11 +1176,16 @@ class JarvisLive: print(f"[Vision] 🖥️ Screen: {len(img_b):,} bytes") _stall = "screen" self._pending_vision = (img_b, mime_t, user_text, angle) + # The image is attached to this same exchange, so there is + # nothing to stall for and nothing to announce. Asking for an + # acknowledgement here is what produced two spoken answers — + # the model filled that turn by answering the question from + # imagination, then answered it again once it could see. result = ( - f"[VISION_ACTIVE] {_stall.capitalize()} captured. " - f"Immediately say ONE short natural sentence in the user's own language, " - f"telling them you are looking at their {_stall} right now. " - f"Do NOT describe or guess content — the actual image arrives in the NEXT message." + f"[VISION_ACTIVE] {_stall.capitalize()} captured and attached to this " + f"same exchange. Do not acknowledge and do not answer yet — the image " + f"is arriving with this result. Reply once, from what you actually see " + f"in it." ) elif name == "close_camera": @@ -898,9 +1262,20 @@ class JarvisLive: self.ui.set_state("LISTENING") print(f"[JARVIS] 📤 {name} → {str(result)[:80]}") + + # A tool that declared itself NON_BLOCKING also says when its answer may + # re-enter the conversation. Without this the model finishes whatever it + # was saying and then reads the result out on top of it — which, for + # something like a phone call already ringing, is exactly the noise the + # non-blocking call was meant to avoid. Tools that declared nothing get + # the API default and behave as they always have. + _sched = (self._action_registry.scheduling(name) + or self._plugin_registry.scheduling(name)) + _extra = {"scheduling": _sched} if _sched else {} return types.FunctionResponse( id=fc.id, name=name, - response={"result": result} + response={"result": result}, + **_extra ) async def _send_realtime(self): @@ -937,7 +1312,52 @@ class JarvisLive: return with self._speaking_lock: jarvis_speaking = self._is_speaking - if not jarvis_speaking and not self.ui.muted and not self._phone_active: + + # ── Barge-in ───────────────────────────────────────────────────── + # While JARVIS talks the mic is not streamed, but it is still worth + # listening to locally: if the user starts speaking, cut the answer + # short the way a person would stop when interrupted. + # + # The whole difficulty is echo — on speakers the mic hears JARVIS. + # So the test is not "is the mic loud" but "is the mic louder than + # the echo of what we are playing right now", sustained long enough + # that a cough or a keystroke cannot trigger it. + if jarvis_speaking: + # Nothing is streamed while JARVIS talks. + # + # Interrupting by voice used to live here: `EchoGuard` can pick a + # user out from under our own echo, and `core/echo.py` still does + # that for the tail below. Re-enabling is small — classify each + # block here and call interrupt() after `required_blocks` of + # agreement — but it depends on the listener's room, so it stays + # out until it can be tried on real hardware. + return + + # ── Echo tail ──────────────────────────────────────────────────── + # The speaking flag has dropped but the speakers have not finished. + # Sending this to the model is how an assistant hears itself, decides + # it was addressed, and answers its own last sentence. The microphone + # stays OPEN — the guard only drops blocks that are our own voice, so + # replying the instant it stops still works. + if self._tail_active(): + try: + if not self._echo.is_user_speech( + indata, SEND_SAMPLE_RATE, _pcm_level(indata)): + return + self._tail_until = 0.0 # a real voice ends the tail early + except Exception: + return + elif self._echo._hist: + self._echo.reset() + + # ── Push-to-talk ───────────────────────────────────────────────── + # When it is on the microphone is closed by default and the chord + # opens it, which is the whole point: nothing leaves the machine + # unless you are holding the key. + if self._ptt_enabled and not self._ptt_held: + return + + if not self.ui.muted and not self._phone_active: data = indata.tobytes() loop.call_soon_threadsafe( self.out_queue.put_nowait, @@ -993,6 +1413,49 @@ class JarvisLive: print(f"[JARVIS] ❌ Mic: {e}") raise + async def _flush_pending_vision(self) -> bool: + """Send a captured frame immediately after its tool response. + + The frame is already in hand by the time `screen_process` returns — the + capture happened inside the tool call. The old flow still made the model + speak a turn first and only injected the image on that turn's + turn_complete, which cost a whole extra round trip AND produced two + spoken answers: one improvised without the picture, then the real one. + Sending it here means the model has the tool result and the image before + it generates anything, so the user gets one answer, sooner. + """ + if not (self._pending_vision and self.session): + return False + + import base64 as _b64 + img_b, mime_t, question, angle = self._pending_vision + self._pending_vision = None + b64 = _b64.b64encode(img_b).decode("ascii") + print(f"[Vision] 📤 {len(img_b):,} bytes (angle={angle}) → main session") + + # Label the source. Without it the image arrives carrying nothing but + # the user's own sentence, and a screenshot of this app — which has a + # face in the middle of it — got read as a photo of the user. What the + # label *means* is explained once, in the generated [SELF] block. + src = ("[IMAGE SOURCE: WEBCAM]" if angle == "camera" + else "[IMAGE SOURCE: SCREEN CAPTURE]") + await self.session.send_client_content( + turns={"role": "user", "parts": [ + {"inline_data": {"mime_type": mime_t, "data": b64}}, + {"text": f"{src}\n\n{question}"}, + ]}, + turn_complete=True, + ) + + if self._vision_cam_active: + # Camera: stay busy until JARVIS has finished speaking the answer, + # then close the preview. + self._vision_cam_active = False + self._vision_close_pending = True + else: + self._vision_busy = False + return True + async def _receive_audio(self): print("[JARVIS] 👂 Recv started") out_buf, in_buf = [], [] @@ -1032,8 +1495,20 @@ class JarvisLive: if sc.output_transcription and sc.output_transcription.text: txt = _clean_transcript(sc.output_transcription.text) - if txt and txt != (out_buf[-1] if out_buf else ""): + # A turn that involves a tool call passes through + # several turn_completes, and the API re-sends the + # tail of the transcript across them. Comparing only + # against the previous chunk missed that — once + # out_buf had been flushed and emptied, the repeat + # sailed straight back in, which logged the answer + # twice AND made the avatar mouth it twice. + if txt and not _is_repeat_chunk(txt, out_buf): out_buf.append(txt) + # Hand the words to the mouth as they arrive, so + # the avatar can form the consonants the audio + # alone cannot show. Pure string work — it adds + # nothing measurable to the response path. + self._visemes.feed_text(txt) if sc.input_transcription and sc.input_transcription.text: txt = _clean_transcript(sc.input_transcription.text) @@ -1051,10 +1526,12 @@ class JarvisLive: self._interrupted = False in_buf = [] out_buf = [] + self._visemes.reset() continue full_in = " ".join(in_buf).strip() if full_in: + self._last_out_logged = "" # new exchange self.ui.write_log(f"You: {full_in}") self._session_log.append(f"User: {full_in}") if self._dashboard: @@ -1066,7 +1543,14 @@ class JarvisLive: in_buf = [] full_out = " ".join(out_buf).strip() + # Second line of defence: even if a repeat slips + # into a *fresh* buffer after a flush, never log the + # same answer (or a tail of it) twice in a row. + if full_out and len(full_out) >= _REPEAT_MIN and self._last_out_logged: + if full_out in self._last_out_logged: + full_out = "" if full_out: + self._last_out_logged = full_out self.ui.write_log(f"{self._asst_name}: {full_out}") self._session_log.append(f"{self._asst_name}: {full_out}") if self._dashboard: @@ -1077,29 +1561,7 @@ class JarvisLive: })) out_buf = [] - # Vision injection: model finished tool-response turn → now send the image - if self._pending_vision and self.session: - import base64 as _b64 - img_b, mime_t, question, angle = self._pending_vision - self._pending_vision = None - b64 = _b64.b64encode(img_b).decode("ascii") - print(f"[Vision] 📤 {len(img_b):,} bytes (angle={angle}) → main session") - await self.session.send_client_content( - turns={"role": "user", "parts": [ - {"inline_data": {"mime_type": mime_t, "data": b64}}, - {"text": question}, - ]}, - turn_complete=True, - ) - # Mark next turn_complete behaviour depending on angle - if self._vision_cam_active: - # Camera: keep busy until JARVIS finishes speaking the answer - self._vision_cam_active = False - self._vision_close_pending = True - else: - # Screen-only: no camera to close; release busy flag now - self._vision_busy = False - elif self._vision_close_pending: + if self._vision_close_pending: # This turn_complete IS the vision answer — close camera + release busy flag self._vision_close_pending = False self._vision_busy = False @@ -1117,6 +1579,7 @@ class JarvisLive: await self.session.send_tool_response( function_responses=fn_responses ) + await self._flush_pending_vision() except Exception as e: print(f"[JARVIS] ❌ Recv: {e}") traceback.print_exc() @@ -1153,6 +1616,19 @@ class JarvisLive: self.ui.write_log(f"SYS: Speaker '{_spk_name}' unavailable — using system default.") stream = _open_spk(None) + # Ask the device how far behind the speakers actually are, rather than + # assuming. This is what the echo tail is sized from, so a machine with a + # large audio buffer gets a correspondingly longer guard — and one with a + # tiny buffer is not penalised with a delay it does not need. + try: + lat = float(getattr(stream, "latency", 0.0) or 0.0) + if 0.0 < lat < 1.0: + self._out_latency = lat + print(f"[JARVIS] 🔊 Output latency {self._out_latency*1000:.0f} ms " + f"→ echo tail {(self._out_latency + _TAIL_MARGIN)*1000:.0f} ms") + except Exception: + pass + try: while True: try: @@ -1182,10 +1658,53 @@ class JarvisLive: except asyncio.QueueEmpty: break - # Drive the HUD waveform from JARVIS's own voice while speaking. + # Drive the HUD waveform and the avatar's mouth from JARVIS's + # own voice. The batch is up to 200 ms long, so we hand over a + # *schedule* of 20 ms viseme frames instead of a single averaged + # level and let the HUD play it out in step with the audio. try: - self.ui.set_audio_level(_pcm_level( - np.frombuffer(bytes(batch), dtype=np.int16))) + pcm = np.frombuffer(bytes(batch), dtype=np.int16) + hop = _VIS_HOP / RECEIVE_SAMPLE_RATE + frames = _pcm_visemes(pcm, sr=RECEIVE_SAMPLE_RATE) + # When does this batch become audible? The stream was + # started at launch and its callback has been pulling + # silence ever since, so the first bytes of a reply reach + # the speaker about one callback period later — NOT one + # buffer later. `stream.latency` reports the buffer's + # capacity, which is how much can be queued ahead, and on + # Windows that is commonly 300-500 ms. Anchoring on it put + # the entire schedule a buffer late; that is the half second + # of lag, and it grew with whatever the device reported. + # + # After the anchor nothing needs measuring: the device + # consumes at exactly realtime, so each batch sounds one + # batch-duration after the one before it. The cursor is + # re-anchored only when it leaves the range physically + # possible — behind `now` means the device drained and this + # batch starts a fresh stretch of speech, while further + # ahead than the buffer can hold means it has drifted. + now = time.time() + horizon = self._out_latency + _CURSOR_SLACK + if not (now <= self._play_cursor <= now + horizon): + self._play_cursor = now + _FIRST_SOUND + at = self._play_cursor + # Advance by the batch's own duration whether or not it + # yielded frames, so a block too short to analyse cannot + # shift everything after it out of step with the audio. + self._play_cursor += pcm.size / RECEIVE_SAMPLE_RATE + if frames: + frames = self._visemes.frames(frames, hop) + self.ui.push_visemes(frames, hop, at) + # Barge-in needs to know what we are playing, not just + # how loud: the guard subtracts this from the microphone. + self._out_level = max(f[0] for f in frames) + self._echo.note_output(pcm, RECEIVE_SAMPLE_RATE, + self._out_level) + else: + lvl = _pcm_level(pcm) + self.ui.set_audio_level(lvl) + self._out_level = lvl + self._echo.note_output(pcm, RECEIVE_SAMPLE_RATE, lvl) except Exception: pass @@ -1266,7 +1785,7 @@ class JarvisLive: turns={"role": "user", "parts": [{"text": p1}]}, turn_complete=True, ) - self.ui.write_log("SYS: Briefing phase 1 (greeting) sent.") + print("[JARVIS] Briefing phase 1 (greeting) sent.") # ── Phase 2: fire as soon as Phase 1 audio is done ─────────────────── async def _deliver_news(): @@ -1296,14 +1815,18 @@ class JarvisLive: await asyncio.sleep(1.0) try: - news_text = await asyncio.wait_for(news_done, timeout=4.0) - except Exception: + news_text = await asyncio.wait_for(news_done, timeout=8.0) + except Exception as e: + self.ui.write_log(f"SYS: News fetch timed out/failed: {e!r}") news_text = "" if not self.session: return - if news_text and len(news_text) > 60: + failed = (not news_text) or news_text.startswith( + ("No news found", "Search failed", "Please provide") + ) + if not failed: # Show on UI content panel immediately self.ui.show_content("NEWS — top world news today", news_text) @@ -1313,6 +1836,9 @@ class JarvisLive: f"is displayed on screen. Do not call any tools.{lang_str}" ) else: + self.ui.write_log( + f"SYS: News unavailable — backend returned: {news_text[:120]!r}" + ) p2 = ( "News headlines could not be fetched right now. " f"Let the user know briefly.{lang_str}" @@ -1322,10 +1848,11 @@ class JarvisLive: turns={"role": "user", "parts": [{"text": p2}]}, turn_complete=True, ) - self.ui.write_log("SYS: Briefing phase 2 (news) sent.") + print("[JARVIS] Briefing phase 2 (news) sent.") except Exception as e: print(f"[Briefing] Phase 2 error: {e}") - self.ui.write_log(f"SYS: Briefing phase 2 failed: {e}") + print(f"[JARVIS] Briefing phase 2 failed: {e}") + self.ui.write_log("SYS: Could not fetch the news for the briefing.") asyncio.create_task(_deliver_news()) @@ -1350,14 +1877,10 @@ class JarvisLive: "Output ONLY the summary text, nothing else:\n\n" + convo ) try: - from google import genai as _genai - client = _genai.Client(api_key=_get_api_key()) - resp = await asyncio.to_thread( - client.models.generate_content, - model="gemini-flash-latest", - contents=prompt, + from core import gemini + summary = await asyncio.to_thread( + gemini.text, prompt, gemini.SMART, None, 30_000, ) - summary = (resp.text or "").strip() if summary: save_session_summary(summary, lang) except Exception as e: @@ -1412,7 +1935,7 @@ class JarvisLive: turns={"role": "user", "parts": [{"text": msg}]}, turn_complete=True, ) - self.ui.write_log(f"SYS: Monitor alert sent.") + print("[JARVIS] Monitor alert sent.") await asyncio.sleep(6) # gap between consecutive alerts except Exception as e: print(f"[Monitor] ⚠️ Background check error: {e}") @@ -1455,7 +1978,7 @@ class JarvisLive: turns={"role": "user", "parts": [{"text": prompt}]}, turn_complete=True, ) - self.ui.write_log("SYS: Proactive check-in.") + print("[JARVIS] Proactive check-in.") except Exception as e: print(f"[Proactive] ⚠️ {e}") @@ -1672,6 +2195,22 @@ class JarvisLive: print(f"[JARVIS] Error ({type(e).__name__}): {e}") traceback.print_exc() + # Turn-taking / media / thinking knobs rejected by the server + # (preview API drift) — drop them first, because they are the + # newest fields and the cheapest to lose. Proactive audio is + # tried again on the next pass if the error persists. + if self._tuned_live and ( + "INVALID_ARGUMENT" in err_str + or "Unknown name" in err_str + or "unexpected keyword" in err_str + or "realtime_input" in err_str.lower() + or "media_resolution" in err_str.lower() + or "thinking" in err_str.lower() + ): + self._tuned_live = False + print("[JARVIS] Live tuning rejected — reconnecting without it.") + continue + # Proactive audio rejected by the server (preview API drift) — # drop it and reconnect with the plain config. if self._enhanced_live and ( diff --git a/memory/config_manager.py b/memory/config_manager.py index 0e96195..915303e 100644 --- a/memory/config_manager.py +++ b/memory/config_manager.py @@ -121,6 +121,150 @@ def save_wake_word_enabled(enabled: bool) -> None: CONFIG_FILE.write_text(json.dumps(data, indent=4), encoding="utf-8") +def get_push_to_talk_enabled() -> bool: + """Hold-a-key-to-speak. When on, the mic is closed unless the chord is held.""" + return load_api_keys().get("push_to_talk_enabled", False) + + +def save_push_to_talk_enabled(enabled: bool) -> None: + _save_flag("push_to_talk_enabled", enabled) + + +HUD_STYLES = ("face", "core") + + +def get_hud_style() -> str: + """Which centrepiece the HUD draws: the animated head, or the reactor core. + + Taste, not capability — both render in the same software painter and cost + about the same. Defaults to the head because that is what MARK LIV shipped + with; anyone who preferred the older look can switch back in ⚙ and the + choice survives a restart. + """ + v = str(load_api_keys().get("hud_style", "face")).strip().lower() + return v if v in HUD_STYLES else "face" + + +def save_hud_style(style: str) -> None: + s = str(style or "").strip().lower() + _save_flag("hud_style", s if s in HUD_STYLES else "face") + + +# ── Live-session tuning ────────────────────────────────────────────────────── +# Everything here is optional and has a working default, so an untouched +# config behaves exactly like a configured one. Each value is also a way out: +# if a future model dislikes one of these, set it back and nothing else changes. + +def get_thinking_enabled() -> bool: + """Whether the Live model may spend tokens thinking before it answers. + + Off by default. A voice assistant is judged on how fast it starts talking, + and the reasoning that actually needs deliberation in this app is delegated + to the planning tools, which run on a separate non-Live model. + """ + return bool(load_api_keys().get("thinking_enabled", False)) + + +def save_thinking_enabled(enabled: bool) -> None: + _save_flag("thinking_enabled", enabled) + + +def get_turn_tuning() -> dict: + """How eagerly the server decides you have stopped speaking. + + OFF by default, and that default was earned. Cutting turns shorter looks + like a free speed win and is not: proactive audio has to judge whether an + utterance was even addressed to the assistant, and a turn clipped early + gives it less to judge, so it stays quiet — and the reply to your first + sentence only arrives once your second one has given it enough context. + That reads as the assistant being a turn behind, which is far worse than + the fraction of a second the tuning saves. + + Turn it on with "turn_tuning": {"enabled": true} if your own microphone and + speaking pace suit it. `silence_ms` is the one that is felt: the pause the + server sits through before accepting your turn is over. + """ + cfg = load_api_keys().get("turn_tuning") + cfg = cfg if isinstance(cfg, dict) else {} + + def _int(key, default, lo, hi): + try: + return max(lo, min(hi, int(cfg.get(key, default)))) + except (TypeError, ValueError): + return default + + return { + "enabled": bool(cfg.get("enabled", False)), + "silence_ms": _int("silence_ms", 550, 200, 3000), + "prefix_ms": _int("prefix_ms", 150, 0, 1000), + # "high" = quicker to decide speech has ended. + "end_sensitivity": str(cfg.get("end_sensitivity", "high")).lower(), + "start_sensitivity": str(cfg.get("start_sensitivity", "default")).lower(), + } + + +def save_turn_tuning(values: dict) -> None: + ensure_config_dir() + data: dict = {} + if CONFIG_FILE.exists(): + try: + data = json.loads(CONFIG_FILE.read_text(encoding="utf-8")) + except Exception: + data = {} + cur = data.get("turn_tuning") + cur = dict(cur) if isinstance(cur, dict) else {} + cur.update(values or {}) + data["turn_tuning"] = cur + CONFIG_FILE.write_text(json.dumps(data, indent=4), encoding="utf-8") + + +def get_proactive_audio_enabled() -> bool: + """Whether the model gets to decide an utterance was not aimed at it and + stay quiet. + + On by default — it is what stops the assistant answering the room. But it + is also the first thing to switch off if replies ever seem to arrive a turn + late: what looks like lag is usually the model having judged your previous + sentence as not addressed to it, and only changing its mind once the next + one arrives. + """ + return bool(load_api_keys().get("proactive_audio", True)) + + +def save_proactive_audio_enabled(enabled: bool) -> None: + _save_flag("proactive_audio", enabled) + + +MEDIA_RESOLUTIONS = ("default", "low", "medium", "high") + + +def get_media_resolution() -> str: + """How finely the model tokenises the screenshots and camera frames it is + sent. 'medium' keeps on-screen text readable at a fraction of the tokens a + full-resolution frame costs; 'low' is cheaper still but starts losing small + text, which is most of what screen captures are for.""" + v = str(load_api_keys().get("media_resolution", "medium")).strip().lower() + return v if v in MEDIA_RESOLUTIONS else "medium" + + +def save_media_resolution(value: str) -> None: + v = str(value or "").strip().lower() + _save_flag("media_resolution", v if v in MEDIA_RESOLUTIONS else "medium") + + +def _save_flag(key: str, value) -> None: + """Read-modify-write one key without disturbing the rest of the config.""" + ensure_config_dir() + data: dict = {} + if CONFIG_FILE.exists(): + try: + data = json.loads(CONFIG_FILE.read_text(encoding="utf-8")) + except Exception: + data = {} + data[key] = bool(value) if isinstance(value, bool) else value + CONFIG_FILE.write_text(json.dumps(data, indent=4), encoding="utf-8") + + def get_brief_enabled() -> bool: return load_api_keys().get("morning_brief_enabled", True) @@ -189,7 +333,7 @@ def get_plugin_enabled(plugin_name: str) -> bool: # and the settings UI renders + persists them WITHOUT any core edit — keeping the # drop-in model intact. Values live under plugin_config[][]. # A namespace defaults to the plugin name, but a suite of plugins (e.g. the -# printer control/watchdog/autoeject trio) can share ONE namespace. +# several printer plugins) can share ONE namespace. def get_plugin_config(namespace: str) -> dict: """All stored values for a namespace (empty dict if none set yet).""" cfg = load_api_keys().get("plugin_config") diff --git a/readme.md b/readme.md index a0b3b95..6397c4f 100644 --- a/readme.md +++ b/readme.md @@ -1,17 +1,21 @@ -# ⚙️ MARK LIII (53) +# ⚙️ MARK LIV (54) ### The Ultimate Cross-Platform Personal AI Assistant — By FatihMakes > 📺 **[Watch the full setup video on YouTube](https://www.youtube.com/@FatihMakes)** -A real-time voice AI that can hear, see, understand, and control your computer — on any OS. Supports Windows, macOS, and Linux. Built on the Gemini Live API for native audio streaming, delivering zero subscriptions and total digital autonomy. +A real-time voice AI that can hear, see, speak, and control your computer — on any OS. Supports Windows, macOS, and Linux. Built on the Gemini Live API for native audio streaming, delivering zero subscriptions and total digital autonomy. --- ## ✨ Overview -**MARK LIII is the hands-free & scalable release.** Say **"Hey Jarvis"** and it wakes; stay quiet and it slips back to sleep on its own — while asleep, your microphone never leaves the machine, so an off-hand *"I'll be right there"* to someone in the room no longer sets it off. Under the hood it now runs on the faster **Gemini 3.1 Flash Live** engine, and the moment you ask for something that takes a beat — analysing a file, searching the web — it answers instantly *("On it — going through that now…")* so you never wonder whether it heard you. +**MARK LIV is the release where JARVIS gets a face.** A holographic head sits at the centre of the HUD and **speaks your assistant's words with real lip-sync** — not a jaw flapping to the volume meter, but actual mouth shapes: lips closing on *m*, *b*, *p*, spreading on *i*, rounding on *u*. Brows ride the sentence, the eyes flick between fixation points, and it blinks. Turn the sound down and you can follow roughly what it just said. -It's also built to grow: every skill — bundled or drop-in — now **describes itself in its own file**, so adding a tool is a one-file operation and the core stays lean. +It ships as **zero extra dependencies and one 25 KB asset**. The face is real measured human geometry; everything else — the skull, the rig, the lighting — is generated at startup and drawn in software, so it looks identical on a gaming rig and a 2013 laptop, with no GPU driver in the loop. + +The face is also the fastest status indicator in the app: it looks away while thinking, meets your eyes while listening, and lets its lids fall while asleep. + +Underneath, Mark LIV rebuilt how the assistant knows itself — what it is, what machine it runs on, what it can do and, new, **what it cannot do** — all assembled from the live system at session start rather than written into a prompt that goes stale. It's not just an assistant — it's an extension of your digital life. @@ -22,10 +26,18 @@ It's not just an assistant — it's an extension of your digital life. ### Core Features | Feature | Description | |---|---| -| 🎙️ Wake Word | Local **"Hey Jarvis"** detection — sleeps until called, auto-sleeps after 2 min of silence, and never streams audio while asleep. Opt-in, one-click download, toggle & manual sleep/wake from the UI | +| 🧑‍🎤 Holographic Avatar | An animated human head in the HUD — real facial geometry, lit and drawn in software, no GPU or extra packages | +| 👄 Real Lip-Sync | ~50 mouth shapes a second from the audio's formants **and** the transcript — closures, spreads and rounds, not a volume meter | +| 🌍 Language-Free Mouth | Articulation is derived by Unicode reduction, so Latin, Cyrillic and Greek scripts all work from one rule set — and scripts that hide pronunciation fall back cleanly | +| 🙂 Facial Acting | Brows track the phrase, gaze saccades between fixations, natural blinking, a small nod on stressed syllables | +| 😐 Face as Status | Looks away while thinking, meets your eyes while listening, lids fall while asleep, glances down at new content | +| 🎚️ Push-to-Talk | Hold **Ctrl+Space** and the mic opens — closed the rest of the time. Truly global on Windows, window-scoped elsewhere | +| 🔇 Self-Echo Guard | Never answers its own last sentence: the tail of its own voice is recognised and dropped without muting you | +| 🪪 Runtime Self-Knowledge | Name, OS, abilities **and limits** are generated from the live system each session — rename it or add a plugin and it knows | +| 🎙️ Wake Word | Local **"Hey Jarvis"** detection — sleeps until called, auto-sleeps after 2 min of silence, and never streams audio while asleep | | ⚡ Instant Acknowledgment | Speaks a short, context-aware reply in **your language** the instant a longer task starts — no more silent waiting | | 🚀 Faster Live Engine | Runs on **Gemini 3.1 Flash Live** — roughly 2× faster time-to-first-word than the previous model | -| 🧩 Self-Describing Skills | Actions and plugins share one shape (`TOOL` / `PLUGIN` dict + `run()`), auto-discovered at launch — adding or moving a skill is a single file, no core edits | +| 🧩 Self-Describing Skills | Actions and plugins share one shape (`TOOL` / `PLUGIN` dict + `run()`), auto-discovered at launch — adding a skill is a single file | | 🧠 Recallable Memory | No size limit and nothing silently forgotten — the prompt carries what fits, the rest is looked up on demand from a local search | | 👁️ Memory Panel | See every fact JARVIS has stored about you, when it learned it, and delete any of it in one click | | ↩️ Undo | Take back what the assistant did — files it moved, renamed, created or wrote, and settings it changed | @@ -34,13 +46,13 @@ It's not just an assistant — it's an extension of your digital life. | 🔗 Session Continuity | A dropped connection, a voice change or a device change no longer wipes the conversation | | 🧩 Plugin System | Drop a single `.py` file into `plugins/` — JARVIS learns a new skill on next launch | | 🎙️ Real-time Voice | Ultra-low latency conversation in any language via Gemini Live API | -| 🎨 Live Theming | Recolour the entire HUD from a hue wheel or hex — applied instantly across every panel | -| 〰️ Reactive HUD | Waveform and reactor core pulse to real audio — your mic while listening, JARVIS while speaking | +| 🎨 Live Theming | Recolour the entire HUD from a hue wheel or hex — the avatar retints with it | +| 〰️ Reactive HUD | Waveform pulses to real audio — your mic while listening, JARVIS while speaking | | 🎙️ Voice Picker | Choose from 5 native Gemini voices and switch live from the UI — no restart | | ♾️ Unlimited Sessions | Sliding-window context compression — one conversation can last for hours | | 🖥️ System Control | Launch apps, adjust volume/brightness, WiFi, shortcuts, power — all by voice | | 🧩 Autonomous Tasks | High-level planning for complex multi-step goals via agent mode | -| 👁️ Visual Awareness | Real-time screen capture and webcam vision piped into your main Gemini session | +| 👁️ Visual Awareness | Screen capture and webcam vision piped into your main Gemini session, labelled by source | | 🧠 Persistent Memory | Deeply remembers projects, preferences, and personal context across sessions | | ⌨️ Hybrid Input | Seamlessly switch between keyboard typing and voice commands | | 🌅 Morning Briefing | On first boot: greets you, reads the time, recaps yesterday, and fetches live news | @@ -68,23 +80,86 @@ It's not just an assistant — it's an extension of your digital life. --- -## 🆕 What's New in Mark LIII +## 🆕 What's New in Mark LIV -Mark LIII is about making JARVIS **hands-free, faster, and easy to extend** — all universal: no hardcoded language, no bundled asset files, works the same on Windows, macOS and Linux. +No hardcoded language, no GPU requirement, no new dependencies — identical on Windows, macOS and Linux. -### 🎙️ Wake Word — "Hey Jarvis" -JARVIS can now sit quietly until you call it. Turn on **⚙ → WAKE WORD** (a one-click, opt-in download of a tiny local model) and it goes to sleep: the microphone is processed **only on your machine** by a local detector, and nothing is sent to the cloud until it hears **"Hey Jarvis."** Once awake it listens normally, then **auto-sleeps after 2 minutes** of silence. You can also **sleep/wake it by clicking** in the settings. Because it's a *local* gate, background chatter — *"I'm coming!"* to someone at home — never wakes it. It costs **zero** when off (the model isn't even loaded), and the detection runs in its own thread, so nothing else in the app slows down. +### The face -### ⚡ Instant Acknowledgment -No more silent gaps. When you ask for something that takes a moment — reading an uploaded file, a web/research search, building code — JARVIS **immediately** says one short, natural sentence *in your language* (*"Right away — going through that file now."*) and *then* runs the tool. Instant actions (opening an app, volume) stay snappy with no chatter. +#### 🧑‍🎤 A real head, rendered in software +The centre of the HUD is now an animated human head. Its face is **real measured human geometry** — MediaPipe's canonical face model, with actual eyelids, nostrils, lips and cheekbones. The skull, neck, jaw rig and lip rig are generated around it at startup, and the whole thing is lit and drawn with **QPainter**, which means: -### 🚀 Faster Live Engine — Gemini 3.1 Flash Live -The live session moved to **`gemini-3.1-flash-live-preview`**, cutting the time-to-first-word roughly in half while keeping tools, all five voices, transcription, session resumption and sliding-window compression intact. +* **no new dependencies** — it runs on the PyQt6 and numpy the app already needed; +* **no OpenGL, no shaders, no GPU driver** to disagree with you — a VM, a remote desktop session and an old integrated laptop all render the same picture; +* **one 25 KB asset**, and everything else is a formula. -### 🧩 Self-Describing Skills — a Scalable Core -Every bundled **action** now carries its own `TOOL` declaration in its own file (exactly like a drop-in **plugin's** `PLUGIN` dict), and the core auto-discovers them at launch. `main.py` no longer holds a giant list of tool definitions and dispatch branches — it shrank by hundreds of lines. Adding a new built-in skill, or promoting an `actions/*.py` file into a shareable plugin, is now just… moving a file. +It breathes, sways, blinks, and retints instantly when you change the HUD colour. -> Built on the Mark LI/LII foundation: the **🧩 Plugin System**, **♾️ Unlimited Sessions**, **🎨 Live Theming**, **〰️ Reactive HUD** and **🎙️ Voice Picker** are all still here. +#### 👄 Lip-sync you can actually read +The old mouth opened to the volume meter, five times a second. The new one produces **~50 mouth shapes per second** from two sources at once: + +* **the audio** — a formant read of each 20 ms slice gives openness (how far the jaw drops) and width (spread for *i* and *e*, rounded for *u* and *o*). This is physics, so it is language-independent by construction. +* **the transcript** — because no spectrum can tell you the lips are *closed*. /m/, /b/ and /p/ look identical to a filter bank and completely different on a face. The words supply the shape; the audio supplies the timing and the force. + +#### 🌍 One rule set, every alphabet +There is no per-language table. Every character is reduced to a bare Latin letter — Unicode decomposition strips accents (é, ü, ş, ğ, ế, ñ, å…), and Cyrillic and Greek transliterate — then articulation is looked up on the sound. + +**Turkish, English, German, French, Spanish, Polish, Vietnamese, Czech, Russian, Ukrainian and Greek all work from the same twenty-odd rules.** Scripts whose spelling doesn't reveal pronunciation (Arabic, Chinese, Japanese, Hindi, Korean, Hebrew, Thai) are detected automatically and the mouth runs on the audio-only shape — less detail, never wrong. Adding a language costs nothing, because there is nothing to add. + +#### 🙂 It acts while it talks +Brows ride the *phrase*, not the syllable, with a slow asymmetry between them. The eyes make real saccades between fixation points, more often while speaking. It blinks. Loud syllables tip the head. Everything relaxes to neutral in silence — and the mouth **only** moves for the assistant's own voice, never for yours. + +#### ◉ Two HUDs, one toggle +Not everyone wants a face looking back at them. ⚙ → **HUD** swaps the centrepiece between the animated head and a **reactor core** — a gauge ring, three arcs that turn at a rate the state sets, a spectrum ring driven by the real audio level, and a core that brightens with the voice. Both render in the same software painter and cost the same; the choice is taste, and it survives a restart. + +Nothing on the core moves for decoration. The rings speed up when JARVIS is thinking, the spikes are the actual waveform, and the colour is the state — the same language the face speaks, without the face. + +#### 🗣️ It answers before it works +Some replies used to open with three or four seconds of silence: not because a tool was slow, but because JARVIS was still *writing the tool call* — a set of quiz questions, the findings from a contract. The tool was instant; the composing was not, and from the user's side those are the same thing. + +The rule is now about the silence rather than the tool: if a gap would form, say one sentence naming what you are starting, then do it. It applies to anything that takes a moment to run **or** a moment to write, without a list of which tools those are. + +#### 😐 The face is a status light +You read a gaze faster than you read a word, so the head tells you what the assistant is doing before it says anything. It **looks away and holds it while thinking** — brows drawn down, blinking suppressed, the way concentration actually looks — **meets your eyes while listening**, and **lets its lids fall while asleep**. When something appears in the content panel below, it **glances down at it**: a wordless "that landed". + +### Talking to it + +#### 🎚️ Push-to-talk +Wake word is hands-free, but in a meeting or a noisy room a key is faster and never mishears. Turn on **⚙ → PUSH-TO-TALK** and the microphone stays **closed** until you hold **Ctrl+Space** — nothing leaves the machine while you are not holding it. Holding the chord also wakes the assistant, so it doubles as a silent alternative to saying the wake word. + +On **Windows** the chord is genuinely global: it works while any other application has focus, implemented by polling two virtual-key codes thirty times a second, with **no new dependency** and no message loop. On **macOS and Linux** there is no dependency-free way to read global key state, so the chord is bound inside the window instead — and the app **says so in the log** rather than pretending otherwise. + +#### 🔇 It no longer talks itself into replying +Writing audio to a device returns when the buffer *accepts* the sound, not when the speaker has finished with it — so for a moment after a reply "ends", it is still in the room. Streaming the microphone during that gap is how an assistant hears its own last sentence, decides it was addressed, and answers itself. + +Mark LIV holds a guard open across that gap, sized from the **device's own reported latency** rather than a guessed constant, so a machine with a large audio buffer gets a longer guard and one with a small buffer is not penalised. The microphone is **not muted** during it: both streams are reduced to band energies and as much of what was just played is subtracted from the microphone as fits, so only your assistant's own voice is dropped — replying the instant it stops still works. + +> Interrupting it mid-sentence by voice is built on the same machinery and is deliberately **switched off** in this release. It depends too much on the listener's room to ship without testing on real hardware. + +### How it understands itself + +#### 🪪 It knows what it is, and what it isn't +Who it is, what machine it runs on, what it can do and **what it cannot do** are assembled from the live system at session start — the configured name, the real OS, the tools actually discovered. Install a plugin and it knows it gained an ability; remove one and it stops claiming it. + +The **limits** half is the important one: it knows its sight is a single frame on demand rather than a live feed, that it acts on this machine only, and that anything outside its tool list should be stated plainly instead of improvised. + +All prompt wording lives in `core/prompt.txt` with `{tokens}` the app fills in — so you can rewrite the personality without touching Python, and a stray brace in your own wording can't break startup. + +### 🩹 Fixes +* Answers were sometimes **logged and spoken twice** — the Live API re-sends the tail of a transcript across the several turn-completes a tool call produces. Now de-duplicated at both the chunk and the flush level. +* Asking JARVIS to look at the screen produced **two different answers** — the flow made it speak once *before* the image arrived, so it improvised, and again after. The frame is now attached to the same exchange as its tool result: one turn, one answer, one fewer round trip. +* Screen captures were **unlabelled**, so a screenshot of this app — which has a face in the middle of it — could be read as a photo of the user. Images now carry their source. +* On a non-UTF-8 console (cp1254, cp1251, cp932…) the emoji in the status lines **crashed the session on startup**. Streams are reconfigured at launch, so it starts the same way in every locale. +* The HUD kept rendering the avatar **while the window was hidden or minimised**. It now stops, and resumes mid-motion rather than snapping. +* Activity-log lines were fixed amber and ignored the theme; they now follow the accent colour. +* Dependencies had no upper bounds, so the next major release of any of them would break every fresh clone. The load-bearing ones are now capped. +* The mouth **ran ahead of the words, then behind them**. Three separate faults compounded: each 200 ms of audio only produced 160 ms of mouth shapes, every new batch overwrote the previous one instead of continuing it, and the schedule was anchored to the moment audio was *handed to the device* rather than the moment it becomes *audible*. The mouth now follows one continuous playback clock — measured at **15 ms** of timing error whether the sound card buffers 100 ms or 500 ms. +* The jaw was driven by the **waveform meter**, which deliberately holds peaks so the bars don't flicker. That hold spanned exactly the consonants the mouth needed to close on. It now reads the audio's own 20 ms levels, so the gaps between words are real gaps. +* Mouth timing was measured in **frames rather than seconds**, so the same constants meant three different mouths at 60, 30 and 20 fps, and a closure shorter than one frame could vanish entirely. Timing is now in seconds and the mouth is stepped once per 20 ms of audio, not once per repaint. +* The **brows barely moved** — 6 px of travel on a 250 px head, because the rig weights halved an already small constant. Derived from the anatomy instead: 19 px. +* The activity log opened with **a dozen lines of plumbing** — one per plugin loaded, plus wake-word and briefing status. The console still carries the full boot transcript; the log now shows your conversation, state changes and anything you have to act on, and nothing else. + +> Built on the Mark LI–LIII foundation: the **🧩 Plugin System**, **♾️ Unlimited Sessions**, **🎨 Live Theming**, **🎙️ Wake Word** and **🧩 Self-Describing Skills** are all still here. --- @@ -194,21 +269,22 @@ It is held in memory only, deliberately: writing it to disk would make a fresh l | **LI** | Plugin system · affective dialog · proactive audio · unlimited sessions | | **LII** | Voice picker · live theming · reactive HUD · recallable memory · undo · real confirmation · audio device picker · session continuity | | **LIII** | Wake word · Gemini 3.1 Flash Live · instant acknowledgment · self-describing action/plugin architecture | +| **LIV** | Holographic avatar · viseme lip-sync · facial acting · face-as-status · push-to-talk · self-echo guard · runtime self-knowledge & limits | | *shared* | The last five above also shipped to LIII, LIV and LV at the same time — moving up a Mark never loses them | -| **LIV+** | Plugin files: email · quiz mode · calendar · home assistant · 3D-printer · and more | +| **LV+** | Interrupt by voice · conversation history · plugin files: email · quiz mode · calendar · home assistant · 3D-printer | --- ## ⚡ Quick Start ```bash -git clone https://github.com/FatihMakes/Mark-LIII.git -cd Mark-LIII +git clone https://github.com/FatihMakes/Mark-LIV.git +cd Mark-LIV python setup.py # installs deps for YOUR OS + the browser automation engine python main.py ``` -`setup.py` only ever installs what your operating system needs — the Windows-only libraries are skipped automatically on macOS and Linux (and vice-versa). Prefer to do it by hand? `pip install -r requirements.txt` works too. +`setup.py` only ever installs what your operating system needs — the Windows-only libraries are skipped automatically on macOS and Linux, and vice-versa. It also checks your Python version up front, so a wrong interpreter fails with a sentence instead of a wall of pip output. Prefer to do it by hand? `pip install -r requirements.txt` works too. > ⚠️ **Installation Note:** If you hit a `ModuleNotFoundError` for an OS-specific package, install it with `pip install `. The optional **wake word** engine is *not* installed here — grab it in one click from **⚙ → WAKE WORD** inside the app. @@ -219,10 +295,11 @@ python main.py | Requirement | Details | | --- | --- | | **OS** | Windows 10/11, macOS, or Linux | -| **Python** | 3.11 or 3.12 | +| **Python** | 3.11, 3.12 or 3.13 | | **Microphone** | Required for voice interaction (and for the "Hey Jarvis" wake word) | | **Speakers** | Required for voice replies | | **API Key** | Free Gemini API key (entered on first launch → `config/api_keys.json`) | +| **GPU** | **Not required.** The avatar is rendered in software | | **Wake word** *(optional)* | One-click download from ⚙ → WAKE WORD (`openwakeword`, a few MB, fully local) | --- @@ -230,17 +307,22 @@ python main.py ## 🗂️ Project Structure ``` -Mark LIII/ -├── main.py # Core loop — Gemini Live session, audio I/O, wake/sleep state, tool dispatch -├── ui.py # PyQt6 HUD — reactive waveform, log panel, settings drawer, plugin manager, camera feed -├── setup.py # OS-aware installer (skips wrong-OS dependencies) +Mark LIV/ +├── main.py # Core loop — Gemini Live session, audio I/O, viseme extraction, tool dispatch +├── ui.py # PyQt6 HUD — avatar canvas, waveform, log panel, settings drawer, camera feed +├── setup.py # OS-aware installer (skips wrong-OS dependencies, checks your Python) +├── .gitignore # Keeps your API key, TLS key and memories out of the repository ├── plugins/ +│ ├── quiz.py # Interactive quiz — JARVIS writes the questions, you answer on screen +│ ├── document_review.py # Contracts and policies in plain language, ordered by what matters +│ ├── _google_core.py # Shared OAuth for the Gmail/Calendar plugins (not a plugin itself) +│ ├── _printer_core.py # Shared printer connectivity (not a plugin itself) │ ├── _template.py # Copy this to write a new plugin — one file, drop in, done │ └── ... # Drop-in skills (each self-describes via a PLUGIN dict + run()) ├── actions/ # Bundled skills — each self-describes via a TOOL dict + handler │ ├── web_search.py # Gemini + DDG parallel search (news, research, price, compare) │ ├── screen_processor.py # Screen & webcam capture for vision -│ ├── background_monitor.py # User-configured topic watching — daily DDG check, no crypto +│ ├── background_monitor.py # User-configured topic watching — daily DDG check │ ├── proactive.py # Proactive 2.0 — time/context/rotation-aware check-ins │ ├── reminder.py # OS-native scheduled notifications │ ├── system_monitor.py # CPU / RAM / GPU / temperature telemetry @@ -261,9 +343,15 @@ Mark LIII/ ├── memory/ │ ├── memory_manager.py # Load/save long_term.json — sessions, monitors, identity │ ├── config_manager.py # api_keys.json access — key, OS, name, voice, colour, toggles -│ └── long_term.json # Persistent store: identity, preferences, projects, sessions, monitors +│ └── long_term.json # Persistent store — created on first run ├── core/ -│ ├── prompt.txt # Assistant personality and tool-routing rules +│ ├── prompt.txt # All prompt wording — {tokens} are filled from the live system at startup +│ ├── avatar.py # Avatar renderer — lighting, pose, expression, mouth (QPainter) +│ ├── avatar_mesh.py # Head geometry — loads the face, generates skull/neck/rigs +│ ├── face_model.obj # The face itself (MediaPipe canonical model, Apache-2.0, 25 KB) +│ ├── viseme.py # Transcript → mouth shapes, fused with the audio's timing +│ ├── echo.py # Tells your voice from the assistant's own echo; self-calibrating +│ ├── hotkey.py # Push-to-talk chord — global on Windows, windowed fallback elsewhere │ ├── undo.py # One shared undo stack — actions register how to reverse themselves │ ├── confirm.py # Irreversible-action gate — the token is issued by the UI, not the model │ ├── audio_devices.py # Microphone / speaker list — filtered, measured, resolved by name @@ -271,11 +359,36 @@ Mark LIII/ │ ├── action_loader.py # Bundled-action engine — the built-in twin of plugin_loader │ └── wake_word.py # Local "Hey Jarvis" detector — own thread, offline, opt-in └── config/ - └── api_keys.json # API key, OS setting, assistant name, user name, voice, UI colour, toggles + ├── api_keys.json # API key, name, voice, colour, toggles — created on first launch (git-ignored) + └── certs/ # Self-signed TLS pair for the phone dashboard — generated locally (git-ignored) ``` --- +## 🙏 Third-Party Assets + +| Asset | Source | Licence | +| --- | --- | --- | +| `core/face_model.obj` | [MediaPipe](https://github.com/google-ai-edge/mediapipe) canonical face model — 468 vertices of measured human face geometry | Apache License 2.0 | + +--- + +## 🔒 Your Data + +Everything stays on your machine. There is no MARK server, no telemetry and no account. + +| What | Where | Notes | +|---|---|---| +| Gemini API key, plugin credentials | `config/api_keys.json` | **Plaintext.** Anyone with your user account can read it. Treat it like a password file. | +| Dashboard TLS certificate + private key | `config/certs/` | Generated locally, self-signed, never leaves the machine. | +| What the assistant remembers about you | `memory/long_term.json` | Delete the file to make it forget everything. | + +All three are listed in `.gitignore`, so a fork or a pull request cannot leak them by accident. **If you have already committed `config/api_keys.json` anywhere public, revoke that key** at [aistudio.google.com](https://aistudio.google.com/app/apikey) and generate a new one — removing the file in a later commit does not remove it from the history. + +Your voice is streamed to Google's Gemini Live API while a session is open; that is the one thing that leaves your computer, and it stops when you mute or close the app. + +--- + ## ⚠️ License Personal and non-commercial use only. diff --git a/requirements.txt b/requirements.txt index 482ec54..401132d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,59 +1,92 @@ -# ── Core: UI, audio, Gemini Live ────────────────────────────────────────────── -PyQt6 -sounddevice -numpy -google-genai>=2.8.0 - -# ── Web search & browser control ────────────────────────────────────────────── -requests -beautifulsoup4 -duckduckgo-search -playwright - -# ── Automation / input control ──────────────────────────────────────────────── -pyautogui -pyperclip -pygetwindow - -# ── Vision & media ──────────────────────────────────────────────────────────── -pillow -opencv-python -mss - -# ── System, files & documents ───────────────────────────────────────────────── -psutil -send2trash -youtube-transcript-api -python-pptx -openpyxl - -# ── Remote dashboard (control from your phone) ──────────────────────────────── -fastapi -uvicorn[standard] -cryptography -python-multipart -qrcode[pil] - -# ── Plugin extras: Gmail / Google Calendar (OAuth) ──────────────────────────── -google-api-python-client -google-auth-oauthlib - -# ── Plugin extras: Tuya / Smart Life smart light (home_assistant) ───────────── -tinytuya - -# ── Windows-only (pip skips these automatically on macOS / Linux) ───────────── -comtypes; sys_platform == "win32" -pycaw; sys_platform == "win32" -win10toast; sys_platform == "win32" -pywinauto; sys_platform == "win32" -pywin32; sys_platform == "win32" -wmi; sys_platform == "win32" - -# ── Optional: local wake word ("Hey Jarvis") ────────────────────────────────── -# Deliberately NOT installed by default — it's opt-in. Enable it (and download -# the tiny model) with one click from ⚙ → WAKE WORD in the app, or install it -# yourself with: pip install openwakeword -# -# macOS / Linux OS actions (volume, brightness, reminders) shell out to native -# tools (pactl / brightnessctl / osascript / systemd), so no extra Python -# packages are needed there — nothing Windows-specific is ever pulled in. +# ── MARK LIV — Python dependencies ──────────────────────────────────────────── +# pip skips any line whose sys_platform marker does not match your OS, so this +# one file serves Windows, macOS and Linux. `python setup.py` runs it for you. + +# Version bounds: the lower bound is what the app is developed against, the +# upper bound stops the next MAJOR release of a dependency from breaking every +# fresh clone on the day it ships. Minor and patch updates flow in freely. + +# ── Core: UI, audio, Gemini Live ────────────────────────────────────────────── +# PyQt6 and numpy also carry the holographic avatar: the head is rendered in +# software with QPainter and posed with numpy, so the face costs no extra +# package, no OpenGL and no GPU. Nothing below was added for it. +PyQt6>=6.6,<7 +sounddevice>=0.4,<1 +numpy>=1.24,<3 +google-genai>=2.8.0,<3 + +# ── Web search & browser control ────────────────────────────────────────────── +requests>=2.31,<3 +beautifulsoup4>=4.12,<5 +playwright>=1.40,<2 + +# ── Automation / input control ──────────────────────────────────────────────── +pyautogui +pyperclip +pygetwindow + +# ── Vision & media ──────────────────────────────────────────────────────────── +pillow>=10,<13 +opencv-python>=4.8,<5 +mss>=9,<11 + +# ── System, files & documents ───────────────────────────────────────────────── +psutil>=5.9,<8 +send2trash +youtube-transcript-api +python-pptx +openpyxl + +# ── Remote dashboard (control from your phone) ──────────────────────────────── +fastapi>=0.110,<1 +uvicorn[standard]>=0.27,<1 +cryptography>=42,<50 +python-multipart>=0.0.9,<1 +qrcode[pil]>=7,<9 + +# ── Plugin extras: Gmail / Google Calendar (OAuth) ──────────────────────────── +google-api-python-client +google-auth-oauthlib + +# ── Plugin extras: Tuya / Smart Life smart light (home_assistant) ───────────── +tinytuya + +# ── Windows-only (pip skips these automatically on macOS / Linux) ───────────── +comtypes; sys_platform == "win32" +pycaw; sys_platform == "win32" +win10toast; sys_platform == "win32" +pywinauto; sys_platform == "win32" +pywin32; sys_platform == "win32" +wmi; sys_platform == "win32" + +# ── Documents, media & hardware readouts ────────────────────────────────────── +# Every one of these is imported inside a try/except at the point of use, so a +# missing package disables exactly one feature and never stops the app. They are +# declared because a feature that silently does nothing is worse than a slightly +# longer install. +ddgs>=9,<10 # web_search — the maintained successor to + # duckduckgo-search, which is tried first and + # falls back to the old name automatically +pdfplumber # file_processor — PDF text and tables +PyPDF2 # file_processor — PDF fallback reader +python-docx # file_processor — .docx reading +paho-mqtt # printer_control plugin — MQTT printers +pynvml # system_monitor + the HUD's GPU meter (NVIDIA) + +# ── Optional: heavy extras, install only if you want the feature ────────────── +# Left out on purpose — together they are several hundred megabytes, which is a +# lot to download for a fresh clone that may never use them. +# +# pip install pandas # file_processor: spreadsheet/CSV analysis +# pip install pydub # file_processor: audio metadata and conversion +# # (also needs ffmpeg on PATH for most formats) +# pip install mediapipe # pushup_counter: elbow-angle rep detection + +# ── Optional: local wake word ("Hey Jarvis") ────────────────────────────────── +# Deliberately NOT installed by default — it's opt-in. Enable it (and download +# the tiny model) with one click from ⚙ → WAKE WORD in the app, or install it +# yourself with: pip install openwakeword +# +# macOS / Linux OS actions (volume, brightness, reminders) shell out to native +# tools (pactl / brightnessctl / osascript / systemd), so no extra Python +# packages are needed there — nothing Windows-specific is ever pulled in. diff --git a/setup.py b/setup.py index f16f856..d7bba74 100644 --- a/setup.py +++ b/setup.py @@ -1,72 +1,129 @@ -""" -MARK LIII — one-time setup. - -Installs the Python dependencies for THIS operating system only: the OS-specific -packages in requirements.txt carry `sys_platform` markers, so a macOS or Linux -user never pulls Windows-only libraries (and vice-versa). Then it fetches the -Playwright browsers needed for web automation (current-OS builds only). - -The optional local wake word ("Hey Jarvis") is NOT installed here — it's a -one-click, opt-in download from ⚙ → WAKE WORD inside the app. -""" -import platform -import subprocess -import sys -from pathlib import Path - -OS = platform.system() # "Windows" | "Darwin" | "Linux" - - -def _run(label: str, args: list[str]) -> None: - print(f"\n▶ {label}") - subprocess.run(args, check=True) - - -def main() -> None: - print(f"⚙ MARK LIII setup — detected OS: {OS or 'unknown'}") - - # requirements.txt filters OS-specific extras by itself via pip markers. - _run("Installing Python dependencies (OS-specific extras auto-filtered)…", - [sys.executable, "-m", "pip", "install", "-r", "requirements.txt"]) - - # Chromium covers Chrome/Edge/Opera/Brave/Vivaldi; Firefox for Firefox. - # (Safari automation additionally needs: python -m playwright install webkit) - _run("Installing Playwright browsers (chromium + firefox)…", - [sys.executable, "-m", "playwright", "install", "chromium", "firefox"]) - - # ── OS-specific post-install notes ──────────────────────────────────────── - if OS == "Windows": - try: - import win32com.client # noqa: F401 - except ImportError: - postinstall = Path(sys.executable).parent / "Scripts" / "pywin32_postinstall.py" - print( - "\n⚠️ pywin32 did not register correctly — desktop-shortcut " - "creation will use a slower fallback. To fix it, run:\n" - f' "{sys.executable}" -m pip install --force-reinstall pywin32\n' - f' "{sys.executable}" "{postinstall}" -install' - ) - elif OS == "Linux": - print( - "\nℹ️ Linux note — a few voice-controlled OS actions shell out to " - "native tools. Install the ones you'll use via your package manager:\n" - " • volume → pulseaudio-utils (pactl)\n" - " • brightness → brightnessctl\n" - " • reminders → systemd (systemd-run) or 'at'\n" - " • open URLs → xdg-utils (xdg-open)" - ) - elif OS == "Darwin": - print( - "\nℹ️ macOS note — volume, brightness and reminders use the built-in " - "'osascript' / LaunchAgents, so no extra tools are required.\n" - " For Safari automation only: python -m playwright install webkit" - ) - - print("\n✅ Setup complete!") - print(" 1) Launch it: python main.py") - print(" 2) Paste your free Gemini API key when the setup screen appears.") - print(" 3) (Optional) Enable 'Hey Jarvis' from ⚙ → WAKE WORD.") - - -if __name__ == "__main__": - main() +""" +MARK LIV — one-time setup. + +Installs the Python dependencies for THIS operating system only: the OS-specific +packages in requirements.txt carry `sys_platform` markers, so a macOS or Linux +user never pulls Windows-only libraries (and vice-versa). Then it fetches the +Playwright browsers needed for web automation (current-OS builds only). + +Two things it deliberately does NOT install: + * the optional local wake word ("Hey Jarvis") — one-click, opt-in, from + ⚙ → WAKE WORD inside the app; + * anything for the avatar — the holographic head renders in software on the + PyQt6 and numpy already listed here. No GPU, no OpenGL, no extra packages. +""" +import platform +import subprocess +import sys +from pathlib import Path + +OS = platform.system() # "Windows" | "Darwin" | "Linux" +HERE = Path(__file__).resolve().parent + +MIN_PY = (3, 11) # hard floor: below this the syntax used here won't parse +MAX_PY = (3, 13) # highest version this is actually tested on + + +def _run(label: str, args: list[str]) -> None: + print(f"\n▶ {label}") + subprocess.run(args, check=True) + + +def _check_python() -> None: + """Fail immediately and clearly rather than deep inside a pip resolver. + + A wrong interpreter is the single most common way this install goes sideways, + and the error it produces on its own names a wheel, not the real problem. + """ + v = sys.version_info[:2] + if v > MAX_PY: + # Newer is a warning, not a wall. Turning away someone who installed + # today's Python is a worse first impression than a version that + # turns out to work fine, and if a wheel really is missing pip says + # so plainly. + print(f"\n⚠️ Python {v[0]}.{v[1]} is newer than the " + f"{MAX_PY[0]}.{MAX_PY[1]} this is tested on. Continuing — if a " + f"package has no wheel yet, install Python " + f"{MAX_PY[0]}.{MAX_PY[1]} and run setup with that.") + return + if v < MIN_PY: + print(f"\n❌ Python {v[0]}.{v[1]} detected — MARK LIV needs at " + f"least Python {MIN_PY[0]}.{MIN_PY[1]}.") + print(" Install a supported version and run setup with it, e.g.:") + print(f" py -{MIN_PY[0]}.{MIN_PY[1]} setup.py (Windows)") + print(f" python{MIN_PY[0]}.{MIN_PY[1]} setup.py (macOS / Linux)") + sys.exit(1) + + +def _check_assets() -> None: + """The avatar's face is a shipped file; a truncated clone should say so.""" + face = HERE / "core" / "face_model.obj" + if not face.exists() or face.stat().st_size < 4096: + print( + "\n⚠️ core/face_model.obj is missing or truncated — the avatar will " + "fall back to the plain glowing core.\n" + " Re-clone the repository, or fetch that one file again." + ) + + +def main() -> None: + print(f"⚙ MARK LIV setup — detected OS: {OS or 'unknown'}, " + f"Python {sys.version_info[0]}.{sys.version_info[1]}") + _check_python() + + # requirements.txt filters OS-specific extras by itself via pip markers. + _run("Installing Python dependencies (OS-specific extras auto-filtered)…", + [sys.executable, "-m", "pip", "install", "-r", "requirements.txt"]) + + # Chromium covers Chrome/Edge/Opera/Brave/Vivaldi; Firefox for Firefox. + # (Safari automation additionally needs: python -m playwright install webkit) + # Not fatal: these are a few hundred megabytes from a CDN that a corporate + # network or a flaky connection can refuse, and everything except browser + # automation works without them. Failing the whole install there would send + # a user away from a working app. + try: + _run("Installing Playwright browsers (chromium + firefox)…", + [sys.executable, "-m", "playwright", "install", "chromium", "firefox"]) + except (subprocess.CalledProcessError, FileNotFoundError) as e: + print(f"\n⚠️ Playwright browsers were not installed ({e}).") + print(" Everything except browser automation works. Retry later with:") + print(f' {sys.executable} -m playwright install chromium firefox') + + _check_assets() + + # ── OS-specific post-install notes ──────────────────────────────────────── + if OS == "Windows": + try: + import win32com.client # noqa: F401 + except ImportError: + postinstall = Path(sys.executable).parent / "Scripts" / "pywin32_postinstall.py" + print( + "\n⚠️ pywin32 did not register correctly — desktop-shortcut " + "creation will use a slower fallback. To fix it, run:\n" + f' "{sys.executable}" -m pip install --force-reinstall pywin32\n' + f' "{sys.executable}" "{postinstall}" -install' + ) + elif OS == "Linux": + print( + "\nℹ️ Linux note — a few voice-controlled OS actions shell out to " + "native tools. Install the ones you'll use via your package manager:\n" + " • volume → pulseaudio-utils (pactl)\n" + " • brightness → brightnessctl\n" + " • reminders → systemd (systemd-run) or 'at'\n" + " • open URLs → xdg-utils (xdg-open)" + ) + elif OS == "Darwin": + print( + "\nℹ️ macOS note — volume, brightness and reminders use the built-in " + "'osascript' / LaunchAgents, so no extra tools are required.\n" + " For Safari automation only: python -m playwright install webkit" + ) + + print("\n✅ Setup complete!") + print(" 1) Launch it: python main.py") + print(" 2) Paste your free Gemini API key when the setup screen appears.") + print(" 3) (Optional) Enable 'Hey Jarvis' from ⚙ → WAKE WORD.") + + +if __name__ == "__main__": + main() diff --git a/ui.py b/ui.py index e78f867..4d6fd61 100644 --- a/ui.py +++ b/ui.py @@ -19,7 +19,7 @@ else: _WIN_HIDE: dict = {} from PyQt6.QtCore import ( - QEasingCurve, QMimeData, QObject, QParallelAnimationGroup, QPointF, + QEasingCurve, QLineF, QMimeData, QObject, QParallelAnimationGroup, QPointF, QPropertyAnimation, QRect, QRectF, QSize, Qt, QTimer, QUrl, pyqtSignal, ) from PyQt6.QtGui import ( @@ -33,14 +33,11 @@ from PyQt6.QtWidgets import ( QStackedWidget, QTextEdit, QVBoxLayout, QWidget, QProgressBar, ) -# ── Which Mark this is ─────────────────────────────────────────────────────── -# One constant, read by the window title, the header badge and the PROTOCOL -# panel. It used to be typed separately in each of those places, and they drifted: -# Mark 52 and 53 shipped showing "PROTOCOL XLIX" — the number from Mark 49 — and -# Mark 55 shipped titled "MARK 54". Deriving the protocol from the name means a -# release bump is this one line. -APP_VERSION = "MARK LIII" -APP_PROTOCOL = APP_VERSION.split()[-1] +try: + from core.avatar import HoloAvatar +except Exception: # pragma: no cover — HUD must never die over cosmetics + HoloAvatar = None + def _base_dir() -> Path: if getattr(sys, "frozen", False): @@ -60,6 +57,11 @@ def _read_full_config() -> dict: return {} +# Single source of truth for the release name — the window title, the header +# badge and the readme must never disagree again. +APP_VERSION = "MARK LIV" +APP_PROTOCOL = APP_VERSION.split()[-1] + _DEFAULT_W, _DEFAULT_H = 980, 700 _MIN_W, _MIN_H = 820, 580 _LEFT_W = 148 @@ -389,36 +391,53 @@ class HudCanvas(QWidget): self.state = "INITIALISING" self._assistant_name = assistant_name + # The holographic head that fills the HUD. If it could not be imported + # we fall back to the old glowing core so the panel is never empty. + self._avatar = None + if HoloAvatar is not None: + try: + self._avatar = HoloAvatar() + except Exception: + self._avatar = None + + # Which centrepiece to draw. Read once here and changed live by the + # settings toggle; the avatar object is kept either way so switching + # back is instant and costs no reload. + try: + from memory.config_manager import get_hud_style + self.hud_style = get_hud_style() + except Exception: + self.hud_style = "face" + self._core_phase = 0.0 + self._tick = 0 self._scale = 1.0 self._tgt_scale = 1.0 self._halo = 55.0 self._tgt_halo = 55.0 self._last_t = time.time() - self._scan = 0.0 - self._scan2 = 180.0 - self._rings = [0.0, 120.0, 240.0] - self._pulses: list[float] = [0.0, 50.0, 100.0] + self._step_t = time.time() self._blink = True self._blink_tick = 0 - self._particles: list[list[float]] = [] - self._face_px: QPixmap | None = None + # Rescaled-face cache: the smooth rescale is expensive, so we keep the # last result and only rebuild it when the (quantised) size changes. - self._face_cache: QPixmap | None = None - self._face_cache_sz = -1 + # Static grid-dot layer, pre-rendered once per size/theme into a pixmap # so paintEvent blits it in one call instead of thousands of drawPoint()s. self._grid_cache: QPixmap | None = None self._grid_key = None # Repaint throttle counter (idle frames drop to ~20 Hz — see _step()). self._paint_tick = 0 - self._load_face(face_path) # Live audio reactivity: _live_amp is written from the audio threads # (0.0–1.0), _amp_disp is the smoothed value the paint code reads. self._live_amp = 0.0 self._amp_disp = 0.0 + # (frames, start_time, hop) posted by the playback thread — see + # push_visemes(). None means "no schedule; use the plain level". + self._visemes = None + self._vis_i = None # first schedule frame not yet handed to the mouth self._base_scale = 1.0 # slow "breathing" target; amp is added per-frame self._base_halo = 55.0 @@ -426,6 +445,67 @@ class HudCanvas(QWidget): self._tmr.timeout.connect(self._step) self._tmr.start(16) + def glance(self, dx: float, dy: float, hold: float = 1.1) -> None: + """Ask the avatar to look somewhere for a moment (see HoloAvatar.glance).""" + try: + if self._avatar is not None: + self._avatar.glance(dx, dy, hold) + except Exception: + pass + + def push_visemes(self, frames, hop: float, at: float) -> None: + """Thread-safe: hand over a schedule of (level, openness, width) frames. + + The playback thread writes up to 200 ms of audio in one go, so a single + averaged level would only move the mouth five times a second — enough to + flap, nowhere near enough to articulate. It instead posts the whole + slice's worth of 20 ms frames here and `_step()` plays them out against + the wall clock, in step with the audio going to the speakers. + + `at` is the wall-clock time this batch will *begin to sound*, which the + caller tracks as a playback cursor. It is not the time of the call, and + the difference is the whole point: `stream.write` returns once the buffer + accepts the samples, so consecutive batches are handed over far faster + than they play. Anchoring each one to "now" made every batch start while + its predecessor was still sounding, so each schedule replaced the last + after a couple of frames and the mouth only ever played the opening + instant of every 200 ms — the reason it did not match the words. + + Successive batches are therefore *appended* into one continuous + timeline, not swapped in. A paragraph is one schedule; the mouth stops + falling into a gap at every chunk boundary and having to climb back out. + """ + try: + if not frames: + return + hop = max(1e-3, float(hop)) + at = float(at) + new = list(frames) + cur = self._visemes + if cur is not None: + old, t0, ohop = cur + if abs(ohop - hop) < 1e-6: + # Where in the existing timeline does this batch land? + i = int(round((at - t0) / hop)) + if 0 <= i <= len(old) + 1: + # Continues (or slightly overlaps) what is already + # queued: extend rather than restart. Drop whatever has + # already been played so the list cannot grow without + # bound over a long reply. + merged = old[:i] + new + played = int((time.time() - t0) / hop) - 2 + if played > 60: + merged = merged[played:] + t0 += played * hop + if self._vis_i is not None: + self._vis_i = max(0, self._vis_i - played) + self._visemes = (merged, t0, hop) + return + self._visemes = (new, at, hop) + self._vis_i = None + except Exception: + pass + def set_audio_level(self, level: float) -> None: """Thread-safe entry point for the audio threads. Stores the louder of the incoming level and the current value so brief gaps between chunks @@ -441,26 +521,6 @@ class HudCanvas(QWidget): if lv > self._live_amp: self._live_amp = lv - def _load_face(self, path: str): - try: - from PIL import Image, ImageDraw - import io - img = Image.open(path).convert("RGBA") - sz = min(img.size) - img = img.resize((sz, sz), Image.LANCZOS) - mk = Image.new("L", (sz, sz), 0) - ImageDraw.Draw(mk).ellipse((2, 2, sz - 2, sz - 2), fill=255) - img.putalpha(mk) - buf = io.BytesIO() - img.save(buf, format="PNG") - px = QPixmap(); px.loadFromData(buf.getvalue()) - self._face_px = px - except Exception: - self._face_px = None - # New source image → drop the rescaled cache so it rebuilds on next paint. - self._face_cache = None - self._face_cache_sz = -1 - def _make_grid(self, W: int, H: int) -> QPixmap: """Pre-render the static grid-dot background into a transparent pixmap so paintEvent can blit it once per frame instead of running a nested @@ -480,69 +540,81 @@ class HudCanvas(QWidget): now = time.time() # ── Live audio reactivity ──────────────────────────────────────────── + # A viseme schedule, if one is playing, gives both the level and the + # mouth shape for this exact instant; otherwise fall back to the peak + # level the audio threads pushed in. + v_open = v_wide = v_level = None + v_seq = None + sched = self._visemes + if sched is not None: + frames, t0, hop = sched + i = int((now - t0) / hop) + if 0 <= i < len(frames): + # Hand over *every* frame since the last tick, not just the one + # under the cursor. This timer runs at 60 Hz but the paint is + # throttled and the machine may be busy, so a tick can span two + # or three 20 ms frames — and a consonant closure is only two + # frames long. Sampling one and discarding the rest is how the + # closures between words went missing. + j = self._vis_i if self._vis_i is not None else i + v_seq = frames[max(0, j):i + 1] + self._vis_i = max(j, i + 1) + v_level, v_open, v_wide = frames[i] + if v_seq: + peak = max(f[0] for f in v_seq) + if peak > self._live_amp: + self._live_amp = peak + elif i >= len(frames): + self._visemes = None # schedule spent + self._vis_i = None + # Audio threads push peaks into _live_amp; decay it toward silence so # gaps between chunks fade out instead of freezing, then smooth it. self._live_amp *= 0.86 self._amp_disp += (self._live_amp - self._amp_disp) * 0.45 amp = self._amp_disp - # Slow "breathing" base target (random shimmer), refreshed on a timer. - if now - self._last_t > (0.12 if self.speaking else 0.5): - if self.speaking: - self._base_scale = 1.03 - self._base_halo = 122.0 - elif self.muted: - self._base_scale = random.uniform(0.998, 1.002) - self._base_halo = random.uniform(15, 28) - else: - self._base_scale = random.uniform(1.001, 1.008) - self._base_halo = random.uniform(48, 68) - self._last_t = now + # The avatar animates off the very same smoothed level the waveform + # uses — one audio source, so the mouth can never drift out of sync. + dt = now - self._step_t + self._step_t = now + # Integrated, not derived from absolute time: multiplying wall-clock by + # a rate that changes with state jumps the rings the instant JARVIS + # starts talking. Same lesson the head's sway taught. + self._core_phase += min(0.10, max(0.0, dt)) - # Every frame, the live audio level lifts the target on top of the base - # — this is what makes the core visibly pulse to the actual voice. - if self.muted: - self._tgt_scale, self._tgt_halo = self._base_scale, self._base_halo - elif self.speaking: - self._tgt_scale = self._base_scale + amp * 0.13 - self._tgt_halo = self._base_halo + amp * 95.0 + if self._avatar is not None and self.hud_style == "face": + self._avatar.step(dt, amp, speaking=self.speaking, + muted=self.muted, state=self.state, + v_open=v_open, v_wide=v_wide or 0.0, + v_level=v_level, v_seq=v_seq, + v_hop=(sched[2] if sched is not None else 0.02)) else: - self._tgt_scale = self._base_scale + amp * 0.06 - self._tgt_halo = self._base_halo + amp * 75.0 + # Fallback core: slow "breathing" base target, lifted by the level. + if now - self._last_t > (0.12 if self.speaking else 0.5): + if self.speaking: + self._base_scale = 1.03 + self._base_halo = 122.0 + elif self.muted: + self._base_scale = random.uniform(0.998, 1.002) + self._base_halo = random.uniform(15, 28) + else: + self._base_scale = random.uniform(1.001, 1.008) + self._base_halo = random.uniform(48, 68) + self._last_t = now - sp = 0.38 if self.speaking else (0.30 if amp > 0.02 else 0.15) - self._scale += (self._tgt_scale - self._scale) * sp - self._halo += (self._tgt_halo - self._halo) * sp + if self.muted: + self._tgt_scale, self._tgt_halo = self._base_scale, self._base_halo + elif self.speaking: + self._tgt_scale = self._base_scale + amp * 0.13 + self._tgt_halo = self._base_halo + amp * 95.0 + else: + self._tgt_scale = self._base_scale + amp * 0.06 + self._tgt_halo = self._base_halo + amp * 75.0 - # Rings/scanners spin faster while speaking, reacting to loudness. - boost = 1.0 + amp * 1.6 - speeds = ([1.3, -0.9, 2.0] if self.speaking else [0.55, -0.35, 0.9]) - for i, spd in enumerate(speeds): - self._rings[i] = (self._rings[i] + spd * boost) % 360 - - self._scan = (self._scan + (3.0 if self.speaking else 1.3) * boost) % 360 - self._scan2 = (self._scan2 + (-2.0 if self.speaking else -0.75) * boost) % 360 - - fw = min(self.width(), self.height()) - lim = fw * 0.74 - spd = 4.2 if self.speaking else 2.0 - self._pulses = [r + spd for r in self._pulses if r + spd < lim] - if len(self._pulses) < 3 and random.random() < (0.07 if self.speaking else 0.025): - self._pulses.append(0.0) - - if self.speaking and random.random() < 0.28: - cx, cy = self.width() / 2, self.height() / 2 - ang = random.uniform(0, 2 * math.pi) - r_s = fw * 0.28 - self._particles.append([ - cx + math.cos(ang) * r_s, cy + math.sin(ang) * r_s, - math.cos(ang) * random.uniform(0.9, 2.4), - math.sin(ang) * random.uniform(0.9, 2.4) - 0.4, 1.0, - ]) - self._particles = [ - [p[0]+p[2], p[1]+p[3], p[2]*0.97, p[3]*0.97, p[4]-0.028] - for p in self._particles if p[4] > 0 - ] + sp = 0.38 if self.speaking else (0.30 if amp > 0.02 else 0.15) + self._scale += (self._tgt_scale - self._scale) * sp + self._halo += (self._tgt_halo - self._halo) * sp self._blink_tick += 1 if self._blink_tick >= 38: @@ -553,15 +625,196 @@ class HudCanvas(QWidget): _blinked = False # Repaint throttling — advancing the animation state above is cheap at - # 60 Hz, but the paint is heavy. Repaint every frame while something is - # actually happening (speaking, audio, thinking) or when the blink - # toggles; otherwise drop to ~20 Hz so an idle HUD stops pinning a CPU - # core. The visuals stay smooth because the state keeps stepping. - self._paint_tick = (self._paint_tick + 1) % 3 + # 60 Hz, but the paint is heavy. Active (speaking, audio, thinking) runs + # at ~30 Hz, which is the frame rate animation has used for talking + # characters forever and is indistinguishable here; idle drops to ~20 Hz + # so a sleeping HUD stops pinning a CPU core. The visuals stay smooth + # either way because the animation state keeps stepping at 60 Hz. + self._paint_tick = (self._paint_tick + 1) % 6 active = (self.speaking or amp > 0.02 or self.state in ("THINKING", "PROCESSING")) - if active or _blinked or self._paint_tick == 0: - self.update() + if _blinked or (self._paint_tick % 2 == 0 if active + else self._paint_tick % 3 == 0): + # Nothing is on screen when the window is hidden or minimised, so + # rendering the avatar into it is pure waste — and this app is meant + # to sit running all day. The animation state above keeps stepping, + # so it picks up mid-motion instead of snapping when you come back. + if self._on_screen(): + self.update() + + def _on_screen(self) -> bool: + """True only when this canvas can actually be seen by the user.""" + try: + if not self.isVisible(): + return False + win = self.window() + return not (win.isMinimized() or win.isHidden()) + except Exception: + return True # never let a visibility check stop the HUD drawing + + # ── reactor core ───────────────────────────────────────────────────────── + # The centrepiece for anyone who did not want a face looking back at them. + # Built from the same budget as the head — software QPainter, no GPU — and + # from the same principle: everything on it means something. The rings turn + # at a rate the state sets, the spectrum ring is the real audio level, and + # the core brightens with the voice. Nothing here is decoration that moves + # for its own sake, which is what made the old glowing orb feel dead. + + def _core_colours(self): + if self.muted: + return qcol(C.MUTED_C), qcol(C.MUTED_C) + if self.speaking: + return qcol(C.PRI), qcol(C.ACC) + if self.state in ("THINKING", "PROCESSING"): + return qcol(C.PRI), qcol(C.ACC2) + if self.state == "LISTENING": + return qcol(C.PRI), qcol(C.GREEN) + return qcol(C.PRI), qcol(C.PRI_DIM) + + def _paint_core(self, p: QPainter, cx: float, cy: float, r: float, + W: float = 0.0, H: float = 0.0): + """Draw the reactor at (cx, cy) with outer radius r, using the whole + canvas (W x H) for the marks that frame it.""" + main, acc = self._core_colours() + bg = qcol(C.BG) + amp = self._amp_disp + t = self._core_phase + live = (self.speaking or amp > 0.04) and not self.muted + + def blend(col: QColor, a: float) -> QColor: + """Pre-mix onto the background instead of asking Qt to composite. + The raster engine's opaque path is several times faster than its + translucent one, and everything here is a line or an arc.""" + k = max(0.0, min(1.0, a)) + return QColor(int(bg.red() + (col.red() - bg.red()) * k), + int(bg.green() + (col.green() - bg.green()) * k), + int(bg.blue() + (col.blue() - bg.blue()) * k)) + + p.setBrush(Qt.BrushStyle.NoBrush) + + # 1. The atmosphere. One radial gradient doing what a stack of discs did + # badly: a wide, soft body of light that gives the thing presence + # before any detail is read. This single element decides whether the + # HUD looks vast or looks small, so it is drawn first and drawn big. + # Concentrated rather than spread: a gradient reaching the outer rim + # washes the whole disc a flat dim blue and reads as fog. Ending it at + # two thirds leaves it a body of light with somewhere to fall off to, + # which is what makes it look lit rather than tinted. + lift = 1.0 + 0.55 * amp + (0.18 if self.speaking else 0.0) + p.setPen(Qt.PenStyle.NoPen) + for gr, a0, a1 in ((r * 0.70, 0.30, 0.0), (r * 0.34, 0.34, 0.0)): + g = QRadialGradient(cx, cy, gr) + g.setColorAt(0.00, blend(main, min(0.95, a0 * lift))) + g.setColorAt(0.45, blend(main, min(0.95, a0 * lift * 0.52))) + g.setColorAt(0.78, blend(main, min(0.95, a0 * lift * 0.18))) + g.setColorAt(1.00, blend(main, a1)) + p.setBrush(QBrush(g)) + p.drawEllipse(QRectF(cx - gr, cy - gr, gr * 2, gr * 2)) + p.setBrush(Qt.BrushStyle.NoBrush) + + # 2. Frame marks at the corners of the whole canvas, not of the circle. + # They are what set the scale: the eye reads the reactor as filling + # the room rather than sitting in the middle of it. + if W > 40 and H > 40: + m, arm = min(W, H) * 0.035, min(W, H) * 0.055 + p.setPen(QPen(blend(main, 0.45), 1.4)) + for sx, sy in ((1, 1), (-1, 1), (1, -1), (-1, -1)): + x = cx + sx * (W / 2 - m) + y = cy + sy * (H / 2 - m) + p.drawLine(QLineF(x, y, x - sx * arm, y)) + p.drawLine(QLineF(x, y, x, y - sy * arm)) + + # 3. Crosshair across the full canvas, broken around the core so it + # frames the reactor rather than crossing it. + p.setPen(QPen(blend(main, 0.16), 1)) + gap = r * 0.62 + if W > 40: + p.drawLine(QLineF(cx - W / 2, cy, cx - gap, cy)) + p.drawLine(QLineF(cx + gap, cy, cx + W / 2, cy)) + if H > 40: + p.drawLine(QLineF(cx, cy - H / 2, cx, cy - gap)) + p.drawLine(QLineF(cx, cy + gap, cx, cy + H / 2)) + + # 4. Two thin outer circles. Sparse on purpose — a dense ring reads as a + # grey band at this size, and restraint is what made the original + # look expensive. + for rr, a in ((1.00, 0.34), (0.93, 0.16)): + rad = r * rr + p.setPen(QPen(blend(main, a), 1)) + p.drawEllipse(QRectF(cx - rad, cy - rad, rad * 2, rad * 2)) + + # 5. Long, sparse graduations: 24 majors reaching well in from the rim, + # with shorter minors between them. + major, minor = [], [] + for i in range(72): + a = math.radians(i * 5.0) + ca, sa = math.cos(a), math.sin(a) + if i % 3 == 0: + major.append(QLineF(cx + ca * r * 0.885, cy + sa * r * 0.885, + cx + ca * r * 0.985, cy + sa * r * 0.985)) + else: + minor.append(QLineF(cx + ca * r * 0.945, cy + sa * r * 0.945, + cx + ca * r * 0.985, cy + sa * r * 0.985)) + p.setPen(QPen(blend(main, 0.42), 1.3)) + p.drawLines(major) + p.setPen(QPen(blend(main, 0.18), 1)) + p.drawLines(minor) + + # 6. Sweeping arcs. Long spans, not dashes — the original's grandeur + # came from a few big strokes. Speed is the state: idle drifts, + # thinking hurries, speaking runs. + rate = 1.0 + (1.9 if self.state in ("THINKING", "PROCESSING") else 0.0) \ + + (1.2 if self.speaking else 0.0) + for k, (rr, span, count, dirn, col, a, wid) in enumerate(( + (0.955, 118, 2, +1, acc, 0.75, 2.0), + (0.845, 82, 3, -1, main, 0.38, 1.3), + (0.760, 150, 1, +1, acc, 0.45, 1.6), + (0.660, 64, 4, -1, main, 0.26, 1.1), + (0.545, 128, 2, +1, main, 0.30, 1.2))): + rad = r * rr + p.setPen(QPen(blend(col, a), wid)) + box = QRectF(cx - rad, cy - rad, rad * 2, rad * 2) + base = (t * rate * (9 + k * 6) * dirn) % 360.0 + for sgm in range(count): + p.drawArc(box, int((base + sgm * (360.0 / count)) * 16), + int(span * 16)) + + # 7. The voice, as a ring of graduations that grow with it. Kept out at + # a wide radius so it never crowds the middle. + n = 60 + ring = r * 0.415 + spikes = [] + for i in range(n): + a = math.radians(i * (360.0 / n)) + ca, sa = math.cos(a), math.sin(a) + wob = 0.5 + 0.5 * math.sin(t * 2.3 + i * 0.42) + idle = 0.018 + 0.012 * math.sin(t * 1.2 + i * 0.7) + h = r * (idle + (amp * 0.20 * wob if live else 0.0)) + spikes.append(QLineF(cx + ca * ring, cy + sa * ring, + cx + ca * (ring + h), cy + sa * (ring + h))) + p.setPen(QPen(blend(acc if live else main, 0.25 + 0.5 * amp), 1.6)) + p.drawLines(spikes) + + # 8. The inner ring the name sits in. + inner = r * 0.355 + p.setPen(QPen(blend(acc, 0.30 + 0.45 * amp), 1.5)) + p.drawEllipse(QRectF(cx - inner, cy - inner, inner * 2, inner * 2)) + + # 9. The name, sized from the string rather than from the radius alone: + # "J.A.R.V.I.S" and a name someone renamed to "MAX" are very + # different widths, and a fixed fraction of r spills one of them past + # the ring it is supposed to sit inside. + name = self._assistant_name or "" + if name: + space = max(1.0, r * 0.018) + fsz = max(8, int(min(r * 0.105, + (inner * 1.75) / max(1, len(name)) * 1.6 - space))) + f = QFont("Courier New", fsz, QFont.Weight.Bold) + f.setLetterSpacing(QFont.SpacingType.AbsoluteSpacing, space) + p.setFont(f) + p.setPen(QPen(blend(qcol(C.WHITE), 0.6 + 0.4 * min(1.0, amp * 2)), 1)) + p.drawText(QRectF(cx - r, cy - fsz, r * 2, fsz * 2), + Qt.AlignmentFlag.AlignCenter, name) def paintEvent(self, _): p = QPainter(self) @@ -582,119 +835,43 @@ class HudCanvas(QWidget): self._grid_key = _gkey p.drawPixmap(0, 0, self._grid_cache) - r_face = fw * 0.31 + # ── holographic head ──────────────────────────────────────────────── + # Sized to the band between the top of the canvas and the status line, + # capped by width, so it fills the HUD at any window size — including + # fullscreen — without ever colliding with the status text below. + _sy_status = cy + fw * 0.40 + if self._avatar is not None and self.hud_style == "face": + _band_t = 12.0 + _band_h = max(60.0, _sy_status - 12.0 - _band_t) + _r_head = min(fw * 0.355, _band_h / (self._avatar.SPAN + 0.08)) + _head_cy = _band_t + (_band_h - self._avatar.SPAN * _r_head) / 2.0 + _r_head - # halo glow - for i in range(10): - r = r_face * (1.8 - i * 0.08) - frc = 1.0 - i / 10 - a = max(0, min(255, int(self._halo * 0.085 * frc))) - col = qcol(C.MUTED_C if self.muted else C.PRI, a) - p.setPen(QPen(col, 1.5)); p.setBrush(Qt.BrushStyle.NoBrush) - p.drawEllipse(QRectF(cx - r, cy - r, r * 2, r * 2)) + if self.muted: + _main = _acc = qcol(C.MUTED_C) + else: + _main = qcol(C.PRI) + if self.speaking: + _acc = qcol(C.ACC) + elif self.state in ("THINKING", "PROCESSING"): + _acc = qcol(C.ACC2) + elif self.state == "LISTENING": + _acc = qcol(C.GREEN) + else: + _acc = qcol(C.PRI) + self._avatar.paint(p, cx, _head_cy, _r_head, _main, _acc, qcol(C.BG)) - # pulse rings - for pr in self._pulses: - a = max(0, int(230 * (1.0 - pr / (fw * 0.74)))) - col = qcol(C.MUTED_C if self.muted else C.PRI, a) - p.setPen(QPen(col, 1.5)); p.setBrush(Qt.BrushStyle.NoBrush) - p.drawEllipse(QRectF(cx - pr, cy - pr, pr * 2, pr * 2)) - - # spinning arc rings - for idx, (r_frac, w_r, arc_l, gap) in enumerate( - [(0.48, 3, 115, 78), (0.40, 2, 78, 55), (0.32, 1, 56, 40)] - ): - ring_r = fw * r_frac - base = self._rings[idx] - a_val = max(0, min(255, int(self._halo * (1.0 - idx * 0.18)))) - col = qcol(C.MUTED_C if self.muted else C.PRI, a_val) - p.setPen(QPen(col, w_r)); p.setBrush(Qt.BrushStyle.NoBrush) - angle = base - rect = QRectF(cx - ring_r, cy - ring_r, ring_r * 2, ring_r * 2) - while angle < base + 360: - p.drawArc(rect, int(angle * 16), int(arc_l * 16)) - angle += arc_l + gap - - # scanners - sr = fw * 0.50 - sa = min(255, int(self._halo * 1.5)) - ex = 75 if self.speaking else 44 - p.setPen(QPen(qcol(C.MUTED_C if self.muted else C.PRI, sa), 2.5)) - p.setBrush(Qt.BrushStyle.NoBrush) - srect = QRectF(cx - sr, cy - sr, sr * 2, sr * 2) - p.drawArc(srect, int(self._scan * 16), int(ex * 16)) - p.setPen(QPen(qcol(C.ACC, sa // 2), 1.5)) - p.drawArc(srect, int(self._scan2 * 16), int(ex * 16)) - - # tick marks - t_out, t_in = fw * 0.497, fw * 0.474 - p.setPen(QPen(qcol(C.PRI, 140), 1)) - for deg in range(0, 360, 10): - rad = math.radians(deg) - inn = t_in if deg % 30 == 0 else t_in + 6 - p.drawLine( - QPointF(cx + t_out * math.cos(rad), cy - t_out * math.sin(rad)), - QPointF(cx + inn * math.cos(rad), cy - inn * math.sin(rad)), - ) - - # crosshair - ch_r, gap_h = fw * 0.51, fw * 0.16 - p.setPen(QPen(qcol(C.PRI, int(self._halo * 0.5)), 1)) - p.drawLine(QPointF(cx - ch_r, cy), QPointF(cx - gap_h, cy)) - p.drawLine(QPointF(cx + gap_h, cy), QPointF(cx + ch_r, cy)) - p.drawLine(QPointF(cx, cy - ch_r), QPointF(cx, cy - gap_h)) - p.drawLine(QPointF(cx, cy + gap_h), QPointF(cx, cy + ch_r)) - - # corner brackets - bl = 24 - bc = qcol(C.PRI, 210) - hl, hr = cx - fw // 2, cx + fw // 2 - ht, hb = cy - fw // 2, cy + fw // 2 - p.setPen(QPen(bc, 2)) - for bx, by, dx, dy in [(hl,ht,1,1),(hr,ht,-1,1),(hl,hb,1,-1),(hr,hb,-1,-1)]: - p.drawLine(QPointF(bx, by), QPointF(bx + dx * bl, by)) - p.drawLine(QPointF(bx, by), QPointF(bx, by + dy * bl)) - - # face - if self._face_px: - fsz = int(fw * 0.62 * self._scale) - # Quantise the target size so the expensive smooth rescale only runs - # when it visibly changes — not on every 1 px "breathing" step. - q_sz = max(1, (fsz // 4) * 4) - if self._face_cache is None or self._face_cache_sz != q_sz: - self._face_cache = self._face_px.scaled( - q_sz, q_sz, - Qt.AspectRatioMode.KeepAspectRatio, - Qt.TransformationMode.SmoothTransformation, - ) - self._face_cache_sz = q_sz - scaled = self._face_cache - p.drawPixmap(int(cx - scaled.width() / 2), - int(cy - scaled.height() / 2), scaled) + # reactor core — the other centrepiece, and the fallback if the head + # could not be built. There is no third path: the old face.png branch + # was unreachable (no such file ships) and the bare orb it fell through + # to is what this replaces. else: - orb_r = int(fw * 0.27 * self._scale) - oc = (200, 0, 50) if self.muted else (0, 60, 110) - for i in range(8, 0, -1): - r2 = int(orb_r * i / 8) - frc = i / 8 - a = max(0, min(255, int(self._halo * 1.1 * frc))) - p.setBrush(QBrush(QColor(int(oc[0]*frc), int(oc[1]*frc), int(oc[2]*frc), a))) - p.setPen(Qt.PenStyle.NoPen) - p.drawEllipse(QRectF(cx - r2, cy - r2, r2 * 2, r2 * 2)) - p.setPen(QPen(qcol(C.PRI, min(255, int(self._halo * 2))), 1)) - p.setFont(QFont("Courier New", 13, QFont.Weight.Bold)) - p.drawText(QRectF(cx - 80, cy - 14, 160, 28), - Qt.AlignmentFlag.AlignCenter, self._assistant_name) - - # particles - for pt in self._particles: - a = max(0, min(255, int(pt[4] * 255))) - p.setPen(Qt.PenStyle.NoPen) - p.setBrush(QBrush(qcol(C.PRI, a))) - p.drawEllipse(QPointF(pt[0], pt[1]), 2.5, 2.5) + _band_t = 12.0 + _band_h = max(60.0, _sy_status - 12.0 - _band_t) + _r = min(W * 0.46, _band_h / 2.0) + self._paint_core(p, cx, _band_t + _band_h / 2.0, _r, W, _band_h) # status text - sy = cy + fw * 0.40 + sy = _sy_status if self.muted: txt, col = "⊘ MUTED", qcol(C.MUTED_C) elif self.speaking: @@ -876,7 +1053,12 @@ class LogWidget(QTextEdit): "ai": qcol(C.PRI), "err": qcol(C.RED), "file": qcol(C.GREEN), - "sys": qcol(C.ACC2), + # SYS lines are the bulk of the log. Amber fought the cyan HUD + # and, being a fixed status colour rather than a hue-linked one, + # stayed amber even after the accent picker retinted everything + # else. TEXT_MED follows the theme and drops the contrast to a + # level you can read past. + "sys": qcol(C.TEXT_MED), }.get(self._tag, qcol(C.TEXT)) fmt.setForeground(QBrush(col)) cur.movePosition(cur.MoveOperation.End) @@ -2746,6 +2928,9 @@ class MainWindow(QMainWindow): _confirm_sig = pyqtSignal(str, str) # (title, detail) — irreversible-action gate _confirm_hide_sig = pyqtSignal() _wake_dl_sig = pyqtSignal(bool, str) # wake-word install finished (ok, message) + _quiz_sig = pyqtSignal(str, object, object) # (topic, questions, grader) + _quiz_hide_sig = pyqtSignal() + _review_sig = pyqtSignal(str, str, object, object) # document review payload def __init__(self, face_path: str): super().__init__() @@ -2781,6 +2966,8 @@ class MainWindow(QMainWindow): self.get_plugin_settings = None # callable: () -> list[dict] settings schemas, set by JarvisLive self.on_wake_toggle = None # callable: (enable: bool) -> str, set by JarvisLive self.on_wake_manual = None # callable: () -> None — manual sleep/wake + self.on_push_to_talk = None # callable: (enable: bool) -> str scope + self.ptt_hold = None # callable: (held: bool) -> None — windowed chord self.wake_get_state = None # callable: () -> dict {enabled, awake, ready} self._muted = False self._current_file: str | None = None @@ -2807,6 +2994,7 @@ class MainWindow(QMainWindow): self.hud = HudCanvas(face_path, _display) self.hud.setSizePolicy(QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Expanding) self._content_panel = self._build_content_panel() + self._quiz_panel = self._build_quiz_panel() # Live camera container — replaces HUD when camera stream is active _cam_cont = QWidget() @@ -2859,6 +3047,7 @@ class MainWindow(QMainWindow): """) self._center_split.addWidget(self._hud_cam_stack) self._center_split.addWidget(self._content_panel) + self._center_split.addWidget(self._quiz_panel) self._center_split.setStretchFactor(0, 3) self._center_split.setStretchFactor(1, 1) self._center_split.setCollapsible(0, False) @@ -2898,6 +3087,9 @@ class MainWindow(QMainWindow): self._cam_frame_sig.connect(self._on_cam_frame) self._clipboard_sig.connect(self._show_clipboard_panel) self._wake_dl_sig.connect(self._on_wake_install_done) + self._quiz_sig.connect(self._show_quiz) + self._quiz_hide_sig.connect(self._hide_quiz) + self._review_sig.connect(self._show_review) self._cam_stop = threading.Event() # Camera preview overlay (child of central widget, positioned in resizeEvent) @@ -3476,7 +3668,7 @@ class MainWindow(QMainWindow): self._title_lbl.setFont(QFont("Courier New", 17, QFont.Weight.Bold)) self._title_lbl.setStyleSheet(f"color: {C.PRI}; background: transparent;") mid.addWidget(self._title_lbl) - _sub_text = ("Just A Rather Very Intelligent System" + _sub_text = ("A Friendly Assistant" if _disp in ("JARVIS", "J.A.R.V.I.S") else "Personal AI Assistant") self._sub_lbl = QLabel(_sub_text) @@ -3752,6 +3944,23 @@ class MainWindow(QMainWindow): self._wake_btn.setStyleSheet(_BTN_STYLE_DIM) self._wake_sleep_btn.hide() + self._ptt_btn = QPushButton() + self._ptt_btn.setFixedHeight(26) + self._ptt_btn.setFont(QFont("Courier New", 7)) + self._ptt_btn.setCursor(Qt.CursorShape.PointingHandCursor) + self._ptt_btn.clicked.connect(self._toggle_ptt) + lay.addWidget(self._ptt_btn) + + self._refresh_talk_btns() + + self._hud_btn = QPushButton() + self._hud_btn.setFixedHeight(26) + self._hud_btn.setFont(QFont("Courier New", 7)) + self._hud_btn.setCursor(Qt.CursorShape.PointingHandCursor) + self._hud_btn.clicked.connect(self._toggle_hud_style) + lay.addWidget(self._hud_btn) + self._refresh_hud_btn() + audio_btn = QPushButton("🎧 AUDIO DEVICES") audio_btn.setFixedHeight(26) audio_btn.setFont(QFont("Courier New", 7)) @@ -3928,6 +4137,9 @@ class MainWindow(QMainWindow): def _show_content(self, title: str, text: str): """Slot — runs on Qt main thread. Updates and shows the content panel.""" import time as _time + # The panel opens below the head, so the head looks down at it. It is a + # tiny thing that answers "did that land?" before you read a word. + self.hud.glance(0.0, -0.85, hold=1.3) self._content_title_lbl.setText(title.upper()[:48]) self._content_ts_lbl.setText(_time.strftime("%H:%M:%S")) self._content_display.setPlainText(text) @@ -3940,6 +4152,361 @@ class MainWindow(QMainWindow): total = self._center_split.height() self._center_split.setSizes([max(total - 220, 120), 220]) + # ── document review ────────────────────────────────────────────────────── + # Rendered as rich text into the content panel that already exists, rather + # than into a panel of its own. A review is read, not clicked, so QTextEdit + # gives scrolling, selection and copy for nothing, and the HUD gains no + # widget it has to lay out. Severity decides colour and order here because + # that is presentation; the plugin supplies no styling and knows no palette, + # which is also what lets a re-theme repaint a review correctly. + + # Severity is marked by a symbol and a colour, not by a word. The findings + # themselves are in the user's language, and "[SERIOUS]" sitting inside a + # Turkish sentence is the kind of seam this project tries not to have — + # while translating the tag would mean a table per language, which is worse. + # A shape carries it in every language, and shape plus colour still reads + # for someone who cannot separate red from amber. What the marks mean + # arrives the way everything else does: JARVIS says it out loud. + _REVIEW_MARKS = {"serious": ("RED", "▲"), "caution": ("ACC2", "●"), "note": ("PRI_DIM", "·")} + + @staticmethod + def _esc(s) -> str: + return (str(s or "").replace("&", "&").replace("<", "<") + .replace(">", ">").replace("\n", "
")) + + def _show_review(self, title: str, summary: str, findings, unclear): + """Slot — Qt main thread. Lays a document review into the content panel.""" + e = self._esc + parts = [f'
'] + + if summary: + parts.append( + f'
{e(summary)}
') + + for f in (findings or []): + key, mark = self._REVIEW_MARKS.get(f.get("severity"), ("PRI_DIM", "·")) + colour = getattr(C, key) + parts.append(f'
') + parts.append( + f'{mark} ' + f'' + f'{e(f.get("heading"))}') + if f.get("detail"): + parts.append(f'
{e(f["detail"])}
') + if f.get("quote"): + # The document's own wording, visually separated from the + # explanation so the two are never mistaken for each other. + parts.append( + f'
' + f'“{e(f["quote"])}”
') + if f.get("suggestion"): + parts.append( + f'
' + f'→ {e(f["suggestion"])}
') + parts.append('
') + + if unclear: + parts.append( + f'
' + 'The document does not settle:
') + for u in unclear: + parts.append( + f'
' + f'· {e(u)}
') + parts.append('
') + + import time as _time + self.hud.glance(0.0, -0.85, hold=1.3) + # Left as written, not upper-cased. The other content-panel titles are + # the app's own English labels, but this one is the document's name in + # the user's language, and str.upper() applies English casing rules to + # it: Turkish "Sözleşmesi" comes back "SÖZLEŞMESI", having lost the + # dotted capital İ. Python has no locale-aware upper to reach for, and + # imposing one language's rules on all of them is the bug, not the fix. + self._content_title_lbl.setText((title or "Document")[:48]) + self._content_ts_lbl.setText(_time.strftime("%H:%M:%S")) + self._content_display.setHtml("".join(parts)) + self._content_display.moveCursor( + self._content_display.textCursor().MoveOperation.Start) + first_show = not self._content_panel.isVisible() + self._content_panel.show() + if first_show: + total = self._center_split.height() + self._center_split.setSizes([max(total - 260, 120), 260, 0]) + + # ── quiz panel ─────────────────────────────────────────────────────────── + # An interactive twin of the content panel. The plugin only ever hands over + # questions; everything about asking, marking and reporting happens here, + # and the finished result is pushed back into the conversation the same way + # a dropped file is — as a message JARVIS reads and responds to. That keeps + # the tool call short (it returns the moment the board is up) and leaves the + # talking to the assistant, in the user's own language. + + def _quiz_btn(self, text: str, primary: bool = False) -> QPushButton: + b = QPushButton(text) + b.setFont(QFont("Courier New", 8)) + b.setCursor(Qt.CursorShape.PointingHandCursor) + b.setMinimumHeight(24) + edge = C.BORDER_B if primary else C.BORDER + col = C.PRI if primary else C.TEXT_MED + b.setStyleSheet(f""" + QPushButton {{ + background: {C.PANEL2}; color: {col}; + border: 1px solid {edge}; border-radius: 2px; + padding: 3px 9px; text-align: left; + }} + QPushButton:hover {{ color: {C.WHITE}; border-color: {C.PRI_DIM}; }} + QPushButton:disabled {{ color: {C.TEXT_DIM}; border-color: {C.BORDER}; }} + """) + return b + + def _build_quiz_panel(self) -> QWidget: + w = QWidget() + w.setObjectName("QuizPanel") + w.setStyleSheet(f""" + QWidget#QuizPanel {{ + background: {C.PANEL}; + border-top: 1px solid {C.BORDER_B}; + }} + """) + w.hide() + + lay = QVBoxLayout(w) + lay.setContentsMargins(12, 7, 12, 8) + lay.setSpacing(6) + + hdr = QHBoxLayout(); hdr.setSpacing(6) + dot = QLabel("◈") + dot.setFont(QFont("Courier New", 9, QFont.Weight.Bold)) + dot.setStyleSheet(f"color: {C.PRI}; background: transparent;") + hdr.addWidget(dot) + + self._quiz_title_lbl = QLabel("QUIZ") + self._quiz_title_lbl.setFont(QFont("Courier New", 8, QFont.Weight.Bold)) + self._quiz_title_lbl.setStyleSheet( + f"color: {C.PRI}; background: transparent; letter-spacing: 1px;") + hdr.addWidget(self._quiz_title_lbl) + hdr.addStretch() + + self._quiz_count_lbl = QLabel("") + self._quiz_count_lbl.setFont(QFont("Courier New", 7)) + self._quiz_count_lbl.setStyleSheet(f"color: {C.TEXT_DIM}; background: transparent;") + hdr.addWidget(self._quiz_count_lbl) + + quit_btn = QPushButton("DISMISS ✕") + quit_btn.setFont(QFont("Courier New", 7)) + quit_btn.setFixedHeight(18) + quit_btn.setCursor(Qt.CursorShape.PointingHandCursor) + quit_btn.setStyleSheet(f""" + QPushButton {{ + background: transparent; color: {C.TEXT_DIM}; + border: 1px solid {C.BORDER}; border-radius: 2px; padding: 0 5px; + }} + QPushButton:hover {{ color: {C.TEXT}; border-color: {C.BORDER_B}; }} + """) + quit_btn.clicked.connect(self._hide_quiz) + hdr.addWidget(quit_btn) + lay.addLayout(hdr) + + rule = QFrame(); rule.setFixedHeight(1) + rule.setStyleSheet(f"background: {C.BORDER};") + lay.addWidget(rule) + + self._quiz_q_lbl = QLabel("") + self._quiz_q_lbl.setWordWrap(True) + self._quiz_q_lbl.setFont(QFont("Courier New", 9)) + self._quiz_q_lbl.setStyleSheet(f"color: {C.WHITE}; background: transparent;") + lay.addWidget(self._quiz_q_lbl) + + self._quiz_answers = QWidget() + self._quiz_answers.setStyleSheet("background: transparent;") + self._quiz_answers_lay = QVBoxLayout(self._quiz_answers) + self._quiz_answers_lay.setContentsMargins(0, 2, 0, 0) + self._quiz_answers_lay.setSpacing(4) + lay.addWidget(self._quiz_answers) + + self._quiz_note_lbl = QLabel("") + self._quiz_note_lbl.setWordWrap(True) + self._quiz_note_lbl.setFont(QFont("Courier New", 8)) + self._quiz_note_lbl.setStyleSheet(f"color: {C.TEXT_MED}; background: transparent;") + self._quiz_note_lbl.hide() + lay.addWidget(self._quiz_note_lbl) + + foot = QHBoxLayout() + foot.addStretch() + self._quiz_next_btn = self._quiz_btn("NEXT →", primary=True) + self._quiz_next_btn.setFixedWidth(110) + self._quiz_next_btn.clicked.connect(self._quiz_next) + self._quiz_next_btn.hide() + foot.addWidget(self._quiz_next_btn) + lay.addLayout(foot) + + self._quiz = None + return w + + def _show_quiz(self, topic: str, questions, grader=None): + """Slot — Qt main thread. Puts a fresh quiz on the board.""" + if not questions: + return + self._quiz = { + "topic": topic or "", + "questions": list(questions), + "grader": grader, + "i": 0, + "results": [], + "answered": False, + } + self._quiz_title_lbl.setText((topic or "quiz").upper()[:48]) + self.hud.glance(0.0, -0.85, hold=1.3) + first_show = not self._quiz_panel.isVisible() + self._quiz_panel.show() + if first_show: + total = self._center_split.height() + self._center_split.setSizes([max(total - 250, 120), 0, 250]) + self._quiz_render() + + def _hide_quiz(self): + self._quiz = None + self._quiz_panel.hide() + + def _quiz_clear_answers(self): + while self._quiz_answers_lay.count(): + item = self._quiz_answers_lay.takeAt(0) + child = item.widget() + if child is not None: + child.setParent(None) + child.deleteLater() + + def _quiz_render(self): + q = self._quiz["questions"][self._quiz["i"]] + n, total = self._quiz["i"] + 1, len(self._quiz["questions"]) + self._quiz_count_lbl.setText(f"{n} / {total}") + self._quiz_q_lbl.setText(q.get("question", "")) + self._quiz_note_lbl.hide() + self._quiz_next_btn.hide() + self._quiz["answered"] = False + self._quiz_clear_answers() + + opts = q.get("options") or [] + if opts: + for text in opts: + b = self._quiz_btn(" " + text) + b.clicked.connect(lambda _=False, t=text: self._quiz_submit(t)) + self._quiz_answers_lay.addWidget(b) + else: + row = QWidget(); row.setStyleSheet("background: transparent;") + h = QHBoxLayout(row); h.setContentsMargins(0, 0, 0, 0); h.setSpacing(6) + field = QLineEdit() + field.setFont(QFont("Courier New", 9)) + field.setPlaceholderText("your answer") + field.setStyleSheet(f""" + QLineEdit {{ + background: {C.PANEL2}; color: {C.WHITE}; + border: 1px solid {C.BORDER}; border-radius: 2px; padding: 4px 7px; + }} + QLineEdit:focus {{ border-color: {C.PRI_DIM}; }} + """) + send = self._quiz_btn("ANSWER", primary=True) + send.setFixedWidth(90) + field.returnPressed.connect(lambda: self._quiz_submit(field.text())) + send.clicked.connect(lambda: self._quiz_submit(field.text())) + h.addWidget(field, stretch=1) + h.addWidget(send) + self._quiz_answers_lay.addWidget(row) + field.setFocus() + + def _quiz_submit(self, given: str): + if self._quiz is None or self._quiz["answered"]: + return + self._quiz["answered"] = True + q = self._quiz["questions"][self._quiz["i"]] + grader = self._quiz.get("grader") + verdict = None + if callable(grader): + try: + verdict = grader(q, given) + except Exception: + verdict = None + self._quiz["results"].append({ + "question": q.get("question", ""), + "type": q.get("type", ""), + "given": str(given or "").strip(), + "answer": q.get("answer", ""), + "correct": verdict, + }) + + for i in range(self._quiz_answers_lay.count()): + wdg = self._quiz_answers_lay.itemAt(i).widget() + if wdg is not None: + wdg.setEnabled(False) + + if verdict is True: + mark, colour = "✓ correct", C.GREEN + elif verdict is False: + mark, colour = "✕ " + str(q.get("answer", "")), C.RED + else: + # Open answers and near-miss gap-fills are JARVIS's to judge. Saying + # so is honest; marking it wrong here would be a guess. + mark, colour = "… noted — I'll go over this one with you", C.ACC2 + note = q.get("note") or "" + self._quiz_note_lbl.setText(mark + (("\n" + note) if note else "")) + self._quiz_note_lbl.setStyleSheet(f"color: {colour}; background: transparent;") + self._quiz_note_lbl.show() + + last = self._quiz["i"] >= len(self._quiz["questions"]) - 1 + self._quiz_next_btn.setText("FINISH →" if last else "NEXT →") + self._quiz_next_btn.show() + self._quiz_next_btn.setFocus() + + def _quiz_next(self): + if self._quiz is None: + return + if self._quiz["i"] >= len(self._quiz["questions"]) - 1: + self._quiz_finish() + else: + self._quiz["i"] += 1 + self._quiz_render() + + def _quiz_finish(self): + if self._quiz is None: + return + topic = self._quiz["topic"] + results = self._quiz["results"] + right = sum(1 for r in results if r["correct"] is True) + unsure = sum(1 for r in results if r["correct"] is None) + total = len(results) + self._quiz_panel.hide() + self._quiz = None + + self._log.append_log(f"QUIZ: {topic or 'quiz'} — {right}/{total} correct") + + # Hand it back to JARVIS as a message, not as a tool return: the tool + # call ended minutes ago. This is the same channel a dropped file uses. + lines = [f"[QUIZ_DONE] topic={topic or 'general'} | " + f"auto-marked {right}/{total} correct" + + (f", {unsure} still need your marking" if unsure else "")] + for i, r in enumerate(results, 1): + state = ("correct" if r["correct"] is True + else "wrong" if r["correct"] is False else "NEEDS MARKING") + lines.append( + f"{i}. [{r['type']}] {r['question']} | they answered: " + f"{r['given'] or '(blank)'} | expected: {r['answer']} | {state}") + lines.append( + "Mark every question flagged NEEDS MARKING yourself — accept an answer " + "that means the same thing. Then tell them how they did in their own " + "language: the score, what they got wrong and why, in a couple of " + "sentences. Offer another round only if it fits. " + "Remember something only if it would still matter next week — that they " + "are working through a subject, or keep missing the same thing. A score " + "from one session is not worth a memory, and a memory per quiz would " + "bury the things that are.") + msg = "\n".join(lines) + if self.on_text_command: + threading.Thread(target=self.on_text_command, args=(msg,), daemon=True).start() + def _build_footer(self) -> QWidget: w = QWidget() w.setFixedHeight(22) @@ -4169,6 +4736,129 @@ class MainWindow(QMainWindow): self._wake_btn.setStyleSheet(_off) self._wake_sleep_btn.hide() + def _refresh_talk_btns(self): + """Repaint the push-to-talk row from the saved setting.""" + if not hasattr(self, "_ptt_btn"): + return + from core.hotkey import chord_label + from memory.config_manager import get_push_to_talk_enabled + _on = f""" + QPushButton {{ background: #001a08; color: {C.GREEN}; + border: 1px solid {C.GREEN_D}; border-radius: 3px; + text-align: left; padding: 0 8px; }} + QPushButton:hover {{ background: #002010; }}""" + _off = f""" + QPushButton {{ background: transparent; color: {C.TEXT_DIM}; + border: 1px solid {C.BORDER}; border-radius: 3px; + text-align: left; padding: 0 8px; }} + QPushButton:hover {{ color: {C.TEXT}; border: 1px solid {C.BORDER_B}; }}""" + + ptt = get_push_to_talk_enabled() + self._ptt_btn.setText(f"🎚 PUSH-TO-TALK: {chord_label()}" if ptt + else "🎚 PUSH-TO-TALK: OFF") + self._ptt_btn.setStyleSheet(_on if ptt else _off) + self._ptt_btn.setToolTip( + "Microphone stays closed until you hold the key — nothing is sent " + "while you are not holding it." if ptt + else "Hold a key to talk instead of streaming the mic continuously.") + + + def _refresh_hud_btn(self): + from memory.config_manager import get_hud_style + face = get_hud_style() == "face" + # Neither state is "off", so both read as active — this is a choice + # between two things, not a switch with a disabled side. + style = f""" + QPushButton {{ background: {C.PANEL2}; color: {C.PRI}; + border: 1px solid {C.BORDER_A}; border-radius: 3px; + text-align: left; padding: 0 8px; }} + QPushButton:hover {{ color: {C.WHITE}; border: 1px solid {C.BORDER_B}; }}""" + self._hud_btn.setText("🧑 HUD: ANIMATED FACE" if face + else "◉ HUD: REACTOR CORE") + self._hud_btn.setStyleSheet(style) + self._hud_btn.setToolTip( + "An animated head that speaks your words and shows what JARVIS is " + "doing. Tap to switch to the reactor core." + if face else + "A reactor core that turns with the state and moves with your voice. " + "Tap to switch to the animated head.") + + def _toggle_hud_style(self): + """Swap the centrepiece. Both objects stay in memory, so the change is + instant and switching back costs nothing.""" + from memory.config_manager import get_hud_style, save_hud_style + want = "core" if get_hud_style() == "face" else "face" + save_hud_style(want) + try: + self.hud.hud_style = want + self.hud.update() + except Exception: + pass + self._refresh_hud_btn() + self._log.append_log( + "SYS: HUD switched to the animated face." if want == "face" + else "SYS: HUD switched to the reactor core.") + + def _toggle_ptt(self): + from memory.config_manager import (get_push_to_talk_enabled, + save_push_to_talk_enabled) + want = not get_push_to_talk_enabled() + save_push_to_talk_enabled(want) + scope = None + if self.on_push_to_talk: + try: + scope = self.on_push_to_talk(want) + except Exception as e: + self._log.append_log(f"ERR: Push-to-talk failed — {e}") + save_push_to_talk_enabled(False) + want = False + self._apply_ptt_shortcut(want and scope != "global") + self._refresh_talk_btns() + + def _apply_ptt_shortcut(self, needed: bool): + """Bind the chord inside the window when no global hook is available. + + On macOS and Linux there is no dependency-free way to read global key + state, so the chord is at least live whenever this window has focus. + Qt gives no key-release for a QShortcut, so a press latches the mic open + and a short timer closes it; held down, auto-repeat keeps pushing that + timer out, which behaves like holding a key. + """ + from PyQt6.QtGui import QKeySequence, QShortcut + from core.hotkey import qt_sequence + + if not needed: + sc = getattr(self, "_ptt_sc", None) + if sc is not None: + sc.setEnabled(False) + self._ptt_sc = None + self._ptt_hold(False) + return + if getattr(self, "_ptt_sc", None) is not None: + return + + self._ptt_release = QTimer(self) + self._ptt_release.setSingleShot(True) + self._ptt_release.setInterval(420) + self._ptt_release.timeout.connect(lambda: self._ptt_hold(False)) + + def _press(): + self._ptt_hold(True) + self._ptt_release.start() + + self._ptt_sc = QShortcut(QKeySequence(qt_sequence()), self) + self._ptt_sc.setAutoRepeat(True) + self._ptt_sc.activated.connect(_press) + + def _ptt_hold(self, held: bool): + """Report a windowed press/release to whoever owns the microphone.""" + cb = getattr(self, "ptt_hold", None) + if cb: + try: + cb(bool(held)) + except Exception: + pass + def _toggle_wake_word(self): st = self._wake_state() if not st["ready"]: @@ -4642,6 +5332,39 @@ class JarvisUI: except Exception: pass + def glance(self, dx: float, dy: float, hold: float = 1.1) -> None: + """Ask the avatar to look somewhere for a moment (see HoloAvatar.glance).""" + try: + if self._avatar is not None: + self._avatar.glance(dx, dy, hold) + except Exception: + pass + + @property + def ptt_hold(self): + return self._win.ptt_hold + + @ptt_hold.setter + def ptt_hold(self, cb): + self._win.ptt_hold = cb + + @property + def on_push_to_talk(self): + return self._win.on_push_to_talk + + @on_push_to_talk.setter + def on_push_to_talk(self, cb): + self._win.on_push_to_talk = cb + + def push_visemes(self, frames, hop: float, at: float) -> None: + """Thread-safe: post a schedule of (level, openness, width) mouth frames + for JARVIS's own speech. `at` is the wall-clock time the batch begins to + sound, not the time of the call. See HudCanvas.push_visemes().""" + try: + self._win.hud.push_visemes(frames, hop, at) + except Exception: + pass + def notify_phone_connected(self) -> None: self._win.notify_phone_connected() @@ -4659,6 +5382,33 @@ class JarvisUI: """Thread-safe: display content in the panel below the HUD.""" self._win._content_sig.emit(title[:48], text[:4000]) + def show_quiz(self, topic: str, questions, grade=None) -> None: + """Thread-safe: put an interactive quiz on the board. + + `grade(question, given)` decides each answer — the plugin supplies it so + the marking rules live with the questions rather than being duplicated + here. Returning None from it means "JARVIS should judge this one", which + is how open answers and near-miss gap-fills are handled. + + Returns immediately: the user answers at their own pace and the finished + result is delivered back through on_text_command. + """ + self._win._quiz_sig.emit(str(topic or ""), list(questions or []), grade) + + def hide_quiz(self) -> None: + """Thread-safe: clear any quiz currently on the board.""" + self._win._quiz_hide_sig.emit() + + def show_review(self, title: str, summary: str, findings, unclear=None) -> None: + """Thread-safe: lay a document review into the panel below the HUD. + + `findings` is a list of {heading, detail, severity, quote, suggestion}; + severity is one of 'serious' / 'caution' / 'note' and decides colour and + order here, so the caller supplies no styling of its own. + """ + self._win._review_sig.emit(str(title or ""), str(summary or ""), + list(findings or []), list(unclear or [])) + def prompt_reconfig(self): """Thread-safe: show the API key setup overlay (e.g. after an auth error).""" self._win._ready = False