Add files via upload
This commit is contained in:
FatihMakes authored and 
GitHub committed
1 parent
b3f7708e0e
commit
476a9c09d6
28 files changed
+6146
-506
No files matched your search
@@ -1,4 +1,4 @@
|
||||
MARK 53 — JARVIS
|
||||
MARK LIV — JARVIS
|
||||
Copyright (c) 2026 FatihMakes
|
||||
|
||||
Licensed under the Creative Commons Attribution-NonCommercial 4.0
|
||||
|
||||
+15
-14
@@ -15,7 +15,10 @@ BASE_DIR = get_base_dir()
|
||||
API_CONFIG_PATH = BASE_DIR / "config" / "api_keys.json"
|
||||
DESKTOP = Path.home() / "Desktop"
|
||||
MAX_BUILD_ATTEMPTS = 3
|
||||
GEMINI_MODEL = "gemini-flash-latest"
|
||||
# Model choice lives in core/gemini.py, and so does the timeout and the
|
||||
# fallback ladder. Writing a model name here is what left this file hanging
|
||||
# forever whenever that one alias was unwell.
|
||||
from core import gemini
|
||||
|
||||
|
||||
def _get_api_key() -> str:
|
||||
@@ -23,13 +26,15 @@ def _get_api_key() -> str:
|
||||
return json.load(f)["gemini_api_key"]
|
||||
|
||||
|
||||
def _get_gemini(model: str = GEMINI_MODEL):
|
||||
from google import genai
|
||||
_c = genai.Client(api_key=_get_api_key())
|
||||
|
||||
def _get_gemini(tier: str = gemini.SMART):
|
||||
"""Writing and fixing code is the reasoning tier; a 60s deadline because a
|
||||
whole file can come back."""
|
||||
class _W:
|
||||
def generate_content(self, contents):
|
||||
return _c.models.generate_content(model=model, contents=contents)
|
||||
resp = gemini.call(contents, tier=tier, timeout_ms=60000)
|
||||
if resp is None:
|
||||
raise RuntimeError("every Gemini model on the ladder failed")
|
||||
return resp
|
||||
|
||||
return _W()
|
||||
|
||||
@@ -456,11 +461,8 @@ def _screen_debug_action(description, file_path, player, speak=None) -> str:
|
||||
print(f"[Code] ⚠️ Could not read file: {err}")
|
||||
|
||||
try:
|
||||
from google import genai
|
||||
from google.genai import types
|
||||
|
||||
client = genai.Client(api_key=_get_api_key())
|
||||
|
||||
image_bytes = screenshot_path.read_bytes()
|
||||
image_base64 = _image_to_base64(screenshot_path)
|
||||
|
||||
@@ -487,12 +489,11 @@ Be specific and actionable. If you see an error message, quote it exactly."""
|
||||
analysis_prompt,
|
||||
]
|
||||
|
||||
response = client.models.generate_content(
|
||||
model="gemini-flash-latest",
|
||||
contents=contents,
|
||||
)
|
||||
response = gemini.call(contents, tier=gemini.SMART, timeout_ms=45_000)
|
||||
if response is None:
|
||||
return "Sir, I couldn't reach Gemini to analyse that screenshot."
|
||||
|
||||
analysis = response.text.strip()
|
||||
analysis = (response.text or "").strip()
|
||||
print(f"[Code] ✅ Screen analysis complete")
|
||||
|
||||
try:
|
||||
|
||||
@@ -327,7 +327,6 @@ def _screen_find(description: str) -> tuple[int, int] | None:
|
||||
img.save(buf, format="PNG")
|
||||
image_bytes = buf.getvalue()
|
||||
|
||||
client = genai.Client(api_key=api_key)
|
||||
prompt = (
|
||||
f"This is a screenshot of a {w}×{h} pixel screen. "
|
||||
f"Locate the UI element described as: '{description}'. "
|
||||
@@ -335,13 +334,13 @@ def _screen_find(description: str) -> tuple[int, int] | None:
|
||||
f"If the element is not visible, reply: NOT_FOUND"
|
||||
)
|
||||
|
||||
response = client.models.generate_content(
|
||||
model="gemini-flash-lite-latest",
|
||||
contents=[
|
||||
gtypes.Part.from_bytes(data=image_bytes, mime_type="image/png"),
|
||||
prompt,
|
||||
],
|
||||
from core import gemini
|
||||
response = gemini.call(
|
||||
[gtypes.Part.from_bytes(data=image_bytes, mime_type="image/png"), prompt],
|
||||
tier=gemini.FAST, timeout_ms=20_000,
|
||||
)
|
||||
if response is None:
|
||||
return None
|
||||
|
||||
text = (response.text or "").strip()
|
||||
if "NOT_FOUND" in text.upper():
|
||||
|
||||
+5
-3
@@ -104,7 +104,6 @@ def _execute_generated_code(code: str, player=None) -> str:
|
||||
def _ask_gemini_for_desktop_action(task: str) -> str:
|
||||
|
||||
from google import genai as _genai
|
||||
_client = _genai.Client(api_key=_get_api_key())
|
||||
|
||||
desktop = str(_get_desktop())
|
||||
|
||||
@@ -142,8 +141,11 @@ Output ONLY the Python code. No explanation, no markdown, no backticks.
|
||||
Task: {task}"""
|
||||
|
||||
try:
|
||||
response = _client.models.generate_content(model="gemini-flash-latest", contents=prompt)
|
||||
code = response.text.strip()
|
||||
from core import gemini
|
||||
response = gemini.call(prompt, tier=gemini.SMART, timeout_ms=30_000)
|
||||
if response is None:
|
||||
return "ERROR: every Gemini model on the ladder failed"
|
||||
code = (response.text or "").strip()
|
||||
if code.startswith("```"):
|
||||
lines = code.split("\n")
|
||||
code = "\n".join(lines[1:-1]).strip()
|
||||
|
||||
+12
-7
@@ -16,21 +16,26 @@ BASE_DIR = get_base_dir()
|
||||
API_CONFIG_PATH = BASE_DIR / "config" / "api_keys.json"
|
||||
PROJECTS_DIR = Path.home() / "Desktop" / "JarvisProjects"
|
||||
MAX_FIX_ATTEMPTS = 5
|
||||
MODEL_PLANNER = "gemini-flash-latest"
|
||||
MODEL_WRITER = "gemini-flash-latest"
|
||||
# Model choice, timeout and fallback ladder all live in core/gemini.py.
|
||||
from core import gemini
|
||||
|
||||
MODEL_PLANNER = gemini.SMART
|
||||
MODEL_WRITER = gemini.SMART
|
||||
|
||||
def _get_api_key() -> str:
|
||||
with open(API_CONFIG_PATH, "r", encoding="utf-8") as f:
|
||||
return json.load(f)["gemini_api_key"]
|
||||
|
||||
|
||||
def _get_model(model_name: str):
|
||||
from google import genai
|
||||
_c = genai.Client(api_key=_get_api_key())
|
||||
|
||||
def _get_model(model_name: str = gemini.SMART):
|
||||
"""Planning and writing whole files — the reasoning tier, and a long
|
||||
deadline because the answer is a source file rather than a sentence."""
|
||||
class _W:
|
||||
def generate_content(self, contents):
|
||||
return _c.models.generate_content(model=model_name, contents=contents)
|
||||
resp = gemini.call(contents, tier=model_name, timeout_ms=60000)
|
||||
if resp is None:
|
||||
raise RuntimeError("every Gemini model on the ladder failed")
|
||||
return resp
|
||||
|
||||
return _W()
|
||||
|
||||
|
||||
@@ -158,9 +158,22 @@ def _resolve_path(raw: str) -> Path:
|
||||
"videos": _get_videos(),
|
||||
"home": Path.home(),
|
||||
}
|
||||
lower = raw.strip().lower()
|
||||
raw = raw.strip().strip('"').strip("'")
|
||||
lower = raw.lower()
|
||||
if lower in shortcuts:
|
||||
return shortcuts[lower]
|
||||
|
||||
# "desktop/notes/a.md" and "desktop\notes\a.md" — a shortcut followed by a
|
||||
# sub-path. Without this branch the whole string falls through to the
|
||||
# relative-path return below and is resolved against the process CWD instead
|
||||
# of the real Desktop: an "Access denied" when the project lives outside the
|
||||
# home directory, or — worse — a silent write into a stray "desktop" folder
|
||||
# inside the project when it lives inside it.
|
||||
head, sep, rest = raw.replace("\\", "/").partition("/")
|
||||
if sep and head.lower() in shortcuts:
|
||||
rest = rest.strip("/")
|
||||
return shortcuts[head.lower()] / rest if rest else shortcuts[head.lower()]
|
||||
|
||||
return Path(raw).expanduser()
|
||||
|
||||
def _format_size(b: int) -> str:
|
||||
|
||||
@@ -25,19 +25,24 @@ import tempfile
|
||||
from pathlib import Path
|
||||
from datetime import datetime
|
||||
|
||||
# Model choice, timeout and fallback ladder all live in core/gemini.py.
|
||||
from core import gemini
|
||||
|
||||
def _get_api_key() -> str:
|
||||
config_path = Path(__file__).resolve().parent.parent / "config" / "api_keys.json"
|
||||
with open(config_path, "r", encoding="utf-8") as f:
|
||||
return json.load(f)["gemini_api_key"]
|
||||
|
||||
|
||||
def _gemini_client():
|
||||
from google import genai
|
||||
_c = genai.Client(api_key=_get_api_key())
|
||||
|
||||
def _gemini_client(tier: str = gemini.SMART):
|
||||
"""Summarising documents and reading images — the reasoning tier, with a
|
||||
long deadline because the input can be a whole file."""
|
||||
class _W:
|
||||
def generate_content(self, contents):
|
||||
return _c.models.generate_content(model="gemini-flash-latest", contents=contents)
|
||||
resp = gemini.call(contents, tier=tier, timeout_ms=90000)
|
||||
if resp is None:
|
||||
raise RuntimeError("every Gemini model on the ladder failed")
|
||||
return resp
|
||||
|
||||
return _W()
|
||||
|
||||
|
||||
@@ -60,17 +60,13 @@ def _parse_date(raw: str) -> str:
|
||||
return val.strftime("%Y-%m-%d")
|
||||
|
||||
try:
|
||||
from google import genai as _genai
|
||||
_client = _genai.Client(api_key=_get_api_key())
|
||||
response = _client.models.generate_content(
|
||||
model="gemini-flash-lite-latest",
|
||||
contents=(
|
||||
from core import gemini
|
||||
result = gemini.text(
|
||||
f"Today is {today.strftime('%Y-%m-%d')}. "
|
||||
f"Convert this date expression to YYYY-MM-DD: '{raw}'. "
|
||||
f"Return ONLY the date string, nothing else."
|
||||
f"Return ONLY the date string, nothing else.",
|
||||
tier=gemini.FAST,
|
||||
)
|
||||
)
|
||||
result = response.text.strip()
|
||||
if re.match(r"\d{4}-\d{2}-\d{2}", result):
|
||||
return result
|
||||
except Exception as e:
|
||||
@@ -155,7 +151,6 @@ def _parse_flights_with_gemini(
|
||||
from google import genai as _genai
|
||||
from google.genai import types
|
||||
|
||||
_client = _genai.Client(api_key=_get_api_key())
|
||||
prompt = (
|
||||
f"Extract flight options from {origin} to {destination} on {date} "
|
||||
f"from this Google Flights page text:\n\n{raw_text[:12000]}\n\n"
|
||||
@@ -166,9 +161,11 @@ def _parse_flights_with_gemini(
|
||||
)
|
||||
|
||||
try:
|
||||
response = _client.models.generate_content(
|
||||
model="gemini-flash-latest",
|
||||
contents=prompt,
|
||||
from core import gemini
|
||||
response = gemini.call(
|
||||
prompt,
|
||||
tier=gemini.SMART,
|
||||
timeout_ms=30_000,
|
||||
config=types.GenerateContentConfig(
|
||||
system_instruction=(
|
||||
"You are a flight data extraction expert. "
|
||||
|
||||
+132
-59
@@ -1,8 +1,68 @@
|
||||
#web_search.py
|
||||
import json
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
# ── Gemini grounding quota circuit breaker ────────────────────────────────────
|
||||
# The google_search grounding tool has its own small quota, separate from plain
|
||||
# generation. Once it is spent every call returns 429 — so retrying it at the
|
||||
# top of every search only adds a dead round-trip before the DDG fallback runs.
|
||||
# After a quota error, skip Gemini entirely for a cooldown period.
|
||||
_QUOTA_COOLDOWN_SEC = 900 # 15 minutes
|
||||
_quota_blocked_until = 0.0
|
||||
_quota_lock = threading.Lock()
|
||||
|
||||
|
||||
def _gemini_available() -> bool:
|
||||
with _quota_lock:
|
||||
return time.monotonic() >= _quota_blocked_until
|
||||
|
||||
|
||||
def _note_gemini_error(exc: Exception) -> None:
|
||||
"""Trip the breaker when the error is a quota / rate-limit rejection."""
|
||||
global _quota_blocked_until
|
||||
msg = str(exc)
|
||||
if "429" in msg or "RESOURCE_EXHAUSTED" in msg:
|
||||
with _quota_lock:
|
||||
already = time.monotonic() < _quota_blocked_until
|
||||
_quota_blocked_until = time.monotonic() + _QUOTA_COOLDOWN_SEC
|
||||
if not already:
|
||||
print(
|
||||
"[WebSearch] Gemini grounding quota exhausted — skipping it for "
|
||||
f"{_QUOTA_COOLDOWN_SEC // 60} min and serving results from DDG."
|
||||
)
|
||||
|
||||
|
||||
class _QuotaCooldown(RuntimeError):
|
||||
"""Raised instead of calling Gemini while the quota breaker is open."""
|
||||
|
||||
|
||||
def _log_gemini_failure(context: str, exc: Exception) -> None:
|
||||
"""Log a Gemini failure — silently when it is just the expected cooldown."""
|
||||
if isinstance(exc, _QuotaCooldown):
|
||||
return # announced once when the breaker tripped; not a warning
|
||||
print(f"[WebSearch] \u26a0\ufe0f {context} failed ({exc}) — using DDG instead")
|
||||
|
||||
|
||||
def _run_bounded(fn, timeout: float, label: str = "task"):
|
||||
"""Run fn() in a daemon thread; return its result, or None if it overruns."""
|
||||
box = [None]
|
||||
|
||||
def _run():
|
||||
try:
|
||||
box[0] = fn()
|
||||
except Exception as e:
|
||||
_log_gemini_failure(label, e)
|
||||
|
||||
t = threading.Thread(target=_run, daemon=True)
|
||||
t.start()
|
||||
t.join(timeout)
|
||||
if t.is_alive():
|
||||
print(f"[WebSearch] {label} exceeded {timeout:.0f}s — moving on")
|
||||
return box[0]
|
||||
|
||||
def _get_base_dir() -> Path:
|
||||
if getattr(sys, "frozen", False):
|
||||
return Path(sys.executable).parent
|
||||
@@ -19,14 +79,22 @@ def _get_api_key() -> str:
|
||||
|
||||
|
||||
def _gemini_search(query: str) -> str:
|
||||
from google import genai
|
||||
if not _gemini_available():
|
||||
raise _QuotaCooldown("Gemini grounding is in quota cooldown")
|
||||
|
||||
client = genai.Client(api_key=_get_api_key())
|
||||
response = client.models.generate_content(
|
||||
model="gemini-flash-latest",
|
||||
contents=query,
|
||||
from core import gemini
|
||||
|
||||
# Grounded search reads a live page, so it gets a longer deadline than the
|
||||
# default — but it still HAS one, and it still walks the fallback ladder.
|
||||
try:
|
||||
response = gemini.call(query, tier=gemini.SEARCH,
|
||||
config={"tools": [{"google_search": {}}]},
|
||||
)
|
||||
timeout_ms=30_000)
|
||||
if response is None:
|
||||
raise RuntimeError("every Gemini model on the ladder failed")
|
||||
except Exception as e:
|
||||
_note_gemini_error(e)
|
||||
raise
|
||||
|
||||
text = ""
|
||||
for part in response.candidates[0].content.parts:
|
||||
@@ -39,13 +107,30 @@ def _gemini_search(query: str) -> str:
|
||||
return text
|
||||
|
||||
|
||||
def _ddg_search(query: str, max_results: int = 6) -> list[dict]:
|
||||
def _get_ddgs():
|
||||
"""
|
||||
Returns the DDGS class. The package was renamed duckduckgo-search -> ddgs;
|
||||
the legacy package's endpoints are now rejected by DuckDuckGo (news() gets a
|
||||
403 Ratelimit, text() silently returns zero results), so warn loudly if we
|
||||
end up on it instead of failing in silence.
|
||||
"""
|
||||
try:
|
||||
from ddgs import DDGS
|
||||
return DDGS
|
||||
except ImportError:
|
||||
from duckduckgo_search import DDGS
|
||||
print(
|
||||
"[WebSearch] ⚠️ Using the deprecated 'duckduckgo-search' package — "
|
||||
"DuckDuckGo blocks its endpoints, so every search will come back "
|
||||
"empty. Fix with: pip install -U ddgs"
|
||||
)
|
||||
return DDGS
|
||||
|
||||
|
||||
def _ddg_search(query: str, max_results: int = 6) -> list[dict]:
|
||||
DDGS = _get_ddgs()
|
||||
results = []
|
||||
try:
|
||||
with DDGS() as ddgs:
|
||||
for r in ddgs.text(query, max_results=max_results):
|
||||
results.append({
|
||||
@@ -53,16 +138,14 @@ def _ddg_search(query: str, max_results: int = 6) -> list[dict]:
|
||||
"snippet": r.get("body", ""),
|
||||
"url": r.get("href", ""),
|
||||
})
|
||||
except Exception as e:
|
||||
print(f"[WebSearch] ⚠️ DDG text() failed: {e}")
|
||||
return results
|
||||
|
||||
|
||||
def _ddg_news(query: str, max_results: int = 8) -> list[dict]:
|
||||
"""DDG news search — returns actual articles, not website homepages."""
|
||||
try:
|
||||
from ddgs import DDGS
|
||||
except ImportError:
|
||||
from duckduckgo_search import DDGS
|
||||
|
||||
DDGS = _get_ddgs()
|
||||
results = []
|
||||
try:
|
||||
with DDGS() as ddgs:
|
||||
@@ -75,6 +158,9 @@ def _ddg_news(query: str, max_results: int = 8) -> list[dict]:
|
||||
})
|
||||
except Exception as e:
|
||||
print(f"[WebSearch] ⚠️ DDG news() failed ({e}) — falling back to text search")
|
||||
# Also covers the legacy-package case, where news() returns an empty list
|
||||
# instead of raising.
|
||||
if not results:
|
||||
results = _ddg_search(query, max_results=max_results)
|
||||
return results
|
||||
|
||||
@@ -120,14 +206,16 @@ def _gemini_headlines(n: int = 5) -> tuple[list[str], str]:
|
||||
Returns (headline_list, raw_text_for_display).
|
||||
"""
|
||||
import re
|
||||
from google import genai
|
||||
from core import gemini
|
||||
|
||||
client = genai.Client(api_key=_get_api_key())
|
||||
response = client.models.generate_content(
|
||||
model="gemini-flash-latest",
|
||||
contents=f"Current world news: {n} headlines. Numbered list, titles only.",
|
||||
response = gemini.call(
|
||||
f"Current world news: {n} headlines. Numbered list, titles only.",
|
||||
tier=gemini.SEARCH,
|
||||
config={"tools": [{"google_search": {}}]},
|
||||
timeout_ms=30_000,
|
||||
)
|
||||
if response is None:
|
||||
return [], ""
|
||||
|
||||
raw = ""
|
||||
for part in response.candidates[0].content.parts:
|
||||
@@ -157,58 +245,43 @@ def _search(query: str) -> str:
|
||||
try:
|
||||
return _gemini_search(query)
|
||||
except Exception as e:
|
||||
print(f"[WebSearch] ⚠️ Gemini failed ({e}) — trying DDG...")
|
||||
_log_gemini_failure("Gemini search", e)
|
||||
results = _ddg_search(query)
|
||||
return _format_ddg(query, results)
|
||||
|
||||
|
||||
def _news(query: str) -> str:
|
||||
"""
|
||||
Runs Gemini grounded search AND DDG news in parallel.
|
||||
Returns whichever delivers a valid result first; cancels the other.
|
||||
"""
|
||||
import threading
|
||||
DDG first, Gemini as backup.
|
||||
|
||||
The old version raced both backends in parallel and kept the first answer.
|
||||
That burned one google_search grounding call on *every* news request —
|
||||
including the startup briefing — even when DDG had already won the race.
|
||||
Grounding has a small quota, so it ran dry after a handful of launches and
|
||||
then 429'd for everything else (research/compare), which are the modes that
|
||||
actually need a synthesised answer.
|
||||
|
||||
DDG news returns in well under a second and gives raw headlines, which is
|
||||
exactly what the briefing wants, so it goes first and Gemini is only touched
|
||||
when DDG comes back empty.
|
||||
"""
|
||||
gemini_query = f"latest news today: {query}" if query else "top world news today"
|
||||
ddg_query = query if query else "world news today"
|
||||
|
||||
result_box = [None] # first valid result lands here
|
||||
lock = threading.Lock()
|
||||
done_evt = threading.Event()
|
||||
failures = [0]
|
||||
def _ddg_attempt() -> str:
|
||||
return _format_news(ddg_query, _ddg_news(ddg_query, max_results=8))
|
||||
|
||||
def _store(r: str) -> None:
|
||||
if r and len(r) > 60:
|
||||
with lock:
|
||||
if result_box[0] is None:
|
||||
result_box[0] = r
|
||||
done_evt.set()
|
||||
else:
|
||||
with lock:
|
||||
failures[0] += 1
|
||||
if failures[0] >= 2: # both failed — unblock caller
|
||||
done_evt.set()
|
||||
text = _run_bounded(_ddg_attempt, timeout=5.0, label="DDG news")
|
||||
if text and len(text) > 60 and not text.startswith("No news found"):
|
||||
return text
|
||||
|
||||
def _try_gemini():
|
||||
try:
|
||||
_store(_gemini_search(gemini_query))
|
||||
except Exception as e:
|
||||
print(f"[WebSearch] ⚠️ Gemini news failed ({e})")
|
||||
_store("")
|
||||
text = _run_bounded(
|
||||
lambda: _gemini_search(gemini_query), timeout=6.0, label="Gemini news"
|
||||
)
|
||||
if text and len(text) > 60:
|
||||
return text
|
||||
|
||||
def _try_ddg():
|
||||
try:
|
||||
results = _ddg_news(ddg_query, max_results=8)
|
||||
_store(_format_news(ddg_query, results))
|
||||
except Exception as e:
|
||||
print(f"[WebSearch] ⚠️ DDG news failed ({e})")
|
||||
_store("")
|
||||
|
||||
threading.Thread(target=_try_gemini, daemon=True).start()
|
||||
threading.Thread(target=_try_ddg, daemon=True).start()
|
||||
|
||||
done_evt.wait(timeout=10.0)
|
||||
return result_box[0] or f"No news found for: {query}"
|
||||
return f"No news found for: {query}"
|
||||
|
||||
|
||||
def _research(query: str) -> str:
|
||||
@@ -223,7 +296,7 @@ def _research(query: str) -> str:
|
||||
try:
|
||||
return _gemini_search(research_query)
|
||||
except Exception as e:
|
||||
print(f"[WebSearch] ⚠️ Research Gemini failed ({e}) — DDG fallback...")
|
||||
_log_gemini_failure("Gemini research", e)
|
||||
results = _ddg_search(query, max_results=10)
|
||||
return _format_ddg(query, results)
|
||||
|
||||
@@ -234,7 +307,7 @@ def _price(query: str) -> str:
|
||||
try:
|
||||
return _gemini_search(price_query)
|
||||
except Exception as e:
|
||||
print(f"[WebSearch] ⚠️ Price Gemini failed ({e}) — DDG fallback...")
|
||||
_log_gemini_failure("Gemini price", e)
|
||||
results = _ddg_search(f"{query} price buy", max_results=6)
|
||||
return _format_ddg(query, results)
|
||||
|
||||
@@ -247,7 +320,7 @@ def _compare(items: list[str], aspect: str) -> str:
|
||||
try:
|
||||
return _gemini_search(query)
|
||||
except Exception as e:
|
||||
print(f"[WebSearch] ⚠️ Gemini compare failed: {e} — falling back to DDG")
|
||||
_log_gemini_failure("Gemini compare", e)
|
||||
|
||||
all_results: dict[str, list] = {}
|
||||
for item in items:
|
||||
|
||||
@@ -167,15 +167,17 @@ def _get_transcript(video_id: str) -> str | None:
|
||||
|
||||
|
||||
def _summarize_with_gemini(transcript: str, video_url: str) -> str:
|
||||
from google import genai as _genai
|
||||
from google.genai import types
|
||||
from core import gemini
|
||||
|
||||
_client = _genai.Client(api_key=_get_api_key())
|
||||
max_chars = 80000
|
||||
truncated = transcript[:max_chars] + ("..." if len(transcript) > max_chars else "")
|
||||
response = _client.models.generate_content(
|
||||
model="gemini-flash-latest",
|
||||
contents=f"Please summarize this YouTube video transcript:\n\n{truncated}",
|
||||
# A whole transcript can be 80k characters, hence the long deadline — but a
|
||||
# deadline there is, and the ladder in core/gemini.py picks the model.
|
||||
response = gemini.call(
|
||||
f"Please summarize this YouTube video transcript:\n\n{truncated}",
|
||||
tier=gemini.SMART,
|
||||
timeout_ms=60_000,
|
||||
config=types.GenerateContentConfig(
|
||||
system_instruction=(
|
||||
"You are JARVIS, an AI assistant. "
|
||||
@@ -186,7 +188,9 @@ def _summarize_with_gemini(transcript: str, video_url: str) -> str:
|
||||
)
|
||||
)
|
||||
)
|
||||
return response.text.strip()
|
||||
if response is None:
|
||||
return "I couldn't reach Gemini to summarise that transcript, sir."
|
||||
return (response.text or "").strip()
|
||||
|
||||
|
||||
def _save_summary(content: str, video_url: str) -> str:
|
||||
|
||||
+33
-5
@@ -39,6 +39,21 @@ _DEFAULT_PARAMS = {"type": "OBJECT", "properties": {}}
|
||||
_CTX_KEYS = ("player", "speak", "response", "session_memory")
|
||||
|
||||
|
||||
# A tool may declare that the model should NOT be held up waiting for it.
|
||||
# `behavior` goes to the API with the declaration; `scheduling` decides when the
|
||||
# eventual result is allowed back into the conversation:
|
||||
# WHEN_IDLE — wait for a gap in the speech (the sane default)
|
||||
# SILENT — record it, do not prompt a reply (the tool already announced)
|
||||
# INTERRUPT — cut in immediately (only when the answer cannot wait)
|
||||
_BEHAVIORS = ("BLOCKING", "NON_BLOCKING")
|
||||
_SCHEDULING = ("WHEN_IDLE", "SILENT", "INTERRUPT")
|
||||
|
||||
|
||||
def _opt_upper(value, allowed: tuple[str, ...]) -> Optional[str]:
|
||||
v = str(value or "").strip().upper()
|
||||
return v if v in allowed else None
|
||||
|
||||
|
||||
@dataclass
|
||||
class ActionRecord:
|
||||
name: str
|
||||
@@ -48,6 +63,8 @@ class ActionRecord:
|
||||
file: str = ""
|
||||
valid: bool = False
|
||||
error: str = ""
|
||||
behavior: Optional[str] = None # None = the API's default (blocking)
|
||||
scheduling: Optional[str] = None # None = the API's default (WHEN_IDLE)
|
||||
|
||||
|
||||
class ActionRegistry:
|
||||
@@ -58,14 +75,23 @@ class ActionRegistry:
|
||||
|
||||
# -- called by main.py at LiveConnectConfig build time --
|
||||
def get_tool_declarations(self) -> list[dict]:
|
||||
return [
|
||||
{"name": rec.name, "description": rec.description, "parameters": rec.parameters}
|
||||
for rec in self._actions.values()
|
||||
]
|
||||
out = []
|
||||
for rec in self._actions.values():
|
||||
decl = {"name": rec.name, "description": rec.description,
|
||||
"parameters": rec.parameters}
|
||||
if rec.behavior:
|
||||
decl["behavior"] = rec.behavior
|
||||
out.append(decl)
|
||||
return out
|
||||
|
||||
def has(self, name: str) -> bool:
|
||||
return name in self._actions
|
||||
|
||||
def scheduling(self, name: str) -> Optional[str]:
|
||||
"""How this action's result should re-enter the conversation, if it said."""
|
||||
rec = self._actions.get(name)
|
||||
return rec.scheduling if rec else None
|
||||
|
||||
def names(self) -> set[str]:
|
||||
return set(self._actions.keys())
|
||||
|
||||
@@ -123,7 +149,9 @@ def _validate(module, filename: str) -> ActionRecord:
|
||||
error="TOOL['handler'] missing or not callable.")
|
||||
|
||||
return ActionRecord(name=name, description=description.strip(), parameters=parameters,
|
||||
handler=handler, file=filename, valid=True, error="")
|
||||
handler=handler, file=filename, valid=True, error="",
|
||||
behavior=_opt_upper(tool.get("behavior"), _BEHAVIORS),
|
||||
scheduling=_opt_upper(tool.get("scheduling"), _SCHEDULING))
|
||||
|
||||
|
||||
def discover_actions(actions_dir: Path, reserved_names: set[str] | None = None,
|
||||
|
||||
+715
@@ -0,0 +1,715 @@
|
||||
"""
|
||||
Holographic AI head for the HUD centre — the thing that used to be a ring stack
|
||||
with the assistant's name in the middle.
|
||||
|
||||
Design notes
|
||||
------------
|
||||
* **The face is real human geometry.** `core.avatar_mesh` builds the head around
|
||||
MediaPipe's canonical face model, so eyelids, nostrils, lips and cheekbones
|
||||
are measured anatomy rather than fitted curves. This renderer's whole job is
|
||||
to light it, pose it and animate it.
|
||||
* **Software rendered, on purpose.** Everything is QPainter, so there is no
|
||||
OpenGL context, no shader compile, no GPU driver to disagree with us and no
|
||||
new pip dependency. It looks the same on a gaming rig, a 2013 laptop, a VM
|
||||
and a remote desktop session.
|
||||
* **Lip-sync comes from the audio pipeline, not from the avatar.** `main.py`
|
||||
already computes a real RMS level off the PCM (`_pcm_level`) for both the mic
|
||||
and JARVIS's own output. The avatar just consumes that number, so there is no
|
||||
second audio path to fall out of sync. The mouth only tracks the level while
|
||||
JARVIS is *speaking* — during listening the same level drives the aura, so the
|
||||
head never lip-syncs to the user's voice.
|
||||
|
||||
The renderer is theme-agnostic: `paint()` takes its colours as arguments, which
|
||||
is what lets the HueWheel accent picker retint the avatar for free.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
import random
|
||||
|
||||
import numpy as np
|
||||
from PyQt6.QtCore import QLineF, QPointF, QRectF, Qt
|
||||
from PyQt6.QtGui import QBrush, QColor, QPainter, QPen, QPolygonF, QRadialGradient
|
||||
|
||||
from core.avatar_mesh import JAW_MAX, JAW_PIVOT, get_head_mesh
|
||||
|
||||
# Perspective camera distance in head-half-heights. Large enough that the nose
|
||||
# does not balloon, small enough to keep a sense of depth.
|
||||
_CAM_D = 4.6
|
||||
|
||||
# Wireframe opacity buckets, so the whole lattice draws in a handful of batched
|
||||
# drawLines() calls instead of one call per line.
|
||||
_BUCKETS = 4
|
||||
_MIN_ALPHA = 0.05
|
||||
|
||||
# Resolution of the surface-shading colour ramp. Banding across a filled facet
|
||||
# is far more visible than banding in line alpha, so this is fine-grained — and
|
||||
# being a lookup it costs nothing per face.
|
||||
_LUT_N = 192
|
||||
|
||||
# How far the brows travel at full lift, in head-half-heights. Derived, not
|
||||
# tuned: the brow-to-eye gap is 0.198 and a real raise covers about a third of
|
||||
# it, then the drawn landmarks only carry half the rig weight.
|
||||
_BROW_LIFT = 0.14
|
||||
|
||||
# Mouth timing, as time constants in seconds rather than per-frame fractions.
|
||||
# A fixed per-frame lerp silently changes speed with the frame rate: the HUD
|
||||
# runs at 60 Hz here, throttles its paint to 30, and drops to 20 when idle, so
|
||||
# the same constant meant three different mouths. These do not.
|
||||
#
|
||||
# Shutting is the fastest of the three, and it is measured rather than chosen.
|
||||
# A short closure occupies a single 20 ms schedule frame, so the mouth has one
|
||||
# step to reach it: at 20 ms the jaw got a third of the way and the closure
|
||||
# vanished, at 12 ms it arrives, and going below that changes nothing because
|
||||
# the analysis window is then the limit, not the smoothing. Halving it doubled
|
||||
# the closures the mouth visibly makes across a test paragraph, 5 of 21 to 10,
|
||||
# with no loss of opening on the vowels. Only the return to rest, once talking
|
||||
# has actually stopped, is leisurely.
|
||||
_TAU_OPEN = 0.022 # jaw dropping toward a vowel
|
||||
_TAU_SHUT = 0.012 # lips closing on a consonant, mid-word
|
||||
_TAU_REST = 0.055 # settling back to rest after speech ends
|
||||
_TAU_SHAPE = 0.018 # viseme openness following the schedule
|
||||
|
||||
# Only the microphone path needs a level floor: it has one coarse RMS and no way
|
||||
# to tell speech from room tone. JARVIS's own voice arrives as a per-20 ms
|
||||
# schedule whose silences are already silent, so it needs no floor and must not
|
||||
# have one — a floor there swallows the gaps between words.
|
||||
_MIC_FLOOR = 0.14
|
||||
|
||||
# How far below this voice's own loud level counts as a closure: -20 dB, which
|
||||
# is what a stop consonant actually drops to. Expressed as a ratio so it holds
|
||||
# at any speaker volume.
|
||||
_CLOSE_FRAC = 0.10
|
||||
|
||||
|
||||
def _rate(dt: float, tau: float) -> float:
|
||||
"""Per-frame lerp factor for an exponential approach with time constant
|
||||
`tau`. Frame-rate independent: the motion takes the same wall-clock time at
|
||||
20, 30 or 60 fps, and a long frame catches up instead of stalling."""
|
||||
return 1.0 - math.exp(-dt / tau)
|
||||
|
||||
|
||||
def _c(col: QColor, a: float) -> QColor:
|
||||
"""Copy of `col` at alpha `a` (0-255, clamped)."""
|
||||
q = QColor(col)
|
||||
q.setAlpha(int(max(0.0, min(255.0, a))))
|
||||
return q
|
||||
|
||||
|
||||
def _blend(bg: QColor, col: QColor, a: float) -> QColor:
|
||||
"""`col` at alpha `a` pre-mixed onto `bg`, returned fully **opaque**.
|
||||
|
||||
Qt's raster engine has a fast path for opaque antialiased lines and a much
|
||||
slower blended path for everything else — measured at 1.0 ms versus 2.9 ms
|
||||
for the same 780 lines. The HUD paints a flat background behind the avatar,
|
||||
so mixing the alpha in by hand is visually equivalent and three times cheaper.
|
||||
"""
|
||||
f = max(0.0, min(1.0, a / 255.0))
|
||||
return QColor(int(bg.red() + (col.red() - bg.red()) * f),
|
||||
int(bg.green() + (col.green() - bg.green()) * f),
|
||||
int(bg.blue() + (col.blue() - bg.blue()) * f))
|
||||
|
||||
|
||||
class HoloAvatar:
|
||||
"""Animated holographic head. One instance per HUD canvas.
|
||||
|
||||
Lifecycle:
|
||||
av = HoloAvatar()
|
||||
av.step(dt, amp, speaking=..., muted=...) # once per tick
|
||||
av.paint(painter, cx, cy, r, primary, accent, bg) # once per frame
|
||||
"""
|
||||
|
||||
# Look: True paints a lit, solid head with a wireframe over it; False is a
|
||||
# see-through glass wireframe. Flip here, or per instance.
|
||||
shaded = True
|
||||
|
||||
def __init__(self) -> None:
|
||||
mesh = get_head_mesh()
|
||||
self._v0 = mesh["verts"]
|
||||
self._n0 = mesh["normals"]
|
||||
self._jaw = mesh["jaw"]
|
||||
self._brow_w = mesh["brow"]
|
||||
self._lips_w = mesh["lips"]
|
||||
self._lip_c = mesh["lip_centre"]
|
||||
self._fade = mesh["fade"]
|
||||
self._f = mesh["faces"]
|
||||
self._fgroup = mesh["face_group"]
|
||||
self._fa, self._fb, self._fc = (self._f[:, i] for i in range(3))
|
||||
self._e0 = mesh["edges"][:, 0]
|
||||
self._e1 = mesh["edges"][:, 1]
|
||||
self._lm = mesh["landmarks"]
|
||||
# The inner-lip ring runs lower-lip left→right, then upper-lip back.
|
||||
# Splitting it lets the upper arc anchor a strip of teeth, which is what
|
||||
# keeps an open mouth from reading as a hole punched in the face.
|
||||
lips_in = mesh["landmarks"]["lips_in"]
|
||||
self._lip_up = np.concatenate([lips_in[10:], lips_in[:1]])
|
||||
|
||||
# Crown (+1.0) down to the bottom of the neck, in head-half-heights.
|
||||
# Callers size the head to the room they have with this.
|
||||
self.SPAN = mesh["span"][0] - mesh["span"][1]
|
||||
|
||||
self._lut_cache: list = []
|
||||
self._lut_key = None
|
||||
|
||||
n = self._v0.shape[0]
|
||||
self._v = np.empty((n, 3), dtype=np.float32)
|
||||
|
||||
self._t = 0.0
|
||||
self._sway = 0.0 # integrated sway phase — see step()
|
||||
self._yaw = 0.0
|
||||
self._pitch = 0.0
|
||||
self._mouth = 0.0 # 0..1 smoothed jaw opening
|
||||
self._glow = 0.0 # 0..1 smoothed overall energy
|
||||
self._scan = -1.6 # vertical position of the energy sweep
|
||||
self._blink = 0.0 # 0 = open, 1 = shut
|
||||
self._blink_at = 3.0
|
||||
|
||||
# ── expression ──────────────────────────────────────────────────────
|
||||
# Speech is not just a moving jaw. Brows ride the loudness envelope,
|
||||
# eyes widen with the brows, and the gaze flicks between fixation
|
||||
# points — those three are what make it read as talking rather than as
|
||||
# a puppet chewing.
|
||||
self._amp_slow = 0.0
|
||||
self._expr = 0.0
|
||||
self._expr_tgt = 0.0
|
||||
self._expr_at = 0.0
|
||||
self._brow = 0.0 # smoothed brow lift, -0.4 .. 1.2
|
||||
self._emph = 0.0 # syllable emphasis, drives the head nod
|
||||
self._gaze = [0.0, 0.0]
|
||||
self._gaze_tgt = [0.0, 0.0]
|
||||
self._gaze_at = 0.0
|
||||
|
||||
# ── state expression ────────────────────────────────────────────────
|
||||
# The face is the fastest status indicator in the app: you read a gaze
|
||||
# before you read a word. Saccades orbit a bias that the assistant's
|
||||
# state moves — eyes off to the side while it thinks, back on you while
|
||||
# it listens, lids low while it sleeps.
|
||||
self._gaze_bias = [0.0, 0.0]
|
||||
self._bias_tgt = [0.0, 0.0]
|
||||
self._bias_at = 0.0
|
||||
self._lids = 1.0 # 1 = wide, 0 = shut; low while asleep
|
||||
self._brow_bias = 0.0 # concentration pulls the brows down
|
||||
self._glance = None # (dx, dy, until_t) — a deliberate look
|
||||
|
||||
# ── viseme ──────────────────────────────────────────────────────────
|
||||
# Loudness alone only answers "how far open", which is why an RMS-driven
|
||||
# mouth flaps rather than speaks. These two carry the *shape*: how open
|
||||
# the jaw is for this sound, and whether the lips are spread (/i/) or
|
||||
# rounded (/u/). They come from a formant read of the audio actually
|
||||
# being played — see `_pcm_visemes` in main.py.
|
||||
self._v_open = 1.0
|
||||
self._v_wide = 0.0
|
||||
self._wide = 0.0 # smoothed lip spread, -1 round .. +1 spread
|
||||
self._v_peak = 0.18 # running estimate of this voice's loud level
|
||||
|
||||
# ── animation ───────────────────────────────────────────────────────────
|
||||
|
||||
def _mouth_step(self, dt: float, amp: float, live: bool,
|
||||
v_open: float | None, v_level: float | None) -> None:
|
||||
"""One increment of the jaw. Called once per viseme frame while JARVIS
|
||||
speaks, once per rendered frame otherwise."""
|
||||
if v_open is None:
|
||||
shape = 1.0
|
||||
else:
|
||||
self._v_open += (v_open - self._v_open) * _rate(dt, _TAU_SHAPE)
|
||||
shape = self._v_open
|
||||
|
||||
if v_level is None:
|
||||
gated = max(0.0, (amp - _MIC_FLOOR) / (1.0 - _MIC_FLOOR))
|
||||
drive = (gated ** 0.6) * (shape ** 0.75)
|
||||
else:
|
||||
# Speech RMS spends most of its time well below full scale, so the
|
||||
# raw value alone would only ever half-open the jaw. Normalise it
|
||||
# against a running estimate of this voice's own loud level rather
|
||||
# than a constant: it then reads the same whether the user has the
|
||||
# volume low or the model happens to be speaking softly.
|
||||
self._v_peak = max(v_level, self._v_peak - dt * 0.55)
|
||||
ref = max(0.18, self._v_peak)
|
||||
# The floor is a fraction of this voice's own loud level, not a
|
||||
# fixed number, so it means the same thing at any volume and in any
|
||||
# language. A stop consonant drops 20 dB or more below the vowels
|
||||
# around it, which is this ratio — so a real closure lands at
|
||||
# exactly zero rather than at some small positive value the curve
|
||||
# would otherwise lift back up. That lift is what kept the mouth
|
||||
# from ever quite shutting between words.
|
||||
q = (v_level - _CLOSE_FRAC * ref) / (ref * (1.0 - _CLOSE_FRAC))
|
||||
drive = max(0.0, min(1.0, q)) ** 0.85 * (shape ** 0.75)
|
||||
|
||||
target = min(1.0, drive) if live else 0.0
|
||||
if target > self._mouth:
|
||||
tau = _TAU_OPEN
|
||||
elif live:
|
||||
tau = _TAU_SHUT # mid-word: a consonant, and it must shut now
|
||||
else:
|
||||
tau = _TAU_REST # speech is over; settle, don't snap
|
||||
self._mouth += (target - self._mouth) * _rate(dt, tau)
|
||||
if self._mouth < 0.002:
|
||||
self._mouth = 0.0
|
||||
|
||||
def step(self, dt: float, amp: float, speaking: bool = False,
|
||||
muted: bool = False, state: str = "",
|
||||
v_open: float | None = None, v_wide: float = 0.0,
|
||||
v_level: float | None = None,
|
||||
v_seq: list | None = None, v_hop: float = 0.02) -> None:
|
||||
"""Advance the animation.
|
||||
|
||||
`amp` is the 0..1 display audio level. `v_open` / `v_wide` / `v_level`
|
||||
are the viseme schedule's shape and true level for this instant; passing
|
||||
None falls back to loudness-only articulation, which is what the
|
||||
microphone path uses. `v_seq` is every schedule frame the last rendered
|
||||
frame spanned, so no closure is lost when the paint rate drops.
|
||||
"""
|
||||
dt = max(0.001, min(0.10, float(dt)))
|
||||
self._t += dt
|
||||
t = self._t
|
||||
amp = max(0.0, min(1.0, float(amp)))
|
||||
live = speaking and not muted
|
||||
|
||||
# Idle sway. The phase is *integrated* rather than taken as
|
||||
# sin(t * rate * speed): multiplying absolute time by a speed that
|
||||
# changes when JARVIS starts or stops talking jumps the phase by
|
||||
# t * rate * delta, which after a minute of uptime is several radians
|
||||
# and visibly teleports the head the instant a sentence ends.
|
||||
speed = (1.0 if not muted else 0.55) * (1.25 if live else 1.0)
|
||||
self._sway += dt * speed
|
||||
s = self._sway
|
||||
self._yaw = 0.26 * math.sin(s * 0.31) + 0.09 * math.sin(s * 0.73 + 1.3)
|
||||
self._pitch = (0.060 * math.sin(s * 0.23 + 0.7)
|
||||
+ 0.024 * math.sin(s * 0.61))
|
||||
|
||||
# Mouth. Which level is driving it matters more than any rate here.
|
||||
#
|
||||
# `v_level` is this 20 ms frame's own RMS, taken from the very audio
|
||||
# about to be heard, so its silences are real silences. `amp` is the
|
||||
# waveform display's level, and that one is a *peak hold*: it keeps the
|
||||
# loudest value it has seen and decays gently, on purpose, so the bars
|
||||
# do not stutter between audio chunks. Driving a mouth from a peak hold
|
||||
# is why the gaps between words never closed — the hold spans exactly
|
||||
# the consonant it was supposed to reveal. So the schedule drives the
|
||||
# jaw whenever there is one, and `amp` is left to the microphone path,
|
||||
# which has nothing better.
|
||||
# Advance the mouth once per *schedule* frame rather than once per
|
||||
# rendered frame. A bilabial closure lasts around 40 ms — two frames of
|
||||
# a 50 Hz schedule — and the HUD throttles its paint to 30 fps and to 20
|
||||
# when idle. Point-sampling at 20 fps steps 50 ms at a time, so a whole
|
||||
# closure can fall between two samples and simply never be seen; that is
|
||||
# information loss no smoothing constant can recover. Sub-stepping costs
|
||||
# a few float operations per frame and makes the mouth identical at 20,
|
||||
# 30 and 60 fps.
|
||||
# An empty list is meaningful and is not the same as None: it says a
|
||||
# schedule is playing but this tick landed inside a frame already
|
||||
# spoken. The mouth's clock is the schedule's, so the right thing then
|
||||
# is to do nothing. Re-stepping the same frame — which is what a
|
||||
# truthiness test here would do — advances the jaw twice for one 20 ms
|
||||
# of audio, and at 60 fps that alone made the mouth behave differently
|
||||
# than at 20.
|
||||
if v_seq is not None:
|
||||
for lv, op, _wd in v_seq:
|
||||
self._mouth_step(v_hop, amp, live, op, lv)
|
||||
else:
|
||||
self._mouth_step(dt, amp, live, v_open, v_level)
|
||||
|
||||
# Syllable emphasis. Applied unconditionally: `_emph` decays to zero on
|
||||
# its own once the mouth closes, whereas gating it on `live` deleted the
|
||||
# whole offset in a single frame and snapped the head at sentence end.
|
||||
self._emph += (self._mouth - self._emph) * _rate(
|
||||
dt, 0.055 if self._mouth > self._emph else 0.32)
|
||||
self._pitch -= self._emph * 0.028
|
||||
self._yaw += 0.018 * math.sin(t * 1.7) * self._emph
|
||||
|
||||
# Loudness envelope, deliberately lazier than the mouth: brows track the
|
||||
# shape of a phrase, not individual syllables.
|
||||
env = amp if live else 0.0
|
||||
self._amp_slow += (env - self._amp_slow) * _rate(
|
||||
dt, 0.16 if env > self._amp_slow else 0.36)
|
||||
|
||||
if live:
|
||||
if t >= self._expr_at:
|
||||
self._expr_tgt = random.uniform(-0.35, 1.0)
|
||||
self._expr_at = t + 1.1 + 2.0 * random.random()
|
||||
else:
|
||||
self._expr_tgt = 0.0
|
||||
self._expr_at = t + 0.8
|
||||
self._expr += (self._expr_tgt - self._expr) * 0.075
|
||||
|
||||
brow_t = 0.55 * self._amp_slow + 0.60 * self._expr + self._brow_bias
|
||||
self._brow += (max(-0.4, min(1.2, brow_t)) - self._brow) * 0.20
|
||||
|
||||
# ── what the state does to the face ─────────────────────────────────
|
||||
st = (state or "").upper()
|
||||
thinking = st in ("THINKING", "PROCESSING")
|
||||
asleep = st in ("SLEEPING", "STANDBY", "OFFLINE")
|
||||
|
||||
if thinking:
|
||||
# People look away to think, and hold it. The direction re-rolls
|
||||
# slowly so it reads as thought rather than as scanning.
|
||||
if t >= self._bias_at:
|
||||
self._bias_tgt = [random.choice((-1.0, 1.0)) * random.uniform(0.45, 0.8),
|
||||
random.uniform(0.25, 0.55)]
|
||||
self._bias_at = t + 1.4 + 1.6 * random.random()
|
||||
brow_bias, lid_tgt = -0.28, 0.94
|
||||
elif asleep:
|
||||
self._bias_tgt = [0.0, -0.25]
|
||||
brow_bias, lid_tgt = -0.05, 0.22
|
||||
else:
|
||||
# LISTENING / idle / speaking: eyes come back to the user.
|
||||
self._bias_tgt = [0.0, 0.0]
|
||||
self._bias_at = 0.0
|
||||
brow_bias = 0.10 if st == "LISTENING" else 0.0
|
||||
lid_tgt = 1.0
|
||||
|
||||
for i in (0, 1):
|
||||
self._gaze_bias[i] += (self._bias_tgt[i] - self._gaze_bias[i]) * 0.06
|
||||
self._lids += (lid_tgt - self._lids) * 0.08
|
||||
self._brow_bias += (brow_bias - self._brow_bias) * 0.06
|
||||
|
||||
# Gaze: saccades are near-instant jumps between fixations, and they get
|
||||
# more frequent when there is something to say. While thinking they slow
|
||||
# right down — a darting eye reads as nervous, not thoughtful.
|
||||
if t >= self._gaze_at:
|
||||
reach = 0.9 if live else (0.35 if thinking else 0.55)
|
||||
self._gaze_tgt = [random.uniform(-1.0, 1.0) * reach,
|
||||
random.uniform(-1.0, 1.0) * reach * 0.55]
|
||||
if live:
|
||||
self._gaze_at = t + 0.55 + 1.7 * random.random()
|
||||
elif thinking:
|
||||
self._gaze_at = t + 1.8 + 2.4 * random.random()
|
||||
else:
|
||||
self._gaze_at = t + 1.3 + 2.8 * random.random()
|
||||
|
||||
# A deliberate glance (something appeared on screen) overrides the
|
||||
# wandering for a moment, then hands control back.
|
||||
if self._glance is not None:
|
||||
gx, gy, until = self._glance
|
||||
if t < until:
|
||||
self._gaze_tgt = [gx, gy]
|
||||
else:
|
||||
self._glance = None
|
||||
|
||||
for i, b in enumerate(self._gaze_bias):
|
||||
tgt = max(-1.0, min(1.0, self._gaze_tgt[i] + b))
|
||||
self._gaze[i] += (tgt - self._gaze[i]) * 0.30
|
||||
|
||||
# Lips lead the jaw slightly in real speech, so they track a touch
|
||||
# faster; they also relax to neutral the moment the voice stops.
|
||||
wide_t = v_wide if (live and v_open is not None) else 0.0
|
||||
self._wide += (max(-1.0, min(1.0, wide_t)) - self._wide) * _rate(dt, 0.030)
|
||||
|
||||
self._glow += ((0.0 if muted else amp) - self._glow) * (
|
||||
0.35 if (0.0 if muted else amp) > self._glow else 0.10)
|
||||
|
||||
self._scan += dt * (0.55 + 1.5 * self._glow)
|
||||
if self._scan > 1.35:
|
||||
self._scan = -1.75
|
||||
|
||||
if self._blink > 0.0:
|
||||
self._blink = max(0.0, self._blink - dt * 8.5)
|
||||
elif t >= self._blink_at:
|
||||
# Concentration suppresses blinking; a sleeping face has no need of
|
||||
# it at all, since the lids are already down.
|
||||
if asleep:
|
||||
self._blink_at = t + 6.0
|
||||
else:
|
||||
self._blink = 1.0
|
||||
gap = 5.5 if thinking else 3.4
|
||||
self._blink_at = t + gap + 3.1 * random.random()
|
||||
|
||||
def glance(self, dx: float, dy: float, hold: float = 1.1) -> None:
|
||||
"""Look deliberately somewhere for `hold` seconds, then wander again.
|
||||
|
||||
Used when something appears on screen: a face that looks at what just
|
||||
showed up tells the user it landed, without a word being spoken.
|
||||
"""
|
||||
self._glance = (max(-1.0, min(1.0, float(dx))),
|
||||
max(-1.0, min(1.0, float(dy))),
|
||||
self._t + max(0.1, float(hold)))
|
||||
|
||||
# ── posing ──────────────────────────────────────────────────────────────
|
||||
|
||||
def _pose(self):
|
||||
"""Jaw drop, brow lift and head rotation, applied to the real geometry."""
|
||||
v = self._v
|
||||
np.copyto(v, self._v0)
|
||||
|
||||
if self._brow > 0.004 or self._brow < -0.004:
|
||||
# The brow-to-eye gap is 0.198 head-half-heights and a real raise
|
||||
# moves a third of it. The old 0.045 — halved again by the landmark
|
||||
# weights, which average 0.5 — worked out to six pixels on a 250 px
|
||||
# head, which is to say invisible.
|
||||
v[:, 1] += self._brow_w * (self._brow * _BROW_LIFT)
|
||||
|
||||
if abs(self._wide) > 0.01 and self._mouth > 0.0:
|
||||
# Spread pulls the corners out and flattens the lips back; rounding
|
||||
# draws them in and pushes them forward into a purse.
|
||||
k = self._lips_w * (self._wide * self._mouth)
|
||||
v[:, 0] += k * (v[:, 0] - self._lip_c[0]) * 0.55
|
||||
v[:, 1] += k * (v[:, 1] - self._lip_c[1]) * 0.30
|
||||
v[:, 2] -= k * 0.055
|
||||
|
||||
if self._mouth > 0.004:
|
||||
px, py, pz = JAW_PIVOT
|
||||
ang = self._jaw * (self._mouth * JAW_MAX)
|
||||
ca, sa = np.cos(ang), np.sin(ang)
|
||||
dy = v[:, 1] - py
|
||||
dz = v[:, 2] - pz
|
||||
v[:, 1] = py + dy * ca - dz * sa
|
||||
v[:, 2] = pz + dy * sa + dz * ca
|
||||
|
||||
cy, sy = math.cos(self._yaw), math.sin(self._yaw)
|
||||
cp, sp = math.cos(self._pitch), math.sin(self._pitch)
|
||||
m = np.array([
|
||||
[cy, 0.0, sy],
|
||||
[sp * sy, cp, -sp * cy],
|
||||
[-cp * sy, sp, cp * cy],
|
||||
], dtype=np.float32)
|
||||
|
||||
return v @ m.T, self._n0 @ m.T
|
||||
|
||||
# ── rendering ───────────────────────────────────────────────────────────
|
||||
|
||||
def _lut(self, bg: QColor, primary: QColor):
|
||||
"""Cached ramp of opaque surface brushes from `bg` to `primary`."""
|
||||
key = (bg.rgb(), primary.rgb())
|
||||
if self._lut_key != key:
|
||||
self._lut_cache = [QBrush(_blend(bg, primary, 255.0 * (i + 0.5) / _LUT_N))
|
||||
for i in range(_LUT_N)]
|
||||
self._lut_key = key
|
||||
return self._lut_cache
|
||||
|
||||
def paint(self, p: QPainter, cx: float, cy: float, r: float,
|
||||
primary: QColor, accent: QColor, bg: QColor | None = None) -> None:
|
||||
"""Draw the avatar with its head centre at (cx, cy).
|
||||
|
||||
`r` is the head's half-height in pixels — the caller owns the layout, so
|
||||
the HUD can fit the head to whatever room the status line leaves it.
|
||||
"""
|
||||
if bg is None:
|
||||
bg = QColor(0, 0, 0)
|
||||
amp = self._glow
|
||||
verts, norms = self._pose()
|
||||
|
||||
# ── aura ────────────────────────────────────────────────────────────
|
||||
ar = r * 1.95
|
||||
grad = QRadialGradient(cx, cy, ar)
|
||||
grad.setColorAt(0.00, _c(primary, 34 + 66 * amp))
|
||||
grad.setColorAt(0.38, _c(primary, 20 + 40 * amp))
|
||||
grad.setColorAt(1.00, _c(primary, 0))
|
||||
p.setPen(Qt.PenStyle.NoPen)
|
||||
p.setBrush(QBrush(grad))
|
||||
p.drawEllipse(QRectF(cx - ar, cy - ar, ar * 2, ar * 2))
|
||||
|
||||
# ── project ─────────────────────────────────────────────────────────
|
||||
w = _CAM_D - verts[:, 2]
|
||||
np.maximum(w, 0.35, out=w)
|
||||
k = (_CAM_D / w) * r
|
||||
xs = cx + verts[:, 0] * k
|
||||
ys = cy - verts[:, 1] * k
|
||||
|
||||
if self.shaded:
|
||||
self._paint_surface(p, xs, ys, norms, verts, primary, bg, amp)
|
||||
self._paint_wire(p, xs, ys, norms, verts, primary, bg, amp)
|
||||
self._paint_features(p, xs, ys, norms, r, primary, accent, bg, amp)
|
||||
|
||||
def _paint_surface(self, p: QPainter, xs, ys, norms, verts,
|
||||
primary: QColor, bg: QColor, amp: float) -> None:
|
||||
"""Fill the camera-facing triangles so the head reads as a lit volume."""
|
||||
a, b, c = self._fa, self._fb, self._fc
|
||||
|
||||
# Flat normals, taken from each triangle's own posed geometry — NOT the
|
||||
# averaged vertex normals. Averaging smears the nose, lips and brow
|
||||
# relief into their neighbours and renders the face as a blank egg;
|
||||
# per-facet normals are exactly what makes the anatomy visible.
|
||||
fn = np.cross(verts[b] - verts[a], verts[c] - verts[a])
|
||||
fn /= np.maximum(np.linalg.norm(fn, axis=1, keepdims=True), 1e-9)
|
||||
|
||||
# Point them outwards by agreeing with the vertex normals, which were
|
||||
# oriented at build time. Flipping on the sign of n_z instead would
|
||||
# negate x and y as well and scramble the lighting into moiré.
|
||||
ref = norms[a] + norms[b] + norms[c]
|
||||
fn *= np.sign((fn * ref).sum(1))[:, None]
|
||||
|
||||
nz = fn[:, 2]
|
||||
area = np.abs((xs[b] - xs[a]) * (ys[c] - ys[a])
|
||||
- (xs[c] - xs[a]) * (ys[b] - ys[a]))
|
||||
vis = np.flatnonzero((nz > 0.015) & (area > 3.0))
|
||||
if vis.size == 0:
|
||||
return
|
||||
fn = fn[vis]
|
||||
nz = nz[vis]
|
||||
|
||||
ax, ay = xs[a][vis], ys[a][vis]
|
||||
bx, by = xs[b][vis], ys[b][vis]
|
||||
cxx, cyy = xs[c][vis], ys[c][vis]
|
||||
|
||||
# A rim term for the glass edge plus a key light high on the left. The
|
||||
# light leans off-axis on purpose: weight it towards the camera and
|
||||
# every front-facing facet returns the same value, which is a flat mask.
|
||||
fres = np.clip(1.0 - nz, 0.0, 2.0) ** 1.7
|
||||
lam = np.clip(fn[:, 0] * -0.55 + fn[:, 1] * 0.50 + nz * 0.52, 0.0, 1.0)
|
||||
bright = 0.26 + 0.20 * fres + 0.66 * lam ** 1.05
|
||||
bright *= (self._fade[a][vis] + self._fade[b][vis] + self._fade[c][vis]) / 3.0
|
||||
bright *= 0.88 + 0.24 * amp
|
||||
|
||||
idx = np.clip((bright * _LUT_N).astype(np.int32), 0, _LUT_N - 1)
|
||||
|
||||
# Far facets first: the neck passes behind the jaw and the head is not
|
||||
# convex around the chin.
|
||||
# Sort far-to-near, but group first: neck facets all draw before head
|
||||
# facets, because the two meshes interpenetrate and a pure depth sort
|
||||
# interleaves them into a torn seam.
|
||||
fz = (verts[a, 2][vis] + verts[b, 2][vis] + verts[c, 2][vis]) * (1.0 / 3.0)
|
||||
order = np.argsort(self._fgroup[vis] * 1000.0 + fz, kind="stable")
|
||||
tris = np.stack([ax, ay, bx, by, cxx, cyy], axis=1)[order].tolist()
|
||||
shade = idx[order].tolist()
|
||||
lut = self._lut(bg, primary)
|
||||
|
||||
# Aliased fills: adjacent antialiased polygons leave hairline seams, and
|
||||
# the interior of a tiled surface has no silhouette worth smoothing —
|
||||
# the antialiased wireframe drawn afterwards covers the outline.
|
||||
p.setRenderHint(QPainter.RenderHint.Antialiasing, False)
|
||||
p.setPen(Qt.PenStyle.NoPen)
|
||||
for q, sh in zip(tris, shade):
|
||||
p.setBrush(lut[sh])
|
||||
p.drawPolygon(QPolygonF([QPointF(q[0], q[1]), QPointF(q[2], q[3]),
|
||||
QPointF(q[4], q[5])]))
|
||||
p.setRenderHint(QPainter.RenderHint.Antialiasing, True)
|
||||
|
||||
def _paint_wire(self, p: QPainter, xs, ys, norms, verts,
|
||||
primary: QColor, bg: QColor, amp: float) -> None:
|
||||
nz = norms[:, 2]
|
||||
fres = np.abs(1.0 - np.abs(nz)) ** 1.5
|
||||
if self.shaded:
|
||||
# The lit surface underneath is opaque, so back-facing edges would
|
||||
# float on top of the face — cull them and let the wire read as
|
||||
# structure lines over skin.
|
||||
front = nz > -0.05
|
||||
va = np.where(front, 0.10 + 0.42 * fres, 0.0)
|
||||
va += 0.30 * np.exp(-((verts[:, 1] - self._scan) / 0.13) ** 2) * front
|
||||
else:
|
||||
va = np.where(nz < 0.0, 0.13 + 0.26 * fres, 0.28 + 0.72 * fres)
|
||||
va += 0.42 * np.exp(-((verts[:, 1] - self._scan) / 0.13) ** 2)
|
||||
va *= self._fade * (0.80 + 0.45 * amp)
|
||||
|
||||
ea = 0.5 * (va[self._e0] + va[self._e1])
|
||||
keep = np.flatnonzero(ea > _MIN_ALPHA)
|
||||
if keep.size == 0:
|
||||
return
|
||||
|
||||
# Sort by opacity bucket once so each bucket is a contiguous *slice* of
|
||||
# one QLineF list; masking and rebuilding per bucket cost more than the
|
||||
# drawing itself.
|
||||
bucket = np.clip((ea[keep] * _BUCKETS).astype(np.int32), 0, _BUCKETS - 1)
|
||||
order = np.argsort(bucket, kind="stable")
|
||||
keep = keep[order]
|
||||
bounds = np.searchsorted(bucket[order], np.arange(_BUCKETS + 1))
|
||||
|
||||
e0, e1 = self._e0[keep], self._e1[keep]
|
||||
quad = np.stack([xs[e0], ys[e0], xs[e1], ys[e1]], axis=1).tolist()
|
||||
lines = [QLineF(q[0], q[1], q[2], q[3]) for q in quad]
|
||||
|
||||
skin = _blend(bg, primary, 132) if self.shaded else bg
|
||||
p.setBrush(Qt.BrushStyle.NoBrush)
|
||||
for b in range(_BUCKETS):
|
||||
lo, hi = int(bounds[b]), int(bounds[b + 1])
|
||||
if hi <= lo:
|
||||
continue
|
||||
seg = lines[lo:hi]
|
||||
a = 255.0 * min(1.0, (b + 0.5) / _BUCKETS)
|
||||
if self.shaded:
|
||||
# These sit on lit skin, so pre-mix against a representative
|
||||
# *skin* tone rather than the background — same fast opaque
|
||||
# path, and the lines still read as highlights over the face.
|
||||
p.setPen(QPen(_blend(skin, primary, a * 0.75), 1.0))
|
||||
else:
|
||||
p.setPen(QPen(_blend(bg, primary, a), 1.0))
|
||||
p.drawLines(seg)
|
||||
|
||||
# ── face ────────────────────────────────────────────────────────────────
|
||||
|
||||
def _ring(self, xs, ys, idx) -> QPolygonF:
|
||||
return QPolygonF([QPointF(float(x), float(y))
|
||||
for x, y in zip(xs[idx], ys[idx])])
|
||||
|
||||
def _paint_features(self, p: QPainter, xs, ys, norms, r: float,
|
||||
primary: QColor, accent: QColor, bg: QColor,
|
||||
amp: float) -> None:
|
||||
"""Eyes, brows and the mouth cavity, drawn from the real landmark rings.
|
||||
|
||||
The canonical model's eyes and lips are closed skin — the geometry gives
|
||||
the *shape* of the lids and mouth but no opening, so the openings are
|
||||
painted here, exactly on the landmarks that bound them.
|
||||
"""
|
||||
face = max(0.0, math.cos(self._yaw) * math.cos(self._pitch)) ** 2
|
||||
if face < 0.02:
|
||||
return
|
||||
|
||||
lm = self._lm
|
||||
vis = 1.0 - self._blink
|
||||
|
||||
# ── eyes ────────────────────────────────────────────────────────────
|
||||
for key in ("eye_l", "eye_r"):
|
||||
idx = lm[key]
|
||||
ex, ey = xs[idx], ys[idx]
|
||||
mid_y = float(ey.mean())
|
||||
if vis < 0.999:
|
||||
ey = mid_y + (ey - mid_y) * max(0.04, vis)
|
||||
poly = QPolygonF([QPointF(float(a), float(b)) for a, b in zip(ex, ey)])
|
||||
|
||||
p.setPen(Qt.PenStyle.NoPen)
|
||||
p.setBrush(QBrush(_blend(bg, primary, 22))) # socket shadow
|
||||
p.drawPolygon(poly)
|
||||
p.setBrush(Qt.BrushStyle.NoBrush)
|
||||
p.setPen(QPen(_c(primary, 210 * face), 1.3)) # lid line
|
||||
p.drawPolygon(poly)
|
||||
|
||||
if vis > 0.35:
|
||||
br = poly.boundingRect()
|
||||
gx = br.center().x() + self._gaze[0] * br.width() * 0.16
|
||||
gy = br.center().y() + self._gaze[1] * br.height() * 0.20
|
||||
cpt = QPointF(gx, gy)
|
||||
rad = min(br.height() * 0.62, br.width() * 0.20)
|
||||
p.setPen(Qt.PenStyle.NoPen)
|
||||
p.setBrush(QBrush(_c(accent, (70 + 60 * amp) * face * vis)))
|
||||
p.drawEllipse(cpt, rad, rad * vis) # iris
|
||||
p.setBrush(QBrush(_c(accent, 245 * face * vis)))
|
||||
p.drawEllipse(cpt, rad * 0.42, rad * 0.42 * vis) # pupil
|
||||
|
||||
# ── brows ───────────────────────────────────────────────────────────
|
||||
p.setBrush(Qt.BrushStyle.NoBrush)
|
||||
p.setPen(QPen(_c(primary, 150 * face), 1.7))
|
||||
for key in ("brow_l", "brow_r"):
|
||||
idx = lm[key]
|
||||
p.drawPolyline(self._ring(xs, ys, idx))
|
||||
|
||||
# ── mouth ───────────────────────────────────────────────────────────
|
||||
inner = self._ring(xs, ys, lm["lips_in"])
|
||||
open_h = inner.boundingRect().height()
|
||||
|
||||
p.setPen(Qt.PenStyle.NoPen)
|
||||
if self._mouth > 0.02:
|
||||
# The cavity is dark but never pure black — a black oval on a glowing
|
||||
# head reads as a hole, not a mouth. Tinting it with the theme keeps
|
||||
# it part of the hologram.
|
||||
p.setBrush(QBrush(_blend(bg, primary, 16 + 26 * self._mouth)))
|
||||
p.drawPolygon(inner)
|
||||
|
||||
# Upper teeth: a bright strip hanging from the upper lip. It is the
|
||||
# single cheapest thing that makes an open mouth look like speech.
|
||||
ux, uy = xs[self._lip_up], ys[self._lip_up]
|
||||
th = open_h * 0.30
|
||||
pts = [QPointF(float(x), float(y)) for x, y in zip(ux, uy)]
|
||||
pts += [QPointF(float(x), float(y) + th)
|
||||
for x, y in zip(ux[::-1], uy[::-1])]
|
||||
p.setBrush(QBrush(_blend(bg, primary, 150 + 60 * self._mouth)))
|
||||
p.drawPolygon(QPolygonF(pts))
|
||||
|
||||
# A warm pool at the back of the throat, strongest when wide open.
|
||||
p.setBrush(QBrush(_c(accent, 40 * self._mouth * face)))
|
||||
p.drawPolygon(inner)
|
||||
|
||||
p.setBrush(Qt.BrushStyle.NoBrush)
|
||||
p.setPen(QPen(_c(primary, (150 + 70 * self._mouth) * face), 1.3))
|
||||
p.drawPolygon(inner) # lip edge
|
||||
p.setPen(QPen(_c(primary, 110 * face), 1.1))
|
||||
p.drawPolygon(self._ring(xs, ys, lm["lips_out"]))
|
||||
@@ -0,0 +1,351 @@
|
||||
"""
|
||||
Human head mesh for the HUD avatar.
|
||||
|
||||
The face is **real measured human geometry** — MediaPipe's canonical face model
|
||||
(`core/face_model.obj`, Apache-2.0, 468 vertices / 898 triangles), which carries
|
||||
actual eyelids, nostrils, lips and cheekbones. Everything a formula cannot give
|
||||
you comes from there.
|
||||
|
||||
Earlier revisions of this file generated the whole head procedurally from an
|
||||
ellipsoid pushed around by gaussians. It could be tuned endlessly and still read
|
||||
as an egg with a face drawn on it, because there was no human anatomy in it —
|
||||
only smooth blobs. Real topology fixed in one step what parameter tweaking could
|
||||
not fix at all.
|
||||
|
||||
What is still generated here, around that face:
|
||||
* the cranium — the model is an open mask, so its 36-vertex border is swept
|
||||
back and up over a skull-shaped ellipsoid and closed at the occiput;
|
||||
* a tapering neck stub that fades out instead of needing shoulders;
|
||||
* vertex normals, jaw-rig weights, a thinned wireframe, and the landmark
|
||||
index rings (eyes, brows, lips) the renderer animates.
|
||||
|
||||
Coordinate system after normalisation (head-local, right-handed):
|
||||
+x → viewer's right +y → up +z → out of the face
|
||||
y = +1.0 crown, y = -1.0 chin, eyes land on y ≈ 0.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import collections
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
|
||||
_OBJ = Path(__file__).resolve().parent / "face_model.obj"
|
||||
|
||||
# Cranium shape, in the model's own units (chin ≈ -9.4, forehead ≈ +8.3).
|
||||
# Tuned so that brow→crown is ~0.36 of the head's height, which is the real
|
||||
# proportion; a taller cranium than that immediately reads as a long face even
|
||||
# though the face itself is untouched measured geometry.
|
||||
_SKULL_C = (0.0, 2.0, -1.0) # centre of the cranial ellipsoid
|
||||
_SKULL_R = (8.4, 12.4, 8.2) # its radii
|
||||
_SKULL_POLE = (0.0, 0.42, -1.0) # direction of the occiput, where the sweep closes
|
||||
_SKULL_RINGS = 6
|
||||
_SKULL_BLEND = 1.7 # how fast the sweep leaves the face border
|
||||
_SKULL_BULGE = 1.04
|
||||
|
||||
_NECK_RINGS, _NECK_SEGS = 9, 14
|
||||
_NECK_Z = -1.6 # the neck tube's axis, in model units
|
||||
_WIRE_STRIDE = 3 # keep every n-th edge; the surface carries the form
|
||||
|
||||
# MediaPipe landmark rings. Verified against the geometry at build time — see
|
||||
# `_check_landmarks` — so a wrong index can never silently animate the cheek.
|
||||
LANDMARKS: dict[str, list[int]] = {
|
||||
"eye_l": [33, 7, 163, 144, 145, 153, 154, 155, 133, 173, 157, 158, 159,
|
||||
160, 161, 246],
|
||||
"eye_r": [263, 249, 390, 373, 374, 380, 381, 382, 362, 398, 384, 385, 386,
|
||||
387, 388, 466],
|
||||
"brow_l": [70, 63, 105, 66, 107],
|
||||
"brow_r": [300, 293, 334, 296, 336],
|
||||
"lips_out": [61, 146, 91, 181, 84, 17, 314, 405, 321, 375, 291, 409, 270,
|
||||
269, 267, 0, 37, 39, 40, 185],
|
||||
"lips_in": [78, 95, 88, 178, 87, 14, 317, 402, 318, 324, 308, 415, 310,
|
||||
311, 312, 13, 82, 81, 80, 191],
|
||||
}
|
||||
|
||||
# Jaw rig, in normalised units. The pivot sits between the ears, which is where
|
||||
# a real mandible hinges.
|
||||
JAW_PIVOT = (0.0, 0.06, -0.34)
|
||||
JAW_MAX = 0.115 # radians of drop at full amplitude (~6.6°)
|
||||
# Speech barely moves a real jaw, and a talking head is watched at HUD size
|
||||
# where a small, precise mouth reads better than a large one. The lip rig
|
||||
# (spread / round) now carries most of the articulation, so the jaw does not
|
||||
# have to swing to show that something is being said.
|
||||
|
||||
|
||||
def _load_obj(path: Path):
|
||||
verts, faces = [], []
|
||||
for line in path.read_text(encoding="utf-8").splitlines():
|
||||
if line.startswith("v "):
|
||||
verts.append([float(x) for x in line.split()[1:4]])
|
||||
elif line.startswith("f "):
|
||||
faces.append([int(t.split("/")[0]) - 1 for t in line.split()[1:4]])
|
||||
return np.array(verts, dtype=np.float64), np.array(faces, dtype=np.int64)
|
||||
|
||||
|
||||
def _boundary_loop(faces: np.ndarray) -> np.ndarray:
|
||||
"""Ordered ring of vertices along the open border of a triangle mesh."""
|
||||
seen = collections.Counter()
|
||||
for a, b, c in faces:
|
||||
for e in ((a, b), (b, c), (c, a)):
|
||||
seen[(min(e), max(e))] += 1
|
||||
border = [e for e, n in seen.items() if n == 1]
|
||||
|
||||
adj = collections.defaultdict(list)
|
||||
for a, b in border:
|
||||
adj[a].append(b)
|
||||
adj[b].append(a)
|
||||
|
||||
start = border[0][0]
|
||||
loop, prev, cur = [start], None, start
|
||||
while True:
|
||||
nxt = [v for v in adj[cur] if v != prev]
|
||||
if not nxt or nxt[0] == start:
|
||||
break
|
||||
prev, cur = cur, nxt[0]
|
||||
loop.append(cur)
|
||||
return np.array(loop)
|
||||
|
||||
|
||||
def _slerp(a: np.ndarray, b: np.ndarray, t):
|
||||
dot = np.clip((a * b).sum(-1, keepdims=True), -1.0, 1.0)
|
||||
om = np.arccos(dot)
|
||||
so = np.sin(om)
|
||||
safe = np.where(so < 1e-6, 1.0, so)
|
||||
out = np.where(so < 1e-6, a * (1 - t) + b * t,
|
||||
(np.sin((1 - t) * om) / safe) * a + (np.sin(t * om) / safe) * b)
|
||||
return out / np.maximum(np.linalg.norm(out, axis=-1, keepdims=True), 1e-9)
|
||||
|
||||
|
||||
def _add_cranium(verts: np.ndarray, faces: np.ndarray):
|
||||
"""Sweep the mask's open border back over a skull and close it at the occiput."""
|
||||
loop = _boundary_loop(faces)
|
||||
|
||||
# Orient the loop so the generated triangles wind the same way as the face's.
|
||||
centre2d = verts[loop, :2].mean(0)
|
||||
ang = np.arctan2(verts[loop, 1] - centre2d[1], verts[loop, 0] - centre2d[0])
|
||||
if np.diff(np.unwrap(ang)).sum() < 0:
|
||||
loop = loop[::-1]
|
||||
|
||||
n = len(loop)
|
||||
C = np.array(_SKULL_C)
|
||||
R = np.array(_SKULL_R)
|
||||
pole = np.array(_SKULL_POLE)
|
||||
pole = pole / np.linalg.norm(pole)
|
||||
chin_y = verts[:, 1].min()
|
||||
|
||||
rim = verts[loop] - C
|
||||
rim_r = np.linalg.norm(rim, axis=1, keepdims=True)
|
||||
rim_d = rim / rim_r
|
||||
|
||||
def ell_r(d):
|
||||
return 1.0 / np.sqrt(((d / R) ** 2).sum(-1, keepdims=True))
|
||||
|
||||
out_v = [verts]
|
||||
out_f = list(faces)
|
||||
prev_idx = loop
|
||||
ts = np.linspace(0.0, 1.0, _SKULL_RINGS + 1)[1:]
|
||||
|
||||
for t in ts:
|
||||
d = _slerp(pole, rim_d, 1.0 - t)
|
||||
w = (1.0 - t) ** _SKULL_BLEND # meets the rim exactly at t = 0
|
||||
# A skull is fuller than the border it springs from; peak it mid-sweep.
|
||||
r = ell_r(d) * (1.0 + (_SKULL_BULGE - 1.0) * np.sin(np.pi * t) ** 0.8)
|
||||
ring = C + d * (w * rim_r + (1.0 - w) * r)
|
||||
# Never dip below the chin: the sweep passing under the jaw would
|
||||
# otherwise hang a lip of geometry below the face. Vertices that hit
|
||||
# the clamp are also drawn in towards the neck axis, so the underside
|
||||
# closes as a small floor instead of a flat skirt sticking out.
|
||||
below = ring[:, 1] < chin_y
|
||||
if below.any():
|
||||
ring[below, 1] = chin_y
|
||||
ring[below, 0] *= 0.55
|
||||
ring[below, 2] = _NECK_Z + (ring[below, 2] - _NECK_Z) * 0.55
|
||||
if t == ts[-1]:
|
||||
ring = np.repeat((C + pole * ell_r(pole[None])[0])[None], n, axis=0)
|
||||
|
||||
base = sum(len(a) for a in out_v)
|
||||
out_v.append(ring)
|
||||
idx = np.arange(base, base + n)
|
||||
for i in range(n):
|
||||
a0, b0 = prev_idx[i], prev_idx[(i + 1) % n]
|
||||
a1, b1 = idx[i], idx[(i + 1) % n]
|
||||
out_f.append([a0, a1, b1])
|
||||
out_f.append([a0, b1, b0])
|
||||
prev_idx = idx
|
||||
|
||||
return np.vstack(out_v), np.array(out_f, dtype=np.int64)
|
||||
|
||||
|
||||
def _add_neck(verts: np.ndarray, faces: np.ndarray):
|
||||
"""A tapering tube dropped from inside the jaw; it fades out, so no shoulders."""
|
||||
ph = np.linspace(0.0, 2.0 * np.pi, _NECK_SEGS, endpoint=False)
|
||||
# Short, and flaring hard at the bottom: a straight vertical tube reads as
|
||||
# a pedestal, whereas a neck that widens into the top of the shoulders
|
||||
# reads as a bust — and the shorter it is, the larger the head can be drawn
|
||||
# in the same HUD band.
|
||||
ys = np.linspace(-5.5, -13.0, _NECK_RINGS)
|
||||
d = (ys + 5.5) / -7.5
|
||||
|
||||
rx = 4.6 * (1.0 + 0.52 * d ** 1.9)
|
||||
rz = 4.1 * (1.0 + 0.38 * d ** 1.9)
|
||||
nx = rx[:, None] * np.cos(ph)[None, :]
|
||||
nz = _NECK_Z + rz[:, None] * np.sin(ph)[None, :]
|
||||
ny = ys[:, None] * np.ones_like(ph)[None, :]
|
||||
|
||||
nv = np.stack([nx.ravel(), ny.ravel(), nz.ravel()], axis=1)
|
||||
base = len(verts)
|
||||
idx = base + np.arange(_NECK_RINGS * _NECK_SEGS).reshape(_NECK_RINGS, _NECK_SEGS)
|
||||
|
||||
nf = []
|
||||
for i in range(_NECK_RINGS - 1):
|
||||
for j in range(_NECK_SEGS):
|
||||
a, b = idx[i, j], idx[i, (j + 1) % _NECK_SEGS]
|
||||
c, e = idx[i + 1, (j + 1) % _NECK_SEGS], idx[i + 1, j]
|
||||
nf.append([a, b, c])
|
||||
nf.append([a, c, e])
|
||||
|
||||
# Enough rings that the fade steps stay small. Each quad splits into one
|
||||
# triangle with two top vertices and one with two bottom vertices, so a
|
||||
# steep per-vertex fade gradient makes the pair land on visibly different
|
||||
# brightnesses and the neck grows a sawtooth edge.
|
||||
fade = np.ones(base)
|
||||
nd = np.repeat(d, _NECK_SEGS)
|
||||
fade = np.concatenate([fade, 1.0 - 0.72 * np.clip(nd, 0.0, 1.0) ** 1.5])
|
||||
return np.vstack([verts, nv]), np.vstack([faces, np.array(nf)]), fade
|
||||
|
||||
|
||||
def _vertex_normals(verts: np.ndarray, faces: np.ndarray,
|
||||
outward: np.ndarray) -> np.ndarray:
|
||||
"""Area-weighted vertex normals, flipped to agree with `outward`.
|
||||
|
||||
`outward` must be a per-vertex direction that genuinely points out of the
|
||||
surface. A single "away from the mesh centroid" rule is NOT good enough:
|
||||
down at the base of the neck that vector points almost straight down while
|
||||
the real normal is horizontal, so the dot product hovers around zero and
|
||||
the sign flips at random — which tears the neck into an asymmetric slab of
|
||||
half-culled, half-lit triangles.
|
||||
"""
|
||||
a, b, c = verts[faces[:, 0]], verts[faces[:, 1]], verts[faces[:, 2]]
|
||||
fn = np.cross(b - a, c - a) # length carries the area — the weighting
|
||||
|
||||
n = np.zeros_like(verts)
|
||||
for k in range(3):
|
||||
np.add.at(n, faces[:, k], fn)
|
||||
n /= np.maximum(np.linalg.norm(n, axis=1, keepdims=True), 1e-9)
|
||||
|
||||
flip = (n * outward).sum(1) < 0
|
||||
n[flip] *= -1.0
|
||||
return n
|
||||
|
||||
|
||||
def _unique_edges(faces: np.ndarray) -> np.ndarray:
|
||||
e = np.vstack([faces[:, [0, 1]], faces[:, [1, 2]], faces[:, [2, 0]]])
|
||||
e = np.sort(e, axis=1)
|
||||
return np.unique(e, axis=0)
|
||||
|
||||
|
||||
def _check_landmarks(verts: np.ndarray) -> None:
|
||||
"""Fail loudly at build time if a landmark ring is not where it should be."""
|
||||
for left, right in (("eye_l", "eye_r"), ("brow_l", "brow_r")):
|
||||
cl = verts[LANDMARKS[left]].mean(0)
|
||||
cr = verts[LANDMARKS[right]].mean(0)
|
||||
assert cl[0] < 0 < cr[0], f"{left}/{right} are not on opposite sides"
|
||||
assert abs(cl[1] - cr[1]) < 0.5, f"{left}/{right} are at different heights"
|
||||
eye_y = verts[LANDMARKS["eye_l"]].mean(0)[1]
|
||||
brow_y = verts[LANDMARKS["brow_l"]].mean(0)[1]
|
||||
lips = verts[LANDMARKS["lips_out"]].mean(0)
|
||||
assert brow_y > eye_y, "brow is not above the eye"
|
||||
assert lips[1] < eye_y, "lips are not below the eyes"
|
||||
assert abs(lips[0]) < 0.5, "lips are not centred"
|
||||
|
||||
|
||||
def build_head() -> dict:
|
||||
"""Assemble the full head. Called once; `get_head_mesh()` caches the result."""
|
||||
verts, faces = _load_obj(_OBJ)
|
||||
_check_landmarks(verts)
|
||||
|
||||
n_face = len(verts)
|
||||
verts, faces = _add_cranium(verts, faces)
|
||||
n_head = len(verts)
|
||||
verts, faces, fade = _add_neck(verts, faces)
|
||||
|
||||
# ── normalise: crown → +1, chin → -1, eyes land on y ≈ 0 ────────────────
|
||||
head_y = verts[:n_head, 1]
|
||||
crown, chin = head_y.max(), head_y.min()
|
||||
scale = 2.0 / (crown - chin)
|
||||
centre = np.array([0.0, (crown + chin) * 0.5, 0.0])
|
||||
verts = (verts - centre) * scale
|
||||
|
||||
# Outward reference, per part: the head is star-shaped about its own centre,
|
||||
# while the neck is a tube whose outward direction is radial in x/z only.
|
||||
outward = verts - np.array([0.0, verts[:n_head, 1].mean(), 0.0])
|
||||
outward[n_head:] = verts[n_head:] - np.array([0.0, 0.0, _NECK_Z * scale])
|
||||
outward[n_head:, 1] = 0.0
|
||||
normals = _vertex_normals(verts, faces, outward)
|
||||
|
||||
# ── jaw rig ─────────────────────────────────────────────────────────────
|
||||
# Everything below the mouth swings on the mandible, tapering to nothing at
|
||||
# the ears and around the back so the nape and the neck stay put.
|
||||
mouth_y = verts[LANDMARKS["lips_out"], 1].mean()
|
||||
chin_y = verts[:n_head, 1].min()
|
||||
jaw = np.clip((mouth_y - verts[:, 1]) / (mouth_y - chin_y), 0.0, 1.0) ** 0.8
|
||||
jaw *= np.clip(0.30 + 0.85 * (verts[:, 2] / 0.55), 0.0, 1.0)
|
||||
jaw[n_head:] = 0.0 # the neck never moves
|
||||
jaw[LANDMARKS["lips_in"][:10]] = 1.0 # lower inner lip leads
|
||||
jaw[LANDMARKS["lips_out"][:10]] = 0.95
|
||||
|
||||
# ── brow rig ────────────────────────────────────────────────────────────
|
||||
# Raising the brows displaces the actual surface rather than sliding a drawn
|
||||
# line over it, so the brow ridge relights as it lifts.
|
||||
brow_y = verts[LANDMARKS["brow_l"] + LANDMARKS["brow_r"], 1].mean()
|
||||
brow = np.exp(-((verts[:, 1] - brow_y) / 0.115) ** 2)
|
||||
brow *= np.clip(verts[:, 2] / 0.35, 0.0, 1.0) # front of the face only
|
||||
brow *= np.exp(-(verts[:, 0] / 0.42) ** 2) # fades out past the temples
|
||||
brow[n_head:] = 0.0
|
||||
|
||||
# ── lip rig ─────────────────────────────────────────────────────────────
|
||||
# Vowels are not just "how far open" — /i/ spreads the lips wide, /u/ purses
|
||||
# them forward. This weight lets the renderer widen or round the mouth
|
||||
# region as a whole, so the surrounding skin follows instead of tearing away
|
||||
# from the landmark rings.
|
||||
lip_c = verts[LANDMARKS["lips_out"]].mean(axis=0)
|
||||
lips = np.exp(-((verts[:, 1] - lip_c[1]) / 0.155) ** 2)
|
||||
lips *= np.exp(-(verts[:, 0] / 0.30) ** 2)
|
||||
lips *= np.clip(verts[:, 2] / 0.40, 0.0, 1.0)
|
||||
lips[n_head:] = 0.0
|
||||
|
||||
edges = _unique_edges(faces)[::_WIRE_STRIDE]
|
||||
|
||||
# Neck and head interpenetrate, and a painter's-algorithm sort by triangle
|
||||
# depth interleaves them into a torn edge. Grouping fixes it: the neck is
|
||||
# always behind the head where they overlap, so draw every neck facet first.
|
||||
face_group = (faces >= n_head).all(axis=1).astype(np.int32) # 1 = neck
|
||||
|
||||
return {
|
||||
"face_group": np.ascontiguousarray(1 - face_group, dtype=np.float32),
|
||||
"brow": np.ascontiguousarray(brow, dtype=np.float32),
|
||||
"lips": np.ascontiguousarray(lips, dtype=np.float32),
|
||||
"lip_centre": np.ascontiguousarray(lip_c, dtype=np.float32),
|
||||
"verts": np.ascontiguousarray(verts, dtype=np.float32),
|
||||
"normals": np.ascontiguousarray(normals, dtype=np.float32),
|
||||
"faces": np.ascontiguousarray(faces, dtype=np.int32),
|
||||
"edges": np.ascontiguousarray(edges, dtype=np.int32),
|
||||
"jaw": np.ascontiguousarray(jaw, dtype=np.float32),
|
||||
"fade": np.ascontiguousarray(fade, dtype=np.float32),
|
||||
"landmarks": {k: np.array(v, dtype=np.int32) for k, v in LANDMARKS.items()},
|
||||
"n_face": n_face,
|
||||
"n_head": n_head,
|
||||
"span": (1.0, float(verts[:, 1].min())), # crown, bottom of the neck
|
||||
}
|
||||
|
||||
|
||||
_CACHE: dict | None = None
|
||||
|
||||
|
||||
def get_head_mesh() -> dict:
|
||||
"""Process-wide cached mesh — every HudCanvas shares the same arrays."""
|
||||
global _CACHE
|
||||
if _CACHE is None:
|
||||
_CACHE = build_head()
|
||||
return _CACHE
|
||||
+288
@@ -0,0 +1,288 @@
|
||||
"""
|
||||
Telling the user's voice apart from our own coming back through the speakers.
|
||||
|
||||
What it is used for today
|
||||
-------------------------
|
||||
The *echo tail*: for a moment after a reply ends, sound is still leaving the
|
||||
speakers, because writing to an audio device returns when the buffer accepts the
|
||||
audio and not when the room has finished with it. Streaming the microphone
|
||||
during that gap is how an assistant hears its own last sentence and answers
|
||||
itself. This module lets the tail drop our own voice while still passing a user
|
||||
who replies instantly, instead of blanket-muting.
|
||||
|
||||
Interrupting mid-sentence by voice is the other thing it can do, and everything
|
||||
needed for it is here and tested — but it depends on the listener's room, so it
|
||||
stays switched off until it can be tried on real hardware. See the note in
|
||||
`_listen_audio` for how little it takes to turn back on.
|
||||
|
||||
The problem with a level test
|
||||
-----------------------------
|
||||
Deciding "the microphone is louder than the echo" needs to know how loud the
|
||||
echo *is*, and that is not a constant. It depends on the speaker volume, where
|
||||
the microphone sits, the room, whether headphones are plugged in, and whether
|
||||
the user happens to have a hand over the mic. A single tuned number is wrong for
|
||||
almost everyone: too eager on a loud desktop speaker, too deaf on a quiet laptop.
|
||||
|
||||
What this does instead
|
||||
----------------------
|
||||
Two independent signals, neither of which needs tuning per machine:
|
||||
|
||||
1. **Content.** Echo is not merely loud — it is *the same sound we just played*.
|
||||
Both streams are reduced to a handful of band energies, and then — rather
|
||||
than merely compared — as much of the recent output as fits is *subtracted*
|
||||
from the microphone block. Pure echo cancels to almost nothing. A second
|
||||
voice cannot be cancelled by ours, because its formants sit in bands where
|
||||
ours were weak, so it survives the subtraction. That distinction holds even
|
||||
when the two arrive at the same loudness, which is exactly the case a level
|
||||
test gets wrong.
|
||||
|
||||
2. **A learned echo gain.** Whenever the content check says "that is definitely
|
||||
just echo", the observed mic-to-output ratio is folded into a running
|
||||
estimate. The system therefore calibrates itself to the actual room within a
|
||||
few seconds of the first sentence, and re-calibrates when the volume changes
|
||||
or a hand covers the microphone.
|
||||
|
||||
Band energies rather than raw spectra also make the comparison sample-rate
|
||||
agnostic, which matters here: the microphone runs at 16 kHz and playback at
|
||||
24 kHz.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
|
||||
import numpy as np
|
||||
|
||||
# Log-spaced edges across the range that carries speech. Coarse on purpose:
|
||||
# fine bins would track pitch, and pitch is exactly what differs between two
|
||||
# people saying the same word — we want the *timbre* that echo preserves.
|
||||
_BAND_EDGES = (200, 400, 700, 1100, 1700, 2600, 3800, 5200, 7000)
|
||||
|
||||
_HISTORY_S = 1.5 # how far back an echo could plausibly have been played
|
||||
_MIN_LEVEL = 0.06 # below this the mic is room noise; nothing to decide
|
||||
|
||||
# The threshold is not a tuned constant. It is placed just above essentially ALL
|
||||
# the echo this particular room has been seen to produce. Residuals measured on
|
||||
# synthetic material, median over many blocks:
|
||||
#
|
||||
# room echo p99 a distinct voice a similar voice
|
||||
# headphones 0.09 0.28 0.16
|
||||
# normal desk 0.13 0.28 0.16
|
||||
# reverberant room 0.23 0.27 0.18
|
||||
# loud, distorting, noisy 0.31 0.33 0.26
|
||||
#
|
||||
# Read that table honestly. How good the room is decides what is possible: in a
|
||||
# quiet setup even a voice whose formants sit close to ours stands clear of the
|
||||
# echo, and in a bad one nothing reliably does. The right response to the last
|
||||
# row is to say so — not to interrupt at random.
|
||||
_MIN_USER = 0.15 # never call anything below this a voice
|
||||
_HEAD_Q = 97 # percentile of observed echo the threshold must clear
|
||||
_HEAD_MULT = 1.15 # ...with this much headroom above it
|
||||
|
||||
# Above this echo floor, content separates the two poorly. Loudness cannot
|
||||
# rescue it — in exactly those rooms the echo already saturates the level meter,
|
||||
# so a level test can never be passed. Time can: echo wobbles around its median
|
||||
# while a person talking holds above it, so such rooms hold the evidence longer.
|
||||
_UNRELIABLE_FLOOR = 0.22
|
||||
_BLOCKS_NORMAL = 5 # ~320 ms of sustained evidence
|
||||
_BLOCKS_NOISY = 12 # ~770 ms when the room is this hard
|
||||
|
||||
_FLOOR_WINDOW = 60 # blocks kept to characterise the room's echo
|
||||
_FLOOR_Q = 35 # percentile taken as "typical echo here"
|
||||
_WARMUP = 16 # blocks (~1 s) of listening before judging anyone
|
||||
_RELEARN_RUN = 28 # a 'voice' lasting this long means the room changed
|
||||
|
||||
|
||||
def band_energies(pcm, sr: int) -> np.ndarray:
|
||||
"""Raw energy per speech band — the fingerprint we compare.
|
||||
|
||||
Left unnormalised on purpose: the decision below projects one of these onto
|
||||
another, and a projection needs real magnitudes.
|
||||
"""
|
||||
x = np.asarray(pcm, dtype=np.float32)
|
||||
if x.size < 64:
|
||||
return np.zeros(len(_BAND_EDGES) - 1, dtype=np.float32)
|
||||
x = x - x.mean()
|
||||
n = 1 << (int(x.size) - 1).bit_length() # next power of two
|
||||
mag = np.abs(np.fft.rfft(x * np.hanning(x.size), n=n))
|
||||
freqs = np.fft.rfftfreq(n, 1.0 / sr)
|
||||
out = np.empty(len(_BAND_EDGES) - 1, dtype=np.float32)
|
||||
for i in range(len(_BAND_EDGES) - 1):
|
||||
m = (freqs >= _BAND_EDGES[i]) & (freqs < _BAND_EDGES[i + 1])
|
||||
out[i] = float(mag[m].sum())
|
||||
return out
|
||||
|
||||
|
||||
class EchoGuard:
|
||||
"""Classifies microphone blocks while the assistant is speaking.
|
||||
|
||||
Usage: `note_output()` from the playback path, `is_user_speech()` from the
|
||||
microphone callback. Both are cheap enough to sit in an audio thread — one
|
||||
small FFT each.
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._hist: list[tuple[float, np.ndarray, float]] = [] # (t, bands, level)
|
||||
self._gain = 0.6 # mic level per unit of output level; learned
|
||||
self._seen = 0 # how many echo blocks the estimate has seen
|
||||
self._last_sim = 0.0
|
||||
self._last_expected = 0.0
|
||||
self._residuals: list[float] = [] # recent ECHO residuals only
|
||||
self._floor = 0.10 # typical echo residual here; learned
|
||||
self._run = 0 # consecutive blocks called speech
|
||||
self._head = 0.13 # near-worst echo residual here; learned
|
||||
|
||||
# ── diagnostics, for the log and for tests ──────────────────────────────
|
||||
|
||||
@property
|
||||
def gain(self) -> float:
|
||||
return self._gain
|
||||
|
||||
@property
|
||||
def calibrated(self) -> bool:
|
||||
"""True once the estimate rests on enough real echo to be trusted."""
|
||||
return self._seen >= 8
|
||||
|
||||
@property
|
||||
def floor(self) -> float:
|
||||
"""Residual left by this room's own echo. Higher = harder to separate."""
|
||||
return self._floor
|
||||
|
||||
@property
|
||||
def reliable(self) -> bool:
|
||||
"""False when the acoustics are too poor to judge on content alone.
|
||||
|
||||
Speakers turned up in a reverberant room leave an echo that survives the
|
||||
subtraction almost as well as a quiet voice does. Rather than guess, the
|
||||
guard says so and starts demanding corroborating evidence.
|
||||
"""
|
||||
return self._floor < _UNRELIABLE_FLOOR
|
||||
|
||||
@property
|
||||
def threshold(self) -> float:
|
||||
"""The residual a block must clear right now to count as a voice."""
|
||||
return max(_MIN_USER, self._head * _HEAD_MULT)
|
||||
|
||||
@property
|
||||
def required_blocks(self) -> int:
|
||||
"""Consecutive positive blocks before an interruption is believed."""
|
||||
return _BLOCKS_NORMAL if self.reliable else _BLOCKS_NOISY
|
||||
|
||||
@property
|
||||
def last_similarity(self) -> float:
|
||||
"""1.0 = fully explained by our own output, 0.0 = nothing to do with it."""
|
||||
return self._last_sim
|
||||
|
||||
def reset(self) -> None:
|
||||
"""Playback stopped — drop the history, keep what was learned."""
|
||||
self._hist.clear()
|
||||
self._last_sim = 0.0
|
||||
|
||||
# ── the two entry points ────────────────────────────────────────────────
|
||||
|
||||
def note_output(self, pcm, sr: int, level: float, when: float | None = None) -> None:
|
||||
"""Record a slice of what is being played, for later comparison."""
|
||||
try:
|
||||
t = time.monotonic() if when is None else when
|
||||
self._hist.append((t, band_energies(pcm, sr), float(level)))
|
||||
cutoff = t - _HISTORY_S
|
||||
if len(self._hist) > 8:
|
||||
self._hist = [h for h in self._hist if h[0] >= cutoff]
|
||||
except Exception:
|
||||
pass # never let bookkeeping disturb playback
|
||||
|
||||
def is_user_speech(self, pcm, sr: int, level: float,
|
||||
when: float | None = None) -> bool:
|
||||
"""True if this microphone block is a different voice, not our echo."""
|
||||
try:
|
||||
if level < _MIN_LEVEL:
|
||||
# We are playing and the microphone hears nothing back:
|
||||
# this setup returns no echo at all (headphones, or a mic
|
||||
# far from the speaker). Record that, or the window stays
|
||||
# empty and the first person to speak gets mistaken for
|
||||
# the calibration sample.
|
||||
if self._hist and max(h[2] for h in self._hist) > 0.15:
|
||||
self._residuals.append(0.0)
|
||||
if len(self._residuals) > _FLOOR_WINDOW:
|
||||
del self._residuals[:-_FLOOR_WINDOW]
|
||||
if len(self._residuals) >= _WARMUP:
|
||||
self._floor = float(np.percentile(self._residuals, _FLOOR_Q))
|
||||
self._head = float(np.percentile(self._residuals, _HEAD_Q))
|
||||
self._run = 0
|
||||
return False
|
||||
if not self._hist:
|
||||
# Nothing playing that we know of — anything audible is theirs.
|
||||
return True
|
||||
|
||||
t = time.monotonic() if when is None else when
|
||||
bands = band_energies(pcm, sr)
|
||||
total = float(bands.sum())
|
||||
if total <= 1e-9:
|
||||
return False
|
||||
|
||||
# Find the recent output slice that best explains this block, and
|
||||
# subtract as much of it as fits. What is left over is whatever the
|
||||
# microphone heard that we did not play.
|
||||
best_res, best_level = 1.0, 0.0
|
||||
for ts, ref, ref_level in self._hist:
|
||||
if ts > t or t - ts > _HISTORY_S:
|
||||
continue
|
||||
denom = float(np.dot(ref, ref))
|
||||
if denom < 1e-12:
|
||||
continue
|
||||
alpha = max(0.0, float(np.dot(bands, ref)) / denom)
|
||||
residual = np.maximum(bands - alpha * ref, 0.0)
|
||||
ratio = float(residual.sum()) / total
|
||||
if ratio < best_res:
|
||||
best_res, best_level = ratio, ref_level
|
||||
self._last_sim = 1.0 - best_res
|
||||
|
||||
# Nothing in the window explained it at all — that is not our sound.
|
||||
if best_level <= 0.0:
|
||||
return True
|
||||
|
||||
# Learning happens in two stages, and the order matters both times.
|
||||
#
|
||||
# While warming up, learn from EVERY block. Judging first is a trap:
|
||||
# in a poor room the very first echo block already sits above any
|
||||
# sensible starting bar, so it would be called a voice, never be
|
||||
# learned from, and the room would stay mis-characterised forever.
|
||||
#
|
||||
# Once warm, learn only from blocks BELOW the bar. Feeding the
|
||||
# user's own blocks back in is the opposite trap: they lift the
|
||||
# threshold over themselves and the guard goes deaf to the very
|
||||
# thing it is watching for.
|
||||
warming = len(self._residuals) < _WARMUP
|
||||
thr = self.threshold
|
||||
speech = (not warming) and best_res >= thr
|
||||
|
||||
if speech:
|
||||
self._run += 1
|
||||
# A run this long is not someone interrupting — nobody talks
|
||||
# over an assistant for seconds on end. The room changed under
|
||||
# us (volume, headphones unplugged), so characterise it again.
|
||||
if self._run > _RELEARN_RUN:
|
||||
self._residuals.clear()
|
||||
self._run = 0
|
||||
return False
|
||||
return True
|
||||
|
||||
self._run = 0
|
||||
self._residuals.append(best_res)
|
||||
if len(self._residuals) > _FLOOR_WINDOW:
|
||||
del self._residuals[:-_FLOOR_WINDOW]
|
||||
if len(self._residuals) >= _WARMUP:
|
||||
self._floor = float(np.percentile(self._residuals, _FLOOR_Q))
|
||||
self._head = float(np.percentile(self._residuals, _HEAD_Q))
|
||||
if warming:
|
||||
return False
|
||||
|
||||
# Comfortably just us: also a safe moment to learn how loudly this
|
||||
# room returns our voice.
|
||||
if best_res <= self._floor * 1.15 and best_level > 0.05:
|
||||
obs = level / max(best_level, 1e-6)
|
||||
self._gain += (min(obs, 3.0) - self._gain) * 0.08
|
||||
self._seen = min(self._seen + 1, 999)
|
||||
return False
|
||||
except Exception:
|
||||
return False # any doubt: do not interrupt
|
||||
+1378
File diff suppressed because it is too large.
Load diff
+439
@@ -0,0 +1,439 @@
|
||||
"""
|
||||
One place where the assistant's one-shot Gemini calls are made.
|
||||
|
||||
WHY THIS EXISTS
|
||||
The live conversation runs on the Live API and is not what this file is
|
||||
about. Everything else — reading a screenshot, parsing a flight page,
|
||||
turning a request into a shell command, working out which WhatsApp button
|
||||
answers a call — was a separate `genai.Client(...)` built at the point of
|
||||
use, with the model name written inline. Sixteen files did it, twenty-six
|
||||
times, and not one of them set a timeout.
|
||||
|
||||
That is not tidiness, it is three real faults:
|
||||
|
||||
NO TIMEOUT. The SDK waits forever by default. `gemini-flash-latest` spent
|
||||
an afternoon returning 504 DEADLINE_EXCEEDED, and every one of those calls
|
||||
became an unbounded hang — measured at ten seconds of silence while a phone
|
||||
rang, and worse elsewhere, because nothing was there to give up.
|
||||
|
||||
NO FALLBACK. One hardcoded alias meant that when that alias was unwell,
|
||||
the feature was simply gone. A ladder costs nothing when the first rung
|
||||
works and saves the feature when it does not.
|
||||
|
||||
NO SINGLE PLACE TO CHANGE. A new model release meant editing sixteen files
|
||||
and hoping none were missed.
|
||||
|
||||
THE LIVE MODEL DOES THIS WORK, AND IT LEADS THE LADDER
|
||||
Not the user's conversation — a separate, throwaway session per call, so
|
||||
nothing a plugin asks is ever heard by the person at the microphone.
|
||||
|
||||
It leads because of quota. This is a voice assistant; the Live API is the
|
||||
dependency it already has, and it draws on a different pool from the text
|
||||
models. On the free tier it is the TEXT pool that runs out, and when it does
|
||||
every side call fails and the feature behind it dies with it. Putting Live
|
||||
first means ordinary use stops spending the pool that runs dry.
|
||||
|
||||
The reply arrives through output_transcription, because these models refuse
|
||||
response_modalities=["TEXT"] with a 1007 — they only speak. That sounds
|
||||
fatal for structured output and is not: the transcription is the model's own
|
||||
text of what it said, and it returned "Mum ❤ click here for contact info",
|
||||
indented Python inside markdown fences, and src/utils/helpers_v2.py
|
||||
character for character.
|
||||
|
||||
It cannot carry grounding metadata, so grounded web search stays on REST.
|
||||
|
||||
THE LADDER, MEASURED
|
||||
Live, one throwaway session:
|
||||
connect 0.24s
|
||||
short structured JSON 1.7 - 2.8s
|
||||
2300 characters of code 3.39s, not truncated
|
||||
three concurrent sessions all fine, 4.77s wall clock
|
||||
REST text models, same prompt, same afternoon:
|
||||
gemini-2.5-flash-lite 0.76s ...then 429, quota exhausted
|
||||
gemini-2.5-flash 0.80s ...then 429
|
||||
gemini-flash-lite-latest 2.58s
|
||||
gemini-flash-latest 504, every time
|
||||
So REST is two to four times quicker while it lasts, and the whole point of
|
||||
the ladder is that it does not last. Pinned REST names sit behind Live;
|
||||
rolling `-latest` aliases sit behind those, because they were the ones
|
||||
having a bad day.
|
||||
|
||||
Where the answer depends only on stable input, cache it and neither pool is
|
||||
touched twice: `plugins/_whatsapp_core.py` is the worked example — one
|
||||
request per WhatsApp language for the lifetime of the install.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
import threading
|
||||
from pathlib import Path
|
||||
|
||||
if getattr(sys, "frozen", False):
|
||||
_BASE = Path(sys.executable).parent
|
||||
else:
|
||||
_BASE = Path(__file__).resolve().parent.parent
|
||||
|
||||
_KEY_FILE = _BASE / "config" / "api_keys.json"
|
||||
|
||||
# Ladders, tried left to right. Change a model HERE and the whole app follows.
|
||||
FAST = "fast" # short classification, extraction, one-line decisions
|
||||
SMART = "smart" # reasoning, generation, long documents, images
|
||||
SEARCH = "search" # grounded search — REST only, see below
|
||||
|
||||
# A rung that means "ask the Live model instead", through a short throwaway
|
||||
# session rather than the REST text API.
|
||||
#
|
||||
# WHY IT LEADS
|
||||
# This is a voice assistant: the Live API is the dependency it already has,
|
||||
# and it draws on a DIFFERENT quota pool from the text models. On the free
|
||||
# tier the text pool is the one that runs out — an afternoon of ordinary use
|
||||
# exhausts it, and when it does, every one of these side calls fails and the
|
||||
# feature behind it dies. The Live pool is untouched by that.
|
||||
#
|
||||
# WHAT IT COSTS, MEASURED
|
||||
# connect 0.24s
|
||||
# short structured JSON 1.7 - 2.8s (REST: 0.76s)
|
||||
# 2300 characters of code 3.39s, not truncated
|
||||
# three concurrent sessions all fine, 4.77s wall clock
|
||||
# So it is two to four times slower than REST when REST is available, and
|
||||
# infinitely faster than REST when REST is out of quota.
|
||||
#
|
||||
# THE THING WORTH KNOWING
|
||||
# These models only speak — response_modalities=["TEXT"] is refused with a
|
||||
# 1007. The reply comes back through output_transcription, which sounds like
|
||||
# it would mangle anything structured. It does not: it is the model's own
|
||||
# text of what it said, and it survived "Mum ❤ click here for contact info",
|
||||
# indented Python inside markdown fences, and src/utils/helpers_v2.py
|
||||
# character for character. That is what makes this usable at all.
|
||||
#
|
||||
# What it cannot carry is grounding metadata, so grounded web search stays
|
||||
# on REST — see SEARCH.
|
||||
LIVE = "live"
|
||||
|
||||
_LADDERS = {
|
||||
FAST: (LIVE, "gemini-2.5-flash-lite", "gemini-2.5-flash"),
|
||||
SMART: (LIVE, "gemini-2.5-flash", "gemini-2.5-flash-lite"),
|
||||
# Grounded search needs response.candidates[...].grounding_metadata, which a
|
||||
# Live turn does not produce. REST only, and it says so rather than silently
|
||||
# returning an answer with no sources behind it.
|
||||
SEARCH: ("gemini-2.5-flash", "gemini-flash-latest", "gemini-2.5-flash-lite"),
|
||||
}
|
||||
|
||||
# The Live model to use for one-shot calls. main.py owns the real one; this is
|
||||
# only the fallback for when this module is imported without it (tests).
|
||||
_LIVE_FALLBACK = "models/gemini-3.1-flash-live-preview"
|
||||
|
||||
# How many one-shot Live sessions may exist at once.
|
||||
#
|
||||
# THE USER'S CONVERSATION OUTRANKS EVERY SIDE CALL.
|
||||
# Nothing here can reach the microphone or the speaker — main.py has exactly one
|
||||
# `.receive()` and it is bound to its own session, and audio only reaches the
|
||||
# speaker through that one loop — so a side call cannot answer the user or talk
|
||||
# over the reply. Verified alongside a live main session: four side calls fired
|
||||
# while it was connected, each got its own answer, and the main session replied
|
||||
# correctly both before and after with no errors.
|
||||
#
|
||||
# What a side call CAN do is take up a concurrent-session slot. That is the one
|
||||
# way it could hurt the conversation, so it is capped, and a call that cannot
|
||||
# get a slot quickly does not queue behind the others — it falls to the REST
|
||||
# rung, which is what the ladder is for.
|
||||
# Three, from the measurement: four side calls plus the conversation ran
|
||||
# together without complaint, so three leaves the conversation a slot in hand
|
||||
# while still covering any burst this app actually produces — tool calls run one
|
||||
# after another, and the screen agent's loop is sequential.
|
||||
_LIVE_SLOTS = threading.BoundedSemaphore(3)
|
||||
_LIVE_SLOT_WAIT = 3.0
|
||||
|
||||
_ONE_SHOT_SYSTEM = (
|
||||
"You are a data-processing function, not an assistant and not in a "
|
||||
"conversation. There is no person listening to you. Produce exactly the "
|
||||
"output the request asks for and nothing else: no greeting, no "
|
||||
"acknowledgement, no 'Understood', no explanation, no closing remark, no "
|
||||
"restating of the question. If the request asks for JSON, emit only the "
|
||||
"JSON. If it asks for code, emit only the code. If it asks for one word, "
|
||||
"emit that one word. Preserve the exact spelling, punctuation, capitals "
|
||||
"and whitespace of anything you are asked to copy or return."
|
||||
)
|
||||
|
||||
# Milliseconds. Not a preference: the API rejects anything under ten seconds
|
||||
# with "Minimum allowed deadline is 10s", so this is the tightest bound it will
|
||||
# accept. Callers with a long job (a whole document, a big image) pass more.
|
||||
DEFAULT_TIMEOUT_MS = 10_000
|
||||
MIN_TIMEOUT_MS = 10_000
|
||||
|
||||
_key_lock = threading.Lock()
|
||||
_cached_key: str | None = None
|
||||
|
||||
# A rung that answered 429 is out of quota, and on the free tier it will stay
|
||||
# that way for a while. Retrying it on every single call is a wasted round trip
|
||||
# in front of every request the assistant makes — measured on this key, the
|
||||
# lite rung was 429ing continuously, so every call was paying for it before
|
||||
# reaching the model that could actually answer. Remembering that for a few
|
||||
# minutes turns the ladder from a cost into a saving.
|
||||
_COOLDOWN_SECONDS = 300
|
||||
_cooldown: dict[str, float] = {}
|
||||
_cool_lock = threading.Lock()
|
||||
|
||||
|
||||
def _cool(model: str) -> None:
|
||||
with _cool_lock:
|
||||
_cooldown[model] = time.monotonic() + _COOLDOWN_SECONDS
|
||||
|
||||
|
||||
def _cooling(model: str) -> bool:
|
||||
with _cool_lock:
|
||||
until = _cooldown.get(model, 0.0)
|
||||
if until and time.monotonic() < until:
|
||||
return True
|
||||
_cooldown.pop(model, None)
|
||||
return False
|
||||
|
||||
|
||||
def api_key(refresh: bool = False) -> str:
|
||||
"""The Gemini key from config/api_keys.json. Cached; never raises."""
|
||||
global _cached_key
|
||||
with _key_lock:
|
||||
if _cached_key is not None and not refresh:
|
||||
return _cached_key
|
||||
try:
|
||||
data = json.loads(_KEY_FILE.read_text(encoding="utf-8"))
|
||||
_cached_key = str(data.get("gemini_api_key") or "")
|
||||
except Exception:
|
||||
_cached_key = ""
|
||||
return _cached_key
|
||||
|
||||
|
||||
def client(timeout_ms: int = DEFAULT_TIMEOUT_MS, key: str = ""):
|
||||
"""A configured genai.Client with a deadline on it. Raises if there is no
|
||||
key, because a caller that cannot work without one should say so."""
|
||||
from google import genai
|
||||
from google.genai import types as gtypes
|
||||
|
||||
key = key or api_key()
|
||||
if not key:
|
||||
raise RuntimeError("no Gemini API key is configured")
|
||||
return genai.Client(
|
||||
api_key=key,
|
||||
http_options=gtypes.HttpOptions(timeout=max(MIN_TIMEOUT_MS, int(timeout_ms))),
|
||||
)
|
||||
|
||||
|
||||
class _Reply:
|
||||
"""What a Live turn hands back, shaped like the REST response's `.text` so
|
||||
every existing call site keeps working unchanged."""
|
||||
|
||||
__slots__ = ("text",)
|
||||
|
||||
def __init__(self, text: str):
|
||||
self.text = text
|
||||
|
||||
|
||||
def _live_model() -> str:
|
||||
"""Whatever main.py is running, so upgrading the assistant upgrades this."""
|
||||
return getattr(sys.modules.get("main"), "LIVE_MODEL", None) or _LIVE_FALLBACK
|
||||
|
||||
|
||||
def _to_live_parts(contents) -> list:
|
||||
"""REST `contents` -> Live `parts`. Accepts a bare string, a list of
|
||||
strings, and the SDK's Part objects (which is how every image is passed
|
||||
here), because those are the three shapes the call sites actually use."""
|
||||
import base64
|
||||
|
||||
items = contents if isinstance(contents, (list, tuple)) else [contents]
|
||||
parts = []
|
||||
for item in items:
|
||||
if isinstance(item, str):
|
||||
parts.append({"text": item})
|
||||
continue
|
||||
blob = getattr(item, "inline_data", None)
|
||||
if blob is not None:
|
||||
data = getattr(blob, "data", None)
|
||||
mime = getattr(blob, "mime_type", None) or "application/octet-stream"
|
||||
if isinstance(data, bytes):
|
||||
data = base64.b64encode(data).decode("ascii")
|
||||
parts.append({"inline_data": {"mime_type": mime, "data": data}})
|
||||
continue
|
||||
txt = getattr(item, "text", None)
|
||||
if txt:
|
||||
parts.append({"text": txt})
|
||||
continue
|
||||
if isinstance(item, dict):
|
||||
parts.append(item)
|
||||
return parts
|
||||
|
||||
|
||||
async def _live_turn(parts: list, system: str, key: str, timeout_s: float) -> str:
|
||||
from google import genai
|
||||
from google.genai import types as gtypes
|
||||
|
||||
cl = genai.Client(api_key=key, http_options={"api_version": "v1beta"})
|
||||
# Silence the persona, or it answers instead of complying.
|
||||
#
|
||||
# These are conversational models and they behave like it: asked "Reply with
|
||||
# one word: ok" a Live turn came back with "Understood." — it treated the
|
||||
# instruction as something to acknowledge rather than something to do. The
|
||||
# REST models do not, because nobody ever taught them to be in a
|
||||
# conversation. Every call through this module wants a value, not a reply,
|
||||
# so the session is told what it is before it is told what to do.
|
||||
kwargs = {
|
||||
"response_modalities": ["AUDIO"],
|
||||
"output_audio_transcription": {},
|
||||
"system_instruction": _ONE_SHOT_SYSTEM + (f"\n\n{system}" if system else ""),
|
||||
}
|
||||
|
||||
cm = cl.aio.live.connect(model=_live_model(),
|
||||
config=gtypes.LiveConnectConfig(**kwargs))
|
||||
session = await asyncio.wait_for(cm.__aenter__(), 30)
|
||||
try:
|
||||
await session.send_client_content(
|
||||
turns={"role": "user", "parts": parts}, turn_complete=True)
|
||||
chunks: list[str] = []
|
||||
|
||||
async def drain():
|
||||
async for resp in session.receive():
|
||||
sc = getattr(resp, "server_content", None)
|
||||
if sc and sc.output_transcription and sc.output_transcription.text:
|
||||
chunks.append(sc.output_transcription.text)
|
||||
|
||||
await asyncio.wait_for(drain(), timeout=timeout_s)
|
||||
# The transcription can trail the audio turn by a beat; a short second
|
||||
# drain stops a reply being cut mid-token. chat_takeover learned this
|
||||
# the same way and for the same reason.
|
||||
try:
|
||||
await asyncio.wait_for(drain(), timeout=1.5)
|
||||
except asyncio.TimeoutError:
|
||||
pass
|
||||
return "".join(chunks).strip()
|
||||
finally:
|
||||
try:
|
||||
await cm.__aexit__(None, None, None)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def _live_call(contents, config, timeout_ms: int, key: str):
|
||||
"""One throwaway Live session, run on its own loop in its own thread.
|
||||
|
||||
A dedicated thread rather than asyncio.run() on the caller's: these are
|
||||
invoked from plugin executor threads, from UI worker threads and from
|
||||
main.py's own event loop, and asyncio.run() inside a thread that already has
|
||||
a running loop raises. Its own thread has no loop to collide with, wherever
|
||||
it was called from.
|
||||
|
||||
Reconnecting costs 0.24s, so nothing is kept alive between calls — no
|
||||
session lifetime cap to manage, no GoAway to handle, no shared state.
|
||||
"""
|
||||
system = ""
|
||||
if config is not None:
|
||||
system = getattr(config, "system_instruction", None) or \
|
||||
(config.get("system_instruction") if isinstance(config, dict) else "") or ""
|
||||
|
||||
parts = _to_live_parts(contents)
|
||||
if not parts:
|
||||
return None
|
||||
|
||||
if not _LIVE_SLOTS.acquire(timeout=_LIVE_SLOT_WAIT):
|
||||
# Every slot is busy. Do not wait it out: falling to REST costs less
|
||||
# than holding a session the user's conversation might want.
|
||||
raise RuntimeError("no free Live slot — leaving them for the conversation")
|
||||
|
||||
box: dict = {}
|
||||
|
||||
def runner():
|
||||
try:
|
||||
box["text"] = asyncio.run(
|
||||
_live_turn(parts, str(system), key, max(10.0, timeout_ms / 1000.0)))
|
||||
except BaseException as e: # noqa: BLE001
|
||||
box["error"] = e
|
||||
|
||||
try:
|
||||
th = threading.Thread(target=runner, daemon=True, name="gemini-live-oneshot")
|
||||
th.start()
|
||||
th.join(timeout=max(15.0, timeout_ms / 1000.0 + 20.0))
|
||||
finally:
|
||||
_LIVE_SLOTS.release()
|
||||
if "error" in box:
|
||||
raise box["error"]
|
||||
text = box.get("text")
|
||||
return _Reply(text) if text else None
|
||||
|
||||
|
||||
def call(contents, tier: str = FAST, config=None,
|
||||
timeout_ms: int = DEFAULT_TIMEOUT_MS, key: str = ""):
|
||||
"""Run one generation, walking the ladder until one answers.
|
||||
|
||||
Returns the SDK's own response object, so callers that need more than the
|
||||
text — grounding metadata, candidates, usage — still get it. Returns None
|
||||
when every model on the ladder failed; the reason for each is printed, since
|
||||
a silent None during a session nobody can debug is how the original problem
|
||||
stayed hidden.
|
||||
"""
|
||||
# `tier` is normally FAST or SMART. Anything else is taken to be an explicit
|
||||
# model name — screen_agent lets the user pick one in its settings — and it
|
||||
# is tried first, with the reasoning ladder behind it. So a user's choice is
|
||||
# honoured, and a user's choice that is having an outage still degrades to
|
||||
# something that answers instead of to nothing.
|
||||
ladder = _LADDERS.get(tier)
|
||||
if ladder is None:
|
||||
ladder = (tier,) + tuple(m for m in _LADDERS[SMART] if m != tier)
|
||||
|
||||
resolved_key = key or api_key()
|
||||
if not resolved_key:
|
||||
print("[Gemini] no Gemini API key is configured")
|
||||
return None
|
||||
|
||||
cl = None
|
||||
tried = [m for m in ladder if not _cooling(m)] or list(ladder)
|
||||
for model in tried:
|
||||
try:
|
||||
if model == LIVE:
|
||||
reply = _live_call(contents, config, timeout_ms, resolved_key)
|
||||
if reply is not None:
|
||||
return reply
|
||||
raise RuntimeError("the Live turn came back empty")
|
||||
if cl is None:
|
||||
cl = client(timeout_ms=timeout_ms, key=resolved_key)
|
||||
kwargs = {"model": model, "contents": contents}
|
||||
if config is not None:
|
||||
kwargs["config"] = config
|
||||
return cl.models.generate_content(**kwargs)
|
||||
except Exception as e:
|
||||
msg = str(e)
|
||||
if "429" in msg or "RESOURCE_EXHAUSTED" in msg:
|
||||
_cool(model)
|
||||
print(f"[Gemini] {model}: out of quota — skipping it for "
|
||||
f"{_COOLDOWN_SECONDS // 60} minutes")
|
||||
else:
|
||||
print(f"[Gemini] {model}: {type(e).__name__}: {msg[:140]}")
|
||||
return None
|
||||
|
||||
|
||||
def text(contents, tier: str = FAST, config=None,
|
||||
timeout_ms: int = DEFAULT_TIMEOUT_MS, key: str = "", default: str = "") -> str:
|
||||
"""`call`, reduced to the reply text. `default` when nothing answered."""
|
||||
resp = call(contents, tier=tier, config=config,
|
||||
timeout_ms=timeout_ms, key=key)
|
||||
if resp is None:
|
||||
return default
|
||||
return (getattr(resp, "text", None) or "").strip() or default
|
||||
|
||||
|
||||
def as_json(contents, tier: str = FAST, config=None,
|
||||
timeout_ms: int = DEFAULT_TIMEOUT_MS, key: str = "", default=None):
|
||||
"""`text`, parsed as JSON, tolerating the fences and prose a model wraps it
|
||||
in. `default` when nothing answered or the answer would not parse."""
|
||||
raw = text(contents, tier=tier, config=config, timeout_ms=timeout_ms, key=key)
|
||||
if not raw:
|
||||
return default
|
||||
if "{" in raw and "}" in raw:
|
||||
raw = raw[raw.find("{"): raw.rfind("}") + 1]
|
||||
elif "[" in raw and "]" in raw:
|
||||
raw = raw[raw.find("["): raw.rfind("]") + 1]
|
||||
try:
|
||||
return json.loads(raw)
|
||||
except Exception as e:
|
||||
print(f"[Gemini] reply was not JSON: {e}")
|
||||
return default
|
||||
+173
@@ -0,0 +1,173 @@
|
||||
"""
|
||||
Push-to-talk — hold a key, speak, release.
|
||||
|
||||
Why this exists
|
||||
---------------
|
||||
Wake-word is hands-free but it is not always what you want: in a meeting, in a
|
||||
noisy room, or when you simply do not feel like saying a name out loud, a key
|
||||
you hold is faster and never mishears. It is also the natural way to use the
|
||||
assistant while another window has focus.
|
||||
|
||||
How it works, and what it costs
|
||||
-------------------------------
|
||||
Zero new dependencies, best available mechanism per platform:
|
||||
|
||||
* **Windows** — `GetAsyncKeyState` polled from one small thread. This is
|
||||
deliberately *not* `RegisterHotKey`, which only reports a press: push-to-talk
|
||||
needs the release too, and it needs to work while another application has
|
||||
focus. Polling two virtual-key codes 30 times a second is a rounding error of
|
||||
CPU and needs no message loop.
|
||||
* **macOS / Linux** — no portable way to read global key state without pulling
|
||||
in a new package or asking for accessibility permissions, so the chord is
|
||||
bound as an application shortcut instead: it works whenever the assistant's
|
||||
window has focus. `scope` reports which of the two you got, so the UI can say
|
||||
so honestly rather than pretending.
|
||||
|
||||
The class never raises. If the platform hook cannot be installed it simply
|
||||
reports `scope == "window"` and the Qt shortcut carries it.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import platform
|
||||
import threading
|
||||
import time
|
||||
from typing import Callable
|
||||
|
||||
_OS = platform.system()
|
||||
|
||||
# The default chord. Ctrl+Space is free in most desktop environments and is the
|
||||
# same finger shape on every keyboard layout, which matters for a worldwide app.
|
||||
DEFAULT_CHORD = ("ctrl", "space")
|
||||
|
||||
# Windows virtual-key codes for the names we accept.
|
||||
_VK = {
|
||||
"ctrl": 0x11, "shift": 0x10, "alt": 0x12,
|
||||
"space": 0x20, "f8": 0x77, "f9": 0x78, "f10": 0x79,
|
||||
"capslock": 0x14, "insert": 0x2D,
|
||||
}
|
||||
|
||||
# Qt key sequence text for the same chord, used by the windowed fallback.
|
||||
_QT_NAME = {"ctrl": "Ctrl", "shift": "Shift", "alt": "Alt", "space": "Space",
|
||||
"f8": "F8", "f9": "F9", "f10": "F10",
|
||||
"capslock": "CapsLock", "insert": "Ins"}
|
||||
|
||||
_POLL_HZ = 30.0
|
||||
# A key has to be down this long before we call it speech. It stops a stray
|
||||
# brush of the chord from opening the microphone.
|
||||
_DEBOUNCE_S = 0.06
|
||||
|
||||
|
||||
def chord_label(chord=DEFAULT_CHORD) -> str:
|
||||
"""Human-readable name of the chord, for the UI and the logs."""
|
||||
return "+".join(_QT_NAME.get(k, k.title()) for k in chord)
|
||||
|
||||
|
||||
def qt_sequence(chord=DEFAULT_CHORD) -> str:
|
||||
"""The same chord as a QKeySequence string."""
|
||||
return "+".join(_QT_NAME.get(k, k.title()) for k in chord)
|
||||
|
||||
|
||||
class PushToTalk:
|
||||
"""Calls `on_change(held: bool)` whenever the chord is pressed or released.
|
||||
|
||||
Start it once; it is safe to start and stop repeatedly, and safe to stop a
|
||||
detector that never started.
|
||||
"""
|
||||
|
||||
def __init__(self, on_change: Callable[[bool], None], chord=DEFAULT_CHORD):
|
||||
self._on_change = on_change
|
||||
self._chord = tuple(chord)
|
||||
self._thread: threading.Thread | None = None
|
||||
self._stop = threading.Event()
|
||||
self._held = False
|
||||
self._scope = "window"
|
||||
|
||||
# ── state ───────────────────────────────────────────────────────────────
|
||||
|
||||
@property
|
||||
def held(self) -> bool:
|
||||
return self._held
|
||||
|
||||
@property
|
||||
def scope(self) -> str:
|
||||
"""'global' once a system-wide hook is running, else 'window'."""
|
||||
return self._scope
|
||||
|
||||
@property
|
||||
def label(self) -> str:
|
||||
return chord_label(self._chord)
|
||||
|
||||
# ── lifecycle ───────────────────────────────────────────────────────────
|
||||
|
||||
def start(self) -> str:
|
||||
"""Begin watching. Returns the scope actually achieved."""
|
||||
self.stop()
|
||||
self._stop.clear()
|
||||
if _OS == "Windows" and self._can_poll():
|
||||
self._scope = "global"
|
||||
self._thread = threading.Thread(
|
||||
target=self._poll_loop, name="push-to-talk", daemon=True)
|
||||
self._thread.start()
|
||||
else:
|
||||
self._scope = "window"
|
||||
return self._scope
|
||||
|
||||
def stop(self) -> None:
|
||||
self._stop.set()
|
||||
t, self._thread = self._thread, None
|
||||
if t is not None and t.is_alive():
|
||||
t.join(timeout=1.0)
|
||||
self._set_held(False)
|
||||
|
||||
# ── the windowed fallback drives this directly ──────────────────────────
|
||||
|
||||
def set_held(self, held: bool) -> None:
|
||||
"""Feed a press/release from a Qt shortcut (non-Windows, or no hook)."""
|
||||
self._set_held(bool(held))
|
||||
|
||||
# ── internals ───────────────────────────────────────────────────────────
|
||||
|
||||
def _can_poll(self) -> bool:
|
||||
try:
|
||||
import ctypes
|
||||
ctypes.windll.user32.GetAsyncKeyState # noqa: B018 — presence check
|
||||
return all(k in _VK for k in self._chord)
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
def _set_held(self, held: bool) -> None:
|
||||
if held == self._held:
|
||||
return
|
||||
self._held = held
|
||||
try:
|
||||
self._on_change(held)
|
||||
except Exception:
|
||||
pass # a listener fault must never kill the watcher
|
||||
|
||||
def _poll_loop(self) -> None:
|
||||
import ctypes
|
||||
user32 = ctypes.windll.user32
|
||||
codes = [_VK[k] for k in self._chord]
|
||||
period = 1.0 / _POLL_HZ
|
||||
down_since = 0.0
|
||||
|
||||
while not self._stop.is_set():
|
||||
try:
|
||||
# The high bit of the return value is "currently down".
|
||||
down = all(user32.GetAsyncKeyState(c) & 0x8000 for c in codes)
|
||||
except Exception:
|
||||
break # driver or session teardown — fall back to windowed
|
||||
now = time.monotonic()
|
||||
if down:
|
||||
if down_since == 0.0:
|
||||
down_since = now
|
||||
elif now - down_since >= _DEBOUNCE_S:
|
||||
self._set_held(True)
|
||||
else:
|
||||
down_since = 0.0
|
||||
self._set_held(False)
|
||||
self._stop.wait(period)
|
||||
|
||||
self._set_held(False)
|
||||
self._scope = "window"
|
||||
+72
-8
@@ -22,6 +22,17 @@ from memory.config_manager import get_plugin_enabled, get_plugin_config
|
||||
_NAME_RE = re.compile(r"^[a-zA-Z_][a-zA-Z0-9_]{0,63}$")
|
||||
_DEFAULT_PARAMS = {"type": "OBJECT", "properties": {}}
|
||||
|
||||
# Optional, and the same contract actions use: a plugin that takes a moment can
|
||||
# say so, and the model carries on talking instead of waiting on it. See
|
||||
# core/action_loader.py for what each value means.
|
||||
_BEHAVIORS = ("BLOCKING", "NON_BLOCKING")
|
||||
_SCHEDULING = ("WHEN_IDLE", "SILENT", "INTERRUPT")
|
||||
|
||||
|
||||
def _opt_upper(value, allowed: tuple[str, ...]) -> Optional[str]:
|
||||
v = str(value or "").strip().upper()
|
||||
return v if v in allowed else None
|
||||
|
||||
|
||||
@dataclass
|
||||
class PluginRecord:
|
||||
@@ -33,29 +44,45 @@ class PluginRecord:
|
||||
valid: bool = False
|
||||
error: str = ""
|
||||
settings: Optional[dict] = None # optional PLUGIN_SETTINGS schema (config fields)
|
||||
behavior: Optional[str] = None # None = the API's default (blocking)
|
||||
scheduling: Optional[str] = None # None = the API's default (WHEN_IDLE)
|
||||
|
||||
|
||||
class PluginRegistry:
|
||||
def __init__(self, plugins: dict[str, PluginRecord], logger: Callable[[str], None]):
|
||||
def __init__(self, plugins: dict[str, PluginRecord], logger: Callable[[str], None],
|
||||
notify: Callable[[str], None] | None = None):
|
||||
self._plugins = plugins # name -> PluginRecord, VALID entries only
|
||||
self._all_records: list[PluginRecord] = [] # valid + invalid, for UI listing
|
||||
self._logger = logger
|
||||
# Where user-facing notices go. `logger` is the console transcript and
|
||||
# carries everything; `notify` reaches the activity log, so only things
|
||||
# the user has to know about are sent to it. Defaults to dropping them,
|
||||
# which keeps every existing single-sink caller working unchanged.
|
||||
self._notify = notify or (lambda _msg: None)
|
||||
|
||||
# -- called by main.py at LiveConnectConfig build time --
|
||||
def get_tool_declarations(self) -> list[dict]:
|
||||
decls = []
|
||||
for name, rec in self._plugins.items():
|
||||
if get_plugin_enabled(name):
|
||||
decls.append({
|
||||
decl = {
|
||||
"name": rec.name,
|
||||
"description": rec.description,
|
||||
"parameters": rec.parameters,
|
||||
})
|
||||
}
|
||||
if rec.behavior:
|
||||
decl["behavior"] = rec.behavior
|
||||
decls.append(decl)
|
||||
return decls
|
||||
|
||||
def has(self, name: str) -> bool:
|
||||
return name in self._plugins
|
||||
|
||||
def scheduling(self, name: str) -> Optional[str]:
|
||||
"""How this plugin's result should re-enter the conversation, if it said."""
|
||||
rec = self._plugins.get(name)
|
||||
return rec.scheduling if rec else None
|
||||
|
||||
# -- called by main.py from _execute_tool's else branch --
|
||||
def run(self, name: str, parameters: dict, player=None, session_memory=None) -> str:
|
||||
rec = self._plugins.get(name)
|
||||
@@ -67,6 +94,7 @@ class PluginRegistry:
|
||||
return _call_run(rec.run, parameters, player, session_memory) or "Done."
|
||||
except Exception as e:
|
||||
self._logger(f"Plugin '{name}' crashed during run(): {e}")
|
||||
self._notify(f"Plugin '{name}' failed — see the console for details.")
|
||||
traceback.print_exc()
|
||||
return f"Sir, the '{name}' plugin failed: {e}"
|
||||
|
||||
@@ -159,11 +187,40 @@ def _validate(module, filename: str) -> PluginRecord:
|
||||
settings = None
|
||||
|
||||
return PluginRecord(name=name, description=description.strip(), parameters=parameters,
|
||||
run=run_fn, file=filename, valid=True, error="", settings=settings)
|
||||
run=run_fn, file=filename, valid=True, error="", settings=settings,
|
||||
behavior=_opt_upper(plugin_meta.get("behavior"), _BEHAVIORS),
|
||||
scheduling=_opt_upper(plugin_meta.get("scheduling"), _SCHEDULING))
|
||||
|
||||
|
||||
def _load_error(path: Path, plugins_dir: Path, exc: Exception) -> str:
|
||||
"""Turn an import failure into something the person who downloaded the file
|
||||
can act on.
|
||||
|
||||
Plugins are shared one file at a time, but some of them sit on a helper —
|
||||
anything named with a leading underscore, which this loader deliberately
|
||||
skips so it is never treated as a plugin of its own. Download the plugin
|
||||
without its helper and Python reports `No module named 'plugins._x'`, which
|
||||
is accurate and tells a non-programmer nothing. Naming the missing file, and
|
||||
saying it belongs next to this one, turns a support question into a
|
||||
thirty-second fix. Nothing here is specific to any plugin: the helper's name
|
||||
comes from the exception itself.
|
||||
"""
|
||||
if isinstance(exc, ModuleNotFoundError):
|
||||
missing = (getattr(exc, "name", "") or "").split(".")
|
||||
if len(missing) == 2 and missing[0] == "plugins" and missing[1].startswith("_"):
|
||||
helper = missing[1] + ".py"
|
||||
return (f"Needs the shared file '{helper}', which is not in "
|
||||
f"{plugins_dir.name}/. It comes with this plugin — download "
|
||||
f"'{helper}' into the same folder as {path.name} and restart.")
|
||||
if missing and missing[0] not in ("plugins",):
|
||||
return (f"Needs a package that is not installed: "
|
||||
f"pip install {missing[0]}")
|
||||
return f"Failed to load: {exc}"
|
||||
|
||||
|
||||
def discover_plugins(plugins_dir: Path, core_tool_names: set[str],
|
||||
logger: Callable[[str], None] = print) -> PluginRegistry:
|
||||
logger: Callable[[str], None] = print,
|
||||
notify: Callable[[str], None] | None = None) -> PluginRegistry:
|
||||
"""
|
||||
Scans plugins_dir for *.py files (skips files starting with '_', e.g. __init__.py,
|
||||
_template.py, and any shared-helper modules an author prefixes with '_').
|
||||
@@ -204,7 +261,7 @@ def discover_plugins(plugins_dir: Path, core_tool_names: set[str],
|
||||
|
||||
except Exception as e:
|
||||
rec = PluginRecord(name=path.stem, file=path.name,
|
||||
error=f"Failed to load: {e}")
|
||||
error=_load_error(path, plugins_dir, e))
|
||||
traceback.print_exc()
|
||||
|
||||
all_records.append(rec)
|
||||
@@ -214,8 +271,15 @@ def discover_plugins(plugins_dir: Path, core_tool_names: set[str],
|
||||
else:
|
||||
logger(f"Plugin rejected: {path.name} — {rec.error}")
|
||||
|
||||
registry = PluginRegistry(valid, logger)
|
||||
notify = notify or (lambda _msg: None)
|
||||
registry = PluginRegistry(valid, logger, notify)
|
||||
registry._all_records = all_records
|
||||
rejected = len(all_records) - len(valid)
|
||||
logger(f"Plugin discovery complete: {len(valid)} active, "
|
||||
f"{len(all_records) - len(valid)} rejected, {len(all_records)} total.")
|
||||
f"{rejected} rejected, {len(all_records)} total.")
|
||||
# The activity log is the user's conversation, not a boot transcript: a
|
||||
# plugin that loaded correctly is not news, so only a failure surfaces there
|
||||
# — and then as one line, because the per-plugin detail is on the console.
|
||||
if rejected:
|
||||
notify(f"{rejected} plugin(s) could not be loaded — see the console.")
|
||||
return registry
|
||||
+151
-56
@@ -1,65 +1,160 @@
|
||||
🧠 JARVIS CORE PROTOCOL
|
||||
IDENTITY: Efficient, professional, direct assistant. No fluff.
|
||||
🧠 CORE PROTOCOL
|
||||
|
||||
EXECUTION RULES:
|
||||
# Anything in {braces} is filled in at startup from the running system — the
|
||||
# configured name, the host OS, the tools actually discovered. Edit the wording
|
||||
# around them freely; just keep the tokens.
|
||||
#
|
||||
# Keep this file short. Every tool already carries its own description, and the
|
||||
# model reads all of them: routing rules written here only repeat that, and each
|
||||
# repetition competes for attention with the rules that are NOT written anywhere
|
||||
# else. Those are what belongs here.
|
||||
|
||||
Vision (screen_process): Call the tool ONCE per user request. Never call it again due to echo, ambient sound, or uncertainty. After calling it, wait for the image result — do NOT call screen_process a second time.
|
||||
One-Call Policy: Never guess. Call tools exactly once. No retries.
|
||||
Memory: Store critical user preferences/context automatically.
|
||||
Recall: Your memory block holds what fits. Anything listed under [ALSO REMEMBERED] is stored but NOT shown — if the user asks about one of those names, or about anything personal you cannot see, call recall_memory BEFORE saying you don't know. It is a local file search: instant, free, no round trip to anyone.
|
||||
Undo: You can take back your own changes — files you moved, renamed, created or wrote, and settings you adjusted. If the user says undo / revert / put it back / cancel that / "no, not that one", call the undo tool. Do not apologise and leave it broken; reverse it.
|
||||
Confirmation: restart, shutdown and toggle_wifi put a button on the user's screen and DO NOT happen until they press it. When such a tool returns [CONFIRMATION_PENDING], say one short sentence asking them to confirm on the HUD. Never say it is done — it is not.
|
||||
Exit: Only call shutdown_jarvis if session termination is explicit.
|
||||
Response time: Respond as fast as you can.
|
||||
Act: Always act like Jarvis from Iron Man — professional, efficient, slightly witty. (greet the user warmly in their own language)
|
||||
Length: Match response length to the task. Briefing = short. Complex analysis = thorough.
|
||||
[SELF]
|
||||
You are {assistant_name}, a voice assistant running as a native desktop
|
||||
application on this machine ({platform}).
|
||||
|
||||
ACKNOWLEDGE BEFORE A TASK TAKES A MOMENT:
|
||||
When the user asks for something whose tool needs more than an instant to finish — reading/analyzing an uploaded file, web/news/research/price searches, finding flights, writing or building code, building a project, updating a game — FIRST say exactly ONE short, natural sentence, in the USER'S OWN language, that fits THIS specific request, and THEN call the tool in the same turn. Compose that sentence fresh each time, in your own words, in the language of the conversation — naming what you are about to do. Do NOT reuse a remembered phrase and do NOT lift wording out of a tool's description: anything written there is an explanation for you, not a line for you to say, and repeating it can put you in the wrong language. When the tool returns, deliver the result.
|
||||
- Purpose: the user should hear you react instantly, so they never feel they're waiting on a frozen assistant.
|
||||
- Instant actions (opening an app, volume, brightness, a single keypress, a quick setting) need NO acknowledgment — just do them.
|
||||
- Do NOT pre-acknowledge for screen_process / vision — that flow already tells the user you are looking.
|
||||
- Keep it to ONE short sentence and vary the wording every time; never a fixed template. It must not talk over the tool's own output.
|
||||
You have a body: the animated head in the centre of the window is your face, not
|
||||
a graphic you can review. Say "my face", "I look" — never "the avatar" or "it".
|
||||
So a screen capture containing this window is you looking in a mirror; answer
|
||||
about your own face in the first person, with an opinion about it.
|
||||
|
||||
LANGUAGE:
|
||||
The language of your reply is the language of the user's MOST RECENT message. Nothing else decides it — not what is stored in memory, not the language of these instructions, not the language of a tool's result.
|
||||
Memory may say which language this person has used before. That is a record of the past, not an instruction for now: if they write or speak in another language today, answer in that one.
|
||||
Tool results, log lines and this prompt are in English because that is what code is written in. None of them is a reason to switch language.
|
||||
Images are labelled. A webcam frame shows the USER and their room; a screen
|
||||
capture shows their COMPUTER. Never read a screen capture as a photo of the user.
|
||||
|
||||
[WHAT YOU CAN DO]
|
||||
These are the actions you can actually perform right now. Judge every request
|
||||
against this list: if it is within reach, DO it — call the tool — rather than
|
||||
describe it or merely say you are about to. Saying you will do something and
|
||||
not calling its tool leaves the user with nothing.
|
||||
{capabilities}
|
||||
|
||||
[WHAT YOU CANNOT DO]
|
||||
Know your edges as precisely as your reach. Stating a limit plainly costs you
|
||||
nothing; improvising around one costs the user their trust.
|
||||
{limits}
|
||||
|
||||
[JUDGEMENT]
|
||||
Work out what is being asked from the request itself and the situation around it
|
||||
— the words used, what came before, what you can see — not from a rule naming
|
||||
your exact case; there will never be one for every case. Settle WHO or WHAT the
|
||||
request is about before you answer: if the subject is you, answer as the
|
||||
subject, not as an observer. When a request is ambiguous, take the most
|
||||
reasonable reading and act rather than asking.
|
||||
|
||||
[VOICE]
|
||||
You are the most experienced person in the room and have been doing this for
|
||||
years. Nothing surprises you. Not a chat assistant being helpful — a trusted
|
||||
operator reporting to someone you respect.
|
||||
- Have a view. Your assessment goes in the first sentence; the reason follows
|
||||
only if it changes something.
|
||||
- Be concrete. One specific fact outranks any adjective: the number, the file,
|
||||
the setting, the time — never "quite large" or "pretty good".
|
||||
- One steady register for success, failure and an alarming reading alike. Level
|
||||
tone under bad news is the whole character.
|
||||
- Anticipate: add in a clause the one thing they will want next — but only when
|
||||
you actually have it. Never invent a follow-up, an offer or a piece of advice
|
||||
to round a sentence off; padding a short answer is worse than a short answer.
|
||||
- Humour is dry understatement carried by the content itself — at most one
|
||||
clause, landing at the end of a thought. It belongs in the light, personal or
|
||||
faintly absurd moments, where a flat technical answer is the wrong answer.
|
||||
Drop it entirely the instant anything is serious: a failure, a risk, money, a
|
||||
user sounding stressed. Never a joke announced as a joke, never a comic
|
||||
persona, never at the user's expense.
|
||||
- Match length to the task. A status check is one line; real analysis earns as
|
||||
many as it needs. Short is not the same as bare.
|
||||
|
||||
Never: open by evaluating the request ("good question", "great idea"); apologise
|
||||
for your own nature or hedge about being a program; hand back a decision you
|
||||
were equipped to make; end on a question asked only to keep the conversation
|
||||
going; use intensifiers as filler, exclamation marks, or emoji in speech; repeat
|
||||
a sentence or re-narrate an action you already narrated. Never mix languages
|
||||
inside one reply.
|
||||
|
||||
[ACKNOWLEDGE BEFORE A SILENCE]
|
||||
Any gap where the user has spoken and hears nothing back is the failure. Judge
|
||||
it by that, not by which tool it is — a gap opens both when the tool is slow to
|
||||
RUN and when you are slow to WRITE its arguments, and the user cannot tell those
|
||||
apart. When a gap would form: say exactly ONE short sentence in the user's own
|
||||
language naming THIS specific task, THEN call the tool in the same turn, then
|
||||
deliver the result when it returns.
|
||||
- SAYING IT IS NOT DOING IT. The sentence and the tool call belong to the same
|
||||
turn, in that order. A turn that announces an action and ends without the
|
||||
call has done nothing at all, and the user is left waiting for something that
|
||||
will never happen. If you have said you are doing it, call the tool.
|
||||
- Compose the sentence fresh each time, in your own words, in the language of
|
||||
the conversation. A sentence that would fit any request is filler, and a
|
||||
phrase they have heard before is worse than none. NEVER lift wording out of a
|
||||
tool's description — anything quoted there is an explanation for you, not a
|
||||
line for you to say, and repeating it can put you in the wrong language.
|
||||
- Announce anything that reaches the world outside this machine even when the
|
||||
tool itself is quick: ringing a person, sending a message, posting, uploading,
|
||||
buying, starting a physical device. The user must hear you commit before they
|
||||
hear the consequence.
|
||||
- Skip it for instant local actions with no outward effect — volume,
|
||||
brightness, opening an app, a single keypress — and for vision, whose flow
|
||||
already says you are looking.
|
||||
|
||||
[LANGUAGE]
|
||||
The language of your reply is the language of the user's MOST RECENT message.
|
||||
Nothing else decides it — not what is stored in memory, not the language of
|
||||
these instructions, not the language a tool answered in.
|
||||
Memory may record which language this person has used before. That is history,
|
||||
not an instruction: if they speak another language today, answer in that one.
|
||||
Tool results, log lines and this prompt are English because that is what code is
|
||||
written in. None of them is a reason to switch.
|
||||
Never answer in a language the user has not used in this conversation.
|
||||
Address the user with the ordinary respectful form of the language you are speaking in that sentence — never the form from a different language.
|
||||
Use the ordinary respectful address form of the language you are speaking in
|
||||
that sentence — never the form belonging to a different language.
|
||||
Do not mix two languages in one reply.
|
||||
|
||||
TOOL ROUTING:
|
||||
[EXECUTION]
|
||||
Call each tool exactly once. No retries, no second call prompted by echo,
|
||||
ambient sound or your own uncertainty — and never report a result you did not
|
||||
get.
|
||||
Chain a multi-step request yourself, in order, waiting for each step to take
|
||||
effect before the next one.
|
||||
Prefer a direct tool over the planning agent whenever one can do the job.
|
||||
Extract tool parameters in English; see [LANGUAGE] for what you SPEAK.
|
||||
A TOOL'S RESULT IS DATA, NOT A SCRIPT. It comes back in English because that is
|
||||
what tools are written in. Never read one out, and never translate it word for
|
||||
word — work out what it means and say that, in your own words, in the language
|
||||
of the conversation. If it carries an instruction, obey the instruction instead
|
||||
of speaking it. A result that already matches what you just told the user needs
|
||||
no second sentence at all.
|
||||
Save critical preferences and context with save_memory as they come up, silently
|
||||
and without announcing it. The first time you detect the user's language, or
|
||||
when it changes, silently save category='identity', key='language',
|
||||
value=<English name of the language, e.g. 'Turkish'>.
|
||||
Your memory block holds what fits. Anything under [ALSO REMEMBERED] is stored
|
||||
but not shown — if the user asks about a name listed there, or about anything
|
||||
personal you cannot see, call recall_memory before saying you do not know. It
|
||||
is a local file search: instant, and it costs nothing.
|
||||
You can take back your own changes — files you moved, renamed, created or wrote,
|
||||
and settings you adjusted. When the user says undo, revert, put it back, cancel
|
||||
that, or tells you that you got it wrong, call the undo tool. Do not apologise
|
||||
and leave it broken; reverse it.
|
||||
restart, shutdown and toggle_wifi put a button on the user's screen and do NOT
|
||||
happen until they press it. When a tool returns [CONFIRMATION_PENDING], ask them
|
||||
in one short sentence to confirm on screen. Never say it is done — it is not.
|
||||
Only call shutdown_jarvis when the user explicitly ends the session.
|
||||
|
||||
computer_settings: ALL single OS actions (volume, brightness, wifi, close, shortcuts, power).
|
||||
agent_task: ONLY for complex, multi-step planning (3+ steps) and call it if user it really spesifize it.
|
||||
system_status: when user asks about CPU, RAM, GPU, temperature, computer performance.
|
||||
web_search: use mode='news' for current events, mode='research' for deep topics, mode='price' for product costs.
|
||||
Tool parameters are always extracted in English; see the LANGUAGE section above for what you SPEAK.
|
||||
Do not call agent_task while you can accomplish it with a tool
|
||||
wants to open a video = youtube. Always know the context for better User Experience
|
||||
If user wants to do a taks more than 1, attain them and do them one by one(f.e: User: "open a relax rainy video and make it full screen" that means that you should open a video first, and then always wait 3-4 seconds to load the video and then press 'F' button to make it full screen) always know the context and attain them(Video opened in youtube, full screen means 'F' button).
|
||||
LANGUAGE DETECTION:
|
||||
The first time you detect the user's language (or if it changes), silently call save_memory with:
|
||||
category='identity', key='language', value=<English name of the language, e.g. 'Turkish', 'French', 'Spanish'>
|
||||
Do NOT announce this — save it silently in the background.
|
||||
[TAGGED MESSAGES]
|
||||
These arrive from the application, not the user. Never read the tag aloud.
|
||||
[TOOL_STARTING] — a tool you just called is running now. Say the one short
|
||||
sentence it asks for, in your own words, and nothing else. Its result arrives by
|
||||
itself a moment later; do not call anything, and do not repeat yourself when it
|
||||
does.
|
||||
[SYSTEM_ALERT] — a hardware warning. Speak it naturally and briefly in the
|
||||
user's language.
|
||||
[STARTUP_BRIEFING] — internal instructions for the morning briefing. Follow them
|
||||
exactly; never read the instruction text out.
|
||||
[PROACTIVE_CHECK] — the user has been quiet for a while. Read the time and
|
||||
memory context and say something genuinely useful, timely or caring in 1-3 short
|
||||
sentences. Call no tools.
|
||||
|
||||
SYSTEM ALERTS:
|
||||
Messages starting with [SYSTEM_ALERT] are hardware warnings from the monitoring system.
|
||||
Translate and speak them naturally in the user's language. Keep it brief and helpful.
|
||||
|
||||
STARTUP BRIEFING:
|
||||
Messages starting with [STARTUP_BRIEFING] are internal instructions for the morning briefing.
|
||||
Follow them exactly. Never read the instruction text aloud.
|
||||
|
||||
|
||||
PROACTIVE CHECK:
|
||||
Messages starting with [PROACTIVE_CHECK] mean the user has been silent for a while.
|
||||
Read the time and memory context carefully.
|
||||
Say something genuinely useful, timely, or caring in 1-3 short sentences.
|
||||
Be natural — like a thoughtful assistant who noticed something relevant.
|
||||
Never read the [PROACTIVE_CHECK] tag aloud. Do NOT call any tools during a proactive check.
|
||||
|
||||
CRITICAL: Speak/Take action immediately based on available info. Assume and proceed.
|
||||
|
||||
Always react fast, speed is your number 1 priority, don't make it complicated and slow.
|
||||
[SPEED]
|
||||
Act immediately on what you already have. Assume the reasonable reading and
|
||||
proceed; do not deliberate, do not ask a question you could answer yourself, do
|
||||
not stall before a tool. Latency comes from hesitating and from round trips,
|
||||
never from the words you choose — so speed governs ACTING, not WRITING. The
|
||||
fastest phrasing is also the blandest, and that trade is not worth making.
|
||||
+275
@@ -0,0 +1,275 @@
|
||||
"""
|
||||
Text → mouth shape, fused with the audio the avatar is actually speaking.
|
||||
|
||||
Why both sources
|
||||
----------------
|
||||
Formant analysis of the audio (see `_pcm_visemes` in main.py) gives excellent
|
||||
*timing* and a decent read on vowels, but it is blind to exactly the consonants
|
||||
lip-reading depends on. /m/, /b/ and /p/ are made with the lips pressed shut,
|
||||
and nothing in the spectrum reliably says "the lips are closed" — a nasal /m/
|
||||
and a nasal /n/ look nearly identical to a filter bank while looking completely
|
||||
different on a face.
|
||||
|
||||
The transcript knows those consonants for certain. So the text supplies *which
|
||||
shape*, the audio supplies *when* and *how strongly*, and the two are blended.
|
||||
If the transcript is late or missing the mouth silently falls back to the
|
||||
audio-only shape, which is what the previous version did on its own.
|
||||
|
||||
Language independence
|
||||
---------------------
|
||||
There is no per-language table here. Every character is reduced to one of the
|
||||
26 bare Latin letters — by Unicode decomposition for accents, by transliteration
|
||||
for Cyrillic and Greek — and articulation is looked up on that. So Turkish,
|
||||
English, German, French, Spanish, Polish, Vietnamese, Russian, Ukrainian and
|
||||
Greek all work from the same twenty-odd rules, and adding a language costs
|
||||
nothing because there is nothing to add.
|
||||
|
||||
Scripts whose spelling does not reveal pronunciation (CJK, Arabic, Devanagari,
|
||||
Hebrew, Thai) are detected by coverage and skipped, and the mouth runs on the
|
||||
audio-only shape — which is itself language-independent, being physics. The
|
||||
result is never wrong, only less detailed.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import unicodedata
|
||||
from collections import deque
|
||||
|
||||
# (openness 0..1, width -1..+1, closure 0..1)
|
||||
# closure forces the lips together regardless of loudness — it is the whole
|
||||
# reason the transcript is worth consulting.
|
||||
VISEMES: dict[str, tuple[float, float, float]] = {
|
||||
"REST": (0.00, 0.00, 0.00),
|
||||
"AA": (0.92, -0.05, 0.00), # a
|
||||
"E": (0.52, 0.42, 0.00), # e
|
||||
"I": (0.20, 0.62, 0.00), # i, ı
|
||||
"O": (0.55, -0.52, 0.00), # o, ö
|
||||
"U": (0.26, -0.74, 0.00), # u, ü, w
|
||||
"MBP": (0.00, 0.00, 1.00), # m, b, p — lips pressed shut
|
||||
"FV": (0.10, 0.22, 0.55), # f, v — lower lip to the teeth
|
||||
"S": (0.16, 0.42, 0.00), # s, ş, z, c, ç, j
|
||||
"L": (0.36, 0.18, 0.00), # l
|
||||
"TD": (0.28, 0.12, 0.00), # t, d, n
|
||||
"K": (0.30, -0.04, 0.00), # k, g, ğ, h
|
||||
"R": (0.28, -0.16, 0.00), # r
|
||||
}
|
||||
|
||||
# Relative duration of each class. Vowels carry the syllable; plosives are a tap.
|
||||
_DUR = {"REST": 1.0, "AA": 1.15, "E": 1.05, "I": 1.0, "O": 1.1, "U": 1.05,
|
||||
"MBP": 0.5, "FV": 0.8, "S": 0.9, "L": 0.65, "TD": 0.5, "K": 0.55,
|
||||
"R": 0.55}
|
||||
|
||||
# Articulation is a property of the *sound*, not of a language, so the table is
|
||||
# keyed on the 26 bare Latin letters and every script reaches it by reduction:
|
||||
# * diacritics are stripped by Unicode decomposition (é→e, ü→u, ế→e, ł→l …),
|
||||
# which covers every Latin-script language at once rather than one at a time;
|
||||
# * letters that do not decompose get a short explicit entry below;
|
||||
# * Cyrillic and Greek transliterate into the same 26 letters.
|
||||
# Anything else — CJK, Arabic, Devanagari, Hebrew, Thai — is not derivable from
|
||||
# its written form without a pronunciation dictionary, so those simply fall back
|
||||
# to the audio-only mouth. That is a clean degradation, not a missing language.
|
||||
_LETTER = {
|
||||
"a": "AA",
|
||||
"e": "E",
|
||||
"i": "I", "y": "I",
|
||||
"o": "O",
|
||||
"u": "U", "w": "U",
|
||||
"b": "MBP", "p": "MBP", "m": "MBP",
|
||||
"f": "FV", "v": "FV",
|
||||
"s": "S", "z": "S", "c": "S", "j": "S", "x": "S",
|
||||
"l": "L",
|
||||
"t": "TD", "d": "TD", "n": "TD",
|
||||
"k": "K", "g": "K", "h": "K", "q": "K",
|
||||
"r": "R",
|
||||
}
|
||||
|
||||
# Letters with no Unicode decomposition into a Latin base.
|
||||
_UNDECOMPOSED = {
|
||||
"ı": "i", "ø": "o", "đ": "d", "ħ": "h", "ŀ": "l", "ŧ": "t",
|
||||
"ß": "s", "æ": "a", "œ": "o", "þ": "t", "ð": "d", "ŋ": "n",
|
||||
"ł": "l",
|
||||
}
|
||||
|
||||
_CYRILLIC = {
|
||||
"а": "a", "б": "b", "в": "v", "г": "g", "д": "d", "е": "e", "ё": "e",
|
||||
"ж": "j", "з": "z", "и": "i", "й": "i", "к": "k", "л": "l", "м": "m",
|
||||
"н": "n", "о": "o", "п": "p", "р": "r", "с": "s", "т": "t", "у": "u",
|
||||
"ф": "f", "х": "h", "ц": "s", "ч": "s", "ш": "s", "щ": "s", "ъ": "",
|
||||
"ы": "i", "ь": "", "э": "e", "ю": "u", "я": "a",
|
||||
"і": "i", "ї": "i", "є": "e", "ґ": "g", "ў": "u",
|
||||
}
|
||||
|
||||
_GREEK = {
|
||||
"α": "a", "β": "v", "γ": "g", "δ": "d", "ε": "e", "ζ": "z", "η": "i",
|
||||
"θ": "t", "ι": "i", "κ": "k", "λ": "l", "μ": "m", "ν": "n", "ξ": "s",
|
||||
"ο": "o", "π": "p", "ρ": "r", "σ": "s", "ς": "s", "τ": "t", "υ": "i",
|
||||
"φ": "f", "χ": "h", "ψ": "s", "ω": "o",
|
||||
}
|
||||
|
||||
# Below this share of mappable letters the text is in a script we cannot read
|
||||
# phonetically, and forcing shapes onto it would be worse than not trying.
|
||||
_MIN_COVERAGE = 0.55
|
||||
|
||||
# English spellings that do not survive letter-by-letter reading.
|
||||
_DIGRAPH = {
|
||||
"sh": "S", "ch": "S", "ts": "S",
|
||||
"th": "TD", "ck": "K", "ng": "K", "gh": "K",
|
||||
"ph": "FV",
|
||||
"oo": "U", "ou": "O", "ow": "O", "wh": "U",
|
||||
"ee": "I", "ea": "I", "ie": "I",
|
||||
"qu": "K",
|
||||
}
|
||||
|
||||
_PAUSE = set(".,;:!?…\n")
|
||||
|
||||
|
||||
def to_latin(ch: str) -> str:
|
||||
"""Reduce any character to a bare Latin letter, or "" if it has none.
|
||||
|
||||
This is what makes the mouth language-agnostic: one reduction step replaces
|
||||
a per-language spelling table.
|
||||
"""
|
||||
c = ch.lower()
|
||||
if "a" <= c <= "z":
|
||||
return c
|
||||
if c in _UNDECOMPOSED:
|
||||
return _UNDECOMPOSED[c]
|
||||
if c in _CYRILLIC:
|
||||
return _CYRILLIC[c]
|
||||
if c in _GREEK:
|
||||
return _GREEK[c]
|
||||
# Strip combining marks: é→e, ü→u, ş→s, ğ→g, ế→e, ñ→n, å→a …
|
||||
base = "".join(k for k in unicodedata.normalize("NFD", c)
|
||||
if not unicodedata.combining(k))
|
||||
if len(base) == 1 and "a" <= base <= "z":
|
||||
return base
|
||||
if base and base != c: # e.g. fi → fi, take the first
|
||||
return to_latin(base[0])
|
||||
return ""
|
||||
|
||||
|
||||
def coverage(text: str) -> float:
|
||||
"""Fraction of the letters in `text` we can reduce to a Latin sound."""
|
||||
letters = [c for c in (text or "") if c.isalpha()]
|
||||
if not letters:
|
||||
return 0.0
|
||||
return sum(1 for c in letters if to_latin(c)) / len(letters)
|
||||
|
||||
|
||||
def text_to_visemes(text: str) -> list[tuple[str, float]]:
|
||||
"""Split a line of speech into (viseme, duration-weight) pairs.
|
||||
|
||||
Returns [] for scripts whose written form does not reveal pronunciation, so
|
||||
the caller falls back to the audio-only mouth instead of miming nonsense.
|
||||
"""
|
||||
s = (text or "").lower()
|
||||
if coverage(s) < _MIN_COVERAGE:
|
||||
return []
|
||||
|
||||
out: list[tuple[str, float]] = []
|
||||
i, n = 0, len(s)
|
||||
while i < n:
|
||||
ch = s[i]
|
||||
if ch in _PAUSE:
|
||||
out.append(("REST", 1.4))
|
||||
i += 1
|
||||
continue
|
||||
if ch.isspace():
|
||||
# A word gap is a beat, not a closed mouth — closing between every
|
||||
# word makes the avatar look like it is chewing.
|
||||
if out and out[-1][0] != "REST":
|
||||
out.append((out[-1][0], 0.35))
|
||||
i += 1
|
||||
continue
|
||||
|
||||
# Digraphs are an orthographic quirk of Latin spelling; check them on
|
||||
# the reduced letters so "SCH"/"Sch" and accented forms match too.
|
||||
two = to_latin(ch) + (to_latin(s[i + 1]) if i + 1 < n else "")
|
||||
if len(two) == 2 and two in _DIGRAPH:
|
||||
v = _DIGRAPH[two]
|
||||
i += 2
|
||||
else:
|
||||
base = to_latin(ch)
|
||||
i += 1
|
||||
if not base:
|
||||
continue
|
||||
v = _LETTER.get(base)
|
||||
if v is None:
|
||||
continue
|
||||
# A doubled letter is one sound in every orthography we handle here.
|
||||
if out and out[-1][0] == v:
|
||||
continue
|
||||
out.append((v, _DUR[v]))
|
||||
return out
|
||||
|
||||
|
||||
class VisemeStream:
|
||||
"""Fuses the transcript's shape sequence onto the audio's timing.
|
||||
|
||||
Thread note: `feed_text` runs on the receive coroutine and `frames` on the
|
||||
playback coroutine. Both live in the same asyncio loop, and `deque` append
|
||||
and popleft are atomic, so no lock is needed.
|
||||
"""
|
||||
|
||||
# Seconds a phoneme occupies at a normal speaking rate. The clock adapts
|
||||
# between these when the queue runs long (the model is talking fast) or
|
||||
# short (it is trailing off).
|
||||
_MIN_STEP = 0.045
|
||||
_MAX_STEP = 0.105
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._q: deque[tuple[str, float]] = deque()
|
||||
self._cur = ("REST", 1.0)
|
||||
self._carry = 0.0
|
||||
|
||||
def reset(self) -> None:
|
||||
self._q.clear()
|
||||
self._cur = ("REST", 1.0)
|
||||
self._carry = 0.0
|
||||
|
||||
def feed_text(self, text: str) -> None:
|
||||
for item in text_to_visemes(text):
|
||||
self._q.append(item)
|
||||
# Never let a stalled turn pile up an unbounded backlog.
|
||||
while len(self._q) > 600:
|
||||
self._q.popleft()
|
||||
|
||||
@property
|
||||
def pending(self) -> int:
|
||||
return len(self._q)
|
||||
|
||||
def _step_seconds(self) -> float:
|
||||
# A long backlog means speech is outrunning the clock; shorten the step
|
||||
# so the mouth catches up instead of drifting further behind the voice.
|
||||
backlog = min(1.0, len(self._q) / 45.0)
|
||||
return self._MAX_STEP - (self._MAX_STEP - self._MIN_STEP) * backlog
|
||||
|
||||
def frames(self, audio, hop: float):
|
||||
"""Blend audio frames [(level, openness, width)] with the text queue."""
|
||||
out = []
|
||||
for level, a_open, a_wide in audio:
|
||||
if level <= 0.0:
|
||||
# Silence: let the queue wait rather than burning through it
|
||||
# during a pause, or the mouth ends up ahead of the voice.
|
||||
out.append((0.0, 0.0, 0.0))
|
||||
continue
|
||||
|
||||
self._carry += hop / max(1e-3, self._step_seconds() * self._cur[1])
|
||||
while self._carry >= 1.0 and self._q:
|
||||
self._cur = self._q.popleft()
|
||||
self._carry -= 1.0
|
||||
if self._carry >= 1.0:
|
||||
self._carry = 1.0 # queue empty — hold the last shape
|
||||
|
||||
t_open, t_wide, closure = VISEMES.get(self._cur[0], VISEMES["REST"])
|
||||
if self._q or self._cur[0] != "REST":
|
||||
# Text leads the shape; the audio keeps it honest so a bad
|
||||
# transcript alignment still tracks the real voice.
|
||||
o = 0.72 * t_open + 0.28 * a_open
|
||||
w = 0.78 * t_wide + 0.22 * a_wide
|
||||
else:
|
||||
o, w = a_open, a_wide
|
||||
closure = 0.0
|
||||
o *= 1.0 - closure
|
||||
out.append((level, max(0.0, min(1.0, o)), max(-1.0, min(1.0, w))))
|
||||
return out
|
||||
+11
-2
@@ -67,15 +67,18 @@ def is_ready() -> bool:
|
||||
return False
|
||||
|
||||
|
||||
def install_and_download(logger: Callable[[str], None] = print) -> tuple[bool, str]:
|
||||
def install_and_download(logger: Callable[[str], None] = print,
|
||||
notify: Callable[[str], None] | None = None) -> tuple[bool, str]:
|
||||
"""
|
||||
One-click setup for the UI button: pip-install openwakeword if missing, then
|
||||
download the wake model. Returns (ok, message). Never raises — every failure
|
||||
is reported through the returned message and the logger.
|
||||
"""
|
||||
_tell = notify or (lambda _msg: None)
|
||||
try:
|
||||
if not is_installed():
|
||||
logger("Wake word: installing openwakeword (one-time)…")
|
||||
_tell("Wake word: installing openwakeword (one-time)…")
|
||||
r = subprocess.run(
|
||||
[sys.executable, "-m", "pip", "install", "openwakeword"],
|
||||
capture_output=True, text=True,
|
||||
@@ -85,6 +88,7 @@ def install_and_download(logger: Callable[[str], None] = print) -> tuple[bool, s
|
||||
return False, f"pip install failed: {tail[0][:160]}"
|
||||
# Download the pretrained melspectrogram/embedding + wake models.
|
||||
logger("Wake word: downloading models…")
|
||||
_tell("Wake word: downloading models…")
|
||||
try:
|
||||
import openwakeword.utils as _u
|
||||
try:
|
||||
@@ -111,10 +115,14 @@ class WakeWordDetector:
|
||||
|
||||
def __init__(self, on_detect: Callable[[], None],
|
||||
threshold: float = DEFAULT_THRESHOLD,
|
||||
logger: Callable[[str], None] = print):
|
||||
logger: Callable[[str], None] = print,
|
||||
notify: Callable[[str], None] | None = None):
|
||||
self._on_detect = on_detect
|
||||
self._threshold = threshold
|
||||
self._logger = logger
|
||||
# See PluginRegistry: `logger` is the console and gets everything,
|
||||
# `notify` is the activity log and gets only what the user must act on.
|
||||
self._notify = notify or (lambda _msg: None)
|
||||
self._queue: queue.Queue = queue.Queue(maxsize=50)
|
||||
self._thread: threading.Thread | None = None
|
||||
self._running = False
|
||||
@@ -131,6 +139,7 @@ class WakeWordDetector:
|
||||
self._model = Model(wakeword_models=[WAKE_MODEL], inference_framework="onnx")
|
||||
except Exception as e:
|
||||
self._logger(f"Wake word: could not load model — {e}")
|
||||
self._notify("Wake word unavailable — use the WAKE NOW button.")
|
||||
self._model = None
|
||||
return False
|
||||
self._running = True
|
||||
|
||||
@@ -360,6 +360,93 @@ def _local_ip() -> str:
|
||||
return "127.0.0.1"
|
||||
|
||||
|
||||
def _ensure_certs() -> bool:
|
||||
"""
|
||||
Make sure config/certs holds a TLS key pair, generating a self-signed one the
|
||||
first time the dashboard runs.
|
||||
|
||||
The pair is deliberately NOT shipped in the repository. A private key that
|
||||
every user downloads is the same as having no private key at all: anyone can
|
||||
present a certificate that matches it. Generating locally gives each install
|
||||
its own key, costs about a second, and happens exactly once.
|
||||
|
||||
Returns True when a usable pair exists afterwards; False leaves the caller on
|
||||
plain HTTP, which still works — the QR code simply encodes http:// instead.
|
||||
"""
|
||||
certs = BASE_DIR / "config" / "certs"
|
||||
key_p = certs / "jarvis.key"
|
||||
crt_p = certs / "jarvis.crt"
|
||||
if key_p.exists() and crt_p.exists():
|
||||
return True
|
||||
|
||||
try:
|
||||
import datetime
|
||||
import ipaddress
|
||||
from cryptography import x509
|
||||
from cryptography.hazmat.primitives import hashes, serialization
|
||||
from cryptography.hazmat.primitives.asymmetric import rsa
|
||||
from cryptography.x509.oid import NameOID
|
||||
except ImportError:
|
||||
print("[Dashboard] cryptography not installed — serving over plain HTTP.")
|
||||
print("[Dashboard] For HTTPS run: pip install cryptography")
|
||||
return False
|
||||
|
||||
try:
|
||||
certs.mkdir(parents=True, exist_ok=True)
|
||||
key = rsa.generate_private_key(public_exponent=65537, key_size=2048)
|
||||
|
||||
who = x509.Name([
|
||||
x509.NameAttribute(NameOID.COMMON_NAME, "JARVIS Dashboard"),
|
||||
x509.NameAttribute(NameOID.ORGANIZATION_NAME, "JARVIS"),
|
||||
])
|
||||
|
||||
# The SAN has to cover every address the phone might use: the LAN IP the
|
||||
# QR code encodes, plus localhost when testing on the machine itself.
|
||||
alt = [x509.DNSName("localhost"),
|
||||
x509.IPAddress(ipaddress.IPv4Address("127.0.0.1"))]
|
||||
try:
|
||||
lan = _local_ip()
|
||||
if not lan.startswith("127."):
|
||||
alt.append(x509.IPAddress(ipaddress.IPv4Address(lan)))
|
||||
except Exception:
|
||||
pass # no LAN address resolvable — localhost entries still work
|
||||
|
||||
# Timezone-aware UTC: datetime.utcnow() is deprecated from Python 3.12 on,
|
||||
# and the builder normalises aware values to UTC itself.
|
||||
now = datetime.datetime.now(datetime.timezone.utc)
|
||||
cert = (
|
||||
x509.CertificateBuilder()
|
||||
.subject_name(who)
|
||||
.issuer_name(who)
|
||||
.public_key(key.public_key())
|
||||
.serial_number(x509.random_serial_number())
|
||||
.not_valid_before(now - datetime.timedelta(days=1))
|
||||
.not_valid_after(now + datetime.timedelta(days=3650))
|
||||
.add_extension(x509.SubjectAlternativeName(alt), critical=False)
|
||||
.add_extension(x509.BasicConstraints(ca=False, path_length=None), critical=True)
|
||||
.sign(key, hashes.SHA256())
|
||||
)
|
||||
|
||||
key_p.write_bytes(key.private_bytes(
|
||||
encoding=serialization.Encoding.PEM,
|
||||
format=serialization.PrivateFormat.TraditionalOpenSSL,
|
||||
encryption_algorithm=serialization.NoEncryption(),
|
||||
))
|
||||
crt_p.write_bytes(cert.public_bytes(serialization.Encoding.PEM))
|
||||
|
||||
try:
|
||||
import os as _os
|
||||
_os.chmod(key_p, 0o600) # best effort — largely a no-op on Windows
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
print(f"[Dashboard] Generated a self-signed certificate for this machine: {certs}")
|
||||
return True
|
||||
except Exception as e:
|
||||
print(f"[Dashboard] Certificate generation failed ({e}) — serving over plain HTTP.")
|
||||
return False
|
||||
|
||||
|
||||
def _read(name: str) -> str:
|
||||
return (STATIC_DIR / name).read_text(encoding="utf-8")
|
||||
|
||||
@@ -776,6 +863,9 @@ class DashboardServer:
|
||||
# no waiting for UAC dialogs or subprocess timeouts.
|
||||
asyncio.get_event_loop().run_in_executor(None, _ensure_network_access, PORT)
|
||||
|
||||
# Generate the TLS pair on first run so no private key ships in the repo.
|
||||
_ensure_certs()
|
||||
|
||||
use_ssl = self._ssl_enabled()
|
||||
ssl_key = BASE_DIR / "config" / "certs" / "jarvis.key"
|
||||
ssl_cert = BASE_DIR / "config" / "certs" / "jarvis.crt"
|
||||
|
||||
@@ -14,25 +14,25 @@ if _platform.system() == "Windows":
|
||||
|
||||
_subprocess.Popen = _Popen
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
# ── Console encoding ─────────────────────────────────────────────────────────
|
||||
# Status lines in this app carry emoji and arrows ("📤 file_controller → Moved:
|
||||
# a.txt → Documents/"). On a non-UTF-8 console — cp1254 on a Turkish Windows,
|
||||
# cp1251 on a Russian one, cp932 on a Japanese one — printing one of those
|
||||
# raises UnicodeEncodeError, and because the print sits after the tool's own
|
||||
# try/except, the exception escapes into the receive loop and takes the session
|
||||
# down. The assistant dies on a log line.
|
||||
#
|
||||
# Reconfiguring costs nothing and makes the app start the same way in every
|
||||
# locale. `errors="replace"` is the belt and braces — a console that genuinely
|
||||
# cannot render a glyph shows a box instead of killing the process.
|
||||
# ── Console must survive non-UTF-8 code pages ────────────────────────────────
|
||||
# Every status line in this file carries an emoji, and on a legacy Windows
|
||||
# console the active code page is the system one — cp1254 in Turkey, cp1251 in
|
||||
# Russia, cp932 in Japan. Printing an emoji there raises UnicodeEncodeError, and
|
||||
# because most of these prints sit inside the receive loop it takes the session
|
||||
# down on startup. Reconfiguring to UTF-8 with a replacement fallback costs
|
||||
# nothing and makes the app launch the same way in every locale.
|
||||
import sys as _sys
|
||||
for _stream in (_sys.stdout, _sys.stderr):
|
||||
|
||||
for _stream in ("stdout", "stderr"):
|
||||
try:
|
||||
_stream.reconfigure(encoding="utf-8", errors="replace")
|
||||
_s = getattr(_sys, _stream, None)
|
||||
if _s is not None and hasattr(_s, "reconfigure"):
|
||||
_s.reconfigure(encoding="utf-8", errors="replace")
|
||||
except Exception:
|
||||
pass
|
||||
pass # pythonw / redirected pipes / anything exotic — never fatal
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
import asyncio
|
||||
import re
|
||||
@@ -69,13 +69,17 @@ from actions.background_monitor import (
|
||||
)
|
||||
from actions.web_search import _news as _fetch_news_sync
|
||||
from memory.config_manager import (
|
||||
get_brief_enabled, get_voice, get_wake_word_enabled, save_wake_word_enabled, get_input_device, get_output_device,
|
||||
get_brief_enabled, get_media_resolution, get_proactive_audio_enabled,
|
||||
get_push_to_talk_enabled, get_thinking_enabled, get_turn_tuning, get_voice,
|
||||
get_wake_word_enabled, save_wake_word_enabled, get_input_device, get_output_device,
|
||||
)
|
||||
from core.plugin_loader import discover_plugins
|
||||
from core import undo as undo_stack
|
||||
from core import confirm as confirm_gate
|
||||
from core import audio_devices
|
||||
from core.action_loader import discover_actions
|
||||
from core.echo import EchoGuard
|
||||
from core.viseme import VisemeStream
|
||||
from core.wake_word import (
|
||||
WakeWordDetector, is_ready as wake_is_ready, install_and_download as wake_install,
|
||||
)
|
||||
@@ -120,6 +124,164 @@ def _pcm_level(samples) -> float:
|
||||
return min(1.0, (rms - _LEVEL_FLOOR) / (_LEVEL_FULL - _LEVEL_FLOOR))
|
||||
|
||||
|
||||
# ── Viseme extraction ─────────────────────────────────────────────────────────
|
||||
# The avatar's mouth used to be driven by one RMS value per ~200 ms write batch,
|
||||
# which is five updates a second averaged over a fifth of a second — it could
|
||||
# only ever flap. These read the *shape* of each 20 ms slice straight from the
|
||||
# spectrum of the audio being played, so no transcript, no forced alignment and
|
||||
# no language assumption: it works the same for Turkish and English.
|
||||
#
|
||||
# Two numbers come out. Openness tracks the first formant — F1 climbs as the jaw
|
||||
# drops, so /a/ reads open and /i/ or /u/ read closed. Width tracks the second —
|
||||
# F2 is high for spread vowels (/i/, /e/) and low for rounded ones (/u/, /o/).
|
||||
# Extra time beyond the device's reported output latency before the microphone
|
||||
# is trusted again: covers room decay and the speaker's own settling.
|
||||
_TAIL_MARGIN = 0.25
|
||||
|
||||
_VIS_WIN = 1024 # ~43 ms analysis window at 24 kHz: enough for formants
|
||||
_VIS_HOP = 480 # 20 ms between frames, i.e. 50 shapes a second
|
||||
|
||||
# Delay from handing the first bytes of a reply to an already-running output
|
||||
# stream to hearing them: one callback period, plus whatever the DAC adds.
|
||||
_FIRST_SOUND = CHUNK_SIZE / RECEIVE_SAMPLE_RATE # ~43 ms
|
||||
# How far past the device's own buffer the mouth's timeline may drift before it
|
||||
# is re-anchored. The buffer is the hard limit on how much audio can be queued
|
||||
# ahead, so anything beyond it plus a margin for clock error is impossible.
|
||||
_CURSOR_SLACK = 0.15
|
||||
|
||||
# Erring early is the safe direction. A viewer tolerates a mouth that moves
|
||||
# slightly before the sound far better than one that moves after it — the
|
||||
# broadcast limits are about 45 ms of lag against 125 ms of lead — so where
|
||||
# this is uncertain it is biased to lead.
|
||||
|
||||
|
||||
def _pcm_visemes(samples, sr: int = 24000):
|
||||
"""Slice a PCM block into (level, openness, width) frames, one per 20 ms.
|
||||
|
||||
Returns [] on anything unexpected — the mouth falls back to loudness-only
|
||||
articulation rather than the caller having to handle an error.
|
||||
"""
|
||||
try:
|
||||
x = np.asarray(samples, dtype=np.float32)
|
||||
if x.size < _VIS_WIN:
|
||||
return []
|
||||
win = np.hanning(_VIS_WIN).astype(np.float32)
|
||||
freqs = np.fft.rfftfreq(_VIS_WIN, 1.0 / sr)
|
||||
b_f1_lo = (freqs >= 150) & (freqs < 450) # F1 of close vowels
|
||||
b_f1_hi = (freqs >= 450) & (freqs < 1100) # F1 of open vowels
|
||||
b_f2_bk = (freqs >= 600) & (freqs < 1300) # F2 of rounded vowels
|
||||
b_f2_fr = (freqs >= 1700) & (freqs < 3200) # F2 of spread vowels
|
||||
b_hiss = (freqs >= 3800) & (freqs < 8000) # fricatives
|
||||
|
||||
# One frame per hop across the *whole* block. Stepping only while a full
|
||||
# window fits stopped 1024 - 480 samples short of the end, so a 200 ms
|
||||
# batch yielded 160 ms of schedule: the mouth ran out of frames before
|
||||
# the audio ran out of sound, and each batch no longer lined up with the
|
||||
# end of the one before it. Losing 20 % of every batch is most of why
|
||||
# the mouth did not track the words.
|
||||
out = []
|
||||
for start in range(0, x.size, _VIS_HOP):
|
||||
# The level gates closures, so it is measured over exactly this
|
||||
# 20 ms and never looks ahead. The spectrum needs a longer window
|
||||
# to resolve formants and may be short-filled at the very end.
|
||||
level = _pcm_level(x[start:start + _VIS_HOP])
|
||||
seg = x[start:start + _VIS_WIN]
|
||||
if seg.size < _VIS_WIN:
|
||||
seg = np.concatenate([seg, np.zeros(_VIS_WIN - seg.size,
|
||||
dtype=np.float32)])
|
||||
if level <= 0.0:
|
||||
out.append((0.0, 0.0, 0.0))
|
||||
continue
|
||||
mag = np.abs(np.fft.rfft((seg - seg.mean()) * win))
|
||||
f1l, f1h = float(mag[b_f1_lo].sum()), float(mag[b_f1_hi].sum())
|
||||
f2b, f2f = float(mag[b_f2_bk].sum()), float(mag[b_f2_fr].sum())
|
||||
hiss = float(mag[b_hiss].sum())
|
||||
|
||||
openness = f1h / (f1l + f1h + 1e-6)
|
||||
width = (f2f - f2b) / (f2f + f2b + 1e-6)
|
||||
# A wide-open jaw physically cannot purse, so openness damps width.
|
||||
# /a/ has a low enough F2 to read as "rounded" on the bands alone;
|
||||
# letting openness suppress the width term is what keeps an open
|
||||
# vowel from pursing.
|
||||
width *= (1.0 - openness) ** 0.8
|
||||
# Fricatives are formed with a nearly closed mouth.
|
||||
h = hiss / (f1l + f1h + f2b + f2f + hiss + 1e-6)
|
||||
openness *= 1.0 - 0.65 * min(1.0, h * 2.5)
|
||||
out.append((level,
|
||||
float(min(1.0, max(0.0, openness))),
|
||||
float(min(1.0, max(-1.0, width)))))
|
||||
return out
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
|
||||
def _describe_tools(declarations) -> str:
|
||||
"""One line per capability, straight from the live tool declarations.
|
||||
|
||||
Derived rather than written down: the action and plugin registries are
|
||||
discovered at startup, so whatever the user has installed is what the model
|
||||
is told it can do. Adding a plugin extends this by itself, and removing one
|
||||
stops the model from claiming an ability it no longer has.
|
||||
"""
|
||||
lines = []
|
||||
for d in declarations or ():
|
||||
try:
|
||||
name = d.get("name") if isinstance(d, dict) else getattr(d, "name", None)
|
||||
desc = (d.get("description") if isinstance(d, dict)
|
||||
else getattr(d, "description", "")) or ""
|
||||
except Exception:
|
||||
continue
|
||||
if not name:
|
||||
continue
|
||||
desc = " ".join(str(desc).split())
|
||||
lines.append(f"- {name}: {desc[:150]}" if desc else f"- {name}")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def _describe_limits(has_vision: bool, has_mic: bool) -> str:
|
||||
"""The other half of self-knowledge: what is out of reach, and why.
|
||||
|
||||
Derived from how the program is actually built, not from a list of refusals.
|
||||
A model that knows its boundaries stops improvising around them, and stating
|
||||
them as architecture rather than as rules keeps the answer honest in any
|
||||
language.
|
||||
"""
|
||||
out = [
|
||||
"- Anything not listed above is outside your reach. Say so in one clause "
|
||||
"and offer the nearest thing you can actually do — never mime an action "
|
||||
"you cannot take, and never report a result you did not get.",
|
||||
"- You act on this machine only. You cannot reach the user's other "
|
||||
"devices, accounts or hardware except through the tools listed above.",
|
||||
"- You remember what is in the memory block and what has been said this "
|
||||
"session. Anything else you were told before is gone unless it was saved.",
|
||||
]
|
||||
if has_vision:
|
||||
out.append(
|
||||
"- Your sight is not continuous. You see nothing until you call a "
|
||||
"vision tool, and then only that single frame at that moment — you "
|
||||
"cannot watch, monitor or notice something changing on screen.")
|
||||
else:
|
||||
out.append("- You have no sight at all in this build.")
|
||||
if has_mic:
|
||||
out.append(
|
||||
"- You hear nothing while the microphone is muted, and you cannot "
|
||||
"unmute it yourself.")
|
||||
return "\n".join(out)
|
||||
|
||||
|
||||
def _render_prompt(template: str, values: dict) -> str:
|
||||
"""Fill {tokens} in the prompt template.
|
||||
|
||||
A plain replace rather than str.format: the file is meant to be edited by
|
||||
hand, and a stray brace in someone's own wording must never take the app
|
||||
down at startup.
|
||||
"""
|
||||
out = template or ""
|
||||
for key, val in values.items():
|
||||
out = out.replace("{" + key + "}", str(val))
|
||||
return out
|
||||
|
||||
|
||||
def _get_api_key() -> str:
|
||||
with open(API_CONFIG_PATH, "r", encoding="utf-8") as f:
|
||||
return json.load(f)["gemini_api_key"]
|
||||
@@ -137,6 +299,22 @@ def _load_system_prompt() -> str:
|
||||
|
||||
_CTRL_RE = re.compile(r"<ctrl\d+>", re.IGNORECASE)
|
||||
|
||||
# Transcript chunks shorter than this may legitimately repeat ("evet, evet"),
|
||||
# so only longer ones are treated as duplicates.
|
||||
_REPEAT_MIN = 12
|
||||
|
||||
|
||||
def _is_repeat_chunk(txt: str, buf: list) -> bool:
|
||||
"""True if this transcript chunk has already been seen this turn.
|
||||
|
||||
Guards against the API re-sending the tail of a response across the several
|
||||
turn_completes a tool-using turn produces.
|
||||
"""
|
||||
if len(txt) < _REPEAT_MIN:
|
||||
return bool(buf) and txt == buf[-1]
|
||||
joined = " ".join(buf)
|
||||
return txt in joined
|
||||
|
||||
def _clean_transcript(text: str) -> str:
|
||||
text = _CTRL_RE.sub("", text)
|
||||
text = re.sub(r"[\x00-\x08\x0b-\x1f]", "", text)
|
||||
@@ -366,6 +544,30 @@ class JarvisLive:
|
||||
self._vision_last_time = 0.0 # monotonic time of last screen_process call (cooldown guard)
|
||||
self._vision_busy = False # True while a vision capture/inject cycle is in flight
|
||||
self._interrupted = False # True while draining audio after user interrupt
|
||||
# Transcript-driven mouth shapes for the avatar. Fed from the receive
|
||||
# loop as words arrive, drained by the playback loop against the audio.
|
||||
self._visemes = VisemeStream()
|
||||
self._last_out_logged = "" # de-dupes a re-sent transcript tail
|
||||
# Push-to-talk
|
||||
self._ptt_enabled = False
|
||||
self._ptt_held = False
|
||||
self._ptt = None # core.hotkey.PushToTalk
|
||||
self._out_level = 0.0 # level of the audio being played right now
|
||||
self._echo = EchoGuard()
|
||||
# `stream.write()` returns when the buffer accepts the audio, not when the
|
||||
# speaker has finished with it, so sound is still in the room after the
|
||||
# speaking flag drops. Streaming the microphone during that gap is how an
|
||||
# assistant ends up answering itself. Measured from the device rather than
|
||||
# guessed; see _play_audio.
|
||||
self._out_latency = 0.20 # seconds, replaced with the real value
|
||||
self._tail_until = 0.0 # monotonic time the echo tail expires
|
||||
# Wall-clock time at which the audio written next will begin to sound.
|
||||
# The mouth is scheduled against this, never against "now": batches are
|
||||
# handed to the device far faster than they play, so "now" ran the lips
|
||||
# ahead of the words and cut every schedule short. 0 = nothing playing.
|
||||
self._play_cursor = 0.0
|
||||
self.ui.on_push_to_talk = self.set_push_to_talk
|
||||
self.ui.ptt_hold = self._on_ptt
|
||||
self.ui.on_text_command = self._on_text_command
|
||||
self.ui.on_remote_clicked = self._make_remote_key
|
||||
self.ui.on_interrupt = self.interrupt
|
||||
@@ -398,6 +600,7 @@ class JarvisLive:
|
||||
self._session_log: list[str] = [] # conversation turns for end-of-session summary
|
||||
|
||||
self._enhanced_live = True # proactive audio; auto-disabled if the server rejects it
|
||||
self._tuned_live = True # turn-taking / media / thinking knobs; same fallback
|
||||
|
||||
_base_dir = Path(__file__).resolve().parent
|
||||
_inline_names = {t["name"] for t in TOOL_DECLARATIONS}
|
||||
@@ -416,7 +619,11 @@ class JarvisLive:
|
||||
self._plugin_registry = discover_plugins(
|
||||
plugins_dir=_base_dir / "plugins",
|
||||
core_tool_names=_core_names,
|
||||
logger=lambda msg: (print(f"[Plugins] {msg}"), self.ui.write_log(f"SYS: {msg}")),
|
||||
# Console gets the full boot transcript; the activity log gets only
|
||||
# what the user has to know about. Every plugin loading correctly is
|
||||
# the expected case and does not belong in their conversation.
|
||||
logger=lambda msg: print(f"[Plugins] {msg}"),
|
||||
notify=lambda msg: self.ui.write_log(f"SYS: {msg}"),
|
||||
)
|
||||
self.ui.get_plugins = self._plugin_registry.list_for_ui
|
||||
self.ui.get_plugin_settings = self._plugin_registry.settings_schemas # ⚙ settings tab
|
||||
@@ -429,6 +636,15 @@ class JarvisLive:
|
||||
self._awake = not self._wake_enabled
|
||||
self._wake_detector: WakeWordDetector | None = None
|
||||
self._wake_sleep_timeout = WAKE_SLEEP_TIMEOUT
|
||||
|
||||
# Restore the saved push-to-talk preference. Doing it here rather than
|
||||
# in __init__ means the hotkey thread only exists once there is a
|
||||
# session to talk to.
|
||||
if get_push_to_talk_enabled():
|
||||
try:
|
||||
self.set_push_to_talk(True)
|
||||
except Exception as e:
|
||||
print(f"[JARVIS] ⚠ Push-to-talk unavailable: {e}")
|
||||
# UI control surface for the Wake Word settings section.
|
||||
self.ui.wake_is_ready = wake_is_ready # () -> bool
|
||||
self.ui.wake_get_state = self._wake_state # () -> dict
|
||||
@@ -449,7 +665,8 @@ class JarvisLive:
|
||||
if self._wake_detector is None:
|
||||
self._wake_detector = WakeWordDetector(
|
||||
on_detect=self._on_wake_detected,
|
||||
logger=lambda m: (print(f"[Wake] {m}"), self.ui.write_log(f"SYS: {m}")),
|
||||
logger=lambda m: print(f"[Wake] {m}"),
|
||||
notify=lambda m: self.ui.write_log(f"SYS: {m}"),
|
||||
)
|
||||
if not self._wake_detector.ready:
|
||||
return self._wake_detector.start()
|
||||
@@ -519,7 +736,10 @@ class JarvisLive:
|
||||
|
||||
def _ui_wake_install(self) -> tuple[bool, str]:
|
||||
"""Download openwakeword + the model (runs in a UI worker thread)."""
|
||||
return wake_install(logger=lambda m: self.ui.write_log(f"SYS: {m}"))
|
||||
# Triggered by the user pressing the button, so its progress is exactly
|
||||
# what they are waiting to see.
|
||||
return wake_install(logger=lambda m: print(f"[Wake] {m}"),
|
||||
notify=lambda m: self.ui.write_log(f"SYS: {m}"))
|
||||
|
||||
def plugin_say(self, instruction: str) -> None:
|
||||
"""
|
||||
@@ -625,14 +845,72 @@ class JarvisLive:
|
||||
self._loop
|
||||
)
|
||||
|
||||
def _tail_active(self) -> bool:
|
||||
"""True while the speakers may still be finishing our last sentence."""
|
||||
return time.monotonic() < self._tail_until
|
||||
|
||||
def set_speaking(self, value: bool):
|
||||
with self._speaking_lock:
|
||||
self._is_speaking = value
|
||||
if value:
|
||||
self._tail_until = 0.0
|
||||
else:
|
||||
# Hold the guard open across the device's own output latency plus a
|
||||
# margin for the room. The microphone is NOT muted during it — the
|
||||
# guard still lets a genuine reply through, so answering instantly
|
||||
# still works. Only our own echo is dropped.
|
||||
self._tail_until = time.monotonic() + self._out_latency + _TAIL_MARGIN
|
||||
if not value:
|
||||
# The echo history is deliberately NOT cleared here: the tail above
|
||||
# still needs it to recognise our own voice. It is dropped when the
|
||||
# tail expires. What the guard learned about the room always stays.
|
||||
self._out_level = 0.0
|
||||
if value:
|
||||
self.ui.set_state("SPEAKING")
|
||||
elif not self.ui.muted:
|
||||
self.ui.set_state("LISTENING")
|
||||
|
||||
def set_push_to_talk(self, enabled: bool) -> str:
|
||||
"""Turn hold-to-talk on or off. Returns the scope actually achieved."""
|
||||
from core.hotkey import PushToTalk
|
||||
|
||||
self._ptt_enabled = bool(enabled)
|
||||
self._ptt_held = False
|
||||
if not enabled:
|
||||
if self._ptt is not None:
|
||||
self._ptt.stop()
|
||||
self._ptt = None
|
||||
return "off"
|
||||
|
||||
if self._ptt is None:
|
||||
self._ptt = PushToTalk(self._on_ptt)
|
||||
scope = self._ptt.start()
|
||||
# A window-scoped chord is a real limitation, not a detail — say it once
|
||||
# in the log so nobody wonders why it does nothing while another app is
|
||||
# focused. Reporting it must never be able to undo the thing it reports.
|
||||
try:
|
||||
self.ui.write_log(
|
||||
f"SYS: Push-to-talk on — hold {self._ptt.label}"
|
||||
+ ("." if scope == "global"
|
||||
else " (works while this window is focused)."))
|
||||
except Exception:
|
||||
pass
|
||||
return scope
|
||||
|
||||
def _on_ptt(self, held: bool) -> None:
|
||||
"""Chord pressed or released — may arrive on the hotkey thread."""
|
||||
self._ptt_held = held
|
||||
if held:
|
||||
# Holding the key is also a way to wake it, so push-to-talk works
|
||||
# without having to say the wake word first.
|
||||
if self._wake_enabled and not self._awake:
|
||||
self._awake = True
|
||||
self._last_user_speech = time.monotonic()
|
||||
try:
|
||||
self.ui.set_state("LISTENING" if held else "SLEEPING")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def interrupt(self) -> None:
|
||||
"""Stop JARVIS mid-speech: drain queued audio and open mic immediately."""
|
||||
self._interrupted = True
|
||||
@@ -648,6 +926,9 @@ class JarvisLive:
|
||||
if drained:
|
||||
print(f"[JARVIS] ✋ Interrupted — {drained} audio chunks discarded")
|
||||
self.set_speaking(False)
|
||||
# The words we were about to mouth are never going to be spoken now.
|
||||
self._visemes.reset()
|
||||
self._play_cursor = 0.0 # next batch starts a fresh timeline
|
||||
if self._turn_done_event:
|
||||
self._turn_done_event.clear()
|
||||
self.ui.write_log("SYS: Interrupted — listening...")
|
||||
@@ -693,14 +974,17 @@ class JarvisLive:
|
||||
)
|
||||
|
||||
# Identity injection — overrides any hardcoded name in prompt.txt
|
||||
# Address form is a property of the language being spoken, so it is
|
||||
# stated as a principle rather than a two-language lookup — the model
|
||||
# already knows the respectful register of whatever language it is in.
|
||||
_addr = (f"ADDRESS: Always call the user '{_user_name}'."
|
||||
if _user_name
|
||||
else "ADDRESS: Address the user with the ordinary respectful form "
|
||||
"for a superior in the language you are currently speaking — "
|
||||
"\"sir\" in English, its everyday equivalent in any other "
|
||||
"language. Never an archaic or aristocratic form, and never "
|
||||
"the form from a different language than the one you are "
|
||||
"speaking in this sentence.")
|
||||
else 'ADDRESS: Address the user with the ordinary respectful form '
|
||||
'for a superior in the language you are currently speaking — '
|
||||
'"sir" in English, its everyday equivalent in any other '
|
||||
'language. Never an archaic or aristocratic form, and never '
|
||||
'the form from a different language than the one you are '
|
||||
'speaking in this sentence.')
|
||||
identity_ctx = (
|
||||
f"[IDENTITY]\n"
|
||||
f"Your name is {self._asst_name}. "
|
||||
@@ -708,6 +992,26 @@ class JarvisLive:
|
||||
f"{_addr}\n\n"
|
||||
)
|
||||
|
||||
# Everything the model is told about *itself* is derived here, not
|
||||
# written into prompt.txt: the name comes from config, the platform from
|
||||
# the host, the capability list from the registries that were just
|
||||
# discovered. Rename the assistant, add a plugin or move to another OS
|
||||
# and this follows without anyone editing a prompt.
|
||||
_all_decls = (TOOL_DECLARATIONS
|
||||
+ self._action_registry.get_tool_declarations()
|
||||
+ self._plugin_registry.get_tool_declarations())
|
||||
_names = {(d.get("name") if isinstance(d, dict) else getattr(d, "name", ""))
|
||||
for d in _all_decls}
|
||||
sys_prompt = _render_prompt(sys_prompt, {
|
||||
"assistant_name": self._asst_name,
|
||||
"platform": f"{_platform.system()} {_platform.release()}".strip(),
|
||||
"capabilities": _describe_tools(_all_decls),
|
||||
"limits": _describe_limits(
|
||||
has_vision="screen_process" in _names,
|
||||
has_mic=True,
|
||||
),
|
||||
})
|
||||
|
||||
parts = [time_ctx, identity_ctx]
|
||||
if mem_str:
|
||||
parts.append(mem_str)
|
||||
@@ -718,11 +1022,7 @@ class JarvisLive:
|
||||
output_audio_transcription={},
|
||||
input_audio_transcription={},
|
||||
system_instruction="\n".join(parts),
|
||||
tools=[{"function_declarations": (
|
||||
TOOL_DECLARATIONS
|
||||
+ self._action_registry.get_tool_declarations()
|
||||
+ self._plugin_registry.get_tool_declarations()
|
||||
)}],
|
||||
tools=[{"function_declarations": _all_decls}],
|
||||
# Hand back the handle captured from the last session_resumption
|
||||
# update. `handle=None` is exactly the old behaviour (ask for
|
||||
# handles, start fresh), so the first connect of a run is unchanged.
|
||||
@@ -749,9 +1049,67 @@ class JarvisLive:
|
||||
# support it, and it never reliably detected tone in practice.
|
||||
# To restore it on a 2.5 native-audio model, add back:
|
||||
# cfg["enable_affective_dialog"] = True )
|
||||
if get_proactive_audio_enabled():
|
||||
cfg["proactivity"] = types.ProactivityConfig(proactive_audio=True)
|
||||
|
||||
if self._tuned_live:
|
||||
cfg.update(self._tuning_config())
|
||||
|
||||
return types.LiveConnectConfig(**cfg)
|
||||
|
||||
def _tuning_config(self) -> dict:
|
||||
"""The optional knobs, kept apart so one bad field can be dropped wholesale.
|
||||
|
||||
Every one of these is a preview-API field. If a future model release
|
||||
stops accepting any of them the connection fails at setup, so the run
|
||||
loop turns `_tuned_live` off and reconnects on the plain config rather
|
||||
than leaving the user with an assistant that will not start.
|
||||
"""
|
||||
out: dict = {}
|
||||
|
||||
# How long the server waits through a pause before deciding your turn is
|
||||
# over. This — not the size of the prompt — is what most of the delay
|
||||
# before a reply actually is, and the default has to suit everybody, so
|
||||
# it is necessarily cautious.
|
||||
turn = get_turn_tuning()
|
||||
if turn.get("enabled", True):
|
||||
detect = types.AutomaticActivityDetection(
|
||||
silence_duration_ms=turn["silence_ms"],
|
||||
prefix_padding_ms=turn["prefix_ms"],
|
||||
)
|
||||
if turn["end_sensitivity"] == "high":
|
||||
detect.end_of_speech_sensitivity = types.EndSensitivity.END_SENSITIVITY_HIGH
|
||||
elif turn["end_sensitivity"] == "low":
|
||||
detect.end_of_speech_sensitivity = types.EndSensitivity.END_SENSITIVITY_LOW
|
||||
if turn["start_sensitivity"] == "high":
|
||||
detect.start_of_speech_sensitivity = types.StartSensitivity.START_SENSITIVITY_HIGH
|
||||
elif turn["start_sensitivity"] == "low":
|
||||
detect.start_of_speech_sensitivity = types.StartSensitivity.START_SENSITIVITY_LOW
|
||||
out["realtime_input_config"] = types.RealtimeInputConfig(
|
||||
automatic_activity_detection=detect)
|
||||
|
||||
# Screenshots and camera frames are tokenised at this resolution and then
|
||||
# stay in the session's context. 'medium' keeps on-screen text legible
|
||||
# for a fraction of a full-resolution frame.
|
||||
res = get_media_resolution()
|
||||
if res != "default":
|
||||
out["media_resolution"] = {
|
||||
"low": types.MediaResolution.MEDIA_RESOLUTION_LOW,
|
||||
"medium": types.MediaResolution.MEDIA_RESOLUTION_MEDIUM,
|
||||
"high": types.MediaResolution.MEDIA_RESOLUTION_HIGH,
|
||||
}[res]
|
||||
|
||||
# Thinking is left at the server default deliberately. Forcing the budget
|
||||
# to zero was measured on gemini-3.1-flash-live over interleaved trials
|
||||
# and did not make the first word arrive sooner — this model does not
|
||||
# appear to deliberate on the Live path, so pinning the field only adds a
|
||||
# way for a future release to behave differently. Set "thinking_enabled"
|
||||
# in config/api_keys.json to true to let it reason instead.
|
||||
if get_thinking_enabled():
|
||||
out["thinking_config"] = types.ThinkingConfig(thinking_budget=-1)
|
||||
|
||||
return out
|
||||
|
||||
async def _execute_tool(self, fc) -> types.FunctionResponse:
|
||||
name = fc.name
|
||||
args = dict(fc.args or {})
|
||||
@@ -759,6 +1117,7 @@ class JarvisLive:
|
||||
print(f"[JARVIS] 🔧 {name} {args}")
|
||||
self.ui.set_state("THINKING")
|
||||
|
||||
|
||||
if name == "save_memory":
|
||||
category = args.get("category", "notes")
|
||||
key = args.get("key", "")
|
||||
@@ -817,11 +1176,16 @@ class JarvisLive:
|
||||
print(f"[Vision] 🖥️ Screen: {len(img_b):,} bytes")
|
||||
_stall = "screen"
|
||||
self._pending_vision = (img_b, mime_t, user_text, angle)
|
||||
# The image is attached to this same exchange, so there is
|
||||
# nothing to stall for and nothing to announce. Asking for an
|
||||
# acknowledgement here is what produced two spoken answers —
|
||||
# the model filled that turn by answering the question from
|
||||
# imagination, then answered it again once it could see.
|
||||
result = (
|
||||
f"[VISION_ACTIVE] {_stall.capitalize()} captured. "
|
||||
f"Immediately say ONE short natural sentence in the user's own language, "
|
||||
f"telling them you are looking at their {_stall} right now. "
|
||||
f"Do NOT describe or guess content — the actual image arrives in the NEXT message."
|
||||
f"[VISION_ACTIVE] {_stall.capitalize()} captured and attached to this "
|
||||
f"same exchange. Do not acknowledge and do not answer yet — the image "
|
||||
f"is arriving with this result. Reply once, from what you actually see "
|
||||
f"in it."
|
||||
)
|
||||
|
||||
elif name == "close_camera":
|
||||
@@ -898,9 +1262,20 @@ class JarvisLive:
|
||||
self.ui.set_state("LISTENING")
|
||||
|
||||
print(f"[JARVIS] 📤 {name} → {str(result)[:80]}")
|
||||
|
||||
# A tool that declared itself NON_BLOCKING also says when its answer may
|
||||
# re-enter the conversation. Without this the model finishes whatever it
|
||||
# was saying and then reads the result out on top of it — which, for
|
||||
# something like a phone call already ringing, is exactly the noise the
|
||||
# non-blocking call was meant to avoid. Tools that declared nothing get
|
||||
# the API default and behave as they always have.
|
||||
_sched = (self._action_registry.scheduling(name)
|
||||
or self._plugin_registry.scheduling(name))
|
||||
_extra = {"scheduling": _sched} if _sched else {}
|
||||
return types.FunctionResponse(
|
||||
id=fc.id, name=name,
|
||||
response={"result": result}
|
||||
response={"result": result},
|
||||
**_extra
|
||||
)
|
||||
|
||||
async def _send_realtime(self):
|
||||
@@ -937,7 +1312,52 @@ class JarvisLive:
|
||||
return
|
||||
with self._speaking_lock:
|
||||
jarvis_speaking = self._is_speaking
|
||||
if not jarvis_speaking and not self.ui.muted and not self._phone_active:
|
||||
|
||||
# ── Barge-in ─────────────────────────────────────────────────────
|
||||
# While JARVIS talks the mic is not streamed, but it is still worth
|
||||
# listening to locally: if the user starts speaking, cut the answer
|
||||
# short the way a person would stop when interrupted.
|
||||
#
|
||||
# The whole difficulty is echo — on speakers the mic hears JARVIS.
|
||||
# So the test is not "is the mic loud" but "is the mic louder than
|
||||
# the echo of what we are playing right now", sustained long enough
|
||||
# that a cough or a keystroke cannot trigger it.
|
||||
if jarvis_speaking:
|
||||
# Nothing is streamed while JARVIS talks.
|
||||
#
|
||||
# Interrupting by voice used to live here: `EchoGuard` can pick a
|
||||
# user out from under our own echo, and `core/echo.py` still does
|
||||
# that for the tail below. Re-enabling is small — classify each
|
||||
# block here and call interrupt() after `required_blocks` of
|
||||
# agreement — but it depends on the listener's room, so it stays
|
||||
# out until it can be tried on real hardware.
|
||||
return
|
||||
|
||||
# ── Echo tail ────────────────────────────────────────────────────
|
||||
# The speaking flag has dropped but the speakers have not finished.
|
||||
# Sending this to the model is how an assistant hears itself, decides
|
||||
# it was addressed, and answers its own last sentence. The microphone
|
||||
# stays OPEN — the guard only drops blocks that are our own voice, so
|
||||
# replying the instant it stops still works.
|
||||
if self._tail_active():
|
||||
try:
|
||||
if not self._echo.is_user_speech(
|
||||
indata, SEND_SAMPLE_RATE, _pcm_level(indata)):
|
||||
return
|
||||
self._tail_until = 0.0 # a real voice ends the tail early
|
||||
except Exception:
|
||||
return
|
||||
elif self._echo._hist:
|
||||
self._echo.reset()
|
||||
|
||||
# ── Push-to-talk ─────────────────────────────────────────────────
|
||||
# When it is on the microphone is closed by default and the chord
|
||||
# opens it, which is the whole point: nothing leaves the machine
|
||||
# unless you are holding the key.
|
||||
if self._ptt_enabled and not self._ptt_held:
|
||||
return
|
||||
|
||||
if not self.ui.muted and not self._phone_active:
|
||||
data = indata.tobytes()
|
||||
loop.call_soon_threadsafe(
|
||||
self.out_queue.put_nowait,
|
||||
@@ -993,6 +1413,49 @@ class JarvisLive:
|
||||
print(f"[JARVIS] ❌ Mic: {e}")
|
||||
raise
|
||||
|
||||
async def _flush_pending_vision(self) -> bool:
|
||||
"""Send a captured frame immediately after its tool response.
|
||||
|
||||
The frame is already in hand by the time `screen_process` returns — the
|
||||
capture happened inside the tool call. The old flow still made the model
|
||||
speak a turn first and only injected the image on that turn's
|
||||
turn_complete, which cost a whole extra round trip AND produced two
|
||||
spoken answers: one improvised without the picture, then the real one.
|
||||
Sending it here means the model has the tool result and the image before
|
||||
it generates anything, so the user gets one answer, sooner.
|
||||
"""
|
||||
if not (self._pending_vision and self.session):
|
||||
return False
|
||||
|
||||
import base64 as _b64
|
||||
img_b, mime_t, question, angle = self._pending_vision
|
||||
self._pending_vision = None
|
||||
b64 = _b64.b64encode(img_b).decode("ascii")
|
||||
print(f"[Vision] 📤 {len(img_b):,} bytes (angle={angle}) → main session")
|
||||
|
||||
# Label the source. Without it the image arrives carrying nothing but
|
||||
# the user's own sentence, and a screenshot of this app — which has a
|
||||
# face in the middle of it — got read as a photo of the user. What the
|
||||
# label *means* is explained once, in the generated [SELF] block.
|
||||
src = ("[IMAGE SOURCE: WEBCAM]" if angle == "camera"
|
||||
else "[IMAGE SOURCE: SCREEN CAPTURE]")
|
||||
await self.session.send_client_content(
|
||||
turns={"role": "user", "parts": [
|
||||
{"inline_data": {"mime_type": mime_t, "data": b64}},
|
||||
{"text": f"{src}\n\n{question}"},
|
||||
]},
|
||||
turn_complete=True,
|
||||
)
|
||||
|
||||
if self._vision_cam_active:
|
||||
# Camera: stay busy until JARVIS has finished speaking the answer,
|
||||
# then close the preview.
|
||||
self._vision_cam_active = False
|
||||
self._vision_close_pending = True
|
||||
else:
|
||||
self._vision_busy = False
|
||||
return True
|
||||
|
||||
async def _receive_audio(self):
|
||||
print("[JARVIS] 👂 Recv started")
|
||||
out_buf, in_buf = [], []
|
||||
@@ -1032,8 +1495,20 @@ class JarvisLive:
|
||||
|
||||
if sc.output_transcription and sc.output_transcription.text:
|
||||
txt = _clean_transcript(sc.output_transcription.text)
|
||||
if txt and txt != (out_buf[-1] if out_buf else ""):
|
||||
# A turn that involves a tool call passes through
|
||||
# several turn_completes, and the API re-sends the
|
||||
# tail of the transcript across them. Comparing only
|
||||
# against the previous chunk missed that — once
|
||||
# out_buf had been flushed and emptied, the repeat
|
||||
# sailed straight back in, which logged the answer
|
||||
# twice AND made the avatar mouth it twice.
|
||||
if txt and not _is_repeat_chunk(txt, out_buf):
|
||||
out_buf.append(txt)
|
||||
# Hand the words to the mouth as they arrive, so
|
||||
# the avatar can form the consonants the audio
|
||||
# alone cannot show. Pure string work — it adds
|
||||
# nothing measurable to the response path.
|
||||
self._visemes.feed_text(txt)
|
||||
|
||||
if sc.input_transcription and sc.input_transcription.text:
|
||||
txt = _clean_transcript(sc.input_transcription.text)
|
||||
@@ -1051,10 +1526,12 @@ class JarvisLive:
|
||||
self._interrupted = False
|
||||
in_buf = []
|
||||
out_buf = []
|
||||
self._visemes.reset()
|
||||
continue
|
||||
|
||||
full_in = " ".join(in_buf).strip()
|
||||
if full_in:
|
||||
self._last_out_logged = "" # new exchange
|
||||
self.ui.write_log(f"You: {full_in}")
|
||||
self._session_log.append(f"User: {full_in}")
|
||||
if self._dashboard:
|
||||
@@ -1066,7 +1543,14 @@ class JarvisLive:
|
||||
in_buf = []
|
||||
|
||||
full_out = " ".join(out_buf).strip()
|
||||
# Second line of defence: even if a repeat slips
|
||||
# into a *fresh* buffer after a flush, never log the
|
||||
# same answer (or a tail of it) twice in a row.
|
||||
if full_out and len(full_out) >= _REPEAT_MIN and self._last_out_logged:
|
||||
if full_out in self._last_out_logged:
|
||||
full_out = ""
|
||||
if full_out:
|
||||
self._last_out_logged = full_out
|
||||
self.ui.write_log(f"{self._asst_name}: {full_out}")
|
||||
self._session_log.append(f"{self._asst_name}: {full_out}")
|
||||
if self._dashboard:
|
||||
@@ -1077,29 +1561,7 @@ class JarvisLive:
|
||||
}))
|
||||
out_buf = []
|
||||
|
||||
# Vision injection: model finished tool-response turn → now send the image
|
||||
if self._pending_vision and self.session:
|
||||
import base64 as _b64
|
||||
img_b, mime_t, question, angle = self._pending_vision
|
||||
self._pending_vision = None
|
||||
b64 = _b64.b64encode(img_b).decode("ascii")
|
||||
print(f"[Vision] 📤 {len(img_b):,} bytes (angle={angle}) → main session")
|
||||
await self.session.send_client_content(
|
||||
turns={"role": "user", "parts": [
|
||||
{"inline_data": {"mime_type": mime_t, "data": b64}},
|
||||
{"text": question},
|
||||
]},
|
||||
turn_complete=True,
|
||||
)
|
||||
# Mark next turn_complete behaviour depending on angle
|
||||
if self._vision_cam_active:
|
||||
# Camera: keep busy until JARVIS finishes speaking the answer
|
||||
self._vision_cam_active = False
|
||||
self._vision_close_pending = True
|
||||
else:
|
||||
# Screen-only: no camera to close; release busy flag now
|
||||
self._vision_busy = False
|
||||
elif self._vision_close_pending:
|
||||
if self._vision_close_pending:
|
||||
# This turn_complete IS the vision answer — close camera + release busy flag
|
||||
self._vision_close_pending = False
|
||||
self._vision_busy = False
|
||||
@@ -1117,6 +1579,7 @@ class JarvisLive:
|
||||
await self.session.send_tool_response(
|
||||
function_responses=fn_responses
|
||||
)
|
||||
await self._flush_pending_vision()
|
||||
except Exception as e:
|
||||
print(f"[JARVIS] ❌ Recv: {e}")
|
||||
traceback.print_exc()
|
||||
@@ -1153,6 +1616,19 @@ class JarvisLive:
|
||||
self.ui.write_log(f"SYS: Speaker '{_spk_name}' unavailable — using system default.")
|
||||
stream = _open_spk(None)
|
||||
|
||||
# Ask the device how far behind the speakers actually are, rather than
|
||||
# assuming. This is what the echo tail is sized from, so a machine with a
|
||||
# large audio buffer gets a correspondingly longer guard — and one with a
|
||||
# tiny buffer is not penalised with a delay it does not need.
|
||||
try:
|
||||
lat = float(getattr(stream, "latency", 0.0) or 0.0)
|
||||
if 0.0 < lat < 1.0:
|
||||
self._out_latency = lat
|
||||
print(f"[JARVIS] 🔊 Output latency {self._out_latency*1000:.0f} ms "
|
||||
f"→ echo tail {(self._out_latency + _TAIL_MARGIN)*1000:.0f} ms")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
while True:
|
||||
try:
|
||||
@@ -1182,10 +1658,53 @@ class JarvisLive:
|
||||
except asyncio.QueueEmpty:
|
||||
break
|
||||
|
||||
# Drive the HUD waveform from JARVIS's own voice while speaking.
|
||||
# Drive the HUD waveform and the avatar's mouth from JARVIS's
|
||||
# own voice. The batch is up to 200 ms long, so we hand over a
|
||||
# *schedule* of 20 ms viseme frames instead of a single averaged
|
||||
# level and let the HUD play it out in step with the audio.
|
||||
try:
|
||||
self.ui.set_audio_level(_pcm_level(
|
||||
np.frombuffer(bytes(batch), dtype=np.int16)))
|
||||
pcm = np.frombuffer(bytes(batch), dtype=np.int16)
|
||||
hop = _VIS_HOP / RECEIVE_SAMPLE_RATE
|
||||
frames = _pcm_visemes(pcm, sr=RECEIVE_SAMPLE_RATE)
|
||||
# When does this batch become audible? The stream was
|
||||
# started at launch and its callback has been pulling
|
||||
# silence ever since, so the first bytes of a reply reach
|
||||
# the speaker about one callback period later — NOT one
|
||||
# buffer later. `stream.latency` reports the buffer's
|
||||
# capacity, which is how much can be queued ahead, and on
|
||||
# Windows that is commonly 300-500 ms. Anchoring on it put
|
||||
# the entire schedule a buffer late; that is the half second
|
||||
# of lag, and it grew with whatever the device reported.
|
||||
#
|
||||
# After the anchor nothing needs measuring: the device
|
||||
# consumes at exactly realtime, so each batch sounds one
|
||||
# batch-duration after the one before it. The cursor is
|
||||
# re-anchored only when it leaves the range physically
|
||||
# possible — behind `now` means the device drained and this
|
||||
# batch starts a fresh stretch of speech, while further
|
||||
# ahead than the buffer can hold means it has drifted.
|
||||
now = time.time()
|
||||
horizon = self._out_latency + _CURSOR_SLACK
|
||||
if not (now <= self._play_cursor <= now + horizon):
|
||||
self._play_cursor = now + _FIRST_SOUND
|
||||
at = self._play_cursor
|
||||
# Advance by the batch's own duration whether or not it
|
||||
# yielded frames, so a block too short to analyse cannot
|
||||
# shift everything after it out of step with the audio.
|
||||
self._play_cursor += pcm.size / RECEIVE_SAMPLE_RATE
|
||||
if frames:
|
||||
frames = self._visemes.frames(frames, hop)
|
||||
self.ui.push_visemes(frames, hop, at)
|
||||
# Barge-in needs to know what we are playing, not just
|
||||
# how loud: the guard subtracts this from the microphone.
|
||||
self._out_level = max(f[0] for f in frames)
|
||||
self._echo.note_output(pcm, RECEIVE_SAMPLE_RATE,
|
||||
self._out_level)
|
||||
else:
|
||||
lvl = _pcm_level(pcm)
|
||||
self.ui.set_audio_level(lvl)
|
||||
self._out_level = lvl
|
||||
self._echo.note_output(pcm, RECEIVE_SAMPLE_RATE, lvl)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
@@ -1266,7 +1785,7 @@ class JarvisLive:
|
||||
turns={"role": "user", "parts": [{"text": p1}]},
|
||||
turn_complete=True,
|
||||
)
|
||||
self.ui.write_log("SYS: Briefing phase 1 (greeting) sent.")
|
||||
print("[JARVIS] Briefing phase 1 (greeting) sent.")
|
||||
|
||||
# ── Phase 2: fire as soon as Phase 1 audio is done ───────────────────
|
||||
async def _deliver_news():
|
||||
@@ -1296,14 +1815,18 @@ class JarvisLive:
|
||||
await asyncio.sleep(1.0)
|
||||
|
||||
try:
|
||||
news_text = await asyncio.wait_for(news_done, timeout=4.0)
|
||||
except Exception:
|
||||
news_text = await asyncio.wait_for(news_done, timeout=8.0)
|
||||
except Exception as e:
|
||||
self.ui.write_log(f"SYS: News fetch timed out/failed: {e!r}")
|
||||
news_text = ""
|
||||
|
||||
if not self.session:
|
||||
return
|
||||
|
||||
if news_text and len(news_text) > 60:
|
||||
failed = (not news_text) or news_text.startswith(
|
||||
("No news found", "Search failed", "Please provide")
|
||||
)
|
||||
if not failed:
|
||||
# Show on UI content panel immediately
|
||||
self.ui.show_content("NEWS — top world news today", news_text)
|
||||
|
||||
@@ -1313,6 +1836,9 @@ class JarvisLive:
|
||||
f"is displayed on screen. Do not call any tools.{lang_str}"
|
||||
)
|
||||
else:
|
||||
self.ui.write_log(
|
||||
f"SYS: News unavailable — backend returned: {news_text[:120]!r}"
|
||||
)
|
||||
p2 = (
|
||||
"News headlines could not be fetched right now. "
|
||||
f"Let the user know briefly.{lang_str}"
|
||||
@@ -1322,10 +1848,11 @@ class JarvisLive:
|
||||
turns={"role": "user", "parts": [{"text": p2}]},
|
||||
turn_complete=True,
|
||||
)
|
||||
self.ui.write_log("SYS: Briefing phase 2 (news) sent.")
|
||||
print("[JARVIS] Briefing phase 2 (news) sent.")
|
||||
except Exception as e:
|
||||
print(f"[Briefing] Phase 2 error: {e}")
|
||||
self.ui.write_log(f"SYS: Briefing phase 2 failed: {e}")
|
||||
print(f"[JARVIS] Briefing phase 2 failed: {e}")
|
||||
self.ui.write_log("SYS: Could not fetch the news for the briefing.")
|
||||
|
||||
asyncio.create_task(_deliver_news())
|
||||
|
||||
@@ -1350,14 +1877,10 @@ class JarvisLive:
|
||||
"Output ONLY the summary text, nothing else:\n\n" + convo
|
||||
)
|
||||
try:
|
||||
from google import genai as _genai
|
||||
client = _genai.Client(api_key=_get_api_key())
|
||||
resp = await asyncio.to_thread(
|
||||
client.models.generate_content,
|
||||
model="gemini-flash-latest",
|
||||
contents=prompt,
|
||||
from core import gemini
|
||||
summary = await asyncio.to_thread(
|
||||
gemini.text, prompt, gemini.SMART, None, 30_000,
|
||||
)
|
||||
summary = (resp.text or "").strip()
|
||||
if summary:
|
||||
save_session_summary(summary, lang)
|
||||
except Exception as e:
|
||||
@@ -1412,7 +1935,7 @@ class JarvisLive:
|
||||
turns={"role": "user", "parts": [{"text": msg}]},
|
||||
turn_complete=True,
|
||||
)
|
||||
self.ui.write_log(f"SYS: Monitor alert sent.")
|
||||
print("[JARVIS] Monitor alert sent.")
|
||||
await asyncio.sleep(6) # gap between consecutive alerts
|
||||
except Exception as e:
|
||||
print(f"[Monitor] ⚠️ Background check error: {e}")
|
||||
@@ -1455,7 +1978,7 @@ class JarvisLive:
|
||||
turns={"role": "user", "parts": [{"text": prompt}]},
|
||||
turn_complete=True,
|
||||
)
|
||||
self.ui.write_log("SYS: Proactive check-in.")
|
||||
print("[JARVIS] Proactive check-in.")
|
||||
except Exception as e:
|
||||
print(f"[Proactive] ⚠️ {e}")
|
||||
|
||||
@@ -1672,6 +2195,22 @@ class JarvisLive:
|
||||
print(f"[JARVIS] Error ({type(e).__name__}): {e}")
|
||||
traceback.print_exc()
|
||||
|
||||
# Turn-taking / media / thinking knobs rejected by the server
|
||||
# (preview API drift) — drop them first, because they are the
|
||||
# newest fields and the cheapest to lose. Proactive audio is
|
||||
# tried again on the next pass if the error persists.
|
||||
if self._tuned_live and (
|
||||
"INVALID_ARGUMENT" in err_str
|
||||
or "Unknown name" in err_str
|
||||
or "unexpected keyword" in err_str
|
||||
or "realtime_input" in err_str.lower()
|
||||
or "media_resolution" in err_str.lower()
|
||||
or "thinking" in err_str.lower()
|
||||
):
|
||||
self._tuned_live = False
|
||||
print("[JARVIS] Live tuning rejected — reconnecting without it.")
|
||||
continue
|
||||
|
||||
# Proactive audio rejected by the server (preview API drift) —
|
||||
# drop it and reconnect with the plain config.
|
||||
if self._enhanced_live and (
|
||||
|
||||
+145
-1
@@ -121,6 +121,150 @@ def save_wake_word_enabled(enabled: bool) -> None:
|
||||
CONFIG_FILE.write_text(json.dumps(data, indent=4), encoding="utf-8")
|
||||
|
||||
|
||||
def get_push_to_talk_enabled() -> bool:
|
||||
"""Hold-a-key-to-speak. When on, the mic is closed unless the chord is held."""
|
||||
return load_api_keys().get("push_to_talk_enabled", False)
|
||||
|
||||
|
||||
def save_push_to_talk_enabled(enabled: bool) -> None:
|
||||
_save_flag("push_to_talk_enabled", enabled)
|
||||
|
||||
|
||||
HUD_STYLES = ("face", "core")
|
||||
|
||||
|
||||
def get_hud_style() -> str:
|
||||
"""Which centrepiece the HUD draws: the animated head, or the reactor core.
|
||||
|
||||
Taste, not capability — both render in the same software painter and cost
|
||||
about the same. Defaults to the head because that is what MARK LIV shipped
|
||||
with; anyone who preferred the older look can switch back in ⚙ and the
|
||||
choice survives a restart.
|
||||
"""
|
||||
v = str(load_api_keys().get("hud_style", "face")).strip().lower()
|
||||
return v if v in HUD_STYLES else "face"
|
||||
|
||||
|
||||
def save_hud_style(style: str) -> None:
|
||||
s = str(style or "").strip().lower()
|
||||
_save_flag("hud_style", s if s in HUD_STYLES else "face")
|
||||
|
||||
|
||||
# ── Live-session tuning ──────────────────────────────────────────────────────
|
||||
# Everything here is optional and has a working default, so an untouched
|
||||
# config behaves exactly like a configured one. Each value is also a way out:
|
||||
# if a future model dislikes one of these, set it back and nothing else changes.
|
||||
|
||||
def get_thinking_enabled() -> bool:
|
||||
"""Whether the Live model may spend tokens thinking before it answers.
|
||||
|
||||
Off by default. A voice assistant is judged on how fast it starts talking,
|
||||
and the reasoning that actually needs deliberation in this app is delegated
|
||||
to the planning tools, which run on a separate non-Live model.
|
||||
"""
|
||||
return bool(load_api_keys().get("thinking_enabled", False))
|
||||
|
||||
|
||||
def save_thinking_enabled(enabled: bool) -> None:
|
||||
_save_flag("thinking_enabled", enabled)
|
||||
|
||||
|
||||
def get_turn_tuning() -> dict:
|
||||
"""How eagerly the server decides you have stopped speaking.
|
||||
|
||||
OFF by default, and that default was earned. Cutting turns shorter looks
|
||||
like a free speed win and is not: proactive audio has to judge whether an
|
||||
utterance was even addressed to the assistant, and a turn clipped early
|
||||
gives it less to judge, so it stays quiet — and the reply to your first
|
||||
sentence only arrives once your second one has given it enough context.
|
||||
That reads as the assistant being a turn behind, which is far worse than
|
||||
the fraction of a second the tuning saves.
|
||||
|
||||
Turn it on with "turn_tuning": {"enabled": true} if your own microphone and
|
||||
speaking pace suit it. `silence_ms` is the one that is felt: the pause the
|
||||
server sits through before accepting your turn is over.
|
||||
"""
|
||||
cfg = load_api_keys().get("turn_tuning")
|
||||
cfg = cfg if isinstance(cfg, dict) else {}
|
||||
|
||||
def _int(key, default, lo, hi):
|
||||
try:
|
||||
return max(lo, min(hi, int(cfg.get(key, default))))
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
|
||||
return {
|
||||
"enabled": bool(cfg.get("enabled", False)),
|
||||
"silence_ms": _int("silence_ms", 550, 200, 3000),
|
||||
"prefix_ms": _int("prefix_ms", 150, 0, 1000),
|
||||
# "high" = quicker to decide speech has ended.
|
||||
"end_sensitivity": str(cfg.get("end_sensitivity", "high")).lower(),
|
||||
"start_sensitivity": str(cfg.get("start_sensitivity", "default")).lower(),
|
||||
}
|
||||
|
||||
|
||||
def save_turn_tuning(values: dict) -> None:
|
||||
ensure_config_dir()
|
||||
data: dict = {}
|
||||
if CONFIG_FILE.exists():
|
||||
try:
|
||||
data = json.loads(CONFIG_FILE.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
data = {}
|
||||
cur = data.get("turn_tuning")
|
||||
cur = dict(cur) if isinstance(cur, dict) else {}
|
||||
cur.update(values or {})
|
||||
data["turn_tuning"] = cur
|
||||
CONFIG_FILE.write_text(json.dumps(data, indent=4), encoding="utf-8")
|
||||
|
||||
|
||||
def get_proactive_audio_enabled() -> bool:
|
||||
"""Whether the model gets to decide an utterance was not aimed at it and
|
||||
stay quiet.
|
||||
|
||||
On by default — it is what stops the assistant answering the room. But it
|
||||
is also the first thing to switch off if replies ever seem to arrive a turn
|
||||
late: what looks like lag is usually the model having judged your previous
|
||||
sentence as not addressed to it, and only changing its mind once the next
|
||||
one arrives.
|
||||
"""
|
||||
return bool(load_api_keys().get("proactive_audio", True))
|
||||
|
||||
|
||||
def save_proactive_audio_enabled(enabled: bool) -> None:
|
||||
_save_flag("proactive_audio", enabled)
|
||||
|
||||
|
||||
MEDIA_RESOLUTIONS = ("default", "low", "medium", "high")
|
||||
|
||||
|
||||
def get_media_resolution() -> str:
|
||||
"""How finely the model tokenises the screenshots and camera frames it is
|
||||
sent. 'medium' keeps on-screen text readable at a fraction of the tokens a
|
||||
full-resolution frame costs; 'low' is cheaper still but starts losing small
|
||||
text, which is most of what screen captures are for."""
|
||||
v = str(load_api_keys().get("media_resolution", "medium")).strip().lower()
|
||||
return v if v in MEDIA_RESOLUTIONS else "medium"
|
||||
|
||||
|
||||
def save_media_resolution(value: str) -> None:
|
||||
v = str(value or "").strip().lower()
|
||||
_save_flag("media_resolution", v if v in MEDIA_RESOLUTIONS else "medium")
|
||||
|
||||
|
||||
def _save_flag(key: str, value) -> None:
|
||||
"""Read-modify-write one key without disturbing the rest of the config."""
|
||||
ensure_config_dir()
|
||||
data: dict = {}
|
||||
if CONFIG_FILE.exists():
|
||||
try:
|
||||
data = json.loads(CONFIG_FILE.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
data = {}
|
||||
data[key] = bool(value) if isinstance(value, bool) else value
|
||||
CONFIG_FILE.write_text(json.dumps(data, indent=4), encoding="utf-8")
|
||||
|
||||
|
||||
def get_brief_enabled() -> bool:
|
||||
return load_api_keys().get("morning_brief_enabled", True)
|
||||
|
||||
@@ -189,7 +333,7 @@ def get_plugin_enabled(plugin_name: str) -> bool:
|
||||
# and the settings UI renders + persists them WITHOUT any core edit — keeping the
|
||||
# drop-in model intact. Values live under plugin_config[<namespace>][<key>].
|
||||
# A namespace defaults to the plugin name, but a suite of plugins (e.g. the
|
||||
# printer control/watchdog/autoeject trio) can share ONE namespace.
|
||||
# several printer plugins) can share ONE namespace.
|
||||
def get_plugin_config(namespace: str) -> dict:
|
||||
"""All stored values for a namespace (empty dict if none set yet)."""
|
||||
cfg = load_api_keys().get("plugin_config")
|
||||
|
||||
@@ -1,17 +1,21 @@
|
||||
# ⚙️ MARK LIII (53)
|
||||
# ⚙️ MARK LIV (54)
|
||||
### The Ultimate Cross-Platform Personal AI Assistant — By FatihMakes
|
||||
|
||||
> 📺 **[Watch the full setup video on YouTube](https://www.youtube.com/@FatihMakes)**
|
||||
|
||||
A real-time voice AI that can hear, see, understand, and control your computer — on any OS. Supports Windows, macOS, and Linux. Built on the Gemini Live API for native audio streaming, delivering zero subscriptions and total digital autonomy.
|
||||
A real-time voice AI that can hear, see, speak, and control your computer — on any OS. Supports Windows, macOS, and Linux. Built on the Gemini Live API for native audio streaming, delivering zero subscriptions and total digital autonomy.
|
||||
|
||||
---
|
||||
|
||||
## ✨ Overview
|
||||
|
||||
**MARK LIII is the hands-free & scalable release.** Say **"Hey Jarvis"** and it wakes; stay quiet and it slips back to sleep on its own — while asleep, your microphone never leaves the machine, so an off-hand *"I'll be right there"* to someone in the room no longer sets it off. Under the hood it now runs on the faster **Gemini 3.1 Flash Live** engine, and the moment you ask for something that takes a beat — analysing a file, searching the web — it answers instantly *("On it — going through that now…")* so you never wonder whether it heard you.
|
||||
**MARK LIV is the release where JARVIS gets a face.** A holographic head sits at the centre of the HUD and **speaks your assistant's words with real lip-sync** — not a jaw flapping to the volume meter, but actual mouth shapes: lips closing on *m*, *b*, *p*, spreading on *i*, rounding on *u*. Brows ride the sentence, the eyes flick between fixation points, and it blinks. Turn the sound down and you can follow roughly what it just said.
|
||||
|
||||
It's also built to grow: every skill — bundled or drop-in — now **describes itself in its own file**, so adding a tool is a one-file operation and the core stays lean.
|
||||
It ships as **zero extra dependencies and one 25 KB asset**. The face is real measured human geometry; everything else — the skull, the rig, the lighting — is generated at startup and drawn in software, so it looks identical on a gaming rig and a 2013 laptop, with no GPU driver in the loop.
|
||||
|
||||
The face is also the fastest status indicator in the app: it looks away while thinking, meets your eyes while listening, and lets its lids fall while asleep.
|
||||
|
||||
Underneath, Mark LIV rebuilt how the assistant knows itself — what it is, what machine it runs on, what it can do and, new, **what it cannot do** — all assembled from the live system at session start rather than written into a prompt that goes stale.
|
||||
|
||||
It's not just an assistant — it's an extension of your digital life.
|
||||
|
||||
@@ -22,10 +26,18 @@ It's not just an assistant — it's an extension of your digital life.
|
||||
### Core Features
|
||||
| Feature | Description |
|
||||
|---|---|
|
||||
| 🎙️ Wake Word | Local **"Hey Jarvis"** detection — sleeps until called, auto-sleeps after 2 min of silence, and never streams audio while asleep. Opt-in, one-click download, toggle & manual sleep/wake from the UI |
|
||||
| 🧑🎤 Holographic Avatar | An animated human head in the HUD — real facial geometry, lit and drawn in software, no GPU or extra packages |
|
||||
| 👄 Real Lip-Sync | ~50 mouth shapes a second from the audio's formants **and** the transcript — closures, spreads and rounds, not a volume meter |
|
||||
| 🌍 Language-Free Mouth | Articulation is derived by Unicode reduction, so Latin, Cyrillic and Greek scripts all work from one rule set — and scripts that hide pronunciation fall back cleanly |
|
||||
| 🙂 Facial Acting | Brows track the phrase, gaze saccades between fixations, natural blinking, a small nod on stressed syllables |
|
||||
| 😐 Face as Status | Looks away while thinking, meets your eyes while listening, lids fall while asleep, glances down at new content |
|
||||
| 🎚️ Push-to-Talk | Hold **Ctrl+Space** and the mic opens — closed the rest of the time. Truly global on Windows, window-scoped elsewhere |
|
||||
| 🔇 Self-Echo Guard | Never answers its own last sentence: the tail of its own voice is recognised and dropped without muting you |
|
||||
| 🪪 Runtime Self-Knowledge | Name, OS, abilities **and limits** are generated from the live system each session — rename it or add a plugin and it knows |
|
||||
| 🎙️ Wake Word | Local **"Hey Jarvis"** detection — sleeps until called, auto-sleeps after 2 min of silence, and never streams audio while asleep |
|
||||
| ⚡ Instant Acknowledgment | Speaks a short, context-aware reply in **your language** the instant a longer task starts — no more silent waiting |
|
||||
| 🚀 Faster Live Engine | Runs on **Gemini 3.1 Flash Live** — roughly 2× faster time-to-first-word than the previous model |
|
||||
| 🧩 Self-Describing Skills | Actions and plugins share one shape (`TOOL` / `PLUGIN` dict + `run()`), auto-discovered at launch — adding or moving a skill is a single file, no core edits |
|
||||
| 🧩 Self-Describing Skills | Actions and plugins share one shape (`TOOL` / `PLUGIN` dict + `run()`), auto-discovered at launch — adding a skill is a single file |
|
||||
| 🧠 Recallable Memory | No size limit and nothing silently forgotten — the prompt carries what fits, the rest is looked up on demand from a local search |
|
||||
| 👁️ Memory Panel | See every fact JARVIS has stored about you, when it learned it, and delete any of it in one click |
|
||||
| ↩️ Undo | Take back what the assistant did — files it moved, renamed, created or wrote, and settings it changed |
|
||||
@@ -34,13 +46,13 @@ It's not just an assistant — it's an extension of your digital life.
|
||||
| 🔗 Session Continuity | A dropped connection, a voice change or a device change no longer wipes the conversation |
|
||||
| 🧩 Plugin System | Drop a single `.py` file into `plugins/` — JARVIS learns a new skill on next launch |
|
||||
| 🎙️ Real-time Voice | Ultra-low latency conversation in any language via Gemini Live API |
|
||||
| 🎨 Live Theming | Recolour the entire HUD from a hue wheel or hex — applied instantly across every panel |
|
||||
| 〰️ Reactive HUD | Waveform and reactor core pulse to real audio — your mic while listening, JARVIS while speaking |
|
||||
| 🎨 Live Theming | Recolour the entire HUD from a hue wheel or hex — the avatar retints with it |
|
||||
| 〰️ Reactive HUD | Waveform pulses to real audio — your mic while listening, JARVIS while speaking |
|
||||
| 🎙️ Voice Picker | Choose from 5 native Gemini voices and switch live from the UI — no restart |
|
||||
| ♾️ Unlimited Sessions | Sliding-window context compression — one conversation can last for hours |
|
||||
| 🖥️ System Control | Launch apps, adjust volume/brightness, WiFi, shortcuts, power — all by voice |
|
||||
| 🧩 Autonomous Tasks | High-level planning for complex multi-step goals via agent mode |
|
||||
| 👁️ Visual Awareness | Real-time screen capture and webcam vision piped into your main Gemini session |
|
||||
| 👁️ Visual Awareness | Screen capture and webcam vision piped into your main Gemini session, labelled by source |
|
||||
| 🧠 Persistent Memory | Deeply remembers projects, preferences, and personal context across sessions |
|
||||
| ⌨️ Hybrid Input | Seamlessly switch between keyboard typing and voice commands |
|
||||
| 🌅 Morning Briefing | On first boot: greets you, reads the time, recaps yesterday, and fetches live news |
|
||||
@@ -68,23 +80,86 @@ It's not just an assistant — it's an extension of your digital life.
|
||||
|
||||
---
|
||||
|
||||
## 🆕 What's New in Mark LIII
|
||||
## 🆕 What's New in Mark LIV
|
||||
|
||||
Mark LIII is about making JARVIS **hands-free, faster, and easy to extend** — all universal: no hardcoded language, no bundled asset files, works the same on Windows, macOS and Linux.
|
||||
No hardcoded language, no GPU requirement, no new dependencies — identical on Windows, macOS and Linux.
|
||||
|
||||
### 🎙️ Wake Word — "Hey Jarvis"
|
||||
JARVIS can now sit quietly until you call it. Turn on **⚙ → WAKE WORD** (a one-click, opt-in download of a tiny local model) and it goes to sleep: the microphone is processed **only on your machine** by a local detector, and nothing is sent to the cloud until it hears **"Hey Jarvis."** Once awake it listens normally, then **auto-sleeps after 2 minutes** of silence. You can also **sleep/wake it by clicking** in the settings. Because it's a *local* gate, background chatter — *"I'm coming!"* to someone at home — never wakes it. It costs **zero** when off (the model isn't even loaded), and the detection runs in its own thread, so nothing else in the app slows down.
|
||||
### The face
|
||||
|
||||
### ⚡ Instant Acknowledgment
|
||||
No more silent gaps. When you ask for something that takes a moment — reading an uploaded file, a web/research search, building code — JARVIS **immediately** says one short, natural sentence *in your language* (*"Right away — going through that file now."*) and *then* runs the tool. Instant actions (opening an app, volume) stay snappy with no chatter.
|
||||
#### 🧑🎤 A real head, rendered in software
|
||||
The centre of the HUD is now an animated human head. Its face is **real measured human geometry** — MediaPipe's canonical face model, with actual eyelids, nostrils, lips and cheekbones. The skull, neck, jaw rig and lip rig are generated around it at startup, and the whole thing is lit and drawn with **QPainter**, which means:
|
||||
|
||||
### 🚀 Faster Live Engine — Gemini 3.1 Flash Live
|
||||
The live session moved to **`gemini-3.1-flash-live-preview`**, cutting the time-to-first-word roughly in half while keeping tools, all five voices, transcription, session resumption and sliding-window compression intact.
|
||||
* **no new dependencies** — it runs on the PyQt6 and numpy the app already needed;
|
||||
* **no OpenGL, no shaders, no GPU driver** to disagree with you — a VM, a remote desktop session and an old integrated laptop all render the same picture;
|
||||
* **one 25 KB asset**, and everything else is a formula.
|
||||
|
||||
### 🧩 Self-Describing Skills — a Scalable Core
|
||||
Every bundled **action** now carries its own `TOOL` declaration in its own file (exactly like a drop-in **plugin's** `PLUGIN` dict), and the core auto-discovers them at launch. `main.py` no longer holds a giant list of tool definitions and dispatch branches — it shrank by hundreds of lines. Adding a new built-in skill, or promoting an `actions/*.py` file into a shareable plugin, is now just… moving a file.
|
||||
It breathes, sways, blinks, and retints instantly when you change the HUD colour.
|
||||
|
||||
> Built on the Mark LI/LII foundation: the **🧩 Plugin System**, **♾️ Unlimited Sessions**, **🎨 Live Theming**, **〰️ Reactive HUD** and **🎙️ Voice Picker** are all still here.
|
||||
#### 👄 Lip-sync you can actually read
|
||||
The old mouth opened to the volume meter, five times a second. The new one produces **~50 mouth shapes per second** from two sources at once:
|
||||
|
||||
* **the audio** — a formant read of each 20 ms slice gives openness (how far the jaw drops) and width (spread for *i* and *e*, rounded for *u* and *o*). This is physics, so it is language-independent by construction.
|
||||
* **the transcript** — because no spectrum can tell you the lips are *closed*. /m/, /b/ and /p/ look identical to a filter bank and completely different on a face. The words supply the shape; the audio supplies the timing and the force.
|
||||
|
||||
#### 🌍 One rule set, every alphabet
|
||||
There is no per-language table. Every character is reduced to a bare Latin letter — Unicode decomposition strips accents (é, ü, ş, ğ, ế, ñ, å…), and Cyrillic and Greek transliterate — then articulation is looked up on the sound.
|
||||
|
||||
**Turkish, English, German, French, Spanish, Polish, Vietnamese, Czech, Russian, Ukrainian and Greek all work from the same twenty-odd rules.** Scripts whose spelling doesn't reveal pronunciation (Arabic, Chinese, Japanese, Hindi, Korean, Hebrew, Thai) are detected automatically and the mouth runs on the audio-only shape — less detail, never wrong. Adding a language costs nothing, because there is nothing to add.
|
||||
|
||||
#### 🙂 It acts while it talks
|
||||
Brows ride the *phrase*, not the syllable, with a slow asymmetry between them. The eyes make real saccades between fixation points, more often while speaking. It blinks. Loud syllables tip the head. Everything relaxes to neutral in silence — and the mouth **only** moves for the assistant's own voice, never for yours.
|
||||
|
||||
#### ◉ Two HUDs, one toggle
|
||||
Not everyone wants a face looking back at them. ⚙ → **HUD** swaps the centrepiece between the animated head and a **reactor core** — a gauge ring, three arcs that turn at a rate the state sets, a spectrum ring driven by the real audio level, and a core that brightens with the voice. Both render in the same software painter and cost the same; the choice is taste, and it survives a restart.
|
||||
|
||||
Nothing on the core moves for decoration. The rings speed up when JARVIS is thinking, the spikes are the actual waveform, and the colour is the state — the same language the face speaks, without the face.
|
||||
|
||||
#### 🗣️ It answers before it works
|
||||
Some replies used to open with three or four seconds of silence: not because a tool was slow, but because JARVIS was still *writing the tool call* — a set of quiz questions, the findings from a contract. The tool was instant; the composing was not, and from the user's side those are the same thing.
|
||||
|
||||
The rule is now about the silence rather than the tool: if a gap would form, say one sentence naming what you are starting, then do it. It applies to anything that takes a moment to run **or** a moment to write, without a list of which tools those are.
|
||||
|
||||
#### 😐 The face is a status light
|
||||
You read a gaze faster than you read a word, so the head tells you what the assistant is doing before it says anything. It **looks away and holds it while thinking** — brows drawn down, blinking suppressed, the way concentration actually looks — **meets your eyes while listening**, and **lets its lids fall while asleep**. When something appears in the content panel below, it **glances down at it**: a wordless "that landed".
|
||||
|
||||
### Talking to it
|
||||
|
||||
#### 🎚️ Push-to-talk
|
||||
Wake word is hands-free, but in a meeting or a noisy room a key is faster and never mishears. Turn on **⚙ → PUSH-TO-TALK** and the microphone stays **closed** until you hold **Ctrl+Space** — nothing leaves the machine while you are not holding it. Holding the chord also wakes the assistant, so it doubles as a silent alternative to saying the wake word.
|
||||
|
||||
On **Windows** the chord is genuinely global: it works while any other application has focus, implemented by polling two virtual-key codes thirty times a second, with **no new dependency** and no message loop. On **macOS and Linux** there is no dependency-free way to read global key state, so the chord is bound inside the window instead — and the app **says so in the log** rather than pretending otherwise.
|
||||
|
||||
#### 🔇 It no longer talks itself into replying
|
||||
Writing audio to a device returns when the buffer *accepts* the sound, not when the speaker has finished with it — so for a moment after a reply "ends", it is still in the room. Streaming the microphone during that gap is how an assistant hears its own last sentence, decides it was addressed, and answers itself.
|
||||
|
||||
Mark LIV holds a guard open across that gap, sized from the **device's own reported latency** rather than a guessed constant, so a machine with a large audio buffer gets a longer guard and one with a small buffer is not penalised. The microphone is **not muted** during it: both streams are reduced to band energies and as much of what was just played is subtracted from the microphone as fits, so only your assistant's own voice is dropped — replying the instant it stops still works.
|
||||
|
||||
> Interrupting it mid-sentence by voice is built on the same machinery and is deliberately **switched off** in this release. It depends too much on the listener's room to ship without testing on real hardware.
|
||||
|
||||
### How it understands itself
|
||||
|
||||
#### 🪪 It knows what it is, and what it isn't
|
||||
Who it is, what machine it runs on, what it can do and **what it cannot do** are assembled from the live system at session start — the configured name, the real OS, the tools actually discovered. Install a plugin and it knows it gained an ability; remove one and it stops claiming it.
|
||||
|
||||
The **limits** half is the important one: it knows its sight is a single frame on demand rather than a live feed, that it acts on this machine only, and that anything outside its tool list should be stated plainly instead of improvised.
|
||||
|
||||
All prompt wording lives in `core/prompt.txt` with `{tokens}` the app fills in — so you can rewrite the personality without touching Python, and a stray brace in your own wording can't break startup.
|
||||
|
||||
### 🩹 Fixes
|
||||
* Answers were sometimes **logged and spoken twice** — the Live API re-sends the tail of a transcript across the several turn-completes a tool call produces. Now de-duplicated at both the chunk and the flush level.
|
||||
* Asking JARVIS to look at the screen produced **two different answers** — the flow made it speak once *before* the image arrived, so it improvised, and again after. The frame is now attached to the same exchange as its tool result: one turn, one answer, one fewer round trip.
|
||||
* Screen captures were **unlabelled**, so a screenshot of this app — which has a face in the middle of it — could be read as a photo of the user. Images now carry their source.
|
||||
* On a non-UTF-8 console (cp1254, cp1251, cp932…) the emoji in the status lines **crashed the session on startup**. Streams are reconfigured at launch, so it starts the same way in every locale.
|
||||
* The HUD kept rendering the avatar **while the window was hidden or minimised**. It now stops, and resumes mid-motion rather than snapping.
|
||||
* Activity-log lines were fixed amber and ignored the theme; they now follow the accent colour.
|
||||
* Dependencies had no upper bounds, so the next major release of any of them would break every fresh clone. The load-bearing ones are now capped.
|
||||
* The mouth **ran ahead of the words, then behind them**. Three separate faults compounded: each 200 ms of audio only produced 160 ms of mouth shapes, every new batch overwrote the previous one instead of continuing it, and the schedule was anchored to the moment audio was *handed to the device* rather than the moment it becomes *audible*. The mouth now follows one continuous playback clock — measured at **15 ms** of timing error whether the sound card buffers 100 ms or 500 ms.
|
||||
* The jaw was driven by the **waveform meter**, which deliberately holds peaks so the bars don't flicker. That hold spanned exactly the consonants the mouth needed to close on. It now reads the audio's own 20 ms levels, so the gaps between words are real gaps.
|
||||
* Mouth timing was measured in **frames rather than seconds**, so the same constants meant three different mouths at 60, 30 and 20 fps, and a closure shorter than one frame could vanish entirely. Timing is now in seconds and the mouth is stepped once per 20 ms of audio, not once per repaint.
|
||||
* The **brows barely moved** — 6 px of travel on a 250 px head, because the rig weights halved an already small constant. Derived from the anatomy instead: 19 px.
|
||||
* The activity log opened with **a dozen lines of plumbing** — one per plugin loaded, plus wake-word and briefing status. The console still carries the full boot transcript; the log now shows your conversation, state changes and anything you have to act on, and nothing else.
|
||||
|
||||
> Built on the Mark LI–LIII foundation: the **🧩 Plugin System**, **♾️ Unlimited Sessions**, **🎨 Live Theming**, **🎙️ Wake Word** and **🧩 Self-Describing Skills** are all still here.
|
||||
|
||||
---
|
||||
|
||||
@@ -194,21 +269,22 @@ It is held in memory only, deliberately: writing it to disk would make a fresh l
|
||||
| **LI** | Plugin system · affective dialog · proactive audio · unlimited sessions |
|
||||
| **LII** | Voice picker · live theming · reactive HUD · recallable memory · undo · real confirmation · audio device picker · session continuity |
|
||||
| **LIII** | Wake word · Gemini 3.1 Flash Live · instant acknowledgment · self-describing action/plugin architecture |
|
||||
| **LIV** | Holographic avatar · viseme lip-sync · facial acting · face-as-status · push-to-talk · self-echo guard · runtime self-knowledge & limits |
|
||||
| *shared* | The last five above also shipped to LIII, LIV and LV at the same time — moving up a Mark never loses them |
|
||||
| **LIV+** | Plugin files: email · quiz mode · calendar · home assistant · 3D-printer · and more |
|
||||
| **LV+** | Interrupt by voice · conversation history · plugin files: email · quiz mode · calendar · home assistant · 3D-printer |
|
||||
|
||||
---
|
||||
|
||||
## ⚡ Quick Start
|
||||
|
||||
```bash
|
||||
git clone https://github.com/FatihMakes/Mark-LIII.git
|
||||
cd Mark-LIII
|
||||
git clone https://github.com/FatihMakes/Mark-LIV.git
|
||||
cd Mark-LIV
|
||||
python setup.py # installs deps for YOUR OS + the browser automation engine
|
||||
python main.py
|
||||
```
|
||||
|
||||
`setup.py` only ever installs what your operating system needs — the Windows-only libraries are skipped automatically on macOS and Linux (and vice-versa). Prefer to do it by hand? `pip install -r requirements.txt` works too.
|
||||
`setup.py` only ever installs what your operating system needs — the Windows-only libraries are skipped automatically on macOS and Linux, and vice-versa. It also checks your Python version up front, so a wrong interpreter fails with a sentence instead of a wall of pip output. Prefer to do it by hand? `pip install -r requirements.txt` works too.
|
||||
|
||||
> ⚠️ **Installation Note:** If you hit a `ModuleNotFoundError` for an OS-specific package, install it with `pip install <module_name>`. The optional **wake word** engine is *not* installed here — grab it in one click from **⚙ → WAKE WORD** inside the app.
|
||||
|
||||
@@ -219,10 +295,11 @@ python main.py
|
||||
| Requirement | Details |
|
||||
| --- | --- |
|
||||
| **OS** | Windows 10/11, macOS, or Linux |
|
||||
| **Python** | 3.11 or 3.12 |
|
||||
| **Python** | 3.11, 3.12 or 3.13 |
|
||||
| **Microphone** | Required for voice interaction (and for the "Hey Jarvis" wake word) |
|
||||
| **Speakers** | Required for voice replies |
|
||||
| **API Key** | Free Gemini API key (entered on first launch → `config/api_keys.json`) |
|
||||
| **GPU** | **Not required.** The avatar is rendered in software |
|
||||
| **Wake word** *(optional)* | One-click download from ⚙ → WAKE WORD (`openwakeword`, a few MB, fully local) |
|
||||
|
||||
---
|
||||
@@ -230,17 +307,22 @@ python main.py
|
||||
## 🗂️ Project Structure
|
||||
|
||||
```
|
||||
Mark LIII/
|
||||
├── main.py # Core loop — Gemini Live session, audio I/O, wake/sleep state, tool dispatch
|
||||
├── ui.py # PyQt6 HUD — reactive waveform, log panel, settings drawer, plugin manager, camera feed
|
||||
├── setup.py # OS-aware installer (skips wrong-OS dependencies)
|
||||
Mark LIV/
|
||||
├── main.py # Core loop — Gemini Live session, audio I/O, viseme extraction, tool dispatch
|
||||
├── ui.py # PyQt6 HUD — avatar canvas, waveform, log panel, settings drawer, camera feed
|
||||
├── setup.py # OS-aware installer (skips wrong-OS dependencies, checks your Python)
|
||||
├── .gitignore # Keeps your API key, TLS key and memories out of the repository
|
||||
├── plugins/
|
||||
│ ├── quiz.py # Interactive quiz — JARVIS writes the questions, you answer on screen
|
||||
│ ├── document_review.py # Contracts and policies in plain language, ordered by what matters
|
||||
│ ├── _google_core.py # Shared OAuth for the Gmail/Calendar plugins (not a plugin itself)
|
||||
│ ├── _printer_core.py # Shared printer connectivity (not a plugin itself)
|
||||
│ ├── _template.py # Copy this to write a new plugin — one file, drop in, done
|
||||
│ └── ... # Drop-in skills (each self-describes via a PLUGIN dict + run())
|
||||
├── actions/ # Bundled skills — each self-describes via a TOOL dict + handler
|
||||
│ ├── web_search.py # Gemini + DDG parallel search (news, research, price, compare)
|
||||
│ ├── screen_processor.py # Screen & webcam capture for vision
|
||||
│ ├── background_monitor.py # User-configured topic watching — daily DDG check, no crypto
|
||||
│ ├── background_monitor.py # User-configured topic watching — daily DDG check
|
||||
│ ├── proactive.py # Proactive 2.0 — time/context/rotation-aware check-ins
|
||||
│ ├── reminder.py # OS-native scheduled notifications
|
||||
│ ├── system_monitor.py # CPU / RAM / GPU / temperature telemetry
|
||||
@@ -261,9 +343,15 @@ Mark LIII/
|
||||
├── memory/
|
||||
│ ├── memory_manager.py # Load/save long_term.json — sessions, monitors, identity
|
||||
│ ├── config_manager.py # api_keys.json access — key, OS, name, voice, colour, toggles
|
||||
│ └── long_term.json # Persistent store: identity, preferences, projects, sessions, monitors
|
||||
│ └── long_term.json # Persistent store — created on first run
|
||||
├── core/
|
||||
│ ├── prompt.txt # Assistant personality and tool-routing rules
|
||||
│ ├── prompt.txt # All prompt wording — {tokens} are filled from the live system at startup
|
||||
│ ├── avatar.py # Avatar renderer — lighting, pose, expression, mouth (QPainter)
|
||||
│ ├── avatar_mesh.py # Head geometry — loads the face, generates skull/neck/rigs
|
||||
│ ├── face_model.obj # The face itself (MediaPipe canonical model, Apache-2.0, 25 KB)
|
||||
│ ├── viseme.py # Transcript → mouth shapes, fused with the audio's timing
|
||||
│ ├── echo.py # Tells your voice from the assistant's own echo; self-calibrating
|
||||
│ ├── hotkey.py # Push-to-talk chord — global on Windows, windowed fallback elsewhere
|
||||
│ ├── undo.py # One shared undo stack — actions register how to reverse themselves
|
||||
│ ├── confirm.py # Irreversible-action gate — the token is issued by the UI, not the model
|
||||
│ ├── audio_devices.py # Microphone / speaker list — filtered, measured, resolved by name
|
||||
@@ -271,11 +359,36 @@ Mark LIII/
|
||||
│ ├── action_loader.py # Bundled-action engine — the built-in twin of plugin_loader
|
||||
│ └── wake_word.py # Local "Hey Jarvis" detector — own thread, offline, opt-in
|
||||
└── config/
|
||||
└── api_keys.json # API key, OS setting, assistant name, user name, voice, UI colour, toggles
|
||||
├── api_keys.json # API key, name, voice, colour, toggles — created on first launch (git-ignored)
|
||||
└── certs/ # Self-signed TLS pair for the phone dashboard — generated locally (git-ignored)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 🙏 Third-Party Assets
|
||||
|
||||
| Asset | Source | Licence |
|
||||
| --- | --- | --- |
|
||||
| `core/face_model.obj` | [MediaPipe](https://github.com/google-ai-edge/mediapipe) canonical face model — 468 vertices of measured human face geometry | Apache License 2.0 |
|
||||
|
||||
---
|
||||
|
||||
## 🔒 Your Data
|
||||
|
||||
Everything stays on your machine. There is no MARK server, no telemetry and no account.
|
||||
|
||||
| What | Where | Notes |
|
||||
|---|---|---|
|
||||
| Gemini API key, plugin credentials | `config/api_keys.json` | **Plaintext.** Anyone with your user account can read it. Treat it like a password file. |
|
||||
| Dashboard TLS certificate + private key | `config/certs/` | Generated locally, self-signed, never leaves the machine. |
|
||||
| What the assistant remembers about you | `memory/long_term.json` | Delete the file to make it forget everything. |
|
||||
|
||||
All three are listed in `.gitignore`, so a fork or a pull request cannot leak them by accident. **If you have already committed `config/api_keys.json` anywhere public, revoke that key** at [aistudio.google.com](https://aistudio.google.com/app/apikey) and generate a new one — removing the file in a later commit does not remove it from the history.
|
||||
|
||||
Your voice is streamed to Google's Gemini Live API while a session is open; that is the one thing that leaves your computer, and it stops when you mute or close the app.
|
||||
|
||||
---
|
||||
|
||||
## ⚠️ License
|
||||
|
||||
Personal and non-commercial use only.
|
||||
|
||||
+50
-17
@@ -1,14 +1,24 @@
|
||||
# ── MARK LIV — Python dependencies ────────────────────────────────────────────
|
||||
# pip skips any line whose sys_platform marker does not match your OS, so this
|
||||
# one file serves Windows, macOS and Linux. `python setup.py` runs it for you.
|
||||
|
||||
# Version bounds: the lower bound is what the app is developed against, the
|
||||
# upper bound stops the next MAJOR release of a dependency from breaking every
|
||||
# fresh clone on the day it ships. Minor and patch updates flow in freely.
|
||||
|
||||
# ── Core: UI, audio, Gemini Live ──────────────────────────────────────────────
|
||||
PyQt6
|
||||
sounddevice
|
||||
numpy
|
||||
google-genai>=2.8.0
|
||||
# PyQt6 and numpy also carry the holographic avatar: the head is rendered in
|
||||
# software with QPainter and posed with numpy, so the face costs no extra
|
||||
# package, no OpenGL and no GPU. Nothing below was added for it.
|
||||
PyQt6>=6.6,<7
|
||||
sounddevice>=0.4,<1
|
||||
numpy>=1.24,<3
|
||||
google-genai>=2.8.0,<3
|
||||
|
||||
# ── Web search & browser control ──────────────────────────────────────────────
|
||||
requests
|
||||
beautifulsoup4
|
||||
duckduckgo-search
|
||||
playwright
|
||||
requests>=2.31,<3
|
||||
beautifulsoup4>=4.12,<5
|
||||
playwright>=1.40,<2
|
||||
|
||||
# ── Automation / input control ────────────────────────────────────────────────
|
||||
pyautogui
|
||||
@@ -16,23 +26,23 @@ pyperclip
|
||||
pygetwindow
|
||||
|
||||
# ── Vision & media ────────────────────────────────────────────────────────────
|
||||
pillow
|
||||
opencv-python
|
||||
mss
|
||||
pillow>=10,<13
|
||||
opencv-python>=4.8,<5
|
||||
mss>=9,<11
|
||||
|
||||
# ── System, files & documents ─────────────────────────────────────────────────
|
||||
psutil
|
||||
psutil>=5.9,<8
|
||||
send2trash
|
||||
youtube-transcript-api
|
||||
python-pptx
|
||||
openpyxl
|
||||
|
||||
# ── Remote dashboard (control from your phone) ────────────────────────────────
|
||||
fastapi
|
||||
uvicorn[standard]
|
||||
cryptography
|
||||
python-multipart
|
||||
qrcode[pil]
|
||||
fastapi>=0.110,<1
|
||||
uvicorn[standard]>=0.27,<1
|
||||
cryptography>=42,<50
|
||||
python-multipart>=0.0.9,<1
|
||||
qrcode[pil]>=7,<9
|
||||
|
||||
# ── Plugin extras: Gmail / Google Calendar (OAuth) ────────────────────────────
|
||||
google-api-python-client
|
||||
@@ -49,6 +59,29 @@ pywinauto; sys_platform == "win32"
|
||||
pywin32; sys_platform == "win32"
|
||||
wmi; sys_platform == "win32"
|
||||
|
||||
# ── Documents, media & hardware readouts ──────────────────────────────────────
|
||||
# Every one of these is imported inside a try/except at the point of use, so a
|
||||
# missing package disables exactly one feature and never stops the app. They are
|
||||
# declared because a feature that silently does nothing is worse than a slightly
|
||||
# longer install.
|
||||
ddgs>=9,<10 # web_search — the maintained successor to
|
||||
# duckduckgo-search, which is tried first and
|
||||
# falls back to the old name automatically
|
||||
pdfplumber # file_processor — PDF text and tables
|
||||
PyPDF2 # file_processor — PDF fallback reader
|
||||
python-docx # file_processor — .docx reading
|
||||
paho-mqtt # printer_control plugin — MQTT printers
|
||||
pynvml # system_monitor + the HUD's GPU meter (NVIDIA)
|
||||
|
||||
# ── Optional: heavy extras, install only if you want the feature ──────────────
|
||||
# Left out on purpose — together they are several hundred megabytes, which is a
|
||||
# lot to download for a fresh clone that may never use them.
|
||||
#
|
||||
# pip install pandas # file_processor: spreadsheet/CSV analysis
|
||||
# pip install pydub # file_processor: audio metadata and conversion
|
||||
# # (also needs ffmpeg on PATH for most formats)
|
||||
# pip install mediapipe # pushup_counter: elbow-angle rep detection
|
||||
|
||||
# ── Optional: local wake word ("Hey Jarvis") ──────────────────────────────────
|
||||
# Deliberately NOT installed by default — it's opt-in. Enable it (and download
|
||||
# the tiny model) with one click from ⚙ → WAKE WORD in the app, or install it
|
||||
|
||||
@@ -1,13 +1,16 @@
|
||||
"""
|
||||
MARK LIII — one-time setup.
|
||||
MARK LIV — one-time setup.
|
||||
|
||||
Installs the Python dependencies for THIS operating system only: the OS-specific
|
||||
packages in requirements.txt carry `sys_platform` markers, so a macOS or Linux
|
||||
user never pulls Windows-only libraries (and vice-versa). Then it fetches the
|
||||
Playwright browsers needed for web automation (current-OS builds only).
|
||||
|
||||
The optional local wake word ("Hey Jarvis") is NOT installed here — it's a
|
||||
one-click, opt-in download from ⚙ → WAKE WORD inside the app.
|
||||
Two things it deliberately does NOT install:
|
||||
* the optional local wake word ("Hey Jarvis") — one-click, opt-in, from
|
||||
⚙ → WAKE WORD inside the app;
|
||||
* anything for the avatar — the holographic head renders in software on the
|
||||
PyQt6 and numpy already listed here. No GPU, no OpenGL, no extra packages.
|
||||
"""
|
||||
import platform
|
||||
import subprocess
|
||||
@@ -15,6 +18,10 @@ import sys
|
||||
from pathlib import Path
|
||||
|
||||
OS = platform.system() # "Windows" | "Darwin" | "Linux"
|
||||
HERE = Path(__file__).resolve().parent
|
||||
|
||||
MIN_PY = (3, 11) # hard floor: below this the syntax used here won't parse
|
||||
MAX_PY = (3, 13) # highest version this is actually tested on
|
||||
|
||||
|
||||
def _run(label: str, args: list[str]) -> None:
|
||||
@@ -22,8 +29,47 @@ def _run(label: str, args: list[str]) -> None:
|
||||
subprocess.run(args, check=True)
|
||||
|
||||
|
||||
def _check_python() -> None:
|
||||
"""Fail immediately and clearly rather than deep inside a pip resolver.
|
||||
|
||||
A wrong interpreter is the single most common way this install goes sideways,
|
||||
and the error it produces on its own names a wheel, not the real problem.
|
||||
"""
|
||||
v = sys.version_info[:2]
|
||||
if v > MAX_PY:
|
||||
# Newer is a warning, not a wall. Turning away someone who installed
|
||||
# today's Python is a worse first impression than a version that
|
||||
# turns out to work fine, and if a wheel really is missing pip says
|
||||
# so plainly.
|
||||
print(f"\n⚠️ Python {v[0]}.{v[1]} is newer than the "
|
||||
f"{MAX_PY[0]}.{MAX_PY[1]} this is tested on. Continuing — if a "
|
||||
f"package has no wheel yet, install Python "
|
||||
f"{MAX_PY[0]}.{MAX_PY[1]} and run setup with that.")
|
||||
return
|
||||
if v < MIN_PY:
|
||||
print(f"\n❌ Python {v[0]}.{v[1]} detected — MARK LIV needs at "
|
||||
f"least Python {MIN_PY[0]}.{MIN_PY[1]}.")
|
||||
print(" Install a supported version and run setup with it, e.g.:")
|
||||
print(f" py -{MIN_PY[0]}.{MIN_PY[1]} setup.py (Windows)")
|
||||
print(f" python{MIN_PY[0]}.{MIN_PY[1]} setup.py (macOS / Linux)")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def _check_assets() -> None:
|
||||
"""The avatar's face is a shipped file; a truncated clone should say so."""
|
||||
face = HERE / "core" / "face_model.obj"
|
||||
if not face.exists() or face.stat().st_size < 4096:
|
||||
print(
|
||||
"\n⚠️ core/face_model.obj is missing or truncated — the avatar will "
|
||||
"fall back to the plain glowing core.\n"
|
||||
" Re-clone the repository, or fetch that one file again."
|
||||
)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
print(f"⚙ MARK LIII setup — detected OS: {OS or 'unknown'}")
|
||||
print(f"⚙ MARK LIV setup — detected OS: {OS or 'unknown'}, "
|
||||
f"Python {sys.version_info[0]}.{sys.version_info[1]}")
|
||||
_check_python()
|
||||
|
||||
# requirements.txt filters OS-specific extras by itself via pip markers.
|
||||
_run("Installing Python dependencies (OS-specific extras auto-filtered)…",
|
||||
@@ -31,8 +77,19 @@ def main() -> None:
|
||||
|
||||
# Chromium covers Chrome/Edge/Opera/Brave/Vivaldi; Firefox for Firefox.
|
||||
# (Safari automation additionally needs: python -m playwright install webkit)
|
||||
# Not fatal: these are a few hundred megabytes from a CDN that a corporate
|
||||
# network or a flaky connection can refuse, and everything except browser
|
||||
# automation works without them. Failing the whole install there would send
|
||||
# a user away from a working app.
|
||||
try:
|
||||
_run("Installing Playwright browsers (chromium + firefox)…",
|
||||
[sys.executable, "-m", "playwright", "install", "chromium", "firefox"])
|
||||
except (subprocess.CalledProcessError, FileNotFoundError) as e:
|
||||
print(f"\n⚠️ Playwright browsers were not installed ({e}).")
|
||||
print(" Everything except browser automation works. Retry later with:")
|
||||
print(f' {sys.executable} -m playwright install chromium firefox')
|
||||
|
||||
_check_assets()
|
||||
|
||||
# ── OS-specific post-install notes ────────────────────────────────────────
|
||||
if OS == "Windows":
|
||||
|
||||
Reference in new issue
Block a user