""" One place where the assistant's one-shot Gemini calls are made. WHY THIS EXISTS The live conversation runs on the Live API and is not what this file is about. Everything else — reading a screenshot, parsing a flight page, turning a request into a shell command, working out which WhatsApp button answers a call — was a separate `genai.Client(...)` built at the point of use, with the model name written inline. Sixteen files did it, twenty-six times, and not one of them set a timeout. That is not tidiness, it is three real faults: NO TIMEOUT. The SDK waits forever by default. `gemini-flash-latest` spent an afternoon returning 504 DEADLINE_EXCEEDED, and every one of those calls became an unbounded hang — measured at ten seconds of silence while a phone rang, and worse elsewhere, because nothing was there to give up. NO FALLBACK. One hardcoded alias meant that when that alias was unwell, the feature was simply gone. A ladder costs nothing when the first rung works and saves the feature when it does not. NO SINGLE PLACE TO CHANGE. A new model release meant editing sixteen files and hoping none were missed. THE LIVE MODEL DOES THIS WORK, AND IT LEADS THE LADDER Not the user's conversation — a separate, throwaway session per call, so nothing a plugin asks is ever heard by the person at the microphone. It leads because of quota. This is a voice assistant; the Live API is the dependency it already has, and it draws on a different pool from the text models. On the free tier it is the TEXT pool that runs out, and when it does every side call fails and the feature behind it dies with it. Putting Live first means ordinary use stops spending the pool that runs dry. The reply arrives through output_transcription, because these models refuse response_modalities=["TEXT"] with a 1007 — they only speak. That sounds fatal for structured output and is not: the transcription is the model's own text of what it said, and it returned "Mum ❤ click here for contact info", indented Python inside markdown fences, and src/utils/helpers_v2.py character for character. It cannot carry grounding metadata, so grounded web search stays on REST. THE LADDER, MEASURED Live, one throwaway session: connect 0.24s short structured JSON 1.7 - 2.8s 2300 characters of code 3.39s, not truncated three concurrent sessions all fine, 4.77s wall clock REST text models, same prompt, same afternoon: gemini-2.5-flash-lite 0.76s ...then 429, quota exhausted gemini-2.5-flash 0.80s ...then 429 gemini-flash-lite-latest 2.58s gemini-flash-latest 504, every time So REST is two to four times quicker while it lasts, and the whole point of the ladder is that it does not last. Pinned REST names sit behind Live; rolling `-latest` aliases sit behind those, because they were the ones having a bad day. Where the answer depends only on stable input, cache it and neither pool is touched twice: `plugins/_whatsapp_core.py` is the worked example — one request per WhatsApp language for the lifetime of the install. """ from __future__ import annotations import asyncio import json import sys import time import threading from pathlib import Path if getattr(sys, "frozen", False): _BASE = Path(sys.executable).parent else: _BASE = Path(__file__).resolve().parent.parent _KEY_FILE = _BASE / "config" / "api_keys.json" # Ladders, tried left to right. Change a model HERE and the whole app follows. FAST = "fast" # short classification, extraction, one-line decisions SMART = "smart" # reasoning, generation, long documents, images SEARCH = "search" # grounded search — REST only, see below # A rung that means "ask the Live model instead", through a short throwaway # session rather than the REST text API. # # WHY IT LEADS # This is a voice assistant: the Live API is the dependency it already has, # and it draws on a DIFFERENT quota pool from the text models. On the free # tier the text pool is the one that runs out — an afternoon of ordinary use # exhausts it, and when it does, every one of these side calls fails and the # feature behind it dies. The Live pool is untouched by that. # # WHAT IT COSTS, MEASURED # connect 0.24s # short structured JSON 1.7 - 2.8s (REST: 0.76s) # 2300 characters of code 3.39s, not truncated # three concurrent sessions all fine, 4.77s wall clock # So it is two to four times slower than REST when REST is available, and # infinitely faster than REST when REST is out of quota. # # THE THING WORTH KNOWING # These models only speak — response_modalities=["TEXT"] is refused with a # 1007. The reply comes back through output_transcription, which sounds like # it would mangle anything structured. It does not: it is the model's own # text of what it said, and it survived "Mum ❤ click here for contact info", # indented Python inside markdown fences, and src/utils/helpers_v2.py # character for character. That is what makes this usable at all. # # What it cannot carry is grounding metadata, so grounded web search stays # on REST — see SEARCH. LIVE = "live" _LADDERS = { FAST: (LIVE, "gemini-2.5-flash-lite", "gemini-2.5-flash"), SMART: (LIVE, "gemini-2.5-flash", "gemini-2.5-flash-lite"), # Grounded search needs response.candidates[...].grounding_metadata, which a # Live turn does not produce. REST only, and it says so rather than silently # returning an answer with no sources behind it. SEARCH: ("gemini-2.5-flash", "gemini-flash-latest", "gemini-2.5-flash-lite"), } # The Live model to use for one-shot calls. main.py owns the real one; this is # only the fallback for when this module is imported without it (tests). _LIVE_FALLBACK = "models/gemini-3.1-flash-live-preview" # How many one-shot Live sessions may exist at once. # # THE USER'S CONVERSATION OUTRANKS EVERY SIDE CALL. # Nothing here can reach the microphone or the speaker — main.py has exactly one # `.receive()` and it is bound to its own session, and audio only reaches the # speaker through that one loop — so a side call cannot answer the user or talk # over the reply. Verified alongside a live main session: four side calls fired # while it was connected, each got its own answer, and the main session replied # correctly both before and after with no errors. # # What a side call CAN do is take up a concurrent-session slot. That is the one # way it could hurt the conversation, so it is capped, and a call that cannot # get a slot quickly does not queue behind the others — it falls to the REST # rung, which is what the ladder is for. # Three, from the measurement: four side calls plus the conversation ran # together without complaint, so three leaves the conversation a slot in hand # while still covering any burst this app actually produces — tool calls run one # after another, and the screen agent's loop is sequential. _LIVE_SLOTS = threading.BoundedSemaphore(3) _LIVE_SLOT_WAIT = 3.0 _ONE_SHOT_SYSTEM = ( "You are a data-processing function, not an assistant and not in a " "conversation. There is no person listening to you. Produce exactly the " "output the request asks for and nothing else: no greeting, no " "acknowledgement, no 'Understood', no explanation, no closing remark, no " "restating of the question. If the request asks for JSON, emit only the " "JSON. If it asks for code, emit only the code. If it asks for one word, " "emit that one word. Preserve the exact spelling, punctuation, capitals " "and whitespace of anything you are asked to copy or return." ) # Milliseconds. Not a preference: the API rejects anything under ten seconds # with "Minimum allowed deadline is 10s", so this is the tightest bound it will # accept. Callers with a long job (a whole document, a big image) pass more. DEFAULT_TIMEOUT_MS = 10_000 MIN_TIMEOUT_MS = 10_000 _key_lock = threading.Lock() _cached_key: str | None = None # A rung that answered 429 is out of quota, and on the free tier it will stay # that way for a while. Retrying it on every single call is a wasted round trip # in front of every request the assistant makes — measured on this key, the # lite rung was 429ing continuously, so every call was paying for it before # reaching the model that could actually answer. Remembering that for a few # minutes turns the ladder from a cost into a saving. _COOLDOWN_SECONDS = 300 _cooldown: dict[str, float] = {} _cool_lock = threading.Lock() def _cool(model: str) -> None: with _cool_lock: _cooldown[model] = time.monotonic() + _COOLDOWN_SECONDS def _cooling(model: str) -> bool: with _cool_lock: until = _cooldown.get(model, 0.0) if until and time.monotonic() < until: return True _cooldown.pop(model, None) return False def api_key(refresh: bool = False) -> str: """The Gemini key from config/api_keys.json. Cached; never raises.""" global _cached_key with _key_lock: if _cached_key is not None and not refresh: return _cached_key try: data = json.loads(_KEY_FILE.read_text(encoding="utf-8")) _cached_key = str(data.get("gemini_api_key") or "") except Exception: _cached_key = "" return _cached_key def client(timeout_ms: int = DEFAULT_TIMEOUT_MS, key: str = ""): """A configured genai.Client with a deadline on it. Raises if there is no key, because a caller that cannot work without one should say so.""" from google import genai from google.genai import types as gtypes key = key or api_key() if not key: raise RuntimeError("no Gemini API key is configured") return genai.Client( api_key=key, http_options=gtypes.HttpOptions(timeout=max(MIN_TIMEOUT_MS, int(timeout_ms))), ) class _Reply: """What a Live turn hands back, shaped like the REST response's `.text` so every existing call site keeps working unchanged.""" __slots__ = ("text",) def __init__(self, text: str): self.text = text def _live_model() -> str: """Whatever main.py is running, so upgrading the assistant upgrades this.""" return getattr(sys.modules.get("main"), "LIVE_MODEL", None) or _LIVE_FALLBACK def _to_live_parts(contents) -> list: """REST `contents` -> Live `parts`. Accepts a bare string, a list of strings, and the SDK's Part objects (which is how every image is passed here), because those are the three shapes the call sites actually use.""" import base64 items = contents if isinstance(contents, (list, tuple)) else [contents] parts = [] for item in items: if isinstance(item, str): parts.append({"text": item}) continue blob = getattr(item, "inline_data", None) if blob is not None: data = getattr(blob, "data", None) mime = getattr(blob, "mime_type", None) or "application/octet-stream" if isinstance(data, bytes): data = base64.b64encode(data).decode("ascii") parts.append({"inline_data": {"mime_type": mime, "data": data}}) continue txt = getattr(item, "text", None) if txt: parts.append({"text": txt}) continue if isinstance(item, dict): parts.append(item) return parts async def _live_turn(parts: list, system: str, key: str, timeout_s: float) -> str: from google import genai from google.genai import types as gtypes cl = genai.Client(api_key=key, http_options={"api_version": "v1beta"}) # Silence the persona, or it answers instead of complying. # # These are conversational models and they behave like it: asked "Reply with # one word: ok" a Live turn came back with "Understood." — it treated the # instruction as something to acknowledge rather than something to do. The # REST models do not, because nobody ever taught them to be in a # conversation. Every call through this module wants a value, not a reply, # so the session is told what it is before it is told what to do. kwargs = { "response_modalities": ["AUDIO"], "output_audio_transcription": {}, "system_instruction": _ONE_SHOT_SYSTEM + (f"\n\n{system}" if system else ""), } cm = cl.aio.live.connect(model=_live_model(), config=gtypes.LiveConnectConfig(**kwargs)) session = await asyncio.wait_for(cm.__aenter__(), 30) try: await session.send_client_content( turns={"role": "user", "parts": parts}, turn_complete=True) chunks: list[str] = [] async def drain(): async for resp in session.receive(): sc = getattr(resp, "server_content", None) if sc and sc.output_transcription and sc.output_transcription.text: chunks.append(sc.output_transcription.text) await asyncio.wait_for(drain(), timeout=timeout_s) # The transcription can trail the audio turn by a beat; a short second # drain stops a reply being cut mid-token. chat_takeover learned this # the same way and for the same reason. try: await asyncio.wait_for(drain(), timeout=1.5) except asyncio.TimeoutError: pass return "".join(chunks).strip() finally: try: await cm.__aexit__(None, None, None) except Exception: pass def _live_call(contents, config, timeout_ms: int, key: str): """One throwaway Live session, run on its own loop in its own thread. A dedicated thread rather than asyncio.run() on the caller's: these are invoked from plugin executor threads, from UI worker threads and from main.py's own event loop, and asyncio.run() inside a thread that already has a running loop raises. Its own thread has no loop to collide with, wherever it was called from. Reconnecting costs 0.24s, so nothing is kept alive between calls — no session lifetime cap to manage, no GoAway to handle, no shared state. """ system = "" if config is not None: system = getattr(config, "system_instruction", None) or \ (config.get("system_instruction") if isinstance(config, dict) else "") or "" parts = _to_live_parts(contents) if not parts: return None if not _LIVE_SLOTS.acquire(timeout=_LIVE_SLOT_WAIT): # Every slot is busy. Do not wait it out: falling to REST costs less # than holding a session the user's conversation might want. raise RuntimeError("no free Live slot — leaving them for the conversation") box: dict = {} def runner(): try: box["text"] = asyncio.run( _live_turn(parts, str(system), key, max(10.0, timeout_ms / 1000.0))) except BaseException as e: # noqa: BLE001 box["error"] = e try: th = threading.Thread(target=runner, daemon=True, name="gemini-live-oneshot") th.start() th.join(timeout=max(15.0, timeout_ms / 1000.0 + 20.0)) finally: _LIVE_SLOTS.release() if "error" in box: raise box["error"] text = box.get("text") return _Reply(text) if text else None def call(contents, tier: str = FAST, config=None, timeout_ms: int = DEFAULT_TIMEOUT_MS, key: str = ""): """Run one generation, walking the ladder until one answers. Returns the SDK's own response object, so callers that need more than the text — grounding metadata, candidates, usage — still get it. Returns None when every model on the ladder failed; the reason for each is printed, since a silent None during a session nobody can debug is how the original problem stayed hidden. """ # `tier` is normally FAST or SMART. Anything else is taken to be an explicit # model name — screen_agent lets the user pick one in its settings — and it # is tried first, with the reasoning ladder behind it. So a user's choice is # honoured, and a user's choice that is having an outage still degrades to # something that answers instead of to nothing. ladder = _LADDERS.get(tier) if ladder is None: ladder = (tier,) + tuple(m for m in _LADDERS[SMART] if m != tier) resolved_key = key or api_key() if not resolved_key: print("[Gemini] no Gemini API key is configured") return None cl = None tried = [m for m in ladder if not _cooling(m)] or list(ladder) for model in tried: try: if model == LIVE: reply = _live_call(contents, config, timeout_ms, resolved_key) if reply is not None: return reply raise RuntimeError("the Live turn came back empty") if cl is None: cl = client(timeout_ms=timeout_ms, key=resolved_key) kwargs = {"model": model, "contents": contents} if config is not None: kwargs["config"] = config return cl.models.generate_content(**kwargs) except Exception as e: msg = str(e) if "429" in msg or "RESOURCE_EXHAUSTED" in msg: _cool(model) print(f"[Gemini] {model}: out of quota — skipping it for " f"{_COOLDOWN_SECONDS // 60} minutes") else: print(f"[Gemini] {model}: {type(e).__name__}: {msg[:140]}") return None def text(contents, tier: str = FAST, config=None, timeout_ms: int = DEFAULT_TIMEOUT_MS, key: str = "", default: str = "") -> str: """`call`, reduced to the reply text. `default` when nothing answered.""" resp = call(contents, tier=tier, config=config, timeout_ms=timeout_ms, key=key) if resp is None: return default return (getattr(resp, "text", None) or "").strip() or default def as_json(contents, tier: str = FAST, config=None, timeout_ms: int = DEFAULT_TIMEOUT_MS, key: str = "", default=None): """`text`, parsed as JSON, tolerating the fences and prose a model wraps it in. `default` when nothing answered or the answer would not parse.""" raw = text(contents, tier=tier, config=config, timeout_ms=timeout_ms, key=key) if not raw: return default if "{" in raw and "}" in raw: raw = raw[raw.find("{"): raw.rfind("}") + 1] elif "[" in raw and "]" in raw: raw = raw[raw.find("["): raw.rfind("]") + 1] try: return json.loads(raw) except Exception as e: print(f"[Gemini] reply was not JSON: {e}") return default