Press Remote Control in JARVIS to get a new QR code.
-Connecting to JARVIS…
-""") - - @app.post("/api/device-login") - async def device_login_ep(req: Request): - """Return a fresh auth token for a previously paired device token.""" - try: - body = await req.json() - except Exception: - return JSONResponse({"ok": False}, status_code=400) - dev_tok = (body.get("device_token") or "").strip() - if not dev_tok or dev_tok not in self._device_sessions: - return JSONResponse({"ok": False}, status_code=401) - session_key = self._device_sessions[dev_tok]["session_key"] - tok = secrets.token_urlsafe(32) - self._tokens.add(tok) - self._token_keys[tok] = session_key - self._aes_key(session_key) - if self._connect_callback: - self._connect_callback() - asyncio.create_task(self.broadcast( - {"type": "sys", "text": "Known device reconnected automatically."} - )) - return JSONResponse({"ok": True, "token": tok, "key": session_key}) - - @app.post("/api/revoke-devices") - async def revoke_devices(req: Request): - """Invalidate all persistent device tokens (admin action).""" - if not _auth(req): - return JSONResponse({"error": "Unauthorized"}, status_code=401) - count = len(self._device_sessions) - self._device_sessions.clear() - return JSONResponse({"ok": True, "revoked": count}) - - @app.post("/api/command") - async def command(req: Request): - if not _auth(req): - return JSONResponse({"error": "Unauthorized"}, status_code=401) - body = await req.json() - token = req.headers.get("authorization", "").removeprefix("Bearer ").strip() - enc = body.get("enc", "") - if enc: - text = self._decrypt(token, enc) - if text is None: - return JSONResponse({"error": "Decryption failed"}, status_code=400) - else: - text = (body.get("text") or "").strip() - if text: - await self._command_queue.put(text) - if self._wake_callback: - self._wake_callback() - return JSONResponse({"ok": True}) - - @app.post("/api/wake") - async def wake_ep(req: Request): - if not _auth(req): - return JSONResponse({"error": "Unauthorized"}, status_code=401) - if self._wake_callback: - self._wake_callback() - return JSONResponse({"ok": True}) - - # ── Phone mic real-time audio → Gemini Live ────────────────────────── - - @app.websocket("/ws/phone-audio") - async def phone_audio_ws(websocket: WebSocket, token: str = ""): - tok = token.strip() - if not tok or tok not in self._tokens: - await websocket.close(code=4001) - return - await websocket.accept() - asyncio.create_task(self.broadcast( - {"type": "sys", "text": "Phone microphone live."} - )) - try: - while True: - data = await websocket.receive_bytes() - try: - self._phone_audio_queue.put_nowait( - {"data": data, "mime_type": "audio/pcm"} - ) - except asyncio.QueueFull: - pass # drop frame rather than block - except WebSocketDisconnect: - pass - finally: - asyncio.create_task(self.broadcast( - {"type": "sys", "text": "Phone microphone stopped."} - )) - - # ── File sharing ────────────────────────────────────────────────────── - - def _safe_filename(raw: str) -> str: - name = Path(raw).name # strip path components - name = re.sub(r'[<>:"/\\|?*\x00-\x1f]', '_', name).strip(". ") - return name or "upload" - - if _UPLOAD_OK: - @app.post("/api/upload") - async def upload_file(req: Request, file: UploadFile = FastAPIFile(...)): - if not _auth(req): - return JSONResponse({"error": "Unauthorized"}, status_code=401) - - safe = _safe_filename(file.filename or "upload") - dest = self._uploads_dir / safe - stem, suffix = Path(safe).stem, Path(safe).suffix - counter = 1 - while dest.exists(): - dest = self._uploads_dir / f"{stem}_{counter}{suffix}" - counter += 1 - - size = 0 - max_bytes = MAX_UPLOAD_MB * 1024 * 1024 - try: - with open(dest, "wb") as fout: - while True: - chunk = await file.read(65536) - if not chunk: - break - size += len(chunk) - if size > max_bytes: - fout.close() - dest.unlink(missing_ok=True) - return JSONResponse( - {"error": f"File too large (max {MAX_UPLOAD_MB} MB)"}, - status_code=413, - ) - fout.write(chunk) - except Exception as exc: - try: - dest.unlink(missing_ok=True) - except Exception: - pass - return JSONResponse({"error": str(exc)}, status_code=500) - - asyncio.create_task(self.broadcast({ - "type": "file_received", - "name": dest.name, - "size": size, - "saved_to": str(self._uploads_dir), - })) - return JSONResponse({"ok": True, "name": dest.name, "size": size}) - else: - @app.post("/api/upload") - async def upload_unavailable(req: Request): - return JSONResponse( - {"error": "File uploads require: pip install python-multipart"}, - status_code=503, - ) - - @app.get("/api/files") - async def list_files(req: Request): - if not _auth(req): - return JSONResponse({"error": "Unauthorized"}, status_code=401) - files = [] - try: - for f in sorted( - (p for p in self._uploads_dir.iterdir() if p.is_file()), - key=lambda p: p.stat().st_mtime, - reverse=True, - ): - files.append({"name": f.name, "size": f.stat().st_size}) - except Exception: - pass - return JSONResponse({"files": files}) - - @app.get("/uploads/{filename}") - async def download_file(filename: str, token: str = ""): - # Auth via query param — browser can't send custom headers - tok = token.strip() - if not tok or tok not in self._tokens: - return JSONResponse({"error": "Unauthorized"}, status_code=401) - safe = re.sub(r'[/\\]', '', filename) - path = self._uploads_dir / safe - if not path.exists() or not path.is_file(): - return JSONResponse({"error": "Not found"}, status_code=404) - return FileResponse(str(path), filename=safe) - - @app.websocket("/ws") - async def ws_ep(websocket: WebSocket, token: str = ""): - tok = token.strip() - if not tok or tok not in self._tokens: - await websocket.close(code=4001) - return - await websocket.accept() - self._clients.add(websocket) - for entry in self._history[-50:]: - try: - await websocket.send_json(entry) - except Exception: - break - try: - while True: - data = await websocket.receive_json() - if data.get("type") == "command": - enc = data.get("enc", "") - t = self._decrypt(tok, enc) if enc else (data.get("text") or "").strip() - if t: - await self._command_queue.put(t) - if self._wake_callback: - self._wake_callback() - except WebSocketDisconnect: - pass - finally: - self._clients.discard(websocket) - - return app - - # ── serve ───────────────────────────────────────────────────────────── - - async def _serve_alias(self) -> None: - """Second HTTPS server on PORT+1 sharing the same app and in-memory state. - Chrome HTTPS-upgrades any bare IP:PORT the user types, so this port also needs TLS. - User types IP:8001 → Chrome tries https → self-signed cert warning → accept once → done.""" - ssl_key = BASE_DIR / "config" / "certs" / "jarvis.key" - ssl_cert = BASE_DIR / "config" / "certs" / "jarvis.crt" - asyncio.get_event_loop().run_in_executor(None, _ensure_network_access, PORT + 1) - cfg = uvicorn.Config( - self.app, host="0.0.0.0", port=PORT + 1, log_level="warning", - ssl_keyfile=str(ssl_key), ssl_certfile=str(ssl_cert), - ) - print(f"[Dashboard] Manual entry: {self._ip}:{PORT + 1} (type in browser, accept cert once)") - await uvicorn.Server(cfg).serve() - - async def serve(self) -> None: - if not _DEPS_OK: - print("[Dashboard] fastapi/uvicorn not installed — dashboard disabled.") - print("[Dashboard] Run: pip install fastapi 'uvicorn[standard]' cryptography") - return - - # Firewall setup runs in a thread — uvicorn starts immediately, - # no waiting for UAC dialogs or subprocess timeouts. - asyncio.get_event_loop().run_in_executor(None, _ensure_network_access, PORT) - - # Generate the TLS pair on first run so no private key ships in the repo. - _ensure_certs() - - use_ssl = self._ssl_enabled() - ssl_key = BASE_DIR / "config" / "certs" / "jarvis.key" - ssl_cert = BASE_DIR / "config" / "certs" / "jarvis.crt" - - if use_ssl: - asyncio.create_task(self._serve_alias()) - - cfg = uvicorn.Config( - self.app, host="0.0.0.0", port=PORT, log_level="warning", - **({"ssl_keyfile": str(ssl_key), "ssl_certfile": str(ssl_cert)} if use_ssl else {}), - ) - - proto = "https" if use_ssl else "http" - print(f"[Dashboard] {proto}://{self._ip}:{PORT}") - print("[Dashboard] Press 'Remote Control' in JARVIS UI to get the QR code.") - await uvicorn.Server(cfg).serve() +""" +dashboard/server.py — JARVIS Local HTTP Dashboard + +Plain HTTP on port 8000 (no SSL warnings, no firewall issues). +Security at the application layer: AES-256-CBC with session-key-derived key. +CryptoJS is auto-downloaded once and served locally — no CDN needed after that. + +Install deps: pip install fastapi "uvicorn[standard]" cryptography +""" + +import asyncio +import base64 +import hashlib +import re +import secrets +import socket +import string +import time +from pathlib import Path + +_DEPS_OK = False +try: + from fastapi import FastAPI, WebSocket, WebSocketDisconnect, Request + from fastapi.responses import HTMLResponse, JSONResponse, FileResponse + import uvicorn + _DEPS_OK = True +except ImportError: + pass + +# python-multipart is required for file uploads — optional dependency +_UPLOAD_OK = False +try: + from fastapi import UploadFile, File as FastAPIFile + _UPLOAD_OK = True +except Exception: + pass + +BASE_DIR = Path(__file__).resolve().parent.parent +STATIC_DIR = Path(__file__).parent / "static" +PORT = 8000 +MAX_UPLOAD_MB = 500 + + +def _make_uploads_dir() -> Path: + """Return (and create) the cross-platform uploads folder.""" + for candidate in [ + Path.home() / "Downloads" / "JARVIS Uploads", + Path.home() / "Documents" / "JARVIS Uploads", + BASE_DIR / "uploads", + ]: + try: + candidate.mkdir(parents=True, exist_ok=True) + return candidate + except Exception: + pass + return BASE_DIR / "uploads" + + +UPLOADS_DIR = _make_uploads_dir() + +def _get_gemini_key() -> str | None: + try: + import json as _json + with open(BASE_DIR / "config" / "api_keys.json", "r", encoding="utf-8") as f: + return _json.load(f).get("gemini_api_key") + except Exception: + return None + +_KEY_CHARS = [c for c in (string.ascii_uppercase + string.digits) + if c not in ('O', 'I', 'L', '0', '1')] + +# ── AES-256-CBC ─────────────────────────────────────────────────────────────── +_AES_SALT = b'JARVIS-DASHBOARD-v1' + + +def _derive_key(session_key: str) -> bytes: + """SHA-256(sessionKey‖salt) → 32-byte AES-256 key (microseconds, no PBKDF2 needed).""" + return hashlib.sha256(session_key.encode('utf-8') + _AES_SALT).digest() + + +def _decrypt_cbc(aes_key: bytes, enc_b64: str) -> str: + """Decrypt base64(IV[16] ‖ ciphertext) with AES-256-CBC + PKCS7.""" + from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes + from cryptography.hazmat.primitives import padding as sym_pad + raw = base64.b64decode(enc_b64) + iv, ct = raw[:16], raw[16:] + dec = Cipher(algorithms.AES(aes_key), modes.CBC(iv)).decryptor() + padded = dec.update(ct) + dec.finalize() + unpadder = sym_pad.PKCS7(128).unpadder() + return (unpadder.update(padded) + unpadder.finalize()).decode('utf-8') + + +# ── CryptoJS (auto-download once, served locally) ───────────────────────────── +_CRYPTOJS_CDN = ("https://cdnjs.cloudflare.com/ajax/libs/" + "crypto-js/4.2.0/crypto-js.min.js") +_CRYPTOJS_FILE = STATIC_DIR / "crypto-js.min.js" + + +def _ensure_network_access(port: int) -> None: + """Cross-platform, best-effort: open port in the OS firewall for LAN access. + + Runs in a background thread — never blocks uvicorn startup. + + Windows : writes a .bat file, runs it elevated via Windows ShellExecuteW + (native UAC dialog, guaranteed to appear). One-time setup. + macOS : osascript admin dialog if the Application Firewall is on. + Linux : pkexec GUI → sudo -n → prints manual command as fallback. + """ + import sys, subprocess, os, tempfile, threading + + # ── Windows ────────────────────────────────────────────────────────────── + if sys.platform == "win32": + import ctypes, time + + port_rule = f"JARVIS Dashboard Port {port}" + prog_rule = "JARVIS Dashboard Python" + py_exe = sys.executable + + def _netsh_rule_exists(name: str) -> bool: + try: + r = subprocess.run( + ["netsh", "advfirewall", "firewall", "show", "rule", f"name={name}"], + capture_output=True, text=True, timeout=5, + ) + return r.returncode == 0 and "No rules match" not in r.stdout + except Exception: + return False + + def _network_is_public() -> bool: + try: + r = subprocess.run( + ["powershell", "-NoProfile", "-NonInteractive", "-Command", + "(Get-NetConnectionProfile | " + "Where-Object {$_.NetworkCategory -eq 'Public'} | " + "Measure-Object).Count"], + capture_output=True, text=True, timeout=6, + ) + return r.stdout.strip() not in ("", "0") + except Exception: + return False + + need_port = not _netsh_rule_exists(port_rule) + need_prog = not _netsh_rule_exists(prog_rule) + need_private = _network_is_public() + + if not need_port and not need_prog and not need_private: + return # already fully configured + + # Build a .bat file — netsh + powershell, runs fast when elevated + bat_lines = ["@echo off"] + if need_private: + bat_lines.append( + 'powershell -NoProfile -NonInteractive -Command "' + 'Get-NetConnectionProfile | ' + "Where-Object {$_.NetworkCategory -eq 'Public'} | " + 'Set-NetConnectionProfile -NetworkCategory Private"' + ) + if need_port: + bat_lines.append( + f'netsh advfirewall firewall add rule ' + f'name="{port_rule}" protocol=TCP dir=in ' + f'localport={port} action=allow' + ) + if need_prog: + bat_lines.append( + f'netsh advfirewall firewall add rule ' + f'name="{prog_rule}" dir=in action=allow ' + f'program="{py_exe}" enable=yes' + ) + + bat_body = "\r\n".join(bat_lines) + "\r\n" + fd, bat_path = tempfile.mkstemp(suffix=".bat", prefix="jarvis_fw_") + try: + os.write(fd, bat_body.encode("mbcs")) # Windows cmd.exe expects ANSI + os.close(fd) + except Exception: + try: + os.close(fd) + except Exception: + pass + return + + # ── Try running directly (succeeds when already admin) ──────────────── + try: + r = subprocess.run( + [bat_path], capture_output=True, timeout=8, shell=True + ) + if r.returncode == 0: + print(f"[Dashboard] Firewall configured for port {port}.") + try: + os.unlink(bat_path) + except Exception: + pass + return + except Exception: + pass + + # ── ShellExecuteW: native UAC elevation (most reliable on Windows) ──── + # ShellExecuteW with verb "runas" always shows the UAC dialog regardless + # of UAC level settings. Non-blocking — uvicorn is already running. + print("[Dashboard] One-time network setup required.") + print("[Dashboard] >>> A Windows security dialog will appear — click 'Yes' <<<") + try: + ret = ctypes.windll.shell32.ShellExecuteW( + None, # hwnd (no parent window) + "runas", # verb (request elevation) + bat_path, # file (our .bat) + None, # params + None, # working dir + 0, # SW_HIDE (run without a visible cmd window) + ) + if int(ret) > 32: + # ShellExecuteW returns immediately; bat finishes in ~1 second. + # Sleep briefly so the rules are in place before the first retry. + time.sleep(2) + print(f"[Dashboard] Network setup complete — port {port} is open.") + print("[Dashboard] Refresh your phone browser to connect.") + else: + print("[Dashboard] Setup was not allowed.") + print("[Dashboard] Phone connections may fail until JARVIS is run as Administrator.") + except Exception as e: + print(f"[Dashboard] Firewall setup error: {e}") + finally: + # Cleanup after the bat has had time to run + def _cleanup(path: str) -> None: + time.sleep(5) + try: + os.unlink(path) + except Exception: + pass + threading.Thread(target=_cleanup, args=(bat_path,), daemon=True).start() + return + + # ── macOS ───────────────────────────────────────────────────────────────── + if sys.platform == "darwin": + fw_ctl = "/usr/libexec/ApplicationFirewall/socketfilterfw" + try: + r = subprocess.run( + [fw_ctl, "--getglobalstate"], capture_output=True, text=True, timeout=5, + ) + if "disabled" in r.stdout.lower(): + return # firewall off — nothing to do + + py = sys.executable + listed = subprocess.run( + [fw_ctl, "--listapps"], capture_output=True, text=True, timeout=5, + ) + if py in listed.stdout: + return # already allowed + + print("[Dashboard] One-time network setup — enter your password in the macOS dialog.") + subprocess.run( + ["osascript", "-e", + f'do shell script "{fw_ctl} --add {py} && {fw_ctl} --unblockapp {py}"' + f' with administrator privileges'], + timeout=60, + ) + except Exception: + pass # macOS firewall is off by default — silent failure is fine + return + + # ── Linux ───────────────────────────────────────────────────────────────── + def _privileged(cmd: list[str]) -> bool: + for prefix in (["pkexec"], ["sudo", "-n"]): + try: + r = subprocess.run(prefix + cmd, capture_output=True, timeout=30) + if r.returncode == 0: + return True + except Exception: + pass + return False + + try: # ufw + r = subprocess.run(["ufw", "status"], capture_output=True, text=True, timeout=5) + if "active" in r.stdout.lower(): + if _privileged(["ufw", "allow", f"{port}/tcp"]): + print(f"[Dashboard] ufw: port {port} allowed.") + else: + print(f"[Dashboard] Run manually: sudo ufw allow {port}/tcp") + return + except FileNotFoundError: + pass + + try: # firewalld + r = subprocess.run( + ["firewall-cmd", "--state"], capture_output=True, text=True, timeout=5, + ) + if "running" in r.stdout.lower(): + ok = (_privileged(["firewall-cmd", "--add-port", f"{port}/tcp", "--permanent"]) + and _privileged(["firewall-cmd", "--reload"])) + if ok: + print(f"[Dashboard] firewalld: port {port} allowed.") + else: + print(f"[Dashboard] Run manually: sudo firewall-cmd --add-port={port}/tcp --permanent && sudo firewall-cmd --reload") + return + except FileNotFoundError: + pass + + try: # iptables (not persistent but works until reboot) + r = subprocess.run(["iptables", "-L", "INPUT", "-n"], capture_output=True, timeout=5) + if r.returncode == 0: + if _privileged(["iptables", "-A", "INPUT", "-p", "tcp", "--dport", str(port), "-j", "ACCEPT"]): + print(f"[Dashboard] iptables: port {port} opened.") + else: + print(f"[Dashboard] Run manually: sudo iptables -A INPUT -p tcp --dport {port} -j ACCEPT") + except FileNotFoundError: + pass # no iptables means firewall is probably off — nothing to do + + +def _ensure_crypto_js() -> None: + if _CRYPTOJS_FILE.exists(): + return + try: + import urllib.request + print("[Dashboard] Downloading CryptoJS (one-time setup)…") + urllib.request.urlretrieve(_CRYPTOJS_CDN, str(_CRYPTOJS_FILE)) + print("[Dashboard] CryptoJS cached — will serve locally from now on.") + except Exception as e: + print(f"[Dashboard] CryptoJS download failed: {e}") + print(f"[Dashboard] Encryption will fall back to CDN load on client.") + + +_ensure_crypto_js() + + +# ── helpers ─────────────────────────────────────────────────────────────────── + +def _local_ip() -> str: + """Return the best LAN-facing IPv4 address, no internet required.""" + # Method 1: route trick (fast, works when internet is available) + for probe in ("8.8.8.8", "1.1.1.1", "192.168.1.1"): + try: + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + s.settimeout(0.5) + s.connect((probe, 80)) + ip = s.getsockname()[0] + s.close() + if not ip.startswith("127."): + return ip + except Exception: + pass + + # Method 2: hostname resolution (works offline on most systems) + try: + ip = socket.gethostbyname(socket.gethostname()) + if not ip.startswith("127."): + return ip + except Exception: + pass + + # Method 3: enumerate all interfaces (fully offline, no external deps) + try: + for info in socket.getaddrinfo(socket.gethostname(), None, socket.AF_INET): + ip = info[4][0] + if not ip.startswith("127.") and not ip.startswith("169.254."): + return ip + except Exception: + pass + + return "127.0.0.1" + + +def _read(name: str) -> str: + return (STATIC_DIR / name).read_text(encoding="utf-8") + + +# ── DashboardServer ─────────────────────────────────────────────────────────── + +class DashboardServer: + + def __init__(self): + self._ip = _local_ip() + self._tokens: set[str] = set() + self._token_keys: dict[str, str] = {} # auth_token → session_key + self._aes_cache: dict[str, bytes]= {} # session_key → AES bytes + self._clients: set[WebSocket] = set() + self._history: list[dict] = [] + self._command_queue = asyncio.Queue() + self._wake_callback = None + self._connect_callback = None + self._pending_keys: dict[str, float] = {} + self._device_sessions: dict[str, dict] = {} # device_token → {session_key} + self._phone_audio_queue: asyncio.Queue = asyncio.Queue(maxsize=200) + self._uploads_dir = UPLOADS_DIR + self._login_html = _read("login.html") + self._app_html = _read("app.html") + self.app = self._build_app() + + # ── one-time key management ─────────────────────────────────────────── + + def new_key(self, expiry_secs: int = 600) -> str: + now = time.time() + self._pending_keys = {k: v for k, v in self._pending_keys.items() if v > now} + key = ''.join(secrets.choice(_KEY_CHARS) for _ in range(6)) + self._pending_keys[key] = now + expiry_secs + return key + + @staticmethod + def _ssl_enabled() -> bool: + certs = BASE_DIR / "config" / "certs" + return (certs / "jarvis.key").exists() and (certs / "jarvis.crt").exists() + + def get_url(self) -> str: + proto = "https" if self._ssl_enabled() else "http" + return f"{proto}://{self._ip}:{PORT}" + + def get_manual_url(self) -> str: + """URL for manual browser entry. When HTTPS active, points to alias port (also HTTPS).""" + if self._ssl_enabled(): + return f"{self._ip}:{PORT + 1}" + return f"{self._ip}:{PORT}" + + def _aes_key(self, session_key: str) -> bytes: + if session_key not in self._aes_cache: + self._aes_cache[session_key] = _derive_key(session_key) + return self._aes_cache[session_key] + + def _decrypt(self, token: str, enc_b64: str) -> str | None: + sk = self._token_keys.get(token) + if not sk: + return None + try: + return _decrypt_cbc(self._aes_key(sk), enc_b64) + except Exception: + return None + + # ── callbacks ──────────────────────────────────────────────────────── + + def set_wake_callback(self, fn) -> None: + self._wake_callback = fn + + def set_connect_callback(self, fn) -> None: + self._connect_callback = fn + + # ── broadcast ──────────────────────────────────────────────────────── + + async def broadcast(self, msg: dict) -> None: + self._history.append(msg) + if len(self._history) > 300: + self._history = self._history[-300:] + dead: set[WebSocket] = set() + for ws in list(self._clients): + try: + await ws.send_json(msg) + except Exception: + dead.add(ws) + self._clients -= dead + + # ── FastAPI app ─────────────────────────────────────────────────────── + + def _build_app(self) -> "FastAPI": + app = FastAPI(docs_url=None, redoc_url=None) + + def _auth(req: Request) -> bool: + tok = req.headers.get("authorization", "").removeprefix("Bearer ").strip() + return bool(tok) and tok in self._tokens + + # serve CryptoJS from local cache, fallback to CDN redirect + @app.get("/static/crypto.js") + async def serve_crypto(): + if _CRYPTOJS_FILE.exists(): + return FileResponse(str(_CRYPTOJS_FILE), + media_type="application/javascript") + from fastapi.responses import RedirectResponse + return RedirectResponse(_CRYPTOJS_CDN) + + @app.get("/login", response_class=HTMLResponse) + async def login_page(): + return HTMLResponse(self._login_html) + + @app.get("/", response_class=HTMLResponse) + async def index(): + # Auth is handled client-side via sessionStorage bearer token. + # Server-side header auth can't work here because browser navigations + # don't send custom headers (location.href doesn't carry Authorization). + html = (self._app_html + .replace("__IP__", self._ip) + .replace("__PORT__", str(PORT))) + return HTMLResponse(html) + + @app.post("/login") + async def login(req: Request): + body = await req.json() + entered = str(body.get("pin", "")).strip().upper() + now = time.time() + if entered in self._pending_keys and self._pending_keys[entered] > now: + del self._pending_keys[entered] # one-time use + tok = secrets.token_urlsafe(32) + self._tokens.add(tok) + self._token_keys[tok] = entered + self._aes_key(entered) # pre-derive & cache + if self._connect_callback: + self._connect_callback() + asyncio.create_task(self.broadcast( + {"type": "sys", "text": "Remote connection established."} + )) + # Bearer token in response body — no cookies needed (works on any browser/HTTP) + return JSONResponse({"ok": True, "token": tok}) + return JSONResponse({"ok": False, "error": "Invalid or expired key"}, + status_code=401) + + @app.get("/auto-login") + async def auto_login(key: str = ""): + """QR code target — validates one-time key, creates session, redirects phone.""" + now = time.time() + if not key or key not in self._pending_keys or self._pending_keys[key] <= now: + return HTMLResponse(""" + + +Press Remote Control in JARVIS to get a new QR code.
+Connecting to JARVIS…
+""") + + @app.post("/api/device-login") + async def device_login_ep(req: Request): + """Return a fresh auth token for a previously paired device token.""" + try: + body = await req.json() + except Exception: + return JSONResponse({"ok": False}, status_code=400) + dev_tok = (body.get("device_token") or "").strip() + if not dev_tok or dev_tok not in self._device_sessions: + return JSONResponse({"ok": False}, status_code=401) + session_key = self._device_sessions[dev_tok]["session_key"] + tok = secrets.token_urlsafe(32) + self._tokens.add(tok) + self._token_keys[tok] = session_key + self._aes_key(session_key) + if self._connect_callback: + self._connect_callback() + asyncio.create_task(self.broadcast( + {"type": "sys", "text": "Known device reconnected automatically."} + )) + return JSONResponse({"ok": True, "token": tok, "key": session_key}) + + @app.post("/api/revoke-devices") + async def revoke_devices(req: Request): + """Invalidate all persistent device tokens (admin action).""" + if not _auth(req): + return JSONResponse({"error": "Unauthorized"}, status_code=401) + count = len(self._device_sessions) + self._device_sessions.clear() + return JSONResponse({"ok": True, "revoked": count}) + + @app.post("/api/command") + async def command(req: Request): + if not _auth(req): + return JSONResponse({"error": "Unauthorized"}, status_code=401) + body = await req.json() + token = req.headers.get("authorization", "").removeprefix("Bearer ").strip() + enc = body.get("enc", "") + if enc: + text = self._decrypt(token, enc) + if text is None: + return JSONResponse({"error": "Decryption failed"}, status_code=400) + else: + text = (body.get("text") or "").strip() + if text: + await self._command_queue.put(text) + if self._wake_callback: + self._wake_callback() + return JSONResponse({"ok": True}) + + @app.post("/api/wake") + async def wake_ep(req: Request): + if not _auth(req): + return JSONResponse({"error": "Unauthorized"}, status_code=401) + if self._wake_callback: + self._wake_callback() + return JSONResponse({"ok": True}) + + # ── Phone mic real-time audio → Gemini Live ────────────────────────── + + @app.websocket("/ws/phone-audio") + async def phone_audio_ws(websocket: WebSocket, token: str = ""): + tok = token.strip() + if not tok or tok not in self._tokens: + await websocket.close(code=4001) + return + await websocket.accept() + asyncio.create_task(self.broadcast( + {"type": "sys", "text": "Phone microphone live."} + )) + try: + while True: + data = await websocket.receive_bytes() + try: + self._phone_audio_queue.put_nowait( + {"data": data, "mime_type": "audio/pcm"} + ) + except asyncio.QueueFull: + pass # drop frame rather than block + except WebSocketDisconnect: + pass + finally: + asyncio.create_task(self.broadcast( + {"type": "sys", "text": "Phone microphone stopped."} + )) + + # ── File sharing ────────────────────────────────────────────────────── + + def _safe_filename(raw: str) -> str: + name = Path(raw).name # strip path components + name = re.sub(r'[<>:"/\\|?*\x00-\x1f]', '_', name).strip(". ") + return name or "upload" + + if _UPLOAD_OK: + @app.post("/api/upload") + async def upload_file(req: Request, file: UploadFile = FastAPIFile(...)): + if not _auth(req): + return JSONResponse({"error": "Unauthorized"}, status_code=401) + + safe = _safe_filename(file.filename or "upload") + dest = self._uploads_dir / safe + stem, suffix = Path(safe).stem, Path(safe).suffix + counter = 1 + while dest.exists(): + dest = self._uploads_dir / f"{stem}_{counter}{suffix}" + counter += 1 + + size = 0 + max_bytes = MAX_UPLOAD_MB * 1024 * 1024 + try: + with open(dest, "wb") as fout: + while True: + chunk = await file.read(65536) + if not chunk: + break + size += len(chunk) + if size > max_bytes: + fout.close() + dest.unlink(missing_ok=True) + return JSONResponse( + {"error": f"File too large (max {MAX_UPLOAD_MB} MB)"}, + status_code=413, + ) + fout.write(chunk) + except Exception as exc: + try: + dest.unlink(missing_ok=True) + except Exception: + pass + return JSONResponse({"error": str(exc)}, status_code=500) + + asyncio.create_task(self.broadcast({ + "type": "file_received", + "name": dest.name, + "size": size, + "saved_to": str(self._uploads_dir), + })) + return JSONResponse({"ok": True, "name": dest.name, "size": size}) + else: + @app.post("/api/upload") + async def upload_unavailable(req: Request): + return JSONResponse( + {"error": "File uploads require: pip install python-multipart"}, + status_code=503, + ) + + @app.get("/api/files") + async def list_files(req: Request): + if not _auth(req): + return JSONResponse({"error": "Unauthorized"}, status_code=401) + files = [] + try: + for f in sorted( + (p for p in self._uploads_dir.iterdir() if p.is_file()), + key=lambda p: p.stat().st_mtime, + reverse=True, + ): + files.append({"name": f.name, "size": f.stat().st_size}) + except Exception: + pass + return JSONResponse({"files": files}) + + @app.get("/uploads/{filename}") + async def download_file(filename: str, token: str = ""): + # Auth via query param — browser can't send custom headers + tok = token.strip() + if not tok or tok not in self._tokens: + return JSONResponse({"error": "Unauthorized"}, status_code=401) + safe = re.sub(r'[/\\]', '', filename) + path = self._uploads_dir / safe + if not path.exists() or not path.is_file(): + return JSONResponse({"error": "Not found"}, status_code=404) + return FileResponse(str(path), filename=safe) + + @app.websocket("/ws") + async def ws_ep(websocket: WebSocket, token: str = ""): + tok = token.strip() + if not tok or tok not in self._tokens: + await websocket.close(code=4001) + return + await websocket.accept() + self._clients.add(websocket) + for entry in self._history[-50:]: + try: + await websocket.send_json(entry) + except Exception: + break + try: + while True: + data = await websocket.receive_json() + if data.get("type") == "command": + enc = data.get("enc", "") + t = self._decrypt(tok, enc) if enc else (data.get("text") or "").strip() + if t: + await self._command_queue.put(t) + if self._wake_callback: + self._wake_callback() + except WebSocketDisconnect: + pass + finally: + self._clients.discard(websocket) + + return app + + # ── serve ───────────────────────────────────────────────────────────── + + async def _serve_alias(self) -> None: + """Second HTTPS server on PORT+1 sharing the same app and in-memory state. + Chrome HTTPS-upgrades any bare IP:PORT the user types, so this port also needs TLS. + User types IP:8001 → Chrome tries https → self-signed cert warning → accept once → done.""" + ssl_key = BASE_DIR / "config" / "certs" / "jarvis.key" + ssl_cert = BASE_DIR / "config" / "certs" / "jarvis.crt" + asyncio.get_event_loop().run_in_executor(None, _ensure_network_access, PORT + 1) + cfg = uvicorn.Config( + self.app, host="0.0.0.0", port=PORT + 1, log_level="warning", + ssl_keyfile=str(ssl_key), ssl_certfile=str(ssl_cert), + ) + print(f"[Dashboard] Manual entry: {self._ip}:{PORT + 1} (type in browser, accept cert once)") + await uvicorn.Server(cfg).serve() + + async def serve(self) -> None: + if not _DEPS_OK: + print("[Dashboard] fastapi/uvicorn not installed — dashboard disabled.") + print("[Dashboard] Run: pip install fastapi 'uvicorn[standard]' cryptography") + return + + # Firewall setup runs in a thread — uvicorn starts immediately, + # no waiting for UAC dialogs or subprocess timeouts. + asyncio.get_event_loop().run_in_executor(None, _ensure_network_access, PORT) + + use_ssl = self._ssl_enabled() + ssl_key = BASE_DIR / "config" / "certs" / "jarvis.key" + ssl_cert = BASE_DIR / "config" / "certs" / "jarvis.crt" + + if use_ssl: + asyncio.create_task(self._serve_alias()) + + cfg = uvicorn.Config( + self.app, host="0.0.0.0", port=PORT, log_level="warning", + **({"ssl_keyfile": str(ssl_key), "ssl_certfile": str(ssl_cert)} if use_ssl else {}), + ) + + proto = "https" if use_ssl else "http" + print(f"[Dashboard] {proto}://{self._ip}:{PORT}") + print("[Dashboard] Press 'Remote Control' in JARVIS UI to get the QR code.") + await uvicorn.Server(cfg).serve() diff --git a/main.py b/main.py index 6a09363..32b58b1 100644 --- a/main.py +++ b/main.py @@ -1,22 +1,6 @@ import platform as _platform import subprocess as _subprocess -# ── Console encoding ───────────────────────────────────────────────────────── -# Windows consoles default to a legacy codepage — cp1254 in Turkey, cp1251 in -# Russia, cp932 in Japan. Printing an emoji there raises UnicodeEncodeError, and -# several of these prints sit inside except handlers, so the handler itself dies -# and skips the recovery code after it. Reconfiguring to UTF-8 with a -# replacement fallback costs nothing and makes the app behave in every locale. -import sys as _sys - -for _stream in ("stdout", "stderr"): - try: - _s = getattr(_sys, _stream, None) - if _s is not None and hasattr(_s, "reconfigure"): - _s.reconfigure(encoding="utf-8", errors="replace") - except Exception: - pass # pythonw / redirected pipes / anything exotic — never fatal - # ── Nuclear: force CREATE_NO_WINDOW on EVERY subprocess call on Windows ─────── # This patches Popen itself, so no per-file flag is needed anywhere. if _platform.system() == "Windows": @@ -32,6 +16,24 @@ if _platform.system() == "Windows": # ───────────────────────────────────────────────────────────────────────────── +# ── Console encoding ───────────────────────────────────────────────────────── +# Status lines in this app carry emoji and arrows ("📤 file_controller → Moved: +# a.txt → Documents/"). On a non-UTF-8 console — cp1254 on a Turkish Windows, +# cp1251 on a Russian one, cp932 on a Japanese one — printing one of those +# raises UnicodeEncodeError, and because the print sits after the tool's own +# try/except, the exception escapes into the receive loop and takes the session +# down. The assistant dies on a log line. +# +# Reconfiguring costs nothing and makes the app start the same way in every +# locale. `errors="replace"` is the belt and braces — a console that genuinely +# cannot render a glyph shows a box instead of killing the process. +import sys as _sys +for _stream in (_sys.stdout, _sys.stderr): + try: + _stream.reconfigure(encoding="utf-8", errors="replace") + except Exception: + pass + import asyncio import re import threading @@ -53,23 +55,13 @@ from memory.memory_manager import ( search_memory, set_trim_notifier, ) -from actions.file_processor import file_processor -from actions.flight_finder import flight_finder -from actions.open_app import open_app -from actions.weather_report import weather_action -from actions.send_message import send_message -from actions.reminder import reminder -from actions.computer_settings import computer_settings +# The file-backed tools (open_app, web_search, browser_control, …) are no longer +# imported or declared here — they self-describe via a TOOL dict in their own +# actions/*.py file and are auto-discovered by core.action_loader at startup. +# Only tools that are tied to live-session state stay inline in this file +# (screen_process, close_camera, save_memory, manage_monitor, shutdown_jarvis, +# system_status). from actions.screen_processor import _capture_camera, _capture_screen -from actions.youtube_video import youtube_video -from actions.desktop import desktop_control -from actions.browser_control import browser_control -from actions.file_controller import file_controller -from actions.code_helper import code_helper -from actions.dev_agent import dev_agent -from actions.web_search import web_search as web_search_action -from actions.computer_control import computer_control -from actions.game_updater import game_updater from actions.system_monitor import SystemMonitor, get_system_status from actions.proactive import ProactiveEngine from actions.background_monitor import ( @@ -77,12 +69,20 @@ from actions.background_monitor import ( ) from actions.web_search import _news as _fetch_news_sync from memory.config_manager import ( - get_brief_enabled, get_voice, get_input_device, get_output_device, + get_brief_enabled, get_voice, get_wake_word_enabled, save_wake_word_enabled, get_input_device, get_output_device, ) from core.plugin_loader import discover_plugins from core import undo as undo_stack from core import confirm as confirm_gate from core import audio_devices +from core.action_loader import discover_actions +from core.wake_word import ( + WakeWordDetector, is_ready as wake_is_ready, install_and_download as wake_install, +) + +# How long the assistant stays awake with no user speech before it auto-sleeps +# again (wake-word mode only). +WAKE_SLEEP_TIMEOUT = 120.0 # seconds (2 minutes) def get_base_dir(): if getattr(sys, "frozen", False): @@ -92,9 +92,9 @@ def get_base_dir(): BASE_DIR = get_base_dir() API_CONFIG_PATH = BASE_DIR / "config" / "api_keys.json" PROMPT_PATH = BASE_DIR / "core" / "prompt.txt" -LIVE_MODEL = "models/gemini-2.5-flash-native-audio-preview-12-2025" +LIVE_MODEL = "models/gemini-3.1-flash-live-preview" CHANNELS = 1 -SEND_SAMPLE_RATE = 16000 +SEND_SAMPLE_RATE = 16000 RECEIVE_SAMPLE_RATE = 24000 CHUNK_SIZE = 1024 @@ -143,44 +143,12 @@ def _clean_transcript(text: str) -> str: return text.strip() TOOL_DECLARATIONS = [ - { - "name": "open_app", - "description": ( - "Opens any application on the computer. " - "Use this whenever the user asks to open, launch, or start any app, " - "website, or program. Always call this tool — never just say you opened it." - ), - "parameters": { - "type": "OBJECT", - "properties": { - "app_name": { - "type": "STRING", - "description": "Exact name of the application (e.g. 'WhatsApp', 'Chrome', 'Spotify')" - } - }, - "required": ["app_name"] - } - }, - { - "name": "web_search", - "description": ( - "Searches the web. Use for ANY question about current facts, events, prices, " - "or topics — always prefer this over guessing. " - "Modes: 'search' (default), 'news' (latest headlines on a topic), " - "'research' (deep comprehensive answer), 'price' (product cost lookup), " - "'compare' (side-by-side comparison of items)." - ), - "parameters": { - "type": "OBJECT", - "properties": { - "query": {"type": "STRING", "description": "Search query or topic"}, - "mode": {"type": "STRING", "description": "search | news | research | price | compare"}, - "items": {"type": "ARRAY", "items": {"type": "STRING"}, "description": "Items to compare (compare mode)"}, - "aspect": {"type": "STRING", "description": "Comparison aspect: price | specs | reviews | features"}, - }, - "required": ["query"] - } - }, + # ── Inline tools ───────────────────────────────────────────────────────── + # These stay here (rather than in an actions/*.py TOOL dict) because their + # handling is woven into live-session state — vision capture/injection, + # camera stream, memory writes, the monitor engine, and shutdown. All other + # tools live in their own action file and are auto-discovered by + # core.action_loader (see JarvisLive.__init__). { "name": "system_status", "description": ( @@ -193,61 +161,6 @@ TOOL_DECLARATIONS = [ "properties": {}, } }, - { - "name": "weather_report", - "description": "Gives the weather report to user", - "parameters": { - "type": "OBJECT", - "properties": { - "city": {"type": "STRING", "description": "City name"} - }, - "required": ["city"] - } - }, - { - "name": "send_message", - "description": "Sends a text message via WhatsApp, Telegram, or other messaging platform.", - "parameters": { - "type": "OBJECT", - "properties": { - "receiver": {"type": "STRING", "description": "Recipient contact name"}, - "message_text": {"type": "STRING", "description": "The message to send"}, - "platform": {"type": "STRING", "description": "Platform: WhatsApp, Telegram, etc."} - }, - "required": ["receiver", "message_text", "platform"] - } - }, - { - "name": "reminder", - "description": "Sets a timed reminder using Task Scheduler.", - "parameters": { - "type": "OBJECT", - "properties": { - "date": {"type": "STRING", "description": "Date in YYYY-MM-DD format"}, - "time": {"type": "STRING", "description": "Time in HH:MM format (24h)"}, - "message": {"type": "STRING", "description": "Reminder message text"} - }, - "required": ["date", "time", "message"] - } - }, - { - "name": "youtube_video", - "description": ( - "Controls YouTube. Use for: playing videos, summarizing a video's content, " - "getting video info, or showing trending videos." - ), - "parameters": { - "type": "OBJECT", - "properties": { - "action": {"type": "STRING", "description": "play | summarize | get_info | trending (default: play)"}, - "query": {"type": "STRING", "description": "Search query for play action"}, - "save": {"type": "BOOLEAN", "description": "Save summary to Notepad (summarize only)"}, - "region": {"type": "STRING", "description": "Country code for trending e.g. TR, US"}, - "url": {"type": "STRING", "description": "Video URL for get_info action"}, - }, - "required": [] - } - }, { "name": "screen_process", "description": ( @@ -271,225 +184,11 @@ TOOL_DECLARATIONS = [ "name": "close_camera", "description": ( "Closes the live camera view shown on screen. " - "Call when user says: close camera, stop camera, turn off camera, " - "kamerayı kapat, kapat, creepy, etc." + "Call when the user says (in ANY language): close camera, stop camera, " + "turn off camera, that's creepy, etc." ), "parameters": {"type": "OBJECT", "properties": {}, "required": []} }, - { - "name": "computer_settings", - "description": ( - "Controls the computer: volume, brightness, window management, keyboard shortcuts, " - "typing text on screen, closing apps, fullscreen, dark mode, WiFi, restart, shutdown, " - "scrolling, tab management, zoom, screenshots, lock screen, refresh/reload page. " - "Use for ANY single computer control command. " - "restart, shutdown and toggle_wifi put a confirmation on the user's screen " - "and do NOT happen until they press it — never claim they are done. " - "Volume, brightness and dark mode can be reversed with the `undo` tool." - ), - "parameters": { - "type": "OBJECT", - "properties": { - # The exact vocabulary, spelled out. - # - # This used to say only "The action to perform", so the model - # usually filled `description` instead — and computer_settings - # then made a SECOND Gemini call, inside the tool, purely to - # translate that sentence into one of these names. Every - # "turn the volume down" cost two model round trips. - "action": { - "type": "STRING", - "description": ( - "The exact action. Prefer this over `description` — pick one of: " - "volume_up | volume_down | volume_set | mute | " - "brightness_up | brightness_down | sleep_display | " - "pause_video | close_app | close_window | full_screen | " - "minimize | maximize | snap_left | snap_right | " - "switch_window | show_desktop | task_manager | focus_search | " - "refresh_page | close_tab | new_tab | next_tab | prev_tab | " - "go_back | go_forward | zoom_in | zoom_out | zoom_reset | " - "find_on_page | scroll_up | scroll_down | scroll_top | " - "scroll_bottom | page_up | page_down | copy | paste | cut | " - "undo | redo | select_all | save | enter | escape | press_key | " - "type_text | screenshot | lock_screen | open_settings | " - "file_explorer | open_run | dark_mode | toggle_wifi | " - "restart | shutdown" - ), - }, - "description": { - "type": "STRING", - "description": ( - "Fallback only, when no action name above fits. " - "Resolved locally — no extra model call." - ), - }, - "value": {"type": "STRING", "description": "Optional value: volume level 0-100, text to type, key name, etc."} - }, - "required": [] - } - }, - { - "name": "browser_control", - "description": ( - "Controls any web browser. Use for: opening websites, searching the web, " - "clicking elements, filling forms, scrolling, screenshots, navigation, any web-based task. " - "Simple open/search requests launch the user's own browser normally (their real profile " - "and logged-in accounts); interactive actions (click, type, fill_form...) attach an " - "automation browser. " - "Always pass the 'browser' parameter when the user specifies a browser (e.g. 'open in Edge', " - "'use Firefox', 'open Chrome'). Multiple browsers can run simultaneously." - ), - "parameters": { - "type": "OBJECT", - "properties": { - "action": {"type": "STRING", "description": "go_to | search | click | type | scroll | fill_form | smart_click | smart_type | get_text | get_url | press | new_tab | close_tab | screenshot | back | forward | reload | switch | list_browsers | close | close_all"}, - "browser": {"type": "STRING", "description": "Target browser: chrome | edge | firefox | opera | operagx | brave | vivaldi | safari. Omit to use the currently active browser."}, - "url": {"type": "STRING", "description": "URL for go_to / new_tab action"}, - "query": {"type": "STRING", "description": "Search query for search action"}, - "engine": {"type": "STRING", "description": "Search engine: google | bing | duckduckgo | yandex (default: google)"}, - "selector": {"type": "STRING", "description": "CSS selector for click/type"}, - "text": {"type": "STRING", "description": "Text to click or type"}, - "description": {"type": "STRING", "description": "Element description for smart_click/smart_type"}, - "direction": {"type": "STRING", "description": "up | down for scroll"}, - "amount": {"type": "INTEGER", "description": "Scroll amount in pixels (default: 500)"}, - "key": {"type": "STRING", "description": "Key name for press action (e.g. Enter, Escape, F5)"}, - "path": {"type": "STRING", "description": "Save path for screenshot"}, - "incognito": {"type": "BOOLEAN", "description": "Open in private/incognito mode"}, - "clear_first": {"type": "BOOLEAN", "description": "Clear field before typing (default: true)"}, - }, - "required": ["action"] - } - }, - { - "name": "file_controller", - "description": "Manages files and folders: list, create, delete, move, copy, rename, read, write, find, disk usage.", - "parameters": { - "type": "OBJECT", - "properties": { - "action": {"type": "STRING", "description": "list | create_file | create_folder | delete | move | copy | rename | read | write | find | largest | disk_usage | organize_desktop | info"}, - "path": {"type": "STRING", "description": "File/folder path or shortcut: desktop, downloads, documents, home"}, - "destination": {"type": "STRING", "description": "Destination path for move/copy"}, - "new_name": {"type": "STRING", "description": "New name for rename"}, - "content": {"type": "STRING", "description": "Content for create_file/write"}, - "name": {"type": "STRING", "description": "File name to search for"}, - "extension": {"type": "STRING", "description": "File extension to search (e.g. .pdf)"}, - "count": {"type": "INTEGER", "description": "Number of results for largest"}, - }, - "required": ["action"] - } - }, - { - "name": "desktop_control", - "description": "Controls the desktop: wallpaper, organize, clean, list, stats.", - "parameters": { - "type": "OBJECT", - "properties": { - "action": {"type": "STRING", "description": "wallpaper | wallpaper_url | organize | clean | list | stats | task"}, - "path": {"type": "STRING", "description": "Image path for wallpaper"}, - "url": {"type": "STRING", "description": "Image URL for wallpaper_url"}, - "mode": {"type": "STRING", "description": "by_type or by_date for organize"}, - "task": {"type": "STRING", "description": "Natural language desktop task"}, - }, - "required": ["action"] - } - }, - { - "name": "code_helper", - "description": "Writes, edits, explains, runs, or builds code files.", - "parameters": { - "type": "OBJECT", - "properties": { - "action": {"type": "STRING", "description": "write | edit | explain | run | build | auto (default: auto)"}, - "description": {"type": "STRING", "description": "What the code should do or what change to make"}, - "language": {"type": "STRING", "description": "Programming language (default: python)"}, - "output_path": {"type": "STRING", "description": "Where to save the file"}, - "file_path": {"type": "STRING", "description": "Path to existing file for edit/explain/run/build"}, - "code": {"type": "STRING", "description": "Raw code string for explain"}, - "args": {"type": "STRING", "description": "CLI arguments for run/build"}, - "timeout": {"type": "INTEGER", "description": "Execution timeout in seconds (default: 30)"}, - }, - "required": ["action"] - } - }, - { - "name": "dev_agent", - "description": "Builds complete multi-file projects from scratch: plans, writes files, installs deps, opens VSCode, runs and fixes errors.", - "parameters": { - "type": "OBJECT", - "properties": { - "description": {"type": "STRING", "description": "What the project should do"}, - "language": {"type": "STRING", "description": "Programming language (default: python)"}, - "project_name": {"type": "STRING", "description": "Optional project folder name"}, - "timeout": {"type": "INTEGER", "description": "Run timeout in seconds (default: 30)"}, - }, - "required": ["description"] - } - }, - { - "name": "computer_control", - "description": "Direct computer control: type, click, hotkeys, scroll, move mouse, screenshots, find elements on screen.", - "parameters": { - "type": "OBJECT", - "properties": { - "action": {"type": "STRING", "description": "type | smart_type | click | double_click | right_click | hotkey | press | scroll | move | copy | paste | screenshot | wait | clear_field | focus_window | screen_find | screen_click | random_data | user_data"}, - "text": {"type": "STRING", "description": "Text to type or paste"}, - "x": {"type": "INTEGER", "description": "X coordinate"}, - "y": {"type": "INTEGER", "description": "Y coordinate"}, - "keys": {"type": "STRING", "description": "Key combination e.g. 'ctrl+c'"}, - "key": {"type": "STRING", "description": "Single key e.g. 'enter'"}, - "direction": {"type": "STRING", "description": "up | down | left | right"}, - "amount": {"type": "INTEGER", "description": "Scroll amount (default: 3)"}, - "seconds": {"type": "NUMBER", "description": "Seconds to wait"}, - "title": {"type": "STRING", "description": "Window title for focus_window"}, - "description": {"type": "STRING", "description": "Element description for screen_find/screen_click"}, - "type": {"type": "STRING", "description": "Data type for random_data"}, - "field": {"type": "STRING", "description": "Field for user_data: name|email|city"}, - "clear_first": {"type": "BOOLEAN", "description": "Clear field before typing (default: true)"}, - "path": {"type": "STRING", "description": "Save path for screenshot"}, - }, - "required": ["action"] - } - }, - { - "name": "game_updater", - "description": ( - "THE ONLY tool for ANY Steam or Epic Games request. " - "Use for: installing, downloading, updating games, listing installed games, " - "checking download status, scheduling updates. " - "ALWAYS call directly for any Steam/Epic/game request. " - "NEVER use browser_control or web_search for Steam/Epic." - ), - "parameters": { - "type": "OBJECT", - "properties": { - "action": {"type": "STRING", "description": "update | install | list | download_status | schedule | cancel_schedule | schedule_status (default: update)"}, - "platform": {"type": "STRING", "description": "steam | epic | both (default: both)"}, - "game_name": {"type": "STRING", "description": "Game name (partial match supported)"}, - "app_id": {"type": "STRING", "description": "Steam AppID for install (optional)"}, - "hour": {"type": "INTEGER", "description": "Hour for scheduled update 0-23 (default: 3)"}, - "minute": {"type": "INTEGER", "description": "Minute for scheduled update 0-59 (default: 0)"}, - "shutdown_when_done": {"type": "BOOLEAN", "description": "Shut down PC when download finishes"}, - }, - "required": [] - } - }, - { - "name": "flight_finder", - "description": "Searches Google Flights and speaks the best options.", - "parameters": { - "type": "OBJECT", - "properties": { - "origin": {"type": "STRING", "description": "Departure city or airport code"}, - "destination": {"type": "STRING", "description": "Arrival city or airport code"}, - "date": {"type": "STRING", "description": "Departure date (any format)"}, - "return_date": {"type": "STRING", "description": "Return date for round trips"}, - "passengers": {"type": "INTEGER", "description": "Number of passengers (default: 1)"}, - "cabin": {"type": "STRING", "description": "economy | premium | business | first"}, - "save": {"type": "BOOLEAN", "description": "Save results to Notepad"}, - }, - "required": ["origin", "destination", "date"] - } - }, { "name": "manage_monitor", "description": ( @@ -528,72 +227,6 @@ TOOL_DECLARATIONS = [ "properties": {}, } }, - { - "name": "file_processor", - "description": ( - "Processes any file that the user has uploaded or dropped onto the interface. " - "Use this when the user refers to an uploaded file and wants an action on it. " - "Supports: images (describe/ocr/resize/compress/convert), " - "PDFs (summarize/extract_text/to_word), " - "Word docs & text files (summarize/fix/reformat/translate), " - "CSV/Excel (analyze/stats/filter/sort/convert), " - "JSON/XML (validate/format/analyze), " - "code files (explain/review/fix/optimize/run/document/test), " - "audio (transcribe/trim/convert/info), " - "video (trim/extract_audio/extract_frame/compress/transcribe/info), " - "archives (list/extract), " - "presentations (summarize/extract_text). " - "ALWAYS call this tool when a file has been uploaded and the user gives a command about it. " - "If the user's command is ambiguous, pick the most logical action for that file type." - ), - "parameters": { - "type": "OBJECT", - "properties": { - "file_path": { - "type": "STRING", - "description": "Full path to the uploaded file. Leave empty to use the currently uploaded file." - }, - "action": { - "type": "STRING", - "description": ( - "What to do with the file. Examples by type:\n" - "image: describe | ocr | resize | compress | convert | info\n" - "pdf: summarize | extract_text | to_word | info\n" - "docx/txt: summarize | fix | reformat | translate_hint | word_count | to_bullet\n" - "csv/excel: analyze | stats | filter | sort | convert | info\n" - "json: validate | format | analyze | to_csv\n" - "code: explain | review | fix | optimize | run | document | test\n" - "audio: transcribe | trim | convert | info\n" - "video: trim | extract_audio | extract_frame | compress | transcribe | info | convert\n" - "archive: list | extract\n" - "pptx: summarize | extract_text | analyze" - ) - }, - "instruction": { - "type": "STRING", - "description": "Free-form instruction if action doesn't cover it. E.g. 'translate this to Turkish', 'find all email addresses'" - }, - "format": { - "type": "STRING", - "description": "Target format for conversion. E.g. 'mp3', 'pdf', 'csv', 'png'" - }, - "width": {"type": "INTEGER", "description": "Target width for image resize"}, - "height": {"type": "INTEGER", "description": "Target height for image resize"}, - "scale": {"type": "NUMBER", "description": "Scale factor for image resize (e.g. 0.5)"}, - "quality": {"type": "INTEGER", "description": "Quality 1-100 for image/video compress"}, - "start": {"type": "STRING", "description": "Start time for trim: seconds or HH:MM:SS"}, - "end": {"type": "STRING", "description": "End time for trim: seconds or HH:MM:SS"}, - "timestamp": {"type": "STRING", "description": "Timestamp for video frame extraction HH:MM:SS"}, - "column": {"type": "STRING", "description": "Column name for CSV filter/sort"}, - "value": {"type": "STRING", "description": "Filter value for CSV filter"}, - "condition": {"type": "STRING", "description": "Filter condition: equals|contains|gt|lt"}, - "ascending": {"type": "BOOLEAN", "description": "Sort order for CSV sort (default: true)"}, - "save": {"type": "BOOLEAN", "description": "Save result to file (default: true)"}, - "destination": {"type": "STRING", "description": "Output folder for archive extract"}, - }, - "required": [] - } -}, { "name": "save_memory", "description": ( @@ -717,14 +350,13 @@ def _keep_context_of(exc: BaseException) -> bool: class JarvisLive: - def __init__(self, ui: JarvisUI): self.ui = ui - self._asst_name = "JARVIS" # updated each session from config + self._asst_name = "JARVI S" # updated each session from config self.session = None self.audio_in_queue = None self.out_queue = None - self._loop = None + self._loop = None self._is_speaking = False self._speaking_lock = threading.Lock() self._phone_active = False # True while phone mic is streaming; pauses PC mic @@ -765,16 +397,130 @@ class JarvisLive: self._last_user_speech = time.monotonic() # updated on every user utterance self._session_log: list[str] = [] # conversation turns for end-of-session summary - self._enhanced_live = True # affective dialog + proactive audio; auto-disabled if the server rejects them - _core_names = {t["name"] for t in TOOL_DECLARATIONS} + self._enhanced_live = True # proactive audio; auto-disabled if the server rejects it + + _base_dir = Path(__file__).resolve().parent + _inline_names = {t["name"] for t in TOOL_DECLARATIONS} + + # File-backed tools: every actions/*.py with a TOOL dict, discovered the + # same way plugins are. Reserved names = the inline tools above, so an + # action can never shadow one. + self._action_registry = discover_actions( + actions_dir=_base_dir / "actions", + reserved_names=_inline_names, + logger=lambda msg: print(f"[Actions] {msg}"), + ) + + # Plugins must not collide with either an inline tool or a discovered action. + _core_names = _inline_names | self._action_registry.names() self._plugin_registry = discover_plugins( - plugins_dir=Path(__file__).resolve().parent / "plugins", + plugins_dir=_base_dir / "plugins", core_tool_names=_core_names, logger=lambda msg: (print(f"[Plugins] {msg}"), self.ui.write_log(f"SYS: {msg}")), ) self.ui.get_plugins = self._plugin_registry.list_for_ui + self.ui.get_plugin_settings = self._plugin_registry.settings_schemas # ⚙ settings tab self.ui.request_say = self.plugin_say # plugins: mid-task speech channel + # ── Wake word ──────────────────────────────────────────────────────── + # _awake gates the mic (see _listen_audio) and the background speakers. + # It is True whenever wake word is OFF, so default behaviour is unchanged. + self._wake_enabled = get_wake_word_enabled() + self._awake = not self._wake_enabled + self._wake_detector: WakeWordDetector | None = None + self._wake_sleep_timeout = WAKE_SLEEP_TIMEOUT + # UI control surface for the Wake Word settings section. + self.ui.wake_is_ready = wake_is_ready # () -> bool + self.ui.wake_get_state = self._wake_state # () -> dict + self.ui.on_wake_toggle = self._ui_wake_toggle # (enable: bool) -> str + self.ui.on_wake_manual = self._ui_wake_manual # () -> toggle awake/asleep + self.ui.on_wake_install = self._ui_wake_install # () -> (ok, msg) + + # ── Wake word: state machine ───────────────────────────────────────────── + + def _wake_state(self) -> dict: + # A loaded, running detector is definitively ready; otherwise fall back + # to the cheap on-disk model-file check (no Model construction). + ready = bool(self._wake_detector and self._wake_detector.ready) or wake_is_ready() + return {"enabled": self._wake_enabled, "awake": self._awake, "ready": ready} + + def _ensure_wake_detector(self) -> bool: + """Load the detector once (model loads on first start). Idempotent.""" + if self._wake_detector is None: + self._wake_detector = WakeWordDetector( + on_detect=self._on_wake_detected, + logger=lambda m: (print(f"[Wake] {m}"), self.ui.write_log(f"SYS: {m}")), + ) + if not self._wake_detector.ready: + return self._wake_detector.start() + return True + + def _on_wake_detected(self) -> None: + """Called from the detector thread when 'Hey Jarvis' is heard.""" + self.wake(reason="wake word") + + def wake(self, reason: str = "wake word") -> None: + if self._awake: + return + self._awake = True + self._last_user_speech = time.monotonic() # start the auto-sleep clock now + if not self.ui.muted: + self.ui.set_state("LISTENING") + self.ui.write_log(f"SYS: Awake — {reason}.") + + def sleep(self, reason: str = "timeout") -> None: + if not self._awake: + return + self._awake = False + self.set_speaking(False) + self.ui.set_state("SLEEPING") + self.ui.write_log(f"SYS: Sleeping — {reason}. Say 'Hey Jarvis' to wake me.") + + async def _run_sleep_watch(self) -> None: + """Auto-sleep after the configured silence window (wake-word mode only).""" + while True: + await asyncio.sleep(5) + if not self._wake_enabled or not self._awake: + continue + with self._speaking_lock: + speaking = self._is_speaking + if speaking: + continue + if (time.monotonic() - self._last_user_speech) > self._wake_sleep_timeout: + self.sleep(reason="no speech for 2 minutes") + + # ── Wake word: UI callbacks (called from the Qt thread) ────────────────── + + def _ui_wake_toggle(self, enable: bool) -> str: + """Enable/disable wake word from the settings UI. Returns a status token: + 'enabled' | 'disabled' | 'need_download'.""" + if enable: + if not wake_is_ready(): + return "need_download" + self._wake_enabled = True + save_wake_word_enabled(True) + self._ensure_wake_detector() + self.sleep(reason="wake word enabled") + return "enabled" + else: + self._wake_enabled = False + save_wake_word_enabled(False) + self.wake(reason="wake word disabled") + return "disabled" + + def _ui_wake_manual(self) -> None: + """Manual sleep/wake button in the UI.""" + if not self._wake_enabled: + return + if self._awake: + self.sleep(reason="you tapped sleep") + else: + self.wake(reason="you tapped wake") + + def _ui_wake_install(self) -> tuple[bool, str]: + """Download openwakeword + the model (runs in a UI worker thread).""" + return wake_install(logger=lambda m: self.ui.write_log(f"SYS: {m}")) + def plugin_say(self, instruction: str) -> None: """ Thread-safe speech channel for plugins: lets a plugin ask JARVIS to @@ -791,7 +537,7 @@ class JarvisLive: async def _say(): try: await self.session.send_client_content( - turns={"parts": [{"text": instruction}]}, + turns={"role": "user", "parts": [{"text": instruction}]}, turn_complete=True, ) except Exception as e: @@ -865,9 +611,15 @@ class JarvisLive: def _on_text_command(self, text: str): if not self._loop or not self.session: return + # Respect wake-word sleep: a typed command must not be answered while + # asleep either (the sleep gate is not just for the mic). Wake first with + # "Hey Jarvis" or the WAKE NOW button. + if self._wake_enabled and not self._awake: + self.ui.write_log("SYS: I'm asleep — say 'Hey Jarvis' or tap WAKE NOW first.") + return asyncio.run_coroutine_threadsafe( self.session.send_client_content( - turns={"parts": [{"text": text}]}, + turns={"role": "user", "parts": [{"text": text}]}, turn_complete=True ), self._loop @@ -905,7 +657,7 @@ class JarvisLive: return asyncio.run_coroutine_threadsafe( self.session.send_client_content( - turns={"parts": [{"text": text}]}, + turns={"role": "user", "parts": [{"text": text}]}, turn_complete=True ), self._loop @@ -966,7 +718,11 @@ class JarvisLive: output_audio_transcription={}, input_audio_transcription={}, system_instruction="\n".join(parts), - tools=[{"function_declarations": TOOL_DECLARATIONS + self._plugin_registry.get_tool_declarations()}], + tools=[{"function_declarations": ( + TOOL_DECLARATIONS + + self._action_registry.get_tool_declarations() + + self._plugin_registry.get_tool_declarations() + )}], # Hand back the handle captured from the last session_resumption # update. `handle=None` is exactly the old behaviour (ask for # handles, start fresh), so the first connect of a run is unchanged. @@ -987,10 +743,12 @@ class JarvisLive: ), ) if self._enhanced_live: - # Affective dialog: JARVIS hears tone/emotion and adapts its voice. # Proactive audio: JARVIS stays silent when speech isn't addressed # to it (background chatter, talking to someone else in the room). - cfg["enable_affective_dialog"] = True + # (Affective dialog was dropped: gemini-3.1-flash-live does not + # support it, and it never reliably detected tone in practice. + # To restore it on a 2.5 native-audio model, add back: + # cfg["enable_affective_dialog"] = True ) cfg["proactivity"] = types.ProactivityConfig(proactive_audio=True) return types.LiveConnectConfig(**cfg) @@ -1035,34 +793,6 @@ class JarvisLive: else: result = await loop.run_in_executor(None, undo_stack.undo_last) - elif name == "open_app": - r = await loop.run_in_executor(None, lambda: open_app(parameters=args, response=None, player=self.ui)) - result = r or f"Opened {args.get('app_name')}." - - elif name == "weather_report": - r = await loop.run_in_executor(None, lambda: weather_action(parameters=args, player=self.ui)) - result = r or "Weather delivered." - - elif name == "browser_control": - r = await loop.run_in_executor(None, lambda: browser_control(parameters=args, player=self.ui)) - result = r or "Done." - - elif name == "file_controller": - r = await loop.run_in_executor(None, lambda: file_controller(parameters=args, player=self.ui)) - result = r or "Done." - - elif name == "send_message": - r = await loop.run_in_executor(None, lambda: send_message(parameters=args, response=None, player=self.ui, session_memory=None)) - result = r or f"Message sent to {args.get('receiver')}." - - elif name == "reminder": - r = await loop.run_in_executor(None, lambda: reminder(parameters=args, response=None, player=self.ui)) - result = r or "Reminder set." - - elif name == "youtube_video": - r = await loop.run_in_executor(None, lambda: youtube_video(parameters=args, response=None, player=self.ui)) - result = r or "Done." - elif name == "screen_process": import time as _t_mod _now = _t_mod.monotonic() @@ -1098,52 +828,6 @@ class JarvisLive: self.ui.stop_camera_stream() result = "Camera closed." - elif name == "computer_settings": - r = await loop.run_in_executor(None, lambda: computer_settings(parameters=args, response=None, player=self.ui)) - result = r or "Done." - - elif name == "desktop_control": - r = await loop.run_in_executor(None, lambda: desktop_control(parameters=args, player=self.ui)) - result = r or "Done." - - elif name == "code_helper": - r = await loop.run_in_executor(None, lambda: code_helper(parameters=args, player=self.ui, speak=self.speak)) - result = r or "Done." - - elif name == "dev_agent": - r = await loop.run_in_executor(None, lambda: dev_agent(parameters=args, player=self.ui, speak=self.speak)) - result = r or "Done." - - elif name == "web_search": - r = await loop.run_in_executor(None, lambda: web_search_action(parameters=args, player=self.ui)) - result = r or "Done." - # Mirror results to the on-screen content panel - _mode = args.get("mode", "search") - if r and not r.startswith("No results") and not r.startswith("Search failed"): - _query = args.get("query") or ", ".join(args.get("items", [])) - _label = f"{_mode.upper()} — {_query[:38]}" if _query else _mode.upper() - self.ui.show_content(_label, r) - elif name == "file_processor": - if not args.get("file_path") and self.ui.current_file: - args["file_path"] = self.ui.current_file - r = await loop.run_in_executor( - None, - lambda: file_processor(parameters=args, player=self.ui, speak=self.speak) - ) - result = r or "Done." - - elif name == "computer_control": - r = await loop.run_in_executor(None, lambda: computer_control(parameters=args, player=self.ui)) - result = r or "Done." - - elif name == "game_updater": - r = await loop.run_in_executor(None, lambda: game_updater(parameters=args, player=self.ui, speak=self.speak)) - result = r or "Done." - - elif name == "flight_finder": - r = await loop.run_in_executor(None, lambda: flight_finder(parameters=args, player=self.ui)) - result = r or "Done." - elif name == "system_status": r = await loop.run_in_executor(None, get_system_status) result = str(r) @@ -1168,7 +852,7 @@ class JarvisLive: if self.session: try: await self.session.send_client_content( - turns={"parts": [{"text": "Say a brief natural goodbye to the user."}]}, + turns={"role": "user", "parts": [{"text": "Say a brief natural goodbye to the user."}]}, turn_complete=True, ) except Exception: @@ -1178,6 +862,23 @@ class JarvisLive: _os._exit(0) asyncio.create_task(_do_shutdown()) + elif self._action_registry.has(name): + # file_processor: fall back to the currently-uploaded file when none is given + if name == "file_processor" and not args.get("file_path") and self.ui.current_file: + args["file_path"] = self.ui.current_file + _ctx = {"player": self.ui, "speak": self.speak, + "response": None, "session_memory": None} + r = await loop.run_in_executor(None, lambda: self._action_registry.run(name, args, _ctx)) + result = r or "Done." + # web_search: mirror results to the on-screen content panel + if (name == "web_search" and r + and not r.startswith("No results") + and not r.startswith("Search failed")): + _mode = args.get("mode", "search") + _query = args.get("query") or ", ".join(args.get("items", [])) + _label = f"{_mode.upper()} — {_query[:38]}" if _query else _mode.upper() + self.ui.show_content(_label, r) + else: if self._plugin_registry.has(name): r = await loop.run_in_executor( @@ -1205,13 +906,35 @@ class JarvisLive: async def _send_realtime(self): while True: msg = await self.out_queue.get() - await self.session.send_realtime_input(media=msg) + # Gemini 3.x Live rejects the old realtime_input.media_chunks field + # (what `media=...` maps to) and closes the socket with a 1007. Send + # mic / phone PCM through the new `audio` field instead. Queue items + # are {"data":