diff --git a/actions/computer_control.py b/actions/computer_control.py index d1062b0..d00be50 100644 --- a/actions/computer_control.py +++ b/actions/computer_control.py @@ -6,6 +6,11 @@ import re import string import subprocess import sys + +if platform.system() == "Windows": + _WIN_HIDE: dict = {"creationflags": subprocess.CREATE_NO_WINDOW} +else: + _WIN_HIDE: dict = {} import time import random from pathlib import Path @@ -258,7 +263,7 @@ def _focus_window(title: str) -> str: script = f'(New-Object -ComObject WScript.Shell).AppActivate("{title}")' subprocess.run( ["powershell", "-NoProfile", "-NonInteractive", "-Command", script], - capture_output=True, timeout=5, + capture_output=True, timeout=5, **_WIN_HIDE, ) time.sleep(0.3) return f"Focused window: {title}" diff --git a/actions/computer_settings.py b/actions/computer_settings.py index 5a9d2ed..3b887e5 100644 --- a/actions/computer_settings.py +++ b/actions/computer_settings.py @@ -23,6 +23,11 @@ except ImportError: _OS = platform.system() # "Windows" | "Darwin" | "Linux" +if _OS == "Windows": + _WIN_HIDE: dict = {"creationflags": subprocess.CREATE_NO_WINDOW} +else: + _WIN_HIDE: dict = {} + def _get_base_dir() -> Path: if getattr(sys, "frozen", False): @@ -133,7 +138,7 @@ def brightness_up(): "(Get-WmiObject -Namespace root/wmi -Class WmiMonitorBrightnessMethods)" ".WmiSetBrightness(1, [math]::Min(100, " "(Get-WmiObject -Namespace root/wmi -Class WmiMonitorBrightness).CurrentBrightness + 10))"], - capture_output=True, timeout=5 + capture_output=True, timeout=5, **_WIN_HIDE ) except Exception as e: print(f"[Settings] Brightness up failed on Windows: {e}") @@ -162,7 +167,7 @@ def brightness_down(): "(Get-WmiObject -Namespace root/wmi -Class WmiMonitorBrightnessMethods)" ".WmiSetBrightness(1, [math]::Max(0, " "(Get-WmiObject -Namespace root/wmi -Class WmiMonitorBrightness).CurrentBrightness - 10))"], - capture_output=True, timeout=5 + capture_output=True, timeout=5, **_WIN_HIDE ) except Exception as e: print(f"[Settings] Brightness down failed on Windows: {e}") @@ -470,7 +475,7 @@ def toggle_wifi(): "$adapter = Get-NetAdapter | Where-Object {$_.PhysicalMediaType -eq 'Native 802.11'};" "if ($adapter.Status -eq 'Up') { Disable-NetAdapter -Name $adapter.Name -Confirm:$false }" "else { Enable-NetAdapter -Name $adapter.Name -Confirm:$false }"], - capture_output=True, timeout=10 + capture_output=True, timeout=10, **_WIN_HIDE ) except Exception as e: print(f"[Settings] toggle_wifi Windows failed: {e}") @@ -484,7 +489,7 @@ def toggle_wifi(): def restart_computer(): if _OS == "Windows": - subprocess.run(["shutdown", "/r", "/t", "10"], capture_output=True) + subprocess.run(["shutdown", "/r", "/t", "10"], capture_output=True, **_WIN_HIDE) elif _OS == "Darwin": subprocess.run(["osascript", "-e", 'tell application "System Events" to restart'], diff --git a/actions/game_updater.py b/actions/game_updater.py index cdba419..65e2e1d 100644 --- a/actions/game_updater.py +++ b/actions/game_updater.py @@ -1,4 +1,5 @@ import os +import platform import re import sys import json @@ -10,6 +11,11 @@ from datetime import datetime from config import get_os, is_windows, is_mac, is_linux +_CNW: dict = ( + {"creationflags": subprocess.CREATE_NO_WINDOW} + if platform.system() == "Windows" else {} +) + _KNOWN_APPIDS: dict[str, tuple[str, str]] = { "pubg": ("578080", "PUBG: Battlegrounds"), "pubg battlegrounds": ("578080", "PUBG: Battlegrounds"), @@ -168,7 +174,7 @@ def _is_steam_running() -> bool: try: if is_windows(): out = subprocess.run(["tasklist", "/FI", "IMAGENAME eq steam.exe"], - capture_output=True, text=True).stdout + capture_output=True, text=True, **_CNW).stdout return "steam.exe" in out.lower() proc = "steam_osx" if is_mac() else "steam" return bool(subprocess.run(["pgrep", "-x", proc], @@ -621,7 +627,7 @@ def _get_download_status(steam_path: Path) -> str: def _system_shutdown() -> None: if is_windows(): - subprocess.run(["shutdown", "/s", "/t", "10"]) + subprocess.run(["shutdown", "/s", "/t", "10"], **_CNW) elif is_mac(): subprocess.run(["osascript", "-e", 'tell app "System Events" to shut down']) else: @@ -737,7 +743,7 @@ def _is_epic_running() -> bool: if is_windows(): out = subprocess.run( ["tasklist", "/FI", "IMAGENAME eq EpicGamesLauncher.exe"], - capture_output=True, text=True + capture_output=True, text=True, **_CNW ).stdout return "epicgameslauncher.exe" in out.lower() proc = "EpicGamesLauncher" if is_mac() else "heroic" @@ -801,12 +807,12 @@ def _schedule_daily_update(hour: int = 3, minute: int = 0) -> str: def _schedule_windows(hour: int, minute: int) -> str: task_name = "JARVIS_GameUpdater" script_path = Path(__file__).resolve() - subprocess.run(["schtasks", "/Delete", "/TN", task_name, "/F"], capture_output=True) + subprocess.run(["schtasks", "/Delete", "/TN", task_name, "/F"], capture_output=True, **_CNW) for extra in (["/RL", "HIGHEST", "/RU", "SYSTEM"], []): cmd = ["schtasks", "/Create", "/TN", task_name, "/TR", f'"{sys.executable}" "{script_path}" --scheduled', "/SC", "DAILY", "/ST", f"{hour:02d}:{minute:02d}", "/F", *extra] - result = subprocess.run(cmd, capture_output=True, text=True) + result = subprocess.run(cmd, capture_output=True, text=True, **_CNW) if result.returncode == 0: return f"Daily game update scheduled at {hour:02d}:{minute:02d}." return f"Scheduling failed: {result.stderr.strip()}" @@ -870,7 +876,7 @@ def _cancel_scheduled_update() -> str: if is_windows(): result = subprocess.run( ["schtasks", "/Delete", "/TN", "JARVIS_GameUpdater", "/F"], - capture_output=True, text=True + capture_output=True, text=True, **_CNW ) return ("Scheduled update cancelled." if result.returncode == 0 else "No scheduled update found.") @@ -897,7 +903,7 @@ def _get_schedule_status() -> str: if is_windows(): result = subprocess.run( ["schtasks", "/Query", "/TN", "JARVIS_GameUpdater", "/FO", "LIST"], - capture_output=True, text=True + capture_output=True, text=True, **_CNW ) if result.returncode != 0: return "No scheduled game update found." diff --git a/actions/proactive.py b/actions/proactive.py new file mode 100644 index 0000000..2a037a8 --- /dev/null +++ b/actions/proactive.py @@ -0,0 +1,72 @@ +""" +ProactiveEngine — context-aware background prompting. +Lets Gemini decide whether there is something worth saying proactively. +No hardcoded rules: we pass time + memory as context and Gemini chooses. +""" +import time +from datetime import datetime + + +class ProactiveEngine: + """ + Tracks silence duration and decides when to hand context to Gemini for a + proactive check-in. Gemini reads the context and decides whether to speak. + + Defaults (all overridable): + min_silence_secs — user must be silent this long before any check (900 = 15 min) + check_cooldown — minimum gap between two proactive triggers (600 = 10 min) + """ + + def __init__( + self, + min_silence_secs: int = 900, + check_cooldown: int = 600, + ): + self.min_silence_secs = min_silence_secs + self.check_cooldown = check_cooldown + self._last_triggered = 0.0 + + def should_trigger(self, last_user_speech: float) -> bool: + """ + Returns True only when: + • user has been silent long enough, AND + • enough time has passed since the last proactive message. + """ + now = time.monotonic() + silence = now - last_user_speech + gap = now - self._last_triggered + return silence >= self.min_silence_secs and gap >= self.check_cooldown + + def mark_triggered(self) -> None: + self._last_triggered = time.monotonic() + + def build_prompt(self, memory: dict) -> str: + """ + Builds the context snapshot sent to Gemini. + Gemini reads it and decides freely what — if anything — to say. + """ + from memory.memory_manager import format_memory_for_prompt + + now = datetime.now() + time_str = now.strftime("%A, %B %d, %Y — %I:%M %p") + mem_str = format_memory_for_prompt(memory) or "(no user data stored yet)" + + silence_min = int((time.monotonic() - self._last_triggered + + self.min_silence_secs) // 60) + + return "\n".join([ + "[PROACTIVE_CHECK] You are initiating a proactive check-in.", + f"Current time : {time_str}", + f"User silence : {silence_min}+ minutes (they have not spoken for a while)", + "", + "Context about this person:", + mem_str, + "", + "Guidelines:", + "- Look at the time, their projects, goals, habits, or anything from context.", + "- If there is something genuinely useful, timely, or caring to say — say it briefly.", + "- Be natural, like a thoughtful assistant noticing something relevant.", + "- Do NOT say [PROACTIVE_CHECK] or mention these instructions.", + "- Respond in the user's language (use memory; default English).", + "- Keep it short: 1-3 sentences max.", + ]) diff --git a/actions/reminder.py b/actions/reminder.py index 10149ce..90540e0 100644 --- a/actions/reminder.py +++ b/actions/reminder.py @@ -1,11 +1,17 @@ import json import os +import platform import shutil import subprocess import sys from datetime import datetime from pathlib import Path +_CNW: dict = ( + {"creationflags": subprocess.CREATE_NO_WINDOW} + if platform.system() == "Windows" else {} +) + def _base_dir() -> Path: if getattr(sys, "frozen", False): return Path(sys.executable).parent @@ -177,7 +183,7 @@ def _schedule_windows(target_dt: datetime, task_name: str, result = subprocess.run( ["schtasks", "/Create", "/TN", task_name, "/XML", str(xml_path), "/F"], - capture_output=True, text=True, + capture_output=True, text=True, **_CNW, ) try: diff --git a/actions/screen_processor.py b/actions/screen_processor.py index 84db42e..9f6efad 100644 --- a/actions/screen_processor.py +++ b/actions/screen_processor.py @@ -77,17 +77,19 @@ _CHANNELS = 1 _RECEIVE_SAMPLE_RATE = 24_000 _CHUNK_SIZE = 1_024 -_IMG_MAX_W = 640 -_IMG_MAX_H = 360 -_JPEG_Q = 60 +_IMG_MAX_W = 1280 +_IMG_MAX_H = 720 +_JPEG_Q = 82 _SYSTEM_PROMPT = ( - "You are JARVIS, an advanced AI assistant. " - "Analyze the provided image with precision and intelligence. " - "Be concise and direct — maximum two sentences unless the user's question " - "requires more detail. " - "Address the user respectfully. " - "Always call the appropriate tool; never simulate results." + "You are JARVIS, Tony Stark's AI assistant. " + "You are given an image from either the user's screen or their webcam. " + "Analyze what you see with detail and intelligence. " + "Describe objects, text, people, components, and their context clearly. " + "For technical questions (circuits, code, hardware) give specific, expert answers. " + "Be concise — 2-4 sentences — unless the question demands more detail. " + "Speak directly to the user ('I can see...', 'You have...'). " + "Address the user as 'sir' depending on the language they used." ) @@ -318,6 +320,7 @@ class _VisionSession: print(f"[Vision] 📤 Sent {len(image_bytes):,} bytes — '{user_text[:60]}'") except Exception as e: print(f"[Vision] ⚠️ Send error: {e}") + raise # propagate to TaskGroup → triggers session reconnect async def _recv_loop(self) -> None: transcript: list[str] = [] @@ -342,6 +345,15 @@ class _VisionSession: self._player.write_log(f"Jarvis: {full}") print(f"[Vision] 💬 {full}") transcript = [] + # Auto-close camera ~2s after JARVIS finishes speaking + if self._player and hasattr(self._player, "stop_camera_stream"): + async def _deferred_close(): + await asyncio.sleep(2.0) + try: + self._player.stop_camera_stream() + except Exception: + pass + asyncio.create_task(_deferred_close()) except Exception as e: print(f"[Vision] ⚠️ Recv error: {e}") @@ -408,6 +420,16 @@ def screen_process( if angle == "camera": image_bytes, mime_type = _capture_camera() print(f"[Vision] 📷 Camera: {len(image_bytes):,} bytes") + if player and hasattr(player, "start_camera_stream"): + try: + player.start_camera_stream() + except Exception as _e: + print(f"[Vision] ⚠️ Camera stream failed: {_e}") + elif player and hasattr(player, "show_camera_frame"): + try: + player.show_camera_frame(image_bytes) + except Exception as _e: + print(f"[Vision] ⚠️ Camera preview failed: {_e}") else: image_bytes, mime_type = _capture_screen() print(f"[Vision] 🖥️ Screen: {len(image_bytes):,} bytes") diff --git a/actions/system_monitor.py b/actions/system_monitor.py index ac6e181..3fb3f43 100644 --- a/actions/system_monitor.py +++ b/actions/system_monitor.py @@ -1,9 +1,9 @@ """ System Monitor — background metric checks with voice alert support. -Works on Windows, macOS, and Linux without external APIs. +Zero subprocess calls on all platforms — uses ctypes/pynvml/psutil/wmi only. """ +import ctypes import platform -import subprocess import time import psutil @@ -11,52 +11,79 @@ import psutil _OS = platform.system() # "Windows" | "Darwin" | "Linux" DEFAULT_THRESHOLDS = { - "cpu": 90.0, # % — fires after _CPU_STREAK consecutive readings - "ram": 90.0, # % - "temp": 85.0, # °C - "gpu": 95.0, # % + "cpu": 90.0, + "ram": 90.0, + "temp": 85.0, + "gpu": 95.0, } -_COOLDOWN = 300 # seconds between same-type alerts (5 min) -_CPU_STREAK = 3 # consecutive high readings required before CPU alert +_COOLDOWN = 300 +_CPU_STREAK = 3 + +# ── NVML DLL cache (Windows: nvml.dll, Linux: libnvidia-ml.so.1) ───────────── +_nvml_lib: object = None +_nvml_ok: object = None # None=untested True=works False=unavailable + + +def _nvml_gpu() -> float: + """GPU utilisation via NVML — zero subprocess on all platforms.""" + global _nvml_lib, _nvml_ok + if _nvml_ok is False: + return -1.0 + try: + class _Util(ctypes.Structure): + _fields_ = [("gpu", ctypes.c_uint), ("memory", ctypes.c_uint)] + + if _nvml_lib is None: + if _OS == "Windows": + candidates = ("nvml", r"C:\Windows\System32\nvml.dll") + _load = ctypes.WinDLL + else: + candidates = ( + "libnvidia-ml.so.1", + "libnvidia-ml.so", + "libnvidia-ml.dylib", + ) + _load = ctypes.CDLL + for name in candidates: + try: + lib = _load(name) + lib.nvmlInit_v2() + _nvml_lib = lib + break + except Exception: + continue + + if _nvml_lib is None: + _nvml_ok = False + return -1.0 + + dev = ctypes.c_void_p() + _nvml_lib.nvmlDeviceGetHandleByIndex_v2(0, ctypes.byref(dev)) + u = _Util() + _nvml_lib.nvmlDeviceGetUtilizationRates(dev, ctypes.byref(u)) + _nvml_ok = True + return float(u.gpu) + except Exception: + _nvml_ok = False + return -1.0 def _get_gpu_usage() -> float: - # NVIDIA + # pynvml — subprocess-free, works everywhere if installed try: - r = subprocess.run( - ["nvidia-smi", "--query-gpu=utilization.gpu", "--format=csv,noheader,nounits"], - capture_output=True, text=True, timeout=2, - ) - if r.returncode == 0: - vals = [float(v.strip()) for v in r.stdout.strip().split("\n") if v.strip()] - return sum(vals) / len(vals) if vals else -1.0 + import pynvml # type: ignore + pynvml.nvmlInit() + h = pynvml.nvmlDeviceGetHandleByIndex(0) + return float(pynvml.nvmlDeviceGetUtilizationRates(h).gpu) except Exception: pass - # AMD (Linux) - if _OS == "Linux": - try: - r = subprocess.run( - ["rocm-smi", "--showuse", "--csv"], - capture_output=True, text=True, timeout=2, - ) - if r.returncode == 0: - for line in r.stdout.strip().split("\n"): - parts = line.split(",") - if len(parts) >= 2: - try: - return float(parts[1].strip().replace("%", "")) - except ValueError: - pass - except Exception: - pass - - return -1.0 + return _nvml_gpu() def _get_cpu_temp() -> float: - # psutil sensors (Linux + some Windows drivers) + # psutil — works on Linux; occasionally Windows with proper drivers try: temps = psutil.sensors_temperatures() for name in ["coretemp", "k10temp", "cpu_thermal", "acpitz", @@ -69,32 +96,14 @@ def _get_cpu_temp() -> float: except Exception: pass - # Windows — WMI thermal zone + # Windows: wmi module (pure Python COM, zero subprocess) if _OS == "Windows": try: - r = subprocess.run( - ["powershell", "-Command", - "(Get-WmiObject MSAcpi_ThermalZoneTemperature " - "-Namespace root/wmi).CurrentTemperature"], - capture_output=True, text=True, timeout=3, - ) - if r.returncode == 0 and r.stdout.strip(): - raw = float(r.stdout.strip().split("\n")[0]) - return (raw / 10.0) - 273.15 - except Exception: - pass - - # macOS — osx-cpu-temp (optional CLI tool) - if _OS == "Darwin": - try: - r = subprocess.run( - ["osx-cpu-temp"], capture_output=True, text=True, timeout=2, - ) - if r.returncode == 0: - import re - m = re.search(r"([\d.]+)", r.stdout) - if m: - return float(m.group(1)) + import wmi # type: ignore + w = wmi.WMI(namespace="root/wmi") + tz = w.MSAcpi_ThermalZoneTemperature() + if tz: + return (tz[0].CurrentTemperature / 10.0) - 273.15 except Exception: pass @@ -102,10 +111,7 @@ def _get_cpu_temp() -> float: def get_system_status() -> dict: - """ - Snapshot of current system metrics. - Used by the system_status tool so the user can ask verbally. - """ + """Snapshot of current system metrics for the system_status tool.""" cpu = psutil.cpu_percent(interval=0.2) ram = psutil.virtual_memory() temp = _get_cpu_temp() @@ -131,8 +137,7 @@ def get_system_status() -> dict: class SystemMonitor: """ Stateful monitor — cooldown state persists across session reconnections. - Call check() periodically; it returns a [SYSTEM_ALERT] string when a - threshold is exceeded, or None when everything is fine. + Call check() periodically; returns a [SYSTEM_ALERT] string or None. """ def __init__(self, thresholds: dict | None = None): @@ -147,10 +152,6 @@ class SystemMonitor: self._last_alert[key] = time.monotonic() def check(self) -> str | None: - """ - Reads current metrics synchronously. - Returns an instruction string for Jarvis to speak, or None. - """ try: cpu = psutil.cpu_percent(interval=None) ram = psutil.virtual_memory().percent @@ -161,7 +162,6 @@ class SystemMonitor: alerts: list[str] = [] - # CPU — require consecutive readings to avoid transient spikes if cpu >= self.thresholds["cpu"]: self._cpu_streak += 1 if self._cpu_streak >= _CPU_STREAK and self._can_alert("cpu"): diff --git a/actions/web_search.py b/actions/web_search.py index cd29d10..c00d448 100644 --- a/actions/web_search.py +++ b/actions/web_search.py @@ -56,6 +56,29 @@ def _ddg_search(query: str, max_results: int = 6) -> list[dict]: return results +def _ddg_news(query: str, max_results: int = 8) -> list[dict]: + """DDG news search — returns actual articles, not website homepages.""" + try: + from ddgs import DDGS + except ImportError: + from duckduckgo_search import DDGS + + results = [] + try: + with DDGS() as ddgs: + for r in ddgs.news(query, max_results=max_results): + results.append({ + "title": r.get("title", ""), + "snippet": r.get("body", ""), + "url": r.get("url", ""), + "source": r.get("source", ""), + }) + except Exception as e: + print(f"[WebSearch] ⚠️ DDG news() failed ({e}) — falling back to text search") + results = _ddg_search(query, max_results=max_results) + return results + + def _format_ddg(query: str, results: list[dict]) -> str: if not results: return f"No results found for: {query}" @@ -69,6 +92,25 @@ def _format_ddg(query: str, results: list[dict]) -> str: return "\n".join(lines).strip() +def _format_news(query: str, results: list[dict]) -> str: + if not results: + return f"No news found for: {query}" + + lines = [f"Latest news: {query}\n"] + for i, r in enumerate(results, 1): + title = r.get("title", "") + if not title: + continue + src = f" [{r['source']}]" if r.get("source") else "" + lines.append(f"{i}. {title}{src}") + if r.get("snippet"): + lines.append(f" {r['snippet'][:140]}") + if r.get("url"): + lines.append(f" {r['url']}") + lines.append("") + return "\n".join(lines).strip() + + # ── Briefing helper ──────────────────────────────────────────────────────────── def _gemini_headlines(n: int = 5) -> tuple[list[str], str]: @@ -121,14 +163,52 @@ def _search(query: str) -> str: def _news(query: str) -> str: - """Latest news — forces recency in the query.""" - news_query = f"latest news today: {query}" if query else "top world news today" - try: - return _gemini_search(news_query) - except Exception as e: - print(f"[WebSearch] ⚠️ News Gemini failed ({e}) — DDG fallback...") - results = _ddg_search(news_query, max_results=8) - return _format_ddg(news_query, results) + """ + Runs Gemini grounded search AND DDG news in parallel. + Returns whichever delivers a valid result first; cancels the other. + """ + import threading + + gemini_query = f"latest news today: {query}" if query else "top world news today" + ddg_query = query if query else "world news today" + + result_box = [None] # first valid result lands here + lock = threading.Lock() + done_evt = threading.Event() + failures = [0] + + def _store(r: str) -> None: + if r and len(r) > 60: + with lock: + if result_box[0] is None: + result_box[0] = r + done_evt.set() + else: + with lock: + failures[0] += 1 + if failures[0] >= 2: # both failed — unblock caller + done_evt.set() + + def _try_gemini(): + try: + _store(_gemini_search(gemini_query)) + except Exception as e: + print(f"[WebSearch] ⚠️ Gemini news failed ({e})") + _store("") + + def _try_ddg(): + try: + results = _ddg_news(ddg_query, max_results=8) + _store(_format_news(ddg_query, results)) + except Exception as e: + print(f"[WebSearch] ⚠️ DDG news failed ({e})") + _store("") + + threading.Thread(target=_try_gemini, daemon=True).start() + threading.Thread(target=_try_ddg, daemon=True).start() + + done_evt.wait(timeout=10.0) + return result_box[0] or f"No news found for: {query}" def _research(query: str) -> str: diff --git a/config/jarvis.ico b/config/jarvis.ico new file mode 100644 index 0000000..b40500f Binary files /dev/null and b/config/jarvis.ico differ diff --git a/core/prompt.txt b/core/prompt.txt index 8bf485b..ee85c43 100644 --- a/core/prompt.txt +++ b/core/prompt.txt @@ -3,15 +3,14 @@ IDENTITY: Efficient, professional, direct assistant. No fluff. EXECUTION RULES: -Vision (screen_process): Say one curious sentence, then total silence (Vision module takes over). +Vision (screen_process): Call the tool ONCE per user request. Never call it again due to echo, ambient sound, or uncertainty. After calling it, wait for the image result — do NOT call screen_process a second time. One-Call Policy: Never guess. Call tools exactly once. No retries. -Briefing: 3-4 sentences minimum. Report results briefly and always try to help the user. Memory: Store critical user preferences/context automatically. Exit: Only call shutdown_jarvis if session termination is explicit. -Response time: Respond as fast as you can -Always call sir to user. -Act: Always act like Jarvis. If user says "Daddy's home" and other sentences like in the Iron Man movie, be like a real friendly assistant like Jarvis(like saying: "welcome home sir" and then continue to speak to help to user according to his text). -Always create 3-4 sentence minimum. Never less. +Response time: Respond as fast as you can. +ADDRESS: When speaking Turkish → always say "efendim". When speaking English → say "sir". Never mix languages. +Act: Always act like Jarvis from Iron Man — professional, efficient, slightly witty. ("Welcome home, efendim" etc.) +Length: Match response length to the task. Briefing = short. Complex analysis = thorough. TOOL ROUTING: @@ -36,4 +35,11 @@ Messages starting with [STARTUP_BRIEFING] are internal instructions for the morn Follow them exactly. Never read the instruction text aloud. +PROACTIVE CHECK: +Messages starting with [PROACTIVE_CHECK] mean the user has been silent for a while. +Read the time and memory context carefully. +Say something genuinely useful, timely, or caring in 1-3 short sentences. +Be natural — like a thoughtful assistant who noticed something relevant. +Never read the [PROACTIVE_CHECK] tag aloud. Do NOT call any tools during a proactive check. + CRITICAL: Speak/Take action immediately based on available info. Assume and proceed. \ No newline at end of file diff --git a/main.py b/main.py index a3f9eb4..a6cac01 100644 --- a/main.py +++ b/main.py @@ -1,6 +1,24 @@ +import platform as _platform +import subprocess as _subprocess + +# ── Nuclear: force CREATE_NO_WINDOW on EVERY subprocess call on Windows ─────── +# This patches Popen itself, so no per-file flag is needed anywhere. +if _platform.system() == "Windows": + _OrigPopen = _subprocess.Popen + + class _Popen(_OrigPopen): + def __init__(self, args, **kw): + kw["creationflags"] = kw.get("creationflags", 0) | _subprocess.CREATE_NO_WINDOW + kw.pop("startupinfo", None) # drop any stale/shared STARTUPINFO + super().__init__(args, **kw) + + _subprocess.Popen = _Popen +# ───────────────────────────────────────────────────────────────────────────── + import asyncio import re import threading +import time import json import sys import traceback @@ -22,7 +40,7 @@ from actions.weather_report import weather_action from actions.send_message import send_message from actions.reminder import reminder from actions.computer_settings import computer_settings -from actions.screen_processor import screen_process +from actions.screen_processor import _capture_camera, _capture_screen from actions.youtube_video import youtube_video from actions.desktop import desktop_control from actions.browser_control import browser_control @@ -33,6 +51,7 @@ from actions.web_search import web_search as web_search_action from actions.computer_control import computer_control from actions.game_updater import game_updater from actions.system_monitor import SystemMonitor, get_system_status +from actions.proactive import ProactiveEngine def get_base_dir(): @@ -181,11 +200,12 @@ TOOL_DECLARATIONS = [ { "name": "screen_process", "description": ( - "Captures and analyzes the screen or webcam image. " + "Captures the screen or webcam image and lets you analyze it. " "MUST be called when user asks what is on screen, what you see, " - "analyze my screen, look at camera, etc. " + "look at camera, analyze my screen, etc. " "You have NO visual ability without this tool. " - "After calling this tool, stay SILENT — the vision module speaks directly." + "After the image is captured it is sent directly to you — describe what you see and answer the user's question. " + "When using camera: the live view stays open until user says close it or calls close_camera." ), "parameters": { "type": "OBJECT", @@ -196,6 +216,15 @@ TOOL_DECLARATIONS = [ "required": ["text"] } }, + { + "name": "close_camera", + "description": ( + "Closes the live camera view shown on screen. " + "Call when user says: close camera, stop camera, turn off camera, " + "kamerayı kapat, kapat, creepy, etc." + ), + "parameters": {"type": "OBJECT", "properties": {}, "required": []} + }, { "name": "computer_settings", "description": ( @@ -491,19 +520,28 @@ class JarvisLive: def __init__(self, ui: JarvisUI): self.ui = ui - self.session = None - self.audio_in_queue = None - self.out_queue = None - self._loop = None - self._is_speaking = False - self._speaking_lock = threading.Lock() - self._phone_active = False # True while phone mic is streaming; pauses PC mic - self.ui.on_text_command = self._on_text_command + self.session = None + self.audio_in_queue = None + self.out_queue = None + self._loop = None + self._is_speaking = False + self._speaking_lock = threading.Lock() + self._phone_active = False # True while phone mic is streaming; pauses PC mic + self._pending_vision = None # (img_bytes, mime_type, question, angle) to inject after tool response + self._vision_cam_active = False # True if camera was opened for vision → auto-close after response + self._vision_close_pending = False # True after vision injected; next turn_complete closes camera + self._vision_last_time = 0.0 # monotonic time of last screen_process call (cooldown guard) + self._vision_busy = False # True while a vision capture/inject cycle is in flight + self._interrupted = False # True while draining audio after user interrupt + self.ui.on_text_command = self._on_text_command self.ui.on_remote_clicked = self._make_remote_key + self.ui.on_interrupt = self.interrupt self._turn_done_event: asyncio.Event | None = None self._dashboard = None - self._briefing_sent = False # morning briefing fires once per process - self._sys_monitor = SystemMonitor() # persistent cooldown state + self._briefing_sent = False # morning briefing fires once per process + self._sys_monitor = SystemMonitor() # persistent cooldown state + self._proactive = ProactiveEngine() + self._last_user_speech = time.monotonic() # updated on every user utterance def _make_remote_key(self): """Called from Qt main thread when user presses Remote Control.""" @@ -537,6 +575,25 @@ class JarvisLive: elif not self.ui.muted: self.ui.set_state("LISTENING") + def interrupt(self) -> None: + """Stop JARVIS mid-speech: drain queued audio and open mic immediately.""" + self._interrupted = True + q = self.audio_in_queue + if q: + drained = 0 + while True: + try: + q.get_nowait() + drained += 1 + except Exception: + break + if drained: + print(f"[JARVIS] ✋ Interrupted — {drained} audio chunks discarded") + self.set_speaking(False) + if self._turn_done_event: + self._turn_done_event.clear() + self.ui.write_log("SYS: Interrupted — listening...") + def speak(self, text: str): if not self._loop or not self.session: return @@ -643,13 +700,40 @@ class JarvisLive: result = r or "Done." elif name == "screen_process": - threading.Thread( - target=screen_process, - kwargs={"parameters": args, "response": None, - "player": self.ui, "session_memory": None}, - daemon=True - ).start() - result = "Vision module activated. Stay completely silent — vision module will speak directly." + import time as _t_mod + _now = _t_mod.monotonic() + _cooldown = 4.0 # seconds — covers echo window after speaking ends + if self._vision_busy or (_now - self._vision_last_time) < _cooldown: + _wait = max(0, _cooldown - (_now - self._vision_last_time)) + print(f"[Vision] ⏳ Cooldown active ({_wait:.1f}s remaining) — ignoring duplicate call") + result = "Vision is still processing the previous request. I will not call this again." + else: + self._vision_busy = True + self._vision_last_time = _now + angle = args.get("angle", "screen").lower() + user_text = args.get("text", "What do you see?") + if angle == "camera": + img_b, mime_t = await loop.run_in_executor(None, _capture_camera) + self.ui.start_camera_stream() + self._vision_cam_active = True + print(f"[Vision] 📷 Camera: {len(img_b):,} bytes") + _stall = "camera" + else: + img_b, mime_t = await loop.run_in_executor(None, _capture_screen) + print(f"[Vision] 🖥️ Screen: {len(img_b):,} bytes") + _stall = "screen" + self._pending_vision = (img_b, mime_t, user_text, angle) + result = ( + f"[VISION_ACTIVE] {_stall.capitalize()} captured. " + f"Immediately say ONE natural sentence in the user's language " + f"(e.g. 'Looking at your {_stall} now, sir' / " + f"'{'Kameraya' if _stall == 'camera' else 'Ekrana'} bakıyorum efendim'). " + f"Do NOT describe or guess content — the actual image arrives in the NEXT message." + ) + + elif name == "close_camera": + self.ui.stop_camera_stream() + result = "Camera closed." elif name == "computer_settings": r = await loop.run_in_executor(None, lambda: computer_settings(parameters=args, response=None, player=self.ui)) @@ -670,12 +754,12 @@ class JarvisLive: elif name == "web_search": r = await loop.run_in_executor(None, lambda: web_search_action(parameters=args, player=self.ui)) result = r or "Done." - # Mirror substantial results to the on-screen content panel - if r and len(r) > 120: - mode = args.get("mode", "search").upper() - query = args.get("query") or ", ".join(args.get("items", [])) - label = f"{mode} — {query[:38]}" if query else mode - self.ui.show_content(label, r) + # Mirror results to the on-screen content panel + _mode = args.get("mode", "search") + if r and not r.startswith("No results") and not r.startswith("Search failed"): + _query = args.get("query") or ", ".join(args.get("items", [])) + _label = f"{_mode.upper()} — {_query[:38]}" if _query else _mode.upper() + self.ui.show_content(_label, r) elif name == "file_processor": if not args.get("file_path") and self.ui.current_file: args["file_path"] = self.ui.current_file @@ -770,27 +854,44 @@ class JarvisLive: async for response in self.session.receive(): if response.data: - if self._turn_done_event and self._turn_done_event.is_set(): - self._turn_done_event.clear() - self.audio_in_queue.put_nowait(response.data) + if self._interrupted: + pass # discard: interrupted + else: + if self._turn_done_event and self._turn_done_event.is_set(): + self._turn_done_event.clear() + # Split into ~50 ms chunks so interrupt() stops audio within 50 ms + # (24000 Hz × 2 bytes/sample × 0.05 s = 2400 bytes per slice) + _audio_data = response.data + _SLICE = 2400 + for _i in range(0, len(_audio_data), _SLICE): + self.audio_in_queue.put_nowait(_audio_data[_i : _i + _SLICE]) if response.server_content: sc = response.server_content if sc.output_transcription and sc.output_transcription.text: txt = _clean_transcript(sc.output_transcription.text) - if txt: + if txt and txt != (out_buf[-1] if out_buf else ""): out_buf.append(txt) if sc.input_transcription and sc.input_transcription.text: txt = _clean_transcript(sc.input_transcription.text) if txt: in_buf.append(txt) + self._last_user_speech = time.monotonic() if sc.turn_complete: if self._turn_done_event: self._turn_done_event.set() + # If this turn_complete ends an interrupted response, clear the + # flag and skip all further processing for that turn. + if self._interrupted: + self._interrupted = False + in_buf = [] + out_buf = [] + continue + full_in = " ".join(in_buf).strip() if full_in: self.ui.write_log(f"You: {full_in}") @@ -813,6 +914,37 @@ class JarvisLive: })) out_buf = [] + # Vision injection: model finished tool-response turn → now send the image + if self._pending_vision and self.session: + import base64 as _b64 + img_b, mime_t, question, angle = self._pending_vision + self._pending_vision = None + b64 = _b64.b64encode(img_b).decode("ascii") + print(f"[Vision] 📤 {len(img_b):,} bytes (angle={angle}) → main session") + await self.session.send_client_content( + turns={"parts": [ + {"inline_data": {"mime_type": mime_t, "data": b64}}, + {"text": question}, + ]}, + turn_complete=True, + ) + # Mark next turn_complete behaviour depending on angle + if self._vision_cam_active: + # Camera: keep busy until JARVIS finishes speaking the answer + self._vision_cam_active = False + self._vision_close_pending = True + else: + # Screen-only: no camera to close; release busy flag now + self._vision_busy = False + elif self._vision_close_pending: + # This turn_complete IS the vision answer — close camera + release busy flag + self._vision_close_pending = False + self._vision_busy = False + async def _cam_close(): + await asyncio.sleep(2.0) + self.ui.stop_camera_stream() + asyncio.create_task(_cam_close()) + if response.tool_call: fn_responses = [] for fc in response.tool_call.function_calls: @@ -855,7 +987,10 @@ class JarvisLive: self._turn_done_event.clear() continue self.set_speaking(True) - await asyncio.to_thread(stream.write, chunk) + try: + await asyncio.to_thread(stream.write, chunk) + except (RuntimeError, asyncio.CancelledError): + break # executor shutting down — exit cleanly except Exception as e: print(f"[JARVIS] ❌ Play: {e}") raise @@ -890,21 +1025,16 @@ class JarvisLive: from datetime import datetime time_str = datetime.now().strftime("%H:%M") - # ── Phase 1: instant greeting — zero data needed ────────────────────── - p1_lines = [ - "[STARTUP_GREETING] Greet the user immediately. Keep it to 1-2 short sentences.", - f"Current time: {time_str}.", - "- Say hello and mention the time naturally.", - "- Say you are checking today's headlines and will share them in a moment.", - "- Do NOT call any tools. Do NOT say [STARTUP_GREETING].", - "- Respond in " - + (f"language: {lang}." if lang else "the user's language (default: English)."), - ] - if name: - p1_lines.append(f"- Address the user as {name}.") + # ── Phase 1: instant greeting — one simple sentence ────────────────── + lang_clause = f" Respond in {lang}." if lang else "" + name_clause = f" Address the user as {name}." if name else "" + p1 = ( + f"Greet the user, mention it is {time_str}, and say you are fetching today's news headlines now. " + f"One short sentence only. Do not call any tools.{lang_clause}{name_clause}" + ) await self.session.send_client_content( - turns={"parts": [{"text": '\n'.join(p1_lines)}]}, + turns={"parts": [{"text": p1}]}, turn_complete=True, ) self.ui.write_log("SYS: Briefing phase 1 (greeting) sent.") @@ -920,74 +1050,27 @@ class JarvisLive: async def _briefing_news_phase(self, lang: str) -> None: """ - Fetches headlines (DDG → Gemini fallback), shows them on screen, - then injects a short 2-headline summary into the Live session. - Waits enough time for the phase-1 greeting to finish playing first. + Sends phase-2 (news) to Gemini ~1.5 s after phase-1 is dispatched so + Gemini starts working on it while phase-1 audio is still playing. """ - from actions.web_search import _ddg_search, _gemini_headlines + lang_str = f" Respond in {lang}." if lang else "" - fetch_start = asyncio.get_event_loop().time() - headlines: list[str] = [] - full_news = "" - - # 1) DDG — ~0.6 s when available - try: - results = await asyncio.wait_for( - asyncio.to_thread(_ddg_search, "world news today", 6), - timeout=4.0, - ) - if results: - headlines = [r["title"] for r in results if r.get("title")][:6] - full_news = "\n\n".join( - f"• {r.get('title','')}\n {r.get('snippet','')}\n {r.get('url','')}" - for r in results - ) - except Exception as e: - print(f"[Briefing] DDG: {e}") - - # 2) Gemini grounded search — reliable fallback - if not headlines: - try: - headlines, full_news = await asyncio.wait_for( - asyncio.to_thread(_gemini_headlines, 5), - timeout=8.0, - ) - except Exception as e: - print(f"[Briefing] Gemini headlines: {e}") - - # Show full list on screen immediately when data arrives - if full_news: - self.ui.show_content("NEWS — latest headlines", full_news) - - if not headlines or not self.session: - return - - # Ensure the phase-1 greeting (≈ 3 s of speech) has finished before we speak again - elapsed = asyncio.get_event_loop().time() - fetch_start - wait_more = max(0.0, 3.5 - elapsed) - if wait_more > 0: - await asyncio.sleep(wait_more) + # 1.5 s is enough for Gemini to finish generating phase-1 audio on its + # side (turn_complete) while the greeting is still being played locally. + await asyncio.sleep(1.5) if not self.session: return - headlines_text = "\n".join(f"{i+1}. {h}" for i, h in enumerate(headlines)) - p2_lines = [ - "[BRIEFING_NEWS] Today's headlines are already displayed on screen.", - "Data:", - headlines_text, - "", - "Voice rules:", - "- Mention ONLY 2 headlines — one short sentence each.", - "- Tell the user the full list is visible on screen.", - "- Ask if they need anything.", - "- Do NOT say [BRIEFING_NEWS].", - "- Respond in " - + (f"language: {lang}." if lang else "the user's language."), - ] + p2 = ( + "[BRIEFING] Call web_search with mode='news' and query='top world news today' " + "to find actual recent news articles with real event headlines (not just website names). " + "After the search, say ONE specific news event from the results in one sentence, " + f"then say the full list is displayed on screen.{lang_str}" + ) await self.session.send_client_content( - turns={"parts": [{"text": '\n'.join(p2_lines)}]}, + turns={"parts": [{"text": p2}]}, turn_complete=True, ) self.ui.write_log("SYS: Briefing phase 2 (news) sent.") @@ -1008,6 +1091,41 @@ class JarvisLive: except Exception as e: print(f"[Monitor] ⚠️ Could not send alert: {e}") + # ── Proactive mode ────────────────────────────────────────────────────────── + + async def _run_proactive_mode(self) -> None: + """ + Background task: periodically checks if the user has been silent long enough, + then hands time + memory context to Gemini so it can decide what (if anything) + to say proactively. No hardcoded rules — Gemini makes the call. + """ + while True: + await asyncio.sleep(60) # evaluate once per minute + + if not self.session: + continue + + with self._speaking_lock: + speaking = self._is_speaking + if speaking: + continue + + if not self._proactive.should_trigger(self._last_user_speech): + continue + + self._proactive.mark_triggered() + + try: + memory = await asyncio.to_thread(load_memory) + prompt = self._proactive.build_prompt(memory) + await self.session.send_client_content( + turns={"parts": [{"text": prompt}]}, + turn_complete=True, + ) + self.ui.write_log("SYS: Proactive check-in.") + except Exception as e: + print(f"[Proactive] ⚠️ {e}") + # ── Phone audio relay ──────────────────────────────────────────────────────── async def _relay_phone_audio(self) -> None: @@ -1067,11 +1185,6 @@ class JarvisLive: async def run(self): self._loop = asyncio.get_event_loop() - client = genai.Client( - api_key=_get_api_key(), - http_options={"api_version": "v1beta"} - ) - # Start dashboard (optional — needs: pip install fastapi "uvicorn[standard]" cryptography) try: from dashboard.server import DashboardServer @@ -1090,6 +1203,12 @@ class JarvisLive: self.ui.set_state("THINKING") config = self._build_config() + # Fresh client on every reconnect — avoids stale HTTP session state + client = genai.Client( + api_key=_get_api_key(), + http_options={"api_version": "v1beta"} + ) + async with ( client.aio.live.connect(model=LIVE_MODEL, config=config) as session, asyncio.TaskGroup() as tg, @@ -1099,6 +1218,14 @@ class JarvisLive: self.out_queue = asyncio.Queue(maxsize=200) self._turn_done_event = asyncio.Event() + # Reset transient state that must not carry over from a previous session + self._pending_vision = None + self._vision_cam_active = False + self._vision_close_pending = False + self._vision_busy = False + self._vision_last_time = 0.0 + self._interrupted = False + print("[JARVIS] Connected.") self.ui.set_state("LISTENING") self.ui.write_log("SYS: JARVIS online.") @@ -1111,6 +1238,7 @@ class JarvisLive: tg.create_task(self._receive_audio()) tg.create_task(self._play_audio()) tg.create_task(self._run_system_monitor()) + tg.create_task(self._run_proactive_mode()) if self._dashboard: tg.create_task(self._relay_phone_audio()) @@ -1119,9 +1247,45 @@ class JarvisLive: self._briefing_sent = True tg.create_task(self._send_startup_briefing()) - except Exception as e: - print(f"[JARVIS] Error: {e}") + except KeyboardInterrupt: + raise + except SystemExit: + raise + except BaseException as e: + # Catches both Exception and BaseExceptionGroup (Python 3.11+ + # TaskGroup raises BaseExceptionGroup when tasks are cancelled + # externally, which `except Exception` would miss, letting the + # exception escape the while-loop and causing asyncio.run() to + # start shutdown — resulting in "executor after shutdown" errors). + err_str = str(e) + print(f"[JARVIS] Error ({type(e).__name__}): {e}") traceback.print_exc() + + # Invalid API key — stop hammering the API, prompt re-configuration + if "API key not valid" in err_str or "1007" in err_str: + self.ui.write_log("ERR: API key invalid — please re-enter your key.") + self.ui.set_state("SLEEPING") + self.ui.prompt_reconfig() + while not self.ui._win._ready: + await asyncio.sleep(1) + print("[JARVIS] New API key saved — reconnecting...") + _conn_backoff = 3 + continue + + # Network / timeout errors — log clearly and back off + is_net_err = any(k in err_str for k in ( + "TimeoutError", "timed out", "getaddrinfo", "CancelledError", + "ConnectionRefusedError", "OSError", "Cannot connect", + )) + if is_net_err: + _conn_backoff = min(getattr(self, "_conn_backoff", 3) * 2, 60) + self._conn_backoff = _conn_backoff + self.ui.write_log( + f"NET: Bağlantı kurulamadı — {_conn_backoff}s sonra tekrar deneniyor. " + "(VPN gerekiyor olabilir)" + ) + else: + self._conn_backoff = 3 finally: self.session = None @@ -1131,8 +1295,9 @@ class JarvisLive: if self._dashboard: await self._dashboard.broadcast({"type": "status", "state": "sleeping"}) - print("[JARVIS] Reconnecting in 3s...") - await asyncio.sleep(3) + delay = getattr(self, "_conn_backoff", 3) + print(f"[JARVIS] Reconnecting in {delay}s...") + await asyncio.sleep(delay) def main(): ui = JarvisUI("face.png") diff --git a/readme.md b/readme.md index 0d829b8..6fa6670 100644 --- a/readme.md +++ b/readme.md @@ -1,86 +1,157 @@ -# 🤖 MARK XLVII (47) -### The Ultimate Cross-Platform Personal AI Assistant — By FatihMakes - -> 📺 **[Watch the full setup video on YouTube](https://www.youtube.com/watch?v=BhOsnGC_sAA)** - -A real-time voice AI that can hear, see, understand, and control your computer — on any OS. Supporting Windows, macOS, and Linux. Built with Gemini integration for maximum stability and performance, delivering zero subscriptions and total digital autonomy. - ---- - -## ✨ Overview - -MARK XLVII represents the pinnacle of the Jarvis series, evolving into a proactive, deeply integrated system. It bridges the gap between the operating system, real-time web intelligence, and hardware metrics. Through natural dialogue, Mark 47 monitors your hardware, prepares your day, and visualizes complex web data through an adaptive interface. - -It's not just an assistant — it's an extension of your digital life. - ---- - -## 🚀 Capabilities - -### Core Features -| Feature | Description | -|---|---| -| 🎙️ Real-time Voice | Ultra-low latency conversation in any language | -| 🖥️ System Control | Launch apps, manage files, execute terminal commands | -| 🧩 Autonomous Tasks | High-level planning for complex, multi-step goals | -| 👁️ Visual Awareness | Real-time screen processing and webcam vision | -| 🧠 Persistent Memory | Deeply remembers your projects, preferences, and personal context | -| ⌨️ Hybrid Input | Seamlessly switch between keyboard typing and voice commands | - ---- - -## 🆕 What's New in XLVII - -- 🌅 **Morning Briefing Mode** — Automatically triggers once on first boot to read the time, pull local/global news, and check memory for your city to deliver a personalized weather update. -- 🎛️ **Audio-Visual System Monitor** — Background telemetry checks CPU, RAM, GPU, and temps every 10 seconds, delivering localized voice warnings with a 5-minute cooldown when thresholds are breached. -- 🔍 **Advanced Multi-Modal Search** — Overhauled web search featuring specific modes (`news`, `research`, `price`, `compare`, `search`) that prioritize Gemini Grounded Search with an automatic DuckDuckGo fallback. -- 🖼️ **Dynamic Content Panel** — A new scrollable display layer beneath the HUD that automatically expands to render high-density web results (over 120 characters) with active timestamps. -- 🗣️ **Silent Language Memory** — Instantly and silently captures your spoken language to save it under `identity/language`, ensuring all subsequent boots and briefings adapt to your dialect automatically. - ---- - -## ⚡ Quick Start - -```bash -git clone [https://github.com/FatihMakes/Mark-XLVII.git](https://github.com/FatihMakes/Mark-XLVII.git) -cd Mark-XLVII -pip install -r requirements.txt -playwright install -python main.py -``` - -> ⚠️ **Installation Note:** To keep the repository lightweight, some OS-specific dependencies are not bundled in `requirements.txt`. If you run into a `ModuleNotFoundError`, simply install the missing package via `pip install ` for your specific system. - ---- - -## 📋 Requirements - -| Requirement | Details | -| --- | --- | -| **OS** | Windows 10/11, macOS, or Linux | -| **Python** | 3.11 or 3.12 | -| **Microphone** | Required for voice interaction | -| **API Key** | Free Gemini API key | - ---- - -## ⚠️ License - -Personal and non-commercial use only. -Licensed under **[Creative Commons BY-NC 4.0](https://creativecommons.org/licenses/by-nc/4.0/)**. - ---- - -## 👤 Connect with the Creator - -Engineered by a developer building a real-world JARVIS-style assistant. -⭐ **Star the repository to support the journey to Mark 100.** - -| Platform | Link | -| --- | --- | -| YouTube | [@FatihMakes](https://www.youtube.com/@FatihMakes) | -| Instagram | [@fatihmakes](https://www.instagram.com/fatihmakes) | - -``` - -``` \ No newline at end of file +# ⚙️ MARK XLVIII (48) +### The Ultimate Cross-Platform Personal AI Assistant — By FatihMakes + +> 📺 **[Watch the full setup video on YouTube](https://www.youtube.com/@FatihMakes)** + +A real-time voice AI that can hear, see, understand, and control your computer — on any OS. Supports Windows, macOS, and Linux. Built on the Gemini Live API for native audio streaming, delivering zero subscriptions and total digital autonomy. + +--- + +## ✨ Overview + +MARK XLVIII is a direct evolution of Mark XLVII — sharper, faster, and more natural to talk to. This build focused entirely on removing every friction point: the silence while JARVIS "thinks," the awkward double-responses, the interrupts that took four seconds to land, the news results that linked to homepages instead of articles. Everything that made the experience feel like software instead of an assistant has been addressed. + +It's not just an assistant — it's an extension of your digital life. + +--- + +## 🚀 Capabilities + +### Core Features +| Feature | Description | +|---|---| +| 🎙️ Real-time Voice | Ultra-low latency conversation in any language via Gemini Live API | +| 🖥️ System Control | Launch apps, adjust volume/brightness, WiFi, shortcuts, power — all by voice | +| 🧩 Autonomous Tasks | High-level planning for complex multi-step goals via agent mode | +| 👁️ Visual Awareness | Real-time screen capture and webcam vision piped into your main Gemini session | +| 🧠 Persistent Memory | Deeply remembers projects, preferences, and personal context across sessions | +| ⌨️ Hybrid Input | Seamlessly switch between keyboard typing and voice commands | +| 🌅 Morning Briefing | On first boot: greets you, reads the time, fetches live news headlines, and checks weather | +| 🔔 Proactive Check-ins | After 15 minutes of silence JARVIS checks context and offers something genuinely useful — no hardcoded rules, Gemini decides | +| 📊 Hardware Monitoring | Continuous CPU, RAM, GPU and temperature telemetry with localized voice alerts when thresholds are breached | +| 🌤️ Weather Report | Live weather data for your city, personalized from memory | +| 🗺️ Dynamic Content Panel | Scrollable display layer beneath the HUD that renders web results, news, and search data with timestamps | +| 🔍 Multi-Mode Web Search | `news` / `research` / `price` / `compare` / `search` — Gemini Grounded first, DDG fallback | +| ⏰ Smart Reminders | OS-native scheduled notifications (Windows Task Scheduler / macOS LaunchAgent / Linux systemd) | +| ✈️ Flight Finder | Live flight price and availability lookup | +| 🎮 Game Updater | Checks and triggers game updates on demand | +| 📂 File Processor | Read, summarize, and answer questions about local files | +| 💻 Code Helper | Inline code review, debugging, and generation | +| 🌐 Browser Control | Open URLs, navigate tabs, and interact with the browser by voice | +| 📨 Send Message | Compose and send messages through integrated messaging apps | +| 🎬 YouTube Control | Search, play, and control YouTube playback by voice | +| 🖱️ Desktop Control | Taskbar, window management, and desktop-level operations | +| 🧑‍💻 Silent Language Memory | Detects spoken language on first use and saves it silently — all future sessions and briefings adapt automatically | + +--- + +## 🆕 What's New in XLVIII + +### ✋ Instant Interrupt — ESC or Button +Press **Escape** or click the `INTERRUPT` button to cut JARVIS off mid-sentence and return to listening. Previously, cancelling a response required waiting for the full audio buffer to drain — sometimes four seconds. In Mark XLVIII, audio is split into ~50 ms chunks (2400 bytes each) so the interrupt fires within one slice. The interrupt drains the audio queue, sets a flag, and clears the turn — listening resumes in under 100 ms. + +### 👁️ Immediate Vision Acknowledgment +When you ask JARVIS to look at your screen or camera, it no longer goes silent while processing. The tool now instructs Gemini to immediately say a natural sentence ("Looking at your screen now, sir" / "Ekrana bakıyorum efendim") while the image capture runs. The actual analysis follows as a second response. No more awkward silence. + +### 📰 Parallel News Search — First Result Wins +News queries now run **Gemini Grounded Search and DuckDuckGo news simultaneously** in two daemon threads. Whichever delivers a valid result first wins; the other is silently discarded. Previously, a Gemini 503 error would stall the search for several seconds before falling back to DDG. Now the fallback happens in parallel — total news fetch time drops to whichever backend is fastest at that moment. + +### 🗞️ Real News Articles (Not Homepages) +DDG news search was previously using `ddgs.text()`, which returned website homepage URLs for news queries. Mark XLVIII switches to `ddgs.news()`, which returns actual article URLs, titles, snippets, and source names — exactly what you want in a news briefing. + +### 🌅 Two-Phase Startup Briefing — Runs Concurrently +The startup briefing now sends Phase 2 (news fetch) while Phase 1 audio (the greeting) is still playing. Previously, Phase 2 waited for Phase 1 to fully complete. The 1.5-second overlap means the news headline is ready by the time JARVIS finishes saying "Good morning." + +### 🔁 Smarter Reconnection — Exponential Backoff +Network timeouts now use exponential backoff: 3s → 6s → 12s → 60s (capped). Each retry shows a Turkish-language status message in the UI ("Bağlantı kurulamadı — Xs sonra tekrar deneniyor"). Previously, a dropped connection would loop tightly and show no useful information. + +### 🛡️ Vision Cooldown & Echo Guard +Screen capture is guarded against echo loops. If JARVIS speaks about the screen and the microphone picks up its own voice, a secondary `screen_process` call can be triggered. Mark XLVIII blocks duplicate calls with a 4-second cooldown and a `_vision_busy` flag. Both are fully reset when a new session connects — fixing a bug where the cooldown would persist across reconnects and block legitimate requests. + +### 🌐 Language-Aware Address — Never Mixed +The system prompt now enforces: **Turkish → efendim**, **English → sir**, never mixed. Mark XLVII's prompt said "Always call sir to user" which caused JARVIS to say "sir" mid-Turkish sentence. Fixed. + +### 🪟 Zero Terminal Windows +A subprocess monkey-patch at startup sets `CREATE_NO_WINDOW` on every child process launched by the app. No PowerShell, no CMD, no terminal flash — ever. Applies to all actions including reminders, system commands, and scheduler calls. + +### 🔄 Session State Isolation +All transient vision and interrupt flags (`_pending_vision`, `_vision_busy`, `_vision_cam_active`, `_vision_close_pending`, `_interrupted`) are fully reset whenever a new Gemini session connects. Previously, state from a crashed session could carry over and leave JARVIS in a broken state until restart. + +--- + +## ⚡ Quick Start + +```bash +git clone https://github.com/FatihMakes/Mark-XLVIII.git +cd Mark-XLVIII +pip install -r requirements.txt +python main.py +``` + +> ⚠️ **Installation Note:** Some OS-specific dependencies are not bundled in `requirements.txt` to keep the repo lightweight. If you hit a `ModuleNotFoundError`, install the missing package with `pip install `. + +--- + +## 📋 Requirements + +| Requirement | Details | +| --- | --- | +| **OS** | Windows 10/11, macOS, or Linux | +| **Python** | 3.11 or 3.12 | +| **Microphone** | Required for voice interaction | +| **API Key** | Free Gemini API key (`config/api_keys.json`) | + +--- + +## 🗂️ Project Structure + +``` +Mark XLVIII/ +├── main.py # Core loop — Gemini Live session, audio I/O, tool dispatch +├── ui.py # PyQt6 HUD — waveform, log panel, interrupt button, camera feed +├── setup.py # First-run configuration wizard +├── actions/ +│ ├── web_search.py # Gemini + DDG parallel search (news, research, price, compare) +│ ├── screen_processor.py # Screen capture & webcam vision via Gemini Live +│ ├── reminder.py # OS-native scheduled notifications +│ ├── system_monitor.py # CPU / RAM / GPU / temperature telemetry +│ ├── computer_settings.py # Volume, brightness, WiFi, power +│ ├── computer_control.py # Keyboard shortcuts, mouse, window management +│ ├── open_app.py # Application launcher +│ ├── browser_control.py # Web browser control +│ ├── file_controller.py # File system operations +│ ├── file_processor.py # Document reading and summarization +│ ├── send_message.py # Messaging integration +│ ├── weather_report.py # Live weather data +│ ├── flight_finder.py # Flight search +│ ├── youtube_video.py # YouTube playback control +│ ├── game_updater.py # Game update management +│ ├── code_helper.py # Code review and generation +│ ├── dev_agent.py # Developer task agent +│ ├── desktop.py # Desktop and taskbar control +│ └── proactive.py # Proactive silence-break suggestions +├── memory/ # Persistent key-value memory store +├── core/ +│ └── prompt.txt # JARVIS personality and tool-routing rules +└── config/ + └── api_keys.json # API key and system configuration +``` + +--- + +## ⚠️ License + +Personal and non-commercial use only. +Licensed under **[Creative Commons BY-NC 4.0](https://creativecommons.org/licenses/by-nc/4.0/)**. + +--- + +## 👤 Connect with the Creator + +Engineered by a developer building a real-world JARVIS-style assistant. +⭐ **Star the repository to support the journey to Mark 100.** + +| Platform | Link | +| --- | --- | +| YouTube | [@FatihMakes](https://www.youtube.com/@FatihMakes) | +| Instagram | [@fatihmakes](https://www.instagram.com/fatihmakes) | diff --git a/ui.py b/ui.py index bcac795..d92bae3 100644 --- a/ui.py +++ b/ui.py @@ -13,6 +13,11 @@ from pathlib import Path import psutil +if platform.system() == "Windows": + _WIN_HIDE: dict = {"creationflags": subprocess.CREATE_NO_WINDOW} +else: + _WIN_HIDE: dict = {} + from PyQt6.QtCore import ( QEasingCurve, QMimeData, QObject, QPointF, QRectF, QSize, Qt, QTimer, QUrl, pyqtSignal, @@ -24,8 +29,8 @@ from PyQt6.QtGui import ( ) from PyQt6.QtWidgets import ( QApplication, QFileDialog, QFrame, QHBoxLayout, QLabel, QLineEdit, - QMainWindow, QPushButton, QScrollArea, QSizePolicy, QTextEdit, - QVBoxLayout, QWidget, QProgressBar, + QMainWindow, QPushButton, QScrollArea, QSizePolicy, QSplitter, + QStackedWidget, QTextEdit, QVBoxLayout, QWidget, QProgressBar, ) def _base_dir() -> Path: @@ -72,6 +77,51 @@ class C: def qcol(h: str, a: int = 255) -> QColor: c = QColor(h); c.setAlpha(a); return c + +# ── Windows GPU via NVML DLL (no subprocess, no console window) ────────────── +_nvml_lib: object = None # cached ctypes DLL +_nvml_ok: object = None # None=untested, True=works, False=unavailable + + +def _nvml_gpu_windows() -> float: + """Return NVIDIA GPU utilisation % using nvml.dll directly — zero subprocess.""" + global _nvml_lib, _nvml_ok + if _nvml_ok is False: + return -1.0 + try: + import ctypes + + class _Util(ctypes.Structure): + _fields_ = [("gpu", ctypes.c_uint), ("memory", ctypes.c_uint)] + + if _nvml_lib is None: + for dll_name in ("nvml", r"C:\Windows\System32\nvml.dll"): + try: + lib = ctypes.WinDLL(dll_name) + lib.nvmlInit_v2() + _nvml_lib = lib + break + except Exception: + continue + + if _nvml_lib is None: + import pynvml # type: ignore + pynvml.nvmlInit() + h = pynvml.nvmlDeviceGetHandleByIndex(0) + _nvml_ok = True + return float(pynvml.nvmlDeviceGetUtilizationRates(h).gpu) + + dev = ctypes.c_void_p() + _nvml_lib.nvmlDeviceGetHandleByIndex_v2(0, ctypes.byref(dev)) + util = _Util() + _nvml_lib.nvmlDeviceGetUtilizationRates(dev, ctypes.byref(util)) + _nvml_ok = True + return float(util.gpu) + except Exception: + _nvml_ok = False + return -1.0 + + class _SysMetrics: def __init__(self): self.cpu = 0.0 @@ -122,112 +172,65 @@ class _SysMetrics: self.tmp = tmp def _get_gpu(self) -> float: - # NVIDIA + # pynvml — subprocess-free, works on all platforms if installed try: - r = subprocess.run( - ["nvidia-smi", "--query-gpu=utilization.gpu", - "--format=csv,noheader,nounits"], - capture_output=True, text=True, timeout=2 - ) - if r.returncode == 0: - vals = [float(v.strip()) for v in r.stdout.strip().split("\n") if v.strip()] - if vals: - return sum(vals) / len(vals) + import pynvml # type: ignore + pynvml.nvmlInit() + h = pynvml.nvmlDeviceGetHandleByIndex(0) + return float(pynvml.nvmlDeviceGetUtilizationRates(h).gpu) except Exception: pass - # AMD (Linux) - if _OS == "Linux": - try: - r = subprocess.run( - ["rocm-smi", "--showuse", "--csv"], - capture_output=True, text=True, timeout=2 - ) - if r.returncode == 0: - for line in r.stdout.strip().split("\n"): - parts = line.split(",") - if len(parts) >= 2: - try: - return float(parts[1].strip().replace("%", "")) - except ValueError: - pass - except Exception: - pass + # Windows: nvml.dll via ctypes (already cached in _nvml_gpu_windows) + if _OS == "Windows": + return _nvml_gpu_windows() - # Intel GPU (Linux) - try: - r = subprocess.run( - ["intel_gpu_top", "-J", "-s", "500"], - capture_output=True, text=True, timeout=1 - ) - if r.returncode == 0 and "Render/3D" in r.stdout: - import re - m = re.search(r'"busy":\s*([\d.]+)', r.stdout) - if m: - return float(m.group(1)) - except Exception: - pass + # Linux / macOS: libnvidia-ml shared lib via ctypes + try: + import ctypes + _lib = "libnvidia-ml.so.1" if _OS == "Linux" else "libnvidia-ml.dylib" - # macOS — powermetrics (GPU Engine) - if _OS == "Darwin": - try: - r = subprocess.run( - ["sudo", "-n", "powermetrics", "-n", "1", "-i", "500", - "--samplers", "gpu_power"], - capture_output=True, text=True, timeout=2 - ) - if r.returncode == 0 and "GPU" in r.stdout: - import re - m = re.search(r'GPU\s+Active:\s+([\d.]+)%', r.stdout) - if m: - return float(m.group(1)) - except Exception: - pass + class _Util(ctypes.Structure): + _fields_ = [("gpu", ctypes.c_uint), ("memory", ctypes.c_uint)] - return -1.0 + nv = ctypes.CDLL(_lib) + nv.nvmlInit_v2() + dev = ctypes.c_void_p() + nv.nvmlDeviceGetHandleByIndex_v2(0, ctypes.byref(dev)) + u = _Util() + nv.nvmlDeviceGetUtilizationRates(dev, ctypes.byref(u)) + return float(u.gpu) + except Exception: + pass + + return -1.0 # N/A — zero subprocess on all platforms def _get_temp(self) -> float: + # psutil — works on Linux; occasionally Windows with driver support try: temps = psutil.sensors_temperatures() - candidates = ["coretemp", "k10temp", "cpu_thermal", "acpitz", - "cpu-thermal", "zenpower", "it8688"] - for name in candidates: - if name in temps: - entries = temps[name] - if entries: - return entries[0].current + for name in ["coretemp", "k10temp", "cpu_thermal", "acpitz", + "cpu-thermal", "zenpower", "it8688"]: + if name in temps and temps[name]: + return temps[name][0].current for entries in temps.values(): if entries: return entries[0].current except Exception: pass - if _OS == "Darwin": - try: - r = subprocess.run( - ["osx-cpu-temp"], capture_output=True, text=True, timeout=2 - ) - if r.returncode == 0: - import re - m = re.search(r"([\d.]+)", r.stdout) - if m: - return float(m.group(1)) - except Exception: - pass + # Windows: wmi module (pure Python COM, zero subprocess) if _OS == "Windows": try: - r = subprocess.run( - ["powershell", "-Command", - "(Get-WmiObject MSAcpi_ThermalZoneTemperature -Namespace root/wmi).CurrentTemperature"], - capture_output=True, text=True, timeout=3 - ) - if r.returncode == 0 and r.stdout.strip(): - raw = float(r.stdout.strip().split("\n")[0]) - return (raw / 10.0) - 273.15 + import wmi # type: ignore + w = wmi.WMI(namespace="root/wmi") + tz = w.MSAcpi_ThermalZoneTemperature() + if tz: + return (tz[0].CurrentTemperature / 10.0) - 273.15 except Exception: pass - return -1.0 + return -1.0 # N/A — zero subprocess on all platforms def snapshot(self) -> dict: with self._lock: @@ -855,6 +858,73 @@ class _DropCanvas(QWidget): z.mousePressEvent(e) +class _CameraPreview(QWidget): + """Floating overlay that briefly shows what the camera captured.""" + + _W, _H = 244, 188 + + def __init__(self, parent=None): + super().__init__(parent) + self.setAttribute(Qt.WidgetAttribute.WA_StyledBackground, True) + self.setStyleSheet(f""" + _CameraPreview {{ + background: rgba(0, 6, 10, 242); + border: 1px solid {C.PRI}; + border-radius: 6px; + }} + """) + self.setFixedWidth(self._W) + + lay = QVBoxLayout(self) + lay.setContentsMargins(6, 5, 6, 6) + lay.setSpacing(4) + + hdr = QHBoxLayout() + title = QLabel("◈ VISUAL INPUT") + title.setFont(QFont("Courier New", 7, QFont.Weight.Bold)) + title.setStyleSheet(f"color: {C.PRI}; background: transparent;") + hdr.addWidget(title) + hdr.addStretch() + close_btn = QPushButton("✕") + close_btn.setFixedSize(16, 16) + close_btn.setFont(QFont("Courier New", 8)) + close_btn.setStyleSheet( + f"color: {C.TEXT_DIM}; background: transparent; border: none;" + ) + close_btn.setCursor(Qt.CursorShape.PointingHandCursor) + close_btn.clicked.connect(self.hide) + hdr.addWidget(close_btn) + lay.addLayout(hdr) + + self._img_lbl = QLabel() + self._img_lbl.setAlignment(Qt.AlignmentFlag.AlignCenter) + self._img_lbl.setStyleSheet("background: transparent;") + lay.addWidget(self._img_lbl) + + self._timer = QTimer(self) + self._timer.setSingleShot(True) + self._timer.timeout.connect(self.hide) + + self.hide() + + def show_frame(self, img_bytes: bytes) -> None: + px = QPixmap() + px.loadFromData(img_bytes) + if not px.isNull(): + max_w = self._W - 12 + scaled = px.scaled( + max_w, 160, + Qt.AspectRatioMode.KeepAspectRatio, + Qt.TransformationMode.SmoothTransformation, + ) + self._img_lbl.setPixmap(scaled) + self._img_lbl.setFixedSize(scaled.width(), scaled.height()) + self.adjustSize() + self.show() + self.raise_() + self._timer.start(6_000) # auto-dismiss after 6 s + + class SetupOverlay(QWidget): done = pyqtSignal(str, str) @@ -1216,10 +1286,15 @@ class MainWindow(QMainWindow): _log_sig = pyqtSignal(str) _state_sig = pyqtSignal(str) _content_sig = pyqtSignal(str, str) # (title, text) — thread-safe content display + _reconfig_sig = pyqtSignal() # trigger setup overlay from any thread + _camera_sig = pyqtSignal(bytes) # show camera frame preview (small overlay) + _cam_stream_sig = pyqtSignal(bool) # True=start live stream, False=stop + _cam_frame_sig = pyqtSignal(bytes) # live camera frame → HUD area def __init__(self, face_path: str): super().__init__() - self.setWindowTitle("J.A.R.V.I.S — MARK XLVII") + self._face_path = face_path + self.setWindowTitle("J.A.R.V.I.S — MARK XLVIII") self.setMinimumSize(_MIN_W, _MIN_H) self.resize(_DEFAULT_W, _DEFAULT_H) @@ -1229,9 +1304,10 @@ class MainWindow(QMainWindow): (screen.height() - _DEFAULT_H) // 2, ) - self.on_text_command = None + self.on_text_command = None self.on_remote_clicked = None # callable: () -> (url, key) | None - self._muted = False + self.on_interrupt = None # callable: () -> None — stop JARVIS mid-speech + self._muted = False self._current_file: str | None = None self._remote_overlay: RemoteKeyOverlay | None = None @@ -1251,18 +1327,66 @@ class MainWindow(QMainWindow): self._left_panel = self._build_left_panel() body.addWidget(self._left_panel, stretch=0) - # Center column: HUD on top + content panel below - _center = QWidget() - _center.setStyleSheet(f"background: {C.BG};") - _center_lay = QVBoxLayout(_center) - _center_lay.setContentsMargins(0, 0, 0, 0) - _center_lay.setSpacing(0) + # Center column: HUD + resizable content panel via QSplitter self.hud = HudCanvas(face_path) self.hud.setSizePolicy(QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Expanding) - _center_lay.addWidget(self.hud, stretch=1) self._content_panel = self._build_content_panel() - _center_lay.addWidget(self._content_panel) - body.addWidget(_center, stretch=5) + + # Live camera container — replaces HUD when camera stream is active + _cam_cont = QWidget() + _cam_cont.setStyleSheet("background: #000308;") + _cam_v = QVBoxLayout(_cam_cont) + _cam_v.setContentsMargins(0, 0, 0, 0) + _cam_v.setSpacing(0) + _cam_hdr = QHBoxLayout() + _cam_hdr.setContentsMargins(8, 5, 8, 5) + _cam_title = QLabel("◈ CAMERA FEED") + _cam_title.setFont(QFont("Courier New", 8, QFont.Weight.Bold)) + _cam_title.setStyleSheet(f"color: {C.PRI}; background: transparent;") + _cam_hdr.addWidget(_cam_title) + _cam_hdr.addStretch() + _cam_x = QPushButton("✕ CLOSE") + _cam_x.setFont(QFont("Courier New", 8, QFont.Weight.Bold)) + _cam_x.setCursor(Qt.CursorShape.PointingHandCursor) + _cam_x.setStyleSheet(f""" + QPushButton {{ + color: {C.TEXT_DIM}; background: transparent; + border: none; padding: 2px 6px; + }} + QPushButton:hover {{ color: {C.PRI}; }} + """) + _cam_x.clicked.connect(self.stop_camera_stream) + _cam_hdr.addWidget(_cam_x) + _cam_v.addLayout(_cam_hdr) + self._cam_live_lbl = QLabel() + self._cam_live_lbl.setAlignment(Qt.AlignmentFlag.AlignCenter) + self._cam_live_lbl.setStyleSheet("background: transparent;") + self._cam_live_lbl.setSizePolicy( + QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Expanding + ) + _cam_v.addWidget(self._cam_live_lbl, stretch=1) + + # Stack: 0 = animated HUD, 1 = live camera + self._hud_cam_stack = QStackedWidget() + self._hud_cam_stack.addWidget(self.hud) + self._hud_cam_stack.addWidget(_cam_cont) + + self._center_split = QSplitter(Qt.Orientation.Vertical) + self._center_split.setStyleSheet(f""" + QSplitter::handle {{ + background: {C.BORDER}; + height: 4px; + }} + QSplitter::handle:hover {{ + background: {C.PRI_DIM}; + }} + """) + self._center_split.addWidget(self._hud_cam_stack) + self._center_split.addWidget(self._content_panel) + self._center_split.setStretchFactor(0, 3) + self._center_split.setStretchFactor(1, 1) + self._center_split.setCollapsible(0, False) + body.addWidget(self._center_split, stretch=5) self._right_panel = self._build_right_panel() body.addWidget(self._right_panel, stretch=0) @@ -1284,6 +1408,14 @@ class MainWindow(QMainWindow): self._log_sig.connect(self._log.append_log) self._state_sig.connect(self._apply_state) self._content_sig.connect(self._show_content) + self._reconfig_sig.connect(self._show_setup) + self._camera_sig.connect(self._show_camera_frame) + self._cam_stream_sig.connect(self._on_cam_stream) + self._cam_frame_sig.connect(self._on_cam_frame) + self._cam_stop = threading.Event() + + # Camera preview overlay (child of central widget, positioned in resizeEvent) + self._cam_preview = _CameraPreview(self.centralWidget()) self._overlay: SetupOverlay | None = None self._ready = self._check_config() @@ -1294,6 +1426,347 @@ class MainWindow(QMainWindow): sc_mute.activated.connect(self._toggle_mute) sc_full = QShortcut(QKeySequence("F11"), self) sc_full.activated.connect(self._toggle_fullscreen) + sc_intr = QShortcut(QKeySequence("Escape"), self) + sc_intr.activated.connect(self._do_interrupt) + + def _show_camera_frame(self, img_bytes: bytes): + """Slot — display camera preview overlay (main thread).""" + self._cam_preview.show_frame(img_bytes) + cw = self.centralWidget() + pw = _CameraPreview._W + ph = self._cam_preview.height() + self._cam_preview.setGeometry( + cw.width() - _RIGHT_W - pw - 12, + cw.height() - ph - 28, + pw, ph, + ) + + # --- Live camera stream in HUD area ------------------------------------ + def _on_cam_stream(self, start: bool) -> None: + if start: + self._hud_cam_stack.setCurrentIndex(1) + else: + self._hud_cam_stack.setCurrentIndex(0) + self._cam_live_lbl.clear() + + def _on_cam_frame(self, data: bytes) -> None: + px = QPixmap() + px.loadFromData(data) + if not px.isNull(): + w, h = self._cam_live_lbl.width(), self._cam_live_lbl.height() + if w > 1 and h > 1: + self._cam_live_lbl.setPixmap( + px.scaled(w, h, + Qt.AspectRatioMode.KeepAspectRatio, + Qt.TransformationMode.SmoothTransformation) + ) + + def start_camera_stream(self) -> None: + self._cam_stop.clear() + self._cam_stream_sig.emit(True) + t = threading.Thread(target=self._cam_loop, daemon=True, name="cam-stream") + t.start() + + def _cam_loop(self) -> None: + try: + import cv2 + # Reuse camera index detected by screen_processor (cached in api_keys.json) + cam_idx = 0 + try: + import json as _j + cfg = _j.loads((CONFIG_DIR / "api_keys.json").read_text()) + cam_idx = int(cfg.get("camera_index", 0)) + except Exception: + pass + try: + backend = cv2.CAP_DSHOW if _OS == "Windows" else cv2.CAP_ANY + except AttributeError: + backend = 0 + cap = cv2.VideoCapture(cam_idx, backend) + if not cap.isOpened(): + cap = cv2.VideoCapture(0) + if not cap.isOpened(): + return + # warm-up frames + for _ in range(5): + cap.read() + while not self._cam_stop.wait(0.033) and cap.isOpened(): + ret, frame = cap.read() + if ret and frame is not None: + _, buf = cv2.imencode(".jpg", frame, [cv2.IMWRITE_JPEG_QUALITY, 65]) + self._cam_frame_sig.emit(buf.tobytes()) + cap.release() + except Exception as e: + print(f"[Camera] Stream error: {e}") + finally: + self._cam_stream_sig.emit(False) + + def stop_camera_stream(self) -> None: + self._cam_stop.set() + + # ------------------------------------------------------------------ + # Icon generation — arc-reactor style, rendered with Pillow + # ------------------------------------------------------------------ + @staticmethod + def _build_jarvis_icon(out_path: Path) -> bool: + """ + Render a JARVIS arc-reactor icon at 4× resolution and downsample + for crisp results at all sizes. Saves a multi-res .ico to out_path. + Returns True on success. + """ + try: + import math + import PIL.Image + import PIL.ImageDraw + import PIL.ImageFilter + except ImportError: + return False + + CYAN = (0, 212, 255) + DIM = (0, 100, 140) + DARK = (0, 6, 10) + GLOW = (0, 160, 200) + WHITE = (220, 240, 255) + + def _render(sz: int) -> PIL.Image.Image: + S = sz * 4 # draw at 4× then downscale + img = PIL.Image.new("RGBA", (S, S), (0, 0, 0, 0)) + d = PIL.ImageDraw.Draw(img) + cx = cy = S // 2 + + # ── filled background circle ────────────────────────────────── + R = S // 2 - 2 + d.ellipse([cx-R, cy-R, cx+R, cy+R], fill=(*DARK, 255)) + + # ── outer border ring ───────────────────────────────────────── + lw = max(2, S // 40) + d.ellipse([cx-R, cy-R, cx+R, cy+R], + outline=(*CYAN, 220), width=lw) + + # ── mid decorative ring ─────────────────────────────────────── + R2 = int(R * 0.72) + d.ellipse([cx-R2, cy-R2, cx+R2, cy+R2], + outline=(*DIM, 180), width=max(1, lw // 2)) + + # ── 6 radial spokes (hex bolt) ──────────────────────────────── + R_inner = int(R * 0.30) + R_outer = int(R * 0.62) + spoke_w = max(1, S // 80) + for i in range(6): + angle = math.radians(i * 60 - 30) + x1 = cx + int(R_inner * math.cos(angle)) + y1 = cy + int(R_inner * math.sin(angle)) + x2 = cx + int(R_outer * math.cos(angle)) + y2 = cy + int(R_outer * math.sin(angle)) + d.line([x1, y1, x2, y2], fill=(*GLOW, 200), width=spoke_w) + + # ── 6 tick marks on outer ring ──────────────────────────────── + for i in range(6): + angle = math.radians(i * 60) + for dr in range(lw * 2): + rx = (R - lw - dr) + d.point( + [cx + int(rx * math.cos(angle)), + cy + int(rx * math.sin(angle))], + fill=(*WHITE, 220), + ) + + # ── inner glowing ring ──────────────────────────────────────── + Ri = int(R * 0.26) + d.ellipse([cx-Ri, cy-Ri, cx+Ri, cy+Ri], + outline=(*CYAN, 255), width=max(2, lw)) + + # ── bright glow soft blur applied before core ───────────────── + # (draw a slightly larger cyan circle on a separate layer) + glow_layer = PIL.Image.new("RGBA", (S, S), (0, 0, 0, 0)) + gd = PIL.ImageDraw.Draw(glow_layer) + Rc = int(R * 0.13) + gd.ellipse([cx-Rc*2, cy-Rc*2, cx+Rc*2, cy+Rc*2], + fill=(*CYAN, 110)) + glow_layer = glow_layer.filter(PIL.ImageFilter.GaussianBlur(S // 14)) + img = PIL.Image.alpha_composite(img, glow_layer) + d = PIL.ImageDraw.Draw(img) + + # ── core dot ────────────────────────────────────────────────── + d.ellipse([cx-Rc, cy-Rc, cx+Rc, cy+Rc], fill=(*WHITE, 255)) + + # ── downscale to target size ────────────────────────────────── + return img.resize((sz, sz), PIL.Image.LANCZOS) + + try: + sizes = [256, 128, 64, 48, 32, 16] + frames = [_render(s) for s in sizes] + frames[0].save( + out_path, + format="ICO", + append_images=frames[1:], + sizes=[(s, s) for s in sizes], + ) + return True + except Exception as e: + print(f"[Shortcut] ⚠️ Icon generation failed: {e}") + return False + + @staticmethod + def _create_lnk_windows(lnk: str, target: str, args: str, + work_dir: str, icon_loc: str) -> None: + """ + Create a Windows .lnk shortcut WITHOUT launching PowerShell or cmd. + Tries win32com (pywin32) first; falls back to wscript.exe + VBScript. + wscript.exe is a GUI-mode host — it never opens a console window. + """ + # ── Option 1: pywin32 (pure Python COM, zero subprocess) ────────── + try: + from win32com.client import Dispatch # type: ignore + sh = Dispatch("WScript.Shell") + sc = sh.CreateShortCut(lnk) + sc.TargetPath = target + sc.Arguments = f'"{args}"' + sc.WorkingDirectory = work_dir + sc.Description = "J.A.R.V.I.S AI Assistant" + sc.IconLocation = icon_loc + sc.save() + return + except ImportError: + pass + + # ── Option 2: wscript.exe + VBScript (always available on Windows, + # GUI-mode executable — never opens a console window) ──────────── + vbs = "\n".join([ + 'Set ws = CreateObject("WScript.Shell")', + f'Set sc = ws.CreateShortcut("{lnk}")', + f'sc.TargetPath = "{target}"', + f'sc.Arguments = Chr(34) & "{args}" & Chr(34)', + f'sc.WorkingDirectory = "{work_dir}"', + 'sc.Description = "J.A.R.V.I.S AI Assistant"', + f'sc.IconLocation = "{icon_loc}"', + 'sc.Save', + ]) + import tempfile + fd, tmp = tempfile.mkstemp(suffix=".vbs") + try: + with os.fdopen(fd, "w", encoding="utf-8") as f: + f.write(vbs) + proc = subprocess.Popen( + ["wscript.exe", "/nologo", tmp], + creationflags=subprocess.DETACHED_PROCESS | subprocess.CREATE_NO_WINDOW, + ) + proc.wait(timeout=10) + finally: + try: + os.unlink(tmp) + except Exception: + pass + + def _create_desktop_shortcut(self): + """ + Create a desktop shortcut on Windows / macOS / Linux. + Never opens a terminal, console, or PowerShell window on any platform. + """ + import stat as _stat + script = Path(__file__).resolve().parent / "main.py" + python = Path(sys.executable) + desktop = Path.home() / "Desktop" + + # Arc-reactor icon (.ico — also exported as .png for Linux/macOS) + ico_path = Path(__file__).resolve().parent / "config" / "jarvis.ico" + if not ico_path.exists(): + self._build_jarvis_icon(ico_path) + + try: + _os = platform.system() + + # ── Windows ─────────────────────────────────────────────────────── + if _os == "Windows": + pythonw = python.parent / "pythonw.exe" + target = str(pythonw if pythonw.exists() else python) + lnk = str(desktop / "J.A.R.V.I.S.lnk") + icon_loc = str(ico_path) if ico_path.exists() else f"{target},0" + self._create_lnk_windows(lnk, target, str(script), + str(script.parent), icon_loc) + + # ── macOS — proper .app bundle (no Terminal window) ─────────────── + elif _os == "Darwin": + app = desktop / "J.A.R.V.I.S.app" + mac_dir = app / "Contents" / "MacOS" + res_dir = app / "Contents" / "Resources" + mac_dir.mkdir(parents=True, exist_ok=True) + res_dir.mkdir(exist_ok=True) + + # Launcher executable (bash — runs as background process, + # macOS does NOT open Terminal for executables inside .app bundles) + launcher = mac_dir / "JARVIS" + launcher.write_text( + "#!/usr/bin/env bash\n" + f'cd "{script.parent}"\n' + f'exec "{python}" "{script}"\n' + ) + launcher.chmod(launcher.stat().st_mode + | _stat.S_IEXEC | _stat.S_IXGRP | _stat.S_IXOTH) + + # Minimal Info.plist (required for .app recognition) + (app / "Contents" / "Info.plist").write_text( + '\n' + '\n' + '\n' + ' CFBundleExecutableJARVIS\n' + ' CFBundleIdentifier' + 'com.jarvis.assistant\n' + ' CFBundleNameJ.A.R.V.I.S\n' + ' CFBundlePackageTypeAPPL\n' + ' CFBundleVersion1.0\n' + '\n' + ) + + # Optional: copy icon as .icns (skip silently if Pillow is missing) + try: + import PIL.Image + icns = res_dir / "AppIcon.icns" + PIL.Image.open(ico_path).save(icns, format="ICNS") + # Inject icon reference into plist + plist = app / "Contents" / "Info.plist" + txt = plist.read_text() + plist.write_text( + txt.replace( + '', + ' CFBundleIconFile' + 'AppIcon\n\n', + ) + ) + except Exception: + pass # icon is optional + + # ── Linux — .desktop file (Terminal=false, no console) ──────────── + else: + # Export .ico → .png for better desktop integration + png_path = ico_path.with_suffix(".png") + if not png_path.exists() and ico_path.exists(): + try: + import PIL.Image + PIL.Image.open(ico_path).resize( + (256, 256), PIL.Image.LANCZOS + ).save(png_path, format="PNG") + except Exception: + png_path = ico_path # fallback to .ico + + icon_line = f"Icon={png_path}\n" if png_path.exists() else "" + desk = desktop / "J.A.R.V.I.S.desktop" + desk.write_text( + "[Desktop Entry]\n" + "Name=J.A.R.V.I.S\n" + f"Exec={python} {script}\n" + f"Path={script.parent}\n" + "Type=Application\n" + "Terminal=false\n" + "Categories=Utility;\n" + + icon_line + ) + desk.chmod(desk.stat().st_mode | 0o755) + + self._log.append_log("SYS: Desktop shortcut created.") + except Exception as e: + self._log.append_log(f"ERR: Shortcut failed — {e}") def _toggle_fullscreen(self): if self.isFullScreen(): @@ -1318,6 +1791,14 @@ class MainWindow(QMainWindow): (cw.height() - oh) // 2, ow, oh, ) + # Camera preview — bottom-right corner of the center/HUD area + pw = _CameraPreview._W + ph = self._cam_preview.height() or _CameraPreview._H + self._cam_preview.setGeometry( + cw.width() - _RIGHT_W - pw - 12, + cw.height() - ph - 28, + pw, ph, + ) def _update_metrics(self): snap = _metrics.snapshot() @@ -1383,7 +1864,7 @@ class MainWindow(QMainWindow): l.setStyleSheet(f"color: {color}; background: transparent;") return l - lay.addWidget(_badge("MARK XLVII", C.PRI_DIM)) + lay.addWidget(_badge("MARK XLVIII", C.PRI_DIM)) lay.addStretch() mid = QVBoxLayout(); mid.setSpacing(1) @@ -1470,12 +1951,14 @@ class MainWindow(QMainWindow): ip_lay.addWidget(os_lbl) lay.addWidget(info_panel) + lay.addSpacing(4) + lay.addStretch() for txt, col in [ ("AI CORE\nACTIVE", C.GREEN), ("SEC\nCLEARED", C.PRI), - ("PROTOCOL\nXXXVIII", C.TEXT_DIM), + ("PROTOCOL\nXLVIII", C.TEXT_DIM), ]: lbl = QLabel(txt) lbl.setFont(QFont("Courier New", 7, QFont.Weight.Bold)) @@ -1527,6 +2010,25 @@ class MainWindow(QMainWindow): lay.addWidget(_sec("COMMAND INPUT")) lay.addLayout(self._build_input_row()) + self._interrupt_btn = QPushButton("✋ INTERRUPT [ESC]") + self._interrupt_btn.setFixedHeight(34) + self._interrupt_btn.setFont(QFont("Courier New", 8, QFont.Weight.Bold)) + self._interrupt_btn.setCursor(Qt.CursorShape.PointingHandCursor) + self._interrupt_btn.setStyleSheet(f""" + QPushButton {{ + background: #140008; color: {C.MUTED_C}; + border: 1px solid {C.MUTED_C}; border-radius: 3px; + }} + QPushButton:hover {{ + background: #200010; border: 1px solid #ff6688; + }} + QPushButton:pressed {{ + background: #300018; + }} + """) + self._interrupt_btn.clicked.connect(self._do_interrupt) + lay.addWidget(self._interrupt_btn) + self._mute_btn = QPushButton("🎙 MICROPHONE ACTIVE") self._mute_btn.setFixedHeight(30) self._mute_btn.setFont(QFont("Courier New", 8, QFont.Weight.Bold)) @@ -1567,6 +2069,22 @@ class MainWindow(QMainWindow): fs_btn.clicked.connect(self._toggle_fullscreen) lay.addWidget(fs_btn) + sc_btn = QPushButton("⊞ CREATE DESKTOP SHORTCUT") + sc_btn.setFixedHeight(26) + sc_btn.setFont(QFont("Courier New", 7)) + sc_btn.setCursor(Qt.CursorShape.PointingHandCursor) + sc_btn.setStyleSheet(f""" + QPushButton {{ + background: transparent; color: {C.TEXT_DIM}; + border: 1px solid {C.BORDER}; border-radius: 3px; + }} + QPushButton:hover {{ + color: {C.ACC2}; border: 1px solid {C.BORDER_B}; + }} + """) + sc_btn.clicked.connect(self._create_desktop_shortcut) + lay.addWidget(sc_btn) + return w def _build_input_row(self) -> QHBoxLayout: @@ -1663,7 +2181,10 @@ class MainWindow(QMainWindow): self._content_display = QTextEdit() self._content_display.setReadOnly(True) self._content_display.setFont(QFont("Courier New", 8)) - self._content_display.setFixedHeight(155) + self._content_display.setMinimumHeight(60) + self._content_display.setSizePolicy( + QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Expanding + ) self._content_display.setStyleSheet(f""" QTextEdit {{ background: {C.DARK}; @@ -1693,13 +2214,14 @@ class MainWindow(QMainWindow): self._content_title_lbl.setText(title.upper()[:48]) self._content_ts_lbl.setText(_time.strftime("%H:%M:%S")) self._content_display.setPlainText(text) - # Scroll to top - cur = self._content_display.textCursor() - cur.moveToStart() if hasattr(cur, "moveToStart") else None self._content_display.moveCursor( self._content_display.textCursor().MoveOperation.Start ) + first_show = not self._content_panel.isVisible() self._content_panel.show() + if first_show: + total = self._center_split.height() + self._center_split.setSizes([max(total - 220, 120), 220]) def _build_footer(self) -> QWidget: w = QWidget() @@ -1714,7 +2236,7 @@ class MainWindow(QMainWindow): lay.addWidget(_fl("[F4] Mute · [F11] Fullscreen")) lay.addStretch() - lay.addWidget(_fl("FatihMakes Industries · MARK XLVII · CLASSIFIED")) + lay.addWidget(_fl("FatihMakes Industries · MARK XLVIII · CLASSIFIED")) lay.addStretch() lay.addWidget(_fl("© STARK INDUSTRIES", C.PRI_DIM)) return w @@ -1769,6 +2291,10 @@ class MainWindow(QMainWindow): self._remote_overlay = ov self._log.append_log(f"SYS: Remote key generated — manual: {manual or url}") + def _do_interrupt(self): + if self.on_interrupt: + self.on_interrupt() + def _toggle_mute(self): self._muted = not self._muted self.hud.muted = self._muted @@ -1891,6 +2417,14 @@ class JarvisUI: def on_remote_clicked(self, cb): self._win.on_remote_clicked = cb + @property + def on_interrupt(self): + return self._win.on_interrupt + + @on_interrupt.setter + def on_interrupt(self, cb): + self._win.on_interrupt = cb + def notify_phone_connected(self) -> None: self._win.notify_phone_connected() @@ -1908,6 +2442,23 @@ class JarvisUI: """Thread-safe: display content in the panel below the HUD.""" self._win._content_sig.emit(title[:48], text[:4000]) + def prompt_reconfig(self): + """Thread-safe: show the API key setup overlay (e.g. after an auth error).""" + self._win._ready = False + self._win._reconfig_sig.emit() + + def show_camera_frame(self, img_bytes: bytes): + """Thread-safe: show a webcam frame in the small overlay (screen captures).""" + self._win._camera_sig.emit(img_bytes) + + def start_camera_stream(self) -> None: + """Thread-safe: start live camera feed in the full HUD area.""" + self._win.start_camera_stream() + + def stop_camera_stream(self) -> None: + """Thread-safe: stop the live camera feed.""" + self._win.stop_camera_stream() + def start_speaking(self): self.set_state("SPEAKING")