""" MARK XL — Screen / Camera Processor Replaces Gemini Live vision session with a direct Ollama vision-model call. The analysis text is returned (and optionally spoken via the `speak` callback). """ from __future__ import annotations import base64 import io import json import sys from pathlib import Path from typing import Optional, Callable try: import cv2 _CV2 = True except ImportError: _CV2 = False try: import mss import mss.tools _MSS = True except ImportError: _MSS = False try: import PIL.Image _PIL = True except ImportError: _PIL = False import platform import requests def _base_dir() -> Path: if getattr(sys, "frozen", False): return Path(sys.executable).parent return Path(__file__).resolve().parent.parent _BASE = _base_dir() _CONFIG_PATH = _BASE / "config" / "api_keys.json" _IMG_MAX_W = 640 _IMG_MAX_H = 360 _JPEG_Q = 60 def _load_config() -> dict: try: return json.loads(_CONFIG_PATH.read_text(encoding="utf-8")) except Exception: return {} def _save_config_key(key: str, value) -> None: try: cfg = _load_config() cfg[key] = value _CONFIG_PATH.write_text(json.dumps(cfg, indent=4), encoding="utf-8") except Exception as e: print(f"[Vision] ⚠️ Could not save config key '{key}': {e}") def _get_os() -> str: s = platform.system().lower() if s == "darwin": return "mac" if s == "windows": return "windows" return "linux" # --------------------------------------------------------------------------- # Image capture helpers (unchanged from original) # --------------------------------------------------------------------------- def _compress(img_bytes: bytes, source_format: str = "PNG") -> tuple[bytes, str]: if not _PIL: return img_bytes, f"image/{source_format.lower()}" try: img = PIL.Image.open(io.BytesIO(img_bytes)).convert("RGB") img.thumbnail((_IMG_MAX_W, _IMG_MAX_H), PIL.Image.BILINEAR) buf = io.BytesIO() img.save(buf, format="JPEG", quality=_JPEG_Q, optimize=False) return buf.getvalue(), "image/jpeg" except Exception as e: print(f"[Vision] ⚠️ Image compress failed: {e}") return img_bytes, f"image/{source_format.lower()}" def _capture_screen() -> tuple[bytes, str]: if not _MSS: raise RuntimeError("mss is not installed. Run: pip install mss") with mss.mss() as sct: monitors = sct.monitors target = monitors[1] if len(monitors) > 1 else monitors[0] shot = sct.grab(target) png = mss.tools.to_png(shot.rgb, shot.size) return _compress(png, "PNG") def _cv2_backend() -> int: if not _CV2: return 0 os_name = _get_os() if os_name == "windows": return cv2.CAP_DSHOW if os_name == "mac": return cv2.CAP_AVFOUNDATION return cv2.CAP_ANY def _probe_camera(index: int, backend: int, warmup: int = 5) -> bool: if not _CV2: return False import numpy as np cap = cv2.VideoCapture(index, backend) if not cap.isOpened(): cap.release(); return False for _ in range(warmup): cap.read() ret, frame = cap.read() cap.release() if not ret or frame is None: return False return bool(np.mean(frame) > 8) def _detect_camera_index() -> int: backend = _cv2_backend() print("[Vision] 🔍 Auto-detecting camera…") for idx in range(6): if _probe_camera(idx, backend): print(f"[Vision] ✅ Camera found at index {idx}") _save_config_key("camera_index", idx) return idx print(f"[Vision] ⚠️ Camera index {idx}: no usable frame") print("[Vision] ⚠️ No camera found — defaulting to index 0") _save_config_key("camera_index", 0) return 0 def _get_camera_index() -> int: cfg = _load_config() if "camera_index" in cfg: return int(cfg["camera_index"]) return _detect_camera_index() def _capture_camera() -> tuple[bytes, str]: if not _CV2: raise RuntimeError("OpenCV (cv2) is not installed. Run: pip install opencv-python") import numpy as np index = _get_camera_index() backend = _cv2_backend() cap = cv2.VideoCapture(index, backend) if not cap.isOpened(): raise RuntimeError(f"Camera index {index} could not be opened.") for _ in range(10): cap.read() ret, frame = cap.read() cap.release() if not ret or frame is None: raise RuntimeError("Camera returned no frame.") if _PIL: rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) img = PIL.Image.fromarray(rgb) img.thumbnail((_IMG_MAX_W, _IMG_MAX_H), PIL.Image.BILINEAR) buf = io.BytesIO() img.save(buf, format="JPEG", quality=_JPEG_Q) return buf.getvalue(), "image/jpeg" _, buf = cv2.imencode(".jpg", frame, [cv2.IMWRITE_JPEG_QUALITY, _JPEG_Q]) return buf.tobytes(), "image/jpeg" # --------------------------------------------------------------------------- # Vision analysis via Ollama # --------------------------------------------------------------------------- _SYSTEM_PROMPT = ( "You are JARVIS, an advanced AI assistant. " "Analyze the provided image with precision and intelligence. " "Be concise and direct — maximum two sentences unless the user's question " "requires more detail. " "Address the user respectfully." ) def _call_vision(image_bytes: bytes, mime: str, user_text: str) -> str: cfg = _load_config() url = cfg.get("llm_url", "http://localhost:11434").rstrip("/") vision_model = cfg.get("vision_model") or cfg.get("llm_model", "llava") b64 = base64.b64encode(image_bytes).decode("ascii") payload = { "model": vision_model, "stream": False, "messages": [ {"role": "system", "content": _SYSTEM_PROMPT}, { "role": "user", "content": user_text, "images": [b64], }, ], } try: resp = requests.post(f"{url}/api/chat", json=payload, timeout=60) resp.raise_for_status() return (resp.json().get("message", {}).get("content") or "").strip() except requests.exceptions.ConnectionError: return "Cannot connect to Ollama. Make sure Ollama is running." except Exception as e: return f"Vision analysis failed: {e}" # --------------------------------------------------------------------------- # Public entry point # --------------------------------------------------------------------------- def screen_process( parameters: dict, response=None, player=None, session_memory=None, speak: Optional[Callable[[str], None]] = None, ) -> str: """ Capture screen or camera, analyse with Ollama vision model. Returns the analysis text (str). Optionally speaks via `speak` callback and logs to `player`. """ params = parameters or {} user_text = (params.get("text") or params.get("user_text") or "").strip() angle = params.get("angle", "screen").lower().strip() if not user_text: user_text = "What do you see? Describe briefly." if player: player.write_log(f"SYS: Vision [{angle}] — {user_text[:60]}") # Capture try: if angle == "camera": image_bytes, mime = _capture_camera() print(f"[Vision] 📷 Camera: {len(image_bytes):,} bytes") else: image_bytes, mime = _capture_screen() print(f"[Vision] 🖥️ Screen: {len(image_bytes):,} bytes") except Exception as e: msg = f"Capture error: {e}" print(f"[Vision] ❌ {msg}") if player: player.write_log(f"ERR: {msg}") return msg # Analyse analysis = _call_vision(image_bytes, mime, user_text) print(f"[Vision] 💬 {analysis[:120]}") if player: player.write_log(f"Jarvis: {analysis}") if speak and analysis: speak(analysis) return analysis