273 lines
7.9 KiB
Python
273 lines
7.9 KiB
Python
"""
|
|
MARK XL — Screen / Camera Processor
|
|
Replaces Gemini Live vision session with a direct Ollama vision-model call.
|
|
The analysis text is returned (and optionally spoken via the `speak` callback).
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import base64
|
|
import io
|
|
import json
|
|
import sys
|
|
from pathlib import Path
|
|
from typing import Optional, Callable
|
|
|
|
try:
|
|
import cv2
|
|
_CV2 = True
|
|
except ImportError:
|
|
_CV2 = False
|
|
|
|
try:
|
|
import mss
|
|
import mss.tools
|
|
_MSS = True
|
|
except ImportError:
|
|
_MSS = False
|
|
|
|
try:
|
|
import PIL.Image
|
|
_PIL = True
|
|
except ImportError:
|
|
_PIL = False
|
|
|
|
import platform
|
|
|
|
import requests
|
|
|
|
|
|
def _base_dir() -> Path:
|
|
if getattr(sys, "frozen", False):
|
|
return Path(sys.executable).parent
|
|
return Path(__file__).resolve().parent.parent
|
|
|
|
|
|
_BASE = _base_dir()
|
|
_CONFIG_PATH = _BASE / "config" / "api_keys.json"
|
|
|
|
_IMG_MAX_W = 640
|
|
_IMG_MAX_H = 360
|
|
_JPEG_Q = 60
|
|
|
|
|
|
def _load_config() -> dict:
|
|
try:
|
|
return json.loads(_CONFIG_PATH.read_text(encoding="utf-8"))
|
|
except Exception:
|
|
return {}
|
|
|
|
|
|
def _save_config_key(key: str, value) -> None:
|
|
try:
|
|
cfg = _load_config()
|
|
cfg[key] = value
|
|
_CONFIG_PATH.write_text(json.dumps(cfg, indent=4), encoding="utf-8")
|
|
except Exception as e:
|
|
print(f"[Vision] ⚠️ Could not save config key '{key}': {e}")
|
|
|
|
|
|
def _get_os() -> str:
|
|
s = platform.system().lower()
|
|
if s == "darwin": return "mac"
|
|
if s == "windows": return "windows"
|
|
return "linux"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Image capture helpers (unchanged from original)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _compress(img_bytes: bytes, source_format: str = "PNG") -> tuple[bytes, str]:
|
|
if not _PIL:
|
|
return img_bytes, f"image/{source_format.lower()}"
|
|
try:
|
|
img = PIL.Image.open(io.BytesIO(img_bytes)).convert("RGB")
|
|
img.thumbnail((_IMG_MAX_W, _IMG_MAX_H), PIL.Image.BILINEAR)
|
|
buf = io.BytesIO()
|
|
img.save(buf, format="JPEG", quality=_JPEG_Q, optimize=False)
|
|
return buf.getvalue(), "image/jpeg"
|
|
except Exception as e:
|
|
print(f"[Vision] ⚠️ Image compress failed: {e}")
|
|
return img_bytes, f"image/{source_format.lower()}"
|
|
|
|
|
|
def _capture_screen() -> tuple[bytes, str]:
|
|
if not _MSS:
|
|
raise RuntimeError("mss is not installed. Run: pip install mss")
|
|
with mss.mss() as sct:
|
|
monitors = sct.monitors
|
|
target = monitors[1] if len(monitors) > 1 else monitors[0]
|
|
shot = sct.grab(target)
|
|
png = mss.tools.to_png(shot.rgb, shot.size)
|
|
return _compress(png, "PNG")
|
|
|
|
|
|
def _cv2_backend() -> int:
|
|
if not _CV2:
|
|
return 0
|
|
os_name = _get_os()
|
|
if os_name == "windows":
|
|
return cv2.CAP_DSHOW
|
|
if os_name == "mac":
|
|
return cv2.CAP_AVFOUNDATION
|
|
return cv2.CAP_ANY
|
|
|
|
|
|
def _probe_camera(index: int, backend: int, warmup: int = 5) -> bool:
|
|
if not _CV2:
|
|
return False
|
|
import numpy as np
|
|
cap = cv2.VideoCapture(index, backend)
|
|
if not cap.isOpened():
|
|
cap.release(); return False
|
|
for _ in range(warmup):
|
|
cap.read()
|
|
ret, frame = cap.read()
|
|
cap.release()
|
|
if not ret or frame is None:
|
|
return False
|
|
return bool(np.mean(frame) > 8)
|
|
|
|
|
|
def _detect_camera_index() -> int:
|
|
backend = _cv2_backend()
|
|
print("[Vision] 🔍 Auto-detecting camera…")
|
|
for idx in range(6):
|
|
if _probe_camera(idx, backend):
|
|
print(f"[Vision] ✅ Camera found at index {idx}")
|
|
_save_config_key("camera_index", idx)
|
|
return idx
|
|
print(f"[Vision] ⚠️ Camera index {idx}: no usable frame")
|
|
print("[Vision] ⚠️ No camera found — defaulting to index 0")
|
|
_save_config_key("camera_index", 0)
|
|
return 0
|
|
|
|
|
|
def _get_camera_index() -> int:
|
|
cfg = _load_config()
|
|
if "camera_index" in cfg:
|
|
return int(cfg["camera_index"])
|
|
return _detect_camera_index()
|
|
|
|
|
|
def _capture_camera() -> tuple[bytes, str]:
|
|
if not _CV2:
|
|
raise RuntimeError("OpenCV (cv2) is not installed. Run: pip install opencv-python")
|
|
import numpy as np
|
|
index = _get_camera_index()
|
|
backend = _cv2_backend()
|
|
cap = cv2.VideoCapture(index, backend)
|
|
if not cap.isOpened():
|
|
raise RuntimeError(f"Camera index {index} could not be opened.")
|
|
for _ in range(10):
|
|
cap.read()
|
|
ret, frame = cap.read()
|
|
cap.release()
|
|
if not ret or frame is None:
|
|
raise RuntimeError("Camera returned no frame.")
|
|
if _PIL:
|
|
rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
|
|
img = PIL.Image.fromarray(rgb)
|
|
img.thumbnail((_IMG_MAX_W, _IMG_MAX_H), PIL.Image.BILINEAR)
|
|
buf = io.BytesIO()
|
|
img.save(buf, format="JPEG", quality=_JPEG_Q)
|
|
return buf.getvalue(), "image/jpeg"
|
|
_, buf = cv2.imencode(".jpg", frame, [cv2.IMWRITE_JPEG_QUALITY, _JPEG_Q])
|
|
return buf.tobytes(), "image/jpeg"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Vision analysis via Ollama
|
|
# ---------------------------------------------------------------------------
|
|
|
|
_SYSTEM_PROMPT = (
|
|
"You are JARVIS, an advanced AI assistant. "
|
|
"Analyze the provided image with precision and intelligence. "
|
|
"Be concise and direct — maximum two sentences unless the user's question "
|
|
"requires more detail. "
|
|
"Address the user respectfully."
|
|
)
|
|
|
|
|
|
def _call_vision(image_bytes: bytes, mime: str, user_text: str) -> str:
|
|
cfg = _load_config()
|
|
url = cfg.get("llm_url", "http://localhost:11434").rstrip("/")
|
|
vision_model = cfg.get("vision_model") or cfg.get("llm_model", "llava")
|
|
|
|
b64 = base64.b64encode(image_bytes).decode("ascii")
|
|
|
|
payload = {
|
|
"model": vision_model,
|
|
"stream": False,
|
|
"messages": [
|
|
{"role": "system", "content": _SYSTEM_PROMPT},
|
|
{
|
|
"role": "user",
|
|
"content": user_text,
|
|
"images": [b64],
|
|
},
|
|
],
|
|
}
|
|
try:
|
|
resp = requests.post(f"{url}/api/chat", json=payload, timeout=60)
|
|
resp.raise_for_status()
|
|
return (resp.json().get("message", {}).get("content") or "").strip()
|
|
except requests.exceptions.ConnectionError:
|
|
return "Cannot connect to Ollama. Make sure Ollama is running."
|
|
except Exception as e:
|
|
return f"Vision analysis failed: {e}"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Public entry point
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def screen_process(
|
|
parameters: dict,
|
|
response=None,
|
|
player=None,
|
|
session_memory=None,
|
|
speak: Optional[Callable[[str], None]] = None,
|
|
) -> str:
|
|
"""
|
|
Capture screen or camera, analyse with Ollama vision model.
|
|
|
|
Returns the analysis text (str).
|
|
Optionally speaks via `speak` callback and logs to `player`.
|
|
"""
|
|
params = parameters or {}
|
|
user_text = (params.get("text") or params.get("user_text") or "").strip()
|
|
angle = params.get("angle", "screen").lower().strip()
|
|
|
|
if not user_text:
|
|
user_text = "What do you see? Describe briefly."
|
|
|
|
if player:
|
|
player.write_log(f"SYS: Vision [{angle}] — {user_text[:60]}")
|
|
|
|
# Capture
|
|
try:
|
|
if angle == "camera":
|
|
image_bytes, mime = _capture_camera()
|
|
print(f"[Vision] 📷 Camera: {len(image_bytes):,} bytes")
|
|
else:
|
|
image_bytes, mime = _capture_screen()
|
|
print(f"[Vision] 🖥️ Screen: {len(image_bytes):,} bytes")
|
|
except Exception as e:
|
|
msg = f"Capture error: {e}"
|
|
print(f"[Vision] ❌ {msg}")
|
|
if player: player.write_log(f"ERR: {msg}")
|
|
return msg
|
|
|
|
# Analyse
|
|
analysis = _call_vision(image_bytes, mime, user_text)
|
|
print(f"[Vision] 💬 {analysis[:120]}")
|
|
|
|
if player:
|
|
player.write_log(f"Jarvis: {analysis}")
|
|
|
|
if speak and analysis:
|
|
speak(analysis)
|
|
|
|
return analysis
|