Add files via upload
This commit is contained in:
1 parent
320d933f08
commit
cd70cb7796
27 files changed
+5807
-3538
No files matched your search
+298
-260
@@ -1,151 +1,186 @@
|
||||
"""
|
||||
actions/screen_processor.py — Gemini Live API — IMAGE-ONLY SESSION v8
|
||||
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
||||
v8 Changes:
|
||||
- Auto camera detection on first use — saves to config
|
||||
- No hardcoded camera index
|
||||
- mic_loop removed (no double response issue)
|
||||
- Image-only session, no conflict with main.py
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import io
|
||||
import json
|
||||
import re
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import threading
|
||||
import cv2
|
||||
import mss
|
||||
import mss.tools
|
||||
import pyaudio
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
import numpy as np
|
||||
import sounddevice as sd
|
||||
|
||||
try:
|
||||
import cv2
|
||||
_CV2 = True
|
||||
except ImportError:
|
||||
_CV2 = False
|
||||
|
||||
try:
|
||||
import mss
|
||||
import mss.tools
|
||||
_MSS = True
|
||||
except ImportError:
|
||||
_MSS = False
|
||||
|
||||
try:
|
||||
import PIL.Image
|
||||
_PIL_OK = True
|
||||
_PIL = True
|
||||
except ImportError:
|
||||
_PIL_OK = False
|
||||
_PIL = False
|
||||
|
||||
from google import genai
|
||||
from google.genai import types
|
||||
from google.genai import types as gtypes
|
||||
|
||||
def get_base_dir():
|
||||
def _base_dir() -> Path:
|
||||
if getattr(sys, "frozen", False):
|
||||
return Path(sys.executable).parent
|
||||
return Path(__file__).resolve().parent.parent
|
||||
|
||||
BASE_DIR = get_base_dir()
|
||||
API_CONFIG_PATH = BASE_DIR / "config" / "api_keys.json"
|
||||
|
||||
LIVE_MODEL = "models/gemini-2.5-flash-native-audio-preview-12-2025"
|
||||
FORMAT = pyaudio.paInt16
|
||||
CHANNELS = 1
|
||||
RECEIVE_SAMPLE_RATE = 24000
|
||||
CHUNK_SIZE = 1024
|
||||
_BASE = _base_dir()
|
||||
_CONFIG_PATH = _BASE / "config" / "api_keys.json"
|
||||
|
||||
IMG_MAX_W = 640
|
||||
IMG_MAX_H = 360
|
||||
JPEG_Q = 55
|
||||
|
||||
SYSTEM_PROMPT = (
|
||||
"You are JARVIS from Iron Man movies. "
|
||||
"Analyze images with technical precision and intelligence. "
|
||||
"Help the user in a way they can understand — don't be overly complex. "
|
||||
"Be concise, smart, and helpful like Tony Stark's AI assistant. "
|
||||
"Respond in maximum 2 short sentences. Speed is priority. "
|
||||
"Address the user as 'sir' for a tone of respect. "
|
||||
"Ask if the user needs any further help with their problem."
|
||||
)
|
||||
def _load_config() -> dict:
|
||||
try:
|
||||
return json.loads(_CONFIG_PATH.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
|
||||
def _save_config_key(key: str, value) -> None:
|
||||
try:
|
||||
cfg = _load_config()
|
||||
cfg[key] = value
|
||||
_CONFIG_PATH.write_text(json.dumps(cfg, indent=4), encoding="utf-8")
|
||||
except Exception as e:
|
||||
print(f"[Vision] ⚠️ Could not save config key '{key}': {e}")
|
||||
|
||||
|
||||
def _get_api_key() -> str:
|
||||
key = _load_config().get("gemini_api_key", "")
|
||||
if not key:
|
||||
raise RuntimeError("gemini_api_key not found in config.")
|
||||
return key
|
||||
|
||||
|
||||
def _get_os() -> str:
|
||||
return _load_config().get("os_system", "windows").lower()
|
||||
|
||||
_LIVE_MODEL = "models/gemini-2.5-flash-native-audio-preview-12-2025"
|
||||
_CHANNELS = 1
|
||||
_RECEIVE_SAMPLE_RATE = 24_000
|
||||
_CHUNK_SIZE = 1_024
|
||||
|
||||
_IMG_MAX_W = 640
|
||||
_IMG_MAX_H = 360
|
||||
_JPEG_Q = 60
|
||||
|
||||
_SYSTEM_PROMPT = (
|
||||
"You are JARVIS, an advanced AI assistant. "
|
||||
"Analyze the provided image with precision and intelligence. "
|
||||
"Be concise and direct — maximum two sentences unless the user's question "
|
||||
"requires more detail. "
|
||||
"Address the user respectfully. "
|
||||
"Always call the appropriate tool; never simulate results."
|
||||
)
|
||||
|
||||
|
||||
def _compress(img_bytes: bytes, source_format: str = "PNG") -> tuple[bytes, str]:
|
||||
if not _PIL:
|
||||
return img_bytes, f"image/{source_format.lower()}"
|
||||
|
||||
try:
|
||||
with open(API_CONFIG_PATH, "r", encoding="utf-8") as f:
|
||||
keys = json.load(f)
|
||||
key = keys.get("gemini_api_key", "")
|
||||
if not key:
|
||||
raise ValueError("gemini_api_key not found")
|
||||
return key
|
||||
img = PIL.Image.open(io.BytesIO(img_bytes)).convert("RGB")
|
||||
img.thumbnail((_IMG_MAX_W, _IMG_MAX_H), PIL.Image.BILINEAR)
|
||||
buf = io.BytesIO()
|
||||
img.save(buf, format="JPEG", quality=_JPEG_Q, optimize=False)
|
||||
return buf.getvalue(), "image/jpeg"
|
||||
except Exception as e:
|
||||
raise RuntimeError(f"Could not load API key: {e}")
|
||||
print(f"[Vision] ⚠️ Image compress failed: {e}")
|
||||
return img_bytes, f"image/{source_format.lower()}"
|
||||
|
||||
def _capture_screen() -> tuple[bytes, str]:
|
||||
|
||||
if not _MSS:
|
||||
raise RuntimeError("mss is not installed. Run: pip install mss")
|
||||
|
||||
with mss.mss() as sct:
|
||||
monitors = sct.monitors # [0] = all combined, [1..n] = real screens
|
||||
target = monitors[1] if len(monitors) > 1 else monitors[0]
|
||||
shot = sct.grab(target)
|
||||
png = mss.tools.to_png(shot.rgb, shot.size)
|
||||
|
||||
return _compress(png, "PNG")
|
||||
|
||||
|
||||
def _cv2_backend() -> int:
|
||||
"""Return the best OpenCV camera backend for the current OS."""
|
||||
if not _CV2:
|
||||
return 0
|
||||
os_name = _get_os()
|
||||
if os_name == "windows":
|
||||
return cv2.CAP_DSHOW
|
||||
if os_name == "mac":
|
||||
return cv2.CAP_AVFOUNDATION
|
||||
return cv2.CAP_ANY
|
||||
|
||||
|
||||
def _probe_camera(index: int, backend: int, warmup: int = 5) -> bool:
|
||||
|
||||
if not _CV2:
|
||||
return False
|
||||
cap = cv2.VideoCapture(index, backend)
|
||||
if not cap.isOpened():
|
||||
cap.release()
|
||||
return False
|
||||
for _ in range(warmup):
|
||||
cap.read()
|
||||
ret, frame = cap.read()
|
||||
cap.release()
|
||||
if not ret or frame is None:
|
||||
return False
|
||||
return bool(np.mean(frame) > 8)
|
||||
|
||||
|
||||
def _detect_camera_index() -> int:
|
||||
|
||||
backend = _cv2_backend()
|
||||
print("[Vision] 🔍 Auto-detecting camera...")
|
||||
for idx in range(6):
|
||||
if _probe_camera(idx, backend):
|
||||
print(f"[Vision] ✅ Camera found at index {idx}")
|
||||
_save_config_key("camera_index", idx)
|
||||
return idx
|
||||
print(f"[Vision] ⚠️ Camera index {idx}: no usable frame")
|
||||
|
||||
print("[Vision] ⚠️ No camera found — defaulting to index 0")
|
||||
_save_config_key("camera_index", 0)
|
||||
return 0
|
||||
|
||||
|
||||
def _get_camera_index() -> int:
|
||||
"""
|
||||
Reads camera index from config.
|
||||
If not set, auto-detects the best camera and saves it for future use.
|
||||
Runs only once — after that, config value is used directly.
|
||||
"""
|
||||
try:
|
||||
with open(API_CONFIG_PATH, "r", encoding="utf-8") as f:
|
||||
cfg = json.load(f)
|
||||
if "camera_index" in cfg:
|
||||
return int(cfg["camera_index"])
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
print("[Camera] 🔍 No camera index in config. Auto-detecting...")
|
||||
best_index = 0
|
||||
|
||||
for idx in range(6):
|
||||
cap = cv2.VideoCapture(idx, cv2.CAP_DSHOW)
|
||||
if not cap.isOpened():
|
||||
cap.release()
|
||||
continue
|
||||
|
||||
for _ in range(5):
|
||||
cap.read()
|
||||
|
||||
ret, frame = cap.read()
|
||||
cap.release()
|
||||
|
||||
if ret and frame is not None and frame.mean() > 5:
|
||||
best_index = idx
|
||||
print(f"[Camera] ✅ Camera found at index {idx} — saving to config.")
|
||||
break
|
||||
else:
|
||||
print(f"[Camera] ⚠️ Index {idx}: no valid frame (black or empty).")
|
||||
|
||||
try:
|
||||
cfg = {}
|
||||
if API_CONFIG_PATH.exists():
|
||||
with open(API_CONFIG_PATH, "r", encoding="utf-8") as f:
|
||||
cfg = json.load(f)
|
||||
cfg["camera_index"] = best_index
|
||||
with open(API_CONFIG_PATH, "w", encoding="utf-8") as f:
|
||||
json.dump(cfg, f, indent=4)
|
||||
print(f"[Camera] 💾 Camera index {best_index} saved to config.")
|
||||
except Exception as e:
|
||||
print(f"[Camera] ⚠️ Could not save camera index: {e}")
|
||||
|
||||
return best_index
|
||||
cfg = _load_config()
|
||||
if "camera_index" in cfg:
|
||||
return int(cfg["camera_index"])
|
||||
return _detect_camera_index()
|
||||
|
||||
|
||||
def _to_jpeg(img_bytes: bytes) -> bytes:
|
||||
if not _PIL_OK:
|
||||
return img_bytes
|
||||
img = PIL.Image.open(io.BytesIO(img_bytes)).convert("RGB")
|
||||
img.thumbnail([IMG_MAX_W, IMG_MAX_H], PIL.Image.BILINEAR)
|
||||
buf = io.BytesIO()
|
||||
img.save(buf, format="JPEG", quality=JPEG_Q, optimize=False)
|
||||
return buf.getvalue()
|
||||
def _capture_camera() -> tuple[bytes, str]:
|
||||
if not _CV2:
|
||||
raise RuntimeError("OpenCV (cv2) is not installed. Run: pip install opencv-python")
|
||||
|
||||
index = _get_camera_index()
|
||||
backend = _cv2_backend()
|
||||
cap = cv2.VideoCapture(index, backend)
|
||||
|
||||
def _capture_screenshot() -> bytes:
|
||||
with mss.mss() as sct:
|
||||
shot = sct.grab(sct.monitors[1])
|
||||
png_bytes = mss.tools.to_png(shot.rgb, shot.size)
|
||||
return _to_jpeg(png_bytes)
|
||||
|
||||
|
||||
def _capture_camera() -> bytes:
|
||||
camera_index = _get_camera_index()
|
||||
cap = cv2.VideoCapture(camera_index, cv2.CAP_DSHOW)
|
||||
if not cap.isOpened():
|
||||
raise RuntimeError(f"Camera could not be opened: index {camera_index}")
|
||||
raise RuntimeError(f"Camera index {index} could not be opened.")
|
||||
|
||||
for _ in range(10):
|
||||
cap.read()
|
||||
@@ -154,127 +189,140 @@ def _capture_camera() -> bytes:
|
||||
cap.release()
|
||||
|
||||
if not ret or frame is None:
|
||||
raise RuntimeError("Could not capture camera frame.")
|
||||
raise RuntimeError("Camera returned no frame.")
|
||||
|
||||
if _PIL_OK:
|
||||
if _PIL:
|
||||
rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
|
||||
img = PIL.Image.fromarray(rgb)
|
||||
img.thumbnail([IMG_MAX_W, IMG_MAX_H], PIL.Image.BILINEAR)
|
||||
img.thumbnail((_IMG_MAX_W, _IMG_MAX_H), PIL.Image.BILINEAR)
|
||||
buf = io.BytesIO()
|
||||
img.save(buf, format="JPEG", quality=JPEG_Q, optimize=False)
|
||||
return buf.getvalue()
|
||||
img.save(buf, format="JPEG", quality=_JPEG_Q)
|
||||
return buf.getvalue(), "image/jpeg"
|
||||
|
||||
_, buf = cv2.imencode(".jpg", frame, [cv2.IMWRITE_JPEG_QUALITY, JPEG_Q])
|
||||
return buf.tobytes()
|
||||
|
||||
|
||||
class _LiveSession:
|
||||
"""
|
||||
Image-only analysis session.
|
||||
No microphone — no conflict with main.py session.
|
||||
Sends image + question, plays audio response.
|
||||
"""
|
||||
_, buf = cv2.imencode(".jpg", frame, [cv2.IMWRITE_JPEG_QUALITY, _JPEG_Q])
|
||||
return buf.tobytes(), "image/jpeg"
|
||||
|
||||
class _VisionSession:
|
||||
def __init__(self):
|
||||
self._loop: asyncio.AbstractEventLoop | None = None
|
||||
self._thread: threading.Thread | None = None
|
||||
self._session = None
|
||||
self._out_queue: asyncio.Queue | None = None
|
||||
self._audio_in: asyncio.Queue | None = None
|
||||
self._ready: threading.Event = threading.Event()
|
||||
self._player = None
|
||||
self._pya = pyaudio.PyAudio()
|
||||
self._send_lock: asyncio.Lock | None = None
|
||||
self._loop: Optional[asyncio.AbstractEventLoop] = None
|
||||
self._thread: Optional[threading.Thread] = None
|
||||
self._session = None
|
||||
self._out_queue: Optional[asyncio.Queue] = None
|
||||
self._audio_in: Optional[asyncio.Queue] = None
|
||||
self._ready_evt: threading.Event = threading.Event()
|
||||
self._player = None
|
||||
self._lock: threading.Lock = threading.Lock()
|
||||
|
||||
def start(self, player=None):
|
||||
if self._thread and self._thread.is_alive():
|
||||
def start(self, player=None, timeout: float = 25.0) -> None:
|
||||
with self._lock:
|
||||
if self._thread and self._thread.is_alive():
|
||||
if player is not None:
|
||||
self._player = player
|
||||
return
|
||||
self._player = player
|
||||
self._thread = threading.Thread(
|
||||
target=self._run_event_loop,
|
||||
daemon=True,
|
||||
name="VisionSessionThread",
|
||||
)
|
||||
self._thread.start()
|
||||
|
||||
if not self._ready_evt.wait(timeout=timeout):
|
||||
raise RuntimeError(f"Vision session did not connect within {timeout}s.")
|
||||
print("[Vision] ✅ Session ready")
|
||||
|
||||
def analyze(self, image_bytes: bytes, mime_type: str, user_text: str) -> None:
|
||||
if not self._loop or not self._out_queue:
|
||||
print("[Vision] ⚠️ Session not started — dropping request")
|
||||
return
|
||||
self._player = player
|
||||
self._thread = threading.Thread(
|
||||
target=self._run_loop, daemon=True, name="VisionSessionThread"
|
||||
asyncio.run_coroutine_threadsafe(
|
||||
self._out_queue.put((image_bytes, mime_type, user_text)),
|
||||
self._loop,
|
||||
)
|
||||
self._thread.start()
|
||||
ok = self._ready.wait(timeout=20)
|
||||
if not ok:
|
||||
raise RuntimeError("Vision session did not start within 20s.")
|
||||
print("[ScreenProcess] ✅ Vision session ready (no mic)")
|
||||
|
||||
def _run_loop(self):
|
||||
def is_ready(self) -> bool:
|
||||
return self._session is not None
|
||||
|
||||
def _run_event_loop(self) -> None:
|
||||
self._loop = asyncio.new_event_loop()
|
||||
asyncio.set_event_loop(self._loop)
|
||||
self._loop.run_until_complete(self._main())
|
||||
self._loop.run_until_complete(self._session_loop())
|
||||
|
||||
async def _main(self):
|
||||
async def _session_loop(self) -> None:
|
||||
self._out_queue = asyncio.Queue(maxsize=30)
|
||||
self._audio_in = asyncio.Queue()
|
||||
self._send_lock = asyncio.Lock()
|
||||
|
||||
client = genai.Client(
|
||||
api_key=_get_api_key(),
|
||||
http_options={"api_version": "v1beta"}
|
||||
http_options={"api_version": "v1beta"},
|
||||
)
|
||||
|
||||
config = types.LiveConnectConfig(
|
||||
config = gtypes.LiveConnectConfig(
|
||||
response_modalities=["AUDIO"],
|
||||
output_audio_transcription={},
|
||||
system_instruction=SYSTEM_PROMPT,
|
||||
speech_config=types.SpeechConfig(
|
||||
voice_config=types.VoiceConfig(
|
||||
prebuilt_voice_config=types.PrebuiltVoiceConfig(
|
||||
system_instruction=_SYSTEM_PROMPT,
|
||||
speech_config=gtypes.SpeechConfig(
|
||||
voice_config=gtypes.VoiceConfig(
|
||||
prebuilt_voice_config=gtypes.PrebuiltVoiceConfig(
|
||||
voice_name="Charon"
|
||||
)
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
backoff = 2.0
|
||||
while True:
|
||||
try:
|
||||
print("[ScreenProcess] 🔌 Vision session connecting...")
|
||||
async with client.aio.live.connect(model=LIVE_MODEL, config=config) as session:
|
||||
print("[Vision] 🔌 Connecting...")
|
||||
async with client.aio.live.connect(
|
||||
model=_LIVE_MODEL, config=config
|
||||
) as session:
|
||||
self._session = session
|
||||
self._ready.set()
|
||||
print("[ScreenProcess] ✅ Vision session connected")
|
||||
self._ready_evt.set()
|
||||
backoff = 2.0
|
||||
print("[Vision] ✅ Connected")
|
||||
|
||||
async with asyncio.TaskGroup() as tg:
|
||||
tg.create_task(self._send_loop())
|
||||
tg.create_task(self._recv_loop())
|
||||
tg.create_task(self._play_loop())
|
||||
|
||||
except Exception as e:
|
||||
print(f"[ScreenProcess] ⚠️ Disconnected: {e} — reconnecting...")
|
||||
except* Exception as eg:
|
||||
for exc in eg.exceptions:
|
||||
print(f"[Vision] ⚠️ Session error: {exc}")
|
||||
finally:
|
||||
self._session = None
|
||||
self._ready.clear()
|
||||
await asyncio.sleep(2)
|
||||
self._ready.set()
|
||||
self._ready_evt.clear()
|
||||
|
||||
# ── Send loop ──
|
||||
print(f"[Vision] 🔄 Reconnecting in {backoff:.0f}s...")
|
||||
await asyncio.sleep(backoff)
|
||||
backoff = min(backoff * 1.5, 30.0)
|
||||
self._ready_evt.set()
|
||||
|
||||
async def _send_loop(self):
|
||||
"""Sends (image_bytes, mime_type, user_text) tuples from queue."""
|
||||
async def _send_loop(self) -> None:
|
||||
while True:
|
||||
item = await self._out_queue.get()
|
||||
if self._session:
|
||||
image_bytes, mime_type, user_text = item
|
||||
try:
|
||||
b64 = base64.b64encode(image_bytes).decode("utf-8")
|
||||
await self._session.send_client_content(
|
||||
turns={
|
||||
"parts": [
|
||||
{"inline_data": {"mime_type": mime_type, "data": b64}},
|
||||
{"text": user_text}
|
||||
]
|
||||
},
|
||||
turn_complete=True
|
||||
)
|
||||
print("[ScreenProcess] ✅ Image sent")
|
||||
except Exception as e:
|
||||
print(f"[ScreenProcess] ⚠️ Send error: {e}")
|
||||
image_bytes, mime_type, user_text = await self._out_queue.get()
|
||||
if not self._session:
|
||||
print("[Vision] ⚠️ No session — dropping image")
|
||||
continue
|
||||
try:
|
||||
b64 = base64.b64encode(image_bytes).decode("ascii")
|
||||
await self._session.send_client_content(
|
||||
turns={
|
||||
"parts": [
|
||||
{"inline_data": {"mime_type": mime_type, "data": b64}},
|
||||
{"text": user_text},
|
||||
]
|
||||
},
|
||||
turn_complete=True,
|
||||
)
|
||||
print(f"[Vision] 📤 Sent {len(image_bytes):,} bytes — '{user_text[:60]}'")
|
||||
except Exception as e:
|
||||
print(f"[Vision] ⚠️ Send error: {e}")
|
||||
|
||||
async def _recv_loop(self):
|
||||
transcript_buf: list[str] = []
|
||||
async def _recv_loop(self) -> None:
|
||||
transcript: list[str] = []
|
||||
try:
|
||||
async for response in self._session.receive():
|
||||
|
||||
if response.data:
|
||||
await self._audio_in.put(response.data)
|
||||
|
||||
@@ -285,120 +333,110 @@ class _LiveSession:
|
||||
if sc.output_transcription and sc.output_transcription.text:
|
||||
chunk = sc.output_transcription.text.strip()
|
||||
if chunk:
|
||||
transcript_buf.append(chunk)
|
||||
transcript.append(chunk)
|
||||
|
||||
if sc.turn_complete:
|
||||
if transcript_buf and self._player:
|
||||
full = re.sub(r'\s+', ' ', " ".join(transcript_buf)).strip()
|
||||
if transcript and self._player:
|
||||
full = re.sub(r"\s+", " ", " ".join(transcript)).strip()
|
||||
if full:
|
||||
self._player.write_log(f"Jarvis: {full}")
|
||||
print(f"[ScreenProcess] 💬 {full}")
|
||||
transcript_buf = []
|
||||
print(f"[Vision] 💬 {full}")
|
||||
transcript = []
|
||||
|
||||
except Exception as e:
|
||||
print(f"[ScreenProcess] ⚠️ Recv error: {e}")
|
||||
transcript_buf = []
|
||||
await asyncio.sleep(0.3)
|
||||
print(f"[Vision] ⚠️ Recv error: {e}")
|
||||
raise
|
||||
|
||||
async def _play_loop(self):
|
||||
stream = await asyncio.to_thread(
|
||||
self._pya.open,
|
||||
format=FORMAT, channels=CHANNELS,
|
||||
rate=RECEIVE_SAMPLE_RATE, output=True,
|
||||
async def _play_loop(self) -> None:
|
||||
stream = sd.RawOutputStream(
|
||||
samplerate=_RECEIVE_SAMPLE_RATE,
|
||||
channels=_CHANNELS,
|
||||
dtype="int16",
|
||||
blocksize=_CHUNK_SIZE,
|
||||
)
|
||||
stream.start()
|
||||
try:
|
||||
while True:
|
||||
chunk = await self._audio_in.get()
|
||||
await asyncio.to_thread(stream.write, chunk)
|
||||
except Exception as e:
|
||||
print(f"[Vision] ❌ Play error: {e}")
|
||||
raise
|
||||
finally:
|
||||
stream.stop()
|
||||
stream.close()
|
||||
|
||||
def analyze(self, image_bytes: bytes, mime_type: str, user_text: str):
|
||||
"""Called from main thread — puts image into async queue."""
|
||||
if not self._loop:
|
||||
return
|
||||
asyncio.run_coroutine_threadsafe(
|
||||
self._out_queue.put((image_bytes, mime_type, user_text)),
|
||||
self._loop
|
||||
)
|
||||
|
||||
def is_ready(self) -> bool:
|
||||
return self._session is not None
|
||||
|
||||
_live = _LiveSession()
|
||||
_started = False
|
||||
_start_lock = threading.Lock()
|
||||
_session = _VisionSession()
|
||||
_session_lock = threading.Lock()
|
||||
_session_up = False
|
||||
|
||||
|
||||
def _ensure_started(player=None):
|
||||
global _started
|
||||
with _start_lock:
|
||||
if not _started:
|
||||
_live.start(player=player)
|
||||
_started = True
|
||||
def _ensure_session(player=None) -> None:
|
||||
global _session_up
|
||||
with _session_lock:
|
||||
if not _session_up:
|
||||
_session.start(player=player)
|
||||
_session_up = True
|
||||
elif player is not None:
|
||||
_live._player = player
|
||||
_session._player = player
|
||||
|
||||
|
||||
def screen_process(
|
||||
parameters: dict,
|
||||
response: str | None = None,
|
||||
response=None,
|
||||
player=None,
|
||||
session_memory=None,
|
||||
) -> bool:
|
||||
user_text = (parameters or {}).get("text") or (parameters or {}).get("user_text", "")
|
||||
user_text = (user_text or "").strip()
|
||||
|
||||
params = parameters or {}
|
||||
user_text = (params.get("text") or params.get("user_text") or "").strip()
|
||||
angle = params.get("angle", "screen").lower().strip()
|
||||
|
||||
if not user_text:
|
||||
print("[ScreenProcess] ⚠️ No user_text provided.")
|
||||
print("[Vision] ⚠️ No question provided — aborting")
|
||||
return False
|
||||
|
||||
angle = (parameters or {}).get("angle", "screen").lower().strip()
|
||||
print(f"[ScreenProcess] angle={angle!r} text={user_text!r}")
|
||||
print(f"[Vision] ▶ angle={angle!r} question='{user_text[:80]}'")
|
||||
|
||||
_ensure_started(player=player)
|
||||
try:
|
||||
_ensure_session(player=player)
|
||||
except Exception as e:
|
||||
print(f"[Vision] ❌ Could not start session: {e}")
|
||||
return False
|
||||
|
||||
try:
|
||||
if angle == "camera":
|
||||
image_bytes = _capture_camera()
|
||||
mime_type = "image/jpeg"
|
||||
print("[ScreenProcess] 📷 Camera captured")
|
||||
image_bytes, mime_type = _capture_camera()
|
||||
print(f"[Vision] 📷 Camera: {len(image_bytes):,} bytes")
|
||||
else:
|
||||
image_bytes = _capture_screenshot()
|
||||
mime_type = "image/jpeg" if _PIL_OK else "image/png"
|
||||
print("[ScreenProcess] 🖥️ Screen captured")
|
||||
image_bytes, mime_type = _capture_screen()
|
||||
print(f"[Vision] 🖥️ Screen: {len(image_bytes):,} bytes")
|
||||
except Exception as e:
|
||||
import traceback; traceback.print_exc()
|
||||
print(f"[ScreenProcess] ❌ Capture error: {e}")
|
||||
print(f"[Vision] ❌ Capture error: {e}")
|
||||
return False
|
||||
|
||||
print(f"[ScreenProcess] 📦 {len(image_bytes)} bytes → sending")
|
||||
_live.analyze(image_bytes, mime_type, user_text)
|
||||
_session.analyze(image_bytes, mime_type, user_text)
|
||||
return True
|
||||
|
||||
|
||||
def warmup_session(player=None):
|
||||
"""
|
||||
Optional: pre-warm the session.
|
||||
Do NOT call from main.py — causes double session issue.
|
||||
Only use when testing screen_processor.py standalone.
|
||||
"""
|
||||
def warmup_session(player=None) -> None:
|
||||
try:
|
||||
_ensure_started(player=player)
|
||||
_ensure_session(player=player)
|
||||
except Exception as e:
|
||||
print(f"[ScreenProcess] ⚠️ Warmup error: {e}")
|
||||
|
||||
print(f"[Vision] ⚠️ Warmup failed: {e}")
|
||||
|
||||
if __name__ == "__main__":
|
||||
print("[TEST] screen_processor.py v8 — image-only session")
|
||||
print("=" * 50)
|
||||
mode = input("screen / camera (default: screen): ").strip().lower() or "screen"
|
||||
request = input("Question (Enter for default): ").strip() or "What do you see? Be brief."
|
||||
print("[TEST] screen_processor.py")
|
||||
print("=" * 52)
|
||||
mode = input("angle — screen / camera (default: screen): ").strip().lower() or "screen"
|
||||
q = input("Question (Enter = default): ").strip() or "What do you see? Be brief."
|
||||
|
||||
t0 = time.perf_counter()
|
||||
warmup_session()
|
||||
print(f"Session ready — {time.perf_counter()-t0:.2f}s\n")
|
||||
print(f"Session ready in {time.perf_counter()-t0:.2f}s\n")
|
||||
|
||||
t1 = time.perf_counter()
|
||||
result = screen_process({"angle": mode, "text": request}, player=None)
|
||||
print(f"Sent — {time.perf_counter()-t1:.3f}s | audio incoming...")
|
||||
time.sleep(8)
|
||||
print(f"\n{'✅' if result else '❌'}")
|
||||
t1 = time.perf_counter()
|
||||
ok = screen_process({"angle": mode, "text": q})
|
||||
print(f"Queued in {time.perf_counter()-t1:.3f}s — waiting for audio...")
|
||||
time.sleep(10)
|
||||
print("Done." if ok else "Failed.")
|
||||
Reference in new issue
Block a user