Add files via upload

actions/calorie_counter.py

@@ -0,0 +1,253 @@
+"""
+JARVIS plugin — Calorie Counter (webcam vision).
+
+Hold food up to the camera and ask "how many calories is this?" —
+JARVIS switches the HUD to the live camera with an animated scan bar,
+photographs the food, analyzes it with Gemini, speaks a short summary
+in your language and shows the full nutrition breakdown in the
+content panel.
+
+Visuals rely on MainWindow's camera signals (_cam_stream_sig /
+_cam_frame_sig, present since Mark LI). If they're ever missing the
+plugin still works — just without the camera view.
+"""
+
+import json
+import threading
+import time
+from pathlib import Path
+
+import cv2
+import numpy as np
+
+BASE_DIR = Path(__file__).resolve().parent.parent
+
+PLUGIN = {
+    "name": "calorie_counter",
+    "description": (
+        "Analyzes FOOD through the WEBCAM and reports calories and nutrition "
+        "(carbs, sugar, fiber, protein, fat). Use whenever the user asks about "
+        "the calories or nutritional value of food they are physically holding "
+        "or showing right now — e.g. 'bu elimdeki tabak kaç kalori', 'how many "
+        "calories is this', 'şunun besin değeri ne'. This tool takes its OWN "
+        "camera photo — NEVER use screen_process for food-calorie questions. "
+        "Pass the user's exact spoken words in 'query'."
+    ),
+    "parameters": {
+        "type": "OBJECT",
+        "properties": {
+            "query": {
+                "type": "STRING",
+                "description": "The user's exact request, verbatim, in their own language.",
+            }
+        },
+        "required": ["query"],
+    },
+}
+
+_MODEL             = "gemini-flash-latest"
+_LIVE_SCAN_SECONDS = 1.8     # live preview before the photo is taken
+_FPS               = 25
+_ANIM_MAX_SECONDS  = 25      # animator safety stop
+_SCAN_COLOR        = (255, 190, 40)    # JARVIS cyan-blue (BGR)
+_SCAN_CORE         = (255, 235, 130)   # bright core line (BGR)
+
+
+# ── config helpers (same pattern as actions/web_search.py) ──────────────────
+
+def _config() -> dict:
+    try:
+        return json.loads(
+            (BASE_DIR / "config" / "api_keys.json").read_text(encoding="utf-8")
+        )
+    except Exception:
+        return {}
+
+
+def _open_camera():
+    import platform
+    try:
+        backend = cv2.CAP_DSHOW if platform.system() == "Windows" else cv2.CAP_ANY
+    except AttributeError:
+        backend = 0
+    cap = cv2.VideoCapture(int(_config().get("camera_index", 0)), backend)
+    if not cap.isOpened():
+        cap = cv2.VideoCapture(0)
+    if not cap.isOpened():
+        return None
+    for _ in range(6):  # warm-up frames
+        cap.read()
+    return cap
+
+
+# ── scan-bar rendering ───────────────────────────────────────────────────────
+
+def _draw_scan_bar(frame: np.ndarray, phase: float) -> np.ndarray:
+    """Return a copy of the BGR frame with an animated scan bar sweeping
+    bottom → top → bottom (triangle wave on `phase`)."""
+    h, w = frame.shape[:2]
+    pos = phase % 2.0
+    pos = pos if pos <= 1.0 else 2.0 - pos
+    y    = int((1.0 - pos) * (h - 1))
+    band = max(6, h // 14)
+
+    out = frame.copy()
+    y0, y1 = max(0, y - band), min(h, y + band)
+    if y1 > y0:
+        region = out[y0:y1].astype(np.float32)
+        falloff = 1.0 - (np.abs(np.arange(y0, y1) - y).astype(np.float32) / band)
+        alpha = falloff[:, None, None] * 0.55
+        glow = np.empty_like(region)
+        glow[:] = _SCAN_COLOR
+        out[y0:y1] = np.clip(region * (1 - alpha) + glow * alpha, 0, 255).astype(np.uint8)
+    cv2.line(out, (0, y), (w, y), _SCAN_CORE, 2)
+    return out
+
+
+def _emit_frame(frame_sig, frame: np.ndarray) -> None:
+    ok, buf = cv2.imencode(".jpg", frame, [cv2.IMWRITE_JPEG_QUALITY, 70])
+    if ok:
+        frame_sig.emit(buf.tobytes())
+
+
+# ── Gemini ───────────────────────────────────────────────────────────────────
+
+def _analyze(photo: np.ndarray, query: str, api_key: str) -> dict:
+    from google import genai
+    from google.genai import types as gtypes
+
+    # match screen_processor's upload size: max 1280 wide
+    h, w = photo.shape[:2]
+    if w > 1280:
+        photo = cv2.resize(photo, (1280, int(h * 1280 / w)), interpolation=cv2.INTER_AREA)
+    ok, jpg = cv2.imencode(".jpg", photo, [cv2.IMWRITE_JPEG_QUALITY, 85])
+    if not ok:
+        raise RuntimeError("could not encode photo")
+
+    prompt = (
+        "You are a nutrition analysis engine. Look at the food in this photo.\n"
+        f'User\'s request (respond in the SAME language as this request): "{query}"\n'
+        "Return ONLY minified JSON — no markdown fences, no extra text — with keys:\n"
+        ' "food": short name of the dish/foods (null if NO food is visible),\n'
+        ' "portion": estimated portion size as short text,\n'
+        ' "calories_kcal": number (total estimate),\n'
+        ' "carbs_g", "sugar_g", "fiber_g", "protein_g", "fat_g": numbers,\n'
+        ' "panel_text": multi-line plain-text nutrition breakdown in the user\'s'
+        " language — food name, portion, total calories, then one line per macro"
+        " (carbs, sugar, fiber, protein, fat) with units; if several foods are"
+        " visible add a short per-item calorie list; max 25 lines,\n"
+        ' "spoken_summary": 1-2 conversational sentences in the user\'s language'
+        " naming the food and total calories, mentioning it is an estimate."
+        " If no food is visible, politely say so instead."
+    )
+
+    client = genai.Client(api_key=api_key)
+    resp = client.models.generate_content(
+        model=_MODEL,
+        contents=[
+            gtypes.Part.from_bytes(data=jpg.tobytes(), mime_type="image/jpeg"),
+            prompt,
+        ],
+    )
+    text = (resp.text or "").strip()
+
+    # tolerate accidental fences / prose around the JSON
+    if "{" in text and "}" in text:
+        text = text[text.find("{"): text.rfind("}") + 1]
+    return json.loads(text)
+
+
+# ── entry point ──────────────────────────────────────────────────────────────
+
+def run(parameters: dict, player=None, session_memory=None) -> str:
+    query = (parameters.get("query") or "").strip() or "How many calories is this food?"
+
+    api_key = _config().get("gemini_api_key")
+    if not api_key:
+        return "I can't run the nutrition scan — no API key is configured."
+
+    def _log(msg: str) -> None:
+        if player:
+            try:
+                player.write_log(msg)
+            except Exception:
+                pass
+
+    # HUD camera signals (graceful if unavailable)
+    win        = getattr(player, "_win", None) if player else None
+    frame_sig  = getattr(win, "_cam_frame_sig", None)
+    stream_sig = getattr(win, "_cam_stream_sig", None)
+
+    cap = _open_camera()
+    if cap is None:
+        return ("I couldn't access the camera — it may be in use by another "
+                "feature or application.")
+
+    photo = None
+    stop_anim = threading.Event()
+    animator = None
+    view_open = False
+    try:
+        if stream_sig:
+            stream_sig.emit(True)   # HUD logo → live camera view
+            view_open = True
+        _log("JARVIS: Nutrition scan started.")
+
+        # Phase 1 — live preview with scan bar (user positions the food)
+        t0 = time.time()
+        last = None
+        while time.time() - t0 < _LIVE_SCAN_SECONDS:
+            ok, frm = cap.read()
+            if ok and frm is not None:
+                last = frm
+                if frame_sig:
+                    _emit_frame(frame_sig, _draw_scan_bar(frm, (time.time() - t0) * 1.2))
+            time.sleep(1.0 / _FPS)
+        ok, frm = cap.read()
+        photo = frm if ok and frm is not None else last
+    finally:
+        try:
+            cap.release()   # release BEFORE anything else — avoid device conflicts
+        except Exception:
+            pass
+
+    if photo is None:
+        if view_open:
+            stream_sig.emit(False)
+        return "I couldn't capture a picture of the food, sorry."
+
+    # Phase 2 — freeze frame, keep the scan bar sweeping while Gemini analyzes
+    if frame_sig:
+        def _animate():
+            a0 = time.time()
+            while (not stop_anim.wait(1.0 / _FPS)
+                   and time.time() - a0 < _ANIM_MAX_SECONDS):
+                _emit_frame(frame_sig, _draw_scan_bar(photo, (time.time() - a0) * 1.2))
+        animator = threading.Thread(target=_animate, daemon=True,
+                                    name="calorie-scan-anim")
+        animator.start()
+
+    try:
+        data = _analyze(photo, query, api_key)
+    except Exception as e:
+        return f"The nutrition analysis failed: {e}"
+    finally:
+        stop_anim.set()
+        if animator:
+            animator.join(timeout=1)
+        if view_open:
+            stream_sig.emit(False)  # back to the JARVIS HUD
+
+    spoken = (data.get("spoken_summary") or "").strip()
+
+    if data.get("food"):
+        panel = (data.get("panel_text") or "").strip()
+        if player and panel:
+            try:
+                player.show_content("🍽 NUTRITION SCAN", panel)
+            except Exception:
+                pass
+        _log(f"JARVIS: Nutrition scan complete — {data.get('food')}"
+             f" ≈ {data.get('calories_kcal')} kcal.")
+
+    return spoken or "The scan finished, but I couldn't read the result."

actions/code_helper.py

@@ -15,7 +15,7 @@ BASE_DIR           = get_base_dir()
 API_CONFIG_PATH    = BASE_DIR / "config" / "api_keys.json"
 DESKTOP            = Path.home() / "Desktop"
 MAX_BUILD_ATTEMPTS = 3
-GEMINI_MODEL       = "gemini-2.5-flash"
+GEMINI_MODEL       = "gemini-flash-latest"
 
 
 def _get_api_key() -> str:
@@ -488,7 +488,7 @@ Be specific and actionable. If you see an error message, quote it exactly."""
         ]
 
         response = client.models.generate_content(
-            model="gemini-2.5-flash",
+            model="gemini-flash-latest",
             contents=contents,
         )
 

actions/computer_control.py

@@ -336,7 +336,7 @@ def _screen_find(description: str) -> tuple[int, int] | None:
         )
 
         response = client.models.generate_content(
-            model="gemini-2.5-flash-lite",
+            model="gemini-flash-lite-latest",
             contents=[
                 gtypes.Part.from_bytes(data=image_bytes, mime_type="image/png"),
                 prompt,

actions/computer_settings.py

@@ -613,7 +613,7 @@ Rules:
 - Return ONLY the JSON, no explanation, no markdown."""
 
     try:
-        resp = _client.models.generate_content(model="gemini-2.5-flash-lite", contents=prompt)
+        resp = _client.models.generate_content(model="gemini-flash-lite-latest", contents=prompt)
         text = re.sub(r"```(?:json)?", "", resp.text).strip().rstrip("`").strip()
         return json.loads(text)
     except Exception as e:

actions/desktop.py

@@ -142,7 +142,7 @@ Output ONLY the Python code. No explanation, no markdown, no backticks.
 Task: {task}"""
 
     try:
-        response = _client.models.generate_content(model="gemini-2.5-flash", contents=prompt)
+        response = _client.models.generate_content(model="gemini-flash-latest", contents=prompt)
         code = response.text.strip()
         if code.startswith("```"):
             lines = code.split("\n")

actions/dev_agent.py

@@ -16,8 +16,8 @@ BASE_DIR         = get_base_dir()
 API_CONFIG_PATH  = BASE_DIR / "config" / "api_keys.json"
 PROJECTS_DIR     = Path.home() / "Desktop" / "JarvisProjects"
 MAX_FIX_ATTEMPTS = 5
-MODEL_PLANNER    = "gemini-2.5-flash"
-MODEL_WRITER     = "gemini-2.5-flash"
+MODEL_PLANNER    = "gemini-flash-latest"
+MODEL_WRITER     = "gemini-flash-latest"
 
 def _get_api_key() -> str:
     with open(API_CONFIG_PATH, "r", encoding="utf-8") as f:

actions/file_processor.py

@@ -37,7 +37,7 @@ def _gemini_client():
 
     class _W:
         def generate_content(self, contents):
-            return _c.models.generate_content(model="gemini-2.5-flash", contents=contents)
+            return _c.models.generate_content(model="gemini-flash-latest", contents=contents)
 
     return _W()
 

actions/flight_finder.py

@@ -65,7 +65,7 @@ def _parse_date(raw: str) -> str:
         from google import genai as _genai
         _client  = _genai.Client(api_key=_get_api_key())
         response = _client.models.generate_content(
-            model="gemini-2.5-flash-lite",
+            model="gemini-flash-lite-latest",
             contents=(
                 f"Today is {today.strftime('%Y-%m-%d')}. "
                 f"Convert this date expression to YYYY-MM-DD: '{raw}'. "
@@ -169,7 +169,7 @@ def _parse_flights_with_gemini(
 
     try:
         response = _client.models.generate_content(
-            model="gemini-2.5-flash",
+            model="gemini-flash-latest",
             contents=prompt,
             config=types.GenerateContentConfig(
                 system_instruction=(

actions/pushup_counter.py

@@ -0,0 +1,377 @@
+"""
+JARVIS plugin — Pushup Counter (webcam vision).
+
+Say "Jarvis, count my pushups" (any language) — the HUD switches to the
+live camera, a 5-second countdown lets you get into position, then JARVIS
+counts your reps live on screen and finally delivers the result with
+playful, teasing motivational commentary in your language.
+
+Detection (two-tier):
+  • If `mediapipe` is installed (pip install mediapipe): elbow-angle
+    tracking — accurate from front or side view.
+  • Otherwise: face-motion tracking with OpenCV only (zero extra
+    dependencies) — place the camera on the floor in front of you so your
+    head moves toward/away from it with each rep.
+
+Session ends automatically: target reached, ~20 s without a rep, or the
+3-minute safety cap. Just stand up when you're done.
+
+Sessions are saved to memory/long_term.json ("pushup_sessions") so future
+versions can track progress over time.
+"""
+
+import json
+import time
+from datetime import date
+from pathlib import Path
+
+import cv2
+import numpy as np
+
+BASE_DIR = Path(__file__).resolve().parent.parent
+
+PLUGIN = {
+    "name": "pushup_counter",
+    "description": (
+        "Counts the user's pushups live through the WEBCAM. Use whenever the "
+        "user says they are about to do pushups and wants them counted — e.g. "
+        "'şınav çekeceğim say', 'count my pushups', 'pushup challenge'. "
+        "If the user mentions a goal ('20 şınav çekeceğim'), pass it as "
+        "'target'. The tool runs the WHOLE workout and only returns when the "
+        "session ends (this can take a few minutes) — do not call other tools "
+        "meanwhile. When the result comes back, deliver it in the user's "
+        "language with playful, lightly TEASING motivational commentary "
+        "(e.g. for 10 reps: congratulate them for being 'a perfectly average "
+        "human'). Never use screen_process for pushup counting."
+    ),
+    "parameters": {
+        "type": "OBJECT",
+        "properties": {
+            "query": {
+                "type": "STRING",
+                "description": "The user's exact request, verbatim, in their own language.",
+            },
+            "target": {
+                "type": "NUMBER",
+                "description": "Rep goal if the user stated one (e.g. 20). Omit otherwise.",
+            },
+        },
+        "required": ["query"],
+    },
+}
+
+_COUNTDOWN_SECONDS   = 5
+_MAX_SESSION_SECONDS = 180
+_IDLE_END_SECONDS    = 20    # end after this long without a rep (once started)
+_IDLE_ABORT_SECONDS  = 40    # abort if NO rep ever happens
+_FPS                 = 20
+_CYAN   = (255, 190, 40)     # JARVIS cyan-blue (BGR)
+_BRIGHT = (255, 235, 130)
+
+
+# ── config / camera (same pattern as calorie_counter) ───────────────────────
+
+def _config() -> dict:
+    try:
+        return json.loads(
+            (BASE_DIR / "config" / "api_keys.json").read_text(encoding="utf-8")
+        )
+    except Exception:
+        return {}
+
+
+def _open_camera():
+    import platform
+    try:
+        backend = cv2.CAP_DSHOW if platform.system() == "Windows" else cv2.CAP_ANY
+    except AttributeError:
+        backend = 0
+    cap = cv2.VideoCapture(int(_config().get("camera_index", 0)), backend)
+    if not cap.isOpened():
+        cap = cv2.VideoCapture(0)
+    if not cap.isOpened():
+        return None
+    for _ in range(6):
+        cap.read()
+    return cap
+
+
+def _emit_frame(frame_sig, frame: np.ndarray) -> None:
+    ok, buf = cv2.imencode(".jpg", frame, [cv2.IMWRITE_JPEG_QUALITY, 70])
+    if ok:
+        frame_sig.emit(buf.tobytes())
+
+
+# ── rep counting state machine ───────────────────────────────────────────────
+
+class _RepCounter:
+    """Counts full down→up cycles of a raw 'down-ness' signal (bigger =
+    lower). Tracks the signal's own min/max range (with slow decay) so it
+    self-calibrates to any camera distance or body size."""
+
+    def __init__(self, min_rel_amplitude: float = 0.18):
+        self._min_rel = min_rel_amplitude
+        self.lo = None
+        self.hi = None
+        self.state = "up"
+        self.reps = 0
+
+    def update(self, v: float) -> bool:
+        if self.lo is None:
+            self.lo = self.hi = float(v)
+            return False
+        # decay range toward the current value, then expand to include it
+        self.lo = min(self.lo + (v - self.lo) * 0.002, v)
+        self.hi = max(self.hi + (v - self.hi) * 0.002, v)
+        rng = self.hi - self.lo
+        if rng < max(abs(self.hi), 1e-6) * self._min_rel:
+            return False   # not enough movement observed yet
+        pos = (v - self.lo) / rng
+        if self.state == "up" and pos > 0.70:
+            self.state = "down"
+        elif self.state == "down" and pos < 0.30:
+            self.state = "up"
+            self.reps += 1
+            return True
+        return False
+
+
+# ── detectors: return a 'down-ness' signal (bigger = lower), or None ────────
+
+def _make_pose_detector():
+    """MediaPipe elbow-angle tracking. Returns fn(frame)->float|None,
+    or None if mediapipe isn't installed."""
+    try:
+        import mediapipe as mp
+        pose = mp.solutions.pose.Pose(model_complexity=0,
+                                      min_detection_confidence=0.5,
+                                      min_tracking_confidence=0.5)
+        L = mp.solutions.pose.PoseLandmark
+    except Exception:
+        # mediapipe missing, or an incompatible version without the
+        # legacy solutions API — face tracking takes over either way
+        return None
+
+    def _angle(a, b, c) -> float:
+        ba = np.array([a.x - b.x, a.y - b.y])
+        bc = np.array([c.x - b.x, c.y - b.y])
+        cos = np.dot(ba, bc) / (np.linalg.norm(ba) * np.linalg.norm(bc) + 1e-9)
+        return float(np.degrees(np.arccos(np.clip(cos, -1.0, 1.0))))
+
+    def _signal(frame: np.ndarray):
+        res = pose.process(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB))
+        lm = getattr(res, "pose_landmarks", None)
+        if not lm:
+            return None
+        pts = lm.landmark
+        best = None
+        for s, e, w in ((L.LEFT_SHOULDER, L.LEFT_ELBOW, L.LEFT_WRIST),
+                        (L.RIGHT_SHOULDER, L.RIGHT_ELBOW, L.RIGHT_WRIST)):
+            vis = min(pts[s].visibility, pts[e].visibility, pts[w].visibility)
+            if vis < 0.5:
+                continue
+            ang = _angle(pts[s], pts[e], pts[w])   # 180 = arm straight
+            if best is None or vis > best[1]:
+                best = (ang, vis)
+        if best is None:
+            return None
+        # straight arm (170°) -> 0.0 (up) … bent arm (60°) -> 1.0 (down)
+        return float(np.clip((170.0 - best[0]) / 110.0, 0.0, 1.0))
+
+    return _signal
+
+
+def _make_face_detector():
+    """OpenCV-only fallback: face area grows as the user lowers toward a
+    floor-placed camera. Returns fn(frame)->float|None."""
+    cascade = cv2.CascadeClassifier(
+        cv2.data.haarcascades + "haarcascade_frontalface_default.xml"
+    )
+
+    def _signal(frame: np.ndarray):
+        gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
+        small = cv2.resize(gray, (0, 0), fx=0.5, fy=0.5)
+        faces = cascade.detectMultiScale(small, 1.15, 4, minSize=(30, 30))
+        if len(faces) == 0:
+            return None
+        x, y, w, h = max(faces, key=lambda f: f[2] * f[3])
+        return float(w * h)   # bigger face = closer = down
+
+    return _signal
+
+
+# ── on-frame overlay (ASCII/digits only — cv2.putText can't render more) ────
+
+def _draw_overlay(frame, reps, target, state, flash_until, countdown=None):
+    out = frame.copy()
+    h, w = out.shape[:2]
+    if countdown is not None:
+        txt = str(countdown)
+        size = cv2.getTextSize(txt, cv2.FONT_HERSHEY_DUPLEX, 6.0, 8)[0]
+        cv2.putText(out, txt, ((w - size[0]) // 2, (h + size[1]) // 2),
+                    cv2.FONT_HERSHEY_DUPLEX, 6.0, _BRIGHT, 8, cv2.LINE_AA)
+        return out
+    big = time.time() < flash_until
+    scale = 4.0 if big else 2.8
+    color = _BRIGHT if big else _CYAN
+    label = f"{reps}" + (f"/{int(target)}" if target else "")
+    cv2.putText(out, label, (24, 90), cv2.FONT_HERSHEY_DUPLEX,
+                scale, color, 6 if big else 4, cv2.LINE_AA)
+    # state hint bar at the bottom: fills while DOWN
+    bar_w = int(w * (0.9 if state == "down" else 0.15))
+    cv2.rectangle(out, (0, h - 10), (bar_w, h), _CYAN, -1)
+    return out
+
+
+# ── persistence (memory/long_term.json, additive — other keys preserved) ────
+
+def _record_session(reps: int, seconds: float) -> int:
+    """Append session, return today's total reps. Never raises."""
+    try:
+        path = BASE_DIR / "memory" / "long_term.json"
+        data = {}
+        if path.exists():
+            try:
+                data = json.loads(path.read_text(encoding="utf-8"))
+            except Exception:
+                data = {}
+        sessions = data.get("pushup_sessions")
+        if not isinstance(sessions, list):
+            sessions = []
+        today = date.today().isoformat()
+        sessions.append({"date": today, "reps": reps, "seconds": round(seconds)})
+        data["pushup_sessions"] = sessions[-100:]
+        path.write_text(json.dumps(data, indent=2, ensure_ascii=False),
+                        encoding="utf-8")
+        return sum(s.get("reps", 0) for s in sessions if s.get("date") == today)
+    except Exception:
+        return reps
+
+
+# ── entry point ──────────────────────────────────────────────────────────────
+
+def run(parameters: dict, player=None, session_memory=None) -> str:
+    try:
+        target = int(parameters.get("target") or 0)
+    except Exception:
+        target = 0
+
+    def _log(msg: str) -> None:
+        if player:
+            try:
+                player.write_log(msg)
+            except Exception:
+                pass
+
+    win        = getattr(player, "_win", None) if player else None
+    frame_sig  = getattr(win, "_cam_frame_sig", None)
+    stream_sig = getattr(win, "_cam_stream_sig", None)
+
+    detector = _make_pose_detector()
+    mode = "elbow tracking (mediapipe)"
+    if detector is None:
+        detector = _make_face_detector()
+        mode = "face-motion tracking (place camera on the floor in front of you)"
+
+    cap = _open_camera()
+    if cap is None:
+        return ("I couldn't access the camera — it may be in use by another "
+                "feature or application.")
+
+    counter = _RepCounter()
+    flash_until = 0.0
+    view_open = False
+    started = time.time()
+    last_rep_time = started
+    end_reason = "finished"
+    try:
+        if stream_sig:
+            stream_sig.emit(True)
+            view_open = True
+        _log(f"JARVIS: Pushup session started — {mode}.")
+
+        # Phase 1 — countdown so the user can get into position
+        t0 = time.time()
+        while time.time() - t0 < _COUNTDOWN_SECONDS:
+            ok, frm = cap.read()
+            if ok and frm is not None and frame_sig:
+                remaining = _COUNTDOWN_SECONDS - int(time.time() - t0)
+                _emit_frame(frame_sig,
+                            _draw_overlay(frm, 0, target, "up", 0,
+                                          countdown=max(1, remaining)))
+            time.sleep(1.0 / _FPS)
+
+        # Phase 2 — count
+        started = time.time()
+        last_rep_time = started
+        while True:
+            now = time.time()
+            if now - started > _MAX_SESSION_SECONDS:
+                end_reason = "time limit"
+                break
+            idle = now - last_rep_time
+            if counter.reps > 0 and idle > _IDLE_END_SECONDS:
+                end_reason = "user stopped"
+                break
+            if counter.reps == 0 and idle > _IDLE_ABORT_SECONDS:
+                end_reason = "no pushups detected"
+                break
+            if target and counter.reps >= target:
+                end_reason = "target reached"
+                break
+
+            ok, frm = cap.read()
+            if not ok or frm is None:
+                time.sleep(1.0 / _FPS)
+                continue
+            sig = detector(frm)
+            if sig is not None and counter.update(sig):
+                last_rep_time = time.time()
+                flash_until = last_rep_time + 0.6
+                if counter.reps % 10 == 0 or (target and counter.reps == target):
+                    _log(f"JARVIS: Pushup {counter.reps} 💪")
+            if frame_sig:
+                _emit_frame(frame_sig,
+                            _draw_overlay(frm, counter.reps, target,
+                                          counter.state, flash_until))
+            time.sleep(1.0 / _FPS)
+    finally:
+        try:
+            cap.release()
+        except Exception:
+            pass
+        if view_open:
+            stream_sig.emit(False)
+
+    reps = counter.reps
+    seconds = max(1.0, time.time() - started)
+
+    if reps == 0:
+        return ("The session ended without a single detected pushup "
+                f"({end_reason}). Tell the user — with gentle teasing — that "
+                "zero pushups were counted, and remind them the camera must "
+                "clearly see them (or their face, in face-tracking mode).")
+
+    today_total = _record_session(reps, seconds)
+    pace = reps / (seconds / 60.0)
+
+    if player:
+        try:
+            player.show_content(
+                "💪 PUSHUP SESSION",
+                (f"Reps: {reps}" + (f" / {target}" if target else "") + "\n"
+                 f"Duration: {int(seconds)} s\n"
+                 f"Pace: {pace:.1f} reps/min\n"
+                 f"Today's total: {today_total}\n"
+                 f"Ended: {end_reason}\n"
+                 f"Mode: {mode}"),
+            )
+        except Exception:
+            pass
+
+    return (f"Pushup session finished ({end_reason}). Reps: {reps}"
+            + (f" of the {target} targeted" if target else "")
+            + f". Duration: {int(seconds)} seconds ({pace:.1f} reps/min)."
+            f" Today's total: {today_total}. Deliver this to the user in "
+            "their language with playful, lightly teasing motivational "
+            "commentary about the number.")

actions/upload_video.py

@@ -0,0 +1,344 @@
+#upload_video.py
+"""
+Hardcoded automation: uploads the video sitting on the Desktop (named 'video')
+to TikTok Studio, using a fixed, always-identical sequence of steps — same
+order, every single time, no improvisation.
+
+The user gives only a rough caption BRIEF ("write that I made a plugin system
+in the new version") — this plugin expands it with Gemini into a full,
+SEO-keyword-rich TikTok caption in the user's own language before the
+automation starts, so no Gemini latency lands in the middle of the click
+sequence.
+
+Opera GX is launched the normal way (Win key, type, Enter) — exactly like a
+human opening it — so it comes up as the user's real, already-logged-in
+browser instead of a separate automated instance with a blank profile.
+
+Flow (fixed): Opera GX → new tab → tiktok.com/tiktokstudio/upload → click
+screen center → Tab → Enter (file picker opens) → Ctrl+F → type 'video' →
+open the found file → Tab → Ctrl+A → paste caption → 16×Tab → Enter. Done.
+
+pyautogui.FAILSAFE stays on: slamming the mouse into the top-left corner
+aborts the automation instantly.
+"""
+import json
+import sys
+import threading
+import time
+from pathlib import Path
+
+try:
+    import pyautogui
+    pyautogui.FAILSAFE = True
+    pyautogui.PAUSE    = 0.05
+    _PYAUTOGUI = True
+except ImportError:
+    _PYAUTOGUI = False
+
+try:
+    import pyperclip
+    _PYPERCLIP = True
+except ImportError:
+    _PYPERCLIP = False
+
+
+PLUGIN = {
+    "name": "upload_video",
+    "description": (
+        "Uploads the video file named 'video' on the user's Desktop to TIKTOK "
+        "with a fully automated browser flow (Opera GX + TikTok Studio). Use "
+        "when the user asks to upload/post/share their video to TikTok — e.g. "
+        "'upload this video to TikTok', 'videoyu tiktok'a yükle'. "
+        "CAPTION RULE — STRICT: 'description' must contain the user's OWN "
+        "caption idea, passed VERBATIM in their language. NEVER invent, "
+        "guess, or auto-fill it yourself. If the user's request contains no "
+        "caption topic, DO NOT call this tool yet — first ASK the user what "
+        "the caption should be about, and call only after they answer. The "
+        "single exception: if the user explicitly delegates it ('sen yaz', "
+        "'default olsun', 'you decide'), you may then pass your own brief. "
+        "The tool itself expands the brief into a full SEO caption — do NOT "
+        "expand it yourself. Call this tool EXACTLY ONCE per request. "
+        "The tool returns IMMEDIATELY while the upload keeps running in the "
+        "background — when the result arrives, announce the start exactly as "
+        "it instructs (uploading now + hands off mouse/keyboard), then wait: "
+        "progress remarks and the completion announcement arrive by "
+        "themselves. Do NOT call the tool again to check status. Never use "
+        "browser_control or computer_control for TikTok uploads."
+    ),
+    "parameters": {
+        "type": "OBJECT",
+        "properties": {
+            "description": {
+                "type": "STRING",
+                "description": ("The user's rough caption brief, verbatim, in "
+                                "their own language."),
+            }
+        },
+        "required": ["description"],
+    },
+}
+
+_POST_COOLDOWN_SEC = 20.0  # ignore repeat calls for this long after a successful post
+_last_post_at      = 0.0
+_upload_running    = False  # a background upload is in progress right now
+
+
+def _get_base_dir() -> Path:
+    if getattr(sys, "frozen", False):
+        return Path(sys.executable).parent
+    return Path(__file__).resolve().parent.parent
+
+
+BASE_DIR        = _get_base_dir()
+API_CONFIG_PATH = BASE_DIR / "config" / "api_keys.json"
+
+
+def _get_api_key() -> str:
+    with open(API_CONFIG_PATH, "r", encoding="utf-8") as f:
+        return json.load(f)["gemini_api_key"]
+
+
+def _require_pyautogui() -> None:
+    if not _PYAUTOGUI:
+        raise RuntimeError("PyAutoGUI not installed. Run: pip install pyautogui")
+
+
+def _paste(text: str) -> None:
+    """Types text via clipboard paste so non-ASCII characters and emoji survive."""
+    _require_pyautogui()
+    if _PYPERCLIP:
+        pyperclip.copy(text)
+        time.sleep(0.15)
+        pyautogui.hotkey("ctrl", "v")
+    else:
+        pyautogui.typewrite(text, interval=0.02)
+    time.sleep(0.2)
+
+
+def _generate_caption(brief: str) -> str:
+    """Expands the user's rough brief into a LONG, SEO-heavy TikTok caption
+    with Gemini, in the same language as the brief, keyword-targeted at
+    Tier-1 audiences (US/UK/CA/AU/DE...). Falls back to the brief itself if
+    the call fails — the upload must never be blocked by a caption error."""
+    try:
+        from google import genai
+        client = genai.Client(api_key=_get_api_key())
+        prompt = (
+            "You are a TikTok SEO copywriter. The user gives you only a rough "
+            "TITLE/BRIEF; you turn it into ONE ready-to-post, LONG caption in "
+            "the SAME language as the brief:\n"
+            f'BRIEF: "{brief}"\n'
+            "Rules:\n"
+            "- LONG and substantial: at least 1000 characters of body text "
+            "(not counting hashtags). Expand the brief into a detailed, "
+            "engaging story/description — this is for search ranking, not a "
+            "one-liner.\n"
+            "- Strong scroll-stopping hook as the first line.\n"
+            "- Weave in plenty of high-search-volume SEO keywords and phrases "
+            "naturally throughout the body, chosen to rank with TIER-1 "
+            "audiences (US, UK, Canada, Australia, Germany...). Keywords may "
+            "be in English even if the body language differs.\n"
+            "- End with a call-to-action (follow/comment/share), then 15-25 "
+            "hashtags: mix broad high-volume Tier-1 English hashtags with "
+            "niche topic hashtags.\n"
+            "- Plain text only — no quotes, no markdown, no explanations.\n"
+            "- Maximum 3500 characters total."
+        )
+        r = client.models.generate_content(model="gemini-flash-latest", contents=prompt)
+        caption = (r.text or "").strip()
+        return caption[:3900] if caption else brief
+    except Exception as e:
+        print(f"[UploadVideo] Caption generation failed, using brief as-is: {e}")
+        return brief
+
+
+def _run_upload_flow(brief: str, caption_holder: dict,
+                     caption_ready: threading.Event) -> str:
+    _require_pyautogui()
+    w, h = pyautogui.size()
+
+    # Open Opera GX exactly like a human would (Win key -> type -> Enter)
+    pyautogui.press("win")
+    time.sleep(0.7)
+    pyautogui.write("opera gx", interval=0.05)
+    time.sleep(0.9)
+    pyautogui.press("enter")
+    time.sleep(3.0)  # give Opera GX time to come to the foreground
+
+    # New tab -> TikTok Studio upload page
+    pyautogui.hotkey("ctrl", "t")
+    time.sleep(0.6)
+    _paste("https://www.tiktok.com/tiktokstudio/upload")
+    time.sleep(0.2)
+    pyautogui.press("enter")
+    time.sleep(5.0)  # page load
+
+    # Click dead center of the screen (upload dropzone area), then Tab + Enter
+    # to activate the 'Select video' control -> native file picker opens
+    pyautogui.click(w // 2, h // 2)
+    time.sleep(0.8)
+    pyautogui.press("tab")
+    time.sleep(0.3)
+    pyautogui.press("enter")
+    time.sleep(2.0)
+
+    # Ctrl+F is a fixed OS-level shortcut that always opens the Open dialog's
+    # search box regardless of screen/resolution -- no lookup needed.
+    pyautogui.hotkey("ctrl", "f")
+    time.sleep(0.6)
+    pyautogui.write("video", interval=0.03)
+    time.sleep(1.5)  # let Explorer filter the results down to the matching file
+
+    # Open the found file. 180x120 on the reference 1920x1080 layout, scaled
+    # to the actual resolution. Double-click = select + open in one motion
+    # (same proven pattern as the old Instagram flow's file pick).
+    pyautogui.doubleClick(int(w * 240 / 1920), int(h * 135 / 1080))
+    time.sleep(4.0)  # TikTok starts processing the upload; caption box appears
+
+    # Caption: Tab once into the field, select the auto-filled text, replace it.
+    # The SEO caption has been generating in a background thread since the very
+    # first keystroke — by now (~20s in) it's almost always ready; wait max 15s
+    # more, then fall back to the raw brief so the flow never stalls.
+    caption_ready.wait(timeout=15)
+    caption = (caption_holder.get("caption") or brief).strip()
+    pyautogui.press("tab")
+    pyautogui.press("tab")
+    pyautogui.hotkey("ctrl", "a")
+    time.sleep(0.3)
+    _paste(caption)
+    time.sleep(3)
+
+    # 16 Tabs down to the Post button, Enter -> published
+    time.sleep(6)
+    for _ in range(17):
+        pyautogui.press("tab")
+        time.sleep(0.12)
+    time.sleep(1)
+    pyautogui.press("enter")
+    time.sleep(1)
+    pyautogui.click(int(w * 1089 / 1920), int(h * 653 / 1080))
+    pyautogui.click(int(w * 1200 / 1920), int(h * 655 / 1080))
+
+    return "OK"
+
+
+def run(parameters: dict, player=None, session_memory=None) -> str:
+    global _last_post_at, _upload_running
+
+    if _upload_running:
+        return ("The upload is STILL RUNNING right now — this call was "
+                "ignored. Do not call the tool again; briefly tell the user, "
+                "in their language, that it is still in progress.")
+
+    since_last = time.monotonic() - _last_post_at
+    if since_last < _POST_COOLDOWN_SEC:
+        print(f"[UploadVideo] Ignoring duplicate call ({since_last:.0f}s after last post)")
+        return ("The video was ALREADY posted successfully — this duplicate "
+                "call was ignored. Do not repeat any status message. Simply "
+                "confirm ONCE, in the user's language and in past tense, that "
+                "the video is posted (e.g. 'I posted the video, sir — you're "
+                "welcome.').")
+
+    brief = ((parameters or {}).get("description") or "").strip()
+    print(f"[UploadVideo] brief={'<empty>' if not brief else brief[:40]!r}")
+    if not brief:
+        return ("I need a caption idea before I share the video, sir. What "
+                "should it be about, or shall I write one myself?")
+
+    # Kick off SEO caption generation in the BACKGROUND — the click sequence
+    # starts immediately (within ~1s); the caption is only needed ~20s later
+    # at the paste step, by which time it's ready.
+    caption_holder: dict = {}
+    caption_ready = threading.Event()
+
+    def _gen():
+        try:
+            caption_holder["caption"] = _generate_caption(brief)
+        finally:
+            caption_ready.set()
+
+    threading.Thread(target=_gen, daemon=True, name="tiktok-caption").start()
+
+    def _log(msg: str) -> None:
+        if player:
+            try:
+                player.write_log(msg)
+            except Exception:
+                pass
+
+    _log("JARVIS: TikTok upload starting — please don't touch the mouse "
+         "or keyboard.")
+
+    say = getattr(player, "request_say", None) if player else None
+
+    # ── No speech channel (older core): fall back to the blocking flow ──────
+    if not say:
+        try:
+            result = _run_upload_flow(brief, caption_holder, caption_ready)
+        except Exception as e:
+            print(f"[UploadVideo] Error: {e}")
+            return f"An error occurred while sharing the video, sir: {e}"
+        if result != "OK":
+            return result
+        _last_post_at = time.monotonic()
+        return ("The video has been posted to TikTok successfully — announce "
+                "it in the user's language, in PAST tense, warm and brief. "
+                "Do NOT call upload_video again.")
+
+    # ── Two-phase mode (same trick as instant vision acknowledgment): the
+    #    tool returns IMMEDIATELY so Gemini can speak right away; the actual
+    #    automation, the mid-upload small talk and the completion announcement
+    #    all run in background threads through the plugin speech channel. ────
+    stop_chatter = threading.Event()
+
+    def _chatter():
+        remarks = [
+            ("[UPLOAD STATUS — speak now] The TikTok upload is underway in "
+             "the background. Make ONE warm, personal small-talk remark to "
+             "the user in their language — for example, kindly remind them "
+             "not to spend too much time on social media and to keep some "
+             "time for themselves, or a similar caring thought. One or two "
+             "sentences, natural tone, do not mention tools or steps."),
+            ("[UPLOAD STATUS — speak now] Still uploading, almost done. Make "
+             "ONE more brief, warm remark to the user in their language — a "
+             "different caring or encouraging thought than before, never "
+             "repeat earlier phrasing. One sentence, do not mention tools "
+             "or steps."),
+        ]
+        for delay, remark in zip((10.0, 12.0), remarks):
+            if stop_chatter.wait(delay):
+                return
+            try:
+                say(remark)
+            except Exception:
+                return
+
+    def _worker():
+        global _last_post_at, _upload_running
+        try:
+            result = _run_upload_flow(brief, caption_holder, caption_ready)
+            stop_chatter.set()
+            if result == "OK":
+                _last_post_at = time.monotonic()
+                posted = (caption_holder.get("caption") or brief)[:60]
+                _log(f"[UploadVideo] Posted to TikTok: \"{posted}...\"")
+                say("[UPLOAD FINISHED — speak now] The video has just been "
+                    "posted to TikTok successfully. Announce it to the user "
+                    "in their language, PAST tense, warm and brief — e.g. "
+                    "'I posted the video, sir — you're welcome.'")
+            else:
+                say(f"[UPLOAD PROBLEM — speak now] The upload could not be "
+                    f"completed: {result}. Tell the user in their language, "
+                    "briefly.")
+        except Exception as e:
+            print(f"[UploadVideo] Error: {e}")
+            stop_chatter.set()
+            say(f"[UPLOAD FAILED — speak now] The TikTok upload failed: {e}. "
+                "Briefly apologize and tell the user in their language.")
+        finally:
+            stop_chatter.set()
+            _upload_running = False
+
+    _upload_running = True
+    threading.Thread(target=_worker, daemon=True, name="tiktok-upload").start()
+    threading.Thread(target=_chatter, daemon=True, name="tiktok-chatter").start()

actions/web_search.py

@@ -1,308 +1,308 @@
-#web_search.py
-import json
-import sys
-from pathlib import Path
-
-def _get_base_dir() -> Path:
-    if getattr(sys, "frozen", False):
-        return Path(sys.executable).parent
-    return Path(__file__).resolve().parent.parent
-
-
-BASE_DIR        = _get_base_dir()
-API_CONFIG_PATH = BASE_DIR / "config" / "api_keys.json"
-
-
-def _get_api_key() -> str:
-    with open(API_CONFIG_PATH, "r", encoding="utf-8") as f:
-        return json.load(f)["gemini_api_key"]
-
-
-def _gemini_search(query: str) -> str:
-    from google import genai
-
-    client   = genai.Client(api_key=_get_api_key())
-    response = client.models.generate_content(
-        model="gemini-2.5-flash",
-        contents=query,
-        config={"tools": [{"google_search": {}}]},
-    )
-
-    text = ""
-    for part in response.candidates[0].content.parts:
-        if hasattr(part, "text") and part.text:
-            text += part.text
-
-    text = text.strip()
-    if not text:
-        raise ValueError("Gemini returned an empty response.")
-    return text
-
-
-def _ddg_search(query: str, max_results: int = 6) -> list[dict]:
-    try:
-        from ddgs import DDGS
-    except ImportError:
-        from duckduckgo_search import DDGS
-
-    results = []
-    with DDGS() as ddgs:
-        for r in ddgs.text(query, max_results=max_results):
-            results.append({
-                "title":   r.get("title",  ""),
-                "snippet": r.get("body",   ""),
-                "url":     r.get("href",   ""),
-            })
-    return results
-
-
-def _ddg_news(query: str, max_results: int = 8) -> list[dict]:
-    """DDG news search — returns actual articles, not website homepages."""
-    try:
-        from ddgs import DDGS
-    except ImportError:
-        from duckduckgo_search import DDGS
-
-    results = []
-    try:
-        with DDGS() as ddgs:
-            for r in ddgs.news(query, max_results=max_results):
-                results.append({
-                    "title":   r.get("title",  ""),
-                    "snippet": r.get("body",   ""),
-                    "url":     r.get("url",    ""),
-                    "source":  r.get("source", ""),
-                })
-    except Exception as e:
-        print(f"[WebSearch] ⚠️ DDG news() failed ({e}) — falling back to text search")
-        results = _ddg_search(query, max_results=max_results)
-    return results
-
-
-def _format_ddg(query: str, results: list[dict]) -> str:
-    if not results:
-        return f"No results found for: {query}"
-
-    lines = [f"Search results for: {query}\n"]
-    for i, r in enumerate(results, 1):
-        if r.get("title"):   lines.append(f"{i}. {r['title']}")
-        if r.get("snippet"): lines.append(f"   {r['snippet']}")
-        if r.get("url"):     lines.append(f"   Source: {r['url']}")
-        lines.append("")
-    return "\n".join(lines).strip()
-
-
-def _format_news(query: str, results: list[dict]) -> str:
-    if not results:
-        return f"No news found for: {query}"
-
-    lines = [f"Latest news: {query}\n"]
-    for i, r in enumerate(results, 1):
-        title = r.get("title", "")
-        if not title:
-            continue
-        src = f"  [{r['source']}]" if r.get("source") else ""
-        lines.append(f"{i}. {title}{src}")
-        if r.get("snippet"):
-            lines.append(f"   {r['snippet'][:140]}")
-        if r.get("url"):
-            lines.append(f"   {r['url']}")
-        lines.append("")
-    return "\n".join(lines).strip()
-
-
-# ── Briefing helper ────────────────────────────────────────────────────────────
-
-def _gemini_headlines(n: int = 5) -> tuple[list[str], str]:
-    """
-    Fetches current headlines via Gemini grounded search.
-    Optimised for speed: minimal prompt + strict token cap.
-    Returns (headline_list, raw_text_for_display).
-    """
-    import re
-    from google import genai
-
-    client = genai.Client(api_key=_get_api_key())
-    response = client.models.generate_content(
-        model="gemini-2.5-flash",
-        contents=f"Current world news: {n} headlines. Numbered list, titles only.",
-        config={"tools": [{"google_search": {}}]},
-    )
-
-    raw = ""
-    for part in response.candidates[0].content.parts:
-        if hasattr(part, "text") and part.text:
-            raw += part.text
-
-    headlines = []
-    for line in raw.strip().split("\n"):
-        line = line.strip()
-        if not line:
-            continue
-        # Only accept lines that begin with a number — skips preamble/closing sentences
-        if not re.match(r'^[\d]+[.\)\-]', line):
-            continue
-        clean = re.sub(r'^[\d]+[.\)\-]\s*', '', line)
-        clean = re.sub(r'^\*+\s*',          '', clean).strip()
-        if clean and len(clean) > 10:
-            headlines.append(clean)
-
-    return headlines[:n], raw.strip()
-
-
-# ── Modes ──────────────────────────────────────────────────────────────────────
-
-def _search(query: str) -> str:
-    """Default search — Gemini grounded, DDG fallback."""
-    try:
-        return _gemini_search(query)
-    except Exception as e:
-        print(f"[WebSearch] ⚠️ Gemini failed ({e}) — trying DDG...")
-        results = _ddg_search(query)
-        return _format_ddg(query, results)
-
-
-def _news(query: str) -> str:
-    """
-    Runs Gemini grounded search AND DDG news in parallel.
-    Returns whichever delivers a valid result first; cancels the other.
-    """
-    import threading
-
-    gemini_query = f"latest news today: {query}" if query else "top world news today"
-    ddg_query    = query if query else "world news today"
-
-    result_box  = [None]   # first valid result lands here
-    lock        = threading.Lock()
-    done_evt    = threading.Event()
-    failures    = [0]
-
-    def _store(r: str) -> None:
-        if r and len(r) > 60:
-            with lock:
-                if result_box[0] is None:
-                    result_box[0] = r
-            done_evt.set()
-        else:
-            with lock:
-                failures[0] += 1
-                if failures[0] >= 2:   # both failed — unblock caller
-                    done_evt.set()
-
-    def _try_gemini():
-        try:
-            _store(_gemini_search(gemini_query))
-        except Exception as e:
-            print(f"[WebSearch] ⚠️ Gemini news failed ({e})")
-            _store("")
-
-    def _try_ddg():
-        try:
-            results = _ddg_news(ddg_query, max_results=8)
-            _store(_format_news(ddg_query, results))
-        except Exception as e:
-            print(f"[WebSearch] ⚠️ DDG news failed ({e})")
-            _store("")
-
-    threading.Thread(target=_try_gemini, daemon=True).start()
-    threading.Thread(target=_try_ddg,    daemon=True).start()
-
-    done_evt.wait(timeout=10.0)
-    return result_box[0] or f"No news found for: {query}"
-
-
-def _research(query: str) -> str:
-    """
-    Deep dive — asks Gemini for a comprehensive answer with context.
-    Falls back to a wider DDG fetch.
-    """
-    research_query = (
-        f"Comprehensive, detailed explanation of: {query}. "
-        "Include background context, key facts, current state, and important nuances."
-    )
-    try:
-        return _gemini_search(research_query)
-    except Exception as e:
-        print(f"[WebSearch] ⚠️ Research Gemini failed ({e}) — DDG fallback...")
-        results = _ddg_search(query, max_results=10)
-        return _format_ddg(query, results)
-
-
-def _price(query: str) -> str:
-    """Product price lookup — searches for current market prices."""
-    price_query = f"current price of {query} — how much does it cost today"
-    try:
-        return _gemini_search(price_query)
-    except Exception as e:
-        print(f"[WebSearch] ⚠️ Price Gemini failed ({e}) — DDG fallback...")
-        results = _ddg_search(f"{query} price buy", max_results=6)
-        return _format_ddg(query, results)
-
-
-def _compare(items: list[str], aspect: str) -> str:
-    query = (
-        f"Compare {', '.join(items)} in terms of {aspect}. "
-        "Give specific facts and data."
-    )
-    try:
-        return _gemini_search(query)
-    except Exception as e:
-        print(f"[WebSearch] ⚠️ Gemini compare failed: {e} — falling back to DDG")
-
-    all_results: dict[str, list] = {}
-    for item in items:
-        try:
-            all_results[item] = _ddg_search(f"{item} {aspect}", max_results=3)
-        except Exception:
-            all_results[item] = []
-
-    lines = [f"Comparison — {aspect.upper()}", "─" * 40]
-    for item in items:
-        lines.append(f"\n▸ {item}")
-        for r in all_results.get(item, [])[:2]:
-            if r.get("snippet"):
-                lines.append(f"  • {r['snippet']}")
-            if r.get("url"):
-                lines.append(f"    {r['url']}")
-    return "\n".join(lines)
-
-
-# ── Public entry point ─────────────────────────────────────────────────────────
-
-def web_search(
-    parameters:     dict,
-    response=None,
-    player=None,
-    session_memory=None,
-) -> str:
-    params = parameters or {}
-    query  = params.get("query", "").strip()
-    mode   = params.get("mode",  "search").lower().strip()
-    items  = params.get("items", [])
-    aspect = params.get("aspect", "general").strip() or "general"
-
-    if not query and not items:
-        return "Please provide a search query."
-
-    if items and mode not in ("compare",):
-        mode = "compare"
-
-    if player:
-        player.write_log(f"[Search:{mode}] {query or ', '.join(items)}")
-
-    print(f"[WebSearch] 🔍 mode={mode!r}  query={query!r}")
-
-    try:
-        if mode == "compare" and items:
-            return _compare(items, aspect)
-        if mode == "news":
-            return _news(query)
-        if mode == "research":
-            return _research(query)
-        if mode == "price":
-            return _price(query)
-        return _search(query)
-
-    except Exception as e:
-        print(f"[WebSearch] ❌ All backends failed: {e}")
-        return f"Search failed: {e}"
+#web_search.py
+import json
+import sys
+from pathlib import Path
+
+def _get_base_dir() -> Path:
+    if getattr(sys, "frozen", False):
+        return Path(sys.executable).parent
+    return Path(__file__).resolve().parent.parent
+
+
+BASE_DIR        = _get_base_dir()
+API_CONFIG_PATH = BASE_DIR / "config" / "api_keys.json"
+
+
+def _get_api_key() -> str:
+    with open(API_CONFIG_PATH, "r", encoding="utf-8") as f:
+        return json.load(f)["gemini_api_key"]
+
+
+def _gemini_search(query: str) -> str:
+    from google import genai
+
+    client   = genai.Client(api_key=_get_api_key())
+    response = client.models.generate_content(
+        model="gemini-flash-latest",
+        contents=query,
+        config={"tools": [{"google_search": {}}]},
+    )
+
+    text = ""
+    for part in response.candidates[0].content.parts:
+        if hasattr(part, "text") and part.text:
+            text += part.text
+
+    text = text.strip()
+    if not text:
+        raise ValueError("Gemini returned an empty response.")
+    return text
+
+
+def _ddg_search(query: str, max_results: int = 6) -> list[dict]:
+    try:
+        from ddgs import DDGS
+    except ImportError:
+        from duckduckgo_search import DDGS
+
+    results = []
+    with DDGS() as ddgs:
+        for r in ddgs.text(query, max_results=max_results):
+            results.append({
+                "title":   r.get("title",  ""),
+                "snippet": r.get("body",   ""),
+                "url":     r.get("href",   ""),
+            })
+    return results
+
+
+def _ddg_news(query: str, max_results: int = 8) -> list[dict]:
+    """DDG news search — returns actual articles, not website homepages."""
+    try:
+        from ddgs import DDGS
+    except ImportError:
+        from duckduckgo_search import DDGS
+
+    results = []
+    try:
+        with DDGS() as ddgs:
+            for r in ddgs.news(query, max_results=max_results):
+                results.append({
+                    "title":   r.get("title",  ""),
+                    "snippet": r.get("body",   ""),
+                    "url":     r.get("url",    ""),
+                    "source":  r.get("source", ""),
+                })
+    except Exception as e:
+        print(f"[WebSearch] ⚠️ DDG news() failed ({e}) — falling back to text search")
+        results = _ddg_search(query, max_results=max_results)
+    return results
+
+
+def _format_ddg(query: str, results: list[dict]) -> str:
+    if not results:
+        return f"No results found for: {query}"
+
+    lines = [f"Search results for: {query}\n"]
+    for i, r in enumerate(results, 1):
+        if r.get("title"):   lines.append(f"{i}. {r['title']}")
+        if r.get("snippet"): lines.append(f"   {r['snippet']}")
+        if r.get("url"):     lines.append(f"   Source: {r['url']}")
+        lines.append("")
+    return "\n".join(lines).strip()
+
+
+def _format_news(query: str, results: list[dict]) -> str:
+    if not results:
+        return f"No news found for: {query}"
+
+    lines = [f"Latest news: {query}\n"]
+    for i, r in enumerate(results, 1):
+        title = r.get("title", "")
+        if not title:
+            continue
+        src = f"  [{r['source']}]" if r.get("source") else ""
+        lines.append(f"{i}. {title}{src}")
+        if r.get("snippet"):
+            lines.append(f"   {r['snippet'][:140]}")
+        if r.get("url"):
+            lines.append(f"   {r['url']}")
+        lines.append("")
+    return "\n".join(lines).strip()
+
+
+# ── Briefing helper ────────────────────────────────────────────────────────────
+
+def _gemini_headlines(n: int = 5) -> tuple[list[str], str]:
+    """
+    Fetches current headlines via Gemini grounded search.
+    Optimised for speed: minimal prompt + strict token cap.
+    Returns (headline_list, raw_text_for_display).
+    """
+    import re
+    from google import genai
+
+    client = genai.Client(api_key=_get_api_key())
+    response = client.models.generate_content(
+        model="gemini-flash-latest",
+        contents=f"Current world news: {n} headlines. Numbered list, titles only.",
+        config={"tools": [{"google_search": {}}]},
+    )
+
+    raw = ""
+    for part in response.candidates[0].content.parts:
+        if hasattr(part, "text") and part.text:
+            raw += part.text
+
+    headlines = []
+    for line in raw.strip().split("\n"):
+        line = line.strip()
+        if not line:
+            continue
+        # Only accept lines that begin with a number — skips preamble/closing sentences
+        if not re.match(r'^[\d]+[.\)\-]', line):
+            continue
+        clean = re.sub(r'^[\d]+[.\)\-]\s*', '', line)
+        clean = re.sub(r'^\*+\s*',          '', clean).strip()
+        if clean and len(clean) > 10:
+            headlines.append(clean)
+
+    return headlines[:n], raw.strip()
+
+
+# ── Modes ──────────────────────────────────────────────────────────────────────
+
+def _search(query: str) -> str:
+    """Default search — Gemini grounded, DDG fallback."""
+    try:
+        return _gemini_search(query)
+    except Exception as e:
+        print(f"[WebSearch] ⚠️ Gemini failed ({e}) — trying DDG...")
+        results = _ddg_search(query)
+        return _format_ddg(query, results)
+
+
+def _news(query: str) -> str:
+    """
+    Runs Gemini grounded search AND DDG news in parallel.
+    Returns whichever delivers a valid result first; cancels the other.
+    """
+    import threading
+
+    gemini_query = f"latest news today: {query}" if query else "top world news today"
+    ddg_query    = query if query else "world news today"
+
+    result_box  = [None]   # first valid result lands here
+    lock        = threading.Lock()
+    done_evt    = threading.Event()
+    failures    = [0]
+
+    def _store(r: str) -> None:
+        if r and len(r) > 60:
+            with lock:
+                if result_box[0] is None:
+                    result_box[0] = r
+            done_evt.set()
+        else:
+            with lock:
+                failures[0] += 1
+                if failures[0] >= 2:   # both failed — unblock caller
+                    done_evt.set()
+
+    def _try_gemini():
+        try:
+            _store(_gemini_search(gemini_query))
+        except Exception as e:
+            print(f"[WebSearch] ⚠️ Gemini news failed ({e})")
+            _store("")
+
+    def _try_ddg():
+        try:
+            results = _ddg_news(ddg_query, max_results=8)
+            _store(_format_news(ddg_query, results))
+        except Exception as e:
+            print(f"[WebSearch] ⚠️ DDG news failed ({e})")
+            _store("")
+
+    threading.Thread(target=_try_gemini, daemon=True).start()
+    threading.Thread(target=_try_ddg,    daemon=True).start()
+
+    done_evt.wait(timeout=10.0)
+    return result_box[0] or f"No news found for: {query}"
+
+
+def _research(query: str) -> str:
+    """
+    Deep dive — asks Gemini for a comprehensive answer with context.
+    Falls back to a wider DDG fetch.
+    """
+    research_query = (
+        f"Comprehensive, detailed explanation of: {query}. "
+        "Include background context, key facts, current state, and important nuances."
+    )
+    try:
+        return _gemini_search(research_query)
+    except Exception as e:
+        print(f"[WebSearch] ⚠️ Research Gemini failed ({e}) — DDG fallback...")
+        results = _ddg_search(query, max_results=10)
+        return _format_ddg(query, results)
+
+
+def _price(query: str) -> str:
+    """Product price lookup — searches for current market prices."""
+    price_query = f"current price of {query} — how much does it cost today"
+    try:
+        return _gemini_search(price_query)
+    except Exception as e:
+        print(f"[WebSearch] ⚠️ Price Gemini failed ({e}) — DDG fallback...")
+        results = _ddg_search(f"{query} price buy", max_results=6)
+        return _format_ddg(query, results)
+
+
+def _compare(items: list[str], aspect: str) -> str:
+    query = (
+        f"Compare {', '.join(items)} in terms of {aspect}. "
+        "Give specific facts and data."
+    )
+    try:
+        return _gemini_search(query)
+    except Exception as e:
+        print(f"[WebSearch] ⚠️ Gemini compare failed: {e} — falling back to DDG")
+
+    all_results: dict[str, list] = {}
+    for item in items:
+        try:
+            all_results[item] = _ddg_search(f"{item} {aspect}", max_results=3)
+        except Exception:
+            all_results[item] = []
+
+    lines = [f"Comparison — {aspect.upper()}", "─" * 40]
+    for item in items:
+        lines.append(f"\n▸ {item}")
+        for r in all_results.get(item, [])[:2]:
+            if r.get("snippet"):
+                lines.append(f"  • {r['snippet']}")
+            if r.get("url"):
+                lines.append(f"    {r['url']}")
+    return "\n".join(lines)
+
+
+# ── Public entry point ─────────────────────────────────────────────────────────
+
+def web_search(
+    parameters:     dict,
+    response=None,
+    player=None,
+    session_memory=None,
+) -> str:
+    params = parameters or {}
+    query  = params.get("query", "").strip()
+    mode   = params.get("mode",  "search").lower().strip()
+    items  = params.get("items", [])
+    aspect = params.get("aspect", "general").strip() or "general"
+
+    if not query and not items:
+        return "Please provide a search query."
+
+    if items and mode not in ("compare",):
+        mode = "compare"
+
+    if player:
+        player.write_log(f"[Search:{mode}] {query or ', '.join(items)}")
+
+    print(f"[WebSearch] 🔍 mode={mode!r}  query={query!r}")
+
+    try:
+        if mode == "compare" and items:
+            return _compare(items, aspect)
+        if mode == "news":
+            return _news(query)
+        if mode == "research":
+            return _research(query)
+        if mode == "price":
+            return _price(query)
+        return _search(query)
+
+    except Exception as e:
+        print(f"[WebSearch] ❌ All backends failed: {e}")
+        return f"Search failed: {e}"

actions/youtube_video.py

@@ -174,7 +174,7 @@ def _summarize_with_gemini(transcript: str, video_url: str) -> str:
     max_chars = 80000
     truncated = transcript[:max_chars] + ("..." if len(transcript) > max_chars else "")
     response  = _client.models.generate_content(
-        model="gemini-2.5-flash",
+        model="gemini-flash-latest",
         contents=f"Please summarize this YouTube video transcript:\n\n{truncated}",
         config=types.GenerateContentConfig(
             system_instruction=(

core/__init__.py

@@ -1 +0,0 @@
-# core package

core/plugin_loader.py

@@ -0,0 +1,188 @@
+"""
+Plugin discovery, validation, collision detection, and dispatch.
+
+Discovery runs once (JarvisLive.__init__ calls discover_plugins()); the resulting
+PluginRegistry is cached for the process lifetime. Enable/disable state is re-read
+from config on every call to get_tool_declarations() / run() / list_for_ui(), so
+toggling a plugin does not require restarting the app or re-importing anything.
+"""
+from __future__ import annotations
+
+import importlib.util
+import inspect
+import re
+import sys
+import traceback
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Callable, Optional
+
+from memory.config_manager import get_plugin_enabled
+
+_NAME_RE = re.compile(r"^[a-zA-Z_][a-zA-Z0-9_]{0,63}$")
+_DEFAULT_PARAMS = {"type": "OBJECT", "properties": {}}
+
+
+@dataclass
+class PluginRecord:
+    name: str
+    description: str = ""
+    parameters: dict = field(default_factory=lambda: dict(_DEFAULT_PARAMS))
+    run: Optional[Callable] = None
+    file: str = ""
+    valid: bool = False
+    error: str = ""
+
+
+class PluginRegistry:
+    def __init__(self, plugins: dict[str, PluginRecord], logger: Callable[[str], None]):
+        self._plugins = plugins          # name -> PluginRecord, VALID entries only
+        self._all_records: list[PluginRecord] = []   # valid + invalid, for UI listing
+        self._logger = logger
+
+    # -- called by main.py at LiveConnectConfig build time --
+    def get_tool_declarations(self) -> list[dict]:
+        decls = []
+        for name, rec in self._plugins.items():
+            if get_plugin_enabled(name):
+                decls.append({
+                    "name": rec.name,
+                    "description": rec.description,
+                    "parameters": rec.parameters,
+                })
+        return decls
+
+    def has(self, name: str) -> bool:
+        return name in self._plugins
+
+    # -- called by main.py from _execute_tool's else branch --
+    def run(self, name: str, parameters: dict, player=None, session_memory=None) -> str:
+        rec = self._plugins.get(name)
+        if rec is None or not rec.valid:
+            return f"Plugin '{name}' is not available."
+        if not get_plugin_enabled(name):
+            return f"The '{name}' plugin is currently disabled."
+        try:
+            return _call_run(rec.run, parameters, player, session_memory) or "Done."
+        except Exception as e:
+            self._logger(f"Plugin '{name}' crashed during run(): {e}")
+            traceback.print_exc()
+            return f"Sir, the '{name}' plugin failed: {e}"
+
+    # -- called by ui.py's Plugin Manager overlay --
+    def list_for_ui(self) -> list[dict]:
+        out = []
+        for rec in self._all_records:
+            out.append({
+                "name": rec.name,
+                "description": rec.description,
+                "file": rec.file,
+                "valid": rec.valid,
+                "error": rec.error,
+                "enabled": get_plugin_enabled(rec.name) if rec.valid else False,
+            })
+        return out
+
+
+def _call_run(run_fn, parameters, player, session_memory):
+    """Invoke run() passing only the kwargs it actually declares (or all of them
+    if it has **kwargs), so a minimal `def run(parameters):` plugin still works."""
+    sig = inspect.signature(run_fn)
+    has_var_kw = any(p.kind == inspect.Parameter.VAR_KEYWORD for p in sig.parameters.values())
+    kwargs = {}
+    if has_var_kw or "player" in sig.parameters:
+        kwargs["player"] = player
+    if has_var_kw or "session_memory" in sig.parameters:
+        kwargs["session_memory"] = session_memory
+    return run_fn(parameters, **kwargs)
+
+
+def _validate(module, filename: str) -> PluginRecord:
+    """Returns a PluginRecord; .valid=False + .error set on any problem. Never raises."""
+    plugin_meta = getattr(module, "PLUGIN", None)
+    if not isinstance(plugin_meta, dict):
+        return PluginRecord(name=Path(filename).stem, file=filename,
+                             error="Missing PLUGIN dict constant.")
+
+    name = plugin_meta.get("name")
+    if not isinstance(name, str) or not _NAME_RE.match(name):
+        return PluginRecord(name=str(name or Path(filename).stem), file=filename,
+                             error="PLUGIN['name'] missing or not a valid identifier "
+                                   "(letters/digits/underscore, must start with letter/underscore).")
+
+    description = plugin_meta.get("description")
+    if not isinstance(description, str) or not description.strip():
+        return PluginRecord(name=name, file=filename,
+                             error="PLUGIN['description'] missing or empty.")
+
+    parameters = plugin_meta.get("parameters", _DEFAULT_PARAMS)
+    if not isinstance(parameters, dict) or parameters.get("type") != "OBJECT":
+        return PluginRecord(name=name, file=filename,
+                             error="PLUGIN['parameters'] must be a dict with \"type\": \"OBJECT\".")
+
+    run_fn = getattr(module, "run", None)
+    if not callable(run_fn):
+        return PluginRecord(name=name, file=filename,
+                             error="Missing callable run(parameters, ...) function.")
+
+    return PluginRecord(name=name, description=description.strip(), parameters=parameters,
+                         run=run_fn, file=filename, valid=True, error="")
+
+
+def discover_plugins(plugins_dir: Path, core_tool_names: set[str],
+                      logger: Callable[[str], None] = print) -> PluginRegistry:
+    """
+    Scans plugins_dir for *.py files (skips files starting with '_', e.g. __init__.py,
+    _template.py, and any shared-helper modules an author prefixes with '_').
+    Import errors, validation errors, and name collisions are logged and the offending
+    file is skipped — they NEVER raise out of this function and never abort the scan
+    of remaining files.
+    """
+    plugins_dir.mkdir(parents=True, exist_ok=True)
+    valid: dict[str, PluginRecord] = {}
+    all_records: list[PluginRecord] = []
+
+    files = sorted(plugins_dir.glob("*.py"), key=lambda p: p.name)  # deterministic order
+    for path in files:
+        if path.name.startswith("_"):
+            continue
+        try:
+            module_name = f"plugins.{path.stem}"
+            spec = importlib.util.spec_from_file_location(module_name, path)
+            if spec is None or spec.loader is None:
+                raise ImportError("could not build import spec")
+            module = importlib.util.module_from_spec(spec)
+            sys.modules[module_name] = module
+            try:
+                spec.loader.exec_module(module)
+            except Exception:
+                sys.modules.pop(module_name, None)
+                raise
+
+            rec = _validate(module, path.name)
+
+            if rec.valid and rec.name in core_tool_names:
+                rec = PluginRecord(name=rec.name, file=path.name,
+                                    error=f"Name '{rec.name}' collides with a core tool — rejected.")
+            elif rec.valid and rec.name in valid:
+                other = valid[rec.name].file
+                rec = PluginRecord(name=rec.name, file=path.name,
+                                    error=f"Name '{rec.name}' already used by plugin '{other}' — rejected.")
+
+        except Exception as e:
+            rec = PluginRecord(name=path.stem, file=path.name,
+                                error=f"Failed to load: {e}")
+            traceback.print_exc()
+
+        all_records.append(rec)
+        if rec.valid:
+            valid[rec.name] = rec
+            logger(f"Plugin loaded: {rec.name} ({path.name})")
+        else:
+            logger(f"Plugin rejected: {path.name} — {rec.error}")
+
+    registry = PluginRegistry(valid, logger)
+    registry._all_records = all_records
+    logger(f"Plugin discovery complete: {len(valid)} active, "
+           f"{len(all_records) - len(valid)} rejected, {len(all_records)} total.")
+    return registry

main.py

@@ -10,9 +10,10 @@ if _platform.system() == "Windows":
         def __init__(self, args, **kw):
             kw["creationflags"] = kw.get("creationflags", 0) | _subprocess.CREATE_NO_WINDOW
             kw.pop("startupinfo", None)   # drop any stale/shared STARTUPINFO
-            super().__init__(args, **kw)
+            super().__init__(args, **                       kw)
 
     _subprocess.Popen = _Popen
+
 # ─────────────────────────────────────────────────────────────────────────────
 
 import asyncio
@@ -58,14 +59,13 @@ from actions.background_monitor import (
 )
 from actions.web_search        import _news as _fetch_news_sync
 from memory.config_manager     import get_brief_enabled
-
+from core.plugin_loader        import discover_plugins
 
 def get_base_dir():
     if getattr(sys, "frozen", False):
         return Path(sys.executable).parent
     return Path(__file__).resolve().parent
 
-
 BASE_DIR        = get_base_dir()
 API_CONFIG_PATH = BASE_DIR / "config" / "api_keys.json"
 PROMPT_PATH     = BASE_DIR / "core" / "prompt.txt"
@@ -547,9 +547,6 @@ TOOL_DECLARATIONS = [
     },
 ]
 
-# --- Plugin system ---
-
-
 class JarvisLive:
 
     def __init__(self, ui: JarvisUI):
@@ -579,6 +576,43 @@ class JarvisLive:
         self._last_user_speech = time.monotonic()  # updated on every user utterance
         self._session_log: list[str] = []          # conversation turns for end-of-session summary
 
+        self._enhanced_live = True  # affective dialog + proactive audio; auto-disabled if the server rejects them
+        _core_names = {t["name"] for t in TOOL_DECLARATIONS}
+        self._plugin_registry = discover_plugins(
+            plugins_dir=Path(__file__).resolve().parent / "plugins",
+            core_tool_names=_core_names,
+            logger=lambda msg: (print(f"[Plugins] {msg}"), self.ui.write_log(f"SYS: {msg}")),
+        )
+        self.ui.get_plugins = self._plugin_registry.list_for_ui
+        self.ui.request_say = self.plugin_say   # plugins: mid-task speech channel
+
+    def plugin_say(self, instruction: str) -> None:
+        """
+        Thread-safe speech channel for plugins: lets a plugin ask JARVIS to
+        say something short WHILE its run() is still executing (plugins block
+        their executor thread, so they can't speak through the tool response
+        until they finish). The instruction is injected into the Live session
+        exactly like a proactive check-in; Gemini phrases it naturally in the
+        user's language. Silently a no-op when no session is connected.
+        """
+        loop = getattr(self, "_loop", None)
+        if not loop or not self.session:
+            return
+
+        async def _say():
+            try:
+                await self.session.send_client_content(
+                    turns={"parts": [{"text": instruction}]},
+                    turn_complete=True,
+                )
+            except Exception as e:
+                print(f"[PluginSay] {e}")
+
+        try:
+            asyncio.run_coroutine_threadsafe(_say(), loop)
+        except Exception as e:
+            print(f"[PluginSay] {e}")
+
     def _make_remote_key(self):
         """Called from Qt main thread when user presses Remote Control."""
         if self._dashboard is None:
@@ -687,13 +721,18 @@ class JarvisLive:
             parts.append(mem_str)
         parts.append(sys_prompt)
 
-        return types.LiveConnectConfig(
+        cfg = dict(
             response_modalities=["AUDIO"],
             output_audio_transcription={},
             input_audio_transcription={},
             system_instruction="\n".join(parts),
-            tools=[{"function_declarations": TOOL_DECLARATIONS}],
+            tools=[{"function_declarations": TOOL_DECLARATIONS + self._plugin_registry.get_tool_declarations()}],
             session_resumption=types.SessionResumptionConfig(),
+            # Sliding-window compression: session never dies from a full context
+            # window — JARVIS can stay in one conversation for hours
+            context_window_compression=types.ContextWindowCompressionConfig(
+                sliding_window=types.SlidingWindow(),
+            ),
             speech_config=types.SpeechConfig(
                 voice_config=types.VoiceConfig(
                     prebuilt_voice_config=types.PrebuiltVoiceConfig(
@@ -702,6 +741,13 @@ class JarvisLive:
                 )
             ),
         )
+        if self._enhanced_live:
+            # Affective dialog: JARVIS hears tone/emotion and adapts its voice.
+            # Proactive audio: JARVIS stays silent when speech isn't addressed
+            # to it (background chatter, talking to someone else in the room).
+            cfg["enable_affective_dialog"] = True
+            cfg["proactivity"] = types.ProactivityConfig(proactive_audio=True)
+        return types.LiveConnectConfig(**cfg)
 
     async def _execute_tool(self, fc) -> types.FunctionResponse:
         name = fc.name
@@ -872,7 +918,14 @@ class JarvisLive:
                 asyncio.create_task(_do_shutdown())
 
             else:
-                result = f"Unknown tool: {name}"
+                if self._plugin_registry.has(name):
+                    r = await loop.run_in_executor(
+                        None,
+                        lambda: self._plugin_registry.run(name, args, player=self.ui, session_memory=None)
+                    )
+                    result = r or "Done."
+                else:
+                    result = f"Unknown tool: {name}"
 
         except Exception as e:
             result = f"Tool '{name}' failed: {e}"
@@ -1236,7 +1289,7 @@ class JarvisLive:
             client = _genai.Client(api_key=_get_api_key())
             resp   = await asyncio.to_thread(
                 client.models.generate_content,
-                model="gemini-2.5-flash",
+                model="gemini-flash-latest",
                 contents=prompt,
             )
             summary = (resp.text or "").strip()
@@ -1419,9 +1472,11 @@ class JarvisLive:
                 config = self._build_config()
 
                 # Fresh client on every reconnect — avoids stale HTTP session state
+                # v1alpha carries the enhanced audio features (affective dialog,
+                # proactive audio); if they get rejected we fall back to v1beta.
                 client = genai.Client(
                     api_key=_get_api_key(),
-                    http_options={"api_version": "v1beta"}
+                    http_options={"api_version": "v1alpha" if self._enhanced_live else "v1beta"}
                 )
 
                 async with (
@@ -1477,6 +1532,21 @@ class JarvisLive:
                 print(f"[JARVIS] Error ({type(e).__name__}): {e}")
                 traceback.print_exc()
 
+                # Enhanced audio features rejected by the server (preview API
+                # drift) — drop them and reconnect with the plain config.
+                if self._enhanced_live and (
+                    "INVALID_ARGUMENT" in err_str
+                    or "affective" in err_str.lower()
+                    or "proactiv" in err_str.lower()
+                    or "Unknown name" in err_str
+                    or "unexpected keyword" in err_str
+                ):
+                    self._enhanced_live = False
+                    self.ui.write_log(
+                        "SYS: Advanced audio features unavailable — reconnecting without them."
+                    )
+                    continue
+
                 # Invalid API key — stop hammering the API, prompt re-configuration
                 if "API key not valid" in err_str or "1007" in err_str:
                     self.ui.write_log("ERR: API key invalid — please re-enter your key.")

memory/config_manager.py

@@ -88,4 +88,25 @@ def save_brief_enabled(enabled: bool) -> None:
         except Exception:
             data = {}
     data["morning_brief_enabled"] = enabled
+    CONFIG_FILE.write_text(json.dumps(data, indent=4), encoding="utf-8")
+
+
+def get_plugin_enabled(plugin_name: str) -> bool:
+    """Plugins are enabled by default the moment they're discovered (opt-out model)."""
+    return load_api_keys().get("plugins_enabled", {}).get(plugin_name, True)
+
+
+def save_plugin_enabled(plugin_name: str, enabled: bool) -> None:
+    ensure_config_dir()
+    data: dict = {}
+    if CONFIG_FILE.exists():
+        try:
+            data = json.loads(CONFIG_FILE.read_text(encoding="utf-8"))
+        except Exception:
+            data = {}
+    plugins_cfg = data.get("plugins_enabled")
+    if not isinstance(plugins_cfg, dict):
+        plugins_cfg = {}
+    plugins_cfg[plugin_name] = enabled
+    data["plugins_enabled"] = plugins_cfg
     CONFIG_FILE.write_text(json.dumps(data, indent=4), encoding="utf-8")

plugins/_template.py

@@ -0,0 +1,45 @@
+"""
+Drop-in JARVIS plugin template.
+
+Copy this file, rename it (no leading underscore), fill in PLUGIN and run().
+No other file needs to change — JARVIS discovers this automatically at startup.
+"""
+
+PLUGIN = {
+    "name": "my_plugin",                     # snake_case, unique, ^[a-zA-Z_][a-zA-Z0-9_]{0,63}$
+    "description": (
+        "One or two sentences Gemini uses to decide when to call this tool. "
+        "Be explicit about trigger phrases and, if it could be confused with "
+        "another tool, say which tool NOT to use instead (see game_updater's "
+        "description in main.py for the pattern)."
+    ),
+    "parameters": {
+        "type": "OBJECT",
+        "properties": {
+            "example_arg": {"type": "STRING", "description": "What this argument means"},
+        },
+        "required": [],   # omit or leave empty for a zero-argument tool
+    },
+}
+
+def run(parameters: dict, player=None, session_memory=None) -> str:
+    """
+    parameters: dict of the args Gemini extracted, matching PLUGIN['parameters'].
+    player: the JarvisUI instance — use player.write_log(f"JARVIS: ...") to log,
+            same as actions/*.py. May be None.
+    session_memory: reserved, usually None today (core tools mostly pass None too).
+    Return a short natural-language string — this is spoken back to the user.
+    Never raise: catch your own errors and return a spoken error string instead
+    (the loader also catches exceptions as a second safety net, but don't rely on it).
+    """
+    example_arg = parameters.get("example_arg", "")
+    try:
+        result_text = f"Did the thing with {example_arg}."
+    except Exception as e:
+        return f"Sir, my_plugin failed: {e}"
+    if player:
+        try:
+            player.write_log(f"JARVIS: {result_text}")
+        except Exception:
+            pass
+    return result_text

readme.md

@@ -1,4 +1,4 @@
-# ⚙️ MARK L (50)
+# ⚙️ MARK LI (51)
 ### The Ultimate Cross-Platform Personal AI Assistant — By FatihMakes
 
 > 📺 **[Watch the full setup video on YouTube](https://www.youtube.com/@FatihMakes)**
@@ -9,7 +9,7 @@ A real-time voice AI that can hear, see, understand, and control your computer
 
 ## ✨ Overview
 
-MARK L is where the assistant stops being a tool and starts being a presence. It remembers yesterday's conversation, watches the topics you care about, and speaks first when it has something worth saying. The goal of this build was continuity — JARVIS should feel like it never fully left, even after you close it.
+MARK LI is the final form of the core: an assistant you extend without ever touching its engine. Drop a single plugin file into the `plugins/` folder and JARVIS learns a new skill on the next launch — no code changes, no configuration, no risk. On top of that, the voice itself got smarter: JARVIS now hears the emotion in your voice, knows when you're talking to someone else in the room and stays silent, and can hold one conversation for hours without losing the thread.
 
 It's not just an assistant — it's an extension of your digital life.
 
@@ -20,7 +20,11 @@ It's not just an assistant — it's an extension of your digital life.
 ### Core Features
 | Feature | Description |
 |---|---|
+| 🧩 Plugin System | Drop a single `.py` file into `plugins/` — JARVIS learns a new skill on next launch |
 | 🎙️ Real-time Voice | Ultra-low latency conversation in any language via Gemini Live API |
+| 💓 Affective Dialog | Hears the emotion in your voice and adapts its tone in response |
+| 🤫 Proactive Audio | Knows when you're not talking to it — background chatter never triggers a reply |
+| ♾️ Unlimited Sessions | Sliding-window context compression — one conversation can last for hours |
 | 🖥️ System Control | Launch apps, adjust volume/brightness, WiFi, shortcuts, power — all by voice |
 | 🧩 Autonomous Tasks | High-level planning for complex multi-step goals via agent mode |
 | 👁️ Visual Awareness | Real-time screen capture and webcam vision piped into your main Gemini session |
@@ -51,34 +55,29 @@ It's not just an assistant — it's an extension of your digital life.
 
 ---
 
-## 🆕 What's New in Mark L
+## 🆕 What's New in Mark LI
 
-### 🗓️ Session Memory — JARVIS Remembers Yesterday
-At the end of every session, JARVIS generates a 1-2 sentence summary of what was discussed and saves it to memory. The next morning, it's mentioned naturally in the briefing:
-> *"Good morning, sir — it's 09:15. Yesterday you were working on the Mark L background monitoring feature. Fetching today's headlines now."*
+### 🧩 Plugin System — Extend JARVIS Without Touching a Single Core File
+The headline feature of Mark LI, and the reason it's the final architecture version. Every new capability from now on ships as a single `.py` file:
 
-The summary is consumed immediately after use — it never repeats in future briefings and adds zero long-term bloat to memory.
+1. Download a plugin file (e.g. `calorie_counter.py`)
+2. Drop it into the `plugins/` folder
+3. Restart JARVIS — done. The skill is live, by voice, in any language.
 
-### 👁️‍🗨️ Background Monitoring — JARVIS Watches While You're Away
-Tell JARVIS to monitor any topic and it checks for new developments once a day using DuckDuckGo news. When a headline changes, it reports back naturally in your language:
-> *"Efendim, takip ettiğiniz yapay zeka haberlerinde bir gelişme var: Google yeni bir model duyurdu."*
+Each plugin declares its own Gemini tool schema and logic in one file. The engine auto-discovers it at startup, registers it with the Live session, and lists it in the new **🧩 Plugin Manager** panel where every plugin gets its own persistent ON/OFF toggle.
 
-Fully opt-in — JARVIS monitors nothing without being explicitly asked. Crypto, financial, and trading topics are blocked at the code level regardless of what is requested. Same headline never triggers twice.
+Safety is built in at three layers: a broken or badly written plugin can **never** crash JARVIS — it simply shows up as "BROKEN" in the manager with the error explained, while every other tool and plugin keeps working. Name collisions with core tools are detected and rejected automatically. Want to write your own? Copy `plugins/_template.py` and fill in two things: the `PLUGIN` dict and the `run()` function.
 
-### 🔔 Proactive System 2.0 — Context-Aware, Time-Aware, Non-Repetitive
-The proactive engine was rebuilt from the ground up. Instead of a generic check-in after 15 minutes of silence, JARVIS now:
-- Knows the **time of day** — morning tone differs from evening tone
-- Knows your **active projects** from memory and can ask how something is going
-- Knows your **monitored topics** and can bring one up naturally
-- Knows **what you were just talking about** (last 8 conversation turns)
-- **Rotates** between three focus areas so it never opens with the same line twice
-- Has a 20-minute cooldown (up from 10) — less intrusive, more meaningful
+### 💓 Affective Dialog — JARVIS Hears How You Feel
+Powered by Gemini Live's native audio understanding, JARVIS now picks up the emotion in your voice — excitement, frustration, fatigue — and adapts its own tone in response. Late-night tired questions get calm answers; excited announcements get energy back.
 
-### 👁️ Instant Vision Acknowledgment — No More Silent Waiting
-When you ask JARVIS to look at your screen or camera, it no longer goes silent while processing. It immediately says something natural ("Looking at your screen now, sir" / "Ekrana bakıyorum efendim") while the capture runs. The actual analysis follows as the next response.
+### 🤫 Proactive Audio — Knows When You're Not Talking to It
+The biggest quality-of-life upgrade for an always-listening assistant: JARVIS can now tell when speech isn't addressed to it. Talking to someone in the room, taking a phone call, TV in the background — it stays silent instead of interjecting. No wake word needed, no accidental replies.
 
-### 📰 Parallel News Search — First Result Wins
-News queries now run Gemini Grounded Search and DuckDuckGo news simultaneously in two threads. Whichever delivers a valid result first is used; the other is silently discarded. A Gemini 503 error no longer delays results — the DDG fallback is already running in parallel.
+### ♾️ Unlimited Session Length — The Conversation Never Dies
+Sliding-window context compression means the Live session no longer terminates when the context window fills up. Combined with session resumption, JARVIS holds one continuous conversation for hours without losing the thread.
+
+All three Live API upgrades degrade gracefully: if the preview API ever rejects them, JARVIS automatically reconnects with the standard configuration — users never see a crash.
 
 ---
 
@@ -89,15 +88,16 @@ News queries now run Gemini Grounded Search and DuckDuckGo news simultaneously i
 | **XLVIII** | Instant interrupt · parallel news · two-phase briefing · exponential backoff · vision cooldown |
 | **XLIX** | Auto-start · clipboard intelligence · assistant customization |
 | **L** | Session memory · background monitoring · proactive 2.0 · instant vision · parallel news search |
-| **LI+** | Plugin system · email · quiz mode · calorie counter · calendar |
+| **LI** | Plugin system · affective dialog · proactive audio · unlimited sessions |
+| **LII+** | Plugin files: email · quiz mode · calorie counter · calendar · and more |
 
 ---
 
 ## ⚡ Quick Start
 
 ```bash
-git clone https://github.com/FatihMakes/Mark-L.git
-cd Mark-L
+git clone https://github.com/FatihMakes/Mark-LI.git
+cd Mark-LI
 pip install -r requirements.txt
 python main.py
 ```
@@ -120,10 +120,12 @@ python main.py
 ## 🗂️ Project Structure
 
 ```
-Mark L/
+Mark LI/
 ├── main.py                   # Core loop — Gemini Live session, audio I/O, tool dispatch
-├── ui.py                     # PyQt6 HUD — waveform, log panel, interrupt button, camera feed
+├── ui.py                     # PyQt6 HUD — waveform, log panel, plugin manager, camera feed
 ├── setup.py                  # First-run configuration wizard
+├── plugins/
+│   └── _template.py          # Copy this to write a new plugin — one file, drop in, done
 ├── actions/
 │   ├── web_search.py         # Gemini + DDG parallel search (news, research, price, compare)
 │   ├── screen_processor.py   # Screen capture & webcam vision via Gemini Live
@@ -149,7 +151,8 @@ Mark L/
 │   ├── memory_manager.py     # Load/save long_term.json — sessions, monitors, identity
 │   └── long_term.json        # Persistent store: identity, preferences, projects, sessions, monitors
 ├── core/
-│   └── prompt.txt            # Assistant personality and tool-routing rules
+│   ├── prompt.txt            # Assistant personality and tool-routing rules
+│   └── plugin_loader.py      # Plugin engine — discovery, validation, crash isolation
 └── config/
     └── api_keys.json         # API key, OS setting, assistant name, user name
 ```

requirements.txt

@@ -1,6 +1,6 @@
 PyQt6
 sounddevice
-google-genai
+google-genai>=2.8.0
 google-generativeai
 pillow
 requests

setup.py

@@ -22,5 +22,5 @@ if platform.system() == "Windows":
             f'    "{sys.executable}" "{postinstall}" -install\n'
         )
 
-print("\n✅ Setup complete! Run 'python main.py' to start MARK L.")
+print("\n✅ Setup complete! Run 'python main.py' to start MARK LI.")
 

ui.py

@@ -1412,6 +1412,117 @@ class CustomizeOverlay(QWidget):
         self.hide()
 
 
+class PluginManagerOverlay(QWidget):
+    """Floating overlay — lists discovered plugins with per-plugin ON/OFF toggles."""
+
+    _OW = 420
+
+    def __init__(self, plugins: list[dict], parent=None):
+        super().__init__(parent)
+        self.setAttribute(Qt.WidgetAttribute.WA_StyledBackground, True)
+        self.setStyleSheet(f"""
+            PluginManagerOverlay {{
+                background: rgba(0, 6, 10, 245);
+                border: 1px solid {C.BORDER_B};
+                border-radius: 6px;
+            }}
+        """)
+        self.setFixedWidth(self._OW)
+
+        lay = QVBoxLayout(self)
+        lay.setContentsMargins(20, 16, 20, 16)
+        lay.setSpacing(6)
+
+        hdr = QLabel("🧩  PLUGIN MANAGER")
+        hdr.setFont(QFont("Courier New", 12, QFont.Weight.Bold))
+        hdr.setStyleSheet(f"color: {C.PRI}; background: transparent;")
+        lay.addWidget(hdr)
+        sep = QFrame(); sep.setFrameShape(QFrame.Shape.HLine)
+        sep.setStyleSheet(f"color: {C.BORDER}; margin: 2px 0;")
+        lay.addWidget(sep)
+
+        if not plugins:
+            empty = QLabel("No plugins found in /plugins.")
+            empty.setFont(QFont("Courier New", 8))
+            empty.setStyleSheet(f"color: {C.TEXT_DIM}; background: transparent;")
+            lay.addWidget(empty)
+
+        for p in plugins:
+            lay.addLayout(self._build_row(p))
+
+        lay.addSpacing(4)
+        close_btn = QPushButton("CLOSE")
+        close_btn.setFixedHeight(30)
+        close_btn.setFont(QFont("Courier New", 9))
+        close_btn.setCursor(Qt.CursorShape.PointingHandCursor)
+        close_btn.setStyleSheet(f"""
+            QPushButton {{
+                background: transparent; color: {C.TEXT_MED};
+                border: 1px solid {C.BORDER}; border-radius: 3px;
+            }}
+            QPushButton:hover {{ color: {C.TEXT}; border-color: {C.BORDER_B}; }}
+        """)
+        close_btn.clicked.connect(self.hide)
+        lay.addWidget(close_btn)
+        self.adjustSize()
+
+    def _build_row(self, p: dict) -> QHBoxLayout:
+        row = QHBoxLayout(); row.setSpacing(6)
+
+        label_text = p["name"] if p["valid"] else f"{p['name']}  (⚠ {p['file']})"
+        lbl = QLabel(label_text)
+        lbl.setFont(QFont("Courier New", 8))
+        lbl.setStyleSheet(f"color: {C.TEXT if p['valid'] else C.TEXT_DIM}; background: transparent;")
+        lbl.setToolTip(p["description"] if p["valid"] else p["error"])
+        lbl.setWordWrap(False)
+        row.addWidget(lbl, stretch=1)
+
+        btn = QPushButton()
+        btn.setFixedSize(72, 24)
+        btn.setFont(QFont("Courier New", 7, QFont.Weight.Bold))
+        if not p["valid"]:
+            btn.setText("BROKEN")
+            btn.setEnabled(False)
+            btn.setStyleSheet(f"""
+                QPushButton {{
+                    background: transparent; color: {C.TEXT_DIM};
+                    border: 1px solid {C.BORDER}; border-radius: 3px;
+                }}
+            """)
+        else:
+            btn.setCursor(Qt.CursorShape.PointingHandCursor)
+            self._style_toggle(btn, p["enabled"])
+            btn.clicked.connect(lambda _, name=p["name"], b=btn: self._toggle(name, b))
+        row.addWidget(btn)
+        return row
+
+    def _style_toggle(self, btn: QPushButton, enabled: bool):
+        if enabled:
+            btn.setText("ON")
+            btn.setStyleSheet(f"""
+                QPushButton {{
+                    background: #001a08; color: {C.GREEN};
+                    border: 1px solid {C.GREEN_D}; border-radius: 3px;
+                }}
+                QPushButton:hover {{ background: #002010; }}
+            """)
+        else:
+            btn.setText("OFF")
+            btn.setStyleSheet(f"""
+                QPushButton {{
+                    background: transparent; color: {C.TEXT_DIM};
+                    border: 1px solid {C.BORDER}; border-radius: 3px;
+                }}
+                QPushButton:hover {{ color: {C.TEXT}; border-color: {C.BORDER_B}; }}
+            """)
+
+    def _toggle(self, name: str, btn: QPushButton):
+        from memory.config_manager import get_plugin_enabled, save_plugin_enabled
+        new_val = not get_plugin_enabled(name)
+        save_plugin_enabled(name, new_val)
+        self._style_toggle(btn, new_val)
+
+
 class ClipboardPanel(QWidget):
     """Floating panel shown when text is copied — offers quick Jarvis actions."""
 
@@ -1750,7 +1861,7 @@ class MainWindow(QMainWindow):
         if _ui_color and _ui_color.lower() != DEFAULT_UI_COLOR:
             apply_ui_accent(_ui_color)
 
-        self.setWindowTitle(f"{_display} — MARK XLIX")
+        self.setWindowTitle(f"{_display} — MARK LI")
         self.setMinimumSize(_MIN_W, _MIN_H)
         self.resize(_DEFAULT_W, _DEFAULT_H)
 
@@ -1763,6 +1874,7 @@ class MainWindow(QMainWindow):
         self.on_text_command   = None
         self.on_remote_clicked = None   # callable: () -> (url, key) | None
         self.on_interrupt      = None   # callable: () -> None — stop JARVIS mid-speech
+        self.get_plugins       = None   # callable: () -> list[dict], set by JarvisLive
         self._muted            = False
         self._current_file: str | None = None
         self._remote_overlay: RemoteKeyOverlay | None = None
@@ -2427,7 +2539,7 @@ class MainWindow(QMainWindow):
             l.setStyleSheet(f"color: {color}; background: transparent;")
             return l
 
-        lay.addWidget(_badge("MARK XLIX", C.PRI_DIM))
+        lay.addWidget(_badge("MARK LI", C.PRI_DIM))
         lay.addSpacing(8)
         self._drawer_btn = QPushButton("⚙")
         self._drawer_btn.setFixedSize(26, 26)
@@ -2710,6 +2822,14 @@ class MainWindow(QMainWindow):
         self._brief_btn.clicked.connect(self._toggle_brief)
         lay.addWidget(self._brief_btn)
 
+        plugin_btn = QPushButton("🧩  PLUGINS")
+        plugin_btn.setFixedHeight(26)
+        plugin_btn.setFont(QFont("Courier New", 7))
+        plugin_btn.setCursor(Qt.CursorShape.PointingHandCursor)
+        plugin_btn.setStyleSheet(_BTN_STYLE_DIM)
+        plugin_btn.clicked.connect(self._open_plugin_manager)
+        lay.addWidget(plugin_btn)
+
         w.adjustSize()
         return w
 
@@ -3099,7 +3219,7 @@ class MainWindow(QMainWindow):
         """Update all name/theme-dependent UI elements and persist to config."""
         self._assistant_name = name.strip() or "JARVIS"
         display = self._assistant_name.upper()
-        self.setWindowTitle(f"{display} — MARK XLIX")
+        self.setWindowTitle(f"{display} — MARK LI")
         self._title_lbl.setText(display)
         if display in ("JARVIS", "J.A.R.V.I.S"):
             self._sub_lbl.setText("Just A Rather Very Intelligent System")
@@ -3129,6 +3249,20 @@ class MainWindow(QMainWindow):
         except Exception as e:
             self._log.append_log(f"ERR: Config save failed — {e}")
 
+    def _open_plugin_manager(self):
+        plugins = self.get_plugins() if self.get_plugins else []
+        cw = self.centralWidget()
+        ov = PluginManagerOverlay(plugins, parent=cw)
+        ov.adjustSize()
+        ov.setGeometry(
+            (cw.width()  - ov.width())  // 2,
+            (cw.height() - ov.height()) // 2,
+            ov.width(), ov.height(),
+        )
+        ov.show()
+        ov.raise_()
+        self._plugin_manager_overlay = ov   # keep a reference so it isn't GC'd
+
     # ── Clipboard intelligence ───────────────────────────────────────────────────
 
     def _on_clipboard_changed(self):
@@ -3293,6 +3427,14 @@ class JarvisUI:
     def on_interrupt(self, cb):
         self._win.on_interrupt = cb
 
+    @property
+    def get_plugins(self):
+        return self._win.get_plugins
+
+    @get_plugins.setter
+    def get_plugins(self, cb):
+        self._win.get_plugins = cb
+
     def notify_phone_connected(self) -> None:
         self._win.notify_phone_connected()