From 679cdbea395f32b156b1aa375432db49103d4bb6 Mon Sep 17 00:00:00 2001 From: FatihMakes Date: Sun, 10 May 2026 17:23:10 +0300 Subject: [PATCH] Change --- actions/browser_control.py | 520 ++++++++++++++++++++ actions/code_helper.py | 583 ++++++++++++++++++++++ actions/computer_control.py | 478 ++++++++++++++++++ actions/computer_settings.py | 672 +++++++++++++++++++++++++ actions/desktop.py | 457 +++++++++++++++++ actions/dev_agent.py | 597 +++++++++++++++++++++++ actions/file_controller.py | 482 ++++++++++++++++++ actions/file_processor.py | 833 +++++++++++++++++++++++++++++++ actions/flight_finder.py | 339 +++++++++++++ actions/game_updater.py | 817 +++++++++++++++++++++++++++++++ actions/open_app.py | 208 ++++++++ actions/reminder.py | 156 ++++++ actions/screen_processor.py | 367 ++++++++++++++ actions/send_message.py | 215 ++++++++ actions/weather_report.py | 62 +++ actions/web_search.py | 137 ++++++ actions/youtube_video.py | 418 ++++++++++++++++ agent/error_handler.py | 197 ++++++++ agent/executor.py | 400 +++++++++++++++ agent/planner.py | 278 +++++++++++ agent/task_queue.py | 221 +++++++++ config/__init__.py | 16 + core/prompt.txt | 35 ++ main.py | 916 +++++++++++++++++++++++++++++++++-- memory/__init__.py | 1 + memory/config_manager.py | 59 +++ memory/memory_manager.py | 316 ++++++++++++ or_client.py | 376 ++++++++++++++ 28 files changed, 10124 insertions(+), 32 deletions(-) create mode 100644 actions/browser_control.py create mode 100644 actions/code_helper.py create mode 100644 actions/computer_control.py create mode 100644 actions/computer_settings.py create mode 100644 actions/desktop.py create mode 100644 actions/dev_agent.py create mode 100644 actions/file_controller.py create mode 100644 actions/file_processor.py create mode 100644 actions/flight_finder.py create mode 100644 actions/game_updater.py create mode 100644 actions/open_app.py create mode 100644 actions/reminder.py create mode 100644 actions/screen_processor.py create mode 100644 actions/send_message.py create mode 100644 actions/weather_report.py create mode 100644 actions/web_search.py create mode 100644 actions/youtube_video.py create mode 100644 agent/error_handler.py create mode 100644 agent/executor.py create mode 100644 agent/planner.py create mode 100644 agent/task_queue.py create mode 100644 config/__init__.py create mode 100644 core/prompt.txt create mode 100644 memory/__init__.py create mode 100644 memory/config_manager.py create mode 100644 memory/memory_manager.py create mode 100644 or_client.py diff --git a/actions/browser_control.py b/actions/browser_control.py new file mode 100644 index 0000000..4f424cd --- /dev/null +++ b/actions/browser_control.py @@ -0,0 +1,520 @@ +import asyncio +import threading +import concurrent.futures +import platform +import shutil +import subprocess +from pathlib import Path +from playwright.async_api import async_playwright, TimeoutError as PlaywrightTimeout + + +def _get_default_browser_id() -> str: + """Returns raw default browser identifier string for current OS.""" + system = platform.system() + try: + if system == "Windows": + import winreg + key = winreg.OpenKey( + winreg.HKEY_CURRENT_USER, + r"Software\Microsoft\Windows\Shell\Associations\UrlAssociations\http\UserChoice" + ) + prog_id = winreg.QueryValueEx(key, "ProgId")[0].lower() + winreg.CloseKey(key) + return prog_id + + elif system == "Darwin": + result = subprocess.run( + ["defaults", "read", + "com.apple.LaunchServices/com.apple.launchservices.secure", + "LSHandlers"], + capture_output=True, text=True, timeout=5 + ) + return result.stdout.lower() + + elif system == "Linux": + result = subprocess.run( + ["xdg-settings", "get", "default-web-browser"], + capture_output=True, text=True, timeout=5 + ) + return result.stdout.lower() + + except Exception: + pass + + return "" + + +_BROWSER_BINARIES = { + "Windows": { + "opera": ["opera.exe"], + "brave": ["brave.exe"], + "vivaldi": ["vivaldi.exe"], + "chrome": ["chrome.exe"], + "firefox": ["firefox.exe"], + }, + "Darwin": { + "opera": ["opera"], + "brave": ["brave browser", "brave"], + "vivaldi": ["vivaldi"], + "chrome": ["google chrome", "google-chrome"], + "firefox": ["firefox"], + }, + "Linux": { + "opera": ["opera", "opera-stable"], + "brave": ["brave-browser", "brave"], + "vivaldi": ["vivaldi-stable", "vivaldi"], + "chrome": ["google-chrome", "google-chrome-stable", "chromium-browser", "chromium"], + "firefox": ["firefox"], + }, +} + + +def _get_opera_executable() -> str | None: + if platform.system() != "Windows": + return None + try: + import winreg + candidate_keys = [ + r"SOFTWARE\Microsoft\Windows\CurrentVersion\App Paths\opera.exe", + r"SOFTWARE\Microsoft\Windows\CurrentVersion\App Paths\launcher.exe", + r"SOFTWARE\Clients\StartMenuInternet\OperaStable\shell\open\command", + r"SOFTWARE\Clients\StartMenuInternet\OperaGXStable\shell\open\command", + ] + for key_path in candidate_keys: + for hive in [winreg.HKEY_LOCAL_MACHINE, winreg.HKEY_CURRENT_USER]: + try: + key = winreg.OpenKey(hive, key_path) + val = winreg.QueryValue(key, None) + winreg.CloseKey(key) + exe = val.strip().strip('"').split('"')[0].split(" --")[0].strip() + if exe and Path(exe).exists(): + print(f"[Browser] πŸ” Opera found via registry: {exe}") + return exe + except Exception: + continue + except Exception: + pass + return None + + +def _find_browser_executable(prog_id: str) -> tuple: + """ + Returns (engine_name, exe_path, channel, is_opera). + is_opera=True β†’ extra args needed to prevent private-mode launch. + """ + system = platform.system() + os_bins = _BROWSER_BINARIES.get(system, {}) + + if any(x in prog_id for x in ["firefox", "mozilla"]): + return "firefox", None, None, False + + if "safari" in prog_id: + return "webkit", None, None, False + + if "edge" in prog_id: + return "chromium", None, "msedge", False + + if "opera" in prog_id: + exe = _get_opera_executable() + if exe: + return "chromium", exe, None, True + for binary in os_bins.get("opera", []): + path = shutil.which(binary) + if path: + return "chromium", path, None, True + + browser_patterns = { + "brave": ["brave"], + "vivaldi": ["vivaldi"], + "chrome": ["chrome"], + } + for browser_name, patterns in browser_patterns.items(): + if not any(p in prog_id for p in patterns): + continue + binaries = os_bins.get(browser_name, []) + for binary in binaries: + path = shutil.which(binary) + if path: + print(f"[Browser] πŸ” Found {browser_name} at: {path}") + return "chromium", path, None, False + + if "chrome" in prog_id or not prog_id: + return "chromium", None, "chrome", False + + return "chromium", None, None, False + + +class _BrowserThread: + + def __init__(self): + self._loop = None + self._thread = None + self._ready = threading.Event() + self._playwright = None + self._browser = None + self._context = None + self._page = None + self._engine_name = "chromium" + self._exe_path = None + self._channel = None + self._is_opera = False + + def start(self): + if self._thread and self._thread.is_alive(): + return + self._thread = threading.Thread( + target=self._run_loop, daemon=True, name="BrowserThread" + ) + self._thread.start() + self._ready.wait(timeout=15) + + def _run_loop(self): + self._loop = asyncio.new_event_loop() + asyncio.set_event_loop(self._loop) + self._loop.run_until_complete(self._init()) + self._ready.set() + self._loop.run_forever() + + async def _init(self): + self._playwright = await async_playwright().start() + + def run(self, coro, timeout: int = 30): + if not self._loop: + raise RuntimeError("BrowserThread not started.") + future = asyncio.run_coroutine_threadsafe(coro, self._loop) + return future.result(timeout=timeout) + + # ── TarayΔ±cΔ± ve sayfa yΓΆnetimi ─────────────────────────────────────────── + + async def _launch_browser_if_needed(self): + """ + TarayΔ±cΔ±yΔ± başlatΔ±r. Zaten aΓ§Δ±ksa hiΓ§bir şey yapmaz. + Her zaman default tarayΔ±cΔ±yΔ± kullanΔ±r, ΓΆzel sekme aΓ§maz. + """ + if self._browser and self._browser.is_connected(): + return + + prog_id = _get_default_browser_id() + self._engine_name, self._exe_path, self._channel, self._is_opera = _find_browser_executable(prog_id) + engine = getattr(self._playwright, self._engine_name) + + # Temel chromium argΓΌmanlarΔ± + chromium_args = ["--start-maximized"] + + if self._is_opera: + # Opera GX bazΔ± sΓΌrΓΌmlerde varsayΔ±lan olarak private modda başlar. + # Aşağıdaki flag'ler bunu engeller. + chromium_args += [ + "--disable-features=OperaPrivacyMode", + "--no-private", + ] + print("[Browser] 🎭 Opera detected β€” disabling private-mode flags") + + launch_kwargs = {"headless": False} + if self._engine_name == "chromium": + launch_kwargs["args"] = chromium_args + if self._exe_path: + launch_kwargs["executable_path"] = self._exe_path + elif self._channel: + launch_kwargs["channel"] = self._channel + + try: + self._browser = await engine.launch(**launch_kwargs) + print( + f"[Browser] βœ… Launched ({self._engine_name}" + f"{' / ' + self._channel if self._channel else ''}" + f"{' / ' + self._exe_path if self._exe_path else ''})" + ) + except Exception as e: + print(f"[Browser] ⚠️ Launch failed ({e}), falling back to built-in Chromium") + self._browser = await self._playwright.chromium.launch( + headless=False, + args=["--start-maximized"] + ) + + async def _get_page(self): + """ + Mevcut sayfayΔ± dΓΆndΓΌrΓΌr. + - TarayΔ±cΔ± kapalΔ±ysa aΓ§ar. + - Context yoksa oluşturur. + - Sayfa kapalΔ±ysa yeni sekme aΓ§ar (aynΔ± pencerede). + - Sayfa zaten aΓ§Δ±ksa aynΔ± sayfayΔ± dΓΆndΓΌrΓΌr (yeni pencere aΓ§maz). + """ + await self._launch_browser_if_needed() + + if self._context is None: + self._context = await self._browser.new_context( + viewport=None, + user_agent=( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + "AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/120.0.0.0 Safari/537.36" + ) + ) + + if self._page is None or self._page.is_closed(): + self._page = await self._context.new_page() + + return self._page + + # ── Eylemler ───────────────────────────────────────────────────────────── + + async def _go_to(self, url: str) -> str: + if not url.startswith("http"): + url = "https://" + url + page = await self._get_page() + try: + await page.goto(url, wait_until="domcontentloaded", timeout=15000) + return f"Opened: {page.url}" + except PlaywrightTimeout: + return f"Timeout loading: {url}" + except Exception as e: + return f"Navigation error: {e}" + + async def _search(self, query: str, engine: str = "google") -> str: + engines = { + "google": f"https://www.google.com/search?q={query.replace(' ', '+')}", + "bing": f"https://www.bing.com/search?q={query.replace(' ', '+')}", + "duckduckgo": f"https://duckduckgo.com/?q={query.replace(' ', '+')}", + } + url = engines.get(engine.lower(), engines["google"]) + return await self._go_to(url) + + async def _click(self, selector=None, text=None) -> str: + page = await self._get_page() + try: + if text: + await page.get_by_text(text, exact=False).first.click(timeout=8000) + return f"Clicked: '{text}'" + elif selector: + await page.click(selector, timeout=8000) + return f"Clicked: {selector}" + return "No selector or text provided." + except PlaywrightTimeout: + return "Element not found or not clickable." + except Exception as e: + return f"Click error: {e}" + + async def _type(self, selector=None, text: str = "", clear_first: bool = True) -> str: + page = await self._get_page() + try: + element = page.locator(selector).first if selector else page.locator(":focus") + if clear_first: + await element.clear() + await element.type(text, delay=50) + return "Text typed." + except Exception as e: + return f"Type error: {e}" + + async def _scroll(self, direction: str = "down", amount: int = 500) -> str: + page = await self._get_page() + try: + y = amount if direction == "down" else -amount + await page.mouse.wheel(0, y) + return f"Scrolled {direction}." + except Exception as e: + return f"Scroll error: {e}" + + async def _press(self, key: str) -> str: + page = await self._get_page() + try: + await page.keyboard.press(key) + return f"Pressed: {key}" + except Exception as e: + return f"Key error: {e}" + + async def _get_text(self) -> str: + page = await self._get_page() + try: + text = await page.inner_text("body") + return text[:4000] if len(text) > 4000 else text + except Exception as e: + return f"Could not get page text: {e}" + + async def _fill_form(self, fields: dict) -> str: + page = await self._get_page() + results = [] + for selector, value in fields.items(): + try: + el = page.locator(selector).first + await el.clear() + await el.type(str(value), delay=40) + results.append(f"βœ“ {selector}") + except Exception as e: + results.append(f"βœ— {selector}: {e}") + return "Form filled: " + ", ".join(results) + + async def _smart_click(self, description: str) -> str: + page = await self._get_page() + desc_lower = description.lower() + + role_hints = { + "button": ["button", "buton", "btn"], + "link": ["link", "bağlantΔ±"], + "searchbox": ["search", "arama"], + "textbox": ["input", "field", "alan"], + } + for role, keywords in role_hints.items(): + if any(k in desc_lower for k in keywords): + try: + await page.get_by_role(role).first.click(timeout=5000) + return f"Clicked ({role}): '{description}'" + except Exception: + pass + + try: + await page.get_by_text(description, exact=False).first.click(timeout=5000) + return f"Clicked (text): '{description}'" + except Exception: + pass + + try: + await page.get_by_placeholder(description, exact=False).first.click(timeout=5000) + return f"Clicked (placeholder): '{description}'" + except Exception: + pass + + return f"Could not find: '{description}'" + + async def _smart_type(self, description: str, text: str) -> str: + page = await self._get_page() + + for method, locator in [ + ("placeholder", page.get_by_placeholder(description, exact=False)), + ("label", page.get_by_label(description, exact=False)), + ("role", page.get_by_role("textbox")), + ]: + try: + el = locator.first + await el.clear() + await el.type(text, delay=50) + return f"Typed into ({method}): '{description}'" + except Exception: + continue + + return f"Could not find input: '{description}'" + + async def _close_browser(self) -> str: + if self._browser: + await self._browser.close() + self._browser = None + self._context = None + self._page = None + + if self._playwright: + await self._playwright.stop() + self._playwright = None + + return "Browser closed." + + +# ── Singleton browser thread ───────────────────────────────────────────────── + +_bt = _BrowserThread() +_bt_started = False +_bt_lock = threading.Lock() + + +def _ensure_started(): + global _bt_started + with _bt_lock: + if not _bt_started: + _bt.start() + _bt_started = True + + +# ── Public API ─────────────────────────────────────────────────────────────── + +def browser_control( + parameters: dict, + response=None, + player=None, + session_memory=None +) -> str: + """ + Browser controller β€” auto-detects and uses system default browser. + Always reuses the existing browser window/page; never opens incognito. + + parameters: + action : go_to | search | click | type | scroll | fill_form | + smart_click | smart_type | get_text | press | close + url : URL for go_to + query : search query + engine : google | bing | duckduckgo (default: google) + selector : CSS selector for click/type + text : text to click or type + description : element description for smart_click/smart_type + direction : up | down for scroll + amount : scroll amount in pixels (default: 500) + key : key name for press (e.g. Enter, Escape, Tab) + fields : {selector: value} dict for fill_form + clear_first : bool, clear input before typing (default: True) + """ + _ensure_started() + + action = (parameters or {}).get("action", "").lower().strip() + result = "Unknown action." + + try: + if action == "go_to": + result = _bt.run(_bt._go_to(parameters.get("url", ""))) + + elif action == "search": + result = _bt.run(_bt._search( + parameters.get("query", ""), + parameters.get("engine", "google"), + )) + + elif action == "click": + result = _bt.run(_bt._click( + selector=parameters.get("selector"), + text=parameters.get("text"), + )) + + elif action == "type": + result = _bt.run(_bt._type( + selector=parameters.get("selector"), + text=parameters.get("text", ""), + clear_first=parameters.get("clear_first", True), + )) + + elif action == "scroll": + result = _bt.run(_bt._scroll( + direction=parameters.get("direction", "down"), + amount=parameters.get("amount", 500), + )) + + elif action == "fill_form": + result = _bt.run(_bt._fill_form(parameters.get("fields", {}))) + + elif action == "smart_click": + result = _bt.run(_bt._smart_click(parameters.get("description", ""))) + + elif action == "smart_type": + result = _bt.run(_bt._smart_type( + parameters.get("description", ""), + parameters.get("text", ""), + )) + + elif action == "get_text": + result = _bt.run(_bt._get_text()) + + elif action == "press": + result = _bt.run(_bt._press(parameters.get("key", "Enter"))) + + elif action == "close": + result = _bt.run(_bt._close_browser()) + + else: + result = f"Unknown action: {action}" + + except concurrent.futures.TimeoutError: + result = "Browser action timed out." + except Exception as e: + result = f"Browser error: {e}" + + print(f"[Browser] {result[:80]}") + if player: + player.write_log(f"[browser] {result[:60]}") + + return result \ No newline at end of file diff --git a/actions/code_helper.py b/actions/code_helper.py new file mode 100644 index 0000000..561fb34 --- /dev/null +++ b/actions/code_helper.py @@ -0,0 +1,583 @@ +# actions/code_helper.py +# AI-powered code assistant β€” writes, edits, explains, runs, builds, debugs, and optimizes code. +# +# Actions: +# write β†’ Describe what you want, Gemini writes it, saves to file +# edit β†’ Read existing file, apply natural language change +# explain β†’ Explain what a piece of code or file does +# run β†’ Execute a script file, return output +# build β†’ Write β†’ Run β†’ Fix loop (max 3 attempts), speaks when done +# screen_debug β†’ Screenshot al, ekrandaki kodu/hatayΔ± Gemini ile analiz et ve dΓΌzelt +# optimize β†’ Mevcut kodu Gemini ile optimize et (performans, okunabilirlik, best practices) +# auto β†’ (default) Intent auto-detected from context + +import subprocess +import sys +import json +import re +import time +from pathlib import Path + + +def get_base_dir(): + if getattr(sys, "frozen", False): + return Path(sys.executable).parent + return Path(__file__).resolve().parent.parent + +BASE_DIR = get_base_dir() +API_CONFIG_PATH = BASE_DIR / "config" / "api_keys.json" +DESKTOP = Path.home() / "Desktop" +MAX_BUILD_ATTEMPTS = 3 +GEMINI_MODEL = "gemini-2.5-flash" + + +def _get_api_key() -> str: + with open(API_CONFIG_PATH, "r", encoding="utf-8") as f: + return json.load(f)["gemini_api_key"] + + +def _get_gemini(model: str = GEMINI_MODEL): + import google.generativeai as genai + genai.configure(api_key=_get_api_key()) + return genai.GenerativeModel(model) + + +def _clean_code(text: str) -> str: + text = text.strip() + text = re.sub(r"^```[a-zA-Z]*\n?", "", text) + text = re.sub(r"\n?```$", "", text) + return text.strip() + + +def _resolve_save_path(output_path: str, language: str) -> Path: + ext_map = { + "python": ".py", "py": ".py", + "javascript": ".js", "js": ".js", + "typescript": ".ts", "ts": ".ts", + "html": ".html", "css": ".css", + "java": ".java", "cpp": ".cpp", "c": ".c", + "bash": ".sh", "shell": ".sh", "powershell": ".ps1", + "sql": ".sql", "json": ".json", "rust": ".rs", "go": ".go", + } + if output_path: + p = Path(output_path) + return p if p.is_absolute() else DESKTOP / p + ext = ext_map.get((language or "python").lower(), ".py") + return DESKTOP / f"jarvis_code{ext}" + + +def _read_file(file_path: str) -> tuple[str, str]: + if not file_path: + return "", "No file path provided." + p = Path(file_path) + if not p.exists(): + return "", f"File not found: {file_path}" + try: + return p.read_text(encoding="utf-8"), "" + except Exception as e: + return "", f"Could not read file: {e}" + + +def _save_file(path: Path, content: str) -> str: + try: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + return f"Saved to: {path}" + except Exception as e: + return f"Could not save: {e}" + + +def _preview(code: str, lines: int = 10) -> str: + all_lines = code.splitlines() + preview = "\n".join(all_lines[:lines]) + suffix = f"\n... ({len(all_lines) - lines} more lines)" if len(all_lines) > lines else "" + return preview + suffix + + +def _has_error(output: str) -> bool: + error_signals = ["error", "exception", "traceback", "syntaxerror", + "nameerror", "typeerror", "stderr", "failed", "crash"] + return any(s in output.lower() for s in error_signals) + + +def _take_screenshot() -> Path | None: + try: + import pyautogui + screenshot_path = Path.home() / "Desktop" / f"jarvis_debug_{int(time.time())}.png" + screenshot = pyautogui.screenshot() + screenshot.save(str(screenshot_path)) + print(f"[Code] πŸ“Έ Screenshot: {screenshot_path}") + return screenshot_path + except Exception as e: + print(f"[Code] ⚠️ Screenshot failed: {e}") + return None + + +def _image_to_base64(path: Path) -> str: + import base64 + return base64.b64encode(path.read_bytes()).decode("utf-8") + + +def _detect_intent(description: str, file_path: str, code: str) -> str: + desc = (description or "").lower() + + screen_kw = ["ekrandaki", "screen", "ekranda", "bu hatayΔ±", "why am i getting", + "neden hata", "what's wrong", "ne yanlış", "screenshot", "gΓΆrΓΌntΓΌ"] + if any(k in desc for k in screen_kw): + return "screen_debug" + + # Optimize keywords + optimize_kw = ["optimize", "refactor", "clean up", "improve", "temizle", + "iyileştir", "daha iyi", "make it better", "hΔ±zlandΔ±r"] + if any(k in desc for k in optimize_kw) and (code or file_path): + return "optimize" + + if file_path: + p = Path(file_path) + edit_kw = ["edit", "update", "modify", "change", "add", "remove", + "refactor", "fix", "rename", "replace", "dΓΌzenle", "değiştir"] + run_kw = ["run", "execute", "launch", "start", "Γ§alıştΔ±r"] + build_kw = ["build", "make it work", "try", "attempt"] + + if p.exists() and any(k in desc for k in edit_kw): + return "edit" + if p.exists() and any(k in desc for k in run_kw): + return "run" + if any(k in desc for k in build_kw): + return "build" + if p.exists(): + return "explain" + + explain_kw = ["explain", "what does", "describe", "analyze", "aΓ§Δ±kla", "ne yapΔ±yor"] + if any(k in desc for k in explain_kw) and (code or file_path): + return "explain" + + build_kw = ["build", "make it work", "try and", "attempt"] + if any(k in desc for k in build_kw): + return "build" + + return "write" + +def _write(description: str, language: str, output_path: str, player=None) -> tuple[str, Path]: + lang = language or "python" + model = _get_gemini() + + prompt = f"""You are an expert {lang} developer. +Write clean, working, well-commented {lang} code for the description below. + +Rules: +- Output ONLY the code. No explanation, no markdown, no backticks. +- Add helpful inline comments. +- Handle errors and edge cases properly. +- Use modern best practices. + +Description: {description} + +Code:""" + + response = model.generate_content(prompt) + code = _clean_code(response.text) + path = _resolve_save_path(output_path, lang) + _save_file(path, code) + return code, path + + +def _fix_code(code: str, error_output: str, description: str) -> str: + model = _get_gemini() + prompt = f"""You are an expert debugger. +The code below failed with the following error. Fix it. +Return ONLY the corrected code β€” no explanation, no markdown, no backticks. + +Original goal: {description} + +Error: +{error_output[:2000]} + +Broken code: +{code} + +Fixed code:""" + + response = model.generate_content(prompt) + return _clean_code(response.text) + + +def _run_file(path: Path, args: list, timeout: int) -> str: + interpreters = { + ".py": [sys.executable], + ".js": ["node"], + ".ts": ["ts-node"], + ".sh": ["bash"], + ".ps1": ["powershell", "-File"], + ".rb": ["ruby"], + ".php": ["php"], + } + interp = interpreters.get(path.suffix.lower()) + if not interp: + return f"No interpreter for {path.suffix}." + + try: + result = subprocess.run( + interp + [str(path)] + (args or []), + capture_output=True, text=True, + encoding="utf-8", errors="replace", + timeout=timeout, cwd=str(path.parent) + ) + output = result.stdout.strip() + error = result.stderr.strip() + parts = [] + if output: parts.append(f"Output:\n{output}") + if error: parts.append(f"Stderr:\n{error}") + return "\n\n".join(parts) if parts else "Executed with no output." + + except subprocess.TimeoutExpired: + return f"Timed out after {timeout}s." + except FileNotFoundError: + return f"Interpreter not found: {interp[0]}." + except Exception as e: + return f"Execution error: {e}" + + +def _build(description, language, output_path, args, timeout, speak=None, player=None) -> str: + if not description: + return "Please describe what you want me to build, sir." + + if player: + player.write_log("[Code] Build started...") + + lang = language or "python" + + try: + code, path = _write(description, lang, output_path, player) + print(f"[Code] βœ… Written: {path}") + except Exception as e: + msg = f"Could not write initial code: {e}" + if speak: speak(msg) + return msg + + last_output = "" + for attempt in range(1, MAX_BUILD_ATTEMPTS + 1): + print(f"[Code] πŸ”„ Attempt {attempt}/{MAX_BUILD_ATTEMPTS}") + if player: + player.write_log(f"[Code] Attempt {attempt}...") + + last_output = _run_file(path, args, timeout) + + if not _has_error(last_output): + msg = ( + f"Build complete, sir. " + f"The code is working after {attempt} attempt{'s' if attempt > 1 else ''}. " + f"Saved to {path}." + ) + if speak: speak(msg) + return f"{msg}\n\nOutput:\n{last_output}" + + print(f"[Code] ⚠️ Error on attempt {attempt}, fixing...") + if player: + player.write_log(f"[Code] Fixing (attempt {attempt})...") + + try: + code = _fix_code(code, last_output, description) + _save_file(path, code) + except Exception as e: + msg = f"Could not fix code on attempt {attempt}: {e}" + if speak: speak(msg) + return msg + + msg = ( + f"I was unable to build a working version after {MAX_BUILD_ATTEMPTS} attempts, sir. " + f"The last error was: {last_output[:200]}" + ) + if speak: speak(msg) + return f"{msg}\n\nLast code saved to: {path}" + +def _write_action(description, language, output_path, player) -> str: + if not description: + return "Please describe what you want me to write, sir." + if player: + player.write_log("[Code] Writing code...") + try: + code, path = _write(description, language, output_path, player) + print(f"[Code] βœ… Written: {path}") + return f"Code written. Saved to: {path}\n\nPreview:\n{_preview(code)}" + except Exception as e: + return f"Could not generate code: {e}" + + +def _edit_action(file_path, instruction, player) -> str: + if not file_path: + return "Please provide a file path to edit, sir." + if not instruction: + return "Please describe what change to make, sir." + + content, err = _read_file(file_path) + if err: + return err + + if player: + player.write_log("[Code] Editing file...") + + model = _get_gemini() + prompt = f"""You are an expert code editor. +Apply the following change to the code below. +Return ONLY the complete updated code β€” no explanation, no markdown, no backticks. + +Change: {instruction} + +Original code: +{content} + +Updated code:""" + + try: + response = model.generate_content(prompt) + edited = _clean_code(response.text) + except Exception as e: + return f"Could not edit code: {e}" + + status = _save_file(Path(file_path), edited) + print(f"[Code] βœ… Edited: {file_path}") + return f"File edited. {status}\n\nPreview:\n{_preview(edited)}" + + +def _explain_action(file_path, code, player) -> str: + if file_path and not code: + code, err = _read_file(file_path) + if err: + return err + if not code: + return "Please provide code or a file path to explain, sir." + + if player: + player.write_log("[Code] Analyzing code...") + + model = _get_gemini() + prompt = f"""Explain what this code does in simple, clear language. +Focus on: what it does, how it works, and any important details. +Be concise β€” 3 to 6 sentences maximum. + +Code: +{code[:4000]} + +Explanation:""" + + try: + response = model.generate_content(prompt) + return response.text.strip() + except Exception as e: + return f"Could not explain code: {e}" + + +def _run_action(file_path, args, timeout, player) -> str: + if not file_path: + return "Please provide a file path to run, sir." + p = Path(file_path) + if not p.exists(): + return f"File not found: {file_path}" + if player: + player.write_log(f"[Code] Running {p.name}...") + return _run_file(p, args, timeout) + + +def _optimize_action(file_path, code, language, output_path, player) -> str: + + if file_path and not code: + code, err = _read_file(file_path) + if err: + return err + if not code: + return "Please provide code or a file path to optimize, sir." + + if player: + player.write_log("[Code] Optimizing code...") + + lang = language or "python" + model = _get_gemini() + + prompt = f"""You are an expert {lang} developer and code reviewer. +Optimize the following code for: +1. Performance β€” eliminate unnecessary operations, use efficient data structures +2. Readability β€” clear variable names, proper formatting, logical structure +3. Best practices β€” modern {lang} patterns, error handling, type hints if applicable +4. Remove dead code, redundant comments, and unnecessary complexity + +Return ONLY the optimized code β€” no explanation, no markdown, no backticks. + +Original code: +{code[:6000]} + +Optimized code:""" + + try: + response = model.generate_content(prompt) + optimized = _clean_code(response.text) + except Exception as e: + return f"Could not optimize code: {e}" + + # Kaydet + if file_path: + save_path = Path(file_path) + else: + save_path = _resolve_save_path(output_path, lang) + + status = _save_file(save_path, optimized) + print(f"[Code] βœ… Optimized: {save_path}") + + original_lines = len(code.splitlines()) + optimized_lines = len(optimized.splitlines()) + diff = original_lines - optimized_lines + + return ( + f"Code optimized. {status}\n" + f"Lines: {original_lines} β†’ {optimized_lines} " + f"({'βˆ’' if diff > 0 else '+'}{abs(diff)} lines)\n\n" + f"Preview:\n{_preview(optimized)}" + ) + + +def _screen_debug_action(description, file_path, player, speak=None) -> str: + + if player: + player.write_log("[Code] Taking screenshot for analysis...") + + print("[Code] πŸ“Έ Capturing screen for debug...") + + + screenshot_path = _take_screenshot() + if not screenshot_path: + return "Could not take screenshot, sir. Please make sure PyAutoGUI is installed." + + + file_content = "" + if file_path: + file_content, err = _read_file(file_path) + if err: + print(f"[Code] ⚠️ Could not read file: {err}") + + try: + from google import genai + from google.genai import types + + client = genai.Client(api_key=_get_api_key()) + + image_bytes = screenshot_path.read_bytes() + image_base64 = _image_to_base64(screenshot_path) + + user_question = description or "What error or problem do you see on the screen? How can it be fixed?" + + context = "" + if file_content: + context = f"\n\nAdditionally, here is the related file content:\n```\n{file_content[:4000]}\n```" + + analysis_prompt = f"""You are an expert programmer and debugger analyzing a screenshot. + +User's question: {user_question}{context} + +Please: +1. Identify any errors, exceptions, or problems visible on the screen +2. Explain what is causing the problem in simple terms +3. Provide a concrete fix or solution +4. If there's code visible, show the corrected version + +Be specific and actionable. If you see an error message, quote it exactly.""" + + contents = [ + types.Part.from_bytes(data=image_bytes, mime_type="image/png"), + analysis_prompt, + ] + + response = client.models.generate_content( + model="gemini-2.5-flash", + contents=contents, + ) + + analysis = response.text.strip() + print(f"[Code] βœ… Screen analysis complete") + + try: + screenshot_path.unlink() + except Exception: + pass + + if file_path and file_content: + + code_match = re.search(r"```[a-zA-Z]*\n(.*?)```", analysis, re.DOTALL) + if code_match: + fixed_code = code_match.group(1).strip() + save_path = Path(file_path) + _save_file(save_path, fixed_code) + analysis += f"\n\nβœ… Fixed code has been saved to: {file_path}" + print(f"[Code] βœ… Fixed code saved: {file_path}") + + return analysis + + except Exception as e: + + try: + screenshot_path.unlink() + except Exception: + pass + return f"Screen analysis failed: {e}" + + +def code_helper( + parameters: dict, + response=None, + player=None, + session_memory=None, + speak=None +) -> str: + """ + Called from main.py. + + parameters: + action : write | edit | explain | run | build | screen_debug | optimize | auto + description : What the code should do / what change to make / what problem to analyze + language : Programming language (default: python) + output_path : Where to save β€” user specifies full path or filename + file_path : Path to existing file (edit / explain / run / build / optimize) + code : Raw code string (explain/optimize without a file) + args : CLI argument list for run/build + timeout : Execution timeout in seconds (default: 30) + """ + p = parameters or {} + action = p.get("action", "auto").lower().strip() + description = p.get("description", "").strip() + language = p.get("language", "python").strip() + output_path = p.get("output_path", "").strip() + file_path = p.get("file_path", "").strip() + code = p.get("code", "").strip() + args = p.get("args", []) + timeout = int(p.get("timeout", 30)) + + if action == "auto": + action = _detect_intent(description, file_path, code) + print(f"[Code] πŸ€– Auto-detected: {action}") + + if action == "write": + return _write_action(description, language, output_path, player) + + elif action == "edit": + return _edit_action( + file_path, + description or p.get("instruction", ""), + player + ) + + elif action == "explain": + return _explain_action(file_path, code, player) + + elif action == "run": + return _run_action(file_path, args, timeout, player) + + elif action == "build": + return _build(description, language, output_path, args, timeout, speak, player) + + elif action == "optimize": + return _optimize_action(file_path, code, language, output_path, player) + + elif action == "screen_debug": + return _screen_debug_action(description, file_path, player, speak) + + else: + return f"Unknown action: '{action}'. Use write, edit, explain, run, build, optimize, or screen_debug." \ No newline at end of file diff --git a/actions/computer_control.py b/actions/computer_control.py new file mode 100644 index 0000000..4203792 --- /dev/null +++ b/actions/computer_control.py @@ -0,0 +1,478 @@ +#computer_control.py +import io +import json +import re +import string +import subprocess +import sys +import time +import random +from pathlib import Path + +try: + import pyautogui + pyautogui.FAILSAFE = True + pyautogui.PAUSE = 0.05 + _PYAUTOGUI = True +except ImportError: + _PYAUTOGUI = False + +try: + import pyperclip + _PYPERCLIP = True +except ImportError: + _PYPERCLIP = False + +def _base_dir() -> Path: + if getattr(sys, "frozen", False): + return Path(sys.executable).parent + return Path(__file__).resolve().parent.parent + + +_BASE = _base_dir() +_CONFIG_PATH = _BASE / "config" / "api_keys.json" +_MEMORY_PATH = _BASE / "memory" / "long_term.json" + +def _load_config() -> dict: + try: + return json.loads(_CONFIG_PATH.read_text(encoding="utf-8")) + except Exception: + return {} + +def _get_os() -> str: + return _load_config().get("os_system", "windows").lower() + +_SAFE_SCREENSHOT_ROOTS = ( + Path.home(), +) + +def _safe_screenshot_path(requested: str | None) -> Path: + fallback = Path.home() / "Desktop" / "jarvis_screenshot.png" + if not requested: + return fallback + try: + p = Path(requested).expanduser().resolve() + for root in _SAFE_SCREENSHOT_ROOTS: + if p.is_relative_to(root.resolve()): + p.parent.mkdir(parents=True, exist_ok=True) + return p + except Exception: + pass + return fallback + +def _require_pyautogui(): + if not _PYAUTOGUI: + raise RuntimeError("PyAutoGUI not installed. Run: pip install pyautogui") + +_FIRST_NAMES = [ + "Alex", "Jordan", "Taylor", "Morgan", "Casey", "Riley", "Drew", "Quinn", + "Avery", "Blake", "Cameron", "Dakota", "Emerson", "Finley", "Harper", +] +_LAST_NAMES = [ + "Smith", "Johnson", "Williams", "Brown", "Jones", "Garcia", "Miller", + "Davis", "Wilson", "Moore", "Taylor", "Anderson", "Thomas", "Jackson", +] +_DOMAINS = ["gmail.com", "yahoo.com", "outlook.com", "proton.me", "mail.com"] + + +def _random_data(data_type: str) -> str: + dt = data_type.lower().strip() + + if dt == "first_name": + return random.choice(_FIRST_NAMES) + + if dt == "last_name": + return random.choice(_LAST_NAMES) + + if dt == "name": + return f"{random.choice(_FIRST_NAMES)} {random.choice(_LAST_NAMES)}" + + if dt == "email": + first = random.choice(_FIRST_NAMES).lower() + last = random.choice(_LAST_NAMES).lower() + num = random.randint(10, 999) + return f"{first}.{last}{num}@{random.choice(_DOMAINS)}" + + if dt == "username": + return f"{random.choice(_FIRST_NAMES).lower()}{random.randint(100, 9999)}" + + if dt == "password": + chars = string.ascii_letters + string.digits + "!@#$%" + raw = ( + random.choice(string.ascii_uppercase) + + random.choice(string.digits) + + random.choice("!@#$%") + + "".join(random.choices(chars, k=9)) + ) + return "".join(random.sample(raw, len(raw))) + + if dt == "phone": + return f"+1{random.randint(200,999)}{random.randint(1_000_000, 9_999_999)}" + + if dt == "birthday": + y = random.randint(1980, 2000) + m = random.randint(1, 12) + d = random.randint(1, 28) + return f"{m:02d}/{d:02d}/{y}" + + if dt == "address": + num = random.randint(100, 9999) + street = random.choice(["Main St", "Oak Ave", "Park Blvd", "Elm St", "Cedar Ln"]) + return f"{num} {street}" + + if dt == "zip_code": + return str(random.randint(10000, 99999)) + + if dt == "city": + return random.choice(["New York", "Los Angeles", "Chicago", "Houston", "Phoenix"]) + + return f"random_{data_type}_{random.randint(1000, 9999)}" + +def _user_profile() -> dict: + """Read identity fields from long-term memory.""" + try: + if _MEMORY_PATH.exists(): + data = json.loads(_MEMORY_PATH.read_text(encoding="utf-8")) + identity = data.get("identity", {}) + return {k: v.get("value", "") for k, v in identity.items()} + except Exception: + pass + return {} + +def _type(text: str, interval: float = 0.03) -> str: + _require_pyautogui() + time.sleep(0.3) + pyautogui.typewrite(text, interval=interval) + return f"Typed: {text[:60]}{'…' if len(text) > 60 else ''}" + + +def _smart_type(text: str, clear_first: bool = True) -> str: + _require_pyautogui() + if clear_first: + _clear_field() + time.sleep(0.1) + + if len(text) > 20 and _PYPERCLIP: + pyperclip.copy(text) + time.sleep(0.1) + pyautogui.hotkey("ctrl", "v") + return f"Smart-typed (clipboard): {text[:60]}{'…' if len(text) > 60 else ''}" + + pyautogui.typewrite(text, interval=0.04) + return f"Smart-typed: {text[:60]}{'…' if len(text) > 60 else ''}" + + +def _click(x=None, y=None, button: str = "left", clicks: int = 1) -> str: + _require_pyautogui() + if x is not None and y is not None: + pyautogui.click(x, y, button=button, clicks=clicks) + return f"{'Double-c' if clicks == 2 else 'C'}licked ({x}, {y}) [{button}]" + pyautogui.click(button=button, clicks=clicks) + return f"Clicked at current position [{button}]" + + +def _hotkey(*keys) -> str: + _require_pyautogui() + pyautogui.hotkey(*keys) + return f"Hotkey: {'+'.join(keys)}" + + +def _press(key: str) -> str: + _require_pyautogui() + pyautogui.press(key) + return f"Pressed: {key}" + + +def _scroll(direction: str = "down", amount: int = 3) -> str: + _require_pyautogui() + vertical = direction in ("up", "down") + clicks = amount if direction in ("up", "right") else -amount + pyautogui.scroll(clicks) if vertical else pyautogui.hscroll(clicks) + return f"Scrolled {direction} Γ—{amount}" + + +def _move(x: int, y: int, duration: float = 0.3) -> str: + _require_pyautogui() + pyautogui.moveTo(x, y, duration=duration) + return f"Mouse β†’ ({x}, {y})" + + +def _drag(x1: int, y1: int, x2: int, y2: int, duration: float = 0.5) -> str: + _require_pyautogui() + pyautogui.moveTo(x1, y1, duration=0.2) + pyautogui.dragTo(x2, y2, duration=duration, button="left") + return f"Dragged ({x1},{y1}) β†’ ({x2},{y2})" + + +def _clipboard_get() -> str: + if _PYPERCLIP: + return pyperclip.paste() + _hotkey("ctrl", "c") + time.sleep(0.2) + return "(copied β€” pyperclip unavailable for read)" + + +def _clipboard_paste(text: str) -> str: + if _PYPERCLIP: + pyperclip.copy(text) + time.sleep(0.1) + _require_pyautogui() + pyautogui.hotkey("ctrl", "v") + return f"Pasted: {text[:60]}{'…' if len(text) > 60 else ''}" + return "pyperclip not available" + + +def _screenshot(save_path: str | None = None) -> str: + _require_pyautogui() + path = _safe_screenshot_path(save_path) + img = pyautogui.screenshot() + img.save(str(path)) + return f"Screenshot saved: {path}" + + +def _clear_field() -> str: + _require_pyautogui() + pyautogui.hotkey("ctrl", "a") + time.sleep(0.1) + pyautogui.press("delete") + return "Field cleared" + +def _focus_window(title: str) -> str: + os_name = _get_os() + + if os_name == "windows": + try: + script = f'(New-Object -ComObject WScript.Shell).AppActivate("{title}")' + subprocess.run( + ["powershell", "-NoProfile", "-NonInteractive", "-Command", script], + capture_output=True, timeout=5, + ) + time.sleep(0.3) + return f"Focused window: {title}" + except Exception as e: + return f"focus_window (Windows) failed: {e}" + + if os_name == "mac": + script = ( + f'tell application "System Events" to ' + f'set frontmost of (first process whose name contains "{title}") to true' + ) + try: + subprocess.run( + ["osascript", "-e", script], + capture_output=True, timeout=5, + ) + time.sleep(0.3) + return f"Focused window: {title}" + except Exception as e: + return f"focus_window (macOS) failed: {e}" + + if os_name == "linux": + try: + result = subprocess.run( + ["wmctrl", "-a", title], + capture_output=True, timeout=5, + ) + if result.returncode == 0: + time.sleep(0.3) + return f"Focused window: {title}" + except FileNotFoundError: + pass + try: + result = subprocess.run( + ["xdotool", "search", "--name", title, "windowactivate"], + capture_output=True, timeout=5, + ) + time.sleep(0.3) + return f"Focused window: {title}" + except FileNotFoundError: + return "focus_window (Linux) requires wmctrl or xdotool" + except Exception as e: + return f"focus_window (Linux) failed: {e}" + + return f"focus_window: unknown OS '{os_name}'" +def _screen_find(description: str) -> tuple[int, int] | None: + try: + import base64 + from or_client import client + + _require_pyautogui() + w, h = pyautogui.size() + img = pyautogui.screenshot() + buf = io.BytesIO() + img.save(buf, format="PNG") + b64 = base64.b64encode(buf.getvalue()).decode() + + text = client.vision( + f"This is a screenshot of a {w}Γ—{h} pixel screen. " + f"Locate the UI element: '{description}'. " + f"Reply ONLY with center coordinates as: x,y β€” or NOT_FOUND", + image_b64=b64, + mime="image/png", + ) + + if "NOT_FOUND" in text.upper(): + return None + match = re.search(r"(\d+)\s*,\s*(\d+)", text) + if match: + return int(match.group(1)), int(match.group(2)) + except Exception as e: + print(f"[ComputerControl] ⚠️ screen_find failed: {e}") + return None + +def computer_control( + parameters: dict, + response=None, + player=None, + session_memory=None, +) -> str: + """ + Dispatch table for all computer control actions. + + parameters keys (all optional unless noted): + action : (required) one of the actions listed below + text : text to type or paste + x, y : screen coordinates + button : 'left' | 'right' (default: left) + keys : hotkey string, e.g. 'ctrl+c' + key : single key name, e.g. 'enter' + direction : 'up' | 'down' | 'left' | 'right' + amount : scroll amount (default: 3) + seconds : wait duration + title : window title fragment for focus_window + description : natural-language element description for screen_find/click + type : data type for random_data + field : memory field name for user_data + clear_first : bool, clear field before typing (default: true) + path : save path for screenshot (must be inside home dir) + + Actions: + type β€” type text at cursor + smart_type β€” clear field + type (clipboard-backed) + click β€” left click + double_click β€” double left click + right_click β€” right click + move β€” move mouse + drag β€” click-drag between two points + hotkey β€” key combination + press β€” single key + scroll β€” scroll the wheel + copy β€” read clipboard + paste β€” write + paste clipboard + screenshot β€” capture screen (safe path only) + wait β€” sleep N seconds + clear_field β€” select-all + delete + focus_window β€” bring window to foreground + screen_find β€” AI element finder (returns x,y) + screen_click β€” AI element finder + click + random_data β€” generate fake form data + user_data β€” pull real data from memory + """ + params = parameters or {} + action = params.get("action", "").lower().strip() + + if not action: + return "No action specified for computer_control." + + if player: + player.write_log(f"[Computer] {action}") + + print(f"[ComputerControl] β–Ά {action} {params}") + + try: + + if action == "type": + return _type(params.get("text", "")) + + if action == "smart_type": + return _smart_type( + params.get("text", ""), + clear_first=params.get("clear_first", True), + ) + + if action in ("click", "left_click"): + return _click(params.get("x"), params.get("y"), "left", 1) + + if action == "double_click": + return _click(params.get("x"), params.get("y"), "left", 2) + + if action == "right_click": + return _click(params.get("x"), params.get("y"), "right", 1) + + if action == "move": + return _move(int(params.get("x", 0)), int(params.get("y", 0))) + + if action == "drag": + return _drag( + int(params.get("x1", 0)), int(params.get("y1", 0)), + int(params.get("x2", 0)), int(params.get("y2", 0)), + ) + + if action == "hotkey": + raw = params.get("keys", "") + keys = [k.strip() for k in raw.split("+")] if isinstance(raw, str) else raw + return _hotkey(*keys) + + if action == "press": + return _press(params.get("key", "enter")) + + if action == "scroll": + return _scroll( + direction=params.get("direction", "down"), + amount=int(params.get("amount", 3)), + ) + + if action == "copy": + return _clipboard_get() + + if action == "paste": + return _clipboard_paste(params.get("text", "")) + + if action == "screenshot": + return _screenshot(params.get("path")) + + if action == "screen_find": + coords = _screen_find(params.get("description", "")) + return f"{coords[0]},{coords[1]}" if coords else "NOT_FOUND" + + if action == "screen_click": + desc = params.get("description", "") + coords = _screen_find(desc) + if coords: + time.sleep(0.2) + _click(x=coords[0], y=coords[1]) + return f"Clicked '{desc}' at {coords}" + return f"Element not found on screen: '{desc}'" + + if action == "wait": + secs = float(params.get("seconds", 1.0)) + secs = min(secs, 30.0) + time.sleep(secs) + return f"Waited {secs}s" + + if action == "clear_field": + return _clear_field() + + if action == "focus_window": + return _focus_window(params.get("title", "")) + + if action == "random_data": + dt = params.get("type", "name") + result = _random_data(dt) + print(f"[ComputerControl] 🎲 random {dt} β†’ {result}") + return result + + if action == "user_data": + field = params.get("field", "name") + profile = _user_profile() + value = profile.get(field, "") + if not value: + value = _random_data(field) + print(f"[ComputerControl] ⚠️ No '{field}' in memory, using random: {value}") + return value + + return f"Unknown action: '{action}'" + + except Exception as e: + print(f"[ComputerControl] ❌ {action}: {e}") + return f"computer_control '{action}' failed: {e}" \ No newline at end of file diff --git a/actions/computer_settings.py b/actions/computer_settings.py new file mode 100644 index 0000000..7b24951 --- /dev/null +++ b/actions/computer_settings.py @@ -0,0 +1,672 @@ +#computer_settings.py +import json +import re +import sys +import time +import subprocess +import platform +from pathlib import Path + +try: + import pyautogui + pyautogui.FAILSAFE = True + pyautogui.PAUSE = 0.05 + _PYAUTOGUI = True +except ImportError: + _PYAUTOGUI = False + +try: + import pyperclip + _PYPERCLIP = True +except ImportError: + _PYPERCLIP = False + +_OS = platform.system() # "Windows" | "Darwin" | "Linux" + + +def _get_base_dir() -> Path: + if getattr(sys, "frozen", False): + return Path(sys.executable).parent + return Path(__file__).resolve().parent.parent + +def _get_macos_wifi_interface() -> str: + try: + result = subprocess.run( + ["networksetup", "-listallhardwareports"], + capture_output=True, text=True, timeout=5 + ) + lines = result.stdout.splitlines() + for i, line in enumerate(lines): + if "Wi-Fi" in line or "AirPort" in line: + for j in range(i, min(i + 4, len(lines))): + if lines[j].startswith("Device:"): + return lines[j].split(":", 1)[1].strip() + except Exception: + pass + return "en0" + +def volume_up(): + if _OS == "Windows": + for _ in range(5): pyautogui.press("volumeup") + elif _OS == "Darwin": + subprocess.run(["osascript", "-e", + "set volume output volume (output volume of (get volume settings) + 10)"], + capture_output=True) + else: + subprocess.run(["pactl", "set-sink-volume", "@DEFAULT_SINK@", "+10%"], + capture_output=True) + +def volume_down(): + if _OS == "Windows": + for _ in range(5): pyautogui.press("volumedown") + elif _OS == "Darwin": + subprocess.run(["osascript", "-e", + "set volume output volume (output volume of (get volume settings) - 10)"], + capture_output=True) + else: + subprocess.run(["pactl", "set-sink-volume", "@DEFAULT_SINK@", "-10%"], + capture_output=True) + +def volume_mute(): + if _OS == "Windows": + pyautogui.press("volumemute") + elif _OS == "Darwin": + subprocess.run(["osascript", "-e", "set volume with output muted"], + capture_output=True) + else: + subprocess.run(["pactl", "set-sink-mute", "@DEFAULT_SINK@", "toggle"], + capture_output=True) + +def volume_set(value: int): + value = max(0, min(100, int(value))) + if _OS == "Windows": + try: + import math + from ctypes import cast, POINTER + from comtypes import CLSCTX_ALL + from pycaw.pycaw import AudioUtilities, IAudioEndpointVolume + devices = AudioUtilities.GetSpeakers() + interface = devices.Activate(IAudioEndpointVolume._iid_, CLSCTX_ALL, None) + vol = cast(interface, POINTER(IAudioEndpointVolume)) + vol_db = -65.25 if value == 0 else max(-65.25, 20 * math.log10(value / 100)) + vol.SetMasterVolumeLevel(vol_db, None) + return + except Exception as e: + print(f"[Settings] pycaw failed, using keypress fallback: {e}") + pyautogui.press("volumemute") + pyautogui.press("volumemute") + elif _OS == "Darwin": + subprocess.run(["osascript", "-e", f"set volume output volume {value}"], + capture_output=True) + return + else: + subprocess.run(["pactl", "set-sink-volume", "@DEFAULT_SINK@", f"{value}%"], + capture_output=True) + return + +def brightness_up(): + if _OS == "Darwin": + subprocess.run(["osascript", "-e", + 'tell application "System Events" to key code 144'], + capture_output=True) + elif _OS == "Linux": + if subprocess.run(["which", "brightnessctl"], + capture_output=True).returncode == 0: + subprocess.run(["brightnessctl", "set", "+10%"], capture_output=True) + else: + subprocess.run( + 'xrandr --output $(xrandr | grep " connected" | head -1 | cut -d " " -f1)' + ' --brightness $(python3 -c "import subprocess; ' + 'b=float(subprocess.check_output([\"xrandr\",\"--verbose\"]).decode()' + '.split(\"Brightness:\")[1].split()[0]); print(min(1.0,b+0.1))")', + shell=True, capture_output=True + ) + else: + try: + subprocess.run( + ["powershell", "-Command", + "(Get-WmiObject -Namespace root/wmi -Class WmiMonitorBrightnessMethods)" + ".WmiSetBrightness(1, [math]::Min(100, " + "(Get-WmiObject -Namespace root/wmi -Class WmiMonitorBrightness).CurrentBrightness + 10))"], + capture_output=True, timeout=5 + ) + except Exception as e: + print(f"[Settings] Brightness up failed on Windows: {e}") + +def brightness_down(): + if _OS == "Darwin": + subprocess.run(["osascript", "-e", + 'tell application "System Events" to key code 145'], + capture_output=True) + elif _OS == "Linux": + if subprocess.run(["which", "brightnessctl"], + capture_output=True).returncode == 0: + subprocess.run(["brightnessctl", "set", "10%-"], capture_output=True) + else: + subprocess.run( + 'xrandr --output $(xrandr | grep " connected" | head -1 | cut -d " " -f1)' + ' --brightness $(python3 -c "import subprocess; ' + 'b=float(subprocess.check_output([\"xrandr\",\"--verbose\"]).decode()' + '.split(\"Brightness:\")[1].split()[0]); print(max(0.1,b-0.1))")', + shell=True, capture_output=True + ) + else: + try: + subprocess.run( + ["powershell", "-Command", + "(Get-WmiObject -Namespace root/wmi -Class WmiMonitorBrightnessMethods)" + ".WmiSetBrightness(1, [math]::Max(0, " + "(Get-WmiObject -Namespace root/wmi -Class WmiMonitorBrightness).CurrentBrightness - 10))"], + capture_output=True, timeout=5 + ) + except Exception as e: + print(f"[Settings] Brightness down failed on Windows: {e}") + +def close_app(): + if _OS == "Darwin": pyautogui.hotkey("command", "q") + else: pyautogui.hotkey("alt", "f4") + +def close_window(): + if _OS == "Darwin": pyautogui.hotkey("command", "w") + else: pyautogui.hotkey("ctrl", "w") + +def full_screen(): + if _OS == "Darwin": pyautogui.hotkey("ctrl", "command", "f") + else: pyautogui.press("f11") + +def minimize_window(): + if _OS == "Darwin": pyautogui.hotkey("command", "m") + else: pyautogui.hotkey("win", "down") + +def maximize_window(): + if _OS == "Darwin": + subprocess.run(["osascript", "-e", + 'tell application "System Events" to keystroke "f" ' + 'using {control down, command down}'], + capture_output=True) + elif _OS == "Windows": + pyautogui.hotkey("win", "up") + else: + try: + subprocess.run(["wmctrl", "-r", ":ACTIVE:", "-b", "add,maximized_vert,maximized_horz"], + capture_output=True) + except Exception: + pyautogui.hotkey("super", "up") + +def snap_left(): + if _OS == "Windows": + pyautogui.hotkey("win", "left") + elif _OS == "Linux": + try: + subprocess.run(["wmctrl", "-r", ":ACTIVE:", "-e", "0,0,0,960,1080"], + capture_output=True) + except Exception: + pass + +def snap_right(): + if _OS == "Windows": + pyautogui.hotkey("win", "right") + elif _OS == "Linux": + try: + subprocess.run(["wmctrl", "-r", ":ACTIVE:", "-e", "0,960,0,960,1080"], + capture_output=True) + except Exception: + pass + +def switch_window(): + if _OS == "Darwin": pyautogui.hotkey("command", "tab") + else: pyautogui.hotkey("alt", "tab") + +def show_desktop(): + if _OS == "Darwin": pyautogui.hotkey("fn", "f11") + elif _OS == "Windows": pyautogui.hotkey("win", "d") + else: pyautogui.hotkey("super", "d") + +def open_task_manager(): + if _OS == "Windows": + pyautogui.hotkey("ctrl", "shift", "esc") + elif _OS == "Darwin": + subprocess.Popen(["open", "-a", "Activity Monitor"]) + else: + for cmd in [["gnome-system-monitor"], ["xfce4-taskmanager"], ["htop"]]: + if subprocess.run(["which", cmd[0]], capture_output=True).returncode == 0: + subprocess.Popen(cmd) + break + + +def focus_search(): + if _OS == "Darwin": pyautogui.hotkey("command", "l") + else: pyautogui.hotkey("ctrl", "l") + +def pause_video(): pyautogui.press("space") + +def refresh_page(): + if _OS == "Darwin": pyautogui.hotkey("command", "r") + else: pyautogui.press("f5") + +def close_tab(): + if _OS == "Darwin": pyautogui.hotkey("command", "w") + else: pyautogui.hotkey("ctrl", "w") + +def new_tab(): + if _OS == "Darwin": pyautogui.hotkey("command", "t") + else: pyautogui.hotkey("ctrl", "t") + +def next_tab(): + if _OS == "Darwin": pyautogui.hotkey("command", "shift", "bracketright") + else: pyautogui.hotkey("ctrl", "tab") + +def prev_tab(): + if _OS == "Darwin": pyautogui.hotkey("command", "shift", "bracketleft") + else: pyautogui.hotkey("ctrl", "shift", "tab") + +def go_back(): + if _OS == "Darwin": pyautogui.hotkey("command", "left") + else: pyautogui.hotkey("alt", "left") + +def go_forward(): + if _OS == "Darwin": pyautogui.hotkey("command", "right") + else: pyautogui.hotkey("alt", "right") + +def zoom_in(): + if _OS == "Darwin": pyautogui.hotkey("command", "equal") + else: pyautogui.hotkey("ctrl", "equal") + +def zoom_out(): + if _OS == "Darwin": pyautogui.hotkey("command", "minus") + else: pyautogui.hotkey("ctrl", "minus") + +def zoom_reset(): + if _OS == "Darwin": pyautogui.hotkey("command", "0") + else: pyautogui.hotkey("ctrl", "0") + +def find_on_page(): + if _OS == "Darwin": pyautogui.hotkey("command", "f") + else: pyautogui.hotkey("ctrl", "f") + +def reload_page_n(n: int): + for _ in range(max(1, n)): + refresh_page() + time.sleep(0.8) + + +def scroll_up(amount: int = 500): pyautogui.scroll(amount) +def scroll_down(amount: int = 500): pyautogui.scroll(-amount) + +def scroll_top(): + if _OS == "Darwin": pyautogui.hotkey("command", "up") + else: pyautogui.hotkey("ctrl", "home") + +def scroll_bottom(): + if _OS == "Darwin": pyautogui.hotkey("command", "down") + else: pyautogui.hotkey("ctrl", "end") + +def page_up(): pyautogui.press("pageup") +def page_down(): pyautogui.press("pagedown") + + +def copy(): + if _OS == "Darwin": pyautogui.hotkey("command", "c") + else: pyautogui.hotkey("ctrl", "c") + +def paste(): + if _OS == "Darwin": pyautogui.hotkey("command", "v") + else: pyautogui.hotkey("ctrl", "v") + +def cut(): + if _OS == "Darwin": pyautogui.hotkey("command", "x") + else: pyautogui.hotkey("ctrl", "x") + +def undo(): + if _OS == "Darwin": pyautogui.hotkey("command", "z") + else: pyautogui.hotkey("ctrl", "z") + +def redo(): + if _OS == "Darwin": pyautogui.hotkey("command", "shift", "z") + else: pyautogui.hotkey("ctrl", "y") + +def select_all(): + if _OS == "Darwin": pyautogui.hotkey("command", "a") + else: pyautogui.hotkey("ctrl", "a") + +def save_file(): + if _OS == "Darwin": pyautogui.hotkey("command", "s") + else: pyautogui.hotkey("ctrl", "s") + +def press_enter(): pyautogui.press("enter") +def press_escape(): pyautogui.press("escape") +def press_key(key: str): pyautogui.press(key) + +def type_text(text: str, press_enter_after: bool = False): + if not text: + return + if _PYPERCLIP: + pyperclip.copy(str(text)) + time.sleep(0.15) + paste() + else: + pyautogui.write(str(text), interval=0.03) + if press_enter_after: + time.sleep(0.1) + pyautogui.press("enter") + +def take_screenshot(): + if _OS == "Windows": + pyautogui.hotkey("win", "shift", "s") + elif _OS == "Darwin": + pyautogui.hotkey("command", "shift", "3") + else: + for cmd in [["scrot"], ["gnome-screenshot"], ["import", "-window", "root", "screenshot.png"]]: + if subprocess.run(["which", cmd[0]], capture_output=True).returncode == 0: + subprocess.Popen(cmd) + return + pyautogui.hotkey("ctrl", "print_screen") + +def lock_screen(): + if _OS == "Windows": + pyautogui.hotkey("win", "l") + elif _OS == "Darwin": + subprocess.run(["pmset", "displaysleepnow"], capture_output=True) + else: + for cmd in [ + ["gnome-screensaver-command", "-l"], + ["xdg-screensaver", "lock"], + ["loginctl", "lock-session"], + ]: + if subprocess.run(["which", cmd[0]], capture_output=True).returncode == 0: + subprocess.run(cmd, capture_output=True) + return + +def open_system_settings(): + if _OS == "Windows": + pyautogui.hotkey("win", "i") + elif _OS == "Darwin": + subprocess.Popen(["open", "-a", "System Preferences"]) + else: + for cmd in [["gnome-control-center"], ["xfce4-settings-manager"], ["kcmshell5"]]: + if subprocess.run(["which", cmd[0]], capture_output=True).returncode == 0: + subprocess.Popen(cmd) + return + +def open_file_explorer(): + if _OS == "Windows": + pyautogui.hotkey("win", "e") + elif _OS == "Darwin": + subprocess.Popen(["open", str(Path.home())]) + else: + for cmd in [["nautilus"], ["thunar"], ["dolphin"], ["nemo"]]: + if subprocess.run(["which", cmd[0]], capture_output=True).returncode == 0: + subprocess.Popen(cmd) + return + subprocess.Popen(["xdg-open", str(Path.home())]) + +def sleep_display(): + if _OS == "Windows": + try: + import ctypes + ctypes.windll.user32.SendMessageW(0xFFFF, 0x0112, 0xF170, 2) + except Exception as e: + print(f"[Settings] sleep_display failed: {e}") + elif _OS == "Darwin": + subprocess.run(["pmset", "displaysleepnow"], capture_output=True) + else: + subprocess.run(["xset", "dpms", "force", "off"], capture_output=True) + +def open_run(): + if _OS == "Windows": + pyautogui.hotkey("win", "r") + +def dark_mode(): + if _OS == "Darwin": + subprocess.run(["osascript", "-e", + 'tell app "System Events" to tell appearance preferences ' + 'to set dark mode to not dark mode'], + capture_output=True) + elif _OS == "Windows": + try: + import winreg + key_path = r"SOFTWARE\Microsoft\Windows\CurrentVersion\Themes\Personalize" + key = winreg.OpenKey(winreg.HKEY_CURRENT_USER, key_path, 0, winreg.KEY_ALL_ACCESS) + current, _ = winreg.QueryValueEx(key, "AppsUseLightTheme") + winreg.SetValueEx(key, "AppsUseLightTheme", 0, winreg.REG_DWORD, 1 - current) + winreg.SetValueEx(key, "SystemUsesLightTheme", 0, winreg.REG_DWORD, 1 - current) + winreg.CloseKey(key) + except Exception as e: + print(f"[Settings] dark_mode registry failed: {e}") + else: + try: + result = subprocess.run( + ["gsettings", "get", "org.gnome.desktop.interface", "color-scheme"], + capture_output=True, text=True + ) + current = result.stdout.strip() + new_scheme = "'default'" if "dark" in current else "'prefer-dark'" + subprocess.run( + ["gsettings", "set", "org.gnome.desktop.interface", "color-scheme", new_scheme], + capture_output=True + ) + except Exception as e: + print(f"[Settings] dark_mode Linux failed: {e}") + +def toggle_wifi(): + if _OS == "Darwin": + iface = _get_macos_wifi_interface() + result = subprocess.run( + ["networksetup", "-getairportpower", iface], + capture_output=True, text=True + ) + state = "off" if "On" in result.stdout else "on" + subprocess.run(["networksetup", "-setairportpower", iface, state], + capture_output=True) + elif _OS == "Windows": + try: + subprocess.run( + ["powershell", "-Command", + "$adapter = Get-NetAdapter | Where-Object {$_.PhysicalMediaType -eq 'Native 802.11'};" + "if ($adapter.Status -eq 'Up') { Disable-NetAdapter -Name $adapter.Name -Confirm:$false }" + "else { Enable-NetAdapter -Name $adapter.Name -Confirm:$false }"], + capture_output=True, timeout=10 + ) + except Exception as e: + print(f"[Settings] toggle_wifi Windows failed: {e}") + else: + try: + result = subprocess.run(["nmcli", "radio", "wifi"], capture_output=True, text=True) + state = "off" if "enabled" in result.stdout else "on" + subprocess.run(["nmcli", "radio", "wifi", state], capture_output=True) + except Exception as e: + print(f"[Settings] toggle_wifi Linux failed: {e}") + +def restart_computer(): + if _OS == "Windows": + subprocess.run(["shutdown", "/r", "/t", "10"], capture_output=True) + elif _OS == "Darwin": + subprocess.run(["osascript", "-e", + 'tell application "System Events" to restart'], + capture_output=True) + else: + subprocess.run(["systemctl", "reboot"], capture_output=True) + +def shutdown_computer(): + if _OS == "Windows": + subprocess.run(["shutdown", "/s", "/t", "10"], capture_output=True) + elif _OS == "Darwin": + subprocess.run(["osascript", "-e", + 'tell application "System Events" to shut down'], + capture_output=True) + else: + subprocess.run(["systemctl", "poweroff"], capture_output=True) + +ACTION_MAP: dict[str, callable] = { + "volume_up": volume_up, + "volume_down": volume_down, + "mute": volume_mute, + "unmute": volume_mute, + "toggle_mute": volume_mute, + "brightness_up": brightness_up, + "brightness_down": brightness_down, + "sleep_display": sleep_display, + "screen_off": sleep_display, + "pause_video": pause_video, + "play_pause": pause_video, + "close_app": close_app, + "close_window": close_window, + "full_screen": full_screen, + "fullscreen": full_screen, + "minimize": minimize_window, + "maximize": maximize_window, + "snap_left": snap_left, + "snap_right": snap_right, + "switch_window": switch_window, + "show_desktop": show_desktop, + "task_manager": open_task_manager, + "focus_search": focus_search, + "refresh_page": refresh_page, + "reload": refresh_page, + "close_tab": close_tab, + "new_tab": new_tab, + "next_tab": next_tab, + "prev_tab": prev_tab, + "go_back": go_back, + "go_forward": go_forward, + "zoom_in": zoom_in, + "zoom_out": zoom_out, + "zoom_reset": zoom_reset, + "find_on_page": find_on_page, + "scroll_up": scroll_up, + "scroll_down": scroll_down, + "scroll_top": scroll_top, + "scroll_bottom": scroll_bottom, + "page_up": page_up, + "page_down": page_down, + "copy": copy, + "paste": paste, + "cut": cut, + "undo": undo, + "redo": redo, + "select_all": select_all, + "save": save_file, + "enter": press_enter, + "escape": press_escape, + "screenshot": take_screenshot, + "lock_screen": lock_screen, + "open_settings": open_system_settings, + "file_explorer": open_file_explorer, + "open_run": open_run, + "dark_mode": dark_mode, + "toggle_wifi": toggle_wifi, + "restart": restart_computer, + "shutdown": shutdown_computer, +} + +_DANGEROUS_ACTIONS = {"restart", "shutdown"} + + +def _detect_action(description: str) -> dict: + from or_client import client + + available = ", ".join(sorted(ACTION_MAP.keys())) + \ + ", volume_set, type_text, press_key, reload_n" + + prompt = f"""You are an intent detector for a computer control assistant. +The user issued a command (possibly in any language): "{description}" +Available actions: {available} +Return ONLY a valid JSON object: {{"action": "action_name", "value": null_or_value}} +Rules: +- For volume_set: value is an integer 0-100. +- For type_text: value is the exact text to type. +- For press_key: value is the key name (e.g. "f5", "tab", "enter"). +- For reload_n: value is an integer. +- Return ONLY the JSON, no explanation, no markdown.""" + + try: + raw = client.chat_json(prompt, system="Return only valid JSON. No extra text.") + return raw + except Exception as e: + print(f"[Settings] Intent detection failed: {e}") + return {"action": description.lower().replace(" ", "_"), "value": None} + +def computer_settings( + parameters: dict = None, + response=None, + player=None, + session_memory=None, +) -> str: + if not _PYAUTOGUI: + return "pyautogui is not installed. Run: pip install pyautogui" + + params = parameters or {} + raw_action = params.get("action", "").strip() + description = params.get("description", "").strip() + value = params.get("value", None) + + if not raw_action and description: + detected = _detect_action(description) + raw_action = detected.get("action", "") + if value is None: + value = detected.get("value") + + action = raw_action.lower().strip().replace(" ", "_").replace("-", "_") + + if not action: + return "No action could be determined." + + print(f"[Settings] Action: {action} Value: {value} OS: {_OS}") + if player: + player.write_log(f"[Settings] {action}") + + if action in _DANGEROUS_ACTIONS: + confirmed = str(params.get("confirmed", "")).lower() + if confirmed not in ("yes", "true", "1", "confirm"): + return ( + f"This will {action} the computer. " + f"Please confirm by calling again with confirmed=yes." + ) + + if action == "volume_set": + try: + volume_set(int(value or 50)) + return f"Volume set to {value}%." + except Exception as e: + return f"Could not set volume: {e}" + + if action in ("type_text", "write_on_screen", "type", "write"): + text = str(value or params.get("text", "")).strip() + if not text: + return "No text provided to type." + enter_after = str(params.get("press_enter", "false")).lower() in ("true", "1", "yes") + type_text(text, press_enter_after=enter_after) + return f"Typed: {text[:80]}" + + if action == "press_key": + key = str(value or params.get("key", "")).strip() + if not key: + return "No key specified." + press_key(key) + return f"Pressed: {key}" + + if action in ("reload_n", "refresh_n", "reload_page_n"): + try: + reload_page_n(int(value or 1)) + return f"Reloaded {value or 1} time(s)." + except Exception as e: + return f"Reload failed: {e}" + + if action == "scroll_up": + scroll_up(int(value or 500)) + return "Scrolled up." + + if action == "scroll_down": + scroll_down(int(value or 500)) + return "Scrolled down." + + func = ACTION_MAP.get(action) + if not func: + return f"Unknown action: '{raw_action}'." + + try: + func() + return f"Done: {action}." + except Exception as e: + print(f"[Settings] Action failed ({action}): {e}") + return f"Action failed ({action}): {e}" \ No newline at end of file diff --git a/actions/desktop.py b/actions/desktop.py new file mode 100644 index 0000000..d172e98 --- /dev/null +++ b/actions/desktop.py @@ -0,0 +1,457 @@ +#desktop.py +import os +import sys +import json +import shutil +import subprocess +import tempfile +import platform +from pathlib import Path +from datetime import datetime + +try: + import pyautogui + _PYAUTOGUI = True +except ImportError: + _PYAUTOGUI = False + +_OS = platform.system() # "Windows" | "Darwin" | "Linux" + + +def _get_base_dir() -> Path: + if getattr(sys, "frozen", False): + return Path(sys.executable).parent + return Path(__file__).resolve().parent.parent + +def _get_desktop() -> Path: + if _OS == "Linux": + xdg = os.environ.get("XDG_DESKTOP_DIR", "") + if xdg and Path(xdg).exists(): + return Path(xdg) + return Path.home() / "Desktop" + +def _build_sandbox() -> dict: + import time + + safe_builtins = { + "print": print, + "len": len, "str": str, "int": int, "float": float, + "bool": bool, "list": list, "dict": dict, "tuple": tuple, + "range": range, "enumerate": enumerate, "sorted": sorted, + "isinstance": isinstance, "hasattr": hasattr, "getattr": getattr, + "max": max, "min": min, "sum": sum, "abs": abs, + "zip": zip, "map": map, "filter": filter, + } + + sandbox = { + "__builtins__": safe_builtins, + "Path": Path, + "time": time, + "shutil": type("shutil", (), { + "copy2": shutil.copy2, + "copytree": shutil.copytree, + "disk_usage": shutil.disk_usage, + })(), + "os_path": os.path, + } + + if _PYAUTOGUI: + sandbox["pyautogui"] = pyautogui + + if _OS == "Windows": + try: + import ctypes + import winreg + sandbox["ctypes"] = ctypes + sandbox["winreg"] = type("winreg", (), { + # Sadece okuma + "OpenKey": winreg.OpenKey, + "QueryValueEx": winreg.QueryValueEx, + "HKEY_CURRENT_USER": winreg.HKEY_CURRENT_USER, + })() + except ImportError: + pass + + return sandbox + + +def _execute_generated_code(code: str, player=None) -> str: + if not code or code.strip() == "UNSAFE": + return "This action cannot be performed safely." + + # Kod temizleme + if code.startswith("```"): + lines = code.split("\n") + code = "\n".join(lines[1:-1]).strip() + + sandbox = _build_sandbox() + output_lines = [] + sandbox["__builtins__"]["print"] = lambda *a: output_lines.append(" ".join(str(x) for x in a)) + + try: + exec(compile(code, "", "exec"), sandbox) + return "\n".join(output_lines) if output_lines else "Done." + except Exception as e: + print(f"[Desktop] Exec error: {e}\nCode:\n{code[:300]}") + return f"Execution error: {e}" + +def _ask_gemini_for_desktop_action(task: str) -> str: + from or_client import client + + desktop = str(_get_desktop()) + os_specific = { + "Windows": "- ctypes (Windows API calls, read-only)\n- winreg (registry READ only)", + "Darwin": "- subprocess is NOT available; use pyautogui or Path only", + "Linux": "- subprocess is NOT available; use pyautogui or Path only", + }.get(_OS, "") + + prompt = f"""You are a desktop automation assistant. +Current OS: {_OS} +Desktop path: {desktop} +Generate safe Python code to accomplish the task below. +Allowed modules ONLY: +- pyautogui (mouse, keyboard β€” if needed) +- pathlib.Path (file/folder inspection only, no deletion) +- shutil.copy2, shutil.copytree, shutil.disk_usage (NO move, NO rmtree) +- os_path (os.path equivalent, read-only) +- time.sleep +{os_specific} +Hard rules: +- NO file deletion, NO subprocess, NO exec/eval inside the code +- NO import statements, NO file write except explicitly requested +- If task cannot be done safely with these tools, output exactly: UNSAFE +Output ONLY the Python code. No explanation, no markdown, no backticks. +Task: {task}""" + + try: + return client.chat(prompt, system="You are a code generator. Output only raw Python code.") + except Exception as e: + return f"ERROR: {e}" + +def set_wallpaper(image_path: str) -> str: + path = Path(image_path).expanduser().resolve() + if not path.exists(): + return f"Image not found: {image_path}" + if path.suffix.lower() not in {".jpg", ".jpeg", ".png", ".bmp", ".webp"}: + return f"Unsupported format: {path.suffix}. Use jpg, png, bmp or webp." + + try: + if _OS == "Windows": + import ctypes + if path.suffix.lower() in {".webp", ".png"}: + try: + from PIL import Image + bmp_path = Path(tempfile.mktemp(suffix=".bmp")) + Image.open(path).convert("RGB").save(bmp_path, "BMP") + path = bmp_path + except ImportError: + pass + ctypes.windll.user32.SystemParametersInfoW(20, 0, str(path), 3) + return f"Wallpaper set: {path.name}" + + elif _OS == "Darwin": + script = ( + f'tell application "System Events" to tell every desktop to ' + f'set picture to POSIX file "{path}"' + ) + subprocess.run(["osascript", "-e", script], capture_output=True) + return f"Wallpaper set: {path.name}" + + else: + desktop_env = os.environ.get("XDG_CURRENT_DESKTOP", "").lower() + uri = f"file://{path}" + + if "gnome" in desktop_env or "unity" in desktop_env: + subprocess.run([ + "gsettings", "set", "org.gnome.desktop.background", + "picture-uri", uri + ], capture_output=True) + subprocess.run([ + "gsettings", "set", "org.gnome.desktop.background", + "picture-uri-dark", uri + ], capture_output=True) + + elif "kde" in desktop_env: + # KDE Plasma + script = f""" +var allDesktops = desktops(); +for (var i = 0; i < allDesktops.length; i++) {{ + d = allDesktops[i]; + d.wallpaperPlugin = "org.kde.image"; + d.currentConfigGroup = ["Wallpaper", "org.kde.image", "General"]; + d.writeConfig("Image", "file://{path}"); +}} +""" + subprocess.run( + ["qdbus", "org.kde.plasmashell", "/PlasmaShell", + "org.kde.PlasmaShell.evaluateScript", script], + capture_output=True + ) + + elif "xfce" in desktop_env: + subprocess.run([ + "xfconf-query", "-c", "xfce4-desktop", + "-p", "/backdrop/screen0/monitor0/workspace0/last-image", + "-s", str(path) + ], capture_output=True) + + else: + result = subprocess.run( + ["feh", "--bg-scale", str(path)], + capture_output=True + ) + if result.returncode != 0: + return ( + f"Could not set wallpaper automatically on {desktop_env}. " + f"Try manually or install 'feh'." + ) + + return f"Wallpaper set: {path.name}" + + except Exception as e: + return f"Could not set wallpaper: {e}" + + +def set_wallpaper_from_url(url: str) -> str: + try: + import urllib.request + suffix = Path(url.split("?")[0]).suffix or ".jpg" + tmp = Path(tempfile.mktemp(suffix=suffix)) + urllib.request.urlretrieve(url, str(tmp)) + result = set_wallpaper(str(tmp)) + try: + tmp.unlink() + except Exception: + pass + return result + except Exception as e: + return f"Could not download wallpaper: {e}" + + +def get_current_wallpaper() -> str: + try: + if _OS == "Windows": + import winreg + key = winreg.OpenKey( + winreg.HKEY_CURRENT_USER, r"Control Panel\Desktop" + ) + val, _ = winreg.QueryValueEx(key, "Wallpaper") + winreg.CloseKey(key) + return f"Current wallpaper: {val}" + + elif _OS == "Darwin": + script = ( + 'tell application "System Events" to get picture of desktop 1' + ) + result = subprocess.run( + ["osascript", "-e", script], + capture_output=True, text=True + ) + return f"Current wallpaper: {result.stdout.strip()}" + + else: + desktop_env = os.environ.get("XDG_CURRENT_DESKTOP", "").lower() + if "gnome" in desktop_env or "unity" in desktop_env: + result = subprocess.run( + ["gsettings", "get", "org.gnome.desktop.background", "picture-uri"], + capture_output=True, text=True + ) + return f"Current wallpaper: {result.stdout.strip()}" + return "Wallpaper path retrieval not supported for this desktop environment." + + except Exception as e: + return f"Could not get wallpaper: {e}" + +FILE_TYPE_MAP = { + "Images": {".jpg", ".jpeg", ".png", ".gif", ".bmp", ".webp", ".svg", ".ico", ".heic"}, + "Documents": {".pdf", ".doc", ".docx", ".txt", ".xls", ".xlsx", + ".ppt", ".pptx", ".csv", ".odt", ".ods", ".odp"}, + "Videos": {".mp4", ".avi", ".mkv", ".mov", ".wmv", ".flv", ".webm", ".m4v"}, + "Music": {".mp3", ".wav", ".flac", ".aac", ".ogg", ".wma", ".m4a"}, + "Archives": {".zip", ".rar", ".7z", ".tar", ".gz", ".bz2", ".xz"}, + "Code": {".py", ".js", ".ts", ".html", ".css", ".json", ".xml", + ".cpp", ".java", ".cs", ".go", ".rs", ".sh", ".php"}, + "Executables": {".exe", ".msi", ".bat", ".cmd", ".sh", ".appimage", ".deb", ".rpm"}, +} + +_SKIP_EXTENSIONS = { + "Windows": {".lnk", ".url"}, + "Darwin": {".webloc"}, + "Linux": {".desktop"}, +} + + +def organize_desktop(mode: str = "by_type") -> str: + desktop = _get_desktop() + skip_exts = _SKIP_EXTENSIONS.get(_OS, set()) + moved, skipped = [], [] + + for item in desktop.iterdir(): + if item.is_dir() or item.name.startswith("."): + continue + if item.suffix.lower() in skip_exts: + continue + + if mode == "by_date": + mtime = datetime.fromtimestamp(item.stat().st_mtime) + folder_name = mtime.strftime("%Y-%m") + else: + ext = item.suffix.lower() + folder_name = "Others" + for folder, exts in FILE_TYPE_MAP.items(): + if ext in exts: + folder_name = folder + break + + target_dir = desktop / folder_name + target_dir.mkdir(exist_ok=True) + new_path = target_dir / item.name + + if new_path.exists(): + skipped.append(item.name) + continue + + shutil.move(str(item), str(new_path)) + moved.append(f"{item.name} β†’ {folder_name}/") + + result = f"Desktop organized ({mode}): {len(moved)} files moved." + if moved: + result += "\n" + "\n".join(moved[:8]) + if len(moved) > 8: + result += f"\n... and {len(moved) - 8} more." + if skipped: + result += f"\n{len(skipped)} file(s) skipped (name conflict)." + return result + + +def list_desktop() -> str: + desktop = _get_desktop() + items = [] + for item in sorted(desktop.iterdir()): + if item.name.startswith("."): + continue + if item.is_dir(): + try: + count = len(list(item.iterdir())) + except PermissionError: + count = "?" + items.append(f"πŸ“ {item.name}/ ({count} items)") + else: + size = item.stat().st_size + size_str = ( + f"{size / 1024:.1f} KB" if size < 1024 * 1024 + else f"{size / 1024 / 1024:.1f} MB" + ) + items.append(f"πŸ“„ {item.name} ({size_str})") + + if not items: + return "Desktop is empty." + return f"Desktop ({len(items)} items):\n" + "\n".join(items) + + +def clean_desktop() -> str: + desktop = _get_desktop() + skip_exts = _SKIP_EXTENSIONS.get(_OS, set()) + today = datetime.now().strftime("%Y-%m-%d") + archive_dir = desktop / f"Desktop Archive {today}" + archive_dir.mkdir(exist_ok=True) + + moved = 0 + for item in desktop.iterdir(): + if item.is_dir() or item.name.startswith("."): + continue + if item.suffix.lower() in skip_exts: + continue + new_path = archive_dir / item.name + if not new_path.exists(): + shutil.move(str(item), str(new_path)) + moved += 1 + + return f"Desktop cleaned: {moved} files archived to '{archive_dir.name}'." + + +def get_desktop_stats() -> str: + desktop = _get_desktop() + files = [i for i in desktop.iterdir() if i.is_file()] + folders = [i for i in desktop.iterdir() if i.is_dir()] + total_size = sum(f.stat().st_size for f in files if f.exists()) + size_str = ( + f"{total_size / 1024:.1f} KB" if total_size < 1024 * 1024 + else f"{total_size / 1024 / 1024:.1f} MB" + ) + return ( + f"Desktop stats ({_OS}):\n" + f" Files : {len(files)}\n" + f" Folders : {len(folders)}\n" + f" Size : {size_str}\n" + f" Path : {desktop}" + ) + +def desktop_control( + parameters: dict = None, + response=None, + player=None, + session_memory=None, +) -> str: + """ + parameters: + action : wallpaper | wallpaper_url | current_wallpaper | + organize | clean | list | stats | + task (AI-powered) + path : image path for 'wallpaper' + url : image URL for 'wallpaper_url' + mode : 'by_type' or 'by_date' for 'organize' + task : natural language description for AI-powered actions + """ + params = parameters or {} + action = params.get("action", "").lower().strip() + task = params.get("task", "").strip() + + if player: + player.write_log(f"[desktop] {action or task[:40]}") + + try: + if action == "wallpaper": + path = params.get("path", "") + return set_wallpaper(path) if path else "No image path provided." + + elif action == "wallpaper_url": + url = params.get("url", "") + return set_wallpaper_from_url(url) if url else "No URL provided." + + elif action == "current_wallpaper": + return get_current_wallpaper() + + elif action == "organize": + return organize_desktop(params.get("mode", "by_type")) + + elif action == "clean": + return clean_desktop() + + elif action == "list": + return list_desktop() + + elif action == "stats": + return get_desktop_stats() + + elif action == "task" or task: + actual_task = task or params.get("description", "") + if not actual_task: + return "Please describe what you want to do on the desktop." + + print(f"[Desktop] Asking Gemini: {actual_task}") + if player: + player.write_log("[Desktop] Generating action...") + + code = _ask_gemini_for_desktop_action(actual_task) + return _execute_generated_code(code, player=player) + + else: + if action: + code = _ask_gemini_for_desktop_action(action) + return _execute_generated_code(code, player=player) + return "No action or task specified." + + except Exception as e: + print(f"[Desktop] Error: {e}") + return f"Desktop control error: {e}" \ No newline at end of file diff --git a/actions/dev_agent.py b/actions/dev_agent.py new file mode 100644 index 0000000..5365cd2 --- /dev/null +++ b/actions/dev_agent.py @@ -0,0 +1,597 @@ +import subprocess +import sys +import json +import re +import time +from pathlib import Path + + +def get_base_dir(): + if getattr(sys, "frozen", False): + return Path(sys.executable).parent + return Path(__file__).resolve().parent.parent + + +BASE_DIR = get_base_dir() +API_CONFIG_PATH = BASE_DIR / "config" / "api_keys.json" +PROJECTS_DIR = Path.home() / "Desktop" / "JarvisProjects" +MAX_FIX_ATTEMPTS = 5 +MODEL_PLANNER = "gemini-2.5-flash" +MODEL_WRITER = "gemini-2.5-flash" + +def _get_api_key() -> str: + with open(API_CONFIG_PATH, "r", encoding="utf-8") as f: + return json.load(f)["gemini_api_key"] + + +def _get_model(model_name: str): + import google.generativeai as genai + genai.configure(api_key=_get_api_key()) + return genai.GenerativeModel(model_name) + + +def _strip_fences(text: str) -> str: + text = text.strip() + text = re.sub(r"^```[a-zA-Z]*\r?\n?", "", text) + text = re.sub(r"\r?\n?```\s*$", "", text) + return text.strip() + + +def _is_rate_limit(error: Exception) -> bool: + msg = str(error).lower() + return "429" in msg or "quota" in msg or "resource_exhausted" in msg + + +def _parse_traceback(output: str, project_files: list[str]) -> tuple[str | None, int | None]: + + pattern = re.compile(r'File ["\']([^"\']+\.py)["\'],\s+line\s+(\d+)', re.IGNORECASE) + matches = pattern.findall(output) + + for raw_path, line_str in reversed(matches): + raw_name = Path(raw_path).name + for pf in project_files: + if Path(pf).name == raw_name or pf == raw_path or raw_path.endswith(pf): + return pf, int(line_str) + + return None, None + + +def _classify_error(output: str) -> str: + + low = output.lower() + + if any(x in low for x in ("no module named", "modulenotfounderror", "importerror")): + return "dependency_error" + + if "syntaxerror" in low or "invalid syntax" in low: + return "syntax_error" + + if "cannot import" in low or "importerror" in low: + return "import_error" + + if any(x in low for x in ( + "traceback", "exception", "error:", "nameerror", "typeerror", + "attributeerror", "valueerror", "keyerror", "indexerror", + "zerodivisionerror", "filenotfounderror", "permissionerror", + )): + return "runtime_error" + + return "none" + + +def _has_error(output: str, run_command: str) -> bool: + + low = output.lower() + + if "timed out" in low: + return False + + if not output.strip(): + return False + + error_type = _classify_error(output) + return error_type != "none" + +class RateLimitError(Exception): + pass + + +def _plan_project(description: str, language: str) -> dict: + model = _get_model(MODEL_PLANNER) + + prompt = f"""You are a senior software architect. Create a minimal, complete file plan for this project. + +Language: {language} +Description: {description} + +Return ONLY valid JSON β€” no markdown, no explanation: +{{ + "project_name": "snake_case_name", + "entry_point": "main.py", + "files": [ + {{ + "path": "main.py", + "description": "Entry point β€” what it does and which modules it imports", + "imports": ["utils.helpers", "core.engine"] + }}, + {{ + "path": "utils/helpers.py", + "description": "Helper utilities β€” what functions it exposes", + "imports": [] + }} + ], + "run_command": "python main.py", + "dependencies": ["requests"] +}} + +Critical rules: +1. List files in DEPENDENCY ORDER β€” files with no imports come first, entry point comes last. +2. The "imports" field must list every other project module this file imports (dot-notation, e.g. "utils.helpers"). +3. Keep it minimal β€” only files truly needed. +4. Entry point must be in the files list. +5. Use relative paths only (e.g. "utils/helpers.py", not absolute paths). +6. Standard library modules (os, sys, json, etc.) do NOT go in "dependencies". + +JSON:""" + + try: + response = model.generate_content(prompt) + raw = _strip_fences(response.text) + return json.loads(raw) + except json.JSONDecodeError as e: + raise ValueError(f"Planner returned invalid JSON: {e}\nRaw: {response.text[:300]}") + except Exception as e: + if _is_rate_limit(e): + raise RateLimitError(str(e)) + raise + +def _write_file( + file_info: dict, + project_description: str, + all_files: list[dict], + language: str, + project_dir: Path, + already_written: dict[str, str], +) -> str: + model = _get_model(MODEL_WRITER) + + file_path = file_info["path"] + file_desc = file_info.get("description", "") + file_imports = file_info.get("imports", []) + + file_list = "\n".join( + f" [{i+1}] {f['path']}: {f.get('description', '')}" + for i, f in enumerate(all_files) + ) + + dependency_context = "" + for dep_dotted in file_imports: + dep_path = dep_dotted.replace(".", "/") + ".py" + if dep_path in already_written: + code_snippet = already_written[dep_path][:2000] + dependency_context += f"\n\n--- {dep_path} (you must import from this) ---\n{code_snippet}" + + lang_rules = "" + if language.lower() == "python": + lang_rules = """ +Python-specific rules: +- Use type hints for all function signatures. +- Add docstrings for all public functions and classes. +- Use if __name__ == "__main__": guard in the entry point. +- For relative imports within the project, use: from utils.helpers import foo (match the project structure exactly). +- Do NOT use implicit relative imports (from . import ...) unless it's a proper package with __init__.py. +- If this is a package subdirectory, create __init__.py files where needed.""" + elif language.lower() in ("javascript", "typescript", "js", "ts"): + lang_rules = """ +JS/TS-specific rules: +- Use ES modules (import/export), not CommonJS (require). +- Add JSDoc comments for all exported functions. +- Handle promise rejections with try/catch in async functions.""" + + prompt = f"""You are a senior {language} developer writing production-quality code for a real project. + +Project goal: {project_description} + +Complete project file structure (in dependency order): +{file_list} + +{f"Dependencies this file must import from other project files:{dependency_context}" if dependency_context else ""} + +Your task: Write the complete, working code for: {file_path} +Purpose of this file: {file_desc} +{f"This file imports from: {', '.join(file_imports)}" if file_imports else "This file has no project-internal imports."} + +{lang_rules} + +General rules: +- Output ONLY raw code. Absolutely no explanation, no markdown, no triple backticks. +- Write COMPLETE, RUNNABLE code β€” no placeholders, no "# TODO", no "pass" stubs. +- Every import must either be from the standard library, listed dependencies, or the project files shown above. +- Match import paths EXACTLY to the file paths in the project structure (e.g. if file is "utils/helpers.py", import as "from utils.helpers import ..."). +- Use proper error handling (try/except) where I/O or network calls are made. +- The code must work correctly when the project entry point is run from the project root directory. + +Code for {file_path}:""" + + try: + response = model.generate_content(prompt) + code = _strip_fences(response.text) + + full_path = project_dir / file_path + full_path.parent.mkdir(parents=True, exist_ok=True) + full_path.write_text(code, encoding="utf-8") + + print(f"[DevAgent] βœ… Written: {file_path} ({len(code)} chars)") + return code + + except Exception as e: + if _is_rate_limit(e): + raise RateLimitError(str(e)) + raise + +def _install_dependencies(dependencies: list[str], project_dir: Path) -> str: + if not dependencies: + return "No external dependencies." + + to_install = [] + for dep in dependencies: + pkg_name = re.split(r"[>= bool: + vscode_candidates = [ + "code", + rf"C:\Users\{Path.home().name}\AppData\Local\Programs\Microsoft VS Code\bin\code.cmd", + r"C:\Program Files\Microsoft VS Code\bin\code.cmd", + ] + for cmd in vscode_candidates: + try: + subprocess.Popen( + [cmd, str(project_dir)], + shell=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL + ) + time.sleep(1.5) + print(f"[DevAgent] πŸ’» VSCode opened: {project_dir}") + return True + except Exception: + continue + return False + +def _run_project(run_command: str, project_dir: Path, timeout: int = 30) -> str: + print(f"[DevAgent] πŸš€ Running: {run_command}") + try: + parts = run_command.split() + if parts[0].lower() == "python": + parts[0] = sys.executable + + result = subprocess.run( + parts, + capture_output=True, text=True, + encoding="utf-8", errors="replace", + timeout=timeout, + cwd=str(project_dir) + ) + + stdout = result.stdout.strip() + stderr = result.stderr.strip() + + combined_parts = [] + if stdout: + combined_parts.append(f"STDOUT:\n{stdout}") + if stderr: + combined_parts.append(f"STDERR:\n{stderr}") + + return "\n\n".join(combined_parts) if combined_parts else "Ran with no output." + + except subprocess.TimeoutExpired: + return f"Timed out after {timeout}s β€” long-running app (server/GUI) is likely working." + except FileNotFoundError as e: + return f"Command not found: {e}" + except Exception as e: + return f"Run error: {e}" + +def _try_auto_install(error_output: str, project_dir: Path) -> bool: + """ModuleNotFoundError varsa eksik paketi otomatik kurmaya Γ§alışır.""" + pattern = re.compile( + r"No module named ['\"]([a-zA-Z0-9_\-\.]+)['\"]", re.IGNORECASE + ) + match = pattern.search(error_output) + if not match: + return False + + pkg = match.group(1).replace("_", "-").split(".")[0] + print(f"[DevAgent] πŸ”§ Auto-installing missing package: {pkg}") + try: + result = subprocess.run( + [sys.executable, "-m", "pip", "install", pkg], + capture_output=True, text=True, + encoding="utf-8", errors="replace", + timeout=60, cwd=str(project_dir) + ) + return result.returncode == 0 + except Exception: + return False + +def _fix_files( + error_output: str, + project_description: str, + all_files: list[dict], + file_codes: dict[str, str], + language: str, + project_dir: Path, + entry_point: str, +) -> dict[str, str]: + + model = _get_model(MODEL_PLANNER) + + error_file, error_line = _parse_traceback(error_output, list(file_codes.keys())) + error_type = _classify_error(error_output) + + files_to_fix: list[str] = [] + + if error_file: + files_to_fix.append(error_file) + if error_type == "import_error": + for fi in all_files: + if error_file.replace("/", ".").replace(".py", "") in fi.get("imports", []): + p = fi["path"] + if p not in files_to_fix: + files_to_fix.append(p) + else: + files_to_fix.append(entry_point) + + updated_codes: dict[str, str] = {} + + for fix_path in files_to_fix: + current_code = file_codes.get(fix_path, "") + + other_ctx = "" + for fp, code in file_codes.items(): + if fp != fix_path and code: + snippet = code[:1500] + ("..." if len(code) > 1500 else "") + other_ctx += f"\n--- {fp} ---\n{snippet}\n" + + line_hint = f"\nError appears to be near line {error_line} in this file." if ( + error_line and fix_path == error_file + ) else "" + + prompt = f"""You are an expert {language} debugger. Fix the broken file below. + +Project goal: {project_description} + +All project files: +{chr(10).join(f" - {f['path']}: {f.get('description', '')}" for f in all_files)} + +Other files for context (read-only β€” fix only the target file): +{other_ctx[:3500]} + +File to fix: {fix_path}{line_hint} +Error type: {error_type} + +Error output: +{error_output[:2500]} + +Current (broken) code: +{current_code} + +Rules: +- Output ONLY the complete fixed code. No explanation, no markdown, no backticks. +- Fix ALL errors visible in the error output. +- Keep all existing correct logic β€” do not remove working features. +- Ensure import paths match the actual project file structure exactly. +- Do NOT introduce new bugs or remove error handling. + +Fixed code for {fix_path}:""" + + try: + response = model.generate_content(prompt) + fixed = _strip_fences(response.text) + + full_path = project_dir / fix_path + full_path.parent.mkdir(parents=True, exist_ok=True) + full_path.write_text(fixed, encoding="utf-8") + + updated_codes[fix_path] = fixed + print(f"[DevAgent] πŸ”§ Fixed: {fix_path}") + + except Exception as e: + if _is_rate_limit(e): + raise RateLimitError(str(e)) + print(f"[DevAgent] ⚠️ Could not fix {fix_path}: {e}") + + return updated_codes + +def _build_project( + description: str, + language: str, + project_name: str, + timeout: int, + speak=None, + player=None, +) -> str: + + def log(msg: str): + print(f"[DevAgent] {msg}") + if player: + player.write_log(f"[DevAgent] {msg}") + + log("Planning project structure...") + try: + plan = _plan_project(description, language) + except RateLimitError: + msg = "Rate limit reached, sir. Please try again in a moment." + if speak: speak(msg) + return msg + except ValueError as e: + msg = f"Planning failed: {e}" + if speak: speak(msg) + return msg + + proj_name = project_name or plan.get("project_name", "jarvis_project") + proj_name = re.sub(r"[^\w\-]", "_", proj_name) + project_dir = PROJECTS_DIR / proj_name + project_dir.mkdir(parents=True, exist_ok=True) + + files = plan.get("files", []) + entry_point = plan.get("entry_point", "main.py") + run_command = plan.get("run_command", f"python {entry_point}") + dependencies = plan.get("dependencies", []) + + log(f"Project: {proj_name} | Files: {len(files)} | Entry: {entry_point}") + + def _dep_sort_key(fi: dict) -> int: + return len(fi.get("imports", [])) + + sorted_files = sorted(files, key=_dep_sort_key) + + file_codes: dict[str, str] = {} + + for file_info in sorted_files: + file_path = file_info.get("path", "") + if not file_path: + continue + + log(f"Writing {file_path}...") + for attempt in range(2): + try: + code = _write_file( + file_info=file_info, + project_description=description, + all_files=files, + language=language, + project_dir=project_dir, + already_written=file_codes, + ) + file_codes[file_path] = code + time.sleep(0.4) + break + except RateLimitError: + if attempt == 0: + log("Rate limit β€” waiting 20s...") + time.sleep(20) + else: + log(f"Rate limit retry failed for {file_path}, skipping.") + except Exception as e: + log(f"Failed to write {file_path}: {e}") + break + + if not file_codes: + msg = "I could not write any project files, sir." + if speak: speak(msg) + return msg + + if dependencies: + install_result = _install_dependencies(dependencies, project_dir) + log(install_result) + + _open_vscode(project_dir) + + last_output = "" + auto_installs = 0 + + for attempt in range(1, MAX_FIX_ATTEMPTS + 1): + log(f"Running project (attempt {attempt}/{MAX_FIX_ATTEMPTS})...") + last_output = _run_project(run_command, project_dir, timeout) + log(f"Output preview: {last_output[:150]}") + + if not _has_error(last_output, run_command): + msg = ( + f"Project '{proj_name}' is working, sir. " + f"Built in {attempt} attempt{'s' if attempt > 1 else ''}. " + f"Saved to: {project_dir}" + ) + if speak: speak(msg) + return f"{msg}\n\nOutput:\n{last_output}" + + if attempt == MAX_FIX_ATTEMPTS: + break + + error_type = _classify_error(last_output) + if error_type == "dependency_error" and auto_installs < 3: + installed = _try_auto_install(last_output, project_dir) + if installed: + auto_installs += 1 + log("Missing dependency installed, retrying...") + time.sleep(1) + continue + + log(f"Fixing errors (type: {error_type})...") + try: + updated = _fix_files( + error_output=last_output, + project_description=description, + all_files=files, + file_codes=file_codes, + language=language, + project_dir=project_dir, + entry_point=entry_point, + ) + file_codes.update(updated) + time.sleep(1) + except RateLimitError: + msg = "Rate limit reached during fix. Project saved, check it manually in VSCode." + if speak: speak(msg) + return msg + except Exception as e: + log(f"Fix step failed: {e}") + + msg = ( + f"I couldn't fully fix '{proj_name}' after {MAX_FIX_ATTEMPTS} attempts, sir. " + f"Project is saved at {project_dir} β€” open it in VSCode and check manually." + ) + if speak: speak(msg) + return f"{msg}\n\nLast error:\n{last_output[:600]}" + + +def dev_agent( + parameters: dict, + response=None, + player=None, + session_memory=None, + speak=None, +) -> str: + p = parameters or {} + description = p.get("description", "").strip() + language = p.get("language", "python").strip() + project_name = p.get("project_name", "").strip() + timeout = int(p.get("timeout", 30)) + + if not description: + return "Please describe the project you want me to build, sir." + + return _build_project( + description = description, + language = language, + project_name = project_name, + timeout = timeout, + speak = speak, + player = player, + ) \ No newline at end of file diff --git a/actions/file_controller.py b/actions/file_controller.py new file mode 100644 index 0000000..7750076 --- /dev/null +++ b/actions/file_controller.py @@ -0,0 +1,482 @@ +# actions/file_controller.py +# File management β€” create, delete, move, rename, list, find, organize + +import shutil +from pathlib import Path +from datetime import datetime +import send2trash + +def _get_desktop() -> Path: + """Returns desktop path β€” works on Windows, Mac, Linux.""" + return Path.home() / "Desktop" + + +def _get_downloads() -> Path: + return Path.home() / "Downloads" + + +def _resolve_path(raw: str) -> Path: + """ + Resolves a path from user input. + Supports shortcuts: 'desktop', 'downloads', 'documents', 'home' + """ + shortcuts = { + "desktop": Path.home() / "Desktop", + "downloads": Path.home() / "Downloads", + "documents": Path.home() / "Documents", + "pictures": Path.home() / "Pictures", + "music": Path.home() / "Music", + "videos": Path.home() / "Videos", + "home": Path.home(), + } + lower = raw.strip().lower() + if lower in shortcuts: + return shortcuts[lower] + return Path(raw).expanduser() + + +def _format_size(bytes_size: int) -> str: + """Converts bytes to human readable format.""" + for unit in ["B", "KB", "MB", "GB", "TB"]: + if bytes_size < 1024: + return f"{bytes_size:.1f} {unit}" + bytes_size /= 1024 + return f"{bytes_size:.1f} TB" + + +def list_files(path: str = "desktop", show_hidden: bool = False) -> str: + """Lists files and folders in a directory.""" + try: + target = _resolve_path(path) + if not target.exists(): + return f"Path not found: {target}" + if not target.is_dir(): + return f"Not a directory: {target}" + + items = [] + for item in sorted(target.iterdir()): + if not show_hidden and item.name.startswith("."): + continue + if item.is_dir(): + items.append(f"πŸ“ {item.name}/") + else: + size = _format_size(item.stat().st_size) + items.append(f"πŸ“„ {item.name} ({size})") + + if not items: + return f"Directory is empty: {target}" + + return f"Contents of {target.name}/ ({len(items)} items):\n" + "\n".join(items) + + except PermissionError: + return f"Permission denied: {path}" + except Exception as e: + return f"Error listing files: {e}" + + +def create_file(path: str, content: str = "") -> str: + """Creates a new file with optional content.""" + try: + target = Path(path).expanduser() + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(content, encoding="utf-8") + return f"File created: {target.name}" + except Exception as e: + return f"Could not create file: {e}" + + +def create_folder(path: str) -> str: + """Creates a new folder (and parent folders if needed).""" + try: + target = Path(path).expanduser() + target.mkdir(parents=True, exist_ok=True) + return f"Folder created: {target}" + except Exception as e: + return f"Could not create folder: {e}" + + +def delete_file(path: str, confirm: bool = True) -> str: + """ + Deletes a file or folder. + Moves to Recycle Bin on Windows if possible, otherwise permanent delete. + """ + try: + target = Path(path).expanduser() + if not target.exists(): + return f"Not found: {path}" + + try: + + send2trash.send2trash(str(target)) + return f"Moved to Recycle Bin: {target.name}" + except ImportError: + pass + + # Fallback: permanent delete + if target.is_dir(): + shutil.rmtree(target) + return f"Folder deleted permanently: {target.name}" + else: + target.unlink() + return f"File deleted permanently: {target.name}" + + except PermissionError: + return f"Permission denied: {path}" + except Exception as e: + return f"Could not delete: {e}" + + +def move_file(source: str, destination: str) -> str: + """Moves a file or folder to a new location.""" + try: + src = Path(source).expanduser() + dst = _resolve_path(destination) + + if not src.exists(): + return f"Source not found: {source}" + + if dst.is_dir(): + dst = dst / src.name + + dst.parent.mkdir(parents=True, exist_ok=True) + shutil.move(str(src), str(dst)) + return f"Moved: {src.name} β†’ {dst.parent.name}/" + + except Exception as e: + return f"Could not move: {e}" + + +def copy_file(source: str, destination: str) -> str: + """Copies a file or folder.""" + try: + src = Path(source).expanduser() + dst = _resolve_path(destination) + + if not src.exists(): + return f"Source not found: {source}" + + if dst.is_dir(): + dst = dst / src.name + + dst.parent.mkdir(parents=True, exist_ok=True) + + if src.is_dir(): + shutil.copytree(str(src), str(dst)) + else: + shutil.copy2(str(src), str(dst)) + + return f"Copied: {src.name} β†’ {dst.parent.name}/" + + except Exception as e: + return f"Could not copy: {e}" + + +def rename_file(path: str, new_name: str) -> str: + """Renames a file or folder.""" + try: + target = Path(path).expanduser() + new_path = target.parent / new_name + + if not target.exists(): + return f"Not found: {path}" + if new_path.exists(): + return f"A file named '{new_name}' already exists." + + target.rename(new_path) + return f"Renamed: {target.name} β†’ {new_name}" + + except Exception as e: + return f"Could not rename: {e}" + + +def read_file(path: str, max_chars: int = 3000) -> str: + """Reads and returns the content of a text file.""" + try: + target = Path(path).expanduser() + if not target.exists(): + return f"File not found: {path}" + if not target.is_file(): + return f"Not a file: {path}" + + content = target.read_text(encoding="utf-8", errors="ignore") + if len(content) > max_chars: + content = content[:max_chars] + f"\n\n... (truncated, {len(content)} total chars)" + return content + + except Exception as e: + return f"Could not read file: {e}" + + +def write_file(path: str, content: str, append: bool = False) -> str: + """Writes or appends content to a file.""" + try: + target = Path(path).expanduser() + target.parent.mkdir(parents=True, exist_ok=True) + mode = "a" if append else "w" + with open(target, mode, encoding="utf-8") as f: + f.write(content) + action = "Appended to" if append else "Written to" + return f"{action}: {target.name}" + except Exception as e: + return f"Could not write file: {e}" + + +def find_files(name: str = "", extension: str = "", path: str = "home", + max_results: int = 20) -> str: + """ + Searches for files by name or extension. + Example: find_files(extension=".pdf", path="documents") + """ + try: + search_path = _resolve_path(path) + if not search_path.exists(): + return f"Search path not found: {path}" + + results = [] + pattern = f"*{extension}" if extension else "*" + + for item in search_path.rglob(pattern): + if item.is_file(): + if name and name.lower() not in item.name.lower(): + continue + size = _format_size(item.stat().st_size) + results.append(f"πŸ“„ {item.name} ({size}) β€” {item.parent}") + if len(results) >= max_results: + break + + if not results: + query = name or extension or "files" + return f"No {query} found in {search_path.name}/" + + return f"Found {len(results)} file(s):\n" + "\n".join(results) + + except Exception as e: + return f"Search error: {e}" + + +def get_largest_files(path: str = "home", count: int = 10) -> str: + """Returns the largest files in a directory.""" + try: + search_path = _resolve_path(path) + if not search_path.exists(): + return f"Path not found: {path}" + + files = [] + for item in search_path.rglob("*"): + if item.is_file(): + try: + files.append((item.stat().st_size, item)) + except Exception: + continue + + files.sort(reverse=True) + top = files[:count] + + if not top: + return "No files found." + + lines = [f"Top {len(top)} largest files in {search_path.name}/:\n"] + for size, f in top: + lines.append(f" {_format_size(size):>10} {f.name} ({f.parent})") + + return "\n".join(lines) + + except Exception as e: + return f"Error: {e}" + + +def get_disk_usage(path: str = "home") -> str: + """Returns disk usage information.""" + try: + target = _resolve_path(path) + usage = shutil.disk_usage(target) + total = _format_size(usage.total) + used = _format_size(usage.used) + free = _format_size(usage.free) + pct = usage.used / usage.total * 100 + + return ( + f"Disk usage for {target}:\n" + f" Total : {total}\n" + f" Used : {used} ({pct:.1f}%)\n" + f" Free : {free}" + ) + except Exception as e: + return f"Could not get disk usage: {e}" + + +def organize_desktop() -> str: + """ + Organizes the desktop by grouping files into folders by type. + Creates folders: Images, Documents, Videos, Music, Archives, Others + """ + try: + desktop = _get_desktop() + type_map = { + "Images": [".jpg", ".jpeg", ".png", ".gif", ".bmp", ".webp", ".svg", ".ico"], + "Documents": [".pdf", ".doc", ".docx", ".txt", ".xls", ".xlsx", ".ppt", ".pptx", ".csv"], + "Videos": [".mp4", ".avi", ".mkv", ".mov", ".wmv", ".flv", ".webm"], + "Music": [".mp3", ".wav", ".flac", ".aac", ".ogg", ".wma"], + "Archives": [".zip", ".rar", ".7z", ".tar", ".gz"], + "Code": [".py", ".js", ".html", ".css", ".json", ".xml", ".ts", ".cpp", ".java"], + } + + moved = [] + skipped = [] + + for item in desktop.iterdir(): + + if item.is_dir() or item.name.startswith("."): + continue + + ext = item.suffix.lower() + target_dir = None + + for folder, extensions in type_map.items(): + if ext in extensions: + target_dir = desktop / folder + break + + if target_dir is None: + target_dir = desktop / "Others" + + target_dir.mkdir(exist_ok=True) + new_path = target_dir / item.name + + if new_path.exists(): + skipped.append(item.name) + continue + + shutil.move(str(item), str(new_path)) + moved.append(f"{item.name} β†’ {target_dir.name}/") + + result = f"Desktop organized. {len(moved)} files moved." + if moved: + result += "\n" + "\n".join(moved[:10]) + if len(moved) > 10: + result += f"\n... and {len(moved)-10} more." + if skipped: + result += f"\n{len(skipped)} files skipped (already exist)." + + return result + + except Exception as e: + return f"Could not organize desktop: {e}" + + +def get_file_info(path: str) -> str: + """Returns detailed information about a file.""" + try: + target = Path(path).expanduser() + if not target.exists(): + return f"Not found: {path}" + + stat = target.stat() + info = { + "Name": target.name, + "Type": "Folder" if target.is_dir() else "File", + "Size": _format_size(stat.st_size), + "Location": str(target.parent), + "Created": datetime.fromtimestamp(stat.st_ctime).strftime("%Y-%m-%d %H:%M"), + "Modified": datetime.fromtimestamp(stat.st_mtime).strftime("%Y-%m-%d %H:%M"), + "Extension": target.suffix or "None", + } + + return "\n".join(f" {k}: {v}" for k, v in info.items()) + + except Exception as e: + return f"Could not get file info: {e}" + +def file_controller( + parameters: dict, + response=None, + player=None, + session_memory=None +) -> str: + action = (parameters or {}).get("action", "").lower().strip() + path = (parameters or {}).get("path", "desktop") + name = (parameters or {}).get("name", "") + content = (parameters or {}).get("content", "") + + def _full_path(p: str, n: str) -> str: + base = _resolve_path(p) + if n: + return str(base / n) + return str(base) + + result = "Unknown action." + + try: + if action == "list": + result = list_files(path) + + elif action == "create_file": + full = _full_path(path, name) + result = create_file(full, content=content) + + elif action == "create_folder": + full = _full_path(path, name) + result = create_folder(full) + + elif action == "delete": + full = _full_path(path, name) + result = delete_file(full) + + elif action == "move": + full = _full_path(path, name) + result = move_file(full, parameters.get("destination", "")) + + elif action == "copy": + full = _full_path(path, name) + result = copy_file(full, parameters.get("destination", "")) + + elif action == "rename": + full = _full_path(path, name) + result = rename_file(full, parameters.get("new_name", "")) + + elif action == "read": + full = _full_path(path, name) + result = read_file(full) + + elif action == "write": + full = _full_path(path, name) + result = write_file( + full, + content=content, + append=parameters.get("append", False) + ) + + elif action == "find": + result = find_files( + name=name or parameters.get("name", ""), + extension=parameters.get("extension", ""), + path=path, + max_results=parameters.get("max_results", 20) + ) + + elif action == "largest": + result = get_largest_files( + path=path, + count=parameters.get("count", 10) + ) + + elif action == "disk_usage": + result = get_disk_usage(path) + + elif action == "organize_desktop": + result = organize_desktop() + + elif action == "info": + full = _full_path(path, name) + result = get_file_info(full) + + else: + result = f"Unknown action: '{action}'" + + except Exception as e: + result = f"File controller error: {e}" + + if player: + player.write_log(f"[file] {result[:60]}") + + return result \ No newline at end of file diff --git a/actions/file_processor.py b/actions/file_processor.py new file mode 100644 index 0000000..df5b811 --- /dev/null +++ b/actions/file_processor.py @@ -0,0 +1,833 @@ +""" +file_processor.py β€” JARVIS Universal File Processor + +Supported types: + image β†’ describe, ocr, resize, convert, compress, crop + pdf β†’ summarize, extract_text, extract_pages, to_word + docx β†’ summarize, extract_text, reformat, translate_hint + txt/md β†’ summarize, reformat, translate_hint, word_count + csv β†’ analyze, filter, sort, convert, stats + xlsx β†’ analyze, filter, convert, stats + json β†’ validate, format, extract, convert + code β†’ explain, review, fix, run, document + audio β†’ transcribe, trim, convert, info + video β†’ trim, extract_audio, extract_frame, info, compress + zip β†’ list, extract + pptx β†’ summarize, extract_text, to_pdf +""" + +import os +import re +import json +import shutil +import subprocess +import tempfile +from pathlib import Path +from datetime import datetime + +import google.generativeai as genai + + +def _get_api_key() -> str: + config_path = Path(__file__).resolve().parent.parent / "config" / "api_keys.json" + with open(config_path, "r", encoding="utf-8") as f: + return json.load(f)["gemini_api_key"] + + +def _gemini_client(): + genai.configure(api_key=_get_api_key()) + return genai.GenerativeModel("gemini-2.5-flash") + + +def _detect_type(path: Path) -> str: + ext = path.suffix.lower().lstrip(".") + image_exts = {"jpg", "jpeg", "png", "gif", "webp", "bmp", "tiff", "svg", "ico"} + video_exts = {"mp4", "avi", "mov", "mkv", "wmv", "flv", "webm", "m4v", "3gp"} + audio_exts = {"mp3", "wav", "ogg", "m4a", "aac", "flac", "wma", "opus"} + code_exts = {"py", "js", "ts", "jsx", "tsx", "html", "css", "java", "c", + "cpp", "cs", "go", "rs", "rb", "php", "swift", "kt", "sh", + "bash", "ps1", "lua", "r", "m", "sql", "yaml", "toml"} + archive_exts = {"zip", "rar", "tar", "gz", "7z", "bz2", "xz"} + + if ext in image_exts: return "image" + if ext in video_exts: return "video" + if ext in audio_exts: return "audio" + if ext in code_exts: return "code" + if ext in archive_exts: return "archive" + if ext == "pdf": return "pdf" + if ext in ("docx", "doc"): return "docx" + if ext in ("txt", "md", "rst", "log"): return "text" + if ext in ("csv", "tsv"): return "csv" + if ext in ("xlsx", "xls", "ods"): return "excel" + if ext == "json": return "json" + if ext == "xml": return "xml" + if ext in ("pptx", "ppt"): return "pptx" + return "unknown" + + +def _file_size_str(path: Path) -> str: + size = path.stat().st_size + if size < 1024: return f"{size} B" + if size < 1024**2: return f"{size/1024:.1f} KB" + if size < 1024**3: return f"{size/1024**2:.1f} MB" + return f"{size/1024**3:.1f} GB" + +def _output_path(src: Path, suffix: str, new_ext: str = None) -> Path: + ext = new_ext or src.suffix + name = f"{src.stem}_{suffix}{ext}" + return src.parent / name + +def _process_image(path: Path, action: str, params: dict, speak=None) -> str: + try: + from PIL import Image + except ImportError: + return "Pillow is not installed. Run: pip install Pillow" + + action = action or "describe" + + if action in ("describe", "ocr", "analyze", "read", "extract_text"): + try: + model = _gemini_client() + img = Image.open(path) + prompt = { + "describe": "Describe this image in detail.", + "ocr": "Extract all text visible in this image. Return only the text, formatted clearly.", + "analyze": "Analyze this image thoroughly: objects, colors, composition, any text, context.", + "read": "Read all text in this image, preserving structure and formatting.", + "extract_text": "Extract all text from this image.", + }.get(action, "Describe this image.") + + if params.get("instruction"): + prompt = params["instruction"] + + response = model.generate_content([prompt, img]) + result = response.text.strip() + + if len(result) > 500 and params.get("save", True): + out = _output_path(path, "result", ".txt") + out.write_text(result, encoding="utf-8") + return f"{result[:300]}...\n\nFull result saved to: {out}" + return result + except Exception as e: + return f"AI image analysis failed: {e}" + + if action == "resize": + width = int(params.get("width", 0)) + height = int(params.get("height", 0)) + scale = float(params.get("scale", 0)) + try: + img = Image.open(path) + w, h = img.size + if scale: + new_size = (int(w * scale), int(h * scale)) + elif width and height: + new_size = (width, height) + elif width: + new_size = (width, int(h * width / w)) + elif height: + new_size = (int(w * height / h), height) + else: + return "Please specify width, height, or scale." + out = _output_path(path, f"resized_{new_size[0]}x{new_size[1]}") + img.resize(new_size, Image.LANCZOS).save(out) + return f"Resized from {w}x{h} to {new_size[0]}x{new_size[1]}. Saved: {out.name}" + except Exception as e: + return f"Resize failed: {e}" + + if action == "convert": + fmt = params.get("format", "png").lower().strip(".") + fmt_map = {"jpg": "JPEG", "jpeg": "JPEG", "png": "PNG", + "webp": "WEBP", "bmp": "BMP", "tiff": "TIFF"} + pil_fmt = fmt_map.get(fmt, fmt.upper()) + try: + img = Image.open(path).convert("RGB") if fmt == "jpg" else Image.open(path) + out = _output_path(path, "converted", f".{fmt}") + img.save(out, pil_fmt) + return f"Converted to {fmt.upper()}. Saved: {out.name}" + except Exception as e: + return f"Convert failed: {e}" + + if action == "compress": + quality = int(params.get("quality", 70)) + try: + img = Image.open(path).convert("RGB") + out = _output_path(path, f"compressed_q{quality}", ".jpg") + img.save(out, "JPEG", quality=quality, optimize=True) + before = _file_size_str(path) + after = _file_size_str(out) + return f"Compressed: {before} β†’ {after}. Saved: {out.name}" + except Exception as e: + return f"Compress failed: {e}" + + if action == "info": + try: + img = Image.open(path) + return (f"Image info: {img.format}, {img.size[0]}x{img.size[1]}px, " + f"mode: {img.mode}, size: {_file_size_str(path)}") + except Exception as e: + return f"Info failed: {e}" + + return _process_image(path, "describe", {"instruction": f"{action}: {params}"}) + +def _process_pdf(path: Path, action: str, params: dict, speak=None) -> str: + action = action or "summarize" + + def _extract_pdf_text(max_chars=50000) -> str: + text = "" + try: + import pdfplumber + with pdfplumber.open(path) as pdf: + for page in pdf.pages: + text += (page.extract_text() or "") + "\n" + except ImportError: + try: + import PyPDF2 + with open(path, "rb") as f: + reader = PyPDF2.PdfReader(f) + for page in reader.pages: + text += page.extract_text() + "\n" + except ImportError: + return "" + return text[:max_chars] + + if action in ("summarize", "extract_text", "translate_hint", "analyze", "reformat"): + text = _extract_pdf_text() + if not text.strip(): + return "Could not extract text from PDF (may be scanned/image-based)." + + if action == "extract_text": + out = _output_path(path, "text", ".txt") + out.write_text(text, encoding="utf-8") + return f"Text extracted ({len(text)} chars). Saved: {out.name}" + + prompt_map = { + "summarize": f"Summarize this PDF document concisely:\n\n{text}", + "analyze": f"Analyze this document thoroughly:\n\n{text}", + "translate_hint": f"What language is this document in and what does it say? Summarize:\n\n{text}", + "reformat": f"Reformat this text cleanly with proper structure:\n\n{text}", + } + try: + model = _gemini_client() + response = model.generate_content(prompt_map.get(action, f"Analyze:\n\n{text}")) + result = response.text.strip() + if len(result) > 600 and params.get("save", True): + out = _output_path(path, action, ".txt") + out.write_text(result, encoding="utf-8") + return f"{result[:400]}...\n\nFull result saved: {out.name}" + return result + except Exception as e: + return f"AI analysis failed: {e}" + + if action == "info": + try: + import pdfplumber + with pdfplumber.open(path) as pdf: + pages = len(pdf.pages) + return f"PDF: {pages} pages, size: {_file_size_str(path)}" + except Exception: + return f"PDF size: {_file_size_str(path)}" + + if action == "to_word": + text = _extract_pdf_text() + if not text: + return "Could not extract text to convert." + try: + from docx import Document + doc = Document() + doc.add_heading(path.stem, 0) + for para in text.split("\n\n"): + if para.strip(): + doc.add_paragraph(para.strip()) + out = _output_path(path, "converted", ".docx") + doc.save(out) + return f"Converted to Word document. Saved: {out.name}" + except ImportError: + return "python-docx not installed. Run: pip install python-docx" + + return f"Unknown PDF action: '{action}'. Try: summarize, extract_text, info, to_word" + +def _process_text_doc(path: Path, file_type: str, action: str, + params: dict, speak=None) -> str: + action = action or "summarize" + + def _read_content() -> str: + if file_type == "docx": + try: + from docx import Document + doc = Document(path) + return "\n".join(p.text for p in doc.paragraphs) + except ImportError: + return "python-docx not installed." + except Exception as e: + return f"Read failed: {e}" + else: + return path.read_text(encoding="utf-8", errors="ignore") + + content = _read_content() + if not content.strip(): + return "File appears to be empty." + + if action == "word_count": + words = len(content.split()) + chars = len(content) + lines = content.count("\n") + return f"Word count: {words} words, {chars} characters, {lines} lines." + + if action == "extract_text": + if file_type != "txt": + out = _output_path(path, "extracted", ".txt") + out.write_text(content, encoding="utf-8") + return f"Text extracted. Saved: {out.name}" + return content[:2000] + + instruction = params.get("instruction", "") + prompt_map = { + "summarize": f"Summarize this document concisely:\n\n{content[:40000]}", + "analyze": f"Analyze this document:\n\n{content[:40000]}", + "reformat": f"Reformat this text with clean structure, proper headings and paragraphs:\n\n{content[:40000]}", + "fix": f"Fix grammar, spelling and style issues in this text:\n\n{content[:40000]}", + "translate_hint": f"What language is this and what does it say? Summarize:\n\n{content[:10000]}", + "to_bullet": f"Convert this text into a clear bullet-point summary:\n\n{content[:40000]}", + "custom": f"{instruction}\n\n{content[:40000]}", + } + + if action not in prompt_map: + + action = "custom" + instruction = action + + try: + model = _gemini_client() + response = model.generate_content(prompt_map[action]) + result = response.text.strip() + if len(result) > 600 and params.get("save", True): + out = _output_path(path, action, ".txt") + out.write_text(result, encoding="utf-8") + return f"{result[:400]}...\n\nFull result saved: {out.name}" + return result + except Exception as e: + return f"AI processing failed: {e}" + + +def _process_data(path: Path, file_type: str, action: str, + params: dict, speak=None) -> str: + try: + import pandas as pd + except ImportError: + return "pandas not installed. Run: pip install pandas openpyxl" + + action = action or "analyze" + + try: + if file_type == "csv": + df = pd.read_csv(path, encoding="utf-8", errors="replace") + else: + df = pd.read_excel(path) + except Exception as e: + return f"Could not read file: {e}" + + if action == "info": + return (f"Rows: {len(df)}, Columns: {len(df.columns)}\n" + f"Columns: {', '.join(df.columns.tolist())}\n" + f"Size: {_file_size_str(path)}") + + if action == "stats": + try: + desc = df.describe(include="all").to_string() + return f"Statistics:\n{desc[:2000]}" + except Exception as e: + return f"Stats failed: {e}" + + if action == "analyze": + preview = df.head(50).to_string() + prompt = (f"Analyze this dataset. Columns: {list(df.columns)}\n" + f"Rows: {len(df)}\nPreview:\n{preview}\n\n" + f"Give insights, patterns, and notable findings.") + try: + model = _gemini_client() + response = model.generate_content(prompt) + return response.text.strip() + except Exception as e: + return f"AI analysis failed: {e}" + + if action in ("convert", "to_csv", "to_excel", "to_json"): + fmt = {"to_csv": "csv", "to_excel": "xlsx", "to_json": "json", + "convert": params.get("format", "csv")}.get(action, "csv") + try: + if fmt == "csv": + out = _output_path(path, "converted", ".csv") + df.to_csv(out, index=False, encoding="utf-8") + elif fmt == "xlsx": + out = _output_path(path, "converted", ".xlsx") + df.to_excel(out, index=False) + elif fmt == "json": + out = _output_path(path, "converted", ".json") + df.to_json(out, orient="records", force_ascii=False, indent=2) + return f"Converted to {fmt.upper()}. Saved: {out.name}" + except Exception as e: + return f"Convert failed: {e}" + + if action == "filter": + col = params.get("column", "") + value = params.get("value", "") + condition = params.get("condition", "equals") + if not col or col not in df.columns: + return f"Column '{col}' not found. Available: {', '.join(df.columns)}" + try: + if condition == "equals": filtered = df[df[col] == value] + elif condition == "contains": filtered = df[df[col].astype(str).str.contains(str(value), case=False)] + elif condition == "gt": filtered = df[df[col] > float(value)] + elif condition == "lt": filtered = df[df[col] < float(value)] + else: filtered = df[df[col] == value] + out = _output_path(path, "filtered", ".csv") + filtered.to_csv(out, index=False) + return f"Filtered: {len(filtered)} rows match. Saved: {out.name}" + except Exception as e: + return f"Filter failed: {e}" + + if action == "sort": + col = params.get("column", df.columns[0]) + asc = params.get("ascending", True) + try: + sorted_df = df.sort_values(col, ascending=asc) + out = _output_path(path, "sorted", path.suffix) + sorted_df.to_csv(out, index=False) + return f"Sorted by '{col}'. Saved: {out.name}" + except Exception as e: + return f"Sort failed: {e}" + + preview = df.head(30).to_string() + try: + model = _gemini_client() + response = model.generate_content( + f"Task: {action}\nDataset ({len(df)} rows, cols: {list(df.columns)}):\n{preview}" + ) + return response.text.strip() + except Exception as e: + return f"Processing failed: {e}" + + +def _process_json(path: Path, action: str, params: dict, speak=None) -> str: + action = action or "analyze" + try: + content = path.read_text(encoding="utf-8") + data = json.loads(content) + except Exception as e: + return f"Invalid JSON: {e}" + + if action == "validate": + return f"Valid JSON. Type: {type(data).__name__}, size: {_file_size_str(path)}" + + if action == "format": + out = _output_path(path, "formatted", ".json") + out.write_text(json.dumps(data, indent=2, ensure_ascii=False), encoding="utf-8") + return f"Formatted JSON saved: {out.name}" + + if action in ("analyze", "summarize", "extract"): + preview = json.dumps(data, indent=2, ensure_ascii=False)[:8000] + prompt = f"Task: {action} this JSON data:\n{preview}" + if params.get("instruction"): + prompt = f"{params['instruction']}\n\nJSON data:\n{preview}" + try: + model = _gemini_client() + response = model.generate_content(prompt) + return response.text.strip() + except Exception as e: + return f"AI processing failed: {e}" + + if action == "to_csv": + try: + import pandas as pd + if isinstance(data, list): + df = pd.DataFrame(data) + out = _output_path(path, "converted", ".csv") + df.to_csv(out, index=False) + return f"Converted to CSV. Saved: {out.name}" + return "JSON must be an array of objects to convert to CSV." + except ImportError: + return "pandas not installed." + + return _process_json(path, "analyze", {"instruction": action}) + +def _process_code(path: Path, action: str, params: dict, speak=None) -> str: + action = action or "explain" + content = path.read_text(encoding="utf-8", errors="ignore") + ext = path.suffix.lstrip(".") + + if action == "run": + if ext == "py": + try: + result = subprocess.run( + ["python", str(path)], + capture_output=True, text=True, timeout=30 + ) + out = result.stdout or result.stderr + return f"Output:\n{out[:2000]}" if out else "No output." + except subprocess.TimeoutExpired: + return "Execution timed out (30s)." + except Exception as e: + return f"Run failed: {e}" + return f"Direct execution not supported for .{ext} files." + + if action == "info": + lines = content.count("\n") + words = len(content.split()) + return f"Code file: {lines} lines, {words} words, {_file_size_str(path)}" + + prompt_map = { + "explain": f"Explain this {ext} code clearly:\n\n```{ext}\n{content[:30000]}\n```", + "review": f"Review this {ext} code for bugs, issues, and improvements:\n\n```{ext}\n{content[:30000]}\n```", + "fix": f"Fix any bugs in this {ext} code and return the corrected version:\n\n```{ext}\n{content[:30000]}\n```", + "optimize": f"Optimize this {ext} code for performance and readability:\n\n```{ext}\n{content[:30000]}\n```", + "document": f"Add proper documentation/comments to this {ext} code:\n\n```{ext}\n{content[:30000]}\n```", + "summarize": f"Summarize what this {ext} code does:\n\n```{ext}\n{content[:30000]}\n```", + "test": f"Write unit tests for this {ext} code:\n\n```{ext}\n{content[:30000]}\n```", + } + + instruction = params.get("instruction", "") + if action not in prompt_map: + prompt = f"{action}\n\n```{ext}\n{content[:30000]}\n```" + if instruction: + prompt = f"{instruction}\n\n```{ext}\n{content[:30000]}\n```" + else: + prompt = prompt_map[action] + + try: + model = _gemini_client() + response = model.generate_content(prompt) + result = response.text.strip() + + if action in ("fix", "optimize", "document") and params.get("save", True): + out = _output_path(path, action) + code_match = re.search(r"```(?:\w+)?\n(.*?)```", result, re.DOTALL) + code_to_save = code_match.group(1) if code_match else result + out.write_text(code_to_save, encoding="utf-8") + return f"{result[:400]}...\n\nSaved: {out.name}" + return result + except Exception as e: + return f"AI processing failed: {e}" + +def _process_audio(path: Path, action: str, params: dict, speak=None) -> str: + action = action or "transcribe" + + if action == "info": + try: + from pydub import AudioSegment + audio = AudioSegment.from_file(path) + duration = len(audio) / 1000 + mins, secs = divmod(int(duration), 60) + return (f"Audio: {mins}m {secs}s, " + f"{audio.channels} ch, " + f"{audio.frame_rate}Hz, " + f"{_file_size_str(path)}") + except ImportError: + return f"Audio file: {_file_size_str(path)} (install pydub for more info)" + except Exception as e: + return f"Info failed: {e}" + + if action == "transcribe": + try: + model = _gemini_client() + content = path.read_bytes() + mime = { + "mp3": "audio/mp3", "wav": "audio/wav", + "ogg": "audio/ogg", "m4a": "audio/mp4", + "aac": "audio/aac", "flac": "audio/flac", + }.get(path.suffix.lstrip(".").lower(), "audio/mpeg") + response = model.generate_content([ + "Transcribe all speech in this audio file accurately.", + {"mime_type": mime, "data": content} + ]) + result = response.text.strip() + if params.get("save", True): + out = _output_path(path, "transcript", ".txt") + out.write_text(result, encoding="utf-8") + return f"Transcription saved: {out.name}\n\nPreview: {result[:300]}" + return result + except Exception as e: + return f"Transcription failed: {e}" + + if action == "convert": + fmt = params.get("format", "mp3").lstrip(".") + try: + from pydub import AudioSegment + audio = AudioSegment.from_file(path) + out = _output_path(path, "converted", f".{fmt}") + audio.export(out, format=fmt) + return f"Converted to {fmt.upper()}. Saved: {out.name}" + except ImportError: + return "pydub not installed. Run: pip install pydub" + except Exception as e: + return f"Convert failed: {e}" + + if action == "trim": + start = float(params.get("start", 0)) + end = float(params.get("end", 0)) + try: + from pydub import AudioSegment + audio = AudioSegment.from_file(path) + end_ms = int(end * 1000) if end else len(audio) + trimmed = audio[int(start * 1000):end_ms] + out = _output_path(path, f"trim_{int(start)}s_{int(end)}s") + trimmed.export(out, format=path.suffix.lstrip(".")) + return f"Trimmed audio ({int(start)}s–{int(end)}s). Saved: {out.name}" + except ImportError: + return "pydub not installed." + except Exception as e: + return f"Trim failed: {e}" + + return f"Unknown audio action: '{action}'. Try: transcribe, info, convert, trim" + +def _process_video(path: Path, action: str, params: dict, speak=None) -> str: + action = action or "info" + + + def _ffmpeg_available() -> bool: + try: + subprocess.run(["ffmpeg", "-version"], capture_output=True, timeout=3) + return True + except Exception: + return False + + if action == "info": + try: + result = subprocess.run( + ["ffprobe", "-v", "quiet", "-print_format", "json", + "-show_format", "-show_streams", str(path)], + capture_output=True, text=True, timeout=10 + ) + data = json.loads(result.stdout) + fmt = data.get("format", {}) + duration = float(fmt.get("duration", 0)) + mins, secs = divmod(int(duration), 60) + size = _file_size_str(path) + streams = data.get("streams", []) + video_s = next((s for s in streams if s["codec_type"] == "video"), {}) + w = video_s.get("width", "?") + h = video_s.get("height", "?") + fps = video_s.get("r_frame_rate", "?") + return f"Video: {mins}m {secs}s, {w}x{h}, {fps} fps, {size}" + except Exception: + return f"Video file: {_file_size_str(path)}" + + if action == "extract_audio": + if not _ffmpeg_available(): + return "ffmpeg not found. Install ffmpeg to extract audio." + out = _output_path(path, "audio", ".mp3") + try: + subprocess.run( + ["ffmpeg", "-i", str(path), "-q:a", "0", "-map", "a", str(out), "-y"], + capture_output=True, timeout=300 + ) + return f"Audio extracted. Saved: {out.name}" + except Exception as e: + return f"Extract audio failed: {e}" + + if action == "trim": + start = params.get("start", "00:00:00") + end = params.get("end", "") + if not _ffmpeg_available(): + return "ffmpeg not found." + out = _output_path(path, f"trim", path.suffix) + try: + cmd = ["ffmpeg", "-i", str(path), "-ss", str(start)] + if end: + cmd += ["-to", str(end)] + cmd += ["-c", "copy", str(out), "-y"] + subprocess.run(cmd, capture_output=True, timeout=600) + return f"Trimmed video saved: {out.name}" + except Exception as e: + return f"Trim failed: {e}" + + if action == "extract_frame": + timestamp = params.get("timestamp", "00:00:01") + if not _ffmpeg_available(): + return "ffmpeg not found." + out = _output_path(path, f"frame_{timestamp.replace(':', '')}", ".jpg") + try: + subprocess.run( + ["ffmpeg", "-i", str(path), "-ss", timestamp, + "-vframes", "1", str(out), "-y"], + capture_output=True, timeout=30 + ) + return f"Frame extracted at {timestamp}. Saved: {out.name}" + except Exception as e: + return f"Extract frame failed: {e}" + + if action == "compress": + crf = int(params.get("quality", 28)) + if not _ffmpeg_available(): + return "ffmpeg not found." + out = _output_path(path, f"compressed_crf{crf}", ".mp4") + try: + subprocess.run( + ["ffmpeg", "-i", str(path), + "-c:v", "libx264", "-crf", str(crf), + "-preset", "medium", "-c:a", "copy", + str(out), "-y"], + capture_output=True, timeout=1800 + ) + before = _file_size_str(path) + after = _file_size_str(out) + return f"Compressed: {before} β†’ {after}. Saved: {out.name}" + except Exception as e: + return f"Compress failed: {e}" + + if action == "transcribe": + if not _ffmpeg_available(): + return "ffmpeg not found. Needed for video transcription." + tmp_audio = Path(tempfile.mktemp(suffix=".mp3")) + try: + subprocess.run( + ["ffmpeg", "-i", str(path), "-q:a", "0", "-map", "a", + str(tmp_audio), "-y"], + capture_output=True, timeout=300 + ) + result = _process_audio(tmp_audio, "transcribe", params, speak) + return result + except Exception as e: + return f"Video transcription failed: {e}" + finally: + if tmp_audio.exists(): + tmp_audio.unlink() + + if action == "convert": + fmt = params.get("format", "mp4").lstrip(".") + if not _ffmpeg_available(): + return "ffmpeg not found." + out = _output_path(path, "converted", f".{fmt}") + try: + subprocess.run( + ["ffmpeg", "-i", str(path), str(out), "-y"], + capture_output=True, timeout=1800 + ) + return f"Converted to {fmt.upper()}. Saved: {out.name}" + except Exception as e: + return f"Convert failed: {e}" + + return f"Unknown video action: '{action}'. Try: info, trim, extract_audio, extract_frame, compress, transcribe, convert" + +def _process_archive(path: Path, action: str, params: dict, speak=None) -> str: + action = action or "list" + + if action == "list": + try: + import zipfile, tarfile + ext = path.suffix.lower() + if ext == ".zip": + with zipfile.ZipFile(path) as z: + names = z.namelist() + elif ext in (".tar", ".gz", ".bz2", ".xz"): + with tarfile.open(path) as t: + names = t.getnames() + else: + return f"Unsupported archive format: {ext}" + preview = "\n".join(names[:30]) + suffix = f"\n... and {len(names)-30} more" if len(names) > 30 else "" + return f"Archive contains {len(names)} files:\n{preview}{suffix}" + except Exception as e: + return f"List failed: {e}" + + if action == "extract": + dest = Path(params.get("destination", str(path.parent / path.stem))) + dest.mkdir(parents=True, exist_ok=True) + try: + shutil.unpack_archive(path, dest) + return f"Extracted to: {dest}" + except Exception as e: + return f"Extract failed: {e}" + + return f"Unknown archive action: '{action}'. Try: list, extract" + +def _process_pptx(path: Path, action: str, params: dict, speak=None) -> str: + action = action or "summarize" + + def _read_pptx_text() -> str: + try: + from pptx import Presentation + prs = Presentation(path) + text = [] + for i, slide in enumerate(prs.slides, 1): + slide_text = f"\n--- Slide {i} ---\n" + for shape in slide.shapes: + if hasattr(shape, "text") and shape.text.strip(): + slide_text += shape.text.strip() + "\n" + text.append(slide_text) + return "\n".join(text) + except ImportError: + return "python-pptx not installed." + + if action in ("summarize", "extract_text", "analyze"): + text = _read_pptx_text() + if action == "extract_text": + out = _output_path(path, "text", ".txt") + out.write_text(text, encoding="utf-8") + return f"Text extracted. Saved: {out.name}" + try: + model = _gemini_client() + prompt = f"{'Summarize' if action == 'summarize' else 'Analyze'} this presentation:\n{text[:30000]}" + response = model.generate_content(prompt) + return response.text.strip() + except Exception as e: + return f"AI processing failed: {e}" + + return f"Unknown PPTX action: '{action}'. Try: summarize, extract_text, analyze" + +def file_processor(parameters: dict, player=None, speak=None) -> str: + file_path_str = parameters.get("file_path", "").strip() + if not file_path_str: + return "No file path provided." + + path = Path(file_path_str) + if not path.exists(): + return f"File not found: {file_path_str}" + if not path.is_file(): + return f"Path is not a file: {file_path_str}" + + file_type = _detect_type(path) + action = (parameters.get("action") or "").lower().strip() + instruction = parameters.get("instruction", "") + params = {**parameters, "instruction": instruction} + + log_msg = f"[FileProcessor] {file_type.upper()} | {path.name} | action={action or 'auto'}" + print(log_msg) + if player: + player.write_log(log_msg) + + if file_type == "unknown": + try: + content = path.read_text(encoding="utf-8", errors="ignore")[:10000] + model = _gemini_client() + prompt = f"File: {path.name}\nContent preview:\n{content}\n\nTask: {action or instruction or 'Describe what this file contains and what can be done with it.'}" + response = model.generate_content(prompt) + return response.text.strip() + except Exception as e: + return f"Unknown file type ({path.suffix}). Could not process: {e}" + + dispatch = { + "image": _process_image, + "pdf": _process_pdf, + "docx": lambda p, a, pm, s: _process_text_doc(p, "docx", a, pm, s), + "text": lambda p, a, pm, s: _process_text_doc(p, "text", a, pm, s), + "csv": lambda p, a, pm, s: _process_data(p, "csv", a, pm, s), + "excel": lambda p, a, pm, s: _process_data(p, "excel", a, pm, s), + "json": _process_json, + "xml": lambda p, a, pm, s: _process_json(p, a, pm, s), + "code": _process_code, + "audio": _process_audio, + "video": _process_video, + "archive": _process_archive, + "pptx": _process_pptx, + } + + handler = dispatch.get(file_type) + if not handler: + return f"Unsupported file type: {file_type}" + + try: + result = handler(path, action, params, speak) + return result or "Done." + except Exception as e: + import traceback + traceback.print_exc() + return f"Processing failed: {e}" \ No newline at end of file diff --git a/actions/flight_finder.py b/actions/flight_finder.py new file mode 100644 index 0000000..f0bfdae --- /dev/null +++ b/actions/flight_finder.py @@ -0,0 +1,339 @@ +#flight_finder.py +import json +import re +import subprocess +import sys +from datetime import datetime, timedelta +from pathlib import Path + +from config import is_windows, is_mac, is_linux + +def _get_base_dir() -> Path: + if getattr(sys, "frozen", False): + return Path(sys.executable).parent + return Path(__file__).resolve().parent.parent + + +BASE_DIR = _get_base_dir() +API_CONFIG_PATH = BASE_DIR / "config" / "api_keys.json" + +_MONTH_MAP: dict[str, int] = { + + "january": 1, "february": 2, "march": 3, "april": 4, + "may": 5, "june": 6, "july": 7, "august": 8, + "september": 9, "october": 10, "november": 11, "december": 12, + "ocak": 1, "şubat": 2, "mart": 3, "nisan": 4, + "mayΔ±s": 5, "haziran": 6, "temmuz": 7, "ağustos": 8, + "eylΓΌl": 9, "ekim": 10, "kasΔ±m": 11, "aralΔ±k": 12, +} + +_RELATIVE_MAP_KEYS = { + "today", "bugΓΌn", + "tomorrow", "yarΔ±n", +} + + +def _parse_date(raw: str) -> str: + + raw = raw.strip() + lower = raw.lower() + today = datetime.now() + + if re.match(r"\d{4}-\d{2}-\d{2}", raw): + return raw + for fmt in ("%d/%m/%Y", "%m/%d/%Y", "%d.%m.%Y", "%d-%m-%Y"): + try: + return datetime.strptime(raw, fmt).strftime("%Y-%m-%d") + except ValueError: + pass + + relative = { + "today": today, "bugΓΌn": today, + "tomorrow": today + timedelta(days=1), + "yarΔ±n": today + timedelta(days=1), + } + for key, val in relative.items(): + if key in lower: + return val.strftime("%Y-%m-%d") +# replace the try/except genai block with: + try: + from or_client import client + result = client.chat( + f"Today is {today.strftime('%Y-%m-%d')}. " + f"Convert this date expression to YYYY-MM-DD: '{raw}'. " + f"Return ONLY the date string, nothing else.", + system="You are a date converter. Return only the YYYY-MM-DD string." + ) + result = result.strip() + if re.match(r"\d{4}-\d{2}-\d{2}", result): + return result + except Exception as e: + print(f"[FlightFinder] ⚠️ date parse failed: {e}") + for month_name, month_num in _MONTH_MAP.items(): + if month_name in lower: + day_match = re.search(r"\d{1,2}", raw) + if day_match: + day = int(day_match.group()) + year = today.year if month_num >= today.month else today.year + 1 + return f"{year}-{month_num:02d}-{day:02d}" + + # Last resort: today + print(f"[FlightFinder] ⚠️ Could not parse date '{raw}' β€” using today.") + return today.strftime("%Y-%m-%d") + +_CABIN_CODE: dict[str, str] = { + "economy": "1", + "premium": "2", + "business": "3", + "first": "4", +} + +def _build_google_flights_url( + origin: str, + destination: str, + date: str, + return_date: str | None = None, + passengers: int = 1, + cabin: str = "economy", +) -> str: + from urllib.parse import quote_plus + + cabin_code = _CABIN_CODE.get(cabin.lower(), "1") + + origin_enc = quote_plus(origin) + destination_enc = quote_plus(destination) + + if return_date: + trip = f"Flights+from+{origin_enc}+to+{destination_enc}+on+{date}+returning+{return_date}" + else: + trip = f"Flights+from+{origin_enc}+to+{destination_enc}+on+{date}" + + return ( + f"https://www.google.com/travel/flights" + f"?q={trip}" + f"&curr=USD" + f"&cabin={cabin_code}" + f"&adults={passengers}" + ) + +def _search_flights_browser( + origin: str, + destination: str, + date: str, + return_date: str | None, + passengers: int, + cabin: str, +) -> tuple[str, str]: + import time + from actions.browser_control import browser_control + + url = _build_google_flights_url( + origin, destination, date, return_date, passengers, cabin + ) + + print(f"[FlightFinder] 🌐 Opening: {url}") + browser_control({"action": "go_to", "url": url}) + time.sleep(5) + + raw = browser_control({"action": "get_text"}) + return (raw or ""), url +def _parse_flights_with_gemini( + raw_text: str, + origin: str, + destination: str, + date: str, +) -> list[dict]: + from or_client import client + + prompt = ( + f"Extract flight options from {origin} to {destination} on {date} " + f"from this Google Flights page text:\n\n{raw_text[:12000]}\n\n" + f"Return a JSON array of up to 5 flights:\n" + f'[{{"airline":"...","departure":"HH:MM","arrival":"HH:MM",' + f'"duration":"Xh Ym","stops":0,"price":"...","currency":"USD"}}]\n' + f"If no flights found, return: []" + ) + + try: + result = client.chat_json(prompt, system="Return only valid JSON. No extra text.") + return result if isinstance(result, list) else [] + except Exception as e: + print(f"[FlightFinder] ⚠️ parse failed: {e}") + return [] + +def _format_spoken( + flights: list[dict], + origin: str, + destination: str, + date: str, +) -> str: + if not flights: + return ( + f"I couldn't find any flights from {origin} to {destination} " + f"on {date}, sir. The page may not have loaded correctly." + ) + + lines = [f"Here are the top flights from {origin} to {destination} on {date}, sir."] + + for i, f in enumerate(flights[:5], 1): + airline = f.get("airline", "Unknown airline") + departure = f.get("departure", "--:--") + arrival = f.get("arrival", "--:--") + duration = f.get("duration", "") + stops = f.get("stops", 0) + price = f.get("price", "") + currency = f.get("currency", "") + + stop_str = "non-stop" if stops == 0 else f"{stops} stop{'s' if stops > 1 else ''}" + price_str = f"{price} {currency}".strip() if price else "price unavailable" + dur_str = f", {duration}" if duration else "" + + lines.append( + f"Option {i}: {airline}, departing {departure}, " + f"arriving {arrival}{dur_str}, {stop_str}, {price_str}." + ) + + # Cheapest β€” strip non-digits for comparison + priced = [f for f in flights if f.get("price")] + if priced: + cheapest = min( + priced, + key=lambda x: int(re.sub(r"[^\d]", "", str(x["price"])) or "999999"), + ) + lines.append( + f"The cheapest option is {cheapest.get('airline')} " + f"at {cheapest.get('price')} {cheapest.get('currency', '')}." + ) + + return " ".join(lines) + + +def _format_text_report( + flights: list[dict], + origin: str, + destination: str, + date: str, + return_date: str | None, + page_url: str, +) -> str: + lines = [ + "JARVIS β€” Flight Search Results", + "─" * 50, + f"Route : {origin} β†’ {destination}", + f"Date : {date}", + ] + if return_date: + lines.append(f"Return : {return_date}") + lines += [ + f"Searched : {datetime.now().strftime('%Y-%m-%d %H:%M')}", + f"Source : {page_url}", + "─" * 50, + "", + ] + + if not flights: + lines.append("No flights found.") + else: + for i, f in enumerate(flights, 1): + stops = f.get("stops", 0) + stop_str = "Non-stop" if stops == 0 else f"{stops} stop(s)" + lines += [ + f"Flight {i}:", + f" Airline : {f.get('airline', 'N/A')}", + f" Departure : {f.get('departure', 'N/A')}", + f" Arrival : {f.get('arrival', 'N/A')}", + f" Duration : {f.get('duration', 'N/A')}", + f" Stops : {stop_str}", + f" Price : {f.get('price', 'N/A')} {f.get('currency', '')}", + "", + ] + + return "\n".join(lines) + +def _save_to_desktop(content: str, origin: str, destination: str) -> str: + ts = datetime.now().strftime("%Y%m%d_%H%M%S") + filename = f"flights_{origin}_{destination}_{ts}.txt".replace(" ", "_") + desktop = Path.home() / "Desktop" + desktop.mkdir(parents=True, exist_ok=True) + filepath = desktop / filename + + filepath.write_text(content, encoding="utf-8") + print(f"[FlightFinder] πŸ’Ύ Saved: {filepath}") + + try: + if is_windows(): + subprocess.Popen(["notepad.exe", str(filepath)]) + elif is_mac(): + subprocess.Popen(["open", "-t", str(filepath)]) + else: + subprocess.Popen(["xdg-open", str(filepath)]) + except Exception as e: + print(f"[FlightFinder] ⚠️ Could not open text editor: {e}") + + return str(filepath) + + +def flight_finder(parameters: dict, player=None, speak=None) -> str: + params = parameters or {} + + origin = params.get("origin", "").strip() + destination = params.get("destination", "").strip() + date_raw = params.get("date", "").strip() + return_raw = (params.get("return_date") or "").strip() + passengers = max(1, int(params.get("passengers", 1))) + cabin = params.get("cabin", "economy").strip().lower() + save = bool(params.get("save", False)) + + if not origin or not destination: + return "Please provide both origin and destination, sir." + if not date_raw: + return "Please provide a departure date, sir." + + # Normalise cabin value + if cabin not in _CABIN_CODE: + cabin = "economy" + + date = _parse_date(date_raw) + return_date = _parse_date(return_raw) if return_raw else None + + if player: + player.write_log(f"[FlightFinder] {origin} β†’ {destination} on {date}") + + if speak: + speak(f"Searching flights from {origin} to {destination} on {date}, sir.") + + print( + f"[FlightFinder] ▢️ {origin} β†’ {destination} | {date}" + f"{' β†’ ' + return_date if return_date else ''}" + f" | {cabin} | {passengers} pax" + ) + + try: + raw_text, page_url = _search_flights_browser( + origin, destination, date, return_date, passengers, cabin + ) + + if not raw_text: + return "Could not retrieve flight data, sir. The page may not have loaded." + + if speak: + speak("Analysing the results now, sir.") + + flights = _parse_flights_with_gemini(raw_text, origin, destination, date) + spoken = _format_spoken(flights, origin, destination, date) + + if speak: + speak(spoken) + + result = spoken + + if save and flights: + report = _format_text_report(flights, origin, destination, date, return_date, page_url) + saved_path = _save_to_desktop(report, origin, destination) + result += f" Results saved to Desktop: {saved_path}" + + return result + + except Exception as e: + print(f"[FlightFinder] ❌ {e}") + return f"Flight search failed, sir: {e}" \ No newline at end of file diff --git a/actions/game_updater.py b/actions/game_updater.py new file mode 100644 index 0000000..4ff1d4d --- /dev/null +++ b/actions/game_updater.py @@ -0,0 +1,817 @@ +import os +import re +import sys +import json +import time +import subprocess +import threading +import winreg +from pathlib import Path +from datetime import datetime + + +def _find_steam_path() -> Path | None: + registry_keys = [ + (winreg.HKEY_LOCAL_MACHINE, r"SOFTWARE\WOW6432Node\Valve\Steam"), + (winreg.HKEY_LOCAL_MACHINE, r"SOFTWARE\Valve\Steam"), + (winreg.HKEY_CURRENT_USER, r"SOFTWARE\Valve\Steam"), + ] + for hive, key_path in registry_keys: + try: + key = winreg.OpenKey(hive, key_path) + val, _ = winreg.QueryValueEx(key, "InstallPath") + winreg.CloseKey(key) + p = Path(val) + if p.exists() and (p / "steam.exe").exists(): + return p + except Exception: + continue + for p in [ + Path(os.environ.get("ProgramFiles(x86)", "")) / "Steam", + Path(os.environ.get("ProgramFiles", "")) / "Steam", + Path("C:/Steam"), Path("D:/Steam"), Path("E:/Steam"), Path("F:/Steam"), + ]: + if p.exists() and (p / "steam.exe").exists(): + return p + return None + + +def _find_epic_path() -> Path | None: + registry_keys = [ + (winreg.HKEY_LOCAL_MACHINE, r"SOFTWARE\WOW6432Node\EpicGames\EpicGamesLauncher"), + (winreg.HKEY_LOCAL_MACHINE, r"SOFTWARE\EpicGames\EpicGamesLauncher"), + (winreg.HKEY_CURRENT_USER, r"SOFTWARE\EpicGames\EpicGamesLauncher"), + ] + for hive, key_path in registry_keys: + try: + key = winreg.OpenKey(hive, key_path) + val, _ = winreg.QueryValueEx(key, "AppDataPath") + winreg.CloseKey(key) + exe = Path(val) / "Binaries" / "Win64" / "EpicGamesLauncher.exe" + if exe.exists(): + return exe.parent + except Exception: + continue + for p in [ + Path(os.environ.get("ProgramFiles(x86)", "")) / "Epic Games" / "Launcher" / "Portal" / "Binaries" / "Win64", + Path(os.environ.get("ProgramFiles", "")) / "Epic Games" / "Launcher" / "Portal" / "Binaries" / "Win64", + Path(os.environ.get("LOCALAPPDATA", "")) / "EpicGamesLauncher" / "Portal" / "Binaries" / "Win64", + ]: + if p.exists() and (p / "EpicGamesLauncher.exe").exists(): + return p + return None + + +def _get_steam_libraries(steam_path: Path) -> list[Path]: + libraries = [steam_path / "steamapps"] + vdf_path = steam_path / "steamapps" / "libraryfolders.vdf" + if not vdf_path.exists(): + return libraries + try: + content = vdf_path.read_text(encoding="utf-8", errors="ignore") + for raw_path in re.findall(r'"path"\s+"([^"]+)"', content): + lib = Path(raw_path.replace("\\\\", "/")) / "steamapps" + if lib.exists() and lib not in libraries: + libraries.append(lib) + except Exception: + pass + return libraries + + +def _get_steam_games(steam_path: Path) -> list[dict]: + games = [] + for lib in _get_steam_libraries(steam_path): + for acf in lib.glob("appmanifest_*.acf"): + try: + content = acf.read_text(encoding="utf-8", errors="ignore") + app_id = re.search(r'"appid"\s+"(\d+)"', content) + name = re.search(r'"name"\s+"([^"]+)"', content) + state = re.search(r'"StateFlags"\s+"(\d+)"', content) + size = re.search(r'"SizeOnDisk"\s+"(\d+)"', content) + if app_id and name: + games.append({ + "id": app_id.group(1), + "name": name.group(1), + "state": int(state.group(1)) if state else 0, + "size": int(size.group(1)) if size else 0, + "lib": str(lib), + }) + except Exception: + continue + return games + + +def _is_steam_running() -> bool: + try: + out = subprocess.run(["tasklist", "/FI", "IMAGENAME eq steam.exe"], + capture_output=True, text=True).stdout + return "steam.exe" in out.lower() + except Exception: + return False + + +def _get_steam_window_rect() -> tuple[int, int, int, int] | None: + try: + import pygetwindow as gw + for w in gw.getAllWindows(): + if "steam" in w.title.lower() and w.width > 200 and w.visible: + return w.left, w.top, w.width, w.height + except Exception: + pass + return None + + +def _click_first_profile_by_screenshot() -> bool: + try: + import pyautogui + import numpy as np + + time.sleep(1.5) + + win = _get_steam_window_rect() + if not win: + print("[GameUpdater] ⚠️ Steam window not found") + return False + + wx, wy, ww, wh = win + screenshot = pyautogui.screenshot(region=(wx, wy, ww, wh)) + img = np.array(screenshot) + + h, w = img.shape[:2] + + search_y1 = h // 3 + search_y2 = h * 3 // 4 + search_x1 = w // 5 + search_x2 = w * 4 // 5 + region = img[search_y1:search_y2, search_x1:search_x2] + + r = region[:, :, 0].astype(int) + g = region[:, :, 1].astype(int) + b = region[:, :, 2].astype(int) + + max_c = np.maximum(np.maximum(r, g), b) + min_c = np.minimum(np.minimum(r, g), b) + sat = max_c - min_c + + colorful = (max_c > 60) & (sat > 40) + + if not colorful.any(): + print("[GameUpdater] ⚠️ No colorful avatar region found β€” clicking estimated center-left") + pyautogui.click(wx + ww // 2 - ww // 6, wy + wh // 2) + return True + + cols = np.where(colorful.any(axis=0))[0] + rows = np.where(colorful.any(axis=1))[0] + + if len(cols) == 0 or len(rows) == 0: + return False + + avatar_w = min(90, (region.shape[1]) // 4) + first_col = int(cols[0]) + block_cols = cols[cols < first_col + avatar_w] + + center_x_local = int(block_cols.mean()) + center_y_local = int(rows.mean()) + + abs_x = wx + search_x1 + center_x_local + abs_y = wy + search_y1 + center_y_local + + print(f"[GameUpdater] 🎯 First profile avatar at ({abs_x}, {abs_y}) β€” clicking") + pyautogui.click(abs_x, abs_y) + return True + + except ImportError as e: + print(f"[GameUpdater] ⚠️ Missing library: {e}") + return False + except Exception as e: + print(f"[GameUpdater] ⚠️ Screenshot profile detection failed: {e}") + return False + + +def _handle_steam_profile_selection() -> bool: + print("[GameUpdater] πŸ” Checking for 'Who's playing?' dialog...") + + win = _get_steam_window_rect() + if not win: + return False + + wx, wy, ww, wh = win + + try: + import pyautogui + import numpy as np + + screenshot = pyautogui.screenshot(region=(wx, wy, ww, wh)) + img = np.array(screenshot) + + is_small_window = ww < 900 and wh < 700 + + top_region = img[:wh // 3, :, :] + white_pixels = int(np.sum( + (top_region[:, :, 0] > 200) & + (top_region[:, :, 1] > 200) & + (top_region[:, :, 2] > 200) + )) + has_white_text = white_pixels > 100 + + if not is_small_window and not has_white_text: + print("[GameUpdater] ℹ️ No profile dialog detected β€” Steam already logged in") + return False + + except ImportError: + pass + except Exception: + pass + + print("[GameUpdater] πŸ‘€ Profile selection detected β€” clicking first profile") + return _click_first_profile_by_screenshot() + + +def _ensure_steam_running(steam_path: Path) -> bool: + if _is_steam_running(): + return True + + steam_exe = steam_path / "steam.exe" + if not steam_exe.exists(): + print("[GameUpdater] ❌ steam.exe not found") + return False + + print("[GameUpdater] πŸš€ Starting Steam...") + subprocess.Popen([str(steam_exe)]) + + for _ in range(20): + time.sleep(1) + if _is_steam_running(): + print("[GameUpdater] βœ… Steam running") + time.sleep(4) + _handle_steam_profile_selection() + time.sleep(2) + return True + + print("[GameUpdater] ⚠️ Steam did not start in time") + return False + + +def _update_steam_games(steam_path: Path, game_name: str = None) -> str: + if not _ensure_steam_running(steam_path): + return "Could not start Steam." + + steam_exe = steam_path / "steam.exe" + games = _get_steam_games(steam_path) + + if not games: + return "No Steam games found." + + if game_name: + name_lower = game_name.lower() + matched = [g for g in games if name_lower in g["name"].lower()] + if not matched: + available = ", ".join(g["name"] for g in games[:5]) + return f"Game '{game_name}' not found. Installed: {available}..." + targets = matched + else: + targets = games + + already_updated, already_running, update_started, errors = [], [], [], [] + + for game in targets: + state = game["state"] + name = game["name"] + if state == 4: + already_updated.append(name) + elif state == 1026: + already_running.append(name) + else: + try: + subprocess.Popen([str(steam_exe), f"steam://update/{game['id']}"]) + update_started.append(name) + time.sleep(0.3) + except Exception as e: + errors.append(f"{name}: {e}") + + parts = [] + if update_started: + names = ", ".join(update_started[:3]) + suffix = f" and {len(update_started) - 3} more" if len(update_started) > 3 else "" + parts.append(f"Update started for: {names}{suffix}.") + if already_running: + parts.append(f"Already updating: {', '.join(already_running)}.") + if already_updated: + parts.append( + f"{already_updated[0]} is already up to date." + if game_name else + f"{len(already_updated)} game(s) already up to date." + ) + if errors: + parts.append(f"Errors: {'; '.join(errors)}.") + return " ".join(parts) if parts else "No games to update." + + +_KNOWN_APPIDS: dict[str, tuple[str, str]] = { + "pubg": ("578080", "PUBG: Battlegrounds"), + "pubg battlegrounds": ("578080", "PUBG: Battlegrounds"), + "pubg: battlegrounds": ("578080", "PUBG: Battlegrounds"), + "battlegrounds": ("578080", "PUBG: Battlegrounds"), + "gta5": ("271590", "Grand Theft Auto V"), + "gta v": ("271590", "Grand Theft Auto V"), + "grand theft auto v": ("271590", "Grand Theft Auto V"), + "cs2": ("730", "Counter-Strike 2"), + "csgo": ("730", "Counter-Strike 2"), + "counter-strike 2": ("730", "Counter-Strike 2"), + "counter strike 2": ("730", "Counter-Strike 2"), + "dota2": ("570", "Dota 2"), + "dota 2": ("570", "Dota 2"), + "rust": ("252490", "Rust"), + "valheim": ("892970", "Valheim"), + "cyberpunk": ("1091500", "Cyberpunk 2077"), + "cyberpunk 2077": ("1091500", "Cyberpunk 2077"), + "elden ring": ("1245620", "ELDEN RING"), + "minecraft": ("1672970", "Minecraft Launcher"), + "apex legends": ("1172470", "Apex Legends"), + "apex": ("1172470", "Apex Legends"), + "fortnite": ("1517990", "Fortnite"), + "goose goose duck": ("1568590", "Goose Goose Duck"), + "among us": ("945360", "Among Us"), + "fall guys": ("1097150", "Fall Guys"), + "rocket league": ("252950", "Rocket League"), + "warframe": ("230410", "Warframe"), + "destiny 2": ("1085660", "Destiny 2"), + "team fortress 2": ("440", "Team Fortress 2"), + "tf2": ("440", "Team Fortress 2"), + "left 4 dead 2": ("550", "Left 4 Dead 2"), + "l4d2": ("550", "Left 4 Dead 2"), + "paladins": ("444090", "Paladins"), + "smite": ("386360", "SMITE"), + "war thunder": ("236390", "War Thunder"), + "world of warships": ("552990", "World of Warships"), + "path of exile": ("238960", "Path of Exile"), + "poe": ("238960", "Path of Exile"), + "lost ark": ("1599340", "Lost Ark"), + "new world": ("1063730", "New World: Aeternum"), +} + + +def _search_steam_appid(game_name: str) -> tuple[str | None, str | None]: + name_lower = game_name.lower().strip() + + steam_path = _find_steam_path() + if steam_path: + for g in _get_steam_games(steam_path): + if name_lower in g["name"].lower(): + return g["id"], g["name"] + + if name_lower in _KNOWN_APPIDS: + app_id, canonical = _KNOWN_APPIDS[name_lower] + print(f"[GameUpdater] πŸ“– Known: {canonical} ({app_id})") + return app_id, canonical + + for key, (app_id, canonical) in _KNOWN_APPIDS.items(): + if name_lower in key or key in name_lower: + print(f"[GameUpdater] πŸ“– Known partial: {canonical} ({app_id})") + return app_id, canonical + + try: + import urllib.request, urllib.parse + query = urllib.parse.quote(game_name) + url = f"https://store.steampowered.com/api/storesearch/?term={query}&l=english&cc=US" + req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0"}) + with urllib.request.urlopen(req, timeout=6) as resp: + items = json.loads(resp.read().decode()).get("items", []) + if items: + best = items[0] + print(f"[GameUpdater] 🌐 Store API: {best['name']} ({best['id']})") + return str(best["id"]), best["name"] + except Exception as e: + print(f"[GameUpdater] ⚠️ AppID search failed: {e}") + + return None, None + + +def _find_best_drive() -> dict | None: + import shutil, string + drives = [] + for letter in string.ascii_uppercase: + drive_path = f"{letter}:\\" + if os.path.exists(drive_path): + try: + gb = shutil.disk_usage(drive_path).free / (1024 ** 3) + if gb > 0: + drives.append({"letter": letter, "path": drive_path, "free_gb": gb}) + except Exception: + continue + return max(drives, key=lambda d: d["free_gb"]) if drives else None + + +def _select_drive_in_dialog(dialog, drive_letter: str) -> bool: + target = drive_letter.upper() + for control_type in ("ListItem", "RadioButton"): + try: + for ctrl in dialog.descendants(control_type=control_type): + if target in ctrl.window_text().upper(): + ctrl.click_input() + print(f"[GameUpdater] βœ… Drive via {control_type}: {ctrl.window_text()}") + return True + except Exception: + continue + try: + for combo in dialog.descendants(control_type="ComboBox"): + try: + combo.expand() + time.sleep(0.15) + for idx, txt in enumerate(combo.texts()): + if target in txt.upper(): + combo.select(idx) + return True + combo.collapse() + except Exception: + continue + except Exception: + pass + try: + for ctrl in dialog.descendants(): + txt = ctrl.window_text().upper() + if f"{target}:" in txt and len(txt) < 80: + ctrl.click_input() + return True + except Exception: + pass + return False + + +def _click_button(window, keywords: list[str]) -> bool: + try: + for btn in window.descendants(control_type="Button"): + try: + txt = btn.window_text().lower().strip() + if txt in keywords or any(kw in txt for kw in keywords): + btn.click_input() + return True + except Exception: + continue + except Exception: + pass + return False + + +def _handle_install_dialog(game_name: str) -> str: + best_drive = _find_best_drive() + if not best_drive: + return f"Install dialog opened for '{game_name}'. Could not detect drives." + + drive_letter = best_drive["letter"] + drive_label = f"{drive_letter}:" + print(f"[GameUpdater] πŸ† Target drive: {drive_label} ({best_drive['free_gb']:.1f} GB free)") + + try: + from pywinauto import Application, findwindows + dialog = None + for _ in range(40): + time.sleep(0.5) + try: + for hwnd in findwindows.find_windows(title_re=r"(?i)(install|yΓΌkle|steam)", visible_only=True): + try: + app = Application(backend="uia").connect(handle=hwnd) + win = app.window(handle=hwnd) + rect = win.rectangle() + if win.is_visible() and rect.width() > 300 and rect.height() > 200: + all_text = " ".join(c.window_text() for c in win.descendants() if c.window_text()).upper() + if any(x in all_text for x in ("C:", "D:", "E:", "F:", "INSTALL", "YÜKLE")): + dialog = win + break + except Exception: + continue + except Exception: + pass + if dialog: + break + + if not dialog: + raise RuntimeError("Dialog not found") + + dialog.set_focus() + time.sleep(0.4) + drive_selected = _select_drive_in_dialog(dialog, drive_letter) + install_clicked = _click_button(dialog, ["install", "yΓΌkle", "next", "ileri", "ok", "tamam"]) + + if install_clicked: + suffix = f"Selected {drive_label} and" if drive_selected else "Default drive used, but" + return f"{suffix} clicked Install for '{game_name}'." + return f"Please click Install manually in Steam for '{game_name}'." + + except ImportError: + return _handle_install_dialog_pyautogui(game_name, best_drive) + except Exception as e: + print(f"[GameUpdater] ⚠️ pywinauto failed: {e}") + return _handle_install_dialog_pyautogui(game_name, best_drive) + + +def _handle_install_dialog_pyautogui(game_name: str, best_drive: dict) -> str: + try: + import pyautogui + import pygetwindow as gw + except ImportError: + return f"Install dialog opened for '{game_name}'. Please select '{best_drive['letter']}:' and click Install manually." + + pyautogui.FAILSAFE = False + drive_label = f"{best_drive['letter']}:" + install_win = None + + for _ in range(30): + time.sleep(0.5) + for w in gw.getAllWindows(): + if ("install" in w.title.lower() or "steam" in w.title.lower()) and w.width > 300 and w.visible: + install_win = w + break + if install_win: + break + + if not install_win: + return f"Please select '{drive_label}' and click Install in Steam for '{game_name}'." + + try: + install_win.activate() + time.sleep(0.4) + except Exception: + pass + + wx, wy = install_win.left, install_win.top + ww, wh = install_win.width, install_win.height + pyautogui.click(wx + int(ww * 0.35), wy + int(wh * 0.45)) + time.sleep(0.2) + pyautogui.typewrite(best_drive["letter"], interval=0.05) + time.sleep(0.2) + pyautogui.click(wx + int(ww * 0.72), wy + int(wh * 0.88)) + return f"Attempted drive {drive_label} selection and Install click for '{game_name}'." + + +def _install_steam_game(steam_path: Path, game_name: str = None, app_id: str = None) -> str: + if not _ensure_steam_running(steam_path): + return "Could not start Steam." + + steam_exe = steam_path / "steam.exe" + installed_games = _get_steam_games(steam_path) + + already = None + if app_id: + already = next((g for g in installed_games if g["id"] == str(app_id)), None) + elif game_name: + name_lower = game_name.lower() + already = next((g for g in installed_games if name_lower in g["name"].lower()), None) + else: + return "Please specify a game name or AppID." + + if already: + state = already["state"] + name = already["name"] + if state == 4: + return f"'{name}' is already installed and up to date." + if state == 1026: + return f"'{name}' is currently downloading or updating." + if state in (6, 516): + subprocess.Popen([str(steam_exe), f"steam://update/{already['id']}"]) + return f"'{name}' has a pending update. Update started." + return f"'{name}' is already installed." + + if not app_id and game_name: + found_id, found_name = _search_steam_appid(game_name) + if not found_id: + return f"Could not find '{game_name}' on Steam. Try providing the AppID directly." + app_id = found_id + game_name = found_name or game_name + print(f"[GameUpdater] πŸ” Installing: {game_name} (AppID: {app_id})") + + try: + subprocess.Popen([str(steam_exe), f"steam://install/{app_id}"]) + threading.Thread(target=_handle_install_dialog, args=(game_name or str(app_id),), daemon=True).start() + return f"Install started for '{game_name}'. Steam will open the download dialog." + except Exception as e: + return f"Install failed: {e}" + + +def _get_download_status(steam_path: Path) -> str: + games = _get_steam_games(steam_path) + active = [g for g in games if g["state"] == 1026] + pending = [g for g in games if g["state"] in (6, 516)] + lines = [] + if active: + lines.append(f"Downloading: {', '.join(g['name'] for g in active)}.") + if pending: + names = ", ".join(g["name"] for g in pending[:5]) + suffix = f" and {len(pending) - 5} more" if len(pending) > 5 else "" + lines.append(f"Pending updates: {names}{suffix}.") + return " ".join(lines) if lines else "No active downloads or pending updates." + + +def _watch_and_shutdown(steam_path: Path, speak=None, check_interval: int = 30, timeout_hours: int = 12): + print("[GameUpdater] πŸ‘οΈ Watching downloads for shutdown...") + deadline = time.time() + timeout_hours * 3600 + + for _ in range(24): + time.sleep(5) + active = [g for g in _get_steam_games(steam_path) if g["state"] == 1026] + if active: + names = ", ".join(g["name"] for g in active) + if speak: speak(f"Download started for {names}. I'll shut down when done.") + break + else: + return + + while time.time() < deadline: + time.sleep(check_interval) + if not any(g["state"] == 1026 for g in _get_steam_games(steam_path)): + if speak: speak("Download complete. Shutting down now.") + time.sleep(5) + subprocess.run(["shutdown", "/s", "/t", "10"]) + return + + if speak: speak("Download taking too long. Cancelling auto-shutdown.") + + +def _get_epic_games() -> list[dict]: + manifests_path = (Path(os.environ.get("PROGRAMDATA", "C:/ProgramData")) + / "Epic" / "EpicGamesLauncher" / "Data" / "Manifests") + if not manifests_path.exists(): + return [] + games = [] + for item_file in manifests_path.glob("*.item"): + try: + data = json.loads(item_file.read_text(encoding="utf-8")) + name = data.get("DisplayName") or data.get("AppName", "") + if name: + games.append({"id": data.get("AppName", ""), "name": name}) + except Exception: + continue + return games + + +def _is_epic_running() -> bool: + try: + return "epicgameslauncher.exe" in subprocess.run( + ["tasklist", "/FI", "IMAGENAME eq EpicGamesLauncher.exe"], + capture_output=True, text=True + ).stdout.lower() + except Exception: + return False + + +def _update_epic_games(epic_path: Path, game_name: str = None) -> str: + epic_exe = epic_path / "EpicGamesLauncher.exe" + if not epic_exe.exists(): + return "Epic Games Launcher not found." + games = _get_epic_games() + if game_name: + name_lower = game_name.lower() + matched = [g for g in games if name_lower in g["name"].lower()] + if not matched: + return f"'{game_name}' not found in Epic." + try: + subprocess.Popen([str(epic_exe), f"com.epicgames.launcher://apps/{matched[0]['id']}?action=launch&silent=true"]) + return f"Opened Epic for '{matched[0]['name']}'." + except Exception as e: + return f"Epic update failed: {e}" + else: + try: + if _is_epic_running(): + for g in games[:10]: + subprocess.Popen([str(epic_exe), f"com.epicgames.launcher://apps/{g['id']}?action=launch&silent=true"]) + time.sleep(0.5) + return f"Triggered update check for {len(games)} Epic game(s)." + else: + subprocess.Popen([str(epic_exe)]) + return f"Epic Games Launcher opened. {len(games)} game(s) will be checked." + except Exception as e: + return f"Epic update failed: {e}" + + +def _schedule_daily_update(hour: int = 3, minute: int = 0) -> str: + task_name = "JARVIS_GameUpdater" + script_path = Path(__file__).resolve() + subprocess.run(["schtasks", "/Delete", "/TN", task_name, "/F"], capture_output=True) + for extra in (["/RL", "HIGHEST", "/RU", "SYSTEM"], []): + cmd = ["schtasks", "/Create", "/TN", task_name, + "/TR", f'"{sys.executable}" "{script_path}" --scheduled', + "/SC", "DAILY", "/ST", f"{hour:02d}:{minute:02d}", "/F", *extra] + result = subprocess.run(cmd, capture_output=True, text=True) + if result.returncode == 0: + return f"Daily game update scheduled at {hour:02d}:{minute:02d}." + return f"Scheduling failed: {result.stderr.strip()}" + + +def _cancel_scheduled_update() -> str: + result = subprocess.run(["schtasks", "/Delete", "/TN", "JARVIS_GameUpdater", "/F"], + capture_output=True, text=True) + return "Scheduled update cancelled." if result.returncode == 0 else "No scheduled update found." + + +def _get_schedule_status() -> str: + result = subprocess.run(["schtasks", "/Query", "/TN", "JARVIS_GameUpdater", "/FO", "LIST"], + capture_output=True, text=True) + if result.returncode != 0: + return "No scheduled game update found." + for line in result.stdout.strip().split("\n"): + if any(k in line for k in ("Next Run", "Sonraki", "Prochaine", "PrΓ³xima", "NΓ€chste")): + return f"Game update scheduled. {line.strip()}" + return "Game update is scheduled." + + +def game_updater(parameters: dict, player=None, speak=None) -> str: + p = parameters or {} + action = p.get("action", "update").lower().strip() + platform = p.get("platform", "both").lower().strip() + game_name = (p.get("game_name") or "").strip() or None + app_id = (p.get("app_id") or "").strip() or None + hour = int(p.get("hour", 3)) + minute = int(p.get("minute", 0)) + shutdown = str(p.get("shutdown_when_done", "false")).lower() == "true" + + results = [] + + if action == "schedule": return _schedule_daily_update(hour=hour, minute=minute) + if action == "cancel_schedule": return _cancel_scheduled_update() + if action == "schedule_status": return _get_schedule_status() + + if action == "list": + if platform in ("steam", "both"): + steam_path = _find_steam_path() + if steam_path: + games = _get_steam_games(steam_path) + if games: + names = ", ".join(g["name"] for g in games[:8]) + suffix = f" and {len(games) - 8} more" if len(games) > 8 else "" + results.append(f"Steam ({len(games)} games): {names}{suffix}.") + else: + results.append("Steam: No games found.") + else: + results.append("Steam: Not installed.") + if platform in ("epic", "both"): + games = _get_epic_games() + if games: + names = ", ".join(g["name"] for g in games[:8]) + suffix = f" and {len(games) - 8} more" if len(games) > 8 else "" + results.append(f"Epic ({len(games)} games): {names}{suffix}.") + else: + results.append("Epic: No games found.") + return " | ".join(results) or "No platforms found." + + if action == "download_status": + if platform in ("steam", "both"): + steam_path = _find_steam_path() + results.append(_get_download_status(steam_path) if steam_path else "Steam: Not installed.") + if platform in ("epic", "both"): + results.append("Epic download status not available directly.") + return " ".join(results) + + if action in ("install", "update"): + if platform in ("steam", "both"): + steam_path = _find_steam_path() + if not steam_path: + results.append("Steam: Not installed.") + else: + if game_name: + installed = _get_steam_games(steam_path) + name_lower = game_name.lower() + is_installed = any(name_lower in g["name"].lower() for g in installed) + + if not is_installed: + msg = _install_steam_game(steam_path, game_name=game_name, app_id=app_id) + if shutdown: + threading.Thread(target=_watch_and_shutdown, + kwargs={"steam_path": steam_path, "speak": speak}, + daemon=True).start() + msg += " Auto-shutdown enabled." + if player: player.write_log(f"[GameUpdater] {msg[:100]}") + if speak: speak(msg) + return msg + else: + results.append(f"Steam: {_update_steam_games(steam_path, game_name=game_name)}") + else: + if action == "install": + results.append("Steam: Please specify a game name to install.") + else: + results.append(f"Steam: {_update_steam_games(steam_path)}") + + if shutdown: + threading.Thread(target=_watch_and_shutdown, + kwargs={"steam_path": steam_path, "speak": speak}, + daemon=True).start() + results.append("Auto-shutdown enabled.") + + if platform in ("epic", "both"): + epic_path = _find_epic_path() + if epic_path: + results.append(f"Epic: {_update_epic_games(epic_path, game_name=game_name)}") + else: + results.append("Epic: Not installed.") + + output = " | ".join(results) or "Nothing to do." + if player: player.write_log(f"[GameUpdater] {output[:100]}") + if speak: speak(output) + return output + + return f"Unknown action: '{action}'." + + +if __name__ == "__main__": + if "--scheduled" in sys.argv: + print(f"[GameUpdater] πŸ• Scheduled run at {datetime.now().strftime('%H:%M')}") + print(f"[GameUpdater] βœ… {game_updater({'action': 'update', 'platform': 'both'})}") \ No newline at end of file diff --git a/actions/open_app.py b/actions/open_app.py new file mode 100644 index 0000000..cf823b8 --- /dev/null +++ b/actions/open_app.py @@ -0,0 +1,208 @@ +# actions/open_app.py +# MARK XXV β€” Cross-Platform App Launcher + +import time +import subprocess +import platform +import shutil + +try: + import psutil + _PSUTIL = True +except ImportError: + _PSUTIL = False + +_APP_ALIASES = { + "whatsapp": {"Windows": "WhatsApp", "Darwin": "WhatsApp", "Linux": "whatsapp"}, + "chrome": {"Windows": "chrome", "Darwin": "Google Chrome", "Linux": "google-chrome"}, + "google chrome": {"Windows": "chrome", "Darwin": "Google Chrome", "Linux": "google-chrome"}, + "firefox": {"Windows": "firefox", "Darwin": "Firefox", "Linux": "firefox"}, + "spotify": {"Windows": "Spotify", "Darwin": "Spotify", "Linux": "spotify"}, + "vscode": {"Windows": "code", "Darwin": "Visual Studio Code", "Linux": "code"}, + "visual studio code": {"Windows": "code", "Darwin": "Visual Studio Code", "Linux": "code"}, + "discord": {"Windows": "Discord", "Darwin": "Discord", "Linux": "discord"}, + "telegram": {"Windows": "Telegram", "Darwin": "Telegram", "Linux": "telegram"}, + "instagram": {"Windows": "Instagram", "Darwin": "Instagram", "Linux": "instagram"}, + "tiktok": {"Windows": "TikTok", "Darwin": "TikTok", "Linux": "tiktok"}, + "notepad": {"Windows": "notepad.exe", "Darwin": "TextEdit", "Linux": "gedit"}, + "calculator": {"Windows": "calc.exe", "Darwin": "Calculator", "Linux": "gnome-calculator"}, + "terminal": {"Windows": "cmd.exe", "Darwin": "Terminal", "Linux": "gnome-terminal"}, + "cmd": {"Windows": "cmd.exe", "Darwin": "Terminal", "Linux": "bash"}, + "explorer": {"Windows": "explorer.exe", "Darwin": "Finder", "Linux": "nautilus"}, + "file explorer": {"Windows": "explorer.exe", "Darwin": "Finder", "Linux": "nautilus"}, + "paint": {"Windows": "mspaint.exe", "Darwin": "Preview", "Linux": "gimp"}, + "word": {"Windows": "winword", "Darwin": "Microsoft Word", "Linux": "libreoffice --writer"}, + "excel": {"Windows": "excel", "Darwin": "Microsoft Excel", "Linux": "libreoffice --calc"}, + "powerpoint": {"Windows": "powerpnt", "Darwin": "Microsoft PowerPoint","Linux": "libreoffice --impress"}, + "vlc": {"Windows": "vlc", "Darwin": "VLC", "Linux": "vlc"}, + "zoom": {"Windows": "Zoom", "Darwin": "zoom.us", "Linux": "zoom"}, + "slack": {"Windows": "Slack", "Darwin": "Slack", "Linux": "slack"}, + "steam": {"Windows": "steam", "Darwin": "Steam", "Linux": "steam"}, + "task manager": {"Windows": "taskmgr.exe", "Darwin": "Activity Monitor", "Linux": "gnome-system-monitor"}, + "settings": {"Windows": "ms-settings:", "Darwin": "System Preferences", "Linux": "gnome-control-center"}, + "powershell": {"Windows": "powershell.exe", "Darwin": "Terminal", "Linux": "bash"}, + "edge": {"Windows": "msedge", "Darwin": "Microsoft Edge", "Linux": "microsoft-edge"}, + "brave": {"Windows": "brave", "Darwin": "Brave Browser", "Linux": "brave-browser"}, + "obsidian": {"Windows": "Obsidian", "Darwin": "Obsidian", "Linux": "obsidian"}, + "notion": {"Windows": "Notion", "Darwin": "Notion", "Linux": "notion"}, + "blender": {"Windows": "blender", "Darwin": "Blender", "Linux": "blender"}, + "capcut": {"Windows": "CapCut", "Darwin": "CapCut", "Linux": "capcut"}, + "postman": {"Windows": "Postman", "Darwin": "Postman", "Linux": "postman"}, + "figma": {"Windows": "Figma", "Darwin": "Figma", "Linux": "figma"}, +} + + +def _normalize(raw: str) -> str: + system = platform.system() + key = raw.lower().strip() + if key in _APP_ALIASES: + return _APP_ALIASES[key].get(system, raw) + for alias_key, os_map in _APP_ALIASES.items(): + if alias_key in key or key in alias_key: + return os_map.get(system, raw) + return raw + + +def _is_running(app_name: str) -> bool: + if not _PSUTIL: + return True + app_lower = app_name.lower().replace(" ", "").replace(".exe", "") + try: + for proc in psutil.process_iter(["name"]): + try: + proc_name = proc.info["name"].lower().replace(" ", "").replace(".exe", "") + if app_lower in proc_name or proc_name in app_lower: + return True + except (psutil.NoSuchProcess, psutil.AccessDenied): + continue + except Exception: + pass + return False + + +def _launch_windows(app_name: str) -> bool: + try: + import pyautogui + pyautogui.PAUSE = 0.1 + pyautogui.press("win") + time.sleep(0.6) + pyautogui.write(app_name, interval=0.05) + time.sleep(0.8) + pyautogui.press("enter") + time.sleep(3.0) + return True + except Exception as e: + print(f"[open_app] ⚠️ Windows launch failed: {e}") + return False + +def _launch_macos(app_name: str) -> bool: + try: + result = subprocess.run(["open", "-a", app_name], capture_output=True, timeout=8) + if result.returncode == 0: + time.sleep(1.0) + return True + except Exception: + pass + + try: + result = subprocess.run(["open", "-a", f"{app_name}.app"], capture_output=True, timeout=8) + if result.returncode == 0: + time.sleep(1.0) + return True + except Exception: + pass + + try: + import pyautogui + pyautogui.hotkey("command", "space") + time.sleep(0.6) + pyautogui.write(app_name, interval=0.05) + time.sleep(0.8) + pyautogui.press("enter") + time.sleep(1.5) + return True + except Exception as e: + print(f"[open_app] ⚠️ macOS Spotlight failed: {e}") + return False + + + +def _launch_linux(app_name: str) -> bool: + binary = ( + shutil.which(app_name) or + shutil.which(app_name.lower()) or + shutil.which(app_name.lower().replace(" ", "-")) + ) + if binary: + try: + subprocess.Popen([binary], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + time.sleep(1.0) + return True + except Exception: + pass + + try: + subprocess.run(["xdg-open", app_name], capture_output=True, timeout=5) + return True + except Exception: + pass + + try: + desktop_name = app_name.lower().replace(" ", "-") + subprocess.run(["gtk-launch", desktop_name], capture_output=True, timeout=5) + return True + except Exception: + pass + + return False + + +_OS_LAUNCHERS = { + "Windows": _launch_windows, + "Darwin": _launch_macos, + "Linux": _launch_linux, +} + + +def open_app( + parameters=None, + response=None, + player=None, + session_memory=None, +) -> str: + app_name = (parameters or {}).get("app_name", "").strip() + + if not app_name: + return "Please specify which application to open, sir." + + system = platform.system() + launcher = _OS_LAUNCHERS.get(system) + + if launcher is None: + return f"Unsupported OS: {system}" + + normalized = _normalize(app_name) + print(f"[open_app] πŸš€ Launching: {app_name} β†’ {normalized} ({system})") + + if player: + player.write_log(f"[open_app] {app_name}") + + try: + success = launcher(normalized) + + if success: + return f"Opened {app_name} successfully, sir." + + if normalized != app_name: + success = launcher(app_name) + if success: + return f"Opened {app_name} successfully, sir." + + return ( + f"I tried to open {app_name}, sir, but couldn't confirm it launched. " + f"It may still be loading or might not be installed." + ) + + except Exception as e: + print(f"[open_app] ❌ {e}") + return f"Failed to open {app_name}, sir: {e}" \ No newline at end of file diff --git a/actions/reminder.py b/actions/reminder.py new file mode 100644 index 0000000..a5fae31 --- /dev/null +++ b/actions/reminder.py @@ -0,0 +1,156 @@ +# actions/reminder.py + +import subprocess +import os +import sys +from datetime import datetime + + +def reminder( + parameters: dict, + response: str | None = None, + player=None, + session_memory=None +) -> str: + """ + Sets a timed reminder using Windows Task Scheduler. + + parameters: + - date (str) YYYY-MM-DD + - time (str) HH:MM + - message (str) + + Returns a result string β€” Live API voices it automatically. + No edge_speak needed. + """ + + date_str = parameters.get("date") + time_str = parameters.get("time") + message = parameters.get("message", "Reminder") + + if not date_str or not time_str: + return "I need both a date and a time to set a reminder." + + try: + target_dt = datetime.strptime(f"{date_str} {time_str}", "%Y-%m-%d %H:%M") + + if target_dt <= datetime.now(): + return "That time is already in the past." + + task_name = f"MARKReminder_{target_dt.strftime('%Y%m%d_%H%M')}" + safe_message = message.replace('"', '').replace("'", "").strip()[:200] + + python_exe = sys.executable + if python_exe.lower().endswith("python.exe"): + pythonw = python_exe.replace("python.exe", "pythonw.exe") + if os.path.exists(pythonw): + python_exe = pythonw + + temp_dir = os.environ.get("TEMP", "C:\\Temp") + notify_script = os.path.join(temp_dir, f"{task_name}.pyw") + project_root = os.path.abspath( + os.path.join(os.path.dirname(__file__), "..") + ) + + script_code = f'''import sys, os, time +sys.path.insert(0, r"{project_root}") + +try: + import winsound + for freq in [800, 1000, 1200]: + winsound.Beep(freq, 200) + time.sleep(0.1) +except Exception: + pass + +try: + from win10toast import ToastNotifier + ToastNotifier().show_toast( + "MARK Reminder", + "{safe_message}", + duration=15, + threaded=False + ) +except Exception: + try: + import subprocess + subprocess.run(["msg", "*", "/TIME:30", "{safe_message}"], shell=True) + except Exception: + pass + +time.sleep(3) +try: + os.remove(__file__) +except Exception: + pass +''' + with open(notify_script, "w", encoding="utf-8") as f: + f.write(script_code) + + xml_content = f''' + + + MARK Reminder: {safe_message} + + + + {target_dt.strftime("%Y-%m-%dT%H:%M:%S")} + true + + + + + {python_exe} + "{notify_script}" + + + + IgnoreNew + false + false + true + true + PT5M + true + + + + InteractiveToken + LeastPrivilege + + +''' + + xml_path = os.path.join(temp_dir, f"{task_name}.xml") + with open(xml_path, "w", encoding="utf-16") as f: + f.write(xml_content) + + result = subprocess.run( + f'schtasks /Create /TN "{task_name}" /XML "{xml_path}" /F', + shell=True, capture_output=True, text=True + ) + + try: + os.remove(xml_path) + except Exception: + pass + + if result.returncode != 0: + err = result.stderr.strip() or result.stdout.strip() + print(f"[Reminder] ❌ schtasks failed: {err}") + try: + os.remove(notify_script) + except Exception: + pass + return "I couldn't schedule the reminder due to a system error." + + if player: + player.write_log(f"[reminder] set for {date_str} {time_str}") + + return f"Reminder set for {target_dt.strftime('%B %d at %I:%M %p')}." + + except ValueError: + return "I couldn't understand that date or time format." + + except Exception as e: + return f"Something went wrong while scheduling the reminder: {str(e)[:80]}" \ No newline at end of file diff --git a/actions/screen_processor.py b/actions/screen_processor.py new file mode 100644 index 0000000..fdea686 --- /dev/null +++ b/actions/screen_processor.py @@ -0,0 +1,367 @@ +import asyncio +import base64 +import io +import json +import re +import os +import sys +import time +import threading +import cv2 +import mss +import mss.tools +import sounddevice as sd +import numpy as np +from pathlib import Path + +try: + import PIL.Image + _PIL_OK = True +except ImportError: + _PIL_OK = False + +from google import genai +from google.genai import types + +def get_base_dir(): + if getattr(sys, "frozen", False): + return Path(sys.executable).parent + return Path(__file__).resolve().parent.parent + +BASE_DIR = get_base_dir() +API_CONFIG_PATH = BASE_DIR / "config" / "api_keys.json" + +LIVE_MODEL = "models/gemini-2.5-flash-native-audio-preview-12-2025" +CHANNELS = 1 +RECEIVE_SAMPLE_RATE = 24000 +CHUNK_SIZE = 1024 + +IMG_MAX_W = 640 +IMG_MAX_H = 360 +JPEG_Q = 55 + +SYSTEM_PROMPT = ( + "You are JARVIS from Iron Man movies. " + "Analyze images with technical precision and intelligence. " + "Help the user in a way they can understand β€” don't be overly complex. " + "Be concise, smart, and helpful like Tony Stark's AI assistant. " + "Respond in maximum 2 short sentences. Speed is priority. " + "Address the user as 'sir' for a tone of respect. " + "Ask if the user needs any further help with their problem." +) + + +def _get_api_key() -> str: + try: + with open(API_CONFIG_PATH, "r", encoding="utf-8") as f: + keys = json.load(f) + key = keys.get("gemini_api_key", "") + if not key: + raise ValueError("gemini_api_key not found") + return key + except Exception as e: + raise RuntimeError(f"Could not load API key: {e}") + + +def _get_camera_index() -> int: + try: + with open(API_CONFIG_PATH, "r", encoding="utf-8") as f: + cfg = json.load(f) + if "camera_index" in cfg: + return int(cfg["camera_index"]) + except Exception: + pass + + print("[Camera] πŸ” No camera index in config. Auto-detecting...") + best_index = 0 + + for idx in range(6): + cap = cv2.VideoCapture(idx, cv2.CAP_DSHOW) + if not cap.isOpened(): + cap.release() + continue + for _ in range(5): + cap.read() + ret, frame = cap.read() + cap.release() + if ret and frame is not None and frame.mean() > 5: + best_index = idx + print(f"[Camera] βœ… Camera found at index {idx} β€” saving to config.") + break + else: + print(f"[Camera] ⚠️ Index {idx}: no valid frame.") + + try: + cfg = {} + if API_CONFIG_PATH.exists(): + with open(API_CONFIG_PATH, "r", encoding="utf-8") as f: + cfg = json.load(f) + cfg["camera_index"] = best_index + with open(API_CONFIG_PATH, "w", encoding="utf-8") as f: + json.dump(cfg, f, indent=4) + print(f"[Camera] πŸ’Ύ Camera index {best_index} saved to config.") + except Exception as e: + print(f"[Camera] ⚠️ Could not save camera index: {e}") + + return best_index + + +def _to_jpeg(img_bytes: bytes) -> bytes: + if not _PIL_OK: + return img_bytes + img = PIL.Image.open(io.BytesIO(img_bytes)).convert("RGB") + img.thumbnail([IMG_MAX_W, IMG_MAX_H], PIL.Image.BILINEAR) + buf = io.BytesIO() + img.save(buf, format="JPEG", quality=JPEG_Q, optimize=False) + return buf.getvalue() + + +def _capture_screenshot() -> bytes: + with mss.mss() as sct: + shot = sct.grab(sct.monitors[1]) + png_bytes = mss.tools.to_png(shot.rgb, shot.size) + return _to_jpeg(png_bytes) + + +def _capture_camera() -> bytes: + camera_index = _get_camera_index() + cap = cv2.VideoCapture(camera_index, cv2.CAP_DSHOW) + if not cap.isOpened(): + raise RuntimeError(f"Camera could not be opened: index {camera_index}") + for _ in range(10): + cap.read() + ret, frame = cap.read() + cap.release() + if not ret or frame is None: + raise RuntimeError("Could not capture camera frame.") + if _PIL_OK: + rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) + img = PIL.Image.fromarray(rgb) + img.thumbnail([IMG_MAX_W, IMG_MAX_H], PIL.Image.BILINEAR) + buf = io.BytesIO() + img.save(buf, format="JPEG", quality=JPEG_Q, optimize=False) + return buf.getvalue() + _, buf = cv2.imencode(".jpg", frame, [cv2.IMWRITE_JPEG_QUALITY, JPEG_Q]) + return buf.tobytes() + + +class _LiveSession: + + def __init__(self): + self._loop: asyncio.AbstractEventLoop | None = None + self._thread: threading.Thread | None = None + self._session = None + self._out_queue: asyncio.Queue | None = None + self._audio_in: asyncio.Queue | None = None + self._ready: threading.Event = threading.Event() + self._player = None + self._send_lock: asyncio.Lock | None = None + + def start(self, player=None): + if self._thread and self._thread.is_alive(): + return + self._player = player + self._thread = threading.Thread( + target=self._run_loop, daemon=True, name="VisionSessionThread" + ) + self._thread.start() + ok = self._ready.wait(timeout=20) + if not ok: + raise RuntimeError("Vision session did not start within 20s.") + print("[ScreenProcess] βœ… Vision session ready (no mic)") + + def _run_loop(self): + self._loop = asyncio.new_event_loop() + asyncio.set_event_loop(self._loop) + self._loop.run_until_complete(self._main()) + + async def _main(self): + self._out_queue = asyncio.Queue(maxsize=30) + self._audio_in = asyncio.Queue() + self._send_lock = asyncio.Lock() + + client = genai.Client( + api_key=_get_api_key(), + http_options={"api_version": "v1beta"} + ) + + config = types.LiveConnectConfig( + response_modalities=["AUDIO"], + output_audio_transcription={}, + system_instruction=SYSTEM_PROMPT, + speech_config=types.SpeechConfig( + voice_config=types.VoiceConfig( + prebuilt_voice_config=types.PrebuiltVoiceConfig( + voice_name="Charon" + ) + ) + ), + ) + + while True: + try: + print("[ScreenProcess] πŸ”Œ Vision session connecting...") + async with client.aio.live.connect(model=LIVE_MODEL, config=config) as session: + self._session = session + self._ready.set() + print("[ScreenProcess] βœ… Vision session connected") + async with asyncio.TaskGroup() as tg: + tg.create_task(self._send_loop()) + tg.create_task(self._recv_loop()) + tg.create_task(self._play_loop()) + except Exception as e: + print(f"[ScreenProcess] ⚠️ Disconnected: {e} β€” reconnecting...") + self._session = None + self._ready.clear() + await asyncio.sleep(2) + self._ready.set() + + async def _send_loop(self): + while True: + item = await self._out_queue.get() + if self._session: + image_bytes, mime_type, user_text = item + try: + b64 = base64.b64encode(image_bytes).decode("utf-8") + await self._session.send_client_content( + turns={ + "parts": [ + {"inline_data": {"mime_type": mime_type, "data": b64}}, + {"text": user_text} + ] + }, + turn_complete=True + ) + print("[ScreenProcess] βœ… Image sent") + except Exception as e: + print(f"[ScreenProcess] ⚠️ Send error: {e}") + + async def _recv_loop(self): + transcript_buf: list[str] = [] + try: + async for response in self._session.receive(): + if response.data: + await self._audio_in.put(response.data) + sc = response.server_content + if not sc: + continue + if sc.output_transcription and sc.output_transcription.text: + chunk = sc.output_transcription.text.strip() + if chunk: + transcript_buf.append(chunk) + if sc.turn_complete: + if transcript_buf and self._player: + full = re.sub(r'\s+', ' ', " ".join(transcript_buf)).strip() + if full: + self._player.write_log(f"Jarvis: {full}") + print(f"[ScreenProcess] πŸ’¬ {full}") + transcript_buf = [] + except Exception as e: + print(f"[ScreenProcess] ⚠️ Recv error: {e}") + transcript_buf = [] + await asyncio.sleep(0.3) + + async def _play_loop(self): + stream = sd.RawOutputStream( + samplerate=RECEIVE_SAMPLE_RATE, + channels=CHANNELS, + dtype="int16", + blocksize=CHUNK_SIZE, + ) + stream.start() + try: + while True: + chunk = await self._audio_in.get() + await asyncio.to_thread(stream.write, chunk) + except Exception as e: + print(f"[ScreenProcess] ❌ Play error: {e}") + raise + finally: + stream.stop() + stream.close() + + def analyze(self, image_bytes: bytes, mime_type: str, user_text: str): + if not self._loop: + return + asyncio.run_coroutine_threadsafe( + self._out_queue.put((image_bytes, mime_type, user_text)), + self._loop + ) + + def is_ready(self) -> bool: + return self._session is not None + + +_live = _LiveSession() +_started = False +_start_lock = threading.Lock() + + +def _ensure_started(player=None): + global _started + with _start_lock: + if not _started: + _live.start(player=player) + _started = True + elif player is not None: + _live._player = player + + +def screen_process( + parameters: dict, + response: str | None = None, + player=None, + session_memory=None, +) -> bool: + user_text = (parameters or {}).get("text") or (parameters or {}).get("user_text", "") + user_text = (user_text or "").strip() + if not user_text: + print("[ScreenProcess] ⚠️ No user_text provided.") + return False + + angle = (parameters or {}).get("angle", "screen").lower().strip() + print(f"[ScreenProcess] angle={angle!r} text={user_text!r}") + + _ensure_started(player=player) + + try: + if angle == "camera": + image_bytes = _capture_camera() + mime_type = "image/jpeg" + print("[ScreenProcess] πŸ“· Camera captured") + else: + image_bytes = _capture_screenshot() + mime_type = "image/jpeg" if _PIL_OK else "image/png" + print("[ScreenProcess] πŸ–₯️ Screen captured") + except Exception as e: + import traceback; traceback.print_exc() + print(f"[ScreenProcess] ❌ Capture error: {e}") + return False + + print(f"[ScreenProcess] πŸ“¦ {len(image_bytes)} bytes β†’ sending") + _live.analyze(image_bytes, mime_type, user_text) + return True + + +def warmup_session(player=None): + try: + _ensure_started(player=player) + except Exception as e: + print(f"[ScreenProcess] ⚠️ Warmup error: {e}") + + +if __name__ == "__main__": + print("[TEST] screen_processor.py v8 β€” image-only session") + print("=" * 50) + mode = input("screen / camera (default: screen): ").strip().lower() or "screen" + request = input("Question (Enter for default): ").strip() or "What do you see? Be brief." + + t0 = time.perf_counter() + warmup_session() + print(f"Session ready β€” {time.perf_counter()-t0:.2f}s\n") + + t1 = time.perf_counter() + result = screen_process({"angle": mode, "text": request}, player=None) + print(f"Sent β€” {time.perf_counter()-t1:.3f}s | audio incoming...") + time.sleep(8) + print(f"\n{'βœ…' if result else '❌'}") diff --git a/actions/send_message.py b/actions/send_message.py new file mode 100644 index 0000000..7708290 --- /dev/null +++ b/actions/send_message.py @@ -0,0 +1,215 @@ +# actions/send_message.py +# Universal messaging β€” WhatsApp & Instagram +# Uses visual element detection (pyautogui + screen search) instead of +# hardcoded tab/click sequences β€” works on any screen resolution. + +import time +import pyautogui +from pathlib import Path + +pyautogui.FAILSAFE = True +pyautogui.PAUSE = 0.08 + +def _open_app(app_name: str) -> bool: + """Opens an app via Windows search.""" + try: + pyautogui.press("win") + time.sleep(0.4) + pyautogui.write(app_name, interval=0.04) + time.sleep(0.5) + pyautogui.press("enter") + time.sleep(2.0) + return True + except Exception as e: + print(f"[SendMessage] Could not open {app_name}: {e}") + return False + + +def _search_contact(contact: str, platform: str): + """ + Searches for a contact inside the messaging app. + Uses Ctrl+F (universal search shortcut) then types contact name. + """ + time.sleep(0.5) + pyautogui.hotkey("ctrl", "f") + time.sleep(0.4) + pyautogui.hotkey("ctrl", "a") + pyautogui.write(contact, interval=0.04) + time.sleep(0.8) + pyautogui.press("enter") + time.sleep(0.6) + + +def _type_and_send(message: str): + """Types message and sends it.""" + pyautogui.press("tab") + time.sleep(0.2) + pyautogui.hotkey("ctrl", "a") + pyautogui.write(message, interval=0.03) + time.sleep(0.2) + pyautogui.press("enter") + time.sleep(0.3) + + +def _send_whatsapp(receiver: str, message: str) -> str: + """ + Sends a WhatsApp message via the Windows desktop app. + Steps: Open WhatsApp β†’ Search contact β†’ Click β†’ Type β†’ Send + """ + try: + if not _open_app("WhatsApp"): + return "Could not open WhatsApp." + + time.sleep(1.5) + + pyautogui.hotkey("ctrl", "f") + time.sleep(0.4) + pyautogui.hotkey("ctrl", "a") + pyautogui.write(receiver, interval=0.04) + time.sleep(1.0) + + pyautogui.press("enter") + time.sleep(0.8) + + pyautogui.write(message, interval=0.03) + time.sleep(0.2) + pyautogui.press("enter") + + return f"Message sent to {receiver} via WhatsApp." + + except Exception as e: + return f"WhatsApp error: {e}" + + +def _send_instagram(receiver: str, message: str) -> str: + """ + Sends an Instagram DM via browser (instagram.com). + Steps: Open Chrome β†’ Go to instagram.com/direct β†’ Search contact β†’ Send + """ + try: + import webbrowser + + webbrowser.open("https://www.instagram.com/direct/new/") + time.sleep(3.5) + + pyautogui.write(receiver, interval=0.05) + time.sleep(1.5) + + pyautogui.press("down") + time.sleep(0.3) + pyautogui.press("enter") + time.sleep(0.5) + + for _ in range(3): + pyautogui.press("tab") + time.sleep(0.1) + pyautogui.press("enter") + time.sleep(1.5) + + pyautogui.write(message, interval=0.04) + time.sleep(0.2) + pyautogui.press("enter") + + return f"Message sent to {receiver} via Instagram." + + except Exception as e: + return f"Instagram error: {e}" + +def _send_telegram(receiver: str, message: str) -> str: + """Sends a Telegram message via Windows desktop app.""" + try: + if not _open_app("Telegram"): + return "Could not open Telegram." + + time.sleep(1.5) + + pyautogui.hotkey("ctrl", "f") + time.sleep(0.4) + pyautogui.write(receiver, interval=0.04) + time.sleep(1.0) + pyautogui.press("enter") + time.sleep(0.8) + + pyautogui.write(message, interval=0.03) + time.sleep(0.2) + pyautogui.press("enter") + + return f"Message sent to {receiver} via Telegram." + + except Exception as e: + return f"Telegram error: {e}" + + + +def _send_generic(platform: str, receiver: str, message: str) -> str: + """ + For any other platform not explicitly supported. + Opens the app, searches for contact, types and sends. + Works for: Messenger, Discord, Signal, etc. + """ + try: + if not _open_app(platform): + return f"Could not open {platform}." + + time.sleep(1.5) + pyautogui.hotkey("ctrl", "f") + time.sleep(0.4) + pyautogui.write(receiver, interval=0.04) + time.sleep(1.0) + pyautogui.press("enter") + time.sleep(0.8) + pyautogui.write(message, interval=0.03) + time.sleep(0.2) + pyautogui.press("enter") + + return f"Message sent to {receiver} via {platform}." + + except Exception as e: + return f"{platform} error: {e}" + +def send_message( + parameters: dict, + response=None, + player=None, + session_memory=None +) -> str: + """ + Called from main.py. + + parameters: + receiver : Contact name to send to + message_text : The message content + platform : whatsapp | instagram | telegram | + Default: whatsapp + """ + params = parameters or {} + receiver = params.get("receiver", "").strip() + message_text = params.get("message_text", "").strip() + platform = params.get("platform", "whatsapp").strip().lower() + + if not receiver: + return "Please specify who to send the message to, sir." + if not message_text: + return "Please specify what message to send, sir." + + print(f"[SendMessage] πŸ“¨ {platform} β†’ {receiver}: {message_text[:40]}") + if player: + player.write_log(f"[msg] Sending to {receiver} via {platform}...") + + if "whatsapp" in platform or "wp" in platform or "wapp" in platform: + result = _send_whatsapp(receiver, message_text) + + elif "instagram" in platform or "ig" in platform or "insta" in platform: + result = _send_instagram(receiver, message_text) + + elif "telegram" in platform or "tg" in platform: + result = _send_telegram(receiver, message_text) + + else: + result = _send_generic(platform, receiver, message_text) + + print(f"[SendMessage] βœ… {result}") + if player: + player.write_log(f"[msg] {result}") + + return result \ No newline at end of file diff --git a/actions/weather_report.py b/actions/weather_report.py new file mode 100644 index 0000000..e49d817 --- /dev/null +++ b/actions/weather_report.py @@ -0,0 +1,62 @@ +# actions/weather_report.py + +import webbrowser +from urllib.parse import quote_plus + + +def weather_action( + parameters: dict, + player=None, + session_memory=None +): + """ + Weather report action. + Opens a Google weather search and gives a short spoken confirmation. + """ + + city = parameters.get("city") + time = parameters.get("time") + if not city or not isinstance(city, str): + msg = "Sir, the city is missing for the weather report." + _speak_and_log(msg, player) + return msg + + city = city.strip() + + if not time or not isinstance(time, str): + time = "today" + else: + time = time.strip() + + search_query = f"weather in {city} {time}" + encoded_query = quote_plus(search_query) + url = f"https://www.google.com/search?q={encoded_query}" + + try: + webbrowser.open(url) + except Exception: + msg = f"Sir, I couldn't open the browser for the weather report." + _speak_and_log(msg, player) + return msg + + msg = f"Showing the weather for {city}, {time}, sir." + _speak_and_log(msg, player) + + if session_memory: + try: + session_memory.set_last_search( + query=search_query, + response=msg + ) + except Exception: + pass + + return msg + + +def _speak_and_log(message: str, player=None): + if player: + try: + player.write_log(f"JARVIS: {message}") + except Exception: + pass \ No newline at end of file diff --git a/actions/web_search.py b/actions/web_search.py new file mode 100644 index 0000000..0be6d42 --- /dev/null +++ b/actions/web_search.py @@ -0,0 +1,137 @@ +#web_search.py +import json +import sys +from pathlib import Path + +def _get_base_dir() -> Path: + if getattr(sys, "frozen", False): + return Path(sys.executable).parent + return Path(__file__).resolve().parent.parent + + +BASE_DIR = _get_base_dir() +API_CONFIG_PATH = BASE_DIR / "config" / "api_keys.json" + + +def _get_api_key() -> str: + with open(API_CONFIG_PATH, "r", encoding="utf-8") as f: + return json.load(f)["gemini_api_key"] + + +def _gemini_search(query: str) -> str: + from google import genai + + client = genai.Client(api_key=_get_api_key()) + response = client.models.generate_content( + model="gemini-2.5-flash", + contents=query, + config={"tools": [{"google_search": {}}]}, + ) + + text = "" + for part in response.candidates[0].content.parts: + if hasattr(part, "text") and part.text: + text += part.text + + text = text.strip() + if not text: + raise ValueError("Gemini returned an empty response.") + return text + + +def _ddg_search(query: str, max_results: int = 6) -> list[dict]: + try: + from ddgs import DDGS + except ImportError: + from duckduckgo_search import DDGS + + results = [] + with DDGS() as ddgs: + for r in ddgs.text(query, max_results=max_results): + results.append({ + "title": r.get("title", ""), + "snippet": r.get("body", ""), + "url": r.get("href", ""), + }) + return results + + +def _format_ddg(query: str, results: list[dict]) -> str: + if not results: + return f"No results found for: {query}" + + lines = [f"Search results for: {query}\n"] + for i, r in enumerate(results, 1): + if r.get("title"): lines.append(f"{i}. {r['title']}") + if r.get("snippet"): lines.append(f" {r['snippet']}") + if r.get("url"): lines.append(f" {r['url']}") + lines.append("") + return "\n".join(lines).strip() + +def _compare(items: list[str], aspect: str) -> str: + query = ( + f"Compare {', '.join(items)} in terms of {aspect}. " + "Give specific facts and data." + ) + try: + return _gemini_search(query) + except Exception as e: + print(f"[WebSearch] ⚠️ Gemini compare failed: {e} β€” falling back to DDG") + + # DDG fallback: fetch results per item and merge + all_results: dict[str, list] = {} + for item in items: + try: + all_results[item] = _ddg_search(f"{item} {aspect}", max_results=3) + except Exception: + all_results[item] = [] + + lines = [f"Comparison β€” {aspect.upper()}", "─" * 40] + for item in items: + lines.append(f"\nβ–Έ {item}") + for r in all_results.get(item, [])[:2]: + if r.get("snippet"): + lines.append(f" β€’ {r['snippet']}") + return "\n".join(lines) + +def web_search( + parameters: dict, + response=None, + player=None, + session_memory=None, +) -> str: + params = parameters or {} + query = params.get("query", "").strip() + mode = params.get("mode", "search").lower().strip() + items = params.get("items", []) + aspect = params.get("aspect", "general").strip() or "general" + + if not query and not items: + return "Please provide a search query, sir." + + if items and mode != "compare": + mode = "compare" + + if player: + player.write_log(f"[Search] {query or ', '.join(items)}") + + print(f"[WebSearch] πŸ” Query: {query!r} Mode: {mode}") +# replace: result = _gemini_search(query) block with: + try: + from or_client import client + result = client.chat( + query, + system="You are a web search assistant. Answer factually and concisely." + ) + print("[WebSearch] βœ… OpenRouter OK.") + return result + except Exception as e: + print(f"[WebSearch] ⚠️ OpenRouter failed ({e}) β€” trying DDG...") + results = _ddg_search(query) + result = _format_ddg(query, results) + print(f"[WebSearch] βœ… DDG: {len(results)} result(s).") + return result + + except Exception as e: + print(f"[WebSearch] ❌ All backends failed: {e}") + return f"Search failed, sir: {e}" \ No newline at end of file diff --git a/actions/youtube_video.py b/actions/youtube_video.py new file mode 100644 index 0000000..1289ecb --- /dev/null +++ b/actions/youtube_video.py @@ -0,0 +1,418 @@ +#youtube_video.py +import json +import re +import sys +import time +import subprocess +import shutil +from pathlib import Path +from datetime import datetime +from urllib.parse import quote_plus + +import pyautogui +import numpy as np + +try: + import requests + _REQUESTS_OK = True +except ImportError: + _REQUESTS_OK = False + +try: + from youtube_transcript_api import YouTubeTranscriptApi + _TRANSCRIPT_OK = True +except ImportError: + _TRANSCRIPT_OK = False + +from config import get_os, is_windows, is_mac, is_linux + + +def _get_base_dir() -> Path: + if getattr(sys, "frozen", False): + return Path(sys.executable).parent + return Path(__file__).resolve().parent.parent + + +BASE_DIR = _get_base_dir() +API_CONFIG_PATH = BASE_DIR / "config" / "api_keys.json" + +HEADERS = { + "User-Agent": ( + "Mozilla/5.0 (X11; Linux x86_64) " + "AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/124.0.0.0 Safari/537.36" + ), + "Accept-Language": "en-US,en;q=0.9", +} + +_YT_VIDEO_FILTER = "EgIQAQ%3D%3D" + +def _open_url(url: str) -> None: + try: + if is_mac(): + subprocess.Popen(["open", url]) + elif is_linux(): + subprocess.Popen(["xdg-open", url]) + else: + subprocess.Popen(["cmd", "/c", "start", "", url], shell=False) + except Exception as e: + print(f"[YouTube] ⚠️ open_url failed: {e}") + +def _scrape_first_video_url(query: str) -> str | None: + + if not _REQUESTS_OK: + return None + + search_url = ( + f"https://www.youtube.com/results" + f"?search_query={quote_plus(query)}" + f"&sp={_YT_VIDEO_FILTER}" + ) + + try: + r = requests.get(search_url, headers=HEADERS, timeout=10) + html = r.text + + video_ids = re.findall(r'"videoId":"([A-Za-z0-9_-]{11})"', html) + + seen = set() + for vid in video_ids: + if vid in seen: + continue + seen.add(vid) + + if f'/shorts/{vid}' in html: + continue + return f"https://www.youtube.com/watch?v={vid}" + + except Exception as e: + print(f"[YouTube] ⚠️ scrape_first_video_url failed: {e}") + + return None + +def _extract_video_id(url: str) -> str | None: + match = re.search( + r"(?:v=|\/v\/|youtu\.be\/|\/embed\/|\/shorts\/)([A-Za-z0-9_-]{11})", url + ) + return match.group(1) if match else None + + +def _is_valid_youtube_url(url: str) -> bool: + return bool(re.search(r"(youtube\.com|youtu\.be)", url or "")) + + +def _ask_for_url(prompt_text: str = "YouTube video URL:") -> str | None: + try: + import tkinter as tk + from tkinter import simpledialog + + root = tk._default_root + if root is None: + root = tk.Tk() + root.withdraw() + + url = simpledialog.askstring("J.A.R.V.I.S", prompt_text, parent=root) + return url.strip() if url else None + except Exception as e: + print(f"[YouTube] ⚠️ URL dialog failed: {e}") + return None + + +def _get_transcript(video_id: str) -> str | None: + if not _TRANSCRIPT_OK: + return None + try: + transcript_list = YouTubeTranscriptApi.list_transcripts(video_id) + transcript = None + + lang_priority = ["en", "tr", "de", "fr", "es", "it", "pt", "ru", "ja", "ko", "ar", "zh"] + + try: + transcript = transcript_list.find_manually_created_transcript(lang_priority) + except Exception: + pass + + if transcript is None: + try: + transcript = transcript_list.find_generated_transcript(lang_priority) + except Exception: + for t in transcript_list: + transcript = t + break + + if transcript is None: + return None + + fetched = transcript.fetch() + return " ".join(entry["text"] for entry in fetched) + + except Exception as e: + print(f"[YouTube] ⚠️ Transcript fetch failed: {e}") + return None + + +def _summarize_with_gemini(transcript: str, video_url: str) -> str: + from or_client import client + + max_chars = 80000 + truncated = transcript[:max_chars] + ("..." if len(transcript) > max_chars else "") + + return client.chat( + f"Please summarize this YouTube video transcript:\n\n{truncated}", + system=( + "You are JARVIS, an AI assistant. " + "Summarize YouTube video transcripts clearly and concisely. " + "Structure: 1-sentence overview, then 3-5 key points. " + "Be direct. Address the user as 'sir'. " + "Match the language of the transcript." + ), + max_tokens=2048, + ) + + +def _save_summary(content: str, video_url: str) -> str: + ts = datetime.now().strftime("%Y%m%d_%H%M%S") + filename = f"youtube_summary_{ts}.txt" + desktop = Path.home() / "Desktop" + desktop.mkdir(parents=True, exist_ok=True) + filepath = desktop / filename + + header = ( + f"JARVIS β€” YouTube Summary\n" + f"{'─' * 50}\n" + f"URL : {video_url}\n" + f"Date : {datetime.now().strftime('%Y-%m-%d %H:%M')}\n" + f"{'─' * 50}\n\n" + ) + filepath.write_text(header + content, encoding="utf-8") + + try: + if is_windows(): + subprocess.Popen(["notepad.exe", str(filepath)]) + elif is_mac(): + subprocess.Popen(["open", "-t", str(filepath)]) + else: + subprocess.Popen(["xdg-open", str(filepath)]) + except Exception as e: + print(f"[YouTube] ⚠️ Could not open text editor: {e}") + + return str(filepath) + + +def _scrape_video_info(video_id: str) -> dict: + if not _REQUESTS_OK: + return {} + url = f"https://www.youtube.com/watch?v={video_id}" + try: + r = requests.get(url, headers=HEADERS, timeout=12) + html = r.text + info = {} + + for key, pattern in [ + ("title", r'"title":\{"runs":\[\{"text":"([^"]+)"'), + ("channel", r'"ownerChannelName":"([^"]+)"'), + ("views", r'"viewCount":"(\d+)"'), + ("duration", r'"lengthSeconds":"(\d+)"'), + ("likes", r'"label":"([0-9,]+ likes)"'), + ]: + match = re.search(pattern, html) + if match: + raw = match.group(1) + if key == "views": + info[key] = f"{int(raw):,}" + elif key == "duration": + secs = int(raw) + info[key] = f"{secs // 60}:{secs % 60:02d}" + else: + info[key] = raw + + return info + except Exception as e: + print(f"[YouTube] ⚠️ Info scrape failed: {e}") + return {} + + +def _scrape_trending(region: str = "TR", max_results: int = 8) -> list[dict]: + if not _REQUESTS_OK: + return [] + url = f"https://www.youtube.com/feed/trending?gl={region.upper()}" + try: + r = requests.get(url, headers=HEADERS, timeout=12) + html = r.text + + titles = re.findall(r'"title":\{"runs":\[\{"text":"([^"]+)"\}\]', html) + channels = re.findall(r'"ownerText":\{"runs":\[\{"text":"([^"]+)"', html) + + results, seen = [], set() + for i, title in enumerate(titles): + if title in seen or len(title) < 5: + continue + seen.add(title) + channel = channels[i] if i < len(channels) else "Unknown" + results.append({"rank": len(results) + 1, "title": title, "channel": channel}) + if len(results) >= max_results: + break + + return results + except Exception as e: + print(f"[YouTube] ⚠️ Trending scrape failed: {e}") + return [] + +def _handle_play(parameters: dict, player) -> str: + query = parameters.get("query", "").strip() + if not query: + return "Please tell me what you'd like to watch, sir." + + if player: + player.write_log(f"[YouTube] Searching: {query}") + + print(f"[YouTube] πŸ” Scraping first non-Shorts video for: {query}") + + video_url = _scrape_first_video_url(query) + + if video_url: + print(f"[YouTube] ▢️ Opening: {video_url}") + _open_url(video_url) + return f"Playing: {query}" + + print(f"[YouTube] ⚠️ Scrape failed, opening filtered search page") + fallback_url = ( + f"https://www.youtube.com/results" + f"?search_query={quote_plus(query)}" + f"&sp={_YT_VIDEO_FILTER}" + ) + _open_url(fallback_url) + return f"Opened YouTube search for: {query} (manual selection required)" + + +def _handle_summarize(parameters: dict, player, speak) -> str: + if not _TRANSCRIPT_OK: + return "youtube-transcript-api is not installed. Run: pip install youtube-transcript-api" + + url = _ask_for_url("Please paste the YouTube video URL:") + if not url: + return "No URL provided, sir. Summary cancelled." + if not _is_valid_youtube_url(url): + return "That doesn't appear to be a valid YouTube URL, sir." + + video_id = _extract_video_id(url) + if not video_id: + return "Could not extract video ID from that URL, sir." + + if player: + player.write_log(f"[YouTube] Summarizing: {url}") + if speak: + speak("Fetching the transcript now, sir. One moment.") + + transcript = _get_transcript(video_id) + if not transcript: + return "I couldn't retrieve a transcript for that video, sir." + + if speak: + speak("Transcript retrieved. Generating summary now.") + + try: + summary = _summarize_with_gemini(transcript, url) + except Exception as e: + return f"Summary generation failed, sir: {e}" + + if speak: + speak(summary) + + if parameters.get("save", False): + saved_path = _save_summary(summary, url) + return f"Summary complete and saved to Desktop: {saved_path}" + + return summary + + +def _handle_get_info(parameters: dict, player, speak) -> str: + url = parameters.get("url", "").strip() + if not url: + url = _ask_for_url("Please paste the YouTube video URL:") + if not url or not _is_valid_youtube_url(url): + return "Please provide a valid YouTube URL, sir." + + video_id = _extract_video_id(url) + if not video_id: + return "Could not extract video ID, sir." + + if player: + player.write_log(f"[YouTube] Getting info: {url}") + + info = _scrape_video_info(video_id) + if not info: + return "Could not retrieve video information, sir." + + lines = [ + f"{key.capitalize()}: {info[key]}" + for key in ("title", "channel", "views", "duration", "likes") + if key in info + ] + result = "\n".join(lines) + + if speak: + speak(f"Here's the video info, sir. {result.replace(chr(10), '. ')}") + + return result + + +def _handle_trending(parameters: dict, player, speak) -> str: + region = parameters.get("region", "TR").upper() + + if player: + player.write_log(f"[YouTube] Trending: {region}") + + trending = _scrape_trending(region=region, max_results=8) + if not trending: + return f"Could not fetch trending videos for region {region}, sir." + + lines = [f"Top trending videos in {region}:"] + lines += [f"{v['rank']}. {v['title']} β€” {v['channel']}" for v in trending] + result = "\n".join(lines) + + if speak: + top3 = trending[:3] + spoken = "Here are the top trending videos, sir. " + ". ".join( + f"Number {v['rank']}: {v['title']} by {v['channel']}" for v in top3 + ) + speak(spoken) + + return result + +_ACTION_MAP = { + "play": _handle_play, + "summarize": _handle_summarize, + "get_info": _handle_get_info, + "trending": _handle_trending, +} + + +def youtube_video( + parameters: dict, + response=None, + player=None, + session_memory=None, + speak=None, +) -> str: + params = parameters or {} + action = params.get("action", "play").lower().strip() + + if player: + player.write_log(f"[YouTube] Action: {action}") + print(f"[YouTube] ▢️ Action: {action} Params: {params}") + + handler = _ACTION_MAP.get(action) + if handler is None: + return ( + f"Unknown YouTube action: '{action}'. " + "Available: play, summarize, get_info, trending." + ) + + try: + if action == "play": + return handler(params, player) or "Done." + return handler(params, player, speak) or "Done." + except Exception as e: + print(f"[YouTube] ❌ Error in {action}: {e}") + return f"YouTube {action} failed, sir: {e}" \ No newline at end of file diff --git a/agent/error_handler.py b/agent/error_handler.py new file mode 100644 index 0000000..2ef61c7 --- /dev/null +++ b/agent/error_handler.py @@ -0,0 +1,197 @@ +import json +import re +import sys +from pathlib import Path +from enum import Enum + + +def get_base_dir() -> Path: + if getattr(sys, "frozen", False): + return Path(sys.executable).parent + return Path(__file__).resolve().parent.parent + + +BASE_DIR = get_base_dir() +API_CONFIG_PATH = BASE_DIR / "config" / "api_keys.json" + + +class ErrorDecision(Enum): + RETRY = "retry" + SKIP = "skip" + REPLAN = "replan" + ABORT = "abort" + + +ERROR_ANALYST_PROMPT = """You are the error recovery module of MARK XXV AI assistant. + +A task step has failed. Analyze the error and decide what to do. + +DECISIONS: +- retry : Transient error (network timeout, temporary file lock, race condition). + The same step can succeed if tried again. +- skip : This step is not critical and the task can succeed without it. +- replan : The approach was wrong. A different tool or method should be tried. +- abort : The task is fundamentally impossible or unsafe to continue. + +Also provide: +- A brief explanation of WHY it failed (1 sentence) +- A fix suggestion if decision is replan (what to try instead) +- Max retries: how many times to retry if decision is retry (1 or 2) + +Return ONLY valid JSON: +{ + "decision": "retry|skip|replan|abort", + "reason": "why it failed", + "fix_suggestion": "what to try instead (for replan)", + "max_retries": 1, + "user_message": "Short message to tell the user (max 15 words)" +} +""" + + +def _get_api_key() -> str: + with open(API_CONFIG_PATH, "r", encoding="utf-8") as f: + return json.load(f)["gemini_api_key"] + + +def analyze_error( + step: dict, + error: str, + attempt: int = 1, + max_attempts: int = 2 +) -> dict: + """ + Analyzes a failed step and returns a recovery decision. + + Args: + step : The step dict that failed + error : Error message/traceback + attempt : Current attempt number + max_attempts : How many times we've already tried + + Returns: + { + "decision": ErrorDecision, + "reason": str, + "fix_suggestion": str, + "max_retries": int, + "user_message": str + } + """ + import google.generativeai as genai + + if attempt >= max_attempts: + print(f"[ErrorHandler] ⚠️ Max attempts reached for step {step.get('step')} β€” forcing replan") + return { + "decision": ErrorDecision.REPLAN, + "reason": f"Failed {attempt} times: {error[:100]}", + "fix_suggestion": "Try a completely different approach or tool", + "max_retries": 0, + "user_message": "Trying a different approach, sir." + } + + genai.configure(api_key=_get_api_key()) + model = genai.GenerativeModel( + model_name="gemini-2.5-flash-lite", + system_instruction=ERROR_ANALYST_PROMPT + ) + + prompt = f"""Failed step: +Tool: {step.get('tool')} +Description: {step.get('description')} +Parameters: {json.dumps(step.get('parameters', {}), indent=2)} +Critical: {step.get('critical', False)} + +Error: +{error[:500]} + +Attempt number: {attempt}""" + + try: + response = model.generate_content(prompt) + text = response.text.strip() + text = re.sub(r"```(?:json)?", "", text).strip().rstrip("`").strip() + + result = json.loads(text) + decision_str = result.get("decision", "replan").lower() + decision_map = { + "retry": ErrorDecision.RETRY, + "skip": ErrorDecision.SKIP, + "replan": ErrorDecision.REPLAN, + "abort": ErrorDecision.ABORT, + } + result["decision"] = decision_map.get(decision_str, ErrorDecision.REPLAN) + + + if step.get("critical") and result["decision"] == ErrorDecision.SKIP: + result["decision"] = ErrorDecision.REPLAN + result["user_message"] = "This step is critical β€” finding alternative approach, sir." + + print(f"[ErrorHandler] Decision: {result['decision'].value} β€” {result.get('reason', '')}") + return result + + except Exception as e: + print(f"[ErrorHandler] ⚠️ Analysis failed: {e} β€” defaulting to replan") + return { + "decision": ErrorDecision.REPLAN, + "reason": str(e), + "fix_suggestion": "Try alternative approach", + "max_retries": 1, + "user_message": "Encountered an issue, adjusting approach, sir." + } + + +def generate_fix(step: dict, error: str, fix_suggestion: str) -> dict: + """ + When decision is REPLAN and a fix suggestion exists, + generates a replacement step using generated_code as fallback. + + Returns a modified step dict. + """ + import google.generativeai as genai + + genai.configure(api_key=_get_api_key()) + model = genai.GenerativeModel(model_name="gemini-2.0-flash") + + prompt = f"""A task step failed. Generate a replacement step. + +Original step: +Tool: {step.get('tool')} +Description: {step.get('description')} +Parameters: {json.dumps(step.get('parameters', {}), indent=2)} + +Error: {error[:300]} +Fix suggestion: {fix_suggestion} + +Write a Python script that accomplishes the same goal differently. +Return ONLY the Python code, no explanation.""" + + try: + response = model.generate_content(prompt) + code = response.text.strip() + code = re.sub(r"```(?:python)?", "", code).strip().rstrip("`").strip() + + return { + "step": step.get("step"), + "tool": "code_helper", + "description": f"Auto-fix for: {step.get('description')}", + "parameters": { + "action": "run", + "description": fix_suggestion, + "code": code, + "language": "python" + }, + "depends_on": step.get("depends_on", []), + "critical": step.get("critical", False) + } + + except Exception as e: + print(f"[ErrorHandler] ⚠️ Fix generation failed: {e}") + return { + "step": step.get("step"), + "tool": "generated_code", + "description": f"Fallback for: {step.get('description')}", + "parameters": {"description": step.get("description", "")}, + "depends_on": step.get("depends_on", []), + "critical": step.get("critical", False) + } \ No newline at end of file diff --git a/agent/executor.py b/agent/executor.py new file mode 100644 index 0000000..4c89fad --- /dev/null +++ b/agent/executor.py @@ -0,0 +1,400 @@ +import json +import re +import sys +import threading +import subprocess +import tempfile +import os +from pathlib import Path +from typing import Callable + +from agent.planner import create_plan, replan +from agent.error_handler import analyze_error, generate_fix, ErrorDecision + + +def get_base_dir() -> Path: + if getattr(sys, "frozen", False): + return Path(sys.executable).parent + return Path(__file__).resolve().parent.parent + + +BASE_DIR = get_base_dir() +API_CONFIG_PATH = BASE_DIR / "config" / "api_keys.json" + + +def _get_api_key() -> str: + with open(API_CONFIG_PATH, "r", encoding="utf-8") as f: + return json.load(f)["gemini_api_key"] + +def _run_generated_code(description: str, speak: Callable | None = None) -> str: + import google.generativeai as genai + + if speak: + speak("Writing custom code for this task, sir.") + + home = Path.home() + desktop = home / "Desktop" + downloads = home / "Downloads" + documents = home / "Documents" + + if not desktop.exists(): + try: + import winreg + key = winreg.OpenKey(winreg.HKEY_CURRENT_USER, + r"Software\Microsoft\Windows\CurrentVersion\Explorer\Shell Folders") + desktop = Path(winreg.QueryValueEx(key, "Desktop")[0]) + except Exception: + pass + + genai.configure(api_key=_get_api_key()) + model = genai.GenerativeModel( + model_name="gemini-2.5-flash", + system_instruction=( + "You are an expert Python developer. " + "Write clean, complete, working Python code. " + "Use standard library + common packages. " + "Install missing packages with subprocess + pip if needed. " + "Return ONLY the Python code. No explanation, no markdown, no backticks.\n\n" + f"SYSTEM PATHS:\n" + f" Desktop = r'{desktop}'\n" + f" Downloads = r'{downloads}'\n" + f" Documents = r'{documents}'\n" + f" Home = r'{home}'\n" + ) + ) + + try: + response = model.generate_content( + f"Write Python code to accomplish this task:\n\n{description}" + ) + code = response.text.strip() + code = re.sub(r"```(?:python)?", "", code).strip().rstrip("`").strip() + + with tempfile.NamedTemporaryFile( + mode="w", suffix=".py", delete=False, encoding="utf-8" + ) as f: + f.write(code) + tmp_path = f.name + + print(f"[Executor] 🐍 Running generated code: {tmp_path}") + + result = subprocess.run( + [sys.executable, tmp_path], + capture_output=True, text=True, + timeout=120, cwd=str(Path.home()) + ) + + try: + os.unlink(tmp_path) + except Exception: + pass + + output = result.stdout.strip() + error = result.stderr.strip() + + if result.returncode == 0 and output: + return output + elif result.returncode == 0: + return "Task completed successfully." + elif error: + raise RuntimeError(f"Code error: {error[:400]}") + return "Completed." + + except subprocess.TimeoutExpired: + raise RuntimeError("Generated code timed out after 120 seconds.") + except RuntimeError: + raise + except Exception as e: + raise RuntimeError(f"Generated code failed: {e}") + +def _inject_context(params: dict, tool: str, step_results: dict, goal: str = "") -> dict: + if not step_results: + return params + + params = dict(params) + + if tool == "file_controller" and params.get("action") in ("write", "create_file"): + content = params.get("content", "") + if not content or len(content) < 50: + all_results = [ + v for v in step_results.values() + if v and len(v) > 100 and v not in ("Done.", "Completed.") + ] + if all_results: + combined = "\n\n---\n\n".join(all_results) + translated = _translate_to_goal_language(combined, goal) + params["content"] = translated + print(f"[Executor] πŸ’‰ Injected + translated content") + + return params +def _detect_language(text: str) -> str: + import google.generativeai as genai + genai.configure(api_key=_get_api_key()) + model = genai.GenerativeModel("gemini-2.5-flash-lite") + try: + response = model.generate_content( + f"What language is this text written in? " + f"Reply with ONLY the language name in English (e.g. Turkish, English, French).\n\n" + f"Text: {text[:200]}" + ) + return response.text.strip() + except Exception: + return "English" + + +def _translate_to_goal_language(content: str, goal: str) -> str: + if not goal: + return content + try: + import google.generativeai as genai + genai.configure(api_key=_get_api_key()) + model = genai.GenerativeModel("gemini-2.5-flash") + + target_lang = _detect_language(goal) + print(f"[Executor] 🌐 Translating to: {target_lang}") + + prompt = ( + f"You are a professional translator. " + f"Translate the following text into {target_lang}.\n" + f"IMPORTANT:\n" + f"- Translate EVERYTHING, leave nothing in English\n" + f"- Keep all facts, numbers, and data intact\n" + f"- Keep the structure and formatting\n" + f"- Output ONLY the translated text, nothing else\n\n" + f"Text to translate:\n{content[:4000]}" + ) + response = model.generate_content(prompt) + translated = response.text.strip() + print(f"[Executor] βœ… Translation done ({target_lang})") + return translated + except Exception as e: + print(f"[Executor] ⚠️ Translation failed: {e}") + return content + +def _call_tool(tool: str, parameters: dict, speak: Callable | None) -> str: + + if tool == "open_app": + from actions.open_app import open_app + return open_app(parameters=parameters, player=None) or "Done." + + elif tool == "web_search": + from actions.web_search import web_search + return web_search(parameters=parameters, player=None) or "Done." + elif tool == "game_updater": + from actions.game_updater import game_updater + return game_updater(parameters=parameters, player=None, speak=speak) or "Done." + elif tool == "browser_control": + from actions.browser_control import browser_control + return browser_control(parameters=parameters, player=None) or "Done." + + elif tool == "file_controller": + from actions.file_controller import file_controller + return file_controller(parameters=parameters, player=None) or "Done." + + elif tool == "cmd_control": + from actions.cmd_control import cmd_control + return cmd_control(parameters=parameters, player=None) or "Done." + + elif tool == "code_helper": + from actions.code_helper import code_helper + return code_helper(parameters=parameters, player=None, speak=speak) or "Done." + + elif tool == "dev_agent": + from actions.dev_agent import dev_agent + return dev_agent(parameters=parameters, player=None, speak=speak) or "Done." + + elif tool == "screen_process": + from actions.screen_processor import screen_process + screen_process(parameters=parameters, player=None) + return "Screen captured and analyzed." + + elif tool == "send_message": + from actions.send_message import send_message + return send_message(parameters=parameters, player=None) or "Done." + + elif tool == "reminder": + from actions.reminder import reminder + return reminder(parameters=parameters, player=None) or "Done." + + elif tool == "youtube_video": + from actions.youtube_video import youtube_video + return youtube_video(parameters=parameters, player=None) or "Done." + + elif tool == "weather_report": + from actions.weather_report import weather_action + return weather_action(parameters=parameters, player=None) or "Done." + + elif tool == "computer_settings": + from actions.computer_settings import computer_settings + return computer_settings(parameters=parameters, player=None) or "Done." + + elif tool == "desktop_control": + from actions.desktop import desktop_control + return desktop_control(parameters=parameters, player=None) or "Done." + + elif tool == "computer_control": + from actions.computer_control import computer_control + return computer_control(parameters=parameters, player=None) or "Done." + + elif tool == "generated_code": + description = parameters.get("description", "") + if not description: + raise ValueError("generated_code requires a 'description' parameter.") + return _run_generated_code(description, speak=speak) + + elif tool == "flight_finder": + from actions.flight_finder import flight_finder + return flight_finder(parameters=parameters, player=None, speak=speak) or "Done." + + else: + print(f"[Executor] ⚠️ Unknown tool '{tool}' β€” falling back to generated_code") + return _run_generated_code(f"Accomplish this task: {parameters}", speak=speak) + +class AgentExecutor: + + MAX_REPLAN_ATTEMPTS = 2 + + def execute( + self, + goal: str, + speak: Callable | None = None, + cancel_flag: threading.Event | None = None, + ) -> str: + print(f"\n[Executor] 🎯 Goal: {goal}") + + replan_attempts = 0 + completed_steps = [] + step_results = {} + plan = create_plan(goal) + + while True: + steps = plan.get("steps", []) + + if not steps: + msg = "I couldn't create a valid plan for this task, sir." + if speak: speak(msg) + return msg + + success = True + failed_step = None + failed_error = "" + + for step in steps: + if cancel_flag and cancel_flag.is_set(): + if speak: speak("Task cancelled, sir.") + return "Task cancelled." + + step_num = step.get("step", "?") + tool = step.get("tool", "generated_code") + desc = step.get("description", "") + params = step.get("parameters", {}) + + params = _inject_context(params, tool, step_results, goal=goal) + + print(f"\n[Executor] ▢️ Step {step_num}: [{tool}] {desc}") + + attempt = 1 + step_ok = False + + while attempt <= 3: + if cancel_flag and cancel_flag.is_set(): + break + try: + result = _call_tool(tool, params, speak) + step_results[step_num] = result + completed_steps.append(step) + print(f"[Executor] βœ… Step {step_num} done: {str(result)[:100]}") + step_ok = True + break + + except Exception as e: + error_msg = str(e) + print(f"[Executor] ❌ Step {step_num} attempt {attempt} failed: {error_msg}") + + recovery = analyze_error(step, error_msg, attempt=attempt) + decision = recovery["decision"] + user_msg = recovery.get("user_message", "") + + if speak and user_msg: + speak(user_msg) + + if decision == ErrorDecision.RETRY: + attempt += 1 + import time; time.sleep(2) + continue + + elif decision == ErrorDecision.SKIP: + print(f"[Executor] ⏭️ Skipping step {step_num}") + completed_steps.append(step) + step_ok = True + break + + elif decision == ErrorDecision.ABORT: + msg = f"Task aborted, sir. {recovery.get('reason', '')}" + if speak: speak(msg) + return msg + + else: + fix_suggestion = recovery.get("fix_suggestion", "") + if fix_suggestion and tool != "generated_code": + try: + fixed_step = generate_fix(step, error_msg, fix_suggestion) + if speak: speak("Trying an alternative approach, sir.") + res = _call_tool( + fixed_step["tool"], + fixed_step["parameters"], + speak + ) + step_results[step_num] = res + completed_steps.append(step) + step_ok = True + break + except Exception as fix_err: + print(f"[Executor] ⚠️ Fix failed: {fix_err}") + + failed_step = step + failed_error = error_msg + success = False + break + + if not step_ok and not failed_step: + failed_step = step + failed_error = "Max retries exceeded" + success = False + + if not success: + break + + if success: + return self._summarize(goal, completed_steps, speak) + + if replan_attempts >= self.MAX_REPLAN_ATTEMPTS: + msg = f"Task failed after {replan_attempts} replan attempts, sir." + if speak: speak(msg) + return msg + + if speak: speak("Adjusting my approach, sir.") + + replan_attempts += 1 + plan = replan(goal, completed_steps, failed_step, failed_error) + + def _summarize(self, goal: str, completed_steps: list, speak: Callable | None) -> str: + fallback = f"All done, sir. Completed {len(completed_steps)} steps for: {goal[:60]}." + try: + import google.generativeai as genai + genai.configure(api_key=_get_api_key()) + model = genai.GenerativeModel(model_name="gemini-2.5-flash-lite") + steps_str = "\n".join(f"- {s.get('description', '')}" for s in completed_steps) + prompt = ( + f'User goal: "{goal}"\n' + f"Completed steps:\n{steps_str}\n\n" + "Write a single natural sentence summarizing what was accomplished. " + "Address the user as 'sir'. Be direct and positive." + ) + response = model.generate_content(prompt) + summary = response.text.strip() + if speak: speak(summary) + return summary + except Exception: + if speak: speak(fallback) + return fallback \ No newline at end of file diff --git a/agent/planner.py b/agent/planner.py new file mode 100644 index 0000000..e9df604 --- /dev/null +++ b/agent/planner.py @@ -0,0 +1,278 @@ +import json +import re +import sys +from pathlib import Path + + +def get_base_dir() -> Path: + if getattr(sys, "frozen", False): + return Path(sys.executable).parent + return Path(__file__).resolve().parent.parent + + +BASE_DIR = get_base_dir() +API_CONFIG_PATH = BASE_DIR / "config" / "api_keys.json" + + +PLANNER_PROMPT = """You are the planning module of MARK XXV, a personal AI assistant. +Your job: break any user goal into a sequence of steps using ONLY the tools listed below. + +ABSOLUTE RULES: +- NEVER use generated_code or write Python scripts. It does not exist. +- NEVER reference previous step results in parameters. Every step is independent. +- Use web_search for ANY information retrieval, research, or current data. +- Use file_controller to save content to disk. +- Use cmd_control to open files or run system commands. +- Max 5 steps. Use the minimum steps needed. + +AVAILABLE TOOLS AND THEIR PARAMETERS: + +open_app + app_name: string (required) + +web_search + query: string (required) β€” write a clear, focused search query + mode: "search" or "compare" (optional, default: search) + items: list of strings (optional, for compare mode) + aspect: string (optional, for compare mode) + +game_updater + action: "update" | "install" | "list" | "download_status" | "schedule" (required) + platform: "steam" | "epic" | "both" (optional, default: both) + game_name: string (optional) + app_id: string (optional) + shutdown_when_done: boolean (optional) + +browser_control + action: "go_to" | "search" | "click" | "type" | "scroll" | "get_text" | "press" | "close" (required) + url: string (for go_to) + query: string (for search) + text: string (for click/type) + direction: "up" | "down" (for scroll) + +file_controller + action: "write" | "create_file" | "read" | "list" | "delete" | "move" | "copy" | "find" | "disk_usage" (required) + path: string β€” use "desktop" for Desktop folder + name: string β€” filename + content: string β€” file content (for write/create_file) + +cmd_control + task: string (required) β€” natural language description of what to do + visible: boolean (optional) + +computer_settings + action: string (required) + description: string β€” natural language description + value: string (optional) + +computer_control + action: "type" | "click" | "hotkey" | "press" | "scroll" | "screenshot" | "screen_find" | "screen_click" (required) + text: string (for type) + x, y: int (for click) + keys: string (for hotkey, e.g. "ctrl+c") + key: string (for press) + direction: "up" | "down" (for scroll) + description: string (for screen_find/screen_click) + +screen_process + text: string (required) β€” what to analyze or ask about the screen + angle: "screen" | "camera" (optional) + +send_message + receiver: string (required) + message_text: string (required) + platform: string (required) + +reminder + date: string YYYY-MM-DD (required) + time: string HH:MM (required) + message: string (required) + +desktop_control + action: "wallpaper" | "organize" | "clean" | "list" | "task" (required) + path: string (optional) + task: string (optional) + +youtube_video + action: "play" | "summarize" | "trending" (required) + query: string (for play) + +weather_report + city: string (required) + +flight_finder + origin: string (required) + destination: string (required) + date: string (required) + +code_helper + action: "write" | "edit" | "run" | "explain" (required) + description: string (required) + language: string (optional) + output_path: string (optional) + file_path: string (optional) + +dev_agent + description: string (required) + language: string (optional) +EXAMPLES: + +Goal: "research mechanical engineering and save it to a notepad file" +Steps: + +web_search | query: "mechanical engineering overview definition history" +web_search | query: "mechanical engineering applications and future trends" +file_controller | action: write, path: desktop, name: mechanical_engineering.txt, content: "MECHANICAL ENGINEERING RESEARCH\n\nThis file will be filled with web research results." +cmd_control | task: "open mechanical_engineering.txt on desktop with notepad" + +Goal: "What is the price of Bitcoin" +Steps: + +web_search | query: "Bitcoin price today USD" + +Goal: "List the files on the desktop and find the largest 5 files" +Steps: + +file_controller | action: list, path: desktop +file_controller | action: largest, path: desktop, count: 5 + +Goal: "Install PUBG from Steam" +Steps: + +game_updater | action: install, platform: steam, game_name: "PUBG" + +Goal: "Update all my Steam games" +Steps: + +game_updater | action: update, platform: steam + +Goal: "Send John a message on WhatsApp saying there is a meeting tomorrow" +Steps: + +send_message | receiver: John, message_text: "There is a meeting tomorrow", platform: WhatsApp + +Goal: "Open the clock and set a reminder for 30 minutes later" +Steps: + +reminder | date: [today], time: [now+30min], message: "Reminder" + +OUTPUT β€” return ONLY valid JSON, no markdown, no explanation, no code blocks: +{ + "goal": "...", + "steps": [ + { + "step": 1, + "tool": "tool_name", + "description": "what this step does", + "parameters": {}, + "critical": true + } + ] +} +""" + + +def _get_api_key() -> str: + with open(API_CONFIG_PATH, "r", encoding="utf-8") as f: + return json.load(f)["gemini_api_key"] + + +def create_plan(goal: str, context: str = "") -> dict: + import google.generativeai as genai + + genai.configure(api_key=_get_api_key()) + model = genai.GenerativeModel( + model_name="gemini-2.5-flash-lite", + system_instruction=PLANNER_PROMPT + ) + + user_input = f"Goal: {goal}" + if context: + user_input += f"\n\nContext: {context}" + + try: + response = model.generate_content(user_input) + text = response.text.strip() + text = re.sub(r"```(?:json)?", "", text).strip().rstrip("`").strip() + + plan = json.loads(text) + + if "steps" not in plan or not isinstance(plan["steps"], list): + raise ValueError("Invalid plan structure") + + for step in plan["steps"]: + if step.get("tool") in ("generated_code",): + print(f"[Planner] ⚠️ generated_code detected in step {step.get('step')} β€” replacing with web_search") + desc = step.get("description", goal) + step["tool"] = "web_search" + step["parameters"] = {"query": desc[:200]} + + print(f"[Planner] βœ… Plan: {len(plan['steps'])} steps") + for s in plan["steps"]: + print(f" Step {s['step']}: [{s['tool']}] {s['description']}") + + return plan + + except json.JSONDecodeError as e: + print(f"[Planner] ⚠️ JSON parse failed: {e}") + return _fallback_plan(goal) + except Exception as e: + print(f"[Planner] ⚠️ Planning failed: {e}") + return _fallback_plan(goal) + + +def _fallback_plan(goal: str) -> dict: + print("[Planner] πŸ”„ Fallback plan") + return { + "goal": goal, + "steps": [ + { + "step": 1, + "tool": "web_search", + "description": f"Search for: {goal}", + "parameters": {"query": goal}, + "critical": True + } + ] + } + + +def replan(goal: str, completed_steps: list, failed_step: dict, error: str) -> dict: + import google.generativeai as genai + + genai.configure(api_key=_get_api_key()) + model = genai.GenerativeModel( + model_name="gemini-2.5-flash", + system_instruction=PLANNER_PROMPT + ) + + completed_summary = "\n".join( + f" - Step {s['step']} ({s['tool']}): DONE" for s in completed_steps + ) + + prompt = f"""Goal: {goal} + +Already completed: +{completed_summary if completed_summary else ' (none)'} + +Failed step: [{failed_step.get('tool')}] {failed_step.get('description')} +Error: {error} + +Create a REVISED plan for the remaining work only. Do not repeat completed steps.""" + + try: + response = model.generate_content(prompt) + text = response.text.strip() + text = re.sub(r"```(?:json)?", "", text).strip().rstrip("`").strip() + plan = json.loads(text) + + for step in plan.get("steps", []): + if step.get("tool") == "generated_code": + step["tool"] = "web_search" + step["parameters"] = {"query": step.get("description", goal)[:200]} + + print(f"[Planner] πŸ”„ Revised plan: {len(plan['steps'])} steps") + return plan + except Exception as e: + print(f"[Planner] ⚠️ Replan failed: {e}") + return _fallback_plan(goal) \ No newline at end of file diff --git a/agent/task_queue.py b/agent/task_queue.py new file mode 100644 index 0000000..36b78b2 --- /dev/null +++ b/agent/task_queue.py @@ -0,0 +1,221 @@ +import threading +import time +import uuid +from dataclasses import dataclass, field +from enum import Enum +from typing import Callable, Any + + +class TaskStatus(Enum): + PENDING = "pending" + RUNNING = "running" + COMPLETED = "completed" + FAILED = "failed" + CANCELLED = "cancelled" + + +class TaskPriority(Enum): + LOW = 3 + NORMAL = 2 + HIGH = 1 + + +@dataclass(order=True) +class Task: + priority: int + created_at: float = field(compare=False) + task_id: str = field(compare=False) + goal: str = field(compare=False) + status: TaskStatus = field(compare=False, default=TaskStatus.PENDING) + result: Any = field(compare=False, default=None) + error: str = field(compare=False, default="") + speak: Any = field(compare=False, default=None) + on_complete: Any = field(compare=False, default=None) + cancel_flag: threading.Event = field(compare=False, default_factory=threading.Event) + + +class TaskQueue: + def __init__(self, max_concurrent: int = 1): + self._queue: list[Task] = [] + self._lock: threading.Lock = threading.Lock() + self._condition: threading.Condition = threading.Condition(self._lock) + self._tasks: dict[str, Task] = {} + self._running: bool = False + self._worker_thread: threading.Thread | None = None + self._max_concurrent = max_concurrent + self._active_count = 0 + self._executor = None + + def _get_executor(self): + if self._executor is None: + from agent.executor import AgentExecutor + self._executor = AgentExecutor() + return self._executor + + def start(self) -> None: + if self._running: + return + self._running = True + self._worker_thread = threading.Thread( + target=self._worker_loop, + daemon=True, + name="AgentTaskQueue" + ) + self._worker_thread.start() + print("[TaskQueue] βœ… Started") + + def stop(self) -> None: + self._running = False + with self._condition: + self._condition.notify_all() + print("[TaskQueue] πŸ”΄ Stopped") + + def submit( + self, + goal: str, + priority: TaskPriority = TaskPriority.NORMAL, + speak: Callable | None = None, + on_complete: Callable | None = None, + ) -> str: + + task_id = str(uuid.uuid4())[:8] + task = Task( + priority = priority.value, + created_at = time.time(), + task_id = task_id, + goal = goal, + speak = speak, + on_complete = on_complete, + ) + + with self._condition: + self._queue.append(task) + self._queue.sort(key=lambda t: (t.priority, t.created_at)) + self._tasks[task_id] = task + self._condition.notify() + + print(f"[TaskQueue] πŸ“₯ Task queued: [{task_id}] {goal[:60]}") + return task_id + + def cancel(self, task_id: str) -> bool: + + with self._lock: + task = self._tasks.get(task_id) + if not task: + return False + if task.status in (TaskStatus.COMPLETED, TaskStatus.FAILED, TaskStatus.CANCELLED): + return False + + task.cancel_flag.set() + task.status = TaskStatus.CANCELLED + print(f"[TaskQueue] 🚫 Task cancelled: [{task_id}]") + return True + + def get_status(self, task_id: str) -> dict | None: + with self._lock: + task = self._tasks.get(task_id) + if not task: + return None + return { + "task_id": task.task_id, + "goal": task.goal, + "status": task.status.value, + "result": task.result, + "error": task.error, + } + + def get_all_statuses(self) -> list[dict]: + with self._lock: + return [ + { + "task_id": t.task_id, + "goal": t.goal[:50], + "status": t.status.value, + } + for t in self._tasks.values() + ] + + def pending_count(self) -> int: + with self._lock: + return sum(1 for t in self._queue if t.status == TaskStatus.PENDING) + + def _worker_loop(self) -> None: + while self._running: + task = None + + with self._condition: + while self._running and not self._next_task(): + self._condition.wait(timeout=1.0) + task = self._next_task() + if task: + task.status = TaskStatus.RUNNING + self._active_count += 1 + try: + self._queue.remove(task) + except ValueError: + pass + + if task: + threading.Thread( + target=self._run_task, + args=(task,), + daemon=True, + name=f"AgentTask-{task.task_id}" + ).start() + + def _next_task(self) -> Task | None: + if self._active_count >= self._max_concurrent: + return None + for task in self._queue: + if task.status == TaskStatus.PENDING and not task.cancel_flag.is_set(): + return task + return None + + def _run_task(self, task: Task) -> None: + print(f"[TaskQueue] ▢️ Running: [{task.task_id}] {task.goal[:60]}") + try: + executor = self._get_executor() + result = executor.execute( + goal = task.goal, + speak = task.speak, + cancel_flag = task.cancel_flag, + ) + + with self._lock: + if task.cancel_flag.is_set(): + task.status = TaskStatus.CANCELLED + else: + task.status = TaskStatus.COMPLETED + task.result = result + self._active_count -= 1 + + if task.on_complete and not task.cancel_flag.is_set(): + try: + task.on_complete(task.task_id, result) + except Exception as e: + print(f"[TaskQueue] ⚠️ on_complete callback error: {e}") + + print(f"[TaskQueue] βœ… Completed: [{task.task_id}]") + + except Exception as e: + with self._lock: + task.status = TaskStatus.FAILED + task.error = str(e) + self._active_count -= 1 + print(f"[TaskQueue] ❌ Failed: [{task.task_id}] {e}") + + with self._condition: + self._condition.notify() + +_queue = TaskQueue() +_queue_started = False +_queue_lock = threading.Lock() + + +def get_queue() -> TaskQueue: + global _queue_started + with _queue_lock: + if not _queue_started: + _queue.start() + _queue_started = True + return _queue \ No newline at end of file diff --git a/config/__init__.py b/config/__init__.py new file mode 100644 index 0000000..2660a04 --- /dev/null +++ b/config/__init__.py @@ -0,0 +1,16 @@ +import json, os +from pathlib import Path + +_CONFIG_PATH = Path(__file__).parent / "api_keys.json" + +def get_config() -> dict: + with open(_CONFIG_PATH, "r", encoding="utf-8") as f: + return json.load(f) + +def get_os() -> str: + """Returns: 'windows' | 'mac' | 'linux'""" + return get_config().get("os_system", "windows").lower() + +def is_windows() -> bool: return get_os() == "windows" +def is_mac() -> bool: return get_os() == "mac" +def is_linux() -> bool: return get_os() == "linux" \ No newline at end of file diff --git a/core/prompt.txt b/core/prompt.txt new file mode 100644 index 0000000..118fae5 --- /dev/null +++ b/core/prompt.txt @@ -0,0 +1,35 @@ +You are JARVIS, a sharp and efficient AI assistant. Calm, direct, professional. + +RULES: +- Always use the correct tool. Never simulate or guess results. +- Call each tool EXACTLY ONCE. Never retry a successful action. +- Do NOT say "I'm doing X" before calling a tool. Just call it, then report the result briefly. +- Do NOT repeat yourself. Say something once and stop. +- Keep responses to 1-2 sentences max unless reporting data. +- Never ask unnecessary questions. Make a reasonable assumption and proceed. +- Speak and take action according to user's information, for better experience. +- If a action tool called, tell that you're doing this thing and then call the tool, don't speak after the action's tool. +- Attain important things in the memory for a better user experience. +- game_updater β†’ ANY Steam or Epic request: install, download, update, list. + Call DIRECTLY β€” never web_search first, never agent_task. +- For slow tools (web_search, screen_process, browser_control, flight_finder, + game_updater, cmd_control, dev_agent, code_helper, agent_task): say TWO short + sentence before calling the tool, then stay silent after β€” result is already reported. +- For screen_process: say something curious before calling, then stay completely + silent β€” the vision module speaks directly. +- Only call shutdown_jarvis if the user clearly intends to end the assistant session. +Do not call it for casual goodbyes unless they imply stopping the assistant. + +LANGUAGE: +- Respond in the same language the user spoke. +- Always extract tool parameters in English. +- Example: "Δ°stanbul hava durumu?" β†’ weather_report city:"Istanbul", reply in Turkish. + +TOOL SELECTION: +- computer_settings β†’ volume, brightness, screen, refresh, scroll, minimize, maximize, + close, screenshot, lock, restart, shutdown, wifi, zoom, tabs, keyboard shortcuts. + Use this for ANY single computer control command. NEVER route these to agent_task. +- agent_task β†’ ONLY for complex multi-step goals that require planning across + multiple tools (e.g. "research X and save to file", "create a project"). +- Simple rule: if it can be done in ONE action β†’ computer_settings. + If it needs 3+ steps across different tools β†’ agent_task. \ No newline at end of file diff --git a/main.py b/main.py index 211bf21..f231c8c 100644 --- a/main.py +++ b/main.py @@ -1,35 +1,887 @@ -You are JARVIS, a sharp and efficient AI assistant. Calm, direct, professional. +import asyncio +import threading +import json +import sys +import traceback +from pathlib import Path -RULES: -- Always use the correct tool. Never simulate or guess results. -- Call each tool EXACTLY ONCE. Never retry a successful action. -- Do NOT say "I'm doing X" before calling a tool. Just call it, then report the result briefly. -- Do NOT repeat yourself. Say something once and stop. -- Keep responses to 1-2 sentences max unless reporting data. -- Never ask unnecessary questions. Make a reasonable assumption and proceed. -- Speak and take action according to user's information, for better experience. -- If a action tool called, tell that you're doing this thing and then call the tool, don't speak after the action's tool. -- Attain important things in the memory for a better user experience. -- game_updater β†’ ANY Steam or Epic request: install, download, update, list. - Call DIRECTLY β€” never web_search first, never agent_task. -- For slow tools (web_search, screen_process, browser_control, flight_finder, - game_updater, cmd_control, dev_agent, code_helper, agent_task): say TWO short - sentence before calling the tool, then stay silent after β€” result is already reported. -- For screen_process: say something curious before calling, then stay completely - silent β€” the vision module speaks directly. -- Only call shutdown_jarvis if the user clearly intends to end the assistant session. -Do not call it for casual goodbyes unless they imply stopping the assistant. +import sounddevice as sd +from google import genai +from google.genai import types +from ui import JarvisUI +from memory.memory_manager import ( + load_memory, update_memory, format_memory_for_prompt, + should_extract_memory, extract_memory +) -LANGUAGE: -- Respond in the same language the user spoke. -- Always extract tool parameters in English. -- Example: "Δ°stanbul hava durumu?" β†’ weather_report city:"Istanbul", reply in Turkish. +from actions.file_processor import file_processor +from actions.flight_finder import flight_finder +from actions.open_app import open_app +from actions.weather_report import weather_action +from actions.send_message import send_message +from actions.reminder import reminder +from actions.computer_settings import computer_settings +from actions.screen_processor import screen_process +from actions.youtube_video import youtube_video +from actions.desktop import desktop_control +from actions.browser_control import browser_control +from actions.file_controller import file_controller +from actions.code_helper import code_helper +from actions.dev_agent import dev_agent +from actions.web_search import web_search as web_search_action +from actions.computer_control import computer_control +from actions.game_updater import game_updater -TOOL SELECTION: -- computer_settings β†’ volume, brightness, screen, refresh, scroll, minimize, maximize, - close, screenshot, lock, restart, shutdown, wifi, zoom, tabs, keyboard shortcuts. - Use this for ANY single computer control command. NEVER route these to agent_task. -- agent_task β†’ ONLY for complex multi-step goals that require planning across - multiple tools (e.g. "research X and save to file", "create a project"). -- Simple rule: if it can be done in ONE action β†’ computer_settings. - If it needs 3+ steps across different tools β†’ agent_task. \ No newline at end of file + +def get_base_dir(): + if getattr(sys, "frozen", False): + return Path(sys.executable).parent + return Path(__file__).resolve().parent + + +BASE_DIR = get_base_dir() +API_CONFIG_PATH = BASE_DIR / "config" / "api_keys.json" +PROMPT_PATH = BASE_DIR / "core" / "prompt.txt" +LIVE_MODEL = "models/gemini-2.5-flash-native-audio-preview-12-2025" +CHANNELS = 1 +SEND_SAMPLE_RATE = 16000 +RECEIVE_SAMPLE_RATE = 24000 +CHUNK_SIZE = 1024 + + +def _get_api_key() -> str: + with open(API_CONFIG_PATH, "r", encoding="utf-8") as f: + return json.load(f)["gemini_api_key"] + + +def _load_system_prompt() -> str: + try: + return PROMPT_PATH.read_text(encoding="utf-8") + except Exception: + return ( + "You are JARVIS, Tony Stark's AI assistant. " + "Be concise, direct, and always use the provided tools to complete tasks. " + "Never simulate or guess results β€” always call the appropriate tool." + ) + +_last_memory_input = "" + +def _update_memory_async(user_text: str, jarvis_text: str) -> None: + global _last_memory_input + + user_text = (user_text or "").strip() + jarvis_text = (jarvis_text or "").strip() + + if len(user_text) < 5 or user_text == _last_memory_input: + return + _last_memory_input = user_text + + try: + api_key = _get_api_key() + if not should_extract_memory(user_text, jarvis_text, api_key): + return + data = extract_memory(user_text, jarvis_text, api_key) + if data: + update_memory(data) + print(f"[Memory] βœ… {list(data.keys())}") + except Exception as e: + if "429" not in str(e): + print(f"[Memory] ⚠️ {e}") + +TOOL_DECLARATIONS = [ + { + "name": "open_app", + "description": ( + "Opens any application on the Windows computer. " + "Use this whenever the user asks to open, launch, or start any app, " + "website, or program. Always call this tool β€” never just say you opened it." + ), + "parameters": { + "type": "OBJECT", + "properties": { + "app_name": { + "type": "STRING", + "description": "Exact name of the application (e.g. 'WhatsApp', 'Chrome', 'Spotify')" + } + }, + "required": ["app_name"] + } + }, + { + "name": "web_search", + "description": "Searches the web for any information.", + "parameters": { + "type": "OBJECT", + "properties": { + "query": {"type": "STRING", "description": "Search query"}, + "mode": {"type": "STRING", "description": "search (default) or compare"}, + "items": {"type": "ARRAY", "items": {"type": "STRING"}, "description": "Items to compare"}, + "aspect": {"type": "STRING", "description": "price | specs | reviews"} + }, + "required": ["query"] + } + }, + { + "name": "weather_report", + "description": "Gives the weather report to user", + "parameters": { + "type": "OBJECT", + "properties": { + "city": {"type": "STRING", "description": "City name"} + }, + "required": ["city"] + } + }, + { + "name": "send_message", + "description": "Sends a text message via WhatsApp, Telegram, or other messaging platform.", + "parameters": { + "type": "OBJECT", + "properties": { + "receiver": {"type": "STRING", "description": "Recipient contact name"}, + "message_text": {"type": "STRING", "description": "The message to send"}, + "platform": {"type": "STRING", "description": "Platform: WhatsApp, Telegram, etc."} + }, + "required": ["receiver", "message_text", "platform"] + } + }, + { + "name": "reminder", + "description": "Sets a timed reminder using Windows Task Scheduler.", + "parameters": { + "type": "OBJECT", + "properties": { + "date": {"type": "STRING", "description": "Date in YYYY-MM-DD format"}, + "time": {"type": "STRING", "description": "Time in HH:MM format (24h)"}, + "message": {"type": "STRING", "description": "Reminder message text"} + }, + "required": ["date", "time", "message"] + } + }, + { + "name": "youtube_video", + "description": ( + "Controls YouTube. Use for: playing videos, summarizing a video's content, " + "getting video info, or showing trending videos." + ), + "parameters": { + "type": "OBJECT", + "properties": { + "action": {"type": "STRING", "description": "play | summarize | get_info | trending (default: play)"}, + "query": {"type": "STRING", "description": "Search query for play action"}, + "save": {"type": "BOOLEAN", "description": "Save summary to Notepad (summarize only)"}, + "region": {"type": "STRING", "description": "Country code for trending e.g. TR, US"}, + "url": {"type": "STRING", "description": "Video URL for get_info action"}, + }, + "required": [] + } + }, + { + "name": "screen_process", + "description": ( + "Captures and analyzes the screen or webcam image. " + "MUST be called when user asks what is on screen, what you see, " + "analyze my screen, look at camera, etc. " + "You have NO visual ability without this tool. " + "After calling this tool, stay SILENT β€” the vision module speaks directly." + ), + "parameters": { + "type": "OBJECT", + "properties": { + "angle": {"type": "STRING", "description": "'screen' to capture display, 'camera' for webcam. Default: 'screen'"}, + "text": {"type": "STRING", "description": "The question or instruction about the captured image"} + }, + "required": ["text"] + } + }, + { + "name": "computer_settings", + "description": ( + "Controls the computer: volume, brightness, window management, keyboard shortcuts, " + "typing text on screen, closing apps, fullscreen, dark mode, WiFi, restart, shutdown, " + "scrolling, tab management, zoom, screenshots, lock screen, refresh/reload page. " + "Use for ANY single computer control command. NEVER route to agent_task." + ), + "parameters": { + "type": "OBJECT", + "properties": { + "action": {"type": "STRING", "description": "The action to perform"}, + "description": {"type": "STRING", "description": "Natural language description of what to do"}, + "value": {"type": "STRING", "description": "Optional value: volume level, text to type, etc."} + }, + "required": [] + } + }, + { + "name": "browser_control", + "description": ( + "Controls the web browser. Use for: opening websites, searching the web, " + "clicking elements, filling forms, scrolling, any web-based task." + ), + "parameters": { + "type": "OBJECT", + "properties": { + "action": {"type": "STRING", "description": "go_to | search | click | type | scroll | fill_form | smart_click | smart_type | get_text | press | close"}, + "url": {"type": "STRING", "description": "URL for go_to action"}, + "query": {"type": "STRING", "description": "Search query for search action"}, + "selector": {"type": "STRING", "description": "CSS selector for click/type"}, + "text": {"type": "STRING", "description": "Text to click or type"}, + "description": {"type": "STRING", "description": "Element description for smart_click/smart_type"}, + "direction": {"type": "STRING", "description": "up or down for scroll"}, + "key": {"type": "STRING", "description": "Key name for press action"}, + "incognito": {"type": "BOOLEAN", "description": "Open in private/incognito mode"}, + }, + "required": ["action"] + } + }, + { + "name": "file_controller", + "description": "Manages files and folders: list, create, delete, move, copy, rename, read, write, find, disk usage.", + "parameters": { + "type": "OBJECT", + "properties": { + "action": {"type": "STRING", "description": "list | create_file | create_folder | delete | move | copy | rename | read | write | find | largest | disk_usage | organize_desktop | info"}, + "path": {"type": "STRING", "description": "File/folder path or shortcut: desktop, downloads, documents, home"}, + "destination": {"type": "STRING", "description": "Destination path for move/copy"}, + "new_name": {"type": "STRING", "description": "New name for rename"}, + "content": {"type": "STRING", "description": "Content for create_file/write"}, + "name": {"type": "STRING", "description": "File name to search for"}, + "extension": {"type": "STRING", "description": "File extension to search (e.g. .pdf)"}, + "count": {"type": "INTEGER", "description": "Number of results for largest"}, + }, + "required": ["action"] + } + }, + { + "name": "desktop_control", + "description": "Controls the desktop: wallpaper, organize, clean, list, stats.", + "parameters": { + "type": "OBJECT", + "properties": { + "action": {"type": "STRING", "description": "wallpaper | wallpaper_url | organize | clean | list | stats | task"}, + "path": {"type": "STRING", "description": "Image path for wallpaper"}, + "url": {"type": "STRING", "description": "Image URL for wallpaper_url"}, + "mode": {"type": "STRING", "description": "by_type or by_date for organize"}, + "task": {"type": "STRING", "description": "Natural language desktop task"}, + }, + "required": ["action"] + } + }, + { + "name": "code_helper", + "description": "Writes, edits, explains, runs, or builds code files.", + "parameters": { + "type": "OBJECT", + "properties": { + "action": {"type": "STRING", "description": "write | edit | explain | run | build | auto (default: auto)"}, + "description": {"type": "STRING", "description": "What the code should do or what change to make"}, + "language": {"type": "STRING", "description": "Programming language (default: python)"}, + "output_path": {"type": "STRING", "description": "Where to save the file"}, + "file_path": {"type": "STRING", "description": "Path to existing file for edit/explain/run/build"}, + "code": {"type": "STRING", "description": "Raw code string for explain"}, + "args": {"type": "STRING", "description": "CLI arguments for run/build"}, + "timeout": {"type": "INTEGER", "description": "Execution timeout in seconds (default: 30)"}, + }, + "required": ["action"] + } + }, + { + "name": "dev_agent", + "description": "Builds complete multi-file projects from scratch: plans, writes files, installs deps, opens VSCode, runs and fixes errors.", + "parameters": { + "type": "OBJECT", + "properties": { + "description": {"type": "STRING", "description": "What the project should do"}, + "language": {"type": "STRING", "description": "Programming language (default: python)"}, + "project_name": {"type": "STRING", "description": "Optional project folder name"}, + "timeout": {"type": "INTEGER", "description": "Run timeout in seconds (default: 30)"}, + }, + "required": ["description"] + } + }, + { + "name": "agent_task", + "description": ( + "Executes complex multi-step tasks requiring multiple different tools. " + "Examples: 'research X and save to file', 'find and organize files'. " + "DO NOT use for single commands. NEVER use for Steam/Epic β€” use game_updater." + ), + "parameters": { + "type": "OBJECT", + "properties": { + "goal": {"type": "STRING", "description": "Complete description of what to accomplish"}, + "priority": {"type": "STRING", "description": "low | normal | high (default: normal)"} + }, + "required": ["goal"] + } + }, + { + "name": "computer_control", + "description": "Direct computer control: type, click, hotkeys, scroll, move mouse, screenshots, find elements on screen.", + "parameters": { + "type": "OBJECT", + "properties": { + "action": {"type": "STRING", "description": "type | smart_type | click | double_click | right_click | hotkey | press | scroll | move | copy | paste | screenshot | wait | clear_field | focus_window | screen_find | screen_click | random_data | user_data"}, + "text": {"type": "STRING", "description": "Text to type or paste"}, + "x": {"type": "INTEGER", "description": "X coordinate"}, + "y": {"type": "INTEGER", "description": "Y coordinate"}, + "keys": {"type": "STRING", "description": "Key combination e.g. 'ctrl+c'"}, + "key": {"type": "STRING", "description": "Single key e.g. 'enter'"}, + "direction": {"type": "STRING", "description": "up | down | left | right"}, + "amount": {"type": "INTEGER", "description": "Scroll amount (default: 3)"}, + "seconds": {"type": "NUMBER", "description": "Seconds to wait"}, + "title": {"type": "STRING", "description": "Window title for focus_window"}, + "description": {"type": "STRING", "description": "Element description for screen_find/screen_click"}, + "type": {"type": "STRING", "description": "Data type for random_data"}, + "field": {"type": "STRING", "description": "Field for user_data: name|email|city"}, + "clear_first": {"type": "BOOLEAN", "description": "Clear field before typing (default: true)"}, + "path": {"type": "STRING", "description": "Save path for screenshot"}, + }, + "required": ["action"] + } + }, + { + "name": "game_updater", + "description": ( + "THE ONLY tool for ANY Steam or Epic Games request. " + "Use for: installing, downloading, updating games, listing installed games, " + "checking download status, scheduling updates. " + "ALWAYS call directly for any Steam/Epic/game request. " + "NEVER use agent_task, browser_control, or web_search for Steam/Epic." + ), + "parameters": { + "type": "OBJECT", + "properties": { + "action": {"type": "STRING", "description": "update | install | list | download_status | schedule | cancel_schedule | schedule_status (default: update)"}, + "platform": {"type": "STRING", "description": "steam | epic | both (default: both)"}, + "game_name": {"type": "STRING", "description": "Game name (partial match supported)"}, + "app_id": {"type": "STRING", "description": "Steam AppID for install (optional)"}, + "hour": {"type": "INTEGER", "description": "Hour for scheduled update 0-23 (default: 3)"}, + "minute": {"type": "INTEGER", "description": "Minute for scheduled update 0-59 (default: 0)"}, + "shutdown_when_done": {"type": "BOOLEAN", "description": "Shut down PC when download finishes"}, + }, + "required": [] + } + }, + { + "name": "flight_finder", + "description": "Searches Google Flights and speaks the best options.", + "parameters": { + "type": "OBJECT", + "properties": { + "origin": {"type": "STRING", "description": "Departure city or airport code"}, + "destination": {"type": "STRING", "description": "Arrival city or airport code"}, + "date": {"type": "STRING", "description": "Departure date (any format)"}, + "return_date": {"type": "STRING", "description": "Return date for round trips"}, + "passengers": {"type": "INTEGER", "description": "Number of passengers (default: 1)"}, + "cabin": {"type": "STRING", "description": "economy | premium | business | first"}, + "save": {"type": "BOOLEAN", "description": "Save results to Notepad"}, + }, + "required": ["origin", "destination", "date"] + } + }, + { + "name": "file_processor", + "description": ( + "Processes any file that the user has uploaded or dropped onto the interface. " + "Use this when the user refers to an uploaded file and wants an action on it. " + "Supports: images (describe/ocr/resize/compress/convert), " + "PDFs (summarize/extract_text/to_word), " + "Word docs & text files (summarize/fix/reformat/translate), " + "CSV/Excel (analyze/stats/filter/sort/convert), " + "JSON/XML (validate/format/analyze), " + "code files (explain/review/fix/optimize/run/document/test), " + "audio (transcribe/trim/convert/info), " + "video (trim/extract_audio/extract_frame/compress/transcribe/info), " + "archives (list/extract), " + "presentations (summarize/extract_text). " + "ALWAYS call this tool when a file has been uploaded and the user gives a command about it. " + "If the user's command is ambiguous, pick the most logical action for that file type." + ), + "parameters": { + "type": "OBJECT", + "properties": { + "file_path": { + "type": "STRING", + "description": "Full path to the uploaded file. Leave empty to use the currently uploaded file." + }, + "action": { + "type": "STRING", + "description": ( + "What to do with the file. Examples by type:\n" + "image: describe | ocr | resize | compress | convert | info\n" + "pdf: summarize | extract_text | to_word | info\n" + "docx/txt: summarize | fix | reformat | translate_hint | word_count | to_bullet\n" + "csv/excel: analyze | stats | filter | sort | convert | info\n" + "json: validate | format | analyze | to_csv\n" + "code: explain | review | fix | optimize | run | document | test\n" + "audio: transcribe | trim | convert | info\n" + "video: trim | extract_audio | extract_frame | compress | transcribe | info | convert\n" + "archive: list | extract\n" + "pptx: summarize | extract_text | analyze" + ) + }, + "instruction": { + "type": "STRING", + "description": "Free-form instruction if action doesn't cover it. E.g. 'translate this to Turkish', 'find all email addresses'" + }, + "format": { + "type": "STRING", + "description": "Target format for conversion. E.g. 'mp3', 'pdf', 'csv', 'png'" + }, + "width": {"type": "INTEGER", "description": "Target width for image resize"}, + "height": {"type": "INTEGER", "description": "Target height for image resize"}, + "scale": {"type": "NUMBER", "description": "Scale factor for image resize (e.g. 0.5)"}, + "quality": {"type": "INTEGER", "description": "Quality 1-100 for image/video compress"}, + "start": {"type": "STRING", "description": "Start time for trim: seconds or HH:MM:SS"}, + "end": {"type": "STRING", "description": "End time for trim: seconds or HH:MM:SS"}, + "timestamp": {"type": "STRING", "description": "Timestamp for video frame extraction HH:MM:SS"}, + "column": {"type": "STRING", "description": "Column name for CSV filter/sort"}, + "value": {"type": "STRING", "description": "Filter value for CSV filter"}, + "condition": {"type": "STRING", "description": "Filter condition: equals|contains|gt|lt"}, + "ascending": {"type": "BOOLEAN", "description": "Sort order for CSV sort (default: true)"}, + "save": {"type": "BOOLEAN", "description": "Save result to file (default: true)"}, + "destination": {"type": "STRING", "description": "Output folder for archive extract"}, + }, + "required": [] + } +}, + { + "name": "shutdown_jarvis", + "description": ( + "Shuts down the assistant completely. " + "Call this when the user expresses intent to end the conversation, " + "close the assistant, say goodbye, or stop Jarvis. " + "The user can say this in ANY language." + ), + "parameters": { + "type": "OBJECT", + "properties": {}, + } + }, + { + "name": "save_memory", + "description": ( + "Save an important personal fact about the user to long-term memory. " + "Call this silently whenever the user reveals something worth remembering: " + "name, age, city, job, preferences, hobbies, relationships, projects, or future plans. " + "Do NOT call for: weather, reminders, searches, or one-time commands. " + "Do NOT announce that you are saving β€” just call it silently. " + "Values must be in English regardless of the conversation language." + ), + "parameters": { + "type": "OBJECT", + "properties": { + "category": { + "type": "STRING", + "description": ( + "identity β€” name, age, birthday, city, job, language, nationality | " + "preferences β€” favorite food/color/music/film/game/sport, hobbies | " + "projects β€” active projects, goals, things being built | " + "relationships β€” friends, family, partner, colleagues | " + "wishes β€” future plans, things to buy, travel dreams | " + "notes β€” habits, schedule, anything else worth remembering" + ) + }, + "key": {"type": "STRING", "description": "Short snake_case key (e.g. name, favorite_food, sister_name)"}, + "value": {"type": "STRING", "description": "Concise value in English (e.g. Fatih, pizza, older sister)"}, + }, + "required": ["category", "key", "value"] + } + }, +] + + +class JarvisLive: + + def __init__(self, ui: JarvisUI): + self.ui = ui + self.session = None + self.audio_in_queue = None + self.out_queue = None + self._loop = None + self._is_speaking = False + self._speaking_lock = threading.Lock() + self.ui.on_text_command = self._on_text_command + + def _on_text_command(self, text: str): + if not self._loop or not self.session: + return + asyncio.run_coroutine_threadsafe( + self.session.send_client_content( + turns={"parts": [{"text": text}]}, + turn_complete=True + ), + self._loop + ) + + def set_speaking(self, value: bool): + with self._speaking_lock: + self._is_speaking = value + if value: + self.ui.set_state("SPEAKING") + elif not self.ui.muted: + self.ui.set_state("LISTENING") + + def speak(self, text: str): + if not self._loop or not self.session: + return + asyncio.run_coroutine_threadsafe( + self.session.send_client_content( + turns={"parts": [{"text": text}]}, + turn_complete=True + ), + self._loop + ) + + def speak_error(self, tool_name: str, error: str): + short = str(error)[:120] + self.ui.write_log(f"ERR: {tool_name} β€” {short}") + self.speak(f"Sir, {tool_name} encountered an error. {short}") + + def _build_config(self) -> types.LiveConnectConfig: + from datetime import datetime + + memory = load_memory() + mem_str = format_memory_for_prompt(memory) + sys_prompt = _load_system_prompt() + + now = datetime.now() + time_str = now.strftime("%A, %B %d, %Y β€” %I:%M %p") + time_ctx = ( + f"[CURRENT DATE & TIME]\n" + f"Right now it is: {time_str}\n" + f"Use this to calculate exact times for reminders.\n\n" + ) + + parts = [time_ctx] + if mem_str: + parts.append(mem_str) + parts.append(sys_prompt) + + return types.LiveConnectConfig( + response_modalities=["AUDIO"], + output_audio_transcription={}, + input_audio_transcription={}, + system_instruction="\n".join(parts), + tools=[{"function_declarations": TOOL_DECLARATIONS}], + session_resumption=types.SessionResumptionConfig(), + speech_config=types.SpeechConfig( + voice_config=types.VoiceConfig( + prebuilt_voice_config=types.PrebuiltVoiceConfig( + voice_name="Charon" + ) + ) + ), + ) + + async def _execute_tool(self, fc) -> types.FunctionResponse: + name = fc.name + args = dict(fc.args or {}) + + print(f"[JARVIS] πŸ”§ {name} {args}") + self.ui.set_state("THINKING") + if name == "save_memory": + category = args.get("category", "notes") + key = args.get("key", "") + value = args.get("value", "") + if key and value: + update_memory({category: {key: {"value": value}}}) + print(f"[Memory] πŸ’Ύ save_memory: {category}/{key} = {value}") + if not self.ui.muted: + self.ui.set_state("LISTENING") + return types.FunctionResponse( + id=fc.id, name=name, + response={"result": "ok", "silent": True} + ) + + loop = asyncio.get_event_loop() + result = "Done." + + try: + if name == "open_app": + r = await loop.run_in_executor(None, lambda: open_app(parameters=args, response=None, player=self.ui)) + result = r or f"Opened {args.get('app_name')}." + + elif name == "weather_report": + r = await loop.run_in_executor(None, lambda: weather_action(parameters=args, player=self.ui)) + result = r or "Weather delivered." + + elif name == "browser_control": + r = await loop.run_in_executor(None, lambda: browser_control(parameters=args, player=self.ui)) + result = r or "Done." + + elif name == "file_controller": + r = await loop.run_in_executor(None, lambda: file_controller(parameters=args, player=self.ui)) + result = r or "Done." + + elif name == "send_message": + r = await loop.run_in_executor(None, lambda: send_message(parameters=args, response=None, player=self.ui, session_memory=None)) + result = r or f"Message sent to {args.get('receiver')}." + + elif name == "reminder": + r = await loop.run_in_executor(None, lambda: reminder(parameters=args, response=None, player=self.ui)) + result = r or "Reminder set." + + elif name == "youtube_video": + r = await loop.run_in_executor(None, lambda: youtube_video(parameters=args, response=None, player=self.ui)) + result = r or "Done." + elif name == "file_processor": + if not args.get("file_path") and self.ui.current_file: + args["file_path"] = self.ui.current_file + r = await loop.run_in_executor( + None, + lambda: file_processor(parameters=args, player=self.ui, speak=self.speak) + ) + result = r or "Done." + + + elif name == "screen_process": + threading.Thread( + target=screen_process, + kwargs={"parameters": args, "response": None, + "player": self.ui, "session_memory": None}, + daemon=True + ).start() + result = "Vision module activated. Stay completely silent β€” vision module will speak directly." + + elif name == "computer_settings": + r = await loop.run_in_executor(None, lambda: computer_settings(parameters=args, response=None, player=self.ui)) + result = r or "Done." + + elif name == "desktop_control": + r = await loop.run_in_executor(None, lambda: desktop_control(parameters=args, player=self.ui)) + result = r or "Done." + + elif name == "code_helper": + r = await loop.run_in_executor(None, lambda: code_helper(parameters=args, player=self.ui, speak=self.speak)) + result = r or "Done." + + elif name == "dev_agent": + r = await loop.run_in_executor(None, lambda: dev_agent(parameters=args, player=self.ui, speak=self.speak)) + result = r or "Done." + + elif name == "agent_task": + from agent.task_queue import get_queue, TaskPriority + priority_map = {"low": TaskPriority.LOW, "normal": TaskPriority.NORMAL, "high": TaskPriority.HIGH} + priority = priority_map.get(args.get("priority", "normal").lower(), TaskPriority.NORMAL) + task_id = get_queue().submit(goal=args.get("goal", ""), priority=priority, speak=self.speak) + result = f"Task started (ID: {task_id})." + + elif name == "web_search": + r = await loop.run_in_executor(None, lambda: web_search_action(parameters=args, player=self.ui)) + result = r or "Done." + + elif name == "computer_control": + r = await loop.run_in_executor(None, lambda: computer_control(parameters=args, player=self.ui)) + result = r or "Done." + + elif name == "game_updater": + r = await loop.run_in_executor(None, lambda: game_updater(parameters=args, player=self.ui, speak=self.speak)) + result = r or "Done." + + elif name == "flight_finder": + r = await loop.run_in_executor(None, lambda: flight_finder(parameters=args, player=self.ui)) + result = r or "Done." + elif name == "shutdown_jarvis": + self.ui.write_log("SYS: Shutdown requested.") + self.speak("Goodbye, sir.") + + def _shutdown(): + import time, sys, os + time.sleep(1) + os._exit(0) + + threading.Thread(target=_shutdown, daemon=True).start() + else: + result = f"Unknown tool: {name}" + + except Exception as e: + result = f"Tool '{name}' failed: {e}" + traceback.print_exc() + self.speak_error(name, e) + + if not self.ui.muted: + self.ui.set_state("LISTENING") + + print(f"[JARVIS] πŸ“€ {name} β†’ {str(result)[:80]}") + + return types.FunctionResponse( + id=fc.id, name=name, + response={"result": result} + ) + + async def _send_realtime(self): + while True: + msg = await self.out_queue.get() + await self.session.send_realtime_input(media=msg) + + async def _listen_audio(self): + print("[JARVIS] 🎀 Mic started") + loop = asyncio.get_event_loop() + + def callback(indata, frames, time_info, status): + with self._speaking_lock: + jarvis_speaking = self._is_speaking + if not jarvis_speaking and not self.ui.muted: + data = indata.tobytes() + loop.call_soon_threadsafe( + self.out_queue.put_nowait, + {"data": data, "mime_type": "audio/pcm"} + ) + + try: + with sd.InputStream( + samplerate=SEND_SAMPLE_RATE, + channels=CHANNELS, + dtype="int16", + blocksize=CHUNK_SIZE, + callback=callback, + ): + print("[JARVIS] 🎀 Mic stream open") + while True: + await asyncio.sleep(0.1) + except Exception as e: + print(f"[JARVIS] ❌ Mic: {e}") + raise + + async def _receive_audio(self): + print("[JARVIS] πŸ‘‚ Recv started") + out_buf, in_buf = [], [] + + try: + while True: + async for response in self.session.receive(): + + if response.data: + self.audio_in_queue.put_nowait(response.data) + + if response.server_content: + sc = response.server_content + + if sc.output_transcription and sc.output_transcription.text: + self.set_speaking(True) + txt = sc.output_transcription.text.strip() + if txt: + out_buf.append(txt) + + if sc.input_transcription and sc.input_transcription.text: + txt = sc.input_transcription.text.strip() + if txt: + in_buf.append(txt) + + if sc.turn_complete: + self.set_speaking(False) + + full_in = " ".join(in_buf).strip() + if full_in: + self.ui.write_log(f"You: {full_in}") + in_buf = [] + + full_out = " ".join(out_buf).strip() + if full_out: + self.ui.write_log(f"Jarvis: {full_out}") + out_buf = [] + + if full_in and len(full_in) > 5: + threading.Thread( + target=_update_memory_async, + args=(full_in, full_out), + daemon=True + ).start() + + if response.tool_call: + fn_responses = [] + for fc in response.tool_call.function_calls: + print(f"[JARVIS] πŸ“ž {fc.name}") + fr = await self._execute_tool(fc) + fn_responses.append(fr) + await self.session.send_tool_response( + function_responses=fn_responses + ) + + except Exception as e: + print(f"[JARVIS] ❌ Recv: {e}") + traceback.print_exc() + raise + + async def _play_audio(self): + print("[JARVIS] πŸ”Š Play started") + loop = asyncio.get_event_loop() + + stream = sd.RawOutputStream( + samplerate=RECEIVE_SAMPLE_RATE, + channels=CHANNELS, + dtype="int16", + blocksize=CHUNK_SIZE, + ) + stream.start() + try: + while True: + chunk = await self.audio_in_queue.get() + self.set_speaking(True) + await asyncio.to_thread(stream.write, chunk) + except Exception as e: + print(f"[JARVIS] ❌ Play: {e}") + raise + finally: + self.set_speaking(False) + stream.stop() + stream.close() + + async def run(self): + client = genai.Client( + api_key=_get_api_key(), + http_options={"api_version": "v1beta"} + ) + + while True: + try: + print("[JARVIS] πŸ”Œ Connecting...") + self.ui.set_state("THINKING") + config = self._build_config() + + async with ( + client.aio.live.connect(model=LIVE_MODEL, config=config) as session, + asyncio.TaskGroup() as tg, + ): + self.session = session + self._loop = asyncio.get_event_loop() + self.audio_in_queue = asyncio.Queue() + self.out_queue = asyncio.Queue(maxsize=10) + + print("[JARVIS] βœ… Connected.") + self.ui.set_state("LISTENING") + self.ui.write_log("SYS: JARVIS online.") + + tg.create_task(self._send_realtime()) + tg.create_task(self._listen_audio()) + tg.create_task(self._receive_audio()) + tg.create_task(self._play_audio()) + + except Exception as e: + print(f"[JARVIS] ⚠️ {e}") + traceback.print_exc() + + self.set_speaking(False) + self.ui.set_state("THINKING") + print("[JARVIS] πŸ”„ Reconnecting in 3s...") + await asyncio.sleep(3) + +def main(): + ui = JarvisUI("face.png") + + def runner(): + ui.wait_for_api_key() + jarvis = JarvisLive(ui) + try: + asyncio.run(jarvis.run()) + except KeyboardInterrupt: + print("\nπŸ”΄ Shutting down...") + + threading.Thread(target=runner, daemon=True).start() + ui.root.mainloop() + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/memory/__init__.py b/memory/__init__.py new file mode 100644 index 0000000..452b936 --- /dev/null +++ b/memory/__init__.py @@ -0,0 +1 @@ +#memory package \ No newline at end of file diff --git a/memory/config_manager.py b/memory/config_manager.py new file mode 100644 index 0000000..16551f2 --- /dev/null +++ b/memory/config_manager.py @@ -0,0 +1,59 @@ +import json +import sys +from pathlib import Path + + +def get_base_dir() -> Path: + if getattr(sys, "frozen", False): + return Path(sys.executable).parent + return Path(__file__).resolve().parent.parent + + +BASE_DIR = get_base_dir() +CONFIG_DIR = BASE_DIR / "config" +CONFIG_FILE = CONFIG_DIR / "api_keys.json" + + +def ensure_config_dir() -> None: + CONFIG_DIR.mkdir(parents=True, exist_ok=True) + + +def config_exists() -> bool: + return CONFIG_FILE.exists() + + +def save_api_keys(gemini_api_key: str) -> None: + ensure_config_dir() + + data: dict = {} + if CONFIG_FILE.exists(): + try: + data = json.loads(CONFIG_FILE.read_text(encoding="utf-8")) + except Exception: + data = {} + + data["gemini_api_key"] = gemini_api_key.strip() + + CONFIG_FILE.write_text( + json.dumps(data, indent=2), + encoding="utf-8" + ) + + +def load_api_keys() -> dict: + if not CONFIG_FILE.exists(): + return {} + try: + return json.loads(CONFIG_FILE.read_text(encoding="utf-8")) + except Exception as e: + print(f"❌ Failed to load api_keys.json: {e}") + return {} + + +def get_gemini_key() -> str | None: + return load_api_keys().get("gemini_api_key") + + +def is_configured() -> bool: + key = get_gemini_key() + return bool(key and len(key) > 15) \ No newline at end of file diff --git a/memory/memory_manager.py b/memory/memory_manager.py new file mode 100644 index 0000000..705cd7d --- /dev/null +++ b/memory/memory_manager.py @@ -0,0 +1,316 @@ +import json +import re +from datetime import datetime +from threading import Lock +from pathlib import Path +import sys + + +def get_base_dir() -> Path: + if getattr(sys, "frozen", False): + return Path(sys.executable).parent + return Path(__file__).resolve().parent.parent + + +BASE_DIR = get_base_dir() +MEMORY_PATH = BASE_DIR / "memory" / "long_term.json" +_lock = Lock() +MAX_VALUE_LENGTH = 380 +MEMORY_MAX_CHARS = 2200 + + +def _empty_memory() -> dict: + return { + "identity": {}, + "preferences": {}, + "projects": {}, + "relationships": {}, + "wishes": {}, + "notes": {} + } + + +def load_memory() -> dict: + if not MEMORY_PATH.exists(): + return _empty_memory() + + with _lock: + try: + data = json.loads(MEMORY_PATH.read_text(encoding="utf-8")) + if isinstance(data, dict): + base = _empty_memory() + for key in base: + if key not in data: + data[key] = {} + return data + return _empty_memory() + except Exception as e: + print(f"[Memory] ⚠️ Load error: {e}") + return _empty_memory() + + +def _all_entries(memory: dict) -> list[tuple]: + entries = [] + for cat, items in memory.items(): + if not isinstance(items, dict): + continue + for key, entry in items.items(): + if isinstance(entry, dict) and "value" in entry: + entries.append((cat, key, entry)) + return entries + + +def _trim_to_limit(memory: dict) -> dict: + serialized = json.dumps(memory, ensure_ascii=False) + if len(serialized) <= MEMORY_MAX_CHARS: + return memory + + entries = _all_entries(memory) + entries.sort(key=lambda t: t[2].get("updated", "0000-00-00")) + + for cat, key, _ in entries: + if len(json.dumps(memory, ensure_ascii=False)) <= MEMORY_MAX_CHARS: + break + del memory[cat][key] + print(f"[Memory] πŸ—‘οΈ Trimmed {cat}/{key} (limit: {MEMORY_MAX_CHARS} chars)") + + return memory + + +def save_memory(memory: dict) -> None: + if not isinstance(memory, dict): + return + + memory = _trim_to_limit(memory) + + MEMORY_PATH.parent.mkdir(parents=True, exist_ok=True) + with _lock: + MEMORY_PATH.write_text( + json.dumps(memory, indent=2, ensure_ascii=False), + encoding="utf-8" + ) + + +def _truncate_value(val: str) -> str: + if isinstance(val, str) and len(val) > MAX_VALUE_LENGTH: + return val[:MAX_VALUE_LENGTH].rstrip() + "…" + return val + + +def _recursive_update(target: dict, updates: dict) -> bool: + changed = False + for key, value in updates.items(): + if value is None: + continue + if isinstance(value, str) and not value.strip(): + continue + + if isinstance(value, dict) and "value" not in value: + if key not in target or not isinstance(target[key], dict): + target[key] = {} + changed = True + if _recursive_update(target[key], value): + changed = True + else: + if isinstance(value, dict) and "value" in value: + new_val = _truncate_value(str(value["value"])) + else: + new_val = _truncate_value(str(value)) + + entry = {"value": new_val, "updated": datetime.now().strftime("%Y-%m-%d")} + existing = target.get(key, {}) + if not isinstance(existing, dict) or existing.get("value") != new_val: + target[key] = entry + changed = True + + return changed + + +def update_memory(memory_update: dict) -> dict: + if not isinstance(memory_update, dict) or not memory_update: + return load_memory() + + memory = load_memory() + if _recursive_update(memory, memory_update): + save_memory(memory) + print(f"[Memory] πŸ’Ύ Saved: {list(memory_update.keys())}") + return memory + + +def should_extract_memory(user_text: str, jarvis_text: str, api_key: str = "") -> bool: + try: + from or_client import client + + combined = f"User: {user_text[:300]}\nJarvis: {jarvis_text[:1000]}" + + result = client.chat( + f"Does this conversation contain ANY of the following?\n" + f"- Personal facts (name, age, city, job, birthday, nationality)\n" + f"- Preferences or favorites (food, color, music, sport, game, film, book, etc.)\n" + f"- Active projects or goals the user is working on\n" + f"- People in the user's life (friends, family, partner, colleagues)\n" + f"- Things the user wants to do or buy in the future\n" + f"- Any other fact worth remembering long-term\n\n" + f"Reply only YES or NO.\n\nConversation:\n{combined}", + system="You are a memory relevance checker. Reply only YES or NO.", + max_tokens=5, + temperature=0.0, + ) + return "YES" in result.upper() + + except Exception as e: + print(f"[Memory] ⚠️ Stage1 check failed: {e}") + return False + + +def extract_memory(user_text: str, jarvis_text: str, api_key: str = "") -> dict: + try: + from or_client import client + + combined = f"User: {user_text[:600]}\nJarvis: {jarvis_text[:300]}" + + raw = client.chat( + f"Extract ALL memorable personal facts from this conversation. Any language.\n" + f"Return ONLY valid JSON. Use {{}} if truly nothing is worth saving.\n\n" + f"Category guide:\n" + f" identity β†’ name, age, birthday, city, country, job, school, nationality, language\n" + f" preferences β†’ ANY favorite or preferred thing:\n" + f" favorite_food, favorite_color, favorite_music, favorite_film,\n" + f" favorite_game, favorite_sport, favorite_book, favorite_artist,\n" + f" favorite_country, hobbies, interests, dislikes, etc.\n" + f" projects β†’ projects being built, ongoing work, goals, ideas in progress\n" + f" (e.g. mark_xxv: 'Building a JARVIS-like AI assistant')\n" + f" relationships β†’ people mentioned: friends, family, partner, colleagues\n" + f" (e.g. best_friend_ali: 'Best friend, met in university')\n" + f" wishes β†’ future plans, things to buy, travel plans, dreams\n" + f" notes β†’ anything else worth remembering (habits, schedule, etc.)\n\n" + f"IMPORTANT:\n" + f"- Be LIBERAL: if something MIGHT be worth remembering, include it.\n" + f"- Extract from BOTH user and Jarvis turns.\n" + f"- Skip: weather, reminders, search results, one-time commands.\n" + f"- Use concise English values regardless of conversation language.\n\n" + f"Format:\n" + f'{{"identity":{{"name":{{"value":"Ali"}}}},\n' + f' "preferences":{{"favorite_color":{{"value":"blue"}}}},\n' + f' "projects":{{"mark_xxv":{{"value":"JARVIS-like AI assistant"}}}},\n' + f' "relationships":{{"friend_yusuf":{{"value":"close friend"}}}},\n' + f' "wishes":{{"buy_guitar":{{"value":"wants an acoustic guitar"}}}},\n' + f' "notes":{{"works_at_night":{{"value":"usually active late at night"}}}}}}\n\n' + f"Conversation:\n{combined}\n\nJSON:", + system="Return ONLY valid JSON. No markdown, no explanation, no extra text.", + max_tokens=1024, + temperature=0.2, + ) + + clean = raw.strip() + clean = re.sub(r"```(?:json)?", "", clean).strip().rstrip("`").strip() + + if not clean or clean == "{}": + return {} + + return json.loads(clean) + + except json.JSONDecodeError: + return {} + except Exception as e: + if "429" not in str(e): + print(f"[Memory] ⚠️ Extract failed: {e}") + return {} + + +def format_memory_for_prompt(memory: dict | None) -> str: + if not memory: + return "" + + lines = [] + + identity = memory.get("identity", {}) + id_fields = ["name", "age", "birthday", "city", "job", "language", "school", "nationality"] + for field in id_fields: + entry = identity.get(field) + if entry: + val = entry.get("value") if isinstance(entry, dict) else entry + if val: + lines.append(f"{field.title()}: {val}") + for key, entry in identity.items(): + if key in id_fields: + continue + val = entry.get("value") if isinstance(entry, dict) else entry + if val: + lines.append(f"{key.replace('_', ' ').title()}: {val}") + + prefs = memory.get("preferences", {}) + if prefs: + lines.append("") + lines.append("Preferences:") + for key, entry in list(prefs.items())[:15]: + val = entry.get("value") if isinstance(entry, dict) else entry + if val: + lines.append(f" - {key.replace('_', ' ').title()}: {val}") + + projects = memory.get("projects", {}) + if projects: + lines.append("") + lines.append("Active Projects / Goals:") + for key, entry in list(projects.items())[:8]: + val = entry.get("value") if isinstance(entry, dict) else entry + if val: + lines.append(f" - {key.replace('_', ' ').title()}: {val}") + + rels = memory.get("relationships", {}) + if rels: + lines.append("") + lines.append("People in their life:") + for key, entry in list(rels.items())[:10]: + val = entry.get("value") if isinstance(entry, dict) else entry + if val: + lines.append(f" - {key.replace('_', ' ').title()}: {val}") + + wishes = memory.get("wishes", {}) + if wishes: + lines.append("") + lines.append("Wishes / Plans / Wants:") + for key, entry in list(wishes.items())[:8]: + val = entry.get("value") if isinstance(entry, dict) else entry + if val: + lines.append(f" - {key.replace('_', ' ').title()}: {val}") + + notes = memory.get("notes", {}) + if notes: + lines.append("") + lines.append("Other notes:") + for key, entry in list(notes.items())[:8]: + val = entry.get("value") if isinstance(entry, dict) else entry + if val: + lines.append(f" - {key}: {val}") + + if not lines: + return "" + + header = "[WHAT YOU KNOW ABOUT THIS PERSON β€” use naturally, never recite like a list]\n" + result = header + "\n".join(lines) + if len(result) > 2000: + result = result[:1997] + "…" + + return result + "\n" + + +def remember(key: str, value: str, category: str = "notes") -> str: + valid = {"identity", "preferences", "projects", "relationships", "wishes", "notes"} + if category not in valid: + category = "notes" + update_memory({category: {key: {"value": value}}}) + return f"Remembered: {category}/{key} = {value}" + + +def forget(key: str, category: str = "notes") -> str: + memory = load_memory() + cat = memory.get(category, {}) + if key in cat: + del cat[key] + memory[category] = cat + save_memory(memory) + return f"Forgotten: {category}/{key}" + return f"Not found: {category}/{key}" + +forget_memory = forget \ No newline at end of file diff --git a/or_client.py b/or_client.py new file mode 100644 index 0000000..08be1d2 --- /dev/null +++ b/or_client.py @@ -0,0 +1,376 @@ +import json +import sys +import time +import base64 +import logging +from pathlib import Path +from typing import Optional + +import requests + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger("openrouter_client") + +def _get_base_dir() -> Path: + if getattr(sys, "frozen", False): + return Path(sys.executable).parent + return Path(__file__).resolve().parent + + +BASE_DIR = _get_base_dir() +API_KEY_PATH = BASE_DIR / "config" / "api_keys.json" + +def _load_api_key() -> str: + try: + with open(API_KEY_PATH, "r", encoding="utf-8") as f: + data = json.load(f) + key = data.get("openrouter_api_key", "").strip() + if not key: + raise ValueError("openrouter_api_key is empty in api_keys.json") + return key + except FileNotFoundError: + raise RuntimeError(f"api_keys.json not found at: {API_KEY_PATH}") + except Exception as e: + raise RuntimeError(f"Failed to load OpenRouter API key: {e}") + +TEXT_MODELS: list[str] = [ + "nvidia/nemotron-3-super-120b-a12b:free", + "nousresearch/hermes-3-llama-3.1-405b:free", + "minimax/minimax-m2.5:free", + "meta-llama/llama-3.3-70b-instruct:free", + "qwen/qwen3-next-80b-a3b-instruct:free", + "qwen/qwen3-coder:free", + "google/gemma-4-31b-it:free", + "google/gemma-4-26b-a4b-it:free", + "google/gemma-3-27b-it:free", + "arcee-ai/trinity-large-preview:free", + "z-ai/glm-4.5-air:free", + "nvidia/nemotron-3-nano-30b-a3b:free", + "cognitivecomputations/dolphin-mistral-24b-venice-edition:free", + "google/gemma-3-12b-it:free", + "nvidia/nemotron-nano-12b-v2-vl:free", + "nvidia/nemotron-nano-9b-v2:free", + "google/gemma-3-4b-it:free", + "google/gemma-3n-e4b-it:free", + "meta-llama/llama-3.2-3b-instruct:free", + "google/gemma-3n-e2b-it:free", + "liquid/lfm-2.5-1.2b-instruct:free", + "liquid/lfm-2.5-1.2b-thinking:free", +] + +VISION_MODELS: list[str] = [ + "nvidia/nemotron-nano-12b-v2-vl:free", + "nvidia/llama-nemotron-embed-vl-1b-v2:free", + "google/gemma-4-31b-it:free", + "google/gemma-4-26b-a4b-it:free", + "google/gemma-3n-e4b-it:free", + "google/gemma-3n-e2b-it:free", + "meta-llama/llama-3.3-70b-instruct:free", + "nvidia/nemotron-3-super-120b-a12b:free", +] + +API_URL = "https://openrouter.ai/api/v1/chat/completions" +DEFAULT_MAX_TOKENS = 4096 +DEFAULT_TEMPERATURE = 0.7 +REQUEST_TIMEOUT = 60 # seconds per request +MAX_RETRIES_PER_MODEL = 2 # attempts before moving to next model +RETRY_DELAY = 2 # seconds between retries +RATE_LIMIT_COOLDOWN = 60 # seconds before retrying a rate-limited model + +_rate_limited: dict[str, float] = {} + +class OpenRouterClient: + + def __init__(self) -> None: + self.api_key = _load_api_key() + self._headers = { + "Authorization": f"Bearer {self.api_key}", + "Content-Type": "application/json", + "HTTP-Referer": "https://github.com/mark-xxv", + "X-Title": "MARK XXV", + } + + def _is_rate_limited(self, model: str) -> bool: + ts = _rate_limited.get(model) + if ts is None: + return False + if time.time() - ts > RATE_LIMIT_COOLDOWN: + del _rate_limited[model] + return False + return True + + def _mark_rate_limited(self, model: str) -> None: + _rate_limited[model] = time.time() + logger.warning( + f"[OpenRouter] Rate limited: {model} β€” " + f"cooling down for {RATE_LIMIT_COOLDOWN}s" + ) + + def _call( + self, + model: str, + messages: list[dict], + max_tokens: int = DEFAULT_MAX_TOKENS, + temperature: float = DEFAULT_TEMPERATURE, + response_format: Optional[dict] = None, + ) -> Optional[str]: + payload: dict = { + "model": model, + "messages": messages, + "max_tokens": max_tokens, + "temperature": temperature, + } + if response_format: + payload["response_format"] = response_format + + for attempt in range(1, MAX_RETRIES_PER_MODEL + 1): + try: + resp = requests.post( + API_URL, + headers=self._headers, + json=payload, + timeout=REQUEST_TIMEOUT, + ) + + if resp.status_code == 429: + self._mark_rate_limited(model) + return None + + if resp.status_code == 200: + data = resp.json() + content = ( + data.get("choices", [{}])[0] + .get("message", {}) + .get("content", "") + ) + return content.strip() if content else None + + logger.warning( + f"[OpenRouter] {model} β†’ HTTP {resp.status_code} " + f"(attempt {attempt}/{MAX_RETRIES_PER_MODEL})" + ) + + except requests.exceptions.Timeout: + logger.warning( + f"[OpenRouter] {model} β†’ Timeout " + f"(attempt {attempt}/{MAX_RETRIES_PER_MODEL})" + ) + except Exception as e: + logger.error(f"[OpenRouter] {model} β†’ Unexpected error: {e}") + + if attempt < MAX_RETRIES_PER_MODEL: + time.sleep(RETRY_DELAY) + + return None + + def _call_with_fallback( + self, + pool: list[str], + messages: list[dict], + model: Optional[str] = None, + max_tokens: int = DEFAULT_MAX_TOKENS, + temperature: float = DEFAULT_TEMPERATURE, + response_format: Optional[dict] = None, + ) -> str: + if model and not self._is_rate_limited(model): + result = self._call(model, messages, max_tokens, temperature, response_format) + if result: + return result + logger.info( + f"[OpenRouter] Requested model failed, " + f"falling back to pool: {model}" + ) + + for m in pool: + if self._is_rate_limited(m): + continue + logger.info(f"[OpenRouter] Trying: {m}") + result = self._call(m, messages, max_tokens, temperature, response_format) + if result: + logger.info(f"[OpenRouter] βœ“ Success: {m}") + return result + + raise RuntimeError( + "[OpenRouter] All models failed or are rate-limited. " + "Check your API key and network connection." + ) + + def chat( + self, + prompt: str, + system: str = ( + "You are a component of MARK XXV, an AI assistant inspired by JARVIS. " + "Be concise, helpful, and precise." + ), + model: Optional[str] = None, + max_tokens: int = DEFAULT_MAX_TOKENS, + temperature: float = DEFAULT_TEMPERATURE, + ) -> str: + messages = [ + {"role": "system", "content": system}, + {"role": "user", "content": prompt}, + ] + return self._call_with_fallback( + TEXT_MODELS, messages, model, max_tokens, temperature + ) + + def chat_json( + self, + prompt: str, + system: str = ( + "Return ONLY valid JSON. " + "No markdown fences, no extra text, no explanation." + ), + model: Optional[str] = None, + max_tokens: int = DEFAULT_MAX_TOKENS, + ) -> dict: + messages = [ + {"role": "system", "content": system}, + {"role": "user", "content": prompt}, + ] + raw = self._call_with_fallback( + TEXT_MODELS, messages, model, max_tokens, temperature=0.2 + ) + + clean = raw.strip() + if clean.startswith("```"): + parts = clean.split("```") + clean = parts[1] if len(parts) > 1 else clean + if clean.startswith("json"): + clean = clean[4:] + clean = clean.strip().rstrip("`").strip() + + try: + return json.loads(clean) + except json.JSONDecodeError as e: + logger.error( + f"[OpenRouter] JSON parse failed: {e}\n" + f"Raw response (first 300 chars): {raw[:300]}" + ) + raise ValueError( + f"Model returned unparseable JSON: {e}\n" + f"Raw output: {raw[:200]}" + ) + + def vision( + self, + prompt: str, + image_b64: str, + mime: str = "image/png", + system: str = "Analyze the image and describe what you see clearly and concisely.", + model: Optional[str] = None, + max_tokens: int = 1024, + ) -> str: + messages = [ + {"role": "system", "content": system}, + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": { + "url": f"data:{mime};base64,{image_b64}" + }, + }, + {"type": "text", "text": prompt}, + ], + }, + ] + return self._call_with_fallback( + VISION_MODELS, messages, model, max_tokens, temperature=0.2 + ) + + def vision_from_file( + self, + prompt: str, + image_path: str, + system: str = "Analyze the image and describe what you see clearly and concisely.", + model: Optional[str] = None, + max_tokens: int = 1024, + ) -> str: + path = Path(image_path) + mime_map = { + ".png": "image/png", + ".jpg": "image/jpeg", + ".jpeg": "image/jpeg", + ".webp": "image/webp", + ".gif": "image/gif", + } + mime = mime_map.get(path.suffix.lower(), "image/png") + + with open(path, "rb") as f: + image_b64 = base64.b64encode(f.read()).decode("utf-8") + + return self.vision(prompt, image_b64, mime, system, model, max_tokens) + + def multi_turn( + self, + messages: list[dict], + model: Optional[str] = None, + max_tokens: int = DEFAULT_MAX_TOKENS, + temperature: float = DEFAULT_TEMPERATURE, + ) -> str: + + return self._call_with_fallback( + TEXT_MODELS, messages, model, max_tokens, temperature + ) + + def available_models(self) -> dict: + return { + "text_models": TEXT_MODELS, + "vision_models": VISION_MODELS, + "rate_limited": list(_rate_limited.keys()), + "total_text": len(TEXT_MODELS), + "total_vision": len(VISION_MODELS), + } + +client = OpenRouterClient() + +if __name__ == "__main__": + print("=" * 55) + print(" MARK XXV β€” OpenRouter Client Self-Test") + print("=" * 55) + + print("\n[TEST 1] Basic chat...") + try: + reply = client.chat("Introduce yourself in one sentence.") + print(f" Response : {reply}") + print(f" Status : PASS βœ“") + except Exception as e: + print(f" Status : FAIL βœ— β€” {e}") + + print("\n[TEST 2] JSON mode...") + try: + data = client.chat_json( + 'List 3 programming languages. Format: {"languages": ["a", "b", "c"]}', + system="Return only valid JSON. No extra text." + ) + print(f" Response : {data}") + print(f" Status : PASS βœ“") + except Exception as e: + print(f" Status : FAIL βœ— β€” {e}") + + print("\n[TEST 3] Multi-turn conversation...") + try: + history = [ + {"role": "system", "content": "You are a helpful assistant. Be brief."}, + {"role": "user", "content": "My name is Tony."}, + {"role": "assistant", "content": "Hello Tony, how can I help you?"}, + {"role": "user", "content": "What is my name?"}, + ] + reply = client.multi_turn(history) + print(f" Response : {reply}") + print(f" Status : PASS βœ“") + except Exception as e: + print(f" Status : FAIL βœ— β€” {e}") + + print("\n[TEST 4] Model pool info...") + info = client.available_models() + print(f" Text models : {info['total_text']}") + print(f" Vision models : {info['total_vision']}") + print(f" Rate limited : {info['rate_limited'] or 'none'}") + print(f" Status : PASS βœ“") + + print("\n" + "=" * 55) + print(" All tests complete.") + print("=" * 55) \ No newline at end of file