From a8674a28748e5c7f7745e6589d392e5cd6230ed6 Mon Sep 17 00:00:00 2001 From: FatihMakes Date: Mon, 2 Mar 2026 04:30:10 +0300 Subject: [PATCH] Add files via upload --- actions/browser_control.py | 476 +++++++++++++++++++ actions/cmd_control.py | 230 ++++++++++ actions/code_helper.py | 583 ++++++++++++++++++++++++ actions/computer_control.py | 554 +++++++++++++++++++++++ actions/computer_settings.py | 679 ++++++++++++++++++++++++++++ actions/desktop.py | 417 +++++++++++++++++ actions/dev_agent.py | 491 ++++++++++++++++++++ actions/file_controller.py | 482 ++++++++++++++++++++ actions/flight_finder.py | 419 +++++++++++++++++ actions/open_app.py | 208 +++++++++ actions/reminder.py | 156 +++++++ actions/screen_processor.py | 404 +++++++++++++++++ actions/send_message.py | 215 +++++++++ actions/weather_report.py | 62 +++ actions/web_search.py | 135 ++++++ actions/youtube_video.py | 517 +++++++++++++++++++++ agent/error_handler.py | 198 ++++++++ agent/executor.py | 398 ++++++++++++++++ agent/planner.py | 258 +++++++++++ agent/task_queue.py | 221 +++++++++ config/__init__.py | 0 core/prompt.txt | 23 + main.py | 853 +++++++++++++++++++++++++++++++++++ memory/__init__.py | 1 + memory/config_manager.py | 59 +++ memory/memory_manager.py | 151 +++++++ readme.md | 59 +++ requirements.txt | 19 + setup.py | 11 + ui.py | 380 ++++++++++++++++ 30 files changed, 8659 insertions(+) create mode 100644 actions/browser_control.py create mode 100644 actions/cmd_control.py create mode 100644 actions/code_helper.py create mode 100644 actions/computer_control.py create mode 100644 actions/computer_settings.py create mode 100644 actions/desktop.py create mode 100644 actions/dev_agent.py create mode 100644 actions/file_controller.py create mode 100644 actions/flight_finder.py create mode 100644 actions/open_app.py create mode 100644 actions/reminder.py create mode 100644 actions/screen_processor.py create mode 100644 actions/send_message.py create mode 100644 actions/weather_report.py create mode 100644 actions/web_search.py create mode 100644 actions/youtube_video.py create mode 100644 agent/error_handler.py create mode 100644 agent/executor.py create mode 100644 agent/planner.py create mode 100644 agent/task_queue.py create mode 100644 config/__init__.py create mode 100644 core/prompt.txt create mode 100644 main.py create mode 100644 memory/__init__.py create mode 100644 memory/config_manager.py create mode 100644 memory/memory_manager.py create mode 100644 readme.md create mode 100644 requirements.txt create mode 100644 setup.py create mode 100644 ui.py diff --git a/actions/browser_control.py b/actions/browser_control.py new file mode 100644 index 0000000..2cf9878 --- /dev/null +++ b/actions/browser_control.py @@ -0,0 +1,476 @@ +import asyncio +import threading +import concurrent.futures +import platform +import shutil +import subprocess +from pathlib import Path +from playwright.async_api import async_playwright, TimeoutError as PlaywrightTimeout + +def _get_default_browser_id() -> str: + """Returns raw default browser identifier string for current OS.""" + system = platform.system() + try: + if system == "Windows": + import winreg + key = winreg.OpenKey( + winreg.HKEY_CURRENT_USER, + r"Software\Microsoft\Windows\Shell\Associations\UrlAssociations\http\UserChoice" + ) + prog_id = winreg.QueryValueEx(key, "ProgId")[0].lower() + winreg.CloseKey(key) + return prog_id + + elif system == "Darwin": + result = subprocess.run( + ["defaults", "read", + "com.apple.LaunchServices/com.apple.launchservices.secure", + "LSHandlers"], + capture_output=True, text=True, timeout=5 + ) + return result.stdout.lower() + + elif system == "Linux": + result = subprocess.run( + ["xdg-settings", "get", "default-web-browser"], + capture_output=True, text=True, timeout=5 + ) + return result.stdout.lower() + + except Exception: + pass + + return "" + +_BROWSER_BINARIES = { + "Windows": { + "opera": ["opera.exe"], + "brave": ["brave.exe"], + "vivaldi": ["vivaldi.exe"], + "chrome": ["chrome.exe"], + "firefox": ["firefox.exe"], + }, + "Darwin": { + "opera": ["opera"], + "brave": ["brave browser", "brave"], + "vivaldi": ["vivaldi"], + "chrome": ["google chrome", "google-chrome"], + "firefox": ["firefox"], + }, + "Linux": { + "opera": ["opera", "opera-stable"], + "brave": ["brave-browser", "brave"], + "vivaldi": ["vivaldi-stable", "vivaldi"], + "chrome": ["google-chrome", "google-chrome-stable", "chromium-browser", "chromium"], + "firefox": ["firefox"], + }, +} + + +def _get_opera_executable() -> str | None: + if platform.system() != "Windows": + return None + try: + import winreg + candidate_keys = [ + r"SOFTWARE\Microsoft\Windows\CurrentVersion\App Paths\opera.exe", + r"SOFTWARE\Microsoft\Windows\CurrentVersion\App Paths\launcher.exe", + r"SOFTWARE\Clients\StartMenuInternet\OperaStable\shell\open\command", + r"SOFTWARE\Clients\StartMenuInternet\OperaGXStable\shell\open\command", + ] + for key_path in candidate_keys: + for hive in [winreg.HKEY_LOCAL_MACHINE, winreg.HKEY_CURRENT_USER]: + try: + key = winreg.OpenKey(hive, key_path) + val = winreg.QueryValue(key, None) + winreg.CloseKey(key) + # Strip quotes and args + exe = val.strip().strip('"').split('"')[0].split(" --")[0].strip() + if exe and Path(exe).exists(): + print(f"[Browser] 🔍 Opera found via registry: {exe}") + return exe + except Exception: + continue + except Exception: + pass + return None + + +def _find_browser_executable(prog_id: str) -> tuple: + system = platform.system() + os_bins = _BROWSER_BINARIES.get(system, {}) + + if any(x in prog_id for x in ["firefox", "mozilla"]): + return "firefox", None, None + + if "safari" in prog_id: + return "webkit", None, None + + if "edge" in prog_id: + return "chromium", None, "msedge" + + if "opera" in prog_id: + exe = _get_opera_executable() + if exe: + return "chromium", exe, None + for binary in os_bins.get("opera", []): + path = shutil.which(binary) + if path: + return "chromium", path, None + + browser_patterns = { + "brave": ["brave"], + "vivaldi": ["vivaldi"], + "chrome": ["chrome"], + } + for browser_name, patterns in browser_patterns.items(): + if not any(p in prog_id for p in patterns): + continue + binaries = os_bins.get(browser_name, []) + for binary in binaries: + path = shutil.which(binary) + if path: + print(f"[Browser] 🔍 Found {browser_name} at: {path}") + return "chromium", path, None + + if "chrome" in prog_id or not prog_id: + return "chromium", None, "chrome" + + + return "chromium", None, None + + +class _BrowserThread: + + + def __init__(self): + self._loop = None + self._thread = None + self._ready = threading.Event() + self._playwright = None + self._browser = None + self._context = None + self._page = None + + def start(self): + if self._thread and self._thread.is_alive(): + return + self._thread = threading.Thread( + target=self._run_loop, daemon=True, name="BrowserThread" + ) + self._thread.start() + self._ready.wait(timeout=15) + + def _run_loop(self): + self._loop = asyncio.new_event_loop() + asyncio.set_event_loop(self._loop) + self._loop.run_until_complete(self._init()) + self._ready.set() + self._loop.run_forever() + + async def _init(self): + self._playwright = await async_playwright().start() + + def run(self, coro, timeout: int = 30): + if not self._loop: + raise RuntimeError("BrowserThread not started.") + future = asyncio.run_coroutine_threadsafe(coro, self._loop) + return future.result(timeout=timeout) + + async def _get_page(self): + if self._page is None or self._page.is_closed(): + await self._launch() + return self._page + + async def _launch(self): + prog_id = _get_default_browser_id() + engine_name, exe_path, channel = _find_browser_executable(prog_id) + engine = getattr(self._playwright, engine_name) + + launch_kwargs = {"headless": False} + + if engine_name == "chromium": + launch_kwargs["args"] = ["--start-maximized"] + + if exe_path: + launch_kwargs["executable_path"] = exe_path + elif channel: + launch_kwargs["channel"] = channel + + try: + if self._browser is None or not self._browser.is_connected(): + self._browser = await engine.launch(**launch_kwargs) + print( + f"[Browser] ✅ Launched ({engine_name}" + f"{' / ' + channel if channel else ''}" + f"{' / ' + exe_path if exe_path else ''})" + ) + except Exception as e: + print(f"[Browser] ⚠️ Launch failed ({e}), falling back to built-in Chromium") + self._browser = await self._playwright.chromium.launch( + headless=False, + args=["--start-maximized"] + ) + + self._context = await self._browser.new_context( + viewport=None, + user_agent=( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + "AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/120.0.0.0 Safari/537.36" + ) + ) + self._page = await self._context.new_page() + + async def _close(self): + if self._browser: + await self._browser.close() + self._browser = None + self._page = None + if self._playwright: + await self._playwright.stop() + self._playwright = None + + async def _go_to(self, url: str) -> str: + if not url.startswith("http"): + url = "https://" + url + page = await self._get_page() + try: + await page.goto(url, wait_until="domcontentloaded", timeout=15000) + return f"Opened: {page.url}" + except PlaywrightTimeout: + return f"Timeout loading: {url}" + except Exception as e: + return f"Navigation error: {e}" + + async def _search(self, query: str, engine: str = "google") -> str: + engines = { + "google": f"https://www.google.com/search?q={query.replace(' ', '+')}", + "bing": f"https://www.bing.com/search?q={query.replace(' ', '+')}", + "duckduckgo": f"https://duckduckgo.com/?q={query.replace(' ', '+')}", + } + url = engines.get(engine.lower(), engines["google"]) + return await self._go_to(url) + + async def _click(self, selector=None, text=None) -> str: + page = await self._get_page() + try: + if text: + await page.get_by_text(text, exact=False).first.click(timeout=8000) + return f"Clicked: '{text}'" + elif selector: + await page.click(selector, timeout=8000) + return f"Clicked: {selector}" + return "No selector or text provided." + except PlaywrightTimeout: + return "Element not found or not clickable." + except Exception as e: + return f"Click error: {e}" + + async def _type(self, selector=None, text: str = "", clear_first: bool = True) -> str: + page = await self._get_page() + try: + element = page.locator(selector).first if selector else page.locator(":focus") + if clear_first: + await element.clear() + await element.type(text, delay=50) + return "Text typed." + except Exception as e: + return f"Type error: {e}" + + async def _scroll(self, direction: str = "down", amount: int = 500) -> str: + page = await self._get_page() + try: + y = amount if direction == "down" else -amount + await page.mouse.wheel(0, y) + return f"Scrolled {direction}." + except Exception as e: + return f"Scroll error: {e}" + + async def _press(self, key: str) -> str: + page = await self._get_page() + try: + await page.keyboard.press(key) + return f"Pressed: {key}" + except Exception as e: + return f"Key error: {e}" + + async def _get_text(self) -> str: + page = await self._get_page() + try: + text = await page.inner_text("body") + return text[:4000] if len(text) > 4000 else text + except Exception as e: + return f"Could not get page text: {e}" + + async def _fill_form(self, fields: dict) -> str: + page = await self._get_page() + results = [] + for selector, value in fields.items(): + try: + el = page.locator(selector).first + await el.clear() + await el.type(str(value), delay=40) + results.append(f"✓ {selector}") + except Exception as e: + results.append(f"✗ {selector}: {e}") + return "Form filled: " + ", ".join(results) + + async def _smart_click(self, description: str) -> str: + page = await self._get_page() + desc_lower = description.lower() + + role_hints = { + "button": ["button", "buton", "btn"], + "link": ["link", "bağlantı"], + "searchbox": ["search", "arama"], + "textbox": ["input", "field", "alan"], + } + for role, keywords in role_hints.items(): + if any(k in desc_lower for k in keywords): + try: + await page.get_by_role(role).first.click(timeout=5000) + return f"Clicked ({role}): '{description}'" + except Exception: + pass + + try: + await page.get_by_text(description, exact=False).first.click(timeout=5000) + return f"Clicked (text): '{description}'" + except Exception: + pass + + try: + await page.get_by_placeholder(description, exact=False).first.click(timeout=5000) + return f"Clicked (placeholder): '{description}'" + except Exception: + pass + + return f"Could not find: '{description}'" + + async def _smart_type(self, description: str, text: str) -> str: + page = await self._get_page() + + for method, locator in [ + ("placeholder", page.get_by_placeholder(description, exact=False)), + ("label", page.get_by_label(description, exact=False)), + ("role", page.get_by_role("textbox")), + ]: + try: + el = locator.first + await el.clear() + await el.type(text, delay=50) + return f"Typed into ({method}): '{description}'" + except Exception: + continue + + return f"Could not find input: '{description}'" + + async def _close_browser(self) -> str: + await self._close() + return "Browser closed." + +_bt = _BrowserThread() +_bt_started = False +_bt_lock = threading.Lock() + + +def _ensure_started(): + global _bt_started + with _bt_lock: + if not _bt_started: + _bt.start() + _bt_started = True + +def browser_control( + parameters: dict, + response=None, + player=None, + session_memory=None +) -> str: + """ + Browser controller — auto-detects and uses system default browser. + + parameters: + action : go_to | search | click | type | scroll | fill_form | + smart_click | smart_type | get_text | press | close + url : URL for go_to + query : search query + engine : google | bing | duckduckgo (default: google) + selector : CSS selector for click/type + text : text to click or type + description : element description for smart_click/smart_type + direction : up | down for scroll + amount : scroll amount in pixels (default: 500) + key : key name for press (e.g. Enter, Escape, Tab) + fields : {selector: value} dict for fill_form + clear_first : bool, clear input before typing (default: True) + """ + _ensure_started() + + action = (parameters or {}).get("action", "").lower().strip() + result = "Unknown action." + + try: + if action == "go_to": + result = _bt.run(_bt._go_to(parameters.get("url", ""))) + + elif action == "search": + result = _bt.run(_bt._search( + parameters.get("query", ""), + parameters.get("engine", "google") + )) + + elif action == "click": + result = _bt.run(_bt._click( + selector=parameters.get("selector"), + text=parameters.get("text") + )) + + elif action == "type": + result = _bt.run(_bt._type( + selector=parameters.get("selector"), + text=parameters.get("text", ""), + clear_first=parameters.get("clear_first", True) + )) + + elif action == "scroll": + result = _bt.run(_bt._scroll( + direction=parameters.get("direction", "down"), + amount=parameters.get("amount", 500) + )) + + elif action == "fill_form": + result = _bt.run(_bt._fill_form(parameters.get("fields", {}))) + + elif action == "smart_click": + result = _bt.run(_bt._smart_click(parameters.get("description", ""))) + + elif action == "smart_type": + result = _bt.run(_bt._smart_type( + parameters.get("description", ""), + parameters.get("text", "") + )) + + elif action == "get_text": + result = _bt.run(_bt._get_text()) + + elif action == "press": + result = _bt.run(_bt._press(parameters.get("key", "Enter"))) + + elif action == "close": + result = _bt.run(_bt._close_browser()) + + else: + result = f"Unknown action: {action}" + + except concurrent.futures.TimeoutError: + result = "Browser action timed out." + except Exception as e: + result = f"Browser error: {e}" + + print(f"[Browser] {result[:80]}") + if player: + player.write_log(f"[browser] {result[:60]}") + + return result \ No newline at end of file diff --git a/actions/cmd_control.py b/actions/cmd_control.py new file mode 100644 index 0000000..69cb2cc --- /dev/null +++ b/actions/cmd_control.py @@ -0,0 +1,230 @@ +import subprocess +import sys +import json +import re +from pathlib import Path + + +def get_base_dir(): + if getattr(sys, "frozen", False): + return Path(sys.executable).parent + return Path(__file__).resolve().parent.parent + +BASE_DIR = get_base_dir() +API_CONFIG_PATH = BASE_DIR / "config" / "api_keys.json" + + +def _get_api_key() -> str: + with open(API_CONFIG_PATH, "r", encoding="utf-8") as f: + return json.load(f)["gemini_api_key"] + + +def _get_platform() -> str: + if sys.platform == "win32": return "windows" + if sys.platform == "darwin": return "macos" + return "linux" + +WIN_COMMAND_MAP = [ + (["disk space", "disk usage", "storage", "free space", "c drive space"], + "wmic logicaldisk get caption,freespace,size /format:list", False), + (["running processes", "list processes", "show processes", "active processes", "tasklist"], + "tasklist /fo table", False), + (["ip address", "my ip", "network info", "ipconfig"], + "ipconfig /all", False), + (["ping", "internet connection", "connected to internet"], + "ping -n 4 google.com", False), + (["open ports", "listening ports", "netstat"], + "netstat -an | findstr LISTENING", False), + (["wifi networks", "available wifi", "wireless networks"], + "netsh wlan show networks", False), + (["system info", "computer info", "hardware info", "pc info", "specs"], + "systeminfo", False), + (["cpu usage", "processor usage"], + "wmic cpu get loadpercentage", False), + (["memory usage", "ram usage"], + "wmic OS get FreePhysicalMemory,TotalVisibleMemorySize /Value", False), + (["windows version", "os version"], + "ver", False), + (["installed programs", "installed software", "installed apps"], + "wmic product get name,version /format:table", False), + (["battery", "battery level", "power status"], + "powershell (Get-WmiObject -Class Win32_Battery).EstimatedChargeRemaining", False), + (["current time", "what time", "system time"], + "time /t", False), + (["current date", "what date", "system date"], + "date /t", False), + (["desktop files", "files on desktop"], + f'dir "{Path.home() / "Desktop"}"', False), + (["downloads", "files in downloads"], + f'dir "{Path.home() / "Downloads"}"', False), + (["large files", "biggest files", "largest files"], + 'powershell "Get-ChildItem C:\\ -Recurse -ErrorAction SilentlyContinue | Sort-Object Length -Descending | Select-Object -First 10 FullName,Length | Format-Table"', False), +] + +def _find_hardcoded(task: str) -> str | None: + task_lower = task.lower() + + if "notepad" in task_lower or any(ext in task_lower for ext in [".txt", ".log", ".md", ".csv"]): + file_match = re.search(r'[\"\']?([\S]+\.(?:txt|log|md|csv|json|xml))[\"\']?', task, re.IGNORECASE) + if file_match: + filename = file_match.group(1) + desktop = Path.home() / "Desktop" + filepath = Path(filename) if Path(filename).is_absolute() else desktop / filename + return f'notepad "{filepath}"' + if "notepad" in task_lower: + return "notepad" + pip_match = re.search(r"install\s+([\w\-]+)", task_lower) + if pip_match: + package = pip_match.group(1) + return f"pip install {package}" + + for keywords, command, _ in WIN_COMMAND_MAP: + if command and any(kw in task_lower for kw in keywords): + return command + + return None + +BLOCKED_PATTERNS = [ + r"\brm\s+-rf\b", r"\brmdir\s+/s\b", r"\bdel\s+/[fqs]", + r"\bformat\b", r"\bdiskpart\b", r"\bfdisk\b", + r"\breg\s+(delete|add)\b", r"\bbcdedit\b", + r"\bnet\s+localgroup\b", + r"\bshutdown\b", r"\brestart-computer\b", + r"\bstop-process\b", r"\bkill\s+-9\b", r"\btaskkill\b", + r"\beval\b", r"\b__import__\b", +] +_BLOCKED_RE = re.compile("|".join(BLOCKED_PATTERNS), re.IGNORECASE) + + +def _is_safe(command: str) -> tuple[bool, str]: + match = _BLOCKED_RE.search(command) + if match: + return False, f"Blocked pattern: '{match.group()}'" + return True, "OK" + +def _ask_gemini(task: str) -> str: + try: + import google.generativeai as genai + genai.configure(api_key=_get_api_key()) + model = genai.GenerativeModel("gemini-2.5-flash-lite") + + prompt = ( + f"Convert this request to a single Windows CMD command.\n" + f"Output ONLY the command. No explanation, no markdown, no backticks.\n" + f"If unsafe or impossible, output: UNSAFE\n\n" + f"Request: {task}\n\nCommand:" + ) + response = model.generate_content(prompt) + command = response.text.strip().strip("`").strip() + if command.startswith("```"): + lines = command.split("\n") + command = "\n".join(lines[1:-1]).strip() + return command + except Exception as e: + return f"ERROR: {e}" + +def _run_silent(command: str, timeout: int = 20) -> str: + try: + platform = _get_platform() + if platform == "windows": + is_ps = command.strip().lower().startswith("powershell") + if is_ps: + cmd_inner = re.sub(r'^powershell\s+"?', '', command, flags=re.IGNORECASE).rstrip('"') + result = subprocess.run( + ["powershell", "-NoProfile", "-Command", cmd_inner], + capture_output=True, text=True, + encoding="utf-8", errors="replace", timeout=timeout + ) + else: + result = subprocess.run( + ["cmd", "/c", command], + capture_output=True, text=True, + encoding="cp1252", errors="replace", + timeout=timeout, cwd=str(Path.home()) + ) + else: + shell = "/bin/zsh" if platform == "macos" else "/bin/bash" + result = subprocess.run( + command, shell=True, executable=shell, + capture_output=True, text=True, + errors="replace", timeout=timeout, + cwd=str(Path.home()) + ) + + output = result.stdout.strip() + error = result.stderr.strip() + if output: return output[:2000] + if error: return f"[stderr]: {error[:500]}" + return "Command executed with no output." + + except subprocess.TimeoutExpired: + return f"Command timed out after {timeout}s." + except Exception as e: + return f"Execution error: {e}" + + +def _run_visible(command: str) -> None: + try: + platform = _get_platform() + if platform == "windows": + subprocess.Popen( + f'cmd /k "{command}"', + creationflags=subprocess.CREATE_NEW_CONSOLE + ) + elif platform == "macos": + subprocess.Popen(["osascript", "-e", + f'tell application "Terminal" to do script "{command}"']) + else: + for term in ["gnome-terminal", "xterm", "konsole"]: + try: + subprocess.Popen([term, "--", "bash", "-c", f"{command}; exec bash"]) + break + except FileNotFoundError: + continue + except Exception as e: + print(f"[CMD] ⚠️ Terminal open failed: {e}") + + +def cmd_control( + parameters: dict, + response=None, + player=None, + session_memory=None +) -> str: + task = (parameters or {}).get("task", "").strip() + command = (parameters or {}).get("command", "").strip() + visible = (parameters or {}).get("visible", True) + + if not task and not command: + return "Please describe what you want to do, sir." + + if not command: + command = _find_hardcoded(task) + if command: + print(f"[CMD] ⚡ Hardcoded: {command[:80]}") + else: + print(f"[CMD] 🤖 Gemini fallback for: {task}") + command = _ask_gemini(task) + print(f"[CMD] ✅ Generated: {command[:80]}") + if command == "UNSAFE": + return "I cannot generate a safe command for that request, sir." + if command.startswith("ERROR:"): + return f"Could not generate command: {command}" + + safe, reason = _is_safe(command) + if not safe: + return f"Blocked for safety: {reason}" + + if player: + player.write_log(f"[CMD] {command[:60]}") + + if any(x in command.lower() for x in ["notepad", "explorer", "start "]): + subprocess.Popen(command, shell=True) + return f"Opened: {command}" + + if visible: + _run_visible(command) + output = _run_silent(command) + return f"Terminal opened.\n\nOutput:\n{output}" + else: + return _run_silent(command) \ No newline at end of file diff --git a/actions/code_helper.py b/actions/code_helper.py new file mode 100644 index 0000000..7247aa1 --- /dev/null +++ b/actions/code_helper.py @@ -0,0 +1,583 @@ +# actions/code_helper.py +# AI-powered code assistant — writes, edits, explains, runs, builds, debugs, and optimizes code. +# +# Actions: +# write → Describe what you want, Gemini writes it, saves to file +# edit → Read existing file, apply natural language change +# explain → Explain what a piece of code or file does +# run → Execute a script file, return output +# build → Write → Run → Fix loop (max 3 attempts), speaks when done +# screen_debug → Screenshot al, ekrandaki kodu/hatayı Gemini ile analiz et ve düzelt +# optimize → Mevcut kodu Gemini ile optimize et (performans, okunabilirlik, best practices) +# auto → (default) Intent auto-detected from context + +import subprocess +import sys +import json +import re +import time +from pathlib import Path + + +def get_base_dir(): + if getattr(sys, "frozen", False): + return Path(sys.executable).parent + return Path(__file__).resolve().parent.parent + +BASE_DIR = get_base_dir() +API_CONFIG_PATH = BASE_DIR / "config" / "api_keys.json" +DESKTOP = Path.home() / "Desktop" +MAX_BUILD_ATTEMPTS = 3 +GEMINI_MODEL = "gemini-2.5-flash" + + +def _get_api_key() -> str: + with open(API_CONFIG_PATH, "r", encoding="utf-8") as f: + return json.load(f)["gemini_api_key"] + + +def _get_gemini(model: str = GEMINI_MODEL): + import google.generativeai as genai + genai.configure(api_key=_get_api_key()) + return genai.GenerativeModel(model) + + +def _clean_code(text: str) -> str: + text = text.strip() + text = re.sub(r"^```[a-zA-Z]*\n?", "", text) + text = re.sub(r"\n?```$", "", text) + return text.strip() + + +def _resolve_save_path(output_path: str, language: str) -> Path: + ext_map = { + "python": ".py", "py": ".py", + "javascript": ".js", "js": ".js", + "typescript": ".ts", "ts": ".ts", + "html": ".html", "css": ".css", + "java": ".java", "cpp": ".cpp", "c": ".c", + "bash": ".sh", "shell": ".sh", "powershell": ".ps1", + "sql": ".sql", "json": ".json", "rust": ".rs", "go": ".go", + } + if output_path: + p = Path(output_path) + return p if p.is_absolute() else DESKTOP / p + ext = ext_map.get((language or "python").lower(), ".py") + return DESKTOP / f"jarvis_code{ext}" + + +def _read_file(file_path: str) -> tuple[str, str]: + if not file_path: + return "", "No file path provided." + p = Path(file_path) + if not p.exists(): + return "", f"File not found: {file_path}" + try: + return p.read_text(encoding="utf-8"), "" + except Exception as e: + return "", f"Could not read file: {e}" + + +def _save_file(path: Path, content: str) -> str: + try: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + return f"Saved to: {path}" + except Exception as e: + return f"Could not save: {e}" + + +def _preview(code: str, lines: int = 10) -> str: + all_lines = code.splitlines() + preview = "\n".join(all_lines[:lines]) + suffix = f"\n... ({len(all_lines) - lines} more lines)" if len(all_lines) > lines else "" + return preview + suffix + + +def _has_error(output: str) -> bool: + error_signals = ["error", "exception", "traceback", "syntaxerror", + "nameerror", "typeerror", "stderr", "failed", "crash"] + return any(s in output.lower() for s in error_signals) + + +def _take_screenshot() -> Path | None: + try: + import pyautogui + screenshot_path = Path.home() / "Desktop" / f"jarvis_debug_{int(time.time())}.png" + screenshot = pyautogui.screenshot() + screenshot.save(str(screenshot_path)) + print(f"[Code] 📸 Screenshot: {screenshot_path}") + return screenshot_path + except Exception as e: + print(f"[Code] ⚠️ Screenshot failed: {e}") + return None + + +def _image_to_base64(path: Path) -> str: + import base64 + return base64.b64encode(path.read_bytes()).decode("utf-8") + + +def _detect_intent(description: str, file_path: str, code: str) -> str: + desc = (description or "").lower() + + screen_kw = ["ekrandaki", "screen", "ekranda", "bu hatayı", "why am i getting", + "neden hata", "what's wrong", "ne yanlış", "screenshot", "görüntü"] + if any(k in desc for k in screen_kw): + return "screen_debug" + + # Optimize keywords + optimize_kw = ["optimize", "refactor", "clean up", "improve", "temizle", + "iyileştir", "daha iyi", "make it better", "hızlandır"] + if any(k in desc for k in optimize_kw) and (code or file_path): + return "optimize" + + if file_path: + p = Path(file_path) + edit_kw = ["edit", "update", "modify", "change", "add", "remove", + "refactor", "fix", "rename", "replace", "düzenle", "değiştir"] + run_kw = ["run", "execute", "launch", "start", "çalıştır"] + build_kw = ["build", "make it work", "try", "attempt"] + + if p.exists() and any(k in desc for k in edit_kw): + return "edit" + if p.exists() and any(k in desc for k in run_kw): + return "run" + if any(k in desc for k in build_kw): + return "build" + if p.exists(): + return "explain" + + explain_kw = ["explain", "what does", "describe", "analyze", "açıkla", "ne yapıyor"] + if any(k in desc for k in explain_kw) and (code or file_path): + return "explain" + + build_kw = ["build", "make it work", "try and", "attempt"] + if any(k in desc for k in build_kw): + return "build" + + return "write" + +def _write(description: str, language: str, output_path: str, player=None) -> tuple[str, Path]: + lang = language or "python" + model = _get_gemini() + + prompt = f"""You are an expert {lang} developer. +Write clean, working, well-commented {lang} code for the description below. + +Rules: +- Output ONLY the code. No explanation, no markdown, no backticks. +- Add helpful inline comments. +- Handle errors and edge cases properly. +- Use modern best practices. + +Description: {description} + +Code:""" + + response = model.generate_content(prompt) + code = _clean_code(response.text) + path = _resolve_save_path(output_path, lang) + _save_file(path, code) + return code, path + + +def _fix_code(code: str, error_output: str, description: str) -> str: + model = _get_gemini() + prompt = f"""You are an expert debugger. +The code below failed with the following error. Fix it. +Return ONLY the corrected code — no explanation, no markdown, no backticks. + +Original goal: {description} + +Error: +{error_output[:2000]} + +Broken code: +{code} + +Fixed code:""" + + response = model.generate_content(prompt) + return _clean_code(response.text) + + +def _run_file(path: Path, args: list, timeout: int) -> str: + interpreters = { + ".py": [sys.executable], + ".js": ["node"], + ".ts": ["ts-node"], + ".sh": ["bash"], + ".ps1": ["powershell", "-File"], + ".rb": ["ruby"], + ".php": ["php"], + } + interp = interpreters.get(path.suffix.lower()) + if not interp: + return f"No interpreter for {path.suffix}." + + try: + result = subprocess.run( + interp + [str(path)] + (args or []), + capture_output=True, text=True, + encoding="utf-8", errors="replace", + timeout=timeout, cwd=str(path.parent) + ) + output = result.stdout.strip() + error = result.stderr.strip() + parts = [] + if output: parts.append(f"Output:\n{output}") + if error: parts.append(f"Stderr:\n{error}") + return "\n\n".join(parts) if parts else "Executed with no output." + + except subprocess.TimeoutExpired: + return f"Timed out after {timeout}s." + except FileNotFoundError: + return f"Interpreter not found: {interp[0]}." + except Exception as e: + return f"Execution error: {e}" + + +def _build(description, language, output_path, args, timeout, speak=None, player=None) -> str: + if not description: + return "Please describe what you want me to build, sir." + + if player: + player.write_log("[Code] Build started...") + + lang = language or "python" + + try: + code, path = _write(description, lang, output_path, player) + print(f"[Code] ✅ Written: {path}") + except Exception as e: + msg = f"Could not write initial code: {e}" + if speak: speak(msg) + return msg + + last_output = "" + for attempt in range(1, MAX_BUILD_ATTEMPTS + 1): + print(f"[Code] 🔄 Attempt {attempt}/{MAX_BUILD_ATTEMPTS}") + if player: + player.write_log(f"[Code] Attempt {attempt}...") + + last_output = _run_file(path, args, timeout) + + if not _has_error(last_output): + msg = ( + f"Build complete, sir. " + f"The code is working after {attempt} attempt{'s' if attempt > 1 else ''}. " + f"Saved to {path}." + ) + if speak: speak(msg) + return f"{msg}\n\nOutput:\n{last_output}" + + print(f"[Code] ⚠️ Error on attempt {attempt}, fixing...") + if player: + player.write_log(f"[Code] Fixing (attempt {attempt})...") + + try: + code = _fix_code(code, last_output, description) + _save_file(path, code) + except Exception as e: + msg = f"Could not fix code on attempt {attempt}: {e}" + if speak: speak(msg) + return msg + + msg = ( + f"I was unable to build a working version after {MAX_BUILD_ATTEMPTS} attempts, sir. " + f"The last error was: {last_output[:200]}" + ) + if speak: speak(msg) + return f"{msg}\n\nLast code saved to: {path}" + +def _write_action(description, language, output_path, player) -> str: + if not description: + return "Please describe what you want me to write, sir." + if player: + player.write_log("[Code] Writing code...") + try: + code, path = _write(description, language, output_path, player) + print(f"[Code] ✅ Written: {path}") + return f"Code written. Saved to: {path}\n\nPreview:\n{_preview(code)}" + except Exception as e: + return f"Could not generate code: {e}" + + +def _edit_action(file_path, instruction, player) -> str: + if not file_path: + return "Please provide a file path to edit, sir." + if not instruction: + return "Please describe what change to make, sir." + + content, err = _read_file(file_path) + if err: + return err + + if player: + player.write_log("[Code] Editing file...") + + model = _get_gemini() + prompt = f"""You are an expert code editor. +Apply the following change to the code below. +Return ONLY the complete updated code — no explanation, no markdown, no backticks. + +Change: {instruction} + +Original code: +{content} + +Updated code:""" + + try: + response = model.generate_content(prompt) + edited = _clean_code(response.text) + except Exception as e: + return f"Could not edit code: {e}" + + status = _save_file(Path(file_path), edited) + print(f"[Code] ✅ Edited: {file_path}") + return f"File edited. {status}\n\nPreview:\n{_preview(edited)}" + + +def _explain_action(file_path, code, player) -> str: + if file_path and not code: + code, err = _read_file(file_path) + if err: + return err + if not code: + return "Please provide code or a file path to explain, sir." + + if player: + player.write_log("[Code] Analyzing code...") + + model = _get_gemini() + prompt = f"""Explain what this code does in simple, clear language. +Focus on: what it does, how it works, and any important details. +Be concise — 3 to 6 sentences maximum. + +Code: +{code[:4000]} + +Explanation:""" + + try: + response = model.generate_content(prompt) + return response.text.strip() + except Exception as e: + return f"Could not explain code: {e}" + + +def _run_action(file_path, args, timeout, player) -> str: + if not file_path: + return "Please provide a file path to run, sir." + p = Path(file_path) + if not p.exists(): + return f"File not found: {file_path}" + if player: + player.write_log(f"[Code] Running {p.name}...") + return _run_file(p, args, timeout) + + +def _optimize_action(file_path, code, language, output_path, player) -> str: + + if file_path and not code: + code, err = _read_file(file_path) + if err: + return err + if not code: + return "Please provide code or a file path to optimize, sir." + + if player: + player.write_log("[Code] Optimizing code...") + + lang = language or "python" + model = _get_gemini() + + prompt = f"""You are an expert {lang} developer and code reviewer. +Optimize the following code for: +1. Performance — eliminate unnecessary operations, use efficient data structures +2. Readability — clear variable names, proper formatting, logical structure +3. Best practices — modern {lang} patterns, error handling, type hints if applicable +4. Remove dead code, redundant comments, and unnecessary complexity + +Return ONLY the optimized code — no explanation, no markdown, no backticks. + +Original code: +{code[:6000]} + +Optimized code:""" + + try: + response = model.generate_content(prompt) + optimized = _clean_code(response.text) + except Exception as e: + return f"Could not optimize code: {e}" + + # Kaydet + if file_path: + save_path = Path(file_path) + else: + save_path = _resolve_save_path(output_path, lang) + + status = _save_file(save_path, optimized) + print(f"[Code] ✅ Optimized: {save_path}") + + original_lines = len(code.splitlines()) + optimized_lines = len(optimized.splitlines()) + diff = original_lines - optimized_lines + + return ( + f"Code optimized. {status}\n" + f"Lines: {original_lines} → {optimized_lines} " + f"({'−' if diff > 0 else '+'}{abs(diff)} lines)\n\n" + f"Preview:\n{_preview(optimized)}" + ) + + +def _screen_debug_action(description, file_path, player, speak=None) -> str: + + if player: + player.write_log("[Code] Taking screenshot for analysis...") + + print("[Code] 📸 Capturing screen for debug...") + + + screenshot_path = _take_screenshot() + if not screenshot_path: + return "Could not take screenshot, sir. Please make sure PyAutoGUI is installed." + + + file_content = "" + if file_path: + file_content, err = _read_file(file_path) + if err: + print(f"[Code] ⚠️ Could not read file: {err}") + + try: + from google import genai + from google.genai import types + + client = genai.Client(api_key=_get_api_key()) + + image_bytes = screenshot_path.read_bytes() + image_base64 = _image_to_base64(screenshot_path) + + user_question = description or "What error or problem do you see on the screen? How can it be fixed?" + + context = "" + if file_content: + context = f"\n\nAdditionally, here is the related file content:\n```\n{file_content[:4000]}\n```" + + analysis_prompt = f"""You are an expert programmer and debugger analyzing a screenshot. + +User's question: {user_question}{context} + +Please: +1. Identify any errors, exceptions, or problems visible on the screen +2. Explain what is causing the problem in simple terms +3. Provide a concrete fix or solution +4. If there's code visible, show the corrected version + +Be specific and actionable. If you see an error message, quote it exactly.""" + + contents = [ + types.Part.from_bytes(data=image_bytes, mime_type="image/png"), + analysis_prompt, + ] + + response = client.models.generate_content( + model="gemini-2.5-flash", + contents=contents, + ) + + analysis = response.text.strip() + print(f"[Code] ✅ Screen analysis complete") + + try: + screenshot_path.unlink() + except Exception: + pass + + if file_path and file_content: + + code_match = re.search(r"```[a-zA-Z]*\n(.*?)```", analysis, re.DOTALL) + if code_match: + fixed_code = code_match.group(1).strip() + save_path = Path(file_path) + _save_file(save_path, fixed_code) + analysis += f"\n\n✅ Fixed code has been saved to: {file_path}" + print(f"[Code] ✅ Fixed code saved: {file_path}") + + return analysis + + except Exception as e: + + try: + screenshot_path.unlink() + except Exception: + pass + return f"Screen analysis failed: {e}" + + +def code_helper( + parameters: dict, + response=None, + player=None, + session_memory=None, + speak=None +) -> str: + """ + Called from main.py. + + parameters: + action : write | edit | explain | run | build | screen_debug | optimize | auto + description : What the code should do / what change to make / what problem to analyze + language : Programming language (default: python) + output_path : Where to save — user specifies full path or filename + file_path : Path to existing file (edit / explain / run / build / optimize) + code : Raw code string (explain/optimize without a file) + args : CLI argument list for run/build + timeout : Execution timeout in seconds (default: 30) + """ + p = parameters or {} + action = p.get("action", "auto").lower().strip() + description = p.get("description", "").strip() + language = p.get("language", "python").strip() + output_path = p.get("output_path", "").strip() + file_path = p.get("file_path", "").strip() + code = p.get("code", "").strip() + args = p.get("args", []) + timeout = int(p.get("timeout", 30)) + + if action == "auto": + action = _detect_intent(description, file_path, code) + print(f"[Code] 🤖 Auto-detected: {action}") + + if action == "write": + return _write_action(description, language, output_path, player) + + elif action == "edit": + return _edit_action( + file_path, + description or p.get("instruction", ""), + player + ) + + elif action == "explain": + return _explain_action(file_path, code, player) + + elif action == "run": + return _run_action(file_path, args, timeout, player) + + elif action == "build": + return _build(description, language, output_path, args, timeout, speak, player) + + elif action == "optimize": + return _optimize_action(file_path, code, language, output_path, player) + + elif action == "screen_debug": + return _screen_debug_action(description, file_path, player, speak) + + else: + return f"Unknown action: '{action}'. Use write, edit, explain, run, build, optimize, or screen_debug." \ No newline at end of file diff --git a/actions/computer_control.py b/actions/computer_control.py new file mode 100644 index 0000000..90db975 --- /dev/null +++ b/actions/computer_control.py @@ -0,0 +1,554 @@ +# actions/computer_control.py +# MARK XXV — Computer Control +# +# Atomic computer control functions using PyAutoGUI + keyboard + clipboard. +# Used by the agent when no existing action file covers the task. +# +# Capabilities: +# - Type text anywhere (active window, forms, fields) +# - Mouse click, double-click, right-click, drag +# - Keyboard shortcuts and key combinations +# - Scroll (up/down/left/right) +# - Window management (minimize, maximize, close, focus) +# - Clipboard (copy, paste, get content) +# - Screenshot + locate element on screen +# - Wait / smart wait for element to appear +# - Random data generation (name, email, username, password, phone, address) +# - Hotkey sequences +# - Find and click image/element on screen + +import json +import sys +import time +import random +import string +import subprocess +import platform +from pathlib import Path + +try: + import pyautogui + pyautogui.FAILSAFE = True + pyautogui.PAUSE = 0.05 + _PYAUTOGUI = True +except ImportError: + _PYAUTOGUI = False + +try: + import pyperclip + _PYPERCLIP = True +except ImportError: + _PYPERCLIP = False + + +def get_base_dir() -> Path: + if getattr(sys, "frozen", False): + return Path(sys.executable).parent + return Path(__file__).resolve().parent.parent + + +BASE_DIR = get_base_dir() +API_CONFIG_PATH = BASE_DIR / "config" / "api_keys.json" + + +def _load_user_profile() -> dict: + """Load user profile from long_term.json for form filling.""" + memory_path = BASE_DIR / "memory" / "long_term.json" + try: + if memory_path.exists(): + data = json.loads(memory_path.read_text(encoding="utf-8")) + identity = data.get("identity", {}) + return { + "name": identity.get("name", {}).get("value", ""), + "age": identity.get("age", {}).get("value", ""), + "city": identity.get("city", {}).get("value", ""), + "email": identity.get("email", {}).get("value", ""), + } + except Exception: + pass + return {} + + +def _ensure_pyautogui(): + if not _PYAUTOGUI: + raise RuntimeError( + "PyAutoGUI not installed. Run: pip install pyautogui" + ) + + +_FIRST_NAMES = [ + "Alex", "Jordan", "Taylor", "Morgan", "Casey", "Riley", "Drew", "Quinn", + "Avery", "Blake", "Cameron", "Dakota", "Emerson", "Finley", "Harper" +] +_LAST_NAMES = [ + "Smith", "Johnson", "Williams", "Brown", "Jones", "Garcia", "Miller", + "Davis", "Wilson", "Moore", "Taylor", "Anderson", "Thomas", "Jackson" +] +_DOMAINS = ["gmail.com", "yahoo.com", "outlook.com", "proton.me", "mail.com"] + + +def generate_random_data(data_type: str) -> str: + """ + Generates random realistic data for form filling. + + Types: name | first_name | last_name | email | username | + password | phone | birthday | address | zip_code + """ + dt = data_type.lower().strip() + + if dt == "first_name": + return random.choice(_FIRST_NAMES) + + elif dt == "last_name": + return random.choice(_LAST_NAMES) + + elif dt == "name": + return f"{random.choice(_FIRST_NAMES)} {random.choice(_LAST_NAMES)}" + + elif dt == "email": + first = random.choice(_FIRST_NAMES).lower() + last = random.choice(_LAST_NAMES).lower() + num = random.randint(10, 999) + return f"{first}.{last}{num}@{random.choice(_DOMAINS)}" + + elif dt == "username": + first = random.choice(_FIRST_NAMES).lower() + num = random.randint(100, 9999) + return f"{first}{num}" + + elif dt == "password": + chars = string.ascii_letters + string.digits + "!@#$%" + pwd = ( + random.choice(string.ascii_uppercase) + + random.choice(string.digits) + + random.choice("!@#$%") + + "".join(random.choices(chars, k=9)) + ) + return "".join(random.sample(pwd, len(pwd))) + + elif dt == "phone": + return f"+1{random.randint(200,999)}{random.randint(1000000,9999999)}" + + elif dt == "birthday": + year = random.randint(1980, 2000) + month = random.randint(1, 12) + day = random.randint(1, 28) + return f"{month:02d}/{day:02d}/{year}" + + elif dt == "address": + num = random.randint(100, 9999) + street = random.choice(["Main St", "Oak Ave", "Park Blvd", "Elm St", "Cedar Ln"]) + return f"{num} {street}" + + elif dt == "zip_code": + return str(random.randint(10000, 99999)) + + elif dt == "city": + return random.choice(["New York", "Los Angeles", "Chicago", "Houston", "Phoenix"]) + + return f"random_{data_type}_{random.randint(1000,9999)}" + + +def _type_text(text: str, interval: float = 0.03) -> str: + """Types text at the current cursor position.""" + _ensure_pyautogui() + time.sleep(0.3) + pyautogui.typewrite(text, interval=interval) + return f"Typed: {text[:50]}{'...' if len(text) > 50 else ''}" + + +def _click(x: int = None, y: int = None, button: str = "left", + clicks: int = 1, image: str = None) -> str: + """ + Clicks at coordinates or on a screen image. + If image path given, locates it on screen and clicks. + """ + _ensure_pyautogui() + + if image: + try: + loc = pyautogui.locateCenterOnScreen(image, confidence=0.8) + if loc: + pyautogui.click(loc.x, loc.y, button=button, clicks=clicks) + return f"Clicked image: {image}" + return f"Image not found on screen: {image}" + except Exception as e: + return f"Image click failed: {e}" + + if x is not None and y is not None: + pyautogui.click(x, y, button=button, clicks=clicks) + return f"Clicked ({x}, {y}) with {button} button" + + pyautogui.click(button=button, clicks=clicks) + return f"Clicked at current position" + + +def _hotkey(*keys) -> str: + """Presses a key combination. E.g. hotkey('ctrl', 'c')""" + _ensure_pyautogui() + pyautogui.hotkey(*keys) + return f"Hotkey: {'+'.join(keys)}" + + +def _press(key: str) -> str: + """Presses a single key.""" + _ensure_pyautogui() + pyautogui.press(key) + return f"Pressed: {key}" + + +def _scroll(direction: str = "down", amount: int = 3) -> str: + """Scrolls in the specified direction.""" + _ensure_pyautogui() + clicks = amount if direction in ("up", "right") else -amount + if direction in ("up", "down"): + pyautogui.scroll(clicks) + else: + pyautogui.hscroll(clicks) + return f"Scrolled {direction} {amount} times" + + +def _move_mouse(x: int, y: int, duration: float = 0.3) -> str: + """Moves mouse to coordinates.""" + _ensure_pyautogui() + pyautogui.moveTo(x, y, duration=duration) + return f"Mouse moved to ({x}, {y})" + + +def _drag(x1: int, y1: int, x2: int, y2: int, duration: float = 0.5) -> str: + """Drags from (x1,y1) to (x2,y2).""" + _ensure_pyautogui() + pyautogui.drag(x1 - pyautogui.position()[0], y1 - pyautogui.position()[1]) + pyautogui.dragTo(x2, y2, duration=duration) + return f"Dragged from ({x1},{y1}) to ({x2},{y2})" + + +def _clipboard_copy() -> str: + """Gets current clipboard content.""" + if _PYPERCLIP: + return pyperclip.paste() + _hotkey("ctrl", "c") + time.sleep(0.2) + return "Copied to clipboard" + + +def _clipboard_set(text: str) -> str: + """Sets clipboard content and pastes it.""" + if _PYPERCLIP: + pyperclip.copy(text) + time.sleep(0.1) + _hotkey("ctrl", "v") + return f"Pasted: {text[:50]}" + return "pyperclip not available" + + +def _screenshot(save_path: str = None) -> str: + """Takes a screenshot.""" + _ensure_pyautogui() + if not save_path: + save_path = str(Path.home() / "Desktop" / "screenshot.png") + img = pyautogui.screenshot() + img.save(save_path) + return f"Screenshot saved: {save_path}" + + +def _wait(seconds: float) -> str: + """Waits for specified seconds.""" + time.sleep(seconds) + return f"Waited {seconds}s" + + +def _wait_for_image(image_path: str, timeout: int = 10) -> str: + """Waits until an image appears on screen (up to timeout seconds).""" + _ensure_pyautogui() + start = time.time() + while time.time() - start < timeout: + try: + loc = pyautogui.locateCenterOnScreen(image_path, confidence=0.8) + if loc: + return f"Image found at ({loc.x}, {loc.y})" + except Exception: + pass + time.sleep(0.5) + return f"Image not found within {timeout}s: {image_path}" + + +def _get_screen_size() -> str: + """Returns current screen resolution.""" + _ensure_pyautogui() + w, h = pyautogui.size() + return f"{w}x{h}" + + +def _focus_window(title: str) -> str: + """Brings a window to focus by title (Windows).""" + if platform.system() == "Windows": + try: + script = f'(New-Object -ComObject WScript.Shell).AppActivate("{title}")' + subprocess.run( + ["powershell", "-NoProfile", "-Command", script], + capture_output=True, timeout=5 + ) + time.sleep(0.3) + return f"Focused window: {title}" + except Exception as e: + return f"Could not focus window: {e}" + return "Window focus only supported on Windows" + + +def _select_all() -> str: + return _hotkey("ctrl", "a") + + +def _clear_field() -> str: + """Selects all and deletes — clears an input field.""" + _hotkey("ctrl", "a") + time.sleep(0.1) + _press("delete") + return "Field cleared" + + +def _smart_type(text: str, clear_first: bool = True) -> str: + """ + Types text into the currently focused field. + Optionally clears the field first. + Uses clipboard for long text (faster, more reliable). + """ + _ensure_pyautogui() + + if clear_first: + _clear_field() + time.sleep(0.1) + + if len(text) > 20 and _PYPERCLIP: + pyperclip.copy(text) + time.sleep(0.1) + pyautogui.hotkey("ctrl", "v") + return f"Smart-typed (clipboard): {text[:50]}" + else: + pyautogui.typewrite(text, interval=0.04) + return f"Smart-typed: {text[:50]}" + + +def _analyze_screen_for_element(description: str) -> tuple[int, int] | None: + """ + Takes a screenshot and asks Gemini to find the coordinates + of a described element on screen. Returns (x, y) or None. + """ + try: + import google.generativeai as genai + import io + + cfg_path = API_CONFIG_PATH + with open(cfg_path, "r") as f: + api_key = json.load(f)["gemini_api_key"] + + genai.configure(api_key=api_key) + model = genai.GenerativeModel("gemini-2.5-flash-lite") + + + _ensure_pyautogui() + w, h = pyautogui.size() + img = pyautogui.screenshot() + buf = io.BytesIO() + img.save(buf, format="PNG") + buf.seek(0) + + prompt = ( + f"This is a screenshot of a computer screen ({w}x{h} pixels). " + f"Find the element: '{description}'. " + f"Return ONLY: x,y (the center coordinates of the element). " + f"If not found, return: NOT_FOUND" + ) + + response = model.generate_content([ + {"mime_type": "image/png", "data": buf.getvalue()}, + prompt + ]) + + text = response.text.strip() + if "NOT_FOUND" in text: + return None + + import re + match = re.search(r"(\d+)\s*,\s*(\d+)", text) + if match: + return int(match.group(1)), int(match.group(2)) + + except Exception as e: + print(f"[ComputerControl] ⚠️ Screen analysis failed: {e}") + + return None + +def computer_control( + parameters: dict, + response=None, + player=None, + session_memory=None, +) -> str: + """ + Universal computer control action. + + Actions: + type : Type text at current cursor position + smart_type : Clear field + type (clipboard-based for long text) + click : Click at coordinates or on image + double_click : Double click + right_click : Right click + hotkey : Key combination (e.g. ctrl+c) + press : Single key press + scroll : Scroll up/down/left/right + move : Move mouse to coordinates + drag : Drag from one point to another + copy : Get clipboard content + paste : Set and paste clipboard content + screenshot : Take a screenshot + wait : Wait N seconds + wait_image : Wait for image to appear on screen + clear_field : Select all + delete in current field + focus_window : Bring window to foreground + screen_find : AI-powered element finder — returns coordinates + screen_click : AI-powered element finder + click + random_data : Generate random data for forms + user_data : Get user's real data from memory + """ + action = (parameters or {}).get("action", "").lower().strip() + + if not action: + return "Please specify an action for computer_control, sir." + + if player: + player.write_log(f"[Computer] {action}") + + print(f"[ComputerControl] ▶️ Action: {action} Params: {parameters}") + + try: + if action == "type": + text = parameters.get("text", "") + return _type_text(text) + + elif action == "smart_type": + text = parameters.get("text", "") + clear_first = parameters.get("clear_first", True) + return _smart_type(text, clear_first=clear_first) + + elif action in ("click", "left_click"): + return _click( + x=parameters.get("x"), + y=parameters.get("y"), + button="left", + clicks=1, + image=parameters.get("image") + ) + + elif action == "double_click": + return _click( + x=parameters.get("x"), + y=parameters.get("y"), + button="left", + clicks=2, + image=parameters.get("image") + ) + + elif action == "right_click": + return _click( + x=parameters.get("x"), + y=parameters.get("y"), + button="right", + clicks=1 + ) + + elif action == "move": + return _move_mouse( + x=int(parameters.get("x", 0)), + y=int(parameters.get("y", 0)), + duration=float(parameters.get("duration", 0.3)) + ) + + elif action == "drag": + return _drag( + x1=int(parameters.get("x1", 0)), + y1=int(parameters.get("y1", 0)), + x2=int(parameters.get("x2", 0)), + y2=int(parameters.get("y2", 0)) + ) + + elif action == "hotkey": + keys = parameters.get("keys", "") + if isinstance(keys, str): + keys = [k.strip() for k in keys.split("+")] + return _hotkey(*keys) + + elif action == "press": + return _press(parameters.get("key", "enter")) + + elif action == "scroll": + return _scroll( + direction=parameters.get("direction", "down"), + amount=int(parameters.get("amount", 3)) + ) + + elif action == "copy": + return _clipboard_copy() + + elif action == "paste": + return _clipboard_set(parameters.get("text", "")) + + elif action == "screenshot": + return _screenshot(parameters.get("path")) + + elif action == "wait": + return _wait(float(parameters.get("seconds", 1.0))) + + elif action == "wait_image": + return _wait_for_image( + parameters.get("image", ""), + timeout=int(parameters.get("timeout", 10)) + ) + + elif action == "clear_field": + return _clear_field() + + elif action == "focus_window": + return _focus_window(parameters.get("title", "")) + + elif action == "screen_size": + return _get_screen_size() + + elif action == "screen_find": + description = parameters.get("description", "") + coords = _analyze_screen_for_element(description) + if coords: + return f"{coords[0]},{coords[1]}" + return "NOT_FOUND" + + elif action == "screen_click": + description = parameters.get("description", "") + coords = _analyze_screen_for_element(description) + if coords: + time.sleep(0.2) + _click(x=coords[0], y=coords[1]) + return f"Found and clicked: {description} at {coords}" + return f"Could not find on screen: {description}" + + elif action == "random_data": + data_type = parameters.get("type", "name") + result = generate_random_data(data_type) + print(f"[ComputerControl] 🎲 Random {data_type}: {result}") + return result + + elif action == "user_data": + field = parameters.get("field", "name") + profile = _load_user_profile() + value = profile.get(field, "") + if not value: + value = generate_random_data(field) + print(f"[ComputerControl] ⚠️ No user {field} in memory, using random: {value}") + return value + + else: + return f"Unknown computer_control action: '{action}'" + + except Exception as e: + print(f"[ComputerControl] ❌ Error: {e}") + return f"computer_control failed: {e}" \ No newline at end of file diff --git a/actions/computer_settings.py b/actions/computer_settings.py new file mode 100644 index 0000000..3325eb5 --- /dev/null +++ b/actions/computer_settings.py @@ -0,0 +1,679 @@ +# actions/computer_settings.py +# MARK XXV — Computer Settings & UI Controls +# +# Kullanıcı "sesi aç", "uygulamayı kapat", "tam ekran yap", "şunu yaz" gibi +# bilgisayar kontrol komutları verdiğinde bu dosya devreye girer. +# +# - Intent detection: Gemini ile (multi-language, hardcoded keyword yok) +# - Cross-platform: Windows / macOS / Linux +# - pyautogui + platform-specific API'ler + +import time +import subprocess +import sys +import platform +from pathlib import Path + +try: + import pyautogui + pyautogui.FAILSAFE = True + pyautogui.PAUSE = 0.05 + _PYAUTOGUI = True +except ImportError: + _PYAUTOGUI = False + +try: + import pyperclip + _PYPERCLIP = True +except ImportError: + _PYPERCLIP = False + +_OS = platform.system() + +def get_base_dir() -> Path: + if getattr(sys, "frozen", False): + return Path(sys.executable).parent + return Path(__file__).resolve().parent.parent + +BASE_DIR = get_base_dir() +API_CONFIG_PATH = BASE_DIR / "config" / "api_keys.json" + +import json +def _get_api_key() -> str: + with open(API_CONFIG_PATH, "r", encoding="utf-8") as f: + return json.load(f)["gemini_api_key"] + + +def volume_up(): + if _OS == "Windows": + for _ in range(5): pyautogui.press("volumeup") + elif _OS == "Darwin": + subprocess.run(["osascript", "-e", "set volume output volume (output volume of (get volume settings) + 10)"]) + else: + subprocess.run(["pactl", "set-sink-volume", "@DEFAULT_SINK@", "+10%"]) + +def volume_down(): + if _OS == "Windows": + for _ in range(5): pyautogui.press("volumedown") + elif _OS == "Darwin": + subprocess.run(["osascript", "-e", "set volume output volume (output volume of (get volume settings) - 10)"]) + else: + subprocess.run(["pactl", "set-sink-volume", "@DEFAULT_SINK@", "-10%"]) + +def volume_mute(): + if _OS == "Windows": + pyautogui.press("volumemute") + elif _OS == "Darwin": + subprocess.run(["osascript", "-e", "set volume with output muted"]) + else: + subprocess.run(["pactl", "set-sink-mute", "@DEFAULT_SINK@", "toggle"]) + +def volume_set(value: int): + value = max(0, min(100, value)) + if _OS == "Windows": + try: + from ctypes import cast, POINTER + from comtypes import CLSCTX_ALL + from pycaw.pycaw import AudioUtilities, IAudioEndpointVolume + import math + devices = AudioUtilities.GetSpeakers() + interface = devices.Activate(IAudioEndpointVolume._iid_, CLSCTX_ALL, None) + vol = cast(interface, POINTER(IAudioEndpointVolume)) + vol_db = -65.25 if value == 0 else max(-65.25, 20 * math.log10(value / 100)) + vol.SetMasterVolumeLevel(vol_db, None) + print(f"[Settings] 🔊 Volume → {value}%") + return + except Exception as e: + print(f"[Settings] ⚠️ pycaw failed: {e}") + elif _OS == "Darwin": + subprocess.run(["osascript", "-e", f"set volume output volume {value}"]) + return + else: + subprocess.run(["pactl", "set-sink-volume", "@DEFAULT_SINK@", f"{value}%"]) + return + +def brightness_up(): + if _OS == "Windows": + pyautogui.hotkey("win", "a") + time.sleep(0.3) + elif _OS == "Darwin": + subprocess.run(["osascript", "-e", "tell application \"System Events\" to key code 144"]) + else: + subprocess.run(["brightnessctl", "set", "+10%"]) + +def brightness_down(): + if _OS == "Windows": + pyautogui.hotkey("win", "a") + time.sleep(0.3) + elif _OS == "Darwin": + subprocess.run(["osascript", "-e", "tell application \"System Events\" to key code 145"]) + else: + subprocess.run(["brightnessctl", "set", "10%-"]) + + +def close_app(): + if _OS == "Darwin": + pyautogui.hotkey("command", "q") + else: + pyautogui.hotkey("alt", "f4") + +def close_window(): + if _OS == "Darwin": + pyautogui.hotkey("command", "w") + else: + pyautogui.hotkey("ctrl", "w") + +def full_screen(): + if _OS == "Darwin": + pyautogui.hotkey("ctrl", "command", "f") + else: + pyautogui.press("f11") + +def minimize_window(): + if _OS == "Darwin": + pyautogui.hotkey("command", "m") + else: + pyautogui.hotkey("win", "down") + +def maximize_window(): + if _OS == "Darwin": + subprocess.run(["osascript", "-e", + 'tell application "System Events" to keystroke "f" using {control down, command down}']) + else: + pyautogui.hotkey("win", "up") + +def snap_left(): + if _OS == "Windows": pyautogui.hotkey("win", "left") + +def snap_right(): + if _OS == "Windows": pyautogui.hotkey("win", "right") + +def switch_window(): + if _OS == "Darwin": + pyautogui.hotkey("command", "tab") + else: + pyautogui.hotkey("alt", "tab") + +def show_desktop(): + if _OS == "Darwin": + pyautogui.hotkey("fn", "f11") + elif _OS == "Windows": + pyautogui.hotkey("win", "d") + else: + pyautogui.hotkey("super", "d") + +def open_task_manager(): + if _OS == "Windows": + pyautogui.hotkey("ctrl", "shift", "esc") + elif _OS == "Darwin": + subprocess.Popen(["open", "-a", "Activity Monitor"]) + else: + subprocess.Popen(["gnome-system-monitor"]) + +def open_task_view(): + if _OS == "Windows": + pyautogui.hotkey("win", "tab") + + +def focus_search(): + if _OS == "Darwin": pyautogui.hotkey("command", "l") + else: pyautogui.hotkey("ctrl", "l") + +def pause_video(): pyautogui.press("space") +def refresh_page(): + if _OS == "Darwin": pyautogui.hotkey("command", "r") + else: pyautogui.press("f5") + +def close_tab(): + if _OS == "Darwin": pyautogui.hotkey("command", "w") + else: pyautogui.hotkey("ctrl", "w") + +def new_tab(): + if _OS == "Darwin": pyautogui.hotkey("command", "t") + else: pyautogui.hotkey("ctrl", "t") + +def next_tab(): + if _OS == "Darwin": pyautogui.hotkey("command", "shift", "bracketright") + else: pyautogui.hotkey("ctrl", "tab") + +def prev_tab(): + if _OS == "Darwin": pyautogui.hotkey("command", "shift", "bracketleft") + else: pyautogui.hotkey("ctrl", "shift", "tab") + +def go_back(): + if _OS == "Darwin": pyautogui.hotkey("command", "left") + else: pyautogui.hotkey("alt", "left") + +def go_forward(): + if _OS == "Darwin": pyautogui.hotkey("command", "right") + else: pyautogui.hotkey("alt", "right") + +def zoom_in(): + if _OS == "Darwin": pyautogui.hotkey("command", "equal") + else: pyautogui.hotkey("ctrl", "equal") + +def zoom_out(): + if _OS == "Darwin": pyautogui.hotkey("command", "minus") + else: pyautogui.hotkey("ctrl", "minus") + +def zoom_reset(): + if _OS == "Darwin": pyautogui.hotkey("command", "0") + else: pyautogui.hotkey("ctrl", "0") + +def find_on_page(): + if _OS == "Darwin": pyautogui.hotkey("command", "f") + else: pyautogui.hotkey("ctrl", "f") + +def reload_page_n(n: int): + for _ in range(n): + refresh_page() + time.sleep(0.8) + + +def scroll_up(amount: int = 500): pyautogui.scroll(amount) +def scroll_down(amount: int = 500): pyautogui.scroll(-amount) +def scroll_top(): pyautogui.hotkey("ctrl", "home") if _OS != "Darwin" else pyautogui.hotkey("command", "up") +def scroll_bottom(): pyautogui.hotkey("ctrl", "end") if _OS != "Darwin" else pyautogui.hotkey("command", "down") +def page_up(): pyautogui.press("pageup") +def page_down(): pyautogui.press("pagedown") + + +def copy(): + if _OS == "Darwin": pyautogui.hotkey("command", "c") + else: pyautogui.hotkey("ctrl", "c") + +def paste(): + if _OS == "Darwin": pyautogui.hotkey("command", "v") + else: pyautogui.hotkey("ctrl", "v") + +def cut(): + if _OS == "Darwin": pyautogui.hotkey("command", "x") + else: pyautogui.hotkey("ctrl", "x") + +def undo(): + if _OS == "Darwin": pyautogui.hotkey("command", "z") + else: pyautogui.hotkey("ctrl", "z") + +def redo(): + if _OS == "Darwin": pyautogui.hotkey("command", "shift", "z") + else: pyautogui.hotkey("ctrl", "y") + +def select_all(): + if _OS == "Darwin": pyautogui.hotkey("command", "a") + else: pyautogui.hotkey("ctrl", "a") + +def save_file(): + if _OS == "Darwin": pyautogui.hotkey("command", "s") + else: pyautogui.hotkey("ctrl", "s") + +def press_enter(): pyautogui.press("enter") +def press_escape(): pyautogui.press("escape") +def press_key(key: str): pyautogui.press(key) + +def type_text(text: str, press_enter_after: bool = False): + if not text: + return + if _PYPERCLIP: + pyperclip.copy(text) + time.sleep(0.1) + paste() + else: + pyautogui.write(str(text), interval=0.03) + if press_enter_after: + time.sleep(0.1) + pyautogui.press("enter") + +def write_on_screen(text: str): + type_text(text) + +def take_screenshot(): + if _OS == "Windows": + pyautogui.hotkey("win", "shift", "s") + elif _OS == "Darwin": + pyautogui.hotkey("command", "shift", "3") + else: + pyautogui.hotkey("ctrl", "print_screen") + +def lock_screen(): + if _OS == "Windows": + pyautogui.hotkey("win", "l") + elif _OS == "Darwin": + subprocess.run(["pmset", "displaysleepnow"]) + else: + subprocess.run(["gnome-screensaver-command", "-l"]) + +def open_system_settings(): + if _OS == "Windows": + pyautogui.hotkey("win", "i") + elif _OS == "Darwin": + subprocess.Popen(["open", "-a", "System Preferences"]) + else: + subprocess.Popen(["gnome-control-center"]) + +def open_file_explorer(): + if _OS == "Windows": + pyautogui.hotkey("win", "e") + elif _OS == "Darwin": + subprocess.Popen(["open", Path.home()]) + else: + subprocess.Popen(["xdg-open", Path.home()]) + +def open_run(): + if _OS == "Windows": + pyautogui.hotkey("win", "r") + +def sleep_display(): + if _OS == "Windows": + try: + import ctypes + ctypes.windll.user32.SendMessageW(0xFFFF, 0x0112, 0xF170, 2) + except Exception: + pass + elif _OS == "Darwin": + subprocess.run(["pmset", "displaysleepnow"]) + else: + subprocess.run(["xset", "dpms", "force", "off"]) + +def restart_computer(): + if _OS == "Windows": + subprocess.run(["shutdown", "/r", "/t", "5"]) + elif _OS == "Darwin": + subprocess.run(["osascript", "-e", 'tell application "System Events" to restart']) + else: + subprocess.run(["sudo", "reboot"]) + +def shutdown_computer(): + if _OS == "Windows": + subprocess.run(["shutdown", "/s", "/t", "5"]) + elif _OS == "Darwin": + subprocess.run(["osascript", "-e", 'tell application "System Events" to shut down']) + else: + subprocess.run(["sudo", "shutdown", "-h", "now"]) + +def dark_mode(): + if _OS == "Windows": + pyautogui.hotkey("win", "a") + time.sleep(0.3) + elif _OS == "Darwin": + subprocess.run(["osascript", "-e", + 'tell app "System Events" to tell appearance preferences to set dark mode to not dark mode']) + +def toggle_wifi(): + if _OS == "Windows": + pyautogui.hotkey("win", "a") + time.sleep(0.3) + elif _OS == "Darwin": + subprocess.run(["networksetup", "-setairportpower", "en0", "toggle"]) + else: + subprocess.run(["nmcli", "radio", "wifi"]) + +ACTION_MAP = { + "volume_up": volume_up, + "volume_down": volume_down, + "mute": volume_mute, + "unmute": volume_mute, + "volume_increase": volume_up, + "volume_decrease": volume_down, + "increase_volume": volume_up, + "decrease_volume": volume_down, + "turn_up_volume": volume_up, + "turn_down_volume": volume_down, + "louder": volume_up, + "quieter": volume_down, + "silence": volume_mute, + "toggle_mute": volume_mute, + "brightness_up": brightness_up, + "brightness_down": brightness_down, + "increase_brightness": brightness_up, + "decrease_brightness": brightness_down, + "brighter": brightness_up, + "dimmer": brightness_down, + "dim_screen": brightness_down, + "brighten_screen": brightness_up, + "sleep_display": sleep_display, + "turn_off_screen": sleep_display, + "screen_off": sleep_display, + "display_off": sleep_display, + "change_screen": sleep_display, + "screen_sleep": sleep_display, + "monitor_off": sleep_display, + "turn_off_monitor": sleep_display, + "pause_video": pause_video, + "play_video": pause_video, + "pause": pause_video, + "play": pause_video, + "toggle_play": pause_video, + "stop_video": pause_video, + "resume_video": pause_video, + "close_app": close_app, + "close_window": close_window, + "quit_app": close_app, + "exit_app": close_app, + "kill_app": close_app, + "full_screen": full_screen, + "fullscreen": full_screen, + "toggle_fullscreen": full_screen, + "minimize": minimize_window, + "minimize_window": minimize_window, + "maximize": maximize_window, + "maximize_window": maximize_window, + "restore_window": maximize_window, + "snap_left": snap_left, + "snap_right": snap_right, + "window_left": snap_left, + "window_right": snap_right, + "switch_window": switch_window, + "alt_tab": switch_window, + "next_window": switch_window, + "show_desktop": show_desktop, + "desktop": show_desktop, + "hide_windows": show_desktop, + "task_manager": open_task_manager, + "open_task_manager": open_task_manager, + "task_view": open_task_view, + "screenshot": take_screenshot, + "take_screenshot": take_screenshot, + "capture_screen": take_screenshot, + "lock_screen": lock_screen, + "lock": lock_screen, + "open_settings": open_system_settings, + "system_settings": open_system_settings, + "settings": open_system_settings, + "preferences": open_system_settings, + "file_explorer": open_file_explorer, + "open_explorer": open_file_explorer, + "explorer": open_file_explorer, + "open_files": open_file_explorer, + "run": open_run, + "open_run": open_run, + "restart": restart_computer, + "restart_computer": restart_computer, + "reboot": restart_computer, + "reboot_computer": restart_computer, + "shutdown": shutdown_computer, + "shut_down": shutdown_computer, + "power_off": shutdown_computer, + "turn_off_computer": shutdown_computer, + "dark_mode": dark_mode, + "toggle_dark_mode": dark_mode, + "night_mode": dark_mode, + "toggle_wifi": toggle_wifi, + "wifi": toggle_wifi, + "wifi_toggle": toggle_wifi, + "focus_search": focus_search, + "address_bar": focus_search, + "url_bar": focus_search, + "refresh_page": refresh_page, + "reload_page": refresh_page, + "reload": refresh_page, + "refresh": refresh_page, + "close_tab": close_tab, + "new_tab": new_tab, + "open_tab": new_tab, + "next_tab": next_tab, + "prev_tab": prev_tab, + "previous_tab": prev_tab, + "go_back": go_back, + "back": go_back, + "go_forward": go_forward, + "forward": go_forward, + "zoom_in": zoom_in, + "zoom_out": zoom_out, + "zoom_reset": zoom_reset, + "reset_zoom": zoom_reset, + "find_on_page": find_on_page, + "search_page": find_on_page, + "scroll_up": scroll_up, + "scroll_down": scroll_down, + "scroll_top": scroll_top, + "scroll_bottom": scroll_bottom, + "top_of_page": scroll_top, + "bottom_of_page": scroll_bottom, + "page_up": page_up, + "page_down": page_down, + "copy": copy, + "paste": paste, + "cut": cut, + "undo": undo, + "redo": redo, + "select_all": select_all, + "save": save_file, + "save_file": save_file, + "enter": press_enter, + "press_enter": press_enter, + "escape": press_escape, + "press_escape": press_escape, + "cancel": press_escape, +} + +def _detect_action(description: str) -> dict: + """ + Gemini ile kullanıcının ne yapmak istediğini anlar. + Herhangi bir dilde çalışır. + Döner: {"action": str, "value": optional} + """ + import google.generativeai as genai + genai.configure(api_key=_get_api_key()) + model = genai.GenerativeModel("gemini-2.5-flash-lite") + + available = ", ".join(sorted(ACTION_MAP.keys())) + ", volume_set, type_text, write_on_screen, reload_n, press_key" + + prompt = f"""The user wants to control their computer. Detect their intent. + +User said (in any language): "{description}" + +Available actions: {available} + +Return ONLY valid JSON: +{{"action": "action_name", "value": null_or_value}} + +Examples: +- "turn up the volume" → {{"action": "volume_up", "value": null}} +- "set volume to 60" → {{"action": "volume_set", "value": 60}} +- "sesi 80 yap" → {{"action": "volume_set", "value": 80}} +- "close the app" → {{"action": "close_app", "value": null}} +- "uygulamayı kapat" → {{"action": "close_app", "value": null}} +- "type hello world" → {{"action": "type_text", "value": "hello world"}} +- "write good morning on screen" → {{"action": "write_on_screen", "value": "good morning"}} +- "reload page 3 times" → {{"action": "reload_n", "value": 3}} +- "tam ekran yap" → {{"action": "full_screen", "value": null}} +- "sesi kıs" → {{"action": "volume_down", "value": null}} +- "sesi aç" → {{"action": "volume_up", "value": null}} +- "sustur" → {{"action": "mute", "value": null}} +- "monte le son" → {{"action": "volume_up", "value": null}} +- "ekranı kapat" → {{"action": "sleep_display", "value": null}} +- "monitörü kapat" → {{"action": "sleep_display", "value": null}} +- "turn off screen" → {{"action": "sleep_display", "value": null}} +- "turn off monitor" → {{"action": "sleep_display", "value": null}} +- "bilgisayarı yeniden başlat" → {{"action": "restart", "value": null}} +- "restart the computer" → {{"action": "restart", "value": null}} +- "bilgisayarı kapat" → {{"action": "shutdown", "value": null}} +- "shut down" → {{"action": "shutdown", "value": null}} +- "ekranı kilitle" → {{"action": "lock_screen", "value": null}} +- "lock the screen" → {{"action": "lock_screen", "value": null}} +- "küçült" → {{"action": "minimize", "value": null}} +- "minimize the window" → {{"action": "minimize", "value": null}} +- "büyüt" → {{"action": "maximize", "value": null}} +- "parlaklığı artır" → {{"action": "brightness_up", "value": null}} +- "parlaklığı azalt" → {{"action": "brightness_down", "value": null}} +- "increase brightness" → {{"action": "brightness_up", "value": null}} +- "wifi'yi aç" → {{"action": "toggle_wifi", "value": null}} +- "toggle wifi" → {{"action": "toggle_wifi", "value": null}} +- "masaüstünü göster" → {{"action": "show_desktop", "value": null}} +- "show desktop" → {{"action": "show_desktop", "value": null}} +- "yeni sekme aç" → {{"action": "new_tab", "value": null}} +- "sekmeyi kapat" → {{"action": "close_tab", "value": null}} +- "geri git" → {{"action": "go_back", "value": null}} +- "ileri git" → {{"action": "go_forward", "value": null}} +- "sayfayı yenile" → {{"action": "refresh_page", "value": null}} +- "yakınlaştır" → {{"action": "zoom_in", "value": null}} +- "uzaklaştır" → {{"action": "zoom_out", "value": null}} +- "kaydet" → {{"action": "save", "value": null}} +- "geri al" → {{"action": "undo", "value": null}} +- "screenshot al" → {{"action": "screenshot", "value": null}} +- "ekran görüntüsü al" → {{"action": "screenshot", "value": null}} +- "aşağı kaydır" → {{"action": "scroll_down", "value": null}} +- "yukarı kaydır" → {{"action": "scroll_up", "value": null}} +- "karanlık mod" → {{"action": "dark_mode", "value": null}} +- "press f5" → {{"action": "press_key", "value": "f5"}} +- "enter'a bas" → {{"action": "enter", "value": null}} +- "escape'e bas" → {{"action": "escape", "value": null}} + +IMPORTANT: +- Always return one of the available actions listed above. +- If the user's intent is clear but uses different wording, map it to the closest action. +- Never invent new action names not in the available list. +- Return ONLY the JSON object, no explanation, no markdown.""" + + try: + response = model.generate_content(prompt) + text = response.text.strip() + text = __import__("re").sub(r"```(?:json)?", "", text).strip().rstrip("`").strip() + return json.loads(text) + except Exception as e: + print(f"[Settings] ⚠️ Intent detection failed: {e}") + return {"action": description.lower().replace(" ", "_"), "value": None} + +def computer_settings( + parameters: dict, + response=None, + player=None, + session_memory=None, +) -> str: + """ + Bilgisayar ayarları ve UI kontrolleri. + + parameters: + action : İşlem adı (verilmezse description'dan Gemini ile tespit edilir) + description : Kullanıcının doğal dil komutu (herhangi bir dilde) + value : İşleme özgü değer (ses seviyesi, yazılacak metin, tekrar sayısı vb.) + """ + if not _PYAUTOGUI: + return "pyautogui is not installed. Run: pip install pyautogui" + + params = parameters or {} + raw_action = params.get("action", "").strip() + description = params.get("description", "").strip() + value = params.get("value", None) + + if not raw_action and description: + detected = _detect_action(description) + raw_action = detected.get("action", "") + if value is None: + value = detected.get("value") + + action = raw_action.lower().strip().replace(" ", "_").replace("-", "_") + + if not action: + return "No action could be determined, sir." + + print(f"[Settings] ⚙️ Action: {action} Value: {value}") + + + if action == "volume_set": + try: + volume_set(int(value or 50)) + return f"Volume set to {value}%." + except Exception as e: + return f"Could not set volume: {e}" + + if action in ("type_text", "write_on_screen", "type", "write"): + text = str(value or params.get("text", "")) + if not text: + return "No text provided to type, sir." + enter_after = bool(params.get("press_enter", False)) + type_text(text, press_enter_after=enter_after) + return f"Typed: {text[:60]}" + + if action == "press_key": + key = str(value or params.get("key", "")) + if not key: + return "No key specified, sir." + press_key(key) + return f"Pressed: {key}" + + if action in ("reload_n", "refresh_n", "reload_page_n"): + try: + n = int(value or 1) + reload_page_n(n) + return f"Reloaded page {n} time{'s' if n > 1 else ''}." + except Exception as e: + return f"Could not reload: {e}" + + if action in ("scroll_up", "scroll_down"): + try: + amount = int(value or 500) + scroll_up(amount) if action == "scroll_up" else scroll_down(amount) + return f"Scrolled {'up' if action == 'scroll_up' else 'down'}." + except Exception as e: + return f"Scroll failed: {e}" + + func = ACTION_MAP.get(action) + if not func: + return f"Unknown action: '{raw_action}', sir." + + try: + func() + return f"Done: {action}." + except Exception as e: + return f"Action failed ({action}): {e}" \ No newline at end of file diff --git a/actions/desktop.py b/actions/desktop.py new file mode 100644 index 0000000..a5cfeba --- /dev/null +++ b/actions/desktop.py @@ -0,0 +1,417 @@ +# actions/desktop.py +# AI-powered desktop & wallpaper management +# +# Flow for unknown tasks: +# User request → Gemini generates Python/pyautogui code → Safety check → Execute +# +# Built-in: wallpaper change, icon arrangement, desktop cleanup, organize by type + +import os +import sys +import json +import shutil +import subprocess +import ctypes +import tempfile +import pyautogui +from pathlib import Path +from datetime import datetime + + +def get_base_dir(): + if getattr(sys, "frozen", False): + return Path(sys.executable).parent + return Path(__file__).resolve().parent.parent + +BASE_DIR = get_base_dir() +API_CONFIG_PATH = BASE_DIR / "config" / "api_keys.json" + + +def _get_api_key() -> str: + with open(API_CONFIG_PATH, "r", encoding="utf-8") as f: + return json.load(f)["gemini_api_key"] + + +def _get_desktop() -> Path: + return Path.home() / "Desktop" + + +BLOCKED_KEYWORDS = [ + "os.remove", "shutil.rmtree", "shutil.rm", + "subprocess.run", "subprocess.Popen", "subprocess.call", + "os.system", "exec(", "eval(", + "import os", "import subprocess", + "__import__", "open(", + "sys.exit", "quit()", +] + + +def _is_safe_code(code: str) -> tuple[bool, str]: + code_lower = code.lower() + for keyword in BLOCKED_KEYWORDS: + if keyword.lower() in code_lower: + return False, f"Blocked operation: '{keyword}'" + return True, "OK" + + +def _ask_gemini_for_desktop_action(task: str) -> str: + """ + Asks Gemini to generate safe Python/pyautogui code + to accomplish a desktop-related task. + """ + import google.generativeai as genai + + genai.configure(api_key=_get_api_key()) + model = genai.GenerativeModel("gemini-2.5-flash") + + desktop = str(_get_desktop()) + + prompt = f"""You are a Windows desktop automation expert. +Generate safe Python code using ONLY these allowed modules: +- pyautogui (mouse, keyboard, screenshots) +- pathlib.Path (already imported as Path) +- shutil (ONLY: copy2, copytree, move, disk_usage) +- os.path (ONLY: exists, join, dirname, basename, splitext) +- time (sleep only) +- ctypes (Windows API calls only) +- winreg (registry READ only) + +Desktop path: {desktop} + +Rules: +- Output ONLY the Python code. No explanation, no markdown, no backticks. +- NO file deletion (os.remove, shutil.rmtree, unlink, etc.) +- NO subprocess calls +- NO exec() or eval() +- NO file write operations +- If task cannot be done safely, output exactly: UNSAFE + +Task: {task} + +Python code:""" + + try: + response = model.generate_content(prompt) + code = response.text.strip() + if code.startswith("```"): + lines = code.split("\n") + code = "\n".join(lines[1:-1]).strip() + return code + except Exception as e: + return f"ERROR: {e}" + + +def _execute_generated_code(code: str) -> str: + """Safely executes Gemini-generated desktop automation code.""" + safe, reason = _is_safe_code(code) + if not safe: + return f"⛔ Blocked for safety: {reason}" + + allowed_globals = { + "pyautogui": pyautogui, + "Path": Path, + "shutil": shutil, + "ctypes": ctypes, + "time": __import__("time"), + "os": type("os", (), { + "path": os.path, + "listdir": os.listdir, + "getcwd": os.getcwd, + "environ": os.environ, + })(), + "__builtins__": { + "print": print, + "len": len, + "str": str, + "int": int, + "float": float, + "bool": bool, + "list": list, + "dict": dict, + "range": range, + "enumerate": enumerate, + "sorted": sorted, + "isinstance": isinstance, + "hasattr": hasattr, + "getattr": getattr, + "max": max, + "min": min, + "sum": sum, + } + } + + output_lines = [] + allowed_globals["print"] = lambda *args: output_lines.append(" ".join(str(a) for a in args)) + + try: + exec(code, allowed_globals) + return "\n".join(output_lines) if output_lines else "Task completed successfully." + except Exception as e: + return f"Execution error: {e}\n\nCode attempted:\n{code[:200]}" + +def set_wallpaper(image_path: str) -> str: + """Sets desktop wallpaper from a local image path.""" + path = Path(image_path).expanduser().resolve() + if not path.exists(): + return f"Image not found: {image_path}" + if path.suffix.lower() not in [".jpg", ".jpeg", ".png", ".bmp"]: + return f"Unsupported format: {path.suffix}. Use jpg, png or bmp." + + try: + if sys.platform == "win32": + + abs_path = str(path.resolve()) + ctypes.windll.user32.SystemParametersInfoW(20, 0, abs_path, 3) + return f"Wallpaper set: {path.name}" + + elif sys.platform == "darwin": + script = f'tell application "Finder" to set desktop picture to POSIX file "{path}"' + subprocess.run(["osascript", "-e", script]) + return f"Wallpaper set: {path.name}" + + else: + subprocess.run(["gsettings", "set", "org.gnome.desktop.background", + "picture-uri", f"file://{path}"]) + return f"Wallpaper set: {path.name}" + + except Exception as e: + return f"Could not set wallpaper: {e}" + + +def set_wallpaper_from_web(url: str) -> str: + """Downloads an image from URL and sets it as wallpaper.""" + try: + import urllib.request + suffix = Path(url.split("?")[0]).suffix or ".jpg" + tmp = Path(tempfile.mktemp(suffix=suffix)) + urllib.request.urlretrieve(url, str(tmp)) + result = set_wallpaper(str(tmp)) + return result + except Exception as e: + return f"Could not download wallpaper: {e}" + + +def get_current_wallpaper() -> str: + """Returns the current wallpaper path.""" + try: + if sys.platform == "win32": + import winreg + key = winreg.OpenKey(winreg.HKEY_CURRENT_USER, + r"Control Panel\Desktop") + val, _ = winreg.QueryValueEx(key, "Wallpaper") + return f"Current wallpaper: {val}" + else: + return "Wallpaper path retrieval not supported on this OS." + except Exception as e: + return f"Could not get wallpaper: {e}" + + +FILE_TYPE_MAP = { + "Images": [".jpg", ".jpeg", ".png", ".gif", ".bmp", ".webp", ".svg", ".ico", ".heic"], + "Documents": [".pdf", ".doc", ".docx", ".txt", ".xls", ".xlsx", ".ppt", ".pptx", ".csv", ".odt"], + "Videos": [".mp4", ".avi", ".mkv", ".mov", ".wmv", ".flv", ".webm", ".m4v"], + "Music": [".mp3", ".wav", ".flac", ".aac", ".ogg", ".wma", ".m4a"], + "Archives": [".zip", ".rar", ".7z", ".tar", ".gz", ".bz2"], + "Code": [".py", ".js", ".html", ".css", ".json", ".xml", ".ts", ".cpp", ".java", ".cs", ".php"], + "Executables": [".exe", ".msi", ".bat", ".cmd", ".sh"], +} + + +def organize_desktop(mode: str = "by_type") -> str: + """ + Organizes desktop files. + mode: 'by_type' — groups by file type (Images, Documents, etc.) + 'by_date' — groups by month (2024-01, 2024-02, etc.) + """ + desktop = _get_desktop() + moved = [] + skipped = [] + + for item in desktop.iterdir(): + if item.is_dir() or item.name.startswith("."): + continue + + if item.suffix.lower() == ".lnk": + continue + + if mode == "by_date": + mtime = datetime.fromtimestamp(item.stat().st_mtime) + folder_name = mtime.strftime("%Y-%m") + else: + ext = item.suffix.lower() + folder_name = "Others" + for folder, exts in FILE_TYPE_MAP.items(): + if ext in exts: + folder_name = folder + break + + target_dir = desktop / folder_name + target_dir.mkdir(exist_ok=True) + new_path = target_dir / item.name + + if new_path.exists(): + skipped.append(item.name) + continue + + shutil.move(str(item), str(new_path)) + moved.append(f"{item.name} → {folder_name}/") + + result = f"Desktop organized ({mode}). {len(moved)} files moved." + if moved: + preview = moved[:8] + result += "\n" + "\n".join(preview) + if len(moved) > 8: + result += f"\n... and {len(moved)-8} more." + if skipped: + result += f"\n{len(skipped)} files skipped (name conflict)." + return result + + +def list_desktop() -> str: + """Lists everything on the desktop.""" + desktop = _get_desktop() + items = [] + + for item in sorted(desktop.iterdir()): + if item.name.startswith("."): + continue + if item.is_dir(): + count = len(list(item.iterdir())) + items.append(f"📁 {item.name}/ ({count} items)") + else: + size = item.stat().st_size + size_str = f"{size/1024:.1f} KB" if size < 1024*1024 else f"{size/1024/1024:.1f} MB" + items.append(f"📄 {item.name} ({size_str})") + + if not items: + return "Desktop is empty." + return f"Desktop ({len(items)} items):\n" + "\n".join(items) + + +def clean_desktop() -> str: + """ + Moves all files on desktop into a 'Desktop Archive' folder + with today's date — fast cleanup without deleting anything. + """ + desktop = _get_desktop() + today = datetime.now().strftime("%Y-%m-%d") + archive_dir = desktop / f"Desktop Archive {today}" + archive_dir.mkdir(exist_ok=True) + + moved = 0 + for item in desktop.iterdir(): + if item.is_dir() or item.name.startswith("."): + continue + if item.suffix.lower() == ".lnk": + continue + new_path = archive_dir / item.name + if not new_path.exists(): + shutil.move(str(item), str(new_path)) + moved += 1 + + return f"Desktop cleaned. {moved} files moved to '{archive_dir.name}'." + + +def get_desktop_stats() -> str: + """Returns stats about the desktop.""" + desktop = _get_desktop() + files = [i for i in desktop.iterdir() if i.is_file()] + folders = [i for i in desktop.iterdir() if i.is_dir()] + total_size = sum(f.stat().st_size for f in files) + size_str = f"{total_size/1024:.1f} KB" if total_size < 1024*1024 else f"{total_size/1024/1024:.1f} MB" + + return ( + f"Desktop stats:\n" + f" Files : {len(files)}\n" + f" Folders : {len(folders)}\n" + f" Total size: {size_str}" + ) + + +def desktop_control( + parameters: dict, + response=None, + player=None, + session_memory=None +) -> str: + """ + Called from main.py. + + parameters: + action : wallpaper | wallpaper_url | current_wallpaper | + organize | clean | list | stats | + task (AI-powered — anything else) + + path : image path for 'wallpaper' + url : image URL for 'wallpaper_url' + mode : 'by_type' or 'by_date' for 'organize' + task : Natural language description for AI-powered actions. + Example: "arrange icons by size" + "show me what's on my desktop" + "move all screenshots to a folder" + """ + action = (parameters or {}).get("action", "").lower().strip() + task = (parameters or {}).get("task", "").strip() + + result = "Unknown action." + + try: + if action == "wallpaper": + path = parameters.get("path", "") + result = set_wallpaper(path) if path else "No image path provided." + + elif action == "wallpaper_url": + url = parameters.get("url", "") + result = set_wallpaper_from_web(url) if url else "No URL provided." + + elif action == "current_wallpaper": + result = get_current_wallpaper() + + elif action == "organize": + mode = parameters.get("mode", "by_type") + result = organize_desktop(mode) + + elif action == "clean": + result = clean_desktop() + + elif action == "list": + result = list_desktop() + + elif action == "stats": + result = get_desktop_stats() + + elif action == "task" or task: + actual_task = task or parameters.get("description", "") + if not actual_task: + return "Please describe what you want to do on the desktop, sir." + + print(f"[Desktop] 🤖 Asking Gemini: {actual_task}") + if player: + player.write_log("[Desktop] Generating action...") + + code = _ask_gemini_for_desktop_action(actual_task) + + if code == "UNSAFE": + result = "I cannot perform that desktop action safely, sir." + elif code.startswith("ERROR:"): + result = f"Could not generate action: {code}" + else: + print(f"[Desktop] ✅ Generated code:\n{code[:200]}") + result = _execute_generated_code(code) + + else: + full_task = task or action + if full_task: + code = _ask_gemini_for_desktop_action(full_task) + result = _execute_generated_code(code) if code not in ("UNSAFE",) else "Cannot do that safely." + else: + result = "No action or task specified." + + except Exception as e: + result = f"Desktop control error: {e}" + + print(f"[Desktop] {result[:100]}") + if player: + player.write_log(f"[desktop] {result[:60]}") + + return result \ No newline at end of file diff --git a/actions/dev_agent.py b/actions/dev_agent.py new file mode 100644 index 0000000..cce8f4f --- /dev/null +++ b/actions/dev_agent.py @@ -0,0 +1,491 @@ +# actions/dev_agent.py +# AI-powered development agent — plans, builds, and debugs full projects. +# +# Flow: +# Describe project → Gemini plans file structure → Files written one by one +# → VSCode opened → Entry point executed → Error? → Identify file → Fix → Retry +# → Speaks only when done (success or failure) +# +# Models: +# Planning : gemini-2.5-flash (architecture, structure, debugging) +# Writing : gemini-2.5-flash-lite (fast file generation) + +import subprocess +import sys +import json +import re +import time +from pathlib import Path + +def get_base_dir(): + if getattr(sys, "frozen", False): + return Path(sys.executable).parent + return Path(__file__).resolve().parent.parent + +BASE_DIR = get_base_dir() +API_CONFIG_PATH = BASE_DIR / "config" / "api_keys.json" +PROJECTS_DIR = Path.home() / "Desktop" / "JarvisProjects" +MAX_FIX_ATTEMPTS = 4 +MODEL_PLANNER = "gemini-2.5-flash" +MODEL_WRITER = "gemini-2.5-flash-lite" + + +def _get_api_key() -> str: + with open(API_CONFIG_PATH, "r", encoding="utf-8") as f: + return json.load(f)["gemini_api_key"] + + +def _get_model(model_name: str): + import google.generativeai as genai + genai.configure(api_key=_get_api_key()) + return genai.GenerativeModel(model_name) + + +def _clean_code(text: str) -> str: + text = text.strip() + text = re.sub(r"^```[a-zA-Z]*\n?", "", text) + text = re.sub(r"\n?```$", "", text) + return text.strip() + + +def _clean_json(text: str) -> str: + text = text.strip() + text = re.sub(r"^```[a-zA-Z]*\n?", "", text) + text = re.sub(r"\n?```$", "", text) + return text.strip() + + +def _is_rate_limit(error: Exception) -> bool: + return "429" in str(error) or "quota" in str(error).lower() + + +def _get_interpreter(path: Path) -> list[str] | None: + return { + ".py": [sys.executable], + ".js": ["node"], + ".ts": ["ts-node"], + ".sh": ["bash"], + ".ps1": ["powershell", "-File"], + ".rb": ["ruby"], + ".php": ["php"], + }.get(path.suffix.lower()) + + +def _has_error(output: str) -> bool: + if "timed out" in output.lower(): + return False + signals = ["error", "exception", "traceback", "syntaxerror", + "nameerror", "typeerror", "importerror", "stderr", "failed"] + return any(s in output.lower() for s in signals) + +def _identify_error_file(error_output: str, project_files: list[str]) -> str | None: + """ + Try to find which file caused the error from traceback. + Returns filename or None. + """ + for line in error_output.splitlines(): + for f in project_files: + if Path(f).name in line or f in line: + return f + return None + +def _plan_project(description: str, language: str) -> dict: + """ + Ask Gemini to plan the full project structure. + + Returns: + { + "project_name": "my_app", + "entry_point": "main.py", + "files": [ + { + "path": "main.py", + "description": "Entry point, starts the app" + }, + { + "path": "utils/helpers.py", + "description": "Utility functions" + } + ], + "run_command": "python main.py", + "dependencies": ["requests", "flask"] + } + """ + model = _get_model(MODEL_PLANNER) + + prompt = f"""You are a senior software architect. +Plan the complete file structure for the following project. + +Language: {language} +Description: {description} + +Return ONLY a valid JSON object with this exact structure: +{{ + "project_name": "short_snake_case_name", + "entry_point": "main.py", + "files": [ + {{"path": "main.py", "description": "what this file does"}}, + {{"path": "utils/helpers.py", "description": "what this file does"}} + ], + "run_command": "python main.py", + "dependencies": ["package1", "package2"] +}} + +Rules: +- Keep it simple. Only include files that are truly necessary. +- No explanation, no markdown, no backticks. Pure JSON only. +- Entry point must be one of the files listed. +- Use relative paths only. + +JSON:""" + + try: + response = model.generate_content(prompt) + raw = _clean_json(response.text) + return json.loads(raw) + except json.JSONDecodeError as e: + raise ValueError(f"Planner returned invalid JSON: {e}\nRaw: {response.text[:300]}") + + +def _write_file( + file_path: str, + file_description: str, + project_description: str, + all_files: list[dict], + language: str, + project_dir: Path +) -> str: + """Write one file. Returns the generated code.""" + model = _get_model(MODEL_WRITER) + + file_list = "\n".join( + f" - {f['path']}: {f['description']}" for f in all_files + ) + + prompt = f"""You are an expert {language} developer. +Write the code for ONE specific file in a larger project. + +Project goal: {project_description} + +All files in this project: +{file_list} + +Now write ONLY the file: {file_path} +Purpose of this file: {file_description} + +Rules: +- Output ONLY the code for this file. No explanation, no markdown, no backticks. +- Import from other project files using relative imports where needed. +- Add helpful inline comments. +- Handle errors properly. +- Use modern best practices. + +Code for {file_path}:""" + + try: + response = model.generate_content(prompt) + code = _clean_code(response.text) + + # Save file + full_path = project_dir / file_path + full_path.parent.mkdir(parents=True, exist_ok=True) + full_path.write_text(code, encoding="utf-8") + + print(f"[DevAgent] ✅ Written: {file_path}") + return code + + except Exception as e: + if _is_rate_limit(e): + raise RateLimitError(str(e)) + raise + + +def _install_dependencies(dependencies: list[str], project_dir: Path) -> str: + if not dependencies: + return "No dependencies to install." + + print(f"[DevAgent] 📦 Installing: {dependencies}") + + try: + result = subprocess.run( + [sys.executable, "-m", "pip", "install"] + dependencies, + capture_output=True, text=True, + encoding="utf-8", errors="replace", + timeout=120, cwd=str(project_dir) + ) + if result.returncode == 0: + return f"Installed: {', '.join(dependencies)}" + return f"Install warning: {result.stderr[:200]}" + except subprocess.TimeoutExpired: + return "Dependency install timed out." + except Exception as e: + return f"Install error: {e}" + + +def _open_vscode(project_dir: Path) -> bool: + vscode_paths = [ + "code", + r"C:\Users\{}\AppData\Local\Programs\Microsoft VS Code\bin\code.cmd".format( + Path.home().name + ), + ] + for cmd in vscode_paths: + try: + subprocess.Popen( + [cmd, str(project_dir)], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + shell=True + ) + time.sleep(2) + print(f"[DevAgent] 💻 VSCode opened: {project_dir}") + return True + except Exception: + continue + print("[DevAgent] ⚠️ VSCode not found.") + return False + + +def _run_project(run_command: str, project_dir: Path, timeout: int = 30) -> str: + """Run the project entry point, return output.""" + print(f"[DevAgent] 🚀 Running: {run_command}") + + try: + parts = run_command.split() + if parts[0] == "python": + parts[0] = sys.executable + + result = subprocess.run( + parts, + capture_output=True, text=True, + encoding="utf-8", errors="replace", + timeout=timeout, cwd=str(project_dir) + ) + + output = result.stdout.strip() + error = result.stderr.strip() + + parts_out = [] + if output: parts_out.append(f"Output:\n{output}") + if error: parts_out.append(f"Stderr:\n{error}") + return "\n\n".join(parts_out) if parts_out else "Ran with no output." + + except subprocess.TimeoutExpired: + return f"Timed out after {timeout}s. (Long-running app may be working fine.)" + except FileNotFoundError as e: + return f"Command not found: {e}" + except Exception as e: + return f"Run error: {e}" + +def _fix_file( + file_path: str, + current_code: str, + error_output: str, + project_description: str, + all_files: list[dict], + language: str, + project_dir: Path +) -> str: + """Ask Gemini to fix a specific file based on error output.""" + model = _get_model(MODEL_PLANNER) + + file_list = "\n".join( + f" - {f['path']}: {f['description']}" for f in all_files + ) + + prompt = f"""You are an expert {language} debugger. +Fix the file below. It caused an error when the project was run. + +Project goal: {project_description} + +All files in this project: +{file_list} + +File to fix: {file_path} + +Error output: +{error_output[:3000]} + +Current code: +{current_code} + +Return ONLY the fixed code — no explanation, no markdown, no backticks. + +Fixed code:""" + + try: + response = model.generate_content(prompt) + fixed = _clean_code(response.text) + + full_path = project_dir / file_path + full_path.write_text(fixed, encoding="utf-8") + + print(f"[DevAgent] 🔧 Fixed: {file_path}") + return fixed + + except Exception as e: + if _is_rate_limit(e): + raise RateLimitError(str(e)) + raise + +class RateLimitError(Exception): + pass +def _build_project( + description: str, + language: str, + project_name: str, + timeout: int, + speak=None, + player=None +) -> str: + """ + Full build loop: + Plan → Write files → Install deps → Open VSCode → Run → Fix loop + """ + + def log(msg: str): + print(f"[DevAgent] {msg}") + if player: + player.write_log(f"[DevAgent] {msg}") + + log("Planning project structure...") + try: + plan = _plan_project(description, language) + except RateLimitError: + msg = "You have reached the rate limit, sir. Please try again shortly." + if speak: speak(msg) + return msg + except ValueError as e: + msg = f"Planning failed: {e}" + if speak: speak(msg) + return msg + + proj_name = project_name or plan.get("project_name", "jarvis_project") + proj_name = re.sub(r"[^\w\-]", "_", proj_name) + project_dir = PROJECTS_DIR / proj_name + project_dir.mkdir(parents=True, exist_ok=True) + + files = plan.get("files", []) + entry_point = plan.get("entry_point", "main.py") + run_command = plan.get("run_command", f"python {entry_point}") + dependencies = plan.get("dependencies", []) + + log(f"Project: {proj_name} | Files: {len(files)} | Entry: {entry_point}") + + file_codes: dict[str, str] = {} + + for file_info in files: + file_path = file_info.get("path", "") + file_desc = file_info.get("description", "") + if not file_path: + continue + + log(f"Writing {file_path}...") + try: + code = _write_file( + file_path, file_desc, description, + files, language, project_dir + ) + file_codes[file_path] = code + except RateLimitError: + msg = "You have reached the rate limit, sir. Please try again shortly." + if speak: speak(msg) + return msg + except Exception as e: + log(f"Failed to write {file_path}: {e}") + continue + + if not file_codes: + msg = "I could not write any files for this project, sir." + if speak: speak(msg) + return msg + + if dependencies: + log(f"Installing dependencies: {dependencies}") + _install_dependencies(dependencies, project_dir) + + _open_vscode(project_dir) + + last_output = "" + for attempt in range(1, MAX_FIX_ATTEMPTS + 1): + log(f"Running project (attempt {attempt}/{MAX_FIX_ATTEMPTS})...") + + last_output = _run_project(run_command, project_dir, timeout) + log(f"Output: {last_output[:150]}") + + if not _has_error(last_output): + msg = ( + f"Project '{proj_name}' is working, sir. " + f"Built in {attempt} attempt{'s' if attempt > 1 else ''}. " + f"Opened in VSCode at {project_dir}." + ) + if speak: speak(msg) + return f"{msg}\n\nOutput:\n{last_output}" + + if attempt == MAX_FIX_ATTEMPTS: + break + + error_file = _identify_error_file(last_output, list(file_codes.keys())) + if not error_file: + error_file = entry_point + + log(f"Error in '{error_file}', fixing...") + + try: + fixed = _fix_file( + error_file, + file_codes.get(error_file, ""), + last_output, + description, + files, + language, + project_dir + ) + file_codes[error_file] = fixed + except RateLimitError: + msg = "You have reached the rate limit, sir. Please try again shortly." + if speak: speak(msg) + return msg + except Exception as e: + log(f"Fix failed: {e}") + + msg = ( + f"I was unable to get '{proj_name}' working after {MAX_FIX_ATTEMPTS} attempts, sir. " + f"The project is saved at {project_dir} — you can open it in VSCode and check manually." + ) + if speak: speak(msg) + return f"{msg}\n\nLast error:\n{last_output[:500]}" + +def dev_agent( + parameters: dict, + response=None, + player=None, + session_memory=None, + speak=None +) -> str: + """ + Called from main.py. + + parameters: + description : What the project should do (required) + language : Programming language (default: python) + project_name : Optional folder name (auto-generated if not given) + timeout : Run timeout in seconds (default: 30) + """ + p = parameters or {} + description = p.get("description", "").strip() + language = p.get("language", "python").strip() + project_name = p.get("project_name", "").strip() + timeout = int(p.get("timeout", 30)) + + if not description: + return "Please describe the project you want me to build, sir." + + return _build_project( + description = description, + language = language, + project_name = project_name, + timeout = timeout, + speak = speak, + player = player + ) \ No newline at end of file diff --git a/actions/file_controller.py b/actions/file_controller.py new file mode 100644 index 0000000..1dc5b64 --- /dev/null +++ b/actions/file_controller.py @@ -0,0 +1,482 @@ +# actions/file_controller.py +# File management — create, delete, move, rename, list, find, organize + +import shutil +from pathlib import Path +from datetime import datetime +import send2trash + +def _get_desktop() -> Path: + """Returns desktop path — works on Windows, Mac, Linux.""" + return Path.home() / "Desktop" + + +def _get_downloads() -> Path: + return Path.home() / "Downloads" + + +def _resolve_path(raw: str) -> Path: + """ + Resolves a path from user input. + Supports shortcuts: 'desktop', 'downloads', 'documents', 'home' + """ + shortcuts = { + "desktop": Path.home() / "Desktop", + "downloads": Path.home() / "Downloads", + "documents": Path.home() / "Documents", + "pictures": Path.home() / "Pictures", + "music": Path.home() / "Music", + "videos": Path.home() / "Videos", + "home": Path.home(), + } + lower = raw.strip().lower() + if lower in shortcuts: + return shortcuts[lower] + return Path(raw).expanduser() + + +def _format_size(bytes_size: int) -> str: + """Converts bytes to human readable format.""" + for unit in ["B", "KB", "MB", "GB", "TB"]: + if bytes_size < 1024: + return f"{bytes_size:.1f} {unit}" + bytes_size /= 1024 + return f"{bytes_size:.1f} TB" + + +def list_files(path: str = "desktop", show_hidden: bool = False) -> str: + """Lists files and folders in a directory.""" + try: + target = _resolve_path(path) + if not target.exists(): + return f"Path not found: {target}" + if not target.is_dir(): + return f"Not a directory: {target}" + + items = [] + for item in sorted(target.iterdir()): + if not show_hidden and item.name.startswith("."): + continue + if item.is_dir(): + items.append(f"📁 {item.name}/") + else: + size = _format_size(item.stat().st_size) + items.append(f"📄 {item.name} ({size})") + + if not items: + return f"Directory is empty: {target}" + + return f"Contents of {target.name}/ ({len(items)} items):\n" + "\n".join(items) + + except PermissionError: + return f"Permission denied: {path}" + except Exception as e: + return f"Error listing files: {e}" + + +def create_file(path: str, content: str = "") -> str: + """Creates a new file with optional content.""" + try: + target = Path(path).expanduser() + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(content, encoding="utf-8") + return f"File created: {target.name}" + except Exception as e: + return f"Could not create file: {e}" + + +def create_folder(path: str) -> str: + """Creates a new folder (and parent folders if needed).""" + try: + target = Path(path).expanduser() + target.mkdir(parents=True, exist_ok=True) + return f"Folder created: {target}" + except Exception as e: + return f"Could not create folder: {e}" + + +def delete_file(path: str, confirm: bool = True) -> str: + """ + Deletes a file or folder. + Moves to Recycle Bin on Windows if possible, otherwise permanent delete. + """ + try: + target = Path(path).expanduser() + if not target.exists(): + return f"Not found: {path}" + + try: + + send2trash.send2trash(str(target)) + return f"Moved to Recycle Bin: {target.name}" + except ImportError: + pass + + # Fallback: permanent delete + if target.is_dir(): + shutil.rmtree(target) + return f"Folder deleted permanently: {target.name}" + else: + target.unlink() + return f"File deleted permanently: {target.name}" + + except PermissionError: + return f"Permission denied: {path}" + except Exception as e: + return f"Could not delete: {e}" + + +def move_file(source: str, destination: str) -> str: + """Moves a file or folder to a new location.""" + try: + src = Path(source).expanduser() + dst = _resolve_path(destination) + + if not src.exists(): + return f"Source not found: {source}" + + if dst.is_dir(): + dst = dst / src.name + + dst.parent.mkdir(parents=True, exist_ok=True) + shutil.move(str(src), str(dst)) + return f"Moved: {src.name} → {dst.parent.name}/" + + except Exception as e: + return f"Could not move: {e}" + + +def copy_file(source: str, destination: str) -> str: + """Copies a file or folder.""" + try: + src = Path(source).expanduser() + dst = _resolve_path(destination) + + if not src.exists(): + return f"Source not found: {source}" + + if dst.is_dir(): + dst = dst / src.name + + dst.parent.mkdir(parents=True, exist_ok=True) + + if src.is_dir(): + shutil.copytree(str(src), str(dst)) + else: + shutil.copy2(str(src), str(dst)) + + return f"Copied: {src.name} → {dst.parent.name}/" + + except Exception as e: + return f"Could not copy: {e}" + + +def rename_file(path: str, new_name: str) -> str: + """Renames a file or folder.""" + try: + target = Path(path).expanduser() + new_path = target.parent / new_name + + if not target.exists(): + return f"Not found: {path}" + if new_path.exists(): + return f"A file named '{new_name}' already exists." + + target.rename(new_path) + return f"Renamed: {target.name} → {new_name}" + + except Exception as e: + return f"Could not rename: {e}" + + +def read_file(path: str, max_chars: int = 3000) -> str: + """Reads and returns the content of a text file.""" + try: + target = Path(path).expanduser() + if not target.exists(): + return f"File not found: {path}" + if not target.is_file(): + return f"Not a file: {path}" + + content = target.read_text(encoding="utf-8", errors="ignore") + if len(content) > max_chars: + content = content[:max_chars] + f"\n\n... (truncated, {len(content)} total chars)" + return content + + except Exception as e: + return f"Could not read file: {e}" + + +def write_file(path: str, content: str, append: bool = False) -> str: + """Writes or appends content to a file.""" + try: + target = Path(path).expanduser() + target.parent.mkdir(parents=True, exist_ok=True) + mode = "a" if append else "w" + with open(target, mode, encoding="utf-8") as f: + f.write(content) + action = "Appended to" if append else "Written to" + return f"{action}: {target.name}" + except Exception as e: + return f"Could not write file: {e}" + + +def find_files(name: str = "", extension: str = "", path: str = "home", + max_results: int = 20) -> str: + """ + Searches for files by name or extension. + Example: find_files(extension=".pdf", path="documents") + """ + try: + search_path = _resolve_path(path) + if not search_path.exists(): + return f"Search path not found: {path}" + + results = [] + pattern = f"*{extension}" if extension else "*" + + for item in search_path.rglob(pattern): + if item.is_file(): + if name and name.lower() not in item.name.lower(): + continue + size = _format_size(item.stat().st_size) + results.append(f"📄 {item.name} ({size}) — {item.parent}") + if len(results) >= max_results: + break + + if not results: + query = name or extension or "files" + return f"No {query} found in {search_path.name}/" + + return f"Found {len(results)} file(s):\n" + "\n".join(results) + + except Exception as e: + return f"Search error: {e}" + + +def get_largest_files(path: str = "home", count: int = 10) -> str: + """Returns the largest files in a directory.""" + try: + search_path = _resolve_path(path) + if not search_path.exists(): + return f"Path not found: {path}" + + files = [] + for item in search_path.rglob("*"): + if item.is_file(): + try: + files.append((item.stat().st_size, item)) + except Exception: + continue + + files.sort(reverse=True) + top = files[:count] + + if not top: + return "No files found." + + lines = [f"Top {len(top)} largest files in {search_path.name}/:\n"] + for size, f in top: + lines.append(f" {_format_size(size):>10} {f.name} ({f.parent})") + + return "\n".join(lines) + + except Exception as e: + return f"Error: {e}" + + +def get_disk_usage(path: str = "home") -> str: + """Returns disk usage information.""" + try: + target = _resolve_path(path) + usage = shutil.disk_usage(target) + total = _format_size(usage.total) + used = _format_size(usage.used) + free = _format_size(usage.free) + pct = usage.used / usage.total * 100 + + return ( + f"Disk usage for {target}:\n" + f" Total : {total}\n" + f" Used : {used} ({pct:.1f}%)\n" + f" Free : {free}" + ) + except Exception as e: + return f"Could not get disk usage: {e}" + + +def organize_desktop() -> str: + """ + Organizes the desktop by grouping files into folders by type. + Creates folders: Images, Documents, Videos, Music, Archives, Others + """ + try: + desktop = _get_desktop() + type_map = { + "Images": [".jpg", ".jpeg", ".png", ".gif", ".bmp", ".webp", ".svg", ".ico"], + "Documents": [".pdf", ".doc", ".docx", ".txt", ".xls", ".xlsx", ".ppt", ".pptx", ".csv"], + "Videos": [".mp4", ".avi", ".mkv", ".mov", ".wmv", ".flv", ".webm"], + "Music": [".mp3", ".wav", ".flac", ".aac", ".ogg", ".wma"], + "Archives": [".zip", ".rar", ".7z", ".tar", ".gz"], + "Code": [".py", ".js", ".html", ".css", ".json", ".xml", ".ts", ".cpp", ".java"], + } + + moved = [] + skipped = [] + + for item in desktop.iterdir(): + + if item.is_dir() or item.name.startswith("."): + continue + + ext = item.suffix.lower() + target_dir = None + + for folder, extensions in type_map.items(): + if ext in extensions: + target_dir = desktop / folder + break + + if target_dir is None: + target_dir = desktop / "Others" + + target_dir.mkdir(exist_ok=True) + new_path = target_dir / item.name + + if new_path.exists(): + skipped.append(item.name) + continue + + shutil.move(str(item), str(new_path)) + moved.append(f"{item.name} → {target_dir.name}/") + + result = f"Desktop organized. {len(moved)} files moved." + if moved: + result += "\n" + "\n".join(moved[:10]) + if len(moved) > 10: + result += f"\n... and {len(moved)-10} more." + if skipped: + result += f"\n{len(skipped)} files skipped (already exist)." + + return result + + except Exception as e: + return f"Could not organize desktop: {e}" + + +def get_file_info(path: str) -> str: + """Returns detailed information about a file.""" + try: + target = Path(path).expanduser() + if not target.exists(): + return f"Not found: {path}" + + stat = target.stat() + info = { + "Name": target.name, + "Type": "Folder" if target.is_dir() else "File", + "Size": _format_size(stat.st_size), + "Location": str(target.parent), + "Created": datetime.fromtimestamp(stat.st_ctime).strftime("%Y-%m-%d %H:%M"), + "Modified": datetime.fromtimestamp(stat.st_mtime).strftime("%Y-%m-%d %H:%M"), + "Extension": target.suffix or "None", + } + + return "\n".join(f" {k}: {v}" for k, v in info.items()) + + except Exception as e: + return f"Could not get file info: {e}" + +def file_controller( + parameters: dict, + response=None, + player=None, + session_memory=None +) -> str: + action = (parameters or {}).get("action", "").lower().strip() + path = (parameters or {}).get("path", "desktop") + name = (parameters or {}).get("name", "") + content = (parameters or {}).get("content", "") + + def _full_path(p: str, n: str) -> str: + base = _resolve_path(p) + if n: + return str(base / n) + return str(base) + + result = "Unknown action." + + try: + if action == "list": + result = list_files(path) + + elif action == "create_file": + full = _full_path(path, name) + result = create_file(full, content=content) + + elif action == "create_folder": + full = _full_path(path, name) + result = create_folder(full) + + elif action == "delete": + full = _full_path(path, name) + result = delete_file(full) + + elif action == "move": + full = _full_path(path, name) + result = move_file(full, parameters.get("destination", "")) + + elif action == "copy": + full = _full_path(path, name) + result = copy_file(full, parameters.get("destination", "")) + + elif action == "rename": + full = _full_path(path, name) + result = rename_file(full, parameters.get("new_name", "")) + + elif action == "read": + full = _full_path(path, name) + result = read_file(full) + + elif action == "write": + full = _full_path(path, name) + result = write_file( + full, + content=content, + append=parameters.get("append", False) + ) + + elif action == "find": + result = find_files( + name=name or parameters.get("name", ""), + extension=parameters.get("extension", ""), + path=path, + max_results=parameters.get("max_results", 20) + ) + + elif action == "largest": + result = get_largest_files( + path=path, + count=parameters.get("count", 10) + ) + + elif action == "disk_usage": + result = get_disk_usage(path) + + elif action == "organize_desktop": + result = organize_desktop() + + elif action == "info": + full = _full_path(path, name) + result = get_file_info(full) + + else: + result = f"Unknown action: '{action}'" + + except Exception as e: + result = f"File controller error: {e}" + + if player: + player.write_log(f"[file] {result[:60]}") + + return result \ No newline at end of file diff --git a/actions/flight_finder.py b/actions/flight_finder.py new file mode 100644 index 0000000..5722d49 --- /dev/null +++ b/actions/flight_finder.py @@ -0,0 +1,419 @@ +# actions/flight_finder.py +# MARK XXV — Flight Finder +# +# Searches for flights using Google Flights via browser_control. +# Results are spoken by JARVIS. Optionally saved to Notepad or opened in browser. +# +# Flow: +# 1. Parse origin, destination, date, passengers from parameters +# 2. Open Google Flights via browser_control +# 3. Fill in search fields +# 4. Scrape results via get_text +# 5. Parse with Gemini → structured flight data +# 6. Speak top results +# 7. Optionally save to Notepad or keep browser open +# +# Cross-platform: Windows, macOS, Linux +# No API key required — Google Flights is free to access + +import json +import re +import sys +import subprocess +import platform +from datetime import datetime, timedelta +from pathlib import Path + + +def get_base_dir() -> Path: + if getattr(sys, "frozen", False): + return Path(sys.executable).parent + return Path(__file__).resolve().parent.parent + + +BASE_DIR = get_base_dir() +API_CONFIG_PATH = BASE_DIR / "config" / "api_keys.json" + + +def _get_api_key() -> str: + with open(API_CONFIG_PATH, "r", encoding="utf-8") as f: + return json.load(f)["gemini_api_key"] + + +def _parse_date(raw: str) -> str: + """ + Converts natural language date to YYYY-MM-DD. + Handles: '15 Mart', 'March 15', '2025-03-15', 'next friday', etc. + Falls back to Gemini for ambiguous inputs. + """ + raw = raw.strip() + + if re.match(r"\d{4}-\d{2}-\d{2}", raw): + return raw + + for fmt in ("%d/%m/%Y", "%m/%d/%Y", "%d.%m.%Y", "%d-%m-%Y"): + try: + return datetime.strptime(raw, fmt).strftime("%Y-%m-%d") + except ValueError: + pass + + + today = datetime.now() + lower = raw.lower() + relative_map = { + "today": today, + "tomorrow": today + timedelta(days=1), + "bugün": today, + "yarın": today + timedelta(days=1), + } + for key, val in relative_map.items(): + if key in lower: + return val.strftime("%Y-%m-%d") + + try: + import google.generativeai as genai + genai.configure(api_key=_get_api_key()) + model = genai.GenerativeModel("gemini-2.5-flash-lite") + today_str = today.strftime("%Y-%m-%d") + response = model.generate_content( + f"Today is {today_str}. Convert this date to YYYY-MM-DD format: '{raw}'. " + f"Return ONLY the date string, nothing else." + ) + result = response.text.strip() + if re.match(r"\d{4}-\d{2}-\d{2}", result): + return result + except Exception: + pass + + month_map = { + "january": 1, "february": 2, "march": 3, "april": 4, + "may": 5, "june": 6, "july": 7, "august": 8, + "september": 9,"october": 10, "november": 11, "december": 12, + "january": 1, "ocak": 1, "şubat": 2, "mart": 3, + "nisan": 4, "mayıs": 5, "haziran": 6, "temmuz": 7, + "ağustos": 8, "eylül": 9, "ekim": 10, "kasım": 11, + "aralık": 12, + } + for month_name, month_num in month_map.items(): + if month_name in lower: + day_match = re.search(r"\d{1,2}", raw) + if day_match: + day = int(day_match.group()) + year = today.year if month_num >= today.month else today.year + 1 + return f"{year}-{month_num:02d}-{day:02d}" + + return today.strftime("%Y-%m-%d") + + + +def _build_google_flights_url( + origin: str, + destination: str, + date: str, + return_date: str | None = None, + passengers: int = 1, + cabin: str = "economy", +) -> str: + """ + Builds a Google Flights URL with pre-filled search parameters. + Uses the direct search URL format. + """ + cabin_map = { + "economy": "1", + "premium": "2", + "business": "3", + "first": "4", + } + cabin_code = cabin_map.get(cabin.lower(), "1") + + base = "https://www.google.com/travel/flights" + + if return_date: + url = ( + f"{base}?q=Flights+from+{origin}+to+{destination}" + f"+on+{date}+returning+{return_date}" + f"&curr=TRY" + ) + else: + url = ( + f"{base}?q=Flights+from+{origin}+to+{destination}+on+{date}" + f"&curr=TRY" + ) + + return url + + + +def _search_flights_browser( + origin: str, + destination: str, + date: str, + return_date: str | None, + passengers: int, + cabin: str, +) -> tuple[str, str]: + """ + Opens Google Flights in browser, waits for results, scrapes text. + Returns (raw_text, page_url). + """ + from actions.browser_control import browser_control + import time + + url = _build_google_flights_url( + origin, destination, date, return_date, passengers, cabin + ) + + print(f"[FlightFinder] 🌐 Opening: {url}") + browser_control({"action": "go_to", "url": url}) + time.sleep(5) + + result = browser_control({"action": "get_text"}) + return result or "", url + + +def _parse_flights_with_gemini( + raw_text: str, + origin: str, + destination: str, + date: str, +) -> list[dict]: + """ + Sends raw page text to Gemini and extracts structured flight data. + Returns list of flight dicts. + """ + import google.generativeai as genai + + genai.configure(api_key=_get_api_key()) + model = genai.GenerativeModel( + model_name="gemini-2.5-flash", + system_instruction=( + "You are a flight data extraction expert. " + "Extract flight information from raw webpage text. " + "Return ONLY valid JSON array. No explanation, no markdown." + ) + ) + + truncated = raw_text[:12000] + + prompt = ( + f"Extract flight options from {origin} to {destination} on {date} " + f"from this Google Flights page text:\n\n{truncated}\n\n" + f"Return a JSON array of up to 5 flights:\n" + f'[{{"airline": "...", "departure": "HH:MM", "arrival": "HH:MM", ' + f'"duration": "Xh Ym", "stops": 0, "price": "...", "currency": "..."}}]\n' + f"If no flights found, return: []" + ) + + try: + response = model.generate_content(prompt) + text = response.text.strip() + text = re.sub(r"```(?:json)?", "", text).strip().rstrip("`").strip() + flights = json.loads(text) + return flights if isinstance(flights, list) else [] + except Exception as e: + print(f"[FlightFinder] ⚠️ Parse failed: {e}") + return [] + + + +def _format_spoken( + flights: list[dict], + origin: str, + destination: str, + date: str, +) -> str: + """Formats flights for spoken output — concise and natural.""" + if not flights: + return ( + f"I couldn't find any flights from {origin} to {destination} " + f"on {date}, sir. The page may not have loaded correctly." + ) + + lines = [f"Here are the flights from {origin} to {destination} on {date}, sir."] + + for i, f in enumerate(flights[:5], 1): + airline = f.get("airline", "Unknown airline") + departure = f.get("departure", "--:--") + arrival = f.get("arrival", "--:--") + duration = f.get("duration", "") + stops = f.get("stops", 0) + price = f.get("price", "") + currency = f.get("currency", "") + + stop_str = "non-stop" if stops == 0 else f"{stops} stop{'s' if stops > 1 else ''}" + price_str = f"{price} {currency}".strip() if price else "price unavailable" + dur_str = f", {duration}" if duration else "" + + lines.append( + f"Option {i}: {airline}, departing {departure}, arriving {arrival}" + f"{dur_str}, {stop_str}, {price_str}." + ) + + cheapest = min( + (f for f in flights if f.get("price")), + key=lambda x: re.sub(r"[^\d]", "", str(x.get("price", "99999"))) or "99999", + default=None, + ) + if cheapest: + lines.append( + f"The cheapest option is {cheapest.get('airline')} " + f"at {cheapest.get('price')} {cheapest.get('currency', '')}." + ) + + return " ".join(lines) + + +def _format_notepad( + flights: list[dict], + origin: str, + destination: str, + date: str, + return_date: str | None, + page_url: str, +) -> str: + """Formats flights for Notepad — detailed and readable.""" + from datetime import datetime as dt + + lines = [ + "JARVIS — Flight Search Results", + "─" * 50, + f"Route : {origin} → {destination}", + f"Date : {date}", + ] + if return_date: + lines.append(f"Return : {return_date}") + lines += [ + f"Searched : {dt.now().strftime('%Y-%m-%d %H:%M')}", + f"Source : {page_url}", + "─" * 50, + "", + ] + + if not flights: + lines.append("No flights found.") + else: + for i, f in enumerate(flights, 1): + stops = f.get("stops", 0) + stop_str = "Non-stop" if stops == 0 else f"{stops} stop(s)" + lines += [ + f"Flight {i}:", + f" Airline : {f.get('airline', 'N/A')}", + f" Departure : {f.get('departure', 'N/A')}", + f" Arrival : {f.get('arrival', 'N/A')}", + f" Duration : {f.get('duration', 'N/A')}", + f" Stops : {stop_str}", + f" Price : {f.get('price', 'N/A')} {f.get('currency', '')}", + "", + ] + + return "\n".join(lines) + + +def _save_to_notepad(content: str, origin: str, destination: str) -> str: + """Saves flight results to Desktop and opens in default text editor.""" + from datetime import datetime + + ts = datetime.now().strftime("%Y%m%d_%H%M%S") + filename = f"flights_{origin}_{destination}_{ts}.txt".replace(" ", "_") + desktop = Path.home() / "Desktop" + desktop.mkdir(parents=True, exist_ok=True) + filepath = desktop / filename + + filepath.write_text(content, encoding="utf-8") + print(f"[FlightFinder] 💾 Saved: {filepath}") + + system = platform.system() + open_fn = { + "Windows": lambda p: subprocess.Popen(["notepad.exe", str(p)]), + "Darwin": lambda p: subprocess.Popen(["open", "-t", str(p)]), + "Linux": lambda p: subprocess.Popen(["xdg-open", str(p)]), + } + opener = open_fn.get(system) + if opener: + opener(filepath) + + return str(filepath) + +def flight_finder( + parameters: dict, + response=None, + player=None, + session_memory=None, + speak=None, +) -> str: + """ + Flight Finder — searches Google Flights and speaks results. + + Parameters: + origin (str, required) — departure city or airport (e.g. "Istanbul", "IST") + destination (str, required) — arrival city or airport (e.g. "London", "LHR") + date (str, required) — departure date (any format: "15 Mart", "March 15", "2025-03-15") + return_date (str, optional) — return date for round trips + passengers (int, optional) — number of passengers (default: 1) + cabin (str, optional) — economy | premium | business | first (default: economy) + save (bool, optional) — save results to Notepad (default: False) + show_browser (bool, optional) — keep browser open after search (default: True) + + Examples: + flight_finder({"origin": "Istanbul", "destination": "London", "date": "15 Mart"}) + flight_finder({"origin": "IST", "destination": "JFK", "date": "2025-04-01", + "return_date": "2025-04-15", "cabin": "business", "save": True}) + """ + params = parameters or {} + + origin = params.get("origin", "").strip() + destination = params.get("destination", "").strip() + date_raw = params.get("date", "").strip() + return_raw = params.get("return_date", "").strip() + passengers = int(params.get("passengers", 1)) + cabin = params.get("cabin", "economy").strip() + save = params.get("save", False) + + if not origin or not destination: + return "Please provide both origin and destination, sir." + if not date_raw: + return "Please provide a departure date, sir." + + date = _parse_date(date_raw) + return_date = _parse_date(return_raw) if return_raw else None + + if player: + player.write_log(f"[FlightFinder] {origin} → {destination} on {date}") + + if speak: + speak(f"Searching flights from {origin} to {destination} on {date}, sir.") + + print(f"[FlightFinder] ▶️ {origin} → {destination} | {date} | {cabin} | {passengers} pax") + + try: + + raw_text, page_url = _search_flights_browser( + origin, destination, date, return_date, passengers, cabin + ) + + if not raw_text: + return "Could not retrieve flight data, sir. The page may not have loaded." + + if speak: + speak("Analysing the results now.") + + flights = _parse_flights_with_gemini(raw_text, origin, destination, date) + + spoken = _format_spoken(flights, origin, destination, date) + if speak: + speak(spoken) + + result = spoken + + if save and flights: + notepad_content = _format_notepad( + flights, origin, destination, date, return_date, page_url + ) + saved_path = _save_to_notepad(notepad_content, origin, destination) + result += f" Results saved to Desktop: {saved_path}" + + return result + + except Exception as e: + print(f"[FlightFinder] ❌ Error: {e}") + return f"Flight search failed, sir: {e}" \ No newline at end of file diff --git a/actions/open_app.py b/actions/open_app.py new file mode 100644 index 0000000..b526817 --- /dev/null +++ b/actions/open_app.py @@ -0,0 +1,208 @@ +# actions/open_app.py +# MARK XXV — Cross-Platform App Launcher + +import time +import subprocess +import platform +import shutil + +try: + import psutil + _PSUTIL = True +except ImportError: + _PSUTIL = False + +_APP_ALIASES = { + "whatsapp": {"Windows": "WhatsApp", "Darwin": "WhatsApp", "Linux": "whatsapp"}, + "chrome": {"Windows": "chrome", "Darwin": "Google Chrome", "Linux": "google-chrome"}, + "google chrome": {"Windows": "chrome", "Darwin": "Google Chrome", "Linux": "google-chrome"}, + "firefox": {"Windows": "firefox", "Darwin": "Firefox", "Linux": "firefox"}, + "spotify": {"Windows": "Spotify", "Darwin": "Spotify", "Linux": "spotify"}, + "vscode": {"Windows": "code", "Darwin": "Visual Studio Code", "Linux": "code"}, + "visual studio code": {"Windows": "code", "Darwin": "Visual Studio Code", "Linux": "code"}, + "discord": {"Windows": "Discord", "Darwin": "Discord", "Linux": "discord"}, + "telegram": {"Windows": "Telegram", "Darwin": "Telegram", "Linux": "telegram"}, + "instagram": {"Windows": "Instagram", "Darwin": "Instagram", "Linux": "instagram"}, + "tiktok": {"Windows": "TikTok", "Darwin": "TikTok", "Linux": "tiktok"}, + "notepad": {"Windows": "notepad.exe", "Darwin": "TextEdit", "Linux": "gedit"}, + "calculator": {"Windows": "calc.exe", "Darwin": "Calculator", "Linux": "gnome-calculator"}, + "terminal": {"Windows": "cmd.exe", "Darwin": "Terminal", "Linux": "gnome-terminal"}, + "cmd": {"Windows": "cmd.exe", "Darwin": "Terminal", "Linux": "bash"}, + "explorer": {"Windows": "explorer.exe", "Darwin": "Finder", "Linux": "nautilus"}, + "file explorer": {"Windows": "explorer.exe", "Darwin": "Finder", "Linux": "nautilus"}, + "paint": {"Windows": "mspaint.exe", "Darwin": "Preview", "Linux": "gimp"}, + "word": {"Windows": "winword", "Darwin": "Microsoft Word", "Linux": "libreoffice --writer"}, + "excel": {"Windows": "excel", "Darwin": "Microsoft Excel", "Linux": "libreoffice --calc"}, + "powerpoint": {"Windows": "powerpnt", "Darwin": "Microsoft PowerPoint","Linux": "libreoffice --impress"}, + "vlc": {"Windows": "vlc", "Darwin": "VLC", "Linux": "vlc"}, + "zoom": {"Windows": "Zoom", "Darwin": "zoom.us", "Linux": "zoom"}, + "slack": {"Windows": "Slack", "Darwin": "Slack", "Linux": "slack"}, + "steam": {"Windows": "steam", "Darwin": "Steam", "Linux": "steam"}, + "task manager": {"Windows": "taskmgr.exe", "Darwin": "Activity Monitor", "Linux": "gnome-system-monitor"}, + "settings": {"Windows": "ms-settings:", "Darwin": "System Preferences", "Linux": "gnome-control-center"}, + "powershell": {"Windows": "powershell.exe", "Darwin": "Terminal", "Linux": "bash"}, + "edge": {"Windows": "msedge", "Darwin": "Microsoft Edge", "Linux": "microsoft-edge"}, + "brave": {"Windows": "brave", "Darwin": "Brave Browser", "Linux": "brave-browser"}, + "obsidian": {"Windows": "Obsidian", "Darwin": "Obsidian", "Linux": "obsidian"}, + "notion": {"Windows": "Notion", "Darwin": "Notion", "Linux": "notion"}, + "blender": {"Windows": "blender", "Darwin": "Blender", "Linux": "blender"}, + "capcut": {"Windows": "CapCut", "Darwin": "CapCut", "Linux": "capcut"}, + "postman": {"Windows": "Postman", "Darwin": "Postman", "Linux": "postman"}, + "figma": {"Windows": "Figma", "Darwin": "Figma", "Linux": "figma"}, +} + + +def _normalize(raw: str) -> str: + system = platform.system() + key = raw.lower().strip() + if key in _APP_ALIASES: + return _APP_ALIASES[key].get(system, raw) + for alias_key, os_map in _APP_ALIASES.items(): + if alias_key in key or key in alias_key: + return os_map.get(system, raw) + return raw + + +def _is_running(app_name: str) -> bool: + if not _PSUTIL: + return True + app_lower = app_name.lower().replace(" ", "").replace(".exe", "") + try: + for proc in psutil.process_iter(["name"]): + try: + proc_name = proc.info["name"].lower().replace(" ", "").replace(".exe", "") + if app_lower in proc_name or proc_name in app_lower: + return True + except (psutil.NoSuchProcess, psutil.AccessDenied): + continue + except Exception: + pass + return False + + +def _launch_windows(app_name: str) -> bool: + try: + import pyautogui + pyautogui.PAUSE = 0.1 + pyautogui.press("win") + time.sleep(0.6) + pyautogui.write(app_name, interval=0.05) + time.sleep(0.8) + pyautogui.press("enter") + time.sleep(3.0) + return True + except Exception as e: + print(f"[open_app] ⚠️ Windows launch failed: {e}") + return False + +def _launch_macos(app_name: str) -> bool: + try: + result = subprocess.run(["open", "-a", app_name], capture_output=True, timeout=8) + if result.returncode == 0: + time.sleep(1.0) + return True + except Exception: + pass + + try: + result = subprocess.run(["open", "-a", f"{app_name}.app"], capture_output=True, timeout=8) + if result.returncode == 0: + time.sleep(1.0) + return True + except Exception: + pass + + try: + import pyautogui + pyautogui.hotkey("command", "space") + time.sleep(0.6) + pyautogui.write(app_name, interval=0.05) + time.sleep(0.8) + pyautogui.press("enter") + time.sleep(1.5) + return True + except Exception as e: + print(f"[open_app] ⚠️ macOS Spotlight failed: {e}") + return False + + + +def _launch_linux(app_name: str) -> bool: + binary = ( + shutil.which(app_name) or + shutil.which(app_name.lower()) or + shutil.which(app_name.lower().replace(" ", "-")) + ) + if binary: + try: + subprocess.Popen([binary], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + time.sleep(1.0) + return True + except Exception: + pass + + try: + subprocess.run(["xdg-open", app_name], capture_output=True, timeout=5) + return True + except Exception: + pass + + try: + desktop_name = app_name.lower().replace(" ", "-") + subprocess.run(["gtk-launch", desktop_name], capture_output=True, timeout=5) + return True + except Exception: + pass + + return False + + +_OS_LAUNCHERS = { + "Windows": _launch_windows, + "Darwin": _launch_macos, + "Linux": _launch_linux, +} + + +def open_app( + parameters=None, + response=None, + player=None, + session_memory=None, +) -> str: + app_name = (parameters or {}).get("app_name", "").strip() + + if not app_name: + return "Please specify which application to open, sir." + + system = platform.system() + launcher = _OS_LAUNCHERS.get(system) + + if launcher is None: + return f"Unsupported OS: {system}" + + normalized = _normalize(app_name) + print(f"[open_app] 🚀 Launching: {app_name} → {normalized} ({system})") + + if player: + player.write_log(f"[open_app] {app_name}") + + try: + success = launcher(normalized) + + if success: + return f"Opened {app_name} successfully, sir." + + if normalized != app_name: + success = launcher(app_name) + if success: + return f"Opened {app_name} successfully, sir." + + return ( + f"I tried to open {app_name}, sir, but couldn't confirm it launched. " + f"It may still be loading or might not be installed." + ) + + except Exception as e: + print(f"[open_app] ❌ {e}") + return f"Failed to open {app_name}, sir: {e}" \ No newline at end of file diff --git a/actions/reminder.py b/actions/reminder.py new file mode 100644 index 0000000..330805d --- /dev/null +++ b/actions/reminder.py @@ -0,0 +1,156 @@ +# actions/reminder.py + +import subprocess +import os +import sys +from datetime import datetime + + +def reminder( + parameters: dict, + response: str | None = None, + player=None, + session_memory=None +) -> str: + """ + Sets a timed reminder using Windows Task Scheduler. + + parameters: + - date (str) YYYY-MM-DD + - time (str) HH:MM + - message (str) + + Returns a result string — Live API voices it automatically. + No edge_speak needed. + """ + + date_str = parameters.get("date") + time_str = parameters.get("time") + message = parameters.get("message", "Reminder") + + if not date_str or not time_str: + return "I need both a date and a time to set a reminder." + + try: + target_dt = datetime.strptime(f"{date_str} {time_str}", "%Y-%m-%d %H:%M") + + if target_dt <= datetime.now(): + return "That time is already in the past." + + task_name = f"MARKReminder_{target_dt.strftime('%Y%m%d_%H%M')}" + safe_message = message.replace('"', '').replace("'", "").strip()[:200] + + python_exe = sys.executable + if python_exe.lower().endswith("python.exe"): + pythonw = python_exe.replace("python.exe", "pythonw.exe") + if os.path.exists(pythonw): + python_exe = pythonw + + temp_dir = os.environ.get("TEMP", "C:\\Temp") + notify_script = os.path.join(temp_dir, f"{task_name}.pyw") + project_root = os.path.abspath( + os.path.join(os.path.dirname(__file__), "..") + ) + + script_code = f'''import sys, os, time +sys.path.insert(0, r"{project_root}") + +try: + import winsound + for freq in [800, 1000, 1200]: + winsound.Beep(freq, 200) + time.sleep(0.1) +except Exception: + pass + +try: + from win10toast import ToastNotifier + ToastNotifier().show_toast( + "MARK Reminder", + "{safe_message}", + duration=15, + threaded=False + ) +except Exception: + try: + import subprocess + subprocess.run(["msg", "*", "/TIME:30", "{safe_message}"], shell=True) + except Exception: + pass + +time.sleep(3) +try: + os.remove(__file__) +except Exception: + pass +''' + with open(notify_script, "w", encoding="utf-8") as f: + f.write(script_code) + + xml_content = f''' + + + MARK Reminder: {safe_message} + + + + {target_dt.strftime("%Y-%m-%dT%H:%M:%S")} + true + + + + + {python_exe} + "{notify_script}" + + + + IgnoreNew + false + false + true + true + PT5M + true + + + + InteractiveToken + LeastPrivilege + + +''' + + xml_path = os.path.join(temp_dir, f"{task_name}.xml") + with open(xml_path, "w", encoding="utf-16") as f: + f.write(xml_content) + + result = subprocess.run( + f'schtasks /Create /TN "{task_name}" /XML "{xml_path}" /F', + shell=True, capture_output=True, text=True + ) + + try: + os.remove(xml_path) + except Exception: + pass + + if result.returncode != 0: + err = result.stderr.strip() or result.stdout.strip() + print(f"[Reminder] ❌ schtasks failed: {err}") + try: + os.remove(notify_script) + except Exception: + pass + return "I couldn't schedule the reminder due to a system error." + + if player: + player.write_log(f"[reminder] set for {date_str} {time_str}") + + return f"Reminder set for {target_dt.strftime('%B %d at %I:%M %p')}." + + except ValueError: + return "I couldn't understand that date or time format." + + except Exception as e: + return f"Something went wrong while scheduling the reminder: {str(e)[:80]}" \ No newline at end of file diff --git a/actions/screen_processor.py b/actions/screen_processor.py new file mode 100644 index 0000000..49fdf49 --- /dev/null +++ b/actions/screen_processor.py @@ -0,0 +1,404 @@ +""" +actions/screen_processor.py — Gemini Live API — IMAGE-ONLY SESSION v8 +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +v8 Changes: + - Auto camera detection on first use — saves to config + - No hardcoded camera index + - mic_loop removed (no double response issue) + - Image-only session, no conflict with main.py +""" + +import asyncio +import base64 +import io +import json +import re +import os +import sys +import time +import threading +import cv2 +import mss +import mss.tools +import pyaudio +from pathlib import Path + +try: + import PIL.Image + _PIL_OK = True +except ImportError: + _PIL_OK = False + +from google import genai +from google.genai import types + +def get_base_dir(): + if getattr(sys, "frozen", False): + return Path(sys.executable).parent + return Path(__file__).resolve().parent.parent + +BASE_DIR = get_base_dir() +API_CONFIG_PATH = BASE_DIR / "config" / "api_keys.json" + +LIVE_MODEL = "models/gemini-2.5-flash-native-audio-preview-12-2025" +FORMAT = pyaudio.paInt16 +CHANNELS = 1 +RECEIVE_SAMPLE_RATE = 24000 +CHUNK_SIZE = 1024 + +IMG_MAX_W = 640 +IMG_MAX_H = 360 +JPEG_Q = 55 + +SYSTEM_PROMPT = ( + "You are JARVIS from Iron Man movies. " + "Analyze images with technical precision and intelligence. " + "Help the user in a way they can understand — don't be overly complex. " + "Be concise, smart, and helpful like Tony Stark's AI assistant. " + "Respond in maximum 2 short sentences. Speed is priority. " + "Address the user as 'sir' for a tone of respect. " + "Ask if the user needs any further help with their problem." +) + + +def _get_api_key() -> str: + try: + with open(API_CONFIG_PATH, "r", encoding="utf-8") as f: + keys = json.load(f) + key = keys.get("gemini_api_key", "") + if not key: + raise ValueError("gemini_api_key not found") + return key + except Exception as e: + raise RuntimeError(f"Could not load API key: {e}") + + +def _get_camera_index() -> int: + """ + Reads camera index from config. + If not set, auto-detects the best camera and saves it for future use. + Runs only once — after that, config value is used directly. + """ + try: + with open(API_CONFIG_PATH, "r", encoding="utf-8") as f: + cfg = json.load(f) + if "camera_index" in cfg: + return int(cfg["camera_index"]) + except Exception: + pass + + print("[Camera] 🔍 No camera index in config. Auto-detecting...") + best_index = 0 + + for idx in range(6): + cap = cv2.VideoCapture(idx, cv2.CAP_DSHOW) + if not cap.isOpened(): + cap.release() + continue + + for _ in range(5): + cap.read() + + ret, frame = cap.read() + cap.release() + + if ret and frame is not None and frame.mean() > 5: + best_index = idx + print(f"[Camera] ✅ Camera found at index {idx} — saving to config.") + break + else: + print(f"[Camera] ⚠️ Index {idx}: no valid frame (black or empty).") + + try: + cfg = {} + if API_CONFIG_PATH.exists(): + with open(API_CONFIG_PATH, "r", encoding="utf-8") as f: + cfg = json.load(f) + cfg["camera_index"] = best_index + with open(API_CONFIG_PATH, "w", encoding="utf-8") as f: + json.dump(cfg, f, indent=4) + print(f"[Camera] 💾 Camera index {best_index} saved to config.") + except Exception as e: + print(f"[Camera] ⚠️ Could not save camera index: {e}") + + return best_index + + +def _to_jpeg(img_bytes: bytes) -> bytes: + if not _PIL_OK: + return img_bytes + img = PIL.Image.open(io.BytesIO(img_bytes)).convert("RGB") + img.thumbnail([IMG_MAX_W, IMG_MAX_H], PIL.Image.BILINEAR) + buf = io.BytesIO() + img.save(buf, format="JPEG", quality=JPEG_Q, optimize=False) + return buf.getvalue() + + +def _capture_screenshot() -> bytes: + with mss.mss() as sct: + shot = sct.grab(sct.monitors[1]) + png_bytes = mss.tools.to_png(shot.rgb, shot.size) + return _to_jpeg(png_bytes) + + +def _capture_camera() -> bytes: + camera_index = _get_camera_index() + cap = cv2.VideoCapture(camera_index, cv2.CAP_DSHOW) + if not cap.isOpened(): + raise RuntimeError(f"Camera could not be opened: index {camera_index}") + + for _ in range(10): + cap.read() + + ret, frame = cap.read() + cap.release() + + if not ret or frame is None: + raise RuntimeError("Could not capture camera frame.") + + if _PIL_OK: + rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) + img = PIL.Image.fromarray(rgb) + img.thumbnail([IMG_MAX_W, IMG_MAX_H], PIL.Image.BILINEAR) + buf = io.BytesIO() + img.save(buf, format="JPEG", quality=JPEG_Q, optimize=False) + return buf.getvalue() + + _, buf = cv2.imencode(".jpg", frame, [cv2.IMWRITE_JPEG_QUALITY, JPEG_Q]) + return buf.tobytes() + + +class _LiveSession: + """ + Image-only analysis session. + No microphone — no conflict with main.py session. + Sends image + question, plays audio response. + """ + + def __init__(self): + self._loop: asyncio.AbstractEventLoop | None = None + self._thread: threading.Thread | None = None + self._session = None + self._out_queue: asyncio.Queue | None = None + self._audio_in: asyncio.Queue | None = None + self._ready: threading.Event = threading.Event() + self._player = None + self._pya = pyaudio.PyAudio() + self._send_lock: asyncio.Lock | None = None + + def start(self, player=None): + if self._thread and self._thread.is_alive(): + return + self._player = player + self._thread = threading.Thread( + target=self._run_loop, daemon=True, name="VisionSessionThread" + ) + self._thread.start() + ok = self._ready.wait(timeout=20) + if not ok: + raise RuntimeError("Vision session did not start within 20s.") + print("[ScreenProcess] ✅ Vision session ready (no mic)") + + def _run_loop(self): + self._loop = asyncio.new_event_loop() + asyncio.set_event_loop(self._loop) + self._loop.run_until_complete(self._main()) + + async def _main(self): + self._out_queue = asyncio.Queue(maxsize=30) + self._audio_in = asyncio.Queue() + self._send_lock = asyncio.Lock() + + client = genai.Client( + api_key=_get_api_key(), + http_options={"api_version": "v1beta"} + ) + + config = types.LiveConnectConfig( + response_modalities=["AUDIO"], + output_audio_transcription={}, + system_instruction=SYSTEM_PROMPT, + speech_config=types.SpeechConfig( + voice_config=types.VoiceConfig( + prebuilt_voice_config=types.PrebuiltVoiceConfig( + voice_name="Charon" + ) + ) + ), + ) + + while True: + try: + print("[ScreenProcess] 🔌 Vision session connecting...") + async with client.aio.live.connect(model=LIVE_MODEL, config=config) as session: + self._session = session + self._ready.set() + print("[ScreenProcess] ✅ Vision session connected") + + async with asyncio.TaskGroup() as tg: + tg.create_task(self._send_loop()) + tg.create_task(self._recv_loop()) + tg.create_task(self._play_loop()) + + except Exception as e: + print(f"[ScreenProcess] ⚠️ Disconnected: {e} — reconnecting...") + self._session = None + self._ready.clear() + await asyncio.sleep(2) + self._ready.set() + + # ── Send loop ── + + async def _send_loop(self): + """Sends (image_bytes, mime_type, user_text) tuples from queue.""" + while True: + item = await self._out_queue.get() + if self._session: + image_bytes, mime_type, user_text = item + try: + b64 = base64.b64encode(image_bytes).decode("utf-8") + await self._session.send_client_content( + turns={ + "parts": [ + {"inline_data": {"mime_type": mime_type, "data": b64}}, + {"text": user_text} + ] + }, + turn_complete=True + ) + print("[ScreenProcess] ✅ Image sent") + except Exception as e: + print(f"[ScreenProcess] ⚠️ Send error: {e}") + + async def _recv_loop(self): + transcript_buf: list[str] = [] + try: + async for response in self._session.receive(): + + if response.data: + await self._audio_in.put(response.data) + + sc = response.server_content + if not sc: + continue + + if sc.output_transcription and sc.output_transcription.text: + chunk = sc.output_transcription.text.strip() + if chunk: + transcript_buf.append(chunk) + + if sc.turn_complete: + if transcript_buf and self._player: + full = re.sub(r'\s+', ' ', " ".join(transcript_buf)).strip() + if full: + self._player.write_log(f"Jarvis: {full}") + print(f"[ScreenProcess] 💬 {full}") + transcript_buf = [] + + except Exception as e: + print(f"[ScreenProcess] ⚠️ Recv error: {e}") + transcript_buf = [] + await asyncio.sleep(0.3) + + async def _play_loop(self): + stream = await asyncio.to_thread( + self._pya.open, + format=FORMAT, channels=CHANNELS, + rate=RECEIVE_SAMPLE_RATE, output=True, + ) + try: + while True: + chunk = await self._audio_in.get() + await asyncio.to_thread(stream.write, chunk) + finally: + stream.close() + + def analyze(self, image_bytes: bytes, mime_type: str, user_text: str): + """Called from main thread — puts image into async queue.""" + if not self._loop: + return + asyncio.run_coroutine_threadsafe( + self._out_queue.put((image_bytes, mime_type, user_text)), + self._loop + ) + + def is_ready(self) -> bool: + return self._session is not None + +_live = _LiveSession() +_started = False +_start_lock = threading.Lock() + + +def _ensure_started(player=None): + global _started + with _start_lock: + if not _started: + _live.start(player=player) + _started = True + elif player is not None: + _live._player = player + +def screen_process( + parameters: dict, + response: str | None = None, + player=None, + session_memory=None, +) -> bool: + user_text = (parameters or {}).get("text") or (parameters or {}).get("user_text", "") + user_text = (user_text or "").strip() + if not user_text: + print("[ScreenProcess] ⚠️ No user_text provided.") + return False + + angle = (parameters or {}).get("angle", "screen").lower().strip() + print(f"[ScreenProcess] angle={angle!r} text={user_text!r}") + + _ensure_started(player=player) + + try: + if angle == "camera": + image_bytes = _capture_camera() + mime_type = "image/jpeg" + print("[ScreenProcess] 📷 Camera captured") + else: + image_bytes = _capture_screenshot() + mime_type = "image/jpeg" if _PIL_OK else "image/png" + print("[ScreenProcess] 🖥️ Screen captured") + except Exception as e: + import traceback; traceback.print_exc() + print(f"[ScreenProcess] ❌ Capture error: {e}") + return False + + print(f"[ScreenProcess] 📦 {len(image_bytes)} bytes → sending") + _live.analyze(image_bytes, mime_type, user_text) + return True + + +def warmup_session(player=None): + """ + Optional: pre-warm the session. + Do NOT call from main.py — causes double session issue. + Only use when testing screen_processor.py standalone. + """ + try: + _ensure_started(player=player) + except Exception as e: + print(f"[ScreenProcess] ⚠️ Warmup error: {e}") + + +if __name__ == "__main__": + print("[TEST] screen_processor.py v8 — image-only session") + print("=" * 50) + mode = input("screen / camera (default: screen): ").strip().lower() or "screen" + request = input("Question (Enter for default): ").strip() or "What do you see? Be brief." + + t0 = time.perf_counter() + warmup_session() + print(f"Session ready — {time.perf_counter()-t0:.2f}s\n") + + t1 = time.perf_counter() + result = screen_process({"angle": mode, "text": request}, player=None) + print(f"Sent — {time.perf_counter()-t1:.3f}s | audio incoming...") + time.sleep(8) + print(f"\n{'✅' if result else '❌'}") \ No newline at end of file diff --git a/actions/send_message.py b/actions/send_message.py new file mode 100644 index 0000000..1fd9bf6 --- /dev/null +++ b/actions/send_message.py @@ -0,0 +1,215 @@ +# actions/send_message.py +# Universal messaging — WhatsApp & Instagram +# Uses visual element detection (pyautogui + screen search) instead of +# hardcoded tab/click sequences — works on any screen resolution. + +import time +import pyautogui +from pathlib import Path + +pyautogui.FAILSAFE = True +pyautogui.PAUSE = 0.08 + +def _open_app(app_name: str) -> bool: + """Opens an app via Windows search.""" + try: + pyautogui.press("win") + time.sleep(0.4) + pyautogui.write(app_name, interval=0.04) + time.sleep(0.5) + pyautogui.press("enter") + time.sleep(2.0) + return True + except Exception as e: + print(f"[SendMessage] Could not open {app_name}: {e}") + return False + + +def _search_contact(contact: str, platform: str): + """ + Searches for a contact inside the messaging app. + Uses Ctrl+F (universal search shortcut) then types contact name. + """ + time.sleep(0.5) + pyautogui.hotkey("ctrl", "f") + time.sleep(0.4) + pyautogui.hotkey("ctrl", "a") + pyautogui.write(contact, interval=0.04) + time.sleep(0.8) + pyautogui.press("enter") + time.sleep(0.6) + + +def _type_and_send(message: str): + """Types message and sends it.""" + pyautogui.press("tab") + time.sleep(0.2) + pyautogui.hotkey("ctrl", "a") + pyautogui.write(message, interval=0.03) + time.sleep(0.2) + pyautogui.press("enter") + time.sleep(0.3) + + +def _send_whatsapp(receiver: str, message: str) -> str: + """ + Sends a WhatsApp message via the Windows desktop app. + Steps: Open WhatsApp → Search contact → Click → Type → Send + """ + try: + if not _open_app("WhatsApp"): + return "Could not open WhatsApp." + + time.sleep(1.5) + + pyautogui.hotkey("ctrl", "f") + time.sleep(0.4) + pyautogui.hotkey("ctrl", "a") + pyautogui.write(receiver, interval=0.04) + time.sleep(1.0) + + pyautogui.press("enter") + time.sleep(0.8) + + pyautogui.write(message, interval=0.03) + time.sleep(0.2) + pyautogui.press("enter") + + return f"Message sent to {receiver} via WhatsApp." + + except Exception as e: + return f"WhatsApp error: {e}" + + +def _send_instagram(receiver: str, message: str) -> str: + """ + Sends an Instagram DM via browser (instagram.com). + Steps: Open Chrome → Go to instagram.com/direct → Search contact → Send + """ + try: + import webbrowser + + webbrowser.open("https://www.instagram.com/direct/new/") + time.sleep(3.5) + + pyautogui.write(receiver, interval=0.05) + time.sleep(1.5) + + pyautogui.press("down") + time.sleep(0.3) + pyautogui.press("enter") + time.sleep(0.5) + + for _ in range(3): + pyautogui.press("tab") + time.sleep(0.1) + pyautogui.press("enter") + time.sleep(1.5) + + pyautogui.write(message, interval=0.04) + time.sleep(0.2) + pyautogui.press("enter") + + return f"Message sent to {receiver} via Instagram." + + except Exception as e: + return f"Instagram error: {e}" + +def _send_telegram(receiver: str, message: str) -> str: + """Sends a Telegram message via Windows desktop app.""" + try: + if not _open_app("Telegram"): + return "Could not open Telegram." + + time.sleep(1.5) + + pyautogui.hotkey("ctrl", "f") + time.sleep(0.4) + pyautogui.write(receiver, interval=0.04) + time.sleep(1.0) + pyautogui.press("enter") + time.sleep(0.8) + + pyautogui.write(message, interval=0.03) + time.sleep(0.2) + pyautogui.press("enter") + + return f"Message sent to {receiver} via Telegram." + + except Exception as e: + return f"Telegram error: {e}" + + + +def _send_generic(platform: str, receiver: str, message: str) -> str: + """ + For any other platform not explicitly supported. + Opens the app, searches for contact, types and sends. + Works for: Messenger, Discord, Signal, etc. + """ + try: + if not _open_app(platform): + return f"Could not open {platform}." + + time.sleep(1.5) + pyautogui.hotkey("ctrl", "f") + time.sleep(0.4) + pyautogui.write(receiver, interval=0.04) + time.sleep(1.0) + pyautogui.press("enter") + time.sleep(0.8) + pyautogui.write(message, interval=0.03) + time.sleep(0.2) + pyautogui.press("enter") + + return f"Message sent to {receiver} via {platform}." + + except Exception as e: + return f"{platform} error: {e}" + +def send_message( + parameters: dict, + response=None, + player=None, + session_memory=None +) -> str: + """ + Called from main.py. + + parameters: + receiver : Contact name to send to + message_text : The message content + platform : whatsapp | instagram | telegram | + Default: whatsapp + """ + params = parameters or {} + receiver = params.get("receiver", "").strip() + message_text = params.get("message_text", "").strip() + platform = params.get("platform", "whatsapp").strip().lower() + + if not receiver: + return "Please specify who to send the message to, sir." + if not message_text: + return "Please specify what message to send, sir." + + print(f"[SendMessage] 📨 {platform} → {receiver}: {message_text[:40]}") + if player: + player.write_log(f"[msg] Sending to {receiver} via {platform}...") + + if "whatsapp" in platform or "wp" in platform or "wapp" in platform: + result = _send_whatsapp(receiver, message_text) + + elif "instagram" in platform or "ig" in platform or "insta" in platform: + result = _send_instagram(receiver, message_text) + + elif "telegram" in platform or "tg" in platform: + result = _send_telegram(receiver, message_text) + + else: + result = _send_generic(platform, receiver, message_text) + + print(f"[SendMessage] ✅ {result}") + if player: + player.write_log(f"[msg] {result}") + + return result \ No newline at end of file diff --git a/actions/weather_report.py b/actions/weather_report.py new file mode 100644 index 0000000..2ac70a2 --- /dev/null +++ b/actions/weather_report.py @@ -0,0 +1,62 @@ +# actions/weather_report.py + +import webbrowser +from urllib.parse import quote_plus + + +def weather_action( + parameters: dict, + player=None, + session_memory=None +): + """ + Weather report action. + Opens a Google weather search and gives a short spoken confirmation. + """ + + city = parameters.get("city") + time = parameters.get("time") + if not city or not isinstance(city, str): + msg = "Sir, the city is missing for the weather report." + _speak_and_log(msg, player) + return msg + + city = city.strip() + + if not time or not isinstance(time, str): + time = "today" + else: + time = time.strip() + + search_query = f"weather in {city} {time}" + encoded_query = quote_plus(search_query) + url = f"https://www.google.com/search?q={encoded_query}" + + try: + webbrowser.open(url) + except Exception: + msg = f"Sir, I couldn't open the browser for the weather report." + _speak_and_log(msg, player) + return msg + + msg = f"Showing the weather for {city}, {time}, sir." + _speak_and_log(msg, player) + + if session_memory: + try: + session_memory.set_last_search( + query=search_query, + response=msg + ) + except Exception: + pass + + return msg + + +def _speak_and_log(message: str, player=None): + if player: + try: + player.write_log(f"JARVIS: {message}") + except Exception: + pass \ No newline at end of file diff --git a/actions/web_search.py b/actions/web_search.py new file mode 100644 index 0000000..4fb67e9 --- /dev/null +++ b/actions/web_search.py @@ -0,0 +1,135 @@ +# actions/web_search.py +# MARK XXV — Web Search +# Primary: Gemini google_search (yeni google.genai SDK) +# Fallback: DuckDuckGo (ddgs) + +import json +import sys +from pathlib import Path + + +def get_base_dir() -> Path: + if getattr(sys, "frozen", False): + return Path(sys.executable).parent + return Path(__file__).resolve().parent.parent + +BASE_DIR = get_base_dir() +API_CONFIG_PATH = BASE_DIR / "config" / "api_keys.json" + +def _get_api_key() -> str: + with open(API_CONFIG_PATH, "r", encoding="utf-8") as f: + return json.load(f)["gemini_api_key"] + + +def _gemini_search(query: str) -> str: + from google import genai + + client = genai.Client(api_key=_get_api_key()) + response = client.models.generate_content( + model="gemini-2.5-flash-lite", + contents=query, + config={"tools": [{"google_search": {}}]} + ) + text = "" + for part in response.candidates[0].content.parts: + if hasattr(part, "text") and part.text: + text += part.text + if not text.strip(): + raise ValueError("Empty response") + return text.strip() + + + +def _ddg_search(query: str, max_results: int = 6) -> list: + try: + from ddgs import DDGS + except ImportError: + from duckduckgo_search import DDGS + results = [] + with DDGS() as ddgs: + for r in ddgs.text(query, max_results=max_results): + results.append({ + "title": r.get("title", ""), + "snippet": r.get("body", ""), + "url": r.get("href", ""), + }) + return results + +def _format_ddg(query: str, results: list) -> str: + if not results: + return f"No results found for: {query}" + lines = [f"Search results for: {query}\n"] + for i, r in enumerate(results, 1): + if r.get("title"): lines.append(f"{i}. {r['title']}") + if r.get("snippet"): lines.append(f" {r['snippet']}") + if r.get("url"): lines.append(f" {r['url']}") + lines.append("") + return "\n".join(lines).strip() + + +def _compare(items: list, aspect: str) -> str: + query = f"Compare {', '.join(items)} in terms of {aspect}. Give specific facts and data." + try: + return _gemini_search(query) + except Exception as e: + print(f"[WebSearch] ⚠️ Gemini compare failed: {e}") + all_results = {} + for item in items: + try: + all_results[item] = _ddg_search(f"{item} {aspect}", max_results=3) + except Exception: + all_results[item] = [] + lines = [f"Comparison — {aspect.upper()}\n{'─'*40}"] + for item in items: + lines.append(f"\n▸ {item}") + for r in all_results.get(item, [])[:2]: + if r.get("snippet"): + lines.append(f" • {r['snippet']}") + return "\n".join(lines) + + +def web_search( + parameters: dict, + response=None, + player=None, + session_memory=None, +) -> str: + params = parameters or {} + query = params.get("query", "").strip() + mode = params.get("mode", "search").lower() + items = params.get("items", []) + aspect = params.get("aspect", "general") + + if not query and not items: + return "Please provide a search query, sir." + + if items and mode != "compare": + mode = "compare" + + if player: + player.write_log(f"[Search] {query or ', '.join(items)}") + + print(f"[WebSearch] 🔍 Query: {query!r} Mode: {mode}") + + try: + if mode == "compare" and items: + print(f"[WebSearch] 📊 Comparing: {items}") + result = _compare(items, aspect) + print("[WebSearch] ✅ Compare done.") + return result + + print("[WebSearch] 🌐 Gemini search...") + try: + result = _gemini_search(query) + print("[WebSearch] ✅ Gemini OK.") + return result + except Exception as e: + print(f"[WebSearch] ⚠️ Gemini failed ({e}), trying DDG...") + results = _ddg_search(query) + result = _format_ddg(query, results) + print(f"[WebSearch] ✅ DDG: {len(results)} results.") + return result + + except Exception as e: + print(f"[WebSearch] ❌ Failed: {e}") + return f"Search failed, sir: {e}" \ No newline at end of file diff --git a/actions/youtube_video.py b/actions/youtube_video.py new file mode 100644 index 0000000..5db42bd --- /dev/null +++ b/actions/youtube_video.py @@ -0,0 +1,517 @@ +# actions/youtube_video.py +# MARK XXV — YouTube Controller +# +# Features: +# - play : Search and play a YouTube video (original feature, preserved) +# - summarize : Get transcript → Gemini summary → speak + optional Notepad +# - get_info : Video title, channel, views, duration (scraping) +# - trending : Fetch trending videos for a region + +import json +import re +import sys +import time +import threading +from pathlib import Path + +import pyautogui +import numpy as np +import cv2 +from PIL import ImageGrab + +try: + import requests + from bs4 import BeautifulSoup + _REQUESTS_OK = True +except ImportError: + _REQUESTS_OK = False + +try: + from youtube_transcript_api import YouTubeTranscriptApi + _TRANSCRIPT_OK = True +except ImportError: + _TRANSCRIPT_OK = False + + +def get_base_dir() -> Path: + if getattr(sys, "frozen", False): + return Path(sys.executable).parent + return Path(__file__).resolve().parent.parent + + +BASE_DIR = get_base_dir() +API_CONFIG_PATH = BASE_DIR / "config" / "api_keys.json" + +HEADERS = { + "User-Agent": ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + "AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/120.0.0.0 Safari/537.36" + ), + "Accept-Language": "en-US,en;q=0.9", +} + + +def _get_api_key() -> str: + with open(API_CONFIG_PATH, "r", encoding="utf-8") as f: + return json.load(f)["gemini_api_key"] + + +def open_browser(): + pyautogui.press("win") + time.sleep(0.4) + pyautogui.write("browser", interval=0.05) + time.sleep(0.3) + pyautogui.press("enter") + time.sleep(2.0) + + +def find_video_thumbnails() -> list[tuple[int, int]]: + + try: + screenshot = ImageGrab.grab() + img = np.array(screenshot) + screen_h, screen_w = img.shape[:2] + + roi_top = int(screen_h * 0.10) + roi_bottom = int(screen_h * 0.75) + roi_left = int(screen_w * 0.20) + roi_right = int(screen_w * 0.80) + roi = img[roi_top:roi_bottom, roi_left:roi_right] + + gray = cv2.cvtColor(roi, cv2.COLOR_RGB2GRAY) + edges = cv2.Canny(gray, 30, 100) + kernel = np.ones((3, 3), np.uint8) + edges = cv2.dilate(edges, kernel, iterations=2) + + contours, _ = cv2.findContours( + edges, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE + ) + + candidates = [] + for c in contours: + x, y, w, h = cv2.boundingRect(c) + area = w * h + ratio = w / h if h > 0 else 0 + if area < 15000: + continue + if not (1.4 < ratio < 2.2): + continue + center_x = x + w // 2 + roi_left + center_y = y + h // 2 + roi_top + candidates.append((center_x, center_y, area)) + + filtered = [] + for cx, cy, area in sorted(candidates, key=lambda c: c[1]): + if not any(abs(cx - fx) < 80 and abs(cy - fy) < 80 for fx, fy in filtered): + filtered.append((cx, cy)) + + return filtered + + except Exception as e: + print(f"[YouTube] ⚠️ Thumbnail detection failed: {e}") + return [] + + +def _extract_video_id(url: str) -> str | None: + patterns = [ + r"(?:v=|\/v\/|youtu\.be\/|\/embed\/|\/shorts\/)([A-Za-z0-9_-]{11})", + ] + for pattern in patterns: + match = re.search(pattern, url) + if match: + return match.group(1) + return None + + +def _ask_for_url(prompt_text: str = "YouTube video URL:") -> str | None: + + try: + import tkinter as tk + from tkinter import simpledialog + + root = tk._default_root + if root is None: + root = tk.Tk() + root.withdraw() + + url = simpledialog.askstring( + "J.A.R.V.I.S", + prompt_text, + parent=root + ) + return url.strip() if url else None + except Exception as e: + print(f"[YouTube] ⚠️ URL dialog failed: {e}") + return None + + +def _is_valid_youtube_url(url: str) -> bool: + return bool(re.search(r"(youtube\.com|youtu\.be)", url or "")) + +def _get_transcript(video_id: str) -> str | None: + + if not _TRANSCRIPT_OK: + print("[YouTube] ⚠️ youtube-transcript-api not installed.") + return None + + try: + transcript_list = YouTubeTranscriptApi.list_transcripts(video_id) + + transcript = None + try: + transcript = transcript_list.find_manually_created_transcript( + ["en", "tr", "de", "fr", "es", "it", "pt", "ru", "ja", "ko", "ar", "zh"] + ) + except Exception: + pass + + if transcript is None: + try: + transcript = transcript_list.find_generated_transcript( + ["en", "tr", "de", "fr", "es", "it", "pt", "ru", "ja", "ko", "ar", "zh"] + ) + except Exception: + for t in transcript_list: + transcript = t + break + + if transcript is None: + return None + + fetched = transcript.fetch() + text = " ".join(entry["text"] for entry in fetched) + print(f"[YouTube] 📝 Transcript: {len(text)} chars") + return text + + except Exception as e: + print(f"[YouTube] ⚠️ Transcript fetch failed: {e}") + return None + +def _summarize_with_gemini(transcript: str, video_url: str) -> str: + import google.generativeai as genai + + genai.configure(api_key=_get_api_key()) + model = genai.GenerativeModel( + model_name="gemini-3.0-flash", + system_instruction=( + "You are JARVIS, Tony Stark's AI assistant. " + "Summarize YouTube video transcripts clearly and concisely. " + "Structure: 1-sentence overview, then 3-5 key points. " + "Be direct. Address the user as 'sir'. " + "Match the language of the transcript." + ) + ) + + max_chars = 80000 + truncated = transcript[:max_chars] + ("..." if len(transcript) > max_chars else "") + + response = model.generate_content( + f"Please summarize this YouTube video transcript:\n\n{truncated}" + ) + return response.text.strip() + +def _save_to_notepad(content: str, video_url: str) -> str: + import subprocess + import platform + from datetime import datetime + + ts = datetime.now().strftime("%Y%m%d_%H%M%S") + filename = f"youtube_summary_{ts}.txt" + desktop = Path.home() / "Desktop" + desktop.mkdir(parents=True, exist_ok=True) + filepath = desktop / filename + + header = ( + f"JARVIS — YouTube Summary\n" + f"{'─' * 50}\n" + f"URL : {video_url}\n" + f"Date : {datetime.now().strftime('%Y-%m-%d %H:%M')}\n" + f"{'─' * 50}\n\n" + ) + + filepath.write_text(header + content, encoding="utf-8") + print(f"[YouTube] 💾 Summary saved: {filepath}") + + system = platform.system() + open_fn = { + "Windows": lambda p: subprocess.Popen(["notepad.exe", str(p)]), + "Darwin": lambda p: subprocess.Popen(["open", "-t", str(p)]), + "Linux": lambda p: subprocess.Popen(["xdg-open", str(p)]), + } + opener = open_fn.get(system) + if opener: + opener(filepath) + + return str(filepath) + +def _scrape_video_info(video_id: str) -> dict: + + if not _REQUESTS_OK: + return {} + + url = f"https://www.youtube.com/watch?v={video_id}" + try: + r = requests.get(url, headers=HEADERS, timeout=12) + html = r.text + + info = {} + + title_match = re.search(r'"title":\{"runs":\[\{"text":"([^"]+)"', html) + if title_match: + info["title"] = title_match.group(1) + + channel_match = re.search(r'"ownerChannelName":"([^"]+)"', html) + if channel_match: + info["channel"] = channel_match.group(1) + + views_match = re.search(r'"viewCount":"(\d+)"', html) + if views_match: + views = int(views_match.group(1)) + info["views"] = f"{views:,}" + + duration_match = re.search(r'"lengthSeconds":"(\d+)"', html) + if duration_match: + secs = int(duration_match.group(1)) + info["duration"] = f"{secs // 60}:{secs % 60:02d}" + + likes_match = re.search(r'"label":"([0-9,]+ likes)"', html) + if likes_match: + info["likes"] = likes_match.group(1) + + return info + + except Exception as e: + print(f"[YouTube] ⚠️ Info scrape failed: {e}") + return {} + +def _scrape_trending(region: str = "TR", max_results: int = 8) -> list[dict]: + + if not _REQUESTS_OK: + return [] + + url = f"https://www.youtube.com/feed/trending?gl={region.upper()}" + try: + r = requests.get(url, headers=HEADERS, timeout=12) + html = r.text + + titles = re.findall(r'"title":\{"runs":\[\{"text":"([^"]+)"\}\]', html) + channels = re.findall(r'"ownerText":\{"runs":\[\{"text":"([^"]+)"', html) + + results = [] + seen = set() + for i, title in enumerate(titles): + if title in seen or len(title) < 5: + continue + seen.add(title) + channel = channels[i] if i < len(channels) else "Unknown" + results.append({"rank": len(results) + 1, "title": title, "channel": channel}) + if len(results) >= max_results: + break + + return results + + except Exception as e: + print(f"[YouTube] ⚠️ Trending scrape failed: {e}") + return [] + +def _handle_play(parameters: dict, player) -> str: + query = parameters.get("query", "").strip() + if not query: + return "Please tell me what you'd like to watch, sir." + + if player: + player.write_log(f"[YouTube] Searching: {query}") + + open_browser() + + search_query = query.replace(" ", "+") + url = f"https://www.youtube.com/results?search_query={search_query}" + + pyautogui.hotkey("ctrl", "l") + time.sleep(0.3) + pyautogui.write(url, interval=0.02) + pyautogui.press("enter") + time.sleep(3.5) + + thumbnails = find_video_thumbnails() + + if len(thumbnails) >= 2: + x, y = thumbnails[1] + print(f"[YouTube] 🎯 Clicking 2nd thumbnail at ({x}, {y})") + pyautogui.click(x, y) + return f"Playing YouTube video for: {query}" + + elif len(thumbnails) == 1: + x, y = thumbnails[0] + print(f"[YouTube] ⚠️ One thumbnail found, clicking at ({x}, {y})") + pyautogui.click(x, y) + return f"Playing YouTube video for: {query}" + + else: + print("[YouTube] ⚠️ No thumbnails found, using fallback position") + screen_w, screen_h = pyautogui.size() + pyautogui.click(screen_w // 2, int(screen_h * 0.45)) + return f"Attempted to play YouTube video for: {query}" + + +def _handle_summarize(parameters: dict, player, speak) -> str: + + if not _TRANSCRIPT_OK: + return ( + "youtube-transcript-api is not installed, sir. " + "Please run: pip install youtube-transcript-api" + ) + + url = _ask_for_url("Please paste the YouTube video URL:") + if not url: + return "No URL provided, sir. Summary cancelled." + + if not _is_valid_youtube_url(url): + return "That doesn't appear to be a valid YouTube URL, sir." + + video_id = _extract_video_id(url) + if not video_id: + return "Could not extract video ID from that URL, sir." + + if player: + player.write_log(f"[YouTube] Summarizing: {url}") + + if speak: + speak("Fetching the transcript now, sir. One moment.") + + transcript = _get_transcript(video_id) + if not transcript: + return ( + "I couldn't retrieve a transcript for that video, sir. " + "It may not have subtitles available." + ) + + if speak: + speak("Transcript retrieved. Generating summary now.") + + try: + summary = _summarize_with_gemini(transcript, url) + except Exception as e: + return f"Summary generation failed, sir: {e}" + + if speak: + speak(summary) + + save = parameters.get("save", False) + if save: + saved_path = _save_to_notepad(summary, url) + return f"Summary complete and saved to Desktop: {saved_path}" + + return summary + + +def _handle_get_info(parameters: dict, player, speak) -> str: + url = parameters.get("url", "").strip() + + if not url: + url = _ask_for_url("Please paste the YouTube video URL:") + if not url or not _is_valid_youtube_url(url): + return "Please provide a valid YouTube URL, sir." + + video_id = _extract_video_id(url) + if not video_id: + return "Could not extract video ID, sir." + + if player: + player.write_log(f"[YouTube] Getting info: {url}") + + info = _scrape_video_info(video_id) + if not info: + return "Could not retrieve video information, sir." + + lines = [] + for key in ("title", "channel", "views", "duration", "likes"): + if key in info: + lines.append(f"{key.capitalize()}: {info[key]}") + + result = "\n".join(lines) + if speak: + speak(f"Here's the video info, sir. {result.replace(chr(10), '. ')}") + + return result + + +def _handle_trending(parameters: dict, player, speak) -> str: + region = parameters.get("region", "TR").upper() + + if player: + player.write_log(f"[YouTube] Trending: {region}") + + trending = _scrape_trending(region=region, max_results=8) + + if not trending: + return f"Could not fetch trending videos for region {region}, sir." + + lines = [f"Top trending videos in {region}:"] + for item in trending: + lines.append(f"{item['rank']}. {item['title']} — {item['channel']}") + + result = "\n".join(lines) + + if speak: + top3 = trending[:3] + spoken = "Here are the top trending videos, sir. " + ". ".join( + f"Number {v['rank']}: {v['title']} by {v['channel']}" + for v in top3 + ) + speak(spoken) + + return result + + +_ACTION_MAP = { + "play": _handle_play, + "summarize": _handle_summarize, + "get_info": _handle_get_info, + "trending": _handle_trending, +} + +def youtube_video( + parameters: dict, + response=None, + player=None, + session_memory=None, + speak=None, +) -> str: + """ + YouTube controller. + + Actions: + play : Search and play a video (default if no action given) + parameters: query(str) + summarize : Summarize a video via transcript + Gemini + parameters: save(bool, optional) — save summary to Notepad + get_info : Get video metadata (title, channel, views, duration) + parameters: url(str, optional — dialog if omitted) + trending : Show trending videos + parameters: region(str, default "TR") — ISO country code + + Agent chain: + summarize can be chained after any action that produces a YouTube URL. + """ + params = parameters or {} + action = params.get("action", "play").lower().strip() + + if player: + player.write_log(f"[YouTube] Action: {action}") + + print(f"[YouTube] ▶️ Action: {action} Params: {params}") + + handler = _ACTION_MAP.get(action) + if handler is None: + return f"Unknown YouTube action: '{action}'. Available: play, summarize, get_info, trending." + + try: + if action == "play": + return handler(params, player) or "Done." + return handler(params, player, speak) or "Done." + + except Exception as e: + print(f"[YouTube] ❌ Error in {action}: {e}") + return f"YouTube {action} failed, sir: {e}" \ No newline at end of file diff --git a/agent/error_handler.py b/agent/error_handler.py new file mode 100644 index 0000000..5d3cc5d --- /dev/null +++ b/agent/error_handler.py @@ -0,0 +1,198 @@ +import json +import re +import sys +from pathlib import Path +from enum import Enum + + +def get_base_dir() -> Path: + if getattr(sys, "frozen", False): + return Path(sys.executable).parent + return Path(__file__).resolve().parent.parent + + +BASE_DIR = get_base_dir() +API_CONFIG_PATH = BASE_DIR / "config" / "api_keys.json" + + +class ErrorDecision(Enum): + RETRY = "retry" + SKIP = "skip" + REPLAN = "replan" + ABORT = "abort" + + +ERROR_ANALYST_PROMPT = """You are the error recovery module of MARK XXV AI assistant. + +A task step has failed. Analyze the error and decide what to do. + +DECISIONS: +- retry : Transient error (network timeout, temporary file lock, race condition). + The same step can succeed if tried again. +- skip : This step is not critical and the task can succeed without it. +- replan : The approach was wrong. A different tool or method should be tried. +- abort : The task is fundamentally impossible or unsafe to continue. + +Also provide: +- A brief explanation of WHY it failed (1 sentence) +- A fix suggestion if decision is replan (what to try instead) +- Max retries: how many times to retry if decision is retry (1 or 2) + +Return ONLY valid JSON: +{ + "decision": "retry|skip|replan|abort", + "reason": "why it failed", + "fix_suggestion": "what to try instead (for replan)", + "max_retries": 1, + "user_message": "Short message to tell the user (max 15 words)" +} +""" + + +def _get_api_key() -> str: + with open(API_CONFIG_PATH, "r", encoding="utf-8") as f: + return json.load(f)["gemini_api_key"] + + +def analyze_error( + step: dict, + error: str, + attempt: int = 1, + max_attempts: int = 2 +) -> dict: + """ + Analyzes a failed step and returns a recovery decision. + + Args: + step : The step dict that failed + error : Error message/traceback + attempt : Current attempt number + max_attempts : How many times we've already tried + + Returns: + { + "decision": ErrorDecision, + "reason": str, + "fix_suggestion": str, + "max_retries": int, + "user_message": str + } + """ + import google.generativeai as genai + + # If we've already retried enough, escalate to replan + if attempt >= max_attempts: + print(f"[ErrorHandler] ⚠️ Max attempts reached for step {step.get('step')} — forcing replan") + return { + "decision": ErrorDecision.REPLAN, + "reason": f"Failed {attempt} times: {error[:100]}", + "fix_suggestion": "Try a completely different approach or tool", + "max_retries": 0, + "user_message": "Trying a different approach, sir." + } + + genai.configure(api_key=_get_api_key()) + model = genai.GenerativeModel( + model_name="gemini-2.5-flash-lite", + system_instruction=ERROR_ANALYST_PROMPT + ) + + prompt = f"""Failed step: +Tool: {step.get('tool')} +Description: {step.get('description')} +Parameters: {json.dumps(step.get('parameters', {}), indent=2)} +Critical: {step.get('critical', False)} + +Error: +{error[:500]} + +Attempt number: {attempt}""" + + try: + response = model.generate_content(prompt) + text = response.text.strip() + text = re.sub(r"```(?:json)?", "", text).strip().rstrip("`").strip() + + result = json.loads(text) + decision_str = result.get("decision", "replan").lower() + decision_map = { + "retry": ErrorDecision.RETRY, + "skip": ErrorDecision.SKIP, + "replan": ErrorDecision.REPLAN, + "abort": ErrorDecision.ABORT, + } + result["decision"] = decision_map.get(decision_str, ErrorDecision.REPLAN) + + + if step.get("critical") and result["decision"] == ErrorDecision.SKIP: + result["decision"] = ErrorDecision.REPLAN + result["user_message"] = "This step is critical — finding alternative approach, sir." + + print(f"[ErrorHandler] Decision: {result['decision'].value} — {result.get('reason', '')}") + return result + + except Exception as e: + print(f"[ErrorHandler] ⚠️ Analysis failed: {e} — defaulting to replan") + return { + "decision": ErrorDecision.REPLAN, + "reason": str(e), + "fix_suggestion": "Try alternative approach", + "max_retries": 1, + "user_message": "Encountered an issue, adjusting approach, sir." + } + + +def generate_fix(step: dict, error: str, fix_suggestion: str) -> dict: + """ + When decision is REPLAN and a fix suggestion exists, + generates a replacement step using generated_code as fallback. + + Returns a modified step dict. + """ + import google.generativeai as genai + + genai.configure(api_key=_get_api_key()) + model = genai.GenerativeModel(model_name="gemini-2.0-flash") + + prompt = f"""A task step failed. Generate a replacement step. + +Original step: +Tool: {step.get('tool')} +Description: {step.get('description')} +Parameters: {json.dumps(step.get('parameters', {}), indent=2)} + +Error: {error[:300]} +Fix suggestion: {fix_suggestion} + +Write a Python script that accomplishes the same goal differently. +Return ONLY the Python code, no explanation.""" + + try: + response = model.generate_content(prompt) + code = response.text.strip() + code = re.sub(r"```(?:python)?", "", code).strip().rstrip("`").strip() + + return { + "step": step.get("step"), + "tool": "code_helper", + "description": f"Auto-fix for: {step.get('description')}", + "parameters": { + "action": "run", + "description": fix_suggestion, + "code": code, + "language": "python" + }, + "depends_on": step.get("depends_on", []), + "critical": step.get("critical", False) + } + + except Exception as e: + print(f"[ErrorHandler] ⚠️ Fix generation failed: {e}") + return { + "step": step.get("step"), + "tool": "generated_code", + "description": f"Fallback for: {step.get('description')}", + "parameters": {"description": step.get("description", "")}, + "depends_on": step.get("depends_on", []), + "critical": step.get("critical", False) + } \ No newline at end of file diff --git a/agent/executor.py b/agent/executor.py new file mode 100644 index 0000000..d2565f0 --- /dev/null +++ b/agent/executor.py @@ -0,0 +1,398 @@ +import json +import re +import sys +import threading +import subprocess +import tempfile +import os +from pathlib import Path +from typing import Callable + +from agent.planner import create_plan, replan +from agent.error_handler import analyze_error, generate_fix, ErrorDecision + + +def get_base_dir() -> Path: + if getattr(sys, "frozen", False): + return Path(sys.executable).parent + return Path(__file__).resolve().parent.parent + + +BASE_DIR = get_base_dir() +API_CONFIG_PATH = BASE_DIR / "config" / "api_keys.json" + + +def _get_api_key() -> str: + with open(API_CONFIG_PATH, "r", encoding="utf-8") as f: + return json.load(f)["gemini_api_key"] + +def _run_generated_code(description: str, speak: Callable | None = None) -> str: + import google.generativeai as genai + + if speak: + speak("Writing custom code for this task, sir.") + + home = Path.home() + desktop = home / "Desktop" + downloads = home / "Downloads" + documents = home / "Documents" + + if not desktop.exists(): + try: + import winreg + key = winreg.OpenKey(winreg.HKEY_CURRENT_USER, + r"Software\Microsoft\Windows\CurrentVersion\Explorer\Shell Folders") + desktop = Path(winreg.QueryValueEx(key, "Desktop")[0]) + except Exception: + pass + + genai.configure(api_key=_get_api_key()) + model = genai.GenerativeModel( + model_name="gemini-2.5-flash", + system_instruction=( + "You are an expert Python developer. " + "Write clean, complete, working Python code. " + "Use standard library + common packages. " + "Install missing packages with subprocess + pip if needed. " + "Return ONLY the Python code. No explanation, no markdown, no backticks.\n\n" + f"SYSTEM PATHS:\n" + f" Desktop = r'{desktop}'\n" + f" Downloads = r'{downloads}'\n" + f" Documents = r'{documents}'\n" + f" Home = r'{home}'\n" + ) + ) + + try: + response = model.generate_content( + f"Write Python code to accomplish this task:\n\n{description}" + ) + code = response.text.strip() + code = re.sub(r"```(?:python)?", "", code).strip().rstrip("`").strip() + + with tempfile.NamedTemporaryFile( + mode="w", suffix=".py", delete=False, encoding="utf-8" + ) as f: + f.write(code) + tmp_path = f.name + + print(f"[Executor] 🐍 Running generated code: {tmp_path}") + + result = subprocess.run( + [sys.executable, tmp_path], + capture_output=True, text=True, + timeout=120, cwd=str(Path.home()) + ) + + try: + os.unlink(tmp_path) + except Exception: + pass + + output = result.stdout.strip() + error = result.stderr.strip() + + if result.returncode == 0 and output: + return output + elif result.returncode == 0: + return "Task completed successfully." + elif error: + raise RuntimeError(f"Code error: {error[:400]}") + return "Completed." + + except subprocess.TimeoutExpired: + raise RuntimeError("Generated code timed out after 120 seconds.") + except RuntimeError: + raise + except Exception as e: + raise RuntimeError(f"Generated code failed: {e}") + +def _inject_context(params: dict, tool: str, step_results: dict, goal: str = "") -> dict: + if not step_results: + return params + + params = dict(params) + + if tool == "file_controller" and params.get("action") in ("write", "create_file"): + content = params.get("content", "") + if not content or len(content) < 50: + all_results = [ + v for v in step_results.values() + if v and len(v) > 100 and v not in ("Done.", "Completed.") + ] + if all_results: + combined = "\n\n---\n\n".join(all_results) + translated = _translate_to_goal_language(combined, goal) + params["content"] = translated + print(f"[Executor] 💉 Injected + translated content") + + return params +def _detect_language(text: str) -> str: + import google.generativeai as genai + genai.configure(api_key=_get_api_key()) + model = genai.GenerativeModel("gemini-2.5-flash-lite") + try: + response = model.generate_content( + f"What language is this text written in? " + f"Reply with ONLY the language name in English (e.g. Turkish, English, French).\n\n" + f"Text: {text[:200]}" + ) + return response.text.strip() + except Exception: + return "English" + + +def _translate_to_goal_language(content: str, goal: str) -> str: + if not goal: + return content + try: + import google.generativeai as genai + genai.configure(api_key=_get_api_key()) + model = genai.GenerativeModel("gemini-2.5-flash") + + target_lang = _detect_language(goal) + print(f"[Executor] 🌐 Translating to: {target_lang}") + + prompt = ( + f"You are a professional translator. " + f"Translate the following text into {target_lang}.\n" + f"IMPORTANT:\n" + f"- Translate EVERYTHING, leave nothing in English\n" + f"- Keep all facts, numbers, and data intact\n" + f"- Keep the structure and formatting\n" + f"- Output ONLY the translated text, nothing else\n\n" + f"Text to translate:\n{content[:4000]}" + ) + response = model.generate_content(prompt) + translated = response.text.strip() + print(f"[Executor] ✅ Translation done ({target_lang})") + return translated + except Exception as e: + print(f"[Executor] ⚠️ Translation failed: {e}") + return content + +def _call_tool(tool: str, parameters: dict, speak: Callable | None) -> str: + + if tool == "open_app": + from actions.open_app import open_app + return open_app(parameters=parameters, player=None) or "Done." + + elif tool == "web_search": + from actions.web_search import web_search + return web_search(parameters=parameters, player=None) or "Done." + + elif tool == "browser_control": + from actions.browser_control import browser_control + return browser_control(parameters=parameters, player=None) or "Done." + + elif tool == "file_controller": + from actions.file_controller import file_controller + return file_controller(parameters=parameters, player=None) or "Done." + + elif tool == "cmd_control": + from actions.cmd_control import cmd_control + return cmd_control(parameters=parameters, player=None) or "Done." + + elif tool == "code_helper": + from actions.code_helper import code_helper + return code_helper(parameters=parameters, player=None, speak=speak) or "Done." + + elif tool == "dev_agent": + from actions.dev_agent import dev_agent + return dev_agent(parameters=parameters, player=None, speak=speak) or "Done." + + elif tool == "screen_process": + from actions.screen_processor import screen_process + screen_process(parameters=parameters, player=None) + return "Screen captured and analyzed." + + elif tool == "send_message": + from actions.send_message import send_message + return send_message(parameters=parameters, player=None) or "Done." + + elif tool == "reminder": + from actions.reminder import reminder + return reminder(parameters=parameters, player=None) or "Done." + + elif tool == "youtube_video": + from actions.youtube_video import youtube_video + return youtube_video(parameters=parameters, player=None) or "Done." + + elif tool == "weather_report": + from actions.weather_report import weather_action + return weather_action(parameters=parameters, player=None) or "Done." + + elif tool == "computer_settings": + from actions.computer_settings import computer_settings + return computer_settings(parameters=parameters, player=None) or "Done." + + elif tool == "desktop_control": + from actions.desktop import desktop_control + return desktop_control(parameters=parameters, player=None) or "Done." + + elif tool == "computer_control": + from actions.computer_control import computer_control + return computer_control(parameters=parameters, player=None) or "Done." + + elif tool == "generated_code": + description = parameters.get("description", "") + if not description: + raise ValueError("generated_code requires a 'description' parameter.") + return _run_generated_code(description, speak=speak) + + elif tool == "flight_finder": + from actions.flight_finder import flight_finder + return flight_finder(parameters=parameters, player=None, speak=speak) or "Done." + + else: + print(f"[Executor] ⚠️ Unknown tool '{tool}' — falling back to generated_code") + return _run_generated_code(f"Accomplish this task: {parameters}", speak=speak) + +class AgentExecutor: + + MAX_REPLAN_ATTEMPTS = 2 + + def execute( + self, + goal: str, + speak: Callable | None = None, + cancel_flag: threading.Event | None = None, + ) -> str: + print(f"\n[Executor] 🎯 Goal: {goal}") + + replan_attempts = 0 + completed_steps = [] + step_results = {} + plan = create_plan(goal) + + while True: + steps = plan.get("steps", []) + + if not steps: + msg = "I couldn't create a valid plan for this task, sir." + if speak: speak(msg) + return msg + + success = True + failed_step = None + failed_error = "" + + for step in steps: + if cancel_flag and cancel_flag.is_set(): + if speak: speak("Task cancelled, sir.") + return "Task cancelled." + + step_num = step.get("step", "?") + tool = step.get("tool", "generated_code") + desc = step.get("description", "") + params = step.get("parameters", {}) + + params = _inject_context(params, tool, step_results, goal=goal) + + print(f"\n[Executor] ▶️ Step {step_num}: [{tool}] {desc}") + + attempt = 1 + step_ok = False + + while attempt <= 3: + if cancel_flag and cancel_flag.is_set(): + break + try: + result = _call_tool(tool, params, speak) + step_results[step_num] = result + completed_steps.append(step) + print(f"[Executor] ✅ Step {step_num} done: {str(result)[:100]}") + step_ok = True + break + + except Exception as e: + error_msg = str(e) + print(f"[Executor] ❌ Step {step_num} attempt {attempt} failed: {error_msg}") + + recovery = analyze_error(step, error_msg, attempt=attempt) + decision = recovery["decision"] + user_msg = recovery.get("user_message", "") + + if speak and user_msg: + speak(user_msg) + + if decision == ErrorDecision.RETRY: + attempt += 1 + import time; time.sleep(2) + continue + + elif decision == ErrorDecision.SKIP: + print(f"[Executor] ⏭️ Skipping step {step_num}") + completed_steps.append(step) + step_ok = True + break + + elif decision == ErrorDecision.ABORT: + msg = f"Task aborted, sir. {recovery.get('reason', '')}" + if speak: speak(msg) + return msg + + else: + fix_suggestion = recovery.get("fix_suggestion", "") + if fix_suggestion and tool != "generated_code": + try: + fixed_step = generate_fix(step, error_msg, fix_suggestion) + if speak: speak("Trying an alternative approach, sir.") + res = _call_tool( + fixed_step["tool"], + fixed_step["parameters"], + speak + ) + step_results[step_num] = res + completed_steps.append(step) + step_ok = True + break + except Exception as fix_err: + print(f"[Executor] ⚠️ Fix failed: {fix_err}") + + failed_step = step + failed_error = error_msg + success = False + break + + if not step_ok and not failed_step: + failed_step = step + failed_error = "Max retries exceeded" + success = False + + if not success: + break + + if success: + return self._summarize(goal, completed_steps, speak) + + if replan_attempts >= self.MAX_REPLAN_ATTEMPTS: + msg = f"Task failed after {replan_attempts} replan attempts, sir." + if speak: speak(msg) + return msg + + if speak: speak("Adjusting my approach, sir.") + + replan_attempts += 1 + plan = replan(goal, completed_steps, failed_step, failed_error) + + def _summarize(self, goal: str, completed_steps: list, speak: Callable | None) -> str: + fallback = f"All done, sir. Completed {len(completed_steps)} steps for: {goal[:60]}." + try: + import google.generativeai as genai + genai.configure(api_key=_get_api_key()) + model = genai.GenerativeModel(model_name="gemini-2.5-flash-lite") + steps_str = "\n".join(f"- {s.get('description', '')}" for s in completed_steps) + prompt = ( + f'User goal: "{goal}"\n' + f"Completed steps:\n{steps_str}\n\n" + "Write a single natural sentence summarizing what was accomplished. " + "Address the user as 'sir'. Be direct and positive." + ) + response = model.generate_content(prompt) + summary = response.text.strip() + if speak: speak(summary) + return summary + except Exception: + if speak: speak(fallback) + return fallback \ No newline at end of file diff --git a/agent/planner.py b/agent/planner.py new file mode 100644 index 0000000..be63315 --- /dev/null +++ b/agent/planner.py @@ -0,0 +1,258 @@ +import json +import re +import sys +from pathlib import Path + + +def get_base_dir() -> Path: + if getattr(sys, "frozen", False): + return Path(sys.executable).parent + return Path(__file__).resolve().parent.parent + + +BASE_DIR = get_base_dir() +API_CONFIG_PATH = BASE_DIR / "config" / "api_keys.json" + + +PLANNER_PROMPT = """You are the planning module of MARK XXV, a personal AI assistant. +Your job: break any user goal into a sequence of steps using ONLY the tools listed below. + +ABSOLUTE RULES: +- NEVER use generated_code or write Python scripts. It does not exist. +- NEVER reference previous step results in parameters. Every step is independent. +- Use web_search for ANY information retrieval, research, or current data. +- Use file_controller to save content to disk. +- Use cmd_control to open files or run system commands. +- Max 5 steps. Use the minimum steps needed. + +AVAILABLE TOOLS AND THEIR PARAMETERS: + +open_app + app_name: string (required) + +web_search + query: string (required) — write a clear, focused search query + mode: "search" or "compare" (optional, default: search) + items: list of strings (optional, for compare mode) + aspect: string (optional, for compare mode) + +browser_control + action: "go_to" | "search" | "click" | "type" | "scroll" | "get_text" | "press" | "close" (required) + url: string (for go_to) + query: string (for search) + text: string (for click/type) + direction: "up" | "down" (for scroll) + +file_controller + action: "write" | "create_file" | "read" | "list" | "delete" | "move" | "copy" | "find" | "disk_usage" (required) + path: string — use "desktop" for Desktop folder + name: string — filename + content: string — file content (for write/create_file) + +cmd_control + task: string (required) — natural language description of what to do + visible: boolean (optional) + +computer_settings + action: string (required) + description: string — natural language description + value: string (optional) + +computer_control + action: "type" | "click" | "hotkey" | "press" | "scroll" | "screenshot" | "screen_find" | "screen_click" (required) + text: string (for type) + x, y: int (for click) + keys: string (for hotkey, e.g. "ctrl+c") + key: string (for press) + direction: "up" | "down" (for scroll) + description: string (for screen_find/screen_click) + +screen_process + text: string (required) — what to analyze or ask about the screen + angle: "screen" | "camera" (optional) + +send_message + receiver: string (required) + message_text: string (required) + platform: string (required) + +reminder + date: string YYYY-MM-DD (required) + time: string HH:MM (required) + message: string (required) + +desktop_control + action: "wallpaper" | "organize" | "clean" | "list" | "task" (required) + path: string (optional) + task: string (optional) + +youtube_video + action: "play" | "summarize" | "trending" (required) + query: string (for play) + +weather_report + city: string (required) + +flight_finder + origin: string (required) + destination: string (required) + date: string (required) + +code_helper + action: "write" | "edit" | "run" | "explain" (required) + description: string (required) + language: string (optional) + output_path: string (optional) + file_path: string (optional) + +dev_agent + description: string (required) + language: string (optional) + +EXAMPLES: + +Goal: "makine mühendisliği hakkında araştırma yap ve not defterine kaydet" +Steps: + 1. web_search | query: "mechanical engineering overview definition history" + 2. web_search | query: "mechanical engineering applications and future trends" + 3. file_controller | action: write, path: desktop, name: makine_muhendisligi.txt, content: "MAKINE MUHENDISLIGI ARASTIRMASI\n\nBu dosya web arastirmasi sonuclari ile doldurulacak." + 4. cmd_control | task: "open makine_muhendisligi.txt on desktop with notepad" + +Goal: "Bitcoin fiyatı nedir" +Steps: + 1. web_search | query: "Bitcoin price today USD" + +Goal: "Masaüstündeki dosyaları listele ve en büyük 5 dosyayı bul" +Steps: + 1. file_controller | action: list, path: desktop + 2. file_controller | action: largest, path: desktop, count: 5 + +Goal: "WhatsApp'tan Ahmet'e yarın toplantı var de" +Steps: + 1. send_message | receiver: Ahmet, message_text: "Yarın toplantı var", platform: WhatsApp + +Goal: "Saati aç ve 30 dakika sonraya hatırlatıcı kur" +Steps: + 1. reminder | date: [today], time: [now+30min], message: "Hatırlatıcı" + +OUTPUT — return ONLY valid JSON, no markdown, no explanation, no code blocks: +{ + "goal": "...", + "steps": [ + { + "step": 1, + "tool": "tool_name", + "description": "what this step does", + "parameters": {}, + "critical": true + } + ] +} +""" + + +def _get_api_key() -> str: + with open(API_CONFIG_PATH, "r", encoding="utf-8") as f: + return json.load(f)["gemini_api_key"] + + +def create_plan(goal: str, context: str = "") -> dict: + import google.generativeai as genai + + genai.configure(api_key=_get_api_key()) + model = genai.GenerativeModel( + model_name="gemini-2.5-flash-lite", + system_instruction=PLANNER_PROMPT + ) + + user_input = f"Goal: {goal}" + if context: + user_input += f"\n\nContext: {context}" + + try: + response = model.generate_content(user_input) + text = response.text.strip() + text = re.sub(r"```(?:json)?", "", text).strip().rstrip("`").strip() + + plan = json.loads(text) + + if "steps" not in plan or not isinstance(plan["steps"], list): + raise ValueError("Invalid plan structure") + + for step in plan["steps"]: + if step.get("tool") in ("generated_code",): + print(f"[Planner] ⚠️ generated_code detected in step {step.get('step')} — replacing with web_search") + desc = step.get("description", goal) + step["tool"] = "web_search" + step["parameters"] = {"query": desc[:200]} + + print(f"[Planner] ✅ Plan: {len(plan['steps'])} steps") + for s in plan["steps"]: + print(f" Step {s['step']}: [{s['tool']}] {s['description']}") + + return plan + + except json.JSONDecodeError as e: + print(f"[Planner] ⚠️ JSON parse failed: {e}") + return _fallback_plan(goal) + except Exception as e: + print(f"[Planner] ⚠️ Planning failed: {e}") + return _fallback_plan(goal) + + +def _fallback_plan(goal: str) -> dict: + print("[Planner] 🔄 Fallback plan") + return { + "goal": goal, + "steps": [ + { + "step": 1, + "tool": "web_search", + "description": f"Search for: {goal}", + "parameters": {"query": goal}, + "critical": True + } + ] + } + + +def replan(goal: str, completed_steps: list, failed_step: dict, error: str) -> dict: + import google.generativeai as genai + + genai.configure(api_key=_get_api_key()) + model = genai.GenerativeModel( + model_name="gemini-2.5-flash", + system_instruction=PLANNER_PROMPT + ) + + completed_summary = "\n".join( + f" - Step {s['step']} ({s['tool']}): DONE" for s in completed_steps + ) + + prompt = f"""Goal: {goal} + +Already completed: +{completed_summary if completed_summary else ' (none)'} + +Failed step: [{failed_step.get('tool')}] {failed_step.get('description')} +Error: {error} + +Create a REVISED plan for the remaining work only. Do not repeat completed steps.""" + + try: + response = model.generate_content(prompt) + text = response.text.strip() + text = re.sub(r"```(?:json)?", "", text).strip().rstrip("`").strip() + plan = json.loads(text) + + # generated_code kontrolü + for step in plan.get("steps", []): + if step.get("tool") == "generated_code": + step["tool"] = "web_search" + step["parameters"] = {"query": step.get("description", goal)[:200]} + + print(f"[Planner] 🔄 Revised plan: {len(plan['steps'])} steps") + return plan + except Exception as e: + print(f"[Planner] ⚠️ Replan failed: {e}") + return _fallback_plan(goal) \ No newline at end of file diff --git a/agent/task_queue.py b/agent/task_queue.py new file mode 100644 index 0000000..90c0e84 --- /dev/null +++ b/agent/task_queue.py @@ -0,0 +1,221 @@ +import threading +import time +import uuid +from dataclasses import dataclass, field +from enum import Enum +from typing import Callable, Any + + +class TaskStatus(Enum): + PENDING = "pending" + RUNNING = "running" + COMPLETED = "completed" + FAILED = "failed" + CANCELLED = "cancelled" + + +class TaskPriority(Enum): + LOW = 3 + NORMAL = 2 + HIGH = 1 + + +@dataclass(order=True) +class Task: + priority: int + created_at: float = field(compare=False) + task_id: str = field(compare=False) + goal: str = field(compare=False) + status: TaskStatus = field(compare=False, default=TaskStatus.PENDING) + result: Any = field(compare=False, default=None) + error: str = field(compare=False, default="") + speak: Any = field(compare=False, default=None) + on_complete: Any = field(compare=False, default=None) + cancel_flag: threading.Event = field(compare=False, default_factory=threading.Event) + + +class TaskQueue: + def __init__(self, max_concurrent: int = 1): + self._queue: list[Task] = [] + self._lock: threading.Lock = threading.Lock() + self._condition: threading.Condition = threading.Condition(self._lock) + self._tasks: dict[str, Task] = {} + self._running: bool = False + self._worker_thread: threading.Thread | None = None + self._max_concurrent = max_concurrent + self._active_count = 0 + self._executor = None + + def _get_executor(self): + if self._executor is None: + from agent.executor import AgentExecutor + self._executor = AgentExecutor() + return self._executor + + def start(self) -> None: + if self._running: + return + self._running = True + self._worker_thread = threading.Thread( + target=self._worker_loop, + daemon=True, + name="AgentTaskQueue" + ) + self._worker_thread.start() + print("[TaskQueue] ✅ Started") + + def stop(self) -> None: + self._running = False + with self._condition: + self._condition.notify_all() + print("[TaskQueue] 🔴 Stopped") + + def submit( + self, + goal: str, + priority: TaskPriority = TaskPriority.NORMAL, + speak: Callable | None = None, + on_complete: Callable | None = None, + ) -> str: + + task_id = str(uuid.uuid4())[:8] + task = Task( + priority = priority.value, + created_at = time.time(), + task_id = task_id, + goal = goal, + speak = speak, + on_complete = on_complete, + ) + + with self._condition: + self._queue.append(task) + self._queue.sort(key=lambda t: (t.priority, t.created_at)) + self._tasks[task_id] = task + self._condition.notify() + + print(f"[TaskQueue] 📥 Task queued: [{task_id}] {goal[:60]}") + return task_id + + def cancel(self, task_id: str) -> bool: + + with self._lock: + task = self._tasks.get(task_id) + if not task: + return False + if task.status in (TaskStatus.COMPLETED, TaskStatus.FAILED, TaskStatus.CANCELLED): + return False + + task.cancel_flag.set() + task.status = TaskStatus.CANCELLED + print(f"[TaskQueue] 🚫 Task cancelled: [{task_id}]") + return True + + def get_status(self, task_id: str) -> dict | None: + with self._lock: + task = self._tasks.get(task_id) + if not task: + return None + return { + "task_id": task.task_id, + "goal": task.goal, + "status": task.status.value, + "result": task.result, + "error": task.error, + } + + def get_all_statuses(self) -> list[dict]: + with self._lock: + return [ + { + "task_id": t.task_id, + "goal": t.goal[:50], + "status": t.status.value, + } + for t in self._tasks.values() + ] + + def pending_count(self) -> int: + with self._lock: + return sum(1 for t in self._queue if t.status == TaskStatus.PENDING) + + def _worker_loop(self) -> None: + while self._running: + task = None + + with self._condition: + while self._running and not self._next_task(): + self._condition.wait(timeout=1.0) + task = self._next_task() + if task: + task.status = TaskStatus.RUNNING + self._active_count += 1 + try: + self._queue.remove(task) + except ValueError: + pass + + if task: + threading.Thread( + target=self._run_task, + args=(task,), + daemon=True, + name=f"AgentTask-{task.task_id}" + ).start() + + def _next_task(self) -> Task | None: + if self._active_count >= self._max_concurrent: + return None + for task in self._queue: + if task.status == TaskStatus.PENDING and not task.cancel_flag.is_set(): + return task + return None + + def _run_task(self, task: Task) -> None: + print(f"[TaskQueue] ▶️ Running: [{task.task_id}] {task.goal[:60]}") + try: + executor = self._get_executor() + result = executor.execute( + goal = task.goal, + speak = task.speak, + cancel_flag = task.cancel_flag, + ) + + with self._lock: + if task.cancel_flag.is_set(): + task.status = TaskStatus.CANCELLED + else: + task.status = TaskStatus.COMPLETED + task.result = result + self._active_count -= 1 + + if task.on_complete and not task.cancel_flag.is_set(): + try: + task.on_complete(task.task_id, result) + except Exception as e: + print(f"[TaskQueue] ⚠️ on_complete callback error: {e}") + + print(f"[TaskQueue] ✅ Completed: [{task.task_id}]") + + except Exception as e: + with self._lock: + task.status = TaskStatus.FAILED + task.error = str(e) + self._active_count -= 1 + print(f"[TaskQueue] ❌ Failed: [{task.task_id}] {e}") + + with self._condition: + self._condition.notify() + +_queue = TaskQueue() +_queue_started = False +_queue_lock = threading.Lock() + + +def get_queue() -> TaskQueue: + global _queue_started + with _queue_lock: + if not _queue_started: + _queue.start() + _queue_started = True + return _queue \ No newline at end of file diff --git a/config/__init__.py b/config/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/core/prompt.txt b/core/prompt.txt new file mode 100644 index 0000000..f5df8a8 --- /dev/null +++ b/core/prompt.txt @@ -0,0 +1,23 @@ +You are JARVIS, a sharp and efficient AI assistant. Calm, direct, professional. + +RULES: +- Always use the correct tool. Never simulate or guess results. +- Call each tool EXACTLY ONCE. Never retry a successful action. +- Do NOT say "I'm doing X" before calling a tool. Just call it, then report the result briefly. +- Do NOT repeat yourself. Say something once and stop. +- Keep responses to 1-2 sentences max unless reporting data. +- Never ask unnecessary questions. Make a reasonable assumption and proceed. + +LANGUAGE: +- Respond in the same language the user spoke. +- Always extract tool parameters in English. +- Example: "İstanbul hava durumu?" → weather_report city:"Istanbul", reply in Turkish. + +TOOL SELECTION: +- computer_settings → volume, brightness, screen, refresh, scroll, minimize, maximize, + close, screenshot, lock, restart, shutdown, wifi, zoom, tabs, keyboard shortcuts. + Use this for ANY single computer control command. NEVER route these to agent_task. +- agent_task → ONLY for complex multi-step goals that require planning across + multiple tools (e.g. "research X and save to file", "create a project"). +- Simple rule: if it can be done in ONE action → computer_settings. + If it needs 3+ steps across different tools → agent_task. \ No newline at end of file diff --git a/main.py b/main.py new file mode 100644 index 0000000..30a3ea5 --- /dev/null +++ b/main.py @@ -0,0 +1,853 @@ +import asyncio +import threading +import json +import re +import sys +import traceback +from pathlib import Path + +import pyaudio +from google import genai +from google.genai import types +import time +from ui import JarvisUI +from memory.memory_manager import load_memory, update_memory, format_memory_for_prompt + +from agent.task_queue import get_queue + +from actions.flight_finder import flight_finder +from actions.open_app import open_app +from actions.weather_report import weather_action +from actions.send_message import send_message +from actions.reminder import reminder +from actions.computer_settings import computer_settings +from actions.screen_processor import screen_process +from actions.youtube_video import youtube_video +from actions.cmd_control import cmd_control +from actions.desktop import desktop_control +from actions.browser_control import browser_control +from actions.file_controller import file_controller +from actions.code_helper import code_helper +from actions.dev_agent import dev_agent +from actions.web_search import web_search as web_search_action +from actions.computer_control import computer_control + +def get_base_dir(): + if getattr(sys, "frozen", False): + return Path(sys.executable).parent + return Path(__file__).resolve().parent + +BASE_DIR = get_base_dir() +API_CONFIG_PATH = BASE_DIR / "config" / "api_keys.json" +PROMPT_PATH = BASE_DIR / "core" / "prompt.txt" +LIVE_MODEL = "models/gemini-2.5-flash-native-audio-preview-12-2025" +FORMAT = pyaudio.paInt16 +CHANNELS = 1 +SEND_SAMPLE_RATE = 16000 +RECEIVE_SAMPLE_RATE = 24000 +CHUNK_SIZE = 1024 + +pya = pyaudio.PyAudio() + +def _get_api_key() -> str: + with open(API_CONFIG_PATH, "r", encoding="utf-8") as f: + return json.load(f)["gemini_api_key"] + + with open(API_CONFIG_PATH, "r", encoding="utf-8") as f: + return json.load(f)["gemini_api_key"] + +def _load_system_prompt() -> str: + try: + return PROMPT_PATH.read_text(encoding="utf-8") + except Exception: + return ( + "You are JARVIS, Tony Stark's AI assistant. " + "Be concise, direct, and always use the provided tools to complete tasks. " + "Never simulate or guess results — always call the appropriate tool." + ) + +_memory_turn_counter = 0 +_memory_turn_lock = threading.Lock() +_MEMORY_EVERY_N_TURNS = 5 +_last_memory_input = "" + + +def _update_memory_async(user_text: str, jarvis_text: str) -> None: + """ + Multilingual memory updater. + Model : gemini-2.5-flash-lite (lowest cost) + Stage 1: Quick YES/NO check → ~5 tokens output + Stage 2: Full extraction → only if Stage 1 says YES + Result : ~80% fewer API calls vs original + """ + global _memory_turn_counter, _last_memory_input + + with _memory_turn_lock: + _memory_turn_counter += 1 + current_count = _memory_turn_counter + + if current_count % _MEMORY_EVERY_N_TURNS != 0: + return + + text = user_text.strip() + if len(text) < 10: + return + if text == _last_memory_input: + return + _last_memory_input = text + + try: + import google.generativeai as genai + genai.configure(api_key=_get_api_key()) + model = genai.GenerativeModel("gemini-2.5-flash-lite") + + check = model.generate_content( + f"Does this message contain personal facts about the user " + f"(name, age, city, job, hobby, relationship, birthday, preference)? " + f"Reply only YES or NO.\n\nMessage: {text[:300]}" + ) + if "YES" not in check.text.upper(): + return + + raw = model.generate_content( + f"Extract personal facts from this message. Any language.\n" + f"Return ONLY valid JSON or {{}} if nothing found.\n" + f"Extract: name, age, birthday, city, job, hobbies, preferences, relationships, language.\n" + f"Skip: weather, reminders, search results, commands.\n\n" + f"Format:\n" + f'{{"identity":{{"name":{{"value":"..."}}}}}}, ' + f'"preferences":{{"hobby":{{"value":"..."}}}}, ' + f'"notes":{{"job":{{"value":"..."}}}}}}\n\n' + f"Message: {text[:500]}\n\nJSON:" + ).text.strip() + + raw = re.sub(r"```(?:json)?", "", raw).strip().rstrip("`").strip() + if not raw or raw == "{}": + return + + data = json.loads(raw) + if data: + update_memory(data) + print(f"[Memory] ✅ Updated: {list(data.keys())}") + + except json.JSONDecodeError: + pass + except Exception as e: + if "429" not in str(e): + print(f"[Memory] ⚠️ {e}") + + +TOOL_DECLARATIONS = [ + { + "name": "open_app", + "description": ( + "Opens any application on the Windows computer. " + "Use this whenever the user asks to open, launch, or start any app, " + "website, or program. Always call this tool — never just say you opened it." + ), + "parameters": { + "type": "OBJECT", + "properties": { + "app_name": { + "type": "STRING", + "description": "Exact name of the application (e.g. 'WhatsApp', 'Chrome', 'Spotify')" + } + }, + "required": ["app_name"] + } + }, +{ + "name": "web_search", + "description": "Searches the web for any information.", + "parameters": { + "type": "OBJECT", + "properties": { + "query": {"type": "STRING", "description": "Search query"}, + "mode": {"type": "STRING", "description": "search (default) or compare"}, + "items": {"type": "ARRAY", "items": {"type": "STRING"}, "description": "Items to compare"}, + "aspect": {"type": "STRING", "description": "price | specs | reviews"} + }, + "required": ["query"] + } +}, + { + "name": "weather_report", + "description": "Gets real-time weather information for a city.", + "parameters": { + "type": "OBJECT", + "properties": { + "city": {"type": "STRING", "description": "City name"} + }, + "required": ["city"] + } + }, + { + "name": "send_message", + "description": "Sends a text message via WhatsApp, Telegram, or other messaging platform.", + "parameters": { + "type": "OBJECT", + "properties": { + "receiver": {"type": "STRING", "description": "Recipient contact name"}, + "message_text": {"type": "STRING", "description": "The message to send"}, + "platform": {"type": "STRING", "description": "Platform: WhatsApp, Telegram, etc."} + }, + "required": ["receiver", "message_text", "platform"] + } + }, + { + "name": "reminder", + "description": "Sets a timed reminder using Windows Task Scheduler.", + "parameters": { + "type": "OBJECT", + "properties": { + "date": {"type": "STRING", "description": "Date in YYYY-MM-DD format"}, + "time": {"type": "STRING", "description": "Time in HH:MM format (24h)"}, + "message": {"type": "STRING", "description": "Reminder message text"} + }, + "required": ["date", "time", "message"] + } + }, + { + "name": "youtube_video", + "description": ( + "Controls YouTube. Use for: playing videos, summarizing a video's content, " + "getting video info, or showing trending videos." + ), + "parameters": { + "type": "OBJECT", + "properties": { + "action": { + "type": "STRING", + "description": "play | summarize | get_info | trending (default: play)" + }, + "query": {"type": "STRING", "description": "Search query for play action"}, + "save": {"type": "BOOLEAN", "description": "Save summary to Notepad (summarize only)"}, + "region": {"type": "STRING", "description": "Country code for trending e.g. TR, US"}, + "url": {"type": "STRING", "description": "Video URL for get_info action"}, + }, + "required": [] + } + }, + { + "name": "screen_process", + "description": ( + "Captures and analyzes the screen or webcam image. " + "MUST be called when user asks what is on screen, what you see, " + "analyze my screen, look at camera, etc. " + "You have NO visual ability without this tool. " + "After calling this tool, stay SILENT — the vision module speaks directly." + ), + "parameters": { + "type": "OBJECT", + "properties": { + "angle": { + "type": "STRING", + "description": "'screen' to capture display, 'camera' for webcam. Default: 'screen'" + }, + "text": { + "type": "STRING", + "description": "The question or instruction about the captured image" + } + }, + "required": ["text"] + } + }, + { + "name": "computer_settings", + "description": ( + "Controls the computer: volume, brightness, window management, keyboard shortcuts, " + "typing text on screen, closing apps, fullscreen, dark mode, WiFi, restart, shutdown, " + "scrolling, tab management, zoom, screenshots, lock screen, refresh/reload page. " + "ALSO use for repeated actions: 'refresh 10 times', 'reload page 5 times' → action: reload_n, value: 10. " + "Use for ANY single computer control command — even if repeated N times. " + "NEVER route simple computer commands to agent_task." + ), + "parameters": { + "type": "OBJECT", + "properties": { + "action": {"type": "STRING", "description": "The action to perform (if known). For repeated reload: 'reload_n'"}, + "description": {"type": "STRING", "description": "Natural language description of what to do"}, + "value": {"type": "STRING", "description": "Optional value: volume level, text to type, number of times, etc."} + }, + "required": [] + } +}, + { + "name": "browser_control", + "description": ( + "Controls the web browser. Use for: opening websites, searching the web, " + "clicking elements, filling forms, scrolling, finding cheapest products, " + "booking flights, any web-based task." + ), + "parameters": { + "type": "OBJECT", + "properties": { + "action": {"type": "STRING", "description": "go_to | search | click | type | scroll | fill_form | smart_click | smart_type | get_text | press | close"}, + "url": {"type": "STRING", "description": "URL for go_to action"}, + "query": {"type": "STRING", "description": "Search query for search action"}, + "selector": {"type": "STRING", "description": "CSS selector for click/type"}, + "text": {"type": "STRING", "description": "Text to click or type"}, + "description": {"type": "STRING", "description": "Element description for smart_click/smart_type"}, + "direction": {"type": "STRING", "description": "up or down for scroll"}, + "key": {"type": "STRING", "description": "Key name for press action"}, + }, + "required": ["action"] + } + }, + { + "name": "file_controller", + "description": ( + "Manages files and folders. Use for: listing files, creating/deleting/moving/copying " + "files, reading file contents, finding files by name or extension, checking disk usage, " + "organizing the desktop, getting file info." + ), + "parameters": { + "type": "OBJECT", + "properties": { + "action": {"type": "STRING", "description": "list | create_file | create_folder | delete | move | copy | rename | read | write | find | largest | disk_usage | organize_desktop | info"}, + "path": {"type": "STRING", "description": "File/folder path or shortcut: desktop, downloads, documents, home"}, + "destination": {"type": "STRING", "description": "Destination path for move/copy"}, + "new_name": {"type": "STRING", "description": "New name for rename"}, + "content": {"type": "STRING", "description": "Content for create_file/write"}, + "name": {"type": "STRING", "description": "File name to search for"}, + "extension": {"type": "STRING", "description": "File extension to search (e.g. .pdf)"}, + "count": {"type": "INTEGER", "description": "Number of results for largest"}, + }, + "required": ["action"] + } + }, + { + "name": "cmd_control", + "description": ( + "Runs CMD/terminal commands by understanding natural language. " + "Use when user wants to: find large files, check disk space, list processes, " + "get system info, navigate folders, check network, find files by name, " + "or do ANYTHING in the command line they don't know how to do themselves." + ), + "parameters": { + "type": "OBJECT", + "properties": { + "task": {"type": "STRING", "description": "Natural language description of what to do. Example: 'find the 10 largest files on C drive'"}, + "visible": {"type": "BOOLEAN", "description": "Open visible CMD window so user can see. Default: true"}, + "command": {"type": "STRING", "description": "Optional: exact command if already known"}, + }, + "required": ["task"] + } + }, + { + "name": "desktop_control", + "description": ( + "Controls the desktop. Use for: changing wallpaper, organizing desktop files, " + "cleaning the desktop, listing desktop contents, or ANY other desktop-related task " + "the user describes in natural language." + ), + "parameters": { + "type": "OBJECT", + "properties": { + "action": {"type": "STRING", "description": "wallpaper | wallpaper_url | organize | clean | list | stats | task"}, + "path": {"type": "STRING", "description": "Image path for wallpaper"}, + "url": {"type": "STRING", "description": "Image URL for wallpaper_url"}, + "mode": {"type": "STRING", "description": "by_type or by_date for organize"}, + "task": {"type": "STRING", "description": "Natural language description of any desktop task"}, + }, + "required": ["action"] + } + }, + { + "name": "code_helper", + "description": ( + "Writes, edits, explains, runs, or self-builds code files. " + "Use for ANY coding request: writing a script, fixing a file, " + "editing existing code, running a file, or building and testing automatically." + ), + "parameters": { + "type": "OBJECT", + "properties": { + "action": {"type": "STRING", "description": "write | edit | explain | run | build | auto (default: auto)"}, + "description": {"type": "STRING", "description": "What the code should do, or what change to make"}, + "language": {"type": "STRING", "description": "Programming language (default: python)"}, + "output_path": {"type": "STRING", "description": "Where to save the file (full path or filename)"}, + "file_path": {"type": "STRING", "description": "Path to existing file for edit / explain / run / build"}, + "code": {"type": "STRING", "description": "Raw code string for explain"}, + "args": {"type": "STRING", "description": "CLI arguments for run/build"}, + "timeout": {"type": "INTEGER", "description": "Execution timeout in seconds (default: 30)"}, + }, + "required": ["action"] + } + }, + { + "name": "dev_agent", + "description": ( + "Builds complete multi-file projects from scratch. " + "Plans structure, writes all files, installs dependencies, " + "opens VSCode, runs the project, and fixes errors automatically. " + "Use for any project larger than a single script." + ), + "parameters": { + "type": "OBJECT", + "properties": { + "description": {"type": "STRING", "description": "What the project should do"}, + "language": {"type": "STRING", "description": "Programming language (default: python)"}, + "project_name": {"type": "STRING", "description": "Optional project folder name"}, + "timeout": {"type": "INTEGER", "description": "Run timeout in seconds (default: 30)"}, + }, + "required": ["description"] + } + }, + { + "name": "agent_task", + "description": ( + "Executes complex multi-step tasks that require MULTIPLE DIFFERENT tools. " + "Always respond to the user in the language they spoke. " + "Examples: 'research X and save to file', 'find files and organize them', " + "'fill a form on a website', 'write and test code'. " + "DO NOT use for simple computer commands like volume, refresh, close, scroll, " + "minimize, screenshot, restart, shutdown — use computer_settings for those. " + "DO NOT use if the task can be done with a single tool call." + ), + "parameters": { + "type": "OBJECT", + "properties": { + "goal": { + "type": "STRING", + "description": "Complete description of what needs to be accomplished" + }, + "priority": { + "type": "STRING", + "description": "low | normal | high (default: normal)" + } + }, + "required": ["goal"] + } +}, + { + "name": "computer_control", + "description": ( + "Direct computer control: type text, click buttons, use keyboard shortcuts, " + "scroll, move mouse, take screenshots, fill forms, find elements on screen. " + "Use when the user wants to interact with any app on the computer directly. " + "Can generate random data for forms or use user's real info from memory." + ), + "parameters": { + "type": "OBJECT", + "properties": { + "action": {"type": "STRING", "description": "type | smart_type | click | double_click | right_click | hotkey | press | scroll | move | copy | paste | screenshot | wait | clear_field | focus_window | screen_find | screen_click | random_data | user_data"}, + "text": {"type": "STRING", "description": "Text to type or paste"}, + "x": {"type": "INTEGER", "description": "X coordinate for click/move"}, + "y": {"type": "INTEGER", "description": "Y coordinate for click/move"}, + "keys": {"type": "STRING", "description": "Key combination e.g. 'ctrl+c'"}, + "key": {"type": "STRING", "description": "Single key to press e.g. 'enter'"}, + "direction": {"type": "STRING", "description": "Scroll direction: up | down | left | right"}, + "amount": {"type": "INTEGER", "description": "Scroll amount (default: 3)"}, + "seconds": {"type": "NUMBER", "description": "Seconds to wait"}, + "title": {"type": "STRING", "description": "Window title for focus_window"}, + "description": {"type": "STRING", "description": "Element description for screen_find/screen_click"}, + "type": {"type": "STRING", "description": "Data type for random_data: name|email|username|password|phone|birthday|address"}, + "field": {"type": "STRING", "description": "Field for user_data: name|email|city"}, + "clear_first": {"type": "BOOLEAN", "description": "Clear field before typing (default: true)"}, + "path": {"type": "STRING", "description": "Save path for screenshot"}, + }, + "required": ["action"] + } +}, + +{ + "name": "flight_finder", + "description": ( + "Searches for flights on Google Flights and speaks the best options. " + "Use when user asks about flights, plane tickets, uçuş, bilet, etc." + ), + "parameters": { + "type": "OBJECT", + "properties": { + "origin": {"type": "STRING", "description": "Departure city or airport code"}, + "destination": {"type": "STRING", "description": "Arrival city or airport code"}, + "date": {"type": "STRING", "description": "Departure date (any format)"}, + "return_date": {"type": "STRING", "description": "Return date for round trips"}, + "passengers": {"type": "INTEGER", "description": "Number of passengers (default: 1)"}, + "cabin": {"type": "STRING", "description": "economy | premium | business | first"}, + "save": {"type": "BOOLEAN", "description": "Save results to Notepad"}, + }, + "required": ["origin", "destination", "date"] + } +} +] + +class JarvisLive: + + def __init__(self, ui: JarvisUI): + self.ui = ui + self.session = None + self.audio_in_queue = None + self.out_queue = None + self._loop = None + + def speak(self, text: str): + """Thread-safe speak — any thread can call this.""" + if not self._loop or not self.session: + return + asyncio.run_coroutine_threadsafe( + self.session.send_client_content( + turns={"parts": [{"text": text}]}, + turn_complete=True + ), + self._loop + ) + + def _build_config(self) -> types.LiveConnectConfig: + from datetime import datetime + + memory = load_memory() + mem_str = format_memory_for_prompt(memory) + + sys_prompt = _load_system_prompt() + + now = datetime.now() + time_str = now.strftime("%A, %B %d, %Y — %I:%M %p") + time_ctx = ( + f"[CURRENT DATE & TIME]\n" + f"Right now it is: {time_str}\n" + f"Use this to calculate exact times for reminders. " + f"If user says 'in 2 minutes', add 2 minutes to this time.\n\n" + ) + + if mem_str: + sys_prompt = time_ctx + mem_str + "\n\n" + sys_prompt + else: + sys_prompt = time_ctx + sys_prompt + + return types.LiveConnectConfig( + response_modalities=["AUDIO"], + output_audio_transcription={}, + input_audio_transcription={}, + system_instruction=sys_prompt, + tools=[{"function_declarations": TOOL_DECLARATIONS}], + session_resumption=types.SessionResumptionConfig(), + speech_config=types.SpeechConfig( + voice_config=types.VoiceConfig( + prebuilt_voice_config=types.PrebuiltVoiceConfig( + voice_name="Charon" + ) + ) + ), + ) + + async def _execute_tool(self, fc) -> types.FunctionResponse: + name = fc.name + args = dict(fc.args or {}) + + print(f"[JARVIS] 🔧 TOOL: {name} ARGS: {args}") + + loop = asyncio.get_event_loop() + result = "Done." + + try: + if name == "open_app": + r = await loop.run_in_executor( + None, lambda: open_app(parameters=args, response=None, player=self.ui) + ) + result = r or f"Opened {args.get('app_name')} successfully." + + elif name == "weather_report": + r = await loop.run_in_executor( + None, lambda: weather_action(parameters=args, player=self.ui) + ) + result = r or f"Weather report for {args.get('city')} delivered." + + elif name == "browser_control": + r = await loop.run_in_executor( + None, lambda: browser_control(parameters=args, player=self.ui) + ) + result = r or "Browser action completed." + + elif name == "file_controller": + r = await loop.run_in_executor( + None, lambda: file_controller(parameters=args, player=self.ui) + ) + result = r or "File operation completed." + + elif name == "send_message": + r = await loop.run_in_executor( + None, lambda: send_message( + parameters=args, response=None, + player=self.ui, session_memory=None + ) + ) + result = r or f"Message sent to {args.get('receiver')}." + + elif name == "reminder": + r = await loop.run_in_executor( + None, lambda: reminder(parameters=args, response=None, player=self.ui) + ) + result = r or f"Reminder set for {args.get('date')} at {args.get('time')}." + + elif name == "youtube_video": + r = await loop.run_in_executor( + None, lambda: youtube_video(parameters=args, response=None, player=self.ui) + ) + result = r or "Done." + + elif name == "screen_process": + threading.Thread( + target=screen_process, + kwargs={"parameters": args, "response": None, + "player": self.ui, "session_memory": None}, + daemon=True + ).start() + result = ( + "Vision module activated. " + "Stay completely silent — vision module will speak directly." + ) + + elif name == "computer_settings": + r = await loop.run_in_executor( + None, lambda: computer_settings( + parameters=args, response=None, player=self.ui + ) + ) + result = r or "Done." + + elif name == "cmd_control": + r = await loop.run_in_executor( + None, lambda: cmd_control(parameters=args, player=self.ui) + ) + result = r or "Command executed." + + elif name == "desktop_control": + r = await loop.run_in_executor( + None, lambda: desktop_control(parameters=args, player=self.ui) + ) + result = r or "Desktop action completed." + elif name == "code_helper": + r = await loop.run_in_executor( + None, lambda: code_helper( + parameters=args, + player=self.ui, + speak=self.speak + ) + ) + result = r or "Done." + + elif name == "dev_agent": + r = await loop.run_in_executor( + None, lambda: dev_agent( + parameters=args, + player=self.ui, + speak=self.speak + ) + ) + result = r or "Done." + elif name == "agent_task": + goal = args.get("goal", "") + priority_str = args.get("priority", "normal").lower() + + from agent.task_queue import get_queue, TaskPriority + priority_map = { + "low": TaskPriority.LOW, + "normal": TaskPriority.NORMAL, + "high": TaskPriority.HIGH, + } + priority = priority_map.get(priority_str, TaskPriority.NORMAL) + + queue = get_queue() + task_id = queue.submit( + goal=goal, + priority=priority, + speak=self.speak, + ) + result = f"Task started (ID: {task_id}). I'll update you as I make progress, sir." + + elif name == "web_search": + r = await loop.run_in_executor( + None, lambda: web_search_action(parameters=args, player=self.ui) + ) + result = r or "Search completed." + elif name == "computer_control": + r = await loop.run_in_executor( + None, lambda: computer_control(parameters=args, player=self.ui) + ) + result = r or "Done." + + elif name == "flight_finder": + r = await loop.run_in_executor( + None, lambda: flight_finder(parameters=args, player=self.ui) + ) + result = r or "Done." + + else: + result = f"Unknown tool: {name}" + + except Exception as e: + result = f"Tool '{name}' failed: {e}" + traceback.print_exc() + + print(f"[JARVIS] 📤 {name} → {result[:80]}") + + return types.FunctionResponse( + id=fc.id, + name=name, + response={"result": result} + ) + + async def _send_realtime(self): + while True: + msg = await self.out_queue.get() + await self.session.send_realtime_input(media=msg) + + async def _listen_audio(self): + print("[JARVIS] 🎤 Mic started") + stream = await asyncio.to_thread( + pya.open, + format=FORMAT, + channels=CHANNELS, + rate=SEND_SAMPLE_RATE, + input=True, + frames_per_buffer=CHUNK_SIZE, + ) + try: + while True: + data = await asyncio.to_thread( + stream.read, CHUNK_SIZE, exception_on_overflow=False + ) + await self.out_queue.put({"data": data, "mime_type": "audio/pcm"}) + except Exception as e: + print(f"[JARVIS] ❌ Mic error: {e}") + raise + finally: + stream.close() + + async def _receive_audio(self): + print("[JARVIS] 👂 Recv started") + out_buf = [] + in_buf = [] + + try: + while True: + turn = self.session.receive() + async for response in turn: + + if response.data: + self.audio_in_queue.put_nowait(response.data) + + if response.server_content: + sc = response.server_content + + if sc.input_transcription and sc.input_transcription.text: + txt = sc.input_transcription.text.strip() + if txt: + in_buf.append(txt) + + if sc.output_transcription and sc.output_transcription.text: + txt = sc.output_transcription.text.strip() + if txt: + out_buf.append(txt) + + if sc.turn_complete: + full_in = "" + full_out = "" + + if in_buf: + full_in = " ".join(in_buf).strip() + if full_in: + self.ui.write_log(f"You: {full_in}") + in_buf = [] + + if out_buf: + full_out = " ".join(out_buf).strip() + if full_out: + self.ui.write_log(f"Jarvis: {full_out}") + out_buf = [] + + if full_in and len(full_in) > 5: + threading.Thread( + target=_update_memory_async, + args=(full_in, full_out), + daemon=True + ).start() + + if response.tool_call: + fn_responses = [] + for fc in response.tool_call.function_calls: + print(f"[JARVIS] 📞 Tool call: {fc.name}") + fr = await self._execute_tool(fc) + fn_responses.append(fr) + await self.session.send_tool_response( + function_responses=fn_responses + ) + + except Exception as e: + print(f"[JARVIS] ❌ Recv error: {e}") + traceback.print_exc() + raise + + async def _play_audio(self): + print("[JARVIS] 🔊 Play started") + stream = await asyncio.to_thread( + pya.open, + format=FORMAT, + channels=CHANNELS, + rate=RECEIVE_SAMPLE_RATE, + output=True, + ) + try: + while True: + chunk = await self.audio_in_queue.get() + await asyncio.to_thread(stream.write, chunk) + except Exception as e: + print(f"[JARVIS] ❌ Play error: {e}") + raise + finally: + stream.close() + + async def run(self): + client = genai.Client( + api_key=_get_api_key(), + http_options={"api_version": "v1beta"} + ) + + while True: + try: + print("[JARVIS] 🔌 Connecting...") + config = self._build_config() + + async with ( + client.aio.live.connect(model=LIVE_MODEL, config=config) as session, + asyncio.TaskGroup() as tg, + ): + self.session = session + self._loop = asyncio.get_event_loop() + self.audio_in_queue = asyncio.Queue() + self.out_queue = asyncio.Queue(maxsize=10) + + print("[JARVIS] ✅ Connected.") + self.ui.write_log("JARVIS online.") + + tg.create_task(self._send_realtime()) + tg.create_task(self._listen_audio()) + tg.create_task(self._receive_audio()) + tg.create_task(self._play_audio()) + + except Exception as e: + print(f"[JARVIS] ⚠️ Error: {e}") + traceback.print_exc() + + print("[JARVIS] 🔄 Reconnecting in 3s...") + await asyncio.sleep(3) + +def main(): + ui = JarvisUI("face.png") + + def runner(): + ui.wait_for_api_key() + + jarvis = JarvisLive(ui) + try: + asyncio.run(jarvis.run()) + except KeyboardInterrupt: + print("\n🔴 Shutting down...") + + threading.Thread(target=runner, daemon=True).start() + ui.root.mainloop() + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/memory/__init__.py b/memory/__init__.py new file mode 100644 index 0000000..452b936 --- /dev/null +++ b/memory/__init__.py @@ -0,0 +1 @@ +#memory package \ No newline at end of file diff --git a/memory/config_manager.py b/memory/config_manager.py new file mode 100644 index 0000000..dbd3ec3 --- /dev/null +++ b/memory/config_manager.py @@ -0,0 +1,59 @@ +import json +import sys +from pathlib import Path + + +def get_base_dir() -> Path: + if getattr(sys, "frozen", False): + return Path(sys.executable).parent + return Path(__file__).resolve().parent.parent + + +BASE_DIR = get_base_dir() +CONFIG_DIR = BASE_DIR / "config" +CONFIG_FILE = CONFIG_DIR / "api_keys.json" + + +def ensure_config_dir() -> None: + CONFIG_DIR.mkdir(parents=True, exist_ok=True) + + +def config_exists() -> bool: + return CONFIG_FILE.exists() + + +def save_api_keys(gemini_api_key: str) -> None: + ensure_config_dir() + + data: dict = {} + if CONFIG_FILE.exists(): + try: + data = json.loads(CONFIG_FILE.read_text(encoding="utf-8")) + except Exception: + data = {} + + data["gemini_api_key"] = gemini_api_key.strip() + + CONFIG_FILE.write_text( + json.dumps(data, indent=2), + encoding="utf-8" + ) + + +def load_api_keys() -> dict: + if not CONFIG_FILE.exists(): + return {} + try: + return json.loads(CONFIG_FILE.read_text(encoding="utf-8")) + except Exception as e: + print(f"❌ Failed to load api_keys.json: {e}") + return {} + + +def get_gemini_key() -> str | None: + return load_api_keys().get("gemini_api_key") + + +def is_configured() -> bool: + key = get_gemini_key() + return bool(key and len(key) > 15) \ No newline at end of file diff --git a/memory/memory_manager.py b/memory/memory_manager.py new file mode 100644 index 0000000..2bc6327 --- /dev/null +++ b/memory/memory_manager.py @@ -0,0 +1,151 @@ +import json +from threading import Lock +from pathlib import Path +import sys + + +def get_base_dir() -> Path: + if getattr(sys, "frozen", False): + return Path(sys.executable).parent + return Path(__file__).resolve().parent.parent + + +BASE_DIR = get_base_dir() +MEMORY_PATH = BASE_DIR / "memory" / "long_term.json" +_lock = Lock() + +MAX_VALUE_LENGTH = 300 + +def _empty_memory() -> dict: + return { + "identity": {}, + "preferences": {}, + "relationships": {}, + "notes": {} + } + +def load_memory() -> dict: + if not MEMORY_PATH.exists(): + return _empty_memory() + + with _lock: + try: + data = json.loads(MEMORY_PATH.read_text(encoding="utf-8")) + if isinstance(data, dict): + return data + return _empty_memory() + except Exception as e: + print(f"[Memory] ⚠️ Load error: {e}") + return _empty_memory() + + +def save_memory(memory: dict) -> None: + if not isinstance(memory, dict): + return + + MEMORY_PATH.parent.mkdir(parents=True, exist_ok=True) + + with _lock: + MEMORY_PATH.write_text( + json.dumps(memory, indent=2, ensure_ascii=False), + encoding="utf-8" + ) + +def _truncate_value(val: str) -> str: + if isinstance(val, str) and len(val) > MAX_VALUE_LENGTH: + return val[:MAX_VALUE_LENGTH].rstrip() + "…" + return val + + +def _recursive_update(target: dict, updates: dict) -> bool: + changed = False + + for key, value in updates.items(): + if value is None: + continue + if isinstance(value, str) and not value.strip(): + continue + + if isinstance(value, dict) and "value" not in value: + if key not in target or not isinstance(target[key], dict): + target[key] = {} + changed = True + if _recursive_update(target[key], value): + changed = True + else: + if isinstance(value, dict) and "value" in value: + entry = {"value": _truncate_value(str(value["value"]))} + else: + entry = {"value": _truncate_value(str(value))} + + if key not in target or target[key] != entry: + target[key] = entry + changed = True + + return changed + + +def update_memory(memory_update: dict) -> dict: + + if not isinstance(memory_update, dict) or not memory_update: + return load_memory() + + memory = load_memory() + + if _recursive_update(memory, memory_update): + save_memory(memory) + print(f"[Memory] 💾 Saved: {list(memory_update.keys())}") + + return memory + + + +def format_memory_for_prompt(memory: dict | None) -> str: + if not memory: + return "" + + lines = [] + + # Identity + identity = memory.get("identity", {}) + name = identity.get("name", {}).get("value") + age = identity.get("age", {}).get("value") + bday = identity.get("birthday", {}).get("value") + city = identity.get("city", {}).get("value") + if name: lines.append(f"Name: {name}") + if age: lines.append(f"Age: {age}") + if bday: lines.append(f"Birthday: {bday}") + if city: lines.append(f"City: {city}") + + prefs = memory.get("preferences", {}) + for i, (key, entry) in enumerate(prefs.items()): + if i >= 5: + break + val = entry.get("value") if isinstance(entry, dict) else entry + if val: + lines.append(f"{key.replace('_', ' ').title()}: {val}") + + rels = memory.get("relationships", {}) + for i, (key, entry) in enumerate(rels.items()): + if i >= 5: + break + val = entry.get("value") if isinstance(entry, dict) else entry + if val: + lines.append(f"{key.title()}: {val}") + + notes = memory.get("notes", {}) + for i, (key, entry) in enumerate(notes.items()): + if i >= 5: + break + val = entry.get("value") if isinstance(entry, dict) else entry + if val: + lines.append(f"{key}: {val}") + + if not lines: + return "" + + result = "[USER MEMORY]\n" + "\n".join(f"- {l}" for l in lines) + if len(result) > 800: + result = result[:797] + "…" + + return result + "\n" \ No newline at end of file diff --git a/readme.md b/readme.md new file mode 100644 index 0000000..5af02b8 --- /dev/null +++ b/readme.md @@ -0,0 +1,59 @@ +# 🤖 MARK XXX + +**Watch The Detailed Video To Set-up This Model**: https://www.youtube.com/watch?v=-YjbWjv1tJg + +### Next-Generation Personal AI Assistant — By FatihMakes + +A real-time voice AI that can hear, see, understand, and control your Windows computer. +Local execution. Zero subscriptions(Unless you want to increase request by buying requests from Google AI Studio). +Built for intelligent automation. + +## ✨ Overview + +**MARK XXX** is an advanced voice-driven AI assistant designed to turn your computer into an interactive intelligent system. + +Speak naturally — it listens, understands context, responds with a human-like voice, and executes tasks across your system automatically. + +Designed for speed, autonomy, and real-world usability. + + +## 🚀 Capabilities + +* **Real-time voice interaction** — Natural conversation with instant response +* **System control** — Launch apps, manage files, execute commands +* **Autonomous task execution** — Plans and completes multi-step workflows +* **Visual awareness** — Screen analysis and webcam understanding +* **Persistent memory** — Learns preferences and remembers context +* **Integrated tools** — Web search, weather, reminders, messaging, code help, image generation + +## ⚡ Quick Start + +```bash +git clone https://github.com/FatihMakes +cd mark-xxx +python setup.py +python main.py +``` + +Enter your free Gemini API key on first launch. +System ready in minutes. + +If you got some problems or questions to ask or just want to support; + +YouTube Account: [text](https://www.youtube.com/@FatihMakes) +Instagram Account: [text](https://www.instagram.com/fatihmakes/) + +## 📋 Requirements + +* Windows 10 / 11 +* Python 3.10 or newer +* Microphone +* Gemini API key + +## ⚠️ License + +Personal and non-commercial use only. +Licensed under **Creative Commons BY-NC 4.0**. + +Engineered by a 17-year-old building a real JARVIS-style assistant. +⭐ Star the repository to support the project. \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..242bd6d --- /dev/null +++ b/requirements.txt @@ -0,0 +1,19 @@ +pyaudio +google-genai +google-generativeai +pillow +requests +beautifulsoup4 +playwright +pyautogui +pyperclip +opencv-python +numpy +mss +psutil +send2trash +comtypes +pycaw +win10toast +duckduckgo-search +youtube-transcript-api \ No newline at end of file diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..5eb9976 --- /dev/null +++ b/setup.py @@ -0,0 +1,11 @@ +import subprocess +import sys + +print("Installing requirements...") +subprocess.run([sys.executable, "-m", "pip", "install", "-r", "requirements.txt"], check=True) + +print("Installing Playwright browsers...") +subprocess.run([sys.executable, "-m", "playwright", "install"], check=True) + +print("\n✅ Setup complete! Run 'python main.py' to start MARK XXV.") + diff --git a/ui.py b/ui.py new file mode 100644 index 0000000..d67876c --- /dev/null +++ b/ui.py @@ -0,0 +1,380 @@ +import os, json, time, math, random, threading +import tkinter as tk +from collections import deque +from PIL import Image, ImageTk, ImageDraw +import sys +from pathlib import Path + + +def get_base_dir(): + if getattr(sys, "frozen", False): + return Path(sys.executable).parent + return Path(__file__).resolve().parent + + +BASE_DIR = get_base_dir() +CONFIG_DIR = BASE_DIR / "config" +API_FILE = CONFIG_DIR / "api_keys.json" + +SYSTEM_NAME = "J.A.R.V.I.S" +MODEL_BADGE = "MARK XXX" + +C_BG = "#000000" +C_PRI = "#00d4ff" +C_MID = "#007a99" +C_DIM = "#003344" +C_DIMMER = "#001520" +C_ACC = "#ff6600" +C_ACC2 = "#ffcc00" +C_TEXT = "#8ffcff" +C_PANEL = "#010c10" +C_GREEN = "#00ff88" +C_RED = "#ff3333" + + +class JarvisUI: + def __init__(self, face_path, size=None): + self.root = tk.Tk() + self.root.title("J.A.R.V.I.S — MARK XXX") + self.root.resizable(False, False) + + sw = self.root.winfo_screenwidth() + sh = self.root.winfo_screenheight() + W = min(sw, 984) + H = min(sh, 816) + self.root.geometry(f"{W}x{H}+{(sw-W)//2}+{(sh-H)//2}") + self.root.configure(bg=C_BG) + + self.W = W + self.H = H + + self.FACE_SZ = min(int(H * 0.54), 400) + self.FCX = W // 2 + self.FCY = int(H * 0.13) + self.FACE_SZ // 2 + + self.speaking = False + self.scale = 1.0 + self.target_scale = 1.0 + self.halo_a = 60.0 + self.target_halo = 60.0 + self.last_t = time.time() + self.tick = 0 + self.scan_angle = 0.0 + self.scan2_angle = 180.0 + self.rings_spin = [0.0, 120.0, 240.0] + self.pulse_r = [0.0, self.FACE_SZ * 0.26, self.FACE_SZ * 0.52] + self.status_text = "INITIALISING" + self.status_blink = True + + self.typing_queue = deque() + self.is_typing = False + + self._face_pil = None + self._has_face = False + self._face_scale_cache = None + self._load_face(face_path) + + self.bg = tk.Canvas(self.root, width=W, height=H, + bg=C_BG, highlightthickness=0) + self.bg.place(x=0, y=0) + + LW = int(W * 0.72) + LH = 138 + self.log_frame = tk.Frame(self.root, bg=C_PANEL, + highlightbackground=C_MID, + highlightthickness=1) + self.log_frame.place(x=(W - LW) // 2, y=H - LH - 36, width=LW, height=LH) + self.log_text = tk.Text(self.log_frame, fg=C_TEXT, bg=C_PANEL, + insertbackground=C_TEXT, borderwidth=0, + wrap="word", font=("Courier", 10), padx=10, pady=6) + self.log_text.pack(fill="both", expand=True) + self.log_text.configure(state="disabled") + self.log_text.tag_config("you", foreground="#e8e8e8") + self.log_text.tag_config("ai", foreground=C_PRI) + self.log_text.tag_config("sys", foreground=C_ACC2) + + self._api_key_ready = self._api_keys_exist() + if not self._api_key_ready: + self._show_setup_ui() + + self._animate() + self.root.protocol("WM_DELETE_WINDOW", lambda: os._exit(0)) + + def _load_face(self, path): + FW = self.FACE_SZ + try: + img = Image.open(path).convert("RGBA").resize((FW, FW), Image.LANCZOS) + mask = Image.new("L", (FW, FW), 0) + ImageDraw.Draw(mask).ellipse((2, 2, FW - 2, FW - 2), fill=255) + img.putalpha(mask) + self._face_pil = img + self._has_face = True + except Exception: + self._has_face = False + + @staticmethod + def _ac(r, g, b, a): + f = a / 255.0 + return f"#{int(r*f):02x}{int(g*f):02x}{int(b*f):02x}" + + def _animate(self): + self.tick += 1 + t = self.tick + now = time.time() + + if now - self.last_t > (0.14 if self.speaking else 0.55): + if self.speaking: + self.target_scale = random.uniform(1.05, 1.11) + self.target_halo = random.uniform(138, 182) + else: + self.target_scale = random.uniform(1.001, 1.007) + self.target_halo = random.uniform(50, 68) + self.last_t = now + + sp = 0.35 if self.speaking else 0.16 + self.scale += (self.target_scale - self.scale) * sp + self.halo_a += (self.target_halo - self.halo_a) * sp + + for i, spd in enumerate([1.2, -0.8, 1.9] if self.speaking else [0.5, -0.3, 0.82]): + self.rings_spin[i] = (self.rings_spin[i] + spd) % 360 + + self.scan_angle = (self.scan_angle + (2.8 if self.speaking else 1.2)) % 360 + self.scan2_angle = (self.scan2_angle + (-1.7 if self.speaking else -0.68)) % 360 + + pspd = 3.8 if self.speaking else 1.8 + limit = self.FACE_SZ * 0.72 + new_p = [r + pspd for r in self.pulse_r if r + pspd < limit] + if len(new_p) < 3 and random.random() < (0.06 if self.speaking else 0.022): + new_p.append(0.0) + self.pulse_r = new_p + + if t % 40 == 0: + self.status_blink = not self.status_blink + + self._draw() + self.root.after(16, self._animate) + + def _draw(self): + c = self.bg + W, H = self.W, self.H + t = self.tick + FCX = self.FCX + FCY = self.FCY + FW = self.FACE_SZ + c.delete("all") + + for x in range(0, W, 44): + for y in range(0, H, 44): + c.create_rectangle(x, y, x+1, y+1, fill=C_DIMMER, outline="") + + for r in range(int(FW * 0.54), int(FW * 0.28), -22): + frac = 1.0 - (r - FW * 0.28) / (FW * 0.26) + ga = max(0, min(255, int(self.halo_a * 0.09 * frac))) + gh = f"{ga:02x}" + c.create_oval(FCX-r, FCY-r, FCX+r, FCY+r, + outline=f"#00{gh}ff", width=2) + + for pr in self.pulse_r: + pa = max(0, int(220 * (1.0 - pr / (FW * 0.72)))) + r = int(pr) + c.create_oval(FCX-r, FCY-r, FCX+r, FCY+r, + outline=self._ac(0, 212, 255, pa), width=2) + + for idx, (r_frac, w_ring, arc_l, gap) in enumerate([ + (0.47, 3, 110, 75), (0.39, 2, 75, 55), (0.31, 1, 55, 38)]): + ring_r = int(FW * r_frac) + base_a = self.rings_spin[idx] + a_val = max(0, min(255, int(self.halo_a * (1.0 - idx * 0.18)))) + col = self._ac(0, 212, 255, a_val) + for s in range(360 // (arc_l + gap)): + start = (base_a + s * (arc_l + gap)) % 360 + c.create_arc(FCX-ring_r, FCY-ring_r, FCX+ring_r, FCY+ring_r, + start=start, extent=arc_l, + outline=col, width=w_ring, style="arc") + + sr = int(FW * 0.49) + scan_a = min(255, int(self.halo_a * 1.4)) + arc_ext = 70 if self.speaking else 42 + c.create_arc(FCX-sr, FCY-sr, FCX+sr, FCY+sr, + start=self.scan_angle, extent=arc_ext, + outline=self._ac(0, 212, 255, scan_a), width=3, style="arc") + c.create_arc(FCX-sr, FCY-sr, FCX+sr, FCY+sr, + start=self.scan2_angle, extent=arc_ext, + outline=self._ac(255, 100, 0, scan_a // 2), width=2, style="arc") + + t_out = int(FW * 0.495) + t_in = int(FW * 0.472) + a_mk = self._ac(0, 212, 255, 155) + for deg in range(0, 360, 10): + rad = math.radians(deg) + inn = t_in if deg % 30 == 0 else t_in + 5 + c.create_line(FCX + t_out * math.cos(rad), FCY - t_out * math.sin(rad), + FCX + inn * math.cos(rad), FCY - inn * math.sin(rad), + fill=a_mk, width=1) + + ch_r = int(FW * 0.50) + gap = int(FW * 0.15) + ch_a = self._ac(0, 212, 255, int(self.halo_a * 0.55)) + for x1, y1, x2, y2 in [ + (FCX - ch_r, FCY, FCX - gap, FCY), (FCX + gap, FCY, FCX + ch_r, FCY), + (FCX, FCY - ch_r, FCX, FCY - gap), (FCX, FCY + gap, FCX, FCY + ch_r)]: + c.create_line(x1, y1, x2, y2, fill=ch_a, width=1) + + blen = 22 + bc = self._ac(0, 212, 255, 200) + hl = FCX - FW // 2; hr = FCX + FW // 2 + ht = FCY - FW // 2; hb = FCY + FW // 2 + for bx, by, sdx, sdy in [(hl, ht, 1, 1), (hr, ht, -1, 1), + (hl, hb, 1, -1), (hr, hb, -1, -1)]: + c.create_line(bx, by, bx + sdx * blen, by, fill=bc, width=2) + c.create_line(bx, by, bx, by + sdy * blen, fill=bc, width=2) + + if self._has_face: + fw = int(FW * self.scale) + if (self._face_scale_cache is None or + abs(self._face_scale_cache[0] - self.scale) > 0.004): + scaled = self._face_pil.resize((fw, fw), Image.BILINEAR) + tk_img = ImageTk.PhotoImage(scaled) + self._face_scale_cache = (self.scale, tk_img) + c.create_image(FCX, FCY, image=self._face_scale_cache[1]) + else: + orb_r = int(FW * 0.27 * self.scale) + for i in range(7, 0, -1): + r2 = int(orb_r * i / 7) + frac = i / 7 + ga = max(0, min(255, int(self.halo_a * 1.1 * frac))) + c.create_oval(FCX-r2, FCY-r2, FCX+r2, FCY+r2, + fill=self._ac(0, int(65*frac), int(120*frac), ga), + outline="") + c.create_text(FCX, FCY, text=SYSTEM_NAME, + fill=self._ac(0, 212, 255, min(255, int(self.halo_a * 2))), + font=("Courier", 14, "bold")) + + HDR = 62 + c.create_rectangle(0, 0, W, HDR, fill="#00080d", outline="") + c.create_line(0, HDR, W, HDR, fill=C_MID, width=1) + c.create_text(W // 2, 22, text=SYSTEM_NAME, + fill=C_PRI, font=("Courier", 18, "bold")) + c.create_text(W // 2, 44, text="Just A Rather Very Intelligent System", + fill=C_MID, font=("Courier", 9)) + c.create_text(16, 31, text=MODEL_BADGE, + fill=C_DIM, font=("Courier", 9), anchor="w") + c.create_text(W - 16, 31, text=time.strftime("%H:%M:%S"), + fill=C_PRI, font=("Courier", 14, "bold"), anchor="e") + + + sy = FCY + FW // 2 + 45 + if self.speaking: + stat, sc = "● SPEAKING", C_ACC + else: + sym = "●" if self.status_blink else "○" + stat, sc = f"{sym} {self.status_text}", C_PRI + + c.create_text(W // 2, sy, text=stat, + fill=sc, font=("Courier", 11, "bold")) + + wy = sy + 22 + N = 32 + BH = 18 + bw = 8 + total_w = N * bw + wx0 = (W - total_w) // 2 + for i in range(N): + hb = random.randint(3, BH) if self.speaking else int(3 + 2 * math.sin(t * 0.08 + i * 0.55)) + col = (C_PRI if hb > BH * 0.6 else C_MID) if self.speaking else C_DIM + bx = wx0 + i * bw + c.create_rectangle(bx, wy + BH - hb, bx + bw - 1, wy + BH, + fill=col, outline="") + + c.create_rectangle(0, H - 28, W, H, fill="#00080d", outline="") + c.create_line(0, H - 28, W, H - 28, fill=C_DIM, width=1) + c.create_text(W // 2, H - 14, fill=C_DIM, font=("Courier", 8), + text="FatihMakes Industries · CLASSIFIED · MARK XXX") + + def write_log(self, text: str): + self.typing_queue.append(text) + tl = text.lower() + self.status_text = ("PROCESSING" if tl.startswith("you:") + else "RESPONDING" if tl.startswith("ai:") + else self.status_text) + if not self.is_typing: + self._start_typing() + + def _start_typing(self): + if not self.typing_queue: + self.is_typing = False + if not self.speaking: + self.status_text = "ONLINE" + return + self.is_typing = True + text = self.typing_queue.popleft() + tl = text.lower() + tag = "you" if tl.startswith("you:") else "ai" if tl.startswith("ai:") else "sys" + self.log_text.configure(state="normal") + self._type_char(text, 0, tag) + + def _type_char(self, text, i, tag): + if i < len(text): + self.log_text.insert(tk.END, text[i], tag) + self.log_text.see(tk.END) + self.root.after(8, self._type_char, text, i + 1, tag) + else: + self.log_text.insert(tk.END, "\n") + self.log_text.configure(state="disabled") + self.root.after(25, self._start_typing) + + def start_speaking(self): + self.speaking = True + self.status_text = "SPEAKING" + + def stop_speaking(self): + self.speaking = False + self.status_text = "ONLINE" + + def _api_keys_exist(self): + return API_FILE.exists() + + def wait_for_api_key(self): + """Block until API key is saved (called from main thread before starting JARVIS).""" + while not self._api_key_ready: + time.sleep(0.1) + + def _show_setup_ui(self): + self.setup_frame = tk.Frame( + self.root, bg="#00080d", + highlightbackground=C_PRI, highlightthickness=1 + ) + self.setup_frame.place(relx=0.5, rely=0.5, anchor="center") + + tk.Label(self.setup_frame, text="◈ INITIALISATION REQUIRED", + fg=C_PRI, bg="#00080d", font=("Courier", 13, "bold")).pack(pady=(18, 4)) + tk.Label(self.setup_frame, + text="Enter your Gemini API key to boot J.A.R.V.I.S.", + fg=C_MID, bg="#00080d", font=("Courier", 9)).pack(pady=(0, 10)) + + tk.Label(self.setup_frame, text="GEMINI API KEY", + fg=C_DIM, bg="#00080d", font=("Courier", 9)).pack(pady=(8, 2)) + self.gemini_entry = tk.Entry( + self.setup_frame, width=52, fg=C_TEXT, bg="#000d12", + insertbackground=C_TEXT, borderwidth=0, font=("Courier", 10), show="*" + ) + self.gemini_entry.pack(pady=(0, 4)) + + tk.Button( + self.setup_frame, text="▸ INITIALISE SYSTEMS", + command=self._save_api_keys, bg=C_BG, fg=C_PRI, + activebackground="#003344", font=("Courier", 10), + borderwidth=0, pady=8 + ).pack(pady=14) + + def _save_api_keys(self): + gemini = self.gemini_entry.get().strip() + if not gemini: + return + os.makedirs(CONFIG_DIR, exist_ok=True) + with open(API_FILE, "w", encoding="utf-8") as f: + json.dump({"gemini_api_key": gemini}, f, indent=4) + self.setup_frame.destroy() + self._api_key_ready = True + self.status_text = "ONLINE" + self.write_log("SYS: Systems initialised. JARVIS online.") \ No newline at end of file