Add files via upload

This commit is contained in:
FatihMakes authored and GitHub committed 2026-03-02 04:30:10 +03:00
commit a8674a2874
30 files changed
+8659

No files matched your search

+476
View File
@@ -0,0 +1,476 @@
import asyncio
import threading
import concurrent.futures
import platform
import shutil
import subprocess
from pathlib import Path
from playwright.async_api import async_playwright, TimeoutError as PlaywrightTimeout
def _get_default_browser_id() -> str:
"""Returns raw default browser identifier string for current OS."""
system = platform.system()
try:
if system == "Windows":
import winreg
key = winreg.OpenKey(
winreg.HKEY_CURRENT_USER,
r"Software\Microsoft\Windows\Shell\Associations\UrlAssociations\http\UserChoice"
)
prog_id = winreg.QueryValueEx(key, "ProgId")[0].lower()
winreg.CloseKey(key)
return prog_id
elif system == "Darwin":
result = subprocess.run(
["defaults", "read",
"com.apple.LaunchServices/com.apple.launchservices.secure",
"LSHandlers"],
capture_output=True, text=True, timeout=5
)
return result.stdout.lower()
elif system == "Linux":
result = subprocess.run(
["xdg-settings", "get", "default-web-browser"],
capture_output=True, text=True, timeout=5
)
return result.stdout.lower()
except Exception:
pass
return ""
_BROWSER_BINARIES = {
"Windows": {
"opera": ["opera.exe"],
"brave": ["brave.exe"],
"vivaldi": ["vivaldi.exe"],
"chrome": ["chrome.exe"],
"firefox": ["firefox.exe"],
},
"Darwin": {
"opera": ["opera"],
"brave": ["brave browser", "brave"],
"vivaldi": ["vivaldi"],
"chrome": ["google chrome", "google-chrome"],
"firefox": ["firefox"],
},
"Linux": {
"opera": ["opera", "opera-stable"],
"brave": ["brave-browser", "brave"],
"vivaldi": ["vivaldi-stable", "vivaldi"],
"chrome": ["google-chrome", "google-chrome-stable", "chromium-browser", "chromium"],
"firefox": ["firefox"],
},
}
def _get_opera_executable() -> str | None:
if platform.system() != "Windows":
return None
try:
import winreg
candidate_keys = [
r"SOFTWARE\Microsoft\Windows\CurrentVersion\App Paths\opera.exe",
r"SOFTWARE\Microsoft\Windows\CurrentVersion\App Paths\launcher.exe",
r"SOFTWARE\Clients\StartMenuInternet\OperaStable\shell\open\command",
r"SOFTWARE\Clients\StartMenuInternet\OperaGXStable\shell\open\command",
]
for key_path in candidate_keys:
for hive in [winreg.HKEY_LOCAL_MACHINE, winreg.HKEY_CURRENT_USER]:
try:
key = winreg.OpenKey(hive, key_path)
val = winreg.QueryValue(key, None)
winreg.CloseKey(key)
# Strip quotes and args
exe = val.strip().strip('"').split('"')[0].split(" --")[0].strip()
if exe and Path(exe).exists():
print(f"[Browser] 🔍 Opera found via registry: {exe}")
return exe
except Exception:
continue
except Exception:
pass
return None
def _find_browser_executable(prog_id: str) -> tuple:
system = platform.system()
os_bins = _BROWSER_BINARIES.get(system, {})
if any(x in prog_id for x in ["firefox", "mozilla"]):
return "firefox", None, None
if "safari" in prog_id:
return "webkit", None, None
if "edge" in prog_id:
return "chromium", None, "msedge"
if "opera" in prog_id:
exe = _get_opera_executable()
if exe:
return "chromium", exe, None
for binary in os_bins.get("opera", []):
path = shutil.which(binary)
if path:
return "chromium", path, None
browser_patterns = {
"brave": ["brave"],
"vivaldi": ["vivaldi"],
"chrome": ["chrome"],
}
for browser_name, patterns in browser_patterns.items():
if not any(p in prog_id for p in patterns):
continue
binaries = os_bins.get(browser_name, [])
for binary in binaries:
path = shutil.which(binary)
if path:
print(f"[Browser] 🔍 Found {browser_name} at: {path}")
return "chromium", path, None
if "chrome" in prog_id or not prog_id:
return "chromium", None, "chrome"
return "chromium", None, None
class _BrowserThread:
def __init__(self):
self._loop = None
self._thread = None
self._ready = threading.Event()
self._playwright = None
self._browser = None
self._context = None
self._page = None
def start(self):
if self._thread and self._thread.is_alive():
return
self._thread = threading.Thread(
target=self._run_loop, daemon=True, name="BrowserThread"
)
self._thread.start()
self._ready.wait(timeout=15)
def _run_loop(self):
self._loop = asyncio.new_event_loop()
asyncio.set_event_loop(self._loop)
self._loop.run_until_complete(self._init())
self._ready.set()
self._loop.run_forever()
async def _init(self):
self._playwright = await async_playwright().start()
def run(self, coro, timeout: int = 30):
if not self._loop:
raise RuntimeError("BrowserThread not started.")
future = asyncio.run_coroutine_threadsafe(coro, self._loop)
return future.result(timeout=timeout)
async def _get_page(self):
if self._page is None or self._page.is_closed():
await self._launch()
return self._page
async def _launch(self):
prog_id = _get_default_browser_id()
engine_name, exe_path, channel = _find_browser_executable(prog_id)
engine = getattr(self._playwright, engine_name)
launch_kwargs = {"headless": False}
if engine_name == "chromium":
launch_kwargs["args"] = ["--start-maximized"]
if exe_path:
launch_kwargs["executable_path"] = exe_path
elif channel:
launch_kwargs["channel"] = channel
try:
if self._browser is None or not self._browser.is_connected():
self._browser = await engine.launch(**launch_kwargs)
print(
f"[Browser] ✅ Launched ({engine_name}"
f"{' / ' + channel if channel else ''}"
f"{' / ' + exe_path if exe_path else ''})"
)
except Exception as e:
print(f"[Browser] ⚠️ Launch failed ({e}), falling back to built-in Chromium")
self._browser = await self._playwright.chromium.launch(
headless=False,
args=["--start-maximized"]
)
self._context = await self._browser.new_context(
viewport=None,
user_agent=(
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
"AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/120.0.0.0 Safari/537.36"
)
)
self._page = await self._context.new_page()
async def _close(self):
if self._browser:
await self._browser.close()
self._browser = None
self._page = None
if self._playwright:
await self._playwright.stop()
self._playwright = None
async def _go_to(self, url: str) -> str:
if not url.startswith("http"):
url = "https://" + url
page = await self._get_page()
try:
await page.goto(url, wait_until="domcontentloaded", timeout=15000)
return f"Opened: {page.url}"
except PlaywrightTimeout:
return f"Timeout loading: {url}"
except Exception as e:
return f"Navigation error: {e}"
async def _search(self, query: str, engine: str = "google") -> str:
engines = {
"google": f"https://www.google.com/search?q={query.replace(' ', '+')}",
"bing": f"https://www.bing.com/search?q={query.replace(' ', '+')}",
"duckduckgo": f"https://duckduckgo.com/?q={query.replace(' ', '+')}",
}
url = engines.get(engine.lower(), engines["google"])
return await self._go_to(url)
async def _click(self, selector=None, text=None) -> str:
page = await self._get_page()
try:
if text:
await page.get_by_text(text, exact=False).first.click(timeout=8000)
return f"Clicked: '{text}'"
elif selector:
await page.click(selector, timeout=8000)
return f"Clicked: {selector}"
return "No selector or text provided."
except PlaywrightTimeout:
return "Element not found or not clickable."
except Exception as e:
return f"Click error: {e}"
async def _type(self, selector=None, text: str = "", clear_first: bool = True) -> str:
page = await self._get_page()
try:
element = page.locator(selector).first if selector else page.locator(":focus")
if clear_first:
await element.clear()
await element.type(text, delay=50)
return "Text typed."
except Exception as e:
return f"Type error: {e}"
async def _scroll(self, direction: str = "down", amount: int = 500) -> str:
page = await self._get_page()
try:
y = amount if direction == "down" else -amount
await page.mouse.wheel(0, y)
return f"Scrolled {direction}."
except Exception as e:
return f"Scroll error: {e}"
async def _press(self, key: str) -> str:
page = await self._get_page()
try:
await page.keyboard.press(key)
return f"Pressed: {key}"
except Exception as e:
return f"Key error: {e}"
async def _get_text(self) -> str:
page = await self._get_page()
try:
text = await page.inner_text("body")
return text[:4000] if len(text) > 4000 else text
except Exception as e:
return f"Could not get page text: {e}"
async def _fill_form(self, fields: dict) -> str:
page = await self._get_page()
results = []
for selector, value in fields.items():
try:
el = page.locator(selector).first
await el.clear()
await el.type(str(value), delay=40)
results.append(f"✓ {selector}")
except Exception as e:
results.append(f"✗ {selector}: {e}")
return "Form filled: " + ", ".join(results)
async def _smart_click(self, description: str) -> str:
page = await self._get_page()
desc_lower = description.lower()
role_hints = {
"button": ["button", "buton", "btn"],
"link": ["link", "bağlantı"],
"searchbox": ["search", "arama"],
"textbox": ["input", "field", "alan"],
}
for role, keywords in role_hints.items():
if any(k in desc_lower for k in keywords):
try:
await page.get_by_role(role).first.click(timeout=5000)
return f"Clicked ({role}): '{description}'"
except Exception:
pass
try:
await page.get_by_text(description, exact=False).first.click(timeout=5000)
return f"Clicked (text): '{description}'"
except Exception:
pass
try:
await page.get_by_placeholder(description, exact=False).first.click(timeout=5000)
return f"Clicked (placeholder): '{description}'"
except Exception:
pass
return f"Could not find: '{description}'"
async def _smart_type(self, description: str, text: str) -> str:
page = await self._get_page()
for method, locator in [
("placeholder", page.get_by_placeholder(description, exact=False)),
("label", page.get_by_label(description, exact=False)),
("role", page.get_by_role("textbox")),
]:
try:
el = locator.first
await el.clear()
await el.type(text, delay=50)
return f"Typed into ({method}): '{description}'"
except Exception:
continue
return f"Could not find input: '{description}'"
async def _close_browser(self) -> str:
await self._close()
return "Browser closed."
_bt = _BrowserThread()
_bt_started = False
_bt_lock = threading.Lock()
def _ensure_started():
global _bt_started
with _bt_lock:
if not _bt_started:
_bt.start()
_bt_started = True
def browser_control(
parameters: dict,
response=None,
player=None,
session_memory=None
) -> str:
"""
Browser controller — auto-detects and uses system default browser.
parameters:
action : go_to | search | click | type | scroll | fill_form |
smart_click | smart_type | get_text | press | close
url : URL for go_to
query : search query
engine : google | bing | duckduckgo (default: google)
selector : CSS selector for click/type
text : text to click or type
description : element description for smart_click/smart_type
direction : up | down for scroll
amount : scroll amount in pixels (default: 500)
key : key name for press (e.g. Enter, Escape, Tab)
fields : {selector: value} dict for fill_form
clear_first : bool, clear input before typing (default: True)
"""
_ensure_started()
action = (parameters or {}).get("action", "").lower().strip()
result = "Unknown action."
try:
if action == "go_to":
result = _bt.run(_bt._go_to(parameters.get("url", "")))
elif action == "search":
result = _bt.run(_bt._search(
parameters.get("query", ""),
parameters.get("engine", "google")
))
elif action == "click":
result = _bt.run(_bt._click(
selector=parameters.get("selector"),
text=parameters.get("text")
))
elif action == "type":
result = _bt.run(_bt._type(
selector=parameters.get("selector"),
text=parameters.get("text", ""),
clear_first=parameters.get("clear_first", True)
))
elif action == "scroll":
result = _bt.run(_bt._scroll(
direction=parameters.get("direction", "down"),
amount=parameters.get("amount", 500)
))
elif action == "fill_form":
result = _bt.run(_bt._fill_form(parameters.get("fields", {})))
elif action == "smart_click":
result = _bt.run(_bt._smart_click(parameters.get("description", "")))
elif action == "smart_type":
result = _bt.run(_bt._smart_type(
parameters.get("description", ""),
parameters.get("text", "")
))
elif action == "get_text":
result = _bt.run(_bt._get_text())
elif action == "press":
result = _bt.run(_bt._press(parameters.get("key", "Enter")))
elif action == "close":
result = _bt.run(_bt._close_browser())
else:
result = f"Unknown action: {action}"
except concurrent.futures.TimeoutError:
result = "Browser action timed out."
except Exception as e:
result = f"Browser error: {e}"
print(f"[Browser] {result[:80]}")
if player:
player.write_log(f"[browser] {result[:60]}")
return result
+230
View File
@@ -0,0 +1,230 @@
import subprocess
import sys
import json
import re
from pathlib import Path
def get_base_dir():
if getattr(sys, "frozen", False):
return Path(sys.executable).parent
return Path(__file__).resolve().parent.parent
BASE_DIR = get_base_dir()
API_CONFIG_PATH = BASE_DIR / "config" / "api_keys.json"
def _get_api_key() -> str:
with open(API_CONFIG_PATH, "r", encoding="utf-8") as f:
return json.load(f)["gemini_api_key"]
def _get_platform() -> str:
if sys.platform == "win32": return "windows"
if sys.platform == "darwin": return "macos"
return "linux"
WIN_COMMAND_MAP = [
(["disk space", "disk usage", "storage", "free space", "c drive space"],
"wmic logicaldisk get caption,freespace,size /format:list", False),
(["running processes", "list processes", "show processes", "active processes", "tasklist"],
"tasklist /fo table", False),
(["ip address", "my ip", "network info", "ipconfig"],
"ipconfig /all", False),
(["ping", "internet connection", "connected to internet"],
"ping -n 4 google.com", False),
(["open ports", "listening ports", "netstat"],
"netstat -an | findstr LISTENING", False),
(["wifi networks", "available wifi", "wireless networks"],
"netsh wlan show networks", False),
(["system info", "computer info", "hardware info", "pc info", "specs"],
"systeminfo", False),
(["cpu usage", "processor usage"],
"wmic cpu get loadpercentage", False),
(["memory usage", "ram usage"],
"wmic OS get FreePhysicalMemory,TotalVisibleMemorySize /Value", False),
(["windows version", "os version"],
"ver", False),
(["installed programs", "installed software", "installed apps"],
"wmic product get name,version /format:table", False),
(["battery", "battery level", "power status"],
"powershell (Get-WmiObject -Class Win32_Battery).EstimatedChargeRemaining", False),
(["current time", "what time", "system time"],
"time /t", False),
(["current date", "what date", "system date"],
"date /t", False),
(["desktop files", "files on desktop"],
f'dir "{Path.home() / "Desktop"}"', False),
(["downloads", "files in downloads"],
f'dir "{Path.home() / "Downloads"}"', False),
(["large files", "biggest files", "largest files"],
'powershell "Get-ChildItem C:\\ -Recurse -ErrorAction SilentlyContinue | Sort-Object Length -Descending | Select-Object -First 10 FullName,Length | Format-Table"', False),
]
def _find_hardcoded(task: str) -> str | None:
task_lower = task.lower()
if "notepad" in task_lower or any(ext in task_lower for ext in [".txt", ".log", ".md", ".csv"]):
file_match = re.search(r'[\"\']?([\S]+\.(?:txt|log|md|csv|json|xml))[\"\']?', task, re.IGNORECASE)
if file_match:
filename = file_match.group(1)
desktop = Path.home() / "Desktop"
filepath = Path(filename) if Path(filename).is_absolute() else desktop / filename
return f'notepad "{filepath}"'
if "notepad" in task_lower:
return "notepad"
pip_match = re.search(r"install\s+([\w\-]+)", task_lower)
if pip_match:
package = pip_match.group(1)
return f"pip install {package}"
for keywords, command, _ in WIN_COMMAND_MAP:
if command and any(kw in task_lower for kw in keywords):
return command
return None
BLOCKED_PATTERNS = [
r"\brm\s+-rf\b", r"\brmdir\s+/s\b", r"\bdel\s+/[fqs]",
r"\bformat\b", r"\bdiskpart\b", r"\bfdisk\b",
r"\breg\s+(delete|add)\b", r"\bbcdedit\b",
r"\bnet\s+localgroup\b",
r"\bshutdown\b", r"\brestart-computer\b",
r"\bstop-process\b", r"\bkill\s+-9\b", r"\btaskkill\b",
r"\beval\b", r"\b__import__\b",
]
_BLOCKED_RE = re.compile("|".join(BLOCKED_PATTERNS), re.IGNORECASE)
def _is_safe(command: str) -> tuple[bool, str]:
match = _BLOCKED_RE.search(command)
if match:
return False, f"Blocked pattern: '{match.group()}'"
return True, "OK"
def _ask_gemini(task: str) -> str:
try:
import google.generativeai as genai
genai.configure(api_key=_get_api_key())
model = genai.GenerativeModel("gemini-2.5-flash-lite")
prompt = (
f"Convert this request to a single Windows CMD command.\n"
f"Output ONLY the command. No explanation, no markdown, no backticks.\n"
f"If unsafe or impossible, output: UNSAFE\n\n"
f"Request: {task}\n\nCommand:"
)
response = model.generate_content(prompt)
command = response.text.strip().strip("`").strip()
if command.startswith("```"):
lines = command.split("\n")
command = "\n".join(lines[1:-1]).strip()
return command
except Exception as e:
return f"ERROR: {e}"
def _run_silent(command: str, timeout: int = 20) -> str:
try:
platform = _get_platform()
if platform == "windows":
is_ps = command.strip().lower().startswith("powershell")
if is_ps:
cmd_inner = re.sub(r'^powershell\s+"?', '', command, flags=re.IGNORECASE).rstrip('"')
result = subprocess.run(
["powershell", "-NoProfile", "-Command", cmd_inner],
capture_output=True, text=True,
encoding="utf-8", errors="replace", timeout=timeout
)
else:
result = subprocess.run(
["cmd", "/c", command],
capture_output=True, text=True,
encoding="cp1252", errors="replace",
timeout=timeout, cwd=str(Path.home())
)
else:
shell = "/bin/zsh" if platform == "macos" else "/bin/bash"
result = subprocess.run(
command, shell=True, executable=shell,
capture_output=True, text=True,
errors="replace", timeout=timeout,
cwd=str(Path.home())
)
output = result.stdout.strip()
error = result.stderr.strip()
if output: return output[:2000]
if error: return f"[stderr]: {error[:500]}"
return "Command executed with no output."
except subprocess.TimeoutExpired:
return f"Command timed out after {timeout}s."
except Exception as e:
return f"Execution error: {e}"
def _run_visible(command: str) -> None:
try:
platform = _get_platform()
if platform == "windows":
subprocess.Popen(
f'cmd /k "{command}"',
creationflags=subprocess.CREATE_NEW_CONSOLE
)
elif platform == "macos":
subprocess.Popen(["osascript", "-e",
f'tell application "Terminal" to do script "{command}"'])
else:
for term in ["gnome-terminal", "xterm", "konsole"]:
try:
subprocess.Popen([term, "--", "bash", "-c", f"{command}; exec bash"])
break
except FileNotFoundError:
continue
except Exception as e:
print(f"[CMD] ⚠️ Terminal open failed: {e}")
def cmd_control(
parameters: dict,
response=None,
player=None,
session_memory=None
) -> str:
task = (parameters or {}).get("task", "").strip()
command = (parameters or {}).get("command", "").strip()
visible = (parameters or {}).get("visible", True)
if not task and not command:
return "Please describe what you want to do, sir."
if not command:
command = _find_hardcoded(task)
if command:
print(f"[CMD] ⚡ Hardcoded: {command[:80]}")
else:
print(f"[CMD] 🤖 Gemini fallback for: {task}")
command = _ask_gemini(task)
print(f"[CMD] ✅ Generated: {command[:80]}")
if command == "UNSAFE":
return "I cannot generate a safe command for that request, sir."
if command.startswith("ERROR:"):
return f"Could not generate command: {command}"
safe, reason = _is_safe(command)
if not safe:
return f"Blocked for safety: {reason}"
if player:
player.write_log(f"[CMD] {command[:60]}")
if any(x in command.lower() for x in ["notepad", "explorer", "start "]):
subprocess.Popen(command, shell=True)
return f"Opened: {command}"
if visible:
_run_visible(command)
output = _run_silent(command)
return f"Terminal opened.\n\nOutput:\n{output}"
else:
return _run_silent(command)
+583
View File
@@ -0,0 +1,583 @@
# actions/code_helper.py
# AI-powered code assistant — writes, edits, explains, runs, builds, debugs, and optimizes code.
#
# Actions:
# write → Describe what you want, Gemini writes it, saves to file
# edit → Read existing file, apply natural language change
# explain → Explain what a piece of code or file does
# run → Execute a script file, return output
# build → Write → Run → Fix loop (max 3 attempts), speaks when done
# screen_debug → Screenshot al, ekrandaki kodu/hatayı Gemini ile analiz et ve düzelt
# optimize → Mevcut kodu Gemini ile optimize et (performans, okunabilirlik, best practices)
# auto → (default) Intent auto-detected from context
import subprocess
import sys
import json
import re
import time
from pathlib import Path
def get_base_dir():
if getattr(sys, "frozen", False):
return Path(sys.executable).parent
return Path(__file__).resolve().parent.parent
BASE_DIR = get_base_dir()
API_CONFIG_PATH = BASE_DIR / "config" / "api_keys.json"
DESKTOP = Path.home() / "Desktop"
MAX_BUILD_ATTEMPTS = 3
GEMINI_MODEL = "gemini-2.5-flash"
def _get_api_key() -> str:
with open(API_CONFIG_PATH, "r", encoding="utf-8") as f:
return json.load(f)["gemini_api_key"]
def _get_gemini(model: str = GEMINI_MODEL):
import google.generativeai as genai
genai.configure(api_key=_get_api_key())
return genai.GenerativeModel(model)
def _clean_code(text: str) -> str:
text = text.strip()
text = re.sub(r"^```[a-zA-Z]*\n?", "", text)
text = re.sub(r"\n?```$", "", text)
return text.strip()
def _resolve_save_path(output_path: str, language: str) -> Path:
ext_map = {
"python": ".py", "py": ".py",
"javascript": ".js", "js": ".js",
"typescript": ".ts", "ts": ".ts",
"html": ".html", "css": ".css",
"java": ".java", "cpp": ".cpp", "c": ".c",
"bash": ".sh", "shell": ".sh", "powershell": ".ps1",
"sql": ".sql", "json": ".json", "rust": ".rs", "go": ".go",
}
if output_path:
p = Path(output_path)
return p if p.is_absolute() else DESKTOP / p
ext = ext_map.get((language or "python").lower(), ".py")
return DESKTOP / f"jarvis_code{ext}"
def _read_file(file_path: str) -> tuple[str, str]:
if not file_path:
return "", "No file path provided."
p = Path(file_path)
if not p.exists():
return "", f"File not found: {file_path}"
try:
return p.read_text(encoding="utf-8"), ""
except Exception as e:
return "", f"Could not read file: {e}"
def _save_file(path: Path, content: str) -> str:
try:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(content, encoding="utf-8")
return f"Saved to: {path}"
except Exception as e:
return f"Could not save: {e}"
def _preview(code: str, lines: int = 10) -> str:
all_lines = code.splitlines()
preview = "\n".join(all_lines[:lines])
suffix = f"\n... ({len(all_lines) - lines} more lines)" if len(all_lines) > lines else ""
return preview + suffix
def _has_error(output: str) -> bool:
error_signals = ["error", "exception", "traceback", "syntaxerror",
"nameerror", "typeerror", "stderr", "failed", "crash"]
return any(s in output.lower() for s in error_signals)
def _take_screenshot() -> Path | None:
try:
import pyautogui
screenshot_path = Path.home() / "Desktop" / f"jarvis_debug_{int(time.time())}.png"
screenshot = pyautogui.screenshot()
screenshot.save(str(screenshot_path))
print(f"[Code] 📸 Screenshot: {screenshot_path}")
return screenshot_path
except Exception as e:
print(f"[Code] ⚠️ Screenshot failed: {e}")
return None
def _image_to_base64(path: Path) -> str:
import base64
return base64.b64encode(path.read_bytes()).decode("utf-8")
def _detect_intent(description: str, file_path: str, code: str) -> str:
desc = (description or "").lower()
screen_kw = ["ekrandaki", "screen", "ekranda", "bu hatayı", "why am i getting",
"neden hata", "what's wrong", "ne yanlış", "screenshot", "görüntü"]
if any(k in desc for k in screen_kw):
return "screen_debug"
# Optimize keywords
optimize_kw = ["optimize", "refactor", "clean up", "improve", "temizle",
"iyileştir", "daha iyi", "make it better", "hızlandır"]
if any(k in desc for k in optimize_kw) and (code or file_path):
return "optimize"
if file_path:
p = Path(file_path)
edit_kw = ["edit", "update", "modify", "change", "add", "remove",
"refactor", "fix", "rename", "replace", "düzenle", "değiştir"]
run_kw = ["run", "execute", "launch", "start", "çalıştır"]
build_kw = ["build", "make it work", "try", "attempt"]
if p.exists() and any(k in desc for k in edit_kw):
return "edit"
if p.exists() and any(k in desc for k in run_kw):
return "run"
if any(k in desc for k in build_kw):
return "build"
if p.exists():
return "explain"
explain_kw = ["explain", "what does", "describe", "analyze", "açıkla", "ne yapıyor"]
if any(k in desc for k in explain_kw) and (code or file_path):
return "explain"
build_kw = ["build", "make it work", "try and", "attempt"]
if any(k in desc for k in build_kw):
return "build"
return "write"
def _write(description: str, language: str, output_path: str, player=None) -> tuple[str, Path]:
lang = language or "python"
model = _get_gemini()
prompt = f"""You are an expert {lang} developer.
Write clean, working, well-commented {lang} code for the description below.
Rules:
- Output ONLY the code. No explanation, no markdown, no backticks.
- Add helpful inline comments.
- Handle errors and edge cases properly.
- Use modern best practices.
Description: {description}
Code:"""
response = model.generate_content(prompt)
code = _clean_code(response.text)
path = _resolve_save_path(output_path, lang)
_save_file(path, code)
return code, path
def _fix_code(code: str, error_output: str, description: str) -> str:
model = _get_gemini()
prompt = f"""You are an expert debugger.
The code below failed with the following error. Fix it.
Return ONLY the corrected code — no explanation, no markdown, no backticks.
Original goal: {description}
Error:
{error_output[:2000]}
Broken code:
{code}
Fixed code:"""
response = model.generate_content(prompt)
return _clean_code(response.text)
def _run_file(path: Path, args: list, timeout: int) -> str:
interpreters = {
".py": [sys.executable],
".js": ["node"],
".ts": ["ts-node"],
".sh": ["bash"],
".ps1": ["powershell", "-File"],
".rb": ["ruby"],
".php": ["php"],
}
interp = interpreters.get(path.suffix.lower())
if not interp:
return f"No interpreter for {path.suffix}."
try:
result = subprocess.run(
interp + [str(path)] + (args or []),
capture_output=True, text=True,
encoding="utf-8", errors="replace",
timeout=timeout, cwd=str(path.parent)
)
output = result.stdout.strip()
error = result.stderr.strip()
parts = []
if output: parts.append(f"Output:\n{output}")
if error: parts.append(f"Stderr:\n{error}")
return "\n\n".join(parts) if parts else "Executed with no output."
except subprocess.TimeoutExpired:
return f"Timed out after {timeout}s."
except FileNotFoundError:
return f"Interpreter not found: {interp[0]}."
except Exception as e:
return f"Execution error: {e}"
def _build(description, language, output_path, args, timeout, speak=None, player=None) -> str:
if not description:
return "Please describe what you want me to build, sir."
if player:
player.write_log("[Code] Build started...")
lang = language or "python"
try:
code, path = _write(description, lang, output_path, player)
print(f"[Code] ✅ Written: {path}")
except Exception as e:
msg = f"Could not write initial code: {e}"
if speak: speak(msg)
return msg
last_output = ""
for attempt in range(1, MAX_BUILD_ATTEMPTS + 1):
print(f"[Code] 🔄 Attempt {attempt}/{MAX_BUILD_ATTEMPTS}")
if player:
player.write_log(f"[Code] Attempt {attempt}...")
last_output = _run_file(path, args, timeout)
if not _has_error(last_output):
msg = (
f"Build complete, sir. "
f"The code is working after {attempt} attempt{'s' if attempt > 1 else ''}. "
f"Saved to {path}."
)
if speak: speak(msg)
return f"{msg}\n\nOutput:\n{last_output}"
print(f"[Code] ⚠️ Error on attempt {attempt}, fixing...")
if player:
player.write_log(f"[Code] Fixing (attempt {attempt})...")
try:
code = _fix_code(code, last_output, description)
_save_file(path, code)
except Exception as e:
msg = f"Could not fix code on attempt {attempt}: {e}"
if speak: speak(msg)
return msg
msg = (
f"I was unable to build a working version after {MAX_BUILD_ATTEMPTS} attempts, sir. "
f"The last error was: {last_output[:200]}"
)
if speak: speak(msg)
return f"{msg}\n\nLast code saved to: {path}"
def _write_action(description, language, output_path, player) -> str:
if not description:
return "Please describe what you want me to write, sir."
if player:
player.write_log("[Code] Writing code...")
try:
code, path = _write(description, language, output_path, player)
print(f"[Code] ✅ Written: {path}")
return f"Code written. Saved to: {path}\n\nPreview:\n{_preview(code)}"
except Exception as e:
return f"Could not generate code: {e}"
def _edit_action(file_path, instruction, player) -> str:
if not file_path:
return "Please provide a file path to edit, sir."
if not instruction:
return "Please describe what change to make, sir."
content, err = _read_file(file_path)
if err:
return err
if player:
player.write_log("[Code] Editing file...")
model = _get_gemini()
prompt = f"""You are an expert code editor.
Apply the following change to the code below.
Return ONLY the complete updated code — no explanation, no markdown, no backticks.
Change: {instruction}
Original code:
{content}
Updated code:"""
try:
response = model.generate_content(prompt)
edited = _clean_code(response.text)
except Exception as e:
return f"Could not edit code: {e}"
status = _save_file(Path(file_path), edited)
print(f"[Code] ✅ Edited: {file_path}")
return f"File edited. {status}\n\nPreview:\n{_preview(edited)}"
def _explain_action(file_path, code, player) -> str:
if file_path and not code:
code, err = _read_file(file_path)
if err:
return err
if not code:
return "Please provide code or a file path to explain, sir."
if player:
player.write_log("[Code] Analyzing code...")
model = _get_gemini()
prompt = f"""Explain what this code does in simple, clear language.
Focus on: what it does, how it works, and any important details.
Be concise — 3 to 6 sentences maximum.
Code:
{code[:4000]}
Explanation:"""
try:
response = model.generate_content(prompt)
return response.text.strip()
except Exception as e:
return f"Could not explain code: {e}"
def _run_action(file_path, args, timeout, player) -> str:
if not file_path:
return "Please provide a file path to run, sir."
p = Path(file_path)
if not p.exists():
return f"File not found: {file_path}"
if player:
player.write_log(f"[Code] Running {p.name}...")
return _run_file(p, args, timeout)
def _optimize_action(file_path, code, language, output_path, player) -> str:
if file_path and not code:
code, err = _read_file(file_path)
if err:
return err
if not code:
return "Please provide code or a file path to optimize, sir."
if player:
player.write_log("[Code] Optimizing code...")
lang = language or "python"
model = _get_gemini()
prompt = f"""You are an expert {lang} developer and code reviewer.
Optimize the following code for:
1. Performance — eliminate unnecessary operations, use efficient data structures
2. Readability — clear variable names, proper formatting, logical structure
3. Best practices — modern {lang} patterns, error handling, type hints if applicable
4. Remove dead code, redundant comments, and unnecessary complexity
Return ONLY the optimized code — no explanation, no markdown, no backticks.
Original code:
{code[:6000]}
Optimized code:"""
try:
response = model.generate_content(prompt)
optimized = _clean_code(response.text)
except Exception as e:
return f"Could not optimize code: {e}"
# Kaydet
if file_path:
save_path = Path(file_path)
else:
save_path = _resolve_save_path(output_path, lang)
status = _save_file(save_path, optimized)
print(f"[Code] ✅ Optimized: {save_path}")
original_lines = len(code.splitlines())
optimized_lines = len(optimized.splitlines())
diff = original_lines - optimized_lines
return (
f"Code optimized. {status}\n"
f"Lines: {original_lines} → {optimized_lines} "
f"({'−' if diff > 0 else '+'}{abs(diff)} lines)\n\n"
f"Preview:\n{_preview(optimized)}"
)
def _screen_debug_action(description, file_path, player, speak=None) -> str:
if player:
player.write_log("[Code] Taking screenshot for analysis...")
print("[Code] 📸 Capturing screen for debug...")
screenshot_path = _take_screenshot()
if not screenshot_path:
return "Could not take screenshot, sir. Please make sure PyAutoGUI is installed."
file_content = ""
if file_path:
file_content, err = _read_file(file_path)
if err:
print(f"[Code] ⚠️ Could not read file: {err}")
try:
from google import genai
from google.genai import types
client = genai.Client(api_key=_get_api_key())
image_bytes = screenshot_path.read_bytes()
image_base64 = _image_to_base64(screenshot_path)
user_question = description or "What error or problem do you see on the screen? How can it be fixed?"
context = ""
if file_content:
context = f"\n\nAdditionally, here is the related file content:\n```\n{file_content[:4000]}\n```"
analysis_prompt = f"""You are an expert programmer and debugger analyzing a screenshot.
User's question: {user_question}{context}
Please:
1. Identify any errors, exceptions, or problems visible on the screen
2. Explain what is causing the problem in simple terms
3. Provide a concrete fix or solution
4. If there's code visible, show the corrected version
Be specific and actionable. If you see an error message, quote it exactly."""
contents = [
types.Part.from_bytes(data=image_bytes, mime_type="image/png"),
analysis_prompt,
]
response = client.models.generate_content(
model="gemini-2.5-flash",
contents=contents,
)
analysis = response.text.strip()
print(f"[Code] ✅ Screen analysis complete")
try:
screenshot_path.unlink()
except Exception:
pass
if file_path and file_content:
code_match = re.search(r"```[a-zA-Z]*\n(.*?)```", analysis, re.DOTALL)
if code_match:
fixed_code = code_match.group(1).strip()
save_path = Path(file_path)
_save_file(save_path, fixed_code)
analysis += f"\n\n✅ Fixed code has been saved to: {file_path}"
print(f"[Code] ✅ Fixed code saved: {file_path}")
return analysis
except Exception as e:
try:
screenshot_path.unlink()
except Exception:
pass
return f"Screen analysis failed: {e}"
def code_helper(
parameters: dict,
response=None,
player=None,
session_memory=None,
speak=None
) -> str:
"""
Called from main.py.
parameters:
action : write | edit | explain | run | build | screen_debug | optimize | auto
description : What the code should do / what change to make / what problem to analyze
language : Programming language (default: python)
output_path : Where to save — user specifies full path or filename
file_path : Path to existing file (edit / explain / run / build / optimize)
code : Raw code string (explain/optimize without a file)
args : CLI argument list for run/build
timeout : Execution timeout in seconds (default: 30)
"""
p = parameters or {}
action = p.get("action", "auto").lower().strip()
description = p.get("description", "").strip()
language = p.get("language", "python").strip()
output_path = p.get("output_path", "").strip()
file_path = p.get("file_path", "").strip()
code = p.get("code", "").strip()
args = p.get("args", [])
timeout = int(p.get("timeout", 30))
if action == "auto":
action = _detect_intent(description, file_path, code)
print(f"[Code] 🤖 Auto-detected: {action}")
if action == "write":
return _write_action(description, language, output_path, player)
elif action == "edit":
return _edit_action(
file_path,
description or p.get("instruction", ""),
player
)
elif action == "explain":
return _explain_action(file_path, code, player)
elif action == "run":
return _run_action(file_path, args, timeout, player)
elif action == "build":
return _build(description, language, output_path, args, timeout, speak, player)
elif action == "optimize":
return _optimize_action(file_path, code, language, output_path, player)
elif action == "screen_debug":
return _screen_debug_action(description, file_path, player, speak)
else:
return f"Unknown action: '{action}'. Use write, edit, explain, run, build, optimize, or screen_debug."
+554
View File
@@ -0,0 +1,554 @@
# actions/computer_control.py
# MARK XXV — Computer Control
#
# Atomic computer control functions using PyAutoGUI + keyboard + clipboard.
# Used by the agent when no existing action file covers the task.
#
# Capabilities:
# - Type text anywhere (active window, forms, fields)
# - Mouse click, double-click, right-click, drag
# - Keyboard shortcuts and key combinations
# - Scroll (up/down/left/right)
# - Window management (minimize, maximize, close, focus)
# - Clipboard (copy, paste, get content)
# - Screenshot + locate element on screen
# - Wait / smart wait for element to appear
# - Random data generation (name, email, username, password, phone, address)
# - Hotkey sequences
# - Find and click image/element on screen
import json
import sys
import time
import random
import string
import subprocess
import platform
from pathlib import Path
try:
import pyautogui
pyautogui.FAILSAFE = True
pyautogui.PAUSE = 0.05
_PYAUTOGUI = True
except ImportError:
_PYAUTOGUI = False
try:
import pyperclip
_PYPERCLIP = True
except ImportError:
_PYPERCLIP = False
def get_base_dir() -> Path:
if getattr(sys, "frozen", False):
return Path(sys.executable).parent
return Path(__file__).resolve().parent.parent
BASE_DIR = get_base_dir()
API_CONFIG_PATH = BASE_DIR / "config" / "api_keys.json"
def _load_user_profile() -> dict:
"""Load user profile from long_term.json for form filling."""
memory_path = BASE_DIR / "memory" / "long_term.json"
try:
if memory_path.exists():
data = json.loads(memory_path.read_text(encoding="utf-8"))
identity = data.get("identity", {})
return {
"name": identity.get("name", {}).get("value", ""),
"age": identity.get("age", {}).get("value", ""),
"city": identity.get("city", {}).get("value", ""),
"email": identity.get("email", {}).get("value", ""),
}
except Exception:
pass
return {}
def _ensure_pyautogui():
if not _PYAUTOGUI:
raise RuntimeError(
"PyAutoGUI not installed. Run: pip install pyautogui"
)
_FIRST_NAMES = [
"Alex", "Jordan", "Taylor", "Morgan", "Casey", "Riley", "Drew", "Quinn",
"Avery", "Blake", "Cameron", "Dakota", "Emerson", "Finley", "Harper"
]
_LAST_NAMES = [
"Smith", "Johnson", "Williams", "Brown", "Jones", "Garcia", "Miller",
"Davis", "Wilson", "Moore", "Taylor", "Anderson", "Thomas", "Jackson"
]
_DOMAINS = ["gmail.com", "yahoo.com", "outlook.com", "proton.me", "mail.com"]
def generate_random_data(data_type: str) -> str:
"""
Generates random realistic data for form filling.
Types: name | first_name | last_name | email | username |
password | phone | birthday | address | zip_code
"""
dt = data_type.lower().strip()
if dt == "first_name":
return random.choice(_FIRST_NAMES)
elif dt == "last_name":
return random.choice(_LAST_NAMES)
elif dt == "name":
return f"{random.choice(_FIRST_NAMES)} {random.choice(_LAST_NAMES)}"
elif dt == "email":
first = random.choice(_FIRST_NAMES).lower()
last = random.choice(_LAST_NAMES).lower()
num = random.randint(10, 999)
return f"{first}.{last}{num}@{random.choice(_DOMAINS)}"
elif dt == "username":
first = random.choice(_FIRST_NAMES).lower()
num = random.randint(100, 9999)
return f"{first}{num}"
elif dt == "password":
chars = string.ascii_letters + string.digits + "!@#$%"
pwd = (
random.choice(string.ascii_uppercase) +
random.choice(string.digits) +
random.choice("!@#$%") +
"".join(random.choices(chars, k=9))
)
return "".join(random.sample(pwd, len(pwd)))
elif dt == "phone":
return f"+1{random.randint(200,999)}{random.randint(1000000,9999999)}"
elif dt == "birthday":
year = random.randint(1980, 2000)
month = random.randint(1, 12)
day = random.randint(1, 28)
return f"{month:02d}/{day:02d}/{year}"
elif dt == "address":
num = random.randint(100, 9999)
street = random.choice(["Main St", "Oak Ave", "Park Blvd", "Elm St", "Cedar Ln"])
return f"{num} {street}"
elif dt == "zip_code":
return str(random.randint(10000, 99999))
elif dt == "city":
return random.choice(["New York", "Los Angeles", "Chicago", "Houston", "Phoenix"])
return f"random_{data_type}_{random.randint(1000,9999)}"
def _type_text(text: str, interval: float = 0.03) -> str:
"""Types text at the current cursor position."""
_ensure_pyautogui()
time.sleep(0.3)
pyautogui.typewrite(text, interval=interval)
return f"Typed: {text[:50]}{'...' if len(text) > 50 else ''}"
def _click(x: int = None, y: int = None, button: str = "left",
clicks: int = 1, image: str = None) -> str:
"""
Clicks at coordinates or on a screen image.
If image path given, locates it on screen and clicks.
"""
_ensure_pyautogui()
if image:
try:
loc = pyautogui.locateCenterOnScreen(image, confidence=0.8)
if loc:
pyautogui.click(loc.x, loc.y, button=button, clicks=clicks)
return f"Clicked image: {image}"
return f"Image not found on screen: {image}"
except Exception as e:
return f"Image click failed: {e}"
if x is not None and y is not None:
pyautogui.click(x, y, button=button, clicks=clicks)
return f"Clicked ({x}, {y}) with {button} button"
pyautogui.click(button=button, clicks=clicks)
return f"Clicked at current position"
def _hotkey(*keys) -> str:
"""Presses a key combination. E.g. hotkey('ctrl', 'c')"""
_ensure_pyautogui()
pyautogui.hotkey(*keys)
return f"Hotkey: {'+'.join(keys)}"
def _press(key: str) -> str:
"""Presses a single key."""
_ensure_pyautogui()
pyautogui.press(key)
return f"Pressed: {key}"
def _scroll(direction: str = "down", amount: int = 3) -> str:
"""Scrolls in the specified direction."""
_ensure_pyautogui()
clicks = amount if direction in ("up", "right") else -amount
if direction in ("up", "down"):
pyautogui.scroll(clicks)
else:
pyautogui.hscroll(clicks)
return f"Scrolled {direction} {amount} times"
def _move_mouse(x: int, y: int, duration: float = 0.3) -> str:
"""Moves mouse to coordinates."""
_ensure_pyautogui()
pyautogui.moveTo(x, y, duration=duration)
return f"Mouse moved to ({x}, {y})"
def _drag(x1: int, y1: int, x2: int, y2: int, duration: float = 0.5) -> str:
"""Drags from (x1,y1) to (x2,y2)."""
_ensure_pyautogui()
pyautogui.drag(x1 - pyautogui.position()[0], y1 - pyautogui.position()[1])
pyautogui.dragTo(x2, y2, duration=duration)
return f"Dragged from ({x1},{y1}) to ({x2},{y2})"
def _clipboard_copy() -> str:
"""Gets current clipboard content."""
if _PYPERCLIP:
return pyperclip.paste()
_hotkey("ctrl", "c")
time.sleep(0.2)
return "Copied to clipboard"
def _clipboard_set(text: str) -> str:
"""Sets clipboard content and pastes it."""
if _PYPERCLIP:
pyperclip.copy(text)
time.sleep(0.1)
_hotkey("ctrl", "v")
return f"Pasted: {text[:50]}"
return "pyperclip not available"
def _screenshot(save_path: str = None) -> str:
"""Takes a screenshot."""
_ensure_pyautogui()
if not save_path:
save_path = str(Path.home() / "Desktop" / "screenshot.png")
img = pyautogui.screenshot()
img.save(save_path)
return f"Screenshot saved: {save_path}"
def _wait(seconds: float) -> str:
"""Waits for specified seconds."""
time.sleep(seconds)
return f"Waited {seconds}s"
def _wait_for_image(image_path: str, timeout: int = 10) -> str:
"""Waits until an image appears on screen (up to timeout seconds)."""
_ensure_pyautogui()
start = time.time()
while time.time() - start < timeout:
try:
loc = pyautogui.locateCenterOnScreen(image_path, confidence=0.8)
if loc:
return f"Image found at ({loc.x}, {loc.y})"
except Exception:
pass
time.sleep(0.5)
return f"Image not found within {timeout}s: {image_path}"
def _get_screen_size() -> str:
"""Returns current screen resolution."""
_ensure_pyautogui()
w, h = pyautogui.size()
return f"{w}x{h}"
def _focus_window(title: str) -> str:
"""Brings a window to focus by title (Windows)."""
if platform.system() == "Windows":
try:
script = f'(New-Object -ComObject WScript.Shell).AppActivate("{title}")'
subprocess.run(
["powershell", "-NoProfile", "-Command", script],
capture_output=True, timeout=5
)
time.sleep(0.3)
return f"Focused window: {title}"
except Exception as e:
return f"Could not focus window: {e}"
return "Window focus only supported on Windows"
def _select_all() -> str:
return _hotkey("ctrl", "a")
def _clear_field() -> str:
"""Selects all and deletes — clears an input field."""
_hotkey("ctrl", "a")
time.sleep(0.1)
_press("delete")
return "Field cleared"
def _smart_type(text: str, clear_first: bool = True) -> str:
"""
Types text into the currently focused field.
Optionally clears the field first.
Uses clipboard for long text (faster, more reliable).
"""
_ensure_pyautogui()
if clear_first:
_clear_field()
time.sleep(0.1)
if len(text) > 20 and _PYPERCLIP:
pyperclip.copy(text)
time.sleep(0.1)
pyautogui.hotkey("ctrl", "v")
return f"Smart-typed (clipboard): {text[:50]}"
else:
pyautogui.typewrite(text, interval=0.04)
return f"Smart-typed: {text[:50]}"
def _analyze_screen_for_element(description: str) -> tuple[int, int] | None:
"""
Takes a screenshot and asks Gemini to find the coordinates
of a described element on screen. Returns (x, y) or None.
"""
try:
import google.generativeai as genai
import io
cfg_path = API_CONFIG_PATH
with open(cfg_path, "r") as f:
api_key = json.load(f)["gemini_api_key"]
genai.configure(api_key=api_key)
model = genai.GenerativeModel("gemini-2.5-flash-lite")
_ensure_pyautogui()
w, h = pyautogui.size()
img = pyautogui.screenshot()
buf = io.BytesIO()
img.save(buf, format="PNG")
buf.seek(0)
prompt = (
f"This is a screenshot of a computer screen ({w}x{h} pixels). "
f"Find the element: '{description}'. "
f"Return ONLY: x,y (the center coordinates of the element). "
f"If not found, return: NOT_FOUND"
)
response = model.generate_content([
{"mime_type": "image/png", "data": buf.getvalue()},
prompt
])
text = response.text.strip()
if "NOT_FOUND" in text:
return None
import re
match = re.search(r"(\d+)\s*,\s*(\d+)", text)
if match:
return int(match.group(1)), int(match.group(2))
except Exception as e:
print(f"[ComputerControl] ⚠️ Screen analysis failed: {e}")
return None
def computer_control(
parameters: dict,
response=None,
player=None,
session_memory=None,
) -> str:
"""
Universal computer control action.
Actions:
type : Type text at current cursor position
smart_type : Clear field + type (clipboard-based for long text)
click : Click at coordinates or on image
double_click : Double click
right_click : Right click
hotkey : Key combination (e.g. ctrl+c)
press : Single key press
scroll : Scroll up/down/left/right
move : Move mouse to coordinates
drag : Drag from one point to another
copy : Get clipboard content
paste : Set and paste clipboard content
screenshot : Take a screenshot
wait : Wait N seconds
wait_image : Wait for image to appear on screen
clear_field : Select all + delete in current field
focus_window : Bring window to foreground
screen_find : AI-powered element finder — returns coordinates
screen_click : AI-powered element finder + click
random_data : Generate random data for forms
user_data : Get user's real data from memory
"""
action = (parameters or {}).get("action", "").lower().strip()
if not action:
return "Please specify an action for computer_control, sir."
if player:
player.write_log(f"[Computer] {action}")
print(f"[ComputerControl] ▶️ Action: {action} Params: {parameters}")
try:
if action == "type":
text = parameters.get("text", "")
return _type_text(text)
elif action == "smart_type":
text = parameters.get("text", "")
clear_first = parameters.get("clear_first", True)
return _smart_type(text, clear_first=clear_first)
elif action in ("click", "left_click"):
return _click(
x=parameters.get("x"),
y=parameters.get("y"),
button="left",
clicks=1,
image=parameters.get("image")
)
elif action == "double_click":
return _click(
x=parameters.get("x"),
y=parameters.get("y"),
button="left",
clicks=2,
image=parameters.get("image")
)
elif action == "right_click":
return _click(
x=parameters.get("x"),
y=parameters.get("y"),
button="right",
clicks=1
)
elif action == "move":
return _move_mouse(
x=int(parameters.get("x", 0)),
y=int(parameters.get("y", 0)),
duration=float(parameters.get("duration", 0.3))
)
elif action == "drag":
return _drag(
x1=int(parameters.get("x1", 0)),
y1=int(parameters.get("y1", 0)),
x2=int(parameters.get("x2", 0)),
y2=int(parameters.get("y2", 0))
)
elif action == "hotkey":
keys = parameters.get("keys", "")
if isinstance(keys, str):
keys = [k.strip() for k in keys.split("+")]
return _hotkey(*keys)
elif action == "press":
return _press(parameters.get("key", "enter"))
elif action == "scroll":
return _scroll(
direction=parameters.get("direction", "down"),
amount=int(parameters.get("amount", 3))
)
elif action == "copy":
return _clipboard_copy()
elif action == "paste":
return _clipboard_set(parameters.get("text", ""))
elif action == "screenshot":
return _screenshot(parameters.get("path"))
elif action == "wait":
return _wait(float(parameters.get("seconds", 1.0)))
elif action == "wait_image":
return _wait_for_image(
parameters.get("image", ""),
timeout=int(parameters.get("timeout", 10))
)
elif action == "clear_field":
return _clear_field()
elif action == "focus_window":
return _focus_window(parameters.get("title", ""))
elif action == "screen_size":
return _get_screen_size()
elif action == "screen_find":
description = parameters.get("description", "")
coords = _analyze_screen_for_element(description)
if coords:
return f"{coords[0]},{coords[1]}"
return "NOT_FOUND"
elif action == "screen_click":
description = parameters.get("description", "")
coords = _analyze_screen_for_element(description)
if coords:
time.sleep(0.2)
_click(x=coords[0], y=coords[1])
return f"Found and clicked: {description} at {coords}"
return f"Could not find on screen: {description}"
elif action == "random_data":
data_type = parameters.get("type", "name")
result = generate_random_data(data_type)
print(f"[ComputerControl] 🎲 Random {data_type}: {result}")
return result
elif action == "user_data":
field = parameters.get("field", "name")
profile = _load_user_profile()
value = profile.get(field, "")
if not value:
value = generate_random_data(field)
print(f"[ComputerControl] ⚠️ No user {field} in memory, using random: {value}")
return value
else:
return f"Unknown computer_control action: '{action}'"
except Exception as e:
print(f"[ComputerControl] ❌ Error: {e}")
return f"computer_control failed: {e}"
+679
View File
@@ -0,0 +1,679 @@
# actions/computer_settings.py
# MARK XXV — Computer Settings & UI Controls
#
# Kullanıcı "sesi aç", "uygulamayı kapat", "tam ekran yap", "şunu yaz" gibi
# bilgisayar kontrol komutları verdiğinde bu dosya devreye girer.
#
# - Intent detection: Gemini ile (multi-language, hardcoded keyword yok)
# - Cross-platform: Windows / macOS / Linux
# - pyautogui + platform-specific API'ler
import time
import subprocess
import sys
import platform
from pathlib import Path
try:
import pyautogui
pyautogui.FAILSAFE = True
pyautogui.PAUSE = 0.05
_PYAUTOGUI = True
except ImportError:
_PYAUTOGUI = False
try:
import pyperclip
_PYPERCLIP = True
except ImportError:
_PYPERCLIP = False
_OS = platform.system()
def get_base_dir() -> Path:
if getattr(sys, "frozen", False):
return Path(sys.executable).parent
return Path(__file__).resolve().parent.parent
BASE_DIR = get_base_dir()
API_CONFIG_PATH = BASE_DIR / "config" / "api_keys.json"
import json
def _get_api_key() -> str:
with open(API_CONFIG_PATH, "r", encoding="utf-8") as f:
return json.load(f)["gemini_api_key"]
def volume_up():
if _OS == "Windows":
for _ in range(5): pyautogui.press("volumeup")
elif _OS == "Darwin":
subprocess.run(["osascript", "-e", "set volume output volume (output volume of (get volume settings) + 10)"])
else:
subprocess.run(["pactl", "set-sink-volume", "@DEFAULT_SINK@", "+10%"])
def volume_down():
if _OS == "Windows":
for _ in range(5): pyautogui.press("volumedown")
elif _OS == "Darwin":
subprocess.run(["osascript", "-e", "set volume output volume (output volume of (get volume settings) - 10)"])
else:
subprocess.run(["pactl", "set-sink-volume", "@DEFAULT_SINK@", "-10%"])
def volume_mute():
if _OS == "Windows":
pyautogui.press("volumemute")
elif _OS == "Darwin":
subprocess.run(["osascript", "-e", "set volume with output muted"])
else:
subprocess.run(["pactl", "set-sink-mute", "@DEFAULT_SINK@", "toggle"])
def volume_set(value: int):
value = max(0, min(100, value))
if _OS == "Windows":
try:
from ctypes import cast, POINTER
from comtypes import CLSCTX_ALL
from pycaw.pycaw import AudioUtilities, IAudioEndpointVolume
import math
devices = AudioUtilities.GetSpeakers()
interface = devices.Activate(IAudioEndpointVolume._iid_, CLSCTX_ALL, None)
vol = cast(interface, POINTER(IAudioEndpointVolume))
vol_db = -65.25 if value == 0 else max(-65.25, 20 * math.log10(value / 100))
vol.SetMasterVolumeLevel(vol_db, None)
print(f"[Settings] 🔊 Volume → {value}%")
return
except Exception as e:
print(f"[Settings] ⚠️ pycaw failed: {e}")
elif _OS == "Darwin":
subprocess.run(["osascript", "-e", f"set volume output volume {value}"])
return
else:
subprocess.run(["pactl", "set-sink-volume", "@DEFAULT_SINK@", f"{value}%"])
return
def brightness_up():
if _OS == "Windows":
pyautogui.hotkey("win", "a")
time.sleep(0.3)
elif _OS == "Darwin":
subprocess.run(["osascript", "-e", "tell application \"System Events\" to key code 144"])
else:
subprocess.run(["brightnessctl", "set", "+10%"])
def brightness_down():
if _OS == "Windows":
pyautogui.hotkey("win", "a")
time.sleep(0.3)
elif _OS == "Darwin":
subprocess.run(["osascript", "-e", "tell application \"System Events\" to key code 145"])
else:
subprocess.run(["brightnessctl", "set", "10%-"])
def close_app():
if _OS == "Darwin":
pyautogui.hotkey("command", "q")
else:
pyautogui.hotkey("alt", "f4")
def close_window():
if _OS == "Darwin":
pyautogui.hotkey("command", "w")
else:
pyautogui.hotkey("ctrl", "w")
def full_screen():
if _OS == "Darwin":
pyautogui.hotkey("ctrl", "command", "f")
else:
pyautogui.press("f11")
def minimize_window():
if _OS == "Darwin":
pyautogui.hotkey("command", "m")
else:
pyautogui.hotkey("win", "down")
def maximize_window():
if _OS == "Darwin":
subprocess.run(["osascript", "-e",
'tell application "System Events" to keystroke "f" using {control down, command down}'])
else:
pyautogui.hotkey("win", "up")
def snap_left():
if _OS == "Windows": pyautogui.hotkey("win", "left")
def snap_right():
if _OS == "Windows": pyautogui.hotkey("win", "right")
def switch_window():
if _OS == "Darwin":
pyautogui.hotkey("command", "tab")
else:
pyautogui.hotkey("alt", "tab")
def show_desktop():
if _OS == "Darwin":
pyautogui.hotkey("fn", "f11")
elif _OS == "Windows":
pyautogui.hotkey("win", "d")
else:
pyautogui.hotkey("super", "d")
def open_task_manager():
if _OS == "Windows":
pyautogui.hotkey("ctrl", "shift", "esc")
elif _OS == "Darwin":
subprocess.Popen(["open", "-a", "Activity Monitor"])
else:
subprocess.Popen(["gnome-system-monitor"])
def open_task_view():
if _OS == "Windows":
pyautogui.hotkey("win", "tab")
def focus_search():
if _OS == "Darwin": pyautogui.hotkey("command", "l")
else: pyautogui.hotkey("ctrl", "l")
def pause_video(): pyautogui.press("space")
def refresh_page():
if _OS == "Darwin": pyautogui.hotkey("command", "r")
else: pyautogui.press("f5")
def close_tab():
if _OS == "Darwin": pyautogui.hotkey("command", "w")
else: pyautogui.hotkey("ctrl", "w")
def new_tab():
if _OS == "Darwin": pyautogui.hotkey("command", "t")
else: pyautogui.hotkey("ctrl", "t")
def next_tab():
if _OS == "Darwin": pyautogui.hotkey("command", "shift", "bracketright")
else: pyautogui.hotkey("ctrl", "tab")
def prev_tab():
if _OS == "Darwin": pyautogui.hotkey("command", "shift", "bracketleft")
else: pyautogui.hotkey("ctrl", "shift", "tab")
def go_back():
if _OS == "Darwin": pyautogui.hotkey("command", "left")
else: pyautogui.hotkey("alt", "left")
def go_forward():
if _OS == "Darwin": pyautogui.hotkey("command", "right")
else: pyautogui.hotkey("alt", "right")
def zoom_in():
if _OS == "Darwin": pyautogui.hotkey("command", "equal")
else: pyautogui.hotkey("ctrl", "equal")
def zoom_out():
if _OS == "Darwin": pyautogui.hotkey("command", "minus")
else: pyautogui.hotkey("ctrl", "minus")
def zoom_reset():
if _OS == "Darwin": pyautogui.hotkey("command", "0")
else: pyautogui.hotkey("ctrl", "0")
def find_on_page():
if _OS == "Darwin": pyautogui.hotkey("command", "f")
else: pyautogui.hotkey("ctrl", "f")
def reload_page_n(n: int):
for _ in range(n):
refresh_page()
time.sleep(0.8)
def scroll_up(amount: int = 500): pyautogui.scroll(amount)
def scroll_down(amount: int = 500): pyautogui.scroll(-amount)
def scroll_top(): pyautogui.hotkey("ctrl", "home") if _OS != "Darwin" else pyautogui.hotkey("command", "up")
def scroll_bottom(): pyautogui.hotkey("ctrl", "end") if _OS != "Darwin" else pyautogui.hotkey("command", "down")
def page_up(): pyautogui.press("pageup")
def page_down(): pyautogui.press("pagedown")
def copy():
if _OS == "Darwin": pyautogui.hotkey("command", "c")
else: pyautogui.hotkey("ctrl", "c")
def paste():
if _OS == "Darwin": pyautogui.hotkey("command", "v")
else: pyautogui.hotkey("ctrl", "v")
def cut():
if _OS == "Darwin": pyautogui.hotkey("command", "x")
else: pyautogui.hotkey("ctrl", "x")
def undo():
if _OS == "Darwin": pyautogui.hotkey("command", "z")
else: pyautogui.hotkey("ctrl", "z")
def redo():
if _OS == "Darwin": pyautogui.hotkey("command", "shift", "z")
else: pyautogui.hotkey("ctrl", "y")
def select_all():
if _OS == "Darwin": pyautogui.hotkey("command", "a")
else: pyautogui.hotkey("ctrl", "a")
def save_file():
if _OS == "Darwin": pyautogui.hotkey("command", "s")
else: pyautogui.hotkey("ctrl", "s")
def press_enter(): pyautogui.press("enter")
def press_escape(): pyautogui.press("escape")
def press_key(key: str): pyautogui.press(key)
def type_text(text: str, press_enter_after: bool = False):
if not text:
return
if _PYPERCLIP:
pyperclip.copy(text)
time.sleep(0.1)
paste()
else:
pyautogui.write(str(text), interval=0.03)
if press_enter_after:
time.sleep(0.1)
pyautogui.press("enter")
def write_on_screen(text: str):
type_text(text)
def take_screenshot():
if _OS == "Windows":
pyautogui.hotkey("win", "shift", "s")
elif _OS == "Darwin":
pyautogui.hotkey("command", "shift", "3")
else:
pyautogui.hotkey("ctrl", "print_screen")
def lock_screen():
if _OS == "Windows":
pyautogui.hotkey("win", "l")
elif _OS == "Darwin":
subprocess.run(["pmset", "displaysleepnow"])
else:
subprocess.run(["gnome-screensaver-command", "-l"])
def open_system_settings():
if _OS == "Windows":
pyautogui.hotkey("win", "i")
elif _OS == "Darwin":
subprocess.Popen(["open", "-a", "System Preferences"])
else:
subprocess.Popen(["gnome-control-center"])
def open_file_explorer():
if _OS == "Windows":
pyautogui.hotkey("win", "e")
elif _OS == "Darwin":
subprocess.Popen(["open", Path.home()])
else:
subprocess.Popen(["xdg-open", Path.home()])
def open_run():
if _OS == "Windows":
pyautogui.hotkey("win", "r")
def sleep_display():
if _OS == "Windows":
try:
import ctypes
ctypes.windll.user32.SendMessageW(0xFFFF, 0x0112, 0xF170, 2)
except Exception:
pass
elif _OS == "Darwin":
subprocess.run(["pmset", "displaysleepnow"])
else:
subprocess.run(["xset", "dpms", "force", "off"])
def restart_computer():
if _OS == "Windows":
subprocess.run(["shutdown", "/r", "/t", "5"])
elif _OS == "Darwin":
subprocess.run(["osascript", "-e", 'tell application "System Events" to restart'])
else:
subprocess.run(["sudo", "reboot"])
def shutdown_computer():
if _OS == "Windows":
subprocess.run(["shutdown", "/s", "/t", "5"])
elif _OS == "Darwin":
subprocess.run(["osascript", "-e", 'tell application "System Events" to shut down'])
else:
subprocess.run(["sudo", "shutdown", "-h", "now"])
def dark_mode():
if _OS == "Windows":
pyautogui.hotkey("win", "a")
time.sleep(0.3)
elif _OS == "Darwin":
subprocess.run(["osascript", "-e",
'tell app "System Events" to tell appearance preferences to set dark mode to not dark mode'])
def toggle_wifi():
if _OS == "Windows":
pyautogui.hotkey("win", "a")
time.sleep(0.3)
elif _OS == "Darwin":
subprocess.run(["networksetup", "-setairportpower", "en0", "toggle"])
else:
subprocess.run(["nmcli", "radio", "wifi"])
ACTION_MAP = {
"volume_up": volume_up,
"volume_down": volume_down,
"mute": volume_mute,
"unmute": volume_mute,
"volume_increase": volume_up,
"volume_decrease": volume_down,
"increase_volume": volume_up,
"decrease_volume": volume_down,
"turn_up_volume": volume_up,
"turn_down_volume": volume_down,
"louder": volume_up,
"quieter": volume_down,
"silence": volume_mute,
"toggle_mute": volume_mute,
"brightness_up": brightness_up,
"brightness_down": brightness_down,
"increase_brightness": brightness_up,
"decrease_brightness": brightness_down,
"brighter": brightness_up,
"dimmer": brightness_down,
"dim_screen": brightness_down,
"brighten_screen": brightness_up,
"sleep_display": sleep_display,
"turn_off_screen": sleep_display,
"screen_off": sleep_display,
"display_off": sleep_display,
"change_screen": sleep_display,
"screen_sleep": sleep_display,
"monitor_off": sleep_display,
"turn_off_monitor": sleep_display,
"pause_video": pause_video,
"play_video": pause_video,
"pause": pause_video,
"play": pause_video,
"toggle_play": pause_video,
"stop_video": pause_video,
"resume_video": pause_video,
"close_app": close_app,
"close_window": close_window,
"quit_app": close_app,
"exit_app": close_app,
"kill_app": close_app,
"full_screen": full_screen,
"fullscreen": full_screen,
"toggle_fullscreen": full_screen,
"minimize": minimize_window,
"minimize_window": minimize_window,
"maximize": maximize_window,
"maximize_window": maximize_window,
"restore_window": maximize_window,
"snap_left": snap_left,
"snap_right": snap_right,
"window_left": snap_left,
"window_right": snap_right,
"switch_window": switch_window,
"alt_tab": switch_window,
"next_window": switch_window,
"show_desktop": show_desktop,
"desktop": show_desktop,
"hide_windows": show_desktop,
"task_manager": open_task_manager,
"open_task_manager": open_task_manager,
"task_view": open_task_view,
"screenshot": take_screenshot,
"take_screenshot": take_screenshot,
"capture_screen": take_screenshot,
"lock_screen": lock_screen,
"lock": lock_screen,
"open_settings": open_system_settings,
"system_settings": open_system_settings,
"settings": open_system_settings,
"preferences": open_system_settings,
"file_explorer": open_file_explorer,
"open_explorer": open_file_explorer,
"explorer": open_file_explorer,
"open_files": open_file_explorer,
"run": open_run,
"open_run": open_run,
"restart": restart_computer,
"restart_computer": restart_computer,
"reboot": restart_computer,
"reboot_computer": restart_computer,
"shutdown": shutdown_computer,
"shut_down": shutdown_computer,
"power_off": shutdown_computer,
"turn_off_computer": shutdown_computer,
"dark_mode": dark_mode,
"toggle_dark_mode": dark_mode,
"night_mode": dark_mode,
"toggle_wifi": toggle_wifi,
"wifi": toggle_wifi,
"wifi_toggle": toggle_wifi,
"focus_search": focus_search,
"address_bar": focus_search,
"url_bar": focus_search,
"refresh_page": refresh_page,
"reload_page": refresh_page,
"reload": refresh_page,
"refresh": refresh_page,
"close_tab": close_tab,
"new_tab": new_tab,
"open_tab": new_tab,
"next_tab": next_tab,
"prev_tab": prev_tab,
"previous_tab": prev_tab,
"go_back": go_back,
"back": go_back,
"go_forward": go_forward,
"forward": go_forward,
"zoom_in": zoom_in,
"zoom_out": zoom_out,
"zoom_reset": zoom_reset,
"reset_zoom": zoom_reset,
"find_on_page": find_on_page,
"search_page": find_on_page,
"scroll_up": scroll_up,
"scroll_down": scroll_down,
"scroll_top": scroll_top,
"scroll_bottom": scroll_bottom,
"top_of_page": scroll_top,
"bottom_of_page": scroll_bottom,
"page_up": page_up,
"page_down": page_down,
"copy": copy,
"paste": paste,
"cut": cut,
"undo": undo,
"redo": redo,
"select_all": select_all,
"save": save_file,
"save_file": save_file,
"enter": press_enter,
"press_enter": press_enter,
"escape": press_escape,
"press_escape": press_escape,
"cancel": press_escape,
}
def _detect_action(description: str) -> dict:
"""
Gemini ile kullanıcının ne yapmak istediğini anlar.
Herhangi bir dilde çalışır.
Döner: {"action": str, "value": optional}
"""
import google.generativeai as genai
genai.configure(api_key=_get_api_key())
model = genai.GenerativeModel("gemini-2.5-flash-lite")
available = ", ".join(sorted(ACTION_MAP.keys())) + ", volume_set, type_text, write_on_screen, reload_n, press_key"
prompt = f"""The user wants to control their computer. Detect their intent.
User said (in any language): "{description}"
Available actions: {available}
Return ONLY valid JSON:
{{"action": "action_name", "value": null_or_value}}
Examples:
- "turn up the volume" → {{"action": "volume_up", "value": null}}
- "set volume to 60" → {{"action": "volume_set", "value": 60}}
- "sesi 80 yap" → {{"action": "volume_set", "value": 80}}
- "close the app" → {{"action": "close_app", "value": null}}
- "uygulamayı kapat" → {{"action": "close_app", "value": null}}
- "type hello world" → {{"action": "type_text", "value": "hello world"}}
- "write good morning on screen" → {{"action": "write_on_screen", "value": "good morning"}}
- "reload page 3 times" → {{"action": "reload_n", "value": 3}}
- "tam ekran yap" → {{"action": "full_screen", "value": null}}
- "sesi kıs" → {{"action": "volume_down", "value": null}}
- "sesi aç" → {{"action": "volume_up", "value": null}}
- "sustur" → {{"action": "mute", "value": null}}
- "monte le son" → {{"action": "volume_up", "value": null}}
- "ekranı kapat" → {{"action": "sleep_display", "value": null}}
- "monitörü kapat" → {{"action": "sleep_display", "value": null}}
- "turn off screen" → {{"action": "sleep_display", "value": null}}
- "turn off monitor" → {{"action": "sleep_display", "value": null}}
- "bilgisayarı yeniden başlat" → {{"action": "restart", "value": null}}
- "restart the computer" → {{"action": "restart", "value": null}}
- "bilgisayarı kapat" → {{"action": "shutdown", "value": null}}
- "shut down" → {{"action": "shutdown", "value": null}}
- "ekranı kilitle" → {{"action": "lock_screen", "value": null}}
- "lock the screen" → {{"action": "lock_screen", "value": null}}
- "küçült" → {{"action": "minimize", "value": null}}
- "minimize the window" → {{"action": "minimize", "value": null}}
- "büyüt" → {{"action": "maximize", "value": null}}
- "parlaklığı artır" → {{"action": "brightness_up", "value": null}}
- "parlaklığı azalt" → {{"action": "brightness_down", "value": null}}
- "increase brightness" → {{"action": "brightness_up", "value": null}}
- "wifi'yi aç" → {{"action": "toggle_wifi", "value": null}}
- "toggle wifi" → {{"action": "toggle_wifi", "value": null}}
- "masaüstünü göster" → {{"action": "show_desktop", "value": null}}
- "show desktop" → {{"action": "show_desktop", "value": null}}
- "yeni sekme aç" → {{"action": "new_tab", "value": null}}
- "sekmeyi kapat" → {{"action": "close_tab", "value": null}}
- "geri git" → {{"action": "go_back", "value": null}}
- "ileri git" → {{"action": "go_forward", "value": null}}
- "sayfayı yenile" → {{"action": "refresh_page", "value": null}}
- "yakınlaştır" → {{"action": "zoom_in", "value": null}}
- "uzaklaştır" → {{"action": "zoom_out", "value": null}}
- "kaydet" → {{"action": "save", "value": null}}
- "geri al" → {{"action": "undo", "value": null}}
- "screenshot al" → {{"action": "screenshot", "value": null}}
- "ekran görüntüsü al" → {{"action": "screenshot", "value": null}}
- "aşağı kaydır" → {{"action": "scroll_down", "value": null}}
- "yukarı kaydır" → {{"action": "scroll_up", "value": null}}
- "karanlık mod" → {{"action": "dark_mode", "value": null}}
- "press f5" → {{"action": "press_key", "value": "f5"}}
- "enter'a bas" → {{"action": "enter", "value": null}}
- "escape'e bas" → {{"action": "escape", "value": null}}
IMPORTANT:
- Always return one of the available actions listed above.
- If the user's intent is clear but uses different wording, map it to the closest action.
- Never invent new action names not in the available list.
- Return ONLY the JSON object, no explanation, no markdown."""
try:
response = model.generate_content(prompt)
text = response.text.strip()
text = __import__("re").sub(r"```(?:json)?", "", text).strip().rstrip("`").strip()
return json.loads(text)
except Exception as e:
print(f"[Settings] ⚠️ Intent detection failed: {e}")
return {"action": description.lower().replace(" ", "_"), "value": None}
def computer_settings(
parameters: dict,
response=None,
player=None,
session_memory=None,
) -> str:
"""
Bilgisayar ayarları ve UI kontrolleri.
parameters:
action : İşlem adı (verilmezse description'dan Gemini ile tespit edilir)
description : Kullanıcının doğal dil komutu (herhangi bir dilde)
value : İşleme özgü değer (ses seviyesi, yazılacak metin, tekrar sayısı vb.)
"""
if not _PYAUTOGUI:
return "pyautogui is not installed. Run: pip install pyautogui"
params = parameters or {}
raw_action = params.get("action", "").strip()
description = params.get("description", "").strip()
value = params.get("value", None)
if not raw_action and description:
detected = _detect_action(description)
raw_action = detected.get("action", "")
if value is None:
value = detected.get("value")
action = raw_action.lower().strip().replace(" ", "_").replace("-", "_")
if not action:
return "No action could be determined, sir."
print(f"[Settings] ⚙️ Action: {action} Value: {value}")
if action == "volume_set":
try:
volume_set(int(value or 50))
return f"Volume set to {value}%."
except Exception as e:
return f"Could not set volume: {e}"
if action in ("type_text", "write_on_screen", "type", "write"):
text = str(value or params.get("text", ""))
if not text:
return "No text provided to type, sir."
enter_after = bool(params.get("press_enter", False))
type_text(text, press_enter_after=enter_after)
return f"Typed: {text[:60]}"
if action == "press_key":
key = str(value or params.get("key", ""))
if not key:
return "No key specified, sir."
press_key(key)
return f"Pressed: {key}"
if action in ("reload_n", "refresh_n", "reload_page_n"):
try:
n = int(value or 1)
reload_page_n(n)
return f"Reloaded page {n} time{'s' if n > 1 else ''}."
except Exception as e:
return f"Could not reload: {e}"
if action in ("scroll_up", "scroll_down"):
try:
amount = int(value or 500)
scroll_up(amount) if action == "scroll_up" else scroll_down(amount)
return f"Scrolled {'up' if action == 'scroll_up' else 'down'}."
except Exception as e:
return f"Scroll failed: {e}"
func = ACTION_MAP.get(action)
if not func:
return f"Unknown action: '{raw_action}', sir."
try:
func()
return f"Done: {action}."
except Exception as e:
return f"Action failed ({action}): {e}"
+417
View File
@@ -0,0 +1,417 @@
# actions/desktop.py
# AI-powered desktop & wallpaper management
#
# Flow for unknown tasks:
# User request → Gemini generates Python/pyautogui code → Safety check → Execute
#
# Built-in: wallpaper change, icon arrangement, desktop cleanup, organize by type
import os
import sys
import json
import shutil
import subprocess
import ctypes
import tempfile
import pyautogui
from pathlib import Path
from datetime import datetime
def get_base_dir():
if getattr(sys, "frozen", False):
return Path(sys.executable).parent
return Path(__file__).resolve().parent.parent
BASE_DIR = get_base_dir()
API_CONFIG_PATH = BASE_DIR / "config" / "api_keys.json"
def _get_api_key() -> str:
with open(API_CONFIG_PATH, "r", encoding="utf-8") as f:
return json.load(f)["gemini_api_key"]
def _get_desktop() -> Path:
return Path.home() / "Desktop"
BLOCKED_KEYWORDS = [
"os.remove", "shutil.rmtree", "shutil.rm",
"subprocess.run", "subprocess.Popen", "subprocess.call",
"os.system", "exec(", "eval(",
"import os", "import subprocess",
"__import__", "open(",
"sys.exit", "quit()",
]
def _is_safe_code(code: str) -> tuple[bool, str]:
code_lower = code.lower()
for keyword in BLOCKED_KEYWORDS:
if keyword.lower() in code_lower:
return False, f"Blocked operation: '{keyword}'"
return True, "OK"
def _ask_gemini_for_desktop_action(task: str) -> str:
"""
Asks Gemini to generate safe Python/pyautogui code
to accomplish a desktop-related task.
"""
import google.generativeai as genai
genai.configure(api_key=_get_api_key())
model = genai.GenerativeModel("gemini-2.5-flash")
desktop = str(_get_desktop())
prompt = f"""You are a Windows desktop automation expert.
Generate safe Python code using ONLY these allowed modules:
- pyautogui (mouse, keyboard, screenshots)
- pathlib.Path (already imported as Path)
- shutil (ONLY: copy2, copytree, move, disk_usage)
- os.path (ONLY: exists, join, dirname, basename, splitext)
- time (sleep only)
- ctypes (Windows API calls only)
- winreg (registry READ only)
Desktop path: {desktop}
Rules:
- Output ONLY the Python code. No explanation, no markdown, no backticks.
- NO file deletion (os.remove, shutil.rmtree, unlink, etc.)
- NO subprocess calls
- NO exec() or eval()
- NO file write operations
- If task cannot be done safely, output exactly: UNSAFE
Task: {task}
Python code:"""
try:
response = model.generate_content(prompt)
code = response.text.strip()
if code.startswith("```"):
lines = code.split("\n")
code = "\n".join(lines[1:-1]).strip()
return code
except Exception as e:
return f"ERROR: {e}"
def _execute_generated_code(code: str) -> str:
"""Safely executes Gemini-generated desktop automation code."""
safe, reason = _is_safe_code(code)
if not safe:
return f"⛔ Blocked for safety: {reason}"
allowed_globals = {
"pyautogui": pyautogui,
"Path": Path,
"shutil": shutil,
"ctypes": ctypes,
"time": __import__("time"),
"os": type("os", (), {
"path": os.path,
"listdir": os.listdir,
"getcwd": os.getcwd,
"environ": os.environ,
})(),
"__builtins__": {
"print": print,
"len": len,
"str": str,
"int": int,
"float": float,
"bool": bool,
"list": list,
"dict": dict,
"range": range,
"enumerate": enumerate,
"sorted": sorted,
"isinstance": isinstance,
"hasattr": hasattr,
"getattr": getattr,
"max": max,
"min": min,
"sum": sum,
}
}
output_lines = []
allowed_globals["print"] = lambda *args: output_lines.append(" ".join(str(a) for a in args))
try:
exec(code, allowed_globals)
return "\n".join(output_lines) if output_lines else "Task completed successfully."
except Exception as e:
return f"Execution error: {e}\n\nCode attempted:\n{code[:200]}"
def set_wallpaper(image_path: str) -> str:
"""Sets desktop wallpaper from a local image path."""
path = Path(image_path).expanduser().resolve()
if not path.exists():
return f"Image not found: {image_path}"
if path.suffix.lower() not in [".jpg", ".jpeg", ".png", ".bmp"]:
return f"Unsupported format: {path.suffix}. Use jpg, png or bmp."
try:
if sys.platform == "win32":
abs_path = str(path.resolve())
ctypes.windll.user32.SystemParametersInfoW(20, 0, abs_path, 3)
return f"Wallpaper set: {path.name}"
elif sys.platform == "darwin":
script = f'tell application "Finder" to set desktop picture to POSIX file "{path}"'
subprocess.run(["osascript", "-e", script])
return f"Wallpaper set: {path.name}"
else:
subprocess.run(["gsettings", "set", "org.gnome.desktop.background",
"picture-uri", f"file://{path}"])
return f"Wallpaper set: {path.name}"
except Exception as e:
return f"Could not set wallpaper: {e}"
def set_wallpaper_from_web(url: str) -> str:
"""Downloads an image from URL and sets it as wallpaper."""
try:
import urllib.request
suffix = Path(url.split("?")[0]).suffix or ".jpg"
tmp = Path(tempfile.mktemp(suffix=suffix))
urllib.request.urlretrieve(url, str(tmp))
result = set_wallpaper(str(tmp))
return result
except Exception as e:
return f"Could not download wallpaper: {e}"
def get_current_wallpaper() -> str:
"""Returns the current wallpaper path."""
try:
if sys.platform == "win32":
import winreg
key = winreg.OpenKey(winreg.HKEY_CURRENT_USER,
r"Control Panel\Desktop")
val, _ = winreg.QueryValueEx(key, "Wallpaper")
return f"Current wallpaper: {val}"
else:
return "Wallpaper path retrieval not supported on this OS."
except Exception as e:
return f"Could not get wallpaper: {e}"
FILE_TYPE_MAP = {
"Images": [".jpg", ".jpeg", ".png", ".gif", ".bmp", ".webp", ".svg", ".ico", ".heic"],
"Documents": [".pdf", ".doc", ".docx", ".txt", ".xls", ".xlsx", ".ppt", ".pptx", ".csv", ".odt"],
"Videos": [".mp4", ".avi", ".mkv", ".mov", ".wmv", ".flv", ".webm", ".m4v"],
"Music": [".mp3", ".wav", ".flac", ".aac", ".ogg", ".wma", ".m4a"],
"Archives": [".zip", ".rar", ".7z", ".tar", ".gz", ".bz2"],
"Code": [".py", ".js", ".html", ".css", ".json", ".xml", ".ts", ".cpp", ".java", ".cs", ".php"],
"Executables": [".exe", ".msi", ".bat", ".cmd", ".sh"],
}
def organize_desktop(mode: str = "by_type") -> str:
"""
Organizes desktop files.
mode: 'by_type' — groups by file type (Images, Documents, etc.)
'by_date' — groups by month (2024-01, 2024-02, etc.)
"""
desktop = _get_desktop()
moved = []
skipped = []
for item in desktop.iterdir():
if item.is_dir() or item.name.startswith("."):
continue
if item.suffix.lower() == ".lnk":
continue
if mode == "by_date":
mtime = datetime.fromtimestamp(item.stat().st_mtime)
folder_name = mtime.strftime("%Y-%m")
else:
ext = item.suffix.lower()
folder_name = "Others"
for folder, exts in FILE_TYPE_MAP.items():
if ext in exts:
folder_name = folder
break
target_dir = desktop / folder_name
target_dir.mkdir(exist_ok=True)
new_path = target_dir / item.name
if new_path.exists():
skipped.append(item.name)
continue
shutil.move(str(item), str(new_path))
moved.append(f"{item.name} → {folder_name}/")
result = f"Desktop organized ({mode}). {len(moved)} files moved."
if moved:
preview = moved[:8]
result += "\n" + "\n".join(preview)
if len(moved) > 8:
result += f"\n... and {len(moved)-8} more."
if skipped:
result += f"\n{len(skipped)} files skipped (name conflict)."
return result
def list_desktop() -> str:
"""Lists everything on the desktop."""
desktop = _get_desktop()
items = []
for item in sorted(desktop.iterdir()):
if item.name.startswith("."):
continue
if item.is_dir():
count = len(list(item.iterdir()))
items.append(f"📁 {item.name}/ ({count} items)")
else:
size = item.stat().st_size
size_str = f"{size/1024:.1f} KB" if size < 1024*1024 else f"{size/1024/1024:.1f} MB"
items.append(f"📄 {item.name} ({size_str})")
if not items:
return "Desktop is empty."
return f"Desktop ({len(items)} items):\n" + "\n".join(items)
def clean_desktop() -> str:
"""
Moves all files on desktop into a 'Desktop Archive' folder
with today's date — fast cleanup without deleting anything.
"""
desktop = _get_desktop()
today = datetime.now().strftime("%Y-%m-%d")
archive_dir = desktop / f"Desktop Archive {today}"
archive_dir.mkdir(exist_ok=True)
moved = 0
for item in desktop.iterdir():
if item.is_dir() or item.name.startswith("."):
continue
if item.suffix.lower() == ".lnk":
continue
new_path = archive_dir / item.name
if not new_path.exists():
shutil.move(str(item), str(new_path))
moved += 1
return f"Desktop cleaned. {moved} files moved to '{archive_dir.name}'."
def get_desktop_stats() -> str:
"""Returns stats about the desktop."""
desktop = _get_desktop()
files = [i for i in desktop.iterdir() if i.is_file()]
folders = [i for i in desktop.iterdir() if i.is_dir()]
total_size = sum(f.stat().st_size for f in files)
size_str = f"{total_size/1024:.1f} KB" if total_size < 1024*1024 else f"{total_size/1024/1024:.1f} MB"
return (
f"Desktop stats:\n"
f" Files : {len(files)}\n"
f" Folders : {len(folders)}\n"
f" Total size: {size_str}"
)
def desktop_control(
parameters: dict,
response=None,
player=None,
session_memory=None
) -> str:
"""
Called from main.py.
parameters:
action : wallpaper | wallpaper_url | current_wallpaper |
organize | clean | list | stats |
task (AI-powered — anything else)
path : image path for 'wallpaper'
url : image URL for 'wallpaper_url'
mode : 'by_type' or 'by_date' for 'organize'
task : Natural language description for AI-powered actions.
Example: "arrange icons by size"
"show me what's on my desktop"
"move all screenshots to a folder"
"""
action = (parameters or {}).get("action", "").lower().strip()
task = (parameters or {}).get("task", "").strip()
result = "Unknown action."
try:
if action == "wallpaper":
path = parameters.get("path", "")
result = set_wallpaper(path) if path else "No image path provided."
elif action == "wallpaper_url":
url = parameters.get("url", "")
result = set_wallpaper_from_web(url) if url else "No URL provided."
elif action == "current_wallpaper":
result = get_current_wallpaper()
elif action == "organize":
mode = parameters.get("mode", "by_type")
result = organize_desktop(mode)
elif action == "clean":
result = clean_desktop()
elif action == "list":
result = list_desktop()
elif action == "stats":
result = get_desktop_stats()
elif action == "task" or task:
actual_task = task or parameters.get("description", "")
if not actual_task:
return "Please describe what you want to do on the desktop, sir."
print(f"[Desktop] 🤖 Asking Gemini: {actual_task}")
if player:
player.write_log("[Desktop] Generating action...")
code = _ask_gemini_for_desktop_action(actual_task)
if code == "UNSAFE":
result = "I cannot perform that desktop action safely, sir."
elif code.startswith("ERROR:"):
result = f"Could not generate action: {code}"
else:
print(f"[Desktop] ✅ Generated code:\n{code[:200]}")
result = _execute_generated_code(code)
else:
full_task = task or action
if full_task:
code = _ask_gemini_for_desktop_action(full_task)
result = _execute_generated_code(code) if code not in ("UNSAFE",) else "Cannot do that safely."
else:
result = "No action or task specified."
except Exception as e:
result = f"Desktop control error: {e}"
print(f"[Desktop] {result[:100]}")
if player:
player.write_log(f"[desktop] {result[:60]}")
return result
+491
View File
@@ -0,0 +1,491 @@
# actions/dev_agent.py
# AI-powered development agent — plans, builds, and debugs full projects.
#
# Flow:
# Describe project → Gemini plans file structure → Files written one by one
# → VSCode opened → Entry point executed → Error? → Identify file → Fix → Retry
# → Speaks only when done (success or failure)
#
# Models:
# Planning : gemini-2.5-flash (architecture, structure, debugging)
# Writing : gemini-2.5-flash-lite (fast file generation)
import subprocess
import sys
import json
import re
import time
from pathlib import Path
def get_base_dir():
if getattr(sys, "frozen", False):
return Path(sys.executable).parent
return Path(__file__).resolve().parent.parent
BASE_DIR = get_base_dir()
API_CONFIG_PATH = BASE_DIR / "config" / "api_keys.json"
PROJECTS_DIR = Path.home() / "Desktop" / "JarvisProjects"
MAX_FIX_ATTEMPTS = 4
MODEL_PLANNER = "gemini-2.5-flash"
MODEL_WRITER = "gemini-2.5-flash-lite"
def _get_api_key() -> str:
with open(API_CONFIG_PATH, "r", encoding="utf-8") as f:
return json.load(f)["gemini_api_key"]
def _get_model(model_name: str):
import google.generativeai as genai
genai.configure(api_key=_get_api_key())
return genai.GenerativeModel(model_name)
def _clean_code(text: str) -> str:
text = text.strip()
text = re.sub(r"^```[a-zA-Z]*\n?", "", text)
text = re.sub(r"\n?```$", "", text)
return text.strip()
def _clean_json(text: str) -> str:
text = text.strip()
text = re.sub(r"^```[a-zA-Z]*\n?", "", text)
text = re.sub(r"\n?```$", "", text)
return text.strip()
def _is_rate_limit(error: Exception) -> bool:
return "429" in str(error) or "quota" in str(error).lower()
def _get_interpreter(path: Path) -> list[str] | None:
return {
".py": [sys.executable],
".js": ["node"],
".ts": ["ts-node"],
".sh": ["bash"],
".ps1": ["powershell", "-File"],
".rb": ["ruby"],
".php": ["php"],
}.get(path.suffix.lower())
def _has_error(output: str) -> bool:
if "timed out" in output.lower():
return False
signals = ["error", "exception", "traceback", "syntaxerror",
"nameerror", "typeerror", "importerror", "stderr", "failed"]
return any(s in output.lower() for s in signals)
def _identify_error_file(error_output: str, project_files: list[str]) -> str | None:
"""
Try to find which file caused the error from traceback.
Returns filename or None.
"""
for line in error_output.splitlines():
for f in project_files:
if Path(f).name in line or f in line:
return f
return None
def _plan_project(description: str, language: str) -> dict:
"""
Ask Gemini to plan the full project structure.
Returns:
{
"project_name": "my_app",
"entry_point": "main.py",
"files": [
{
"path": "main.py",
"description": "Entry point, starts the app"
},
{
"path": "utils/helpers.py",
"description": "Utility functions"
}
],
"run_command": "python main.py",
"dependencies": ["requests", "flask"]
}
"""
model = _get_model(MODEL_PLANNER)
prompt = f"""You are a senior software architect.
Plan the complete file structure for the following project.
Language: {language}
Description: {description}
Return ONLY a valid JSON object with this exact structure:
{{
"project_name": "short_snake_case_name",
"entry_point": "main.py",
"files": [
{{"path": "main.py", "description": "what this file does"}},
{{"path": "utils/helpers.py", "description": "what this file does"}}
],
"run_command": "python main.py",
"dependencies": ["package1", "package2"]
}}
Rules:
- Keep it simple. Only include files that are truly necessary.
- No explanation, no markdown, no backticks. Pure JSON only.
- Entry point must be one of the files listed.
- Use relative paths only.
JSON:"""
try:
response = model.generate_content(prompt)
raw = _clean_json(response.text)
return json.loads(raw)
except json.JSONDecodeError as e:
raise ValueError(f"Planner returned invalid JSON: {e}\nRaw: {response.text[:300]}")
def _write_file(
file_path: str,
file_description: str,
project_description: str,
all_files: list[dict],
language: str,
project_dir: Path
) -> str:
"""Write one file. Returns the generated code."""
model = _get_model(MODEL_WRITER)
file_list = "\n".join(
f" - {f['path']}: {f['description']}" for f in all_files
)
prompt = f"""You are an expert {language} developer.
Write the code for ONE specific file in a larger project.
Project goal: {project_description}
All files in this project:
{file_list}
Now write ONLY the file: {file_path}
Purpose of this file: {file_description}
Rules:
- Output ONLY the code for this file. No explanation, no markdown, no backticks.
- Import from other project files using relative imports where needed.
- Add helpful inline comments.
- Handle errors properly.
- Use modern best practices.
Code for {file_path}:"""
try:
response = model.generate_content(prompt)
code = _clean_code(response.text)
# Save file
full_path = project_dir / file_path
full_path.parent.mkdir(parents=True, exist_ok=True)
full_path.write_text(code, encoding="utf-8")
print(f"[DevAgent] ✅ Written: {file_path}")
return code
except Exception as e:
if _is_rate_limit(e):
raise RateLimitError(str(e))
raise
def _install_dependencies(dependencies: list[str], project_dir: Path) -> str:
if not dependencies:
return "No dependencies to install."
print(f"[DevAgent] 📦 Installing: {dependencies}")
try:
result = subprocess.run(
[sys.executable, "-m", "pip", "install"] + dependencies,
capture_output=True, text=True,
encoding="utf-8", errors="replace",
timeout=120, cwd=str(project_dir)
)
if result.returncode == 0:
return f"Installed: {', '.join(dependencies)}"
return f"Install warning: {result.stderr[:200]}"
except subprocess.TimeoutExpired:
return "Dependency install timed out."
except Exception as e:
return f"Install error: {e}"
def _open_vscode(project_dir: Path) -> bool:
vscode_paths = [
"code",
r"C:\Users\{}\AppData\Local\Programs\Microsoft VS Code\bin\code.cmd".format(
Path.home().name
),
]
for cmd in vscode_paths:
try:
subprocess.Popen(
[cmd, str(project_dir)],
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
shell=True
)
time.sleep(2)
print(f"[DevAgent] 💻 VSCode opened: {project_dir}")
return True
except Exception:
continue
print("[DevAgent] ⚠️ VSCode not found.")
return False
def _run_project(run_command: str, project_dir: Path, timeout: int = 30) -> str:
"""Run the project entry point, return output."""
print(f"[DevAgent] 🚀 Running: {run_command}")
try:
parts = run_command.split()
if parts[0] == "python":
parts[0] = sys.executable
result = subprocess.run(
parts,
capture_output=True, text=True,
encoding="utf-8", errors="replace",
timeout=timeout, cwd=str(project_dir)
)
output = result.stdout.strip()
error = result.stderr.strip()
parts_out = []
if output: parts_out.append(f"Output:\n{output}")
if error: parts_out.append(f"Stderr:\n{error}")
return "\n\n".join(parts_out) if parts_out else "Ran with no output."
except subprocess.TimeoutExpired:
return f"Timed out after {timeout}s. (Long-running app may be working fine.)"
except FileNotFoundError as e:
return f"Command not found: {e}"
except Exception as e:
return f"Run error: {e}"
def _fix_file(
file_path: str,
current_code: str,
error_output: str,
project_description: str,
all_files: list[dict],
language: str,
project_dir: Path
) -> str:
"""Ask Gemini to fix a specific file based on error output."""
model = _get_model(MODEL_PLANNER)
file_list = "\n".join(
f" - {f['path']}: {f['description']}" for f in all_files
)
prompt = f"""You are an expert {language} debugger.
Fix the file below. It caused an error when the project was run.
Project goal: {project_description}
All files in this project:
{file_list}
File to fix: {file_path}
Error output:
{error_output[:3000]}
Current code:
{current_code}
Return ONLY the fixed code — no explanation, no markdown, no backticks.
Fixed code:"""
try:
response = model.generate_content(prompt)
fixed = _clean_code(response.text)
full_path = project_dir / file_path
full_path.write_text(fixed, encoding="utf-8")
print(f"[DevAgent] 🔧 Fixed: {file_path}")
return fixed
except Exception as e:
if _is_rate_limit(e):
raise RateLimitError(str(e))
raise
class RateLimitError(Exception):
pass
def _build_project(
description: str,
language: str,
project_name: str,
timeout: int,
speak=None,
player=None
) -> str:
"""
Full build loop:
Plan → Write files → Install deps → Open VSCode → Run → Fix loop
"""
def log(msg: str):
print(f"[DevAgent] {msg}")
if player:
player.write_log(f"[DevAgent] {msg}")
log("Planning project structure...")
try:
plan = _plan_project(description, language)
except RateLimitError:
msg = "You have reached the rate limit, sir. Please try again shortly."
if speak: speak(msg)
return msg
except ValueError as e:
msg = f"Planning failed: {e}"
if speak: speak(msg)
return msg
proj_name = project_name or plan.get("project_name", "jarvis_project")
proj_name = re.sub(r"[^\w\-]", "_", proj_name)
project_dir = PROJECTS_DIR / proj_name
project_dir.mkdir(parents=True, exist_ok=True)
files = plan.get("files", [])
entry_point = plan.get("entry_point", "main.py")
run_command = plan.get("run_command", f"python {entry_point}")
dependencies = plan.get("dependencies", [])
log(f"Project: {proj_name} | Files: {len(files)} | Entry: {entry_point}")
file_codes: dict[str, str] = {}
for file_info in files:
file_path = file_info.get("path", "")
file_desc = file_info.get("description", "")
if not file_path:
continue
log(f"Writing {file_path}...")
try:
code = _write_file(
file_path, file_desc, description,
files, language, project_dir
)
file_codes[file_path] = code
except RateLimitError:
msg = "You have reached the rate limit, sir. Please try again shortly."
if speak: speak(msg)
return msg
except Exception as e:
log(f"Failed to write {file_path}: {e}")
continue
if not file_codes:
msg = "I could not write any files for this project, sir."
if speak: speak(msg)
return msg
if dependencies:
log(f"Installing dependencies: {dependencies}")
_install_dependencies(dependencies, project_dir)
_open_vscode(project_dir)
last_output = ""
for attempt in range(1, MAX_FIX_ATTEMPTS + 1):
log(f"Running project (attempt {attempt}/{MAX_FIX_ATTEMPTS})...")
last_output = _run_project(run_command, project_dir, timeout)
log(f"Output: {last_output[:150]}")
if not _has_error(last_output):
msg = (
f"Project '{proj_name}' is working, sir. "
f"Built in {attempt} attempt{'s' if attempt > 1 else ''}. "
f"Opened in VSCode at {project_dir}."
)
if speak: speak(msg)
return f"{msg}\n\nOutput:\n{last_output}"
if attempt == MAX_FIX_ATTEMPTS:
break
error_file = _identify_error_file(last_output, list(file_codes.keys()))
if not error_file:
error_file = entry_point
log(f"Error in '{error_file}', fixing...")
try:
fixed = _fix_file(
error_file,
file_codes.get(error_file, ""),
last_output,
description,
files,
language,
project_dir
)
file_codes[error_file] = fixed
except RateLimitError:
msg = "You have reached the rate limit, sir. Please try again shortly."
if speak: speak(msg)
return msg
except Exception as e:
log(f"Fix failed: {e}")
msg = (
f"I was unable to get '{proj_name}' working after {MAX_FIX_ATTEMPTS} attempts, sir. "
f"The project is saved at {project_dir} — you can open it in VSCode and check manually."
)
if speak: speak(msg)
return f"{msg}\n\nLast error:\n{last_output[:500]}"
def dev_agent(
parameters: dict,
response=None,
player=None,
session_memory=None,
speak=None
) -> str:
"""
Called from main.py.
parameters:
description : What the project should do (required)
language : Programming language (default: python)
project_name : Optional folder name (auto-generated if not given)
timeout : Run timeout in seconds (default: 30)
"""
p = parameters or {}
description = p.get("description", "").strip()
language = p.get("language", "python").strip()
project_name = p.get("project_name", "").strip()
timeout = int(p.get("timeout", 30))
if not description:
return "Please describe the project you want me to build, sir."
return _build_project(
description = description,
language = language,
project_name = project_name,
timeout = timeout,
speak = speak,
player = player
)
+482
View File
@@ -0,0 +1,482 @@
# actions/file_controller.py
# File management — create, delete, move, rename, list, find, organize
import shutil
from pathlib import Path
from datetime import datetime
import send2trash
def _get_desktop() -> Path:
"""Returns desktop path — works on Windows, Mac, Linux."""
return Path.home() / "Desktop"
def _get_downloads() -> Path:
return Path.home() / "Downloads"
def _resolve_path(raw: str) -> Path:
"""
Resolves a path from user input.
Supports shortcuts: 'desktop', 'downloads', 'documents', 'home'
"""
shortcuts = {
"desktop": Path.home() / "Desktop",
"downloads": Path.home() / "Downloads",
"documents": Path.home() / "Documents",
"pictures": Path.home() / "Pictures",
"music": Path.home() / "Music",
"videos": Path.home() / "Videos",
"home": Path.home(),
}
lower = raw.strip().lower()
if lower in shortcuts:
return shortcuts[lower]
return Path(raw).expanduser()
def _format_size(bytes_size: int) -> str:
"""Converts bytes to human readable format."""
for unit in ["B", "KB", "MB", "GB", "TB"]:
if bytes_size < 1024:
return f"{bytes_size:.1f} {unit}"
bytes_size /= 1024
return f"{bytes_size:.1f} TB"
def list_files(path: str = "desktop", show_hidden: bool = False) -> str:
"""Lists files and folders in a directory."""
try:
target = _resolve_path(path)
if not target.exists():
return f"Path not found: {target}"
if not target.is_dir():
return f"Not a directory: {target}"
items = []
for item in sorted(target.iterdir()):
if not show_hidden and item.name.startswith("."):
continue
if item.is_dir():
items.append(f"📁 {item.name}/")
else:
size = _format_size(item.stat().st_size)
items.append(f"📄 {item.name} ({size})")
if not items:
return f"Directory is empty: {target}"
return f"Contents of {target.name}/ ({len(items)} items):\n" + "\n".join(items)
except PermissionError:
return f"Permission denied: {path}"
except Exception as e:
return f"Error listing files: {e}"
def create_file(path: str, content: str = "") -> str:
"""Creates a new file with optional content."""
try:
target = Path(path).expanduser()
target.parent.mkdir(parents=True, exist_ok=True)
target.write_text(content, encoding="utf-8")
return f"File created: {target.name}"
except Exception as e:
return f"Could not create file: {e}"
def create_folder(path: str) -> str:
"""Creates a new folder (and parent folders if needed)."""
try:
target = Path(path).expanduser()
target.mkdir(parents=True, exist_ok=True)
return f"Folder created: {target}"
except Exception as e:
return f"Could not create folder: {e}"
def delete_file(path: str, confirm: bool = True) -> str:
"""
Deletes a file or folder.
Moves to Recycle Bin on Windows if possible, otherwise permanent delete.
"""
try:
target = Path(path).expanduser()
if not target.exists():
return f"Not found: {path}"
try:
send2trash.send2trash(str(target))
return f"Moved to Recycle Bin: {target.name}"
except ImportError:
pass
# Fallback: permanent delete
if target.is_dir():
shutil.rmtree(target)
return f"Folder deleted permanently: {target.name}"
else:
target.unlink()
return f"File deleted permanently: {target.name}"
except PermissionError:
return f"Permission denied: {path}"
except Exception as e:
return f"Could not delete: {e}"
def move_file(source: str, destination: str) -> str:
"""Moves a file or folder to a new location."""
try:
src = Path(source).expanduser()
dst = _resolve_path(destination)
if not src.exists():
return f"Source not found: {source}"
if dst.is_dir():
dst = dst / src.name
dst.parent.mkdir(parents=True, exist_ok=True)
shutil.move(str(src), str(dst))
return f"Moved: {src.name} → {dst.parent.name}/"
except Exception as e:
return f"Could not move: {e}"
def copy_file(source: str, destination: str) -> str:
"""Copies a file or folder."""
try:
src = Path(source).expanduser()
dst = _resolve_path(destination)
if not src.exists():
return f"Source not found: {source}"
if dst.is_dir():
dst = dst / src.name
dst.parent.mkdir(parents=True, exist_ok=True)
if src.is_dir():
shutil.copytree(str(src), str(dst))
else:
shutil.copy2(str(src), str(dst))
return f"Copied: {src.name} → {dst.parent.name}/"
except Exception as e:
return f"Could not copy: {e}"
def rename_file(path: str, new_name: str) -> str:
"""Renames a file or folder."""
try:
target = Path(path).expanduser()
new_path = target.parent / new_name
if not target.exists():
return f"Not found: {path}"
if new_path.exists():
return f"A file named '{new_name}' already exists."
target.rename(new_path)
return f"Renamed: {target.name} → {new_name}"
except Exception as e:
return f"Could not rename: {e}"
def read_file(path: str, max_chars: int = 3000) -> str:
"""Reads and returns the content of a text file."""
try:
target = Path(path).expanduser()
if not target.exists():
return f"File not found: {path}"
if not target.is_file():
return f"Not a file: {path}"
content = target.read_text(encoding="utf-8", errors="ignore")
if len(content) > max_chars:
content = content[:max_chars] + f"\n\n... (truncated, {len(content)} total chars)"
return content
except Exception as e:
return f"Could not read file: {e}"
def write_file(path: str, content: str, append: bool = False) -> str:
"""Writes or appends content to a file."""
try:
target = Path(path).expanduser()
target.parent.mkdir(parents=True, exist_ok=True)
mode = "a" if append else "w"
with open(target, mode, encoding="utf-8") as f:
f.write(content)
action = "Appended to" if append else "Written to"
return f"{action}: {target.name}"
except Exception as e:
return f"Could not write file: {e}"
def find_files(name: str = "", extension: str = "", path: str = "home",
max_results: int = 20) -> str:
"""
Searches for files by name or extension.
Example: find_files(extension=".pdf", path="documents")
"""
try:
search_path = _resolve_path(path)
if not search_path.exists():
return f"Search path not found: {path}"
results = []
pattern = f"*{extension}" if extension else "*"
for item in search_path.rglob(pattern):
if item.is_file():
if name and name.lower() not in item.name.lower():
continue
size = _format_size(item.stat().st_size)
results.append(f"📄 {item.name} ({size}) — {item.parent}")
if len(results) >= max_results:
break
if not results:
query = name or extension or "files"
return f"No {query} found in {search_path.name}/"
return f"Found {len(results)} file(s):\n" + "\n".join(results)
except Exception as e:
return f"Search error: {e}"
def get_largest_files(path: str = "home", count: int = 10) -> str:
"""Returns the largest files in a directory."""
try:
search_path = _resolve_path(path)
if not search_path.exists():
return f"Path not found: {path}"
files = []
for item in search_path.rglob("*"):
if item.is_file():
try:
files.append((item.stat().st_size, item))
except Exception:
continue
files.sort(reverse=True)
top = files[:count]
if not top:
return "No files found."
lines = [f"Top {len(top)} largest files in {search_path.name}/:\n"]
for size, f in top:
lines.append(f" {_format_size(size):>10} {f.name} ({f.parent})")
return "\n".join(lines)
except Exception as e:
return f"Error: {e}"
def get_disk_usage(path: str = "home") -> str:
"""Returns disk usage information."""
try:
target = _resolve_path(path)
usage = shutil.disk_usage(target)
total = _format_size(usage.total)
used = _format_size(usage.used)
free = _format_size(usage.free)
pct = usage.used / usage.total * 100
return (
f"Disk usage for {target}:\n"
f" Total : {total}\n"
f" Used : {used} ({pct:.1f}%)\n"
f" Free : {free}"
)
except Exception as e:
return f"Could not get disk usage: {e}"
def organize_desktop() -> str:
"""
Organizes the desktop by grouping files into folders by type.
Creates folders: Images, Documents, Videos, Music, Archives, Others
"""
try:
desktop = _get_desktop()
type_map = {
"Images": [".jpg", ".jpeg", ".png", ".gif", ".bmp", ".webp", ".svg", ".ico"],
"Documents": [".pdf", ".doc", ".docx", ".txt", ".xls", ".xlsx", ".ppt", ".pptx", ".csv"],
"Videos": [".mp4", ".avi", ".mkv", ".mov", ".wmv", ".flv", ".webm"],
"Music": [".mp3", ".wav", ".flac", ".aac", ".ogg", ".wma"],
"Archives": [".zip", ".rar", ".7z", ".tar", ".gz"],
"Code": [".py", ".js", ".html", ".css", ".json", ".xml", ".ts", ".cpp", ".java"],
}
moved = []
skipped = []
for item in desktop.iterdir():
if item.is_dir() or item.name.startswith("."):
continue
ext = item.suffix.lower()
target_dir = None
for folder, extensions in type_map.items():
if ext in extensions:
target_dir = desktop / folder
break
if target_dir is None:
target_dir = desktop / "Others"
target_dir.mkdir(exist_ok=True)
new_path = target_dir / item.name
if new_path.exists():
skipped.append(item.name)
continue
shutil.move(str(item), str(new_path))
moved.append(f"{item.name} → {target_dir.name}/")
result = f"Desktop organized. {len(moved)} files moved."
if moved:
result += "\n" + "\n".join(moved[:10])
if len(moved) > 10:
result += f"\n... and {len(moved)-10} more."
if skipped:
result += f"\n{len(skipped)} files skipped (already exist)."
return result
except Exception as e:
return f"Could not organize desktop: {e}"
def get_file_info(path: str) -> str:
"""Returns detailed information about a file."""
try:
target = Path(path).expanduser()
if not target.exists():
return f"Not found: {path}"
stat = target.stat()
info = {
"Name": target.name,
"Type": "Folder" if target.is_dir() else "File",
"Size": _format_size(stat.st_size),
"Location": str(target.parent),
"Created": datetime.fromtimestamp(stat.st_ctime).strftime("%Y-%m-%d %H:%M"),
"Modified": datetime.fromtimestamp(stat.st_mtime).strftime("%Y-%m-%d %H:%M"),
"Extension": target.suffix or "None",
}
return "\n".join(f" {k}: {v}" for k, v in info.items())
except Exception as e:
return f"Could not get file info: {e}"
def file_controller(
parameters: dict,
response=None,
player=None,
session_memory=None
) -> str:
action = (parameters or {}).get("action", "").lower().strip()
path = (parameters or {}).get("path", "desktop")
name = (parameters or {}).get("name", "")
content = (parameters or {}).get("content", "")
def _full_path(p: str, n: str) -> str:
base = _resolve_path(p)
if n:
return str(base / n)
return str(base)
result = "Unknown action."
try:
if action == "list":
result = list_files(path)
elif action == "create_file":
full = _full_path(path, name)
result = create_file(full, content=content)
elif action == "create_folder":
full = _full_path(path, name)
result = create_folder(full)
elif action == "delete":
full = _full_path(path, name)
result = delete_file(full)
elif action == "move":
full = _full_path(path, name)
result = move_file(full, parameters.get("destination", ""))
elif action == "copy":
full = _full_path(path, name)
result = copy_file(full, parameters.get("destination", ""))
elif action == "rename":
full = _full_path(path, name)
result = rename_file(full, parameters.get("new_name", ""))
elif action == "read":
full = _full_path(path, name)
result = read_file(full)
elif action == "write":
full = _full_path(path, name)
result = write_file(
full,
content=content,
append=parameters.get("append", False)
)
elif action == "find":
result = find_files(
name=name or parameters.get("name", ""),
extension=parameters.get("extension", ""),
path=path,
max_results=parameters.get("max_results", 20)
)
elif action == "largest":
result = get_largest_files(
path=path,
count=parameters.get("count", 10)
)
elif action == "disk_usage":
result = get_disk_usage(path)
elif action == "organize_desktop":
result = organize_desktop()
elif action == "info":
full = _full_path(path, name)
result = get_file_info(full)
else:
result = f"Unknown action: '{action}'"
except Exception as e:
result = f"File controller error: {e}"
if player:
player.write_log(f"[file] {result[:60]}")
return result
+419
View File
@@ -0,0 +1,419 @@
# actions/flight_finder.py
# MARK XXV — Flight Finder
#
# Searches for flights using Google Flights via browser_control.
# Results are spoken by JARVIS. Optionally saved to Notepad or opened in browser.
#
# Flow:
# 1. Parse origin, destination, date, passengers from parameters
# 2. Open Google Flights via browser_control
# 3. Fill in search fields
# 4. Scrape results via get_text
# 5. Parse with Gemini → structured flight data
# 6. Speak top results
# 7. Optionally save to Notepad or keep browser open
#
# Cross-platform: Windows, macOS, Linux
# No API key required — Google Flights is free to access
import json
import re
import sys
import subprocess
import platform
from datetime import datetime, timedelta
from pathlib import Path
def get_base_dir() -> Path:
if getattr(sys, "frozen", False):
return Path(sys.executable).parent
return Path(__file__).resolve().parent.parent
BASE_DIR = get_base_dir()
API_CONFIG_PATH = BASE_DIR / "config" / "api_keys.json"
def _get_api_key() -> str:
with open(API_CONFIG_PATH, "r", encoding="utf-8") as f:
return json.load(f)["gemini_api_key"]
def _parse_date(raw: str) -> str:
"""
Converts natural language date to YYYY-MM-DD.
Handles: '15 Mart', 'March 15', '2025-03-15', 'next friday', etc.
Falls back to Gemini for ambiguous inputs.
"""
raw = raw.strip()
if re.match(r"\d{4}-\d{2}-\d{2}", raw):
return raw
for fmt in ("%d/%m/%Y", "%m/%d/%Y", "%d.%m.%Y", "%d-%m-%Y"):
try:
return datetime.strptime(raw, fmt).strftime("%Y-%m-%d")
except ValueError:
pass
today = datetime.now()
lower = raw.lower()
relative_map = {
"today": today,
"tomorrow": today + timedelta(days=1),
"bugün": today,
"yarın": today + timedelta(days=1),
}
for key, val in relative_map.items():
if key in lower:
return val.strftime("%Y-%m-%d")
try:
import google.generativeai as genai
genai.configure(api_key=_get_api_key())
model = genai.GenerativeModel("gemini-2.5-flash-lite")
today_str = today.strftime("%Y-%m-%d")
response = model.generate_content(
f"Today is {today_str}. Convert this date to YYYY-MM-DD format: '{raw}'. "
f"Return ONLY the date string, nothing else."
)
result = response.text.strip()
if re.match(r"\d{4}-\d{2}-\d{2}", result):
return result
except Exception:
pass
month_map = {
"january": 1, "february": 2, "march": 3, "april": 4,
"may": 5, "june": 6, "july": 7, "august": 8,
"september": 9,"october": 10, "november": 11, "december": 12,
"january": 1, "ocak": 1, "şubat": 2, "mart": 3,
"nisan": 4, "mayıs": 5, "haziran": 6, "temmuz": 7,
"ağustos": 8, "eylül": 9, "ekim": 10, "kasım": 11,
"aralık": 12,
}
for month_name, month_num in month_map.items():
if month_name in lower:
day_match = re.search(r"\d{1,2}", raw)
if day_match:
day = int(day_match.group())
year = today.year if month_num >= today.month else today.year + 1
return f"{year}-{month_num:02d}-{day:02d}"
return today.strftime("%Y-%m-%d")
def _build_google_flights_url(
origin: str,
destination: str,
date: str,
return_date: str | None = None,
passengers: int = 1,
cabin: str = "economy",
) -> str:
"""
Builds a Google Flights URL with pre-filled search parameters.
Uses the direct search URL format.
"""
cabin_map = {
"economy": "1",
"premium": "2",
"business": "3",
"first": "4",
}
cabin_code = cabin_map.get(cabin.lower(), "1")
base = "https://www.google.com/travel/flights"
if return_date:
url = (
f"{base}?q=Flights+from+{origin}+to+{destination}"
f"+on+{date}+returning+{return_date}"
f"&curr=TRY"
)
else:
url = (
f"{base}?q=Flights+from+{origin}+to+{destination}+on+{date}"
f"&curr=TRY"
)
return url
def _search_flights_browser(
origin: str,
destination: str,
date: str,
return_date: str | None,
passengers: int,
cabin: str,
) -> tuple[str, str]:
"""
Opens Google Flights in browser, waits for results, scrapes text.
Returns (raw_text, page_url).
"""
from actions.browser_control import browser_control
import time
url = _build_google_flights_url(
origin, destination, date, return_date, passengers, cabin
)
print(f"[FlightFinder] 🌐 Opening: {url}")
browser_control({"action": "go_to", "url": url})
time.sleep(5)
result = browser_control({"action": "get_text"})
return result or "", url
def _parse_flights_with_gemini(
raw_text: str,
origin: str,
destination: str,
date: str,
) -> list[dict]:
"""
Sends raw page text to Gemini and extracts structured flight data.
Returns list of flight dicts.
"""
import google.generativeai as genai
genai.configure(api_key=_get_api_key())
model = genai.GenerativeModel(
model_name="gemini-2.5-flash",
system_instruction=(
"You are a flight data extraction expert. "
"Extract flight information from raw webpage text. "
"Return ONLY valid JSON array. No explanation, no markdown."
)
)
truncated = raw_text[:12000]
prompt = (
f"Extract flight options from {origin} to {destination} on {date} "
f"from this Google Flights page text:\n\n{truncated}\n\n"
f"Return a JSON array of up to 5 flights:\n"
f'[{{"airline": "...", "departure": "HH:MM", "arrival": "HH:MM", '
f'"duration": "Xh Ym", "stops": 0, "price": "...", "currency": "..."}}]\n'
f"If no flights found, return: []"
)
try:
response = model.generate_content(prompt)
text = response.text.strip()
text = re.sub(r"```(?:json)?", "", text).strip().rstrip("`").strip()
flights = json.loads(text)
return flights if isinstance(flights, list) else []
except Exception as e:
print(f"[FlightFinder] ⚠️ Parse failed: {e}")
return []
def _format_spoken(
flights: list[dict],
origin: str,
destination: str,
date: str,
) -> str:
"""Formats flights for spoken output — concise and natural."""
if not flights:
return (
f"I couldn't find any flights from {origin} to {destination} "
f"on {date}, sir. The page may not have loaded correctly."
)
lines = [f"Here are the flights from {origin} to {destination} on {date}, sir."]
for i, f in enumerate(flights[:5], 1):
airline = f.get("airline", "Unknown airline")
departure = f.get("departure", "--:--")
arrival = f.get("arrival", "--:--")
duration = f.get("duration", "")
stops = f.get("stops", 0)
price = f.get("price", "")
currency = f.get("currency", "")
stop_str = "non-stop" if stops == 0 else f"{stops} stop{'s' if stops > 1 else ''}"
price_str = f"{price} {currency}".strip() if price else "price unavailable"
dur_str = f", {duration}" if duration else ""
lines.append(
f"Option {i}: {airline}, departing {departure}, arriving {arrival}"
f"{dur_str}, {stop_str}, {price_str}."
)
cheapest = min(
(f for f in flights if f.get("price")),
key=lambda x: re.sub(r"[^\d]", "", str(x.get("price", "99999"))) or "99999",
default=None,
)
if cheapest:
lines.append(
f"The cheapest option is {cheapest.get('airline')} "
f"at {cheapest.get('price')} {cheapest.get('currency', '')}."
)
return " ".join(lines)
def _format_notepad(
flights: list[dict],
origin: str,
destination: str,
date: str,
return_date: str | None,
page_url: str,
) -> str:
"""Formats flights for Notepad — detailed and readable."""
from datetime import datetime as dt
lines = [
"JARVIS — Flight Search Results",
"─" * 50,
f"Route : {origin} → {destination}",
f"Date : {date}",
]
if return_date:
lines.append(f"Return : {return_date}")
lines += [
f"Searched : {dt.now().strftime('%Y-%m-%d %H:%M')}",
f"Source : {page_url}",
"─" * 50,
"",
]
if not flights:
lines.append("No flights found.")
else:
for i, f in enumerate(flights, 1):
stops = f.get("stops", 0)
stop_str = "Non-stop" if stops == 0 else f"{stops} stop(s)"
lines += [
f"Flight {i}:",
f" Airline : {f.get('airline', 'N/A')}",
f" Departure : {f.get('departure', 'N/A')}",
f" Arrival : {f.get('arrival', 'N/A')}",
f" Duration : {f.get('duration', 'N/A')}",
f" Stops : {stop_str}",
f" Price : {f.get('price', 'N/A')} {f.get('currency', '')}",
"",
]
return "\n".join(lines)
def _save_to_notepad(content: str, origin: str, destination: str) -> str:
"""Saves flight results to Desktop and opens in default text editor."""
from datetime import datetime
ts = datetime.now().strftime("%Y%m%d_%H%M%S")
filename = f"flights_{origin}_{destination}_{ts}.txt".replace(" ", "_")
desktop = Path.home() / "Desktop"
desktop.mkdir(parents=True, exist_ok=True)
filepath = desktop / filename
filepath.write_text(content, encoding="utf-8")
print(f"[FlightFinder] 💾 Saved: {filepath}")
system = platform.system()
open_fn = {
"Windows": lambda p: subprocess.Popen(["notepad.exe", str(p)]),
"Darwin": lambda p: subprocess.Popen(["open", "-t", str(p)]),
"Linux": lambda p: subprocess.Popen(["xdg-open", str(p)]),
}
opener = open_fn.get(system)
if opener:
opener(filepath)
return str(filepath)
def flight_finder(
parameters: dict,
response=None,
player=None,
session_memory=None,
speak=None,
) -> str:
"""
Flight Finder — searches Google Flights and speaks results.
Parameters:
origin (str, required) — departure city or airport (e.g. "Istanbul", "IST")
destination (str, required) — arrival city or airport (e.g. "London", "LHR")
date (str, required) — departure date (any format: "15 Mart", "March 15", "2025-03-15")
return_date (str, optional) — return date for round trips
passengers (int, optional) — number of passengers (default: 1)
cabin (str, optional) — economy | premium | business | first (default: economy)
save (bool, optional) — save results to Notepad (default: False)
show_browser (bool, optional) — keep browser open after search (default: True)
Examples:
flight_finder({"origin": "Istanbul", "destination": "London", "date": "15 Mart"})
flight_finder({"origin": "IST", "destination": "JFK", "date": "2025-04-01",
"return_date": "2025-04-15", "cabin": "business", "save": True})
"""
params = parameters or {}
origin = params.get("origin", "").strip()
destination = params.get("destination", "").strip()
date_raw = params.get("date", "").strip()
return_raw = params.get("return_date", "").strip()
passengers = int(params.get("passengers", 1))
cabin = params.get("cabin", "economy").strip()
save = params.get("save", False)
if not origin or not destination:
return "Please provide both origin and destination, sir."
if not date_raw:
return "Please provide a departure date, sir."
date = _parse_date(date_raw)
return_date = _parse_date(return_raw) if return_raw else None
if player:
player.write_log(f"[FlightFinder] {origin} → {destination} on {date}")
if speak:
speak(f"Searching flights from {origin} to {destination} on {date}, sir.")
print(f"[FlightFinder] ▶️ {origin} → {destination} | {date} | {cabin} | {passengers} pax")
try:
raw_text, page_url = _search_flights_browser(
origin, destination, date, return_date, passengers, cabin
)
if not raw_text:
return "Could not retrieve flight data, sir. The page may not have loaded."
if speak:
speak("Analysing the results now.")
flights = _parse_flights_with_gemini(raw_text, origin, destination, date)
spoken = _format_spoken(flights, origin, destination, date)
if speak:
speak(spoken)
result = spoken
if save and flights:
notepad_content = _format_notepad(
flights, origin, destination, date, return_date, page_url
)
saved_path = _save_to_notepad(notepad_content, origin, destination)
result += f" Results saved to Desktop: {saved_path}"
return result
except Exception as e:
print(f"[FlightFinder] ❌ Error: {e}")
return f"Flight search failed, sir: {e}"
+208
View File
@@ -0,0 +1,208 @@
# actions/open_app.py
# MARK XXV — Cross-Platform App Launcher
import time
import subprocess
import platform
import shutil
try:
import psutil
_PSUTIL = True
except ImportError:
_PSUTIL = False
_APP_ALIASES = {
"whatsapp": {"Windows": "WhatsApp", "Darwin": "WhatsApp", "Linux": "whatsapp"},
"chrome": {"Windows": "chrome", "Darwin": "Google Chrome", "Linux": "google-chrome"},
"google chrome": {"Windows": "chrome", "Darwin": "Google Chrome", "Linux": "google-chrome"},
"firefox": {"Windows": "firefox", "Darwin": "Firefox", "Linux": "firefox"},
"spotify": {"Windows": "Spotify", "Darwin": "Spotify", "Linux": "spotify"},
"vscode": {"Windows": "code", "Darwin": "Visual Studio Code", "Linux": "code"},
"visual studio code": {"Windows": "code", "Darwin": "Visual Studio Code", "Linux": "code"},
"discord": {"Windows": "Discord", "Darwin": "Discord", "Linux": "discord"},
"telegram": {"Windows": "Telegram", "Darwin": "Telegram", "Linux": "telegram"},
"instagram": {"Windows": "Instagram", "Darwin": "Instagram", "Linux": "instagram"},
"tiktok": {"Windows": "TikTok", "Darwin": "TikTok", "Linux": "tiktok"},
"notepad": {"Windows": "notepad.exe", "Darwin": "TextEdit", "Linux": "gedit"},
"calculator": {"Windows": "calc.exe", "Darwin": "Calculator", "Linux": "gnome-calculator"},
"terminal": {"Windows": "cmd.exe", "Darwin": "Terminal", "Linux": "gnome-terminal"},
"cmd": {"Windows": "cmd.exe", "Darwin": "Terminal", "Linux": "bash"},
"explorer": {"Windows": "explorer.exe", "Darwin": "Finder", "Linux": "nautilus"},
"file explorer": {"Windows": "explorer.exe", "Darwin": "Finder", "Linux": "nautilus"},
"paint": {"Windows": "mspaint.exe", "Darwin": "Preview", "Linux": "gimp"},
"word": {"Windows": "winword", "Darwin": "Microsoft Word", "Linux": "libreoffice --writer"},
"excel": {"Windows": "excel", "Darwin": "Microsoft Excel", "Linux": "libreoffice --calc"},
"powerpoint": {"Windows": "powerpnt", "Darwin": "Microsoft PowerPoint","Linux": "libreoffice --impress"},
"vlc": {"Windows": "vlc", "Darwin": "VLC", "Linux": "vlc"},
"zoom": {"Windows": "Zoom", "Darwin": "zoom.us", "Linux": "zoom"},
"slack": {"Windows": "Slack", "Darwin": "Slack", "Linux": "slack"},
"steam": {"Windows": "steam", "Darwin": "Steam", "Linux": "steam"},
"task manager": {"Windows": "taskmgr.exe", "Darwin": "Activity Monitor", "Linux": "gnome-system-monitor"},
"settings": {"Windows": "ms-settings:", "Darwin": "System Preferences", "Linux": "gnome-control-center"},
"powershell": {"Windows": "powershell.exe", "Darwin": "Terminal", "Linux": "bash"},
"edge": {"Windows": "msedge", "Darwin": "Microsoft Edge", "Linux": "microsoft-edge"},
"brave": {"Windows": "brave", "Darwin": "Brave Browser", "Linux": "brave-browser"},
"obsidian": {"Windows": "Obsidian", "Darwin": "Obsidian", "Linux": "obsidian"},
"notion": {"Windows": "Notion", "Darwin": "Notion", "Linux": "notion"},
"blender": {"Windows": "blender", "Darwin": "Blender", "Linux": "blender"},
"capcut": {"Windows": "CapCut", "Darwin": "CapCut", "Linux": "capcut"},
"postman": {"Windows": "Postman", "Darwin": "Postman", "Linux": "postman"},
"figma": {"Windows": "Figma", "Darwin": "Figma", "Linux": "figma"},
}
def _normalize(raw: str) -> str:
system = platform.system()
key = raw.lower().strip()
if key in _APP_ALIASES:
return _APP_ALIASES[key].get(system, raw)
for alias_key, os_map in _APP_ALIASES.items():
if alias_key in key or key in alias_key:
return os_map.get(system, raw)
return raw
def _is_running(app_name: str) -> bool:
if not _PSUTIL:
return True
app_lower = app_name.lower().replace(" ", "").replace(".exe", "")
try:
for proc in psutil.process_iter(["name"]):
try:
proc_name = proc.info["name"].lower().replace(" ", "").replace(".exe", "")
if app_lower in proc_name or proc_name in app_lower:
return True
except (psutil.NoSuchProcess, psutil.AccessDenied):
continue
except Exception:
pass
return False
def _launch_windows(app_name: str) -> bool:
try:
import pyautogui
pyautogui.PAUSE = 0.1
pyautogui.press("win")
time.sleep(0.6)
pyautogui.write(app_name, interval=0.05)
time.sleep(0.8)
pyautogui.press("enter")
time.sleep(3.0)
return True
except Exception as e:
print(f"[open_app] ⚠️ Windows launch failed: {e}")
return False
def _launch_macos(app_name: str) -> bool:
try:
result = subprocess.run(["open", "-a", app_name], capture_output=True, timeout=8)
if result.returncode == 0:
time.sleep(1.0)
return True
except Exception:
pass
try:
result = subprocess.run(["open", "-a", f"{app_name}.app"], capture_output=True, timeout=8)
if result.returncode == 0:
time.sleep(1.0)
return True
except Exception:
pass
try:
import pyautogui
pyautogui.hotkey("command", "space")
time.sleep(0.6)
pyautogui.write(app_name, interval=0.05)
time.sleep(0.8)
pyautogui.press("enter")
time.sleep(1.5)
return True
except Exception as e:
print(f"[open_app] ⚠️ macOS Spotlight failed: {e}")
return False
def _launch_linux(app_name: str) -> bool:
binary = (
shutil.which(app_name) or
shutil.which(app_name.lower()) or
shutil.which(app_name.lower().replace(" ", "-"))
)
if binary:
try:
subprocess.Popen([binary], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
time.sleep(1.0)
return True
except Exception:
pass
try:
subprocess.run(["xdg-open", app_name], capture_output=True, timeout=5)
return True
except Exception:
pass
try:
desktop_name = app_name.lower().replace(" ", "-")
subprocess.run(["gtk-launch", desktop_name], capture_output=True, timeout=5)
return True
except Exception:
pass
return False
_OS_LAUNCHERS = {
"Windows": _launch_windows,
"Darwin": _launch_macos,
"Linux": _launch_linux,
}
def open_app(
parameters=None,
response=None,
player=None,
session_memory=None,
) -> str:
app_name = (parameters or {}).get("app_name", "").strip()
if not app_name:
return "Please specify which application to open, sir."
system = platform.system()
launcher = _OS_LAUNCHERS.get(system)
if launcher is None:
return f"Unsupported OS: {system}"
normalized = _normalize(app_name)
print(f"[open_app] 🚀 Launching: {app_name} → {normalized} ({system})")
if player:
player.write_log(f"[open_app] {app_name}")
try:
success = launcher(normalized)
if success:
return f"Opened {app_name} successfully, sir."
if normalized != app_name:
success = launcher(app_name)
if success:
return f"Opened {app_name} successfully, sir."
return (
f"I tried to open {app_name}, sir, but couldn't confirm it launched. "
f"It may still be loading or might not be installed."
)
except Exception as e:
print(f"[open_app] ❌ {e}")
return f"Failed to open {app_name}, sir: {e}"
+156
View File
@@ -0,0 +1,156 @@
# actions/reminder.py
import subprocess
import os
import sys
from datetime import datetime
def reminder(
parameters: dict,
response: str | None = None,
player=None,
session_memory=None
) -> str:
"""
Sets a timed reminder using Windows Task Scheduler.
parameters:
- date (str) YYYY-MM-DD
- time (str) HH:MM
- message (str)
Returns a result string — Live API voices it automatically.
No edge_speak needed.
"""
date_str = parameters.get("date")
time_str = parameters.get("time")
message = parameters.get("message", "Reminder")
if not date_str or not time_str:
return "I need both a date and a time to set a reminder."
try:
target_dt = datetime.strptime(f"{date_str} {time_str}", "%Y-%m-%d %H:%M")
if target_dt <= datetime.now():
return "That time is already in the past."
task_name = f"MARKReminder_{target_dt.strftime('%Y%m%d_%H%M')}"
safe_message = message.replace('"', '').replace("'", "").strip()[:200]
python_exe = sys.executable
if python_exe.lower().endswith("python.exe"):
pythonw = python_exe.replace("python.exe", "pythonw.exe")
if os.path.exists(pythonw):
python_exe = pythonw
temp_dir = os.environ.get("TEMP", "C:\\Temp")
notify_script = os.path.join(temp_dir, f"{task_name}.pyw")
project_root = os.path.abspath(
os.path.join(os.path.dirname(__file__), "..")
)
script_code = f'''import sys, os, time
sys.path.insert(0, r"{project_root}")
try:
import winsound
for freq in [800, 1000, 1200]:
winsound.Beep(freq, 200)
time.sleep(0.1)
except Exception:
pass
try:
from win10toast import ToastNotifier
ToastNotifier().show_toast(
"MARK Reminder",
"{safe_message}",
duration=15,
threaded=False
)
except Exception:
try:
import subprocess
subprocess.run(["msg", "*", "/TIME:30", "{safe_message}"], shell=True)
except Exception:
pass
time.sleep(3)
try:
os.remove(__file__)
except Exception:
pass
'''
with open(notify_script, "w", encoding="utf-8") as f:
f.write(script_code)
xml_content = f'''<?xml version="1.0" encoding="UTF-16"?>
<Task version="1.2" xmlns="http://schemas.microsoft.com/windows/2004/02/mit/task">
<RegistrationInfo>
<Description>MARK Reminder: {safe_message}</Description>
</RegistrationInfo>
<Triggers>
<TimeTrigger>
<StartBoundary>{target_dt.strftime("%Y-%m-%dT%H:%M:%S")}</StartBoundary>
<Enabled>true</Enabled>
</TimeTrigger>
</Triggers>
<Actions>
<Exec>
<Command>{python_exe}</Command>
<Arguments>"{notify_script}"</Arguments>
</Exec>
</Actions>
<Settings>
<MultipleInstancesPolicy>IgnoreNew</MultipleInstancesPolicy>
<DisallowStartIfOnBatteries>false</DisallowStartIfOnBatteries>
<StopIfGoingOnBatteries>false</StopIfGoingOnBatteries>
<StartWhenAvailable>true</StartWhenAvailable>
<WakeToRun>true</WakeToRun>
<ExecutionTimeLimit>PT5M</ExecutionTimeLimit>
<Enabled>true</Enabled>
</Settings>
<Principals>
<Principal>
<LogonType>InteractiveToken</LogonType>
<RunLevel>LeastPrivilege</RunLevel>
</Principal>
</Principals>
</Task>'''
xml_path = os.path.join(temp_dir, f"{task_name}.xml")
with open(xml_path, "w", encoding="utf-16") as f:
f.write(xml_content)
result = subprocess.run(
f'schtasks /Create /TN "{task_name}" /XML "{xml_path}" /F',
shell=True, capture_output=True, text=True
)
try:
os.remove(xml_path)
except Exception:
pass
if result.returncode != 0:
err = result.stderr.strip() or result.stdout.strip()
print(f"[Reminder] ❌ schtasks failed: {err}")
try:
os.remove(notify_script)
except Exception:
pass
return "I couldn't schedule the reminder due to a system error."
if player:
player.write_log(f"[reminder] set for {date_str} {time_str}")
return f"Reminder set for {target_dt.strftime('%B %d at %I:%M %p')}."
except ValueError:
return "I couldn't understand that date or time format."
except Exception as e:
return f"Something went wrong while scheduling the reminder: {str(e)[:80]}"
+404
View File
@@ -0,0 +1,404 @@
"""
actions/screen_processor.py — Gemini Live API — IMAGE-ONLY SESSION v8
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
v8 Changes:
- Auto camera detection on first use — saves to config
- No hardcoded camera index
- mic_loop removed (no double response issue)
- Image-only session, no conflict with main.py
"""
import asyncio
import base64
import io
import json
import re
import os
import sys
import time
import threading
import cv2
import mss
import mss.tools
import pyaudio
from pathlib import Path
try:
import PIL.Image
_PIL_OK = True
except ImportError:
_PIL_OK = False
from google import genai
from google.genai import types
def get_base_dir():
if getattr(sys, "frozen", False):
return Path(sys.executable).parent
return Path(__file__).resolve().parent.parent
BASE_DIR = get_base_dir()
API_CONFIG_PATH = BASE_DIR / "config" / "api_keys.json"
LIVE_MODEL = "models/gemini-2.5-flash-native-audio-preview-12-2025"
FORMAT = pyaudio.paInt16
CHANNELS = 1
RECEIVE_SAMPLE_RATE = 24000
CHUNK_SIZE = 1024
IMG_MAX_W = 640
IMG_MAX_H = 360
JPEG_Q = 55
SYSTEM_PROMPT = (
"You are JARVIS from Iron Man movies. "
"Analyze images with technical precision and intelligence. "
"Help the user in a way they can understand — don't be overly complex. "
"Be concise, smart, and helpful like Tony Stark's AI assistant. "
"Respond in maximum 2 short sentences. Speed is priority. "
"Address the user as 'sir' for a tone of respect. "
"Ask if the user needs any further help with their problem."
)
def _get_api_key() -> str:
try:
with open(API_CONFIG_PATH, "r", encoding="utf-8") as f:
keys = json.load(f)
key = keys.get("gemini_api_key", "")
if not key:
raise ValueError("gemini_api_key not found")
return key
except Exception as e:
raise RuntimeError(f"Could not load API key: {e}")
def _get_camera_index() -> int:
"""
Reads camera index from config.
If not set, auto-detects the best camera and saves it for future use.
Runs only once — after that, config value is used directly.
"""
try:
with open(API_CONFIG_PATH, "r", encoding="utf-8") as f:
cfg = json.load(f)
if "camera_index" in cfg:
return int(cfg["camera_index"])
except Exception:
pass
print("[Camera] 🔍 No camera index in config. Auto-detecting...")
best_index = 0
for idx in range(6):
cap = cv2.VideoCapture(idx, cv2.CAP_DSHOW)
if not cap.isOpened():
cap.release()
continue
for _ in range(5):
cap.read()
ret, frame = cap.read()
cap.release()
if ret and frame is not None and frame.mean() > 5:
best_index = idx
print(f"[Camera] ✅ Camera found at index {idx} — saving to config.")
break
else:
print(f"[Camera] ⚠️ Index {idx}: no valid frame (black or empty).")
try:
cfg = {}
if API_CONFIG_PATH.exists():
with open(API_CONFIG_PATH, "r", encoding="utf-8") as f:
cfg = json.load(f)
cfg["camera_index"] = best_index
with open(API_CONFIG_PATH, "w", encoding="utf-8") as f:
json.dump(cfg, f, indent=4)
print(f"[Camera] 💾 Camera index {best_index} saved to config.")
except Exception as e:
print(f"[Camera] ⚠️ Could not save camera index: {e}")
return best_index
def _to_jpeg(img_bytes: bytes) -> bytes:
if not _PIL_OK:
return img_bytes
img = PIL.Image.open(io.BytesIO(img_bytes)).convert("RGB")
img.thumbnail([IMG_MAX_W, IMG_MAX_H], PIL.Image.BILINEAR)
buf = io.BytesIO()
img.save(buf, format="JPEG", quality=JPEG_Q, optimize=False)
return buf.getvalue()
def _capture_screenshot() -> bytes:
with mss.mss() as sct:
shot = sct.grab(sct.monitors[1])
png_bytes = mss.tools.to_png(shot.rgb, shot.size)
return _to_jpeg(png_bytes)
def _capture_camera() -> bytes:
camera_index = _get_camera_index()
cap = cv2.VideoCapture(camera_index, cv2.CAP_DSHOW)
if not cap.isOpened():
raise RuntimeError(f"Camera could not be opened: index {camera_index}")
for _ in range(10):
cap.read()
ret, frame = cap.read()
cap.release()
if not ret or frame is None:
raise RuntimeError("Could not capture camera frame.")
if _PIL_OK:
rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
img = PIL.Image.fromarray(rgb)
img.thumbnail([IMG_MAX_W, IMG_MAX_H], PIL.Image.BILINEAR)
buf = io.BytesIO()
img.save(buf, format="JPEG", quality=JPEG_Q, optimize=False)
return buf.getvalue()
_, buf = cv2.imencode(".jpg", frame, [cv2.IMWRITE_JPEG_QUALITY, JPEG_Q])
return buf.tobytes()
class _LiveSession:
"""
Image-only analysis session.
No microphone — no conflict with main.py session.
Sends image + question, plays audio response.
"""
def __init__(self):
self._loop: asyncio.AbstractEventLoop | None = None
self._thread: threading.Thread | None = None
self._session = None
self._out_queue: asyncio.Queue | None = None
self._audio_in: asyncio.Queue | None = None
self._ready: threading.Event = threading.Event()
self._player = None
self._pya = pyaudio.PyAudio()
self._send_lock: asyncio.Lock | None = None
def start(self, player=None):
if self._thread and self._thread.is_alive():
return
self._player = player
self._thread = threading.Thread(
target=self._run_loop, daemon=True, name="VisionSessionThread"
)
self._thread.start()
ok = self._ready.wait(timeout=20)
if not ok:
raise RuntimeError("Vision session did not start within 20s.")
print("[ScreenProcess] ✅ Vision session ready (no mic)")
def _run_loop(self):
self._loop = asyncio.new_event_loop()
asyncio.set_event_loop(self._loop)
self._loop.run_until_complete(self._main())
async def _main(self):
self._out_queue = asyncio.Queue(maxsize=30)
self._audio_in = asyncio.Queue()
self._send_lock = asyncio.Lock()
client = genai.Client(
api_key=_get_api_key(),
http_options={"api_version": "v1beta"}
)
config = types.LiveConnectConfig(
response_modalities=["AUDIO"],
output_audio_transcription={},
system_instruction=SYSTEM_PROMPT,
speech_config=types.SpeechConfig(
voice_config=types.VoiceConfig(
prebuilt_voice_config=types.PrebuiltVoiceConfig(
voice_name="Charon"
)
)
),
)
while True:
try:
print("[ScreenProcess] 🔌 Vision session connecting...")
async with client.aio.live.connect(model=LIVE_MODEL, config=config) as session:
self._session = session
self._ready.set()
print("[ScreenProcess] ✅ Vision session connected")
async with asyncio.TaskGroup() as tg:
tg.create_task(self._send_loop())
tg.create_task(self._recv_loop())
tg.create_task(self._play_loop())
except Exception as e:
print(f"[ScreenProcess] ⚠️ Disconnected: {e} — reconnecting...")
self._session = None
self._ready.clear()
await asyncio.sleep(2)
self._ready.set()
# ── Send loop ──
async def _send_loop(self):
"""Sends (image_bytes, mime_type, user_text) tuples from queue."""
while True:
item = await self._out_queue.get()
if self._session:
image_bytes, mime_type, user_text = item
try:
b64 = base64.b64encode(image_bytes).decode("utf-8")
await self._session.send_client_content(
turns={
"parts": [
{"inline_data": {"mime_type": mime_type, "data": b64}},
{"text": user_text}
]
},
turn_complete=True
)
print("[ScreenProcess] ✅ Image sent")
except Exception as e:
print(f"[ScreenProcess] ⚠️ Send error: {e}")
async def _recv_loop(self):
transcript_buf: list[str] = []
try:
async for response in self._session.receive():
if response.data:
await self._audio_in.put(response.data)
sc = response.server_content
if not sc:
continue
if sc.output_transcription and sc.output_transcription.text:
chunk = sc.output_transcription.text.strip()
if chunk:
transcript_buf.append(chunk)
if sc.turn_complete:
if transcript_buf and self._player:
full = re.sub(r'\s+', ' ', " ".join(transcript_buf)).strip()
if full:
self._player.write_log(f"Jarvis: {full}")
print(f"[ScreenProcess] 💬 {full}")
transcript_buf = []
except Exception as e:
print(f"[ScreenProcess] ⚠️ Recv error: {e}")
transcript_buf = []
await asyncio.sleep(0.3)
async def _play_loop(self):
stream = await asyncio.to_thread(
self._pya.open,
format=FORMAT, channels=CHANNELS,
rate=RECEIVE_SAMPLE_RATE, output=True,
)
try:
while True:
chunk = await self._audio_in.get()
await asyncio.to_thread(stream.write, chunk)
finally:
stream.close()
def analyze(self, image_bytes: bytes, mime_type: str, user_text: str):
"""Called from main thread — puts image into async queue."""
if not self._loop:
return
asyncio.run_coroutine_threadsafe(
self._out_queue.put((image_bytes, mime_type, user_text)),
self._loop
)
def is_ready(self) -> bool:
return self._session is not None
_live = _LiveSession()
_started = False
_start_lock = threading.Lock()
def _ensure_started(player=None):
global _started
with _start_lock:
if not _started:
_live.start(player=player)
_started = True
elif player is not None:
_live._player = player
def screen_process(
parameters: dict,
response: str | None = None,
player=None,
session_memory=None,
) -> bool:
user_text = (parameters or {}).get("text") or (parameters or {}).get("user_text", "")
user_text = (user_text or "").strip()
if not user_text:
print("[ScreenProcess] ⚠️ No user_text provided.")
return False
angle = (parameters or {}).get("angle", "screen").lower().strip()
print(f"[ScreenProcess] angle={angle!r} text={user_text!r}")
_ensure_started(player=player)
try:
if angle == "camera":
image_bytes = _capture_camera()
mime_type = "image/jpeg"
print("[ScreenProcess] 📷 Camera captured")
else:
image_bytes = _capture_screenshot()
mime_type = "image/jpeg" if _PIL_OK else "image/png"
print("[ScreenProcess] 🖥️ Screen captured")
except Exception as e:
import traceback; traceback.print_exc()
print(f"[ScreenProcess] ❌ Capture error: {e}")
return False
print(f"[ScreenProcess] 📦 {len(image_bytes)} bytes → sending")
_live.analyze(image_bytes, mime_type, user_text)
return True
def warmup_session(player=None):
"""
Optional: pre-warm the session.
Do NOT call from main.py — causes double session issue.
Only use when testing screen_processor.py standalone.
"""
try:
_ensure_started(player=player)
except Exception as e:
print(f"[ScreenProcess] ⚠️ Warmup error: {e}")
if __name__ == "__main__":
print("[TEST] screen_processor.py v8 — image-only session")
print("=" * 50)
mode = input("screen / camera (default: screen): ").strip().lower() or "screen"
request = input("Question (Enter for default): ").strip() or "What do you see? Be brief."
t0 = time.perf_counter()
warmup_session()
print(f"Session ready — {time.perf_counter()-t0:.2f}s\n")
t1 = time.perf_counter()
result = screen_process({"angle": mode, "text": request}, player=None)
print(f"Sent — {time.perf_counter()-t1:.3f}s | audio incoming...")
time.sleep(8)
print(f"\n{'✅' if result else '❌'}")
+215
View File
@@ -0,0 +1,215 @@
# actions/send_message.py
# Universal messaging — WhatsApp & Instagram
# Uses visual element detection (pyautogui + screen search) instead of
# hardcoded tab/click sequences — works on any screen resolution.
import time
import pyautogui
from pathlib import Path
pyautogui.FAILSAFE = True
pyautogui.PAUSE = 0.08
def _open_app(app_name: str) -> bool:
"""Opens an app via Windows search."""
try:
pyautogui.press("win")
time.sleep(0.4)
pyautogui.write(app_name, interval=0.04)
time.sleep(0.5)
pyautogui.press("enter")
time.sleep(2.0)
return True
except Exception as e:
print(f"[SendMessage] Could not open {app_name}: {e}")
return False
def _search_contact(contact: str, platform: str):
"""
Searches for a contact inside the messaging app.
Uses Ctrl+F (universal search shortcut) then types contact name.
"""
time.sleep(0.5)
pyautogui.hotkey("ctrl", "f")
time.sleep(0.4)
pyautogui.hotkey("ctrl", "a")
pyautogui.write(contact, interval=0.04)
time.sleep(0.8)
pyautogui.press("enter")
time.sleep(0.6)
def _type_and_send(message: str):
"""Types message and sends it."""
pyautogui.press("tab")
time.sleep(0.2)
pyautogui.hotkey("ctrl", "a")
pyautogui.write(message, interval=0.03)
time.sleep(0.2)
pyautogui.press("enter")
time.sleep(0.3)
def _send_whatsapp(receiver: str, message: str) -> str:
"""
Sends a WhatsApp message via the Windows desktop app.
Steps: Open WhatsApp → Search contact → Click → Type → Send
"""
try:
if not _open_app("WhatsApp"):
return "Could not open WhatsApp."
time.sleep(1.5)
pyautogui.hotkey("ctrl", "f")
time.sleep(0.4)
pyautogui.hotkey("ctrl", "a")
pyautogui.write(receiver, interval=0.04)
time.sleep(1.0)
pyautogui.press("enter")
time.sleep(0.8)
pyautogui.write(message, interval=0.03)
time.sleep(0.2)
pyautogui.press("enter")
return f"Message sent to {receiver} via WhatsApp."
except Exception as e:
return f"WhatsApp error: {e}"
def _send_instagram(receiver: str, message: str) -> str:
"""
Sends an Instagram DM via browser (instagram.com).
Steps: Open Chrome → Go to instagram.com/direct → Search contact → Send
"""
try:
import webbrowser
webbrowser.open("https://www.instagram.com/direct/new/")
time.sleep(3.5)
pyautogui.write(receiver, interval=0.05)
time.sleep(1.5)
pyautogui.press("down")
time.sleep(0.3)
pyautogui.press("enter")
time.sleep(0.5)
for _ in range(3):
pyautogui.press("tab")
time.sleep(0.1)
pyautogui.press("enter")
time.sleep(1.5)
pyautogui.write(message, interval=0.04)
time.sleep(0.2)
pyautogui.press("enter")
return f"Message sent to {receiver} via Instagram."
except Exception as e:
return f"Instagram error: {e}"
def _send_telegram(receiver: str, message: str) -> str:
"""Sends a Telegram message via Windows desktop app."""
try:
if not _open_app("Telegram"):
return "Could not open Telegram."
time.sleep(1.5)
pyautogui.hotkey("ctrl", "f")
time.sleep(0.4)
pyautogui.write(receiver, interval=0.04)
time.sleep(1.0)
pyautogui.press("enter")
time.sleep(0.8)
pyautogui.write(message, interval=0.03)
time.sleep(0.2)
pyautogui.press("enter")
return f"Message sent to {receiver} via Telegram."
except Exception as e:
return f"Telegram error: {e}"
def _send_generic(platform: str, receiver: str, message: str) -> str:
"""
For any other platform not explicitly supported.
Opens the app, searches for contact, types and sends.
Works for: Messenger, Discord, Signal, etc.
"""
try:
if not _open_app(platform):
return f"Could not open {platform}."
time.sleep(1.5)
pyautogui.hotkey("ctrl", "f")
time.sleep(0.4)
pyautogui.write(receiver, interval=0.04)
time.sleep(1.0)
pyautogui.press("enter")
time.sleep(0.8)
pyautogui.write(message, interval=0.03)
time.sleep(0.2)
pyautogui.press("enter")
return f"Message sent to {receiver} via {platform}."
except Exception as e:
return f"{platform} error: {e}"
def send_message(
parameters: dict,
response=None,
player=None,
session_memory=None
) -> str:
"""
Called from main.py.
parameters:
receiver : Contact name to send to
message_text : The message content
platform : whatsapp | instagram | telegram | <any app name>
Default: whatsapp
"""
params = parameters or {}
receiver = params.get("receiver", "").strip()
message_text = params.get("message_text", "").strip()
platform = params.get("platform", "whatsapp").strip().lower()
if not receiver:
return "Please specify who to send the message to, sir."
if not message_text:
return "Please specify what message to send, sir."
print(f"[SendMessage] 📨 {platform} → {receiver}: {message_text[:40]}")
if player:
player.write_log(f"[msg] Sending to {receiver} via {platform}...")
if "whatsapp" in platform or "wp" in platform or "wapp" in platform:
result = _send_whatsapp(receiver, message_text)
elif "instagram" in platform or "ig" in platform or "insta" in platform:
result = _send_instagram(receiver, message_text)
elif "telegram" in platform or "tg" in platform:
result = _send_telegram(receiver, message_text)
else:
result = _send_generic(platform, receiver, message_text)
print(f"[SendMessage] ✅ {result}")
if player:
player.write_log(f"[msg] {result}")
return result
+62
View File
@@ -0,0 +1,62 @@
# actions/weather_report.py
import webbrowser
from urllib.parse import quote_plus
def weather_action(
parameters: dict,
player=None,
session_memory=None
):
"""
Weather report action.
Opens a Google weather search and gives a short spoken confirmation.
"""
city = parameters.get("city")
time = parameters.get("time")
if not city or not isinstance(city, str):
msg = "Sir, the city is missing for the weather report."
_speak_and_log(msg, player)
return msg
city = city.strip()
if not time or not isinstance(time, str):
time = "today"
else:
time = time.strip()
search_query = f"weather in {city} {time}"
encoded_query = quote_plus(search_query)
url = f"https://www.google.com/search?q={encoded_query}"
try:
webbrowser.open(url)
except Exception:
msg = f"Sir, I couldn't open the browser for the weather report."
_speak_and_log(msg, player)
return msg
msg = f"Showing the weather for {city}, {time}, sir."
_speak_and_log(msg, player)
if session_memory:
try:
session_memory.set_last_search(
query=search_query,
response=msg
)
except Exception:
pass
return msg
def _speak_and_log(message: str, player=None):
if player:
try:
player.write_log(f"JARVIS: {message}")
except Exception:
pass
+135
View File
@@ -0,0 +1,135 @@
# actions/web_search.py
# MARK XXV — Web Search
# Primary: Gemini google_search (yeni google.genai SDK)
# Fallback: DuckDuckGo (ddgs)
import json
import sys
from pathlib import Path
def get_base_dir() -> Path:
if getattr(sys, "frozen", False):
return Path(sys.executable).parent
return Path(__file__).resolve().parent.parent
BASE_DIR = get_base_dir()
API_CONFIG_PATH = BASE_DIR / "config" / "api_keys.json"
def _get_api_key() -> str:
with open(API_CONFIG_PATH, "r", encoding="utf-8") as f:
return json.load(f)["gemini_api_key"]
def _gemini_search(query: str) -> str:
from google import genai
client = genai.Client(api_key=_get_api_key())
response = client.models.generate_content(
model="gemini-2.5-flash-lite",
contents=query,
config={"tools": [{"google_search": {}}]}
)
text = ""
for part in response.candidates[0].content.parts:
if hasattr(part, "text") and part.text:
text += part.text
if not text.strip():
raise ValueError("Empty response")
return text.strip()
def _ddg_search(query: str, max_results: int = 6) -> list:
try:
from ddgs import DDGS
except ImportError:
from duckduckgo_search import DDGS
results = []
with DDGS() as ddgs:
for r in ddgs.text(query, max_results=max_results):
results.append({
"title": r.get("title", ""),
"snippet": r.get("body", ""),
"url": r.get("href", ""),
})
return results
def _format_ddg(query: str, results: list) -> str:
if not results:
return f"No results found for: {query}"
lines = [f"Search results for: {query}\n"]
for i, r in enumerate(results, 1):
if r.get("title"): lines.append(f"{i}. {r['title']}")
if r.get("snippet"): lines.append(f" {r['snippet']}")
if r.get("url"): lines.append(f" {r['url']}")
lines.append("")
return "\n".join(lines).strip()
def _compare(items: list, aspect: str) -> str:
query = f"Compare {', '.join(items)} in terms of {aspect}. Give specific facts and data."
try:
return _gemini_search(query)
except Exception as e:
print(f"[WebSearch] ⚠️ Gemini compare failed: {e}")
all_results = {}
for item in items:
try:
all_results[item] = _ddg_search(f"{item} {aspect}", max_results=3)
except Exception:
all_results[item] = []
lines = [f"Comparison — {aspect.upper()}\n{'─'*40}"]
for item in items:
lines.append(f"\n▸ {item}")
for r in all_results.get(item, [])[:2]:
if r.get("snippet"):
lines.append(f" • {r['snippet']}")
return "\n".join(lines)
def web_search(
parameters: dict,
response=None,
player=None,
session_memory=None,
) -> str:
params = parameters or {}
query = params.get("query", "").strip()
mode = params.get("mode", "search").lower()
items = params.get("items", [])
aspect = params.get("aspect", "general")
if not query and not items:
return "Please provide a search query, sir."
if items and mode != "compare":
mode = "compare"
if player:
player.write_log(f"[Search] {query or ', '.join(items)}")
print(f"[WebSearch] 🔍 Query: {query!r} Mode: {mode}")
try:
if mode == "compare" and items:
print(f"[WebSearch] 📊 Comparing: {items}")
result = _compare(items, aspect)
print("[WebSearch] ✅ Compare done.")
return result
print("[WebSearch] 🌐 Gemini search...")
try:
result = _gemini_search(query)
print("[WebSearch] ✅ Gemini OK.")
return result
except Exception as e:
print(f"[WebSearch] ⚠️ Gemini failed ({e}), trying DDG...")
results = _ddg_search(query)
result = _format_ddg(query, results)
print(f"[WebSearch] ✅ DDG: {len(results)} results.")
return result
except Exception as e:
print(f"[WebSearch] ❌ Failed: {e}")
return f"Search failed, sir: {e}"
+517
View File
@@ -0,0 +1,517 @@
# actions/youtube_video.py
# MARK XXV — YouTube Controller
#
# Features:
# - play : Search and play a YouTube video (original feature, preserved)
# - summarize : Get transcript → Gemini summary → speak + optional Notepad
# - get_info : Video title, channel, views, duration (scraping)
# - trending : Fetch trending videos for a region
import json
import re
import sys
import time
import threading
from pathlib import Path
import pyautogui
import numpy as np
import cv2
from PIL import ImageGrab
try:
import requests
from bs4 import BeautifulSoup
_REQUESTS_OK = True
except ImportError:
_REQUESTS_OK = False
try:
from youtube_transcript_api import YouTubeTranscriptApi
_TRANSCRIPT_OK = True
except ImportError:
_TRANSCRIPT_OK = False
def get_base_dir() -> Path:
if getattr(sys, "frozen", False):
return Path(sys.executable).parent
return Path(__file__).resolve().parent.parent
BASE_DIR = get_base_dir()
API_CONFIG_PATH = BASE_DIR / "config" / "api_keys.json"
HEADERS = {
"User-Agent": (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
"AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/120.0.0.0 Safari/537.36"
),
"Accept-Language": "en-US,en;q=0.9",
}
def _get_api_key() -> str:
with open(API_CONFIG_PATH, "r", encoding="utf-8") as f:
return json.load(f)["gemini_api_key"]
def open_browser():
pyautogui.press("win")
time.sleep(0.4)
pyautogui.write("browser", interval=0.05)
time.sleep(0.3)
pyautogui.press("enter")
time.sleep(2.0)
def find_video_thumbnails() -> list[tuple[int, int]]:
try:
screenshot = ImageGrab.grab()
img = np.array(screenshot)
screen_h, screen_w = img.shape[:2]
roi_top = int(screen_h * 0.10)
roi_bottom = int(screen_h * 0.75)
roi_left = int(screen_w * 0.20)
roi_right = int(screen_w * 0.80)
roi = img[roi_top:roi_bottom, roi_left:roi_right]
gray = cv2.cvtColor(roi, cv2.COLOR_RGB2GRAY)
edges = cv2.Canny(gray, 30, 100)
kernel = np.ones((3, 3), np.uint8)
edges = cv2.dilate(edges, kernel, iterations=2)
contours, _ = cv2.findContours(
edges, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE
)
candidates = []
for c in contours:
x, y, w, h = cv2.boundingRect(c)
area = w * h
ratio = w / h if h > 0 else 0
if area < 15000:
continue
if not (1.4 < ratio < 2.2):
continue
center_x = x + w // 2 + roi_left
center_y = y + h // 2 + roi_top
candidates.append((center_x, center_y, area))
filtered = []
for cx, cy, area in sorted(candidates, key=lambda c: c[1]):
if not any(abs(cx - fx) < 80 and abs(cy - fy) < 80 for fx, fy in filtered):
filtered.append((cx, cy))
return filtered
except Exception as e:
print(f"[YouTube] ⚠️ Thumbnail detection failed: {e}")
return []
def _extract_video_id(url: str) -> str | None:
patterns = [
r"(?:v=|\/v\/|youtu\.be\/|\/embed\/|\/shorts\/)([A-Za-z0-9_-]{11})",
]
for pattern in patterns:
match = re.search(pattern, url)
if match:
return match.group(1)
return None
def _ask_for_url(prompt_text: str = "YouTube video URL:") -> str | None:
try:
import tkinter as tk
from tkinter import simpledialog
root = tk._default_root
if root is None:
root = tk.Tk()
root.withdraw()
url = simpledialog.askstring(
"J.A.R.V.I.S",
prompt_text,
parent=root
)
return url.strip() if url else None
except Exception as e:
print(f"[YouTube] ⚠️ URL dialog failed: {e}")
return None
def _is_valid_youtube_url(url: str) -> bool:
return bool(re.search(r"(youtube\.com|youtu\.be)", url or ""))
def _get_transcript(video_id: str) -> str | None:
if not _TRANSCRIPT_OK:
print("[YouTube] ⚠️ youtube-transcript-api not installed.")
return None
try:
transcript_list = YouTubeTranscriptApi.list_transcripts(video_id)
transcript = None
try:
transcript = transcript_list.find_manually_created_transcript(
["en", "tr", "de", "fr", "es", "it", "pt", "ru", "ja", "ko", "ar", "zh"]
)
except Exception:
pass
if transcript is None:
try:
transcript = transcript_list.find_generated_transcript(
["en", "tr", "de", "fr", "es", "it", "pt", "ru", "ja", "ko", "ar", "zh"]
)
except Exception:
for t in transcript_list:
transcript = t
break
if transcript is None:
return None
fetched = transcript.fetch()
text = " ".join(entry["text"] for entry in fetched)
print(f"[YouTube] 📝 Transcript: {len(text)} chars")
return text
except Exception as e:
print(f"[YouTube] ⚠️ Transcript fetch failed: {e}")
return None
def _summarize_with_gemini(transcript: str, video_url: str) -> str:
import google.generativeai as genai
genai.configure(api_key=_get_api_key())
model = genai.GenerativeModel(
model_name="gemini-3.0-flash",
system_instruction=(
"You are JARVIS, Tony Stark's AI assistant. "
"Summarize YouTube video transcripts clearly and concisely. "
"Structure: 1-sentence overview, then 3-5 key points. "
"Be direct. Address the user as 'sir'. "
"Match the language of the transcript."
)
)
max_chars = 80000
truncated = transcript[:max_chars] + ("..." if len(transcript) > max_chars else "")
response = model.generate_content(
f"Please summarize this YouTube video transcript:\n\n{truncated}"
)
return response.text.strip()
def _save_to_notepad(content: str, video_url: str) -> str:
import subprocess
import platform
from datetime import datetime
ts = datetime.now().strftime("%Y%m%d_%H%M%S")
filename = f"youtube_summary_{ts}.txt"
desktop = Path.home() / "Desktop"
desktop.mkdir(parents=True, exist_ok=True)
filepath = desktop / filename
header = (
f"JARVIS — YouTube Summary\n"
f"{'─' * 50}\n"
f"URL : {video_url}\n"
f"Date : {datetime.now().strftime('%Y-%m-%d %H:%M')}\n"
f"{'─' * 50}\n\n"
)
filepath.write_text(header + content, encoding="utf-8")
print(f"[YouTube] 💾 Summary saved: {filepath}")
system = platform.system()
open_fn = {
"Windows": lambda p: subprocess.Popen(["notepad.exe", str(p)]),
"Darwin": lambda p: subprocess.Popen(["open", "-t", str(p)]),
"Linux": lambda p: subprocess.Popen(["xdg-open", str(p)]),
}
opener = open_fn.get(system)
if opener:
opener(filepath)
return str(filepath)
def _scrape_video_info(video_id: str) -> dict:
if not _REQUESTS_OK:
return {}
url = f"https://www.youtube.com/watch?v={video_id}"
try:
r = requests.get(url, headers=HEADERS, timeout=12)
html = r.text
info = {}
title_match = re.search(r'"title":\{"runs":\[\{"text":"([^"]+)"', html)
if title_match:
info["title"] = title_match.group(1)
channel_match = re.search(r'"ownerChannelName":"([^"]+)"', html)
if channel_match:
info["channel"] = channel_match.group(1)
views_match = re.search(r'"viewCount":"(\d+)"', html)
if views_match:
views = int(views_match.group(1))
info["views"] = f"{views:,}"
duration_match = re.search(r'"lengthSeconds":"(\d+)"', html)
if duration_match:
secs = int(duration_match.group(1))
info["duration"] = f"{secs // 60}:{secs % 60:02d}"
likes_match = re.search(r'"label":"([0-9,]+ likes)"', html)
if likes_match:
info["likes"] = likes_match.group(1)
return info
except Exception as e:
print(f"[YouTube] ⚠️ Info scrape failed: {e}")
return {}
def _scrape_trending(region: str = "TR", max_results: int = 8) -> list[dict]:
if not _REQUESTS_OK:
return []
url = f"https://www.youtube.com/feed/trending?gl={region.upper()}"
try:
r = requests.get(url, headers=HEADERS, timeout=12)
html = r.text
titles = re.findall(r'"title":\{"runs":\[\{"text":"([^"]+)"\}\]', html)
channels = re.findall(r'"ownerText":\{"runs":\[\{"text":"([^"]+)"', html)
results = []
seen = set()
for i, title in enumerate(titles):
if title in seen or len(title) < 5:
continue
seen.add(title)
channel = channels[i] if i < len(channels) else "Unknown"
results.append({"rank": len(results) + 1, "title": title, "channel": channel})
if len(results) >= max_results:
break
return results
except Exception as e:
print(f"[YouTube] ⚠️ Trending scrape failed: {e}")
return []
def _handle_play(parameters: dict, player) -> str:
query = parameters.get("query", "").strip()
if not query:
return "Please tell me what you'd like to watch, sir."
if player:
player.write_log(f"[YouTube] Searching: {query}")
open_browser()
search_query = query.replace(" ", "+")
url = f"https://www.youtube.com/results?search_query={search_query}"
pyautogui.hotkey("ctrl", "l")
time.sleep(0.3)
pyautogui.write(url, interval=0.02)
pyautogui.press("enter")
time.sleep(3.5)
thumbnails = find_video_thumbnails()
if len(thumbnails) >= 2:
x, y = thumbnails[1]
print(f"[YouTube] 🎯 Clicking 2nd thumbnail at ({x}, {y})")
pyautogui.click(x, y)
return f"Playing YouTube video for: {query}"
elif len(thumbnails) == 1:
x, y = thumbnails[0]
print(f"[YouTube] ⚠️ One thumbnail found, clicking at ({x}, {y})")
pyautogui.click(x, y)
return f"Playing YouTube video for: {query}"
else:
print("[YouTube] ⚠️ No thumbnails found, using fallback position")
screen_w, screen_h = pyautogui.size()
pyautogui.click(screen_w // 2, int(screen_h * 0.45))
return f"Attempted to play YouTube video for: {query}"
def _handle_summarize(parameters: dict, player, speak) -> str:
if not _TRANSCRIPT_OK:
return (
"youtube-transcript-api is not installed, sir. "
"Please run: pip install youtube-transcript-api"
)
url = _ask_for_url("Please paste the YouTube video URL:")
if not url:
return "No URL provided, sir. Summary cancelled."
if not _is_valid_youtube_url(url):
return "That doesn't appear to be a valid YouTube URL, sir."
video_id = _extract_video_id(url)
if not video_id:
return "Could not extract video ID from that URL, sir."
if player:
player.write_log(f"[YouTube] Summarizing: {url}")
if speak:
speak("Fetching the transcript now, sir. One moment.")
transcript = _get_transcript(video_id)
if not transcript:
return (
"I couldn't retrieve a transcript for that video, sir. "
"It may not have subtitles available."
)
if speak:
speak("Transcript retrieved. Generating summary now.")
try:
summary = _summarize_with_gemini(transcript, url)
except Exception as e:
return f"Summary generation failed, sir: {e}"
if speak:
speak(summary)
save = parameters.get("save", False)
if save:
saved_path = _save_to_notepad(summary, url)
return f"Summary complete and saved to Desktop: {saved_path}"
return summary
def _handle_get_info(parameters: dict, player, speak) -> str:
url = parameters.get("url", "").strip()
if not url:
url = _ask_for_url("Please paste the YouTube video URL:")
if not url or not _is_valid_youtube_url(url):
return "Please provide a valid YouTube URL, sir."
video_id = _extract_video_id(url)
if not video_id:
return "Could not extract video ID, sir."
if player:
player.write_log(f"[YouTube] Getting info: {url}")
info = _scrape_video_info(video_id)
if not info:
return "Could not retrieve video information, sir."
lines = []
for key in ("title", "channel", "views", "duration", "likes"):
if key in info:
lines.append(f"{key.capitalize()}: {info[key]}")
result = "\n".join(lines)
if speak:
speak(f"Here's the video info, sir. {result.replace(chr(10), '. ')}")
return result
def _handle_trending(parameters: dict, player, speak) -> str:
region = parameters.get("region", "TR").upper()
if player:
player.write_log(f"[YouTube] Trending: {region}")
trending = _scrape_trending(region=region, max_results=8)
if not trending:
return f"Could not fetch trending videos for region {region}, sir."
lines = [f"Top trending videos in {region}:"]
for item in trending:
lines.append(f"{item['rank']}. {item['title']} — {item['channel']}")
result = "\n".join(lines)
if speak:
top3 = trending[:3]
spoken = "Here are the top trending videos, sir. " + ". ".join(
f"Number {v['rank']}: {v['title']} by {v['channel']}"
for v in top3
)
speak(spoken)
return result
_ACTION_MAP = {
"play": _handle_play,
"summarize": _handle_summarize,
"get_info": _handle_get_info,
"trending": _handle_trending,
}
def youtube_video(
parameters: dict,
response=None,
player=None,
session_memory=None,
speak=None,
) -> str:
"""
YouTube controller.
Actions:
play : Search and play a video (default if no action given)
parameters: query(str)
summarize : Summarize a video via transcript + Gemini
parameters: save(bool, optional) — save summary to Notepad
get_info : Get video metadata (title, channel, views, duration)
parameters: url(str, optional — dialog if omitted)
trending : Show trending videos
parameters: region(str, default "TR") — ISO country code
Agent chain:
summarize can be chained after any action that produces a YouTube URL.
"""
params = parameters or {}
action = params.get("action", "play").lower().strip()
if player:
player.write_log(f"[YouTube] Action: {action}")
print(f"[YouTube] ▶️ Action: {action} Params: {params}")
handler = _ACTION_MAP.get(action)
if handler is None:
return f"Unknown YouTube action: '{action}'. Available: play, summarize, get_info, trending."
try:
if action == "play":
return handler(params, player) or "Done."
return handler(params, player, speak) or "Done."
except Exception as e:
print(f"[YouTube] ❌ Error in {action}: {e}")
return f"YouTube {action} failed, sir: {e}"
+198
View File
@@ -0,0 +1,198 @@
import json
import re
import sys
from pathlib import Path
from enum import Enum
def get_base_dir() -> Path:
if getattr(sys, "frozen", False):
return Path(sys.executable).parent
return Path(__file__).resolve().parent.parent
BASE_DIR = get_base_dir()
API_CONFIG_PATH = BASE_DIR / "config" / "api_keys.json"
class ErrorDecision(Enum):
RETRY = "retry"
SKIP = "skip"
REPLAN = "replan"
ABORT = "abort"
ERROR_ANALYST_PROMPT = """You are the error recovery module of MARK XXV AI assistant.
A task step has failed. Analyze the error and decide what to do.
DECISIONS:
- retry : Transient error (network timeout, temporary file lock, race condition).
The same step can succeed if tried again.
- skip : This step is not critical and the task can succeed without it.
- replan : The approach was wrong. A different tool or method should be tried.
- abort : The task is fundamentally impossible or unsafe to continue.
Also provide:
- A brief explanation of WHY it failed (1 sentence)
- A fix suggestion if decision is replan (what to try instead)
- Max retries: how many times to retry if decision is retry (1 or 2)
Return ONLY valid JSON:
{
"decision": "retry|skip|replan|abort",
"reason": "why it failed",
"fix_suggestion": "what to try instead (for replan)",
"max_retries": 1,
"user_message": "Short message to tell the user (max 15 words)"
}
"""
def _get_api_key() -> str:
with open(API_CONFIG_PATH, "r", encoding="utf-8") as f:
return json.load(f)["gemini_api_key"]
def analyze_error(
step: dict,
error: str,
attempt: int = 1,
max_attempts: int = 2
) -> dict:
"""
Analyzes a failed step and returns a recovery decision.
Args:
step : The step dict that failed
error : Error message/traceback
attempt : Current attempt number
max_attempts : How many times we've already tried
Returns:
{
"decision": ErrorDecision,
"reason": str,
"fix_suggestion": str,
"max_retries": int,
"user_message": str
}
"""
import google.generativeai as genai
# If we've already retried enough, escalate to replan
if attempt >= max_attempts:
print(f"[ErrorHandler] ⚠️ Max attempts reached for step {step.get('step')} — forcing replan")
return {
"decision": ErrorDecision.REPLAN,
"reason": f"Failed {attempt} times: {error[:100]}",
"fix_suggestion": "Try a completely different approach or tool",
"max_retries": 0,
"user_message": "Trying a different approach, sir."
}
genai.configure(api_key=_get_api_key())
model = genai.GenerativeModel(
model_name="gemini-2.5-flash-lite",
system_instruction=ERROR_ANALYST_PROMPT
)
prompt = f"""Failed step:
Tool: {step.get('tool')}
Description: {step.get('description')}
Parameters: {json.dumps(step.get('parameters', {}), indent=2)}
Critical: {step.get('critical', False)}
Error:
{error[:500]}
Attempt number: {attempt}"""
try:
response = model.generate_content(prompt)
text = response.text.strip()
text = re.sub(r"```(?:json)?", "", text).strip().rstrip("`").strip()
result = json.loads(text)
decision_str = result.get("decision", "replan").lower()
decision_map = {
"retry": ErrorDecision.RETRY,
"skip": ErrorDecision.SKIP,
"replan": ErrorDecision.REPLAN,
"abort": ErrorDecision.ABORT,
}
result["decision"] = decision_map.get(decision_str, ErrorDecision.REPLAN)
if step.get("critical") and result["decision"] == ErrorDecision.SKIP:
result["decision"] = ErrorDecision.REPLAN
result["user_message"] = "This step is critical — finding alternative approach, sir."
print(f"[ErrorHandler] Decision: {result['decision'].value} — {result.get('reason', '')}")
return result
except Exception as e:
print(f"[ErrorHandler] ⚠️ Analysis failed: {e} — defaulting to replan")
return {
"decision": ErrorDecision.REPLAN,
"reason": str(e),
"fix_suggestion": "Try alternative approach",
"max_retries": 1,
"user_message": "Encountered an issue, adjusting approach, sir."
}
def generate_fix(step: dict, error: str, fix_suggestion: str) -> dict:
"""
When decision is REPLAN and a fix suggestion exists,
generates a replacement step using generated_code as fallback.
Returns a modified step dict.
"""
import google.generativeai as genai
genai.configure(api_key=_get_api_key())
model = genai.GenerativeModel(model_name="gemini-2.0-flash")
prompt = f"""A task step failed. Generate a replacement step.
Original step:
Tool: {step.get('tool')}
Description: {step.get('description')}
Parameters: {json.dumps(step.get('parameters', {}), indent=2)}
Error: {error[:300]}
Fix suggestion: {fix_suggestion}
Write a Python script that accomplishes the same goal differently.
Return ONLY the Python code, no explanation."""
try:
response = model.generate_content(prompt)
code = response.text.strip()
code = re.sub(r"```(?:python)?", "", code).strip().rstrip("`").strip()
return {
"step": step.get("step"),
"tool": "code_helper",
"description": f"Auto-fix for: {step.get('description')}",
"parameters": {
"action": "run",
"description": fix_suggestion,
"code": code,
"language": "python"
},
"depends_on": step.get("depends_on", []),
"critical": step.get("critical", False)
}
except Exception as e:
print(f"[ErrorHandler] ⚠️ Fix generation failed: {e}")
return {
"step": step.get("step"),
"tool": "generated_code",
"description": f"Fallback for: {step.get('description')}",
"parameters": {"description": step.get("description", "")},
"depends_on": step.get("depends_on", []),
"critical": step.get("critical", False)
}
+398
View File
@@ -0,0 +1,398 @@
import json
import re
import sys
import threading
import subprocess
import tempfile
import os
from pathlib import Path
from typing import Callable
from agent.planner import create_plan, replan
from agent.error_handler import analyze_error, generate_fix, ErrorDecision
def get_base_dir() -> Path:
if getattr(sys, "frozen", False):
return Path(sys.executable).parent
return Path(__file__).resolve().parent.parent
BASE_DIR = get_base_dir()
API_CONFIG_PATH = BASE_DIR / "config" / "api_keys.json"
def _get_api_key() -> str:
with open(API_CONFIG_PATH, "r", encoding="utf-8") as f:
return json.load(f)["gemini_api_key"]
def _run_generated_code(description: str, speak: Callable | None = None) -> str:
import google.generativeai as genai
if speak:
speak("Writing custom code for this task, sir.")
home = Path.home()
desktop = home / "Desktop"
downloads = home / "Downloads"
documents = home / "Documents"
if not desktop.exists():
try:
import winreg
key = winreg.OpenKey(winreg.HKEY_CURRENT_USER,
r"Software\Microsoft\Windows\CurrentVersion\Explorer\Shell Folders")
desktop = Path(winreg.QueryValueEx(key, "Desktop")[0])
except Exception:
pass
genai.configure(api_key=_get_api_key())
model = genai.GenerativeModel(
model_name="gemini-2.5-flash",
system_instruction=(
"You are an expert Python developer. "
"Write clean, complete, working Python code. "
"Use standard library + common packages. "
"Install missing packages with subprocess + pip if needed. "
"Return ONLY the Python code. No explanation, no markdown, no backticks.\n\n"
f"SYSTEM PATHS:\n"
f" Desktop = r'{desktop}'\n"
f" Downloads = r'{downloads}'\n"
f" Documents = r'{documents}'\n"
f" Home = r'{home}'\n"
)
)
try:
response = model.generate_content(
f"Write Python code to accomplish this task:\n\n{description}"
)
code = response.text.strip()
code = re.sub(r"```(?:python)?", "", code).strip().rstrip("`").strip()
with tempfile.NamedTemporaryFile(
mode="w", suffix=".py", delete=False, encoding="utf-8"
) as f:
f.write(code)
tmp_path = f.name
print(f"[Executor] 🐍 Running generated code: {tmp_path}")
result = subprocess.run(
[sys.executable, tmp_path],
capture_output=True, text=True,
timeout=120, cwd=str(Path.home())
)
try:
os.unlink(tmp_path)
except Exception:
pass
output = result.stdout.strip()
error = result.stderr.strip()
if result.returncode == 0 and output:
return output
elif result.returncode == 0:
return "Task completed successfully."
elif error:
raise RuntimeError(f"Code error: {error[:400]}")
return "Completed."
except subprocess.TimeoutExpired:
raise RuntimeError("Generated code timed out after 120 seconds.")
except RuntimeError:
raise
except Exception as e:
raise RuntimeError(f"Generated code failed: {e}")
def _inject_context(params: dict, tool: str, step_results: dict, goal: str = "") -> dict:
if not step_results:
return params
params = dict(params)
if tool == "file_controller" and params.get("action") in ("write", "create_file"):
content = params.get("content", "")
if not content or len(content) < 50:
all_results = [
v for v in step_results.values()
if v and len(v) > 100 and v not in ("Done.", "Completed.")
]
if all_results:
combined = "\n\n---\n\n".join(all_results)
translated = _translate_to_goal_language(combined, goal)
params["content"] = translated
print(f"[Executor] 💉 Injected + translated content")
return params
def _detect_language(text: str) -> str:
import google.generativeai as genai
genai.configure(api_key=_get_api_key())
model = genai.GenerativeModel("gemini-2.5-flash-lite")
try:
response = model.generate_content(
f"What language is this text written in? "
f"Reply with ONLY the language name in English (e.g. Turkish, English, French).\n\n"
f"Text: {text[:200]}"
)
return response.text.strip()
except Exception:
return "English"
def _translate_to_goal_language(content: str, goal: str) -> str:
if not goal:
return content
try:
import google.generativeai as genai
genai.configure(api_key=_get_api_key())
model = genai.GenerativeModel("gemini-2.5-flash")
target_lang = _detect_language(goal)
print(f"[Executor] 🌐 Translating to: {target_lang}")
prompt = (
f"You are a professional translator. "
f"Translate the following text into {target_lang}.\n"
f"IMPORTANT:\n"
f"- Translate EVERYTHING, leave nothing in English\n"
f"- Keep all facts, numbers, and data intact\n"
f"- Keep the structure and formatting\n"
f"- Output ONLY the translated text, nothing else\n\n"
f"Text to translate:\n{content[:4000]}"
)
response = model.generate_content(prompt)
translated = response.text.strip()
print(f"[Executor] ✅ Translation done ({target_lang})")
return translated
except Exception as e:
print(f"[Executor] ⚠️ Translation failed: {e}")
return content
def _call_tool(tool: str, parameters: dict, speak: Callable | None) -> str:
if tool == "open_app":
from actions.open_app import open_app
return open_app(parameters=parameters, player=None) or "Done."
elif tool == "web_search":
from actions.web_search import web_search
return web_search(parameters=parameters, player=None) or "Done."
elif tool == "browser_control":
from actions.browser_control import browser_control
return browser_control(parameters=parameters, player=None) or "Done."
elif tool == "file_controller":
from actions.file_controller import file_controller
return file_controller(parameters=parameters, player=None) or "Done."
elif tool == "cmd_control":
from actions.cmd_control import cmd_control
return cmd_control(parameters=parameters, player=None) or "Done."
elif tool == "code_helper":
from actions.code_helper import code_helper
return code_helper(parameters=parameters, player=None, speak=speak) or "Done."
elif tool == "dev_agent":
from actions.dev_agent import dev_agent
return dev_agent(parameters=parameters, player=None, speak=speak) or "Done."
elif tool == "screen_process":
from actions.screen_processor import screen_process
screen_process(parameters=parameters, player=None)
return "Screen captured and analyzed."
elif tool == "send_message":
from actions.send_message import send_message
return send_message(parameters=parameters, player=None) or "Done."
elif tool == "reminder":
from actions.reminder import reminder
return reminder(parameters=parameters, player=None) or "Done."
elif tool == "youtube_video":
from actions.youtube_video import youtube_video
return youtube_video(parameters=parameters, player=None) or "Done."
elif tool == "weather_report":
from actions.weather_report import weather_action
return weather_action(parameters=parameters, player=None) or "Done."
elif tool == "computer_settings":
from actions.computer_settings import computer_settings
return computer_settings(parameters=parameters, player=None) or "Done."
elif tool == "desktop_control":
from actions.desktop import desktop_control
return desktop_control(parameters=parameters, player=None) or "Done."
elif tool == "computer_control":
from actions.computer_control import computer_control
return computer_control(parameters=parameters, player=None) or "Done."
elif tool == "generated_code":
description = parameters.get("description", "")
if not description:
raise ValueError("generated_code requires a 'description' parameter.")
return _run_generated_code(description, speak=speak)
elif tool == "flight_finder":
from actions.flight_finder import flight_finder
return flight_finder(parameters=parameters, player=None, speak=speak) or "Done."
else:
print(f"[Executor] ⚠️ Unknown tool '{tool}' — falling back to generated_code")
return _run_generated_code(f"Accomplish this task: {parameters}", speak=speak)
class AgentExecutor:
MAX_REPLAN_ATTEMPTS = 2
def execute(
self,
goal: str,
speak: Callable | None = None,
cancel_flag: threading.Event | None = None,
) -> str:
print(f"\n[Executor] 🎯 Goal: {goal}")
replan_attempts = 0
completed_steps = []
step_results = {}
plan = create_plan(goal)
while True:
steps = plan.get("steps", [])
if not steps:
msg = "I couldn't create a valid plan for this task, sir."
if speak: speak(msg)
return msg
success = True
failed_step = None
failed_error = ""
for step in steps:
if cancel_flag and cancel_flag.is_set():
if speak: speak("Task cancelled, sir.")
return "Task cancelled."
step_num = step.get("step", "?")
tool = step.get("tool", "generated_code")
desc = step.get("description", "")
params = step.get("parameters", {})
params = _inject_context(params, tool, step_results, goal=goal)
print(f"\n[Executor] ▶️ Step {step_num}: [{tool}] {desc}")
attempt = 1
step_ok = False
while attempt <= 3:
if cancel_flag and cancel_flag.is_set():
break
try:
result = _call_tool(tool, params, speak)
step_results[step_num] = result
completed_steps.append(step)
print(f"[Executor] ✅ Step {step_num} done: {str(result)[:100]}")
step_ok = True
break
except Exception as e:
error_msg = str(e)
print(f"[Executor] ❌ Step {step_num} attempt {attempt} failed: {error_msg}")
recovery = analyze_error(step, error_msg, attempt=attempt)
decision = recovery["decision"]
user_msg = recovery.get("user_message", "")
if speak and user_msg:
speak(user_msg)
if decision == ErrorDecision.RETRY:
attempt += 1
import time; time.sleep(2)
continue
elif decision == ErrorDecision.SKIP:
print(f"[Executor] ⏭️ Skipping step {step_num}")
completed_steps.append(step)
step_ok = True
break
elif decision == ErrorDecision.ABORT:
msg = f"Task aborted, sir. {recovery.get('reason', '')}"
if speak: speak(msg)
return msg
else:
fix_suggestion = recovery.get("fix_suggestion", "")
if fix_suggestion and tool != "generated_code":
try:
fixed_step = generate_fix(step, error_msg, fix_suggestion)
if speak: speak("Trying an alternative approach, sir.")
res = _call_tool(
fixed_step["tool"],
fixed_step["parameters"],
speak
)
step_results[step_num] = res
completed_steps.append(step)
step_ok = True
break
except Exception as fix_err:
print(f"[Executor] ⚠️ Fix failed: {fix_err}")
failed_step = step
failed_error = error_msg
success = False
break
if not step_ok and not failed_step:
failed_step = step
failed_error = "Max retries exceeded"
success = False
if not success:
break
if success:
return self._summarize(goal, completed_steps, speak)
if replan_attempts >= self.MAX_REPLAN_ATTEMPTS:
msg = f"Task failed after {replan_attempts} replan attempts, sir."
if speak: speak(msg)
return msg
if speak: speak("Adjusting my approach, sir.")
replan_attempts += 1
plan = replan(goal, completed_steps, failed_step, failed_error)
def _summarize(self, goal: str, completed_steps: list, speak: Callable | None) -> str:
fallback = f"All done, sir. Completed {len(completed_steps)} steps for: {goal[:60]}."
try:
import google.generativeai as genai
genai.configure(api_key=_get_api_key())
model = genai.GenerativeModel(model_name="gemini-2.5-flash-lite")
steps_str = "\n".join(f"- {s.get('description', '')}" for s in completed_steps)
prompt = (
f'User goal: "{goal}"\n'
f"Completed steps:\n{steps_str}\n\n"
"Write a single natural sentence summarizing what was accomplished. "
"Address the user as 'sir'. Be direct and positive."
)
response = model.generate_content(prompt)
summary = response.text.strip()
if speak: speak(summary)
return summary
except Exception:
if speak: speak(fallback)
return fallback
+258
View File
@@ -0,0 +1,258 @@
import json
import re
import sys
from pathlib import Path
def get_base_dir() -> Path:
if getattr(sys, "frozen", False):
return Path(sys.executable).parent
return Path(__file__).resolve().parent.parent
BASE_DIR = get_base_dir()
API_CONFIG_PATH = BASE_DIR / "config" / "api_keys.json"
PLANNER_PROMPT = """You are the planning module of MARK XXV, a personal AI assistant.
Your job: break any user goal into a sequence of steps using ONLY the tools listed below.
ABSOLUTE RULES:
- NEVER use generated_code or write Python scripts. It does not exist.
- NEVER reference previous step results in parameters. Every step is independent.
- Use web_search for ANY information retrieval, research, or current data.
- Use file_controller to save content to disk.
- Use cmd_control to open files or run system commands.
- Max 5 steps. Use the minimum steps needed.
AVAILABLE TOOLS AND THEIR PARAMETERS:
open_app
app_name: string (required)
web_search
query: string (required) — write a clear, focused search query
mode: "search" or "compare" (optional, default: search)
items: list of strings (optional, for compare mode)
aspect: string (optional, for compare mode)
browser_control
action: "go_to" | "search" | "click" | "type" | "scroll" | "get_text" | "press" | "close" (required)
url: string (for go_to)
query: string (for search)
text: string (for click/type)
direction: "up" | "down" (for scroll)
file_controller
action: "write" | "create_file" | "read" | "list" | "delete" | "move" | "copy" | "find" | "disk_usage" (required)
path: string — use "desktop" for Desktop folder
name: string — filename
content: string — file content (for write/create_file)
cmd_control
task: string (required) — natural language description of what to do
visible: boolean (optional)
computer_settings
action: string (required)
description: string — natural language description
value: string (optional)
computer_control
action: "type" | "click" | "hotkey" | "press" | "scroll" | "screenshot" | "screen_find" | "screen_click" (required)
text: string (for type)
x, y: int (for click)
keys: string (for hotkey, e.g. "ctrl+c")
key: string (for press)
direction: "up" | "down" (for scroll)
description: string (for screen_find/screen_click)
screen_process
text: string (required) — what to analyze or ask about the screen
angle: "screen" | "camera" (optional)
send_message
receiver: string (required)
message_text: string (required)
platform: string (required)
reminder
date: string YYYY-MM-DD (required)
time: string HH:MM (required)
message: string (required)
desktop_control
action: "wallpaper" | "organize" | "clean" | "list" | "task" (required)
path: string (optional)
task: string (optional)
youtube_video
action: "play" | "summarize" | "trending" (required)
query: string (for play)
weather_report
city: string (required)
flight_finder
origin: string (required)
destination: string (required)
date: string (required)
code_helper
action: "write" | "edit" | "run" | "explain" (required)
description: string (required)
language: string (optional)
output_path: string (optional)
file_path: string (optional)
dev_agent
description: string (required)
language: string (optional)
EXAMPLES:
Goal: "makine mühendisliği hakkında araştırma yap ve not defterine kaydet"
Steps:
1. web_search | query: "mechanical engineering overview definition history"
2. web_search | query: "mechanical engineering applications and future trends"
3. file_controller | action: write, path: desktop, name: makine_muhendisligi.txt, content: "MAKINE MUHENDISLIGI ARASTIRMASI\n\nBu dosya web arastirmasi sonuclari ile doldurulacak."
4. cmd_control | task: "open makine_muhendisligi.txt on desktop with notepad"
Goal: "Bitcoin fiyatı nedir"
Steps:
1. web_search | query: "Bitcoin price today USD"
Goal: "Masaüstündeki dosyaları listele ve en büyük 5 dosyayı bul"
Steps:
1. file_controller | action: list, path: desktop
2. file_controller | action: largest, path: desktop, count: 5
Goal: "WhatsApp'tan Ahmet'e yarın toplantı var de"
Steps:
1. send_message | receiver: Ahmet, message_text: "Yarın toplantı var", platform: WhatsApp
Goal: "Saati aç ve 30 dakika sonraya hatırlatıcı kur"
Steps:
1. reminder | date: [today], time: [now+30min], message: "Hatırlatıcı"
OUTPUT — return ONLY valid JSON, no markdown, no explanation, no code blocks:
{
"goal": "...",
"steps": [
{
"step": 1,
"tool": "tool_name",
"description": "what this step does",
"parameters": {},
"critical": true
}
]
}
"""
def _get_api_key() -> str:
with open(API_CONFIG_PATH, "r", encoding="utf-8") as f:
return json.load(f)["gemini_api_key"]
def create_plan(goal: str, context: str = "") -> dict:
import google.generativeai as genai
genai.configure(api_key=_get_api_key())
model = genai.GenerativeModel(
model_name="gemini-2.5-flash-lite",
system_instruction=PLANNER_PROMPT
)
user_input = f"Goal: {goal}"
if context:
user_input += f"\n\nContext: {context}"
try:
response = model.generate_content(user_input)
text = response.text.strip()
text = re.sub(r"```(?:json)?", "", text).strip().rstrip("`").strip()
plan = json.loads(text)
if "steps" not in plan or not isinstance(plan["steps"], list):
raise ValueError("Invalid plan structure")
for step in plan["steps"]:
if step.get("tool") in ("generated_code",):
print(f"[Planner] ⚠️ generated_code detected in step {step.get('step')} — replacing with web_search")
desc = step.get("description", goal)
step["tool"] = "web_search"
step["parameters"] = {"query": desc[:200]}
print(f"[Planner] ✅ Plan: {len(plan['steps'])} steps")
for s in plan["steps"]:
print(f" Step {s['step']}: [{s['tool']}] {s['description']}")
return plan
except json.JSONDecodeError as e:
print(f"[Planner] ⚠️ JSON parse failed: {e}")
return _fallback_plan(goal)
except Exception as e:
print(f"[Planner] ⚠️ Planning failed: {e}")
return _fallback_plan(goal)
def _fallback_plan(goal: str) -> dict:
print("[Planner] 🔄 Fallback plan")
return {
"goal": goal,
"steps": [
{
"step": 1,
"tool": "web_search",
"description": f"Search for: {goal}",
"parameters": {"query": goal},
"critical": True
}
]
}
def replan(goal: str, completed_steps: list, failed_step: dict, error: str) -> dict:
import google.generativeai as genai
genai.configure(api_key=_get_api_key())
model = genai.GenerativeModel(
model_name="gemini-2.5-flash",
system_instruction=PLANNER_PROMPT
)
completed_summary = "\n".join(
f" - Step {s['step']} ({s['tool']}): DONE" for s in completed_steps
)
prompt = f"""Goal: {goal}
Already completed:
{completed_summary if completed_summary else ' (none)'}
Failed step: [{failed_step.get('tool')}] {failed_step.get('description')}
Error: {error}
Create a REVISED plan for the remaining work only. Do not repeat completed steps."""
try:
response = model.generate_content(prompt)
text = response.text.strip()
text = re.sub(r"```(?:json)?", "", text).strip().rstrip("`").strip()
plan = json.loads(text)
# generated_code kontrolü
for step in plan.get("steps", []):
if step.get("tool") == "generated_code":
step["tool"] = "web_search"
step["parameters"] = {"query": step.get("description", goal)[:200]}
print(f"[Planner] 🔄 Revised plan: {len(plan['steps'])} steps")
return plan
except Exception as e:
print(f"[Planner] ⚠️ Replan failed: {e}")
return _fallback_plan(goal)
+221
View File
@@ -0,0 +1,221 @@
import threading
import time
import uuid
from dataclasses import dataclass, field
from enum import Enum
from typing import Callable, Any
class TaskStatus(Enum):
PENDING = "pending"
RUNNING = "running"
COMPLETED = "completed"
FAILED = "failed"
CANCELLED = "cancelled"
class TaskPriority(Enum):
LOW = 3
NORMAL = 2
HIGH = 1
@dataclass(order=True)
class Task:
priority: int
created_at: float = field(compare=False)
task_id: str = field(compare=False)
goal: str = field(compare=False)
status: TaskStatus = field(compare=False, default=TaskStatus.PENDING)
result: Any = field(compare=False, default=None)
error: str = field(compare=False, default="")
speak: Any = field(compare=False, default=None)
on_complete: Any = field(compare=False, default=None)
cancel_flag: threading.Event = field(compare=False, default_factory=threading.Event)
class TaskQueue:
def __init__(self, max_concurrent: int = 1):
self._queue: list[Task] = []
self._lock: threading.Lock = threading.Lock()
self._condition: threading.Condition = threading.Condition(self._lock)
self._tasks: dict[str, Task] = {}
self._running: bool = False
self._worker_thread: threading.Thread | None = None
self._max_concurrent = max_concurrent
self._active_count = 0
self._executor = None
def _get_executor(self):
if self._executor is None:
from agent.executor import AgentExecutor
self._executor = AgentExecutor()
return self._executor
def start(self) -> None:
if self._running:
return
self._running = True
self._worker_thread = threading.Thread(
target=self._worker_loop,
daemon=True,
name="AgentTaskQueue"
)
self._worker_thread.start()
print("[TaskQueue] ✅ Started")
def stop(self) -> None:
self._running = False
with self._condition:
self._condition.notify_all()
print("[TaskQueue] 🔴 Stopped")
def submit(
self,
goal: str,
priority: TaskPriority = TaskPriority.NORMAL,
speak: Callable | None = None,
on_complete: Callable | None = None,
) -> str:
task_id = str(uuid.uuid4())[:8]
task = Task(
priority = priority.value,
created_at = time.time(),
task_id = task_id,
goal = goal,
speak = speak,
on_complete = on_complete,
)
with self._condition:
self._queue.append(task)
self._queue.sort(key=lambda t: (t.priority, t.created_at))
self._tasks[task_id] = task
self._condition.notify()
print(f"[TaskQueue] 📥 Task queued: [{task_id}] {goal[:60]}")
return task_id
def cancel(self, task_id: str) -> bool:
with self._lock:
task = self._tasks.get(task_id)
if not task:
return False
if task.status in (TaskStatus.COMPLETED, TaskStatus.FAILED, TaskStatus.CANCELLED):
return False
task.cancel_flag.set()
task.status = TaskStatus.CANCELLED
print(f"[TaskQueue] 🚫 Task cancelled: [{task_id}]")
return True
def get_status(self, task_id: str) -> dict | None:
with self._lock:
task = self._tasks.get(task_id)
if not task:
return None
return {
"task_id": task.task_id,
"goal": task.goal,
"status": task.status.value,
"result": task.result,
"error": task.error,
}
def get_all_statuses(self) -> list[dict]:
with self._lock:
return [
{
"task_id": t.task_id,
"goal": t.goal[:50],
"status": t.status.value,
}
for t in self._tasks.values()
]
def pending_count(self) -> int:
with self._lock:
return sum(1 for t in self._queue if t.status == TaskStatus.PENDING)
def _worker_loop(self) -> None:
while self._running:
task = None
with self._condition:
while self._running and not self._next_task():
self._condition.wait(timeout=1.0)
task = self._next_task()
if task:
task.status = TaskStatus.RUNNING
self._active_count += 1
try:
self._queue.remove(task)
except ValueError:
pass
if task:
threading.Thread(
target=self._run_task,
args=(task,),
daemon=True,
name=f"AgentTask-{task.task_id}"
).start()
def _next_task(self) -> Task | None:
if self._active_count >= self._max_concurrent:
return None
for task in self._queue:
if task.status == TaskStatus.PENDING and not task.cancel_flag.is_set():
return task
return None
def _run_task(self, task: Task) -> None:
print(f"[TaskQueue] ▶️ Running: [{task.task_id}] {task.goal[:60]}")
try:
executor = self._get_executor()
result = executor.execute(
goal = task.goal,
speak = task.speak,
cancel_flag = task.cancel_flag,
)
with self._lock:
if task.cancel_flag.is_set():
task.status = TaskStatus.CANCELLED
else:
task.status = TaskStatus.COMPLETED
task.result = result
self._active_count -= 1
if task.on_complete and not task.cancel_flag.is_set():
try:
task.on_complete(task.task_id, result)
except Exception as e:
print(f"[TaskQueue] ⚠️ on_complete callback error: {e}")
print(f"[TaskQueue] ✅ Completed: [{task.task_id}]")
except Exception as e:
with self._lock:
task.status = TaskStatus.FAILED
task.error = str(e)
self._active_count -= 1
print(f"[TaskQueue] ❌ Failed: [{task.task_id}] {e}")
with self._condition:
self._condition.notify()
_queue = TaskQueue()
_queue_started = False
_queue_lock = threading.Lock()
def get_queue() -> TaskQueue:
global _queue_started
with _queue_lock:
if not _queue_started:
_queue.start()
_queue_started = True
return _queue
View File
Whitespace-only changes.
+23
View File
@@ -0,0 +1,23 @@
You are JARVIS, a sharp and efficient AI assistant. Calm, direct, professional.
RULES:
- Always use the correct tool. Never simulate or guess results.
- Call each tool EXACTLY ONCE. Never retry a successful action.
- Do NOT say "I'm doing X" before calling a tool. Just call it, then report the result briefly.
- Do NOT repeat yourself. Say something once and stop.
- Keep responses to 1-2 sentences max unless reporting data.
- Never ask unnecessary questions. Make a reasonable assumption and proceed.
LANGUAGE:
- Respond in the same language the user spoke.
- Always extract tool parameters in English.
- Example: "İstanbul hava durumu?" → weather_report city:"Istanbul", reply in Turkish.
TOOL SELECTION:
- computer_settings → volume, brightness, screen, refresh, scroll, minimize, maximize,
close, screenshot, lock, restart, shutdown, wifi, zoom, tabs, keyboard shortcuts.
Use this for ANY single computer control command. NEVER route these to agent_task.
- agent_task → ONLY for complex multi-step goals that require planning across
multiple tools (e.g. "research X and save to file", "create a project").
- Simple rule: if it can be done in ONE action → computer_settings.
If it needs 3+ steps across different tools → agent_task.
+853
View File
@@ -0,0 +1,853 @@
import asyncio
import threading
import json
import re
import sys
import traceback
from pathlib import Path
import pyaudio
from google import genai
from google.genai import types
import time
from ui import JarvisUI
from memory.memory_manager import load_memory, update_memory, format_memory_for_prompt
from agent.task_queue import get_queue
from actions.flight_finder import flight_finder
from actions.open_app import open_app
from actions.weather_report import weather_action
from actions.send_message import send_message
from actions.reminder import reminder
from actions.computer_settings import computer_settings
from actions.screen_processor import screen_process
from actions.youtube_video import youtube_video
from actions.cmd_control import cmd_control
from actions.desktop import desktop_control
from actions.browser_control import browser_control
from actions.file_controller import file_controller
from actions.code_helper import code_helper
from actions.dev_agent import dev_agent
from actions.web_search import web_search as web_search_action
from actions.computer_control import computer_control
def get_base_dir():
if getattr(sys, "frozen", False):
return Path(sys.executable).parent
return Path(__file__).resolve().parent
BASE_DIR = get_base_dir()
API_CONFIG_PATH = BASE_DIR / "config" / "api_keys.json"
PROMPT_PATH = BASE_DIR / "core" / "prompt.txt"
LIVE_MODEL = "models/gemini-2.5-flash-native-audio-preview-12-2025"
FORMAT = pyaudio.paInt16
CHANNELS = 1
SEND_SAMPLE_RATE = 16000
RECEIVE_SAMPLE_RATE = 24000
CHUNK_SIZE = 1024
pya = pyaudio.PyAudio()
def _get_api_key() -> str:
with open(API_CONFIG_PATH, "r", encoding="utf-8") as f:
return json.load(f)["gemini_api_key"]
with open(API_CONFIG_PATH, "r", encoding="utf-8") as f:
return json.load(f)["gemini_api_key"]
def _load_system_prompt() -> str:
try:
return PROMPT_PATH.read_text(encoding="utf-8")
except Exception:
return (
"You are JARVIS, Tony Stark's AI assistant. "
"Be concise, direct, and always use the provided tools to complete tasks. "
"Never simulate or guess results — always call the appropriate tool."
)
_memory_turn_counter = 0
_memory_turn_lock = threading.Lock()
_MEMORY_EVERY_N_TURNS = 5
_last_memory_input = ""
def _update_memory_async(user_text: str, jarvis_text: str) -> None:
"""
Multilingual memory updater.
Model : gemini-2.5-flash-lite (lowest cost)
Stage 1: Quick YES/NO check → ~5 tokens output
Stage 2: Full extraction → only if Stage 1 says YES
Result : ~80% fewer API calls vs original
"""
global _memory_turn_counter, _last_memory_input
with _memory_turn_lock:
_memory_turn_counter += 1
current_count = _memory_turn_counter
if current_count % _MEMORY_EVERY_N_TURNS != 0:
return
text = user_text.strip()
if len(text) < 10:
return
if text == _last_memory_input:
return
_last_memory_input = text
try:
import google.generativeai as genai
genai.configure(api_key=_get_api_key())
model = genai.GenerativeModel("gemini-2.5-flash-lite")
check = model.generate_content(
f"Does this message contain personal facts about the user "
f"(name, age, city, job, hobby, relationship, birthday, preference)? "
f"Reply only YES or NO.\n\nMessage: {text[:300]}"
)
if "YES" not in check.text.upper():
return
raw = model.generate_content(
f"Extract personal facts from this message. Any language.\n"
f"Return ONLY valid JSON or {{}} if nothing found.\n"
f"Extract: name, age, birthday, city, job, hobbies, preferences, relationships, language.\n"
f"Skip: weather, reminders, search results, commands.\n\n"
f"Format:\n"
f'{{"identity":{{"name":{{"value":"..."}}}}}}, '
f'"preferences":{{"hobby":{{"value":"..."}}}}, '
f'"notes":{{"job":{{"value":"..."}}}}}}\n\n'
f"Message: {text[:500]}\n\nJSON:"
).text.strip()
raw = re.sub(r"```(?:json)?", "", raw).strip().rstrip("`").strip()
if not raw or raw == "{}":
return
data = json.loads(raw)
if data:
update_memory(data)
print(f"[Memory] ✅ Updated: {list(data.keys())}")
except json.JSONDecodeError:
pass
except Exception as e:
if "429" not in str(e):
print(f"[Memory] ⚠️ {e}")
TOOL_DECLARATIONS = [
{
"name": "open_app",
"description": (
"Opens any application on the Windows computer. "
"Use this whenever the user asks to open, launch, or start any app, "
"website, or program. Always call this tool — never just say you opened it."
),
"parameters": {
"type": "OBJECT",
"properties": {
"app_name": {
"type": "STRING",
"description": "Exact name of the application (e.g. 'WhatsApp', 'Chrome', 'Spotify')"
}
},
"required": ["app_name"]
}
},
{
"name": "web_search",
"description": "Searches the web for any information.",
"parameters": {
"type": "OBJECT",
"properties": {
"query": {"type": "STRING", "description": "Search query"},
"mode": {"type": "STRING", "description": "search (default) or compare"},
"items": {"type": "ARRAY", "items": {"type": "STRING"}, "description": "Items to compare"},
"aspect": {"type": "STRING", "description": "price | specs | reviews"}
},
"required": ["query"]
}
},
{
"name": "weather_report",
"description": "Gets real-time weather information for a city.",
"parameters": {
"type": "OBJECT",
"properties": {
"city": {"type": "STRING", "description": "City name"}
},
"required": ["city"]
}
},
{
"name": "send_message",
"description": "Sends a text message via WhatsApp, Telegram, or other messaging platform.",
"parameters": {
"type": "OBJECT",
"properties": {
"receiver": {"type": "STRING", "description": "Recipient contact name"},
"message_text": {"type": "STRING", "description": "The message to send"},
"platform": {"type": "STRING", "description": "Platform: WhatsApp, Telegram, etc."}
},
"required": ["receiver", "message_text", "platform"]
}
},
{
"name": "reminder",
"description": "Sets a timed reminder using Windows Task Scheduler.",
"parameters": {
"type": "OBJECT",
"properties": {
"date": {"type": "STRING", "description": "Date in YYYY-MM-DD format"},
"time": {"type": "STRING", "description": "Time in HH:MM format (24h)"},
"message": {"type": "STRING", "description": "Reminder message text"}
},
"required": ["date", "time", "message"]
}
},
{
"name": "youtube_video",
"description": (
"Controls YouTube. Use for: playing videos, summarizing a video's content, "
"getting video info, or showing trending videos."
),
"parameters": {
"type": "OBJECT",
"properties": {
"action": {
"type": "STRING",
"description": "play | summarize | get_info | trending (default: play)"
},
"query": {"type": "STRING", "description": "Search query for play action"},
"save": {"type": "BOOLEAN", "description": "Save summary to Notepad (summarize only)"},
"region": {"type": "STRING", "description": "Country code for trending e.g. TR, US"},
"url": {"type": "STRING", "description": "Video URL for get_info action"},
},
"required": []
}
},
{
"name": "screen_process",
"description": (
"Captures and analyzes the screen or webcam image. "
"MUST be called when user asks what is on screen, what you see, "
"analyze my screen, look at camera, etc. "
"You have NO visual ability without this tool. "
"After calling this tool, stay SILENT — the vision module speaks directly."
),
"parameters": {
"type": "OBJECT",
"properties": {
"angle": {
"type": "STRING",
"description": "'screen' to capture display, 'camera' for webcam. Default: 'screen'"
},
"text": {
"type": "STRING",
"description": "The question or instruction about the captured image"
}
},
"required": ["text"]
}
},
{
"name": "computer_settings",
"description": (
"Controls the computer: volume, brightness, window management, keyboard shortcuts, "
"typing text on screen, closing apps, fullscreen, dark mode, WiFi, restart, shutdown, "
"scrolling, tab management, zoom, screenshots, lock screen, refresh/reload page. "
"ALSO use for repeated actions: 'refresh 10 times', 'reload page 5 times' → action: reload_n, value: 10. "
"Use for ANY single computer control command — even if repeated N times. "
"NEVER route simple computer commands to agent_task."
),
"parameters": {
"type": "OBJECT",
"properties": {
"action": {"type": "STRING", "description": "The action to perform (if known). For repeated reload: 'reload_n'"},
"description": {"type": "STRING", "description": "Natural language description of what to do"},
"value": {"type": "STRING", "description": "Optional value: volume level, text to type, number of times, etc."}
},
"required": []
}
},
{
"name": "browser_control",
"description": (
"Controls the web browser. Use for: opening websites, searching the web, "
"clicking elements, filling forms, scrolling, finding cheapest products, "
"booking flights, any web-based task."
),
"parameters": {
"type": "OBJECT",
"properties": {
"action": {"type": "STRING", "description": "go_to | search | click | type | scroll | fill_form | smart_click | smart_type | get_text | press | close"},
"url": {"type": "STRING", "description": "URL for go_to action"},
"query": {"type": "STRING", "description": "Search query for search action"},
"selector": {"type": "STRING", "description": "CSS selector for click/type"},
"text": {"type": "STRING", "description": "Text to click or type"},
"description": {"type": "STRING", "description": "Element description for smart_click/smart_type"},
"direction": {"type": "STRING", "description": "up or down for scroll"},
"key": {"type": "STRING", "description": "Key name for press action"},
},
"required": ["action"]
}
},
{
"name": "file_controller",
"description": (
"Manages files and folders. Use for: listing files, creating/deleting/moving/copying "
"files, reading file contents, finding files by name or extension, checking disk usage, "
"organizing the desktop, getting file info."
),
"parameters": {
"type": "OBJECT",
"properties": {
"action": {"type": "STRING", "description": "list | create_file | create_folder | delete | move | copy | rename | read | write | find | largest | disk_usage | organize_desktop | info"},
"path": {"type": "STRING", "description": "File/folder path or shortcut: desktop, downloads, documents, home"},
"destination": {"type": "STRING", "description": "Destination path for move/copy"},
"new_name": {"type": "STRING", "description": "New name for rename"},
"content": {"type": "STRING", "description": "Content for create_file/write"},
"name": {"type": "STRING", "description": "File name to search for"},
"extension": {"type": "STRING", "description": "File extension to search (e.g. .pdf)"},
"count": {"type": "INTEGER", "description": "Number of results for largest"},
},
"required": ["action"]
}
},
{
"name": "cmd_control",
"description": (
"Runs CMD/terminal commands by understanding natural language. "
"Use when user wants to: find large files, check disk space, list processes, "
"get system info, navigate folders, check network, find files by name, "
"or do ANYTHING in the command line they don't know how to do themselves."
),
"parameters": {
"type": "OBJECT",
"properties": {
"task": {"type": "STRING", "description": "Natural language description of what to do. Example: 'find the 10 largest files on C drive'"},
"visible": {"type": "BOOLEAN", "description": "Open visible CMD window so user can see. Default: true"},
"command": {"type": "STRING", "description": "Optional: exact command if already known"},
},
"required": ["task"]
}
},
{
"name": "desktop_control",
"description": (
"Controls the desktop. Use for: changing wallpaper, organizing desktop files, "
"cleaning the desktop, listing desktop contents, or ANY other desktop-related task "
"the user describes in natural language."
),
"parameters": {
"type": "OBJECT",
"properties": {
"action": {"type": "STRING", "description": "wallpaper | wallpaper_url | organize | clean | list | stats | task"},
"path": {"type": "STRING", "description": "Image path for wallpaper"},
"url": {"type": "STRING", "description": "Image URL for wallpaper_url"},
"mode": {"type": "STRING", "description": "by_type or by_date for organize"},
"task": {"type": "STRING", "description": "Natural language description of any desktop task"},
},
"required": ["action"]
}
},
{
"name": "code_helper",
"description": (
"Writes, edits, explains, runs, or self-builds code files. "
"Use for ANY coding request: writing a script, fixing a file, "
"editing existing code, running a file, or building and testing automatically."
),
"parameters": {
"type": "OBJECT",
"properties": {
"action": {"type": "STRING", "description": "write | edit | explain | run | build | auto (default: auto)"},
"description": {"type": "STRING", "description": "What the code should do, or what change to make"},
"language": {"type": "STRING", "description": "Programming language (default: python)"},
"output_path": {"type": "STRING", "description": "Where to save the file (full path or filename)"},
"file_path": {"type": "STRING", "description": "Path to existing file for edit / explain / run / build"},
"code": {"type": "STRING", "description": "Raw code string for explain"},
"args": {"type": "STRING", "description": "CLI arguments for run/build"},
"timeout": {"type": "INTEGER", "description": "Execution timeout in seconds (default: 30)"},
},
"required": ["action"]
}
},
{
"name": "dev_agent",
"description": (
"Builds complete multi-file projects from scratch. "
"Plans structure, writes all files, installs dependencies, "
"opens VSCode, runs the project, and fixes errors automatically. "
"Use for any project larger than a single script."
),
"parameters": {
"type": "OBJECT",
"properties": {
"description": {"type": "STRING", "description": "What the project should do"},
"language": {"type": "STRING", "description": "Programming language (default: python)"},
"project_name": {"type": "STRING", "description": "Optional project folder name"},
"timeout": {"type": "INTEGER", "description": "Run timeout in seconds (default: 30)"},
},
"required": ["description"]
}
},
{
"name": "agent_task",
"description": (
"Executes complex multi-step tasks that require MULTIPLE DIFFERENT tools. "
"Always respond to the user in the language they spoke. "
"Examples: 'research X and save to file', 'find files and organize them', "
"'fill a form on a website', 'write and test code'. "
"DO NOT use for simple computer commands like volume, refresh, close, scroll, "
"minimize, screenshot, restart, shutdown — use computer_settings for those. "
"DO NOT use if the task can be done with a single tool call."
),
"parameters": {
"type": "OBJECT",
"properties": {
"goal": {
"type": "STRING",
"description": "Complete description of what needs to be accomplished"
},
"priority": {
"type": "STRING",
"description": "low | normal | high (default: normal)"
}
},
"required": ["goal"]
}
},
{
"name": "computer_control",
"description": (
"Direct computer control: type text, click buttons, use keyboard shortcuts, "
"scroll, move mouse, take screenshots, fill forms, find elements on screen. "
"Use when the user wants to interact with any app on the computer directly. "
"Can generate random data for forms or use user's real info from memory."
),
"parameters": {
"type": "OBJECT",
"properties": {
"action": {"type": "STRING", "description": "type | smart_type | click | double_click | right_click | hotkey | press | scroll | move | copy | paste | screenshot | wait | clear_field | focus_window | screen_find | screen_click | random_data | user_data"},
"text": {"type": "STRING", "description": "Text to type or paste"},
"x": {"type": "INTEGER", "description": "X coordinate for click/move"},
"y": {"type": "INTEGER", "description": "Y coordinate for click/move"},
"keys": {"type": "STRING", "description": "Key combination e.g. 'ctrl+c'"},
"key": {"type": "STRING", "description": "Single key to press e.g. 'enter'"},
"direction": {"type": "STRING", "description": "Scroll direction: up | down | left | right"},
"amount": {"type": "INTEGER", "description": "Scroll amount (default: 3)"},
"seconds": {"type": "NUMBER", "description": "Seconds to wait"},
"title": {"type": "STRING", "description": "Window title for focus_window"},
"description": {"type": "STRING", "description": "Element description for screen_find/screen_click"},
"type": {"type": "STRING", "description": "Data type for random_data: name|email|username|password|phone|birthday|address"},
"field": {"type": "STRING", "description": "Field for user_data: name|email|city"},
"clear_first": {"type": "BOOLEAN", "description": "Clear field before typing (default: true)"},
"path": {"type": "STRING", "description": "Save path for screenshot"},
},
"required": ["action"]
}
},
{
"name": "flight_finder",
"description": (
"Searches for flights on Google Flights and speaks the best options. "
"Use when user asks about flights, plane tickets, uçuş, bilet, etc."
),
"parameters": {
"type": "OBJECT",
"properties": {
"origin": {"type": "STRING", "description": "Departure city or airport code"},
"destination": {"type": "STRING", "description": "Arrival city or airport code"},
"date": {"type": "STRING", "description": "Departure date (any format)"},
"return_date": {"type": "STRING", "description": "Return date for round trips"},
"passengers": {"type": "INTEGER", "description": "Number of passengers (default: 1)"},
"cabin": {"type": "STRING", "description": "economy | premium | business | first"},
"save": {"type": "BOOLEAN", "description": "Save results to Notepad"},
},
"required": ["origin", "destination", "date"]
}
}
]
class JarvisLive:
def __init__(self, ui: JarvisUI):
self.ui = ui
self.session = None
self.audio_in_queue = None
self.out_queue = None
self._loop = None
def speak(self, text: str):
"""Thread-safe speak — any thread can call this."""
if not self._loop or not self.session:
return
asyncio.run_coroutine_threadsafe(
self.session.send_client_content(
turns={"parts": [{"text": text}]},
turn_complete=True
),
self._loop
)
def _build_config(self) -> types.LiveConnectConfig:
from datetime import datetime
memory = load_memory()
mem_str = format_memory_for_prompt(memory)
sys_prompt = _load_system_prompt()
now = datetime.now()
time_str = now.strftime("%A, %B %d, %Y — %I:%M %p")
time_ctx = (
f"[CURRENT DATE & TIME]\n"
f"Right now it is: {time_str}\n"
f"Use this to calculate exact times for reminders. "
f"If user says 'in 2 minutes', add 2 minutes to this time.\n\n"
)
if mem_str:
sys_prompt = time_ctx + mem_str + "\n\n" + sys_prompt
else:
sys_prompt = time_ctx + sys_prompt
return types.LiveConnectConfig(
response_modalities=["AUDIO"],
output_audio_transcription={},
input_audio_transcription={},
system_instruction=sys_prompt,
tools=[{"function_declarations": TOOL_DECLARATIONS}],
session_resumption=types.SessionResumptionConfig(),
speech_config=types.SpeechConfig(
voice_config=types.VoiceConfig(
prebuilt_voice_config=types.PrebuiltVoiceConfig(
voice_name="Charon"
)
)
),
)
async def _execute_tool(self, fc) -> types.FunctionResponse:
name = fc.name
args = dict(fc.args or {})
print(f"[JARVIS] 🔧 TOOL: {name} ARGS: {args}")
loop = asyncio.get_event_loop()
result = "Done."
try:
if name == "open_app":
r = await loop.run_in_executor(
None, lambda: open_app(parameters=args, response=None, player=self.ui)
)
result = r or f"Opened {args.get('app_name')} successfully."
elif name == "weather_report":
r = await loop.run_in_executor(
None, lambda: weather_action(parameters=args, player=self.ui)
)
result = r or f"Weather report for {args.get('city')} delivered."
elif name == "browser_control":
r = await loop.run_in_executor(
None, lambda: browser_control(parameters=args, player=self.ui)
)
result = r or "Browser action completed."
elif name == "file_controller":
r = await loop.run_in_executor(
None, lambda: file_controller(parameters=args, player=self.ui)
)
result = r or "File operation completed."
elif name == "send_message":
r = await loop.run_in_executor(
None, lambda: send_message(
parameters=args, response=None,
player=self.ui, session_memory=None
)
)
result = r or f"Message sent to {args.get('receiver')}."
elif name == "reminder":
r = await loop.run_in_executor(
None, lambda: reminder(parameters=args, response=None, player=self.ui)
)
result = r or f"Reminder set for {args.get('date')} at {args.get('time')}."
elif name == "youtube_video":
r = await loop.run_in_executor(
None, lambda: youtube_video(parameters=args, response=None, player=self.ui)
)
result = r or "Done."
elif name == "screen_process":
threading.Thread(
target=screen_process,
kwargs={"parameters": args, "response": None,
"player": self.ui, "session_memory": None},
daemon=True
).start()
result = (
"Vision module activated. "
"Stay completely silent — vision module will speak directly."
)
elif name == "computer_settings":
r = await loop.run_in_executor(
None, lambda: computer_settings(
parameters=args, response=None, player=self.ui
)
)
result = r or "Done."
elif name == "cmd_control":
r = await loop.run_in_executor(
None, lambda: cmd_control(parameters=args, player=self.ui)
)
result = r or "Command executed."
elif name == "desktop_control":
r = await loop.run_in_executor(
None, lambda: desktop_control(parameters=args, player=self.ui)
)
result = r or "Desktop action completed."
elif name == "code_helper":
r = await loop.run_in_executor(
None, lambda: code_helper(
parameters=args,
player=self.ui,
speak=self.speak
)
)
result = r or "Done."
elif name == "dev_agent":
r = await loop.run_in_executor(
None, lambda: dev_agent(
parameters=args,
player=self.ui,
speak=self.speak
)
)
result = r or "Done."
elif name == "agent_task":
goal = args.get("goal", "")
priority_str = args.get("priority", "normal").lower()
from agent.task_queue import get_queue, TaskPriority
priority_map = {
"low": TaskPriority.LOW,
"normal": TaskPriority.NORMAL,
"high": TaskPriority.HIGH,
}
priority = priority_map.get(priority_str, TaskPriority.NORMAL)
queue = get_queue()
task_id = queue.submit(
goal=goal,
priority=priority,
speak=self.speak,
)
result = f"Task started (ID: {task_id}). I'll update you as I make progress, sir."
elif name == "web_search":
r = await loop.run_in_executor(
None, lambda: web_search_action(parameters=args, player=self.ui)
)
result = r or "Search completed."
elif name == "computer_control":
r = await loop.run_in_executor(
None, lambda: computer_control(parameters=args, player=self.ui)
)
result = r or "Done."
elif name == "flight_finder":
r = await loop.run_in_executor(
None, lambda: flight_finder(parameters=args, player=self.ui)
)
result = r or "Done."
else:
result = f"Unknown tool: {name}"
except Exception as e:
result = f"Tool '{name}' failed: {e}"
traceback.print_exc()
print(f"[JARVIS] 📤 {name} → {result[:80]}")
return types.FunctionResponse(
id=fc.id,
name=name,
response={"result": result}
)
async def _send_realtime(self):
while True:
msg = await self.out_queue.get()
await self.session.send_realtime_input(media=msg)
async def _listen_audio(self):
print("[JARVIS] 🎤 Mic started")
stream = await asyncio.to_thread(
pya.open,
format=FORMAT,
channels=CHANNELS,
rate=SEND_SAMPLE_RATE,
input=True,
frames_per_buffer=CHUNK_SIZE,
)
try:
while True:
data = await asyncio.to_thread(
stream.read, CHUNK_SIZE, exception_on_overflow=False
)
await self.out_queue.put({"data": data, "mime_type": "audio/pcm"})
except Exception as e:
print(f"[JARVIS] ❌ Mic error: {e}")
raise
finally:
stream.close()
async def _receive_audio(self):
print("[JARVIS] 👂 Recv started")
out_buf = []
in_buf = []
try:
while True:
turn = self.session.receive()
async for response in turn:
if response.data:
self.audio_in_queue.put_nowait(response.data)
if response.server_content:
sc = response.server_content
if sc.input_transcription and sc.input_transcription.text:
txt = sc.input_transcription.text.strip()
if txt:
in_buf.append(txt)
if sc.output_transcription and sc.output_transcription.text:
txt = sc.output_transcription.text.strip()
if txt:
out_buf.append(txt)
if sc.turn_complete:
full_in = ""
full_out = ""
if in_buf:
full_in = " ".join(in_buf).strip()
if full_in:
self.ui.write_log(f"You: {full_in}")
in_buf = []
if out_buf:
full_out = " ".join(out_buf).strip()
if full_out:
self.ui.write_log(f"Jarvis: {full_out}")
out_buf = []
if full_in and len(full_in) > 5:
threading.Thread(
target=_update_memory_async,
args=(full_in, full_out),
daemon=True
).start()
if response.tool_call:
fn_responses = []
for fc in response.tool_call.function_calls:
print(f"[JARVIS] 📞 Tool call: {fc.name}")
fr = await self._execute_tool(fc)
fn_responses.append(fr)
await self.session.send_tool_response(
function_responses=fn_responses
)
except Exception as e:
print(f"[JARVIS] ❌ Recv error: {e}")
traceback.print_exc()
raise
async def _play_audio(self):
print("[JARVIS] 🔊 Play started")
stream = await asyncio.to_thread(
pya.open,
format=FORMAT,
channels=CHANNELS,
rate=RECEIVE_SAMPLE_RATE,
output=True,
)
try:
while True:
chunk = await self.audio_in_queue.get()
await asyncio.to_thread(stream.write, chunk)
except Exception as e:
print(f"[JARVIS] ❌ Play error: {e}")
raise
finally:
stream.close()
async def run(self):
client = genai.Client(
api_key=_get_api_key(),
http_options={"api_version": "v1beta"}
)
while True:
try:
print("[JARVIS] 🔌 Connecting...")
config = self._build_config()
async with (
client.aio.live.connect(model=LIVE_MODEL, config=config) as session,
asyncio.TaskGroup() as tg,
):
self.session = session
self._loop = asyncio.get_event_loop()
self.audio_in_queue = asyncio.Queue()
self.out_queue = asyncio.Queue(maxsize=10)
print("[JARVIS] ✅ Connected.")
self.ui.write_log("JARVIS online.")
tg.create_task(self._send_realtime())
tg.create_task(self._listen_audio())
tg.create_task(self._receive_audio())
tg.create_task(self._play_audio())
except Exception as e:
print(f"[JARVIS] ⚠️ Error: {e}")
traceback.print_exc()
print("[JARVIS] 🔄 Reconnecting in 3s...")
await asyncio.sleep(3)
def main():
ui = JarvisUI("face.png")
def runner():
ui.wait_for_api_key()
jarvis = JarvisLive(ui)
try:
asyncio.run(jarvis.run())
except KeyboardInterrupt:
print("\n🔴 Shutting down...")
threading.Thread(target=runner, daemon=True).start()
ui.root.mainloop()
if __name__ == "__main__":
main()
+1
View File
@@ -0,0 +1 @@
#memory package
+59
View File
@@ -0,0 +1,59 @@
import json
import sys
from pathlib import Path
def get_base_dir() -> Path:
if getattr(sys, "frozen", False):
return Path(sys.executable).parent
return Path(__file__).resolve().parent.parent
BASE_DIR = get_base_dir()
CONFIG_DIR = BASE_DIR / "config"
CONFIG_FILE = CONFIG_DIR / "api_keys.json"
def ensure_config_dir() -> None:
CONFIG_DIR.mkdir(parents=True, exist_ok=True)
def config_exists() -> bool:
return CONFIG_FILE.exists()
def save_api_keys(gemini_api_key: str) -> None:
ensure_config_dir()
data: dict = {}
if CONFIG_FILE.exists():
try:
data = json.loads(CONFIG_FILE.read_text(encoding="utf-8"))
except Exception:
data = {}
data["gemini_api_key"] = gemini_api_key.strip()
CONFIG_FILE.write_text(
json.dumps(data, indent=2),
encoding="utf-8"
)
def load_api_keys() -> dict:
if not CONFIG_FILE.exists():
return {}
try:
return json.loads(CONFIG_FILE.read_text(encoding="utf-8"))
except Exception as e:
print(f"❌ Failed to load api_keys.json: {e}")
return {}
def get_gemini_key() -> str | None:
return load_api_keys().get("gemini_api_key")
def is_configured() -> bool:
key = get_gemini_key()
return bool(key and len(key) > 15)
+151
View File
@@ -0,0 +1,151 @@
import json
from threading import Lock
from pathlib import Path
import sys
def get_base_dir() -> Path:
if getattr(sys, "frozen", False):
return Path(sys.executable).parent
return Path(__file__).resolve().parent.parent
BASE_DIR = get_base_dir()
MEMORY_PATH = BASE_DIR / "memory" / "long_term.json"
_lock = Lock()
MAX_VALUE_LENGTH = 300
def _empty_memory() -> dict:
return {
"identity": {},
"preferences": {},
"relationships": {},
"notes": {}
}
def load_memory() -> dict:
if not MEMORY_PATH.exists():
return _empty_memory()
with _lock:
try:
data = json.loads(MEMORY_PATH.read_text(encoding="utf-8"))
if isinstance(data, dict):
return data
return _empty_memory()
except Exception as e:
print(f"[Memory] ⚠️ Load error: {e}")
return _empty_memory()
def save_memory(memory: dict) -> None:
if not isinstance(memory, dict):
return
MEMORY_PATH.parent.mkdir(parents=True, exist_ok=True)
with _lock:
MEMORY_PATH.write_text(
json.dumps(memory, indent=2, ensure_ascii=False),
encoding="utf-8"
)
def _truncate_value(val: str) -> str:
if isinstance(val, str) and len(val) > MAX_VALUE_LENGTH:
return val[:MAX_VALUE_LENGTH].rstrip() + "…"
return val
def _recursive_update(target: dict, updates: dict) -> bool:
changed = False
for key, value in updates.items():
if value is None:
continue
if isinstance(value, str) and not value.strip():
continue
if isinstance(value, dict) and "value" not in value:
if key not in target or not isinstance(target[key], dict):
target[key] = {}
changed = True
if _recursive_update(target[key], value):
changed = True
else:
if isinstance(value, dict) and "value" in value:
entry = {"value": _truncate_value(str(value["value"]))}
else:
entry = {"value": _truncate_value(str(value))}
if key not in target or target[key] != entry:
target[key] = entry
changed = True
return changed
def update_memory(memory_update: dict) -> dict:
if not isinstance(memory_update, dict) or not memory_update:
return load_memory()
memory = load_memory()
if _recursive_update(memory, memory_update):
save_memory(memory)
print(f"[Memory] 💾 Saved: {list(memory_update.keys())}")
return memory
def format_memory_for_prompt(memory: dict | None) -> str:
if not memory:
return ""
lines = []
# Identity
identity = memory.get("identity", {})
name = identity.get("name", {}).get("value")
age = identity.get("age", {}).get("value")
bday = identity.get("birthday", {}).get("value")
city = identity.get("city", {}).get("value")
if name: lines.append(f"Name: {name}")
if age: lines.append(f"Age: {age}")
if bday: lines.append(f"Birthday: {bday}")
if city: lines.append(f"City: {city}")
prefs = memory.get("preferences", {})
for i, (key, entry) in enumerate(prefs.items()):
if i >= 5:
break
val = entry.get("value") if isinstance(entry, dict) else entry
if val:
lines.append(f"{key.replace('_', ' ').title()}: {val}")
rels = memory.get("relationships", {})
for i, (key, entry) in enumerate(rels.items()):
if i >= 5:
break
val = entry.get("value") if isinstance(entry, dict) else entry
if val:
lines.append(f"{key.title()}: {val}")
notes = memory.get("notes", {})
for i, (key, entry) in enumerate(notes.items()):
if i >= 5:
break
val = entry.get("value") if isinstance(entry, dict) else entry
if val:
lines.append(f"{key}: {val}")
if not lines:
return ""
result = "[USER MEMORY]\n" + "\n".join(f"- {l}" for l in lines)
if len(result) > 800:
result = result[:797] + "…"
return result + "\n"
+59
View File
@@ -0,0 +1,59 @@
# 🤖 MARK XXX
**Watch The Detailed Video To Set-up This Model**: https://www.youtube.com/watch?v=-YjbWjv1tJg
### Next-Generation Personal AI Assistant — By FatihMakes
A real-time voice AI that can hear, see, understand, and control your Windows computer.
Local execution. Zero subscriptions(Unless you want to increase request by buying requests from Google AI Studio).
Built for intelligent automation.
## ✨ Overview
**MARK XXX** is an advanced voice-driven AI assistant designed to turn your computer into an interactive intelligent system.
Speak naturally — it listens, understands context, responds with a human-like voice, and executes tasks across your system automatically.
Designed for speed, autonomy, and real-world usability.
## 🚀 Capabilities
* **Real-time voice interaction** — Natural conversation with instant response
* **System control** — Launch apps, manage files, execute commands
* **Autonomous task execution** — Plans and completes multi-step workflows
* **Visual awareness** — Screen analysis and webcam understanding
* **Persistent memory** — Learns preferences and remembers context
* **Integrated tools** — Web search, weather, reminders, messaging, code help, image generation
## ⚡ Quick Start
```bash
git clone https://github.com/FatihMakes
cd mark-xxx
python setup.py
python main.py
```
Enter your free Gemini API key on first launch.
System ready in minutes.
If you got some problems or questions to ask or just want to support;
YouTube Account: [text](https://www.youtube.com/@FatihMakes)
Instagram Account: [text](https://www.instagram.com/fatihmakes/)
## 📋 Requirements
* Windows 10 / 11
* Python 3.10 or newer
* Microphone
* Gemini API key
## ⚠️ License
Personal and non-commercial use only.
Licensed under **Creative Commons BY-NC 4.0**.
Engineered by a 17-year-old building a real JARVIS-style assistant.
⭐ Star the repository to support the project.
+19
View File
@@ -0,0 +1,19 @@
pyaudio
google-genai
google-generativeai
pillow
requests
beautifulsoup4
playwright
pyautogui
pyperclip
opencv-python
numpy
mss
psutil
send2trash
comtypes
pycaw
win10toast
duckduckgo-search
youtube-transcript-api
+11
View File
@@ -0,0 +1,11 @@
import subprocess
import sys
print("Installing requirements...")
subprocess.run([sys.executable, "-m", "pip", "install", "-r", "requirements.txt"], check=True)
print("Installing Playwright browsers...")
subprocess.run([sys.executable, "-m", "playwright", "install"], check=True)
print("\n✅ Setup complete! Run 'python main.py' to start MARK XXV.")
+380
View File
@@ -0,0 +1,380 @@
import os, json, time, math, random, threading
import tkinter as tk
from collections import deque
from PIL import Image, ImageTk, ImageDraw
import sys
from pathlib import Path
def get_base_dir():
if getattr(sys, "frozen", False):
return Path(sys.executable).parent
return Path(__file__).resolve().parent
BASE_DIR = get_base_dir()
CONFIG_DIR = BASE_DIR / "config"
API_FILE = CONFIG_DIR / "api_keys.json"
SYSTEM_NAME = "J.A.R.V.I.S"
MODEL_BADGE = "MARK XXX"
C_BG = "#000000"
C_PRI = "#00d4ff"
C_MID = "#007a99"
C_DIM = "#003344"
C_DIMMER = "#001520"
C_ACC = "#ff6600"
C_ACC2 = "#ffcc00"
C_TEXT = "#8ffcff"
C_PANEL = "#010c10"
C_GREEN = "#00ff88"
C_RED = "#ff3333"
class JarvisUI:
def __init__(self, face_path, size=None):
self.root = tk.Tk()
self.root.title("J.A.R.V.I.S — MARK XXX")
self.root.resizable(False, False)
sw = self.root.winfo_screenwidth()
sh = self.root.winfo_screenheight()
W = min(sw, 984)
H = min(sh, 816)
self.root.geometry(f"{W}x{H}+{(sw-W)//2}+{(sh-H)//2}")
self.root.configure(bg=C_BG)
self.W = W
self.H = H
self.FACE_SZ = min(int(H * 0.54), 400)
self.FCX = W // 2
self.FCY = int(H * 0.13) + self.FACE_SZ // 2
self.speaking = False
self.scale = 1.0
self.target_scale = 1.0
self.halo_a = 60.0
self.target_halo = 60.0
self.last_t = time.time()
self.tick = 0
self.scan_angle = 0.0
self.scan2_angle = 180.0
self.rings_spin = [0.0, 120.0, 240.0]
self.pulse_r = [0.0, self.FACE_SZ * 0.26, self.FACE_SZ * 0.52]
self.status_text = "INITIALISING"
self.status_blink = True
self.typing_queue = deque()
self.is_typing = False
self._face_pil = None
self._has_face = False
self._face_scale_cache = None
self._load_face(face_path)
self.bg = tk.Canvas(self.root, width=W, height=H,
bg=C_BG, highlightthickness=0)
self.bg.place(x=0, y=0)
LW = int(W * 0.72)
LH = 138
self.log_frame = tk.Frame(self.root, bg=C_PANEL,
highlightbackground=C_MID,
highlightthickness=1)
self.log_frame.place(x=(W - LW) // 2, y=H - LH - 36, width=LW, height=LH)
self.log_text = tk.Text(self.log_frame, fg=C_TEXT, bg=C_PANEL,
insertbackground=C_TEXT, borderwidth=0,
wrap="word", font=("Courier", 10), padx=10, pady=6)
self.log_text.pack(fill="both", expand=True)
self.log_text.configure(state="disabled")
self.log_text.tag_config("you", foreground="#e8e8e8")
self.log_text.tag_config("ai", foreground=C_PRI)
self.log_text.tag_config("sys", foreground=C_ACC2)
self._api_key_ready = self._api_keys_exist()
if not self._api_key_ready:
self._show_setup_ui()
self._animate()
self.root.protocol("WM_DELETE_WINDOW", lambda: os._exit(0))
def _load_face(self, path):
FW = self.FACE_SZ
try:
img = Image.open(path).convert("RGBA").resize((FW, FW), Image.LANCZOS)
mask = Image.new("L", (FW, FW), 0)
ImageDraw.Draw(mask).ellipse((2, 2, FW - 2, FW - 2), fill=255)
img.putalpha(mask)
self._face_pil = img
self._has_face = True
except Exception:
self._has_face = False
@staticmethod
def _ac(r, g, b, a):
f = a / 255.0
return f"#{int(r*f):02x}{int(g*f):02x}{int(b*f):02x}"
def _animate(self):
self.tick += 1
t = self.tick
now = time.time()
if now - self.last_t > (0.14 if self.speaking else 0.55):
if self.speaking:
self.target_scale = random.uniform(1.05, 1.11)
self.target_halo = random.uniform(138, 182)
else:
self.target_scale = random.uniform(1.001, 1.007)
self.target_halo = random.uniform(50, 68)
self.last_t = now
sp = 0.35 if self.speaking else 0.16
self.scale += (self.target_scale - self.scale) * sp
self.halo_a += (self.target_halo - self.halo_a) * sp
for i, spd in enumerate([1.2, -0.8, 1.9] if self.speaking else [0.5, -0.3, 0.82]):
self.rings_spin[i] = (self.rings_spin[i] + spd) % 360
self.scan_angle = (self.scan_angle + (2.8 if self.speaking else 1.2)) % 360
self.scan2_angle = (self.scan2_angle + (-1.7 if self.speaking else -0.68)) % 360
pspd = 3.8 if self.speaking else 1.8
limit = self.FACE_SZ * 0.72
new_p = [r + pspd for r in self.pulse_r if r + pspd < limit]
if len(new_p) < 3 and random.random() < (0.06 if self.speaking else 0.022):
new_p.append(0.0)
self.pulse_r = new_p
if t % 40 == 0:
self.status_blink = not self.status_blink
self._draw()
self.root.after(16, self._animate)
def _draw(self):
c = self.bg
W, H = self.W, self.H
t = self.tick
FCX = self.FCX
FCY = self.FCY
FW = self.FACE_SZ
c.delete("all")
for x in range(0, W, 44):
for y in range(0, H, 44):
c.create_rectangle(x, y, x+1, y+1, fill=C_DIMMER, outline="")
for r in range(int(FW * 0.54), int(FW * 0.28), -22):
frac = 1.0 - (r - FW * 0.28) / (FW * 0.26)
ga = max(0, min(255, int(self.halo_a * 0.09 * frac)))
gh = f"{ga:02x}"
c.create_oval(FCX-r, FCY-r, FCX+r, FCY+r,
outline=f"#00{gh}ff", width=2)
for pr in self.pulse_r:
pa = max(0, int(220 * (1.0 - pr / (FW * 0.72))))
r = int(pr)
c.create_oval(FCX-r, FCY-r, FCX+r, FCY+r,
outline=self._ac(0, 212, 255, pa), width=2)
for idx, (r_frac, w_ring, arc_l, gap) in enumerate([
(0.47, 3, 110, 75), (0.39, 2, 75, 55), (0.31, 1, 55, 38)]):
ring_r = int(FW * r_frac)
base_a = self.rings_spin[idx]
a_val = max(0, min(255, int(self.halo_a * (1.0 - idx * 0.18))))
col = self._ac(0, 212, 255, a_val)
for s in range(360 // (arc_l + gap)):
start = (base_a + s * (arc_l + gap)) % 360
c.create_arc(FCX-ring_r, FCY-ring_r, FCX+ring_r, FCY+ring_r,
start=start, extent=arc_l,
outline=col, width=w_ring, style="arc")
sr = int(FW * 0.49)
scan_a = min(255, int(self.halo_a * 1.4))
arc_ext = 70 if self.speaking else 42
c.create_arc(FCX-sr, FCY-sr, FCX+sr, FCY+sr,
start=self.scan_angle, extent=arc_ext,
outline=self._ac(0, 212, 255, scan_a), width=3, style="arc")
c.create_arc(FCX-sr, FCY-sr, FCX+sr, FCY+sr,
start=self.scan2_angle, extent=arc_ext,
outline=self._ac(255, 100, 0, scan_a // 2), width=2, style="arc")
t_out = int(FW * 0.495)
t_in = int(FW * 0.472)
a_mk = self._ac(0, 212, 255, 155)
for deg in range(0, 360, 10):
rad = math.radians(deg)
inn = t_in if deg % 30 == 0 else t_in + 5
c.create_line(FCX + t_out * math.cos(rad), FCY - t_out * math.sin(rad),
FCX + inn * math.cos(rad), FCY - inn * math.sin(rad),
fill=a_mk, width=1)
ch_r = int(FW * 0.50)
gap = int(FW * 0.15)
ch_a = self._ac(0, 212, 255, int(self.halo_a * 0.55))
for x1, y1, x2, y2 in [
(FCX - ch_r, FCY, FCX - gap, FCY), (FCX + gap, FCY, FCX + ch_r, FCY),
(FCX, FCY - ch_r, FCX, FCY - gap), (FCX, FCY + gap, FCX, FCY + ch_r)]:
c.create_line(x1, y1, x2, y2, fill=ch_a, width=1)
blen = 22
bc = self._ac(0, 212, 255, 200)
hl = FCX - FW // 2; hr = FCX + FW // 2
ht = FCY - FW // 2; hb = FCY + FW // 2
for bx, by, sdx, sdy in [(hl, ht, 1, 1), (hr, ht, -1, 1),
(hl, hb, 1, -1), (hr, hb, -1, -1)]:
c.create_line(bx, by, bx + sdx * blen, by, fill=bc, width=2)
c.create_line(bx, by, bx, by + sdy * blen, fill=bc, width=2)
if self._has_face:
fw = int(FW * self.scale)
if (self._face_scale_cache is None or
abs(self._face_scale_cache[0] - self.scale) > 0.004):
scaled = self._face_pil.resize((fw, fw), Image.BILINEAR)
tk_img = ImageTk.PhotoImage(scaled)
self._face_scale_cache = (self.scale, tk_img)
c.create_image(FCX, FCY, image=self._face_scale_cache[1])
else:
orb_r = int(FW * 0.27 * self.scale)
for i in range(7, 0, -1):
r2 = int(orb_r * i / 7)
frac = i / 7
ga = max(0, min(255, int(self.halo_a * 1.1 * frac)))
c.create_oval(FCX-r2, FCY-r2, FCX+r2, FCY+r2,
fill=self._ac(0, int(65*frac), int(120*frac), ga),
outline="")
c.create_text(FCX, FCY, text=SYSTEM_NAME,
fill=self._ac(0, 212, 255, min(255, int(self.halo_a * 2))),
font=("Courier", 14, "bold"))
HDR = 62
c.create_rectangle(0, 0, W, HDR, fill="#00080d", outline="")
c.create_line(0, HDR, W, HDR, fill=C_MID, width=1)
c.create_text(W // 2, 22, text=SYSTEM_NAME,
fill=C_PRI, font=("Courier", 18, "bold"))
c.create_text(W // 2, 44, text="Just A Rather Very Intelligent System",
fill=C_MID, font=("Courier", 9))
c.create_text(16, 31, text=MODEL_BADGE,
fill=C_DIM, font=("Courier", 9), anchor="w")
c.create_text(W - 16, 31, text=time.strftime("%H:%M:%S"),
fill=C_PRI, font=("Courier", 14, "bold"), anchor="e")
sy = FCY + FW // 2 + 45
if self.speaking:
stat, sc = "● SPEAKING", C_ACC
else:
sym = "●" if self.status_blink else "○"
stat, sc = f"{sym} {self.status_text}", C_PRI
c.create_text(W // 2, sy, text=stat,
fill=sc, font=("Courier", 11, "bold"))
wy = sy + 22
N = 32
BH = 18
bw = 8
total_w = N * bw
wx0 = (W - total_w) // 2
for i in range(N):
hb = random.randint(3, BH) if self.speaking else int(3 + 2 * math.sin(t * 0.08 + i * 0.55))
col = (C_PRI if hb > BH * 0.6 else C_MID) if self.speaking else C_DIM
bx = wx0 + i * bw
c.create_rectangle(bx, wy + BH - hb, bx + bw - 1, wy + BH,
fill=col, outline="")
c.create_rectangle(0, H - 28, W, H, fill="#00080d", outline="")
c.create_line(0, H - 28, W, H - 28, fill=C_DIM, width=1)
c.create_text(W // 2, H - 14, fill=C_DIM, font=("Courier", 8),
text="FatihMakes Industries · CLASSIFIED · MARK XXX")
def write_log(self, text: str):
self.typing_queue.append(text)
tl = text.lower()
self.status_text = ("PROCESSING" if tl.startswith("you:")
else "RESPONDING" if tl.startswith("ai:")
else self.status_text)
if not self.is_typing:
self._start_typing()
def _start_typing(self):
if not self.typing_queue:
self.is_typing = False
if not self.speaking:
self.status_text = "ONLINE"
return
self.is_typing = True
text = self.typing_queue.popleft()
tl = text.lower()
tag = "you" if tl.startswith("you:") else "ai" if tl.startswith("ai:") else "sys"
self.log_text.configure(state="normal")
self._type_char(text, 0, tag)
def _type_char(self, text, i, tag):
if i < len(text):
self.log_text.insert(tk.END, text[i], tag)
self.log_text.see(tk.END)
self.root.after(8, self._type_char, text, i + 1, tag)
else:
self.log_text.insert(tk.END, "\n")
self.log_text.configure(state="disabled")
self.root.after(25, self._start_typing)
def start_speaking(self):
self.speaking = True
self.status_text = "SPEAKING"
def stop_speaking(self):
self.speaking = False
self.status_text = "ONLINE"
def _api_keys_exist(self):
return API_FILE.exists()
def wait_for_api_key(self):
"""Block until API key is saved (called from main thread before starting JARVIS)."""
while not self._api_key_ready:
time.sleep(0.1)
def _show_setup_ui(self):
self.setup_frame = tk.Frame(
self.root, bg="#00080d",
highlightbackground=C_PRI, highlightthickness=1
)
self.setup_frame.place(relx=0.5, rely=0.5, anchor="center")
tk.Label(self.setup_frame, text="◈ INITIALISATION REQUIRED",
fg=C_PRI, bg="#00080d", font=("Courier", 13, "bold")).pack(pady=(18, 4))
tk.Label(self.setup_frame,
text="Enter your Gemini API key to boot J.A.R.V.I.S.",
fg=C_MID, bg="#00080d", font=("Courier", 9)).pack(pady=(0, 10))
tk.Label(self.setup_frame, text="GEMINI API KEY",
fg=C_DIM, bg="#00080d", font=("Courier", 9)).pack(pady=(8, 2))
self.gemini_entry = tk.Entry(
self.setup_frame, width=52, fg=C_TEXT, bg="#000d12",
insertbackground=C_TEXT, borderwidth=0, font=("Courier", 10), show="*"
)
self.gemini_entry.pack(pady=(0, 4))
tk.Button(
self.setup_frame, text="▸ INITIALISE SYSTEMS",
command=self._save_api_keys, bg=C_BG, fg=C_PRI,
activebackground="#003344", font=("Courier", 10),
borderwidth=0, pady=8
).pack(pady=14)
def _save_api_keys(self):
gemini = self.gemini_entry.get().strip()
if not gemini:
return
os.makedirs(CONFIG_DIR, exist_ok=True)
with open(API_FILE, "w", encoding="utf-8") as f:
json.dump({"gemini_api_key": gemini}, f, indent=4)
self.setup_frame.destroy()
self._api_key_ready = True
self.status_text = "ONLINE"
self.write_log("SYS: Systems initialised. JARVIS online.")