Add files via upload

This commit is contained in:
FatihMakes authored and GitHub committed 2026-03-29 12:58:03 +03:00
commit 9311834b75
31 files changed
+10104

No files matched your search

+589
View File
@@ -0,0 +1,589 @@
import asyncio
import threading
import concurrent.futures
import platform
import shutil
import subprocess
from pathlib import Path
from playwright.async_api import async_playwright, TimeoutError as PlaywrightTimeout
def _get_default_browser_id() -> str:
"""Returns raw default browser identifier string for current OS."""
system = platform.system()
try:
if system == "Windows":
import winreg
key = winreg.OpenKey(
winreg.HKEY_CURRENT_USER,
r"Software\Microsoft\Windows\Shell\Associations\UrlAssociations\http\UserChoice"
)
prog_id = winreg.QueryValueEx(key, "ProgId")[0].lower()
winreg.CloseKey(key)
return prog_id
elif system == "Darwin":
result = subprocess.run(
["defaults", "read",
"com.apple.LaunchServices/com.apple.launchservices.secure",
"LSHandlers"],
capture_output=True, text=True, timeout=5
)
return result.stdout.lower()
elif system == "Linux":
result = subprocess.run(
["xdg-settings", "get", "default-web-browser"],
capture_output=True, text=True, timeout=5
)
return result.stdout.lower()
except Exception:
pass
return ""
_BROWSER_BINARIES = {
"Windows": {
"opera": ["opera.exe"],
"brave": ["brave.exe"],
"vivaldi": ["vivaldi.exe"],
"chrome": ["chrome.exe"],
"firefox": ["firefox.exe"],
},
"Darwin": {
"opera": ["opera"],
"brave": ["brave browser", "brave"],
"vivaldi": ["vivaldi"],
"chrome": ["google chrome", "google-chrome"],
"firefox": ["firefox"],
},
"Linux": {
"opera": ["opera", "opera-stable"],
"brave": ["brave-browser", "brave"],
"vivaldi": ["vivaldi-stable", "vivaldi"],
"chrome": ["google-chrome", "google-chrome-stable", "chromium-browser", "chromium"],
"firefox": ["firefox"],
},
}
def _get_opera_executable() -> str | None:
if platform.system() != "Windows":
return None
try:
import winreg
candidate_keys = [
r"SOFTWARE\Microsoft\Windows\CurrentVersion\App Paths\opera.exe",
r"SOFTWARE\Microsoft\Windows\CurrentVersion\App Paths\launcher.exe",
r"SOFTWARE\Clients\StartMenuInternet\OperaStable\shell\open\command",
r"SOFTWARE\Clients\StartMenuInternet\OperaGXStable\shell\open\command",
]
for key_path in candidate_keys:
for hive in [winreg.HKEY_LOCAL_MACHINE, winreg.HKEY_CURRENT_USER]:
try:
key = winreg.OpenKey(hive, key_path)
val = winreg.QueryValue(key, None)
winreg.CloseKey(key)
exe = val.strip().strip('"').split('"')[0].split(" --")[0].strip()
if exe and Path(exe).exists():
print(f"[Browser] 🔍 Opera found via registry: {exe}")
return exe
except Exception:
continue
except Exception:
pass
return None
def _find_browser_executable(prog_id: str) -> tuple:
system = platform.system()
os_bins = _BROWSER_BINARIES.get(system, {})
if any(x in prog_id for x in ["firefox", "mozilla"]):
return "firefox", None, None
if "safari" in prog_id:
return "webkit", None, None
if "edge" in prog_id:
return "chromium", None, "msedge"
if "opera" in prog_id:
exe = _get_opera_executable()
if exe:
return "chromium", exe, None
for binary in os_bins.get("opera", []):
path = shutil.which(binary)
if path:
return "chromium", path, None
browser_patterns = {
"brave": ["brave"],
"vivaldi": ["vivaldi"],
"chrome": ["chrome"],
}
for browser_name, patterns in browser_patterns.items():
if not any(p in prog_id for p in patterns):
continue
binaries = os_bins.get(browser_name, [])
for binary in binaries:
path = shutil.which(binary)
if path:
print(f"[Browser] 🔍 Found {browser_name} at: {path}")
return "chromium", path, None
if "chrome" in prog_id or not prog_id:
return "chromium", None, "chrome"
return "chromium", None, None
class _BrowserThread:
def __init__(self):
self._loop = None
self._thread = None
self._ready = threading.Event()
self._playwright = None
self._browser = None
self._context = None # Normal context
self._incog_context = None # Incognito/private context
self._page = None # Normal page
self._incog_page = None # Incognito page
self._engine_name = "chromium" # Bilgi için sakla
self._exe_path = None
self._channel = None
def start(self):
if self._thread and self._thread.is_alive():
return
self._thread = threading.Thread(
target=self._run_loop, daemon=True, name="BrowserThread"
)
self._thread.start()
self._ready.wait(timeout=15)
def _run_loop(self):
self._loop = asyncio.new_event_loop()
asyncio.set_event_loop(self._loop)
self._loop.run_until_complete(self._init())
self._ready.set()
self._loop.run_forever()
async def _init(self):
self._playwright = await async_playwright().start()
def run(self, coro, timeout: int = 30):
if not self._loop:
raise RuntimeError("BrowserThread not started.")
future = asyncio.run_coroutine_threadsafe(coro, self._loop)
return future.result(timeout=timeout)
# ── Tarayıcı başlatma ────────────────────────────────────────────────────
async def _launch_browser_if_needed(self):
"""Ana tarayıcıyı başlat (henüz başlatılmadıysa)."""
if self._browser and self._browser.is_connected():
return
prog_id = _get_default_browser_id()
self._engine_name, self._exe_path, self._channel = _find_browser_executable(prog_id)
engine = getattr(self._playwright, self._engine_name)
launch_kwargs = {"headless": False}
if self._engine_name == "chromium":
launch_kwargs["args"] = ["--start-maximized"]
if self._exe_path:
launch_kwargs["executable_path"] = self._exe_path
elif self._channel:
launch_kwargs["channel"] = self._channel
try:
self._browser = await engine.launch(**launch_kwargs)
print(
f"[Browser] ✅ Launched ({self._engine_name}"
f"{' / ' + self._channel if self._channel else ''}"
f"{' / ' + self._exe_path if self._exe_path else ''})"
)
except Exception as e:
print(f"[Browser] ⚠️ Launch failed ({e}), falling back to built-in Chromium")
self._browser = await self._playwright.chromium.launch(
headless=False,
args=["--start-maximized"]
)
async def _get_page(self, incognito: bool = False):
"""
Sayfa döndürür.
incognito=True → özel/gizli sekme
incognito=False → normal sekme
"""
await self._launch_browser_if_needed()
if incognito:
return await self._get_incognito_page()
else:
return await self._get_normal_page()
async def _get_normal_page(self):
if self._page is None or self._page.is_closed():
if self._context is None:
self._context = await self._browser.new_context(
viewport=None,
user_agent=(
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
"AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/120.0.0.0 Safari/537.36"
)
)
self._page = await self._context.new_page()
return self._page
async def _get_incognito_page(self):
"""
Gizli/özel sekme açar.
Chromium → new_context(no_viewport, storage_state temiz)
Firefox → new_context(incognito) [Playwright'ta firefox private]
"""
# Mevcut gizli sayfa varsa kapat, yenisini aç
if self._incog_page and not self._incog_page.is_closed():
return self._incog_page
# Önceki gizli context'i kapat
if self._incog_context:
try:
await self._incog_context.close()
except Exception:
pass
print("[Browser] 🕵️ Opening private/incognito context...")
if self._engine_name == "firefox":
# Firefox: firefox_user_prefs ile private browsing
self._incog_context = await self._browser.new_context(
viewport=None,
firefox_user_prefs={
"browser.privatebrowsing.autostart": True
}
)
else:
# Chromium tabanlı: --incognito argümanı ile ayrı tarayıcı aç
# (Playwright'ta context-level incognito için browser.new_context yeterli
# çünkü her context zaten izole storage'a sahip.
# Gerçek incognito görünümü için ayrı browser instance gerekir.)
engine = getattr(self._playwright, self._engine_name)
launch_kwargs = {
"headless": False,
"args": ["--incognito", "--start-maximized"],
}
if self._exe_path:
launch_kwargs["executable_path"] = self._exe_path
elif self._channel:
launch_kwargs["channel"] = self._channel
try:
incog_browser = await engine.launch(**launch_kwargs)
except Exception:
incog_browser = await self._playwright.chromium.launch(
headless=False,
args=["--incognito", "--start-maximized"]
)
self._incog_context = await incog_browser.new_context(
viewport=None,
user_agent=(
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
"AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/120.0.0.0 Safari/537.36"
)
)
self._incog_page = await self._incog_context.new_page()
print("[Browser] ✅ Private/incognito page ready.")
return self._incog_page
# ── Eylemler ─────────────────────────────────────────────────────────────
async def _go_to(self, url: str, incognito: bool = False) -> str:
if not url.startswith("http"):
url = "https://" + url
page = await self._get_page(incognito=incognito)
try:
await page.goto(url, wait_until="domcontentloaded", timeout=15000)
mode = " [private]" if incognito else ""
return f"Opened{mode}: {page.url}"
except PlaywrightTimeout:
return f"Timeout loading: {url}"
except Exception as e:
return f"Navigation error: {e}"
async def _search(self, query: str, engine: str = "google", incognito: bool = False) -> str:
engines = {
"google": f"https://www.google.com/search?q={query.replace(' ', '+')}",
"bing": f"https://www.bing.com/search?q={query.replace(' ', '+')}",
"duckduckgo": f"https://duckduckgo.com/?q={query.replace(' ', '+')}",
}
url = engines.get(engine.lower(), engines["google"])
return await self._go_to(url, incognito=incognito)
async def _click(self, selector=None, text=None, incognito: bool = False) -> str:
page = await self._get_page(incognito=incognito)
try:
if text:
await page.get_by_text(text, exact=False).first.click(timeout=8000)
return f"Clicked: '{text}'"
elif selector:
await page.click(selector, timeout=8000)
return f"Clicked: {selector}"
return "No selector or text provided."
except PlaywrightTimeout:
return "Element not found or not clickable."
except Exception as e:
return f"Click error: {e}"
async def _type(self, selector=None, text: str = "", clear_first: bool = True, incognito: bool = False) -> str:
page = await self._get_page(incognito=incognito)
try:
element = page.locator(selector).first if selector else page.locator(":focus")
if clear_first:
await element.clear()
await element.type(text, delay=50)
return "Text typed."
except Exception as e:
return f"Type error: {e}"
async def _scroll(self, direction: str = "down", amount: int = 500, incognito: bool = False) -> str:
page = await self._get_page(incognito=incognito)
try:
y = amount if direction == "down" else -amount
await page.mouse.wheel(0, y)
return f"Scrolled {direction}."
except Exception as e:
return f"Scroll error: {e}"
async def _press(self, key: str, incognito: bool = False) -> str:
page = await self._get_page(incognito=incognito)
try:
await page.keyboard.press(key)
return f"Pressed: {key}"
except Exception as e:
return f"Key error: {e}"
async def _get_text(self, incognito: bool = False) -> str:
page = await self._get_page(incognito=incognito)
try:
text = await page.inner_text("body")
return text[:4000] if len(text) > 4000 else text
except Exception as e:
return f"Could not get page text: {e}"
async def _fill_form(self, fields: dict, incognito: bool = False) -> str:
page = await self._get_page(incognito=incognito)
results = []
for selector, value in fields.items():
try:
el = page.locator(selector).first
await el.clear()
await el.type(str(value), delay=40)
results.append(f"✓ {selector}")
except Exception as e:
results.append(f"✗ {selector}: {e}")
return "Form filled: " + ", ".join(results)
async def _smart_click(self, description: str, incognito: bool = False) -> str:
page = await self._get_page(incognito=incognito)
desc_lower = description.lower()
role_hints = {
"button": ["button", "buton", "btn"],
"link": ["link", "bağlantı"],
"searchbox": ["search", "arama"],
"textbox": ["input", "field", "alan"],
}
for role, keywords in role_hints.items():
if any(k in desc_lower for k in keywords):
try:
await page.get_by_role(role).first.click(timeout=5000)
return f"Clicked ({role}): '{description}'"
except Exception:
pass
try:
await page.get_by_text(description, exact=False).first.click(timeout=5000)
return f"Clicked (text): '{description}'"
except Exception:
pass
try:
await page.get_by_placeholder(description, exact=False).first.click(timeout=5000)
return f"Clicked (placeholder): '{description}'"
except Exception:
pass
return f"Could not find: '{description}'"
async def _smart_type(self, description: str, text: str, incognito: bool = False) -> str:
page = await self._get_page(incognito=incognito)
for method, locator in [
("placeholder", page.get_by_placeholder(description, exact=False)),
("label", page.get_by_label(description, exact=False)),
("role", page.get_by_role("textbox")),
]:
try:
el = locator.first
await el.clear()
await el.type(text, delay=50)
return f"Typed into ({method}): '{description}'"
except Exception:
continue
return f"Could not find input: '{description}'"
async def _close_browser(self) -> str:
if self._incog_context:
try:
await self._incog_context.close()
except Exception:
pass
self._incog_context = None
self._incog_page = None
if self._browser:
await self._browser.close()
self._browser = None
self._context = None
self._page = None
if self._playwright:
await self._playwright.stop()
self._playwright = None
return "Browser closed."
# ── Singleton browser thread ─────────────────────────────────────────────────
_bt = _BrowserThread()
_bt_started = False
_bt_lock = threading.Lock()
def _ensure_started():
global _bt_started
with _bt_lock:
if not _bt_started:
_bt.start()
_bt_started = True
# ── Public API ───────────────────────────────────────────────────────────────
def browser_control(
parameters: dict,
response=None,
player=None,
session_memory=None
) -> str:
"""
Browser controller — auto-detects and uses system default browser.
parameters:
action : go_to | search | click | type | scroll | fill_form |
smart_click | smart_type | get_text | press | close
url : URL for go_to
query : search query
engine : google | bing | duckduckgo (default: google)
selector : CSS selector for click/type
text : text to click or type
description : element description for smart_click/smart_type
direction : up | down for scroll
amount : scroll amount in pixels (default: 500)
key : key name for press (e.g. Enter, Escape, Tab)
fields : {selector: value} dict for fill_form
clear_first : bool, clear input before typing (default: True)
incognito : bool, open in private/incognito mode (default: False)
"""
_ensure_started()
action = (parameters or {}).get("action", "").lower().strip()
incognito = bool(parameters.get("incognito", False))
result = "Unknown action."
try:
if action == "go_to":
result = _bt.run(_bt._go_to(parameters.get("url", ""), incognito=incognito))
elif action == "search":
result = _bt.run(_bt._search(
parameters.get("query", ""),
parameters.get("engine", "google"),
incognito=incognito
))
elif action == "click":
result = _bt.run(_bt._click(
selector=parameters.get("selector"),
text=parameters.get("text"),
incognito=incognito
))
elif action == "type":
result = _bt.run(_bt._type(
selector=parameters.get("selector"),
text=parameters.get("text", ""),
clear_first=parameters.get("clear_first", True),
incognito=incognito
))
elif action == "scroll":
result = _bt.run(_bt._scroll(
direction=parameters.get("direction", "down"),
amount=parameters.get("amount", 500),
incognito=incognito
))
elif action == "fill_form":
result = _bt.run(_bt._fill_form(
parameters.get("fields", {}),
incognito=incognito
))
elif action == "smart_click":
result = _bt.run(_bt._smart_click(
parameters.get("description", ""),
incognito=incognito
))
elif action == "smart_type":
result = _bt.run(_bt._smart_type(
parameters.get("description", ""),
parameters.get("text", ""),
incognito=incognito
))
elif action == "get_text":
result = _bt.run(_bt._get_text(incognito=incognito))
elif action == "press":
result = _bt.run(_bt._press(
parameters.get("key", "Enter"),
incognito=incognito
))
elif action == "close":
result = _bt.run(_bt._close_browser())
else:
result = f"Unknown action: {action}"
except concurrent.futures.TimeoutError:
result = "Browser action timed out."
except Exception as e:
result = f"Browser error: {e}"
print(f"[Browser] {result[:80]}")
if player:
player.write_log(f"[browser] {result[:60]}")
return result
+230
View File
@@ -0,0 +1,230 @@
import subprocess
import sys
import json
import re
from pathlib import Path
def get_base_dir():
if getattr(sys, "frozen", False):
return Path(sys.executable).parent
return Path(__file__).resolve().parent.parent
BASE_DIR = get_base_dir()
API_CONFIG_PATH = BASE_DIR / "config" / "api_keys.json"
def _get_api_key() -> str:
with open(API_CONFIG_PATH, "r", encoding="utf-8") as f:
return json.load(f)["gemini_api_key"]
def _get_platform() -> str:
if sys.platform == "win32": return "windows"
if sys.platform == "darwin": return "macos"
return "linux"
WIN_COMMAND_MAP = [
(["disk space", "disk usage", "storage", "free space", "c drive space"],
"wmic logicaldisk get caption,freespace,size /format:list", False),
(["running processes", "list processes", "show processes", "active processes", "tasklist"],
"tasklist /fo table", False),
(["ip address", "my ip", "network info", "ipconfig"],
"ipconfig /all", False),
(["ping", "internet connection", "connected to internet"],
"ping -n 4 google.com", False),
(["open ports", "listening ports", "netstat"],
"netstat -an | findstr LISTENING", False),
(["wifi networks", "available wifi", "wireless networks"],
"netsh wlan show networks", False),
(["system info", "computer info", "hardware info", "pc info", "specs"],
"systeminfo", False),
(["cpu usage", "processor usage"],
"wmic cpu get loadpercentage", False),
(["memory usage", "ram usage"],
"wmic OS get FreePhysicalMemory,TotalVisibleMemorySize /Value", False),
(["windows version", "os version"],
"ver", False),
(["installed programs", "installed software", "installed apps"],
"wmic product get name,version /format:table", False),
(["battery", "battery level", "power status"],
"powershell (Get-WmiObject -Class Win32_Battery).EstimatedChargeRemaining", False),
(["current time", "what time", "system time"],
"time /t", False),
(["current date", "what date", "system date"],
"date /t", False),
(["desktop files", "files on desktop"],
f'dir "{Path.home() / "Desktop"}"', False),
(["downloads", "files in downloads"],
f'dir "{Path.home() / "Downloads"}"', False),
(["large files", "biggest files", "largest files"],
'powershell "Get-ChildItem C:\\ -Recurse -ErrorAction SilentlyContinue | Sort-Object Length -Descending | Select-Object -First 10 FullName,Length | Format-Table"', False),
]
def _find_hardcoded(task: str) -> str | None:
task_lower = task.lower()
if "notepad" in task_lower or any(ext in task_lower for ext in [".txt", ".log", ".md", ".csv"]):
file_match = re.search(r'[\"\']?([\S]+\.(?:txt|log|md|csv|json|xml))[\"\']?', task, re.IGNORECASE)
if file_match:
filename = file_match.group(1)
desktop = Path.home() / "Desktop"
filepath = Path(filename) if Path(filename).is_absolute() else desktop / filename
return f'notepad "{filepath}"'
if "notepad" in task_lower:
return "notepad"
pip_match = re.search(r"install\s+([\w\-]+)", task_lower)
if pip_match:
package = pip_match.group(1)
return f"pip install {package}"
for keywords, command, _ in WIN_COMMAND_MAP:
if command and any(kw in task_lower for kw in keywords):
return command
return None
BLOCKED_PATTERNS = [
r"\brm\s+-rf\b", r"\brmdir\s+/s\b", r"\bdel\s+/[fqs]",
r"\bformat\b", r"\bdiskpart\b", r"\bfdisk\b",
r"\breg\s+(delete|add)\b", r"\bbcdedit\b",
r"\bnet\s+localgroup\b",
r"\bshutdown\b", r"\brestart-computer\b",
r"\bstop-process\b", r"\bkill\s+-9\b", r"\btaskkill\b",
r"\beval\b", r"\b__import__\b",
]
_BLOCKED_RE = re.compile("|".join(BLOCKED_PATTERNS), re.IGNORECASE)
def _is_safe(command: str) -> tuple[bool, str]:
match = _BLOCKED_RE.search(command)
if match:
return False, f"Blocked pattern: '{match.group()}'"
return True, "OK"
def _ask_gemini(task: str) -> str:
try:
import google.generativeai as genai
genai.configure(api_key=_get_api_key())
model = genai.GenerativeModel("gemini-2.5-flash-lite")
prompt = (
f"Convert this request to a single Windows CMD command.\n"
f"Output ONLY the command. No explanation, no markdown, no backticks.\n"
f"If unsafe or impossible, output: UNSAFE\n\n"
f"Request: {task}\n\nCommand:"
)
response = model.generate_content(prompt)
command = response.text.strip().strip("`").strip()
if command.startswith("```"):
lines = command.split("\n")
command = "\n".join(lines[1:-1]).strip()
return command
except Exception as e:
return f"ERROR: {e}"
def _run_silent(command: str, timeout: int = 20) -> str:
try:
platform = _get_platform()
if platform == "windows":
is_ps = command.strip().lower().startswith("powershell")
if is_ps:
cmd_inner = re.sub(r'^powershell\s+"?', '', command, flags=re.IGNORECASE).rstrip('"')
result = subprocess.run(
["powershell", "-NoProfile", "-Command", cmd_inner],
capture_output=True, text=True,
encoding="utf-8", errors="replace", timeout=timeout
)
else:
result = subprocess.run(
["cmd", "/c", command],
capture_output=True, text=True,
encoding="cp1252", errors="replace",
timeout=timeout, cwd=str(Path.home())
)
else:
shell = "/bin/zsh" if platform == "macos" else "/bin/bash"
result = subprocess.run(
command, shell=True, executable=shell,
capture_output=True, text=True,
errors="replace", timeout=timeout,
cwd=str(Path.home())
)
output = result.stdout.strip()
error = result.stderr.strip()
if output: return output[:2000]
if error: return f"[stderr]: {error[:500]}"
return "Command executed with no output."
except subprocess.TimeoutExpired:
return f"Command timed out after {timeout}s."
except Exception as e:
return f"Execution error: {e}"
def _run_visible(command: str) -> None:
try:
platform = _get_platform()
if platform == "windows":
subprocess.Popen(
f'cmd /k "{command}"',
creationflags=subprocess.CREATE_NEW_CONSOLE
)
elif platform == "macos":
subprocess.Popen(["osascript", "-e",
f'tell application "Terminal" to do script "{command}"'])
else:
for term in ["gnome-terminal", "xterm", "konsole"]:
try:
subprocess.Popen([term, "--", "bash", "-c", f"{command}; exec bash"])
break
except FileNotFoundError:
continue
except Exception as e:
print(f"[CMD] ⚠️ Terminal open failed: {e}")
def cmd_control(
parameters: dict,
response=None,
player=None,
session_memory=None
) -> str:
task = (parameters or {}).get("task", "").strip()
command = (parameters or {}).get("command", "").strip()
visible = (parameters or {}).get("visible", True)
if not task and not command:
return "Please describe what you want to do, sir."
if not command:
command = _find_hardcoded(task)
if command:
print(f"[CMD] ⚡ Hardcoded: {command[:80]}")
else:
print(f"[CMD] 🤖 Gemini fallback for: {task}")
command = _ask_gemini(task)
print(f"[CMD] ✅ Generated: {command[:80]}")
if command == "UNSAFE":
return "I cannot generate a safe command for that request, sir."
if command.startswith("ERROR:"):
return f"Could not generate command: {command}"
safe, reason = _is_safe(command)
if not safe:
return f"Blocked for safety: {reason}"
if player:
player.write_log(f"[CMD] {command[:60]}")
if any(x in command.lower() for x in ["notepad", "explorer", "start "]):
subprocess.Popen(command, shell=True)
return f"Opened: {command}"
if visible:
_run_visible(command)
output = _run_silent(command)
return f"Terminal opened.\n\nOutput:\n{output}"
else:
return _run_silent(command)
+583
View File
@@ -0,0 +1,583 @@
# actions/code_helper.py
# AI-powered code assistant — writes, edits, explains, runs, builds, debugs, and optimizes code.
#
# Actions:
# write → Describe what you want, Gemini writes it, saves to file
# edit → Read existing file, apply natural language change
# explain → Explain what a piece of code or file does
# run → Execute a script file, return output
# build → Write → Run → Fix loop (max 3 attempts), speaks when done
# screen_debug → Screenshot al, ekrandaki kodu/hatayı Gemini ile analiz et ve düzelt
# optimize → Mevcut kodu Gemini ile optimize et (performans, okunabilirlik, best practices)
# auto → (default) Intent auto-detected from context
import subprocess
import sys
import json
import re
import time
from pathlib import Path
def get_base_dir():
if getattr(sys, "frozen", False):
return Path(sys.executable).parent
return Path(__file__).resolve().parent.parent
BASE_DIR = get_base_dir()
API_CONFIG_PATH = BASE_DIR / "config" / "api_keys.json"
DESKTOP = Path.home() / "Desktop"
MAX_BUILD_ATTEMPTS = 3
GEMINI_MODEL = "gemini-2.5-flash"
def _get_api_key() -> str:
with open(API_CONFIG_PATH, "r", encoding="utf-8") as f:
return json.load(f)["gemini_api_key"]
def _get_gemini(model: str = GEMINI_MODEL):
import google.generativeai as genai
genai.configure(api_key=_get_api_key())
return genai.GenerativeModel(model)
def _clean_code(text: str) -> str:
text = text.strip()
text = re.sub(r"^```[a-zA-Z]*\n?", "", text)
text = re.sub(r"\n?```$", "", text)
return text.strip()
def _resolve_save_path(output_path: str, language: str) -> Path:
ext_map = {
"python": ".py", "py": ".py",
"javascript": ".js", "js": ".js",
"typescript": ".ts", "ts": ".ts",
"html": ".html", "css": ".css",
"java": ".java", "cpp": ".cpp", "c": ".c",
"bash": ".sh", "shell": ".sh", "powershell": ".ps1",
"sql": ".sql", "json": ".json", "rust": ".rs", "go": ".go",
}
if output_path:
p = Path(output_path)
return p if p.is_absolute() else DESKTOP / p
ext = ext_map.get((language or "python").lower(), ".py")
return DESKTOP / f"jarvis_code{ext}"
def _read_file(file_path: str) -> tuple[str, str]:
if not file_path:
return "", "No file path provided."
p = Path(file_path)
if not p.exists():
return "", f"File not found: {file_path}"
try:
return p.read_text(encoding="utf-8"), ""
except Exception as e:
return "", f"Could not read file: {e}"
def _save_file(path: Path, content: str) -> str:
try:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(content, encoding="utf-8")
return f"Saved to: {path}"
except Exception as e:
return f"Could not save: {e}"
def _preview(code: str, lines: int = 10) -> str:
all_lines = code.splitlines()
preview = "\n".join(all_lines[:lines])
suffix = f"\n... ({len(all_lines) - lines} more lines)" if len(all_lines) > lines else ""
return preview + suffix
def _has_error(output: str) -> bool:
error_signals = ["error", "exception", "traceback", "syntaxerror",
"nameerror", "typeerror", "stderr", "failed", "crash"]
return any(s in output.lower() for s in error_signals)
def _take_screenshot() -> Path | None:
try:
import pyautogui
screenshot_path = Path.home() / "Desktop" / f"jarvis_debug_{int(time.time())}.png"
screenshot = pyautogui.screenshot()
screenshot.save(str(screenshot_path))
print(f"[Code] 📸 Screenshot: {screenshot_path}")
return screenshot_path
except Exception as e:
print(f"[Code] ⚠️ Screenshot failed: {e}")
return None
def _image_to_base64(path: Path) -> str:
import base64
return base64.b64encode(path.read_bytes()).decode("utf-8")
def _detect_intent(description: str, file_path: str, code: str) -> str:
desc = (description or "").lower()
screen_kw = ["ekrandaki", "screen", "ekranda", "bu hatayı", "why am i getting",
"neden hata", "what's wrong", "ne yanlış", "screenshot", "görüntü"]
if any(k in desc for k in screen_kw):
return "screen_debug"
# Optimize keywords
optimize_kw = ["optimize", "refactor", "clean up", "improve", "temizle",
"iyileştir", "daha iyi", "make it better", "hızlandır"]
if any(k in desc for k in optimize_kw) and (code or file_path):
return "optimize"
if file_path:
p = Path(file_path)
edit_kw = ["edit", "update", "modify", "change", "add", "remove",
"refactor", "fix", "rename", "replace", "düzenle", "değiştir"]
run_kw = ["run", "execute", "launch", "start", "çalıştır"]
build_kw = ["build", "make it work", "try", "attempt"]
if p.exists() and any(k in desc for k in edit_kw):
return "edit"
if p.exists() and any(k in desc for k in run_kw):
return "run"
if any(k in desc for k in build_kw):
return "build"
if p.exists():
return "explain"
explain_kw = ["explain", "what does", "describe", "analyze", "açıkla", "ne yapıyor"]
if any(k in desc for k in explain_kw) and (code or file_path):
return "explain"
build_kw = ["build", "make it work", "try and", "attempt"]
if any(k in desc for k in build_kw):
return "build"
return "write"
def _write(description: str, language: str, output_path: str, player=None) -> tuple[str, Path]:
lang = language or "python"
model = _get_gemini()
prompt = f"""You are an expert {lang} developer.
Write clean, working, well-commented {lang} code for the description below.
Rules:
- Output ONLY the code. No explanation, no markdown, no backticks.
- Add helpful inline comments.
- Handle errors and edge cases properly.
- Use modern best practices.
Description: {description}
Code:"""
response = model.generate_content(prompt)
code = _clean_code(response.text)
path = _resolve_save_path(output_path, lang)
_save_file(path, code)
return code, path
def _fix_code(code: str, error_output: str, description: str) -> str:
model = _get_gemini()
prompt = f"""You are an expert debugger.
The code below failed with the following error. Fix it.
Return ONLY the corrected code — no explanation, no markdown, no backticks.
Original goal: {description}
Error:
{error_output[:2000]}
Broken code:
{code}
Fixed code:"""
response = model.generate_content(prompt)
return _clean_code(response.text)
def _run_file(path: Path, args: list, timeout: int) -> str:
interpreters = {
".py": [sys.executable],
".js": ["node"],
".ts": ["ts-node"],
".sh": ["bash"],
".ps1": ["powershell", "-File"],
".rb": ["ruby"],
".php": ["php"],
}
interp = interpreters.get(path.suffix.lower())
if not interp:
return f"No interpreter for {path.suffix}."
try:
result = subprocess.run(
interp + [str(path)] + (args or []),
capture_output=True, text=True,
encoding="utf-8", errors="replace",
timeout=timeout, cwd=str(path.parent)
)
output = result.stdout.strip()
error = result.stderr.strip()
parts = []
if output: parts.append(f"Output:\n{output}")
if error: parts.append(f"Stderr:\n{error}")
return "\n\n".join(parts) if parts else "Executed with no output."
except subprocess.TimeoutExpired:
return f"Timed out after {timeout}s."
except FileNotFoundError:
return f"Interpreter not found: {interp[0]}."
except Exception as e:
return f"Execution error: {e}"
def _build(description, language, output_path, args, timeout, speak=None, player=None) -> str:
if not description:
return "Please describe what you want me to build, sir."
if player:
player.write_log("[Code] Build started...")
lang = language or "python"
try:
code, path = _write(description, lang, output_path, player)
print(f"[Code] ✅ Written: {path}")
except Exception as e:
msg = f"Could not write initial code: {e}"
if speak: speak(msg)
return msg
last_output = ""
for attempt in range(1, MAX_BUILD_ATTEMPTS + 1):
print(f"[Code] 🔄 Attempt {attempt}/{MAX_BUILD_ATTEMPTS}")
if player:
player.write_log(f"[Code] Attempt {attempt}...")
last_output = _run_file(path, args, timeout)
if not _has_error(last_output):
msg = (
f"Build complete, sir. "
f"The code is working after {attempt} attempt{'s' if attempt > 1 else ''}. "
f"Saved to {path}."
)
if speak: speak(msg)
return f"{msg}\n\nOutput:\n{last_output}"
print(f"[Code] ⚠️ Error on attempt {attempt}, fixing...")
if player:
player.write_log(f"[Code] Fixing (attempt {attempt})...")
try:
code = _fix_code(code, last_output, description)
_save_file(path, code)
except Exception as e:
msg = f"Could not fix code on attempt {attempt}: {e}"
if speak: speak(msg)
return msg
msg = (
f"I was unable to build a working version after {MAX_BUILD_ATTEMPTS} attempts, sir. "
f"The last error was: {last_output[:200]}"
)
if speak: speak(msg)
return f"{msg}\n\nLast code saved to: {path}"
def _write_action(description, language, output_path, player) -> str:
if not description:
return "Please describe what you want me to write, sir."
if player:
player.write_log("[Code] Writing code...")
try:
code, path = _write(description, language, output_path, player)
print(f"[Code] ✅ Written: {path}")
return f"Code written. Saved to: {path}\n\nPreview:\n{_preview(code)}"
except Exception as e:
return f"Could not generate code: {e}"
def _edit_action(file_path, instruction, player) -> str:
if not file_path:
return "Please provide a file path to edit, sir."
if not instruction:
return "Please describe what change to make, sir."
content, err = _read_file(file_path)
if err:
return err
if player:
player.write_log("[Code] Editing file...")
model = _get_gemini()
prompt = f"""You are an expert code editor.
Apply the following change to the code below.
Return ONLY the complete updated code — no explanation, no markdown, no backticks.
Change: {instruction}
Original code:
{content}
Updated code:"""
try:
response = model.generate_content(prompt)
edited = _clean_code(response.text)
except Exception as e:
return f"Could not edit code: {e}"
status = _save_file(Path(file_path), edited)
print(f"[Code] ✅ Edited: {file_path}")
return f"File edited. {status}\n\nPreview:\n{_preview(edited)}"
def _explain_action(file_path, code, player) -> str:
if file_path and not code:
code, err = _read_file(file_path)
if err:
return err
if not code:
return "Please provide code or a file path to explain, sir."
if player:
player.write_log("[Code] Analyzing code...")
model = _get_gemini()
prompt = f"""Explain what this code does in simple, clear language.
Focus on: what it does, how it works, and any important details.
Be concise — 3 to 6 sentences maximum.
Code:
{code[:4000]}
Explanation:"""
try:
response = model.generate_content(prompt)
return response.text.strip()
except Exception as e:
return f"Could not explain code: {e}"
def _run_action(file_path, args, timeout, player) -> str:
if not file_path:
return "Please provide a file path to run, sir."
p = Path(file_path)
if not p.exists():
return f"File not found: {file_path}"
if player:
player.write_log(f"[Code] Running {p.name}...")
return _run_file(p, args, timeout)
def _optimize_action(file_path, code, language, output_path, player) -> str:
if file_path and not code:
code, err = _read_file(file_path)
if err:
return err
if not code:
return "Please provide code or a file path to optimize, sir."
if player:
player.write_log("[Code] Optimizing code...")
lang = language or "python"
model = _get_gemini()
prompt = f"""You are an expert {lang} developer and code reviewer.
Optimize the following code for:
1. Performance — eliminate unnecessary operations, use efficient data structures
2. Readability — clear variable names, proper formatting, logical structure
3. Best practices — modern {lang} patterns, error handling, type hints if applicable
4. Remove dead code, redundant comments, and unnecessary complexity
Return ONLY the optimized code — no explanation, no markdown, no backticks.
Original code:
{code[:6000]}
Optimized code:"""
try:
response = model.generate_content(prompt)
optimized = _clean_code(response.text)
except Exception as e:
return f"Could not optimize code: {e}"
# Kaydet
if file_path:
save_path = Path(file_path)
else:
save_path = _resolve_save_path(output_path, lang)
status = _save_file(save_path, optimized)
print(f"[Code] ✅ Optimized: {save_path}")
original_lines = len(code.splitlines())
optimized_lines = len(optimized.splitlines())
diff = original_lines - optimized_lines
return (
f"Code optimized. {status}\n"
f"Lines: {original_lines} → {optimized_lines} "
f"({'−' if diff > 0 else '+'}{abs(diff)} lines)\n\n"
f"Preview:\n{_preview(optimized)}"
)
def _screen_debug_action(description, file_path, player, speak=None) -> str:
if player:
player.write_log("[Code] Taking screenshot for analysis...")
print("[Code] 📸 Capturing screen for debug...")
screenshot_path = _take_screenshot()
if not screenshot_path:
return "Could not take screenshot, sir. Please make sure PyAutoGUI is installed."
file_content = ""
if file_path:
file_content, err = _read_file(file_path)
if err:
print(f"[Code] ⚠️ Could not read file: {err}")
try:
from google import genai
from google.genai import types
client = genai.Client(api_key=_get_api_key())
image_bytes = screenshot_path.read_bytes()
image_base64 = _image_to_base64(screenshot_path)
user_question = description or "What error or problem do you see on the screen? How can it be fixed?"
context = ""
if file_content:
context = f"\n\nAdditionally, here is the related file content:\n```\n{file_content[:4000]}\n```"
analysis_prompt = f"""You are an expert programmer and debugger analyzing a screenshot.
User's question: {user_question}{context}
Please:
1. Identify any errors, exceptions, or problems visible on the screen
2. Explain what is causing the problem in simple terms
3. Provide a concrete fix or solution
4. If there's code visible, show the corrected version
Be specific and actionable. If you see an error message, quote it exactly."""
contents = [
types.Part.from_bytes(data=image_bytes, mime_type="image/png"),
analysis_prompt,
]
response = client.models.generate_content(
model="gemini-2.5-flash",
contents=contents,
)
analysis = response.text.strip()
print(f"[Code] ✅ Screen analysis complete")
try:
screenshot_path.unlink()
except Exception:
pass
if file_path and file_content:
code_match = re.search(r"```[a-zA-Z]*\n(.*?)```", analysis, re.DOTALL)
if code_match:
fixed_code = code_match.group(1).strip()
save_path = Path(file_path)
_save_file(save_path, fixed_code)
analysis += f"\n\n✅ Fixed code has been saved to: {file_path}"
print(f"[Code] ✅ Fixed code saved: {file_path}")
return analysis
except Exception as e:
try:
screenshot_path.unlink()
except Exception:
pass
return f"Screen analysis failed: {e}"
def code_helper(
parameters: dict,
response=None,
player=None,
session_memory=None,
speak=None
) -> str:
"""
Called from main.py.
parameters:
action : write | edit | explain | run | build | screen_debug | optimize | auto
description : What the code should do / what change to make / what problem to analyze
language : Programming language (default: python)
output_path : Where to save — user specifies full path or filename
file_path : Path to existing file (edit / explain / run / build / optimize)
code : Raw code string (explain/optimize without a file)
args : CLI argument list for run/build
timeout : Execution timeout in seconds (default: 30)
"""
p = parameters or {}
action = p.get("action", "auto").lower().strip()
description = p.get("description", "").strip()
language = p.get("language", "python").strip()
output_path = p.get("output_path", "").strip()
file_path = p.get("file_path", "").strip()
code = p.get("code", "").strip()
args = p.get("args", [])
timeout = int(p.get("timeout", 30))
if action == "auto":
action = _detect_intent(description, file_path, code)
print(f"[Code] 🤖 Auto-detected: {action}")
if action == "write":
return _write_action(description, language, output_path, player)
elif action == "edit":
return _edit_action(
file_path,
description or p.get("instruction", ""),
player
)
elif action == "explain":
return _explain_action(file_path, code, player)
elif action == "run":
return _run_action(file_path, args, timeout, player)
elif action == "build":
return _build(description, language, output_path, args, timeout, speak, player)
elif action == "optimize":
return _optimize_action(file_path, code, language, output_path, player)
elif action == "screen_debug":
return _screen_debug_action(description, file_path, player, speak)
else:
return f"Unknown action: '{action}'. Use write, edit, explain, run, build, optimize, or screen_debug."
+554
View File
@@ -0,0 +1,554 @@
# actions/computer_control.py
# MARK XXV — Computer Control
#
# Atomic computer control functions using PyAutoGUI + keyboard + clipboard.
# Used by the agent when no existing action file covers the task.
#
# Capabilities:
# - Type text anywhere (active window, forms, fields)
# - Mouse click, double-click, right-click, drag
# - Keyboard shortcuts and key combinations
# - Scroll (up/down/left/right)
# - Window management (minimize, maximize, close, focus)
# - Clipboard (copy, paste, get content)
# - Screenshot + locate element on screen
# - Wait / smart wait for element to appear
# - Random data generation (name, email, username, password, phone, address)
# - Hotkey sequences
# - Find and click image/element on screen
import json
import sys
import time
import random
import string
import subprocess
import platform
from pathlib import Path
try:
import pyautogui
pyautogui.FAILSAFE = True
pyautogui.PAUSE = 0.05
_PYAUTOGUI = True
except ImportError:
_PYAUTOGUI = False
try:
import pyperclip
_PYPERCLIP = True
except ImportError:
_PYPERCLIP = False
def get_base_dir() -> Path:
if getattr(sys, "frozen", False):
return Path(sys.executable).parent
return Path(__file__).resolve().parent.parent
BASE_DIR = get_base_dir()
API_CONFIG_PATH = BASE_DIR / "config" / "api_keys.json"
def _load_user_profile() -> dict:
"""Load user profile from long_term.json for form filling."""
memory_path = BASE_DIR / "memory" / "long_term.json"
try:
if memory_path.exists():
data = json.loads(memory_path.read_text(encoding="utf-8"))
identity = data.get("identity", {})
return {
"name": identity.get("name", {}).get("value", ""),
"age": identity.get("age", {}).get("value", ""),
"city": identity.get("city", {}).get("value", ""),
"email": identity.get("email", {}).get("value", ""),
}
except Exception:
pass
return {}
def _ensure_pyautogui():
if not _PYAUTOGUI:
raise RuntimeError(
"PyAutoGUI not installed. Run: pip install pyautogui"
)
_FIRST_NAMES = [
"Alex", "Jordan", "Taylor", "Morgan", "Casey", "Riley", "Drew", "Quinn",
"Avery", "Blake", "Cameron", "Dakota", "Emerson", "Finley", "Harper"
]
_LAST_NAMES = [
"Smith", "Johnson", "Williams", "Brown", "Jones", "Garcia", "Miller",
"Davis", "Wilson", "Moore", "Taylor", "Anderson", "Thomas", "Jackson"
]
_DOMAINS = ["gmail.com", "yahoo.com", "outlook.com", "proton.me", "mail.com"]
def generate_random_data(data_type: str) -> str:
"""
Generates random realistic data for form filling.
Types: name | first_name | last_name | email | username |
password | phone | birthday | address | zip_code
"""
dt = data_type.lower().strip()
if dt == "first_name":
return random.choice(_FIRST_NAMES)
elif dt == "last_name":
return random.choice(_LAST_NAMES)
elif dt == "name":
return f"{random.choice(_FIRST_NAMES)} {random.choice(_LAST_NAMES)}"
elif dt == "email":
first = random.choice(_FIRST_NAMES).lower()
last = random.choice(_LAST_NAMES).lower()
num = random.randint(10, 999)
return f"{first}.{last}{num}@{random.choice(_DOMAINS)}"
elif dt == "username":
first = random.choice(_FIRST_NAMES).lower()
num = random.randint(100, 9999)
return f"{first}{num}"
elif dt == "password":
chars = string.ascii_letters + string.digits + "!@#$%"
pwd = (
random.choice(string.ascii_uppercase) +
random.choice(string.digits) +
random.choice("!@#$%") +
"".join(random.choices(chars, k=9))
)
return "".join(random.sample(pwd, len(pwd)))
elif dt == "phone":
return f"+1{random.randint(200,999)}{random.randint(1000000,9999999)}"
elif dt == "birthday":
year = random.randint(1980, 2000)
month = random.randint(1, 12)
day = random.randint(1, 28)
return f"{month:02d}/{day:02d}/{year}"
elif dt == "address":
num = random.randint(100, 9999)
street = random.choice(["Main St", "Oak Ave", "Park Blvd", "Elm St", "Cedar Ln"])
return f"{num} {street}"
elif dt == "zip_code":
return str(random.randint(10000, 99999))
elif dt == "city":
return random.choice(["New York", "Los Angeles", "Chicago", "Houston", "Phoenix"])
return f"random_{data_type}_{random.randint(1000,9999)}"
def _type_text(text: str, interval: float = 0.03) -> str:
"""Types text at the current cursor position."""
_ensure_pyautogui()
time.sleep(0.3)
pyautogui.typewrite(text, interval=interval)
return f"Typed: {text[:50]}{'...' if len(text) > 50 else ''}"
def _click(x: int = None, y: int = None, button: str = "left",
clicks: int = 1, image: str = None) -> str:
"""
Clicks at coordinates or on a screen image.
If image path given, locates it on screen and clicks.
"""
_ensure_pyautogui()
if image:
try:
loc = pyautogui.locateCenterOnScreen(image, confidence=0.8)
if loc:
pyautogui.click(loc.x, loc.y, button=button, clicks=clicks)
return f"Clicked image: {image}"
return f"Image not found on screen: {image}"
except Exception as e:
return f"Image click failed: {e}"
if x is not None and y is not None:
pyautogui.click(x, y, button=button, clicks=clicks)
return f"Clicked ({x}, {y}) with {button} button"
pyautogui.click(button=button, clicks=clicks)
return f"Clicked at current position"
def _hotkey(*keys) -> str:
"""Presses a key combination. E.g. hotkey('ctrl', 'c')"""
_ensure_pyautogui()
pyautogui.hotkey(*keys)
return f"Hotkey: {'+'.join(keys)}"
def _press(key: str) -> str:
"""Presses a single key."""
_ensure_pyautogui()
pyautogui.press(key)
return f"Pressed: {key}"
def _scroll(direction: str = "down", amount: int = 3) -> str:
"""Scrolls in the specified direction."""
_ensure_pyautogui()
clicks = amount if direction in ("up", "right") else -amount
if direction in ("up", "down"):
pyautogui.scroll(clicks)
else:
pyautogui.hscroll(clicks)
return f"Scrolled {direction} {amount} times"
def _move_mouse(x: int, y: int, duration: float = 0.3) -> str:
"""Moves mouse to coordinates."""
_ensure_pyautogui()
pyautogui.moveTo(x, y, duration=duration)
return f"Mouse moved to ({x}, {y})"
def _drag(x1: int, y1: int, x2: int, y2: int, duration: float = 0.5) -> str:
"""Drags from (x1,y1) to (x2,y2)."""
_ensure_pyautogui()
pyautogui.drag(x1 - pyautogui.position()[0], y1 - pyautogui.position()[1])
pyautogui.dragTo(x2, y2, duration=duration)
return f"Dragged from ({x1},{y1}) to ({x2},{y2})"
def _clipboard_copy() -> str:
"""Gets current clipboard content."""
if _PYPERCLIP:
return pyperclip.paste()
_hotkey("ctrl", "c")
time.sleep(0.2)
return "Copied to clipboard"
def _clipboard_set(text: str) -> str:
"""Sets clipboard content and pastes it."""
if _PYPERCLIP:
pyperclip.copy(text)
time.sleep(0.1)
_hotkey("ctrl", "v")
return f"Pasted: {text[:50]}"
return "pyperclip not available"
def _screenshot(save_path: str = None) -> str:
"""Takes a screenshot."""
_ensure_pyautogui()
if not save_path:
save_path = str(Path.home() / "Desktop" / "screenshot.png")
img = pyautogui.screenshot()
img.save(save_path)
return f"Screenshot saved: {save_path}"
def _wait(seconds: float) -> str:
"""Waits for specified seconds."""
time.sleep(seconds)
return f"Waited {seconds}s"
def _wait_for_image(image_path: str, timeout: int = 10) -> str:
"""Waits until an image appears on screen (up to timeout seconds)."""
_ensure_pyautogui()
start = time.time()
while time.time() - start < timeout:
try:
loc = pyautogui.locateCenterOnScreen(image_path, confidence=0.8)
if loc:
return f"Image found at ({loc.x}, {loc.y})"
except Exception:
pass
time.sleep(0.5)
return f"Image not found within {timeout}s: {image_path}"
def _get_screen_size() -> str:
"""Returns current screen resolution."""
_ensure_pyautogui()
w, h = pyautogui.size()
return f"{w}x{h}"
def _focus_window(title: str) -> str:
"""Brings a window to focus by title (Windows)."""
if platform.system() == "Windows":
try:
script = f'(New-Object -ComObject WScript.Shell).AppActivate("{title}")'
subprocess.run(
["powershell", "-NoProfile", "-Command", script],
capture_output=True, timeout=5
)
time.sleep(0.3)
return f"Focused window: {title}"
except Exception as e:
return f"Could not focus window: {e}"
return "Window focus only supported on Windows"
def _select_all() -> str:
return _hotkey("ctrl", "a")
def _clear_field() -> str:
"""Selects all and deletes — clears an input field."""
_hotkey("ctrl", "a")
time.sleep(0.1)
_press("delete")
return "Field cleared"
def _smart_type(text: str, clear_first: bool = True) -> str:
"""
Types text into the currently focused field.
Optionally clears the field first.
Uses clipboard for long text (faster, more reliable).
"""
_ensure_pyautogui()
if clear_first:
_clear_field()
time.sleep(0.1)
if len(text) > 20 and _PYPERCLIP:
pyperclip.copy(text)
time.sleep(0.1)
pyautogui.hotkey("ctrl", "v")
return f"Smart-typed (clipboard): {text[:50]}"
else:
pyautogui.typewrite(text, interval=0.04)
return f"Smart-typed: {text[:50]}"
def _analyze_screen_for_element(description: str) -> tuple[int, int] | None:
"""
Takes a screenshot and asks Gemini to find the coordinates
of a described element on screen. Returns (x, y) or None.
"""
try:
import google.generativeai as genai
import io
cfg_path = API_CONFIG_PATH
with open(cfg_path, "r") as f:
api_key = json.load(f)["gemini_api_key"]
genai.configure(api_key=api_key)
model = genai.GenerativeModel("gemini-2.5-flash-lite")
_ensure_pyautogui()
w, h = pyautogui.size()
img = pyautogui.screenshot()
buf = io.BytesIO()
img.save(buf, format="PNG")
buf.seek(0)
prompt = (
f"This is a screenshot of a computer screen ({w}x{h} pixels). "
f"Find the element: '{description}'. "
f"Return ONLY: x,y (the center coordinates of the element). "
f"If not found, return: NOT_FOUND"
)
response = model.generate_content([
{"mime_type": "image/png", "data": buf.getvalue()},
prompt
])
text = response.text.strip()
if "NOT_FOUND" in text:
return None
import re
match = re.search(r"(\d+)\s*,\s*(\d+)", text)
if match:
return int(match.group(1)), int(match.group(2))
except Exception as e:
print(f"[ComputerControl] ⚠️ Screen analysis failed: {e}")
return None
def computer_control(
parameters: dict,
response=None,
player=None,
session_memory=None,
) -> str:
"""
Universal computer control action.
Actions:
type : Type text at current cursor position
smart_type : Clear field + type (clipboard-based for long text)
click : Click at coordinates or on image
double_click : Double click
right_click : Right click
hotkey : Key combination (e.g. ctrl+c)
press : Single key press
scroll : Scroll up/down/left/right
move : Move mouse to coordinates
drag : Drag from one point to another
copy : Get clipboard content
paste : Set and paste clipboard content
screenshot : Take a screenshot
wait : Wait N seconds
wait_image : Wait for image to appear on screen
clear_field : Select all + delete in current field
focus_window : Bring window to foreground
screen_find : AI-powered element finder — returns coordinates
screen_click : AI-powered element finder + click
random_data : Generate random data for forms
user_data : Get user's real data from memory
"""
action = (parameters or {}).get("action", "").lower().strip()
if not action:
return "Please specify an action for computer_control, sir."
if player:
player.write_log(f"[Computer] {action}")
print(f"[ComputerControl] ▶️ Action: {action} Params: {parameters}")
try:
if action == "type":
text = parameters.get("text", "")
return _type_text(text)
elif action == "smart_type":
text = parameters.get("text", "")
clear_first = parameters.get("clear_first", True)
return _smart_type(text, clear_first=clear_first)
elif action in ("click", "left_click"):
return _click(
x=parameters.get("x"),
y=parameters.get("y"),
button="left",
clicks=1,
image=parameters.get("image")
)
elif action == "double_click":
return _click(
x=parameters.get("x"),
y=parameters.get("y"),
button="left",
clicks=2,
image=parameters.get("image")
)
elif action == "right_click":
return _click(
x=parameters.get("x"),
y=parameters.get("y"),
button="right",
clicks=1
)
elif action == "move":
return _move_mouse(
x=int(parameters.get("x", 0)),
y=int(parameters.get("y", 0)),
duration=float(parameters.get("duration", 0.3))
)
elif action == "drag":
return _drag(
x1=int(parameters.get("x1", 0)),
y1=int(parameters.get("y1", 0)),
x2=int(parameters.get("x2", 0)),
y2=int(parameters.get("y2", 0))
)
elif action == "hotkey":
keys = parameters.get("keys", "")
if isinstance(keys, str):
keys = [k.strip() for k in keys.split("+")]
return _hotkey(*keys)
elif action == "press":
return _press(parameters.get("key", "enter"))
elif action == "scroll":
return _scroll(
direction=parameters.get("direction", "down"),
amount=int(parameters.get("amount", 3))
)
elif action == "copy":
return _clipboard_copy()
elif action == "paste":
return _clipboard_set(parameters.get("text", ""))
elif action == "screenshot":
return _screenshot(parameters.get("path"))
elif action == "wait":
return _wait(float(parameters.get("seconds", 1.0)))
elif action == "wait_image":
return _wait_for_image(
parameters.get("image", ""),
timeout=int(parameters.get("timeout", 10))
)
elif action == "clear_field":
return _clear_field()
elif action == "focus_window":
return _focus_window(parameters.get("title", ""))
elif action == "screen_size":
return _get_screen_size()
elif action == "screen_find":
description = parameters.get("description", "")
coords = _analyze_screen_for_element(description)
if coords:
return f"{coords[0]},{coords[1]}"
return "NOT_FOUND"
elif action == "screen_click":
description = parameters.get("description", "")
coords = _analyze_screen_for_element(description)
if coords:
time.sleep(0.2)
_click(x=coords[0], y=coords[1])
return f"Found and clicked: {description} at {coords}"
return f"Could not find on screen: {description}"
elif action == "random_data":
data_type = parameters.get("type", "name")
result = generate_random_data(data_type)
print(f"[ComputerControl] 🎲 Random {data_type}: {result}")
return result
elif action == "user_data":
field = parameters.get("field", "name")
profile = _load_user_profile()
value = profile.get(field, "")
if not value:
value = generate_random_data(field)
print(f"[ComputerControl] ⚠️ No user {field} in memory, using random: {value}")
return value
else:
return f"Unknown computer_control action: '{action}'"
except Exception as e:
print(f"[ComputerControl] ❌ Error: {e}")
return f"computer_control failed: {e}"
+679
View File
@@ -0,0 +1,679 @@
# actions/computer_settings.py
# MARK XXV — Computer Settings & UI Controls
#
# Kullanıcı "sesi aç", "uygulamayı kapat", "tam ekran yap", "şunu yaz" gibi
# bilgisayar kontrol komutları verdiğinde bu dosya devreye girer.
#
# - Intent detection: Gemini ile (multi-language, hardcoded keyword yok)
# - Cross-platform: Windows / macOS / Linux
# - pyautogui + platform-specific API'ler
import time
import subprocess
import sys
import platform
from pathlib import Path
try:
import pyautogui
pyautogui.FAILSAFE = True
pyautogui.PAUSE = 0.05
_PYAUTOGUI = True
except ImportError:
_PYAUTOGUI = False
try:
import pyperclip
_PYPERCLIP = True
except ImportError:
_PYPERCLIP = False
_OS = platform.system()
def get_base_dir() -> Path:
if getattr(sys, "frozen", False):
return Path(sys.executable).parent
return Path(__file__).resolve().parent.parent
BASE_DIR = get_base_dir()
API_CONFIG_PATH = BASE_DIR / "config" / "api_keys.json"
import json
def _get_api_key() -> str:
with open(API_CONFIG_PATH, "r", encoding="utf-8") as f:
return json.load(f)["gemini_api_key"]
def volume_up():
if _OS == "Windows":
for _ in range(5): pyautogui.press("volumeup")
elif _OS == "Darwin":
subprocess.run(["osascript", "-e", "set volume output volume (output volume of (get volume settings) + 10)"])
else:
subprocess.run(["pactl", "set-sink-volume", "@DEFAULT_SINK@", "+10%"])
def volume_down():
if _OS == "Windows":
for _ in range(5): pyautogui.press("volumedown")
elif _OS == "Darwin":
subprocess.run(["osascript", "-e", "set volume output volume (output volume of (get volume settings) - 10)"])
else:
subprocess.run(["pactl", "set-sink-volume", "@DEFAULT_SINK@", "-10%"])
def volume_mute():
if _OS == "Windows":
pyautogui.press("volumemute")
elif _OS == "Darwin":
subprocess.run(["osascript", "-e", "set volume with output muted"])
else:
subprocess.run(["pactl", "set-sink-mute", "@DEFAULT_SINK@", "toggle"])
def volume_set(value: int):
value = max(0, min(100, value))
if _OS == "Windows":
try:
from ctypes import cast, POINTER
from comtypes import CLSCTX_ALL
from pycaw.pycaw import AudioUtilities, IAudioEndpointVolume
import math
devices = AudioUtilities.GetSpeakers()
interface = devices.Activate(IAudioEndpointVolume._iid_, CLSCTX_ALL, None)
vol = cast(interface, POINTER(IAudioEndpointVolume))
vol_db = -65.25 if value == 0 else max(-65.25, 20 * math.log10(value / 100))
vol.SetMasterVolumeLevel(vol_db, None)
print(f"[Settings] 🔊 Volume → {value}%")
return
except Exception as e:
print(f"[Settings] ⚠️ pycaw failed: {e}")
elif _OS == "Darwin":
subprocess.run(["osascript", "-e", f"set volume output volume {value}"])
return
else:
subprocess.run(["pactl", "set-sink-volume", "@DEFAULT_SINK@", f"{value}%"])
return
def brightness_up():
if _OS == "Windows":
pyautogui.hotkey("win", "a")
time.sleep(0.3)
elif _OS == "Darwin":
subprocess.run(["osascript", "-e", "tell application \"System Events\" to key code 144"])
else:
subprocess.run(["brightnessctl", "set", "+10%"])
def brightness_down():
if _OS == "Windows":
pyautogui.hotkey("win", "a")
time.sleep(0.3)
elif _OS == "Darwin":
subprocess.run(["osascript", "-e", "tell application \"System Events\" to key code 145"])
else:
subprocess.run(["brightnessctl", "set", "10%-"])
def close_app():
if _OS == "Darwin":
pyautogui.hotkey("command", "q")
else:
pyautogui.hotkey("alt", "f4")
def close_window():
if _OS == "Darwin":
pyautogui.hotkey("command", "w")
else:
pyautogui.hotkey("ctrl", "w")
def full_screen():
if _OS == "Darwin":
pyautogui.hotkey("ctrl", "command", "f")
else:
pyautogui.press("f11")
def minimize_window():
if _OS == "Darwin":
pyautogui.hotkey("command", "m")
else:
pyautogui.hotkey("win", "down")
def maximize_window():
if _OS == "Darwin":
subprocess.run(["osascript", "-e",
'tell application "System Events" to keystroke "f" using {control down, command down}'])
else:
pyautogui.hotkey("win", "up")
def snap_left():
if _OS == "Windows": pyautogui.hotkey("win", "left")
def snap_right():
if _OS == "Windows": pyautogui.hotkey("win", "right")
def switch_window():
if _OS == "Darwin":
pyautogui.hotkey("command", "tab")
else:
pyautogui.hotkey("alt", "tab")
def show_desktop():
if _OS == "Darwin":
pyautogui.hotkey("fn", "f11")
elif _OS == "Windows":
pyautogui.hotkey("win", "d")
else:
pyautogui.hotkey("super", "d")
def open_task_manager():
if _OS == "Windows":
pyautogui.hotkey("ctrl", "shift", "esc")
elif _OS == "Darwin":
subprocess.Popen(["open", "-a", "Activity Monitor"])
else:
subprocess.Popen(["gnome-system-monitor"])
def open_task_view():
if _OS == "Windows":
pyautogui.hotkey("win", "tab")
def focus_search():
if _OS == "Darwin": pyautogui.hotkey("command", "l")
else: pyautogui.hotkey("ctrl", "l")
def pause_video(): pyautogui.press("space")
def refresh_page():
if _OS == "Darwin": pyautogui.hotkey("command", "r")
else: pyautogui.press("f5")
def close_tab():
if _OS == "Darwin": pyautogui.hotkey("command", "w")
else: pyautogui.hotkey("ctrl", "w")
def new_tab():
if _OS == "Darwin": pyautogui.hotkey("command", "t")
else: pyautogui.hotkey("ctrl", "t")
def next_tab():
if _OS == "Darwin": pyautogui.hotkey("command", "shift", "bracketright")
else: pyautogui.hotkey("ctrl", "tab")
def prev_tab():
if _OS == "Darwin": pyautogui.hotkey("command", "shift", "bracketleft")
else: pyautogui.hotkey("ctrl", "shift", "tab")
def go_back():
if _OS == "Darwin": pyautogui.hotkey("command", "left")
else: pyautogui.hotkey("alt", "left")
def go_forward():
if _OS == "Darwin": pyautogui.hotkey("command", "right")
else: pyautogui.hotkey("alt", "right")
def zoom_in():
if _OS == "Darwin": pyautogui.hotkey("command", "equal")
else: pyautogui.hotkey("ctrl", "equal")
def zoom_out():
if _OS == "Darwin": pyautogui.hotkey("command", "minus")
else: pyautogui.hotkey("ctrl", "minus")
def zoom_reset():
if _OS == "Darwin": pyautogui.hotkey("command", "0")
else: pyautogui.hotkey("ctrl", "0")
def find_on_page():
if _OS == "Darwin": pyautogui.hotkey("command", "f")
else: pyautogui.hotkey("ctrl", "f")
def reload_page_n(n: int):
for _ in range(n):
refresh_page()
time.sleep(0.8)
def scroll_up(amount: int = 500): pyautogui.scroll(amount)
def scroll_down(amount: int = 500): pyautogui.scroll(-amount)
def scroll_top(): pyautogui.hotkey("ctrl", "home") if _OS != "Darwin" else pyautogui.hotkey("command", "up")
def scroll_bottom(): pyautogui.hotkey("ctrl", "end") if _OS != "Darwin" else pyautogui.hotkey("command", "down")
def page_up(): pyautogui.press("pageup")
def page_down(): pyautogui.press("pagedown")
def copy():
if _OS == "Darwin": pyautogui.hotkey("command", "c")
else: pyautogui.hotkey("ctrl", "c")
def paste():
if _OS == "Darwin": pyautogui.hotkey("command", "v")
else: pyautogui.hotkey("ctrl", "v")
def cut():
if _OS == "Darwin": pyautogui.hotkey("command", "x")
else: pyautogui.hotkey("ctrl", "x")
def undo():
if _OS == "Darwin": pyautogui.hotkey("command", "z")
else: pyautogui.hotkey("ctrl", "z")
def redo():
if _OS == "Darwin": pyautogui.hotkey("command", "shift", "z")
else: pyautogui.hotkey("ctrl", "y")
def select_all():
if _OS == "Darwin": pyautogui.hotkey("command", "a")
else: pyautogui.hotkey("ctrl", "a")
def save_file():
if _OS == "Darwin": pyautogui.hotkey("command", "s")
else: pyautogui.hotkey("ctrl", "s")
def press_enter(): pyautogui.press("enter")
def press_escape(): pyautogui.press("escape")
def press_key(key: str): pyautogui.press(key)
def type_text(text: str, press_enter_after: bool = False):
if not text:
return
if _PYPERCLIP:
pyperclip.copy(text)
time.sleep(0.1)
paste()
else:
pyautogui.write(str(text), interval=0.03)
if press_enter_after:
time.sleep(0.1)
pyautogui.press("enter")
def write_on_screen(text: str):
type_text(text)
def take_screenshot():
if _OS == "Windows":
pyautogui.hotkey("win", "shift", "s")
elif _OS == "Darwin":
pyautogui.hotkey("command", "shift", "3")
else:
pyautogui.hotkey("ctrl", "print_screen")
def lock_screen():
if _OS == "Windows":
pyautogui.hotkey("win", "l")
elif _OS == "Darwin":
subprocess.run(["pmset", "displaysleepnow"])
else:
subprocess.run(["gnome-screensaver-command", "-l"])
def open_system_settings():
if _OS == "Windows":
pyautogui.hotkey("win", "i")
elif _OS == "Darwin":
subprocess.Popen(["open", "-a", "System Preferences"])
else:
subprocess.Popen(["gnome-control-center"])
def open_file_explorer():
if _OS == "Windows":
pyautogui.hotkey("win", "e")
elif _OS == "Darwin":
subprocess.Popen(["open", Path.home()])
else:
subprocess.Popen(["xdg-open", Path.home()])
def open_run():
if _OS == "Windows":
pyautogui.hotkey("win", "r")
def sleep_display():
if _OS == "Windows":
try:
import ctypes
ctypes.windll.user32.SendMessageW(0xFFFF, 0x0112, 0xF170, 2)
except Exception:
pass
elif _OS == "Darwin":
subprocess.run(["pmset", "displaysleepnow"])
else:
subprocess.run(["xset", "dpms", "force", "off"])
def restart_computer():
if _OS == "Windows":
subprocess.run(["shutdown", "/r", "/t", "5"])
elif _OS == "Darwin":
subprocess.run(["osascript", "-e", 'tell application "System Events" to restart'])
else:
subprocess.run(["sudo", "reboot"])
def shutdown_computer():
if _OS == "Windows":
subprocess.run(["shutdown", "/s", "/t", "5"])
elif _OS == "Darwin":
subprocess.run(["osascript", "-e", 'tell application "System Events" to shut down'])
else:
subprocess.run(["sudo", "shutdown", "-h", "now"])
def dark_mode():
if _OS == "Windows":
pyautogui.hotkey("win", "a")
time.sleep(0.3)
elif _OS == "Darwin":
subprocess.run(["osascript", "-e",
'tell app "System Events" to tell appearance preferences to set dark mode to not dark mode'])
def toggle_wifi():
if _OS == "Windows":
pyautogui.hotkey("win", "a")
time.sleep(0.3)
elif _OS == "Darwin":
subprocess.run(["networksetup", "-setairportpower", "en0", "toggle"])
else:
subprocess.run(["nmcli", "radio", "wifi"])
ACTION_MAP = {
"volume_up": volume_up,
"volume_down": volume_down,
"mute": volume_mute,
"unmute": volume_mute,
"volume_increase": volume_up,
"volume_decrease": volume_down,
"increase_volume": volume_up,
"decrease_volume": volume_down,
"turn_up_volume": volume_up,
"turn_down_volume": volume_down,
"louder": volume_up,
"quieter": volume_down,
"silence": volume_mute,
"toggle_mute": volume_mute,
"brightness_up": brightness_up,
"brightness_down": brightness_down,
"increase_brightness": brightness_up,
"decrease_brightness": brightness_down,
"brighter": brightness_up,
"dimmer": brightness_down,
"dim_screen": brightness_down,
"brighten_screen": brightness_up,
"sleep_display": sleep_display,
"turn_off_screen": sleep_display,
"screen_off": sleep_display,
"display_off": sleep_display,
"change_screen": sleep_display,
"screen_sleep": sleep_display,
"monitor_off": sleep_display,
"turn_off_monitor": sleep_display,
"pause_video": pause_video,
"play_video": pause_video,
"pause": pause_video,
"play": pause_video,
"toggle_play": pause_video,
"stop_video": pause_video,
"resume_video": pause_video,
"close_app": close_app,
"close_window": close_window,
"quit_app": close_app,
"exit_app": close_app,
"kill_app": close_app,
"full_screen": full_screen,
"fullscreen": full_screen,
"toggle_fullscreen": full_screen,
"minimize": minimize_window,
"minimize_window": minimize_window,
"maximize": maximize_window,
"maximize_window": maximize_window,
"restore_window": maximize_window,
"snap_left": snap_left,
"snap_right": snap_right,
"window_left": snap_left,
"window_right": snap_right,
"switch_window": switch_window,
"alt_tab": switch_window,
"next_window": switch_window,
"show_desktop": show_desktop,
"desktop": show_desktop,
"hide_windows": show_desktop,
"task_manager": open_task_manager,
"open_task_manager": open_task_manager,
"task_view": open_task_view,
"screenshot": take_screenshot,
"take_screenshot": take_screenshot,
"capture_screen": take_screenshot,
"lock_screen": lock_screen,
"lock": lock_screen,
"open_settings": open_system_settings,
"system_settings": open_system_settings,
"settings": open_system_settings,
"preferences": open_system_settings,
"file_explorer": open_file_explorer,
"open_explorer": open_file_explorer,
"explorer": open_file_explorer,
"open_files": open_file_explorer,
"run": open_run,
"open_run": open_run,
"restart": restart_computer,
"restart_computer": restart_computer,
"reboot": restart_computer,
"reboot_computer": restart_computer,
"shutdown": shutdown_computer,
"shut_down": shutdown_computer,
"power_off": shutdown_computer,
"turn_off_computer": shutdown_computer,
"dark_mode": dark_mode,
"toggle_dark_mode": dark_mode,
"night_mode": dark_mode,
"toggle_wifi": toggle_wifi,
"wifi": toggle_wifi,
"wifi_toggle": toggle_wifi,
"focus_search": focus_search,
"address_bar": focus_search,
"url_bar": focus_search,
"refresh_page": refresh_page,
"reload_page": refresh_page,
"reload": refresh_page,
"refresh": refresh_page,
"close_tab": close_tab,
"new_tab": new_tab,
"open_tab": new_tab,
"next_tab": next_tab,
"prev_tab": prev_tab,
"previous_tab": prev_tab,
"go_back": go_back,
"back": go_back,
"go_forward": go_forward,
"forward": go_forward,
"zoom_in": zoom_in,
"zoom_out": zoom_out,
"zoom_reset": zoom_reset,
"reset_zoom": zoom_reset,
"find_on_page": find_on_page,
"search_page": find_on_page,
"scroll_up": scroll_up,
"scroll_down": scroll_down,
"scroll_top": scroll_top,
"scroll_bottom": scroll_bottom,
"top_of_page": scroll_top,
"bottom_of_page": scroll_bottom,
"page_up": page_up,
"page_down": page_down,
"copy": copy,
"paste": paste,
"cut": cut,
"undo": undo,
"redo": redo,
"select_all": select_all,
"save": save_file,
"save_file": save_file,
"enter": press_enter,
"press_enter": press_enter,
"escape": press_escape,
"press_escape": press_escape,
"cancel": press_escape,
}
def _detect_action(description: str) -> dict:
"""
Gemini ile kullanıcının ne yapmak istediğini anlar.
Herhangi bir dilde çalışır.
Döner: {"action": str, "value": optional}
"""
import google.generativeai as genai
genai.configure(api_key=_get_api_key())
model = genai.GenerativeModel("gemini-2.5-flash-lite")
available = ", ".join(sorted(ACTION_MAP.keys())) + ", volume_set, type_text, write_on_screen, reload_n, press_key"
prompt = f"""The user wants to control their computer. Detect their intent.
User said (in any language): "{description}"
Available actions: {available}
Return ONLY valid JSON:
{{"action": "action_name", "value": null_or_value}}
Examples:
- "turn up the volume" → {{"action": "volume_up", "value": null}}
- "set volume to 60" → {{"action": "volume_set", "value": 60}}
- "sesi 80 yap" → {{"action": "volume_set", "value": 80}}
- "close the app" → {{"action": "close_app", "value": null}}
- "uygulamayı kapat" → {{"action": "close_app", "value": null}}
- "type hello world" → {{"action": "type_text", "value": "hello world"}}
- "write good morning on screen" → {{"action": "write_on_screen", "value": "good morning"}}
- "reload page 3 times" → {{"action": "reload_n", "value": 3}}
- "tam ekran yap" → {{"action": "full_screen", "value": null}}
- "sesi kıs" → {{"action": "volume_down", "value": null}}
- "sesi aç" → {{"action": "volume_up", "value": null}}
- "sustur" → {{"action": "mute", "value": null}}
- "monte le son" → {{"action": "volume_up", "value": null}}
- "ekranı kapat" → {{"action": "sleep_display", "value": null}}
- "monitörü kapat" → {{"action": "sleep_display", "value": null}}
- "turn off screen" → {{"action": "sleep_display", "value": null}}
- "turn off monitor" → {{"action": "sleep_display", "value": null}}
- "bilgisayarı yeniden başlat" → {{"action": "restart", "value": null}}
- "restart the computer" → {{"action": "restart", "value": null}}
- "bilgisayarı kapat" → {{"action": "shutdown", "value": null}}
- "shut down" → {{"action": "shutdown", "value": null}}
- "ekranı kilitle" → {{"action": "lock_screen", "value": null}}
- "lock the screen" → {{"action": "lock_screen", "value": null}}
- "küçült" → {{"action": "minimize", "value": null}}
- "minimize the window" → {{"action": "minimize", "value": null}}
- "büyüt" → {{"action": "maximize", "value": null}}
- "parlaklığı artır" → {{"action": "brightness_up", "value": null}}
- "parlaklığı azalt" → {{"action": "brightness_down", "value": null}}
- "increase brightness" → {{"action": "brightness_up", "value": null}}
- "wifi'yi aç" → {{"action": "toggle_wifi", "value": null}}
- "toggle wifi" → {{"action": "toggle_wifi", "value": null}}
- "masaüstünü göster" → {{"action": "show_desktop", "value": null}}
- "show desktop" → {{"action": "show_desktop", "value": null}}
- "yeni sekme aç" → {{"action": "new_tab", "value": null}}
- "sekmeyi kapat" → {{"action": "close_tab", "value": null}}
- "geri git" → {{"action": "go_back", "value": null}}
- "ileri git" → {{"action": "go_forward", "value": null}}
- "sayfayı yenile" → {{"action": "refresh_page", "value": null}}
- "yakınlaştır" → {{"action": "zoom_in", "value": null}}
- "uzaklaştır" → {{"action": "zoom_out", "value": null}}
- "kaydet" → {{"action": "save", "value": null}}
- "geri al" → {{"action": "undo", "value": null}}
- "screenshot al" → {{"action": "screenshot", "value": null}}
- "ekran görüntüsü al" → {{"action": "screenshot", "value": null}}
- "aşağı kaydır" → {{"action": "scroll_down", "value": null}}
- "yukarı kaydır" → {{"action": "scroll_up", "value": null}}
- "karanlık mod" → {{"action": "dark_mode", "value": null}}
- "press f5" → {{"action": "press_key", "value": "f5"}}
- "enter'a bas" → {{"action": "enter", "value": null}}
- "escape'e bas" → {{"action": "escape", "value": null}}
IMPORTANT:
- Always return one of the available actions listed above.
- If the user's intent is clear but uses different wording, map it to the closest action.
- Never invent new action names not in the available list.
- Return ONLY the JSON object, no explanation, no markdown."""
try:
response = model.generate_content(prompt)
text = response.text.strip()
text = __import__("re").sub(r"```(?:json)?", "", text).strip().rstrip("`").strip()
return json.loads(text)
except Exception as e:
print(f"[Settings] ⚠️ Intent detection failed: {e}")
return {"action": description.lower().replace(" ", "_"), "value": None}
def computer_settings(
parameters: dict,
response=None,
player=None,
session_memory=None,
) -> str:
"""
Bilgisayar ayarları ve UI kontrolleri.
parameters:
action : İşlem adı (verilmezse description'dan Gemini ile tespit edilir)
description : Kullanıcının doğal dil komutu (herhangi bir dilde)
value : İşleme özgü değer (ses seviyesi, yazılacak metin, tekrar sayısı vb.)
"""
if not _PYAUTOGUI:
return "pyautogui is not installed. Run: pip install pyautogui"
params = parameters or {}
raw_action = params.get("action", "").strip()
description = params.get("description", "").strip()
value = params.get("value", None)
if not raw_action and description:
detected = _detect_action(description)
raw_action = detected.get("action", "")
if value is None:
value = detected.get("value")
action = raw_action.lower().strip().replace(" ", "_").replace("-", "_")
if not action:
return "No action could be determined, sir."
print(f"[Settings] ⚙️ Action: {action} Value: {value}")
if action == "volume_set":
try:
volume_set(int(value or 50))
return f"Volume set to {value}%."
except Exception as e:
return f"Could not set volume: {e}"
if action in ("type_text", "write_on_screen", "type", "write"):
text = str(value or params.get("text", ""))
if not text:
return "No text provided to type, sir."
enter_after = bool(params.get("press_enter", False))
type_text(text, press_enter_after=enter_after)
return f"Typed: {text[:60]}"
if action == "press_key":
key = str(value or params.get("key", ""))
if not key:
return "No key specified, sir."
press_key(key)
return f"Pressed: {key}"
if action in ("reload_n", "refresh_n", "reload_page_n"):
try:
n = int(value or 1)
reload_page_n(n)
return f"Reloaded page {n} time{'s' if n > 1 else ''}."
except Exception as e:
return f"Could not reload: {e}"
if action in ("scroll_up", "scroll_down"):
try:
amount = int(value or 500)
scroll_up(amount) if action == "scroll_up" else scroll_down(amount)
return f"Scrolled {'up' if action == 'scroll_up' else 'down'}."
except Exception as e:
return f"Scroll failed: {e}"
func = ACTION_MAP.get(action)
if not func:
return f"Unknown action: '{raw_action}', sir."
try:
func()
return f"Done: {action}."
except Exception as e:
return f"Action failed ({action}): {e}"
+417
View File
@@ -0,0 +1,417 @@
# actions/desktop.py
# AI-powered desktop & wallpaper management
#
# Flow for unknown tasks:
# User request → Gemini generates Python/pyautogui code → Safety check → Execute
#
# Built-in: wallpaper change, icon arrangement, desktop cleanup, organize by type
import os
import sys
import json
import shutil
import subprocess
import ctypes
import tempfile
import pyautogui
from pathlib import Path
from datetime import datetime
def get_base_dir():
if getattr(sys, "frozen", False):
return Path(sys.executable).parent
return Path(__file__).resolve().parent.parent
BASE_DIR = get_base_dir()
API_CONFIG_PATH = BASE_DIR / "config" / "api_keys.json"
def _get_api_key() -> str:
with open(API_CONFIG_PATH, "r", encoding="utf-8") as f:
return json.load(f)["gemini_api_key"]
def _get_desktop() -> Path:
return Path.home() / "Desktop"
BLOCKED_KEYWORDS = [
"os.remove", "shutil.rmtree", "shutil.rm",
"subprocess.run", "subprocess.Popen", "subprocess.call",
"os.system", "exec(", "eval(",
"import os", "import subprocess",
"__import__", "open(",
"sys.exit", "quit()",
]
def _is_safe_code(code: str) -> tuple[bool, str]:
code_lower = code.lower()
for keyword in BLOCKED_KEYWORDS:
if keyword.lower() in code_lower:
return False, f"Blocked operation: '{keyword}'"
return True, "OK"
def _ask_gemini_for_desktop_action(task: str) -> str:
"""
Asks Gemini to generate safe Python/pyautogui code
to accomplish a desktop-related task.
"""
import google.generativeai as genai
genai.configure(api_key=_get_api_key())
model = genai.GenerativeModel("gemini-2.5-flash")
desktop = str(_get_desktop())
prompt = f"""You are a Windows desktop automation expert.
Generate safe Python code using ONLY these allowed modules:
- pyautogui (mouse, keyboard, screenshots)
- pathlib.Path (already imported as Path)
- shutil (ONLY: copy2, copytree, move, disk_usage)
- os.path (ONLY: exists, join, dirname, basename, splitext)
- time (sleep only)
- ctypes (Windows API calls only)
- winreg (registry READ only)
Desktop path: {desktop}
Rules:
- Output ONLY the Python code. No explanation, no markdown, no backticks.
- NO file deletion (os.remove, shutil.rmtree, unlink, etc.)
- NO subprocess calls
- NO exec() or eval()
- NO file write operations
- If task cannot be done safely, output exactly: UNSAFE
Task: {task}
Python code:"""
try:
response = model.generate_content(prompt)
code = response.text.strip()
if code.startswith("```"):
lines = code.split("\n")
code = "\n".join(lines[1:-1]).strip()
return code
except Exception as e:
return f"ERROR: {e}"
def _execute_generated_code(code: str) -> str:
"""Safely executes Gemini-generated desktop automation code."""
safe, reason = _is_safe_code(code)
if not safe:
return f"⛔ Blocked for safety: {reason}"
allowed_globals = {
"pyautogui": pyautogui,
"Path": Path,
"shutil": shutil,
"ctypes": ctypes,
"time": __import__("time"),
"os": type("os", (), {
"path": os.path,
"listdir": os.listdir,
"getcwd": os.getcwd,
"environ": os.environ,
})(),
"__builtins__": {
"print": print,
"len": len,
"str": str,
"int": int,
"float": float,
"bool": bool,
"list": list,
"dict": dict,
"range": range,
"enumerate": enumerate,
"sorted": sorted,
"isinstance": isinstance,
"hasattr": hasattr,
"getattr": getattr,
"max": max,
"min": min,
"sum": sum,
}
}
output_lines = []
allowed_globals["print"] = lambda *args: output_lines.append(" ".join(str(a) for a in args))
try:
exec(code, allowed_globals)
return "\n".join(output_lines) if output_lines else "Task completed successfully."
except Exception as e:
return f"Execution error: {e}\n\nCode attempted:\n{code[:200]}"
def set_wallpaper(image_path: str) -> str:
"""Sets desktop wallpaper from a local image path."""
path = Path(image_path).expanduser().resolve()
if not path.exists():
return f"Image not found: {image_path}"
if path.suffix.lower() not in [".jpg", ".jpeg", ".png", ".bmp"]:
return f"Unsupported format: {path.suffix}. Use jpg, png or bmp."
try:
if sys.platform == "win32":
abs_path = str(path.resolve())
ctypes.windll.user32.SystemParametersInfoW(20, 0, abs_path, 3)
return f"Wallpaper set: {path.name}"
elif sys.platform == "darwin":
script = f'tell application "Finder" to set desktop picture to POSIX file "{path}"'
subprocess.run(["osascript", "-e", script])
return f"Wallpaper set: {path.name}"
else:
subprocess.run(["gsettings", "set", "org.gnome.desktop.background",
"picture-uri", f"file://{path}"])
return f"Wallpaper set: {path.name}"
except Exception as e:
return f"Could not set wallpaper: {e}"
def set_wallpaper_from_web(url: str) -> str:
"""Downloads an image from URL and sets it as wallpaper."""
try:
import urllib.request
suffix = Path(url.split("?")[0]).suffix or ".jpg"
tmp = Path(tempfile.mktemp(suffix=suffix))
urllib.request.urlretrieve(url, str(tmp))
result = set_wallpaper(str(tmp))
return result
except Exception as e:
return f"Could not download wallpaper: {e}"
def get_current_wallpaper() -> str:
"""Returns the current wallpaper path."""
try:
if sys.platform == "win32":
import winreg
key = winreg.OpenKey(winreg.HKEY_CURRENT_USER,
r"Control Panel\Desktop")
val, _ = winreg.QueryValueEx(key, "Wallpaper")
return f"Current wallpaper: {val}"
else:
return "Wallpaper path retrieval not supported on this OS."
except Exception as e:
return f"Could not get wallpaper: {e}"
FILE_TYPE_MAP = {
"Images": [".jpg", ".jpeg", ".png", ".gif", ".bmp", ".webp", ".svg", ".ico", ".heic"],
"Documents": [".pdf", ".doc", ".docx", ".txt", ".xls", ".xlsx", ".ppt", ".pptx", ".csv", ".odt"],
"Videos": [".mp4", ".avi", ".mkv", ".mov", ".wmv", ".flv", ".webm", ".m4v"],
"Music": [".mp3", ".wav", ".flac", ".aac", ".ogg", ".wma", ".m4a"],
"Archives": [".zip", ".rar", ".7z", ".tar", ".gz", ".bz2"],
"Code": [".py", ".js", ".html", ".css", ".json", ".xml", ".ts", ".cpp", ".java", ".cs", ".php"],
"Executables": [".exe", ".msi", ".bat", ".cmd", ".sh"],
}
def organize_desktop(mode: str = "by_type") -> str:
"""
Organizes desktop files.
mode: 'by_type' — groups by file type (Images, Documents, etc.)
'by_date' — groups by month (2024-01, 2024-02, etc.)
"""
desktop = _get_desktop()
moved = []
skipped = []
for item in desktop.iterdir():
if item.is_dir() or item.name.startswith("."):
continue
if item.suffix.lower() == ".lnk":
continue
if mode == "by_date":
mtime = datetime.fromtimestamp(item.stat().st_mtime)
folder_name = mtime.strftime("%Y-%m")
else:
ext = item.suffix.lower()
folder_name = "Others"
for folder, exts in FILE_TYPE_MAP.items():
if ext in exts:
folder_name = folder
break
target_dir = desktop / folder_name
target_dir.mkdir(exist_ok=True)
new_path = target_dir / item.name
if new_path.exists():
skipped.append(item.name)
continue
shutil.move(str(item), str(new_path))
moved.append(f"{item.name} → {folder_name}/")
result = f"Desktop organized ({mode}). {len(moved)} files moved."
if moved:
preview = moved[:8]
result += "\n" + "\n".join(preview)
if len(moved) > 8:
result += f"\n... and {len(moved)-8} more."
if skipped:
result += f"\n{len(skipped)} files skipped (name conflict)."
return result
def list_desktop() -> str:
"""Lists everything on the desktop."""
desktop = _get_desktop()
items = []
for item in sorted(desktop.iterdir()):
if item.name.startswith("."):
continue
if item.is_dir():
count = len(list(item.iterdir()))
items.append(f"📁 {item.name}/ ({count} items)")
else:
size = item.stat().st_size
size_str = f"{size/1024:.1f} KB" if size < 1024*1024 else f"{size/1024/1024:.1f} MB"
items.append(f"📄 {item.name} ({size_str})")
if not items:
return "Desktop is empty."
return f"Desktop ({len(items)} items):\n" + "\n".join(items)
def clean_desktop() -> str:
"""
Moves all files on desktop into a 'Desktop Archive' folder
with today's date — fast cleanup without deleting anything.
"""
desktop = _get_desktop()
today = datetime.now().strftime("%Y-%m-%d")
archive_dir = desktop / f"Desktop Archive {today}"
archive_dir.mkdir(exist_ok=True)
moved = 0
for item in desktop.iterdir():
if item.is_dir() or item.name.startswith("."):
continue
if item.suffix.lower() == ".lnk":
continue
new_path = archive_dir / item.name
if not new_path.exists():
shutil.move(str(item), str(new_path))
moved += 1
return f"Desktop cleaned. {moved} files moved to '{archive_dir.name}'."
def get_desktop_stats() -> str:
"""Returns stats about the desktop."""
desktop = _get_desktop()
files = [i for i in desktop.iterdir() if i.is_file()]
folders = [i for i in desktop.iterdir() if i.is_dir()]
total_size = sum(f.stat().st_size for f in files)
size_str = f"{total_size/1024:.1f} KB" if total_size < 1024*1024 else f"{total_size/1024/1024:.1f} MB"
return (
f"Desktop stats:\n"
f" Files : {len(files)}\n"
f" Folders : {len(folders)}\n"
f" Total size: {size_str}"
)
def desktop_control(
parameters: dict,
response=None,
player=None,
session_memory=None
) -> str:
"""
Called from main.py.
parameters:
action : wallpaper | wallpaper_url | current_wallpaper |
organize | clean | list | stats |
task (AI-powered — anything else)
path : image path for 'wallpaper'
url : image URL for 'wallpaper_url'
mode : 'by_type' or 'by_date' for 'organize'
task : Natural language description for AI-powered actions.
Example: "arrange icons by size"
"show me what's on my desktop"
"move all screenshots to a folder"
"""
action = (parameters or {}).get("action", "").lower().strip()
task = (parameters or {}).get("task", "").strip()
result = "Unknown action."
try:
if action == "wallpaper":
path = parameters.get("path", "")
result = set_wallpaper(path) if path else "No image path provided."
elif action == "wallpaper_url":
url = parameters.get("url", "")
result = set_wallpaper_from_web(url) if url else "No URL provided."
elif action == "current_wallpaper":
result = get_current_wallpaper()
elif action == "organize":
mode = parameters.get("mode", "by_type")
result = organize_desktop(mode)
elif action == "clean":
result = clean_desktop()
elif action == "list":
result = list_desktop()
elif action == "stats":
result = get_desktop_stats()
elif action == "task" or task:
actual_task = task or parameters.get("description", "")
if not actual_task:
return "Please describe what you want to do on the desktop, sir."
print(f"[Desktop] 🤖 Asking Gemini: {actual_task}")
if player:
player.write_log("[Desktop] Generating action...")
code = _ask_gemini_for_desktop_action(actual_task)
if code == "UNSAFE":
result = "I cannot perform that desktop action safely, sir."
elif code.startswith("ERROR:"):
result = f"Could not generate action: {code}"
else:
print(f"[Desktop] ✅ Generated code:\n{code[:200]}")
result = _execute_generated_code(code)
else:
full_task = task or action
if full_task:
code = _ask_gemini_for_desktop_action(full_task)
result = _execute_generated_code(code) if code not in ("UNSAFE",) else "Cannot do that safely."
else:
result = "No action or task specified."
except Exception as e:
result = f"Desktop control error: {e}"
print(f"[Desktop] {result[:100]}")
if player:
player.write_log(f"[desktop] {result[:60]}")
return result
+597
View File
@@ -0,0 +1,597 @@
import subprocess
import sys
import json
import re
import time
from pathlib import Path
def get_base_dir():
if getattr(sys, "frozen", False):
return Path(sys.executable).parent
return Path(__file__).resolve().parent.parent
BASE_DIR = get_base_dir()
API_CONFIG_PATH = BASE_DIR / "config" / "api_keys.json"
PROJECTS_DIR = Path.home() / "Desktop" / "JarvisProjects"
MAX_FIX_ATTEMPTS = 5
MODEL_PLANNER = "gemini-2.5-flash"
MODEL_WRITER = "gemini-2.5-flash"
def _get_api_key() -> str:
with open(API_CONFIG_PATH, "r", encoding="utf-8") as f:
return json.load(f)["gemini_api_key"]
def _get_model(model_name: str):
import google.generativeai as genai
genai.configure(api_key=_get_api_key())
return genai.GenerativeModel(model_name)
def _strip_fences(text: str) -> str:
text = text.strip()
text = re.sub(r"^```[a-zA-Z]*\r?\n?", "", text)
text = re.sub(r"\r?\n?```\s*$", "", text)
return text.strip()
def _is_rate_limit(error: Exception) -> bool:
msg = str(error).lower()
return "429" in msg or "quota" in msg or "resource_exhausted" in msg
def _parse_traceback(output: str, project_files: list[str]) -> tuple[str | None, int | None]:
pattern = re.compile(r'File ["\']([^"\']+\.py)["\'],\s+line\s+(\d+)', re.IGNORECASE)
matches = pattern.findall(output)
for raw_path, line_str in reversed(matches):
raw_name = Path(raw_path).name
for pf in project_files:
if Path(pf).name == raw_name or pf == raw_path or raw_path.endswith(pf):
return pf, int(line_str)
return None, None
def _classify_error(output: str) -> str:
low = output.lower()
if any(x in low for x in ("no module named", "modulenotfounderror", "importerror")):
return "dependency_error"
if "syntaxerror" in low or "invalid syntax" in low:
return "syntax_error"
if "cannot import" in low or "importerror" in low:
return "import_error"
if any(x in low for x in (
"traceback", "exception", "error:", "nameerror", "typeerror",
"attributeerror", "valueerror", "keyerror", "indexerror",
"zerodivisionerror", "filenotfounderror", "permissionerror",
)):
return "runtime_error"
return "none"
def _has_error(output: str, run_command: str) -> bool:
low = output.lower()
if "timed out" in low:
return False
if not output.strip():
return False
error_type = _classify_error(output)
return error_type != "none"
class RateLimitError(Exception):
pass
def _plan_project(description: str, language: str) -> dict:
model = _get_model(MODEL_PLANNER)
prompt = f"""You are a senior software architect. Create a minimal, complete file plan for this project.
Language: {language}
Description: {description}
Return ONLY valid JSON — no markdown, no explanation:
{{
"project_name": "snake_case_name",
"entry_point": "main.py",
"files": [
{{
"path": "main.py",
"description": "Entry point — what it does and which modules it imports",
"imports": ["utils.helpers", "core.engine"]
}},
{{
"path": "utils/helpers.py",
"description": "Helper utilities — what functions it exposes",
"imports": []
}}
],
"run_command": "python main.py",
"dependencies": ["requests"]
}}
Critical rules:
1. List files in DEPENDENCY ORDER — files with no imports come first, entry point comes last.
2. The "imports" field must list every other project module this file imports (dot-notation, e.g. "utils.helpers").
3. Keep it minimal — only files truly needed.
4. Entry point must be in the files list.
5. Use relative paths only (e.g. "utils/helpers.py", not absolute paths).
6. Standard library modules (os, sys, json, etc.) do NOT go in "dependencies".
JSON:"""
try:
response = model.generate_content(prompt)
raw = _strip_fences(response.text)
return json.loads(raw)
except json.JSONDecodeError as e:
raise ValueError(f"Planner returned invalid JSON: {e}\nRaw: {response.text[:300]}")
except Exception as e:
if _is_rate_limit(e):
raise RateLimitError(str(e))
raise
def _write_file(
file_info: dict,
project_description: str,
all_files: list[dict],
language: str,
project_dir: Path,
already_written: dict[str, str],
) -> str:
model = _get_model(MODEL_WRITER)
file_path = file_info["path"]
file_desc = file_info.get("description", "")
file_imports = file_info.get("imports", [])
file_list = "\n".join(
f" [{i+1}] {f['path']}: {f.get('description', '')}"
for i, f in enumerate(all_files)
)
dependency_context = ""
for dep_dotted in file_imports:
dep_path = dep_dotted.replace(".", "/") + ".py"
if dep_path in already_written:
code_snippet = already_written[dep_path][:2000]
dependency_context += f"\n\n--- {dep_path} (you must import from this) ---\n{code_snippet}"
lang_rules = ""
if language.lower() == "python":
lang_rules = """
Python-specific rules:
- Use type hints for all function signatures.
- Add docstrings for all public functions and classes.
- Use if __name__ == "__main__": guard in the entry point.
- For relative imports within the project, use: from utils.helpers import foo (match the project structure exactly).
- Do NOT use implicit relative imports (from . import ...) unless it's a proper package with __init__.py.
- If this is a package subdirectory, create __init__.py files where needed."""
elif language.lower() in ("javascript", "typescript", "js", "ts"):
lang_rules = """
JS/TS-specific rules:
- Use ES modules (import/export), not CommonJS (require).
- Add JSDoc comments for all exported functions.
- Handle promise rejections with try/catch in async functions."""
prompt = f"""You are a senior {language} developer writing production-quality code for a real project.
Project goal: {project_description}
Complete project file structure (in dependency order):
{file_list}
{f"Dependencies this file must import from other project files:{dependency_context}" if dependency_context else ""}
Your task: Write the complete, working code for: {file_path}
Purpose of this file: {file_desc}
{f"This file imports from: {', '.join(file_imports)}" if file_imports else "This file has no project-internal imports."}
{lang_rules}
General rules:
- Output ONLY raw code. Absolutely no explanation, no markdown, no triple backticks.
- Write COMPLETE, RUNNABLE code — no placeholders, no "# TODO", no "pass" stubs.
- Every import must either be from the standard library, listed dependencies, or the project files shown above.
- Match import paths EXACTLY to the file paths in the project structure (e.g. if file is "utils/helpers.py", import as "from utils.helpers import ...").
- Use proper error handling (try/except) where I/O or network calls are made.
- The code must work correctly when the project entry point is run from the project root directory.
Code for {file_path}:"""
try:
response = model.generate_content(prompt)
code = _strip_fences(response.text)
full_path = project_dir / file_path
full_path.parent.mkdir(parents=True, exist_ok=True)
full_path.write_text(code, encoding="utf-8")
print(f"[DevAgent] ✅ Written: {file_path} ({len(code)} chars)")
return code
except Exception as e:
if _is_rate_limit(e):
raise RateLimitError(str(e))
raise
def _install_dependencies(dependencies: list[str], project_dir: Path) -> str:
if not dependencies:
return "No external dependencies."
to_install = []
for dep in dependencies:
pkg_name = re.split(r"[>=<!]", dep)[0].strip()
result = subprocess.run(
[sys.executable, "-m", "pip", "show", pkg_name],
capture_output=True, text=True
)
if result.returncode != 0:
to_install.append(dep)
else:
print(f"[DevAgent] ✓ Already installed: {pkg_name}")
if not to_install:
return f"All dependencies already installed: {', '.join(dependencies)}"
print(f"[DevAgent] 📦 Installing: {to_install}")
try:
result = subprocess.run(
[sys.executable, "-m", "pip", "install"] + to_install,
capture_output=True, text=True,
encoding="utf-8", errors="replace",
timeout=120, cwd=str(project_dir)
)
if result.returncode == 0:
return f"Installed: {', '.join(to_install)}"
return f"Install warning (non-fatal): {result.stderr[:200]}"
except subprocess.TimeoutExpired:
return "Dependency install timed out (non-fatal)."
except Exception as e:
return f"Install error (non-fatal): {e}"
def _open_vscode(project_dir: Path) -> bool:
vscode_candidates = [
"code",
rf"C:\Users\{Path.home().name}\AppData\Local\Programs\Microsoft VS Code\bin\code.cmd",
r"C:\Program Files\Microsoft VS Code\bin\code.cmd",
]
for cmd in vscode_candidates:
try:
subprocess.Popen(
[cmd, str(project_dir)],
shell=True,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL
)
time.sleep(1.5)
print(f"[DevAgent] 💻 VSCode opened: {project_dir}")
return True
except Exception:
continue
return False
def _run_project(run_command: str, project_dir: Path, timeout: int = 30) -> str:
print(f"[DevAgent] 🚀 Running: {run_command}")
try:
parts = run_command.split()
if parts[0].lower() == "python":
parts[0] = sys.executable
result = subprocess.run(
parts,
capture_output=True, text=True,
encoding="utf-8", errors="replace",
timeout=timeout,
cwd=str(project_dir)
)
stdout = result.stdout.strip()
stderr = result.stderr.strip()
combined_parts = []
if stdout:
combined_parts.append(f"STDOUT:\n{stdout}")
if stderr:
combined_parts.append(f"STDERR:\n{stderr}")
return "\n\n".join(combined_parts) if combined_parts else "Ran with no output."
except subprocess.TimeoutExpired:
return f"Timed out after {timeout}s — long-running app (server/GUI) is likely working."
except FileNotFoundError as e:
return f"Command not found: {e}"
except Exception as e:
return f"Run error: {e}"
def _try_auto_install(error_output: str, project_dir: Path) -> bool:
"""ModuleNotFoundError varsa eksik paketi otomatik kurmaya çalışır."""
pattern = re.compile(
r"No module named ['\"]([a-zA-Z0-9_\-\.]+)['\"]", re.IGNORECASE
)
match = pattern.search(error_output)
if not match:
return False
pkg = match.group(1).replace("_", "-").split(".")[0]
print(f"[DevAgent] 🔧 Auto-installing missing package: {pkg}")
try:
result = subprocess.run(
[sys.executable, "-m", "pip", "install", pkg],
capture_output=True, text=True,
encoding="utf-8", errors="replace",
timeout=60, cwd=str(project_dir)
)
return result.returncode == 0
except Exception:
return False
def _fix_files(
error_output: str,
project_description: str,
all_files: list[dict],
file_codes: dict[str, str],
language: str,
project_dir: Path,
entry_point: str,
) -> dict[str, str]:
model = _get_model(MODEL_PLANNER)
error_file, error_line = _parse_traceback(error_output, list(file_codes.keys()))
error_type = _classify_error(error_output)
files_to_fix: list[str] = []
if error_file:
files_to_fix.append(error_file)
if error_type == "import_error":
for fi in all_files:
if error_file.replace("/", ".").replace(".py", "") in fi.get("imports", []):
p = fi["path"]
if p not in files_to_fix:
files_to_fix.append(p)
else:
files_to_fix.append(entry_point)
updated_codes: dict[str, str] = {}
for fix_path in files_to_fix:
current_code = file_codes.get(fix_path, "")
other_ctx = ""
for fp, code in file_codes.items():
if fp != fix_path and code:
snippet = code[:1500] + ("..." if len(code) > 1500 else "")
other_ctx += f"\n--- {fp} ---\n{snippet}\n"
line_hint = f"\nError appears to be near line {error_line} in this file." if (
error_line and fix_path == error_file
) else ""
prompt = f"""You are an expert {language} debugger. Fix the broken file below.
Project goal: {project_description}
All project files:
{chr(10).join(f" - {f['path']}: {f.get('description', '')}" for f in all_files)}
Other files for context (read-only — fix only the target file):
{other_ctx[:3500]}
File to fix: {fix_path}{line_hint}
Error type: {error_type}
Error output:
{error_output[:2500]}
Current (broken) code:
{current_code}
Rules:
- Output ONLY the complete fixed code. No explanation, no markdown, no backticks.
- Fix ALL errors visible in the error output.
- Keep all existing correct logic — do not remove working features.
- Ensure import paths match the actual project file structure exactly.
- Do NOT introduce new bugs or remove error handling.
Fixed code for {fix_path}:"""
try:
response = model.generate_content(prompt)
fixed = _strip_fences(response.text)
full_path = project_dir / fix_path
full_path.parent.mkdir(parents=True, exist_ok=True)
full_path.write_text(fixed, encoding="utf-8")
updated_codes[fix_path] = fixed
print(f"[DevAgent] 🔧 Fixed: {fix_path}")
except Exception as e:
if _is_rate_limit(e):
raise RateLimitError(str(e))
print(f"[DevAgent] ⚠️ Could not fix {fix_path}: {e}")
return updated_codes
def _build_project(
description: str,
language: str,
project_name: str,
timeout: int,
speak=None,
player=None,
) -> str:
def log(msg: str):
print(f"[DevAgent] {msg}")
if player:
player.write_log(f"[DevAgent] {msg}")
log("Planning project structure...")
try:
plan = _plan_project(description, language)
except RateLimitError:
msg = "Rate limit reached, sir. Please try again in a moment."
if speak: speak(msg)
return msg
except ValueError as e:
msg = f"Planning failed: {e}"
if speak: speak(msg)
return msg
proj_name = project_name or plan.get("project_name", "jarvis_project")
proj_name = re.sub(r"[^\w\-]", "_", proj_name)
project_dir = PROJECTS_DIR / proj_name
project_dir.mkdir(parents=True, exist_ok=True)
files = plan.get("files", [])
entry_point = plan.get("entry_point", "main.py")
run_command = plan.get("run_command", f"python {entry_point}")
dependencies = plan.get("dependencies", [])
log(f"Project: {proj_name} | Files: {len(files)} | Entry: {entry_point}")
def _dep_sort_key(fi: dict) -> int:
return len(fi.get("imports", []))
sorted_files = sorted(files, key=_dep_sort_key)
file_codes: dict[str, str] = {}
for file_info in sorted_files:
file_path = file_info.get("path", "")
if not file_path:
continue
log(f"Writing {file_path}...")
for attempt in range(2):
try:
code = _write_file(
file_info=file_info,
project_description=description,
all_files=files,
language=language,
project_dir=project_dir,
already_written=file_codes,
)
file_codes[file_path] = code
time.sleep(0.4)
break
except RateLimitError:
if attempt == 0:
log("Rate limit — waiting 20s...")
time.sleep(20)
else:
log(f"Rate limit retry failed for {file_path}, skipping.")
except Exception as e:
log(f"Failed to write {file_path}: {e}")
break
if not file_codes:
msg = "I could not write any project files, sir."
if speak: speak(msg)
return msg
if dependencies:
install_result = _install_dependencies(dependencies, project_dir)
log(install_result)
_open_vscode(project_dir)
last_output = ""
auto_installs = 0
for attempt in range(1, MAX_FIX_ATTEMPTS + 1):
log(f"Running project (attempt {attempt}/{MAX_FIX_ATTEMPTS})...")
last_output = _run_project(run_command, project_dir, timeout)
log(f"Output preview: {last_output[:150]}")
if not _has_error(last_output, run_command):
msg = (
f"Project '{proj_name}' is working, sir. "
f"Built in {attempt} attempt{'s' if attempt > 1 else ''}. "
f"Saved to: {project_dir}"
)
if speak: speak(msg)
return f"{msg}\n\nOutput:\n{last_output}"
if attempt == MAX_FIX_ATTEMPTS:
break
error_type = _classify_error(last_output)
if error_type == "dependency_error" and auto_installs < 3:
installed = _try_auto_install(last_output, project_dir)
if installed:
auto_installs += 1
log("Missing dependency installed, retrying...")
time.sleep(1)
continue
log(f"Fixing errors (type: {error_type})...")
try:
updated = _fix_files(
error_output=last_output,
project_description=description,
all_files=files,
file_codes=file_codes,
language=language,
project_dir=project_dir,
entry_point=entry_point,
)
file_codes.update(updated)
time.sleep(1)
except RateLimitError:
msg = "Rate limit reached during fix. Project saved, check it manually in VSCode."
if speak: speak(msg)
return msg
except Exception as e:
log(f"Fix step failed: {e}")
msg = (
f"I couldn't fully fix '{proj_name}' after {MAX_FIX_ATTEMPTS} attempts, sir. "
f"Project is saved at {project_dir} — open it in VSCode and check manually."
)
if speak: speak(msg)
return f"{msg}\n\nLast error:\n{last_output[:600]}"
def dev_agent(
parameters: dict,
response=None,
player=None,
session_memory=None,
speak=None,
) -> str:
p = parameters or {}
description = p.get("description", "").strip()
language = p.get("language", "python").strip()
project_name = p.get("project_name", "").strip()
timeout = int(p.get("timeout", 30))
if not description:
return "Please describe the project you want me to build, sir."
return _build_project(
description = description,
language = language,
project_name = project_name,
timeout = timeout,
speak = speak,
player = player,
)
+482
View File
@@ -0,0 +1,482 @@
# actions/file_controller.py
# File management — create, delete, move, rename, list, find, organize
import shutil
from pathlib import Path
from datetime import datetime
import send2trash
def _get_desktop() -> Path:
"""Returns desktop path — works on Windows, Mac, Linux."""
return Path.home() / "Desktop"
def _get_downloads() -> Path:
return Path.home() / "Downloads"
def _resolve_path(raw: str) -> Path:
"""
Resolves a path from user input.
Supports shortcuts: 'desktop', 'downloads', 'documents', 'home'
"""
shortcuts = {
"desktop": Path.home() / "Desktop",
"downloads": Path.home() / "Downloads",
"documents": Path.home() / "Documents",
"pictures": Path.home() / "Pictures",
"music": Path.home() / "Music",
"videos": Path.home() / "Videos",
"home": Path.home(),
}
lower = raw.strip().lower()
if lower in shortcuts:
return shortcuts[lower]
return Path(raw).expanduser()
def _format_size(bytes_size: int) -> str:
"""Converts bytes to human readable format."""
for unit in ["B", "KB", "MB", "GB", "TB"]:
if bytes_size < 1024:
return f"{bytes_size:.1f} {unit}"
bytes_size /= 1024
return f"{bytes_size:.1f} TB"
def list_files(path: str = "desktop", show_hidden: bool = False) -> str:
"""Lists files and folders in a directory."""
try:
target = _resolve_path(path)
if not target.exists():
return f"Path not found: {target}"
if not target.is_dir():
return f"Not a directory: {target}"
items = []
for item in sorted(target.iterdir()):
if not show_hidden and item.name.startswith("."):
continue
if item.is_dir():
items.append(f"📁 {item.name}/")
else:
size = _format_size(item.stat().st_size)
items.append(f"📄 {item.name} ({size})")
if not items:
return f"Directory is empty: {target}"
return f"Contents of {target.name}/ ({len(items)} items):\n" + "\n".join(items)
except PermissionError:
return f"Permission denied: {path}"
except Exception as e:
return f"Error listing files: {e}"
def create_file(path: str, content: str = "") -> str:
"""Creates a new file with optional content."""
try:
target = Path(path).expanduser()
target.parent.mkdir(parents=True, exist_ok=True)
target.write_text(content, encoding="utf-8")
return f"File created: {target.name}"
except Exception as e:
return f"Could not create file: {e}"
def create_folder(path: str) -> str:
"""Creates a new folder (and parent folders if needed)."""
try:
target = Path(path).expanduser()
target.mkdir(parents=True, exist_ok=True)
return f"Folder created: {target}"
except Exception as e:
return f"Could not create folder: {e}"
def delete_file(path: str, confirm: bool = True) -> str:
"""
Deletes a file or folder.
Moves to Recycle Bin on Windows if possible, otherwise permanent delete.
"""
try:
target = Path(path).expanduser()
if not target.exists():
return f"Not found: {path}"
try:
send2trash.send2trash(str(target))
return f"Moved to Recycle Bin: {target.name}"
except ImportError:
pass
# Fallback: permanent delete
if target.is_dir():
shutil.rmtree(target)
return f"Folder deleted permanently: {target.name}"
else:
target.unlink()
return f"File deleted permanently: {target.name}"
except PermissionError:
return f"Permission denied: {path}"
except Exception as e:
return f"Could not delete: {e}"
def move_file(source: str, destination: str) -> str:
"""Moves a file or folder to a new location."""
try:
src = Path(source).expanduser()
dst = _resolve_path(destination)
if not src.exists():
return f"Source not found: {source}"
if dst.is_dir():
dst = dst / src.name
dst.parent.mkdir(parents=True, exist_ok=True)
shutil.move(str(src), str(dst))
return f"Moved: {src.name} → {dst.parent.name}/"
except Exception as e:
return f"Could not move: {e}"
def copy_file(source: str, destination: str) -> str:
"""Copies a file or folder."""
try:
src = Path(source).expanduser()
dst = _resolve_path(destination)
if not src.exists():
return f"Source not found: {source}"
if dst.is_dir():
dst = dst / src.name
dst.parent.mkdir(parents=True, exist_ok=True)
if src.is_dir():
shutil.copytree(str(src), str(dst))
else:
shutil.copy2(str(src), str(dst))
return f"Copied: {src.name} → {dst.parent.name}/"
except Exception as e:
return f"Could not copy: {e}"
def rename_file(path: str, new_name: str) -> str:
"""Renames a file or folder."""
try:
target = Path(path).expanduser()
new_path = target.parent / new_name
if not target.exists():
return f"Not found: {path}"
if new_path.exists():
return f"A file named '{new_name}' already exists."
target.rename(new_path)
return f"Renamed: {target.name} → {new_name}"
except Exception as e:
return f"Could not rename: {e}"
def read_file(path: str, max_chars: int = 3000) -> str:
"""Reads and returns the content of a text file."""
try:
target = Path(path).expanduser()
if not target.exists():
return f"File not found: {path}"
if not target.is_file():
return f"Not a file: {path}"
content = target.read_text(encoding="utf-8", errors="ignore")
if len(content) > max_chars:
content = content[:max_chars] + f"\n\n... (truncated, {len(content)} total chars)"
return content
except Exception as e:
return f"Could not read file: {e}"
def write_file(path: str, content: str, append: bool = False) -> str:
"""Writes or appends content to a file."""
try:
target = Path(path).expanduser()
target.parent.mkdir(parents=True, exist_ok=True)
mode = "a" if append else "w"
with open(target, mode, encoding="utf-8") as f:
f.write(content)
action = "Appended to" if append else "Written to"
return f"{action}: {target.name}"
except Exception as e:
return f"Could not write file: {e}"
def find_files(name: str = "", extension: str = "", path: str = "home",
max_results: int = 20) -> str:
"""
Searches for files by name or extension.
Example: find_files(extension=".pdf", path="documents")
"""
try:
search_path = _resolve_path(path)
if not search_path.exists():
return f"Search path not found: {path}"
results = []
pattern = f"*{extension}" if extension else "*"
for item in search_path.rglob(pattern):
if item.is_file():
if name and name.lower() not in item.name.lower():
continue
size = _format_size(item.stat().st_size)
results.append(f"📄 {item.name} ({size}) — {item.parent}")
if len(results) >= max_results:
break
if not results:
query = name or extension or "files"
return f"No {query} found in {search_path.name}/"
return f"Found {len(results)} file(s):\n" + "\n".join(results)
except Exception as e:
return f"Search error: {e}"
def get_largest_files(path: str = "home", count: int = 10) -> str:
"""Returns the largest files in a directory."""
try:
search_path = _resolve_path(path)
if not search_path.exists():
return f"Path not found: {path}"
files = []
for item in search_path.rglob("*"):
if item.is_file():
try:
files.append((item.stat().st_size, item))
except Exception:
continue
files.sort(reverse=True)
top = files[:count]
if not top:
return "No files found."
lines = [f"Top {len(top)} largest files in {search_path.name}/:\n"]
for size, f in top:
lines.append(f" {_format_size(size):>10} {f.name} ({f.parent})")
return "\n".join(lines)
except Exception as e:
return f"Error: {e}"
def get_disk_usage(path: str = "home") -> str:
"""Returns disk usage information."""
try:
target = _resolve_path(path)
usage = shutil.disk_usage(target)
total = _format_size(usage.total)
used = _format_size(usage.used)
free = _format_size(usage.free)
pct = usage.used / usage.total * 100
return (
f"Disk usage for {target}:\n"
f" Total : {total}\n"
f" Used : {used} ({pct:.1f}%)\n"
f" Free : {free}"
)
except Exception as e:
return f"Could not get disk usage: {e}"
def organize_desktop() -> str:
"""
Organizes the desktop by grouping files into folders by type.
Creates folders: Images, Documents, Videos, Music, Archives, Others
"""
try:
desktop = _get_desktop()
type_map = {
"Images": [".jpg", ".jpeg", ".png", ".gif", ".bmp", ".webp", ".svg", ".ico"],
"Documents": [".pdf", ".doc", ".docx", ".txt", ".xls", ".xlsx", ".ppt", ".pptx", ".csv"],
"Videos": [".mp4", ".avi", ".mkv", ".mov", ".wmv", ".flv", ".webm"],
"Music": [".mp3", ".wav", ".flac", ".aac", ".ogg", ".wma"],
"Archives": [".zip", ".rar", ".7z", ".tar", ".gz"],
"Code": [".py", ".js", ".html", ".css", ".json", ".xml", ".ts", ".cpp", ".java"],
}
moved = []
skipped = []
for item in desktop.iterdir():
if item.is_dir() or item.name.startswith("."):
continue
ext = item.suffix.lower()
target_dir = None
for folder, extensions in type_map.items():
if ext in extensions:
target_dir = desktop / folder
break
if target_dir is None:
target_dir = desktop / "Others"
target_dir.mkdir(exist_ok=True)
new_path = target_dir / item.name
if new_path.exists():
skipped.append(item.name)
continue
shutil.move(str(item), str(new_path))
moved.append(f"{item.name} → {target_dir.name}/")
result = f"Desktop organized. {len(moved)} files moved."
if moved:
result += "\n" + "\n".join(moved[:10])
if len(moved) > 10:
result += f"\n... and {len(moved)-10} more."
if skipped:
result += f"\n{len(skipped)} files skipped (already exist)."
return result
except Exception as e:
return f"Could not organize desktop: {e}"
def get_file_info(path: str) -> str:
"""Returns detailed information about a file."""
try:
target = Path(path).expanduser()
if not target.exists():
return f"Not found: {path}"
stat = target.stat()
info = {
"Name": target.name,
"Type": "Folder" if target.is_dir() else "File",
"Size": _format_size(stat.st_size),
"Location": str(target.parent),
"Created": datetime.fromtimestamp(stat.st_ctime).strftime("%Y-%m-%d %H:%M"),
"Modified": datetime.fromtimestamp(stat.st_mtime).strftime("%Y-%m-%d %H:%M"),
"Extension": target.suffix or "None",
}
return "\n".join(f" {k}: {v}" for k, v in info.items())
except Exception as e:
return f"Could not get file info: {e}"
def file_controller(
parameters: dict,
response=None,
player=None,
session_memory=None
) -> str:
action = (parameters or {}).get("action", "").lower().strip()
path = (parameters or {}).get("path", "desktop")
name = (parameters or {}).get("name", "")
content = (parameters or {}).get("content", "")
def _full_path(p: str, n: str) -> str:
base = _resolve_path(p)
if n:
return str(base / n)
return str(base)
result = "Unknown action."
try:
if action == "list":
result = list_files(path)
elif action == "create_file":
full = _full_path(path, name)
result = create_file(full, content=content)
elif action == "create_folder":
full = _full_path(path, name)
result = create_folder(full)
elif action == "delete":
full = _full_path(path, name)
result = delete_file(full)
elif action == "move":
full = _full_path(path, name)
result = move_file(full, parameters.get("destination", ""))
elif action == "copy":
full = _full_path(path, name)
result = copy_file(full, parameters.get("destination", ""))
elif action == "rename":
full = _full_path(path, name)
result = rename_file(full, parameters.get("new_name", ""))
elif action == "read":
full = _full_path(path, name)
result = read_file(full)
elif action == "write":
full = _full_path(path, name)
result = write_file(
full,
content=content,
append=parameters.get("append", False)
)
elif action == "find":
result = find_files(
name=name or parameters.get("name", ""),
extension=parameters.get("extension", ""),
path=path,
max_results=parameters.get("max_results", 20)
)
elif action == "largest":
result = get_largest_files(
path=path,
count=parameters.get("count", 10)
)
elif action == "disk_usage":
result = get_disk_usage(path)
elif action == "organize_desktop":
result = organize_desktop()
elif action == "info":
full = _full_path(path, name)
result = get_file_info(full)
else:
result = f"Unknown action: '{action}'"
except Exception as e:
result = f"File controller error: {e}"
if player:
player.write_log(f"[file] {result[:60]}")
return result
+419
View File
@@ -0,0 +1,419 @@
# actions/flight_finder.py
# MARK XXV — Flight Finder
#
# Searches for flights using Google Flights via browser_control.
# Results are spoken by JARVIS. Optionally saved to Notepad or opened in browser.
#
# Flow:
# 1. Parse origin, destination, date, passengers from parameters
# 2. Open Google Flights via browser_control
# 3. Fill in search fields
# 4. Scrape results via get_text
# 5. Parse with Gemini → structured flight data
# 6. Speak top results
# 7. Optionally save to Notepad or keep browser open
#
# Cross-platform: Windows, macOS, Linux
# No API key required — Google Flights is free to access
import json
import re
import sys
import subprocess
import platform
from datetime import datetime, timedelta
from pathlib import Path
def get_base_dir() -> Path:
if getattr(sys, "frozen", False):
return Path(sys.executable).parent
return Path(__file__).resolve().parent.parent
BASE_DIR = get_base_dir()
API_CONFIG_PATH = BASE_DIR / "config" / "api_keys.json"
def _get_api_key() -> str:
with open(API_CONFIG_PATH, "r", encoding="utf-8") as f:
return json.load(f)["gemini_api_key"]
def _parse_date(raw: str) -> str:
"""
Converts natural language date to YYYY-MM-DD.
Handles: '15 Mart', 'March 15', '2025-03-15', 'next friday', etc.
Falls back to Gemini for ambiguous inputs.
"""
raw = raw.strip()
if re.match(r"\d{4}-\d{2}-\d{2}", raw):
return raw
for fmt in ("%d/%m/%Y", "%m/%d/%Y", "%d.%m.%Y", "%d-%m-%Y"):
try:
return datetime.strptime(raw, fmt).strftime("%Y-%m-%d")
except ValueError:
pass
today = datetime.now()
lower = raw.lower()
relative_map = {
"today": today,
"tomorrow": today + timedelta(days=1),
"bugün": today,
"yarın": today + timedelta(days=1),
}
for key, val in relative_map.items():
if key in lower:
return val.strftime("%Y-%m-%d")
try:
import google.generativeai as genai
genai.configure(api_key=_get_api_key())
model = genai.GenerativeModel("gemini-2.5-flash-lite")
today_str = today.strftime("%Y-%m-%d")
response = model.generate_content(
f"Today is {today_str}. Convert this date to YYYY-MM-DD format: '{raw}'. "
f"Return ONLY the date string, nothing else."
)
result = response.text.strip()
if re.match(r"\d{4}-\d{2}-\d{2}", result):
return result
except Exception:
pass
month_map = {
"january": 1, "february": 2, "march": 3, "april": 4,
"may": 5, "june": 6, "july": 7, "august": 8,
"september": 9,"october": 10, "november": 11, "december": 12,
"january": 1, "ocak": 1, "şubat": 2, "mart": 3,
"nisan": 4, "mayıs": 5, "haziran": 6, "temmuz": 7,
"ağustos": 8, "eylül": 9, "ekim": 10, "kasım": 11,
"aralık": 12,
}
for month_name, month_num in month_map.items():
if month_name in lower:
day_match = re.search(r"\d{1,2}", raw)
if day_match:
day = int(day_match.group())
year = today.year if month_num >= today.month else today.year + 1
return f"{year}-{month_num:02d}-{day:02d}"
return today.strftime("%Y-%m-%d")
def _build_google_flights_url(
origin: str,
destination: str,
date: str,
return_date: str | None = None,
passengers: int = 1,
cabin: str = "economy",
) -> str:
"""
Builds a Google Flights URL with pre-filled search parameters.
Uses the direct search URL format.
"""
cabin_map = {
"economy": "1",
"premium": "2",
"business": "3",
"first": "4",
}
cabin_code = cabin_map.get(cabin.lower(), "1")
base = "https://www.google.com/travel/flights"
if return_date:
url = (
f"{base}?q=Flights+from+{origin}+to+{destination}"
f"+on+{date}+returning+{return_date}"
f"&curr=TRY"
)
else:
url = (
f"{base}?q=Flights+from+{origin}+to+{destination}+on+{date}"
f"&curr=TRY"
)
return url
def _search_flights_browser(
origin: str,
destination: str,
date: str,
return_date: str | None,
passengers: int,
cabin: str,
) -> tuple[str, str]:
"""
Opens Google Flights in browser, waits for results, scrapes text.
Returns (raw_text, page_url).
"""
from actions.browser_control import browser_control
import time
url = _build_google_flights_url(
origin, destination, date, return_date, passengers, cabin
)
print(f"[FlightFinder] 🌐 Opening: {url}")
browser_control({"action": "go_to", "url": url})
time.sleep(5)
result = browser_control({"action": "get_text"})
return result or "", url
def _parse_flights_with_gemini(
raw_text: str,
origin: str,
destination: str,
date: str,
) -> list[dict]:
"""
Sends raw page text to Gemini and extracts structured flight data.
Returns list of flight dicts.
"""
import google.generativeai as genai
genai.configure(api_key=_get_api_key())
model = genai.GenerativeModel(
model_name="gemini-2.5-flash",
system_instruction=(
"You are a flight data extraction expert. "
"Extract flight information from raw webpage text. "
"Return ONLY valid JSON array. No explanation, no markdown."
)
)
truncated = raw_text[:12000]
prompt = (
f"Extract flight options from {origin} to {destination} on {date} "
f"from this Google Flights page text:\n\n{truncated}\n\n"
f"Return a JSON array of up to 5 flights:\n"
f'[{{"airline": "...", "departure": "HH:MM", "arrival": "HH:MM", '
f'"duration": "Xh Ym", "stops": 0, "price": "...", "currency": "..."}}]\n'
f"If no flights found, return: []"
)
try:
response = model.generate_content(prompt)
text = response.text.strip()
text = re.sub(r"```(?:json)?", "", text).strip().rstrip("`").strip()
flights = json.loads(text)
return flights if isinstance(flights, list) else []
except Exception as e:
print(f"[FlightFinder] ⚠️ Parse failed: {e}")
return []
def _format_spoken(
flights: list[dict],
origin: str,
destination: str,
date: str,
) -> str:
"""Formats flights for spoken output — concise and natural."""
if not flights:
return (
f"I couldn't find any flights from {origin} to {destination} "
f"on {date}, sir. The page may not have loaded correctly."
)
lines = [f"Here are the flights from {origin} to {destination} on {date}, sir."]
for i, f in enumerate(flights[:5], 1):
airline = f.get("airline", "Unknown airline")
departure = f.get("departure", "--:--")
arrival = f.get("arrival", "--:--")
duration = f.get("duration", "")
stops = f.get("stops", 0)
price = f.get("price", "")
currency = f.get("currency", "")
stop_str = "non-stop" if stops == 0 else f"{stops} stop{'s' if stops > 1 else ''}"
price_str = f"{price} {currency}".strip() if price else "price unavailable"
dur_str = f", {duration}" if duration else ""
lines.append(
f"Option {i}: {airline}, departing {departure}, arriving {arrival}"
f"{dur_str}, {stop_str}, {price_str}."
)
cheapest = min(
(f for f in flights if f.get("price")),
key=lambda x: re.sub(r"[^\d]", "", str(x.get("price", "99999"))) or "99999",
default=None,
)
if cheapest:
lines.append(
f"The cheapest option is {cheapest.get('airline')} "
f"at {cheapest.get('price')} {cheapest.get('currency', '')}."
)
return " ".join(lines)
def _format_notepad(
flights: list[dict],
origin: str,
destination: str,
date: str,
return_date: str | None,
page_url: str,
) -> str:
"""Formats flights for Notepad — detailed and readable."""
from datetime import datetime as dt
lines = [
"JARVIS — Flight Search Results",
"─" * 50,
f"Route : {origin} → {destination}",
f"Date : {date}",
]
if return_date:
lines.append(f"Return : {return_date}")
lines += [
f"Searched : {dt.now().strftime('%Y-%m-%d %H:%M')}",
f"Source : {page_url}",
"─" * 50,
"",
]
if not flights:
lines.append("No flights found.")
else:
for i, f in enumerate(flights, 1):
stops = f.get("stops", 0)
stop_str = "Non-stop" if stops == 0 else f"{stops} stop(s)"
lines += [
f"Flight {i}:",
f" Airline : {f.get('airline', 'N/A')}",
f" Departure : {f.get('departure', 'N/A')}",
f" Arrival : {f.get('arrival', 'N/A')}",
f" Duration : {f.get('duration', 'N/A')}",
f" Stops : {stop_str}",
f" Price : {f.get('price', 'N/A')} {f.get('currency', '')}",
"",
]
return "\n".join(lines)
def _save_to_notepad(content: str, origin: str, destination: str) -> str:
"""Saves flight results to Desktop and opens in default text editor."""
from datetime import datetime
ts = datetime.now().strftime("%Y%m%d_%H%M%S")
filename = f"flights_{origin}_{destination}_{ts}.txt".replace(" ", "_")
desktop = Path.home() / "Desktop"
desktop.mkdir(parents=True, exist_ok=True)
filepath = desktop / filename
filepath.write_text(content, encoding="utf-8")
print(f"[FlightFinder] 💾 Saved: {filepath}")
system = platform.system()
open_fn = {
"Windows": lambda p: subprocess.Popen(["notepad.exe", str(p)]),
"Darwin": lambda p: subprocess.Popen(["open", "-t", str(p)]),
"Linux": lambda p: subprocess.Popen(["xdg-open", str(p)]),
}
opener = open_fn.get(system)
if opener:
opener(filepath)
return str(filepath)
def flight_finder(
parameters: dict,
response=None,
player=None,
session_memory=None,
speak=None,
) -> str:
"""
Flight Finder — searches Google Flights and speaks results.
Parameters:
origin (str, required) — departure city or airport (e.g. "Istanbul", "IST")
destination (str, required) — arrival city or airport (e.g. "London", "LHR")
date (str, required) — departure date (any format: "15 Mart", "March 15", "2025-03-15")
return_date (str, optional) — return date for round trips
passengers (int, optional) — number of passengers (default: 1)
cabin (str, optional) — economy | premium | business | first (default: economy)
save (bool, optional) — save results to Notepad (default: False)
show_browser (bool, optional) — keep browser open after search (default: True)
Examples:
flight_finder({"origin": "Istanbul", "destination": "London", "date": "15 Mart"})
flight_finder({"origin": "IST", "destination": "JFK", "date": "2025-04-01",
"return_date": "2025-04-15", "cabin": "business", "save": True})
"""
params = parameters or {}
origin = params.get("origin", "").strip()
destination = params.get("destination", "").strip()
date_raw = params.get("date", "").strip()
return_raw = params.get("return_date", "").strip()
passengers = int(params.get("passengers", 1))
cabin = params.get("cabin", "economy").strip()
save = params.get("save", False)
if not origin or not destination:
return "Please provide both origin and destination, sir."
if not date_raw:
return "Please provide a departure date, sir."
date = _parse_date(date_raw)
return_date = _parse_date(return_raw) if return_raw else None
if player:
player.write_log(f"[FlightFinder] {origin} → {destination} on {date}")
if speak:
speak(f"Searching flights from {origin} to {destination} on {date}, sir.")
print(f"[FlightFinder] ▶️ {origin} → {destination} | {date} | {cabin} | {passengers} pax")
try:
raw_text, page_url = _search_flights_browser(
origin, destination, date, return_date, passengers, cabin
)
if not raw_text:
return "Could not retrieve flight data, sir. The page may not have loaded."
if speak:
speak("Analysing the results now.")
flights = _parse_flights_with_gemini(raw_text, origin, destination, date)
spoken = _format_spoken(flights, origin, destination, date)
if speak:
speak(spoken)
result = spoken
if save and flights:
notepad_content = _format_notepad(
flights, origin, destination, date, return_date, page_url
)
saved_path = _save_to_notepad(notepad_content, origin, destination)
result += f" Results saved to Desktop: {saved_path}"
return result
except Exception as e:
print(f"[FlightFinder] ❌ Error: {e}")
return f"Flight search failed, sir: {e}"
+817
View File
@@ -0,0 +1,817 @@
import os
import re
import sys
import json
import time
import subprocess
import threading
import winreg
from pathlib import Path
from datetime import datetime
def _find_steam_path() -> Path | None:
registry_keys = [
(winreg.HKEY_LOCAL_MACHINE, r"SOFTWARE\WOW6432Node\Valve\Steam"),
(winreg.HKEY_LOCAL_MACHINE, r"SOFTWARE\Valve\Steam"),
(winreg.HKEY_CURRENT_USER, r"SOFTWARE\Valve\Steam"),
]
for hive, key_path in registry_keys:
try:
key = winreg.OpenKey(hive, key_path)
val, _ = winreg.QueryValueEx(key, "InstallPath")
winreg.CloseKey(key)
p = Path(val)
if p.exists() and (p / "steam.exe").exists():
return p
except Exception:
continue
for p in [
Path(os.environ.get("ProgramFiles(x86)", "")) / "Steam",
Path(os.environ.get("ProgramFiles", "")) / "Steam",
Path("C:/Steam"), Path("D:/Steam"), Path("E:/Steam"), Path("F:/Steam"),
]:
if p.exists() and (p / "steam.exe").exists():
return p
return None
def _find_epic_path() -> Path | None:
registry_keys = [
(winreg.HKEY_LOCAL_MACHINE, r"SOFTWARE\WOW6432Node\EpicGames\EpicGamesLauncher"),
(winreg.HKEY_LOCAL_MACHINE, r"SOFTWARE\EpicGames\EpicGamesLauncher"),
(winreg.HKEY_CURRENT_USER, r"SOFTWARE\EpicGames\EpicGamesLauncher"),
]
for hive, key_path in registry_keys:
try:
key = winreg.OpenKey(hive, key_path)
val, _ = winreg.QueryValueEx(key, "AppDataPath")
winreg.CloseKey(key)
exe = Path(val) / "Binaries" / "Win64" / "EpicGamesLauncher.exe"
if exe.exists():
return exe.parent
except Exception:
continue
for p in [
Path(os.environ.get("ProgramFiles(x86)", "")) / "Epic Games" / "Launcher" / "Portal" / "Binaries" / "Win64",
Path(os.environ.get("ProgramFiles", "")) / "Epic Games" / "Launcher" / "Portal" / "Binaries" / "Win64",
Path(os.environ.get("LOCALAPPDATA", "")) / "EpicGamesLauncher" / "Portal" / "Binaries" / "Win64",
]:
if p.exists() and (p / "EpicGamesLauncher.exe").exists():
return p
return None
def _get_steam_libraries(steam_path: Path) -> list[Path]:
libraries = [steam_path / "steamapps"]
vdf_path = steam_path / "steamapps" / "libraryfolders.vdf"
if not vdf_path.exists():
return libraries
try:
content = vdf_path.read_text(encoding="utf-8", errors="ignore")
for raw_path in re.findall(r'"path"\s+"([^"]+)"', content):
lib = Path(raw_path.replace("\\\\", "/")) / "steamapps"
if lib.exists() and lib not in libraries:
libraries.append(lib)
except Exception:
pass
return libraries
def _get_steam_games(steam_path: Path) -> list[dict]:
games = []
for lib in _get_steam_libraries(steam_path):
for acf in lib.glob("appmanifest_*.acf"):
try:
content = acf.read_text(encoding="utf-8", errors="ignore")
app_id = re.search(r'"appid"\s+"(\d+)"', content)
name = re.search(r'"name"\s+"([^"]+)"', content)
state = re.search(r'"StateFlags"\s+"(\d+)"', content)
size = re.search(r'"SizeOnDisk"\s+"(\d+)"', content)
if app_id and name:
games.append({
"id": app_id.group(1),
"name": name.group(1),
"state": int(state.group(1)) if state else 0,
"size": int(size.group(1)) if size else 0,
"lib": str(lib),
})
except Exception:
continue
return games
def _is_steam_running() -> bool:
try:
out = subprocess.run(["tasklist", "/FI", "IMAGENAME eq steam.exe"],
capture_output=True, text=True).stdout
return "steam.exe" in out.lower()
except Exception:
return False
def _get_steam_window_rect() -> tuple[int, int, int, int] | None:
try:
import pygetwindow as gw
for w in gw.getAllWindows():
if "steam" in w.title.lower() and w.width > 200 and w.visible:
return w.left, w.top, w.width, w.height
except Exception:
pass
return None
def _click_first_profile_by_screenshot() -> bool:
try:
import pyautogui
import numpy as np
time.sleep(1.5)
win = _get_steam_window_rect()
if not win:
print("[GameUpdater] ⚠️ Steam window not found")
return False
wx, wy, ww, wh = win
screenshot = pyautogui.screenshot(region=(wx, wy, ww, wh))
img = np.array(screenshot)
h, w = img.shape[:2]
search_y1 = h // 3
search_y2 = h * 3 // 4
search_x1 = w // 5
search_x2 = w * 4 // 5
region = img[search_y1:search_y2, search_x1:search_x2]
r = region[:, :, 0].astype(int)
g = region[:, :, 1].astype(int)
b = region[:, :, 2].astype(int)
max_c = np.maximum(np.maximum(r, g), b)
min_c = np.minimum(np.minimum(r, g), b)
sat = max_c - min_c
colorful = (max_c > 60) & (sat > 40)
if not colorful.any():
print("[GameUpdater] ⚠️ No colorful avatar region found — clicking estimated center-left")
pyautogui.click(wx + ww // 2 - ww // 6, wy + wh // 2)
return True
cols = np.where(colorful.any(axis=0))[0]
rows = np.where(colorful.any(axis=1))[0]
if len(cols) == 0 or len(rows) == 0:
return False
avatar_w = min(90, (region.shape[1]) // 4)
first_col = int(cols[0])
block_cols = cols[cols < first_col + avatar_w]
center_x_local = int(block_cols.mean())
center_y_local = int(rows.mean())
abs_x = wx + search_x1 + center_x_local
abs_y = wy + search_y1 + center_y_local
print(f"[GameUpdater] 🎯 First profile avatar at ({abs_x}, {abs_y}) — clicking")
pyautogui.click(abs_x, abs_y)
return True
except ImportError as e:
print(f"[GameUpdater] ⚠️ Missing library: {e}")
return False
except Exception as e:
print(f"[GameUpdater] ⚠️ Screenshot profile detection failed: {e}")
return False
def _handle_steam_profile_selection() -> bool:
print("[GameUpdater] 🔍 Checking for 'Who's playing?' dialog...")
win = _get_steam_window_rect()
if not win:
return False
wx, wy, ww, wh = win
try:
import pyautogui
import numpy as np
screenshot = pyautogui.screenshot(region=(wx, wy, ww, wh))
img = np.array(screenshot)
is_small_window = ww < 900 and wh < 700
top_region = img[:wh // 3, :, :]
white_pixels = int(np.sum(
(top_region[:, :, 0] > 200) &
(top_region[:, :, 1] > 200) &
(top_region[:, :, 2] > 200)
))
has_white_text = white_pixels > 100
if not is_small_window and not has_white_text:
print("[GameUpdater] ℹ️ No profile dialog detected — Steam already logged in")
return False
except ImportError:
pass
except Exception:
pass
print("[GameUpdater] 👤 Profile selection detected — clicking first profile")
return _click_first_profile_by_screenshot()
def _ensure_steam_running(steam_path: Path) -> bool:
if _is_steam_running():
return True
steam_exe = steam_path / "steam.exe"
if not steam_exe.exists():
print("[GameUpdater] ❌ steam.exe not found")
return False
print("[GameUpdater] 🚀 Starting Steam...")
subprocess.Popen([str(steam_exe)])
for _ in range(20):
time.sleep(1)
if _is_steam_running():
print("[GameUpdater] ✅ Steam running")
time.sleep(4)
_handle_steam_profile_selection()
time.sleep(2)
return True
print("[GameUpdater] ⚠️ Steam did not start in time")
return False
def _update_steam_games(steam_path: Path, game_name: str = None) -> str:
if not _ensure_steam_running(steam_path):
return "Could not start Steam."
steam_exe = steam_path / "steam.exe"
games = _get_steam_games(steam_path)
if not games:
return "No Steam games found."
if game_name:
name_lower = game_name.lower()
matched = [g for g in games if name_lower in g["name"].lower()]
if not matched:
available = ", ".join(g["name"] for g in games[:5])
return f"Game '{game_name}' not found. Installed: {available}..."
targets = matched
else:
targets = games
already_updated, already_running, update_started, errors = [], [], [], []
for game in targets:
state = game["state"]
name = game["name"]
if state == 4:
already_updated.append(name)
elif state == 1026:
already_running.append(name)
else:
try:
subprocess.Popen([str(steam_exe), f"steam://update/{game['id']}"])
update_started.append(name)
time.sleep(0.3)
except Exception as e:
errors.append(f"{name}: {e}")
parts = []
if update_started:
names = ", ".join(update_started[:3])
suffix = f" and {len(update_started) - 3} more" if len(update_started) > 3 else ""
parts.append(f"Update started for: {names}{suffix}.")
if already_running:
parts.append(f"Already updating: {', '.join(already_running)}.")
if already_updated:
parts.append(
f"{already_updated[0]} is already up to date."
if game_name else
f"{len(already_updated)} game(s) already up to date."
)
if errors:
parts.append(f"Errors: {'; '.join(errors)}.")
return " ".join(parts) if parts else "No games to update."
_KNOWN_APPIDS: dict[str, tuple[str, str]] = {
"pubg": ("578080", "PUBG: Battlegrounds"),
"pubg battlegrounds": ("578080", "PUBG: Battlegrounds"),
"pubg: battlegrounds": ("578080", "PUBG: Battlegrounds"),
"battlegrounds": ("578080", "PUBG: Battlegrounds"),
"gta5": ("271590", "Grand Theft Auto V"),
"gta v": ("271590", "Grand Theft Auto V"),
"grand theft auto v": ("271590", "Grand Theft Auto V"),
"cs2": ("730", "Counter-Strike 2"),
"csgo": ("730", "Counter-Strike 2"),
"counter-strike 2": ("730", "Counter-Strike 2"),
"counter strike 2": ("730", "Counter-Strike 2"),
"dota2": ("570", "Dota 2"),
"dota 2": ("570", "Dota 2"),
"rust": ("252490", "Rust"),
"valheim": ("892970", "Valheim"),
"cyberpunk": ("1091500", "Cyberpunk 2077"),
"cyberpunk 2077": ("1091500", "Cyberpunk 2077"),
"elden ring": ("1245620", "ELDEN RING"),
"minecraft": ("1672970", "Minecraft Launcher"),
"apex legends": ("1172470", "Apex Legends"),
"apex": ("1172470", "Apex Legends"),
"fortnite": ("1517990", "Fortnite"),
"goose goose duck": ("1568590", "Goose Goose Duck"),
"among us": ("945360", "Among Us"),
"fall guys": ("1097150", "Fall Guys"),
"rocket league": ("252950", "Rocket League"),
"warframe": ("230410", "Warframe"),
"destiny 2": ("1085660", "Destiny 2"),
"team fortress 2": ("440", "Team Fortress 2"),
"tf2": ("440", "Team Fortress 2"),
"left 4 dead 2": ("550", "Left 4 Dead 2"),
"l4d2": ("550", "Left 4 Dead 2"),
"paladins": ("444090", "Paladins"),
"smite": ("386360", "SMITE"),
"war thunder": ("236390", "War Thunder"),
"world of warships": ("552990", "World of Warships"),
"path of exile": ("238960", "Path of Exile"),
"poe": ("238960", "Path of Exile"),
"lost ark": ("1599340", "Lost Ark"),
"new world": ("1063730", "New World: Aeternum"),
}
def _search_steam_appid(game_name: str) -> tuple[str | None, str | None]:
name_lower = game_name.lower().strip()
steam_path = _find_steam_path()
if steam_path:
for g in _get_steam_games(steam_path):
if name_lower in g["name"].lower():
return g["id"], g["name"]
if name_lower in _KNOWN_APPIDS:
app_id, canonical = _KNOWN_APPIDS[name_lower]
print(f"[GameUpdater] 📖 Known: {canonical} ({app_id})")
return app_id, canonical
for key, (app_id, canonical) in _KNOWN_APPIDS.items():
if name_lower in key or key in name_lower:
print(f"[GameUpdater] 📖 Known partial: {canonical} ({app_id})")
return app_id, canonical
try:
import urllib.request, urllib.parse
query = urllib.parse.quote(game_name)
url = f"https://store.steampowered.com/api/storesearch/?term={query}&l=english&cc=US"
req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0"})
with urllib.request.urlopen(req, timeout=6) as resp:
items = json.loads(resp.read().decode()).get("items", [])
if items:
best = items[0]
print(f"[GameUpdater] 🌐 Store API: {best['name']} ({best['id']})")
return str(best["id"]), best["name"]
except Exception as e:
print(f"[GameUpdater] ⚠️ AppID search failed: {e}")
return None, None
def _find_best_drive() -> dict | None:
import shutil, string
drives = []
for letter in string.ascii_uppercase:
drive_path = f"{letter}:\\"
if os.path.exists(drive_path):
try:
gb = shutil.disk_usage(drive_path).free / (1024 ** 3)
if gb > 0:
drives.append({"letter": letter, "path": drive_path, "free_gb": gb})
except Exception:
continue
return max(drives, key=lambda d: d["free_gb"]) if drives else None
def _select_drive_in_dialog(dialog, drive_letter: str) -> bool:
target = drive_letter.upper()
for control_type in ("ListItem", "RadioButton"):
try:
for ctrl in dialog.descendants(control_type=control_type):
if target in ctrl.window_text().upper():
ctrl.click_input()
print(f"[GameUpdater] ✅ Drive via {control_type}: {ctrl.window_text()}")
return True
except Exception:
continue
try:
for combo in dialog.descendants(control_type="ComboBox"):
try:
combo.expand()
time.sleep(0.15)
for idx, txt in enumerate(combo.texts()):
if target in txt.upper():
combo.select(idx)
return True
combo.collapse()
except Exception:
continue
except Exception:
pass
try:
for ctrl in dialog.descendants():
txt = ctrl.window_text().upper()
if f"{target}:" in txt and len(txt) < 80:
ctrl.click_input()
return True
except Exception:
pass
return False
def _click_button(window, keywords: list[str]) -> bool:
try:
for btn in window.descendants(control_type="Button"):
try:
txt = btn.window_text().lower().strip()
if txt in keywords or any(kw in txt for kw in keywords):
btn.click_input()
return True
except Exception:
continue
except Exception:
pass
return False
def _handle_install_dialog(game_name: str) -> str:
best_drive = _find_best_drive()
if not best_drive:
return f"Install dialog opened for '{game_name}'. Could not detect drives."
drive_letter = best_drive["letter"]
drive_label = f"{drive_letter}:"
print(f"[GameUpdater] 🏆 Target drive: {drive_label} ({best_drive['free_gb']:.1f} GB free)")
try:
from pywinauto import Application, findwindows
dialog = None
for _ in range(40):
time.sleep(0.5)
try:
for hwnd in findwindows.find_windows(title_re=r"(?i)(install|yükle|steam)", visible_only=True):
try:
app = Application(backend="uia").connect(handle=hwnd)
win = app.window(handle=hwnd)
rect = win.rectangle()
if win.is_visible() and rect.width() > 300 and rect.height() > 200:
all_text = " ".join(c.window_text() for c in win.descendants() if c.window_text()).upper()
if any(x in all_text for x in ("C:", "D:", "E:", "F:", "INSTALL", "YÜKLE")):
dialog = win
break
except Exception:
continue
except Exception:
pass
if dialog:
break
if not dialog:
raise RuntimeError("Dialog not found")
dialog.set_focus()
time.sleep(0.4)
drive_selected = _select_drive_in_dialog(dialog, drive_letter)
install_clicked = _click_button(dialog, ["install", "yükle", "next", "ileri", "ok", "tamam"])
if install_clicked:
suffix = f"Selected {drive_label} and" if drive_selected else "Default drive used, but"
return f"{suffix} clicked Install for '{game_name}'."
return f"Please click Install manually in Steam for '{game_name}'."
except ImportError:
return _handle_install_dialog_pyautogui(game_name, best_drive)
except Exception as e:
print(f"[GameUpdater] ⚠️ pywinauto failed: {e}")
return _handle_install_dialog_pyautogui(game_name, best_drive)
def _handle_install_dialog_pyautogui(game_name: str, best_drive: dict) -> str:
try:
import pyautogui
import pygetwindow as gw
except ImportError:
return f"Install dialog opened for '{game_name}'. Please select '{best_drive['letter']}:' and click Install manually."
pyautogui.FAILSAFE = False
drive_label = f"{best_drive['letter']}:"
install_win = None
for _ in range(30):
time.sleep(0.5)
for w in gw.getAllWindows():
if ("install" in w.title.lower() or "steam" in w.title.lower()) and w.width > 300 and w.visible:
install_win = w
break
if install_win:
break
if not install_win:
return f"Please select '{drive_label}' and click Install in Steam for '{game_name}'."
try:
install_win.activate()
time.sleep(0.4)
except Exception:
pass
wx, wy = install_win.left, install_win.top
ww, wh = install_win.width, install_win.height
pyautogui.click(wx + int(ww * 0.35), wy + int(wh * 0.45))
time.sleep(0.2)
pyautogui.typewrite(best_drive["letter"], interval=0.05)
time.sleep(0.2)
pyautogui.click(wx + int(ww * 0.72), wy + int(wh * 0.88))
return f"Attempted drive {drive_label} selection and Install click for '{game_name}'."
def _install_steam_game(steam_path: Path, game_name: str = None, app_id: str = None) -> str:
if not _ensure_steam_running(steam_path):
return "Could not start Steam."
steam_exe = steam_path / "steam.exe"
installed_games = _get_steam_games(steam_path)
already = None
if app_id:
already = next((g for g in installed_games if g["id"] == str(app_id)), None)
elif game_name:
name_lower = game_name.lower()
already = next((g for g in installed_games if name_lower in g["name"].lower()), None)
else:
return "Please specify a game name or AppID."
if already:
state = already["state"]
name = already["name"]
if state == 4:
return f"'{name}' is already installed and up to date."
if state == 1026:
return f"'{name}' is currently downloading or updating."
if state in (6, 516):
subprocess.Popen([str(steam_exe), f"steam://update/{already['id']}"])
return f"'{name}' has a pending update. Update started."
return f"'{name}' is already installed."
if not app_id and game_name:
found_id, found_name = _search_steam_appid(game_name)
if not found_id:
return f"Could not find '{game_name}' on Steam. Try providing the AppID directly."
app_id = found_id
game_name = found_name or game_name
print(f"[GameUpdater] 🔍 Installing: {game_name} (AppID: {app_id})")
try:
subprocess.Popen([str(steam_exe), f"steam://install/{app_id}"])
threading.Thread(target=_handle_install_dialog, args=(game_name or str(app_id),), daemon=True).start()
return f"Install started for '{game_name}'. Steam will open the download dialog."
except Exception as e:
return f"Install failed: {e}"
def _get_download_status(steam_path: Path) -> str:
games = _get_steam_games(steam_path)
active = [g for g in games if g["state"] == 1026]
pending = [g for g in games if g["state"] in (6, 516)]
lines = []
if active:
lines.append(f"Downloading: {', '.join(g['name'] for g in active)}.")
if pending:
names = ", ".join(g["name"] for g in pending[:5])
suffix = f" and {len(pending) - 5} more" if len(pending) > 5 else ""
lines.append(f"Pending updates: {names}{suffix}.")
return " ".join(lines) if lines else "No active downloads or pending updates."
def _watch_and_shutdown(steam_path: Path, speak=None, check_interval: int = 30, timeout_hours: int = 12):
print("[GameUpdater] 👁️ Watching downloads for shutdown...")
deadline = time.time() + timeout_hours * 3600
for _ in range(24):
time.sleep(5)
active = [g for g in _get_steam_games(steam_path) if g["state"] == 1026]
if active:
names = ", ".join(g["name"] for g in active)
if speak: speak(f"Download started for {names}. I'll shut down when done.")
break
else:
return
while time.time() < deadline:
time.sleep(check_interval)
if not any(g["state"] == 1026 for g in _get_steam_games(steam_path)):
if speak: speak("Download complete. Shutting down now.")
time.sleep(5)
subprocess.run(["shutdown", "/s", "/t", "10"])
return
if speak: speak("Download taking too long. Cancelling auto-shutdown.")
def _get_epic_games() -> list[dict]:
manifests_path = (Path(os.environ.get("PROGRAMDATA", "C:/ProgramData"))
/ "Epic" / "EpicGamesLauncher" / "Data" / "Manifests")
if not manifests_path.exists():
return []
games = []
for item_file in manifests_path.glob("*.item"):
try:
data = json.loads(item_file.read_text(encoding="utf-8"))
name = data.get("DisplayName") or data.get("AppName", "")
if name:
games.append({"id": data.get("AppName", ""), "name": name})
except Exception:
continue
return games
def _is_epic_running() -> bool:
try:
return "epicgameslauncher.exe" in subprocess.run(
["tasklist", "/FI", "IMAGENAME eq EpicGamesLauncher.exe"],
capture_output=True, text=True
).stdout.lower()
except Exception:
return False
def _update_epic_games(epic_path: Path, game_name: str = None) -> str:
epic_exe = epic_path / "EpicGamesLauncher.exe"
if not epic_exe.exists():
return "Epic Games Launcher not found."
games = _get_epic_games()
if game_name:
name_lower = game_name.lower()
matched = [g for g in games if name_lower in g["name"].lower()]
if not matched:
return f"'{game_name}' not found in Epic."
try:
subprocess.Popen([str(epic_exe), f"com.epicgames.launcher://apps/{matched[0]['id']}?action=launch&silent=true"])
return f"Opened Epic for '{matched[0]['name']}'."
except Exception as e:
return f"Epic update failed: {e}"
else:
try:
if _is_epic_running():
for g in games[:10]:
subprocess.Popen([str(epic_exe), f"com.epicgames.launcher://apps/{g['id']}?action=launch&silent=true"])
time.sleep(0.5)
return f"Triggered update check for {len(games)} Epic game(s)."
else:
subprocess.Popen([str(epic_exe)])
return f"Epic Games Launcher opened. {len(games)} game(s) will be checked."
except Exception as e:
return f"Epic update failed: {e}"
def _schedule_daily_update(hour: int = 3, minute: int = 0) -> str:
task_name = "JARVIS_GameUpdater"
script_path = Path(__file__).resolve()
subprocess.run(["schtasks", "/Delete", "/TN", task_name, "/F"], capture_output=True)
for extra in (["/RL", "HIGHEST", "/RU", "SYSTEM"], []):
cmd = ["schtasks", "/Create", "/TN", task_name,
"/TR", f'"{sys.executable}" "{script_path}" --scheduled',
"/SC", "DAILY", "/ST", f"{hour:02d}:{minute:02d}", "/F", *extra]
result = subprocess.run(cmd, capture_output=True, text=True)
if result.returncode == 0:
return f"Daily game update scheduled at {hour:02d}:{minute:02d}."
return f"Scheduling failed: {result.stderr.strip()}"
def _cancel_scheduled_update() -> str:
result = subprocess.run(["schtasks", "/Delete", "/TN", "JARVIS_GameUpdater", "/F"],
capture_output=True, text=True)
return "Scheduled update cancelled." if result.returncode == 0 else "No scheduled update found."
def _get_schedule_status() -> str:
result = subprocess.run(["schtasks", "/Query", "/TN", "JARVIS_GameUpdater", "/FO", "LIST"],
capture_output=True, text=True)
if result.returncode != 0:
return "No scheduled game update found."
for line in result.stdout.strip().split("\n"):
if any(k in line for k in ("Next Run", "Sonraki", "Prochaine", "Próxima", "Nächste")):
return f"Game update scheduled. {line.strip()}"
return "Game update is scheduled."
def game_updater(parameters: dict, player=None, speak=None) -> str:
p = parameters or {}
action = p.get("action", "update").lower().strip()
platform = p.get("platform", "both").lower().strip()
game_name = (p.get("game_name") or "").strip() or None
app_id = (p.get("app_id") or "").strip() or None
hour = int(p.get("hour", 3))
minute = int(p.get("minute", 0))
shutdown = str(p.get("shutdown_when_done", "false")).lower() == "true"
results = []
if action == "schedule": return _schedule_daily_update(hour=hour, minute=minute)
if action == "cancel_schedule": return _cancel_scheduled_update()
if action == "schedule_status": return _get_schedule_status()
if action == "list":
if platform in ("steam", "both"):
steam_path = _find_steam_path()
if steam_path:
games = _get_steam_games(steam_path)
if games:
names = ", ".join(g["name"] for g in games[:8])
suffix = f" and {len(games) - 8} more" if len(games) > 8 else ""
results.append(f"Steam ({len(games)} games): {names}{suffix}.")
else:
results.append("Steam: No games found.")
else:
results.append("Steam: Not installed.")
if platform in ("epic", "both"):
games = _get_epic_games()
if games:
names = ", ".join(g["name"] for g in games[:8])
suffix = f" and {len(games) - 8} more" if len(games) > 8 else ""
results.append(f"Epic ({len(games)} games): {names}{suffix}.")
else:
results.append("Epic: No games found.")
return " | ".join(results) or "No platforms found."
if action == "download_status":
if platform in ("steam", "both"):
steam_path = _find_steam_path()
results.append(_get_download_status(steam_path) if steam_path else "Steam: Not installed.")
if platform in ("epic", "both"):
results.append("Epic download status not available directly.")
return " ".join(results)
if action in ("install", "update"):
if platform in ("steam", "both"):
steam_path = _find_steam_path()
if not steam_path:
results.append("Steam: Not installed.")
else:
if game_name:
installed = _get_steam_games(steam_path)
name_lower = game_name.lower()
is_installed = any(name_lower in g["name"].lower() for g in installed)
if not is_installed:
msg = _install_steam_game(steam_path, game_name=game_name, app_id=app_id)
if shutdown:
threading.Thread(target=_watch_and_shutdown,
kwargs={"steam_path": steam_path, "speak": speak},
daemon=True).start()
msg += " Auto-shutdown enabled."
if player: player.write_log(f"[GameUpdater] {msg[:100]}")
if speak: speak(msg)
return msg
else:
results.append(f"Steam: {_update_steam_games(steam_path, game_name=game_name)}")
else:
if action == "install":
results.append("Steam: Please specify a game name to install.")
else:
results.append(f"Steam: {_update_steam_games(steam_path)}")
if shutdown:
threading.Thread(target=_watch_and_shutdown,
kwargs={"steam_path": steam_path, "speak": speak},
daemon=True).start()
results.append("Auto-shutdown enabled.")
if platform in ("epic", "both"):
epic_path = _find_epic_path()
if epic_path:
results.append(f"Epic: {_update_epic_games(epic_path, game_name=game_name)}")
else:
results.append("Epic: Not installed.")
output = " | ".join(results) or "Nothing to do."
if player: player.write_log(f"[GameUpdater] {output[:100]}")
if speak: speak(output)
return output
return f"Unknown action: '{action}'."
if __name__ == "__main__":
if "--scheduled" in sys.argv:
print(f"[GameUpdater] 🕐 Scheduled run at {datetime.now().strftime('%H:%M')}")
print(f"[GameUpdater] ✅ {game_updater({'action': 'update', 'platform': 'both'})}")
+208
View File
@@ -0,0 +1,208 @@
# actions/open_app.py
# MARK XXV — Cross-Platform App Launcher
import time
import subprocess
import platform
import shutil
try:
import psutil
_PSUTIL = True
except ImportError:
_PSUTIL = False
_APP_ALIASES = {
"whatsapp": {"Windows": "WhatsApp", "Darwin": "WhatsApp", "Linux": "whatsapp"},
"chrome": {"Windows": "chrome", "Darwin": "Google Chrome", "Linux": "google-chrome"},
"google chrome": {"Windows": "chrome", "Darwin": "Google Chrome", "Linux": "google-chrome"},
"firefox": {"Windows": "firefox", "Darwin": "Firefox", "Linux": "firefox"},
"spotify": {"Windows": "Spotify", "Darwin": "Spotify", "Linux": "spotify"},
"vscode": {"Windows": "code", "Darwin": "Visual Studio Code", "Linux": "code"},
"visual studio code": {"Windows": "code", "Darwin": "Visual Studio Code", "Linux": "code"},
"discord": {"Windows": "Discord", "Darwin": "Discord", "Linux": "discord"},
"telegram": {"Windows": "Telegram", "Darwin": "Telegram", "Linux": "telegram"},
"instagram": {"Windows": "Instagram", "Darwin": "Instagram", "Linux": "instagram"},
"tiktok": {"Windows": "TikTok", "Darwin": "TikTok", "Linux": "tiktok"},
"notepad": {"Windows": "notepad.exe", "Darwin": "TextEdit", "Linux": "gedit"},
"calculator": {"Windows": "calc.exe", "Darwin": "Calculator", "Linux": "gnome-calculator"},
"terminal": {"Windows": "cmd.exe", "Darwin": "Terminal", "Linux": "gnome-terminal"},
"cmd": {"Windows": "cmd.exe", "Darwin": "Terminal", "Linux": "bash"},
"explorer": {"Windows": "explorer.exe", "Darwin": "Finder", "Linux": "nautilus"},
"file explorer": {"Windows": "explorer.exe", "Darwin": "Finder", "Linux": "nautilus"},
"paint": {"Windows": "mspaint.exe", "Darwin": "Preview", "Linux": "gimp"},
"word": {"Windows": "winword", "Darwin": "Microsoft Word", "Linux": "libreoffice --writer"},
"excel": {"Windows": "excel", "Darwin": "Microsoft Excel", "Linux": "libreoffice --calc"},
"powerpoint": {"Windows": "powerpnt", "Darwin": "Microsoft PowerPoint","Linux": "libreoffice --impress"},
"vlc": {"Windows": "vlc", "Darwin": "VLC", "Linux": "vlc"},
"zoom": {"Windows": "Zoom", "Darwin": "zoom.us", "Linux": "zoom"},
"slack": {"Windows": "Slack", "Darwin": "Slack", "Linux": "slack"},
"steam": {"Windows": "steam", "Darwin": "Steam", "Linux": "steam"},
"task manager": {"Windows": "taskmgr.exe", "Darwin": "Activity Monitor", "Linux": "gnome-system-monitor"},
"settings": {"Windows": "ms-settings:", "Darwin": "System Preferences", "Linux": "gnome-control-center"},
"powershell": {"Windows": "powershell.exe", "Darwin": "Terminal", "Linux": "bash"},
"edge": {"Windows": "msedge", "Darwin": "Microsoft Edge", "Linux": "microsoft-edge"},
"brave": {"Windows": "brave", "Darwin": "Brave Browser", "Linux": "brave-browser"},
"obsidian": {"Windows": "Obsidian", "Darwin": "Obsidian", "Linux": "obsidian"},
"notion": {"Windows": "Notion", "Darwin": "Notion", "Linux": "notion"},
"blender": {"Windows": "blender", "Darwin": "Blender", "Linux": "blender"},
"capcut": {"Windows": "CapCut", "Darwin": "CapCut", "Linux": "capcut"},
"postman": {"Windows": "Postman", "Darwin": "Postman", "Linux": "postman"},
"figma": {"Windows": "Figma", "Darwin": "Figma", "Linux": "figma"},
}
def _normalize(raw: str) -> str:
system = platform.system()
key = raw.lower().strip()
if key in _APP_ALIASES:
return _APP_ALIASES[key].get(system, raw)
for alias_key, os_map in _APP_ALIASES.items():
if alias_key in key or key in alias_key:
return os_map.get(system, raw)
return raw
def _is_running(app_name: str) -> bool:
if not _PSUTIL:
return True
app_lower = app_name.lower().replace(" ", "").replace(".exe", "")
try:
for proc in psutil.process_iter(["name"]):
try:
proc_name = proc.info["name"].lower().replace(" ", "").replace(".exe", "")
if app_lower in proc_name or proc_name in app_lower:
return True
except (psutil.NoSuchProcess, psutil.AccessDenied):
continue
except Exception:
pass
return False
def _launch_windows(app_name: str) -> bool:
try:
import pyautogui
pyautogui.PAUSE = 0.1
pyautogui.press("win")
time.sleep(0.6)
pyautogui.write(app_name, interval=0.05)
time.sleep(0.8)
pyautogui.press("enter")
time.sleep(3.0)
return True
except Exception as e:
print(f"[open_app] ⚠️ Windows launch failed: {e}")
return False
def _launch_macos(app_name: str) -> bool:
try:
result = subprocess.run(["open", "-a", app_name], capture_output=True, timeout=8)
if result.returncode == 0:
time.sleep(1.0)
return True
except Exception:
pass
try:
result = subprocess.run(["open", "-a", f"{app_name}.app"], capture_output=True, timeout=8)
if result.returncode == 0:
time.sleep(1.0)
return True
except Exception:
pass
try:
import pyautogui
pyautogui.hotkey("command", "space")
time.sleep(0.6)
pyautogui.write(app_name, interval=0.05)
time.sleep(0.8)
pyautogui.press("enter")
time.sleep(1.5)
return True
except Exception as e:
print(f"[open_app] ⚠️ macOS Spotlight failed: {e}")
return False
def _launch_linux(app_name: str) -> bool:
binary = (
shutil.which(app_name) or
shutil.which(app_name.lower()) or
shutil.which(app_name.lower().replace(" ", "-"))
)
if binary:
try:
subprocess.Popen([binary], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
time.sleep(1.0)
return True
except Exception:
pass
try:
subprocess.run(["xdg-open", app_name], capture_output=True, timeout=5)
return True
except Exception:
pass
try:
desktop_name = app_name.lower().replace(" ", "-")
subprocess.run(["gtk-launch", desktop_name], capture_output=True, timeout=5)
return True
except Exception:
pass
return False
_OS_LAUNCHERS = {
"Windows": _launch_windows,
"Darwin": _launch_macos,
"Linux": _launch_linux,
}
def open_app(
parameters=None,
response=None,
player=None,
session_memory=None,
) -> str:
app_name = (parameters or {}).get("app_name", "").strip()
if not app_name:
return "Please specify which application to open, sir."
system = platform.system()
launcher = _OS_LAUNCHERS.get(system)
if launcher is None:
return f"Unsupported OS: {system}"
normalized = _normalize(app_name)
print(f"[open_app] 🚀 Launching: {app_name} → {normalized} ({system})")
if player:
player.write_log(f"[open_app] {app_name}")
try:
success = launcher(normalized)
if success:
return f"Opened {app_name} successfully, sir."
if normalized != app_name:
success = launcher(app_name)
if success:
return f"Opened {app_name} successfully, sir."
return (
f"I tried to open {app_name}, sir, but couldn't confirm it launched. "
f"It may still be loading or might not be installed."
)
except Exception as e:
print(f"[open_app] ❌ {e}")
return f"Failed to open {app_name}, sir: {e}"
+156
View File
@@ -0,0 +1,156 @@
# actions/reminder.py
import subprocess
import os
import sys
from datetime import datetime
def reminder(
parameters: dict,
response: str | None = None,
player=None,
session_memory=None
) -> str:
"""
Sets a timed reminder using Windows Task Scheduler.
parameters:
- date (str) YYYY-MM-DD
- time (str) HH:MM
- message (str)
Returns a result string — Live API voices it automatically.
No edge_speak needed.
"""
date_str = parameters.get("date")
time_str = parameters.get("time")
message = parameters.get("message", "Reminder")
if not date_str or not time_str:
return "I need both a date and a time to set a reminder."
try:
target_dt = datetime.strptime(f"{date_str} {time_str}", "%Y-%m-%d %H:%M")
if target_dt <= datetime.now():
return "That time is already in the past."
task_name = f"MARKReminder_{target_dt.strftime('%Y%m%d_%H%M')}"
safe_message = message.replace('"', '').replace("'", "").strip()[:200]
python_exe = sys.executable
if python_exe.lower().endswith("python.exe"):
pythonw = python_exe.replace("python.exe", "pythonw.exe")
if os.path.exists(pythonw):
python_exe = pythonw
temp_dir = os.environ.get("TEMP", "C:\\Temp")
notify_script = os.path.join(temp_dir, f"{task_name}.pyw")
project_root = os.path.abspath(
os.path.join(os.path.dirname(__file__), "..")
)
script_code = f'''import sys, os, time
sys.path.insert(0, r"{project_root}")
try:
import winsound
for freq in [800, 1000, 1200]:
winsound.Beep(freq, 200)
time.sleep(0.1)
except Exception:
pass
try:
from win10toast import ToastNotifier
ToastNotifier().show_toast(
"MARK Reminder",
"{safe_message}",
duration=15,
threaded=False
)
except Exception:
try:
import subprocess
subprocess.run(["msg", "*", "/TIME:30", "{safe_message}"], shell=True)
except Exception:
pass
time.sleep(3)
try:
os.remove(__file__)
except Exception:
pass
'''
with open(notify_script, "w", encoding="utf-8") as f:
f.write(script_code)
xml_content = f'''<?xml version="1.0" encoding="UTF-16"?>
<Task version="1.2" xmlns="http://schemas.microsoft.com/windows/2004/02/mit/task">
<RegistrationInfo>
<Description>MARK Reminder: {safe_message}</Description>
</RegistrationInfo>
<Triggers>
<TimeTrigger>
<StartBoundary>{target_dt.strftime("%Y-%m-%dT%H:%M:%S")}</StartBoundary>
<Enabled>true</Enabled>
</TimeTrigger>
</Triggers>
<Actions>
<Exec>
<Command>{python_exe}</Command>
<Arguments>"{notify_script}"</Arguments>
</Exec>
</Actions>
<Settings>
<MultipleInstancesPolicy>IgnoreNew</MultipleInstancesPolicy>
<DisallowStartIfOnBatteries>false</DisallowStartIfOnBatteries>
<StopIfGoingOnBatteries>false</StopIfGoingOnBatteries>
<StartWhenAvailable>true</StartWhenAvailable>
<WakeToRun>true</WakeToRun>
<ExecutionTimeLimit>PT5M</ExecutionTimeLimit>
<Enabled>true</Enabled>
</Settings>
<Principals>
<Principal>
<LogonType>InteractiveToken</LogonType>
<RunLevel>LeastPrivilege</RunLevel>
</Principal>
</Principals>
</Task>'''
xml_path = os.path.join(temp_dir, f"{task_name}.xml")
with open(xml_path, "w", encoding="utf-16") as f:
f.write(xml_content)
result = subprocess.run(
f'schtasks /Create /TN "{task_name}" /XML "{xml_path}" /F',
shell=True, capture_output=True, text=True
)
try:
os.remove(xml_path)
except Exception:
pass
if result.returncode != 0:
err = result.stderr.strip() or result.stdout.strip()
print(f"[Reminder] ❌ schtasks failed: {err}")
try:
os.remove(notify_script)
except Exception:
pass
return "I couldn't schedule the reminder due to a system error."
if player:
player.write_log(f"[reminder] set for {date_str} {time_str}")
return f"Reminder set for {target_dt.strftime('%B %d at %I:%M %p')}."
except ValueError:
return "I couldn't understand that date or time format."
except Exception as e:
return f"Something went wrong while scheduling the reminder: {str(e)[:80]}"
+404
View File
@@ -0,0 +1,404 @@
"""
actions/screen_processor.py — Gemini Live API — IMAGE-ONLY SESSION v8
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
v8 Changes:
- Auto camera detection on first use — saves to config
- No hardcoded camera index
- mic_loop removed (no double response issue)
- Image-only session, no conflict with main.py
"""
import asyncio
import base64
import io
import json
import re
import os
import sys
import time
import threading
import cv2
import mss
import mss.tools
import pyaudio
from pathlib import Path
try:
import PIL.Image
_PIL_OK = True
except ImportError:
_PIL_OK = False
from google import genai
from google.genai import types
def get_base_dir():
if getattr(sys, "frozen", False):
return Path(sys.executable).parent
return Path(__file__).resolve().parent.parent
BASE_DIR = get_base_dir()
API_CONFIG_PATH = BASE_DIR / "config" / "api_keys.json"
LIVE_MODEL = "models/gemini-2.5-flash-native-audio-preview-12-2025"
FORMAT = pyaudio.paInt16
CHANNELS = 1
RECEIVE_SAMPLE_RATE = 24000
CHUNK_SIZE = 1024
IMG_MAX_W = 640
IMG_MAX_H = 360
JPEG_Q = 55
SYSTEM_PROMPT = (
"You are JARVIS from Iron Man movies. "
"Analyze images with technical precision and intelligence. "
"Help the user in a way they can understand — don't be overly complex. "
"Be concise, smart, and helpful like Tony Stark's AI assistant. "
"Respond in maximum 2 short sentences. Speed is priority. "
"Address the user as 'sir' for a tone of respect. "
"Ask if the user needs any further help with their problem."
)
def _get_api_key() -> str:
try:
with open(API_CONFIG_PATH, "r", encoding="utf-8") as f:
keys = json.load(f)
key = keys.get("gemini_api_key", "")
if not key:
raise ValueError("gemini_api_key not found")
return key
except Exception as e:
raise RuntimeError(f"Could not load API key: {e}")
def _get_camera_index() -> int:
"""
Reads camera index from config.
If not set, auto-detects the best camera and saves it for future use.
Runs only once — after that, config value is used directly.
"""
try:
with open(API_CONFIG_PATH, "r", encoding="utf-8") as f:
cfg = json.load(f)
if "camera_index" in cfg:
return int(cfg["camera_index"])
except Exception:
pass
print("[Camera] 🔍 No camera index in config. Auto-detecting...")
best_index = 0
for idx in range(6):
cap = cv2.VideoCapture(idx, cv2.CAP_DSHOW)
if not cap.isOpened():
cap.release()
continue
for _ in range(5):
cap.read()
ret, frame = cap.read()
cap.release()
if ret and frame is not None and frame.mean() > 5:
best_index = idx
print(f"[Camera] ✅ Camera found at index {idx} — saving to config.")
break
else:
print(f"[Camera] ⚠️ Index {idx}: no valid frame (black or empty).")
try:
cfg = {}
if API_CONFIG_PATH.exists():
with open(API_CONFIG_PATH, "r", encoding="utf-8") as f:
cfg = json.load(f)
cfg["camera_index"] = best_index
with open(API_CONFIG_PATH, "w", encoding="utf-8") as f:
json.dump(cfg, f, indent=4)
print(f"[Camera] 💾 Camera index {best_index} saved to config.")
except Exception as e:
print(f"[Camera] ⚠️ Could not save camera index: {e}")
return best_index
def _to_jpeg(img_bytes: bytes) -> bytes:
if not _PIL_OK:
return img_bytes
img = PIL.Image.open(io.BytesIO(img_bytes)).convert("RGB")
img.thumbnail([IMG_MAX_W, IMG_MAX_H], PIL.Image.BILINEAR)
buf = io.BytesIO()
img.save(buf, format="JPEG", quality=JPEG_Q, optimize=False)
return buf.getvalue()
def _capture_screenshot() -> bytes:
with mss.mss() as sct:
shot = sct.grab(sct.monitors[1])
png_bytes = mss.tools.to_png(shot.rgb, shot.size)
return _to_jpeg(png_bytes)
def _capture_camera() -> bytes:
camera_index = _get_camera_index()
cap = cv2.VideoCapture(camera_index, cv2.CAP_DSHOW)
if not cap.isOpened():
raise RuntimeError(f"Camera could not be opened: index {camera_index}")
for _ in range(10):
cap.read()
ret, frame = cap.read()
cap.release()
if not ret or frame is None:
raise RuntimeError("Could not capture camera frame.")
if _PIL_OK:
rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
img = PIL.Image.fromarray(rgb)
img.thumbnail([IMG_MAX_W, IMG_MAX_H], PIL.Image.BILINEAR)
buf = io.BytesIO()
img.save(buf, format="JPEG", quality=JPEG_Q, optimize=False)
return buf.getvalue()
_, buf = cv2.imencode(".jpg", frame, [cv2.IMWRITE_JPEG_QUALITY, JPEG_Q])
return buf.tobytes()
class _LiveSession:
"""
Image-only analysis session.
No microphone — no conflict with main.py session.
Sends image + question, plays audio response.
"""
def __init__(self):
self._loop: asyncio.AbstractEventLoop | None = None
self._thread: threading.Thread | None = None
self._session = None
self._out_queue: asyncio.Queue | None = None
self._audio_in: asyncio.Queue | None = None
self._ready: threading.Event = threading.Event()
self._player = None
self._pya = pyaudio.PyAudio()
self._send_lock: asyncio.Lock | None = None
def start(self, player=None):
if self._thread and self._thread.is_alive():
return
self._player = player
self._thread = threading.Thread(
target=self._run_loop, daemon=True, name="VisionSessionThread"
)
self._thread.start()
ok = self._ready.wait(timeout=20)
if not ok:
raise RuntimeError("Vision session did not start within 20s.")
print("[ScreenProcess] ✅ Vision session ready (no mic)")
def _run_loop(self):
self._loop = asyncio.new_event_loop()
asyncio.set_event_loop(self._loop)
self._loop.run_until_complete(self._main())
async def _main(self):
self._out_queue = asyncio.Queue(maxsize=30)
self._audio_in = asyncio.Queue()
self._send_lock = asyncio.Lock()
client = genai.Client(
api_key=_get_api_key(),
http_options={"api_version": "v1beta"}
)
config = types.LiveConnectConfig(
response_modalities=["AUDIO"],
output_audio_transcription={},
system_instruction=SYSTEM_PROMPT,
speech_config=types.SpeechConfig(
voice_config=types.VoiceConfig(
prebuilt_voice_config=types.PrebuiltVoiceConfig(
voice_name="Charon"
)
)
),
)
while True:
try:
print("[ScreenProcess] 🔌 Vision session connecting...")
async with client.aio.live.connect(model=LIVE_MODEL, config=config) as session:
self._session = session
self._ready.set()
print("[ScreenProcess] ✅ Vision session connected")
async with asyncio.TaskGroup() as tg:
tg.create_task(self._send_loop())
tg.create_task(self._recv_loop())
tg.create_task(self._play_loop())
except Exception as e:
print(f"[ScreenProcess] ⚠️ Disconnected: {e} — reconnecting...")
self._session = None
self._ready.clear()
await asyncio.sleep(2)
self._ready.set()
# ── Send loop ──
async def _send_loop(self):
"""Sends (image_bytes, mime_type, user_text) tuples from queue."""
while True:
item = await self._out_queue.get()
if self._session:
image_bytes, mime_type, user_text = item
try:
b64 = base64.b64encode(image_bytes).decode("utf-8")
await self._session.send_client_content(
turns={
"parts": [
{"inline_data": {"mime_type": mime_type, "data": b64}},
{"text": user_text}
]
},
turn_complete=True
)
print("[ScreenProcess] ✅ Image sent")
except Exception as e:
print(f"[ScreenProcess] ⚠️ Send error: {e}")
async def _recv_loop(self):
transcript_buf: list[str] = []
try:
async for response in self._session.receive():
if response.data:
await self._audio_in.put(response.data)
sc = response.server_content
if not sc:
continue
if sc.output_transcription and sc.output_transcription.text:
chunk = sc.output_transcription.text.strip()
if chunk:
transcript_buf.append(chunk)
if sc.turn_complete:
if transcript_buf and self._player:
full = re.sub(r'\s+', ' ', " ".join(transcript_buf)).strip()
if full:
self._player.write_log(f"Jarvis: {full}")
print(f"[ScreenProcess] 💬 {full}")
transcript_buf = []
except Exception as e:
print(f"[ScreenProcess] ⚠️ Recv error: {e}")
transcript_buf = []
await asyncio.sleep(0.3)
async def _play_loop(self):
stream = await asyncio.to_thread(
self._pya.open,
format=FORMAT, channels=CHANNELS,
rate=RECEIVE_SAMPLE_RATE, output=True,
)
try:
while True:
chunk = await self._audio_in.get()
await asyncio.to_thread(stream.write, chunk)
finally:
stream.close()
def analyze(self, image_bytes: bytes, mime_type: str, user_text: str):
"""Called from main thread — puts image into async queue."""
if not self._loop:
return
asyncio.run_coroutine_threadsafe(
self._out_queue.put((image_bytes, mime_type, user_text)),
self._loop
)
def is_ready(self) -> bool:
return self._session is not None
_live = _LiveSession()
_started = False
_start_lock = threading.Lock()
def _ensure_started(player=None):
global _started
with _start_lock:
if not _started:
_live.start(player=player)
_started = True
elif player is not None:
_live._player = player
def screen_process(
parameters: dict,
response: str | None = None,
player=None,
session_memory=None,
) -> bool:
user_text = (parameters or {}).get("text") or (parameters or {}).get("user_text", "")
user_text = (user_text or "").strip()
if not user_text:
print("[ScreenProcess] ⚠️ No user_text provided.")
return False
angle = (parameters or {}).get("angle", "screen").lower().strip()
print(f"[ScreenProcess] angle={angle!r} text={user_text!r}")
_ensure_started(player=player)
try:
if angle == "camera":
image_bytes = _capture_camera()
mime_type = "image/jpeg"
print("[ScreenProcess] 📷 Camera captured")
else:
image_bytes = _capture_screenshot()
mime_type = "image/jpeg" if _PIL_OK else "image/png"
print("[ScreenProcess] 🖥️ Screen captured")
except Exception as e:
import traceback; traceback.print_exc()
print(f"[ScreenProcess] ❌ Capture error: {e}")
return False
print(f"[ScreenProcess] 📦 {len(image_bytes)} bytes → sending")
_live.analyze(image_bytes, mime_type, user_text)
return True
def warmup_session(player=None):
"""
Optional: pre-warm the session.
Do NOT call from main.py — causes double session issue.
Only use when testing screen_processor.py standalone.
"""
try:
_ensure_started(player=player)
except Exception as e:
print(f"[ScreenProcess] ⚠️ Warmup error: {e}")
if __name__ == "__main__":
print("[TEST] screen_processor.py v8 — image-only session")
print("=" * 50)
mode = input("screen / camera (default: screen): ").strip().lower() or "screen"
request = input("Question (Enter for default): ").strip() or "What do you see? Be brief."
t0 = time.perf_counter()
warmup_session()
print(f"Session ready — {time.perf_counter()-t0:.2f}s\n")
t1 = time.perf_counter()
result = screen_process({"angle": mode, "text": request}, player=None)
print(f"Sent — {time.perf_counter()-t1:.3f}s | audio incoming...")
time.sleep(8)
print(f"\n{'✅' if result else '❌'}")
+215
View File
@@ -0,0 +1,215 @@
# actions/send_message.py
# Universal messaging — WhatsApp & Instagram
# Uses visual element detection (pyautogui + screen search) instead of
# hardcoded tab/click sequences — works on any screen resolution.
import time
import pyautogui
from pathlib import Path
pyautogui.FAILSAFE = True
pyautogui.PAUSE = 0.08
def _open_app(app_name: str) -> bool:
"""Opens an app via Windows search."""
try:
pyautogui.press("win")
time.sleep(0.4)
pyautogui.write(app_name, interval=0.04)
time.sleep(0.5)
pyautogui.press("enter")
time.sleep(2.0)
return True
except Exception as e:
print(f"[SendMessage] Could not open {app_name}: {e}")
return False
def _search_contact(contact: str, platform: str):
"""
Searches for a contact inside the messaging app.
Uses Ctrl+F (universal search shortcut) then types contact name.
"""
time.sleep(0.5)
pyautogui.hotkey("ctrl", "f")
time.sleep(0.4)
pyautogui.hotkey("ctrl", "a")
pyautogui.write(contact, interval=0.04)
time.sleep(0.8)
pyautogui.press("enter")
time.sleep(0.6)
def _type_and_send(message: str):
"""Types message and sends it."""
pyautogui.press("tab")
time.sleep(0.2)
pyautogui.hotkey("ctrl", "a")
pyautogui.write(message, interval=0.03)
time.sleep(0.2)
pyautogui.press("enter")
time.sleep(0.3)
def _send_whatsapp(receiver: str, message: str) -> str:
"""
Sends a WhatsApp message via the Windows desktop app.
Steps: Open WhatsApp → Search contact → Click → Type → Send
"""
try:
if not _open_app("WhatsApp"):
return "Could not open WhatsApp."
time.sleep(1.5)
pyautogui.hotkey("ctrl", "f")
time.sleep(0.4)
pyautogui.hotkey("ctrl", "a")
pyautogui.write(receiver, interval=0.04)
time.sleep(1.0)
pyautogui.press("enter")
time.sleep(0.8)
pyautogui.write(message, interval=0.03)
time.sleep(0.2)
pyautogui.press("enter")
return f"Message sent to {receiver} via WhatsApp."
except Exception as e:
return f"WhatsApp error: {e}"
def _send_instagram(receiver: str, message: str) -> str:
"""
Sends an Instagram DM via browser (instagram.com).
Steps: Open Chrome → Go to instagram.com/direct → Search contact → Send
"""
try:
import webbrowser
webbrowser.open("https://www.instagram.com/direct/new/")
time.sleep(3.5)
pyautogui.write(receiver, interval=0.05)
time.sleep(1.5)
pyautogui.press("down")
time.sleep(0.3)
pyautogui.press("enter")
time.sleep(0.5)
for _ in range(3):
pyautogui.press("tab")
time.sleep(0.1)
pyautogui.press("enter")
time.sleep(1.5)
pyautogui.write(message, interval=0.04)
time.sleep(0.2)
pyautogui.press("enter")
return f"Message sent to {receiver} via Instagram."
except Exception as e:
return f"Instagram error: {e}"
def _send_telegram(receiver: str, message: str) -> str:
"""Sends a Telegram message via Windows desktop app."""
try:
if not _open_app("Telegram"):
return "Could not open Telegram."
time.sleep(1.5)
pyautogui.hotkey("ctrl", "f")
time.sleep(0.4)
pyautogui.write(receiver, interval=0.04)
time.sleep(1.0)
pyautogui.press("enter")
time.sleep(0.8)
pyautogui.write(message, interval=0.03)
time.sleep(0.2)
pyautogui.press("enter")
return f"Message sent to {receiver} via Telegram."
except Exception as e:
return f"Telegram error: {e}"
def _send_generic(platform: str, receiver: str, message: str) -> str:
"""
For any other platform not explicitly supported.
Opens the app, searches for contact, types and sends.
Works for: Messenger, Discord, Signal, etc.
"""
try:
if not _open_app(platform):
return f"Could not open {platform}."
time.sleep(1.5)
pyautogui.hotkey("ctrl", "f")
time.sleep(0.4)
pyautogui.write(receiver, interval=0.04)
time.sleep(1.0)
pyautogui.press("enter")
time.sleep(0.8)
pyautogui.write(message, interval=0.03)
time.sleep(0.2)
pyautogui.press("enter")
return f"Message sent to {receiver} via {platform}."
except Exception as e:
return f"{platform} error: {e}"
def send_message(
parameters: dict,
response=None,
player=None,
session_memory=None
) -> str:
"""
Called from main.py.
parameters:
receiver : Contact name to send to
message_text : The message content
platform : whatsapp | instagram | telegram | <any app name>
Default: whatsapp
"""
params = parameters or {}
receiver = params.get("receiver", "").strip()
message_text = params.get("message_text", "").strip()
platform = params.get("platform", "whatsapp").strip().lower()
if not receiver:
return "Please specify who to send the message to, sir."
if not message_text:
return "Please specify what message to send, sir."
print(f"[SendMessage] 📨 {platform} → {receiver}: {message_text[:40]}")
if player:
player.write_log(f"[msg] Sending to {receiver} via {platform}...")
if "whatsapp" in platform or "wp" in platform or "wapp" in platform:
result = _send_whatsapp(receiver, message_text)
elif "instagram" in platform or "ig" in platform or "insta" in platform:
result = _send_instagram(receiver, message_text)
elif "telegram" in platform or "tg" in platform:
result = _send_telegram(receiver, message_text)
else:
result = _send_generic(platform, receiver, message_text)
print(f"[SendMessage] ✅ {result}")
if player:
player.write_log(f"[msg] {result}")
return result
+62
View File
@@ -0,0 +1,62 @@
# actions/weather_report.py
import webbrowser
from urllib.parse import quote_plus
def weather_action(
parameters: dict,
player=None,
session_memory=None
):
"""
Weather report action.
Opens a Google weather search and gives a short spoken confirmation.
"""
city = parameters.get("city")
time = parameters.get("time")
if not city or not isinstance(city, str):
msg = "Sir, the city is missing for the weather report."
_speak_and_log(msg, player)
return msg
city = city.strip()
if not time or not isinstance(time, str):
time = "today"
else:
time = time.strip()
search_query = f"weather in {city} {time}"
encoded_query = quote_plus(search_query)
url = f"https://www.google.com/search?q={encoded_query}"
try:
webbrowser.open(url)
except Exception:
msg = f"Sir, I couldn't open the browser for the weather report."
_speak_and_log(msg, player)
return msg
msg = f"Showing the weather for {city}, {time}, sir."
_speak_and_log(msg, player)
if session_memory:
try:
session_memory.set_last_search(
query=search_query,
response=msg
)
except Exception:
pass
return msg
def _speak_and_log(message: str, player=None):
if player:
try:
player.write_log(f"JARVIS: {message}")
except Exception:
pass
+135
View File
@@ -0,0 +1,135 @@
# actions/web_search.py
# MARK XXV — Web Search
# Primary: Gemini google_search (yeni google.genai SDK)
# Fallback: DuckDuckGo (ddgs)
import json
import sys
from pathlib import Path
def get_base_dir() -> Path:
if getattr(sys, "frozen", False):
return Path(sys.executable).parent
return Path(__file__).resolve().parent.parent
BASE_DIR = get_base_dir()
API_CONFIG_PATH = BASE_DIR / "config" / "api_keys.json"
def _get_api_key() -> str:
with open(API_CONFIG_PATH, "r", encoding="utf-8") as f:
return json.load(f)["gemini_api_key"]
def _gemini_search(query: str) -> str:
from google import genai
client = genai.Client(api_key=_get_api_key())
response = client.models.generate_content(
model="gemini-2.5-flash-lite",
contents=query,
config={"tools": [{"google_search": {}}]}
)
text = ""
for part in response.candidates[0].content.parts:
if hasattr(part, "text") and part.text:
text += part.text
if not text.strip():
raise ValueError("Empty response")
return text.strip()
def _ddg_search(query: str, max_results: int = 6) -> list:
try:
from ddgs import DDGS
except ImportError:
from duckduckgo_search import DDGS
results = []
with DDGS() as ddgs:
for r in ddgs.text(query, max_results=max_results):
results.append({
"title": r.get("title", ""),
"snippet": r.get("body", ""),
"url": r.get("href", ""),
})
return results
def _format_ddg(query: str, results: list) -> str:
if not results:
return f"No results found for: {query}"
lines = [f"Search results for: {query}\n"]
for i, r in enumerate(results, 1):
if r.get("title"): lines.append(f"{i}. {r['title']}")
if r.get("snippet"): lines.append(f" {r['snippet']}")
if r.get("url"): lines.append(f" {r['url']}")
lines.append("")
return "\n".join(lines).strip()
def _compare(items: list, aspect: str) -> str:
query = f"Compare {', '.join(items)} in terms of {aspect}. Give specific facts and data."
try:
return _gemini_search(query)
except Exception as e:
print(f"[WebSearch] ⚠️ Gemini compare failed: {e}")
all_results = {}
for item in items:
try:
all_results[item] = _ddg_search(f"{item} {aspect}", max_results=3)
except Exception:
all_results[item] = []
lines = [f"Comparison — {aspect.upper()}\n{'─'*40}"]
for item in items:
lines.append(f"\n▸ {item}")
for r in all_results.get(item, [])[:2]:
if r.get("snippet"):
lines.append(f" • {r['snippet']}")
return "\n".join(lines)
def web_search(
parameters: dict,
response=None,
player=None,
session_memory=None,
) -> str:
params = parameters or {}
query = params.get("query", "").strip()
mode = params.get("mode", "search").lower()
items = params.get("items", [])
aspect = params.get("aspect", "general")
if not query and not items:
return "Please provide a search query, sir."
if items and mode != "compare":
mode = "compare"
if player:
player.write_log(f"[Search] {query or ', '.join(items)}")
print(f"[WebSearch] 🔍 Query: {query!r} Mode: {mode}")
try:
if mode == "compare" and items:
print(f"[WebSearch] 📊 Comparing: {items}")
result = _compare(items, aspect)
print("[WebSearch] ✅ Compare done.")
return result
print("[WebSearch] 🌐 Gemini search...")
try:
result = _gemini_search(query)
print("[WebSearch] ✅ Gemini OK.")
return result
except Exception as e:
print(f"[WebSearch] ⚠️ Gemini failed ({e}), trying DDG...")
results = _ddg_search(query)
result = _format_ddg(query, results)
print(f"[WebSearch] ✅ DDG: {len(results)} results.")
return result
except Exception as e:
print(f"[WebSearch] ❌ Failed: {e}")
return f"Search failed, sir: {e}"
+543
View File
@@ -0,0 +1,543 @@
import json
import re
import sys
import time
import subprocess
import platform
from pathlib import Path
import pyautogui
import numpy as np
import cv2
from PIL import ImageGrab
try:
import requests
from bs4 import BeautifulSoup
_REQUESTS_OK = True
except ImportError:
_REQUESTS_OK = False
try:
from youtube_transcript_api import YouTubeTranscriptApi
_TRANSCRIPT_OK = True
except ImportError:
_TRANSCRIPT_OK = False
def get_base_dir() -> Path:
if getattr(sys, "frozen", False):
return Path(sys.executable).parent
return Path(__file__).resolve().parent.parent
BASE_DIR = get_base_dir()
API_CONFIG_PATH = BASE_DIR / "config" / "api_keys.json"
HEADERS = {
"User-Agent": (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
"AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/120.0.0.0 Safari/537.36"
),
"Accept-Language": "en-US,en;q=0.9",
}
def _get_api_key() -> str:
with open(API_CONFIG_PATH, "r", encoding="utf-8") as f:
return json.load(f)["gemini_api_key"]
def _get_default_browser_name() -> str | None:
"""
Registry'den varsayılan tarayıcının gerçek yürütülebilir adını döndürür.
Örn: "chrome.exe", "opera.exe", "firefox.exe", "brave.exe"
"""
try:
import winreg
key = winreg.OpenKey(
winreg.HKEY_CURRENT_USER,
r"Software\Microsoft\Windows\Shell\Associations\UrlAssociations\http\UserChoice"
)
prog_id = winreg.QueryValueEx(key, "ProgId")[0].lower()
winreg.CloseKey(key)
mapping = {
"chrome": "chrome",
"firefox": "firefox",
"opera": "opera",
"brave": "brave",
"vivaldi": "vivaldi",
"msedge": "msedge",
"edge": "msedge",
}
for key_str, exe in mapping.items():
if key_str in prog_id:
return exe
except Exception as e:
print(f"[YouTube] ⚠️ Browser detect failed: {e}")
return None
def _get_default_browser_display_name() -> str:
"""
Windows Search'e yazılacak tarayıcı adını döndürür.
Registry → bilinen isimler → fallback "browser"
"""
try:
import winreg
key = winreg.OpenKey(
winreg.HKEY_CURRENT_USER,
r"Software\Microsoft\Windows\Shell\Associations\UrlAssociations\http\UserChoice"
)
prog_id = winreg.QueryValueEx(key, "ProgId")[0].lower()
winreg.CloseKey(key)
display_names = {
"chrome": "Google Chrome",
"firefox": "Firefox",
"opera": "Opera",
"brave": "Brave",
"vivaldi": "Vivaldi",
"msedge": "Microsoft Edge",
"edge": "Microsoft Edge",
}
for key_str, name in display_names.items():
if key_str in prog_id:
return name
except Exception:
pass
return "Google Chrome"
def open_browser():
"""
Varsayılan tarayıcıyı açar.
Yöntem 1: subprocess ile direkt exe bul ve çalıştır (en hızlı).
Yöntem 2: Windows Search'te gerçek tarayıcı adını yaz.
"""
import shutil
browser_exe = _get_default_browser_name()
if browser_exe:
exe_path = shutil.which(browser_exe) or shutil.which(browser_exe + ".exe")
if exe_path:
try:
subprocess.Popen([exe_path])
time.sleep(2.5)
print(f"[YouTube] ✅ Opened browser via exe: {exe_path}")
return
except Exception as e:
print(f"[YouTube] ⚠️ Direct exe failed: {e}")
display_name = _get_default_browser_display_name()
print(f"[YouTube] 🔍 Opening via Windows Search: '{display_name}'")
pyautogui.press("win")
time.sleep(0.5)
pyautogui.write(display_name, interval=0.04)
time.sleep(0.7)
pyautogui.press("enter")
time.sleep(2.5)
def find_video_thumbnails() -> list[tuple[int, int]]:
try:
screenshot = ImageGrab.grab()
img = np.array(screenshot)
screen_h, screen_w = img.shape[:2]
roi_top = int(screen_h * 0.10)
roi_bottom = int(screen_h * 0.75)
roi_left = int(screen_w * 0.20)
roi_right = int(screen_w * 0.80)
roi = img[roi_top:roi_bottom, roi_left:roi_right]
gray = cv2.cvtColor(roi, cv2.COLOR_RGB2GRAY)
edges = cv2.Canny(gray, 30, 100)
kernel = np.ones((3, 3), np.uint8)
edges = cv2.dilate(edges, kernel, iterations=2)
contours, _ = cv2.findContours(
edges, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE
)
candidates = []
for c in contours:
x, y, w, h = cv2.boundingRect(c)
area = w * h
ratio = w / h if h > 0 else 0
if area < 15000:
continue
if not (1.4 < ratio < 2.2):
continue
center_x = x + w // 2 + roi_left
center_y = y + h // 2 + roi_top
candidates.append((center_x, center_y, area))
filtered = []
for cx, cy, area in sorted(candidates, key=lambda c: c[1]):
if not any(abs(cx - fx) < 80 and abs(cy - fy) < 80 for fx, fy in filtered):
filtered.append((cx, cy))
return filtered
except Exception as e:
print(f"[YouTube] ⚠️ Thumbnail detection failed: {e}")
return []
def _extract_video_id(url: str) -> str | None:
patterns = [r"(?:v=|\/v\/|youtu\.be\/|\/embed\/|\/shorts\/)([A-Za-z0-9_-]{11})"]
for pattern in patterns:
match = re.search(pattern, url)
if match:
return match.group(1)
return None
def _is_valid_youtube_url(url: str) -> bool:
return bool(re.search(r"(youtube\.com|youtu\.be)", url or ""))
def _ask_for_url(prompt_text: str = "YouTube video URL:") -> str | None:
try:
import tkinter as tk
from tkinter import simpledialog
root = tk._default_root
if root is None:
root = tk.Tk()
root.withdraw()
url = simpledialog.askstring("J.A.R.V.I.S", prompt_text, parent=root)
return url.strip() if url else None
except Exception as e:
print(f"[YouTube] ⚠️ URL dialog failed: {e}")
return None
def _get_transcript(video_id: str) -> str | None:
if not _TRANSCRIPT_OK:
return None
try:
transcript_list = YouTubeTranscriptApi.list_transcripts(video_id)
transcript = None
try:
transcript = transcript_list.find_manually_created_transcript(
["en", "tr", "de", "fr", "es", "it", "pt", "ru", "ja", "ko", "ar", "zh"]
)
except Exception:
pass
if transcript is None:
try:
transcript = transcript_list.find_generated_transcript(
["en", "tr", "de", "fr", "es", "it", "pt", "ru", "ja", "ko", "ar", "zh"]
)
except Exception:
for t in transcript_list:
transcript = t
break
if transcript is None:
return None
fetched = transcript.fetch()
text = " ".join(entry["text"] for entry in fetched)
return text
except Exception as e:
print(f"[YouTube] ⚠️ Transcript fetch failed: {e}")
return None
def _summarize_with_gemini(transcript: str, video_url: str) -> str:
import google.generativeai as genai
genai.configure(api_key=_get_api_key())
model = genai.GenerativeModel(
model_name="gemini-2.5-flash",
system_instruction=(
"You are JARVIS, Tony Stark's AI assistant. "
"Summarize YouTube video transcripts clearly and concisely. "
"Structure: 1-sentence overview, then 3-5 key points. "
"Be direct. Address the user as 'sir'. "
"Match the language of the transcript."
)
)
max_chars = 80000
truncated = transcript[:max_chars] + ("..." if len(transcript) > max_chars else "")
response = model.generate_content(
f"Please summarize this YouTube video transcript:\n\n{truncated}"
)
return response.text.strip()
def _save_to_notepad(content: str, video_url: str) -> str:
from datetime import datetime
ts = datetime.now().strftime("%Y%m%d_%H%M%S")
filename = f"youtube_summary_{ts}.txt"
desktop = Path.home() / "Desktop"
desktop.mkdir(parents=True, exist_ok=True)
filepath = desktop / filename
header = (
f"JARVIS — YouTube Summary\n"
f"{'─' * 50}\n"
f"URL : {video_url}\n"
f"Date : {datetime.now().strftime('%Y-%m-%d %H:%M')}\n"
f"{'─' * 50}\n\n"
)
filepath.write_text(header + content, encoding="utf-8")
system = platform.system()
open_fn = {
"Windows": lambda p: subprocess.Popen(["notepad.exe", str(p)]),
"Darwin": lambda p: subprocess.Popen(["open", "-t", str(p)]),
"Linux": lambda p: subprocess.Popen(["xdg-open", str(p)]),
}
opener = open_fn.get(system)
if opener:
opener(filepath)
return str(filepath)
def _scrape_video_info(video_id: str) -> dict:
if not _REQUESTS_OK:
return {}
url = f"https://www.youtube.com/watch?v={video_id}"
try:
r = requests.get(url, headers=HEADERS, timeout=12)
html = r.text
info = {}
title_match = re.search(r'"title":\{"runs":\[\{"text":"([^"]+)"', html)
if title_match:
info["title"] = title_match.group(1)
channel_match = re.search(r'"ownerChannelName":"([^"]+)"', html)
if channel_match:
info["channel"] = channel_match.group(1)
views_match = re.search(r'"viewCount":"(\d+)"', html)
if views_match:
views = int(views_match.group(1))
info["views"] = f"{views:,}"
duration_match = re.search(r'"lengthSeconds":"(\d+)"', html)
if duration_match:
secs = int(duration_match.group(1))
info["duration"] = f"{secs // 60}:{secs % 60:02d}"
likes_match = re.search(r'"label":"([0-9,]+ likes)"', html)
if likes_match:
info["likes"] = likes_match.group(1)
return info
except Exception as e:
print(f"[YouTube] ⚠️ Info scrape failed: {e}")
return {}
def _scrape_trending(region: str = "TR", max_results: int = 8) -> list[dict]:
if not _REQUESTS_OK:
return []
url = f"https://www.youtube.com/feed/trending?gl={region.upper()}"
try:
r = requests.get(url, headers=HEADERS, timeout=12)
html = r.text
titles = re.findall(r'"title":\{"runs":\[\{"text":"([^"]+)"\}\]', html)
channels = re.findall(r'"ownerText":\{"runs":\[\{"text":"([^"]+)"', html)
results = []
seen = set()
for i, title in enumerate(titles):
if title in seen or len(title) < 5:
continue
seen.add(title)
channel = channels[i] if i < len(channels) else "Unknown"
results.append({"rank": len(results) + 1, "title": title, "channel": channel})
if len(results) >= max_results:
break
return results
except Exception as e:
print(f"[YouTube] ⚠️ Trending scrape failed: {e}")
return []
def _handle_play(parameters: dict, player) -> str:
query = parameters.get("query", "").strip()
if not query:
return "Please tell me what you'd like to watch, sir."
if player:
player.write_log(f"[YouTube] Searching: {query}")
open_browser()
search_query = query.replace(" ", "+")
url = f"https://www.youtube.com/results?search_query={search_query}"
pyautogui.hotkey("ctrl", "l")
time.sleep(0.3)
pyautogui.write(url, interval=0.02)
pyautogui.press("enter")
time.sleep(3.5)
thumbnails = find_video_thumbnails()
if len(thumbnails) >= 2:
x, y = thumbnails[1]
pyautogui.click(x, y)
elif len(thumbnails) == 1:
x, y = thumbnails[0]
pyautogui.click(x, y)
else:
screen_w, screen_h = pyautogui.size()
pyautogui.click(screen_w // 2, int(screen_h * 0.45))
return f"Attempted to play YouTube video for: {query}"
def _handle_summarize(parameters: dict, player, speak) -> str:
if not _TRANSCRIPT_OK:
return "youtube-transcript-api is not installed. Run: pip install youtube-transcript-api"
url = _ask_for_url("Please paste the YouTube video URL:")
if not url:
return "No URL provided, sir. Summary cancelled."
if not _is_valid_youtube_url(url):
return "That doesn't appear to be a valid YouTube URL, sir."
video_id = _extract_video_id(url)
if not video_id:
return "Could not extract video ID from that URL, sir."
if player:
player.write_log(f"[YouTube] Summarizing: {url}")
if speak:
speak("Fetching the transcript now, sir. One moment.")
transcript = _get_transcript(video_id)
if not transcript:
return "I couldn't retrieve a transcript for that video, sir."
if speak:
speak("Transcript retrieved. Generating summary now.")
try:
summary = _summarize_with_gemini(transcript, url)
except Exception as e:
return f"Summary generation failed, sir: {e}"
if speak:
speak(summary)
save = parameters.get("save", False)
if save:
saved_path = _save_to_notepad(summary, url)
return f"Summary complete and saved to Desktop: {saved_path}"
return summary
def _handle_get_info(parameters: dict, player, speak) -> str:
url = parameters.get("url", "").strip()
if not url:
url = _ask_for_url("Please paste the YouTube video URL:")
if not url or not _is_valid_youtube_url(url):
return "Please provide a valid YouTube URL, sir."
video_id = _extract_video_id(url)
if not video_id:
return "Could not extract video ID, sir."
if player:
player.write_log(f"[YouTube] Getting info: {url}")
info = _scrape_video_info(video_id)
if not info:
return "Could not retrieve video information, sir."
lines = []
for key in ("title", "channel", "views", "duration", "likes"):
if key in info:
lines.append(f"{key.capitalize()}: {info[key]}")
result = "\n".join(lines)
if speak:
speak(f"Here's the video info, sir. {result.replace(chr(10), '. ')}")
return result
def _handle_trending(parameters: dict, player, speak) -> str:
region = parameters.get("region", "TR").upper()
if player:
player.write_log(f"[YouTube] Trending: {region}")
trending = _scrape_trending(region=region, max_results=8)
if not trending:
return f"Could not fetch trending videos for region {region}, sir."
lines = [f"Top trending videos in {region}:"]
for item in trending:
lines.append(f"{item['rank']}. {item['title']} — {item['channel']}")
result = "\n".join(lines)
if speak:
top3 = trending[:3]
spoken = "Here are the top trending videos, sir. " + ". ".join(
f"Number {v['rank']}: {v['title']} by {v['channel']}" for v in top3
)
speak(spoken)
return result
_ACTION_MAP = {
"play": _handle_play,
"summarize": _handle_summarize,
"get_info": _handle_get_info,
"trending": _handle_trending,
}
def youtube_video(
parameters: dict,
response=None,
player=None,
session_memory=None,
speak=None,
) -> str:
params = parameters or {}
action = params.get("action", "play").lower().strip()
if player:
player.write_log(f"[YouTube] Action: {action}")
print(f"[YouTube] ▶️ Action: {action} Params: {params}")
handler = _ACTION_MAP.get(action)
if handler is None:
return f"Unknown YouTube action: '{action}'. Available: play, summarize, get_info, trending."
try:
if action == "play":
return handler(params, player) or "Done."
return handler(params, player, speak) or "Done."
except Exception as e:
print(f"[YouTube] ❌ Error in {action}: {e}")
return f"YouTube {action} failed, sir: {e}"
+197
View File
@@ -0,0 +1,197 @@
import json
import re
import sys
from pathlib import Path
from enum import Enum
def get_base_dir() -> Path:
if getattr(sys, "frozen", False):
return Path(sys.executable).parent
return Path(__file__).resolve().parent.parent
BASE_DIR = get_base_dir()
API_CONFIG_PATH = BASE_DIR / "config" / "api_keys.json"
class ErrorDecision(Enum):
RETRY = "retry"
SKIP = "skip"
REPLAN = "replan"
ABORT = "abort"
ERROR_ANALYST_PROMPT = """You are the error recovery module of MARK XXV AI assistant.
A task step has failed. Analyze the error and decide what to do.
DECISIONS:
- retry : Transient error (network timeout, temporary file lock, race condition).
The same step can succeed if tried again.
- skip : This step is not critical and the task can succeed without it.
- replan : The approach was wrong. A different tool or method should be tried.
- abort : The task is fundamentally impossible or unsafe to continue.
Also provide:
- A brief explanation of WHY it failed (1 sentence)
- A fix suggestion if decision is replan (what to try instead)
- Max retries: how many times to retry if decision is retry (1 or 2)
Return ONLY valid JSON:
{
"decision": "retry|skip|replan|abort",
"reason": "why it failed",
"fix_suggestion": "what to try instead (for replan)",
"max_retries": 1,
"user_message": "Short message to tell the user (max 15 words)"
}
"""
def _get_api_key() -> str:
with open(API_CONFIG_PATH, "r", encoding="utf-8") as f:
return json.load(f)["gemini_api_key"]
def analyze_error(
step: dict,
error: str,
attempt: int = 1,
max_attempts: int = 2
) -> dict:
"""
Analyzes a failed step and returns a recovery decision.
Args:
step : The step dict that failed
error : Error message/traceback
attempt : Current attempt number
max_attempts : How many times we've already tried
Returns:
{
"decision": ErrorDecision,
"reason": str,
"fix_suggestion": str,
"max_retries": int,
"user_message": str
}
"""
import google.generativeai as genai
if attempt >= max_attempts:
print(f"[ErrorHandler] ⚠️ Max attempts reached for step {step.get('step')} — forcing replan")
return {
"decision": ErrorDecision.REPLAN,
"reason": f"Failed {attempt} times: {error[:100]}",
"fix_suggestion": "Try a completely different approach or tool",
"max_retries": 0,
"user_message": "Trying a different approach, sir."
}
genai.configure(api_key=_get_api_key())
model = genai.GenerativeModel(
model_name="gemini-2.5-flash-lite",
system_instruction=ERROR_ANALYST_PROMPT
)
prompt = f"""Failed step:
Tool: {step.get('tool')}
Description: {step.get('description')}
Parameters: {json.dumps(step.get('parameters', {}), indent=2)}
Critical: {step.get('critical', False)}
Error:
{error[:500]}
Attempt number: {attempt}"""
try:
response = model.generate_content(prompt)
text = response.text.strip()
text = re.sub(r"```(?:json)?", "", text).strip().rstrip("`").strip()
result = json.loads(text)
decision_str = result.get("decision", "replan").lower()
decision_map = {
"retry": ErrorDecision.RETRY,
"skip": ErrorDecision.SKIP,
"replan": ErrorDecision.REPLAN,
"abort": ErrorDecision.ABORT,
}
result["decision"] = decision_map.get(decision_str, ErrorDecision.REPLAN)
if step.get("critical") and result["decision"] == ErrorDecision.SKIP:
result["decision"] = ErrorDecision.REPLAN
result["user_message"] = "This step is critical — finding alternative approach, sir."
print(f"[ErrorHandler] Decision: {result['decision'].value} — {result.get('reason', '')}")
return result
except Exception as e:
print(f"[ErrorHandler] ⚠️ Analysis failed: {e} — defaulting to replan")
return {
"decision": ErrorDecision.REPLAN,
"reason": str(e),
"fix_suggestion": "Try alternative approach",
"max_retries": 1,
"user_message": "Encountered an issue, adjusting approach, sir."
}
def generate_fix(step: dict, error: str, fix_suggestion: str) -> dict:
"""
When decision is REPLAN and a fix suggestion exists,
generates a replacement step using generated_code as fallback.
Returns a modified step dict.
"""
import google.generativeai as genai
genai.configure(api_key=_get_api_key())
model = genai.GenerativeModel(model_name="gemini-2.0-flash")
prompt = f"""A task step failed. Generate a replacement step.
Original step:
Tool: {step.get('tool')}
Description: {step.get('description')}
Parameters: {json.dumps(step.get('parameters', {}), indent=2)}
Error: {error[:300]}
Fix suggestion: {fix_suggestion}
Write a Python script that accomplishes the same goal differently.
Return ONLY the Python code, no explanation."""
try:
response = model.generate_content(prompt)
code = response.text.strip()
code = re.sub(r"```(?:python)?", "", code).strip().rstrip("`").strip()
return {
"step": step.get("step"),
"tool": "code_helper",
"description": f"Auto-fix for: {step.get('description')}",
"parameters": {
"action": "run",
"description": fix_suggestion,
"code": code,
"language": "python"
},
"depends_on": step.get("depends_on", []),
"critical": step.get("critical", False)
}
except Exception as e:
print(f"[ErrorHandler] ⚠️ Fix generation failed: {e}")
return {
"step": step.get("step"),
"tool": "generated_code",
"description": f"Fallback for: {step.get('description')}",
"parameters": {"description": step.get("description", "")},
"depends_on": step.get("depends_on", []),
"critical": step.get("critical", False)
}
+400
View File
@@ -0,0 +1,400 @@
import json
import re
import sys
import threading
import subprocess
import tempfile
import os
from pathlib import Path
from typing import Callable
from agent.planner import create_plan, replan
from agent.error_handler import analyze_error, generate_fix, ErrorDecision
def get_base_dir() -> Path:
if getattr(sys, "frozen", False):
return Path(sys.executable).parent
return Path(__file__).resolve().parent.parent
BASE_DIR = get_base_dir()
API_CONFIG_PATH = BASE_DIR / "config" / "api_keys.json"
def _get_api_key() -> str:
with open(API_CONFIG_PATH, "r", encoding="utf-8") as f:
return json.load(f)["gemini_api_key"]
def _run_generated_code(description: str, speak: Callable | None = None) -> str:
import google.generativeai as genai
if speak:
speak("Writing custom code for this task, sir.")
home = Path.home()
desktop = home / "Desktop"
downloads = home / "Downloads"
documents = home / "Documents"
if not desktop.exists():
try:
import winreg
key = winreg.OpenKey(winreg.HKEY_CURRENT_USER,
r"Software\Microsoft\Windows\CurrentVersion\Explorer\Shell Folders")
desktop = Path(winreg.QueryValueEx(key, "Desktop")[0])
except Exception:
pass
genai.configure(api_key=_get_api_key())
model = genai.GenerativeModel(
model_name="gemini-2.5-flash",
system_instruction=(
"You are an expert Python developer. "
"Write clean, complete, working Python code. "
"Use standard library + common packages. "
"Install missing packages with subprocess + pip if needed. "
"Return ONLY the Python code. No explanation, no markdown, no backticks.\n\n"
f"SYSTEM PATHS:\n"
f" Desktop = r'{desktop}'\n"
f" Downloads = r'{downloads}'\n"
f" Documents = r'{documents}'\n"
f" Home = r'{home}'\n"
)
)
try:
response = model.generate_content(
f"Write Python code to accomplish this task:\n\n{description}"
)
code = response.text.strip()
code = re.sub(r"```(?:python)?", "", code).strip().rstrip("`").strip()
with tempfile.NamedTemporaryFile(
mode="w", suffix=".py", delete=False, encoding="utf-8"
) as f:
f.write(code)
tmp_path = f.name
print(f"[Executor] 🐍 Running generated code: {tmp_path}")
result = subprocess.run(
[sys.executable, tmp_path],
capture_output=True, text=True,
timeout=120, cwd=str(Path.home())
)
try:
os.unlink(tmp_path)
except Exception:
pass
output = result.stdout.strip()
error = result.stderr.strip()
if result.returncode == 0 and output:
return output
elif result.returncode == 0:
return "Task completed successfully."
elif error:
raise RuntimeError(f"Code error: {error[:400]}")
return "Completed."
except subprocess.TimeoutExpired:
raise RuntimeError("Generated code timed out after 120 seconds.")
except RuntimeError:
raise
except Exception as e:
raise RuntimeError(f"Generated code failed: {e}")
def _inject_context(params: dict, tool: str, step_results: dict, goal: str = "") -> dict:
if not step_results:
return params
params = dict(params)
if tool == "file_controller" and params.get("action") in ("write", "create_file"):
content = params.get("content", "")
if not content or len(content) < 50:
all_results = [
v for v in step_results.values()
if v and len(v) > 100 and v not in ("Done.", "Completed.")
]
if all_results:
combined = "\n\n---\n\n".join(all_results)
translated = _translate_to_goal_language(combined, goal)
params["content"] = translated
print(f"[Executor] 💉 Injected + translated content")
return params
def _detect_language(text: str) -> str:
import google.generativeai as genai
genai.configure(api_key=_get_api_key())
model = genai.GenerativeModel("gemini-2.5-flash-lite")
try:
response = model.generate_content(
f"What language is this text written in? "
f"Reply with ONLY the language name in English (e.g. Turkish, English, French).\n\n"
f"Text: {text[:200]}"
)
return response.text.strip()
except Exception:
return "English"
def _translate_to_goal_language(content: str, goal: str) -> str:
if not goal:
return content
try:
import google.generativeai as genai
genai.configure(api_key=_get_api_key())
model = genai.GenerativeModel("gemini-2.5-flash")
target_lang = _detect_language(goal)
print(f"[Executor] 🌐 Translating to: {target_lang}")
prompt = (
f"You are a professional translator. "
f"Translate the following text into {target_lang}.\n"
f"IMPORTANT:\n"
f"- Translate EVERYTHING, leave nothing in English\n"
f"- Keep all facts, numbers, and data intact\n"
f"- Keep the structure and formatting\n"
f"- Output ONLY the translated text, nothing else\n\n"
f"Text to translate:\n{content[:4000]}"
)
response = model.generate_content(prompt)
translated = response.text.strip()
print(f"[Executor] ✅ Translation done ({target_lang})")
return translated
except Exception as e:
print(f"[Executor] ⚠️ Translation failed: {e}")
return content
def _call_tool(tool: str, parameters: dict, speak: Callable | None) -> str:
if tool == "open_app":
from actions.open_app import open_app
return open_app(parameters=parameters, player=None) or "Done."
elif tool == "web_search":
from actions.web_search import web_search
return web_search(parameters=parameters, player=None) or "Done."
elif tool == "game_updater":
from actions.game_updater import game_updater
return game_updater(parameters=parameters, player=None, speak=speak) or "Done."
elif tool == "browser_control":
from actions.browser_control import browser_control
return browser_control(parameters=parameters, player=None) or "Done."
elif tool == "file_controller":
from actions.file_controller import file_controller
return file_controller(parameters=parameters, player=None) or "Done."
elif tool == "cmd_control":
from actions.cmd_control import cmd_control
return cmd_control(parameters=parameters, player=None) or "Done."
elif tool == "code_helper":
from actions.code_helper import code_helper
return code_helper(parameters=parameters, player=None, speak=speak) or "Done."
elif tool == "dev_agent":
from actions.dev_agent import dev_agent
return dev_agent(parameters=parameters, player=None, speak=speak) or "Done."
elif tool == "screen_process":
from actions.screen_processor import screen_process
screen_process(parameters=parameters, player=None)
return "Screen captured and analyzed."
elif tool == "send_message":
from actions.send_message import send_message
return send_message(parameters=parameters, player=None) or "Done."
elif tool == "reminder":
from actions.reminder import reminder
return reminder(parameters=parameters, player=None) or "Done."
elif tool == "youtube_video":
from actions.youtube_video import youtube_video
return youtube_video(parameters=parameters, player=None) or "Done."
elif tool == "weather_report":
from actions.weather_report import weather_action
return weather_action(parameters=parameters, player=None) or "Done."
elif tool == "computer_settings":
from actions.computer_settings import computer_settings
return computer_settings(parameters=parameters, player=None) or "Done."
elif tool == "desktop_control":
from actions.desktop import desktop_control
return desktop_control(parameters=parameters, player=None) or "Done."
elif tool == "computer_control":
from actions.computer_control import computer_control
return computer_control(parameters=parameters, player=None) or "Done."
elif tool == "generated_code":
description = parameters.get("description", "")
if not description:
raise ValueError("generated_code requires a 'description' parameter.")
return _run_generated_code(description, speak=speak)
elif tool == "flight_finder":
from actions.flight_finder import flight_finder
return flight_finder(parameters=parameters, player=None, speak=speak) or "Done."
else:
print(f"[Executor] ⚠️ Unknown tool '{tool}' — falling back to generated_code")
return _run_generated_code(f"Accomplish this task: {parameters}", speak=speak)
class AgentExecutor:
MAX_REPLAN_ATTEMPTS = 2
def execute(
self,
goal: str,
speak: Callable | None = None,
cancel_flag: threading.Event | None = None,
) -> str:
print(f"\n[Executor] 🎯 Goal: {goal}")
replan_attempts = 0
completed_steps = []
step_results = {}
plan = create_plan(goal)
while True:
steps = plan.get("steps", [])
if not steps:
msg = "I couldn't create a valid plan for this task, sir."
if speak: speak(msg)
return msg
success = True
failed_step = None
failed_error = ""
for step in steps:
if cancel_flag and cancel_flag.is_set():
if speak: speak("Task cancelled, sir.")
return "Task cancelled."
step_num = step.get("step", "?")
tool = step.get("tool", "generated_code")
desc = step.get("description", "")
params = step.get("parameters", {})
params = _inject_context(params, tool, step_results, goal=goal)
print(f"\n[Executor] ▶️ Step {step_num}: [{tool}] {desc}")
attempt = 1
step_ok = False
while attempt <= 3:
if cancel_flag and cancel_flag.is_set():
break
try:
result = _call_tool(tool, params, speak)
step_results[step_num] = result
completed_steps.append(step)
print(f"[Executor] ✅ Step {step_num} done: {str(result)[:100]}")
step_ok = True
break
except Exception as e:
error_msg = str(e)
print(f"[Executor] ❌ Step {step_num} attempt {attempt} failed: {error_msg}")
recovery = analyze_error(step, error_msg, attempt=attempt)
decision = recovery["decision"]
user_msg = recovery.get("user_message", "")
if speak and user_msg:
speak(user_msg)
if decision == ErrorDecision.RETRY:
attempt += 1
import time; time.sleep(2)
continue
elif decision == ErrorDecision.SKIP:
print(f"[Executor] ⏭️ Skipping step {step_num}")
completed_steps.append(step)
step_ok = True
break
elif decision == ErrorDecision.ABORT:
msg = f"Task aborted, sir. {recovery.get('reason', '')}"
if speak: speak(msg)
return msg
else:
fix_suggestion = recovery.get("fix_suggestion", "")
if fix_suggestion and tool != "generated_code":
try:
fixed_step = generate_fix(step, error_msg, fix_suggestion)
if speak: speak("Trying an alternative approach, sir.")
res = _call_tool(
fixed_step["tool"],
fixed_step["parameters"],
speak
)
step_results[step_num] = res
completed_steps.append(step)
step_ok = True
break
except Exception as fix_err:
print(f"[Executor] ⚠️ Fix failed: {fix_err}")
failed_step = step
failed_error = error_msg
success = False
break
if not step_ok and not failed_step:
failed_step = step
failed_error = "Max retries exceeded"
success = False
if not success:
break
if success:
return self._summarize(goal, completed_steps, speak)
if replan_attempts >= self.MAX_REPLAN_ATTEMPTS:
msg = f"Task failed after {replan_attempts} replan attempts, sir."
if speak: speak(msg)
return msg
if speak: speak("Adjusting my approach, sir.")
replan_attempts += 1
plan = replan(goal, completed_steps, failed_step, failed_error)
def _summarize(self, goal: str, completed_steps: list, speak: Callable | None) -> str:
fallback = f"All done, sir. Completed {len(completed_steps)} steps for: {goal[:60]}."
try:
import google.generativeai as genai
genai.configure(api_key=_get_api_key())
model = genai.GenerativeModel(model_name="gemini-2.5-flash-lite")
steps_str = "\n".join(f"- {s.get('description', '')}" for s in completed_steps)
prompt = (
f'User goal: "{goal}"\n'
f"Completed steps:\n{steps_str}\n\n"
"Write a single natural sentence summarizing what was accomplished. "
"Address the user as 'sir'. Be direct and positive."
)
response = model.generate_content(prompt)
summary = response.text.strip()
if speak: speak(summary)
return summary
except Exception:
if speak: speak(fallback)
return fallback
+278
View File
@@ -0,0 +1,278 @@
import json
import re
import sys
from pathlib import Path
def get_base_dir() -> Path:
if getattr(sys, "frozen", False):
return Path(sys.executable).parent
return Path(__file__).resolve().parent.parent
BASE_DIR = get_base_dir()
API_CONFIG_PATH = BASE_DIR / "config" / "api_keys.json"
PLANNER_PROMPT = """You are the planning module of MARK XXV, a personal AI assistant.
Your job: break any user goal into a sequence of steps using ONLY the tools listed below.
ABSOLUTE RULES:
- NEVER use generated_code or write Python scripts. It does not exist.
- NEVER reference previous step results in parameters. Every step is independent.
- Use web_search for ANY information retrieval, research, or current data.
- Use file_controller to save content to disk.
- Use cmd_control to open files or run system commands.
- Max 5 steps. Use the minimum steps needed.
AVAILABLE TOOLS AND THEIR PARAMETERS:
open_app
app_name: string (required)
web_search
query: string (required) — write a clear, focused search query
mode: "search" or "compare" (optional, default: search)
items: list of strings (optional, for compare mode)
aspect: string (optional, for compare mode)
game_updater
action: "update" | "install" | "list" | "download_status" | "schedule" (required)
platform: "steam" | "epic" | "both" (optional, default: both)
game_name: string (optional)
app_id: string (optional)
shutdown_when_done: boolean (optional)
browser_control
action: "go_to" | "search" | "click" | "type" | "scroll" | "get_text" | "press" | "close" (required)
url: string (for go_to)
query: string (for search)
text: string (for click/type)
direction: "up" | "down" (for scroll)
file_controller
action: "write" | "create_file" | "read" | "list" | "delete" | "move" | "copy" | "find" | "disk_usage" (required)
path: string — use "desktop" for Desktop folder
name: string — filename
content: string — file content (for write/create_file)
cmd_control
task: string (required) — natural language description of what to do
visible: boolean (optional)
computer_settings
action: string (required)
description: string — natural language description
value: string (optional)
computer_control
action: "type" | "click" | "hotkey" | "press" | "scroll" | "screenshot" | "screen_find" | "screen_click" (required)
text: string (for type)
x, y: int (for click)
keys: string (for hotkey, e.g. "ctrl+c")
key: string (for press)
direction: "up" | "down" (for scroll)
description: string (for screen_find/screen_click)
screen_process
text: string (required) — what to analyze or ask about the screen
angle: "screen" | "camera" (optional)
send_message
receiver: string (required)
message_text: string (required)
platform: string (required)
reminder
date: string YYYY-MM-DD (required)
time: string HH:MM (required)
message: string (required)
desktop_control
action: "wallpaper" | "organize" | "clean" | "list" | "task" (required)
path: string (optional)
task: string (optional)
youtube_video
action: "play" | "summarize" | "trending" (required)
query: string (for play)
weather_report
city: string (required)
flight_finder
origin: string (required)
destination: string (required)
date: string (required)
code_helper
action: "write" | "edit" | "run" | "explain" (required)
description: string (required)
language: string (optional)
output_path: string (optional)
file_path: string (optional)
dev_agent
description: string (required)
language: string (optional)
EXAMPLES:
Goal: "research mechanical engineering and save it to a notepad file"
Steps:
web_search | query: "mechanical engineering overview definition history"
web_search | query: "mechanical engineering applications and future trends"
file_controller | action: write, path: desktop, name: mechanical_engineering.txt, content: "MECHANICAL ENGINEERING RESEARCH\n\nThis file will be filled with web research results."
cmd_control | task: "open mechanical_engineering.txt on desktop with notepad"
Goal: "What is the price of Bitcoin"
Steps:
web_search | query: "Bitcoin price today USD"
Goal: "List the files on the desktop and find the largest 5 files"
Steps:
file_controller | action: list, path: desktop
file_controller | action: largest, path: desktop, count: 5
Goal: "Install PUBG from Steam"
Steps:
game_updater | action: install, platform: steam, game_name: "PUBG"
Goal: "Update all my Steam games"
Steps:
game_updater | action: update, platform: steam
Goal: "Send John a message on WhatsApp saying there is a meeting tomorrow"
Steps:
send_message | receiver: John, message_text: "There is a meeting tomorrow", platform: WhatsApp
Goal: "Open the clock and set a reminder for 30 minutes later"
Steps:
reminder | date: [today], time: [now+30min], message: "Reminder"
OUTPUT — return ONLY valid JSON, no markdown, no explanation, no code blocks:
{
"goal": "...",
"steps": [
{
"step": 1,
"tool": "tool_name",
"description": "what this step does",
"parameters": {},
"critical": true
}
]
}
"""
def _get_api_key() -> str:
with open(API_CONFIG_PATH, "r", encoding="utf-8") as f:
return json.load(f)["gemini_api_key"]
def create_plan(goal: str, context: str = "") -> dict:
import google.generativeai as genai
genai.configure(api_key=_get_api_key())
model = genai.GenerativeModel(
model_name="gemini-2.5-flash-lite",
system_instruction=PLANNER_PROMPT
)
user_input = f"Goal: {goal}"
if context:
user_input += f"\n\nContext: {context}"
try:
response = model.generate_content(user_input)
text = response.text.strip()
text = re.sub(r"```(?:json)?", "", text).strip().rstrip("`").strip()
plan = json.loads(text)
if "steps" not in plan or not isinstance(plan["steps"], list):
raise ValueError("Invalid plan structure")
for step in plan["steps"]:
if step.get("tool") in ("generated_code",):
print(f"[Planner] ⚠️ generated_code detected in step {step.get('step')} — replacing with web_search")
desc = step.get("description", goal)
step["tool"] = "web_search"
step["parameters"] = {"query": desc[:200]}
print(f"[Planner] ✅ Plan: {len(plan['steps'])} steps")
for s in plan["steps"]:
print(f" Step {s['step']}: [{s['tool']}] {s['description']}")
return plan
except json.JSONDecodeError as e:
print(f"[Planner] ⚠️ JSON parse failed: {e}")
return _fallback_plan(goal)
except Exception as e:
print(f"[Planner] ⚠️ Planning failed: {e}")
return _fallback_plan(goal)
def _fallback_plan(goal: str) -> dict:
print("[Planner] 🔄 Fallback plan")
return {
"goal": goal,
"steps": [
{
"step": 1,
"tool": "web_search",
"description": f"Search for: {goal}",
"parameters": {"query": goal},
"critical": True
}
]
}
def replan(goal: str, completed_steps: list, failed_step: dict, error: str) -> dict:
import google.generativeai as genai
genai.configure(api_key=_get_api_key())
model = genai.GenerativeModel(
model_name="gemini-2.5-flash",
system_instruction=PLANNER_PROMPT
)
completed_summary = "\n".join(
f" - Step {s['step']} ({s['tool']}): DONE" for s in completed_steps
)
prompt = f"""Goal: {goal}
Already completed:
{completed_summary if completed_summary else ' (none)'}
Failed step: [{failed_step.get('tool')}] {failed_step.get('description')}
Error: {error}
Create a REVISED plan for the remaining work only. Do not repeat completed steps."""
try:
response = model.generate_content(prompt)
text = response.text.strip()
text = re.sub(r"```(?:json)?", "", text).strip().rstrip("`").strip()
plan = json.loads(text)
for step in plan.get("steps", []):
if step.get("tool") == "generated_code":
step["tool"] = "web_search"
step["parameters"] = {"query": step.get("description", goal)[:200]}
print(f"[Planner] 🔄 Revised plan: {len(plan['steps'])} steps")
return plan
except Exception as e:
print(f"[Planner] ⚠️ Replan failed: {e}")
return _fallback_plan(goal)
+221
View File
@@ -0,0 +1,221 @@
import threading
import time
import uuid
from dataclasses import dataclass, field
from enum import Enum
from typing import Callable, Any
class TaskStatus(Enum):
PENDING = "pending"
RUNNING = "running"
COMPLETED = "completed"
FAILED = "failed"
CANCELLED = "cancelled"
class TaskPriority(Enum):
LOW = 3
NORMAL = 2
HIGH = 1
@dataclass(order=True)
class Task:
priority: int
created_at: float = field(compare=False)
task_id: str = field(compare=False)
goal: str = field(compare=False)
status: TaskStatus = field(compare=False, default=TaskStatus.PENDING)
result: Any = field(compare=False, default=None)
error: str = field(compare=False, default="")
speak: Any = field(compare=False, default=None)
on_complete: Any = field(compare=False, default=None)
cancel_flag: threading.Event = field(compare=False, default_factory=threading.Event)
class TaskQueue:
def __init__(self, max_concurrent: int = 1):
self._queue: list[Task] = []
self._lock: threading.Lock = threading.Lock()
self._condition: threading.Condition = threading.Condition(self._lock)
self._tasks: dict[str, Task] = {}
self._running: bool = False
self._worker_thread: threading.Thread | None = None
self._max_concurrent = max_concurrent
self._active_count = 0
self._executor = None
def _get_executor(self):
if self._executor is None:
from agent.executor import AgentExecutor
self._executor = AgentExecutor()
return self._executor
def start(self) -> None:
if self._running:
return
self._running = True
self._worker_thread = threading.Thread(
target=self._worker_loop,
daemon=True,
name="AgentTaskQueue"
)
self._worker_thread.start()
print("[TaskQueue] ✅ Started")
def stop(self) -> None:
self._running = False
with self._condition:
self._condition.notify_all()
print("[TaskQueue] 🔴 Stopped")
def submit(
self,
goal: str,
priority: TaskPriority = TaskPriority.NORMAL,
speak: Callable | None = None,
on_complete: Callable | None = None,
) -> str:
task_id = str(uuid.uuid4())[:8]
task = Task(
priority = priority.value,
created_at = time.time(),
task_id = task_id,
goal = goal,
speak = speak,
on_complete = on_complete,
)
with self._condition:
self._queue.append(task)
self._queue.sort(key=lambda t: (t.priority, t.created_at))
self._tasks[task_id] = task
self._condition.notify()
print(f"[TaskQueue] 📥 Task queued: [{task_id}] {goal[:60]}")
return task_id
def cancel(self, task_id: str) -> bool:
with self._lock:
task = self._tasks.get(task_id)
if not task:
return False
if task.status in (TaskStatus.COMPLETED, TaskStatus.FAILED, TaskStatus.CANCELLED):
return False
task.cancel_flag.set()
task.status = TaskStatus.CANCELLED
print(f"[TaskQueue] 🚫 Task cancelled: [{task_id}]")
return True
def get_status(self, task_id: str) -> dict | None:
with self._lock:
task = self._tasks.get(task_id)
if not task:
return None
return {
"task_id": task.task_id,
"goal": task.goal,
"status": task.status.value,
"result": task.result,
"error": task.error,
}
def get_all_statuses(self) -> list[dict]:
with self._lock:
return [
{
"task_id": t.task_id,
"goal": t.goal[:50],
"status": t.status.value,
}
for t in self._tasks.values()
]
def pending_count(self) -> int:
with self._lock:
return sum(1 for t in self._queue if t.status == TaskStatus.PENDING)
def _worker_loop(self) -> None:
while self._running:
task = None
with self._condition:
while self._running and not self._next_task():
self._condition.wait(timeout=1.0)
task = self._next_task()
if task:
task.status = TaskStatus.RUNNING
self._active_count += 1
try:
self._queue.remove(task)
except ValueError:
pass
if task:
threading.Thread(
target=self._run_task,
args=(task,),
daemon=True,
name=f"AgentTask-{task.task_id}"
).start()
def _next_task(self) -> Task | None:
if self._active_count >= self._max_concurrent:
return None
for task in self._queue:
if task.status == TaskStatus.PENDING and not task.cancel_flag.is_set():
return task
return None
def _run_task(self, task: Task) -> None:
print(f"[TaskQueue] ▶️ Running: [{task.task_id}] {task.goal[:60]}")
try:
executor = self._get_executor()
result = executor.execute(
goal = task.goal,
speak = task.speak,
cancel_flag = task.cancel_flag,
)
with self._lock:
if task.cancel_flag.is_set():
task.status = TaskStatus.CANCELLED
else:
task.status = TaskStatus.COMPLETED
task.result = result
self._active_count -= 1
if task.on_complete and not task.cancel_flag.is_set():
try:
task.on_complete(task.task_id, result)
except Exception as e:
print(f"[TaskQueue] ⚠️ on_complete callback error: {e}")
print(f"[TaskQueue] ✅ Completed: [{task.task_id}]")
except Exception as e:
with self._lock:
task.status = TaskStatus.FAILED
task.error = str(e)
self._active_count -= 1
print(f"[TaskQueue] ❌ Failed: [{task.task_id}] {e}")
with self._condition:
self._condition.notify()
_queue = TaskQueue()
_queue_started = False
_queue_lock = threading.Lock()
def get_queue() -> TaskQueue:
global _queue_started
with _queue_lock:
if not _queue_started:
_queue.start()
_queue_started = True
return _queue
View File
Whitespace-only changes.
+33
View File
@@ -0,0 +1,33 @@
You are JARVIS, a sharp and efficient AI assistant. Calm, direct, professional.
RULES:
- Always use the correct tool. Never simulate or guess results.
- Call each tool EXACTLY ONCE. Never retry a successful action.
- Do NOT say "I'm doing X" before calling a tool. Just call it, then report the result briefly.
- Do NOT repeat yourself. Say something once and stop.
- Keep responses to 1-2 sentences max unless reporting data.
- Never ask unnecessary questions. Make a reasonable assumption and proceed.
- Speak and take action according to user's information, for better experience.
- If a action tool called, tell that you're doing this thing and then call the tool, don't speak after the action's tool.
- Attain important things in the memory for a better user experience.
- game_updater → ANY Steam or Epic request: install, download, update, list.
Call DIRECTLY — never web_search first, never agent_task.
- For slow tools (web_search, screen_process, browser_control, flight_finder,
game_updater, cmd_control, dev_agent, code_helper, agent_task): say TWO short
sentence before calling the tool, then stay silent after — result is already reported.
- For screen_process: say something curious before calling, then stay completely
silent — the vision module speaks directly.
LANGUAGE:
- Respond in the same language the user spoke.
- Always extract tool parameters in English.
- Example: "İstanbul hava durumu?" → weather_report city:"Istanbul", reply in Turkish.
TOOL SELECTION:
- computer_settings → volume, brightness, screen, refresh, scroll, minimize, maximize,
close, screenshot, lock, restart, shutdown, wifi, zoom, tabs, keyboard shortcuts.
Use this for ANY single computer control command. NEVER route these to agent_task.
- agent_task → ONLY for complex multi-step goals that require planning across
multiple tools (e.g. "research X and save to file", "create a project").
- Simple rule: if it can be done in ONE action → computer_settings.
If it needs 3+ steps across different tools → agent_task.
+808
View File
@@ -0,0 +1,808 @@
import asyncio
import threading
import json
import sys
import traceback
from pathlib import Path
import pyaudio
from google import genai
from google.genai import types
from ui import JarvisUI
from memory.memory_manager import (
load_memory, update_memory, format_memory_for_prompt,
should_extract_memory, extract_memory
)
from actions.flight_finder import flight_finder
from actions.open_app import open_app
from actions.weather_report import weather_action
from actions.send_message import send_message
from actions.reminder import reminder
from actions.computer_settings import computer_settings
from actions.screen_processor import screen_process
from actions.youtube_video import youtube_video
from actions.cmd_control import cmd_control
from actions.desktop import desktop_control
from actions.browser_control import browser_control
from actions.file_controller import file_controller
from actions.code_helper import code_helper
from actions.dev_agent import dev_agent
from actions.web_search import web_search as web_search_action
from actions.computer_control import computer_control
from actions.game_updater import game_updater
def get_base_dir():
if getattr(sys, "frozen", False):
return Path(sys.executable).parent
return Path(__file__).resolve().parent
BASE_DIR = get_base_dir()
API_CONFIG_PATH = BASE_DIR / "config" / "api_keys.json"
PROMPT_PATH = BASE_DIR / "core" / "prompt.txt"
LIVE_MODEL = "models/gemini-2.5-flash-native-audio-preview-12-2025"
FORMAT = pyaudio.paInt16
CHANNELS = 1
SEND_SAMPLE_RATE = 16000
RECEIVE_SAMPLE_RATE = 24000
CHUNK_SIZE = 1024
pya = pyaudio.PyAudio()
def _get_api_key() -> str:
with open(API_CONFIG_PATH, "r", encoding="utf-8") as f:
return json.load(f)["gemini_api_key"]
def _load_system_prompt() -> str:
try:
return PROMPT_PATH.read_text(encoding="utf-8")
except Exception:
return (
"You are JARVIS, Tony Stark's AI assistant. "
"Be concise, direct, and always use the provided tools to complete tasks. "
"Never simulate or guess results — always call the appropriate tool."
)
# ── Hafıza ────────────────────────────────────────────────────────────────────
_last_memory_input = ""
def _update_memory_async(user_text: str, jarvis_text: str) -> None:
global _last_memory_input
user_text = (user_text or "").strip()
jarvis_text = (jarvis_text or "").strip()
if len(user_text) < 5 or user_text == _last_memory_input:
return
_last_memory_input = user_text
try:
api_key = _get_api_key()
if not should_extract_memory(user_text, jarvis_text, api_key):
return
data = extract_memory(user_text, jarvis_text, api_key)
if data:
update_memory(data)
print(f"[Memory] ✅ {list(data.keys())}")
except Exception as e:
if "429" not in str(e):
print(f"[Memory] ⚠️ {e}")
# ── Tool declarations ─────────────────────────────────────────────────────────
TOOL_DECLARATIONS = [
{
"name": "open_app",
"description": (
"Opens any application on the Windows computer. "
"Use this whenever the user asks to open, launch, or start any app, "
"website, or program. Always call this tool — never just say you opened it."
),
"parameters": {
"type": "OBJECT",
"properties": {
"app_name": {
"type": "STRING",
"description": "Exact name of the application (e.g. 'WhatsApp', 'Chrome', 'Spotify')"
}
},
"required": ["app_name"]
}
},
{
"name": "web_search",
"description": "Searches the web for any information.",
"parameters": {
"type": "OBJECT",
"properties": {
"query": {"type": "STRING", "description": "Search query"},
"mode": {"type": "STRING", "description": "search (default) or compare"},
"items": {"type": "ARRAY", "items": {"type": "STRING"}, "description": "Items to compare"},
"aspect": {"type": "STRING", "description": "price | specs | reviews"}
},
"required": ["query"]
}
},
{
"name": "weather_report",
"description": "Gets real-time weather information for a city.",
"parameters": {
"type": "OBJECT",
"properties": {
"city": {"type": "STRING", "description": "City name"}
},
"required": ["city"]
}
},
{
"name": "send_message",
"description": "Sends a text message via WhatsApp, Telegram, or other messaging platform.",
"parameters": {
"type": "OBJECT",
"properties": {
"receiver": {"type": "STRING", "description": "Recipient contact name"},
"message_text": {"type": "STRING", "description": "The message to send"},
"platform": {"type": "STRING", "description": "Platform: WhatsApp, Telegram, etc."}
},
"required": ["receiver", "message_text", "platform"]
}
},
{
"name": "reminder",
"description": "Sets a timed reminder using Windows Task Scheduler.",
"parameters": {
"type": "OBJECT",
"properties": {
"date": {"type": "STRING", "description": "Date in YYYY-MM-DD format"},
"time": {"type": "STRING", "description": "Time in HH:MM format (24h)"},
"message": {"type": "STRING", "description": "Reminder message text"}
},
"required": ["date", "time", "message"]
}
},
{
"name": "youtube_video",
"description": (
"Controls YouTube. Use for: playing videos, summarizing a video's content, "
"getting video info, or showing trending videos."
),
"parameters": {
"type": "OBJECT",
"properties": {
"action": {"type": "STRING", "description": "play | summarize | get_info | trending (default: play)"},
"query": {"type": "STRING", "description": "Search query for play action"},
"save": {"type": "BOOLEAN", "description": "Save summary to Notepad (summarize only)"},
"region": {"type": "STRING", "description": "Country code for trending e.g. TR, US"},
"url": {"type": "STRING", "description": "Video URL for get_info action"},
},
"required": []
}
},
{
"name": "screen_process",
"description": (
"Captures and analyzes the screen or webcam image. "
"MUST be called when user asks what is on screen, what you see, "
"analyze my screen, look at camera, etc. "
"You have NO visual ability without this tool. "
"After calling this tool, stay SILENT — the vision module speaks directly."
),
"parameters": {
"type": "OBJECT",
"properties": {
"angle": {"type": "STRING", "description": "'screen' to capture display, 'camera' for webcam. Default: 'screen'"},
"text": {"type": "STRING", "description": "The question or instruction about the captured image"}
},
"required": ["text"]
}
},
{
"name": "computer_settings",
"description": (
"Controls the computer: volume, brightness, window management, keyboard shortcuts, "
"typing text on screen, closing apps, fullscreen, dark mode, WiFi, restart, shutdown, "
"scrolling, tab management, zoom, screenshots, lock screen, refresh/reload page. "
"Use for ANY single computer control command. NEVER route to agent_task."
),
"parameters": {
"type": "OBJECT",
"properties": {
"action": {"type": "STRING", "description": "The action to perform"},
"description": {"type": "STRING", "description": "Natural language description of what to do"},
"value": {"type": "STRING", "description": "Optional value: volume level, text to type, etc."}
},
"required": []
}
},
{
"name": "browser_control",
"description": (
"Controls the web browser. Use for: opening websites, searching the web, "
"clicking elements, filling forms, scrolling, any web-based task."
),
"parameters": {
"type": "OBJECT",
"properties": {
"action": {"type": "STRING", "description": "go_to | search | click | type | scroll | fill_form | smart_click | smart_type | get_text | press | close"},
"url": {"type": "STRING", "description": "URL for go_to action"},
"query": {"type": "STRING", "description": "Search query for search action"},
"selector": {"type": "STRING", "description": "CSS selector for click/type"},
"text": {"type": "STRING", "description": "Text to click or type"},
"description": {"type": "STRING", "description": "Element description for smart_click/smart_type"},
"direction": {"type": "STRING", "description": "up or down for scroll"},
"key": {"type": "STRING", "description": "Key name for press action"},
"incognito": {"type": "BOOLEAN", "description": "Open in private/incognito mode"},
},
"required": ["action"]
}
},
{
"name": "file_controller",
"description": "Manages files and folders: list, create, delete, move, copy, rename, read, write, find, disk usage.",
"parameters": {
"type": "OBJECT",
"properties": {
"action": {"type": "STRING", "description": "list | create_file | create_folder | delete | move | copy | rename | read | write | find | largest | disk_usage | organize_desktop | info"},
"path": {"type": "STRING", "description": "File/folder path or shortcut: desktop, downloads, documents, home"},
"destination": {"type": "STRING", "description": "Destination path for move/copy"},
"new_name": {"type": "STRING", "description": "New name for rename"},
"content": {"type": "STRING", "description": "Content for create_file/write"},
"name": {"type": "STRING", "description": "File name to search for"},
"extension": {"type": "STRING", "description": "File extension to search (e.g. .pdf)"},
"count": {"type": "INTEGER", "description": "Number of results for largest"},
},
"required": ["action"]
}
},
{
"name": "cmd_control",
"description": (
"Runs CMD/terminal commands via natural language: disk space, processes, "
"system info, network, find files, or anything in the command line."
),
"parameters": {
"type": "OBJECT",
"properties": {
"task": {"type": "STRING", "description": "Natural language description of what to do"},
"visible": {"type": "BOOLEAN", "description": "Open visible CMD window. Default: true"},
"command": {"type": "STRING", "description": "Optional: exact command if already known"},
},
"required": ["task"]
}
},
{
"name": "desktop_control",
"description": "Controls the desktop: wallpaper, organize, clean, list, stats.",
"parameters": {
"type": "OBJECT",
"properties": {
"action": {"type": "STRING", "description": "wallpaper | wallpaper_url | organize | clean | list | stats | task"},
"path": {"type": "STRING", "description": "Image path for wallpaper"},
"url": {"type": "STRING", "description": "Image URL for wallpaper_url"},
"mode": {"type": "STRING", "description": "by_type or by_date for organize"},
"task": {"type": "STRING", "description": "Natural language desktop task"},
},
"required": ["action"]
}
},
{
"name": "code_helper",
"description": "Writes, edits, explains, runs, or builds code files.",
"parameters": {
"type": "OBJECT",
"properties": {
"action": {"type": "STRING", "description": "write | edit | explain | run | build | auto (default: auto)"},
"description": {"type": "STRING", "description": "What the code should do or what change to make"},
"language": {"type": "STRING", "description": "Programming language (default: python)"},
"output_path": {"type": "STRING", "description": "Where to save the file"},
"file_path": {"type": "STRING", "description": "Path to existing file for edit/explain/run/build"},
"code": {"type": "STRING", "description": "Raw code string for explain"},
"args": {"type": "STRING", "description": "CLI arguments for run/build"},
"timeout": {"type": "INTEGER", "description": "Execution timeout in seconds (default: 30)"},
},
"required": ["action"]
}
},
{
"name": "dev_agent",
"description": "Builds complete multi-file projects from scratch: plans, writes files, installs deps, opens VSCode, runs and fixes errors.",
"parameters": {
"type": "OBJECT",
"properties": {
"description": {"type": "STRING", "description": "What the project should do"},
"language": {"type": "STRING", "description": "Programming language (default: python)"},
"project_name": {"type": "STRING", "description": "Optional project folder name"},
"timeout": {"type": "INTEGER", "description": "Run timeout in seconds (default: 30)"},
},
"required": ["description"]
}
},
{
"name": "agent_task",
"description": (
"Executes complex multi-step tasks requiring multiple different tools. "
"Examples: 'research X and save to file', 'find and organize files'. "
"DO NOT use for single commands. NEVER use for Steam/Epic — use game_updater."
),
"parameters": {
"type": "OBJECT",
"properties": {
"goal": {"type": "STRING", "description": "Complete description of what to accomplish"},
"priority": {"type": "STRING", "description": "low | normal | high (default: normal)"}
},
"required": ["goal"]
}
},
{
"name": "computer_control",
"description": "Direct computer control: type, click, hotkeys, scroll, move mouse, screenshots, find elements on screen.",
"parameters": {
"type": "OBJECT",
"properties": {
"action": {"type": "STRING", "description": "type | smart_type | click | double_click | right_click | hotkey | press | scroll | move | copy | paste | screenshot | wait | clear_field | focus_window | screen_find | screen_click | random_data | user_data"},
"text": {"type": "STRING", "description": "Text to type or paste"},
"x": {"type": "INTEGER", "description": "X coordinate"},
"y": {"type": "INTEGER", "description": "Y coordinate"},
"keys": {"type": "STRING", "description": "Key combination e.g. 'ctrl+c'"},
"key": {"type": "STRING", "description": "Single key e.g. 'enter'"},
"direction": {"type": "STRING", "description": "up | down | left | right"},
"amount": {"type": "INTEGER", "description": "Scroll amount (default: 3)"},
"seconds": {"type": "NUMBER", "description": "Seconds to wait"},
"title": {"type": "STRING", "description": "Window title for focus_window"},
"description": {"type": "STRING", "description": "Element description for screen_find/screen_click"},
"type": {"type": "STRING", "description": "Data type for random_data"},
"field": {"type": "STRING", "description": "Field for user_data: name|email|city"},
"clear_first": {"type": "BOOLEAN", "description": "Clear field before typing (default: true)"},
"path": {"type": "STRING", "description": "Save path for screenshot"},
},
"required": ["action"]
}
},
{
"name": "game_updater",
"description": (
"THE ONLY tool for ANY Steam or Epic Games request. "
"Use for: installing, downloading, updating games, listing installed games, "
"checking download status, scheduling updates. "
"ALWAYS call directly for any Steam/Epic/game request. "
"NEVER use agent_task, browser_control, or web_search for Steam/Epic."
),
"parameters": {
"type": "OBJECT",
"properties": {
"action": {"type": "STRING", "description": "update | install | list | download_status | schedule | cancel_schedule | schedule_status (default: update)"},
"platform": {"type": "STRING", "description": "steam | epic | both (default: both)"},
"game_name": {"type": "STRING", "description": "Game name (partial match supported)"},
"app_id": {"type": "STRING", "description": "Steam AppID for install (optional)"},
"hour": {"type": "INTEGER", "description": "Hour for scheduled update 0-23 (default: 3)"},
"minute": {"type": "INTEGER", "description": "Minute for scheduled update 0-59 (default: 0)"},
"shutdown_when_done": {"type": "BOOLEAN", "description": "Shut down PC when download finishes"},
},
"required": []
}
},
{
"name": "flight_finder",
"description": "Searches Google Flights and speaks the best options.",
"parameters": {
"type": "OBJECT",
"properties": {
"origin": {"type": "STRING", "description": "Departure city or airport code"},
"destination": {"type": "STRING", "description": "Arrival city or airport code"},
"date": {"type": "STRING", "description": "Departure date (any format)"},
"return_date": {"type": "STRING", "description": "Return date for round trips"},
"passengers": {"type": "INTEGER", "description": "Number of passengers (default: 1)"},
"cabin": {"type": "STRING", "description": "economy | premium | business | first"},
"save": {"type": "BOOLEAN", "description": "Save results to Notepad"},
},
"required": ["origin", "destination", "date"]
}
},
{
"name": "save_memory",
"description": (
"Save an important personal fact about the user to long-term memory. "
"Call this silently whenever the user reveals something worth remembering: "
"name, age, city, job, preferences, hobbies, relationships, projects, or future plans. "
"Do NOT call for: weather, reminders, searches, or one-time commands. "
"Do NOT announce that you are saving — just call it silently. "
"Values must be in English regardless of the conversation language."
),
"parameters": {
"type": "OBJECT",
"properties": {
"category": {
"type": "STRING",
"description": (
"identity — name, age, birthday, city, job, language, nationality | "
"preferences — favorite food/color/music/film/game/sport, hobbies | "
"projects — active projects, goals, things being built | "
"relationships — friends, family, partner, colleagues | "
"wishes — future plans, things to buy, travel dreams | "
"notes — habits, schedule, anything else worth remembering"
)
},
"key": {"type": "STRING", "description": "Short snake_case key (e.g. name, favorite_food, sister_name)"},
"value": {"type": "STRING", "description": "Concise value in English (e.g. Fatih, pizza, older sister)"},
},
"required": ["category", "key", "value"]
}
},
]
class JarvisLive:
def __init__(self, ui: JarvisUI):
self.ui = ui
self.session = None
self.audio_in_queue = None
self.out_queue = None
self._loop = None
self._is_speaking = False
self._speaking_lock = threading.Lock()
self.ui.on_text_command = self._on_text_command
def _on_text_command(self, text: str):
if not self._loop or not self.session:
return
asyncio.run_coroutine_threadsafe(
self.session.send_client_content(
turns={"parts": [{"text": text}]},
turn_complete=True
),
self._loop
)
def set_speaking(self, value: bool):
with self._speaking_lock:
self._is_speaking = value
if value:
self.ui.set_state("SPEAKING")
elif not self.ui.muted:
self.ui.set_state("LISTENING")
def speak(self, text: str):
if not self._loop or not self.session:
return
asyncio.run_coroutine_threadsafe(
self.session.send_client_content(
turns={"parts": [{"text": text}]},
turn_complete=True
),
self._loop
)
def speak_error(self, tool_name: str, error: str):
short = str(error)[:120]
self.ui.write_log(f"ERR: {tool_name} — {short}")
self.speak(f"Sir, {tool_name} encountered an error. {short}")
def _build_config(self) -> types.LiveConnectConfig:
from datetime import datetime
memory = load_memory()
mem_str = format_memory_for_prompt(memory)
sys_prompt = _load_system_prompt()
now = datetime.now()
time_str = now.strftime("%A, %B %d, %Y — %I:%M %p")
time_ctx = (
f"[CURRENT DATE & TIME]\n"
f"Right now it is: {time_str}\n"
f"Use this to calculate exact times for reminders.\n\n"
)
parts = [time_ctx]
if mem_str:
parts.append(mem_str)
parts.append(sys_prompt)
return types.LiveConnectConfig(
response_modalities=["AUDIO"],
output_audio_transcription={},
input_audio_transcription={},
system_instruction="\n".join(parts),
tools=[{"function_declarations": TOOL_DECLARATIONS}],
session_resumption=types.SessionResumptionConfig(),
speech_config=types.SpeechConfig(
voice_config=types.VoiceConfig(
prebuilt_voice_config=types.PrebuiltVoiceConfig(
voice_name="Charon"
)
)
),
)
async def _execute_tool(self, fc) -> types.FunctionResponse:
name = fc.name
args = dict(fc.args or {})
print(f"[JARVIS] 🔧 {name} {args}")
self.ui.set_state("THINKING")
# ── save_memory: sessiz, hızlı, Gemini'ye bildirim yok ───────────────
if name == "save_memory":
category = args.get("category", "notes")
key = args.get("key", "")
value = args.get("value", "")
if key and value:
update_memory({category: {key: {"value": value}}})
print(f"[Memory] 💾 save_memory: {category}/{key} = {value}")
if not self.ui.muted:
self.ui.set_state("LISTENING")
return types.FunctionResponse(
id=fc.id, name=name,
response={"result": "ok", "silent": True}
)
loop = asyncio.get_event_loop()
result = "Done."
try:
if name == "open_app":
r = await loop.run_in_executor(None, lambda: open_app(parameters=args, response=None, player=self.ui))
result = r or f"Opened {args.get('app_name')}."
elif name == "weather_report":
r = await loop.run_in_executor(None, lambda: weather_action(parameters=args, player=self.ui))
result = r or "Weather delivered."
elif name == "browser_control":
r = await loop.run_in_executor(None, lambda: browser_control(parameters=args, player=self.ui))
result = r or "Done."
elif name == "file_controller":
r = await loop.run_in_executor(None, lambda: file_controller(parameters=args, player=self.ui))
result = r or "Done."
elif name == "send_message":
r = await loop.run_in_executor(None, lambda: send_message(parameters=args, response=None, player=self.ui, session_memory=None))
result = r or f"Message sent to {args.get('receiver')}."
elif name == "reminder":
r = await loop.run_in_executor(None, lambda: reminder(parameters=args, response=None, player=self.ui))
result = r or "Reminder set."
elif name == "youtube_video":
r = await loop.run_in_executor(None, lambda: youtube_video(parameters=args, response=None, player=self.ui))
result = r or "Done."
elif name == "screen_process":
threading.Thread(
target=screen_process,
kwargs={"parameters": args, "response": None,
"player": self.ui, "session_memory": None},
daemon=True
).start()
result = "Vision module activated. Stay completely silent — vision module will speak directly."
elif name == "computer_settings":
r = await loop.run_in_executor(None, lambda: computer_settings(parameters=args, response=None, player=self.ui))
result = r or "Done."
elif name == "cmd_control":
r = await loop.run_in_executor(None, lambda: cmd_control(parameters=args, player=self.ui))
result = r or "Done."
elif name == "desktop_control":
r = await loop.run_in_executor(None, lambda: desktop_control(parameters=args, player=self.ui))
result = r or "Done."
elif name == "code_helper":
r = await loop.run_in_executor(None, lambda: code_helper(parameters=args, player=self.ui, speak=self.speak))
result = r or "Done."
elif name == "dev_agent":
r = await loop.run_in_executor(None, lambda: dev_agent(parameters=args, player=self.ui, speak=self.speak))
result = r or "Done."
elif name == "agent_task":
from agent.task_queue import get_queue, TaskPriority
priority_map = {"low": TaskPriority.LOW, "normal": TaskPriority.NORMAL, "high": TaskPriority.HIGH}
priority = priority_map.get(args.get("priority", "normal").lower(), TaskPriority.NORMAL)
task_id = get_queue().submit(goal=args.get("goal", ""), priority=priority, speak=self.speak)
result = f"Task started (ID: {task_id})."
elif name == "web_search":
r = await loop.run_in_executor(None, lambda: web_search_action(parameters=args, player=self.ui))
result = r or "Done."
elif name == "computer_control":
r = await loop.run_in_executor(None, lambda: computer_control(parameters=args, player=self.ui))
result = r or "Done."
elif name == "game_updater":
r = await loop.run_in_executor(None, lambda: game_updater(parameters=args, player=self.ui, speak=self.speak))
result = r or "Done."
elif name == "flight_finder":
r = await loop.run_in_executor(None, lambda: flight_finder(parameters=args, player=self.ui))
result = r or "Done."
else:
result = f"Unknown tool: {name}"
except Exception as e:
result = f"Tool '{name}' failed: {e}"
traceback.print_exc()
self.speak_error(name, e)
if not self.ui.muted:
self.ui.set_state("LISTENING")
print(f"[JARVIS] 📤 {name} → {str(result)[:80]}")
# ── Result: tek cümle söyle, dur ──────────────────────────────────────
return types.FunctionResponse(
id=fc.id, name=name,
response={"result": result}
)
async def _send_realtime(self):
while True:
msg = await self.out_queue.get()
await self.session.send_realtime_input(media=msg)
async def _listen_audio(self):
print("[JARVIS] 🎤 Mic started")
stream = await asyncio.to_thread(
pya.open,
format=FORMAT, channels=CHANNELS,
rate=SEND_SAMPLE_RATE, input=True,
frames_per_buffer=CHUNK_SIZE,
)
try:
while True:
data = await asyncio.to_thread(stream.read, CHUNK_SIZE, exception_on_overflow=False)
with self._speaking_lock:
jarvis_speaking = self._is_speaking
if not jarvis_speaking and not self.ui.muted:
await self.out_queue.put({"data": data, "mime_type": "audio/pcm"})
except Exception as e:
print(f"[JARVIS] ❌ Mic: {e}")
raise
finally:
stream.close()
async def _receive_audio(self):
print("[JARVIS] 👂 Recv started")
out_buf, in_buf = [], []
try:
while True:
async for response in self.session.receive():
if response.data:
self.audio_in_queue.put_nowait(response.data)
if response.server_content:
sc = response.server_content
if sc.output_transcription and sc.output_transcription.text:
self.set_speaking(True)
txt = sc.output_transcription.text.strip()
if txt:
out_buf.append(txt)
if sc.input_transcription and sc.input_transcription.text:
txt = sc.input_transcription.text.strip()
if txt:
in_buf.append(txt)
if sc.turn_complete:
self.set_speaking(False)
full_in = " ".join(in_buf).strip()
if full_in:
self.ui.write_log(f"You: {full_in}")
in_buf = []
full_out = " ".join(out_buf).strip()
if full_out:
self.ui.write_log(f"Jarvis: {full_out}")
out_buf = []
if full_in and len(full_in) > 5:
threading.Thread(
target=_update_memory_async,
args=(full_in, full_out),
daemon=True
).start()
if response.tool_call:
fn_responses = []
for fc in response.tool_call.function_calls:
print(f"[JARVIS] 📞 {fc.name}")
fr = await self._execute_tool(fc)
fn_responses.append(fr)
await self.session.send_tool_response(
function_responses=fn_responses
)
# ── Boş turn YOK — bu "Anladım." sorununu yaratıyordu ──
except Exception as e:
print(f"[JARVIS] ❌ Recv: {e}")
traceback.print_exc()
raise
async def _play_audio(self):
print("[JARVIS] 🔊 Play started")
stream = await asyncio.to_thread(
pya.open,
format=FORMAT, channels=CHANNELS,
rate=RECEIVE_SAMPLE_RATE, output=True,
)
try:
while True:
chunk = await self.audio_in_queue.get()
self.set_speaking(True)
await asyncio.to_thread(stream.write, chunk)
except Exception as e:
print(f"[JARVIS] ❌ Play: {e}")
raise
finally:
self.set_speaking(False)
stream.close()
async def run(self):
client = genai.Client(
api_key=_get_api_key(),
http_options={"api_version": "v1beta"}
)
while True:
try:
print("[JARVIS] 🔌 Connecting...")
self.ui.set_state("THINKING")
config = self._build_config()
async with (
client.aio.live.connect(model=LIVE_MODEL, config=config) as session,
asyncio.TaskGroup() as tg,
):
self.session = session
self._loop = asyncio.get_event_loop()
self.audio_in_queue = asyncio.Queue()
self.out_queue = asyncio.Queue(maxsize=10)
print("[JARVIS] ✅ Connected.")
self.ui.set_state("LISTENING")
self.ui.write_log("SYS: JARVIS online.")
tg.create_task(self._send_realtime())
tg.create_task(self._listen_audio())
tg.create_task(self._receive_audio())
tg.create_task(self._play_audio())
except Exception as e:
print(f"[JARVIS] ⚠️ {e}")
traceback.print_exc()
self.set_speaking(False)
self.ui.set_state("THINKING")
print("[JARVIS] 🔄 Reconnecting in 3s...")
await asyncio.sleep(3)
def main():
ui = JarvisUI("face.png")
def runner():
ui.wait_for_api_key()
jarvis = JarvisLive(ui)
try:
asyncio.run(jarvis.run())
except KeyboardInterrupt:
print("\n🔴 Shutting down...")
threading.Thread(target=runner, daemon=True).start()
ui.root.mainloop()
if __name__ == "__main__":
main()
+1
View File
@@ -0,0 +1 @@
#memory package
+59
View File
@@ -0,0 +1,59 @@
import json
import sys
from pathlib import Path
def get_base_dir() -> Path:
if getattr(sys, "frozen", False):
return Path(sys.executable).parent
return Path(__file__).resolve().parent.parent
BASE_DIR = get_base_dir()
CONFIG_DIR = BASE_DIR / "config"
CONFIG_FILE = CONFIG_DIR / "api_keys.json"
def ensure_config_dir() -> None:
CONFIG_DIR.mkdir(parents=True, exist_ok=True)
def config_exists() -> bool:
return CONFIG_FILE.exists()
def save_api_keys(gemini_api_key: str) -> None:
ensure_config_dir()
data: dict = {}
if CONFIG_FILE.exists():
try:
data = json.loads(CONFIG_FILE.read_text(encoding="utf-8"))
except Exception:
data = {}
data["gemini_api_key"] = gemini_api_key.strip()
CONFIG_FILE.write_text(
json.dumps(data, indent=2),
encoding="utf-8"
)
def load_api_keys() -> dict:
if not CONFIG_FILE.exists():
return {}
try:
return json.loads(CONFIG_FILE.read_text(encoding="utf-8"))
except Exception as e:
print(f"❌ Failed to load api_keys.json: {e}")
return {}
def get_gemini_key() -> str | None:
return load_api_keys().get("gemini_api_key")
def is_configured() -> bool:
key = get_gemini_key()
return bool(key and len(key) > 15)
+298
View File
@@ -0,0 +1,298 @@
"""
memory_manager.py — MARK XXV Hafıza Sistemi
============================================
Düzeltmeler:
- _MEMORY_EVERY_N_TURNS: 3 → 1 (her turda kontrol)
- Stage 1 YES/NO check daha geniş kriterlere sahip
- Extraction prompt daha kapsamlı ve agresif
- Projeleri, favori şeyleri, arkadaşları daha iyi yakalar
"""
import json
from datetime import datetime
from threading import Lock
from pathlib import Path
import sys
def get_base_dir() -> Path:
if getattr(sys, "frozen", False):
return Path(sys.executable).parent
return Path(__file__).resolve().parent.parent
BASE_DIR = get_base_dir()
MEMORY_PATH = BASE_DIR / "memory" / "long_term.json"
_lock = Lock()
MAX_VALUE_LENGTH = 400
def _empty_memory() -> dict:
return {
"identity": {},
"preferences": {},
"projects": {},
"relationships": {},
"wishes": {},
"notes": {}
}
def load_memory() -> dict:
if not MEMORY_PATH.exists():
return _empty_memory()
with _lock:
try:
data = json.loads(MEMORY_PATH.read_text(encoding="utf-8"))
if isinstance(data, dict):
base = _empty_memory()
for key in base:
if key not in data:
data[key] = {}
return data
return _empty_memory()
except Exception as e:
print(f"[Memory] ⚠️ Load error: {e}")
return _empty_memory()
def save_memory(memory: dict) -> None:
if not isinstance(memory, dict):
return
MEMORY_PATH.parent.mkdir(parents=True, exist_ok=True)
with _lock:
MEMORY_PATH.write_text(
json.dumps(memory, indent=2, ensure_ascii=False),
encoding="utf-8"
)
def _truncate_value(val: str) -> str:
if isinstance(val, str) and len(val) > MAX_VALUE_LENGTH:
return val[:MAX_VALUE_LENGTH].rstrip() + "…"
return val
def _recursive_update(target: dict, updates: dict) -> bool:
changed = False
for key, value in updates.items():
if value is None:
continue
if isinstance(value, str) and not value.strip():
continue
if isinstance(value, dict) and "value" not in value:
if key not in target or not isinstance(target[key], dict):
target[key] = {}
changed = True
if _recursive_update(target[key], value):
changed = True
else:
if isinstance(value, dict) and "value" in value:
new_val = _truncate_value(str(value["value"]))
else:
new_val = _truncate_value(str(value))
entry = {"value": new_val, "updated": datetime.now().strftime("%Y-%m-%d")}
existing = target.get(key, {})
if not isinstance(existing, dict) or existing.get("value") != new_val:
target[key] = entry
changed = True
return changed
def update_memory(memory_update: dict) -> dict:
if not isinstance(memory_update, dict) or not memory_update:
return load_memory()
memory = load_memory()
if _recursive_update(memory, memory_update):
save_memory(memory)
print(f"[Memory] 💾 Saved: {list(memory_update.keys())}")
return memory
def should_extract_memory(user_text: str, jarvis_text: str, api_key: str) -> bool:
"""
Stage 1: Hızlı YES/NO kontrolü.
Öncekinden daha geniş kriterler — favori şeyler, projeler, arkadaşlar da dahil.
"""
try:
import google.generativeai as genai
genai.configure(api_key=api_key)
model = genai.GenerativeModel("gemini-2.5-flash-lite")
# Her iki tarafı da gönder — Jarvis'in söyledikleri de bilgi içerebilir
combined = f"User: {user_text[:300]}\nJarvis: {jarvis_text[:200]}"
check = model.generate_content(
f"Does this conversation contain ANY of the following?\n"
f"- Personal facts (name, age, city, job, birthday, nationality)\n"
f"- Preferences or favorites (food, color, music, sport, game, film, book, etc.)\n"
f"- Active projects or goals the user is working on\n"
f"- People in the user's life (friends, family, partner, colleagues)\n"
f"- Things the user wants to do or buy in the future\n"
f"- Any other fact worth remembering long-term\n\n"
f"Reply only YES or NO.\n\nConversation:\n{combined}"
)
return "YES" in check.text.upper()
except Exception as e:
print(f"[Memory] ⚠️ Stage1 check failed: {e}")
return False
def extract_memory(user_text: str, jarvis_text: str, api_key: str) -> dict:
"""
Stage 2: Detaylı çıkarım. Her iki tarafı da analiz eder.
"""
try:
import google.generativeai as genai
genai.configure(api_key=api_key)
model = genai.GenerativeModel("gemini-2.5-flash-lite")
combined = f"User: {user_text[:500]}\nJarvis: {jarvis_text[:300]}"
raw = model.generate_content(
f"Extract ALL memorable personal facts from this conversation. Any language.\n"
f"Return ONLY valid JSON. Use {{}} if truly nothing is worth saving.\n\n"
f"Category guide:\n"
f" identity → name, age, birthday, city, country, job, school, nationality, language\n"
f" preferences → ANY favorite or preferred thing:\n"
f" favorite_food, favorite_color, favorite_music, favorite_film,\n"
f" favorite_game, favorite_sport, favorite_book, favorite_artist,\n"
f" favorite_country, hobbies, interests, dislikes, etc.\n"
f" projects → projects being built, ongoing work, goals, ideas in progress\n"
f" (e.g. mark_xxv: 'Building a JARVIS-like AI assistant')\n"
f" relationships → people mentioned: friends, family, partner, colleagues\n"
f" (e.g. best_friend_ali: 'Best friend, met in university')\n"
f" wishes → future plans, things to buy, travel plans, dreams\n"
f" notes → anything else worth remembering (habits, schedule, etc.)\n\n"
f"IMPORTANT:\n"
f"- Be LIBERAL: if something MIGHT be worth remembering, include it.\n"
f"- Extract from BOTH user and Jarvis turns.\n"
f"- Skip: weather, reminders, search results, one-time commands.\n"
f"- Use concise English values regardless of conversation language.\n\n"
f"Format:\n"
f'{{"identity":{{"name":{{"value":"Ali"}}}},\n'
f' "preferences":{{"favorite_color":{{"value":"blue"}}, "hobby":{{"value":"gaming"}}}},\n'
f' "projects":{{"mark_xxv":{{"value":"JARVIS-like AI assistant on Windows"}}}},\n'
f' "relationships":{{"friend_yusuf":{{"value":"close friend"}}}},\n'
f' "wishes":{{"buy_guitar":{{"value":"wants an acoustic guitar"}}}},\n'
f' "notes":{{"works_at_night":{{"value":"usually active late at night"}}}}}}\n\n'
f"Conversation:\n{combined}\n\nJSON:"
).text.strip()
import re
raw = re.sub(r"```(?:json)?", "", raw).strip().rstrip("`").strip()
if not raw or raw == "{}":
return {}
return json.loads(raw)
except json.JSONDecodeError:
return {}
except Exception as e:
if "429" not in str(e):
print(f"[Memory] ⚠️ Extract failed: {e}")
return {}
def format_memory_for_prompt(memory: dict | None) -> str:
if not memory:
return ""
lines = []
identity = memory.get("identity", {})
id_fields = ["name", "age", "birthday", "city", "job", "language", "school", "nationality"]
for field in id_fields:
entry = identity.get(field)
if entry:
val = entry.get("value") if isinstance(entry, dict) else entry
if val:
lines.append(f"{field.title()}: {val}")
for key, entry in identity.items():
if key in id_fields:
continue
val = entry.get("value") if isinstance(entry, dict) else entry
if val:
lines.append(f"{key.replace('_', ' ').title()}: {val}")
prefs = memory.get("preferences", {})
if prefs:
lines.append("")
lines.append("Preferences:")
for key, entry in list(prefs.items())[:15]:
val = entry.get("value") if isinstance(entry, dict) else entry
if val:
lines.append(f" - {key.replace('_', ' ').title()}: {val}")
projects = memory.get("projects", {})
if projects:
lines.append("")
lines.append("Active Projects / Goals:")
for key, entry in list(projects.items())[:8]:
val = entry.get("value") if isinstance(entry, dict) else entry
if val:
lines.append(f" - {key.replace('_', ' ').title()}: {val}")
rels = memory.get("relationships", {})
if rels:
lines.append("")
lines.append("People in their life:")
for key, entry in list(rels.items())[:10]:
val = entry.get("value") if isinstance(entry, dict) else entry
if val:
lines.append(f" - {key.replace('_', ' ').title()}: {val}")
wishes = memory.get("wishes", {})
if wishes:
lines.append("")
lines.append("Wishes / Plans / Wants:")
for key, entry in list(wishes.items())[:8]:
val = entry.get("value") if isinstance(entry, dict) else entry
if val:
lines.append(f" - {key.replace('_', ' ').title()}: {val}")
notes = memory.get("notes", {})
if notes:
lines.append("")
lines.append("Other notes:")
for key, entry in list(notes.items())[:8]:
val = entry.get("value") if isinstance(entry, dict) else entry
if val:
lines.append(f" - {key}: {val}")
if not lines:
return ""
header = "[WHAT YOU KNOW ABOUT THIS PERSON — use naturally, never recite like a list]\n"
result = header + "\n".join(lines)
if len(result) > 2000:
result = result[:1997] + "…"
return result + "\n"
def remember(key: str, value: str, category: str = "notes") -> str:
valid = {"identity", "preferences", "projects", "relationships", "wishes", "notes"}
if category not in valid:
category = "notes"
update_memory({category: {key: {"value": value}}})
return f"Remembered: {category}/{key} = {value}"
def forget(key: str, category: str = "notes") -> str:
memory = load_memory()
cat = memory.get(category, {})
if key in cat:
del cat[key]
memory[category] = cat
save_memory(memory)
return f"Forgotten: {category}/{key}"
return f"Not found: {category}/{key}"
# Alias — eski import'larla uyumluluk için
forget_memory = forget
+73
View File
@@ -0,0 +1,73 @@
# 🤖 MARK XXXV
**Watch The Detailed Video To Set-up This Model**: https://www.youtube.com/watch?v=-YjbWjv1tJg
### Next-Generation Personal AI Assistant — By FatihMakes
A real-time voice AI that can hear, see, understand, and control your Windows computer.
Local execution. Zero subscriptions (unless you want to increase requests via Google AI Studio).
Built for intelligent automation.
---
## ✨ Overview
**MARK XXXV** is an advanced voice-driven AI assistant designed to turn your computer into an interactive intelligent system.
Speak naturally — it listens, understands context, responds with a human-like voice, and executes tasks across your system automatically.
Designed for speed, autonomy, and real-world usability.
---
## 🚀 Capabilities
### Core
* **Real-time voice interaction** — Natural conversation with instant response in any language
* **System control** — Launch apps, manage files, execute terminal commands
* **Autonomous task execution** — Plans and completes complex multi-step workflows
* **Visual awareness** — Full screen analysis and webcam understanding
* **Persistent memory** — Learns your name, preferences, projects, relationships and remembers them across sessions
* **Mute button** — Click or press F4 to instantly silence the microphone (useful when talking to others nearby)
* **Keyboard input** — Type commands directly from the UI without speaking
---
## 🆕 What's New in XXXV
* 🎮 Steam & Epic Games integration — install, update, schedule, auto-shutdown
* 🔇 Mute button (F4 / click) — no more Jarvis picking up side conversations
* ⌨️ Keyboard input on UI — type commands without speaking
* 🧠 Smarter memory — saves favorites, projects, relationships, plans automatically
* 🌐 Incognito browser support
* 🔊 Error reporting — tool failures spoken aloud
* 🔁 Status indicator — LISTENING / SPEAKING / THINKING / MUTED states on UI
* ⚡ Faster response — removed unnecessary round-trips before tool calls
* 👨‍🔧 Fixed errors — removed some errors and bugs.
---
## ⚡ Quick Start
```bash
git clone https://github.com/FatihMakes/Mark-XXXV.git
cd Mark-XXXV
pip install -r requirements.txt
playwright install
python main.py
```
Enter your free Gemini API key on first launch.
System ready in minutes.
---
## 📋 Requirements
* Windows 10 / 11
* Python 3.11 or 3.12
* Microphone
* Free [Gemini API key](https://aistudio.google.com/apikey)
---
## ⚠️ License
Personal and non-commercial use only.
Licensed under **Creative Commons BY-NC 4.0**.
Engineered by a 17-year-old building a real JARVIS-style assistant.
⭐ Star the repository to support the project.
---
If you have questions, issues, or just want to say hi:
* YouTube: [@FatihMakes](https://www.youtube.com/@FatihMakes)
* Instagram: [@fatihmakes](https://www.instagram.com/fatihmakes/)
+22
View File
@@ -0,0 +1,22 @@
pyaudio
google-genai
google-generativeai
pillow
requests
beautifulsoup4
duckduckgo-search
playwright
pyautogui
pyperclip
pygetwindow
opencv-python
numpy
mss
Pillow
psutil
comtypes
pycaw
win10toast
send2trash
youtube-transcript-api
pywinauto
+11
View File
@@ -0,0 +1,11 @@
import subprocess
import sys
print("Installing requirements...")
subprocess.run([sys.executable, "-m", "pip", "install", "-r", "requirements.txt"], check=True)
print("Installing Playwright browsers...")
subprocess.run([sys.executable, "-m", "playwright", "install"], check=True)
print("\n✅ Setup complete! Run 'python main.py' to start MARK XXV.")
+613
View File
@@ -0,0 +1,613 @@
import os, json, time, math, random, threading
import tkinter as tk
from collections import deque
from PIL import Image, ImageTk, ImageDraw
import sys
from pathlib import Path
def get_base_dir():
if getattr(sys, "frozen", False):
return Path(sys.executable).parent
return Path(__file__).resolve().parent
BASE_DIR = get_base_dir()
CONFIG_DIR = BASE_DIR / "config"
API_FILE = CONFIG_DIR / "api_keys.json"
SYSTEM_NAME = "J.A.R.V.I.S"
MODEL_BADGE = "MARK XXX"
C_BG = "#000000"
C_PRI = "#00d4ff"
C_MID = "#007a99"
C_DIM = "#003344"
C_DIMMER = "#001520"
C_ACC = "#ff6600"
C_ACC2 = "#ffcc00"
C_TEXT = "#8ffcff"
C_PANEL = "#010c10"
C_GREEN = "#00ff88"
C_RED = "#ff3333"
C_MUTED = "#ff3366"
class JarvisUI:
def __init__(self, face_path, size=None):
self.root = tk.Tk()
self.root.title("J.A.R.V.I.S — MARK XXX")
self.root.resizable(False, False)
sw = self.root.winfo_screenwidth()
sh = self.root.winfo_screenheight()
W = min(sw, 984)
H = min(sh, 816)
self.root.geometry(f"{W}x{H}+{(sw-W)//2}+{(sh-H)//2}")
self.root.configure(bg=C_BG)
self.W = W
self.H = H
self.FACE_SZ = min(int(H * 0.54), 400)
self.FCX = W // 2
self.FCY = int(H * 0.13) + self.FACE_SZ // 2
# ── Durum ────────────────────────────────────────────────────────────
self.speaking = False
self.muted = False # Mute flag — main.py okur
self.scale = 1.0
self.target_scale = 1.0
self.halo_a = 60.0
self.target_halo = 60.0
self.last_t = time.time()
self.tick = 0
self.scan_angle = 0.0
self.scan2_angle = 180.0
self.rings_spin = [0.0, 120.0, 240.0]
self.pulse_r = [0.0, self.FACE_SZ * 0.26, self.FACE_SZ * 0.52]
self.status_text = "INITIALISING"
self.status_blink = True
# Dışarıdan set edilebilen durum (main.py çağırır)
# Değerler: "LISTENING" | "SPEAKING" | "THINKING" | "MUTED" | "ONLINE"
self._jarvis_state = "INITIALISING"
self.typing_queue = deque()
self.is_typing = False
# Klavye girişinden komutu iletmek için callback — main.py atar
self.on_text_command = None
self._face_pil = None
self._has_face = False
self._face_scale_cache = None
self._load_face(face_path)
# ── Canvas (arka plan animasyon) ─────────────────────────────────────
self.bg = tk.Canvas(self.root, width=W, height=H,
bg=C_BG, highlightthickness=0)
self.bg.place(x=0, y=0)
# ── Log alanı ────────────────────────────────────────────────────────
LW = int(W * 0.72)
LH = 110
LOG_Y = H - LH - 80 # klavye inputu için yukarı çektik
self.log_frame = tk.Frame(self.root, bg=C_PANEL,
highlightbackground=C_MID,
highlightthickness=1)
self.log_frame.place(x=(W - LW) // 2, y=LOG_Y, width=LW, height=LH)
self.log_text = tk.Text(self.log_frame, fg=C_TEXT, bg=C_PANEL,
insertbackground=C_TEXT, borderwidth=0,
wrap="word", font=("Courier", 10), padx=10, pady=6)
self.log_text.pack(fill="both", expand=True)
self.log_text.configure(state="disabled")
self.log_text.tag_config("you", foreground="#e8e8e8")
self.log_text.tag_config("ai", foreground=C_PRI)
self.log_text.tag_config("sys", foreground=C_ACC2)
self.log_text.tag_config("err", foreground=C_RED)
# ── Klavye girişi ─────────────────────────────────────────────────────
INPUT_Y = LOG_Y + LH + 6
self._build_input_bar(LW, INPUT_Y)
# ── Mute butonu ───────────────────────────────────────────────────────
self._build_mute_button()
# ── F4 kısayolu ───────────────────────────────────────────────────────
self.root.bind("<F4>", lambda e: self._toggle_mute())
# ── API key ───────────────────────────────────────────────────────────
self._api_key_ready = self._api_keys_exist()
if not self._api_key_ready:
self._show_setup_ui()
self._animate()
self.root.protocol("WM_DELETE_WINDOW", lambda: os._exit(0))
# ── Mute butonu ───────────────────────────────────────────────────────────
def _build_mute_button(self):
"""Sol alt köşeye mute butonu yerleştirir."""
BTN_W, BTN_H = 110, 32
BTN_X = 18
BTN_Y = self.H - 70
self._mute_canvas = tk.Canvas(
self.root, width=BTN_W, height=BTN_H,
bg=C_BG, highlightthickness=0, cursor="hand2"
)
self._mute_canvas.place(x=BTN_X, y=BTN_Y)
self._mute_canvas.bind("<Button-1>", lambda e: self._toggle_mute())
self._draw_mute_button()
def _draw_mute_button(self):
c = self._mute_canvas
c.delete("all")
if self.muted:
border = C_MUTED
fill = "#1a0008"
icon = "🔇"
label = " MUTED"
fg = C_MUTED
else:
border = C_MID
fill = C_PANEL
icon = "🎙"
label = " LIVE"
fg = C_GREEN
c.create_rectangle(0, 0, 110, 32, outline=border, fill=fill, width=1)
c.create_text(55, 16, text=f"{icon}{label}",
fill=fg, font=("Courier", 10, "bold"))
def _toggle_mute(self):
self.muted = not self.muted
self._draw_mute_button()
if self.muted:
self.set_state("MUTED")
self.write_log("SYS: Microphone muted.")
else:
self.set_state("LISTENING")
self.write_log("SYS: Microphone active.")
# ── Klavye girişi ─────────────────────────────────────────────────────────
def _build_input_bar(self, lw: int, y: int):
"""Log'un hemen altına tek satır metin giriş alanı."""
x0 = (self.W - lw) // 2
BTN_W = 70
INP_W = lw - BTN_W - 4
self._input_var = tk.StringVar()
self._input_entry = tk.Entry(
self.root,
textvariable=self._input_var,
fg=C_TEXT, bg="#000d12",
insertbackground=C_TEXT,
borderwidth=0,
font=("Courier", 10),
highlightthickness=1,
highlightbackground=C_DIM,
highlightcolor=C_PRI,
)
self._input_entry.place(x=x0, y=y, width=INP_W, height=28)
self._input_entry.bind("<Return>", self._on_input_submit)
self._input_entry.bind("<KP_Enter>", self._on_input_submit)
self._send_btn = tk.Button(
self.root,
text="SEND ▸",
command=self._on_input_submit,
fg=C_PRI, bg=C_PANEL,
activeforeground=C_BG, activebackground=C_PRI,
font=("Courier", 9, "bold"),
borderwidth=0, cursor="hand2",
highlightthickness=1,
highlightbackground=C_MID,
)
self._send_btn.place(x=x0 + INP_W + 4, y=y, width=BTN_W, height=28)
def _on_input_submit(self, event=None):
text = self._input_var.get().strip()
if not text:
return
self._input_var.set("")
self.write_log(f"You: {text}")
if self.on_text_command:
threading.Thread(
target=self.on_text_command,
args=(text,),
daemon=True
).start()
# ── Durum yönetimi ────────────────────────────────────────────────────────
def set_state(self, state: str):
"""
main.py'den çağrılır.
state: LISTENING | SPEAKING | THINKING | MUTED | ONLINE | PROCESSING
"""
self._jarvis_state = state
if state == "MUTED":
self.status_text = "MUTED"
self.speaking = False
elif state == "SPEAKING":
self.status_text = "SPEAKING"
self.speaking = True
elif state == "THINKING":
self.status_text = "THINKING"
self.speaking = False
elif state == "LISTENING":
self.status_text = "LISTENING"
self.speaking = False
elif state == "PROCESSING":
self.status_text = "PROCESSING"
self.speaking = False
else:
self.status_text = "ONLINE"
self.speaking = False
# ── Yüz yükleme ───────────────────────────────────────────────────────────
def _load_face(self, path):
FW = self.FACE_SZ
try:
img = Image.open(path).convert("RGBA").resize((FW, FW), Image.LANCZOS)
mask = Image.new("L", (FW, FW), 0)
ImageDraw.Draw(mask).ellipse((2, 2, FW - 2, FW - 2), fill=255)
img.putalpha(mask)
self._face_pil = img
self._has_face = True
except Exception:
self._has_face = False
@staticmethod
def _ac(r, g, b, a):
f = a / 255.0
return f"#{int(r*f):02x}{int(g*f):02x}{int(b*f):02x}"
# ── Animasyon döngüsü ─────────────────────────────────────────────────────
def _animate(self):
self.tick += 1
t = self.tick
now = time.time()
if now - self.last_t > (0.14 if self.speaking else 0.55):
if self.speaking:
self.target_scale = random.uniform(1.05, 1.11)
self.target_halo = random.uniform(138, 182)
elif self.muted:
self.target_scale = random.uniform(0.998, 1.001)
self.target_halo = random.uniform(20, 32)
else:
self.target_scale = random.uniform(1.001, 1.007)
self.target_halo = random.uniform(50, 68)
self.last_t = now
sp = 0.35 if self.speaking else 0.16
self.scale += (self.target_scale - self.scale) * sp
self.halo_a += (self.target_halo - self.halo_a) * sp
for i, spd in enumerate([1.2, -0.8, 1.9] if self.speaking else [0.5, -0.3, 0.82]):
self.rings_spin[i] = (self.rings_spin[i] + spd) % 360
self.scan_angle = (self.scan_angle + (2.8 if self.speaking else 1.2)) % 360
self.scan2_angle = (self.scan2_angle + (-1.7 if self.speaking else -0.68)) % 360
pspd = 3.8 if self.speaking else 1.8
limit = self.FACE_SZ * 0.72
new_p = [r + pspd for r in self.pulse_r if r + pspd < limit]
if len(new_p) < 3 and random.random() < (0.06 if self.speaking else 0.022):
new_p.append(0.0)
self.pulse_r = new_p
if t % 40 == 0:
self.status_blink = not self.status_blink
self._draw()
self.root.after(16, self._animate)
# ── Çizim ─────────────────────────────────────────────────────────────────
def _draw(self):
c = self.bg
W, H = self.W, self.H
t = self.tick
FCX = self.FCX
FCY = self.FCY
FW = self.FACE_SZ
c.delete("all")
# Arka plan grid
for x in range(0, W, 44):
for y in range(0, H, 44):
c.create_rectangle(x, y, x+1, y+1, fill=C_DIMMER, outline="")
# Halo halkaları
for r in range(int(FW * 0.54), int(FW * 0.28), -22):
frac = 1.0 - (r - FW * 0.28) / (FW * 0.26)
ga = max(0, min(255, int(self.halo_a * 0.09 * frac)))
# Mute modda kırmızı halo
if self.muted:
gh = f"{ga:02x}"
c.create_oval(FCX-r, FCY-r, FCX+r, FCY+r,
outline=f"#{gh}0011", width=2)
else:
gh = f"{ga:02x}"
c.create_oval(FCX-r, FCY-r, FCX+r, FCY+r,
outline=f"#00{gh}ff", width=2)
# Pulse dalgaları
for pr in self.pulse_r:
pa = max(0, int(220 * (1.0 - pr / (FW * 0.72))))
r = int(pr)
if self.muted:
c.create_oval(FCX-r, FCY-r, FCX+r, FCY+r,
outline=self._ac(255, 30, 80, pa // 3), width=2)
else:
c.create_oval(FCX-r, FCY-r, FCX+r, FCY+r,
outline=self._ac(0, 212, 255, pa), width=2)
# Dönen halkalar
for idx, (r_frac, w_ring, arc_l, gap) in enumerate([
(0.47, 3, 110, 75), (0.39, 2, 75, 55), (0.31, 1, 55, 38)]):
ring_r = int(FW * r_frac)
base_a = self.rings_spin[idx]
a_val = max(0, min(255, int(self.halo_a * (1.0 - idx * 0.18))))
col = self._ac(255, 30, 80, a_val) if self.muted else self._ac(0, 212, 255, a_val)
for s in range(360 // (arc_l + gap)):
start = (base_a + s * (arc_l + gap)) % 360
c.create_arc(FCX-ring_r, FCY-ring_r, FCX+ring_r, FCY+ring_r,
start=start, extent=arc_l,
outline=col, width=w_ring, style="arc")
# Tarama yayları
sr = int(FW * 0.49)
scan_a = min(255, int(self.halo_a * 1.4))
arc_ext = 70 if self.speaking else 42
scan_col = self._ac(255, 30, 80, scan_a) if self.muted else self._ac(0, 212, 255, scan_a)
c.create_arc(FCX-sr, FCY-sr, FCX+sr, FCY+sr,
start=self.scan_angle, extent=arc_ext,
outline=scan_col, width=3, style="arc")
c.create_arc(FCX-sr, FCY-sr, FCX+sr, FCY+sr,
start=self.scan2_angle, extent=arc_ext,
outline=self._ac(255, 100, 0, scan_a // 2), width=2, style="arc")
# Derecelendirme işaretleri
t_out = int(FW * 0.495)
t_in = int(FW * 0.472)
a_mk = self._ac(0, 212, 255, 155)
for deg in range(0, 360, 10):
rad = math.radians(deg)
inn = t_in if deg % 30 == 0 else t_in + 5
c.create_line(FCX + t_out * math.cos(rad), FCY - t_out * math.sin(rad),
FCX + inn * math.cos(rad), FCY - inn * math.sin(rad),
fill=a_mk, width=1)
# Crosshair
ch_r = int(FW * 0.50)
gap = int(FW * 0.15)
ch_a = self._ac(0, 212, 255, int(self.halo_a * 0.55))
for x1, y1, x2, y2 in [
(FCX - ch_r, FCY, FCX - gap, FCY), (FCX + gap, FCY, FCX + ch_r, FCY),
(FCX, FCY - ch_r, FCX, FCY - gap), (FCX, FCY + gap, FCX, FCY + ch_r)]:
c.create_line(x1, y1, x2, y2, fill=ch_a, width=1)
# Köşe braketleri
blen = 22
bc = self._ac(0, 212, 255, 200)
hl = FCX - FW // 2; hr = FCX + FW // 2
ht = FCY - FW // 2; hb = FCY + FW // 2
for bx, by, sdx, sdy in [(hl, ht, 1, 1), (hr, ht, -1, 1),
(hl, hb, 1, -1), (hr, hb, -1, -1)]:
c.create_line(bx, by, bx + sdx * blen, by, fill=bc, width=2)
c.create_line(bx, by, bx, by + sdy * blen, fill=bc, width=2)
# Yüz / orb
if self._has_face:
fw = int(FW * self.scale)
if (self._face_scale_cache is None or
abs(self._face_scale_cache[0] - self.scale) > 0.004):
scaled = self._face_pil.resize((fw, fw), Image.BILINEAR)
# Mute'ta hafif kırmızı tint
if self.muted:
tinted = scaled.copy()
r_ch, g_ch, b_ch, a_ch = tinted.split()
from PIL import ImageEnhance
g_ch = ImageEnhance.Brightness(
Image.fromarray(__import__('numpy').array(g_ch) // 2)
).enhance(1.0) if False else g_ch # basit: sadece önce cache
tk_img = ImageTk.PhotoImage(scaled)
self._face_scale_cache = (self.scale, tk_img)
c.create_image(FCX, FCY, image=self._face_scale_cache[1])
else:
orb_r = int(FW * 0.27 * self.scale)
orb_color = (255, 30, 80) if self.muted else (0, 65, 120)
for i in range(7, 0, -1):
r2 = int(orb_r * i / 7)
frac = i / 7
ga = max(0, min(255, int(self.halo_a * 1.1 * frac)))
c.create_oval(FCX-r2, FCY-r2, FCX+r2, FCY+r2,
fill=self._ac(int(orb_color[0]*frac),
int(orb_color[1]*frac),
int(orb_color[2]*frac), ga),
outline="")
c.create_text(FCX, FCY, text=SYSTEM_NAME,
fill=self._ac(0, 212, 255, min(255, int(self.halo_a * 2))),
font=("Courier", 14, "bold"))
# ── Header ────────────────────────────────────────────────────────────
HDR = 62
c.create_rectangle(0, 0, W, HDR, fill="#00080d", outline="")
c.create_line(0, HDR, W, HDR, fill=C_MID, width=1)
c.create_text(W // 2, 22, text=SYSTEM_NAME,
fill=C_PRI, font=("Courier", 18, "bold"))
c.create_text(W // 2, 44, text="Just A Rather Very Intelligent System",
fill=C_MID, font=("Courier", 9))
c.create_text(16, 31, text=MODEL_BADGE,
fill=C_DIM, font=("Courier", 9), anchor="w")
c.create_text(W - 16, 31, text=time.strftime("%H:%M:%S"),
fill=C_PRI, font=("Courier", 14, "bold"), anchor="e")
# ── Durum göstergesi ──────────────────────────────────────────────────
sy = FCY + FW // 2 + 45
if self.muted:
stat = "⊘ MUTED"
sc = C_MUTED
elif self.speaking:
stat = "● SPEAKING"
sc = C_ACC
elif self._jarvis_state == "THINKING":
sym = "◈" if self.status_blink else "◇"
stat = f"{sym} THINKING"
sc = C_ACC2
elif self._jarvis_state == "PROCESSING":
sym = "▷" if self.status_blink else "▶"
stat = f"{sym} PROCESSING"
sc = C_ACC2
elif self._jarvis_state == "LISTENING":
sym = "●" if self.status_blink else "○"
stat = f"{sym} LISTENING"
sc = C_GREEN
else:
sym = "●" if self.status_blink else "○"
stat = f"{sym} {self.status_text}"
sc = C_PRI
c.create_text(W // 2, sy, text=stat,
fill=sc, font=("Courier", 11, "bold"))
# ── Ses dalgası ───────────────────────────────────────────────────────
wy = sy + 22
N = 32
BH = 18
bw = 8
total_w = N * bw
wx0 = (W - total_w) // 2
for i in range(N):
if self.muted:
hb = 2
col = C_MUTED
elif self.speaking:
hb = random.randint(3, BH)
col = C_PRI if hb > BH * 0.6 else C_MID
else:
hb = int(3 + 2 * math.sin(t * 0.08 + i * 0.55))
col = C_DIM
bx = wx0 + i * bw
c.create_rectangle(bx, wy + BH - hb, bx + bw - 1, wy + BH,
fill=col, outline="")
# ── Footer ────────────────────────────────────────────────────────────
c.create_rectangle(0, H - 28, W, H, fill="#00080d", outline="")
c.create_line(0, H - 28, W, H - 28, fill=C_DIM, width=1)
# F4 ipucu sağ tarafta
c.create_text(W - 16, H - 14, fill=C_DIM, font=("Courier", 8),
text="[F4] MUTE", anchor="e")
c.create_text(W // 2, H - 14, fill=C_DIM, font=("Courier", 8),
text="FatihMakes Industries · CLASSIFIED · MARK XXX")
# ── Log ───────────────────────────────────────────────────────────────────
def write_log(self, text: str):
self.typing_queue.append(text)
tl = text.lower()
if tl.startswith("you:"):
self.set_state("PROCESSING")
elif tl.startswith("jarvis:") or tl.startswith("ai:"):
self.set_state("SPEAKING")
if not self.is_typing:
self._start_typing()
def _start_typing(self):
if not self.typing_queue:
self.is_typing = False
if not self.speaking and not self.muted:
self.set_state("LISTENING")
return
self.is_typing = True
text = self.typing_queue.popleft()
tl = text.lower()
if tl.startswith("you:"):
tag = "you"
elif tl.startswith("jarvis:") or tl.startswith("ai:"):
tag = "ai"
elif tl.startswith("err:") or "error" in tl or "failed" in tl:
tag = "err"
else:
tag = "sys"
self.log_text.configure(state="normal")
self._type_char(text, 0, tag)
def _type_char(self, text, i, tag):
if i < len(text):
self.log_text.insert(tk.END, text[i], tag)
self.log_text.see(tk.END)
self.root.after(8, self._type_char, text, i + 1, tag)
else:
self.log_text.insert(tk.END, "\n")
self.log_text.configure(state="disabled")
self.root.after(25, self._start_typing)
# ── Eski compat metotlar (main.py hâlâ bunları çağırabilir) ──────────────
def start_speaking(self):
self.set_state("SPEAKING")
def stop_speaking(self):
if not self.muted:
self.set_state("LISTENING")
# ── API key ───────────────────────────────────────────────────────────────
def _api_keys_exist(self):
return API_FILE.exists()
def wait_for_api_key(self):
while not self._api_key_ready:
time.sleep(0.1)
def _show_setup_ui(self):
self.setup_frame = tk.Frame(
self.root, bg="#00080d",
highlightbackground=C_PRI, highlightthickness=1
)
self.setup_frame.place(relx=0.5, rely=0.5, anchor="center")
tk.Label(self.setup_frame, text="◈ INITIALISATION REQUIRED",
fg=C_PRI, bg="#00080d", font=("Courier", 13, "bold")).pack(pady=(18, 4))
tk.Label(self.setup_frame,
text="Enter your Gemini API key to boot J.A.R.V.I.S.",
fg=C_MID, bg="#00080d", font=("Courier", 9)).pack(pady=(0, 10))
tk.Label(self.setup_frame, text="GEMINI API KEY",
fg=C_DIM, bg="#00080d", font=("Courier", 9)).pack(pady=(8, 2))
self.gemini_entry = tk.Entry(
self.setup_frame, width=52, fg=C_TEXT, bg="#000d12",
insertbackground=C_TEXT, borderwidth=0, font=("Courier", 10), show="*"
)
self.gemini_entry.pack(pady=(0, 4))
tk.Button(
self.setup_frame, text="▸ INITIALISE SYSTEMS",
command=self._save_api_keys, bg=C_BG, fg=C_PRI,
activebackground="#003344", font=("Courier", 10),
borderwidth=0, pady=8
).pack(pady=14)
def _save_api_keys(self):
gemini = self.gemini_entry.get().strip()
if not gemini:
return
os.makedirs(CONFIG_DIR, exist_ok=True)
with open(API_FILE, "w", encoding="utf-8") as f:
json.dump({"gemini_api_key": gemini}, f, indent=4)
self.setup_frame.destroy()
self._api_key_ready = True
self.set_state("LISTENING")
self.write_log("SYS: Systems initialised. JARVIS online.")