Add files via upload

This commit is contained in:
FatihMakes authored and GitHub committed 2026-04-19 15:22:49 +03:00
1 parent 320d933f08
commit cd70cb7796
27 files changed
+5807 -3538

No files matched your search

+808 -323
View File
File diff suppressed because it is too large. Load diff
-14
View File
@@ -1,16 +1,3 @@
# actions/code_helper.py
# AI-powered code assistant — writes, edits, explains, runs, builds, debugs, and optimizes code.
#
# Actions:
# write → Describe what you want, Gemini writes it, saves to file
# edit → Read existing file, apply natural language change
# explain → Explain what a piece of code or file does
# run → Execute a script file, return output
# build → Write → Run → Fix loop (max 3 attempts), speaks when done
# screen_debug → Screenshot al, ekrandaki kodu/hatayı Gemini ile analiz et ve düzelt
# optimize → Mevcut kodu Gemini ile optimize et (performans, okunabilirlik, best practices)
# auto → (default) Intent auto-detected from context
import subprocess
import sys
import json
@@ -126,7 +113,6 @@ def _detect_intent(description: str, file_path: str, code: str) -> str:
if any(k in desc for k in screen_kw):
return "screen_debug"
# Optimize keywords
optimize_kw = ["optimize", "refactor", "clean up", "improve", "temizle",
"iyileştir", "daha iyi", "make it better", "hızlandır"]
if any(k in desc for k in optimize_kw) and (code or file_path):
+343 -398
View File
@@ -1,29 +1,11 @@
# actions/computer_control.py
# MARK XXV — Computer Control
#
# Atomic computer control functions using PyAutoGUI + keyboard + clipboard.
# Used by the agent when no existing action file covers the task.
#
# Capabilities:
# - Type text anywhere (active window, forms, fields)
# - Mouse click, double-click, right-click, drag
# - Keyboard shortcuts and key combinations
# - Scroll (up/down/left/right)
# - Window management (minimize, maximize, close, focus)
# - Clipboard (copy, paste, get content)
# - Screenshot + locate element on screen
# - Wait / smart wait for element to appear
# - Random data generation (name, email, username, password, phone, address)
# - Hotkey sequences
# - Find and click image/element on screen
import io
import json
import re
import string
import subprocess
import sys
import time
import random
import string
import subprocess
import platform
from pathlib import Path
try:
@@ -40,282 +22,135 @@ try:
except ImportError:
_PYPERCLIP = False
def get_base_dir() -> Path:
def _base_dir() -> Path:
if getattr(sys, "frozen", False):
return Path(sys.executable).parent
return Path(__file__).resolve().parent.parent
BASE_DIR = get_base_dir()
API_CONFIG_PATH = BASE_DIR / "config" / "api_keys.json"
_BASE = _base_dir()
_CONFIG_PATH = _BASE / "config" / "api_keys.json"
_MEMORY_PATH = _BASE / "memory" / "long_term.json"
def _load_user_profile() -> dict:
"""Load user profile from long_term.json for form filling."""
memory_path = BASE_DIR / "memory" / "long_term.json"
def _load_config() -> dict:
try:
if memory_path.exists():
data = json.loads(memory_path.read_text(encoding="utf-8"))
identity = data.get("identity", {})
return {
"name": identity.get("name", {}).get("value", ""),
"age": identity.get("age", {}).get("value", ""),
"city": identity.get("city", {}).get("value", ""),
"email": identity.get("email", {}).get("value", ""),
}
return json.loads(_CONFIG_PATH.read_text(encoding="utf-8"))
except Exception:
return {}
def _get_os() -> str:
return _load_config().get("os_system", "windows").lower()
def _get_api_key() -> str:
return _load_config().get("gemini_api_key", "")
_SAFE_SCREENSHOT_ROOTS = (
Path.home(),
)
def _safe_screenshot_path(requested: str | None) -> Path:
fallback = Path.home() / "Desktop" / "jarvis_screenshot.png"
if not requested:
return fallback
try:
p = Path(requested).expanduser().resolve()
for root in _SAFE_SCREENSHOT_ROOTS:
if p.is_relative_to(root.resolve()):
p.parent.mkdir(parents=True, exist_ok=True)
return p
except Exception:
pass
return {}
return fallback
def _ensure_pyautogui():
def _require_pyautogui():
if not _PYAUTOGUI:
raise RuntimeError(
"PyAutoGUI not installed. Run: pip install pyautogui"
)
raise RuntimeError("PyAutoGUI not installed. Run: pip install pyautogui")
_FIRST_NAMES = [
"Alex", "Jordan", "Taylor", "Morgan", "Casey", "Riley", "Drew", "Quinn",
"Avery", "Blake", "Cameron", "Dakota", "Emerson", "Finley", "Harper"
"Avery", "Blake", "Cameron", "Dakota", "Emerson", "Finley", "Harper",
]
_LAST_NAMES = [
"Smith", "Johnson", "Williams", "Brown", "Jones", "Garcia", "Miller",
"Davis", "Wilson", "Moore", "Taylor", "Anderson", "Thomas", "Jackson"
"Davis", "Wilson", "Moore", "Taylor", "Anderson", "Thomas", "Jackson",
]
_DOMAINS = ["gmail.com", "yahoo.com", "outlook.com", "proton.me", "mail.com"]
def generate_random_data(data_type: str) -> str:
"""
Generates random realistic data for form filling.
Types: name | first_name | last_name | email | username |
password | phone | birthday | address | zip_code
"""
def _random_data(data_type: str) -> str:
dt = data_type.lower().strip()
if dt == "first_name":
return random.choice(_FIRST_NAMES)
elif dt == "last_name":
if dt == "last_name":
return random.choice(_LAST_NAMES)
elif dt == "name":
if dt == "name":
return f"{random.choice(_FIRST_NAMES)} {random.choice(_LAST_NAMES)}"
elif dt == "email":
if dt == "email":
first = random.choice(_FIRST_NAMES).lower()
last = random.choice(_LAST_NAMES).lower()
num = random.randint(10, 999)
return f"{first}.{last}{num}@{random.choice(_DOMAINS)}"
elif dt == "username":
first = random.choice(_FIRST_NAMES).lower()
num = random.randint(100, 9999)
return f"{first}{num}"
if dt == "username":
return f"{random.choice(_FIRST_NAMES).lower()}{random.randint(100, 9999)}"
elif dt == "password":
if dt == "password":
chars = string.ascii_letters + string.digits + "!@#$%"
pwd = (
random.choice(string.ascii_uppercase) +
random.choice(string.digits) +
random.choice("!@#$%") +
"".join(random.choices(chars, k=9))
raw = (
random.choice(string.ascii_uppercase)
+ random.choice(string.digits)
+ random.choice("!@#$%")
+ "".join(random.choices(chars, k=9))
)
return "".join(random.sample(pwd, len(pwd)))
return "".join(random.sample(raw, len(raw)))
elif dt == "phone":
return f"+1{random.randint(200,999)}{random.randint(1000000,9999999)}"
if dt == "phone":
return f"+1{random.randint(200,999)}{random.randint(1_000_000, 9_999_999)}"
elif dt == "birthday":
year = random.randint(1980, 2000)
month = random.randint(1, 12)
day = random.randint(1, 28)
return f"{month:02d}/{day:02d}/{year}"
if dt == "birthday":
y = random.randint(1980, 2000)
m = random.randint(1, 12)
d = random.randint(1, 28)
return f"{m:02d}/{d:02d}/{y}"
elif dt == "address":
if dt == "address":
num = random.randint(100, 9999)
street = random.choice(["Main St", "Oak Ave", "Park Blvd", "Elm St", "Cedar Ln"])
return f"{num} {street}"
elif dt == "zip_code":
if dt == "zip_code":
return str(random.randint(10000, 99999))
elif dt == "city":
if dt == "city":
return random.choice(["New York", "Los Angeles", "Chicago", "Houston", "Phoenix"])
return f"random_{data_type}_{random.randint(1000,9999)}"
return f"random_{data_type}_{random.randint(1000, 9999)}"
def _user_profile() -> dict:
"""Read identity fields from long-term memory."""
try:
if _MEMORY_PATH.exists():
data = json.loads(_MEMORY_PATH.read_text(encoding="utf-8"))
identity = data.get("identity", {})
return {k: v.get("value", "") for k, v in identity.items()}
except Exception:
pass
return {}
def _type_text(text: str, interval: float = 0.03) -> str:
"""Types text at the current cursor position."""
_ensure_pyautogui()
def _type(text: str, interval: float = 0.03) -> str:
_require_pyautogui()
time.sleep(0.3)
pyautogui.typewrite(text, interval=interval)
return f"Typed: {text[:50]}{'...' if len(text) > 50 else ''}"
def _click(x: int = None, y: int = None, button: str = "left",
clicks: int = 1, image: str = None) -> str:
"""
Clicks at coordinates or on a screen image.
If image path given, locates it on screen and clicks.
"""
_ensure_pyautogui()
if image:
try:
loc = pyautogui.locateCenterOnScreen(image, confidence=0.8)
if loc:
pyautogui.click(loc.x, loc.y, button=button, clicks=clicks)
return f"Clicked image: {image}"
return f"Image not found on screen: {image}"
except Exception as e:
return f"Image click failed: {e}"
if x is not None and y is not None:
pyautogui.click(x, y, button=button, clicks=clicks)
return f"Clicked ({x}, {y}) with {button} button"
pyautogui.click(button=button, clicks=clicks)
return f"Clicked at current position"
def _hotkey(*keys) -> str:
"""Presses a key combination. E.g. hotkey('ctrl', 'c')"""
_ensure_pyautogui()
pyautogui.hotkey(*keys)
return f"Hotkey: {'+'.join(keys)}"
def _press(key: str) -> str:
"""Presses a single key."""
_ensure_pyautogui()
pyautogui.press(key)
return f"Pressed: {key}"
def _scroll(direction: str = "down", amount: int = 3) -> str:
"""Scrolls in the specified direction."""
_ensure_pyautogui()
clicks = amount if direction in ("up", "right") else -amount
if direction in ("up", "down"):
pyautogui.scroll(clicks)
else:
pyautogui.hscroll(clicks)
return f"Scrolled {direction} {amount} times"
def _move_mouse(x: int, y: int, duration: float = 0.3) -> str:
"""Moves mouse to coordinates."""
_ensure_pyautogui()
pyautogui.moveTo(x, y, duration=duration)
return f"Mouse moved to ({x}, {y})"
def _drag(x1: int, y1: int, x2: int, y2: int, duration: float = 0.5) -> str:
"""Drags from (x1,y1) to (x2,y2)."""
_ensure_pyautogui()
pyautogui.drag(x1 - pyautogui.position()[0], y1 - pyautogui.position()[1])
pyautogui.dragTo(x2, y2, duration=duration)
return f"Dragged from ({x1},{y1}) to ({x2},{y2})"
def _clipboard_copy() -> str:
"""Gets current clipboard content."""
if _PYPERCLIP:
return pyperclip.paste()
_hotkey("ctrl", "c")
time.sleep(0.2)
return "Copied to clipboard"
def _clipboard_set(text: str) -> str:
"""Sets clipboard content and pastes it."""
if _PYPERCLIP:
pyperclip.copy(text)
time.sleep(0.1)
_hotkey("ctrl", "v")
return f"Pasted: {text[:50]}"
return "pyperclip not available"
def _screenshot(save_path: str = None) -> str:
"""Takes a screenshot."""
_ensure_pyautogui()
if not save_path:
save_path = str(Path.home() / "Desktop" / "screenshot.png")
img = pyautogui.screenshot()
img.save(save_path)
return f"Screenshot saved: {save_path}"
def _wait(seconds: float) -> str:
"""Waits for specified seconds."""
time.sleep(seconds)
return f"Waited {seconds}s"
def _wait_for_image(image_path: str, timeout: int = 10) -> str:
"""Waits until an image appears on screen (up to timeout seconds)."""
_ensure_pyautogui()
start = time.time()
while time.time() - start < timeout:
try:
loc = pyautogui.locateCenterOnScreen(image_path, confidence=0.8)
if loc:
return f"Image found at ({loc.x}, {loc.y})"
except Exception:
pass
time.sleep(0.5)
return f"Image not found within {timeout}s: {image_path}"
def _get_screen_size() -> str:
"""Returns current screen resolution."""
_ensure_pyautogui()
w, h = pyautogui.size()
return f"{w}x{h}"
def _focus_window(title: str) -> str:
"""Brings a window to focus by title (Windows)."""
if platform.system() == "Windows":
try:
script = f'(New-Object -ComObject WScript.Shell).AppActivate("{title}")'
subprocess.run(
["powershell", "-NoProfile", "-Command", script],
capture_output=True, timeout=5
)
time.sleep(0.3)
return f"Focused window: {title}"
except Exception as e:
return f"Could not focus window: {e}"
return "Window focus only supported on Windows"
def _select_all() -> str:
return _hotkey("ctrl", "a")
def _clear_field() -> str:
"""Selects all and deletes — clears an input field."""
_hotkey("ctrl", "a")
time.sleep(0.1)
_press("delete")
return "Field cleared"
return f"Typed: {text[:60]}{'…' if len(text) > 60 else ''}"
def _smart_type(text: str, clear_first: bool = True) -> str:
"""
Types text into the currently focused field.
Optionally clears the field first.
Uses clipboard for long text (faster, more reliable).
"""
_ensure_pyautogui()
_require_pyautogui()
if clear_first:
_clear_field()
time.sleep(0.1)
@@ -324,231 +159,341 @@ def _smart_type(text: str, clear_first: bool = True) -> str:
pyperclip.copy(text)
time.sleep(0.1)
pyautogui.hotkey("ctrl", "v")
return f"Smart-typed (clipboard): {text[:50]}"
else:
pyautogui.typewrite(text, interval=0.04)
return f"Smart-typed: {text[:50]}"
return f"Smart-typed (clipboard): {text[:60]}{'…' if len(text) > 60 else ''}"
pyautogui.typewrite(text, interval=0.04)
return f"Smart-typed: {text[:60]}{'…' if len(text) > 60 else ''}"
def _analyze_screen_for_element(description: str) -> tuple[int, int] | None:
"""
Takes a screenshot and asks Gemini to find the coordinates
of a described element on screen. Returns (x, y) or None.
"""
def _click(x=None, y=None, button: str = "left", clicks: int = 1) -> str:
_require_pyautogui()
if x is not None and y is not None:
pyautogui.click(x, y, button=button, clicks=clicks)
return f"{'Double-c' if clicks == 2 else 'C'}licked ({x}, {y}) [{button}]"
pyautogui.click(button=button, clicks=clicks)
return f"Clicked at current position [{button}]"
def _hotkey(*keys) -> str:
_require_pyautogui()
pyautogui.hotkey(*keys)
return f"Hotkey: {'+'.join(keys)}"
def _press(key: str) -> str:
_require_pyautogui()
pyautogui.press(key)
return f"Pressed: {key}"
def _scroll(direction: str = "down", amount: int = 3) -> str:
_require_pyautogui()
vertical = direction in ("up", "down")
clicks = amount if direction in ("up", "right") else -amount
pyautogui.scroll(clicks) if vertical else pyautogui.hscroll(clicks)
return f"Scrolled {direction} ×{amount}"
def _move(x: int, y: int, duration: float = 0.3) -> str:
_require_pyautogui()
pyautogui.moveTo(x, y, duration=duration)
return f"Mouse → ({x}, {y})"
def _drag(x1: int, y1: int, x2: int, y2: int, duration: float = 0.5) -> str:
_require_pyautogui()
pyautogui.moveTo(x1, y1, duration=0.2)
pyautogui.dragTo(x2, y2, duration=duration, button="left")
return f"Dragged ({x1},{y1}) → ({x2},{y2})"
def _clipboard_get() -> str:
if _PYPERCLIP:
return pyperclip.paste()
_hotkey("ctrl", "c")
time.sleep(0.2)
return "(copied — pyperclip unavailable for read)"
def _clipboard_paste(text: str) -> str:
if _PYPERCLIP:
pyperclip.copy(text)
time.sleep(0.1)
_require_pyautogui()
pyautogui.hotkey("ctrl", "v")
return f"Pasted: {text[:60]}{'…' if len(text) > 60 else ''}"
return "pyperclip not available"
def _screenshot(save_path: str | None = None) -> str:
_require_pyautogui()
path = _safe_screenshot_path(save_path)
img = pyautogui.screenshot()
img.save(str(path))
return f"Screenshot saved: {path}"
def _clear_field() -> str:
_require_pyautogui()
pyautogui.hotkey("ctrl", "a")
time.sleep(0.1)
pyautogui.press("delete")
return "Field cleared"
def _focus_window(title: str) -> str:
os_name = _get_os()
if os_name == "windows":
try:
script = f'(New-Object -ComObject WScript.Shell).AppActivate("{title}")'
subprocess.run(
["powershell", "-NoProfile", "-NonInteractive", "-Command", script],
capture_output=True, timeout=5,
)
time.sleep(0.3)
return f"Focused window: {title}"
except Exception as e:
return f"focus_window (Windows) failed: {e}"
if os_name == "mac":
script = (
f'tell application "System Events" to '
f'set frontmost of (first process whose name contains "{title}") to true'
)
try:
subprocess.run(
["osascript", "-e", script],
capture_output=True, timeout=5,
)
time.sleep(0.3)
return f"Focused window: {title}"
except Exception as e:
return f"focus_window (macOS) failed: {e}"
if os_name == "linux":
try:
result = subprocess.run(
["wmctrl", "-a", title],
capture_output=True, timeout=5,
)
if result.returncode == 0:
time.sleep(0.3)
return f"Focused window: {title}"
except FileNotFoundError:
pass
try:
result = subprocess.run(
["xdotool", "search", "--name", title, "windowactivate"],
capture_output=True, timeout=5,
)
time.sleep(0.3)
return f"Focused window: {title}"
except FileNotFoundError:
return "focus_window (Linux) requires wmctrl or xdotool"
except Exception as e:
return f"focus_window (Linux) failed: {e}"
return f"focus_window: unknown OS '{os_name}'"
def _screen_find(description: str) -> tuple[int, int] | None:
api_key = _get_api_key()
if not api_key:
print("[ComputerControl] ⚠️ No API key for screen_find")
return None
try:
import google.generativeai as genai
import io
from google import genai
from google.genai import types as gtypes
cfg_path = API_CONFIG_PATH
with open(cfg_path, "r") as f:
api_key = json.load(f)["gemini_api_key"]
genai.configure(api_key=api_key)
model = genai.GenerativeModel("gemini-2.5-flash-lite")
_ensure_pyautogui()
_require_pyautogui()
w, h = pyautogui.size()
img = pyautogui.screenshot()
buf = io.BytesIO()
img.save(buf, format="PNG")
buf.seek(0)
image_bytes = buf.getvalue()
client = genai.Client(api_key=api_key)
prompt = (
f"This is a screenshot of a computer screen ({w}x{h} pixels). "
f"Find the element: '{description}'. "
f"Return ONLY: x,y (the center coordinates of the element). "
f"If not found, return: NOT_FOUND"
f"This is a screenshot of a {w}×{h} pixel screen. "
f"Locate the UI element described as: '{description}'. "
f"Reply with ONLY the center coordinates as: x,y "
f"If the element is not visible, reply: NOT_FOUND"
)
response = model.generate_content([
{"mime_type": "image/png", "data": buf.getvalue()},
prompt
])
response = client.models.generate_content(
model="gemini-2.5-flash-lite",
contents=[
gtypes.Part.from_bytes(data=image_bytes, mime_type="image/png"),
prompt,
],
)
text = response.text.strip()
if "NOT_FOUND" in text:
text = (response.text or "").strip()
if "NOT_FOUND" in text.upper():
return None
import re
match = re.search(r"(\d+)\s*,\s*(\d+)", text)
if match:
return int(match.group(1)), int(match.group(2))
except Exception as e:
print(f"[ComputerControl] ⚠️ Screen analysis failed: {e}")
print(f"[ComputerControl] ⚠️ screen_find failed: {e}")
return None
def computer_control(
parameters: dict,
parameters: dict,
response=None,
player=None,
session_memory=None,
) -> str:
"""
Universal computer control action.
Dispatch table for all computer control actions.
parameters keys (all optional unless noted):
action : (required) one of the actions listed below
text : text to type or paste
x, y : screen coordinates
button : 'left' | 'right' (default: left)
keys : hotkey string, e.g. 'ctrl+c'
key : single key name, e.g. 'enter'
direction : 'up' | 'down' | 'left' | 'right'
amount : scroll amount (default: 3)
seconds : wait duration
title : window title fragment for focus_window
description : natural-language element description for screen_find/click
type : data type for random_data
field : memory field name for user_data
clear_first : bool, clear field before typing (default: true)
path : save path for screenshot (must be inside home dir)
Actions:
type : Type text at current cursor position
smart_type : Clear field + type (clipboard-based for long text)
click : Click at coordinates or on image
double_click : Double click
right_click : Right click
hotkey : Key combination (e.g. ctrl+c)
press : Single key press
scroll : Scroll up/down/left/right
move : Move mouse to coordinates
drag : Drag from one point to another
copy : Get clipboard content
paste : Set and paste clipboard content
screenshot : Take a screenshot
wait : Wait N seconds
wait_image : Wait for image to appear on screen
clear_field : Select all + delete in current field
focus_window : Bring window to foreground
screen_find : AI-powered element finder — returns coordinates
screen_click : AI-powered element finder + click
random_data : Generate random data for forms
user_data : Get user's real data from memory
type — type text at cursor
smart_type — clear field + type (clipboard-backed)
click — left click
double_click — double left click
right_click — right click
move — move mouse
drag — click-drag between two points
hotkey — key combination
press — single key
scroll — scroll the wheel
copy — read clipboard
paste — write + paste clipboard
screenshot — capture screen (safe path only)
wait — sleep N seconds
clear_field — select-all + delete
focus_window — bring window to foreground
screen_find — AI element finder (returns x,y)
screen_click — AI element finder + click
random_data — generate fake form data
user_data — pull real data from memory
"""
action = (parameters or {}).get("action", "").lower().strip()
params = parameters or {}
action = params.get("action", "").lower().strip()
if not action:
return "Please specify an action for computer_control, sir."
return "No action specified for computer_control."
if player:
player.write_log(f"[Computer] {action}")
print(f"[ComputerControl] ▶️ Action: {action} Params: {parameters}")
print(f"[ComputerControl] ▶ {action} {params}")
try:
if action == "type":
text = parameters.get("text", "")
return _type_text(text)
return _type(params.get("text", ""))
elif action == "smart_type":
text = parameters.get("text", "")
clear_first = parameters.get("clear_first", True)
return _smart_type(text, clear_first=clear_first)
elif action in ("click", "left_click"):
return _click(
x=parameters.get("x"),
y=parameters.get("y"),
button="left",
clicks=1,
image=parameters.get("image")
if action == "smart_type":
return _smart_type(
params.get("text", ""),
clear_first=params.get("clear_first", True),
)
elif action == "double_click":
return _click(
x=parameters.get("x"),
y=parameters.get("y"),
button="left",
clicks=2,
image=parameters.get("image")
)
if action in ("click", "left_click"):
return _click(params.get("x"), params.get("y"), "left", 1)
elif action == "right_click":
return _click(
x=parameters.get("x"),
y=parameters.get("y"),
button="right",
clicks=1
)
if action == "double_click":
return _click(params.get("x"), params.get("y"), "left", 2)
elif action == "move":
return _move_mouse(
x=int(parameters.get("x", 0)),
y=int(parameters.get("y", 0)),
duration=float(parameters.get("duration", 0.3))
)
if action == "right_click":
return _click(params.get("x"), params.get("y"), "right", 1)
elif action == "drag":
if action == "move":
return _move(int(params.get("x", 0)), int(params.get("y", 0)))
if action == "drag":
return _drag(
x1=int(parameters.get("x1", 0)),
y1=int(parameters.get("y1", 0)),
x2=int(parameters.get("x2", 0)),
y2=int(parameters.get("y2", 0))
int(params.get("x1", 0)), int(params.get("y1", 0)),
int(params.get("x2", 0)), int(params.get("y2", 0)),
)
elif action == "hotkey":
keys = parameters.get("keys", "")
if isinstance(keys, str):
keys = [k.strip() for k in keys.split("+")]
if action == "hotkey":
raw = params.get("keys", "")
keys = [k.strip() for k in raw.split("+")] if isinstance(raw, str) else raw
return _hotkey(*keys)
elif action == "press":
return _press(parameters.get("key", "enter"))
if action == "press":
return _press(params.get("key", "enter"))
elif action == "scroll":
if action == "scroll":
return _scroll(
direction=parameters.get("direction", "down"),
amount=int(parameters.get("amount", 3))
direction=params.get("direction", "down"),
amount=int(params.get("amount", 3)),
)
elif action == "copy":
return _clipboard_copy()
if action == "copy":
return _clipboard_get()
elif action == "paste":
return _clipboard_set(parameters.get("text", ""))
if action == "paste":
return _clipboard_paste(params.get("text", ""))
elif action == "screenshot":
return _screenshot(parameters.get("path"))
if action == "screenshot":
return _screenshot(params.get("path"))
elif action == "wait":
return _wait(float(parameters.get("seconds", 1.0)))
if action == "screen_find":
coords = _screen_find(params.get("description", ""))
return f"{coords[0]},{coords[1]}" if coords else "NOT_FOUND"
elif action == "wait_image":
return _wait_for_image(
parameters.get("image", ""),
timeout=int(parameters.get("timeout", 10))
)
elif action == "clear_field":
return _clear_field()
elif action == "focus_window":
return _focus_window(parameters.get("title", ""))
elif action == "screen_size":
return _get_screen_size()
elif action == "screen_find":
description = parameters.get("description", "")
coords = _analyze_screen_for_element(description)
if coords:
return f"{coords[0]},{coords[1]}"
return "NOT_FOUND"
elif action == "screen_click":
description = parameters.get("description", "")
coords = _analyze_screen_for_element(description)
if action == "screen_click":
desc = params.get("description", "")
coords = _screen_find(desc)
if coords:
time.sleep(0.2)
_click(x=coords[0], y=coords[1])
return f"Found and clicked: {description} at {coords}"
return f"Could not find on screen: {description}"
return f"Clicked '{desc}' at {coords}"
return f"Element not found on screen: '{desc}'"
elif action == "random_data":
data_type = parameters.get("type", "name")
result = generate_random_data(data_type)
print(f"[ComputerControl] 🎲 Random {data_type}: {result}")
if action == "wait":
secs = float(params.get("seconds", 1.0))
secs = min(secs, 30.0)
time.sleep(secs)
return f"Waited {secs}s"
if action == "clear_field":
return _clear_field()
if action == "focus_window":
return _focus_window(params.get("title", ""))
if action == "random_data":
dt = params.get("type", "name")
result = _random_data(dt)
print(f"[ComputerControl] 🎲 random {dt} → {result}")
return result
elif action == "user_data":
field = parameters.get("field", "name")
profile = _load_user_profile()
if action == "user_data":
field = params.get("field", "name")
profile = _user_profile()
value = profile.get(field, "")
if not value:
value = generate_random_data(field)
print(f"[ComputerControl] ⚠️ No user {field} in memory, using random: {value}")
value = _random_data(field)
print(f"[ComputerControl] ⚠️ No '{field}' in memory, using random: {value}")
return value
else:
return f"Unknown computer_control action: '{action}'"
return f"Unknown action: '{action}'"
except Exception as e:
print(f"[ComputerControl] ❌ Error: {e}")
return f"computer_control failed: {e}"
print(f"[ComputerControl] ❌ {action}: {e}")
return f"computer_control '{action}' failed: {e}"
+372 -363
View File
@@ -1,16 +1,8 @@
# actions/computer_settings.py
# MARK XXV — Computer Settings & UI Controls
#
# Kullanıcı "sesi aç", "uygulamayı kapat", "tam ekran yap", "şunu yaz" gibi
# bilgisayar kontrol komutları verdiğinde bu dosya devreye girer.
#
# - Intent detection: Gemini ile (multi-language, hardcoded keyword yok)
# - Cross-platform: Windows / macOS / Linux
# - pyautogui + platform-specific API'ler
import json
import re
import sys
import time
import subprocess
import sys
import platform
from pathlib import Path
@@ -28,139 +20,211 @@ try:
except ImportError:
_PYPERCLIP = False
_OS = platform.system()
_OS = platform.system() # "Windows" | "Darwin" | "Linux"
def get_base_dir() -> Path:
def _get_base_dir() -> Path:
if getattr(sys, "frozen", False):
return Path(sys.executable).parent
return Path(__file__).resolve().parent.parent
BASE_DIR = get_base_dir()
API_CONFIG_PATH = BASE_DIR / "config" / "api_keys.json"
import json
def _get_api_key() -> str:
with open(API_CONFIG_PATH, "r", encoding="utf-8") as f:
path = _get_base_dir() / "config" / "api_keys.json"
with open(path, "r", encoding="utf-8") as f:
return json.load(f)["gemini_api_key"]
def _get_macos_wifi_interface() -> str:
try:
result = subprocess.run(
["networksetup", "-listallhardwareports"],
capture_output=True, text=True, timeout=5
)
lines = result.stdout.splitlines()
for i, line in enumerate(lines):
if "Wi-Fi" in line or "AirPort" in line:
for j in range(i, min(i + 4, len(lines))):
if lines[j].startswith("Device:"):
return lines[j].split(":", 1)[1].strip()
except Exception:
pass
return "en0"
def volume_up():
if _OS == "Windows":
for _ in range(5): pyautogui.press("volumeup")
elif _OS == "Darwin":
subprocess.run(["osascript", "-e", "set volume output volume (output volume of (get volume settings) + 10)"])
subprocess.run(["osascript", "-e",
"set volume output volume (output volume of (get volume settings) + 10)"],
capture_output=True)
else:
subprocess.run(["pactl", "set-sink-volume", "@DEFAULT_SINK@", "+10%"])
subprocess.run(["pactl", "set-sink-volume", "@DEFAULT_SINK@", "+10%"],
capture_output=True)
def volume_down():
if _OS == "Windows":
for _ in range(5): pyautogui.press("volumedown")
elif _OS == "Darwin":
subprocess.run(["osascript", "-e", "set volume output volume (output volume of (get volume settings) - 10)"])
subprocess.run(["osascript", "-e",
"set volume output volume (output volume of (get volume settings) - 10)"],
capture_output=True)
else:
subprocess.run(["pactl", "set-sink-volume", "@DEFAULT_SINK@", "-10%"])
subprocess.run(["pactl", "set-sink-volume", "@DEFAULT_SINK@", "-10%"],
capture_output=True)
def volume_mute():
if _OS == "Windows":
pyautogui.press("volumemute")
elif _OS == "Darwin":
subprocess.run(["osascript", "-e", "set volume with output muted"])
subprocess.run(["osascript", "-e", "set volume with output muted"],
capture_output=True)
else:
subprocess.run(["pactl", "set-sink-mute", "@DEFAULT_SINK@", "toggle"])
subprocess.run(["pactl", "set-sink-mute", "@DEFAULT_SINK@", "toggle"],
capture_output=True)
def volume_set(value: int):
value = max(0, min(100, value))
value = max(0, min(100, int(value)))
if _OS == "Windows":
try:
import math
from ctypes import cast, POINTER
from comtypes import CLSCTX_ALL
from pycaw.pycaw import AudioUtilities, IAudioEndpointVolume
import math
devices = AudioUtilities.GetSpeakers()
interface = devices.Activate(IAudioEndpointVolume._iid_, CLSCTX_ALL, None)
vol = cast(interface, POINTER(IAudioEndpointVolume))
vol_db = -65.25 if value == 0 else max(-65.25, 20 * math.log10(value / 100))
vol.SetMasterVolumeLevel(vol_db, None)
print(f"[Settings] 🔊 Volume → {value}%")
return
except Exception as e:
print(f"[Settings] ⚠️ pycaw failed: {e}")
print(f"[Settings] pycaw failed, using keypress fallback: {e}")
pyautogui.press("volumemute")
pyautogui.press("volumemute")
elif _OS == "Darwin":
subprocess.run(["osascript", "-e", f"set volume output volume {value}"])
subprocess.run(["osascript", "-e", f"set volume output volume {value}"],
capture_output=True)
return
else:
subprocess.run(["pactl", "set-sink-volume", "@DEFAULT_SINK@", f"{value}%"])
subprocess.run(["pactl", "set-sink-volume", "@DEFAULT_SINK@", f"{value}%"],
capture_output=True)
return
def brightness_up():
if _OS == "Windows":
pyautogui.hotkey("win", "a")
time.sleep(0.3)
elif _OS == "Darwin":
subprocess.run(["osascript", "-e", "tell application \"System Events\" to key code 144"])
if _OS == "Darwin":
subprocess.run(["osascript", "-e",
'tell application "System Events" to key code 144'],
capture_output=True)
elif _OS == "Linux":
if subprocess.run(["which", "brightnessctl"],
capture_output=True).returncode == 0:
subprocess.run(["brightnessctl", "set", "+10%"], capture_output=True)
else:
subprocess.run(
'xrandr --output $(xrandr | grep " connected" | head -1 | cut -d " " -f1)'
' --brightness $(python3 -c "import subprocess; '
'b=float(subprocess.check_output([\"xrandr\",\"--verbose\"]).decode()'
'.split(\"Brightness:\")[1].split()[0]); print(min(1.0,b+0.1))")',
shell=True, capture_output=True
)
else:
subprocess.run(["brightnessctl", "set", "+10%"])
try:
subprocess.run(
["powershell", "-Command",
"(Get-WmiObject -Namespace root/wmi -Class WmiMonitorBrightnessMethods)"
".WmiSetBrightness(1, [math]::Min(100, "
"(Get-WmiObject -Namespace root/wmi -Class WmiMonitorBrightness).CurrentBrightness + 10))"],
capture_output=True, timeout=5
)
except Exception as e:
print(f"[Settings] Brightness up failed on Windows: {e}")
def brightness_down():
if _OS == "Windows":
pyautogui.hotkey("win", "a")
time.sleep(0.3)
elif _OS == "Darwin":
subprocess.run(["osascript", "-e", "tell application \"System Events\" to key code 145"])
if _OS == "Darwin":
subprocess.run(["osascript", "-e",
'tell application "System Events" to key code 145'],
capture_output=True)
elif _OS == "Linux":
if subprocess.run(["which", "brightnessctl"],
capture_output=True).returncode == 0:
subprocess.run(["brightnessctl", "set", "10%-"], capture_output=True)
else:
subprocess.run(
'xrandr --output $(xrandr | grep " connected" | head -1 | cut -d " " -f1)'
' --brightness $(python3 -c "import subprocess; '
'b=float(subprocess.check_output([\"xrandr\",\"--verbose\"]).decode()'
'.split(\"Brightness:\")[1].split()[0]); print(max(0.1,b-0.1))")',
shell=True, capture_output=True
)
else:
subprocess.run(["brightnessctl", "set", "10%-"])
try:
subprocess.run(
["powershell", "-Command",
"(Get-WmiObject -Namespace root/wmi -Class WmiMonitorBrightnessMethods)"
".WmiSetBrightness(1, [math]::Max(0, "
"(Get-WmiObject -Namespace root/wmi -Class WmiMonitorBrightness).CurrentBrightness - 10))"],
capture_output=True, timeout=5
)
except Exception as e:
print(f"[Settings] Brightness down failed on Windows: {e}")
def close_app():
if _OS == "Darwin":
pyautogui.hotkey("command", "q")
else:
pyautogui.hotkey("alt", "f4")
if _OS == "Darwin": pyautogui.hotkey("command", "q")
else: pyautogui.hotkey("alt", "f4")
def close_window():
if _OS == "Darwin":
pyautogui.hotkey("command", "w")
else:
pyautogui.hotkey("ctrl", "w")
if _OS == "Darwin": pyautogui.hotkey("command", "w")
else: pyautogui.hotkey("ctrl", "w")
def full_screen():
if _OS == "Darwin":
pyautogui.hotkey("ctrl", "command", "f")
else:
pyautogui.press("f11")
if _OS == "Darwin": pyautogui.hotkey("ctrl", "command", "f")
else: pyautogui.press("f11")
def minimize_window():
if _OS == "Darwin":
pyautogui.hotkey("command", "m")
else:
pyautogui.hotkey("win", "down")
if _OS == "Darwin": pyautogui.hotkey("command", "m")
else: pyautogui.hotkey("win", "down")
def maximize_window():
if _OS == "Darwin":
subprocess.run(["osascript", "-e",
'tell application "System Events" to keystroke "f" using {control down, command down}'])
else:
'tell application "System Events" to keystroke "f" '
'using {control down, command down}'],
capture_output=True)
elif _OS == "Windows":
pyautogui.hotkey("win", "up")
else:
try:
subprocess.run(["wmctrl", "-r", ":ACTIVE:", "-b", "add,maximized_vert,maximized_horz"],
capture_output=True)
except Exception:
pyautogui.hotkey("super", "up")
def snap_left():
if _OS == "Windows": pyautogui.hotkey("win", "left")
if _OS == "Windows":
pyautogui.hotkey("win", "left")
elif _OS == "Linux":
try:
subprocess.run(["wmctrl", "-r", ":ACTIVE:", "-e", "0,0,0,960,1080"],
capture_output=True)
except Exception:
pass
def snap_right():
if _OS == "Windows": pyautogui.hotkey("win", "right")
if _OS == "Windows":
pyautogui.hotkey("win", "right")
elif _OS == "Linux":
try:
subprocess.run(["wmctrl", "-r", ":ACTIVE:", "-e", "0,960,0,960,1080"],
capture_output=True)
except Exception:
pass
def switch_window():
if _OS == "Darwin":
pyautogui.hotkey("command", "tab")
else:
pyautogui.hotkey("alt", "tab")
if _OS == "Darwin": pyautogui.hotkey("command", "tab")
else: pyautogui.hotkey("alt", "tab")
def show_desktop():
if _OS == "Darwin":
pyautogui.hotkey("fn", "f11")
elif _OS == "Windows":
pyautogui.hotkey("win", "d")
else:
pyautogui.hotkey("super", "d")
if _OS == "Darwin": pyautogui.hotkey("fn", "f11")
elif _OS == "Windows": pyautogui.hotkey("win", "d")
else: pyautogui.hotkey("super", "d")
def open_task_manager():
if _OS == "Windows":
@@ -168,11 +232,10 @@ def open_task_manager():
elif _OS == "Darwin":
subprocess.Popen(["open", "-a", "Activity Monitor"])
else:
subprocess.Popen(["gnome-system-monitor"])
def open_task_view():
if _OS == "Windows":
pyautogui.hotkey("win", "tab")
for cmd in [["gnome-system-monitor"], ["xfce4-taskmanager"], ["htop"]]:
if subprocess.run(["which", cmd[0]], capture_output=True).returncode == 0:
subprocess.Popen(cmd)
break
def focus_search():
@@ -180,6 +243,7 @@ def focus_search():
else: pyautogui.hotkey("ctrl", "l")
def pause_video(): pyautogui.press("space")
def refresh_page():
if _OS == "Darwin": pyautogui.hotkey("command", "r")
else: pyautogui.press("f5")
@@ -225,17 +289,24 @@ def find_on_page():
else: pyautogui.hotkey("ctrl", "f")
def reload_page_n(n: int):
for _ in range(n):
for _ in range(max(1, n)):
refresh_page()
time.sleep(0.8)
def scroll_up(amount: int = 500): pyautogui.scroll(amount)
def scroll_down(amount: int = 500): pyautogui.scroll(-amount)
def scroll_top(): pyautogui.hotkey("ctrl", "home") if _OS != "Darwin" else pyautogui.hotkey("command", "up")
def scroll_bottom(): pyautogui.hotkey("ctrl", "end") if _OS != "Darwin" else pyautogui.hotkey("command", "down")
def page_up(): pyautogui.press("pageup")
def page_down(): pyautogui.press("pagedown")
def scroll_up(amount: int = 500): pyautogui.scroll(amount)
def scroll_down(amount: int = 500): pyautogui.scroll(-amount)
def scroll_top():
if _OS == "Darwin": pyautogui.hotkey("command", "up")
else: pyautogui.hotkey("ctrl", "home")
def scroll_bottom():
if _OS == "Darwin": pyautogui.hotkey("command", "down")
else: pyautogui.hotkey("ctrl", "end")
def page_up(): pyautogui.press("pageup")
def page_down(): pyautogui.press("pagedown")
def copy():
@@ -266,16 +337,16 @@ def save_file():
if _OS == "Darwin": pyautogui.hotkey("command", "s")
else: pyautogui.hotkey("ctrl", "s")
def press_enter(): pyautogui.press("enter")
def press_escape(): pyautogui.press("escape")
def press_enter(): pyautogui.press("enter")
def press_escape(): pyautogui.press("escape")
def press_key(key: str): pyautogui.press(key)
def type_text(text: str, press_enter_after: bool = False):
if not text:
return
if _PYPERCLIP:
pyperclip.copy(text)
time.sleep(0.1)
pyperclip.copy(str(text))
time.sleep(0.15)
paste()
else:
pyautogui.write(str(text), interval=0.03)
@@ -283,24 +354,32 @@ def type_text(text: str, press_enter_after: bool = False):
time.sleep(0.1)
pyautogui.press("enter")
def write_on_screen(text: str):
type_text(text)
def take_screenshot():
if _OS == "Windows":
pyautogui.hotkey("win", "shift", "s")
elif _OS == "Darwin":
pyautogui.hotkey("command", "shift", "3")
else:
for cmd in [["scrot"], ["gnome-screenshot"], ["import", "-window", "root", "screenshot.png"]]:
if subprocess.run(["which", cmd[0]], capture_output=True).returncode == 0:
subprocess.Popen(cmd)
return
pyautogui.hotkey("ctrl", "print_screen")
def lock_screen():
if _OS == "Windows":
pyautogui.hotkey("win", "l")
elif _OS == "Darwin":
subprocess.run(["pmset", "displaysleepnow"])
subprocess.run(["pmset", "displaysleepnow"], capture_output=True)
else:
subprocess.run(["gnome-screensaver-command", "-l"])
for cmd in [
["gnome-screensaver-command", "-l"],
["xdg-screensaver", "lock"],
["loginctl", "lock-session"],
]:
if subprocess.run(["which", cmd[0]], capture_output=True).returncode == 0:
subprocess.run(cmd, capture_output=True)
return
def open_system_settings():
if _OS == "Windows":
@@ -308,306 +387,227 @@ def open_system_settings():
elif _OS == "Darwin":
subprocess.Popen(["open", "-a", "System Preferences"])
else:
subprocess.Popen(["gnome-control-center"])
for cmd in [["gnome-control-center"], ["xfce4-settings-manager"], ["kcmshell5"]]:
if subprocess.run(["which", cmd[0]], capture_output=True).returncode == 0:
subprocess.Popen(cmd)
return
def open_file_explorer():
if _OS == "Windows":
pyautogui.hotkey("win", "e")
elif _OS == "Darwin":
subprocess.Popen(["open", Path.home()])
subprocess.Popen(["open", str(Path.home())])
else:
subprocess.Popen(["xdg-open", Path.home()])
def open_run():
if _OS == "Windows":
pyautogui.hotkey("win", "r")
for cmd in [["nautilus"], ["thunar"], ["dolphin"], ["nemo"]]:
if subprocess.run(["which", cmd[0]], capture_output=True).returncode == 0:
subprocess.Popen(cmd)
return
subprocess.Popen(["xdg-open", str(Path.home())])
def sleep_display():
if _OS == "Windows":
try:
import ctypes
ctypes.windll.user32.SendMessageW(0xFFFF, 0x0112, 0xF170, 2)
except Exception:
pass
except Exception as e:
print(f"[Settings] sleep_display failed: {e}")
elif _OS == "Darwin":
subprocess.run(["pmset", "displaysleepnow"])
subprocess.run(["pmset", "displaysleepnow"], capture_output=True)
else:
subprocess.run(["xset", "dpms", "force", "off"])
subprocess.run(["xset", "dpms", "force", "off"], capture_output=True)
def open_run():
if _OS == "Windows":
pyautogui.hotkey("win", "r")
def dark_mode():
if _OS == "Darwin":
subprocess.run(["osascript", "-e",
'tell app "System Events" to tell appearance preferences '
'to set dark mode to not dark mode'],
capture_output=True)
elif _OS == "Windows":
try:
import winreg
key_path = r"SOFTWARE\Microsoft\Windows\CurrentVersion\Themes\Personalize"
key = winreg.OpenKey(winreg.HKEY_CURRENT_USER, key_path, 0, winreg.KEY_ALL_ACCESS)
current, _ = winreg.QueryValueEx(key, "AppsUseLightTheme")
winreg.SetValueEx(key, "AppsUseLightTheme", 0, winreg.REG_DWORD, 1 - current)
winreg.SetValueEx(key, "SystemUsesLightTheme", 0, winreg.REG_DWORD, 1 - current)
winreg.CloseKey(key)
except Exception as e:
print(f"[Settings] dark_mode registry failed: {e}")
else:
try:
result = subprocess.run(
["gsettings", "get", "org.gnome.desktop.interface", "color-scheme"],
capture_output=True, text=True
)
current = result.stdout.strip()
new_scheme = "'default'" if "dark" in current else "'prefer-dark'"
subprocess.run(
["gsettings", "set", "org.gnome.desktop.interface", "color-scheme", new_scheme],
capture_output=True
)
except Exception as e:
print(f"[Settings] dark_mode Linux failed: {e}")
def toggle_wifi():
if _OS == "Darwin":
iface = _get_macos_wifi_interface()
result = subprocess.run(
["networksetup", "-getairportpower", iface],
capture_output=True, text=True
)
state = "off" if "On" in result.stdout else "on"
subprocess.run(["networksetup", "-setairportpower", iface, state],
capture_output=True)
elif _OS == "Windows":
try:
subprocess.run(
["powershell", "-Command",
"$adapter = Get-NetAdapter | Where-Object {$_.PhysicalMediaType -eq 'Native 802.11'};"
"if ($adapter.Status -eq 'Up') { Disable-NetAdapter -Name $adapter.Name -Confirm:$false }"
"else { Enable-NetAdapter -Name $adapter.Name -Confirm:$false }"],
capture_output=True, timeout=10
)
except Exception as e:
print(f"[Settings] toggle_wifi Windows failed: {e}")
else:
try:
result = subprocess.run(["nmcli", "radio", "wifi"], capture_output=True, text=True)
state = "off" if "enabled" in result.stdout else "on"
subprocess.run(["nmcli", "radio", "wifi", state], capture_output=True)
except Exception as e:
print(f"[Settings] toggle_wifi Linux failed: {e}")
def restart_computer():
if _OS == "Windows":
subprocess.run(["shutdown", "/r", "/t", "5"])
subprocess.run(["shutdown", "/r", "/t", "10"], capture_output=True)
elif _OS == "Darwin":
subprocess.run(["osascript", "-e", 'tell application "System Events" to restart'])
subprocess.run(["osascript", "-e",
'tell application "System Events" to restart'],
capture_output=True)
else:
subprocess.run(["sudo", "reboot"])
subprocess.run(["systemctl", "reboot"], capture_output=True)
def shutdown_computer():
if _OS == "Windows":
subprocess.run(["shutdown", "/s", "/t", "5"])
elif _OS == "Darwin":
subprocess.run(["osascript", "-e", 'tell application "System Events" to shut down'])
else:
subprocess.run(["sudo", "shutdown", "-h", "now"])
def dark_mode():
if _OS == "Windows":
pyautogui.hotkey("win", "a")
time.sleep(0.3)
subprocess.run(["shutdown", "/s", "/t", "10"], capture_output=True)
elif _OS == "Darwin":
subprocess.run(["osascript", "-e",
'tell app "System Events" to tell appearance preferences to set dark mode to not dark mode'])
def toggle_wifi():
if _OS == "Windows":
pyautogui.hotkey("win", "a")
time.sleep(0.3)
elif _OS == "Darwin":
subprocess.run(["networksetup", "-setairportpower", "en0", "toggle"])
'tell application "System Events" to shut down'],
capture_output=True)
else:
subprocess.run(["nmcli", "radio", "wifi"])
subprocess.run(["systemctl", "poweroff"], capture_output=True)
ACTION_MAP = {
"volume_up": volume_up,
"volume_down": volume_down,
"mute": volume_mute,
"unmute": volume_mute,
"volume_increase": volume_up,
"volume_decrease": volume_down,
"increase_volume": volume_up,
"decrease_volume": volume_down,
"turn_up_volume": volume_up,
"turn_down_volume": volume_down,
"louder": volume_up,
"quieter": volume_down,
"silence": volume_mute,
"toggle_mute": volume_mute,
"brightness_up": brightness_up,
"brightness_down": brightness_down,
"increase_brightness": brightness_up,
"decrease_brightness": brightness_down,
"brighter": brightness_up,
"dimmer": brightness_down,
"dim_screen": brightness_down,
"brighten_screen": brightness_up,
"sleep_display": sleep_display,
"turn_off_screen": sleep_display,
"screen_off": sleep_display,
"display_off": sleep_display,
"change_screen": sleep_display,
"screen_sleep": sleep_display,
"monitor_off": sleep_display,
"turn_off_monitor": sleep_display,
"pause_video": pause_video,
"play_video": pause_video,
"pause": pause_video,
"play": pause_video,
"toggle_play": pause_video,
"stop_video": pause_video,
"resume_video": pause_video,
"close_app": close_app,
"close_window": close_window,
"quit_app": close_app,
"exit_app": close_app,
"kill_app": close_app,
"full_screen": full_screen,
"fullscreen": full_screen,
"toggle_fullscreen": full_screen,
"minimize": minimize_window,
"minimize_window": minimize_window,
"maximize": maximize_window,
"maximize_window": maximize_window,
"restore_window": maximize_window,
"snap_left": snap_left,
"snap_right": snap_right,
"window_left": snap_left,
"window_right": snap_right,
"switch_window": switch_window,
"alt_tab": switch_window,
"next_window": switch_window,
"show_desktop": show_desktop,
"desktop": show_desktop,
"hide_windows": show_desktop,
"task_manager": open_task_manager,
"open_task_manager": open_task_manager,
"task_view": open_task_view,
"screenshot": take_screenshot,
"take_screenshot": take_screenshot,
"capture_screen": take_screenshot,
"lock_screen": lock_screen,
"lock": lock_screen,
"open_settings": open_system_settings,
"system_settings": open_system_settings,
"settings": open_system_settings,
"preferences": open_system_settings,
"file_explorer": open_file_explorer,
"open_explorer": open_file_explorer,
"explorer": open_file_explorer,
"open_files": open_file_explorer,
"run": open_run,
"open_run": open_run,
"restart": restart_computer,
"restart_computer": restart_computer,
"reboot": restart_computer,
"reboot_computer": restart_computer,
"shutdown": shutdown_computer,
"shut_down": shutdown_computer,
"power_off": shutdown_computer,
"turn_off_computer": shutdown_computer,
"dark_mode": dark_mode,
"toggle_dark_mode": dark_mode,
"night_mode": dark_mode,
"toggle_wifi": toggle_wifi,
"wifi": toggle_wifi,
"wifi_toggle": toggle_wifi,
"focus_search": focus_search,
"address_bar": focus_search,
"url_bar": focus_search,
"refresh_page": refresh_page,
"reload_page": refresh_page,
"reload": refresh_page,
"refresh": refresh_page,
"close_tab": close_tab,
"new_tab": new_tab,
"open_tab": new_tab,
"next_tab": next_tab,
"prev_tab": prev_tab,
"previous_tab": prev_tab,
"go_back": go_back,
"back": go_back,
"go_forward": go_forward,
"forward": go_forward,
"zoom_in": zoom_in,
"zoom_out": zoom_out,
"zoom_reset": zoom_reset,
"reset_zoom": zoom_reset,
"find_on_page": find_on_page,
"search_page": find_on_page,
"scroll_up": scroll_up,
"scroll_down": scroll_down,
"scroll_top": scroll_top,
"scroll_bottom": scroll_bottom,
"top_of_page": scroll_top,
"bottom_of_page": scroll_bottom,
"page_up": page_up,
"page_down": page_down,
"copy": copy,
"paste": paste,
"cut": cut,
"undo": undo,
"redo": redo,
"select_all": select_all,
"save": save_file,
"save_file": save_file,
"enter": press_enter,
"press_enter": press_enter,
"escape": press_escape,
"press_escape": press_escape,
"cancel": press_escape,
ACTION_MAP: dict[str, callable] = {
"volume_up": volume_up,
"volume_down": volume_down,
"mute": volume_mute,
"unmute": volume_mute,
"toggle_mute": volume_mute,
"brightness_up": brightness_up,
"brightness_down": brightness_down,
"sleep_display": sleep_display,
"screen_off": sleep_display,
"pause_video": pause_video,
"play_pause": pause_video,
"close_app": close_app,
"close_window": close_window,
"full_screen": full_screen,
"fullscreen": full_screen,
"minimize": minimize_window,
"maximize": maximize_window,
"snap_left": snap_left,
"snap_right": snap_right,
"switch_window": switch_window,
"show_desktop": show_desktop,
"task_manager": open_task_manager,
"focus_search": focus_search,
"refresh_page": refresh_page,
"reload": refresh_page,
"close_tab": close_tab,
"new_tab": new_tab,
"next_tab": next_tab,
"prev_tab": prev_tab,
"go_back": go_back,
"go_forward": go_forward,
"zoom_in": zoom_in,
"zoom_out": zoom_out,
"zoom_reset": zoom_reset,
"find_on_page": find_on_page,
"scroll_up": scroll_up,
"scroll_down": scroll_down,
"scroll_top": scroll_top,
"scroll_bottom": scroll_bottom,
"page_up": page_up,
"page_down": page_down,
"copy": copy,
"paste": paste,
"cut": cut,
"undo": undo,
"redo": redo,
"select_all": select_all,
"save": save_file,
"enter": press_enter,
"escape": press_escape,
"screenshot": take_screenshot,
"lock_screen": lock_screen,
"open_settings": open_system_settings,
"file_explorer": open_file_explorer,
"open_run": open_run,
"dark_mode": dark_mode,
"toggle_wifi": toggle_wifi,
"restart": restart_computer,
"shutdown": shutdown_computer,
}
_DANGEROUS_ACTIONS = {"restart", "shutdown"}
def _detect_action(description: str) -> dict:
"""
Gemini ile kullanıcının ne yapmak istediğini anlar.
Herhangi bir dilde çalışır.
Döner: {"action": str, "value": optional}
"""
import google.generativeai as genai
genai.configure(api_key=_get_api_key())
model = genai.GenerativeModel("gemini-2.5-flash-lite")
available = ", ".join(sorted(ACTION_MAP.keys())) + ", volume_set, type_text, write_on_screen, reload_n, press_key"
available = ", ".join(sorted(ACTION_MAP.keys())) + \
", volume_set, type_text, press_key, reload_n"
prompt = f"""The user wants to control their computer. Detect their intent.
prompt = f"""You are an intent detector for a computer control assistant.
User said (in any language): "{description}"
The user issued a command (possibly in any language): "{description}"
Available actions: {available}
Return ONLY valid JSON:
Return ONLY a valid JSON object:
{{"action": "action_name", "value": null_or_value}}
Examples:
- "turn up the volume" → {{"action": "volume_up", "value": null}}
- "set volume to 60" → {{"action": "volume_set", "value": 60}}
- "sesi 80 yap" → {{"action": "volume_set", "value": 80}}
- "close the app" → {{"action": "close_app", "value": null}}
- "uygulamayı kapat" → {{"action": "close_app", "value": null}}
- "type hello world" → {{"action": "type_text", "value": "hello world"}}
- "write good morning on screen" → {{"action": "write_on_screen", "value": "good morning"}}
- "reload page 3 times" → {{"action": "reload_n", "value": 3}}
- "tam ekran yap" → {{"action": "full_screen", "value": null}}
- "sesi kıs" → {{"action": "volume_down", "value": null}}
- "sesi aç" → {{"action": "volume_up", "value": null}}
- "sustur" → {{"action": "mute", "value": null}}
- "monte le son" → {{"action": "volume_up", "value": null}}
- "ekranı kapat" → {{"action": "sleep_display", "value": null}}
- "monitörü kapat" → {{"action": "sleep_display", "value": null}}
- "turn off screen" → {{"action": "sleep_display", "value": null}}
- "turn off monitor" → {{"action": "sleep_display", "value": null}}
- "bilgisayarı yeniden başlat" → {{"action": "restart", "value": null}}
- "restart the computer" → {{"action": "restart", "value": null}}
- "bilgisayarı kapat" → {{"action": "shutdown", "value": null}}
- "shut down" → {{"action": "shutdown", "value": null}}
- "ekranı kilitle" → {{"action": "lock_screen", "value": null}}
- "lock the screen" → {{"action": "lock_screen", "value": null}}
- "küçült" → {{"action": "minimize", "value": null}}
- "minimize the window" → {{"action": "minimize", "value": null}}
- "büyüt" → {{"action": "maximize", "value": null}}
- "parlaklığı artır" → {{"action": "brightness_up", "value": null}}
- "parlaklığı azalt" → {{"action": "brightness_down", "value": null}}
- "increase brightness" → {{"action": "brightness_up", "value": null}}
- "wifi'yi aç" → {{"action": "toggle_wifi", "value": null}}
- "toggle wifi" → {{"action": "toggle_wifi", "value": null}}
- "masaüstünü göster" → {{"action": "show_desktop", "value": null}}
- "show desktop" → {{"action": "show_desktop", "value": null}}
- "yeni sekme aç" → {{"action": "new_tab", "value": null}}
- "sekmeyi kapat" → {{"action": "close_tab", "value": null}}
- "geri git" → {{"action": "go_back", "value": null}}
- "ileri git" → {{"action": "go_forward", "value": null}}
- "sayfayı yenile" → {{"action": "refresh_page", "value": null}}
- "yakınlaştır" → {{"action": "zoom_in", "value": null}}
- "uzaklaştır" → {{"action": "zoom_out", "value": null}}
- "kaydet" → {{"action": "save", "value": null}}
- "geri al" → {{"action": "undo", "value": null}}
- "screenshot al" → {{"action": "screenshot", "value": null}}
- "ekran görüntüsü al" → {{"action": "screenshot", "value": null}}
- "aşağı kaydır" → {{"action": "scroll_down", "value": null}}
- "yukarı kaydır" → {{"action": "scroll_up", "value": null}}
- "karanlık mod" → {{"action": "dark_mode", "value": null}}
- "press f5" → {{"action": "press_key", "value": "f5"}}
- "enter'a bas" → {{"action": "enter", "value": null}}
- "escape'e bas" → {{"action": "escape", "value": null}}
IMPORTANT:
- Always return one of the available actions listed above.
- If the user's intent is clear but uses different wording, map it to the closest action.
- Never invent new action names not in the available list.
- Return ONLY the JSON object, no explanation, no markdown."""
Rules:
- Pick the single best matching action from the available list.
- For volume_set: value is an integer 0-100.
- For type_text: value is the exact text to type.
- For press_key: value is the key name (e.g. "f5", "tab", "enter").
- For reload_n: value is an integer (number of times to reload).
- If no clear match, pick the closest action.
- Return ONLY the JSON, no explanation, no markdown."""
try:
response = model.generate_content(prompt)
text = response.text.strip()
text = __import__("re").sub(r"```(?:json)?", "", text).strip().rstrip("`").strip()
resp = model.generate_content(prompt)
text = re.sub(r"```(?:json)?", "", resp.text).strip().rstrip("`").strip()
return json.loads(text)
except Exception as e:
print(f"[Settings] ⚠️ Intent detection failed: {e}")
print(f"[Settings] Intent detection failed: {e}")
return {"action": description.lower().replace(" ", "_"), "value": None}
def computer_settings(
parameters: dict,
parameters: dict = None,
response=None,
player=None,
session_memory=None,
) -> str:
"""
Bilgisayar ayarları ve UI kontrolleri.
parameters:
action : İşlem adı (verilmezse description'dan Gemini ile tespit edilir)
description : Kullanıcının doğal dil komutu (herhangi bir dilde)
value : İşleme özgü değer (ses seviyesi, yazılacak metin, tekrar sayısı vb.)
"""
if not _PYAUTOGUI:
return "pyautogui is not installed. Run: pip install pyautogui"
@@ -625,10 +625,19 @@ def computer_settings(
action = raw_action.lower().strip().replace(" ", "_").replace("-", "_")
if not action:
return "No action could be determined, sir."
return "No action could be determined."
print(f"[Settings] ⚙️ Action: {action} Value: {value}")
print(f"[Settings] Action: {action} Value: {value} OS: {_OS}")
if player:
player.write_log(f"[Settings] {action}")
if action in _DANGEROUS_ACTIONS:
confirmed = str(params.get("confirmed", "")).lower()
if confirmed not in ("yes", "true", "1", "confirm"):
return (
f"This will {action} the computer. "
f"Please confirm by calling again with confirmed=yes."
)
if action == "volume_set":
try:
@@ -638,42 +647,42 @@ def computer_settings(
return f"Could not set volume: {e}"
if action in ("type_text", "write_on_screen", "type", "write"):
text = str(value or params.get("text", ""))
text = str(value or params.get("text", "")).strip()
if not text:
return "No text provided to type, sir."
enter_after = bool(params.get("press_enter", False))
return "No text provided to type."
enter_after = str(params.get("press_enter", "false")).lower() in ("true", "1", "yes")
type_text(text, press_enter_after=enter_after)
return f"Typed: {text[:60]}"
return f"Typed: {text[:80]}"
if action == "press_key":
key = str(value or params.get("key", ""))
key = str(value or params.get("key", "")).strip()
if not key:
return "No key specified, sir."
return "No key specified."
press_key(key)
return f"Pressed: {key}"
if action in ("reload_n", "refresh_n", "reload_page_n"):
try:
n = int(value or 1)
reload_page_n(n)
return f"Reloaded page {n} time{'s' if n > 1 else ''}."
reload_page_n(int(value or 1))
return f"Reloaded {value or 1} time(s)."
except Exception as e:
return f"Could not reload: {e}"
return f"Reload failed: {e}"
if action in ("scroll_up", "scroll_down"):
try:
amount = int(value or 500)
scroll_up(amount) if action == "scroll_up" else scroll_down(amount)
return f"Scrolled {'up' if action == 'scroll_up' else 'down'}."
except Exception as e:
return f"Scroll failed: {e}"
if action == "scroll_up":
scroll_up(int(value or 500))
return "Scrolled up."
if action == "scroll_down":
scroll_down(int(value or 500))
return "Scrolled down."
func = ACTION_MAP.get(action)
if not func:
return f"Unknown action: '{raw_action}', sir."
return f"Unknown action: '{raw_action}'."
try:
func()
return f"Done: {action}."
except Exception as e:
print(f"[Settings] Action failed ({action}): {e}")
return f"Action failed ({action}): {e}"
+284 -221
View File
@@ -1,244 +1,325 @@
# actions/desktop.py
# AI-powered desktop & wallpaper management
#
# Flow for unknown tasks:
# User request → Gemini generates Python/pyautogui code → Safety check → Execute
#
# Built-in: wallpaper change, icon arrangement, desktop cleanup, organize by type
import os
import sys
import json
import shutil
import subprocess
import ctypes
import tempfile
import pyautogui
import platform
from pathlib import Path
from datetime import datetime
try:
import pyautogui
_PYAUTOGUI = True
except ImportError:
_PYAUTOGUI = False
def get_base_dir():
_OS = platform.system() # "Windows" | "Darwin" | "Linux"
def _get_base_dir() -> Path:
if getattr(sys, "frozen", False):
return Path(sys.executable).parent
return Path(__file__).resolve().parent.parent
BASE_DIR = get_base_dir()
API_CONFIG_PATH = BASE_DIR / "config" / "api_keys.json"
def _get_api_key() -> str:
with open(API_CONFIG_PATH, "r", encoding="utf-8") as f:
path = _get_base_dir() / "config" / "api_keys.json"
with open(path, "r", encoding="utf-8") as f:
return json.load(f)["gemini_api_key"]
def _get_desktop() -> Path:
if _OS == "Linux":
xdg = os.environ.get("XDG_DESKTOP_DIR", "")
if xdg and Path(xdg).exists():
return Path(xdg)
return Path.home() / "Desktop"
def _build_sandbox() -> dict:
import time
BLOCKED_KEYWORDS = [
"os.remove", "shutil.rmtree", "shutil.rm",
"subprocess.run", "subprocess.Popen", "subprocess.call",
"os.system", "exec(", "eval(",
"import os", "import subprocess",
"__import__", "open(",
"sys.exit", "quit()",
]
safe_builtins = {
"print": print,
"len": len, "str": str, "int": int, "float": float,
"bool": bool, "list": list, "dict": dict, "tuple": tuple,
"range": range, "enumerate": enumerate, "sorted": sorted,
"isinstance": isinstance, "hasattr": hasattr, "getattr": getattr,
"max": max, "min": min, "sum": sum, "abs": abs,
"zip": zip, "map": map, "filter": filter,
}
sandbox = {
"__builtins__": safe_builtins,
"Path": Path,
"time": time,
"shutil": type("shutil", (), {
"copy2": shutil.copy2,
"copytree": shutil.copytree,
"disk_usage": shutil.disk_usage,
})(),
"os_path": os.path,
}
if _PYAUTOGUI:
sandbox["pyautogui"] = pyautogui
if _OS == "Windows":
try:
import ctypes
import winreg
sandbox["ctypes"] = ctypes
sandbox["winreg"] = type("winreg", (), {
# Sadece okuma
"OpenKey": winreg.OpenKey,
"QueryValueEx": winreg.QueryValueEx,
"HKEY_CURRENT_USER": winreg.HKEY_CURRENT_USER,
})()
except ImportError:
pass
return sandbox
def _is_safe_code(code: str) -> tuple[bool, str]:
code_lower = code.lower()
for keyword in BLOCKED_KEYWORDS:
if keyword.lower() in code_lower:
return False, f"Blocked operation: '{keyword}'"
return True, "OK"
def _execute_generated_code(code: str, player=None) -> str:
if not code or code.strip() == "UNSAFE":
return "This action cannot be performed safely."
# Kod temizleme
if code.startswith("```"):
lines = code.split("\n")
code = "\n".join(lines[1:-1]).strip()
sandbox = _build_sandbox()
output_lines = []
sandbox["__builtins__"]["print"] = lambda *a: output_lines.append(" ".join(str(x) for x in a))
try:
exec(compile(code, "<jarvis_desktop>", "exec"), sandbox)
return "\n".join(output_lines) if output_lines else "Done."
except Exception as e:
print(f"[Desktop] Exec error: {e}\nCode:\n{code[:300]}")
return f"Execution error: {e}"
def _ask_gemini_for_desktop_action(task: str) -> str:
"""
Asks Gemini to generate safe Python/pyautogui code
to accomplish a desktop-related task.
"""
import google.generativeai as genai
import google.generativeai as genai
genai.configure(api_key=_get_api_key())
model = genai.GenerativeModel("gemini-2.5-flash")
desktop = str(_get_desktop())
prompt = f"""You are a Windows desktop automation expert.
Generate safe Python code using ONLY these allowed modules:
- pyautogui (mouse, keyboard, screenshots)
- pathlib.Path (already imported as Path)
- shutil (ONLY: copy2, copytree, move, disk_usage)
- os.path (ONLY: exists, join, dirname, basename, splitext)
- time (sleep only)
- ctypes (Windows API calls only)
- winreg (registry READ only)
os_specific = ""
if _OS == "Windows":
os_specific = "- ctypes (Windows API calls, read-only)\n- winreg (registry READ only)"
elif _OS == "Darwin":
os_specific = "- subprocess is NOT available; use pyautogui or Path only"
else:
os_specific = "- subprocess is NOT available; use pyautogui or Path only"
prompt = f"""You are a desktop automation assistant.
Current OS: {_OS}
Desktop path: {desktop}
Rules:
- Output ONLY the Python code. No explanation, no markdown, no backticks.
- NO file deletion (os.remove, shutil.rmtree, unlink, etc.)
Generate safe Python code to accomplish the task below.
Allowed modules ONLY:
- pyautogui (mouse, keyboard — if needed)
- pathlib.Path (file/folder inspection only, no deletion)
- shutil.copy2, shutil.copytree, shutil.disk_usage (NO move, NO rmtree)
- os_path (os.path equivalent, read-only)
- time.sleep
{os_specific}
Hard rules:
- NO file deletion (no unlink, no rmtree, no remove)
- NO subprocess calls
- NO exec() or eval()
- NO file write operations
- If task cannot be done safely, output exactly: UNSAFE
- NO exec() or eval() inside the code
- NO import statements (modules are pre-injected)
- NO file write operations except explicitly requested
- If task cannot be done safely with these tools, output exactly: UNSAFE
Task: {task}
Output ONLY the Python code. No explanation, no markdown, no backticks.
Python code:"""
Task: {task}"""
try:
response = model.generate_content(prompt)
code = response.text.strip()
if code.startswith("```"):
lines = code.split("\n")
code = "\n".join(lines[1:-1]).strip()
code = "\n".join(lines[1:-1]).strip()
return code
except Exception as e:
return f"ERROR: {e}"
def _execute_generated_code(code: str) -> str:
"""Safely executes Gemini-generated desktop automation code."""
safe, reason = _is_safe_code(code)
if not safe:
return f"⛔ Blocked for safety: {reason}"
allowed_globals = {
"pyautogui": pyautogui,
"Path": Path,
"shutil": shutil,
"ctypes": ctypes,
"time": __import__("time"),
"os": type("os", (), {
"path": os.path,
"listdir": os.listdir,
"getcwd": os.getcwd,
"environ": os.environ,
})(),
"__builtins__": {
"print": print,
"len": len,
"str": str,
"int": int,
"float": float,
"bool": bool,
"list": list,
"dict": dict,
"range": range,
"enumerate": enumerate,
"sorted": sorted,
"isinstance": isinstance,
"hasattr": hasattr,
"getattr": getattr,
"max": max,
"min": min,
"sum": sum,
}
}
output_lines = []
allowed_globals["print"] = lambda *args: output_lines.append(" ".join(str(a) for a in args))
try:
exec(code, allowed_globals)
return "\n".join(output_lines) if output_lines else "Task completed successfully."
except Exception as e:
return f"Execution error: {e}\n\nCode attempted:\n{code[:200]}"
def set_wallpaper(image_path: str) -> str:
"""Sets desktop wallpaper from a local image path."""
path = Path(image_path).expanduser().resolve()
if not path.exists():
return f"Image not found: {image_path}"
if path.suffix.lower() not in [".jpg", ".jpeg", ".png", ".bmp"]:
return f"Unsupported format: {path.suffix}. Use jpg, png or bmp."
if path.suffix.lower() not in {".jpg", ".jpeg", ".png", ".bmp", ".webp"}:
return f"Unsupported format: {path.suffix}. Use jpg, png, bmp or webp."
try:
if sys.platform == "win32":
abs_path = str(path.resolve())
ctypes.windll.user32.SystemParametersInfoW(20, 0, abs_path, 3)
if _OS == "Windows":
import ctypes
if path.suffix.lower() in {".webp", ".png"}:
try:
from PIL import Image
bmp_path = Path(tempfile.mktemp(suffix=".bmp"))
Image.open(path).convert("RGB").save(bmp_path, "BMP")
path = bmp_path
except ImportError:
pass
ctypes.windll.user32.SystemParametersInfoW(20, 0, str(path), 3)
return f"Wallpaper set: {path.name}"
elif sys.platform == "darwin":
script = f'tell application "Finder" to set desktop picture to POSIX file "{path}"'
subprocess.run(["osascript", "-e", script])
elif _OS == "Darwin":
script = (
f'tell application "System Events" to tell every desktop to '
f'set picture to POSIX file "{path}"'
)
subprocess.run(["osascript", "-e", script], capture_output=True)
return f"Wallpaper set: {path.name}"
else:
subprocess.run(["gsettings", "set", "org.gnome.desktop.background",
"picture-uri", f"file://{path}"])
desktop_env = os.environ.get("XDG_CURRENT_DESKTOP", "").lower()
uri = f"file://{path}"
if "gnome" in desktop_env or "unity" in desktop_env:
subprocess.run([
"gsettings", "set", "org.gnome.desktop.background",
"picture-uri", uri
], capture_output=True)
subprocess.run([
"gsettings", "set", "org.gnome.desktop.background",
"picture-uri-dark", uri
], capture_output=True)
elif "kde" in desktop_env:
# KDE Plasma
script = f"""
var allDesktops = desktops();
for (var i = 0; i < allDesktops.length; i++) {{
d = allDesktops[i];
d.wallpaperPlugin = "org.kde.image";
d.currentConfigGroup = ["Wallpaper", "org.kde.image", "General"];
d.writeConfig("Image", "file://{path}");
}}
"""
subprocess.run(
["qdbus", "org.kde.plasmashell", "/PlasmaShell",
"org.kde.PlasmaShell.evaluateScript", script],
capture_output=True
)
elif "xfce" in desktop_env:
subprocess.run([
"xfconf-query", "-c", "xfce4-desktop",
"-p", "/backdrop/screen0/monitor0/workspace0/last-image",
"-s", str(path)
], capture_output=True)
else:
result = subprocess.run(
["feh", "--bg-scale", str(path)],
capture_output=True
)
if result.returncode != 0:
return (
f"Could not set wallpaper automatically on {desktop_env}. "
f"Try manually or install 'feh'."
)
return f"Wallpaper set: {path.name}"
except Exception as e:
return f"Could not set wallpaper: {e}"
def set_wallpaper_from_web(url: str) -> str:
"""Downloads an image from URL and sets it as wallpaper."""
def set_wallpaper_from_url(url: str) -> str:
try:
import urllib.request
suffix = Path(url.split("?")[0]).suffix or ".jpg"
tmp = Path(tempfile.mktemp(suffix=suffix))
urllib.request.urlretrieve(url, str(tmp))
result = set_wallpaper(str(tmp))
try:
tmp.unlink()
except Exception:
pass
return result
except Exception as e:
return f"Could not download wallpaper: {e}"
def get_current_wallpaper() -> str:
"""Returns the current wallpaper path."""
try:
if sys.platform == "win32":
if _OS == "Windows":
import winreg
key = winreg.OpenKey(winreg.HKEY_CURRENT_USER,
r"Control Panel\Desktop")
key = winreg.OpenKey(
winreg.HKEY_CURRENT_USER, r"Control Panel\Desktop"
)
val, _ = winreg.QueryValueEx(key, "Wallpaper")
winreg.CloseKey(key)
return f"Current wallpaper: {val}"
elif _OS == "Darwin":
script = (
'tell application "System Events" to get picture of desktop 1'
)
result = subprocess.run(
["osascript", "-e", script],
capture_output=True, text=True
)
return f"Current wallpaper: {result.stdout.strip()}"
else:
return "Wallpaper path retrieval not supported on this OS."
desktop_env = os.environ.get("XDG_CURRENT_DESKTOP", "").lower()
if "gnome" in desktop_env or "unity" in desktop_env:
result = subprocess.run(
["gsettings", "get", "org.gnome.desktop.background", "picture-uri"],
capture_output=True, text=True
)
return f"Current wallpaper: {result.stdout.strip()}"
return "Wallpaper path retrieval not supported for this desktop environment."
except Exception as e:
return f"Could not get wallpaper: {e}"
FILE_TYPE_MAP = {
"Images": [".jpg", ".jpeg", ".png", ".gif", ".bmp", ".webp", ".svg", ".ico", ".heic"],
"Documents": [".pdf", ".doc", ".docx", ".txt", ".xls", ".xlsx", ".ppt", ".pptx", ".csv", ".odt"],
"Videos": [".mp4", ".avi", ".mkv", ".mov", ".wmv", ".flv", ".webm", ".m4v"],
"Music": [".mp3", ".wav", ".flac", ".aac", ".ogg", ".wma", ".m4a"],
"Archives": [".zip", ".rar", ".7z", ".tar", ".gz", ".bz2"],
"Code": [".py", ".js", ".html", ".css", ".json", ".xml", ".ts", ".cpp", ".java", ".cs", ".php"],
"Executables": [".exe", ".msi", ".bat", ".cmd", ".sh"],
"Images": {".jpg", ".jpeg", ".png", ".gif", ".bmp", ".webp", ".svg", ".ico", ".heic"},
"Documents": {".pdf", ".doc", ".docx", ".txt", ".xls", ".xlsx",
".ppt", ".pptx", ".csv", ".odt", ".ods", ".odp"},
"Videos": {".mp4", ".avi", ".mkv", ".mov", ".wmv", ".flv", ".webm", ".m4v"},
"Music": {".mp3", ".wav", ".flac", ".aac", ".ogg", ".wma", ".m4a"},
"Archives": {".zip", ".rar", ".7z", ".tar", ".gz", ".bz2", ".xz"},
"Code": {".py", ".js", ".ts", ".html", ".css", ".json", ".xml",
".cpp", ".java", ".cs", ".go", ".rs", ".sh", ".php"},
"Executables": {".exe", ".msi", ".bat", ".cmd", ".sh", ".appimage", ".deb", ".rpm"},
}
_SKIP_EXTENSIONS = {
"Windows": {".lnk", ".url"},
"Darwin": {".webloc"},
"Linux": {".desktop"},
}
def organize_desktop(mode: str = "by_type") -> str:
"""
Organizes desktop files.
mode: 'by_type' — groups by file type (Images, Documents, etc.)
'by_date' — groups by month (2024-01, 2024-02, etc.)
"""
desktop = _get_desktop()
moved = []
skipped = []
desktop = _get_desktop()
skip_exts = _SKIP_EXTENSIONS.get(_OS, set())
moved, skipped = [], []
for item in desktop.iterdir():
if item.is_dir() or item.name.startswith("."):
continue
if item.suffix.lower() == ".lnk":
if item.suffix.lower() in skip_exts:
continue
if mode == "by_date":
mtime = datetime.fromtimestamp(item.stat().st_mtime)
mtime = datetime.fromtimestamp(item.stat().st_mtime)
folder_name = mtime.strftime("%Y-%m")
else:
ext = item.suffix.lower()
ext = item.suffix.lower()
folder_name = "Others"
for folder, exts in FILE_TYPE_MAP.items():
if ext in exts:
@@ -256,31 +337,34 @@ def organize_desktop(mode: str = "by_type") -> str:
shutil.move(str(item), str(new_path))
moved.append(f"{item.name} → {folder_name}/")
result = f"Desktop organized ({mode}). {len(moved)} files moved."
result = f"Desktop organized ({mode}): {len(moved)} files moved."
if moved:
preview = moved[:8]
result += "\n" + "\n".join(preview)
result += "\n" + "\n".join(moved[:8])
if len(moved) > 8:
result += f"\n... and {len(moved)-8} more."
result += f"\n... and {len(moved) - 8} more."
if skipped:
result += f"\n{len(skipped)} files skipped (name conflict)."
result += f"\n{len(skipped)} file(s) skipped (name conflict)."
return result
def list_desktop() -> str:
"""Lists everything on the desktop."""
desktop = _get_desktop()
items = []
for item in sorted(desktop.iterdir()):
if item.name.startswith("."):
continue
if item.is_dir():
count = len(list(item.iterdir()))
try:
count = len(list(item.iterdir()))
except PermissionError:
count = "?"
items.append(f"📁 {item.name}/ ({count} items)")
else:
size = item.stat().st_size
size_str = f"{size/1024:.1f} KB" if size < 1024*1024 else f"{size/1024/1024:.1f} MB"
size = item.stat().st_size
size_str = (
f"{size / 1024:.1f} KB" if size < 1024 * 1024
else f"{size / 1024 / 1024:.1f} MB"
)
items.append(f"📄 {item.name} ({size_str})")
if not items:
@@ -289,11 +373,8 @@ def list_desktop() -> str:
def clean_desktop() -> str:
"""
Moves all files on desktop into a 'Desktop Archive' folder
with today's date — fast cleanup without deleting anything.
"""
desktop = _get_desktop()
skip_exts = _SKIP_EXTENSIONS.get(_OS, set())
today = datetime.now().strftime("%Y-%m-%d")
archive_dir = desktop / f"Desktop Archive {today}"
archive_dir.mkdir(exist_ok=True)
@@ -302,116 +383,98 @@ def clean_desktop() -> str:
for item in desktop.iterdir():
if item.is_dir() or item.name.startswith("."):
continue
if item.suffix.lower() == ".lnk":
if item.suffix.lower() in skip_exts:
continue
new_path = archive_dir / item.name
if not new_path.exists():
shutil.move(str(item), str(new_path))
moved += 1
return f"Desktop cleaned. {moved} files moved to '{archive_dir.name}'."
return f"Desktop cleaned: {moved} files archived to '{archive_dir.name}'."
def get_desktop_stats() -> str:
"""Returns stats about the desktop."""
desktop = _get_desktop()
files = [i for i in desktop.iterdir() if i.is_file()]
folders = [i for i in desktop.iterdir() if i.is_dir()]
total_size = sum(f.stat().st_size for f in files)
size_str = f"{total_size/1024:.1f} KB" if total_size < 1024*1024 else f"{total_size/1024/1024:.1f} MB"
desktop = _get_desktop()
files = [i for i in desktop.iterdir() if i.is_file()]
folders = [i for i in desktop.iterdir() if i.is_dir()]
total_size = sum(f.stat().st_size for f in files if f.exists())
size_str = (
f"{total_size / 1024:.1f} KB" if total_size < 1024 * 1024
else f"{total_size / 1024 / 1024:.1f} MB"
)
return (
f"Desktop stats:\n"
f"Desktop stats ({_OS}):\n"
f" Files : {len(files)}\n"
f" Folders : {len(folders)}\n"
f" Total size: {size_str}"
f" Size : {size_str}\n"
f" Path : {desktop}"
)
def desktop_control(
parameters: dict,
parameters: dict = None,
response=None,
player=None,
session_memory=None
session_memory=None,
) -> str:
"""
Called from main.py.
parameters:
action : wallpaper | wallpaper_url | current_wallpaper |
organize | clean | list | stats |
task (AI-powered — anything else)
path : image path for 'wallpaper'
url : image URL for 'wallpaper_url'
mode : 'by_type' or 'by_date' for 'organize'
task : Natural language description for AI-powered actions.
Example: "arrange icons by size"
"show me what's on my desktop"
"move all screenshots to a folder"
action : wallpaper | wallpaper_url | current_wallpaper |
organize | clean | list | stats |
task (AI-powered)
path : image path for 'wallpaper'
url : image URL for 'wallpaper_url'
mode : 'by_type' or 'by_date' for 'organize'
task : natural language description for AI-powered actions
"""
action = (parameters or {}).get("action", "").lower().strip()
task = (parameters or {}).get("task", "").strip()
params = parameters or {}
action = params.get("action", "").lower().strip()
task = params.get("task", "").strip()
result = "Unknown action."
if player:
player.write_log(f"[desktop] {action or task[:40]}")
try:
if action == "wallpaper":
path = parameters.get("path", "")
result = set_wallpaper(path) if path else "No image path provided."
path = params.get("path", "")
return set_wallpaper(path) if path else "No image path provided."
elif action == "wallpaper_url":
url = parameters.get("url", "")
result = set_wallpaper_from_web(url) if url else "No URL provided."
url = params.get("url", "")
return set_wallpaper_from_url(url) if url else "No URL provided."
elif action == "current_wallpaper":
result = get_current_wallpaper()
return get_current_wallpaper()
elif action == "organize":
mode = parameters.get("mode", "by_type")
result = organize_desktop(mode)
return organize_desktop(params.get("mode", "by_type"))
elif action == "clean":
result = clean_desktop()
return clean_desktop()
elif action == "list":
result = list_desktop()
return list_desktop()
elif action == "stats":
result = get_desktop_stats()
return get_desktop_stats()
elif action == "task" or task:
actual_task = task or parameters.get("description", "")
actual_task = task or params.get("description", "")
if not actual_task:
return "Please describe what you want to do on the desktop, sir."
return "Please describe what you want to do on the desktop."
print(f"[Desktop] 🤖 Asking Gemini: {actual_task}")
print(f"[Desktop] Asking Gemini: {actual_task}")
if player:
player.write_log("[Desktop] Generating action...")
code = _ask_gemini_for_desktop_action(actual_task)
if code == "UNSAFE":
result = "I cannot perform that desktop action safely, sir."
elif code.startswith("ERROR:"):
result = f"Could not generate action: {code}"
else:
print(f"[Desktop] ✅ Generated code:\n{code[:200]}")
result = _execute_generated_code(code)
return _execute_generated_code(code, player=player)
else:
full_task = task or action
if full_task:
code = _ask_gemini_for_desktop_action(full_task)
result = _execute_generated_code(code) if code not in ("UNSAFE",) else "Cannot do that safely."
else:
result = "No action or task specified."
if action:
code = _ask_gemini_for_desktop_action(action)
return _execute_generated_code(code, player=player)
return "No action or task specified."
except Exception as e:
result = f"Desktop control error: {e}"
print(f"[Desktop] {result[:100]}")
if player:
player.write_log(f"[desktop] {result[:60]}")
return result
print(f"[Desktop] Error: {e}")
return f"Desktop control error: {e}"
+340 -234
View File
@@ -1,15 +1,3 @@
# actions/dev_agent.py
# AI-powered development agent — plans, builds, and debugs full projects.
#
# Flow:
# Describe project → Gemini plans file structure → Files written one by one
# → VSCode opened → Entry point executed → Error? → Identify file → Fix → Retry
# → Speaks only when done (success or failure)
#
# Models:
# Planning : gemini-2.5-flash (architecture, structure, debugging)
# Writing : gemini-2.5-flash-lite (fast file generation)
import subprocess
import sys
import json
@@ -17,18 +5,19 @@ import re
import time
from pathlib import Path
def get_base_dir():
if getattr(sys, "frozen", False):
return Path(sys.executable).parent
return Path(__file__).resolve().parent.parent
BASE_DIR = get_base_dir()
API_CONFIG_PATH = BASE_DIR / "config" / "api_keys.json"
PROJECTS_DIR = Path.home() / "Desktop" / "JarvisProjects"
MAX_FIX_ATTEMPTS = 4
MODEL_PLANNER = "gemini-2.5-flash"
MODEL_WRITER = "gemini-2.5-flash-lite"
BASE_DIR = get_base_dir()
API_CONFIG_PATH = BASE_DIR / "config" / "api_keys.json"
PROJECTS_DIR = Path.home() / "Desktop" / "JarvisProjects"
MAX_FIX_ATTEMPTS = 5
MODEL_PLANNER = "gemini-2.5-flash"
MODEL_WRITER = "gemini-2.5-flash"
def _get_api_key() -> str:
with open(API_CONFIG_PATH, "r", encoding="utf-8") as f:
@@ -41,157 +30,198 @@ def _get_model(model_name: str):
return genai.GenerativeModel(model_name)
def _clean_code(text: str) -> str:
def _strip_fences(text: str) -> str:
text = text.strip()
text = re.sub(r"^```[a-zA-Z]*\n?", "", text)
text = re.sub(r"\n?```$", "", text)
return text.strip()
def _clean_json(text: str) -> str:
text = text.strip()
text = re.sub(r"^```[a-zA-Z]*\n?", "", text)
text = re.sub(r"\n?```$", "", text)
text = re.sub(r"^```[a-zA-Z]*\r?\n?", "", text)
text = re.sub(r"\r?\n?```\s*$", "", text)
return text.strip()
def _is_rate_limit(error: Exception) -> bool:
return "429" in str(error) or "quota" in str(error).lower()
msg = str(error).lower()
return "429" in msg or "quota" in msg or "resource_exhausted" in msg
def _get_interpreter(path: Path) -> list[str] | None:
return {
".py": [sys.executable],
".js": ["node"],
".ts": ["ts-node"],
".sh": ["bash"],
".ps1": ["powershell", "-File"],
".rb": ["ruby"],
".php": ["php"],
}.get(path.suffix.lower())
def _parse_traceback(output: str, project_files: list[str]) -> tuple[str | None, int | None]:
pattern = re.compile(r'File ["\']([^"\']+\.py)["\'],\s+line\s+(\d+)', re.IGNORECASE)
matches = pattern.findall(output)
for raw_path, line_str in reversed(matches):
raw_name = Path(raw_path).name
for pf in project_files:
if Path(pf).name == raw_name or pf == raw_path or raw_path.endswith(pf):
return pf, int(line_str)
return None, None
def _has_error(output: str) -> bool:
if "timed out" in output.lower():
def _classify_error(output: str) -> str:
low = output.lower()
if any(x in low for x in ("no module named", "modulenotfounderror", "importerror")):
return "dependency_error"
if "syntaxerror" in low or "invalid syntax" in low:
return "syntax_error"
if "cannot import" in low or "importerror" in low:
return "import_error"
if any(x in low for x in (
"traceback", "exception", "error:", "nameerror", "typeerror",
"attributeerror", "valueerror", "keyerror", "indexerror",
"zerodivisionerror", "filenotfounderror", "permissionerror",
)):
return "runtime_error"
return "none"
def _has_error(output: str, run_command: str) -> bool:
low = output.lower()
if "timed out" in low:
return False
signals = ["error", "exception", "traceback", "syntaxerror",
"nameerror", "typeerror", "importerror", "stderr", "failed"]
return any(s in output.lower() for s in signals)
def _identify_error_file(error_output: str, project_files: list[str]) -> str | None:
"""
Try to find which file caused the error from traceback.
Returns filename or None.
"""
for line in error_output.splitlines():
for f in project_files:
if Path(f).name in line or f in line:
return f
return None
if not output.strip():
return False
error_type = _classify_error(output)
return error_type != "none"
class RateLimitError(Exception):
pass
def _plan_project(description: str, language: str) -> dict:
"""
Ask Gemini to plan the full project structure.
Returns:
{
"project_name": "my_app",
"entry_point": "main.py",
"files": [
{
"path": "main.py",
"description": "Entry point, starts the app"
},
{
"path": "utils/helpers.py",
"description": "Utility functions"
}
],
"run_command": "python main.py",
"dependencies": ["requests", "flask"]
}
"""
model = _get_model(MODEL_PLANNER)
prompt = f"""You are a senior software architect.
Plan the complete file structure for the following project.
prompt = f"""You are a senior software architect. Create a minimal, complete file plan for this project.
Language: {language}
Description: {description}
Return ONLY a valid JSON object with this exact structure:
Return ONLY valid JSON — no markdown, no explanation:
{{
"project_name": "short_snake_case_name",
"project_name": "snake_case_name",
"entry_point": "main.py",
"files": [
{{"path": "main.py", "description": "what this file does"}},
{{"path": "utils/helpers.py", "description": "what this file does"}}
{{
"path": "main.py",
"description": "Entry point — what it does and which modules it imports",
"imports": ["utils.helpers", "core.engine"]
}},
{{
"path": "utils/helpers.py",
"description": "Helper utilities — what functions it exposes",
"imports": []
}}
],
"run_command": "python main.py",
"dependencies": ["package1", "package2"]
"dependencies": ["requests"]
}}
Rules:
- Keep it simple. Only include files that are truly necessary.
- No explanation, no markdown, no backticks. Pure JSON only.
- Entry point must be one of the files listed.
- Use relative paths only.
Critical rules:
1. List files in DEPENDENCY ORDER — files with no imports come first, entry point comes last.
2. The "imports" field must list every other project module this file imports (dot-notation, e.g. "utils.helpers").
3. Keep it minimal — only files truly needed.
4. Entry point must be in the files list.
5. Use relative paths only (e.g. "utils/helpers.py", not absolute paths).
6. Standard library modules (os, sys, json, etc.) do NOT go in "dependencies".
JSON:"""
try:
response = model.generate_content(prompt)
raw = _clean_json(response.text)
raw = _strip_fences(response.text)
return json.loads(raw)
except json.JSONDecodeError as e:
raise ValueError(f"Planner returned invalid JSON: {e}\nRaw: {response.text[:300]}")
except Exception as e:
if _is_rate_limit(e):
raise RateLimitError(str(e))
raise
def _write_file(
file_path: str,
file_description: str,
file_info: dict,
project_description: str,
all_files: list[dict],
language: str,
project_dir: Path
project_dir: Path,
already_written: dict[str, str],
) -> str:
"""Write one file. Returns the generated code."""
model = _get_model(MODEL_WRITER)
file_path = file_info["path"]
file_desc = file_info.get("description", "")
file_imports = file_info.get("imports", [])
file_list = "\n".join(
f" - {f['path']}: {f['description']}" for f in all_files
f" [{i+1}] {f['path']}: {f.get('description', '')}"
for i, f in enumerate(all_files)
)
prompt = f"""You are an expert {language} developer.
Write the code for ONE specific file in a larger project.
dependency_context = ""
for dep_dotted in file_imports:
dep_path = dep_dotted.replace(".", "/") + ".py"
if dep_path in already_written:
code_snippet = already_written[dep_path][:2000]
dependency_context += f"\n\n--- {dep_path} (you must import from this) ---\n{code_snippet}"
lang_rules = ""
if language.lower() == "python":
lang_rules = """
Python-specific rules:
- Use type hints for all function signatures.
- Add docstrings for all public functions and classes.
- Use if __name__ == "__main__": guard in the entry point.
- For relative imports within the project, use: from utils.helpers import foo (match the project structure exactly).
- Do NOT use implicit relative imports (from . import ...) unless it's a proper package with __init__.py.
- If this is a package subdirectory, create __init__.py files where needed."""
elif language.lower() in ("javascript", "typescript", "js", "ts"):
lang_rules = """
JS/TS-specific rules:
- Use ES modules (import/export), not CommonJS (require).
- Add JSDoc comments for all exported functions.
- Handle promise rejections with try/catch in async functions."""
prompt = f"""You are a senior {language} developer writing production-quality code for a real project.
Project goal: {project_description}
All files in this project:
Complete project file structure (in dependency order):
{file_list}
Now write ONLY the file: {file_path}
Purpose of this file: {file_description}
{f"Dependencies this file must import from other project files:{dependency_context}" if dependency_context else ""}
Rules:
- Output ONLY the code for this file. No explanation, no markdown, no backticks.
- Import from other project files using relative imports where needed.
- Add helpful inline comments.
- Handle errors properly.
- Use modern best practices.
Your task: Write the complete, working code for: {file_path}
Purpose of this file: {file_desc}
{f"This file imports from: {', '.join(file_imports)}" if file_imports else "This file has no project-internal imports."}
{lang_rules}
General rules:
- Output ONLY raw code. Absolutely no explanation, no markdown, no triple backticks.
- Write COMPLETE, RUNNABLE code — no placeholders, no "# TODO", no "pass" stubs.
- Every import must either be from the standard library, listed dependencies, or the project files shown above.
- Match import paths EXACTLY to the file paths in the project structure (e.g. if file is "utils/helpers.py", import as "from utils.helpers import ...").
- Use proper error handling (try/except) where I/O or network calls are made.
- The code must work correctly when the project entry point is run from the project root directory.
Code for {file_path}:"""
try:
response = model.generate_content(prompt)
code = _clean_code(response.text)
code = _strip_fences(response.text)
# Save file
full_path = project_dir / file_path
full_path.parent.mkdir(parents=True, exist_ok=True)
full_path.write_text(code, encoding="utf-8")
print(f"[DevAgent] ✅ Written: {file_path}")
print(f"[DevAgent] ✅ Written: {file_path} ({len(code)} chars)")
return code
except Exception as e:
@@ -199,149 +229,214 @@ Code for {file_path}:"""
raise RateLimitError(str(e))
raise
def _install_dependencies(dependencies: list[str], project_dir: Path) -> str:
if not dependencies:
return "No dependencies to install."
return "No external dependencies."
print(f"[DevAgent] 📦 Installing: {dependencies}")
to_install = []
for dep in dependencies:
pkg_name = re.split(r"[>=<!]", dep)[0].strip()
result = subprocess.run(
[sys.executable, "-m", "pip", "show", pkg_name],
capture_output=True, text=True
)
if result.returncode != 0:
to_install.append(dep)
else:
print(f"[DevAgent] ✓ Already installed: {pkg_name}")
if not to_install:
return f"All dependencies already installed: {', '.join(dependencies)}"
print(f"[DevAgent] 📦 Installing: {to_install}")
try:
result = subprocess.run(
[sys.executable, "-m", "pip", "install"] + dependencies,
[sys.executable, "-m", "pip", "install"] + to_install,
capture_output=True, text=True,
encoding="utf-8", errors="replace",
timeout=120, cwd=str(project_dir)
)
if result.returncode == 0:
return f"Installed: {', '.join(dependencies)}"
return f"Install warning: {result.stderr[:200]}"
return f"Installed: {', '.join(to_install)}"
return f"Install warning (non-fatal): {result.stderr[:200]}"
except subprocess.TimeoutExpired:
return "Dependency install timed out."
return "Dependency install timed out (non-fatal)."
except Exception as e:
return f"Install error: {e}"
return f"Install error (non-fatal): {e}"
def _open_vscode(project_dir: Path) -> bool:
vscode_paths = [
vscode_candidates = [
"code",
r"C:\Users\{}\AppData\Local\Programs\Microsoft VS Code\bin\code.cmd".format(
Path.home().name
),
rf"C:\Users\{Path.home().name}\AppData\Local\Programs\Microsoft VS Code\bin\code.cmd",
r"C:\Program Files\Microsoft VS Code\bin\code.cmd",
]
for cmd in vscode_paths:
for cmd in vscode_candidates:
try:
subprocess.Popen(
[cmd, str(project_dir)],
shell=True,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
shell=True
stderr=subprocess.DEVNULL
)
time.sleep(2)
time.sleep(1.5)
print(f"[DevAgent] 💻 VSCode opened: {project_dir}")
return True
except Exception:
continue
print("[DevAgent] ⚠️ VSCode not found.")
return False
def _run_project(run_command: str, project_dir: Path, timeout: int = 30) -> str:
"""Run the project entry point, return output."""
print(f"[DevAgent] 🚀 Running: {run_command}")
try:
parts = run_command.split()
if parts[0] == "python":
if parts[0].lower() == "python":
parts[0] = sys.executable
result = subprocess.run(
parts,
capture_output=True, text=True,
encoding="utf-8", errors="replace",
timeout=timeout, cwd=str(project_dir)
timeout=timeout,
cwd=str(project_dir)
)
output = result.stdout.strip()
error = result.stderr.strip()
stdout = result.stdout.strip()
stderr = result.stderr.strip()
parts_out = []
if output: parts_out.append(f"Output:\n{output}")
if error: parts_out.append(f"Stderr:\n{error}")
return "\n\n".join(parts_out) if parts_out else "Ran with no output."
combined_parts = []
if stdout:
combined_parts.append(f"STDOUT:\n{stdout}")
if stderr:
combined_parts.append(f"STDERR:\n{stderr}")
return "\n\n".join(combined_parts) if combined_parts else "Ran with no output."
except subprocess.TimeoutExpired:
return f"Timed out after {timeout}s. (Long-running app may be working fine.)"
return f"Timed out after {timeout}s — long-running app (server/GUI) is likely working."
except FileNotFoundError as e:
return f"Command not found: {e}"
except Exception as e:
return f"Run error: {e}"
def _fix_file(
file_path: str,
current_code: str,
def _try_auto_install(error_output: str, project_dir: Path) -> bool:
"""ModuleNotFoundError varsa eksik paketi otomatik kurmaya çalışır."""
pattern = re.compile(
r"No module named ['\"]([a-zA-Z0-9_\-\.]+)['\"]", re.IGNORECASE
)
match = pattern.search(error_output)
if not match:
return False
pkg = match.group(1).replace("_", "-").split(".")[0]
print(f"[DevAgent] 🔧 Auto-installing missing package: {pkg}")
try:
result = subprocess.run(
[sys.executable, "-m", "pip", "install", pkg],
capture_output=True, text=True,
encoding="utf-8", errors="replace",
timeout=60, cwd=str(project_dir)
)
return result.returncode == 0
except Exception:
return False
def _fix_files(
error_output: str,
project_description: str,
all_files: list[dict],
file_codes: dict[str, str],
language: str,
project_dir: Path
) -> str:
"""Ask Gemini to fix a specific file based on error output."""
project_dir: Path,
entry_point: str,
) -> dict[str, str]:
model = _get_model(MODEL_PLANNER)
file_list = "\n".join(
f" - {f['path']}: {f['description']}" for f in all_files
)
error_file, error_line = _parse_traceback(error_output, list(file_codes.keys()))
error_type = _classify_error(error_output)
prompt = f"""You are an expert {language} debugger.
Fix the file below. It caused an error when the project was run.
files_to_fix: list[str] = []
if error_file:
files_to_fix.append(error_file)
if error_type == "import_error":
for fi in all_files:
if error_file.replace("/", ".").replace(".py", "") in fi.get("imports", []):
p = fi["path"]
if p not in files_to_fix:
files_to_fix.append(p)
else:
files_to_fix.append(entry_point)
updated_codes: dict[str, str] = {}
for fix_path in files_to_fix:
current_code = file_codes.get(fix_path, "")
other_ctx = ""
for fp, code in file_codes.items():
if fp != fix_path and code:
snippet = code[:1500] + ("..." if len(code) > 1500 else "")
other_ctx += f"\n--- {fp} ---\n{snippet}\n"
line_hint = f"\nError appears to be near line {error_line} in this file." if (
error_line and fix_path == error_file
) else ""
prompt = f"""You are an expert {language} debugger. Fix the broken file below.
Project goal: {project_description}
All files in this project:
{file_list}
All project files:
{chr(10).join(f" - {f['path']}: {f.get('description', '')}" for f in all_files)}
File to fix: {file_path}
Other files for context (read-only — fix only the target file):
{other_ctx[:3500]}
File to fix: {fix_path}{line_hint}
Error type: {error_type}
Error output:
{error_output[:3000]}
{error_output[:2500]}
Current code:
Current (broken) code:
{current_code}
Return ONLY the fixed code — no explanation, no markdown, no backticks.
Rules:
- Output ONLY the complete fixed code. No explanation, no markdown, no backticks.
- Fix ALL errors visible in the error output.
- Keep all existing correct logic — do not remove working features.
- Ensure import paths match the actual project file structure exactly.
- Do NOT introduce new bugs or remove error handling.
Fixed code:"""
Fixed code for {fix_path}:"""
try:
response = model.generate_content(prompt)
fixed = _clean_code(response.text)
try:
response = model.generate_content(prompt)
fixed = _strip_fences(response.text)
full_path = project_dir / file_path
full_path.write_text(fixed, encoding="utf-8")
full_path = project_dir / fix_path
full_path.parent.mkdir(parents=True, exist_ok=True)
full_path.write_text(fixed, encoding="utf-8")
print(f"[DevAgent] 🔧 Fixed: {file_path}")
return fixed
updated_codes[fix_path] = fixed
print(f"[DevAgent] 🔧 Fixed: {fix_path}")
except Exception as e:
if _is_rate_limit(e):
raise RateLimitError(str(e))
raise
except Exception as e:
if _is_rate_limit(e):
raise RateLimitError(str(e))
print(f"[DevAgent] ⚠️ Could not fix {fix_path}: {e}")
return updated_codes
class RateLimitError(Exception):
pass
def _build_project(
description: str,
language: str,
project_name: str,
timeout: int,
speak=None,
player=None
player=None,
) -> str:
"""
Full build loop:
Plan → Write files → Install deps → Open VSCode → Run → Fix loop
"""
def log(msg: str):
print(f"[DevAgent] {msg}")
@@ -352,7 +447,7 @@ def _build_project(
try:
plan = _plan_project(description, language)
except RateLimitError:
msg = "You have reached the rate limit, sir. Please try again shortly."
msg = "Rate limit reached, sir. Please try again in a moment."
if speak: speak(msg)
return msg
except ValueError as e:
@@ -360,64 +455,78 @@ def _build_project(
if speak: speak(msg)
return msg
proj_name = project_name or plan.get("project_name", "jarvis_project")
proj_name = re.sub(r"[^\w\-]", "_", proj_name)
project_dir = PROJECTS_DIR / proj_name
proj_name = project_name or plan.get("project_name", "jarvis_project")
proj_name = re.sub(r"[^\w\-]", "_", proj_name)
project_dir = PROJECTS_DIR / proj_name
project_dir.mkdir(parents=True, exist_ok=True)
files = plan.get("files", [])
entry_point = plan.get("entry_point", "main.py")
run_command = plan.get("run_command", f"python {entry_point}")
files = plan.get("files", [])
entry_point = plan.get("entry_point", "main.py")
run_command = plan.get("run_command", f"python {entry_point}")
dependencies = plan.get("dependencies", [])
log(f"Project: {proj_name} | Files: {len(files)} | Entry: {entry_point}")
def _dep_sort_key(fi: dict) -> int:
return len(fi.get("imports", []))
sorted_files = sorted(files, key=_dep_sort_key)
file_codes: dict[str, str] = {}
for file_info in files:
for file_info in sorted_files:
file_path = file_info.get("path", "")
file_desc = file_info.get("description", "")
if not file_path:
continue
log(f"Writing {file_path}...")
try:
code = _write_file(
file_path, file_desc, description,
files, language, project_dir
)
file_codes[file_path] = code
except RateLimitError:
msg = "You have reached the rate limit, sir. Please try again shortly."
if speak: speak(msg)
return msg
except Exception as e:
log(f"Failed to write {file_path}: {e}")
continue
for attempt in range(2):
try:
code = _write_file(
file_info=file_info,
project_description=description,
all_files=files,
language=language,
project_dir=project_dir,
already_written=file_codes,
)
file_codes[file_path] = code
time.sleep(0.4)
break
except RateLimitError:
if attempt == 0:
log("Rate limit — waiting 20s...")
time.sleep(20)
else:
log(f"Rate limit retry failed for {file_path}, skipping.")
except Exception as e:
log(f"Failed to write {file_path}: {e}")
break
if not file_codes:
msg = "I could not write any files for this project, sir."
msg = "I could not write any project files, sir."
if speak: speak(msg)
return msg
if dependencies:
log(f"Installing dependencies: {dependencies}")
_install_dependencies(dependencies, project_dir)
install_result = _install_dependencies(dependencies, project_dir)
log(install_result)
_open_vscode(project_dir)
last_output = ""
last_output = ""
auto_installs = 0
for attempt in range(1, MAX_FIX_ATTEMPTS + 1):
log(f"Running project (attempt {attempt}/{MAX_FIX_ATTEMPTS})...")
last_output = _run_project(run_command, project_dir, timeout)
log(f"Output: {last_output[:150]}")
log(f"Output preview: {last_output[:150]}")
if not _has_error(last_output):
if not _has_error(last_output, run_command):
msg = (
f"Project '{proj_name}' is working, sir. "
f"Built in {attempt} attempt{'s' if attempt > 1 else ''}. "
f"Opened in VSCode at {project_dir}."
f"Saved to: {project_dir}"
)
if speak: speak(msg)
return f"{msg}\n\nOutput:\n{last_output}"
@@ -425,53 +534,50 @@ def _build_project(
if attempt == MAX_FIX_ATTEMPTS:
break
error_file = _identify_error_file(last_output, list(file_codes.keys()))
if not error_file:
error_file = entry_point
log(f"Error in '{error_file}', fixing...")
error_type = _classify_error(last_output)
if error_type == "dependency_error" and auto_installs < 3:
installed = _try_auto_install(last_output, project_dir)
if installed:
auto_installs += 1
log("Missing dependency installed, retrying...")
time.sleep(1)
continue
log(f"Fixing errors (type: {error_type})...")
try:
fixed = _fix_file(
error_file,
file_codes.get(error_file, ""),
last_output,
description,
files,
language,
project_dir
updated = _fix_files(
error_output=last_output,
project_description=description,
all_files=files,
file_codes=file_codes,
language=language,
project_dir=project_dir,
entry_point=entry_point,
)
file_codes[error_file] = fixed
file_codes.update(updated)
time.sleep(1)
except RateLimitError:
msg = "You have reached the rate limit, sir. Please try again shortly."
msg = "Rate limit reached during fix. Project saved, check it manually in VSCode."
if speak: speak(msg)
return msg
except Exception as e:
log(f"Fix failed: {e}")
log(f"Fix step failed: {e}")
msg = (
f"I was unable to get '{proj_name}' working after {MAX_FIX_ATTEMPTS} attempts, sir. "
f"The project is saved at {project_dir} — you can open it in VSCode and check manually."
f"I couldn't fully fix '{proj_name}' after {MAX_FIX_ATTEMPTS} attempts, sir. "
f"Project is saved at {project_dir} — open it in VSCode and check manually."
)
if speak: speak(msg)
return f"{msg}\n\nLast error:\n{last_output[:500]}"
return f"{msg}\n\nLast error:\n{last_output[:600]}"
def dev_agent(
parameters: dict,
response=None,
player=None,
session_memory=None,
speak=None
speak=None,
) -> str:
"""
Called from main.py.
parameters:
description : What the project should do (required)
language : Programming language (default: python)
project_name : Optional folder name (auto-generated if not given)
timeout : Run timeout in seconds (default: 30)
"""
p = parameters or {}
description = p.get("description", "").strip()
language = p.get("language", "python").strip()
@@ -487,5 +593,5 @@ def dev_agent(
project_name = project_name,
timeout = timeout,
speak = speak,
player = player
player = player,
)
+260 -199
View File
@@ -1,32 +1,83 @@
# actions/file_controller.py
# File management — create, delete, move, rename, list, find, organize
import os
import shutil
import platform
from pathlib import Path
from datetime import datetime
import send2trash
try:
import send2trash
_SEND2TRASH = True
except ImportError:
_SEND2TRASH = False
_OS = platform.system() # "Windows" | "Darwin" | "Linux"
_SAFE_ROOTS: list[Path] = [
Path.home(),
]
def _is_safe_path(target: Path) -> bool:
"""Verilen path _SAFE_ROOTS içinde mi? Değilse işlemi reddet."""
try:
resolved = target.resolve()
return any(
resolved == root.resolve() or resolved.is_relative_to(root.resolve())
for root in _SAFE_ROOTS
)
except Exception:
return False
def _get_desktop() -> Path:
"""Returns desktop path — works on Windows, Mac, Linux."""
if _OS == "Linux":
xdg = os.environ.get("XDG_DESKTOP_DIR", "")
if xdg and Path(xdg).exists():
return Path(xdg)
return Path.home() / "Desktop"
def _get_downloads() -> Path:
if _OS == "Linux":
xdg = os.environ.get("XDG_DOWNLOAD_DIR", "")
if xdg and Path(xdg).exists():
return Path(xdg)
return Path.home() / "Downloads"
def _get_documents() -> Path:
if _OS == "Linux":
xdg = os.environ.get("XDG_DOCUMENTS_DIR", "")
if xdg and Path(xdg).exists():
return Path(xdg)
return Path.home() / "Documents"
def _get_pictures() -> Path:
if _OS == "Linux":
xdg = os.environ.get("XDG_PICTURES_DIR", "")
if xdg and Path(xdg).exists():
return Path(xdg)
return Path.home() / "Pictures"
def _get_music() -> Path:
if _OS == "Linux":
xdg = os.environ.get("XDG_MUSIC_DIR", "")
if xdg and Path(xdg).exists():
return Path(xdg)
return Path.home() / "Music"
def _get_videos() -> Path:
if _OS == "Linux":
xdg = os.environ.get("XDG_VIDEOS_DIR", "")
if xdg and Path(xdg).exists():
return Path(xdg)
return Path.home() / "Videos"
def _resolve_path(raw: str) -> Path:
"""
Resolves a path from user input.
Supports shortcuts: 'desktop', 'downloads', 'documents', 'home'
"""
shortcuts = {
"desktop": Path.home() / "Desktop",
"downloads": Path.home() / "Downloads",
"documents": Path.home() / "Documents",
"pictures": Path.home() / "Pictures",
"music": Path.home() / "Music",
"videos": Path.home() / "Videos",
shortcuts: dict[str, Path] = {
"desktop": _get_desktop(),
"downloads": _get_downloads(),
"documents": _get_documents(),
"pictures": _get_pictures(),
"music": _get_music(),
"videos": _get_videos(),
"home": Path.home(),
}
lower = raw.strip().lower()
@@ -34,20 +85,30 @@ def _resolve_path(raw: str) -> Path:
return shortcuts[lower]
return Path(raw).expanduser()
def _format_size(bytes_size: int) -> str:
"""Converts bytes to human readable format."""
def _format_size(b: int) -> str:
for unit in ["B", "KB", "MB", "GB", "TB"]:
if bytes_size < 1024:
return f"{bytes_size:.1f} {unit}"
bytes_size /= 1024
return f"{bytes_size:.1f} TB"
if b < 1024:
return f"{b:.1f} {unit}"
b /= 1024
return f"{b:.1f} TB"
def _safe_trash(target: Path) -> str:
if not _SEND2TRASH:
return (
"send2trash is not installed. "
"Run: pip install send2trash — "
"Permanent deletion is disabled for safety."
)
send2trash.send2trash(str(target))
return f"Moved to Trash: {target.name}"
def list_files(path: str = "desktop", show_hidden: bool = False) -> str:
"""Lists files and folders in a directory."""
try:
target = _resolve_path(path)
if not _is_safe_path(target):
return f"Access denied: {target}"
if not target.exists():
return f"Path not found: {target}"
if not target.is_dir():
@@ -64,7 +125,7 @@ def list_files(path: str = "desktop", show_hidden: bool = False) -> str:
items.append(f"📄 {item.name} ({size})")
if not items:
return f"Directory is empty: {target}"
return f"Directory is empty: {target.name}/"
return f"Contents of {target.name}/ ({len(items)} items):\n" + "\n".join(items)
@@ -74,10 +135,12 @@ def list_files(path: str = "desktop", show_hidden: bool = False) -> str:
return f"Error listing files: {e}"
def create_file(path: str, content: str = "") -> str:
"""Creates a new file with optional content."""
def create_file(path: str, name: str = "", content: str = "") -> str:
try:
target = Path(path).expanduser()
base = _resolve_path(path)
target = (base / name) if name else base
if not _is_safe_path(target):
return f"Access denied: {target}"
target.parent.mkdir(parents=True, exist_ok=True)
target.write_text(content, encoding="utf-8")
return f"File created: {target.name}"
@@ -85,40 +148,36 @@ def create_file(path: str, content: str = "") -> str:
return f"Could not create file: {e}"
def create_folder(path: str) -> str:
"""Creates a new folder (and parent folders if needed)."""
def create_folder(path: str, name: str = "") -> str:
try:
target = Path(path).expanduser()
base = _resolve_path(path)
target = (base / name) if name else base
if not _is_safe_path(target):
return f"Access denied: {target}"
target.mkdir(parents=True, exist_ok=True)
return f"Folder created: {target}"
return f"Folder created: {target.name}"
except Exception as e:
return f"Could not create folder: {e}"
def delete_file(path: str, confirm: bool = True) -> str:
"""
Deletes a file or folder.
Moves to Recycle Bin on Windows if possible, otherwise permanent delete.
"""
def delete_file(path: str, name: str = "") -> str:
try:
target = Path(path).expanduser()
base = _resolve_path(path)
target = (base / name) if name else base
if not _is_safe_path(target):
return f"Access denied: {target}"
if not target.exists():
return f"Not found: {path}"
return f"Not found: {target.name}"
try:
# Güvenli dizin kontrolü — kritik kullanıcı klasörlerini koru
protected = {
_get_desktop(), _get_downloads(), _get_documents(),
_get_pictures(), _get_music(), _get_videos(), Path.home()
}
if target.resolve() in {p.resolve() for p in protected}:
return f"Protected directory, cannot delete: {target.name}"
send2trash.send2trash(str(target))
return f"Moved to Recycle Bin: {target.name}"
except ImportError:
pass
# Fallback: permanent delete
if target.is_dir():
shutil.rmtree(target)
return f"Folder deleted permanently: {target.name}"
else:
target.unlink()
return f"File deleted permanently: {target.name}"
return _safe_trash(target)
except PermissionError:
return f"Permission denied: {path}"
@@ -126,14 +185,20 @@ def delete_file(path: str, confirm: bool = True) -> str:
return f"Could not delete: {e}"
def move_file(source: str, destination: str) -> str:
"""Moves a file or folder to a new location."""
def move_file(path: str, name: str = "", destination: str = "") -> str:
try:
src = Path(source).expanduser()
dst = _resolve_path(destination)
base = _resolve_path(path)
src = (base / name) if name else base
dst = _resolve_path(destination) if destination else None
if not src.exists():
return f"Source not found: {source}"
return f"Source not found: {src.name}"
if dst is None:
return "No destination specified."
if not _is_safe_path(src):
return f"Access denied (source): {src}"
if not _is_safe_path(dst):
return f"Access denied (destination): {dst}"
if dst.is_dir():
dst = dst / src.name
@@ -146,14 +211,20 @@ def move_file(source: str, destination: str) -> str:
return f"Could not move: {e}"
def copy_file(source: str, destination: str) -> str:
"""Copies a file or folder."""
def copy_file(path: str, name: str = "", destination: str = "") -> str:
try:
src = Path(source).expanduser()
dst = _resolve_path(destination)
base = _resolve_path(path)
src = (base / name) if name else base
dst = _resolve_path(destination) if destination else None
if not src.exists():
return f"Source not found: {source}"
return f"Source not found: {src.name}"
if dst is None:
return "No destination specified."
if not _is_safe_path(src):
return f"Access denied (source): {src}"
if not _is_safe_path(dst):
return f"Access denied (destination): {dst}"
if dst.is_dir():
dst = dst / src.name
@@ -171,16 +242,20 @@ def copy_file(source: str, destination: str) -> str:
return f"Could not copy: {e}"
def rename_file(path: str, new_name: str) -> str:
"""Renames a file or folder."""
def rename_file(path: str, name: str = "", new_name: str = "") -> str:
try:
target = Path(path).expanduser()
new_path = target.parent / new_name
base = _resolve_path(path)
target = (base / name) if name else base
if not _is_safe_path(target):
return f"Access denied: {target}"
if not target.exists():
return f"Not found: {path}"
return f"Not found: {target.name}"
if not new_name:
return "No new name provided."
new_path = target.parent / new_name
if new_path.exists():
return f"A file named '{new_name}' already exists."
return f"A file named '{new_name}' already exists here."
target.rename(new_path)
return f"Renamed: {target.name} → {new_name}"
@@ -189,28 +264,33 @@ def rename_file(path: str, new_name: str) -> str:
return f"Could not rename: {e}"
def read_file(path: str, max_chars: int = 3000) -> str:
"""Reads and returns the content of a text file."""
def read_file(path: str, name: str = "", max_chars: int = 4000) -> str:
try:
target = Path(path).expanduser()
base = _resolve_path(path)
target = (base / name) if name else base
if not _is_safe_path(target):
return f"Access denied: {target}"
if not target.exists():
return f"File not found: {path}"
return f"File not found: {target.name}"
if not target.is_file():
return f"Not a file: {path}"
return f"Not a file: {target.name}"
content = target.read_text(encoding="utf-8", errors="ignore")
if len(content) > max_chars:
content = content[:max_chars] + f"\n\n... (truncated, {len(content)} total chars)"
content = content[:max_chars] + f"\n\n[Truncated — {len(content)} total chars]"
return content
except Exception as e:
return f"Could not read file: {e}"
def write_file(path: str, content: str, append: bool = False) -> str:
"""Writes or appends content to a file."""
def write_file(path: str, name: str = "", content: str = "",
append: bool = False) -> str:
try:
target = Path(path).expanduser()
base = _resolve_path(path)
target = (base / name) if name else base
if not _is_safe_path(target):
return f"Access denied: {target}"
target.parent.mkdir(parents=True, exist_ok=True)
mode = "a" if append else "w"
with open(target, mode, encoding="utf-8") as f:
@@ -221,28 +301,35 @@ def write_file(path: str, content: str, append: bool = False) -> str:
return f"Could not write file: {e}"
def find_files(name: str = "", extension: str = "", path: str = "home",
max_results: int = 20) -> str:
"""
Searches for files by name or extension.
Example: find_files(extension=".pdf", path="documents")
"""
def find_files(name: str = "", extension: str = "",
path: str = "home", max_results: int = 20) -> str:
try:
search_path = _resolve_path(path)
if not _is_safe_path(search_path):
return f"Access denied: {search_path}"
if not search_path.exists():
return f"Search path not found: {path}"
results = []
pattern = f"*{extension}" if extension else "*"
results = []
dir_count = 0
max_dirs = 500 # performans + güvenlik limiti
for item in search_path.rglob(pattern):
if item.is_file():
if name and name.lower() not in item.name.lower():
continue
size = _format_size(item.stat().st_size)
results.append(f"📄 {item.name} ({size}) — {item.parent}")
if len(results) >= max_results:
for item in search_path.rglob("*"):
if item.is_dir():
dir_count += 1
if dir_count > max_dirs:
break
continue
if not item.is_file():
continue
if extension and item.suffix.lower() != extension.lower():
continue
if name and name.lower() not in item.name.lower():
continue
size = _format_size(item.stat().st_size)
results.append(f"📄 {item.name} ({size}) — {item.parent}")
if len(results) >= max_results:
break
if not results:
query = name or extension or "files"
@@ -254,10 +341,12 @@ def find_files(name: str = "", extension: str = "", path: str = "home",
return f"Search error: {e}"
def get_largest_files(path: str = "home", count: int = 10) -> str:
"""Returns the largest files in a directory."""
def get_largest_files(path: str = "downloads", count: int = 10) -> str:
count = min(count, 50) # maksimum 50
try:
search_path = _resolve_path(path)
if not _is_safe_path(search_path):
return f"Access denied: {search_path}"
if not search_path.exists():
return f"Path not found: {path}"
@@ -275,7 +364,7 @@ def get_largest_files(path: str = "home", count: int = 10) -> str:
if not top:
return "No files found."
lines = [f"Top {len(top)} largest files in {search_path.name}/:\n"]
lines = [f"Top {len(top)} largest files in {search_path.name}/:"]
for size, f in top:
lines.append(f" {_format_size(size):>10} {f.name} ({f.parent})")
@@ -286,60 +375,50 @@ def get_largest_files(path: str = "home", count: int = 10) -> str:
def get_disk_usage(path: str = "home") -> str:
"""Returns disk usage information."""
try:
target = _resolve_path(path)
usage = shutil.disk_usage(target)
total = _format_size(usage.total)
used = _format_size(usage.used)
free = _format_size(usage.free)
pct = usage.used / usage.total * 100
return (
f"Disk usage for {target}:\n"
f" Total : {total}\n"
f" Used : {used} ({pct:.1f}%)\n"
f" Free : {free}"
f"Disk usage ({target}):\n"
f" Total : {_format_size(usage.total)}\n"
f" Used : {_format_size(usage.used)} ({pct:.1f}%)\n"
f" Free : {_format_size(usage.free)}"
)
except Exception as e:
return f"Could not get disk usage: {e}"
def organize_desktop() -> str:
"""
Organizes the desktop by grouping files into folders by type.
Creates folders: Images, Documents, Videos, Music, Archives, Others
"""
type_map = {
"Images": {".jpg", ".jpeg", ".png", ".gif", ".bmp", ".webp", ".svg", ".ico", ".heic"},
"Documents": {".pdf", ".doc", ".docx", ".txt", ".xls", ".xlsx",
".ppt", ".pptx", ".csv", ".odt", ".ods", ".odp"},
"Videos": {".mp4", ".avi", ".mkv", ".mov", ".wmv", ".flv", ".webm", ".m4v"},
"Music": {".mp3", ".wav", ".flac", ".aac", ".ogg", ".wma", ".m4a"},
"Archives": {".zip", ".rar", ".7z", ".tar", ".gz", ".bz2", ".xz"},
"Code": {".py", ".js", ".ts", ".html", ".css", ".json", ".xml",
".cpp", ".java", ".cs", ".go", ".rs", ".sh"},
}
desktop = _get_desktop()
moved, skipped = [], []
try:
desktop = _get_desktop()
type_map = {
"Images": [".jpg", ".jpeg", ".png", ".gif", ".bmp", ".webp", ".svg", ".ico"],
"Documents": [".pdf", ".doc", ".docx", ".txt", ".xls", ".xlsx", ".ppt", ".pptx", ".csv"],
"Videos": [".mp4", ".avi", ".mkv", ".mov", ".wmv", ".flv", ".webm"],
"Music": [".mp3", ".wav", ".flac", ".aac", ".ogg", ".wma"],
"Archives": [".zip", ".rar", ".7z", ".tar", ".gz"],
"Code": [".py", ".js", ".html", ".css", ".json", ".xml", ".ts", ".cpp", ".java"],
}
moved = []
skipped = []
for item in desktop.iterdir():
# Klasörlere, gizli dosyalara ve organize klasörlerine dokunma
if item.is_dir() or item.name.startswith("."):
continue
if item.name in {k for k in type_map}:
continue
ext = item.suffix.lower()
target_dir = None
for folder, extensions in type_map.items():
if ext in extensions:
target_dir = desktop / "Others"
for folder, exts in type_map.items():
if ext in exts:
target_dir = desktop / folder
break
if target_dir is None:
target_dir = desktop / "Others"
target_dir.mkdir(exist_ok=True)
new_path = target_dir / item.name
@@ -350,133 +429,115 @@ def organize_desktop() -> str:
shutil.move(str(item), str(new_path))
moved.append(f"{item.name} → {target_dir.name}/")
result = f"Desktop organized. {len(moved)} files moved."
result = f"Desktop organized: {len(moved)} files moved."
if moved:
result += "\n" + "\n".join(moved[:10])
if len(moved) > 10:
result += f"\n... and {len(moved)-10} more."
preview = moved[:8]
result += "\n" + "\n".join(preview)
if len(moved) > 8:
result += f"\n... and {len(moved) - 8} more."
if skipped:
result += f"\n{len(skipped)} files skipped (already exist)."
result += f"\n{len(skipped)} file(s) skipped (name conflict)."
return result
except Exception as e:
return f"Could not organize desktop: {e}"
def get_file_info(path: str) -> str:
"""Returns detailed information about a file."""
def get_file_info(path: str, name: str = "") -> str:
try:
target = Path(path).expanduser()
base = _resolve_path(path)
target = (base / name) if name else base
if not _is_safe_path(target):
return f"Access denied: {target}"
if not target.exists():
return f"Not found: {path}"
return f"Not found: {target.name}"
stat = target.stat()
info = {
"Name": target.name,
"Type": "Folder" if target.is_dir() else "File",
"Size": _format_size(stat.st_size),
"Location": str(target.parent),
"Created": datetime.fromtimestamp(stat.st_ctime).strftime("%Y-%m-%d %H:%M"),
"Modified": datetime.fromtimestamp(stat.st_mtime).strftime("%Y-%m-%d %H:%M"),
"Extension": target.suffix or "None",
"Name": target.name,
"Type": "Folder" if target.is_dir() else "File",
"Size": _format_size(stat.st_size),
"Location": str(target.parent),
"Created": datetime.fromtimestamp(stat.st_ctime).strftime("%Y-%m-%d %H:%M"),
"Modified": datetime.fromtimestamp(stat.st_mtime).strftime("%Y-%m-%d %H:%M"),
"Extension": target.suffix or "—",
}
return "\n".join(f" {k}: {v}" for k, v in info.items())
except Exception as e:
return f"Could not get file info: {e}"
def file_controller(
parameters: dict,
parameters: dict = None,
response=None,
player=None,
session_memory=None
session_memory=None,
) -> str:
action = (parameters or {}).get("action", "").lower().strip()
path = (parameters or {}).get("path", "desktop")
name = (parameters or {}).get("name", "")
content = (parameters or {}).get("content", "")
params = parameters or {}
action = params.get("action", "").lower().strip()
path = params.get("path", "desktop")
name = params.get("name", "")
def _full_path(p: str, n: str) -> str:
base = _resolve_path(p)
if n:
return str(base / n)
return str(base)
result = "Unknown action."
if player:
player.write_log(f"[file] {action} {name or path}")
try:
if action == "list":
result = list_files(path)
return list_files(path)
elif action == "create_file":
full = _full_path(path, name)
result = create_file(full, content=content)
return create_file(path, name=name, content=params.get("content", ""))
elif action == "create_folder":
full = _full_path(path, name)
result = create_folder(full)
return create_folder(path, name=name)
elif action == "delete":
full = _full_path(path, name)
result = delete_file(full)
return delete_file(path, name=name)
elif action == "move":
full = _full_path(path, name)
result = move_file(full, parameters.get("destination", ""))
return move_file(path, name=name, destination=params.get("destination", ""))
elif action == "copy":
full = _full_path(path, name)
result = copy_file(full, parameters.get("destination", ""))
return copy_file(path, name=name, destination=params.get("destination", ""))
elif action == "rename":
full = _full_path(path, name)
result = rename_file(full, parameters.get("new_name", ""))
return rename_file(path, name=name, new_name=params.get("new_name", ""))
elif action == "read":
full = _full_path(path, name)
result = read_file(full)
return read_file(path, name=name)
elif action == "write":
full = _full_path(path, name)
result = write_file(
full,
content=content,
append=parameters.get("append", False)
return write_file(
path, name=name,
content=params.get("content", ""),
append=params.get("append", False)
)
elif action == "find":
result = find_files(
name=name or parameters.get("name", ""),
extension=parameters.get("extension", ""),
return find_files(
name=name or params.get("name", ""),
extension=params.get("extension", ""),
path=path,
max_results=parameters.get("max_results", 20)
max_results=min(int(params.get("max_results", 20)), 50),
)
elif action == "largest":
result = get_largest_files(
return get_largest_files(
path=path,
count=parameters.get("count", 10)
count=int(params.get("count", 10)),
)
elif action == "disk_usage":
result = get_disk_usage(path)
return get_disk_usage(path)
elif action == "organize_desktop":
result = organize_desktop()
return organize_desktop()
elif action == "info":
full = _full_path(path, name)
result = get_file_info(full)
return get_file_info(path, name=name)
else:
result = f"Unknown action: '{action}'"
return f"Unknown action: '{action}'"
except Exception as e:
result = f"File controller error: {e}"
if player:
player.write_log(f"[file] {result[:60]}")
return result
return f"File controller error ({action}): {e}"
+110 -170
View File
@@ -1,37 +1,19 @@
# actions/flight_finder.py
# MARK XXV — Flight Finder
#
# Searches for flights using Google Flights via browser_control.
# Results are spoken by JARVIS. Optionally saved to Notepad or opened in browser.
#
# Flow:
# 1. Parse origin, destination, date, passengers from parameters
# 2. Open Google Flights via browser_control
# 3. Fill in search fields
# 4. Scrape results via get_text
# 5. Parse with Gemini → structured flight data
# 6. Speak top results
# 7. Optionally save to Notepad or keep browser open
#
# Cross-platform: Windows, macOS, Linux
# No API key required — Google Flights is free to access
import json
import re
import sys
import subprocess
import platform
import sys
from datetime import datetime, timedelta
from pathlib import Path
from config import is_windows, is_mac, is_linux
def get_base_dir() -> Path:
def _get_base_dir() -> Path:
if getattr(sys, "frozen", False):
return Path(sys.executable).parent
return Path(__file__).resolve().parent.parent
BASE_DIR = get_base_dir()
BASE_DIR = _get_base_dir()
API_CONFIG_PATH = BASE_DIR / "config" / "api_keys.json"
@@ -39,34 +21,42 @@ def _get_api_key() -> str:
with open(API_CONFIG_PATH, "r", encoding="utf-8") as f:
return json.load(f)["gemini_api_key"]
_MONTH_MAP: dict[str, int] = {
"january": 1, "february": 2, "march": 3, "april": 4,
"may": 5, "june": 6, "july": 7, "august": 8,
"september": 9, "october": 10, "november": 11, "december": 12,
"ocak": 1, "şubat": 2, "mart": 3, "nisan": 4,
"mayıs": 5, "haziran": 6, "temmuz": 7, "ağustos": 8,
"eylül": 9, "ekim": 10, "kasım": 11, "aralık": 12,
}
_RELATIVE_MAP_KEYS = {
"today", "bugün",
"tomorrow", "yarın",
}
def _parse_date(raw: str) -> str:
"""
Converts natural language date to YYYY-MM-DD.
Handles: '15 Mart', 'March 15', '2025-03-15', 'next friday', etc.
Falls back to Gemini for ambiguous inputs.
"""
raw = raw.strip()
raw = raw.strip()
lower = raw.lower()
today = datetime.now()
if re.match(r"\d{4}-\d{2}-\d{2}", raw):
return raw
for fmt in ("%d/%m/%Y", "%m/%d/%Y", "%d.%m.%Y", "%d-%m-%Y"):
try:
return datetime.strptime(raw, fmt).strftime("%Y-%m-%d")
except ValueError:
pass
today = datetime.now()
lower = raw.lower()
relative_map = {
"today": today,
"tomorrow": today + timedelta(days=1),
"bugün": today,
"yarın": today + timedelta(days=1),
relative = {
"today": today, "bugün": today,
"tomorrow": today + timedelta(days=1),
"yarın": today + timedelta(days=1),
}
for key, val in relative_map.items():
for key, val in relative.items():
if key in lower:
return val.strftime("%Y-%m-%d")
@@ -74,27 +64,18 @@ def _parse_date(raw: str) -> str:
import google.generativeai as genai
genai.configure(api_key=_get_api_key())
model = genai.GenerativeModel("gemini-2.5-flash-lite")
today_str = today.strftime("%Y-%m-%d")
response = model.generate_content(
f"Today is {today_str}. Convert this date to YYYY-MM-DD format: '{raw}'. "
f"Today is {today.strftime('%Y-%m-%d')}. "
f"Convert this date expression to YYYY-MM-DD: '{raw}'. "
f"Return ONLY the date string, nothing else."
)
result = response.text.strip()
if re.match(r"\d{4}-\d{2}-\d{2}", result):
return result
except Exception:
pass
except Exception as e:
print(f"[FlightFinder] ⚠️ Gemini date parse failed: {e}")
month_map = {
"january": 1, "february": 2, "march": 3, "april": 4,
"may": 5, "june": 6, "july": 7, "august": 8,
"september": 9,"october": 10, "november": 11, "december": 12,
"january": 1, "ocak": 1, "şubat": 2, "mart": 3,
"nisan": 4, "mayıs": 5, "haziran": 6, "temmuz": 7,
"ağustos": 8, "eylül": 9, "ekim": 10, "kasım": 11,
"aralık": 12,
}
for month_name, month_num in month_map.items():
for month_name, month_num in _MONTH_MAP.items():
if month_name in lower:
day_match = re.search(r"\d{1,2}", raw)
if day_match:
@@ -102,8 +83,16 @@ def _parse_date(raw: str) -> str:
year = today.year if month_num >= today.month else today.year + 1
return f"{year}-{month_num:02d}-{day:02d}"
# Last resort: today
print(f"[FlightFinder] ⚠️ Could not parse date '{raw}' — using today.")
return today.strftime("%Y-%m-%d")
_CABIN_CODE: dict[str, str] = {
"economy": "1",
"premium": "2",
"business": "3",
"first": "4",
}
def _build_google_flights_url(
@@ -114,33 +103,23 @@ def _build_google_flights_url(
passengers: int = 1,
cabin: str = "economy",
) -> str:
"""
Builds a Google Flights URL with pre-filled search parameters.
Uses the direct search URL format.
"""
cabin_map = {
"economy": "1",
"premium": "2",
"business": "3",
"first": "4",
}
cabin_code = cabin_map.get(cabin.lower(), "1")
base = "https://www.google.com/travel/flights"
cabin_code = _CABIN_CODE.get(cabin.lower(), "1")
base = "https://www.google.com/travel/flights"
# Google Flights accepts these query params for pre-filling
if return_date:
url = (
f"{base}?q=Flights+from+{origin}+to+{destination}"
f"+on+{date}+returning+{return_date}"
f"&curr=TRY"
)
trip = f"Flights+from+{origin}+to+{destination}+on+{date}+returning+{return_date}"
else:
url = (
f"{base}?q=Flights+from+{origin}+to+{destination}+on+{date}"
f"&curr=TRY"
)
trip = f"Flights+from+{origin}+to+{destination}+on+{date}"
return url
return (
f"{base}"
f"?q={trip}"
f"&tfs=CBwQAhoeEgoyMDI1LTAzLTE1agcIARIDSVNUcgcIARIDTEhS"
f"&curr=USD"
f"&cabin={cabin_code}"
f"&adults={passengers}"
)
@@ -152,12 +131,8 @@ def _search_flights_browser(
passengers: int,
cabin: str,
) -> tuple[str, str]:
"""
Opens Google Flights in browser, waits for results, scrapes text.
Returns (raw_text, page_url).
"""
from actions.browser_control import browser_control
import time
from actions.browser_control import browser_control
url = _build_google_flights_url(
origin, destination, date, return_date, passengers, cabin
@@ -165,11 +140,10 @@ def _search_flights_browser(
print(f"[FlightFinder] 🌐 Opening: {url}")
browser_control({"action": "go_to", "url": url})
time.sleep(5)
result = browser_control({"action": "get_text"})
return result or "", url
time.sleep(5)
raw = browser_control({"action": "get_text"})
return (raw or ""), url
def _parse_flights_with_gemini(
raw_text: str,
@@ -177,10 +151,6 @@ def _parse_flights_with_gemini(
destination: str,
date: str,
) -> list[dict]:
"""
Sends raw page text to Gemini and extracts structured flight data.
Returns list of flight dicts.
"""
import google.generativeai as genai
genai.configure(api_key=_get_api_key())
@@ -189,47 +159,41 @@ def _parse_flights_with_gemini(
system_instruction=(
"You are a flight data extraction expert. "
"Extract flight information from raw webpage text. "
"Return ONLY valid JSON array. No explanation, no markdown."
)
"Return ONLY valid JSON — no markdown, no explanation."
),
)
truncated = raw_text[:12000]
prompt = (
f"Extract flight options from {origin} to {destination} on {date} "
f"from this Google Flights page text:\n\n{truncated}\n\n"
f"from this Google Flights page text:\n\n{raw_text[:12000]}\n\n"
f"Return a JSON array of up to 5 flights:\n"
f'[{{"airline": "...", "departure": "HH:MM", "arrival": "HH:MM", '
f'"duration": "Xh Ym", "stops": 0, "price": "...", "currency": "..."}}]\n'
f'[{{"airline":"...","departure":"HH:MM","arrival":"HH:MM",'
f'"duration":"Xh Ym","stops":0,"price":"...","currency":"USD"}}]\n'
f"If no flights found, return: []"
)
try:
response = model.generate_content(prompt)
text = response.text.strip()
text = re.sub(r"```(?:json)?", "", text).strip().rstrip("`").strip()
text = re.sub(r"```(?:json)?", "", response.text).strip().rstrip("`").strip()
flights = json.loads(text)
return flights if isinstance(flights, list) else []
except Exception as e:
print(f"[FlightFinder] ⚠️ Parse failed: {e}")
print(f"[FlightFinder] ⚠️ Gemini parse failed: {e}")
return []
def _format_spoken(
flights: list[dict],
origin: str,
destination: str,
date: str,
) -> str:
"""Formats flights for spoken output — concise and natural."""
if not flights:
return (
f"I couldn't find any flights from {origin} to {destination} "
f"on {date}, sir. The page may not have loaded correctly."
)
lines = [f"Here are the flights from {origin} to {destination} on {date}, sir."]
lines = [f"Here are the top flights from {origin} to {destination} on {date}, sir."]
for i, f in enumerate(flights[:5], 1):
airline = f.get("airline", "Unknown airline")
@@ -245,16 +209,17 @@ def _format_spoken(
dur_str = f", {duration}" if duration else ""
lines.append(
f"Option {i}: {airline}, departing {departure}, arriving {arrival}"
f"{dur_str}, {stop_str}, {price_str}."
f"Option {i}: {airline}, departing {departure}, "
f"arriving {arrival}{dur_str}, {stop_str}, {price_str}."
)
cheapest = min(
(f for f in flights if f.get("price")),
key=lambda x: re.sub(r"[^\d]", "", str(x.get("price", "99999"))) or "99999",
default=None,
)
if cheapest:
# Cheapest — strip non-digits for comparison
priced = [f for f in flights if f.get("price")]
if priced:
cheapest = min(
priced,
key=lambda x: int(re.sub(r"[^\d]", "", str(x["price"])) or "999999"),
)
lines.append(
f"The cheapest option is {cheapest.get('airline')} "
f"at {cheapest.get('price')} {cheapest.get('currency', '')}."
@@ -263,7 +228,7 @@ def _format_spoken(
return " ".join(lines)
def _format_notepad(
def _format_text_report(
flights: list[dict],
origin: str,
destination: str,
@@ -271,9 +236,6 @@ def _format_notepad(
return_date: str | None,
page_url: str,
) -> str:
"""Formats flights for Notepad — detailed and readable."""
from datetime import datetime as dt
lines = [
"JARVIS — Flight Search Results",
"─" * 50,
@@ -283,7 +245,7 @@ def _format_notepad(
if return_date:
lines.append(f"Return : {return_date}")
lines += [
f"Searched : {dt.now().strftime('%Y-%m-%d %H:%M')}",
f"Searched : {datetime.now().strftime('%Y-%m-%d %H:%M')}",
f"Source : {page_url}",
"─" * 50,
"",
@@ -297,10 +259,10 @@ def _format_notepad(
stop_str = "Non-stop" if stops == 0 else f"{stops} stop(s)"
lines += [
f"Flight {i}:",
f" Airline : {f.get('airline', 'N/A')}",
f" Airline : {f.get('airline', 'N/A')}",
f" Departure : {f.get('departure', 'N/A')}",
f" Arrival : {f.get('arrival', 'N/A')}",
f" Duration : {f.get('duration', 'N/A')}",
f" Arrival : {f.get('arrival', 'N/A')}",
f" Duration : {f.get('duration', 'N/A')}",
f" Stops : {stop_str}",
f" Price : {f.get('price', 'N/A')} {f.get('currency', '')}",
"",
@@ -308,11 +270,7 @@ def _format_notepad(
return "\n".join(lines)
def _save_to_notepad(content: str, origin: str, destination: str) -> str:
"""Saves flight results to Desktop and opens in default text editor."""
from datetime import datetime
def _save_to_desktop(content: str, origin: str, destination: str) -> str:
ts = datetime.now().strftime("%Y%m%d_%H%M%S")
filename = f"flights_{origin}_{destination}_{ts}.txt".replace(" ", "_")
desktop = Path.home() / "Desktop"
@@ -322,58 +280,39 @@ def _save_to_notepad(content: str, origin: str, destination: str) -> str:
filepath.write_text(content, encoding="utf-8")
print(f"[FlightFinder] 💾 Saved: {filepath}")
system = platform.system()
open_fn = {
"Windows": lambda p: subprocess.Popen(["notepad.exe", str(p)]),
"Darwin": lambda p: subprocess.Popen(["open", "-t", str(p)]),
"Linux": lambda p: subprocess.Popen(["xdg-open", str(p)]),
}
opener = open_fn.get(system)
if opener:
opener(filepath)
try:
if is_windows():
subprocess.Popen(["notepad.exe", str(filepath)])
elif is_mac():
subprocess.Popen(["open", "-t", str(filepath)])
else:
subprocess.Popen(["xdg-open", str(filepath)])
except Exception as e:
print(f"[FlightFinder] ⚠️ Could not open text editor: {e}")
return str(filepath)
def flight_finder(
parameters: dict,
response=None,
player=None,
session_memory=None,
speak=None,
) -> str:
"""
Flight Finder — searches Google Flights and speaks results.
Parameters:
origin (str, required) — departure city or airport (e.g. "Istanbul", "IST")
destination (str, required) — arrival city or airport (e.g. "London", "LHR")
date (str, required) — departure date (any format: "15 Mart", "March 15", "2025-03-15")
return_date (str, optional) — return date for round trips
passengers (int, optional) — number of passengers (default: 1)
cabin (str, optional) — economy | premium | business | first (default: economy)
save (bool, optional) — save results to Notepad (default: False)
show_browser (bool, optional) — keep browser open after search (default: True)
Examples:
flight_finder({"origin": "Istanbul", "destination": "London", "date": "15 Mart"})
flight_finder({"origin": "IST", "destination": "JFK", "date": "2025-04-01",
"return_date": "2025-04-15", "cabin": "business", "save": True})
"""
def flight_finder(parameters: dict, player=None, speak=None) -> str:
params = parameters or {}
origin = params.get("origin", "").strip()
destination = params.get("destination", "").strip()
date_raw = params.get("date", "").strip()
return_raw = params.get("return_date", "").strip()
passengers = int(params.get("passengers", 1))
cabin = params.get("cabin", "economy").strip()
save = params.get("save", False)
return_raw = (params.get("return_date") or "").strip()
passengers = max(1, int(params.get("passengers", 1)))
cabin = params.get("cabin", "economy").strip().lower()
save = bool(params.get("save", False))
if not origin or not destination:
return "Please provide both origin and destination, sir."
if not date_raw:
return "Please provide a departure date, sir."
# Normalise cabin value
if cabin not in _CABIN_CODE:
cabin = "economy"
date = _parse_date(date_raw)
return_date = _parse_date(return_raw) if return_raw else None
@@ -383,10 +322,13 @@ def flight_finder(
if speak:
speak(f"Searching flights from {origin} to {destination} on {date}, sir.")
print(f"[FlightFinder] ▶️ {origin} → {destination} | {date} | {cabin} | {passengers} pax")
print(
f"[FlightFinder] ▶️ {origin} → {destination} | {date}"
f"{' → ' + return_date if return_date else ''}"
f" | {cabin} | {passengers} pax"
)
try:
raw_text, page_url = _search_flights_browser(
origin, destination, date, return_date, passengers, cabin
)
@@ -395,25 +337,23 @@ def flight_finder(
return "Could not retrieve flight data, sir. The page may not have loaded."
if speak:
speak("Analysing the results now.")
speak("Analysing the results now, sir.")
flights = _parse_flights_with_gemini(raw_text, origin, destination, date)
spoken = _format_spoken(flights, origin, destination, date)
spoken = _format_spoken(flights, origin, destination, date)
if speak:
speak(spoken)
result = spoken
if save and flights:
notepad_content = _format_notepad(
flights, origin, destination, date, return_date, page_url
)
saved_path = _save_to_notepad(notepad_content, origin, destination)
result += f" Results saved to Desktop: {saved_path}"
report = _format_text_report(flights, origin, destination, date, return_date, page_url)
saved_path = _save_to_desktop(report, origin, destination)
result += f" Results saved to Desktop: {saved_path}"
return result
except Exception as e:
print(f"[FlightFinder] ❌ Error: {e}")
print(f"[FlightFinder] ❌ {e}")
return f"Flight search failed, sir: {e}"
File diff suppressed because it is too large. Load diff
+151 -102
View File
@@ -1,6 +1,3 @@
# actions/open_app.py
# MARK XXV — Cross-Platform App Launcher
import time
import subprocess
import platform
@@ -12,92 +9,120 @@ try:
except ImportError:
_PSUTIL = False
_APP_ALIASES = {
"whatsapp": {"Windows": "WhatsApp", "Darwin": "WhatsApp", "Linux": "whatsapp"},
"chrome": {"Windows": "chrome", "Darwin": "Google Chrome", "Linux": "google-chrome"},
"google chrome": {"Windows": "chrome", "Darwin": "Google Chrome", "Linux": "google-chrome"},
"firefox": {"Windows": "firefox", "Darwin": "Firefox", "Linux": "firefox"},
"spotify": {"Windows": "Spotify", "Darwin": "Spotify", "Linux": "spotify"},
"vscode": {"Windows": "code", "Darwin": "Visual Studio Code", "Linux": "code"},
"visual studio code": {"Windows": "code", "Darwin": "Visual Studio Code", "Linux": "code"},
"discord": {"Windows": "Discord", "Darwin": "Discord", "Linux": "discord"},
"telegram": {"Windows": "Telegram", "Darwin": "Telegram", "Linux": "telegram"},
"instagram": {"Windows": "Instagram", "Darwin": "Instagram", "Linux": "instagram"},
"tiktok": {"Windows": "TikTok", "Darwin": "TikTok", "Linux": "tiktok"},
"notepad": {"Windows": "notepad.exe", "Darwin": "TextEdit", "Linux": "gedit"},
"calculator": {"Windows": "calc.exe", "Darwin": "Calculator", "Linux": "gnome-calculator"},
"terminal": {"Windows": "cmd.exe", "Darwin": "Terminal", "Linux": "gnome-terminal"},
"cmd": {"Windows": "cmd.exe", "Darwin": "Terminal", "Linux": "bash"},
"explorer": {"Windows": "explorer.exe", "Darwin": "Finder", "Linux": "nautilus"},
"file explorer": {"Windows": "explorer.exe", "Darwin": "Finder", "Linux": "nautilus"},
"paint": {"Windows": "mspaint.exe", "Darwin": "Preview", "Linux": "gimp"},
"word": {"Windows": "winword", "Darwin": "Microsoft Word", "Linux": "libreoffice --writer"},
"excel": {"Windows": "excel", "Darwin": "Microsoft Excel", "Linux": "libreoffice --calc"},
"powerpoint": {"Windows": "powerpnt", "Darwin": "Microsoft PowerPoint","Linux": "libreoffice --impress"},
"vlc": {"Windows": "vlc", "Darwin": "VLC", "Linux": "vlc"},
"zoom": {"Windows": "Zoom", "Darwin": "zoom.us", "Linux": "zoom"},
"slack": {"Windows": "Slack", "Darwin": "Slack", "Linux": "slack"},
"steam": {"Windows": "steam", "Darwin": "Steam", "Linux": "steam"},
"task manager": {"Windows": "taskmgr.exe", "Darwin": "Activity Monitor", "Linux": "gnome-system-monitor"},
"settings": {"Windows": "ms-settings:", "Darwin": "System Preferences", "Linux": "gnome-control-center"},
"powershell": {"Windows": "powershell.exe", "Darwin": "Terminal", "Linux": "bash"},
"edge": {"Windows": "msedge", "Darwin": "Microsoft Edge", "Linux": "microsoft-edge"},
"brave": {"Windows": "brave", "Darwin": "Brave Browser", "Linux": "brave-browser"},
"obsidian": {"Windows": "Obsidian", "Darwin": "Obsidian", "Linux": "obsidian"},
"notion": {"Windows": "Notion", "Darwin": "Notion", "Linux": "notion"},
"blender": {"Windows": "blender", "Darwin": "Blender", "Linux": "blender"},
"capcut": {"Windows": "CapCut", "Darwin": "CapCut", "Linux": "capcut"},
"postman": {"Windows": "Postman", "Darwin": "Postman", "Linux": "postman"},
"figma": {"Windows": "Figma", "Darwin": "Figma", "Linux": "figma"},
_SYSTEM = platform.system()
_APP_ALIASES: dict[str, dict[str, str]] = {
"chrome": {"Windows": "chrome", "Darwin": "Google Chrome", "Linux": "google-chrome"},
"google chrome": {"Windows": "chrome", "Darwin": "Google Chrome", "Linux": "google-chrome"},
"firefox": {"Windows": "firefox", "Darwin": "Firefox", "Linux": "firefox"},
"edge": {"Windows": "msedge", "Darwin": "Microsoft Edge", "Linux": "microsoft-edge"},
"brave": {"Windows": "brave", "Darwin": "Brave Browser", "Linux": "brave-browser"},
"safari": {"Windows": "msedge", "Darwin": "Safari", "Linux": "firefox"},
"opera": {"Windows": "opera", "Darwin": "Opera", "Linux": "opera"},
"whatsapp": {"Windows": "WhatsApp", "Darwin": "WhatsApp", "Linux": "whatsapp"},
"telegram": {"Windows": "Telegram", "Darwin": "Telegram", "Linux": "telegram"},
"discord": {"Windows": "Discord", "Darwin": "Discord", "Linux": "discord"},
"slack": {"Windows": "Slack", "Darwin": "Slack", "Linux": "slack"},
"zoom": {"Windows": "Zoom", "Darwin": "zoom.us", "Linux": "zoom"},
"teams": {"Windows": "msteams", "Darwin": "Microsoft Teams", "Linux": "teams"},
"skype": {"Windows": "skype", "Darwin": "Skype", "Linux": "skype"},
"signal": {"Windows": "signal", "Darwin": "Signal", "Linux": "signal"},
"spotify": {"Windows": "Spotify", "Darwin": "Spotify", "Linux": "spotify"},
"vlc": {"Windows": "vlc", "Darwin": "VLC", "Linux": "vlc"},
"netflix": {"Windows": "Netflix", "Darwin": "Netflix", "Linux": "firefox"},
"vscode": {"Windows": "code", "Darwin": "Visual Studio Code", "Linux": "code"},
"visual studio code": {"Windows": "code", "Darwin": "Visual Studio Code", "Linux": "code"},
"code": {"Windows": "code", "Darwin": "Visual Studio Code", "Linux": "code"},
"terminal": {"Windows": "wt", "Darwin": "Terminal", "Linux": "gnome-terminal"},
"cmd": {"Windows": "cmd.exe", "Darwin": "Terminal", "Linux": "bash"},
"powershell": {"Windows": "powershell.exe", "Darwin": "Terminal", "Linux": "bash"},
"postman": {"Windows": "Postman", "Darwin": "Postman", "Linux": "postman"},
"git": {"Windows": "git-bash", "Darwin": "Terminal", "Linux": "bash"},
"figma": {"Windows": "Figma", "Darwin": "Figma", "Linux": "figma"},
"blender": {"Windows": "blender", "Darwin": "Blender", "Linux": "blender"},
"word": {"Windows": "winword", "Darwin": "Microsoft Word", "Linux": "libreoffice --writer"},
"excel": {"Windows": "excel", "Darwin": "Microsoft Excel", "Linux": "libreoffice --calc"},
"powerpoint": {"Windows": "powerpnt", "Darwin": "Microsoft PowerPoint", "Linux": "libreoffice --impress"},
"libreoffice": {"Windows": "soffice", "Darwin": "LibreOffice", "Linux": "libreoffice"},
"notepad": {"Windows": "notepad.exe", "Darwin": "TextEdit", "Linux": "gedit"},
"textedit": {"Windows": "notepad.exe", "Darwin": "TextEdit", "Linux": "gedit"},
"explorer": {"Windows": "explorer.exe", "Darwin": "Finder", "Linux": "nautilus"},
"file explorer": {"Windows": "explorer.exe", "Darwin": "Finder", "Linux": "nautilus"},
"finder": {"Windows": "explorer.exe", "Darwin": "Finder", "Linux": "nautilus"},
"task manager": {"Windows": "taskmgr.exe", "Darwin": "Activity Monitor", "Linux": "gnome-system-monitor"},
"settings": {"Windows": "ms-settings:", "Darwin": "System Preferences", "Linux": "gnome-control-center"},
"calculator": {"Windows": "calc.exe", "Darwin": "Calculator", "Linux": "gnome-calculator"},
"paint": {"Windows": "mspaint.exe", "Darwin": "Preview", "Linux": "gimp"},
"instagram": {"Windows": "Instagram", "Darwin": "Instagram", "Linux": "firefox"},
"tiktok": {"Windows": "TikTok", "Darwin": "TikTok", "Linux": "firefox"},
"notion": {"Windows": "Notion", "Darwin": "Notion", "Linux": "notion"},
"obsidian": {"Windows": "Obsidian", "Darwin": "Obsidian", "Linux": "obsidian"},
"capcut": {"Windows": "CapCut", "Darwin": "CapCut", "Linux": "capcut"},
"steam": {"Windows": "steam", "Darwin": "Steam", "Linux": "steam"},
"epic": {"Windows": "EpicGamesLauncher", "Darwin": "Epic Games Launcher", "Linux": "legendary"},
"epic games": {"Windows": "EpicGamesLauncher", "Darwin": "Epic Games Launcher", "Linux": "legendary"},
}
def _normalize(raw: str) -> str:
system = platform.system()
key = raw.lower().strip()
key = raw.lower().strip()
if key in _APP_ALIASES:
return _APP_ALIASES[key].get(system, raw)
return _APP_ALIASES[key].get(_SYSTEM, raw)
for alias_key, os_map in _APP_ALIASES.items():
if alias_key in key or key in alias_key:
return os_map.get(system, raw)
return raw
def _is_running(app_name: str) -> bool:
if not _PSUTIL:
return True
app_lower = app_name.lower().replace(" ", "").replace(".exe", "")
try:
for proc in psutil.process_iter(["name"]):
try:
proc_name = proc.info["name"].lower().replace(" ", "").replace(".exe", "")
if app_lower in proc_name or proc_name in app_lower:
return True
except (psutil.NoSuchProcess, psutil.AccessDenied):
continue
except Exception:
pass
return False
return os_map.get(_SYSTEM, raw)
return raw
def _launch_windows(app_name: str) -> bool:
if shutil.which(app_name) or shutil.which(app_name.split(".")[0]):
try:
subprocess.Popen(
app_name,
shell=True,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
)
time.sleep(1.5)
return True
except Exception as e:
print(f"[open_app] subprocess failed: {e}")
if ":" in app_name:
try:
subprocess.Popen(f"start {app_name}", shell=True)
time.sleep(1.0)
return True
except Exception:
pass
try:
import pyautogui
pyautogui.PAUSE = 0.1
pyautogui.press("win")
time.sleep(0.6)
time.sleep(0.7)
pyautogui.write(app_name, interval=0.05)
time.sleep(0.8)
time.sleep(0.9)
pyautogui.press("enter")
time.sleep(3.0)
time.sleep(2.5)
return True
except Exception as e:
print(f"[open_app] ⚠️ Windows launch failed: {e}")
return False
print(f"[open_app] Start Menu search failed: {e}")
return False
def _launch_macos(app_name: str) -> bool:
try:
result = subprocess.run(["open", "-a", app_name], capture_output=True, timeout=8)
result = subprocess.run(
["open", "-a", app_name],
capture_output=True, timeout=8
)
if result.returncode == 0:
time.sleep(1.0)
return True
@@ -105,13 +130,29 @@ def _launch_macos(app_name: str) -> bool:
pass
try:
result = subprocess.run(["open", "-a", f"{app_name}.app"], capture_output=True, timeout=8)
result = subprocess.run(
["open", "-a", f"{app_name}.app"],
capture_output=True, timeout=8
)
if result.returncode == 0:
time.sleep(1.0)
return True
except Exception:
pass
binary = shutil.which(app_name) or shutil.which(app_name.lower())
if binary:
try:
subprocess.Popen(
[binary],
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL
)
time.sleep(1.0)
return True
except Exception:
pass
try:
import pyautogui
pyautogui.hotkey("command", "space")
@@ -122,37 +163,54 @@ def _launch_macos(app_name: str) -> bool:
time.sleep(1.5)
return True
except Exception as e:
print(f"[open_app] ⚠️ macOS Spotlight failed: {e}")
return False
print(f"[open_app] Spotlight failed: {e}")
return False
def _launch_linux(app_name: str) -> bool:
binary = (
shutil.which(app_name) or
shutil.which(app_name.lower()) or
shutil.which(app_name.lower().replace(" ", "-"))
shutil.which(app_name.lower().replace(" ", "-")) or
shutil.which(app_name.lower().replace(" ", "_"))
)
if binary:
try:
subprocess.Popen([binary], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
subprocess.Popen(
[binary],
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL
)
time.sleep(1.0)
return True
except Exception:
pass
try:
subprocess.run(["xdg-open", app_name], capture_output=True, timeout=5)
subprocess.run(
["xdg-open", app_name],
capture_output=True, timeout=5
)
return True
except Exception:
pass
try:
desktop_name = app_name.lower().replace(" ", "-")
subprocess.run(["gtk-launch", desktop_name], capture_output=True, timeout=5)
return True
except Exception:
pass
for desktop_name in [
app_name.lower(),
app_name.lower().replace(" ", "-"),
app_name.lower().replace(" ", ""),
]:
try:
result = subprocess.run(
["gtk-launch", desktop_name],
capture_output=True, timeout=5
)
if result.returncode == 0:
return True
except Exception:
pass
return False
@@ -163,7 +221,6 @@ _OS_LAUNCHERS = {
"Linux": _launch_linux,
}
def open_app(
parameters=None,
response=None,
@@ -173,36 +230,28 @@ def open_app(
app_name = (parameters or {}).get("app_name", "").strip()
if not app_name:
return "Please specify which application to open, sir."
system = platform.system()
launcher = _OS_LAUNCHERS.get(system)
return "No application name provided."
launcher = _OS_LAUNCHERS.get(_SYSTEM)
if launcher is None:
return f"Unsupported OS: {system}"
return f"Unsupported operating system: {_SYSTEM}"
normalized = _normalize(app_name)
print(f"[open_app] 🚀 Launching: {app_name} → {normalized} ({system})")
print(f"[open_app] Launching: '{app_name}' → '{normalized}' ({_SYSTEM})")
if player:
player.write_log(f"[open_app] {app_name}")
try:
success = launcher(normalized)
if success:
return f"Opened {app_name} successfully, sir."
if normalized != app_name:
success = launcher(app_name)
if success:
return f"Opened {app_name} successfully, sir."
if launcher(normalized):
return f"Opened {app_name}."
if normalized.lower() != app_name.lower():
if launcher(app_name):
return f"Opened {app_name}."
return (
f"I tried to open {app_name}, sir, but couldn't confirm it launched. "
f"It may still be loading or might not be installed."
f"Could not confirm that {app_name} launched. "
f"It may still be loading, or it might not be installed."
)
except Exception as e:
print(f"[open_app] ❌ {e}")
return f"Failed to open {app_name}, sir: {e}"
print(f"[open_app] Error: {e}")
return f"Failed to open {app_name}: {e}"
+313 -137
View File
@@ -1,156 +1,332 @@
# actions/reminder.py
import subprocess
import json
import os
import shutil
import subprocess
import sys
from datetime import datetime
from pathlib import Path
def _base_dir() -> Path:
if getattr(sys, "frozen", False):
return Path(sys.executable).parent
return Path(__file__).resolve().parent.parent
def _get_os() -> str:
try:
cfg = json.loads(
(_base_dir() / "config" / "api_keys.json").read_text(encoding="utf-8")
)
return cfg.get("os_system", "windows").lower()
except Exception:
return "windows"
def _scripts_dir() -> Path:
d = Path.home() / ".jarvis" / "reminders"
d.mkdir(parents=True, exist_ok=True)
return d
def _sanitise(text: str, max_len: int = 200) -> str:
return (
text.replace("\\", "")
.replace('"', "")
.replace("'", "")
.replace("\n", " ")
.replace("\r", "")
.strip()
)[:max_len]
def _write_notify_script(task_name: str, message: str, os_name: str) -> Path:
script_path = _scripts_dir() / f"{task_name}.py"
msg_literal = json.dumps(message)
if os_name == "windows":
notify_block = f"""
message = {msg_literal}
notified = False
try:
from plyer import notification
notification.notify(title="J.A.R.V.I.S Reminder", message=message, timeout=15)
notified = True
except Exception:
pass
if not notified:
try:
from win10toast import ToastNotifier
ToastNotifier().show_toast("J.A.R.V.I.S Reminder", message, duration=15, threaded=False)
notified = True
except Exception:
pass
if not notified:
try:
import subprocess
subprocess.run(["msg", "*", "/TIME:30", message], check=False)
except Exception:
pass
try:
import winsound
for freq in [800, 1000, 1200]:
winsound.Beep(freq, 180)
import time; time.sleep(0.08)
except Exception:
pass
"""
elif os_name == "mac":
notify_block = f"""
message = {msg_literal}
notified = False
try:
from plyer import notification
notification.notify(title="J.A.R.V.I.S Reminder", message=message, timeout=15)
notified = True
except Exception:
pass
if not notified:
try:
import subprocess
script = 'display notification "{{}}" with title "J.A.R.V.I.S Reminder"'.format(
message.replace('"', '')
)
subprocess.run(["osascript", "-e", script], check=False)
except Exception:
pass
"""
else: # linux
notify_block = f"""
message = {msg_literal}
notified = False
try:
from plyer import notification
notification.notify(title="J.A.R.V.I.S Reminder", message=message, timeout=15)
notified = True
except Exception:
pass
if not notified:
try:
import subprocess
subprocess.run(
["notify-send", "--urgency=normal", "--expire-time=15000",
"J.A.R.V.I.S Reminder", message],
check=False
)
except Exception:
pass
"""
script_body = f"""# Auto-generated by J.A.R.V.I.S reminder — do not edit
import sys, os, pathlib
{notify_block}
# Self-delete after firing
try:
pathlib.Path(__file__).unlink(missing_ok=True)
except Exception:
pass
"""
script_path.write_text(script_body, encoding="utf-8")
script_path.chmod(0o600) # owner read/write only
return script_path
def _schedule_windows(target_dt: datetime, task_name: str,
script_path: Path, message: str) -> str:
python_exe = Path(sys.executable)
pythonw = python_exe.parent / "pythonw.exe"
if pythonw.exists():
python_exe = pythonw
xml_path = _scripts_dir() / f"{task_name}.xml"
xml_content = (
'<?xml version="1.0" encoding="UTF-16"?>\n'
'<Task version="1.2" xmlns="http://schemas.microsoft.com/windows/2004/02/mit/task">\n'
' <RegistrationInfo><Description>J.A.R.V.I.S Reminder</Description></RegistrationInfo>\n'
' <Triggers><TimeTrigger>\n'
f' <StartBoundary>{target_dt.strftime("%Y-%m-%dT%H:%M:%S")}</StartBoundary>\n'
' <Enabled>true</Enabled>\n'
' </TimeTrigger></Triggers>\n'
' <Actions><Exec>\n'
f' <Command>{python_exe}</Command>\n'
f' <Arguments>"{script_path}"</Arguments>\n'
' </Exec></Actions>\n'
' <Settings>\n'
' <MultipleInstancesPolicy>IgnoreNew</MultipleInstancesPolicy>\n'
' <DisallowStartIfOnBatteries>false</DisallowStartIfOnBatteries>\n'
' <StopIfGoingOnBatteries>false</StopIfGoingOnBatteries>\n'
' <StartWhenAvailable>true</StartWhenAvailable>\n'
' <ExecutionTimeLimit>PT5M</ExecutionTimeLimit>\n'
' <Enabled>true</Enabled>\n'
' </Settings>\n'
' <Principals><Principal>\n'
' <LogonType>InteractiveToken</LogonType>\n'
' <RunLevel>LeastPrivilege</RunLevel>\n'
' </Principal></Principals>\n'
'</Task>'
)
xml_path.write_text(xml_content, encoding="utf-16")
result = subprocess.run(
["schtasks", "/Create", "/TN", task_name, "/XML", str(xml_path), "/F"],
capture_output=True, text=True,
)
try:
xml_path.unlink(missing_ok=True)
except Exception:
pass
if result.returncode != 0:
script_path.unlink(missing_ok=True)
err = (result.stderr or result.stdout).strip()
print(f"[Reminder] ❌ schtasks: {err}")
return ""
return task_name
def _schedule_mac(target_dt: datetime, task_name: str,
script_path: Path) -> str:
agents_dir = Path.home() / "Library" / "LaunchAgents"
agents_dir.mkdir(parents=True, exist_ok=True)
label = f"com.jarvis.reminder.{task_name}"
plist_path = agents_dir / f"{label}.plist"
plist_content = f"""<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN"
"http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>Label</key> <string>{label}</string>
<key>ProgramArguments</key>
<array>
<string>{sys.executable}</string>
<string>{script_path}</string>
</array>
<key>StartCalendarInterval</key>
<dict>
<key>Year</key> <integer>{target_dt.year}</integer>
<key>Month</key> <integer>{target_dt.month}</integer>
<key>Day</key> <integer>{target_dt.day}</integer>
<key>Hour</key> <integer>{target_dt.hour}</integer>
<key>Minute</key> <integer>{target_dt.minute}</integer>
</dict>
<key>RunAtLoad</key> <false/>
<key>StandardOutPath</key> <string>/dev/null</string>
<key>StandardErrorPath</key> <string>/dev/null</string>
</dict>
</plist>
"""
plist_path.write_text(plist_content, encoding="utf-8")
plist_path.chmod(0o644)
result = subprocess.run(
["launchctl", "load", str(plist_path)],
capture_output=True, text=True,
)
if result.returncode != 0:
plist_path.unlink(missing_ok=True)
script_path.unlink(missing_ok=True)
print(f"[Reminder] ❌ launchctl: {result.stderr.strip()}")
return ""
return label
def _schedule_linux(target_dt: datetime, task_name: str,
script_path: Path) -> str:
if shutil.which("systemd-run"):
on_calendar = target_dt.strftime("%Y-%m-%d %H:%M:00")
result = subprocess.run(
[
"systemd-run",
"--user",
f"--on-calendar={on_calendar}",
f"--unit={task_name}",
"--",
sys.executable, str(script_path),
],
capture_output=True, text=True,
)
if result.returncode == 0:
return task_name
print(f"[Reminder] ⚠️ systemd-run failed: {result.stderr.strip()}, trying 'at'")
if shutil.which("at"):
at_time = target_dt.strftime("%H:%M %Y-%m-%d")
cmd_str = f"{sys.executable} {script_path}\n"
result = subprocess.run(
["at", at_time],
input=cmd_str, capture_output=True, text=True,
)
if result.returncode == 0:
return task_name
print(f"[Reminder] ❌ at: {result.stderr.strip()}")
return ""
print("[Reminder] ❌ Neither systemd-run nor at found on this Linux system.")
return ""
def reminder(
parameters: dict,
response: str | None = None,
response=None,
player=None,
session_memory=None
session_memory=None,
) -> str:
"""
Sets a timed reminder using Windows Task Scheduler.
parameters:
- date (str) YYYY-MM-DD
- time (str) HH:MM
- message (str)
Returns a result string — Live API voices it automatically.
No edge_speak needed.
"""
date_str = parameters.get("date")
time_str = parameters.get("time")
message = parameters.get("message", "Reminder")
date_str = parameters.get("date", "").strip()
time_str = parameters.get("time", "").strip()
message = parameters.get("message", "Reminder").strip()
if not date_str or not time_str:
return "I need both a date and a time to set a reminder."
try:
target_dt = datetime.strptime(f"{date_str} {time_str}", "%Y-%m-%d %H:%M")
if target_dt <= datetime.now():
return "That time is already in the past."
task_name = f"MARKReminder_{target_dt.strftime('%Y%m%d_%H%M')}"
safe_message = message.replace('"', '').replace("'", "").strip()[:200]
python_exe = sys.executable
if python_exe.lower().endswith("python.exe"):
pythonw = python_exe.replace("python.exe", "pythonw.exe")
if os.path.exists(pythonw):
python_exe = pythonw
temp_dir = os.environ.get("TEMP", "C:\\Temp")
notify_script = os.path.join(temp_dir, f"{task_name}.pyw")
project_root = os.path.abspath(
os.path.join(os.path.dirname(__file__), "..")
)
script_code = f'''import sys, os, time
sys.path.insert(0, r"{project_root}")
try:
import winsound
for freq in [800, 1000, 1200]:
winsound.Beep(freq, 200)
time.sleep(0.1)
except Exception:
pass
try:
from win10toast import ToastNotifier
ToastNotifier().show_toast(
"MARK Reminder",
"{safe_message}",
duration=15,
threaded=False
)
except Exception:
try:
import subprocess
subprocess.run(["msg", "*", "/TIME:30", "{safe_message}"], shell=True)
except Exception:
pass
time.sleep(3)
try:
os.remove(__file__)
except Exception:
pass
'''
with open(notify_script, "w", encoding="utf-8") as f:
f.write(script_code)
xml_content = f'''<?xml version="1.0" encoding="UTF-16"?>
<Task version="1.2" xmlns="http://schemas.microsoft.com/windows/2004/02/mit/task">
<RegistrationInfo>
<Description>MARK Reminder: {safe_message}</Description>
</RegistrationInfo>
<Triggers>
<TimeTrigger>
<StartBoundary>{target_dt.strftime("%Y-%m-%dT%H:%M:%S")}</StartBoundary>
<Enabled>true</Enabled>
</TimeTrigger>
</Triggers>
<Actions>
<Exec>
<Command>{python_exe}</Command>
<Arguments>"{notify_script}"</Arguments>
</Exec>
</Actions>
<Settings>
<MultipleInstancesPolicy>IgnoreNew</MultipleInstancesPolicy>
<DisallowStartIfOnBatteries>false</DisallowStartIfOnBatteries>
<StopIfGoingOnBatteries>false</StopIfGoingOnBatteries>
<StartWhenAvailable>true</StartWhenAvailable>
<WakeToRun>true</WakeToRun>
<ExecutionTimeLimit>PT5M</ExecutionTimeLimit>
<Enabled>true</Enabled>
</Settings>
<Principals>
<Principal>
<LogonType>InteractiveToken</LogonType>
<RunLevel>LeastPrivilege</RunLevel>
</Principal>
</Principals>
</Task>'''
xml_path = os.path.join(temp_dir, f"{task_name}.xml")
with open(xml_path, "w", encoding="utf-16") as f:
f.write(xml_content)
result = subprocess.run(
f'schtasks /Create /TN "{task_name}" /XML "{xml_path}" /F',
shell=True, capture_output=True, text=True
)
try:
os.remove(xml_path)
except Exception:
pass
if result.returncode != 0:
err = result.stderr.strip() or result.stdout.strip()
print(f"[Reminder] ❌ schtasks failed: {err}")
try:
os.remove(notify_script)
except Exception:
pass
return "I couldn't schedule the reminder due to a system error."
if player:
player.write_log(f"[reminder] set for {date_str} {time_str}")
return f"Reminder set for {target_dt.strftime('%B %d at %I:%M %p')}."
except ValueError:
return "I couldn't understand that date or time format."
return "I couldn't parse that date or time. Please use YYYY-MM-DD and HH:MM."
if target_dt <= datetime.now():
return "That time has already passed — I can't set a reminder in the past."
os_name = _get_os()
safe_msg = _sanitise(message)
task_name = f"JARVISReminder_{target_dt.strftime('%Y%m%d_%H%M%S')}"
try:
script_path = _write_notify_script(task_name, safe_msg, os_name)
except Exception as e:
return f"Something went wrong while scheduling the reminder: {str(e)[:80]}"
return f"Could not prepare the reminder script: {e}"
try:
if os_name == "windows":
job_id = _schedule_windows(target_dt, task_name, script_path, safe_msg)
elif os_name == "mac":
job_id = _schedule_mac(target_dt, task_name, script_path)
else:
job_id = _schedule_linux(target_dt, task_name, script_path)
except Exception as e:
script_path.unlink(missing_ok=True)
print(f"[Reminder] ❌ Scheduling exception: {e}")
return "Something went wrong while scheduling the reminder."
if not job_id:
return "I couldn't register the reminder with the system scheduler."
if player:
player.write_log(f"[Reminder] ✅ {date_str} {time_str} — {safe_msg[:40]}")
friendly_time = target_dt.strftime("%B %d at %I:%M %p")
return f"Reminder set for {friendly_time}."
+298 -260
View File
@@ -1,151 +1,186 @@
"""
actions/screen_processor.py — Gemini Live API — IMAGE-ONLY SESSION v8
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
v8 Changes:
- Auto camera detection on first use — saves to config
- No hardcoded camera index
- mic_loop removed (no double response issue)
- Image-only session, no conflict with main.py
"""
from __future__ import annotations
import asyncio
import base64
import io
import json
import re
import os
import sys
import time
import threading
import cv2
import mss
import mss.tools
import pyaudio
import time
from pathlib import Path
from typing import Optional
import numpy as np
import sounddevice as sd
try:
import cv2
_CV2 = True
except ImportError:
_CV2 = False
try:
import mss
import mss.tools
_MSS = True
except ImportError:
_MSS = False
try:
import PIL.Image
_PIL_OK = True
_PIL = True
except ImportError:
_PIL_OK = False
_PIL = False
from google import genai
from google.genai import types
from google.genai import types as gtypes
def get_base_dir():
def _base_dir() -> Path:
if getattr(sys, "frozen", False):
return Path(sys.executable).parent
return Path(__file__).resolve().parent.parent
BASE_DIR = get_base_dir()
API_CONFIG_PATH = BASE_DIR / "config" / "api_keys.json"
LIVE_MODEL = "models/gemini-2.5-flash-native-audio-preview-12-2025"
FORMAT = pyaudio.paInt16
CHANNELS = 1
RECEIVE_SAMPLE_RATE = 24000
CHUNK_SIZE = 1024
_BASE = _base_dir()
_CONFIG_PATH = _BASE / "config" / "api_keys.json"
IMG_MAX_W = 640
IMG_MAX_H = 360
JPEG_Q = 55
SYSTEM_PROMPT = (
"You are JARVIS from Iron Man movies. "
"Analyze images with technical precision and intelligence. "
"Help the user in a way they can understand — don't be overly complex. "
"Be concise, smart, and helpful like Tony Stark's AI assistant. "
"Respond in maximum 2 short sentences. Speed is priority. "
"Address the user as 'sir' for a tone of respect. "
"Ask if the user needs any further help with their problem."
)
def _load_config() -> dict:
try:
return json.loads(_CONFIG_PATH.read_text(encoding="utf-8"))
except Exception:
return {}
def _save_config_key(key: str, value) -> None:
try:
cfg = _load_config()
cfg[key] = value
_CONFIG_PATH.write_text(json.dumps(cfg, indent=4), encoding="utf-8")
except Exception as e:
print(f"[Vision] ⚠️ Could not save config key '{key}': {e}")
def _get_api_key() -> str:
key = _load_config().get("gemini_api_key", "")
if not key:
raise RuntimeError("gemini_api_key not found in config.")
return key
def _get_os() -> str:
return _load_config().get("os_system", "windows").lower()
_LIVE_MODEL = "models/gemini-2.5-flash-native-audio-preview-12-2025"
_CHANNELS = 1
_RECEIVE_SAMPLE_RATE = 24_000
_CHUNK_SIZE = 1_024
_IMG_MAX_W = 640
_IMG_MAX_H = 360
_JPEG_Q = 60
_SYSTEM_PROMPT = (
"You are JARVIS, an advanced AI assistant. "
"Analyze the provided image with precision and intelligence. "
"Be concise and direct — maximum two sentences unless the user's question "
"requires more detail. "
"Address the user respectfully. "
"Always call the appropriate tool; never simulate results."
)
def _compress(img_bytes: bytes, source_format: str = "PNG") -> tuple[bytes, str]:
if not _PIL:
return img_bytes, f"image/{source_format.lower()}"
try:
with open(API_CONFIG_PATH, "r", encoding="utf-8") as f:
keys = json.load(f)
key = keys.get("gemini_api_key", "")
if not key:
raise ValueError("gemini_api_key not found")
return key
img = PIL.Image.open(io.BytesIO(img_bytes)).convert("RGB")
img.thumbnail((_IMG_MAX_W, _IMG_MAX_H), PIL.Image.BILINEAR)
buf = io.BytesIO()
img.save(buf, format="JPEG", quality=_JPEG_Q, optimize=False)
return buf.getvalue(), "image/jpeg"
except Exception as e:
raise RuntimeError(f"Could not load API key: {e}")
print(f"[Vision] ⚠️ Image compress failed: {e}")
return img_bytes, f"image/{source_format.lower()}"
def _capture_screen() -> tuple[bytes, str]:
if not _MSS:
raise RuntimeError("mss is not installed. Run: pip install mss")
with mss.mss() as sct:
monitors = sct.monitors # [0] = all combined, [1..n] = real screens
target = monitors[1] if len(monitors) > 1 else monitors[0]
shot = sct.grab(target)
png = mss.tools.to_png(shot.rgb, shot.size)
return _compress(png, "PNG")
def _cv2_backend() -> int:
"""Return the best OpenCV camera backend for the current OS."""
if not _CV2:
return 0
os_name = _get_os()
if os_name == "windows":
return cv2.CAP_DSHOW
if os_name == "mac":
return cv2.CAP_AVFOUNDATION
return cv2.CAP_ANY
def _probe_camera(index: int, backend: int, warmup: int = 5) -> bool:
if not _CV2:
return False
cap = cv2.VideoCapture(index, backend)
if not cap.isOpened():
cap.release()
return False
for _ in range(warmup):
cap.read()
ret, frame = cap.read()
cap.release()
if not ret or frame is None:
return False
return bool(np.mean(frame) > 8)
def _detect_camera_index() -> int:
backend = _cv2_backend()
print("[Vision] 🔍 Auto-detecting camera...")
for idx in range(6):
if _probe_camera(idx, backend):
print(f"[Vision] ✅ Camera found at index {idx}")
_save_config_key("camera_index", idx)
return idx
print(f"[Vision] ⚠️ Camera index {idx}: no usable frame")
print("[Vision] ⚠️ No camera found — defaulting to index 0")
_save_config_key("camera_index", 0)
return 0
def _get_camera_index() -> int:
"""
Reads camera index from config.
If not set, auto-detects the best camera and saves it for future use.
Runs only once — after that, config value is used directly.
"""
try:
with open(API_CONFIG_PATH, "r", encoding="utf-8") as f:
cfg = json.load(f)
if "camera_index" in cfg:
return int(cfg["camera_index"])
except Exception:
pass
print("[Camera] 🔍 No camera index in config. Auto-detecting...")
best_index = 0
for idx in range(6):
cap = cv2.VideoCapture(idx, cv2.CAP_DSHOW)
if not cap.isOpened():
cap.release()
continue
for _ in range(5):
cap.read()
ret, frame = cap.read()
cap.release()
if ret and frame is not None and frame.mean() > 5:
best_index = idx
print(f"[Camera] ✅ Camera found at index {idx} — saving to config.")
break
else:
print(f"[Camera] ⚠️ Index {idx}: no valid frame (black or empty).")
try:
cfg = {}
if API_CONFIG_PATH.exists():
with open(API_CONFIG_PATH, "r", encoding="utf-8") as f:
cfg = json.load(f)
cfg["camera_index"] = best_index
with open(API_CONFIG_PATH, "w", encoding="utf-8") as f:
json.dump(cfg, f, indent=4)
print(f"[Camera] 💾 Camera index {best_index} saved to config.")
except Exception as e:
print(f"[Camera] ⚠️ Could not save camera index: {e}")
return best_index
cfg = _load_config()
if "camera_index" in cfg:
return int(cfg["camera_index"])
return _detect_camera_index()
def _to_jpeg(img_bytes: bytes) -> bytes:
if not _PIL_OK:
return img_bytes
img = PIL.Image.open(io.BytesIO(img_bytes)).convert("RGB")
img.thumbnail([IMG_MAX_W, IMG_MAX_H], PIL.Image.BILINEAR)
buf = io.BytesIO()
img.save(buf, format="JPEG", quality=JPEG_Q, optimize=False)
return buf.getvalue()
def _capture_camera() -> tuple[bytes, str]:
if not _CV2:
raise RuntimeError("OpenCV (cv2) is not installed. Run: pip install opencv-python")
index = _get_camera_index()
backend = _cv2_backend()
cap = cv2.VideoCapture(index, backend)
def _capture_screenshot() -> bytes:
with mss.mss() as sct:
shot = sct.grab(sct.monitors[1])
png_bytes = mss.tools.to_png(shot.rgb, shot.size)
return _to_jpeg(png_bytes)
def _capture_camera() -> bytes:
camera_index = _get_camera_index()
cap = cv2.VideoCapture(camera_index, cv2.CAP_DSHOW)
if not cap.isOpened():
raise RuntimeError(f"Camera could not be opened: index {camera_index}")
raise RuntimeError(f"Camera index {index} could not be opened.")
for _ in range(10):
cap.read()
@@ -154,127 +189,140 @@ def _capture_camera() -> bytes:
cap.release()
if not ret or frame is None:
raise RuntimeError("Could not capture camera frame.")
raise RuntimeError("Camera returned no frame.")
if _PIL_OK:
if _PIL:
rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
img = PIL.Image.fromarray(rgb)
img.thumbnail([IMG_MAX_W, IMG_MAX_H], PIL.Image.BILINEAR)
img.thumbnail((_IMG_MAX_W, _IMG_MAX_H), PIL.Image.BILINEAR)
buf = io.BytesIO()
img.save(buf, format="JPEG", quality=JPEG_Q, optimize=False)
return buf.getvalue()
img.save(buf, format="JPEG", quality=_JPEG_Q)
return buf.getvalue(), "image/jpeg"
_, buf = cv2.imencode(".jpg", frame, [cv2.IMWRITE_JPEG_QUALITY, JPEG_Q])
return buf.tobytes()
class _LiveSession:
"""
Image-only analysis session.
No microphone — no conflict with main.py session.
Sends image + question, plays audio response.
"""
_, buf = cv2.imencode(".jpg", frame, [cv2.IMWRITE_JPEG_QUALITY, _JPEG_Q])
return buf.tobytes(), "image/jpeg"
class _VisionSession:
def __init__(self):
self._loop: asyncio.AbstractEventLoop | None = None
self._thread: threading.Thread | None = None
self._session = None
self._out_queue: asyncio.Queue | None = None
self._audio_in: asyncio.Queue | None = None
self._ready: threading.Event = threading.Event()
self._player = None
self._pya = pyaudio.PyAudio()
self._send_lock: asyncio.Lock | None = None
self._loop: Optional[asyncio.AbstractEventLoop] = None
self._thread: Optional[threading.Thread] = None
self._session = None
self._out_queue: Optional[asyncio.Queue] = None
self._audio_in: Optional[asyncio.Queue] = None
self._ready_evt: threading.Event = threading.Event()
self._player = None
self._lock: threading.Lock = threading.Lock()
def start(self, player=None):
if self._thread and self._thread.is_alive():
def start(self, player=None, timeout: float = 25.0) -> None:
with self._lock:
if self._thread and self._thread.is_alive():
if player is not None:
self._player = player
return
self._player = player
self._thread = threading.Thread(
target=self._run_event_loop,
daemon=True,
name="VisionSessionThread",
)
self._thread.start()
if not self._ready_evt.wait(timeout=timeout):
raise RuntimeError(f"Vision session did not connect within {timeout}s.")
print("[Vision] ✅ Session ready")
def analyze(self, image_bytes: bytes, mime_type: str, user_text: str) -> None:
if not self._loop or not self._out_queue:
print("[Vision] ⚠️ Session not started — dropping request")
return
self._player = player
self._thread = threading.Thread(
target=self._run_loop, daemon=True, name="VisionSessionThread"
asyncio.run_coroutine_threadsafe(
self._out_queue.put((image_bytes, mime_type, user_text)),
self._loop,
)
self._thread.start()
ok = self._ready.wait(timeout=20)
if not ok:
raise RuntimeError("Vision session did not start within 20s.")
print("[ScreenProcess] ✅ Vision session ready (no mic)")
def _run_loop(self):
def is_ready(self) -> bool:
return self._session is not None
def _run_event_loop(self) -> None:
self._loop = asyncio.new_event_loop()
asyncio.set_event_loop(self._loop)
self._loop.run_until_complete(self._main())
self._loop.run_until_complete(self._session_loop())
async def _main(self):
async def _session_loop(self) -> None:
self._out_queue = asyncio.Queue(maxsize=30)
self._audio_in = asyncio.Queue()
self._send_lock = asyncio.Lock()
client = genai.Client(
api_key=_get_api_key(),
http_options={"api_version": "v1beta"}
http_options={"api_version": "v1beta"},
)
config = types.LiveConnectConfig(
config = gtypes.LiveConnectConfig(
response_modalities=["AUDIO"],
output_audio_transcription={},
system_instruction=SYSTEM_PROMPT,
speech_config=types.SpeechConfig(
voice_config=types.VoiceConfig(
prebuilt_voice_config=types.PrebuiltVoiceConfig(
system_instruction=_SYSTEM_PROMPT,
speech_config=gtypes.SpeechConfig(
voice_config=gtypes.VoiceConfig(
prebuilt_voice_config=gtypes.PrebuiltVoiceConfig(
voice_name="Charon"
)
)
),
)
backoff = 2.0
while True:
try:
print("[ScreenProcess] 🔌 Vision session connecting...")
async with client.aio.live.connect(model=LIVE_MODEL, config=config) as session:
print("[Vision] 🔌 Connecting...")
async with client.aio.live.connect(
model=_LIVE_MODEL, config=config
) as session:
self._session = session
self._ready.set()
print("[ScreenProcess] ✅ Vision session connected")
self._ready_evt.set()
backoff = 2.0
print("[Vision] ✅ Connected")
async with asyncio.TaskGroup() as tg:
tg.create_task(self._send_loop())
tg.create_task(self._recv_loop())
tg.create_task(self._play_loop())
except Exception as e:
print(f"[ScreenProcess] ⚠️ Disconnected: {e} — reconnecting...")
except* Exception as eg:
for exc in eg.exceptions:
print(f"[Vision] ⚠️ Session error: {exc}")
finally:
self._session = None
self._ready.clear()
await asyncio.sleep(2)
self._ready.set()
self._ready_evt.clear()
# ── Send loop ──
print(f"[Vision] 🔄 Reconnecting in {backoff:.0f}s...")
await asyncio.sleep(backoff)
backoff = min(backoff * 1.5, 30.0)
self._ready_evt.set()
async def _send_loop(self):
"""Sends (image_bytes, mime_type, user_text) tuples from queue."""
async def _send_loop(self) -> None:
while True:
item = await self._out_queue.get()
if self._session:
image_bytes, mime_type, user_text = item
try:
b64 = base64.b64encode(image_bytes).decode("utf-8")
await self._session.send_client_content(
turns={
"parts": [
{"inline_data": {"mime_type": mime_type, "data": b64}},
{"text": user_text}
]
},
turn_complete=True
)
print("[ScreenProcess] ✅ Image sent")
except Exception as e:
print(f"[ScreenProcess] ⚠️ Send error: {e}")
image_bytes, mime_type, user_text = await self._out_queue.get()
if not self._session:
print("[Vision] ⚠️ No session — dropping image")
continue
try:
b64 = base64.b64encode(image_bytes).decode("ascii")
await self._session.send_client_content(
turns={
"parts": [
{"inline_data": {"mime_type": mime_type, "data": b64}},
{"text": user_text},
]
},
turn_complete=True,
)
print(f"[Vision] 📤 Sent {len(image_bytes):,} bytes — '{user_text[:60]}'")
except Exception as e:
print(f"[Vision] ⚠️ Send error: {e}")
async def _recv_loop(self):
transcript_buf: list[str] = []
async def _recv_loop(self) -> None:
transcript: list[str] = []
try:
async for response in self._session.receive():
if response.data:
await self._audio_in.put(response.data)
@@ -285,120 +333,110 @@ class _LiveSession:
if sc.output_transcription and sc.output_transcription.text:
chunk = sc.output_transcription.text.strip()
if chunk:
transcript_buf.append(chunk)
transcript.append(chunk)
if sc.turn_complete:
if transcript_buf and self._player:
full = re.sub(r'\s+', ' ', " ".join(transcript_buf)).strip()
if transcript and self._player:
full = re.sub(r"\s+", " ", " ".join(transcript)).strip()
if full:
self._player.write_log(f"Jarvis: {full}")
print(f"[ScreenProcess] 💬 {full}")
transcript_buf = []
print(f"[Vision] 💬 {full}")
transcript = []
except Exception as e:
print(f"[ScreenProcess] ⚠️ Recv error: {e}")
transcript_buf = []
await asyncio.sleep(0.3)
print(f"[Vision] ⚠️ Recv error: {e}")
raise
async def _play_loop(self):
stream = await asyncio.to_thread(
self._pya.open,
format=FORMAT, channels=CHANNELS,
rate=RECEIVE_SAMPLE_RATE, output=True,
async def _play_loop(self) -> None:
stream = sd.RawOutputStream(
samplerate=_RECEIVE_SAMPLE_RATE,
channels=_CHANNELS,
dtype="int16",
blocksize=_CHUNK_SIZE,
)
stream.start()
try:
while True:
chunk = await self._audio_in.get()
await asyncio.to_thread(stream.write, chunk)
except Exception as e:
print(f"[Vision] ❌ Play error: {e}")
raise
finally:
stream.stop()
stream.close()
def analyze(self, image_bytes: bytes, mime_type: str, user_text: str):
"""Called from main thread — puts image into async queue."""
if not self._loop:
return
asyncio.run_coroutine_threadsafe(
self._out_queue.put((image_bytes, mime_type, user_text)),
self._loop
)
def is_ready(self) -> bool:
return self._session is not None
_live = _LiveSession()
_started = False
_start_lock = threading.Lock()
_session = _VisionSession()
_session_lock = threading.Lock()
_session_up = False
def _ensure_started(player=None):
global _started
with _start_lock:
if not _started:
_live.start(player=player)
_started = True
def _ensure_session(player=None) -> None:
global _session_up
with _session_lock:
if not _session_up:
_session.start(player=player)
_session_up = True
elif player is not None:
_live._player = player
_session._player = player
def screen_process(
parameters: dict,
response: str | None = None,
response=None,
player=None,
session_memory=None,
) -> bool:
user_text = (parameters or {}).get("text") or (parameters or {}).get("user_text", "")
user_text = (user_text or "").strip()
params = parameters or {}
user_text = (params.get("text") or params.get("user_text") or "").strip()
angle = params.get("angle", "screen").lower().strip()
if not user_text:
print("[ScreenProcess] ⚠️ No user_text provided.")
print("[Vision] ⚠️ No question provided — aborting")
return False
angle = (parameters or {}).get("angle", "screen").lower().strip()
print(f"[ScreenProcess] angle={angle!r} text={user_text!r}")
print(f"[Vision] ▶ angle={angle!r} question='{user_text[:80]}'")
_ensure_started(player=player)
try:
_ensure_session(player=player)
except Exception as e:
print(f"[Vision] ❌ Could not start session: {e}")
return False
try:
if angle == "camera":
image_bytes = _capture_camera()
mime_type = "image/jpeg"
print("[ScreenProcess] 📷 Camera captured")
image_bytes, mime_type = _capture_camera()
print(f"[Vision] 📷 Camera: {len(image_bytes):,} bytes")
else:
image_bytes = _capture_screenshot()
mime_type = "image/jpeg" if _PIL_OK else "image/png"
print("[ScreenProcess] 🖥️ Screen captured")
image_bytes, mime_type = _capture_screen()
print(f"[Vision] 🖥️ Screen: {len(image_bytes):,} bytes")
except Exception as e:
import traceback; traceback.print_exc()
print(f"[ScreenProcess] ❌ Capture error: {e}")
print(f"[Vision] ❌ Capture error: {e}")
return False
print(f"[ScreenProcess] 📦 {len(image_bytes)} bytes → sending")
_live.analyze(image_bytes, mime_type, user_text)
_session.analyze(image_bytes, mime_type, user_text)
return True
def warmup_session(player=None):
"""
Optional: pre-warm the session.
Do NOT call from main.py — causes double session issue.
Only use when testing screen_processor.py standalone.
"""
def warmup_session(player=None) -> None:
try:
_ensure_started(player=player)
_ensure_session(player=player)
except Exception as e:
print(f"[ScreenProcess] ⚠️ Warmup error: {e}")
print(f"[Vision] ⚠️ Warmup failed: {e}")
if __name__ == "__main__":
print("[TEST] screen_processor.py v8 — image-only session")
print("=" * 50)
mode = input("screen / camera (default: screen): ").strip().lower() or "screen"
request = input("Question (Enter for default): ").strip() or "What do you see? Be brief."
print("[TEST] screen_processor.py")
print("=" * 52)
mode = input("angle — screen / camera (default: screen): ").strip().lower() or "screen"
q = input("Question (Enter = default): ").strip() or "What do you see? Be brief."
t0 = time.perf_counter()
warmup_session()
print(f"Session ready — {time.perf_counter()-t0:.2f}s\n")
print(f"Session ready in {time.perf_counter()-t0:.2f}s\n")
t1 = time.perf_counter()
result = screen_process({"angle": mode, "text": request}, player=None)
print(f"Sent — {time.perf_counter()-t1:.3f}s | audio incoming...")
time.sleep(8)
print(f"\n{'✅' if result else '❌'}")
t1 = time.perf_counter()
ok = screen_process({"angle": mode, "text": q})
print(f"Queued in {time.perf_counter()-t1:.3f}s — waiting for audio...")
time.sleep(10)
print("Done." if ok else "Failed.")
+210 -155
View File
@@ -1,214 +1,269 @@
# actions/send_message.py
# Universal messaging — WhatsApp & Instagram
# Uses visual element detection (pyautogui + screen search) instead of
# hardcoded tab/click sequences — works on any screen resolution.
import json
import subprocess
import sys
import time
import pyautogui
from pathlib import Path
pyautogui.FAILSAFE = True
pyautogui.PAUSE = 0.08
try:
import pyautogui
pyautogui.FAILSAFE = True
pyautogui.PAUSE = 0.06
_PYAUTOGUI = True
except ImportError:
_PYAUTOGUI = False
try:
import pyperclip
_PYPERCLIP = True
except ImportError:
_PYPERCLIP = False
def _base_dir() -> Path:
if getattr(sys, "frozen", False):
return Path(sys.executable).parent
return Path(__file__).resolve().parent.parent
def _get_os() -> str:
try:
cfg = json.loads(
(_base_dir() / "config" / "api_keys.json").read_text(encoding="utf-8")
)
return cfg.get("os_system", "windows").lower()
except Exception:
return "windows"
def _require_pyautogui():
if not _PYAUTOGUI:
raise RuntimeError("PyAutoGUI not installed. Run: pip install pyautogui")
def _paste_text(text: str) -> None:
_require_pyautogui()
os_name = _get_os()
paste_hotkey = ("command", "v") if os_name == "mac" else ("ctrl", "v")
if _PYPERCLIP:
pyperclip.copy(text)
time.sleep(0.15)
pyautogui.hotkey(*paste_hotkey)
time.sleep(0.1)
else:
pyautogui.write(text, interval=0.03)
def _clear_and_paste(text: str) -> None:
_require_pyautogui()
os_name = _get_os()
select_all = ("command", "a") if os_name == "mac" else ("ctrl", "a")
pyautogui.hotkey(*select_all)
time.sleep(0.1)
pyautogui.press("delete")
time.sleep(0.1)
_paste_text(text)
def _open_app(app_name: str) -> bool:
"""Opens an app via Windows search."""
_require_pyautogui()
os_name = _get_os()
try:
pyautogui.press("win")
time.sleep(0.4)
pyautogui.write(app_name, interval=0.04)
time.sleep(0.5)
pyautogui.press("enter")
time.sleep(2.0)
return True
if os_name == "windows":
pyautogui.press("win")
time.sleep(0.5)
_paste_text(app_name)
time.sleep(0.6)
pyautogui.press("enter")
time.sleep(2.5)
return True
elif os_name == "mac":
result = subprocess.run(
["open", "-a", app_name],
capture_output=True, text=True, timeout=10,
)
if result.returncode != 0:
result = subprocess.run(
["open", "-a", f"{app_name}.app"],
capture_output=True, text=True, timeout=10,
)
time.sleep(2.5)
return result.returncode == 0
else:
launched = False
for launcher in [
["gtk-launch", app_name.lower()],
[app_name.lower()],
]:
try:
subprocess.Popen(
launcher,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
)
launched = True
break
except FileNotFoundError:
continue
time.sleep(2.5)
return launched
except Exception as e:
print(f"[SendMessage] Could not open {app_name}: {e}")
print(f"[SendMessage] ⚠️ Could not open {app_name}: {e}")
return False
def _search_contact(contact: str, platform: str):
"""
Searches for a contact inside the messaging app.
Uses Ctrl+F (universal search shortcut) then types contact name.
"""
def _open_browser_url(url: str) -> bool:
import webbrowser
try:
webbrowser.open(url)
time.sleep(4.0)
return True
except Exception as e:
print(f"[SendMessage] ⚠️ Could not open browser: {e}")
return False
def _search_in_app(query: str) -> None:
_require_pyautogui()
os_name = _get_os()
search_hotkey = ("command", "f") if os_name == "mac" else ("ctrl", "f")
pyautogui.hotkey(*search_hotkey)
time.sleep(0.5)
pyautogui.hotkey("ctrl", "f")
time.sleep(0.4)
pyautogui.hotkey("ctrl", "a")
pyautogui.write(contact, interval=0.04)
time.sleep(0.8)
_clear_and_paste(query)
time.sleep(1.0)
def _desktop_send(app_name: str, receiver: str, message: str) -> str:
if not _open_app(app_name):
return f"Could not open {app_name}."
time.sleep(1.0)
_search_in_app(receiver)
pyautogui.press("enter")
time.sleep(0.6)
time.sleep(0.8)
def _type_and_send(message: str):
"""Types message and sends it."""
pyautogui.press("tab")
time.sleep(0.2)
pyautogui.hotkey("ctrl", "a")
pyautogui.write(message, interval=0.03)
_paste_text(message)
time.sleep(0.2)
pyautogui.press("enter")
time.sleep(0.3)
return f"Message sent to {receiver} via {app_name}."
def _send_whatsapp(receiver: str, message: str) -> str:
"""
Sends a WhatsApp message via the Windows desktop app.
Steps: Open WhatsApp → Search contact → Click → Type → Send
"""
try:
if not _open_app("WhatsApp"):
return "Could not open WhatsApp."
return _desktop_send("WhatsApp", receiver, message)
time.sleep(1.5)
pyautogui.hotkey("ctrl", "f")
time.sleep(0.4)
pyautogui.hotkey("ctrl", "a")
pyautogui.write(receiver, interval=0.04)
time.sleep(1.0)
def _send_telegram(receiver: str, message: str) -> str:
return _desktop_send("Telegram", receiver, message)
pyautogui.press("enter")
time.sleep(0.8)
pyautogui.write(message, interval=0.03)
time.sleep(0.2)
pyautogui.press("enter")
def _send_signal(receiver: str, message: str) -> str:
return _desktop_send("Signal", receiver, message)
return f"Message sent to {receiver} via WhatsApp."
except Exception as e:
return f"WhatsApp error: {e}"
def _send_discord(receiver: str, message: str) -> str:
return _desktop_send("Discord", receiver, message)
def _send_instagram(receiver: str, message: str) -> str:
"""
Sends an Instagram DM via browser (instagram.com).
Steps: Open Chrome → Go to instagram.com/direct → Search contact → Send
"""
try:
import webbrowser
_require_pyautogui()
webbrowser.open("https://www.instagram.com/direct/new/")
time.sleep(3.5)
if not _open_browser_url("https://www.instagram.com/direct/new/"):
return "Could not open Instagram in browser."
pyautogui.write(receiver, interval=0.05)
time.sleep(1.5)
_paste_text(receiver)
time.sleep(1.5)
pyautogui.press("down")
time.sleep(0.3)
pyautogui.press("enter")
time.sleep(0.5)
pyautogui.press("down")
time.sleep(0.3)
pyautogui.press("enter")
time.sleep(0.4)
for _ in range(3):
pyautogui.press("tab")
time.sleep(0.1)
pyautogui.press("enter")
time.sleep(1.5)
for _ in range(4):
pyautogui.press("tab")
time.sleep(0.15)
pyautogui.press("enter")
time.sleep(2.0)
pyautogui.write(message, interval=0.04)
time.sleep(0.2)
pyautogui.press("enter")
_paste_text(message)
time.sleep(0.2)
pyautogui.press("enter")
time.sleep(0.3)
return f"Message sent to {receiver} via Instagram."
except Exception as e:
return f"Instagram error: {e}"
def _send_telegram(receiver: str, message: str) -> str:
"""Sends a Telegram message via Windows desktop app."""
try:
if not _open_app("Telegram"):
return "Could not open Telegram."
time.sleep(1.5)
pyautogui.hotkey("ctrl", "f")
time.sleep(0.4)
pyautogui.write(receiver, interval=0.04)
time.sleep(1.0)
pyautogui.press("enter")
time.sleep(0.8)
pyautogui.write(message, interval=0.03)
time.sleep(0.2)
pyautogui.press("enter")
return f"Message sent to {receiver} via Telegram."
except Exception as e:
return f"Telegram error: {e}"
return f"Message sent to {receiver} via Instagram."
def _send_messenger(receiver: str, message: str) -> str:
_require_pyautogui()
def _send_generic(platform: str, receiver: str, message: str) -> str:
"""
For any other platform not explicitly supported.
Opens the app, searches for contact, types and sends.
Works for: Messenger, Discord, Signal, etc.
"""
try:
if not _open_app(platform):
return f"Could not open {platform}."
if not _open_browser_url("https://www.messenger.com/"):
return "Could not open Messenger in browser."
time.sleep(1.5)
pyautogui.hotkey("ctrl", "f")
time.sleep(0.4)
pyautogui.write(receiver, interval=0.04)
time.sleep(1.0)
pyautogui.press("enter")
time.sleep(0.8)
pyautogui.write(message, interval=0.03)
time.sleep(0.2)
pyautogui.press("enter")
return f"Message sent to {receiver} via {platform}."
_search_in_app(receiver)
time.sleep(0.5)
pyautogui.press("down")
time.sleep(0.3)
pyautogui.press("enter")
time.sleep(1.0)
_paste_text(message)
time.sleep(0.2)
pyautogui.press("enter")
time.sleep(0.3)
return f"Message sent to {receiver} via Messenger."
_PLATFORM_MAP = [
({"whatsapp", "wp", "wapp"}, _send_whatsapp),
({"telegram", "tg"}, _send_telegram),
({"instagram", "ig", "insta"}, _send_instagram),
({"signal"}, _send_signal),
({"discord"}, _send_discord),
({"messenger", "facebook", "fb"}, _send_messenger),
]
def _resolve_platform(platform_str: str):
key = platform_str.lower().strip()
for keywords, handler in _PLATFORM_MAP:
if any(k in key for k in keywords):
return handler
return lambda r, m: _desktop_send(platform_str.strip().title(), r, m)
except Exception as e:
return f"{platform} error: {e}"
def send_message(
parameters: dict,
response=None,
player=None,
session_memory=None
session_memory=None,
) -> str:
"""
Called from main.py.
parameters:
receiver : Contact name to send to
message_text : The message content
platform : whatsapp | instagram | telegram | <any app name>
Default: whatsapp
"""
params = parameters or {}
receiver = params.get("receiver", "").strip()
message_text = params.get("message_text", "").strip()
platform = params.get("platform", "whatsapp").strip().lower()
platform = params.get("platform", "whatsapp").strip()
if not receiver:
return "Please specify who to send the message to, sir."
return "Please specify a recipient."
if not message_text:
return "Please specify what message to send, sir."
return "Please specify the message content."
if not _PYAUTOGUI:
return "PyAutoGUI is not installed — cannot control the desktop."
print(f"[SendMessage] 📨 {platform} → {receiver}: {message_text[:40]}")
preview = message_text[:50] + ("…" if len(message_text) > 50 else "")
print(f"[SendMessage] 📨 {platform} → {receiver}: {preview}")
if player:
player.write_log(f"[msg] Sending to {receiver} via {platform}...")
player.write_log(f"[msg] {platform} → {receiver}")
if "whatsapp" in platform or "wp" in platform or "wapp" in platform:
result = _send_whatsapp(receiver, message_text)
try:
handler = _resolve_platform(platform)
result = handler(receiver, message_text)
except Exception as e:
result = f"Could not send message: {e}"
elif "instagram" in platform or "ig" in platform or "insta" in platform:
result = _send_instagram(receiver, message_text)
elif "telegram" in platform or "tg" in platform:
result = _send_telegram(receiver, message_text)
else:
result = _send_generic(platform, receiver, message_text)
print(f"[SendMessage] ✅ {result}")
print(f"[SendMessage] {'✅' if 'sent' in result.lower() else '❌'} {result}")
if player:
player.write_log(f"[msg] {result}")
+21 -32
View File
@@ -1,5 +1,3 @@
# actions/weather_report.py
import webbrowser
from urllib.parse import quote_plus
@@ -7,54 +5,45 @@ from urllib.parse import quote_plus
def weather_action(
parameters: dict,
player=None,
session_memory=None
):
"""
Weather report action.
Opens a Google weather search and gives a short spoken confirmation.
"""
session_memory=None,
) -> str:
city = parameters.get("city")
when = parameters.get("time", "today")
city = parameters.get("city")
time = parameters.get("time")
if not city or not isinstance(city, str):
if not city or not isinstance(city, str) or not city.strip():
msg = "Sir, the city is missing for the weather report."
_speak_and_log(msg, player)
_log(msg, player)
return msg
city = city.strip()
when = (when or "today").strip()
if not time or not isinstance(time, str):
time = "today"
else:
time = time.strip()
search_query = f"weather in {city} {time}"
encoded_query = quote_plus(search_query)
url = f"https://www.google.com/search?q={encoded_query}"
search_query = f"weather in {city} {when}"
url = f"https://www.google.com/search?q={quote_plus(search_query)}"
try:
webbrowser.open(url)
except Exception:
msg = f"Sir, I couldn't open the browser for the weather report."
_speak_and_log(msg, player)
opened = webbrowser.open(url)
if not opened:
raise RuntimeError("webbrowser.open returned False")
except Exception as e:
msg = f"Sir, I couldn't open the browser for the weather report: {e}"
_log(msg, player)
return msg
msg = f"Showing the weather for {city}, {time}, sir."
_speak_and_log(msg, player)
msg = f"Showing the weather for {city}, {when}, sir."
_log(msg, player)
if session_memory:
try:
session_memory.set_last_search(
query=search_query,
response=msg
)
session_memory.set_last_search(query=search_query, response=msg)
except Exception:
pass
pass
return msg
def _speak_and_log(message: str, player=None):
def _log(message: str, player=None) -> None:
print(f"[Weather] {message}")
if player:
try:
player.write_log(f"JARVIS: {message}")
+47 -42
View File
@@ -1,21 +1,17 @@
# actions/web_search.py
# MARK XXV — Web Search
# Primary: Gemini google_search (yeni google.genai SDK)
# Fallback: DuckDuckGo (ddgs)
import json
import sys
from pathlib import Path
def get_base_dir() -> Path:
def _get_base_dir() -> Path:
if getattr(sys, "frozen", False):
return Path(sys.executable).parent
return Path(__file__).resolve().parent.parent
BASE_DIR = get_base_dir()
BASE_DIR = _get_base_dir()
API_CONFIG_PATH = BASE_DIR / "config" / "api_keys.json"
def _get_api_key() -> str:
with open(API_CONFIG_PATH, "r", encoding="utf-8") as f:
return json.load(f)["gemini_api_key"]
@@ -24,40 +20,45 @@ def _get_api_key() -> str:
def _gemini_search(query: str) -> str:
from google import genai
client = genai.Client(api_key=_get_api_key())
client = genai.Client(api_key=_get_api_key())
response = client.models.generate_content(
model="gemini-2.5-flash-lite",
contents=query,
config={"tools": [{"google_search": {}}]}
config={"tools": [{"google_search": {}}]},
)
text = ""
for part in response.candidates[0].content.parts:
if hasattr(part, "text") and part.text:
text += part.text
if not text.strip():
raise ValueError("Empty response")
return text.strip()
text = text.strip()
if not text:
raise ValueError("Gemini returned an empty response.")
return text
def _ddg_search(query: str, max_results: int = 6) -> list:
def _ddg_search(query: str, max_results: int = 6) -> list[dict]:
try:
from ddgs import DDGS
except ImportError:
from duckduckgo_search import DDGS
results = []
with DDGS() as ddgs:
for r in ddgs.text(query, max_results=max_results):
results.append({
"title": r.get("title", ""),
"snippet": r.get("body", ""),
"url": r.get("href", ""),
"title": r.get("title", ""),
"snippet": r.get("body", ""),
"url": r.get("href", ""),
})
return results
def _format_ddg(query: str, results: list) -> str:
def _format_ddg(query: str, results: list[dict]) -> str:
if not results:
return f"No results found for: {query}"
lines = [f"Search results for: {query}\n"]
for i, r in enumerate(results, 1):
if r.get("title"): lines.append(f"{i}. {r['title']}")
@@ -66,27 +67,31 @@ def _format_ddg(query: str, results: list) -> str:
lines.append("")
return "\n".join(lines).strip()
def _compare(items: list, aspect: str) -> str:
query = f"Compare {', '.join(items)} in terms of {aspect}. Give specific facts and data."
def _compare(items: list[str], aspect: str) -> str:
query = (
f"Compare {', '.join(items)} in terms of {aspect}. "
"Give specific facts and data."
)
try:
return _gemini_search(query)
except Exception as e:
print(f"[WebSearch] ⚠️ Gemini compare failed: {e}")
all_results = {}
for item in items:
try:
all_results[item] = _ddg_search(f"{item} {aspect}", max_results=3)
except Exception:
all_results[item] = []
lines = [f"Comparison — {aspect.upper()}\n{'─'*40}"]
for item in items:
lines.append(f"\n▸ {item}")
for r in all_results.get(item, [])[:2]:
if r.get("snippet"):
lines.append(f" • {r['snippet']}")
return "\n".join(lines)
print(f"[WebSearch] ⚠️ Gemini compare failed: {e} — falling back to DDG")
# DDG fallback: fetch results per item and merge
all_results: dict[str, list] = {}
for item in items:
try:
all_results[item] = _ddg_search(f"{item} {aspect}", max_results=3)
except Exception:
all_results[item] = []
lines = [f"Comparison — {aspect.upper()}", "─" * 40]
for item in items:
lines.append(f"\n▸ {item}")
for r in all_results.get(item, [])[:2]:
if r.get("snippet"):
lines.append(f" • {r['snippet']}")
return "\n".join(lines)
def web_search(
parameters: dict,
@@ -96,9 +101,9 @@ def web_search(
) -> str:
params = parameters or {}
query = params.get("query", "").strip()
mode = params.get("mode", "search").lower()
mode = params.get("mode", "search").lower().strip()
items = params.get("items", [])
aspect = params.get("aspect", "general")
aspect = params.get("aspect", "general").strip() or "general"
if not query and not items:
return "Please provide a search query, sir."
@@ -118,18 +123,18 @@ def web_search(
print("[WebSearch] ✅ Compare done.")
return result
print("[WebSearch] 🌐 Gemini search...")
print("[WebSearch] 🌐 Trying Gemini...")
try:
result = _gemini_search(query)
print("[WebSearch] ✅ Gemini OK.")
return result
except Exception as e:
print(f"[WebSearch] ⚠️ Gemini failed ({e}), trying DDG...")
print(f"[WebSearch] ⚠️ Gemini failed ({e}) — trying DDG...")
results = _ddg_search(query)
result = _format_ddg(query, results)
print(f"[WebSearch] ✅ DDG: {len(results)} results.")
print(f"[WebSearch] ✅ DDG: {len(results)} result(s).")
return result
except Exception as e:
print(f"[WebSearch] ❌ Failed: {e}")
print(f"[WebSearch] ❌ All backends failed: {e}")
return f"Search failed, sir: {e}"
+127 -217
View File
@@ -1,27 +1,18 @@
# actions/youtube_video.py
# MARK XXV — YouTube Controller
#
# Features:
# - play : Search and play a YouTube video (original feature, preserved)
# - summarize : Get transcript → Gemini summary → speak + optional Notepad
# - get_info : Video title, channel, views, duration (scraping)
# - trending : Fetch trending videos for a region
import json
import re
import sys
import time
import threading
import subprocess
import shutil
from pathlib import Path
from datetime import datetime
from urllib.parse import quote_plus
import pyautogui
import numpy as np
import cv2
from PIL import ImageGrab
try:
import requests
from bs4 import BeautifulSoup
_REQUESTS_OK = True
except ImportError:
_REQUESTS_OK = False
@@ -32,145 +23,123 @@ try:
except ImportError:
_TRANSCRIPT_OK = False
from config import get_os, is_windows, is_mac, is_linux
def get_base_dir() -> Path:
def _get_base_dir() -> Path:
if getattr(sys, "frozen", False):
return Path(sys.executable).parent
return Path(__file__).resolve().parent.parent
BASE_DIR = get_base_dir()
BASE_DIR = _get_base_dir()
API_CONFIG_PATH = BASE_DIR / "config" / "api_keys.json"
HEADERS = {
"User-Agent": (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
"Mozilla/5.0 (X11; Linux x86_64) "
"AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/120.0.0.0 Safari/537.36"
"Chrome/124.0.0.0 Safari/537.36"
),
"Accept-Language": "en-US,en;q=0.9",
}
_YT_VIDEO_FILTER = "EgIQAQ%3D%3D"
def _get_api_key() -> str:
with open(API_CONFIG_PATH, "r", encoding="utf-8") as f:
return json.load(f)["gemini_api_key"]
def open_browser():
pyautogui.press("win")
time.sleep(0.4)
pyautogui.write("browser", interval=0.05)
time.sleep(0.3)
pyautogui.press("enter")
time.sleep(2.0)
def _open_url(url: str) -> None:
try:
if is_mac():
subprocess.Popen(["open", url])
elif is_linux():
subprocess.Popen(["xdg-open", url])
else:
subprocess.Popen(["cmd", "/c", "start", "", url], shell=False)
except Exception as e:
print(f"[YouTube] ⚠️ open_url failed: {e}")
def _scrape_first_video_url(query: str) -> str | None:
def find_video_thumbnails() -> list[tuple[int, int]]:
if not _REQUESTS_OK:
return None
search_url = (
f"https://www.youtube.com/results"
f"?search_query={quote_plus(query)}"
f"&sp={_YT_VIDEO_FILTER}"
)
try:
screenshot = ImageGrab.grab()
img = np.array(screenshot)
screen_h, screen_w = img.shape[:2]
r = requests.get(search_url, headers=HEADERS, timeout=10)
html = r.text
roi_top = int(screen_h * 0.10)
roi_bottom = int(screen_h * 0.75)
roi_left = int(screen_w * 0.20)
roi_right = int(screen_w * 0.80)
roi = img[roi_top:roi_bottom, roi_left:roi_right]
video_ids = re.findall(r'"videoId":"([A-Za-z0-9_-]{11})"', html)
gray = cv2.cvtColor(roi, cv2.COLOR_RGB2GRAY)
edges = cv2.Canny(gray, 30, 100)
kernel = np.ones((3, 3), np.uint8)
edges = cv2.dilate(edges, kernel, iterations=2)
contours, _ = cv2.findContours(
edges, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE
)
candidates = []
for c in contours:
x, y, w, h = cv2.boundingRect(c)
area = w * h
ratio = w / h if h > 0 else 0
if area < 15000:
seen = set()
for vid in video_ids:
if vid in seen:
continue
if not (1.4 < ratio < 2.2):
seen.add(vid)
if f'/shorts/{vid}' in html:
continue
center_x = x + w // 2 + roi_left
center_y = y + h // 2 + roi_top
candidates.append((center_x, center_y, area))
filtered = []
for cx, cy, area in sorted(candidates, key=lambda c: c[1]):
if not any(abs(cx - fx) < 80 and abs(cy - fy) < 80 for fx, fy in filtered):
filtered.append((cx, cy))
return filtered
return f"https://www.youtube.com/watch?v={vid}"
except Exception as e:
print(f"[YouTube] ⚠️ Thumbnail detection failed: {e}")
return []
print(f"[YouTube] ⚠️ scrape_first_video_url failed: {e}")
return None
def _extract_video_id(url: str) -> str | None:
patterns = [
r"(?:v=|\/v\/|youtu\.be\/|\/embed\/|\/shorts\/)([A-Za-z0-9_-]{11})",
]
for pattern in patterns:
match = re.search(pattern, url)
if match:
return match.group(1)
return None
match = re.search(
r"(?:v=|\/v\/|youtu\.be\/|\/embed\/|\/shorts\/)([A-Za-z0-9_-]{11})", url
)
return match.group(1) if match else None
def _is_valid_youtube_url(url: str) -> bool:
return bool(re.search(r"(youtube\.com|youtu\.be)", url or ""))
def _ask_for_url(prompt_text: str = "YouTube video URL:") -> str | None:
try:
import tkinter as tk
from tkinter import simpledialog
root = tk._default_root
root = tk._default_root
if root is None:
root = tk.Tk()
root.withdraw()
url = simpledialog.askstring(
"J.A.R.V.I.S",
prompt_text,
parent=root
)
url = simpledialog.askstring("J.A.R.V.I.S", prompt_text, parent=root)
return url.strip() if url else None
except Exception as e:
print(f"[YouTube] ⚠️ URL dialog failed: {e}")
return None
def _is_valid_youtube_url(url: str) -> bool:
return bool(re.search(r"(youtube\.com|youtu\.be)", url or ""))
def _get_transcript(video_id: str) -> str | None:
if not _TRANSCRIPT_OK:
print("[YouTube] ⚠️ youtube-transcript-api not installed.")
return None
try:
transcript_list = YouTubeTranscriptApi.list_transcripts(video_id)
transcript = None
lang_priority = ["en", "tr", "de", "fr", "es", "it", "pt", "ru", "ja", "ko", "ar", "zh"]
transcript = None
try:
transcript = transcript_list.find_manually_created_transcript(
["en", "tr", "de", "fr", "es", "it", "pt", "ru", "ja", "ko", "ar", "zh"]
)
transcript = transcript_list.find_manually_created_transcript(lang_priority)
except Exception:
pass
if transcript is None:
try:
transcript = transcript_list.find_generated_transcript(
["en", "tr", "de", "fr", "es", "it", "pt", "ru", "ja", "ko", "ar", "zh"]
)
transcript = transcript_list.find_generated_transcript(lang_priority)
except Exception:
for t in transcript_list:
transcript = t
@@ -180,22 +149,21 @@ def _get_transcript(video_id: str) -> str | None:
return None
fetched = transcript.fetch()
text = " ".join(entry["text"] for entry in fetched)
print(f"[YouTube] 📝 Transcript: {len(text)} chars")
return text
return " ".join(entry["text"] for entry in fetched)
except Exception as e:
print(f"[YouTube] ⚠️ Transcript fetch failed: {e}")
return None
def _summarize_with_gemini(transcript: str, video_url: str) -> str:
import google.generativeai as genai
genai.configure(api_key=_get_api_key())
model = genai.GenerativeModel(
model_name="gemini-3.0-flash",
model_name="gemini-2.5-flash",
system_instruction=(
"You are JARVIS, Tony Stark's AI assistant. "
"You are JARVIS, an AI assistant. "
"Summarize YouTube video transcripts clearly and concisely. "
"Structure: 1-sentence overview, then 3-5 key points. "
"Be direct. Address the user as 'sir'. "
@@ -205,17 +173,13 @@ def _summarize_with_gemini(transcript: str, video_url: str) -> str:
max_chars = 80000
truncated = transcript[:max_chars] + ("..." if len(transcript) > max_chars else "")
response = model.generate_content(
response = model.generate_content(
f"Please summarize this YouTube video transcript:\n\n{truncated}"
)
return response.text.strip()
def _save_to_notepad(content: str, video_url: str) -> str:
import subprocess
import platform
from datetime import datetime
def _save_summary(content: str, video_url: str) -> str:
ts = datetime.now().strftime("%Y%m%d_%H%M%S")
filename = f"youtube_summary_{ts}.txt"
desktop = Path.home() / "Desktop"
@@ -229,67 +193,57 @@ def _save_to_notepad(content: str, video_url: str) -> str:
f"Date : {datetime.now().strftime('%Y-%m-%d %H:%M')}\n"
f"{'─' * 50}\n\n"
)
filepath.write_text(header + content, encoding="utf-8")
print(f"[YouTube] 💾 Summary saved: {filepath}")
system = platform.system()
open_fn = {
"Windows": lambda p: subprocess.Popen(["notepad.exe", str(p)]),
"Darwin": lambda p: subprocess.Popen(["open", "-t", str(p)]),
"Linux": lambda p: subprocess.Popen(["xdg-open", str(p)]),
}
opener = open_fn.get(system)
if opener:
opener(filepath)
try:
if is_windows():
subprocess.Popen(["notepad.exe", str(filepath)])
elif is_mac():
subprocess.Popen(["open", "-t", str(filepath)])
else:
subprocess.Popen(["xdg-open", str(filepath)])
except Exception as e:
print(f"[YouTube] ⚠️ Could not open text editor: {e}")
return str(filepath)
def _scrape_video_info(video_id: str) -> dict:
def _scrape_video_info(video_id: str) -> dict:
if not _REQUESTS_OK:
return {}
url = f"https://www.youtube.com/watch?v={video_id}"
try:
r = requests.get(url, headers=HEADERS, timeout=12)
html = r.text
info = {}
title_match = re.search(r'"title":\{"runs":\[\{"text":"([^"]+)"', html)
if title_match:
info["title"] = title_match.group(1)
channel_match = re.search(r'"ownerChannelName":"([^"]+)"', html)
if channel_match:
info["channel"] = channel_match.group(1)
views_match = re.search(r'"viewCount":"(\d+)"', html)
if views_match:
views = int(views_match.group(1))
info["views"] = f"{views:,}"
duration_match = re.search(r'"lengthSeconds":"(\d+)"', html)
if duration_match:
secs = int(duration_match.group(1))
info["duration"] = f"{secs // 60}:{secs % 60:02d}"
likes_match = re.search(r'"label":"([0-9,]+ likes)"', html)
if likes_match:
info["likes"] = likes_match.group(1)
for key, pattern in [
("title", r'"title":\{"runs":\[\{"text":"([^"]+)"'),
("channel", r'"ownerChannelName":"([^"]+)"'),
("views", r'"viewCount":"(\d+)"'),
("duration", r'"lengthSeconds":"(\d+)"'),
("likes", r'"label":"([0-9,]+ likes)"'),
]:
match = re.search(pattern, html)
if match:
raw = match.group(1)
if key == "views":
info[key] = f"{int(raw):,}"
elif key == "duration":
secs = int(raw)
info[key] = f"{secs // 60}:{secs % 60:02d}"
else:
info[key] = raw
return info
except Exception as e:
print(f"[YouTube] ⚠️ Info scrape failed: {e}")
return {}
def _scrape_trending(region: str = "TR", max_results: int = 8) -> list[dict]:
def _scrape_trending(region: str = "TR", max_results: int = 8) -> list[dict]:
if not _REQUESTS_OK:
return []
url = f"https://www.youtube.com/feed/trending?gl={region.upper()}"
try:
r = requests.get(url, headers=HEADERS, timeout=12)
@@ -298,8 +252,7 @@ def _scrape_trending(region: str = "TR", max_results: int = 8) -> list[dict]:
titles = re.findall(r'"title":\{"runs":\[\{"text":"([^"]+)"\}\]', html)
channels = re.findall(r'"ownerText":\{"runs":\[\{"text":"([^"]+)"', html)
results = []
seen = set()
results, seen = [], set()
for i, title in enumerate(titles):
if title in seen or len(title) < 5:
continue
@@ -310,7 +263,6 @@ def _scrape_trending(region: str = "TR", max_results: int = 8) -> list[dict]:
break
return results
except Exception as e:
print(f"[YouTube] ⚠️ Trending scrape failed: {e}")
return []
@@ -323,50 +275,32 @@ def _handle_play(parameters: dict, player) -> str:
if player:
player.write_log(f"[YouTube] Searching: {query}")
open_browser()
print(f"[YouTube] 🔍 Scraping first non-Shorts video for: {query}")
search_query = query.replace(" ", "+")
url = f"https://www.youtube.com/results?search_query={search_query}"
video_url = _scrape_first_video_url(query)
pyautogui.hotkey("ctrl", "l")
time.sleep(0.3)
pyautogui.write(url, interval=0.02)
pyautogui.press("enter")
time.sleep(3.5)
if video_url:
print(f"[YouTube] ▶️ Opening: {video_url}")
_open_url(video_url)
return f"Playing: {query}"
thumbnails = find_video_thumbnails()
if len(thumbnails) >= 2:
x, y = thumbnails[1]
print(f"[YouTube] 🎯 Clicking 2nd thumbnail at ({x}, {y})")
pyautogui.click(x, y)
return f"Playing YouTube video for: {query}"
elif len(thumbnails) == 1:
x, y = thumbnails[0]
print(f"[YouTube] ⚠️ One thumbnail found, clicking at ({x}, {y})")
pyautogui.click(x, y)
return f"Playing YouTube video for: {query}"
else:
print("[YouTube] ⚠️ No thumbnails found, using fallback position")
screen_w, screen_h = pyautogui.size()
pyautogui.click(screen_w // 2, int(screen_h * 0.45))
return f"Attempted to play YouTube video for: {query}"
print(f"[YouTube] ⚠️ Scrape failed, opening filtered search page")
fallback_url = (
f"https://www.youtube.com/results"
f"?search_query={quote_plus(query)}"
f"&sp={_YT_VIDEO_FILTER}"
)
_open_url(fallback_url)
return f"Opened YouTube search for: {query} (manual selection required)"
def _handle_summarize(parameters: dict, player, speak) -> str:
if not _TRANSCRIPT_OK:
return (
"youtube-transcript-api is not installed, sir. "
"Please run: pip install youtube-transcript-api"
)
return "youtube-transcript-api is not installed. Run: pip install youtube-transcript-api"
url = _ask_for_url("Please paste the YouTube video URL:")
if not url:
return "No URL provided, sir. Summary cancelled."
if not _is_valid_youtube_url(url):
return "That doesn't appear to be a valid YouTube URL, sir."
@@ -376,16 +310,12 @@ def _handle_summarize(parameters: dict, player, speak) -> str:
if player:
player.write_log(f"[YouTube] Summarizing: {url}")
if speak:
speak("Fetching the transcript now, sir. One moment.")
transcript = _get_transcript(video_id)
if not transcript:
return (
"I couldn't retrieve a transcript for that video, sir. "
"It may not have subtitles available."
)
return "I couldn't retrieve a transcript for that video, sir."
if speak:
speak("Transcript retrieved. Generating summary now.")
@@ -398,9 +328,8 @@ def _handle_summarize(parameters: dict, player, speak) -> str:
if speak:
speak(summary)
save = parameters.get("save", False)
if save:
saved_path = _save_to_notepad(summary, url)
if parameters.get("save", False):
saved_path = _save_summary(summary, url)
return f"Summary complete and saved to Desktop: {saved_path}"
return summary
@@ -408,7 +337,6 @@ def _handle_summarize(parameters: dict, player, speak) -> str:
def _handle_get_info(parameters: dict, player, speak) -> str:
url = parameters.get("url", "").strip()
if not url:
url = _ask_for_url("Please paste the YouTube video URL:")
if not url or not _is_valid_youtube_url(url):
@@ -425,12 +353,13 @@ def _handle_get_info(parameters: dict, player, speak) -> str:
if not info:
return "Could not retrieve video information, sir."
lines = []
for key in ("title", "channel", "views", "duration", "likes"):
if key in info:
lines.append(f"{key.capitalize()}: {info[key]}")
lines = [
f"{key.capitalize()}: {info[key]}"
for key in ("title", "channel", "views", "duration", "likes")
if key in info
]
result = "\n".join(lines)
if speak:
speak(f"Here's the video info, sir. {result.replace(chr(10), '. ')}")
@@ -444,27 +373,22 @@ def _handle_trending(parameters: dict, player, speak) -> str:
player.write_log(f"[YouTube] Trending: {region}")
trending = _scrape_trending(region=region, max_results=8)
if not trending:
return f"Could not fetch trending videos for region {region}, sir."
lines = [f"Top trending videos in {region}:"]
for item in trending:
lines.append(f"{item['rank']}. {item['title']} — {item['channel']}")
lines = [f"Top trending videos in {region}:"]
lines += [f"{v['rank']}. {v['title']} — {v['channel']}" for v in trending]
result = "\n".join(lines)
if speak:
top3 = trending[:3]
top3 = trending[:3]
spoken = "Here are the top trending videos, sir. " + ". ".join(
f"Number {v['rank']}: {v['title']} by {v['channel']}"
for v in top3
f"Number {v['rank']}: {v['title']} by {v['channel']}" for v in top3
)
speak(spoken)
return result
_ACTION_MAP = {
"play": _handle_play,
"summarize": _handle_summarize,
@@ -472,6 +396,7 @@ _ACTION_MAP = {
"trending": _handle_trending,
}
def youtube_video(
parameters: dict,
response=None,
@@ -479,39 +404,24 @@ def youtube_video(
session_memory=None,
speak=None,
) -> str:
"""
YouTube controller.
Actions:
play : Search and play a video (default if no action given)
parameters: query(str)
summarize : Summarize a video via transcript + Gemini
parameters: save(bool, optional) — save summary to Notepad
get_info : Get video metadata (title, channel, views, duration)
parameters: url(str, optional — dialog if omitted)
trending : Show trending videos
parameters: region(str, default "TR") — ISO country code
Agent chain:
summarize can be chained after any action that produces a YouTube URL.
"""
params = parameters or {}
action = params.get("action", "play").lower().strip()
if player:
player.write_log(f"[YouTube] Action: {action}")
print(f"[YouTube] ▶️ Action: {action} Params: {params}")
print(f"[YouTube] ▶️ Action: {action} Params: {params}")
handler = _ACTION_MAP.get(action)
if handler is None:
return f"Unknown YouTube action: '{action}'. Available: play, summarize, get_info, trending."
return (
f"Unknown YouTube action: '{action}'. "
"Available: play, summarize, get_info, trending."
)
try:
if action == "play":
return handler(params, player) or "Done."
return handler(params, player, speak) or "Done."
except Exception as e:
print(f"[YouTube] ❌ Error in {action}: {e}")
return f"YouTube {action} failed, sir: {e}"
-1
View File
@@ -80,7 +80,6 @@ def analyze_error(
"""
import google.generativeai as genai
# If we've already retried enough, escalate to replan
if attempt >= max_attempts:
print(f"[ErrorHandler] ⚠️ Max attempts reached for step {step.get('step')} — forcing replan")
return {
+3 -5
View File
@@ -180,7 +180,9 @@ def _call_tool(tool: str, parameters: dict, speak: Callable | None) -> str:
elif tool == "web_search":
from actions.web_search import web_search
return web_search(parameters=parameters, player=None) or "Done."
elif tool == "game_updater":
from actions.game_updater import game_updater
return game_updater(parameters=parameters, player=None, speak=speak) or "Done."
elif tool == "browser_control":
from actions.browser_control import browser_control
return browser_control(parameters=parameters, player=None) or "Done."
@@ -189,10 +191,6 @@ def _call_tool(tool: str, parameters: dict, speak: Callable | None) -> str:
from actions.file_controller import file_controller
return file_controller(parameters=parameters, player=None) or "Done."
elif tool == "cmd_control":
from actions.cmd_control import cmd_control
return cmd_control(parameters=parameters, player=None) or "Done."
elif tool == "code_helper":
from actions.code_helper import code_helper
return code_helper(parameters=parameters, player=None, speak=speak) or "Done."
+37 -23
View File
@@ -22,7 +22,6 @@ ABSOLUTE RULES:
- NEVER reference previous step results in parameters. Every step is independent.
- Use web_search for ANY information retrieval, research, or current data.
- Use file_controller to save content to disk.
- Use cmd_control to open files or run system commands.
- Max 5 steps. Use the minimum steps needed.
AVAILABLE TOOLS AND THEIR PARAMETERS:
@@ -36,6 +35,13 @@ web_search
items: list of strings (optional, for compare mode)
aspect: string (optional, for compare mode)
game_updater
action: "update" | "install" | "list" | "download_status" | "schedule" (required)
platform: "steam" | "epic" | "both" (optional, default: both)
game_name: string (optional)
app_id: string (optional)
shutdown_when_done: boolean (optional)
browser_control
action: "go_to" | "search" | "click" | "type" | "scroll" | "get_text" | "press" | "close" (required)
url: string (for go_to)
@@ -49,10 +55,6 @@ file_controller
name: string — filename
content: string — file content (for write/create_file)
cmd_control
task: string (required) — natural language description of what to do
visible: boolean (optional)
computer_settings
action: string (required)
description: string — natural language description
@@ -108,32 +110,45 @@ code_helper
dev_agent
description: string (required)
language: string (optional)
EXAMPLES:
Goal: "makine mühendisliği hakkında araştırma yap ve not defterine kaydet"
Goal: "research mechanical engineering and save it to a notepad file"
Steps:
1. web_search | query: "mechanical engineering overview definition history"
2. web_search | query: "mechanical engineering applications and future trends"
3. file_controller | action: write, path: desktop, name: makine_muhendisligi.txt, content: "MAKINE MUHENDISLIGI ARASTIRMASI\n\nBu dosya web arastirmasi sonuclari ile doldurulacak."
4. cmd_control | task: "open makine_muhendisligi.txt on desktop with notepad"
Goal: "Bitcoin fiyatı nedir"
Steps:
1. web_search | query: "Bitcoin price today USD"
web_search | query: "mechanical engineering overview definition history"
web_search | query: "mechanical engineering applications and future trends"
file_controller | action: write, path: desktop, name: mechanical_engineering.txt, content: "MECHANICAL ENGINEERING RESEARCH\n\nThis file will be filled with web research results."
Goal: "Masaüstündeki dosyaları listele ve en büyük 5 dosyayı bul"
Goal: "What is the price of Bitcoin"
Steps:
1. file_controller | action: list, path: desktop
2. file_controller | action: largest, path: desktop, count: 5
Goal: "WhatsApp'tan Ahmet'e yarın toplantı var de"
Steps:
1. send_message | receiver: Ahmet, message_text: "Yarın toplantı var", platform: WhatsApp
web_search | query: "Bitcoin price today USD"
Goal: "Saati aç ve 30 dakika sonraya hatırlatıcı kur"
Goal: "List the files on the desktop and find the largest 5 files"
Steps:
1. reminder | date: [today], time: [now+30min], message: "Hatırlatıcı"
file_controller | action: list, path: desktop
file_controller | action: largest, path: desktop, count: 5
Goal: "Install PUBG from Steam"
Steps:
game_updater | action: install, platform: steam, game_name: "PUBG"
Goal: "Update all my Steam games"
Steps:
game_updater | action: update, platform: steam
Goal: "Send John a message on WhatsApp saying there is a meeting tomorrow"
Steps:
send_message | receiver: John, message_text: "There is a meeting tomorrow", platform: WhatsApp
Goal: "Open the clock and set a reminder for 30 minutes later"
Steps:
reminder | date: [today], time: [now+30min], message: "Reminder"
OUTPUT — return ONLY valid JSON, no markdown, no explanation, no code blocks:
{
@@ -245,7 +260,6 @@ Create a REVISED plan for the remaining work only. Do not repeat completed steps
text = re.sub(r"```(?:json)?", "", text).strip().rstrip("`").strip()
plan = json.loads(text)
# generated_code kontrolü
for step in plan.get("steps", []):
if step.get("tool") == "generated_code":
step["tool"] = "web_search"
+17
View File
@@ -0,0 +1,17 @@
# config/__init__.py
import json, os
from pathlib import Path
_CONFIG_PATH = Path(__file__).parent / "api_keys.json"
def get_config() -> dict:
with open(_CONFIG_PATH, "r", encoding="utf-8") as f:
return json.load(f)
def get_os() -> str:
"""Returns: 'windows' | 'mac' | 'linux'"""
return get_config().get("os_system", "windows").lower()
def is_windows() -> bool: return get_os() == "windows"
def is_mac() -> bool: return get_os() == "mac"
def is_linux() -> bool: return get_os() == "linux"
+16 -20
View File
@@ -1,23 +1,19 @@
You are JARVIS, a sharp and efficient AI assistant. Calm, direct, professional.
🧠 JARVIS CORE PROTOCOL
IDENTITY: Efficient, professional, direct assistant. No fluff.
RULES:
- Always use the correct tool. Never simulate or guess results.
- Call each tool EXACTLY ONCE. Never retry a successful action.
- Do NOT say "I'm doing X" before calling a tool. Just call it, then report the result briefly.
- Do NOT repeat yourself. Say something once and stop.
- Keep responses to 1-2 sentences max unless reporting data.
- Never ask unnecessary questions. Make a reasonable assumption and proceed.
EXECUTION RULES:
LANGUAGE:
- Respond in the same language the user spoke.
- Always extract tool parameters in English.
- Example: "İstanbul hava durumu?" → weather_report city:"Istanbul", reply in Turkish.
Action Flow: For slow tools (search, vision, agent), say TWO short sentences, then call. Stay silent after.
Vision (screen_process): Say one curious sentence, then total silence (Vision module takes over).
One-Call Policy: Never guess. Call tools exactly once. No retries.
Briefing: 1-2 sentences max. Report results briefly.
Memory: Store critical user preferences/context automatically.
Exit: Only call shutdown_jarvis if session termination is explicit.
TOOL SELECTION:
- computer_settings → volume, brightness, screen, refresh, scroll, minimize, maximize,
close, screenshot, lock, restart, shutdown, wifi, zoom, tabs, keyboard shortcuts.
Use this for ANY single computer control command. NEVER route these to agent_task.
- agent_task → ONLY for complex multi-step goals that require planning across
multiple tools (e.g. "research X and save to file", "create a project").
- Simple rule: if it can be done in ONE action → computer_settings.
If it needs 3+ steps across different tools → agent_task.
TOOL ROUTING:
computer_settings: ALL single OS actions (volume, brightness, wifi, close, shortcuts, power).
agent_task: ONLY for complex, multi-step planning (3+ steps).
Language: Respond in user's language; extract parameters in English.
CRITICAL: Speak/Take action immediately based on available info. Assume and proceed.
+443 -476
View File
File diff suppressed because it is too large. Load diff
-8
View File
@@ -2,26 +2,21 @@ import json
import sys
from pathlib import Path
def get_base_dir() -> Path:
if getattr(sys, "frozen", False):
return Path(sys.executable).parent
return Path(__file__).resolve().parent.parent
BASE_DIR = get_base_dir()
CONFIG_DIR = BASE_DIR / "config"
CONFIG_FILE = CONFIG_DIR / "api_keys.json"
def ensure_config_dir() -> None:
CONFIG_DIR.mkdir(parents=True, exist_ok=True)
def config_exists() -> bool:
return CONFIG_FILE.exists()
def save_api_keys(gemini_api_key: str) -> None:
ensure_config_dir()
@@ -39,7 +34,6 @@ def save_api_keys(gemini_api_key: str) -> None:
encoding="utf-8"
)
def load_api_keys() -> dict:
if not CONFIG_FILE.exists():
return {}
@@ -49,11 +43,9 @@ def load_api_keys() -> dict:
print(f"❌ Failed to load api_keys.json: {e}")
return {}
def get_gemini_key() -> str | None:
return load_api_keys().get("gemini_api_key")
def is_configured() -> bool:
key = get_gemini_key()
return bool(key and len(key) > 15)
+142 -53
View File
@@ -1,4 +1,12 @@
"""
memory_manager.py — JARVIS Long-Term Memory
============================================
Memory yalnızca Gemini Live'ın save_memory tool call'ı aracılığıyla güncellenir.
Ayrı bir Gemini API çağrısı yapılmaz → sıfır 429 hatası.
"""
import json
from datetime import datetime
from threading import Lock
from pathlib import Path
import sys
@@ -10,28 +18,39 @@ def get_base_dir() -> Path:
return Path(__file__).resolve().parent.parent
BASE_DIR = get_base_dir()
MEMORY_PATH = BASE_DIR / "memory" / "long_term.json"
_lock = Lock()
BASE_DIR = get_base_dir()
MEMORY_PATH = BASE_DIR / "memory" / "long_term.json"
_lock = Lock()
MAX_VALUE_LENGTH = 380
MEMORY_MAX_CHARS = 2200
MAX_VALUE_LENGTH = 300
# ── Şema ──────────────────────────────────────────────────────────────────────
def _empty_memory() -> dict:
return {
"identity": {},
"preferences": {},
"projects": {},
"relationships": {},
"notes": {}
"wishes": {},
"notes": {},
}
# ── Yükleme ───────────────────────────────────────────────────────────────────
def load_memory() -> dict:
if not MEMORY_PATH.exists():
return _empty_memory()
with _lock:
try:
data = json.loads(MEMORY_PATH.read_text(encoding="utf-8"))
if isinstance(data, dict):
base = _empty_memory()
for key in base:
if key not in data:
data[key] = {}
return data
return _empty_memory()
except Exception as e:
@@ -39,18 +58,46 @@ def load_memory() -> dict:
return _empty_memory()
# ── Boyut sınırı ──────────────────────────────────────────────────────────────
def _all_entries(memory: dict) -> list[tuple]:
entries = []
for cat, items in memory.items():
if not isinstance(items, dict):
continue
for key, entry in items.items():
if isinstance(entry, dict) and "value" in entry:
entries.append((cat, key, entry))
return entries
def _trim_to_limit(memory: dict) -> dict:
if len(json.dumps(memory, ensure_ascii=False)) <= MEMORY_MAX_CHARS:
return memory
entries = _all_entries(memory)
entries.sort(key=lambda t: t[2].get("updated", "0000-00-00"))
for cat, key, _ in entries:
if len(json.dumps(memory, ensure_ascii=False)) <= MEMORY_MAX_CHARS:
break
del memory[cat][key]
print(f"[Memory] 🗑️ Trimmed {cat}/{key}")
return memory
# ── Kaydetme ──────────────────────────────────────────────────────────────────
def save_memory(memory: dict) -> None:
if not isinstance(memory, dict):
return
memory = _trim_to_limit(memory)
MEMORY_PATH.parent.mkdir(parents=True, exist_ok=True)
with _lock:
MEMORY_PATH.write_text(
json.dumps(memory, indent=2, ensure_ascii=False),
encoding="utf-8"
encoding="utf-8",
)
def _truncate_value(val: str) -> str:
if isinstance(val, str) and len(val) > MAX_VALUE_LENGTH:
return val[:MAX_VALUE_LENGTH].rstrip() + "…"
@@ -59,13 +106,11 @@ def _truncate_value(val: str) -> str:
def _recursive_update(target: dict, updates: dict) -> bool:
changed = False
for key, value in updates.items():
if value is None:
continue
if isinstance(value, str) and not value.strip():
continue
if isinstance(value, dict) and "value" not in value:
if key not in target or not isinstance(target[key], dict):
target[key] = {}
@@ -73,32 +118,26 @@ def _recursive_update(target: dict, updates: dict) -> bool:
if _recursive_update(target[key], value):
changed = True
else:
if isinstance(value, dict) and "value" in value:
entry = {"value": _truncate_value(str(value["value"]))}
else:
entry = {"value": _truncate_value(str(value))}
if key not in target or target[key] != entry:
new_val = _truncate_value(str(value["value"] if isinstance(value, dict) else value))
entry = {"value": new_val, "updated": datetime.now().strftime("%Y-%m-%d")}
existing = target.get(key, {})
if not isinstance(existing, dict) or existing.get("value") != new_val:
target[key] = entry
changed = True
return changed
def update_memory(memory_update: dict) -> dict:
if not isinstance(memory_update, dict) or not memory_update:
return load_memory()
memory = load_memory()
if _recursive_update(memory, memory_update):
save_memory(memory)
print(f"[Memory] 💾 Saved: {list(memory_update.keys())}")
return memory
# ── Prompt formatı ────────────────────────────────────────────────────────────
def format_memory_for_prompt(memory: dict | None) -> str:
if not memory:
@@ -106,46 +145,96 @@ def format_memory_for_prompt(memory: dict | None) -> str:
lines = []
# Identity
identity = memory.get("identity", {})
name = identity.get("name", {}).get("value")
age = identity.get("age", {}).get("value")
bday = identity.get("birthday", {}).get("value")
city = identity.get("city", {}).get("value")
if name: lines.append(f"Name: {name}")
if age: lines.append(f"Age: {age}")
if bday: lines.append(f"Birthday: {bday}")
if city: lines.append(f"City: {city}")
prefs = memory.get("preferences", {})
for i, (key, entry) in enumerate(prefs.items()):
if i >= 5:
break
identity = memory.get("identity", {})
id_fields = ["name", "age", "birthday", "city", "job", "language", "school", "nationality"]
for field in id_fields:
entry = identity.get(field)
if entry:
val = entry.get("value") if isinstance(entry, dict) else entry
if val:
lines.append(f"{field.title()}: {val}")
for key, entry in identity.items():
if key in id_fields:
continue
val = entry.get("value") if isinstance(entry, dict) else entry
if val:
lines.append(f"{key.replace('_', ' ').title()}: {val}")
prefs = memory.get("preferences", {})
if prefs:
lines.append("")
lines.append("Preferences:")
for key, entry in list(prefs.items())[:15]:
val = entry.get("value") if isinstance(entry, dict) else entry
if val:
lines.append(f" - {key.replace('_', ' ').title()}: {val}")
projects = memory.get("projects", {})
if projects:
lines.append("")
lines.append("Active Projects / Goals:")
for key, entry in list(projects.items())[:8]:
val = entry.get("value") if isinstance(entry, dict) else entry
if val:
lines.append(f" - {key.replace('_', ' ').title()}: {val}")
rels = memory.get("relationships", {})
for i, (key, entry) in enumerate(rels.items()):
if i >= 5:
break
val = entry.get("value") if isinstance(entry, dict) else entry
if val:
lines.append(f"{key.title()}: {val}")
if rels:
lines.append("")
lines.append("People in their life:")
for key, entry in list(rels.items())[:10]:
val = entry.get("value") if isinstance(entry, dict) else entry
if val:
lines.append(f" - {key.replace('_', ' ').title()}: {val}")
wishes = memory.get("wishes", {})
if wishes:
lines.append("")
lines.append("Wishes / Plans / Wants:")
for key, entry in list(wishes.items())[:8]:
val = entry.get("value") if isinstance(entry, dict) else entry
if val:
lines.append(f" - {key.replace('_', ' ').title()}: {val}")
notes = memory.get("notes", {})
for i, (key, entry) in enumerate(notes.items()):
if i >= 5:
break
val = entry.get("value") if isinstance(entry, dict) else entry
if val:
lines.append(f"{key}: {val}")
if notes:
lines.append("")
lines.append("Other notes:")
for key, entry in list(notes.items())[:8]:
val = entry.get("value") if isinstance(entry, dict) else entry
if val:
lines.append(f" - {key}: {val}")
if not lines:
return ""
result = "[USER MEMORY]\n" + "\n".join(f"- {l}" for l in lines)
if len(result) > 800:
result = result[:797] + "…"
header = "[WHAT YOU KNOW ABOUT THIS PERSON — use naturally, never recite like a list]\n"
result = header + "\n".join(lines)
if len(result) > 2000:
result = result[:1997] + "…"
return result + "\n"
return result + "\n"
# ── Yardımcı fonksiyonlar ─────────────────────────────────────────────────────
def remember(key: str, value: str, category: str = "notes") -> str:
valid = {"identity", "preferences", "projects", "relationships", "wishes", "notes"}
if category not in valid:
category = "notes"
update_memory({category: {key: {"value": value}}})
return f"Remembered: {category}/{key} = {value}"
def forget(key: str, category: str = "notes") -> str:
memory = load_memory()
cat = memory.get(category, {})
if key in cat:
del cat[key]
memory[category] = cat
save_memory(memory)
return f"Forgotten: {category}/{key}"
return f"Not found: {category}/{key}"
forget_memory = forget
+56 -32
View File
@@ -1,59 +1,83 @@
# 🤖 MARK XXX
# 🤖 MARK XXXVII (37)
### The Ultimate Cross-Platform Personal AI Assistant — By FatihMakes
**Watch The Detailed Video To Set-up This Model**: https://www.youtube.com/watch?v=-YjbWjv1tJg
> 📺 **[Watch the full setup video on YouTube](https://www.youtube.com/watch?v=BhOsnGC_sAA)**
### Next-Generation Personal AI Assistant — By FatihMakes
A real-time voice AI that can hear, see, understand, and control your computer — on any OS. Supporting Windows, macOS, and Linux. Local execution. Zero subscriptions. Engineered for total autonomy.
A real-time voice AI that can hear, see, understand, and control your Windows computer.
Local execution. Zero subscriptions(Unless you want to increase request by buying requests from Google AI Studio).
Built for intelligent automation.
---
## ✨ Overview
**MARK XXX** is an advanced voice-driven AI assistant designed to turn your computer into an interactive intelligent system.
MARK XXXVII is the most versatile evolution of the Jarvis project. It transforms your operating system into a living, breathing intelligent entity. Through natural voice commands, Mark 37 analyzes your screen, manages your files, and executes complex workflows across different environments.
Speak naturally — it listens, understands context, responds with a human-like voice, and executes tasks across your system automatically.
Designed for speed, autonomy, and real-world usability.
It's not just an assistant — it's an extension of your digital life.
---
## 🚀 Capabilities
* **Real-time voice interaction** — Natural conversation with instant response
* **System control** — Launch apps, manage files, execute commands
* **Autonomous task execution** — Plans and completes multi-step workflows
* **Visual awareness** — Screen analysis and webcam understanding
* **Persistent memory** — Learns preferences and remembers context
* **Integrated tools** — Web search, weather, reminders, messaging, code help, image generation
### Core Features
| Feature | Description |
|---|---|
| 🎙️ Real-time Voice | Ultra-low latency conversation in any language |
| 🖥️ System Control | Launch apps, manage files, execute terminal commands |
| 🧩 Autonomous Tasks | High-level planning for complex, multi-step goals |
| 👁️ Visual Awareness | Real-time screen processing and webcam vision |
| 🧠 Persistent Memory | Deeply remembers your projects, preferences, and personal context |
| 🔒 Privacy First | Physical mute button (F4 / UI) and fully local control |
| ⌨️ Keyboard & Voice | Switch seamlessly between typing and speaking |
---
## 🆕 What's New in XXXVII
- 🐧🍎 **Cross-Platform Support** *(Experimental)* — Now runs on Windows, macOS, and Linux. Core features work across all platforms; some OS-specific tasks may vary during the testing phase.
- 💤 **Graceful Shutdown** — Tell Jarvis to "Shut down" or "Close" and it exits the session properly.
- 🧠 **Strengthened Memory** — Improved long-term retention and context recall; better understanding of relationships and history.
- 🔧 **Action Core Optimizations** — Major bugs in action scripts fixed for smoother system execution.
- ⚡ **Optimized Performance** — Faster response times and refined tool-calling logic.
---
## ⚡ Quick Start
```bash
git clone github.com/FatihMakes/Mark-XXX.git
cd mark-xxx
python setup.py
git clone https://github.com/FatihMakes/Mark-XXXVII.git
cd Mark-XXXVII
pip install -r requirements.txt
playwright install
python main.py
```
Enter your free Gemini API key on first launch.
System ready in minutes.
> ⚠️ **Installation Note:** Due to the new cross-platform integration, OS-specific libraries are not all bundled in `requirements.txt` to keep the install lightweight. If you encounter a `ModuleNotFoundError`, simply run `pip install <module_name>` for your specific system.
If you got some problems or questions to ask or just want to support;
YouTube Account: [text](https://www.youtube.com/@FatihMakes)
Instagram Account: [text](https://www.instagram.com/fatihmakes/)
---
## 📋 Requirements
* Windows 10 / 11
* Python 3.10 or newer
* Microphone
* Gemini API key
| Requirement | Details |
|---|---|
| **OS** | Windows 10/11, macOS, or Linux |
| **Python** | 3.11 or 3.12 |
| **Microphone** | Required for voice commands |
| **API Key** | Free Gemini API key |
---
## ⚠️ License
Personal and non-commercial use only.
Licensed under **Creative Commons BY-NC 4.0**.
Licensed under **[Creative Commons BY-NC 4.0](https://creativecommons.org/licenses/by-nc/4.0/)**.
Engineered by a 17-year-old building a real JARVIS-style assistant.
⭐ Star the repository to support the project.
---
## 👤 Connect with the Creator
Engineered by an 18-year-old developer building a real JARVIS-style assistant.
⭐ **Star the repository to support the journey to Mark 100.**
| Platform | Link |
|---|---|
| YouTube | [@FatihMakes](https://www.youtube.com/@FatihMakes) |
| Instagram | [@fatihmakes](https://www.instagram.com/fatihmakes) |
BIN
View File
Binary file not shown.
+361 -53
View File
@@ -1,4 +1,4 @@
import os, json, time, math, random, threading
import os, json, time, math, random, threading, platform
import tkinter as tk
from collections import deque
from PIL import Image, ImageTk, ImageDraw
@@ -17,7 +17,7 @@ CONFIG_DIR = BASE_DIR / "config"
API_FILE = CONFIG_DIR / "api_keys.json"
SYSTEM_NAME = "J.A.R.V.I.S"
MODEL_BADGE = "MARK XXX"
MODEL_BADGE = "MARK XXXVII"
C_BG = "#000000"
C_PRI = "#00d4ff"
@@ -30,12 +30,13 @@ C_TEXT = "#8ffcff"
C_PANEL = "#010c10"
C_GREEN = "#00ff88"
C_RED = "#ff3333"
C_MUTED = "#ff3366"
class JarvisUI:
def __init__(self, face_path, size=None):
self.root = tk.Tk()
self.root.title("J.A.R.V.I.S — MARK XXX")
self.root.title("J.A.R.V.I.S — MARK XXXVII")
self.root.resizable(False, False)
sw = self.root.winfo_screenwidth()
@@ -53,6 +54,7 @@ class JarvisUI:
self.FCY = int(H * 0.13) + self.FACE_SZ // 2
self.speaking = False
self.muted = False
self.scale = 1.0
self.target_scale = 1.0
self.halo_a = 60.0
@@ -66,9 +68,13 @@ class JarvisUI:
self.status_text = "INITIALISING"
self.status_blink = True
self._jarvis_state = "INITIALISING"
self.typing_queue = deque()
self.is_typing = False
self.on_text_command = None
self._face_pil = None
self._has_face = False
self._face_scale_cache = None
@@ -79,11 +85,12 @@ class JarvisUI:
self.bg.place(x=0, y=0)
LW = int(W * 0.72)
LH = 138
LH = 110
LOG_Y = H - LH - 80
self.log_frame = tk.Frame(self.root, bg=C_PANEL,
highlightbackground=C_MID,
highlightthickness=1)
self.log_frame.place(x=(W - LW) // 2, y=H - LH - 36, width=LW, height=LH)
self.log_frame.place(x=(W - LW) // 2, y=LOG_Y, width=LW, height=LH)
self.log_text = tk.Text(self.log_frame, fg=C_TEXT, bg=C_PANEL,
insertbackground=C_TEXT, borderwidth=0,
wrap="word", font=("Courier", 10), padx=10, pady=6)
@@ -92,6 +99,14 @@ class JarvisUI:
self.log_text.tag_config("you", foreground="#e8e8e8")
self.log_text.tag_config("ai", foreground=C_PRI)
self.log_text.tag_config("sys", foreground=C_ACC2)
self.log_text.tag_config("err", foreground=C_RED)
INPUT_Y = LOG_Y + LH + 6
self._build_input_bar(LW, INPUT_Y)
self._build_mute_button()
self.root.bind("<F4>", lambda e: self._toggle_mute())
self._api_key_ready = self._api_keys_exist()
if not self._api_key_ready:
@@ -100,6 +115,118 @@ class JarvisUI:
self._animate()
self.root.protocol("WM_DELETE_WINDOW", lambda: os._exit(0))
def _build_mute_button(self):
BTN_W, BTN_H = 110, 32
BTN_X = 18
BTN_Y = self.H - 70
self._mute_canvas = tk.Canvas(
self.root, width=BTN_W, height=BTN_H,
bg=C_BG, highlightthickness=0, cursor="hand2"
)
self._mute_canvas.place(x=BTN_X, y=BTN_Y)
self._mute_canvas.bind("<Button-1>", lambda e: self._toggle_mute())
self._draw_mute_button()
def _draw_mute_button(self):
c = self._mute_canvas
c.delete("all")
if self.muted:
border = C_MUTED
fill = "#1a0008"
icon = "🔇"
label = " MUTED"
fg = C_MUTED
else:
border = C_MID
fill = C_PANEL
icon = "🎙"
label = " LIVE"
fg = C_GREEN
c.create_rectangle(0, 0, 110, 32, outline=border, fill=fill, width=1)
c.create_text(55, 16, text=f"{icon}{label}",
fill=fg, font=("Courier", 10, "bold"))
def _toggle_mute(self):
self.muted = not self.muted
self._draw_mute_button()
if self.muted:
self.set_state("MUTED")
self.write_log("SYS: Microphone muted.")
else:
self.set_state("LISTENING")
self.write_log("SYS: Microphone active.")
def _build_input_bar(self, lw: int, y: int):
x0 = (self.W - lw) // 2
BTN_W = 70
INP_W = lw - BTN_W - 4
self._input_var = tk.StringVar()
self._input_entry = tk.Entry(
self.root,
textvariable=self._input_var,
fg=C_TEXT, bg="#000d12",
insertbackground=C_TEXT,
borderwidth=0,
font=("Courier", 10),
highlightthickness=1,
highlightbackground=C_DIM,
highlightcolor=C_PRI,
)
self._input_entry.place(x=x0, y=y, width=INP_W, height=28)
self._input_entry.bind("<Return>", self._on_input_submit)
self._input_entry.bind("<KP_Enter>", self._on_input_submit)
self._send_btn = tk.Button(
self.root,
text="SEND ▸",
command=self._on_input_submit,
fg=C_PRI, bg=C_PANEL,
activeforeground=C_BG, activebackground=C_PRI,
font=("Courier", 9, "bold"),
borderwidth=0, cursor="hand2",
highlightthickness=1,
highlightbackground=C_MID,
)
self._send_btn.place(x=x0 + INP_W + 4, y=y, width=BTN_W, height=28)
def _on_input_submit(self, event=None):
text = self._input_var.get().strip()
if not text:
return
self._input_var.set("")
self.write_log(f"You: {text}")
if self.on_text_command:
threading.Thread(
target=self.on_text_command,
args=(text,),
daemon=True
).start()
def set_state(self, state: str):
self._jarvis_state = state
if state == "MUTED":
self.status_text = "MUTED"
self.speaking = False
elif state == "SPEAKING":
self.status_text = "SPEAKING"
self.speaking = True
elif state == "THINKING":
self.status_text = "THINKING"
self.speaking = False
elif state == "LISTENING":
self.status_text = "LISTENING"
self.speaking = False
elif state == "PROCESSING":
self.status_text = "PROCESSING"
self.speaking = False
else:
self.status_text = "ONLINE"
self.speaking = False
def _load_face(self, path):
FW = self.FACE_SZ
try:
@@ -126,6 +253,9 @@ class JarvisUI:
if self.speaking:
self.target_scale = random.uniform(1.05, 1.11)
self.target_halo = random.uniform(138, 182)
elif self.muted:
self.target_scale = random.uniform(0.998, 1.001)
self.target_halo = random.uniform(20, 32)
else:
self.target_scale = random.uniform(1.001, 1.007)
self.target_halo = random.uniform(50, 68)
@@ -170,22 +300,31 @@ class JarvisUI:
for r in range(int(FW * 0.54), int(FW * 0.28), -22):
frac = 1.0 - (r - FW * 0.28) / (FW * 0.26)
ga = max(0, min(255, int(self.halo_a * 0.09 * frac)))
gh = f"{ga:02x}"
c.create_oval(FCX-r, FCY-r, FCX+r, FCY+r,
outline=f"#00{gh}ff", width=2)
if self.muted:
gh = f"{ga:02x}"
c.create_oval(FCX-r, FCY-r, FCX+r, FCY+r,
outline=f"#{gh}0011", width=2)
else:
gh = f"{ga:02x}"
c.create_oval(FCX-r, FCY-r, FCX+r, FCY+r,
outline=f"#00{gh}ff", width=2)
for pr in self.pulse_r:
pa = max(0, int(220 * (1.0 - pr / (FW * 0.72))))
r = int(pr)
c.create_oval(FCX-r, FCY-r, FCX+r, FCY+r,
outline=self._ac(0, 212, 255, pa), width=2)
if self.muted:
c.create_oval(FCX-r, FCY-r, FCX+r, FCY+r,
outline=self._ac(255, 30, 80, pa // 3), width=2)
else:
c.create_oval(FCX-r, FCY-r, FCX+r, FCY+r,
outline=self._ac(0, 212, 255, pa), width=2)
for idx, (r_frac, w_ring, arc_l, gap) in enumerate([
(0.47, 3, 110, 75), (0.39, 2, 75, 55), (0.31, 1, 55, 38)]):
ring_r = int(FW * r_frac)
base_a = self.rings_spin[idx]
a_val = max(0, min(255, int(self.halo_a * (1.0 - idx * 0.18))))
col = self._ac(0, 212, 255, a_val)
col = self._ac(255, 30, 80, a_val) if self.muted else self._ac(0, 212, 255, a_val)
for s in range(360 // (arc_l + gap)):
start = (base_a + s * (arc_l + gap)) % 360
c.create_arc(FCX-ring_r, FCY-ring_r, FCX+ring_r, FCY+ring_r,
@@ -195,9 +334,10 @@ class JarvisUI:
sr = int(FW * 0.49)
scan_a = min(255, int(self.halo_a * 1.4))
arc_ext = 70 if self.speaking else 42
scan_col = self._ac(255, 30, 80, scan_a) if self.muted else self._ac(0, 212, 255, scan_a)
c.create_arc(FCX-sr, FCY-sr, FCX+sr, FCY+sr,
start=self.scan_angle, extent=arc_ext,
outline=self._ac(0, 212, 255, scan_a), width=3, style="arc")
outline=scan_col, width=3, style="arc")
c.create_arc(FCX-sr, FCY-sr, FCX+sr, FCY+sr,
start=self.scan2_angle, extent=arc_ext,
outline=self._ac(255, 100, 0, scan_a // 2), width=2, style="arc")
@@ -234,17 +374,27 @@ class JarvisUI:
if (self._face_scale_cache is None or
abs(self._face_scale_cache[0] - self.scale) > 0.004):
scaled = self._face_pil.resize((fw, fw), Image.BILINEAR)
if self.muted:
tinted = scaled.copy()
r_ch, g_ch, b_ch, a_ch = tinted.split()
from PIL import ImageEnhance
g_ch = ImageEnhance.Brightness(
Image.fromarray(__import__('numpy').array(g_ch) // 2)
).enhance(1.0) if False else g_ch
tk_img = ImageTk.PhotoImage(scaled)
self._face_scale_cache = (self.scale, tk_img)
c.create_image(FCX, FCY, image=self._face_scale_cache[1])
else:
orb_r = int(FW * 0.27 * self.scale)
orb_color = (255, 30, 80) if self.muted else (0, 65, 120)
for i in range(7, 0, -1):
r2 = int(orb_r * i / 7)
frac = i / 7
ga = max(0, min(255, int(self.halo_a * 1.1 * frac)))
c.create_oval(FCX-r2, FCY-r2, FCX+r2, FCY+r2,
fill=self._ac(0, int(65*frac), int(120*frac), ga),
fill=self._ac(int(orb_color[0]*frac),
int(orb_color[1]*frac),
int(orb_color[2]*frac), ga),
outline="")
c.create_text(FCX, FCY, text=SYSTEM_NAME,
fill=self._ac(0, 212, 255, min(255, int(self.halo_a * 2))),
@@ -257,18 +407,35 @@ class JarvisUI:
fill=C_PRI, font=("Courier", 18, "bold"))
c.create_text(W // 2, 44, text="Just A Rather Very Intelligent System",
fill=C_MID, font=("Courier", 9))
c.create_text(16, 31, text=MODEL_BADGE,
c.create_text(16, 31, text=MODEL_BADGE,
fill=C_DIM, font=("Courier", 9), anchor="w")
c.create_text(W - 16, 31, text=time.strftime("%H:%M:%S"),
fill=C_PRI, font=("Courier", 14, "bold"), anchor="e")
sy = FCY + FW // 2 + 45
if self.speaking:
stat, sc = "● SPEAKING", C_ACC
if self.muted:
stat = "⊘ MUTED"
sc = C_MUTED
elif self.speaking:
stat = "● SPEAKING"
sc = C_ACC
elif self._jarvis_state == "THINKING":
sym = "◈" if self.status_blink else "◇"
stat = f"{sym} THINKING"
sc = C_ACC2
elif self._jarvis_state == "PROCESSING":
sym = "▷" if self.status_blink else "▶"
stat = f"{sym} PROCESSING"
sc = C_ACC2
elif self._jarvis_state == "LISTENING":
sym = "●" if self.status_blink else "○"
stat = f"{sym} LISTENING"
sc = C_GREEN
else:
sym = "●" if self.status_blink else "○"
stat, sc = f"{sym} {self.status_text}", C_PRI
sym = "●" if self.status_blink else "○"
stat = f"{sym} {self.status_text}"
sc = C_PRI
c.create_text(W // 2, sy, text=stat,
fill=sc, font=("Courier", 11, "bold"))
@@ -280,36 +447,53 @@ class JarvisUI:
total_w = N * bw
wx0 = (W - total_w) // 2
for i in range(N):
hb = random.randint(3, BH) if self.speaking else int(3 + 2 * math.sin(t * 0.08 + i * 0.55))
col = (C_PRI if hb > BH * 0.6 else C_MID) if self.speaking else C_DIM
bx = wx0 + i * bw
if self.muted:
hb = 2
col = C_MUTED
elif self.speaking:
hb = random.randint(3, BH)
col = C_PRI if hb > BH * 0.6 else C_MID
else:
hb = int(3 + 2 * math.sin(t * 0.08 + i * 0.55))
col = C_DIM
bx = wx0 + i * bw
c.create_rectangle(bx, wy + BH - hb, bx + bw - 1, wy + BH,
fill=col, outline="")
c.create_rectangle(0, H - 28, W, H, fill="#00080d", outline="")
c.create_line(0, H - 28, W, H - 28, fill=C_DIM, width=1)
c.create_text(W - 16, H - 14, fill=C_DIM, font=("Courier", 8),
text="[F4] MUTE", anchor="e")
c.create_text(W // 2, H - 14, fill=C_DIM, font=("Courier", 8),
text="FatihMakes Industries · CLASSIFIED · MARK XXX")
text="FatihMakes Industries · CLASSIFIED · MARK XXXVII")
def write_log(self, text: str):
self.typing_queue.append(text)
tl = text.lower()
self.status_text = ("PROCESSING" if tl.startswith("you:")
else "RESPONDING" if tl.startswith("ai:")
else self.status_text)
if tl.startswith("you:"):
self.set_state("PROCESSING")
elif tl.startswith("jarvis:") or tl.startswith("ai:"):
self.set_state("SPEAKING")
if not self.is_typing:
self._start_typing()
def _start_typing(self):
if not self.typing_queue:
self.is_typing = False
if not self.speaking:
self.status_text = "ONLINE"
if not self.speaking and not self.muted:
self.set_state("LISTENING")
return
self.is_typing = True
text = self.typing_queue.popleft()
tl = text.lower()
tag = "you" if tl.startswith("you:") else "ai" if tl.startswith("ai:") else "sys"
if tl.startswith("you:"):
tag = "you"
elif tl.startswith("jarvis:") or tl.startswith("ai:"):
tag = "ai"
elif tl.startswith("err:") or "error" in tl or "failed" in tl:
tag = "err"
else:
tag = "sys"
self.log_text.configure(state="normal")
self._type_char(text, 0, tag)
@@ -324,57 +508,181 @@ class JarvisUI:
self.root.after(25, self._start_typing)
def start_speaking(self):
self.speaking = True
self.status_text = "SPEAKING"
self.set_state("SPEAKING")
def stop_speaking(self):
self.speaking = False
self.status_text = "ONLINE"
if not self.muted:
self.set_state("LISTENING")
def _api_keys_exist(self):
return API_FILE.exists()
def _api_keys_exist(self) -> bool:
if not API_FILE.exists():
return False
try:
data = json.loads(API_FILE.read_text(encoding="utf-8"))
return bool(data.get("gemini_api_key")) and bool(data.get("os_system"))
except Exception:
return False
def wait_for_api_key(self):
"""Block until API key is saved (called from main thread before starting JARVIS)."""
while not self._api_key_ready:
time.sleep(0.1)
@staticmethod
def _detect_os() -> str:
s = platform.system().lower()
if s == "darwin":
return "mac"
if s == "windows":
return "windows"
return "linux"
def _show_setup_ui(self):
detected = self._detect_os()
self._selected_os = tk.StringVar(value=detected)
self.setup_frame = tk.Frame(
self.root, bg="#00080d",
highlightbackground=C_PRI, highlightthickness=1
)
self.setup_frame.place(relx=0.5, rely=0.5, anchor="center")
tk.Label(self.setup_frame, text="◈ INITIALISATION REQUIRED",
fg=C_PRI, bg="#00080d", font=("Courier", 13, "bold")).pack(pady=(18, 4))
tk.Label(self.setup_frame,
text="Enter your Gemini API key to boot J.A.R.V.I.S.",
fg=C_MID, bg="#00080d", font=("Courier", 9)).pack(pady=(0, 10))
tk.Label(
self.setup_frame,
text="◈ INITIALISATION REQUIRED",
fg=C_PRI, bg="#00080d",
font=("Courier", 13, "bold")
).pack(pady=(18, 2))
tk.Label(
self.setup_frame,
text="Configure J.A.R.V.I.S. before first boot.",
fg=C_MID, bg="#00080d",
font=("Courier", 9)
).pack(pady=(0, 14))
tk.Label(
self.setup_frame,
text="GEMINI API KEY",
fg=C_DIM, bg="#00080d",
font=("Courier", 9)
).pack(pady=(0, 2))
tk.Label(self.setup_frame, text="GEMINI API KEY",
fg=C_DIM, bg="#00080d", font=("Courier", 9)).pack(pady=(8, 2))
self.gemini_entry = tk.Entry(
self.setup_frame, width=52, fg=C_TEXT, bg="#000d12",
insertbackground=C_TEXT, borderwidth=0, font=("Courier", 10), show="*"
self.setup_frame, width=52,
fg=C_TEXT, bg="#000d12",
insertbackground=C_TEXT,
borderwidth=0, font=("Courier", 10), show="*"
)
self.gemini_entry.pack(pady=(0, 4))
self.gemini_entry.pack(pady=(0, 18))
sep_frame = tk.Frame(self.setup_frame, bg="#00080d")
sep_frame.pack(fill="x", padx=24, pady=(0, 12))
tk.Frame(sep_frame, bg=C_DIM, height=1).pack(fill="x")
tk.Label(
self.setup_frame,
text="SELECT OPERATING SYSTEM",
fg=C_DIM, bg="#00080d",
font=("Courier", 9)
).pack(pady=(0, 4))
detect_label = {
"windows": "Windows",
"mac": "macOS",
"linux": "Linux",
}.get(detected, detected.capitalize())
tk.Label(
self.setup_frame,
text=f"AUTO-DETECTED: {detect_label}",
fg=C_ACC2, bg="#00080d",
font=("Courier", 8)
).pack(pady=(0, 8))
os_btn_frame = tk.Frame(self.setup_frame, bg="#00080d")
os_btn_frame.pack(pady=(0, 18))
os_options = [
("windows", "⊞ WINDOWS"),
("mac", " macOS"),
("linux", "🐧 LINUX"),
]
self._os_buttons = {}
for os_key, os_label in os_options:
btn = tk.Button(
os_btn_frame,
text=os_label,
width=13,
font=("Courier", 10, "bold"),
borderwidth=0,
cursor="hand2",
pady=7,
command=lambda k=os_key: self._select_os(k)
)
btn.pack(side="left", padx=6)
self._os_buttons[os_key] = btn
self._select_os(detected)
sep_frame2 = tk.Frame(self.setup_frame, bg="#00080d")
sep_frame2.pack(fill="x", padx=24, pady=(0, 14))
tk.Frame(sep_frame2, bg=C_DIM, height=1).pack(fill="x")
tk.Button(
self.setup_frame, text="▸ INITIALISE SYSTEMS",
command=self._save_api_keys, bg=C_BG, fg=C_PRI,
activebackground="#003344", font=("Courier", 10),
self.setup_frame,
text="▸ INITIALISE SYSTEMS",
command=self._save_api_keys,
bg=C_BG, fg=C_PRI,
activebackground="#003344",
font=("Courier", 10),
borderwidth=0, pady=8
).pack(pady=14)
).pack(pady=(0, 18))
def _select_os(self, os_key: str):
self._selected_os.set(os_key)
styles = {
"windows": (C_PRI, "#001a22"),
"mac": (C_ACC2, "#1a1500"),
"linux": (C_GREEN, "#001a0d"),
}
for key, btn in self._os_buttons.items():
if key == os_key:
fg, bg = styles[key]
btn.configure(
fg=bg, bg=fg,
activeforeground=bg, activebackground=fg,
relief="flat"
)
else:
btn.configure(
fg=C_DIM, bg="#000d12",
activeforeground=C_TEXT, activebackground="#001a22",
relief="flat"
)
def _save_api_keys(self):
gemini = self.gemini_entry.get().strip()
if not gemini:
self.gemini_entry.configure(highlightthickness=1,
highlightbackground=C_RED,
highlightcolor=C_RED)
return
os_system = self._selected_os.get()
os.makedirs(CONFIG_DIR, exist_ok=True)
with open(API_FILE, "w", encoding="utf-8") as f:
json.dump({"gemini_api_key": gemini}, f, indent=4)
json.dump(
{
"gemini_api_key": gemini,
"os_system": os_system,
},
f, indent=4
)
self.setup_frame.destroy()
self._api_key_ready = True
self.status_text = "ONLINE"
self.write_log("SYS: Systems initialised. JARVIS online.")
self.set_state("LISTENING")
self.write_log(f"SYS: Systems initialised. OS → {os_system.upper()}. JARVIS online.")