Add files via upload
This commit is contained in:
FatihMakes authored and 
GitHub committed
1 parent
234dd79273
commit
d929ef03cf
30 files changed
+2331
-1231
No files matched your search
@@ -1,4 +1,4 @@
|
||||
MARK 52 — JARVIS
|
||||
MARK 53 — JARVIS
|
||||
Copyright (c) 2026 FatihMakes
|
||||
|
||||
Licensed under the Creative Commons Attribution-NonCommercial 4.0
|
||||
|
||||
@@ -13,10 +13,10 @@ from pathlib import Path
|
||||
# ── Blocked categories (never monitor regardless of what user says) ────────────
|
||||
|
||||
_BLOCKED = {
|
||||
# Marka / varlık adları — her dilde aynı yazılır
|
||||
# Brand / asset names — spelled the same in every language
|
||||
"bitcoin", "ethereum", "dogecoin", "solana", "binance",
|
||||
"nft", "blockchain", "defi", "altcoin", "memecoin", "coin", "token",
|
||||
# "kripto" kökünün farklı dillerdeki yazılışları
|
||||
# spellings of the "crypto" root across different languages
|
||||
"crypto", "kripto", "cripto", "krypto", "крипто", "仮想通貨", "暗号資産",
|
||||
"cryptocurrency",
|
||||
}
|
||||
|
||||
+103
-31
@@ -369,12 +369,12 @@ _WIN_EXE_HINTS: dict[str, str] = {"chrome": "chrome", "edge": "msedge"}
|
||||
|
||||
def _open_native(url: str, browser_name: Optional[str]) -> str:
|
||||
"""
|
||||
Kullanıcının GERÇEK tarayıcısını normal şekilde açar — kendi profili,
|
||||
giriş yapılmış hesapları ve eklentileriyle. Otomasyon bağlanmaz, bu yüzden
|
||||
about:blank sekmesi veya boş profil ASLA görünmez.
|
||||
url boş ise tarayıcı URL'siz başlatılır (kendi açılış sayfası /
|
||||
oturum geri yükleme ile) — tıpkı kullanıcının kendisi açmış gibi.
|
||||
Windows / macOS / Linux üçünde de çalışır.
|
||||
Opens the user's REAL browser normally — with their own profile,
|
||||
logged-in accounts and extensions. No automation attaches, so an
|
||||
about:blank tab or a blank profile NEVER shows up.
|
||||
If url is empty the browser starts with no URL (its own start page /
|
||||
session restore) — exactly as if the user had opened it themselves.
|
||||
Works on all three of Windows / macOS / Linux.
|
||||
"""
|
||||
url = _normalize_url(url) if url and url.strip() else ""
|
||||
if url == "about:blank":
|
||||
@@ -384,7 +384,7 @@ def _open_native(url: str, browser_name: Optional[str]) -> str:
|
||||
if browser_name:
|
||||
name = _ALIASES.get(browser_name.lower().strip(), browser_name.lower().strip())
|
||||
elif not url:
|
||||
# URL yok → sadece pencere açılacak; varsayılan tarayıcının exe'si gerekir
|
||||
# No URL → only a window will open; needs the default browser's exe
|
||||
name = _detect_default_browser()
|
||||
|
||||
# Specific browser → launch its own executable, exactly like the user would.
|
||||
@@ -443,8 +443,8 @@ def _open_native(url: str, browser_name: Optional[str]) -> str:
|
||||
|
||||
class _BrowserSession:
|
||||
"""
|
||||
Bir tarayıcı örneği için tam oturum.
|
||||
Tüm tarayıcılar launch_persistent_context ile gerçek profil üzerinde açılır.
|
||||
A full session for one browser instance.
|
||||
All browsers open on the real profile via launch_persistent_context.
|
||||
"""
|
||||
|
||||
def __init__(self, browser_name: str):
|
||||
@@ -505,9 +505,9 @@ class _BrowserSession:
|
||||
|
||||
async def _adopt_page(self) -> Page:
|
||||
"""
|
||||
launch_persistent_context zaten bir başlangıç sekmesi açar.
|
||||
Yeni bir boş sekme (about:blank) açmak yerine o sekmeyi devralır —
|
||||
böylece kullanıcı fazladan boş sekme görmez.
|
||||
launch_persistent_context already opens a starting tab.
|
||||
Instead of opening a new blank tab (about:blank), it adopts that tab —
|
||||
so the user never sees an extra blank tab.
|
||||
"""
|
||||
await asyncio.sleep(0.3)
|
||||
pages = self._context.pages
|
||||
@@ -515,8 +515,8 @@ class _BrowserSession:
|
||||
|
||||
async def _launch(self):
|
||||
"""
|
||||
Tarayıcıyı gerçek kullanıcı profiliyle başlatır.
|
||||
Context zaten açıksa hiçbir şey yapmaz.
|
||||
Launches the browser with the real user profile.
|
||||
Does nothing if the context is already open.
|
||||
"""
|
||||
if self._context is not None:
|
||||
return
|
||||
@@ -607,10 +607,10 @@ class _BrowserSession:
|
||||
except Exception as e:
|
||||
print(f"[Browser] ⚠️ Real profile failed for {label}: {e}")
|
||||
|
||||
# Gerçek profil açılamadı (tarayıcı zaten açık / kilitli profil / yeni
|
||||
# Chrome sürümleri otomasyonla gerçek profili engelliyor). Kalıcı
|
||||
# JARVIS otomasyon profiline geçilir — buraya bir kez giriş yapılan
|
||||
# hesaplar sonraki oturumlarda da açık kalır.
|
||||
# The real profile could not be opened (browser already open / locked
|
||||
# profile / newer Chrome versions block the real profile under
|
||||
# automation). Fall back to a persistent JARVIS automation profile —
|
||||
# accounts logged in here once stay logged in on later sessions too.
|
||||
jarvis_profile = str(Path.home() / ".jarvis_profiles" / self.browser_name)
|
||||
Path(jarvis_profile).mkdir(parents=True, exist_ok=True)
|
||||
print(f"[Browser] Retrying with JARVIS profile: {jarvis_profile}")
|
||||
@@ -840,7 +840,7 @@ class _BrowserSession:
|
||||
return f"{self.browser_name} closed."
|
||||
|
||||
class _SessionRegistry:
|
||||
"""Tüm aktif tarayıcı oturumlarını yönetir."""
|
||||
"""Manages all active browser sessions."""
|
||||
|
||||
def __init__(self):
|
||||
self._sessions: dict[str, _BrowserSession] = {}
|
||||
@@ -849,7 +849,7 @@ class _SessionRegistry:
|
||||
self._last_native_url: str = ""
|
||||
|
||||
def has(self, browser_name: str | None = None) -> bool:
|
||||
"""Bu tarayıcı için (veya hiç) aktif bir otomasyon oturumu var mı?"""
|
||||
"""Is there an active automation session for this browser (or any)?"""
|
||||
with self._lock:
|
||||
if not browser_name:
|
||||
return bool(self._sessions)
|
||||
@@ -860,7 +860,7 @@ class _SessionRegistry:
|
||||
self._last_native_url = url
|
||||
|
||||
def pop_native_url(self) -> str:
|
||||
"""Son native açılan URL'yi bir kez döndürür (tekrarı önlemek için tüketilir)."""
|
||||
"""Returns the last natively-opened URL once (consumed to avoid repeats)."""
|
||||
url, self._last_native_url = self._last_native_url, ""
|
||||
return url
|
||||
|
||||
@@ -956,12 +956,12 @@ def browser_control(
|
||||
_log(player, result)
|
||||
return result
|
||||
|
||||
# ── Gezinme HER ZAMAN native ─────────────────────────────────────────────
|
||||
# go_to / search / new_tab siteyi kullanıcının kendi tarayıcısında açar —
|
||||
# kendi profili, giriş yapılmış hesapları ve açılış sayfasıyla; tıpkı
|
||||
# kullanıcının kendisi açmış gibi. about:blank'li kontrollü pencere burada
|
||||
# asla açılmaz. Tek istisna: hâlihazırda süren bir otomasyon akışı varsa
|
||||
# gezinme o pencerede devam eder (çok adımlı görevler bölünmesin diye).
|
||||
# ── Navigation is ALWAYS native ──────────────────────────────────────────
|
||||
# go_to / search / new_tab open the site in the user's own browser —
|
||||
# their own profile, logged-in accounts and start page; exactly as if the
|
||||
# user had opened it themselves. A controlled window with about:blank never
|
||||
# opens here. The only exception: if an automation flow is already running,
|
||||
# navigation continues in that window (so multi-step tasks aren't split).
|
||||
if action in ("go_to", "search", "new_tab"):
|
||||
if _registry.has(browser):
|
||||
sess = _registry.get(browser)
|
||||
@@ -993,10 +993,10 @@ def browser_control(
|
||||
_log(player, result)
|
||||
return result
|
||||
|
||||
# ── Etkileşimli aksiyonlar (tıklama/yazma/okuma…) ────────────────────────
|
||||
# Bunlar fiziksel olarak kontrol edilebilir bir tarayıcı gerektirir;
|
||||
# yalnızca burada otomasyon penceresi açılır ve açılır açılmaz kullanıcının
|
||||
# son gezindiği sayfaya gider — boş sayfada beklemez.
|
||||
# ── Interactive actions (click/type/read…) ───────────────────────────────
|
||||
# These require a physically controllable browser; the automation window
|
||||
# only opens here, and as soon as it opens it goes to the user's last
|
||||
# navigated page — it doesn't sit on a blank page.
|
||||
try:
|
||||
sess = _registry.get(browser)
|
||||
except Exception as e:
|
||||
@@ -1058,3 +1058,75 @@ def _log(player, text: str):
|
||||
print(f"[Browser] {short}")
|
||||
if player:
|
||||
player.write_log(f"[browser] {short[:60]}")
|
||||
|
||||
|
||||
# ── Tool declaration (auto-discovered by core/action_loader.py) ──────────────
|
||||
TOOL = {
|
||||
"name": "browser_control",
|
||||
"description": "Controls any web browser. Use for: opening websites, searching the web, clicking elements, filling forms, scrolling, screenshots, navigation, any web-based task. Simple open/search requests launch the user's own browser normally (their real profile and logged-in accounts); interactive actions (click, type, fill_form...) attach an automation browser. Always pass the 'browser' parameter when the user specifies a browser (e.g. 'open in Edge', 'use Firefox', 'open Chrome'). Multiple browsers can run simultaneously.",
|
||||
"parameters": {
|
||||
"type": "OBJECT",
|
||||
"properties": {
|
||||
"action": {
|
||||
"type": "STRING",
|
||||
"description": "go_to | search | click | type | scroll | fill_form | smart_click | smart_type | get_text | get_url | press | new_tab | close_tab | screenshot | back | forward | reload | switch | list_browsers | close | close_all"
|
||||
},
|
||||
"browser": {
|
||||
"type": "STRING",
|
||||
"description": "Target browser: chrome | edge | firefox | opera | operagx | brave | vivaldi | safari. Omit to use the currently active browser."
|
||||
},
|
||||
"url": {
|
||||
"type": "STRING",
|
||||
"description": "URL for go_to / new_tab action"
|
||||
},
|
||||
"query": {
|
||||
"type": "STRING",
|
||||
"description": "Search query for search action"
|
||||
},
|
||||
"engine": {
|
||||
"type": "STRING",
|
||||
"description": "Search engine: google | bing | duckduckgo | yandex (default: google)"
|
||||
},
|
||||
"selector": {
|
||||
"type": "STRING",
|
||||
"description": "CSS selector for click/type"
|
||||
},
|
||||
"text": {
|
||||
"type": "STRING",
|
||||
"description": "Text to click or type"
|
||||
},
|
||||
"description": {
|
||||
"type": "STRING",
|
||||
"description": "Element description for smart_click/smart_type"
|
||||
},
|
||||
"direction": {
|
||||
"type": "STRING",
|
||||
"description": "up | down for scroll"
|
||||
},
|
||||
"amount": {
|
||||
"type": "INTEGER",
|
||||
"description": "Scroll amount in pixels (default: 500)"
|
||||
},
|
||||
"key": {
|
||||
"type": "STRING",
|
||||
"description": "Key name for press action (e.g. Enter, Escape, F5)"
|
||||
},
|
||||
"path": {
|
||||
"type": "STRING",
|
||||
"description": "Save path for screenshot"
|
||||
},
|
||||
"incognito": {
|
||||
"type": "BOOLEAN",
|
||||
"description": "Open in private/incognito mode"
|
||||
},
|
||||
"clear_first": {
|
||||
"type": "BOOLEAN",
|
||||
"description": "Clear field before typing (default: true)"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"action"
|
||||
]
|
||||
},
|
||||
"handler": browser_control,
|
||||
}
|
||||
+53
-5
@@ -115,10 +115,10 @@ _VALID_INTENTS = {"write", "edit", "explain", "run", "build", "screen_debug", "o
|
||||
|
||||
def _detect_intent(description: str, file_path: str, code: str) -> str:
|
||||
"""
|
||||
Dil bağımsız niyet tespiti — sabit anahtar kelime listesi YOK.
|
||||
Kullanıcı hangi dilde konuşursa konuşsun, açıklama Gemini'ye
|
||||
sınıflandırtılır. API'ye ulaşılamazsa dile bakmayan yapısal
|
||||
ipuçlarına (dosya diskte var mı, kod verilmiş mi) düşülür.
|
||||
Language-independent intent detection — NO fixed keyword list.
|
||||
Whatever language the user speaks, the description is classified by
|
||||
Gemini. If the API is unreachable, it falls back to language-agnostic
|
||||
structural hints (does the file exist on disk, was code provided).
|
||||
"""
|
||||
desc = (description or "").strip()
|
||||
file_exists = bool(file_path) and Path(file_path).exists()
|
||||
@@ -152,7 +152,7 @@ def _detect_intent(description: str, file_path: str, code: str) -> str:
|
||||
except Exception as e:
|
||||
print(f"[Code] Intent classification failed ({e}) — structural fallback")
|
||||
|
||||
# Yapısal geri dönüş — hiçbir dile bağlı değil
|
||||
# Structural fallback — not tied to any language
|
||||
if file_exists:
|
||||
return "edit" if desc else "explain"
|
||||
if code:
|
||||
@@ -582,3 +582,51 @@ def code_helper(
|
||||
|
||||
else:
|
||||
return f"Unknown action: '{action}'. Use write, edit, explain, run, build, optimize, or screen_debug."
|
||||
|
||||
|
||||
# ── Tool declaration (auto-discovered by core/action_loader.py) ──────────────
|
||||
TOOL = {
|
||||
"name": "code_helper",
|
||||
"description": "Writes, edits, explains, runs, or builds code files.",
|
||||
"parameters": {
|
||||
"type": "OBJECT",
|
||||
"properties": {
|
||||
"action": {
|
||||
"type": "STRING",
|
||||
"description": "write | edit | explain | run | build | auto (default: auto)"
|
||||
},
|
||||
"description": {
|
||||
"type": "STRING",
|
||||
"description": "What the code should do or what change to make"
|
||||
},
|
||||
"language": {
|
||||
"type": "STRING",
|
||||
"description": "Programming language (default: python)"
|
||||
},
|
||||
"output_path": {
|
||||
"type": "STRING",
|
||||
"description": "Where to save the file"
|
||||
},
|
||||
"file_path": {
|
||||
"type": "STRING",
|
||||
"description": "Path to existing file for edit/explain/run/build"
|
||||
},
|
||||
"code": {
|
||||
"type": "STRING",
|
||||
"description": "Raw code string for explain"
|
||||
},
|
||||
"args": {
|
||||
"type": "STRING",
|
||||
"description": "CLI arguments for run/build"
|
||||
},
|
||||
"timeout": {
|
||||
"type": "INTEGER",
|
||||
"description": "Execution timeout in seconds (default: 30)"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"action"
|
||||
]
|
||||
},
|
||||
"handler": code_helper,
|
||||
}
|
||||
@@ -512,3 +512,79 @@ def computer_control(
|
||||
except Exception as e:
|
||||
print(f"[ComputerControl] ❌ {action}: {e}")
|
||||
return f"computer_control '{action}' failed: {e}"
|
||||
|
||||
|
||||
# ── Tool declaration (auto-discovered by core/action_loader.py) ──────────────
|
||||
TOOL = {
|
||||
"name": "computer_control",
|
||||
"description": "Direct computer control: type, click, hotkeys, scroll, move mouse, screenshots, find elements on screen.",
|
||||
"parameters": {
|
||||
"type": "OBJECT",
|
||||
"properties": {
|
||||
"action": {
|
||||
"type": "STRING",
|
||||
"description": "type | smart_type | click | double_click | right_click | hotkey | press | scroll | move | copy | paste | screenshot | wait | clear_field | focus_window | screen_find | screen_click | random_data | user_data"
|
||||
},
|
||||
"text": {
|
||||
"type": "STRING",
|
||||
"description": "Text to type or paste"
|
||||
},
|
||||
"x": {
|
||||
"type": "INTEGER",
|
||||
"description": "X coordinate"
|
||||
},
|
||||
"y": {
|
||||
"type": "INTEGER",
|
||||
"description": "Y coordinate"
|
||||
},
|
||||
"keys": {
|
||||
"type": "STRING",
|
||||
"description": "Key combination e.g. 'ctrl+c'"
|
||||
},
|
||||
"key": {
|
||||
"type": "STRING",
|
||||
"description": "Single key e.g. 'enter'"
|
||||
},
|
||||
"direction": {
|
||||
"type": "STRING",
|
||||
"description": "up | down | left | right"
|
||||
},
|
||||
"amount": {
|
||||
"type": "INTEGER",
|
||||
"description": "Scroll amount (default: 3)"
|
||||
},
|
||||
"seconds": {
|
||||
"type": "NUMBER",
|
||||
"description": "Seconds to wait"
|
||||
},
|
||||
"title": {
|
||||
"type": "STRING",
|
||||
"description": "Window title for focus_window"
|
||||
},
|
||||
"description": {
|
||||
"type": "STRING",
|
||||
"description": "Element description for screen_find/screen_click"
|
||||
},
|
||||
"type": {
|
||||
"type": "STRING",
|
||||
"description": "Data type for random_data"
|
||||
},
|
||||
"field": {
|
||||
"type": "STRING",
|
||||
"description": "Field for user_data: name|email|city"
|
||||
},
|
||||
"clear_first": {
|
||||
"type": "BOOLEAN",
|
||||
"description": "Clear field before typing (default: true)"
|
||||
},
|
||||
"path": {
|
||||
"type": "STRING",
|
||||
"description": "Save path for screenshot"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"action"
|
||||
]
|
||||
},
|
||||
"handler": computer_control,
|
||||
}
|
||||
@@ -906,3 +906,57 @@ def computer_settings(
|
||||
lambda: (dark_mode(), "Theme switched back.")[1])
|
||||
|
||||
return f"Done: {action}."
|
||||
|
||||
|
||||
# ── Tool declaration (auto-discovered by core/action_loader.py) ──────────────
|
||||
TOOL = {
|
||||
"name": "computer_settings",
|
||||
"description": "Controls the computer: volume, brightness, window management, keyboard shortcuts, typing text on screen, closing apps, fullscreen, dark mode, WiFi, restart, shutdown, scrolling, tab management, zoom, screenshots, lock screen, refresh/reload page. Use for ANY single computer control command. restart, shutdown and toggle_wifi put a confirmation on the user's screen and do NOT happen until they press it — never claim they are done. Volume, brightness and dark mode can be reversed with the `undo` tool.",
|
||||
"parameters": {
|
||||
"type": "OBJECT",
|
||||
"properties": {
|
||||
# The exact vocabulary, spelled out.
|
||||
#
|
||||
# This used to say only "The action to perform", so the model
|
||||
# usually filled `description` instead — and computer_settings then
|
||||
# made a SECOND Gemini call, inside the tool, purely to translate
|
||||
# that sentence into one of these names. Every "turn the volume
|
||||
# down" cost two model round trips.
|
||||
#
|
||||
# Naming the actions here costs ~600 characters once per session and
|
||||
# removes a whole round trip from every computer command.
|
||||
"action": {
|
||||
"type": "STRING",
|
||||
"description": (
|
||||
"The exact action. Prefer this over `description` — pick one of: "
|
||||
"volume_up | volume_down | volume_set | mute | "
|
||||
"brightness_up | brightness_down | sleep_display | "
|
||||
"pause_video | close_app | close_window | full_screen | "
|
||||
"minimize | maximize | snap_left | snap_right | "
|
||||
"switch_window | show_desktop | task_manager | focus_search | "
|
||||
"refresh_page | close_tab | new_tab | next_tab | prev_tab | "
|
||||
"go_back | go_forward | zoom_in | zoom_out | zoom_reset | "
|
||||
"find_on_page | scroll_up | scroll_down | scroll_top | "
|
||||
"scroll_bottom | page_up | page_down | copy | paste | cut | "
|
||||
"undo | redo | select_all | save | enter | escape | press_key | "
|
||||
"type_text | screenshot | lock_screen | open_settings | "
|
||||
"file_explorer | open_run | dark_mode | toggle_wifi | "
|
||||
"restart | shutdown"
|
||||
)
|
||||
},
|
||||
"description": {
|
||||
"type": "STRING",
|
||||
"description": (
|
||||
"Fallback only, when no action name above fits. "
|
||||
"Resolved locally — no extra model call."
|
||||
)
|
||||
},
|
||||
"value": {
|
||||
"type": "STRING",
|
||||
"description": "Optional value: volume level 0-100, text to type, key name, etc."
|
||||
}
|
||||
},
|
||||
"required": []
|
||||
},
|
||||
"handler": computer_settings,
|
||||
}
|
||||
@@ -478,3 +478,39 @@ def desktop_control(
|
||||
except Exception as e:
|
||||
print(f"[Desktop] Error: {e}")
|
||||
return f"Desktop control error: {e}"
|
||||
|
||||
|
||||
# ── Tool declaration (auto-discovered by core/action_loader.py) ──────────────
|
||||
TOOL = {
|
||||
"name": "desktop_control",
|
||||
"description": "Controls the desktop: wallpaper, organize, clean, list, stats.",
|
||||
"parameters": {
|
||||
"type": "OBJECT",
|
||||
"properties": {
|
||||
"action": {
|
||||
"type": "STRING",
|
||||
"description": "wallpaper | wallpaper_url | organize | clean | list | stats | task"
|
||||
},
|
||||
"path": {
|
||||
"type": "STRING",
|
||||
"description": "Image path for wallpaper"
|
||||
},
|
||||
"url": {
|
||||
"type": "STRING",
|
||||
"description": "Image URL for wallpaper_url"
|
||||
},
|
||||
"mode": {
|
||||
"type": "STRING",
|
||||
"description": "by_type or by_date for organize"
|
||||
},
|
||||
"task": {
|
||||
"type": "STRING",
|
||||
"description": "Natural language desktop task"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"action"
|
||||
]
|
||||
},
|
||||
"handler": desktop_control,
|
||||
}
|
||||
+33
-1
@@ -324,7 +324,7 @@ def _run_project(run_command: str, project_dir: Path, timeout: int = 30) -> str:
|
||||
return f"Run error: {e}"
|
||||
|
||||
def _try_auto_install(error_output: str, project_dir: Path) -> bool:
|
||||
"""ModuleNotFoundError varsa eksik paketi otomatik kurmaya çalışır."""
|
||||
"""If there is a ModuleNotFoundError, tries to auto-install the missing package."""
|
||||
pattern = re.compile(
|
||||
r"No module named ['\"]([a-zA-Z0-9_\-\.]+)['\"]", re.IGNORECASE
|
||||
)
|
||||
@@ -600,3 +600,35 @@ def dev_agent(
|
||||
speak = speak,
|
||||
player = player,
|
||||
)
|
||||
|
||||
|
||||
# ── Tool declaration (auto-discovered by core/action_loader.py) ──────────────
|
||||
TOOL = {
|
||||
"name": "dev_agent",
|
||||
"description": "Builds complete multi-file projects from scratch: plans, writes files, installs deps, opens VSCode, runs and fixes errors.",
|
||||
"parameters": {
|
||||
"type": "OBJECT",
|
||||
"properties": {
|
||||
"description": {
|
||||
"type": "STRING",
|
||||
"description": "What the project should do"
|
||||
},
|
||||
"language": {
|
||||
"type": "STRING",
|
||||
"description": "Programming language (default: python)"
|
||||
},
|
||||
"project_name": {
|
||||
"type": "STRING",
|
||||
"description": "Optional project folder name"
|
||||
},
|
||||
"timeout": {
|
||||
"type": "INTEGER",
|
||||
"description": "Run timeout in seconds (default: 30)"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"description"
|
||||
]
|
||||
},
|
||||
"handler": dev_agent,
|
||||
}
|
||||
+53
-18
@@ -95,7 +95,7 @@ _SAFE_ROOTS: list[Path] = [
|
||||
]
|
||||
|
||||
def _is_safe_path(target: Path) -> bool:
|
||||
"""Verilen path _SAFE_ROOTS içinde mi? Değilse işlemi reddet."""
|
||||
"""Is the given path inside _SAFE_ROOTS? If not, reject the operation."""
|
||||
try:
|
||||
resolved = target.resolve()
|
||||
return any(
|
||||
@@ -158,22 +158,9 @@ def _resolve_path(raw: str) -> Path:
|
||||
"videos": _get_videos(),
|
||||
"home": Path.home(),
|
||||
}
|
||||
raw = raw.strip().strip('"').strip("'")
|
||||
lower = raw.lower()
|
||||
lower = raw.strip().lower()
|
||||
if lower in shortcuts:
|
||||
return shortcuts[lower]
|
||||
|
||||
# "desktop/notes/a.md" and "desktop\notes\a.md" — a shortcut followed by a
|
||||
# sub-path. Without this branch the whole string falls through to the
|
||||
# relative-path return below and is resolved against the process CWD instead
|
||||
# of the real Desktop: an "Access denied" when the project lives outside the
|
||||
# home directory, or — worse — a silent write into a stray "desktop" folder
|
||||
# inside the project when it lives inside it.
|
||||
head, sep, rest = raw.replace("\\", "/").partition("/")
|
||||
if sep and head.lower() in shortcuts:
|
||||
rest = rest.strip("/")
|
||||
return shortcuts[head.lower()] / rest if rest else shortcuts[head.lower()]
|
||||
|
||||
return Path(raw).expanduser()
|
||||
|
||||
def _format_size(b: int) -> str:
|
||||
@@ -275,7 +262,7 @@ def delete_file(path: str, name: str = "") -> str:
|
||||
if not target.exists():
|
||||
return f"Not found: {target.name}"
|
||||
|
||||
# Güvenli dizin kontrolü — kritik kullanıcı klasörlerini koru
|
||||
# Safe-directory check — protect critical user folders
|
||||
protected = {
|
||||
_get_desktop(), _get_downloads(), _get_documents(),
|
||||
_get_pictures(), _get_music(), _get_videos(), Path.home()
|
||||
@@ -461,7 +448,7 @@ def find_files(name: str = "", extension: str = "",
|
||||
|
||||
results = []
|
||||
dir_count = 0
|
||||
max_dirs = 500 # performans + güvenlik limiti
|
||||
max_dirs = 500 # performance + safety limit
|
||||
|
||||
for item in search_path.rglob("*"):
|
||||
if item.is_dir():
|
||||
@@ -556,7 +543,7 @@ def organize_desktop() -> str:
|
||||
|
||||
try:
|
||||
for item in desktop.iterdir():
|
||||
# Klasörlere, gizli dosyalara ve organize klasörlerine dokunma
|
||||
# Leave folders, hidden files and organize-folders untouched
|
||||
if item.is_dir() or item.name.startswith("."):
|
||||
continue
|
||||
if item.name in {k for k in type_map}:
|
||||
@@ -719,3 +706,51 @@ def file_controller(
|
||||
|
||||
except Exception as e:
|
||||
return f"File controller error ({action}): {e}"
|
||||
|
||||
|
||||
# ── Tool declaration (auto-discovered by core/action_loader.py) ──────────────
|
||||
TOOL = {
|
||||
"name": "file_controller",
|
||||
"description": "Manages files and folders: list, create, delete, move, copy, rename, read, write, find, disk usage.",
|
||||
"parameters": {
|
||||
"type": "OBJECT",
|
||||
"properties": {
|
||||
"action": {
|
||||
"type": "STRING",
|
||||
"description": "list | create_file | create_folder | delete | move | copy | rename | read | write | find | largest | disk_usage | organize_desktop | info"
|
||||
},
|
||||
"path": {
|
||||
"type": "STRING",
|
||||
"description": "File/folder path or shortcut: desktop, downloads, documents, home"
|
||||
},
|
||||
"destination": {
|
||||
"type": "STRING",
|
||||
"description": "Destination path for move/copy"
|
||||
},
|
||||
"new_name": {
|
||||
"type": "STRING",
|
||||
"description": "New name for rename"
|
||||
},
|
||||
"content": {
|
||||
"type": "STRING",
|
||||
"description": "Content for create_file/write"
|
||||
},
|
||||
"name": {
|
||||
"type": "STRING",
|
||||
"description": "File name to search for"
|
||||
},
|
||||
"extension": {
|
||||
"type": "STRING",
|
||||
"description": "File extension to search (e.g. .pdf)"
|
||||
},
|
||||
"count": {
|
||||
"type": "INTEGER",
|
||||
"description": "Number of results for largest"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"action"
|
||||
]
|
||||
},
|
||||
"handler": file_controller,
|
||||
}
|
||||
@@ -834,3 +834,85 @@ def file_processor(parameters: dict, player=None, speak=None) -> str:
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
return f"Processing failed: {e}"
|
||||
|
||||
|
||||
# ── Tool declaration (auto-discovered by core/action_loader.py) ──────────────
|
||||
TOOL = {
|
||||
"name": "file_processor",
|
||||
"description": "Processes any file that the user has uploaded or dropped onto the interface. Use this when the user refers to an uploaded file and wants an action on it. Supports: images (describe/ocr/resize/compress/convert), PDFs (summarize/extract_text/to_word), Word docs & text files (summarize/fix/reformat/translate), CSV/Excel (analyze/stats/filter/sort/convert), JSON/XML (validate/format/analyze), code files (explain/review/fix/optimize/run/document/test), audio (transcribe/trim/convert/info), video (trim/extract_audio/extract_frame/compress/transcribe/info), archives (list/extract), presentations (summarize/extract_text). ALWAYS call this tool when a file has been uploaded and the user gives a command about it. If the user's command is ambiguous, pick the most logical action for that file type.",
|
||||
"parameters": {
|
||||
"type": "OBJECT",
|
||||
"properties": {
|
||||
"file_path": {
|
||||
"type": "STRING",
|
||||
"description": "Full path to the uploaded file. Leave empty to use the currently uploaded file."
|
||||
},
|
||||
"action": {
|
||||
"type": "STRING",
|
||||
"description": "What to do with the file. Examples by type:\nimage: describe | ocr | resize | compress | convert | info\npdf: summarize | extract_text | to_word | info\ndocx/txt: summarize | fix | reformat | translate_hint | word_count | to_bullet\ncsv/excel: analyze | stats | filter | sort | convert | info\njson: validate | format | analyze | to_csv\ncode: explain | review | fix | optimize | run | document | test\naudio: transcribe | trim | convert | info\nvideo: trim | extract_audio | extract_frame | compress | transcribe | info | convert\narchive: list | extract\npptx: summarize | extract_text | analyze"
|
||||
},
|
||||
"instruction": {
|
||||
"type": "STRING",
|
||||
"description": "Free-form instruction if action doesn't cover it. E.g. 'translate this to Turkish', 'find all email addresses'"
|
||||
},
|
||||
"format": {
|
||||
"type": "STRING",
|
||||
"description": "Target format for conversion. E.g. 'mp3', 'pdf', 'csv', 'png'"
|
||||
},
|
||||
"width": {
|
||||
"type": "INTEGER",
|
||||
"description": "Target width for image resize"
|
||||
},
|
||||
"height": {
|
||||
"type": "INTEGER",
|
||||
"description": "Target height for image resize"
|
||||
},
|
||||
"scale": {
|
||||
"type": "NUMBER",
|
||||
"description": "Scale factor for image resize (e.g. 0.5)"
|
||||
},
|
||||
"quality": {
|
||||
"type": "INTEGER",
|
||||
"description": "Quality 1-100 for image/video compress"
|
||||
},
|
||||
"start": {
|
||||
"type": "STRING",
|
||||
"description": "Start time for trim: seconds or HH:MM:SS"
|
||||
},
|
||||
"end": {
|
||||
"type": "STRING",
|
||||
"description": "End time for trim: seconds or HH:MM:SS"
|
||||
},
|
||||
"timestamp": {
|
||||
"type": "STRING",
|
||||
"description": "Timestamp for video frame extraction HH:MM:SS"
|
||||
},
|
||||
"column": {
|
||||
"type": "STRING",
|
||||
"description": "Column name for CSV filter/sort"
|
||||
},
|
||||
"value": {
|
||||
"type": "STRING",
|
||||
"description": "Filter value for CSV filter"
|
||||
},
|
||||
"condition": {
|
||||
"type": "STRING",
|
||||
"description": "Filter condition: equals|contains|gt|lt"
|
||||
},
|
||||
"ascending": {
|
||||
"type": "BOOLEAN",
|
||||
"description": "Sort order for CSV sort (default: true)"
|
||||
},
|
||||
"save": {
|
||||
"type": "BOOLEAN",
|
||||
"description": "Save result to file (default: true)"
|
||||
},
|
||||
"destination": {
|
||||
"type": "STRING",
|
||||
"description": "Output folder for archive extract"
|
||||
}
|
||||
},
|
||||
"required": []
|
||||
},
|
||||
"handler": file_processor,
|
||||
}
|
||||
@@ -27,14 +27,13 @@ _MONTH_MAP: dict[str, int] = {
|
||||
"january": 1, "february": 2, "march": 3, "april": 4,
|
||||
"may": 5, "june": 6, "july": 7, "august": 8,
|
||||
"september": 9, "october": 10, "november": 11, "december": 12,
|
||||
"ocak": 1, "şubat": 2, "mart": 3, "nisan": 4,
|
||||
"mayıs": 5, "haziran": 6, "temmuz": 7, "ağustos": 8,
|
||||
"eylül": 9, "ekim": 10, "kasım": 11, "aralık": 12,
|
||||
}
|
||||
|
||||
# English fast-path only — Gemini (below) normalizes date expressions in ANY
|
||||
# language to YYYY-MM-DD, so no other language needs to be hardcoded here.
|
||||
_RELATIVE_MAP_KEYS = {
|
||||
"today", "bugün",
|
||||
"tomorrow", "yarın",
|
||||
"today",
|
||||
"tomorrow",
|
||||
}
|
||||
|
||||
|
||||
@@ -53,9 +52,8 @@ def _parse_date(raw: str) -> str:
|
||||
pass
|
||||
|
||||
relative = {
|
||||
"today": today, "bugün": today,
|
||||
"today": today,
|
||||
"tomorrow": today + timedelta(days=1),
|
||||
"yarın": today + timedelta(days=1),
|
||||
}
|
||||
for key, val in relative.items():
|
||||
if key in lower:
|
||||
@@ -362,3 +360,49 @@ def flight_finder(parameters: dict, player=None, speak=None) -> str:
|
||||
except Exception as e:
|
||||
print(f"[FlightFinder] ❌ {e}")
|
||||
return f"Flight search failed, sir: {e}"
|
||||
|
||||
|
||||
# ── Tool declaration (auto-discovered by core/action_loader.py) ──────────────
|
||||
TOOL = {
|
||||
"name": "flight_finder",
|
||||
"description": "Searches Google Flights and speaks the best options.",
|
||||
"parameters": {
|
||||
"type": "OBJECT",
|
||||
"properties": {
|
||||
"origin": {
|
||||
"type": "STRING",
|
||||
"description": "Departure city or airport code"
|
||||
},
|
||||
"destination": {
|
||||
"type": "STRING",
|
||||
"description": "Arrival city or airport code"
|
||||
},
|
||||
"date": {
|
||||
"type": "STRING",
|
||||
"description": "Departure date (any format)"
|
||||
},
|
||||
"return_date": {
|
||||
"type": "STRING",
|
||||
"description": "Return date for round trips"
|
||||
},
|
||||
"passengers": {
|
||||
"type": "INTEGER",
|
||||
"description": "Number of passengers (default: 1)"
|
||||
},
|
||||
"cabin": {
|
||||
"type": "STRING",
|
||||
"description": "economy | premium | business | first"
|
||||
},
|
||||
"save": {
|
||||
"type": "BOOLEAN",
|
||||
"description": "Save results to Notepad"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"origin",
|
||||
"destination",
|
||||
"date"
|
||||
]
|
||||
},
|
||||
"handler": flight_finder,
|
||||
}
|
||||
+65
-18
@@ -202,7 +202,7 @@ def _click_first_profile_by_screenshot() -> bool:
|
||||
time.sleep(1.5)
|
||||
win = _get_steam_window_rect()
|
||||
if not win:
|
||||
print("[GameUpdater] ⚠️ Steam penceresi bulunamadı")
|
||||
print("[GameUpdater] ⚠️ Steam window not found")
|
||||
return False
|
||||
|
||||
wx, wy, ww, wh = win
|
||||
@@ -220,7 +220,7 @@ def _click_first_profile_by_screenshot() -> bool:
|
||||
colorful = (max_c > 60) & ((max_c - min_c) > 40)
|
||||
|
||||
if not colorful.any():
|
||||
print("[GameUpdater] ⚠️ Avatar rengi bulunamadı — tahminle tıklanıyor")
|
||||
print("[GameUpdater] ⚠️ Avatar colour not found — clicking by guess")
|
||||
pyautogui.click(wx + ww // 2 - ww // 6, wy + wh // 2)
|
||||
return True
|
||||
|
||||
@@ -235,20 +235,20 @@ def _click_first_profile_by_screenshot() -> bool:
|
||||
|
||||
abs_x = wx + search_x1 + int(block_cols.mean())
|
||||
abs_y = wy + search_y1 + int(rows.mean())
|
||||
print(f"[GameUpdater] 🎯 Profil avatarı ({abs_x}, {abs_y}) — tıklanıyor")
|
||||
print(f"[GameUpdater] 🎯 Profile avatar ({abs_x}, {abs_y}) — clicking")
|
||||
pyautogui.click(abs_x, abs_y)
|
||||
return True
|
||||
|
||||
except ImportError as e:
|
||||
print(f"[GameUpdater] ⚠️ Eksik kütüphane: {e}")
|
||||
print(f"[GameUpdater] ⚠️ Missing library: {e}")
|
||||
return False
|
||||
except Exception as e:
|
||||
print(f"[GameUpdater] ⚠️ Profil tespiti başarısız: {e}")
|
||||
print(f"[GameUpdater] ⚠️ Profile detection failed: {e}")
|
||||
return False
|
||||
|
||||
|
||||
def _handle_steam_profile_selection() -> bool:
|
||||
print("[GameUpdater] 🔍 Profil seçim dialogu kontrol ediliyor...")
|
||||
print("[GameUpdater] 🔍 Checking profile-selection dialog...")
|
||||
win = _get_steam_window_rect()
|
||||
if not win:
|
||||
return False
|
||||
@@ -266,14 +266,14 @@ def _handle_steam_profile_selection() -> bool:
|
||||
(top_region[:,:,2] > 200)
|
||||
))
|
||||
if not is_small and white_pixels <= 100:
|
||||
print("[GameUpdater] ℹ️ Profil dialogu yok — Steam zaten giriş yapmış")
|
||||
print("[GameUpdater] ℹ️ No profile dialog — Steam is already logged in")
|
||||
return False
|
||||
except ImportError:
|
||||
pass
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
print("[GameUpdater] 👤 Profil seçimi tespit edildi — ilk profile tıklanıyor")
|
||||
print("[GameUpdater] 👤 Profile selection detected — clicking the first profile")
|
||||
return _click_first_profile_by_screenshot()
|
||||
|
||||
def _find_best_drive() -> dict | None:
|
||||
@@ -298,7 +298,7 @@ def _select_drive_in_dialog(dialog, drive_letter: str) -> bool:
|
||||
for ctrl in dialog.descendants(control_type=control_type):
|
||||
if target in ctrl.window_text().upper():
|
||||
ctrl.click_input()
|
||||
print(f"[GameUpdater] ✅ Sürücü seçildi ({control_type}): {ctrl.window_text()}")
|
||||
print(f"[GameUpdater] ✅ Drive selected ({control_type}): {ctrl.window_text()}")
|
||||
return True
|
||||
except Exception:
|
||||
continue
|
||||
@@ -390,12 +390,17 @@ def _handle_install_dialog(game_name: str) -> str:
|
||||
|
||||
drive_letter = best_drive["letter"]
|
||||
drive_label = f"{drive_letter}:"
|
||||
print(f"[GameUpdater] 🏆 Hedef sürücü: {drive_label} ({best_drive['free_gb']:.1f} GB boş)")
|
||||
print(f"[GameUpdater] 🏆 Target drive: {drive_label} ({best_drive['free_gb']:.1f} GB free)")
|
||||
|
||||
try:
|
||||
from pywinauto import Application, findwindows
|
||||
dialog = None
|
||||
|
||||
# NOTE: this drives Steam's native installer dialog by matching its
|
||||
# on-screen button/title text, which is localized. We match English and
|
||||
# Turkish labels (install/yükle, next/ileri, ok/tamam). On a Steam client
|
||||
# set to another language the match simply fails and the automation
|
||||
# reports it — it never crashes, and manual install still works.
|
||||
for _ in range(40):
|
||||
time.sleep(0.5)
|
||||
try:
|
||||
@@ -423,7 +428,7 @@ def _handle_install_dialog(game_name: str) -> str:
|
||||
break
|
||||
|
||||
if not dialog:
|
||||
raise RuntimeError("Dialog bulunamadı")
|
||||
raise RuntimeError("Dialog not found")
|
||||
|
||||
dialog.set_focus()
|
||||
time.sleep(0.4)
|
||||
@@ -440,7 +445,7 @@ def _handle_install_dialog(game_name: str) -> str:
|
||||
except ImportError:
|
||||
return _handle_install_dialog_pyautogui(game_name, best_drive)
|
||||
except Exception as e:
|
||||
print(f"[GameUpdater] ⚠️ pywinauto başarısız: {e}")
|
||||
print(f"[GameUpdater] ⚠️ pywinauto failed: {e}")
|
||||
return _handle_install_dialog_pyautogui(game_name, best_drive)
|
||||
|
||||
def _ensure_steam_running(steam_path: Path) -> bool:
|
||||
@@ -449,10 +454,10 @@ def _ensure_steam_running(steam_path: Path) -> bool:
|
||||
|
||||
exe = _steam_exe(steam_path)
|
||||
if not exe.exists():
|
||||
print(f"[GameUpdater] ❌ Steam bulunamadı: {exe}")
|
||||
print(f"[GameUpdater] ❌ Steam not found: {exe}")
|
||||
return False
|
||||
|
||||
print("[GameUpdater] 🚀 Steam başlatılıyor...")
|
||||
print("[GameUpdater] 🚀 Starting Steam...")
|
||||
if is_mac():
|
||||
subprocess.Popen(["open", "-a", "Steam"])
|
||||
else:
|
||||
@@ -461,14 +466,14 @@ def _ensure_steam_running(steam_path: Path) -> bool:
|
||||
for _ in range(20):
|
||||
time.sleep(1)
|
||||
if _is_steam_running():
|
||||
print("[GameUpdater] ✅ Steam çalışıyor")
|
||||
print("[GameUpdater] ✅ Steam is running")
|
||||
time.sleep(4)
|
||||
if is_windows():
|
||||
_handle_steam_profile_selection()
|
||||
time.sleep(2)
|
||||
return True
|
||||
|
||||
print("[GameUpdater] ⚠️ Steam başlatılamadı")
|
||||
print("[GameUpdater] ⚠️ Could not start Steam")
|
||||
return False
|
||||
|
||||
def _search_steam_appid(game_name: str) -> tuple[str | None, str | None]:
|
||||
@@ -488,7 +493,7 @@ def _search_steam_appid(game_name: str) -> tuple[str | None, str | None]:
|
||||
|
||||
for key, (app_id, canonical) in _KNOWN_APPIDS.items():
|
||||
if name_lower in key or key in name_lower:
|
||||
print(f"[GameUpdater] 📖 Kısmi eşleşme: {canonical} ({app_id})")
|
||||
print(f"[GameUpdater] 📖 Partial match: {canonical} ({app_id})")
|
||||
return app_id, canonical
|
||||
|
||||
try:
|
||||
@@ -503,7 +508,7 @@ def _search_steam_appid(game_name: str) -> tuple[str | None, str | None]:
|
||||
print(f"[GameUpdater] 🌐 Store API: {best['name']} ({best['id']})")
|
||||
return str(best["id"]), best["name"]
|
||||
except Exception as e:
|
||||
print(f"[GameUpdater] ⚠️ AppID arama başarısız: {e}")
|
||||
print(f"[GameUpdater] ⚠️ AppID lookup failed: {e}")
|
||||
|
||||
return None, None
|
||||
|
||||
@@ -1052,3 +1057,45 @@ if __name__ == "__main__":
|
||||
print(f"[GameUpdater] 🕐 Scheduled run at {datetime.now().strftime('%H:%M')}")
|
||||
result = game_updater({"action": "update", "platform": "both"})
|
||||
print(f"[GameUpdater] ✅ {result}")
|
||||
|
||||
|
||||
# ── Tool declaration (auto-discovered by core/action_loader.py) ──────────────
|
||||
TOOL = {
|
||||
"name": "game_updater",
|
||||
"description": "THE ONLY tool for ANY Steam or Epic Games request. Use for: installing, downloading, updating games, listing installed games, checking download status, scheduling updates. ALWAYS call directly for any Steam/Epic/game request. NEVER use browser_control or web_search for Steam/Epic.",
|
||||
"parameters": {
|
||||
"type": "OBJECT",
|
||||
"properties": {
|
||||
"action": {
|
||||
"type": "STRING",
|
||||
"description": "update | install | list | download_status | schedule | cancel_schedule | schedule_status (default: update)"
|
||||
},
|
||||
"platform": {
|
||||
"type": "STRING",
|
||||
"description": "steam | epic | both (default: both)"
|
||||
},
|
||||
"game_name": {
|
||||
"type": "STRING",
|
||||
"description": "Game name (partial match supported)"
|
||||
},
|
||||
"app_id": {
|
||||
"type": "STRING",
|
||||
"description": "Steam AppID for install (optional)"
|
||||
},
|
||||
"hour": {
|
||||
"type": "INTEGER",
|
||||
"description": "Hour for scheduled update 0-23 (default: 3)"
|
||||
},
|
||||
"minute": {
|
||||
"type": "INTEGER",
|
||||
"description": "Minute for scheduled update 0-59 (default: 0)"
|
||||
},
|
||||
"shutdown_when_done": {
|
||||
"type": "BOOLEAN",
|
||||
"description": "Shut down PC when download finishes"
|
||||
}
|
||||
},
|
||||
"required": []
|
||||
},
|
||||
"handler": game_updater,
|
||||
}
|
||||
@@ -271,3 +271,23 @@ def open_app(
|
||||
except Exception as e:
|
||||
print(f"[open_app] Error: {e}")
|
||||
return f"Failed to open {app_name}: {e}"
|
||||
|
||||
|
||||
# ── Tool declaration (auto-discovered by core/action_loader.py) ──────────────
|
||||
TOOL = {
|
||||
"name": "open_app",
|
||||
"description": "Opens any application on the computer. Use this whenever the user asks to open, launch, or start any app, website, or program. Always call this tool — never just say you opened it.",
|
||||
"parameters": {
|
||||
"type": "OBJECT",
|
||||
"properties": {
|
||||
"app_name": {
|
||||
"type": "STRING",
|
||||
"description": "Exact name of the application (e.g. 'WhatsApp', 'Chrome', 'Spotify')"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"app_name"
|
||||
]
|
||||
},
|
||||
"handler": open_app,
|
||||
}
|
||||
@@ -335,3 +335,33 @@ def reminder(
|
||||
|
||||
friendly_time = target_dt.strftime("%B %d at %I:%M %p")
|
||||
return f"Reminder set for {friendly_time}."
|
||||
|
||||
|
||||
# ── Tool declaration (auto-discovered by core/action_loader.py) ──────────────
|
||||
TOOL = {
|
||||
"name": "reminder",
|
||||
"description": "Sets a timed reminder using Task Scheduler.",
|
||||
"parameters": {
|
||||
"type": "OBJECT",
|
||||
"properties": {
|
||||
"date": {
|
||||
"type": "STRING",
|
||||
"description": "Date in YYYY-MM-DD format"
|
||||
},
|
||||
"time": {
|
||||
"type": "STRING",
|
||||
"description": "Time in HH:MM format (24h)"
|
||||
},
|
||||
"message": {
|
||||
"type": "STRING",
|
||||
"description": "Reminder message text"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"date",
|
||||
"time",
|
||||
"message"
|
||||
]
|
||||
},
|
||||
"handler": reminder,
|
||||
}
|
||||
+9
-290
@@ -1,18 +1,19 @@
|
||||
"""
|
||||
Screen & webcam capture for JARVIS vision.
|
||||
|
||||
Provides the two capture entry points main.py uses — `_capture_screen()` and
|
||||
`_capture_camera()` — plus their helpers (compression, camera auto-detection,
|
||||
config access). main.py grabs a frame here on demand, then injects it into the
|
||||
main Gemini Live session; there is no separate vision session here.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import io
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
import numpy as np
|
||||
import sounddevice as sd
|
||||
|
||||
try:
|
||||
import cv2
|
||||
@@ -33,8 +34,6 @@ try:
|
||||
except ImportError:
|
||||
_PIL = False
|
||||
|
||||
from google import genai
|
||||
from google.genai import types as gtypes
|
||||
|
||||
def _base_dir() -> Path:
|
||||
if getattr(sys, "frozen", False):
|
||||
@@ -62,36 +61,14 @@ def _save_config_key(key: str, value) -> None:
|
||||
print(f"[Vision] ⚠️ Could not save config key '{key}': {e}")
|
||||
|
||||
|
||||
def _get_api_key() -> str:
|
||||
key = _load_config().get("gemini_api_key", "")
|
||||
if not key:
|
||||
raise RuntimeError("gemini_api_key not found in config.")
|
||||
return key
|
||||
|
||||
|
||||
def _get_os() -> str:
|
||||
return _load_config().get("os_system", "windows").lower()
|
||||
|
||||
_LIVE_MODEL = "models/gemini-2.5-flash-native-audio-preview-12-2025"
|
||||
_CHANNELS = 1
|
||||
_RECEIVE_SAMPLE_RATE = 24_000
|
||||
_CHUNK_SIZE = 1_024
|
||||
|
||||
_IMG_MAX_W = 1280
|
||||
_IMG_MAX_H = 720
|
||||
_JPEG_Q = 82
|
||||
|
||||
_SYSTEM_PROMPT = (
|
||||
"You are JARVIS, Tony Stark's AI assistant. "
|
||||
"You are given an image from either the user's screen or their webcam. "
|
||||
"Analyze what you see with detail and intelligence. "
|
||||
"Describe objects, text, people, components, and their context clearly. "
|
||||
"For technical questions (circuits, code, hardware) give specific, expert answers. "
|
||||
"Be concise — 2-4 sentences — unless the question demands more detail. "
|
||||
"Speak directly to the user ('I can see...', 'You have...'). "
|
||||
"Address the user as 'sir' depending on the language they used."
|
||||
)
|
||||
|
||||
|
||||
def _compress(img_bytes: bytes, source_format: str = "PNG") -> tuple[bytes, str]:
|
||||
if not _PIL:
|
||||
@@ -107,6 +84,7 @@ def _compress(img_bytes: bytes, source_format: str = "PNG") -> tuple[bytes, str]
|
||||
print(f"[Vision] ⚠️ Image compress failed: {e}")
|
||||
return img_bytes, f"image/{source_format.lower()}"
|
||||
|
||||
|
||||
def _capture_screen() -> tuple[bytes, str]:
|
||||
|
||||
if not _MSS:
|
||||
@@ -203,262 +181,3 @@ def _capture_camera() -> tuple[bytes, str]:
|
||||
|
||||
_, buf = cv2.imencode(".jpg", frame, [cv2.IMWRITE_JPEG_QUALITY, _JPEG_Q])
|
||||
return buf.tobytes(), "image/jpeg"
|
||||
|
||||
class _VisionSession:
|
||||
def __init__(self):
|
||||
self._loop: Optional[asyncio.AbstractEventLoop] = None
|
||||
self._thread: Optional[threading.Thread] = None
|
||||
self._session = None
|
||||
self._out_queue: Optional[asyncio.Queue] = None
|
||||
self._audio_in: Optional[asyncio.Queue] = None
|
||||
self._ready_evt: threading.Event = threading.Event()
|
||||
self._player = None
|
||||
self._lock: threading.Lock = threading.Lock()
|
||||
|
||||
def start(self, player=None, timeout: float = 25.0) -> None:
|
||||
with self._lock:
|
||||
if self._thread and self._thread.is_alive():
|
||||
if player is not None:
|
||||
self._player = player
|
||||
return
|
||||
self._player = player
|
||||
self._thread = threading.Thread(
|
||||
target=self._run_event_loop,
|
||||
daemon=True,
|
||||
name="VisionSessionThread",
|
||||
)
|
||||
self._thread.start()
|
||||
|
||||
if not self._ready_evt.wait(timeout=timeout):
|
||||
raise RuntimeError(f"Vision session did not connect within {timeout}s.")
|
||||
print("[Vision] ✅ Session ready")
|
||||
|
||||
def analyze(self, image_bytes: bytes, mime_type: str, user_text: str) -> None:
|
||||
if not self._loop or not self._out_queue:
|
||||
print("[Vision] ⚠️ Session not started — dropping request")
|
||||
return
|
||||
asyncio.run_coroutine_threadsafe(
|
||||
self._out_queue.put((image_bytes, mime_type, user_text)),
|
||||
self._loop,
|
||||
)
|
||||
|
||||
def is_ready(self) -> bool:
|
||||
return self._session is not None
|
||||
|
||||
def _run_event_loop(self) -> None:
|
||||
self._loop = asyncio.new_event_loop()
|
||||
asyncio.set_event_loop(self._loop)
|
||||
self._loop.run_until_complete(self._session_loop())
|
||||
|
||||
async def _session_loop(self) -> None:
|
||||
self._out_queue = asyncio.Queue(maxsize=30)
|
||||
self._audio_in = asyncio.Queue()
|
||||
|
||||
client = genai.Client(
|
||||
api_key=_get_api_key(),
|
||||
http_options={"api_version": "v1beta"},
|
||||
)
|
||||
config = gtypes.LiveConnectConfig(
|
||||
response_modalities=["AUDIO"],
|
||||
output_audio_transcription={},
|
||||
system_instruction=_SYSTEM_PROMPT,
|
||||
speech_config=gtypes.SpeechConfig(
|
||||
voice_config=gtypes.VoiceConfig(
|
||||
prebuilt_voice_config=gtypes.PrebuiltVoiceConfig(
|
||||
voice_name="Charon"
|
||||
)
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
backoff = 2.0
|
||||
while True:
|
||||
try:
|
||||
print("[Vision] 🔌 Connecting...")
|
||||
async with client.aio.live.connect(
|
||||
model=_LIVE_MODEL, config=config
|
||||
) as session:
|
||||
self._session = session
|
||||
self._ready_evt.set()
|
||||
backoff = 2.0
|
||||
print("[Vision] ✅ Connected")
|
||||
|
||||
async with asyncio.TaskGroup() as tg:
|
||||
tg.create_task(self._send_loop())
|
||||
tg.create_task(self._recv_loop())
|
||||
tg.create_task(self._play_loop())
|
||||
|
||||
except* Exception as eg:
|
||||
for exc in eg.exceptions:
|
||||
print(f"[Vision] ⚠️ Session error: {exc}")
|
||||
finally:
|
||||
self._session = None
|
||||
self._ready_evt.clear()
|
||||
|
||||
print(f"[Vision] 🔄 Reconnecting in {backoff:.0f}s...")
|
||||
await asyncio.sleep(backoff)
|
||||
backoff = min(backoff * 1.5, 30.0)
|
||||
self._ready_evt.set()
|
||||
|
||||
async def _send_loop(self) -> None:
|
||||
while True:
|
||||
image_bytes, mime_type, user_text = await self._out_queue.get()
|
||||
if not self._session:
|
||||
print("[Vision] ⚠️ No session — dropping image")
|
||||
continue
|
||||
try:
|
||||
b64 = base64.b64encode(image_bytes).decode("ascii")
|
||||
await self._session.send_client_content(
|
||||
turns={
|
||||
"parts": [
|
||||
{"inline_data": {"mime_type": mime_type, "data": b64}},
|
||||
{"text": user_text},
|
||||
]
|
||||
},
|
||||
turn_complete=True,
|
||||
)
|
||||
print(f"[Vision] 📤 Sent {len(image_bytes):,} bytes — '{user_text[:60]}'")
|
||||
except Exception as e:
|
||||
print(f"[Vision] ⚠️ Send error: {e}")
|
||||
raise # propagate to TaskGroup → triggers session reconnect
|
||||
|
||||
async def _recv_loop(self) -> None:
|
||||
transcript: list[str] = []
|
||||
try:
|
||||
async for response in self._session.receive():
|
||||
if response.data:
|
||||
await self._audio_in.put(response.data)
|
||||
|
||||
sc = response.server_content
|
||||
if not sc:
|
||||
continue
|
||||
|
||||
if sc.output_transcription and sc.output_transcription.text:
|
||||
chunk = sc.output_transcription.text.strip()
|
||||
if chunk:
|
||||
transcript.append(chunk)
|
||||
|
||||
if sc.turn_complete:
|
||||
if transcript and self._player:
|
||||
full = re.sub(r"\s+", " ", " ".join(transcript)).strip()
|
||||
if full:
|
||||
self._player.write_log(f"Jarvis: {full}")
|
||||
print(f"[Vision] 💬 {full}")
|
||||
transcript = []
|
||||
# Auto-close camera ~2s after JARVIS finishes speaking
|
||||
if self._player and hasattr(self._player, "stop_camera_stream"):
|
||||
async def _deferred_close():
|
||||
await asyncio.sleep(2.0)
|
||||
try:
|
||||
self._player.stop_camera_stream()
|
||||
except Exception:
|
||||
pass
|
||||
asyncio.create_task(_deferred_close())
|
||||
|
||||
except Exception as e:
|
||||
print(f"[Vision] ⚠️ Recv error: {e}")
|
||||
raise
|
||||
|
||||
async def _play_loop(self) -> None:
|
||||
stream = sd.RawOutputStream(
|
||||
samplerate=_RECEIVE_SAMPLE_RATE,
|
||||
channels=_CHANNELS,
|
||||
dtype="int16",
|
||||
blocksize=_CHUNK_SIZE,
|
||||
)
|
||||
stream.start()
|
||||
try:
|
||||
while True:
|
||||
chunk = await self._audio_in.get()
|
||||
await asyncio.to_thread(stream.write, chunk)
|
||||
except Exception as e:
|
||||
print(f"[Vision] ❌ Play error: {e}")
|
||||
raise
|
||||
finally:
|
||||
stream.stop()
|
||||
stream.close()
|
||||
|
||||
_session = _VisionSession()
|
||||
_session_lock = threading.Lock()
|
||||
_session_up = False
|
||||
|
||||
|
||||
def _ensure_session(player=None) -> None:
|
||||
global _session_up
|
||||
with _session_lock:
|
||||
if not _session_up:
|
||||
_session.start(player=player)
|
||||
_session_up = True
|
||||
elif player is not None:
|
||||
_session._player = player
|
||||
|
||||
|
||||
def screen_process(
|
||||
parameters: dict,
|
||||
response=None,
|
||||
player=None,
|
||||
session_memory=None,
|
||||
) -> bool:
|
||||
|
||||
params = parameters or {}
|
||||
user_text = (params.get("text") or params.get("user_text") or "").strip()
|
||||
angle = params.get("angle", "screen").lower().strip()
|
||||
|
||||
if not user_text:
|
||||
print("[Vision] ⚠️ No question provided — aborting")
|
||||
return False
|
||||
|
||||
print(f"[Vision] ▶ angle={angle!r} question='{user_text[:80]}'")
|
||||
|
||||
try:
|
||||
_ensure_session(player=player)
|
||||
except Exception as e:
|
||||
print(f"[Vision] ❌ Could not start session: {e}")
|
||||
return False
|
||||
|
||||
try:
|
||||
if angle == "camera":
|
||||
image_bytes, mime_type = _capture_camera()
|
||||
print(f"[Vision] 📷 Camera: {len(image_bytes):,} bytes")
|
||||
if player and hasattr(player, "start_camera_stream"):
|
||||
try:
|
||||
player.start_camera_stream()
|
||||
except Exception as _e:
|
||||
print(f"[Vision] ⚠️ Camera stream failed: {_e}")
|
||||
elif player and hasattr(player, "show_camera_frame"):
|
||||
try:
|
||||
player.show_camera_frame(image_bytes)
|
||||
except Exception as _e:
|
||||
print(f"[Vision] ⚠️ Camera preview failed: {_e}")
|
||||
else:
|
||||
image_bytes, mime_type = _capture_screen()
|
||||
print(f"[Vision] 🖥️ Screen: {len(image_bytes):,} bytes")
|
||||
except Exception as e:
|
||||
print(f"[Vision] ❌ Capture error: {e}")
|
||||
return False
|
||||
|
||||
_session.analyze(image_bytes, mime_type, user_text)
|
||||
return True
|
||||
|
||||
|
||||
def warmup_session(player=None) -> None:
|
||||
try:
|
||||
_ensure_session(player=player)
|
||||
except Exception as e:
|
||||
print(f"[Vision] ⚠️ Warmup failed: {e}")
|
||||
|
||||
if __name__ == "__main__":
|
||||
print("[TEST] screen_processor.py")
|
||||
print("=" * 52)
|
||||
mode = input("angle — screen / camera (default: screen): ").strip().lower() or "screen"
|
||||
q = input("Question (Enter = default): ").strip() or "What do you see? Be brief."
|
||||
|
||||
t0 = time.perf_counter()
|
||||
warmup_session()
|
||||
print(f"Session ready in {time.perf_counter()-t0:.2f}s\n")
|
||||
|
||||
t1 = time.perf_counter()
|
||||
ok = screen_process({"angle": mode, "text": q})
|
||||
print(f"Queued in {time.perf_counter()-t1:.3f}s — waiting for audio...")
|
||||
time.sleep(10)
|
||||
print("Done." if ok else "Failed.")
|
||||
@@ -264,3 +264,33 @@ def send_message(
|
||||
player.write_log(f"[msg] {result}")
|
||||
|
||||
return result
|
||||
|
||||
|
||||
# ── Tool declaration (auto-discovered by core/action_loader.py) ──────────────
|
||||
TOOL = {
|
||||
"name": "send_message",
|
||||
"description": "Sends a text message via WhatsApp, Telegram, or other messaging platform.",
|
||||
"parameters": {
|
||||
"type": "OBJECT",
|
||||
"properties": {
|
||||
"receiver": {
|
||||
"type": "STRING",
|
||||
"description": "Recipient contact name"
|
||||
},
|
||||
"message_text": {
|
||||
"type": "STRING",
|
||||
"description": "The message to send"
|
||||
},
|
||||
"platform": {
|
||||
"type": "STRING",
|
||||
"description": "Platform: WhatsApp, Telegram, etc."
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"receiver",
|
||||
"message_text",
|
||||
"platform"
|
||||
]
|
||||
},
|
||||
"handler": send_message,
|
||||
}
|
||||
@@ -49,3 +49,23 @@ def _log(message: str, player=None) -> None:
|
||||
player.write_log(f"JARVIS: {message}")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
# ── Tool declaration (auto-discovered by core/action_loader.py) ──────────────
|
||||
TOOL = {
|
||||
"name": "weather_report",
|
||||
"description": "Gives the weather report to user",
|
||||
"parameters": {
|
||||
"type": "OBJECT",
|
||||
"properties": {
|
||||
"city": {
|
||||
"type": "STRING",
|
||||
"description": "City name"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"city"
|
||||
]
|
||||
},
|
||||
"handler": weather_action,
|
||||
}
|
||||
+83
-118
@@ -1,68 +1,8 @@
|
||||
#web_search.py
|
||||
import json
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
# ── Gemini grounding quota circuit breaker ────────────────────────────────────
|
||||
# The google_search grounding tool has its own small quota, separate from plain
|
||||
# generation. Once it is spent every call returns 429 — so retrying it at the
|
||||
# top of every search only adds a dead round-trip before the DDG fallback runs.
|
||||
# After a quota error, skip Gemini entirely for a cooldown period.
|
||||
_QUOTA_COOLDOWN_SEC = 900 # 15 minutes
|
||||
_quota_blocked_until = 0.0
|
||||
_quota_lock = threading.Lock()
|
||||
|
||||
|
||||
def _gemini_available() -> bool:
|
||||
with _quota_lock:
|
||||
return time.monotonic() >= _quota_blocked_until
|
||||
|
||||
|
||||
def _note_gemini_error(exc: Exception) -> None:
|
||||
"""Trip the breaker when the error is a quota / rate-limit rejection."""
|
||||
global _quota_blocked_until
|
||||
msg = str(exc)
|
||||
if "429" in msg or "RESOURCE_EXHAUSTED" in msg:
|
||||
with _quota_lock:
|
||||
already = time.monotonic() < _quota_blocked_until
|
||||
_quota_blocked_until = time.monotonic() + _QUOTA_COOLDOWN_SEC
|
||||
if not already:
|
||||
print(
|
||||
"[WebSearch] Gemini grounding quota exhausted — skipping it for "
|
||||
f"{_QUOTA_COOLDOWN_SEC // 60} min and serving results from DDG."
|
||||
)
|
||||
|
||||
|
||||
class _QuotaCooldown(RuntimeError):
|
||||
"""Raised instead of calling Gemini while the quota breaker is open."""
|
||||
|
||||
|
||||
def _log_gemini_failure(context: str, exc: Exception) -> None:
|
||||
"""Log a Gemini failure — silently when it is just the expected cooldown."""
|
||||
if isinstance(exc, _QuotaCooldown):
|
||||
return # announced once when the breaker tripped; not a warning
|
||||
print(f"[WebSearch] \u26a0\ufe0f {context} failed ({exc}) — using DDG instead")
|
||||
|
||||
|
||||
def _run_bounded(fn, timeout: float, label: str = "task"):
|
||||
"""Run fn() in a daemon thread; return its result, or None if it overruns."""
|
||||
box = [None]
|
||||
|
||||
def _run():
|
||||
try:
|
||||
box[0] = fn()
|
||||
except Exception as e:
|
||||
_log_gemini_failure(label, e)
|
||||
|
||||
t = threading.Thread(target=_run, daemon=True)
|
||||
t.start()
|
||||
t.join(timeout)
|
||||
if t.is_alive():
|
||||
print(f"[WebSearch] {label} exceeded {timeout:.0f}s — moving on")
|
||||
return box[0]
|
||||
|
||||
def _get_base_dir() -> Path:
|
||||
if getattr(sys, "frozen", False):
|
||||
return Path(sys.executable).parent
|
||||
@@ -79,21 +19,14 @@ def _get_api_key() -> str:
|
||||
|
||||
|
||||
def _gemini_search(query: str) -> str:
|
||||
if not _gemini_available():
|
||||
raise _QuotaCooldown("Gemini grounding is in quota cooldown")
|
||||
|
||||
from google import genai
|
||||
|
||||
client = genai.Client(api_key=_get_api_key())
|
||||
try:
|
||||
response = client.models.generate_content(
|
||||
model="gemini-flash-latest",
|
||||
contents=query,
|
||||
config={"tools": [{"google_search": {}}]},
|
||||
)
|
||||
except Exception as e:
|
||||
_note_gemini_error(e)
|
||||
raise
|
||||
|
||||
text = ""
|
||||
for part in response.candidates[0].content.parts:
|
||||
@@ -106,30 +39,13 @@ def _gemini_search(query: str) -> str:
|
||||
return text
|
||||
|
||||
|
||||
def _get_ddgs():
|
||||
"""
|
||||
Returns the DDGS class. The package was renamed duckduckgo-search -> ddgs;
|
||||
the legacy package's endpoints are now rejected by DuckDuckGo (news() gets a
|
||||
403 Ratelimit, text() silently returns zero results), so warn loudly if we
|
||||
end up on it instead of failing in silence.
|
||||
"""
|
||||
def _ddg_search(query: str, max_results: int = 6) -> list[dict]:
|
||||
try:
|
||||
from ddgs import DDGS
|
||||
return DDGS
|
||||
except ImportError:
|
||||
from duckduckgo_search import DDGS
|
||||
print(
|
||||
"[WebSearch] ⚠️ Using the deprecated 'duckduckgo-search' package — "
|
||||
"DuckDuckGo blocks its endpoints, so every search will come back "
|
||||
"empty. Fix with: pip install -U ddgs"
|
||||
)
|
||||
return DDGS
|
||||
|
||||
|
||||
def _ddg_search(query: str, max_results: int = 6) -> list[dict]:
|
||||
DDGS = _get_ddgs()
|
||||
results = []
|
||||
try:
|
||||
with DDGS() as ddgs:
|
||||
for r in ddgs.text(query, max_results=max_results):
|
||||
results.append({
|
||||
@@ -137,14 +53,16 @@ def _ddg_search(query: str, max_results: int = 6) -> list[dict]:
|
||||
"snippet": r.get("body", ""),
|
||||
"url": r.get("href", ""),
|
||||
})
|
||||
except Exception as e:
|
||||
print(f"[WebSearch] ⚠️ DDG text() failed: {e}")
|
||||
return results
|
||||
|
||||
|
||||
def _ddg_news(query: str, max_results: int = 8) -> list[dict]:
|
||||
"""DDG news search — returns actual articles, not website homepages."""
|
||||
DDGS = _get_ddgs()
|
||||
try:
|
||||
from ddgs import DDGS
|
||||
except ImportError:
|
||||
from duckduckgo_search import DDGS
|
||||
|
||||
results = []
|
||||
try:
|
||||
with DDGS() as ddgs:
|
||||
@@ -157,9 +75,6 @@ def _ddg_news(query: str, max_results: int = 8) -> list[dict]:
|
||||
})
|
||||
except Exception as e:
|
||||
print(f"[WebSearch] ⚠️ DDG news() failed ({e}) — falling back to text search")
|
||||
# Also covers the legacy-package case, where news() returns an empty list
|
||||
# instead of raising.
|
||||
if not results:
|
||||
results = _ddg_search(query, max_results=max_results)
|
||||
return results
|
||||
|
||||
@@ -242,43 +157,58 @@ def _search(query: str) -> str:
|
||||
try:
|
||||
return _gemini_search(query)
|
||||
except Exception as e:
|
||||
_log_gemini_failure("Gemini search", e)
|
||||
print(f"[WebSearch] ⚠️ Gemini failed ({e}) — trying DDG...")
|
||||
results = _ddg_search(query)
|
||||
return _format_ddg(query, results)
|
||||
|
||||
|
||||
def _news(query: str) -> str:
|
||||
"""
|
||||
DDG first, Gemini as backup.
|
||||
|
||||
The old version raced both backends in parallel and kept the first answer.
|
||||
That burned one google_search grounding call on *every* news request —
|
||||
including the startup briefing — even when DDG had already won the race.
|
||||
Grounding has a small quota, so it ran dry after a handful of launches and
|
||||
then 429'd for everything else (research/compare), which are the modes that
|
||||
actually need a synthesised answer.
|
||||
|
||||
DDG news returns in well under a second and gives raw headlines, which is
|
||||
exactly what the briefing wants, so it goes first and Gemini is only touched
|
||||
when DDG comes back empty.
|
||||
Runs Gemini grounded search AND DDG news in parallel.
|
||||
Returns whichever delivers a valid result first; cancels the other.
|
||||
"""
|
||||
import threading
|
||||
|
||||
gemini_query = f"latest news today: {query}" if query else "top world news today"
|
||||
ddg_query = query if query else "world news today"
|
||||
|
||||
def _ddg_attempt() -> str:
|
||||
return _format_news(ddg_query, _ddg_news(ddg_query, max_results=8))
|
||||
result_box = [None] # first valid result lands here
|
||||
lock = threading.Lock()
|
||||
done_evt = threading.Event()
|
||||
failures = [0]
|
||||
|
||||
text = _run_bounded(_ddg_attempt, timeout=5.0, label="DDG news")
|
||||
if text and len(text) > 60 and not text.startswith("No news found"):
|
||||
return text
|
||||
def _store(r: str) -> None:
|
||||
if r and len(r) > 60:
|
||||
with lock:
|
||||
if result_box[0] is None:
|
||||
result_box[0] = r
|
||||
done_evt.set()
|
||||
else:
|
||||
with lock:
|
||||
failures[0] += 1
|
||||
if failures[0] >= 2: # both failed — unblock caller
|
||||
done_evt.set()
|
||||
|
||||
text = _run_bounded(
|
||||
lambda: _gemini_search(gemini_query), timeout=6.0, label="Gemini news"
|
||||
)
|
||||
if text and len(text) > 60:
|
||||
return text
|
||||
def _try_gemini():
|
||||
try:
|
||||
_store(_gemini_search(gemini_query))
|
||||
except Exception as e:
|
||||
print(f"[WebSearch] ⚠️ Gemini news failed ({e})")
|
||||
_store("")
|
||||
|
||||
return f"No news found for: {query}"
|
||||
def _try_ddg():
|
||||
try:
|
||||
results = _ddg_news(ddg_query, max_results=8)
|
||||
_store(_format_news(ddg_query, results))
|
||||
except Exception as e:
|
||||
print(f"[WebSearch] ⚠️ DDG news failed ({e})")
|
||||
_store("")
|
||||
|
||||
threading.Thread(target=_try_gemini, daemon=True).start()
|
||||
threading.Thread(target=_try_ddg, daemon=True).start()
|
||||
|
||||
done_evt.wait(timeout=10.0)
|
||||
return result_box[0] or f"No news found for: {query}"
|
||||
|
||||
|
||||
def _research(query: str) -> str:
|
||||
@@ -293,7 +223,7 @@ def _research(query: str) -> str:
|
||||
try:
|
||||
return _gemini_search(research_query)
|
||||
except Exception as e:
|
||||
_log_gemini_failure("Gemini research", e)
|
||||
print(f"[WebSearch] ⚠️ Research Gemini failed ({e}) — DDG fallback...")
|
||||
results = _ddg_search(query, max_results=10)
|
||||
return _format_ddg(query, results)
|
||||
|
||||
@@ -304,7 +234,7 @@ def _price(query: str) -> str:
|
||||
try:
|
||||
return _gemini_search(price_query)
|
||||
except Exception as e:
|
||||
_log_gemini_failure("Gemini price", e)
|
||||
print(f"[WebSearch] ⚠️ Price Gemini failed ({e}) — DDG fallback...")
|
||||
results = _ddg_search(f"{query} price buy", max_results=6)
|
||||
return _format_ddg(query, results)
|
||||
|
||||
@@ -317,7 +247,7 @@ def _compare(items: list[str], aspect: str) -> str:
|
||||
try:
|
||||
return _gemini_search(query)
|
||||
except Exception as e:
|
||||
_log_gemini_failure("Gemini compare", e)
|
||||
print(f"[WebSearch] ⚠️ Gemini compare failed: {e} — falling back to DDG")
|
||||
|
||||
all_results: dict[str, list] = {}
|
||||
for item in items:
|
||||
@@ -376,3 +306,38 @@ def web_search(
|
||||
except Exception as e:
|
||||
print(f"[WebSearch] ❌ All backends failed: {e}")
|
||||
return f"Search failed: {e}"
|
||||
|
||||
|
||||
# ── Tool declaration (auto-discovered by core/action_loader.py) ──────────────
|
||||
TOOL = {
|
||||
"name": "web_search",
|
||||
"description": "Searches the web. Use for ANY question about current facts, events, prices, or topics — always prefer this over guessing. Modes: 'search' (default), 'news' (latest headlines on a topic), 'research' (deep comprehensive answer), 'price' (product cost lookup), 'compare' (side-by-side comparison of items).",
|
||||
"parameters": {
|
||||
"type": "OBJECT",
|
||||
"properties": {
|
||||
"query": {
|
||||
"type": "STRING",
|
||||
"description": "Search query or topic"
|
||||
},
|
||||
"mode": {
|
||||
"type": "STRING",
|
||||
"description": "search | news | research | price | compare"
|
||||
},
|
||||
"items": {
|
||||
"type": "ARRAY",
|
||||
"items": {
|
||||
"type": "STRING"
|
||||
},
|
||||
"description": "Items to compare (compare mode)"
|
||||
},
|
||||
"aspect": {
|
||||
"type": "STRING",
|
||||
"description": "Comparison aspect: price | specs | reviews | features"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"query"
|
||||
]
|
||||
},
|
||||
"handler": web_search,
|
||||
}
|
||||
@@ -435,3 +435,37 @@ def youtube_video(
|
||||
except Exception as e:
|
||||
print(f"[YouTube] ❌ Error in {action}: {e}")
|
||||
return f"YouTube {action} failed, sir: {e}"
|
||||
|
||||
|
||||
# ── Tool declaration (auto-discovered by core/action_loader.py) ──────────────
|
||||
TOOL = {
|
||||
"name": "youtube_video",
|
||||
"description": "Controls YouTube. Use for: playing videos, summarizing a video's content, getting video info, or showing trending videos.",
|
||||
"parameters": {
|
||||
"type": "OBJECT",
|
||||
"properties": {
|
||||
"action": {
|
||||
"type": "STRING",
|
||||
"description": "play | summarize | get_info | trending (default: play)"
|
||||
},
|
||||
"query": {
|
||||
"type": "STRING",
|
||||
"description": "Search query for play action"
|
||||
},
|
||||
"save": {
|
||||
"type": "BOOLEAN",
|
||||
"description": "Save summary to Notepad (summarize only)"
|
||||
},
|
||||
"region": {
|
||||
"type": "STRING",
|
||||
"description": "Country code for trending e.g. TR, US"
|
||||
},
|
||||
"url": {
|
||||
"type": "STRING",
|
||||
"description": "Video URL for get_info action"
|
||||
}
|
||||
},
|
||||
"required": []
|
||||
},
|
||||
"handler": youtube_video,
|
||||
}
|
||||
@@ -0,0 +1,193 @@
|
||||
"""
|
||||
Action discovery, validation, and dispatch — the built-in twin of plugin_loader.
|
||||
|
||||
Every actions/*.py that exposes a module-level ``TOOL`` dict is auto-discovered
|
||||
here, exactly like a drop-in plugin, so main.py never has to hardcode a tool
|
||||
declaration or a dispatch branch for it. Adding a new bundled action is then the
|
||||
same one-file operation as writing a plugin: define ``TOOL`` and a handler.
|
||||
|
||||
``TOOL`` shape (see actions/open_app.py for a live example):
|
||||
|
||||
TOOL = {
|
||||
"name": "open_app", # unique, ^[a-zA-Z_][a-zA-Z0-9_]{0,63}$
|
||||
"description": "...", # what Gemini reads to route the call
|
||||
"parameters": {"type": "OBJECT", ...}, # Gemini function-declaration schema
|
||||
"handler": open_app, # the callable to run
|
||||
}
|
||||
|
||||
The handler is invoked through signature introspection: it receives ``parameters``
|
||||
plus whichever of ``player`` / ``speak`` / ``response`` / ``session_memory`` it
|
||||
actually declares — so existing action signatures work unchanged.
|
||||
|
||||
Discovery runs once at startup; import errors, validation errors, and name
|
||||
collisions are logged and the offending file is skipped — they NEVER raise out
|
||||
of discover_actions() and never abort the scan of the remaining files.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import inspect
|
||||
import re
|
||||
import sys
|
||||
import traceback
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Callable, Optional
|
||||
|
||||
_NAME_RE = re.compile(r"^[a-zA-Z_][a-zA-Z0-9_]{0,63}$")
|
||||
_DEFAULT_PARAMS = {"type": "OBJECT", "properties": {}}
|
||||
_CTX_KEYS = ("player", "speak", "response", "session_memory")
|
||||
|
||||
|
||||
@dataclass
|
||||
class ActionRecord:
|
||||
name: str
|
||||
description: str = ""
|
||||
parameters: dict = field(default_factory=lambda: dict(_DEFAULT_PARAMS))
|
||||
handler: Optional[Callable] = None
|
||||
file: str = ""
|
||||
valid: bool = False
|
||||
error: str = ""
|
||||
|
||||
|
||||
class ActionRegistry:
|
||||
def __init__(self, actions: dict[str, ActionRecord], logger: Callable[[str], None]):
|
||||
self._actions = actions # name -> ActionRecord, VALID entries only
|
||||
self._all_records: list[ActionRecord] = []
|
||||
self._logger = logger
|
||||
|
||||
# -- called by main.py at LiveConnectConfig build time --
|
||||
def get_tool_declarations(self) -> list[dict]:
|
||||
return [
|
||||
{"name": rec.name, "description": rec.description, "parameters": rec.parameters}
|
||||
for rec in self._actions.values()
|
||||
]
|
||||
|
||||
def has(self, name: str) -> bool:
|
||||
return name in self._actions
|
||||
|
||||
def names(self) -> set[str]:
|
||||
return set(self._actions.keys())
|
||||
|
||||
# -- called by main.py from _execute_tool --
|
||||
def run(self, name: str, parameters: dict, ctx: dict | None = None) -> str:
|
||||
rec = self._actions.get(name)
|
||||
if rec is None or not rec.valid:
|
||||
return f"Action '{name}' is not available."
|
||||
try:
|
||||
return _call_handler(rec.handler, parameters, ctx or {}) or "Done."
|
||||
except Exception as e:
|
||||
self._logger(f"Action '{name}' crashed during run(): {e}")
|
||||
traceback.print_exc()
|
||||
return f"Tool '{name}' failed: {e}"
|
||||
|
||||
|
||||
def _call_handler(fn: Callable, parameters: dict, ctx: dict) -> str:
|
||||
"""Invoke the handler passing only the context kwargs it actually declares
|
||||
(or all of them if it has **kwargs), so each action's existing signature
|
||||
works unchanged."""
|
||||
sig = inspect.signature(fn)
|
||||
has_var_kw = any(p.kind == inspect.Parameter.VAR_KEYWORD for p in sig.parameters.values())
|
||||
kwargs = {}
|
||||
for key in _CTX_KEYS:
|
||||
if has_var_kw or key in sig.parameters:
|
||||
kwargs[key] = ctx.get(key)
|
||||
return fn(parameters=parameters, **kwargs)
|
||||
|
||||
|
||||
def _validate(module, filename: str) -> ActionRecord:
|
||||
"""Returns an ActionRecord; .valid=False + .error set on any problem. Never raises."""
|
||||
tool = getattr(module, "TOOL", None)
|
||||
if not isinstance(tool, dict):
|
||||
return ActionRecord(name=Path(filename).stem, file=filename,
|
||||
error="No module-level TOOL dict (not a discoverable action).")
|
||||
|
||||
name = tool.get("name")
|
||||
if not isinstance(name, str) or not _NAME_RE.match(name):
|
||||
return ActionRecord(name=str(name or Path(filename).stem), file=filename,
|
||||
error="TOOL['name'] missing or not a valid identifier.")
|
||||
|
||||
description = tool.get("description")
|
||||
if not isinstance(description, str) or not description.strip():
|
||||
return ActionRecord(name=name, file=filename,
|
||||
error="TOOL['description'] missing or empty.")
|
||||
|
||||
parameters = tool.get("parameters", _DEFAULT_PARAMS)
|
||||
if not isinstance(parameters, dict) or parameters.get("type") != "OBJECT":
|
||||
return ActionRecord(name=name, file=filename,
|
||||
error="TOOL['parameters'] must be a dict with \"type\": \"OBJECT\".")
|
||||
|
||||
handler = tool.get("handler")
|
||||
if not callable(handler):
|
||||
return ActionRecord(name=name, file=filename,
|
||||
error="TOOL['handler'] missing or not callable.")
|
||||
|
||||
return ActionRecord(name=name, description=description.strip(), parameters=parameters,
|
||||
handler=handler, file=filename, valid=True, error="")
|
||||
|
||||
|
||||
def discover_actions(actions_dir: Path, reserved_names: set[str] | None = None,
|
||||
logger: Callable[[str], None] = print) -> ActionRegistry:
|
||||
"""
|
||||
Scans actions_dir for *.py files (skips files starting with '_'). A file is
|
||||
only treated as an action if it exposes a module-level TOOL dict; files
|
||||
without one (shared helpers, capture-only modules) are silently ignored.
|
||||
Import/validation errors and name collisions are logged and the file is
|
||||
skipped — they NEVER raise out of this function.
|
||||
"""
|
||||
reserved = reserved_names or set()
|
||||
actions_dir.mkdir(parents=True, exist_ok=True)
|
||||
valid: dict[str, ActionRecord] = {}
|
||||
all_records: list[ActionRecord] = []
|
||||
|
||||
files = sorted(actions_dir.glob("*.py"), key=lambda p: p.name) # deterministic order
|
||||
for path in files:
|
||||
if path.name.startswith("_"):
|
||||
continue
|
||||
try:
|
||||
module_name = f"actions.{path.stem}"
|
||||
# Reuse the already-imported module when present so handlers are the
|
||||
# same objects the rest of the app holds.
|
||||
module = sys.modules.get(module_name)
|
||||
if module is None:
|
||||
spec = importlib.util.spec_from_file_location(module_name, path)
|
||||
if spec is None or spec.loader is None:
|
||||
raise ImportError("could not build import spec")
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[module_name] = module
|
||||
try:
|
||||
spec.loader.exec_module(module)
|
||||
except Exception:
|
||||
sys.modules.pop(module_name, None)
|
||||
raise
|
||||
|
||||
if getattr(module, "TOOL", None) is None:
|
||||
continue # not an action file — a helper/capture-only module
|
||||
|
||||
rec = _validate(module, path.name)
|
||||
|
||||
if rec.valid and rec.name in reserved:
|
||||
rec = ActionRecord(name=rec.name, file=path.name,
|
||||
error=f"Name '{rec.name}' collides with a reserved core tool — rejected.")
|
||||
elif rec.valid and rec.name in valid:
|
||||
other = valid[rec.name].file
|
||||
rec = ActionRecord(name=rec.name, file=path.name,
|
||||
error=f"Name '{rec.name}' already used by action '{other}' — rejected.")
|
||||
|
||||
except Exception as e:
|
||||
rec = ActionRecord(name=path.stem, file=path.name,
|
||||
error=f"Failed to load: {e}")
|
||||
traceback.print_exc()
|
||||
|
||||
all_records.append(rec)
|
||||
if rec.valid:
|
||||
valid[rec.name] = rec
|
||||
logger(f"Action loaded: {rec.name} ({path.name})")
|
||||
else:
|
||||
# Only log a rejection if the file actually tried to be an action.
|
||||
logger(f"Action rejected: {path.name} — {rec.error}")
|
||||
|
||||
registry = ActionRegistry(valid, logger)
|
||||
registry._all_records = all_records
|
||||
logger(f"Action discovery complete: {len(valid)} active.")
|
||||
return registry
|
||||
+35
-2
@@ -17,7 +17,7 @@ from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Callable, Optional
|
||||
|
||||
from memory.config_manager import get_plugin_enabled
|
||||
from memory.config_manager import get_plugin_enabled, get_plugin_config
|
||||
|
||||
_NAME_RE = re.compile(r"^[a-zA-Z_][a-zA-Z0-9_]{0,63}$")
|
||||
_DEFAULT_PARAMS = {"type": "OBJECT", "properties": {}}
|
||||
@@ -32,6 +32,7 @@ class PluginRecord:
|
||||
file: str = ""
|
||||
valid: bool = False
|
||||
error: str = ""
|
||||
settings: Optional[dict] = None # optional PLUGIN_SETTINGS schema (config fields)
|
||||
|
||||
|
||||
class PluginRegistry:
|
||||
@@ -69,6 +70,32 @@ class PluginRegistry:
|
||||
traceback.print_exc()
|
||||
return f"Sir, the '{name}' plugin failed: {e}"
|
||||
|
||||
# -- called by ui.py's settings tab to render per-plugin config forms --
|
||||
def settings_schemas(self) -> list[dict]:
|
||||
"""One entry per settings SECTION, for enabled plugins that declare a
|
||||
PLUGIN_SETTINGS schema. Sections are deduped by namespace so a suite of
|
||||
plugins sharing one namespace (e.g. the printer trio) shows a single
|
||||
form. Current stored values are merged in so the UI can pre-fill fields.
|
||||
"""
|
||||
seen: set[str] = set()
|
||||
out: list[dict] = []
|
||||
for name, rec in self._plugins.items():
|
||||
if not rec.settings or not get_plugin_enabled(name):
|
||||
continue
|
||||
ns = rec.settings.get("namespace") or rec.name
|
||||
if ns in seen:
|
||||
continue
|
||||
seen.add(ns)
|
||||
out.append({
|
||||
"plugin": rec.name,
|
||||
"namespace": ns,
|
||||
"title": rec.settings.get("title") or rec.name,
|
||||
"fields": rec.settings.get("fields", []),
|
||||
"values": get_plugin_config(ns),
|
||||
"action": rec.settings.get("action"), # optional test/connect button
|
||||
})
|
||||
return out
|
||||
|
||||
# -- called by ui.py's Plugin Manager overlay --
|
||||
def list_for_ui(self) -> list[dict]:
|
||||
out = []
|
||||
@@ -125,8 +152,14 @@ def _validate(module, filename: str) -> PluginRecord:
|
||||
return PluginRecord(name=name, file=filename,
|
||||
error="Missing callable run(parameters, ...) function.")
|
||||
|
||||
# Optional, self-describing settings schema (rendered by the settings UI).
|
||||
# A malformed schema is ignored, never fatal — the plugin still loads.
|
||||
settings = getattr(module, "PLUGIN_SETTINGS", None)
|
||||
if not (isinstance(settings, dict) and isinstance(settings.get("fields"), list)):
|
||||
settings = None
|
||||
|
||||
return PluginRecord(name=name, description=description.strip(), parameters=parameters,
|
||||
run=run_fn, file=filename, valid=True, error="")
|
||||
run=run_fn, file=filename, valid=True, error="", settings=settings)
|
||||
|
||||
|
||||
def discover_plugins(plugins_dir: Path, core_tool_names: set[str],
|
||||
|
||||
@@ -14,6 +14,13 @@ Response time: Respond as fast as you can.
|
||||
Act: Always act like Jarvis from Iron Man — professional, efficient, slightly witty. (greet the user warmly in their own language)
|
||||
Length: Match response length to the task. Briefing = short. Complex analysis = thorough.
|
||||
|
||||
ACKNOWLEDGE BEFORE A TASK TAKES A MOMENT:
|
||||
When the user asks for something whose tool needs more than an instant to finish — reading/analyzing an uploaded file, web/news/research/price searches, finding flights, writing or building code, building a project, updating a game — FIRST say exactly ONE short, natural sentence, in the USER'S OWN language, that fits THIS specific request, and THEN call the tool in the same turn. Compose that sentence fresh each time, in your own words, in the language of the conversation — naming what you are about to do. Do NOT reuse a remembered phrase and do NOT lift wording out of a tool's description: anything written there is an explanation for you, not a line for you to say, and repeating it can put you in the wrong language. When the tool returns, deliver the result.
|
||||
- Purpose: the user should hear you react instantly, so they never feel they're waiting on a frozen assistant.
|
||||
- Instant actions (opening an app, volume, brightness, a single keypress, a quick setting) need NO acknowledgment — just do them.
|
||||
- Do NOT pre-acknowledge for screen_process / vision — that flow already tells the user you are looking.
|
||||
- Keep it to ONE short sentence and vary the wording every time; never a fixed template. It must not talk over the tool's own output.
|
||||
|
||||
LANGUAGE:
|
||||
The language of your reply is the language of the user's MOST RECENT message. Nothing else decides it — not what is stored in memory, not the language of these instructions, not the language of a tool's result.
|
||||
Memory may say which language this person has used before. That is a record of the past, not an instruction for now: if they write or speak in another language today, answer in that one.
|
||||
|
||||
@@ -0,0 +1,202 @@
|
||||
"""
|
||||
Local wake-word detection for JARVIS ("Hey Jarvis").
|
||||
|
||||
Design goals:
|
||||
• ZERO cost when the feature is off — openwakeword is imported ONLY inside
|
||||
start()/install helpers, never at module load. If the user never enables
|
||||
wake word, none of this touches the app.
|
||||
• ZERO latency on the audio path — the microphone callback only ever does a
|
||||
cheap, non-blocking queue push (feed()); the actual model inference runs in
|
||||
this module's own background thread, so the real-time audio thread and the
|
||||
Gemini stream are never slowed.
|
||||
• Fully local & offline — audio fed here never leaves the machine; there is no
|
||||
network call except the one-time model download the user triggers from the UI.
|
||||
|
||||
openwakeword ships small ONNX models (a few MB each) and runs comfortably on a
|
||||
CPU. The pretrained wake phrase used here is "Hey Jarvis".
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import queue
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
from pathlib import Path
|
||||
from typing import Callable
|
||||
|
||||
# Pretrained openwakeword model that listens for "Hey Jarvis".
|
||||
WAKE_MODEL = "hey_jarvis"
|
||||
# Score in [0,1]; above this counts as a detection. Tunable per environment.
|
||||
DEFAULT_THRESHOLD = 0.5
|
||||
# Mic frames arrive at 16 kHz int16; this is just the detector's input rate.
|
||||
SAMPLE_RATE = 16000
|
||||
|
||||
|
||||
def is_installed() -> bool:
|
||||
"""True if the openwakeword package is importable (no model check)."""
|
||||
try:
|
||||
import importlib.util
|
||||
return importlib.util.find_spec("openwakeword") is not None
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def is_ready() -> bool:
|
||||
"""True if openwakeword is installed AND its model files are present on disk.
|
||||
|
||||
This is a cheap, DETERMINISTIC file-existence check. It deliberately does NOT
|
||||
construct a Model to probe readiness — doing that is slow and, worse, can clash
|
||||
with the detector's own Model when it's already running, which intermittently
|
||||
returned False and made the UI flicker to 'not downloaded'. Never raises.
|
||||
"""
|
||||
if not is_installed():
|
||||
return False
|
||||
try:
|
||||
import openwakeword
|
||||
models_dir = Path(openwakeword.__file__).resolve().parent / "resources" / "models"
|
||||
if not models_dir.is_dir():
|
||||
return False
|
||||
has_wake = (any(models_dir.glob(f"{WAKE_MODEL}*.onnx"))
|
||||
or any(models_dir.glob(f"{WAKE_MODEL}*.tflite")))
|
||||
has_mel = (any(models_dir.glob("melspectrogram*.onnx"))
|
||||
or any(models_dir.glob("melspectrogram*.tflite")))
|
||||
has_emb = (any(models_dir.glob("embedding_model*.onnx"))
|
||||
or any(models_dir.glob("embedding_model*.tflite")))
|
||||
return bool(has_wake and has_mel and has_emb)
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def install_and_download(logger: Callable[[str], None] = print) -> tuple[bool, str]:
|
||||
"""
|
||||
One-click setup for the UI button: pip-install openwakeword if missing, then
|
||||
download the wake model. Returns (ok, message). Never raises — every failure
|
||||
is reported through the returned message and the logger.
|
||||
"""
|
||||
try:
|
||||
if not is_installed():
|
||||
logger("Wake word: installing openwakeword (one-time)…")
|
||||
r = subprocess.run(
|
||||
[sys.executable, "-m", "pip", "install", "openwakeword"],
|
||||
capture_output=True, text=True,
|
||||
)
|
||||
if r.returncode != 0:
|
||||
tail = (r.stderr or r.stdout or "").strip().splitlines()[-1:] or [""]
|
||||
return False, f"pip install failed: {tail[0][:160]}"
|
||||
# Download the pretrained melspectrogram/embedding + wake models.
|
||||
logger("Wake word: downloading models…")
|
||||
try:
|
||||
import openwakeword.utils as _u
|
||||
try:
|
||||
_u.download_models([WAKE_MODEL])
|
||||
except TypeError:
|
||||
_u.download_models() # older signature downloads the default set
|
||||
except Exception as e:
|
||||
return False, f"model download failed: {e}"
|
||||
|
||||
if not is_ready():
|
||||
return False, "installed, but the wake model could not be loaded."
|
||||
logger("Wake word: ready.")
|
||||
return True, "Wake word installed and ready."
|
||||
except Exception as e:
|
||||
return False, f"setup error: {e}"
|
||||
|
||||
|
||||
class WakeWordDetector:
|
||||
"""
|
||||
Runs the wake model in a dedicated thread. The mic thread calls feed() with
|
||||
raw int16 frames; detections invoke on_detect() (called from this thread —
|
||||
the callback must marshal to whatever loop/UI it needs).
|
||||
"""
|
||||
|
||||
def __init__(self, on_detect: Callable[[], None],
|
||||
threshold: float = DEFAULT_THRESHOLD,
|
||||
logger: Callable[[str], None] = print):
|
||||
self._on_detect = on_detect
|
||||
self._threshold = threshold
|
||||
self._logger = logger
|
||||
self._queue: queue.Queue = queue.Queue(maxsize=50)
|
||||
self._thread: threading.Thread | None = None
|
||||
self._running = False
|
||||
self._model = None
|
||||
self._ready = False
|
||||
|
||||
def start(self) -> bool:
|
||||
"""Load the model and spawn the inference thread. Returns True on success.
|
||||
Safe to call again — a no-op if already running. Never raises."""
|
||||
if self._running:
|
||||
return True
|
||||
try:
|
||||
from openwakeword.model import Model
|
||||
self._model = Model(wakeword_models=[WAKE_MODEL], inference_framework="onnx")
|
||||
except Exception as e:
|
||||
self._logger(f"Wake word: could not load model — {e}")
|
||||
self._model = None
|
||||
return False
|
||||
self._running = True
|
||||
self._ready = True
|
||||
self._thread = threading.Thread(target=self._loop, daemon=True, name="WakeWordThread")
|
||||
self._thread.start()
|
||||
self._logger("Wake word: listening for 'Hey Jarvis'.")
|
||||
return True
|
||||
|
||||
def stop(self) -> None:
|
||||
self._running = False
|
||||
# unblock the thread if it's waiting on the queue
|
||||
try:
|
||||
self._queue.put_nowait(None)
|
||||
except Exception:
|
||||
pass
|
||||
self._model = None
|
||||
self._ready = False
|
||||
|
||||
@property
|
||||
def ready(self) -> bool:
|
||||
return self._ready
|
||||
|
||||
def feed(self, frame_int16) -> None:
|
||||
"""Called from the mic callback (real-time thread). Must stay cheap and
|
||||
never block — the frame is copied and dropped if the queue is backed up."""
|
||||
if not self._running:
|
||||
return
|
||||
try:
|
||||
# frame_int16 is a numpy int16 array (possibly 2-D mono) — flatten to 1-D
|
||||
data = frame_int16[:, 0].copy() if getattr(frame_int16, "ndim", 1) > 1 else frame_int16.copy()
|
||||
self._queue.put_nowait(data)
|
||||
except queue.Full:
|
||||
pass
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def _loop(self) -> None:
|
||||
import numpy as np
|
||||
while self._running:
|
||||
try:
|
||||
frame = self._queue.get()
|
||||
if frame is None or not self._running:
|
||||
break
|
||||
scores = self._model.predict(np.asarray(frame, dtype=np.int16))
|
||||
score = 0.0
|
||||
if isinstance(scores, dict):
|
||||
# match the jarvis model regardless of exact key suffix
|
||||
for k, v in scores.items():
|
||||
if "jarvis" in k.lower():
|
||||
score = max(score, float(v))
|
||||
if score == 0.0 and scores:
|
||||
score = max(float(v) for v in scores.values())
|
||||
if score >= self._threshold:
|
||||
# drain any backlog so we don't double-fire on the same utterance
|
||||
self._drain()
|
||||
try:
|
||||
self._on_detect()
|
||||
except Exception as e:
|
||||
self._logger(f"Wake word: on_detect error — {e}")
|
||||
except Exception as e:
|
||||
self._logger(f"Wake word: inference error — {e}")
|
||||
|
||||
def _drain(self) -> None:
|
||||
try:
|
||||
while True:
|
||||
self._queue.get_nowait()
|
||||
except Exception:
|
||||
pass
|
||||
@@ -360,93 +360,6 @@ def _local_ip() -> str:
|
||||
return "127.0.0.1"
|
||||
|
||||
|
||||
def _ensure_certs() -> bool:
|
||||
"""
|
||||
Make sure config/certs holds a TLS key pair, generating a self-signed one the
|
||||
first time the dashboard runs.
|
||||
|
||||
The pair is deliberately NOT shipped in the repository. A private key that
|
||||
every user downloads is the same as having no private key at all: anyone can
|
||||
present a certificate that matches it. Generating locally gives each install
|
||||
its own key, costs about a second, and happens exactly once.
|
||||
|
||||
Returns True when a usable pair exists afterwards; False leaves the caller on
|
||||
plain HTTP, which still works — the QR code simply encodes http:// instead.
|
||||
"""
|
||||
certs = BASE_DIR / "config" / "certs"
|
||||
key_p = certs / "jarvis.key"
|
||||
crt_p = certs / "jarvis.crt"
|
||||
if key_p.exists() and crt_p.exists():
|
||||
return True
|
||||
|
||||
try:
|
||||
import datetime
|
||||
import ipaddress
|
||||
from cryptography import x509
|
||||
from cryptography.hazmat.primitives import hashes, serialization
|
||||
from cryptography.hazmat.primitives.asymmetric import rsa
|
||||
from cryptography.x509.oid import NameOID
|
||||
except ImportError:
|
||||
print("[Dashboard] cryptography not installed — serving over plain HTTP.")
|
||||
print("[Dashboard] For HTTPS run: pip install cryptography")
|
||||
return False
|
||||
|
||||
try:
|
||||
certs.mkdir(parents=True, exist_ok=True)
|
||||
key = rsa.generate_private_key(public_exponent=65537, key_size=2048)
|
||||
|
||||
who = x509.Name([
|
||||
x509.NameAttribute(NameOID.COMMON_NAME, "JARVIS Dashboard"),
|
||||
x509.NameAttribute(NameOID.ORGANIZATION_NAME, "JARVIS"),
|
||||
])
|
||||
|
||||
# The SAN has to cover every address the phone might use: the LAN IP the
|
||||
# QR code encodes, plus localhost when testing on the machine itself.
|
||||
alt = [x509.DNSName("localhost"),
|
||||
x509.IPAddress(ipaddress.IPv4Address("127.0.0.1"))]
|
||||
try:
|
||||
lan = _local_ip()
|
||||
if not lan.startswith("127."):
|
||||
alt.append(x509.IPAddress(ipaddress.IPv4Address(lan)))
|
||||
except Exception:
|
||||
pass # no LAN address resolvable — localhost entries still work
|
||||
|
||||
# Timezone-aware UTC: datetime.utcnow() is deprecated from Python 3.12 on,
|
||||
# and the builder normalises aware values to UTC itself.
|
||||
now = datetime.datetime.now(datetime.timezone.utc)
|
||||
cert = (
|
||||
x509.CertificateBuilder()
|
||||
.subject_name(who)
|
||||
.issuer_name(who)
|
||||
.public_key(key.public_key())
|
||||
.serial_number(x509.random_serial_number())
|
||||
.not_valid_before(now - datetime.timedelta(days=1))
|
||||
.not_valid_after(now + datetime.timedelta(days=3650))
|
||||
.add_extension(x509.SubjectAlternativeName(alt), critical=False)
|
||||
.add_extension(x509.BasicConstraints(ca=False, path_length=None), critical=True)
|
||||
.sign(key, hashes.SHA256())
|
||||
)
|
||||
|
||||
key_p.write_bytes(key.private_bytes(
|
||||
encoding=serialization.Encoding.PEM,
|
||||
format=serialization.PrivateFormat.TraditionalOpenSSL,
|
||||
encryption_algorithm=serialization.NoEncryption(),
|
||||
))
|
||||
crt_p.write_bytes(cert.public_bytes(serialization.Encoding.PEM))
|
||||
|
||||
try:
|
||||
import os as _os
|
||||
_os.chmod(key_p, 0o600) # best effort — largely a no-op on Windows
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
print(f"[Dashboard] Generated a self-signed certificate for this machine: {certs}")
|
||||
return True
|
||||
except Exception as e:
|
||||
print(f"[Dashboard] Certificate generation failed ({e}) — serving over plain HTTP.")
|
||||
return False
|
||||
|
||||
|
||||
def _read(name: str) -> str:
|
||||
return (STATIC_DIR / name).read_text(encoding="utf-8")
|
||||
|
||||
@@ -863,9 +776,6 @@ class DashboardServer:
|
||||
# no waiting for UAC dialogs or subprocess timeouts.
|
||||
asyncio.get_event_loop().run_in_executor(None, _ensure_network_access, PORT)
|
||||
|
||||
# Generate the TLS pair on first run so no private key ships in the repo.
|
||||
_ensure_certs()
|
||||
|
||||
use_ssl = self._ssl_enabled()
|
||||
ssl_key = BASE_DIR / "config" / "certs" / "jarvis.key"
|
||||
ssl_cert = BASE_DIR / "config" / "certs" / "jarvis.crt"
|
||||
|
||||
@@ -104,6 +104,23 @@ def save_voice(voice_name: str) -> None:
|
||||
CONFIG_FILE.write_text(json.dumps(data, indent=4), encoding="utf-8")
|
||||
|
||||
|
||||
def get_wake_word_enabled() -> bool:
|
||||
"""Whether local wake-word gating is on (assistant sleeps until 'Hey Jarvis')."""
|
||||
return load_api_keys().get("wake_word_enabled", False)
|
||||
|
||||
|
||||
def save_wake_word_enabled(enabled: bool) -> None:
|
||||
ensure_config_dir()
|
||||
data: dict = {}
|
||||
if CONFIG_FILE.exists():
|
||||
try:
|
||||
data = json.loads(CONFIG_FILE.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
data = {}
|
||||
data["wake_word_enabled"] = bool(enabled)
|
||||
CONFIG_FILE.write_text(json.dumps(data, indent=4), encoding="utf-8")
|
||||
|
||||
|
||||
def get_brief_enabled() -> bool:
|
||||
return load_api_keys().get("morning_brief_enabled", True)
|
||||
|
||||
@@ -167,6 +184,46 @@ def get_plugin_enabled(plugin_name: str) -> bool:
|
||||
return load_api_keys().get("plugins_enabled", {}).get(plugin_name, True)
|
||||
|
||||
|
||||
# ── Per-plugin settings ("tokens" / connection details) ───────────────────────
|
||||
# Generic store so a plugin can declare its own config fields (PLUGIN_SETTINGS)
|
||||
# and the settings UI renders + persists them WITHOUT any core edit — keeping the
|
||||
# drop-in model intact. Values live under plugin_config[<namespace>][<key>].
|
||||
# A namespace defaults to the plugin name, but a suite of plugins (e.g. the
|
||||
# printer control/watchdog/autoeject trio) can share ONE namespace.
|
||||
def get_plugin_config(namespace: str) -> dict:
|
||||
"""All stored values for a namespace (empty dict if none set yet)."""
|
||||
cfg = load_api_keys().get("plugin_config")
|
||||
val = cfg.get(namespace) if isinstance(cfg, dict) else None
|
||||
return dict(val) if isinstance(val, dict) else {}
|
||||
|
||||
|
||||
def get_plugin_setting(namespace: str, key: str, default=None):
|
||||
"""A single value from a namespace, or `default` if unset."""
|
||||
return get_plugin_config(namespace).get(key, default)
|
||||
|
||||
|
||||
def save_plugin_config(namespace: str, values: dict) -> None:
|
||||
"""Merge `values` into a namespace's stored config (read-modify-write, like
|
||||
every other helper here). Only the provided keys are touched."""
|
||||
ensure_config_dir()
|
||||
data: dict = {}
|
||||
if CONFIG_FILE.exists():
|
||||
try:
|
||||
data = json.loads(CONFIG_FILE.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
data = {}
|
||||
pc = data.get("plugin_config")
|
||||
if not isinstance(pc, dict):
|
||||
pc = {}
|
||||
cur = pc.get(namespace)
|
||||
if not isinstance(cur, dict):
|
||||
cur = {}
|
||||
cur.update(values)
|
||||
pc[namespace] = cur
|
||||
data["plugin_config"] = pc
|
||||
CONFIG_FILE.write_text(json.dumps(data, indent=4), encoding="utf-8")
|
||||
|
||||
|
||||
def save_plugin_enabled(plugin_name: str, enabled: bool) -> None:
|
||||
ensure_config_dir()
|
||||
data: dict = {}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
# ⚙️ MARK LII (52)
|
||||
# ⚙️ MARK LIII (53)
|
||||
### The Ultimate Cross-Platform Personal AI Assistant — By FatihMakes
|
||||
|
||||
> 📺 **[Watch the full setup video on YouTube](https://www.youtube.com/@FatihMakes)**
|
||||
@@ -9,9 +9,9 @@ A real-time voice AI that can hear, see, understand, and control your computer
|
||||
|
||||
## ✨ Overview
|
||||
|
||||
MARK LII is the **personalization** release: the assistant that becomes *yours*. Pick the voice it speaks with, tune the colour of the whole HUD, and watch it power on with a boot chime and a swelling animation like a machine coming to life. The interface now breathes with you too — the waveform and the arc-reactor core pulse to your **real** voice while you speak and to JARVIS's own voice while it answers.
|
||||
**MARK LIII is the hands-free & scalable release.** Say **"Hey Jarvis"** and it wakes; stay quiet and it slips back to sleep on its own — while asleep, your microphone never leaves the machine, so an off-hand *"I'll be right there"* to someone in the room no longer sets it off. Under the hood it now runs on the faster **Gemini 3.1 Flash Live** engine, and the moment you ask for something that takes a beat — analysing a file, searching the web — it answers instantly *("On it — going through that now…")* so you never wonder whether it heard you.
|
||||
|
||||
All of that sits on the Mark LI foundation: a plugin engine you extend without ever touching the core, native audio that hears the emotion in your voice, knows when you're not talking to it, and can hold one conversation for hours.
|
||||
It's also built to grow: every skill — bundled or drop-in — now **describes itself in its own file**, so adding a tool is a one-file operation and the core stays lean.
|
||||
|
||||
It's not just an assistant — it's an extension of your digital life.
|
||||
|
||||
@@ -22,9 +22,10 @@ It's not just an assistant — it's an extension of your digital life.
|
||||
### Core Features
|
||||
| Feature | Description |
|
||||
|---|---|
|
||||
| 🎙️ Voice Picker | Choose from 5 native Gemini voices and switch live from the UI — no restart |
|
||||
| 🎨 Live Theming | Recolour the entire HUD from a hue wheel or hex — applied instantly across every panel |
|
||||
| 〰️ Reactive HUD | Waveform and reactor core pulse to real audio — your mic while listening, JARVIS while speaking |
|
||||
| 🎙️ Wake Word | Local **"Hey Jarvis"** detection — sleeps until called, auto-sleeps after 2 min of silence, and never streams audio while asleep. Opt-in, one-click download, toggle & manual sleep/wake from the UI |
|
||||
| ⚡ Instant Acknowledgment | Speaks a short, context-aware reply in **your language** the instant a longer task starts — no more silent waiting |
|
||||
| 🚀 Faster Live Engine | Runs on **Gemini 3.1 Flash Live** — roughly 2× faster time-to-first-word than the previous model |
|
||||
| 🧩 Self-Describing Skills | Actions and plugins share one shape (`TOOL` / `PLUGIN` dict + `run()`), auto-discovered at launch — adding or moving a skill is a single file, no core edits |
|
||||
| 🧠 Recallable Memory | No size limit and nothing silently forgotten — the prompt carries what fits, the rest is looked up on demand from a local search |
|
||||
| 👁️ Memory Panel | See every fact JARVIS has stored about you, when it learned it, and delete any of it in one click |
|
||||
| ↩️ Undo | Take back what the assistant did — files it moved, renamed, created or wrote, and settings it changed |
|
||||
@@ -33,8 +34,9 @@ It's not just an assistant — it's an extension of your digital life.
|
||||
| 🔗 Session Continuity | A dropped connection, a voice change or a device change no longer wipes the conversation |
|
||||
| 🧩 Plugin System | Drop a single `.py` file into `plugins/` — JARVIS learns a new skill on next launch |
|
||||
| 🎙️ Real-time Voice | Ultra-low latency conversation in any language via Gemini Live API |
|
||||
| 💓 Affective Dialog | Hears the emotion in your voice and adapts its tone in response |
|
||||
| 🤫 Proactive Audio | Knows when you're not talking to it — background chatter never triggers a reply |
|
||||
| 🎨 Live Theming | Recolour the entire HUD from a hue wheel or hex — applied instantly across every panel |
|
||||
| 〰️ Reactive HUD | Waveform and reactor core pulse to real audio — your mic while listening, JARVIS while speaking |
|
||||
| 🎙️ Voice Picker | Choose from 5 native Gemini voices and switch live from the UI — no restart |
|
||||
| ♾️ Unlimited Sessions | Sliding-window context compression — one conversation can last for hours |
|
||||
| 🖥️ System Control | Launch apps, adjust volume/brightness, WiFi, shortcuts, power — all by voice |
|
||||
| 🧩 Autonomous Tasks | High-level planning for complex multi-step goals via agent mode |
|
||||
@@ -66,22 +68,23 @@ It's not just an assistant — it's an extension of your digital life.
|
||||
|
||||
---
|
||||
|
||||
## 🆕 What's New in Mark LII
|
||||
## 🆕 What's New in Mark LIII
|
||||
|
||||
Mark LII is about making JARVIS feel like *your own* machine. Four small-but-delightful upgrades — all universal: no hardcoded values, no bundled asset files, and no assumptions about your language or operating system.
|
||||
Mark LIII is about making JARVIS **hands-free, faster, and easy to extend** — all universal: no hardcoded language, no bundled asset files, works the same on Windows, macOS and Linux.
|
||||
|
||||
### 🎙️ Voice Picker — Give JARVIS the Voice You Want
|
||||
JARVIS is no longer stuck with one voice. Open **⚙ Customise Assistant** and choose between five native Gemini voices — **Charon, Puck, Kore, Fenrir, Aoede** — each with its own character. The switch is live: the session rebuilds itself the instant you apply, so the new voice takes over without you restarting anything, and session resumption keeps your conversation going. The voice names are language-neutral, so the picker reads the same in every locale.
|
||||
### 🎙️ Wake Word — "Hey Jarvis"
|
||||
JARVIS can now sit quietly until you call it. Turn on **⚙ → WAKE WORD** (a one-click, opt-in download of a tiny local model) and it goes to sleep: the microphone is processed **only on your machine** by a local detector, and nothing is sent to the cloud until it hears **"Hey Jarvis."** Once awake it listens normally, then **auto-sleeps after 2 minutes** of silence. You can also **sleep/wake it by clicking** in the settings. Because it's a *local* gate, background chatter — *"I'm coming!"* to someone at home — never wakes it. It costs **zero** when off (the model isn't even loaded), and the detection runs in its own thread, so nothing else in the app slows down.
|
||||
|
||||
### 🎨 Live Theming — Recolour the Entire Interface
|
||||
Drag the hue wheel (or type an exact hex code) and the whole HUD re-themes in real time — panels, borders, the reactor core, the waveform, every button. Make it classic arc-reactor cyan, Iron Man gold, hostile red, or anything in between. Your choice is saved and restored on the next launch.
|
||||
### ⚡ Instant Acknowledgment
|
||||
No more silent gaps. When you ask for something that takes a moment — reading an uploaded file, a web/research search, building code — JARVIS **immediately** says one short, natural sentence *in your language* (*"Right away — going through that file now."*) and *then* runs the tool. Instant actions (opening an app, volume) stay snappy with no chatter.
|
||||
|
||||
### 〰️ Reactive HUD — The Interface Breathes With the Room
|
||||
The waveform and the arc-reactor core now respond to **real audio**, not a random animation. While JARVIS listens, they pulse to your microphone; while JARVIS speaks, they pulse to its own voice — louder speech, taller bars and a brighter, wider core. When the room goes quiet, everything settles back into a gentle idle ripple. It makes the assistant feel genuinely alive and connected to what's happening.
|
||||
### 🚀 Faster Live Engine — Gemini 3.1 Flash Live
|
||||
The live session moved to **`gemini-3.1-flash-live-preview`**, cutting the time-to-first-word roughly in half while keeping tools, all five voices, transcription, session resumption and sliding-window compression intact.
|
||||
|
||||
Every launch now opens with a proper boot: a ~2.4-second cinematic **transform** sound — a reactor spinning up, servos locking into place, and a bright chord confirming "online" — plays as the HUD swells up from a dim point, rings spin up, and a bright pulse sweeps outward. It's synthesized entirely in code (no sound file to ship, identical on Windows, macOS and Linux), and you can turn it on or off any time from the **🔊 BOOT SOUND** toggle in the ⚙ controls. If a machine has no audio output, it simply stays silent — never an error.
|
||||
### 🧩 Self-Describing Skills — a Scalable Core
|
||||
Every bundled **action** now carries its own `TOOL` declaration in its own file (exactly like a drop-in **plugin's** `PLUGIN` dict), and the core auto-discovers them at launch. `main.py` no longer holds a giant list of tool definitions and dispatch branches — it shrank by hundreds of lines. Adding a new built-in skill, or promoting an `actions/*.py` file into a shareable plugin, is now just… moving a file.
|
||||
|
||||
> Built on Mark LI's foundation: the **🧩 Plugin System** (extend JARVIS with a single drop-in file), **💓 Affective Dialog**, **🤫 Proactive Audio**, and **♾️ Unlimited Sessions** are all still here and unchanged.
|
||||
> Built on the Mark LI/LII foundation: the **🧩 Plugin System**, **♾️ Unlimited Sessions**, **🎨 Live Theming**, **〰️ Reactive HUD** and **🎙️ Voice Picker** are all still here.
|
||||
|
||||
---
|
||||
|
||||
@@ -98,7 +101,7 @@ The store was capped at **2,200 characters — the whole memory, not per entry**
|
||||
Storage and prompt budget are now separate problems:
|
||||
|
||||
* **Nothing is deleted.** The cap is a runaway guard normal use never approaches, and if it is ever hit it says so in the activity log instead of on stdout.
|
||||
* **The prompt carries a core, not a dump.** Identity in full, then the most recently updated facts, budgeted — measured at **under 1,000 characters on a memory holding 61 stored facts.** That is *smaller* than the old whole-store cap, so sessions now connect with fewer tokens than before.
|
||||
* **The prompt carries a core, not a dump.** Identity in full, then the most recently updated facts, budgeted — measured at **971 characters on a memory holding 62 stored facts.** That is *smaller* than the old whole-store cap, so sessions now connect with fewer tokens than before.
|
||||
* **The rest is fetched on demand.** A `recall_memory` tool searches the full store locally — no network, no second model, well under a millisecond.
|
||||
|
||||
The part that is easy to get wrong: **a model cannot look something up if it doesn't know the thing exists.** So the prompt also carries an **index of the keys** it had no room for. Without it, "who is Ayşe?" gets "I don't know" while `ayse_sister` sits on disk unread. That index interleaves categories rather than sorting by recency — sorted like the core, a memory with forty preferences pushed the one entry the index existed for off the end.
|
||||
@@ -186,25 +189,28 @@ It is held in memory only, deliberately: writing it to disk would make a fresh l
|
||||
|
||||
| Mark | Focus |
|
||||
|---|---|
|
||||
| **XLVIII** | Instant interrupt · parallel news · two-phase briefing · exponential backoff · vision cooldown |
|
||||
| **XLIX** | Auto-start · clipboard intelligence · assistant customization |
|
||||
| **L** | Session memory · background monitoring · proactive 2.0 · instant vision · parallel news search |
|
||||
| **L** | Session memory · background monitoring · proactive 2.0 · instant vision |
|
||||
| **LI** | Plugin system · affective dialog · proactive audio · unlimited sessions |
|
||||
| **LII** | Voice picker · live theming · reactive HUD |
|
||||
| **LIII+** | Plugin files: email · quiz mode · calendar · and more |
|
||||
| **LII** | Voice picker · live theming · reactive HUD · recallable memory · undo · real confirmation · audio device picker · session continuity |
|
||||
| **LIII** | Wake word · Gemini 3.1 Flash Live · instant acknowledgment · self-describing action/plugin architecture |
|
||||
| *shared* | The last five above also shipped to LIII, LIV and LV at the same time — moving up a Mark never loses them |
|
||||
| **LIV+** | Plugin files: email · quiz mode · calendar · home assistant · 3D-printer · and more |
|
||||
|
||||
---
|
||||
|
||||
## ⚡ Quick Start
|
||||
|
||||
```bash
|
||||
git clone https://github.com/FatihMakes/Mark-LII.git
|
||||
cd Mark-LII
|
||||
pip install -r requirements.txt
|
||||
git clone https://github.com/FatihMakes/Mark-LIII.git
|
||||
cd Mark-LIII
|
||||
python setup.py # installs deps for YOUR OS + the browser automation engine
|
||||
python main.py
|
||||
```
|
||||
|
||||
> ⚠️ **Installation Note:** Some OS-specific dependencies are not bundled in `requirements.txt` to keep the repo lightweight. If you hit a `ModuleNotFoundError`, install the missing package with `pip install <module_name>`.
|
||||
`setup.py` only ever installs what your operating system needs — the Windows-only libraries are skipped automatically on macOS and Linux (and vice-versa). Prefer to do it by hand? `pip install -r requirements.txt` works too.
|
||||
|
||||
> ⚠️ **Installation Note:** If you hit a `ModuleNotFoundError` for an OS-specific package, install it with `pip install <module_name>`. The optional **wake word** engine is *not* installed here — grab it in one click from **⚙ → WAKE WORD** inside the app.
|
||||
|
||||
---
|
||||
|
||||
@@ -214,31 +220,33 @@ python main.py
|
||||
| --- | --- |
|
||||
| **OS** | Windows 10/11, macOS, or Linux |
|
||||
| **Python** | 3.11 or 3.12 |
|
||||
| **Microphone** | Required for voice interaction |
|
||||
| **Microphone** | Required for voice interaction (and for the "Hey Jarvis" wake word) |
|
||||
| **Speakers** | Required for voice replies |
|
||||
| **API Key** | Free Gemini API key (`config/api_keys.json`) |
|
||||
| **API Key** | Free Gemini API key (entered on first launch → `config/api_keys.json`) |
|
||||
| **Wake word** *(optional)* | One-click download from ⚙ → WAKE WORD (`openwakeword`, a few MB, fully local) |
|
||||
|
||||
---
|
||||
|
||||
## 🗂️ Project Structure
|
||||
|
||||
```
|
||||
Mark LII/
|
||||
├── main.py # Core loop — Gemini Live session, audio I/O, live audio levels, tool dispatch
|
||||
├── ui.py # PyQt6 HUD — reactive waveform, boot animation, log panel, plugin manager, camera feed
|
||||
├── setup.py # First-run configuration wizard
|
||||
Mark LIII/
|
||||
├── main.py # Core loop — Gemini Live session, audio I/O, wake/sleep state, tool dispatch
|
||||
├── ui.py # PyQt6 HUD — reactive waveform, log panel, settings drawer, plugin manager, camera feed
|
||||
├── setup.py # OS-aware installer (skips wrong-OS dependencies)
|
||||
├── plugins/
|
||||
│ └── _template.py # Copy this to write a new plugin — one file, drop in, done
|
||||
├── actions/
|
||||
│ ├── _template.py # Copy this to write a new plugin — one file, drop in, done
|
||||
│ └── ... # Drop-in skills (each self-describes via a PLUGIN dict + run())
|
||||
├── actions/ # Bundled skills — each self-describes via a TOOL dict + handler
|
||||
│ ├── web_search.py # Gemini + DDG parallel search (news, research, price, compare)
|
||||
│ ├── screen_processor.py # Screen capture & webcam vision via Gemini Live
|
||||
│ ├── screen_processor.py # Screen & webcam capture for vision
|
||||
│ ├── background_monitor.py # User-configured topic watching — daily DDG check, no crypto
|
||||
│ ├── proactive.py # Proactive 2.0 — time/context/rotation-aware check-ins
|
||||
│ ├── reminder.py # OS-native scheduled notifications
|
||||
│ ├── system_monitor.py # CPU / RAM / GPU / temperature telemetry
|
||||
│ ├── computer_settings.py # Volume, brightness, WiFi, power
|
||||
│ ├── computer_settings.py # Volume, brightness, WiFi, power (per-OS)
|
||||
│ ├── computer_control.py # Keyboard shortcuts, mouse, window management
|
||||
│ ├── open_app.py # Application launcher
|
||||
│ ├── open_app.py # Application launcher (per-OS name map)
|
||||
│ ├── browser_control.py # Web browser control
|
||||
│ ├── file_controller.py # File system operations
|
||||
│ ├── file_processor.py # Document reading and summarization
|
||||
@@ -252,16 +260,18 @@ Mark LII/
|
||||
│ └── desktop.py # Desktop and taskbar control
|
||||
├── memory/
|
||||
│ ├── memory_manager.py # Load/save long_term.json — sessions, monitors, identity
|
||||
│ ├── config_manager.py # api_keys.json access — key, OS, name, voice, colour, plugin toggles
|
||||
│ ├── config_manager.py # api_keys.json access — key, OS, name, voice, colour, toggles
|
||||
│ └── long_term.json # Persistent store: identity, preferences, projects, sessions, monitors
|
||||
├── core/
|
||||
│ ├── prompt.txt # Assistant personality and tool-routing rules
|
||||
│ ├── plugin_loader.py # Plugin engine — discovery, validation, crash isolation
|
||||
│ ├── undo.py # One shared undo stack — actions register how to reverse themselves
|
||||
│ ├── confirm.py # Irreversible-action gate — the token is issued by the UI, not the model
|
||||
│ └── audio_devices.py # Microphone / speaker list — filtered, measured, resolved by name
|
||||
│ ├── audio_devices.py # Microphone / speaker list — filtered, measured, resolved by name
|
||||
│ ├── plugin_loader.py # Plugin engine — discovery, validation, crash isolation
|
||||
│ ├── action_loader.py # Bundled-action engine — the built-in twin of plugin_loader
|
||||
│ └── wake_word.py # Local "Hey Jarvis" detector — own thread, offline, opt-in
|
||||
└── config/
|
||||
└── api_keys.json # API key, OS setting, assistant name, user name, voice, UI colour, audio devices
|
||||
└── api_keys.json # API key, OS setting, assistant name, user name, voice, UI colour, toggles
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
+35
-5
@@ -1,29 +1,59 @@
|
||||
# ── Core: UI, audio, Gemini Live ──────────────────────────────────────────────
|
||||
PyQt6
|
||||
sounddevice
|
||||
numpy
|
||||
google-genai>=2.8.0
|
||||
google-generativeai
|
||||
pillow
|
||||
|
||||
# ── Web search & browser control ──────────────────────────────────────────────
|
||||
requests
|
||||
beautifulsoup4
|
||||
ddgs>=9.0.0
|
||||
duckduckgo-search
|
||||
playwright
|
||||
|
||||
# ── Automation / input control ────────────────────────────────────────────────
|
||||
pyautogui
|
||||
pyperclip
|
||||
pygetwindow
|
||||
|
||||
# ── Vision & media ────────────────────────────────────────────────────────────
|
||||
pillow
|
||||
opencv-python
|
||||
numpy
|
||||
mss
|
||||
|
||||
# ── System, files & documents ─────────────────────────────────────────────────
|
||||
psutil
|
||||
send2trash
|
||||
youtube-transcript-api
|
||||
python-pptx
|
||||
openpyxl
|
||||
|
||||
# ── Remote dashboard (control from your phone) ────────────────────────────────
|
||||
fastapi
|
||||
uvicorn[standard]
|
||||
cryptography
|
||||
python-multipart
|
||||
qrcode[pil]
|
||||
|
||||
# ── Plugin extras: Gmail / Google Calendar (OAuth) ────────────────────────────
|
||||
google-api-python-client
|
||||
google-auth-oauthlib
|
||||
|
||||
# ── Plugin extras: Tuya / Smart Life smart light (home_assistant) ─────────────
|
||||
tinytuya
|
||||
|
||||
# ── Windows-only (pip skips these automatically on macOS / Linux) ─────────────
|
||||
comtypes; sys_platform == "win32"
|
||||
pycaw; sys_platform == "win32"
|
||||
win10toast; sys_platform == "win32"
|
||||
pywinauto; sys_platform == "win32"
|
||||
pywin32; sys_platform == "win32"
|
||||
qrcode[pil]
|
||||
wmi; sys_platform == "win32"
|
||||
|
||||
# ── Optional: local wake word ("Hey Jarvis") ──────────────────────────────────
|
||||
# Deliberately NOT installed by default — it's opt-in. Enable it (and download
|
||||
# the tiny model) with one click from ⚙ → WAKE WORD in the app, or install it
|
||||
# yourself with: pip install openwakeword
|
||||
#
|
||||
# macOS / Linux OS actions (volume, brightness, reminders) shell out to native
|
||||
# tools (pactl / brightnessctl / osascript / systemd), so no extra Python
|
||||
# packages are needed there — nothing Windows-specific is ever pulled in.
|
||||
@@ -1,26 +1,72 @@
|
||||
"""
|
||||
MARK LIII — one-time setup.
|
||||
|
||||
Installs the Python dependencies for THIS operating system only: the OS-specific
|
||||
packages in requirements.txt carry `sys_platform` markers, so a macOS or Linux
|
||||
user never pulls Windows-only libraries (and vice-versa). Then it fetches the
|
||||
Playwright browsers needed for web automation (current-OS builds only).
|
||||
|
||||
The optional local wake word ("Hey Jarvis") is NOT installed here — it's a
|
||||
one-click, opt-in download from ⚙ → WAKE WORD inside the app.
|
||||
"""
|
||||
import platform
|
||||
import subprocess
|
||||
import sys
|
||||
import platform
|
||||
from pathlib import Path
|
||||
|
||||
print("Installing requirements...")
|
||||
subprocess.run([sys.executable, "-m", "pip", "install", "-r", "requirements.txt"], check=True)
|
||||
OS = platform.system() # "Windows" | "Darwin" | "Linux"
|
||||
|
||||
print("Installing Playwright browsers...")
|
||||
subprocess.run([sys.executable, "-m", "playwright", "install"], check=True)
|
||||
|
||||
if platform.system() == "Windows":
|
||||
def _run(label: str, args: list[str]) -> None:
|
||||
print(f"\n▶ {label}")
|
||||
subprocess.run(args, check=True)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
print(f"⚙ MARK LIII setup — detected OS: {OS or 'unknown'}")
|
||||
|
||||
# requirements.txt filters OS-specific extras by itself via pip markers.
|
||||
_run("Installing Python dependencies (OS-specific extras auto-filtered)…",
|
||||
[sys.executable, "-m", "pip", "install", "-r", "requirements.txt"])
|
||||
|
||||
# Chromium covers Chrome/Edge/Opera/Brave/Vivaldi; Firefox for Firefox.
|
||||
# (Safari automation additionally needs: python -m playwright install webkit)
|
||||
_run("Installing Playwright browsers (chromium + firefox)…",
|
||||
[sys.executable, "-m", "playwright", "install", "chromium", "firefox"])
|
||||
|
||||
# ── OS-specific post-install notes ────────────────────────────────────────
|
||||
if OS == "Windows":
|
||||
try:
|
||||
import win32com.client # noqa: F401
|
||||
except ImportError:
|
||||
postinstall = Path(sys.executable).parent / "Scripts" / "pywin32_postinstall.py"
|
||||
print(
|
||||
"\n⚠️ pywin32 did not install correctly — desktop shortcut creation "
|
||||
"will fall back to a slower method that may not work on this machine.\n"
|
||||
" Try fixing it manually with:\n"
|
||||
"\n⚠️ pywin32 did not register correctly — desktop-shortcut "
|
||||
"creation will use a slower fallback. To fix it, run:\n"
|
||||
f' "{sys.executable}" -m pip install --force-reinstall pywin32\n'
|
||||
f' "{sys.executable}" "{postinstall}" -install\n'
|
||||
f' "{sys.executable}" "{postinstall}" -install'
|
||||
)
|
||||
elif OS == "Linux":
|
||||
print(
|
||||
"\nℹ️ Linux note — a few voice-controlled OS actions shell out to "
|
||||
"native tools. Install the ones you'll use via your package manager:\n"
|
||||
" • volume → pulseaudio-utils (pactl)\n"
|
||||
" • brightness → brightnessctl\n"
|
||||
" • reminders → systemd (systemd-run) or 'at'\n"
|
||||
" • open URLs → xdg-utils (xdg-open)"
|
||||
)
|
||||
elif OS == "Darwin":
|
||||
print(
|
||||
"\nℹ️ macOS note — volume, brightness and reminders use the built-in "
|
||||
"'osascript' / LaunchAgents, so no extra tools are required.\n"
|
||||
" For Safari automation only: python -m playwright install webkit"
|
||||
)
|
||||
|
||||
print("\n✅ Setup complete! Run 'python main.py' to start MARK LII.")
|
||||
print("\n✅ Setup complete!")
|
||||
print(" 1) Launch it: python main.py")
|
||||
print(" 2) Paste your free Gemini API key when the setup screen appears.")
|
||||
print(" 3) (Optional) Enable 'Hey Jarvis' from ⚙ → WAKE WORD.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in new issue
Block a user