Files
Mark-LIV/main.py
T

820 lines
37 KiB
Python
Raw Normal View History

2026-03-02 04:30:10 +03:00
import asyncio
2026-04-19 15:22:49 +03:00
import re
2026-03-02 04:30:10 +03:00
import threading
import json
import sys
import traceback
from pathlib import Path
2026-04-19 15:22:49 +03:00
import sounddevice as sd
2026-03-02 04:30:10 +03:00
from google import genai
from google.genai import types
from ui import JarvisUI
2026-04-19 15:22:49 +03:00
from memory.memory_manager import (
load_memory, update_memory, format_memory_for_prompt,
)
from actions.flight_finder import flight_finder
from actions.open_app import open_app
from actions.weather_report import weather_action
from actions.send_message import send_message
from actions.reminder import reminder
2026-03-02 04:30:10 +03:00
from actions.computer_settings import computer_settings
2026-04-19 15:22:49 +03:00
from actions.screen_processor import screen_process
from actions.youtube_video import youtube_video
from actions.desktop import desktop_control
from actions.browser_control import browser_control
from actions.file_controller import file_controller
from actions.code_helper import code_helper
from actions.dev_agent import dev_agent
from actions.web_search import web_search as web_search_action
from actions.computer_control import computer_control
from actions.game_updater import game_updater
2026-03-02 04:30:10 +03:00
def get_base_dir():
if getattr(sys, "frozen", False):
return Path(sys.executable).parent
return Path(__file__).resolve().parent
2026-04-19 15:22:49 +03:00
2026-03-02 04:30:10 +03:00
BASE_DIR = get_base_dir()
API_CONFIG_PATH = BASE_DIR / "config" / "api_keys.json"
PROMPT_PATH = BASE_DIR / "core" / "prompt.txt"
LIVE_MODEL = "models/gemini-2.5-flash-native-audio-preview-12-2025"
CHANNELS = 1
SEND_SAMPLE_RATE = 16000
RECEIVE_SAMPLE_RATE = 24000
CHUNK_SIZE = 1024
def _get_api_key() -> str:
with open(API_CONFIG_PATH, "r", encoding="utf-8") as f:
return json.load(f)["gemini_api_key"]
2026-04-19 15:22:49 +03:00
2026-03-02 04:30:10 +03:00
def _load_system_prompt() -> str:
try:
return PROMPT_PATH.read_text(encoding="utf-8")
except Exception:
return (
"You are JARVIS, Tony Stark's AI assistant. "
"Be concise, direct, and always use the provided tools to complete tasks. "
"Never simulate or guess results — always call the appropriate tool."
)
2026-04-19 15:22:49 +03:00
# ── Transkripsiyon temizleyici ─────────────────────────────────────────────────
_CTRL_RE = re.compile(r"<ctrl\d+>", re.IGNORECASE)
2026-03-02 04:30:10 +03:00
2026-04-19 15:22:49 +03:00
def _clean_transcript(text: str) -> str:
"""Gemini'nin ürettiği <ctrlXX> artefaktlarını ve kontrol karakterlerini temizler."""
text = _CTRL_RE.sub("", text)
text = re.sub(r"[\x00-\x08\x0b-\x1f]", "", text)
return text.strip()
2026-03-02 04:30:10 +03:00
2026-04-19 15:22:49 +03:00
# ── Tool declarations ──────────────────────────────────────────────────────────
2026-03-02 04:30:10 +03:00
TOOL_DECLARATIONS = [
{
"name": "open_app",
"description": (
2026-04-19 15:22:49 +03:00
"Opens any application on the computer. "
2026-03-02 04:30:10 +03:00
"Use this whenever the user asks to open, launch, or start any app, "
"website, or program. Always call this tool — never just say you opened it."
),
"parameters": {
"type": "OBJECT",
"properties": {
"app_name": {
"type": "STRING",
"description": "Exact name of the application (e.g. 'WhatsApp', 'Chrome', 'Spotify')"
}
},
"required": ["app_name"]
}
},
2026-04-19 15:22:49 +03:00
{
"name": "web_search",
"description": "Searches the web for any information.",
"parameters": {
"type": "OBJECT",
"properties": {
"query": {"type": "STRING", "description": "Search query"},
"mode": {"type": "STRING", "description": "search (default) or compare"},
"items": {"type": "ARRAY", "items": {"type": "STRING"}, "description": "Items to compare"},
"aspect": {"type": "STRING", "description": "price | specs | reviews"}
},
"required": ["query"]
}
},
2026-03-02 04:30:10 +03:00
{
"name": "weather_report",
2026-04-19 15:22:49 +03:00
"description": "Gives the weather report to user",
2026-03-02 04:30:10 +03:00
"parameters": {
"type": "OBJECT",
"properties": {
"city": {"type": "STRING", "description": "City name"}
},
"required": ["city"]
}
},
{
"name": "send_message",
"description": "Sends a text message via WhatsApp, Telegram, or other messaging platform.",
"parameters": {
"type": "OBJECT",
"properties": {
"receiver": {"type": "STRING", "description": "Recipient contact name"},
"message_text": {"type": "STRING", "description": "The message to send"},
"platform": {"type": "STRING", "description": "Platform: WhatsApp, Telegram, etc."}
},
"required": ["receiver", "message_text", "platform"]
}
},
{
"name": "reminder",
2026-04-19 15:22:49 +03:00
"description": "Sets a timed reminder using Task Scheduler.",
2026-03-02 04:30:10 +03:00
"parameters": {
"type": "OBJECT",
"properties": {
"date": {"type": "STRING", "description": "Date in YYYY-MM-DD format"},
"time": {"type": "STRING", "description": "Time in HH:MM format (24h)"},
"message": {"type": "STRING", "description": "Reminder message text"}
},
"required": ["date", "time", "message"]
}
},
{
2026-04-19 15:22:49 +03:00
"name": "youtube_video",
"description": (
"Controls YouTube. Use for: playing videos, summarizing a video's content, "
"getting video info, or showing trending videos."
),
"parameters": {
"type": "OBJECT",
"properties": {
"action": {"type": "STRING", "description": "play | summarize | get_info | trending (default: play)"},
"query": {"type": "STRING", "description": "Search query for play action"},
"save": {"type": "BOOLEAN", "description": "Save summary to Notepad (summarize only)"},
"region": {"type": "STRING", "description": "Country code for trending e.g. TR, US"},
"url": {"type": "STRING", "description": "Video URL for get_info action"},
2026-03-02 04:30:10 +03:00
},
2026-04-19 15:22:49 +03:00
"required": []
}
2026-03-02 04:30:10 +03:00
},
{
"name": "screen_process",
"description": (
"Captures and analyzes the screen or webcam image. "
"MUST be called when user asks what is on screen, what you see, "
"analyze my screen, look at camera, etc. "
"You have NO visual ability without this tool. "
"After calling this tool, stay SILENT — the vision module speaks directly."
),
"parameters": {
"type": "OBJECT",
"properties": {
2026-04-19 15:22:49 +03:00
"angle": {"type": "STRING", "description": "'screen' to capture display, 'camera' for webcam. Default: 'screen'"},
"text": {"type": "STRING", "description": "The question or instruction about the captured image"}
2026-03-02 04:30:10 +03:00
},
"required": ["text"]
}
},
{
2026-04-19 15:22:49 +03:00
"name": "computer_settings",
"description": (
"Controls the computer: volume, brightness, window management, keyboard shortcuts, "
"typing text on screen, closing apps, fullscreen, dark mode, WiFi, restart, shutdown, "
"scrolling, tab management, zoom, screenshots, lock screen, refresh/reload page. "
"Use for ANY single computer control command. NEVER route to agent_task."
),
"parameters": {
"type": "OBJECT",
"properties": {
"action": {"type": "STRING", "description": "The action to perform"},
"description": {"type": "STRING", "description": "Natural language description of what to do"},
"value": {"type": "STRING", "description": "Optional value: volume level, text to type, etc."}
},
"required": []
}
},
2026-03-02 04:30:10 +03:00
{
"name": "browser_control",
"description": (
2026-04-19 15:22:49 +03:00
"Controls any web browser. Use for: opening websites, searching the web, "
"clicking elements, filling forms, scrolling, screenshots, navigation, any web-based task. "
"Always pass the 'browser' parameter when the user specifies a browser (e.g. 'open in Edge', "
"'use Firefox', 'open Chrome'). Multiple browsers can run simultaneously."
2026-03-02 04:30:10 +03:00
),
"parameters": {
"type": "OBJECT",
"properties": {
2026-04-19 15:22:49 +03:00
"action": {"type": "STRING", "description": "go_to | search | click | type | scroll | fill_form | smart_click | smart_type | get_text | get_url | press | new_tab | close_tab | screenshot | back | forward | reload | switch | list_browsers | close | close_all"},
"browser": {"type": "STRING", "description": "Target browser: chrome | edge | firefox | opera | operagx | brave | vivaldi | safari. Omit to use the currently active browser."},
"url": {"type": "STRING", "description": "URL for go_to / new_tab action"},
2026-03-02 04:30:10 +03:00
"query": {"type": "STRING", "description": "Search query for search action"},
2026-04-19 15:22:49 +03:00
"engine": {"type": "STRING", "description": "Search engine: google | bing | duckduckgo | yandex (default: google)"},
2026-03-02 04:30:10 +03:00
"selector": {"type": "STRING", "description": "CSS selector for click/type"},
"text": {"type": "STRING", "description": "Text to click or type"},
"description": {"type": "STRING", "description": "Element description for smart_click/smart_type"},
2026-04-19 15:22:49 +03:00
"direction": {"type": "STRING", "description": "up | down for scroll"},
"amount": {"type": "INTEGER", "description": "Scroll amount in pixels (default: 500)"},
"key": {"type": "STRING", "description": "Key name for press action (e.g. Enter, Escape, F5)"},
"path": {"type": "STRING", "description": "Save path for screenshot"},
"incognito": {"type": "BOOLEAN", "description": "Open in private/incognito mode"},
"clear_first": {"type": "BOOLEAN", "description": "Clear field before typing (default: true)"},
2026-03-02 04:30:10 +03:00
},
"required": ["action"]
}
},
{
"name": "file_controller",
2026-04-19 15:22:49 +03:00
"description": "Manages files and folders: list, create, delete, move, copy, rename, read, write, find, disk usage.",
2026-03-02 04:30:10 +03:00
"parameters": {
"type": "OBJECT",
"properties": {
"action": {"type": "STRING", "description": "list | create_file | create_folder | delete | move | copy | rename | read | write | find | largest | disk_usage | organize_desktop | info"},
"path": {"type": "STRING", "description": "File/folder path or shortcut: desktop, downloads, documents, home"},
"destination": {"type": "STRING", "description": "Destination path for move/copy"},
"new_name": {"type": "STRING", "description": "New name for rename"},
"content": {"type": "STRING", "description": "Content for create_file/write"},
"name": {"type": "STRING", "description": "File name to search for"},
"extension": {"type": "STRING", "description": "File extension to search (e.g. .pdf)"},
"count": {"type": "INTEGER", "description": "Number of results for largest"},
},
"required": ["action"]
}
},
{
2026-04-19 15:22:49 +03:00
"name": "desktop_control",
"description": "Controls the desktop: wallpaper, organize, clean, list, stats.",
"parameters": {
"type": "OBJECT",
"properties": {
"action": {"type": "STRING", "description": "wallpaper | wallpaper_url | organize | clean | list | stats | task"},
"path": {"type": "STRING", "description": "Image path for wallpaper"},
"url": {"type": "STRING", "description": "Image URL for wallpaper_url"},
"mode": {"type": "STRING", "description": "by_type or by_date for organize"},
"task": {"type": "STRING", "description": "Natural language desktop task"},
},
"required": ["action"]
}
},
{
"name": "code_helper",
"description": "Writes, edits, explains, runs, or builds code files.",
"parameters": {
"type": "OBJECT",
"properties": {
"action": {"type": "STRING", "description": "write | edit | explain | run | build | auto (default: auto)"},
"description": {"type": "STRING", "description": "What the code should do or what change to make"},
"language": {"type": "STRING", "description": "Programming language (default: python)"},
"output_path": {"type": "STRING", "description": "Where to save the file"},
"file_path": {"type": "STRING", "description": "Path to existing file for edit/explain/run/build"},
"code": {"type": "STRING", "description": "Raw code string for explain"},
"args": {"type": "STRING", "description": "CLI arguments for run/build"},
"timeout": {"type": "INTEGER", "description": "Execution timeout in seconds (default: 30)"},
},
"required": ["action"]
}
},
{
"name": "dev_agent",
"description": "Builds complete multi-file projects from scratch: plans, writes files, installs deps, opens VSCode, runs and fixes errors.",
"parameters": {
"type": "OBJECT",
"properties": {
"description": {"type": "STRING", "description": "What the project should do"},
"language": {"type": "STRING", "description": "Programming language (default: python)"},
"project_name": {"type": "STRING", "description": "Optional project folder name"},
"timeout": {"type": "INTEGER", "description": "Run timeout in seconds (default: 30)"},
},
"required": ["description"]
}
},
{
"name": "agent_task",
2026-03-02 04:30:10 +03:00
"description": (
2026-04-19 15:22:49 +03:00
"Executes complex multi-step tasks requiring multiple different tools. "
"Examples: 'research X and save to file', 'find and organize files'. "
"DO NOT use for single commands. NEVER use for Steam/Epic — use game_updater."
2026-03-02 04:30:10 +03:00
),
"parameters": {
"type": "OBJECT",
"properties": {
2026-04-19 15:22:49 +03:00
"goal": {"type": "STRING", "description": "Complete description of what to accomplish"},
"priority": {"type": "STRING", "description": "low | normal | high (default: normal)"}
2026-03-02 04:30:10 +03:00
},
2026-04-19 15:22:49 +03:00
"required": ["goal"]
2026-03-02 04:30:10 +03:00
}
},
{
2026-04-19 15:22:49 +03:00
"name": "computer_control",
"description": "Direct computer control: type, click, hotkeys, scroll, move mouse, screenshots, find elements on screen.",
"parameters": {
"type": "OBJECT",
"properties": {
"action": {"type": "STRING", "description": "type | smart_type | click | double_click | right_click | hotkey | press | scroll | move | copy | paste | screenshot | wait | clear_field | focus_window | screen_find | screen_click | random_data | user_data"},
"text": {"type": "STRING", "description": "Text to type or paste"},
"x": {"type": "INTEGER", "description": "X coordinate"},
"y": {"type": "INTEGER", "description": "Y coordinate"},
"keys": {"type": "STRING", "description": "Key combination e.g. 'ctrl+c'"},
"key": {"type": "STRING", "description": "Single key e.g. 'enter'"},
"direction": {"type": "STRING", "description": "up | down | left | right"},
"amount": {"type": "INTEGER", "description": "Scroll amount (default: 3)"},
"seconds": {"type": "NUMBER", "description": "Seconds to wait"},
"title": {"type": "STRING", "description": "Window title for focus_window"},
"description": {"type": "STRING", "description": "Element description for screen_find/screen_click"},
"type": {"type": "STRING", "description": "Data type for random_data"},
"field": {"type": "STRING", "description": "Field for user_data: name|email|city"},
"clear_first": {"type": "BOOLEAN", "description": "Clear field before typing (default: true)"},
"path": {"type": "STRING", "description": "Save path for screenshot"},
},
"required": ["action"]
}
},
{
"name": "game_updater",
2026-03-02 04:30:10 +03:00
"description": (
2026-04-19 15:22:49 +03:00
"THE ONLY tool for ANY Steam or Epic Games request. "
"Use for: installing, downloading, updating games, listing installed games, "
"checking download status, scheduling updates. "
"ALWAYS call directly for any Steam/Epic/game request. "
"NEVER use agent_task, browser_control, or web_search for Steam/Epic."
2026-03-02 04:30:10 +03:00
),
"parameters": {
"type": "OBJECT",
"properties": {
2026-04-19 15:22:49 +03:00
"action": {"type": "STRING", "description": "update | install | list | download_status | schedule | cancel_schedule | schedule_status (default: update)"},
"platform": {"type": "STRING", "description": "steam | epic | both (default: both)"},
"game_name": {"type": "STRING", "description": "Game name (partial match supported)"},
"app_id": {"type": "STRING", "description": "Steam AppID for install (optional)"},
"hour": {"type": "INTEGER", "description": "Hour for scheduled update 0-23 (default: 3)"},
"minute": {"type": "INTEGER", "description": "Minute for scheduled update 0-59 (default: 0)"},
"shutdown_when_done": {"type": "BOOLEAN", "description": "Shut down PC when download finishes"},
2026-03-02 04:30:10 +03:00
},
2026-04-19 15:22:49 +03:00
"required": []
2026-03-02 04:30:10 +03:00
}
},
{
2026-04-19 15:22:49 +03:00
"name": "flight_finder",
"description": "Searches Google Flights and speaks the best options.",
"parameters": {
"type": "OBJECT",
"properties": {
"origin": {"type": "STRING", "description": "Departure city or airport code"},
"destination": {"type": "STRING", "description": "Arrival city or airport code"},
"date": {"type": "STRING", "description": "Departure date (any format)"},
"return_date": {"type": "STRING", "description": "Return date for round trips"},
"passengers": {"type": "INTEGER", "description": "Number of passengers (default: 1)"},
"cabin": {"type": "STRING", "description": "economy | premium | business | first"},
"save": {"type": "BOOLEAN", "description": "Save results to Notepad"},
},
"required": ["origin", "destination", "date"]
}
2026-03-02 04:30:10 +03:00
},
{
2026-04-19 15:22:49 +03:00
"name": "shutdown_jarvis",
"description": (
"Shuts down the assistant completely. "
"Call this when the user expresses intent to end the conversation, "
"close the assistant, say goodbye, or stop Jarvis. "
"The user can say this in ANY language."
),
"parameters": {
"type": "OBJECT",
"properties": {},
}
2026-03-02 04:30:10 +03:00
},
{
2026-04-19 15:22:49 +03:00
"name": "save_memory",
"description": (
"Save an important personal fact about the user to long-term memory. "
"Call this silently whenever the user reveals something worth remembering: "
"name, age, city, job, preferences, hobbies, relationships, projects, or future plans. "
"Do NOT call for: weather, reminders, searches, or one-time commands. "
"Do NOT announce that you are saving — just call it silently. "
"Values must be in English regardless of the conversation language."
),
"parameters": {
"type": "OBJECT",
"properties": {
"category": {
"type": "STRING",
"description": (
"identity — name, age, birthday, city, job, language, nationality | "
"preferences — favorite food/color/music/film/game/sport, hobbies | "
"projects — active projects, goals, things being built | "
"relationships — friends, family, partner, colleagues | "
"wishes — future plans, things to buy, travel dreams | "
"notes — habits, schedule, anything else worth remembering"
)
},
"key": {"type": "STRING", "description": "Short snake_case key (e.g. name, favorite_food, sister_name)"},
"value": {"type": "STRING", "description": "Concise value in English (e.g. Fatih, pizza, older sister)"},
2026-03-02 04:30:10 +03:00
},
2026-04-19 15:22:49 +03:00
"required": ["category", "key", "value"]
}
},
2026-03-02 04:30:10 +03:00
]
2026-04-19 15:22:49 +03:00
2026-03-02 04:30:10 +03:00
class JarvisLive:
def __init__(self, ui: JarvisUI):
self.ui = ui
self.session = None
self.audio_in_queue = None
self.out_queue = None
self._loop = None
2026-04-19 15:22:49 +03:00
self._is_speaking = False
self._speaking_lock = threading.Lock()
self.ui.on_text_command = self._on_text_command
self._turn_done_event: asyncio.Event | None = None
def _on_text_command(self, text: str):
if not self._loop or not self.session:
return
asyncio.run_coroutine_threadsafe(
self.session.send_client_content(
turns={"parts": [{"text": text}]},
turn_complete=True
),
self._loop
)
def set_speaking(self, value: bool):
with self._speaking_lock:
self._is_speaking = value
if value:
self.ui.set_state("SPEAKING")
elif not self.ui.muted:
self.ui.set_state("LISTENING")
2026-03-02 04:30:10 +03:00
def speak(self, text: str):
if not self._loop or not self.session:
return
asyncio.run_coroutine_threadsafe(
self.session.send_client_content(
turns={"parts": [{"text": text}]},
turn_complete=True
),
self._loop
2026-04-19 15:22:49 +03:00
)
def speak_error(self, tool_name: str, error: str):
short = str(error)[:120]
self.ui.write_log(f"ERR: {tool_name} — {short}")
self.speak(f"Sir, {tool_name} encountered an error. {short}")
2026-03-02 04:30:10 +03:00
2026-04-19 15:22:49 +03:00
def _build_config(self) -> types.LiveConnectConfig:
from datetime import datetime
2026-03-02 04:30:10 +03:00
2026-04-19 15:22:49 +03:00
memory = load_memory()
mem_str = format_memory_for_prompt(memory)
2026-03-02 04:30:10 +03:00
sys_prompt = _load_system_prompt()
now = datetime.now()
time_str = now.strftime("%A, %B %d, %Y — %I:%M %p")
time_ctx = (
f"[CURRENT DATE & TIME]\n"
f"Right now it is: {time_str}\n"
2026-04-19 15:22:49 +03:00
f"Use this to calculate exact times for reminders.\n\n"
2026-03-02 04:30:10 +03:00
)
2026-04-19 15:22:49 +03:00
parts = [time_ctx]
2026-03-02 04:30:10 +03:00
if mem_str:
2026-04-19 15:22:49 +03:00
parts.append(mem_str)
parts.append(sys_prompt)
2026-03-02 04:30:10 +03:00
return types.LiveConnectConfig(
response_modalities=["AUDIO"],
output_audio_transcription={},
input_audio_transcription={},
2026-04-19 15:22:49 +03:00
system_instruction="\n".join(parts),
2026-03-02 04:30:10 +03:00
tools=[{"function_declarations": TOOL_DECLARATIONS}],
session_resumption=types.SessionResumptionConfig(),
speech_config=types.SpeechConfig(
voice_config=types.VoiceConfig(
prebuilt_voice_config=types.PrebuiltVoiceConfig(
2026-04-19 15:22:49 +03:00
voice_name="Charon"
2026-03-02 04:30:10 +03:00
)
)
),
)
async def _execute_tool(self, fc) -> types.FunctionResponse:
name = fc.name
args = dict(fc.args or {})
2026-04-19 15:22:49 +03:00
print(f"[JARVIS] 🔧 {name} {args}")
self.ui.set_state("THINKING")
# ── save_memory: sessiz ve hızlı ──────────────────────────────────────
if name == "save_memory":
category = args.get("category", "notes")
key = args.get("key", "")
value = args.get("value", "")
if key and value:
update_memory({category: {key: {"value": value}}})
print(f"[Memory] 💾 save_memory: {category}/{key} = {value}")
if not self.ui.muted:
self.ui.set_state("LISTENING")
return types.FunctionResponse(
id=fc.id, name=name,
response={"result": "ok", "silent": True}
)
2026-03-02 04:30:10 +03:00
loop = asyncio.get_event_loop()
result = "Done."
try:
if name == "open_app":
2026-04-19 15:22:49 +03:00
r = await loop.run_in_executor(None, lambda: open_app(parameters=args, response=None, player=self.ui))
result = r or f"Opened {args.get('app_name')}."
2026-03-02 04:30:10 +03:00
elif name == "weather_report":
2026-04-19 15:22:49 +03:00
r = await loop.run_in_executor(None, lambda: weather_action(parameters=args, player=self.ui))
result = r or "Weather delivered."
2026-03-02 04:30:10 +03:00
elif name == "browser_control":
2026-04-19 15:22:49 +03:00
r = await loop.run_in_executor(None, lambda: browser_control(parameters=args, player=self.ui))
result = r or "Done."
2026-03-02 04:30:10 +03:00
elif name == "file_controller":
2026-04-19 15:22:49 +03:00
r = await loop.run_in_executor(None, lambda: file_controller(parameters=args, player=self.ui))
result = r or "Done."
2026-03-02 04:30:10 +03:00
elif name == "send_message":
2026-04-19 15:22:49 +03:00
r = await loop.run_in_executor(None, lambda: send_message(parameters=args, response=None, player=self.ui, session_memory=None))
2026-03-02 04:30:10 +03:00
result = r or f"Message sent to {args.get('receiver')}."
elif name == "reminder":
2026-04-19 15:22:49 +03:00
r = await loop.run_in_executor(None, lambda: reminder(parameters=args, response=None, player=self.ui))
result = r or "Reminder set."
2026-03-02 04:30:10 +03:00
elif name == "youtube_video":
2026-04-19 15:22:49 +03:00
r = await loop.run_in_executor(None, lambda: youtube_video(parameters=args, response=None, player=self.ui))
2026-03-02 04:30:10 +03:00
result = r or "Done."
elif name == "screen_process":
threading.Thread(
target=screen_process,
kwargs={"parameters": args, "response": None,
"player": self.ui, "session_memory": None},
daemon=True
).start()
2026-04-19 15:22:49 +03:00
result = "Vision module activated. Stay completely silent — vision module will speak directly."
2026-03-02 04:30:10 +03:00
elif name == "computer_settings":
2026-04-19 15:22:49 +03:00
r = await loop.run_in_executor(None, lambda: computer_settings(parameters=args, response=None, player=self.ui))
2026-03-02 04:30:10 +03:00
result = r or "Done."
elif name == "desktop_control":
2026-04-19 15:22:49 +03:00
r = await loop.run_in_executor(None, lambda: desktop_control(parameters=args, player=self.ui))
result = r or "Done."
2026-03-02 04:30:10 +03:00
elif name == "code_helper":
2026-04-19 15:22:49 +03:00
r = await loop.run_in_executor(None, lambda: code_helper(parameters=args, player=self.ui, speak=self.speak))
2026-03-02 04:30:10 +03:00
result = r or "Done."
elif name == "dev_agent":
2026-04-19 15:22:49 +03:00
r = await loop.run_in_executor(None, lambda: dev_agent(parameters=args, player=self.ui, speak=self.speak))
2026-03-02 04:30:10 +03:00
result = r or "Done."
2026-04-19 15:22:49 +03:00
elif name == "agent_task":
2026-03-02 04:30:10 +03:00
from agent.task_queue import get_queue, TaskPriority
2026-04-19 15:22:49 +03:00
priority_map = {"low": TaskPriority.LOW, "normal": TaskPriority.NORMAL, "high": TaskPriority.HIGH}
priority = priority_map.get(args.get("priority", "normal").lower(), TaskPriority.NORMAL)
task_id = get_queue().submit(goal=args.get("goal", ""), priority=priority, speak=self.speak)
result = f"Task started (ID: {task_id})."
2026-03-02 04:30:10 +03:00
elif name == "web_search":
2026-04-19 15:22:49 +03:00
r = await loop.run_in_executor(None, lambda: web_search_action(parameters=args, player=self.ui))
result = r or "Done."
2026-03-02 04:30:10 +03:00
elif name == "computer_control":
2026-04-19 15:22:49 +03:00
r = await loop.run_in_executor(None, lambda: computer_control(parameters=args, player=self.ui))
result = r or "Done."
elif name == "game_updater":
r = await loop.run_in_executor(None, lambda: game_updater(parameters=args, player=self.ui, speak=self.speak))
2026-03-02 04:30:10 +03:00
result = r or "Done."
elif name == "flight_finder":
2026-04-19 15:22:49 +03:00
r = await loop.run_in_executor(None, lambda: flight_finder(parameters=args, player=self.ui))
2026-03-02 04:30:10 +03:00
result = r or "Done."
2026-04-19 15:22:49 +03:00
elif name == "shutdown_jarvis":
self.ui.write_log("SYS: Shutdown requested.")
self.speak("Goodbye, sir.")
def _shutdown():
import time, os
time.sleep(1)
os._exit(0)
threading.Thread(target=_shutdown, daemon=True).start()
2026-03-02 04:30:10 +03:00
else:
result = f"Unknown tool: {name}"
2026-04-19 15:22:49 +03:00
2026-03-02 04:30:10 +03:00
except Exception as e:
result = f"Tool '{name}' failed: {e}"
traceback.print_exc()
2026-04-19 15:22:49 +03:00
self.speak_error(name, e)
2026-03-02 04:30:10 +03:00
2026-04-19 15:22:49 +03:00
if not self.ui.muted:
self.ui.set_state("LISTENING")
2026-03-02 04:30:10 +03:00
2026-04-19 15:22:49 +03:00
print(f"[JARVIS] 📤 {name} → {str(result)[:80]}")
2026-03-02 04:30:10 +03:00
return types.FunctionResponse(
2026-04-19 15:22:49 +03:00
id=fc.id, name=name,
2026-03-02 04:30:10 +03:00
response={"result": result}
)
async def _send_realtime(self):
while True:
msg = await self.out_queue.get()
await self.session.send_realtime_input(media=msg)
async def _listen_audio(self):
print("[JARVIS] 🎤 Mic started")
2026-04-19 15:22:49 +03:00
loop = asyncio.get_event_loop()
def callback(indata, frames, time_info, status):
with self._speaking_lock:
jarvis_speaking = self._is_speaking
if not jarvis_speaking and not self.ui.muted:
data = indata.tobytes()
loop.call_soon_threadsafe(
self.out_queue.put_nowait,
{"data": data, "mime_type": "audio/pcm"}
2026-03-02 04:30:10 +03:00
)
2026-04-19 15:22:49 +03:00
try:
with sd.InputStream(
samplerate=SEND_SAMPLE_RATE,
channels=CHANNELS,
dtype="int16",
blocksize=CHUNK_SIZE,
callback=callback,
):
print("[JARVIS] 🎤 Mic stream open")
while True:
await asyncio.sleep(0.1)
2026-03-02 04:30:10 +03:00
except Exception as e:
2026-04-19 15:22:49 +03:00
print(f"[JARVIS] ❌ Mic: {e}")
2026-03-02 04:30:10 +03:00
raise
async def _receive_audio(self):
print("[JARVIS] 👂 Recv started")
2026-04-19 15:22:49 +03:00
out_buf, in_buf = [], []
2026-03-02 04:30:10 +03:00
try:
while True:
2026-04-19 15:22:49 +03:00
async for response in self.session.receive():
2026-03-02 04:30:10 +03:00
if response.data:
2026-04-19 15:22:49 +03:00
if self._turn_done_event and self._turn_done_event.is_set():
self._turn_done_event.clear()
2026-03-02 04:30:10 +03:00
self.audio_in_queue.put_nowait(response.data)
if response.server_content:
sc = response.server_content
if sc.output_transcription and sc.output_transcription.text:
2026-04-19 15:22:49 +03:00
txt = _clean_transcript(sc.output_transcription.text)
2026-03-02 04:30:10 +03:00
if txt:
out_buf.append(txt)
2026-04-19 15:22:49 +03:00
if sc.input_transcription and sc.input_transcription.text:
txt = _clean_transcript(sc.input_transcription.text)
if txt:
in_buf.append(txt)
2026-03-02 04:30:10 +03:00
if sc.turn_complete:
2026-04-19 15:22:49 +03:00
if self._turn_done_event:
self._turn_done_event.set()
2026-03-02 04:30:10 +03:00
2026-04-19 15:22:49 +03:00
full_in = " ".join(in_buf).strip()
if full_in:
self.ui.write_log(f"You: {full_in}")
2026-03-02 04:30:10 +03:00
in_buf = []
2026-04-19 15:22:49 +03:00
full_out = " ".join(out_buf).strip()
if full_out:
self.ui.write_log(f"Jarvis: {full_out}")
2026-03-02 04:30:10 +03:00
out_buf = []
if response.tool_call:
fn_responses = []
for fc in response.tool_call.function_calls:
2026-04-19 15:22:49 +03:00
print(f"[JARVIS] 📞 {fc.name}")
2026-03-02 04:30:10 +03:00
fr = await self._execute_tool(fc)
fn_responses.append(fr)
await self.session.send_tool_response(
function_responses=fn_responses
)
except Exception as e:
2026-04-19 15:22:49 +03:00
print(f"[JARVIS] ❌ Recv: {e}")
2026-03-02 04:30:10 +03:00
traceback.print_exc()
raise
async def _play_audio(self):
print("[JARVIS] 🔊 Play started")
2026-04-19 15:22:49 +03:00
stream = sd.RawOutputStream(
samplerate=RECEIVE_SAMPLE_RATE,
2026-03-02 04:30:10 +03:00
channels=CHANNELS,
2026-04-19 15:22:49 +03:00
dtype="int16",
blocksize=CHUNK_SIZE,
2026-03-02 04:30:10 +03:00
)
2026-04-19 15:22:49 +03:00
stream.start()
2026-03-02 04:30:10 +03:00
try:
while True:
2026-04-19 15:22:49 +03:00
try:
chunk = await asyncio.wait_for(
self.audio_in_queue.get(),
timeout=0.1
)
except asyncio.TimeoutError:
if (
self._turn_done_event
and self._turn_done_event.is_set()
and self.audio_in_queue.empty()
):
self.set_speaking(False)
self._turn_done_event.clear()
continue
self.set_speaking(True)
2026-03-02 04:30:10 +03:00
await asyncio.to_thread(stream.write, chunk)
2026-04-19 15:22:49 +03:00
2026-03-02 04:30:10 +03:00
except Exception as e:
2026-04-19 15:22:49 +03:00
print(f"[JARVIS] ❌ Play: {e}")
2026-03-02 04:30:10 +03:00
raise
finally:
2026-04-19 15:22:49 +03:00
self.set_speaking(False)
stream.stop()
2026-03-02 04:30:10 +03:00
stream.close()
async def run(self):
client = genai.Client(
api_key=_get_api_key(),
http_options={"api_version": "v1beta"}
)
while True:
try:
print("[JARVIS] 🔌 Connecting...")
2026-04-19 15:22:49 +03:00
self.ui.set_state("THINKING")
2026-03-02 04:30:10 +03:00
config = self._build_config()
async with (
client.aio.live.connect(model=LIVE_MODEL, config=config) as session,
asyncio.TaskGroup() as tg,
):
self.session = session
2026-04-19 15:22:49 +03:00
self._loop = asyncio.get_event_loop()
2026-03-02 04:30:10 +03:00
self.audio_in_queue = asyncio.Queue()
self.out_queue = asyncio.Queue(maxsize=10)
2026-04-19 15:22:49 +03:00
self._turn_done_event = asyncio.Event()
2026-03-02 04:30:10 +03:00
print("[JARVIS] ✅ Connected.")
2026-04-19 15:22:49 +03:00
self.ui.set_state("LISTENING")
self.ui.write_log("SYS: JARVIS online.")
2026-03-02 04:30:10 +03:00
tg.create_task(self._send_realtime())
tg.create_task(self._listen_audio())
tg.create_task(self._receive_audio())
tg.create_task(self._play_audio())
except Exception as e:
2026-04-19 15:22:49 +03:00
print(f"[JARVIS] ⚠️ {e}")
2026-03-02 04:30:10 +03:00
traceback.print_exc()
2026-04-19 15:22:49 +03:00
self.set_speaking(False)
self.ui.set_state("THINKING")
2026-03-02 04:30:10 +03:00
print("[JARVIS] 🔄 Reconnecting in 3s...")
await asyncio.sleep(3)
2026-04-19 15:22:49 +03:00
2026-03-02 04:30:10 +03:00
def main():
ui = JarvisUI("face.png")
def runner():
ui.wait_for_api_key()
jarvis = JarvisLive(ui)
try:
asyncio.run(jarvis.run())
except KeyboardInterrupt:
print("\n🔴 Shutting down...")
threading.Thread(target=runner, daemon=True).start()
ui.root.mainloop()
2026-04-19 15:22:49 +03:00
2026-03-02 04:30:10 +03:00
if __name__ == "__main__":
main()