2026-03-02 04:30:10 +03:00
import asyncio
2026-04-19 15:22:49 +03:00
import re
2026-03-02 04:30:10 +03:00
import threading
import json
import sys
import traceback
from pathlib import Path
2026-04-19 15:22:49 +03:00
import sounddevice as sd
2026-03-02 04:30:10 +03:00
from google import genai
from google.genai import types
from ui import JarvisUI
2026-04-19 15:22:49 +03:00
from memory.memory_manager import (
load_memory , update_memory , format_memory_for_prompt ,
)
from actions.flight_finder import flight_finder
from actions.open_app import open_app
from actions.weather_report import weather_action
from actions.send_message import send_message
from actions.reminder import reminder
2026-03-02 04:30:10 +03:00
from actions.computer_settings import computer_settings
2026-04-19 15:22:49 +03:00
from actions.screen_processor import screen_process
from actions.youtube_video import youtube_video
from actions.desktop import desktop_control
from actions.browser_control import browser_control
from actions.file_controller import file_controller
from actions.code_helper import code_helper
from actions.dev_agent import dev_agent
from actions.web_search import web_search as web_search_action
from actions.computer_control import computer_control
from actions.game_updater import game_updater
2026-03-02 04:30:10 +03:00
def get_base_dir ():
if getattr ( sys , "frozen" , False ):
return Path ( sys . executable ) . parent
return Path ( __file__ ) . resolve () . parent
2026-04-19 15:22:49 +03:00
2026-03-02 04:30:10 +03:00
BASE_DIR = get_base_dir ()
API_CONFIG_PATH = BASE_DIR / "config" / "api_keys.json"
PROMPT_PATH = BASE_DIR / "core" / "prompt.txt"
LIVE_MODEL = "models/gemini-2.5-flash-native-audio-preview-12-2025"
CHANNELS = 1
SEND_SAMPLE_RATE = 16000
RECEIVE_SAMPLE_RATE = 24000
CHUNK_SIZE = 1024
def _get_api_key () -> str :
with open ( API_CONFIG_PATH , "r" , encoding = "utf-8" ) as f :
return json . load ( f )[ "gemini_api_key" ]
2026-04-19 15:22:49 +03:00
2026-03-02 04:30:10 +03:00
def _load_system_prompt () -> str :
try :
return PROMPT_PATH . read_text ( encoding = "utf-8" )
except Exception :
return (
"You are JARVIS, Tony Stark's AI assistant. "
"Be concise, direct, and always use the provided tools to complete tasks. "
"Never simulate or guess results — always call the appropriate tool."
)
2026-04-19 15:22:49 +03:00
# ── Transkripsiyon temizleyici ─────────────────────────────────────────────────
_CTRL_RE = re . compile ( r "<ctrl\d+>" , re . IGNORECASE )
2026-03-02 04:30:10 +03:00
2026-04-19 15:22:49 +03:00
def _clean_transcript ( text : str ) -> str :
"""Gemini'nin ürettiği <ctrlXX> artefaktları nı ve kontrol karakterlerini temizler."""
text = _CTRL_RE . sub ( "" , text )
text = re . sub ( r "[\x00-\x08\x0b-\x1f]" , "" , text )
return text . strip ()
2026-03-02 04:30:10 +03:00
2026-04-19 15:22:49 +03:00
# ── Tool declarations ──────────────────────────────────────────────────────────
2026-03-02 04:30:10 +03:00
TOOL_DECLARATIONS = [
{
"name" : "open_app" ,
"description" : (
2026-04-19 15:22:49 +03:00
"Opens any application on the computer. "
2026-03-02 04:30:10 +03:00
"Use this whenever the user asks to open, launch, or start any app, "
"website, or program. Always call this tool — never just say you opened it."
),
"parameters" : {
"type" : "OBJECT" ,
"properties" : {
"app_name" : {
"type" : "STRING" ,
"description" : "Exact name of the application (e.g. 'WhatsApp', 'Chrome', 'Spotify')"
}
},
"required" : [ "app_name" ]
}
},
2026-04-19 15:22:49 +03:00
{
"name" : "web_search" ,
"description" : "Searches the web for any information." ,
"parameters" : {
"type" : "OBJECT" ,
"properties" : {
"query" : { "type" : "STRING" , "description" : "Search query" },
"mode" : { "type" : "STRING" , "description" : "search (default) or compare" },
"items" : { "type" : "ARRAY" , "items" : { "type" : "STRING" }, "description" : "Items to compare" },
"aspect" : { "type" : "STRING" , "description" : "price | specs | reviews" }
},
"required" : [ "query" ]
}
},
2026-03-02 04:30:10 +03:00
{
"name" : "weather_report" ,
2026-04-19 15:22:49 +03:00
"description" : "Gives the weather report to user" ,
2026-03-02 04:30:10 +03:00
"parameters" : {
"type" : "OBJECT" ,
"properties" : {
"city" : { "type" : "STRING" , "description" : "City name" }
},
"required" : [ "city" ]
}
},
{
"name" : "send_message" ,
"description" : "Sends a text message via WhatsApp, Telegram, or other messaging platform." ,
"parameters" : {
"type" : "OBJECT" ,
"properties" : {
"receiver" : { "type" : "STRING" , "description" : "Recipient contact name" },
"message_text" : { "type" : "STRING" , "description" : "The message to send" },
"platform" : { "type" : "STRING" , "description" : "Platform: WhatsApp, Telegram, etc." }
},
"required" : [ "receiver" , "message_text" , "platform" ]
}
},
{
"name" : "reminder" ,
2026-04-19 15:22:49 +03:00
"description" : "Sets a timed reminder using Task Scheduler." ,
2026-03-02 04:30:10 +03:00
"parameters" : {
"type" : "OBJECT" ,
"properties" : {
"date" : { "type" : "STRING" , "description" : "Date in YYYY-MM-DD format" },
"time" : { "type" : "STRING" , "description" : "Time in HH:MM format (24h)" },
"message" : { "type" : "STRING" , "description" : "Reminder message text" }
},
"required" : [ "date" , "time" , "message" ]
}
},
{
2026-04-19 15:22:49 +03:00
"name" : "youtube_video" ,
"description" : (
"Controls YouTube. Use for: playing videos, summarizing a video's content, "
"getting video info, or showing trending videos."
),
"parameters" : {
"type" : "OBJECT" ,
"properties" : {
"action" : { "type" : "STRING" , "description" : "play | summarize | get_info | trending (default: play)" },
"query" : { "type" : "STRING" , "description" : "Search query for play action" },
"save" : { "type" : "BOOLEAN" , "description" : "Save summary to Notepad (summarize only)" },
"region" : { "type" : "STRING" , "description" : "Country code for trending e.g. TR, US" },
"url" : { "type" : "STRING" , "description" : "Video URL for get_info action" },
2026-03-02 04:30:10 +03:00
},
2026-04-19 15:22:49 +03:00
"required" : []
}
2026-03-02 04:30:10 +03:00
},
{
"name" : "screen_process" ,
"description" : (
"Captures and analyzes the screen or webcam image. "
"MUST be called when user asks what is on screen, what you see, "
"analyze my screen, look at camera, etc. "
"You have NO visual ability without this tool. "
"After calling this tool, stay SILENT — the vision module speaks directly."
),
"parameters" : {
"type" : "OBJECT" ,
"properties" : {
2026-04-19 15:22:49 +03:00
"angle" : { "type" : "STRING" , "description" : "'screen' to capture display, 'camera' for webcam. Default: 'screen'" },
"text" : { "type" : "STRING" , "description" : "The question or instruction about the captured image" }
2026-03-02 04:30:10 +03:00
},
"required" : [ "text" ]
}
},
{
2026-04-19 15:22:49 +03:00
"name" : "computer_settings" ,
"description" : (
"Controls the computer: volume, brightness, window management, keyboard shortcuts, "
"typing text on screen, closing apps, fullscreen, dark mode, WiFi, restart, shutdown, "
"scrolling, tab management, zoom, screenshots, lock screen, refresh/reload page. "
"Use for ANY single computer control command. NEVER route to agent_task."
),
"parameters" : {
"type" : "OBJECT" ,
"properties" : {
"action" : { "type" : "STRING" , "description" : "The action to perform" },
"description" : { "type" : "STRING" , "description" : "Natural language description of what to do" },
"value" : { "type" : "STRING" , "description" : "Optional value: volume level, text to type, etc." }
},
"required" : []
}
},
2026-03-02 04:30:10 +03:00
{
"name" : "browser_control" ,
"description" : (
2026-04-19 15:22:49 +03:00
"Controls any web browser. Use for: opening websites, searching the web, "
"clicking elements, filling forms, scrolling, screenshots, navigation, any web-based task. "
"Always pass the 'browser' parameter when the user specifies a browser (e.g. 'open in Edge', "
"'use Firefox', 'open Chrome'). Multiple browsers can run simultaneously."
2026-03-02 04:30:10 +03:00
),
"parameters" : {
"type" : "OBJECT" ,
"properties" : {
2026-04-19 15:22:49 +03:00
"action" : { "type" : "STRING" , "description" : "go_to | search | click | type | scroll | fill_form | smart_click | smart_type | get_text | get_url | press | new_tab | close_tab | screenshot | back | forward | reload | switch | list_browsers | close | close_all" },
"browser" : { "type" : "STRING" , "description" : "Target browser: chrome | edge | firefox | opera | operagx | brave | vivaldi | safari. Omit to use the currently active browser." },
"url" : { "type" : "STRING" , "description" : "URL for go_to / new_tab action" },
2026-03-02 04:30:10 +03:00
"query" : { "type" : "STRING" , "description" : "Search query for search action" },
2026-04-19 15:22:49 +03:00
"engine" : { "type" : "STRING" , "description" : "Search engine: google | bing | duckduckgo | yandex (default: google)" },
2026-03-02 04:30:10 +03:00
"selector" : { "type" : "STRING" , "description" : "CSS selector for click/type" },
"text" : { "type" : "STRING" , "description" : "Text to click or type" },
"description" : { "type" : "STRING" , "description" : "Element description for smart_click/smart_type" },
2026-04-19 15:22:49 +03:00
"direction" : { "type" : "STRING" , "description" : "up | down for scroll" },
"amount" : { "type" : "INTEGER" , "description" : "Scroll amount in pixels (default: 500)" },
"key" : { "type" : "STRING" , "description" : "Key name for press action (e.g. Enter, Escape, F5)" },
"path" : { "type" : "STRING" , "description" : "Save path for screenshot" },
"incognito" : { "type" : "BOOLEAN" , "description" : "Open in private/incognito mode" },
"clear_first" : { "type" : "BOOLEAN" , "description" : "Clear field before typing (default: true)" },
2026-03-02 04:30:10 +03:00
},
"required" : [ "action" ]
}
},
{
"name" : "file_controller" ,
2026-04-19 15:22:49 +03:00
"description" : "Manages files and folders: list, create, delete, move, copy, rename, read, write, find, disk usage." ,
2026-03-02 04:30:10 +03:00
"parameters" : {
"type" : "OBJECT" ,
"properties" : {
"action" : { "type" : "STRING" , "description" : "list | create_file | create_folder | delete | move | copy | rename | read | write | find | largest | disk_usage | organize_desktop | info" },
"path" : { "type" : "STRING" , "description" : "File/folder path or shortcut: desktop, downloads, documents, home" },
"destination" : { "type" : "STRING" , "description" : "Destination path for move/copy" },
"new_name" : { "type" : "STRING" , "description" : "New name for rename" },
"content" : { "type" : "STRING" , "description" : "Content for create_file/write" },
"name" : { "type" : "STRING" , "description" : "File name to search for" },
"extension" : { "type" : "STRING" , "description" : "File extension to search (e.g. .pdf)" },
"count" : { "type" : "INTEGER" , "description" : "Number of results for largest" },
},
"required" : [ "action" ]
}
},
{
2026-04-19 15:22:49 +03:00
"name" : "desktop_control" ,
"description" : "Controls the desktop: wallpaper, organize, clean, list, stats." ,
"parameters" : {
"type" : "OBJECT" ,
"properties" : {
"action" : { "type" : "STRING" , "description" : "wallpaper | wallpaper_url | organize | clean | list | stats | task" },
"path" : { "type" : "STRING" , "description" : "Image path for wallpaper" },
"url" : { "type" : "STRING" , "description" : "Image URL for wallpaper_url" },
"mode" : { "type" : "STRING" , "description" : "by_type or by_date for organize" },
"task" : { "type" : "STRING" , "description" : "Natural language desktop task" },
},
"required" : [ "action" ]
}
},
{
"name" : "code_helper" ,
"description" : "Writes, edits, explains, runs, or builds code files." ,
"parameters" : {
"type" : "OBJECT" ,
"properties" : {
"action" : { "type" : "STRING" , "description" : "write | edit | explain | run | build | auto (default: auto)" },
"description" : { "type" : "STRING" , "description" : "What the code should do or what change to make" },
"language" : { "type" : "STRING" , "description" : "Programming language (default: python)" },
"output_path" : { "type" : "STRING" , "description" : "Where to save the file" },
"file_path" : { "type" : "STRING" , "description" : "Path to existing file for edit/explain/run/build" },
"code" : { "type" : "STRING" , "description" : "Raw code string for explain" },
"args" : { "type" : "STRING" , "description" : "CLI arguments for run/build" },
"timeout" : { "type" : "INTEGER" , "description" : "Execution timeout in seconds (default: 30)" },
},
"required" : [ "action" ]
}
},
{
"name" : "dev_agent" ,
"description" : "Builds complete multi-file projects from scratch: plans, writes files, installs deps, opens VSCode, runs and fixes errors." ,
"parameters" : {
"type" : "OBJECT" ,
"properties" : {
"description" : { "type" : "STRING" , "description" : "What the project should do" },
"language" : { "type" : "STRING" , "description" : "Programming language (default: python)" },
"project_name" : { "type" : "STRING" , "description" : "Optional project folder name" },
"timeout" : { "type" : "INTEGER" , "description" : "Run timeout in seconds (default: 30)" },
},
"required" : [ "description" ]
}
},
{
"name" : "agent_task" ,
2026-03-02 04:30:10 +03:00
"description" : (
2026-04-19 15:22:49 +03:00
"Executes complex multi-step tasks requiring multiple different tools. "
"Examples: 'research X and save to file', 'find and organize files'. "
"DO NOT use for single commands. NEVER use for Steam/Epic — use game_updater."
2026-03-02 04:30:10 +03:00
),
"parameters" : {
"type" : "OBJECT" ,
"properties" : {
2026-04-19 15:22:49 +03:00
"goal" : { "type" : "STRING" , "description" : "Complete description of what to accomplish" },
"priority" : { "type" : "STRING" , "description" : "low | normal | high (default: normal)" }
2026-03-02 04:30:10 +03:00
},
2026-04-19 15:22:49 +03:00
"required" : [ "goal" ]
2026-03-02 04:30:10 +03:00
}
},
{
2026-04-19 15:22:49 +03:00
"name" : "computer_control" ,
"description" : "Direct computer control: type, click, hotkeys, scroll, move mouse, screenshots, find elements on screen." ,
"parameters" : {
"type" : "OBJECT" ,
"properties" : {
"action" : { "type" : "STRING" , "description" : "type | smart_type | click | double_click | right_click | hotkey | press | scroll | move | copy | paste | screenshot | wait | clear_field | focus_window | screen_find | screen_click | random_data | user_data" },
"text" : { "type" : "STRING" , "description" : "Text to type or paste" },
"x" : { "type" : "INTEGER" , "description" : "X coordinate" },
"y" : { "type" : "INTEGER" , "description" : "Y coordinate" },
"keys" : { "type" : "STRING" , "description" : "Key combination e.g. 'ctrl+c'" },
"key" : { "type" : "STRING" , "description" : "Single key e.g. 'enter'" },
"direction" : { "type" : "STRING" , "description" : "up | down | left | right" },
"amount" : { "type" : "INTEGER" , "description" : "Scroll amount (default: 3)" },
"seconds" : { "type" : "NUMBER" , "description" : "Seconds to wait" },
"title" : { "type" : "STRING" , "description" : "Window title for focus_window" },
"description" : { "type" : "STRING" , "description" : "Element description for screen_find/screen_click" },
"type" : { "type" : "STRING" , "description" : "Data type for random_data" },
"field" : { "type" : "STRING" , "description" : "Field for user_data: name|email|city" },
"clear_first" : { "type" : "BOOLEAN" , "description" : "Clear field before typing (default: true)" },
"path" : { "type" : "STRING" , "description" : "Save path for screenshot" },
},
"required" : [ "action" ]
}
},
{
"name" : "game_updater" ,
2026-03-02 04:30:10 +03:00
"description" : (
2026-04-19 15:22:49 +03:00
"THE ONLY tool for ANY Steam or Epic Games request. "
"Use for: installing, downloading, updating games, listing installed games, "
"checking download status, scheduling updates. "
"ALWAYS call directly for any Steam/Epic/game request. "
"NEVER use agent_task, browser_control, or web_search for Steam/Epic."
2026-03-02 04:30:10 +03:00
),
"parameters" : {
"type" : "OBJECT" ,
"properties" : {
2026-04-19 15:22:49 +03:00
"action" : { "type" : "STRING" , "description" : "update | install | list | download_status | schedule | cancel_schedule | schedule_status (default: update)" },
"platform" : { "type" : "STRING" , "description" : "steam | epic | both (default: both)" },
"game_name" : { "type" : "STRING" , "description" : "Game name (partial match supported)" },
"app_id" : { "type" : "STRING" , "description" : "Steam AppID for install (optional)" },
"hour" : { "type" : "INTEGER" , "description" : "Hour for scheduled update 0-23 (default: 3)" },
"minute" : { "type" : "INTEGER" , "description" : "Minute for scheduled update 0-59 (default: 0)" },
"shutdown_when_done" : { "type" : "BOOLEAN" , "description" : "Shut down PC when download finishes" },
2026-03-02 04:30:10 +03:00
},
2026-04-19 15:22:49 +03:00
"required" : []
2026-03-02 04:30:10 +03:00
}
},
{
2026-04-19 15:22:49 +03:00
"name" : "flight_finder" ,
"description" : "Searches Google Flights and speaks the best options." ,
"parameters" : {
"type" : "OBJECT" ,
"properties" : {
"origin" : { "type" : "STRING" , "description" : "Departure city or airport code" },
"destination" : { "type" : "STRING" , "description" : "Arrival city or airport code" },
"date" : { "type" : "STRING" , "description" : "Departure date (any format)" },
"return_date" : { "type" : "STRING" , "description" : "Return date for round trips" },
"passengers" : { "type" : "INTEGER" , "description" : "Number of passengers (default: 1)" },
"cabin" : { "type" : "STRING" , "description" : "economy | premium | business | first" },
"save" : { "type" : "BOOLEAN" , "description" : "Save results to Notepad" },
},
"required" : [ "origin" , "destination" , "date" ]
}
2026-03-02 04:30:10 +03:00
},
{
2026-04-19 15:22:49 +03:00
"name" : "shutdown_jarvis" ,
"description" : (
"Shuts down the assistant completely. "
"Call this when the user expresses intent to end the conversation, "
"close the assistant, say goodbye, or stop Jarvis. "
"The user can say this in ANY language."
),
"parameters" : {
"type" : "OBJECT" ,
"properties" : {},
}
2026-03-02 04:30:10 +03:00
},
{
2026-04-19 15:22:49 +03:00
"name" : "save_memory" ,
"description" : (
"Save an important personal fact about the user to long-term memory. "
"Call this silently whenever the user reveals something worth remembering: "
"name, age, city, job, preferences, hobbies, relationships, projects, or future plans. "
"Do NOT call for: weather, reminders, searches, or one-time commands. "
"Do NOT announce that you are saving — just call it silently. "
"Values must be in English regardless of the conversation language."
),
"parameters" : {
"type" : "OBJECT" ,
"properties" : {
"category" : {
"type" : "STRING" ,
"description" : (
"identity — name, age, birthday, city, job, language, nationality | "
"preferences — favorite food/color/music/film/game/sport, hobbies | "
"projects — active projects, goals, things being built | "
"relationships — friends, family, partner, colleagues | "
"wishes — future plans, things to buy, travel dreams | "
"notes — habits, schedule, anything else worth remembering"
)
},
"key" : { "type" : "STRING" , "description" : "Short snake_case key (e.g. name, favorite_food, sister_name)" },
"value" : { "type" : "STRING" , "description" : "Concise value in English (e.g. Fatih, pizza, older sister)" },
2026-03-02 04:30:10 +03:00
},
2026-04-19 15:22:49 +03:00
"required" : [ "category" , "key" , "value" ]
}
},
2026-03-02 04:30:10 +03:00
]
2026-04-19 15:22:49 +03:00
2026-03-02 04:30:10 +03:00
class JarvisLive :
def __init__ ( self , ui : JarvisUI ):
self . ui = ui
self . session = None
self . audio_in_queue = None
self . out_queue = None
self . _loop = None
2026-04-19 15:22:49 +03:00
self . _is_speaking = False
self . _speaking_lock = threading . Lock ()
self . ui . on_text_command = self . _on_text_command
self . _turn_done_event : asyncio . Event | None = None
def _on_text_command ( self , text : str ):
if not self . _loop or not self . session :
return
asyncio . run_coroutine_threadsafe (
self . session . send_client_content (
turns = { "parts" : [{ "text" : text }]},
turn_complete = True
),
self . _loop
)
def set_speaking ( self , value : bool ):
with self . _speaking_lock :
self . _is_speaking = value
if value :
self . ui . set_state ( "SPEAKING" )
elif not self . ui . muted :
self . ui . set_state ( "LISTENING" )
2026-03-02 04:30:10 +03:00
def speak ( self , text : str ):
if not self . _loop or not self . session :
return
asyncio . run_coroutine_threadsafe (
self . session . send_client_content (
turns = { "parts" : [{ "text" : text }]},
turn_complete = True
),
self . _loop
2026-04-19 15:22:49 +03:00
)
def speak_error ( self , tool_name : str , error : str ):
short = str ( error )[: 120 ]
self . ui . write_log ( f "ERR: { tool_name } — { short } " )
self . speak ( f "Sir, { tool_name } encountered an error. { short } " )
2026-03-02 04:30:10 +03:00
2026-04-19 15:22:49 +03:00
def _build_config ( self ) -> types . LiveConnectConfig :
from datetime import datetime
2026-03-02 04:30:10 +03:00
2026-04-19 15:22:49 +03:00
memory = load_memory ()
mem_str = format_memory_for_prompt ( memory )
2026-03-02 04:30:10 +03:00
sys_prompt = _load_system_prompt ()
now = datetime . now ()
time_str = now . strftime ( "%A, %B %d , %Y — %I:%M %p" )
time_ctx = (
f "[CURRENT DATE & TIME] \n "
f "Right now it is: { time_str } \n "
2026-04-19 15:22:49 +03:00
f "Use this to calculate exact times for reminders. \n\n "
2026-03-02 04:30:10 +03:00
)
2026-04-19 15:22:49 +03:00
parts = [ time_ctx ]
2026-03-02 04:30:10 +03:00
if mem_str :
2026-04-19 15:22:49 +03:00
parts . append ( mem_str )
parts . append ( sys_prompt )
2026-03-02 04:30:10 +03:00
return types . LiveConnectConfig (
response_modalities = [ "AUDIO" ],
output_audio_transcription = {},
input_audio_transcription = {},
2026-04-19 15:22:49 +03:00
system_instruction = " \n " . join ( parts ),
2026-03-02 04:30:10 +03:00
tools = [{ "function_declarations" : TOOL_DECLARATIONS }],
session_resumption = types . SessionResumptionConfig (),
speech_config = types . SpeechConfig (
voice_config = types . VoiceConfig (
prebuilt_voice_config = types . PrebuiltVoiceConfig (
2026-04-19 15:22:49 +03:00
voice_name = "Charon"
2026-03-02 04:30:10 +03:00
)
)
),
)
async def _execute_tool ( self , fc ) -> types . FunctionResponse :
name = fc . name
args = dict ( fc . args or {})
2026-04-19 15:22:49 +03:00
print ( f "[JARVIS] 🔧 { name } { args } " )
self . ui . set_state ( "THINKING" )
# ── save_memory: sessiz ve hı zlı ──────────────────────────────────────
if name == "save_memory" :
category = args . get ( "category" , "notes" )
key = args . get ( "key" , "" )
value = args . get ( "value" , "" )
if key and value :
update_memory ({ category : { key : { "value" : value }}})
print ( f "[Memory] 💾 save_memory: { category } / { key } = { value } " )
if not self . ui . muted :
self . ui . set_state ( "LISTENING" )
return types . FunctionResponse (
id = fc . id , name = name ,
response = { "result" : "ok" , "silent" : True }
)
2026-03-02 04:30:10 +03:00
loop = asyncio . get_event_loop ()
result = "Done."
try :
if name == "open_app" :
2026-04-19 15:22:49 +03:00
r = await loop . run_in_executor ( None , lambda : open_app ( parameters = args , response = None , player = self . ui ))
result = r or f "Opened { args . get ( 'app_name' ) } ."
2026-03-02 04:30:10 +03:00
elif name == "weather_report" :
2026-04-19 15:22:49 +03:00
r = await loop . run_in_executor ( None , lambda : weather_action ( parameters = args , player = self . ui ))
result = r or "Weather delivered."
2026-03-02 04:30:10 +03:00
elif name == "browser_control" :
2026-04-19 15:22:49 +03:00
r = await loop . run_in_executor ( None , lambda : browser_control ( parameters = args , player = self . ui ))
result = r or "Done."
2026-03-02 04:30:10 +03:00
elif name == "file_controller" :
2026-04-19 15:22:49 +03:00
r = await loop . run_in_executor ( None , lambda : file_controller ( parameters = args , player = self . ui ))
result = r or "Done."
2026-03-02 04:30:10 +03:00
elif name == "send_message" :
2026-04-19 15:22:49 +03:00
r = await loop . run_in_executor ( None , lambda : send_message ( parameters = args , response = None , player = self . ui , session_memory = None ))
2026-03-02 04:30:10 +03:00
result = r or f "Message sent to { args . get ( 'receiver' ) } ."
elif name == "reminder" :
2026-04-19 15:22:49 +03:00
r = await loop . run_in_executor ( None , lambda : reminder ( parameters = args , response = None , player = self . ui ))
result = r or "Reminder set."
2026-03-02 04:30:10 +03:00
elif name == "youtube_video" :
2026-04-19 15:22:49 +03:00
r = await loop . run_in_executor ( None , lambda : youtube_video ( parameters = args , response = None , player = self . ui ))
2026-03-02 04:30:10 +03:00
result = r or "Done."
elif name == "screen_process" :
threading . Thread (
target = screen_process ,
kwargs = { "parameters" : args , "response" : None ,
"player" : self . ui , "session_memory" : None },
daemon = True
) . start ()
2026-04-19 15:22:49 +03:00
result = "Vision module activated. Stay completely silent — vision module will speak directly."
2026-03-02 04:30:10 +03:00
elif name == "computer_settings" :
2026-04-19 15:22:49 +03:00
r = await loop . run_in_executor ( None , lambda : computer_settings ( parameters = args , response = None , player = self . ui ))
2026-03-02 04:30:10 +03:00
result = r or "Done."
elif name == "desktop_control" :
2026-04-19 15:22:49 +03:00
r = await loop . run_in_executor ( None , lambda : desktop_control ( parameters = args , player = self . ui ))
result = r or "Done."
2026-03-02 04:30:10 +03:00
elif name == "code_helper" :
2026-04-19 15:22:49 +03:00
r = await loop . run_in_executor ( None , lambda : code_helper ( parameters = args , player = self . ui , speak = self . speak ))
2026-03-02 04:30:10 +03:00
result = r or "Done."
elif name == "dev_agent" :
2026-04-19 15:22:49 +03:00
r = await loop . run_in_executor ( None , lambda : dev_agent ( parameters = args , player = self . ui , speak = self . speak ))
2026-03-02 04:30:10 +03:00
result = r or "Done."
2026-04-19 15:22:49 +03:00
elif name == "agent_task" :
2026-03-02 04:30:10 +03:00
from agent.task_queue import get_queue , TaskPriority
2026-04-19 15:22:49 +03:00
priority_map = { "low" : TaskPriority . LOW , "normal" : TaskPriority . NORMAL , "high" : TaskPriority . HIGH }
priority = priority_map . get ( args . get ( "priority" , "normal" ) . lower (), TaskPriority . NORMAL )
task_id = get_queue () . submit ( goal = args . get ( "goal" , "" ), priority = priority , speak = self . speak )
result = f "Task started (ID: { task_id } )."
2026-03-02 04:30:10 +03:00
elif name == "web_search" :
2026-04-19 15:22:49 +03:00
r = await loop . run_in_executor ( None , lambda : web_search_action ( parameters = args , player = self . ui ))
result = r or "Done."
2026-03-02 04:30:10 +03:00
elif name == "computer_control" :
2026-04-19 15:22:49 +03:00
r = await loop . run_in_executor ( None , lambda : computer_control ( parameters = args , player = self . ui ))
result = r or "Done."
elif name == "game_updater" :
r = await loop . run_in_executor ( None , lambda : game_updater ( parameters = args , player = self . ui , speak = self . speak ))
2026-03-02 04:30:10 +03:00
result = r or "Done."
elif name == "flight_finder" :
2026-04-19 15:22:49 +03:00
r = await loop . run_in_executor ( None , lambda : flight_finder ( parameters = args , player = self . ui ))
2026-03-02 04:30:10 +03:00
result = r or "Done."
2026-04-19 15:22:49 +03:00
elif name == "shutdown_jarvis" :
self . ui . write_log ( "SYS: Shutdown requested." )
self . speak ( "Goodbye, sir." )
def _shutdown ():
import time , os
time . sleep ( 1 )
os . _exit ( 0 )
threading . Thread ( target = _shutdown , daemon = True ) . start ()
2026-03-02 04:30:10 +03:00
else :
result = f "Unknown tool: { name } "
2026-04-19 15:22:49 +03:00
2026-03-02 04:30:10 +03:00
except Exception as e :
result = f "Tool ' { name } ' failed: { e } "
traceback . print_exc ()
2026-04-19 15:22:49 +03:00
self . speak_error ( name , e )
2026-03-02 04:30:10 +03:00
2026-04-19 15:22:49 +03:00
if not self . ui . muted :
self . ui . set_state ( "LISTENING" )
2026-03-02 04:30:10 +03:00
2026-04-19 15:22:49 +03:00
print ( f "[JARVIS] 📤 { name } → { str ( result )[: 80 ] } " )
2026-03-02 04:30:10 +03:00
return types . FunctionResponse (
2026-04-19 15:22:49 +03:00
id = fc . id , name = name ,
2026-03-02 04:30:10 +03:00
response = { "result" : result }
)
async def _send_realtime ( self ):
while True :
msg = await self . out_queue . get ()
await self . session . send_realtime_input ( media = msg )
async def _listen_audio ( self ):
print ( "[JARVIS] 🎤 Mic started" )
2026-04-19 15:22:49 +03:00
loop = asyncio . get_event_loop ()
def callback ( indata , frames , time_info , status ):
with self . _speaking_lock :
jarvis_speaking = self . _is_speaking
if not jarvis_speaking and not self . ui . muted :
data = indata . tobytes ()
loop . call_soon_threadsafe (
self . out_queue . put_nowait ,
{ "data" : data , "mime_type" : "audio/pcm" }
2026-03-02 04:30:10 +03:00
)
2026-04-19 15:22:49 +03:00
try :
with sd . InputStream (
samplerate = SEND_SAMPLE_RATE ,
channels = CHANNELS ,
dtype = "int16" ,
blocksize = CHUNK_SIZE ,
callback = callback ,
):
print ( "[JARVIS] 🎤 Mic stream open" )
while True :
await asyncio . sleep ( 0.1 )
2026-03-02 04:30:10 +03:00
except Exception as e :
2026-04-19 15:22:49 +03:00
print ( f "[JARVIS] ❌ Mic: { e } " )
2026-03-02 04:30:10 +03:00
raise
async def _receive_audio ( self ):
print ( "[JARVIS] 👂 Recv started" )
2026-04-19 15:22:49 +03:00
out_buf , in_buf = [], []
2026-03-02 04:30:10 +03:00
try :
while True :
2026-04-19 15:22:49 +03:00
async for response in self . session . receive ():
2026-03-02 04:30:10 +03:00
if response . data :
2026-04-19 15:22:49 +03:00
if self . _turn_done_event and self . _turn_done_event . is_set ():
self . _turn_done_event . clear ()
2026-03-02 04:30:10 +03:00
self . audio_in_queue . put_nowait ( response . data )
if response . server_content :
sc = response . server_content
if sc . output_transcription and sc . output_transcription . text :
2026-04-19 15:22:49 +03:00
txt = _clean_transcript ( sc . output_transcription . text )
2026-03-02 04:30:10 +03:00
if txt :
out_buf . append ( txt )
2026-04-19 15:22:49 +03:00
if sc . input_transcription and sc . input_transcription . text :
txt = _clean_transcript ( sc . input_transcription . text )
if txt :
in_buf . append ( txt )
2026-03-02 04:30:10 +03:00
if sc . turn_complete :
2026-04-19 15:22:49 +03:00
if self . _turn_done_event :
self . _turn_done_event . set ()
2026-03-02 04:30:10 +03:00
2026-04-19 15:22:49 +03:00
full_in = " " . join ( in_buf ) . strip ()
if full_in :
self . ui . write_log ( f "You: { full_in } " )
2026-03-02 04:30:10 +03:00
in_buf = []
2026-04-19 15:22:49 +03:00
full_out = " " . join ( out_buf ) . strip ()
if full_out :
self . ui . write_log ( f "Jarvis: { full_out } " )
2026-03-02 04:30:10 +03:00
out_buf = []
if response . tool_call :
fn_responses = []
for fc in response . tool_call . function_calls :
2026-04-19 15:22:49 +03:00
print ( f "[JARVIS] 📞 { fc . name } " )
2026-03-02 04:30:10 +03:00
fr = await self . _execute_tool ( fc )
fn_responses . append ( fr )
await self . session . send_tool_response (
function_responses = fn_responses
)
except Exception as e :
2026-04-19 15:22:49 +03:00
print ( f "[JARVIS] ❌ Recv: { e } " )
2026-03-02 04:30:10 +03:00
traceback . print_exc ()
raise
async def _play_audio ( self ):
print ( "[JARVIS] 🔊 Play started" )
2026-04-19 15:22:49 +03:00
stream = sd . RawOutputStream (
samplerate = RECEIVE_SAMPLE_RATE ,
2026-03-02 04:30:10 +03:00
channels = CHANNELS ,
2026-04-19 15:22:49 +03:00
dtype = "int16" ,
blocksize = CHUNK_SIZE ,
2026-03-02 04:30:10 +03:00
)
2026-04-19 15:22:49 +03:00
stream . start ()
2026-03-02 04:30:10 +03:00
try :
while True :
2026-04-19 15:22:49 +03:00
try :
chunk = await asyncio . wait_for (
self . audio_in_queue . get (),
timeout = 0.1
)
except asyncio . TimeoutError :
if (
self . _turn_done_event
and self . _turn_done_event . is_set ()
and self . audio_in_queue . empty ()
):
self . set_speaking ( False )
self . _turn_done_event . clear ()
continue
self . set_speaking ( True )
2026-03-02 04:30:10 +03:00
await asyncio . to_thread ( stream . write , chunk )
2026-04-19 15:22:49 +03:00
2026-03-02 04:30:10 +03:00
except Exception as e :
2026-04-19 15:22:49 +03:00
print ( f "[JARVIS] ❌ Play: { e } " )
2026-03-02 04:30:10 +03:00
raise
finally :
2026-04-19 15:22:49 +03:00
self . set_speaking ( False )
stream . stop ()
2026-03-02 04:30:10 +03:00
stream . close ()
async def run ( self ):
client = genai . Client (
api_key = _get_api_key (),
http_options = { "api_version" : "v1beta" }
)
while True :
try :
print ( "[JARVIS] 🔌 Connecting..." )
2026-04-19 15:22:49 +03:00
self . ui . set_state ( "THINKING" )
2026-03-02 04:30:10 +03:00
config = self . _build_config ()
async with (
client . aio . live . connect ( model = LIVE_MODEL , config = config ) as session ,
asyncio . TaskGroup () as tg ,
):
self . session = session
2026-04-19 15:22:49 +03:00
self . _loop = asyncio . get_event_loop ()
2026-03-02 04:30:10 +03:00
self . audio_in_queue = asyncio . Queue ()
self . out_queue = asyncio . Queue ( maxsize = 10 )
2026-04-19 15:22:49 +03:00
self . _turn_done_event = asyncio . Event ()
2026-03-02 04:30:10 +03:00
print ( "[JARVIS] ✅ Connected." )
2026-04-19 15:22:49 +03:00
self . ui . set_state ( "LISTENING" )
self . ui . write_log ( "SYS: JARVIS online." )
2026-03-02 04:30:10 +03:00
tg . create_task ( self . _send_realtime ())
tg . create_task ( self . _listen_audio ())
tg . create_task ( self . _receive_audio ())
tg . create_task ( self . _play_audio ())
except Exception as e :
2026-04-19 15:22:49 +03:00
print ( f "[JARVIS] ⚠️ { e } " )
2026-03-02 04:30:10 +03:00
traceback . print_exc ()
2026-04-19 15:22:49 +03:00
self . set_speaking ( False )
self . ui . set_state ( "THINKING" )
2026-03-02 04:30:10 +03:00
print ( "[JARVIS] 🔄 Reconnecting in 3s..." )
await asyncio . sleep ( 3 )
2026-04-19 15:22:49 +03:00
2026-03-02 04:30:10 +03:00
def main ():
ui = JarvisUI ( "face.png" )
def runner ():
ui . wait_for_api_key ()
jarvis = JarvisLive ( ui )
try :
asyncio . run ( jarvis . run ())
except KeyboardInterrupt :
print ( " \n 🔴 Shutting down..." )
threading . Thread ( target = runner , daemon = True ) . start ()
ui . root . mainloop ()
2026-04-19 15:22:49 +03:00
2026-03-02 04:30:10 +03:00
if __name__ == "__main__" :
main ()