Files
Mark-LIV/main.py
T
2026-06-14 17:04:27 +03:00

1397 lines
63 KiB
Python

"""
MARK XL — Local LLM Edition
STT (Whisper / Vosk) + Ollama LLM + TTS (EdgeTTS / Kokoro / ElevenLabs)
All Gemini / Google-AI dependencies removed.
"""
# ── Silence verbose logs + block heavy unused backends ─────────────────────
import os as _os
_os.environ.setdefault("TF_CPP_MIN_LOG_LEVEL", "3") # TensorFlow C++ noise
_os.environ.setdefault("TF_ENABLE_ONEDNN_OPTS", "0") # oneDNN banner
_os.environ.setdefault("GRPC_VERBOSITY", "ERROR")
# USE_TF=0 prevents transformers from importing TensorFlow (saves 4-8 s).
# We intentionally do NOT set USE_TORCH or USE_JAX — forcing those values
# breaks transformers' lazy-loader on some versions (AutoModel disappears
# from the namespace). Let transformers auto-detect the available backends.
_os.environ.setdefault("USE_TF", "0")
_os.environ.setdefault("TOKENIZERS_PARALLELISM", "false")
# Offline mode — use cached models, no HuggingFace network calls on startup.
# On first run the model isn't cached yet; tts.py / stt.py detect this and
# temporarily clear these flags to allow the one-time download, then they
# stay in effect for every subsequent launch (fully offline).
_os.environ.setdefault("HF_HUB_OFFLINE", "1")
_os.environ.setdefault("TRANSFORMERS_OFFLINE", "1")
_os.environ.setdefault("HF_DATASETS_OFFLINE", "1")
import warnings as _warnings
_warnings.filterwarnings("ignore", category=UserWarning)
_warnings.filterwarnings("ignore", category=DeprecationWarning)
_warnings.filterwarnings("ignore", category=FutureWarning)
# ───────────────────────────────────────────────────────────────────────────
# ── Bootstrap: auto-install base UI packages before anything else ──────────
# Uses only stdlib so it works even on a completely fresh Python install.
import importlib.util as _ilu
import subprocess as _sp
import sys as _sys
_BASE_PKGS = [
("PyQt6", "PyQt6"),
("psutil", "psutil"),
("numpy", "numpy"),
("sounddevice", "sounddevice"),
("PIL", "pillow"),
("requests", "requests"),
]
def _bootstrap() -> None:
need = [pkg for mod, pkg in _BASE_PKGS if _ilu.find_spec(mod) is None]
if not need:
return
print(f"\n[MARK XL] First-run setup — installing: {', '.join(need)}")
print("[MARK XL] This happens only once.\n")
_sp.run([_sys.executable, "-m", "pip", "install", *need], check=True)
print("\n[MARK XL] Base packages ready — restarting…\n")
# Replace current process with a fresh one (picks up newly installed packages)
_os.execv(_sys.executable, [_sys.executable] + _sys.argv)
_bootstrap()
# ───────────────────────────────────────────────────────────────────────────
import json
import queue
import re
import sys
import threading
import traceback
from datetime import datetime
from pathlib import Path
import numpy as np
import sounddevice as sd
from ui import JarvisUI
from memory.memory_manager import load_memory, update_memory, format_memory_for_prompt
from core.llm_client import call_llm, call_llm_stream, get_llm_settings
from actions.file_processor import file_processor
from actions.flight_finder import flight_finder
from actions.open_app import open_app
from actions.weather_report import weather_action
from actions.send_message import send_message
from actions.reminder import reminder
from actions.computer_settings import computer_settings
from actions.screen_processor import screen_process
from actions.youtube_video import youtube_video
from actions.desktop import desktop_control
from actions.browser_control import browser_control
from actions.file_controller import file_controller
from actions.code_helper import code_helper
from actions.dev_agent import dev_agent
from actions.web_search import web_search as web_search_action
from actions.computer_control import computer_control
from actions.game_updater import game_updater
# ---------------------------------------------------------------------------
# Paths
# ---------------------------------------------------------------------------
def _get_base_dir() -> Path:
if getattr(sys, "frozen", False):
return Path(sys.executable).parent
return Path(__file__).resolve().parent
BASE_DIR = _get_base_dir()
API_CONFIG_PATH = BASE_DIR / "config" / "api_keys.json"
PROMPT_PATH = BASE_DIR / "core" / "prompt.txt"
SAMPLE_RATE_IN = 16_000
BLOCK_SIZE = 1_024
CHANNELS = 1
# ---------------------------------------------------------------------------
# Tool declarations (Gemini format kept for readability;
# converted to OpenAI/Ollama format by _to_ollama_tools())
# ---------------------------------------------------------------------------
TOOL_DECLARATIONS = [
{
"name": "open_app",
"description": (
"Opens or launches any application, website, or program on the computer. "
"ALWAYS use this when the user says: open, launch, start, run, pull up, "
"or 'open X real quick'. Examples: 'open WhatsApp', 'open Chrome', "
"'launch Spotify', 'open calculator', 'pull up WhatsApp'. "
"Do NOT use send_message just because the app is a messaging app — "
"if the user only says to open it, call open_app."
),
"parameters": {
"type": "OBJECT",
"properties": {
"app_name": {"type": "STRING", "description": "Name of the application or website to open"}
},
"required": ["app_name"]
}
},
{
"name": "web_search",
"description": "Searches the web for any information.",
"parameters": {
"type": "OBJECT",
"properties": {
"query": {"type": "STRING", "description": "Search query"},
"mode": {"type": "STRING", "description": "search (default) or compare"},
"items": {"type": "ARRAY", "items": {"type": "STRING"}, "description": "Items to compare"},
"aspect": {"type": "STRING", "description": "price | specs | reviews"}
},
"required": ["query"]
}
},
{
"name": "weather_report",
"description": "Gives the weather report to user",
"parameters": {
"type": "OBJECT",
"properties": {"city": {"type": "STRING", "description": "City name"}},
"required": ["city"]
}
},
{
"name": "send_message",
"description": (
"Sends a message to a specific person via WhatsApp, Telegram, or similar. "
"ONLY use this when the user explicitly provides BOTH a recipient AND message content. "
"Example triggers: 'text John saying I am late', 'send a WhatsApp to mom that dinner is ready'. "
"Do NOT call this if the user only wants to open the app without sending a message."
),
"parameters": {
"type": "OBJECT",
"properties": {
"receiver": {"type": "STRING", "description": "Recipient contact name"},
"message_text": {"type": "STRING", "description": "The exact message text to send"},
"platform": {"type": "STRING", "description": "Platform: WhatsApp, Telegram, etc."}
},
"required": ["receiver", "message_text", "platform"]
}
},
{
"name": "reminder",
"description": "Sets a timed reminder using Task Scheduler.",
"parameters": {
"type": "OBJECT",
"properties": {
"date": {"type": "STRING", "description": "Date in YYYY-MM-DD format"},
"time": {"type": "STRING", "description": "Time in HH:MM format (24h)"},
"message": {"type": "STRING", "description": "Reminder message text"}
},
"required": ["date", "time", "message"]
}
},
{
"name": "youtube_video",
"description": (
"Controls YouTube. Use for: playing videos, summarizing a video's content, "
"getting video info, or showing trending videos."
),
"parameters": {
"type": "OBJECT",
"properties": {
"action": {"type": "STRING", "description": "play | summarize | get_info | trending"},
"query": {"type": "STRING", "description": "Search query for play action"},
"save": {"type": "BOOLEAN", "description": "Save summary to Notepad"},
"region": {"type": "STRING", "description": "Country code for trending e.g. TR, US"},
"url": {"type": "STRING", "description": "Video URL for get_info action"},
},
"required": []
}
},
{
"name": "screen_process",
"description": (
"Captures and analyzes the screen or webcam image. "
"MUST be called when user asks what is on screen, what you see, "
"analyze my screen, look at camera, etc. "
"You have NO visual ability without this tool."
),
"parameters": {
"type": "OBJECT",
"properties": {
"angle": {"type": "STRING", "description": "'screen' or 'camera'. Default: 'screen'"},
"text": {"type": "STRING", "description": "The question about the captured image"}
},
"required": ["text"]
}
},
{
"name": "computer_settings",
"description": (
"Controls the computer: volume, brightness, window management, keyboard shortcuts, "
"typing text on screen, closing apps, fullscreen, dark mode, WiFi, restart, shutdown, "
"scrolling, tab management, zoom, screenshots, lock screen, refresh/reload page."
),
"parameters": {
"type": "OBJECT",
"properties": {
"action": {"type": "STRING", "description": "The action to perform"},
"description": {"type": "STRING", "description": "Natural language description"},
"value": {"type": "STRING", "description": "Optional value"}
},
"required": []
}
},
{
"name": "browser_control",
"description": (
"Controls any web browser. Use for: opening websites, searching the web, "
"clicking elements, filling forms, scrolling, screenshots, navigation."
),
"parameters": {
"type": "OBJECT",
"properties": {
"action": {"type": "STRING", "description": "go_to | search | click | type | scroll | fill_form | smart_click | smart_type | get_text | get_url | press | new_tab | close_tab | screenshot | back | forward | reload | switch | list_browsers | close | close_all"},
"browser": {"type": "STRING", "description": "chrome | edge | firefox | opera | operagx | brave | vivaldi | safari"},
"url": {"type": "STRING", "description": "URL for go_to / new_tab action"},
"query": {"type": "STRING", "description": "Search query"},
"engine": {"type": "STRING", "description": "google | bing | duckduckgo | yandex"},
"selector": {"type": "STRING", "description": "CSS selector for click/type"},
"text": {"type": "STRING", "description": "Text to click or type"},
"description": {"type": "STRING", "description": "Element description for smart_click/smart_type"},
"direction": {"type": "STRING", "description": "up | down for scroll"},
"amount": {"type": "INTEGER", "description": "Scroll amount in pixels"},
"key": {"type": "STRING", "description": "Key name for press"},
"path": {"type": "STRING", "description": "Save path for screenshot"},
"incognito": {"type": "BOOLEAN", "description": "Open in private/incognito mode"},
"clear_first": {"type": "BOOLEAN", "description": "Clear field before typing"},
},
"required": ["action"]
}
},
{
"name": "file_controller",
"description": "Manages files and folders: list, create, delete, move, copy, rename, read, write, find, disk usage.",
"parameters": {
"type": "OBJECT",
"properties": {
"action": {"type": "STRING", "description": "list | create_file | create_folder | delete | move | copy | rename | read | write | find | largest | disk_usage | organize_desktop | info"},
"path": {"type": "STRING", "description": "File/folder path or shortcut: desktop, downloads, documents, home"},
"destination": {"type": "STRING", "description": "Destination path for move/copy"},
"new_name": {"type": "STRING", "description": "New name for rename"},
"content": {"type": "STRING", "description": "Content for create_file/write"},
"name": {"type": "STRING", "description": "File name to search for"},
"extension": {"type": "STRING", "description": "File extension to search"},
"count": {"type": "INTEGER", "description": "Number of results for largest"},
},
"required": ["action"]
}
},
{
"name": "desktop_control",
"description": "Controls the desktop: wallpaper, organize, clean, list, stats.",
"parameters": {
"type": "OBJECT",
"properties": {
"action": {"type": "STRING", "description": "wallpaper | wallpaper_url | organize | clean | list | stats | task"},
"path": {"type": "STRING", "description": "Image path for wallpaper"},
"url": {"type": "STRING", "description": "Image URL for wallpaper_url"},
"mode": {"type": "STRING", "description": "by_type or by_date for organize"},
"task": {"type": "STRING", "description": "Natural language desktop task"},
},
"required": ["action"]
}
},
{
"name": "code_helper",
"description": "Writes, edits, explains, runs, or builds code files.",
"parameters": {
"type": "OBJECT",
"properties": {
"action": {"type": "STRING", "description": "write | edit | explain | run | build | auto"},
"description": {"type": "STRING", "description": "What the code should do"},
"language": {"type": "STRING", "description": "Programming language"},
"output_path": {"type": "STRING", "description": "Where to save the file"},
"file_path": {"type": "STRING", "description": "Path to existing file"},
"code": {"type": "STRING", "description": "Raw code string for explain"},
"args": {"type": "STRING", "description": "CLI arguments"},
"timeout": {"type": "INTEGER", "description": "Execution timeout in seconds"},
},
"required": ["action"]
}
},
{
"name": "dev_agent",
"description": "Builds complete multi-file projects from scratch.",
"parameters": {
"type": "OBJECT",
"properties": {
"description": {"type": "STRING", "description": "What the project should do"},
"language": {"type": "STRING", "description": "Programming language"},
"project_name": {"type": "STRING", "description": "Optional project folder name"},
"timeout": {"type": "INTEGER", "description": "Run timeout in seconds"},
},
"required": ["description"]
}
},
{
"name": "agent_task",
"description": (
"Executes complex multi-step tasks requiring multiple different tools. "
"Examples: 'research X and save to file', 'find and organize files'. "
"DO NOT use for single commands."
),
"parameters": {
"type": "OBJECT",
"properties": {
"goal": {"type": "STRING", "description": "Complete description of what to accomplish"},
"priority": {"type": "STRING", "description": "low | normal | high"}
},
"required": ["goal"]
}
},
{
"name": "computer_control",
"description": "Direct computer control: type, click, hotkeys, scroll, move mouse, screenshots, find elements on screen.",
"parameters": {
"type": "OBJECT",
"properties": {
"action": {"type": "STRING", "description": "type | smart_type | click | double_click | right_click | hotkey | press | scroll | move | copy | paste | screenshot | wait | clear_field | focus_window | screen_find | screen_click | random_data | user_data"},
"text": {"type": "STRING", "description": "Text to type or paste"},
"x": {"type": "INTEGER", "description": "X coordinate"},
"y": {"type": "INTEGER", "description": "Y coordinate"},
"keys": {"type": "STRING", "description": "Key combination e.g. 'ctrl+c'"},
"key": {"type": "STRING", "description": "Single key e.g. 'enter'"},
"direction": {"type": "STRING", "description": "up | down | left | right"},
"amount": {"type": "INTEGER", "description": "Scroll amount"},
"seconds": {"type": "NUMBER", "description": "Seconds to wait"},
"title": {"type": "STRING", "description": "Window title for focus_window"},
"description": {"type": "STRING", "description": "Element description"},
"type": {"type": "STRING", "description": "Data type for random_data"},
"field": {"type": "STRING", "description": "Field for user_data"},
"clear_first": {"type": "BOOLEAN", "description": "Clear field before typing"},
"path": {"type": "STRING", "description": "Save path for screenshot"},
},
"required": ["action"]
}
},
{
"name": "game_updater",
"description": (
"THE ONLY tool for ANY Steam or Epic Games request. "
"Use for: installing, downloading, updating games, listing installed games."
),
"parameters": {
"type": "OBJECT",
"properties": {
"action": {"type": "STRING", "description": "update | install | list | download_status | schedule | cancel_schedule | schedule_status"},
"platform": {"type": "STRING", "description": "steam | epic | both"},
"game_name": {"type": "STRING", "description": "Game name"},
"app_id": {"type": "STRING", "description": "Steam AppID"},
"hour": {"type": "INTEGER", "description": "Hour for scheduled update 0-23"},
"minute": {"type": "INTEGER", "description": "Minute for scheduled update 0-59"},
"shutdown_when_done": {"type": "BOOLEAN", "description": "Shut down PC when done"},
},
"required": []
}
},
{
"name": "flight_finder",
"description": "Searches Google Flights and speaks the best options.",
"parameters": {
"type": "OBJECT",
"properties": {
"origin": {"type": "STRING", "description": "Departure city or airport code"},
"destination": {"type": "STRING", "description": "Arrival city or airport code"},
"date": {"type": "STRING", "description": "Departure date"},
"return_date": {"type": "STRING", "description": "Return date for round trips"},
"passengers": {"type": "INTEGER", "description": "Number of passengers"},
"cabin": {"type": "STRING", "description": "economy | premium | business | first"},
"save": {"type": "BOOLEAN", "description": "Save results to Notepad"},
},
"required": ["origin", "destination", "date"]
}
},
{
"name": "shutdown_jarvis",
"description": (
"Shuts down the assistant completely. "
"Call this when the user expresses intent to end the conversation, "
"close the assistant, say goodbye, or stop Jarvis."
),
"parameters": {"type": "OBJECT", "properties": {}}
},
{
"name": "file_processor",
"description": (
"Processes any file that the user has uploaded or dropped onto the interface. "
"Supports: images, PDFs, Word docs, CSV/Excel, JSON, code files, audio, video, archives."
),
"parameters": {
"type": "OBJECT",
"properties": {
"file_path": {"type": "STRING", "description": "Full path to the uploaded file"},
"action": {"type": "STRING", "description": "What to do with the file"},
"instruction": {"type": "STRING", "description": "Free-form instruction"},
"format": {"type": "STRING", "description": "Target format for conversion"},
"width": {"type": "INTEGER", "description": "Target width for image resize"},
"height": {"type": "INTEGER", "description": "Target height for image resize"},
"scale": {"type": "NUMBER", "description": "Scale factor"},
"quality": {"type": "INTEGER", "description": "Quality 1-100"},
"start": {"type": "STRING", "description": "Start time for trim"},
"end": {"type": "STRING", "description": "End time for trim"},
"timestamp": {"type": "STRING", "description": "Timestamp for video frame extraction"},
"column": {"type": "STRING", "description": "Column name for CSV filter/sort"},
"value": {"type": "STRING", "description": "Filter value"},
"condition": {"type": "STRING", "description": "Filter condition"},
"ascending": {"type": "BOOLEAN", "description": "Sort order"},
"save": {"type": "BOOLEAN", "description": "Save result to file"},
"destination": {"type": "STRING", "description": "Output folder for archive extract"},
},
"required": []
}
},
{
"name": "save_memory",
"description": (
"Save a personal fact about the user to permanent long-term memory. "
"MANDATORY: call this IMMEDIATELY (without asking) whenever the user states or corrects: "
"their name, age, city, job, school, language, nationality, a preference, a goal, or a relationship. "
"Examples: "
"'my name is Fatih' → (identity, name, Fatih) | "
"'not Travis, Fatih' → (identity, name, Fatih) | "
"'I am 22' → (identity, age, 22) | "
"'I live in Ankara' → (identity, city, Ankara) | "
"'I prefer dark mode' → (preferences, ui_theme, dark mode). "
"Call SILENTLY alongside your verbal reply — never announce that you are saving."
),
"parameters": {
"type": "OBJECT",
"properties": {
"category": {
"type": "STRING",
"description": (
"identity (name/age/city/job/school/nationality) | "
"preferences (likes/dislikes/habits) | "
"projects (active work/goals) | "
"relationships (people in their life) | "
"wishes (future plans/wants) | "
"notes (anything else)"
)
},
"key": {"type": "STRING", "description": "Short snake_case key, e.g. 'name', 'age', 'favorite_color'"},
"value": {"type": "STRING", "description": "Concise value in English"},
},
"required": ["category", "key", "value"]
}
},
]
# ---------------------------------------------------------------------------
# Convert Gemini-style declarations to OpenAI/Ollama format
# ---------------------------------------------------------------------------
_TYPE_MAP = {
"OBJECT": "object", "STRING": "string", "ARRAY": "array",
"INTEGER": "integer", "BOOLEAN": "boolean", "NUMBER": "number",
}
def _convert_type(t: str) -> str:
return _TYPE_MAP.get(t, t.lower()) if isinstance(t, str) else t
def _convert_props(props: dict) -> dict:
out = {}
for k, v in props.items():
nv = dict(v)
if "type" in nv:
nv["type"] = _convert_type(nv["type"])
if "items" in nv and isinstance(nv["items"], dict):
nv["items"] = {"type": _convert_type(nv["items"].get("type", "string"))}
out[k] = nv
return out
def _to_ollama_tools(decls: list) -> list:
tools = []
for d in decls:
params = d.get("parameters", {})
new_params: dict = {
"type": "object",
"properties": _convert_props(params.get("properties", {})),
}
req = params.get("required")
if req:
new_params["required"] = req
tools.append({
"type": "function",
"function": {
"name": d["name"],
"description": d["description"],
"parameters": new_params,
},
})
return tools
OLLAMA_TOOLS = _to_ollama_tools(TOOL_DECLARATIONS)
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def _load_config() -> dict:
try:
with open(API_CONFIG_PATH, encoding="utf-8") as f:
return json.load(f)
except Exception:
return {}
def _load_system_prompt() -> str:
try:
return PROMPT_PATH.read_text(encoding="utf-8")
except Exception:
return (
"You are JARVIS, Tony Stark's AI assistant. "
"Be concise, direct, and always use the provided tools to complete tasks. "
"Never simulate or guess results — always call the appropriate tool."
)
# ---------------------------------------------------------------------------
# Voice Activity Detection (used for Whisper listen loop)
# ---------------------------------------------------------------------------
class _VADBuffer:
"""Energy-based VAD: buffers audio until end of utterance."""
def __init__(
self,
sample_rate: int = 16_000,
silence_sec: float = 0.7, # silence after last word → send to STT
speech_thresh: float = 0.008, # RMS above this = speech (0.008 catches voice at 3-4 m; raise if mic picks up too much room noise)
silence_thresh: float = 0.004, # RMS below this = silence (half of speech_thresh — hysteresis prevents mid-sentence cuts)
min_speech_sec: float = 0.3,
max_speech_sec: float = 30.0,
):
self._sr = sample_rate
self._sil_n = int(silence_sec * sample_rate)
self._speech_thresh = speech_thresh
self._sil_thresh = silence_thresh
self._min_n = int(min_speech_sec * sample_rate)
self._max_n = int(max_speech_sec * sample_rate)
self._buf: list[np.ndarray] = []
self._in_spch = False
self._sil_cnt = 0
def process(self, chunk: np.ndarray) -> np.ndarray | None:
"""
Feed one audio chunk (float32 mono).
Returns complete utterance when speech ends, otherwise None.
Uses hysteresis thresholds so the detector doesn't flicker:
- speech starts when RMS > speech_thresh (0.008 = ~3-4 m range)
- speech ends only when RMS < silence_thresh (0.004 = half of start)
The gap between the two thresholds prevents mid-sentence cuts on
natural pauses and quiet consonants.
"""
rms = float(np.sqrt(np.mean(chunk ** 2)))
total_n = sum(len(c) for c in self._buf)
if rms > self._speech_thresh:
self._in_spch = True
self._sil_cnt = 0
self._buf.append(chunk.copy())
elif self._in_spch:
self._buf.append(chunk.copy())
if rms < self._sil_thresh:
self._sil_cnt += len(chunk)
if self._sil_cnt >= self._sil_n or total_n >= self._max_n:
audio = np.concatenate(self._buf)
self._buf = []
self._in_spch = False
self._sil_cnt = 0
if len(audio) >= self._min_n:
return audio
return None
# ---------------------------------------------------------------------------
# JarvisLocal
# ---------------------------------------------------------------------------
class JarvisLocal:
"""
Main assistant class.
Replaces JarvisLive (Gemini Live API) with:
STT (Whisper/Vosk) → Ollama LLM (tool calling) → TTS (Edge/Kokoro/ElevenLabs)
"""
def __init__(self, ui: JarvisUI):
self.ui = ui
self._config = _load_config()
self._stt = None
self._tts = None
self._tts_ready = threading.Event() # set when TTS engine is loaded
self._speaking = False
self._speaking_lock = threading.Lock()
self._text_queue: queue.Queue = queue.Queue()
self._tts_queue: queue.Queue = queue.Queue()
self._conversation: list[dict] = []
self.ui.on_text_command = self._on_text_command
# ------------------------------------------------------------------
# System prompt
# ------------------------------------------------------------------
def _build_system_prompt(self) -> str:
# ── ORDER MATTERS for Ollama KV prefix caching ─────────────────────
# Ollama caches the KV attention state of any stable prompt prefix.
# By putting the STATIC JARVIS protocol text FIRST, Ollama reuses its
# cached KV for all those tokens on every request. Only the small
# dynamic tail (memory + time, ~50-80 tokens) needs re-evaluation.
# This turns a 17-second first-token into a sub-second one after warmup.
#
# Rule: static content first → semi-static memory middle → dynamic time LAST.
sys_p = _load_system_prompt() # static — never changes mid-session
memory = load_memory()
mem_str = format_memory_for_prompt(memory) # semi-static — changes only when user tells facts
now = datetime.now()
time_ctx = (
f"[CURRENT DATE & TIME]\n"
f"Right now it is: {now.strftime('%A, %B %d, %Y — %I:%M %p')}\n"
f"Use this to calculate exact times for reminders."
)
parts = [sys_p]
if mem_str:
parts.append(mem_str)
parts.append(time_ctx)
return "\n\n".join(parts)
# ------------------------------------------------------------------
# Speaking state & TTS
# ------------------------------------------------------------------
# ------------------------------------------------------------------
# TTS queue worker — plays sentences sequentially, no overlaps
# ------------------------------------------------------------------
def _tts_worker(self) -> None:
# Block until TTS engine is loaded. Queued items are preserved
# and played immediately once loading completes — nothing is lost.
self._tts_ready.wait(timeout=120)
while True:
text = self._tts_queue.get()
try:
if text and self._tts:
with self._speaking_lock:
self._speaking = True
self.ui.set_state("SPEAKING")
self._tts.speak(text)
except Exception as e:
print(f"[TTS] speak error: {e}")
finally:
self._tts_queue.task_done()
if self._tts_queue.empty():
with self._speaking_lock:
self._speaking = False
if not self.ui.muted:
self.ui.set_state("LISTENING")
def set_speaking(self, value: bool) -> None:
with self._speaking_lock:
self._speaking = value
if value:
self.ui.set_state("SPEAKING")
elif not self.ui.muted:
self.ui.set_state("LISTENING")
def speak(self, text: str) -> None:
if not text or not self._tts:
return
with self._speaking_lock:
self._speaking = True
self._tts_queue.put(text)
def speak_error(self, tool_name: str, error) -> None:
short = str(error)[:120]
self.ui.write_log(f"ERR: {tool_name} — {short}")
self.speak(f"{tool_name} encountered an error.")
# ------------------------------------------------------------------
# Live reconfigure (called when user clicks Apply in Configure panel)
# ------------------------------------------------------------------
def reconfigure(self, new_config: dict) -> None:
"""Non-blocking: spawns a background thread to install + reload."""
threading.Thread(
target=self._do_reconfigure, args=(new_config,), daemon=True
).start()
def _do_reconfigure(self, new_config: dict) -> None:
old_stt_engine = self._config.get("stt_engine", "whisper").lower()
old_llm_model = self._config.get("llm_model", "")
new_stt_engine = new_config.get("stt_engine", "whisper").lower()
self._config = new_config
# Install any packages required by the new config
try:
from core.installer import install_for_config
install_for_config(new_config, log=self.ui.write_log)
except Exception as e:
self.ui.write_log(f"ERR: Dependency install — {e}")
# TTS: always hot-reload (runs in queue worker, safe to swap)
try:
from core.tts import create_tts_player
self._tts = create_tts_player(new_config)
self._tts_ready.set() # ensure worker isn't blocked
self.ui.write_log("SYS: TTS reconfigured.")
except Exception as e:
self.ui.write_log(f"ERR: TTS reconfigure — {e}")
# STT: hot-reload if same engine type; full restart needed if engine changed
if old_stt_engine == new_stt_engine:
try:
stt_language = new_config.get("stt_language", "auto")
if new_stt_engine == "vosk":
from core.stt import VoskSTT
self._stt = VoskSTT(new_config.get("vosk_model_path"), language=stt_language)
else:
from core.stt import WhisperSTT
self._stt = WhisperSTT(new_config.get("stt_model", "base"), language=stt_language)
self.ui.write_log("SYS: STT reconfigured.")
except Exception as e:
self.ui.write_log(f"ERR: STT reconfigure — {e}")
else:
self.ui.write_log("SYS: STT engine changed — restart required.")
# LLM warmup if model changed
if new_config.get("llm_model", "") != old_llm_model:
self.ui.write_log("SYS: Warming up new LLM model…")
from core.llm_client import warmup_model
warmup_model()
self.ui.write_log("SYS: New LLM model ready.")
if old_stt_engine == new_stt_engine:
self.speak("Configuration applied.")
else:
self.speak("LLM and TTS updated. Restart for speech engine change.")
# ------------------------------------------------------------------
# Text command (from UI input box)
# ------------------------------------------------------------------
def _on_text_command(self, text: str) -> None:
self._text_queue.put(text)
# ------------------------------------------------------------------
# Tool execution (routing unchanged from original)
# ------------------------------------------------------------------
def _execute_tool(self, name: str, args: dict) -> str:
print(f"[JARVIS] 🔧 {name} {args}")
self.ui.set_state("THINKING")
# save_memory is handled silently
if name == "save_memory":
category = args.get("category", "notes")
key = args.get("key", "")
value = args.get("value", "")
if key and value:
update_memory({category: {key: {"value": value}}})
print(f"[Memory] 💾 {category}/{key} = {value}")
if not self.ui.muted:
self.ui.set_state("LISTENING")
return "__SILENT__"
result = "Done."
try:
if name == "open_app":
r = open_app(parameters=args, response=None, player=self.ui)
result = r or f"Opened {args.get('app_name')}."
elif name == "weather_report":
r = weather_action(parameters=args, player=self.ui)
result = r or "Weather delivered."
elif name == "browser_control":
r = browser_control(parameters=args, player=self.ui)
result = r or "Done."
elif name == "file_controller":
r = file_controller(parameters=args, player=self.ui)
result = r or "Done."
elif name == "send_message":
r = send_message(parameters=args, response=None, player=self.ui, session_memory=None)
result = r or f"Message sent to {args.get('receiver')}."
elif name == "reminder":
r = reminder(parameters=args, response=None, player=self.ui)
result = r or "Reminder set."
elif name == "youtube_video":
r = youtube_video(parameters=args, response=None, player=self.ui)
result = r or "Done."
elif name == "screen_process":
# Synchronous call — returns analysis text which the LLM can speak
r = screen_process(parameters=args, response=None, player=self.ui, session_memory=None)
result = r if isinstance(r, str) and r else "Screen analyzed."
elif name == "computer_settings":
r = computer_settings(parameters=args, response=None, player=self.ui)
result = r or "Done."
elif name == "desktop_control":
r = desktop_control(parameters=args, player=self.ui)
result = r or "Done."
elif name == "code_helper":
r = code_helper(parameters=args, player=self.ui, speak=self.speak)
result = r or "Done."
elif name == "dev_agent":
r = dev_agent(parameters=args, player=self.ui, speak=self.speak)
result = r or "Done."
elif name == "agent_task":
from agent.task_queue import get_queue, TaskPriority
priority_map = {
"low": TaskPriority.LOW,
"normal": TaskPriority.NORMAL,
"high": TaskPriority.HIGH,
}
priority = priority_map.get(
args.get("priority", "normal").lower(), TaskPriority.NORMAL
)
task_id = get_queue().submit(
goal=args.get("goal", ""), priority=priority, speak=self.speak
)
result = f"Task started (ID: {task_id})."
elif name == "web_search":
r = web_search_action(parameters=args, player=self.ui)
result = r or "Done."
elif name == "file_processor":
if not args.get("file_path") and self.ui.current_file:
args["file_path"] = self.ui.current_file
r = file_processor(parameters=args, player=self.ui, speak=self.speak)
result = r or "Done."
elif name == "computer_control":
r = computer_control(parameters=args, player=self.ui)
result = r or "Done."
elif name == "game_updater":
r = game_updater(parameters=args, player=self.ui, speak=self.speak)
result = r or "Done."
elif name == "flight_finder":
r = flight_finder(parameters=args, player=self.ui)
result = r or "Done."
elif name == "shutdown_jarvis":
self.ui.write_log("SYS: Shutdown requested.")
def _shutdown():
import time, os
self.speak("Goodbye.")
time.sleep(2.5)
os._exit(0)
threading.Thread(target=_shutdown, daemon=True).start()
return "Shutting down."
else:
result = f"Unknown tool: {name}"
except Exception as e:
result = f"Tool '{name}' failed: {e}"
traceback.print_exc()
self.speak_error(name, e)
if not self.ui.muted:
self.ui.set_state("LISTENING")
print(f"[JARVIS] 📤 {name} → {str(result)[:80]}")
return result
# ------------------------------------------------------------------
# LLM processing loop
# ------------------------------------------------------------------
def _process_message(self, user_text: str) -> None:
"""
Full turn: user_text → LLM stream → TTS (overlapped) → tool execution
Streaming TTS: sentence events are piped to the TTS queue AS they
arrive from the LLM, so Kokoro starts synthesising sentence 1 while
the LLM is still generating sentence 2. This cuts perceived latency
from (LLM_total + TTS_total) down to roughly max(LLM_total, TTS_total).
Tool-call responses never emit sentence events, so the TTS overlap
only kicks in for pure conversational replies — which is exactly when
it matters most.
"""
self.ui.set_state("THINKING")
self.ui.write_log(f"You: {user_text}")
self._conversation.append({"role": "user", "content": user_text})
MAX_HISTORY = 10
if len(self._conversation) > MAX_HISTORY:
self._conversation = self._conversation[-MAX_HISTORY:]
messages = [
{"role": "system", "content": self._build_system_prompt()}
] + list(self._conversation)
# Tools whose output needs a second LLM round to summarise/interpret.
# Everything else returns a user-ready string → speak directly.
_NEEDS_LLM_ROUND = {"web_search", "screen_process", "agent_task"}
MAX_TOOL_ROUNDS = 6
for _round in range(MAX_TOOL_ROUNDS):
final_content = ""
final_tool_calls: list = []
# Sentences already queued to TTS during streaming (may be empty
# for tool-call rounds where the model emits no content).
_streamed: list[str] = []
try:
for event in call_llm_stream(messages, OLLAMA_TOOLS):
if event["type"] == "sentence":
# ── Overlap TTS with LLM generation ─────────────────
# Queue this sentence immediately; the TTS worker
# synthesises it while the LLM is still generating
# the next one.
_streamed.append(event["text"])
self.speak(event["text"])
elif event["type"] == "done":
final_content = event["content"]
final_tool_calls = event["tool_calls"]
except RuntimeError as e:
self.speak_error("LLM", e)
return
# ── No tool calls: pure conversational reply ─────────────────────
if not final_tool_calls:
if _streamed:
# Sentences already queued to TTS — just update history/log.
assistant_msg = {"role": "assistant", "content": final_content}
messages.append(assistant_msg)
self._conversation.append(assistant_msg)
self.ui.write_log(f"Jarvis: {final_content}")
elif final_content:
# Very short response (no sentence boundary) — speak now.
assistant_msg = {"role": "assistant", "content": final_content}
messages.append(assistant_msg)
self._conversation.append(assistant_msg)
self.ui.write_log(f"Jarvis: {final_content}")
self.speak(final_content)
break
# ── Tool calls present ────────────────────────────────────────────
assistant_msg = {
"role": "assistant",
"content": final_content or "",
"tool_calls": final_tool_calls,
}
messages.append(assistant_msg)
self._conversation.append(assistant_msg)
# ── Fast path: save_memory + verbal content in same round ────────
_only_memory = all(
tc.get("function", {}).get("name") == "save_memory"
for tc in final_tool_calls
)
if _only_memory and final_content:
for tc in final_tool_calls:
fn = tc.get("function", {})
targs = fn.get("arguments", {})
if isinstance(targs, str):
try:
targs = json.loads(targs)
except Exception:
targs = {}
self._execute_tool("save_memory", targs)
assistant_msg2 = {"role": "assistant", "content": final_content}
messages.append(assistant_msg2)
self._conversation.append(assistant_msg2)
self.ui.write_log(f"Jarvis: {final_content}")
if not _streamed:
self.speak(final_content)
break
# ── Execute tools ─────────────────────────────────────────────────
all_silent = True
_tool_results: list[tuple[str, str]] = []
for tc in final_tool_calls:
fn = tc.get("function", {})
tname = fn.get("name", "")
targs = fn.get("arguments", {})
if isinstance(targs, str):
try:
targs = json.loads(targs)
except Exception:
targs = {}
tc_id = tc.get("id", "")
self.ui.write_log(f"SYS: ▶ {tname}")
result = self._execute_tool(tname, targs)
if result != "__SILENT__":
all_silent = False
_tool_results.append((tname, result))
tool_msg: dict = {
"role": "tool",
"content": "Done." if result == "__SILENT__" else str(result),
}
if tc_id:
tool_msg["tool_call_id"] = tc_id
messages.append(tool_msg)
self._conversation.append(tool_msg)
# ── Fast-ack: every call was save_memory (silent) ────────────────
if all_silent:
_saved_name: str | None = None
for _tc in final_tool_calls:
_fn = _tc.get("function", {})
if _fn.get("name") == "save_memory":
_a = _fn.get("arguments", {})
if isinstance(_a, str):
try:
_a = json.loads(_a)
except Exception:
_a = {}
if isinstance(_a, dict) and _a.get("key") == "name" and _a.get("value"):
_saved_name = str(_a["value"])
break
_ack = f"Got it, {_saved_name}." if _saved_name else "Noted."
_amsg = {"role": "assistant", "content": _ack}
messages.append(_amsg)
self._conversation.append(_amsg)
self.ui.write_log(f"Jarvis: {_ack}")
self.speak(_ack)
break
# ── Direct-result: speak tool output, skip LLM round ────────────
if _tool_results and not any(n in _NEEDS_LLM_ROUND for n, _ in _tool_results):
_, _reply = _tool_results[-1]
_amsg = {"role": "assistant", "content": _reply}
messages.append(_amsg)
self._conversation.append(_amsg)
self.ui.write_log(f"Jarvis: {_reply}")
self.speak(_reply)
break
if not self.ui.muted:
self.ui.set_state("LISTENING")
# ------------------------------------------------------------------
# STT listening loops
# ------------------------------------------------------------------
def _listen_whisper(self) -> None:
"""Mic → VAD → Whisper → LLM loop."""
vad = _VADBuffer()
q: queue.Queue = queue.Queue(maxsize=200)
def callback(indata, frames, time_info, status):
with self._speaking_lock:
is_speaking = self._speaking
if not is_speaking and not self.ui.muted:
try:
q.put_nowait(indata.copy())
except queue.Full:
pass
try:
with sd.InputStream(
samplerate=SAMPLE_RATE_IN,
channels=CHANNELS,
dtype="float32",
blocksize=BLOCK_SIZE,
callback=callback,
):
self.ui.write_log("SYS: Mic active (Whisper STT).")
while True:
try:
chunk = q.get(timeout=0.1)
audio = vad.process(chunk.flatten())
if audio is not None:
self.ui.set_state("THINKING")
text = self._stt.transcribe(audio)
if text.strip():
self._process_message(text)
except queue.Empty:
pass
except Exception as e:
print(f"[STT-Whisper] Mic error: {e}")
traceback.print_exc()
self.ui.write_log(
f"ERR: Microphone — {e}\n"
" Check: microphone plugged in, OS privacy permissions granted, "
"correct audio device selected in system settings."
)
def _listen_vosk(self) -> None:
"""Mic → Vosk streaming → LLM loop."""
q: queue.Queue = queue.Queue(maxsize=200)
def callback(indata, frames, time_info, status):
with self._speaking_lock:
is_speaking = self._speaking
if not is_speaking and not self.ui.muted:
try:
q.put_nowait(indata.copy())
except queue.Full:
pass
try:
with sd.InputStream(
samplerate=SAMPLE_RATE_IN,
channels=CHANNELS,
dtype="int16",
blocksize=4096,
callback=callback,
):
self.ui.write_log("SYS: Mic active (Vosk STT).")
while True:
try:
chunk = q.get(timeout=0.1)
text, is_final = self._stt.process_chunk(chunk.tobytes())
if is_final and text.strip():
self._process_message(text)
except queue.Empty:
pass
except Exception as e:
print(f"[STT-Vosk] Mic error: {e}")
traceback.print_exc()
self.ui.write_log(
f"ERR: Microphone — {e}\n"
" Check: microphone plugged in, OS privacy permissions granted, "
"correct audio device selected in system settings."
)
# ------------------------------------------------------------------
# Text command loop (UI input box)
# ------------------------------------------------------------------
def _text_command_loop(self) -> None:
while True:
try:
text = self._text_queue.get(timeout=0.5)
if text.strip():
self._process_message(text)
except queue.Empty:
pass
# ------------------------------------------------------------------
# Entry point
# ------------------------------------------------------------------
def run(self) -> None:
"""
Startup strategy — optimised for minimum time-to-interactive:
1. LLM warmup + STT load → parallel, fast (~3s)
2. TTS load → parallel, slow (~20s for Kokoro)
3. Wait only for (1) → go online immediately
4. TTS finishes in BG → queued speech plays automatically
"""
try:
self.ui.on_reconfigure = self.reconfigure
# ── Ollama ────────────────────────────────────────────────────
from core.llm_client import ensure_ollama_running, warmup_model, check_model_available
self.ui.write_log("SYS: Checking Ollama…")
if ensure_ollama_running():
self.ui.write_log("SYS: Ollama OK.")
check_model_available(log=self.ui.write_log)
else:
self.ui.write_log("ERR: Ollama unavailable — run: ollama serve")
# ── Config ────────────────────────────────────────────────────
stt_engine = self._config.get("stt_engine", "whisper").lower()
stt_language = self._config.get("stt_language", "auto")
stt_model = self._config.get("stt_model", "base")
tts_engine = self._config.get("tts_engine", "edgetts").lower()
# ── Startup progress panel ────────────────────────────────────
self.ui.show_startup_panel()
_warmup_done = threading.Event()
_stt_done = threading.Event()
# ── LLM warmup thread ─────────────────────────────────────────
def _do_warmup():
try:
# Pass the STATIC system prompt so Ollama evaluates and caches
# its KV state during startup. Real requests start with the same
# static prefix → Ollama reuses cached KV → first token <1 s
# instead of the ~17 s it takes to re-evaluate 300+ tokens cold.
static_prompt = _load_system_prompt()
ok = warmup_model(system_prompt=static_prompt)
if ok:
self.ui.write_log("SYS: LLM ready.")
else:
# warmup() returned False — check if model is missing
from core.llm_client import check_model_available
if not check_model_available(log=self.ui.write_log):
pass # warning already written by check_model_available
else:
self.ui.write_log("WRN: LLM warmup failed — is Ollama running?")
# GPU usage tip (cross-platform, non-blocking)
try:
import torch
if torch.cuda.is_available():
self.ui.write_log(f"SYS: GPU detected — {torch.cuda.get_device_name(0)}")
else:
self.ui.write_log(
"WRN: CUDA not detected — LLM running on CPU (slow). "
"For GPU speed install CUDA PyTorch: "
"pip install torch --index-url https://download.pytorch.org/whl/cu118"
)
except Exception:
pass # torch not installed or not applicable
self.ui.mark_startup_ready("llm")
except Exception as e:
self.ui.write_log(f"ERR: LLM warmup — {e}")
self.ui.mark_startup_ready("llm", error=True)
finally:
_warmup_done.set()
# ── STT load thread ───────────────────────────────────────────
def _do_stt():
try:
self.ui.write_log(f"SYS: Loading {stt_engine.upper()} STT…")
if stt_engine == "vosk":
from core.stt import VoskSTT
self._stt = VoskSTT(
self._config.get("vosk_model_path"),
language=stt_language,
)
else:
from core.stt import WhisperSTT
self._stt = WhisperSTT(stt_model, language=stt_language)
self.ui.write_log("SYS: STT ready.")
self.ui.mark_startup_ready("stt")
except Exception as e:
self.ui.write_log(f"ERR: STT — {e}")
self.ui.mark_startup_ready("stt", error=True)
finally:
_stt_done.set()
# ── TTS load thread — does NOT block going online ─────────────
def _do_tts():
try:
self.ui.write_log(f"SYS: Loading {tts_engine.upper()} TTS…")
if tts_engine == "kokoro":
self.ui.write_log("SYS: Kokoro — loading model + compiling JIT…")
from core.tts import create_tts_player
self._tts = create_tts_player(self._config)
self._tts_ready.set() # unblock _tts_worker
self.ui.write_log("SYS: TTS ready.")
self.ui.mark_startup_ready("tts")
self.ui.set_startup_status("● All systems ready.")
self.ui.hide_startup_panel()
self.speak("Jarvis fully online.")
except Exception as e:
import traceback as _tb; _tb.print_exc()
self.ui.write_log(f"ERR: TTS — {e}")
self.ui.mark_startup_ready("tts", error=True)
self._tts_ready.set()
# Launch all three simultaneously
self.ui.write_log("SYS: Loading systems in parallel…")
threading.Thread(target=_do_warmup, daemon=True).start()
threading.Thread(target=_do_stt, daemon=True).start()
threading.Thread(target=_do_tts, daemon=True).start()
# ── Wait ONLY for STT + LLM (fast) ────────────────────────────
_warmup_done.wait(timeout=60)
_stt_done.wait(timeout=60)
# ── Go online immediately ──────────────────────────────────────
self.ui.write_log("SYS: JARVIS online.")
self.ui.set_state("LISTENING")
self.ui.set_startup_status("● JARVIS online · Voice loading in background…")
threading.Thread(target=self._tts_worker, daemon=True).start()
threading.Thread(target=self._text_command_loop, daemon=True).start()
# STT loop — blocks this thread forever
if stt_engine == "vosk":
self._listen_vosk()
else:
self._listen_whisper()
except Exception as e:
self.ui.write_log(f"ERR: Init failed — {e}")
traceback.print_exc()
# ---------------------------------------------------------------------------
# Entry
# ---------------------------------------------------------------------------
def main() -> None:
# ── Pre-import torch in background immediately ─────────────────────────
# By the time the TTS thread starts (~5s from now), torch will already
# be in sys.modules — removing it from the TTS critical path entirely.
def _preload_torch():
try:
import torch # noqa: F401 (side-effect import only)
except Exception:
pass
threading.Thread(target=_preload_torch, daemon=True).start()
# ───────────────────────────────────────────────────────────────────────
ui = JarvisUI("face.png")
def runner():
# 1. Wait until the user completes the setup overlay (first run)
# or config already exists (subsequent runs).
ui.wait_for_api_key()
# 2. Install any missing engine packages before loading engines.
# Progress is streamed to the log panel in real time.
ui.write_log("SYS: Checking dependencies…")
cfg = _load_config()
_install_done = threading.Event()
def _do_install():
try:
from core.installer import install_for_config
install_for_config(cfg, log=ui.write_log)
except Exception as e:
ui.write_log(f"ERR: Dependency install — {e}")
finally:
_install_done.set()
threading.Thread(target=_do_install, daemon=True).start()
_install_done.wait() # blocks runner thread; UI remains responsive
# 3. Start the assistant (loads STT / TTS / LLM).
jarvis = JarvisLocal(ui)
try:
jarvis.run()
except KeyboardInterrupt:
print("\n[MARK XL] Shutting down…")
threading.Thread(target=runner, daemon=True).start()
ui.root.mainloop()
if __name__ == "__main__":
main()