1328 lines
59 KiB
Python
1328 lines
59 KiB
Python
"""
|
|
MARK XL — Local LLM Edition
|
|
STT (Whisper / Vosk) + Ollama LLM + TTS (EdgeTTS / Kokoro / ElevenLabs)
|
|
All Gemini / Google-AI dependencies removed.
|
|
"""
|
|
# ── Silence verbose logs + block heavy unused backends ─────────────────────
|
|
import os as _os
|
|
_os.environ.setdefault("TF_CPP_MIN_LOG_LEVEL", "3") # TensorFlow C++ noise
|
|
_os.environ.setdefault("TF_ENABLE_ONEDNN_OPTS", "0") # oneDNN banner
|
|
_os.environ.setdefault("GRPC_VERBOSITY", "ERROR")
|
|
# USE_TF=0 prevents transformers from importing TensorFlow (saves 4-8 s).
|
|
# We intentionally do NOT set USE_TORCH or USE_JAX — forcing those values
|
|
# breaks transformers' lazy-loader on some versions (AutoModel disappears
|
|
# from the namespace). Let transformers auto-detect the available backends.
|
|
_os.environ.setdefault("USE_TF", "0")
|
|
_os.environ.setdefault("TOKENIZERS_PARALLELISM", "false")
|
|
import warnings as _warnings
|
|
_warnings.filterwarnings("ignore", category=UserWarning)
|
|
_warnings.filterwarnings("ignore", category=DeprecationWarning)
|
|
_warnings.filterwarnings("ignore", category=FutureWarning)
|
|
# ───────────────────────────────────────────────────────────────────────────
|
|
|
|
# ── Bootstrap: auto-install base UI packages before anything else ──────────
|
|
# Uses only stdlib so it works even on a completely fresh Python install.
|
|
import importlib.util as _ilu
|
|
import subprocess as _sp
|
|
import sys as _sys
|
|
|
|
_BASE_PKGS = [
|
|
("PyQt6", "PyQt6"),
|
|
("psutil", "psutil"),
|
|
("numpy", "numpy"),
|
|
("sounddevice", "sounddevice"),
|
|
("PIL", "pillow"),
|
|
("requests", "requests"),
|
|
]
|
|
|
|
def _bootstrap() -> None:
|
|
need = [pkg for mod, pkg in _BASE_PKGS if _ilu.find_spec(mod) is None]
|
|
if not need:
|
|
return
|
|
print(f"\n[MARK XL] First-run setup — installing: {', '.join(need)}")
|
|
print("[MARK XL] This happens only once.\n")
|
|
_sp.run([_sys.executable, "-m", "pip", "install", *need], check=True)
|
|
print("\n[MARK XL] Base packages ready — restarting…\n")
|
|
# Replace current process with a fresh one (picks up newly installed packages)
|
|
_os.execv(_sys.executable, [_sys.executable] + _sys.argv)
|
|
|
|
_bootstrap()
|
|
# ───────────────────────────────────────────────────────────────────────────
|
|
|
|
import json
|
|
import queue
|
|
import re
|
|
import sys
|
|
import threading
|
|
import traceback
|
|
from datetime import datetime
|
|
from pathlib import Path
|
|
|
|
import numpy as np
|
|
import sounddevice as sd
|
|
|
|
from ui import JarvisUI
|
|
from memory.memory_manager import load_memory, update_memory, format_memory_for_prompt
|
|
from core.llm_client import call_llm, call_llm_stream, get_llm_settings
|
|
|
|
from actions.file_processor import file_processor
|
|
from actions.flight_finder import flight_finder
|
|
from actions.open_app import open_app
|
|
from actions.weather_report import weather_action
|
|
from actions.send_message import send_message
|
|
from actions.reminder import reminder
|
|
from actions.computer_settings import computer_settings
|
|
from actions.screen_processor import screen_process
|
|
from actions.youtube_video import youtube_video
|
|
from actions.desktop import desktop_control
|
|
from actions.browser_control import browser_control
|
|
from actions.file_controller import file_controller
|
|
from actions.code_helper import code_helper
|
|
from actions.dev_agent import dev_agent
|
|
from actions.web_search import web_search as web_search_action
|
|
from actions.computer_control import computer_control
|
|
from actions.game_updater import game_updater
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Paths
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _get_base_dir() -> Path:
|
|
if getattr(sys, "frozen", False):
|
|
return Path(sys.executable).parent
|
|
return Path(__file__).resolve().parent
|
|
|
|
|
|
BASE_DIR = _get_base_dir()
|
|
API_CONFIG_PATH = BASE_DIR / "config" / "api_keys.json"
|
|
PROMPT_PATH = BASE_DIR / "core" / "prompt.txt"
|
|
|
|
SAMPLE_RATE_IN = 16_000
|
|
BLOCK_SIZE = 1_024
|
|
CHANNELS = 1
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Tool declarations (Gemini format kept for readability;
|
|
# converted to OpenAI/Ollama format by _to_ollama_tools())
|
|
# ---------------------------------------------------------------------------
|
|
|
|
TOOL_DECLARATIONS = [
|
|
{
|
|
"name": "open_app",
|
|
"description": (
|
|
"Opens or launches any application, website, or program on the computer. "
|
|
"ALWAYS use this when the user says: open, launch, start, run, pull up, "
|
|
"or 'open X real quick'. Examples: 'open WhatsApp', 'open Chrome', "
|
|
"'launch Spotify', 'open calculator', 'pull up WhatsApp'. "
|
|
"Do NOT use send_message just because the app is a messaging app — "
|
|
"if the user only says to open it, call open_app."
|
|
),
|
|
"parameters": {
|
|
"type": "OBJECT",
|
|
"properties": {
|
|
"app_name": {"type": "STRING", "description": "Name of the application or website to open"}
|
|
},
|
|
"required": ["app_name"]
|
|
}
|
|
},
|
|
{
|
|
"name": "web_search",
|
|
"description": "Searches the web for any information.",
|
|
"parameters": {
|
|
"type": "OBJECT",
|
|
"properties": {
|
|
"query": {"type": "STRING", "description": "Search query"},
|
|
"mode": {"type": "STRING", "description": "search (default) or compare"},
|
|
"items": {"type": "ARRAY", "items": {"type": "STRING"}, "description": "Items to compare"},
|
|
"aspect": {"type": "STRING", "description": "price | specs | reviews"}
|
|
},
|
|
"required": ["query"]
|
|
}
|
|
},
|
|
{
|
|
"name": "weather_report",
|
|
"description": "Gives the weather report to user",
|
|
"parameters": {
|
|
"type": "OBJECT",
|
|
"properties": {"city": {"type": "STRING", "description": "City name"}},
|
|
"required": ["city"]
|
|
}
|
|
},
|
|
{
|
|
"name": "send_message",
|
|
"description": (
|
|
"Sends a message to a specific person via WhatsApp, Telegram, or similar. "
|
|
"ONLY use this when the user explicitly provides BOTH a recipient AND message content. "
|
|
"Example triggers: 'text John saying I am late', 'send a WhatsApp to mom that dinner is ready'. "
|
|
"Do NOT call this if the user only wants to open the app without sending a message."
|
|
),
|
|
"parameters": {
|
|
"type": "OBJECT",
|
|
"properties": {
|
|
"receiver": {"type": "STRING", "description": "Recipient contact name"},
|
|
"message_text": {"type": "STRING", "description": "The exact message text to send"},
|
|
"platform": {"type": "STRING", "description": "Platform: WhatsApp, Telegram, etc."}
|
|
},
|
|
"required": ["receiver", "message_text", "platform"]
|
|
}
|
|
},
|
|
{
|
|
"name": "reminder",
|
|
"description": "Sets a timed reminder using Task Scheduler.",
|
|
"parameters": {
|
|
"type": "OBJECT",
|
|
"properties": {
|
|
"date": {"type": "STRING", "description": "Date in YYYY-MM-DD format"},
|
|
"time": {"type": "STRING", "description": "Time in HH:MM format (24h)"},
|
|
"message": {"type": "STRING", "description": "Reminder message text"}
|
|
},
|
|
"required": ["date", "time", "message"]
|
|
}
|
|
},
|
|
{
|
|
"name": "youtube_video",
|
|
"description": (
|
|
"Controls YouTube. Use for: playing videos, summarizing a video's content, "
|
|
"getting video info, or showing trending videos."
|
|
),
|
|
"parameters": {
|
|
"type": "OBJECT",
|
|
"properties": {
|
|
"action": {"type": "STRING", "description": "play | summarize | get_info | trending"},
|
|
"query": {"type": "STRING", "description": "Search query for play action"},
|
|
"save": {"type": "BOOLEAN", "description": "Save summary to Notepad"},
|
|
"region": {"type": "STRING", "description": "Country code for trending e.g. TR, US"},
|
|
"url": {"type": "STRING", "description": "Video URL for get_info action"},
|
|
},
|
|
"required": []
|
|
}
|
|
},
|
|
{
|
|
"name": "screen_process",
|
|
"description": (
|
|
"Captures and analyzes the screen or webcam image. "
|
|
"MUST be called when user asks what is on screen, what you see, "
|
|
"analyze my screen, look at camera, etc. "
|
|
"You have NO visual ability without this tool."
|
|
),
|
|
"parameters": {
|
|
"type": "OBJECT",
|
|
"properties": {
|
|
"angle": {"type": "STRING", "description": "'screen' or 'camera'. Default: 'screen'"},
|
|
"text": {"type": "STRING", "description": "The question about the captured image"}
|
|
},
|
|
"required": ["text"]
|
|
}
|
|
},
|
|
{
|
|
"name": "computer_settings",
|
|
"description": (
|
|
"Controls the computer: volume, brightness, window management, keyboard shortcuts, "
|
|
"typing text on screen, closing apps, fullscreen, dark mode, WiFi, restart, shutdown, "
|
|
"scrolling, tab management, zoom, screenshots, lock screen, refresh/reload page."
|
|
),
|
|
"parameters": {
|
|
"type": "OBJECT",
|
|
"properties": {
|
|
"action": {"type": "STRING", "description": "The action to perform"},
|
|
"description": {"type": "STRING", "description": "Natural language description"},
|
|
"value": {"type": "STRING", "description": "Optional value"}
|
|
},
|
|
"required": []
|
|
}
|
|
},
|
|
{
|
|
"name": "browser_control",
|
|
"description": (
|
|
"Controls any web browser. Use for: opening websites, searching the web, "
|
|
"clicking elements, filling forms, scrolling, screenshots, navigation."
|
|
),
|
|
"parameters": {
|
|
"type": "OBJECT",
|
|
"properties": {
|
|
"action": {"type": "STRING", "description": "go_to | search | click | type | scroll | fill_form | smart_click | smart_type | get_text | get_url | press | new_tab | close_tab | screenshot | back | forward | reload | switch | list_browsers | close | close_all"},
|
|
"browser": {"type": "STRING", "description": "chrome | edge | firefox | opera | operagx | brave | vivaldi | safari"},
|
|
"url": {"type": "STRING", "description": "URL for go_to / new_tab action"},
|
|
"query": {"type": "STRING", "description": "Search query"},
|
|
"engine": {"type": "STRING", "description": "google | bing | duckduckgo | yandex"},
|
|
"selector": {"type": "STRING", "description": "CSS selector for click/type"},
|
|
"text": {"type": "STRING", "description": "Text to click or type"},
|
|
"description": {"type": "STRING", "description": "Element description for smart_click/smart_type"},
|
|
"direction": {"type": "STRING", "description": "up | down for scroll"},
|
|
"amount": {"type": "INTEGER", "description": "Scroll amount in pixels"},
|
|
"key": {"type": "STRING", "description": "Key name for press"},
|
|
"path": {"type": "STRING", "description": "Save path for screenshot"},
|
|
"incognito": {"type": "BOOLEAN", "description": "Open in private/incognito mode"},
|
|
"clear_first": {"type": "BOOLEAN", "description": "Clear field before typing"},
|
|
},
|
|
"required": ["action"]
|
|
}
|
|
},
|
|
{
|
|
"name": "file_controller",
|
|
"description": "Manages files and folders: list, create, delete, move, copy, rename, read, write, find, disk usage.",
|
|
"parameters": {
|
|
"type": "OBJECT",
|
|
"properties": {
|
|
"action": {"type": "STRING", "description": "list | create_file | create_folder | delete | move | copy | rename | read | write | find | largest | disk_usage | organize_desktop | info"},
|
|
"path": {"type": "STRING", "description": "File/folder path or shortcut: desktop, downloads, documents, home"},
|
|
"destination": {"type": "STRING", "description": "Destination path for move/copy"},
|
|
"new_name": {"type": "STRING", "description": "New name for rename"},
|
|
"content": {"type": "STRING", "description": "Content for create_file/write"},
|
|
"name": {"type": "STRING", "description": "File name to search for"},
|
|
"extension": {"type": "STRING", "description": "File extension to search"},
|
|
"count": {"type": "INTEGER", "description": "Number of results for largest"},
|
|
},
|
|
"required": ["action"]
|
|
}
|
|
},
|
|
{
|
|
"name": "desktop_control",
|
|
"description": "Controls the desktop: wallpaper, organize, clean, list, stats.",
|
|
"parameters": {
|
|
"type": "OBJECT",
|
|
"properties": {
|
|
"action": {"type": "STRING", "description": "wallpaper | wallpaper_url | organize | clean | list | stats | task"},
|
|
"path": {"type": "STRING", "description": "Image path for wallpaper"},
|
|
"url": {"type": "STRING", "description": "Image URL for wallpaper_url"},
|
|
"mode": {"type": "STRING", "description": "by_type or by_date for organize"},
|
|
"task": {"type": "STRING", "description": "Natural language desktop task"},
|
|
},
|
|
"required": ["action"]
|
|
}
|
|
},
|
|
{
|
|
"name": "code_helper",
|
|
"description": "Writes, edits, explains, runs, or builds code files.",
|
|
"parameters": {
|
|
"type": "OBJECT",
|
|
"properties": {
|
|
"action": {"type": "STRING", "description": "write | edit | explain | run | build | auto"},
|
|
"description": {"type": "STRING", "description": "What the code should do"},
|
|
"language": {"type": "STRING", "description": "Programming language"},
|
|
"output_path": {"type": "STRING", "description": "Where to save the file"},
|
|
"file_path": {"type": "STRING", "description": "Path to existing file"},
|
|
"code": {"type": "STRING", "description": "Raw code string for explain"},
|
|
"args": {"type": "STRING", "description": "CLI arguments"},
|
|
"timeout": {"type": "INTEGER", "description": "Execution timeout in seconds"},
|
|
},
|
|
"required": ["action"]
|
|
}
|
|
},
|
|
{
|
|
"name": "dev_agent",
|
|
"description": "Builds complete multi-file projects from scratch.",
|
|
"parameters": {
|
|
"type": "OBJECT",
|
|
"properties": {
|
|
"description": {"type": "STRING", "description": "What the project should do"},
|
|
"language": {"type": "STRING", "description": "Programming language"},
|
|
"project_name": {"type": "STRING", "description": "Optional project folder name"},
|
|
"timeout": {"type": "INTEGER", "description": "Run timeout in seconds"},
|
|
},
|
|
"required": ["description"]
|
|
}
|
|
},
|
|
{
|
|
"name": "agent_task",
|
|
"description": (
|
|
"Executes complex multi-step tasks requiring multiple different tools. "
|
|
"Examples: 'research X and save to file', 'find and organize files'. "
|
|
"DO NOT use for single commands."
|
|
),
|
|
"parameters": {
|
|
"type": "OBJECT",
|
|
"properties": {
|
|
"goal": {"type": "STRING", "description": "Complete description of what to accomplish"},
|
|
"priority": {"type": "STRING", "description": "low | normal | high"}
|
|
},
|
|
"required": ["goal"]
|
|
}
|
|
},
|
|
{
|
|
"name": "computer_control",
|
|
"description": "Direct computer control: type, click, hotkeys, scroll, move mouse, screenshots, find elements on screen.",
|
|
"parameters": {
|
|
"type": "OBJECT",
|
|
"properties": {
|
|
"action": {"type": "STRING", "description": "type | smart_type | click | double_click | right_click | hotkey | press | scroll | move | copy | paste | screenshot | wait | clear_field | focus_window | screen_find | screen_click | random_data | user_data"},
|
|
"text": {"type": "STRING", "description": "Text to type or paste"},
|
|
"x": {"type": "INTEGER", "description": "X coordinate"},
|
|
"y": {"type": "INTEGER", "description": "Y coordinate"},
|
|
"keys": {"type": "STRING", "description": "Key combination e.g. 'ctrl+c'"},
|
|
"key": {"type": "STRING", "description": "Single key e.g. 'enter'"},
|
|
"direction": {"type": "STRING", "description": "up | down | left | right"},
|
|
"amount": {"type": "INTEGER", "description": "Scroll amount"},
|
|
"seconds": {"type": "NUMBER", "description": "Seconds to wait"},
|
|
"title": {"type": "STRING", "description": "Window title for focus_window"},
|
|
"description": {"type": "STRING", "description": "Element description"},
|
|
"type": {"type": "STRING", "description": "Data type for random_data"},
|
|
"field": {"type": "STRING", "description": "Field for user_data"},
|
|
"clear_first": {"type": "BOOLEAN", "description": "Clear field before typing"},
|
|
"path": {"type": "STRING", "description": "Save path for screenshot"},
|
|
},
|
|
"required": ["action"]
|
|
}
|
|
},
|
|
{
|
|
"name": "game_updater",
|
|
"description": (
|
|
"THE ONLY tool for ANY Steam or Epic Games request. "
|
|
"Use for: installing, downloading, updating games, listing installed games."
|
|
),
|
|
"parameters": {
|
|
"type": "OBJECT",
|
|
"properties": {
|
|
"action": {"type": "STRING", "description": "update | install | list | download_status | schedule | cancel_schedule | schedule_status"},
|
|
"platform": {"type": "STRING", "description": "steam | epic | both"},
|
|
"game_name": {"type": "STRING", "description": "Game name"},
|
|
"app_id": {"type": "STRING", "description": "Steam AppID"},
|
|
"hour": {"type": "INTEGER", "description": "Hour for scheduled update 0-23"},
|
|
"minute": {"type": "INTEGER", "description": "Minute for scheduled update 0-59"},
|
|
"shutdown_when_done": {"type": "BOOLEAN", "description": "Shut down PC when done"},
|
|
},
|
|
"required": []
|
|
}
|
|
},
|
|
{
|
|
"name": "flight_finder",
|
|
"description": "Searches Google Flights and speaks the best options.",
|
|
"parameters": {
|
|
"type": "OBJECT",
|
|
"properties": {
|
|
"origin": {"type": "STRING", "description": "Departure city or airport code"},
|
|
"destination": {"type": "STRING", "description": "Arrival city or airport code"},
|
|
"date": {"type": "STRING", "description": "Departure date"},
|
|
"return_date": {"type": "STRING", "description": "Return date for round trips"},
|
|
"passengers": {"type": "INTEGER", "description": "Number of passengers"},
|
|
"cabin": {"type": "STRING", "description": "economy | premium | business | first"},
|
|
"save": {"type": "BOOLEAN", "description": "Save results to Notepad"},
|
|
},
|
|
"required": ["origin", "destination", "date"]
|
|
}
|
|
},
|
|
{
|
|
"name": "shutdown_jarvis",
|
|
"description": (
|
|
"Shuts down the assistant completely. "
|
|
"Call this when the user expresses intent to end the conversation, "
|
|
"close the assistant, say goodbye, or stop Jarvis."
|
|
),
|
|
"parameters": {"type": "OBJECT", "properties": {}}
|
|
},
|
|
{
|
|
"name": "file_processor",
|
|
"description": (
|
|
"Processes any file that the user has uploaded or dropped onto the interface. "
|
|
"Supports: images, PDFs, Word docs, CSV/Excel, JSON, code files, audio, video, archives."
|
|
),
|
|
"parameters": {
|
|
"type": "OBJECT",
|
|
"properties": {
|
|
"file_path": {"type": "STRING", "description": "Full path to the uploaded file"},
|
|
"action": {"type": "STRING", "description": "What to do with the file"},
|
|
"instruction": {"type": "STRING", "description": "Free-form instruction"},
|
|
"format": {"type": "STRING", "description": "Target format for conversion"},
|
|
"width": {"type": "INTEGER", "description": "Target width for image resize"},
|
|
"height": {"type": "INTEGER", "description": "Target height for image resize"},
|
|
"scale": {"type": "NUMBER", "description": "Scale factor"},
|
|
"quality": {"type": "INTEGER", "description": "Quality 1-100"},
|
|
"start": {"type": "STRING", "description": "Start time for trim"},
|
|
"end": {"type": "STRING", "description": "End time for trim"},
|
|
"timestamp": {"type": "STRING", "description": "Timestamp for video frame extraction"},
|
|
"column": {"type": "STRING", "description": "Column name for CSV filter/sort"},
|
|
"value": {"type": "STRING", "description": "Filter value"},
|
|
"condition": {"type": "STRING", "description": "Filter condition"},
|
|
"ascending": {"type": "BOOLEAN", "description": "Sort order"},
|
|
"save": {"type": "BOOLEAN", "description": "Save result to file"},
|
|
"destination": {"type": "STRING", "description": "Output folder for archive extract"},
|
|
},
|
|
"required": []
|
|
}
|
|
},
|
|
{
|
|
"name": "save_memory",
|
|
"description": (
|
|
"Save a personal fact about the user to permanent long-term memory. "
|
|
"MANDATORY: call this IMMEDIATELY (without asking) whenever the user states or corrects: "
|
|
"their name, age, city, job, school, language, nationality, a preference, a goal, or a relationship. "
|
|
"Examples: "
|
|
"'my name is Fatih' → (identity, name, Fatih) | "
|
|
"'not Travis, Fatih' → (identity, name, Fatih) | "
|
|
"'I am 22' → (identity, age, 22) | "
|
|
"'I live in Ankara' → (identity, city, Ankara) | "
|
|
"'I prefer dark mode' → (preferences, ui_theme, dark mode). "
|
|
"Call SILENTLY alongside your verbal reply — never announce that you are saving."
|
|
),
|
|
"parameters": {
|
|
"type": "OBJECT",
|
|
"properties": {
|
|
"category": {
|
|
"type": "STRING",
|
|
"description": (
|
|
"identity (name/age/city/job/school/nationality) | "
|
|
"preferences (likes/dislikes/habits) | "
|
|
"projects (active work/goals) | "
|
|
"relationships (people in their life) | "
|
|
"wishes (future plans/wants) | "
|
|
"notes (anything else)"
|
|
)
|
|
},
|
|
"key": {"type": "STRING", "description": "Short snake_case key, e.g. 'name', 'age', 'favorite_color'"},
|
|
"value": {"type": "STRING", "description": "Concise value in English"},
|
|
},
|
|
"required": ["category", "key", "value"]
|
|
}
|
|
},
|
|
]
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Convert Gemini-style declarations to OpenAI/Ollama format
|
|
# ---------------------------------------------------------------------------
|
|
|
|
_TYPE_MAP = {
|
|
"OBJECT": "object", "STRING": "string", "ARRAY": "array",
|
|
"INTEGER": "integer", "BOOLEAN": "boolean", "NUMBER": "number",
|
|
}
|
|
|
|
|
|
def _convert_type(t: str) -> str:
|
|
return _TYPE_MAP.get(t, t.lower()) if isinstance(t, str) else t
|
|
|
|
|
|
def _convert_props(props: dict) -> dict:
|
|
out = {}
|
|
for k, v in props.items():
|
|
nv = dict(v)
|
|
if "type" in nv:
|
|
nv["type"] = _convert_type(nv["type"])
|
|
if "items" in nv and isinstance(nv["items"], dict):
|
|
nv["items"] = {"type": _convert_type(nv["items"].get("type", "string"))}
|
|
out[k] = nv
|
|
return out
|
|
|
|
|
|
def _to_ollama_tools(decls: list) -> list:
|
|
tools = []
|
|
for d in decls:
|
|
params = d.get("parameters", {})
|
|
new_params: dict = {
|
|
"type": "object",
|
|
"properties": _convert_props(params.get("properties", {})),
|
|
}
|
|
req = params.get("required")
|
|
if req:
|
|
new_params["required"] = req
|
|
tools.append({
|
|
"type": "function",
|
|
"function": {
|
|
"name": d["name"],
|
|
"description": d["description"],
|
|
"parameters": new_params,
|
|
},
|
|
})
|
|
return tools
|
|
|
|
|
|
OLLAMA_TOOLS = _to_ollama_tools(TOOL_DECLARATIONS)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _load_config() -> dict:
|
|
try:
|
|
with open(API_CONFIG_PATH, encoding="utf-8") as f:
|
|
return json.load(f)
|
|
except Exception:
|
|
return {}
|
|
|
|
|
|
def _load_system_prompt() -> str:
|
|
try:
|
|
return PROMPT_PATH.read_text(encoding="utf-8")
|
|
except Exception:
|
|
return (
|
|
"You are JARVIS, Tony Stark's AI assistant. "
|
|
"Be concise, direct, and always use the provided tools to complete tasks. "
|
|
"Never simulate or guess results — always call the appropriate tool."
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Voice Activity Detection (used for Whisper listen loop)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class _VADBuffer:
|
|
"""Energy-based VAD: buffers audio until end of utterance."""
|
|
|
|
def __init__(
|
|
self,
|
|
sample_rate: int = 16_000,
|
|
silence_sec: float = 1.0, # silence after last word → send to STT (1.0s = fast enough + won't cut mid-sentence with hysteresis)
|
|
speech_thresh: float = 0.015, # RMS above this = speech (raised from 0.010 to kill background noise false-positives)
|
|
silence_thresh: float = 0.008, # RMS below this = silence (hysteresis gap — much lower bar than speech_thresh to avoid mid-sentence cuts)
|
|
min_speech_sec: float = 0.4,
|
|
max_speech_sec: float = 30.0,
|
|
):
|
|
self._sr = sample_rate
|
|
self._sil_n = int(silence_sec * sample_rate)
|
|
self._speech_thresh = speech_thresh
|
|
self._sil_thresh = silence_thresh
|
|
self._min_n = int(min_speech_sec * sample_rate)
|
|
self._max_n = int(max_speech_sec * sample_rate)
|
|
self._buf: list[np.ndarray] = []
|
|
self._in_spch = False
|
|
self._sil_cnt = 0
|
|
def process(self, chunk: np.ndarray) -> np.ndarray | None:
|
|
"""
|
|
Feed one audio chunk (float32 mono).
|
|
Returns complete utterance when speech ends, otherwise None.
|
|
|
|
Uses hysteresis thresholds so the detector doesn't flicker on quiet
|
|
consonants or brief mid-sentence pauses:
|
|
- speech starts when RMS > speech_thresh
|
|
- speech ends only when RMS < silence_thresh (lower bar)
|
|
This prevents natural pauses ("I just... wanted to say") from
|
|
triggering a premature cut.
|
|
"""
|
|
rms = float(np.sqrt(np.mean(chunk ** 2)))
|
|
total_n = sum(len(c) for c in self._buf)
|
|
|
|
if rms > self._speech_thresh:
|
|
self._in_spch = True
|
|
self._sil_cnt = 0
|
|
self._buf.append(chunk.copy())
|
|
elif self._in_spch:
|
|
self._buf.append(chunk.copy())
|
|
if rms < self._sil_thresh:
|
|
self._sil_cnt += len(chunk)
|
|
|
|
if self._sil_cnt >= self._sil_n or total_n >= self._max_n:
|
|
audio = np.concatenate(self._buf)
|
|
self._buf = []
|
|
self._in_spch = False
|
|
self._sil_cnt = 0
|
|
if len(audio) >= self._min_n:
|
|
return audio
|
|
return None
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# JarvisLocal
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class JarvisLocal:
|
|
"""
|
|
Main assistant class.
|
|
Replaces JarvisLive (Gemini Live API) with:
|
|
STT (Whisper/Vosk) → Ollama LLM (tool calling) → TTS (Edge/Kokoro/ElevenLabs)
|
|
"""
|
|
|
|
def __init__(self, ui: JarvisUI):
|
|
self.ui = ui
|
|
self._config = _load_config()
|
|
self._stt = None
|
|
self._tts = None
|
|
self._tts_ready = threading.Event() # set when TTS engine is loaded
|
|
self._speaking = False
|
|
self._speaking_lock = threading.Lock()
|
|
self._text_queue: queue.Queue = queue.Queue()
|
|
self._tts_queue: queue.Queue = queue.Queue()
|
|
self._conversation: list[dict] = []
|
|
|
|
self.ui.on_text_command = self._on_text_command
|
|
|
|
# ------------------------------------------------------------------
|
|
# System prompt
|
|
# ------------------------------------------------------------------
|
|
|
|
def _build_system_prompt(self) -> str:
|
|
# ── ORDER MATTERS for Ollama KV prefix caching ─────────────────────
|
|
# Ollama caches the KV attention state of any stable prompt prefix.
|
|
# By putting the STATIC JARVIS protocol text FIRST, Ollama reuses its
|
|
# cached KV for all those tokens on every request. Only the small
|
|
# dynamic tail (memory + time, ~50-80 tokens) needs re-evaluation.
|
|
# This turns a 17-second first-token into a sub-second one after warmup.
|
|
#
|
|
# Rule: static content first → semi-static memory middle → dynamic time LAST.
|
|
sys_p = _load_system_prompt() # static — never changes mid-session
|
|
memory = load_memory()
|
|
mem_str = format_memory_for_prompt(memory) # semi-static — changes only when user tells facts
|
|
now = datetime.now()
|
|
time_ctx = (
|
|
f"[CURRENT DATE & TIME]\n"
|
|
f"Right now it is: {now.strftime('%A, %B %d, %Y — %I:%M %p')}\n"
|
|
f"Use this to calculate exact times for reminders."
|
|
)
|
|
parts = [sys_p]
|
|
if mem_str:
|
|
parts.append(mem_str)
|
|
parts.append(time_ctx)
|
|
return "\n\n".join(parts)
|
|
|
|
# ------------------------------------------------------------------
|
|
# Speaking state & TTS
|
|
# ------------------------------------------------------------------
|
|
|
|
# ------------------------------------------------------------------
|
|
# TTS queue worker — plays sentences sequentially, no overlaps
|
|
# ------------------------------------------------------------------
|
|
|
|
def _tts_worker(self) -> None:
|
|
# Block until TTS engine is loaded. Queued items are preserved
|
|
# and played immediately once loading completes — nothing is lost.
|
|
self._tts_ready.wait(timeout=120)
|
|
|
|
while True:
|
|
text = self._tts_queue.get()
|
|
try:
|
|
if text and self._tts:
|
|
with self._speaking_lock:
|
|
self._speaking = True
|
|
self.ui.set_state("SPEAKING")
|
|
self._tts.speak(text)
|
|
except Exception as e:
|
|
print(f"[TTS] speak error: {e}")
|
|
finally:
|
|
self._tts_queue.task_done()
|
|
if self._tts_queue.empty():
|
|
with self._speaking_lock:
|
|
self._speaking = False
|
|
if not self.ui.muted:
|
|
self.ui.set_state("LISTENING")
|
|
|
|
def set_speaking(self, value: bool) -> None:
|
|
with self._speaking_lock:
|
|
self._speaking = value
|
|
if value:
|
|
self.ui.set_state("SPEAKING")
|
|
elif not self.ui.muted:
|
|
self.ui.set_state("LISTENING")
|
|
|
|
def speak(self, text: str) -> None:
|
|
if not text or not self._tts:
|
|
return
|
|
with self._speaking_lock:
|
|
self._speaking = True
|
|
self._tts_queue.put(text)
|
|
|
|
def speak_error(self, tool_name: str, error) -> None:
|
|
short = str(error)[:120]
|
|
self.ui.write_log(f"ERR: {tool_name} — {short}")
|
|
self.speak(f"{tool_name} encountered an error.")
|
|
|
|
# ------------------------------------------------------------------
|
|
# Live reconfigure (called when user clicks Apply in Configure panel)
|
|
# ------------------------------------------------------------------
|
|
|
|
def reconfigure(self, new_config: dict) -> None:
|
|
"""Non-blocking: spawns a background thread to install + reload."""
|
|
threading.Thread(
|
|
target=self._do_reconfigure, args=(new_config,), daemon=True
|
|
).start()
|
|
|
|
def _do_reconfigure(self, new_config: dict) -> None:
|
|
old_stt_engine = self._config.get("stt_engine", "whisper").lower()
|
|
old_llm_model = self._config.get("llm_model", "")
|
|
new_stt_engine = new_config.get("stt_engine", "whisper").lower()
|
|
self._config = new_config
|
|
|
|
# Install any packages required by the new config
|
|
try:
|
|
from core.installer import install_for_config
|
|
install_for_config(new_config, log=self.ui.write_log)
|
|
except Exception as e:
|
|
self.ui.write_log(f"ERR: Dependency install — {e}")
|
|
|
|
# TTS: always hot-reload (runs in queue worker, safe to swap)
|
|
try:
|
|
from core.tts import create_tts_player
|
|
self._tts = create_tts_player(new_config)
|
|
self._tts_ready.set() # ensure worker isn't blocked
|
|
self.ui.write_log("SYS: TTS reconfigured.")
|
|
except Exception as e:
|
|
self.ui.write_log(f"ERR: TTS reconfigure — {e}")
|
|
|
|
# STT: hot-reload if same engine type; full restart needed if engine changed
|
|
if old_stt_engine == new_stt_engine:
|
|
try:
|
|
stt_language = new_config.get("stt_language", "auto")
|
|
if new_stt_engine == "vosk":
|
|
from core.stt import VoskSTT
|
|
self._stt = VoskSTT(new_config.get("vosk_model_path"), language=stt_language)
|
|
else:
|
|
from core.stt import WhisperSTT
|
|
self._stt = WhisperSTT(new_config.get("stt_model", "base"), language=stt_language)
|
|
self.ui.write_log("SYS: STT reconfigured.")
|
|
except Exception as e:
|
|
self.ui.write_log(f"ERR: STT reconfigure — {e}")
|
|
else:
|
|
self.ui.write_log("SYS: STT engine changed — restart required.")
|
|
|
|
# LLM warmup if model changed
|
|
if new_config.get("llm_model", "") != old_llm_model:
|
|
self.ui.write_log("SYS: Warming up new LLM model…")
|
|
from core.llm_client import warmup_model
|
|
warmup_model()
|
|
self.ui.write_log("SYS: New LLM model ready.")
|
|
|
|
if old_stt_engine == new_stt_engine:
|
|
self.speak("Configuration applied.")
|
|
else:
|
|
self.speak("LLM and TTS updated. Restart for speech engine change.")
|
|
|
|
# ------------------------------------------------------------------
|
|
# Text command (from UI input box)
|
|
# ------------------------------------------------------------------
|
|
|
|
def _on_text_command(self, text: str) -> None:
|
|
self._text_queue.put(text)
|
|
|
|
# ------------------------------------------------------------------
|
|
# Tool execution (routing unchanged from original)
|
|
# ------------------------------------------------------------------
|
|
|
|
def _execute_tool(self, name: str, args: dict) -> str:
|
|
print(f"[JARVIS] 🔧 {name} {args}")
|
|
self.ui.set_state("THINKING")
|
|
|
|
# save_memory is handled silently
|
|
if name == "save_memory":
|
|
category = args.get("category", "notes")
|
|
key = args.get("key", "")
|
|
value = args.get("value", "")
|
|
if key and value:
|
|
update_memory({category: {key: {"value": value}}})
|
|
print(f"[Memory] 💾 {category}/{key} = {value}")
|
|
if not self.ui.muted:
|
|
self.ui.set_state("LISTENING")
|
|
return "__SILENT__"
|
|
|
|
result = "Done."
|
|
try:
|
|
if name == "open_app":
|
|
r = open_app(parameters=args, response=None, player=self.ui)
|
|
result = r or f"Opened {args.get('app_name')}."
|
|
|
|
elif name == "weather_report":
|
|
r = weather_action(parameters=args, player=self.ui)
|
|
result = r or "Weather delivered."
|
|
|
|
elif name == "browser_control":
|
|
r = browser_control(parameters=args, player=self.ui)
|
|
result = r or "Done."
|
|
|
|
elif name == "file_controller":
|
|
r = file_controller(parameters=args, player=self.ui)
|
|
result = r or "Done."
|
|
|
|
elif name == "send_message":
|
|
r = send_message(parameters=args, response=None, player=self.ui, session_memory=None)
|
|
result = r or f"Message sent to {args.get('receiver')}."
|
|
|
|
elif name == "reminder":
|
|
r = reminder(parameters=args, response=None, player=self.ui)
|
|
result = r or "Reminder set."
|
|
|
|
elif name == "youtube_video":
|
|
r = youtube_video(parameters=args, response=None, player=self.ui)
|
|
result = r or "Done."
|
|
|
|
elif name == "screen_process":
|
|
# Synchronous call — returns analysis text which the LLM can speak
|
|
r = screen_process(parameters=args, response=None, player=self.ui, session_memory=None)
|
|
result = r if isinstance(r, str) and r else "Screen analyzed."
|
|
|
|
elif name == "computer_settings":
|
|
r = computer_settings(parameters=args, response=None, player=self.ui)
|
|
result = r or "Done."
|
|
|
|
elif name == "desktop_control":
|
|
r = desktop_control(parameters=args, player=self.ui)
|
|
result = r or "Done."
|
|
|
|
elif name == "code_helper":
|
|
r = code_helper(parameters=args, player=self.ui, speak=self.speak)
|
|
result = r or "Done."
|
|
|
|
elif name == "dev_agent":
|
|
r = dev_agent(parameters=args, player=self.ui, speak=self.speak)
|
|
result = r or "Done."
|
|
|
|
elif name == "agent_task":
|
|
from agent.task_queue import get_queue, TaskPriority
|
|
priority_map = {
|
|
"low": TaskPriority.LOW,
|
|
"normal": TaskPriority.NORMAL,
|
|
"high": TaskPriority.HIGH,
|
|
}
|
|
priority = priority_map.get(
|
|
args.get("priority", "normal").lower(), TaskPriority.NORMAL
|
|
)
|
|
task_id = get_queue().submit(
|
|
goal=args.get("goal", ""), priority=priority, speak=self.speak
|
|
)
|
|
result = f"Task started (ID: {task_id})."
|
|
|
|
elif name == "web_search":
|
|
r = web_search_action(parameters=args, player=self.ui)
|
|
result = r or "Done."
|
|
|
|
elif name == "file_processor":
|
|
if not args.get("file_path") and self.ui.current_file:
|
|
args["file_path"] = self.ui.current_file
|
|
r = file_processor(parameters=args, player=self.ui, speak=self.speak)
|
|
result = r or "Done."
|
|
|
|
elif name == "computer_control":
|
|
r = computer_control(parameters=args, player=self.ui)
|
|
result = r or "Done."
|
|
|
|
elif name == "game_updater":
|
|
r = game_updater(parameters=args, player=self.ui, speak=self.speak)
|
|
result = r or "Done."
|
|
|
|
elif name == "flight_finder":
|
|
r = flight_finder(parameters=args, player=self.ui)
|
|
result = r or "Done."
|
|
|
|
elif name == "shutdown_jarvis":
|
|
self.ui.write_log("SYS: Shutdown requested.")
|
|
|
|
def _shutdown():
|
|
import time, os
|
|
self.speak("Goodbye.")
|
|
time.sleep(2.5)
|
|
os._exit(0)
|
|
|
|
threading.Thread(target=_shutdown, daemon=True).start()
|
|
return "Shutting down."
|
|
|
|
else:
|
|
result = f"Unknown tool: {name}"
|
|
|
|
except Exception as e:
|
|
result = f"Tool '{name}' failed: {e}"
|
|
traceback.print_exc()
|
|
self.speak_error(name, e)
|
|
|
|
if not self.ui.muted:
|
|
self.ui.set_state("LISTENING")
|
|
|
|
print(f"[JARVIS] 📤 {name} → {str(result)[:80]}")
|
|
return result
|
|
|
|
# ------------------------------------------------------------------
|
|
# LLM processing loop
|
|
# ------------------------------------------------------------------
|
|
|
|
def _process_message(self, user_text: str) -> None:
|
|
"""
|
|
Full turn:
|
|
user_text → Ollama stream (with tools) → execute tools → final spoken response
|
|
|
|
The complete LLM response is collected first, then sent to TTS in one call.
|
|
This gives TTS engines (Kokoro, ElevenLabs) the full text so they can
|
|
apply natural prosody across sentence boundaries — no choppy pauses.
|
|
"""
|
|
self.ui.set_state("THINKING")
|
|
self.ui.write_log(f"You: {user_text}")
|
|
|
|
self._conversation.append({"role": "user", "content": user_text})
|
|
|
|
MAX_HISTORY = 10
|
|
if len(self._conversation) > MAX_HISTORY:
|
|
self._conversation = self._conversation[-MAX_HISTORY:]
|
|
|
|
messages = [
|
|
{"role": "system", "content": self._build_system_prompt()}
|
|
] + list(self._conversation)
|
|
|
|
MAX_TOOL_ROUNDS = 6
|
|
for _round in range(MAX_TOOL_ROUNDS):
|
|
final_content = ""
|
|
final_tool_calls: list = []
|
|
|
|
try:
|
|
for event in call_llm_stream(messages, OLLAMA_TOOLS):
|
|
if event["type"] == "done":
|
|
final_content = event["content"]
|
|
final_tool_calls = event["tool_calls"]
|
|
except RuntimeError as e:
|
|
self.speak_error("LLM", e)
|
|
return
|
|
|
|
if not final_tool_calls:
|
|
# Full response ready — send complete text to TTS in one call
|
|
if final_content:
|
|
assistant_msg = {"role": "assistant", "content": final_content}
|
|
messages.append(assistant_msg)
|
|
self._conversation.append(assistant_msg)
|
|
self.ui.write_log(f"Jarvis: {final_content}")
|
|
self.speak(final_content)
|
|
break
|
|
|
|
# ------- tool calls -------
|
|
assistant_msg = {
|
|
"role": "assistant",
|
|
"content": final_content or "",
|
|
"tool_calls": final_tool_calls,
|
|
}
|
|
messages.append(assistant_msg)
|
|
self._conversation.append(assistant_msg)
|
|
|
|
# ── Fast path: save_memory-only round with verbal content ──────
|
|
# When the LLM calls ONLY save_memory (silent) AND also returned
|
|
# verbal content in the same response, speak the content directly
|
|
# instead of burning another full LLM round just to get a reply.
|
|
_only_memory = all(
|
|
tc.get("function", {}).get("name") == "save_memory"
|
|
for tc in final_tool_calls
|
|
)
|
|
if _only_memory and final_content:
|
|
# Execute the memory saves silently
|
|
for tc in final_tool_calls:
|
|
fn = tc.get("function", {})
|
|
targs = fn.get("arguments", {})
|
|
if isinstance(targs, str):
|
|
try:
|
|
targs = json.loads(targs)
|
|
except Exception:
|
|
targs = {}
|
|
self._execute_tool("save_memory", targs)
|
|
# Use the verbal content that came alongside the saves
|
|
assistant_msg = {"role": "assistant", "content": final_content}
|
|
messages.append(assistant_msg)
|
|
self._conversation.append(assistant_msg)
|
|
self.ui.write_log(f"Jarvis: {final_content}")
|
|
self.speak(final_content)
|
|
break
|
|
|
|
all_silent = True
|
|
for tc in final_tool_calls:
|
|
fn = tc.get("function", {})
|
|
tname = fn.get("name", "")
|
|
targs = fn.get("arguments", {})
|
|
if isinstance(targs, str):
|
|
try:
|
|
targs = json.loads(targs)
|
|
except Exception:
|
|
targs = {}
|
|
|
|
tc_id = tc.get("id", "")
|
|
self.ui.write_log(f"SYS: ▶ {tname}")
|
|
result = self._execute_tool(tname, targs)
|
|
|
|
if result != "__SILENT__":
|
|
all_silent = False
|
|
|
|
tool_msg: dict = {
|
|
"role": "tool",
|
|
"content": "Done." if result == "__SILENT__" else str(result),
|
|
}
|
|
if tc_id:
|
|
tool_msg["tool_call_id"] = tc_id
|
|
|
|
messages.append(tool_msg)
|
|
self._conversation.append(tool_msg)
|
|
|
|
# ── Fast-ack for save_memory-only rounds ─────────────────────────
|
|
# When EVERY tool call was a silent save_memory, skip the extra LLM
|
|
# round entirely. A brief instant acknowledgment feels better than
|
|
# waiting 2-3 s for an unnecessary follow-up inference.
|
|
if all_silent:
|
|
# Extract the saved name (if any) for a personalised ack.
|
|
_saved_name: str | None = None
|
|
for _tc in final_tool_calls:
|
|
_fn = _tc.get("function", {})
|
|
if _fn.get("name") == "save_memory":
|
|
_a = _fn.get("arguments", {})
|
|
if isinstance(_a, str):
|
|
try:
|
|
_a = json.loads(_a)
|
|
except Exception:
|
|
_a = {}
|
|
if isinstance(_a, dict) and _a.get("key") == "name" and _a.get("value"):
|
|
_saved_name = str(_a["value"])
|
|
break
|
|
_ack = f"Got it, {_saved_name}." if _saved_name else "Noted."
|
|
_amsg = {"role": "assistant", "content": _ack}
|
|
messages.append(_amsg)
|
|
self._conversation.append(_amsg)
|
|
self.ui.write_log(f"Jarvis: {_ack}")
|
|
self.speak(_ack)
|
|
break
|
|
|
|
if not self.ui.muted:
|
|
self.ui.set_state("LISTENING")
|
|
|
|
# ------------------------------------------------------------------
|
|
# STT listening loops
|
|
# ------------------------------------------------------------------
|
|
|
|
def _listen_whisper(self) -> None:
|
|
"""Mic → VAD → Whisper → LLM loop."""
|
|
vad = _VADBuffer()
|
|
q: queue.Queue = queue.Queue(maxsize=200)
|
|
|
|
def callback(indata, frames, time_info, status):
|
|
with self._speaking_lock:
|
|
is_speaking = self._speaking
|
|
if not is_speaking and not self.ui.muted:
|
|
try:
|
|
q.put_nowait(indata.copy())
|
|
except queue.Full:
|
|
pass
|
|
|
|
try:
|
|
with sd.InputStream(
|
|
samplerate=SAMPLE_RATE_IN,
|
|
channels=CHANNELS,
|
|
dtype="float32",
|
|
blocksize=BLOCK_SIZE,
|
|
callback=callback,
|
|
):
|
|
self.ui.write_log("SYS: Mic active (Whisper STT).")
|
|
while True:
|
|
try:
|
|
chunk = q.get(timeout=0.1)
|
|
audio = vad.process(chunk.flatten())
|
|
if audio is not None:
|
|
self.ui.set_state("THINKING")
|
|
text = self._stt.transcribe(audio)
|
|
if text.strip():
|
|
self._process_message(text)
|
|
except queue.Empty:
|
|
pass
|
|
except Exception as e:
|
|
print(f"[STT-Whisper] Mic error: {e}")
|
|
traceback.print_exc()
|
|
|
|
def _listen_vosk(self) -> None:
|
|
"""Mic → Vosk streaming → LLM loop."""
|
|
q: queue.Queue = queue.Queue(maxsize=200)
|
|
|
|
def callback(indata, frames, time_info, status):
|
|
with self._speaking_lock:
|
|
is_speaking = self._speaking
|
|
if not is_speaking and not self.ui.muted:
|
|
try:
|
|
q.put_nowait(indata.copy())
|
|
except queue.Full:
|
|
pass
|
|
|
|
try:
|
|
with sd.InputStream(
|
|
samplerate=SAMPLE_RATE_IN,
|
|
channels=CHANNELS,
|
|
dtype="int16",
|
|
blocksize=4096,
|
|
callback=callback,
|
|
):
|
|
self.ui.write_log("SYS: Mic active (Vosk STT).")
|
|
while True:
|
|
try:
|
|
chunk = q.get(timeout=0.1)
|
|
text, is_final = self._stt.process_chunk(chunk.tobytes())
|
|
if is_final and text.strip():
|
|
self._process_message(text)
|
|
except queue.Empty:
|
|
pass
|
|
except Exception as e:
|
|
print(f"[STT-Vosk] Mic error: {e}")
|
|
traceback.print_exc()
|
|
|
|
# ------------------------------------------------------------------
|
|
# Text command loop (UI input box)
|
|
# ------------------------------------------------------------------
|
|
|
|
def _text_command_loop(self) -> None:
|
|
while True:
|
|
try:
|
|
text = self._text_queue.get(timeout=0.5)
|
|
if text.strip():
|
|
self._process_message(text)
|
|
except queue.Empty:
|
|
pass
|
|
|
|
# ------------------------------------------------------------------
|
|
# Entry point
|
|
# ------------------------------------------------------------------
|
|
|
|
def run(self) -> None:
|
|
"""
|
|
Startup strategy — optimised for minimum time-to-interactive:
|
|
|
|
1. LLM warmup + STT load → parallel, fast (~3s)
|
|
2. TTS load → parallel, slow (~20s for Kokoro)
|
|
3. Wait only for (1) → go online immediately
|
|
4. TTS finishes in BG → queued speech plays automatically
|
|
"""
|
|
try:
|
|
self.ui.on_reconfigure = self.reconfigure
|
|
|
|
# ── Ollama ────────────────────────────────────────────────────
|
|
from core.llm_client import ensure_ollama_running, warmup_model
|
|
self.ui.write_log("SYS: Checking Ollama…")
|
|
if ensure_ollama_running():
|
|
self.ui.write_log("SYS: Ollama OK.")
|
|
else:
|
|
self.ui.write_log("ERR: Ollama unavailable — run: ollama serve")
|
|
|
|
# ── Config ────────────────────────────────────────────────────
|
|
stt_engine = self._config.get("stt_engine", "whisper").lower()
|
|
stt_language = self._config.get("stt_language", "auto")
|
|
stt_model = self._config.get("stt_model", "base")
|
|
tts_engine = self._config.get("tts_engine", "edgetts").lower()
|
|
|
|
# ── Startup progress panel ────────────────────────────────────
|
|
self.ui.show_startup_panel()
|
|
|
|
_warmup_done = threading.Event()
|
|
_stt_done = threading.Event()
|
|
|
|
# ── LLM warmup thread ─────────────────────────────────────────
|
|
def _do_warmup():
|
|
try:
|
|
# Pass the STATIC system prompt so Ollama evaluates and caches
|
|
# its KV state during startup. Real requests start with the same
|
|
# static prefix → Ollama reuses cached KV → first token <1 s
|
|
# instead of the ~17 s it takes to re-evaluate 300+ tokens cold.
|
|
static_prompt = _load_system_prompt()
|
|
warmup_model(system_prompt=static_prompt)
|
|
self.ui.write_log("SYS: LLM ready.")
|
|
self.ui.mark_startup_ready("llm")
|
|
except Exception as e:
|
|
self.ui.write_log(f"ERR: LLM warmup — {e}")
|
|
self.ui.mark_startup_ready("llm", error=True)
|
|
finally:
|
|
_warmup_done.set()
|
|
|
|
# ── STT load thread ───────────────────────────────────────────
|
|
def _do_stt():
|
|
try:
|
|
self.ui.write_log(f"SYS: Loading {stt_engine.upper()} STT…")
|
|
if stt_engine == "vosk":
|
|
from core.stt import VoskSTT
|
|
self._stt = VoskSTT(
|
|
self._config.get("vosk_model_path"),
|
|
language=stt_language,
|
|
)
|
|
else:
|
|
from core.stt import WhisperSTT
|
|
self._stt = WhisperSTT(stt_model, language=stt_language)
|
|
self.ui.write_log("SYS: STT ready.")
|
|
self.ui.mark_startup_ready("stt")
|
|
except Exception as e:
|
|
self.ui.write_log(f"ERR: STT — {e}")
|
|
self.ui.mark_startup_ready("stt", error=True)
|
|
finally:
|
|
_stt_done.set()
|
|
|
|
# ── TTS load thread — does NOT block going online ─────────────
|
|
def _do_tts():
|
|
try:
|
|
self.ui.write_log(f"SYS: Loading {tts_engine.upper()} TTS…")
|
|
if tts_engine == "kokoro":
|
|
self.ui.write_log("SYS: Kokoro — loading model + compiling JIT…")
|
|
from core.tts import create_tts_player
|
|
self._tts = create_tts_player(self._config)
|
|
self._tts_ready.set() # unblock _tts_worker
|
|
self.ui.write_log("SYS: TTS ready.")
|
|
self.ui.mark_startup_ready("tts")
|
|
self.ui.set_startup_status("● All systems ready.")
|
|
self.ui.hide_startup_panel()
|
|
self.speak("Jarvis fully online.")
|
|
except Exception as e:
|
|
import traceback as _tb; _tb.print_exc()
|
|
self.ui.write_log(f"ERR: TTS — {e}")
|
|
self.ui.mark_startup_ready("tts", error=True)
|
|
self._tts_ready.set()
|
|
|
|
# Launch all three simultaneously
|
|
self.ui.write_log("SYS: Loading systems in parallel…")
|
|
threading.Thread(target=_do_warmup, daemon=True).start()
|
|
threading.Thread(target=_do_stt, daemon=True).start()
|
|
threading.Thread(target=_do_tts, daemon=True).start()
|
|
|
|
# ── Wait ONLY for STT + LLM (fast) ────────────────────────────
|
|
_warmup_done.wait(timeout=60)
|
|
_stt_done.wait(timeout=60)
|
|
|
|
# ── Go online immediately ──────────────────────────────────────
|
|
self.ui.write_log("SYS: JARVIS online.")
|
|
self.ui.set_state("LISTENING")
|
|
self.ui.set_startup_status("● JARVIS online · Voice loading in background…")
|
|
|
|
threading.Thread(target=self._tts_worker, daemon=True).start()
|
|
threading.Thread(target=self._text_command_loop, daemon=True).start()
|
|
|
|
# STT loop — blocks this thread forever
|
|
if stt_engine == "vosk":
|
|
self._listen_vosk()
|
|
else:
|
|
self._listen_whisper()
|
|
|
|
except Exception as e:
|
|
self.ui.write_log(f"ERR: Init failed — {e}")
|
|
traceback.print_exc()
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Entry
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def main() -> None:
|
|
# ── Pre-import torch in background immediately ─────────────────────────
|
|
# By the time the TTS thread starts (~5s from now), torch will already
|
|
# be in sys.modules — removing it from the TTS critical path entirely.
|
|
def _preload_torch():
|
|
try:
|
|
import torch # noqa: F401 (side-effect import only)
|
|
except Exception:
|
|
pass
|
|
threading.Thread(target=_preload_torch, daemon=True).start()
|
|
# ───────────────────────────────────────────────────────────────────────
|
|
|
|
ui = JarvisUI("face.png")
|
|
|
|
def runner():
|
|
# 1. Wait until the user completes the setup overlay (first run)
|
|
# or config already exists (subsequent runs).
|
|
ui.wait_for_api_key()
|
|
|
|
# 2. Install any missing engine packages before loading engines.
|
|
# Progress is streamed to the log panel in real time.
|
|
ui.write_log("SYS: Checking dependencies…")
|
|
cfg = _load_config()
|
|
_install_done = threading.Event()
|
|
|
|
def _do_install():
|
|
try:
|
|
from core.installer import install_for_config
|
|
install_for_config(cfg, log=ui.write_log)
|
|
except Exception as e:
|
|
ui.write_log(f"ERR: Dependency install — {e}")
|
|
finally:
|
|
_install_done.set()
|
|
|
|
threading.Thread(target=_do_install, daemon=True).start()
|
|
_install_done.wait() # blocks runner thread; UI remains responsive
|
|
|
|
# 3. Start the assistant (loads STT / TTS / LLM).
|
|
jarvis = JarvisLocal(ui)
|
|
try:
|
|
jarvis.run()
|
|
except KeyboardInterrupt:
|
|
print("\n[MARK XL] Shutting down…")
|
|
|
|
threading.Thread(target=runner, daemon=True).start()
|
|
ui.root.mainloop()
|
|
|
|
if __name__ == "__main__":
|
|
main()
|