Files
Mark-LIV/main.py
T
2026-06-06 17:41:38 +03:00

1192 lines
49 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""
MARK XL — Local LLM Edition
STT (Whisper / Vosk) + Ollama LLM + TTS (EdgeTTS / Kokoro / ElevenLabs)
All Gemini / Google-AI dependencies removed.
"""
# ── Bootstrap: auto-install base UI packages before anything else ──────────
# Uses only stdlib so it works even on a completely fresh Python install.
import importlib.util as _ilu
import os as _os
import subprocess as _sp
import sys as _sys
_BASE_PKGS = [
("PyQt6", "PyQt6"),
("psutil", "psutil"),
("numpy", "numpy"),
("sounddevice", "sounddevice"),
("PIL", "pillow"),
("requests", "requests"),
]
def _bootstrap() -> None:
need = [pkg for mod, pkg in _BASE_PKGS if _ilu.find_spec(mod) is None]
if not need:
return
print(f"\n[MARK XL] First-run setup — installing: {', '.join(need)}")
print("[MARK XL] This happens only once.\n")
_sp.run([_sys.executable, "-m", "pip", "install", *need], check=True)
print("\n[MARK XL] Base packages ready — restarting…\n")
# Replace current process with a fresh one (picks up newly installed packages)
_os.execv(_sys.executable, [_sys.executable] + _sys.argv)
_bootstrap()
# ───────────────────────────────────────────────────────────────────────────
import json
import queue
import re
import sys
import threading
import traceback
from datetime import datetime
from pathlib import Path
import numpy as np
import sounddevice as sd
from ui import JarvisUI
from memory.memory_manager import load_memory, update_memory, format_memory_for_prompt
from core.llm_client import call_llm, call_llm_stream, get_llm_settings
from actions.file_processor import file_processor
from actions.flight_finder import flight_finder
from actions.open_app import open_app
from actions.weather_report import weather_action
from actions.send_message import send_message
from actions.reminder import reminder
from actions.computer_settings import computer_settings
from actions.screen_processor import screen_process
from actions.youtube_video import youtube_video
from actions.desktop import desktop_control
from actions.browser_control import browser_control
from actions.file_controller import file_controller
from actions.code_helper import code_helper
from actions.dev_agent import dev_agent
from actions.web_search import web_search as web_search_action
from actions.computer_control import computer_control
from actions.game_updater import game_updater
# ---------------------------------------------------------------------------
# Paths
# ---------------------------------------------------------------------------
def _get_base_dir() -> Path:
if getattr(sys, "frozen", False):
return Path(sys.executable).parent
return Path(__file__).resolve().parent
BASE_DIR = _get_base_dir()
API_CONFIG_PATH = BASE_DIR / "config" / "api_keys.json"
PROMPT_PATH = BASE_DIR / "core" / "prompt.txt"
SAMPLE_RATE_IN = 16_000
BLOCK_SIZE = 1_024
CHANNELS = 1
# ---------------------------------------------------------------------------
# Tool declarations (Gemini format kept for readability;
# converted to OpenAI/Ollama format by _to_ollama_tools())
# ---------------------------------------------------------------------------
TOOL_DECLARATIONS = [
{
"name": "open_app",
"description": (
"Opens any application on the computer. "
"Use this whenever the user asks to open, launch, or start any app, "
"website, or program. Always call this tool — never just say you opened it."
),
"parameters": {
"type": "OBJECT",
"properties": {
"app_name": {"type": "STRING", "description": "Exact name of the application"}
},
"required": ["app_name"]
}
},
{
"name": "web_search",
"description": "Searches the web for any information.",
"parameters": {
"type": "OBJECT",
"properties": {
"query": {"type": "STRING", "description": "Search query"},
"mode": {"type": "STRING", "description": "search (default) or compare"},
"items": {"type": "ARRAY", "items": {"type": "STRING"}, "description": "Items to compare"},
"aspect": {"type": "STRING", "description": "price | specs | reviews"}
},
"required": ["query"]
}
},
{
"name": "weather_report",
"description": "Gives the weather report to user",
"parameters": {
"type": "OBJECT",
"properties": {"city": {"type": "STRING", "description": "City name"}},
"required": ["city"]
}
},
{
"name": "send_message",
"description": "Sends a text message via WhatsApp, Telegram, or other messaging platform.",
"parameters": {
"type": "OBJECT",
"properties": {
"receiver": {"type": "STRING", "description": "Recipient contact name"},
"message_text": {"type": "STRING", "description": "The message to send"},
"platform": {"type": "STRING", "description": "Platform: WhatsApp, Telegram, etc."}
},
"required": ["receiver", "message_text", "platform"]
}
},
{
"name": "reminder",
"description": "Sets a timed reminder using Task Scheduler.",
"parameters": {
"type": "OBJECT",
"properties": {
"date": {"type": "STRING", "description": "Date in YYYY-MM-DD format"},
"time": {"type": "STRING", "description": "Time in HH:MM format (24h)"},
"message": {"type": "STRING", "description": "Reminder message text"}
},
"required": ["date", "time", "message"]
}
},
{
"name": "youtube_video",
"description": (
"Controls YouTube. Use for: playing videos, summarizing a video's content, "
"getting video info, or showing trending videos."
),
"parameters": {
"type": "OBJECT",
"properties": {
"action": {"type": "STRING", "description": "play | summarize | get_info | trending"},
"query": {"type": "STRING", "description": "Search query for play action"},
"save": {"type": "BOOLEAN", "description": "Save summary to Notepad"},
"region": {"type": "STRING", "description": "Country code for trending e.g. TR, US"},
"url": {"type": "STRING", "description": "Video URL for get_info action"},
},
"required": []
}
},
{
"name": "screen_process",
"description": (
"Captures and analyzes the screen or webcam image. "
"MUST be called when user asks what is on screen, what you see, "
"analyze my screen, look at camera, etc. "
"You have NO visual ability without this tool."
),
"parameters": {
"type": "OBJECT",
"properties": {
"angle": {"type": "STRING", "description": "'screen' or 'camera'. Default: 'screen'"},
"text": {"type": "STRING", "description": "The question about the captured image"}
},
"required": ["text"]
}
},
{
"name": "computer_settings",
"description": (
"Controls the computer: volume, brightness, window management, keyboard shortcuts, "
"typing text on screen, closing apps, fullscreen, dark mode, WiFi, restart, shutdown, "
"scrolling, tab management, zoom, screenshots, lock screen, refresh/reload page."
),
"parameters": {
"type": "OBJECT",
"properties": {
"action": {"type": "STRING", "description": "The action to perform"},
"description": {"type": "STRING", "description": "Natural language description"},
"value": {"type": "STRING", "description": "Optional value"}
},
"required": []
}
},
{
"name": "browser_control",
"description": (
"Controls any web browser. Use for: opening websites, searching the web, "
"clicking elements, filling forms, scrolling, screenshots, navigation."
),
"parameters": {
"type": "OBJECT",
"properties": {
"action": {"type": "STRING", "description": "go_to | search | click | type | scroll | fill_form | smart_click | smart_type | get_text | get_url | press | new_tab | close_tab | screenshot | back | forward | reload | switch | list_browsers | close | close_all"},
"browser": {"type": "STRING", "description": "chrome | edge | firefox | opera | operagx | brave | vivaldi | safari"},
"url": {"type": "STRING", "description": "URL for go_to / new_tab action"},
"query": {"type": "STRING", "description": "Search query"},
"engine": {"type": "STRING", "description": "google | bing | duckduckgo | yandex"},
"selector": {"type": "STRING", "description": "CSS selector for click/type"},
"text": {"type": "STRING", "description": "Text to click or type"},
"description": {"type": "STRING", "description": "Element description for smart_click/smart_type"},
"direction": {"type": "STRING", "description": "up | down for scroll"},
"amount": {"type": "INTEGER", "description": "Scroll amount in pixels"},
"key": {"type": "STRING", "description": "Key name for press"},
"path": {"type": "STRING", "description": "Save path for screenshot"},
"incognito": {"type": "BOOLEAN", "description": "Open in private/incognito mode"},
"clear_first": {"type": "BOOLEAN", "description": "Clear field before typing"},
},
"required": ["action"]
}
},
{
"name": "file_controller",
"description": "Manages files and folders: list, create, delete, move, copy, rename, read, write, find, disk usage.",
"parameters": {
"type": "OBJECT",
"properties": {
"action": {"type": "STRING", "description": "list | create_file | create_folder | delete | move | copy | rename | read | write | find | largest | disk_usage | organize_desktop | info"},
"path": {"type": "STRING", "description": "File/folder path or shortcut: desktop, downloads, documents, home"},
"destination": {"type": "STRING", "description": "Destination path for move/copy"},
"new_name": {"type": "STRING", "description": "New name for rename"},
"content": {"type": "STRING", "description": "Content for create_file/write"},
"name": {"type": "STRING", "description": "File name to search for"},
"extension": {"type": "STRING", "description": "File extension to search"},
"count": {"type": "INTEGER", "description": "Number of results for largest"},
},
"required": ["action"]
}
},
{
"name": "desktop_control",
"description": "Controls the desktop: wallpaper, organize, clean, list, stats.",
"parameters": {
"type": "OBJECT",
"properties": {
"action": {"type": "STRING", "description": "wallpaper | wallpaper_url | organize | clean | list | stats | task"},
"path": {"type": "STRING", "description": "Image path for wallpaper"},
"url": {"type": "STRING", "description": "Image URL for wallpaper_url"},
"mode": {"type": "STRING", "description": "by_type or by_date for organize"},
"task": {"type": "STRING", "description": "Natural language desktop task"},
},
"required": ["action"]
}
},
{
"name": "code_helper",
"description": "Writes, edits, explains, runs, or builds code files.",
"parameters": {
"type": "OBJECT",
"properties": {
"action": {"type": "STRING", "description": "write | edit | explain | run | build | auto"},
"description": {"type": "STRING", "description": "What the code should do"},
"language": {"type": "STRING", "description": "Programming language"},
"output_path": {"type": "STRING", "description": "Where to save the file"},
"file_path": {"type": "STRING", "description": "Path to existing file"},
"code": {"type": "STRING", "description": "Raw code string for explain"},
"args": {"type": "STRING", "description": "CLI arguments"},
"timeout": {"type": "INTEGER", "description": "Execution timeout in seconds"},
},
"required": ["action"]
}
},
{
"name": "dev_agent",
"description": "Builds complete multi-file projects from scratch.",
"parameters": {
"type": "OBJECT",
"properties": {
"description": {"type": "STRING", "description": "What the project should do"},
"language": {"type": "STRING", "description": "Programming language"},
"project_name": {"type": "STRING", "description": "Optional project folder name"},
"timeout": {"type": "INTEGER", "description": "Run timeout in seconds"},
},
"required": ["description"]
}
},
{
"name": "agent_task",
"description": (
"Executes complex multi-step tasks requiring multiple different tools. "
"Examples: 'research X and save to file', 'find and organize files'. "
"DO NOT use for single commands."
),
"parameters": {
"type": "OBJECT",
"properties": {
"goal": {"type": "STRING", "description": "Complete description of what to accomplish"},
"priority": {"type": "STRING", "description": "low | normal | high"}
},
"required": ["goal"]
}
},
{
"name": "computer_control",
"description": "Direct computer control: type, click, hotkeys, scroll, move mouse, screenshots, find elements on screen.",
"parameters": {
"type": "OBJECT",
"properties": {
"action": {"type": "STRING", "description": "type | smart_type | click | double_click | right_click | hotkey | press | scroll | move | copy | paste | screenshot | wait | clear_field | focus_window | screen_find | screen_click | random_data | user_data"},
"text": {"type": "STRING", "description": "Text to type or paste"},
"x": {"type": "INTEGER", "description": "X coordinate"},
"y": {"type": "INTEGER", "description": "Y coordinate"},
"keys": {"type": "STRING", "description": "Key combination e.g. 'ctrl+c'"},
"key": {"type": "STRING", "description": "Single key e.g. 'enter'"},
"direction": {"type": "STRING", "description": "up | down | left | right"},
"amount": {"type": "INTEGER", "description": "Scroll amount"},
"seconds": {"type": "NUMBER", "description": "Seconds to wait"},
"title": {"type": "STRING", "description": "Window title for focus_window"},
"description": {"type": "STRING", "description": "Element description"},
"type": {"type": "STRING", "description": "Data type for random_data"},
"field": {"type": "STRING", "description": "Field for user_data"},
"clear_first": {"type": "BOOLEAN", "description": "Clear field before typing"},
"path": {"type": "STRING", "description": "Save path for screenshot"},
},
"required": ["action"]
}
},
{
"name": "game_updater",
"description": (
"THE ONLY tool for ANY Steam or Epic Games request. "
"Use for: installing, downloading, updating games, listing installed games."
),
"parameters": {
"type": "OBJECT",
"properties": {
"action": {"type": "STRING", "description": "update | install | list | download_status | schedule | cancel_schedule | schedule_status"},
"platform": {"type": "STRING", "description": "steam | epic | both"},
"game_name": {"type": "STRING", "description": "Game name"},
"app_id": {"type": "STRING", "description": "Steam AppID"},
"hour": {"type": "INTEGER", "description": "Hour for scheduled update 0-23"},
"minute": {"type": "INTEGER", "description": "Minute for scheduled update 0-59"},
"shutdown_when_done": {"type": "BOOLEAN", "description": "Shut down PC when done"},
},
"required": []
}
},
{
"name": "flight_finder",
"description": "Searches Google Flights and speaks the best options.",
"parameters": {
"type": "OBJECT",
"properties": {
"origin": {"type": "STRING", "description": "Departure city or airport code"},
"destination": {"type": "STRING", "description": "Arrival city or airport code"},
"date": {"type": "STRING", "description": "Departure date"},
"return_date": {"type": "STRING", "description": "Return date for round trips"},
"passengers": {"type": "INTEGER", "description": "Number of passengers"},
"cabin": {"type": "STRING", "description": "economy | premium | business | first"},
"save": {"type": "BOOLEAN", "description": "Save results to Notepad"},
},
"required": ["origin", "destination", "date"]
}
},
{
"name": "shutdown_jarvis",
"description": (
"Shuts down the assistant completely. "
"Call this when the user expresses intent to end the conversation, "
"close the assistant, say goodbye, or stop Jarvis."
),
"parameters": {"type": "OBJECT", "properties": {}}
},
{
"name": "file_processor",
"description": (
"Processes any file that the user has uploaded or dropped onto the interface. "
"Supports: images, PDFs, Word docs, CSV/Excel, JSON, code files, audio, video, archives."
),
"parameters": {
"type": "OBJECT",
"properties": {
"file_path": {"type": "STRING", "description": "Full path to the uploaded file"},
"action": {"type": "STRING", "description": "What to do with the file"},
"instruction": {"type": "STRING", "description": "Free-form instruction"},
"format": {"type": "STRING", "description": "Target format for conversion"},
"width": {"type": "INTEGER", "description": "Target width for image resize"},
"height": {"type": "INTEGER", "description": "Target height for image resize"},
"scale": {"type": "NUMBER", "description": "Scale factor"},
"quality": {"type": "INTEGER", "description": "Quality 1-100"},
"start": {"type": "STRING", "description": "Start time for trim"},
"end": {"type": "STRING", "description": "End time for trim"},
"timestamp": {"type": "STRING", "description": "Timestamp for video frame extraction"},
"column": {"type": "STRING", "description": "Column name for CSV filter/sort"},
"value": {"type": "STRING", "description": "Filter value"},
"condition": {"type": "STRING", "description": "Filter condition"},
"ascending": {"type": "BOOLEAN", "description": "Sort order"},
"save": {"type": "BOOLEAN", "description": "Save result to file"},
"destination": {"type": "STRING", "description": "Output folder for archive extract"},
},
"required": []
}
},
{
"name": "save_memory",
"description": (
"Save an important personal fact about the user to long-term memory. "
"Call this silently whenever the user reveals something worth remembering. "
"Do NOT announce that you are saving — just call it silently."
),
"parameters": {
"type": "OBJECT",
"properties": {
"category": {
"type": "STRING",
"description": (
"identity | preferences | projects | relationships | wishes | notes"
)
},
"key": {"type": "STRING", "description": "Short snake_case key"},
"value": {"type": "STRING", "description": "Concise value in English"},
},
"required": ["category", "key", "value"]
}
},
]
# ---------------------------------------------------------------------------
# Convert Gemini-style declarations to OpenAI/Ollama format
# ---------------------------------------------------------------------------
_TYPE_MAP = {
"OBJECT": "object", "STRING": "string", "ARRAY": "array",
"INTEGER": "integer", "BOOLEAN": "boolean", "NUMBER": "number",
}
def _convert_type(t: str) -> str:
return _TYPE_MAP.get(t, t.lower()) if isinstance(t, str) else t
def _convert_props(props: dict) -> dict:
out = {}
for k, v in props.items():
nv = dict(v)
if "type" in nv:
nv["type"] = _convert_type(nv["type"])
if "items" in nv and isinstance(nv["items"], dict):
nv["items"] = {"type": _convert_type(nv["items"].get("type", "string"))}
out[k] = nv
return out
def _to_ollama_tools(decls: list) -> list:
tools = []
for d in decls:
params = d.get("parameters", {})
new_params: dict = {
"type": "object",
"properties": _convert_props(params.get("properties", {})),
}
req = params.get("required")
if req:
new_params["required"] = req
tools.append({
"type": "function",
"function": {
"name": d["name"],
"description": d["description"],
"parameters": new_params,
},
})
return tools
OLLAMA_TOOLS = _to_ollama_tools(TOOL_DECLARATIONS)
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def _load_config() -> dict:
try:
with open(API_CONFIG_PATH, encoding="utf-8") as f:
return json.load(f)
except Exception:
return {}
def _load_system_prompt() -> str:
try:
return PROMPT_PATH.read_text(encoding="utf-8")
except Exception:
return (
"You are JARVIS, Tony Stark's AI assistant. "
"Be concise, direct, and always use the provided tools to complete tasks. "
"Never simulate or guess results — always call the appropriate tool."
)
# ---------------------------------------------------------------------------
# Voice Activity Detection (used for Whisper listen loop)
# ---------------------------------------------------------------------------
class _VADBuffer:
"""Energy-based VAD: buffers audio until end of utterance."""
def __init__(
self,
sample_rate: int = 16_000,
silence_sec: float = 1.2,
threshold: float = 0.015,
min_speech_sec: float = 0.4,
max_speech_sec: float = 30.0,
):
self._sr = sample_rate
self._sil_n = int(silence_sec * sample_rate)
self._thresh = threshold
self._min_n = int(min_speech_sec * sample_rate)
self._max_n = int(max_speech_sec * sample_rate)
self._buf: list[np.ndarray] = []
self._in_spch = False
self._sil_cnt = 0
def process(self, chunk: np.ndarray) -> np.ndarray | None:
"""
Feed one audio chunk (float32 mono).
Returns complete utterance when speech ends, otherwise None.
"""
rms = float(np.sqrt(np.mean(chunk ** 2)))
total_n = sum(len(c) for c in self._buf)
if rms > self._thresh:
self._in_spch = True
self._sil_cnt = 0
self._buf.append(chunk.copy())
elif self._in_spch:
self._buf.append(chunk.copy())
self._sil_cnt += len(chunk)
if self._sil_cnt >= self._sil_n or total_n >= self._max_n:
audio = np.concatenate(self._buf)
self._buf = []
self._in_spch = False
self._sil_cnt = 0
if len(audio) >= self._min_n:
return audio
return None
# ---------------------------------------------------------------------------
# JarvisLocal
# ---------------------------------------------------------------------------
class JarvisLocal:
"""
Main assistant class.
Replaces JarvisLive (Gemini Live API) with:
STT (Whisper/Vosk) → Ollama LLM (tool calling) → TTS (Edge/Kokoro/ElevenLabs)
"""
def __init__(self, ui: JarvisUI):
self.ui = ui
self._config = _load_config()
self._stt = None
self._tts = None
self._speaking = False
self._speaking_lock = threading.Lock()
self._text_queue: queue.Queue = queue.Queue()
self._tts_queue: queue.Queue = queue.Queue()
self._conversation: list[dict] = []
self.ui.on_text_command = self._on_text_command
# ------------------------------------------------------------------
# System prompt
# ------------------------------------------------------------------
def _build_system_prompt(self) -> str:
memory = load_memory()
mem_str = format_memory_for_prompt(memory)
sys_p = _load_system_prompt()
now = datetime.now()
time_ctx = (
f"[CURRENT DATE & TIME]\n"
f"Right now it is: {now.strftime('%A, %B %d, %Y — %I:%M %p')}\n"
f"Use this to calculate exact times for reminders.\n\n"
)
parts = [time_ctx]
if mem_str:
parts.append(mem_str)
parts.append(sys_p)
return "\n".join(parts)
# ------------------------------------------------------------------
# Speaking state & TTS
# ------------------------------------------------------------------
# ------------------------------------------------------------------
# TTS queue worker — plays sentences sequentially, no overlaps
# ------------------------------------------------------------------
def _tts_worker(self) -> None:
while True:
text = self._tts_queue.get()
n_got = 1 # how many queue items we consumed this round
try:
if text and self._tts:
# ── Kokoro: merge buffered sentences into one inference call ──
# Each separate Kokoro call costs ~0.5–1 s startup.
# Draining the queue here combines "Hello. How are you?" into
# one call instead of two, eliminating the gap between them.
from core.tts import KokoroTTSEngine
engine = getattr(self._tts, "_engine", None)
if isinstance(engine, KokoroTTSEngine):
while True:
try:
nxt = self._tts_queue.get_nowait()
if nxt:
text = text.rstrip() + " " + nxt.lstrip()
n_got += 1
except queue.Empty:
break
with self._speaking_lock:
self._speaking = True
self.ui.set_state("SPEAKING")
self._tts.speak(text)
except Exception as e:
print(f"[TTS] speak error: {e}")
finally:
for _ in range(n_got):
self._tts_queue.task_done()
if self._tts_queue.empty():
with self._speaking_lock:
self._speaking = False
if not self.ui.muted:
self.ui.set_state("LISTENING")
def set_speaking(self, value: bool) -> None:
with self._speaking_lock:
self._speaking = value
if value:
self.ui.set_state("SPEAKING")
elif not self.ui.muted:
self.ui.set_state("LISTENING")
def speak(self, text: str) -> None:
if not text or not self._tts:
return
with self._speaking_lock:
self._speaking = True
self._tts_queue.put(text)
def speak_error(self, tool_name: str, error) -> None:
short = str(error)[:120]
self.ui.write_log(f"ERR: {tool_name} — {short}")
self.speak(f"{tool_name} encountered an error.")
# ------------------------------------------------------------------
# Live reconfigure (called when user clicks Apply in Configure panel)
# ------------------------------------------------------------------
def reconfigure(self, new_config: dict) -> None:
"""Non-blocking: spawns a background thread to install + reload."""
threading.Thread(
target=self._do_reconfigure, args=(new_config,), daemon=True
).start()
def _do_reconfigure(self, new_config: dict) -> None:
old_stt_engine = self._config.get("stt_engine", "whisper").lower()
old_llm_model = self._config.get("llm_model", "")
new_stt_engine = new_config.get("stt_engine", "whisper").lower()
self._config = new_config
# Install any packages required by the new config
try:
from core.installer import install_for_config
install_for_config(new_config, log=self.ui.write_log)
except Exception as e:
self.ui.write_log(f"ERR: Dependency install — {e}")
# TTS: always hot-reload (runs in queue worker, safe to swap)
try:
from core.tts import create_tts_player
self._tts = create_tts_player(new_config)
self.ui.write_log("SYS: TTS reconfigured.")
except Exception as e:
self.ui.write_log(f"ERR: TTS reconfigure — {e}")
# STT: hot-reload if same engine type; full restart needed if engine changed
if old_stt_engine == new_stt_engine:
try:
stt_language = new_config.get("stt_language", "auto")
if new_stt_engine == "vosk":
from core.stt import VoskSTT
self._stt = VoskSTT(new_config.get("vosk_model_path"), language=stt_language)
else:
from core.stt import WhisperSTT
self._stt = WhisperSTT(new_config.get("stt_model", "base"), language=stt_language)
self.ui.write_log("SYS: STT reconfigured.")
except Exception as e:
self.ui.write_log(f"ERR: STT reconfigure — {e}")
else:
self.ui.write_log("SYS: STT engine changed — restart required.")
# LLM warmup if model changed
if new_config.get("llm_model", "") != old_llm_model:
self.ui.write_log("SYS: Warming up new LLM model…")
from core.llm_client import warmup_model
warmup_model()
self.ui.write_log("SYS: New LLM model ready.")
if old_stt_engine == new_stt_engine:
self.speak("Configuration applied.")
else:
self.speak("LLM and TTS updated. Restart for speech engine change.")
# ------------------------------------------------------------------
# Text command (from UI input box)
# ------------------------------------------------------------------
def _on_text_command(self, text: str) -> None:
self._text_queue.put(text)
# ------------------------------------------------------------------
# Tool execution (routing unchanged from original)
# ------------------------------------------------------------------
def _execute_tool(self, name: str, args: dict) -> str:
print(f"[JARVIS] 🔧 {name} {args}")
self.ui.set_state("THINKING")
# save_memory is handled silently
if name == "save_memory":
category = args.get("category", "notes")
key = args.get("key", "")
value = args.get("value", "")
if key and value:
update_memory({category: {key: {"value": value}}})
print(f"[Memory] 💾 {category}/{key} = {value}")
if not self.ui.muted:
self.ui.set_state("LISTENING")
return "__SILENT__"
result = "Done."
try:
if name == "open_app":
r = open_app(parameters=args, response=None, player=self.ui)
result = r or f"Opened {args.get('app_name')}."
elif name == "weather_report":
r = weather_action(parameters=args, player=self.ui)
result = r or "Weather delivered."
elif name == "browser_control":
r = browser_control(parameters=args, player=self.ui)
result = r or "Done."
elif name == "file_controller":
r = file_controller(parameters=args, player=self.ui)
result = r or "Done."
elif name == "send_message":
r = send_message(parameters=args, response=None, player=self.ui, session_memory=None)
result = r or f"Message sent to {args.get('receiver')}."
elif name == "reminder":
r = reminder(parameters=args, response=None, player=self.ui)
result = r or "Reminder set."
elif name == "youtube_video":
r = youtube_video(parameters=args, response=None, player=self.ui)
result = r or "Done."
elif name == "screen_process":
# Synchronous call — returns analysis text which the LLM can speak
r = screen_process(parameters=args, response=None, player=self.ui, session_memory=None)
result = r if isinstance(r, str) and r else "Screen analyzed."
elif name == "computer_settings":
r = computer_settings(parameters=args, response=None, player=self.ui)
result = r or "Done."
elif name == "desktop_control":
r = desktop_control(parameters=args, player=self.ui)
result = r or "Done."
elif name == "code_helper":
r = code_helper(parameters=args, player=self.ui, speak=self.speak)
result = r or "Done."
elif name == "dev_agent":
r = dev_agent(parameters=args, player=self.ui, speak=self.speak)
result = r or "Done."
elif name == "agent_task":
from agent.task_queue import get_queue, TaskPriority
priority_map = {
"low": TaskPriority.LOW,
"normal": TaskPriority.NORMAL,
"high": TaskPriority.HIGH,
}
priority = priority_map.get(
args.get("priority", "normal").lower(), TaskPriority.NORMAL
)
task_id = get_queue().submit(
goal=args.get("goal", ""), priority=priority, speak=self.speak
)
result = f"Task started (ID: {task_id})."
elif name == "web_search":
r = web_search_action(parameters=args, player=self.ui)
result = r or "Done."
elif name == "file_processor":
if not args.get("file_path") and self.ui.current_file:
args["file_path"] = self.ui.current_file
r = file_processor(parameters=args, player=self.ui, speak=self.speak)
result = r or "Done."
elif name == "computer_control":
r = computer_control(parameters=args, player=self.ui)
result = r or "Done."
elif name == "game_updater":
r = game_updater(parameters=args, player=self.ui, speak=self.speak)
result = r or "Done."
elif name == "flight_finder":
r = flight_finder(parameters=args, player=self.ui)
result = r or "Done."
elif name == "shutdown_jarvis":
self.ui.write_log("SYS: Shutdown requested.")
def _shutdown():
import time, os
self.speak("Goodbye.")
time.sleep(2.5)
os._exit(0)
threading.Thread(target=_shutdown, daemon=True).start()
return "Shutting down."
else:
result = f"Unknown tool: {name}"
except Exception as e:
result = f"Tool '{name}' failed: {e}"
traceback.print_exc()
self.speak_error(name, e)
if not self.ui.muted:
self.ui.set_state("LISTENING")
print(f"[JARVIS] 📤 {name} → {str(result)[:80]}")
return result
# ------------------------------------------------------------------
# LLM processing loop
# ------------------------------------------------------------------
def _process_message(self, user_text: str) -> None:
"""
Full turn:
user_text → Ollama stream (with tools) → execute tools → final spoken response
Streaming means TTS starts on the first complete sentence (~1-2 s),
not after the full response is ready.
"""
self.ui.set_state("THINKING")
self.ui.write_log(f"You: {user_text}")
self._conversation.append({"role": "user", "content": user_text})
MAX_HISTORY = 30
if len(self._conversation) > MAX_HISTORY:
self._conversation = self._conversation[-MAX_HISTORY:]
messages = [
{"role": "system", "content": self._build_system_prompt()}
] + list(self._conversation)
MAX_TOOL_ROUNDS = 6
for _round in range(MAX_TOOL_ROUNDS):
final_content = ""
final_tool_calls: list = []
try:
for event in call_llm_stream(messages, OLLAMA_TOOLS):
if event["type"] == "sentence":
self.speak(event["text"])
elif event["type"] == "done":
final_content = event["content"]
final_tool_calls = event["tool_calls"]
except RuntimeError as e:
self.speak_error("LLM", e)
return
if not final_tool_calls:
# Final text response — TTS already started sentence-by-sentence
if final_content:
assistant_msg = {"role": "assistant", "content": final_content}
messages.append(assistant_msg)
self._conversation.append(assistant_msg)
self.ui.write_log(f"Jarvis: {final_content}")
break
# ------- tool calls -------
assistant_msg = {
"role": "assistant",
"content": final_content or "",
"tool_calls": final_tool_calls,
}
messages.append(assistant_msg)
self._conversation.append(assistant_msg)
all_silent = True
for tc in final_tool_calls:
fn = tc.get("function", {})
tname = fn.get("name", "")
targs = fn.get("arguments", {})
if isinstance(targs, str):
try:
targs = json.loads(targs)
except Exception:
targs = {}
tc_id = tc.get("id", "")
self.ui.write_log(f"SYS: ▶ {tname}")
result = self._execute_tool(tname, targs)
if result != "__SILENT__":
all_silent = False
tool_msg: dict = {
"role": "tool",
"content": "Done." if result == "__SILENT__" else str(result),
}
if tc_id:
tool_msg["tool_call_id"] = tc_id
messages.append(tool_msg)
self._conversation.append(tool_msg)
# Even if all tools were silent (e.g. save_memory only),
# always do one more LLM round so the assistant can verbally respond.
if not self.ui.muted:
self.ui.set_state("LISTENING")
# ------------------------------------------------------------------
# STT listening loops
# ------------------------------------------------------------------
def _listen_whisper(self) -> None:
"""Mic → VAD → Whisper → LLM loop."""
vad = _VADBuffer()
q: queue.Queue = queue.Queue(maxsize=200)
def callback(indata, frames, time_info, status):
with self._speaking_lock:
is_speaking = self._speaking
if not is_speaking and not self.ui.muted:
try:
q.put_nowait(indata.copy())
except queue.Full:
pass
try:
with sd.InputStream(
samplerate=SAMPLE_RATE_IN,
channels=CHANNELS,
dtype="float32",
blocksize=BLOCK_SIZE,
callback=callback,
):
self.ui.write_log("SYS: Mic active (Whisper STT).")
while True:
try:
chunk = q.get(timeout=0.1)
audio = vad.process(chunk.flatten())
if audio is not None:
self.ui.set_state("THINKING")
text = self._stt.transcribe(audio)
if text.strip():
self._process_message(text)
except queue.Empty:
pass
except Exception as e:
print(f"[STT-Whisper] Mic error: {e}")
traceback.print_exc()
def _listen_vosk(self) -> None:
"""Mic → Vosk streaming → LLM loop."""
q: queue.Queue = queue.Queue(maxsize=200)
def callback(indata, frames, time_info, status):
with self._speaking_lock:
is_speaking = self._speaking
if not is_speaking and not self.ui.muted:
try:
q.put_nowait(indata.copy())
except queue.Full:
pass
try:
with sd.InputStream(
samplerate=SAMPLE_RATE_IN,
channels=CHANNELS,
dtype="int16",
blocksize=4096,
callback=callback,
):
self.ui.write_log("SYS: Mic active (Vosk STT).")
while True:
try:
chunk = q.get(timeout=0.1)
text, is_final = self._stt.process_chunk(chunk.tobytes())
if is_final and text.strip():
self._process_message(text)
except queue.Empty:
pass
except Exception as e:
print(f"[STT-Vosk] Mic error: {e}")
traceback.print_exc()
# ------------------------------------------------------------------
# Text command loop (UI input box)
# ------------------------------------------------------------------
def _text_command_loop(self) -> None:
while True:
try:
text = self._text_queue.get(timeout=0.5)
if text.strip():
self._process_message(text)
except queue.Empty:
pass
# ------------------------------------------------------------------
# Entry point
# ------------------------------------------------------------------
def run(self) -> None:
"""Initialise STT + TTS, then start the main listen loop."""
try:
# -- Wire reconfigure callback --
self.ui.on_reconfigure = self.reconfigure
# -- Ollama auto-start --
from core.llm_client import ensure_ollama_running, warmup_model
self.ui.write_log("SYS: Checking Ollama…")
if ensure_ollama_running():
self.ui.write_log("SYS: Ollama OK.")
else:
self.ui.write_log("ERR: Ollama unavailable. Start it manually: ollama serve")
self.speak("Ollama is not running. Please start it manually.")
# -- LLM warmup (parallel with STT/TTS loading) --
_warmup_done = threading.Event()
def _do_warmup():
try:
warmup_model()
finally:
_warmup_done.set()
threading.Thread(target=_do_warmup, daemon=True).start()
# -- STT --
stt_engine = self._config.get("stt_engine", "whisper").lower()
stt_language = self._config.get("stt_language", "auto")
stt_model = self._config.get("stt_model", "base")
self.ui.write_log(f"SYS: Loading {stt_engine.upper()} STT…")
if stt_engine == "whisper":
self.ui.write_log(f"SYS: Whisper '{stt_model}' — downloading if not cached…")
elif stt_engine == "vosk":
self.ui.write_log("SYS: Vosk — downloading model if not cached…")
if stt_engine == "vosk":
from core.stt import VoskSTT
self._stt = VoskSTT(
self._config.get("vosk_model_path"),
language=stt_language,
)
else:
from core.stt import WhisperSTT
self._stt = WhisperSTT(stt_model, language=stt_language)
self.ui.write_log("SYS: STT ready.")
# -- TTS --
tts_engine = self._config.get("tts_engine", "edgetts").lower()
self.ui.write_log(f"SYS: Loading {tts_engine.upper()} TTS…")
if tts_engine == "kokoro":
self.ui.write_log("SYS: Kokoro — downloading model (~330 MB) if not cached…")
from core.tts import create_tts_player
self._tts = create_tts_player(self._config)
self.ui.write_log("SYS: TTS ready.")
# -- Wait for LLM warmup to finish --
self.ui.write_log("SYS: Loading LLM into memory…")
_warmup_done.wait(timeout=120)
self.ui.write_log("SYS: LLM ready.")
# Welcome
self.ui.write_log("SYS: JARVIS online.")
self.ui.set_state("LISTENING")
# TTS queue worker (sequential, no overlap)
threading.Thread(target=self._tts_worker, daemon=True).start()
self.speak("JARVIS online. All systems ready.")
# Text-command handler (runs in background)
threading.Thread(target=self._text_command_loop, daemon=True).start()
# STT loop (blocking — keeps this thread alive)
if stt_engine == "vosk":
self._listen_vosk()
else:
self._listen_whisper()
except Exception as e:
self.ui.write_log(f"ERR: Init failed — {e}")
traceback.print_exc()
# ---------------------------------------------------------------------------
# Entry
# ---------------------------------------------------------------------------
def main() -> None:
ui = JarvisUI("face.png")
def runner():
# 1. Wait until the user completes the setup overlay (first run)
# or config already exists (subsequent runs).
ui.wait_for_api_key()
# 2. Install any missing engine packages before loading engines.
# Progress is streamed to the log panel in real time.
ui.write_log("SYS: Checking dependencies…")
cfg = _load_config()
_install_done = threading.Event()
def _do_install():
try:
from core.installer import install_for_config
install_for_config(cfg, log=ui.write_log)
except Exception as e:
ui.write_log(f"ERR: Dependency install — {e}")
finally:
_install_done.set()
threading.Thread(target=_do_install, daemon=True).start()
_install_done.wait() # blocks runner thread; UI remains responsive
# 3. Start the assistant (loads STT / TTS / LLM).
jarvis = JarvisLocal(ui)
try:
jarvis.run()
except KeyboardInterrupt:
print("\n[MARK XL] Shutting down…")
threading.Thread(target=runner, daemon=True).start()
ui.root.mainloop()
if __name__ == "__main__":
main()