1192 lines
49 KiB
Python
1192 lines
49 KiB
Python
"""
|
||
MARK XL — Local LLM Edition
|
||
STT (Whisper / Vosk) + Ollama LLM + TTS (EdgeTTS / Kokoro / ElevenLabs)
|
||
All Gemini / Google-AI dependencies removed.
|
||
"""
|
||
# ── Bootstrap: auto-install base UI packages before anything else ──────────
|
||
# Uses only stdlib so it works even on a completely fresh Python install.
|
||
import importlib.util as _ilu
|
||
import os as _os
|
||
import subprocess as _sp
|
||
import sys as _sys
|
||
|
||
_BASE_PKGS = [
|
||
("PyQt6", "PyQt6"),
|
||
("psutil", "psutil"),
|
||
("numpy", "numpy"),
|
||
("sounddevice", "sounddevice"),
|
||
("PIL", "pillow"),
|
||
("requests", "requests"),
|
||
]
|
||
|
||
def _bootstrap() -> None:
|
||
need = [pkg for mod, pkg in _BASE_PKGS if _ilu.find_spec(mod) is None]
|
||
if not need:
|
||
return
|
||
print(f"\n[MARK XL] First-run setup — installing: {', '.join(need)}")
|
||
print("[MARK XL] This happens only once.\n")
|
||
_sp.run([_sys.executable, "-m", "pip", "install", *need], check=True)
|
||
print("\n[MARK XL] Base packages ready — restarting…\n")
|
||
# Replace current process with a fresh one (picks up newly installed packages)
|
||
_os.execv(_sys.executable, [_sys.executable] + _sys.argv)
|
||
|
||
_bootstrap()
|
||
# ───────────────────────────────────────────────────────────────────────────
|
||
|
||
import json
|
||
import queue
|
||
import re
|
||
import sys
|
||
import threading
|
||
import traceback
|
||
from datetime import datetime
|
||
from pathlib import Path
|
||
|
||
import numpy as np
|
||
import sounddevice as sd
|
||
|
||
from ui import JarvisUI
|
||
from memory.memory_manager import load_memory, update_memory, format_memory_for_prompt
|
||
from core.llm_client import call_llm, call_llm_stream, get_llm_settings
|
||
|
||
from actions.file_processor import file_processor
|
||
from actions.flight_finder import flight_finder
|
||
from actions.open_app import open_app
|
||
from actions.weather_report import weather_action
|
||
from actions.send_message import send_message
|
||
from actions.reminder import reminder
|
||
from actions.computer_settings import computer_settings
|
||
from actions.screen_processor import screen_process
|
||
from actions.youtube_video import youtube_video
|
||
from actions.desktop import desktop_control
|
||
from actions.browser_control import browser_control
|
||
from actions.file_controller import file_controller
|
||
from actions.code_helper import code_helper
|
||
from actions.dev_agent import dev_agent
|
||
from actions.web_search import web_search as web_search_action
|
||
from actions.computer_control import computer_control
|
||
from actions.game_updater import game_updater
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Paths
|
||
# ---------------------------------------------------------------------------
|
||
|
||
def _get_base_dir() -> Path:
|
||
if getattr(sys, "frozen", False):
|
||
return Path(sys.executable).parent
|
||
return Path(__file__).resolve().parent
|
||
|
||
|
||
BASE_DIR = _get_base_dir()
|
||
API_CONFIG_PATH = BASE_DIR / "config" / "api_keys.json"
|
||
PROMPT_PATH = BASE_DIR / "core" / "prompt.txt"
|
||
|
||
SAMPLE_RATE_IN = 16_000
|
||
BLOCK_SIZE = 1_024
|
||
CHANNELS = 1
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Tool declarations (Gemini format kept for readability;
|
||
# converted to OpenAI/Ollama format by _to_ollama_tools())
|
||
# ---------------------------------------------------------------------------
|
||
|
||
TOOL_DECLARATIONS = [
|
||
{
|
||
"name": "open_app",
|
||
"description": (
|
||
"Opens any application on the computer. "
|
||
"Use this whenever the user asks to open, launch, or start any app, "
|
||
"website, or program. Always call this tool — never just say you opened it."
|
||
),
|
||
"parameters": {
|
||
"type": "OBJECT",
|
||
"properties": {
|
||
"app_name": {"type": "STRING", "description": "Exact name of the application"}
|
||
},
|
||
"required": ["app_name"]
|
||
}
|
||
},
|
||
{
|
||
"name": "web_search",
|
||
"description": "Searches the web for any information.",
|
||
"parameters": {
|
||
"type": "OBJECT",
|
||
"properties": {
|
||
"query": {"type": "STRING", "description": "Search query"},
|
||
"mode": {"type": "STRING", "description": "search (default) or compare"},
|
||
"items": {"type": "ARRAY", "items": {"type": "STRING"}, "description": "Items to compare"},
|
||
"aspect": {"type": "STRING", "description": "price | specs | reviews"}
|
||
},
|
||
"required": ["query"]
|
||
}
|
||
},
|
||
{
|
||
"name": "weather_report",
|
||
"description": "Gives the weather report to user",
|
||
"parameters": {
|
||
"type": "OBJECT",
|
||
"properties": {"city": {"type": "STRING", "description": "City name"}},
|
||
"required": ["city"]
|
||
}
|
||
},
|
||
{
|
||
"name": "send_message",
|
||
"description": "Sends a text message via WhatsApp, Telegram, or other messaging platform.",
|
||
"parameters": {
|
||
"type": "OBJECT",
|
||
"properties": {
|
||
"receiver": {"type": "STRING", "description": "Recipient contact name"},
|
||
"message_text": {"type": "STRING", "description": "The message to send"},
|
||
"platform": {"type": "STRING", "description": "Platform: WhatsApp, Telegram, etc."}
|
||
},
|
||
"required": ["receiver", "message_text", "platform"]
|
||
}
|
||
},
|
||
{
|
||
"name": "reminder",
|
||
"description": "Sets a timed reminder using Task Scheduler.",
|
||
"parameters": {
|
||
"type": "OBJECT",
|
||
"properties": {
|
||
"date": {"type": "STRING", "description": "Date in YYYY-MM-DD format"},
|
||
"time": {"type": "STRING", "description": "Time in HH:MM format (24h)"},
|
||
"message": {"type": "STRING", "description": "Reminder message text"}
|
||
},
|
||
"required": ["date", "time", "message"]
|
||
}
|
||
},
|
||
{
|
||
"name": "youtube_video",
|
||
"description": (
|
||
"Controls YouTube. Use for: playing videos, summarizing a video's content, "
|
||
"getting video info, or showing trending videos."
|
||
),
|
||
"parameters": {
|
||
"type": "OBJECT",
|
||
"properties": {
|
||
"action": {"type": "STRING", "description": "play | summarize | get_info | trending"},
|
||
"query": {"type": "STRING", "description": "Search query for play action"},
|
||
"save": {"type": "BOOLEAN", "description": "Save summary to Notepad"},
|
||
"region": {"type": "STRING", "description": "Country code for trending e.g. TR, US"},
|
||
"url": {"type": "STRING", "description": "Video URL for get_info action"},
|
||
},
|
||
"required": []
|
||
}
|
||
},
|
||
{
|
||
"name": "screen_process",
|
||
"description": (
|
||
"Captures and analyzes the screen or webcam image. "
|
||
"MUST be called when user asks what is on screen, what you see, "
|
||
"analyze my screen, look at camera, etc. "
|
||
"You have NO visual ability without this tool."
|
||
),
|
||
"parameters": {
|
||
"type": "OBJECT",
|
||
"properties": {
|
||
"angle": {"type": "STRING", "description": "'screen' or 'camera'. Default: 'screen'"},
|
||
"text": {"type": "STRING", "description": "The question about the captured image"}
|
||
},
|
||
"required": ["text"]
|
||
}
|
||
},
|
||
{
|
||
"name": "computer_settings",
|
||
"description": (
|
||
"Controls the computer: volume, brightness, window management, keyboard shortcuts, "
|
||
"typing text on screen, closing apps, fullscreen, dark mode, WiFi, restart, shutdown, "
|
||
"scrolling, tab management, zoom, screenshots, lock screen, refresh/reload page."
|
||
),
|
||
"parameters": {
|
||
"type": "OBJECT",
|
||
"properties": {
|
||
"action": {"type": "STRING", "description": "The action to perform"},
|
||
"description": {"type": "STRING", "description": "Natural language description"},
|
||
"value": {"type": "STRING", "description": "Optional value"}
|
||
},
|
||
"required": []
|
||
}
|
||
},
|
||
{
|
||
"name": "browser_control",
|
||
"description": (
|
||
"Controls any web browser. Use for: opening websites, searching the web, "
|
||
"clicking elements, filling forms, scrolling, screenshots, navigation."
|
||
),
|
||
"parameters": {
|
||
"type": "OBJECT",
|
||
"properties": {
|
||
"action": {"type": "STRING", "description": "go_to | search | click | type | scroll | fill_form | smart_click | smart_type | get_text | get_url | press | new_tab | close_tab | screenshot | back | forward | reload | switch | list_browsers | close | close_all"},
|
||
"browser": {"type": "STRING", "description": "chrome | edge | firefox | opera | operagx | brave | vivaldi | safari"},
|
||
"url": {"type": "STRING", "description": "URL for go_to / new_tab action"},
|
||
"query": {"type": "STRING", "description": "Search query"},
|
||
"engine": {"type": "STRING", "description": "google | bing | duckduckgo | yandex"},
|
||
"selector": {"type": "STRING", "description": "CSS selector for click/type"},
|
||
"text": {"type": "STRING", "description": "Text to click or type"},
|
||
"description": {"type": "STRING", "description": "Element description for smart_click/smart_type"},
|
||
"direction": {"type": "STRING", "description": "up | down for scroll"},
|
||
"amount": {"type": "INTEGER", "description": "Scroll amount in pixels"},
|
||
"key": {"type": "STRING", "description": "Key name for press"},
|
||
"path": {"type": "STRING", "description": "Save path for screenshot"},
|
||
"incognito": {"type": "BOOLEAN", "description": "Open in private/incognito mode"},
|
||
"clear_first": {"type": "BOOLEAN", "description": "Clear field before typing"},
|
||
},
|
||
"required": ["action"]
|
||
}
|
||
},
|
||
{
|
||
"name": "file_controller",
|
||
"description": "Manages files and folders: list, create, delete, move, copy, rename, read, write, find, disk usage.",
|
||
"parameters": {
|
||
"type": "OBJECT",
|
||
"properties": {
|
||
"action": {"type": "STRING", "description": "list | create_file | create_folder | delete | move | copy | rename | read | write | find | largest | disk_usage | organize_desktop | info"},
|
||
"path": {"type": "STRING", "description": "File/folder path or shortcut: desktop, downloads, documents, home"},
|
||
"destination": {"type": "STRING", "description": "Destination path for move/copy"},
|
||
"new_name": {"type": "STRING", "description": "New name for rename"},
|
||
"content": {"type": "STRING", "description": "Content for create_file/write"},
|
||
"name": {"type": "STRING", "description": "File name to search for"},
|
||
"extension": {"type": "STRING", "description": "File extension to search"},
|
||
"count": {"type": "INTEGER", "description": "Number of results for largest"},
|
||
},
|
||
"required": ["action"]
|
||
}
|
||
},
|
||
{
|
||
"name": "desktop_control",
|
||
"description": "Controls the desktop: wallpaper, organize, clean, list, stats.",
|
||
"parameters": {
|
||
"type": "OBJECT",
|
||
"properties": {
|
||
"action": {"type": "STRING", "description": "wallpaper | wallpaper_url | organize | clean | list | stats | task"},
|
||
"path": {"type": "STRING", "description": "Image path for wallpaper"},
|
||
"url": {"type": "STRING", "description": "Image URL for wallpaper_url"},
|
||
"mode": {"type": "STRING", "description": "by_type or by_date for organize"},
|
||
"task": {"type": "STRING", "description": "Natural language desktop task"},
|
||
},
|
||
"required": ["action"]
|
||
}
|
||
},
|
||
{
|
||
"name": "code_helper",
|
||
"description": "Writes, edits, explains, runs, or builds code files.",
|
||
"parameters": {
|
||
"type": "OBJECT",
|
||
"properties": {
|
||
"action": {"type": "STRING", "description": "write | edit | explain | run | build | auto"},
|
||
"description": {"type": "STRING", "description": "What the code should do"},
|
||
"language": {"type": "STRING", "description": "Programming language"},
|
||
"output_path": {"type": "STRING", "description": "Where to save the file"},
|
||
"file_path": {"type": "STRING", "description": "Path to existing file"},
|
||
"code": {"type": "STRING", "description": "Raw code string for explain"},
|
||
"args": {"type": "STRING", "description": "CLI arguments"},
|
||
"timeout": {"type": "INTEGER", "description": "Execution timeout in seconds"},
|
||
},
|
||
"required": ["action"]
|
||
}
|
||
},
|
||
{
|
||
"name": "dev_agent",
|
||
"description": "Builds complete multi-file projects from scratch.",
|
||
"parameters": {
|
||
"type": "OBJECT",
|
||
"properties": {
|
||
"description": {"type": "STRING", "description": "What the project should do"},
|
||
"language": {"type": "STRING", "description": "Programming language"},
|
||
"project_name": {"type": "STRING", "description": "Optional project folder name"},
|
||
"timeout": {"type": "INTEGER", "description": "Run timeout in seconds"},
|
||
},
|
||
"required": ["description"]
|
||
}
|
||
},
|
||
{
|
||
"name": "agent_task",
|
||
"description": (
|
||
"Executes complex multi-step tasks requiring multiple different tools. "
|
||
"Examples: 'research X and save to file', 'find and organize files'. "
|
||
"DO NOT use for single commands."
|
||
),
|
||
"parameters": {
|
||
"type": "OBJECT",
|
||
"properties": {
|
||
"goal": {"type": "STRING", "description": "Complete description of what to accomplish"},
|
||
"priority": {"type": "STRING", "description": "low | normal | high"}
|
||
},
|
||
"required": ["goal"]
|
||
}
|
||
},
|
||
{
|
||
"name": "computer_control",
|
||
"description": "Direct computer control: type, click, hotkeys, scroll, move mouse, screenshots, find elements on screen.",
|
||
"parameters": {
|
||
"type": "OBJECT",
|
||
"properties": {
|
||
"action": {"type": "STRING", "description": "type | smart_type | click | double_click | right_click | hotkey | press | scroll | move | copy | paste | screenshot | wait | clear_field | focus_window | screen_find | screen_click | random_data | user_data"},
|
||
"text": {"type": "STRING", "description": "Text to type or paste"},
|
||
"x": {"type": "INTEGER", "description": "X coordinate"},
|
||
"y": {"type": "INTEGER", "description": "Y coordinate"},
|
||
"keys": {"type": "STRING", "description": "Key combination e.g. 'ctrl+c'"},
|
||
"key": {"type": "STRING", "description": "Single key e.g. 'enter'"},
|
||
"direction": {"type": "STRING", "description": "up | down | left | right"},
|
||
"amount": {"type": "INTEGER", "description": "Scroll amount"},
|
||
"seconds": {"type": "NUMBER", "description": "Seconds to wait"},
|
||
"title": {"type": "STRING", "description": "Window title for focus_window"},
|
||
"description": {"type": "STRING", "description": "Element description"},
|
||
"type": {"type": "STRING", "description": "Data type for random_data"},
|
||
"field": {"type": "STRING", "description": "Field for user_data"},
|
||
"clear_first": {"type": "BOOLEAN", "description": "Clear field before typing"},
|
||
"path": {"type": "STRING", "description": "Save path for screenshot"},
|
||
},
|
||
"required": ["action"]
|
||
}
|
||
},
|
||
{
|
||
"name": "game_updater",
|
||
"description": (
|
||
"THE ONLY tool for ANY Steam or Epic Games request. "
|
||
"Use for: installing, downloading, updating games, listing installed games."
|
||
),
|
||
"parameters": {
|
||
"type": "OBJECT",
|
||
"properties": {
|
||
"action": {"type": "STRING", "description": "update | install | list | download_status | schedule | cancel_schedule | schedule_status"},
|
||
"platform": {"type": "STRING", "description": "steam | epic | both"},
|
||
"game_name": {"type": "STRING", "description": "Game name"},
|
||
"app_id": {"type": "STRING", "description": "Steam AppID"},
|
||
"hour": {"type": "INTEGER", "description": "Hour for scheduled update 0-23"},
|
||
"minute": {"type": "INTEGER", "description": "Minute for scheduled update 0-59"},
|
||
"shutdown_when_done": {"type": "BOOLEAN", "description": "Shut down PC when done"},
|
||
},
|
||
"required": []
|
||
}
|
||
},
|
||
{
|
||
"name": "flight_finder",
|
||
"description": "Searches Google Flights and speaks the best options.",
|
||
"parameters": {
|
||
"type": "OBJECT",
|
||
"properties": {
|
||
"origin": {"type": "STRING", "description": "Departure city or airport code"},
|
||
"destination": {"type": "STRING", "description": "Arrival city or airport code"},
|
||
"date": {"type": "STRING", "description": "Departure date"},
|
||
"return_date": {"type": "STRING", "description": "Return date for round trips"},
|
||
"passengers": {"type": "INTEGER", "description": "Number of passengers"},
|
||
"cabin": {"type": "STRING", "description": "economy | premium | business | first"},
|
||
"save": {"type": "BOOLEAN", "description": "Save results to Notepad"},
|
||
},
|
||
"required": ["origin", "destination", "date"]
|
||
}
|
||
},
|
||
{
|
||
"name": "shutdown_jarvis",
|
||
"description": (
|
||
"Shuts down the assistant completely. "
|
||
"Call this when the user expresses intent to end the conversation, "
|
||
"close the assistant, say goodbye, or stop Jarvis."
|
||
),
|
||
"parameters": {"type": "OBJECT", "properties": {}}
|
||
},
|
||
{
|
||
"name": "file_processor",
|
||
"description": (
|
||
"Processes any file that the user has uploaded or dropped onto the interface. "
|
||
"Supports: images, PDFs, Word docs, CSV/Excel, JSON, code files, audio, video, archives."
|
||
),
|
||
"parameters": {
|
||
"type": "OBJECT",
|
||
"properties": {
|
||
"file_path": {"type": "STRING", "description": "Full path to the uploaded file"},
|
||
"action": {"type": "STRING", "description": "What to do with the file"},
|
||
"instruction": {"type": "STRING", "description": "Free-form instruction"},
|
||
"format": {"type": "STRING", "description": "Target format for conversion"},
|
||
"width": {"type": "INTEGER", "description": "Target width for image resize"},
|
||
"height": {"type": "INTEGER", "description": "Target height for image resize"},
|
||
"scale": {"type": "NUMBER", "description": "Scale factor"},
|
||
"quality": {"type": "INTEGER", "description": "Quality 1-100"},
|
||
"start": {"type": "STRING", "description": "Start time for trim"},
|
||
"end": {"type": "STRING", "description": "End time for trim"},
|
||
"timestamp": {"type": "STRING", "description": "Timestamp for video frame extraction"},
|
||
"column": {"type": "STRING", "description": "Column name for CSV filter/sort"},
|
||
"value": {"type": "STRING", "description": "Filter value"},
|
||
"condition": {"type": "STRING", "description": "Filter condition"},
|
||
"ascending": {"type": "BOOLEAN", "description": "Sort order"},
|
||
"save": {"type": "BOOLEAN", "description": "Save result to file"},
|
||
"destination": {"type": "STRING", "description": "Output folder for archive extract"},
|
||
},
|
||
"required": []
|
||
}
|
||
},
|
||
{
|
||
"name": "save_memory",
|
||
"description": (
|
||
"Save an important personal fact about the user to long-term memory. "
|
||
"Call this silently whenever the user reveals something worth remembering. "
|
||
"Do NOT announce that you are saving — just call it silently."
|
||
),
|
||
"parameters": {
|
||
"type": "OBJECT",
|
||
"properties": {
|
||
"category": {
|
||
"type": "STRING",
|
||
"description": (
|
||
"identity | preferences | projects | relationships | wishes | notes"
|
||
)
|
||
},
|
||
"key": {"type": "STRING", "description": "Short snake_case key"},
|
||
"value": {"type": "STRING", "description": "Concise value in English"},
|
||
},
|
||
"required": ["category", "key", "value"]
|
||
}
|
||
},
|
||
]
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Convert Gemini-style declarations to OpenAI/Ollama format
|
||
# ---------------------------------------------------------------------------
|
||
|
||
_TYPE_MAP = {
|
||
"OBJECT": "object", "STRING": "string", "ARRAY": "array",
|
||
"INTEGER": "integer", "BOOLEAN": "boolean", "NUMBER": "number",
|
||
}
|
||
|
||
|
||
def _convert_type(t: str) -> str:
|
||
return _TYPE_MAP.get(t, t.lower()) if isinstance(t, str) else t
|
||
|
||
|
||
def _convert_props(props: dict) -> dict:
|
||
out = {}
|
||
for k, v in props.items():
|
||
nv = dict(v)
|
||
if "type" in nv:
|
||
nv["type"] = _convert_type(nv["type"])
|
||
if "items" in nv and isinstance(nv["items"], dict):
|
||
nv["items"] = {"type": _convert_type(nv["items"].get("type", "string"))}
|
||
out[k] = nv
|
||
return out
|
||
|
||
|
||
def _to_ollama_tools(decls: list) -> list:
|
||
tools = []
|
||
for d in decls:
|
||
params = d.get("parameters", {})
|
||
new_params: dict = {
|
||
"type": "object",
|
||
"properties": _convert_props(params.get("properties", {})),
|
||
}
|
||
req = params.get("required")
|
||
if req:
|
||
new_params["required"] = req
|
||
tools.append({
|
||
"type": "function",
|
||
"function": {
|
||
"name": d["name"],
|
||
"description": d["description"],
|
||
"parameters": new_params,
|
||
},
|
||
})
|
||
return tools
|
||
|
||
|
||
OLLAMA_TOOLS = _to_ollama_tools(TOOL_DECLARATIONS)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Helpers
|
||
# ---------------------------------------------------------------------------
|
||
|
||
def _load_config() -> dict:
|
||
try:
|
||
with open(API_CONFIG_PATH, encoding="utf-8") as f:
|
||
return json.load(f)
|
||
except Exception:
|
||
return {}
|
||
|
||
|
||
def _load_system_prompt() -> str:
|
||
try:
|
||
return PROMPT_PATH.read_text(encoding="utf-8")
|
||
except Exception:
|
||
return (
|
||
"You are JARVIS, Tony Stark's AI assistant. "
|
||
"Be concise, direct, and always use the provided tools to complete tasks. "
|
||
"Never simulate or guess results — always call the appropriate tool."
|
||
)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Voice Activity Detection (used for Whisper listen loop)
|
||
# ---------------------------------------------------------------------------
|
||
|
||
class _VADBuffer:
|
||
"""Energy-based VAD: buffers audio until end of utterance."""
|
||
|
||
def __init__(
|
||
self,
|
||
sample_rate: int = 16_000,
|
||
silence_sec: float = 1.2,
|
||
threshold: float = 0.015,
|
||
min_speech_sec: float = 0.4,
|
||
max_speech_sec: float = 30.0,
|
||
):
|
||
self._sr = sample_rate
|
||
self._sil_n = int(silence_sec * sample_rate)
|
||
self._thresh = threshold
|
||
self._min_n = int(min_speech_sec * sample_rate)
|
||
self._max_n = int(max_speech_sec * sample_rate)
|
||
self._buf: list[np.ndarray] = []
|
||
self._in_spch = False
|
||
self._sil_cnt = 0
|
||
|
||
def process(self, chunk: np.ndarray) -> np.ndarray | None:
|
||
"""
|
||
Feed one audio chunk (float32 mono).
|
||
Returns complete utterance when speech ends, otherwise None.
|
||
"""
|
||
rms = float(np.sqrt(np.mean(chunk ** 2)))
|
||
total_n = sum(len(c) for c in self._buf)
|
||
|
||
if rms > self._thresh:
|
||
self._in_spch = True
|
||
self._sil_cnt = 0
|
||
self._buf.append(chunk.copy())
|
||
elif self._in_spch:
|
||
self._buf.append(chunk.copy())
|
||
self._sil_cnt += len(chunk)
|
||
|
||
if self._sil_cnt >= self._sil_n or total_n >= self._max_n:
|
||
audio = np.concatenate(self._buf)
|
||
self._buf = []
|
||
self._in_spch = False
|
||
self._sil_cnt = 0
|
||
if len(audio) >= self._min_n:
|
||
return audio
|
||
return None
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# JarvisLocal
|
||
# ---------------------------------------------------------------------------
|
||
|
||
class JarvisLocal:
|
||
"""
|
||
Main assistant class.
|
||
Replaces JarvisLive (Gemini Live API) with:
|
||
STT (Whisper/Vosk) → Ollama LLM (tool calling) → TTS (Edge/Kokoro/ElevenLabs)
|
||
"""
|
||
|
||
def __init__(self, ui: JarvisUI):
|
||
self.ui = ui
|
||
self._config = _load_config()
|
||
self._stt = None
|
||
self._tts = None
|
||
self._speaking = False
|
||
self._speaking_lock = threading.Lock()
|
||
self._text_queue: queue.Queue = queue.Queue()
|
||
self._tts_queue: queue.Queue = queue.Queue()
|
||
self._conversation: list[dict] = []
|
||
|
||
self.ui.on_text_command = self._on_text_command
|
||
|
||
# ------------------------------------------------------------------
|
||
# System prompt
|
||
# ------------------------------------------------------------------
|
||
|
||
def _build_system_prompt(self) -> str:
|
||
memory = load_memory()
|
||
mem_str = format_memory_for_prompt(memory)
|
||
sys_p = _load_system_prompt()
|
||
now = datetime.now()
|
||
time_ctx = (
|
||
f"[CURRENT DATE & TIME]\n"
|
||
f"Right now it is: {now.strftime('%A, %B %d, %Y — %I:%M %p')}\n"
|
||
f"Use this to calculate exact times for reminders.\n\n"
|
||
)
|
||
parts = [time_ctx]
|
||
if mem_str:
|
||
parts.append(mem_str)
|
||
parts.append(sys_p)
|
||
return "\n".join(parts)
|
||
|
||
# ------------------------------------------------------------------
|
||
# Speaking state & TTS
|
||
# ------------------------------------------------------------------
|
||
|
||
# ------------------------------------------------------------------
|
||
# TTS queue worker — plays sentences sequentially, no overlaps
|
||
# ------------------------------------------------------------------
|
||
|
||
def _tts_worker(self) -> None:
|
||
while True:
|
||
text = self._tts_queue.get()
|
||
n_got = 1 # how many queue items we consumed this round
|
||
|
||
try:
|
||
if text and self._tts:
|
||
# ── Kokoro: merge buffered sentences into one inference call ──
|
||
# Each separate Kokoro call costs ~0.5–1 s startup.
|
||
# Draining the queue here combines "Hello. How are you?" into
|
||
# one call instead of two, eliminating the gap between them.
|
||
from core.tts import KokoroTTSEngine
|
||
engine = getattr(self._tts, "_engine", None)
|
||
if isinstance(engine, KokoroTTSEngine):
|
||
while True:
|
||
try:
|
||
nxt = self._tts_queue.get_nowait()
|
||
if nxt:
|
||
text = text.rstrip() + " " + nxt.lstrip()
|
||
n_got += 1
|
||
except queue.Empty:
|
||
break
|
||
|
||
with self._speaking_lock:
|
||
self._speaking = True
|
||
self.ui.set_state("SPEAKING")
|
||
self._tts.speak(text)
|
||
|
||
except Exception as e:
|
||
print(f"[TTS] speak error: {e}")
|
||
finally:
|
||
for _ in range(n_got):
|
||
self._tts_queue.task_done()
|
||
if self._tts_queue.empty():
|
||
with self._speaking_lock:
|
||
self._speaking = False
|
||
if not self.ui.muted:
|
||
self.ui.set_state("LISTENING")
|
||
|
||
def set_speaking(self, value: bool) -> None:
|
||
with self._speaking_lock:
|
||
self._speaking = value
|
||
if value:
|
||
self.ui.set_state("SPEAKING")
|
||
elif not self.ui.muted:
|
||
self.ui.set_state("LISTENING")
|
||
|
||
def speak(self, text: str) -> None:
|
||
if not text or not self._tts:
|
||
return
|
||
with self._speaking_lock:
|
||
self._speaking = True
|
||
self._tts_queue.put(text)
|
||
|
||
def speak_error(self, tool_name: str, error) -> None:
|
||
short = str(error)[:120]
|
||
self.ui.write_log(f"ERR: {tool_name} — {short}")
|
||
self.speak(f"{tool_name} encountered an error.")
|
||
|
||
# ------------------------------------------------------------------
|
||
# Live reconfigure (called when user clicks Apply in Configure panel)
|
||
# ------------------------------------------------------------------
|
||
|
||
def reconfigure(self, new_config: dict) -> None:
|
||
"""Non-blocking: spawns a background thread to install + reload."""
|
||
threading.Thread(
|
||
target=self._do_reconfigure, args=(new_config,), daemon=True
|
||
).start()
|
||
|
||
def _do_reconfigure(self, new_config: dict) -> None:
|
||
old_stt_engine = self._config.get("stt_engine", "whisper").lower()
|
||
old_llm_model = self._config.get("llm_model", "")
|
||
new_stt_engine = new_config.get("stt_engine", "whisper").lower()
|
||
self._config = new_config
|
||
|
||
# Install any packages required by the new config
|
||
try:
|
||
from core.installer import install_for_config
|
||
install_for_config(new_config, log=self.ui.write_log)
|
||
except Exception as e:
|
||
self.ui.write_log(f"ERR: Dependency install — {e}")
|
||
|
||
# TTS: always hot-reload (runs in queue worker, safe to swap)
|
||
try:
|
||
from core.tts import create_tts_player
|
||
self._tts = create_tts_player(new_config)
|
||
self.ui.write_log("SYS: TTS reconfigured.")
|
||
except Exception as e:
|
||
self.ui.write_log(f"ERR: TTS reconfigure — {e}")
|
||
|
||
# STT: hot-reload if same engine type; full restart needed if engine changed
|
||
if old_stt_engine == new_stt_engine:
|
||
try:
|
||
stt_language = new_config.get("stt_language", "auto")
|
||
if new_stt_engine == "vosk":
|
||
from core.stt import VoskSTT
|
||
self._stt = VoskSTT(new_config.get("vosk_model_path"), language=stt_language)
|
||
else:
|
||
from core.stt import WhisperSTT
|
||
self._stt = WhisperSTT(new_config.get("stt_model", "base"), language=stt_language)
|
||
self.ui.write_log("SYS: STT reconfigured.")
|
||
except Exception as e:
|
||
self.ui.write_log(f"ERR: STT reconfigure — {e}")
|
||
else:
|
||
self.ui.write_log("SYS: STT engine changed — restart required.")
|
||
|
||
# LLM warmup if model changed
|
||
if new_config.get("llm_model", "") != old_llm_model:
|
||
self.ui.write_log("SYS: Warming up new LLM model…")
|
||
from core.llm_client import warmup_model
|
||
warmup_model()
|
||
self.ui.write_log("SYS: New LLM model ready.")
|
||
|
||
if old_stt_engine == new_stt_engine:
|
||
self.speak("Configuration applied.")
|
||
else:
|
||
self.speak("LLM and TTS updated. Restart for speech engine change.")
|
||
|
||
# ------------------------------------------------------------------
|
||
# Text command (from UI input box)
|
||
# ------------------------------------------------------------------
|
||
|
||
def _on_text_command(self, text: str) -> None:
|
||
self._text_queue.put(text)
|
||
|
||
# ------------------------------------------------------------------
|
||
# Tool execution (routing unchanged from original)
|
||
# ------------------------------------------------------------------
|
||
|
||
def _execute_tool(self, name: str, args: dict) -> str:
|
||
print(f"[JARVIS] 🔧 {name} {args}")
|
||
self.ui.set_state("THINKING")
|
||
|
||
# save_memory is handled silently
|
||
if name == "save_memory":
|
||
category = args.get("category", "notes")
|
||
key = args.get("key", "")
|
||
value = args.get("value", "")
|
||
if key and value:
|
||
update_memory({category: {key: {"value": value}}})
|
||
print(f"[Memory] 💾 {category}/{key} = {value}")
|
||
if not self.ui.muted:
|
||
self.ui.set_state("LISTENING")
|
||
return "__SILENT__"
|
||
|
||
result = "Done."
|
||
try:
|
||
if name == "open_app":
|
||
r = open_app(parameters=args, response=None, player=self.ui)
|
||
result = r or f"Opened {args.get('app_name')}."
|
||
|
||
elif name == "weather_report":
|
||
r = weather_action(parameters=args, player=self.ui)
|
||
result = r or "Weather delivered."
|
||
|
||
elif name == "browser_control":
|
||
r = browser_control(parameters=args, player=self.ui)
|
||
result = r or "Done."
|
||
|
||
elif name == "file_controller":
|
||
r = file_controller(parameters=args, player=self.ui)
|
||
result = r or "Done."
|
||
|
||
elif name == "send_message":
|
||
r = send_message(parameters=args, response=None, player=self.ui, session_memory=None)
|
||
result = r or f"Message sent to {args.get('receiver')}."
|
||
|
||
elif name == "reminder":
|
||
r = reminder(parameters=args, response=None, player=self.ui)
|
||
result = r or "Reminder set."
|
||
|
||
elif name == "youtube_video":
|
||
r = youtube_video(parameters=args, response=None, player=self.ui)
|
||
result = r or "Done."
|
||
|
||
elif name == "screen_process":
|
||
# Synchronous call — returns analysis text which the LLM can speak
|
||
r = screen_process(parameters=args, response=None, player=self.ui, session_memory=None)
|
||
result = r if isinstance(r, str) and r else "Screen analyzed."
|
||
|
||
elif name == "computer_settings":
|
||
r = computer_settings(parameters=args, response=None, player=self.ui)
|
||
result = r or "Done."
|
||
|
||
elif name == "desktop_control":
|
||
r = desktop_control(parameters=args, player=self.ui)
|
||
result = r or "Done."
|
||
|
||
elif name == "code_helper":
|
||
r = code_helper(parameters=args, player=self.ui, speak=self.speak)
|
||
result = r or "Done."
|
||
|
||
elif name == "dev_agent":
|
||
r = dev_agent(parameters=args, player=self.ui, speak=self.speak)
|
||
result = r or "Done."
|
||
|
||
elif name == "agent_task":
|
||
from agent.task_queue import get_queue, TaskPriority
|
||
priority_map = {
|
||
"low": TaskPriority.LOW,
|
||
"normal": TaskPriority.NORMAL,
|
||
"high": TaskPriority.HIGH,
|
||
}
|
||
priority = priority_map.get(
|
||
args.get("priority", "normal").lower(), TaskPriority.NORMAL
|
||
)
|
||
task_id = get_queue().submit(
|
||
goal=args.get("goal", ""), priority=priority, speak=self.speak
|
||
)
|
||
result = f"Task started (ID: {task_id})."
|
||
|
||
elif name == "web_search":
|
||
r = web_search_action(parameters=args, player=self.ui)
|
||
result = r or "Done."
|
||
|
||
elif name == "file_processor":
|
||
if not args.get("file_path") and self.ui.current_file:
|
||
args["file_path"] = self.ui.current_file
|
||
r = file_processor(parameters=args, player=self.ui, speak=self.speak)
|
||
result = r or "Done."
|
||
|
||
elif name == "computer_control":
|
||
r = computer_control(parameters=args, player=self.ui)
|
||
result = r or "Done."
|
||
|
||
elif name == "game_updater":
|
||
r = game_updater(parameters=args, player=self.ui, speak=self.speak)
|
||
result = r or "Done."
|
||
|
||
elif name == "flight_finder":
|
||
r = flight_finder(parameters=args, player=self.ui)
|
||
result = r or "Done."
|
||
|
||
elif name == "shutdown_jarvis":
|
||
self.ui.write_log("SYS: Shutdown requested.")
|
||
|
||
def _shutdown():
|
||
import time, os
|
||
self.speak("Goodbye.")
|
||
time.sleep(2.5)
|
||
os._exit(0)
|
||
|
||
threading.Thread(target=_shutdown, daemon=True).start()
|
||
return "Shutting down."
|
||
|
||
else:
|
||
result = f"Unknown tool: {name}"
|
||
|
||
except Exception as e:
|
||
result = f"Tool '{name}' failed: {e}"
|
||
traceback.print_exc()
|
||
self.speak_error(name, e)
|
||
|
||
if not self.ui.muted:
|
||
self.ui.set_state("LISTENING")
|
||
|
||
print(f"[JARVIS] 📤 {name} → {str(result)[:80]}")
|
||
return result
|
||
|
||
# ------------------------------------------------------------------
|
||
# LLM processing loop
|
||
# ------------------------------------------------------------------
|
||
|
||
def _process_message(self, user_text: str) -> None:
|
||
"""
|
||
Full turn:
|
||
user_text → Ollama stream (with tools) → execute tools → final spoken response
|
||
|
||
Streaming means TTS starts on the first complete sentence (~1-2 s),
|
||
not after the full response is ready.
|
||
"""
|
||
self.ui.set_state("THINKING")
|
||
self.ui.write_log(f"You: {user_text}")
|
||
|
||
self._conversation.append({"role": "user", "content": user_text})
|
||
|
||
MAX_HISTORY = 30
|
||
if len(self._conversation) > MAX_HISTORY:
|
||
self._conversation = self._conversation[-MAX_HISTORY:]
|
||
|
||
messages = [
|
||
{"role": "system", "content": self._build_system_prompt()}
|
||
] + list(self._conversation)
|
||
|
||
MAX_TOOL_ROUNDS = 6
|
||
for _round in range(MAX_TOOL_ROUNDS):
|
||
final_content = ""
|
||
final_tool_calls: list = []
|
||
|
||
try:
|
||
for event in call_llm_stream(messages, OLLAMA_TOOLS):
|
||
if event["type"] == "sentence":
|
||
self.speak(event["text"])
|
||
elif event["type"] == "done":
|
||
final_content = event["content"]
|
||
final_tool_calls = event["tool_calls"]
|
||
except RuntimeError as e:
|
||
self.speak_error("LLM", e)
|
||
return
|
||
|
||
if not final_tool_calls:
|
||
# Final text response — TTS already started sentence-by-sentence
|
||
if final_content:
|
||
assistant_msg = {"role": "assistant", "content": final_content}
|
||
messages.append(assistant_msg)
|
||
self._conversation.append(assistant_msg)
|
||
self.ui.write_log(f"Jarvis: {final_content}")
|
||
break
|
||
|
||
# ------- tool calls -------
|
||
assistant_msg = {
|
||
"role": "assistant",
|
||
"content": final_content or "",
|
||
"tool_calls": final_tool_calls,
|
||
}
|
||
messages.append(assistant_msg)
|
||
self._conversation.append(assistant_msg)
|
||
|
||
all_silent = True
|
||
for tc in final_tool_calls:
|
||
fn = tc.get("function", {})
|
||
tname = fn.get("name", "")
|
||
targs = fn.get("arguments", {})
|
||
if isinstance(targs, str):
|
||
try:
|
||
targs = json.loads(targs)
|
||
except Exception:
|
||
targs = {}
|
||
|
||
tc_id = tc.get("id", "")
|
||
self.ui.write_log(f"SYS: ▶ {tname}")
|
||
result = self._execute_tool(tname, targs)
|
||
|
||
if result != "__SILENT__":
|
||
all_silent = False
|
||
|
||
tool_msg: dict = {
|
||
"role": "tool",
|
||
"content": "Done." if result == "__SILENT__" else str(result),
|
||
}
|
||
if tc_id:
|
||
tool_msg["tool_call_id"] = tc_id
|
||
|
||
messages.append(tool_msg)
|
||
self._conversation.append(tool_msg)
|
||
|
||
# Even if all tools were silent (e.g. save_memory only),
|
||
# always do one more LLM round so the assistant can verbally respond.
|
||
|
||
if not self.ui.muted:
|
||
self.ui.set_state("LISTENING")
|
||
|
||
# ------------------------------------------------------------------
|
||
# STT listening loops
|
||
# ------------------------------------------------------------------
|
||
|
||
def _listen_whisper(self) -> None:
|
||
"""Mic → VAD → Whisper → LLM loop."""
|
||
vad = _VADBuffer()
|
||
q: queue.Queue = queue.Queue(maxsize=200)
|
||
|
||
def callback(indata, frames, time_info, status):
|
||
with self._speaking_lock:
|
||
is_speaking = self._speaking
|
||
if not is_speaking and not self.ui.muted:
|
||
try:
|
||
q.put_nowait(indata.copy())
|
||
except queue.Full:
|
||
pass
|
||
|
||
try:
|
||
with sd.InputStream(
|
||
samplerate=SAMPLE_RATE_IN,
|
||
channels=CHANNELS,
|
||
dtype="float32",
|
||
blocksize=BLOCK_SIZE,
|
||
callback=callback,
|
||
):
|
||
self.ui.write_log("SYS: Mic active (Whisper STT).")
|
||
while True:
|
||
try:
|
||
chunk = q.get(timeout=0.1)
|
||
audio = vad.process(chunk.flatten())
|
||
if audio is not None:
|
||
self.ui.set_state("THINKING")
|
||
text = self._stt.transcribe(audio)
|
||
if text.strip():
|
||
self._process_message(text)
|
||
except queue.Empty:
|
||
pass
|
||
except Exception as e:
|
||
print(f"[STT-Whisper] Mic error: {e}")
|
||
traceback.print_exc()
|
||
|
||
def _listen_vosk(self) -> None:
|
||
"""Mic → Vosk streaming → LLM loop."""
|
||
q: queue.Queue = queue.Queue(maxsize=200)
|
||
|
||
def callback(indata, frames, time_info, status):
|
||
with self._speaking_lock:
|
||
is_speaking = self._speaking
|
||
if not is_speaking and not self.ui.muted:
|
||
try:
|
||
q.put_nowait(indata.copy())
|
||
except queue.Full:
|
||
pass
|
||
|
||
try:
|
||
with sd.InputStream(
|
||
samplerate=SAMPLE_RATE_IN,
|
||
channels=CHANNELS,
|
||
dtype="int16",
|
||
blocksize=4096,
|
||
callback=callback,
|
||
):
|
||
self.ui.write_log("SYS: Mic active (Vosk STT).")
|
||
while True:
|
||
try:
|
||
chunk = q.get(timeout=0.1)
|
||
text, is_final = self._stt.process_chunk(chunk.tobytes())
|
||
if is_final and text.strip():
|
||
self._process_message(text)
|
||
except queue.Empty:
|
||
pass
|
||
except Exception as e:
|
||
print(f"[STT-Vosk] Mic error: {e}")
|
||
traceback.print_exc()
|
||
|
||
# ------------------------------------------------------------------
|
||
# Text command loop (UI input box)
|
||
# ------------------------------------------------------------------
|
||
|
||
def _text_command_loop(self) -> None:
|
||
while True:
|
||
try:
|
||
text = self._text_queue.get(timeout=0.5)
|
||
if text.strip():
|
||
self._process_message(text)
|
||
except queue.Empty:
|
||
pass
|
||
|
||
# ------------------------------------------------------------------
|
||
# Entry point
|
||
# ------------------------------------------------------------------
|
||
|
||
def run(self) -> None:
|
||
"""Initialise STT + TTS, then start the main listen loop."""
|
||
try:
|
||
# -- Wire reconfigure callback --
|
||
self.ui.on_reconfigure = self.reconfigure
|
||
|
||
# -- Ollama auto-start --
|
||
from core.llm_client import ensure_ollama_running, warmup_model
|
||
self.ui.write_log("SYS: Checking Ollama…")
|
||
if ensure_ollama_running():
|
||
self.ui.write_log("SYS: Ollama OK.")
|
||
else:
|
||
self.ui.write_log("ERR: Ollama unavailable. Start it manually: ollama serve")
|
||
self.speak("Ollama is not running. Please start it manually.")
|
||
|
||
# -- LLM warmup (parallel with STT/TTS loading) --
|
||
_warmup_done = threading.Event()
|
||
def _do_warmup():
|
||
try:
|
||
warmup_model()
|
||
finally:
|
||
_warmup_done.set()
|
||
threading.Thread(target=_do_warmup, daemon=True).start()
|
||
|
||
# -- STT --
|
||
stt_engine = self._config.get("stt_engine", "whisper").lower()
|
||
stt_language = self._config.get("stt_language", "auto")
|
||
stt_model = self._config.get("stt_model", "base")
|
||
self.ui.write_log(f"SYS: Loading {stt_engine.upper()} STT…")
|
||
if stt_engine == "whisper":
|
||
self.ui.write_log(f"SYS: Whisper '{stt_model}' — downloading if not cached…")
|
||
elif stt_engine == "vosk":
|
||
self.ui.write_log("SYS: Vosk — downloading model if not cached…")
|
||
|
||
if stt_engine == "vosk":
|
||
from core.stt import VoskSTT
|
||
self._stt = VoskSTT(
|
||
self._config.get("vosk_model_path"),
|
||
language=stt_language,
|
||
)
|
||
else:
|
||
from core.stt import WhisperSTT
|
||
self._stt = WhisperSTT(stt_model, language=stt_language)
|
||
|
||
self.ui.write_log("SYS: STT ready.")
|
||
|
||
# -- TTS --
|
||
tts_engine = self._config.get("tts_engine", "edgetts").lower()
|
||
self.ui.write_log(f"SYS: Loading {tts_engine.upper()} TTS…")
|
||
if tts_engine == "kokoro":
|
||
self.ui.write_log("SYS: Kokoro — downloading model (~330 MB) if not cached…")
|
||
from core.tts import create_tts_player
|
||
self._tts = create_tts_player(self._config)
|
||
self.ui.write_log("SYS: TTS ready.")
|
||
|
||
# -- Wait for LLM warmup to finish --
|
||
self.ui.write_log("SYS: Loading LLM into memory…")
|
||
_warmup_done.wait(timeout=120)
|
||
self.ui.write_log("SYS: LLM ready.")
|
||
|
||
# Welcome
|
||
self.ui.write_log("SYS: JARVIS online.")
|
||
self.ui.set_state("LISTENING")
|
||
|
||
# TTS queue worker (sequential, no overlap)
|
||
threading.Thread(target=self._tts_worker, daemon=True).start()
|
||
|
||
self.speak("JARVIS online. All systems ready.")
|
||
|
||
# Text-command handler (runs in background)
|
||
threading.Thread(target=self._text_command_loop, daemon=True).start()
|
||
|
||
# STT loop (blocking — keeps this thread alive)
|
||
if stt_engine == "vosk":
|
||
self._listen_vosk()
|
||
else:
|
||
self._listen_whisper()
|
||
|
||
except Exception as e:
|
||
self.ui.write_log(f"ERR: Init failed — {e}")
|
||
traceback.print_exc()
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Entry
|
||
# ---------------------------------------------------------------------------
|
||
|
||
def main() -> None:
|
||
ui = JarvisUI("face.png")
|
||
|
||
def runner():
|
||
# 1. Wait until the user completes the setup overlay (first run)
|
||
# or config already exists (subsequent runs).
|
||
ui.wait_for_api_key()
|
||
|
||
# 2. Install any missing engine packages before loading engines.
|
||
# Progress is streamed to the log panel in real time.
|
||
ui.write_log("SYS: Checking dependencies…")
|
||
cfg = _load_config()
|
||
_install_done = threading.Event()
|
||
|
||
def _do_install():
|
||
try:
|
||
from core.installer import install_for_config
|
||
install_for_config(cfg, log=ui.write_log)
|
||
except Exception as e:
|
||
ui.write_log(f"ERR: Dependency install — {e}")
|
||
finally:
|
||
_install_done.set()
|
||
|
||
threading.Thread(target=_do_install, daemon=True).start()
|
||
_install_done.wait() # blocks runner thread; UI remains responsive
|
||
|
||
# 3. Start the assistant (loads STT / TTS / LLM).
|
||
jarvis = JarvisLocal(ui)
|
||
try:
|
||
jarvis.run()
|
||
except KeyboardInterrupt:
|
||
print("\n[MARK XL] Shutting down…")
|
||
|
||
threading.Thread(target=runner, daemon=True).start()
|
||
ui.root.mainloop()
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|