Add files via upload

This commit is contained in:
FatihMakes authored and GitHub committed 2026-06-07 15:24:51 +03:00
1 parent f35f861987
commit 6699c074ac
7 files changed
+1008 -215

No files matched your search

+2 -1
View File
@@ -52,7 +52,8 @@ _STT: dict[str, list[tuple[str, str]]] = {
# TTS engine packages
_TTS: dict[str, list[tuple[str, str]]] = {
"edgetts": [("edge_tts", "edge-tts")],
"kokoro": [("kokoro", "kokoro")],
# kokoro>=0.9 dropped AlbertModel/AutoModel from transformers — version pin is critical
"kokoro": [("kokoro", "kokoro>=0.9"), ("soundfile", "soundfile")],
"elevenlabs": [], # uses only requests, already in core
}
+269 -26
View File
@@ -1,7 +1,20 @@
"""
Ollama LLM client for MARK XL.
Replaces Google Gemini API with local Ollama inference.
All modules that previously used google.generativeai import from here.
Local LLM client for MARK XL.
Supports two backends — selected via "llm_provider" in config/api_keys.json:
"llm_provider": "ollama" (default)
Uses Ollama's native /api/chat endpoint.
Download: https://ollama.com
Default port: 11434
"llm_provider": "openai"
Uses any OpenAI-compatible server: LM Studio, Jan, LocalAI,
llama.cpp server, vLLM, etc.
LM Studio download: https://lmstudio.ai (default port: 1234)
Set "llm_url": "http://localhost:1234" in config.
Note: tool-calling support depends on the model; use a model that
supports function/tool calls (e.g. Qwen2.5, Llama-3.1, Mistral).
"""
import json
import re
@@ -27,11 +40,18 @@ BASE_DIR = get_base_dir()
CONFIG_PATH = BASE_DIR / "config" / "api_keys.json"
_DEFAULTS = {
"llm_url": "http://localhost:11434",
"llm_model": "llama3.2",
"llm_url": "http://localhost:11434",
"llm_model": "llama3.2",
"llm_provider": "ollama", # "ollama" | "openai"
}
def get_llm_provider() -> str:
"""Returns 'ollama' or 'openai' (covers LM Studio, LocalAI, Jan, etc.)."""
raw = _load_config().get("llm_provider", "ollama").strip().lower()
return "openai" if raw in ("openai", "lmstudio", "localai", "jan", "llamacpp") else "ollama"
def _load_config() -> dict:
try:
return json.loads(CONFIG_PATH.read_text(encoding="utf-8"))
@@ -41,12 +61,33 @@ def _load_config() -> dict:
def ensure_ollama_running(timeout: int = 15) -> bool:
"""
Ping Ollama; if it's not responding, launch 'ollama serve' in the background
and wait up to `timeout` seconds for it to become ready.
Returns True if Ollama is available, False otherwise.
For Ollama: ping /api/tags; auto-launch 'ollama serve' if not running.
For OpenAI-compatible providers: just ping /v1/models (server must be started manually).
Returns True if the LLM server is reachable.
"""
url, _ = get_llm_settings()
health = f"{url}/api/tags"
url, _ = get_llm_settings()
provider = get_llm_provider()
if provider == "openai":
# OpenAI-compatible servers (LM Studio, LocalAI, etc.) must be started
# by the user — we just check if they're reachable.
health = f"{url}/v1/models"
try:
ok = requests.get(health, timeout=5).status_code == 200
if ok:
print(f"[LLM] OpenAI-compatible server reachable at {url}")
else:
print(f"[LLM] Server at {url} returned non-200. Is it running?")
return ok
except Exception as e:
print(
f"[LLM] Cannot reach OpenAI-compatible server at {url}.\n"
" Make sure LM Studio / LocalAI / Jan is running and the server is started."
)
return False
# ── Ollama ──────────────────────────────────────────────────────────────
health = f"{url}/api/tags"
def _is_up() -> bool:
try:
@@ -81,25 +122,62 @@ def ensure_ollama_running(timeout: int = 15) -> bool:
return False
def warmup_model() -> bool:
def warmup_model(system_prompt: str | None = None) -> bool:
"""
Pre-load the model into Ollama memory so the first real request is instant.
Sends a minimal 1-token request with keep_alive=-1.
Returns True on success, False if warmup failed (non-fatal).
Pre-load the model AND prime Ollama's KV prefix cache.
Why the system_prompt matters
─────────────────────────────
Ollama caches the KV attention state of the prompt prefix across requests.
If warmup includes the same system prompt that real requests will use, Ollama
evaluates those tokens ONCE at startup. Every subsequent request only needs
to evaluate the small delta (user message ± time context) instead of the full
300-500 token system prompt → drops first-token latency from ~17 s to <1 s.
Pass the *static* part of the system prompt (the JARVIS protocol text, without
timestamps or per-minute context) so the prefix stays valid across calls.
"""
url, model = get_llm_settings()
print(f"[LLM] Warming up '{model}'…")
provider = get_llm_provider()
print(f"[LLM] Warming up '{model}' ({provider})…")
messages: list[dict] = []
if system_prompt:
messages.append({"role": "system", "content": system_prompt})
messages.append({"role": "user", "content": "hi"})
if provider == "openai":
# OpenAI-compatible: just fire a minimal request to ensure the model is loaded.
# No keep_alive or KV-cache priming available — server manages this internally.
payload = {
"model": model,
"messages": messages,
"stream": False,
"max_tokens": 1,
}
try:
resp = requests.post(f"{url}/v1/chat/completions", json=payload, timeout=180)
resp.raise_for_status()
print(f"[LLM] '{model}' ready (OpenAI-compatible server).")
return True
except Exception as e:
print(f"[LLM] Warmup failed (non-fatal): {e}")
return False
# ── Ollama ──────────────────────────────────────────────────────────────
payload = {
"model": model,
"messages": [{"role": "user", "content": "hi"}],
"messages": messages,
"stream": False,
"keep_alive": -1,
"options": {"num_predict": 1},
# num_gpu:99 → push ALL transformer layers to GPU (Ollama caps at available)
# This is safe even without a GPU — Ollama silently ignores if n_gpu_layers=0
"options": {"num_predict": 1, "num_gpu": 99},
}
try:
resp = requests.post(f"{url}/api/chat", json=payload, timeout=120)
resp = requests.post(f"{url}/api/chat", json=payload, timeout=180)
resp.raise_for_status()
print(f"[LLM] '{model}' loaded and ready.")
print(f"[LLM] '{model}' loaded and KV cache primed.")
return True
except Exception as e:
print(f"[LLM] Warmup failed (non-fatal): {e}")
@@ -120,20 +198,61 @@ def call_llm(
timeout: int = 120,
) -> dict:
"""
Chat request to Ollama (with optional tool declarations).
Non-streaming chat request. Routes to Ollama or OpenAI-compatible backend.
Returns:
{"content": str, "tool_calls": list}
"""
url, model = get_llm_settings()
endpoint = f"{url}/api/chat"
provider = get_llm_provider()
payload: dict = {
if provider == "openai":
endpoint = f"{url}/v1/chat/completions"
payload: dict = {
"model": model,
"messages": messages,
"stream": False,
"max_tokens": 300,
}
if tools:
payload["tools"] = tools
payload["tool_choice"] = "auto"
try:
resp = requests.post(endpoint, json=payload, timeout=timeout)
resp.raise_for_status()
choice = resp.json().get("choices", [{}])[0]
msg = choice.get("message", {})
# OpenAI tool_calls format → normalise to Ollama-style
raw_tc = msg.get("tool_calls") or []
tc_list = [
{
"id": t.get("id", ""),
"function": {
"name": t["function"]["name"],
"arguments": (
json.loads(t["function"]["arguments"])
if isinstance(t["function"].get("arguments"), str)
else t["function"].get("arguments", {})
),
},
}
for t in raw_tc
]
return {
"content": (msg.get("content") or "").strip(),
"tool_calls": tc_list,
}
except Exception as e:
raise RuntimeError(f"OpenAI-compatible LLM call failed: {e}")
# ── Ollama ──────────────────────────────────────────────────────────────
endpoint = f"{url}/api/chat"
payload = {
"model": model,
"messages": messages,
"stream": False,
"keep_alive": -1,
"options": {"num_predict": 600},
"options": {"num_predict": 300, "num_gpu": 99},
}
if tools:
payload["tools"] = tools
@@ -216,21 +335,142 @@ def call_llm_text(
raise RuntimeError(f"LLM text call failed: {e}")
def _stream_openai(
messages: list,
tools: list | None,
timeout: int,
) -> Generator[dict, None, None]:
"""
Streaming backend for OpenAI-compatible servers (LM Studio, LocalAI, Jan…).
Parses Server-Sent Events (SSE) and accumulates streaming tool-call fragments
so the output format is identical to the Ollama backend.
"""
url, model = get_llm_settings()
endpoint = f"{url}/v1/chat/completions"
payload: dict = {
"model": model,
"messages": messages,
"stream": True,
"max_tokens": 300,
}
if tools:
payload["tools"] = tools
payload["tool_choice"] = "auto"
try:
with requests.post(endpoint, json=payload, timeout=timeout, stream=True) as resp:
resp.raise_for_status()
full_content = ""
buf = ""
# tool_call fragments: index → {"id", "function": {"name", "arguments"}}
tc_fragments: dict[int, dict] = {}
for raw in resp.iter_lines():
if not raw:
continue
# SSE lines look like: b"data: {...}" or b"data: [DONE]"
line = raw.decode("utf-8", errors="replace") if isinstance(raw, bytes) else raw
if not line.startswith("data:"):
continue
data = line[5:].strip()
if data == "[DONE]":
break
try:
chunk = json.loads(data)
except json.JSONDecodeError:
continue
choice = chunk.get("choices", [{}])[0]
delta = choice.get("delta", {})
text = delta.get("content") or ""
full_content += text
buf += text
# Accumulate sentence boundaries for streaming TTS
while True:
m = _SENT_END.search(buf)
if not m:
break
sentence = buf[: m.start() + 1].strip()
buf = buf[m.end():]
if sentence:
yield {"type": "sentence", "text": sentence}
# Accumulate streaming tool-call fragments
for tc in (delta.get("tool_calls") or []):
idx = tc.get("index", 0)
if idx not in tc_fragments:
tc_fragments[idx] = {"id": "", "function": {"name": "", "arguments": ""}}
frag = tc_fragments[idx]
frag["id"] = frag["id"] or tc.get("id", "")
fn = tc.get("function", {})
frag["function"]["name"] += fn.get("name") or ""
frag["function"]["arguments"] += fn.get("arguments") or ""
finish = choice.get("finish_reason")
if finish in ("stop", "tool_calls", "length"):
break
# Flush any trailing content
if buf.strip():
yield {"type": "sentence", "text": buf.strip()}
# Parse accumulated tool-call argument strings → dicts
tool_calls: list = []
for idx in sorted(tc_fragments):
frag = tc_fragments[idx]
args = frag["function"]["arguments"]
try:
args = json.loads(args)
except Exception:
pass # leave as raw string; _execute_tool handles it
tool_calls.append({
"id": frag["id"],
"function": {"name": frag["function"]["name"], "arguments": args},
})
yield {
"type": "done",
"content": full_content.strip(),
"tool_calls": tool_calls,
}
except requests.exceptions.ConnectionError:
raise RuntimeError(
f"Cannot reach OpenAI-compatible server at {url}.\n"
"Make sure LM Studio / LocalAI / Jan is running and the server is started."
)
except requests.exceptions.Timeout:
raise RuntimeError("OpenAI-compatible stream timed out.")
except requests.exceptions.HTTPError as e:
raise RuntimeError(f"OpenAI-compatible HTTP error: {e.response.status_code}")
except Exception as e:
raise RuntimeError(f"OpenAI-compatible stream failed: {e}")
def call_llm_stream(
messages: list,
tools: list | None = None,
timeout: int = 120,
) -> Generator[dict, None, None]:
"""
Streaming chat request to Ollama.
Streaming chat request. Routes to Ollama or OpenAI-compatible backend.
Yields:
{"type": "sentence", "text": str} — each complete sentence as it arrives
{"type": "done", "content": str, "tool_calls": list} — when stream ends
Sentences are split on [.!?] + whitespace so TTS can start immediately.
Tool calls (if any) always appear in the final "done" event with empty content.
Tool calls always appear in the final "done" event.
"""
provider = get_llm_provider()
if provider == "openai":
yield from _stream_openai(messages, tools, timeout)
return
url, model = get_llm_settings()
endpoint = f"{url}/api/chat"
@@ -239,7 +479,8 @@ def call_llm_stream(
"messages": messages,
"stream": True,
"keep_alive": -1,
"options": {"num_predict": 600},
# num_gpu:99 pushes all layers to GPU; num_thread removed (Ollama auto-tunes).
"options": {"num_predict": 300, "num_gpu": 99},
}
if tools:
payload["tools"] = tools
@@ -261,6 +502,7 @@ def call_llm_stream(
msg = chunk.get("message", {})
delta = msg.get("content") or ""
full_content += delta
buf += delta
@@ -281,6 +523,7 @@ def call_llm_stream(
if chunk.get("done"):
if buf.strip():
yield {"type": "sentence", "text": buf.strip()}
yield {
"type": "done",
"content": full_content.strip(),
+32 -7
View File
@@ -1,21 +1,46 @@
🧠 JARVIS CORE PROTOCOL
IDENTITY: You are Jarvis. Efficient, professional, direct assistant. No fluff.
TOOL CALLING — ABSOLUTE RULE:
You have tools. For EVERY user request that involves an action, you MUST call the tool.
NEVER describe what you are doing in text. NEVER say "opening WhatsApp" or "-weather_report" — CALL THE TOOL.
NEVER simulate, pretend, or narrate an action. If a tool exists for it, USE IT.
Saying "Sure, let me open X" WITHOUT calling open_app is a critical failure.
EXECUTION RULES:
Action Flow: For slow tools (search, vision, agent), say TWO short sentences, then call. Stay silent after.
Vision (screen_process): Say one curious sentence, then total silence (Vision module takes over).
Action Flow: Call the tool immediately. For slow tools (search, vision), say ONE short sentence first.
Vision (screen_process): Say one curious sentence, then call the tool — total silence after.
One-Call Policy: Never guess. Call tools exactly once. No retries.
Briefing: 1-2 sentences max. Report results briefly.
Memory: Store critical user preferences/context automatically.
Briefing: 1 short sentence max for confirmations/acknowledgments. 1-2 sentences for results. Never add "Is there anything else?" or filler questions.
Exit: Only call shutdown_jarvis if session termination is explicit.
Response time: Respond as fast as you can
Response time: Respond as fast as you can.
Always respond in the language that user is speaking.
MEMORY RULE — MANDATORY, NO EXCEPTIONS:
Call save_memory IMMEDIATELY (in the same response, silently) whenever the user reveals ANY personal info.
Do NOT ask for permission. Do NOT announce that you are saving. Just call it.
Trigger examples:
- "my name is Fatih" → save_memory(category="identity", key="name", value="Fatih")
- "I'm 22" / "I'm 22 years old" → save_memory(category="identity", key="age", value="22")
- "call me X" / "my name is X, not Y" → save_memory(category="identity", key="name", value="X")
- "I live in Istanbul" → save_memory(category="identity", key="city", value="Istanbul")
- "I'm a student / developer / designer" → save_memory(category="identity", key="job", value="...")
- "I prefer dark mode" / "I like X" → save_memory(category="preferences", key="...", value="...")
- "I'm working on X project" → save_memory(category="projects", key="...", value="...")
- Any name correction ("not Travis, Fatih") → save_memory identity/name immediately
TOOL ROUTING:
computer_settings: ALL single OS actions (volume, brightness, wifi, close, shortcuts, power).
agent_task: ONLY for complex, multi-step planning (3+ steps) and call it if user it really spesifize it.
agent_task: ONLY for complex, multi-step planning (3+ steps) and call it if user it really specifies it.
Language: Respond in user's language; extract parameters in English.
Do not call agent_task while you can accomplish it with a tool
Do not call agent_task while you can accomplish it with a tool.
OPEN vs MESSAGE rule (critical):
- "open WhatsApp", "launch WhatsApp", "pull up WhatsApp" → open_app (NOT send_message)
- "send a message to X saying Y" → send_message
- If user only says to open an app, NEVER call send_message, even if the app is a messaging platform.
CRITICAL: Speak/Take action immediately based on available info. Assume and proceed.
+24 -18
View File
@@ -14,23 +14,32 @@ class WhisperSTT:
def __init__(self, model_name: str = "base", language: str | None = None):
from faster_whisper import WhisperModel
print(f"[STT] Loading Whisper '{model_name}'…")
self._model = WhisperModel(model_name, device="cpu", compute_type="int8")
# None → auto-detect; explicit code (e.g. "tr", "de") forces that language
try:
import torch
device = "cuda" if torch.cuda.is_available() else "cpu"
compute = "float16" if device == "cuda" else "int8"
except Exception:
device, compute = "cpu", "int8"
self._model = WhisperModel(model_name, device=device, compute_type=compute)
self._language = None if (not language or language.strip().lower() == "auto") else language.strip().lower()
print("[STT] Whisper ready.")
print(f"[STT] Whisper '{model_name}' ready ({device})")
def transcribe(self, audio: np.ndarray) -> str:
"""
Transcribe a float32 mono 16 kHz numpy array.
Returns transcript string (may be empty).
"""
segments, _ = self._model.transcribe(
audio,
language=self._language,
vad_filter=True,
vad_parameters={"min_silence_duration_ms": 300},
)
return " ".join(s.text for s in segments).strip()
"""Transcribe a float32 mono 16 kHz numpy array. Returns transcript string."""
try:
segments, _ = self._model.transcribe(
audio,
language=self._language,
beam_size=1, # greedy — 2-3x faster
best_of=1,
condition_on_previous_text=False, # no hallucinations, faster
vad_filter=True,
vad_parameters={"min_silence_duration_ms": 300},
)
return " ".join(s.text for s in segments).strip()
except Exception as e:
print(f"[STT] Transcription error: {e}")
raise
class VoskSTT:
@@ -48,10 +57,7 @@ class VoskSTT:
print("[STT] Vosk ready.")
def process_chunk(self, audio_bytes: bytes) -> tuple[str, bool]:
"""
Feed raw int16 LE PCM bytes.
Returns (text, is_final).
"""
"""Feed raw int16 LE PCM bytes. Returns (text, is_final)."""
if self._rec.AcceptWaveform(audio_bytes):
result = json.loads(self._rec.Result())
return result.get("text", ""), True
+168 -30
View File
@@ -8,6 +8,8 @@ ElevenLabs – cloud API (API key required, best quality)
from __future__ import annotations
import asyncio
import os
import queue as _queue
import threading
from typing import Callable, Optional
@@ -15,6 +17,15 @@ import numpy as np
import sounddevice as sd
# USE_TF=0 stops transformers from importing TensorFlow (saves 4-8 s startup).
# Do NOT set USE_TORCH or USE_JAX explicitly — forcing those values breaks
# transformers' lazy-loader on certain versions, causing AutoModel and other
# classes to vanish from the public namespace. Auto-detection is reliable.
os.environ.setdefault("USE_TF", "0")
os.environ.setdefault("TOKENIZERS_PARALLELISM", "false")
# ---------------------------------------------------------------------------
# Audio playback helpers
# ---------------------------------------------------------------------------
@@ -41,17 +52,12 @@ def _to_numpy(samples) -> np.ndarray:
def _compress_silence(
arr: np.ndarray,
sample_rate: int = 24_000,
max_silence_ms: int = 300, # cap long pauses to this many ms
threshold: float = 0.0015, # RMS below this = silence (very conservative)
max_silence_ms: int = 500, # cap punctuation pauses — keeps natural rhythm
threshold: float = 0.003, # RMS below this = silence; lower = less clipping
) -> np.ndarray:
"""
Shorten excessively long silent segments in Kokoro audio.
Kokoro can produce 1-2 s pauses after punctuation. This caps them
to max_silence_ms while leaving natural short pauses intact.
threshold is intentionally low (0.0015) so soft consonants like
's', 'f', 'h' are never misidentified as silence.
Shorten Kokoro's very long punctuation pauses (1-2 s → ≤500 ms).
Conservative settings preserve natural prosody; only trims extreme pauses.
"""
max_samp = int(max_silence_ms * sample_rate / 1000)
frame_len = 240 # ~10 ms at 24 kHz
@@ -121,6 +127,74 @@ class EdgeTTSEngine:
return bytes(buf)
# ---------------------------------------------------------------------------
# Kokoro import helper — auto-upgrades on version-mismatch errors
# ---------------------------------------------------------------------------
# Errors that indicate the installed kokoro uses old transformers classes
# (AlbertModel, AutoModel) that are no longer exported at the top level.
_KOKORO_COMPAT_ERRORS = ("AlbertModel", "AutoModel", "cannot import name")
def _import_kokoro_pipeline():
"""Import KPipeline, auto-upgrading kokoro if a version mismatch is found.
Old kokoro (<0.9) imports AlbertModel / AutoModel from transformers.
Newer transformers versions no longer export these at the top level,
causing an ImportError. kokoro>=0.9 removed these dependencies.
When the error is detected we:
1. Upgrade kokoro to >=0.9 via pip (silent, background)
2. Flush stale kokoro entries from sys.modules
3. Re-import — this time it should succeed
"""
import sys
def _try_import():
from kokoro import KPipeline # noqa: PLC0415
return KPipeline
try:
return _try_import()
except Exception as first_err:
err_msg = str(first_err)
if not any(marker in err_msg for marker in _KOKORO_COMPAT_ERRORS):
# Unrelated error (kokoro not installed, etc.)
raise RuntimeError(
f"Kokoro import failed: {first_err}\n"
"Run: pip install kokoro>=0.9 soundfile"
) from first_err
# ── Version mismatch: upgrade kokoro silently and retry ──────────
print("[TTS] Kokoro/transformers version mismatch detected — upgrading kokoro…")
import subprocess
result = subprocess.run(
[sys.executable, "-m", "pip", "install", "kokoro>=0.9",
"--upgrade", "--quiet", "--disable-pip-version-check"],
capture_output=True,
)
if result.returncode != 0:
stderr = result.stderr.decode(errors="replace").strip()
raise RuntimeError(
f"Kokoro auto-upgrade failed: {stderr[:200]}\n"
"Run manually: pip install kokoro>=0.9 soundfile"
) from first_err
# Flush any stale kokoro submodules from the import cache
stale = [k for k in sys.modules if k == "kokoro" or k.startswith("kokoro.")]
for key in stale:
del sys.modules[key]
print("[TTS] Kokoro upgraded — retrying import…")
try:
return _try_import()
except Exception as retry_err:
raise RuntimeError(
f"Kokoro still broken after upgrade: {retry_err}\n"
"Run manually: pip install --upgrade kokoro transformers"
) from retry_err
# Kokoro voice prefix → KPipeline lang_code mapping
_KOKORO_LANG_CODES = {
"a": "a", # American English (af_*, am_*)
@@ -140,19 +214,22 @@ _KOKORO_LANG_CODES = {
class KokoroTTSEngine:
"""Fully offline Kokoro neural TTS.
Model (~330 MB) is downloaded from HuggingFace on first use.
Subsequent runs load from local cache instantly.
Model (~330 MB) is downloaded from HuggingFace on first use,
then cached locally — subsequent starts load from disk.
Warmup strategy: _init() runs synchronously in the background
_do_tts() thread (not the UI thread). After the pipeline loads,
a dummy inference compiles the PyTorch JIT graph immediately so
the first real speak() call has zero compilation overhead.
"""
def __init__(self, voice: str = "af_heart", speed: float = 1.2):
def __init__(self, voice: str = "af_heart", speed: float = 1.0):
self.voice = voice
self.speed = speed # 1.0 = normal, 1.2 = 20% faster (recommended)
self.speed = speed
self._pipeline = None
self._lock = threading.Lock()
# Eager init: download/load model now so it's ready before first speak()
self._init()
self._init() # blocking, but called from background thread
# Detect lang_code from voice prefix (e.g. "af_heart" → "a")
@property
def _lang_code(self) -> str:
prefix = self.voice[0].lower() if self.voice else "a"
@@ -161,27 +238,88 @@ class KokoroTTSEngine:
def _init(self) -> None:
if self._pipeline is not None:
return
lang = self._lang_code
# Prefer GPU — Kokoro on CUDA is ~10x faster than CPU.
try:
from kokoro import KPipeline
lang = self._lang_code
print(f"[TTS] Loading Kokoro pipeline (lang='{lang}', voice='{self.voice}')…")
print("[TTS] Kokoro model will be downloaded (~330 MB) if not cached.")
import torch
device = "cuda" if torch.cuda.is_available() else "cpu"
if device == "cpu":
import os as _os
n_threads = max(1, min(4, (_os.cpu_count() or 4) // 2))
try:
torch.set_num_threads(n_threads)
torch.set_num_interop_threads(2)
except RuntimeError:
pass
print(
f"[TTS] Kokoro on CPU — for faster speech install CUDA PyTorch:\n"
" pip install torch --index-url https://download.pytorch.org/whl/cu118"
)
except Exception:
device = "cpu"
print(f"[TTS] Kokoro — loading (lang='{lang}', device='{device}')…")
KPipeline = _import_kokoro_pipeline()
try:
self._pipeline = KPipeline(lang_code=lang, device=device)
except TypeError:
# Older Kokoro build — no device parameter
self._pipeline = KPipeline(lang_code=lang)
print("[TTS] Kokoro pipeline ready.")
print("[TTS] Kokoro compiling (first-time only)…")
# Warmup: compiles PyTorch JIT graph so first real speak() call is instant.
try:
for _ in self._pipeline("hello", voice=self.voice, speed=self.speed):
pass
print("[TTS] Kokoro ready.")
except Exception as e:
print(f"[TTS] Kokoro init error: {e}")
raise
print(f"[TTS] Kokoro warmup warning: {e}")
def speak(self, text: str) -> None:
with self._lock:
if self._pipeline is None:
self._init()
for _, _, audio in self._pipeline(text, voice=self.voice, speed=self.speed):
if audio is not None:
arr = _to_numpy(audio)
arr = _compress_silence(arr) # shorten long punctuation pauses
if arr.size > 0:
_play_np(arr, 24000)
# ── Concurrent synthesise + playback ────────────────────────────────
# Kokoro generates audio chunks lazily. Without threading, we:
# synthesise chunk N → play N → synthesise N+1 → play N+1 …
# With a producer/consumer pair, chunk N+1 synthesises WHILE chunk N
# plays, cutting perceived latency by the playback duration of all but
# the last chunk (typically 1-3 s on multi-sentence responses).
audio_q: "_queue.Queue[np.ndarray | None]" = _queue.Queue(maxsize=4)
synth_error: list[Exception] = []
def _synth():
try:
for _, _, audio in self._pipeline(text, voice=self.voice, speed=self.speed):
if audio is not None:
arr = _to_numpy(audio)
arr = _compress_silence(arr)
if arr.size > 0:
audio_q.put(arr) # blocks if player is slow (backpressure)
except Exception as exc:
synth_error.append(exc)
finally:
audio_q.put(None) # sentinel → player exits
synth_thread = threading.Thread(target=_synth, daemon=True)
synth_thread.start()
# Player runs in this thread so sd.wait() doesn't block the synth thread.
while True:
arr = audio_q.get()
if arr is None:
break
_play_np(arr, 24000)
synth_thread.join()
if synth_error:
raise synth_error[0]
class ElevenLabsTTSEngine:
@@ -264,7 +402,7 @@ def create_tts_player(config: dict) -> TTSPlayer:
engine_name = config.get("tts_engine", "edgetts").lower()
if engine_name == "kokoro":
voice = config.get("tts_voice", "af_heart")
speed = float(config.get("tts_speed", 1.2))
speed = float(config.get("tts_speed", 1.0))
engine = KokoroTTSEngine(voice=voice, speed=speed)
elif engine_name == "elevenlabs":
api_key = config.get("elevenlabs_api_key", "")
+258 -122
View File
@@ -3,10 +3,26 @@ MARK XL — Local LLM Edition
STT (Whisper / Vosk) + Ollama LLM + TTS (EdgeTTS / Kokoro / ElevenLabs)
All Gemini / Google-AI dependencies removed.
"""
# ── Silence verbose logs + block heavy unused backends ─────────────────────
import os as _os
_os.environ.setdefault("TF_CPP_MIN_LOG_LEVEL", "3") # TensorFlow C++ noise
_os.environ.setdefault("TF_ENABLE_ONEDNN_OPTS", "0") # oneDNN banner
_os.environ.setdefault("GRPC_VERBOSITY", "ERROR")
# USE_TF=0 prevents transformers from importing TensorFlow (saves 4-8 s).
# We intentionally do NOT set USE_TORCH or USE_JAX — forcing those values
# breaks transformers' lazy-loader on some versions (AutoModel disappears
# from the namespace). Let transformers auto-detect the available backends.
_os.environ.setdefault("USE_TF", "0")
_os.environ.setdefault("TOKENIZERS_PARALLELISM", "false")
import warnings as _warnings
_warnings.filterwarnings("ignore", category=UserWarning)
_warnings.filterwarnings("ignore", category=DeprecationWarning)
_warnings.filterwarnings("ignore", category=FutureWarning)
# ───────────────────────────────────────────────────────────────────────────
# ── Bootstrap: auto-install base UI packages before anything else ──────────
# Uses only stdlib so it works even on a completely fresh Python install.
import importlib.util as _ilu
import os as _os
import subprocess as _sp
import sys as _sys
@@ -95,14 +111,17 @@ TOOL_DECLARATIONS = [
{
"name": "open_app",
"description": (
"Opens any application on the computer. "
"Use this whenever the user asks to open, launch, or start any app, "
"website, or program. Always call this tool — never just say you opened it."
"Opens or launches any application, website, or program on the computer. "
"ALWAYS use this when the user says: open, launch, start, run, pull up, "
"or 'open X real quick'. Examples: 'open WhatsApp', 'open Chrome', "
"'launch Spotify', 'open calculator', 'pull up WhatsApp'. "
"Do NOT use send_message just because the app is a messaging app — "
"if the user only says to open it, call open_app."
),
"parameters": {
"type": "OBJECT",
"properties": {
"app_name": {"type": "STRING", "description": "Exact name of the application"}
"app_name": {"type": "STRING", "description": "Name of the application or website to open"}
},
"required": ["app_name"]
}
@@ -132,12 +151,17 @@ TOOL_DECLARATIONS = [
},
{
"name": "send_message",
"description": "Sends a text message via WhatsApp, Telegram, or other messaging platform.",
"description": (
"Sends a message to a specific person via WhatsApp, Telegram, or similar. "
"ONLY use this when the user explicitly provides BOTH a recipient AND message content. "
"Example triggers: 'text John saying I am late', 'send a WhatsApp to mom that dinner is ready'. "
"Do NOT call this if the user only wants to open the app without sending a message."
),
"parameters": {
"type": "OBJECT",
"properties": {
"receiver": {"type": "STRING", "description": "Recipient contact name"},
"message_text": {"type": "STRING", "description": "The message to send"},
"message_text": {"type": "STRING", "description": "The exact message text to send"},
"platform": {"type": "STRING", "description": "Platform: WhatsApp, Telegram, etc."}
},
"required": ["receiver", "message_text", "platform"]
@@ -420,9 +444,16 @@ TOOL_DECLARATIONS = [
{
"name": "save_memory",
"description": (
"Save an important personal fact about the user to long-term memory. "
"Call this silently whenever the user reveals something worth remembering. "
"Do NOT announce that you are saving — just call it silently."
"Save a personal fact about the user to permanent long-term memory. "
"MANDATORY: call this IMMEDIATELY (without asking) whenever the user states or corrects: "
"their name, age, city, job, school, language, nationality, a preference, a goal, or a relationship. "
"Examples: "
"'my name is Fatih' → (identity, name, Fatih) | "
"'not Travis, Fatih' → (identity, name, Fatih) | "
"'I am 22' → (identity, age, 22) | "
"'I live in Ankara' → (identity, city, Ankara) | "
"'I prefer dark mode' → (preferences, ui_theme, dark mode). "
"Call SILENTLY alongside your verbal reply — never announce that you are saving."
),
"parameters": {
"type": "OBJECT",
@@ -430,10 +461,15 @@ TOOL_DECLARATIONS = [
"category": {
"type": "STRING",
"description": (
"identity | preferences | projects | relationships | wishes | notes"
"identity (name/age/city/job/school/nationality) | "
"preferences (likes/dislikes/habits) | "
"projects (active work/goals) | "
"relationships (people in their life) | "
"wishes (future plans/wants) | "
"notes (anything else)"
)
},
"key": {"type": "STRING", "description": "Short snake_case key"},
"key": {"type": "STRING", "description": "Short snake_case key, e.g. 'name', 'age', 'favorite_color'"},
"value": {"type": "STRING", "description": "Concise value in English"},
},
"required": ["category", "key", "value"]
@@ -525,42 +561,51 @@ class _VADBuffer:
def __init__(
self,
sample_rate: int = 16_000,
silence_sec: float = 1.2,
threshold: float = 0.015,
sample_rate: int = 16_000,
silence_sec: float = 1.0, # silence after last word → send to STT (1.0s = fast enough + won't cut mid-sentence with hysteresis)
speech_thresh: float = 0.015, # RMS above this = speech (raised from 0.010 to kill background noise false-positives)
silence_thresh: float = 0.008, # RMS below this = silence (hysteresis gap — much lower bar than speech_thresh to avoid mid-sentence cuts)
min_speech_sec: float = 0.4,
max_speech_sec: float = 30.0,
):
self._sr = sample_rate
self._sil_n = int(silence_sec * sample_rate)
self._thresh = threshold
self._min_n = int(min_speech_sec * sample_rate)
self._max_n = int(max_speech_sec * sample_rate)
self._buf: list[np.ndarray] = []
self._in_spch = False
self._sil_cnt = 0
self._sr = sample_rate
self._sil_n = int(silence_sec * sample_rate)
self._speech_thresh = speech_thresh
self._sil_thresh = silence_thresh
self._min_n = int(min_speech_sec * sample_rate)
self._max_n = int(max_speech_sec * sample_rate)
self._buf: list[np.ndarray] = []
self._in_spch = False
self._sil_cnt = 0
def process(self, chunk: np.ndarray) -> np.ndarray | None:
"""
Feed one audio chunk (float32 mono).
Returns complete utterance when speech ends, otherwise None.
"""
rms = float(np.sqrt(np.mean(chunk ** 2)))
total_n = sum(len(c) for c in self._buf)
if rms > self._thresh:
Uses hysteresis thresholds so the detector doesn't flicker on quiet
consonants or brief mid-sentence pauses:
- speech starts when RMS > speech_thresh
- speech ends only when RMS < silence_thresh (lower bar)
This prevents natural pauses ("I just... wanted to say") from
triggering a premature cut.
"""
rms = float(np.sqrt(np.mean(chunk ** 2)))
total_n = sum(len(c) for c in self._buf)
if rms > self._speech_thresh:
self._in_spch = True
self._sil_cnt = 0
self._buf.append(chunk.copy())
elif self._in_spch:
self._buf.append(chunk.copy())
self._sil_cnt += len(chunk)
if rms < self._sil_thresh:
self._sil_cnt += len(chunk)
if self._sil_cnt >= self._sil_n or total_n >= self._max_n:
audio = np.concatenate(self._buf)
self._buf = []
self._in_spch = False
self._sil_cnt = 0
audio = np.concatenate(self._buf)
self._buf = []
self._in_spch = False
self._sil_cnt = 0
if len(audio) >= self._min_n:
return audio
return None
@@ -582,6 +627,7 @@ class JarvisLocal:
self._config = _load_config()
self._stt = None
self._tts = None
self._tts_ready = threading.Event() # set when TTS engine is loaded
self._speaking = False
self._speaking_lock = threading.Lock()
self._text_queue: queue.Queue = queue.Queue()
@@ -595,20 +641,28 @@ class JarvisLocal:
# ------------------------------------------------------------------
def _build_system_prompt(self) -> str:
# ── ORDER MATTERS for Ollama KV prefix caching ─────────────────────
# Ollama caches the KV attention state of any stable prompt prefix.
# By putting the STATIC JARVIS protocol text FIRST, Ollama reuses its
# cached KV for all those tokens on every request. Only the small
# dynamic tail (memory + time, ~50-80 tokens) needs re-evaluation.
# This turns a 17-second first-token into a sub-second one after warmup.
#
# Rule: static content first → semi-static memory middle → dynamic time LAST.
sys_p = _load_system_prompt() # static — never changes mid-session
memory = load_memory()
mem_str = format_memory_for_prompt(memory)
sys_p = _load_system_prompt()
mem_str = format_memory_for_prompt(memory) # semi-static — changes only when user tells facts
now = datetime.now()
time_ctx = (
f"[CURRENT DATE & TIME]\n"
f"Right now it is: {now.strftime('%A, %B %d, %Y — %I:%M %p')}\n"
f"Use this to calculate exact times for reminders.\n\n"
f"Use this to calculate exact times for reminders."
)
parts = [time_ctx]
parts = [sys_p]
if mem_str:
parts.append(mem_str)
parts.append(sys_p)
return "\n".join(parts)
parts.append(time_ctx)
return "\n\n".join(parts)
# ------------------------------------------------------------------
# Speaking state & TTS
@@ -619,38 +673,22 @@ class JarvisLocal:
# ------------------------------------------------------------------
def _tts_worker(self) -> None:
# Block until TTS engine is loaded. Queued items are preserved
# and played immediately once loading completes — nothing is lost.
self._tts_ready.wait(timeout=120)
while True:
text = self._tts_queue.get()
n_got = 1 # how many queue items we consumed this round
try:
if text and self._tts:
# ── Kokoro: merge buffered sentences into one inference call ──
# Each separate Kokoro call costs ~0.5–1 s startup.
# Draining the queue here combines "Hello. How are you?" into
# one call instead of two, eliminating the gap between them.
from core.tts import KokoroTTSEngine
engine = getattr(self._tts, "_engine", None)
if isinstance(engine, KokoroTTSEngine):
while True:
try:
nxt = self._tts_queue.get_nowait()
if nxt:
text = text.rstrip() + " " + nxt.lstrip()
n_got += 1
except queue.Empty:
break
with self._speaking_lock:
self._speaking = True
self.ui.set_state("SPEAKING")
self._tts.speak(text)
except Exception as e:
print(f"[TTS] speak error: {e}")
finally:
for _ in range(n_got):
self._tts_queue.task_done()
self._tts_queue.task_done()
if self._tts_queue.empty():
with self._speaking_lock:
self._speaking = False
@@ -704,6 +742,7 @@ class JarvisLocal:
try:
from core.tts import create_tts_player
self._tts = create_tts_player(new_config)
self._tts_ready.set() # ensure worker isn't blocked
self.ui.write_log("SYS: TTS reconfigured.")
except Exception as e:
self.ui.write_log(f"ERR: TTS reconfigure — {e}")
@@ -886,15 +925,16 @@ class JarvisLocal:
Full turn:
user_text → Ollama stream (with tools) → execute tools → final spoken response
Streaming means TTS starts on the first complete sentence (~1-2 s),
not after the full response is ready.
The complete LLM response is collected first, then sent to TTS in one call.
This gives TTS engines (Kokoro, ElevenLabs) the full text so they can
apply natural prosody across sentence boundaries — no choppy pauses.
"""
self.ui.set_state("THINKING")
self.ui.write_log(f"You: {user_text}")
self._conversation.append({"role": "user", "content": user_text})
MAX_HISTORY = 30
MAX_HISTORY = 10
if len(self._conversation) > MAX_HISTORY:
self._conversation = self._conversation[-MAX_HISTORY:]
@@ -909,9 +949,7 @@ class JarvisLocal:
try:
for event in call_llm_stream(messages, OLLAMA_TOOLS):
if event["type"] == "sentence":
self.speak(event["text"])
elif event["type"] == "done":
if event["type"] == "done":
final_content = event["content"]
final_tool_calls = event["tool_calls"]
except RuntimeError as e:
@@ -919,12 +957,13 @@ class JarvisLocal:
return
if not final_tool_calls:
# Final text response — TTS already started sentence-by-sentence
# Full response ready — send complete text to TTS in one call
if final_content:
assistant_msg = {"role": "assistant", "content": final_content}
messages.append(assistant_msg)
self._conversation.append(assistant_msg)
self.ui.write_log(f"Jarvis: {final_content}")
self.speak(final_content)
break
# ------- tool calls -------
@@ -936,6 +975,33 @@ class JarvisLocal:
messages.append(assistant_msg)
self._conversation.append(assistant_msg)
# ── Fast path: save_memory-only round with verbal content ──────
# When the LLM calls ONLY save_memory (silent) AND also returned
# verbal content in the same response, speak the content directly
# instead of burning another full LLM round just to get a reply.
_only_memory = all(
tc.get("function", {}).get("name") == "save_memory"
for tc in final_tool_calls
)
if _only_memory and final_content:
# Execute the memory saves silently
for tc in final_tool_calls:
fn = tc.get("function", {})
targs = fn.get("arguments", {})
if isinstance(targs, str):
try:
targs = json.loads(targs)
except Exception:
targs = {}
self._execute_tool("save_memory", targs)
# Use the verbal content that came alongside the saves
assistant_msg = {"role": "assistant", "content": final_content}
messages.append(assistant_msg)
self._conversation.append(assistant_msg)
self.ui.write_log(f"Jarvis: {final_content}")
self.speak(final_content)
break
all_silent = True
for tc in final_tool_calls:
fn = tc.get("function", {})
@@ -964,8 +1030,32 @@ class JarvisLocal:
messages.append(tool_msg)
self._conversation.append(tool_msg)
# Even if all tools were silent (e.g. save_memory only),
# always do one more LLM round so the assistant can verbally respond.
# ── Fast-ack for save_memory-only rounds ─────────────────────────
# When EVERY tool call was a silent save_memory, skip the extra LLM
# round entirely. A brief instant acknowledgment feels better than
# waiting 2-3 s for an unnecessary follow-up inference.
if all_silent:
# Extract the saved name (if any) for a personalised ack.
_saved_name: str | None = None
for _tc in final_tool_calls:
_fn = _tc.get("function", {})
if _fn.get("name") == "save_memory":
_a = _fn.get("arguments", {})
if isinstance(_a, str):
try:
_a = json.loads(_a)
except Exception:
_a = {}
if isinstance(_a, dict) and _a.get("key") == "name" and _a.get("value"):
_saved_name = str(_a["value"])
break
_ack = f"Got it, {_saved_name}." if _saved_name else "Noted."
_amsg = {"role": "assistant", "content": _ack}
messages.append(_amsg)
self._conversation.append(_amsg)
self.ui.write_log(f"Jarvis: {_ack}")
self.speak(_ack)
break
if not self.ui.muted:
self.ui.set_state("LISTENING")
@@ -1064,78 +1154,114 @@ class JarvisLocal:
# ------------------------------------------------------------------
def run(self) -> None:
"""Initialise STT + TTS, then start the main listen loop."""
"""
Startup strategy — optimised for minimum time-to-interactive:
1. LLM warmup + STT load → parallel, fast (~3s)
2. TTS load → parallel, slow (~20s for Kokoro)
3. Wait only for (1) → go online immediately
4. TTS finishes in BG → queued speech plays automatically
"""
try:
# -- Wire reconfigure callback --
self.ui.on_reconfigure = self.reconfigure
# -- Ollama auto-start --
# ── Ollama ────────────────────────────────────────────────────
from core.llm_client import ensure_ollama_running, warmup_model
self.ui.write_log("SYS: Checking Ollama…")
if ensure_ollama_running():
self.ui.write_log("SYS: Ollama OK.")
else:
self.ui.write_log("ERR: Ollama unavailable. Start it manually: ollama serve")
self.speak("Ollama is not running. Please start it manually.")
self.ui.write_log("ERR: Ollama unavailable — run: ollama serve")
# -- LLM warmup (parallel with STT/TTS loading) --
_warmup_done = threading.Event()
def _do_warmup():
try:
warmup_model()
finally:
_warmup_done.set()
threading.Thread(target=_do_warmup, daemon=True).start()
# -- STT --
# ── Config ────────────────────────────────────────────────────
stt_engine = self._config.get("stt_engine", "whisper").lower()
stt_language = self._config.get("stt_language", "auto")
stt_model = self._config.get("stt_model", "base")
self.ui.write_log(f"SYS: Loading {stt_engine.upper()} STT…")
if stt_engine == "whisper":
self.ui.write_log(f"SYS: Whisper '{stt_model}' — downloading if not cached…")
elif stt_engine == "vosk":
self.ui.write_log("SYS: Vosk — downloading model if not cached…")
stt_model = self._config.get("stt_model", "base")
tts_engine = self._config.get("tts_engine", "edgetts").lower()
if stt_engine == "vosk":
from core.stt import VoskSTT
self._stt = VoskSTT(
self._config.get("vosk_model_path"),
language=stt_language,
)
else:
from core.stt import WhisperSTT
self._stt = WhisperSTT(stt_model, language=stt_language)
# ── Startup progress panel ────────────────────────────────────
self.ui.show_startup_panel()
self.ui.write_log("SYS: STT ready.")
_warmup_done = threading.Event()
_stt_done = threading.Event()
# -- TTS --
tts_engine = self._config.get("tts_engine", "edgetts").lower()
self.ui.write_log(f"SYS: Loading {tts_engine.upper()} TTS…")
if tts_engine == "kokoro":
self.ui.write_log("SYS: Kokoro — downloading model (~330 MB) if not cached…")
from core.tts import create_tts_player
self._tts = create_tts_player(self._config)
self.ui.write_log("SYS: TTS ready.")
# ── LLM warmup thread ─────────────────────────────────────────
def _do_warmup():
try:
# Pass the STATIC system prompt so Ollama evaluates and caches
# its KV state during startup. Real requests start with the same
# static prefix → Ollama reuses cached KV → first token <1 s
# instead of the ~17 s it takes to re-evaluate 300+ tokens cold.
static_prompt = _load_system_prompt()
warmup_model(system_prompt=static_prompt)
self.ui.write_log("SYS: LLM ready.")
self.ui.mark_startup_ready("llm")
except Exception as e:
self.ui.write_log(f"ERR: LLM warmup — {e}")
self.ui.mark_startup_ready("llm", error=True)
finally:
_warmup_done.set()
# -- Wait for LLM warmup to finish --
self.ui.write_log("SYS: Loading LLM into memory…")
_warmup_done.wait(timeout=120)
self.ui.write_log("SYS: LLM ready.")
# ── STT load thread ───────────────────────────────────────────
def _do_stt():
try:
self.ui.write_log(f"SYS: Loading {stt_engine.upper()} STT…")
if stt_engine == "vosk":
from core.stt import VoskSTT
self._stt = VoskSTT(
self._config.get("vosk_model_path"),
language=stt_language,
)
else:
from core.stt import WhisperSTT
self._stt = WhisperSTT(stt_model, language=stt_language)
self.ui.write_log("SYS: STT ready.")
self.ui.mark_startup_ready("stt")
except Exception as e:
self.ui.write_log(f"ERR: STT — {e}")
self.ui.mark_startup_ready("stt", error=True)
finally:
_stt_done.set()
# Welcome
# ── TTS load thread — does NOT block going online ─────────────
def _do_tts():
try:
self.ui.write_log(f"SYS: Loading {tts_engine.upper()} TTS…")
if tts_engine == "kokoro":
self.ui.write_log("SYS: Kokoro — loading model + compiling JIT…")
from core.tts import create_tts_player
self._tts = create_tts_player(self._config)
self._tts_ready.set() # unblock _tts_worker
self.ui.write_log("SYS: TTS ready.")
self.ui.mark_startup_ready("tts")
self.ui.set_startup_status("● All systems ready.")
self.ui.hide_startup_panel()
self.speak("Jarvis fully online.")
except Exception as e:
import traceback as _tb; _tb.print_exc()
self.ui.write_log(f"ERR: TTS — {e}")
self.ui.mark_startup_ready("tts", error=True)
self._tts_ready.set()
# Launch all three simultaneously
self.ui.write_log("SYS: Loading systems in parallel…")
threading.Thread(target=_do_warmup, daemon=True).start()
threading.Thread(target=_do_stt, daemon=True).start()
threading.Thread(target=_do_tts, daemon=True).start()
# ── Wait ONLY for STT + LLM (fast) ────────────────────────────
_warmup_done.wait(timeout=60)
_stt_done.wait(timeout=60)
# ── Go online immediately ──────────────────────────────────────
self.ui.write_log("SYS: JARVIS online.")
self.ui.set_state("LISTENING")
self.ui.set_startup_status("● JARVIS online · Voice loading in background…")
# TTS queue worker (sequential, no overlap)
threading.Thread(target=self._tts_worker, daemon=True).start()
threading.Thread(target=self._tts_worker, daemon=True).start()
threading.Thread(target=self._text_command_loop, daemon=True).start()
self.speak("JARVIS online. All systems ready.")
# Text-command handler (runs in background)
threading.Thread(target=self._text_command_loop, daemon=True).start()
# STT loop (blocking — keeps this thread alive)
# STT loop — blocks this thread forever
if stt_engine == "vosk":
self._listen_vosk()
else:
@@ -1151,6 +1277,17 @@ class JarvisLocal:
# ---------------------------------------------------------------------------
def main() -> None:
# ── Pre-import torch in background immediately ─────────────────────────
# By the time the TTS thread starts (~5s from now), torch will already
# be in sys.modules — removing it from the TTS critical path entirely.
def _preload_torch():
try:
import torch # noqa: F401 (side-effect import only)
except Exception:
pass
threading.Thread(target=_preload_torch, daemon=True).start()
# ───────────────────────────────────────────────────────────────────────
ui = JarvisUI("face.png")
def runner():
@@ -1186,6 +1323,5 @@ def main() -> None:
threading.Thread(target=runner, daemon=True).start()
ui.root.mainloop()
if __name__ == "__main__":
main()
+255 -11
View File
@@ -884,8 +884,9 @@ class SetupOverlay(QWidget):
QLineEdit:focus {{ border: 1px solid {C.PRI}; }}
"""
self._sel_stt = _init.get("stt_engine", "whisper")
self._sel_tts = _init.get("tts_engine", "edgetts")
self._sel_stt = _init.get("stt_engine", "whisper")
self._sel_tts = _init.get("tts_engine", "edgetts")
self._sel_llm_provider = _init.get("llm_provider", "ollama")
layout = QVBoxLayout(self)
layout.setContentsMargins(22, 16, 22, 16)
@@ -1018,17 +1019,44 @@ class SetupOverlay(QWidget):
layout.addWidget(_sep())
# ── LLM ──────────────────────────────────────────────────────── #
layout.addWidget(_lbl("LOCAL LLM (Ollama)", 7, col=C.TEXT_DIM,
layout.addWidget(_lbl("LOCAL LLM", 7, col=C.TEXT_DIM,
align=Qt.AlignmentFlag.AlignLeft))
# Provider toggle: Ollama vs LM Studio / OpenAI-compatible
llm_prov_row, self._llm_prov_btns = _toggle_row(
[
("ollama", "🦙 Ollama"),
("openai", "🔌 LM Studio / OpenAI"),
],
lambda: self._sel_llm_provider,
self._set_llm_provider,
)
layout.addLayout(llm_prov_row)
# Hint label — changes based on provider
_ollama_hint = "ollama.com · run: ollama pull qwen2.5:3b"
_openai_hint = "lmstudio.ai · start Local Server first, then pick model"
self._llm_hint_lbl = _lbl(
_openai_hint if self._sel_llm_provider == "openai" else _ollama_hint,
7, col=C.TEXT_DIM, align=Qt.AlignmentFlag.AlignLeft
)
layout.addWidget(self._llm_hint_lbl)
llm_row = QHBoxLayout(); llm_row.setSpacing(5)
llm_row.addWidget(_lbl("URL:", 7, col=C.TEXT_MED,
align=Qt.AlignmentFlag.AlignRight))
self._llm_url_input = _input("http://localhost:11434")
self._llm_url_input.setText(_init.get("llm_url", "http://localhost:11434"))
_default_url = _init.get("llm_url",
"http://localhost:1234" if self._sel_llm_provider == "openai"
else "http://localhost:11434")
self._llm_url_input = _input(
"http://localhost:1234" if self._sel_llm_provider == "openai"
else "http://localhost:11434"
)
self._llm_url_input.setText(_default_url)
llm_row.addWidget(self._llm_url_input, stretch=2)
llm_row.addWidget(_lbl("Model:", 7, col=C.TEXT_MED,
align=Qt.AlignmentFlag.AlignRight))
self._llm_model_input = _input("e.g. llama3.2 / qwen2.5 / mistral")
self._llm_model_input = _input("e.g. qwen2.5:3b / llama3.2 / mistral")
self._llm_model_input.setText(_init.get("llm_model", ""))
llm_row.addWidget(self._llm_model_input, stretch=2)
layout.addLayout(llm_row)
@@ -1201,6 +1229,40 @@ class SetupOverlay(QWidget):
if hasattr(self, "_el_key_widget"):
self._el_key_widget.setVisible(key == "elevenlabs")
def _set_llm_provider(self, key: str):
self._sel_llm_provider = key
if not hasattr(self, "_llm_prov_btns"):
return
for k, btn in self._llm_prov_btns.items():
active = (k == key)
btn.setStyleSheet(f"""
QPushButton {{
background: {'#00d4ff' if active else '#000d12'};
color: {'#001a22' if active else C.TEXT_DIM};
border: {'none' if active else f'1px solid {C.BORDER}'};
border-radius: 3px; font-weight: {'bold' if active else 'normal'};
}}
QPushButton:hover {{ color: {C.TEXT}; border: 1px solid {C.BORDER_B}; }}
""")
# Update URL placeholder and hint to match selected provider
if hasattr(self, "_llm_url_input"):
if key == "openai":
self._llm_url_input.setPlaceholderText("http://localhost:1234")
# Only override if it still looks like an Ollama default URL
cur = self._llm_url_input.text().strip()
if not cur or cur == "http://localhost:11434":
self._llm_url_input.setText("http://localhost:1234")
else:
self._llm_url_input.setPlaceholderText("http://localhost:11434")
cur = self._llm_url_input.text().strip()
if not cur or cur == "http://localhost:1234":
self._llm_url_input.setText("http://localhost:11434")
if hasattr(self, "_llm_hint_lbl"):
if key == "openai":
self._llm_hint_lbl.setText("lmstudio.ai · start Local Server first, then pick model")
else:
self._llm_hint_lbl.setText("ollama.com · run: ollama pull qwen2.5:3b")
def _set_stt(self, key: str):
self._sel_stt = key
if not hasattr(self, "_stt_btns"):
@@ -1262,11 +1324,14 @@ class SetupOverlay(QWidget):
tts_voice = self._tts_voice_input.text().strip() or "en-US-GuyNeural"
tts_speed = "1.0"
_provider = getattr(self, "_sel_llm_provider", "ollama")
_default_url = "http://localhost:1234" if _provider == "openai" else "http://localhost:11434"
cfg = {
"stt_engine": self._sel_stt,
"stt_model": stt_model,
"stt_language": self._stt_lang_input.text().strip() or "auto",
"llm_url": self._llm_url_input.text().strip() or "http://localhost:11434",
"llm_provider": _provider,
"llm_url": self._llm_url_input.text().strip() or _default_url,
"llm_model": llm_model,
"tts_engine": self._sel_tts,
"tts_voice": tts_voice,
@@ -1278,9 +1343,130 @@ class SetupOverlay(QWidget):
self.done.emit(json.dumps(cfg))
class StartupPanel(QWidget):
"""Animated startup progress overlay — shown while components initialize."""
def __init__(self, parent=None):
super().__init__(parent)
self.setAttribute(Qt.WidgetAttribute.WA_StyledBackground, True)
self.setStyleSheet(f"""
StartupPanel {{
background: rgba(0, 6, 10, 235);
border: 1px solid {C.BORDER_B};
border-radius: 8px;
}}
""")
lay = QVBoxLayout(self)
lay.setContentsMargins(28, 20, 28, 20)
lay.setSpacing(10)
# ── Title ──────────────────────────────────────────────────────
title = QLabel("◈ SYSTEMS INITIALISING")
title.setFont(QFont("Courier New", 11, QFont.Weight.Bold))
title.setAlignment(Qt.AlignmentFlag.AlignCenter)
title.setStyleSheet(f"color: {C.PRI}; background: transparent;")
lay.addWidget(title)
lay.addSpacing(2)
# ── Component rows ──────────────────────────────────────────────
self._rows: dict[str, dict] = {}
_COMPS = [
("stt", "SPEECH RECOGNITION (STT)", C.GREEN),
("llm", "LANGUAGE MODEL (LLM)", C.ACC2),
("tts", "VOICE SYNTHESIS (TTS)", C.PRI),
]
for key, label, color in _COMPS:
box = QWidget()
box.setStyleSheet(
f"background: {C.PANEL2}; border: 1px solid {C.BORDER}; border-radius: 4px;"
)
box_lay = QVBoxLayout(box)
box_lay.setContentsMargins(10, 6, 10, 6)
box_lay.setSpacing(4)
top = QHBoxLayout()
nm = QLabel(label)
nm.setFont(QFont("Courier New", 8, QFont.Weight.Bold))
nm.setStyleSheet(f"color: {C.TEXT_MED}; background: transparent; border: none;")
top.addWidget(nm)
top.addStretch()
st = QLabel("LOADING…")
st.setFont(QFont("Courier New", 8, QFont.Weight.Bold))
st.setStyleSheet(f"color: {C.TEXT_DIM}; background: transparent; border: none;")
top.addWidget(st)
box_lay.addLayout(top)
bar = QProgressBar()
bar.setFixedHeight(4)
bar.setRange(0, 0) # indeterminate marquee
bar.setTextVisible(False)
bar.setStyleSheet(f"""
QProgressBar {{
background: {C.BAR_BG}; border: none; border-radius: 2px;
}}
QProgressBar::chunk {{
background: qlineargradient(x1:0,y1:0,x2:1,y2:0,
stop:0 {C.BORDER}, stop:1 {color});
border-radius: 2px; width: 60px; margin: 0px;
}}
""")
box_lay.addWidget(bar)
lay.addWidget(box)
self._rows[key] = {"bar": bar, "status": st, "color": color}
lay.addSpacing(4)
# ── Bottom status ───────────────────────────────────────────────
self._status_lbl = QLabel("Initialising components…")
self._status_lbl.setFont(QFont("Courier New", 8))
self._status_lbl.setAlignment(Qt.AlignmentFlag.AlignCenter)
self._status_lbl.setStyleSheet(f"color: {C.TEXT_DIM}; background: transparent;")
self._status_lbl.setWordWrap(True)
lay.addWidget(self._status_lbl)
tip = QLabel("All AI models run 100% locally · No data leaves your device")
tip.setFont(QFont("Courier New", 7))
tip.setAlignment(Qt.AlignmentFlag.AlignCenter)
tip.setStyleSheet(f"color: {C.BORDER}; background: transparent;")
lay.addWidget(tip)
# Called only from the main thread (via MainWindow._startup_sig)
def update_component(self, key: str, status: str) -> None:
if key not in self._rows:
return
row = self._rows[key]
ok = status == "ready"
color = row["color"] if ok else C.RED
label = "READY ✓" if ok else "ERROR ✗"
bar = row["bar"]
bar.setRange(0, 100)
bar.setValue(100)
bar.setStyleSheet(f"""
QProgressBar {{
background: {C.BAR_BG}; border: none; border-radius: 2px;
}}
QProgressBar::chunk {{
background: {color}; border-radius: 2px;
}}
""")
st = row["status"]
st.setText(label)
st.setStyleSheet(f"color: {color}; background: transparent; border: none;")
def set_status(self, text: str) -> None:
self._status_lbl.setText(text)
col = C.GREEN if "online" in text.lower() else C.TEXT_DIM
self._status_lbl.setStyleSheet(f"color: {col}; background: transparent;")
class MainWindow(QMainWindow):
_log_sig = pyqtSignal(str)
_state_sig = pyqtSignal(str)
_log_sig = pyqtSignal(str)
_state_sig = pyqtSignal(str)
_startup_sig = pyqtSignal(str, str) # action, data — thread-safe startup panel control
def __init__(self, face_path: str):
super().__init__()
@@ -1337,8 +1523,10 @@ class MainWindow(QMainWindow):
self._log_sig.connect(self._log.append_log)
self._state_sig.connect(self._apply_state)
self._startup_sig.connect(self._on_startup_sig)
self._overlay: SetupOverlay | None = None
self._startup_panel: StartupPanel | None = None
self._on_reconfigure_cb = None
self._ready = self._check_config()
if not self._ready:
@@ -1357,14 +1545,58 @@ class MainWindow(QMainWindow):
def resizeEvent(self, event):
super().resizeEvent(event)
cw = self.centralWidget()
if self._overlay and self._overlay.isVisible():
ow, oh = 520, 580
cw = self.centralWidget()
self._overlay.setGeometry(
(cw.width() - ow) // 2,
(cw.height() - oh) // 2,
ow, oh,
)
if self._startup_panel and self._startup_panel.isVisible():
pw, ph = 400, 310
self._startup_panel.setGeometry(
(cw.width() - pw) // 2,
(cw.height() - ph) // 2,
pw, ph,
)
# ── Startup panel (thread-safe via _startup_sig) ────────────────────
def _on_startup_sig(self, action: str, data: str) -> None:
"""Runs on main thread — handles all startup panel state changes."""
if action == "show":
self._create_startup_panel()
elif action in ("ready", "error"):
if self._startup_panel:
self._startup_panel.update_component(data, action)
elif action == "status":
if self._startup_panel:
self._startup_panel.set_status(data)
elif action == "hide":
if self._startup_panel:
# Fade out after a short pause so "READY ✓" is visible
QTimer.singleShot(1200, self._destroy_startup_panel)
def _create_startup_panel(self) -> None:
if self._startup_panel and self._startup_panel.isVisible():
return
cw = self.centralWidget()
pw, ph = 400, 310
panel = StartupPanel(cw)
panel.setGeometry(
(cw.width() - pw) // 2,
(cw.height() - ph) // 2,
pw, ph,
)
panel.show()
panel.raise_()
self._startup_panel = panel
def _destroy_startup_panel(self) -> None:
if self._startup_panel:
self._startup_panel.hide()
self._startup_panel.deleteLater()
self._startup_panel = None
def _update_metrics(self):
snap = _metrics.snapshot()
@@ -1716,7 +1948,6 @@ class MainWindow(QMainWindow):
txt = self._input.text().strip()
if not txt: return
self._input.clear()
self._log.append_log(f"You: {txt}")
if self.on_text_command:
threading.Thread(target=self.on_text_command, args=(txt,), daemon=True).start()
@@ -1861,6 +2092,19 @@ class JarvisUI:
def write_log(self, text: str):
self._win._log_sig.emit(text)
# ── Startup panel (all thread-safe) ──────────────────────────────────
def show_startup_panel(self) -> None:
self._win._startup_sig.emit("show", "")
def mark_startup_ready(self, key: str, error: bool = False) -> None:
self._win._startup_sig.emit("error" if error else "ready", key)
def set_startup_status(self, text: str) -> None:
self._win._startup_sig.emit("status", text)
def hide_startup_panel(self) -> None:
self._win._startup_sig.emit("hide", "")
def wait_for_api_key(self):
while not self._win._ready:
time.sleep(0.1)