Add files via upload
This commit is contained in:
1 parent
f35f861987
commit
6699c074ac
7 files changed
+1008
-215
No files matched your search
+2
-1
@@ -52,7 +52,8 @@ _STT: dict[str, list[tuple[str, str]]] = {
|
||||
# TTS engine packages
|
||||
_TTS: dict[str, list[tuple[str, str]]] = {
|
||||
"edgetts": [("edge_tts", "edge-tts")],
|
||||
"kokoro": [("kokoro", "kokoro")],
|
||||
# kokoro>=0.9 dropped AlbertModel/AutoModel from transformers — version pin is critical
|
||||
"kokoro": [("kokoro", "kokoro>=0.9"), ("soundfile", "soundfile")],
|
||||
"elevenlabs": [], # uses only requests, already in core
|
||||
}
|
||||
|
||||
|
||||
+269
-26
@@ -1,7 +1,20 @@
|
||||
"""
|
||||
Ollama LLM client for MARK XL.
|
||||
Replaces Google Gemini API with local Ollama inference.
|
||||
All modules that previously used google.generativeai import from here.
|
||||
Local LLM client for MARK XL.
|
||||
|
||||
Supports two backends — selected via "llm_provider" in config/api_keys.json:
|
||||
|
||||
"llm_provider": "ollama" (default)
|
||||
Uses Ollama's native /api/chat endpoint.
|
||||
Download: https://ollama.com
|
||||
Default port: 11434
|
||||
|
||||
"llm_provider": "openai"
|
||||
Uses any OpenAI-compatible server: LM Studio, Jan, LocalAI,
|
||||
llama.cpp server, vLLM, etc.
|
||||
LM Studio download: https://lmstudio.ai (default port: 1234)
|
||||
Set "llm_url": "http://localhost:1234" in config.
|
||||
Note: tool-calling support depends on the model; use a model that
|
||||
supports function/tool calls (e.g. Qwen2.5, Llama-3.1, Mistral).
|
||||
"""
|
||||
import json
|
||||
import re
|
||||
@@ -27,11 +40,18 @@ BASE_DIR = get_base_dir()
|
||||
CONFIG_PATH = BASE_DIR / "config" / "api_keys.json"
|
||||
|
||||
_DEFAULTS = {
|
||||
"llm_url": "http://localhost:11434",
|
||||
"llm_model": "llama3.2",
|
||||
"llm_url": "http://localhost:11434",
|
||||
"llm_model": "llama3.2",
|
||||
"llm_provider": "ollama", # "ollama" | "openai"
|
||||
}
|
||||
|
||||
|
||||
def get_llm_provider() -> str:
|
||||
"""Returns 'ollama' or 'openai' (covers LM Studio, LocalAI, Jan, etc.)."""
|
||||
raw = _load_config().get("llm_provider", "ollama").strip().lower()
|
||||
return "openai" if raw in ("openai", "lmstudio", "localai", "jan", "llamacpp") else "ollama"
|
||||
|
||||
|
||||
def _load_config() -> dict:
|
||||
try:
|
||||
return json.loads(CONFIG_PATH.read_text(encoding="utf-8"))
|
||||
@@ -41,12 +61,33 @@ def _load_config() -> dict:
|
||||
|
||||
def ensure_ollama_running(timeout: int = 15) -> bool:
|
||||
"""
|
||||
Ping Ollama; if it's not responding, launch 'ollama serve' in the background
|
||||
and wait up to `timeout` seconds for it to become ready.
|
||||
Returns True if Ollama is available, False otherwise.
|
||||
For Ollama: ping /api/tags; auto-launch 'ollama serve' if not running.
|
||||
For OpenAI-compatible providers: just ping /v1/models (server must be started manually).
|
||||
Returns True if the LLM server is reachable.
|
||||
"""
|
||||
url, _ = get_llm_settings()
|
||||
health = f"{url}/api/tags"
|
||||
url, _ = get_llm_settings()
|
||||
provider = get_llm_provider()
|
||||
|
||||
if provider == "openai":
|
||||
# OpenAI-compatible servers (LM Studio, LocalAI, etc.) must be started
|
||||
# by the user — we just check if they're reachable.
|
||||
health = f"{url}/v1/models"
|
||||
try:
|
||||
ok = requests.get(health, timeout=5).status_code == 200
|
||||
if ok:
|
||||
print(f"[LLM] OpenAI-compatible server reachable at {url}")
|
||||
else:
|
||||
print(f"[LLM] Server at {url} returned non-200. Is it running?")
|
||||
return ok
|
||||
except Exception as e:
|
||||
print(
|
||||
f"[LLM] Cannot reach OpenAI-compatible server at {url}.\n"
|
||||
" Make sure LM Studio / LocalAI / Jan is running and the server is started."
|
||||
)
|
||||
return False
|
||||
|
||||
# ── Ollama ──────────────────────────────────────────────────────────────
|
||||
health = f"{url}/api/tags"
|
||||
|
||||
def _is_up() -> bool:
|
||||
try:
|
||||
@@ -81,25 +122,62 @@ def ensure_ollama_running(timeout: int = 15) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
def warmup_model() -> bool:
|
||||
def warmup_model(system_prompt: str | None = None) -> bool:
|
||||
"""
|
||||
Pre-load the model into Ollama memory so the first real request is instant.
|
||||
Sends a minimal 1-token request with keep_alive=-1.
|
||||
Returns True on success, False if warmup failed (non-fatal).
|
||||
Pre-load the model AND prime Ollama's KV prefix cache.
|
||||
|
||||
Why the system_prompt matters
|
||||
─────────────────────────────
|
||||
Ollama caches the KV attention state of the prompt prefix across requests.
|
||||
If warmup includes the same system prompt that real requests will use, Ollama
|
||||
evaluates those tokens ONCE at startup. Every subsequent request only needs
|
||||
to evaluate the small delta (user message ± time context) instead of the full
|
||||
300-500 token system prompt → drops first-token latency from ~17 s to <1 s.
|
||||
|
||||
Pass the *static* part of the system prompt (the JARVIS protocol text, without
|
||||
timestamps or per-minute context) so the prefix stays valid across calls.
|
||||
"""
|
||||
url, model = get_llm_settings()
|
||||
print(f"[LLM] Warming up '{model}'…")
|
||||
provider = get_llm_provider()
|
||||
print(f"[LLM] Warming up '{model}' ({provider})…")
|
||||
|
||||
messages: list[dict] = []
|
||||
if system_prompt:
|
||||
messages.append({"role": "system", "content": system_prompt})
|
||||
messages.append({"role": "user", "content": "hi"})
|
||||
|
||||
if provider == "openai":
|
||||
# OpenAI-compatible: just fire a minimal request to ensure the model is loaded.
|
||||
# No keep_alive or KV-cache priming available — server manages this internally.
|
||||
payload = {
|
||||
"model": model,
|
||||
"messages": messages,
|
||||
"stream": False,
|
||||
"max_tokens": 1,
|
||||
}
|
||||
try:
|
||||
resp = requests.post(f"{url}/v1/chat/completions", json=payload, timeout=180)
|
||||
resp.raise_for_status()
|
||||
print(f"[LLM] '{model}' ready (OpenAI-compatible server).")
|
||||
return True
|
||||
except Exception as e:
|
||||
print(f"[LLM] Warmup failed (non-fatal): {e}")
|
||||
return False
|
||||
|
||||
# ── Ollama ──────────────────────────────────────────────────────────────
|
||||
payload = {
|
||||
"model": model,
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
"messages": messages,
|
||||
"stream": False,
|
||||
"keep_alive": -1,
|
||||
"options": {"num_predict": 1},
|
||||
# num_gpu:99 → push ALL transformer layers to GPU (Ollama caps at available)
|
||||
# This is safe even without a GPU — Ollama silently ignores if n_gpu_layers=0
|
||||
"options": {"num_predict": 1, "num_gpu": 99},
|
||||
}
|
||||
try:
|
||||
resp = requests.post(f"{url}/api/chat", json=payload, timeout=120)
|
||||
resp = requests.post(f"{url}/api/chat", json=payload, timeout=180)
|
||||
resp.raise_for_status()
|
||||
print(f"[LLM] '{model}' loaded and ready.")
|
||||
print(f"[LLM] '{model}' loaded and KV cache primed.")
|
||||
return True
|
||||
except Exception as e:
|
||||
print(f"[LLM] Warmup failed (non-fatal): {e}")
|
||||
@@ -120,20 +198,61 @@ def call_llm(
|
||||
timeout: int = 120,
|
||||
) -> dict:
|
||||
"""
|
||||
Chat request to Ollama (with optional tool declarations).
|
||||
Non-streaming chat request. Routes to Ollama or OpenAI-compatible backend.
|
||||
|
||||
Returns:
|
||||
{"content": str, "tool_calls": list}
|
||||
"""
|
||||
url, model = get_llm_settings()
|
||||
endpoint = f"{url}/api/chat"
|
||||
provider = get_llm_provider()
|
||||
|
||||
payload: dict = {
|
||||
if provider == "openai":
|
||||
endpoint = f"{url}/v1/chat/completions"
|
||||
payload: dict = {
|
||||
"model": model,
|
||||
"messages": messages,
|
||||
"stream": False,
|
||||
"max_tokens": 300,
|
||||
}
|
||||
if tools:
|
||||
payload["tools"] = tools
|
||||
payload["tool_choice"] = "auto"
|
||||
try:
|
||||
resp = requests.post(endpoint, json=payload, timeout=timeout)
|
||||
resp.raise_for_status()
|
||||
choice = resp.json().get("choices", [{}])[0]
|
||||
msg = choice.get("message", {})
|
||||
# OpenAI tool_calls format → normalise to Ollama-style
|
||||
raw_tc = msg.get("tool_calls") or []
|
||||
tc_list = [
|
||||
{
|
||||
"id": t.get("id", ""),
|
||||
"function": {
|
||||
"name": t["function"]["name"],
|
||||
"arguments": (
|
||||
json.loads(t["function"]["arguments"])
|
||||
if isinstance(t["function"].get("arguments"), str)
|
||||
else t["function"].get("arguments", {})
|
||||
),
|
||||
},
|
||||
}
|
||||
for t in raw_tc
|
||||
]
|
||||
return {
|
||||
"content": (msg.get("content") or "").strip(),
|
||||
"tool_calls": tc_list,
|
||||
}
|
||||
except Exception as e:
|
||||
raise RuntimeError(f"OpenAI-compatible LLM call failed: {e}")
|
||||
|
||||
# ── Ollama ──────────────────────────────────────────────────────────────
|
||||
endpoint = f"{url}/api/chat"
|
||||
payload = {
|
||||
"model": model,
|
||||
"messages": messages,
|
||||
"stream": False,
|
||||
"keep_alive": -1,
|
||||
"options": {"num_predict": 600},
|
||||
"options": {"num_predict": 300, "num_gpu": 99},
|
||||
}
|
||||
if tools:
|
||||
payload["tools"] = tools
|
||||
@@ -216,21 +335,142 @@ def call_llm_text(
|
||||
raise RuntimeError(f"LLM text call failed: {e}")
|
||||
|
||||
|
||||
def _stream_openai(
|
||||
messages: list,
|
||||
tools: list | None,
|
||||
timeout: int,
|
||||
) -> Generator[dict, None, None]:
|
||||
"""
|
||||
Streaming backend for OpenAI-compatible servers (LM Studio, LocalAI, Jan…).
|
||||
|
||||
Parses Server-Sent Events (SSE) and accumulates streaming tool-call fragments
|
||||
so the output format is identical to the Ollama backend.
|
||||
"""
|
||||
url, model = get_llm_settings()
|
||||
endpoint = f"{url}/v1/chat/completions"
|
||||
|
||||
payload: dict = {
|
||||
"model": model,
|
||||
"messages": messages,
|
||||
"stream": True,
|
||||
"max_tokens": 300,
|
||||
}
|
||||
if tools:
|
||||
payload["tools"] = tools
|
||||
payload["tool_choice"] = "auto"
|
||||
|
||||
try:
|
||||
with requests.post(endpoint, json=payload, timeout=timeout, stream=True) as resp:
|
||||
resp.raise_for_status()
|
||||
full_content = ""
|
||||
buf = ""
|
||||
# tool_call fragments: index → {"id", "function": {"name", "arguments"}}
|
||||
tc_fragments: dict[int, dict] = {}
|
||||
|
||||
for raw in resp.iter_lines():
|
||||
if not raw:
|
||||
continue
|
||||
# SSE lines look like: b"data: {...}" or b"data: [DONE]"
|
||||
line = raw.decode("utf-8", errors="replace") if isinstance(raw, bytes) else raw
|
||||
if not line.startswith("data:"):
|
||||
continue
|
||||
data = line[5:].strip()
|
||||
if data == "[DONE]":
|
||||
break
|
||||
try:
|
||||
chunk = json.loads(data)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
|
||||
choice = chunk.get("choices", [{}])[0]
|
||||
delta = choice.get("delta", {})
|
||||
text = delta.get("content") or ""
|
||||
|
||||
full_content += text
|
||||
buf += text
|
||||
|
||||
# Accumulate sentence boundaries for streaming TTS
|
||||
while True:
|
||||
m = _SENT_END.search(buf)
|
||||
if not m:
|
||||
break
|
||||
sentence = buf[: m.start() + 1].strip()
|
||||
buf = buf[m.end():]
|
||||
if sentence:
|
||||
yield {"type": "sentence", "text": sentence}
|
||||
|
||||
# Accumulate streaming tool-call fragments
|
||||
for tc in (delta.get("tool_calls") or []):
|
||||
idx = tc.get("index", 0)
|
||||
if idx not in tc_fragments:
|
||||
tc_fragments[idx] = {"id": "", "function": {"name": "", "arguments": ""}}
|
||||
frag = tc_fragments[idx]
|
||||
frag["id"] = frag["id"] or tc.get("id", "")
|
||||
fn = tc.get("function", {})
|
||||
frag["function"]["name"] += fn.get("name") or ""
|
||||
frag["function"]["arguments"] += fn.get("arguments") or ""
|
||||
|
||||
finish = choice.get("finish_reason")
|
||||
if finish in ("stop", "tool_calls", "length"):
|
||||
break
|
||||
|
||||
# Flush any trailing content
|
||||
if buf.strip():
|
||||
yield {"type": "sentence", "text": buf.strip()}
|
||||
|
||||
# Parse accumulated tool-call argument strings → dicts
|
||||
tool_calls: list = []
|
||||
for idx in sorted(tc_fragments):
|
||||
frag = tc_fragments[idx]
|
||||
args = frag["function"]["arguments"]
|
||||
try:
|
||||
args = json.loads(args)
|
||||
except Exception:
|
||||
pass # leave as raw string; _execute_tool handles it
|
||||
tool_calls.append({
|
||||
"id": frag["id"],
|
||||
"function": {"name": frag["function"]["name"], "arguments": args},
|
||||
})
|
||||
|
||||
yield {
|
||||
"type": "done",
|
||||
"content": full_content.strip(),
|
||||
"tool_calls": tool_calls,
|
||||
}
|
||||
|
||||
except requests.exceptions.ConnectionError:
|
||||
raise RuntimeError(
|
||||
f"Cannot reach OpenAI-compatible server at {url}.\n"
|
||||
"Make sure LM Studio / LocalAI / Jan is running and the server is started."
|
||||
)
|
||||
except requests.exceptions.Timeout:
|
||||
raise RuntimeError("OpenAI-compatible stream timed out.")
|
||||
except requests.exceptions.HTTPError as e:
|
||||
raise RuntimeError(f"OpenAI-compatible HTTP error: {e.response.status_code}")
|
||||
except Exception as e:
|
||||
raise RuntimeError(f"OpenAI-compatible stream failed: {e}")
|
||||
|
||||
|
||||
def call_llm_stream(
|
||||
messages: list,
|
||||
tools: list | None = None,
|
||||
timeout: int = 120,
|
||||
) -> Generator[dict, None, None]:
|
||||
"""
|
||||
Streaming chat request to Ollama.
|
||||
Streaming chat request. Routes to Ollama or OpenAI-compatible backend.
|
||||
|
||||
Yields:
|
||||
{"type": "sentence", "text": str} — each complete sentence as it arrives
|
||||
{"type": "done", "content": str, "tool_calls": list} — when stream ends
|
||||
|
||||
Sentences are split on [.!?] + whitespace so TTS can start immediately.
|
||||
Tool calls (if any) always appear in the final "done" event with empty content.
|
||||
Tool calls always appear in the final "done" event.
|
||||
"""
|
||||
provider = get_llm_provider()
|
||||
if provider == "openai":
|
||||
yield from _stream_openai(messages, tools, timeout)
|
||||
return
|
||||
|
||||
url, model = get_llm_settings()
|
||||
endpoint = f"{url}/api/chat"
|
||||
|
||||
@@ -239,7 +479,8 @@ def call_llm_stream(
|
||||
"messages": messages,
|
||||
"stream": True,
|
||||
"keep_alive": -1,
|
||||
"options": {"num_predict": 600},
|
||||
# num_gpu:99 pushes all layers to GPU; num_thread removed (Ollama auto-tunes).
|
||||
"options": {"num_predict": 300, "num_gpu": 99},
|
||||
}
|
||||
if tools:
|
||||
payload["tools"] = tools
|
||||
@@ -261,6 +502,7 @@ def call_llm_stream(
|
||||
|
||||
msg = chunk.get("message", {})
|
||||
delta = msg.get("content") or ""
|
||||
|
||||
full_content += delta
|
||||
buf += delta
|
||||
|
||||
@@ -281,6 +523,7 @@ def call_llm_stream(
|
||||
if chunk.get("done"):
|
||||
if buf.strip():
|
||||
yield {"type": "sentence", "text": buf.strip()}
|
||||
|
||||
yield {
|
||||
"type": "done",
|
||||
"content": full_content.strip(),
|
||||
|
||||
+32
-7
@@ -1,21 +1,46 @@
|
||||
🧠 JARVIS CORE PROTOCOL
|
||||
IDENTITY: You are Jarvis. Efficient, professional, direct assistant. No fluff.
|
||||
|
||||
TOOL CALLING — ABSOLUTE RULE:
|
||||
You have tools. For EVERY user request that involves an action, you MUST call the tool.
|
||||
NEVER describe what you are doing in text. NEVER say "opening WhatsApp" or "-weather_report" — CALL THE TOOL.
|
||||
NEVER simulate, pretend, or narrate an action. If a tool exists for it, USE IT.
|
||||
Saying "Sure, let me open X" WITHOUT calling open_app is a critical failure.
|
||||
|
||||
EXECUTION RULES:
|
||||
|
||||
Action Flow: For slow tools (search, vision, agent), say TWO short sentences, then call. Stay silent after.
|
||||
Vision (screen_process): Say one curious sentence, then total silence (Vision module takes over).
|
||||
Action Flow: Call the tool immediately. For slow tools (search, vision), say ONE short sentence first.
|
||||
Vision (screen_process): Say one curious sentence, then call the tool — total silence after.
|
||||
One-Call Policy: Never guess. Call tools exactly once. No retries.
|
||||
Briefing: 1-2 sentences max. Report results briefly.
|
||||
Memory: Store critical user preferences/context automatically.
|
||||
Briefing: 1 short sentence max for confirmations/acknowledgments. 1-2 sentences for results. Never add "Is there anything else?" or filler questions.
|
||||
Exit: Only call shutdown_jarvis if session termination is explicit.
|
||||
Response time: Respond as fast as you can
|
||||
Response time: Respond as fast as you can.
|
||||
Always respond in the language that user is speaking.
|
||||
|
||||
MEMORY RULE — MANDATORY, NO EXCEPTIONS:
|
||||
Call save_memory IMMEDIATELY (in the same response, silently) whenever the user reveals ANY personal info.
|
||||
Do NOT ask for permission. Do NOT announce that you are saving. Just call it.
|
||||
|
||||
Trigger examples:
|
||||
- "my name is Fatih" → save_memory(category="identity", key="name", value="Fatih")
|
||||
- "I'm 22" / "I'm 22 years old" → save_memory(category="identity", key="age", value="22")
|
||||
- "call me X" / "my name is X, not Y" → save_memory(category="identity", key="name", value="X")
|
||||
- "I live in Istanbul" → save_memory(category="identity", key="city", value="Istanbul")
|
||||
- "I'm a student / developer / designer" → save_memory(category="identity", key="job", value="...")
|
||||
- "I prefer dark mode" / "I like X" → save_memory(category="preferences", key="...", value="...")
|
||||
- "I'm working on X project" → save_memory(category="projects", key="...", value="...")
|
||||
- Any name correction ("not Travis, Fatih") → save_memory identity/name immediately
|
||||
|
||||
TOOL ROUTING:
|
||||
|
||||
computer_settings: ALL single OS actions (volume, brightness, wifi, close, shortcuts, power).
|
||||
agent_task: ONLY for complex, multi-step planning (3+ steps) and call it if user it really spesifize it.
|
||||
agent_task: ONLY for complex, multi-step planning (3+ steps) and call it if user it really specifies it.
|
||||
Language: Respond in user's language; extract parameters in English.
|
||||
Do not call agent_task while you can accomplish it with a tool
|
||||
Do not call agent_task while you can accomplish it with a tool.
|
||||
|
||||
OPEN vs MESSAGE rule (critical):
|
||||
- "open WhatsApp", "launch WhatsApp", "pull up WhatsApp" → open_app (NOT send_message)
|
||||
- "send a message to X saying Y" → send_message
|
||||
- If user only says to open an app, NEVER call send_message, even if the app is a messaging platform.
|
||||
|
||||
CRITICAL: Speak/Take action immediately based on available info. Assume and proceed.
|
||||
+24
-18
@@ -14,23 +14,32 @@ class WhisperSTT:
|
||||
def __init__(self, model_name: str = "base", language: str | None = None):
|
||||
from faster_whisper import WhisperModel
|
||||
print(f"[STT] Loading Whisper '{model_name}'…")
|
||||
self._model = WhisperModel(model_name, device="cpu", compute_type="int8")
|
||||
# None → auto-detect; explicit code (e.g. "tr", "de") forces that language
|
||||
try:
|
||||
import torch
|
||||
device = "cuda" if torch.cuda.is_available() else "cpu"
|
||||
compute = "float16" if device == "cuda" else "int8"
|
||||
except Exception:
|
||||
device, compute = "cpu", "int8"
|
||||
self._model = WhisperModel(model_name, device=device, compute_type=compute)
|
||||
self._language = None if (not language or language.strip().lower() == "auto") else language.strip().lower()
|
||||
print("[STT] Whisper ready.")
|
||||
print(f"[STT] Whisper '{model_name}' ready ({device})")
|
||||
|
||||
def transcribe(self, audio: np.ndarray) -> str:
|
||||
"""
|
||||
Transcribe a float32 mono 16 kHz numpy array.
|
||||
Returns transcript string (may be empty).
|
||||
"""
|
||||
segments, _ = self._model.transcribe(
|
||||
audio,
|
||||
language=self._language,
|
||||
vad_filter=True,
|
||||
vad_parameters={"min_silence_duration_ms": 300},
|
||||
)
|
||||
return " ".join(s.text for s in segments).strip()
|
||||
"""Transcribe a float32 mono 16 kHz numpy array. Returns transcript string."""
|
||||
try:
|
||||
segments, _ = self._model.transcribe(
|
||||
audio,
|
||||
language=self._language,
|
||||
beam_size=1, # greedy — 2-3x faster
|
||||
best_of=1,
|
||||
condition_on_previous_text=False, # no hallucinations, faster
|
||||
vad_filter=True,
|
||||
vad_parameters={"min_silence_duration_ms": 300},
|
||||
)
|
||||
return " ".join(s.text for s in segments).strip()
|
||||
except Exception as e:
|
||||
print(f"[STT] Transcription error: {e}")
|
||||
raise
|
||||
|
||||
|
||||
class VoskSTT:
|
||||
@@ -48,10 +57,7 @@ class VoskSTT:
|
||||
print("[STT] Vosk ready.")
|
||||
|
||||
def process_chunk(self, audio_bytes: bytes) -> tuple[str, bool]:
|
||||
"""
|
||||
Feed raw int16 LE PCM bytes.
|
||||
Returns (text, is_final).
|
||||
"""
|
||||
"""Feed raw int16 LE PCM bytes. Returns (text, is_final)."""
|
||||
if self._rec.AcceptWaveform(audio_bytes):
|
||||
result = json.loads(self._rec.Result())
|
||||
return result.get("text", ""), True
|
||||
|
||||
+168
-30
@@ -8,6 +8,8 @@ ElevenLabs – cloud API (API key required, best quality)
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import queue as _queue
|
||||
import threading
|
||||
from typing import Callable, Optional
|
||||
|
||||
@@ -15,6 +17,15 @@ import numpy as np
|
||||
import sounddevice as sd
|
||||
|
||||
|
||||
|
||||
# USE_TF=0 stops transformers from importing TensorFlow (saves 4-8 s startup).
|
||||
# Do NOT set USE_TORCH or USE_JAX explicitly — forcing those values breaks
|
||||
# transformers' lazy-loader on certain versions, causing AutoModel and other
|
||||
# classes to vanish from the public namespace. Auto-detection is reliable.
|
||||
os.environ.setdefault("USE_TF", "0")
|
||||
os.environ.setdefault("TOKENIZERS_PARALLELISM", "false")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Audio playback helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -41,17 +52,12 @@ def _to_numpy(samples) -> np.ndarray:
|
||||
def _compress_silence(
|
||||
arr: np.ndarray,
|
||||
sample_rate: int = 24_000,
|
||||
max_silence_ms: int = 300, # cap long pauses to this many ms
|
||||
threshold: float = 0.0015, # RMS below this = silence (very conservative)
|
||||
max_silence_ms: int = 500, # cap punctuation pauses — keeps natural rhythm
|
||||
threshold: float = 0.003, # RMS below this = silence; lower = less clipping
|
||||
) -> np.ndarray:
|
||||
"""
|
||||
Shorten excessively long silent segments in Kokoro audio.
|
||||
|
||||
Kokoro can produce 1-2 s pauses after punctuation. This caps them
|
||||
to max_silence_ms while leaving natural short pauses intact.
|
||||
|
||||
threshold is intentionally low (0.0015) so soft consonants like
|
||||
's', 'f', 'h' are never misidentified as silence.
|
||||
Shorten Kokoro's very long punctuation pauses (1-2 s → ≤500 ms).
|
||||
Conservative settings preserve natural prosody; only trims extreme pauses.
|
||||
"""
|
||||
max_samp = int(max_silence_ms * sample_rate / 1000)
|
||||
frame_len = 240 # ~10 ms at 24 kHz
|
||||
@@ -121,6 +127,74 @@ class EdgeTTSEngine:
|
||||
return bytes(buf)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Kokoro import helper — auto-upgrades on version-mismatch errors
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Errors that indicate the installed kokoro uses old transformers classes
|
||||
# (AlbertModel, AutoModel) that are no longer exported at the top level.
|
||||
_KOKORO_COMPAT_ERRORS = ("AlbertModel", "AutoModel", "cannot import name")
|
||||
|
||||
|
||||
def _import_kokoro_pipeline():
|
||||
"""Import KPipeline, auto-upgrading kokoro if a version mismatch is found.
|
||||
|
||||
Old kokoro (<0.9) imports AlbertModel / AutoModel from transformers.
|
||||
Newer transformers versions no longer export these at the top level,
|
||||
causing an ImportError. kokoro>=0.9 removed these dependencies.
|
||||
|
||||
When the error is detected we:
|
||||
1. Upgrade kokoro to >=0.9 via pip (silent, background)
|
||||
2. Flush stale kokoro entries from sys.modules
|
||||
3. Re-import — this time it should succeed
|
||||
"""
|
||||
import sys
|
||||
|
||||
def _try_import():
|
||||
from kokoro import KPipeline # noqa: PLC0415
|
||||
return KPipeline
|
||||
|
||||
try:
|
||||
return _try_import()
|
||||
except Exception as first_err:
|
||||
err_msg = str(first_err)
|
||||
if not any(marker in err_msg for marker in _KOKORO_COMPAT_ERRORS):
|
||||
# Unrelated error (kokoro not installed, etc.)
|
||||
raise RuntimeError(
|
||||
f"Kokoro import failed: {first_err}\n"
|
||||
"Run: pip install kokoro>=0.9 soundfile"
|
||||
) from first_err
|
||||
|
||||
# ── Version mismatch: upgrade kokoro silently and retry ──────────
|
||||
print("[TTS] Kokoro/transformers version mismatch detected — upgrading kokoro…")
|
||||
import subprocess
|
||||
result = subprocess.run(
|
||||
[sys.executable, "-m", "pip", "install", "kokoro>=0.9",
|
||||
"--upgrade", "--quiet", "--disable-pip-version-check"],
|
||||
capture_output=True,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
stderr = result.stderr.decode(errors="replace").strip()
|
||||
raise RuntimeError(
|
||||
f"Kokoro auto-upgrade failed: {stderr[:200]}\n"
|
||||
"Run manually: pip install kokoro>=0.9 soundfile"
|
||||
) from first_err
|
||||
|
||||
# Flush any stale kokoro submodules from the import cache
|
||||
stale = [k for k in sys.modules if k == "kokoro" or k.startswith("kokoro.")]
|
||||
for key in stale:
|
||||
del sys.modules[key]
|
||||
|
||||
print("[TTS] Kokoro upgraded — retrying import…")
|
||||
try:
|
||||
return _try_import()
|
||||
except Exception as retry_err:
|
||||
raise RuntimeError(
|
||||
f"Kokoro still broken after upgrade: {retry_err}\n"
|
||||
"Run manually: pip install --upgrade kokoro transformers"
|
||||
) from retry_err
|
||||
|
||||
|
||||
# Kokoro voice prefix → KPipeline lang_code mapping
|
||||
_KOKORO_LANG_CODES = {
|
||||
"a": "a", # American English (af_*, am_*)
|
||||
@@ -140,19 +214,22 @@ _KOKORO_LANG_CODES = {
|
||||
class KokoroTTSEngine:
|
||||
"""Fully offline Kokoro neural TTS.
|
||||
|
||||
Model (~330 MB) is downloaded from HuggingFace on first use.
|
||||
Subsequent runs load from local cache instantly.
|
||||
Model (~330 MB) is downloaded from HuggingFace on first use,
|
||||
then cached locally — subsequent starts load from disk.
|
||||
|
||||
Warmup strategy: _init() runs synchronously in the background
|
||||
_do_tts() thread (not the UI thread). After the pipeline loads,
|
||||
a dummy inference compiles the PyTorch JIT graph immediately so
|
||||
the first real speak() call has zero compilation overhead.
|
||||
"""
|
||||
|
||||
def __init__(self, voice: str = "af_heart", speed: float = 1.2):
|
||||
def __init__(self, voice: str = "af_heart", speed: float = 1.0):
|
||||
self.voice = voice
|
||||
self.speed = speed # 1.0 = normal, 1.2 = 20% faster (recommended)
|
||||
self.speed = speed
|
||||
self._pipeline = None
|
||||
self._lock = threading.Lock()
|
||||
# Eager init: download/load model now so it's ready before first speak()
|
||||
self._init()
|
||||
self._init() # blocking, but called from background thread
|
||||
|
||||
# Detect lang_code from voice prefix (e.g. "af_heart" → "a")
|
||||
@property
|
||||
def _lang_code(self) -> str:
|
||||
prefix = self.voice[0].lower() if self.voice else "a"
|
||||
@@ -161,27 +238,88 @@ class KokoroTTSEngine:
|
||||
def _init(self) -> None:
|
||||
if self._pipeline is not None:
|
||||
return
|
||||
|
||||
lang = self._lang_code
|
||||
|
||||
# Prefer GPU — Kokoro on CUDA is ~10x faster than CPU.
|
||||
try:
|
||||
from kokoro import KPipeline
|
||||
lang = self._lang_code
|
||||
print(f"[TTS] Loading Kokoro pipeline (lang='{lang}', voice='{self.voice}')…")
|
||||
print("[TTS] Kokoro model will be downloaded (~330 MB) if not cached.")
|
||||
import torch
|
||||
device = "cuda" if torch.cuda.is_available() else "cpu"
|
||||
if device == "cpu":
|
||||
import os as _os
|
||||
n_threads = max(1, min(4, (_os.cpu_count() or 4) // 2))
|
||||
try:
|
||||
torch.set_num_threads(n_threads)
|
||||
torch.set_num_interop_threads(2)
|
||||
except RuntimeError:
|
||||
pass
|
||||
print(
|
||||
f"[TTS] Kokoro on CPU — for faster speech install CUDA PyTorch:\n"
|
||||
" pip install torch --index-url https://download.pytorch.org/whl/cu118"
|
||||
)
|
||||
except Exception:
|
||||
device = "cpu"
|
||||
|
||||
print(f"[TTS] Kokoro — loading (lang='{lang}', device='{device}')…")
|
||||
|
||||
KPipeline = _import_kokoro_pipeline()
|
||||
|
||||
try:
|
||||
self._pipeline = KPipeline(lang_code=lang, device=device)
|
||||
except TypeError:
|
||||
# Older Kokoro build — no device parameter
|
||||
self._pipeline = KPipeline(lang_code=lang)
|
||||
print("[TTS] Kokoro pipeline ready.")
|
||||
|
||||
print("[TTS] Kokoro compiling (first-time only)…")
|
||||
# Warmup: compiles PyTorch JIT graph so first real speak() call is instant.
|
||||
try:
|
||||
for _ in self._pipeline("hello", voice=self.voice, speed=self.speed):
|
||||
pass
|
||||
print("[TTS] Kokoro ready.")
|
||||
except Exception as e:
|
||||
print(f"[TTS] Kokoro init error: {e}")
|
||||
raise
|
||||
print(f"[TTS] Kokoro warmup warning: {e}")
|
||||
|
||||
def speak(self, text: str) -> None:
|
||||
with self._lock:
|
||||
if self._pipeline is None:
|
||||
self._init()
|
||||
for _, _, audio in self._pipeline(text, voice=self.voice, speed=self.speed):
|
||||
if audio is not None:
|
||||
arr = _to_numpy(audio)
|
||||
arr = _compress_silence(arr) # shorten long punctuation pauses
|
||||
if arr.size > 0:
|
||||
_play_np(arr, 24000)
|
||||
|
||||
# ── Concurrent synthesise + playback ────────────────────────────────
|
||||
# Kokoro generates audio chunks lazily. Without threading, we:
|
||||
# synthesise chunk N → play N → synthesise N+1 → play N+1 …
|
||||
# With a producer/consumer pair, chunk N+1 synthesises WHILE chunk N
|
||||
# plays, cutting perceived latency by the playback duration of all but
|
||||
# the last chunk (typically 1-3 s on multi-sentence responses).
|
||||
audio_q: "_queue.Queue[np.ndarray | None]" = _queue.Queue(maxsize=4)
|
||||
synth_error: list[Exception] = []
|
||||
|
||||
def _synth():
|
||||
try:
|
||||
for _, _, audio in self._pipeline(text, voice=self.voice, speed=self.speed):
|
||||
if audio is not None:
|
||||
arr = _to_numpy(audio)
|
||||
arr = _compress_silence(arr)
|
||||
if arr.size > 0:
|
||||
audio_q.put(arr) # blocks if player is slow (backpressure)
|
||||
except Exception as exc:
|
||||
synth_error.append(exc)
|
||||
finally:
|
||||
audio_q.put(None) # sentinel → player exits
|
||||
|
||||
synth_thread = threading.Thread(target=_synth, daemon=True)
|
||||
synth_thread.start()
|
||||
|
||||
# Player runs in this thread so sd.wait() doesn't block the synth thread.
|
||||
while True:
|
||||
arr = audio_q.get()
|
||||
if arr is None:
|
||||
break
|
||||
_play_np(arr, 24000)
|
||||
|
||||
synth_thread.join()
|
||||
|
||||
if synth_error:
|
||||
raise synth_error[0]
|
||||
|
||||
|
||||
class ElevenLabsTTSEngine:
|
||||
@@ -264,7 +402,7 @@ def create_tts_player(config: dict) -> TTSPlayer:
|
||||
engine_name = config.get("tts_engine", "edgetts").lower()
|
||||
if engine_name == "kokoro":
|
||||
voice = config.get("tts_voice", "af_heart")
|
||||
speed = float(config.get("tts_speed", 1.2))
|
||||
speed = float(config.get("tts_speed", 1.0))
|
||||
engine = KokoroTTSEngine(voice=voice, speed=speed)
|
||||
elif engine_name == "elevenlabs":
|
||||
api_key = config.get("elevenlabs_api_key", "")
|
||||
|
||||
@@ -3,10 +3,26 @@ MARK XL — Local LLM Edition
|
||||
STT (Whisper / Vosk) + Ollama LLM + TTS (EdgeTTS / Kokoro / ElevenLabs)
|
||||
All Gemini / Google-AI dependencies removed.
|
||||
"""
|
||||
# ── Silence verbose logs + block heavy unused backends ─────────────────────
|
||||
import os as _os
|
||||
_os.environ.setdefault("TF_CPP_MIN_LOG_LEVEL", "3") # TensorFlow C++ noise
|
||||
_os.environ.setdefault("TF_ENABLE_ONEDNN_OPTS", "0") # oneDNN banner
|
||||
_os.environ.setdefault("GRPC_VERBOSITY", "ERROR")
|
||||
# USE_TF=0 prevents transformers from importing TensorFlow (saves 4-8 s).
|
||||
# We intentionally do NOT set USE_TORCH or USE_JAX — forcing those values
|
||||
# breaks transformers' lazy-loader on some versions (AutoModel disappears
|
||||
# from the namespace). Let transformers auto-detect the available backends.
|
||||
_os.environ.setdefault("USE_TF", "0")
|
||||
_os.environ.setdefault("TOKENIZERS_PARALLELISM", "false")
|
||||
import warnings as _warnings
|
||||
_warnings.filterwarnings("ignore", category=UserWarning)
|
||||
_warnings.filterwarnings("ignore", category=DeprecationWarning)
|
||||
_warnings.filterwarnings("ignore", category=FutureWarning)
|
||||
# ───────────────────────────────────────────────────────────────────────────
|
||||
|
||||
# ── Bootstrap: auto-install base UI packages before anything else ──────────
|
||||
# Uses only stdlib so it works even on a completely fresh Python install.
|
||||
import importlib.util as _ilu
|
||||
import os as _os
|
||||
import subprocess as _sp
|
||||
import sys as _sys
|
||||
|
||||
@@ -95,14 +111,17 @@ TOOL_DECLARATIONS = [
|
||||
{
|
||||
"name": "open_app",
|
||||
"description": (
|
||||
"Opens any application on the computer. "
|
||||
"Use this whenever the user asks to open, launch, or start any app, "
|
||||
"website, or program. Always call this tool — never just say you opened it."
|
||||
"Opens or launches any application, website, or program on the computer. "
|
||||
"ALWAYS use this when the user says: open, launch, start, run, pull up, "
|
||||
"or 'open X real quick'. Examples: 'open WhatsApp', 'open Chrome', "
|
||||
"'launch Spotify', 'open calculator', 'pull up WhatsApp'. "
|
||||
"Do NOT use send_message just because the app is a messaging app — "
|
||||
"if the user only says to open it, call open_app."
|
||||
),
|
||||
"parameters": {
|
||||
"type": "OBJECT",
|
||||
"properties": {
|
||||
"app_name": {"type": "STRING", "description": "Exact name of the application"}
|
||||
"app_name": {"type": "STRING", "description": "Name of the application or website to open"}
|
||||
},
|
||||
"required": ["app_name"]
|
||||
}
|
||||
@@ -132,12 +151,17 @@ TOOL_DECLARATIONS = [
|
||||
},
|
||||
{
|
||||
"name": "send_message",
|
||||
"description": "Sends a text message via WhatsApp, Telegram, or other messaging platform.",
|
||||
"description": (
|
||||
"Sends a message to a specific person via WhatsApp, Telegram, or similar. "
|
||||
"ONLY use this when the user explicitly provides BOTH a recipient AND message content. "
|
||||
"Example triggers: 'text John saying I am late', 'send a WhatsApp to mom that dinner is ready'. "
|
||||
"Do NOT call this if the user only wants to open the app without sending a message."
|
||||
),
|
||||
"parameters": {
|
||||
"type": "OBJECT",
|
||||
"properties": {
|
||||
"receiver": {"type": "STRING", "description": "Recipient contact name"},
|
||||
"message_text": {"type": "STRING", "description": "The message to send"},
|
||||
"message_text": {"type": "STRING", "description": "The exact message text to send"},
|
||||
"platform": {"type": "STRING", "description": "Platform: WhatsApp, Telegram, etc."}
|
||||
},
|
||||
"required": ["receiver", "message_text", "platform"]
|
||||
@@ -420,9 +444,16 @@ TOOL_DECLARATIONS = [
|
||||
{
|
||||
"name": "save_memory",
|
||||
"description": (
|
||||
"Save an important personal fact about the user to long-term memory. "
|
||||
"Call this silently whenever the user reveals something worth remembering. "
|
||||
"Do NOT announce that you are saving — just call it silently."
|
||||
"Save a personal fact about the user to permanent long-term memory. "
|
||||
"MANDATORY: call this IMMEDIATELY (without asking) whenever the user states or corrects: "
|
||||
"their name, age, city, job, school, language, nationality, a preference, a goal, or a relationship. "
|
||||
"Examples: "
|
||||
"'my name is Fatih' → (identity, name, Fatih) | "
|
||||
"'not Travis, Fatih' → (identity, name, Fatih) | "
|
||||
"'I am 22' → (identity, age, 22) | "
|
||||
"'I live in Ankara' → (identity, city, Ankara) | "
|
||||
"'I prefer dark mode' → (preferences, ui_theme, dark mode). "
|
||||
"Call SILENTLY alongside your verbal reply — never announce that you are saving."
|
||||
),
|
||||
"parameters": {
|
||||
"type": "OBJECT",
|
||||
@@ -430,10 +461,15 @@ TOOL_DECLARATIONS = [
|
||||
"category": {
|
||||
"type": "STRING",
|
||||
"description": (
|
||||
"identity | preferences | projects | relationships | wishes | notes"
|
||||
"identity (name/age/city/job/school/nationality) | "
|
||||
"preferences (likes/dislikes/habits) | "
|
||||
"projects (active work/goals) | "
|
||||
"relationships (people in their life) | "
|
||||
"wishes (future plans/wants) | "
|
||||
"notes (anything else)"
|
||||
)
|
||||
},
|
||||
"key": {"type": "STRING", "description": "Short snake_case key"},
|
||||
"key": {"type": "STRING", "description": "Short snake_case key, e.g. 'name', 'age', 'favorite_color'"},
|
||||
"value": {"type": "STRING", "description": "Concise value in English"},
|
||||
},
|
||||
"required": ["category", "key", "value"]
|
||||
@@ -525,42 +561,51 @@ class _VADBuffer:
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
sample_rate: int = 16_000,
|
||||
silence_sec: float = 1.2,
|
||||
threshold: float = 0.015,
|
||||
sample_rate: int = 16_000,
|
||||
silence_sec: float = 1.0, # silence after last word → send to STT (1.0s = fast enough + won't cut mid-sentence with hysteresis)
|
||||
speech_thresh: float = 0.015, # RMS above this = speech (raised from 0.010 to kill background noise false-positives)
|
||||
silence_thresh: float = 0.008, # RMS below this = silence (hysteresis gap — much lower bar than speech_thresh to avoid mid-sentence cuts)
|
||||
min_speech_sec: float = 0.4,
|
||||
max_speech_sec: float = 30.0,
|
||||
):
|
||||
self._sr = sample_rate
|
||||
self._sil_n = int(silence_sec * sample_rate)
|
||||
self._thresh = threshold
|
||||
self._min_n = int(min_speech_sec * sample_rate)
|
||||
self._max_n = int(max_speech_sec * sample_rate)
|
||||
self._buf: list[np.ndarray] = []
|
||||
self._in_spch = False
|
||||
self._sil_cnt = 0
|
||||
|
||||
self._sr = sample_rate
|
||||
self._sil_n = int(silence_sec * sample_rate)
|
||||
self._speech_thresh = speech_thresh
|
||||
self._sil_thresh = silence_thresh
|
||||
self._min_n = int(min_speech_sec * sample_rate)
|
||||
self._max_n = int(max_speech_sec * sample_rate)
|
||||
self._buf: list[np.ndarray] = []
|
||||
self._in_spch = False
|
||||
self._sil_cnt = 0
|
||||
def process(self, chunk: np.ndarray) -> np.ndarray | None:
|
||||
"""
|
||||
Feed one audio chunk (float32 mono).
|
||||
Returns complete utterance when speech ends, otherwise None.
|
||||
"""
|
||||
rms = float(np.sqrt(np.mean(chunk ** 2)))
|
||||
total_n = sum(len(c) for c in self._buf)
|
||||
|
||||
if rms > self._thresh:
|
||||
Uses hysteresis thresholds so the detector doesn't flicker on quiet
|
||||
consonants or brief mid-sentence pauses:
|
||||
- speech starts when RMS > speech_thresh
|
||||
- speech ends only when RMS < silence_thresh (lower bar)
|
||||
This prevents natural pauses ("I just... wanted to say") from
|
||||
triggering a premature cut.
|
||||
"""
|
||||
rms = float(np.sqrt(np.mean(chunk ** 2)))
|
||||
total_n = sum(len(c) for c in self._buf)
|
||||
|
||||
if rms > self._speech_thresh:
|
||||
self._in_spch = True
|
||||
self._sil_cnt = 0
|
||||
self._buf.append(chunk.copy())
|
||||
elif self._in_spch:
|
||||
self._buf.append(chunk.copy())
|
||||
self._sil_cnt += len(chunk)
|
||||
if rms < self._sil_thresh:
|
||||
self._sil_cnt += len(chunk)
|
||||
|
||||
if self._sil_cnt >= self._sil_n or total_n >= self._max_n:
|
||||
audio = np.concatenate(self._buf)
|
||||
self._buf = []
|
||||
self._in_spch = False
|
||||
self._sil_cnt = 0
|
||||
audio = np.concatenate(self._buf)
|
||||
self._buf = []
|
||||
self._in_spch = False
|
||||
self._sil_cnt = 0
|
||||
if len(audio) >= self._min_n:
|
||||
return audio
|
||||
return None
|
||||
@@ -582,6 +627,7 @@ class JarvisLocal:
|
||||
self._config = _load_config()
|
||||
self._stt = None
|
||||
self._tts = None
|
||||
self._tts_ready = threading.Event() # set when TTS engine is loaded
|
||||
self._speaking = False
|
||||
self._speaking_lock = threading.Lock()
|
||||
self._text_queue: queue.Queue = queue.Queue()
|
||||
@@ -595,20 +641,28 @@ class JarvisLocal:
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _build_system_prompt(self) -> str:
|
||||
# ── ORDER MATTERS for Ollama KV prefix caching ─────────────────────
|
||||
# Ollama caches the KV attention state of any stable prompt prefix.
|
||||
# By putting the STATIC JARVIS protocol text FIRST, Ollama reuses its
|
||||
# cached KV for all those tokens on every request. Only the small
|
||||
# dynamic tail (memory + time, ~50-80 tokens) needs re-evaluation.
|
||||
# This turns a 17-second first-token into a sub-second one after warmup.
|
||||
#
|
||||
# Rule: static content first → semi-static memory middle → dynamic time LAST.
|
||||
sys_p = _load_system_prompt() # static — never changes mid-session
|
||||
memory = load_memory()
|
||||
mem_str = format_memory_for_prompt(memory)
|
||||
sys_p = _load_system_prompt()
|
||||
mem_str = format_memory_for_prompt(memory) # semi-static — changes only when user tells facts
|
||||
now = datetime.now()
|
||||
time_ctx = (
|
||||
f"[CURRENT DATE & TIME]\n"
|
||||
f"Right now it is: {now.strftime('%A, %B %d, %Y — %I:%M %p')}\n"
|
||||
f"Use this to calculate exact times for reminders.\n\n"
|
||||
f"Use this to calculate exact times for reminders."
|
||||
)
|
||||
parts = [time_ctx]
|
||||
parts = [sys_p]
|
||||
if mem_str:
|
||||
parts.append(mem_str)
|
||||
parts.append(sys_p)
|
||||
return "\n".join(parts)
|
||||
parts.append(time_ctx)
|
||||
return "\n\n".join(parts)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Speaking state & TTS
|
||||
@@ -619,38 +673,22 @@ class JarvisLocal:
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _tts_worker(self) -> None:
|
||||
# Block until TTS engine is loaded. Queued items are preserved
|
||||
# and played immediately once loading completes — nothing is lost.
|
||||
self._tts_ready.wait(timeout=120)
|
||||
|
||||
while True:
|
||||
text = self._tts_queue.get()
|
||||
n_got = 1 # how many queue items we consumed this round
|
||||
|
||||
try:
|
||||
if text and self._tts:
|
||||
# ── Kokoro: merge buffered sentences into one inference call ──
|
||||
# Each separate Kokoro call costs ~0.5–1 s startup.
|
||||
# Draining the queue here combines "Hello. How are you?" into
|
||||
# one call instead of two, eliminating the gap between them.
|
||||
from core.tts import KokoroTTSEngine
|
||||
engine = getattr(self._tts, "_engine", None)
|
||||
if isinstance(engine, KokoroTTSEngine):
|
||||
while True:
|
||||
try:
|
||||
nxt = self._tts_queue.get_nowait()
|
||||
if nxt:
|
||||
text = text.rstrip() + " " + nxt.lstrip()
|
||||
n_got += 1
|
||||
except queue.Empty:
|
||||
break
|
||||
|
||||
with self._speaking_lock:
|
||||
self._speaking = True
|
||||
self.ui.set_state("SPEAKING")
|
||||
self._tts.speak(text)
|
||||
|
||||
except Exception as e:
|
||||
print(f"[TTS] speak error: {e}")
|
||||
finally:
|
||||
for _ in range(n_got):
|
||||
self._tts_queue.task_done()
|
||||
self._tts_queue.task_done()
|
||||
if self._tts_queue.empty():
|
||||
with self._speaking_lock:
|
||||
self._speaking = False
|
||||
@@ -704,6 +742,7 @@ class JarvisLocal:
|
||||
try:
|
||||
from core.tts import create_tts_player
|
||||
self._tts = create_tts_player(new_config)
|
||||
self._tts_ready.set() # ensure worker isn't blocked
|
||||
self.ui.write_log("SYS: TTS reconfigured.")
|
||||
except Exception as e:
|
||||
self.ui.write_log(f"ERR: TTS reconfigure — {e}")
|
||||
@@ -886,15 +925,16 @@ class JarvisLocal:
|
||||
Full turn:
|
||||
user_text → Ollama stream (with tools) → execute tools → final spoken response
|
||||
|
||||
Streaming means TTS starts on the first complete sentence (~1-2 s),
|
||||
not after the full response is ready.
|
||||
The complete LLM response is collected first, then sent to TTS in one call.
|
||||
This gives TTS engines (Kokoro, ElevenLabs) the full text so they can
|
||||
apply natural prosody across sentence boundaries — no choppy pauses.
|
||||
"""
|
||||
self.ui.set_state("THINKING")
|
||||
self.ui.write_log(f"You: {user_text}")
|
||||
|
||||
self._conversation.append({"role": "user", "content": user_text})
|
||||
|
||||
MAX_HISTORY = 30
|
||||
MAX_HISTORY = 10
|
||||
if len(self._conversation) > MAX_HISTORY:
|
||||
self._conversation = self._conversation[-MAX_HISTORY:]
|
||||
|
||||
@@ -909,9 +949,7 @@ class JarvisLocal:
|
||||
|
||||
try:
|
||||
for event in call_llm_stream(messages, OLLAMA_TOOLS):
|
||||
if event["type"] == "sentence":
|
||||
self.speak(event["text"])
|
||||
elif event["type"] == "done":
|
||||
if event["type"] == "done":
|
||||
final_content = event["content"]
|
||||
final_tool_calls = event["tool_calls"]
|
||||
except RuntimeError as e:
|
||||
@@ -919,12 +957,13 @@ class JarvisLocal:
|
||||
return
|
||||
|
||||
if not final_tool_calls:
|
||||
# Final text response — TTS already started sentence-by-sentence
|
||||
# Full response ready — send complete text to TTS in one call
|
||||
if final_content:
|
||||
assistant_msg = {"role": "assistant", "content": final_content}
|
||||
messages.append(assistant_msg)
|
||||
self._conversation.append(assistant_msg)
|
||||
self.ui.write_log(f"Jarvis: {final_content}")
|
||||
self.speak(final_content)
|
||||
break
|
||||
|
||||
# ------- tool calls -------
|
||||
@@ -936,6 +975,33 @@ class JarvisLocal:
|
||||
messages.append(assistant_msg)
|
||||
self._conversation.append(assistant_msg)
|
||||
|
||||
# ── Fast path: save_memory-only round with verbal content ──────
|
||||
# When the LLM calls ONLY save_memory (silent) AND also returned
|
||||
# verbal content in the same response, speak the content directly
|
||||
# instead of burning another full LLM round just to get a reply.
|
||||
_only_memory = all(
|
||||
tc.get("function", {}).get("name") == "save_memory"
|
||||
for tc in final_tool_calls
|
||||
)
|
||||
if _only_memory and final_content:
|
||||
# Execute the memory saves silently
|
||||
for tc in final_tool_calls:
|
||||
fn = tc.get("function", {})
|
||||
targs = fn.get("arguments", {})
|
||||
if isinstance(targs, str):
|
||||
try:
|
||||
targs = json.loads(targs)
|
||||
except Exception:
|
||||
targs = {}
|
||||
self._execute_tool("save_memory", targs)
|
||||
# Use the verbal content that came alongside the saves
|
||||
assistant_msg = {"role": "assistant", "content": final_content}
|
||||
messages.append(assistant_msg)
|
||||
self._conversation.append(assistant_msg)
|
||||
self.ui.write_log(f"Jarvis: {final_content}")
|
||||
self.speak(final_content)
|
||||
break
|
||||
|
||||
all_silent = True
|
||||
for tc in final_tool_calls:
|
||||
fn = tc.get("function", {})
|
||||
@@ -964,8 +1030,32 @@ class JarvisLocal:
|
||||
messages.append(tool_msg)
|
||||
self._conversation.append(tool_msg)
|
||||
|
||||
# Even if all tools were silent (e.g. save_memory only),
|
||||
# always do one more LLM round so the assistant can verbally respond.
|
||||
# ── Fast-ack for save_memory-only rounds ─────────────────────────
|
||||
# When EVERY tool call was a silent save_memory, skip the extra LLM
|
||||
# round entirely. A brief instant acknowledgment feels better than
|
||||
# waiting 2-3 s for an unnecessary follow-up inference.
|
||||
if all_silent:
|
||||
# Extract the saved name (if any) for a personalised ack.
|
||||
_saved_name: str | None = None
|
||||
for _tc in final_tool_calls:
|
||||
_fn = _tc.get("function", {})
|
||||
if _fn.get("name") == "save_memory":
|
||||
_a = _fn.get("arguments", {})
|
||||
if isinstance(_a, str):
|
||||
try:
|
||||
_a = json.loads(_a)
|
||||
except Exception:
|
||||
_a = {}
|
||||
if isinstance(_a, dict) and _a.get("key") == "name" and _a.get("value"):
|
||||
_saved_name = str(_a["value"])
|
||||
break
|
||||
_ack = f"Got it, {_saved_name}." if _saved_name else "Noted."
|
||||
_amsg = {"role": "assistant", "content": _ack}
|
||||
messages.append(_amsg)
|
||||
self._conversation.append(_amsg)
|
||||
self.ui.write_log(f"Jarvis: {_ack}")
|
||||
self.speak(_ack)
|
||||
break
|
||||
|
||||
if not self.ui.muted:
|
||||
self.ui.set_state("LISTENING")
|
||||
@@ -1064,78 +1154,114 @@ class JarvisLocal:
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def run(self) -> None:
|
||||
"""Initialise STT + TTS, then start the main listen loop."""
|
||||
"""
|
||||
Startup strategy — optimised for minimum time-to-interactive:
|
||||
|
||||
1. LLM warmup + STT load → parallel, fast (~3s)
|
||||
2. TTS load → parallel, slow (~20s for Kokoro)
|
||||
3. Wait only for (1) → go online immediately
|
||||
4. TTS finishes in BG → queued speech plays automatically
|
||||
"""
|
||||
try:
|
||||
# -- Wire reconfigure callback --
|
||||
self.ui.on_reconfigure = self.reconfigure
|
||||
|
||||
# -- Ollama auto-start --
|
||||
# ── Ollama ────────────────────────────────────────────────────
|
||||
from core.llm_client import ensure_ollama_running, warmup_model
|
||||
self.ui.write_log("SYS: Checking Ollama…")
|
||||
if ensure_ollama_running():
|
||||
self.ui.write_log("SYS: Ollama OK.")
|
||||
else:
|
||||
self.ui.write_log("ERR: Ollama unavailable. Start it manually: ollama serve")
|
||||
self.speak("Ollama is not running. Please start it manually.")
|
||||
self.ui.write_log("ERR: Ollama unavailable — run: ollama serve")
|
||||
|
||||
# -- LLM warmup (parallel with STT/TTS loading) --
|
||||
_warmup_done = threading.Event()
|
||||
def _do_warmup():
|
||||
try:
|
||||
warmup_model()
|
||||
finally:
|
||||
_warmup_done.set()
|
||||
threading.Thread(target=_do_warmup, daemon=True).start()
|
||||
|
||||
# -- STT --
|
||||
# ── Config ────────────────────────────────────────────────────
|
||||
stt_engine = self._config.get("stt_engine", "whisper").lower()
|
||||
stt_language = self._config.get("stt_language", "auto")
|
||||
stt_model = self._config.get("stt_model", "base")
|
||||
self.ui.write_log(f"SYS: Loading {stt_engine.upper()} STT…")
|
||||
if stt_engine == "whisper":
|
||||
self.ui.write_log(f"SYS: Whisper '{stt_model}' — downloading if not cached…")
|
||||
elif stt_engine == "vosk":
|
||||
self.ui.write_log("SYS: Vosk — downloading model if not cached…")
|
||||
stt_model = self._config.get("stt_model", "base")
|
||||
tts_engine = self._config.get("tts_engine", "edgetts").lower()
|
||||
|
||||
if stt_engine == "vosk":
|
||||
from core.stt import VoskSTT
|
||||
self._stt = VoskSTT(
|
||||
self._config.get("vosk_model_path"),
|
||||
language=stt_language,
|
||||
)
|
||||
else:
|
||||
from core.stt import WhisperSTT
|
||||
self._stt = WhisperSTT(stt_model, language=stt_language)
|
||||
# ── Startup progress panel ────────────────────────────────────
|
||||
self.ui.show_startup_panel()
|
||||
|
||||
self.ui.write_log("SYS: STT ready.")
|
||||
_warmup_done = threading.Event()
|
||||
_stt_done = threading.Event()
|
||||
|
||||
# -- TTS --
|
||||
tts_engine = self._config.get("tts_engine", "edgetts").lower()
|
||||
self.ui.write_log(f"SYS: Loading {tts_engine.upper()} TTS…")
|
||||
if tts_engine == "kokoro":
|
||||
self.ui.write_log("SYS: Kokoro — downloading model (~330 MB) if not cached…")
|
||||
from core.tts import create_tts_player
|
||||
self._tts = create_tts_player(self._config)
|
||||
self.ui.write_log("SYS: TTS ready.")
|
||||
# ── LLM warmup thread ─────────────────────────────────────────
|
||||
def _do_warmup():
|
||||
try:
|
||||
# Pass the STATIC system prompt so Ollama evaluates and caches
|
||||
# its KV state during startup. Real requests start with the same
|
||||
# static prefix → Ollama reuses cached KV → first token <1 s
|
||||
# instead of the ~17 s it takes to re-evaluate 300+ tokens cold.
|
||||
static_prompt = _load_system_prompt()
|
||||
warmup_model(system_prompt=static_prompt)
|
||||
self.ui.write_log("SYS: LLM ready.")
|
||||
self.ui.mark_startup_ready("llm")
|
||||
except Exception as e:
|
||||
self.ui.write_log(f"ERR: LLM warmup — {e}")
|
||||
self.ui.mark_startup_ready("llm", error=True)
|
||||
finally:
|
||||
_warmup_done.set()
|
||||
|
||||
# -- Wait for LLM warmup to finish --
|
||||
self.ui.write_log("SYS: Loading LLM into memory…")
|
||||
_warmup_done.wait(timeout=120)
|
||||
self.ui.write_log("SYS: LLM ready.")
|
||||
# ── STT load thread ───────────────────────────────────────────
|
||||
def _do_stt():
|
||||
try:
|
||||
self.ui.write_log(f"SYS: Loading {stt_engine.upper()} STT…")
|
||||
if stt_engine == "vosk":
|
||||
from core.stt import VoskSTT
|
||||
self._stt = VoskSTT(
|
||||
self._config.get("vosk_model_path"),
|
||||
language=stt_language,
|
||||
)
|
||||
else:
|
||||
from core.stt import WhisperSTT
|
||||
self._stt = WhisperSTT(stt_model, language=stt_language)
|
||||
self.ui.write_log("SYS: STT ready.")
|
||||
self.ui.mark_startup_ready("stt")
|
||||
except Exception as e:
|
||||
self.ui.write_log(f"ERR: STT — {e}")
|
||||
self.ui.mark_startup_ready("stt", error=True)
|
||||
finally:
|
||||
_stt_done.set()
|
||||
|
||||
# Welcome
|
||||
# ── TTS load thread — does NOT block going online ─────────────
|
||||
def _do_tts():
|
||||
try:
|
||||
self.ui.write_log(f"SYS: Loading {tts_engine.upper()} TTS…")
|
||||
if tts_engine == "kokoro":
|
||||
self.ui.write_log("SYS: Kokoro — loading model + compiling JIT…")
|
||||
from core.tts import create_tts_player
|
||||
self._tts = create_tts_player(self._config)
|
||||
self._tts_ready.set() # unblock _tts_worker
|
||||
self.ui.write_log("SYS: TTS ready.")
|
||||
self.ui.mark_startup_ready("tts")
|
||||
self.ui.set_startup_status("● All systems ready.")
|
||||
self.ui.hide_startup_panel()
|
||||
self.speak("Jarvis fully online.")
|
||||
except Exception as e:
|
||||
import traceback as _tb; _tb.print_exc()
|
||||
self.ui.write_log(f"ERR: TTS — {e}")
|
||||
self.ui.mark_startup_ready("tts", error=True)
|
||||
self._tts_ready.set()
|
||||
|
||||
# Launch all three simultaneously
|
||||
self.ui.write_log("SYS: Loading systems in parallel…")
|
||||
threading.Thread(target=_do_warmup, daemon=True).start()
|
||||
threading.Thread(target=_do_stt, daemon=True).start()
|
||||
threading.Thread(target=_do_tts, daemon=True).start()
|
||||
|
||||
# ── Wait ONLY for STT + LLM (fast) ────────────────────────────
|
||||
_warmup_done.wait(timeout=60)
|
||||
_stt_done.wait(timeout=60)
|
||||
|
||||
# ── Go online immediately ──────────────────────────────────────
|
||||
self.ui.write_log("SYS: JARVIS online.")
|
||||
self.ui.set_state("LISTENING")
|
||||
self.ui.set_startup_status("● JARVIS online · Voice loading in background…")
|
||||
|
||||
# TTS queue worker (sequential, no overlap)
|
||||
threading.Thread(target=self._tts_worker, daemon=True).start()
|
||||
threading.Thread(target=self._tts_worker, daemon=True).start()
|
||||
threading.Thread(target=self._text_command_loop, daemon=True).start()
|
||||
|
||||
self.speak("JARVIS online. All systems ready.")
|
||||
|
||||
# Text-command handler (runs in background)
|
||||
threading.Thread(target=self._text_command_loop, daemon=True).start()
|
||||
|
||||
# STT loop (blocking — keeps this thread alive)
|
||||
# STT loop — blocks this thread forever
|
||||
if stt_engine == "vosk":
|
||||
self._listen_vosk()
|
||||
else:
|
||||
@@ -1151,6 +1277,17 @@ class JarvisLocal:
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def main() -> None:
|
||||
# ── Pre-import torch in background immediately ─────────────────────────
|
||||
# By the time the TTS thread starts (~5s from now), torch will already
|
||||
# be in sys.modules — removing it from the TTS critical path entirely.
|
||||
def _preload_torch():
|
||||
try:
|
||||
import torch # noqa: F401 (side-effect import only)
|
||||
except Exception:
|
||||
pass
|
||||
threading.Thread(target=_preload_torch, daemon=True).start()
|
||||
# ───────────────────────────────────────────────────────────────────────
|
||||
|
||||
ui = JarvisUI("face.png")
|
||||
|
||||
def runner():
|
||||
@@ -1186,6 +1323,5 @@ def main() -> None:
|
||||
threading.Thread(target=runner, daemon=True).start()
|
||||
ui.root.mainloop()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -884,8 +884,9 @@ class SetupOverlay(QWidget):
|
||||
QLineEdit:focus {{ border: 1px solid {C.PRI}; }}
|
||||
"""
|
||||
|
||||
self._sel_stt = _init.get("stt_engine", "whisper")
|
||||
self._sel_tts = _init.get("tts_engine", "edgetts")
|
||||
self._sel_stt = _init.get("stt_engine", "whisper")
|
||||
self._sel_tts = _init.get("tts_engine", "edgetts")
|
||||
self._sel_llm_provider = _init.get("llm_provider", "ollama")
|
||||
|
||||
layout = QVBoxLayout(self)
|
||||
layout.setContentsMargins(22, 16, 22, 16)
|
||||
@@ -1018,17 +1019,44 @@ class SetupOverlay(QWidget):
|
||||
layout.addWidget(_sep())
|
||||
|
||||
# ── LLM ──────────────────────────────────────────────────────── #
|
||||
layout.addWidget(_lbl("LOCAL LLM (Ollama)", 7, col=C.TEXT_DIM,
|
||||
layout.addWidget(_lbl("LOCAL LLM", 7, col=C.TEXT_DIM,
|
||||
align=Qt.AlignmentFlag.AlignLeft))
|
||||
|
||||
# Provider toggle: Ollama vs LM Studio / OpenAI-compatible
|
||||
llm_prov_row, self._llm_prov_btns = _toggle_row(
|
||||
[
|
||||
("ollama", "🦙 Ollama"),
|
||||
("openai", "🔌 LM Studio / OpenAI"),
|
||||
],
|
||||
lambda: self._sel_llm_provider,
|
||||
self._set_llm_provider,
|
||||
)
|
||||
layout.addLayout(llm_prov_row)
|
||||
|
||||
# Hint label — changes based on provider
|
||||
_ollama_hint = "ollama.com · run: ollama pull qwen2.5:3b"
|
||||
_openai_hint = "lmstudio.ai · start Local Server first, then pick model"
|
||||
self._llm_hint_lbl = _lbl(
|
||||
_openai_hint if self._sel_llm_provider == "openai" else _ollama_hint,
|
||||
7, col=C.TEXT_DIM, align=Qt.AlignmentFlag.AlignLeft
|
||||
)
|
||||
layout.addWidget(self._llm_hint_lbl)
|
||||
|
||||
llm_row = QHBoxLayout(); llm_row.setSpacing(5)
|
||||
llm_row.addWidget(_lbl("URL:", 7, col=C.TEXT_MED,
|
||||
align=Qt.AlignmentFlag.AlignRight))
|
||||
self._llm_url_input = _input("http://localhost:11434")
|
||||
self._llm_url_input.setText(_init.get("llm_url", "http://localhost:11434"))
|
||||
_default_url = _init.get("llm_url",
|
||||
"http://localhost:1234" if self._sel_llm_provider == "openai"
|
||||
else "http://localhost:11434")
|
||||
self._llm_url_input = _input(
|
||||
"http://localhost:1234" if self._sel_llm_provider == "openai"
|
||||
else "http://localhost:11434"
|
||||
)
|
||||
self._llm_url_input.setText(_default_url)
|
||||
llm_row.addWidget(self._llm_url_input, stretch=2)
|
||||
llm_row.addWidget(_lbl("Model:", 7, col=C.TEXT_MED,
|
||||
align=Qt.AlignmentFlag.AlignRight))
|
||||
self._llm_model_input = _input("e.g. llama3.2 / qwen2.5 / mistral")
|
||||
self._llm_model_input = _input("e.g. qwen2.5:3b / llama3.2 / mistral")
|
||||
self._llm_model_input.setText(_init.get("llm_model", ""))
|
||||
llm_row.addWidget(self._llm_model_input, stretch=2)
|
||||
layout.addLayout(llm_row)
|
||||
@@ -1201,6 +1229,40 @@ class SetupOverlay(QWidget):
|
||||
if hasattr(self, "_el_key_widget"):
|
||||
self._el_key_widget.setVisible(key == "elevenlabs")
|
||||
|
||||
def _set_llm_provider(self, key: str):
|
||||
self._sel_llm_provider = key
|
||||
if not hasattr(self, "_llm_prov_btns"):
|
||||
return
|
||||
for k, btn in self._llm_prov_btns.items():
|
||||
active = (k == key)
|
||||
btn.setStyleSheet(f"""
|
||||
QPushButton {{
|
||||
background: {'#00d4ff' if active else '#000d12'};
|
||||
color: {'#001a22' if active else C.TEXT_DIM};
|
||||
border: {'none' if active else f'1px solid {C.BORDER}'};
|
||||
border-radius: 3px; font-weight: {'bold' if active else 'normal'};
|
||||
}}
|
||||
QPushButton:hover {{ color: {C.TEXT}; border: 1px solid {C.BORDER_B}; }}
|
||||
""")
|
||||
# Update URL placeholder and hint to match selected provider
|
||||
if hasattr(self, "_llm_url_input"):
|
||||
if key == "openai":
|
||||
self._llm_url_input.setPlaceholderText("http://localhost:1234")
|
||||
# Only override if it still looks like an Ollama default URL
|
||||
cur = self._llm_url_input.text().strip()
|
||||
if not cur or cur == "http://localhost:11434":
|
||||
self._llm_url_input.setText("http://localhost:1234")
|
||||
else:
|
||||
self._llm_url_input.setPlaceholderText("http://localhost:11434")
|
||||
cur = self._llm_url_input.text().strip()
|
||||
if not cur or cur == "http://localhost:1234":
|
||||
self._llm_url_input.setText("http://localhost:11434")
|
||||
if hasattr(self, "_llm_hint_lbl"):
|
||||
if key == "openai":
|
||||
self._llm_hint_lbl.setText("lmstudio.ai · start Local Server first, then pick model")
|
||||
else:
|
||||
self._llm_hint_lbl.setText("ollama.com · run: ollama pull qwen2.5:3b")
|
||||
|
||||
def _set_stt(self, key: str):
|
||||
self._sel_stt = key
|
||||
if not hasattr(self, "_stt_btns"):
|
||||
@@ -1262,11 +1324,14 @@ class SetupOverlay(QWidget):
|
||||
tts_voice = self._tts_voice_input.text().strip() or "en-US-GuyNeural"
|
||||
tts_speed = "1.0"
|
||||
|
||||
_provider = getattr(self, "_sel_llm_provider", "ollama")
|
||||
_default_url = "http://localhost:1234" if _provider == "openai" else "http://localhost:11434"
|
||||
cfg = {
|
||||
"stt_engine": self._sel_stt,
|
||||
"stt_model": stt_model,
|
||||
"stt_language": self._stt_lang_input.text().strip() or "auto",
|
||||
"llm_url": self._llm_url_input.text().strip() or "http://localhost:11434",
|
||||
"llm_provider": _provider,
|
||||
"llm_url": self._llm_url_input.text().strip() or _default_url,
|
||||
"llm_model": llm_model,
|
||||
"tts_engine": self._sel_tts,
|
||||
"tts_voice": tts_voice,
|
||||
@@ -1278,9 +1343,130 @@ class SetupOverlay(QWidget):
|
||||
self.done.emit(json.dumps(cfg))
|
||||
|
||||
|
||||
class StartupPanel(QWidget):
|
||||
"""Animated startup progress overlay — shown while components initialize."""
|
||||
|
||||
def __init__(self, parent=None):
|
||||
super().__init__(parent)
|
||||
self.setAttribute(Qt.WidgetAttribute.WA_StyledBackground, True)
|
||||
self.setStyleSheet(f"""
|
||||
StartupPanel {{
|
||||
background: rgba(0, 6, 10, 235);
|
||||
border: 1px solid {C.BORDER_B};
|
||||
border-radius: 8px;
|
||||
}}
|
||||
""")
|
||||
|
||||
lay = QVBoxLayout(self)
|
||||
lay.setContentsMargins(28, 20, 28, 20)
|
||||
lay.setSpacing(10)
|
||||
|
||||
# ── Title ──────────────────────────────────────────────────────
|
||||
title = QLabel("◈ SYSTEMS INITIALISING")
|
||||
title.setFont(QFont("Courier New", 11, QFont.Weight.Bold))
|
||||
title.setAlignment(Qt.AlignmentFlag.AlignCenter)
|
||||
title.setStyleSheet(f"color: {C.PRI}; background: transparent;")
|
||||
lay.addWidget(title)
|
||||
|
||||
lay.addSpacing(2)
|
||||
|
||||
# ── Component rows ──────────────────────────────────────────────
|
||||
self._rows: dict[str, dict] = {}
|
||||
_COMPS = [
|
||||
("stt", "SPEECH RECOGNITION (STT)", C.GREEN),
|
||||
("llm", "LANGUAGE MODEL (LLM)", C.ACC2),
|
||||
("tts", "VOICE SYNTHESIS (TTS)", C.PRI),
|
||||
]
|
||||
for key, label, color in _COMPS:
|
||||
box = QWidget()
|
||||
box.setStyleSheet(
|
||||
f"background: {C.PANEL2}; border: 1px solid {C.BORDER}; border-radius: 4px;"
|
||||
)
|
||||
box_lay = QVBoxLayout(box)
|
||||
box_lay.setContentsMargins(10, 6, 10, 6)
|
||||
box_lay.setSpacing(4)
|
||||
|
||||
top = QHBoxLayout()
|
||||
nm = QLabel(label)
|
||||
nm.setFont(QFont("Courier New", 8, QFont.Weight.Bold))
|
||||
nm.setStyleSheet(f"color: {C.TEXT_MED}; background: transparent; border: none;")
|
||||
top.addWidget(nm)
|
||||
top.addStretch()
|
||||
|
||||
st = QLabel("LOADING…")
|
||||
st.setFont(QFont("Courier New", 8, QFont.Weight.Bold))
|
||||
st.setStyleSheet(f"color: {C.TEXT_DIM}; background: transparent; border: none;")
|
||||
top.addWidget(st)
|
||||
box_lay.addLayout(top)
|
||||
|
||||
bar = QProgressBar()
|
||||
bar.setFixedHeight(4)
|
||||
bar.setRange(0, 0) # indeterminate marquee
|
||||
bar.setTextVisible(False)
|
||||
bar.setStyleSheet(f"""
|
||||
QProgressBar {{
|
||||
background: {C.BAR_BG}; border: none; border-radius: 2px;
|
||||
}}
|
||||
QProgressBar::chunk {{
|
||||
background: qlineargradient(x1:0,y1:0,x2:1,y2:0,
|
||||
stop:0 {C.BORDER}, stop:1 {color});
|
||||
border-radius: 2px; width: 60px; margin: 0px;
|
||||
}}
|
||||
""")
|
||||
box_lay.addWidget(bar)
|
||||
lay.addWidget(box)
|
||||
self._rows[key] = {"bar": bar, "status": st, "color": color}
|
||||
|
||||
lay.addSpacing(4)
|
||||
|
||||
# ── Bottom status ───────────────────────────────────────────────
|
||||
self._status_lbl = QLabel("Initialising components…")
|
||||
self._status_lbl.setFont(QFont("Courier New", 8))
|
||||
self._status_lbl.setAlignment(Qt.AlignmentFlag.AlignCenter)
|
||||
self._status_lbl.setStyleSheet(f"color: {C.TEXT_DIM}; background: transparent;")
|
||||
self._status_lbl.setWordWrap(True)
|
||||
lay.addWidget(self._status_lbl)
|
||||
|
||||
tip = QLabel("All AI models run 100% locally · No data leaves your device")
|
||||
tip.setFont(QFont("Courier New", 7))
|
||||
tip.setAlignment(Qt.AlignmentFlag.AlignCenter)
|
||||
tip.setStyleSheet(f"color: {C.BORDER}; background: transparent;")
|
||||
lay.addWidget(tip)
|
||||
|
||||
# Called only from the main thread (via MainWindow._startup_sig)
|
||||
def update_component(self, key: str, status: str) -> None:
|
||||
if key not in self._rows:
|
||||
return
|
||||
row = self._rows[key]
|
||||
ok = status == "ready"
|
||||
color = row["color"] if ok else C.RED
|
||||
label = "READY ✓" if ok else "ERROR ✗"
|
||||
|
||||
bar = row["bar"]
|
||||
bar.setRange(0, 100)
|
||||
bar.setValue(100)
|
||||
bar.setStyleSheet(f"""
|
||||
QProgressBar {{
|
||||
background: {C.BAR_BG}; border: none; border-radius: 2px;
|
||||
}}
|
||||
QProgressBar::chunk {{
|
||||
background: {color}; border-radius: 2px;
|
||||
}}
|
||||
""")
|
||||
st = row["status"]
|
||||
st.setText(label)
|
||||
st.setStyleSheet(f"color: {color}; background: transparent; border: none;")
|
||||
|
||||
def set_status(self, text: str) -> None:
|
||||
self._status_lbl.setText(text)
|
||||
col = C.GREEN if "online" in text.lower() else C.TEXT_DIM
|
||||
self._status_lbl.setStyleSheet(f"color: {col}; background: transparent;")
|
||||
|
||||
|
||||
class MainWindow(QMainWindow):
|
||||
_log_sig = pyqtSignal(str)
|
||||
_state_sig = pyqtSignal(str)
|
||||
_log_sig = pyqtSignal(str)
|
||||
_state_sig = pyqtSignal(str)
|
||||
_startup_sig = pyqtSignal(str, str) # action, data — thread-safe startup panel control
|
||||
|
||||
def __init__(self, face_path: str):
|
||||
super().__init__()
|
||||
@@ -1337,8 +1523,10 @@ class MainWindow(QMainWindow):
|
||||
|
||||
self._log_sig.connect(self._log.append_log)
|
||||
self._state_sig.connect(self._apply_state)
|
||||
self._startup_sig.connect(self._on_startup_sig)
|
||||
|
||||
self._overlay: SetupOverlay | None = None
|
||||
self._startup_panel: StartupPanel | None = None
|
||||
self._on_reconfigure_cb = None
|
||||
self._ready = self._check_config()
|
||||
if not self._ready:
|
||||
@@ -1357,14 +1545,58 @@ class MainWindow(QMainWindow):
|
||||
|
||||
def resizeEvent(self, event):
|
||||
super().resizeEvent(event)
|
||||
cw = self.centralWidget()
|
||||
if self._overlay and self._overlay.isVisible():
|
||||
ow, oh = 520, 580
|
||||
cw = self.centralWidget()
|
||||
self._overlay.setGeometry(
|
||||
(cw.width() - ow) // 2,
|
||||
(cw.height() - oh) // 2,
|
||||
ow, oh,
|
||||
)
|
||||
if self._startup_panel and self._startup_panel.isVisible():
|
||||
pw, ph = 400, 310
|
||||
self._startup_panel.setGeometry(
|
||||
(cw.width() - pw) // 2,
|
||||
(cw.height() - ph) // 2,
|
||||
pw, ph,
|
||||
)
|
||||
|
||||
# ── Startup panel (thread-safe via _startup_sig) ────────────────────
|
||||
def _on_startup_sig(self, action: str, data: str) -> None:
|
||||
"""Runs on main thread — handles all startup panel state changes."""
|
||||
if action == "show":
|
||||
self._create_startup_panel()
|
||||
elif action in ("ready", "error"):
|
||||
if self._startup_panel:
|
||||
self._startup_panel.update_component(data, action)
|
||||
elif action == "status":
|
||||
if self._startup_panel:
|
||||
self._startup_panel.set_status(data)
|
||||
elif action == "hide":
|
||||
if self._startup_panel:
|
||||
# Fade out after a short pause so "READY ✓" is visible
|
||||
QTimer.singleShot(1200, self._destroy_startup_panel)
|
||||
|
||||
def _create_startup_panel(self) -> None:
|
||||
if self._startup_panel and self._startup_panel.isVisible():
|
||||
return
|
||||
cw = self.centralWidget()
|
||||
pw, ph = 400, 310
|
||||
panel = StartupPanel(cw)
|
||||
panel.setGeometry(
|
||||
(cw.width() - pw) // 2,
|
||||
(cw.height() - ph) // 2,
|
||||
pw, ph,
|
||||
)
|
||||
panel.show()
|
||||
panel.raise_()
|
||||
self._startup_panel = panel
|
||||
|
||||
def _destroy_startup_panel(self) -> None:
|
||||
if self._startup_panel:
|
||||
self._startup_panel.hide()
|
||||
self._startup_panel.deleteLater()
|
||||
self._startup_panel = None
|
||||
|
||||
def _update_metrics(self):
|
||||
snap = _metrics.snapshot()
|
||||
@@ -1716,7 +1948,6 @@ class MainWindow(QMainWindow):
|
||||
txt = self._input.text().strip()
|
||||
if not txt: return
|
||||
self._input.clear()
|
||||
self._log.append_log(f"You: {txt}")
|
||||
if self.on_text_command:
|
||||
threading.Thread(target=self.on_text_command, args=(txt,), daemon=True).start()
|
||||
|
||||
@@ -1861,6 +2092,19 @@ class JarvisUI:
|
||||
def write_log(self, text: str):
|
||||
self._win._log_sig.emit(text)
|
||||
|
||||
# ── Startup panel (all thread-safe) ──────────────────────────────────
|
||||
def show_startup_panel(self) -> None:
|
||||
self._win._startup_sig.emit("show", "")
|
||||
|
||||
def mark_startup_ready(self, key: str, error: bool = False) -> None:
|
||||
self._win._startup_sig.emit("error" if error else "ready", key)
|
||||
|
||||
def set_startup_status(self, text: str) -> None:
|
||||
self._win._startup_sig.emit("status", text)
|
||||
|
||||
def hide_startup_panel(self) -> None:
|
||||
self._win._startup_sig.emit("hide", "")
|
||||
|
||||
def wait_for_api_key(self):
|
||||
while not self._win._ready:
|
||||
time.sleep(0.1)
|
||||
|
||||
Reference in new issue
Block a user