From 5d8d214e9b98767d4d0d87e3a170b462ce45885a Mon Sep 17 00:00:00 2001 From: FatihMakes Date: Sun, 7 Jun 2026 17:03:02 +0300 Subject: [PATCH] Add files via upload --- core/llm_client.py | 9 ++-- core/prompt.txt | 7 ++- core/stt.py | 17 ++++++- core/tts.py | 22 ++++++++-- main.py | 107 ++++++++++++++++++++++++++++++--------------- 5 files changed, 117 insertions(+), 45 deletions(-) diff --git a/core/llm_client.py b/core/llm_client.py index 4c1c715..8643e44 100644 --- a/core/llm_client.py +++ b/core/llm_client.py @@ -212,7 +212,7 @@ def call_llm( "model": model, "messages": messages, "stream": False, - "max_tokens": 300, + "max_tokens": 150, } if tools: payload["tools"] = tools @@ -252,7 +252,7 @@ def call_llm( "messages": messages, "stream": False, "keep_alive": -1, - "options": {"num_predict": 300, "num_gpu": 99}, + "options": {"num_predict": 150, "num_gpu": 99}, } if tools: payload["tools"] = tools @@ -353,7 +353,7 @@ def _stream_openai( "model": model, "messages": messages, "stream": True, - "max_tokens": 300, + "max_tokens": 150, } if tools: payload["tools"] = tools @@ -479,8 +479,9 @@ def call_llm_stream( "messages": messages, "stream": True, "keep_alive": -1, + # 150 tokens ≈ 100 words ≈ 3-4 sentences — enough for any voice reply. # num_gpu:99 pushes all layers to GPU; num_thread removed (Ollama auto-tunes). - "options": {"num_predict": 300, "num_gpu": 99}, + "options": {"num_predict": 150, "num_gpu": 99}, } if tools: payload["tools"] = tools diff --git a/core/prompt.txt b/core/prompt.txt index ece5fbb..ee3acfd 100644 --- a/core/prompt.txt +++ b/core/prompt.txt @@ -12,7 +12,12 @@ EXECUTION RULES: Action Flow: Call the tool immediately. For slow tools (search, vision), say ONE short sentence first. Vision (screen_process): Say one curious sentence, then call the tool — total silence after. One-Call Policy: Never guess. Call tools exactly once. No retries. -Briefing: 1 short sentence max for confirmations/acknowledgments. 1-2 sentences for results. Never add "Is there anything else?" or filler questions. +Briefing: HARD LIMIT — 3 SHORT sentences maximum for ANY response. No exceptions. + • Confirmations / acks: 1 sentence only. + • Answers / results: 2-3 short sentences max. + • NEVER ask follow-up questions ("Would you like...?", "Is there anything else?"). + • NEVER pad with sympathy openers or filler phrases ("Of course!", "Great question!"). + • If you cannot fit the answer in 3 sentences, give the most important part only. Exit: Only call shutdown_jarvis if session termination is explicit. Response time: Respond as fast as you can. Always respond in the language that user is speaking. diff --git a/core/stt.py b/core/stt.py index dddce58..64efcaf 100644 --- a/core/stt.py +++ b/core/stt.py @@ -12,6 +12,7 @@ class WhisperSTT: """Offline transcription using faster-whisper.""" def __init__(self, model_name: str = "base", language: str | None = None): + import os from faster_whisper import WhisperModel print(f"[STT] Loading Whisper '{model_name}'…") try: @@ -20,7 +21,21 @@ class WhisperSTT: compute = "float16" if device == "cuda" else "int8" except Exception: device, compute = "cpu", "int8" - self._model = WhisperModel(model_name, device=device, compute_type=compute) + + try: + self._model = WhisperModel(model_name, device=device, compute_type=compute) + except Exception as _first_err: + # Offline flag set but model not cached yet → download once, then offline forever + _e = str(_first_err).lower() + if any(k in _e for k in ("offline", "not found", "cache", "localentry", "does not exist")): + print(f"[STT] '{model_name}' not cached — downloading (internet required for first run)…") + os.environ.pop("HF_HUB_OFFLINE", None) + os.environ.pop("TRANSFORMERS_OFFLINE", None) + os.environ.pop("HF_DATASETS_OFFLINE", None) + self._model = WhisperModel(model_name, device=device, compute_type=compute) + else: + raise + self._language = None if (not language or language.strip().lower() == "auto") else language.strip().lower() print(f"[STT] Whisper '{model_name}' ready ({device})") diff --git a/core/tts.py b/core/tts.py index 8643993..de591ad 100644 --- a/core/tts.py +++ b/core/tts.py @@ -264,11 +264,25 @@ class KokoroTTSEngine: KPipeline = _import_kokoro_pipeline() + def _create_pipeline(): + try: + return KPipeline(lang_code=lang, device=device) + except TypeError: + return KPipeline(lang_code=lang) # older build — no device param + try: - self._pipeline = KPipeline(lang_code=lang, device=device) - except TypeError: - # Older Kokoro build — no device parameter - self._pipeline = KPipeline(lang_code=lang) + self._pipeline = _create_pipeline() + except Exception as _first_err: + # Offline flag set but model not cached yet → download once + _e = str(_first_err).lower() + if any(k in _e for k in ("offline", "not found", "cache", "localentry", "does not exist")): + print("[TTS] Kokoro model not cached — downloading (internet required for first run)…") + os.environ.pop("HF_HUB_OFFLINE", None) + os.environ.pop("TRANSFORMERS_OFFLINE", None) + os.environ.pop("HF_DATASETS_OFFLINE", None) + self._pipeline = _create_pipeline() + else: + raise print("[TTS] Kokoro compiling (first-time only)…") # Warmup: compiles PyTorch JIT graph so first real speak() call is instant. diff --git a/main.py b/main.py index 0693a68..0502897 100644 --- a/main.py +++ b/main.py @@ -14,6 +14,13 @@ _os.environ.setdefault("GRPC_VERBOSITY", "ERROR") # from the namespace). Let transformers auto-detect the available backends. _os.environ.setdefault("USE_TF", "0") _os.environ.setdefault("TOKENIZERS_PARALLELISM", "false") +# Offline mode — use cached models, no HuggingFace network calls on startup. +# On first run the model isn't cached yet; tts.py / stt.py detect this and +# temporarily clear these flags to allow the one-time download, then they +# stay in effect for every subsequent launch (fully offline). +_os.environ.setdefault("HF_HUB_OFFLINE", "1") +_os.environ.setdefault("TRANSFORMERS_OFFLINE", "1") +_os.environ.setdefault("HF_DATASETS_OFFLINE", "1") import warnings as _warnings _warnings.filterwarnings("ignore", category=UserWarning) _warnings.filterwarnings("ignore", category=DeprecationWarning) @@ -562,10 +569,10 @@ class _VADBuffer: def __init__( self, sample_rate: int = 16_000, - silence_sec: float = 1.0, # silence after last word → send to STT (1.0s = fast enough + won't cut mid-sentence with hysteresis) - speech_thresh: float = 0.015, # RMS above this = speech (raised from 0.010 to kill background noise false-positives) - silence_thresh: float = 0.008, # RMS below this = silence (hysteresis gap — much lower bar than speech_thresh to avoid mid-sentence cuts) - min_speech_sec: float = 0.4, + silence_sec: float = 0.7, # silence after last word → send to STT + speech_thresh: float = 0.008, # RMS above this = speech (0.008 catches voice at 3-4 m; raise if mic picks up too much room noise) + silence_thresh: float = 0.004, # RMS below this = silence (half of speech_thresh — hysteresis prevents mid-sentence cuts) + min_speech_sec: float = 0.3, max_speech_sec: float = 30.0, ): self._sr = sample_rate @@ -582,12 +589,11 @@ class _VADBuffer: Feed one audio chunk (float32 mono). Returns complete utterance when speech ends, otherwise None. - Uses hysteresis thresholds so the detector doesn't flicker on quiet - consonants or brief mid-sentence pauses: - - speech starts when RMS > speech_thresh - - speech ends only when RMS < silence_thresh (lower bar) - This prevents natural pauses ("I just... wanted to say") from - triggering a premature cut. + Uses hysteresis thresholds so the detector doesn't flicker: + - speech starts when RMS > speech_thresh (0.008 = ~3-4 m range) + - speech ends only when RMS < silence_thresh (0.004 = half of start) + The gap between the two thresholds prevents mid-sentence cuts on + natural pauses and quiet consonants. """ rms = float(np.sqrt(np.mean(chunk ** 2))) total_n = sum(len(c) for c in self._buf) @@ -922,12 +928,16 @@ class JarvisLocal: def _process_message(self, user_text: str) -> None: """ - Full turn: - user_text → Ollama stream (with tools) → execute tools → final spoken response + Full turn: user_text → LLM stream → TTS (overlapped) → tool execution - The complete LLM response is collected first, then sent to TTS in one call. - This gives TTS engines (Kokoro, ElevenLabs) the full text so they can - apply natural prosody across sentence boundaries — no choppy pauses. + Streaming TTS: sentence events are piped to the TTS queue AS they + arrive from the LLM, so Kokoro starts synthesising sentence 1 while + the LLM is still generating sentence 2. This cuts perceived latency + from (LLM_total + TTS_total) down to roughly max(LLM_total, TTS_total). + + Tool-call responses never emit sentence events, so the TTS overlap + only kicks in for pure conversational replies — which is exactly when + it matters most. """ self.ui.set_state("THINKING") self.ui.write_log(f"You: {user_text}") @@ -942,23 +952,44 @@ class JarvisLocal: {"role": "system", "content": self._build_system_prompt()} ] + list(self._conversation) + # Tools whose output needs a second LLM round to summarise/interpret. + # Everything else returns a user-ready string → speak directly. + _NEEDS_LLM_ROUND = {"web_search", "screen_process", "agent_task"} + MAX_TOOL_ROUNDS = 6 for _round in range(MAX_TOOL_ROUNDS): final_content = "" final_tool_calls: list = [] + # Sentences already queued to TTS during streaming (may be empty + # for tool-call rounds where the model emits no content). + _streamed: list[str] = [] try: for event in call_llm_stream(messages, OLLAMA_TOOLS): - if event["type"] == "done": + if event["type"] == "sentence": + # ── Overlap TTS with LLM generation ───────────────── + # Queue this sentence immediately; the TTS worker + # synthesises it while the LLM is still generating + # the next one. + _streamed.append(event["text"]) + self.speak(event["text"]) + elif event["type"] == "done": final_content = event["content"] final_tool_calls = event["tool_calls"] except RuntimeError as e: self.speak_error("LLM", e) return + # ── No tool calls: pure conversational reply ───────────────────── if not final_tool_calls: - # Full response ready — send complete text to TTS in one call - if final_content: + if _streamed: + # Sentences already queued to TTS — just update history/log. + assistant_msg = {"role": "assistant", "content": final_content} + messages.append(assistant_msg) + self._conversation.append(assistant_msg) + self.ui.write_log(f"Jarvis: {final_content}") + elif final_content: + # Very short response (no sentence boundary) — speak now. assistant_msg = {"role": "assistant", "content": final_content} messages.append(assistant_msg) self._conversation.append(assistant_msg) @@ -966,7 +997,7 @@ class JarvisLocal: self.speak(final_content) break - # ------- tool calls ------- + # ── Tool calls present ──────────────────────────────────────────── assistant_msg = { "role": "assistant", "content": final_content or "", @@ -975,16 +1006,12 @@ class JarvisLocal: messages.append(assistant_msg) self._conversation.append(assistant_msg) - # ── Fast path: save_memory-only round with verbal content ────── - # When the LLM calls ONLY save_memory (silent) AND also returned - # verbal content in the same response, speak the content directly - # instead of burning another full LLM round just to get a reply. + # ── Fast path: save_memory + verbal content in same round ──────── _only_memory = all( tc.get("function", {}).get("name") == "save_memory" for tc in final_tool_calls ) if _only_memory and final_content: - # Execute the memory saves silently for tc in final_tool_calls: fn = tc.get("function", {}) targs = fn.get("arguments", {}) @@ -994,15 +1021,18 @@ class JarvisLocal: except Exception: targs = {} self._execute_tool("save_memory", targs) - # Use the verbal content that came alongside the saves - assistant_msg = {"role": "assistant", "content": final_content} - messages.append(assistant_msg) - self._conversation.append(assistant_msg) + assistant_msg2 = {"role": "assistant", "content": final_content} + messages.append(assistant_msg2) + self._conversation.append(assistant_msg2) self.ui.write_log(f"Jarvis: {final_content}") - self.speak(final_content) + if not _streamed: + self.speak(final_content) break - all_silent = True + # ── Execute tools ───────────────────────────────────────────────── + all_silent = True + _tool_results: list[tuple[str, str]] = [] + for tc in final_tool_calls: fn = tc.get("function", {}) tname = fn.get("name", "") @@ -1019,6 +1049,7 @@ class JarvisLocal: if result != "__SILENT__": all_silent = False + _tool_results.append((tname, result)) tool_msg: dict = { "role": "tool", @@ -1030,12 +1061,8 @@ class JarvisLocal: messages.append(tool_msg) self._conversation.append(tool_msg) - # ── Fast-ack for save_memory-only rounds ───────────────────────── - # When EVERY tool call was a silent save_memory, skip the extra LLM - # round entirely. A brief instant acknowledgment feels better than - # waiting 2-3 s for an unnecessary follow-up inference. + # ── Fast-ack: every call was save_memory (silent) ──────────────── if all_silent: - # Extract the saved name (if any) for a personalised ack. _saved_name: str | None = None for _tc in final_tool_calls: _fn = _tc.get("function", {}) @@ -1057,6 +1084,16 @@ class JarvisLocal: self.speak(_ack) break + # ── Direct-result: speak tool output, skip LLM round ──────────── + if _tool_results and not any(n in _NEEDS_LLM_ROUND for n, _ in _tool_results): + _, _reply = _tool_results[-1] + _amsg = {"role": "assistant", "content": _reply} + messages.append(_amsg) + self._conversation.append(_amsg) + self.ui.write_log(f"Jarvis: {_reply}") + self.speak(_reply) + break + if not self.ui.muted: self.ui.set_state("LISTENING")