Add files via upload

This commit is contained in:
FatihMakes authored and GitHub committed 2026-06-07 17:03:02 +03:00
1 parent 6699c074ac
commit 5d8d214e9b
5 files changed
+117 -45

No files matched your search

+5 -4
View File
@@ -212,7 +212,7 @@ def call_llm(
"model": model,
"messages": messages,
"stream": False,
"max_tokens": 300,
"max_tokens": 150,
}
if tools:
payload["tools"] = tools
@@ -252,7 +252,7 @@ def call_llm(
"messages": messages,
"stream": False,
"keep_alive": -1,
"options": {"num_predict": 300, "num_gpu": 99},
"options": {"num_predict": 150, "num_gpu": 99},
}
if tools:
payload["tools"] = tools
@@ -353,7 +353,7 @@ def _stream_openai(
"model": model,
"messages": messages,
"stream": True,
"max_tokens": 300,
"max_tokens": 150,
}
if tools:
payload["tools"] = tools
@@ -479,8 +479,9 @@ def call_llm_stream(
"messages": messages,
"stream": True,
"keep_alive": -1,
# 150 tokens ≈ 100 words ≈ 3-4 sentences — enough for any voice reply.
# num_gpu:99 pushes all layers to GPU; num_thread removed (Ollama auto-tunes).
"options": {"num_predict": 300, "num_gpu": 99},
"options": {"num_predict": 150, "num_gpu": 99},
}
if tools:
payload["tools"] = tools
+6 -1
View File
@@ -12,7 +12,12 @@ EXECUTION RULES:
Action Flow: Call the tool immediately. For slow tools (search, vision), say ONE short sentence first.
Vision (screen_process): Say one curious sentence, then call the tool — total silence after.
One-Call Policy: Never guess. Call tools exactly once. No retries.
Briefing: 1 short sentence max for confirmations/acknowledgments. 1-2 sentences for results. Never add "Is there anything else?" or filler questions.
Briefing: HARD LIMIT — 3 SHORT sentences maximum for ANY response. No exceptions.
• Confirmations / acks: 1 sentence only.
• Answers / results: 2-3 short sentences max.
• NEVER ask follow-up questions ("Would you like...?", "Is there anything else?").
• NEVER pad with sympathy openers or filler phrases ("Of course!", "Great question!").
• If you cannot fit the answer in 3 sentences, give the most important part only.
Exit: Only call shutdown_jarvis if session termination is explicit.
Response time: Respond as fast as you can.
Always respond in the language that user is speaking.
+16 -1
View File
@@ -12,6 +12,7 @@ class WhisperSTT:
"""Offline transcription using faster-whisper."""
def __init__(self, model_name: str = "base", language: str | None = None):
import os
from faster_whisper import WhisperModel
print(f"[STT] Loading Whisper '{model_name}'…")
try:
@@ -20,7 +21,21 @@ class WhisperSTT:
compute = "float16" if device == "cuda" else "int8"
except Exception:
device, compute = "cpu", "int8"
self._model = WhisperModel(model_name, device=device, compute_type=compute)
try:
self._model = WhisperModel(model_name, device=device, compute_type=compute)
except Exception as _first_err:
# Offline flag set but model not cached yet → download once, then offline forever
_e = str(_first_err).lower()
if any(k in _e for k in ("offline", "not found", "cache", "localentry", "does not exist")):
print(f"[STT] '{model_name}' not cached — downloading (internet required for first run)…")
os.environ.pop("HF_HUB_OFFLINE", None)
os.environ.pop("TRANSFORMERS_OFFLINE", None)
os.environ.pop("HF_DATASETS_OFFLINE", None)
self._model = WhisperModel(model_name, device=device, compute_type=compute)
else:
raise
self._language = None if (not language or language.strip().lower() == "auto") else language.strip().lower()
print(f"[STT] Whisper '{model_name}' ready ({device})")
+18 -4
View File
@@ -264,11 +264,25 @@ class KokoroTTSEngine:
KPipeline = _import_kokoro_pipeline()
def _create_pipeline():
try:
return KPipeline(lang_code=lang, device=device)
except TypeError:
return KPipeline(lang_code=lang) # older build — no device param
try:
self._pipeline = KPipeline(lang_code=lang, device=device)
except TypeError:
# Older Kokoro build — no device parameter
self._pipeline = KPipeline(lang_code=lang)
self._pipeline = _create_pipeline()
except Exception as _first_err:
# Offline flag set but model not cached yet → download once
_e = str(_first_err).lower()
if any(k in _e for k in ("offline", "not found", "cache", "localentry", "does not exist")):
print("[TTS] Kokoro model not cached — downloading (internet required for first run)…")
os.environ.pop("HF_HUB_OFFLINE", None)
os.environ.pop("TRANSFORMERS_OFFLINE", None)
os.environ.pop("HF_DATASETS_OFFLINE", None)
self._pipeline = _create_pipeline()
else:
raise
print("[TTS] Kokoro compiling (first-time only)…")
# Warmup: compiles PyTorch JIT graph so first real speak() call is instant.
+72 -35
View File
@@ -14,6 +14,13 @@ _os.environ.setdefault("GRPC_VERBOSITY", "ERROR")
# from the namespace). Let transformers auto-detect the available backends.
_os.environ.setdefault("USE_TF", "0")
_os.environ.setdefault("TOKENIZERS_PARALLELISM", "false")
# Offline mode — use cached models, no HuggingFace network calls on startup.
# On first run the model isn't cached yet; tts.py / stt.py detect this and
# temporarily clear these flags to allow the one-time download, then they
# stay in effect for every subsequent launch (fully offline).
_os.environ.setdefault("HF_HUB_OFFLINE", "1")
_os.environ.setdefault("TRANSFORMERS_OFFLINE", "1")
_os.environ.setdefault("HF_DATASETS_OFFLINE", "1")
import warnings as _warnings
_warnings.filterwarnings("ignore", category=UserWarning)
_warnings.filterwarnings("ignore", category=DeprecationWarning)
@@ -562,10 +569,10 @@ class _VADBuffer:
def __init__(
self,
sample_rate: int = 16_000,
silence_sec: float = 1.0, # silence after last word → send to STT (1.0s = fast enough + won't cut mid-sentence with hysteresis)
speech_thresh: float = 0.015, # RMS above this = speech (raised from 0.010 to kill background noise false-positives)
silence_thresh: float = 0.008, # RMS below this = silence (hysteresis gap — much lower bar than speech_thresh to avoid mid-sentence cuts)
min_speech_sec: float = 0.4,
silence_sec: float = 0.7, # silence after last word → send to STT
speech_thresh: float = 0.008, # RMS above this = speech (0.008 catches voice at 3-4 m; raise if mic picks up too much room noise)
silence_thresh: float = 0.004, # RMS below this = silence (half of speech_thresh — hysteresis prevents mid-sentence cuts)
min_speech_sec: float = 0.3,
max_speech_sec: float = 30.0,
):
self._sr = sample_rate
@@ -582,12 +589,11 @@ class _VADBuffer:
Feed one audio chunk (float32 mono).
Returns complete utterance when speech ends, otherwise None.
Uses hysteresis thresholds so the detector doesn't flicker on quiet
consonants or brief mid-sentence pauses:
- speech starts when RMS > speech_thresh
- speech ends only when RMS < silence_thresh (lower bar)
This prevents natural pauses ("I just... wanted to say") from
triggering a premature cut.
Uses hysteresis thresholds so the detector doesn't flicker:
- speech starts when RMS > speech_thresh (0.008 = ~3-4 m range)
- speech ends only when RMS < silence_thresh (0.004 = half of start)
The gap between the two thresholds prevents mid-sentence cuts on
natural pauses and quiet consonants.
"""
rms = float(np.sqrt(np.mean(chunk ** 2)))
total_n = sum(len(c) for c in self._buf)
@@ -922,12 +928,16 @@ class JarvisLocal:
def _process_message(self, user_text: str) -> None:
"""
Full turn:
user_text → Ollama stream (with tools) → execute tools → final spoken response
Full turn: user_text → LLM stream → TTS (overlapped) → tool execution
The complete LLM response is collected first, then sent to TTS in one call.
This gives TTS engines (Kokoro, ElevenLabs) the full text so they can
apply natural prosody across sentence boundaries — no choppy pauses.
Streaming TTS: sentence events are piped to the TTS queue AS they
arrive from the LLM, so Kokoro starts synthesising sentence 1 while
the LLM is still generating sentence 2. This cuts perceived latency
from (LLM_total + TTS_total) down to roughly max(LLM_total, TTS_total).
Tool-call responses never emit sentence events, so the TTS overlap
only kicks in for pure conversational replies — which is exactly when
it matters most.
"""
self.ui.set_state("THINKING")
self.ui.write_log(f"You: {user_text}")
@@ -942,23 +952,44 @@ class JarvisLocal:
{"role": "system", "content": self._build_system_prompt()}
] + list(self._conversation)
# Tools whose output needs a second LLM round to summarise/interpret.
# Everything else returns a user-ready string → speak directly.
_NEEDS_LLM_ROUND = {"web_search", "screen_process", "agent_task"}
MAX_TOOL_ROUNDS = 6
for _round in range(MAX_TOOL_ROUNDS):
final_content = ""
final_tool_calls: list = []
# Sentences already queued to TTS during streaming (may be empty
# for tool-call rounds where the model emits no content).
_streamed: list[str] = []
try:
for event in call_llm_stream(messages, OLLAMA_TOOLS):
if event["type"] == "done":
if event["type"] == "sentence":
# ── Overlap TTS with LLM generation ─────────────────
# Queue this sentence immediately; the TTS worker
# synthesises it while the LLM is still generating
# the next one.
_streamed.append(event["text"])
self.speak(event["text"])
elif event["type"] == "done":
final_content = event["content"]
final_tool_calls = event["tool_calls"]
except RuntimeError as e:
self.speak_error("LLM", e)
return
# ── No tool calls: pure conversational reply ─────────────────────
if not final_tool_calls:
# Full response ready — send complete text to TTS in one call
if final_content:
if _streamed:
# Sentences already queued to TTS — just update history/log.
assistant_msg = {"role": "assistant", "content": final_content}
messages.append(assistant_msg)
self._conversation.append(assistant_msg)
self.ui.write_log(f"Jarvis: {final_content}")
elif final_content:
# Very short response (no sentence boundary) — speak now.
assistant_msg = {"role": "assistant", "content": final_content}
messages.append(assistant_msg)
self._conversation.append(assistant_msg)
@@ -966,7 +997,7 @@ class JarvisLocal:
self.speak(final_content)
break
# ------- tool calls -------
# ── Tool calls present ────────────────────────────────────────────
assistant_msg = {
"role": "assistant",
"content": final_content or "",
@@ -975,16 +1006,12 @@ class JarvisLocal:
messages.append(assistant_msg)
self._conversation.append(assistant_msg)
# ── Fast path: save_memory-only round with verbal content ──────
# When the LLM calls ONLY save_memory (silent) AND also returned
# verbal content in the same response, speak the content directly
# instead of burning another full LLM round just to get a reply.
# ── Fast path: save_memory + verbal content in same round ────────
_only_memory = all(
tc.get("function", {}).get("name") == "save_memory"
for tc in final_tool_calls
)
if _only_memory and final_content:
# Execute the memory saves silently
for tc in final_tool_calls:
fn = tc.get("function", {})
targs = fn.get("arguments", {})
@@ -994,15 +1021,18 @@ class JarvisLocal:
except Exception:
targs = {}
self._execute_tool("save_memory", targs)
# Use the verbal content that came alongside the saves
assistant_msg = {"role": "assistant", "content": final_content}
messages.append(assistant_msg)
self._conversation.append(assistant_msg)
assistant_msg2 = {"role": "assistant", "content": final_content}
messages.append(assistant_msg2)
self._conversation.append(assistant_msg2)
self.ui.write_log(f"Jarvis: {final_content}")
self.speak(final_content)
if not _streamed:
self.speak(final_content)
break
all_silent = True
# ── Execute tools ─────────────────────────────────────────────────
all_silent = True
_tool_results: list[tuple[str, str]] = []
for tc in final_tool_calls:
fn = tc.get("function", {})
tname = fn.get("name", "")
@@ -1019,6 +1049,7 @@ class JarvisLocal:
if result != "__SILENT__":
all_silent = False
_tool_results.append((tname, result))
tool_msg: dict = {
"role": "tool",
@@ -1030,12 +1061,8 @@ class JarvisLocal:
messages.append(tool_msg)
self._conversation.append(tool_msg)
# ── Fast-ack for save_memory-only rounds ─────────────────────────
# When EVERY tool call was a silent save_memory, skip the extra LLM
# round entirely. A brief instant acknowledgment feels better than
# waiting 2-3 s for an unnecessary follow-up inference.
# ── Fast-ack: every call was save_memory (silent) ────────────────
if all_silent:
# Extract the saved name (if any) for a personalised ack.
_saved_name: str | None = None
for _tc in final_tool_calls:
_fn = _tc.get("function", {})
@@ -1057,6 +1084,16 @@ class JarvisLocal:
self.speak(_ack)
break
# ── Direct-result: speak tool output, skip LLM round ────────────
if _tool_results and not any(n in _NEEDS_LLM_ROUND for n, _ in _tool_results):
_, _reply = _tool_results[-1]
_amsg = {"role": "assistant", "content": _reply}
messages.append(_amsg)
self._conversation.append(_amsg)
self.ui.write_log(f"Jarvis: {_reply}")
self.speak(_reply)
break
if not self.ui.muted:
self.ui.set_state("LISTENING")