Files
Mark-LIV/core/tts.py
T

442 lines
16 KiB
Python
Raw Normal View History

2026-06-06 17:41:38 +03:00
"""
Text-to-Speech engines for MARK XL.
EdgeTTS – free Microsoft TTS (internet required, no API key)
Kokoro – fully offline neural TTS (~330 MB model)
ElevenLabs – cloud API (API key required, best quality)
"""
from __future__ import annotations
import asyncio
2026-06-07 15:24:51 +03:00
import os
import queue as _queue
2026-06-06 17:41:38 +03:00
import threading
from typing import Callable, Optional
import numpy as np
import sounddevice as sd
2026-06-07 15:24:51 +03:00
# USE_TF=0 stops transformers from importing TensorFlow (saves 4-8 s startup).
# Do NOT set USE_TORCH or USE_JAX explicitly — forcing those values breaks
# transformers' lazy-loader on certain versions, causing AutoModel and other
# classes to vanish from the public namespace. Auto-detection is reliable.
os.environ.setdefault("USE_TF", "0")
os.environ.setdefault("TOKENIZERS_PARALLELISM", "false")
2026-06-06 17:41:38 +03:00
# ---------------------------------------------------------------------------
# Audio playback helpers
# ---------------------------------------------------------------------------
def _to_numpy(samples) -> np.ndarray:
"""Convert samples to float32 numpy array.
Handles both numpy arrays and PyTorch tensors (Kokoro >= 0.9).
PyTorch built against numpy 1.x raises RuntimeError('Numpy is not available')
when numpy 2.x is installed. The .tolist() fallback always works regardless
of PyTorch / numpy version pairing.
"""
if hasattr(samples, "detach"): # PyTorch tensor
t = samples.detach().cpu().float()
try:
return t.numpy() # fast path (compatible versions)
except RuntimeError:
# PyTorch/numpy version mismatch — convert via Python list (always safe)
return np.asarray(t.tolist(), dtype=np.float32)
return np.asarray(samples, dtype=np.float32)
def _compress_silence(
arr: np.ndarray,
sample_rate: int = 24_000,
2026-06-07 15:24:51 +03:00
max_silence_ms: int = 500, # cap punctuation pauses — keeps natural rhythm
threshold: float = 0.003, # RMS below this = silence; lower = less clipping
2026-06-06 17:41:38 +03:00
) -> np.ndarray:
"""
2026-06-07 15:24:51 +03:00
Shorten Kokoro's very long punctuation pauses (1-2 s → ≤500 ms).
Conservative settings preserve natural prosody; only trims extreme pauses.
2026-06-06 17:41:38 +03:00
"""
max_samp = int(max_silence_ms * sample_rate / 1000)
frame_len = 240 # ~10 ms at 24 kHz
out: list[np.ndarray] = []
silent_acc = 0
for i in range(0, len(arr), frame_len):
chunk = arr[i : i + frame_len]
if np.sqrt(np.mean(chunk ** 2) + 1e-12) < threshold:
silent_acc += len(chunk)
if silent_acc <= max_samp:
out.append(chunk)
else:
silent_acc = 0
out.append(chunk)
return np.concatenate(out) if out else arr
def _play_np(samples, sample_rate: int) -> None:
"""Play float32 mono (or stereo) audio via sounddevice.
Accepts numpy arrays or PyTorch tensors.
"""
sd.play(_to_numpy(samples), sample_rate)
sd.wait()
def _play_audio_bytes(audio_bytes: bytes) -> None:
"""Decode MP3/WAV/OGG bytes and play via sounddevice (uses miniaudio)."""
import miniaudio
decoded = miniaudio.decode(
audio_bytes,
output_format=miniaudio.SampleFormat.FLOAT32,
nchannels=1,
)
samples = np.array(decoded.samples, dtype=np.float32)
sd.play(samples, decoded.sample_rate)
sd.wait()
# ---------------------------------------------------------------------------
# Engines
# ---------------------------------------------------------------------------
class EdgeTTSEngine:
"""Microsoft EdgeTTS – free, requires internet."""
def __init__(self, voice: str = "en-US-GuyNeural"):
self.voice = voice
def speak(self, text: str) -> None:
loop = asyncio.new_event_loop()
try:
audio_bytes = loop.run_until_complete(self._synth(text))
finally:
loop.close()
if audio_bytes:
_play_audio_bytes(audio_bytes)
async def _synth(self, text: str) -> bytes:
import edge_tts
comm = edge_tts.Communicate(text, self.voice)
buf = bytearray()
async for chunk in comm.stream():
if chunk["type"] == "audio":
buf.extend(chunk["data"])
return bytes(buf)
2026-06-07 15:24:51 +03:00
# ---------------------------------------------------------------------------
# Kokoro import helper — auto-upgrades on version-mismatch errors
# ---------------------------------------------------------------------------
# Errors that indicate the installed kokoro uses old transformers classes
# (AlbertModel, AutoModel) that are no longer exported at the top level.
_KOKORO_COMPAT_ERRORS = ("AlbertModel", "AutoModel", "cannot import name")
def _import_kokoro_pipeline():
"""Import KPipeline, auto-upgrading kokoro if a version mismatch is found.
Old kokoro (<0.9) imports AlbertModel / AutoModel from transformers.
Newer transformers versions no longer export these at the top level,
causing an ImportError. kokoro>=0.9 removed these dependencies.
When the error is detected we:
1. Upgrade kokoro to >=0.9 via pip (silent, background)
2. Flush stale kokoro entries from sys.modules
3. Re-import — this time it should succeed
"""
import sys
def _try_import():
from kokoro import KPipeline # noqa: PLC0415
return KPipeline
try:
return _try_import()
except Exception as first_err:
err_msg = str(first_err)
if not any(marker in err_msg for marker in _KOKORO_COMPAT_ERRORS):
# Unrelated error (kokoro not installed, etc.)
raise RuntimeError(
f"Kokoro import failed: {first_err}\n"
"Run: pip install kokoro>=0.9 soundfile"
) from first_err
# ── Version mismatch: upgrade kokoro silently and retry ──────────
print("[TTS] Kokoro/transformers version mismatch detected — upgrading kokoro…")
import subprocess
result = subprocess.run(
[sys.executable, "-m", "pip", "install", "kokoro>=0.9",
"--upgrade", "--quiet", "--disable-pip-version-check"],
capture_output=True,
)
if result.returncode != 0:
stderr = result.stderr.decode(errors="replace").strip()
raise RuntimeError(
f"Kokoro auto-upgrade failed: {stderr[:200]}\n"
"Run manually: pip install kokoro>=0.9 soundfile"
) from first_err
# Flush any stale kokoro submodules from the import cache
stale = [k for k in sys.modules if k == "kokoro" or k.startswith("kokoro.")]
for key in stale:
del sys.modules[key]
print("[TTS] Kokoro upgraded — retrying import…")
try:
return _try_import()
except Exception as retry_err:
raise RuntimeError(
f"Kokoro still broken after upgrade: {retry_err}\n"
"Run manually: pip install --upgrade kokoro transformers"
) from retry_err
2026-06-06 17:41:38 +03:00
# Kokoro voice prefix → KPipeline lang_code mapping
_KOKORO_LANG_CODES = {
"a": "a", # American English (af_*, am_*)
"b": "b", # British English (bf_*, bm_*)
"j": "j", # Japanese (jf_*, jm_*)
"z": "z", # Mandarin Chinese (zf_*, zm_*)
"s": "s", # Spanish (sf_*, sm_*)
"f": "f", # French (ff_*, fm_*)
"h": "h", # Hindi (hf_*, hm_*)
"i": "i", # Italian (if_*, im_*)
"p": "p", # Brazilian Portuguese
"r": "r", # Russian (rf_*, rm_*)
"e": "e", # German (ef_*, em_*)
}
class KokoroTTSEngine:
"""Fully offline Kokoro neural TTS.
2026-06-07 15:24:51 +03:00
Model (~330 MB) is downloaded from HuggingFace on first use,
then cached locally — subsequent starts load from disk.
Warmup strategy: _init() runs synchronously in the background
_do_tts() thread (not the UI thread). After the pipeline loads,
a dummy inference compiles the PyTorch JIT graph immediately so
the first real speak() call has zero compilation overhead.
2026-06-06 17:41:38 +03:00
"""
2026-06-07 15:24:51 +03:00
def __init__(self, voice: str = "af_heart", speed: float = 1.0):
2026-06-06 17:41:38 +03:00
self.voice = voice
2026-06-07 15:24:51 +03:00
self.speed = speed
2026-06-06 17:41:38 +03:00
self._pipeline = None
self._lock = threading.Lock()
2026-06-07 15:24:51 +03:00
self._init() # blocking, but called from background thread
2026-06-06 17:41:38 +03:00
@property
def _lang_code(self) -> str:
prefix = self.voice[0].lower() if self.voice else "a"
return _KOKORO_LANG_CODES.get(prefix, "a")
def _init(self) -> None:
if self._pipeline is not None:
return
2026-06-07 15:24:51 +03:00
lang = self._lang_code
# Prefer GPU — Kokoro on CUDA is ~10x faster than CPU.
try:
import torch
device = "cuda" if torch.cuda.is_available() else "cpu"
if device == "cpu":
import os as _os
n_threads = max(1, min(4, (_os.cpu_count() or 4) // 2))
try:
torch.set_num_threads(n_threads)
torch.set_num_interop_threads(2)
except RuntimeError:
pass
print(
f"[TTS] Kokoro on CPU — for faster speech install CUDA PyTorch:\n"
" pip install torch --index-url https://download.pytorch.org/whl/cu118"
)
except Exception:
device = "cpu"
print(f"[TTS] Kokoro — loading (lang='{lang}', device='{device}')…")
KPipeline = _import_kokoro_pipeline()
2026-06-07 17:03:02 +03:00
def _create_pipeline():
try:
return KPipeline(lang_code=lang, device=device)
except TypeError:
return KPipeline(lang_code=lang) # older build — no device param
2026-06-06 17:41:38 +03:00
try:
2026-06-07 17:03:02 +03:00
self._pipeline = _create_pipeline()
except Exception as _first_err:
2026-06-14 17:04:27 +03:00
# Offline flag set but model not cached yet → clear flags and download once.
# Keywords cover multiple huggingface_hub error message variants across versions.
2026-06-07 17:03:02 +03:00
_e = str(_first_err).lower()
2026-06-14 17:04:27 +03:00
_offline_keywords = (
"offline", "not found", "cache", "localentry",
"does not exist", "outgoing", "local_files_only",
)
if any(k in _e for k in _offline_keywords):
print("[TTS] Kokoro model not in local cache — downloading (one-time, internet required)…")
2026-06-07 17:03:02 +03:00
os.environ.pop("HF_HUB_OFFLINE", None)
os.environ.pop("TRANSFORMERS_OFFLINE", None)
os.environ.pop("HF_DATASETS_OFFLINE", None)
2026-06-14 17:04:27 +03:00
try:
self._pipeline = _create_pipeline()
except Exception as _dl_err:
raise RuntimeError(
f"Kokoro model download failed.\n"
f"Internet access is required the first time to download the voice model (~330 MB).\n"
f"After the first download it runs fully offline.\n"
f"Tip: Switch to EdgeTTS (free, no download) in the Configure panel if offline.\n"
f"Details: {_dl_err}"
) from _dl_err
2026-06-07 17:03:02 +03:00
else:
raise
2026-06-07 15:24:51 +03:00
print("[TTS] Kokoro compiling (first-time only)…")
# Warmup: compiles PyTorch JIT graph so first real speak() call is instant.
try:
for _ in self._pipeline("hello", voice=self.voice, speed=self.speed):
pass
print("[TTS] Kokoro ready.")
2026-06-06 17:41:38 +03:00
except Exception as e:
2026-06-07 15:24:51 +03:00
print(f"[TTS] Kokoro warmup warning: {e}")
2026-06-06 17:41:38 +03:00
def speak(self, text: str) -> None:
with self._lock:
if self._pipeline is None:
self._init()
2026-06-07 15:24:51 +03:00
# ── Concurrent synthesise + playback ────────────────────────────────
# Kokoro generates audio chunks lazily. Without threading, we:
# synthesise chunk N → play N → synthesise N+1 → play N+1 …
# With a producer/consumer pair, chunk N+1 synthesises WHILE chunk N
# plays, cutting perceived latency by the playback duration of all but
# the last chunk (typically 1-3 s on multi-sentence responses).
audio_q: "_queue.Queue[np.ndarray | None]" = _queue.Queue(maxsize=4)
synth_error: list[Exception] = []
def _synth():
try:
for _, _, audio in self._pipeline(text, voice=self.voice, speed=self.speed):
if audio is not None:
arr = _to_numpy(audio)
arr = _compress_silence(arr)
if arr.size > 0:
audio_q.put(arr) # blocks if player is slow (backpressure)
except Exception as exc:
synth_error.append(exc)
finally:
audio_q.put(None) # sentinel → player exits
synth_thread = threading.Thread(target=_synth, daemon=True)
synth_thread.start()
# Player runs in this thread so sd.wait() doesn't block the synth thread.
while True:
arr = audio_q.get()
if arr is None:
break
_play_np(arr, 24000)
synth_thread.join()
if synth_error:
raise synth_error[0]
2026-06-06 17:41:38 +03:00
class ElevenLabsTTSEngine:
"""ElevenLabs cloud TTS – API key required."""
def __init__(self, api_key: str, voice_id: str = "pNInz6obpgDQGcFmaJgB"):
self.api_key = api_key
self.voice_id = voice_id
def speak(self, text: str) -> None:
import requests
headers = {
"xi-api-key": self.api_key,
"Content-Type": "application/json",
}
payload = {
"text": text,
"model_id": "eleven_multilingual_v2",
"voice_settings": {"stability": 0.5, "similarity_boost": 0.75},
}
resp = requests.post(
f"https://api.elevenlabs.io/v1/text-to-speech/{self.voice_id}",
json=payload, headers=headers, timeout=30,
)
resp.raise_for_status()
_play_audio_bytes(resp.content)
# ---------------------------------------------------------------------------
# Thread-safe player wrapper
# ---------------------------------------------------------------------------
class TTSPlayer:
"""
Wraps any *Engine. Exposes a blocking speak() method
meant to be called from a dedicated background thread.
"""
def __init__(self, engine):
self._engine = engine
self._playing = False
self._lock = threading.Lock()
@property
def is_playing(self) -> bool:
return self._playing
def speak(
self,
text: str,
on_start: Optional[Callable] = None,
on_done: Optional[Callable] = None,
) -> None:
"""Synthesise and play text. BLOCKING – call from a dedicated thread."""
try:
with self._lock:
self._playing = True
if on_start:
on_start()
self._engine.speak(text)
except Exception as e:
print(f"[TTS] Error: {e}")
finally:
with self._lock:
self._playing = False
if on_done:
on_done()
def stop(self) -> None:
sd.stop()
with self._lock:
self._playing = False
# ---------------------------------------------------------------------------
# Factory
# ---------------------------------------------------------------------------
def create_tts_player(config: dict) -> TTSPlayer:
engine_name = config.get("tts_engine", "edgetts").lower()
if engine_name == "kokoro":
voice = config.get("tts_voice", "af_heart")
2026-06-07 15:24:51 +03:00
speed = float(config.get("tts_speed", 1.0))
2026-06-06 17:41:38 +03:00
engine = KokoroTTSEngine(voice=voice, speed=speed)
elif engine_name == "elevenlabs":
api_key = config.get("elevenlabs_api_key", "")
voice_id = config.get("tts_voice", "pNInz6obpgDQGcFmaJgB")
engine = ElevenLabsTTSEngine(api_key=api_key, voice_id=voice_id)
else: # edgetts (default)
voice = config.get("tts_voice", "en-US-GuyNeural")
engine = EdgeTTSEngine(voice=voice)
return TTSPlayer(engine)