443 lines
16 KiB
Python
443 lines
16 KiB
Python
"""
|
||
Text-to-Speech engines for MARK XL.
|
||
|
||
EdgeTTS – free Microsoft TTS (internet required, no API key)
|
||
Kokoro – fully offline neural TTS (~330 MB model)
|
||
ElevenLabs – cloud API (API key required, best quality)
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import asyncio
|
||
import os
|
||
import queue as _queue
|
||
import threading
|
||
from typing import Callable, Optional
|
||
|
||
import numpy as np
|
||
import sounddevice as sd
|
||
|
||
|
||
|
||
# USE_TF=0 stops transformers from importing TensorFlow (saves 4-8 s startup).
|
||
# Do NOT set USE_TORCH or USE_JAX explicitly — forcing those values breaks
|
||
# transformers' lazy-loader on certain versions, causing AutoModel and other
|
||
# classes to vanish from the public namespace. Auto-detection is reliable.
|
||
os.environ.setdefault("USE_TF", "0")
|
||
os.environ.setdefault("TOKENIZERS_PARALLELISM", "false")
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Audio playback helpers
|
||
# ---------------------------------------------------------------------------
|
||
|
||
def _to_numpy(samples) -> np.ndarray:
|
||
"""Convert samples to float32 numpy array.
|
||
|
||
Handles both numpy arrays and PyTorch tensors (Kokoro >= 0.9).
|
||
|
||
PyTorch built against numpy 1.x raises RuntimeError('Numpy is not available')
|
||
when numpy 2.x is installed. The .tolist() fallback always works regardless
|
||
of PyTorch / numpy version pairing.
|
||
"""
|
||
if hasattr(samples, "detach"): # PyTorch tensor
|
||
t = samples.detach().cpu().float()
|
||
try:
|
||
return t.numpy() # fast path (compatible versions)
|
||
except RuntimeError:
|
||
# PyTorch/numpy version mismatch — convert via Python list (always safe)
|
||
return np.asarray(t.tolist(), dtype=np.float32)
|
||
return np.asarray(samples, dtype=np.float32)
|
||
|
||
|
||
def _compress_silence(
|
||
arr: np.ndarray,
|
||
sample_rate: int = 24_000,
|
||
max_silence_ms: int = 500, # cap punctuation pauses — keeps natural rhythm
|
||
threshold: float = 0.003, # RMS below this = silence; lower = less clipping
|
||
) -> np.ndarray:
|
||
"""
|
||
Shorten Kokoro's very long punctuation pauses (1-2 s → ≤500 ms).
|
||
Conservative settings preserve natural prosody; only trims extreme pauses.
|
||
"""
|
||
max_samp = int(max_silence_ms * sample_rate / 1000)
|
||
frame_len = 240 # ~10 ms at 24 kHz
|
||
out: list[np.ndarray] = []
|
||
silent_acc = 0
|
||
|
||
for i in range(0, len(arr), frame_len):
|
||
chunk = arr[i : i + frame_len]
|
||
if np.sqrt(np.mean(chunk ** 2) + 1e-12) < threshold:
|
||
silent_acc += len(chunk)
|
||
if silent_acc <= max_samp:
|
||
out.append(chunk)
|
||
else:
|
||
silent_acc = 0
|
||
out.append(chunk)
|
||
|
||
return np.concatenate(out) if out else arr
|
||
|
||
|
||
def _play_np(samples, sample_rate: int) -> None:
|
||
"""Play float32 mono (or stereo) audio via sounddevice.
|
||
Accepts numpy arrays or PyTorch tensors.
|
||
"""
|
||
sd.play(_to_numpy(samples), sample_rate)
|
||
sd.wait()
|
||
|
||
|
||
def _play_audio_bytes(audio_bytes: bytes) -> None:
|
||
"""Decode MP3/WAV/OGG bytes and play via sounddevice (uses miniaudio)."""
|
||
import miniaudio
|
||
decoded = miniaudio.decode(
|
||
audio_bytes,
|
||
output_format=miniaudio.SampleFormat.FLOAT32,
|
||
nchannels=1,
|
||
)
|
||
samples = np.array(decoded.samples, dtype=np.float32)
|
||
sd.play(samples, decoded.sample_rate)
|
||
sd.wait()
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Engines
|
||
# ---------------------------------------------------------------------------
|
||
|
||
class EdgeTTSEngine:
|
||
"""Microsoft EdgeTTS – free, requires internet."""
|
||
|
||
def __init__(self, voice: str = "en-US-GuyNeural"):
|
||
self.voice = voice
|
||
|
||
def speak(self, text: str) -> None:
|
||
loop = asyncio.new_event_loop()
|
||
try:
|
||
audio_bytes = loop.run_until_complete(self._synth(text))
|
||
finally:
|
||
loop.close()
|
||
if audio_bytes:
|
||
_play_audio_bytes(audio_bytes)
|
||
|
||
async def _synth(self, text: str) -> bytes:
|
||
import edge_tts
|
||
comm = edge_tts.Communicate(text, self.voice)
|
||
buf = bytearray()
|
||
async for chunk in comm.stream():
|
||
if chunk["type"] == "audio":
|
||
buf.extend(chunk["data"])
|
||
return bytes(buf)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Kokoro import helper — auto-upgrades on version-mismatch errors
|
||
# ---------------------------------------------------------------------------
|
||
|
||
# Errors that indicate the installed kokoro uses old transformers classes
|
||
# (AlbertModel, AutoModel) that are no longer exported at the top level.
|
||
_KOKORO_COMPAT_ERRORS = ("AlbertModel", "AutoModel", "cannot import name")
|
||
|
||
|
||
def _import_kokoro_pipeline():
|
||
"""Import KPipeline, auto-upgrading kokoro if a version mismatch is found.
|
||
|
||
Old kokoro (<0.9) imports AlbertModel / AutoModel from transformers.
|
||
Newer transformers versions no longer export these at the top level,
|
||
causing an ImportError. kokoro>=0.9 removed these dependencies.
|
||
|
||
When the error is detected we:
|
||
1. Upgrade kokoro to >=0.9 via pip (silent, background)
|
||
2. Flush stale kokoro entries from sys.modules
|
||
3. Re-import — this time it should succeed
|
||
"""
|
||
import sys
|
||
|
||
def _try_import():
|
||
from kokoro import KPipeline # noqa: PLC0415
|
||
return KPipeline
|
||
|
||
try:
|
||
return _try_import()
|
||
except Exception as first_err:
|
||
err_msg = str(first_err)
|
||
if not any(marker in err_msg for marker in _KOKORO_COMPAT_ERRORS):
|
||
# Unrelated error (kokoro not installed, etc.)
|
||
raise RuntimeError(
|
||
f"Kokoro import failed: {first_err}\n"
|
||
"Run: pip install kokoro>=0.9 soundfile"
|
||
) from first_err
|
||
|
||
# ── Version mismatch: upgrade kokoro silently and retry ──────────
|
||
print("[TTS] Kokoro/transformers version mismatch detected — upgrading kokoro…")
|
||
import subprocess
|
||
result = subprocess.run(
|
||
[sys.executable, "-m", "pip", "install", "kokoro>=0.9",
|
||
"--upgrade", "--quiet", "--disable-pip-version-check"],
|
||
capture_output=True,
|
||
)
|
||
if result.returncode != 0:
|
||
stderr = result.stderr.decode(errors="replace").strip()
|
||
raise RuntimeError(
|
||
f"Kokoro auto-upgrade failed: {stderr[:200]}\n"
|
||
"Run manually: pip install kokoro>=0.9 soundfile"
|
||
) from first_err
|
||
|
||
# Flush any stale kokoro submodules from the import cache
|
||
stale = [k for k in sys.modules if k == "kokoro" or k.startswith("kokoro.")]
|
||
for key in stale:
|
||
del sys.modules[key]
|
||
|
||
print("[TTS] Kokoro upgraded — retrying import…")
|
||
try:
|
||
return _try_import()
|
||
except Exception as retry_err:
|
||
raise RuntimeError(
|
||
f"Kokoro still broken after upgrade: {retry_err}\n"
|
||
"Run manually: pip install --upgrade kokoro transformers"
|
||
) from retry_err
|
||
|
||
|
||
# Kokoro voice prefix → KPipeline lang_code mapping
|
||
_KOKORO_LANG_CODES = {
|
||
"a": "a", # American English (af_*, am_*)
|
||
"b": "b", # British English (bf_*, bm_*)
|
||
"j": "j", # Japanese (jf_*, jm_*)
|
||
"z": "z", # Mandarin Chinese (zf_*, zm_*)
|
||
"s": "s", # Spanish (sf_*, sm_*)
|
||
"f": "f", # French (ff_*, fm_*)
|
||
"h": "h", # Hindi (hf_*, hm_*)
|
||
"i": "i", # Italian (if_*, im_*)
|
||
"p": "p", # Brazilian Portuguese
|
||
"r": "r", # Russian (rf_*, rm_*)
|
||
"e": "e", # German (ef_*, em_*)
|
||
}
|
||
|
||
|
||
class KokoroTTSEngine:
|
||
"""Fully offline Kokoro neural TTS.
|
||
|
||
Model (~330 MB) is downloaded from HuggingFace on first use,
|
||
then cached locally — subsequent starts load from disk.
|
||
|
||
Warmup strategy: _init() runs synchronously in the background
|
||
_do_tts() thread (not the UI thread). After the pipeline loads,
|
||
a dummy inference compiles the PyTorch JIT graph immediately so
|
||
the first real speak() call has zero compilation overhead.
|
||
"""
|
||
|
||
def __init__(self, voice: str = "af_heart", speed: float = 1.0):
|
||
self.voice = voice
|
||
self.speed = speed
|
||
self._pipeline = None
|
||
self._lock = threading.Lock()
|
||
self._init() # blocking, but called from background thread
|
||
|
||
@property
|
||
def _lang_code(self) -> str:
|
||
prefix = self.voice[0].lower() if self.voice else "a"
|
||
return _KOKORO_LANG_CODES.get(prefix, "a")
|
||
|
||
def _init(self) -> None:
|
||
if self._pipeline is not None:
|
||
return
|
||
|
||
lang = self._lang_code
|
||
|
||
# Prefer GPU — Kokoro on CUDA is ~10x faster than CPU.
|
||
try:
|
||
import torch
|
||
device = "cuda" if torch.cuda.is_available() else "cpu"
|
||
if device == "cpu":
|
||
import os as _os
|
||
n_threads = max(1, min(4, (_os.cpu_count() or 4) // 2))
|
||
try:
|
||
torch.set_num_threads(n_threads)
|
||
torch.set_num_interop_threads(2)
|
||
except RuntimeError:
|
||
pass
|
||
print(
|
||
f"[TTS] Kokoro on CPU — for faster speech install CUDA PyTorch:\n"
|
||
" pip install torch --index-url https://download.pytorch.org/whl/cu118"
|
||
)
|
||
except Exception:
|
||
device = "cpu"
|
||
|
||
print(f"[TTS] Kokoro — loading (lang='{lang}', device='{device}')…")
|
||
|
||
KPipeline = _import_kokoro_pipeline()
|
||
|
||
def _create_pipeline():
|
||
try:
|
||
return KPipeline(lang_code=lang, device=device)
|
||
except TypeError:
|
||
return KPipeline(lang_code=lang) # older build — no device param
|
||
|
||
try:
|
||
self._pipeline = _create_pipeline()
|
||
except Exception as _first_err:
|
||
# Offline flag set but model not cached yet → clear flags and download once.
|
||
# Keywords cover multiple huggingface_hub error message variants across versions.
|
||
_e = str(_first_err).lower()
|
||
_offline_keywords = (
|
||
"offline", "not found", "cache", "localentry",
|
||
"does not exist", "outgoing", "local_files_only",
|
||
)
|
||
if any(k in _e for k in _offline_keywords):
|
||
print("[TTS] Kokoro model not in local cache — downloading (one-time, internet required)…")
|
||
os.environ.pop("HF_HUB_OFFLINE", None)
|
||
os.environ.pop("TRANSFORMERS_OFFLINE", None)
|
||
os.environ.pop("HF_DATASETS_OFFLINE", None)
|
||
try:
|
||
self._pipeline = _create_pipeline()
|
||
except Exception as _dl_err:
|
||
raise RuntimeError(
|
||
f"Kokoro model download failed.\n"
|
||
f"Internet access is required the first time to download the voice model (~330 MB).\n"
|
||
f"After the first download it runs fully offline.\n"
|
||
f"Tip: Switch to EdgeTTS (free, no download) in the Configure panel if offline.\n"
|
||
f"Details: {_dl_err}"
|
||
) from _dl_err
|
||
else:
|
||
raise
|
||
|
||
print("[TTS] Kokoro compiling (first-time only)…")
|
||
# Warmup: compiles PyTorch JIT graph so first real speak() call is instant.
|
||
try:
|
||
for _ in self._pipeline("hello", voice=self.voice, speed=self.speed):
|
||
pass
|
||
print("[TTS] Kokoro ready.")
|
||
except Exception as e:
|
||
print(f"[TTS] Kokoro warmup warning: {e}")
|
||
|
||
def speak(self, text: str) -> None:
|
||
with self._lock:
|
||
if self._pipeline is None:
|
||
self._init()
|
||
|
||
# ── Concurrent synthesise + playback ────────────────────────────────
|
||
# Kokoro generates audio chunks lazily. Without threading, we:
|
||
# synthesise chunk N → play N → synthesise N+1 → play N+1 …
|
||
# With a producer/consumer pair, chunk N+1 synthesises WHILE chunk N
|
||
# plays, cutting perceived latency by the playback duration of all but
|
||
# the last chunk (typically 1-3 s on multi-sentence responses).
|
||
audio_q: "_queue.Queue[np.ndarray | None]" = _queue.Queue(maxsize=4)
|
||
synth_error: list[Exception] = []
|
||
|
||
def _synth():
|
||
try:
|
||
for _, _, audio in self._pipeline(text, voice=self.voice, speed=self.speed):
|
||
if audio is not None:
|
||
arr = _to_numpy(audio)
|
||
arr = _compress_silence(arr)
|
||
if arr.size > 0:
|
||
audio_q.put(arr) # blocks if player is slow (backpressure)
|
||
except Exception as exc:
|
||
synth_error.append(exc)
|
||
finally:
|
||
audio_q.put(None) # sentinel → player exits
|
||
|
||
synth_thread = threading.Thread(target=_synth, daemon=True)
|
||
synth_thread.start()
|
||
|
||
# Player runs in this thread so sd.wait() doesn't block the synth thread.
|
||
while True:
|
||
arr = audio_q.get()
|
||
if arr is None:
|
||
break
|
||
_play_np(arr, 24000)
|
||
|
||
synth_thread.join()
|
||
|
||
if synth_error:
|
||
raise synth_error[0]
|
||
|
||
|
||
class ElevenLabsTTSEngine:
|
||
"""ElevenLabs cloud TTS – API key required."""
|
||
|
||
def __init__(self, api_key: str, voice_id: str = "pNInz6obpgDQGcFmaJgB"):
|
||
self.api_key = api_key
|
||
self.voice_id = voice_id
|
||
|
||
def speak(self, text: str) -> None:
|
||
import requests
|
||
headers = {
|
||
"xi-api-key": self.api_key,
|
||
"Content-Type": "application/json",
|
||
}
|
||
payload = {
|
||
"text": text,
|
||
"model_id": "eleven_multilingual_v2",
|
||
"voice_settings": {"stability": 0.5, "similarity_boost": 0.75},
|
||
}
|
||
resp = requests.post(
|
||
f"https://api.elevenlabs.io/v1/text-to-speech/{self.voice_id}",
|
||
json=payload, headers=headers, timeout=30,
|
||
)
|
||
resp.raise_for_status()
|
||
_play_audio_bytes(resp.content)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Thread-safe player wrapper
|
||
# ---------------------------------------------------------------------------
|
||
|
||
class TTSPlayer:
|
||
"""
|
||
Wraps any *Engine. Exposes a blocking speak() method
|
||
meant to be called from a dedicated background thread.
|
||
"""
|
||
|
||
def __init__(self, engine):
|
||
self._engine = engine
|
||
self._playing = False
|
||
self._lock = threading.Lock()
|
||
|
||
@property
|
||
def is_playing(self) -> bool:
|
||
return self._playing
|
||
|
||
def speak(
|
||
self,
|
||
text: str,
|
||
on_start: Optional[Callable] = None,
|
||
on_done: Optional[Callable] = None,
|
||
) -> None:
|
||
"""Synthesise and play text. BLOCKING – call from a dedicated thread."""
|
||
try:
|
||
with self._lock:
|
||
self._playing = True
|
||
if on_start:
|
||
on_start()
|
||
self._engine.speak(text)
|
||
except Exception as e:
|
||
print(f"[TTS] Error: {e}")
|
||
finally:
|
||
with self._lock:
|
||
self._playing = False
|
||
if on_done:
|
||
on_done()
|
||
|
||
def stop(self) -> None:
|
||
sd.stop()
|
||
with self._lock:
|
||
self._playing = False
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Factory
|
||
# ---------------------------------------------------------------------------
|
||
|
||
def create_tts_player(config: dict) -> TTSPlayer:
|
||
engine_name = config.get("tts_engine", "edgetts").lower()
|
||
if engine_name == "kokoro":
|
||
voice = config.get("tts_voice", "af_heart")
|
||
speed = float(config.get("tts_speed", 1.0))
|
||
engine = KokoroTTSEngine(voice=voice, speed=speed)
|
||
elif engine_name == "elevenlabs":
|
||
api_key = config.get("elevenlabs_api_key", "")
|
||
voice_id = config.get("tts_voice", "pNInz6obpgDQGcFmaJgB")
|
||
engine = ElevenLabsTTSEngine(api_key=api_key, voice_id=voice_id)
|
||
else: # edgetts (default)
|
||
voice = config.get("tts_voice", "en-US-GuyNeural")
|
||
engine = EdgeTTSEngine(voice=voice)
|
||
return TTSPlayer(engine)
|