Files
Mark-LIV/core/tts.py
T
2026-06-06 17:41:38 +03:00

277 lines
9.2 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""
Text-to-Speech engines for MARK XL.
EdgeTTS – free Microsoft TTS (internet required, no API key)
Kokoro – fully offline neural TTS (~330 MB model)
ElevenLabs – cloud API (API key required, best quality)
"""
from __future__ import annotations
import asyncio
import threading
from typing import Callable, Optional
import numpy as np
import sounddevice as sd
# ---------------------------------------------------------------------------
# Audio playback helpers
# ---------------------------------------------------------------------------
def _to_numpy(samples) -> np.ndarray:
"""Convert samples to float32 numpy array.
Handles both numpy arrays and PyTorch tensors (Kokoro >= 0.9).
PyTorch built against numpy 1.x raises RuntimeError('Numpy is not available')
when numpy 2.x is installed. The .tolist() fallback always works regardless
of PyTorch / numpy version pairing.
"""
if hasattr(samples, "detach"): # PyTorch tensor
t = samples.detach().cpu().float()
try:
return t.numpy() # fast path (compatible versions)
except RuntimeError:
# PyTorch/numpy version mismatch — convert via Python list (always safe)
return np.asarray(t.tolist(), dtype=np.float32)
return np.asarray(samples, dtype=np.float32)
def _compress_silence(
arr: np.ndarray,
sample_rate: int = 24_000,
max_silence_ms: int = 300, # cap long pauses to this many ms
threshold: float = 0.0015, # RMS below this = silence (very conservative)
) -> np.ndarray:
"""
Shorten excessively long silent segments in Kokoro audio.
Kokoro can produce 1-2 s pauses after punctuation. This caps them
to max_silence_ms while leaving natural short pauses intact.
threshold is intentionally low (0.0015) so soft consonants like
's', 'f', 'h' are never misidentified as silence.
"""
max_samp = int(max_silence_ms * sample_rate / 1000)
frame_len = 240 # ~10 ms at 24 kHz
out: list[np.ndarray] = []
silent_acc = 0
for i in range(0, len(arr), frame_len):
chunk = arr[i : i + frame_len]
if np.sqrt(np.mean(chunk ** 2) + 1e-12) < threshold:
silent_acc += len(chunk)
if silent_acc <= max_samp:
out.append(chunk)
else:
silent_acc = 0
out.append(chunk)
return np.concatenate(out) if out else arr
def _play_np(samples, sample_rate: int) -> None:
"""Play float32 mono (or stereo) audio via sounddevice.
Accepts numpy arrays or PyTorch tensors.
"""
sd.play(_to_numpy(samples), sample_rate)
sd.wait()
def _play_audio_bytes(audio_bytes: bytes) -> None:
"""Decode MP3/WAV/OGG bytes and play via sounddevice (uses miniaudio)."""
import miniaudio
decoded = miniaudio.decode(
audio_bytes,
output_format=miniaudio.SampleFormat.FLOAT32,
nchannels=1,
)
samples = np.array(decoded.samples, dtype=np.float32)
sd.play(samples, decoded.sample_rate)
sd.wait()
# ---------------------------------------------------------------------------
# Engines
# ---------------------------------------------------------------------------
class EdgeTTSEngine:
"""Microsoft EdgeTTS – free, requires internet."""
def __init__(self, voice: str = "en-US-GuyNeural"):
self.voice = voice
def speak(self, text: str) -> None:
loop = asyncio.new_event_loop()
try:
audio_bytes = loop.run_until_complete(self._synth(text))
finally:
loop.close()
if audio_bytes:
_play_audio_bytes(audio_bytes)
async def _synth(self, text: str) -> bytes:
import edge_tts
comm = edge_tts.Communicate(text, self.voice)
buf = bytearray()
async for chunk in comm.stream():
if chunk["type"] == "audio":
buf.extend(chunk["data"])
return bytes(buf)
# Kokoro voice prefix → KPipeline lang_code mapping
_KOKORO_LANG_CODES = {
"a": "a", # American English (af_*, am_*)
"b": "b", # British English (bf_*, bm_*)
"j": "j", # Japanese (jf_*, jm_*)
"z": "z", # Mandarin Chinese (zf_*, zm_*)
"s": "s", # Spanish (sf_*, sm_*)
"f": "f", # French (ff_*, fm_*)
"h": "h", # Hindi (hf_*, hm_*)
"i": "i", # Italian (if_*, im_*)
"p": "p", # Brazilian Portuguese
"r": "r", # Russian (rf_*, rm_*)
"e": "e", # German (ef_*, em_*)
}
class KokoroTTSEngine:
"""Fully offline Kokoro neural TTS.
Model (~330 MB) is downloaded from HuggingFace on first use.
Subsequent runs load from local cache instantly.
"""
def __init__(self, voice: str = "af_heart", speed: float = 1.2):
self.voice = voice
self.speed = speed # 1.0 = normal, 1.2 = 20% faster (recommended)
self._pipeline = None
self._lock = threading.Lock()
# Eager init: download/load model now so it's ready before first speak()
self._init()
# Detect lang_code from voice prefix (e.g. "af_heart" → "a")
@property
def _lang_code(self) -> str:
prefix = self.voice[0].lower() if self.voice else "a"
return _KOKORO_LANG_CODES.get(prefix, "a")
def _init(self) -> None:
if self._pipeline is not None:
return
try:
from kokoro import KPipeline
lang = self._lang_code
print(f"[TTS] Loading Kokoro pipeline (lang='{lang}', voice='{self.voice}')…")
print("[TTS] Kokoro model will be downloaded (~330 MB) if not cached.")
self._pipeline = KPipeline(lang_code=lang)
print("[TTS] Kokoro pipeline ready.")
except Exception as e:
print(f"[TTS] Kokoro init error: {e}")
raise
def speak(self, text: str) -> None:
with self._lock:
if self._pipeline is None:
self._init()
for _, _, audio in self._pipeline(text, voice=self.voice, speed=self.speed):
if audio is not None:
arr = _to_numpy(audio)
arr = _compress_silence(arr) # shorten long punctuation pauses
if arr.size > 0:
_play_np(arr, 24000)
class ElevenLabsTTSEngine:
"""ElevenLabs cloud TTS – API key required."""
def __init__(self, api_key: str, voice_id: str = "pNInz6obpgDQGcFmaJgB"):
self.api_key = api_key
self.voice_id = voice_id
def speak(self, text: str) -> None:
import requests
headers = {
"xi-api-key": self.api_key,
"Content-Type": "application/json",
}
payload = {
"text": text,
"model_id": "eleven_multilingual_v2",
"voice_settings": {"stability": 0.5, "similarity_boost": 0.75},
}
resp = requests.post(
f"https://api.elevenlabs.io/v1/text-to-speech/{self.voice_id}",
json=payload, headers=headers, timeout=30,
)
resp.raise_for_status()
_play_audio_bytes(resp.content)
# ---------------------------------------------------------------------------
# Thread-safe player wrapper
# ---------------------------------------------------------------------------
class TTSPlayer:
"""
Wraps any *Engine. Exposes a blocking speak() method
meant to be called from a dedicated background thread.
"""
def __init__(self, engine):
self._engine = engine
self._playing = False
self._lock = threading.Lock()
@property
def is_playing(self) -> bool:
return self._playing
def speak(
self,
text: str,
on_start: Optional[Callable] = None,
on_done: Optional[Callable] = None,
) -> None:
"""Synthesise and play text. BLOCKING – call from a dedicated thread."""
try:
with self._lock:
self._playing = True
if on_start:
on_start()
self._engine.speak(text)
except Exception as e:
print(f"[TTS] Error: {e}")
finally:
with self._lock:
self._playing = False
if on_done:
on_done()
def stop(self) -> None:
sd.stop()
with self._lock:
self._playing = False
# ---------------------------------------------------------------------------
# Factory
# ---------------------------------------------------------------------------
def create_tts_player(config: dict) -> TTSPlayer:
engine_name = config.get("tts_engine", "edgetts").lower()
if engine_name == "kokoro":
voice = config.get("tts_voice", "af_heart")
speed = float(config.get("tts_speed", 1.2))
engine = KokoroTTSEngine(voice=voice, speed=speed)
elif engine_name == "elevenlabs":
api_key = config.get("elevenlabs_api_key", "")
voice_id = config.get("tts_voice", "pNInz6obpgDQGcFmaJgB")
engine = ElevenLabsTTSEngine(api_key=api_key, voice_id=voice_id)
else: # edgetts (default)
voice = config.get("tts_voice", "en-US-GuyNeural")
engine = EdgeTTSEngine(voice=voice)
return TTSPlayer(engine)