Files
Mark-LIV/core/llm_client.py
T
2026-06-06 17:41:38 +03:00

309 lines
10 KiB
Python

"""
Ollama LLM client for MARK XL.
Replaces Google Gemini API with local Ollama inference.
All modules that previously used google.generativeai import from here.
"""
import json
import re
import subprocess
import sys
import time
from pathlib import Path
from typing import Generator
import requests
# Matches a sentence boundary: [.!?] followed by whitespace, or a blank line.
# Avoids splitting on decimals (3.5) because those have no space after the dot.
_SENT_END = re.compile(r'(?<=[.!?])\s+|(?<=\n)\s*\n')
def get_base_dir() -> Path:
if getattr(sys, "frozen", False):
return Path(sys.executable).parent
return Path(__file__).resolve().parent.parent
BASE_DIR = get_base_dir()
CONFIG_PATH = BASE_DIR / "config" / "api_keys.json"
_DEFAULTS = {
"llm_url": "http://localhost:11434",
"llm_model": "llama3.2",
}
def _load_config() -> dict:
try:
return json.loads(CONFIG_PATH.read_text(encoding="utf-8"))
except Exception:
return {}
def ensure_ollama_running(timeout: int = 15) -> bool:
"""
Ping Ollama; if it's not responding, launch 'ollama serve' in the background
and wait up to `timeout` seconds for it to become ready.
Returns True if Ollama is available, False otherwise.
"""
url, _ = get_llm_settings()
health = f"{url}/api/tags"
def _is_up() -> bool:
try:
return requests.get(health, timeout=3).status_code == 200
except Exception:
return False
if _is_up():
return True
print("[LLM] Ollama not running — launching 'ollama serve'…")
try:
kwargs: dict = {"stdout": subprocess.DEVNULL, "stderr": subprocess.DEVNULL}
if sys.platform == "win32":
kwargs["creationflags"] = subprocess.CREATE_NO_WINDOW
subprocess.Popen(["ollama", "serve"], **kwargs)
except FileNotFoundError:
print("[LLM] 'ollama' command not found. Install Ollama from https://ollama.com")
return False
except Exception as e:
print(f"[LLM] Could not launch Ollama: {e}")
return False
deadline = time.time() + timeout
while time.time() < deadline:
time.sleep(1.0)
if _is_up():
print("[LLM] Ollama started successfully.")
return True
print("[LLM] Ollama did not respond within the timeout.")
return False
def warmup_model() -> bool:
"""
Pre-load the model into Ollama memory so the first real request is instant.
Sends a minimal 1-token request with keep_alive=-1.
Returns True on success, False if warmup failed (non-fatal).
"""
url, model = get_llm_settings()
print(f"[LLM] Warming up '{model}'…")
payload = {
"model": model,
"messages": [{"role": "user", "content": "hi"}],
"stream": False,
"keep_alive": -1,
"options": {"num_predict": 1},
}
try:
resp = requests.post(f"{url}/api/chat", json=payload, timeout=120)
resp.raise_for_status()
print(f"[LLM] '{model}' loaded and ready.")
return True
except Exception as e:
print(f"[LLM] Warmup failed (non-fatal): {e}")
return False
def get_llm_settings() -> tuple[str, str]:
"""Returns (base_url, model_name)."""
cfg = _load_config()
url = cfg.get("llm_url", _DEFAULTS["llm_url"]).rstrip("/")
model = cfg.get("llm_model", _DEFAULTS["llm_model"])
return url, model
def call_llm(
messages: list,
tools: list | None = None,
timeout: int = 120,
) -> dict:
"""
Chat request to Ollama (with optional tool declarations).
Returns:
{"content": str, "tool_calls": list}
"""
url, model = get_llm_settings()
endpoint = f"{url}/api/chat"
payload: dict = {
"model": model,
"messages": messages,
"stream": False,
"keep_alive": -1,
"options": {"num_predict": 600},
}
if tools:
payload["tools"] = tools
try:
resp = requests.post(endpoint, json=payload, timeout=timeout)
resp.raise_for_status()
data = resp.json()
msg = data.get("message", {})
return {
"content": (msg.get("content") or "").strip(),
"tool_calls": msg.get("tool_calls") or [],
}
except requests.exceptions.ConnectionError as e:
print(f"[LLM] ConnectionError — trying to restart Ollama… ({e})")
if ensure_ollama_running():
try:
resp = requests.post(endpoint, json=payload, timeout=timeout)
resp.raise_for_status()
data = resp.json()
msg = data.get("message", {})
return {
"content": (msg.get("content") or "").strip(),
"tool_calls": msg.get("tool_calls") or [],
}
except Exception:
pass
raise RuntimeError(
f"Cannot connect to Ollama at {url}. "
"Make sure Ollama is installed and run: ollama serve"
)
except requests.exceptions.Timeout:
raise RuntimeError("Ollama request timed out after 120 s.")
except requests.exceptions.HTTPError as e:
print(f"[LLM] HTTPError: {e.response.status_code} — {e.response.text[:200]}")
raise RuntimeError(f"Ollama HTTP error: {e.response.status_code}")
except Exception as e:
print(f"[LLM] Unexpected error: {type(e).__name__}: {e}")
raise RuntimeError(f"LLM call failed: {e}")
def call_llm_text(
prompt: str,
system: str | None = None,
model: str | None = None,
timeout: int = 120,
) -> str:
"""
Simple text-only generation (no tools).
Used by planner, executor, error_handler, code_helper, dev_agent.
"""
url, default_model = get_llm_settings()
endpoint = f"{url}/api/chat"
m = model or default_model
messages: list[dict] = []
if system:
messages.append({"role": "system", "content": system})
messages.append({"role": "user", "content": prompt})
payload = {"model": m, "messages": messages, "stream": False, "keep_alive": -1, "options": {"num_predict": 600}}
try:
resp = requests.post(endpoint, json=payload, timeout=timeout)
resp.raise_for_status()
return (resp.json().get("message", {}).get("content") or "").strip()
except requests.exceptions.ConnectionError:
if ensure_ollama_running():
try:
resp = requests.post(endpoint, json=payload, timeout=timeout)
resp.raise_for_status()
return (resp.json().get("message", {}).get("content") or "").strip()
except Exception:
pass
raise RuntimeError(
f"Cannot connect to Ollama at {url}. "
"Make sure Ollama is installed and run: ollama serve"
)
except Exception as e:
raise RuntimeError(f"LLM text call failed: {e}")
def call_llm_stream(
messages: list,
tools: list | None = None,
timeout: int = 120,
) -> Generator[dict, None, None]:
"""
Streaming chat request to Ollama.
Yields:
{"type": "sentence", "text": str} — each complete sentence as it arrives
{"type": "done", "content": str, "tool_calls": list} — when stream ends
Sentences are split on [.!?] + whitespace so TTS can start immediately.
Tool calls (if any) always appear in the final "done" event with empty content.
"""
url, model = get_llm_settings()
endpoint = f"{url}/api/chat"
payload: dict = {
"model": model,
"messages": messages,
"stream": True,
"keep_alive": -1,
"options": {"num_predict": 600},
}
if tools:
payload["tools"] = tools
def _do_stream() -> Generator[dict, None, None]:
with requests.post(endpoint, json=payload, timeout=timeout, stream=True) as resp:
resp.raise_for_status()
full_content = ""
tool_calls: list = []
buf = ""
for raw in resp.iter_lines():
if not raw:
continue
try:
chunk = json.loads(raw)
except json.JSONDecodeError:
continue
msg = chunk.get("message", {})
delta = msg.get("content") or ""
full_content += delta
buf += delta
# Yield complete sentences as they accumulate
while True:
m = _SENT_END.search(buf)
if not m:
break
sentence = buf[: m.start() + 1].strip()
buf = buf[m.end() :]
if sentence:
yield {"type": "sentence", "text": sentence}
tc = msg.get("tool_calls")
if tc:
tool_calls.extend(tc)
if chunk.get("done"):
if buf.strip():
yield {"type": "sentence", "text": buf.strip()}
yield {
"type": "done",
"content": full_content.strip(),
"tool_calls": tool_calls,
}
return
try:
yield from _do_stream()
except requests.exceptions.ConnectionError as e:
print(f"[LLM] Stream ConnectionError — trying to restart Ollama… ({e})")
if ensure_ollama_running():
yield from _do_stream()
return
raise RuntimeError(
f"Cannot connect to Ollama at {url}. "
"Make sure Ollama is installed and run: ollama serve"
)
except requests.exceptions.Timeout:
raise RuntimeError("Ollama stream timed out.")
except requests.exceptions.HTTPError as e:
raise RuntimeError(f"Ollama HTTP error: {e.response.status_code}")
except Exception as e:
print(f"[LLM] Stream error: {type(e).__name__}: {e}")
raise RuntimeError(f"LLM stream failed: {e}")