309 lines
10 KiB
Python
309 lines
10 KiB
Python
"""
|
|
Ollama LLM client for MARK XL.
|
|
Replaces Google Gemini API with local Ollama inference.
|
|
All modules that previously used google.generativeai import from here.
|
|
"""
|
|
import json
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from pathlib import Path
|
|
from typing import Generator
|
|
|
|
import requests
|
|
|
|
# Matches a sentence boundary: [.!?] followed by whitespace, or a blank line.
|
|
# Avoids splitting on decimals (3.5) because those have no space after the dot.
|
|
_SENT_END = re.compile(r'(?<=[.!?])\s+|(?<=\n)\s*\n')
|
|
|
|
def get_base_dir() -> Path:
|
|
if getattr(sys, "frozen", False):
|
|
return Path(sys.executable).parent
|
|
return Path(__file__).resolve().parent.parent
|
|
|
|
|
|
BASE_DIR = get_base_dir()
|
|
CONFIG_PATH = BASE_DIR / "config" / "api_keys.json"
|
|
|
|
_DEFAULTS = {
|
|
"llm_url": "http://localhost:11434",
|
|
"llm_model": "llama3.2",
|
|
}
|
|
|
|
|
|
def _load_config() -> dict:
|
|
try:
|
|
return json.loads(CONFIG_PATH.read_text(encoding="utf-8"))
|
|
except Exception:
|
|
return {}
|
|
|
|
|
|
def ensure_ollama_running(timeout: int = 15) -> bool:
|
|
"""
|
|
Ping Ollama; if it's not responding, launch 'ollama serve' in the background
|
|
and wait up to `timeout` seconds for it to become ready.
|
|
Returns True if Ollama is available, False otherwise.
|
|
"""
|
|
url, _ = get_llm_settings()
|
|
health = f"{url}/api/tags"
|
|
|
|
def _is_up() -> bool:
|
|
try:
|
|
return requests.get(health, timeout=3).status_code == 200
|
|
except Exception:
|
|
return False
|
|
|
|
if _is_up():
|
|
return True
|
|
|
|
print("[LLM] Ollama not running — launching 'ollama serve'…")
|
|
try:
|
|
kwargs: dict = {"stdout": subprocess.DEVNULL, "stderr": subprocess.DEVNULL}
|
|
if sys.platform == "win32":
|
|
kwargs["creationflags"] = subprocess.CREATE_NO_WINDOW
|
|
subprocess.Popen(["ollama", "serve"], **kwargs)
|
|
except FileNotFoundError:
|
|
print("[LLM] 'ollama' command not found. Install Ollama from https://ollama.com")
|
|
return False
|
|
except Exception as e:
|
|
print(f"[LLM] Could not launch Ollama: {e}")
|
|
return False
|
|
|
|
deadline = time.time() + timeout
|
|
while time.time() < deadline:
|
|
time.sleep(1.0)
|
|
if _is_up():
|
|
print("[LLM] Ollama started successfully.")
|
|
return True
|
|
|
|
print("[LLM] Ollama did not respond within the timeout.")
|
|
return False
|
|
|
|
|
|
def warmup_model() -> bool:
|
|
"""
|
|
Pre-load the model into Ollama memory so the first real request is instant.
|
|
Sends a minimal 1-token request with keep_alive=-1.
|
|
Returns True on success, False if warmup failed (non-fatal).
|
|
"""
|
|
url, model = get_llm_settings()
|
|
print(f"[LLM] Warming up '{model}'…")
|
|
payload = {
|
|
"model": model,
|
|
"messages": [{"role": "user", "content": "hi"}],
|
|
"stream": False,
|
|
"keep_alive": -1,
|
|
"options": {"num_predict": 1},
|
|
}
|
|
try:
|
|
resp = requests.post(f"{url}/api/chat", json=payload, timeout=120)
|
|
resp.raise_for_status()
|
|
print(f"[LLM] '{model}' loaded and ready.")
|
|
return True
|
|
except Exception as e:
|
|
print(f"[LLM] Warmup failed (non-fatal): {e}")
|
|
return False
|
|
|
|
|
|
def get_llm_settings() -> tuple[str, str]:
|
|
"""Returns (base_url, model_name)."""
|
|
cfg = _load_config()
|
|
url = cfg.get("llm_url", _DEFAULTS["llm_url"]).rstrip("/")
|
|
model = cfg.get("llm_model", _DEFAULTS["llm_model"])
|
|
return url, model
|
|
|
|
|
|
def call_llm(
|
|
messages: list,
|
|
tools: list | None = None,
|
|
timeout: int = 120,
|
|
) -> dict:
|
|
"""
|
|
Chat request to Ollama (with optional tool declarations).
|
|
|
|
Returns:
|
|
{"content": str, "tool_calls": list}
|
|
"""
|
|
url, model = get_llm_settings()
|
|
endpoint = f"{url}/api/chat"
|
|
|
|
payload: dict = {
|
|
"model": model,
|
|
"messages": messages,
|
|
"stream": False,
|
|
"keep_alive": -1,
|
|
"options": {"num_predict": 600},
|
|
}
|
|
if tools:
|
|
payload["tools"] = tools
|
|
|
|
try:
|
|
resp = requests.post(endpoint, json=payload, timeout=timeout)
|
|
resp.raise_for_status()
|
|
data = resp.json()
|
|
msg = data.get("message", {})
|
|
return {
|
|
"content": (msg.get("content") or "").strip(),
|
|
"tool_calls": msg.get("tool_calls") or [],
|
|
}
|
|
except requests.exceptions.ConnectionError as e:
|
|
print(f"[LLM] ConnectionError — trying to restart Ollama… ({e})")
|
|
if ensure_ollama_running():
|
|
try:
|
|
resp = requests.post(endpoint, json=payload, timeout=timeout)
|
|
resp.raise_for_status()
|
|
data = resp.json()
|
|
msg = data.get("message", {})
|
|
return {
|
|
"content": (msg.get("content") or "").strip(),
|
|
"tool_calls": msg.get("tool_calls") or [],
|
|
}
|
|
except Exception:
|
|
pass
|
|
raise RuntimeError(
|
|
f"Cannot connect to Ollama at {url}. "
|
|
"Make sure Ollama is installed and run: ollama serve"
|
|
)
|
|
except requests.exceptions.Timeout:
|
|
raise RuntimeError("Ollama request timed out after 120 s.")
|
|
except requests.exceptions.HTTPError as e:
|
|
print(f"[LLM] HTTPError: {e.response.status_code} — {e.response.text[:200]}")
|
|
raise RuntimeError(f"Ollama HTTP error: {e.response.status_code}")
|
|
except Exception as e:
|
|
print(f"[LLM] Unexpected error: {type(e).__name__}: {e}")
|
|
raise RuntimeError(f"LLM call failed: {e}")
|
|
|
|
|
|
def call_llm_text(
|
|
prompt: str,
|
|
system: str | None = None,
|
|
model: str | None = None,
|
|
timeout: int = 120,
|
|
) -> str:
|
|
"""
|
|
Simple text-only generation (no tools).
|
|
Used by planner, executor, error_handler, code_helper, dev_agent.
|
|
"""
|
|
url, default_model = get_llm_settings()
|
|
endpoint = f"{url}/api/chat"
|
|
m = model or default_model
|
|
|
|
messages: list[dict] = []
|
|
if system:
|
|
messages.append({"role": "system", "content": system})
|
|
messages.append({"role": "user", "content": prompt})
|
|
|
|
payload = {"model": m, "messages": messages, "stream": False, "keep_alive": -1, "options": {"num_predict": 600}}
|
|
|
|
try:
|
|
resp = requests.post(endpoint, json=payload, timeout=timeout)
|
|
resp.raise_for_status()
|
|
return (resp.json().get("message", {}).get("content") or "").strip()
|
|
except requests.exceptions.ConnectionError:
|
|
if ensure_ollama_running():
|
|
try:
|
|
resp = requests.post(endpoint, json=payload, timeout=timeout)
|
|
resp.raise_for_status()
|
|
return (resp.json().get("message", {}).get("content") or "").strip()
|
|
except Exception:
|
|
pass
|
|
raise RuntimeError(
|
|
f"Cannot connect to Ollama at {url}. "
|
|
"Make sure Ollama is installed and run: ollama serve"
|
|
)
|
|
except Exception as e:
|
|
raise RuntimeError(f"LLM text call failed: {e}")
|
|
|
|
|
|
def call_llm_stream(
|
|
messages: list,
|
|
tools: list | None = None,
|
|
timeout: int = 120,
|
|
) -> Generator[dict, None, None]:
|
|
"""
|
|
Streaming chat request to Ollama.
|
|
|
|
Yields:
|
|
{"type": "sentence", "text": str} — each complete sentence as it arrives
|
|
{"type": "done", "content": str, "tool_calls": list} — when stream ends
|
|
|
|
Sentences are split on [.!?] + whitespace so TTS can start immediately.
|
|
Tool calls (if any) always appear in the final "done" event with empty content.
|
|
"""
|
|
url, model = get_llm_settings()
|
|
endpoint = f"{url}/api/chat"
|
|
|
|
payload: dict = {
|
|
"model": model,
|
|
"messages": messages,
|
|
"stream": True,
|
|
"keep_alive": -1,
|
|
"options": {"num_predict": 600},
|
|
}
|
|
if tools:
|
|
payload["tools"] = tools
|
|
|
|
def _do_stream() -> Generator[dict, None, None]:
|
|
with requests.post(endpoint, json=payload, timeout=timeout, stream=True) as resp:
|
|
resp.raise_for_status()
|
|
full_content = ""
|
|
tool_calls: list = []
|
|
buf = ""
|
|
|
|
for raw in resp.iter_lines():
|
|
if not raw:
|
|
continue
|
|
try:
|
|
chunk = json.loads(raw)
|
|
except json.JSONDecodeError:
|
|
continue
|
|
|
|
msg = chunk.get("message", {})
|
|
delta = msg.get("content") or ""
|
|
full_content += delta
|
|
buf += delta
|
|
|
|
# Yield complete sentences as they accumulate
|
|
while True:
|
|
m = _SENT_END.search(buf)
|
|
if not m:
|
|
break
|
|
sentence = buf[: m.start() + 1].strip()
|
|
buf = buf[m.end() :]
|
|
if sentence:
|
|
yield {"type": "sentence", "text": sentence}
|
|
|
|
tc = msg.get("tool_calls")
|
|
if tc:
|
|
tool_calls.extend(tc)
|
|
|
|
if chunk.get("done"):
|
|
if buf.strip():
|
|
yield {"type": "sentence", "text": buf.strip()}
|
|
yield {
|
|
"type": "done",
|
|
"content": full_content.strip(),
|
|
"tool_calls": tool_calls,
|
|
}
|
|
return
|
|
|
|
try:
|
|
yield from _do_stream()
|
|
except requests.exceptions.ConnectionError as e:
|
|
print(f"[LLM] Stream ConnectionError — trying to restart Ollama… ({e})")
|
|
if ensure_ollama_running():
|
|
yield from _do_stream()
|
|
return
|
|
raise RuntimeError(
|
|
f"Cannot connect to Ollama at {url}. "
|
|
"Make sure Ollama is installed and run: ollama serve"
|
|
)
|
|
except requests.exceptions.Timeout:
|
|
raise RuntimeError("Ollama stream timed out.")
|
|
except requests.exceptions.HTTPError as e:
|
|
raise RuntimeError(f"Ollama HTTP error: {e.response.status_code}")
|
|
except Exception as e:
|
|
print(f"[LLM] Stream error: {type(e).__name__}: {e}")
|
|
raise RuntimeError(f"LLM stream failed: {e}")
|