Add files via upload
This commit is contained in:
1 parent
d3238af8bd
commit
c131a1b4f7
16 files changed
+3036
-505
No files matched your search
@@ -1,4 +1,4 @@
|
||||
MARK 51 — JARVIS
|
||||
MARK LII — JARVIS
|
||||
Copyright (c) 2026 FatihMakes
|
||||
|
||||
Licensed under the Creative Commons Attribution-NonCommercial 4.0
|
||||
|
||||
+240
-38
@@ -21,6 +21,9 @@ try:
|
||||
except ImportError:
|
||||
_PYPERCLIP = False
|
||||
|
||||
from core import confirm
|
||||
from core.undo import push_undo
|
||||
|
||||
_OS = platform.system() # "Windows" | "Darwin" | "Linux"
|
||||
|
||||
if _OS == "Windows":
|
||||
@@ -87,6 +90,75 @@ def volume_mute():
|
||||
subprocess.run(["pactl", "set-sink-mute", "@DEFAULT_SINK@", "toggle"],
|
||||
capture_output=True)
|
||||
|
||||
def volume_get() -> int | None:
|
||||
"""Current master volume 0-100, or None if this platform will not say.
|
||||
|
||||
Undo needs a "before" value, and reading one is cheap on every OS we
|
||||
support. Where it is not readable the action simply is not registered as
|
||||
undoable — a wrong undo is worse than no undo."""
|
||||
try:
|
||||
if _OS == "Windows":
|
||||
import math
|
||||
from ctypes import cast, POINTER
|
||||
from comtypes import CLSCTX_ALL
|
||||
from pycaw.pycaw import AudioUtilities, IAudioEndpointVolume
|
||||
devices = AudioUtilities.GetSpeakers()
|
||||
interface = devices.Activate(IAudioEndpointVolume._iid_, CLSCTX_ALL, None)
|
||||
vol = cast(interface, POINTER(IAudioEndpointVolume))
|
||||
db = vol.GetMasterVolumeLevel()
|
||||
if db <= -65.0:
|
||||
return 0
|
||||
return max(0, min(100, round(10 ** (db / 20) * 100)))
|
||||
if _OS == "Darwin":
|
||||
r = subprocess.run(["osascript", "-e", "output volume of (get volume settings)"],
|
||||
capture_output=True, text=True, timeout=5)
|
||||
return max(0, min(100, int(r.stdout.strip())))
|
||||
r = subprocess.run(["pactl", "get-sink-volume", "@DEFAULT_SINK@"],
|
||||
capture_output=True, text=True, timeout=5)
|
||||
m = re.search(r"(\d+)%", r.stdout)
|
||||
return max(0, min(100, int(m.group(1)))) if m else None
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def brightness_get() -> int | None:
|
||||
"""Current brightness 0-100, or None where it cannot be read."""
|
||||
try:
|
||||
if _OS == "Windows":
|
||||
r = subprocess.run(
|
||||
["powershell", "-Command",
|
||||
"(Get-WmiObject -Namespace root/wmi -Class WmiMonitorBrightness)"
|
||||
".CurrentBrightness"],
|
||||
capture_output=True, text=True, timeout=5, **_WIN_HIDE
|
||||
)
|
||||
return max(0, min(100, int(r.stdout.strip())))
|
||||
if _OS == "Linux" and subprocess.run(
|
||||
["which", "brightnessctl"], capture_output=True).returncode == 0:
|
||||
cur = int(subprocess.run(["brightnessctl", "get"],
|
||||
capture_output=True, text=True, timeout=5).stdout.strip())
|
||||
mx = int(subprocess.run(["brightnessctl", "max"],
|
||||
capture_output=True, text=True, timeout=5).stdout.strip())
|
||||
return max(0, min(100, round(cur * 100 / mx))) if mx else None
|
||||
except Exception:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def brightness_set(value: int) -> None:
|
||||
"""Set brightness to an absolute percentage. Only used to restore a value
|
||||
captured before a change, so it is undo's counterpart to the up/down pair."""
|
||||
value = max(0, min(100, int(value)))
|
||||
if _OS == "Windows":
|
||||
subprocess.run(
|
||||
["powershell", "-Command",
|
||||
"(Get-WmiObject -Namespace root/wmi -Class WmiMonitorBrightnessMethods)"
|
||||
f".WmiSetBrightness(1, {value})"],
|
||||
capture_output=True, timeout=5, **_WIN_HIDE
|
||||
)
|
||||
elif _OS == "Linux":
|
||||
subprocess.run(["brightnessctl", "set", f"{value}%"], capture_output=True)
|
||||
|
||||
|
||||
def volume_set(value: int):
|
||||
value = max(0, min(100, int(value)))
|
||||
if _OS == "Windows":
|
||||
@@ -582,43 +654,132 @@ ACTION_MAP: dict[str, callable] = {
|
||||
"shutdown": shutdown_computer,
|
||||
}
|
||||
|
||||
_DANGEROUS_ACTIONS = {"restart", "shutdown"}
|
||||
# ── What needs a human, and what just needs an undo ──────────────────────────
|
||||
#
|
||||
# The old gate was `_DANGEROUS_ACTIONS = {"restart", "shutdown"}` checked against
|
||||
# a `confirmed` parameter the MODEL filled in — so the model confirmed its own
|
||||
# shutdowns, and everything else (switching off the WiFi this assistant is
|
||||
# talking over) had no gate at all.
|
||||
#
|
||||
# Two lists now, and the split is about reversibility, not about how alarming
|
||||
# the word sounds:
|
||||
#
|
||||
# _IRREVERSIBLE — a human presses a button on the HUD. Nothing else.
|
||||
# everything else — done immediately, with an undo pushed if it can be undone.
|
||||
#
|
||||
# Asking before every action is what makes an assistant unusable, and every
|
||||
# question costs a round trip. Undo is both faster and safer than a prompt.
|
||||
_IRREVERSIBLE = {
|
||||
"restart": ("Restart this computer",
|
||||
"Anything unsaved will be lost. The computer restarts in 10 seconds."),
|
||||
"shutdown": ("Shut this computer down",
|
||||
"Anything unsaved will be lost. The computer powers off in 10 seconds."),
|
||||
# Not obviously destructive, and that is exactly why it was missed: turning
|
||||
# the WiFi off cuts the assistant's own connection to the Live API, so it
|
||||
# cannot be asked to turn it back on.
|
||||
"toggle_wifi": ("Switch WiFi off or on",
|
||||
"If this switches WiFi off, JARVIS loses its connection and "
|
||||
"cannot switch it back on by voice."),
|
||||
}
|
||||
|
||||
# Kept so anything still importing the old name keeps working.
|
||||
_DANGEROUS_ACTIONS = set(_IRREVERSIBLE)
|
||||
|
||||
|
||||
# ── Local intent resolution ──────────────────────────────────────────────────
|
||||
#
|
||||
# This used to be an entire extra Gemini call, made INSIDE the tool: the Live
|
||||
# model called computer_settings, and computer_settings then asked a second
|
||||
# model which action was meant. Every "turn the volume down" paid for two round
|
||||
# trips — and when that second call failed, the fallback was
|
||||
# `description.lower().replace(" ", "_")`, which turns the Turkish for "turn it
|
||||
# down" into `sesi_kis`, i.e. straight to "Unknown action".
|
||||
#
|
||||
# Nothing here needs a language model. The Live model already understands the
|
||||
# sentence; it only needed the vocabulary, which the tool declaration now spells
|
||||
# out in full. What is left is spelling tolerance, and difflib does that in
|
||||
# microseconds instead of ~600 ms and a quota unit.
|
||||
_ALIASES = {
|
||||
"volume_up": ("louder", "raise volume", "turn it up", "increase volume"),
|
||||
"volume_down": ("quieter", "lower volume", "turn it down", "decrease volume"),
|
||||
"mute": ("silence", "sound off", "no sound"),
|
||||
"brightness_up": ("brighter", "raise brightness", "increase brightness"),
|
||||
"brightness_down": ("dimmer", "dim", "lower brightness", "decrease brightness"),
|
||||
"close_window": ("close this", "close it"),
|
||||
"full_screen": ("fullscreen", "maximise screen"),
|
||||
"show_desktop": ("minimise everything", "go to desktop"),
|
||||
"lock_screen": ("lock", "lock the pc", "lock computer"),
|
||||
"sleep_display": ("screen off", "turn off the screen", "display off"),
|
||||
"dark_mode": ("night mode", "light mode", "toggle theme"),
|
||||
"toggle_wifi": ("wifi", "wi-fi", "internet off", "internet on"),
|
||||
"task_manager": ("processes", "task list"),
|
||||
"screenshot": ("capture screen", "take a screenshot", "snip"),
|
||||
"refresh_page": ("refresh", "reload page"),
|
||||
"new_tab": ("open a tab", "open new tab"),
|
||||
"shutdown": ("power off", "turn off the computer", "switch off the pc"),
|
||||
"restart": ("reboot", "restart the pc"),
|
||||
}
|
||||
|
||||
_VALUE_ACTIONS = {"volume_set", "type_text", "press_key", "reload_n",
|
||||
"scroll_up", "scroll_down"}
|
||||
|
||||
|
||||
def _normalise(text: str) -> str:
|
||||
return (text or "").strip().lower().replace("-", "_").replace(" ", "_")
|
||||
|
||||
|
||||
def _detect_action(description: str) -> dict:
|
||||
"""Resolve a free-text description to an action name, locally.
|
||||
|
||||
from google import genai as _genai
|
||||
_client = _genai.Client(api_key=_get_api_key())
|
||||
Returns {"action": name, "value": ...}; `action` is "" when nothing matched,
|
||||
which the caller turns into a message naming real candidates — one round
|
||||
trip, and only in the case that used to cost one anyway."""
|
||||
raw = (description or "").strip()
|
||||
norm = _normalise(raw)
|
||||
if not norm:
|
||||
return {"action": "", "value": None}
|
||||
|
||||
available = ", ".join(sorted(ACTION_MAP.keys())) + \
|
||||
", volume_set, type_text, press_key, reload_n"
|
||||
known = set(ACTION_MAP) | _VALUE_ACTIONS
|
||||
|
||||
prompt = f"""You are an intent detector for a computer control assistant.
|
||||
# 1. Already an action name.
|
||||
if norm in known:
|
||||
return {"action": norm, "value": None}
|
||||
|
||||
The user issued a command (possibly in any language): "{description}"
|
||||
low = raw.lower()
|
||||
|
||||
Available actions: {available}
|
||||
# 2. "set volume to 30", "sesi 30 yap" — a number next to a volume word.
|
||||
num = re.search(r"(\d{1,3})\s*%?", low)
|
||||
if num and any(w in low for w in ("volume", "ses", "sound", "lautstark", "громкость")):
|
||||
return {"action": "volume_set", "value": max(0, min(100, int(num.group(1))))}
|
||||
|
||||
Return ONLY a valid JSON object:
|
||||
{{"action": "action_name", "value": null_or_value}}
|
||||
# 3. Alias phrases.
|
||||
for action, phrases in _ALIASES.items():
|
||||
if any(_normalise(p) == norm or p in low for p in phrases):
|
||||
return {"action": action, "value": None}
|
||||
|
||||
Rules:
|
||||
- Pick the single best matching action from the available list.
|
||||
- For volume_set: value is an integer 0-100.
|
||||
- For type_text: value is the exact text to type.
|
||||
- For press_key: value is the key name (e.g. "f5", "tab", "enter").
|
||||
- For reload_n: value is an integer (number of times to reload).
|
||||
- If no clear match, pick the closest action.
|
||||
- Return ONLY the JSON, no explanation, no markdown."""
|
||||
# 4. Fuzzy match on the action names — catches "fullscren", "volumeup".
|
||||
import difflib
|
||||
close = difflib.get_close_matches(norm, sorted(known), n=1, cutoff=0.72)
|
||||
if close:
|
||||
return {"action": close[0], "value": None}
|
||||
|
||||
try:
|
||||
resp = _client.models.generate_content(model="gemini-flash-lite-latest", contents=prompt)
|
||||
text = re.sub(r"```(?:json)?", "", resp.text).strip().rstrip("`").strip()
|
||||
return json.loads(text)
|
||||
except Exception as e:
|
||||
print(f"[Settings] Intent detection failed: {e}")
|
||||
return {"action": description.lower().replace(" ", "_"), "value": None}
|
||||
# 5. Substring: "increase_the_brightness" contains "brightness".
|
||||
for action in sorted(known, key=len, reverse=True):
|
||||
if len(action) > 4 and (action in norm or norm in action):
|
||||
return {"action": action, "value": None}
|
||||
|
||||
return {"action": "", "value": None}
|
||||
|
||||
|
||||
def _suggest(description: str) -> str:
|
||||
"""What to tell the model when nothing matched. Names real actions so its
|
||||
retry lands, instead of the old 'Unknown action' dead end."""
|
||||
import difflib
|
||||
near = difflib.get_close_matches(_normalise(description),
|
||||
sorted(ACTION_MAP), n=5, cutoff=0.3)
|
||||
hint = ", ".join(near) if near else ", ".join(sorted(ACTION_MAP)[:12])
|
||||
return (f"I could not match '{description}' to a computer action. "
|
||||
f"Call computer_settings again with an exact `action` from: {hint}.")
|
||||
|
||||
def computer_settings(
|
||||
parameters: dict = None,
|
||||
@@ -643,24 +804,39 @@ def computer_settings(
|
||||
action = raw_action.lower().strip().replace(" ", "_").replace("-", "_")
|
||||
|
||||
if not action:
|
||||
return "No action could be determined."
|
||||
return _suggest(description or raw_action)
|
||||
|
||||
print(f"[Settings] Action: {action} Value: {value} OS: {_OS}")
|
||||
if player:
|
||||
player.write_log(f"[Settings] {action}")
|
||||
|
||||
if action in _DANGEROUS_ACTIONS:
|
||||
confirmed = str(params.get("confirmed", "")).lower()
|
||||
if confirmed not in ("yes", "true", "1", "confirm"):
|
||||
return (
|
||||
f"This will {action} the computer. "
|
||||
f"Please confirm by calling again with confirmed=yes."
|
||||
)
|
||||
# ── The gate ─────────────────────────────────────────────────────────────
|
||||
# A human presses a button, or this does not happen. The model can no longer
|
||||
# write its own permission slip, and the action itself is handed to the UI
|
||||
# rather than performed here — so returning early is not "declining", it is
|
||||
# "parked until someone says yes".
|
||||
if action in _IRREVERSIBLE:
|
||||
title, detail = _IRREVERSIBLE[action]
|
||||
func = ACTION_MAP.get(action)
|
||||
if func is None:
|
||||
return f"Unknown action: '{raw_action}'."
|
||||
if confirm.pending_title():
|
||||
return ("There is already a confirmation waiting on screen. "
|
||||
"Ask the user to answer that one first.")
|
||||
return confirm.request(
|
||||
key=action, title=title, detail=detail,
|
||||
run=lambda f=func, a=action: (f(), f"{a} done.")[1],
|
||||
)
|
||||
|
||||
if action == "volume_set":
|
||||
try:
|
||||
volume_set(int(value or 50))
|
||||
return f"Volume set to {value}%."
|
||||
target = int(value if value is not None else 50)
|
||||
before = volume_get()
|
||||
volume_set(target)
|
||||
if before is not None:
|
||||
push_undo(f"volume {before}% → {target}%",
|
||||
lambda b=before: (volume_set(b), f"Back to {b}%.")[1])
|
||||
return f"Volume set to {target}%."
|
||||
except Exception as e:
|
||||
return f"Could not set volume: {e}"
|
||||
|
||||
@@ -696,11 +872,37 @@ def computer_settings(
|
||||
|
||||
func = ACTION_MAP.get(action)
|
||||
if not func:
|
||||
return f"Unknown action: '{raw_action}'."
|
||||
return _suggest(raw_action or description)
|
||||
|
||||
# ── Capture "before" so the change can be taken back ─────────────────────
|
||||
# Read-then-write is the whole mechanism for settings: there is no clever
|
||||
# inverse to compute, just a value to remember. Where the platform will not
|
||||
# tell us the current value, nothing is registered — an undo that restores
|
||||
# a guess is worse than no undo at all.
|
||||
_before = None
|
||||
if action in ("volume_up", "volume_down", "mute", "unmute", "toggle_mute"):
|
||||
_before = ("volume", volume_get())
|
||||
elif action in ("brightness_up", "brightness_down"):
|
||||
_before = ("brightness", brightness_get())
|
||||
|
||||
try:
|
||||
func()
|
||||
return f"Done: {action}."
|
||||
except Exception as e:
|
||||
print(f"[Settings] Action failed ({action}): {e}")
|
||||
return f"Action failed ({action}): {e}"
|
||||
return f"Action failed ({action}): {e}"
|
||||
|
||||
if _before:
|
||||
kind, old = _before
|
||||
if old is not None:
|
||||
if kind == "volume":
|
||||
push_undo(f"volume ({action})",
|
||||
lambda b=old: (volume_set(b), f"Volume back to {b}%.")[1])
|
||||
elif kind == "brightness":
|
||||
push_undo(f"brightness ({action})",
|
||||
lambda b=old: (brightness_set(b), f"Brightness back to {b}%.")[1])
|
||||
elif action == "dark_mode":
|
||||
# A pure toggle: calling it again is the undo.
|
||||
push_undo("dark mode toggled",
|
||||
lambda: (dark_mode(), "Theme switched back.")[1])
|
||||
|
||||
return f"Done: {action}."
|
||||
+173
-7
@@ -10,8 +10,86 @@ try:
|
||||
except ImportError:
|
||||
_SEND2TRASH = False
|
||||
|
||||
from core.undo import push_undo
|
||||
|
||||
_OS = platform.system() # "Windows" | "Darwin" | "Linux"
|
||||
|
||||
# Undo keeps a file's previous contents in memory so `write` can be reversed.
|
||||
# Above this size it does not — a 200 MB log would sit in RAM for the rest of
|
||||
# the session to protect an edit nobody is going to take back.
|
||||
_UNDO_CONTENT_LIMIT = 1_000_000
|
||||
|
||||
|
||||
def _undo_move(src: Path, dst: Path):
|
||||
"""Reverse of a move: put it back where it came from."""
|
||||
def _fn():
|
||||
if not dst.exists():
|
||||
return f"'{dst.name}' is no longer there — nothing moved back."
|
||||
src.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.move(str(dst), str(src))
|
||||
return f"'{src.name}' is back in {src.parent.name}/."
|
||||
return _fn
|
||||
|
||||
|
||||
def _undo_create(target: Path):
|
||||
"""Reverse of a create: remove what we made — and only if we still made it.
|
||||
|
||||
Deliberately refuses to touch a directory that has since been filled: the
|
||||
undo for 'create a folder' is not 'delete whatever ended up in it'."""
|
||||
def _fn():
|
||||
if not target.exists():
|
||||
return f"'{target.name}' is already gone."
|
||||
if target.is_dir():
|
||||
if any(target.iterdir()):
|
||||
return (f"'{target.name}' is not empty any more — "
|
||||
f"leaving it alone rather than deleting your files.")
|
||||
target.rmdir()
|
||||
else:
|
||||
target.unlink()
|
||||
return f"Removed '{target.name}'."
|
||||
return _fn
|
||||
|
||||
|
||||
def _undo_write(target: Path, previous: str | None):
|
||||
"""Reverse of a write: restore the old contents, or remove a file that did
|
||||
not exist before the write created it."""
|
||||
def _fn():
|
||||
if previous is None:
|
||||
if target.exists():
|
||||
target.unlink()
|
||||
return f"Removed '{target.name}' — it did not exist before."
|
||||
return f"'{target.name}' is already gone."
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
target.write_text(previous, encoding="utf-8")
|
||||
return f"Restored the previous contents of '{target.name}'."
|
||||
return _fn
|
||||
|
||||
|
||||
def _restore_from_trash(original: Path) -> str:
|
||||
"""Best-effort undelete.
|
||||
|
||||
delete_file uses send2trash, which is the right call: the file lands in the
|
||||
Recycle Bin / Trash where the person can also find it themselves. Getting it
|
||||
back out again is shell work and only reliable on Windows, where pywin32 is
|
||||
already a dependency. Everywhere else this says where the file is instead of
|
||||
pretending it failed — the file is not lost either way."""
|
||||
if _OS == "Windows":
|
||||
try:
|
||||
import win32com.client
|
||||
shell = win32com.client.Dispatch("Shell.Application")
|
||||
bin_folder = shell.NameSpace(10) # ssfBITBUCKET
|
||||
for item in bin_folder.Items():
|
||||
if str(bin_folder.GetDetailsOf(item, 1)).strip().lower() == \
|
||||
str(original.parent).strip().lower():
|
||||
if str(item.Name).strip().lower() == original.name.strip().lower():
|
||||
item.InvokeVerb("UNDELETE")
|
||||
return f"'{original.name}' restored from the Recycle Bin."
|
||||
except Exception as e:
|
||||
print(f"[file] Recycle Bin restore failed: {e}")
|
||||
return (f"'{original.name}' is in the Recycle Bin — I could not pull it back "
|
||||
f"automatically, but it is there and can be restored by hand.")
|
||||
|
||||
|
||||
_SAFE_ROOTS: list[Path] = [
|
||||
Path.home(),
|
||||
]
|
||||
@@ -85,11 +163,12 @@ def _resolve_path(raw: str) -> Path:
|
||||
if lower in shortcuts:
|
||||
return shortcuts[lower]
|
||||
|
||||
# "desktop/notlar/a.md" ve "desktop\notlar\a.md" — kisayol + alt yol.
|
||||
# Bu dal olmadan tum dize asagidaki goreli-yol dalina dusuyor ve process'in
|
||||
# CWD'sine gore cozuluyordu. Proje home disindaysa _is_safe_path reddedip
|
||||
# "Access denied" veriyor; home icindeyse daha kotusu oluyor ve dosya sessizce
|
||||
# projenin icindeki olmayan bir "desktop" klasorune yaziliyordu.
|
||||
# "desktop/notes/a.md" and "desktop\notes\a.md" — a shortcut followed by a
|
||||
# sub-path. Without this branch the whole string falls through to the
|
||||
# relative-path return below and is resolved against the process CWD instead
|
||||
# of the real Desktop: an "Access denied" when the project lives outside the
|
||||
# home directory, or — worse — a silent write into a stray "desktop" folder
|
||||
# inside the project when it lives inside it.
|
||||
head, sep, rest = raw.replace("\\", "/").partition("/")
|
||||
if sep and head.lower() in shortcuts:
|
||||
rest = rest.strip("/")
|
||||
@@ -154,7 +233,16 @@ def create_file(path: str, name: str = "", content: str = "") -> str:
|
||||
if not _is_safe_path(target):
|
||||
return f"Access denied: {target}"
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
existed = target.exists()
|
||||
previous = None
|
||||
if existed:
|
||||
try:
|
||||
previous = target.read_text(encoding="utf-8", errors="ignore")
|
||||
except Exception:
|
||||
previous = None
|
||||
target.write_text(content, encoding="utf-8")
|
||||
push_undo(f"created {target.name}",
|
||||
_undo_write(target, previous) if existed else _undo_create(target))
|
||||
return f"File created: {target.name}"
|
||||
except Exception as e:
|
||||
return f"Could not create file: {e}"
|
||||
@@ -166,7 +254,13 @@ def create_folder(path: str, name: str = "") -> str:
|
||||
target = (base / name) if name else base
|
||||
if not _is_safe_path(target):
|
||||
return f"Access denied: {target}"
|
||||
already = target.exists()
|
||||
target.mkdir(parents=True, exist_ok=True)
|
||||
# Only offer to undo a folder we actually made. "mkdir -p" on something
|
||||
# that was already there is not a change, and undoing it would delete a
|
||||
# directory the user has had for years.
|
||||
if not already:
|
||||
push_undo(f"created folder {target.name}", _undo_create(target))
|
||||
return f"Folder created: {target.name}"
|
||||
except Exception as e:
|
||||
return f"Could not create folder: {e}"
|
||||
@@ -189,7 +283,12 @@ def delete_file(path: str, name: str = "") -> str:
|
||||
if target.resolve() in {p.resolve() for p in protected}:
|
||||
return f"Protected directory, cannot delete: {target.name}"
|
||||
|
||||
return _safe_trash(target)
|
||||
original = target.resolve()
|
||||
result = _safe_trash(target)
|
||||
if result.startswith("Moved to Trash"):
|
||||
push_undo(f"deleted {original.name}",
|
||||
lambda p=original: _restore_from_trash(p))
|
||||
return result
|
||||
|
||||
except PermissionError:
|
||||
return f"Permission denied: {path}"
|
||||
@@ -216,7 +315,10 @@ def move_file(path: str, name: str = "", destination: str = "") -> str:
|
||||
dst = dst / src.name
|
||||
|
||||
dst.parent.mkdir(parents=True, exist_ok=True)
|
||||
origin = src.resolve()
|
||||
shutil.move(str(src), str(dst))
|
||||
push_undo(f"moved {origin.name} to {dst.parent.name}/",
|
||||
_undo_move(origin, dst.resolve()))
|
||||
return f"Moved: {src.name} → {dst.parent.name}/"
|
||||
|
||||
except Exception as e:
|
||||
@@ -248,6 +350,18 @@ def copy_file(path: str, name: str = "", destination: str = "") -> str:
|
||||
else:
|
||||
shutil.copy2(str(src), str(dst))
|
||||
|
||||
# The undo for a copy is deleting the copy — never the original.
|
||||
_copy = dst.resolve()
|
||||
def _undo_copy():
|
||||
if not _copy.exists():
|
||||
return f"The copy '{_copy.name}' is already gone."
|
||||
if _copy.is_dir():
|
||||
shutil.rmtree(_copy)
|
||||
else:
|
||||
_copy.unlink()
|
||||
return f"Removed the copy in {_copy.parent.name}/."
|
||||
push_undo(f"copied {src.name} to {dst.parent.name}/", _undo_copy)
|
||||
|
||||
return f"Copied: {src.name} → {dst.parent.name}/"
|
||||
|
||||
except Exception as e:
|
||||
@@ -269,7 +383,10 @@ def rename_file(path: str, name: str = "", new_name: str = "") -> str:
|
||||
if new_path.exists():
|
||||
return f"A file named '{new_name}' already exists here."
|
||||
|
||||
old_path = target.resolve()
|
||||
target.rename(new_path)
|
||||
push_undo(f"renamed {old_path.name} to {new_name}",
|
||||
_undo_move(old_path, new_path.resolve()))
|
||||
return f"Renamed: {target.name} → {new_name}"
|
||||
|
||||
except Exception as e:
|
||||
@@ -304,11 +421,31 @@ def write_file(path: str, name: str = "", content: str = "",
|
||||
if not _is_safe_path(target):
|
||||
return f"Access denied: {target}"
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Snapshot before writing. None means "did not exist", which is a
|
||||
# different undo (delete it) from "existed and had this in it".
|
||||
previous: str | None = None
|
||||
undoable = True
|
||||
if target.exists():
|
||||
try:
|
||||
if target.stat().st_size > _UNDO_CONTENT_LIMIT:
|
||||
undoable = False # too large to hold in memory
|
||||
else:
|
||||
previous = target.read_text(encoding="utf-8", errors="ignore")
|
||||
except Exception:
|
||||
undoable = False # binary, locked, unreadable
|
||||
|
||||
mode = "a" if append else "w"
|
||||
with open(target, mode, encoding="utf-8") as f:
|
||||
f.write(content)
|
||||
|
||||
action = "Appended to" if append else "Written to"
|
||||
return f"{action}: {target.name}"
|
||||
if undoable:
|
||||
push_undo(f"wrote to {target.name}", _undo_write(target, previous))
|
||||
return f"{action}: {target.name}"
|
||||
return (f"{action}: {target.name}. "
|
||||
f"(Too large to keep a copy of the old contents, so this one "
|
||||
f"cannot be undone.)")
|
||||
except Exception as e:
|
||||
return f"Could not write file: {e}"
|
||||
|
||||
@@ -415,6 +552,7 @@ def organize_desktop() -> str:
|
||||
|
||||
desktop = _get_desktop()
|
||||
moved, skipped = [], []
|
||||
journal: list[tuple[Path, Path]] = [] # (where it was, where it went)
|
||||
|
||||
try:
|
||||
for item in desktop.iterdir():
|
||||
@@ -438,9 +576,37 @@ def organize_desktop() -> str:
|
||||
skipped.append(item.name)
|
||||
continue
|
||||
|
||||
origin = item.resolve()
|
||||
shutil.move(str(item), str(new_path))
|
||||
journal.append((origin, new_path.resolve()))
|
||||
moved.append(f"{item.name} → {target_dir.name}/")
|
||||
|
||||
# One command, dozens of moves — so one undo that reverses all of them.
|
||||
# Without this, "organize my desktop" is the single least reversible
|
||||
# thing the assistant can do to a person's files, and it was completely
|
||||
# ungated.
|
||||
if journal:
|
||||
def _undo_organize(entries=tuple(journal)):
|
||||
restored = 0
|
||||
for origin, moved_to in entries:
|
||||
try:
|
||||
if moved_to.exists():
|
||||
origin.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.move(str(moved_to), str(origin))
|
||||
restored += 1
|
||||
except Exception as e:
|
||||
print(f"[file] undo organize: {moved_to.name}: {e}")
|
||||
# Clear away the folders we created, but only while they are
|
||||
# empty — anything the user put in since stays.
|
||||
for folder in {m.parent for _o, m in entries}:
|
||||
try:
|
||||
if folder.exists() and folder.is_dir() and not any(folder.iterdir()):
|
||||
folder.rmdir()
|
||||
except Exception:
|
||||
pass
|
||||
return f"{restored} file(s) put back on the desktop."
|
||||
push_undo(f"organized the desktop ({len(journal)} files)", _undo_organize)
|
||||
|
||||
result = f"Desktop organized: {len(moved)} files moved."
|
||||
if moved:
|
||||
preview = moved[:8]
|
||||
|
||||
+127
-124
@@ -1,124 +1,127 @@
|
||||
"""
|
||||
ProactiveEngine 2.0 — context-aware, time-aware, non-repetitive background prompting.
|
||||
Gemini decides what to say; this module decides WHEN and builds a rich context snapshot.
|
||||
"""
|
||||
import time
|
||||
from datetime import datetime
|
||||
|
||||
|
||||
class ProactiveEngine:
|
||||
"""
|
||||
Decides when JARVIS should speak unprompted and builds a context-rich prompt.
|
||||
|
||||
Improvements over 1.0:
|
||||
- Time-of-day awareness (morning / afternoon / evening / night)
|
||||
- Monitor-topic awareness (what the user is tracking)
|
||||
- Recent-session context (last few turns of the current conversation)
|
||||
- Non-repetitive (rotates context focus to avoid same opener)
|
||||
- Smarter silence gate (doesn't fire while JARVIS is speaking)
|
||||
|
||||
Defaults:
|
||||
min_silence_secs — 900 s (15 min) user must be silent before any check
|
||||
check_cooldown — 1200 s (20 min) minimum gap between proactive messages
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
min_silence_secs: int = 900,
|
||||
check_cooldown: int = 1200,
|
||||
):
|
||||
self.min_silence_secs = min_silence_secs
|
||||
self.check_cooldown = check_cooldown
|
||||
self._last_triggered = 0.0
|
||||
self._rotation = 0 # cycles through context focus areas
|
||||
|
||||
# ── Trigger gate ───────────────────────────────────────────────────────────
|
||||
|
||||
def should_trigger(self, last_user_speech: float) -> bool:
|
||||
now = time.monotonic()
|
||||
return (
|
||||
(now - last_user_speech) >= self.min_silence_secs
|
||||
and (now - self._last_triggered) >= self.check_cooldown
|
||||
)
|
||||
|
||||
def mark_triggered(self) -> None:
|
||||
self._last_triggered = time.monotonic()
|
||||
self._rotation += 1
|
||||
|
||||
# ── Prompt builder ─────────────────────────────────────────────────────────
|
||||
|
||||
def build_prompt(
|
||||
self,
|
||||
memory: dict,
|
||||
monitors: list[str] | None = None,
|
||||
recent_turns: list[str] | None = None,
|
||||
) -> str:
|
||||
"""
|
||||
Build a context snapshot for Gemini.
|
||||
Rotates through three focus areas so proactive messages don't repeat.
|
||||
"""
|
||||
from memory.memory_manager import format_memory_for_prompt
|
||||
|
||||
now = datetime.now()
|
||||
hour = now.hour
|
||||
time_str = now.strftime("%A, %B %d, %Y — %I:%M %p")
|
||||
|
||||
# Time-of-day label
|
||||
if 6 <= hour < 12: period = "morning"
|
||||
elif 12 <= hour < 18: period = "afternoon"
|
||||
elif 18 <= hour < 23: period = "evening"
|
||||
else: period = "late night"
|
||||
|
||||
mem_str = format_memory_for_prompt(memory) or "(no stored user data)"
|
||||
|
||||
# Rotating context focus (cycles every trigger)
|
||||
focus_index = self._rotation % 3
|
||||
if focus_index == 0:
|
||||
focus = (
|
||||
"Focus on the user's active projects or goals if any are stored. "
|
||||
"Ask how something is going, or offer a relevant tip."
|
||||
)
|
||||
elif focus_index == 1:
|
||||
focus = (
|
||||
"Focus on the time of day and the user's wellbeing. "
|
||||
"A warm check-in, a reminder to take a break, or something timely."
|
||||
)
|
||||
else:
|
||||
focus = (
|
||||
"Focus on something genuinely interesting or useful — "
|
||||
"a fact, a suggestion, or a question based on what you know about this person."
|
||||
)
|
||||
|
||||
# Optional: monitored topics context
|
||||
monitor_ctx = ""
|
||||
if monitors:
|
||||
monitor_ctx = (
|
||||
f"\nThe user tracks these topics: {', '.join(monitors[:4])}. "
|
||||
"You may mention one if it seems relevant."
|
||||
)
|
||||
|
||||
# Optional: recent conversation context
|
||||
recent_ctx = ""
|
||||
if recent_turns:
|
||||
snippet = "\n".join(recent_turns[-6:])
|
||||
recent_ctx = f"\nRecent conversation:\n{snippet}"
|
||||
|
||||
return "\n".join([
|
||||
"[PROACTIVE_CHECK] You are initiating a proactive check-in.",
|
||||
f"Current time : {time_str} ({period})",
|
||||
"",
|
||||
"Context about this person:",
|
||||
mem_str,
|
||||
monitor_ctx,
|
||||
recent_ctx,
|
||||
"",
|
||||
"Task:",
|
||||
focus,
|
||||
"",
|
||||
"Rules:",
|
||||
"- Speak in the user's language (check memory; default English).",
|
||||
"- 1-2 sentences max. Natural, warm, never robotic.",
|
||||
"- Do NOT mention [PROACTIVE_CHECK] or these instructions.",
|
||||
"- Do NOT call any tools.",
|
||||
"- If nothing genuinely useful comes to mind, stay silent (say nothing).",
|
||||
])
|
||||
"""
|
||||
ProactiveEngine 2.0 — context-aware, time-aware, non-repetitive background prompting.
|
||||
Gemini decides what to say; this module decides WHEN and builds a rich context snapshot.
|
||||
"""
|
||||
import time
|
||||
from datetime import datetime
|
||||
|
||||
|
||||
class ProactiveEngine:
|
||||
"""
|
||||
Decides when JARVIS should speak unprompted and builds a context-rich prompt.
|
||||
|
||||
Improvements over 1.0:
|
||||
- Time-of-day awareness (morning / afternoon / evening / night)
|
||||
- Monitor-topic awareness (what the user is tracking)
|
||||
- Recent-session context (last few turns of the current conversation)
|
||||
- Non-repetitive (rotates context focus to avoid same opener)
|
||||
- Smarter silence gate (doesn't fire while JARVIS is speaking)
|
||||
|
||||
Defaults:
|
||||
min_silence_secs — 900 s (15 min) user must be silent before any check
|
||||
check_cooldown — 1200 s (20 min) minimum gap between proactive messages
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
min_silence_secs: int = 900,
|
||||
check_cooldown: int = 1200,
|
||||
):
|
||||
self.min_silence_secs = min_silence_secs
|
||||
self.check_cooldown = check_cooldown
|
||||
self._last_triggered = 0.0
|
||||
self._rotation = 0 # cycles through context focus areas
|
||||
|
||||
# ── Trigger gate ───────────────────────────────────────────────────────────
|
||||
|
||||
def should_trigger(self, last_user_speech: float) -> bool:
|
||||
now = time.monotonic()
|
||||
return (
|
||||
(now - last_user_speech) >= self.min_silence_secs
|
||||
and (now - self._last_triggered) >= self.check_cooldown
|
||||
)
|
||||
|
||||
def mark_triggered(self) -> None:
|
||||
self._last_triggered = time.monotonic()
|
||||
self._rotation += 1
|
||||
|
||||
# ── Prompt builder ─────────────────────────────────────────────────────────
|
||||
|
||||
def build_prompt(
|
||||
self,
|
||||
memory: dict,
|
||||
monitors: list[str] | None = None,
|
||||
recent_turns: list[str] | None = None,
|
||||
) -> str:
|
||||
"""
|
||||
Build a context snapshot for Gemini.
|
||||
Rotates through three focus areas so proactive messages don't repeat.
|
||||
"""
|
||||
from memory.memory_manager import format_memory_for_prompt
|
||||
|
||||
now = datetime.now()
|
||||
hour = now.hour
|
||||
time_str = now.strftime("%A, %B %d, %Y — %I:%M %p")
|
||||
|
||||
# Time-of-day label
|
||||
if 6 <= hour < 12: period = "morning"
|
||||
elif 12 <= hour < 18: period = "afternoon"
|
||||
elif 18 <= hour < 23: period = "evening"
|
||||
else: period = "late night"
|
||||
|
||||
mem_str = format_memory_for_prompt(memory) or "(no stored user data)"
|
||||
|
||||
# Rotating context focus (cycles every trigger)
|
||||
focus_index = self._rotation % 3
|
||||
if focus_index == 0:
|
||||
focus = (
|
||||
"Focus on the user's active projects or goals if any are stored. "
|
||||
"Ask how something is going, or offer a relevant tip."
|
||||
)
|
||||
elif focus_index == 1:
|
||||
focus = (
|
||||
"Focus on the time of day and the user's wellbeing. "
|
||||
"A warm check-in, a reminder to take a break, or something timely."
|
||||
)
|
||||
else:
|
||||
focus = (
|
||||
"Focus on something genuinely interesting or useful — "
|
||||
"a fact, a suggestion, or a question based on what you know about this person."
|
||||
)
|
||||
|
||||
# Optional: monitored topics context
|
||||
monitor_ctx = ""
|
||||
if monitors:
|
||||
monitor_ctx = (
|
||||
f"\nThe user tracks these topics: {', '.join(monitors[:4])}. "
|
||||
"You may mention one if it seems relevant."
|
||||
)
|
||||
|
||||
# Optional: recent conversation context
|
||||
recent_ctx = ""
|
||||
if recent_turns:
|
||||
snippet = "\n".join(recent_turns[-6:])
|
||||
recent_ctx = f"\nRecent conversation:\n{snippet}"
|
||||
|
||||
return "\n".join([
|
||||
"[PROACTIVE_CHECK] You are initiating a proactive check-in.",
|
||||
f"Current time : {time_str} ({period})",
|
||||
"",
|
||||
"Context about this person:",
|
||||
mem_str,
|
||||
monitor_ctx,
|
||||
recent_ctx,
|
||||
"",
|
||||
"Task:",
|
||||
focus,
|
||||
"",
|
||||
"Rules:",
|
||||
"- Speak the language this person actually uses: the one in the "
|
||||
"recent conversation above, or the remembered one if there is no "
|
||||
"conversation yet. Never default to English because these "
|
||||
"instructions are in English.",
|
||||
"- 1-2 sentences max. Natural, warm, never robotic.",
|
||||
"- Do NOT mention [PROACTIVE_CHECK] or these instructions.",
|
||||
"- Do NOT call any tools.",
|
||||
"- If nothing genuinely useful comes to mind, stay silent (say nothing).",
|
||||
])
|
||||
@@ -43,7 +43,7 @@ def _log_gemini_failure(context: str, exc: Exception) -> None:
|
||||
"""Log a Gemini failure — silently when it is just the expected cooldown."""
|
||||
if isinstance(exc, _QuotaCooldown):
|
||||
return # announced once when the breaker tripped; not a warning
|
||||
print(f"[WebSearch] ⚠️ {context} failed ({exc}) — using DDG instead")
|
||||
print(f"[WebSearch] \u26a0\ufe0f {context} failed ({exc}) — using DDG instead")
|
||||
|
||||
|
||||
def _run_bounded(fn, timeout: float, label: str = "task"):
|
||||
|
||||
@@ -0,0 +1,421 @@
|
||||
"""
|
||||
core/audio_devices.py — pick which microphone and which speakers JARVIS uses.
|
||||
|
||||
WHY
|
||||
Both audio streams in main.py were opened without a `device=` argument, so
|
||||
they always took whatever the operating system called "default". On a laptop
|
||||
with a built-in mic, a webcam mic and a headset that is a coin toss — and on
|
||||
Windows the default *moves on its own* the moment you plug a headset in.
|
||||
"JARVIS can't hear me" almost always means "JARVIS is listening to the
|
||||
monitor's microphone".
|
||||
|
||||
WHY NAMES, NOT INDICES
|
||||
sounddevice identifies devices by integer index, and those indices shift
|
||||
whenever a device appears or disappears. Storing index 3 means that after
|
||||
unplugging a USB interface the saved setting silently points at something
|
||||
else. We store the device *name* and resolve it to an index at open time.
|
||||
|
||||
WHY THIS IS CACHED
|
||||
`sd.query_devices()` talks to the host audio API and can take a few hundred
|
||||
milliseconds on a Windows machine with many endpoints. Mark LV learned this
|
||||
lesson the expensive way — a 2.1-second `openwakeword` import on the Qt
|
||||
thread made the settings drawer look like it was broken. So the list is
|
||||
fetched once on a background thread at startup and served from cache.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
import time
|
||||
|
||||
# The label shown for "let the OS decide", and the value stored in config for
|
||||
# it. Empty string, so an untouched install and a deliberately-default install
|
||||
# are the same thing — nothing changes for anyone who never opens the picker.
|
||||
DEFAULT_LABEL = "System default"
|
||||
DEFAULT_VALUE = ""
|
||||
|
||||
_cache: dict[str, list[str]] | None = None
|
||||
_cache_lock = threading.Lock()
|
||||
|
||||
# Which host API each direction settled on, so resolve() opens the same endpoint
|
||||
# the picker listed. Filled in by _query().
|
||||
_chosen_api: dict = {"input": None, "output": None}
|
||||
|
||||
|
||||
# ── Why the raw list is unusable, and what is filtered out ───────────────────
|
||||
#
|
||||
# `sd.query_devices()` returns one entry per (device × host API), not one per
|
||||
# device. Measured on a normal Windows machine: 41 entries for what the Windows
|
||||
# sound settings show as 4 microphones and 4 speakers. The same Realtek
|
||||
# microphone appears four times — once each under MME, DirectSound, WASAPI and
|
||||
# WDM-KS — and none of the four is labelled to say which is which.
|
||||
#
|
||||
# Handing that to a person is not a choice, it is a quiz. So the list is reduced
|
||||
# the way the operating system's own settings panel does it:
|
||||
#
|
||||
# 1. ONE host API per direction — chosen by measurement, not by reasoning.
|
||||
# See the preference note below.
|
||||
# 2. No pseudo-devices. "Microsoft Sound Mapper", "Primary Sound Driver",
|
||||
# ALSA's "default"/"sysdefault"/"dmix" are aliases for "whatever the OS
|
||||
# picks" — which is precisely the "System default" entry already at the top
|
||||
# of the list. Offering them again as if they were hardware is noise.
|
||||
# 3. No zero-channel or unnamed entries (WASAPI reports one of each).
|
||||
# 4. Deduplicated by name.
|
||||
#
|
||||
# Nothing is hidden that a person could actually want: the same hardware is
|
||||
# still there, listed once, under the name their operating system uses for it.
|
||||
|
||||
# ── Preference order, which is only a starting point ─────────────────────────
|
||||
#
|
||||
# Two earlier versions of this file were wrong in the same way: they decided
|
||||
# which host API to use by reasoning about it instead of measuring it.
|
||||
#
|
||||
# v1 ranked APIs by how clean their device names were and picked WASAPI.
|
||||
# WASAPI in shared mode does not resample — the hardware runs at 48 kHz,
|
||||
# this app streams 16 kHz in and 24 kHz out, and every open failed with
|
||||
# "Invalid sample rate". The picker looked right and did nothing.
|
||||
#
|
||||
# v2 added a rate check and picked DirectSound, which passes that check on
|
||||
# both sides. PortAudio's DirectSound *output* is a silent sink: the
|
||||
# stream opens, every write returns success in ~0 ms, and nothing is ever
|
||||
# heard. Same failure, one layer deeper.
|
||||
#
|
||||
# So this list is a preference, not a promise. Which API actually gets used is
|
||||
# decided below by _usable() (open it for real) and _transport_works() (does
|
||||
# audio actually move), per direction. On this platform that lands on
|
||||
# DirectSound for the microphone and MME for the speakers — a split that no
|
||||
# amount of reasoning would have produced.
|
||||
_PREFERRED_APIS = {
|
||||
"Windows": ("directsound", "mme", "wasapi"),
|
||||
# macOS has only Core Audio, so there is nothing to disambiguate.
|
||||
"Darwin": ("core audio",),
|
||||
# PulseAudio/PipeWire present one clean endpoint per device; raw ALSA
|
||||
# presents dozens of routing permutations of the same card.
|
||||
"Linux": ("pulse", "pipewire", "jack", "alsa"),
|
||||
}
|
||||
|
||||
# ── "It opens" is not "it works" ─────────────────────────────────────────────
|
||||
#
|
||||
# Opening a stream successfully proves nothing. Measured, writing 2.0 s of audio:
|
||||
#
|
||||
# device=None (MME) 2.02 s consumed in real time
|
||||
# DirectSound, any output device 0.00 s swallowed instantly
|
||||
#
|
||||
# No flag or capability field reports this. The only thing that separates a real
|
||||
# sink from a fake one is whether it consumes audio at the rate audio is
|
||||
# consumed at — so that is what gets measured, once per host API per direction,
|
||||
# on the background thread at startup, using silence.
|
||||
#
|
||||
# Each direction is probed **the way main.py actually uses it**. That is not a
|
||||
# detail: DirectSound input passes a callback stream and fails a blocking read,
|
||||
# so an earlier version of this probe rejected a microphone that works perfectly
|
||||
# in the app. Probe the mode you ship, not the mode that is easier to write.
|
||||
_PROBE_SECONDS = {"output": 0.6, "input": 0.35}
|
||||
|
||||
# Cache: {(api_name_or_None, kind): bool}
|
||||
_probe_results: dict = {}
|
||||
|
||||
|
||||
def _transport_works(idx: int, kind: str, api_key) -> bool:
|
||||
"""Does this host API actually move audio, or only pretend to?
|
||||
|
||||
Probed once per API per direction and cached. Output writes silence, so the
|
||||
probe is inaudible; input reads and discards."""
|
||||
if api_key in _probe_results:
|
||||
return _probe_results[api_key]
|
||||
|
||||
ok = False
|
||||
try:
|
||||
import sounddevice as sd
|
||||
rate = _RATES.get(kind, 16000)
|
||||
secs = _PROBE_SECONDS.get(kind, 0.5)
|
||||
|
||||
# Each direction is probed the way main.py actually uses it. That is not
|
||||
# a detail: DirectSound input passes a callback stream and fails a
|
||||
# blocking read, so probing the wrong mode rejected a microphone that
|
||||
# works perfectly in the app.
|
||||
if kind == "output":
|
||||
# main.py writes with stream.write() — a real sink is rate-limited
|
||||
# by the hardware clock, a fake one swallows the buffer instantly.
|
||||
st = sd.RawOutputStream(samplerate=rate, channels=1, dtype="int16",
|
||||
blocksize=1024, device=idx)
|
||||
st.start()
|
||||
t0 = time.monotonic()
|
||||
st.write(bytes(int(rate * secs) * 2)) # silence — inaudible
|
||||
elapsed = time.monotonic() - t0
|
||||
st.stop(); st.close()
|
||||
ok = elapsed > secs * 0.5
|
||||
if not ok:
|
||||
print(f"[Audio] output: host API reports success but moves no "
|
||||
f"audio ({elapsed*1000:.0f} ms for {secs*1000:.0f} ms) "
|
||||
f"— skipping it")
|
||||
else:
|
||||
# main.py reads through a callback — count what arrives.
|
||||
frames = [0]
|
||||
|
||||
def _cb(indata, n, *_a):
|
||||
frames[0] += n
|
||||
|
||||
st = sd.InputStream(samplerate=rate, channels=1, dtype="int16",
|
||||
blocksize=1024, device=idx, callback=_cb)
|
||||
st.start()
|
||||
time.sleep(secs)
|
||||
st.stop(); st.close()
|
||||
ok = frames[0] > rate * secs * 0.3
|
||||
if not ok:
|
||||
print(f"[Audio] input: host API delivered {frames[0]} frames in "
|
||||
f"{secs*1000:.0f} ms — skipping it")
|
||||
except Exception as e:
|
||||
print(f"[Audio] {kind} transport probe failed: {e}")
|
||||
ok = False
|
||||
|
||||
_probe_results[api_key] = ok
|
||||
return ok
|
||||
|
||||
|
||||
def _display_name(name: str, devices) -> str:
|
||||
"""MME truncates device names to 31 characters, so the API that actually
|
||||
carries the audio may not be the one that can spell. If another host API
|
||||
knows a longer name that starts with this one, show that instead — the user
|
||||
reads 'Realtek HD Audio 2nd output (Realtek(R) Audio)' while the stream runs
|
||||
on the endpoint called 'Realtek HD Audio 2nd output (Re'."""
|
||||
if len(name) < 30:
|
||||
return name
|
||||
best = name
|
||||
for dev in devices:
|
||||
other = (dev.get("name") or "").strip()
|
||||
if len(other) > len(best) and other.startswith(name):
|
||||
best = other
|
||||
return best
|
||||
|
||||
# The rates the app opens its streams at. Defaults match main.py; main.py calls
|
||||
# configure() at startup with its own constants so the two can never drift apart
|
||||
# and silently reintroduce the bug above.
|
||||
_RATES = {"input": 16000, "output": 24000}
|
||||
|
||||
|
||||
def configure(input_rate: int, output_rate: int) -> None:
|
||||
"""Tell this module the sample rates the audio streams will use, so the
|
||||
picker can rule out devices that cannot be opened at them.
|
||||
|
||||
Drops any cached list: which devices are usable depends on the rate, so a
|
||||
list built under the old rates would be stale."""
|
||||
global _cache
|
||||
_RATES["input"] = int(input_rate)
|
||||
_RATES["output"] = int(output_rate)
|
||||
with _cache_lock:
|
||||
_cache = None
|
||||
|
||||
|
||||
def _usable(idx: int, kind: str) -> bool:
|
||||
"""Can this device actually be opened at the rate we need?
|
||||
|
||||
Deliberately opens a real stream rather than asking
|
||||
`check_output_settings`, because that function lies: it passed for an MME
|
||||
endpoint that then failed to open with "The specified format is not
|
||||
supported or cannot be translated" [MME error 32]. Opening and immediately
|
||||
closing costs milliseconds and is the only answer that holds."""
|
||||
st = None
|
||||
try:
|
||||
import sounddevice as sd
|
||||
rate = _RATES.get(kind, 16000)
|
||||
if kind == "input":
|
||||
st = sd.InputStream(samplerate=rate, channels=1, dtype="int16",
|
||||
blocksize=1024, device=idx,
|
||||
callback=lambda *_a: None)
|
||||
else:
|
||||
st = sd.RawOutputStream(samplerate=rate, channels=1, dtype="int16",
|
||||
blocksize=1024, device=idx)
|
||||
st.start()
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
finally:
|
||||
if st is not None:
|
||||
try:
|
||||
st.stop(); st.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Aliases for "the default device" and internal routing endpoints. Matched
|
||||
# case-insensitively as substrings against the device name.
|
||||
_PSEUDO_DEVICES = (
|
||||
"sound mapper", # Windows MME
|
||||
"primary sound", # Windows DirectSound ("Primary Sound Capture Driver")
|
||||
"sysdefault", # ALSA
|
||||
"default", # ALSA / PulseAudio alias
|
||||
"dmix", "dsnoop", # ALSA software mixing plugins
|
||||
"surround", # ALSA channel-layout permutations of one card
|
||||
"samplerate", "speexrate", "upmix", "vdownmix", "null",
|
||||
)
|
||||
|
||||
|
||||
def _is_pseudo(name: str) -> bool:
|
||||
low = name.lower()
|
||||
return any(tok in low for tok in _PSEUDO_DEVICES)
|
||||
|
||||
|
||||
def _query() -> dict[str, list[str]]:
|
||||
"""Return {'input': [names...], 'output': [names...]}. Never raises.
|
||||
|
||||
Only real, selectable devices — see the note above."""
|
||||
out: dict[str, list[str]] = {"input": [], "output": []}
|
||||
try:
|
||||
import platform
|
||||
import sounddevice as sd
|
||||
|
||||
devices = list(sd.query_devices())
|
||||
try:
|
||||
apis = [a.get("name", "") for a in sd.query_hostapis()]
|
||||
except Exception:
|
||||
apis = []
|
||||
|
||||
preferred = _PREFERRED_APIS.get(platform.system(), ())
|
||||
|
||||
def _collect(api_filter, kind) -> list[tuple[int, str]]:
|
||||
"""(index, name) for named, non-pseudo devices on one side that can
|
||||
be opened at the rate that side runs at."""
|
||||
chan = "max_input_channels" if kind == "input" else "max_output_channels"
|
||||
found, seen = [], set()
|
||||
for idx, dev in enumerate(devices):
|
||||
name = (dev.get("name") or "").strip()
|
||||
if not name or _is_pseudo(name) or name in seen:
|
||||
continue
|
||||
if dev.get(chan, 0) <= 0:
|
||||
continue
|
||||
if api_filter is not None:
|
||||
api = apis[dev["hostapi"]].lower() if dev.get("hostapi", -1) < len(apis) else ""
|
||||
if api_filter not in api:
|
||||
continue
|
||||
if not _usable(idx, kind):
|
||||
continue
|
||||
seen.add(name)
|
||||
found.append((idx, name))
|
||||
return found
|
||||
|
||||
# Each direction picks its own host API. They are genuinely different
|
||||
# problems — on Windows the microphone works on DirectSound while the
|
||||
# speakers only work on MME — and a single global choice cannot be right
|
||||
# for both.
|
||||
for kind in ("input", "output"):
|
||||
for api_filter in list(preferred) + [None]:
|
||||
found = _collect(api_filter, kind)
|
||||
if not found:
|
||||
continue
|
||||
# One probe per API per direction, cached, on this thread.
|
||||
if not _transport_works(found[0][0], kind, (api_filter, kind)):
|
||||
continue
|
||||
_chosen_api[kind] = api_filter
|
||||
out[kind] = [_display_name(n, devices) for _i, n in found]
|
||||
break
|
||||
if out[kind]:
|
||||
print(f"[Audio] {kind}: using "
|
||||
f"{_chosen_api[kind] or 'any host API'} "
|
||||
f"({len(out[kind])} devices)")
|
||||
return out
|
||||
|
||||
except Exception as e:
|
||||
print(f"[Audio] Device enumeration failed: {e}")
|
||||
return out
|
||||
|
||||
|
||||
def prefetch() -> None:
|
||||
"""Warm the cache on a background thread. Called once at startup so the
|
||||
settings drawer never pays for enumeration on the Qt thread."""
|
||||
def _work():
|
||||
global _cache
|
||||
result = _query()
|
||||
with _cache_lock:
|
||||
_cache = result
|
||||
print(f"[Audio] {len(result['input'])} input / "
|
||||
f"{len(result['output'])} output devices found")
|
||||
threading.Thread(target=_work, daemon=True, name="audio-devices").start()
|
||||
|
||||
|
||||
def list_devices(kind: str, refresh: bool = False) -> list[str]:
|
||||
"""Device names for 'input' or 'output'. Falls back to a synchronous query
|
||||
if the prefetch has not landed yet — correctness over the cache."""
|
||||
global _cache
|
||||
with _cache_lock:
|
||||
cached = None if refresh else _cache
|
||||
if cached is None:
|
||||
cached = _query()
|
||||
with _cache_lock:
|
||||
_cache = cached
|
||||
return list(cached.get(kind, []))
|
||||
|
||||
|
||||
def resolve(name: str, kind: str):
|
||||
"""Turn a saved device name into something sounddevice accepts.
|
||||
|
||||
Returns None for "system default" — which is also what we return when the
|
||||
saved device is gone, because a missing headset must degrade to the built-in
|
||||
speakers, not to a crash on startup.
|
||||
|
||||
Candidates are walked in the same host-API order the picker used, so a name
|
||||
the user chose from the WASAPI list resolves to the WASAPI endpoint. Without
|
||||
that ordering a full name would fall through to MME's truncated copy of the
|
||||
same device — which happens to work, but means the setting quietly refers to
|
||||
a different endpoint than the one on screen."""
|
||||
wanted = (name or "").strip()
|
||||
if not wanted or wanted == DEFAULT_LABEL:
|
||||
return None
|
||||
|
||||
try:
|
||||
import platform
|
||||
import sounddevice as sd
|
||||
|
||||
devices = list(sd.query_devices())
|
||||
try:
|
||||
apis = [a.get("name", "") for a in sd.query_hostapis()]
|
||||
except Exception:
|
||||
apis = []
|
||||
|
||||
want_in = (kind == "input")
|
||||
chan_key = "max_input_channels" if want_in else "max_output_channels"
|
||||
|
||||
def _candidates(api_filter):
|
||||
for idx, dev in enumerate(devices):
|
||||
if dev.get(chan_key, 0) <= 0:
|
||||
continue
|
||||
if api_filter is not None:
|
||||
api = apis[dev["hostapi"]].lower() if dev.get("hostapi", -1) < len(apis) else ""
|
||||
if api_filter not in api:
|
||||
continue
|
||||
yield idx, (dev.get("name") or "").strip()
|
||||
|
||||
# The API the picker settled on for this direction comes first — the
|
||||
# endpoint that was listed must be the endpoint that gets opened, or the
|
||||
# setting means something different from what it says. list_devices()
|
||||
# populates it; calling it here is a no-op once the cache is warm.
|
||||
list_devices(kind)
|
||||
chosen = _chosen_api.get(kind)
|
||||
orders = ([chosen] if chosen is not None else []) \
|
||||
+ [a for a in _PREFERRED_APIS.get(platform.system(), ()) if a != chosen] \
|
||||
+ [None]
|
||||
|
||||
# A candidate only counts if it can be opened at the rate this side runs
|
||||
# at. The prefix match matters because the API that carries the audio is
|
||||
# not always the one that can spell: MME truncates names to 31 characters
|
||||
# while DirectSound and WASAPI do not, so the name shown in the picker
|
||||
# can be longer than the name of the endpoint it actually opens.
|
||||
for api_filter in orders:
|
||||
partial = None
|
||||
for idx, dev_name in _candidates(api_filter):
|
||||
if dev_name == wanted:
|
||||
if _usable(idx, kind):
|
||||
return idx
|
||||
continue
|
||||
if partial is None and (dev_name.startswith(wanted[:24])
|
||||
or wanted.startswith(dev_name[:24])):
|
||||
if _usable(idx, kind):
|
||||
partial = idx
|
||||
if partial is not None:
|
||||
return partial
|
||||
|
||||
print(f"[Audio] Saved {kind} device '{wanted}' cannot be opened at "
|
||||
f"{_RATES.get(kind)} Hz on any host API — using system default")
|
||||
return None
|
||||
except Exception as e:
|
||||
print(f"[Audio] resolve({kind}) failed: {e} — using system default")
|
||||
return None
|
||||
+161
@@ -0,0 +1,161 @@
|
||||
"""
|
||||
core/confirm.py — a confirmation the model cannot forge.
|
||||
|
||||
THE PROBLEM WITH THE OLD GATE
|
||||
computer_settings guarded shutdown and restart like this:
|
||||
|
||||
confirmed = str(params.get("confirmed", "")).lower()
|
||||
if confirmed not in ("yes", "true", "1", "confirm"):
|
||||
return "Please confirm by calling again with confirmed=yes."
|
||||
|
||||
`confirmed` is a tool parameter, which means the *model* writes it. Nothing
|
||||
stops it from sending confirmed=yes on the first call, and nothing checks
|
||||
that a human was ever involved. It is a convention, not a gate — and its
|
||||
coverage was two actions, so deleting files and switching off the WiFi the
|
||||
assistant is talking over went through with no gate at all.
|
||||
|
||||
THE DESIGN HERE
|
||||
The confirmation token is issued by the *interface*, never by the model:
|
||||
|
||||
1. An action calls `request(...)` with a callable that does the real work.
|
||||
2. This module hands the UI a banner with CONFIRM / CANCEL and returns
|
||||
IMMEDIATELY with a sentence for the model to say out loud.
|
||||
3. If — and only if — the user presses CONFIRM, the UI calls `resolve()`,
|
||||
which runs the stored callable off the Qt thread.
|
||||
|
||||
Nothing blocks. The model keeps talking while the banner is up, so this
|
||||
costs no latency at all; in fact it is cheaper than the old gate, which
|
||||
burned two tool round trips (reject, then re-call) on every shutdown.
|
||||
|
||||
WHAT BELONGS HERE AND WHAT DOES NOT
|
||||
Only genuinely irreversible things. Anything that can be reversed should be
|
||||
done at once and pushed onto core/undo.py instead — undo is faster than a
|
||||
question, and an assistant that asks before every action is one nobody uses.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from typing import Callable, Optional
|
||||
|
||||
# A pending confirmation is abandoned after this long. Chosen to outlast a
|
||||
# normal "hang on, let me look at the screen" pause without leaving a live
|
||||
# shutdown button sitting on the HUD for the rest of the day.
|
||||
TIMEOUT_SECONDS = 90.0
|
||||
|
||||
|
||||
@dataclass
|
||||
class _Pending:
|
||||
key: str
|
||||
title: str
|
||||
detail: str
|
||||
run: Callable[[], str]
|
||||
at: float
|
||||
|
||||
|
||||
_pending: Optional[_Pending] = None
|
||||
_lock = threading.Lock()
|
||||
|
||||
# Set once at startup by main.py. Signature: (title, detail) -> None for show,
|
||||
# and () -> None for hide. Both are marshalled onto the Qt thread by the UI.
|
||||
_show_cb: Optional[Callable[[str, str], None]] = None
|
||||
_hide_cb: Optional[Callable[[], None]] = None
|
||||
_log_cb: Optional[Callable[[str], None]] = None
|
||||
|
||||
|
||||
def bind(show, hide, log=None) -> None:
|
||||
"""Wire this module to the HUD. Called once from main.py at startup."""
|
||||
global _show_cb, _hide_cb, _log_cb
|
||||
_show_cb, _hide_cb, _log_cb = show, hide, log
|
||||
|
||||
|
||||
def _log(msg: str) -> None:
|
||||
if _log_cb:
|
||||
try:
|
||||
_log_cb(msg)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def request(key: str, title: str, detail: str, run: Callable[[], str]) -> str:
|
||||
"""Park an irreversible action behind the on-screen gate.
|
||||
|
||||
Returns the sentence the tool should hand back to the model — phrased as an
|
||||
instruction so the assistant asks the user out loud in their own language,
|
||||
rather than reading an English string verbatim."""
|
||||
global _pending
|
||||
|
||||
if _show_cb is None:
|
||||
# No interface bound (headless, or a very early call). Refuse rather
|
||||
# than silently performing something irreversible.
|
||||
return (f"I cannot confirm '{title}' right now because the interface is "
|
||||
f"not available, so I have not done it.")
|
||||
|
||||
with _lock:
|
||||
_pending = _Pending(key=key, title=title, detail=detail,
|
||||
run=run, at=time.monotonic())
|
||||
|
||||
try:
|
||||
_show_cb(title, detail)
|
||||
except Exception as e:
|
||||
with _lock:
|
||||
_pending = None
|
||||
return f"Could not ask for confirmation: {e}. Nothing was done."
|
||||
|
||||
_log(f"SYS: Awaiting confirmation — {title}")
|
||||
return (
|
||||
f"[CONFIRMATION_PENDING] I have put a confirmation on screen for: {title}. "
|
||||
f"Say ONE short sentence in the user's own language telling them you need "
|
||||
f"them to confirm it on the HUD before you do it. Do not claim it is done."
|
||||
)
|
||||
|
||||
|
||||
def resolve(accepted: bool) -> None:
|
||||
"""Called by the UI when the user presses CONFIRM or CANCEL.
|
||||
|
||||
Runs the stored callable on a worker thread — this is invoked from the Qt
|
||||
thread, and shutting the machine down from inside a button handler would
|
||||
freeze the interface on its way out."""
|
||||
global _pending
|
||||
|
||||
with _lock:
|
||||
p, _pending = _pending, None
|
||||
|
||||
if _hide_cb:
|
||||
try:
|
||||
_hide_cb()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if p is None:
|
||||
return
|
||||
|
||||
if time.monotonic() - p.at > TIMEOUT_SECONDS:
|
||||
_log(f"SYS: Confirmation expired — {p.title}")
|
||||
return
|
||||
|
||||
if not accepted:
|
||||
_log(f"SYS: Cancelled — {p.title}")
|
||||
return
|
||||
|
||||
def _worker():
|
||||
try:
|
||||
result = p.run() or "Done."
|
||||
_log(f"SYS: Confirmed — {p.title}. {result}")
|
||||
except Exception as e:
|
||||
_log(f"ERR: {p.title} failed — {e}")
|
||||
|
||||
threading.Thread(target=_worker, daemon=True,
|
||||
name=f"confirm-{p.key}").start()
|
||||
|
||||
|
||||
def pending_title() -> str:
|
||||
"""'' when nothing is waiting. Lets an action avoid stacking two banners."""
|
||||
with _lock:
|
||||
if _pending is None:
|
||||
return ""
|
||||
if time.monotonic() - _pending.at > TIMEOUT_SECONDS:
|
||||
return ""
|
||||
return _pending.title
|
||||
+18
-5
@@ -6,21 +6,32 @@ EXECUTION RULES:
|
||||
Vision (screen_process): Call the tool ONCE per user request. Never call it again due to echo, ambient sound, or uncertainty. After calling it, wait for the image result — do NOT call screen_process a second time.
|
||||
One-Call Policy: Never guess. Call tools exactly once. No retries.
|
||||
Memory: Store critical user preferences/context automatically.
|
||||
Recall: Your memory block holds what fits. Anything listed under [ALSO REMEMBERED] is stored but NOT shown — if the user asks about one of those names, or about anything personal you cannot see, call recall_memory BEFORE saying you don't know. It is a local file search: instant, free, no round trip to anyone.
|
||||
Undo: You can take back your own changes — files you moved, renamed, created or wrote, and settings you adjusted. If the user says undo / revert / put it back / cancel that / "no, not that one", call the undo tool. Do not apologise and leave it broken; reverse it.
|
||||
Confirmation: restart, shutdown and toggle_wifi put a button on the user's screen and DO NOT happen until they press it. When such a tool returns [CONFIRMATION_PENDING], say one short sentence asking them to confirm on the HUD. Never say it is done — it is not.
|
||||
Exit: Only call shutdown_jarvis if session termination is explicit.
|
||||
Response time: Respond as fast as you can.
|
||||
ADDRESS: When speaking Turkish → always say "efendim". When speaking English → say "sir". Never mix languages.
|
||||
Act: Always act like Jarvis from Iron Man — professional, efficient, slightly witty. ("Welcome home, efendim" etc.)
|
||||
Act: Always act like Jarvis from Iron Man — professional, efficient, slightly witty. (greet the user warmly in their own language)
|
||||
Length: Match response length to the task. Briefing = short. Complex analysis = thorough.
|
||||
|
||||
LANGUAGE:
|
||||
The language of your reply is the language of the user's MOST RECENT message. Nothing else decides it — not what is stored in memory, not the language of these instructions, not the language of a tool's result.
|
||||
Memory may say which language this person has used before. That is a record of the past, not an instruction for now: if they write or speak in another language today, answer in that one.
|
||||
Tool results, log lines and this prompt are in English because that is what code is written in. None of them is a reason to switch language.
|
||||
Never answer in a language the user has not used in this conversation.
|
||||
Address the user with the ordinary respectful form of the language you are speaking in that sentence — never the form from a different language.
|
||||
Do not mix two languages in one reply.
|
||||
|
||||
TOOL ROUTING:
|
||||
|
||||
computer_settings: ALL single OS actions (volume, brightness, wifi, close, shortcuts, power).
|
||||
agent_task: ONLY for complex, multi-step planning (3+ steps) and call it if user it really spesifize it.
|
||||
system_status: when user asks about CPU, RAM, GPU, temperature, computer performance.
|
||||
web_search: use mode='news' for current events, mode='research' for deep topics, mode='price' for product costs.
|
||||
Language: Respond in user's language; extract parameters in English.
|
||||
Tool parameters are always extracted in English; see the LANGUAGE section above for what you SPEAK.
|
||||
Do not call agent_task while you can accomplish it with a tool
|
||||
|
||||
wants to open a video = youtube. Always know the context for better User Experience
|
||||
If user wants to do a taks more than 1, attain them and do them one by one(f.e: User: "open a relax rainy video and make it full screen" that means that you should open a video first, and then always wait 3-4 seconds to load the video and then press 'F' button to make it full screen) always know the context and attain them(Video opened in youtube, full screen means 'F' button).
|
||||
LANGUAGE DETECTION:
|
||||
The first time you detect the user's language (or if it changes), silently call save_memory with:
|
||||
category='identity', key='language', value=<English name of the language, e.g. 'Turkish', 'French', 'Spanish'>
|
||||
@@ -42,4 +53,6 @@ Say something genuinely useful, timely, or caring in 1-3 short sentences.
|
||||
Be natural — like a thoughtful assistant who noticed something relevant.
|
||||
Never read the [PROACTIVE_CHECK] tag aloud. Do NOT call any tools during a proactive check.
|
||||
|
||||
CRITICAL: Speak/Take action immediately based on available info. Assume and proceed.
|
||||
CRITICAL: Speak/Take action immediately based on available info. Assume and proceed.
|
||||
|
||||
Always react fast, speed is your number 1 priority, don't make it complicated and slow.
|
||||
+121
@@ -0,0 +1,121 @@
|
||||
"""
|
||||
core/undo.py — one shared undo stack for every action that changes state.
|
||||
|
||||
WHY THIS EXISTS
|
||||
A voice assistant misunderstands. When it does, "sorry" is not a remedy —
|
||||
the file is already in another folder and the brightness is already at 10%.
|
||||
Before this module the only recovery was to fix it by hand.
|
||||
|
||||
The alternative — asking "are you sure?" before everything — is worse. Every
|
||||
confirmation is a round trip to the model and back, and an assistant that
|
||||
checks with you before turning the volume down is one you stop talking to.
|
||||
|
||||
So: act immediately, remember how to reverse it, and let the user say
|
||||
"undo". Confirmation is then reserved for the handful of actions that
|
||||
genuinely cannot be reversed (see core/confirm.py).
|
||||
|
||||
HOW AN ACTION OPTS IN
|
||||
Actions do not have to know anything about this file's internals. They
|
||||
capture the "before" state and hand back a zero-argument callable:
|
||||
|
||||
from core.undo import push_undo
|
||||
old = volume_get()
|
||||
volume_set(new)
|
||||
push_undo(f"volume → {new}%", lambda: volume_set(old))
|
||||
|
||||
Only the action itself can know that the reverse of "move A to B" is "move
|
||||
B to A", which is why this cannot be fully centralised. What IS central is
|
||||
the stack, the ordering, the thread safety and the tool the model calls.
|
||||
|
||||
COST
|
||||
Pushing is a list append behind a lock: microseconds. Nothing in here runs
|
||||
until the user actually asks to undo something. This module cannot make the
|
||||
assistant slower.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Callable
|
||||
|
||||
# How many reversible operations we keep. Ten is roughly "this conversation":
|
||||
# far enough back to catch a mistake you noticed a few commands later, short
|
||||
# enough that a closure holding a file's old contents cannot pile up in RAM.
|
||||
MAX_DEPTH = 10
|
||||
|
||||
|
||||
@dataclass
|
||||
class _Entry:
|
||||
label: str # human sentence, spoken back to the user
|
||||
undo: Callable[[], str] # returns a short result string
|
||||
at: float = field(default_factory=time.monotonic)
|
||||
|
||||
|
||||
_stack: list[_Entry] = []
|
||||
_lock = threading.Lock()
|
||||
|
||||
|
||||
def push_undo(label: str, undo_fn: Callable[[], str]) -> None:
|
||||
"""Record that `label` just happened and `undo_fn()` reverses it.
|
||||
|
||||
Called from action handlers, which run in executor threads — hence the
|
||||
lock. Never raises: a broken undo registration must not take down the
|
||||
action that actually succeeded."""
|
||||
if not callable(undo_fn):
|
||||
return
|
||||
try:
|
||||
with _lock:
|
||||
_stack.append(_Entry(label=str(label)[:120], undo=undo_fn))
|
||||
# Drop the oldest rather than refusing the newest: the recent past
|
||||
# is what people ask to undo.
|
||||
while len(_stack) > MAX_DEPTH:
|
||||
_stack.pop(0)
|
||||
except Exception as e: # pragma: no cover
|
||||
print(f"[Undo] push failed: {e}")
|
||||
|
||||
|
||||
def can_undo() -> bool:
|
||||
with _lock:
|
||||
return bool(_stack)
|
||||
|
||||
|
||||
def peek() -> str:
|
||||
"""Label of the operation that `undo_last()` would reverse, or ''."""
|
||||
with _lock:
|
||||
return _stack[-1].label if _stack else ""
|
||||
|
||||
|
||||
def history() -> list[str]:
|
||||
"""Most recent first — used by the UI panel and the `undo` tool's list mode."""
|
||||
with _lock:
|
||||
return [e.label for e in reversed(_stack)]
|
||||
|
||||
|
||||
def undo_last() -> str:
|
||||
"""Reverse the most recent reversible operation.
|
||||
|
||||
The entry is popped *before* running so a failing undo cannot be retried
|
||||
forever against a world that has already moved on (the file the user is
|
||||
trying to restore may have been deleted by something else since)."""
|
||||
with _lock:
|
||||
entry = _stack.pop() if _stack else None
|
||||
|
||||
if entry is None:
|
||||
return ("There is nothing to undo. I only track things I changed myself — "
|
||||
"files I moved or wrote, and settings I adjusted.")
|
||||
|
||||
try:
|
||||
detail = entry.undo() or ""
|
||||
except Exception as e:
|
||||
return f"Could not undo '{entry.label}': {e}"
|
||||
|
||||
return f"Undone: {entry.label}." + (f" {detail}" if detail else "")
|
||||
|
||||
|
||||
def clear() -> None:
|
||||
"""Forget the stack. Called when the app shuts down so closures holding old
|
||||
file contents do not outlive the session."""
|
||||
with _lock:
|
||||
_stack.clear()
|
||||
@@ -1,18 +1,21 @@
|
||||
import platform as _platform
|
||||
import subprocess as _subprocess
|
||||
|
||||
# ── Console encoding ─────────────────────────────────────────────────────────
|
||||
# Windows consoles default to a legacy codepage — cp1254 in Turkey, cp1251 in
|
||||
# Russia, cp932 in Japan. Printing an emoji there raises UnicodeEncodeError, and
|
||||
# several of these prints sit inside except handlers, so the handler itself dies
|
||||
# and skips the recovery code after it. Reconfiguring to UTF-8 with a
|
||||
# replacement fallback costs nothing and makes the app behave in every locale.
|
||||
import sys as _sys
|
||||
|
||||
# ── Make stdout/stderr UTF-8 tolerant ────────────────────────────────────────
|
||||
# On non-UTF-8 Windows consoles (cp1254/cp1252/cp936...) any print() containing
|
||||
# an emoji raises UnicodeEncodeError. Several of those prints sit inside except
|
||||
# handlers, so the handler itself would blow up and skip the recovery code that
|
||||
# follows it — turning a recoverable error into a silent hang. errors="replace"
|
||||
# makes every print safe.
|
||||
for _stream in (_sys.stdout, _sys.stderr):
|
||||
for _stream in ("stdout", "stderr"):
|
||||
try:
|
||||
_stream.reconfigure(encoding="utf-8", errors="replace")
|
||||
_s = getattr(_sys, _stream, None)
|
||||
if _s is not None and hasattr(_s, "reconfigure"):
|
||||
_s.reconfigure(encoding="utf-8", errors="replace")
|
||||
except Exception:
|
||||
pass # frozen builds may have no real stream attached
|
||||
pass # pythonw / redirected pipes / anything exotic — never fatal
|
||||
|
||||
# ── Nuclear: force CREATE_NO_WINDOW on EVERY subprocess call on Windows ───────
|
||||
# This patches Popen itself, so no per-file flag is needed anywhere.
|
||||
@@ -40,12 +43,14 @@ from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
import sounddevice as sd
|
||||
import numpy as np
|
||||
from google import genai
|
||||
from google.genai import types
|
||||
from ui import JarvisUI
|
||||
from memory.memory_manager import (
|
||||
load_memory, update_memory, format_memory_for_prompt,
|
||||
save_session_summary, pop_last_session,
|
||||
search_memory, set_trim_notifier,
|
||||
)
|
||||
|
||||
from actions.file_processor import file_processor
|
||||
@@ -71,8 +76,13 @@ from actions.background_monitor import (
|
||||
add_monitor, remove_monitor, list_monitors, check_all as monitor_check_all,
|
||||
)
|
||||
from actions.web_search import _news as _fetch_news_sync
|
||||
from memory.config_manager import get_brief_enabled
|
||||
from memory.config_manager import (
|
||||
get_brief_enabled, get_voice, get_input_device, get_output_device,
|
||||
)
|
||||
from core.plugin_loader import discover_plugins
|
||||
from core import undo as undo_stack
|
||||
from core import confirm as confirm_gate
|
||||
from core import audio_devices
|
||||
|
||||
def get_base_dir():
|
||||
if getattr(sys, "frozen", False):
|
||||
@@ -88,6 +98,28 @@ SEND_SAMPLE_RATE = 16000
|
||||
RECEIVE_SAMPLE_RATE = 24000
|
||||
CHUNK_SIZE = 1024
|
||||
|
||||
# RMS below which 16-bit PCM is treated as room silence; above _LEVEL_FULL it
|
||||
# reads as a full-height waveform. Tuned so ordinary speech lands mid-range and
|
||||
# the bars still move for a quiet talker — language- and device-independent.
|
||||
_LEVEL_FLOOR = 60.0
|
||||
_LEVEL_FULL = 2600.0
|
||||
|
||||
|
||||
def _pcm_level(samples) -> float:
|
||||
"""Map a block of int16 PCM samples to a 0.0–1.0 loudness level for the HUD
|
||||
waveform. Returns 0.0 on empty/invalid input so it can never raise."""
|
||||
try:
|
||||
x = np.asarray(samples, dtype=np.float32)
|
||||
if x.size == 0:
|
||||
return 0.0
|
||||
rms = float(np.sqrt(np.mean(x * x)))
|
||||
except Exception:
|
||||
return 0.0
|
||||
if rms <= _LEVEL_FLOOR:
|
||||
return 0.0
|
||||
return min(1.0, (rms - _LEVEL_FLOOR) / (_LEVEL_FULL - _LEVEL_FLOOR))
|
||||
|
||||
|
||||
def _get_api_key() -> str:
|
||||
with open(API_CONFIG_PATH, "r", encoding="utf-8") as f:
|
||||
return json.load(f)["gemini_api_key"]
|
||||
@@ -250,14 +282,48 @@ TOOL_DECLARATIONS = [
|
||||
"Controls the computer: volume, brightness, window management, keyboard shortcuts, "
|
||||
"typing text on screen, closing apps, fullscreen, dark mode, WiFi, restart, shutdown, "
|
||||
"scrolling, tab management, zoom, screenshots, lock screen, refresh/reload page. "
|
||||
"Use for ANY single computer control command."
|
||||
"Use for ANY single computer control command. "
|
||||
"restart, shutdown and toggle_wifi put a confirmation on the user's screen "
|
||||
"and do NOT happen until they press it — never claim they are done. "
|
||||
"Volume, brightness and dark mode can be reversed with the `undo` tool."
|
||||
),
|
||||
"parameters": {
|
||||
"type": "OBJECT",
|
||||
"properties": {
|
||||
"action": {"type": "STRING", "description": "The action to perform"},
|
||||
"description": {"type": "STRING", "description": "Natural language description of what to do"},
|
||||
"value": {"type": "STRING", "description": "Optional value: volume level, text to type, etc."}
|
||||
# The exact vocabulary, spelled out.
|
||||
#
|
||||
# This used to say only "The action to perform", so the model
|
||||
# usually filled `description` instead — and computer_settings
|
||||
# then made a SECOND Gemini call, inside the tool, purely to
|
||||
# translate that sentence into one of these names. Every
|
||||
# "turn the volume down" cost two model round trips.
|
||||
"action": {
|
||||
"type": "STRING",
|
||||
"description": (
|
||||
"The exact action. Prefer this over `description` — pick one of: "
|
||||
"volume_up | volume_down | volume_set | mute | "
|
||||
"brightness_up | brightness_down | sleep_display | "
|
||||
"pause_video | close_app | close_window | full_screen | "
|
||||
"minimize | maximize | snap_left | snap_right | "
|
||||
"switch_window | show_desktop | task_manager | focus_search | "
|
||||
"refresh_page | close_tab | new_tab | next_tab | prev_tab | "
|
||||
"go_back | go_forward | zoom_in | zoom_out | zoom_reset | "
|
||||
"find_on_page | scroll_up | scroll_down | scroll_top | "
|
||||
"scroll_bottom | page_up | page_down | copy | paste | cut | "
|
||||
"undo | redo | select_all | save | enter | escape | press_key | "
|
||||
"type_text | screenshot | lock_screen | open_settings | "
|
||||
"file_explorer | open_run | dark_mode | toggle_wifi | "
|
||||
"restart | shutdown"
|
||||
),
|
||||
},
|
||||
"description": {
|
||||
"type": "STRING",
|
||||
"description": (
|
||||
"Fallback only, when no action name above fits. "
|
||||
"Resolved locally — no extra model call."
|
||||
),
|
||||
},
|
||||
"value": {"type": "STRING", "description": "Optional value: volume level 0-100, text to type, key name, etc."}
|
||||
},
|
||||
"required": []
|
||||
}
|
||||
@@ -558,8 +624,98 @@ TOOL_DECLARATIONS = [
|
||||
"required": ["category", "key", "value"]
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "recall_memory",
|
||||
"description": (
|
||||
"Look up a fact you have stored about the user but which is NOT in "
|
||||
"the memory block of your system prompt. "
|
||||
"The prompt lists the keys it did not have room for under "
|
||||
"'[ALSO REMEMBERED]' — if the user asks about anything named there, "
|
||||
"call this FIRST. "
|
||||
"Also call it before saying you do not know something personal, and "
|
||||
"when the user asks what you remember about them (leave query empty "
|
||||
"for everything). "
|
||||
"This is a local file search: it is instant and costs nothing."
|
||||
),
|
||||
"parameters": {
|
||||
"type": "OBJECT",
|
||||
"properties": {
|
||||
"query": {
|
||||
"type": "STRING",
|
||||
"description": (
|
||||
"Keyword to search for — a name, a topic, a category "
|
||||
"(e.g. 'ayse', 'coffee', 'projects'). "
|
||||
"Leave empty to list everything stored."
|
||||
),
|
||||
},
|
||||
},
|
||||
"required": [],
|
||||
},
|
||||
},
|
||||
{
|
||||
"name": "undo",
|
||||
"description": (
|
||||
"Reverse the last change YOU made to this computer — a file you "
|
||||
"moved, renamed, created or wrote, or a setting you changed such as "
|
||||
"volume, brightness, dark mode or WiFi. "
|
||||
"Call this whenever the user says undo, revert, take it back, put it "
|
||||
"back, cancel that, or tells you that you did the wrong thing, in ANY "
|
||||
"language. "
|
||||
"Use action='list' when they ask what can be undone. "
|
||||
"This only covers your own actions — it is not the Ctrl+Z of whatever "
|
||||
"application is on screen (that is computer_settings with action 'undo')."
|
||||
),
|
||||
"parameters": {
|
||||
"type": "OBJECT",
|
||||
"properties": {
|
||||
"action": {
|
||||
"type": "STRING",
|
||||
"description": "undo (default) — reverse the last change | list — show what can be undone",
|
||||
},
|
||||
},
|
||||
"required": [],
|
||||
},
|
||||
},
|
||||
]
|
||||
|
||||
class _ReconnectSignal(Exception):
|
||||
"""Raised inside the session TaskGroup to force a clean, voluntary reconnect
|
||||
(e.g. the user picked a new voice — the voice is fixed at connect time, so
|
||||
the session must be rebuilt).
|
||||
|
||||
Carries `keep_context`: True for an ordinary rebuild, where the stored
|
||||
resumption handle is replayed and the conversation continues; False when the
|
||||
new session must genuinely start clean (see the voice-change note in
|
||||
_on_voice_change)."""
|
||||
|
||||
def __init__(self, keep_context: bool = True):
|
||||
super().__init__()
|
||||
self.keep_context = keep_context
|
||||
|
||||
|
||||
def _is_reconnect_signal(exc: BaseException) -> bool:
|
||||
"""True if `exc` is a _ReconnectSignal, or a(n) (Base)ExceptionGroup that
|
||||
wraps one — TaskGroup bundles child exceptions into a group."""
|
||||
if isinstance(exc, _ReconnectSignal):
|
||||
return True
|
||||
if isinstance(exc, BaseExceptionGroup):
|
||||
return any(_is_reconnect_signal(sub) for sub in exc.exceptions)
|
||||
return False
|
||||
|
||||
|
||||
def _keep_context_of(exc: BaseException) -> bool:
|
||||
"""Read `keep_context` off a reconnect signal, unwrapping the group the
|
||||
TaskGroup put it in. Defaults to True: an unexpected shape must not silently
|
||||
wipe the conversation."""
|
||||
if isinstance(exc, _ReconnectSignal):
|
||||
return getattr(exc, "keep_context", True)
|
||||
if isinstance(exc, BaseExceptionGroup):
|
||||
for sub in exc.exceptions:
|
||||
if _is_reconnect_signal(sub):
|
||||
return _keep_context_of(sub)
|
||||
return True
|
||||
|
||||
|
||||
class JarvisLive:
|
||||
|
||||
def __init__(self, ui: JarvisUI):
|
||||
@@ -581,6 +737,26 @@ class JarvisLive:
|
||||
self.ui.on_text_command = self._on_text_command
|
||||
self.ui.on_remote_clicked = self._make_remote_key
|
||||
self.ui.on_interrupt = self.interrupt
|
||||
self.ui.on_voice_change = self._on_voice_change # voice picker → rebuild session
|
||||
self.ui.on_audio_device_change = self._on_audio_device_change
|
||||
self._reconnect_event: asyncio.Event | None = None
|
||||
self._reconnect_keep = True # False → next rebuild drops the resumption handle
|
||||
|
||||
# ── Session resumption ─────────────────────────────────────────
|
||||
# The server issues a resumption handle every few seconds and reissues
|
||||
# it as the conversation moves on. Before this, session_resumption was
|
||||
# switched ON in the config and the update was never read, so the handle
|
||||
# was thrown away and EVERY reconnect — a dropped packet, a voice change,
|
||||
# switching microphone — started an empty session. "Unlimited sessions"
|
||||
# leaked through exactly this hole.
|
||||
#
|
||||
# Deliberately in RAM only, never written to disk. Persisting it would
|
||||
# make a fresh launch continue yesterday's conversation, which sounds
|
||||
# appealing but breaks the session-summary flow: _save_session_summary
|
||||
# runs at shutdown and the morning briefing pops it the next day. A
|
||||
# conversation that never ends never produces a summary, and the
|
||||
# "yesterday we talked about…" line silently disappears.
|
||||
self._resume_handle: str | None = None
|
||||
self._turn_done_event: asyncio.Event | None = None
|
||||
self._dashboard = None
|
||||
self._briefing_sent = False # morning briefing fires once per process
|
||||
@@ -626,6 +802,53 @@ class JarvisLive:
|
||||
except Exception as e:
|
||||
print(f"[PluginSay] {e}")
|
||||
|
||||
def request_reconnect(self, keep_context: bool = True, reason: str = ""):
|
||||
"""Thread-safe: ask the run loop to tear down and rebuild the Live
|
||||
session. Called from the Qt thread. No-op until the async loop and
|
||||
reconnect event exist.
|
||||
|
||||
`keep_context=False` drops the resumption handle so the new session
|
||||
starts empty — only for changes the server cannot apply to a resumed
|
||||
session."""
|
||||
loop = getattr(self, "_loop", None)
|
||||
ev = self._reconnect_event
|
||||
self._reconnect_keep = keep_context
|
||||
self._reconnect_reason = reason
|
||||
if loop and ev is not None:
|
||||
loop.call_soon_threadsafe(ev.set)
|
||||
|
||||
def _on_voice_change(self):
|
||||
"""Voice picker applied.
|
||||
|
||||
The voice is baked into the session at connect time, so a rebuild is
|
||||
required. It is rebuilt WITHOUT the resumption handle on purpose:
|
||||
resuming restores the server's own session state, and the safe reading
|
||||
is that it restores the voice with it — which would make the picker
|
||||
appear to do nothing. Losing context here is acceptable because changing
|
||||
voice is a deliberate, rare act; losing it on a dropped packet was not."""
|
||||
self.request_reconnect(keep_context=False, reason="new voice")
|
||||
|
||||
def _on_audio_device_change(self):
|
||||
"""Microphone or speaker changed. Both streams are opened inside the
|
||||
session TaskGroup, so they can only be re-opened by rebuilding it —
|
||||
but the conversation is kept, which is the whole reason resumption
|
||||
landed before this feature did."""
|
||||
self.request_reconnect(keep_context=True, reason="audio device")
|
||||
|
||||
async def _watch_reconnect(self):
|
||||
"""Session-scoped task: when a voluntary reconnect is requested, raise a
|
||||
signal that unwinds the TaskGroup so the run loop rebuilds the session."""
|
||||
assert self._reconnect_event is not None
|
||||
await self._reconnect_event.wait()
|
||||
self._reconnect_event.clear()
|
||||
keep = self._reconnect_keep
|
||||
reason = getattr(self, "_reconnect_reason", "") or "settings"
|
||||
self.ui.write_log(
|
||||
f"SYS: Applying {reason} — reconnecting"
|
||||
+ ("..." if keep else " (starting a fresh conversation)...")
|
||||
)
|
||||
raise _ReconnectSignal(keep_context=keep)
|
||||
|
||||
def _make_remote_key(self):
|
||||
"""Called from Qt main thread when user presses Remote Control."""
|
||||
if self._dashboard is None:
|
||||
@@ -720,8 +943,12 @@ class JarvisLive:
|
||||
# Identity injection — overrides any hardcoded name in prompt.txt
|
||||
_addr = (f"ADDRESS: Always call the user '{_user_name}'."
|
||||
if _user_name
|
||||
else "ADDRESS: When speaking Turkish → always say \"efendim\". "
|
||||
"When speaking English → say \"sir\". Never mix languages.")
|
||||
else "ADDRESS: Address the user with the ordinary respectful form "
|
||||
"for a superior in the language you are currently speaking — "
|
||||
"\"sir\" in English, its everyday equivalent in any other "
|
||||
"language. Never an archaic or aristocratic form, and never "
|
||||
"the form from a different language than the one you are "
|
||||
"speaking in this sentence.")
|
||||
identity_ctx = (
|
||||
f"[IDENTITY]\n"
|
||||
f"Your name is {self._asst_name}. "
|
||||
@@ -740,7 +967,12 @@ class JarvisLive:
|
||||
input_audio_transcription={},
|
||||
system_instruction="\n".join(parts),
|
||||
tools=[{"function_declarations": TOOL_DECLARATIONS + self._plugin_registry.get_tool_declarations()}],
|
||||
session_resumption=types.SessionResumptionConfig(),
|
||||
# Hand back the handle captured from the last session_resumption
|
||||
# update. `handle=None` is exactly the old behaviour (ask for
|
||||
# handles, start fresh), so the first connect of a run is unchanged.
|
||||
session_resumption=types.SessionResumptionConfig(
|
||||
handle=self._resume_handle
|
||||
),
|
||||
# Sliding-window compression: session never dies from a full context
|
||||
# window — JARVIS can stay in one conversation for hours
|
||||
context_window_compression=types.ContextWindowCompressionConfig(
|
||||
@@ -749,7 +981,7 @@ class JarvisLive:
|
||||
speech_config=types.SpeechConfig(
|
||||
voice_config=types.VoiceConfig(
|
||||
prebuilt_voice_config=types.PrebuiltVoiceConfig(
|
||||
voice_name="Charon"
|
||||
voice_name=get_voice()
|
||||
)
|
||||
)
|
||||
),
|
||||
@@ -787,7 +1019,23 @@ class JarvisLive:
|
||||
result = "Done."
|
||||
|
||||
try:
|
||||
if name == "open_app":
|
||||
if name == "recall_memory":
|
||||
# Local file search: no network, no second model. Kept out of
|
||||
# the executor deliberately — it is a dictionary scan over a few
|
||||
# hundred short strings, and a thread hop would cost more than
|
||||
# the work itself.
|
||||
result = search_memory(args.get("query", ""), limit=8)
|
||||
|
||||
elif name == "undo":
|
||||
if str(args.get("action", "")).lower().strip() == "list":
|
||||
items = undo_stack.history()
|
||||
result = ("Things I can undo, most recent first:\n"
|
||||
+ "\n".join(f"{i+1}. {t}" for i, t in enumerate(items))
|
||||
) if items else "I have not changed anything I can undo yet."
|
||||
else:
|
||||
result = await loop.run_in_executor(None, undo_stack.undo_last)
|
||||
|
||||
elif name == "open_app":
|
||||
r = await loop.run_in_executor(None, lambda: open_app(parameters=args, response=None, player=self.ui))
|
||||
result = r or f"Opened {args.get('app_name')}."
|
||||
|
||||
@@ -972,15 +1220,49 @@ class JarvisLive:
|
||||
self.out_queue.put_nowait,
|
||||
{"data": data, "mime_type": "audio/pcm"}
|
||||
)
|
||||
# Feed the live mic level to the HUD so the waveform reacts to
|
||||
# the user's actual voice while listening. Purely cosmetic — any
|
||||
# failure here must never disturb the mic.
|
||||
try:
|
||||
self.ui.set_audio_level(_pcm_level(indata))
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
with sd.InputStream(
|
||||
samplerate=SEND_SAMPLE_RATE,
|
||||
channels=CHANNELS,
|
||||
dtype="int16",
|
||||
blocksize=CHUNK_SIZE,
|
||||
callback=callback,
|
||||
):
|
||||
def _open_mic(dev):
|
||||
return sd.InputStream(
|
||||
samplerate=SEND_SAMPLE_RATE,
|
||||
channels=CHANNELS,
|
||||
dtype="int16",
|
||||
blocksize=CHUNK_SIZE,
|
||||
device=dev,
|
||||
callback=callback,
|
||||
)
|
||||
|
||||
# Which microphone. resolve() returns None for "system default" and
|
||||
# for a saved device that is no longer present — so a headset
|
||||
# unplugged since the last run falls back to the built-in mic
|
||||
# instead of raising on startup and taking the session with it.
|
||||
_mic_name = get_input_device()
|
||||
_mic_dev = audio_devices.resolve(_mic_name, "input")
|
||||
if _mic_dev is not None:
|
||||
print(f"[JARVIS] 🎤 Input device: {_mic_name}")
|
||||
try:
|
||||
_mic_stream = _open_mic(_mic_dev)
|
||||
except Exception as _e:
|
||||
# A device the picker listed but the driver will not open right
|
||||
# now — exclusive mode, a webcam already in use, a virtual mic
|
||||
# whose source went away. Chosen hardware failing must never
|
||||
# mean the assistant cannot hear at all.
|
||||
if _mic_dev is None:
|
||||
raise
|
||||
print(f"[JARVIS] ⚠️ Mic '{_mic_name}' failed: {_e} — using default")
|
||||
self.ui.write_log(
|
||||
f"SYS: Microphone '{_mic_name}' unavailable — using system default."
|
||||
)
|
||||
_mic_stream = _open_mic(None)
|
||||
|
||||
with _mic_stream:
|
||||
print("[JARVIS] 🎤 Mic stream open")
|
||||
while True:
|
||||
await asyncio.sleep(0.1)
|
||||
@@ -996,6 +1278,19 @@ class JarvisLive:
|
||||
while True:
|
||||
async for response in self.session.receive():
|
||||
|
||||
# ── Session resumption ───────────────────────────────────
|
||||
# The server sends this periodically. `resumable` goes false
|
||||
# while a turn is mid-flight — replaying a handle from that
|
||||
# moment is what the flag exists to prevent — so only
|
||||
# resumable handles are kept. This is three lines and it is
|
||||
# the entire fix for "every reconnect forgets everything".
|
||||
_sru = getattr(response, "session_resumption_update", None)
|
||||
if _sru is not None:
|
||||
if getattr(_sru, "resumable", False) and getattr(_sru, "new_handle", None):
|
||||
if self._resume_handle is None:
|
||||
print("[JARVIS] 🔗 Session resumption armed")
|
||||
self._resume_handle = _sru.new_handle
|
||||
|
||||
if response.data:
|
||||
if self._interrupted:
|
||||
pass # discard: interrupted
|
||||
@@ -1107,13 +1402,33 @@ class JarvisLive:
|
||||
async def _play_audio(self):
|
||||
print("[JARVIS] 🔊 Play started")
|
||||
|
||||
stream = sd.RawOutputStream(
|
||||
samplerate=RECEIVE_SAMPLE_RATE,
|
||||
channels=CHANNELS,
|
||||
dtype="int16",
|
||||
blocksize=CHUNK_SIZE,
|
||||
)
|
||||
stream.start()
|
||||
_spk_name = get_output_device()
|
||||
_spk_dev = audio_devices.resolve(_spk_name, "output")
|
||||
if _spk_dev is not None:
|
||||
print(f"[JARVIS] 🔊 Output device: {_spk_name}")
|
||||
|
||||
def _open_spk(dev):
|
||||
st = sd.RawOutputStream(
|
||||
samplerate=RECEIVE_SAMPLE_RATE,
|
||||
channels=CHANNELS,
|
||||
dtype="int16",
|
||||
blocksize=CHUNK_SIZE,
|
||||
device=dev,
|
||||
)
|
||||
st.start()
|
||||
return st
|
||||
|
||||
try:
|
||||
stream = _open_spk(_spk_dev)
|
||||
except Exception as _e:
|
||||
# A chosen output that the host API accepts by name but refuses to
|
||||
# open (exclusive mode, wrong sample rate, device asleep) must not
|
||||
# cost the user their voice. Fall back to the default and say so.
|
||||
if _spk_dev is None:
|
||||
raise
|
||||
print(f"[JARVIS] ⚠️ Output device '{_spk_name}' failed: {_e} — using default")
|
||||
self.ui.write_log(f"SYS: Speaker '{_spk_name}' unavailable — using system default.")
|
||||
stream = _open_spk(None)
|
||||
|
||||
try:
|
||||
while True:
|
||||
@@ -1144,6 +1459,13 @@ class JarvisLive:
|
||||
except asyncio.QueueEmpty:
|
||||
break
|
||||
|
||||
# Drive the HUD waveform from JARVIS's own voice while speaking.
|
||||
try:
|
||||
self.ui.set_audio_level(_pcm_level(
|
||||
np.frombuffer(bytes(batch), dtype=np.int16)))
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
await asyncio.to_thread(stream.write, bytes(batch))
|
||||
except (RuntimeError, asyncio.CancelledError):
|
||||
@@ -1187,7 +1509,12 @@ class JarvisLive:
|
||||
return
|
||||
|
||||
# ── Phase 1: instant greeting ─────────────────────────────────────────
|
||||
lang_clause = f" Respond in {lang}." if lang else ""
|
||||
# The briefing fires before the user has said anything, so the
|
||||
# remembered language is the only signal there is. It is a starting
|
||||
# point, not a setting: the moment they reply, their language wins.
|
||||
lang_clause = (f" Speak this greeting in {lang}, then follow the "
|
||||
f"user's own language from their first reply onward."
|
||||
if lang else "")
|
||||
name_clause = f" Address the user as {name}." if name else ""
|
||||
|
||||
# Inject last session context if available — pop removes it so it's never repeated
|
||||
@@ -1221,7 +1548,9 @@ class JarvisLive:
|
||||
# ── Phase 2: fire as soon as Phase 1 audio is done ───────────────────
|
||||
async def _deliver_news():
|
||||
try:
|
||||
lang_str = f" Respond in {lang}." if lang else ""
|
||||
lang_str = (f" Speak in {lang} unless the user has since "
|
||||
f"spoken another language, in which case use theirs."
|
||||
if lang else "")
|
||||
|
||||
# Wait for news fetch (already running) and Phase 1 turn-complete
|
||||
# in parallel — whichever takes longer determines the wait time
|
||||
@@ -1472,6 +1801,27 @@ class JarvisLive:
|
||||
|
||||
async def run(self):
|
||||
self._loop = asyncio.get_event_loop()
|
||||
self._reconnect_event = asyncio.Event()
|
||||
|
||||
# ── Wire the shared core services to the interface ───────────────────
|
||||
# The confirmation gate is useless without a way to ask, and a memory
|
||||
# trim is invisible without a way to say so. Both are bound once here
|
||||
# rather than passed down through every action signature.
|
||||
confirm_gate.bind(
|
||||
show = self.ui.show_confirm,
|
||||
hide = self.ui.hide_confirm,
|
||||
log = self.ui.write_log,
|
||||
)
|
||||
set_trim_notifier(self.ui.write_log)
|
||||
|
||||
# Tell the device picker the exact rates the streams open at, from the
|
||||
# constants that actually open them — so it can never list a device that
|
||||
# cannot be opened at them.
|
||||
audio_devices.configure(SEND_SAMPLE_RATE, RECEIVE_SAMPLE_RATE)
|
||||
|
||||
# Enumerate audio devices off-thread. The settings drawer must never pay
|
||||
# for host-API enumeration on the Qt thread.
|
||||
audio_devices.prefetch()
|
||||
|
||||
# Start dashboard (optional — needs: pip install fastapi "uvicorn[standard]" cryptography)
|
||||
try:
|
||||
@@ -1489,6 +1839,7 @@ class JarvisLive:
|
||||
try:
|
||||
print("[JARVIS] Connecting...")
|
||||
self.ui.set_state("THINKING")
|
||||
_resumed_with = self._resume_handle is not None
|
||||
config = self._build_config()
|
||||
|
||||
# Fresh client on every reconnect — avoids stale HTTP session state
|
||||
@@ -1517,12 +1868,19 @@ class JarvisLive:
|
||||
self._interrupted = False
|
||||
|
||||
print("[JARVIS] Connected.")
|
||||
if _resumed_with:
|
||||
# Say it plainly: the difference between "it reconnected"
|
||||
# and "it reconnected and still knows what we were doing"
|
||||
# is the whole point, and it is invisible otherwise.
|
||||
self.ui.write_log("SYS: Reconnected — conversation restored.")
|
||||
self.ui.set_state("LISTENING")
|
||||
self.ui.write_log("SYS: JARVIS online.")
|
||||
|
||||
if self._dashboard:
|
||||
await self._dashboard.broadcast({"type": "status", "state": "active"})
|
||||
|
||||
self._reconnect_event.clear() # ignore requests from before this session
|
||||
tg.create_task(self._watch_reconnect())
|
||||
tg.create_task(self._send_realtime())
|
||||
tg.create_task(self._listen_audio())
|
||||
tg.create_task(self._receive_audio())
|
||||
@@ -1548,6 +1906,35 @@ class JarvisLive:
|
||||
# externally, which `except Exception` would miss, letting the
|
||||
# exception escape the while-loop and causing asyncio.run() to
|
||||
# start shutdown — resulting in "executor after shutdown" errors).
|
||||
# Voluntary reconnect (voice change) — not an error. Rebuild the
|
||||
# session immediately with no backoff and no scary logs.
|
||||
if _is_reconnect_signal(e):
|
||||
print("[JARVIS] Voluntary reconnect requested.")
|
||||
if not _keep_context_of(e):
|
||||
# A deliberate clean slate (voice change) — drop the
|
||||
# handle so the next connect really does start empty.
|
||||
self._resume_handle = None
|
||||
self._conn_backoff = 0
|
||||
continue
|
||||
|
||||
# A resumption handle the server will not accept — expired, or
|
||||
# belonging to a session it has since dropped. Without this, the
|
||||
# same dead handle would be replayed on every retry and the
|
||||
# assistant would never come back at all: the feature meant to
|
||||
# survive a reconnect would be the thing preventing one. Drop it
|
||||
# once and let the next attempt start clean.
|
||||
if _resumed_with and (
|
||||
"resum" in str(e).lower()
|
||||
or "handle" in str(e).lower()
|
||||
or "INVALID_ARGUMENT" in str(e)
|
||||
or "NOT_FOUND" in str(e)
|
||||
):
|
||||
print("[JARVIS] 🔗 Resumption handle rejected — starting a fresh session")
|
||||
self.ui.write_log("SYS: Could not restore the conversation — starting fresh.")
|
||||
self._resume_handle = None
|
||||
self._conn_backoff = 0
|
||||
continue
|
||||
|
||||
err_str = str(e)
|
||||
print(f"[JARVIS] Error ({type(e).__name__}): {e}")
|
||||
traceback.print_exc()
|
||||
|
||||
@@ -75,6 +75,35 @@ def save_assistant_config(assistant_name: str, user_name: str) -> None:
|
||||
CONFIG_FILE.write_text(json.dumps(data, indent=4), encoding="utf-8")
|
||||
|
||||
|
||||
# ── Assistant voice ──────────────────────────────────────────────────────────
|
||||
# Gemini Live prebuilt voices. Names are proper nouns — identical in every
|
||||
# language, so this list is safe to show verbatim in any locale.
|
||||
AVAILABLE_VOICES = ["Charon", "Puck", "Kore", "Fenrir", "Aoede"]
|
||||
DEFAULT_VOICE = "Charon"
|
||||
|
||||
|
||||
def get_voice() -> str:
|
||||
"""Return the configured Live voice, falling back to the default if unset
|
||||
or if the stored value is not a voice we recognise."""
|
||||
v = load_api_keys().get("voice_name", DEFAULT_VOICE) or DEFAULT_VOICE
|
||||
return v if v in AVAILABLE_VOICES else DEFAULT_VOICE
|
||||
|
||||
|
||||
def save_voice(voice_name: str) -> None:
|
||||
"""Persist the chosen Live voice. Unknown names collapse to the default so a
|
||||
bad value can never reach the API and break the session."""
|
||||
ensure_config_dir()
|
||||
data: dict = {}
|
||||
if CONFIG_FILE.exists():
|
||||
try:
|
||||
data = json.loads(CONFIG_FILE.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
data = {}
|
||||
v = (voice_name or "").strip()
|
||||
data["voice_name"] = v if v in AVAILABLE_VOICES else DEFAULT_VOICE
|
||||
CONFIG_FILE.write_text(json.dumps(data, indent=4), encoding="utf-8")
|
||||
|
||||
|
||||
def get_brief_enabled() -> bool:
|
||||
return load_api_keys().get("morning_brief_enabled", True)
|
||||
|
||||
@@ -91,6 +120,48 @@ def save_brief_enabled(enabled: bool) -> None:
|
||||
CONFIG_FILE.write_text(json.dumps(data, indent=4), encoding="utf-8")
|
||||
|
||||
|
||||
# ── Audio devices ────────────────────────────────────────────────────────────
|
||||
# Stored as device NAMES, not sounddevice indices. Indices shift every time a
|
||||
# USB device is plugged in or removed, so a saved index silently starts pointing
|
||||
# at a different microphone. The empty string means "system default", which is
|
||||
# both the factory setting and what an unresolvable saved device falls back to —
|
||||
# so unplugging a headset degrades to the built-in speakers instead of crashing.
|
||||
|
||||
def _patch_config(**fields) -> None:
|
||||
"""Read-modify-write one or more keys in api_keys.json.
|
||||
|
||||
Every setter in this file open-coded this. Collapsing it here means a new
|
||||
setting is one line, and there is one place where a corrupt config file is
|
||||
handled instead of nine."""
|
||||
ensure_config_dir()
|
||||
data: dict = {}
|
||||
if CONFIG_FILE.exists():
|
||||
try:
|
||||
data = json.loads(CONFIG_FILE.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
data = {}
|
||||
data.update(fields)
|
||||
CONFIG_FILE.write_text(json.dumps(data, indent=4), encoding="utf-8")
|
||||
|
||||
|
||||
def get_input_device() -> str:
|
||||
"""Microphone device name, or '' for the system default."""
|
||||
return (load_api_keys().get("input_device", "") or "").strip()
|
||||
|
||||
|
||||
def save_input_device(name: str) -> None:
|
||||
_patch_config(input_device=(name or "").strip())
|
||||
|
||||
|
||||
def get_output_device() -> str:
|
||||
"""Speaker device name, or '' for the system default."""
|
||||
return (load_api_keys().get("output_device", "") or "").strip()
|
||||
|
||||
|
||||
def save_output_device(name: str) -> None:
|
||||
_patch_config(output_device=(name or "").strip())
|
||||
|
||||
|
||||
def get_plugin_enabled(plugin_name: str) -> bool:
|
||||
"""Plugins are enabled by default the moment they're discovered (opt-out model)."""
|
||||
return load_api_keys().get("plugins_enabled", {}).get(plugin_name, True)
|
||||
|
||||
+268
-60
@@ -1,4 +1,5 @@
|
||||
import json
|
||||
import re
|
||||
from datetime import datetime
|
||||
from threading import Lock
|
||||
from pathlib import Path
|
||||
@@ -15,7 +16,33 @@ BASE_DIR = get_base_dir()
|
||||
MEMORY_PATH = BASE_DIR / "memory" / "long_term.json"
|
||||
_lock = Lock()
|
||||
MAX_VALUE_LENGTH = 380
|
||||
MEMORY_MAX_CHARS = 2200
|
||||
|
||||
# ── Why there are two very different numbers here ────────────────────────────
|
||||
#
|
||||
# There used to be one: MEMORY_MAX_CHARS = 2200, applied to the whole store. It
|
||||
# was a *storage* limit, and it existed only because the entire memory was
|
||||
# pasted into the system prompt on every connect — so growing the memory grew
|
||||
# every single request. When it filled, _trim_to_limit() deleted the oldest
|
||||
# entries and printed one line to a console nobody reads. A memory described as
|
||||
# "deeply remembers projects, preferences and personal context" was in practice
|
||||
# two pages long, and quietly forgot your sister's name after a few weeks.
|
||||
#
|
||||
# Storage and prompt budget are now separate concerns:
|
||||
#
|
||||
# MEMORY_MAX_CHARS — a runaway guard, not a feature limit. Nothing normal
|
||||
# reaches it; a bug writing in a loop does.
|
||||
# PROMPT_CORE_CHARS — what actually rides in the system prompt every session.
|
||||
# Smaller than the old whole-memory dump, so sessions
|
||||
# start *faster* than before, not slower.
|
||||
#
|
||||
# Everything above the core stays on disk and is fetched on demand by the
|
||||
# recall_memory tool — see search_memory() and format_memory_for_prompt().
|
||||
MEMORY_MAX_CHARS = 200_000
|
||||
PROMPT_CORE_CHARS = 900
|
||||
PROMPT_INDEX_CHARS = 420
|
||||
# Most entries any one category may contribute to the core block, so a person
|
||||
# with forty stored preferences still gets their sister into the prompt.
|
||||
PROMPT_MAX_PER_CATEGORY = 6
|
||||
|
||||
def _empty_memory() -> dict:
|
||||
return {
|
||||
@@ -55,16 +82,37 @@ def _all_entries(memory: dict) -> list[tuple]:
|
||||
return entries
|
||||
|
||||
|
||||
# Set by main.py so a trim can reach the activity log. Deleting something a
|
||||
# person told you and mentioning it only on stdout is how a memory loses trust.
|
||||
_trim_notifier = None
|
||||
|
||||
|
||||
def set_trim_notifier(fn) -> None:
|
||||
"""Register a callable(str) that surfaces trims to the user."""
|
||||
global _trim_notifier
|
||||
_trim_notifier = fn
|
||||
|
||||
|
||||
def _trim_to_limit(memory: dict) -> dict:
|
||||
if len(json.dumps(memory, ensure_ascii=False)) <= MEMORY_MAX_CHARS:
|
||||
return memory
|
||||
entries = _all_entries(memory)
|
||||
entries.sort(key=lambda t: t[2].get("updated", "0000-00-00"))
|
||||
dropped = []
|
||||
for cat, key, _ in entries:
|
||||
if len(json.dumps(memory, ensure_ascii=False)) <= MEMORY_MAX_CHARS:
|
||||
break
|
||||
del memory[cat][key]
|
||||
dropped.append(f"{cat}/{key}")
|
||||
print(f"[Memory] 🗑️ Trimmed {cat}/{key}")
|
||||
if dropped and _trim_notifier:
|
||||
try:
|
||||
_trim_notifier(
|
||||
f"SYS: Memory full — forgot {len(dropped)} oldest entries "
|
||||
f"({', '.join(dropped[:3])}{'…' if len(dropped) > 3 else ''})"
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
return memory
|
||||
|
||||
def save_memory(memory: dict) -> None:
|
||||
@@ -117,81 +165,241 @@ def update_memory(memory_update: dict) -> dict:
|
||||
print(f"[Memory] 💾 Saved: {list(memory_update.keys())}")
|
||||
return memory
|
||||
|
||||
def _entry_value(entry) -> str:
|
||||
"""Accept both the {'value': ..., 'updated': ...} shape and a bare string,
|
||||
because early versions of the store wrote plain strings."""
|
||||
if isinstance(entry, dict):
|
||||
return str(entry.get("value", "") or "").strip()
|
||||
return str(entry or "").strip()
|
||||
|
||||
|
||||
def _pretty(key: str) -> str:
|
||||
return key.replace("_", " ").strip()
|
||||
|
||||
|
||||
# Identity is always in the prompt; these categories compete for the remaining
|
||||
# budget by recency.
|
||||
_CATEGORY_LABELS = {
|
||||
"preferences": "Preferences",
|
||||
"projects": "Active projects / goals",
|
||||
"relationships": "People in their life",
|
||||
"wishes": "Wishes / plans",
|
||||
"notes": "Notes",
|
||||
}
|
||||
|
||||
_IDENTITY_FIELDS = ["name", "age", "birthday", "city", "job",
|
||||
"language", "school", "nationality"]
|
||||
|
||||
|
||||
def format_memory_for_prompt(memory: dict | None) -> str:
|
||||
"""Build the memory block that goes into the system prompt.
|
||||
|
||||
This used to dump everything. It now sends three things:
|
||||
|
||||
1. IDENTITY - always, in full. It is small, and it is wrong for the
|
||||
assistant to have to look up your name.
|
||||
2. RECENT - the most recently updated entries from every other
|
||||
category, up to PROMPT_CORE_CHARS. Recency is the cheapest useful
|
||||
relevance signal available without embeddings.
|
||||
3. AN INDEX - the *keys* of everything else, values omitted.
|
||||
|
||||
Point 3 is what makes recall work at all. A model cannot decide to look
|
||||
something up if it does not know the thing exists: with only points 1 and 2,
|
||||
"who is Ayse?" would get "I don't know" while ayse_sister sat on disk
|
||||
unread. The index costs a few hundred characters and turns recall from a
|
||||
gamble into a lookup.
|
||||
|
||||
Net effect on latency: this block is SMALLER than the old full dump, so
|
||||
every session connects with fewer tokens. Occasionally the model spends one
|
||||
extra round trip on recall_memory - covered by the acknowledgment it
|
||||
already speaks before any slow step."""
|
||||
if not memory:
|
||||
return ""
|
||||
|
||||
lines = []
|
||||
core_lines: list[str] = []
|
||||
|
||||
identity = memory.get("identity", {})
|
||||
id_fields = ["name", "age", "birthday", "city", "job", "language", "school", "nationality"]
|
||||
for field in id_fields:
|
||||
entry = identity.get(field)
|
||||
if entry:
|
||||
val = entry.get("value") if isinstance(entry, dict) else entry
|
||||
if val:
|
||||
lines.append(f"{field.title()}: {val}")
|
||||
for key, entry in identity.items():
|
||||
if key in id_fields:
|
||||
# 1. Identity - always, in full
|
||||
identity = memory.get("identity", {}) or {}
|
||||
for field in _IDENTITY_FIELDS:
|
||||
val = _entry_value(identity.get(field))
|
||||
if not val:
|
||||
continue
|
||||
val = entry.get("value") if isinstance(entry, dict) else entry
|
||||
if field == "language":
|
||||
# Labelled as an observation, not a setting. A bare "Language:
|
||||
# English" line written months ago reads like a standing order and
|
||||
# was one of the reasons a Turkish question came back in English.
|
||||
core_lines.append(
|
||||
f"Has spoken to you in: {val} (an observation about the past — "
|
||||
f"always answer in the language of their CURRENT message)")
|
||||
else:
|
||||
core_lines.append(f"{field.title()}: {val}")
|
||||
for key, entry in identity.items():
|
||||
if key in _IDENTITY_FIELDS:
|
||||
continue
|
||||
val = _entry_value(entry)
|
||||
if val:
|
||||
lines.append(f"{key.replace('_', ' ').title()}: {val}")
|
||||
core_lines.append(f"{_pretty(key).title()}: {val}")
|
||||
|
||||
prefs = memory.get("preferences", {})
|
||||
if prefs:
|
||||
lines.append("")
|
||||
lines.append("Preferences:")
|
||||
for key, entry in list(prefs.items())[:15]:
|
||||
val = entry.get("value") if isinstance(entry, dict) else entry
|
||||
if val:
|
||||
lines.append(f" - {key.replace('_', ' ').title()}: {val}")
|
||||
# 2. Everything else, most recently updated first
|
||||
rest: list[tuple[str, str, str, str]] = [] # (updated, cat, key, value)
|
||||
for cat in _CATEGORY_LABELS:
|
||||
for key, entry in (memory.get(cat, {}) or {}).items():
|
||||
val = _entry_value(entry)
|
||||
if not val:
|
||||
continue
|
||||
updated = (entry.get("updated", "") if isinstance(entry, dict) else "") or "0000-00-00"
|
||||
rest.append((updated, cat, key, val))
|
||||
rest.sort(key=lambda t: t[0], reverse=True)
|
||||
|
||||
projects = memory.get("projects", {})
|
||||
if projects:
|
||||
lines.append("")
|
||||
lines.append("Active Projects / Goals:")
|
||||
for key, entry in list(projects.items())[:8]:
|
||||
val = entry.get("value") if isinstance(entry, dict) else entry
|
||||
if val:
|
||||
lines.append(f" - {key.replace('_', ' ').title()}: {val}")
|
||||
used = sum(len(l) + 1 for l in core_lines)
|
||||
shown: dict[str, list[str]] = {}
|
||||
overflow: dict[str, list[str]] = {}
|
||||
|
||||
rels = memory.get("relationships", {})
|
||||
if rels:
|
||||
lines.append("")
|
||||
lines.append("People in their life:")
|
||||
for key, entry in list(rels.items())[:10]:
|
||||
val = entry.get("value") if isinstance(entry, dict) else entry
|
||||
if val:
|
||||
lines.append(f" - {key.replace('_', ' ').title()}: {val}")
|
||||
# Recency decides order, but no single category may take the whole budget.
|
||||
# Without the cap, someone with forty stored preferences gets a prompt that
|
||||
# is forty preferences and not one person's name — the categories that
|
||||
# matter most in conversation are also the ones that change least often, so
|
||||
# pure recency systematically buries them.
|
||||
per_cat_used: dict[str, int] = {}
|
||||
for _updated, cat, key, val in rest:
|
||||
line = f" - {_pretty(key).title()}: {val}"
|
||||
if (per_cat_used.get(cat, 0) < PROMPT_MAX_PER_CATEGORY
|
||||
and used + len(line) + 1 <= PROMPT_CORE_CHARS):
|
||||
shown.setdefault(cat, []).append(line)
|
||||
per_cat_used[cat] = per_cat_used.get(cat, 0) + 1
|
||||
used += len(line) + 1
|
||||
else:
|
||||
overflow.setdefault(cat, []).append(_pretty(key))
|
||||
|
||||
wishes = memory.get("wishes", {})
|
||||
if wishes:
|
||||
lines.append("")
|
||||
lines.append("Wishes / Plans / Wants:")
|
||||
for key, entry in list(wishes.items())[:8]:
|
||||
val = entry.get("value") if isinstance(entry, dict) else entry
|
||||
if val:
|
||||
lines.append(f" - {key.replace('_', ' ').title()}: {val}")
|
||||
# The index is a table of contents, so it is interleaved across categories
|
||||
# rather than continuing in recency order. Sorted by recency it would list
|
||||
# twenty-four preferences before the first relationship, and the one entry
|
||||
# the index exists for — the old fact the model has no other way to know
|
||||
# about — would fall off the end.
|
||||
indexed: list[str] = []
|
||||
if overflow:
|
||||
cats = [c for c in _CATEGORY_LABELS if overflow.get(c)]
|
||||
cursor = {c: 0 for c in cats}
|
||||
while cats:
|
||||
for cat in list(cats):
|
||||
i = cursor[cat]
|
||||
if i >= len(overflow[cat]):
|
||||
cats.remove(cat)
|
||||
continue
|
||||
indexed.append(overflow[cat][i])
|
||||
cursor[cat] = i + 1
|
||||
|
||||
notes = memory.get("notes", {})
|
||||
if notes:
|
||||
lines.append("")
|
||||
lines.append("Other notes:")
|
||||
for key, entry in list(notes.items())[:8]:
|
||||
val = entry.get("value") if isinstance(entry, dict) else entry
|
||||
if val:
|
||||
lines.append(f" - {key}: {val}")
|
||||
for cat, label in _CATEGORY_LABELS.items():
|
||||
if shown.get(cat):
|
||||
core_lines.append("")
|
||||
core_lines.append(f"{label}:")
|
||||
core_lines.extend(shown[cat])
|
||||
|
||||
if not lines:
|
||||
if not core_lines and not indexed:
|
||||
return ""
|
||||
|
||||
header = "[WHAT YOU KNOW ABOUT THIS PERSON — use naturally, never recite like a list]\n"
|
||||
result = header + "\n".join(lines)
|
||||
if len(result) > 2000:
|
||||
result = result[:1997] + "…"
|
||||
out = [
|
||||
"[WHAT YOU KNOW ABOUT THIS PERSON — use naturally, never recite like a list]",
|
||||
*core_lines,
|
||||
]
|
||||
|
||||
return result + "\n"
|
||||
# 3. The index of what is on disk but not in this prompt
|
||||
if indexed:
|
||||
budget, names = PROMPT_INDEX_CHARS, []
|
||||
for n in indexed:
|
||||
if budget - len(n) - 2 < 0:
|
||||
break
|
||||
names.append(n)
|
||||
budget -= len(n) + 2
|
||||
if names:
|
||||
out.append("")
|
||||
out.append(
|
||||
"[ALSO REMEMBERED — values not shown here. Call recall_memory "
|
||||
"with a keyword to read any of these before saying you do not know]"
|
||||
)
|
||||
out.append(", ".join(names)
|
||||
+ (f" (+{len(indexed) - len(names)} more)"
|
||||
if len(indexed) > len(names) else ""))
|
||||
|
||||
return "\n".join(out) + "\n"
|
||||
|
||||
|
||||
# ── Recall ────────────────────────────────────────────────────────────────────
|
||||
|
||||
def _score(query_words: list[str], cat: str, key: str, value: str) -> int:
|
||||
"""Cheap lexical relevance. No embeddings, no network, no model call - this
|
||||
runs in well under a millisecond, which is the entire point: recall must
|
||||
cost one model round trip, never two."""
|
||||
hay_key = _pretty(key).lower()
|
||||
hay_val = value.lower()
|
||||
score = 0
|
||||
for w in query_words:
|
||||
if not w:
|
||||
continue
|
||||
if w == hay_key:
|
||||
score += 10
|
||||
elif w in hay_key:
|
||||
score += 6
|
||||
if w in hay_val:
|
||||
score += 3
|
||||
if w in cat:
|
||||
score += 1
|
||||
return score
|
||||
|
||||
|
||||
def search_memory(query: str, limit: int = 8) -> str:
|
||||
"""Find stored facts matching `query`. Backs the recall_memory tool.
|
||||
|
||||
An empty query is treated as "show me everything you know", capped - the
|
||||
model asks that when the user says "what do you remember about me?"."""
|
||||
memory = load_memory()
|
||||
words = [w for w in re.split(r"[^\w]+", (query or "").lower()) if len(w) > 1]
|
||||
|
||||
rows: list[tuple[int, str, str, str]] = []
|
||||
for cat, items in memory.items():
|
||||
if not isinstance(items, dict):
|
||||
continue # skip 'sessions', which is a list
|
||||
for key, entry in items.items():
|
||||
val = _entry_value(entry)
|
||||
if not val:
|
||||
continue
|
||||
s = _score(words, cat, key, val) if words else 1
|
||||
if s > 0:
|
||||
rows.append((s, cat, key, val))
|
||||
|
||||
if not rows:
|
||||
return (f"Nothing stored about '{query}'." if query
|
||||
else "I have not stored anything about this person yet.")
|
||||
|
||||
rows.sort(key=lambda r: (-r[0], r[2]))
|
||||
lines = [f"{cat}/{_pretty(key)}: {val}" for _s, cat, key, val in rows[:max(1, limit)]]
|
||||
head = (f"Stored facts matching '{query}':" if query
|
||||
else "Everything currently stored:")
|
||||
more = (f"\n(+{len(rows) - len(lines)} more — search with a narrower keyword)"
|
||||
if len(rows) > len(lines) else "")
|
||||
return head + "\n" + "\n".join(lines) + more
|
||||
|
||||
|
||||
def all_entries_for_ui() -> list[dict]:
|
||||
"""Flat list for the memory panel: what JARVIS knows, and when it learned it.
|
||||
Sorted newest first so the panel opens on what changed most recently."""
|
||||
memory = load_memory()
|
||||
rows = []
|
||||
for cat, items in memory.items():
|
||||
if not isinstance(items, dict):
|
||||
continue
|
||||
for key, entry in items.items():
|
||||
val = _entry_value(entry)
|
||||
if not val:
|
||||
continue
|
||||
rows.append({
|
||||
"category": cat,
|
||||
"key": key,
|
||||
"value": val,
|
||||
"updated": (entry.get("updated", "") if isinstance(entry, dict) else ""),
|
||||
})
|
||||
rows.sort(key=lambda r: (r["updated"] or "0000-00-00"), reverse=True)
|
||||
return rows
|
||||
|
||||
def remember(key: str, value: str, category: str = "notes") -> str:
|
||||
valid = {"identity", "preferences", "projects", "relationships", "wishes", "notes"}
|
||||
|
||||
@@ -1,177 +1,284 @@
|
||||
# ⚙️ MARK LI (51)
|
||||
### The Ultimate Cross-Platform Personal AI Assistant — By FatihMakes
|
||||
|
||||
> 📺 **[Watch the full setup video on YouTube](https://www.youtube.com/@FatihMakes)**
|
||||
|
||||
A real-time voice AI that can hear, see, understand, and control your computer — on any OS. Supports Windows, macOS, and Linux. Built on the Gemini Live API for native audio streaming, delivering zero subscriptions and total digital autonomy.
|
||||
|
||||
---
|
||||
|
||||
## ✨ Overview
|
||||
|
||||
MARK LI is the final form of the core: an assistant you extend without ever touching its engine. Drop a single plugin file into the `plugins/` folder and JARVIS learns a new skill on the next launch — no code changes, no configuration, no risk. On top of that, the voice itself got smarter: JARVIS now hears the emotion in your voice, knows when you're talking to someone else in the room and stays silent, and can hold one conversation for hours without losing the thread.
|
||||
|
||||
It's not just an assistant — it's an extension of your digital life.
|
||||
|
||||
---
|
||||
|
||||
## 🚀 Capabilities
|
||||
|
||||
### Core Features
|
||||
| Feature | Description |
|
||||
|---|---|
|
||||
| 🧩 Plugin System | Drop a single `.py` file into `plugins/` — JARVIS learns a new skill on next launch |
|
||||
| 🎙️ Real-time Voice | Ultra-low latency conversation in any language via Gemini Live API |
|
||||
| 💓 Affective Dialog | Hears the emotion in your voice and adapts its tone in response |
|
||||
| 🤫 Proactive Audio | Knows when you're not talking to it — background chatter never triggers a reply |
|
||||
| ♾️ Unlimited Sessions | Sliding-window context compression — one conversation can last for hours |
|
||||
| 🖥️ System Control | Launch apps, adjust volume/brightness, WiFi, shortcuts, power — all by voice |
|
||||
| 🧩 Autonomous Tasks | High-level planning for complex multi-step goals via agent mode |
|
||||
| 👁️ Visual Awareness | Real-time screen capture and webcam vision piped into your main Gemini session |
|
||||
| 🧠 Persistent Memory | Deeply remembers projects, preferences, and personal context across sessions |
|
||||
| ⌨️ Hybrid Input | Seamlessly switch between keyboard typing and voice commands |
|
||||
| 🌅 Morning Briefing | On first boot: greets you, reads the time, recaps yesterday, and fetches live news |
|
||||
| 🔔 Proactive 2.0 | Time-aware, context-aware check-ins — knows the time of day, your projects, and what you've been discussing |
|
||||
| 🗓️ Session Memory | Summarises each conversation and mentions it naturally next morning — consumed after use, never repeats |
|
||||
| 👁️🗨️ Background Monitoring | User-configured topic watching — checks for new headlines once a day and alerts naturally |
|
||||
| 📊 Hardware Monitoring | Continuous CPU, RAM, GPU and temperature telemetry with localized voice alerts |
|
||||
| 🌤️ Weather Report | Live weather data for your city, personalized from memory |
|
||||
| 🗺️ Dynamic Content Panel | Scrollable display layer beneath the HUD that renders web results, news, and search data |
|
||||
| 🔍 Multi-Mode Web Search | `news` / `research` / `price` / `compare` / `search` — Gemini Grounded first, DDG fallback |
|
||||
| ⏰ Smart Reminders | OS-native scheduled notifications (Windows Task Scheduler / macOS LaunchAgent / Linux systemd) |
|
||||
| ✈️ Flight Finder | Live flight price and availability lookup |
|
||||
| 🎮 Game Updater | Checks and triggers game updates on Steam and Epic Games on demand |
|
||||
| 📂 File Processor | Read, summarize, and answer questions about local files |
|
||||
| 💻 Code Helper | Inline code review, debugging, and generation |
|
||||
| 🌐 Browser Control | Open URLs, navigate tabs, and interact with the browser by voice |
|
||||
| 📨 Send Message | Compose and send messages through WhatsApp, Telegram, and more |
|
||||
| 🎬 YouTube Control | Search, play, and control YouTube playback by voice |
|
||||
| 🖱️ Desktop Control | Taskbar, window management, and desktop-level operations |
|
||||
| 🧑💻 Silent Language Memory | Detects spoken language on first use — all future sessions adapt automatically |
|
||||
| 📱 Remote Dashboard | Control the assistant from your phone via QR code pairing |
|
||||
| ⚡ Auto-Start on Boot | Registers with the OS startup system (registry / LaunchAgent / .desktop) |
|
||||
| 📋 Clipboard Intelligence | Copy any text → floating panel with Translate / Summarise / Explain / Fix |
|
||||
| 🎨 Assistant Customization | Change the assistant name and your name from the UI — takes effect immediately |
|
||||
|
||||
---
|
||||
|
||||
## 🆕 What's New in Mark LI
|
||||
|
||||
### 🧩 Plugin System — Extend JARVIS Without Touching a Single Core File
|
||||
The headline feature of Mark LI, and the reason it's the final architecture version. Every new capability from now on ships as a single `.py` file:
|
||||
|
||||
1. Download a plugin file (e.g. `calorie_counter.py`)
|
||||
2. Drop it into the `plugins/` folder
|
||||
3. Restart JARVIS — done. The skill is live, by voice, in any language.
|
||||
|
||||
Each plugin declares its own Gemini tool schema and logic in one file. The engine auto-discovers it at startup, registers it with the Live session, and lists it in the new **🧩 Plugin Manager** panel where every plugin gets its own persistent ON/OFF toggle.
|
||||
|
||||
Safety is built in at three layers: a broken or badly written plugin can **never** crash JARVIS — it simply shows up as "BROKEN" in the manager with the error explained, while every other tool and plugin keeps working. Name collisions with core tools are detected and rejected automatically. Want to write your own? Copy `plugins/_template.py` and fill in two things: the `PLUGIN` dict and the `run()` function.
|
||||
|
||||
### 💓 Affective Dialog — JARVIS Hears How You Feel
|
||||
Powered by Gemini Live's native audio understanding, JARVIS now picks up the emotion in your voice — excitement, frustration, fatigue — and adapts its own tone in response. Late-night tired questions get calm answers; excited announcements get energy back.
|
||||
|
||||
### 🤫 Proactive Audio — Knows When You're Not Talking to It
|
||||
The biggest quality-of-life upgrade for an always-listening assistant: JARVIS can now tell when speech isn't addressed to it. Talking to someone in the room, taking a phone call, TV in the background — it stays silent instead of interjecting. No wake word needed, no accidental replies.
|
||||
|
||||
### ♾️ Unlimited Session Length — The Conversation Never Dies
|
||||
Sliding-window context compression means the Live session no longer terminates when the context window fills up. Combined with session resumption, JARVIS holds one continuous conversation for hours without losing the thread.
|
||||
|
||||
All three Live API upgrades degrade gracefully: if the preview API ever rejects them, JARVIS automatically reconnects with the standard configuration — users never see a crash.
|
||||
|
||||
---
|
||||
|
||||
## 🗺️ Mark Roadmap
|
||||
|
||||
| Mark | Focus |
|
||||
|---|---|
|
||||
| **XLVIII** | Instant interrupt · parallel news · two-phase briefing · exponential backoff · vision cooldown |
|
||||
| **XLIX** | Auto-start · clipboard intelligence · assistant customization |
|
||||
| **L** | Session memory · background monitoring · proactive 2.0 · instant vision · parallel news search |
|
||||
| **LI** | Plugin system · affective dialog · proactive audio · unlimited sessions |
|
||||
| **LII+** | Plugin files: email · quiz mode · calorie counter · calendar · and more |
|
||||
|
||||
---
|
||||
|
||||
## ⚡ Quick Start
|
||||
|
||||
```bash
|
||||
git clone https://github.com/FatihMakes/Mark-LI.git
|
||||
cd Mark-LI
|
||||
pip install -r requirements.txt
|
||||
python main.py
|
||||
```
|
||||
|
||||
> ⚠️ **Installation Note:** Some OS-specific dependencies are not bundled in `requirements.txt` to keep the repo lightweight. If you hit a `ModuleNotFoundError`, install the missing package with `pip install <module_name>`.
|
||||
|
||||
---
|
||||
|
||||
## 📋 Requirements
|
||||
|
||||
| Requirement | Details |
|
||||
| --- | --- |
|
||||
| **OS** | Windows 10/11, macOS, or Linux |
|
||||
| **Python** | 3.11 or 3.12 |
|
||||
| **Microphone** | Required for voice interaction |
|
||||
| **API Key** | Free Gemini API key (`config/api_keys.json`) |
|
||||
|
||||
---
|
||||
|
||||
## 🗂️ Project Structure
|
||||
|
||||
```
|
||||
Mark LI/
|
||||
├── main.py # Core loop — Gemini Live session, audio I/O, tool dispatch
|
||||
├── ui.py # PyQt6 HUD — waveform, log panel, plugin manager, camera feed
|
||||
├── setup.py # First-run configuration wizard
|
||||
├── plugins/
|
||||
│ └── _template.py # Copy this to write a new plugin — one file, drop in, done
|
||||
├── actions/
|
||||
│ ├── web_search.py # Gemini + DDG parallel search (news, research, price, compare)
|
||||
│ ├── screen_processor.py # Screen capture & webcam vision via Gemini Live
|
||||
│ ├── background_monitor.py # User-configured topic watching — daily DDG check, no crypto
|
||||
│ ├── proactive.py # Proactive 2.0 — time/context/rotation-aware check-ins
|
||||
│ ├── reminder.py # OS-native scheduled notifications
|
||||
│ ├── system_monitor.py # CPU / RAM / GPU / temperature telemetry
|
||||
│ ├── computer_settings.py # Volume, brightness, WiFi, power
|
||||
│ ├── computer_control.py # Keyboard shortcuts, mouse, window management
|
||||
│ ├── open_app.py # Application launcher
|
||||
│ ├── browser_control.py # Web browser control
|
||||
│ ├── file_controller.py # File system operations
|
||||
│ ├── file_processor.py # Document reading and summarization
|
||||
│ ├── send_message.py # Messaging integration
|
||||
│ ├── weather_report.py # Live weather data
|
||||
│ ├── flight_finder.py # Flight search
|
||||
│ ├── youtube_video.py # YouTube playback control
|
||||
│ ├── game_updater.py # Game update management (Steam / Epic)
|
||||
│ ├── code_helper.py # Code review and generation
|
||||
│ ├── dev_agent.py # Developer task agent
|
||||
│ └── desktop.py # Desktop and taskbar control
|
||||
├── memory/
|
||||
│ ├── memory_manager.py # Load/save long_term.json — sessions, monitors, identity
|
||||
│ └── long_term.json # Persistent store: identity, preferences, projects, sessions, monitors
|
||||
├── core/
|
||||
│ ├── prompt.txt # Assistant personality and tool-routing rules
|
||||
│ └── plugin_loader.py # Plugin engine — discovery, validation, crash isolation
|
||||
└── config/
|
||||
└── api_keys.json # API key, OS setting, assistant name, user name
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## ⚠️ License
|
||||
|
||||
Personal and non-commercial use only.
|
||||
Licensed under **[Creative Commons BY-NC 4.0](https://creativecommons.org/licenses/by-nc/4.0/)**.
|
||||
|
||||
---
|
||||
|
||||
## 👤 Connect with the Creator
|
||||
|
||||
Engineered by a developer building a real-world JARVIS-style assistant.
|
||||
⭐ **Star the repository to support the journey to Mark 100.**
|
||||
|
||||
| Platform | Link |
|
||||
| --- | --- |
|
||||
| YouTube | [@FatihMakes](https://www.youtube.com/@FatihMakes) |
|
||||
| Instagram | [@fatihmakes](https://www.instagram.com/fatihmakes) |
|
||||
# ⚙️ MARK LII (52)
|
||||
### The Ultimate Cross-Platform Personal AI Assistant — By FatihMakes
|
||||
|
||||
> 📺 **[Watch the full setup video on YouTube](https://www.youtube.com/@FatihMakes)**
|
||||
|
||||
A real-time voice AI that can hear, see, understand, and control your computer — on any OS. Supports Windows, macOS, and Linux. Built on the Gemini Live API for native audio streaming, delivering zero subscriptions and total digital autonomy.
|
||||
|
||||
---
|
||||
|
||||
## ✨ Overview
|
||||
|
||||
MARK LII is the **personalization** release: the assistant that becomes *yours*. Pick the voice it speaks with, tune the colour of the whole HUD, and watch it power on with a boot chime and a swelling animation like a machine coming to life. The interface now breathes with you too — the waveform and the arc-reactor core pulse to your **real** voice while you speak and to JARVIS's own voice while it answers.
|
||||
|
||||
All of that sits on the Mark LI foundation: a plugin engine you extend without ever touching the core, native audio that hears the emotion in your voice, knows when you're not talking to it, and can hold one conversation for hours.
|
||||
|
||||
It's not just an assistant — it's an extension of your digital life.
|
||||
|
||||
---
|
||||
|
||||
## 🚀 Capabilities
|
||||
|
||||
### Core Features
|
||||
| Feature | Description |
|
||||
|---|---|
|
||||
| 🎙️ Voice Picker | Choose from 5 native Gemini voices and switch live from the UI — no restart |
|
||||
| 🎨 Live Theming | Recolour the entire HUD from a hue wheel or hex — applied instantly across every panel |
|
||||
| 〰️ Reactive HUD | Waveform and reactor core pulse to real audio — your mic while listening, JARVIS while speaking |
|
||||
| 🧠 Recallable Memory | No size limit and nothing silently forgotten — the prompt carries what fits, the rest is looked up on demand from a local search |
|
||||
| 👁️ Memory Panel | See every fact JARVIS has stored about you, when it learned it, and delete any of it in one click |
|
||||
| ↩️ Undo | Take back what the assistant did — files it moved, renamed, created or wrote, and settings it changed |
|
||||
| ⚠️ Real Confirmation | Shutdown, restart and WiFi wait for a button **you** press — the model cannot confirm its own irreversible actions |
|
||||
| 🎧 Audio Device Picker | Choose the microphone and speakers by name, filtered to the short list your OS shows — and measured, so every entry actually works |
|
||||
| 🔗 Session Continuity | A dropped connection, a voice change or a device change no longer wipes the conversation |
|
||||
| 🧩 Plugin System | Drop a single `.py` file into `plugins/` — JARVIS learns a new skill on next launch |
|
||||
| 🎙️ Real-time Voice | Ultra-low latency conversation in any language via Gemini Live API |
|
||||
| 💓 Affective Dialog | Hears the emotion in your voice and adapts its tone in response |
|
||||
| 🤫 Proactive Audio | Knows when you're not talking to it — background chatter never triggers a reply |
|
||||
| ♾️ Unlimited Sessions | Sliding-window context compression — one conversation can last for hours |
|
||||
| 🖥️ System Control | Launch apps, adjust volume/brightness, WiFi, shortcuts, power — all by voice |
|
||||
| 🧩 Autonomous Tasks | High-level planning for complex multi-step goals via agent mode |
|
||||
| 👁️ Visual Awareness | Real-time screen capture and webcam vision piped into your main Gemini session |
|
||||
| 🧠 Persistent Memory | Deeply remembers projects, preferences, and personal context across sessions |
|
||||
| ⌨️ Hybrid Input | Seamlessly switch between keyboard typing and voice commands |
|
||||
| 🌅 Morning Briefing | On first boot: greets you, reads the time, recaps yesterday, and fetches live news |
|
||||
| 🔔 Proactive 2.0 | Time-aware, context-aware check-ins — knows the time of day, your projects, and what you've been discussing |
|
||||
| 🗓️ Session Memory | Summarises each conversation and mentions it naturally next morning — consumed after use, never repeats |
|
||||
| 👁️🗨️ Background Monitoring | User-configured topic watching — checks for new headlines once a day and alerts naturally |
|
||||
| 📊 Hardware Monitoring | Continuous CPU, RAM, GPU and temperature telemetry with localized voice alerts |
|
||||
| 🌤️ Weather Report | Live weather data for your city, personalized from memory |
|
||||
| 🗺️ Dynamic Content Panel | Scrollable display layer beneath the HUD that renders web results, news, and search data |
|
||||
| 🔍 Multi-Mode Web Search | `news` / `research` / `price` / `compare` / `search` — Gemini Grounded first, DDG fallback |
|
||||
| ⏰ Smart Reminders | OS-native scheduled notifications (Windows Task Scheduler / macOS LaunchAgent / Linux systemd) |
|
||||
| ✈️ Flight Finder | Live flight price and availability lookup |
|
||||
| 🎮 Game Updater | Checks and triggers game updates on Steam and Epic Games on demand |
|
||||
| 📂 File Processor | Read, summarize, and answer questions about local files |
|
||||
| 💻 Code Helper | Inline code review, debugging, and generation |
|
||||
| 🌐 Browser Control | Open URLs, navigate tabs, and interact with the browser by voice |
|
||||
| 📨 Send Message | Compose and send messages through WhatsApp, Telegram, and more |
|
||||
| 🎬 YouTube Control | Search, play, and control YouTube playback by voice |
|
||||
| 🖱️ Desktop Control | Taskbar, window management, and desktop-level operations |
|
||||
| 🧑💻 Silent Language Memory | Detects spoken language on first use — all future sessions adapt automatically |
|
||||
| 📱 Remote Dashboard | Control the assistant from your phone via QR code pairing |
|
||||
| ⚡ Auto-Start on Boot | Registers with the OS startup system (registry / LaunchAgent / .desktop) |
|
||||
| 📋 Clipboard Intelligence | Copy any text → floating panel with Translate / Summarise / Explain / Fix |
|
||||
| 🪪 Assistant Customization | Change the assistant name, your name, voice, and colour from the UI — takes effect immediately |
|
||||
|
||||
---
|
||||
|
||||
## 🆕 What's New in Mark LII
|
||||
|
||||
Mark LII is about making JARVIS feel like *your own* machine. Four small-but-delightful upgrades — all universal: no hardcoded values, no bundled asset files, and no assumptions about your language or operating system.
|
||||
|
||||
### 🎙️ Voice Picker — Give JARVIS the Voice You Want
|
||||
JARVIS is no longer stuck with one voice. Open **⚙ Customise Assistant** and choose between five native Gemini voices — **Charon, Puck, Kore, Fenrir, Aoede** — each with its own character. The switch is live: the session rebuilds itself the instant you apply, so the new voice takes over without you restarting anything, and session resumption keeps your conversation going. The voice names are language-neutral, so the picker reads the same in every locale.
|
||||
|
||||
### 🎨 Live Theming — Recolour the Entire Interface
|
||||
Drag the hue wheel (or type an exact hex code) and the whole HUD re-themes in real time — panels, borders, the reactor core, the waveform, every button. Make it classic arc-reactor cyan, Iron Man gold, hostile red, or anything in between. Your choice is saved and restored on the next launch.
|
||||
|
||||
### 〰️ Reactive HUD — The Interface Breathes With the Room
|
||||
The waveform and the arc-reactor core now respond to **real audio**, not a random animation. While JARVIS listens, they pulse to your microphone; while JARVIS speaks, they pulse to its own voice — louder speech, taller bars and a brighter, wider core. When the room goes quiet, everything settles back into a gentle idle ripple. It makes the assistant feel genuinely alive and connected to what's happening.
|
||||
|
||||
Every launch now opens with a proper boot: a ~2.4-second cinematic **transform** sound — a reactor spinning up, servos locking into place, and a bright chord confirming "online" — plays as the HUD swells up from a dim point, rings spin up, and a bright pulse sweeps outward. It's synthesized entirely in code (no sound file to ship, identical on Windows, macOS and Linux), and you can turn it on or off any time from the **🔊 BOOT SOUND** toggle in the ⚙ controls. If a machine has no audio output, it simply stays silent — never an error.
|
||||
|
||||
> Built on Mark LI's foundation: the **🧩 Plugin System** (extend JARVIS with a single drop-in file), **💓 Affective Dialog**, **🤫 Proactive Audio**, and **♾️ Unlimited Sessions** are all still here and unchanged.
|
||||
|
||||
---
|
||||
|
||||
## 🔄 The Foundation Update — in every Mark from LII
|
||||
|
||||
These four landed across **Mark LII, LIII, LIV and LV at the same time**, after each of those releases had already shipped. They are not what any one of those versions originally introduced; they are the floor all of them now stand on, so moving up a Mark never costs you something the one below it had.
|
||||
|
||||
No new dependencies. No bundled asset files. No hardcoded language, and nothing that assumes one operating system.
|
||||
|
||||
### 🧠 A memory that actually remembers
|
||||
|
||||
The store was capped at **2,200 characters — the whole memory, not per entry** — because all of it was pasted into the system prompt on every connect, so growing the memory grew every request. When it filled, the oldest entries were deleted and one line was printed to a console nobody reads. An assistant advertised as remembering "projects, preferences and personal context" was in practice a two-page notepad that quietly forgot your sister's name after a few weeks.
|
||||
|
||||
Storage and prompt budget are now separate problems:
|
||||
|
||||
* **Nothing is deleted.** The cap is a runaway guard normal use never approaches, and if it is ever hit it says so in the activity log instead of on stdout.
|
||||
* **The prompt carries a core, not a dump.** Identity in full, then the most recently updated facts, budgeted — measured at **under 1,000 characters on a memory holding 61 stored facts.** That is *smaller* than the old whole-store cap, so sessions now connect with fewer tokens than before.
|
||||
* **The rest is fetched on demand.** A `recall_memory` tool searches the full store locally — no network, no second model, well under a millisecond.
|
||||
|
||||
The part that is easy to get wrong: **a model cannot look something up if it doesn't know the thing exists.** So the prompt also carries an **index of the keys** it had no room for. Without it, "who is Ayşe?" gets "I don't know" while `ayse_sister` sits on disk unread. That index interleaves categories rather than sorting by recency — sorted like the core, a memory with forty preferences pushed the one entry the index existed for off the end.
|
||||
|
||||
⚙ → **🧠 MEMORY** shows every stored fact, when it was learned, and a ✕ to forget it. Everything stays in `memory/long_term.json` on your machine.
|
||||
|
||||
### ↩️ Undo — it can take back what it did
|
||||
|
||||
JARVIS moves files, renames them, writes to them and changes your settings. None of that had a way back; if it misheard you, the only remedy was to fix it by hand.
|
||||
|
||||
Say **"undo"** — in any language — and it reverses its own last action:
|
||||
|
||||
| | |
|
||||
|---|---|
|
||||
| **Files** | move · rename · create · copy · write · delete · organize desktop |
|
||||
| **Settings** | volume · brightness · dark mode |
|
||||
|
||||
Three things it deliberately does *not* do:
|
||||
|
||||
* **It does not guess.** Settings undo reads the current value *before* changing it. Where a platform won't report that value, nothing is registered — an undo that restores a guess is worse than no undo.
|
||||
* **It does not hoard.** Undoing a write means keeping the old contents in memory, so files over 1 MB are excluded and it says so rather than holding a 200 MB log for the session.
|
||||
* **It does not delete your files to undo a copy.** The reverse of a copy is removing the copy; the reverse of "create a folder" is removing it *only while it's still empty*.
|
||||
|
||||
`organize_desktop` gets special treatment — one command that moves dozens of files, which made it the least reversible thing the assistant could do. It journals every move and puts all of them back in one go, cleaning up the folders it created if they're still empty.
|
||||
|
||||
**Undo costs nothing at runtime.** It appends a closure to a list; nothing in it runs unless you ask.
|
||||
|
||||
### ⚠️ A confirmation the model can't forge
|
||||
|
||||
The old gate read like this:
|
||||
|
||||
```python
|
||||
if action in _DANGEROUS_ACTIONS: # {"restart", "shutdown"}
|
||||
confirmed = str(params.get("confirmed", "")).lower()
|
||||
```
|
||||
|
||||
`confirmed` is a **tool parameter, which means the model fills it in.** Nothing stopped it sending `confirmed=yes` on the first call and nothing checked that a human was ever involved. It was a convention, not a gate. And its coverage was two actions — so `toggle_wifi`, which cuts the assistant's own connection to the Live API and therefore *cannot be asked to undo itself*, went through with no gate at all.
|
||||
|
||||
The token is now issued by the interface. Shutdown, restart and WiFi put a banner on the HUD and **return immediately**; the action runs only if you press CONFIRM. Nothing blocks — JARVIS keeps talking while the banner is up — so this is **cheaper than the old gate**, which burned two tool round trips on every power command.
|
||||
|
||||
> The split between the two mechanisms is about reversibility, not about how alarming a word sounds. Anything undoable is done at once; only the genuinely irreversible asks. An assistant that checks with you before turning the volume down is one you stop talking to.
|
||||
|
||||
### 🎧 It finally asks which microphone
|
||||
|
||||
Both audio streams opened with no device argument at all, so they always took whatever the OS called "default" — and on Windows that *moves on its own* the moment you plug a headset in. "JARVIS can't hear me" almost always meant "JARVIS is listening to the webcam".
|
||||
|
||||
⚙ → **🎧 AUDIO DEVICES** lets you pick the microphone and the speakers by name. Two things matter more than the dropdown:
|
||||
|
||||
**The list is short.** `query_devices()` returns one entry per *device × host API*, not per device — measured on an ordinary Windows machine, **41 entries for what the sound settings show as 4 microphones and 4 speakers.** The same microphone appears four times, under MME, DirectSound, WASAPI and WDM-KS, with nothing to say which is which. That is not a choice, it's a quiz. The picker takes one host API per direction, drops the "Sound Mapper" and "Primary Sound Driver" pseudo-devices that just mean "default", and deduplicates. **41 → 8.**
|
||||
|
||||
**Every entry has been measured, not assumed.** The obvious approach is to pick the host API with the nicest names — WASAPI on Windows, which in shared mode **doesn't resample**, so with 16 kHz in and 24 kHz out against 48 kHz hardware every open failed. Adding a rate check and moving to DirectSound passes that test on both sides, and PortAudio's DirectSound **output is a silent sink**: the stream opens, every write returns success in ~0 ms, and not one sample reaches the speakers.
|
||||
|
||||
| | write(2.0 s) took | |
|
||||
|---|---|---|
|
||||
| MME | **2.02 s** | consumed in real time |
|
||||
| DirectSound | **0.00 s** | swallowed instantly |
|
||||
|
||||
No capability flag reports that. So the app measures it — once per host API per direction, on a background thread at startup, using silence. Two consequences worth stating plainly:
|
||||
|
||||
* **Each direction picks its own host API.** On Windows this lands on DirectSound for the microphone and MME for the speakers — a split no amount of reasoning would have produced.
|
||||
* **The probe runs in the mode the app actually ships.** DirectSound input passes a callback stream and fails a blocking read; probing the wrong mode rejected a microphone that works perfectly.
|
||||
|
||||
Your choice is stored **by name, not by index** — indices shift whenever something is plugged in. If the saved device is gone, it falls back to the system default and says so in the log rather than failing to start.
|
||||
|
||||
### 🔗 It stops forgetting the conversation when the connection drops
|
||||
|
||||
`session_resumption` was switched on in the config and the handle the server sent back was **never read** — so every reconnect started an empty session. A dropped packet, or simply changing the voice, wiped the conversation. "Unlimited sessions" leaked through exactly this hole.
|
||||
|
||||
The handle is captured and replayed now. A network blip, or switching your microphone, keeps the conversation intact.
|
||||
|
||||
It is held in memory only, deliberately: writing it to disk would make a fresh launch continue yesterday's chat, which sounds appealing but breaks the session-summary flow — a conversation that never ends never produces a summary, and the "yesterday we talked about…" line in the morning briefing silently disappears. Changing the **voice** also starts clean on purpose, since resuming restores the server's session state and would likely bring the old voice back with it.
|
||||
|
||||
### 🩹 Fixes that came with it
|
||||
|
||||
* **The assistant could die on a log line.** Status lines carry emoji and arrows (`📤 file_controller → Moved: a.txt → Documents/`). On a non-UTF-8 console — cp1254 on a Turkish Windows, cp1251 on a Russian one, cp932 on a Japanese one — printing one raises `UnicodeEncodeError`, and because that print sits *after* the tool's own `try/except`, it escaped into the receive loop and took the session down.
|
||||
* **Every computer command paid for two model round trips.** `computer_settings` made an *entire second Gemini call, inside the tool*, purely to translate the request into one of its own action names — because the declaration only said "The action to perform", so the model rarely filled it in. When that second call failed, the fallback was `description.lower().replace(" ", "_")`, which turns the Turkish for "turn it down" into `sesi_kis` and straight into "Unknown action". The declaration now names all 56 actions and the rest is spelling tolerance handled locally by `difflib` in microseconds. When nothing matches it suggests real action names instead of dead-ending.
|
||||
* An unresolvable saved audio device, or one the driver refuses to open, falls back to the system default and says so — on both the microphone and the speakers.
|
||||
* A rejected session-resumption handle is dropped after one attempt, so an expired handle can never be replayed on every retry and prevent the reconnect it exists to protect.
|
||||
|
||||
|
||||
|
||||
---
|
||||
|
||||
## 🗺️ Mark Roadmap
|
||||
|
||||
| Mark | Focus |
|
||||
|---|---|
|
||||
| **XLVIII** | Instant interrupt · parallel news · two-phase briefing · exponential backoff · vision cooldown |
|
||||
| **XLIX** | Auto-start · clipboard intelligence · assistant customization |
|
||||
| **L** | Session memory · background monitoring · proactive 2.0 · instant vision · parallel news search |
|
||||
| **LI** | Plugin system · affective dialog · proactive audio · unlimited sessions |
|
||||
| **LII** | Voice picker · live theming · reactive HUD |
|
||||
| **LIII+** | Plugin files: email · quiz mode · calendar · and more |
|
||||
|
||||
---
|
||||
|
||||
## ⚡ Quick Start
|
||||
|
||||
```bash
|
||||
git clone https://github.com/FatihMakes/Mark-LII.git
|
||||
cd Mark-LII
|
||||
pip install -r requirements.txt
|
||||
python main.py
|
||||
```
|
||||
|
||||
> ⚠️ **Installation Note:** Some OS-specific dependencies are not bundled in `requirements.txt` to keep the repo lightweight. If you hit a `ModuleNotFoundError`, install the missing package with `pip install <module_name>`.
|
||||
|
||||
---
|
||||
|
||||
## 📋 Requirements
|
||||
|
||||
| Requirement | Details |
|
||||
| --- | --- |
|
||||
| **OS** | Windows 10/11, macOS, or Linux |
|
||||
| **Python** | 3.11 or 3.12 |
|
||||
| **Microphone** | Required for voice interaction |
|
||||
| **Speakers** | Required for voice replies |
|
||||
| **API Key** | Free Gemini API key (`config/api_keys.json`) |
|
||||
|
||||
---
|
||||
|
||||
## 🗂️ Project Structure
|
||||
|
||||
```
|
||||
Mark LII/
|
||||
├── main.py # Core loop — Gemini Live session, audio I/O, live audio levels, tool dispatch
|
||||
├── ui.py # PyQt6 HUD — reactive waveform, boot animation, log panel, plugin manager, camera feed
|
||||
├── setup.py # First-run configuration wizard
|
||||
├── plugins/
|
||||
│ └── _template.py # Copy this to write a new plugin — one file, drop in, done
|
||||
├── actions/
|
||||
│ ├── web_search.py # Gemini + DDG parallel search (news, research, price, compare)
|
||||
│ ├── screen_processor.py # Screen capture & webcam vision via Gemini Live
|
||||
│ ├── background_monitor.py # User-configured topic watching — daily DDG check, no crypto
|
||||
│ ├── proactive.py # Proactive 2.0 — time/context/rotation-aware check-ins
|
||||
│ ├── reminder.py # OS-native scheduled notifications
|
||||
│ ├── system_monitor.py # CPU / RAM / GPU / temperature telemetry
|
||||
│ ├── computer_settings.py # Volume, brightness, WiFi, power
|
||||
│ ├── computer_control.py # Keyboard shortcuts, mouse, window management
|
||||
│ ├── open_app.py # Application launcher
|
||||
│ ├── browser_control.py # Web browser control
|
||||
│ ├── file_controller.py # File system operations
|
||||
│ ├── file_processor.py # Document reading and summarization
|
||||
│ ├── send_message.py # Messaging integration
|
||||
│ ├── weather_report.py # Live weather data
|
||||
│ ├── flight_finder.py # Flight search
|
||||
│ ├── youtube_video.py # YouTube playback control
|
||||
│ ├── game_updater.py # Game update management (Steam / Epic)
|
||||
│ ├── code_helper.py # Code review and generation
|
||||
│ ├── dev_agent.py # Developer task agent
|
||||
│ └── desktop.py # Desktop and taskbar control
|
||||
├── memory/
|
||||
│ ├── memory_manager.py # Load/save long_term.json — sessions, monitors, identity
|
||||
│ ├── config_manager.py # api_keys.json access — key, OS, name, voice, colour, plugin toggles
|
||||
│ └── long_term.json # Persistent store: identity, preferences, projects, sessions, monitors
|
||||
├── core/
|
||||
│ ├── prompt.txt # Assistant personality and tool-routing rules
|
||||
│ ├── plugin_loader.py # Plugin engine — discovery, validation, crash isolation
|
||||
│ ├── undo.py # One shared undo stack — actions register how to reverse themselves
|
||||
│ ├── confirm.py # Irreversible-action gate — the token is issued by the UI, not the model
|
||||
│ └── audio_devices.py # Microphone / speaker list — filtered, measured, resolved by name
|
||||
└── config/
|
||||
└── api_keys.json # API key, OS setting, assistant name, user name, voice, UI colour, audio devices
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## ⚠️ License
|
||||
|
||||
Personal and non-commercial use only.
|
||||
Licensed under **[Creative Commons BY-NC 4.0](https://creativecommons.org/licenses/by-nc/4.0/)**.
|
||||
|
||||
---
|
||||
|
||||
## 👤 Connect with the Creator
|
||||
|
||||
Engineered by a developer building a real-world JARVIS-style assistant.
|
||||
⭐ **Star the repository to support the journey to Mark 100.**
|
||||
|
||||
| Platform | Link |
|
||||
| --- | --- |
|
||||
| YouTube | [@FatihMakes](https://www.youtube.com/@FatihMakes) |
|
||||
| Instagram | [@fatihmakes](https://www.instagram.com/fatihmakes) |
|
||||
+29
-29
@@ -1,29 +1,29 @@
|
||||
PyQt6
|
||||
sounddevice
|
||||
google-genai>=2.8.0
|
||||
google-generativeai
|
||||
pillow
|
||||
requests
|
||||
beautifulsoup4
|
||||
ddgs>=9.0.0
|
||||
playwright
|
||||
pyautogui
|
||||
pyperclip
|
||||
pygetwindow
|
||||
opencv-python
|
||||
numpy
|
||||
mss
|
||||
psutil
|
||||
send2trash
|
||||
youtube-transcript-api
|
||||
python-pptx
|
||||
fastapi
|
||||
uvicorn[standard]
|
||||
cryptography
|
||||
python-multipart
|
||||
comtypes; sys_platform == "win32"
|
||||
pycaw; sys_platform == "win32"
|
||||
win10toast; sys_platform == "win32"
|
||||
pywinauto; sys_platform == "win32"
|
||||
pywin32; sys_platform == "win32"
|
||||
qrcode[pil]
|
||||
PyQt6
|
||||
sounddevice
|
||||
google-genai>=2.8.0
|
||||
google-generativeai
|
||||
pillow
|
||||
requests
|
||||
beautifulsoup4
|
||||
ddgs>=9.0.0
|
||||
playwright
|
||||
pyautogui
|
||||
pyperclip
|
||||
pygetwindow
|
||||
opencv-python
|
||||
numpy
|
||||
mss
|
||||
psutil
|
||||
send2trash
|
||||
youtube-transcript-api
|
||||
python-pptx
|
||||
fastapi
|
||||
uvicorn[standard]
|
||||
cryptography
|
||||
python-multipart
|
||||
comtypes; sys_platform == "win32"
|
||||
pycaw; sys_platform == "win32"
|
||||
win10toast; sys_platform == "win32"
|
||||
pywinauto; sys_platform == "win32"
|
||||
pywin32; sys_platform == "win32"
|
||||
qrcode[pil]
|
||||
@@ -22,5 +22,5 @@ if platform.system() == "Windows":
|
||||
f' "{sys.executable}" "{postinstall}" -install\n'
|
||||
)
|
||||
|
||||
print("\n✅ Setup complete! Run 'python main.py' to start MARK LI.")
|
||||
print("\n✅ Setup complete! Run 'python main.py' to start MARK LII.")
|
||||
|
||||
@@ -28,7 +28,7 @@ from PyQt6.QtGui import (
|
||||
QPen, QPixmap, QRadialGradient, QShortcut,
|
||||
)
|
||||
from PyQt6.QtWidgets import (
|
||||
QApplication, QFileDialog, QFrame, QHBoxLayout, QLabel, QLineEdit,
|
||||
QApplication, QComboBox, QFileDialog, QFrame, QHBoxLayout, QLabel, QLineEdit,
|
||||
QMainWindow, QPushButton, QScrollArea, QSizePolicy, QSplitter,
|
||||
QStackedWidget, QTextEdit, QVBoxLayout, QWidget, QProgressBar,
|
||||
)
|
||||
@@ -365,10 +365,32 @@ class HudCanvas(QWidget):
|
||||
self._face_px: QPixmap | None = None
|
||||
self._load_face(face_path)
|
||||
|
||||
# Live audio reactivity: _live_amp is written from the audio threads
|
||||
# (0.0–1.0), _amp_disp is the smoothed value the paint code reads.
|
||||
self._live_amp = 0.0
|
||||
self._amp_disp = 0.0
|
||||
self._base_scale = 1.0 # slow "breathing" target; amp is added per-frame
|
||||
self._base_halo = 55.0
|
||||
|
||||
self._tmr = QTimer(self)
|
||||
self._tmr.timeout.connect(self._step)
|
||||
self._tmr.start(16)
|
||||
|
||||
def set_audio_level(self, level: float) -> None:
|
||||
"""Thread-safe entry point for the audio threads. Stores the louder of
|
||||
the incoming level and the current value so brief gaps between chunks
|
||||
don't make the waveform stutter; _step() decays it back down."""
|
||||
try:
|
||||
lv = float(level)
|
||||
except (TypeError, ValueError):
|
||||
return
|
||||
if lv < 0.0:
|
||||
lv = 0.0
|
||||
elif lv > 1.0:
|
||||
lv = 1.0
|
||||
if lv > self._live_amp:
|
||||
self._live_amp = lv
|
||||
|
||||
def _load_face(self, path: str):
|
||||
try:
|
||||
from PIL import Image, ImageDraw
|
||||
@@ -389,28 +411,50 @@ class HudCanvas(QWidget):
|
||||
def _step(self):
|
||||
self._tick += 1
|
||||
now = time.time()
|
||||
|
||||
# ── Live audio reactivity ────────────────────────────────────────────
|
||||
# Audio threads push peaks into _live_amp; decay it toward silence so
|
||||
# gaps between chunks fade out instead of freezing, then smooth it.
|
||||
self._live_amp *= 0.86
|
||||
self._amp_disp += (self._live_amp - self._amp_disp) * 0.45
|
||||
amp = self._amp_disp
|
||||
|
||||
# Slow "breathing" base target (random shimmer), refreshed on a timer.
|
||||
if now - self._last_t > (0.12 if self.speaking else 0.5):
|
||||
if self.speaking:
|
||||
self._tgt_scale = random.uniform(1.06, 1.14)
|
||||
self._tgt_halo = random.uniform(145, 190)
|
||||
self._base_scale = 1.03
|
||||
self._base_halo = 122.0
|
||||
elif self.muted:
|
||||
self._tgt_scale = random.uniform(0.998, 1.002)
|
||||
self._tgt_halo = random.uniform(15, 28)
|
||||
self._base_scale = random.uniform(0.998, 1.002)
|
||||
self._base_halo = random.uniform(15, 28)
|
||||
else:
|
||||
self._tgt_scale = random.uniform(1.001, 1.008)
|
||||
self._tgt_halo = random.uniform(48, 68)
|
||||
self._base_scale = random.uniform(1.001, 1.008)
|
||||
self._base_halo = random.uniform(48, 68)
|
||||
self._last_t = now
|
||||
|
||||
sp = 0.38 if self.speaking else 0.15
|
||||
# Every frame, the live audio level lifts the target on top of the base
|
||||
# — this is what makes the core visibly pulse to the actual voice.
|
||||
if self.muted:
|
||||
self._tgt_scale, self._tgt_halo = self._base_scale, self._base_halo
|
||||
elif self.speaking:
|
||||
self._tgt_scale = self._base_scale + amp * 0.13
|
||||
self._tgt_halo = self._base_halo + amp * 95.0
|
||||
else:
|
||||
self._tgt_scale = self._base_scale + amp * 0.06
|
||||
self._tgt_halo = self._base_halo + amp * 75.0
|
||||
|
||||
sp = 0.38 if self.speaking else (0.30 if amp > 0.02 else 0.15)
|
||||
self._scale += (self._tgt_scale - self._scale) * sp
|
||||
self._halo += (self._tgt_halo - self._halo) * sp
|
||||
|
||||
speeds = [1.3, -0.9, 2.0] if self.speaking else [0.55, -0.35, 0.9]
|
||||
# Rings/scanners spin faster while speaking, reacting to loudness.
|
||||
boost = 1.0 + amp * 1.6
|
||||
speeds = ([1.3, -0.9, 2.0] if self.speaking else [0.55, -0.35, 0.9])
|
||||
for i, spd in enumerate(speeds):
|
||||
self._rings[i] = (self._rings[i] + spd) % 360
|
||||
self._rings[i] = (self._rings[i] + spd * boost) % 360
|
||||
|
||||
self._scan = (self._scan + (3.0 if self.speaking else 1.3)) % 360
|
||||
self._scan2 = (self._scan2 + (-2.0 if self.speaking else -0.75)) % 360
|
||||
self._scan = (self._scan + (3.0 if self.speaking else 1.3) * boost) % 360
|
||||
self._scan2 = (self._scan2 + (-2.0 if self.speaking else -0.75) * boost) % 360
|
||||
|
||||
fw = min(self.width(), self.height())
|
||||
lim = fw * 0.74
|
||||
@@ -441,6 +485,8 @@ class HudCanvas(QWidget):
|
||||
|
||||
def paintEvent(self, _):
|
||||
p = QPainter(self)
|
||||
if not p.isActive(): # device not ready (e.g. 0-size during layout) — skip cleanly
|
||||
return
|
||||
p.setRenderHint(QPainter.RenderHint.Antialiasing)
|
||||
p.fillRect(self.rect(), qcol(C.BG))
|
||||
|
||||
@@ -581,21 +627,30 @@ class HudCanvas(QWidget):
|
||||
p.setFont(QFont("Courier New", 11, QFont.Weight.Bold))
|
||||
p.drawText(QRectF(0, sy, W, 26), Qt.AlignmentFlag.AlignCenter, txt)
|
||||
|
||||
# waveform
|
||||
# waveform — reacts to the real audio level (mic while listening,
|
||||
# JARVIS's own voice while speaking). Falls back to a gentle idle
|
||||
# ripple when there's no sound. _amp_disp is the smoothed 0–1 level.
|
||||
wy = sy + 30
|
||||
N, bw = 36, 8
|
||||
wx0 = (W - N * bw) / 2
|
||||
amp = self._amp_disp
|
||||
mid = (N - 1) / 2.0
|
||||
for i in range(N):
|
||||
if self.muted:
|
||||
hgt, cl = 2, qcol(C.MUTED_C)
|
||||
elif self.speaking:
|
||||
hgt = random.randint(3, 20)
|
||||
cl = qcol(C.PRI) if hgt > 12 else qcol(C.PRI_DIM)
|
||||
else:
|
||||
hgt = int(3 + 2 * math.sin(self._tick * 0.09 + i * 0.6))
|
||||
cl = qcol(C.BORDER_B)
|
||||
env = (1.0 - abs(i - mid) / mid) ** 0.7 # center-weighted hump
|
||||
shimmer = 0.55 + 0.45 * math.sin(self._tick * 0.18 + i * 0.7)
|
||||
idle = 3.0 + 2.0 * math.sin(self._tick * 0.09 + i * 0.6)
|
||||
hgt = int(max(2, min(24, idle + amp * 22.0 * env * shimmer)))
|
||||
if amp > 0.05:
|
||||
cl = qcol(C.PRI) if hgt > 12 else qcol(C.PRI_DIM)
|
||||
else:
|
||||
cl = qcol(C.BORDER_B)
|
||||
p.fillRect(QRectF(wx0 + i * bw, wy + 20 - hgt, bw - 1, hgt), cl)
|
||||
|
||||
p.end() # end deterministically so the backing store never flushes an active painter
|
||||
|
||||
class MetricBar(QWidget):
|
||||
|
||||
def __init__(self, label: str, color: str = C.PRI, parent=None):
|
||||
@@ -614,6 +669,8 @@ class MetricBar(QWidget):
|
||||
|
||||
def paintEvent(self, _):
|
||||
p = QPainter(self)
|
||||
if not p.isActive():
|
||||
return
|
||||
p.setRenderHint(QPainter.RenderHint.Antialiasing)
|
||||
W, H = self.width(), self.height()
|
||||
|
||||
@@ -650,6 +707,8 @@ class MetricBar(QWidget):
|
||||
p.setPen(QPen(bar_col if self._text != "--" else qcol(C.TEXT_DIM), 1))
|
||||
p.drawText(QRectF(0, 4, W - 6, 16), Qt.AlignmentFlag.AlignRight | Qt.AlignmentFlag.AlignVCenter, self._text)
|
||||
|
||||
p.end()
|
||||
|
||||
class LogWidget(QTextEdit):
|
||||
_sig = pyqtSignal(str)
|
||||
|
||||
@@ -857,6 +916,8 @@ class _DropCanvas(QWidget):
|
||||
|
||||
def paintEvent(self, _):
|
||||
p = QPainter(self)
|
||||
if not p.isActive():
|
||||
return
|
||||
p.setRenderHint(QPainter.RenderHint.Antialiasing)
|
||||
z = self._z
|
||||
W, H = self.width(), self.height()
|
||||
@@ -881,6 +942,8 @@ class _DropCanvas(QWidget):
|
||||
elif z._drag_over: self._paint_drag_over(p, W, H)
|
||||
else: self._paint_idle(p, W, H, z._hovering)
|
||||
|
||||
p.end()
|
||||
|
||||
def _paint_idle(self, p, W, H, hover):
|
||||
cx, cy = W / 2, H / 2
|
||||
col = qcol(C.PRI_DIM if not hover else C.PRI)
|
||||
@@ -1194,6 +1257,8 @@ class HueWheel(QWidget):
|
||||
# ── çizim ────────────────────────────────────────────────────────────────
|
||||
def paintEvent(self, _):
|
||||
p = QPainter(self)
|
||||
if not p.isActive():
|
||||
return
|
||||
p.setRenderHint(QPainter.RenderHint.Antialiasing)
|
||||
rect = self._ring_rect()
|
||||
center = rect.center()
|
||||
@@ -1220,6 +1285,7 @@ class HueWheel(QWidget):
|
||||
p.setPen(QPen(QColor("#00060a"), 2))
|
||||
p.setBrush(QBrush(QColor("#ffffff")))
|
||||
p.drawEllipse(QPointF(hx, hy), 7.5, 7.5)
|
||||
p.end()
|
||||
|
||||
# ── fare ─────────────────────────────────────────────────────────────────
|
||||
def mousePressEvent(self, e):
|
||||
@@ -1241,13 +1307,13 @@ class HueWheel(QWidget):
|
||||
|
||||
|
||||
class CustomizeOverlay(QWidget):
|
||||
"""Floating overlay — change assistant name, user name and UI colour."""
|
||||
"""Floating overlay — change assistant name, user name, UI colour and voice."""
|
||||
|
||||
saved = pyqtSignal(str, str, str) # assistant_name, user_name, ui_color
|
||||
_OW, _OH = 400, 500
|
||||
saved = pyqtSignal(str, str, str, str) # assistant_name, user_name, ui_color, voice
|
||||
_OW, _OH = 400, 588
|
||||
|
||||
def __init__(self, assistant_name="JARVIS", user_name="",
|
||||
ui_color=DEFAULT_UI_COLOR, parent=None):
|
||||
ui_color=DEFAULT_UI_COLOR, voice="", parent=None):
|
||||
super().__init__(parent)
|
||||
self.setAttribute(Qt.WidgetAttribute.WA_StyledBackground, True)
|
||||
self.setStyleSheet(f"""
|
||||
@@ -1295,6 +1361,30 @@ class CustomizeOverlay(QWidget):
|
||||
self._user_input.setStyleSheet(_fs)
|
||||
lay.addWidget(self._user_input)
|
||||
|
||||
# ── Assistant voice — Gemini prebuilt voices ─────────────────────────
|
||||
# Names are language-neutral proper nouns, so the row reads the same in
|
||||
# every locale. Selecting one and applying rebuilds the Live session.
|
||||
from memory.config_manager import AVAILABLE_VOICES, DEFAULT_VOICE
|
||||
lay.addSpacing(4)
|
||||
lay.addWidget(_lbl("ASSISTANT VOICE", 8, color=C.TEXT_DIM,
|
||||
align=Qt.AlignmentFlag.AlignLeft))
|
||||
self._sel_voice = (voice or DEFAULT_VOICE)
|
||||
if self._sel_voice not in AVAILABLE_VOICES:
|
||||
self._sel_voice = DEFAULT_VOICE
|
||||
self._voice_btns: dict[str, QPushButton] = {}
|
||||
voice_row = QHBoxLayout(); voice_row.setSpacing(4)
|
||||
for _v in AVAILABLE_VOICES:
|
||||
b = QPushButton(_v)
|
||||
b.setCheckable(True)
|
||||
b.setFixedHeight(28)
|
||||
b.setFont(QFont("Courier New", 8, QFont.Weight.Bold))
|
||||
b.setCursor(Qt.CursorShape.PointingHandCursor)
|
||||
b.clicked.connect(lambda _=False, name=_v: self._on_voice_pick(name))
|
||||
self._voice_btns[_v] = b
|
||||
voice_row.addWidget(b)
|
||||
lay.addLayout(voice_row)
|
||||
self._refresh_voice_btns()
|
||||
|
||||
# ── UI colour — renk çarkı ───────────────────────────────────────────
|
||||
lay.addSpacing(4)
|
||||
clr_hdr = QHBoxLayout()
|
||||
@@ -1367,6 +1457,28 @@ class CustomizeOverlay(QWidget):
|
||||
btn_row.addWidget(cancel_btn)
|
||||
lay.addLayout(btn_row)
|
||||
|
||||
# ── ses seçimi ───────────────────────────────────────────────────────────
|
||||
def _on_voice_pick(self, name: str):
|
||||
self._sel_voice = name
|
||||
self._refresh_voice_btns()
|
||||
|
||||
def _refresh_voice_btns(self):
|
||||
"""Highlight the selected voice pill; dim the rest."""
|
||||
for name, b in self._voice_btns.items():
|
||||
on = (name == self._sel_voice)
|
||||
b.setChecked(on)
|
||||
if on:
|
||||
b.setStyleSheet(f"""
|
||||
QPushButton {{ background: {C.PRI_GHO}; color: {C.PRI};
|
||||
border: 1px solid {C.PRI}; border-radius: 3px; }}
|
||||
""")
|
||||
else:
|
||||
b.setStyleSheet(f"""
|
||||
QPushButton {{ background: transparent; color: {C.TEXT_MED};
|
||||
border: 1px solid {C.BORDER}; border-radius: 3px; }}
|
||||
QPushButton:hover {{ color: {C.TEXT}; border-color: {C.BORDER_B}; }}
|
||||
""")
|
||||
|
||||
# ── renk akışı ───────────────────────────────────────────────────────────
|
||||
def _set_color(self, hx: str, update_wheel: bool = True, preview: bool = True):
|
||||
"""Seçili rengi günceller; hex kutusu + çark senkron kalır, tema canlı önizlenir."""
|
||||
@@ -1408,7 +1520,7 @@ class CustomizeOverlay(QWidget):
|
||||
def _save(self):
|
||||
name = self._name_input.text().strip() or "JARVIS"
|
||||
user = self._user_input.text().strip()
|
||||
self.saved.emit(name, user, self._sel_color or DEFAULT_UI_COLOR)
|
||||
self.saved.emit(name, user, self._sel_color or DEFAULT_UI_COLOR, self._sel_voice)
|
||||
self.hide()
|
||||
|
||||
|
||||
@@ -1523,6 +1635,433 @@ class PluginManagerOverlay(QWidget):
|
||||
self._style_toggle(btn, new_val)
|
||||
|
||||
|
||||
class _HudOverlay(QWidget):
|
||||
"""Base for the floating panels placed by hand over the HUD.
|
||||
|
||||
They are children of the central widget but sit in no layout, so Qt never
|
||||
invalidates the region they occupy when they hide or shrink: the HUD keeps
|
||||
painting around them and their last frame stays on screen as a ghost. Any
|
||||
overlay positioned with _centre_overlay needs this."""
|
||||
|
||||
def hideEvent(self, e):
|
||||
p = self.parentWidget()
|
||||
if p is not None:
|
||||
# Repaint exactly what we were covering, before we stop covering it.
|
||||
p.update(self.geometry())
|
||||
super().hideEvent(e)
|
||||
|
||||
def closeEvent(self, e):
|
||||
p = self.parentWidget()
|
||||
if p is not None:
|
||||
p.update(self.geometry())
|
||||
super().closeEvent(e)
|
||||
|
||||
|
||||
class ConfirmBanner(_HudOverlay):
|
||||
"""The gate in front of an action that cannot be taken back.
|
||||
|
||||
The old confirmation was a tool parameter the model filled in itself, which
|
||||
means it confirmed its own shutdown requests. This is the interface asking,
|
||||
and the answer travels from a human finger to core/confirm.py without the
|
||||
model in the loop. Nothing blocks while it is up: the assistant keeps
|
||||
talking, so this costs no latency — unlike the old gate, which spent two
|
||||
tool round trips on every power command."""
|
||||
|
||||
answered = pyqtSignal(bool)
|
||||
_OW = 430
|
||||
|
||||
def __init__(self, title: str, detail: str, parent=None):
|
||||
super().__init__(parent)
|
||||
self.setAttribute(Qt.WidgetAttribute.WA_StyledBackground, True)
|
||||
self.setStyleSheet(f"""
|
||||
ConfirmBanner {{
|
||||
background: rgba(14, 3, 0, 250);
|
||||
border: 1px solid {C.ACC};
|
||||
border-radius: 6px;
|
||||
}}
|
||||
""")
|
||||
self.setFixedWidth(self._OW)
|
||||
|
||||
lay = QVBoxLayout(self)
|
||||
lay.setContentsMargins(20, 16, 20, 16)
|
||||
lay.setSpacing(8)
|
||||
|
||||
hdr = QLabel("⚠ CONFIRM")
|
||||
hdr.setFont(QFont("Courier New", 11, QFont.Weight.Bold))
|
||||
hdr.setStyleSheet(f"color: {C.ACC}; background: transparent;")
|
||||
lay.addWidget(hdr)
|
||||
|
||||
ttl = QLabel(title)
|
||||
ttl.setWordWrap(True)
|
||||
ttl.setFont(QFont("Courier New", 10, QFont.Weight.Bold))
|
||||
ttl.setStyleSheet(f"color: {C.TEXT}; background: transparent;")
|
||||
lay.addWidget(ttl)
|
||||
|
||||
if detail:
|
||||
dtl = QLabel(detail)
|
||||
dtl.setWordWrap(True)
|
||||
dtl.setFont(QFont("Courier New", 8))
|
||||
dtl.setStyleSheet(f"color: {C.TEXT_MED}; background: transparent;")
|
||||
lay.addWidget(dtl)
|
||||
|
||||
row = QHBoxLayout(); row.setSpacing(8)
|
||||
|
||||
yes = QPushButton("▸ CONFIRM")
|
||||
yes.setFixedHeight(32)
|
||||
yes.setFont(QFont("Courier New", 9, QFont.Weight.Bold))
|
||||
yes.setCursor(Qt.CursorShape.PointingHandCursor)
|
||||
yes.setStyleSheet(f"""
|
||||
QPushButton {{ background: transparent; color: {C.ACC};
|
||||
border: 1px solid {C.ACC}; border-radius: 3px; }}
|
||||
QPushButton:hover {{ background: rgba(255,107,0,40); }}
|
||||
""")
|
||||
yes.clicked.connect(lambda: self.answered.emit(True))
|
||||
row.addWidget(yes)
|
||||
|
||||
no = QPushButton("CANCEL")
|
||||
no.setFixedHeight(32)
|
||||
no.setFont(QFont("Courier New", 9))
|
||||
no.setCursor(Qt.CursorShape.PointingHandCursor)
|
||||
no.setStyleSheet(f"""
|
||||
QPushButton {{ background: transparent; color: {C.TEXT_MED};
|
||||
border: 1px solid {C.BORDER}; border-radius: 3px; }}
|
||||
QPushButton:hover {{ color: {C.TEXT}; border-color: {C.BORDER_B}; }}
|
||||
""")
|
||||
no.clicked.connect(lambda: self.answered.emit(False))
|
||||
row.addWidget(no)
|
||||
lay.addLayout(row)
|
||||
|
||||
# Default focus on CANCEL: if someone hits Enter without reading, the
|
||||
# safe answer wins.
|
||||
no.setDefault(True)
|
||||
no.setFocus()
|
||||
|
||||
|
||||
class AudioDeviceOverlay(_HudOverlay):
|
||||
"""Choose which microphone JARVIS listens to and which speakers it uses.
|
||||
|
||||
Both audio streams used to open with no `device=` at all, so they always
|
||||
took the OS default — which on Windows moves by itself the moment a headset
|
||||
is plugged in. 'JARVIS can't hear me' is usually 'JARVIS is listening to the
|
||||
webcam'."""
|
||||
|
||||
picked = pyqtSignal() # emitted after Apply, when something changed
|
||||
_OW = 460
|
||||
|
||||
def __init__(self, parent=None):
|
||||
super().__init__(parent)
|
||||
from core.audio_devices import list_devices, DEFAULT_LABEL
|
||||
from memory.config_manager import get_input_device, get_output_device
|
||||
|
||||
self.setAttribute(Qt.WidgetAttribute.WA_StyledBackground, True)
|
||||
self.setStyleSheet(f"""
|
||||
AudioDeviceOverlay {{
|
||||
background: rgba(0, 6, 10, 245);
|
||||
border: 1px solid {C.BORDER_B};
|
||||
border-radius: 6px;
|
||||
}}
|
||||
""")
|
||||
self.setFixedWidth(self._OW)
|
||||
|
||||
lay = QVBoxLayout(self)
|
||||
lay.setContentsMargins(20, 16, 20, 16)
|
||||
lay.setSpacing(6)
|
||||
|
||||
hdr = QLabel("🎧 AUDIO DEVICES")
|
||||
hdr.setFont(QFont("Courier New", 12, QFont.Weight.Bold))
|
||||
hdr.setStyleSheet(f"color: {C.PRI}; background: transparent;")
|
||||
lay.addWidget(hdr)
|
||||
|
||||
sep = QFrame(); sep.setFrameShape(QFrame.Shape.HLine)
|
||||
sep.setStyleSheet(f"color: {C.BORDER}; margin: 2px 0;")
|
||||
lay.addWidget(sep)
|
||||
|
||||
_combo_css = (
|
||||
f"QComboBox {{ background: #000d12; color: {C.TEXT}; "
|
||||
f"border: 1px solid {C.BORDER}; border-radius: 3px; padding: 4px 8px; }}"
|
||||
f"QComboBox:hover {{ border-color: {C.BORDER_B}; }}"
|
||||
f"QComboBox QAbstractItemView {{ background: #000d12; color: {C.TEXT}; "
|
||||
f"selection-background-color: {C.PRI_GHO}; border: 1px solid {C.BORDER}; }}"
|
||||
)
|
||||
|
||||
def _row(label: str, kind: str, current: str) -> QComboBox:
|
||||
cap = QLabel(label)
|
||||
cap.setFont(QFont("Courier New", 8))
|
||||
cap.setStyleSheet(f"color: {C.TEXT_DIM}; background: transparent;")
|
||||
lay.addWidget(cap)
|
||||
|
||||
box = QComboBox()
|
||||
box.setFont(QFont("Courier New", 9))
|
||||
box.setFixedHeight(30)
|
||||
box.setStyleSheet(_combo_css)
|
||||
# The list is served from a cache warmed on a background thread at
|
||||
# startup, so opening this panel never blocks the Qt thread on the
|
||||
# host audio API.
|
||||
box.addItem(DEFAULT_LABEL, "")
|
||||
for name in list_devices(kind):
|
||||
box.addItem(name, name)
|
||||
idx = box.findData(current) if current else 0
|
||||
box.setCurrentIndex(idx if idx >= 0 else 0)
|
||||
if current and idx < 0:
|
||||
# Saved device is not plugged in right now. Show it rather than
|
||||
# silently resetting the user's choice to default.
|
||||
box.addItem(f"{current} (not connected)", current)
|
||||
box.setCurrentIndex(box.count() - 1)
|
||||
lay.addWidget(box)
|
||||
return box
|
||||
|
||||
self._in_box = _row("MICROPHONE — what JARVIS hears you with",
|
||||
"input", get_input_device())
|
||||
lay.addSpacing(4)
|
||||
self._out_box = _row("SPEAKERS — what JARVIS talks through",
|
||||
"output", get_output_device())
|
||||
|
||||
note = QLabel("Applying reconnects the session. Your conversation is kept.")
|
||||
note.setWordWrap(True)
|
||||
note.setFont(QFont("Courier New", 7))
|
||||
note.setStyleSheet(f"color: {C.TEXT_DIM}; background: transparent;")
|
||||
lay.addSpacing(6)
|
||||
lay.addWidget(note)
|
||||
|
||||
row = QHBoxLayout(); row.setSpacing(8)
|
||||
ok = QPushButton("▸ APPLY")
|
||||
ok.setFixedHeight(32)
|
||||
ok.setFont(QFont("Courier New", 9, QFont.Weight.Bold))
|
||||
ok.setCursor(Qt.CursorShape.PointingHandCursor)
|
||||
ok.setStyleSheet(f"""
|
||||
QPushButton {{ background: transparent; color: {C.PRI};
|
||||
border: 1px solid {C.PRI_DIM}; border-radius: 3px; }}
|
||||
QPushButton:hover {{ background: {C.PRI_GHO}; border-color: {C.PRI}; }}
|
||||
""")
|
||||
ok.clicked.connect(self._apply)
|
||||
row.addWidget(ok)
|
||||
|
||||
cancel = QPushButton("CLOSE")
|
||||
cancel.setFixedHeight(32)
|
||||
cancel.setFont(QFont("Courier New", 9))
|
||||
cancel.setCursor(Qt.CursorShape.PointingHandCursor)
|
||||
cancel.setStyleSheet(f"""
|
||||
QPushButton {{ background: transparent; color: {C.TEXT_MED};
|
||||
border: 1px solid {C.BORDER}; border-radius: 3px; }}
|
||||
QPushButton:hover {{ color: {C.TEXT}; border-color: {C.BORDER_B}; }}
|
||||
""")
|
||||
cancel.clicked.connect(self.hide)
|
||||
row.addWidget(cancel)
|
||||
lay.addLayout(row)
|
||||
|
||||
def _apply(self):
|
||||
from memory.config_manager import (
|
||||
get_input_device, get_output_device,
|
||||
save_input_device, save_output_device,
|
||||
)
|
||||
new_in = self._in_box.currentData() or ""
|
||||
new_out = self._out_box.currentData() or ""
|
||||
changed = (new_in != get_input_device()) or (new_out != get_output_device())
|
||||
save_input_device(new_in)
|
||||
save_output_device(new_out)
|
||||
self.hide()
|
||||
# Only rebuild the session if something actually moved — a no-op Apply
|
||||
# should not cost a reconnect.
|
||||
if changed:
|
||||
self.picked.emit()
|
||||
|
||||
|
||||
class MemoryOverlay(_HudOverlay):
|
||||
"""Everything JARVIS has stored about you, and when it learned it.
|
||||
|
||||
Memory used to be a 2200-character store that deleted its oldest entries
|
||||
when full and mentioned it only on stdout. The cap is gone; this panel is
|
||||
the other half of that change — a memory you cannot inspect is a memory you
|
||||
cannot trust, and 'delete' has to be something the person can do."""
|
||||
|
||||
_OW = 520
|
||||
|
||||
def __init__(self, parent=None):
|
||||
super().__init__(parent)
|
||||
self.setAttribute(Qt.WidgetAttribute.WA_StyledBackground, True)
|
||||
self.setStyleSheet(f"""
|
||||
MemoryOverlay {{
|
||||
background: rgba(0, 6, 10, 246);
|
||||
border: 1px solid {C.BORDER_B};
|
||||
border-radius: 6px;
|
||||
}}
|
||||
""")
|
||||
self.setFixedWidth(self._OW)
|
||||
|
||||
self._lay = QVBoxLayout(self)
|
||||
self._lay.setContentsMargins(20, 16, 20, 16)
|
||||
self._lay.setSpacing(5)
|
||||
self._rebuild()
|
||||
|
||||
def _clear_layout(self):
|
||||
"""Take every item out of the layout and detach it from the widget tree
|
||||
in this call.
|
||||
|
||||
deleteLater() on its own is not enough: it queues destruction for the
|
||||
next event-loop pass, and until then the old rows are still children of
|
||||
this widget and still paint — which is what drew half of the previous
|
||||
panel over the new one. setParent(None) removes them from the tree now;
|
||||
deleteLater() then frees them safely."""
|
||||
while self._lay.count():
|
||||
item = self._lay.takeAt(0)
|
||||
w = item.widget()
|
||||
if w is not None:
|
||||
# hide() stops it painting in this frame; deleteLater() frees it
|
||||
# safely afterwards. setParent(None) would also stop the paint,
|
||||
# but it turns the widget into a top-level window for the moment
|
||||
# between the two calls, which is not something to leave lying
|
||||
# around inside a click handler.
|
||||
w.hide()
|
||||
w.deleteLater()
|
||||
continue
|
||||
sub = item.layout()
|
||||
if sub is not None:
|
||||
while sub.count():
|
||||
si = sub.takeAt(0)
|
||||
sw = si.widget()
|
||||
if sw is not None:
|
||||
sw.hide()
|
||||
sw.deleteLater()
|
||||
sub.deleteLater()
|
||||
|
||||
def _settle(self, before):
|
||||
"""Size the panel to its content, re-centre it, and repaint what the old
|
||||
size covered.
|
||||
|
||||
The re-size has to happen here rather than at the end of _rebuild
|
||||
because Qt has not polished the freshly-created children at that point,
|
||||
so the size hint it would read is the empty-layout one. Measured: a
|
||||
first adjustSize() returned 32 px for a panel whose content needed 155,
|
||||
and a second call — after the same widgets had been through the event
|
||||
loop — returned 155. So this runs twice: once now, once on the next
|
||||
turn, from _rebuild.
|
||||
|
||||
The re-centre and the repaint are needed because the overlay is placed
|
||||
by hand and is in no layout: shrinking it leaves it off-centre and
|
||||
leaves its former pixels on screen, since nothing tells the parent that
|
||||
region changed. The repaint has to cover the union of the old and new
|
||||
rectangles."""
|
||||
self._lay.invalidate()
|
||||
self._lay.activate()
|
||||
self.updateGeometry()
|
||||
self.adjustSize()
|
||||
|
||||
p = self.parentWidget()
|
||||
if p is None:
|
||||
self.update()
|
||||
return
|
||||
self.move(max(0, (p.width() - self.width()) // 2),
|
||||
max(0, (p.height() - self.height()) // 2))
|
||||
p.update(before.united(self.geometry()))
|
||||
self.update()
|
||||
|
||||
def _rebuild(self):
|
||||
before = self.geometry()
|
||||
self._clear_layout()
|
||||
|
||||
from memory.memory_manager import all_entries_for_ui
|
||||
|
||||
hdr = QLabel("🧠 WHAT JARVIS REMEMBERS")
|
||||
hdr.setFont(QFont("Courier New", 12, QFont.Weight.Bold))
|
||||
hdr.setStyleSheet(f"color: {C.PRI}; background: transparent;")
|
||||
self._lay.addWidget(hdr)
|
||||
|
||||
sep = QFrame(); sep.setFrameShape(QFrame.Shape.HLine)
|
||||
sep.setStyleSheet(f"color: {C.BORDER}; margin: 2px 0;")
|
||||
self._lay.addWidget(sep)
|
||||
|
||||
rows = all_entries_for_ui()
|
||||
|
||||
cap = QLabel(f"{len(rows)} stored facts — newest first. "
|
||||
f"Nothing here is sent anywhere; it lives in "
|
||||
f"memory/long_term.json on this machine.")
|
||||
cap.setWordWrap(True)
|
||||
cap.setFont(QFont("Courier New", 7))
|
||||
cap.setStyleSheet(f"color: {C.TEXT_DIM}; background: transparent;")
|
||||
self._lay.addWidget(cap)
|
||||
|
||||
if not rows:
|
||||
empty = QLabel("Nothing stored yet.")
|
||||
empty.setFont(QFont("Courier New", 9))
|
||||
empty.setStyleSheet(f"color: {C.TEXT_MED}; background: transparent;")
|
||||
self._lay.addWidget(empty)
|
||||
else:
|
||||
scroll = QScrollArea()
|
||||
scroll.setWidgetResizable(True)
|
||||
scroll.setFixedHeight(min(420, 34 * len(rows) + 10))
|
||||
scroll.setStyleSheet(
|
||||
f"QScrollArea {{ border: 1px solid {C.BORDER}; border-radius: 3px; "
|
||||
f"background: transparent; }}"
|
||||
)
|
||||
inner = QWidget()
|
||||
ilay = QVBoxLayout(inner)
|
||||
ilay.setContentsMargins(6, 6, 6, 6)
|
||||
ilay.setSpacing(3)
|
||||
|
||||
for r in rows:
|
||||
line = QHBoxLayout(); line.setSpacing(6)
|
||||
txt = QLabel(f"<b>{r['key'].replace('_', ' ')}</b> "
|
||||
f"<span style='color:{C.TEXT_MED}'>— {r['value']}</span>")
|
||||
txt.setWordWrap(True)
|
||||
txt.setFont(QFont("Courier New", 8))
|
||||
txt.setStyleSheet(f"color: {C.TEXT}; background: transparent;")
|
||||
line.addWidget(txt, 1)
|
||||
|
||||
meta = QLabel(f"{r['category'][:4]} · {r['updated'] or '—'}")
|
||||
meta.setFont(QFont("Courier New", 7))
|
||||
meta.setStyleSheet(f"color: {C.TEXT_DIM}; background: transparent;")
|
||||
line.addWidget(meta)
|
||||
|
||||
rm = QPushButton("✕")
|
||||
rm.setFixedSize(20, 20)
|
||||
rm.setFont(QFont("Courier New", 8, QFont.Weight.Bold))
|
||||
rm.setCursor(Qt.CursorShape.PointingHandCursor)
|
||||
rm.setToolTip("Forget this")
|
||||
rm.setStyleSheet(f"""
|
||||
QPushButton {{ background: transparent; color: {C.TEXT_DIM};
|
||||
border: 1px solid {C.BORDER}; border-radius: 3px; }}
|
||||
QPushButton:hover {{ color: {C.RED}; border-color: {C.RED}; }}
|
||||
""")
|
||||
rm.clicked.connect(
|
||||
lambda _=False, c=r["category"], k=r["key"]: self._forget(c, k))
|
||||
line.addWidget(rm)
|
||||
|
||||
holder = QWidget()
|
||||
holder.setLayout(line)
|
||||
ilay.addWidget(holder)
|
||||
|
||||
ilay.addStretch()
|
||||
scroll.setWidget(inner)
|
||||
self._lay.addWidget(scroll)
|
||||
|
||||
close = QPushButton("CLOSE")
|
||||
close.setFixedHeight(30)
|
||||
close.setFont(QFont("Courier New", 9))
|
||||
close.setCursor(Qt.CursorShape.PointingHandCursor)
|
||||
close.setStyleSheet(f"""
|
||||
QPushButton {{ background: transparent; color: {C.TEXT_MED};
|
||||
border: 1px solid {C.BORDER}; border-radius: 3px; }}
|
||||
QPushButton:hover {{ color: {C.TEXT}; border-color: {C.BORDER_B}; }}
|
||||
""")
|
||||
close.clicked.connect(self.hide)
|
||||
self._lay.addWidget(close)
|
||||
|
||||
self._settle(before)
|
||||
# …and again once Qt has polished the new children, because the size
|
||||
# hint is not final until then. Harmless when the first pass already
|
||||
# got it right: _settle is idempotent.
|
||||
QTimer.singleShot(0, lambda g=before: self._settle(g))
|
||||
|
||||
def _forget(self, category: str, key: str):
|
||||
from memory.memory_manager import forget
|
||||
forget(key, category)
|
||||
# Rebuild on the NEXT event-loop turn, not inside this click handler.
|
||||
# The rebuild destroys the very ✕ button that emitted this signal, and
|
||||
# Qt is entitled to touch the sender after a slot returns; tearing it
|
||||
# down mid-emission is how a widget ends up half-alive on screen.
|
||||
QTimer.singleShot(0, self._rebuild)
|
||||
|
||||
|
||||
class ClipboardPanel(QWidget):
|
||||
"""Floating panel shown when text is copied — offers quick Jarvis actions."""
|
||||
|
||||
@@ -1846,6 +2385,8 @@ class MainWindow(QMainWindow):
|
||||
_cam_stream_sig = pyqtSignal(bool) # True=start live stream, False=stop
|
||||
_cam_frame_sig = pyqtSignal(bytes) # live camera frame → HUD area
|
||||
_clipboard_sig = pyqtSignal(str) # clipboard text changed (thread-safe)
|
||||
_confirm_sig = pyqtSignal(str, str) # (title, detail) — irreversible-action gate
|
||||
_confirm_hide_sig = pyqtSignal()
|
||||
|
||||
def __init__(self, face_path: str):
|
||||
super().__init__()
|
||||
@@ -1861,7 +2402,7 @@ class MainWindow(QMainWindow):
|
||||
if _ui_color and _ui_color.lower() != DEFAULT_UI_COLOR:
|
||||
apply_ui_accent(_ui_color)
|
||||
|
||||
self.setWindowTitle(f"{_display} — MARK LI")
|
||||
self.setWindowTitle(f"{_display} — MARK LII")
|
||||
self.setMinimumSize(_MIN_W, _MIN_H)
|
||||
self.resize(_DEFAULT_W, _DEFAULT_H)
|
||||
|
||||
@@ -1874,6 +2415,9 @@ class MainWindow(QMainWindow):
|
||||
self.on_text_command = None
|
||||
self.on_remote_clicked = None # callable: () -> (url, key) | None
|
||||
self.on_interrupt = None # callable: () -> None — stop JARVIS mid-speech
|
||||
self.on_voice_change = None # callable: () -> None — rebuild session with new voice
|
||||
self.on_audio_device_change = None # callable: () -> None — reopen audio streams
|
||||
self._confirm_overlay = None # live ConfirmBanner, if one is on screen
|
||||
self.get_plugins = None # callable: () -> list[dict], set by JarvisLive
|
||||
self._muted = False
|
||||
self._current_file: str | None = None
|
||||
@@ -1985,6 +2529,8 @@ class MainWindow(QMainWindow):
|
||||
self._content_sig.connect(self._show_content)
|
||||
self._reconfig_sig.connect(self._show_setup)
|
||||
self._camera_sig.connect(self._show_camera_frame)
|
||||
self._confirm_sig.connect(self._show_confirm_banner)
|
||||
self._confirm_hide_sig.connect(self._hide_confirm_banner)
|
||||
self._cam_stream_sig.connect(self._on_cam_stream)
|
||||
self._cam_frame_sig.connect(self._on_cam_frame)
|
||||
self._clipboard_sig.connect(self._show_clipboard_panel)
|
||||
@@ -2539,7 +3085,7 @@ class MainWindow(QMainWindow):
|
||||
l.setStyleSheet(f"color: {color}; background: transparent;")
|
||||
return l
|
||||
|
||||
lay.addWidget(_badge("MARK LI", C.PRI_DIM))
|
||||
lay.addWidget(_badge("MARK LII", C.PRI_DIM))
|
||||
lay.addSpacing(8)
|
||||
self._drawer_btn = QPushButton("⚙")
|
||||
self._drawer_btn.setFixedSize(26, 26)
|
||||
@@ -2822,6 +3368,22 @@ class MainWindow(QMainWindow):
|
||||
self._brief_btn.clicked.connect(self._toggle_brief)
|
||||
lay.addWidget(self._brief_btn)
|
||||
|
||||
audio_btn = QPushButton("🎧 AUDIO DEVICES")
|
||||
audio_btn.setFixedHeight(26)
|
||||
audio_btn.setFont(QFont("Courier New", 7))
|
||||
audio_btn.setCursor(Qt.CursorShape.PointingHandCursor)
|
||||
audio_btn.setStyleSheet(_BTN_STYLE_DIM)
|
||||
audio_btn.clicked.connect(self._open_audio_devices)
|
||||
lay.addWidget(audio_btn)
|
||||
|
||||
mem_btn = QPushButton("🧠 MEMORY")
|
||||
mem_btn.setFixedHeight(26)
|
||||
mem_btn.setFont(QFont("Courier New", 7))
|
||||
mem_btn.setCursor(Qt.CursorShape.PointingHandCursor)
|
||||
mem_btn.setStyleSheet(_BTN_STYLE_DIM)
|
||||
mem_btn.clicked.connect(self._open_memory_panel)
|
||||
lay.addWidget(mem_btn)
|
||||
|
||||
plugin_btn = QPushButton("🧩 PLUGINS")
|
||||
plugin_btn.setFixedHeight(26)
|
||||
plugin_btn.setFont(QFont("Courier New", 7))
|
||||
@@ -3195,6 +3757,7 @@ class MainWindow(QMainWindow):
|
||||
cfg.get("assistant_name", "JARVIS") or "JARVIS",
|
||||
cfg.get("user_name", ""),
|
||||
cfg.get("ui_color", "") or DEFAULT_UI_COLOR,
|
||||
cfg.get("voice_name", ""),
|
||||
parent=cw,
|
||||
)
|
||||
ow, oh = CustomizeOverlay._OW, CustomizeOverlay._OH
|
||||
@@ -3215,11 +3778,12 @@ class MainWindow(QMainWindow):
|
||||
if apply_ui_accent(hex_color):
|
||||
retheme_all_widgets(old, current_palette())
|
||||
|
||||
def _apply_name_update(self, name: str, user_name: str, ui_color: str = ""):
|
||||
def _apply_name_update(self, name: str, user_name: str, ui_color: str = "",
|
||||
voice: str = ""):
|
||||
"""Update all name/theme-dependent UI elements and persist to config."""
|
||||
self._assistant_name = name.strip() or "JARVIS"
|
||||
display = self._assistant_name.upper()
|
||||
self.setWindowTitle(f"{display} — MARK LI")
|
||||
self.setWindowTitle(f"{display} — MARK LII")
|
||||
self._title_lbl.setText(display)
|
||||
if display in ("JARVIS", "J.A.R.V.I.S"):
|
||||
self._sub_lbl.setText("Just A Rather Very Intelligent System")
|
||||
@@ -3236,6 +3800,15 @@ class MainWindow(QMainWindow):
|
||||
retheme_all_widgets(old, current_palette())
|
||||
color_changed = old["PRI"] != C.PRI
|
||||
|
||||
# Voice change → persist and, if it actually changed, rebuild the Live
|
||||
# session so the new voice takes effect (it's fixed at connect time).
|
||||
voice_changed = False
|
||||
if voice:
|
||||
from memory.config_manager import get_voice, save_voice
|
||||
if voice != get_voice():
|
||||
save_voice(voice)
|
||||
voice_changed = True
|
||||
|
||||
try:
|
||||
data = _read_full_config()
|
||||
data["assistant_name"] = self._assistant_name
|
||||
@@ -3246,9 +3819,73 @@ class MainWindow(QMainWindow):
|
||||
self._log.append_log(f"SYS: Identity updated — {display}")
|
||||
if color_changed:
|
||||
self._log.append_log(f"SYS: UI colour applied — {ui_color}")
|
||||
if voice_changed:
|
||||
self._log.append_log(f"SYS: Voice set — {voice}")
|
||||
except Exception as e:
|
||||
self._log.append_log(f"ERR: Config save failed — {e}")
|
||||
|
||||
if voice_changed and self.on_voice_change:
|
||||
self.on_voice_change()
|
||||
|
||||
def _centre_overlay(self, ov) -> None:
|
||||
"""Place a floating overlay in the middle of the HUD and show it."""
|
||||
cw = self.centralWidget()
|
||||
ov.adjustSize()
|
||||
ov.setGeometry(
|
||||
max(0, (cw.width() - ov.width()) // 2),
|
||||
max(0, (cw.height() - ov.height()) // 2),
|
||||
ov.width(), ov.height(),
|
||||
)
|
||||
ov.show()
|
||||
ov.raise_()
|
||||
|
||||
# ── Audio devices ────────────────────────────────────────────────────────
|
||||
|
||||
def _open_audio_devices(self):
|
||||
ov = AudioDeviceOverlay(parent=self.centralWidget())
|
||||
ov.picked.connect(self._on_audio_devices_applied)
|
||||
self._centre_overlay(ov)
|
||||
self._audio_overlay = ov # keep a reference so it isn't GC'd
|
||||
|
||||
def _on_audio_devices_applied(self):
|
||||
self._log.append_log("SYS: Audio devices updated.")
|
||||
if self.on_audio_device_change:
|
||||
self.on_audio_device_change()
|
||||
|
||||
# ── Memory panel ─────────────────────────────────────────────────────────
|
||||
|
||||
def _open_memory_panel(self):
|
||||
ov = MemoryOverlay(parent=self.centralWidget())
|
||||
self._centre_overlay(ov)
|
||||
self._memory_overlay = ov
|
||||
|
||||
# ── Irreversible-action confirmation ─────────────────────────────────────
|
||||
|
||||
def _show_confirm_banner(self, title: str, detail: str):
|
||||
self._hide_confirm_banner()
|
||||
ov = ConfirmBanner(title, detail, parent=self.centralWidget())
|
||||
ov.answered.connect(self._on_confirm_answered)
|
||||
self._centre_overlay(ov)
|
||||
self._confirm_overlay = ov
|
||||
|
||||
def _hide_confirm_banner(self):
|
||||
ov = getattr(self, "_confirm_overlay", None)
|
||||
if ov is not None:
|
||||
ov.hide()
|
||||
ov.deleteLater()
|
||||
self._confirm_overlay = None
|
||||
|
||||
def _on_confirm_answered(self, accepted: bool):
|
||||
# Tear the banner down first: core.confirm.resolve() may be about to
|
||||
# shut the machine down, and a live widget mid-callback is not where you
|
||||
# want to be when that happens.
|
||||
self._hide_confirm_banner()
|
||||
try:
|
||||
from core.confirm import resolve
|
||||
resolve(bool(accepted))
|
||||
except Exception as e:
|
||||
self._log.append_log(f"ERR: Confirmation failed — {e}")
|
||||
|
||||
def _open_plugin_manager(self):
|
||||
plugins = self.get_plugins() if self.get_plugins else []
|
||||
cw = self.centralWidget()
|
||||
@@ -3427,6 +4064,31 @@ class JarvisUI:
|
||||
def on_interrupt(self, cb):
|
||||
self._win.on_interrupt = cb
|
||||
|
||||
@property
|
||||
def on_voice_change(self):
|
||||
return self._win.on_voice_change
|
||||
|
||||
@on_voice_change.setter
|
||||
def on_voice_change(self, cb):
|
||||
self._win.on_voice_change = cb
|
||||
|
||||
@property
|
||||
def on_audio_device_change(self):
|
||||
return self._win.on_audio_device_change
|
||||
|
||||
@on_audio_device_change.setter
|
||||
def on_audio_device_change(self, cb):
|
||||
self._win.on_audio_device_change = cb
|
||||
|
||||
def show_confirm(self, title: str, detail: str) -> None:
|
||||
"""Thread-safe: raise the irreversible-action gate. Called from action
|
||||
handlers running in executor threads, so it goes through a signal."""
|
||||
self._win._confirm_sig.emit(str(title)[:120], str(detail)[:300])
|
||||
|
||||
def hide_confirm(self) -> None:
|
||||
"""Thread-safe: take the gate down."""
|
||||
self._win._confirm_hide_sig.emit()
|
||||
|
||||
@property
|
||||
def get_plugins(self):
|
||||
return self._win.get_plugins
|
||||
@@ -3435,6 +4097,15 @@ class JarvisUI:
|
||||
def get_plugins(self, cb):
|
||||
self._win.get_plugins = cb
|
||||
|
||||
def set_audio_level(self, level: float) -> None:
|
||||
"""Thread-safe: feed a 0.0–1.0 live audio level to the HUD waveform.
|
||||
Called from the audio threads; a plain float store is atomic under the
|
||||
GIL, so no signal/lock is needed for this cosmetic value."""
|
||||
try:
|
||||
self._win.hud.set_audio_level(level)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def notify_phone_connected(self) -> None:
|
||||
self._win.notify_phone_connected()
|
||||
|
||||
|
||||
Reference in new issue
Block a user