Add files via upload
This commit is contained in:
1 parent
ff7a982d82
commit
c36079104e
17 files changed
+2854
-1210
No files matched your search
@@ -1,15 +1,3 @@
|
||||
"""
|
||||
browser_control.py — JARVIS Browser Action Module
|
||||
====================================================
|
||||
Desteklenen tarayıcılar : chrome, edge, firefox, opera, operagx, brave, vivaldi, safari (macOS)
|
||||
Desteklenen platformlar : Windows · macOS · Linux
|
||||
|
||||
Her tarayıcı için bağımsız _BrowserSession örneği tutulur.
|
||||
"switch" action ile aktif tarayıcı değiştirilebilir.
|
||||
|
||||
NOT: Tüm tarayıcılar kullanıcının gerçek profiliyle açılır (hesaplar, çerezler korunur).
|
||||
Gizli/incognito mod kullanılmaz.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -30,12 +18,8 @@ from playwright.async_api import (
|
||||
Playwright,
|
||||
TimeoutError as PlaywrightTimeout,
|
||||
)
|
||||
|
||||
# ── Platform ──────────────────────────────────────────────────────────────────
|
||||
_OS = platform.system() # "Windows" | "Darwin" | "Linux"
|
||||
|
||||
|
||||
# ── URL normalizer ────────────────────────────────────────────────────────────
|
||||
def _normalize_url(url: str) -> str:
|
||||
"""
|
||||
Bare words like "instagram" → "https://instagram.com"
|
||||
@@ -53,7 +37,6 @@ def _normalize_url(url: str) -> str:
|
||||
return "https://" + url
|
||||
|
||||
|
||||
# ── User-Agent ────────────────────────────────────────────────────────────────
|
||||
def _user_agent() -> str:
|
||||
if _OS == "Windows":
|
||||
return (
|
||||
@@ -74,7 +57,6 @@ def _user_agent() -> str:
|
||||
)
|
||||
|
||||
|
||||
# ── Gerçek kullanıcı profil dizini (tüm tarayıcılar) ─────────────────────────
|
||||
def _real_profile_dir(browser: str) -> str:
|
||||
home = Path.home()
|
||||
local = os.environ.get("LOCALAPPDATA", "")
|
||||
@@ -129,8 +111,6 @@ def _real_profile_dir(browser: str) -> str:
|
||||
print(f"[Browser] ⚠️ Real profile not found for {browser}, using: {fallback}")
|
||||
return str(fallback)
|
||||
|
||||
|
||||
# ── Firefox gerçek profil dizini ──────────────────────────────────────────────
|
||||
def _firefox_profile_dir() -> Optional[str]:
|
||||
home = Path.home()
|
||||
|
||||
@@ -170,8 +150,6 @@ def _firefox_profile_dir() -> Optional[str]:
|
||||
return default_path
|
||||
return None
|
||||
|
||||
|
||||
# ── Windows: Opera exe tespiti ────────────────────────────────────────────────
|
||||
def _find_opera_windows() -> Optional[str]:
|
||||
local = os.environ.get("LOCALAPPDATA", "")
|
||||
prog = os.environ.get("PROGRAMFILES", "")
|
||||
@@ -213,8 +191,6 @@ def _find_opera_windows() -> Optional[str]:
|
||||
|
||||
return shutil.which("opera") or None
|
||||
|
||||
|
||||
# ── Genel exe arama (Windows registry) ───────────────────────────────────────
|
||||
def _find_exe_windows(prog_name: str) -> Optional[str]:
|
||||
try:
|
||||
import winreg
|
||||
@@ -237,8 +213,6 @@ def _find_exe_windows(prog_name: str) -> Optional[str]:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
# ── Tarayıcı tanım tablosu ────────────────────────────────────────────────────
|
||||
_BROWSER_SPECS: dict[str, dict] = {
|
||||
"Windows": {
|
||||
"chrome": {"engine": "chromium", "channel": "chrome", "bins": []},
|
||||
@@ -370,8 +344,6 @@ def _detect_default_browser() -> str:
|
||||
return "chrome"
|
||||
|
||||
|
||||
# ── Tek tarayıcı oturumu ──────────────────────────────────────────────────────
|
||||
|
||||
class _BrowserSession:
|
||||
"""
|
||||
Bir tarayıcı örneği için tam oturum.
|
||||
@@ -390,8 +362,6 @@ class _BrowserSession:
|
||||
self._context: BrowserContext | None = None
|
||||
self._page: Page | None = None
|
||||
|
||||
# ── Yaşam döngüsü ─────────────────────────────────────────────────────────
|
||||
|
||||
def start(self):
|
||||
if self._thread and self._thread.is_alive():
|
||||
return
|
||||
@@ -436,8 +406,6 @@ class _BrowserSession:
|
||||
pass
|
||||
self._context = self._page = None
|
||||
|
||||
# ── Tarayıcı başlatma ──────────────────────────────────────────────────────
|
||||
|
||||
async def _launch(self):
|
||||
"""
|
||||
Tarayıcıyı gerçek kullanıcı profiliyle başlatır.
|
||||
@@ -456,7 +424,6 @@ class _BrowserSession:
|
||||
channel = self._spec["channel"]
|
||||
engine_obj = getattr(self._pw, engine_name)
|
||||
|
||||
# ── Firefox ───────────────────────────────────────────────────────────
|
||||
if engine_name == "firefox":
|
||||
profile = _firefox_profile_dir() or str(
|
||||
Path.home() / ".jarvis_profiles" / "firefox"
|
||||
@@ -477,14 +444,11 @@ class _BrowserSession:
|
||||
Path(jarvis).mkdir(parents=True, exist_ok=True)
|
||||
self._context = await engine_obj.launch_persistent_context(jarvis, **kwargs)
|
||||
|
||||
await asyncio.sleep(0.5) # let the browser settle
|
||||
# Always open a dedicated JARVIS tab instead of grabbing pages[0],
|
||||
# which may already be about:blank or a stale restored tab.
|
||||
await asyncio.sleep(0.5)
|
||||
self._page = await self._context.new_page()
|
||||
print(f"[Browser] ✅ Firefox launched")
|
||||
return
|
||||
|
||||
# ── Safari (webkit) ───────────────────────────────────────────────────
|
||||
if engine_name == "webkit":
|
||||
safari_profile = str(Path.home() / ".jarvis_profiles" / "safari")
|
||||
Path(safari_profile).mkdir(parents=True, exist_ok=True)
|
||||
@@ -500,7 +464,6 @@ class _BrowserSession:
|
||||
print(f"[Browser] ✅ Safari launched")
|
||||
return
|
||||
|
||||
# ── Chromium tabanlı ──────────────────────────────────────────────────
|
||||
profile = _real_profile_dir(self.browser_name)
|
||||
|
||||
kwargs = {
|
||||
@@ -528,17 +491,15 @@ class _BrowserSession:
|
||||
+ (f" @ {exe}" if exe else "")
|
||||
)
|
||||
|
||||
# 1) Gerçek profil
|
||||
try:
|
||||
self._context = await engine_obj.launch_persistent_context(profile, **kwargs)
|
||||
await asyncio.sleep(0.5) # let the browser settle before any navigation
|
||||
await asyncio.sleep(0.5)
|
||||
self._page = await self._context.new_page()
|
||||
print(f"[Browser] ✅ Launched [{label}] profile={profile}")
|
||||
return
|
||||
except Exception as e:
|
||||
print(f"[Browser] ⚠️ Real profile failed for {label}: {e}")
|
||||
|
||||
# 2) JARVIS fallback profili
|
||||
jarvis_profile = str(Path.home() / ".jarvis_profiles" / self.browser_name)
|
||||
Path(jarvis_profile).mkdir(parents=True, exist_ok=True)
|
||||
print(f"[Browser] Retrying with JARVIS profile: {jarvis_profile}")
|
||||
@@ -551,7 +512,6 @@ class _BrowserSession:
|
||||
except Exception as e2:
|
||||
raise RuntimeError(f"Could not launch {self.browser_name}: {e2}") from e2
|
||||
|
||||
# ── Sayfa erişimi ─────────────────────────────────────────────────────────
|
||||
|
||||
async def _get_page(self) -> Page:
|
||||
await self._launch()
|
||||
@@ -561,18 +521,8 @@ class _BrowserSession:
|
||||
await asyncio.sleep(0.2)
|
||||
return self._page
|
||||
|
||||
# ── Action'lar ────────────────────────────────────────────────────────────
|
||||
|
||||
async def go_to(self, url: str) -> str:
|
||||
"""
|
||||
Navigate to a URL.
|
||||
|
||||
• Uses a 30-second timeout to handle heavy sites like YouTube.
|
||||
• On PlaywrightTimeout we check whether the URL actually changed;
|
||||
if it did, treat it as success so JARVIS won't retry endlessly.
|
||||
• If the page is still about:blank after navigation, open a fresh tab
|
||||
and try once more (covers stale-page edge cases in Opera GX etc.).
|
||||
"""
|
||||
url = _normalize_url(url)
|
||||
page = await self._get_page()
|
||||
prev_url = page.url
|
||||
@@ -590,7 +540,6 @@ class _BrowserSession:
|
||||
|
||||
result_url = await _do_goto(page)
|
||||
|
||||
# If page didn't move away from blank, retry on a brand-new tab
|
||||
if result_url in ("about:blank", "", None, prev_url) and prev_url in ("about:blank", "", None):
|
||||
print(f"[Browser] Still blank after goto — retrying on new tab: {url}")
|
||||
try:
|
||||
@@ -785,9 +734,6 @@ class _BrowserSession:
|
||||
await self._async_close()
|
||||
return f"{self.browser_name} closed."
|
||||
|
||||
|
||||
# ── Session registry ──────────────────────────────────────────────────────────
|
||||
|
||||
class _SessionRegistry:
|
||||
"""Tüm aktif tarayıcı oturumlarını yönetir."""
|
||||
|
||||
@@ -855,26 +801,12 @@ class _SessionRegistry:
|
||||
|
||||
_registry = _SessionRegistry()
|
||||
|
||||
|
||||
# ── Ana fonksiyon ─────────────────────────────────────────────────────────────
|
||||
|
||||
def browser_control(
|
||||
parameters: dict = None,
|
||||
response=None,
|
||||
player=None,
|
||||
session_memory=None,
|
||||
) -> str:
|
||||
"""
|
||||
Desteklenen action'lar:
|
||||
go_to, search, click, type, scroll, fill_form,
|
||||
smart_click, smart_type, get_text, get_url, press,
|
||||
new_tab, close_tab, screenshot, back, forward, reload,
|
||||
switch, list_browsers, close, close_all
|
||||
|
||||
Parametre:
|
||||
browser : chrome | edge | firefox | opera | operagx | brave | vivaldi | safari
|
||||
(belirtilmezse aktif tarayıcı kullanılır)
|
||||
"""
|
||||
params = parameters or {}
|
||||
action = params.get("action", "").lower().strip()
|
||||
browser = params.get("browser", "").lower().strip() or None
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
#computer_control.py
|
||||
import io
|
||||
import json
|
||||
import re
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
#computer_settings.py
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
#desktop.py
|
||||
import os
|
||||
import sys
|
||||
import json
|
||||
|
||||
@@ -0,0 +1,833 @@
|
||||
"""
|
||||
file_processor.py — JARVIS Universal File Processor
|
||||
|
||||
Supported types:
|
||||
image → describe, ocr, resize, convert, compress, crop
|
||||
pdf → summarize, extract_text, extract_pages, to_word
|
||||
docx → summarize, extract_text, reformat, translate_hint
|
||||
txt/md → summarize, reformat, translate_hint, word_count
|
||||
csv → analyze, filter, sort, convert, stats
|
||||
xlsx → analyze, filter, convert, stats
|
||||
json → validate, format, extract, convert
|
||||
code → explain, review, fix, run, document
|
||||
audio → transcribe, trim, convert, info
|
||||
video → trim, extract_audio, extract_frame, info, compress
|
||||
zip → list, extract
|
||||
pptx → summarize, extract_text, to_pdf
|
||||
"""
|
||||
|
||||
import os
|
||||
import re
|
||||
import json
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from datetime import datetime
|
||||
|
||||
import google.generativeai as genai
|
||||
|
||||
|
||||
def _get_api_key() -> str:
|
||||
config_path = Path(__file__).resolve().parent.parent / "config" / "api_keys.json"
|
||||
with open(config_path, "r", encoding="utf-8") as f:
|
||||
return json.load(f)["gemini_api_key"]
|
||||
|
||||
|
||||
def _gemini_client():
|
||||
genai.configure(api_key=_get_api_key())
|
||||
return genai.GenerativeModel("gemini-2.5-flash")
|
||||
|
||||
|
||||
def _detect_type(path: Path) -> str:
|
||||
ext = path.suffix.lower().lstrip(".")
|
||||
image_exts = {"jpg", "jpeg", "png", "gif", "webp", "bmp", "tiff", "svg", "ico"}
|
||||
video_exts = {"mp4", "avi", "mov", "mkv", "wmv", "flv", "webm", "m4v", "3gp"}
|
||||
audio_exts = {"mp3", "wav", "ogg", "m4a", "aac", "flac", "wma", "opus"}
|
||||
code_exts = {"py", "js", "ts", "jsx", "tsx", "html", "css", "java", "c",
|
||||
"cpp", "cs", "go", "rs", "rb", "php", "swift", "kt", "sh",
|
||||
"bash", "ps1", "lua", "r", "m", "sql", "yaml", "toml"}
|
||||
archive_exts = {"zip", "rar", "tar", "gz", "7z", "bz2", "xz"}
|
||||
|
||||
if ext in image_exts: return "image"
|
||||
if ext in video_exts: return "video"
|
||||
if ext in audio_exts: return "audio"
|
||||
if ext in code_exts: return "code"
|
||||
if ext in archive_exts: return "archive"
|
||||
if ext == "pdf": return "pdf"
|
||||
if ext in ("docx", "doc"): return "docx"
|
||||
if ext in ("txt", "md", "rst", "log"): return "text"
|
||||
if ext in ("csv", "tsv"): return "csv"
|
||||
if ext in ("xlsx", "xls", "ods"): return "excel"
|
||||
if ext == "json": return "json"
|
||||
if ext == "xml": return "xml"
|
||||
if ext in ("pptx", "ppt"): return "pptx"
|
||||
return "unknown"
|
||||
|
||||
|
||||
def _file_size_str(path: Path) -> str:
|
||||
size = path.stat().st_size
|
||||
if size < 1024: return f"{size} B"
|
||||
if size < 1024**2: return f"{size/1024:.1f} KB"
|
||||
if size < 1024**3: return f"{size/1024**2:.1f} MB"
|
||||
return f"{size/1024**3:.1f} GB"
|
||||
|
||||
def _output_path(src: Path, suffix: str, new_ext: str = None) -> Path:
|
||||
ext = new_ext or src.suffix
|
||||
name = f"{src.stem}_{suffix}{ext}"
|
||||
return src.parent / name
|
||||
|
||||
def _process_image(path: Path, action: str, params: dict, speak=None) -> str:
|
||||
try:
|
||||
from PIL import Image
|
||||
except ImportError:
|
||||
return "Pillow is not installed. Run: pip install Pillow"
|
||||
|
||||
action = action or "describe"
|
||||
|
||||
if action in ("describe", "ocr", "analyze", "read", "extract_text"):
|
||||
try:
|
||||
model = _gemini_client()
|
||||
img = Image.open(path)
|
||||
prompt = {
|
||||
"describe": "Describe this image in detail.",
|
||||
"ocr": "Extract all text visible in this image. Return only the text, formatted clearly.",
|
||||
"analyze": "Analyze this image thoroughly: objects, colors, composition, any text, context.",
|
||||
"read": "Read all text in this image, preserving structure and formatting.",
|
||||
"extract_text": "Extract all text from this image.",
|
||||
}.get(action, "Describe this image.")
|
||||
|
||||
if params.get("instruction"):
|
||||
prompt = params["instruction"]
|
||||
|
||||
response = model.generate_content([prompt, img])
|
||||
result = response.text.strip()
|
||||
|
||||
if len(result) > 500 and params.get("save", True):
|
||||
out = _output_path(path, "result", ".txt")
|
||||
out.write_text(result, encoding="utf-8")
|
||||
return f"{result[:300]}...\n\nFull result saved to: {out}"
|
||||
return result
|
||||
except Exception as e:
|
||||
return f"AI image analysis failed: {e}"
|
||||
|
||||
if action == "resize":
|
||||
width = int(params.get("width", 0))
|
||||
height = int(params.get("height", 0))
|
||||
scale = float(params.get("scale", 0))
|
||||
try:
|
||||
img = Image.open(path)
|
||||
w, h = img.size
|
||||
if scale:
|
||||
new_size = (int(w * scale), int(h * scale))
|
||||
elif width and height:
|
||||
new_size = (width, height)
|
||||
elif width:
|
||||
new_size = (width, int(h * width / w))
|
||||
elif height:
|
||||
new_size = (int(w * height / h), height)
|
||||
else:
|
||||
return "Please specify width, height, or scale."
|
||||
out = _output_path(path, f"resized_{new_size[0]}x{new_size[1]}")
|
||||
img.resize(new_size, Image.LANCZOS).save(out)
|
||||
return f"Resized from {w}x{h} to {new_size[0]}x{new_size[1]}. Saved: {out.name}"
|
||||
except Exception as e:
|
||||
return f"Resize failed: {e}"
|
||||
|
||||
if action == "convert":
|
||||
fmt = params.get("format", "png").lower().strip(".")
|
||||
fmt_map = {"jpg": "JPEG", "jpeg": "JPEG", "png": "PNG",
|
||||
"webp": "WEBP", "bmp": "BMP", "tiff": "TIFF"}
|
||||
pil_fmt = fmt_map.get(fmt, fmt.upper())
|
||||
try:
|
||||
img = Image.open(path).convert("RGB") if fmt == "jpg" else Image.open(path)
|
||||
out = _output_path(path, "converted", f".{fmt}")
|
||||
img.save(out, pil_fmt)
|
||||
return f"Converted to {fmt.upper()}. Saved: {out.name}"
|
||||
except Exception as e:
|
||||
return f"Convert failed: {e}"
|
||||
|
||||
if action == "compress":
|
||||
quality = int(params.get("quality", 70))
|
||||
try:
|
||||
img = Image.open(path).convert("RGB")
|
||||
out = _output_path(path, f"compressed_q{quality}", ".jpg")
|
||||
img.save(out, "JPEG", quality=quality, optimize=True)
|
||||
before = _file_size_str(path)
|
||||
after = _file_size_str(out)
|
||||
return f"Compressed: {before} → {after}. Saved: {out.name}"
|
||||
except Exception as e:
|
||||
return f"Compress failed: {e}"
|
||||
|
||||
if action == "info":
|
||||
try:
|
||||
img = Image.open(path)
|
||||
return (f"Image info: {img.format}, {img.size[0]}x{img.size[1]}px, "
|
||||
f"mode: {img.mode}, size: {_file_size_str(path)}")
|
||||
except Exception as e:
|
||||
return f"Info failed: {e}"
|
||||
|
||||
return _process_image(path, "describe", {"instruction": f"{action}: {params}"})
|
||||
|
||||
def _process_pdf(path: Path, action: str, params: dict, speak=None) -> str:
|
||||
action = action or "summarize"
|
||||
|
||||
def _extract_pdf_text(max_chars=50000) -> str:
|
||||
text = ""
|
||||
try:
|
||||
import pdfplumber
|
||||
with pdfplumber.open(path) as pdf:
|
||||
for page in pdf.pages:
|
||||
text += (page.extract_text() or "") + "\n"
|
||||
except ImportError:
|
||||
try:
|
||||
import PyPDF2
|
||||
with open(path, "rb") as f:
|
||||
reader = PyPDF2.PdfReader(f)
|
||||
for page in reader.pages:
|
||||
text += page.extract_text() + "\n"
|
||||
except ImportError:
|
||||
return ""
|
||||
return text[:max_chars]
|
||||
|
||||
if action in ("summarize", "extract_text", "translate_hint", "analyze", "reformat"):
|
||||
text = _extract_pdf_text()
|
||||
if not text.strip():
|
||||
return "Could not extract text from PDF (may be scanned/image-based)."
|
||||
|
||||
if action == "extract_text":
|
||||
out = _output_path(path, "text", ".txt")
|
||||
out.write_text(text, encoding="utf-8")
|
||||
return f"Text extracted ({len(text)} chars). Saved: {out.name}"
|
||||
|
||||
prompt_map = {
|
||||
"summarize": f"Summarize this PDF document concisely:\n\n{text}",
|
||||
"analyze": f"Analyze this document thoroughly:\n\n{text}",
|
||||
"translate_hint": f"What language is this document in and what does it say? Summarize:\n\n{text}",
|
||||
"reformat": f"Reformat this text cleanly with proper structure:\n\n{text}",
|
||||
}
|
||||
try:
|
||||
model = _gemini_client()
|
||||
response = model.generate_content(prompt_map.get(action, f"Analyze:\n\n{text}"))
|
||||
result = response.text.strip()
|
||||
if len(result) > 600 and params.get("save", True):
|
||||
out = _output_path(path, action, ".txt")
|
||||
out.write_text(result, encoding="utf-8")
|
||||
return f"{result[:400]}...\n\nFull result saved: {out.name}"
|
||||
return result
|
||||
except Exception as e:
|
||||
return f"AI analysis failed: {e}"
|
||||
|
||||
if action == "info":
|
||||
try:
|
||||
import pdfplumber
|
||||
with pdfplumber.open(path) as pdf:
|
||||
pages = len(pdf.pages)
|
||||
return f"PDF: {pages} pages, size: {_file_size_str(path)}"
|
||||
except Exception:
|
||||
return f"PDF size: {_file_size_str(path)}"
|
||||
|
||||
if action == "to_word":
|
||||
text = _extract_pdf_text()
|
||||
if not text:
|
||||
return "Could not extract text to convert."
|
||||
try:
|
||||
from docx import Document
|
||||
doc = Document()
|
||||
doc.add_heading(path.stem, 0)
|
||||
for para in text.split("\n\n"):
|
||||
if para.strip():
|
||||
doc.add_paragraph(para.strip())
|
||||
out = _output_path(path, "converted", ".docx")
|
||||
doc.save(out)
|
||||
return f"Converted to Word document. Saved: {out.name}"
|
||||
except ImportError:
|
||||
return "python-docx not installed. Run: pip install python-docx"
|
||||
|
||||
return f"Unknown PDF action: '{action}'. Try: summarize, extract_text, info, to_word"
|
||||
|
||||
def _process_text_doc(path: Path, file_type: str, action: str,
|
||||
params: dict, speak=None) -> str:
|
||||
action = action or "summarize"
|
||||
|
||||
def _read_content() -> str:
|
||||
if file_type == "docx":
|
||||
try:
|
||||
from docx import Document
|
||||
doc = Document(path)
|
||||
return "\n".join(p.text for p in doc.paragraphs)
|
||||
except ImportError:
|
||||
return "python-docx not installed."
|
||||
except Exception as e:
|
||||
return f"Read failed: {e}"
|
||||
else:
|
||||
return path.read_text(encoding="utf-8", errors="ignore")
|
||||
|
||||
content = _read_content()
|
||||
if not content.strip():
|
||||
return "File appears to be empty."
|
||||
|
||||
if action == "word_count":
|
||||
words = len(content.split())
|
||||
chars = len(content)
|
||||
lines = content.count("\n")
|
||||
return f"Word count: {words} words, {chars} characters, {lines} lines."
|
||||
|
||||
if action == "extract_text":
|
||||
if file_type != "txt":
|
||||
out = _output_path(path, "extracted", ".txt")
|
||||
out.write_text(content, encoding="utf-8")
|
||||
return f"Text extracted. Saved: {out.name}"
|
||||
return content[:2000]
|
||||
|
||||
instruction = params.get("instruction", "")
|
||||
prompt_map = {
|
||||
"summarize": f"Summarize this document concisely:\n\n{content[:40000]}",
|
||||
"analyze": f"Analyze this document:\n\n{content[:40000]}",
|
||||
"reformat": f"Reformat this text with clean structure, proper headings and paragraphs:\n\n{content[:40000]}",
|
||||
"fix": f"Fix grammar, spelling and style issues in this text:\n\n{content[:40000]}",
|
||||
"translate_hint": f"What language is this and what does it say? Summarize:\n\n{content[:10000]}",
|
||||
"to_bullet": f"Convert this text into a clear bullet-point summary:\n\n{content[:40000]}",
|
||||
"custom": f"{instruction}\n\n{content[:40000]}",
|
||||
}
|
||||
|
||||
if action not in prompt_map:
|
||||
|
||||
action = "custom"
|
||||
instruction = action
|
||||
|
||||
try:
|
||||
model = _gemini_client()
|
||||
response = model.generate_content(prompt_map[action])
|
||||
result = response.text.strip()
|
||||
if len(result) > 600 and params.get("save", True):
|
||||
out = _output_path(path, action, ".txt")
|
||||
out.write_text(result, encoding="utf-8")
|
||||
return f"{result[:400]}...\n\nFull result saved: {out.name}"
|
||||
return result
|
||||
except Exception as e:
|
||||
return f"AI processing failed: {e}"
|
||||
|
||||
|
||||
def _process_data(path: Path, file_type: str, action: str,
|
||||
params: dict, speak=None) -> str:
|
||||
try:
|
||||
import pandas as pd
|
||||
except ImportError:
|
||||
return "pandas not installed. Run: pip install pandas openpyxl"
|
||||
|
||||
action = action or "analyze"
|
||||
|
||||
try:
|
||||
if file_type == "csv":
|
||||
df = pd.read_csv(path, encoding="utf-8", errors="replace")
|
||||
else:
|
||||
df = pd.read_excel(path)
|
||||
except Exception as e:
|
||||
return f"Could not read file: {e}"
|
||||
|
||||
if action == "info":
|
||||
return (f"Rows: {len(df)}, Columns: {len(df.columns)}\n"
|
||||
f"Columns: {', '.join(df.columns.tolist())}\n"
|
||||
f"Size: {_file_size_str(path)}")
|
||||
|
||||
if action == "stats":
|
||||
try:
|
||||
desc = df.describe(include="all").to_string()
|
||||
return f"Statistics:\n{desc[:2000]}"
|
||||
except Exception as e:
|
||||
return f"Stats failed: {e}"
|
||||
|
||||
if action == "analyze":
|
||||
preview = df.head(50).to_string()
|
||||
prompt = (f"Analyze this dataset. Columns: {list(df.columns)}\n"
|
||||
f"Rows: {len(df)}\nPreview:\n{preview}\n\n"
|
||||
f"Give insights, patterns, and notable findings.")
|
||||
try:
|
||||
model = _gemini_client()
|
||||
response = model.generate_content(prompt)
|
||||
return response.text.strip()
|
||||
except Exception as e:
|
||||
return f"AI analysis failed: {e}"
|
||||
|
||||
if action in ("convert", "to_csv", "to_excel", "to_json"):
|
||||
fmt = {"to_csv": "csv", "to_excel": "xlsx", "to_json": "json",
|
||||
"convert": params.get("format", "csv")}.get(action, "csv")
|
||||
try:
|
||||
if fmt == "csv":
|
||||
out = _output_path(path, "converted", ".csv")
|
||||
df.to_csv(out, index=False, encoding="utf-8")
|
||||
elif fmt == "xlsx":
|
||||
out = _output_path(path, "converted", ".xlsx")
|
||||
df.to_excel(out, index=False)
|
||||
elif fmt == "json":
|
||||
out = _output_path(path, "converted", ".json")
|
||||
df.to_json(out, orient="records", force_ascii=False, indent=2)
|
||||
return f"Converted to {fmt.upper()}. Saved: {out.name}"
|
||||
except Exception as e:
|
||||
return f"Convert failed: {e}"
|
||||
|
||||
if action == "filter":
|
||||
col = params.get("column", "")
|
||||
value = params.get("value", "")
|
||||
condition = params.get("condition", "equals")
|
||||
if not col or col not in df.columns:
|
||||
return f"Column '{col}' not found. Available: {', '.join(df.columns)}"
|
||||
try:
|
||||
if condition == "equals": filtered = df[df[col] == value]
|
||||
elif condition == "contains": filtered = df[df[col].astype(str).str.contains(str(value), case=False)]
|
||||
elif condition == "gt": filtered = df[df[col] > float(value)]
|
||||
elif condition == "lt": filtered = df[df[col] < float(value)]
|
||||
else: filtered = df[df[col] == value]
|
||||
out = _output_path(path, "filtered", ".csv")
|
||||
filtered.to_csv(out, index=False)
|
||||
return f"Filtered: {len(filtered)} rows match. Saved: {out.name}"
|
||||
except Exception as e:
|
||||
return f"Filter failed: {e}"
|
||||
|
||||
if action == "sort":
|
||||
col = params.get("column", df.columns[0])
|
||||
asc = params.get("ascending", True)
|
||||
try:
|
||||
sorted_df = df.sort_values(col, ascending=asc)
|
||||
out = _output_path(path, "sorted", path.suffix)
|
||||
sorted_df.to_csv(out, index=False)
|
||||
return f"Sorted by '{col}'. Saved: {out.name}"
|
||||
except Exception as e:
|
||||
return f"Sort failed: {e}"
|
||||
|
||||
preview = df.head(30).to_string()
|
||||
try:
|
||||
model = _gemini_client()
|
||||
response = model.generate_content(
|
||||
f"Task: {action}\nDataset ({len(df)} rows, cols: {list(df.columns)}):\n{preview}"
|
||||
)
|
||||
return response.text.strip()
|
||||
except Exception as e:
|
||||
return f"Processing failed: {e}"
|
||||
|
||||
|
||||
def _process_json(path: Path, action: str, params: dict, speak=None) -> str:
|
||||
action = action or "analyze"
|
||||
try:
|
||||
content = path.read_text(encoding="utf-8")
|
||||
data = json.loads(content)
|
||||
except Exception as e:
|
||||
return f"Invalid JSON: {e}"
|
||||
|
||||
if action == "validate":
|
||||
return f"Valid JSON. Type: {type(data).__name__}, size: {_file_size_str(path)}"
|
||||
|
||||
if action == "format":
|
||||
out = _output_path(path, "formatted", ".json")
|
||||
out.write_text(json.dumps(data, indent=2, ensure_ascii=False), encoding="utf-8")
|
||||
return f"Formatted JSON saved: {out.name}"
|
||||
|
||||
if action in ("analyze", "summarize", "extract"):
|
||||
preview = json.dumps(data, indent=2, ensure_ascii=False)[:8000]
|
||||
prompt = f"Task: {action} this JSON data:\n{preview}"
|
||||
if params.get("instruction"):
|
||||
prompt = f"{params['instruction']}\n\nJSON data:\n{preview}"
|
||||
try:
|
||||
model = _gemini_client()
|
||||
response = model.generate_content(prompt)
|
||||
return response.text.strip()
|
||||
except Exception as e:
|
||||
return f"AI processing failed: {e}"
|
||||
|
||||
if action == "to_csv":
|
||||
try:
|
||||
import pandas as pd
|
||||
if isinstance(data, list):
|
||||
df = pd.DataFrame(data)
|
||||
out = _output_path(path, "converted", ".csv")
|
||||
df.to_csv(out, index=False)
|
||||
return f"Converted to CSV. Saved: {out.name}"
|
||||
return "JSON must be an array of objects to convert to CSV."
|
||||
except ImportError:
|
||||
return "pandas not installed."
|
||||
|
||||
return _process_json(path, "analyze", {"instruction": action})
|
||||
|
||||
def _process_code(path: Path, action: str, params: dict, speak=None) -> str:
|
||||
action = action or "explain"
|
||||
content = path.read_text(encoding="utf-8", errors="ignore")
|
||||
ext = path.suffix.lstrip(".")
|
||||
|
||||
if action == "run":
|
||||
if ext == "py":
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["python", str(path)],
|
||||
capture_output=True, text=True, timeout=30
|
||||
)
|
||||
out = result.stdout or result.stderr
|
||||
return f"Output:\n{out[:2000]}" if out else "No output."
|
||||
except subprocess.TimeoutExpired:
|
||||
return "Execution timed out (30s)."
|
||||
except Exception as e:
|
||||
return f"Run failed: {e}"
|
||||
return f"Direct execution not supported for .{ext} files."
|
||||
|
||||
if action == "info":
|
||||
lines = content.count("\n")
|
||||
words = len(content.split())
|
||||
return f"Code file: {lines} lines, {words} words, {_file_size_str(path)}"
|
||||
|
||||
prompt_map = {
|
||||
"explain": f"Explain this {ext} code clearly:\n\n```{ext}\n{content[:30000]}\n```",
|
||||
"review": f"Review this {ext} code for bugs, issues, and improvements:\n\n```{ext}\n{content[:30000]}\n```",
|
||||
"fix": f"Fix any bugs in this {ext} code and return the corrected version:\n\n```{ext}\n{content[:30000]}\n```",
|
||||
"optimize": f"Optimize this {ext} code for performance and readability:\n\n```{ext}\n{content[:30000]}\n```",
|
||||
"document": f"Add proper documentation/comments to this {ext} code:\n\n```{ext}\n{content[:30000]}\n```",
|
||||
"summarize": f"Summarize what this {ext} code does:\n\n```{ext}\n{content[:30000]}\n```",
|
||||
"test": f"Write unit tests for this {ext} code:\n\n```{ext}\n{content[:30000]}\n```",
|
||||
}
|
||||
|
||||
instruction = params.get("instruction", "")
|
||||
if action not in prompt_map:
|
||||
prompt = f"{action}\n\n```{ext}\n{content[:30000]}\n```"
|
||||
if instruction:
|
||||
prompt = f"{instruction}\n\n```{ext}\n{content[:30000]}\n```"
|
||||
else:
|
||||
prompt = prompt_map[action]
|
||||
|
||||
try:
|
||||
model = _gemini_client()
|
||||
response = model.generate_content(prompt)
|
||||
result = response.text.strip()
|
||||
|
||||
if action in ("fix", "optimize", "document") and params.get("save", True):
|
||||
out = _output_path(path, action)
|
||||
code_match = re.search(r"```(?:\w+)?\n(.*?)```", result, re.DOTALL)
|
||||
code_to_save = code_match.group(1) if code_match else result
|
||||
out.write_text(code_to_save, encoding="utf-8")
|
||||
return f"{result[:400]}...\n\nSaved: {out.name}"
|
||||
return result
|
||||
except Exception as e:
|
||||
return f"AI processing failed: {e}"
|
||||
|
||||
def _process_audio(path: Path, action: str, params: dict, speak=None) -> str:
|
||||
action = action or "transcribe"
|
||||
|
||||
if action == "info":
|
||||
try:
|
||||
from pydub import AudioSegment
|
||||
audio = AudioSegment.from_file(path)
|
||||
duration = len(audio) / 1000
|
||||
mins, secs = divmod(int(duration), 60)
|
||||
return (f"Audio: {mins}m {secs}s, "
|
||||
f"{audio.channels} ch, "
|
||||
f"{audio.frame_rate}Hz, "
|
||||
f"{_file_size_str(path)}")
|
||||
except ImportError:
|
||||
return f"Audio file: {_file_size_str(path)} (install pydub for more info)"
|
||||
except Exception as e:
|
||||
return f"Info failed: {e}"
|
||||
|
||||
if action == "transcribe":
|
||||
try:
|
||||
model = _gemini_client()
|
||||
content = path.read_bytes()
|
||||
mime = {
|
||||
"mp3": "audio/mp3", "wav": "audio/wav",
|
||||
"ogg": "audio/ogg", "m4a": "audio/mp4",
|
||||
"aac": "audio/aac", "flac": "audio/flac",
|
||||
}.get(path.suffix.lstrip(".").lower(), "audio/mpeg")
|
||||
response = model.generate_content([
|
||||
"Transcribe all speech in this audio file accurately.",
|
||||
{"mime_type": mime, "data": content}
|
||||
])
|
||||
result = response.text.strip()
|
||||
if params.get("save", True):
|
||||
out = _output_path(path, "transcript", ".txt")
|
||||
out.write_text(result, encoding="utf-8")
|
||||
return f"Transcription saved: {out.name}\n\nPreview: {result[:300]}"
|
||||
return result
|
||||
except Exception as e:
|
||||
return f"Transcription failed: {e}"
|
||||
|
||||
if action == "convert":
|
||||
fmt = params.get("format", "mp3").lstrip(".")
|
||||
try:
|
||||
from pydub import AudioSegment
|
||||
audio = AudioSegment.from_file(path)
|
||||
out = _output_path(path, "converted", f".{fmt}")
|
||||
audio.export(out, format=fmt)
|
||||
return f"Converted to {fmt.upper()}. Saved: {out.name}"
|
||||
except ImportError:
|
||||
return "pydub not installed. Run: pip install pydub"
|
||||
except Exception as e:
|
||||
return f"Convert failed: {e}"
|
||||
|
||||
if action == "trim":
|
||||
start = float(params.get("start", 0))
|
||||
end = float(params.get("end", 0))
|
||||
try:
|
||||
from pydub import AudioSegment
|
||||
audio = AudioSegment.from_file(path)
|
||||
end_ms = int(end * 1000) if end else len(audio)
|
||||
trimmed = audio[int(start * 1000):end_ms]
|
||||
out = _output_path(path, f"trim_{int(start)}s_{int(end)}s")
|
||||
trimmed.export(out, format=path.suffix.lstrip("."))
|
||||
return f"Trimmed audio ({int(start)}s–{int(end)}s). Saved: {out.name}"
|
||||
except ImportError:
|
||||
return "pydub not installed."
|
||||
except Exception as e:
|
||||
return f"Trim failed: {e}"
|
||||
|
||||
return f"Unknown audio action: '{action}'. Try: transcribe, info, convert, trim"
|
||||
|
||||
def _process_video(path: Path, action: str, params: dict, speak=None) -> str:
|
||||
action = action or "info"
|
||||
|
||||
|
||||
def _ffmpeg_available() -> bool:
|
||||
try:
|
||||
subprocess.run(["ffmpeg", "-version"], capture_output=True, timeout=3)
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
if action == "info":
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["ffprobe", "-v", "quiet", "-print_format", "json",
|
||||
"-show_format", "-show_streams", str(path)],
|
||||
capture_output=True, text=True, timeout=10
|
||||
)
|
||||
data = json.loads(result.stdout)
|
||||
fmt = data.get("format", {})
|
||||
duration = float(fmt.get("duration", 0))
|
||||
mins, secs = divmod(int(duration), 60)
|
||||
size = _file_size_str(path)
|
||||
streams = data.get("streams", [])
|
||||
video_s = next((s for s in streams if s["codec_type"] == "video"), {})
|
||||
w = video_s.get("width", "?")
|
||||
h = video_s.get("height", "?")
|
||||
fps = video_s.get("r_frame_rate", "?")
|
||||
return f"Video: {mins}m {secs}s, {w}x{h}, {fps} fps, {size}"
|
||||
except Exception:
|
||||
return f"Video file: {_file_size_str(path)}"
|
||||
|
||||
if action == "extract_audio":
|
||||
if not _ffmpeg_available():
|
||||
return "ffmpeg not found. Install ffmpeg to extract audio."
|
||||
out = _output_path(path, "audio", ".mp3")
|
||||
try:
|
||||
subprocess.run(
|
||||
["ffmpeg", "-i", str(path), "-q:a", "0", "-map", "a", str(out), "-y"],
|
||||
capture_output=True, timeout=300
|
||||
)
|
||||
return f"Audio extracted. Saved: {out.name}"
|
||||
except Exception as e:
|
||||
return f"Extract audio failed: {e}"
|
||||
|
||||
if action == "trim":
|
||||
start = params.get("start", "00:00:00")
|
||||
end = params.get("end", "")
|
||||
if not _ffmpeg_available():
|
||||
return "ffmpeg not found."
|
||||
out = _output_path(path, f"trim", path.suffix)
|
||||
try:
|
||||
cmd = ["ffmpeg", "-i", str(path), "-ss", str(start)]
|
||||
if end:
|
||||
cmd += ["-to", str(end)]
|
||||
cmd += ["-c", "copy", str(out), "-y"]
|
||||
subprocess.run(cmd, capture_output=True, timeout=600)
|
||||
return f"Trimmed video saved: {out.name}"
|
||||
except Exception as e:
|
||||
return f"Trim failed: {e}"
|
||||
|
||||
if action == "extract_frame":
|
||||
timestamp = params.get("timestamp", "00:00:01")
|
||||
if not _ffmpeg_available():
|
||||
return "ffmpeg not found."
|
||||
out = _output_path(path, f"frame_{timestamp.replace(':', '')}", ".jpg")
|
||||
try:
|
||||
subprocess.run(
|
||||
["ffmpeg", "-i", str(path), "-ss", timestamp,
|
||||
"-vframes", "1", str(out), "-y"],
|
||||
capture_output=True, timeout=30
|
||||
)
|
||||
return f"Frame extracted at {timestamp}. Saved: {out.name}"
|
||||
except Exception as e:
|
||||
return f"Extract frame failed: {e}"
|
||||
|
||||
if action == "compress":
|
||||
crf = int(params.get("quality", 28))
|
||||
if not _ffmpeg_available():
|
||||
return "ffmpeg not found."
|
||||
out = _output_path(path, f"compressed_crf{crf}", ".mp4")
|
||||
try:
|
||||
subprocess.run(
|
||||
["ffmpeg", "-i", str(path),
|
||||
"-c:v", "libx264", "-crf", str(crf),
|
||||
"-preset", "medium", "-c:a", "copy",
|
||||
str(out), "-y"],
|
||||
capture_output=True, timeout=1800
|
||||
)
|
||||
before = _file_size_str(path)
|
||||
after = _file_size_str(out)
|
||||
return f"Compressed: {before} → {after}. Saved: {out.name}"
|
||||
except Exception as e:
|
||||
return f"Compress failed: {e}"
|
||||
|
||||
if action == "transcribe":
|
||||
if not _ffmpeg_available():
|
||||
return "ffmpeg not found. Needed for video transcription."
|
||||
tmp_audio = Path(tempfile.mktemp(suffix=".mp3"))
|
||||
try:
|
||||
subprocess.run(
|
||||
["ffmpeg", "-i", str(path), "-q:a", "0", "-map", "a",
|
||||
str(tmp_audio), "-y"],
|
||||
capture_output=True, timeout=300
|
||||
)
|
||||
result = _process_audio(tmp_audio, "transcribe", params, speak)
|
||||
return result
|
||||
except Exception as e:
|
||||
return f"Video transcription failed: {e}"
|
||||
finally:
|
||||
if tmp_audio.exists():
|
||||
tmp_audio.unlink()
|
||||
|
||||
if action == "convert":
|
||||
fmt = params.get("format", "mp4").lstrip(".")
|
||||
if not _ffmpeg_available():
|
||||
return "ffmpeg not found."
|
||||
out = _output_path(path, "converted", f".{fmt}")
|
||||
try:
|
||||
subprocess.run(
|
||||
["ffmpeg", "-i", str(path), str(out), "-y"],
|
||||
capture_output=True, timeout=1800
|
||||
)
|
||||
return f"Converted to {fmt.upper()}. Saved: {out.name}"
|
||||
except Exception as e:
|
||||
return f"Convert failed: {e}"
|
||||
|
||||
return f"Unknown video action: '{action}'. Try: info, trim, extract_audio, extract_frame, compress, transcribe, convert"
|
||||
|
||||
def _process_archive(path: Path, action: str, params: dict, speak=None) -> str:
|
||||
action = action or "list"
|
||||
|
||||
if action == "list":
|
||||
try:
|
||||
import zipfile, tarfile
|
||||
ext = path.suffix.lower()
|
||||
if ext == ".zip":
|
||||
with zipfile.ZipFile(path) as z:
|
||||
names = z.namelist()
|
||||
elif ext in (".tar", ".gz", ".bz2", ".xz"):
|
||||
with tarfile.open(path) as t:
|
||||
names = t.getnames()
|
||||
else:
|
||||
return f"Unsupported archive format: {ext}"
|
||||
preview = "\n".join(names[:30])
|
||||
suffix = f"\n... and {len(names)-30} more" if len(names) > 30 else ""
|
||||
return f"Archive contains {len(names)} files:\n{preview}{suffix}"
|
||||
except Exception as e:
|
||||
return f"List failed: {e}"
|
||||
|
||||
if action == "extract":
|
||||
dest = Path(params.get("destination", str(path.parent / path.stem)))
|
||||
dest.mkdir(parents=True, exist_ok=True)
|
||||
try:
|
||||
shutil.unpack_archive(path, dest)
|
||||
return f"Extracted to: {dest}"
|
||||
except Exception as e:
|
||||
return f"Extract failed: {e}"
|
||||
|
||||
return f"Unknown archive action: '{action}'. Try: list, extract"
|
||||
|
||||
def _process_pptx(path: Path, action: str, params: dict, speak=None) -> str:
|
||||
action = action or "summarize"
|
||||
|
||||
def _read_pptx_text() -> str:
|
||||
try:
|
||||
from pptx import Presentation
|
||||
prs = Presentation(path)
|
||||
text = []
|
||||
for i, slide in enumerate(prs.slides, 1):
|
||||
slide_text = f"\n--- Slide {i} ---\n"
|
||||
for shape in slide.shapes:
|
||||
if hasattr(shape, "text") and shape.text.strip():
|
||||
slide_text += shape.text.strip() + "\n"
|
||||
text.append(slide_text)
|
||||
return "\n".join(text)
|
||||
except ImportError:
|
||||
return "python-pptx not installed."
|
||||
|
||||
if action in ("summarize", "extract_text", "analyze"):
|
||||
text = _read_pptx_text()
|
||||
if action == "extract_text":
|
||||
out = _output_path(path, "text", ".txt")
|
||||
out.write_text(text, encoding="utf-8")
|
||||
return f"Text extracted. Saved: {out.name}"
|
||||
try:
|
||||
model = _gemini_client()
|
||||
prompt = f"{'Summarize' if action == 'summarize' else 'Analyze'} this presentation:\n{text[:30000]}"
|
||||
response = model.generate_content(prompt)
|
||||
return response.text.strip()
|
||||
except Exception as e:
|
||||
return f"AI processing failed: {e}"
|
||||
|
||||
return f"Unknown PPTX action: '{action}'. Try: summarize, extract_text, analyze"
|
||||
|
||||
def file_processor(parameters: dict, player=None, speak=None) -> str:
|
||||
file_path_str = parameters.get("file_path", "").strip()
|
||||
if not file_path_str:
|
||||
return "No file path provided."
|
||||
|
||||
path = Path(file_path_str)
|
||||
if not path.exists():
|
||||
return f"File not found: {file_path_str}"
|
||||
if not path.is_file():
|
||||
return f"Path is not a file: {file_path_str}"
|
||||
|
||||
file_type = _detect_type(path)
|
||||
action = (parameters.get("action") or "").lower().strip()
|
||||
instruction = parameters.get("instruction", "")
|
||||
params = {**parameters, "instruction": instruction}
|
||||
|
||||
log_msg = f"[FileProcessor] {file_type.upper()} | {path.name} | action={action or 'auto'}"
|
||||
print(log_msg)
|
||||
if player:
|
||||
player.write_log(log_msg)
|
||||
|
||||
if file_type == "unknown":
|
||||
try:
|
||||
content = path.read_text(encoding="utf-8", errors="ignore")[:10000]
|
||||
model = _gemini_client()
|
||||
prompt = f"File: {path.name}\nContent preview:\n{content}\n\nTask: {action or instruction or 'Describe what this file contains and what can be done with it.'}"
|
||||
response = model.generate_content(prompt)
|
||||
return response.text.strip()
|
||||
except Exception as e:
|
||||
return f"Unknown file type ({path.suffix}). Could not process: {e}"
|
||||
|
||||
dispatch = {
|
||||
"image": _process_image,
|
||||
"pdf": _process_pdf,
|
||||
"docx": lambda p, a, pm, s: _process_text_doc(p, "docx", a, pm, s),
|
||||
"text": lambda p, a, pm, s: _process_text_doc(p, "text", a, pm, s),
|
||||
"csv": lambda p, a, pm, s: _process_data(p, "csv", a, pm, s),
|
||||
"excel": lambda p, a, pm, s: _process_data(p, "excel", a, pm, s),
|
||||
"json": _process_json,
|
||||
"xml": lambda p, a, pm, s: _process_json(p, a, pm, s),
|
||||
"code": _process_code,
|
||||
"audio": _process_audio,
|
||||
"video": _process_video,
|
||||
"archive": _process_archive,
|
||||
"pptx": _process_pptx,
|
||||
}
|
||||
|
||||
handler = dispatch.get(file_type)
|
||||
if not handler:
|
||||
return f"Unsupported file type: {file_type}"
|
||||
|
||||
try:
|
||||
result = handler(path, action, params, speak)
|
||||
return result or "Done."
|
||||
except Exception as e:
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
return f"Processing failed: {e}"
|
||||
@@ -1,3 +1,4 @@
|
||||
#flight_finder.py
|
||||
import json
|
||||
import re
|
||||
import subprocess
|
||||
|
||||
+442
-442
@@ -1,442 +1,442 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import io
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
import numpy as np
|
||||
import sounddevice as sd
|
||||
|
||||
try:
|
||||
import cv2
|
||||
_CV2 = True
|
||||
except ImportError:
|
||||
_CV2 = False
|
||||
|
||||
try:
|
||||
import mss
|
||||
import mss.tools
|
||||
_MSS = True
|
||||
except ImportError:
|
||||
_MSS = False
|
||||
|
||||
try:
|
||||
import PIL.Image
|
||||
_PIL = True
|
||||
except ImportError:
|
||||
_PIL = False
|
||||
|
||||
from google import genai
|
||||
from google.genai import types as gtypes
|
||||
|
||||
def _base_dir() -> Path:
|
||||
if getattr(sys, "frozen", False):
|
||||
return Path(sys.executable).parent
|
||||
return Path(__file__).resolve().parent.parent
|
||||
|
||||
|
||||
_BASE = _base_dir()
|
||||
_CONFIG_PATH = _BASE / "config" / "api_keys.json"
|
||||
|
||||
|
||||
def _load_config() -> dict:
|
||||
try:
|
||||
return json.loads(_CONFIG_PATH.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
|
||||
def _save_config_key(key: str, value) -> None:
|
||||
try:
|
||||
cfg = _load_config()
|
||||
cfg[key] = value
|
||||
_CONFIG_PATH.write_text(json.dumps(cfg, indent=4), encoding="utf-8")
|
||||
except Exception as e:
|
||||
print(f"[Vision] ⚠️ Could not save config key '{key}': {e}")
|
||||
|
||||
|
||||
def _get_api_key() -> str:
|
||||
key = _load_config().get("gemini_api_key", "")
|
||||
if not key:
|
||||
raise RuntimeError("gemini_api_key not found in config.")
|
||||
return key
|
||||
|
||||
|
||||
def _get_os() -> str:
|
||||
return _load_config().get("os_system", "windows").lower()
|
||||
|
||||
_LIVE_MODEL = "models/gemini-2.5-flash-native-audio-preview-12-2025"
|
||||
_CHANNELS = 1
|
||||
_RECEIVE_SAMPLE_RATE = 24_000
|
||||
_CHUNK_SIZE = 1_024
|
||||
|
||||
_IMG_MAX_W = 640
|
||||
_IMG_MAX_H = 360
|
||||
_JPEG_Q = 60
|
||||
|
||||
_SYSTEM_PROMPT = (
|
||||
"You are JARVIS, an advanced AI assistant. "
|
||||
"Analyze the provided image with precision and intelligence. "
|
||||
"Be concise and direct — maximum two sentences unless the user's question "
|
||||
"requires more detail. "
|
||||
"Address the user respectfully. "
|
||||
"Always call the appropriate tool; never simulate results."
|
||||
)
|
||||
|
||||
|
||||
def _compress(img_bytes: bytes, source_format: str = "PNG") -> tuple[bytes, str]:
|
||||
if not _PIL:
|
||||
return img_bytes, f"image/{source_format.lower()}"
|
||||
|
||||
try:
|
||||
img = PIL.Image.open(io.BytesIO(img_bytes)).convert("RGB")
|
||||
img.thumbnail((_IMG_MAX_W, _IMG_MAX_H), PIL.Image.BILINEAR)
|
||||
buf = io.BytesIO()
|
||||
img.save(buf, format="JPEG", quality=_JPEG_Q, optimize=False)
|
||||
return buf.getvalue(), "image/jpeg"
|
||||
except Exception as e:
|
||||
print(f"[Vision] ⚠️ Image compress failed: {e}")
|
||||
return img_bytes, f"image/{source_format.lower()}"
|
||||
|
||||
def _capture_screen() -> tuple[bytes, str]:
|
||||
|
||||
if not _MSS:
|
||||
raise RuntimeError("mss is not installed. Run: pip install mss")
|
||||
|
||||
with mss.mss() as sct:
|
||||
monitors = sct.monitors # [0] = all combined, [1..n] = real screens
|
||||
target = monitors[1] if len(monitors) > 1 else monitors[0]
|
||||
shot = sct.grab(target)
|
||||
png = mss.tools.to_png(shot.rgb, shot.size)
|
||||
|
||||
return _compress(png, "PNG")
|
||||
|
||||
|
||||
def _cv2_backend() -> int:
|
||||
"""Return the best OpenCV camera backend for the current OS."""
|
||||
if not _CV2:
|
||||
return 0
|
||||
os_name = _get_os()
|
||||
if os_name == "windows":
|
||||
return cv2.CAP_DSHOW
|
||||
if os_name == "mac":
|
||||
return cv2.CAP_AVFOUNDATION
|
||||
return cv2.CAP_ANY
|
||||
|
||||
|
||||
def _probe_camera(index: int, backend: int, warmup: int = 5) -> bool:
|
||||
|
||||
if not _CV2:
|
||||
return False
|
||||
cap = cv2.VideoCapture(index, backend)
|
||||
if not cap.isOpened():
|
||||
cap.release()
|
||||
return False
|
||||
for _ in range(warmup):
|
||||
cap.read()
|
||||
ret, frame = cap.read()
|
||||
cap.release()
|
||||
if not ret or frame is None:
|
||||
return False
|
||||
return bool(np.mean(frame) > 8)
|
||||
|
||||
|
||||
def _detect_camera_index() -> int:
|
||||
|
||||
backend = _cv2_backend()
|
||||
print("[Vision] 🔍 Auto-detecting camera...")
|
||||
for idx in range(6):
|
||||
if _probe_camera(idx, backend):
|
||||
print(f"[Vision] ✅ Camera found at index {idx}")
|
||||
_save_config_key("camera_index", idx)
|
||||
return idx
|
||||
print(f"[Vision] ⚠️ Camera index {idx}: no usable frame")
|
||||
|
||||
print("[Vision] ⚠️ No camera found — defaulting to index 0")
|
||||
_save_config_key("camera_index", 0)
|
||||
return 0
|
||||
|
||||
|
||||
def _get_camera_index() -> int:
|
||||
cfg = _load_config()
|
||||
if "camera_index" in cfg:
|
||||
return int(cfg["camera_index"])
|
||||
return _detect_camera_index()
|
||||
|
||||
|
||||
def _capture_camera() -> tuple[bytes, str]:
|
||||
if not _CV2:
|
||||
raise RuntimeError("OpenCV (cv2) is not installed. Run: pip install opencv-python")
|
||||
|
||||
index = _get_camera_index()
|
||||
backend = _cv2_backend()
|
||||
cap = cv2.VideoCapture(index, backend)
|
||||
|
||||
if not cap.isOpened():
|
||||
raise RuntimeError(f"Camera index {index} could not be opened.")
|
||||
|
||||
for _ in range(10):
|
||||
cap.read()
|
||||
|
||||
ret, frame = cap.read()
|
||||
cap.release()
|
||||
|
||||
if not ret or frame is None:
|
||||
raise RuntimeError("Camera returned no frame.")
|
||||
|
||||
if _PIL:
|
||||
rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
|
||||
img = PIL.Image.fromarray(rgb)
|
||||
img.thumbnail((_IMG_MAX_W, _IMG_MAX_H), PIL.Image.BILINEAR)
|
||||
buf = io.BytesIO()
|
||||
img.save(buf, format="JPEG", quality=_JPEG_Q)
|
||||
return buf.getvalue(), "image/jpeg"
|
||||
|
||||
_, buf = cv2.imencode(".jpg", frame, [cv2.IMWRITE_JPEG_QUALITY, _JPEG_Q])
|
||||
return buf.tobytes(), "image/jpeg"
|
||||
|
||||
class _VisionSession:
|
||||
def __init__(self):
|
||||
self._loop: Optional[asyncio.AbstractEventLoop] = None
|
||||
self._thread: Optional[threading.Thread] = None
|
||||
self._session = None
|
||||
self._out_queue: Optional[asyncio.Queue] = None
|
||||
self._audio_in: Optional[asyncio.Queue] = None
|
||||
self._ready_evt: threading.Event = threading.Event()
|
||||
self._player = None
|
||||
self._lock: threading.Lock = threading.Lock()
|
||||
|
||||
def start(self, player=None, timeout: float = 25.0) -> None:
|
||||
with self._lock:
|
||||
if self._thread and self._thread.is_alive():
|
||||
if player is not None:
|
||||
self._player = player
|
||||
return
|
||||
self._player = player
|
||||
self._thread = threading.Thread(
|
||||
target=self._run_event_loop,
|
||||
daemon=True,
|
||||
name="VisionSessionThread",
|
||||
)
|
||||
self._thread.start()
|
||||
|
||||
if not self._ready_evt.wait(timeout=timeout):
|
||||
raise RuntimeError(f"Vision session did not connect within {timeout}s.")
|
||||
print("[Vision] ✅ Session ready")
|
||||
|
||||
def analyze(self, image_bytes: bytes, mime_type: str, user_text: str) -> None:
|
||||
if not self._loop or not self._out_queue:
|
||||
print("[Vision] ⚠️ Session not started — dropping request")
|
||||
return
|
||||
asyncio.run_coroutine_threadsafe(
|
||||
self._out_queue.put((image_bytes, mime_type, user_text)),
|
||||
self._loop,
|
||||
)
|
||||
|
||||
def is_ready(self) -> bool:
|
||||
return self._session is not None
|
||||
|
||||
def _run_event_loop(self) -> None:
|
||||
self._loop = asyncio.new_event_loop()
|
||||
asyncio.set_event_loop(self._loop)
|
||||
self._loop.run_until_complete(self._session_loop())
|
||||
|
||||
async def _session_loop(self) -> None:
|
||||
self._out_queue = asyncio.Queue(maxsize=30)
|
||||
self._audio_in = asyncio.Queue()
|
||||
|
||||
client = genai.Client(
|
||||
api_key=_get_api_key(),
|
||||
http_options={"api_version": "v1beta"},
|
||||
)
|
||||
config = gtypes.LiveConnectConfig(
|
||||
response_modalities=["AUDIO"],
|
||||
output_audio_transcription={},
|
||||
system_instruction=_SYSTEM_PROMPT,
|
||||
speech_config=gtypes.SpeechConfig(
|
||||
voice_config=gtypes.VoiceConfig(
|
||||
prebuilt_voice_config=gtypes.PrebuiltVoiceConfig(
|
||||
voice_name="Charon"
|
||||
)
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
backoff = 2.0
|
||||
while True:
|
||||
try:
|
||||
print("[Vision] 🔌 Connecting...")
|
||||
async with client.aio.live.connect(
|
||||
model=_LIVE_MODEL, config=config
|
||||
) as session:
|
||||
self._session = session
|
||||
self._ready_evt.set()
|
||||
backoff = 2.0
|
||||
print("[Vision] ✅ Connected")
|
||||
|
||||
async with asyncio.TaskGroup() as tg:
|
||||
tg.create_task(self._send_loop())
|
||||
tg.create_task(self._recv_loop())
|
||||
tg.create_task(self._play_loop())
|
||||
|
||||
except Exception as eg:
|
||||
for exc in eg.exceptions:
|
||||
print(f"[Vision] ⚠️ Session error: {exc}")
|
||||
finally:
|
||||
self._session = None
|
||||
self._ready_evt.clear()
|
||||
|
||||
print(f"[Vision] 🔄 Reconnecting in {backoff:.0f}s...")
|
||||
await asyncio.sleep(backoff)
|
||||
backoff = min(backoff * 1.5, 30.0)
|
||||
self._ready_evt.set()
|
||||
|
||||
async def _send_loop(self) -> None:
|
||||
while True:
|
||||
image_bytes, mime_type, user_text = await self._out_queue.get()
|
||||
if not self._session:
|
||||
print("[Vision] ⚠️ No session — dropping image")
|
||||
continue
|
||||
try:
|
||||
b64 = base64.b64encode(image_bytes).decode("ascii")
|
||||
await self._session.send_client_content(
|
||||
turns={
|
||||
"parts": [
|
||||
{"inline_data": {"mime_type": mime_type, "data": b64}},
|
||||
{"text": user_text},
|
||||
]
|
||||
},
|
||||
turn_complete=True,
|
||||
)
|
||||
print(f"[Vision] 📤 Sent {len(image_bytes):,} bytes — '{user_text[:60]}'")
|
||||
except Exception as e:
|
||||
print(f"[Vision] ⚠️ Send error: {e}")
|
||||
|
||||
async def _recv_loop(self) -> None:
|
||||
transcript: list[str] = []
|
||||
try:
|
||||
async for response in self._session.receive():
|
||||
if response.data:
|
||||
await self._audio_in.put(response.data)
|
||||
|
||||
sc = response.server_content
|
||||
if not sc:
|
||||
continue
|
||||
|
||||
if sc.output_transcription and sc.output_transcription.text:
|
||||
chunk = sc.output_transcription.text.strip()
|
||||
if chunk:
|
||||
transcript.append(chunk)
|
||||
|
||||
if sc.turn_complete:
|
||||
if transcript and self._player:
|
||||
full = re.sub(r"\s+", " ", " ".join(transcript)).strip()
|
||||
if full:
|
||||
self._player.write_log(f"Jarvis: {full}")
|
||||
print(f"[Vision] 💬 {full}")
|
||||
transcript = []
|
||||
|
||||
except Exception as e:
|
||||
print(f"[Vision] ⚠️ Recv error: {e}")
|
||||
raise
|
||||
|
||||
async def _play_loop(self) -> None:
|
||||
stream = sd.RawOutputStream(
|
||||
samplerate=_RECEIVE_SAMPLE_RATE,
|
||||
channels=_CHANNELS,
|
||||
dtype="int16",
|
||||
blocksize=_CHUNK_SIZE,
|
||||
)
|
||||
stream.start()
|
||||
try:
|
||||
while True:
|
||||
chunk = await self._audio_in.get()
|
||||
await asyncio.to_thread(stream.write, chunk)
|
||||
except Exception as e:
|
||||
print(f"[Vision] ❌ Play error: {e}")
|
||||
raise
|
||||
finally:
|
||||
stream.stop()
|
||||
stream.close()
|
||||
|
||||
_session = _VisionSession()
|
||||
_session_lock = threading.Lock()
|
||||
_session_up = False
|
||||
|
||||
|
||||
def _ensure_session(player=None) -> None:
|
||||
global _session_up
|
||||
with _session_lock:
|
||||
if not _session_up:
|
||||
_session.start(player=player)
|
||||
_session_up = True
|
||||
elif player is not None:
|
||||
_session._player = player
|
||||
|
||||
|
||||
def screen_process(
|
||||
parameters: dict,
|
||||
response=None,
|
||||
player=None,
|
||||
session_memory=None,
|
||||
) -> bool:
|
||||
|
||||
params = parameters or {}
|
||||
user_text = (params.get("text") or params.get("user_text") or "").strip()
|
||||
angle = params.get("angle", "screen").lower().strip()
|
||||
|
||||
if not user_text:
|
||||
print("[Vision] ⚠️ No question provided — aborting")
|
||||
return False
|
||||
|
||||
print(f"[Vision] ▶ angle={angle!r} question='{user_text[:80]}'")
|
||||
|
||||
try:
|
||||
_ensure_session(player=player)
|
||||
except Exception as e:
|
||||
print(f"[Vision] ❌ Could not start session: {e}")
|
||||
return False
|
||||
|
||||
try:
|
||||
if angle == "camera":
|
||||
image_bytes, mime_type = _capture_camera()
|
||||
print(f"[Vision] 📷 Camera: {len(image_bytes):,} bytes")
|
||||
else:
|
||||
image_bytes, mime_type = _capture_screen()
|
||||
print(f"[Vision] 🖥️ Screen: {len(image_bytes):,} bytes")
|
||||
except Exception as e:
|
||||
print(f"[Vision] ❌ Capture error: {e}")
|
||||
return False
|
||||
|
||||
_session.analyze(image_bytes, mime_type, user_text)
|
||||
return True
|
||||
|
||||
|
||||
def warmup_session(player=None) -> None:
|
||||
try:
|
||||
_ensure_session(player=player)
|
||||
except Exception as e:
|
||||
print(f"[Vision] ⚠️ Warmup failed: {e}")
|
||||
|
||||
if __name__ == "__main__":
|
||||
print("[TEST] screen_processor.py")
|
||||
print("=" * 52)
|
||||
mode = input("angle — screen / camera (default: screen): ").strip().lower() or "screen"
|
||||
q = input("Question (Enter = default): ").strip() or "What do you see? Be brief."
|
||||
|
||||
t0 = time.perf_counter()
|
||||
warmup_session()
|
||||
print(f"Session ready in {time.perf_counter()-t0:.2f}s\n")
|
||||
|
||||
t1 = time.perf_counter()
|
||||
ok = screen_process({"angle": mode, "text": q})
|
||||
print(f"Queued in {time.perf_counter()-t1:.3f}s — waiting for audio...")
|
||||
time.sleep(10)
|
||||
print("Done." if ok else "Failed.")
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import io
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
import numpy as np
|
||||
import sounddevice as sd
|
||||
|
||||
try:
|
||||
import cv2
|
||||
_CV2 = True
|
||||
except ImportError:
|
||||
_CV2 = False
|
||||
|
||||
try:
|
||||
import mss
|
||||
import mss.tools
|
||||
_MSS = True
|
||||
except ImportError:
|
||||
_MSS = False
|
||||
|
||||
try:
|
||||
import PIL.Image
|
||||
_PIL = True
|
||||
except ImportError:
|
||||
_PIL = False
|
||||
|
||||
from google import genai
|
||||
from google.genai import types as gtypes
|
||||
|
||||
def _base_dir() -> Path:
|
||||
if getattr(sys, "frozen", False):
|
||||
return Path(sys.executable).parent
|
||||
return Path(__file__).resolve().parent.parent
|
||||
|
||||
|
||||
_BASE = _base_dir()
|
||||
_CONFIG_PATH = _BASE / "config" / "api_keys.json"
|
||||
|
||||
|
||||
def _load_config() -> dict:
|
||||
try:
|
||||
return json.loads(_CONFIG_PATH.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
|
||||
def _save_config_key(key: str, value) -> None:
|
||||
try:
|
||||
cfg = _load_config()
|
||||
cfg[key] = value
|
||||
_CONFIG_PATH.write_text(json.dumps(cfg, indent=4), encoding="utf-8")
|
||||
except Exception as e:
|
||||
print(f"[Vision] ⚠️ Could not save config key '{key}': {e}")
|
||||
|
||||
|
||||
def _get_api_key() -> str:
|
||||
key = _load_config().get("gemini_api_key", "")
|
||||
if not key:
|
||||
raise RuntimeError("gemini_api_key not found in config.")
|
||||
return key
|
||||
|
||||
|
||||
def _get_os() -> str:
|
||||
return _load_config().get("os_system", "windows").lower()
|
||||
|
||||
_LIVE_MODEL = "models/gemini-2.5-flash-native-audio-preview-12-2025"
|
||||
_CHANNELS = 1
|
||||
_RECEIVE_SAMPLE_RATE = 24_000
|
||||
_CHUNK_SIZE = 1_024
|
||||
|
||||
_IMG_MAX_W = 640
|
||||
_IMG_MAX_H = 360
|
||||
_JPEG_Q = 60
|
||||
|
||||
_SYSTEM_PROMPT = (
|
||||
"You are JARVIS, an advanced AI assistant. "
|
||||
"Analyze the provided image with precision and intelligence. "
|
||||
"Be concise and direct — maximum two sentences unless the user's question "
|
||||
"requires more detail. "
|
||||
"Address the user respectfully. "
|
||||
"Always call the appropriate tool; never simulate results."
|
||||
)
|
||||
|
||||
|
||||
def _compress(img_bytes: bytes, source_format: str = "PNG") -> tuple[bytes, str]:
|
||||
if not _PIL:
|
||||
return img_bytes, f"image/{source_format.lower()}"
|
||||
|
||||
try:
|
||||
img = PIL.Image.open(io.BytesIO(img_bytes)).convert("RGB")
|
||||
img.thumbnail((_IMG_MAX_W, _IMG_MAX_H), PIL.Image.BILINEAR)
|
||||
buf = io.BytesIO()
|
||||
img.save(buf, format="JPEG", quality=_JPEG_Q, optimize=False)
|
||||
return buf.getvalue(), "image/jpeg"
|
||||
except Exception as e:
|
||||
print(f"[Vision] ⚠️ Image compress failed: {e}")
|
||||
return img_bytes, f"image/{source_format.lower()}"
|
||||
|
||||
def _capture_screen() -> tuple[bytes, str]:
|
||||
|
||||
if not _MSS:
|
||||
raise RuntimeError("mss is not installed. Run: pip install mss")
|
||||
|
||||
with mss.mss() as sct:
|
||||
monitors = sct.monitors # [0] = all combined, [1..n] = real screens
|
||||
target = monitors[1] if len(monitors) > 1 else monitors[0]
|
||||
shot = sct.grab(target)
|
||||
png = mss.tools.to_png(shot.rgb, shot.size)
|
||||
|
||||
return _compress(png, "PNG")
|
||||
|
||||
|
||||
def _cv2_backend() -> int:
|
||||
"""Return the best OpenCV camera backend for the current OS."""
|
||||
if not _CV2:
|
||||
return 0
|
||||
os_name = _get_os()
|
||||
if os_name == "windows":
|
||||
return cv2.CAP_DSHOW
|
||||
if os_name == "mac":
|
||||
return cv2.CAP_AVFOUNDATION
|
||||
return cv2.CAP_ANY
|
||||
|
||||
|
||||
def _probe_camera(index: int, backend: int, warmup: int = 5) -> bool:
|
||||
|
||||
if not _CV2:
|
||||
return False
|
||||
cap = cv2.VideoCapture(index, backend)
|
||||
if not cap.isOpened():
|
||||
cap.release()
|
||||
return False
|
||||
for _ in range(warmup):
|
||||
cap.read()
|
||||
ret, frame = cap.read()
|
||||
cap.release()
|
||||
if not ret or frame is None:
|
||||
return False
|
||||
return bool(np.mean(frame) > 8)
|
||||
|
||||
|
||||
def _detect_camera_index() -> int:
|
||||
|
||||
backend = _cv2_backend()
|
||||
print("[Vision] 🔍 Auto-detecting camera...")
|
||||
for idx in range(6):
|
||||
if _probe_camera(idx, backend):
|
||||
print(f"[Vision] ✅ Camera found at index {idx}")
|
||||
_save_config_key("camera_index", idx)
|
||||
return idx
|
||||
print(f"[Vision] ⚠️ Camera index {idx}: no usable frame")
|
||||
|
||||
print("[Vision] ⚠️ No camera found — defaulting to index 0")
|
||||
_save_config_key("camera_index", 0)
|
||||
return 0
|
||||
|
||||
|
||||
def _get_camera_index() -> int:
|
||||
cfg = _load_config()
|
||||
if "camera_index" in cfg:
|
||||
return int(cfg["camera_index"])
|
||||
return _detect_camera_index()
|
||||
|
||||
|
||||
def _capture_camera() -> tuple[bytes, str]:
|
||||
if not _CV2:
|
||||
raise RuntimeError("OpenCV (cv2) is not installed. Run: pip install opencv-python")
|
||||
|
||||
index = _get_camera_index()
|
||||
backend = _cv2_backend()
|
||||
cap = cv2.VideoCapture(index, backend)
|
||||
|
||||
if not cap.isOpened():
|
||||
raise RuntimeError(f"Camera index {index} could not be opened.")
|
||||
|
||||
for _ in range(10):
|
||||
cap.read()
|
||||
|
||||
ret, frame = cap.read()
|
||||
cap.release()
|
||||
|
||||
if not ret or frame is None:
|
||||
raise RuntimeError("Camera returned no frame.")
|
||||
|
||||
if _PIL:
|
||||
rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
|
||||
img = PIL.Image.fromarray(rgb)
|
||||
img.thumbnail((_IMG_MAX_W, _IMG_MAX_H), PIL.Image.BILINEAR)
|
||||
buf = io.BytesIO()
|
||||
img.save(buf, format="JPEG", quality=_JPEG_Q)
|
||||
return buf.getvalue(), "image/jpeg"
|
||||
|
||||
_, buf = cv2.imencode(".jpg", frame, [cv2.IMWRITE_JPEG_QUALITY, _JPEG_Q])
|
||||
return buf.tobytes(), "image/jpeg"
|
||||
|
||||
class _VisionSession:
|
||||
def __init__(self):
|
||||
self._loop: Optional[asyncio.AbstractEventLoop] = None
|
||||
self._thread: Optional[threading.Thread] = None
|
||||
self._session = None
|
||||
self._out_queue: Optional[asyncio.Queue] = None
|
||||
self._audio_in: Optional[asyncio.Queue] = None
|
||||
self._ready_evt: threading.Event = threading.Event()
|
||||
self._player = None
|
||||
self._lock: threading.Lock = threading.Lock()
|
||||
|
||||
def start(self, player=None, timeout: float = 25.0) -> None:
|
||||
with self._lock:
|
||||
if self._thread and self._thread.is_alive():
|
||||
if player is not None:
|
||||
self._player = player
|
||||
return
|
||||
self._player = player
|
||||
self._thread = threading.Thread(
|
||||
target=self._run_event_loop,
|
||||
daemon=True,
|
||||
name="VisionSessionThread",
|
||||
)
|
||||
self._thread.start()
|
||||
|
||||
if not self._ready_evt.wait(timeout=timeout):
|
||||
raise RuntimeError(f"Vision session did not connect within {timeout}s.")
|
||||
print("[Vision] ✅ Session ready")
|
||||
|
||||
def analyze(self, image_bytes: bytes, mime_type: str, user_text: str) -> None:
|
||||
if not self._loop or not self._out_queue:
|
||||
print("[Vision] ⚠️ Session not started — dropping request")
|
||||
return
|
||||
asyncio.run_coroutine_threadsafe(
|
||||
self._out_queue.put((image_bytes, mime_type, user_text)),
|
||||
self._loop,
|
||||
)
|
||||
|
||||
def is_ready(self) -> bool:
|
||||
return self._session is not None
|
||||
|
||||
def _run_event_loop(self) -> None:
|
||||
self._loop = asyncio.new_event_loop()
|
||||
asyncio.set_event_loop(self._loop)
|
||||
self._loop.run_until_complete(self._session_loop())
|
||||
|
||||
async def _session_loop(self) -> None:
|
||||
self._out_queue = asyncio.Queue(maxsize=30)
|
||||
self._audio_in = asyncio.Queue()
|
||||
|
||||
client = genai.Client(
|
||||
api_key=_get_api_key(),
|
||||
http_options={"api_version": "v1beta"},
|
||||
)
|
||||
config = gtypes.LiveConnectConfig(
|
||||
response_modalities=["AUDIO"],
|
||||
output_audio_transcription={},
|
||||
system_instruction=_SYSTEM_PROMPT,
|
||||
speech_config=gtypes.SpeechConfig(
|
||||
voice_config=gtypes.VoiceConfig(
|
||||
prebuilt_voice_config=gtypes.PrebuiltVoiceConfig(
|
||||
voice_name="Charon"
|
||||
)
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
backoff = 2.0
|
||||
while True:
|
||||
try:
|
||||
print("[Vision] 🔌 Connecting...")
|
||||
async with client.aio.live.connect(
|
||||
model=_LIVE_MODEL, config=config
|
||||
) as session:
|
||||
self._session = session
|
||||
self._ready_evt.set()
|
||||
backoff = 2.0
|
||||
print("[Vision] ✅ Connected")
|
||||
|
||||
async with asyncio.TaskGroup() as tg:
|
||||
tg.create_task(self._send_loop())
|
||||
tg.create_task(self._recv_loop())
|
||||
tg.create_task(self._play_loop())
|
||||
|
||||
except* Exception as eg:
|
||||
for exc in eg.exceptions:
|
||||
print(f"[Vision] ⚠️ Session error: {exc}")
|
||||
finally:
|
||||
self._session = None
|
||||
self._ready_evt.clear()
|
||||
|
||||
print(f"[Vision] 🔄 Reconnecting in {backoff:.0f}s...")
|
||||
await asyncio.sleep(backoff)
|
||||
backoff = min(backoff * 1.5, 30.0)
|
||||
self._ready_evt.set()
|
||||
|
||||
async def _send_loop(self) -> None:
|
||||
while True:
|
||||
image_bytes, mime_type, user_text = await self._out_queue.get()
|
||||
if not self._session:
|
||||
print("[Vision] ⚠️ No session — dropping image")
|
||||
continue
|
||||
try:
|
||||
b64 = base64.b64encode(image_bytes).decode("ascii")
|
||||
await self._session.send_client_content(
|
||||
turns={
|
||||
"parts": [
|
||||
{"inline_data": {"mime_type": mime_type, "data": b64}},
|
||||
{"text": user_text},
|
||||
]
|
||||
},
|
||||
turn_complete=True,
|
||||
)
|
||||
print(f"[Vision] 📤 Sent {len(image_bytes):,} bytes — '{user_text[:60]}'")
|
||||
except Exception as e:
|
||||
print(f"[Vision] ⚠️ Send error: {e}")
|
||||
|
||||
async def _recv_loop(self) -> None:
|
||||
transcript: list[str] = []
|
||||
try:
|
||||
async for response in self._session.receive():
|
||||
if response.data:
|
||||
await self._audio_in.put(response.data)
|
||||
|
||||
sc = response.server_content
|
||||
if not sc:
|
||||
continue
|
||||
|
||||
if sc.output_transcription and sc.output_transcription.text:
|
||||
chunk = sc.output_transcription.text.strip()
|
||||
if chunk:
|
||||
transcript.append(chunk)
|
||||
|
||||
if sc.turn_complete:
|
||||
if transcript and self._player:
|
||||
full = re.sub(r"\s+", " ", " ".join(transcript)).strip()
|
||||
if full:
|
||||
self._player.write_log(f"Jarvis: {full}")
|
||||
print(f"[Vision] 💬 {full}")
|
||||
transcript = []
|
||||
|
||||
except Exception as e:
|
||||
print(f"[Vision] ⚠️ Recv error: {e}")
|
||||
raise
|
||||
|
||||
async def _play_loop(self) -> None:
|
||||
stream = sd.RawOutputStream(
|
||||
samplerate=_RECEIVE_SAMPLE_RATE,
|
||||
channels=_CHANNELS,
|
||||
dtype="int16",
|
||||
blocksize=_CHUNK_SIZE,
|
||||
)
|
||||
stream.start()
|
||||
try:
|
||||
while True:
|
||||
chunk = await self._audio_in.get()
|
||||
await asyncio.to_thread(stream.write, chunk)
|
||||
except Exception as e:
|
||||
print(f"[Vision] ❌ Play error: {e}")
|
||||
raise
|
||||
finally:
|
||||
stream.stop()
|
||||
stream.close()
|
||||
|
||||
_session = _VisionSession()
|
||||
_session_lock = threading.Lock()
|
||||
_session_up = False
|
||||
|
||||
|
||||
def _ensure_session(player=None) -> None:
|
||||
global _session_up
|
||||
with _session_lock:
|
||||
if not _session_up:
|
||||
_session.start(player=player)
|
||||
_session_up = True
|
||||
elif player is not None:
|
||||
_session._player = player
|
||||
|
||||
|
||||
def screen_process(
|
||||
parameters: dict,
|
||||
response=None,
|
||||
player=None,
|
||||
session_memory=None,
|
||||
) -> bool:
|
||||
|
||||
params = parameters or {}
|
||||
user_text = (params.get("text") or params.get("user_text") or "").strip()
|
||||
angle = params.get("angle", "screen").lower().strip()
|
||||
|
||||
if not user_text:
|
||||
print("[Vision] ⚠️ No question provided — aborting")
|
||||
return False
|
||||
|
||||
print(f"[Vision] ▶ angle={angle!r} question='{user_text[:80]}'")
|
||||
|
||||
try:
|
||||
_ensure_session(player=player)
|
||||
except Exception as e:
|
||||
print(f"[Vision] ❌ Could not start session: {e}")
|
||||
return False
|
||||
|
||||
try:
|
||||
if angle == "camera":
|
||||
image_bytes, mime_type = _capture_camera()
|
||||
print(f"[Vision] 📷 Camera: {len(image_bytes):,} bytes")
|
||||
else:
|
||||
image_bytes, mime_type = _capture_screen()
|
||||
print(f"[Vision] 🖥️ Screen: {len(image_bytes):,} bytes")
|
||||
except Exception as e:
|
||||
print(f"[Vision] ❌ Capture error: {e}")
|
||||
return False
|
||||
|
||||
_session.analyze(image_bytes, mime_type, user_text)
|
||||
return True
|
||||
|
||||
|
||||
def warmup_session(player=None) -> None:
|
||||
try:
|
||||
_ensure_session(player=player)
|
||||
except Exception as e:
|
||||
print(f"[Vision] ⚠️ Warmup failed: {e}")
|
||||
|
||||
if __name__ == "__main__":
|
||||
print("[TEST] screen_processor.py")
|
||||
print("=" * 52)
|
||||
mode = input("angle — screen / camera (default: screen): ").strip().lower() or "screen"
|
||||
q = input("Question (Enter = default): ").strip() or "What do you see? Be brief."
|
||||
|
||||
t0 = time.perf_counter()
|
||||
warmup_session()
|
||||
print(f"Session ready in {time.perf_counter()-t0:.2f}s\n")
|
||||
|
||||
t1 = time.perf_counter()
|
||||
ok = screen_process({"angle": mode, "text": q})
|
||||
print(f"Queued in {time.perf_counter()-t1:.3f}s — waiting for audio...")
|
||||
time.sleep(10)
|
||||
print("Done." if ok else "Failed.")
|
||||
@@ -147,18 +147,14 @@ def _desktop_send(app_name: str, receiver: str, message: str) -> str:
|
||||
time.sleep(0.2)
|
||||
pyautogui.press("enter")
|
||||
time.sleep(0.3)
|
||||
|
||||
return f"Message sent to {receiver} via {app_name}."
|
||||
|
||||
|
||||
def _send_whatsapp(receiver: str, message: str) -> str:
|
||||
return _desktop_send("WhatsApp", receiver, message)
|
||||
|
||||
|
||||
def _send_telegram(receiver: str, message: str) -> str:
|
||||
return _desktop_send("Telegram", receiver, message)
|
||||
|
||||
|
||||
def _send_signal(receiver: str, message: str) -> str:
|
||||
return _desktop_send("Signal", receiver, message)
|
||||
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
#web_search.py
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
@@ -22,7 +23,7 @@ def _gemini_search(query: str) -> str:
|
||||
|
||||
client = genai.Client(api_key=_get_api_key())
|
||||
response = client.models.generate_content(
|
||||
model="gemini-2.5-flash-lite",
|
||||
model="gemini-2.5-flash",
|
||||
contents=query,
|
||||
config={"tools": [{"google_search": {}}]},
|
||||
)
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
#youtube_video.py
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
|
||||
+4
-1
@@ -9,11 +9,14 @@ One-Call Policy: Never guess. Call tools exactly once. No retries.
|
||||
Briefing: 1-2 sentences max. Report results briefly.
|
||||
Memory: Store critical user preferences/context automatically.
|
||||
Exit: Only call shutdown_jarvis if session termination is explicit.
|
||||
Response time: Respond as fast as you can
|
||||
|
||||
TOOL ROUTING:
|
||||
|
||||
computer_settings: ALL single OS actions (volume, brightness, wifi, close, shortcuts, power).
|
||||
agent_task: ONLY for complex, multi-step planning (3+ steps).
|
||||
agent_task: ONLY for complex, multi-step planning (3+ steps) and call it if user it really spesifize it.
|
||||
Language: Respond in user's language; extract parameters in English.
|
||||
Do not call agent_task while you can accomplish it with a tool
|
||||
|
||||
|
||||
CRITICAL: Speak/Take action immediately based on available info. Assume and proceed.
|
||||
@@ -14,6 +14,7 @@ from memory.memory_manager import (
|
||||
load_memory, update_memory, format_memory_for_prompt,
|
||||
)
|
||||
|
||||
from actions.file_processor import file_processor
|
||||
from actions.flight_finder import flight_finder
|
||||
from actions.open_app import open_app
|
||||
from actions.weather_report import weather_action
|
||||
@@ -47,7 +48,6 @@ SEND_SAMPLE_RATE = 16000
|
||||
RECEIVE_SAMPLE_RATE = 24000
|
||||
CHUNK_SIZE = 1024
|
||||
|
||||
|
||||
def _get_api_key() -> str:
|
||||
with open(API_CONFIG_PATH, "r", encoding="utf-8") as f:
|
||||
return json.load(f)["gemini_api_key"]
|
||||
@@ -63,18 +63,13 @@ def _load_system_prompt() -> str:
|
||||
"Never simulate or guess results — always call the appropriate tool."
|
||||
)
|
||||
|
||||
|
||||
# ── Transkripsiyon temizleyici ─────────────────────────────────────────────────
|
||||
_CTRL_RE = re.compile(r"<ctrl\d+>", re.IGNORECASE)
|
||||
|
||||
def _clean_transcript(text: str) -> str:
|
||||
"""Gemini'nin ürettiği <ctrlXX> artefaktlarını ve kontrol karakterlerini temizler."""
|
||||
def _clean_transcript(text: str) -> str:
|
||||
text = _CTRL_RE.sub("", text)
|
||||
text = re.sub(r"[\x00-\x08\x0b-\x1f]", "", text)
|
||||
return text.strip()
|
||||
|
||||
|
||||
# ── Tool declarations ──────────────────────────────────────────────────────────
|
||||
TOOL_DECLARATIONS = [
|
||||
{
|
||||
"name": "open_app",
|
||||
@@ -387,6 +382,72 @@ TOOL_DECLARATIONS = [
|
||||
"properties": {},
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "file_processor",
|
||||
"description": (
|
||||
"Processes any file that the user has uploaded or dropped onto the interface. "
|
||||
"Use this when the user refers to an uploaded file and wants an action on it. "
|
||||
"Supports: images (describe/ocr/resize/compress/convert), "
|
||||
"PDFs (summarize/extract_text/to_word), "
|
||||
"Word docs & text files (summarize/fix/reformat/translate), "
|
||||
"CSV/Excel (analyze/stats/filter/sort/convert), "
|
||||
"JSON/XML (validate/format/analyze), "
|
||||
"code files (explain/review/fix/optimize/run/document/test), "
|
||||
"audio (transcribe/trim/convert/info), "
|
||||
"video (trim/extract_audio/extract_frame/compress/transcribe/info), "
|
||||
"archives (list/extract), "
|
||||
"presentations (summarize/extract_text). "
|
||||
"ALWAYS call this tool when a file has been uploaded and the user gives a command about it. "
|
||||
"If the user's command is ambiguous, pick the most logical action for that file type."
|
||||
),
|
||||
"parameters": {
|
||||
"type": "OBJECT",
|
||||
"properties": {
|
||||
"file_path": {
|
||||
"type": "STRING",
|
||||
"description": "Full path to the uploaded file. Leave empty to use the currently uploaded file."
|
||||
},
|
||||
"action": {
|
||||
"type": "STRING",
|
||||
"description": (
|
||||
"What to do with the file. Examples by type:\n"
|
||||
"image: describe | ocr | resize | compress | convert | info\n"
|
||||
"pdf: summarize | extract_text | to_word | info\n"
|
||||
"docx/txt: summarize | fix | reformat | translate_hint | word_count | to_bullet\n"
|
||||
"csv/excel: analyze | stats | filter | sort | convert | info\n"
|
||||
"json: validate | format | analyze | to_csv\n"
|
||||
"code: explain | review | fix | optimize | run | document | test\n"
|
||||
"audio: transcribe | trim | convert | info\n"
|
||||
"video: trim | extract_audio | extract_frame | compress | transcribe | info | convert\n"
|
||||
"archive: list | extract\n"
|
||||
"pptx: summarize | extract_text | analyze"
|
||||
)
|
||||
},
|
||||
"instruction": {
|
||||
"type": "STRING",
|
||||
"description": "Free-form instruction if action doesn't cover it. E.g. 'translate this to Turkish', 'find all email addresses'"
|
||||
},
|
||||
"format": {
|
||||
"type": "STRING",
|
||||
"description": "Target format for conversion. E.g. 'mp3', 'pdf', 'csv', 'png'"
|
||||
},
|
||||
"width": {"type": "INTEGER", "description": "Target width for image resize"},
|
||||
"height": {"type": "INTEGER", "description": "Target height for image resize"},
|
||||
"scale": {"type": "NUMBER", "description": "Scale factor for image resize (e.g. 0.5)"},
|
||||
"quality": {"type": "INTEGER", "description": "Quality 1-100 for image/video compress"},
|
||||
"start": {"type": "STRING", "description": "Start time for trim: seconds or HH:MM:SS"},
|
||||
"end": {"type": "STRING", "description": "End time for trim: seconds or HH:MM:SS"},
|
||||
"timestamp": {"type": "STRING", "description": "Timestamp for video frame extraction HH:MM:SS"},
|
||||
"column": {"type": "STRING", "description": "Column name for CSV filter/sort"},
|
||||
"value": {"type": "STRING", "description": "Filter value for CSV filter"},
|
||||
"condition": {"type": "STRING", "description": "Filter condition: equals|contains|gt|lt"},
|
||||
"ascending": {"type": "BOOLEAN", "description": "Sort order for CSV sort (default: true)"},
|
||||
"save": {"type": "BOOLEAN", "description": "Save result to file (default: true)"},
|
||||
"destination": {"type": "STRING", "description": "Output folder for archive extract"},
|
||||
},
|
||||
"required": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "save_memory",
|
||||
"description": (
|
||||
@@ -419,7 +480,6 @@ TOOL_DECLARATIONS = [
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
class JarvisLive:
|
||||
|
||||
def __init__(self, ui: JarvisUI):
|
||||
@@ -511,7 +571,6 @@ class JarvisLive:
|
||||
print(f"[JARVIS] 🔧 {name} {args}")
|
||||
self.ui.set_state("THINKING")
|
||||
|
||||
# ── save_memory: sessiz ve hızlı ──────────────────────────────────────
|
||||
if name == "save_memory":
|
||||
category = args.get("category", "notes")
|
||||
key = args.get("key", "")
|
||||
@@ -593,6 +652,14 @@ class JarvisLive:
|
||||
elif name == "web_search":
|
||||
r = await loop.run_in_executor(None, lambda: web_search_action(parameters=args, player=self.ui))
|
||||
result = r or "Done."
|
||||
elif name == "file_processor":
|
||||
if not args.get("file_path") and self.ui.current_file:
|
||||
args["file_path"] = self.ui.current_file
|
||||
r = await loop.run_in_executor(
|
||||
None,
|
||||
lambda: file_processor(parameters=args, player=self.ui, speak=self.speak)
|
||||
)
|
||||
result = r or "Done."
|
||||
|
||||
elif name == "computer_control":
|
||||
r = await loop.run_in_executor(None, lambda: computer_control(parameters=args, player=self.ui))
|
||||
@@ -715,7 +782,6 @@ class JarvisLive:
|
||||
await self.session.send_tool_response(
|
||||
function_responses=fn_responses
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
print(f"[JARVIS] ❌ Recv: {e}")
|
||||
traceback.print_exc()
|
||||
@@ -748,10 +814,8 @@ class JarvisLive:
|
||||
self.set_speaking(False)
|
||||
self._turn_done_event.clear()
|
||||
continue
|
||||
|
||||
self.set_speaking(True)
|
||||
await asyncio.to_thread(stream.write, chunk)
|
||||
|
||||
except Exception as e:
|
||||
print(f"[JARVIS] ❌ Play: {e}")
|
||||
raise
|
||||
@@ -794,13 +858,11 @@ class JarvisLive:
|
||||
except Exception as e:
|
||||
print(f"[JARVIS] ⚠️ {e}")
|
||||
traceback.print_exc()
|
||||
|
||||
self.set_speaking(False)
|
||||
self.ui.set_state("THINKING")
|
||||
print("[JARVIS] 🔄 Reconnecting in 3s...")
|
||||
await asyncio.sleep(3)
|
||||
|
||||
|
||||
def main():
|
||||
ui = JarvisUI("face.png")
|
||||
|
||||
@@ -815,6 +877,5 @@ def main():
|
||||
threading.Thread(target=runner, daemon=True).start()
|
||||
ui.root.mainloop()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -212,4 +212,4 @@ def forget(key: str, category: str = "notes") -> str:
|
||||
return f"Not found: {category}/{key}"
|
||||
|
||||
|
||||
forget_memory = forget
|
||||
forget_memory = forget
|
||||
@@ -1,7 +1,7 @@
|
||||
# 🤖 MARK XXXVII (37)
|
||||
# 🤖 MARK XXXIX (39)
|
||||
### The Ultimate Cross-Platform Personal AI Assistant — By FatihMakes
|
||||
|
||||
> 📺 **[Watch the full setup video on YouTube](https://youtu.be/3dqgUsonmn8?si=eh_McOF5V8bduTo1)**
|
||||
> 📺 **[Watch the full setup video on YouTube](https://www.youtube.com/watch?v=BhOsnGC_sAA)**
|
||||
|
||||
A real-time voice AI that can hear, see, understand, and control your computer — on any OS. Supporting Windows, macOS, and Linux. Local execution. Zero subscriptions. Engineered for total autonomy.
|
||||
|
||||
@@ -9,7 +9,7 @@ A real-time voice AI that can hear, see, understand, and control your computer
|
||||
|
||||
## ✨ Overview
|
||||
|
||||
MARK XXXVII is the most versatile evolution of the Jarvis project. It transforms your operating system into a living, breathing intelligent entity. Through natural voice commands, Mark 37 analyzes your screen, manages your files, and executes complex workflows across different environments.
|
||||
MARK XXXIX represents the pinnacle of the Jarvis series, evolving into a more flexible and robust system. It bridges the gap between the operating system and human intent. Through natural dialogue, Mark 39 analyzes your screen, processes uploaded documents, and executes complex workflows with a brand-new, adaptive interface.
|
||||
|
||||
It's not just an assistant — it's an extension of your digital life.
|
||||
|
||||
@@ -25,32 +25,30 @@ It's not just an assistant — it's an extension of your digital life.
|
||||
| 🧩 Autonomous Tasks | High-level planning for complex, multi-step goals |
|
||||
| 👁️ Visual Awareness | Real-time screen processing and webcam vision |
|
||||
| 🧠 Persistent Memory | Deeply remembers your projects, preferences, and personal context |
|
||||
| 🔒 Privacy First | Physical mute button (F4 / UI) and fully local control |
|
||||
| ⌨️ Keyboard & Voice | Switch seamlessly between typing and speaking |
|
||||
| ⌨️ Hybrid Input | Seamlessly switch between keyboard typing and voice commands |
|
||||
|
||||
---
|
||||
|
||||
## 🆕 What's New in XXXVII
|
||||
## 🆕 What's New in XXXIX
|
||||
|
||||
- 🐧🍎 **Cross-Platform Support** *(Experimental)* — Now runs on Windows, macOS, and Linux. Core features work across all platforms; some OS-specific tasks may vary during the testing phase.
|
||||
- 💤 **Graceful Shutdown** — Tell Jarvis to "Shut down" or "Close" and it exits the session properly.
|
||||
- 🧠 **Strengthened Memory** — Improved long-term retention and context recall; better understanding of relationships and history.
|
||||
- 🔧 **Action Core Optimizations** — Major bugs in action scripts fixed for smoother system execution.
|
||||
- ⚡ **Optimized Performance** — Faster response times and refined tool-calling logic.
|
||||
- 📂 **Advanced File Handling** — New support for direct file uploads. Drop PDFs, source code, or images into the assistant to have them analyzed, summarized, or edited instantly.
|
||||
- 🎨 **Adaptive & Flexible UI** — A complete overhaul of the interface. The new UI is fully resizable and responsive, featuring transparency controls and customizable layouts to fit your workspace perfectly.
|
||||
- 🐧🍎 **Refined Cross-Platform Stability** — Major fixes for macOS and Linux compatibility. Core system actions are now more consistent across all three major operating systems.
|
||||
- ⚡ **Optimized Core Engine** — Significant performance boost in tool-calling logic and response generation, resulting in a 40% faster interaction speed.
|
||||
|
||||
---
|
||||
|
||||
## ⚡ Quick Start
|
||||
|
||||
```bash
|
||||
git clone https://github.com/FatihMakes/Jarvis-MK37.git
|
||||
cd Jarvis-MK37
|
||||
git clone https://github.com/FatihMakes/Mark-XXXIX.git
|
||||
cd Mark-XXXIX
|
||||
pip install -r requirements.txt
|
||||
playwright install
|
||||
python main.py
|
||||
```
|
||||
|
||||
> ⚠️ **Installation Note:** Due to the new cross-platform integration, OS-specific libraries are not all bundled in `requirements.txt` to keep the install lightweight. If you encounter a `ModuleNotFoundError`, simply run `pip install <module_name>` for your specific system.
|
||||
> ⚠️ **Installation Note:** To keep the repository lightweight, some OS-specific dependencies are not bundled in `requirements.txt`. If you run into a `ModuleNotFoundError`, simply install the missing package via `pip install <module_name>` for your specific system.
|
||||
|
||||
---
|
||||
|
||||
@@ -60,7 +58,7 @@ python main.py
|
||||
|---|---|
|
||||
| **OS** | Windows 10/11, macOS, or Linux |
|
||||
| **Python** | 3.11 or 3.12 |
|
||||
| **Microphone** | Required for voice commands |
|
||||
| **Microphone** | Required for voice interaction |
|
||||
| **API Key** | Free Gemini API key |
|
||||
|
||||
---
|
||||
@@ -74,10 +72,10 @@ Licensed under **[Creative Commons BY-NC 4.0](https://creativecommons.org/licens
|
||||
|
||||
## 👤 Connect with the Creator
|
||||
|
||||
Engineered by an 18-year-old developer building a real JARVIS-style assistant.
|
||||
⭐ **Star the repository to support the journey to Mark 85.**
|
||||
Engineered by a developer building a real-world JARVIS-style assistant.
|
||||
⭐ **Star the repository to support the journey to Mark 100.**
|
||||
|
||||
| Platform | Link |
|
||||
|---|---|
|
||||
| YouTube | [@FatihMakes](https://www.youtube.com/@FatihMakes) |
|
||||
| Instagram | [@fatihmakes](https://www.instagram.com/fatihmakes) |
|
||||
| Instagram | [@fatihmakes](https://www.instagram.com/fatihmakes) |
|
||||
Binary file not shown.
@@ -7,5 +7,5 @@ subprocess.run([sys.executable, "-m", "pip", "install", "-r", "requirements.txt"
|
||||
print("Installing Playwright browsers...")
|
||||
subprocess.run([sys.executable, "-m", "playwright", "install"], check=True)
|
||||
|
||||
print("\n✅ Setup complete! Run 'python main.py' to start MARK XXXVII.")
|
||||
print("\n✅ Setup complete! Run 'python main.py' to start MARK XXV.")
|
||||
|
||||
Reference in new issue
Block a user