Add files via upload

README

@@ -0,0 +1,45 @@
+# 🤖 Jarvis Mark X — Local AI Desktop Assistant (UPDATED)
+
+Jarvis Mark X is a **local, modular AI desktop assistant** inspired by Jarvis and built from scratch using **Python**.
+
+It allows you to control your computer with voice commands and an intent → action system.
+
+## 🚀 Features
+- Open applications
+- Send messages (WhatsApp)
+- Web search 
+- Weather reports
+- Temporary & long-term memory
+- Chatting with good memory
+
+
+## 🎥 Tutorial
+This project is part of a **video tutorial series**.  
+Please watch the tutorial to understand the architecture and how to safely extend the system.
+video: https://youtu.be/w_AGre6-9TM?si=gE5xc4_aKhy4DwnA
+
+## 🔐 Security Notice
+- Runs **100% locally**
+- No remote access by default
+- Jarvis only executes what **YOU code**
+
+⚠️ Do NOT paste unknown code or give full system control to untrusted scripts.
+
+---
+
+## ⚠️ Usage Warning
+- **No commercial use**
+- Do NOT sell, market, or monetize
+- Do NOT present this project as your own on social media
+
+You may use and modify it for **personal and educational purposes**.
+
+---
+
+## 📲 Follow & Contact
+Instagram: https://www.instagram.com/fatihmakes/  
+TikTok: https://www.tiktok.com/@fatihmakes  
+
+If you have issues or suggestions, feel free to reach out.
+
+⭐ Like, follow, and subscribe for future Jarvis upgrades!

REQUIREMENTS.txt

@@ -0,0 +1,22 @@
+# Don't forget to download Python first(We are running this program with Python): https://www.python.org/downloads/
+# Detailed set up (step by step guide): https://youtu.be/w_AGre6-9TM?si=gE5xc4_aKhy4DwnA
+
+# If you have some errors with libraries that's probably because you forgot to install these ones
+# Just copy the text below and paste on the cmd to fix those errors.
+
+# Also you have to paste the API keys on the UI to run the program.
+
+# serpapi: https://serpapi.com/dashboard
+# openrouter: https://openrouter.ai/settings/keys
+
+#by the way, you have to download vosk and change the model_path on speech_to_text.py with your own path.
+# download vosk here: https://alphacephei.com/vosk/models
+
+pip install sounddevice
+pip install soundfile
+pip install vosk
+pip install edge-tts
+pip install google-search-results
+pip install Pillow
+pip install requests
+pip install pyautogui

actions/aircraft_report.py

@@ -0,0 +1,2 @@
+#EXAMPLE ACTION IDEA FOR IMPROVE YOURSELF
+#TAKE THE API KEY FROM A RADAR WEB SITE TO SEE ALL PLANES ON THE AIR

actions/open_app.py

@@ -0,0 +1,63 @@
+import time
+import pyautogui
+from tts import edge_speak
+
+
+def open_app(
+    parameters: dict,
+    response: str | None = None,
+    player=None,
+    session_memory=None
+) -> bool:
+    """
+    Opens an application using Windows search.
+
+    parameters:
+        - app_name (str)
+
+    Memory behavior:
+        - Uses ONLY session memory
+        - No long-term memory writes
+    """
+
+    app_name = (parameters or {}).get("app_name", "").strip()
+
+    if not app_name and session_memory:
+        app_name = session_memory.open_app or ""
+
+    if not app_name:
+        msg = "Sir, I couldn't determine which application to open."
+        if player:
+            player.write_log(msg)
+        edge_speak(msg, player)
+        return False
+
+    if response:
+        if player:
+            player.write_log(response)
+        edge_speak(response, player)
+
+    try:
+        pyautogui.PAUSE = 0.1
+
+
+        pyautogui.press("win")
+        time.sleep(0.3)
+
+        pyautogui.write(app_name, interval=0.03)
+        time.sleep(0.2)
+
+        pyautogui.press("enter")
+        time.sleep(0.6)
+
+        if session_memory:
+            session_memory.set_open_app(app_name)
+
+        return True
+
+    except Exception as e:
+        msg = f"Sir, I failed to open {app_name}."
+        if player:
+            player.write_log(f"{msg} ({e})")
+        edge_speak(msg, player)
+        return False

actions/send_message.py

@@ -0,0 +1,96 @@
+import time
+import pyautogui
+from tts import edge_speak
+
+REQUIRED_PARAMS = ["receiver", "message_text", "platform"]
+
+def send_message(parameters: dict, response: str | None = None, player=None, session_memory=None) -> bool:
+    """
+    Send a message via Windows app (WhatsApp, Telegram, etc.)
+
+    Multi-step support: asks for missing parameters using temporary memory.
+
+    Expected parameters:
+        - receiver (str)
+        - message_text (str)
+        - platform (str, default: "WhatsApp")
+    """
+
+    if session_memory is None:
+        msg = "Session memory missing, cannot proceed."
+        if player:
+            player.write_log(msg)
+        edge_speak(msg, player)
+        return False
+
+    if parameters:
+        session_memory.update_parameters(parameters)
+
+    for param in REQUIRED_PARAMS:
+        value = session_memory.get_parameter(param)
+        if not value:
+        
+            session_memory.set_current_question(param)
+            question_text = ""
+            if param == "receiver":
+                question_text = "Sir, who should I send the message to?"
+            elif param == "message_text":
+                question_text = "Sir, what should I say?"
+            elif param == "platform":
+                question_text = "Sir, which platform should I use? (WhatsApp, Telegram, etc.)"
+            else:
+                question_text = f"Sir, please provide {param}."
+
+            if player:
+                player.write_log("AI :", question_text)
+            edge_speak(question_text, player)
+            return False  
+
+    receiver = session_memory.get_parameter("receiver").strip()
+    platform = session_memory.get_parameter("platform").strip() or "WhatsApp"
+    message_text = session_memory.get_parameter("message_text").strip()
+
+    if response:
+        if player:
+            player.write_log(response)
+        edge_speak(response, player)
+
+    try:
+        pyautogui.PAUSE = 0.1
+
+        pyautogui.press("win")
+        time.sleep(0.3)
+        pyautogui.write(platform, interval=0.03)
+        pyautogui.press("enter")
+        time.sleep(0.6)
+
+        pyautogui.hotkey("ctrl", "f")
+        time.sleep(0.2)
+        pyautogui.write(receiver, interval=0.03)
+        time.sleep(0.2)
+        pyautogui.press("enter")
+        time.sleep(0.2)
+
+        pyautogui.write(message_text, interval=0.03)
+        pyautogui.press("enter")
+
+        session_memory.clear_current_question()
+        session_memory.clear_pending_intent()
+        session_memory.update_parameters({})  
+
+        # -----------------------------
+        # Log success
+        # -----------------------------
+        success_msg = f"Sir, message sent to {receiver} via {platform}."
+        if player:
+            player.write_log(success_msg)
+        edge_speak(success_msg, player)
+
+        return True
+
+    except Exception as e:
+        msg = f"Sir, I failed to send the message. ({e})"
+        if player:
+            player.write_log(msg)
+        edge_speak(msg, player)
+        return False

actions/weather_report.py

@@ -0,0 +1,67 @@
+import webbrowser
+from urllib.parse import quote_plus
+from tts import edge_speak
+
+
+def weather_action(
+    parameters: dict,
+    player=None,
+    session_memory=None
+):
+    """
+    Weather report action.
+    Opens a Google weather search and gives a short spoken confirmation.
+    """
+
+    city = parameters.get("city")
+    time = parameters.get("time")
+    if not city or not isinstance(city, str):
+        msg = "Sir, the city is missing for the weather report."
+        _speak_and_log(msg, player)
+        return msg
+
+    city = city.strip()
+
+    if not time or not isinstance(time, str):
+        time = "today"
+    else:
+        time = time.strip()
+
+    search_query = f"weather in {city} {time}"
+    encoded_query = quote_plus(search_query)
+    url = f"https://www.google.com/search?q={encoded_query}"
+
+    try:
+        webbrowser.open(url)
+    except Exception:
+        msg = f"Sir, I couldn't open the browser for the weather report."
+        _speak_and_log(msg, player)
+        return msg
+
+    msg = f"Showing the weather for {city}, {time}, sir."
+    _speak_and_log(msg, player)
+
+    if session_memory:
+        try:
+            session_memory.set_last_search(
+                query=search_query,
+                response=msg
+            )
+        except Exception:
+            pass  
+
+    return msg
+
+
+def _speak_and_log(message: str, player=None):
+    """Helper: log + TTS safely"""
+    if player:
+        try:
+            player.write_log(f"JARVIS: {message}")
+        except Exception:
+            pass
+
+    try:
+        edge_speak(message)
+    except Exception:
+        pass

actions/web_search.py

@@ -0,0 +1,142 @@
+import re
+from serpapi import Client 
+from tts import edge_speak
+from memory.config_manager import get_serpapi_key
+
+MAX_NEWS_ITEMS = 3
+
+def clean(text: str) -> str:
+    if not text:
+        return ""
+    text = re.sub(r"\s+", " ", text)
+    text = re.sub(r"\(.*?\)|\[.*?\]", "", text)
+    text = text.strip()
+    text = re.sub(r"\.{2,}", ".")
+    text = re.sub(r"\s*—\s*", " - ", text)
+    return text
+
+def is_trash(text: str) -> bool:
+    t = text.lower()
+    trash_patterns = [
+        r"\bstock(?:s)?\b.*\btoday\b",
+        r"\bshare(?:s)?\b.*\bprice\b",
+        r"\binvestor(?:s)?\b",
+        r"\btrading\b",
+        r"\bmarket(?:s)?\b.*\bopen(?:s|ed)?\b",
+        r"\bticker\b",
+        r"\bnyse\b",
+        r"\bnasdaq\b",
+        r"\.\w{2,4}\sis\b",
+    ]
+    spam_keywords = [
+        "click here", "read more", "advertisement", "sponsored",
+        "subscribe", "newsletter", "sign up",
+        "best things to do", "events this week", "calendar",
+        "official website", "visit our", "learn more",
+        "year in review", "trending now", "top 10"
+    ]
+    for pattern in trash_patterns:
+        if re.search(pattern, t):
+            return True
+    return any(keyword in t for keyword in spam_keywords)
+
+def extract_clean_news(result: dict) -> str:
+    title = clean(result.get("title", ""))
+    snippet = clean(result.get("snippet", ""))
+    if not title:
+        return ""
+    if snippet.startswith(title[:30]) or snippet == title:
+        return title
+    if len(snippet) > 120:
+        snippet = snippet[:120]
+        last_period = snippet.rfind(".")
+        last_space = snippet.rfind(" ")
+        if last_period > 80:
+            snippet = snippet[:last_period + 1]
+        elif last_space > 80:
+            snippet = snippet[:last_space] + "..."
+        return f"{title}. {snippet}"
+    return title
+
+def format_news_output(news_items: list) -> str:
+    if len(news_items) == 1:
+        return news_items[0]
+    elif len(news_items) == 2:
+        return f"{news_items[0]}. Also, {news_items[1]}"
+    else:
+        result = news_items[0]
+        for item in news_items[1:-1]:
+            result += f". {item}"
+        result += f". Additionally, {news_items[-1]}"
+        return result
+    
+def serpapi_search(query: str) -> str:
+    api_key = get_serpapi_key()
+    if not api_key:
+        return "Sir, the web search system is not configured."
+
+    clean_query = query
+    if "what happened" in query.lower():
+        clean_query = re.sub(r"what happened (?:in|at|to)\s*", "", query, flags=re.IGNORECASE)
+        clean_query += " news today"
+
+    params = {
+        "q": clean_query,
+        "engine": "google_news",
+        "hl": "en",
+        "gl": "us",
+        "num": 15
+    }
+
+    try:
+        client = Client(api_key=api_key) 
+        data = client.search(params)  
+        results = data.get("news_results", [])
+    except Exception:
+
+        params["engine"] = "google"
+        try:
+            client = Client(api_key=api_key)
+            data = client.search(params)
+            results = data.get("organic_results", [])
+        except Exception:
+            return "Sir, I couldn't connect to the search service."
+
+    if not results:
+        return "Sir, I couldn't find any recent news about that."
+
+    news_items = []
+    for result in results:
+        title = result.get("title", "")
+        snippet = result.get("snippet", "")
+        if is_trash(title) or is_trash(snippet):
+            continue
+        news_text = extract_clean_news(result)
+        if news_text and len(news_text.split()) >= 6:
+            news_items.append(news_text)
+        if len(news_items) >= MAX_NEWS_ITEMS:
+            break
+
+    if not news_items:
+        return "Sir, I found some results but they weren't clear news stories."
+
+    return format_news_output(news_items)
+
+def web_search(parameters, player=None, session_memory=None):
+    query = (parameters or {}).get("query", "").strip()
+    if not query:
+        msg = "Sir, I couldn't understand the search request."
+        edge_speak(msg)
+        return msg
+
+    answer = serpapi_search(query)
+
+    if player:
+        player.write_log(f"AI: {answer}")
+
+    edge_speak(answer)
+
+    if session_memory:
+        session_memory.set_last_search(query, answer)
+
+    return answer

core/prompt.txt

@@ -0,0 +1,44 @@
+You are Jarvis, Fully optimized, Fast react AI Computer Assistant
+
+Intents:
+    - send_message
+    - open_app
+    - search
+    - weather_report
+    - chat
+    
+Parameter Rules:
+    - search → parameters.query (cleaned core question).
+    - weather_report → city, time (time is the day)
+    - if user wants to send a message to someone then return intent: send_message → receiver, message_text, platform (default WhatsApp) and also if user says something like "ask him how he is doing" then appoint this into message_text
+    - open_app → app_name(If user asks, can you open please, create a lovely text and open the app that user asked you to open)
+        - If user the app name incorrect, open the app by correcting it(f.e: Hey Jarvis open what's up = WhatsApp)
+    - Missing a required field → ask one short clarification question, set needs_clarification:true, stop.
+
+Chat Mode:
+    - 1‑2 short sentences, no extra analysis.
+    - Keep only identity, preferences, relationships, emotional_state.
+    - temporary memory
+
+Output format:
+
+    json
+    {
+    "intent": null,
+    "parameters": {},
+    "needs_clarification": false,
+    "text": null,
+    "memory_update": {
+        "identity": {},
+        "preferences": {},
+        "relationships": {},
+        "emotional_state": {}
+    }
+    }
+ 
+Just add really important informations in the memory like;
+    - Name of the user, age, city
+    - The question that user's ask mostly doesn't include memory.
+
+return json format. 
+Create really fast responses. make it quick.

llm.py

@@ -0,0 +1,184 @@
+import os
+import json
+import requests
+import sys
+from pathlib import Path
+
+OPENROUTER_URL = "https://openrouter.ai/api/v1/chat/completions"
+MODEL = "arcee-ai/trinity-large-preview:free"
+
+def get_base_dir():
+    if getattr(sys, "frozen", False):
+        return Path(sys.executable).parent
+    return Path(__file__).resolve().parent
+
+BASE_DIR = get_base_dir()
+
+PROMPT_PATH = BASE_DIR / "core" / "prompt.txt"
+API_CONFIG_PATH = BASE_DIR / "config" / "api_keys.json"
+
+def load_api_keys() -> dict:
+    if not os.path.exists(API_CONFIG_PATH):
+        return {}
+
+    try:
+        with open(API_CONFIG_PATH, "r", encoding="utf-8") as f:
+            return json.load(f)
+    except Exception as e:
+        print(f"❌ Failed to read api_keys.json: {e}")
+        return {}
+
+
+def get_openrouter_key() -> str | None:
+    keys = load_api_keys()
+    return keys.get("openrouter_api_key")
+
+def load_system_prompt() -> str:
+    try:
+        with open(PROMPT_PATH, "r", encoding="utf-8") as f:
+            return f.read()
+    except Exception as e:
+        print(f"⚠️ prompt.txt couldn't be loaded: {e}")
+        return "You are Jarvis, a helpful AI assistant."
+
+
+SYSTEM_PROMPT = load_system_prompt()
+
+def safe_json_parse(text: str) -> dict | None:
+    if not text:
+        return None
+
+    text = text.strip()
+
+    if "```json" in text:
+        try:
+            start = text.index("```json") + 7
+            end = text.index("```", start)
+            text = text[start:end].strip()
+        except:
+            pass
+    elif "```" in text:
+        try:
+            start = text.index("```") + 3
+            end = text.index("```", start)
+            text = text[start:end].strip()
+        except:
+            pass
+
+    try:
+        start = text.index("{")
+        end = text.rindex("}") + 1
+        json_str = text[start:end]
+        return json.loads(json_str)
+    except Exception as e:
+        print(f"⚠️ JSON parse error: {e}")
+        print(f"⚠️ Raw text preview: {text[:200]}")
+        return None
+
+def get_llm_output(user_text: str, memory_block: dict | None = None) -> dict:
+
+    if not user_text or not user_text.strip():
+        return {
+            "intent": "chat",
+            "parameters": {},
+            "needs_clarification": False,
+            "text": "Sir, I didn't catch that.",
+            "memory_update": None
+        }
+
+    api_key = get_openrouter_key()
+    if not api_key:
+        print("❌ OPENROUTER API KEY NOT FOUND")
+        return {
+            "intent": "chat",
+            "parameters": {},
+            "needs_clarification": False,
+            "text": "OpenRouter API key is missing, Sir.",
+            "memory_update": None
+        }
+
+    memory_str = ""
+    if memory_block:
+        memory_str = "\n".join(f"{k}: {v}" for k, v in memory_block.items())
+
+    user_prompt = f"""User message: "{user_text}"
+
+Known user memory:
+{memory_str if memory_str else "No memory available"}"""
+
+    payload = {
+        "model": MODEL,
+        "messages": [
+            {"role": "system", "content": SYSTEM_PROMPT},
+            {"role": "user", "content": user_prompt}
+        ],
+        "temperature": 0.2,
+        "max_tokens": 500
+    }
+
+    headers = {
+        "Authorization": f"Bearer {api_key}",
+        "Content-Type": "application/json",
+        "HTTP-Referer": "http://localhost",
+        "X-Title": "Jarvis-Assistant"
+    }
+
+    try:
+        response = requests.post(
+            OPENROUTER_URL,
+            headers=headers,
+            json=payload,
+            timeout=30
+        )
+
+        if response.status_code != 200:
+            print(f"❌ OpenRouter API Error: {response.text}")
+            return {
+                "intent": "chat",
+                "parameters": {},
+                "needs_clarification": False,
+                "text": f"Sir, API error ({response.status_code}).",
+                "memory_update": None
+            }
+
+        data = response.json()
+        content = data["choices"][0]["message"]["content"]
+
+        parsed = safe_json_parse(content)
+
+        if parsed:
+            return {
+                "intent": parsed.get("intent", "chat"),
+                "parameters": parsed.get("parameters", {}),
+                "needs_clarification": parsed.get("needs_clarification", False),
+                "text": parsed.get("text"),
+                "memory_update": parsed.get("memory_update")
+            }
+
+        return {
+            "intent": "chat",
+            "parameters": {},
+            "needs_clarification": False,
+            "text": content,
+            "memory_update": None
+        }
+
+    except requests.exceptions.Timeout:
+        print("❌ OpenRouter timeout")
+        return {
+            "intent": "chat",
+            "parameters": {},
+            "needs_clarification": False,
+            "text": "Sir, the request timed out.",
+            "memory_update": None
+        }
+
+    except Exception as e:
+        print(f"❌ LLM ERROR: {e}")
+        return {
+            "intent": "chat",
+            "parameters": {},
+            "needs_clarification": False,
+            "text": "Sir, a system error occurred.",
+            "memory_update": None
+        }

main.py

@@ -0,0 +1,186 @@
+import asyncio
+import threading
+
+from speech_to_text import record_voice
+from llm import get_llm_output
+from tts import edge_speak, stop_speaking
+from ui import JarvisUI
+import sys
+from pathlib import Path
+
+from actions.open_app import open_app
+from actions.web_search import web_search
+from actions.weather_report import weather_action
+from actions.send_message import send_message  
+
+from memory.memory_manager import load_memory, update_memory
+from memory.temporary_memory import TemporaryMemory
+
+interrupt_commands = ["mute", "quit", "exit", "stop"]
+
+temp_memory = TemporaryMemory()
+
+def get_base_dir():
+    if getattr(sys, "frozen", False):
+        return Path(sys.executable).parent
+    return Path(__file__).resolve().parent
+
+BASE_DIR = get_base_dir()
+
+async def get_voice_input():
+    return await asyncio.to_thread(record_voice)
+
+async def ai_loop(ui: JarvisUI):
+    while True:
+        
+        user_text = await get_voice_input()
+
+        if not user_text:
+            continue
+
+        if any(cmd in user_text.lower() for cmd in interrupt_commands):
+            stop_speaking()
+            temp_memory.reset()
+            continue
+
+        ui.write_log(f"You: {user_text}")
+
+        if temp_memory.get_current_question():
+            param = temp_memory.get_current_question()
+            temp_memory.update_parameters({param: user_text})
+            temp_memory.clear_current_question()
+            user_text = temp_memory.get_last_user_text()
+
+        temp_memory.set_last_user_text(user_text)
+
+        long_term_memory = load_memory()
+
+        def minimal_memory_for_prompt(memory: dict) -> dict:
+            result = {}
+
+            identity = memory.get("identity", {})
+            preferences = memory.get("preferences", {})
+            relationships = memory.get("relationships", {})
+            emotional_state = memory.get("emotional_state", {})
+
+            if "name" in identity:
+                result["user_name"] = identity["name"].get("value")
+
+            for k in ["favorite_color", "favorite_food", "favorite_music"]:
+                if k in preferences:
+                    val = preferences[k].get("value")
+                    if isinstance(val, dict) and "value" in val:
+                        val = val["value"]
+                    result[k] = val
+
+            for rel, info in relationships.items():
+                if isinstance(info, dict) and "name" in info and "value" in info["name"]:
+                    result[f"{rel}_name"] = info["name"]["value"]
+
+            for event, info in emotional_state.items():
+                if "value" in info:
+                    result[f"emotion_{event}"] = info["value"]
+
+            return {k: v for k, v in result.items() if v}
+
+        memory_for_prompt = minimal_memory_for_prompt(long_term_memory)
+
+        history_lines = temp_memory.get_history_for_prompt()
+        recent_history = "\n".join(history_lines.split("\n")[-5:])
+        if recent_history:
+            memory_for_prompt["recent_conversation"] = recent_history
+
+        if temp_memory.has_pending_intent():
+            memory_for_prompt["_pending_intent"] = temp_memory.pending_intent
+            memory_for_prompt["_collected_params"] = str(temp_memory.get_parameters())
+
+        try:
+            llm_output = get_llm_output(
+                user_text=user_text,
+                memory_block=memory_for_prompt
+            )
+        except Exception as e:
+            ui.write_log(f"AI ERROR: {e}")
+            continue
+
+        intent = llm_output.get("intent", "chat")
+        parameters = llm_output.get("parameters", {})
+        response = llm_output.get("text")
+        memory_update = llm_output.get("memory_update")
+
+        if memory_update and isinstance(memory_update, dict):
+            update_memory(memory_update)
+
+        temp_memory.set_last_ai_response(response)
+
+        if intent == "send_message":
+            temp_memory.set_pending_intent("send_message")
+            temp_memory.update_parameters(parameters)
+
+            if all(temp_memory.get_parameter(p) for p in ["receiver", "message_text", "platform"]):
+                threading.Thread(
+                    target=send_message,
+                    kwargs={
+                        "parameters": temp_memory.get_parameters(),
+                        "player": ui,
+                        "session_memory": temp_memory
+                    },
+                    daemon=True
+                ).start()
+
+        elif intent == "open_app":
+            if parameters.get("app_name"):
+                threading.Thread(
+                    target=open_app,
+                    kwargs={
+                        "parameters": parameters,
+                        "response": response,
+                        "player": ui,
+                        "session_memory": temp_memory
+                    },
+                    daemon=True
+                ).start()
+
+        elif intent == "weather_report":
+            if parameters.get("city"):
+                threading.Thread(
+                    target=weather_action,
+                    kwargs={
+                        "parameters": parameters,
+                        "player": ui,
+                        "session_memory": temp_memory
+                    },
+                    daemon=True
+                ).start()
+
+        elif intent == "search":
+            if parameters.get("query"):
+                threading.Thread(
+                    target=web_search,
+                    kwargs={
+                        "parameters": parameters,
+                        "player": ui,
+                        "session_memory": temp_memory
+                    },
+                    daemon=True
+                ).start()
+
+        else:
+            if response:
+                ui.write_log(f"AI: {response}")
+                edge_speak(response, ui)
+
+        await asyncio.sleep(0.01)
+
+def main():
+    ui = JarvisUI(BASE_DIR / "face.png", size=(900, 900))
+
+    def runner():
+        asyncio.run(ai_loop(ui))
+
+    threading.Thread(target=runner, daemon=True).start()
+    ui.root.mainloop()
+
+
+if __name__ == "__main__":
+    main()

memory/__init__.py

@@ -0,0 +1 @@
+#memory package 

memory/config_manager.py

@@ -0,0 +1,80 @@
+import json
+import sys
+from pathlib import Path
+
+def get_base_dir():
+    if getattr(sys, "frozen", False):
+        return Path(sys.executable).parent
+    return Path(__file__).resolve().parent.parent
+
+
+BASE_DIR = get_base_dir()
+CONFIG_DIR = BASE_DIR / "config"
+CONFIG_FILE = CONFIG_DIR / "api_keys.json"
+
+def ensure_config_dir() -> None:
+    CONFIG_DIR.mkdir(parents=True, exist_ok=True)
+
+
+def config_exists() -> bool:
+    return CONFIG_FILE.exists()
+
+def save_api_keys(
+    openrouter_api_key: str | None = None,
+    serpapi_api_key: str | None = None
+) -> None:
+    """
+    Saves API keys into config/api_keys.json
+    Only overwrites keys that are provided.
+    """
+
+    ensure_config_dir()
+
+    data: dict = {}
+
+    if CONFIG_FILE.exists():
+        try:
+            data = json.loads(CONFIG_FILE.read_text(encoding="utf-8"))
+        except Exception:
+            data = {}
+
+    if openrouter_api_key is not None:
+        data["openrouter_api_key"] = openrouter_api_key.strip()
+
+    if serpapi_api_key is not None:
+        data["serpapi_api_key"] = serpapi_api_key.strip()
+
+    CONFIG_FILE.write_text(
+        json.dumps(data, indent=2),
+        encoding="utf-8"
+    )
+
+def load_api_keys() -> dict:
+    if not CONFIG_FILE.exists():
+        return {}
+
+    try:
+        return json.loads(CONFIG_FILE.read_text(encoding="utf-8"))
+    except Exception as e:
+        print(f"❌ Failed to load api_keys.json: {e}")
+        return {}
+
+def get_openrouter_key() -> str | None:
+    return load_api_keys().get("openrouter_api_key")
+
+
+def get_serpapi_key() -> str | None:
+    return load_api_keys().get("serpapi_api_key")
+
+def is_openrouter_configured() -> bool:
+    key = get_openrouter_key()
+    return bool(key and len(key) > 15)
+
+
+def is_serpapi_configured() -> bool:
+    key = get_serpapi_key()
+    return bool(key and len(key) > 10)
+
+
+def is_fully_configured() -> bool:
+    return is_openrouter_configured() and is_serpapi_configured()

memory/memory_manager.py

@@ -0,0 +1,83 @@
+# memory/memory_manager.py
+import json
+import os
+from threading import Lock
+from datetime import datetime
+
+MEMORY_PATH = "memory/memory.json"
+_lock = Lock()
+
+
+def _empty_memory() -> dict:
+    """Return an empty memory structure."""
+    return {
+        "identity": {},
+        "preferences": {},
+        "relationships": {},
+        "emotional_state": {}
+    }
+
+
+def load_memory() -> dict:
+    """Load memory from disk, return empty if not exists or invalid."""
+    if not os.path.exists(MEMORY_PATH):
+        return _empty_memory()
+
+    with _lock:
+        try:
+            with open(MEMORY_PATH, "r", encoding="utf-8") as f:
+                data = json.load(f)
+                if isinstance(data, dict):
+                    return data
+                return _empty_memory()
+        except Exception:
+            return _empty_memory()
+
+
+def save_memory(memory: dict) -> None:
+    """Save memory to disk safely."""
+    if not isinstance(memory, dict):
+        return
+
+    os.makedirs(os.path.dirname(MEMORY_PATH), exist_ok=True)
+
+    with _lock:
+        with open(MEMORY_PATH, "w", encoding="utf-8") as f:
+            json.dump(memory, f, indent=2, ensure_ascii=False)
+
+
+def _recursive_update(target: dict, updates: dict) -> bool:
+    """Recursively merge updates into target memory. Returns True if changed."""
+    changed = False
+    now = datetime.utcnow().isoformat() + "Z"
+
+    for key, value in updates.items():
+        if value is None or (isinstance(value, str) and not value.strip()):
+            continue
+
+        if isinstance(value, dict) and "value" not in value:
+            if key not in target or not isinstance(target[key], dict):
+                target[key] = {}
+                changed = True
+            if _recursive_update(target[key], value):
+                changed = True
+        else:
+
+            entry = value if isinstance(value, dict) and "value" in value else {"value": value}
+            if key not in target or target[key] != entry:
+                target[key] = entry
+                changed = True
+
+    return changed
+
+
+def update_memory(memory_update: dict) -> dict:
+    """Merge LLM memory update into global memory and save."""
+    if not isinstance(memory_update, dict):
+        return load_memory()
+
+    memory = load_memory()
+    if _recursive_update(memory, memory_update):
+        save_memory(memory)
+
+    return memory

memory/temporary_memory.py

@@ -0,0 +1,136 @@
+# memory/temporary_memory.py
+from typing import Any
+
+
+class TemporaryMemory:
+    """
+    Temporary runtime memory (session-only).
+    Resets when program restarts.
+
+    Purpose:
+      - Track multi-step intents
+      - Keep short conversation context
+      - Store last actions (search / open_app)
+      - Support follow-up questions like:
+          "why did he executed?"
+    """
+
+    def __init__(self, max_history: int = 5):
+        self.max_history = max_history
+        self.reset()
+
+
+    def reset(self):
+        self.pending_intent: str | None = None
+        self.parameters: dict[str, Any] = {}
+        self.current_question: str | None = None
+
+        self.last_user_text: str | None = None
+        self.last_ai_response: str | None = None
+
+        # --- Action memory ---
+        self.last_search: dict | None = None   
+        self.last_opened_app: str | None = None
+
+        # --- Conversation ---
+        self.conversation_history: list[dict[str, str]] = []
+
+
+    def set_pending_intent(self, intent: str):
+        self.pending_intent = intent
+
+    def clear_pending_intent(self):
+        self.pending_intent = None
+        self.parameters = {}
+        self.current_question = None
+
+    def has_pending_intent(self) -> bool:
+        return self.pending_intent is not None
+
+
+    def update_parameters(self, new_params: dict):
+        if not isinstance(new_params, dict):
+            return
+        for k, v in new_params.items():
+            if v not in (None, ""):
+                self.parameters[k] = v
+
+    def get_parameters(self) -> dict:
+        return self.parameters.copy()
+
+    def get_parameter(self, key: str):
+        return self.parameters.get(key)
+
+    def set_current_question(self, param_name: str):
+        self.current_question = param_name
+
+    def get_current_question(self) -> str | None:
+        return self.current_question
+
+    def clear_current_question(self):
+        self.current_question = None
+
+    def set_last_user_text(self, text: str):
+        self.last_user_text = text
+        self._add_to_history("user", text)
+
+    def set_last_ai_response(self, text: str):
+        self.last_ai_response = text
+        self._add_to_history("ai", text)
+
+    def get_last_user_text(self):
+        return self.last_user_text
+
+    def get_last_ai_response(self):
+        return self.last_ai_response
+
+
+    def set_last_search(self, query: str, answer: str):
+        self.last_search = {
+            "query": query,
+            "answer": answer
+        }
+
+    def get_last_search(self):
+        return self.last_search
+
+    def set_open_app(self, app_name: str):
+        self.last_opened_app = app_name
+
+    def get_last_opened_app(self):
+        return self.last_opened_app
+
+    def _add_to_history(self, role: str, text: str):
+        if role not in ("user", "ai"):
+            return
+
+        self.conversation_history.append({
+            "role": role,
+            "text": text
+        })
+
+        if len(self.conversation_history) > self.max_history:
+            self.conversation_history.pop(0)
+
+    def get_history_for_prompt(self) -> str:
+        """
+        Returns compact history for LLM prompt.
+        """
+        return "\n".join(
+            f"{m['role'].capitalize()}: {m['text']}"
+            for m in self.conversation_history
+        )
+
+
+    def get_context_summary(self) -> dict:
+        """
+        Useful for debugging or smarter LLM prompts.
+        """
+        return {
+            "pending_intent": self.pending_intent,
+            "parameters": self.parameters,
+            "last_search": self.last_search,
+            "last_opened_app": self.last_opened_app,
+            "last_user_text": self.last_user_text,
+            "last_ai_response": self.last_ai_response,
+        }

speech_to_text.py

@@ -0,0 +1,39 @@
+# stt.py
+import sounddevice as sd
+import vosk
+import queue
+import sys
+import json
+import threading
+
+MODEL_PATH = "C:/Users/90553/Downloads/vosk/vosk-model-small-en-us-0.15"  # put your path
+model = vosk.Model(MODEL_PATH)
+
+q = queue.Queue()
+stop_listening_flag = threading.Event()
+
+def callback(indata, frames, time, status):
+    if status:
+        print(status, file=sys.stderr)
+    q.put(bytes(indata))
+
+def record_voice(prompt="🎙 I'm listening, sir..."):
+    """
+    Blocking call, returns the first recognized sentence.
+    """
+    print(prompt)
+    rec = vosk.KaldiRecognizer(model, 16000)
+    with sd.RawInputStream(samplerate=16000, blocksize=8000, dtype='int16',
+                           channels=1, callback=callback):
+        while not stop_listening_flag.is_set():
+            try:
+                data = q.get(timeout=0.1)
+            except queue.Empty:
+                continue
+            if rec.AcceptWaveform(data):
+                result = json.loads(rec.Result())
+                text = result.get("text", "")
+                if text.strip():
+                    print("👤 You:", text)
+                    return text
+    return ""

tts.py

@@ -0,0 +1,78 @@
+import io
+import threading
+import asyncio
+import sounddevice as sd
+import soundfile as sf
+import edge_tts
+
+VOICE = "en-US-AndrewMultilingualNeural"  
+
+RATE = "+0%"     
+VOLUME = "+0%"   
+PITCH = "+0Hz"   
+
+stop_speaking_flag = threading.Event()
+
+def edge_speak(text: str, ui=None, blocking=False):
+    if not text or not text.strip():
+        return
+
+    finished_event = threading.Event()
+
+    def _thread():
+        if ui:
+            ui.start_speaking()
+
+        stop_speaking_flag.clear()
+
+        try:
+            asyncio.run(_speak_async(text))
+        except Exception as e:
+            print("EDGE TTS ERROR:", e)
+        finally:
+            if ui:
+                ui.stop_speaking()
+            finished_event.set()
+
+    threading.Thread(target=_thread, daemon=True).start()
+
+    if blocking:
+        finished_event.wait()
+
+async def _speak_async(text: str):
+    communicate = edge_tts.Communicate(
+        text=text.strip(),
+        voice=VOICE,
+        rate=RATE,
+        volume=VOLUME,
+        pitch=PITCH,
+    )
+
+    audio_bytes = io.BytesIO()
+
+    async for chunk in communicate.stream():
+        if stop_speaking_flag.is_set():
+            return
+
+        if chunk["type"] == "audio":
+            audio_bytes.write(chunk["data"])
+
+    audio_bytes.seek(0)
+
+    data, samplerate = sf.read(audio_bytes, dtype="float32")
+
+    channels = data.shape[1] if len(data.shape) > 1 else 1
+
+    with sd.OutputStream(
+        samplerate=samplerate,
+        channels=channels,
+        dtype="float32",
+    ) as stream:
+        block_size = 1024
+        for start in range(0, len(data), block_size):
+            if stop_speaking_flag.is_set():
+                break
+            stream.write(data[start:start + block_size])
+
+def stop_speaking():
+    stop_speaking_flag.set()

ui.py

@@ -0,0 +1,259 @@
+import os
+import json
+import time
+import random
+import tkinter as tk
+from collections import deque
+from PIL import Image, ImageTk, ImageDraw, ImageFilter
+from tkinter.scrolledtext import ScrolledText
+import sys
+from pathlib import Path
+
+def get_base_dir():
+    if getattr(sys, "frozen", False):
+        return Path(sys.executable).parent
+    return Path(__file__).resolve().parent
+
+BASE_DIR = get_base_dir()
+CONFIG_DIR = BASE_DIR / "config"
+API_FILE = CONFIG_DIR / "api_keys.json"
+
+def get_base_dir():
+    if getattr(sys, "frozen", False):
+        return Path(sys.executable).parent
+    return Path(__file__).resolve().parent
+
+BASE_DIR = get_base_dir()
+CONFIG_DIR = BASE_DIR / "config"
+API_FILE = CONFIG_DIR / "api_keys.json"
+class JarvisUI:
+    def __init__(self, face_path, size=(760, 760)):
+        self.root = tk.Tk()
+        self.root.title("J.A.R.V.I.S")
+        self.root.resizable(False, False)
+        self.root.geometry("760x900")
+        self.root.configure(bg="#000000")
+
+        self.size = size
+        self.center_y = 0.42
+
+        self.canvas = tk.Canvas(
+            self.root,
+            width=size[0],
+            height=size[1],
+            bg="#000000",
+            highlightthickness=0
+        )
+        self.canvas.place(relx=0.5, rely=self.center_y, anchor="center")
+
+        self.face_base = (
+            Image.open(face_path)
+            .convert("RGBA")
+            .resize(size, Image.LANCZOS)
+        )
+
+        self.halo_base = self._create_halo(size, radius=220, y_offset=-50)
+
+        self.speaking = False
+        self.scale = 1.0
+        self.target_scale = 1.0
+        self.halo_alpha = 70
+        self.target_halo_alpha = 70
+        self.last_target_time = time.time()
+
+        self.text_box = ScrolledText(
+            self.root,
+            fg="#8ffcff",
+            bg="#000000",
+            insertbackground="#8ffcff",
+            height=12,
+            borderwidth=0,
+            wrap="word",
+            font=("Consolas", 10),
+            padx=12,
+            pady=12
+        )
+        self.text_box.place(relx=0.5, rely=0.86, anchor="center")
+        self.text_box.configure(state="disabled")
+
+        self.typing_queue = deque()
+        self.is_typing = False
+
+        if not self._api_keys_exist():
+            self._show_setup_ui()
+
+        self._animate()
+        self.root.protocol("WM_DELETE_WINDOW", lambda: os._exit(0))
+
+    def _api_keys_exist(self):
+        return os.path.exists(API_FILE)
+
+    def _show_setup_ui(self):
+        self.setup_frame = tk.Frame(
+            self.root,
+            bg="#050505",
+            highlightbackground="#00cfff",
+            highlightthickness=1
+        )
+        self.setup_frame.place(relx=0.5, rely=0.5, anchor="center")
+
+        tk.Label(
+            self.setup_frame,
+            text="JARVIS SETUP",
+            fg="#8ffcff",
+            bg="#050505",
+            font=("Consolas", 14, "bold")
+        ).pack(pady=(15, 10))
+
+        self.openrouter_entry = self._setup_entry(
+            "OpenRouter API Key"
+        )
+        self.serpapi_entry = self._setup_entry(
+            "SerpAPI Key"
+        )
+
+        tk.Button(
+            self.setup_frame,
+            text="SAVE & CONTINUE",
+            command=self._save_api_keys,
+            bg="#000000",
+            fg="#8ffcff",
+            activebackground="#003344",
+            font=("Consolas", 10),
+            borderwidth=0
+        ).pack(pady=15)
+
+    def _setup_entry(self, label_text):
+        tk.Label(
+            self.setup_frame,
+            text=label_text,
+            fg="#8ffcff",
+            bg="#050505",
+            font=("Consolas", 10)
+        ).pack(pady=(8, 2))
+
+        entry = tk.Entry(
+            self.setup_frame,
+            width=46,
+            fg="#8ffcff",
+            bg="#000000",
+            insertbackground="#8ffcff",
+            borderwidth=0,
+            font=("Consolas", 10)
+        )
+        entry.pack(pady=(0, 6))
+        return entry
+
+    def _save_api_keys(self):
+        openrouter_key = self.openrouter_entry.get().strip()
+        serpapi_key = self.serpapi_entry.get().strip()
+
+        if not openrouter_key:
+            return  
+
+        os.makedirs(CONFIG_DIR, exist_ok=True)
+
+        with open(API_FILE, "w", encoding="utf-8") as f:
+            json.dump(
+                {
+                    "openrouter_api_key": openrouter_key,
+                    "serpapi_api_key": serpapi_key
+                },
+                f,
+                indent=4
+            )
+
+        self.setup_frame.destroy()
+        self.write_log("API keys saved successfully.")
+
+    def _create_halo(self, size, radius, y_offset):
+        w, h = size
+        img = Image.new("RGBA", size, (0, 0, 0, 0))
+        draw = ImageDraw.Draw(img)
+
+        cx = w // 2
+        cy = h // 2 + y_offset
+
+        for r in range(radius, 0, -12):
+            alpha = int(70 * (1 - r / radius))
+            draw.ellipse(
+                (cx - r, cy - r, cx + r, cy + r),
+                fill=(0, 180, 255, alpha)
+            )
+
+        return img.filter(ImageFilter.GaussianBlur(30))
+
+    def write_log(self, text: str):
+        self.typing_queue.append(text)
+        if not self.is_typing:
+            self._start_typing()
+
+    def _start_typing(self):
+        if not self.typing_queue:
+            self.is_typing = False
+            return
+
+        self.is_typing = True
+        text = self.typing_queue.popleft()
+
+        self.text_box.configure(state="normal")
+        self._type_char(text, 0)
+
+    def _type_char(self, text, i):
+        if i < len(text):
+            self.text_box.insert(tk.END, text[i])
+            self.text_box.see(tk.END)
+            self.root.after(12, self._type_char, text, i + 1)
+        else:
+            self.text_box.insert(tk.END, "\n")
+            self.text_box.configure(state="disabled")
+            self.root.after(40, self._start_typing)
+
+
+    def start_speaking(self):
+        self.speaking = True
+
+    def stop_speaking(self):
+        self.speaking = False
+
+    def _animate(self):
+        now = time.time()
+
+        if now - self.last_target_time > (0.25 if self.speaking else 0.7):
+            if self.speaking:
+                self.target_scale = random.uniform(1.02, 1.1)
+                self.target_halo_alpha = random.randint(120, 150)
+            else:
+                self.target_scale = random.uniform(1.004, 1.012)
+                self.target_halo_alpha = random.randint(60, 80)
+
+            self.last_target_time = now
+
+        scale_speed = 0.45 if self.speaking else 0.25
+        halo_speed = 0.40 if self.speaking else 0.25
+
+        self.scale += (self.target_scale - self.scale) * scale_speed
+        self.halo_alpha += (self.target_halo_alpha - self.halo_alpha) * halo_speed
+
+        frame = Image.new("RGBA", self.size, (0, 0, 0, 255))
+
+        halo = self.halo_base.copy()
+        halo.putalpha(int(self.halo_alpha))
+        frame.alpha_composite(halo)
+
+        w, h = self.size
+        face = self.face_base.resize(
+            (int(w * self.scale), int(h * self.scale)),
+            Image.LANCZOS
+        )
+
+        fx = (w - face.size[0]) // 2
+        fy = (h - face.size[1]) // 2
+        frame.alpha_composite(face, (fx, fy))
+
+        img = ImageTk.PhotoImage(frame)
+        self.canvas.delete("all")
+        self.canvas.create_image(w // 2, h // 2, image=img)
+        self.canvas.image = img
+
+        self.root.after(16, self._animate)