554 lines
18 KiB
Python
554 lines
18 KiB
Python
# actions/computer_control.py
|
|
# MARK XXV — Computer Control
|
|
#
|
|
# Atomic computer control functions using PyAutoGUI + keyboard + clipboard.
|
|
# Used by the agent when no existing action file covers the task.
|
|
#
|
|
# Capabilities:
|
|
# - Type text anywhere (active window, forms, fields)
|
|
# - Mouse click, double-click, right-click, drag
|
|
# - Keyboard shortcuts and key combinations
|
|
# - Scroll (up/down/left/right)
|
|
# - Window management (minimize, maximize, close, focus)
|
|
# - Clipboard (copy, paste, get content)
|
|
# - Screenshot + locate element on screen
|
|
# - Wait / smart wait for element to appear
|
|
# - Random data generation (name, email, username, password, phone, address)
|
|
# - Hotkey sequences
|
|
# - Find and click image/element on screen
|
|
|
|
import json
|
|
import sys
|
|
import time
|
|
import random
|
|
import string
|
|
import subprocess
|
|
import platform
|
|
from pathlib import Path
|
|
|
|
try:
|
|
import pyautogui
|
|
pyautogui.FAILSAFE = True
|
|
pyautogui.PAUSE = 0.05
|
|
_PYAUTOGUI = True
|
|
except ImportError:
|
|
_PYAUTOGUI = False
|
|
|
|
try:
|
|
import pyperclip
|
|
_PYPERCLIP = True
|
|
except ImportError:
|
|
_PYPERCLIP = False
|
|
|
|
|
|
def get_base_dir() -> Path:
|
|
if getattr(sys, "frozen", False):
|
|
return Path(sys.executable).parent
|
|
return Path(__file__).resolve().parent.parent
|
|
|
|
|
|
BASE_DIR = get_base_dir()
|
|
API_CONFIG_PATH = BASE_DIR / "config" / "api_keys.json"
|
|
|
|
|
|
def _load_user_profile() -> dict:
|
|
"""Load user profile from long_term.json for form filling."""
|
|
memory_path = BASE_DIR / "memory" / "long_term.json"
|
|
try:
|
|
if memory_path.exists():
|
|
data = json.loads(memory_path.read_text(encoding="utf-8"))
|
|
identity = data.get("identity", {})
|
|
return {
|
|
"name": identity.get("name", {}).get("value", ""),
|
|
"age": identity.get("age", {}).get("value", ""),
|
|
"city": identity.get("city", {}).get("value", ""),
|
|
"email": identity.get("email", {}).get("value", ""),
|
|
}
|
|
except Exception:
|
|
pass
|
|
return {}
|
|
|
|
|
|
def _ensure_pyautogui():
|
|
if not _PYAUTOGUI:
|
|
raise RuntimeError(
|
|
"PyAutoGUI not installed. Run: pip install pyautogui"
|
|
)
|
|
|
|
|
|
_FIRST_NAMES = [
|
|
"Alex", "Jordan", "Taylor", "Morgan", "Casey", "Riley", "Drew", "Quinn",
|
|
"Avery", "Blake", "Cameron", "Dakota", "Emerson", "Finley", "Harper"
|
|
]
|
|
_LAST_NAMES = [
|
|
"Smith", "Johnson", "Williams", "Brown", "Jones", "Garcia", "Miller",
|
|
"Davis", "Wilson", "Moore", "Taylor", "Anderson", "Thomas", "Jackson"
|
|
]
|
|
_DOMAINS = ["gmail.com", "yahoo.com", "outlook.com", "proton.me", "mail.com"]
|
|
|
|
|
|
def generate_random_data(data_type: str) -> str:
|
|
"""
|
|
Generates random realistic data for form filling.
|
|
|
|
Types: name | first_name | last_name | email | username |
|
|
password | phone | birthday | address | zip_code
|
|
"""
|
|
dt = data_type.lower().strip()
|
|
|
|
if dt == "first_name":
|
|
return random.choice(_FIRST_NAMES)
|
|
|
|
elif dt == "last_name":
|
|
return random.choice(_LAST_NAMES)
|
|
|
|
elif dt == "name":
|
|
return f"{random.choice(_FIRST_NAMES)} {random.choice(_LAST_NAMES)}"
|
|
|
|
elif dt == "email":
|
|
first = random.choice(_FIRST_NAMES).lower()
|
|
last = random.choice(_LAST_NAMES).lower()
|
|
num = random.randint(10, 999)
|
|
return f"{first}.{last}{num}@{random.choice(_DOMAINS)}"
|
|
|
|
elif dt == "username":
|
|
first = random.choice(_FIRST_NAMES).lower()
|
|
num = random.randint(100, 9999)
|
|
return f"{first}{num}"
|
|
|
|
elif dt == "password":
|
|
chars = string.ascii_letters + string.digits + "!@#$%"
|
|
pwd = (
|
|
random.choice(string.ascii_uppercase) +
|
|
random.choice(string.digits) +
|
|
random.choice("!@#$%") +
|
|
"".join(random.choices(chars, k=9))
|
|
)
|
|
return "".join(random.sample(pwd, len(pwd)))
|
|
|
|
elif dt == "phone":
|
|
return f"+1{random.randint(200,999)}{random.randint(1000000,9999999)}"
|
|
|
|
elif dt == "birthday":
|
|
year = random.randint(1980, 2000)
|
|
month = random.randint(1, 12)
|
|
day = random.randint(1, 28)
|
|
return f"{month:02d}/{day:02d}/{year}"
|
|
|
|
elif dt == "address":
|
|
num = random.randint(100, 9999)
|
|
street = random.choice(["Main St", "Oak Ave", "Park Blvd", "Elm St", "Cedar Ln"])
|
|
return f"{num} {street}"
|
|
|
|
elif dt == "zip_code":
|
|
return str(random.randint(10000, 99999))
|
|
|
|
elif dt == "city":
|
|
return random.choice(["New York", "Los Angeles", "Chicago", "Houston", "Phoenix"])
|
|
|
|
return f"random_{data_type}_{random.randint(1000,9999)}"
|
|
|
|
|
|
def _type_text(text: str, interval: float = 0.03) -> str:
|
|
"""Types text at the current cursor position."""
|
|
_ensure_pyautogui()
|
|
time.sleep(0.3)
|
|
pyautogui.typewrite(text, interval=interval)
|
|
return f"Typed: {text[:50]}{'...' if len(text) > 50 else ''}"
|
|
|
|
|
|
def _click(x: int = None, y: int = None, button: str = "left",
|
|
clicks: int = 1, image: str = None) -> str:
|
|
"""
|
|
Clicks at coordinates or on a screen image.
|
|
If image path given, locates it on screen and clicks.
|
|
"""
|
|
_ensure_pyautogui()
|
|
|
|
if image:
|
|
try:
|
|
loc = pyautogui.locateCenterOnScreen(image, confidence=0.8)
|
|
if loc:
|
|
pyautogui.click(loc.x, loc.y, button=button, clicks=clicks)
|
|
return f"Clicked image: {image}"
|
|
return f"Image not found on screen: {image}"
|
|
except Exception as e:
|
|
return f"Image click failed: {e}"
|
|
|
|
if x is not None and y is not None:
|
|
pyautogui.click(x, y, button=button, clicks=clicks)
|
|
return f"Clicked ({x}, {y}) with {button} button"
|
|
|
|
pyautogui.click(button=button, clicks=clicks)
|
|
return f"Clicked at current position"
|
|
|
|
|
|
def _hotkey(*keys) -> str:
|
|
"""Presses a key combination. E.g. hotkey('ctrl', 'c')"""
|
|
_ensure_pyautogui()
|
|
pyautogui.hotkey(*keys)
|
|
return f"Hotkey: {'+'.join(keys)}"
|
|
|
|
|
|
def _press(key: str) -> str:
|
|
"""Presses a single key."""
|
|
_ensure_pyautogui()
|
|
pyautogui.press(key)
|
|
return f"Pressed: {key}"
|
|
|
|
|
|
def _scroll(direction: str = "down", amount: int = 3) -> str:
|
|
"""Scrolls in the specified direction."""
|
|
_ensure_pyautogui()
|
|
clicks = amount if direction in ("up", "right") else -amount
|
|
if direction in ("up", "down"):
|
|
pyautogui.scroll(clicks)
|
|
else:
|
|
pyautogui.hscroll(clicks)
|
|
return f"Scrolled {direction} {amount} times"
|
|
|
|
|
|
def _move_mouse(x: int, y: int, duration: float = 0.3) -> str:
|
|
"""Moves mouse to coordinates."""
|
|
_ensure_pyautogui()
|
|
pyautogui.moveTo(x, y, duration=duration)
|
|
return f"Mouse moved to ({x}, {y})"
|
|
|
|
|
|
def _drag(x1: int, y1: int, x2: int, y2: int, duration: float = 0.5) -> str:
|
|
"""Drags from (x1,y1) to (x2,y2)."""
|
|
_ensure_pyautogui()
|
|
pyautogui.drag(x1 - pyautogui.position()[0], y1 - pyautogui.position()[1])
|
|
pyautogui.dragTo(x2, y2, duration=duration)
|
|
return f"Dragged from ({x1},{y1}) to ({x2},{y2})"
|
|
|
|
|
|
def _clipboard_copy() -> str:
|
|
"""Gets current clipboard content."""
|
|
if _PYPERCLIP:
|
|
return pyperclip.paste()
|
|
_hotkey("ctrl", "c")
|
|
time.sleep(0.2)
|
|
return "Copied to clipboard"
|
|
|
|
|
|
def _clipboard_set(text: str) -> str:
|
|
"""Sets clipboard content and pastes it."""
|
|
if _PYPERCLIP:
|
|
pyperclip.copy(text)
|
|
time.sleep(0.1)
|
|
_hotkey("ctrl", "v")
|
|
return f"Pasted: {text[:50]}"
|
|
return "pyperclip not available"
|
|
|
|
|
|
def _screenshot(save_path: str = None) -> str:
|
|
"""Takes a screenshot."""
|
|
_ensure_pyautogui()
|
|
if not save_path:
|
|
save_path = str(Path.home() / "Desktop" / "screenshot.png")
|
|
img = pyautogui.screenshot()
|
|
img.save(save_path)
|
|
return f"Screenshot saved: {save_path}"
|
|
|
|
|
|
def _wait(seconds: float) -> str:
|
|
"""Waits for specified seconds."""
|
|
time.sleep(seconds)
|
|
return f"Waited {seconds}s"
|
|
|
|
|
|
def _wait_for_image(image_path: str, timeout: int = 10) -> str:
|
|
"""Waits until an image appears on screen (up to timeout seconds)."""
|
|
_ensure_pyautogui()
|
|
start = time.time()
|
|
while time.time() - start < timeout:
|
|
try:
|
|
loc = pyautogui.locateCenterOnScreen(image_path, confidence=0.8)
|
|
if loc:
|
|
return f"Image found at ({loc.x}, {loc.y})"
|
|
except Exception:
|
|
pass
|
|
time.sleep(0.5)
|
|
return f"Image not found within {timeout}s: {image_path}"
|
|
|
|
|
|
def _get_screen_size() -> str:
|
|
"""Returns current screen resolution."""
|
|
_ensure_pyautogui()
|
|
w, h = pyautogui.size()
|
|
return f"{w}x{h}"
|
|
|
|
|
|
def _focus_window(title: str) -> str:
|
|
"""Brings a window to focus by title (Windows)."""
|
|
if platform.system() == "Windows":
|
|
try:
|
|
script = f'(New-Object -ComObject WScript.Shell).AppActivate("{title}")'
|
|
subprocess.run(
|
|
["powershell", "-NoProfile", "-Command", script],
|
|
capture_output=True, timeout=5
|
|
)
|
|
time.sleep(0.3)
|
|
return f"Focused window: {title}"
|
|
except Exception as e:
|
|
return f"Could not focus window: {e}"
|
|
return "Window focus only supported on Windows"
|
|
|
|
|
|
def _select_all() -> str:
|
|
return _hotkey("ctrl", "a")
|
|
|
|
|
|
def _clear_field() -> str:
|
|
"""Selects all and deletes — clears an input field."""
|
|
_hotkey("ctrl", "a")
|
|
time.sleep(0.1)
|
|
_press("delete")
|
|
return "Field cleared"
|
|
|
|
|
|
def _smart_type(text: str, clear_first: bool = True) -> str:
|
|
"""
|
|
Types text into the currently focused field.
|
|
Optionally clears the field first.
|
|
Uses clipboard for long text (faster, more reliable).
|
|
"""
|
|
_ensure_pyautogui()
|
|
|
|
if clear_first:
|
|
_clear_field()
|
|
time.sleep(0.1)
|
|
|
|
if len(text) > 20 and _PYPERCLIP:
|
|
pyperclip.copy(text)
|
|
time.sleep(0.1)
|
|
pyautogui.hotkey("ctrl", "v")
|
|
return f"Smart-typed (clipboard): {text[:50]}"
|
|
else:
|
|
pyautogui.typewrite(text, interval=0.04)
|
|
return f"Smart-typed: {text[:50]}"
|
|
|
|
|
|
def _analyze_screen_for_element(description: str) -> tuple[int, int] | None:
|
|
"""
|
|
Takes a screenshot and asks Gemini to find the coordinates
|
|
of a described element on screen. Returns (x, y) or None.
|
|
"""
|
|
try:
|
|
import google.generativeai as genai
|
|
import io
|
|
|
|
cfg_path = API_CONFIG_PATH
|
|
with open(cfg_path, "r") as f:
|
|
api_key = json.load(f)["gemini_api_key"]
|
|
|
|
genai.configure(api_key=api_key)
|
|
model = genai.GenerativeModel("gemini-2.5-flash-lite")
|
|
|
|
|
|
_ensure_pyautogui()
|
|
w, h = pyautogui.size()
|
|
img = pyautogui.screenshot()
|
|
buf = io.BytesIO()
|
|
img.save(buf, format="PNG")
|
|
buf.seek(0)
|
|
|
|
prompt = (
|
|
f"This is a screenshot of a computer screen ({w}x{h} pixels). "
|
|
f"Find the element: '{description}'. "
|
|
f"Return ONLY: x,y (the center coordinates of the element). "
|
|
f"If not found, return: NOT_FOUND"
|
|
)
|
|
|
|
response = model.generate_content([
|
|
{"mime_type": "image/png", "data": buf.getvalue()},
|
|
prompt
|
|
])
|
|
|
|
text = response.text.strip()
|
|
if "NOT_FOUND" in text:
|
|
return None
|
|
|
|
import re
|
|
match = re.search(r"(\d+)\s*,\s*(\d+)", text)
|
|
if match:
|
|
return int(match.group(1)), int(match.group(2))
|
|
|
|
except Exception as e:
|
|
print(f"[ComputerControl] ⚠️ Screen analysis failed: {e}")
|
|
|
|
return None
|
|
|
|
def computer_control(
|
|
parameters: dict,
|
|
response=None,
|
|
player=None,
|
|
session_memory=None,
|
|
) -> str:
|
|
"""
|
|
Universal computer control action.
|
|
|
|
Actions:
|
|
type : Type text at current cursor position
|
|
smart_type : Clear field + type (clipboard-based for long text)
|
|
click : Click at coordinates or on image
|
|
double_click : Double click
|
|
right_click : Right click
|
|
hotkey : Key combination (e.g. ctrl+c)
|
|
press : Single key press
|
|
scroll : Scroll up/down/left/right
|
|
move : Move mouse to coordinates
|
|
drag : Drag from one point to another
|
|
copy : Get clipboard content
|
|
paste : Set and paste clipboard content
|
|
screenshot : Take a screenshot
|
|
wait : Wait N seconds
|
|
wait_image : Wait for image to appear on screen
|
|
clear_field : Select all + delete in current field
|
|
focus_window : Bring window to foreground
|
|
screen_find : AI-powered element finder — returns coordinates
|
|
screen_click : AI-powered element finder + click
|
|
random_data : Generate random data for forms
|
|
user_data : Get user's real data from memory
|
|
"""
|
|
action = (parameters or {}).get("action", "").lower().strip()
|
|
|
|
if not action:
|
|
return "Please specify an action for computer_control, sir."
|
|
|
|
if player:
|
|
player.write_log(f"[Computer] {action}")
|
|
|
|
print(f"[ComputerControl] ▶️ Action: {action} Params: {parameters}")
|
|
|
|
try:
|
|
if action == "type":
|
|
text = parameters.get("text", "")
|
|
return _type_text(text)
|
|
|
|
elif action == "smart_type":
|
|
text = parameters.get("text", "")
|
|
clear_first = parameters.get("clear_first", True)
|
|
return _smart_type(text, clear_first=clear_first)
|
|
|
|
elif action in ("click", "left_click"):
|
|
return _click(
|
|
x=parameters.get("x"),
|
|
y=parameters.get("y"),
|
|
button="left",
|
|
clicks=1,
|
|
image=parameters.get("image")
|
|
)
|
|
|
|
elif action == "double_click":
|
|
return _click(
|
|
x=parameters.get("x"),
|
|
y=parameters.get("y"),
|
|
button="left",
|
|
clicks=2,
|
|
image=parameters.get("image")
|
|
)
|
|
|
|
elif action == "right_click":
|
|
return _click(
|
|
x=parameters.get("x"),
|
|
y=parameters.get("y"),
|
|
button="right",
|
|
clicks=1
|
|
)
|
|
|
|
elif action == "move":
|
|
return _move_mouse(
|
|
x=int(parameters.get("x", 0)),
|
|
y=int(parameters.get("y", 0)),
|
|
duration=float(parameters.get("duration", 0.3))
|
|
)
|
|
|
|
elif action == "drag":
|
|
return _drag(
|
|
x1=int(parameters.get("x1", 0)),
|
|
y1=int(parameters.get("y1", 0)),
|
|
x2=int(parameters.get("x2", 0)),
|
|
y2=int(parameters.get("y2", 0))
|
|
)
|
|
|
|
elif action == "hotkey":
|
|
keys = parameters.get("keys", "")
|
|
if isinstance(keys, str):
|
|
keys = [k.strip() for k in keys.split("+")]
|
|
return _hotkey(*keys)
|
|
|
|
elif action == "press":
|
|
return _press(parameters.get("key", "enter"))
|
|
|
|
elif action == "scroll":
|
|
return _scroll(
|
|
direction=parameters.get("direction", "down"),
|
|
amount=int(parameters.get("amount", 3))
|
|
)
|
|
|
|
elif action == "copy":
|
|
return _clipboard_copy()
|
|
|
|
elif action == "paste":
|
|
return _clipboard_set(parameters.get("text", ""))
|
|
|
|
elif action == "screenshot":
|
|
return _screenshot(parameters.get("path"))
|
|
|
|
elif action == "wait":
|
|
return _wait(float(parameters.get("seconds", 1.0)))
|
|
|
|
elif action == "wait_image":
|
|
return _wait_for_image(
|
|
parameters.get("image", ""),
|
|
timeout=int(parameters.get("timeout", 10))
|
|
)
|
|
|
|
elif action == "clear_field":
|
|
return _clear_field()
|
|
|
|
elif action == "focus_window":
|
|
return _focus_window(parameters.get("title", ""))
|
|
|
|
elif action == "screen_size":
|
|
return _get_screen_size()
|
|
|
|
elif action == "screen_find":
|
|
description = parameters.get("description", "")
|
|
coords = _analyze_screen_for_element(description)
|
|
if coords:
|
|
return f"{coords[0]},{coords[1]}"
|
|
return "NOT_FOUND"
|
|
|
|
elif action == "screen_click":
|
|
description = parameters.get("description", "")
|
|
coords = _analyze_screen_for_element(description)
|
|
if coords:
|
|
time.sleep(0.2)
|
|
_click(x=coords[0], y=coords[1])
|
|
return f"Found and clicked: {description} at {coords}"
|
|
return f"Could not find on screen: {description}"
|
|
|
|
elif action == "random_data":
|
|
data_type = parameters.get("type", "name")
|
|
result = generate_random_data(data_type)
|
|
print(f"[ComputerControl] 🎲 Random {data_type}: {result}")
|
|
return result
|
|
|
|
elif action == "user_data":
|
|
field = parameters.get("field", "name")
|
|
profile = _load_user_profile()
|
|
value = profile.get(field, "")
|
|
if not value:
|
|
value = generate_random_data(field)
|
|
print(f"[ComputerControl] ⚠️ No user {field} in memory, using random: {value}")
|
|
return value
|
|
|
|
else:
|
|
return f"Unknown computer_control action: '{action}'"
|
|
|
|
except Exception as e:
|
|
print(f"[ComputerControl] ❌ Error: {e}")
|
|
return f"computer_control failed: {e}" |