#!/usr/bin/env python3 """Run the m3taversal-style direct-claim suite through live VPS GatewayRunner. The harness does not post to Telegram and does not write to the production KB. It copies the live leoclean profile to a temporary profile on the VPS, invokes GatewayRunner._handle_message with Telegram-shaped MessageEvents, and removes the temporary profile after the run. """ from __future__ import annotations import json import re import subprocess import uuid from pathlib import Path from typing import Any import working_leo_open_ended_benchmark as benchmark ROOT = Path(__file__).resolve().parents[1] REPORT_DIR = ROOT / "docs" / "reports" / "leo-working-state-20260709" OUTPUT_JSON = REPORT_DIR / "telegram-handler-direct-claim-suite-current.json" SSH_KEY = Path.home() / ".ssh/livingip_hetzner_20260604_ed25519" VPS = "root@77.42.65.182" DEFAULT_SUITE_MODE = "live_vps_gatewayrunner_temp_profile_direct_claim_suite" DEFAULT_REPORT_PREFIX = "leo-direct-claim-handler-report" DEFAULT_PROMPT_NOTE = "Prompts are the exact DC-01..DC-06 direct-claim benchmark messages." CHAT_ID = "-5146042086" USER_ID = "9070919" USER_NAME = "codex handler direct claim" SECRET_PATTERNS = [ re.compile(r"\b\d{6,}:[A-Za-z0-9_-]{20,}\b"), re.compile(r"(Authorization: Bearer )([A-Za-z0-9._-]+)", re.IGNORECASE), re.compile(r"((?:api[_-]?key|token|secret|password)[=:]\s*)([A-Za-z0-9._-]+)", re.IGNORECASE), ] REMOTE_SCRIPT = r''' import asyncio import json import os import shutil import subprocess import sys import tempfile import traceback from datetime import datetime, timezone from pathlib import Path LIVE_PROFILE = Path("/home/teleo/.hermes/profiles/leoclean") AGENT_ROOT = Path("/home/teleo/.hermes/hermes-agent") CHAT_ID = "__CHAT_ID__" USER_ID = "__USER_ID__" USER_NAME = "__USER_NAME__" RUN_ID = "__RUN_ID__" PROMPTS = __PROMPTS_JSON__ REPORT_PREFIX = __REPORT_PREFIX_JSON__ REPORT_PATH = Path(f"/tmp/{REPORT_PREFIX}-{RUN_ID}.json") def service_state(): raw = subprocess.check_output( [ "systemctl", "show", "leoclean-gateway.service", "-p", "ActiveState", "-p", "SubState", "-p", "MainPID", "-p", "NRestarts", "-p", "ExecMainStartTimestamp", ], text=True, ) result = {} for line in raw.splitlines(): if "=" in line: key, value = line.split("=", 1) result[key] = value return result def db_counts(): sql = """ select jsonb_build_object( 'public.claims', (select count(*) from public.claims), 'public.sources', (select count(*) from public.sources), 'public.claim_evidence', (select count(*) from public.claim_evidence), 'public.claim_edges', (select count(*) from public.claim_edges), 'kb_stage.kb_proposals', (select count(*) from kb_stage.kb_proposals) )::text; """ try: raw = subprocess.check_output( ["docker", "exec", "-i", "teleo-pg", "psql", "-U", "postgres", "-d", "teleo", "-At", "-q"], input=sql, text=True, stderr=subprocess.STDOUT, ) return json.loads(raw.strip()) except Exception as exc: return {"error": str(exc)} def copy_profile() -> Path: tmp_root = Path(tempfile.mkdtemp(prefix="leo-direct-claim-handler-")) target = tmp_root / "profile" def ignore(_dir, names): ignored = { "logs", "cache", "image_cache", "memory_backups", ".pytest_cache", "__pycache__", "gateway.pid", "auth.lock", } return { name for name in names if name in ignored or name.endswith(".lock") or ".bak-" in name or name.endswith(".bak") } try: shutil.copytree(LIVE_PROFILE, target, symlinks=True, ignore=ignore) except Exception: shutil.rmtree(tmp_root, ignore_errors=True) raise return target def remove_tree(path): if not path or not path.exists(): return True try: shutil.rmtree(path) except Exception: for root, dirs, files in os.walk(path, topdown=False): for name in files: p = Path(root) / name try: p.chmod(0o600) p.unlink() except Exception: pass for name in dirs: p = Path(root) / name try: p.chmod(0o700) p.rmdir() except Exception: pass try: path.chmod(0o700) path.rmdir() except Exception: pass return not path.exists() def write_report_snapshot(report): try: REPORT_PATH.write_text(json.dumps(report, sort_keys=True), encoding="utf-8") except Exception: pass def read_database_context_trace(path): if not path or not path.exists(): return [] records = [] for line in path.read_text(encoding="utf-8").splitlines(): try: value = json.loads(line) except json.JSONDecodeError: continue if isinstance(value, dict): records.append(value) return records async def run_suite(): before_service = service_state() before_counts = db_counts() temp_profile = None context_trace_path = None temp_removed = False report = { "generated_at_utc": datetime.now(timezone.utc).isoformat(), "mode": __SUITE_MODE_JSON__, "posted_to_telegram": False, "changed_live_profile": False, "mutates_kb_by_harness": False, "source": { "platform": "telegram", "chat_id": CHAT_ID, "chat_name": "Leo", "chat_type": "group", "user_id": USER_ID, "user_name": USER_NAME, }, "harness_notes": [ __PROMPT_NOTE_JSON__, "The harness uses a temporary copy of the leoclean profile and does not post to Telegram.", "This proves handler-level live VPS GatewayRunner behavior, not human-visible Telegram delivery.", ], "db_counts_before": before_counts, "before_service": before_service, "remote_report_path": str(REPORT_PATH), } write_report_snapshot(report) try: temp_profile = copy_profile() os.environ["HERMES_HOME"] = str(temp_profile) os.environ["HOME"] = "/home/teleo" context_trace_path = temp_profile / "state" / "leo-db-context-trace.jsonl" os.environ["LEO_DB_CONTEXT_TRACE_PATH"] = str(context_trace_path) sys.path.insert(0, str(AGENT_ROOT)) from gateway.config import Platform from gateway.platforms.base import MessageEvent, MessageType from gateway.run import GatewayRunner from gateway.session import SessionSource source = SessionSource( platform=Platform.TELEGRAM, chat_id=CHAT_ID, chat_name="Leo", chat_type="group", user_id=USER_ID, user_name=USER_NAME, ) runner = GatewayRunner() session_key = runner._session_key_for_source(source) authorized = runner._is_user_authorized(source) report["handler"] = { "temp_profile": str(temp_profile), "session_key": session_key, "authorized": authorized, } write_report_snapshot(report) if not authorized: report["results"] = [] report["pass_runtime"] = False report["failure"] = "telegram group source was not authorized by live config" write_report_snapshot(report) return report results = [] for index, prompt in enumerate(PROMPTS, start=1): trace_start = len(read_database_context_trace(context_trace_path)) event = MessageEvent( text=prompt["message"], message_type=MessageType.TEXT, source=source, message_id=f"handler-suite-{RUN_ID}-{prompt['id']}", ) started = datetime.now(timezone.utc) reply = await asyncio.wait_for(runner._handle_message(event), timeout=300) ended = datetime.now(timezone.utc) results.append( { "turn": index, "prompt_id": prompt["id"], "dimension": prompt["dimension"], "prompt": prompt["message"], "reply": reply or "", "started_at_utc": started.isoformat(), "ended_at_utc": ended.isoformat(), "ok": bool((reply or "").strip()), "mutates_kb": False, "evidence_tier": "live_vps_gatewayrunner_temp_profile", "claim_ceiling": "Live VPS GatewayRunner reply from temp leoclean profile; no Telegram post; no production DB apply authorized.", "database_context_trace": read_database_context_trace(context_trace_path)[trace_start:], } ) report["results"] = results write_report_snapshot(report) report["results"] = results report["pass_runtime"] = all(row["ok"] for row in results) return report except Exception as exc: report["pass_runtime"] = False report["error"] = str(exc) report["traceback_tail"] = traceback.format_exc().splitlines()[-12:] write_report_snapshot(report) return report finally: after_counts = db_counts() after_service = service_state() report["db_counts_after"] = after_counts report["db_counts_changed"] = after_counts != before_counts report["service_before_after"] = { "before": before_service, "after": after_service, "unchanged_from_preexisting_live_readback": before_service == after_service, } database_context_trace = read_database_context_trace(context_trace_path) if temp_profile is not None: tmp_root = temp_profile.parent temp_removed = remove_tree(tmp_root) report["temp_profile_removed"] = temp_removed report["database_context_trace"] = database_context_trace report["database_context_injection_count"] = len(report["database_context_trace"]) report["database_context_all_ok"] = bool(report["database_context_trace"]) and all( item.get("status") == "ok" and item.get("injected") is True for item in report["database_context_trace"] ) write_report_snapshot(report) print(json.dumps(asyncio.run(run_suite()), sort_keys=True)) ''' def direct_claim_prompts() -> list[dict[str, Any]]: return [ { "id": prompt["id"], "dimension": prompt["dimension"], "message": prompt["message"], } for prompt in benchmark.M3TAVERSAL_DIRECT_CLAIM_FOLLOWUP_SCENARIOS ] def build_remote_script( run_id: str, *, prompts: list[dict[str, Any]] | None = None, suite_mode: str = DEFAULT_SUITE_MODE, report_prefix: str = DEFAULT_REPORT_PREFIX, prompt_note: str = DEFAULT_PROMPT_NOTE, ) -> str: if not re.fullmatch(r"[A-Za-z0-9._-]+", report_prefix): raise ValueError("report_prefix must contain only letters, numbers, dot, underscore, or hyphen") selected_prompts = direct_claim_prompts() if prompts is None else prompts return ( REMOTE_SCRIPT.replace("__CHAT_ID__", CHAT_ID) .replace("__USER_ID__", USER_ID) .replace("__USER_NAME__", USER_NAME) .replace("__RUN_ID__", run_id) .replace("__PROMPTS_JSON__", json.dumps(selected_prompts)) .replace("__SUITE_MODE_JSON__", json.dumps(suite_mode)) .replace("__REPORT_PREFIX_JSON__", json.dumps(report_prefix)) .replace("__PROMPT_NOTE_JSON__", json.dumps(prompt_note)) ) def redact(text: str) -> str: result = text for pattern in SECRET_PATTERNS: if pattern.groups >= 2: result = pattern.sub(r"\1[REDACTED]", result) else: result = pattern.sub("[REDACTED_TOKEN]", result) return result def fetch_remote_report( run_id: str, *, report_prefix: str = DEFAULT_REPORT_PREFIX, unlink: bool = True, ) -> dict[str, Any] | None: report_path = f"/tmp/{report_prefix}-{run_id}.json" unlink_line = " p.unlink()\n" if unlink else "" proc = subprocess.run( [ "ssh", "-i", str(SSH_KEY), "-o", "BatchMode=yes", "-o", "StrictHostKeyChecking=accept-new", VPS, "sudo -u teleo -H /home/teleo/.hermes/hermes-agent/venv/bin/python -", ], input=( "import json\n" "from pathlib import Path\n" f"p = Path({report_path!r})\n" "if p.exists():\n" " print(p.read_text(encoding='utf-8'))\n" f"{unlink_line}" ), text=True, capture_output=True, timeout=60, ) if proc.returncode != 0 or not proc.stdout.strip(): return None return json.loads(redact(proc.stdout.strip())) def run_remote( *, prompts: list[dict[str, Any]] | None = None, suite_mode: str = DEFAULT_SUITE_MODE, report_prefix: str = DEFAULT_REPORT_PREFIX, prompt_note: str = DEFAULT_PROMPT_NOTE, ) -> dict[str, Any]: run_id = uuid.uuid4().hex[:12] proc = subprocess.run( [ "ssh", "-i", str(SSH_KEY), "-o", "BatchMode=yes", "-o", "StrictHostKeyChecking=accept-new", VPS, "cd /home/teleo && sudo -u teleo -H /home/teleo/.hermes/hermes-agent/venv/bin/python -", ], input=build_remote_script( run_id, prompts=prompts, suite_mode=suite_mode, report_prefix=report_prefix, prompt_note=prompt_note, ), text=True, capture_output=True, timeout=2100, ) result: dict[str, Any] = { "returncode": proc.returncode, "stdout": redact(proc.stdout), "stderr": redact(proc.stderr), "run_id": run_id, } if proc.returncode == 0 and proc.stdout.strip(): for line in reversed(proc.stdout.splitlines()): candidate = line.strip() if not candidate.startswith("{"): continue try: result["parsed"] = json.loads(candidate) break except json.JSONDecodeError: continue if "parsed" not in result: result["parse_error"] = "remote stdout contained no parseable JSON object line" if "parsed" not in result: fetched = fetch_remote_report(run_id, report_prefix=report_prefix) if fetched is not None: result["parsed"] = fetched result["parsed_from_report_file"] = True else: fetch_remote_report(run_id, report_prefix=report_prefix) return result def write_output(remote: dict[str, Any], *, output_json: Path = OUTPUT_JSON) -> dict[str, Any]: REPORT_DIR.mkdir(parents=True, exist_ok=True) parsed = remote.get("parsed") or {} report = parsed if isinstance(parsed, dict) else {} report["source_report_path"] = str(output_json) report["remote_returncode"] = remote.get("returncode") report["remote_run_id"] = remote.get("run_id") if remote.get("stderr"): report["remote_stderr"] = remote["stderr"] output_json.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8") return report def main() -> int: remote = run_remote() report = write_output(remote) print(json.dumps({"json": str(OUTPUT_JSON), "pass_runtime": report.get("pass_runtime")}, indent=2)) return 0 if remote.get("returncode") == 0 and report.get("pass_runtime") else 1 if __name__ == "__main__": raise SystemExit(main())