430 lines
14 KiB
Python
Executable file
430 lines
14 KiB
Python
Executable file
#!/usr/bin/env python3
|
|
"""Run the Cory-style direct-claim suite through live VPS GatewayRunner.
|
|
|
|
The harness does not post to Telegram and does not write to the production KB.
|
|
It copies the live leoclean profile to a temporary profile on the VPS, invokes
|
|
GatewayRunner._handle_message with Telegram-shaped MessageEvents, and removes
|
|
the temporary profile after the run.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import re
|
|
import subprocess
|
|
import uuid
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import working_leo_open_ended_benchmark as benchmark
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
REPORT_DIR = ROOT / "docs" / "reports" / "leo-working-state-20260709"
|
|
OUTPUT_JSON = REPORT_DIR / "telegram-handler-direct-claim-suite-current.json"
|
|
SSH_KEY = Path.home() / ".ssh/livingip_hetzner_20260604_ed25519"
|
|
VPS = "root@77.42.65.182"
|
|
|
|
CHAT_ID = "-5146042086"
|
|
USER_ID = "9070919"
|
|
USER_NAME = "codex handler direct claim"
|
|
|
|
SECRET_PATTERNS = [
|
|
re.compile(r"\b\d{6,}:[A-Za-z0-9_-]{20,}\b"),
|
|
re.compile(r"(Authorization: Bearer )([A-Za-z0-9._-]+)", re.IGNORECASE),
|
|
re.compile(r"((?:api[_-]?key|token|secret|password)[=:]\s*)([A-Za-z0-9._-]+)", re.IGNORECASE),
|
|
]
|
|
|
|
|
|
REMOTE_SCRIPT = r'''
|
|
import asyncio
|
|
import json
|
|
import os
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import traceback
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
|
|
LIVE_PROFILE = Path("/home/teleo/.hermes/profiles/leoclean")
|
|
AGENT_ROOT = Path("/home/teleo/.hermes/hermes-agent")
|
|
CHAT_ID = "__CHAT_ID__"
|
|
USER_ID = "__USER_ID__"
|
|
USER_NAME = "__USER_NAME__"
|
|
RUN_ID = "__RUN_ID__"
|
|
PROMPTS = __PROMPTS_JSON__
|
|
REPORT_PATH = Path(f"/tmp/leo-direct-claim-handler-report-{RUN_ID}.json")
|
|
|
|
|
|
def service_state():
|
|
raw = subprocess.check_output(
|
|
[
|
|
"systemctl",
|
|
"show",
|
|
"leoclean-gateway.service",
|
|
"-p",
|
|
"ActiveState",
|
|
"-p",
|
|
"SubState",
|
|
"-p",
|
|
"MainPID",
|
|
"-p",
|
|
"NRestarts",
|
|
"-p",
|
|
"ExecMainStartTimestamp",
|
|
],
|
|
text=True,
|
|
)
|
|
result = {}
|
|
for line in raw.splitlines():
|
|
if "=" in line:
|
|
key, value = line.split("=", 1)
|
|
result[key] = value
|
|
return result
|
|
|
|
|
|
def db_counts():
|
|
sql = """
|
|
select jsonb_build_object(
|
|
'public.claims', (select count(*) from public.claims),
|
|
'public.sources', (select count(*) from public.sources),
|
|
'public.claim_evidence', (select count(*) from public.claim_evidence),
|
|
'public.claim_edges', (select count(*) from public.claim_edges),
|
|
'kb_stage.kb_proposals', (select count(*) from kb_stage.kb_proposals)
|
|
)::text;
|
|
"""
|
|
try:
|
|
raw = subprocess.check_output(
|
|
["docker", "exec", "-i", "teleo-pg", "psql", "-U", "postgres", "-d", "teleo", "-At", "-q"],
|
|
input=sql,
|
|
text=True,
|
|
stderr=subprocess.STDOUT,
|
|
)
|
|
return json.loads(raw.strip())
|
|
except Exception as exc:
|
|
return {"error": str(exc)}
|
|
|
|
|
|
def copy_profile() -> Path:
|
|
tmp_root = Path(tempfile.mkdtemp(prefix="leo-direct-claim-handler-"))
|
|
target = tmp_root / "profile"
|
|
|
|
def ignore(_dir, names):
|
|
ignored = {
|
|
"logs",
|
|
"cache",
|
|
"image_cache",
|
|
"memory_backups",
|
|
".pytest_cache",
|
|
"__pycache__",
|
|
"gateway.pid",
|
|
"auth.lock",
|
|
}
|
|
return {
|
|
name
|
|
for name in names
|
|
if name in ignored
|
|
or name.endswith(".lock")
|
|
or ".bak-" in name
|
|
or name.endswith(".bak")
|
|
}
|
|
|
|
try:
|
|
shutil.copytree(LIVE_PROFILE, target, symlinks=True, ignore=ignore)
|
|
except Exception:
|
|
shutil.rmtree(tmp_root, ignore_errors=True)
|
|
raise
|
|
return target
|
|
|
|
|
|
def remove_tree(path):
|
|
if not path or not path.exists():
|
|
return True
|
|
try:
|
|
shutil.rmtree(path)
|
|
except Exception:
|
|
for root, dirs, files in os.walk(path, topdown=False):
|
|
for name in files:
|
|
p = Path(root) / name
|
|
try:
|
|
p.chmod(0o600)
|
|
p.unlink()
|
|
except Exception:
|
|
pass
|
|
for name in dirs:
|
|
p = Path(root) / name
|
|
try:
|
|
p.chmod(0o700)
|
|
p.rmdir()
|
|
except Exception:
|
|
pass
|
|
try:
|
|
path.chmod(0o700)
|
|
path.rmdir()
|
|
except Exception:
|
|
pass
|
|
return not path.exists()
|
|
|
|
|
|
def write_report_snapshot(report):
|
|
try:
|
|
REPORT_PATH.write_text(json.dumps(report, sort_keys=True), encoding="utf-8")
|
|
except Exception:
|
|
pass
|
|
|
|
|
|
async def run_suite():
|
|
before_service = service_state()
|
|
before_counts = db_counts()
|
|
temp_profile = None
|
|
temp_removed = False
|
|
report = {
|
|
"generated_at_utc": datetime.now(timezone.utc).isoformat(),
|
|
"mode": "live_vps_gatewayrunner_temp_profile_direct_claim_suite",
|
|
"posted_to_telegram": False,
|
|
"changed_live_profile": False,
|
|
"mutates_kb_by_harness": False,
|
|
"source": {
|
|
"platform": "telegram",
|
|
"chat_id": CHAT_ID,
|
|
"chat_name": "Leo",
|
|
"chat_type": "group",
|
|
"user_id": USER_ID,
|
|
"user_name": USER_NAME,
|
|
},
|
|
"harness_notes": [
|
|
"Prompts are the exact DC-01..DC-06 direct-claim benchmark messages.",
|
|
"The harness uses a temporary copy of the leoclean profile and does not post to Telegram.",
|
|
"This proves handler-level live VPS GatewayRunner behavior, not human-visible Telegram delivery.",
|
|
],
|
|
"db_counts_before": before_counts,
|
|
"before_service": before_service,
|
|
"remote_report_path": str(REPORT_PATH),
|
|
}
|
|
write_report_snapshot(report)
|
|
try:
|
|
temp_profile = copy_profile()
|
|
os.environ["HERMES_HOME"] = str(temp_profile)
|
|
os.environ["HOME"] = "/home/teleo"
|
|
sys.path.insert(0, str(AGENT_ROOT))
|
|
|
|
from gateway.config import Platform
|
|
from gateway.platforms.base import MessageEvent, MessageType
|
|
from gateway.run import GatewayRunner
|
|
from gateway.session import SessionSource
|
|
|
|
source = SessionSource(
|
|
platform=Platform.TELEGRAM,
|
|
chat_id=CHAT_ID,
|
|
chat_name="Leo",
|
|
chat_type="group",
|
|
user_id=USER_ID,
|
|
user_name=USER_NAME,
|
|
)
|
|
runner = GatewayRunner()
|
|
session_key = runner._session_key_for_source(source)
|
|
authorized = runner._is_user_authorized(source)
|
|
report["handler"] = {
|
|
"temp_profile": str(temp_profile),
|
|
"session_key": session_key,
|
|
"authorized": authorized,
|
|
}
|
|
write_report_snapshot(report)
|
|
if not authorized:
|
|
report["results"] = []
|
|
report["pass_runtime"] = False
|
|
report["failure"] = "telegram group source was not authorized by live config"
|
|
write_report_snapshot(report)
|
|
return report
|
|
|
|
results = []
|
|
for index, prompt in enumerate(PROMPTS, start=1):
|
|
event = MessageEvent(
|
|
text=prompt["message"],
|
|
message_type=MessageType.TEXT,
|
|
source=source,
|
|
message_id=f"direct-claim-{RUN_ID}-{prompt['id']}",
|
|
)
|
|
started = datetime.now(timezone.utc)
|
|
reply = await asyncio.wait_for(runner._handle_message(event), timeout=300)
|
|
ended = datetime.now(timezone.utc)
|
|
results.append(
|
|
{
|
|
"turn": index,
|
|
"prompt_id": prompt["id"],
|
|
"dimension": prompt["dimension"],
|
|
"prompt": prompt["message"],
|
|
"reply": reply or "",
|
|
"started_at_utc": started.isoformat(),
|
|
"ended_at_utc": ended.isoformat(),
|
|
"ok": bool((reply or "").strip()),
|
|
"mutates_kb": False,
|
|
"evidence_tier": "live_vps_gatewayrunner_temp_profile",
|
|
"claim_ceiling": "Live VPS GatewayRunner reply from temp leoclean profile; no Telegram post; no production DB apply authorized.",
|
|
}
|
|
)
|
|
report["results"] = results
|
|
write_report_snapshot(report)
|
|
report["results"] = results
|
|
report["pass_runtime"] = all(row["ok"] for row in results)
|
|
return report
|
|
except Exception as exc:
|
|
report["pass_runtime"] = False
|
|
report["error"] = str(exc)
|
|
report["traceback_tail"] = traceback.format_exc().splitlines()[-12:]
|
|
write_report_snapshot(report)
|
|
return report
|
|
finally:
|
|
after_counts = db_counts()
|
|
after_service = service_state()
|
|
report["db_counts_after"] = after_counts
|
|
report["db_counts_changed"] = after_counts != before_counts
|
|
report["service_before_after"] = {
|
|
"before": before_service,
|
|
"after": after_service,
|
|
"unchanged_from_preexisting_live_readback": before_service == after_service,
|
|
}
|
|
if temp_profile is not None:
|
|
tmp_root = temp_profile.parent
|
|
temp_removed = remove_tree(tmp_root)
|
|
report["temp_profile_removed"] = temp_removed
|
|
write_report_snapshot(report)
|
|
|
|
|
|
print(json.dumps(asyncio.run(run_suite()), sort_keys=True))
|
|
'''
|
|
|
|
|
|
def direct_claim_prompts() -> list[dict[str, Any]]:
|
|
return [
|
|
{
|
|
"id": prompt["id"],
|
|
"dimension": prompt["dimension"],
|
|
"message": prompt["message"],
|
|
}
|
|
for prompt in benchmark.CORY_DIRECT_CLAIM_FOLLOWUP_SCENARIOS
|
|
]
|
|
|
|
|
|
def build_remote_script(run_id: str) -> str:
|
|
return (
|
|
REMOTE_SCRIPT.replace("__CHAT_ID__", CHAT_ID)
|
|
.replace("__USER_ID__", USER_ID)
|
|
.replace("__USER_NAME__", USER_NAME)
|
|
.replace("__RUN_ID__", run_id)
|
|
.replace("__PROMPTS_JSON__", json.dumps(direct_claim_prompts()))
|
|
)
|
|
|
|
|
|
def redact(text: str) -> str:
|
|
result = text
|
|
for pattern in SECRET_PATTERNS:
|
|
if pattern.groups >= 2:
|
|
result = pattern.sub(r"\1[REDACTED]", result)
|
|
else:
|
|
result = pattern.sub("[REDACTED_TOKEN]", result)
|
|
return result
|
|
|
|
|
|
def fetch_remote_report(run_id: str, *, unlink: bool = True) -> dict[str, Any] | None:
|
|
report_path = f"/tmp/leo-direct-claim-handler-report-{run_id}.json"
|
|
unlink_line = " p.unlink()\n" if unlink else ""
|
|
proc = subprocess.run(
|
|
[
|
|
"ssh",
|
|
"-i",
|
|
str(SSH_KEY),
|
|
"-o",
|
|
"BatchMode=yes",
|
|
"-o",
|
|
"StrictHostKeyChecking=accept-new",
|
|
VPS,
|
|
"sudo -u teleo -H /home/teleo/.hermes/hermes-agent/venv/bin/python -",
|
|
],
|
|
input=(
|
|
"import json\n"
|
|
"from pathlib import Path\n"
|
|
f"p = Path({report_path!r})\n"
|
|
"if p.exists():\n"
|
|
" print(p.read_text(encoding='utf-8'))\n"
|
|
f"{unlink_line}"
|
|
),
|
|
text=True,
|
|
capture_output=True,
|
|
timeout=60,
|
|
)
|
|
if proc.returncode != 0 or not proc.stdout.strip():
|
|
return None
|
|
return json.loads(redact(proc.stdout.strip()))
|
|
|
|
|
|
def run_remote() -> dict[str, Any]:
|
|
run_id = uuid.uuid4().hex[:12]
|
|
proc = subprocess.run(
|
|
[
|
|
"ssh",
|
|
"-i",
|
|
str(SSH_KEY),
|
|
"-o",
|
|
"BatchMode=yes",
|
|
"-o",
|
|
"StrictHostKeyChecking=accept-new",
|
|
VPS,
|
|
"cd /home/teleo && sudo -u teleo -H /home/teleo/.hermes/hermes-agent/venv/bin/python -",
|
|
],
|
|
input=build_remote_script(run_id),
|
|
text=True,
|
|
capture_output=True,
|
|
timeout=2100,
|
|
)
|
|
result: dict[str, Any] = {
|
|
"returncode": proc.returncode,
|
|
"stdout": redact(proc.stdout),
|
|
"stderr": redact(proc.stderr),
|
|
"run_id": run_id,
|
|
}
|
|
if proc.returncode == 0 and proc.stdout.strip():
|
|
for line in reversed(proc.stdout.splitlines()):
|
|
candidate = line.strip()
|
|
if not candidate.startswith("{"):
|
|
continue
|
|
try:
|
|
result["parsed"] = json.loads(candidate)
|
|
break
|
|
except json.JSONDecodeError:
|
|
continue
|
|
if "parsed" not in result:
|
|
result["parse_error"] = "remote stdout contained no parseable JSON object line"
|
|
if "parsed" not in result:
|
|
fetched = fetch_remote_report(run_id)
|
|
if fetched is not None:
|
|
result["parsed"] = fetched
|
|
result["parsed_from_report_file"] = True
|
|
else:
|
|
fetch_remote_report(run_id)
|
|
return result
|
|
|
|
|
|
def write_output(remote: dict[str, Any]) -> dict[str, Any]:
|
|
REPORT_DIR.mkdir(parents=True, exist_ok=True)
|
|
parsed = remote.get("parsed") or {}
|
|
report = parsed if isinstance(parsed, dict) else {}
|
|
report["source_report_path"] = str(OUTPUT_JSON)
|
|
report["remote_returncode"] = remote.get("returncode")
|
|
report["remote_run_id"] = remote.get("run_id")
|
|
if remote.get("stderr"):
|
|
report["remote_stderr"] = remote["stderr"]
|
|
OUTPUT_JSON.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
|
return report
|
|
|
|
|
|
def main() -> int:
|
|
remote = run_remote()
|
|
report = write_output(remote)
|
|
print(json.dumps({"json": str(OUTPUT_JSON), "pass_runtime": report.get("pass_runtime")}, indent=2))
|
|
return 0 if remote.get("returncode") == 0 and report.get("pass_runtime") else 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|