306 lines
10 KiB
Python
306 lines
10 KiB
Python
#!/usr/bin/env python3
|
|
"""Run the Working Leo direct-claim handler suite on the VPS.
|
|
|
|
This script is intended to be executed on the VPS from the deploy-infra checkout
|
|
as the ``teleo`` user. It does not post to Telegram and does not write to the
|
|
production KB. It copies the live leoclean profile to a temporary profile,
|
|
invokes GatewayRunner._handle_message with Telegram-shaped MessageEvents, and
|
|
removes the temporary profile after the run.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import asyncio
|
|
import json
|
|
import os
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import traceback
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import working_leo_open_ended_benchmark as benchmark
|
|
|
|
LIVE_PROFILE = Path("/home/teleo/.hermes/profiles/leoclean")
|
|
AGENT_ROOT = Path("/home/teleo/.hermes/hermes-agent")
|
|
DEFAULT_CHAT_ID = "-5146042086"
|
|
DEFAULT_USER_ID = "9070919"
|
|
DEFAULT_USER_NAME = "codex handler direct claim"
|
|
|
|
|
|
def direct_claim_prompts() -> list[dict[str, Any]]:
|
|
return [
|
|
{
|
|
"id": prompt["id"],
|
|
"dimension": prompt["dimension"],
|
|
"message": prompt["message"],
|
|
}
|
|
for prompt in benchmark.CORY_DIRECT_CLAIM_FOLLOWUP_SCENARIOS
|
|
]
|
|
|
|
|
|
def run_command(command: list[str], *, input_text: str | None = None) -> subprocess.CompletedProcess[str]:
|
|
return subprocess.run(command, input=input_text, text=True, capture_output=True, check=False)
|
|
|
|
|
|
def service_state() -> dict[str, str]:
|
|
proc = run_command(
|
|
[
|
|
"systemctl",
|
|
"show",
|
|
"leoclean-gateway.service",
|
|
"-p",
|
|
"ActiveState",
|
|
"-p",
|
|
"SubState",
|
|
"-p",
|
|
"MainPID",
|
|
"-p",
|
|
"NRestarts",
|
|
"-p",
|
|
"ExecMainStartTimestamp",
|
|
"--no-pager",
|
|
]
|
|
)
|
|
state = {}
|
|
for line in proc.stdout.splitlines():
|
|
if "=" in line:
|
|
key, value = line.split("=", 1)
|
|
state[key] = value
|
|
return state
|
|
|
|
|
|
def db_counts() -> dict[str, Any]:
|
|
sql = """
|
|
select jsonb_build_object(
|
|
'public.claims', (select count(*) from public.claims),
|
|
'public.sources', (select count(*) from public.sources),
|
|
'public.claim_evidence', (select count(*) from public.claim_evidence),
|
|
'public.claim_edges', (select count(*) from public.claim_edges),
|
|
'kb_stage.kb_proposals', (select count(*) from kb_stage.kb_proposals)
|
|
)::text;
|
|
"""
|
|
proc = run_command(
|
|
["docker", "exec", "-i", "teleo-pg", "psql", "-U", "postgres", "-d", "teleo", "-At", "-q"],
|
|
input_text=sql,
|
|
)
|
|
if proc.returncode == 0 and proc.stdout.strip():
|
|
return json.loads(proc.stdout.strip())
|
|
return {"error": proc.stderr.strip() or proc.stdout.strip() or f"psql exited {proc.returncode}"}
|
|
|
|
|
|
def copy_profile() -> Path:
|
|
tmp_root = Path(tempfile.mkdtemp(prefix="leo-direct-claim-handler-"))
|
|
target = tmp_root / "profile"
|
|
|
|
def ignore(_dir: str, names: list[str]) -> set[str]:
|
|
ignored = {
|
|
"logs",
|
|
"cache",
|
|
"image_cache",
|
|
"memory_backups",
|
|
".pytest_cache",
|
|
"__pycache__",
|
|
"gateway.pid",
|
|
"auth.lock",
|
|
}
|
|
return {
|
|
name
|
|
for name in names
|
|
if name in ignored or name.endswith(".lock") or ".bak-" in name or name.endswith(".bak")
|
|
}
|
|
|
|
try:
|
|
shutil.copytree(LIVE_PROFILE, target, symlinks=True, ignore=ignore)
|
|
except Exception:
|
|
shutil.rmtree(tmp_root, ignore_errors=True)
|
|
raise
|
|
return target
|
|
|
|
|
|
def remove_tree(path: Path | None) -> bool:
|
|
if not path or not path.exists():
|
|
return True
|
|
try:
|
|
shutil.rmtree(path)
|
|
except Exception:
|
|
for root, dirs, files in os.walk(path, topdown=False):
|
|
for name in files:
|
|
p = Path(root) / name
|
|
try:
|
|
p.chmod(0o600)
|
|
p.unlink()
|
|
except Exception:
|
|
pass
|
|
for name in dirs:
|
|
p = Path(root) / name
|
|
try:
|
|
p.chmod(0o700)
|
|
p.rmdir()
|
|
except Exception:
|
|
pass
|
|
try:
|
|
path.chmod(0o700)
|
|
path.rmdir()
|
|
except Exception:
|
|
pass
|
|
return not path.exists()
|
|
|
|
|
|
def write_report_snapshot(report_path: Path, report: dict[str, Any]) -> None:
|
|
try:
|
|
report_path.write_text(json.dumps(report, sort_keys=True), encoding="utf-8")
|
|
except Exception:
|
|
pass
|
|
|
|
|
|
async def run_suite(args: argparse.Namespace) -> dict[str, Any]:
|
|
before_service = service_state()
|
|
before_counts = db_counts()
|
|
temp_profile: Path | None = None
|
|
temp_removed = False
|
|
prompts = direct_claim_prompts()
|
|
report_path = Path(args.remote_report_path)
|
|
report: dict[str, Any] = {
|
|
"generated_at_utc": datetime.now(timezone.utc).isoformat(),
|
|
"mode": "live_vps_gatewayrunner_temp_profile_direct_claim_suite",
|
|
"posted_to_telegram": False,
|
|
"changed_live_profile": False,
|
|
"mutates_kb_by_harness": False,
|
|
"source": {
|
|
"platform": "telegram",
|
|
"chat_id": args.chat_id,
|
|
"chat_name": "Leo",
|
|
"chat_type": "group",
|
|
"user_id": args.user_id,
|
|
"user_name": args.user_name,
|
|
},
|
|
"harness_notes": [
|
|
"Prompts are the exact DC-01..DC-06 direct-claim benchmark messages.",
|
|
"The harness uses a temporary copy of the leoclean profile and does not post to Telegram.",
|
|
"This proves handler-level live VPS GatewayRunner behavior, not human-visible Telegram delivery.",
|
|
],
|
|
"db_counts_before": before_counts,
|
|
"before_service": before_service,
|
|
"remote_report_path": str(report_path),
|
|
}
|
|
write_report_snapshot(report_path, report)
|
|
try:
|
|
temp_profile = copy_profile()
|
|
os.environ["HERMES_HOME"] = str(temp_profile)
|
|
os.environ["HOME"] = "/home/teleo"
|
|
sys.path.insert(0, str(AGENT_ROOT))
|
|
|
|
from gateway.config import Platform
|
|
from gateway.platforms.base import MessageEvent, MessageType
|
|
from gateway.run import GatewayRunner
|
|
from gateway.session import SessionSource
|
|
|
|
source = SessionSource(
|
|
platform=Platform.TELEGRAM,
|
|
chat_id=args.chat_id,
|
|
chat_name="Leo",
|
|
chat_type="group",
|
|
user_id=args.user_id,
|
|
user_name=args.user_name,
|
|
)
|
|
runner = GatewayRunner()
|
|
session_key = runner._session_key_for_source(source)
|
|
authorized = runner._is_user_authorized(source)
|
|
report["handler"] = {
|
|
"temp_profile": str(temp_profile),
|
|
"session_key": session_key,
|
|
"authorized": authorized,
|
|
}
|
|
write_report_snapshot(report_path, report)
|
|
if not authorized:
|
|
report["results"] = []
|
|
report["pass_runtime"] = False
|
|
report["failure"] = "telegram group source was not authorized by live config"
|
|
write_report_snapshot(report_path, report)
|
|
return report
|
|
|
|
results = []
|
|
for index, prompt in enumerate(prompts, start=1):
|
|
event = MessageEvent(
|
|
text=prompt["message"],
|
|
message_type=MessageType.TEXT,
|
|
source=source,
|
|
message_id=f"direct-claim-{args.run_id}-{prompt['id']}",
|
|
)
|
|
started = datetime.now(timezone.utc)
|
|
reply = await asyncio.wait_for(runner._handle_message(event), timeout=args.turn_timeout)
|
|
ended = datetime.now(timezone.utc)
|
|
results.append(
|
|
{
|
|
"turn": index,
|
|
"prompt_id": prompt["id"],
|
|
"dimension": prompt["dimension"],
|
|
"prompt": prompt["message"],
|
|
"reply": reply or "",
|
|
"started_at_utc": started.isoformat(),
|
|
"ended_at_utc": ended.isoformat(),
|
|
"ok": bool((reply or "").strip()),
|
|
"mutates_kb": False,
|
|
"evidence_tier": "live_vps_gatewayrunner_temp_profile",
|
|
"claim_ceiling": (
|
|
"Live VPS GatewayRunner reply from temp leoclean profile; no Telegram post; "
|
|
"no production DB apply authorized."
|
|
),
|
|
}
|
|
)
|
|
report["results"] = results
|
|
write_report_snapshot(report_path, report)
|
|
report["results"] = results
|
|
report["pass_runtime"] = all(row["ok"] for row in results)
|
|
return report
|
|
except Exception as exc:
|
|
report["pass_runtime"] = False
|
|
report["error"] = str(exc)
|
|
report["traceback_tail"] = traceback.format_exc().splitlines()[-12:]
|
|
write_report_snapshot(report_path, report)
|
|
return report
|
|
finally:
|
|
after_counts = db_counts()
|
|
after_service = service_state()
|
|
report["db_counts_after"] = after_counts
|
|
report["db_counts_changed"] = after_counts != before_counts
|
|
report["service_before_after"] = {
|
|
"before": before_service,
|
|
"after": after_service,
|
|
"unchanged_from_preexisting_live_readback": before_service == after_service,
|
|
}
|
|
if temp_profile is not None:
|
|
temp_removed = remove_tree(temp_profile.parent)
|
|
report["temp_profile_removed"] = temp_removed
|
|
write_report_snapshot(report_path, report)
|
|
|
|
|
|
def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--chat-id", default=DEFAULT_CHAT_ID)
|
|
parser.add_argument("--user-id", default=DEFAULT_USER_ID)
|
|
parser.add_argument("--user-name", default=DEFAULT_USER_NAME)
|
|
parser.add_argument("--run-id", default=datetime.now(timezone.utc).strftime("%Y%m%d%H%M%S"))
|
|
parser.add_argument("--turn-timeout", type=int, default=300)
|
|
parser.add_argument(
|
|
"--remote-report-path",
|
|
default="/tmp/leo-direct-claim-handler-report-current.json",
|
|
help="Temporary remote JSON snapshot path; overwritten on each run.",
|
|
)
|
|
return parser.parse_args(argv)
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
args = parse_args(argv)
|
|
report = asyncio.run(run_suite(args))
|
|
print(json.dumps(report, sort_keys=True))
|
|
return 0 if report.get("pass_runtime") else 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|