Add Leo direct-claim remote runner

scripts/record_leo_direct_claim_handler_suite.py

@@ -0,0 +1,79 @@
+#!/usr/bin/env python3
+"""Record a retained direct-claim handler suite JSON emitted by the VPS runner."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import re
+import sys
+from pathlib import Path
+from typing import Any
+
+ROOT = Path(__file__).resolve().parents[1]
+REPORT_DIR = ROOT / "docs" / "reports" / "leo-working-state-20260709"
+DEFAULT_OUT = REPORT_DIR / "telegram-handler-direct-claim-suite-current.json"
+
+SECRET_PATTERNS = [
+    re.compile(r"\b\d{6,}:[A-Za-z0-9_-]{20,}\b"),
+    re.compile(r"(Authorization: Bearer )([A-Za-z0-9._-]+)", re.IGNORECASE),
+    re.compile(r"((?:api[_-]?key|token|secret|password)[=:]\s*)([A-Za-z0-9._-]+)", re.IGNORECASE),
+]
+
+
+def redact(text: str) -> str:
+    result = text
+    for pattern in SECRET_PATTERNS:
+        if pattern.groups >= 2:
+            result = pattern.sub(r"\1[REDACTED]", result)
+        else:
+            result = pattern.sub("[REDACTED_TOKEN]", result)
+    return result
+
+
+def parse_report(raw: str) -> dict[str, Any]:
+    cleaned = redact(raw).strip()
+    for line in reversed(cleaned.splitlines()):
+        candidate = line.strip()
+        if not candidate.startswith("{"):
+            continue
+        return json.loads(candidate)
+    raise ValueError("stdin contained no JSON object line")
+
+
+def record_report(report: dict[str, Any], *, out: Path = DEFAULT_OUT) -> dict[str, Any]:
+    report = dict(report)
+    report["source_report_path"] = str(out)
+    out.parent.mkdir(parents=True, exist_ok=True)
+    out.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8")
+    return report
+
+
+def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--out", type=Path, default=DEFAULT_OUT)
+    return parser.parse_args(argv)
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = parse_args(argv)
+    report = record_report(parse_report(sys.stdin.read()), out=args.out)
+    print(
+        json.dumps(
+            {
+                "out": str(args.out),
+                "pass_runtime": report.get("pass_runtime"),
+                "results": len(report.get("results") or []),
+                "posted_to_telegram": report.get("posted_to_telegram"),
+                "db_counts_changed": report.get("db_counts_changed"),
+                "temp_profile_removed": report.get("temp_profile_removed"),
+            },
+            indent=2,
+            sort_keys=True,
+        )
+    )
+    return 0 if report.get("pass_runtime") else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/run_leo_direct_claim_handler_remote.py

@@ -0,0 +1,306 @@
+#!/usr/bin/env python3
+"""Run the Working Leo direct-claim handler suite on the VPS.
+
+This script is intended to be executed on the VPS from the deploy-infra checkout
+as the ``teleo`` user. It does not post to Telegram and does not write to the
+production KB. It copies the live leoclean profile to a temporary profile,
+invokes GatewayRunner._handle_message with Telegram-shaped MessageEvents, and
+removes the temporary profile after the run.
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import json
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+import traceback
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+import working_leo_open_ended_benchmark as benchmark
+
+LIVE_PROFILE = Path("/home/teleo/.hermes/profiles/leoclean")
+AGENT_ROOT = Path("/home/teleo/.hermes/hermes-agent")
+DEFAULT_CHAT_ID = "-5146042086"
+DEFAULT_USER_ID = "9070919"
+DEFAULT_USER_NAME = "codex handler direct claim"
+
+
+def direct_claim_prompts() -> list[dict[str, Any]]:
+    return [
+        {
+            "id": prompt["id"],
+            "dimension": prompt["dimension"],
+            "message": prompt["message"],
+        }
+        for prompt in benchmark.CORY_DIRECT_CLAIM_FOLLOWUP_SCENARIOS
+    ]
+
+
+def run_command(command: list[str], *, input_text: str | None = None) -> subprocess.CompletedProcess[str]:
+    return subprocess.run(command, input=input_text, text=True, capture_output=True, check=False)
+
+
+def service_state() -> dict[str, str]:
+    proc = run_command(
+        [
+            "systemctl",
+            "show",
+            "leoclean-gateway.service",
+            "-p",
+            "ActiveState",
+            "-p",
+            "SubState",
+            "-p",
+            "MainPID",
+            "-p",
+            "NRestarts",
+            "-p",
+            "ExecMainStartTimestamp",
+            "--no-pager",
+        ]
+    )
+    state = {}
+    for line in proc.stdout.splitlines():
+        if "=" in line:
+            key, value = line.split("=", 1)
+            state[key] = value
+    return state
+
+
+def db_counts() -> dict[str, Any]:
+    sql = """
+select jsonb_build_object(
+  'public.claims', (select count(*) from public.claims),
+  'public.sources', (select count(*) from public.sources),
+  'public.claim_evidence', (select count(*) from public.claim_evidence),
+  'public.claim_edges', (select count(*) from public.claim_edges),
+  'kb_stage.kb_proposals', (select count(*) from kb_stage.kb_proposals)
+)::text;
+"""
+    proc = run_command(
+        ["docker", "exec", "-i", "teleo-pg", "psql", "-U", "postgres", "-d", "teleo", "-At", "-q"],
+        input_text=sql,
+    )
+    if proc.returncode == 0 and proc.stdout.strip():
+        return json.loads(proc.stdout.strip())
+    return {"error": proc.stderr.strip() or proc.stdout.strip() or f"psql exited {proc.returncode}"}
+
+
+def copy_profile() -> Path:
+    tmp_root = Path(tempfile.mkdtemp(prefix="leo-direct-claim-handler-"))
+    target = tmp_root / "profile"
+
+    def ignore(_dir: str, names: list[str]) -> set[str]:
+        ignored = {
+            "logs",
+            "cache",
+            "image_cache",
+            "memory_backups",
+            ".pytest_cache",
+            "__pycache__",
+            "gateway.pid",
+            "auth.lock",
+        }
+        return {
+            name
+            for name in names
+            if name in ignored or name.endswith(".lock") or ".bak-" in name or name.endswith(".bak")
+        }
+
+    try:
+        shutil.copytree(LIVE_PROFILE, target, symlinks=True, ignore=ignore)
+    except Exception:
+        shutil.rmtree(tmp_root, ignore_errors=True)
+        raise
+    return target
+
+
+def remove_tree(path: Path | None) -> bool:
+    if not path or not path.exists():
+        return True
+    try:
+        shutil.rmtree(path)
+    except Exception:
+        for root, dirs, files in os.walk(path, topdown=False):
+            for name in files:
+                p = Path(root) / name
+                try:
+                    p.chmod(0o600)
+                    p.unlink()
+                except Exception:
+                    pass
+            for name in dirs:
+                p = Path(root) / name
+                try:
+                    p.chmod(0o700)
+                    p.rmdir()
+                except Exception:
+                    pass
+        try:
+            path.chmod(0o700)
+            path.rmdir()
+        except Exception:
+            pass
+    return not path.exists()
+
+
+def write_report_snapshot(report_path: Path, report: dict[str, Any]) -> None:
+    try:
+        report_path.write_text(json.dumps(report, sort_keys=True), encoding="utf-8")
+    except Exception:
+        pass
+
+
+async def run_suite(args: argparse.Namespace) -> dict[str, Any]:
+    before_service = service_state()
+    before_counts = db_counts()
+    temp_profile: Path | None = None
+    temp_removed = False
+    prompts = direct_claim_prompts()
+    report_path = Path(args.remote_report_path)
+    report: dict[str, Any] = {
+        "generated_at_utc": datetime.now(timezone.utc).isoformat(),
+        "mode": "live_vps_gatewayrunner_temp_profile_direct_claim_suite",
+        "posted_to_telegram": False,
+        "changed_live_profile": False,
+        "mutates_kb_by_harness": False,
+        "source": {
+            "platform": "telegram",
+            "chat_id": args.chat_id,
+            "chat_name": "Leo",
+            "chat_type": "group",
+            "user_id": args.user_id,
+            "user_name": args.user_name,
+        },
+        "harness_notes": [
+            "Prompts are the exact DC-01..DC-06 direct-claim benchmark messages.",
+            "The harness uses a temporary copy of the leoclean profile and does not post to Telegram.",
+            "This proves handler-level live VPS GatewayRunner behavior, not human-visible Telegram delivery.",
+        ],
+        "db_counts_before": before_counts,
+        "before_service": before_service,
+        "remote_report_path": str(report_path),
+    }
+    write_report_snapshot(report_path, report)
+    try:
+        temp_profile = copy_profile()
+        os.environ["HERMES_HOME"] = str(temp_profile)
+        os.environ["HOME"] = "/home/teleo"
+        sys.path.insert(0, str(AGENT_ROOT))
+
+        from gateway.config import Platform
+        from gateway.platforms.base import MessageEvent, MessageType
+        from gateway.run import GatewayRunner
+        from gateway.session import SessionSource
+
+        source = SessionSource(
+            platform=Platform.TELEGRAM,
+            chat_id=args.chat_id,
+            chat_name="Leo",
+            chat_type="group",
+            user_id=args.user_id,
+            user_name=args.user_name,
+        )
+        runner = GatewayRunner()
+        session_key = runner._session_key_for_source(source)
+        authorized = runner._is_user_authorized(source)
+        report["handler"] = {
+            "temp_profile": str(temp_profile),
+            "session_key": session_key,
+            "authorized": authorized,
+        }
+        write_report_snapshot(report_path, report)
+        if not authorized:
+            report["results"] = []
+            report["pass_runtime"] = False
+            report["failure"] = "telegram group source was not authorized by live config"
+            write_report_snapshot(report_path, report)
+            return report
+
+        results = []
+        for index, prompt in enumerate(prompts, start=1):
+            event = MessageEvent(
+                text=prompt["message"],
+                message_type=MessageType.TEXT,
+                source=source,
+                message_id=f"direct-claim-{args.run_id}-{prompt['id']}",
+            )
+            started = datetime.now(timezone.utc)
+            reply = await asyncio.wait_for(runner._handle_message(event), timeout=args.turn_timeout)
+            ended = datetime.now(timezone.utc)
+            results.append(
+                {
+                    "turn": index,
+                    "prompt_id": prompt["id"],
+                    "dimension": prompt["dimension"],
+                    "prompt": prompt["message"],
+                    "reply": reply or "",
+                    "started_at_utc": started.isoformat(),
+                    "ended_at_utc": ended.isoformat(),
+                    "ok": bool((reply or "").strip()),
+                    "mutates_kb": False,
+                    "evidence_tier": "live_vps_gatewayrunner_temp_profile",
+                    "claim_ceiling": (
+                        "Live VPS GatewayRunner reply from temp leoclean profile; no Telegram post; "
+                        "no production DB apply authorized."
+                    ),
+                }
+            )
+            report["results"] = results
+            write_report_snapshot(report_path, report)
+        report["results"] = results
+        report["pass_runtime"] = all(row["ok"] for row in results)
+        return report
+    except Exception as exc:
+        report["pass_runtime"] = False
+        report["error"] = str(exc)
+        report["traceback_tail"] = traceback.format_exc().splitlines()[-12:]
+        write_report_snapshot(report_path, report)
+        return report
+    finally:
+        after_counts = db_counts()
+        after_service = service_state()
+        report["db_counts_after"] = after_counts
+        report["db_counts_changed"] = after_counts != before_counts
+        report["service_before_after"] = {
+            "before": before_service,
+            "after": after_service,
+            "unchanged_from_preexisting_live_readback": before_service == after_service,
+        }
+        if temp_profile is not None:
+            temp_removed = remove_tree(temp_profile.parent)
+        report["temp_profile_removed"] = temp_removed
+        write_report_snapshot(report_path, report)
+
+
+def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--chat-id", default=DEFAULT_CHAT_ID)
+    parser.add_argument("--user-id", default=DEFAULT_USER_ID)
+    parser.add_argument("--user-name", default=DEFAULT_USER_NAME)
+    parser.add_argument("--run-id", default=datetime.now(timezone.utc).strftime("%Y%m%d%H%M%S"))
+    parser.add_argument("--turn-timeout", type=int, default=300)
+    parser.add_argument(
+        "--remote-report-path",
+        default="/tmp/leo-direct-claim-handler-report-current.json",
+        help="Temporary remote JSON snapshot path; overwritten on each run.",
+    )
+    return parser.parse_args(argv)
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = parse_args(argv)
+    report = asyncio.run(run_suite(args))
+    print(json.dumps(report, sort_keys=True))
+    return 0 if report.get("pass_runtime") else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

tests/test_record_leo_direct_claim_handler_suite.py

@@ -0,0 +1,53 @@
+"""Tests for scripts/record_leo_direct_claim_handler_suite.py."""
+
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(REPO_ROOT / "scripts"))
+
+import record_leo_direct_claim_handler_suite as recorder  # noqa: E402
+import run_leo_direct_claim_handler_remote as remote_runner  # noqa: E402
+
+
+def sample_report() -> dict:
+    return {
+        "mode": "live_vps_gatewayrunner_temp_profile_direct_claim_suite",
+        "posted_to_telegram": False,
+        "changed_live_profile": False,
+        "mutates_kb_by_harness": False,
+        "pass_runtime": True,
+        "db_counts_changed": False,
+        "temp_profile_removed": True,
+        "results": [{"prompt_id": "DC-01", "reply": "ok", "ok": True}],
+    }
+
+
+def test_remote_runner_preserves_direct_claim_prompt_catalog():
+    prompts = remote_runner.direct_claim_prompts()
+
+    assert [prompt["id"] for prompt in prompts] == ["DC-01", "DC-02", "DC-03", "DC-04", "DC-05", "DC-06"]
+    assert prompts[0]["message"] == "Did we actually update the knowledge base, or is it still just proposals?"
+
+
+def test_recorder_parses_last_json_line_and_redacts_token():
+    raw = "noise\n" + json.dumps(sample_report() | {"secret": "token=123456:ABCdefghijklmnopqrstuvwxyz"}) + "\n"
+
+    parsed = recorder.parse_report(raw)
+
+    assert parsed["pass_runtime"] is True
+    assert parsed["secret"] == "token=[REDACTED_TOKEN]"
+
+
+def test_recorder_writes_report_with_source_path(tmp_path: Path):
+    out = tmp_path / "suite.json"
+
+    report = recorder.record_report(sample_report(), out=out)
+
+    assert report["source_report_path"] == str(out)
+    written = json.loads(out.read_text(encoding="utf-8"))
+    assert written["posted_to_telegram"] is False
+    assert written["temp_profile_removed"] is True