576 lines
20 KiB
Python
Executable file
576 lines
20 KiB
Python
Executable file
#!/usr/bin/env python3
|
|
"""Run the m3taversal-style direct-claim suite through live VPS GatewayRunner.
|
|
|
|
The harness does not post to Telegram and does not write to the production KB.
|
|
It copies only static behavior surfaces from the live leoclean profile to a
|
|
temporary profile on the VPS, invokes GatewayRunner._handle_message with
|
|
Telegram-shaped MessageEvents, and removes the temporary profile after the
|
|
run. Prior sessions, memory, state databases, and generated prompt caches are
|
|
excluded so the result can be attributed to the declared runtime inputs.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import re
|
|
import subprocess
|
|
import uuid
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import working_leo_open_ended_benchmark as benchmark
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
REPORT_DIR = ROOT / "docs" / "reports" / "leo-working-state-20260709"
|
|
OUTPUT_JSON = REPORT_DIR / "telegram-handler-direct-claim-suite-current.json"
|
|
SSH_KEY = Path.home() / ".ssh/livingip_hetzner_20260604_ed25519"
|
|
VPS = "root@77.42.65.182"
|
|
|
|
DEFAULT_SUITE_MODE = "live_vps_gatewayrunner_temp_profile_direct_claim_suite"
|
|
DEFAULT_REPORT_PREFIX = "leo-direct-claim-handler-report"
|
|
DEFAULT_PROMPT_NOTE = "Prompts are the exact DC-01..DC-06 direct-claim benchmark messages."
|
|
|
|
CHAT_ID = "-5146042086"
|
|
USER_ID = "9070919"
|
|
USER_NAME = "codex handler direct claim"
|
|
|
|
SECRET_PATTERNS = [
|
|
re.compile(r"\b\d{6,}:[A-Za-z0-9_-]{20,}\b"),
|
|
re.compile(r"(Authorization: Bearer )([A-Za-z0-9._-]+)", re.IGNORECASE),
|
|
re.compile(r"((?:api[_-]?key|token|secret|password)[=:]\s*)([A-Za-z0-9._-]+)", re.IGNORECASE),
|
|
]
|
|
|
|
|
|
REMOTE_SCRIPT = r'''
|
|
import asyncio
|
|
import json
|
|
import os
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import traceback
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
|
|
LIVE_PROFILE = Path("/home/teleo/.hermes/profiles/leoclean")
|
|
AGENT_ROOT = Path("/home/teleo/.hermes/hermes-agent")
|
|
DEPLOY_ROOT = Path("/opt/teleo-eval/workspaces/deploy-infra")
|
|
CHAT_ID = "__CHAT_ID__"
|
|
USER_ID = "__USER_ID__"
|
|
USER_NAME = "__USER_NAME__"
|
|
RUN_ID = "__RUN_ID__"
|
|
PROMPTS = __PROMPTS_JSON__
|
|
REPORT_PREFIX = __REPORT_PREFIX_JSON__
|
|
REPORT_PATH = Path(f"/tmp/{REPORT_PREFIX}-{RUN_ID}.json")
|
|
|
|
|
|
def service_state():
|
|
raw = subprocess.check_output(
|
|
[
|
|
"systemctl",
|
|
"show",
|
|
"leoclean-gateway.service",
|
|
"-p",
|
|
"ActiveState",
|
|
"-p",
|
|
"SubState",
|
|
"-p",
|
|
"MainPID",
|
|
"-p",
|
|
"NRestarts",
|
|
"-p",
|
|
"ExecMainStartTimestamp",
|
|
],
|
|
text=True,
|
|
)
|
|
result = {}
|
|
for line in raw.splitlines():
|
|
if "=" in line:
|
|
key, value = line.split("=", 1)
|
|
result[key] = value
|
|
return result
|
|
|
|
|
|
def db_counts():
|
|
sql = """
|
|
select jsonb_build_object(
|
|
'public.claims', (select count(*) from public.claims),
|
|
'public.sources', (select count(*) from public.sources),
|
|
'public.claim_evidence', (select count(*) from public.claim_evidence),
|
|
'public.claim_edges', (select count(*) from public.claim_edges),
|
|
'kb_stage.kb_proposals', (select count(*) from kb_stage.kb_proposals)
|
|
)::text;
|
|
"""
|
|
try:
|
|
raw = subprocess.check_output(
|
|
["docker", "exec", "-i", "teleo-pg", "psql", "-U", "postgres", "-d", "teleo", "-At", "-q"],
|
|
input=sql,
|
|
text=True,
|
|
stderr=subprocess.STDOUT,
|
|
)
|
|
return json.loads(raw.strip())
|
|
except Exception as exc:
|
|
return {"error": str(exc)}
|
|
|
|
|
|
def copy_profile() -> Path:
|
|
tmp_root = Path(tempfile.mkdtemp(prefix="leo-direct-claim-handler-"))
|
|
target = tmp_root / "profile"
|
|
|
|
def ignore(_dir, names):
|
|
ignored = {
|
|
"logs",
|
|
"cache",
|
|
"image_cache",
|
|
"memory_backups",
|
|
"memories",
|
|
"sessions",
|
|
"state",
|
|
".pytest_cache",
|
|
"__pycache__",
|
|
"gateway.pid",
|
|
"auth.lock",
|
|
".skills_prompt_snapshot.json",
|
|
"state.db",
|
|
"state.db-shm",
|
|
"state.db-wal",
|
|
}
|
|
return {
|
|
name
|
|
for name in names
|
|
if name in ignored
|
|
or name.endswith(".lock")
|
|
or ".bak-" in name
|
|
or name.endswith(".bak")
|
|
}
|
|
|
|
try:
|
|
shutil.copytree(LIVE_PROFILE, target, symlinks=True, ignore=ignore)
|
|
for directory in ("memories", "sessions", "state"):
|
|
(target / directory).mkdir(mode=0o700, exist_ok=True)
|
|
except Exception:
|
|
shutil.rmtree(tmp_root, ignore_errors=True)
|
|
raise
|
|
return target
|
|
|
|
|
|
def remove_tree(path):
|
|
if not path or not path.exists():
|
|
return True
|
|
try:
|
|
shutil.rmtree(path)
|
|
except Exception:
|
|
for root, dirs, files in os.walk(path, topdown=False):
|
|
for name in files:
|
|
p = Path(root) / name
|
|
try:
|
|
p.chmod(0o600)
|
|
p.unlink()
|
|
except Exception:
|
|
pass
|
|
for name in dirs:
|
|
p = Path(root) / name
|
|
try:
|
|
p.chmod(0o700)
|
|
p.rmdir()
|
|
except Exception:
|
|
pass
|
|
try:
|
|
path.chmod(0o700)
|
|
path.rmdir()
|
|
except Exception:
|
|
pass
|
|
return not path.exists()
|
|
|
|
|
|
def write_report_snapshot(report):
|
|
try:
|
|
REPORT_PATH.write_text(json.dumps(report, sort_keys=True), encoding="utf-8")
|
|
except Exception:
|
|
pass
|
|
|
|
|
|
def read_database_context_trace(path):
|
|
if not path or not path.exists():
|
|
return []
|
|
records = []
|
|
for line in path.read_text(encoding="utf-8").splitlines():
|
|
try:
|
|
value = json.loads(line)
|
|
except json.JSONDecodeError:
|
|
continue
|
|
if isinstance(value, dict):
|
|
records.append(value)
|
|
return records
|
|
|
|
|
|
def current_agent_messages(runner, session_key):
|
|
cache = getattr(runner, "_agent_cache", None) or {}
|
|
cache_lock = getattr(runner, "_agent_cache_lock", None)
|
|
if cache_lock is not None:
|
|
with cache_lock:
|
|
cached = cache.get(session_key)
|
|
else:
|
|
cached = cache.get(session_key)
|
|
if not cached:
|
|
return []
|
|
agent = cached[0]
|
|
messages = getattr(agent, "_session_messages", None) or []
|
|
return [dict(message) for message in messages if isinstance(message, dict)]
|
|
|
|
|
|
async def run_suite():
|
|
sys.path.insert(0, str(DEPLOY_ROOT / "scripts"))
|
|
from leo_behavior_manifest import build_manifest as build_behavior_manifest
|
|
from leo_tool_trace import extract_kb_tool_trace
|
|
|
|
before_service = service_state()
|
|
before_counts = db_counts()
|
|
behavior_before = build_behavior_manifest(LIVE_PROFILE, AGENT_ROOT)
|
|
temp_profile = None
|
|
context_trace_path = None
|
|
temp_removed = False
|
|
report = {
|
|
"generated_at_utc": datetime.now(timezone.utc).isoformat(),
|
|
"mode": __SUITE_MODE_JSON__,
|
|
"posted_to_telegram": False,
|
|
"mutates_kb_by_harness": False,
|
|
"live_behavior_manifest_before": behavior_before,
|
|
"temp_profile_seed": {
|
|
"mode": "static_runtime_surfaces_only",
|
|
"excluded": [
|
|
"memories",
|
|
"sessions",
|
|
"state",
|
|
"state.db",
|
|
"state.db-shm",
|
|
"state.db-wal",
|
|
".skills_prompt_snapshot.json",
|
|
],
|
|
"same_session_continuity_starts_fresh": True,
|
|
},
|
|
"source": {
|
|
"platform": "telegram",
|
|
"chat_id": CHAT_ID,
|
|
"chat_name": "Leo",
|
|
"chat_type": "group",
|
|
"user_id": USER_ID,
|
|
"user_name": USER_NAME,
|
|
},
|
|
"harness_notes": [
|
|
__PROMPT_NOTE_JSON__,
|
|
"The harness uses a temporary copy of the leoclean profile and does not post to Telegram.",
|
|
"This proves handler-level live VPS GatewayRunner behavior, not human-visible Telegram delivery.",
|
|
],
|
|
"db_counts_before": before_counts,
|
|
"before_service": before_service,
|
|
"remote_report_path": str(REPORT_PATH),
|
|
}
|
|
write_report_snapshot(report)
|
|
try:
|
|
temp_profile = copy_profile()
|
|
os.environ["HERMES_HOME"] = str(temp_profile)
|
|
os.environ["HOME"] = "/home/teleo"
|
|
context_trace_path = temp_profile / "state" / "leo-db-context-trace.jsonl"
|
|
os.environ["LEO_DB_CONTEXT_TRACE_PATH"] = str(context_trace_path)
|
|
sys.path.insert(0, str(AGENT_ROOT))
|
|
|
|
from gateway.config import Platform
|
|
from gateway.platforms.base import MessageEvent, MessageType
|
|
from gateway.run import GatewayRunner
|
|
from gateway.session import SessionSource
|
|
|
|
source = SessionSource(
|
|
platform=Platform.TELEGRAM,
|
|
chat_id=CHAT_ID,
|
|
chat_name="Leo",
|
|
chat_type="group",
|
|
user_id=USER_ID,
|
|
user_name=USER_NAME,
|
|
)
|
|
runner = GatewayRunner()
|
|
session_key = runner._session_key_for_source(source)
|
|
authorized = runner._is_user_authorized(source)
|
|
report["handler"] = {
|
|
"temp_profile": str(temp_profile),
|
|
"session_key": session_key,
|
|
"authorized": authorized,
|
|
}
|
|
write_report_snapshot(report)
|
|
if not authorized:
|
|
report["results"] = []
|
|
report["pass_runtime"] = False
|
|
report["failure"] = "telegram group source was not authorized by live config"
|
|
write_report_snapshot(report)
|
|
return report
|
|
|
|
results = []
|
|
for index, prompt in enumerate(PROMPTS, start=1):
|
|
trace_start = len(read_database_context_trace(context_trace_path))
|
|
event = MessageEvent(
|
|
text=prompt["message"],
|
|
message_type=MessageType.TEXT,
|
|
source=source,
|
|
message_id=f"handler-suite-{RUN_ID}-{prompt['id']}",
|
|
)
|
|
started = datetime.now(timezone.utc)
|
|
reply = await asyncio.wait_for(runner._handle_message(event), timeout=300)
|
|
ended = datetime.now(timezone.utc)
|
|
database_tool_trace = extract_kb_tool_trace(current_agent_messages(runner, session_key))
|
|
results.append(
|
|
{
|
|
"turn": index,
|
|
"prompt_id": prompt["id"],
|
|
"dimension": prompt["dimension"],
|
|
"prompt": prompt["message"],
|
|
"reply": reply or "",
|
|
"started_at_utc": started.isoformat(),
|
|
"ended_at_utc": ended.isoformat(),
|
|
"ok": bool((reply or "").strip()),
|
|
"mutates_kb": False,
|
|
"evidence_tier": "live_vps_gatewayrunner_temp_profile",
|
|
"claim_ceiling": "Live VPS GatewayRunner reply from temp leoclean profile; no Telegram post; no production DB apply authorized.",
|
|
"database_context_trace": read_database_context_trace(context_trace_path)[trace_start:],
|
|
"database_tool_trace": database_tool_trace,
|
|
}
|
|
)
|
|
report["results"] = results
|
|
write_report_snapshot(report)
|
|
report["results"] = results
|
|
report["pass_runtime"] = all(row["ok"] for row in results)
|
|
return report
|
|
except Exception as exc:
|
|
report["pass_runtime"] = False
|
|
report["error"] = str(exc)
|
|
report["traceback_tail"] = traceback.format_exc().splitlines()[-12:]
|
|
write_report_snapshot(report)
|
|
return report
|
|
finally:
|
|
after_counts = db_counts()
|
|
after_service = service_state()
|
|
behavior_after = build_behavior_manifest(LIVE_PROFILE, AGENT_ROOT)
|
|
report["db_counts_after"] = after_counts
|
|
report["db_counts_changed"] = after_counts != before_counts
|
|
report["live_behavior_manifest_after"] = behavior_after
|
|
report["changed_live_profile"] = behavior_before["behavior_sha256"] != behavior_after["behavior_sha256"]
|
|
report["live_behavior_manifest_unchanged"] = not report["changed_live_profile"]
|
|
report["service_before_after"] = {
|
|
"before": before_service,
|
|
"after": after_service,
|
|
"unchanged_from_preexisting_live_readback": before_service == after_service,
|
|
}
|
|
database_context_trace = read_database_context_trace(context_trace_path)
|
|
if temp_profile is not None:
|
|
tmp_root = temp_profile.parent
|
|
temp_removed = remove_tree(tmp_root)
|
|
report["temp_profile_removed"] = temp_removed
|
|
report["database_context_trace"] = database_context_trace
|
|
context_injections = [
|
|
item for item in database_context_trace if item.get("event", "pre_llm_call") == "pre_llm_call"
|
|
]
|
|
response_validations = [
|
|
item for item in database_context_trace if item.get("event") == "post_llm_call"
|
|
]
|
|
report["database_context_injection_count"] = len(context_injections)
|
|
report["database_context_all_ok"] = bool(context_injections) and all(
|
|
item.get("status") == "ok" and item.get("injected") is True
|
|
for item in context_injections
|
|
)
|
|
report["database_response_validation_count"] = len(response_validations)
|
|
report["database_response_validation_all_ok"] = bool(response_validations) and all(
|
|
item.get("status") == "ok" and item.get("validated") is True
|
|
for item in response_validations
|
|
)
|
|
report["database_response_transform_count"] = sum(
|
|
item.get("transformed") is True for item in response_validations
|
|
)
|
|
tool_traces = [
|
|
result.get("database_tool_trace") or {}
|
|
for result in report.get("results") or []
|
|
]
|
|
report["database_tool_call_proven_count"] = sum(
|
|
trace.get("database_tool_call_proven") is True for trace in tool_traces
|
|
)
|
|
report["database_tool_call_proven"] = any(
|
|
trace.get("database_tool_call_proven") is True for trace in tool_traces
|
|
)
|
|
report["database_retrieval_receipt_proven"] = any(
|
|
trace.get("database_retrieval_receipt_proven") is True for trace in tool_traces
|
|
)
|
|
tool_traces_with_calls = [
|
|
trace for trace in tool_traces if trace.get("database_tool_call_count")
|
|
]
|
|
report["database_tool_calls_read_only"] = bool(tool_traces_with_calls) and all(
|
|
trace.get("database_tool_calls_read_only") is True
|
|
for trace in tool_traces_with_calls
|
|
)
|
|
write_report_snapshot(report)
|
|
|
|
|
|
print(json.dumps(asyncio.run(run_suite()), sort_keys=True))
|
|
'''
|
|
|
|
|
|
def direct_claim_prompts() -> list[dict[str, Any]]:
|
|
return [
|
|
{
|
|
"id": prompt["id"],
|
|
"dimension": prompt["dimension"],
|
|
"message": prompt["message"],
|
|
}
|
|
for prompt in benchmark.M3TAVERSAL_DIRECT_CLAIM_FOLLOWUP_SCENARIOS
|
|
]
|
|
|
|
|
|
def build_remote_script(
|
|
run_id: str,
|
|
*,
|
|
prompts: list[dict[str, Any]] | None = None,
|
|
suite_mode: str = DEFAULT_SUITE_MODE,
|
|
report_prefix: str = DEFAULT_REPORT_PREFIX,
|
|
prompt_note: str = DEFAULT_PROMPT_NOTE,
|
|
) -> str:
|
|
if not re.fullmatch(r"[A-Za-z0-9._-]+", report_prefix):
|
|
raise ValueError("report_prefix must contain only letters, numbers, dot, underscore, or hyphen")
|
|
selected_prompts = direct_claim_prompts() if prompts is None else prompts
|
|
return (
|
|
REMOTE_SCRIPT.replace("__CHAT_ID__", CHAT_ID)
|
|
.replace("__USER_ID__", USER_ID)
|
|
.replace("__USER_NAME__", USER_NAME)
|
|
.replace("__RUN_ID__", run_id)
|
|
.replace("__PROMPTS_JSON__", json.dumps(selected_prompts))
|
|
.replace("__SUITE_MODE_JSON__", json.dumps(suite_mode))
|
|
.replace("__REPORT_PREFIX_JSON__", json.dumps(report_prefix))
|
|
.replace("__PROMPT_NOTE_JSON__", json.dumps(prompt_note))
|
|
)
|
|
|
|
|
|
def redact(text: str) -> str:
|
|
result = text
|
|
for pattern in SECRET_PATTERNS:
|
|
if pattern.groups >= 2:
|
|
result = pattern.sub(r"\1[REDACTED]", result)
|
|
else:
|
|
result = pattern.sub("[REDACTED_TOKEN]", result)
|
|
return result
|
|
|
|
|
|
def fetch_remote_report(
|
|
run_id: str,
|
|
*,
|
|
report_prefix: str = DEFAULT_REPORT_PREFIX,
|
|
unlink: bool = True,
|
|
) -> dict[str, Any] | None:
|
|
report_path = f"/tmp/{report_prefix}-{run_id}.json"
|
|
unlink_line = " p.unlink()\n" if unlink else ""
|
|
proc = subprocess.run(
|
|
[
|
|
"ssh",
|
|
"-i",
|
|
str(SSH_KEY),
|
|
"-o",
|
|
"BatchMode=yes",
|
|
"-o",
|
|
"StrictHostKeyChecking=accept-new",
|
|
VPS,
|
|
"sudo -u teleo -H /home/teleo/.hermes/hermes-agent/venv/bin/python -",
|
|
],
|
|
input=(
|
|
"import json\n"
|
|
"from pathlib import Path\n"
|
|
f"p = Path({report_path!r})\n"
|
|
"if p.exists():\n"
|
|
" print(p.read_text(encoding='utf-8'))\n"
|
|
f"{unlink_line}"
|
|
),
|
|
text=True,
|
|
capture_output=True,
|
|
timeout=60,
|
|
)
|
|
if proc.returncode != 0 or not proc.stdout.strip():
|
|
return None
|
|
return json.loads(redact(proc.stdout.strip()))
|
|
|
|
|
|
def run_remote(
|
|
*,
|
|
prompts: list[dict[str, Any]] | None = None,
|
|
suite_mode: str = DEFAULT_SUITE_MODE,
|
|
report_prefix: str = DEFAULT_REPORT_PREFIX,
|
|
prompt_note: str = DEFAULT_PROMPT_NOTE,
|
|
) -> dict[str, Any]:
|
|
run_id = uuid.uuid4().hex[:12]
|
|
proc = subprocess.run(
|
|
[
|
|
"ssh",
|
|
"-i",
|
|
str(SSH_KEY),
|
|
"-o",
|
|
"BatchMode=yes",
|
|
"-o",
|
|
"StrictHostKeyChecking=accept-new",
|
|
VPS,
|
|
"cd /home/teleo && sudo -u teleo -H /home/teleo/.hermes/hermes-agent/venv/bin/python -",
|
|
],
|
|
input=build_remote_script(
|
|
run_id,
|
|
prompts=prompts,
|
|
suite_mode=suite_mode,
|
|
report_prefix=report_prefix,
|
|
prompt_note=prompt_note,
|
|
),
|
|
text=True,
|
|
capture_output=True,
|
|
timeout=2100,
|
|
)
|
|
result: dict[str, Any] = {
|
|
"returncode": proc.returncode,
|
|
"stdout": redact(proc.stdout),
|
|
"stderr": redact(proc.stderr),
|
|
"run_id": run_id,
|
|
}
|
|
if proc.returncode == 0 and proc.stdout.strip():
|
|
for line in reversed(proc.stdout.splitlines()):
|
|
candidate = line.strip()
|
|
if not candidate.startswith("{"):
|
|
continue
|
|
try:
|
|
result["parsed"] = json.loads(candidate)
|
|
break
|
|
except json.JSONDecodeError:
|
|
continue
|
|
if "parsed" not in result:
|
|
result["parse_error"] = "remote stdout contained no parseable JSON object line"
|
|
if "parsed" not in result:
|
|
fetched = fetch_remote_report(run_id, report_prefix=report_prefix)
|
|
if fetched is not None:
|
|
result["parsed"] = fetched
|
|
result["parsed_from_report_file"] = True
|
|
else:
|
|
fetch_remote_report(run_id, report_prefix=report_prefix)
|
|
return result
|
|
|
|
|
|
def write_output(remote: dict[str, Any], *, output_json: Path = OUTPUT_JSON) -> dict[str, Any]:
|
|
REPORT_DIR.mkdir(parents=True, exist_ok=True)
|
|
parsed = remote.get("parsed") or {}
|
|
report = parsed if isinstance(parsed, dict) else {}
|
|
report["source_report_path"] = str(output_json)
|
|
report["remote_returncode"] = remote.get("returncode")
|
|
report["remote_run_id"] = remote.get("run_id")
|
|
if remote.get("stderr"):
|
|
report["remote_stderr"] = remote["stderr"]
|
|
output_json.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
|
return report
|
|
|
|
|
|
def main() -> int:
|
|
remote = run_remote()
|
|
report = write_output(remote)
|
|
print(json.dumps({"json": str(OUTPUT_JSON), "pass_runtime": report.get("pass_runtime")}, indent=2))
|
|
return 0 if remote.get("returncode") == 0 and report.get("pass_runtime") else 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|