1036 lines
38 KiB
Python
Executable file
1036 lines
38 KiB
Python
Executable file
#!/usr/bin/env python3
|
|
"""Run the m3taversal-style direct-claim suite through live VPS GatewayRunner.
|
|
|
|
The harness does not post to Telegram and does not write to the production KB.
|
|
It copies only static behavior surfaces from the live leoclean profile to a
|
|
temporary profile on the VPS, invokes GatewayRunner._handle_message with
|
|
Telegram-shaped MessageEvents, and removes the temporary profile after the
|
|
run. Prior sessions, memory, state databases, and generated prompt caches are
|
|
excluded so the result can be attributed to the declared runtime inputs.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import re
|
|
import subprocess
|
|
import uuid
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
try:
|
|
import working_leo_open_ended_benchmark as benchmark
|
|
except ImportError: # pragma: no cover - imported as scripts.* in tests
|
|
from scripts import working_leo_open_ended_benchmark as benchmark
|
|
|
|
try:
|
|
from leo_turn_execution_manifest import attach_execution_manifests
|
|
except ImportError: # pragma: no cover - imported as scripts.* in tests
|
|
from scripts.leo_turn_execution_manifest import attach_execution_manifests
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
REPORT_DIR = ROOT / "docs" / "reports" / "leo-working-state-20260709"
|
|
OUTPUT_JSON = REPORT_DIR / "telegram-handler-direct-claim-suite-current.json"
|
|
SSH_KEY = Path.home() / ".ssh/livingip_hetzner_20260604_ed25519"
|
|
VPS = "root@77.42.65.182"
|
|
|
|
DEFAULT_SUITE_MODE = "live_vps_gatewayrunner_temp_profile_direct_claim_suite"
|
|
DEFAULT_REPORT_PREFIX = "leo-direct-claim-handler-report"
|
|
DEFAULT_PROMPT_NOTE = "Prompts are the exact DC-01..DC-06 direct-claim benchmark messages."
|
|
|
|
CHAT_ID = "-5146042086"
|
|
USER_ID = "9070919"
|
|
USER_NAME = "codex handler direct claim"
|
|
|
|
JSON_SECRET_PATTERN = re.compile(
|
|
r'("(?:api[_-]?key|access[_-]?token|refresh[_-]?token|bot[_-]?token|token|secret|password)"\s*:\s*)'
|
|
r'("(?:\\.|[^"\\])*"|null|[^\s,}\]]+)',
|
|
re.IGNORECASE,
|
|
)
|
|
TELEGRAM_TOKEN_PATTERN = re.compile(r"\b\d{6,}:[A-Za-z0-9_-]{20,}\b")
|
|
AUTHORIZATION_PATTERN = re.compile(r"(Authorization:\s*Bearer\s+)([^\s\"']+)", re.IGNORECASE)
|
|
ASSIGNMENT_SECRET_PATTERN = re.compile(
|
|
r"((?:api[_-]?key|access[_-]?token|refresh[_-]?token|bot[_-]?token|token|secret|password)"
|
|
r"\s*[=:]\s*)([^\s,;\"']+)",
|
|
re.IGNORECASE,
|
|
)
|
|
|
|
DB_CONTEXT_PLUGIN = (
|
|
ROOT / "hermes-agent" / "leoclean-plugins" / "vps" / "leo-db-context" / "__init__.py"
|
|
)
|
|
KB_TOOL = ROOT / "hermes-agent" / "leoclean-bin" / "kb_tool.py"
|
|
BEHAVIOR_MANIFEST = ROOT / "scripts" / "leo_behavior_manifest.py"
|
|
|
|
|
|
REMOTE_SCRIPT = r'''
|
|
import asyncio
|
|
import copy
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import traceback
|
|
import types
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
|
|
LIVE_PROFILE = Path("/home/teleo/.hermes/profiles/leoclean")
|
|
AGENT_ROOT = Path("/home/teleo/.hermes/hermes-agent")
|
|
DEPLOY_ROOT = Path("/opt/teleo-eval/workspaces/deploy-infra")
|
|
CHAT_ID = "__CHAT_ID__"
|
|
USER_ID = "__USER_ID__"
|
|
USER_NAME = "__USER_NAME__"
|
|
RUN_ID = "__RUN_ID__"
|
|
PROMPTS = __PROMPTS_JSON__
|
|
REPORT_PREFIX = __REPORT_PREFIX_JSON__
|
|
REPORT_PATH = Path(f"/tmp/{REPORT_PREFIX}-{RUN_ID}.json")
|
|
DB_CONTEXT_PLUGIN_SOURCE = __DB_CONTEXT_PLUGIN_SOURCE_JSON__
|
|
KB_TOOL_SOURCE = __KB_TOOL_SOURCE_JSON__
|
|
BEHAVIOR_MANIFEST_SOURCE = __BEHAVIOR_MANIFEST_SOURCE_JSON__
|
|
TURN_TRACE_PLUGIN_SOURCE = """\
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
|
|
|
|
def _safe_usage(value):
|
|
if not isinstance(value, dict):
|
|
return {}
|
|
safe = {}
|
|
for key in (
|
|
"input_tokens",
|
|
"output_tokens",
|
|
"prompt_tokens",
|
|
"completion_tokens",
|
|
"total_tokens",
|
|
"cache_read_tokens",
|
|
"cache_write_tokens",
|
|
"reasoning_tokens",
|
|
):
|
|
item = value.get(key)
|
|
if isinstance(item, (int, float)) and not isinstance(item, bool):
|
|
safe[key] = item
|
|
return safe
|
|
|
|
|
|
def _sha(value):
|
|
return hashlib.sha256(str(value or "").encode("utf-8")).hexdigest()
|
|
|
|
|
|
def _write(record):
|
|
raw_path = os.getenv("LEO_TURN_API_TRACE_PATH", "").strip()
|
|
if not raw_path:
|
|
return
|
|
path = Path(raw_path)
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
with path.open("a", encoding="utf-8") as handle:
|
|
handle.write(json.dumps(record, sort_keys=True) + "\\n")
|
|
try:
|
|
path.chmod(0o600)
|
|
except OSError:
|
|
pass
|
|
|
|
|
|
def _pre_llm_call(**kwargs):
|
|
_write({
|
|
"generated_at_utc": datetime.now(timezone.utc).isoformat(),
|
|
"event": "turn_pre_llm_call",
|
|
"session_id_sha256": _sha(kwargs.get("session_id")),
|
|
"prompt_sha256": _sha(kwargs.get("user_message")),
|
|
})
|
|
return None
|
|
|
|
|
|
def _post_api_request(**kwargs):
|
|
record = {
|
|
"generated_at_utc": datetime.now(timezone.utc).isoformat(),
|
|
"event": "post_api_request",
|
|
"task_id_sha256": _sha(kwargs.get("task_id")),
|
|
"session_id_sha256": _sha(kwargs.get("session_id")),
|
|
"model": kwargs.get("model"),
|
|
"provider": kwargs.get("provider"),
|
|
"base_url_sha256": _sha(kwargs.get("base_url")),
|
|
"api_mode": kwargs.get("api_mode"),
|
|
"api_call_count": kwargs.get("api_call_count"),
|
|
"finish_reason": kwargs.get("finish_reason"),
|
|
"message_count": kwargs.get("message_count"),
|
|
"response_model": kwargs.get("response_model"),
|
|
"usage": _safe_usage(kwargs.get("usage")),
|
|
"assistant_content_chars": kwargs.get("assistant_content_chars"),
|
|
"assistant_tool_call_count": kwargs.get("assistant_tool_call_count"),
|
|
"provider_response_id": None,
|
|
"provider_response_id_status": "not_exposed_by_hermes_post_api_request_hook",
|
|
}
|
|
_write(record)
|
|
return None
|
|
|
|
|
|
def _post_llm_call(**kwargs):
|
|
_write({
|
|
"generated_at_utc": datetime.now(timezone.utc).isoformat(),
|
|
"event": "turn_post_llm_call",
|
|
"session_id_sha256": _sha(kwargs.get("session_id")),
|
|
"prompt_sha256": _sha(kwargs.get("user_message")),
|
|
"raw_assistant_response_sha256": _sha(kwargs.get("assistant_response")),
|
|
})
|
|
return None
|
|
|
|
|
|
def register(ctx):
|
|
ctx.register_hook("pre_llm_call", _pre_llm_call)
|
|
ctx.register_hook("post_api_request", _post_api_request)
|
|
ctx.register_hook("post_llm_call", _post_llm_call)
|
|
"""
|
|
TURN_TRACE_PLUGIN_MANIFEST = """\
|
|
name: leo-turn-execution-trace
|
|
version: "1.0.0"
|
|
description: Secret-safe model-call trace for no-send Leo harnesses.
|
|
provides_hooks:
|
|
- pre_llm_call
|
|
- post_api_request
|
|
- post_llm_call
|
|
"""
|
|
|
|
|
|
def service_state():
|
|
raw = subprocess.check_output(
|
|
[
|
|
"systemctl",
|
|
"show",
|
|
"leoclean-gateway.service",
|
|
"-p",
|
|
"ActiveState",
|
|
"-p",
|
|
"SubState",
|
|
"-p",
|
|
"MainPID",
|
|
"-p",
|
|
"NRestarts",
|
|
"-p",
|
|
"ExecMainStartTimestamp",
|
|
],
|
|
text=True,
|
|
)
|
|
result = {}
|
|
for line in raw.splitlines():
|
|
if "=" in line:
|
|
key, value = line.split("=", 1)
|
|
result[key] = value
|
|
return result
|
|
|
|
|
|
def db_counts():
|
|
sql = """
|
|
select jsonb_build_object(
|
|
'public.claims', (select count(*) from public.claims),
|
|
'public.sources', (select count(*) from public.sources),
|
|
'public.claim_evidence', (select count(*) from public.claim_evidence),
|
|
'public.claim_edges', (select count(*) from public.claim_edges),
|
|
'kb_stage.kb_proposals', (select count(*) from kb_stage.kb_proposals)
|
|
)::text;
|
|
"""
|
|
try:
|
|
raw = subprocess.check_output(
|
|
["docker", "exec", "-i", "teleo-pg", "psql", "-U", "postgres", "-d", "teleo", "-At", "-q"],
|
|
input=sql,
|
|
text=True,
|
|
stderr=subprocess.STDOUT,
|
|
)
|
|
return json.loads(raw.strip())
|
|
except Exception as exc:
|
|
return {"error": str(exc)}
|
|
|
|
|
|
def db_read_marker():
|
|
sql = """
|
|
select jsonb_build_object(
|
|
'database', current_database(),
|
|
'database_user', current_user,
|
|
'system_identifier', (select system_identifier::text from pg_control_system()),
|
|
'wal_lsn', pg_current_wal_lsn()::text
|
|
)::text;
|
|
"""
|
|
raw = subprocess.check_output(
|
|
["docker", "exec", "-i", "teleo-pg", "psql", "-U", "postgres", "-d", "teleo", "-At", "-q"],
|
|
input=sql,
|
|
text=True,
|
|
stderr=subprocess.STDOUT,
|
|
)
|
|
return json.loads(raw.strip())
|
|
|
|
|
|
def db_fingerprint():
|
|
sql_path = DEPLOY_ROOT / "ops" / "postgres_parity_manifest.sql"
|
|
before = db_read_marker()
|
|
raw = subprocess.check_output(
|
|
["docker", "exec", "-i", "teleo-pg", "psql", "-U", "postgres", "-d", "teleo", "-At", "-q"],
|
|
input=sql_path.read_text(encoding="utf-8"),
|
|
text=True,
|
|
stderr=subprocess.STDOUT,
|
|
timeout=180,
|
|
)
|
|
after = db_read_marker()
|
|
rows = []
|
|
for line in raw.splitlines():
|
|
candidate = line.strip()
|
|
if not candidate or candidate in {"BEGIN", "SET", "ROLLBACK"}:
|
|
continue
|
|
value = json.loads(candidate)
|
|
if isinstance(value, dict):
|
|
rows.append(value)
|
|
tables = {
|
|
f"{row['schema']}.{row['table']}": {
|
|
"row_count": int(row["row_count"]),
|
|
"rowset_md5": row["rowset_md5"],
|
|
}
|
|
for row in rows
|
|
if row.get("kind") == "table"
|
|
}
|
|
singleton = {
|
|
str(row["kind"]): row.get("items", [])
|
|
for row in rows
|
|
if row.get("kind")
|
|
in {
|
|
"schemas",
|
|
"extensions",
|
|
"application_roles",
|
|
"columns",
|
|
"constraints",
|
|
"indexes",
|
|
"sequences",
|
|
"views",
|
|
"functions",
|
|
"triggers",
|
|
"types",
|
|
"policies",
|
|
}
|
|
}
|
|
identity_rows = [row for row in rows if row.get("kind") == "identity"]
|
|
if len(identity_rows) != 1 or not tables:
|
|
raise RuntimeError("canonical database fingerprint manifest is incomplete")
|
|
identity = identity_rows[0]
|
|
stable = {
|
|
"identity": {
|
|
key: identity.get(key)
|
|
for key in (
|
|
"database",
|
|
"server_version_num",
|
|
"server_encoding",
|
|
"database_collation",
|
|
"database_ctype",
|
|
"transaction_read_only",
|
|
)
|
|
},
|
|
"singleton_sha256": {
|
|
key: hashlib.sha256(
|
|
json.dumps(value, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
|
).hexdigest()
|
|
for key, value in sorted(singleton.items())
|
|
},
|
|
"tables": tables,
|
|
}
|
|
return {
|
|
"schema": "livingip.teleoCanonicalDatabaseFingerprint.v1",
|
|
"status": "ok",
|
|
"fingerprint_sha256": hashlib.sha256(
|
|
json.dumps(stable, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
|
).hexdigest(),
|
|
"table_count": len(tables),
|
|
"total_rows": sum(item["row_count"] for item in tables.values()),
|
|
"table_rows_sha256": hashlib.sha256(
|
|
json.dumps(tables, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
|
).hexdigest(),
|
|
"structure_sha256": hashlib.sha256(
|
|
json.dumps(stable["singleton_sha256"], sort_keys=True, separators=(",", ":")).encode("utf-8")
|
|
).hexdigest(),
|
|
"read_consistency": {
|
|
"status": "stable_wal_marker" if before == after else "wal_changed_during_fingerprint",
|
|
"before": before,
|
|
"after": after,
|
|
},
|
|
}
|
|
|
|
|
|
def safe_db_fingerprint():
|
|
try:
|
|
return db_fingerprint()
|
|
except Exception as exc:
|
|
return {
|
|
"schema": "livingip.teleoCanonicalDatabaseFingerprint.v1",
|
|
"status": "error",
|
|
"error": type(exc).__name__,
|
|
}
|
|
|
|
|
|
def copy_profile() -> Path:
|
|
tmp_root = Path(tempfile.mkdtemp(prefix="leo-direct-claim-handler-"))
|
|
target = tmp_root / "profile"
|
|
|
|
def ignore(_dir, names):
|
|
ignored = {
|
|
"logs",
|
|
"cache",
|
|
"image_cache",
|
|
"memory_backups",
|
|
"memories",
|
|
"sessions",
|
|
"state",
|
|
".pytest_cache",
|
|
"__pycache__",
|
|
"gateway.pid",
|
|
"auth.lock",
|
|
".skills_prompt_snapshot.json",
|
|
"state.db",
|
|
"state.db-shm",
|
|
"state.db-wal",
|
|
}
|
|
return {
|
|
name
|
|
for name in names
|
|
if name in ignored
|
|
or name.endswith(".lock")
|
|
or ".bak-" in name
|
|
or name.endswith(".bak")
|
|
}
|
|
|
|
try:
|
|
shutil.copytree(LIVE_PROFILE, target, symlinks=True, ignore=ignore)
|
|
for directory in ("memories", "sessions", "state"):
|
|
(target / directory).mkdir(mode=0o700, exist_ok=True)
|
|
except Exception:
|
|
shutil.rmtree(tmp_root, ignore_errors=True)
|
|
raise
|
|
return target
|
|
|
|
|
|
def install_harness_plugins(profile):
|
|
db_context = profile / "plugins" / "leo-db-context" / "__init__.py"
|
|
db_context.parent.mkdir(parents=True, exist_ok=True)
|
|
db_context.write_text(DB_CONTEXT_PLUGIN_SOURCE, encoding="utf-8")
|
|
|
|
kb_tool = profile / "bin" / "kb_tool.py"
|
|
kb_tool.parent.mkdir(parents=True, exist_ok=True)
|
|
kb_tool.write_text(KB_TOOL_SOURCE, encoding="utf-8")
|
|
kb_tool.chmod(0o700)
|
|
|
|
turn_trace = profile / "plugins" / "leo-turn-execution-trace"
|
|
turn_trace.mkdir(parents=True, exist_ok=True)
|
|
(turn_trace / "__init__.py").write_text(TURN_TRACE_PLUGIN_SOURCE, encoding="utf-8")
|
|
(turn_trace / "plugin.yaml").write_text(TURN_TRACE_PLUGIN_MANIFEST, encoding="utf-8")
|
|
|
|
|
|
def read_jsonl(path):
|
|
if not path or not path.exists():
|
|
return []
|
|
records = []
|
|
for line in path.read_text(encoding="utf-8").splitlines():
|
|
try:
|
|
value = json.loads(line)
|
|
except json.JSONDecodeError:
|
|
continue
|
|
if isinstance(value, dict):
|
|
records.append(value)
|
|
return records
|
|
|
|
|
|
def remove_tree(path):
|
|
if not path or not path.exists():
|
|
return True
|
|
try:
|
|
shutil.rmtree(path)
|
|
except Exception:
|
|
for root, dirs, files in os.walk(path, topdown=False):
|
|
for name in files:
|
|
p = Path(root) / name
|
|
try:
|
|
p.chmod(0o600)
|
|
p.unlink()
|
|
except Exception:
|
|
pass
|
|
for name in dirs:
|
|
p = Path(root) / name
|
|
try:
|
|
p.chmod(0o700)
|
|
p.rmdir()
|
|
except Exception:
|
|
pass
|
|
try:
|
|
path.chmod(0o700)
|
|
path.rmdir()
|
|
except Exception:
|
|
pass
|
|
return not path.exists()
|
|
|
|
|
|
def write_report_snapshot(report):
|
|
try:
|
|
REPORT_PATH.write_text(json.dumps(report, sort_keys=True), encoding="utf-8")
|
|
REPORT_PATH.chmod(0o600)
|
|
except Exception:
|
|
pass
|
|
|
|
|
|
def read_database_context_trace(path):
|
|
return read_jsonl(path)
|
|
|
|
|
|
def current_agent_messages(runner, session_key):
|
|
cache = getattr(runner, "_agent_cache", None) or {}
|
|
cache_lock = getattr(runner, "_agent_cache_lock", None)
|
|
if cache_lock is not None:
|
|
with cache_lock:
|
|
cached = cache.get(session_key)
|
|
else:
|
|
cached = cache.get(session_key)
|
|
if not cached:
|
|
return []
|
|
agent = cached[0]
|
|
messages = getattr(agent, "_session_messages", None) or []
|
|
return [copy.deepcopy(message) for message in messages if isinstance(message, dict)]
|
|
|
|
|
|
def conversation_trace(messages):
|
|
rows = []
|
|
for message in messages:
|
|
content = message.get("content")
|
|
content_text = content if isinstance(content, str) else json.dumps(content, sort_keys=True, default=str)
|
|
tool_calls = message.get("tool_calls") or []
|
|
rows.append({
|
|
"role": message.get("role"),
|
|
"content_sha256": hashlib.sha256(content_text.encode("utf-8")).hexdigest(),
|
|
"content_bytes": len(content_text.encode("utf-8")),
|
|
"tool_calls_sha256": hashlib.sha256(
|
|
json.dumps(tool_calls, sort_keys=True, separators=(",", ":"), default=str).encode("utf-8")
|
|
).hexdigest(),
|
|
"tool_call_count": len(tool_calls),
|
|
})
|
|
return {
|
|
"message_count": len(rows),
|
|
"messages_sha256": hashlib.sha256(
|
|
json.dumps(rows, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
|
).hexdigest(),
|
|
"last_message_role": rows[-1]["role"] if rows else None,
|
|
"last_message_content_sha256": rows[-1]["content_sha256"] if rows else None,
|
|
}
|
|
|
|
|
|
def model_response_trace(messages):
|
|
return [
|
|
{
|
|
"content_sha256": row["content_sha256"],
|
|
"content_bytes": row["content_bytes"],
|
|
"tool_calls_sha256": row["tool_calls_sha256"],
|
|
"tool_call_count": row["tool_call_count"],
|
|
}
|
|
for message, row in zip(messages, conversation_trace_rows(messages))
|
|
if message.get("role") == "assistant"
|
|
]
|
|
|
|
|
|
def conversation_trace_rows(messages):
|
|
rows = []
|
|
for message in messages:
|
|
content = message.get("content")
|
|
content_text = content if isinstance(content, str) else json.dumps(content, sort_keys=True, default=str)
|
|
tool_calls = message.get("tool_calls") or []
|
|
rows.append({
|
|
"role": message.get("role"),
|
|
"content_sha256": hashlib.sha256(content_text.encode("utf-8")).hexdigest(),
|
|
"content_bytes": len(content_text.encode("utf-8")),
|
|
"tool_calls_sha256": hashlib.sha256(
|
|
json.dumps(tool_calls, sort_keys=True, separators=(",", ":"), default=str).encode("utf-8")
|
|
).hexdigest(),
|
|
"tool_call_count": len(tool_calls),
|
|
})
|
|
return rows
|
|
|
|
|
|
async def run_suite():
|
|
sys.path.insert(0, str(DEPLOY_ROOT / "scripts"))
|
|
from leo_tool_trace import extract_kb_tool_trace
|
|
|
|
behavior_manifest_module = types.ModuleType("leo_behavior_manifest_harness")
|
|
behavior_manifest_module.__file__ = "<embedded leo_behavior_manifest.py>"
|
|
exec(compile(BEHAVIOR_MANIFEST_SOURCE, behavior_manifest_module.__file__, "exec"), behavior_manifest_module.__dict__)
|
|
|
|
def build_behavior_manifest(profile):
|
|
return behavior_manifest_module.build_manifest(profile, AGENT_ROOT, DEPLOY_ROOT)
|
|
|
|
before_service = service_state()
|
|
before_counts = db_counts()
|
|
before_fingerprint = safe_db_fingerprint()
|
|
behavior_before = build_behavior_manifest(LIVE_PROFILE)
|
|
temp_profile = None
|
|
context_trace_path = None
|
|
model_trace_path = None
|
|
temp_removed = False
|
|
report = {
|
|
"generated_at_utc": datetime.now(timezone.utc).isoformat(),
|
|
"mode": __SUITE_MODE_JSON__,
|
|
"posted_to_telegram": False,
|
|
"mutates_kb_by_harness": False,
|
|
"live_behavior_manifest_before": behavior_before,
|
|
"temp_profile_seed": {
|
|
"mode": "static_runtime_surfaces_only",
|
|
"excluded": [
|
|
"memories",
|
|
"sessions",
|
|
"state",
|
|
"state.db",
|
|
"state.db-shm",
|
|
"state.db-wal",
|
|
".skills_prompt_snapshot.json",
|
|
],
|
|
"same_session_continuity_starts_fresh": True,
|
|
},
|
|
"source": {
|
|
"platform": "telegram",
|
|
"chat_id": CHAT_ID,
|
|
"chat_name": "Leo",
|
|
"chat_type": "group",
|
|
"user_id": USER_ID,
|
|
"user_name": USER_NAME,
|
|
},
|
|
"harness_notes": [
|
|
__PROMPT_NOTE_JSON__,
|
|
"The harness uses a temporary copy of the leoclean profile and does not post to Telegram.",
|
|
"This proves handler-level live VPS GatewayRunner behavior, not human-visible Telegram delivery.",
|
|
],
|
|
"db_counts_before": before_counts,
|
|
"db_fingerprint_before": before_fingerprint,
|
|
"before_service": before_service,
|
|
"remote_report_path": str(REPORT_PATH),
|
|
}
|
|
write_report_snapshot(report)
|
|
try:
|
|
temp_profile = copy_profile()
|
|
install_harness_plugins(temp_profile)
|
|
os.environ["HERMES_HOME"] = str(temp_profile)
|
|
os.environ["HOME"] = "/home/teleo"
|
|
context_trace_path = temp_profile / "state" / "leo-db-context-trace.jsonl"
|
|
model_trace_path = temp_profile / "state" / "leo-turn-api-trace.jsonl"
|
|
os.environ["LEO_DB_CONTEXT_TRACE_PATH"] = str(context_trace_path)
|
|
os.environ["LEO_TURN_API_TRACE_PATH"] = str(model_trace_path)
|
|
report["executed_behavior_manifest"] = build_behavior_manifest(temp_profile)
|
|
sys.path.insert(0, str(AGENT_ROOT))
|
|
|
|
from gateway.config import Platform
|
|
from gateway.platforms.base import MessageEvent, MessageType
|
|
from gateway.run import GatewayRunner
|
|
from gateway.session import SessionSource
|
|
|
|
source = SessionSource(
|
|
platform=Platform.TELEGRAM,
|
|
chat_id=CHAT_ID,
|
|
chat_name="Leo",
|
|
chat_type="group",
|
|
user_id=USER_ID,
|
|
user_name=USER_NAME,
|
|
)
|
|
runner = GatewayRunner()
|
|
session_key = runner._session_key_for_source(source)
|
|
authorized = runner._is_user_authorized(source)
|
|
report["handler"] = {
|
|
"temp_profile": str(temp_profile),
|
|
"session_key": session_key,
|
|
"authorized": authorized,
|
|
}
|
|
write_report_snapshot(report)
|
|
if not authorized:
|
|
report["results"] = []
|
|
report["pass_runtime"] = False
|
|
report["failure"] = "telegram group source was not authorized by live config"
|
|
write_report_snapshot(report)
|
|
return report
|
|
|
|
results = []
|
|
for index, prompt in enumerate(PROMPTS, start=1):
|
|
trace_start = len(read_database_context_trace(context_trace_path))
|
|
model_trace_start = len(read_jsonl(model_trace_path))
|
|
messages_before = current_agent_messages(runner, session_key)
|
|
event = MessageEvent(
|
|
text=prompt["message"],
|
|
message_type=MessageType.TEXT,
|
|
source=source,
|
|
message_id=f"handler-suite-{RUN_ID}-{prompt['id']}",
|
|
)
|
|
started = datetime.now(timezone.utc)
|
|
reply = await asyncio.wait_for(runner._handle_message(event), timeout=300)
|
|
ended = datetime.now(timezone.utc)
|
|
messages_after = current_agent_messages(runner, session_key)
|
|
before_rows = conversation_trace_rows(messages_before)
|
|
after_rows = conversation_trace_rows(messages_after)
|
|
history_prefix_preserved = after_rows[: len(before_rows)] == before_rows
|
|
new_messages = messages_after[len(messages_before) :] if history_prefix_preserved else messages_after
|
|
database_tool_trace = extract_kb_tool_trace(new_messages)
|
|
results.append(
|
|
{
|
|
"turn": index,
|
|
"prompt_id": prompt["id"],
|
|
"dimension": prompt["dimension"],
|
|
"prompt": prompt["message"],
|
|
"reply": reply or "",
|
|
"started_at_utc": started.isoformat(),
|
|
"ended_at_utc": ended.isoformat(),
|
|
"ok": bool((reply or "").strip()),
|
|
"mutates_kb": False,
|
|
"evidence_tier": "live_vps_gatewayrunner_temp_profile",
|
|
"claim_ceiling": "Live VPS GatewayRunner reply from temp leoclean profile; no Telegram post; no production DB apply authorized.",
|
|
"database_context_trace": read_database_context_trace(context_trace_path)[trace_start:],
|
|
"database_tool_trace": database_tool_trace,
|
|
"model_call_trace": read_jsonl(model_trace_path)[model_trace_start:],
|
|
"model_response_trace": model_response_trace(new_messages),
|
|
"conversation_before": conversation_trace(messages_before),
|
|
"conversation_after": conversation_trace(messages_after),
|
|
"conversation_history_prefix_preserved": history_prefix_preserved,
|
|
}
|
|
)
|
|
report["results"] = results
|
|
write_report_snapshot(report)
|
|
report["results"] = results
|
|
report["pass_runtime"] = all(row["ok"] for row in results)
|
|
return report
|
|
except Exception as exc:
|
|
report["pass_runtime"] = False
|
|
report["error"] = str(exc)
|
|
report["traceback_tail"] = traceback.format_exc().splitlines()[-12:]
|
|
write_report_snapshot(report)
|
|
return report
|
|
finally:
|
|
after_counts = db_counts()
|
|
after_fingerprint = safe_db_fingerprint()
|
|
after_service = service_state()
|
|
behavior_after = build_behavior_manifest(LIVE_PROFILE)
|
|
report["db_counts_after"] = after_counts
|
|
report["db_counts_changed"] = after_counts != before_counts
|
|
report["db_fingerprint_after"] = after_fingerprint
|
|
report["db_fingerprint_unchanged"] = bool(
|
|
before_fingerprint.get("status") == "ok"
|
|
and after_fingerprint.get("status") == "ok"
|
|
and before_fingerprint.get("fingerprint_sha256")
|
|
== after_fingerprint.get("fingerprint_sha256")
|
|
)
|
|
report["live_behavior_manifest_after"] = behavior_after
|
|
report["changed_live_profile"] = behavior_before["behavior_sha256"] != behavior_after["behavior_sha256"]
|
|
report["live_behavior_manifest_unchanged"] = not report["changed_live_profile"]
|
|
report["service_before_after"] = {
|
|
"before": before_service,
|
|
"after": after_service,
|
|
"unchanged_from_preexisting_live_readback": before_service == after_service,
|
|
}
|
|
database_context_trace = read_database_context_trace(context_trace_path)
|
|
model_call_trace = read_jsonl(model_trace_path)
|
|
api_model_trace = [item for item in model_call_trace if item.get("event") == "post_api_request"]
|
|
if temp_profile is not None:
|
|
tmp_root = temp_profile.parent
|
|
temp_removed = remove_tree(tmp_root)
|
|
report["temp_profile_removed"] = temp_removed
|
|
report["database_context_trace"] = database_context_trace
|
|
report["model_call_trace_count"] = len(api_model_trace)
|
|
report["model_call_trace_all_bound"] = bool(api_model_trace) and all(
|
|
item.get("event") == "post_api_request"
|
|
and item.get("model")
|
|
and item.get("provider")
|
|
and item.get("response_model")
|
|
for item in api_model_trace
|
|
)
|
|
context_injections = [
|
|
item for item in database_context_trace if item.get("event", "pre_llm_call") == "pre_llm_call"
|
|
]
|
|
response_validations = [
|
|
item for item in database_context_trace if item.get("event") == "post_llm_call"
|
|
]
|
|
report["database_context_injection_count"] = len(context_injections)
|
|
report["database_context_all_ok"] = bool(context_injections) and all(
|
|
item.get("status") == "ok" and item.get("injected") is True
|
|
for item in context_injections
|
|
)
|
|
report["database_response_validation_count"] = len(response_validations)
|
|
report["database_response_validation_all_ok"] = bool(response_validations) and all(
|
|
item.get("status") == "ok" and item.get("validated") is True
|
|
for item in response_validations
|
|
)
|
|
report["database_response_transform_count"] = sum(
|
|
item.get("transformed") is True for item in response_validations
|
|
)
|
|
tool_traces = [
|
|
result.get("database_tool_trace") or {}
|
|
for result in report.get("results") or []
|
|
]
|
|
report["database_tool_call_proven_count"] = sum(
|
|
trace.get("database_tool_call_proven") is True for trace in tool_traces
|
|
)
|
|
report["database_tool_call_proven"] = any(
|
|
trace.get("database_tool_call_proven") is True for trace in tool_traces
|
|
)
|
|
report["database_retrieval_receipt_proven"] = any(
|
|
trace.get("database_retrieval_receipt_proven") is True for trace in tool_traces
|
|
)
|
|
tool_traces_with_calls = [
|
|
trace for trace in tool_traces if trace.get("database_tool_call_count")
|
|
]
|
|
report["database_tool_calls_read_only"] = bool(tool_traces_with_calls) and all(
|
|
trace.get("database_tool_calls_read_only") is True
|
|
for trace in tool_traces_with_calls
|
|
)
|
|
write_report_snapshot(report)
|
|
|
|
|
|
print(json.dumps(asyncio.run(run_suite()), sort_keys=True))
|
|
'''
|
|
|
|
|
|
def direct_claim_prompts() -> list[dict[str, Any]]:
|
|
return [
|
|
{
|
|
"id": prompt["id"],
|
|
"dimension": prompt["dimension"],
|
|
"message": prompt["message"],
|
|
}
|
|
for prompt in benchmark.M3TAVERSAL_DIRECT_CLAIM_FOLLOWUP_SCENARIOS
|
|
]
|
|
|
|
|
|
def build_remote_script(
|
|
run_id: str,
|
|
*,
|
|
prompts: list[dict[str, Any]] | None = None,
|
|
suite_mode: str = DEFAULT_SUITE_MODE,
|
|
report_prefix: str = DEFAULT_REPORT_PREFIX,
|
|
prompt_note: str = DEFAULT_PROMPT_NOTE,
|
|
) -> str:
|
|
if not re.fullmatch(r"[A-Za-z0-9._-]+", report_prefix):
|
|
raise ValueError("report_prefix must contain only letters, numbers, dot, underscore, or hyphen")
|
|
selected_prompts = direct_claim_prompts() if prompts is None else prompts
|
|
return (
|
|
REMOTE_SCRIPT.replace("__CHAT_ID__", CHAT_ID)
|
|
.replace("__USER_ID__", USER_ID)
|
|
.replace("__USER_NAME__", USER_NAME)
|
|
.replace("__RUN_ID__", run_id)
|
|
.replace("__PROMPTS_JSON__", json.dumps(selected_prompts))
|
|
.replace("__SUITE_MODE_JSON__", json.dumps(suite_mode))
|
|
.replace("__REPORT_PREFIX_JSON__", json.dumps(report_prefix))
|
|
.replace("__PROMPT_NOTE_JSON__", json.dumps(prompt_note))
|
|
.replace("__DB_CONTEXT_PLUGIN_SOURCE_JSON__", json.dumps(DB_CONTEXT_PLUGIN.read_text(encoding="utf-8")))
|
|
.replace("__KB_TOOL_SOURCE_JSON__", json.dumps(KB_TOOL.read_text(encoding="utf-8")))
|
|
.replace("__BEHAVIOR_MANIFEST_SOURCE_JSON__", json.dumps(BEHAVIOR_MANIFEST.read_text(encoding="utf-8")))
|
|
)
|
|
|
|
|
|
def redact(text: str) -> str:
|
|
result = JSON_SECRET_PATTERN.sub(lambda match: f'{match.group(1)}"[REDACTED]"', text)
|
|
result = AUTHORIZATION_PATTERN.sub(lambda match: f"{match.group(1)}[REDACTED]", result)
|
|
result = ASSIGNMENT_SECRET_PATTERN.sub(lambda match: f"{match.group(1)}[REDACTED]", result)
|
|
result = TELEGRAM_TOKEN_PATTERN.sub("[REDACTED_TOKEN]", result)
|
|
return result
|
|
|
|
|
|
def fetch_remote_report(
|
|
run_id: str,
|
|
*,
|
|
report_prefix: str = DEFAULT_REPORT_PREFIX,
|
|
unlink: bool = True,
|
|
) -> dict[str, Any] | None:
|
|
report_path = f"/tmp/{report_prefix}-{run_id}.json"
|
|
unlink_line = " p.unlink()\n" if unlink else ""
|
|
proc = subprocess.run(
|
|
[
|
|
"ssh",
|
|
"-i",
|
|
str(SSH_KEY),
|
|
"-o",
|
|
"BatchMode=yes",
|
|
"-o",
|
|
"StrictHostKeyChecking=accept-new",
|
|
VPS,
|
|
"sudo -u teleo -H /home/teleo/.hermes/hermes-agent/venv/bin/python -",
|
|
],
|
|
input=(
|
|
"import json\n"
|
|
"from pathlib import Path\n"
|
|
f"p = Path({report_path!r})\n"
|
|
"if p.exists():\n"
|
|
" print(p.read_text(encoding='utf-8'))\n"
|
|
f"{unlink_line}"
|
|
),
|
|
text=True,
|
|
capture_output=True,
|
|
timeout=60,
|
|
)
|
|
if proc.returncode != 0 or not proc.stdout.strip():
|
|
return None
|
|
return json.loads(redact(proc.stdout.strip()))
|
|
|
|
|
|
def run_remote(
|
|
*,
|
|
prompts: list[dict[str, Any]] | None = None,
|
|
suite_mode: str = DEFAULT_SUITE_MODE,
|
|
report_prefix: str = DEFAULT_REPORT_PREFIX,
|
|
prompt_note: str = DEFAULT_PROMPT_NOTE,
|
|
) -> dict[str, Any]:
|
|
run_id = uuid.uuid4().hex[:12]
|
|
proc = subprocess.run(
|
|
[
|
|
"ssh",
|
|
"-i",
|
|
str(SSH_KEY),
|
|
"-o",
|
|
"BatchMode=yes",
|
|
"-o",
|
|
"StrictHostKeyChecking=accept-new",
|
|
VPS,
|
|
"cd /home/teleo && sudo -u teleo -H /home/teleo/.hermes/hermes-agent/venv/bin/python -",
|
|
],
|
|
input=build_remote_script(
|
|
run_id,
|
|
prompts=prompts,
|
|
suite_mode=suite_mode,
|
|
report_prefix=report_prefix,
|
|
prompt_note=prompt_note,
|
|
),
|
|
text=True,
|
|
capture_output=True,
|
|
timeout=2100,
|
|
)
|
|
result: dict[str, Any] = {
|
|
"returncode": proc.returncode,
|
|
"stdout": redact(proc.stdout),
|
|
"stderr": redact(proc.stderr),
|
|
"run_id": run_id,
|
|
}
|
|
if proc.returncode == 0 and proc.stdout.strip():
|
|
for line in reversed(proc.stdout.splitlines()):
|
|
candidate = line.strip()
|
|
if not candidate.startswith("{"):
|
|
continue
|
|
try:
|
|
result["parsed"] = json.loads(redact(candidate))
|
|
break
|
|
except json.JSONDecodeError:
|
|
continue
|
|
if "parsed" not in result:
|
|
result["parse_error"] = "remote stdout contained no parseable JSON object line"
|
|
if "parsed" not in result:
|
|
fetched = fetch_remote_report(run_id, report_prefix=report_prefix)
|
|
if fetched is not None:
|
|
result["parsed"] = fetched
|
|
result["parsed_from_report_file"] = True
|
|
else:
|
|
fetch_remote_report(run_id, report_prefix=report_prefix)
|
|
return result
|
|
|
|
|
|
def _tool_trace_is_safe(trace: Any) -> bool:
|
|
if not isinstance(trace, dict):
|
|
return False
|
|
count = trace.get("database_tool_call_count")
|
|
completed = trace.get("database_tool_completed_count")
|
|
calls = trace.get("calls") if isinstance(trace.get("calls"), list) else []
|
|
if not isinstance(count, int) or isinstance(count, bool) or count < 0:
|
|
return False
|
|
if not isinstance(completed, int) or isinstance(completed, bool) or completed != count:
|
|
return False
|
|
if len(calls) != count:
|
|
return False
|
|
if count and trace.get("database_tool_calls_read_only") is not True:
|
|
return False
|
|
for call in calls:
|
|
if not isinstance(call, dict):
|
|
return False
|
|
invocations = call.get("database_invocations")
|
|
result = call.get("result")
|
|
if not isinstance(invocations, list) or not invocations:
|
|
return False
|
|
if not all(isinstance(item, dict) and item.get("access_mode") == "read_only" for item in invocations):
|
|
return False
|
|
if (
|
|
not isinstance(result, dict)
|
|
or result.get("error_detected") is not False
|
|
or result.get("nonempty") is not True
|
|
or not re.fullmatch(r"[0-9a-f]{64}", str(result.get("content_sha256") or ""))
|
|
):
|
|
return False
|
|
return True
|
|
|
|
|
|
def build_safety_gate(report: dict[str, Any]) -> dict[str, Any]:
|
|
results = [item for item in report.get("results") or [] if isinstance(item, dict)]
|
|
handler = report.get("handler") if isinstance(report.get("handler"), dict) else {}
|
|
service = report.get("service_before_after") if isinstance(report.get("service_before_after"), dict) else {}
|
|
execution_summary = (
|
|
report.get("execution_manifest_summary")
|
|
if isinstance(report.get("execution_manifest_summary"), dict)
|
|
else {}
|
|
)
|
|
checks = {
|
|
"remote_command_succeeded": report.get("remote_returncode") == 0,
|
|
"runtime_completed": report.get("pass_runtime") is True,
|
|
"handler_authorized": handler.get("authorized") is True,
|
|
"results_nonempty": bool(results),
|
|
"all_turns_returned_replies": bool(results) and all(item.get("ok") is True for item in results),
|
|
"all_turns_declared_no_mutation": bool(results)
|
|
and all(item.get("mutates_kb") is False for item in results),
|
|
"all_tool_traces_read_only_and_bound": bool(results)
|
|
and all(_tool_trace_is_safe(item.get("database_tool_trace")) for item in results),
|
|
"no_telegram_post": report.get("posted_to_telegram") is False,
|
|
"harness_declared_no_kb_mutation": report.get("mutates_kb_by_harness") is False,
|
|
"database_counts_unchanged": report.get("db_counts_changed") is False,
|
|
"database_fingerprint_unchanged": report.get("db_fingerprint_unchanged") is True,
|
|
"live_behavior_unchanged": report.get("live_behavior_manifest_unchanged") is True,
|
|
"service_unchanged": service.get("unchanged_from_preexisting_live_readback") is True,
|
|
"temporary_profile_removed": report.get("temp_profile_removed") is True,
|
|
"model_trace_metadata_bound": report.get("model_call_trace_all_bound") is True,
|
|
"all_turn_manifests_complete": execution_summary.get("all_turns_attribution_complete") is True,
|
|
"no_runtime_error": not report.get("error") and not report.get("failure"),
|
|
}
|
|
failed = [name for name, passed in checks.items() if not passed]
|
|
return {
|
|
"status": "pass" if not failed else "fail",
|
|
"checks": checks,
|
|
"failed_checks": failed,
|
|
}
|
|
|
|
|
|
def report_passes_safety(report: dict[str, Any]) -> bool:
|
|
return build_safety_gate(report)["status"] == "pass"
|
|
|
|
|
|
def write_output(remote: dict[str, Any], *, output_json: Path = OUTPUT_JSON) -> dict[str, Any]:
|
|
output_json.parent.mkdir(parents=True, exist_ok=True)
|
|
parsed = remote.get("parsed") or {}
|
|
report = parsed if isinstance(parsed, dict) else {}
|
|
report["source_report_path"] = str(output_json)
|
|
report["remote_returncode"] = remote.get("returncode")
|
|
report["remote_run_id"] = remote.get("run_id")
|
|
if remote.get("stderr"):
|
|
report["remote_stderr"] = remote["stderr"]
|
|
attach_execution_manifests(report, repo_root=ROOT)
|
|
report["safety_gate"] = build_safety_gate(report)
|
|
output_json.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
|
return report
|
|
|
|
|
|
def main() -> int:
|
|
remote = run_remote()
|
|
report = write_output(remote)
|
|
print(
|
|
json.dumps(
|
|
{
|
|
"json": str(OUTPUT_JSON),
|
|
"pass_runtime": report.get("pass_runtime"),
|
|
"safety_gate": report.get("safety_gate"),
|
|
},
|
|
indent=2,
|
|
)
|
|
)
|
|
return 0 if report_passes_safety(report) else 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|