471 lines
19 KiB
Python
471 lines
19 KiB
Python
#!/usr/bin/env python3
|
|
"""Collect read-only preflight for the Telegram-visible DC benchmark.
|
|
|
|
This is a pre-send proof collector. It verifies the exact authorization packet,
|
|
then reads the VPS deploy stamp, Leo gateway service state, and current KB row
|
|
counts. It never sends Telegram messages and never mutates the database.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import base64
|
|
import hashlib
|
|
import json
|
|
import shlex
|
|
import subprocess
|
|
from dataclasses import dataclass
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from typing import Any, Callable
|
|
|
|
REPORT_DIR = Path("docs/reports/leo-working-state-20260709")
|
|
DEFAULT_PACKET = REPORT_DIR / "telegram-visible-direct-claim-authorization-packet-current.json"
|
|
DEFAULT_OUT = REPORT_DIR / "telegram-visible-direct-claim-preflight-current.json"
|
|
DEFAULT_MARKDOWN_OUT = REPORT_DIR / "telegram-visible-direct-claim-preflight-current.md"
|
|
DEFAULT_REMOTE_COMMAND_OUT = REPORT_DIR / "telegram-visible-direct-claim-remote-readback-command-current.sh"
|
|
DEFAULT_SSH_TARGET = "root@77.42.65.182"
|
|
DEFAULT_SSH_KEY = Path.home() / ".ssh/livingip_hetzner_20260604_ed25519"
|
|
REMOTE_REPO = "/opt/teleo-eval/workspaces/deploy-infra"
|
|
REMOTE_PACKET = (
|
|
f"{REMOTE_REPO}/docs/reports/leo-working-state-20260709/"
|
|
"telegram-visible-direct-claim-authorization-packet-current.json"
|
|
)
|
|
EXPECTED_PROMPT_IDS = ["DC-01", "DC-02", "DC-03", "DC-04", "DC-05", "DC-06"]
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class CommandResult:
|
|
returncode: int
|
|
stdout: str
|
|
stderr: str
|
|
|
|
|
|
Runner = Callable[[list[str], str, int], CommandResult]
|
|
|
|
|
|
def subprocess_runner(command: list[str], input_text: str, timeout: int) -> CommandResult:
|
|
proc = subprocess.run(command, input=input_text, text=True, capture_output=True, timeout=timeout, check=False)
|
|
return CommandResult(proc.returncode, proc.stdout, proc.stderr)
|
|
|
|
|
|
def _load_json(path: Path) -> dict[str, Any]:
|
|
return json.loads(path.read_text(encoding="utf-8"))
|
|
|
|
|
|
def packet_checks(packet: dict[str, Any]) -> dict[str, bool]:
|
|
target = packet.get("target", {})
|
|
return {
|
|
"packet_ready_to_request_authorization": packet.get("ready_to_request_authorization") is True,
|
|
"packet_not_sent": packet.get("telegram_visible_messages_sent") is False,
|
|
"packet_has_no_authorization_recorded": packet.get("telegram_visible_message_authorization_present") is False,
|
|
"packet_does_not_mutate_db": packet.get("mutates_db") is False,
|
|
"packet_does_not_execute_production_apply": packet.get("production_apply_executed") is False,
|
|
"packet_targets_leo_group": target.get("chat_label") == "Leo" and target.get("chat_id") == "-5146042086",
|
|
"packet_prompt_ids_exact": target.get("expected_prompt_ids") == EXPECTED_PROMPT_IDS,
|
|
"packet_message_count_exact": target.get("message_count") == len(EXPECTED_PROMPT_IDS),
|
|
}
|
|
|
|
|
|
def ssh_command(args: argparse.Namespace, remote_command: str) -> list[str]:
|
|
return [
|
|
"ssh",
|
|
"-i",
|
|
str(args.ssh_key),
|
|
"-o",
|
|
"BatchMode=yes",
|
|
"-o",
|
|
"StrictHostKeyChecking=accept-new",
|
|
"-o",
|
|
f"ConnectTimeout={args.connect_timeout}",
|
|
args.ssh_target,
|
|
remote_command,
|
|
]
|
|
|
|
|
|
def db_counts_sql() -> str:
|
|
return """
|
|
select jsonb_build_object(
|
|
'public.claims', (select count(*) from public.claims),
|
|
'public.sources', (select count(*) from public.sources),
|
|
'public.claim_evidence', (select count(*) from public.claim_evidence),
|
|
'public.claim_edges', (select count(*) from public.claim_edges),
|
|
'kb_stage.kb_proposals', (select count(*) from kb_stage.kb_proposals)
|
|
)::text;
|
|
""".strip()
|
|
|
|
|
|
def deploy_readback_command() -> str:
|
|
repo = shlex.quote(REMOTE_REPO)
|
|
packet = shlex.quote(REMOTE_PACKET)
|
|
return (
|
|
f"cd {repo} && "
|
|
"printf 'deploy_head=' && git rev-parse HEAD && "
|
|
"printf 'last_deploy_sha=' && cat /opt/teleo-eval/.last-deploy-sha && printf '\\n' && "
|
|
f"if [ -s {packet} ]; then "
|
|
"printf 'packet_present=true\\npacket_bytes='; "
|
|
f"wc -c < {packet}; "
|
|
"else printf 'packet_present=false\\npacket_bytes=0\\n'; fi"
|
|
)
|
|
|
|
|
|
def service_readback_command() -> str:
|
|
return (
|
|
"systemctl show leoclean-gateway.service "
|
|
"-p ActiveState -p SubState -p MainPID -p NRestarts -p ExecMainStartTimestamp --no-pager"
|
|
)
|
|
|
|
|
|
def db_counts_readback_command() -> str:
|
|
return "docker exec teleo-pg psql -U postgres -d teleo -At -q -c " + shlex.quote(db_counts_sql())
|
|
|
|
|
|
def parse_key_value_lines(text: str) -> dict[str, str]:
|
|
values = {}
|
|
for line in text.splitlines():
|
|
if "=" not in line:
|
|
continue
|
|
key, value = line.split("=", 1)
|
|
values[key] = value.strip()
|
|
return values
|
|
|
|
|
|
def collect_remote_direct(args: argparse.Namespace, runner: Runner) -> tuple[dict[str, Any] | None, int, str]:
|
|
commands = {
|
|
"deploy": deploy_readback_command(),
|
|
"service": service_readback_command(),
|
|
"db_counts": db_counts_readback_command(),
|
|
}
|
|
results = {
|
|
name: runner(ssh_command(args, command), "", args.timeout)
|
|
for name, command in commands.items()
|
|
}
|
|
stderr = "\n".join(
|
|
f"{name}: {result.stderr.strip()}"
|
|
for name, result in results.items()
|
|
if result.stderr.strip()
|
|
)
|
|
returncode = next((result.returncode for result in results.values() if result.returncode != 0), 0)
|
|
deploy_values = parse_key_value_lines(results["deploy"].stdout)
|
|
service_values = parse_key_value_lines(results["service"].stdout)
|
|
counts = None
|
|
if results["db_counts"].returncode == 0 and results["db_counts"].stdout.strip():
|
|
try:
|
|
counts = json.loads(results["db_counts"].stdout.strip())
|
|
except json.JSONDecodeError:
|
|
counts = None
|
|
try:
|
|
packet_bytes = int(deploy_values.get("packet_bytes", "0") or "0")
|
|
except ValueError:
|
|
packet_bytes = 0
|
|
payload = {
|
|
"deploy_head": {
|
|
"returncode": results["deploy"].returncode,
|
|
"stdout": deploy_values.get("deploy_head", ""),
|
|
"stderr": results["deploy"].stderr.strip(),
|
|
},
|
|
"last_deploy_sha": {
|
|
"returncode": results["deploy"].returncode,
|
|
"stdout": deploy_values.get("last_deploy_sha", ""),
|
|
"stderr": results["deploy"].stderr.strip(),
|
|
},
|
|
"packet_present": deploy_values.get("packet_present") == "true",
|
|
"packet_bytes": packet_bytes,
|
|
"service": {
|
|
"returncode": results["service"].returncode,
|
|
"stderr": results["service"].stderr,
|
|
"state": service_values,
|
|
},
|
|
"db_counts": {
|
|
"returncode": results["db_counts"].returncode,
|
|
"stderr": results["db_counts"].stderr,
|
|
"counts": counts,
|
|
},
|
|
}
|
|
return payload, returncode, stderr
|
|
|
|
|
|
def remote_readback_command_script(args: argparse.Namespace) -> str:
|
|
ssh_prefix = " ".join(
|
|
shlex.quote(part)
|
|
for part in [
|
|
"ssh",
|
|
"-i",
|
|
str(args.ssh_key),
|
|
"-o",
|
|
"BatchMode=yes",
|
|
"-o",
|
|
"StrictHostKeyChecking=accept-new",
|
|
"-o",
|
|
f"ConnectTimeout={int(args.connect_timeout)}",
|
|
args.ssh_target,
|
|
]
|
|
)
|
|
lines = [
|
|
"#!/usr/bin/env bash",
|
|
"set -euo pipefail",
|
|
f"# Remote packet: {REMOTE_PACKET}",
|
|
"# Readback covers deploy head, .last-deploy-sha, leoclean-gateway.service, and DB counts:",
|
|
"# public.claims, public.sources, public.claim_evidence, public.claim_edges, kb_stage.kb_proposals",
|
|
"run_ssh() {",
|
|
f" {ssh_prefix} \"$1\"",
|
|
"}",
|
|
f"DEPLOY_READBACK=\"$(run_ssh {shlex.quote(deploy_readback_command())})\"",
|
|
f"SERVICE_READBACK=\"$(run_ssh {shlex.quote(service_readback_command())})\"",
|
|
f"DB_COUNTS_READBACK=\"$(run_ssh {shlex.quote(db_counts_readback_command())})\"",
|
|
"python3 - \"$DEPLOY_READBACK\" \"$SERVICE_READBACK\" \"$DB_COUNTS_READBACK\" <<'PY' | base64 | tr -d '\\n'",
|
|
"import json",
|
|
"import sys",
|
|
"",
|
|
"def kv(text):",
|
|
" out = {}",
|
|
" for line in text.splitlines():",
|
|
" if '=' in line:",
|
|
" key, value = line.split('=', 1)",
|
|
" out[key] = value.strip()",
|
|
" return out",
|
|
"",
|
|
"deploy = kv(sys.argv[1])",
|
|
"service = kv(sys.argv[2])",
|
|
"counts = json.loads(sys.argv[3])",
|
|
"payload = {",
|
|
" 'deploy_head': {'returncode': 0, 'stdout': deploy.get('deploy_head', ''), 'stderr': ''},",
|
|
" 'last_deploy_sha': {'returncode': 0, 'stdout': deploy.get('last_deploy_sha', ''), 'stderr': ''},",
|
|
" 'packet_present': deploy.get('packet_present') == 'true',",
|
|
" 'packet_bytes': int(deploy.get('packet_bytes') or 0),",
|
|
" 'service': {'returncode': 0, 'stderr': '', 'state': service},",
|
|
" 'db_counts': {'returncode': 0, 'stderr': '', 'counts': counts},",
|
|
"}",
|
|
"print(json.dumps(payload, sort_keys=True))",
|
|
"PY",
|
|
"printf '\\n'",
|
|
"",
|
|
]
|
|
return "\n".join(lines)
|
|
|
|
|
|
def write_remote_readback_command(path: Path, args: argparse.Namespace) -> str:
|
|
script = remote_readback_command_script(args)
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text(script, encoding="utf-8")
|
|
path.chmod(0o755)
|
|
return hashlib.sha256(script.encode("utf-8")).hexdigest()
|
|
|
|
|
|
def remote_payload_from_args(args: argparse.Namespace) -> tuple[dict[str, Any] | None, int, str]:
|
|
if args.remote_json_base64:
|
|
try:
|
|
decoded = base64.b64decode(args.remote_json_base64, validate=True).decode("utf-8")
|
|
return json.loads(decoded), 0, ""
|
|
except Exception as exc: # noqa: BLE001 - retained in artifact as a parse gate
|
|
return None, 2, f"failed to parse --remote-json-base64: {exc}"
|
|
return None, -1, ""
|
|
|
|
|
|
def collect_preflight(args: argparse.Namespace, *, runner: Runner = subprocess_runner) -> dict[str, Any]:
|
|
packet = _load_json(args.packet)
|
|
local_packet_checks = packet_checks(packet)
|
|
supplied_payload, supplied_returncode, supplied_stderr = remote_payload_from_args(args)
|
|
if supplied_returncode >= 0:
|
|
remote_payload = supplied_payload
|
|
remote_returncode = supplied_returncode
|
|
remote_stderr = supplied_stderr
|
|
remote_source = "supplied_remote_json_base64"
|
|
else:
|
|
remote_payload, remote_returncode, remote_stderr = collect_remote_direct(args, runner)
|
|
remote_source = "ssh_direct_commands"
|
|
|
|
remote_checks = {
|
|
"ssh_readback_ok": remote_returncode == 0 and remote_payload is not None,
|
|
"remote_packet_present": bool((remote_payload or {}).get("packet_present")),
|
|
"remote_packet_nonempty": int((remote_payload or {}).get("packet_bytes") or 0) > 0,
|
|
"deploy_head_matches_last_deploy": False,
|
|
"service_active_running": False,
|
|
"db_counts_read": False,
|
|
}
|
|
if remote_payload:
|
|
deploy_head = remote_payload.get("deploy_head", {}).get("stdout")
|
|
last_deploy = remote_payload.get("last_deploy_sha", {}).get("stdout")
|
|
service_state = remote_payload.get("service", {}).get("state", {})
|
|
counts = remote_payload.get("db_counts", {}).get("counts")
|
|
remote_checks.update(
|
|
{
|
|
"deploy_head_matches_last_deploy": bool(deploy_head and deploy_head == last_deploy),
|
|
"service_active_running": service_state.get("ActiveState") == "active"
|
|
and service_state.get("SubState") == "running",
|
|
"db_counts_read": isinstance(counts, dict)
|
|
and all(key in counts for key in EXPECTED_DB_COUNT_KEYS),
|
|
}
|
|
)
|
|
|
|
status = "pass" if all(local_packet_checks.values()) and all(remote_checks.values()) else "fail"
|
|
blocker_readback = {
|
|
"current_canary": (
|
|
"Pre-send Telegram-visible direct-claim benchmark readiness for DC-01 through DC-06 in the "
|
|
"Leo group, expecting packet verification plus VPS deploy/service/DB count readback before "
|
|
"any authorized send."
|
|
),
|
|
"attempted_routes": [
|
|
"local authorization packet validation",
|
|
f"{remote_source} remote VPS readback for deploy stamp, service state, packet presence, and DB counts",
|
|
],
|
|
"exact_gate": "" if status == "pass" else (remote_stderr.strip() or "remote readback checks did not pass"),
|
|
"clear_CTA": (
|
|
f"Run `{args.remote_command_out}` from a full-access shell, then rerun "
|
|
"`scripts/collect_telegram_visible_direct_claim_preflight.py --remote-json-base64 <payload>` "
|
|
"with the base64 output it prints. "
|
|
"Only after this report passes should the exact DC-01 through DC-06 Telegram group messages be "
|
|
"sent under explicit action-time authorization."
|
|
),
|
|
"next_non_user_action": (
|
|
"Rerun this preflight with supplied VPS remote readback JSON, then run the Telegram-visible "
|
|
"DC scorer and service/DB after-readbacks if the user explicitly authorizes the group send."
|
|
),
|
|
}
|
|
return {
|
|
"generated_at_utc": datetime.now(timezone.utc).isoformat(),
|
|
"mode": "telegram_visible_direct_claim_preflight_readonly",
|
|
"status": status,
|
|
"ready_for_authorized_telegram_visible_dc_run": status == "pass",
|
|
"telegram_visible_messages_sent": False,
|
|
"production_apply_executed": False,
|
|
"mutates_db": False,
|
|
"packet_path": str(args.packet),
|
|
"packet_checks": local_packet_checks,
|
|
"remote_checks": remote_checks,
|
|
"remote_source": remote_source,
|
|
"remote": remote_payload,
|
|
"remote_returncode": remote_returncode,
|
|
"remote_stderr": remote_stderr,
|
|
"remote_readback_command_path": str(args.remote_command_out),
|
|
"remote_readback_command_sha256": hashlib.sha256(
|
|
remote_readback_command_script(args).encode("utf-8")
|
|
).hexdigest(),
|
|
"blocker_readback": blocker_readback,
|
|
"claim_ceiling": (
|
|
"This is a read-only preflight for an explicitly authorized Telegram-visible DC benchmark. It does "
|
|
"not send Telegram messages, does not run production apply, and does not prove Telegram-visible DC "
|
|
"behavior until the six-message run is authorized, sent, captured, and scored."
|
|
),
|
|
}
|
|
|
|
|
|
EXPECTED_DB_COUNT_KEYS = [
|
|
"public.claims",
|
|
"public.sources",
|
|
"public.claim_evidence",
|
|
"public.claim_edges",
|
|
"kb_stage.kb_proposals",
|
|
]
|
|
|
|
|
|
def write_json(path: Path, data: dict[str, Any]) -> None:
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text(json.dumps(data, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
|
|
|
|
|
def write_markdown(path: Path, data: dict[str, Any]) -> None:
|
|
remote = data.get("remote") or {}
|
|
service = remote.get("service", {}).get("state", {})
|
|
counts = remote.get("db_counts", {}).get("counts") or {}
|
|
lines = [
|
|
"# Telegram-Visible Direct-Claim Preflight",
|
|
"",
|
|
f"Generated UTC: `{data['generated_at_utc']}`",
|
|
f"Mode: `{data['mode']}`",
|
|
f"Status: `{data['status']}`",
|
|
f"Ready for authorized Telegram-visible DC run: `{data['ready_for_authorized_telegram_visible_dc_run']}`",
|
|
f"Telegram-visible messages sent: `{data['telegram_visible_messages_sent']}`",
|
|
f"Production apply executed: `{data['production_apply_executed']}`",
|
|
f"Mutates DB: `{data['mutates_db']}`",
|
|
f"Remote source: `{data['remote_source']}`",
|
|
f"Remote readback command: `{data['remote_readback_command_path']}`",
|
|
f"Remote readback command SHA256: `{data['remote_readback_command_sha256']}`",
|
|
"",
|
|
"## Packet Checks",
|
|
"",
|
|
]
|
|
lines.extend(f"- `{key}`: `{value}`" for key, value in sorted(data["packet_checks"].items()))
|
|
lines.extend(["", "## Remote Checks", ""])
|
|
lines.extend(f"- `{key}`: `{value}`" for key, value in sorted(data["remote_checks"].items()))
|
|
lines.extend(
|
|
[
|
|
"",
|
|
"## Deploy And Service",
|
|
"",
|
|
f"- Deploy head: `{remote.get('deploy_head', {}).get('stdout', '')}`",
|
|
f"- Last deploy SHA: `{remote.get('last_deploy_sha', {}).get('stdout', '')}`",
|
|
f"- ActiveState: `{service.get('ActiveState', '')}`",
|
|
f"- SubState: `{service.get('SubState', '')}`",
|
|
f"- MainPID: `{service.get('MainPID', '')}`",
|
|
f"- NRestarts: `{service.get('NRestarts', '')}`",
|
|
f"- ExecMainStartTimestamp: `{service.get('ExecMainStartTimestamp', '')}`",
|
|
"",
|
|
"## DB Counts Before Send",
|
|
"",
|
|
]
|
|
)
|
|
lines.extend(f"- `{key}`: `{counts.get(key, '')}`" for key in EXPECTED_DB_COUNT_KEYS)
|
|
if data["status"] != "pass":
|
|
blocker = data.get("blocker_readback", {})
|
|
lines.extend(
|
|
[
|
|
"",
|
|
"## Blocker Readback",
|
|
"",
|
|
f"- Current canary: {blocker.get('current_canary', '')}",
|
|
f"- Exact gate: `{blocker.get('exact_gate', '')}`",
|
|
f"- Clear CTA: {blocker.get('clear_CTA', '')}",
|
|
f"- Next non-user action: {blocker.get('next_non_user_action', '')}",
|
|
]
|
|
)
|
|
lines.extend(["", "## Claim Ceiling", "", data["claim_ceiling"], ""])
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text("\n".join(lines), encoding="utf-8")
|
|
|
|
|
|
def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--packet", type=Path, default=DEFAULT_PACKET)
|
|
parser.add_argument("--ssh-target", default=DEFAULT_SSH_TARGET)
|
|
parser.add_argument("--ssh-key", type=Path, default=DEFAULT_SSH_KEY)
|
|
parser.add_argument("--connect-timeout", type=int, default=10)
|
|
parser.add_argument("--timeout", type=int, default=60)
|
|
parser.add_argument(
|
|
"--remote-json-base64",
|
|
help="Base64-encoded remote readback JSON from an approved top-level SSH command; skips subprocess SSH.",
|
|
)
|
|
parser.add_argument("--out", type=Path, default=DEFAULT_OUT)
|
|
parser.add_argument("--markdown-out", type=Path, default=DEFAULT_MARKDOWN_OUT)
|
|
parser.add_argument("--remote-command-out", type=Path, default=DEFAULT_REMOTE_COMMAND_OUT)
|
|
return parser.parse_args(argv)
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
args = parse_args(argv)
|
|
remote_command_sha256 = write_remote_readback_command(args.remote_command_out, args)
|
|
data = collect_preflight(args)
|
|
if data["remote_readback_command_sha256"] != remote_command_sha256:
|
|
raise RuntimeError("remote readback command hash drifted during collection")
|
|
write_json(args.out, data)
|
|
write_markdown(args.markdown_out, data)
|
|
print(
|
|
json.dumps(
|
|
{
|
|
"out": str(args.out),
|
|
"markdown_out": str(args.markdown_out),
|
|
"remote_command_out": str(args.remote_command_out),
|
|
"status": data["status"],
|
|
"ready_for_authorized_telegram_visible_dc_run": data[
|
|
"ready_for_authorized_telegram_visible_dc_run"
|
|
],
|
|
},
|
|
indent=2,
|
|
sort_keys=True,
|
|
)
|
|
)
|
|
return 0 if data["status"] == "pass" else 2
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|