430 lines
17 KiB
Python
430 lines
17 KiB
Python
#!/usr/bin/env python3
|
|
"""Collect read-only preflight for the Telegram-visible DC benchmark.
|
|
|
|
This is a pre-send proof collector. It verifies the exact authorization packet,
|
|
then reads the VPS deploy stamp, Leo gateway service state, and current KB row
|
|
counts. It never sends Telegram messages and never mutates the database.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import base64
|
|
import hashlib
|
|
import json
|
|
import shlex
|
|
import subprocess
|
|
from dataclasses import dataclass
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from typing import Any, Callable
|
|
|
|
REPORT_DIR = Path("docs/reports/leo-working-state-20260709")
|
|
DEFAULT_PACKET = REPORT_DIR / "telegram-visible-direct-claim-authorization-packet-current.json"
|
|
DEFAULT_OUT = REPORT_DIR / "telegram-visible-direct-claim-preflight-current.json"
|
|
DEFAULT_MARKDOWN_OUT = REPORT_DIR / "telegram-visible-direct-claim-preflight-current.md"
|
|
DEFAULT_REMOTE_COMMAND_OUT = REPORT_DIR / "telegram-visible-direct-claim-remote-readback-command-current.sh"
|
|
DEFAULT_SSH_TARGET = "root@77.42.65.182"
|
|
DEFAULT_SSH_KEY = Path.home() / ".ssh/livingip_hetzner_20260604_ed25519"
|
|
REMOTE_REPO = "/opt/teleo-eval/workspaces/deploy-infra"
|
|
REMOTE_PACKET = (
|
|
f"{REMOTE_REPO}/docs/reports/leo-working-state-20260709/"
|
|
"telegram-visible-direct-claim-authorization-packet-current.json"
|
|
)
|
|
EXPECTED_PROMPT_IDS = ["DC-01", "DC-02", "DC-03", "DC-04", "DC-05", "DC-06"]
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class CommandResult:
|
|
returncode: int
|
|
stdout: str
|
|
stderr: str
|
|
|
|
|
|
Runner = Callable[[list[str], str, int], CommandResult]
|
|
|
|
|
|
REMOTE_SCRIPT = r'''
|
|
import json
|
|
import subprocess
|
|
from pathlib import Path
|
|
|
|
repo = Path("__REMOTE_REPO__")
|
|
packet_path = Path("__REMOTE_PACKET__")
|
|
|
|
|
|
def run(command, **kwargs):
|
|
return subprocess.run(command, text=True, capture_output=True, check=False, **kwargs)
|
|
|
|
|
|
def service_state():
|
|
proc = run([
|
|
"systemctl",
|
|
"show",
|
|
"leoclean-gateway.service",
|
|
"-p",
|
|
"ActiveState",
|
|
"-p",
|
|
"SubState",
|
|
"-p",
|
|
"MainPID",
|
|
"-p",
|
|
"NRestarts",
|
|
"-p",
|
|
"ExecMainStartTimestamp",
|
|
"--no-pager",
|
|
])
|
|
state = {}
|
|
for line in proc.stdout.splitlines():
|
|
if "=" in line:
|
|
key, value = line.split("=", 1)
|
|
state[key] = value
|
|
return {"returncode": proc.returncode, "stderr": proc.stderr, "state": state}
|
|
|
|
|
|
def db_counts():
|
|
sql = """
|
|
select jsonb_build_object(
|
|
'public.claims', (select count(*) from public.claims),
|
|
'public.sources', (select count(*) from public.sources),
|
|
'public.claim_evidence', (select count(*) from public.claim_evidence),
|
|
'public.claim_edges', (select count(*) from public.claim_edges),
|
|
'kb_stage.kb_proposals', (select count(*) from kb_stage.kb_proposals)
|
|
)::text;
|
|
"""
|
|
proc = run(
|
|
["docker", "exec", "-i", "teleo-pg", "psql", "-U", "postgres", "-d", "teleo", "-At", "-q"],
|
|
input=sql,
|
|
)
|
|
parsed = None
|
|
if proc.returncode == 0 and proc.stdout.strip():
|
|
parsed = json.loads(proc.stdout.strip())
|
|
return {"returncode": proc.returncode, "stderr": proc.stderr, "counts": parsed}
|
|
|
|
|
|
def git_value(args):
|
|
proc = run(["git", "-C", str(repo), *args])
|
|
return {"returncode": proc.returncode, "stdout": proc.stdout.strip(), "stderr": proc.stderr.strip()}
|
|
|
|
|
|
payload = {
|
|
"deploy_head": git_value(["rev-parse", "HEAD"]),
|
|
"last_deploy_sha": {
|
|
"returncode": 0,
|
|
"stdout": Path("/opt/teleo-eval/.last-deploy-sha").read_text(encoding="utf-8").strip(),
|
|
"stderr": "",
|
|
},
|
|
"packet_present": packet_path.exists(),
|
|
"packet_bytes": packet_path.stat().st_size if packet_path.exists() else 0,
|
|
"service": service_state(),
|
|
"db_counts": db_counts(),
|
|
}
|
|
print(json.dumps(payload, sort_keys=True))
|
|
'''
|
|
|
|
|
|
def subprocess_runner(command: list[str], input_text: str, timeout: int) -> CommandResult:
|
|
proc = subprocess.run(command, input=input_text, text=True, capture_output=True, timeout=timeout, check=False)
|
|
return CommandResult(proc.returncode, proc.stdout, proc.stderr)
|
|
|
|
|
|
def _load_json(path: Path) -> dict[str, Any]:
|
|
return json.loads(path.read_text(encoding="utf-8"))
|
|
|
|
|
|
def packet_checks(packet: dict[str, Any]) -> dict[str, bool]:
|
|
target = packet.get("target", {})
|
|
return {
|
|
"packet_ready_to_request_authorization": packet.get("ready_to_request_authorization") is True,
|
|
"packet_not_sent": packet.get("telegram_visible_messages_sent") is False,
|
|
"packet_has_no_authorization_recorded": packet.get("telegram_visible_message_authorization_present") is False,
|
|
"packet_does_not_mutate_db": packet.get("mutates_db") is False,
|
|
"packet_does_not_execute_production_apply": packet.get("production_apply_executed") is False,
|
|
"packet_targets_leo_group": target.get("chat_label") == "Leo" and target.get("chat_id") == "-5146042086",
|
|
"packet_prompt_ids_exact": target.get("expected_prompt_ids") == EXPECTED_PROMPT_IDS,
|
|
"packet_message_count_exact": target.get("message_count") == len(EXPECTED_PROMPT_IDS),
|
|
}
|
|
|
|
|
|
def ssh_command(args: argparse.Namespace) -> list[str]:
|
|
return [
|
|
"ssh",
|
|
"-i",
|
|
str(args.ssh_key),
|
|
"-o",
|
|
"BatchMode=yes",
|
|
"-o",
|
|
"StrictHostKeyChecking=accept-new",
|
|
"-o",
|
|
f"ConnectTimeout={args.connect_timeout}",
|
|
args.ssh_target,
|
|
"python3 -",
|
|
]
|
|
|
|
|
|
def remote_script_text() -> str:
|
|
return REMOTE_SCRIPT.replace("__REMOTE_REPO__", REMOTE_REPO).replace("__REMOTE_PACKET__", REMOTE_PACKET)
|
|
|
|
|
|
def remote_readback_command_script(args: argparse.Namespace) -> str:
|
|
lines = [
|
|
"#!/usr/bin/env bash",
|
|
"set -euo pipefail",
|
|
(
|
|
"ssh "
|
|
f"-i {shlex.quote(str(args.ssh_key))} "
|
|
"-o BatchMode=yes "
|
|
"-o StrictHostKeyChecking=accept-new "
|
|
f"-o ConnectTimeout={int(args.connect_timeout)} "
|
|
f"{shlex.quote(args.ssh_target)} "
|
|
'"python3 -" <<\'PY\' | base64 | tr -d \'\\n\''
|
|
),
|
|
remote_script_text().rstrip(),
|
|
"PY",
|
|
"printf '\\n'",
|
|
"",
|
|
]
|
|
return "\n".join(lines)
|
|
|
|
|
|
def write_remote_readback_command(path: Path, args: argparse.Namespace) -> str:
|
|
script = remote_readback_command_script(args)
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text(script, encoding="utf-8")
|
|
path.chmod(0o755)
|
|
return hashlib.sha256(script.encode("utf-8")).hexdigest()
|
|
|
|
|
|
def parse_remote(stdout: str) -> dict[str, Any] | None:
|
|
for line in reversed(stdout.splitlines()):
|
|
line = line.strip()
|
|
if not line.startswith("{"):
|
|
continue
|
|
try:
|
|
return json.loads(line)
|
|
except json.JSONDecodeError:
|
|
continue
|
|
return None
|
|
|
|
|
|
def remote_payload_from_args(args: argparse.Namespace) -> tuple[dict[str, Any] | None, int, str]:
|
|
if args.remote_json_base64:
|
|
try:
|
|
decoded = base64.b64decode(args.remote_json_base64, validate=True).decode("utf-8")
|
|
return json.loads(decoded), 0, ""
|
|
except Exception as exc: # noqa: BLE001 - retained in artifact as a parse gate
|
|
return None, 2, f"failed to parse --remote-json-base64: {exc}"
|
|
return None, -1, ""
|
|
|
|
|
|
def collect_preflight(args: argparse.Namespace, *, runner: Runner = subprocess_runner) -> dict[str, Any]:
|
|
packet = _load_json(args.packet)
|
|
local_packet_checks = packet_checks(packet)
|
|
supplied_payload, supplied_returncode, supplied_stderr = remote_payload_from_args(args)
|
|
if supplied_returncode >= 0:
|
|
remote_payload = supplied_payload
|
|
remote_returncode = supplied_returncode
|
|
remote_stderr = supplied_stderr
|
|
remote_source = "supplied_remote_json_base64"
|
|
else:
|
|
command = ssh_command(args)
|
|
remote_proc = runner(command, remote_script_text(), args.timeout)
|
|
remote_payload = parse_remote(remote_proc.stdout) if remote_proc.returncode == 0 else None
|
|
remote_returncode = remote_proc.returncode
|
|
remote_stderr = remote_proc.stderr
|
|
remote_source = "ssh_subprocess"
|
|
|
|
remote_checks = {
|
|
"ssh_readback_ok": remote_returncode == 0 and remote_payload is not None,
|
|
"remote_packet_present": bool((remote_payload or {}).get("packet_present")),
|
|
"remote_packet_nonempty": int((remote_payload or {}).get("packet_bytes") or 0) > 0,
|
|
"deploy_head_matches_last_deploy": False,
|
|
"service_active_running": False,
|
|
"db_counts_read": False,
|
|
}
|
|
if remote_payload:
|
|
deploy_head = remote_payload.get("deploy_head", {}).get("stdout")
|
|
last_deploy = remote_payload.get("last_deploy_sha", {}).get("stdout")
|
|
service_state = remote_payload.get("service", {}).get("state", {})
|
|
counts = remote_payload.get("db_counts", {}).get("counts")
|
|
remote_checks.update(
|
|
{
|
|
"deploy_head_matches_last_deploy": bool(deploy_head and deploy_head == last_deploy),
|
|
"service_active_running": service_state.get("ActiveState") == "active"
|
|
and service_state.get("SubState") == "running",
|
|
"db_counts_read": isinstance(counts, dict)
|
|
and all(key in counts for key in EXPECTED_DB_COUNT_KEYS),
|
|
}
|
|
)
|
|
|
|
status = "pass" if all(local_packet_checks.values()) and all(remote_checks.values()) else "fail"
|
|
blocker_readback = {
|
|
"current_canary": (
|
|
"Pre-send Telegram-visible direct-claim benchmark readiness for DC-01 through DC-06 in the "
|
|
"Leo group, expecting packet verification plus VPS deploy/service/DB count readback before "
|
|
"any authorized send."
|
|
),
|
|
"attempted_routes": [
|
|
"local authorization packet validation",
|
|
f"{remote_source} remote VPS readback for deploy stamp, service state, packet presence, and DB counts",
|
|
],
|
|
"exact_gate": "" if status == "pass" else (remote_stderr.strip() or "remote readback checks did not pass"),
|
|
"clear_CTA": (
|
|
f"Run `{args.remote_command_out}` from a full-access shell, then rerun "
|
|
"`scripts/collect_telegram_visible_direct_claim_preflight.py --remote-json-base64 <payload>` "
|
|
"with the base64 output it prints. "
|
|
"Only after this report passes should the exact DC-01 through DC-06 Telegram group messages be "
|
|
"sent under explicit action-time authorization."
|
|
),
|
|
"next_non_user_action": (
|
|
"Rerun this preflight with supplied VPS remote readback JSON, then run the Telegram-visible "
|
|
"DC scorer and service/DB after-readbacks if the user explicitly authorizes the group send."
|
|
),
|
|
}
|
|
return {
|
|
"generated_at_utc": datetime.now(timezone.utc).isoformat(),
|
|
"mode": "telegram_visible_direct_claim_preflight_readonly",
|
|
"status": status,
|
|
"ready_for_authorized_telegram_visible_dc_run": status == "pass",
|
|
"telegram_visible_messages_sent": False,
|
|
"production_apply_executed": False,
|
|
"mutates_db": False,
|
|
"packet_path": str(args.packet),
|
|
"packet_checks": local_packet_checks,
|
|
"remote_checks": remote_checks,
|
|
"remote_source": remote_source,
|
|
"remote": remote_payload,
|
|
"remote_returncode": remote_returncode,
|
|
"remote_stderr": remote_stderr,
|
|
"remote_readback_command_path": str(args.remote_command_out),
|
|
"remote_readback_command_sha256": hashlib.sha256(
|
|
remote_readback_command_script(args).encode("utf-8")
|
|
).hexdigest(),
|
|
"blocker_readback": blocker_readback,
|
|
"claim_ceiling": (
|
|
"This is a read-only preflight for an explicitly authorized Telegram-visible DC benchmark. It does "
|
|
"not send Telegram messages, does not run production apply, and does not prove Telegram-visible DC "
|
|
"behavior until the six-message run is authorized, sent, captured, and scored."
|
|
),
|
|
}
|
|
|
|
|
|
EXPECTED_DB_COUNT_KEYS = [
|
|
"public.claims",
|
|
"public.sources",
|
|
"public.claim_evidence",
|
|
"public.claim_edges",
|
|
"kb_stage.kb_proposals",
|
|
]
|
|
|
|
|
|
def write_json(path: Path, data: dict[str, Any]) -> None:
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text(json.dumps(data, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
|
|
|
|
|
def write_markdown(path: Path, data: dict[str, Any]) -> None:
|
|
remote = data.get("remote") or {}
|
|
service = remote.get("service", {}).get("state", {})
|
|
counts = remote.get("db_counts", {}).get("counts") or {}
|
|
lines = [
|
|
"# Telegram-Visible Direct-Claim Preflight",
|
|
"",
|
|
f"Generated UTC: `{data['generated_at_utc']}`",
|
|
f"Mode: `{data['mode']}`",
|
|
f"Status: `{data['status']}`",
|
|
f"Ready for authorized Telegram-visible DC run: `{data['ready_for_authorized_telegram_visible_dc_run']}`",
|
|
f"Telegram-visible messages sent: `{data['telegram_visible_messages_sent']}`",
|
|
f"Production apply executed: `{data['production_apply_executed']}`",
|
|
f"Mutates DB: `{data['mutates_db']}`",
|
|
f"Remote source: `{data['remote_source']}`",
|
|
f"Remote readback command: `{data['remote_readback_command_path']}`",
|
|
f"Remote readback command SHA256: `{data['remote_readback_command_sha256']}`",
|
|
"",
|
|
"## Packet Checks",
|
|
"",
|
|
]
|
|
lines.extend(f"- `{key}`: `{value}`" for key, value in sorted(data["packet_checks"].items()))
|
|
lines.extend(["", "## Remote Checks", ""])
|
|
lines.extend(f"- `{key}`: `{value}`" for key, value in sorted(data["remote_checks"].items()))
|
|
lines.extend(
|
|
[
|
|
"",
|
|
"## Deploy And Service",
|
|
"",
|
|
f"- Deploy head: `{remote.get('deploy_head', {}).get('stdout', '')}`",
|
|
f"- Last deploy SHA: `{remote.get('last_deploy_sha', {}).get('stdout', '')}`",
|
|
f"- ActiveState: `{service.get('ActiveState', '')}`",
|
|
f"- SubState: `{service.get('SubState', '')}`",
|
|
f"- MainPID: `{service.get('MainPID', '')}`",
|
|
f"- NRestarts: `{service.get('NRestarts', '')}`",
|
|
f"- ExecMainStartTimestamp: `{service.get('ExecMainStartTimestamp', '')}`",
|
|
"",
|
|
"## DB Counts Before Send",
|
|
"",
|
|
]
|
|
)
|
|
lines.extend(f"- `{key}`: `{counts.get(key, '')}`" for key in EXPECTED_DB_COUNT_KEYS)
|
|
if data["status"] != "pass":
|
|
blocker = data.get("blocker_readback", {})
|
|
lines.extend(
|
|
[
|
|
"",
|
|
"## Blocker Readback",
|
|
"",
|
|
f"- Current canary: {blocker.get('current_canary', '')}",
|
|
f"- Exact gate: `{blocker.get('exact_gate', '')}`",
|
|
f"- Clear CTA: {blocker.get('clear_CTA', '')}",
|
|
f"- Next non-user action: {blocker.get('next_non_user_action', '')}",
|
|
]
|
|
)
|
|
lines.extend(["", "## Claim Ceiling", "", data["claim_ceiling"], ""])
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text("\n".join(lines), encoding="utf-8")
|
|
|
|
|
|
def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--packet", type=Path, default=DEFAULT_PACKET)
|
|
parser.add_argument("--ssh-target", default=DEFAULT_SSH_TARGET)
|
|
parser.add_argument("--ssh-key", type=Path, default=DEFAULT_SSH_KEY)
|
|
parser.add_argument("--connect-timeout", type=int, default=10)
|
|
parser.add_argument("--timeout", type=int, default=60)
|
|
parser.add_argument(
|
|
"--remote-json-base64",
|
|
help="Base64-encoded remote readback JSON from an approved top-level SSH command; skips subprocess SSH.",
|
|
)
|
|
parser.add_argument("--out", type=Path, default=DEFAULT_OUT)
|
|
parser.add_argument("--markdown-out", type=Path, default=DEFAULT_MARKDOWN_OUT)
|
|
parser.add_argument("--remote-command-out", type=Path, default=DEFAULT_REMOTE_COMMAND_OUT)
|
|
return parser.parse_args(argv)
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
args = parse_args(argv)
|
|
remote_command_sha256 = write_remote_readback_command(args.remote_command_out, args)
|
|
data = collect_preflight(args)
|
|
if data["remote_readback_command_sha256"] != remote_command_sha256:
|
|
raise RuntimeError("remote readback command hash drifted during collection")
|
|
write_json(args.out, data)
|
|
write_markdown(args.markdown_out, data)
|
|
print(
|
|
json.dumps(
|
|
{
|
|
"out": str(args.out),
|
|
"markdown_out": str(args.markdown_out),
|
|
"remote_command_out": str(args.remote_command_out),
|
|
"status": data["status"],
|
|
"ready_for_authorized_telegram_visible_dc_run": data[
|
|
"ready_for_authorized_telegram_visible_dc_run"
|
|
],
|
|
},
|
|
indent=2,
|
|
sort_keys=True,
|
|
)
|
|
)
|
|
return 0 if data["status"] == "pass" else 2
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|