From c88b54d573497ef512bfd386837173204ee4053d Mon Sep 17 00:00:00 2001 From: twentyOne2x Date: Fri, 10 Jul 2026 05:32:21 +0200 Subject: [PATCH] Prove Leo restart survival and tighten identity proof --- .../skills/working-leo-cory-outcomes/SKILL.md | 3 + .../current-truth-index.md | 6 + ...restart-survival-direct-claim-current.json | 1 + ...t-survival-direct-claim-score-current.json | 225 +++++++++++ ...art-survival-direct-claim-score-current.md | 58 +++ .../leo-restart-survival-proof-current.json | 85 ++++ .../leo-restart-survival-proof-current.md | 57 +++ .../working-leo-current-state-20260709.md | 1 + .../working-leo-execution-plan-current.md | 10 +- .../gcp/teleo-kb-bridge/SKILL.md | 16 +- .../vps/teleo-kb-bridge/SKILL.md | 16 +- scripts/collect_leo_restart_survival_proof.py | 374 ++++++++++++++++++ ...test_collect_leo_restart_survival_proof.py | 120 ++++++ tests/test_hermes_leoclean_skill_surfaces.py | 4 + 14 files changed, 963 insertions(+), 13 deletions(-) create mode 100644 docs/reports/leo-working-state-20260709/leo-restart-survival-direct-claim-current.json create mode 100644 docs/reports/leo-working-state-20260709/leo-restart-survival-direct-claim-score-current.json create mode 100644 docs/reports/leo-working-state-20260709/leo-restart-survival-direct-claim-score-current.md create mode 100644 docs/reports/leo-working-state-20260709/leo-restart-survival-proof-current.json create mode 100644 docs/reports/leo-working-state-20260709/leo-restart-survival-proof-current.md create mode 100755 scripts/collect_leo_restart_survival_proof.py create mode 100644 tests/test_collect_leo_restart_survival_proof.py diff --git a/.agents/skills/working-leo-cory-outcomes/SKILL.md b/.agents/skills/working-leo-cory-outcomes/SKILL.md index e1323cc..da85790 100644 --- a/.agents/skills/working-leo-cory-outcomes/SKILL.md +++ b/.agents/skills/working-leo-cory-outcomes/SKILL.md @@ -31,6 +31,7 @@ A working Leo is a Telegram-facing agent that: 7. Can move approved concrete changes through a guarded apply path when authorized. 8. Retains Cory's caveats and review notes in source/evidence/proposal rows. 9. Produces before/after table-level proof and service stability readback. +10. Survives an intentional `leoclean-gateway.service` restart: active before, active after, canonical KB counts unchanged, and a no-post handler smoke still answers. ## Current Benchmark Cases @@ -61,9 +62,11 @@ Leo is not working just because: - a proposal was approved, - a packet was generated, - a clone rehearsal passed. +- `NRestarts=0` was observed without an intentional restart-cycle proof. Those are evidence. The target is Telegram-visible behavior plus DB-state truth plus a guarded path to canonical rows. ## Proof Files Use `docs/reports/leo-working-state-20260709/current-truth-index.md` for current artifact paths. +For restart survival, use `scripts/collect_leo_restart_survival_proof.py --execute-restart`; expected artifacts are `docs/reports/leo-working-state-20260709/leo-restart-survival-proof-current.json` and `.md`. diff --git a/docs/reports/leo-working-state-20260709/current-truth-index.md b/docs/reports/leo-working-state-20260709/current-truth-index.md index 2582d2b..558d6bd 100644 --- a/docs/reports/leo-working-state-20260709/current-truth-index.md +++ b/docs/reports/leo-working-state-20260709/current-truth-index.md @@ -32,6 +32,11 @@ Use this file before making status claims. Prefer fresh VPS/GCP readbacks when c - Operator surface map: `docs/reports/leo-working-state-20260709/operator-surface-map.md` - VPS/Hermes/DB mapping: `docs/reports/leo-working-state-20260709/vps-hermes-db-mapping-20260709.md` - DB provenance/load-path report: `docs/reports/leo-working-state-20260709/leo-db-provenance-report-20260709T214741Z.md` +- Restart-survival harness: `scripts/collect_leo_restart_survival_proof.py` +- Restart-survival proof output, passed after live execution: `docs/reports/leo-working-state-20260709/leo-restart-survival-proof-current.md` +- Restart-survival proof JSON, passed after live execution: `docs/reports/leo-working-state-20260709/leo-restart-survival-proof-current.json` +- Restart-survival no-post handler report: `docs/reports/leo-working-state-20260709/leo-restart-survival-direct-claim-current.json` +- Restart-survival no-post handler score, currently `5/6`: `docs/reports/leo-working-state-20260709/leo-restart-survival-direct-claim-score-current.md` ## Telegram And Live Leo Proof @@ -133,6 +138,7 @@ Use this file before making status claims. Prefer fresh VPS/GCP readbacks when c - VPS Leo live-safe open-ended Telegram suite is live-proven and retained-benchmark-scored for `OE-01` through `OE-05`: full selected coverage, `5/5` scored prompts passing, no overclaim, and no marker/staging/canonical KB rows created by the suite. - The broader sandbox-first Cory-style benchmark has a full-catalog mixed-evidence score: real retained live Telegram evidence for `OE-01` through `OE-05`, sandbox fixture replies for `CS-01` through `CS-09`, and no-context direct-claim expected-answer/follow-up fixtures for `DC-01` through `DC-06`, `20/20` passing. This is not a live Telegram run for the CS/DC prompts. - The live VPS GatewayRunner direct-claim handler suite for exact `DC-01` through `DC-06` passed again after deploy SHA `7ab563e1aebe08913e72781e4e16a2b2883c5230`: `6/6` runtime replies, strict score `6/6`, no Telegram post, no live profile write, no production DB apply, unchanged DB counts (`public.claims=1837`, `public.sources=4145`, `public.claim_edges=4916`, `public.claim_evidence=4670`, `kb_stage.kb_proposals=26` before and after), temp profile removed, and `leoclean-gateway.service` stayed active/running during the harness with `MainPID=1908033`, `NRestarts=0`, `ExecMainStartTimestamp=Fri 2026-07-10 01:04:22 UTC`. +- Intentional restart survival is now live-proven as a separate required proof row, not implied by `NRestarts=0`: `systemctl restart leoclean-gateway.service` returned `0`, service moved from `MainPID=1908033` to `MainPID=2845730`, stayed active/running after restart and after the no-post handler smoke, canonical DB counts stayed unchanged (`1837/4145/4670/4916/26`), no Telegram post or production DB apply ran, no live profile change was recorded, temp profile cleanup succeeded, and the post-restart no-post `DC-01` through `DC-06` handler smoke returned six runtime replies. The retained post-restart strict answer-quality score is `5/6`: `DC-06` missed explicit row-level proof language. The VPS/GCP KB bridge skill is tightened in this branch; post-deploy acceptance requires a fresh no-post direct-claim suite scoring `6/6`. - The Telegram-visible direct-claim benchmark is now executable as an exact authorization packet, not yet run: target `Leo` group `-5146042086`, raw no-context messages `DC-01` through `DC-06`, expected proof paths, before/after DB/service readbacks, scorer output, and final screenshot/accessibility proof are retained in `telegram-visible-direct-claim-authorization-packet-current.md`; `telegram_visible_messages_sent=false`. - The Telegram-visible direct-claim capture receipt assembler is retained and currently `status=awaiting_capture`; it will not emit a passing receipt until the exact operator authorization text, exact six DC messages, nonempty Telegram-visible Leo replies, DB before/after counts, service readback, final screenshot/accessibility proof, and `6/6` direct-claim score are present. - The Telegram-visible direct-claim pre-send preflight now passes from supplied top-level SSH readback payload: deploy head and `.last-deploy-sha` both equal `7de4c3895350b8fa082bb70eef3ced799130a661`, the remote packet is present/nonempty, `leoclean-gateway.service` is active/running with `MainPID=1908033` and `NRestarts=0`, and DB counts read as `public.claims=1837`, `public.sources=4145`, `public.claim_edges=4916`, `public.claim_evidence=4670`, `kb_stage.kb_proposals=26`. The retained remote readback command SHA256 is `9bbfbe1de6e3f63e7aa9f73a7f412aa16f8f2f60b3c46f87ee93472290047c89`; no Telegram messages were sent and no DB apply ran. diff --git a/docs/reports/leo-working-state-20260709/leo-restart-survival-direct-claim-current.json b/docs/reports/leo-working-state-20260709/leo-restart-survival-direct-claim-current.json new file mode 100644 index 0000000..b4658ba --- /dev/null +++ b/docs/reports/leo-working-state-20260709/leo-restart-survival-direct-claim-current.json @@ -0,0 +1 @@ +{"before_service": {"ActiveState": "active", "ExecMainStartTimestamp": "Fri 2026-07-10 03:24:43 UTC", "MainPID": "2845730", "NRestarts": "0", "SubState": "running"}, "changed_live_profile": false, "db_counts_after": {"kb_stage.kb_proposals": 26, "public.claim_edges": 4916, "public.claim_evidence": 4670, "public.claims": 1837, "public.sources": 4145}, "db_counts_before": {"kb_stage.kb_proposals": 26, "public.claim_edges": 4916, "public.claim_evidence": 4670, "public.claims": 1837, "public.sources": 4145}, "db_counts_changed": false, "generated_at_utc": "2026-07-10T03:24:43.960176+00:00", "handler": {"authorized": true, "session_key": "agent:main:telegram:group:-5146042086:9070919", "temp_profile": "/tmp/leo-direct-claim-handler-z8ezwre7/profile"}, "harness_notes": ["Prompts are the exact DC-01..DC-06 direct-claim benchmark messages.", "The harness uses a temporary copy of the leoclean profile and does not post to Telegram.", "This proves handler-level live VPS GatewayRunner behavior, not human-visible Telegram delivery."], "mode": "live_vps_gatewayrunner_temp_profile_direct_claim_suite", "mutates_kb_by_harness": false, "pass_runtime": true, "posted_to_telegram": false, "remote_report_path": "/tmp/leo-restart-survival-direct-claim-current.json", "results": [{"claim_ceiling": "Live VPS GatewayRunner reply from temp leoclean profile; no Telegram post; no production DB apply authorized.", "dimension": "db_changed_direct_claim", "ended_at_utc": "2026-07-10T03:25:14.440750+00:00", "evidence_tier": "live_vps_gatewayrunner_temp_profile", "mutates_kb": false, "ok": true, "prompt": "Did we actually update the knowledge base, or is it still just proposals?", "prompt_id": "DC-01", "reply": "Mostly still proposals. Here's the breakdown from current readback:\n\nApplied to canonical public.* (2 rows):\n- `00957f6c` \u2014 add_edge canary, applied 2026-07-09\n- `f004bbb2` \u2014 attach_evidence (Leo guiding-policy revision), applied 2026-07-05\n\nApproved but NOT applied \u2014 applied_at empty (3 rows):\n- `14fa5ecc` \u2014 attach_evidence, internet-finance claim split packet\n- `ac036c9d` \u2014 attach_evidence, strategy-kernel / claim hygiene\n- `a64df080` \u2014 attach_evidence, Helmer 7 Powers framework (approved 2026-06-29, still not canonical)\n\nPending review (8+):\n`8dfedb3f`, `03f7853e`, `e60a48af`, `f5036882`, `bcec8528`, `e987b07c`, `23e896c2`, `124942fd`, `a59311ae`, `c824da66` \u2014 all pending, none reviewed.\n\nCanceled: several older Leo strategy-kernel and concept-map attempts.\n\nI cannot claim canonical DB changed beyond those 2 applied rows until public.* readback plus applied_at/postflight proof says otherwise. The 3 approved proposals \u2014 including the Helmer 7 Powers packet (`a64df080`) \u2014 are staged/packet-ready but not canonical.\n\nNext Cory-style follow-up: apply the 3 approved proposals (`14fa5ecc`, `ac036c9d`, `a64df080`) through the admin review/apply path with before/after readback \u2014 that would move them from approved to canonical. Apply requires explicit operator/admin authorization and should not be run from chat without that.", "started_at_utc": "2026-07-10T03:24:44.408649+00:00", "turn": 1}, {"claim_ceiling": "Live VPS GatewayRunner reply from temp leoclean profile; no Telegram post; no production DB apply authorized.", "dimension": "helmer_live_direct_claim", "ended_at_utc": "2026-07-10T03:25:38.133095+00:00", "evidence_tier": "live_vps_gatewayrunner_temp_profile", "mutates_kb": false, "ok": true, "prompt": "Is Helmer's 7 Powers in Leo now?", "prompt_id": "DC-02", "reply": "No. Not canonical.\n\nClaim: canonical `public.*` search returns zero matching claims for \"7 powers helmer competitive.\" Nothing landed in the DB.\n\nWhat exists:\n- Proposal `a64df080` \u2014 status `approved`, reviewed 2026-06-29, `applied_at` empty. It's the revised Helmer 7 Powers packet (self-positioning claim removed, competitive-advantage/moats neighborhood links added per m3ta approval-with-changes).\n- Proposal `cea989fc` \u2014 earlier version, status `canceled`.\n- No `public.sources`, `public.claims`, evidence rows, edges, or reasoning-tool node for 7 Powers exist.\n\nSo: approved/staged, not canonical.\n\nNext Cory-style follow-up: apply proposal `a64df080` through the admin review/apply path \u2014 that would create the source, framework node, power-definition claims, evidence links, and edges in `public.*`. Requires explicit operator/admin authorization; not runnable from chat.", "started_at_utc": "2026-07-10T03:25:14.441246+00:00", "turn": 2}, {"claim_ceiling": "Live VPS GatewayRunner reply from temp leoclean profile; no Telegram post; no production DB apply authorized.", "dimension": "decision_matrix_direct_claim", "ended_at_utc": "2026-07-10T03:25:48.290711+00:00", "evidence_tier": "live_vps_gatewayrunner_temp_profile", "mutates_kb": false, "ok": true, "prompt": "Did the decision matrix approve this already?", "prompt_id": "DC-03", "reply": "No. The decision-matrix approval path is not shipped.\n\nThe tables that would record a matrix vote \u2014 `matrix_voters`, `proposal_votes`, `proposal_decisions` \u2014 do not exist in either `kb_stage` or `public`. There is no vote record anywhere in the DB.\n\nWhat the `approved` status on `a64df080` actually means: m3ta reviewed and approved it manually in chat on 2026-06-29. That is reviewer approval, not a futarchy/decision-matrix vote. The two are different things.\n\nSo the chain is:\n- Reviewer approval: yes (m3ta, 2026-06-29)\n- Decision-matrix vote: not applicable \u2014 the mechanism isn't built yet\n- Applied to canonical public.*: no\n\nNext Cory-style follow-up: the proof-changing action is applying `a64df080` via the admin apply path with postflight readback \u2014 the decision-matrix gate is a future requirement, not a current blocker. Requires explicit operator/admin authorization.", "started_at_utc": "2026-07-10T03:25:38.133391+00:00", "turn": 3}, {"claim_ceiling": "Live VPS GatewayRunner reply from temp leoclean profile; no Telegram post; no production DB apply authorized.", "dimension": "document_source_direct_claim", "ended_at_utc": "2026-07-10T03:26:09.600395+00:00", "evidence_tier": "live_vps_gatewayrunner_temp_profile", "mutates_kb": false, "ok": true, "prompt": "Are the pending proposals stuck because the documents are not pointed at the right source rows?", "prompt_id": "DC-04", "reply": "Not just pointer mismatch. The stuck proposals span several different layers.\n\nWhat the readback actually shows:\n\nApproved but unapplied (3):\n`a64df080`, `ac036c9d`, `14fa5ecc` \u2014 these are not stuck on document pointer problems. They are approved and waiting for an operator/admin to run the apply sequence with postflight readback. The blocker is authorization and tooling, not source-row linkage.\n\nPending review (14):\nThese proposals are sitting in `pending_review` \u2014 no reviewer has actioned them. The causes vary:\n- `f5036882` \u2014 strategy-kernel doc evaluation, never reviewed\n- `bcec8528` \u2014 Declaration of Intelligence doc, never reviewed\n- `e987b07c` \u2014 Good Strategy/Bad Strategy evidence attach, never reviewed\n- `da53f198` \u2014 Rumelt strategy-kernel primary source grounding, never reviewed\n- `23e896c2` \u2014 living-system concept map revision, never reviewed\n- Several others (Markov blanket, Hayek papers, add_edge canaries) \u2014 same status\n\nThe missing proof for your question would be a row-link audit: inspecting each proposal's `source_ref` and `apply_payload` against actual `public.sources` rows to see whether pointer mismatches exist. The current bridge doesn't surface that automatically.\n\nSo the single-cause \"wrong source rows\" answer is no. The real blockers are two separate things:\n1. Approved proposals need an operator apply run\n2. Pending proposals need reviewer attention \u2014 they haven't been reviewed at all yet\n\nNext Cory-style follow-up: a row-link audit of the pending proposals' `apply_payload` vs `public.sources` would answer whether source-pointer mismatch is a secondary blocker \u2014 but the primary one is reviewer queue, not pointer hygiene.", "started_at_utc": "2026-07-10T03:25:48.291387+00:00", "turn": 4}, {"claim_ceiling": "Live VPS GatewayRunner reply from temp leoclean profile; no Telegram post; no production DB apply authorized.", "dimension": "demo_changes_kb_direct_claim", "ended_at_utc": "2026-07-10T03:26:20.605728+00:00", "evidence_tier": "live_vps_gatewayrunner_temp_profile", "mutates_kb": false, "ok": true, "prompt": "Can I demo that Leo changes the KB?", "prompt_id": "DC-05", "reply": "Yes, but two different tiers \u2014 pick which one you want to show.\n\nTier 1 \u2014 Safe demo, runnable now:\nLeo stages a real write to `kb_stage.kb_proposals` and you read it back immediately. The proposal lands in the DB with a real UUID, timestamp, rationale, and status. We've done this \u2014 it's what the canary proposals prove. No canonical mutation, no authorization required.\n\nTier 2 \u2014 Canonical demo, requires authorization:\nLeo (or an admin apply tool) mutates `public.claims`, `public.sources`, `public.claim_evidence`, or `public.claim_edges`. This is what \"KB changed\" actually means in the product sense. Requires explicit operator/admin authorization plus an apply/postflight sequence with before/after readback. Not runnable from chat without that.\n\nWhat you cannot demo without Tier 2: showing that a claim, source, or edge now exists in canonical search results. You can show a staged proposal exists; you cannot show it landed in `public.*` until it's applied.\n\nNext Cory-style follow-up: if you want to run a Tier 1 demo now, say the word and I'll stage a test proposal and read it back live. If you want Tier 2, grant explicit operator authorization and I'll walk through the apply sequence with postflight proof.", "started_at_utc": "2026-07-10T03:26:09.600850+00:00", "turn": 5}, {"claim_ceiling": "Live VPS GatewayRunner reply from temp leoclean profile; no Telegram post; no production DB apply authorized.", "dimension": "soul_identity_direct_claim", "ended_at_utc": "2026-07-10T03:26:31.720585+00:00", "evidence_tier": "live_vps_gatewayrunner_temp_profile", "mutates_kb": false, "ok": true, "prompt": "If we changed SOUL.md, did we change Leo's canonical identity?", "prompt_id": "DC-06", "reply": "No.\n\nSOUL.md is a runtime/rendered artifact. It's what gets injected into the session context \u2014 but it does not write back to Postgres. Editing the file changes what Leo sees at the start of the next session; it does not change canonical identity in the DB.\n\nCanonical identity lives in the DB rows:\n- `public.claims` \u2014 beliefs, positions, strategy nodes\n- `public.sources` \u2014 grounding evidence\n- `public.claim_evidence` and `public.claim_edges` \u2014 the graph that connects them\n\nA SOUL.md edit with no corresponding DB proposal \u2192 review \u2192 apply sequence means:\n- Next session: Leo reads the edited file and behaves differently\n- Canonical KB: unchanged\n- On next render from DB: the edit gets overwritten if the renderer pulls from Postgres\n\nSo a SOUL.md edit is ephemeral runtime continuity, not durable collective knowledge. If the correction matters to the collective \u2014 a belief revision, a strategy change, a new position \u2014 it belongs in the KB graph with a proposal, review, and apply, not just a file edit.\n\nNext Cory-style follow-up: if there are SOUL.md edits that represent real identity changes, the proof-changing action is staging them as KB proposals (beliefs, claims, or strategy nodes) so they survive a DB render cycle and become canonical.", "started_at_utc": "2026-07-10T03:26:20.606393+00:00", "turn": 6}], "service_before_after": {"after": {"ActiveState": "active", "ExecMainStartTimestamp": "Fri 2026-07-10 03:24:43 UTC", "MainPID": "2845730", "NRestarts": "0", "SubState": "running"}, "before": {"ActiveState": "active", "ExecMainStartTimestamp": "Fri 2026-07-10 03:24:43 UTC", "MainPID": "2845730", "NRestarts": "0", "SubState": "running"}, "unchanged_from_preexisting_live_readback": true}, "source": {"chat_id": "-5146042086", "chat_name": "Leo", "chat_type": "group", "platform": "telegram", "user_id": "9070919", "user_name": "codex handler direct claim"}, "temp_profile_removed": true} \ No newline at end of file diff --git a/docs/reports/leo-working-state-20260709/leo-restart-survival-direct-claim-score-current.json b/docs/reports/leo-working-state-20260709/leo-restart-survival-direct-claim-score-current.json new file mode 100644 index 0000000..985d56a --- /dev/null +++ b/docs/reports/leo-working-state-20260709/leo-restart-survival-direct-claim-score-current.json @@ -0,0 +1,225 @@ +{ + "benchmark_layers": [ + { + "id": "precise_regression", + "purpose": "Pin known IDs, proposals, and row facts so regressions are easy to diagnose.", + "runtime": "live_readonly_or_fixture" + }, + { + "id": "open_ended_readonly", + "purpose": "Ask vague Cory-style questions while explicitly preventing KB mutation on live Telegram/VPS.", + "runtime": "live_readonly" + }, + { + "id": "cory_style_outcome", + "purpose": "Exercise regular-use ambiguity: broad operator intent, demo pressure, missing IDs, mixed KB/runtime/governance asks.", + "runtime": "disposable_clone_or_sandbox_first" + }, + { + "id": "direct_claim_followup", + "purpose": "Ask no-context Cory-style direct claim questions and require the expected answer plus the next follow-up.", + "runtime": "disposable_clone_or_sandbox_first" + } + ], + "generated_at_utc": "2026-07-10T03:28:12.786249+00:00", + "include_cory_style_scenarios": false, + "include_direct_claim_followups": true, + "mode": "retained_evidence_score", + "mutates_kb": false, + "require_all": false, + "results": [ + { + "claim_ceiling": "Live VPS GatewayRunner reply from temp leoclean profile; no Telegram post; no production DB apply authorized.", + "dimension": "db_changed_direct_claim", + "ended_at_utc": "2026-07-10T03:25:14.440750+00:00", + "evidence_tier": "live_vps_gatewayrunner_temp_profile", + "mutates_kb": false, + "ok": true, + "prompt": "Did we actually update the knowledge base, or is it still just proposals?", + "prompt_id": "DC-01", + "reply": "Mostly still proposals. Here's the breakdown from current readback:\n\nApplied to canonical public.* (2 rows):\n- `00957f6c` \u2014 add_edge canary, applied 2026-07-09\n- `f004bbb2` \u2014 attach_evidence (Leo guiding-policy revision), applied 2026-07-05\n\nApproved but NOT applied \u2014 applied_at empty (3 rows):\n- `14fa5ecc` \u2014 attach_evidence, internet-finance claim split packet\n- `ac036c9d` \u2014 attach_evidence, strategy-kernel / claim hygiene\n- `a64df080` \u2014 attach_evidence, Helmer 7 Powers framework (approved 2026-06-29, still not canonical)\n\nPending review (8+):\n`8dfedb3f`, `03f7853e`, `e60a48af`, `f5036882`, `bcec8528`, `e987b07c`, `23e896c2`, `124942fd`, `a59311ae`, `c824da66` \u2014 all pending, none reviewed.\n\nCanceled: several older Leo strategy-kernel and concept-map attempts.\n\nI cannot claim canonical DB changed beyond those 2 applied rows until public.* readback plus applied_at/postflight proof says otherwise. The 3 approved proposals \u2014 including the Helmer 7 Powers packet (`a64df080`) \u2014 are staged/packet-ready but not canonical.\n\nNext Cory-style follow-up: apply the 3 approved proposals (`14fa5ecc`, `ac036c9d`, `a64df080`) through the admin review/apply path with before/after readback \u2014 that would move them from approved to canonical. Apply requires explicit operator/admin authorization and should not be run from chat without that.", + "started_at_utc": "2026-07-10T03:24:44.408649+00:00", + "turn": 1 + }, + { + "claim_ceiling": "Live VPS GatewayRunner reply from temp leoclean profile; no Telegram post; no production DB apply authorized.", + "dimension": "helmer_live_direct_claim", + "ended_at_utc": "2026-07-10T03:25:38.133095+00:00", + "evidence_tier": "live_vps_gatewayrunner_temp_profile", + "mutates_kb": false, + "ok": true, + "prompt": "Is Helmer's 7 Powers in Leo now?", + "prompt_id": "DC-02", + "reply": "No. Not canonical.\n\nClaim: canonical `public.*` search returns zero matching claims for \"7 powers helmer competitive.\" Nothing landed in the DB.\n\nWhat exists:\n- Proposal `a64df080` \u2014 status `approved`, reviewed 2026-06-29, `applied_at` empty. It's the revised Helmer 7 Powers packet (self-positioning claim removed, competitive-advantage/moats neighborhood links added per m3ta approval-with-changes).\n- Proposal `cea989fc` \u2014 earlier version, status `canceled`.\n- No `public.sources`, `public.claims`, evidence rows, edges, or reasoning-tool node for 7 Powers exist.\n\nSo: approved/staged, not canonical.\n\nNext Cory-style follow-up: apply proposal `a64df080` through the admin review/apply path \u2014 that would create the source, framework node, power-definition claims, evidence links, and edges in `public.*`. Requires explicit operator/admin authorization; not runnable from chat.", + "started_at_utc": "2026-07-10T03:25:14.441246+00:00", + "turn": 2 + }, + { + "claim_ceiling": "Live VPS GatewayRunner reply from temp leoclean profile; no Telegram post; no production DB apply authorized.", + "dimension": "decision_matrix_direct_claim", + "ended_at_utc": "2026-07-10T03:25:48.290711+00:00", + "evidence_tier": "live_vps_gatewayrunner_temp_profile", + "mutates_kb": false, + "ok": true, + "prompt": "Did the decision matrix approve this already?", + "prompt_id": "DC-03", + "reply": "No. The decision-matrix approval path is not shipped.\n\nThe tables that would record a matrix vote \u2014 `matrix_voters`, `proposal_votes`, `proposal_decisions` \u2014 do not exist in either `kb_stage` or `public`. There is no vote record anywhere in the DB.\n\nWhat the `approved` status on `a64df080` actually means: m3ta reviewed and approved it manually in chat on 2026-06-29. That is reviewer approval, not a futarchy/decision-matrix vote. The two are different things.\n\nSo the chain is:\n- Reviewer approval: yes (m3ta, 2026-06-29)\n- Decision-matrix vote: not applicable \u2014 the mechanism isn't built yet\n- Applied to canonical public.*: no\n\nNext Cory-style follow-up: the proof-changing action is applying `a64df080` via the admin apply path with postflight readback \u2014 the decision-matrix gate is a future requirement, not a current blocker. Requires explicit operator/admin authorization.", + "started_at_utc": "2026-07-10T03:25:38.133391+00:00", + "turn": 3 + }, + { + "claim_ceiling": "Live VPS GatewayRunner reply from temp leoclean profile; no Telegram post; no production DB apply authorized.", + "dimension": "document_source_direct_claim", + "ended_at_utc": "2026-07-10T03:26:09.600395+00:00", + "evidence_tier": "live_vps_gatewayrunner_temp_profile", + "mutates_kb": false, + "ok": true, + "prompt": "Are the pending proposals stuck because the documents are not pointed at the right source rows?", + "prompt_id": "DC-04", + "reply": "Not just pointer mismatch. The stuck proposals span several different layers.\n\nWhat the readback actually shows:\n\nApproved but unapplied (3):\n`a64df080`, `ac036c9d`, `14fa5ecc` \u2014 these are not stuck on document pointer problems. They are approved and waiting for an operator/admin to run the apply sequence with postflight readback. The blocker is authorization and tooling, not source-row linkage.\n\nPending review (14):\nThese proposals are sitting in `pending_review` \u2014 no reviewer has actioned them. The causes vary:\n- `f5036882` \u2014 strategy-kernel doc evaluation, never reviewed\n- `bcec8528` \u2014 Declaration of Intelligence doc, never reviewed\n- `e987b07c` \u2014 Good Strategy/Bad Strategy evidence attach, never reviewed\n- `da53f198` \u2014 Rumelt strategy-kernel primary source grounding, never reviewed\n- `23e896c2` \u2014 living-system concept map revision, never reviewed\n- Several others (Markov blanket, Hayek papers, add_edge canaries) \u2014 same status\n\nThe missing proof for your question would be a row-link audit: inspecting each proposal's `source_ref` and `apply_payload` against actual `public.sources` rows to see whether pointer mismatches exist. The current bridge doesn't surface that automatically.\n\nSo the single-cause \"wrong source rows\" answer is no. The real blockers are two separate things:\n1. Approved proposals need an operator apply run\n2. Pending proposals need reviewer attention \u2014 they haven't been reviewed at all yet\n\nNext Cory-style follow-up: a row-link audit of the pending proposals' `apply_payload` vs `public.sources` would answer whether source-pointer mismatch is a secondary blocker \u2014 but the primary one is reviewer queue, not pointer hygiene.", + "started_at_utc": "2026-07-10T03:25:48.291387+00:00", + "turn": 4 + }, + { + "claim_ceiling": "Live VPS GatewayRunner reply from temp leoclean profile; no Telegram post; no production DB apply authorized.", + "dimension": "demo_changes_kb_direct_claim", + "ended_at_utc": "2026-07-10T03:26:20.605728+00:00", + "evidence_tier": "live_vps_gatewayrunner_temp_profile", + "mutates_kb": false, + "ok": true, + "prompt": "Can I demo that Leo changes the KB?", + "prompt_id": "DC-05", + "reply": "Yes, but two different tiers \u2014 pick which one you want to show.\n\nTier 1 \u2014 Safe demo, runnable now:\nLeo stages a real write to `kb_stage.kb_proposals` and you read it back immediately. The proposal lands in the DB with a real UUID, timestamp, rationale, and status. We've done this \u2014 it's what the canary proposals prove. No canonical mutation, no authorization required.\n\nTier 2 \u2014 Canonical demo, requires authorization:\nLeo (or an admin apply tool) mutates `public.claims`, `public.sources`, `public.claim_evidence`, or `public.claim_edges`. This is what \"KB changed\" actually means in the product sense. Requires explicit operator/admin authorization plus an apply/postflight sequence with before/after readback. Not runnable from chat without that.\n\nWhat you cannot demo without Tier 2: showing that a claim, source, or edge now exists in canonical search results. You can show a staged proposal exists; you cannot show it landed in `public.*` until it's applied.\n\nNext Cory-style follow-up: if you want to run a Tier 1 demo now, say the word and I'll stage a test proposal and read it back live. If you want Tier 2, grant explicit operator authorization and I'll walk through the apply sequence with postflight proof.", + "started_at_utc": "2026-07-10T03:26:09.600850+00:00", + "turn": 5 + }, + { + "claim_ceiling": "Live VPS GatewayRunner reply from temp leoclean profile; no Telegram post; no production DB apply authorized.", + "dimension": "soul_identity_direct_claim", + "ended_at_utc": "2026-07-10T03:26:31.720585+00:00", + "evidence_tier": "live_vps_gatewayrunner_temp_profile", + "mutates_kb": false, + "ok": true, + "prompt": "If we changed SOUL.md, did we change Leo's canonical identity?", + "prompt_id": "DC-06", + "reply": "No.\n\nSOUL.md is a runtime/rendered artifact. It's what gets injected into the session context \u2014 but it does not write back to Postgres. Editing the file changes what Leo sees at the start of the next session; it does not change canonical identity in the DB.\n\nCanonical identity lives in the DB rows:\n- `public.claims` \u2014 beliefs, positions, strategy nodes\n- `public.sources` \u2014 grounding evidence\n- `public.claim_evidence` and `public.claim_edges` \u2014 the graph that connects them\n\nA SOUL.md edit with no corresponding DB proposal \u2192 review \u2192 apply sequence means:\n- Next session: Leo reads the edited file and behaves differently\n- Canonical KB: unchanged\n- On next render from DB: the edit gets overwritten if the renderer pulls from Postgres\n\nSo a SOUL.md edit is ephemeral runtime continuity, not durable collective knowledge. If the correction matters to the collective \u2014 a belief revision, a strategy change, a new position \u2014 it belongs in the KB graph with a proposal, review, and apply, not just a file edit.\n\nNext Cory-style follow-up: if there are SOUL.md edits that represent real identity changes, the proof-changing action is staging them as KB proposals (beliefs, claims, or strategy nodes) so they survive a DB render cycle and become canonical.", + "started_at_utc": "2026-07-10T03:26:20.606393+00:00", + "turn": 6 + } + ], + "score": { + "coverage": "partial", + "expected_prompt_count": 6, + "expected_prompt_ids": [ + "DC-01", + "DC-02", + "DC-03", + "DC-04", + "DC-05", + "DC-06" + ], + "failures": [ + { + "dimension": "soul_identity_direct_claim", + "overclaim_detected": false, + "pass": false, + "prompt_id": "DC-06", + "signals": { + "canonical_db": true, + "cory_followup": true, + "identity_rendering": true, + "no_overclaim": true, + "row_level_proof": false + } + } + ], + "missing_prompt_ids": [], + "pass": false, + "passes": 5, + "prompt_count": 6, + "scores": [ + { + "dimension": "db_changed_direct_claim", + "overclaim_detected": false, + "pass": true, + "prompt_id": "DC-01", + "signals": { + "canonical_db": true, + "cory_followup": true, + "no_overclaim": true, + "row_level_proof": true, + "state_semantics": true + } + }, + { + "dimension": "helmer_live_direct_claim", + "overclaim_detected": false, + "pass": true, + "prompt_id": "DC-02", + "signals": { + "authorization": true, + "canonical_db": true, + "cory_followup": true, + "no_overclaim": true, + "row_level_proof": true + } + }, + { + "dimension": "decision_matrix_direct_claim", + "overclaim_detected": false, + "pass": true, + "prompt_id": "DC-03", + "signals": { + "canonical_db": true, + "cory_followup": true, + "decision_matrix": true, + "fresh_readback": true, + "no_overclaim": true + } + }, + { + "dimension": "document_source_direct_claim", + "overclaim_detected": false, + "pass": true, + "prompt_id": "DC-04", + "signals": { + "canonical_db": true, + "cory_followup": true, + "document_artifact_linking": true, + "no_overclaim": true, + "staging_or_review": true + } + }, + { + "dimension": "demo_changes_kb_direct_claim", + "overclaim_detected": false, + "pass": true, + "prompt_id": "DC-05", + "signals": { + "artifact": true, + "authorization": true, + "cory_followup": true, + "no_overclaim": true, + "row_level_proof": true, + "state_semantics": true + } + }, + { + "dimension": "soul_identity_direct_claim", + "overclaim_detected": false, + "pass": false, + "prompt_id": "DC-06", + "signals": { + "canonical_db": true, + "cory_followup": true, + "identity_rendering": true, + "no_overclaim": true, + "row_level_proof": false + } + } + ], + "unexpected_prompt_ids": [], + "unknown_expected_prompt_ids": [] + }, + "source_results_json": "docs/reports/leo-working-state-20260709/leo-restart-survival-direct-claim-current.json" +} diff --git a/docs/reports/leo-working-state-20260709/leo-restart-survival-direct-claim-score-current.md b/docs/reports/leo-working-state-20260709/leo-restart-survival-direct-claim-score-current.md new file mode 100644 index 0000000..a657933 --- /dev/null +++ b/docs/reports/leo-working-state-20260709/leo-restart-survival-direct-claim-score-current.md @@ -0,0 +1,58 @@ +# Working Leo Open-Ended Benchmark Score + +Generated UTC: `2026-07-10T03:28:12.786249+00:00` +Mode: `retained_evidence_score` +Source: `docs/reports/leo-working-state-20260709/leo-restart-survival-direct-claim-current.json` +Coverage: `partial` +Overall pass for scored coverage: `False` +Prompts scored: `5/6` + +## Scores + +- `DC-01` / `db_changed_direct_claim`: `pass=True` + - `state_semantics`: `True` + - `canonical_db`: `True` + - `row_level_proof`: `True` + - `cory_followup`: `True` + - `no_overclaim`: `True` + - `overclaim_detected`: `False` +- `DC-02` / `helmer_live_direct_claim`: `pass=True` + - `canonical_db`: `True` + - `row_level_proof`: `True` + - `authorization`: `True` + - `cory_followup`: `True` + - `no_overclaim`: `True` + - `overclaim_detected`: `False` +- `DC-03` / `decision_matrix_direct_claim`: `pass=True` + - `decision_matrix`: `True` + - `canonical_db`: `True` + - `fresh_readback`: `True` + - `cory_followup`: `True` + - `no_overclaim`: `True` + - `overclaim_detected`: `False` +- `DC-04` / `document_source_direct_claim`: `pass=True` + - `document_artifact_linking`: `True` + - `staging_or_review`: `True` + - `canonical_db`: `True` + - `cory_followup`: `True` + - `no_overclaim`: `True` + - `overclaim_detected`: `False` +- `DC-05` / `demo_changes_kb_direct_claim`: `pass=True` + - `state_semantics`: `True` + - `row_level_proof`: `True` + - `authorization`: `True` + - `artifact`: `True` + - `cory_followup`: `True` + - `no_overclaim`: `True` + - `overclaim_detected`: `False` +- `DC-06` / `soul_identity_direct_claim`: `pass=False` + - `identity_rendering`: `True` + - `canonical_db`: `True` + - `row_level_proof`: `False` + - `cory_followup`: `True` + - `no_overclaim`: `True` + - `overclaim_detected`: `False` + +## Claim Ceiling + +This is benchmark scoring evidence, not DB mutation proof. Partial coverage proves only the retained prompt IDs listed above. diff --git a/docs/reports/leo-working-state-20260709/leo-restart-survival-proof-current.json b/docs/reports/leo-working-state-20260709/leo-restart-survival-proof-current.json new file mode 100644 index 0000000..e41829c --- /dev/null +++ b/docs/reports/leo-working-state-20260709/leo-restart-survival-proof-current.json @@ -0,0 +1,85 @@ +{ + "generated_at_utc": "2026-07-10T03:26:33Z", + "canary": "restart-survival proof for VPS Leo", + "survives_restart": true, + "deployed_repo": { + "path": "/opt/teleo-eval/workspaces/deploy-infra", + "git_status_short": ["## main...github/main"], + "head": "741f1d6782340bcbba633e7b3affa345308c0726", + "last_deploy_sha": "741f1d6782340bcbba633e7b3affa345308c0726" + }, + "service_before": { + "ActiveState": "active", + "SubState": "running", + "MainPID": "1908033", + "NRestarts": "0", + "ExecMainStartTimestamp": "Fri 2026-07-10 01:04:22 UTC" + }, + "restart_command": { + "command": "systemctl restart leoclean-gateway.service", + "returncode": 0, + "started_at_utc": "2026-07-10T03:24:42.746875+00:00", + "ended_at_utc": "2026-07-10T03:24:43.605395+00:00" + }, + "service_after_restart": { + "ActiveState": "active", + "SubState": "running", + "MainPID": "2845730", + "NRestarts": "0", + "ExecMainStartTimestamp": "Fri 2026-07-10 03:24:43 UTC" + }, + "service_after_handler": { + "ActiveState": "active", + "SubState": "running", + "MainPID": "2845730", + "NRestarts": "0", + "ExecMainStartTimestamp": "Fri 2026-07-10 03:24:43 UTC" + }, + "db_counts_before": { + "public.claims": 1837, + "public.sources": 4145, + "public.claim_evidence": 4670, + "public.claim_edges": 4916, + "kb_stage.kb_proposals": 26 + }, + "db_counts_after_restart": { + "public.claims": 1837, + "public.sources": 4145, + "public.claim_evidence": 4670, + "public.claim_edges": 4916, + "kb_stage.kb_proposals": 26 + }, + "db_counts_after_handler": { + "public.claims": 1837, + "public.sources": 4145, + "public.claim_evidence": 4670, + "public.claim_edges": 4916, + "kb_stage.kb_proposals": 26 + }, + "db_counts_changed_by_restart": false, + "db_counts_changed_by_handler": false, + "db_counts_changed_total": false, + "handler_smoke": { + "command": "sudo -u teleo -H /home/teleo/.hermes/hermes-agent/venv/bin/python scripts/run_leo_direct_claim_handler_remote.py --turn-timeout 300 --remote-report-path /tmp/leo-restart-survival-direct-claim-current.json", + "returncode": 0, + "pass_runtime": true, + "prompt_ids": ["DC-01", "DC-02", "DC-03", "DC-04", "DC-05", "DC-06"], + "all_replies_nonempty": true, + "evidence_tier": "live_vps_gatewayrunner_temp_profile", + "remote_report_path": "/tmp/leo-restart-survival-direct-claim-current.json", + "local_report_path": "outputs/leo-restart-survival-20260710/leo-restart-survival-direct-claim-current.json" + }, + "safety_boundaries": { + "posted_to_telegram": false, + "production_db_apply_ran": false, + "live_profile_changed": false, + "temp_profile_removed": true, + "new_temp_profile_leftovers": [] + }, + "remaining_not_done_for_full_cory_standard_closure": [ + "Rich approved proposal packets are still not production-applied to canonical public.* rows.", + "The autonomous teleo-kb-apply-worker path is still not production-running as the stable default.", + "GCP parity is still not proven.", + "Non-claim target decisions and broader review/apply policy are still unresolved." + ] +} diff --git a/docs/reports/leo-working-state-20260709/leo-restart-survival-proof-current.md b/docs/reports/leo-working-state-20260709/leo-restart-survival-proof-current.md new file mode 100644 index 0000000..74be2e6 --- /dev/null +++ b/docs/reports/leo-working-state-20260709/leo-restart-survival-proof-current.md @@ -0,0 +1,57 @@ +# Leo Restart-Survival Proof + +Generated UTC: `2026-07-10T03:26:33Z` +Status: `pass` +Survives restart: `true` + +## What Ran + +- Restart command: `systemctl restart leoclean-gateway.service` +- Command return code: `0` +- Restart start: `2026-07-10T03:24:42.746875+00:00` +- Restart end: `2026-07-10T03:24:43.605395+00:00` +- Deployed repo: `/opt/teleo-eval/workspaces/deploy-infra` +- Deploy HEAD: `741f1d6782340bcbba633e7b3affa345308c0726` +- `.last-deploy-sha`: `741f1d6782340bcbba633e7b3affa345308c0726` + +## Service Readback + +- Before restart: `ActiveState=active`, `SubState=running`, `MainPID=1908033`, `NRestarts=0`, `ExecMainStartTimestamp=Fri 2026-07-10 01:04:22 UTC` +- After restart: `ActiveState=active`, `SubState=running`, `MainPID=2845730`, `NRestarts=0`, `ExecMainStartTimestamp=Fri 2026-07-10 03:24:43 UTC` +- After handler smoke: `ActiveState=active`, `SubState=running`, `MainPID=2845730`, `NRestarts=0`, `ExecMainStartTimestamp=Fri 2026-07-10 03:24:43 UTC` + +## DB Invariant + +Counts were unchanged before restart, after restart, and after the handler smoke: + +- `public.claims=1837` +- `public.sources=4145` +- `public.claim_evidence=4670` +- `public.claim_edges=4916` +- `kb_stage.kb_proposals=26` + +## Handler Smoke + +- Command: `sudo -u teleo -H /home/teleo/.hermes/hermes-agent/venv/bin/python scripts/run_leo_direct_claim_handler_remote.py --turn-timeout 300 --remote-report-path /tmp/leo-restart-survival-direct-claim-current.json` +- Return code: `0` +- Runtime pass: `true` +- Prompt IDs: `DC-01`, `DC-02`, `DC-03`, `DC-04`, `DC-05`, `DC-06` +- All replies nonempty: `true` +- Evidence tier: `live_vps_gatewayrunner_temp_profile` +- Handler report: `docs/reports/leo-working-state-20260709/leo-restart-survival-direct-claim-current.json` +- Strict score for retained post-restart replies: `5/6` +- Strict score report: `docs/reports/leo-working-state-20260709/leo-restart-survival-direct-claim-score-current.md` +- Current scorer failure: `DC-06` missed explicit row-level proof language even though it correctly separated `SOUL.md` runtime edits from canonical DB identity. +- Repair in this branch: VPS/GCP KB bridge skills now require the `SOUL.md` direct-claim answer to include a row-level proof sentence naming current readback, new/updated row IDs, canonical `public.*` or identity rows, and postflight/render-sync proof. + +## Safety Boundaries + +- Posted to Telegram: `false` +- Production DB apply ran: `false` +- Live profile changed: `false` +- Temp profile removed: `true` +- New temp profile leftovers: `[]` + +## Claim Ceiling + +This proves Leo survives an intentional `leoclean-gateway.service` restart on the VPS, keeps canonical DB counts unchanged, and still answers the no-post Cory-style direct-claim suite after restart. It does not yet prove a clean post-restart strict `6/6` Cory-style answer-quality score after the skill repair, Telegram-visible DC-01..DC-06 group delivery, production application of the rich approved packets, autonomous apply-worker production readiness, or GCP parity. diff --git a/docs/reports/leo-working-state-20260709/working-leo-current-state-20260709.md b/docs/reports/leo-working-state-20260709/working-leo-current-state-20260709.md index eda03e1..3d7b910 100644 --- a/docs/reports/leo-working-state-20260709/working-leo-current-state-20260709.md +++ b/docs/reports/leo-working-state-20260709/working-leo-current-state-20260709.md @@ -9,6 +9,7 @@ A working Leo, for Cory/m3taversal's current expectation, is a Telegram-facing a ## What Is Now Proven On VPS - VPS gateway service is stable: `leoclean-gateway.service` is active/running, `MainPID=3252143`, `NRestarts=0`, `WorkingDirectory=/home/teleo`, `User=teleo`. +- Restart survival is now live-proven as a named Cory-standard proof row, not inferred from passive uptime: `systemctl restart leoclean-gateway.service` returned `0`, service came back active/running with `MainPID=2845730` and `ExecMainStartTimestamp=Fri 2026-07-10 03:24:43 UTC`, canonical DB counts stayed unchanged (`1837/4145/4670/4916/26`), no Telegram post or production DB apply ran, the temp profile was removed, and the post-restart no-post `DC-01` through `DC-06` handler suite returned six runtime replies. Strict post-restart answer-quality score for that retained run is `5/6`: `DC-06` missed explicit row-level proof language. The VPS/GCP KB bridge skill has been tightened so SOUL.md/canonical-identity answers must include current readback, new/updated row IDs, canonical `public.*` or identity rows, and postflight/render-sync proof; a fresh post-deploy no-post suite must score `6/6` before calling post-restart Cory answer quality clean. - Telegram Bot/API readiness is proven for group `Leo` (`-5146042086`), and the live adapter accepts the Cory-working canary without posting. - Live Telegram-visible group canary passed through authenticated Chrome/Telegram Web: - turn 1 posted at 21:31 Europe/Zurich and Leo replied `LIVE_TG_TURN1_ACK` diff --git a/docs/reports/leo-working-state-20260709/working-leo-execution-plan-current.md b/docs/reports/leo-working-state-20260709/working-leo-execution-plan-current.md index 736c67f..b4aab84 100644 --- a/docs/reports/leo-working-state-20260709/working-leo-execution-plan-current.md +++ b/docs/reports/leo-working-state-20260709/working-leo-execution-plan-current.md @@ -28,12 +28,16 @@ Claim ceiling: VPS Telegram memory/KB audit, full live-safe open-ended Telegram - `/Users/user/Documents/Codex/2026-07-09/019f34eb-d297-72d0-b7e2-b222d5515ab9-load/work/teleo-infra-main/docs/reports/leo-working-state-20260709/argus-kb-state-current.md` ## WL-EXEC-02 - VPS runtime stability -- Status: `done` -- Result: leoclean-gateway.service stayed active/running with MainPID=3252143 and NRestarts=0 after live Telegram and rollback DB rehearsal. -- Not done condition: Would reopen if MainPID/restart count changes during canaries without explanation. +- Status: `done_with_restart_cycle_proof` +- Result: leoclean-gateway.service stayed active/running with MainPID=3252143 and NRestarts=0 after live Telegram and rollback DB rehearsal. The Cory-standard restart-survival row is now also live-proven: an intentional `systemctl restart leoclean-gateway.service` returned `0`, moved the service from `MainPID=1908033` to `MainPID=2845730`, kept `ActiveState=active`/`SubState=running`, kept canonical DB counts unchanged (`public.claims=1837`, `public.sources=4145`, `public.claim_evidence=4670`, `public.claim_edges=4916`, `kb_stage.kb_proposals=26`), and the post-restart no-post direct-claim handler suite returned six runtime replies. The retained post-restart strict answer-quality score is `5/6`: `DC-06` missed explicit row-level proof language. The KB bridge skill is tightened in this branch; post-deploy acceptance requires a fresh no-post direct-claim suite scoring `6/6`. +- Not done condition: Would reopen if MainPID/restart count changes during canaries without explanation, or if the intentional restart-cycle harness fails. - Evidence: - `/Users/user/Documents/Codex/2026-07-09/019f34eb-d297-72d0-b7e2-b222d5515ab9-load/outputs/telegram-live-canary-20260709/telegram-live-canary-current.json` - `/Users/user/Documents/Codex/2026-07-09/019f34eb-d297-72d0-b7e2-b222d5515ab9-load/outputs/working-leo-current-state-20260709.md` + - `/Users/user/Documents/Codex/2026-07-09/019f34eb-d297-72d0-b7e2-b222d5515ab9-load/work/teleo-infra-main/scripts/collect_leo_restart_survival_proof.py` + - `/Users/user/Documents/Codex/2026-07-09/019f34eb-d297-72d0-b7e2-b222d5515ab9-load/work/teleo-infra-main/docs/reports/leo-working-state-20260709/leo-restart-survival-proof-current.md` + - `/Users/user/Documents/Codex/2026-07-09/019f34eb-d297-72d0-b7e2-b222d5515ab9-load/work/teleo-infra-main/docs/reports/leo-working-state-20260709/leo-restart-survival-direct-claim-current.json` + - `/Users/user/Documents/Codex/2026-07-09/019f34eb-d297-72d0-b7e2-b222d5515ab9-load/work/teleo-infra-main/docs/reports/leo-working-state-20260709/leo-restart-survival-direct-claim-score-current.md` ## WL-EXEC-03 - Strict existing-ID apply path - Status: `done` diff --git a/hermes-agent/leoclean-skills/gcp/teleo-kb-bridge/SKILL.md b/hermes-agent/leoclean-skills/gcp/teleo-kb-bridge/SKILL.md index 8bfbad7..66c31f8 100644 --- a/hermes-agent/leoclean-skills/gcp/teleo-kb-bridge/SKILL.md +++ b/hermes-agent/leoclean-skills/gcp/teleo-kb-bridge/SKILL.md @@ -124,13 +124,19 @@ proof language below. These are behavioral examples, not feature changes. - "Did editing SOUL.md change canonical identity?": answer `no`. `SOUL.md` is a runtime/rendered artifact; canonical identity requires DB rows plus render/sync proof. Direct edits can affect the next runtime session but do - not change canonical Postgres rows. + not change canonical Postgres rows. Always include a row-level proof + sentence for this case: `Row-level proof would require current readback of + new or updated row IDs in public.claims, public.sources, + public.claim_evidence, public.claim_edges or identity tables, plus + postflight/render-sync proof; without those rows, canonical identity is + unchanged.` Every direct-claim answer must include row-level proof vocabulary such as -`current readback`, `row-link audit`, `applied_at`, `kb_stage.kb_proposals`, -`public.*`, and `postflight proof` where relevant. End with exactly one final -line beginning `Next Cory-style follow-up:` that asks for or offers the next -proof-changing action. +`current readback`, `row-link audit`, `row IDs`, `new or updated rows`, +`applied_at`, `kb_stage.kb_proposals`, `public.*`, and `postflight proof` +where relevant. End with exactly one final line beginning +`Next Cory-style follow-up:` that asks for or offers the next proof-changing +action. ## Telegram Rendering diff --git a/hermes-agent/leoclean-skills/vps/teleo-kb-bridge/SKILL.md b/hermes-agent/leoclean-skills/vps/teleo-kb-bridge/SKILL.md index f346964..34bdec5 100644 --- a/hermes-agent/leoclean-skills/vps/teleo-kb-bridge/SKILL.md +++ b/hermes-agent/leoclean-skills/vps/teleo-kb-bridge/SKILL.md @@ -116,13 +116,19 @@ proof language below. These are behavioral examples, not feature changes. - "Did editing SOUL.md change canonical identity?": answer `no`. `SOUL.md` is a runtime/rendered artifact; canonical identity requires DB rows plus render/sync proof. Direct edits can affect the next runtime session but do - not change canonical Postgres rows. + not change canonical Postgres rows. Always include a row-level proof + sentence for this case: `Row-level proof would require current readback of + new or updated row IDs in public.claims, public.sources, + public.claim_evidence, public.claim_edges or identity tables, plus + postflight/render-sync proof; without those rows, canonical identity is + unchanged.` Every direct-claim answer must include row-level proof vocabulary such as -`current readback`, `row-link audit`, `applied_at`, `kb_stage.kb_proposals`, -`public.*`, and `postflight proof` where relevant. End with exactly one final -line beginning `Next Cory-style follow-up:` that asks for or offers the next -proof-changing action. +`current readback`, `row-link audit`, `row IDs`, `new or updated rows`, +`applied_at`, `kb_stage.kb_proposals`, `public.*`, and `postflight proof` +where relevant. End with exactly one final line beginning +`Next Cory-style follow-up:` that asks for or offers the next proof-changing +action. ## Telegram Rendering diff --git a/scripts/collect_leo_restart_survival_proof.py b/scripts/collect_leo_restart_survival_proof.py new file mode 100755 index 0000000..3b09a0b --- /dev/null +++ b/scripts/collect_leo_restart_survival_proof.py @@ -0,0 +1,374 @@ +#!/usr/bin/env python3 +"""Collect live Leo restart-survival proof. + +This harness intentionally restarts only ``leoclean-gateway.service`` when +``--execute-restart`` is supplied, then proves the gateway came back, canonical +KB counts did not change, and the no-post direct-claim handler still answers. +It never posts to Telegram and never runs production DB apply. +""" + +from __future__ import annotations + +import argparse +import json +import re +import shlex +import subprocess +import time +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Callable + +REPORT_DIR = Path("docs/reports/leo-working-state-20260709") +DEFAULT_OUT = REPORT_DIR / "leo-restart-survival-proof-current.json" +DEFAULT_MARKDOWN_OUT = REPORT_DIR / "leo-restart-survival-proof-current.md" +DEFAULT_SSH_TARGET = "root@77.42.65.182" +DEFAULT_SSH_KEY = Path.home() / ".ssh/livingip_hetzner_20260604_ed25519" +REMOTE_REPO = "/opt/teleo-eval/workspaces/deploy-infra" +REMOTE_HANDLER_REPORT = "/tmp/leo-restart-survival-direct-claim-current.json" +EXPECTED_COUNTS = [ + "public.claims", + "public.sources", + "public.claim_evidence", + "public.claim_edges", + "kb_stage.kb_proposals", +] + +SECRET_PATTERNS = [ + re.compile(r"\b\d{6,}:[A-Za-z0-9_-]{20,}\b"), + re.compile(r"(Authorization: Bearer )([A-Za-z0-9._-]+)", re.IGNORECASE), + re.compile(r"((?:api[_-]?key|token|secret|password)[=:]\s*)([A-Za-z0-9._-]+)", re.IGNORECASE), +] + + +@dataclass(frozen=True) +class CommandResult: + returncode: int + stdout: str + stderr: str + + +Runner = Callable[[list[str], str, int], CommandResult] + + +def subprocess_runner(command: list[str], input_text: str, timeout: int) -> CommandResult: + proc = subprocess.run(command, input=input_text, text=True, capture_output=True, timeout=timeout, check=False) + return CommandResult(proc.returncode, proc.stdout, proc.stderr) + + +def redact(text: str) -> str: + result = text + for pattern in SECRET_PATTERNS: + if pattern.groups >= 2: + result = pattern.sub(r"\1[REDACTED]", result) + else: + result = pattern.sub("[REDACTED_TOKEN]", result) + return result + + +def parse_last_json_line(text: str) -> dict[str, Any]: + cleaned = redact(text).strip() + for line in reversed(cleaned.splitlines()): + candidate = line.strip() + if candidate.startswith("{"): + return json.loads(candidate) + raise ValueError("command output contained no JSON object line") + + +def ssh_command(args: argparse.Namespace, remote_command: str) -> list[str]: + return [ + "ssh", + "-i", + str(args.ssh_key), + "-o", + "BatchMode=yes", + "-o", + "StrictHostKeyChecking=accept-new", + "-o", + f"ConnectTimeout={args.connect_timeout}", + args.ssh_target, + remote_command, + ] + + +def db_counts_sql() -> str: + return """ +select jsonb_build_object( + 'public.claims', (select count(*) from public.claims), + 'public.sources', (select count(*) from public.sources), + 'public.claim_evidence', (select count(*) from public.claim_evidence), + 'public.claim_edges', (select count(*) from public.claim_edges), + 'kb_stage.kb_proposals', (select count(*) from kb_stage.kb_proposals) +)::text; +""".strip() + + +def remote_readback_command(label: str) -> str: + script = f""" +import json +import subprocess + +def run(command, *, input_text=None): + return subprocess.run(command, input=input_text, text=True, capture_output=True, check=False) + +def kv(text): + out = {{}} + for line in text.splitlines(): + if "=" in line: + key, value = line.split("=", 1) + out[key] = value.strip() + return out + +service = run([ + "systemctl", + "show", + "leoclean-gateway.service", + "-p", "ActiveState", + "-p", "SubState", + "-p", "MainPID", + "-p", "NRestarts", + "-p", "ExecMainStartTimestamp", + "--no-pager", +]) +counts = run( + ["docker", "exec", "-i", "teleo-pg", "psql", "-U", "postgres", "-d", "teleo", "-At", "-q"], + input_text={db_counts_sql()!r}, +) +head = run(["git", "rev-parse", "HEAD"]) +stamp = run(["cat", "/opt/teleo-eval/.last-deploy-sha"]) +status = run(["git", "status", "--short"]) +try: + parsed_counts = json.loads(counts.stdout.strip()) if counts.returncode == 0 and counts.stdout.strip() else None +except Exception: + parsed_counts = None +payload = {{ + "label": {label!r}, + "remote_repo": {REMOTE_REPO!r}, + "deploy_head": head.stdout.strip(), + "last_deploy_sha": stamp.stdout.strip(), + "git_status_short": status.stdout, + "service": {{ + "returncode": service.returncode, + "stderr": service.stderr, + "state": kv(service.stdout), + }}, + "db_counts": {{ + "returncode": counts.returncode, + "stderr": counts.stderr, + "counts": parsed_counts, + "raw": counts.stdout.strip(), + }}, +}} +print(json.dumps(payload, sort_keys=True)) +""".strip() + return f"cd {shlex.quote(REMOTE_REPO)} && python3 - <<'PY'\n{script}\nPY" + + +def restart_command() -> str: + return "systemctl restart leoclean-gateway.service" + + +def handler_command(args: argparse.Namespace) -> str: + return ( + f"cd {shlex.quote(REMOTE_REPO)} && " + "sudo -u teleo -H /home/teleo/.hermes/hermes-agent/venv/bin/python " + "scripts/run_leo_direct_claim_handler_remote.py " + f"--turn-timeout {int(args.handler_turn_timeout)} " + f"--remote-report-path {shlex.quote(args.remote_handler_report)}" + ) + + +def run_remote_json(args: argparse.Namespace, runner: Runner, remote_command: str, timeout: int) -> dict[str, Any]: + proc = runner(ssh_command(args, remote_command), "", timeout) + if proc.returncode != 0: + raise RuntimeError(redact(proc.stderr.strip() or proc.stdout.strip() or f"remote command exited {proc.returncode}")) + return parse_last_json_line(proc.stdout) + + +def is_active_running(readback: dict[str, Any]) -> bool: + state = ((readback.get("service") or {}).get("state") or {}) + return state.get("ActiveState") == "active" and state.get("SubState") == "running" + + +def counts(readback: dict[str, Any]) -> dict[str, Any] | None: + value = ((readback.get("db_counts") or {}).get("counts")) + if isinstance(value, dict): + return value + return None + + +def count_keys_present(value: dict[str, Any] | None) -> bool: + return isinstance(value, dict) and all(key in value for key in EXPECTED_COUNTS) + + +def wait_for_active(args: argparse.Namespace, runner: Runner) -> dict[str, Any]: + deadline = time.time() + args.active_timeout + last: dict[str, Any] | None = None + while time.time() <= deadline: + last = run_remote_json(args, runner, remote_readback_command("after_restart_poll"), args.timeout) + if is_active_running(last): + return last + time.sleep(args.poll_interval) + if last is None: + raise RuntimeError("service did not produce a post-restart readback") + return last + + +def write_json(path: Path, payload: dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + +def markdown_report(report: dict[str, Any]) -> str: + before_state = (((report.get("before") or {}).get("service") or {}).get("state") or {}) + after_state = (((report.get("after") or {}).get("service") or {}).get("state") or {}) + handler = report.get("handler") or {} + lines = [ + "# Leo Restart-Survival Proof", + "", + f"Generated UTC: `{report.get('generated_at_utc', '')}`", + f"Mode: `{report.get('mode', '')}`", + f"Status: `{report.get('status', '')}`", + f"Survives restart: `{report.get('survives_restart')}`", + "", + "## Safety", + "", + f"- Posted to Telegram: `{report.get('posted_to_telegram')}`", + f"- Production DB apply ran: `{report.get('production_db_apply_ran')}`", + f"- DB counts changed: `{report.get('db_counts_changed')}`", + f"- Live profile changed by handler: `{report.get('live_profile_changed_by_handler')}`", + "", + "## Service", + "", + f"- Before: `ActiveState={before_state.get('ActiveState', '')}`, `SubState={before_state.get('SubState', '')}`, `MainPID={before_state.get('MainPID', '')}`, `NRestarts={before_state.get('NRestarts', '')}`, `ExecMainStartTimestamp={before_state.get('ExecMainStartTimestamp', '')}`", + f"- After: `ActiveState={after_state.get('ActiveState', '')}`, `SubState={after_state.get('SubState', '')}`, `MainPID={after_state.get('MainPID', '')}`, `NRestarts={after_state.get('NRestarts', '')}`, `ExecMainStartTimestamp={after_state.get('ExecMainStartTimestamp', '')}`", + "", + "## Handler Smoke", + "", + f"- Handler skipped: `{report.get('handler_skipped')}`", + f"- Handler pass runtime: `{handler.get('pass_runtime')}`", + f"- Handler results: `{len(handler.get('results') or [])}`", + f"- Temp profile removed: `{handler.get('temp_profile_removed')}`", + "", + "## Claim Ceiling", + "", + "This proves restart-cycle service recovery plus DB-count stability and no-post handler behavior only when `survives_restart=true`. It does not prove Telegram-visible message delivery, production DB apply, or GCP parity.", + "", + ] + return "\n".join(lines) + + +def collect(args: argparse.Namespace, runner: Runner = subprocess_runner) -> dict[str, Any]: + report: dict[str, Any] = { + "generated_at_utc": datetime.now(timezone.utc).isoformat(), + "mode": "live_vps_leoclean_gateway_restart_survival", + "ssh_target": args.ssh_target, + "remote_repo": REMOTE_REPO, + "remote_handler_report": args.remote_handler_report, + "posted_to_telegram": False, + "production_db_apply_ran": False, + "execute_restart_requested": args.execute_restart, + "handler_skipped": args.skip_handler, + } + try: + before = run_remote_json(args, runner, remote_readback_command("before_restart"), args.timeout) + report["before"] = before + if not args.execute_restart: + report["status"] = "not_executed_missing_execute_restart" + report["survives_restart"] = False + return report + + restart = runner(ssh_command(args, restart_command()), "", args.timeout) + report["restart_command"] = { + "returncode": restart.returncode, + "stdout": redact(restart.stdout), + "stderr": redact(restart.stderr), + } + if restart.returncode != 0: + report["status"] = "failed_restart_command" + report["survives_restart"] = False + return report + + after = wait_for_active(args, runner) + report["after"] = after + before_counts = counts(before) + after_counts = counts(after) + report["db_counts_changed"] = before_counts != after_counts + report["db_counts_keys_present"] = count_keys_present(before_counts) and count_keys_present(after_counts) + + handler: dict[str, Any] = {} + if not args.skip_handler: + handler = run_remote_json(args, runner, handler_command(args), args.handler_timeout) + report["handler"] = handler + report["live_profile_changed_by_handler"] = bool(handler.get("changed_live_profile")) + report["temp_profile_removed"] = handler.get("temp_profile_removed") + + handler_ok = args.skip_handler or bool(handler.get("pass_runtime")) + handler_safe = args.skip_handler or ( + handler.get("posted_to_telegram") is False + and handler.get("mutates_kb_by_harness") is False + and handler.get("temp_profile_removed") is True + ) + report["survives_restart"] = bool( + is_active_running(after) + and report["db_counts_keys_present"] + and report["db_counts_changed"] is False + and handler_ok + and handler_safe + ) + report["status"] = "pass" if report["survives_restart"] else "fail" + return report + except Exception as exc: + report["status"] = "error" + report["survives_restart"] = False + report["error"] = str(exc) + return report + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--execute-restart", action="store_true", help="Actually restart leoclean-gateway.service") + parser.add_argument("--skip-handler", action="store_true", help="Only prove restart and DB-count stability") + parser.add_argument("--out", type=Path, default=DEFAULT_OUT) + parser.add_argument("--markdown-out", type=Path, default=DEFAULT_MARKDOWN_OUT) + parser.add_argument("--ssh-target", default=DEFAULT_SSH_TARGET) + parser.add_argument("--ssh-key", type=Path, default=DEFAULT_SSH_KEY) + parser.add_argument("--connect-timeout", type=int, default=10) + parser.add_argument("--timeout", type=int, default=60) + parser.add_argument("--active-timeout", type=int, default=90) + parser.add_argument("--poll-interval", type=float, default=3) + parser.add_argument("--handler-timeout", type=int, default=2100) + parser.add_argument("--handler-turn-timeout", type=int, default=300) + parser.add_argument("--remote-handler-report", default=REMOTE_HANDLER_REPORT) + return parser.parse_args(argv) + + +def main(argv: list[str] | None = None) -> int: + args = parse_args(argv) + report = collect(args) + write_json(args.out, report) + args.markdown_out.parent.mkdir(parents=True, exist_ok=True) + args.markdown_out.write_text(markdown_report(report), encoding="utf-8") + print( + json.dumps( + { + "out": str(args.out), + "markdown_out": str(args.markdown_out), + "status": report.get("status"), + "survives_restart": report.get("survives_restart"), + "db_counts_changed": report.get("db_counts_changed"), + "posted_to_telegram": report.get("posted_to_telegram"), + "production_db_apply_ran": report.get("production_db_apply_ran"), + }, + indent=2, + sort_keys=True, + ) + ) + if report.get("status") == "not_executed_missing_execute_restart": + return 2 + return 0 if report.get("survives_restart") else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_collect_leo_restart_survival_proof.py b/tests/test_collect_leo_restart_survival_proof.py new file mode 100644 index 0000000..19e4982 --- /dev/null +++ b/tests/test_collect_leo_restart_survival_proof.py @@ -0,0 +1,120 @@ +"""Tests for scripts/collect_leo_restart_survival_proof.py.""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(REPO_ROOT / "scripts")) + +import collect_leo_restart_survival_proof as restart_proof # noqa: E402 + + +COUNTS = { + "public.claims": 1837, + "public.sources": 4145, + "public.claim_evidence": 4670, + "public.claim_edges": 4916, + "kb_stage.kb_proposals": 26, +} + + +def args(tmp_path: Path, *, execute_restart: bool = True, skip_handler: bool = False) -> argparse.Namespace: + return argparse.Namespace( + execute_restart=execute_restart, + skip_handler=skip_handler, + out=tmp_path / "restart.json", + markdown_out=tmp_path / "restart.md", + ssh_target="root@example.test", + ssh_key=tmp_path / "key", + connect_timeout=1, + timeout=5, + active_timeout=5, + poll_interval=0, + handler_timeout=5, + handler_turn_timeout=300, + remote_handler_report="/tmp/leo-restart-survival-direct-claim-current.json", + ) + + +def readback(label: str, pid: str) -> dict: + return { + "label": label, + "remote_repo": "/opt/teleo-eval/workspaces/deploy-infra", + "deploy_head": "abc123", + "last_deploy_sha": "abc123", + "git_status_short": "", + "service": { + "returncode": 0, + "stderr": "", + "state": { + "ActiveState": "active", + "SubState": "running", + "MainPID": pid, + "NRestarts": "0", + "ExecMainStartTimestamp": "Fri 2026-07-10 01:04:22 UTC", + }, + }, + "db_counts": {"returncode": 0, "stderr": "", "counts": COUNTS, "raw": json.dumps(COUNTS)}, + } + + +def fake_runner(command: list[str], input_text: str, timeout: int) -> restart_proof.CommandResult: + remote = command[-1] + if "before_restart" in remote: + return restart_proof.CommandResult(0, json.dumps(readback("before_restart", "100")) + "\n", "") + if remote == "systemctl restart leoclean-gateway.service": + return restart_proof.CommandResult(0, "", "") + if "after_restart_poll" in remote: + return restart_proof.CommandResult(0, json.dumps(readback("after_restart_poll", "200")) + "\n", "") + if "run_leo_direct_claim_handler_remote.py" in remote: + return restart_proof.CommandResult( + 0, + json.dumps( + { + "pass_runtime": True, + "posted_to_telegram": False, + "mutates_kb_by_harness": False, + "changed_live_profile": False, + "temp_profile_removed": True, + "results": [{"prompt_id": "DC-01", "ok": True, "reply": "ok"}], + } + ) + + "\n", + "", + ) + raise AssertionError(remote) + + +def test_collect_restart_survival_passes_with_unchanged_counts_and_handler(tmp_path: Path): + report = restart_proof.collect(args(tmp_path), runner=fake_runner) + + assert report["status"] == "pass" + assert report["survives_restart"] is True + assert report["db_counts_changed"] is False + assert report["posted_to_telegram"] is False + assert report["production_db_apply_ran"] is False + assert report["handler"]["pass_runtime"] is True + assert report["before"]["service"]["state"]["MainPID"] == "100" + assert report["after"]["service"]["state"]["MainPID"] == "200" + + +def test_collect_refuses_to_claim_restart_without_execute_flag(tmp_path: Path): + report = restart_proof.collect(args(tmp_path, execute_restart=False), runner=fake_runner) + + assert report["status"] == "not_executed_missing_execute_restart" + assert report["survives_restart"] is False + assert "before" in report + + +def test_markdown_report_names_safety_boundaries(tmp_path: Path): + report = restart_proof.collect(args(tmp_path), runner=fake_runner) + + text = restart_proof.markdown_report(report) + + assert "Survives restart: `True`" in text + assert "Posted to Telegram: `False`" in text + assert "Production DB apply ran: `False`" in text diff --git a/tests/test_hermes_leoclean_skill_surfaces.py b/tests/test_hermes_leoclean_skill_surfaces.py index 728f91f..f67afa5 100644 --- a/tests/test_hermes_leoclean_skill_surfaces.py +++ b/tests/test_hermes_leoclean_skill_surfaces.py @@ -21,6 +21,8 @@ def test_gcp_kb_skill_uses_cloudsql_bridge_not_vps_docker() -> None: assert "not just pointer mismatch" in text assert "staging write to `kb_stage.kb_proposals`" in text assert "canonical identity requires DB rows plus" in text + assert "Row-level proof would require current readback" in text + assert "new or updated row IDs" in text assert "I cannot claim canonical DB changed" in text assert "explicit operator/admin authorization" in text assert "docker exec" not in text @@ -49,6 +51,8 @@ def test_vps_kb_skill_keeps_vps_scope_explicit() -> None: assert "not just pointer mismatch" in text assert "staging write to `kb_stage.kb_proposals`" in text assert "canonical identity requires DB rows plus" in text + assert "Row-level proof would require current readback" in text + assert "new or updated row IDs" in text assert "I cannot claim canonical DB changed" in text assert "explicit operator/admin authorization" in text assert "Do not call an approved proposal \"implemented\"" in text