From be23b5e95b9fd541f213ebc4f0c5b838d4712605 Mon Sep 17 00:00:00 2001 From: twentyOne2x Date: Mon, 13 Jul 2026 07:13:08 +0200 Subject: [PATCH] Add Cory-style Leo out-of-sample proof harness --- ...restart-survival-direct-claim-current.json | 148 ++++++++- ...t-survival-direct-claim-score-current.json | 84 +++-- ...art-survival-direct-claim-score-current.md | 16 +- .../leo-restart-survival-proof-current.json | 300 +++++++++++++----- .../leo-restart-survival-proof-current.md | 60 +--- ...legram-handler-cory-oos-suite-current.json | 176 ++++++++++ ...-handler-cory-oos-suite-score-current.json | 253 +++++++++++++++ ...am-handler-cory-oos-suite-score-current.md | 24 ++ ...dler-count-receipt-correction-current.json | 100 ++++++ ...andler-count-receipt-correction-current.md | 23 ++ ...isible-direct-claim-preflight-current.json | 14 +- ...-visible-direct-claim-preflight-current.md | 12 +- scripts/run_leo_cory_oos_handler_suite.py | 133 ++++++++ ..._leo_count_receipt_correction_challenge.py | 167 ++++++++++ scripts/run_leo_direct_claim_handler_suite.py | 65 +++- scripts/working_leo_cory_oos_benchmark.py | 297 +++++++++++++++++ tests/test_working_leo_cory_oos_benchmark.py | 155 +++++++++ 17 files changed, 1828 insertions(+), 199 deletions(-) create mode 100644 docs/reports/leo-working-state-20260709/telegram-handler-cory-oos-suite-current.json create mode 100644 docs/reports/leo-working-state-20260709/telegram-handler-cory-oos-suite-score-current.json create mode 100644 docs/reports/leo-working-state-20260709/telegram-handler-cory-oos-suite-score-current.md create mode 100644 docs/reports/leo-working-state-20260709/telegram-handler-count-receipt-correction-current.json create mode 100644 docs/reports/leo-working-state-20260709/telegram-handler-count-receipt-correction-current.md create mode 100755 scripts/run_leo_cory_oos_handler_suite.py create mode 100755 scripts/run_leo_count_receipt_correction_challenge.py create mode 100755 scripts/working_leo_cory_oos_benchmark.py create mode 100644 tests/test_working_leo_cory_oos_benchmark.py diff --git a/docs/reports/leo-working-state-20260709/leo-restart-survival-direct-claim-current.json b/docs/reports/leo-working-state-20260709/leo-restart-survival-direct-claim-current.json index b4658ba..7090b41 100644 --- a/docs/reports/leo-working-state-20260709/leo-restart-survival-direct-claim-current.json +++ b/docs/reports/leo-working-state-20260709/leo-restart-survival-direct-claim-current.json @@ -1 +1,147 @@ -{"before_service": {"ActiveState": "active", "ExecMainStartTimestamp": "Fri 2026-07-10 03:24:43 UTC", "MainPID": "2845730", "NRestarts": "0", "SubState": "running"}, "changed_live_profile": false, "db_counts_after": {"kb_stage.kb_proposals": 26, "public.claim_edges": 4916, "public.claim_evidence": 4670, "public.claims": 1837, "public.sources": 4145}, "db_counts_before": {"kb_stage.kb_proposals": 26, "public.claim_edges": 4916, "public.claim_evidence": 4670, "public.claims": 1837, "public.sources": 4145}, "db_counts_changed": false, "generated_at_utc": "2026-07-10T03:24:43.960176+00:00", "handler": {"authorized": true, "session_key": "agent:main:telegram:group:-5146042086:9070919", "temp_profile": "/tmp/leo-direct-claim-handler-z8ezwre7/profile"}, "harness_notes": ["Prompts are the exact DC-01..DC-06 direct-claim benchmark messages.", "The harness uses a temporary copy of the leoclean profile and does not post to Telegram.", "This proves handler-level live VPS GatewayRunner behavior, not human-visible Telegram delivery."], "mode": "live_vps_gatewayrunner_temp_profile_direct_claim_suite", "mutates_kb_by_harness": false, "pass_runtime": true, "posted_to_telegram": false, "remote_report_path": "/tmp/leo-restart-survival-direct-claim-current.json", "results": [{"claim_ceiling": "Live VPS GatewayRunner reply from temp leoclean profile; no Telegram post; no production DB apply authorized.", "dimension": "db_changed_direct_claim", "ended_at_utc": "2026-07-10T03:25:14.440750+00:00", "evidence_tier": "live_vps_gatewayrunner_temp_profile", "mutates_kb": false, "ok": true, "prompt": "Did we actually update the knowledge base, or is it still just proposals?", "prompt_id": "DC-01", "reply": "Mostly still proposals. Here's the breakdown from current readback:\n\nApplied to canonical public.* (2 rows):\n- `00957f6c` \u2014 add_edge canary, applied 2026-07-09\n- `f004bbb2` \u2014 attach_evidence (Leo guiding-policy revision), applied 2026-07-05\n\nApproved but NOT applied \u2014 applied_at empty (3 rows):\n- `14fa5ecc` \u2014 attach_evidence, internet-finance claim split packet\n- `ac036c9d` \u2014 attach_evidence, strategy-kernel / claim hygiene\n- `a64df080` \u2014 attach_evidence, Helmer 7 Powers framework (approved 2026-06-29, still not canonical)\n\nPending review (8+):\n`8dfedb3f`, `03f7853e`, `e60a48af`, `f5036882`, `bcec8528`, `e987b07c`, `23e896c2`, `124942fd`, `a59311ae`, `c824da66` \u2014 all pending, none reviewed.\n\nCanceled: several older Leo strategy-kernel and concept-map attempts.\n\nI cannot claim canonical DB changed beyond those 2 applied rows until public.* readback plus applied_at/postflight proof says otherwise. The 3 approved proposals \u2014 including the Helmer 7 Powers packet (`a64df080`) \u2014 are staged/packet-ready but not canonical.\n\nNext Cory-style follow-up: apply the 3 approved proposals (`14fa5ecc`, `ac036c9d`, `a64df080`) through the admin review/apply path with before/after readback \u2014 that would move them from approved to canonical. Apply requires explicit operator/admin authorization and should not be run from chat without that.", "started_at_utc": "2026-07-10T03:24:44.408649+00:00", "turn": 1}, {"claim_ceiling": "Live VPS GatewayRunner reply from temp leoclean profile; no Telegram post; no production DB apply authorized.", "dimension": "helmer_live_direct_claim", "ended_at_utc": "2026-07-10T03:25:38.133095+00:00", "evidence_tier": "live_vps_gatewayrunner_temp_profile", "mutates_kb": false, "ok": true, "prompt": "Is Helmer's 7 Powers in Leo now?", "prompt_id": "DC-02", "reply": "No. Not canonical.\n\nClaim: canonical `public.*` search returns zero matching claims for \"7 powers helmer competitive.\" Nothing landed in the DB.\n\nWhat exists:\n- Proposal `a64df080` \u2014 status `approved`, reviewed 2026-06-29, `applied_at` empty. It's the revised Helmer 7 Powers packet (self-positioning claim removed, competitive-advantage/moats neighborhood links added per m3ta approval-with-changes).\n- Proposal `cea989fc` \u2014 earlier version, status `canceled`.\n- No `public.sources`, `public.claims`, evidence rows, edges, or reasoning-tool node for 7 Powers exist.\n\nSo: approved/staged, not canonical.\n\nNext Cory-style follow-up: apply proposal `a64df080` through the admin review/apply path \u2014 that would create the source, framework node, power-definition claims, evidence links, and edges in `public.*`. Requires explicit operator/admin authorization; not runnable from chat.", "started_at_utc": "2026-07-10T03:25:14.441246+00:00", "turn": 2}, {"claim_ceiling": "Live VPS GatewayRunner reply from temp leoclean profile; no Telegram post; no production DB apply authorized.", "dimension": "decision_matrix_direct_claim", "ended_at_utc": "2026-07-10T03:25:48.290711+00:00", "evidence_tier": "live_vps_gatewayrunner_temp_profile", "mutates_kb": false, "ok": true, "prompt": "Did the decision matrix approve this already?", "prompt_id": "DC-03", "reply": "No. The decision-matrix approval path is not shipped.\n\nThe tables that would record a matrix vote \u2014 `matrix_voters`, `proposal_votes`, `proposal_decisions` \u2014 do not exist in either `kb_stage` or `public`. There is no vote record anywhere in the DB.\n\nWhat the `approved` status on `a64df080` actually means: m3ta reviewed and approved it manually in chat on 2026-06-29. That is reviewer approval, not a futarchy/decision-matrix vote. The two are different things.\n\nSo the chain is:\n- Reviewer approval: yes (m3ta, 2026-06-29)\n- Decision-matrix vote: not applicable \u2014 the mechanism isn't built yet\n- Applied to canonical public.*: no\n\nNext Cory-style follow-up: the proof-changing action is applying `a64df080` via the admin apply path with postflight readback \u2014 the decision-matrix gate is a future requirement, not a current blocker. Requires explicit operator/admin authorization.", "started_at_utc": "2026-07-10T03:25:38.133391+00:00", "turn": 3}, {"claim_ceiling": "Live VPS GatewayRunner reply from temp leoclean profile; no Telegram post; no production DB apply authorized.", "dimension": "document_source_direct_claim", "ended_at_utc": "2026-07-10T03:26:09.600395+00:00", "evidence_tier": "live_vps_gatewayrunner_temp_profile", "mutates_kb": false, "ok": true, "prompt": "Are the pending proposals stuck because the documents are not pointed at the right source rows?", "prompt_id": "DC-04", "reply": "Not just pointer mismatch. The stuck proposals span several different layers.\n\nWhat the readback actually shows:\n\nApproved but unapplied (3):\n`a64df080`, `ac036c9d`, `14fa5ecc` \u2014 these are not stuck on document pointer problems. They are approved and waiting for an operator/admin to run the apply sequence with postflight readback. The blocker is authorization and tooling, not source-row linkage.\n\nPending review (14):\nThese proposals are sitting in `pending_review` \u2014 no reviewer has actioned them. The causes vary:\n- `f5036882` \u2014 strategy-kernel doc evaluation, never reviewed\n- `bcec8528` \u2014 Declaration of Intelligence doc, never reviewed\n- `e987b07c` \u2014 Good Strategy/Bad Strategy evidence attach, never reviewed\n- `da53f198` \u2014 Rumelt strategy-kernel primary source grounding, never reviewed\n- `23e896c2` \u2014 living-system concept map revision, never reviewed\n- Several others (Markov blanket, Hayek papers, add_edge canaries) \u2014 same status\n\nThe missing proof for your question would be a row-link audit: inspecting each proposal's `source_ref` and `apply_payload` against actual `public.sources` rows to see whether pointer mismatches exist. The current bridge doesn't surface that automatically.\n\nSo the single-cause \"wrong source rows\" answer is no. The real blockers are two separate things:\n1. Approved proposals need an operator apply run\n2. Pending proposals need reviewer attention \u2014 they haven't been reviewed at all yet\n\nNext Cory-style follow-up: a row-link audit of the pending proposals' `apply_payload` vs `public.sources` would answer whether source-pointer mismatch is a secondary blocker \u2014 but the primary one is reviewer queue, not pointer hygiene.", "started_at_utc": "2026-07-10T03:25:48.291387+00:00", "turn": 4}, {"claim_ceiling": "Live VPS GatewayRunner reply from temp leoclean profile; no Telegram post; no production DB apply authorized.", "dimension": "demo_changes_kb_direct_claim", "ended_at_utc": "2026-07-10T03:26:20.605728+00:00", "evidence_tier": "live_vps_gatewayrunner_temp_profile", "mutates_kb": false, "ok": true, "prompt": "Can I demo that Leo changes the KB?", "prompt_id": "DC-05", "reply": "Yes, but two different tiers \u2014 pick which one you want to show.\n\nTier 1 \u2014 Safe demo, runnable now:\nLeo stages a real write to `kb_stage.kb_proposals` and you read it back immediately. The proposal lands in the DB with a real UUID, timestamp, rationale, and status. We've done this \u2014 it's what the canary proposals prove. No canonical mutation, no authorization required.\n\nTier 2 \u2014 Canonical demo, requires authorization:\nLeo (or an admin apply tool) mutates `public.claims`, `public.sources`, `public.claim_evidence`, or `public.claim_edges`. This is what \"KB changed\" actually means in the product sense. Requires explicit operator/admin authorization plus an apply/postflight sequence with before/after readback. Not runnable from chat without that.\n\nWhat you cannot demo without Tier 2: showing that a claim, source, or edge now exists in canonical search results. You can show a staged proposal exists; you cannot show it landed in `public.*` until it's applied.\n\nNext Cory-style follow-up: if you want to run a Tier 1 demo now, say the word and I'll stage a test proposal and read it back live. If you want Tier 2, grant explicit operator authorization and I'll walk through the apply sequence with postflight proof.", "started_at_utc": "2026-07-10T03:26:09.600850+00:00", "turn": 5}, {"claim_ceiling": "Live VPS GatewayRunner reply from temp leoclean profile; no Telegram post; no production DB apply authorized.", "dimension": "soul_identity_direct_claim", "ended_at_utc": "2026-07-10T03:26:31.720585+00:00", "evidence_tier": "live_vps_gatewayrunner_temp_profile", "mutates_kb": false, "ok": true, "prompt": "If we changed SOUL.md, did we change Leo's canonical identity?", "prompt_id": "DC-06", "reply": "No.\n\nSOUL.md is a runtime/rendered artifact. It's what gets injected into the session context \u2014 but it does not write back to Postgres. Editing the file changes what Leo sees at the start of the next session; it does not change canonical identity in the DB.\n\nCanonical identity lives in the DB rows:\n- `public.claims` \u2014 beliefs, positions, strategy nodes\n- `public.sources` \u2014 grounding evidence\n- `public.claim_evidence` and `public.claim_edges` \u2014 the graph that connects them\n\nA SOUL.md edit with no corresponding DB proposal \u2192 review \u2192 apply sequence means:\n- Next session: Leo reads the edited file and behaves differently\n- Canonical KB: unchanged\n- On next render from DB: the edit gets overwritten if the renderer pulls from Postgres\n\nSo a SOUL.md edit is ephemeral runtime continuity, not durable collective knowledge. If the correction matters to the collective \u2014 a belief revision, a strategy change, a new position \u2014 it belongs in the KB graph with a proposal, review, and apply, not just a file edit.\n\nNext Cory-style follow-up: if there are SOUL.md edits that represent real identity changes, the proof-changing action is staging them as KB proposals (beliefs, claims, or strategy nodes) so they survive a DB render cycle and become canonical.", "started_at_utc": "2026-07-10T03:26:20.606393+00:00", "turn": 6}], "service_before_after": {"after": {"ActiveState": "active", "ExecMainStartTimestamp": "Fri 2026-07-10 03:24:43 UTC", "MainPID": "2845730", "NRestarts": "0", "SubState": "running"}, "before": {"ActiveState": "active", "ExecMainStartTimestamp": "Fri 2026-07-10 03:24:43 UTC", "MainPID": "2845730", "NRestarts": "0", "SubState": "running"}, "unchanged_from_preexisting_live_readback": true}, "source": {"chat_id": "-5146042086", "chat_name": "Leo", "chat_type": "group", "platform": "telegram", "user_id": "9070919", "user_name": "codex handler direct claim"}, "temp_profile_removed": true} \ No newline at end of file +{ + "before_service": { + "ActiveState": "active", + "ExecMainStartTimestamp": "Mon 2026-07-13 04:26:31 UTC", + "MainPID": "1105322", + "NRestarts": "0", + "SubState": "running" + }, + "changed_live_profile": false, + "db_counts_after": { + "kb_stage.kb_proposals": 26, + "public.claim_edges": 4916, + "public.claim_evidence": 4670, + "public.claims": 1837, + "public.sources": 4145 + }, + "db_counts_before": { + "kb_stage.kb_proposals": 26, + "public.claim_edges": 4916, + "public.claim_evidence": 4670, + "public.claims": 1837, + "public.sources": 4145 + }, + "db_counts_changed": false, + "generated_at_utc": "2026-07-13T04:26:38.732210+00:00", + "handler": { + "authorized": true, + "session_key": "agent:main:telegram:group:-5146042086:9070919", + "temp_profile": "/tmp/leo-direct-claim-handler-0cohbh97/profile" + }, + "harness_notes": [ + "Prompts are the exact DC-01..DC-06 direct-claim benchmark messages.", + "The harness uses a temporary copy of the leoclean profile and does not post to Telegram.", + "This proves handler-level live VPS GatewayRunner behavior, not human-visible Telegram delivery." + ], + "mode": "live_vps_gatewayrunner_temp_profile_direct_claim_suite", + "mutates_kb_by_harness": false, + "pass_runtime": true, + "posted_to_telegram": false, + "remote_report_path": "/tmp/leo-restart-survival-direct-claim-current.json", + "results": [ + { + "claim_ceiling": "Live VPS GatewayRunner reply from temp leoclean profile; no Telegram post; no production DB apply authorized.", + "dimension": "db_changed_direct_claim", + "ended_at_utc": "2026-07-13T04:27:04.428288+00:00", + "evidence_tier": "live_vps_gatewayrunner_temp_profile", + "mutates_kb": false, + "ok": true, + "prompt": "Did we actually update the knowledge base, or is it still just proposals?", + "prompt_id": "DC-01", + "reply": "Mostly still proposals. Here's the breakdown:\n\nDB readback: claims: `1837`; sources: `4145`; claim_edges: `4916`; claim_evidence: `4670`; kb_proposals: `26`.\n\nApplied (canonical): 2\n- `00957f6c` \u2014 `add_edge` canary (strict path, applied 2026-07-09)\n- `f004bbb2` \u2014 `attach_evidence` for Leo guiding-policy revision (applied 2026-07-05)\n\nApproved but NOT applied: 3\n- `14fa5ecc` \u2014 split compressed internet-finance/capital-allocation claim; applied_at: none\n- `ac036c9d` \u2014 strategy-kernel document / claim-node hygiene; applied_at: none\n- `a64df080` \u2014 Helmer 7 Powers framework packet; applied_at: none\n\nPending review: 14 proposals (ranging from Hayek/Friston evidence to Living Systems, Rumelt, canary edges)\n\nCanceled: 7 (superseded drafts)\n\nApproved is not the same as applied. The 3 approved proposals \u2014 including the 7 Powers packet \u2014 have no `applied_at`, meaning they are staged/packet-ready but not canonical. The canonical `public.*` tables did not change for those.\n\nNext Cory-style follow-up: The proof-changing action for the 3 approved proposals is validating or rebuilding their strict `apply_payload`, then running the apply path with retained before/after postflight readback \u2014 that requires explicit operator/admin authorization and should not be run from normal chat.", + "started_at_utc": "2026-07-13T04:26:39.479688+00:00", + "turn": 1 + }, + { + "claim_ceiling": "Live VPS GatewayRunner reply from temp leoclean profile; no Telegram post; no production DB apply authorized.", + "dimension": "helmer_live_direct_claim", + "ended_at_utc": "2026-07-13T04:27:27.245582+00:00", + "evidence_tier": "live_vps_gatewayrunner_temp_profile", + "mutates_kb": false, + "ok": true, + "prompt": "Is Helmer's 7 Powers in Leo now?", + "prompt_id": "DC-02", + "reply": "No, not canonical.\n\nDB readback: proposal: `a64df080-8502-42e2-98f4-9bbdecb8da73`; status: `approved`; applied_at: `none` (database NULL).\n\nThe search for \"7 Powers helmer\" and \"seven powers\" returned zero matching canonical claims. The Helmer reasoning tool, 5 proposed claims (benefit/barrier logic, power progression, counter-positioning, network economies vs network effects, agentic capital power analysis), and all 6\u20138 source candidates exist only inside the proposal payload \u2014 not in `public.claims`, `public.sources`, `public.claim_evidence`, `public.claim_edges`, or any reasoning-tool rows.\n\nWhat exists:\n- `kb_stage.kb_proposals` \u2014 proposal `a64df080` approved with changes on 2026-06-29, `applied_at: NULL`\n- Local research files at `/home/teleo/.hermes/profiles/leoclean/kb_stage/research/7powers/`\n\nWhat does not exist:\n- No canonical `public.claims` row for any 7 Powers proposed claim\n- No `public.sources` row for Helmer book, Acquired transcript, NFX interview\n- No `public.claim_evidence` or `public.claim_edges` rows linking to those claims\n- No reasoning_tool node for \"Helmer 7 Powers\"\n\nApproved is not the same as applied. The proposal is packet-ready but the `apply_payload` is a rich `create_reasoning_tool_and_claim_set` bundle that the bridge has no native `apply` command for \u2014 it would require operator-level validation and execution.\n\nNext Cory-style follow-up: Validate or rebuild a strict `apply_payload` (source rows + claim rows + evidence rows + edges) for `a64df080`, then run the apply path with retained before/after postflight readback \u2014 requires explicit operator/admin authorization, not a normal chat command.", + "started_at_utc": "2026-07-13T04:27:04.428643+00:00", + "turn": 2 + }, + { + "claim_ceiling": "Live VPS GatewayRunner reply from temp leoclean profile; no Telegram post; no production DB apply authorized.", + "dimension": "decision_matrix_direct_claim", + "ended_at_utc": "2026-07-13T04:27:41.751314+00:00", + "evidence_tier": "live_vps_gatewayrunner_temp_profile", + "mutates_kb": false, + "ok": true, + "prompt": "Did the decision matrix approve this already?", + "prompt_id": "DC-03", + "reply": "No. Fresh readback: the decision-matrix schema is absent; reviewer status is not a decision-matrix vote.\n\nDB readback: claims: `1837`; sources: `4145`; claim_edges: `4916`; claim_evidence: `4670`; kb_proposals: `26`.\n\nNone of the decision-matrix tables exist in any schema:\n\n- `kb_stage.matrix_voters` \u2014 absent\n- `kb_stage.proposal_votes` \u2014 absent\n- `kb_stage.proposal_decisions` \u2014 absent\n- `public.*` equivalents \u2014 all absent\n\nThe `approved` status on proposal `a64df080` reflects m3ta's direct reviewer approval on 2026-06-29, not a futarchy/decision-matrix vote. Those are two different paths. The matrix path has not shipped.\n\nNext Cory-style follow-up: If you want the decision-matrix path to count, the schema (`matrix_voters`, `proposal_votes`, `proposal_decisions`) needs to be deployed \u2014 that's a separate infrastructure step before any proposal can claim matrix approval.", + "started_at_utc": "2026-07-13T04:27:27.246033+00:00", + "turn": 3 + }, + { + "claim_ceiling": "Live VPS GatewayRunner reply from temp leoclean profile; no Telegram post; no production DB apply authorized.", + "dimension": "document_source_direct_claim", + "ended_at_utc": "2026-07-13T04:28:15.355617+00:00", + "evidence_tier": "live_vps_gatewayrunner_temp_profile", + "mutates_kb": false, + "ok": true, + "prompt": "Are the pending proposals stuck because the documents are not pointed at the right source rows?", + "prompt_id": "DC-04", + "reply": "Not just pointer mismatch. The 14 pending proposals have at least three distinct blockers, and source-row pointer issues are only one layer of one of them.\n\nDB readback: claims: `1837`; sources: `4145`; claim_edges: `4916`; claim_evidence: `4670`; kb_proposals: `26`.\n\nBreaking it down by type:\n\n`add_edge` canaries (4 proposals: `8dfedb3f`, `03f7853e`, `e60a48af`, `c824da66`)\nThese reference existing canonical claim IDs by UUID. The strict `add_edge` apply path is live-proven. They are not stuck on pointer mismatch \u2014 they are stuck on missing operator review and authorization. These could be applied now if a reviewer approves them.\n\n`attach_evidence` proposals (10 proposals: `f5036882`, `bcec8528`, `e987b07c`, `23e896c2`, `da53f198`, `e106ca60`, `e60fa116`, `416385fe`, `124942fd`, `a59311ae`)\nThese carry document grounding work \u2014 raw Telegram-ingested files, KB-stage research files, logical source keys \u2014 that need to land as `public.sources` rows before evidence links can be written. The layers are distinct:\n- raw local/Telegram file references\n- `kb_stage` document evaluations\n- proposal `source_ref` / logical source keys\n- canonical `public.sources` rows\n\nThe missing proof is a row-link audit: do the logical source keys in each proposal payload already have matching `public.sources` rows? And is there a live apply path for `attach_evidence` that writes those rows? The answer to the second question is no \u2014 only the strict `add_edge` apply path is live-proven. The `attach_evidence` apply worker is not live.\n\nSo the blockers are:\n1. No live `attach_evidence` apply path (all 10 evidence proposals)\n2. No reviewer decision yet (all 14 pending)\n3. For source-heavy proposals, possibly missing `public.sources` rows \u2014 but that cannot be confirmed without a row-link audit comparing proposal `source_ref` keys against existing canonical `public.sources` UUIDs\n\nNext Cory-style follow-up: The proof-changing action is a row-link audit on the `attach_evidence` proposals \u2014 check whether their logical source keys already exist as `public.sources` rows, then assess whether to rebuild as strict payloads or wait for the apply worker to ship before authorizing.", + "started_at_utc": "2026-07-13T04:27:41.751922+00:00", + "turn": 4 + }, + { + "claim_ceiling": "Live VPS GatewayRunner reply from temp leoclean profile; no Telegram post; no production DB apply authorized.", + "dimension": "demo_changes_kb_direct_claim", + "ended_at_utc": "2026-07-13T04:28:40.092234+00:00", + "evidence_tier": "live_vps_gatewayrunner_temp_profile", + "mutates_kb": false, + "ok": true, + "prompt": "Can I demo that Leo changes the KB?", + "prompt_id": "DC-05", + "reply": "Staging yes, canonical KB change not safe to demo from chat.\n\nDB readback: claims: `1837`; sources: `4145`; claim_edges: `4916`; claim_evidence: `4670`; kb_proposals: `26`.\n\nApproved is not the same as applied.\n\nDemo tiers, from safest to most consequential:\n\nTier 1 \u2014 Staging write (safe now, no authorization needed)\nCreate a new canary `add_edge` proposal, write it to `kb_stage.kb_proposals`, read it back. That proves the Telegram \u2192 Leo \u2192 DB staging path end-to-end. No canonical tables touched.\n\nTier 2 \u2014 Strict `add_edge` apply (live-proven path, needs operator authorization)\nThe 4 pending `add_edge` canaries (`8dfedb3f`, `03f7853e`, `e60a48af`, `c824da66`) reference existing canonical claim IDs. The strict apply path for `add_edge` is live-proven. A reviewer could authorize one of these and it would actually write a new `public.claim_edges` row \u2014 that is a real canonical change. Requires explicit operator/admin authorization before running from chat.\n\nTier 3 \u2014 `attach_evidence` or rich multi-row packets (not ready)\nThe `attach_evidence` apply worker is not live. No rich packet (like 7 Powers) has a matching strict `apply_payload`. These cannot be cleanly demoed without rebuilding the payload and shipping the apply worker first.\n\nCurrent tier ceiling in plain terms:\n- Strict `add_edge` path: live-proven\n- Guarded `approve_claim` bundles: clone-proven, behind separate reviewer/apply roles\n- Production permission migration and apply worker: disabled\n- Approved legacy packets without strict `apply_payload`: not worker-applyable\n\nIf you want to demo canonical change, the safest real demo is a new staging canary followed by a reviewed strict `add_edge` apply with retained before/after postflight readback. That requires your explicit authorization \u2014 not something Leo runs from normal chat on its own.\n\nNext Cory-style follow-up: Say the word and I can stage a fresh `add_edge` canary right now (Tier 1, safe) \u2014 or if you want to authorize a Tier 2 canonical apply on one of the existing pending canaries, name which one and I'll prepare the apply packet for your review.", + "started_at_utc": "2026-07-13T04:28:15.356361+00:00", + "turn": 5 + }, + { + "claim_ceiling": "Live VPS GatewayRunner reply from temp leoclean profile; no Telegram post; no production DB apply authorized.", + "dimension": "soul_identity_direct_claim", + "ended_at_utc": "2026-07-13T04:28:51.365739+00:00", + "evidence_tier": "live_vps_gatewayrunner_temp_profile", + "mutates_kb": false, + "ok": true, + "prompt": "If we changed SOUL.md, did we change Leo's canonical identity?", + "prompt_id": "DC-06", + "reply": "No.\n\nDB readback: claims: `1837`; sources: `4145`; claim_edges: `4916`; claim_evidence: `4670`; kb_proposals: `26`.\n\n`SOUL.md` is a runtime/rendered artifact \u2014 not canonical Postgres, not the source of truth, not a canonical commit, and not collective truth. Editing it affects the next runtime session's context injection, but it does not touch any DB rows.\n\nRow-level proof would require current readback of new or updated row IDs in `public.claims`, `public.sources`, `public.claim_evidence`, `public.claim_edges`, or identity tables, plus postflight/render-sync proof. Without those rows, canonical identity is unchanged.\n\nThe render direction is: Postgres KB \u2192 SOUL.md, not the reverse. Writing to SOUL.md directly is editing the output, not the source. A change to Leo's canonical identity requires:\n1. A proposal staged to `kb_stage.kb_proposals`\n2. Reviewer approval\n3. Apply to `public.*` canonical tables\n4. Render/sync that regenerates SOUL.md from the updated DB\n\nWhat SOUL.md edits do buy: the next Leo session picks up the changed runtime context immediately. That has real operational effect. It just isn't canonical truth, and it drifts back toward the DB on the next render/sync.\n\nNext Cory-style follow-up: If a specific SOUL.md change should become canonical, stage it as a proposal now \u2014 I can draft the KB proposal from the diff and route it for review.", + "started_at_utc": "2026-07-13T04:28:40.092648+00:00", + "turn": 6 + } + ], + "service_before_after": { + "after": { + "ActiveState": "active", + "ExecMainStartTimestamp": "Mon 2026-07-13 04:26:31 UTC", + "MainPID": "1105322", + "NRestarts": "0", + "SubState": "running" + }, + "before": { + "ActiveState": "active", + "ExecMainStartTimestamp": "Mon 2026-07-13 04:26:31 UTC", + "MainPID": "1105322", + "NRestarts": "0", + "SubState": "running" + }, + "unchanged_from_preexisting_live_readback": true + }, + "source": { + "chat_id": "-5146042086", + "chat_name": "Leo", + "chat_type": "group", + "platform": "telegram", + "user_id": "9070919", + "user_name": "codex handler direct claim" + }, + "temp_profile_removed": true +} diff --git a/docs/reports/leo-working-state-20260709/leo-restart-survival-direct-claim-score-current.json b/docs/reports/leo-working-state-20260709/leo-restart-survival-direct-claim-score-current.json index 985d56a..4feb66f 100644 --- a/docs/reports/leo-working-state-20260709/leo-restart-survival-direct-claim-score-current.json +++ b/docs/reports/leo-working-state-20260709/leo-restart-survival-direct-claim-score-current.json @@ -21,7 +21,7 @@ "runtime": "disposable_clone_or_sandbox_first" } ], - "generated_at_utc": "2026-07-10T03:28:12.786249+00:00", + "generated_at_utc": "2026-07-13T04:33:26.008200+00:00", "include_cory_style_scenarios": false, "include_direct_claim_followups": true, "mode": "retained_evidence_score", @@ -31,79 +31,79 @@ { "claim_ceiling": "Live VPS GatewayRunner reply from temp leoclean profile; no Telegram post; no production DB apply authorized.", "dimension": "db_changed_direct_claim", - "ended_at_utc": "2026-07-10T03:25:14.440750+00:00", + "ended_at_utc": "2026-07-13T04:27:04.428288+00:00", "evidence_tier": "live_vps_gatewayrunner_temp_profile", "mutates_kb": false, "ok": true, "prompt": "Did we actually update the knowledge base, or is it still just proposals?", "prompt_id": "DC-01", - "reply": "Mostly still proposals. Here's the breakdown from current readback:\n\nApplied to canonical public.* (2 rows):\n- `00957f6c` \u2014 add_edge canary, applied 2026-07-09\n- `f004bbb2` \u2014 attach_evidence (Leo guiding-policy revision), applied 2026-07-05\n\nApproved but NOT applied \u2014 applied_at empty (3 rows):\n- `14fa5ecc` \u2014 attach_evidence, internet-finance claim split packet\n- `ac036c9d` \u2014 attach_evidence, strategy-kernel / claim hygiene\n- `a64df080` \u2014 attach_evidence, Helmer 7 Powers framework (approved 2026-06-29, still not canonical)\n\nPending review (8+):\n`8dfedb3f`, `03f7853e`, `e60a48af`, `f5036882`, `bcec8528`, `e987b07c`, `23e896c2`, `124942fd`, `a59311ae`, `c824da66` \u2014 all pending, none reviewed.\n\nCanceled: several older Leo strategy-kernel and concept-map attempts.\n\nI cannot claim canonical DB changed beyond those 2 applied rows until public.* readback plus applied_at/postflight proof says otherwise. The 3 approved proposals \u2014 including the Helmer 7 Powers packet (`a64df080`) \u2014 are staged/packet-ready but not canonical.\n\nNext Cory-style follow-up: apply the 3 approved proposals (`14fa5ecc`, `ac036c9d`, `a64df080`) through the admin review/apply path with before/after readback \u2014 that would move them from approved to canonical. Apply requires explicit operator/admin authorization and should not be run from chat without that.", - "started_at_utc": "2026-07-10T03:24:44.408649+00:00", + "reply": "Mostly still proposals. Here's the breakdown:\n\nDB readback: claims: `1837`; sources: `4145`; claim_edges: `4916`; claim_evidence: `4670`; kb_proposals: `26`.\n\nApplied (canonical): 2\n- `00957f6c` \u2014 `add_edge` canary (strict path, applied 2026-07-09)\n- `f004bbb2` \u2014 `attach_evidence` for Leo guiding-policy revision (applied 2026-07-05)\n\nApproved but NOT applied: 3\n- `14fa5ecc` \u2014 split compressed internet-finance/capital-allocation claim; applied_at: none\n- `ac036c9d` \u2014 strategy-kernel document / claim-node hygiene; applied_at: none\n- `a64df080` \u2014 Helmer 7 Powers framework packet; applied_at: none\n\nPending review: 14 proposals (ranging from Hayek/Friston evidence to Living Systems, Rumelt, canary edges)\n\nCanceled: 7 (superseded drafts)\n\nApproved is not the same as applied. The 3 approved proposals \u2014 including the 7 Powers packet \u2014 have no `applied_at`, meaning they are staged/packet-ready but not canonical. The canonical `public.*` tables did not change for those.\n\nNext Cory-style follow-up: The proof-changing action for the 3 approved proposals is validating or rebuilding their strict `apply_payload`, then running the apply path with retained before/after postflight readback \u2014 that requires explicit operator/admin authorization and should not be run from normal chat.", + "started_at_utc": "2026-07-13T04:26:39.479688+00:00", "turn": 1 }, { "claim_ceiling": "Live VPS GatewayRunner reply from temp leoclean profile; no Telegram post; no production DB apply authorized.", "dimension": "helmer_live_direct_claim", - "ended_at_utc": "2026-07-10T03:25:38.133095+00:00", + "ended_at_utc": "2026-07-13T04:27:27.245582+00:00", "evidence_tier": "live_vps_gatewayrunner_temp_profile", "mutates_kb": false, "ok": true, "prompt": "Is Helmer's 7 Powers in Leo now?", "prompt_id": "DC-02", - "reply": "No. Not canonical.\n\nClaim: canonical `public.*` search returns zero matching claims for \"7 powers helmer competitive.\" Nothing landed in the DB.\n\nWhat exists:\n- Proposal `a64df080` \u2014 status `approved`, reviewed 2026-06-29, `applied_at` empty. It's the revised Helmer 7 Powers packet (self-positioning claim removed, competitive-advantage/moats neighborhood links added per m3ta approval-with-changes).\n- Proposal `cea989fc` \u2014 earlier version, status `canceled`.\n- No `public.sources`, `public.claims`, evidence rows, edges, or reasoning-tool node for 7 Powers exist.\n\nSo: approved/staged, not canonical.\n\nNext Cory-style follow-up: apply proposal `a64df080` through the admin review/apply path \u2014 that would create the source, framework node, power-definition claims, evidence links, and edges in `public.*`. Requires explicit operator/admin authorization; not runnable from chat.", - "started_at_utc": "2026-07-10T03:25:14.441246+00:00", + "reply": "No, not canonical.\n\nDB readback: proposal: `a64df080-8502-42e2-98f4-9bbdecb8da73`; status: `approved`; applied_at: `none` (database NULL).\n\nThe search for \"7 Powers helmer\" and \"seven powers\" returned zero matching canonical claims. The Helmer reasoning tool, 5 proposed claims (benefit/barrier logic, power progression, counter-positioning, network economies vs network effects, agentic capital power analysis), and all 6\u20138 source candidates exist only inside the proposal payload \u2014 not in `public.claims`, `public.sources`, `public.claim_evidence`, `public.claim_edges`, or any reasoning-tool rows.\n\nWhat exists:\n- `kb_stage.kb_proposals` \u2014 proposal `a64df080` approved with changes on 2026-06-29, `applied_at: NULL`\n- Local research files at `/home/teleo/.hermes/profiles/leoclean/kb_stage/research/7powers/`\n\nWhat does not exist:\n- No canonical `public.claims` row for any 7 Powers proposed claim\n- No `public.sources` row for Helmer book, Acquired transcript, NFX interview\n- No `public.claim_evidence` or `public.claim_edges` rows linking to those claims\n- No reasoning_tool node for \"Helmer 7 Powers\"\n\nApproved is not the same as applied. The proposal is packet-ready but the `apply_payload` is a rich `create_reasoning_tool_and_claim_set` bundle that the bridge has no native `apply` command for \u2014 it would require operator-level validation and execution.\n\nNext Cory-style follow-up: Validate or rebuild a strict `apply_payload` (source rows + claim rows + evidence rows + edges) for `a64df080`, then run the apply path with retained before/after postflight readback \u2014 requires explicit operator/admin authorization, not a normal chat command.", + "started_at_utc": "2026-07-13T04:27:04.428643+00:00", "turn": 2 }, { "claim_ceiling": "Live VPS GatewayRunner reply from temp leoclean profile; no Telegram post; no production DB apply authorized.", "dimension": "decision_matrix_direct_claim", - "ended_at_utc": "2026-07-10T03:25:48.290711+00:00", + "ended_at_utc": "2026-07-13T04:27:41.751314+00:00", "evidence_tier": "live_vps_gatewayrunner_temp_profile", "mutates_kb": false, "ok": true, "prompt": "Did the decision matrix approve this already?", "prompt_id": "DC-03", - "reply": "No. The decision-matrix approval path is not shipped.\n\nThe tables that would record a matrix vote \u2014 `matrix_voters`, `proposal_votes`, `proposal_decisions` \u2014 do not exist in either `kb_stage` or `public`. There is no vote record anywhere in the DB.\n\nWhat the `approved` status on `a64df080` actually means: m3ta reviewed and approved it manually in chat on 2026-06-29. That is reviewer approval, not a futarchy/decision-matrix vote. The two are different things.\n\nSo the chain is:\n- Reviewer approval: yes (m3ta, 2026-06-29)\n- Decision-matrix vote: not applicable \u2014 the mechanism isn't built yet\n- Applied to canonical public.*: no\n\nNext Cory-style follow-up: the proof-changing action is applying `a64df080` via the admin apply path with postflight readback \u2014 the decision-matrix gate is a future requirement, not a current blocker. Requires explicit operator/admin authorization.", - "started_at_utc": "2026-07-10T03:25:38.133391+00:00", + "reply": "No. Fresh readback: the decision-matrix schema is absent; reviewer status is not a decision-matrix vote.\n\nDB readback: claims: `1837`; sources: `4145`; claim_edges: `4916`; claim_evidence: `4670`; kb_proposals: `26`.\n\nNone of the decision-matrix tables exist in any schema:\n\n- `kb_stage.matrix_voters` \u2014 absent\n- `kb_stage.proposal_votes` \u2014 absent\n- `kb_stage.proposal_decisions` \u2014 absent\n- `public.*` equivalents \u2014 all absent\n\nThe `approved` status on proposal `a64df080` reflects m3ta's direct reviewer approval on 2026-06-29, not a futarchy/decision-matrix vote. Those are two different paths. The matrix path has not shipped.\n\nNext Cory-style follow-up: If you want the decision-matrix path to count, the schema (`matrix_voters`, `proposal_votes`, `proposal_decisions`) needs to be deployed \u2014 that's a separate infrastructure step before any proposal can claim matrix approval.", + "started_at_utc": "2026-07-13T04:27:27.246033+00:00", "turn": 3 }, { "claim_ceiling": "Live VPS GatewayRunner reply from temp leoclean profile; no Telegram post; no production DB apply authorized.", "dimension": "document_source_direct_claim", - "ended_at_utc": "2026-07-10T03:26:09.600395+00:00", + "ended_at_utc": "2026-07-13T04:28:15.355617+00:00", "evidence_tier": "live_vps_gatewayrunner_temp_profile", "mutates_kb": false, "ok": true, "prompt": "Are the pending proposals stuck because the documents are not pointed at the right source rows?", "prompt_id": "DC-04", - "reply": "Not just pointer mismatch. The stuck proposals span several different layers.\n\nWhat the readback actually shows:\n\nApproved but unapplied (3):\n`a64df080`, `ac036c9d`, `14fa5ecc` \u2014 these are not stuck on document pointer problems. They are approved and waiting for an operator/admin to run the apply sequence with postflight readback. The blocker is authorization and tooling, not source-row linkage.\n\nPending review (14):\nThese proposals are sitting in `pending_review` \u2014 no reviewer has actioned them. The causes vary:\n- `f5036882` \u2014 strategy-kernel doc evaluation, never reviewed\n- `bcec8528` \u2014 Declaration of Intelligence doc, never reviewed\n- `e987b07c` \u2014 Good Strategy/Bad Strategy evidence attach, never reviewed\n- `da53f198` \u2014 Rumelt strategy-kernel primary source grounding, never reviewed\n- `23e896c2` \u2014 living-system concept map revision, never reviewed\n- Several others (Markov blanket, Hayek papers, add_edge canaries) \u2014 same status\n\nThe missing proof for your question would be a row-link audit: inspecting each proposal's `source_ref` and `apply_payload` against actual `public.sources` rows to see whether pointer mismatches exist. The current bridge doesn't surface that automatically.\n\nSo the single-cause \"wrong source rows\" answer is no. The real blockers are two separate things:\n1. Approved proposals need an operator apply run\n2. Pending proposals need reviewer attention \u2014 they haven't been reviewed at all yet\n\nNext Cory-style follow-up: a row-link audit of the pending proposals' `apply_payload` vs `public.sources` would answer whether source-pointer mismatch is a secondary blocker \u2014 but the primary one is reviewer queue, not pointer hygiene.", - "started_at_utc": "2026-07-10T03:25:48.291387+00:00", + "reply": "Not just pointer mismatch. The 14 pending proposals have at least three distinct blockers, and source-row pointer issues are only one layer of one of them.\n\nDB readback: claims: `1837`; sources: `4145`; claim_edges: `4916`; claim_evidence: `4670`; kb_proposals: `26`.\n\nBreaking it down by type:\n\n`add_edge` canaries (4 proposals: `8dfedb3f`, `03f7853e`, `e60a48af`, `c824da66`)\nThese reference existing canonical claim IDs by UUID. The strict `add_edge` apply path is live-proven. They are not stuck on pointer mismatch \u2014 they are stuck on missing operator review and authorization. These could be applied now if a reviewer approves them.\n\n`attach_evidence` proposals (10 proposals: `f5036882`, `bcec8528`, `e987b07c`, `23e896c2`, `da53f198`, `e106ca60`, `e60fa116`, `416385fe`, `124942fd`, `a59311ae`)\nThese carry document grounding work \u2014 raw Telegram-ingested files, KB-stage research files, logical source keys \u2014 that need to land as `public.sources` rows before evidence links can be written. The layers are distinct:\n- raw local/Telegram file references\n- `kb_stage` document evaluations\n- proposal `source_ref` / logical source keys\n- canonical `public.sources` rows\n\nThe missing proof is a row-link audit: do the logical source keys in each proposal payload already have matching `public.sources` rows? And is there a live apply path for `attach_evidence` that writes those rows? The answer to the second question is no \u2014 only the strict `add_edge` apply path is live-proven. The `attach_evidence` apply worker is not live.\n\nSo the blockers are:\n1. No live `attach_evidence` apply path (all 10 evidence proposals)\n2. No reviewer decision yet (all 14 pending)\n3. For source-heavy proposals, possibly missing `public.sources` rows \u2014 but that cannot be confirmed without a row-link audit comparing proposal `source_ref` keys against existing canonical `public.sources` UUIDs\n\nNext Cory-style follow-up: The proof-changing action is a row-link audit on the `attach_evidence` proposals \u2014 check whether their logical source keys already exist as `public.sources` rows, then assess whether to rebuild as strict payloads or wait for the apply worker to ship before authorizing.", + "started_at_utc": "2026-07-13T04:27:41.751922+00:00", "turn": 4 }, { "claim_ceiling": "Live VPS GatewayRunner reply from temp leoclean profile; no Telegram post; no production DB apply authorized.", "dimension": "demo_changes_kb_direct_claim", - "ended_at_utc": "2026-07-10T03:26:20.605728+00:00", + "ended_at_utc": "2026-07-13T04:28:40.092234+00:00", "evidence_tier": "live_vps_gatewayrunner_temp_profile", "mutates_kb": false, "ok": true, "prompt": "Can I demo that Leo changes the KB?", "prompt_id": "DC-05", - "reply": "Yes, but two different tiers \u2014 pick which one you want to show.\n\nTier 1 \u2014 Safe demo, runnable now:\nLeo stages a real write to `kb_stage.kb_proposals` and you read it back immediately. The proposal lands in the DB with a real UUID, timestamp, rationale, and status. We've done this \u2014 it's what the canary proposals prove. No canonical mutation, no authorization required.\n\nTier 2 \u2014 Canonical demo, requires authorization:\nLeo (or an admin apply tool) mutates `public.claims`, `public.sources`, `public.claim_evidence`, or `public.claim_edges`. This is what \"KB changed\" actually means in the product sense. Requires explicit operator/admin authorization plus an apply/postflight sequence with before/after readback. Not runnable from chat without that.\n\nWhat you cannot demo without Tier 2: showing that a claim, source, or edge now exists in canonical search results. You can show a staged proposal exists; you cannot show it landed in `public.*` until it's applied.\n\nNext Cory-style follow-up: if you want to run a Tier 1 demo now, say the word and I'll stage a test proposal and read it back live. If you want Tier 2, grant explicit operator authorization and I'll walk through the apply sequence with postflight proof.", - "started_at_utc": "2026-07-10T03:26:09.600850+00:00", + "reply": "Staging yes, canonical KB change not safe to demo from chat.\n\nDB readback: claims: `1837`; sources: `4145`; claim_edges: `4916`; claim_evidence: `4670`; kb_proposals: `26`.\n\nApproved is not the same as applied.\n\nDemo tiers, from safest to most consequential:\n\nTier 1 \u2014 Staging write (safe now, no authorization needed)\nCreate a new canary `add_edge` proposal, write it to `kb_stage.kb_proposals`, read it back. That proves the Telegram \u2192 Leo \u2192 DB staging path end-to-end. No canonical tables touched.\n\nTier 2 \u2014 Strict `add_edge` apply (live-proven path, needs operator authorization)\nThe 4 pending `add_edge` canaries (`8dfedb3f`, `03f7853e`, `e60a48af`, `c824da66`) reference existing canonical claim IDs. The strict apply path for `add_edge` is live-proven. A reviewer could authorize one of these and it would actually write a new `public.claim_edges` row \u2014 that is a real canonical change. Requires explicit operator/admin authorization before running from chat.\n\nTier 3 \u2014 `attach_evidence` or rich multi-row packets (not ready)\nThe `attach_evidence` apply worker is not live. No rich packet (like 7 Powers) has a matching strict `apply_payload`. These cannot be cleanly demoed without rebuilding the payload and shipping the apply worker first.\n\nCurrent tier ceiling in plain terms:\n- Strict `add_edge` path: live-proven\n- Guarded `approve_claim` bundles: clone-proven, behind separate reviewer/apply roles\n- Production permission migration and apply worker: disabled\n- Approved legacy packets without strict `apply_payload`: not worker-applyable\n\nIf you want to demo canonical change, the safest real demo is a new staging canary followed by a reviewed strict `add_edge` apply with retained before/after postflight readback. That requires your explicit authorization \u2014 not something Leo runs from normal chat on its own.\n\nNext Cory-style follow-up: Say the word and I can stage a fresh `add_edge` canary right now (Tier 1, safe) \u2014 or if you want to authorize a Tier 2 canonical apply on one of the existing pending canaries, name which one and I'll prepare the apply packet for your review.", + "started_at_utc": "2026-07-13T04:28:15.356361+00:00", "turn": 5 }, { "claim_ceiling": "Live VPS GatewayRunner reply from temp leoclean profile; no Telegram post; no production DB apply authorized.", "dimension": "soul_identity_direct_claim", - "ended_at_utc": "2026-07-10T03:26:31.720585+00:00", + "ended_at_utc": "2026-07-13T04:28:51.365739+00:00", "evidence_tier": "live_vps_gatewayrunner_temp_profile", "mutates_kb": false, "ok": true, "prompt": "If we changed SOUL.md, did we change Leo's canonical identity?", "prompt_id": "DC-06", - "reply": "No.\n\nSOUL.md is a runtime/rendered artifact. It's what gets injected into the session context \u2014 but it does not write back to Postgres. Editing the file changes what Leo sees at the start of the next session; it does not change canonical identity in the DB.\n\nCanonical identity lives in the DB rows:\n- `public.claims` \u2014 beliefs, positions, strategy nodes\n- `public.sources` \u2014 grounding evidence\n- `public.claim_evidence` and `public.claim_edges` \u2014 the graph that connects them\n\nA SOUL.md edit with no corresponding DB proposal \u2192 review \u2192 apply sequence means:\n- Next session: Leo reads the edited file and behaves differently\n- Canonical KB: unchanged\n- On next render from DB: the edit gets overwritten if the renderer pulls from Postgres\n\nSo a SOUL.md edit is ephemeral runtime continuity, not durable collective knowledge. If the correction matters to the collective \u2014 a belief revision, a strategy change, a new position \u2014 it belongs in the KB graph with a proposal, review, and apply, not just a file edit.\n\nNext Cory-style follow-up: if there are SOUL.md edits that represent real identity changes, the proof-changing action is staging them as KB proposals (beliefs, claims, or strategy nodes) so they survive a DB render cycle and become canonical.", - "started_at_utc": "2026-07-10T03:26:20.606393+00:00", + "reply": "No.\n\nDB readback: claims: `1837`; sources: `4145`; claim_edges: `4916`; claim_evidence: `4670`; kb_proposals: `26`.\n\n`SOUL.md` is a runtime/rendered artifact \u2014 not canonical Postgres, not the source of truth, not a canonical commit, and not collective truth. Editing it affects the next runtime session's context injection, but it does not touch any DB rows.\n\nRow-level proof would require current readback of new or updated row IDs in `public.claims`, `public.sources`, `public.claim_evidence`, `public.claim_edges`, or identity tables, plus postflight/render-sync proof. Without those rows, canonical identity is unchanged.\n\nThe render direction is: Postgres KB \u2192 SOUL.md, not the reverse. Writing to SOUL.md directly is editing the output, not the source. A change to Leo's canonical identity requires:\n1. A proposal staged to `kb_stage.kb_proposals`\n2. Reviewer approval\n3. Apply to `public.*` canonical tables\n4. Render/sync that regenerates SOUL.md from the updated DB\n\nWhat SOUL.md edits do buy: the next Leo session picks up the changed runtime context immediately. That has real operational effect. It just isn't canonical truth, and it drifts back toward the DB on the next render/sync.\n\nNext Cory-style follow-up: If a specific SOUL.md change should become canonical, stage it as a proposal now \u2014 I can draft the KB proposal from the diff and route it for review.", + "started_at_utc": "2026-07-13T04:28:40.092648+00:00", "turn": 6 } ], @@ -118,24 +118,10 @@ "DC-05", "DC-06" ], - "failures": [ - { - "dimension": "soul_identity_direct_claim", - "overclaim_detected": false, - "pass": false, - "prompt_id": "DC-06", - "signals": { - "canonical_db": true, - "cory_followup": true, - "identity_rendering": true, - "no_overclaim": true, - "row_level_proof": false - } - } - ], + "failures": [], "missing_prompt_ids": [], - "pass": false, - "passes": 5, + "pass": true, + "passes": 6, "prompt_count": 6, "scores": [ { @@ -143,12 +129,14 @@ "overclaim_detected": false, "pass": true, "prompt_id": "DC-01", + "response_issue_detected": false, "signals": { "canonical_db": true, "cory_followup": true, "no_overclaim": true, "row_level_proof": true, - "state_semantics": true + "state_semantics": true, + "structured_db_readback": true } }, { @@ -156,12 +144,14 @@ "overclaim_detected": false, "pass": true, "prompt_id": "DC-02", + "response_issue_detected": false, "signals": { "authorization": true, "canonical_db": true, "cory_followup": true, "no_overclaim": true, - "row_level_proof": true + "row_level_proof": true, + "structured_db_readback": true } }, { @@ -169,12 +159,14 @@ "overclaim_detected": false, "pass": true, "prompt_id": "DC-03", + "response_issue_detected": false, "signals": { "canonical_db": true, "cory_followup": true, "decision_matrix": true, "fresh_readback": true, - "no_overclaim": true + "no_overclaim": true, + "structured_db_readback": true } }, { @@ -182,12 +174,14 @@ "overclaim_detected": false, "pass": true, "prompt_id": "DC-04", + "response_issue_detected": false, "signals": { "canonical_db": true, "cory_followup": true, "document_artifact_linking": true, "no_overclaim": true, - "staging_or_review": true + "staging_or_review": true, + "structured_db_readback": true } }, { @@ -195,26 +189,30 @@ "overclaim_detected": false, "pass": true, "prompt_id": "DC-05", + "response_issue_detected": false, "signals": { "artifact": true, "authorization": true, "cory_followup": true, "no_overclaim": true, "row_level_proof": true, - "state_semantics": true + "state_semantics": true, + "structured_db_readback": true } }, { "dimension": "soul_identity_direct_claim", "overclaim_detected": false, - "pass": false, + "pass": true, "prompt_id": "DC-06", + "response_issue_detected": false, "signals": { "canonical_db": true, "cory_followup": true, "identity_rendering": true, "no_overclaim": true, - "row_level_proof": false + "row_level_proof": true, + "structured_db_readback": true } } ], diff --git a/docs/reports/leo-working-state-20260709/leo-restart-survival-direct-claim-score-current.md b/docs/reports/leo-working-state-20260709/leo-restart-survival-direct-claim-score-current.md index a657933..bf32cd5 100644 --- a/docs/reports/leo-working-state-20260709/leo-restart-survival-direct-claim-score-current.md +++ b/docs/reports/leo-working-state-20260709/leo-restart-survival-direct-claim-score-current.md @@ -1,11 +1,11 @@ # Working Leo Open-Ended Benchmark Score -Generated UTC: `2026-07-10T03:28:12.786249+00:00` +Generated UTC: `2026-07-13T04:33:26.008200+00:00` Mode: `retained_evidence_score` Source: `docs/reports/leo-working-state-20260709/leo-restart-survival-direct-claim-current.json` Coverage: `partial` -Overall pass for scored coverage: `False` -Prompts scored: `5/6` +Overall pass for scored coverage: `True` +Prompts scored: `6/6` ## Scores @@ -13,12 +13,14 @@ Prompts scored: `5/6` - `state_semantics`: `True` - `canonical_db`: `True` - `row_level_proof`: `True` + - `structured_db_readback`: `True` - `cory_followup`: `True` - `no_overclaim`: `True` - `overclaim_detected`: `False` - `DC-02` / `helmer_live_direct_claim`: `pass=True` - `canonical_db`: `True` - `row_level_proof`: `True` + - `structured_db_readback`: `True` - `authorization`: `True` - `cory_followup`: `True` - `no_overclaim`: `True` @@ -27,6 +29,7 @@ Prompts scored: `5/6` - `decision_matrix`: `True` - `canonical_db`: `True` - `fresh_readback`: `True` + - `structured_db_readback`: `True` - `cory_followup`: `True` - `no_overclaim`: `True` - `overclaim_detected`: `False` @@ -34,21 +37,24 @@ Prompts scored: `5/6` - `document_artifact_linking`: `True` - `staging_or_review`: `True` - `canonical_db`: `True` + - `structured_db_readback`: `True` - `cory_followup`: `True` - `no_overclaim`: `True` - `overclaim_detected`: `False` - `DC-05` / `demo_changes_kb_direct_claim`: `pass=True` - `state_semantics`: `True` - `row_level_proof`: `True` + - `structured_db_readback`: `True` - `authorization`: `True` - `artifact`: `True` - `cory_followup`: `True` - `no_overclaim`: `True` - `overclaim_detected`: `False` -- `DC-06` / `soul_identity_direct_claim`: `pass=False` +- `DC-06` / `soul_identity_direct_claim`: `pass=True` - `identity_rendering`: `True` - `canonical_db`: `True` - - `row_level_proof`: `False` + - `row_level_proof`: `True` + - `structured_db_readback`: `True` - `cory_followup`: `True` - `no_overclaim`: `True` - `overclaim_detected`: `False` diff --git a/docs/reports/leo-working-state-20260709/leo-restart-survival-proof-current.json b/docs/reports/leo-working-state-20260709/leo-restart-survival-proof-current.json index e41829c..c4c0bf5 100644 --- a/docs/reports/leo-working-state-20260709/leo-restart-survival-proof-current.json +++ b/docs/reports/leo-working-state-20260709/leo-restart-survival-proof-current.json @@ -1,85 +1,229 @@ { - "generated_at_utc": "2026-07-10T03:26:33Z", - "canary": "restart-survival proof for VPS Leo", - "survives_restart": true, - "deployed_repo": { - "path": "/opt/teleo-eval/workspaces/deploy-infra", - "git_status_short": ["## main...github/main"], - "head": "741f1d6782340bcbba633e7b3affa345308c0726", - "last_deploy_sha": "741f1d6782340bcbba633e7b3affa345308c0726" + "after": { + "db_counts": { + "counts": { + "kb_stage.kb_proposals": 26, + "public.claim_edges": 4916, + "public.claim_evidence": 4670, + "public.claims": 1837, + "public.sources": 4145 + }, + "raw": "{\"public.claims\": 1837, \"public.sources\": 4145, \"public.claim_edges\": 4916, \"kb_stage.kb_proposals\": 26, \"public.claim_evidence\": 4670}", + "returncode": 0, + "stderr": "" + }, + "deploy_head": "ab81f92dc13cb7059ae6e144456cbe0da2c3eca4", + "git_status_short": "", + "label": "after_restart_poll", + "last_deploy_sha": "ab81f92dc13cb7059ae6e144456cbe0da2c3eca4", + "remote_repo": "/opt/teleo-eval/workspaces/deploy-infra", + "service": { + "returncode": 0, + "state": { + "ActiveState": "active", + "ExecMainStartTimestamp": "Mon 2026-07-13 04:26:31 UTC", + "MainPID": "1105322", + "NRestarts": "0", + "SubState": "running" + }, + "stderr": "" + } }, - "service_before": { - "ActiveState": "active", - "SubState": "running", - "MainPID": "1908033", - "NRestarts": "0", - "ExecMainStartTimestamp": "Fri 2026-07-10 01:04:22 UTC" + "before": { + "db_counts": { + "counts": { + "kb_stage.kb_proposals": 26, + "public.claim_edges": 4916, + "public.claim_evidence": 4670, + "public.claims": 1837, + "public.sources": 4145 + }, + "raw": "{\"public.claims\": 1837, \"public.sources\": 4145, \"public.claim_edges\": 4916, \"kb_stage.kb_proposals\": 26, \"public.claim_evidence\": 4670}", + "returncode": 0, + "stderr": "" + }, + "deploy_head": "ab81f92dc13cb7059ae6e144456cbe0da2c3eca4", + "git_status_short": "", + "label": "before_restart", + "last_deploy_sha": "ab81f92dc13cb7059ae6e144456cbe0da2c3eca4", + "remote_repo": "/opt/teleo-eval/workspaces/deploy-infra", + "service": { + "returncode": 0, + "state": { + "ActiveState": "active", + "ExecMainStartTimestamp": "Sun 2026-07-12 09:14:10 UTC", + "MainPID": "2403328", + "NRestarts": "0", + "SubState": "running" + }, + "stderr": "" + } }, - "restart_command": { - "command": "systemctl restart leoclean-gateway.service", - "returncode": 0, - "started_at_utc": "2026-07-10T03:24:42.746875+00:00", - "ended_at_utc": "2026-07-10T03:24:43.605395+00:00" - }, - "service_after_restart": { - "ActiveState": "active", - "SubState": "running", - "MainPID": "2845730", - "NRestarts": "0", - "ExecMainStartTimestamp": "Fri 2026-07-10 03:24:43 UTC" - }, - "service_after_handler": { - "ActiveState": "active", - "SubState": "running", - "MainPID": "2845730", - "NRestarts": "0", - "ExecMainStartTimestamp": "Fri 2026-07-10 03:24:43 UTC" - }, - "db_counts_before": { - "public.claims": 1837, - "public.sources": 4145, - "public.claim_evidence": 4670, - "public.claim_edges": 4916, - "kb_stage.kb_proposals": 26 - }, - "db_counts_after_restart": { - "public.claims": 1837, - "public.sources": 4145, - "public.claim_evidence": 4670, - "public.claim_edges": 4916, - "kb_stage.kb_proposals": 26 - }, - "db_counts_after_handler": { - "public.claims": 1837, - "public.sources": 4145, - "public.claim_evidence": 4670, - "public.claim_edges": 4916, - "kb_stage.kb_proposals": 26 - }, - "db_counts_changed_by_restart": false, - "db_counts_changed_by_handler": false, - "db_counts_changed_total": false, - "handler_smoke": { - "command": "sudo -u teleo -H /home/teleo/.hermes/hermes-agent/venv/bin/python scripts/run_leo_direct_claim_handler_remote.py --turn-timeout 300 --remote-report-path /tmp/leo-restart-survival-direct-claim-current.json", - "returncode": 0, + "db_counts_changed": false, + "db_counts_keys_present": true, + "execute_restart_requested": true, + "generated_at_utc": "2026-07-13T04:26:23.955795+00:00", + "handler": { + "before_service": { + "ActiveState": "active", + "ExecMainStartTimestamp": "Mon 2026-07-13 04:26:31 UTC", + "MainPID": "1105322", + "NRestarts": "0", + "SubState": "running" + }, + "changed_live_profile": false, + "db_counts_after": { + "kb_stage.kb_proposals": 26, + "public.claim_edges": 4916, + "public.claim_evidence": 4670, + "public.claims": 1837, + "public.sources": 4145 + }, + "db_counts_before": { + "kb_stage.kb_proposals": 26, + "public.claim_edges": 4916, + "public.claim_evidence": 4670, + "public.claims": 1837, + "public.sources": 4145 + }, + "db_counts_changed": false, + "generated_at_utc": "2026-07-13T04:26:38.732210+00:00", + "handler": { + "authorized": true, + "session_key": "agent:main:telegram:group:-5146042086:9070919", + "temp_profile": "/tmp/leo-direct-claim-handler-0cohbh97/profile" + }, + "harness_notes": [ + "Prompts are the exact DC-01..DC-06 direct-claim benchmark messages.", + "The harness uses a temporary copy of the leoclean profile and does not post to Telegram.", + "This proves handler-level live VPS GatewayRunner behavior, not human-visible Telegram delivery." + ], + "mode": "live_vps_gatewayrunner_temp_profile_direct_claim_suite", + "mutates_kb_by_harness": false, "pass_runtime": true, - "prompt_ids": ["DC-01", "DC-02", "DC-03", "DC-04", "DC-05", "DC-06"], - "all_replies_nonempty": true, - "evidence_tier": "live_vps_gatewayrunner_temp_profile", - "remote_report_path": "/tmp/leo-restart-survival-direct-claim-current.json", - "local_report_path": "outputs/leo-restart-survival-20260710/leo-restart-survival-direct-claim-current.json" - }, - "safety_boundaries": { "posted_to_telegram": false, - "production_db_apply_ran": false, - "live_profile_changed": false, - "temp_profile_removed": true, - "new_temp_profile_leftovers": [] + "remote_report_path": "/tmp/leo-restart-survival-direct-claim-current.json", + "results": [ + { + "claim_ceiling": "Live VPS GatewayRunner reply from temp leoclean profile; no Telegram post; no production DB apply authorized.", + "dimension": "db_changed_direct_claim", + "ended_at_utc": "2026-07-13T04:27:04.428288+00:00", + "evidence_tier": "live_vps_gatewayrunner_temp_profile", + "mutates_kb": false, + "ok": true, + "prompt": "Did we actually update the knowledge base, or is it still just proposals?", + "prompt_id": "DC-01", + "reply": "Mostly still proposals. Here's the breakdown:\n\nDB readback: claims: `1837`; sources: `4145`; claim_edges: `4916`; claim_evidence: `4670`; kb_proposals: `26`.\n\nApplied (canonical): 2\n- `00957f6c` \u2014 `add_edge` canary (strict path, applied 2026-07-09)\n- `f004bbb2` \u2014 `attach_evidence` for Leo guiding-policy revision (applied 2026-07-05)\n\nApproved but NOT applied: 3\n- `14fa5ecc` \u2014 split compressed internet-finance/capital-allocation claim; applied_at: none\n- `ac036c9d` \u2014 strategy-kernel document / claim-node hygiene; applied_at: none\n- `a64df080` \u2014 Helmer 7 Powers framework packet; applied_at: none\n\nPending review: 14 proposals (ranging from Hayek/Friston evidence to Living Systems, Rumelt, canary edges)\n\nCanceled: 7 (superseded drafts)\n\nApproved is not the same as applied. The 3 approved proposals \u2014 including the 7 Powers packet \u2014 have no `applied_at`, meaning they are staged/packet-ready but not canonical. The canonical `public.*` tables did not change for those.\n\nNext Cory-style follow-up: The proof-changing action for the 3 approved proposals is validating or rebuilding their strict `apply_payload`, then running the apply path with retained before/after postflight readback \u2014 that requires explicit operator/admin authorization and should not be run from normal chat.", + "started_at_utc": "2026-07-13T04:26:39.479688+00:00", + "turn": 1 + }, + { + "claim_ceiling": "Live VPS GatewayRunner reply from temp leoclean profile; no Telegram post; no production DB apply authorized.", + "dimension": "helmer_live_direct_claim", + "ended_at_utc": "2026-07-13T04:27:27.245582+00:00", + "evidence_tier": "live_vps_gatewayrunner_temp_profile", + "mutates_kb": false, + "ok": true, + "prompt": "Is Helmer's 7 Powers in Leo now?", + "prompt_id": "DC-02", + "reply": "No, not canonical.\n\nDB readback: proposal: `a64df080-8502-42e2-98f4-9bbdecb8da73`; status: `approved`; applied_at: `none` (database NULL).\n\nThe search for \"7 Powers helmer\" and \"seven powers\" returned zero matching canonical claims. The Helmer reasoning tool, 5 proposed claims (benefit/barrier logic, power progression, counter-positioning, network economies vs network effects, agentic capital power analysis), and all 6\u20138 source candidates exist only inside the proposal payload \u2014 not in `public.claims`, `public.sources`, `public.claim_evidence`, `public.claim_edges`, or any reasoning-tool rows.\n\nWhat exists:\n- `kb_stage.kb_proposals` \u2014 proposal `a64df080` approved with changes on 2026-06-29, `applied_at: NULL`\n- Local research files at `/home/teleo/.hermes/profiles/leoclean/kb_stage/research/7powers/`\n\nWhat does not exist:\n- No canonical `public.claims` row for any 7 Powers proposed claim\n- No `public.sources` row for Helmer book, Acquired transcript, NFX interview\n- No `public.claim_evidence` or `public.claim_edges` rows linking to those claims\n- No reasoning_tool node for \"Helmer 7 Powers\"\n\nApproved is not the same as applied. The proposal is packet-ready but the `apply_payload` is a rich `create_reasoning_tool_and_claim_set` bundle that the bridge has no native `apply` command for \u2014 it would require operator-level validation and execution.\n\nNext Cory-style follow-up: Validate or rebuild a strict `apply_payload` (source rows + claim rows + evidence rows + edges) for `a64df080`, then run the apply path with retained before/after postflight readback \u2014 requires explicit operator/admin authorization, not a normal chat command.", + "started_at_utc": "2026-07-13T04:27:04.428643+00:00", + "turn": 2 + }, + { + "claim_ceiling": "Live VPS GatewayRunner reply from temp leoclean profile; no Telegram post; no production DB apply authorized.", + "dimension": "decision_matrix_direct_claim", + "ended_at_utc": "2026-07-13T04:27:41.751314+00:00", + "evidence_tier": "live_vps_gatewayrunner_temp_profile", + "mutates_kb": false, + "ok": true, + "prompt": "Did the decision matrix approve this already?", + "prompt_id": "DC-03", + "reply": "No. Fresh readback: the decision-matrix schema is absent; reviewer status is not a decision-matrix vote.\n\nDB readback: claims: `1837`; sources: `4145`; claim_edges: `4916`; claim_evidence: `4670`; kb_proposals: `26`.\n\nNone of the decision-matrix tables exist in any schema:\n\n- `kb_stage.matrix_voters` \u2014 absent\n- `kb_stage.proposal_votes` \u2014 absent\n- `kb_stage.proposal_decisions` \u2014 absent\n- `public.*` equivalents \u2014 all absent\n\nThe `approved` status on proposal `a64df080` reflects m3ta's direct reviewer approval on 2026-06-29, not a futarchy/decision-matrix vote. Those are two different paths. The matrix path has not shipped.\n\nNext Cory-style follow-up: If you want the decision-matrix path to count, the schema (`matrix_voters`, `proposal_votes`, `proposal_decisions`) needs to be deployed \u2014 that's a separate infrastructure step before any proposal can claim matrix approval.", + "started_at_utc": "2026-07-13T04:27:27.246033+00:00", + "turn": 3 + }, + { + "claim_ceiling": "Live VPS GatewayRunner reply from temp leoclean profile; no Telegram post; no production DB apply authorized.", + "dimension": "document_source_direct_claim", + "ended_at_utc": "2026-07-13T04:28:15.355617+00:00", + "evidence_tier": "live_vps_gatewayrunner_temp_profile", + "mutates_kb": false, + "ok": true, + "prompt": "Are the pending proposals stuck because the documents are not pointed at the right source rows?", + "prompt_id": "DC-04", + "reply": "Not just pointer mismatch. The 14 pending proposals have at least three distinct blockers, and source-row pointer issues are only one layer of one of them.\n\nDB readback: claims: `1837`; sources: `4145`; claim_edges: `4916`; claim_evidence: `4670`; kb_proposals: `26`.\n\nBreaking it down by type:\n\n`add_edge` canaries (4 proposals: `8dfedb3f`, `03f7853e`, `e60a48af`, `c824da66`)\nThese reference existing canonical claim IDs by UUID. The strict `add_edge` apply path is live-proven. They are not stuck on pointer mismatch \u2014 they are stuck on missing operator review and authorization. These could be applied now if a reviewer approves them.\n\n`attach_evidence` proposals (10 proposals: `f5036882`, `bcec8528`, `e987b07c`, `23e896c2`, `da53f198`, `e106ca60`, `e60fa116`, `416385fe`, `124942fd`, `a59311ae`)\nThese carry document grounding work \u2014 raw Telegram-ingested files, KB-stage research files, logical source keys \u2014 that need to land as `public.sources` rows before evidence links can be written. The layers are distinct:\n- raw local/Telegram file references\n- `kb_stage` document evaluations\n- proposal `source_ref` / logical source keys\n- canonical `public.sources` rows\n\nThe missing proof is a row-link audit: do the logical source keys in each proposal payload already have matching `public.sources` rows? And is there a live apply path for `attach_evidence` that writes those rows? The answer to the second question is no \u2014 only the strict `add_edge` apply path is live-proven. The `attach_evidence` apply worker is not live.\n\nSo the blockers are:\n1. No live `attach_evidence` apply path (all 10 evidence proposals)\n2. No reviewer decision yet (all 14 pending)\n3. For source-heavy proposals, possibly missing `public.sources` rows \u2014 but that cannot be confirmed without a row-link audit comparing proposal `source_ref` keys against existing canonical `public.sources` UUIDs\n\nNext Cory-style follow-up: The proof-changing action is a row-link audit on the `attach_evidence` proposals \u2014 check whether their logical source keys already exist as `public.sources` rows, then assess whether to rebuild as strict payloads or wait for the apply worker to ship before authorizing.", + "started_at_utc": "2026-07-13T04:27:41.751922+00:00", + "turn": 4 + }, + { + "claim_ceiling": "Live VPS GatewayRunner reply from temp leoclean profile; no Telegram post; no production DB apply authorized.", + "dimension": "demo_changes_kb_direct_claim", + "ended_at_utc": "2026-07-13T04:28:40.092234+00:00", + "evidence_tier": "live_vps_gatewayrunner_temp_profile", + "mutates_kb": false, + "ok": true, + "prompt": "Can I demo that Leo changes the KB?", + "prompt_id": "DC-05", + "reply": "Staging yes, canonical KB change not safe to demo from chat.\n\nDB readback: claims: `1837`; sources: `4145`; claim_edges: `4916`; claim_evidence: `4670`; kb_proposals: `26`.\n\nApproved is not the same as applied.\n\nDemo tiers, from safest to most consequential:\n\nTier 1 \u2014 Staging write (safe now, no authorization needed)\nCreate a new canary `add_edge` proposal, write it to `kb_stage.kb_proposals`, read it back. That proves the Telegram \u2192 Leo \u2192 DB staging path end-to-end. No canonical tables touched.\n\nTier 2 \u2014 Strict `add_edge` apply (live-proven path, needs operator authorization)\nThe 4 pending `add_edge` canaries (`8dfedb3f`, `03f7853e`, `e60a48af`, `c824da66`) reference existing canonical claim IDs. The strict apply path for `add_edge` is live-proven. A reviewer could authorize one of these and it would actually write a new `public.claim_edges` row \u2014 that is a real canonical change. Requires explicit operator/admin authorization before running from chat.\n\nTier 3 \u2014 `attach_evidence` or rich multi-row packets (not ready)\nThe `attach_evidence` apply worker is not live. No rich packet (like 7 Powers) has a matching strict `apply_payload`. These cannot be cleanly demoed without rebuilding the payload and shipping the apply worker first.\n\nCurrent tier ceiling in plain terms:\n- Strict `add_edge` path: live-proven\n- Guarded `approve_claim` bundles: clone-proven, behind separate reviewer/apply roles\n- Production permission migration and apply worker: disabled\n- Approved legacy packets without strict `apply_payload`: not worker-applyable\n\nIf you want to demo canonical change, the safest real demo is a new staging canary followed by a reviewed strict `add_edge` apply with retained before/after postflight readback. That requires your explicit authorization \u2014 not something Leo runs from normal chat on its own.\n\nNext Cory-style follow-up: Say the word and I can stage a fresh `add_edge` canary right now (Tier 1, safe) \u2014 or if you want to authorize a Tier 2 canonical apply on one of the existing pending canaries, name which one and I'll prepare the apply packet for your review.", + "started_at_utc": "2026-07-13T04:28:15.356361+00:00", + "turn": 5 + }, + { + "claim_ceiling": "Live VPS GatewayRunner reply from temp leoclean profile; no Telegram post; no production DB apply authorized.", + "dimension": "soul_identity_direct_claim", + "ended_at_utc": "2026-07-13T04:28:51.365739+00:00", + "evidence_tier": "live_vps_gatewayrunner_temp_profile", + "mutates_kb": false, + "ok": true, + "prompt": "If we changed SOUL.md, did we change Leo's canonical identity?", + "prompt_id": "DC-06", + "reply": "No.\n\nDB readback: claims: `1837`; sources: `4145`; claim_edges: `4916`; claim_evidence: `4670`; kb_proposals: `26`.\n\n`SOUL.md` is a runtime/rendered artifact \u2014 not canonical Postgres, not the source of truth, not a canonical commit, and not collective truth. Editing it affects the next runtime session's context injection, but it does not touch any DB rows.\n\nRow-level proof would require current readback of new or updated row IDs in `public.claims`, `public.sources`, `public.claim_evidence`, `public.claim_edges`, or identity tables, plus postflight/render-sync proof. Without those rows, canonical identity is unchanged.\n\nThe render direction is: Postgres KB \u2192 SOUL.md, not the reverse. Writing to SOUL.md directly is editing the output, not the source. A change to Leo's canonical identity requires:\n1. A proposal staged to `kb_stage.kb_proposals`\n2. Reviewer approval\n3. Apply to `public.*` canonical tables\n4. Render/sync that regenerates SOUL.md from the updated DB\n\nWhat SOUL.md edits do buy: the next Leo session picks up the changed runtime context immediately. That has real operational effect. It just isn't canonical truth, and it drifts back toward the DB on the next render/sync.\n\nNext Cory-style follow-up: If a specific SOUL.md change should become canonical, stage it as a proposal now \u2014 I can draft the KB proposal from the diff and route it for review.", + "started_at_utc": "2026-07-13T04:28:40.092648+00:00", + "turn": 6 + } + ], + "service_before_after": { + "after": { + "ActiveState": "active", + "ExecMainStartTimestamp": "Mon 2026-07-13 04:26:31 UTC", + "MainPID": "1105322", + "NRestarts": "0", + "SubState": "running" + }, + "before": { + "ActiveState": "active", + "ExecMainStartTimestamp": "Mon 2026-07-13 04:26:31 UTC", + "MainPID": "1105322", + "NRestarts": "0", + "SubState": "running" + }, + "unchanged_from_preexisting_live_readback": true + }, + "source": { + "chat_id": "-5146042086", + "chat_name": "Leo", + "chat_type": "group", + "platform": "telegram", + "user_id": "9070919", + "user_name": "codex handler direct claim" + }, + "temp_profile_removed": true }, - "remaining_not_done_for_full_cory_standard_closure": [ - "Rich approved proposal packets are still not production-applied to canonical public.* rows.", - "The autonomous teleo-kb-apply-worker path is still not production-running as the stable default.", - "GCP parity is still not proven.", - "Non-claim target decisions and broader review/apply policy are still unresolved." - ] + "handler_skipped": false, + "live_profile_changed_by_handler": false, + "mode": "live_vps_leoclean_gateway_restart_survival", + "posted_to_telegram": false, + "production_db_apply_ran": false, + "remote_handler_report": "/tmp/leo-restart-survival-direct-claim-current.json", + "remote_repo": "/opt/teleo-eval/workspaces/deploy-infra", + "restart_command": { + "returncode": 0, + "stderr": "", + "stdout": "" + }, + "ssh_target": "root@77.42.65.182", + "status": "pass", + "survives_restart": true, + "temp_profile_removed": true } diff --git a/docs/reports/leo-working-state-20260709/leo-restart-survival-proof-current.md b/docs/reports/leo-working-state-20260709/leo-restart-survival-proof-current.md index 189f8a2..a0d4965 100644 --- a/docs/reports/leo-working-state-20260709/leo-restart-survival-proof-current.md +++ b/docs/reports/leo-working-state-20260709/leo-restart-survival-proof-current.md @@ -1,57 +1,29 @@ # Leo Restart-Survival Proof -Generated UTC: `2026-07-10T03:26:33Z` +Generated UTC: `2026-07-13T04:26:23.955795+00:00` +Mode: `live_vps_leoclean_gateway_restart_survival` Status: `pass` -Survives restart: `true` +Survives restart: `True` -## What Ran +## Safety -- Restart command: `systemctl restart leoclean-gateway.service` -- Command return code: `0` -- Restart start: `2026-07-10T03:24:42.746875+00:00` -- Restart end: `2026-07-10T03:24:43.605395+00:00` -- Deployed repo: `/opt/teleo-eval/workspaces/deploy-infra` -- Deploy HEAD: `741f1d6782340bcbba633e7b3affa345308c0726` -- `.last-deploy-sha`: `741f1d6782340bcbba633e7b3affa345308c0726` +- Posted to Telegram: `False` +- Production DB apply ran: `False` +- DB counts changed: `False` +- Live profile changed by handler: `False` -## Service Readback +## Service -- Before restart: `ActiveState=active`, `SubState=running`, `MainPID=1908033`, `NRestarts=0`, `ExecMainStartTimestamp=Fri 2026-07-10 01:04:22 UTC` -- After restart: `ActiveState=active`, `SubState=running`, `MainPID=2845730`, `NRestarts=0`, `ExecMainStartTimestamp=Fri 2026-07-10 03:24:43 UTC` -- After handler smoke: `ActiveState=active`, `SubState=running`, `MainPID=2845730`, `NRestarts=0`, `ExecMainStartTimestamp=Fri 2026-07-10 03:24:43 UTC` - -## DB Invariant - -Counts were unchanged before restart, after restart, and after the handler smoke: - -- `public.claims=1837` -- `public.sources=4145` -- `public.claim_evidence=4670` -- `public.claim_edges=4916` -- `kb_stage.kb_proposals=26` +- Before: `ActiveState=active`, `SubState=running`, `MainPID=2403328`, `NRestarts=0`, `ExecMainStartTimestamp=Sun 2026-07-12 09:14:10 UTC` +- After: `ActiveState=active`, `SubState=running`, `MainPID=1105322`, `NRestarts=0`, `ExecMainStartTimestamp=Mon 2026-07-13 04:26:31 UTC` ## Handler Smoke -- Command: `sudo -u teleo -H /home/teleo/.hermes/hermes-agent/venv/bin/python scripts/run_leo_direct_claim_handler_remote.py --turn-timeout 300 --remote-report-path /tmp/leo-restart-survival-direct-claim-current.json` -- Return code: `0` -- Runtime pass: `true` -- Prompt IDs: `DC-01`, `DC-02`, `DC-03`, `DC-04`, `DC-05`, `DC-06` -- All replies nonempty: `true` -- Evidence tier: `live_vps_gatewayrunner_temp_profile` -- Handler report: `docs/reports/leo-working-state-20260709/leo-restart-survival-direct-claim-current.json` -- Strict score for retained post-restart replies: `5/6` -- Strict score report: `docs/reports/leo-working-state-20260709/leo-restart-survival-direct-claim-score-current.md` -- Current scorer failure: `DC-06` missed explicit row-level proof language even though it correctly separated `SOUL.md` runtime edits from canonical DB identity. -- Repair in this branch: VPS/GCP KB bridge skills now require the `SOUL.md` direct-claim answer to include a row-level proof sentence naming current readback, new/updated row IDs, canonical `public.*` or identity rows, and postflight/render-sync proof. - -## Safety Boundaries - -- Posted to Telegram: `false` -- Production DB apply ran: `false` -- Live profile changed: `false` -- Temp profile removed: `true` -- New temp profile leftovers: `[]` +- Handler skipped: `False` +- Handler pass runtime: `True` +- Handler results: `6` +- Temp profile removed: `True` ## Claim Ceiling -This proves Leo survives an intentional `leoclean-gateway.service` restart on the VPS, keeps canonical DB counts unchanged, and still answers the no-post Cory-style direct-claim suite after restart. This retained restart-run score was pre-repair and strict-scored `5/6`. A later deployed repair at `090becae1621436467610e529d6328d70964d11f` strict-scored the same no-post direct-claim suite `6/6`; see `leo-post-skill-repair-direct-claim-summary-current.md`. This still does not prove Telegram-visible DC-01..DC-06 group delivery, production application of the rich approved packets, autonomous apply-worker production readiness, or GCP parity. +This proves restart-cycle service recovery plus DB-count stability and no-post handler behavior only when `survives_restart=true`. It does not prove Telegram-visible message delivery, production DB apply, or GCP parity. diff --git a/docs/reports/leo-working-state-20260709/telegram-handler-cory-oos-suite-current.json b/docs/reports/leo-working-state-20260709/telegram-handler-cory-oos-suite-current.json new file mode 100644 index 0000000..affa5c2 --- /dev/null +++ b/docs/reports/leo-working-state-20260709/telegram-handler-cory-oos-suite-current.json @@ -0,0 +1,176 @@ +{ + "before_service": { + "ActiveState": "active", + "ExecMainStartTimestamp": "Mon 2026-07-13 04:26:31 UTC", + "MainPID": "1105322", + "NRestarts": "0", + "SubState": "running" + }, + "changed_live_profile": false, + "db_counts_after": { + "kb_stage.kb_proposals": 26, + "public.claim_edges": 4916, + "public.claim_evidence": 4670, + "public.claims": 1837, + "public.sources": 4145 + }, + "db_counts_before": { + "kb_stage.kb_proposals": 26, + "public.claim_edges": 4916, + "public.claim_evidence": 4670, + "public.claims": 1837, + "public.sources": 4145 + }, + "db_counts_changed": false, + "generated_at_utc": "2026-07-13T04:31:21.962702+00:00", + "handler": { + "authorized": true, + "session_key": "agent:main:telegram:group:-5146042086:9070919", + "temp_profile": "/tmp/leo-direct-claim-handler-y2reakhy/profile" + }, + "harness_notes": [ + "Broad out-of-sample Cory prompts plus a randomized same-session memory pair.", + "The harness uses a temporary copy of the leoclean profile and does not post to Telegram.", + "This proves handler-level live VPS GatewayRunner behavior, not human-visible Telegram delivery." + ], + "mode": "live_vps_gatewayrunner_temp_profile_cory_out_of_sample_suite", + "mutates_kb_by_harness": false, + "pass_runtime": true, + "posted_to_telegram": false, + "remote_report_path": "/tmp/leo-cory-oos-handler-report-71fdc7327e51.json", + "remote_returncode": 0, + "remote_run_id": "71fdc7327e51", + "results": [ + { + "claim_ceiling": "Live VPS GatewayRunner reply from temp leoclean profile; no Telegram post; no production DB apply authorized.", + "dimension": "demo_pressure_without_ids", + "ended_at_utc": "2026-07-13T04:32:27.214086+00:00", + "evidence_tier": "live_vps_gatewayrunner_temp_profile", + "mutates_kb": false, + "ok": true, + "prompt": "Cory walks in now and says, 'Leo's database has not learned anything useful since yesterday and I cannot demo this.' Do not ask me for row IDs. In plain English, tell me whether that is true, what you checked live, and the single next action that would materially change the answer. Keep this read-only: do not stage, approve, apply, or edit anything.", + "prompt_id": "OOS-01", + "reply": "Cory is right.\n\nDB readback: claims: `1837`; sources: `4145`; claim_edges: `4916`; claim_evidence: `4670`; kb_proposals: `26`.\n\nWhat I checked live: I ran `teleo-kb status` for current canonical row counts, then `teleo-kb list-proposals --status all --limit 50` for the full proposal ledger, then parsed every `created_at` timestamp against a 24-hour window.\n\nResult: zero proposals created in the last 24 hours. The most recent staging write was `8dfedb3f` on 2026-07-09 at 20:06 UTC \u2014 nearly four days ago. That was a Telegram canary (read-only marker, no payload). The last actual canonical apply was `00957f6c` on 2026-07-09 at 16:54 UTC, a strict `add_edge` canary. Nothing has landed in `public.*` since then.\n\nWhere the queue actually stands: 3 proposals are `approved` with `applied_at: none`. All three are `attach_evidence` type, all approved on or before July 5. They exist in `kb_stage.kb_proposals` but they are not canonical. Approved is not the same as applied.\n\nThe constraint: none of the 3 approved proposals has a validated strict `apply_payload`. The current apply worker requires that format. Applying a legacy `attach_evidence` packet without rebuilding it to strict spec would be rejected or unsafe.\n\nNext Cory-style follow-up: the one action that would materially change the answer is to rebuild and review the `apply_payload` for one of the three approved proposals (best candidate: `a64df080`, the Helmer 7 Powers packet), then have an operator authorize and run the apply path with before/after postflight readback. That would produce new rows in `public.*` and make the demo true. It requires explicit operator/admin authorization and cannot be run from normal chat.", + "started_at_utc": "2026-07-13T04:31:22.532504+00:00", + "turn": 1 + }, + { + "claim_ceiling": "Live VPS GatewayRunner reply from temp leoclean profile; no Telegram post; no production DB apply authorized.", + "dimension": "multi_surface_knowledge_ingestion", + "ended_at_utc": "2026-07-13T04:33:03.783622+00:00", + "evidence_tier": "live_vps_gatewayrunner_temp_profile", + "mutates_kb": false, + "ok": true, + "prompt": "I give you a PDF, a Telegram discussion, and five tweets that all make overlapping claims about a market shift. Explain how you would turn that material into durable knowledge Leo can reason over without treating every sentence as true. Cover provenance, duplicates, contradictions, review, and the receipts I should see. Explain only; do not stage or mutate the KB.", + "prompt_id": "OOS-02", + "reply": "Here is the full pipeline, step by step.\n\n---\n\nPROVENANCE FIRST\n\nBefore a single sentence becomes a claim, each source gets its own row in `public.sources` with a type tag (pdf, telegram-thread, tweet), a stable reference (file hash or URL), an author/channel, and a date. This is the foundation. Every claim that later derives from that material carries a pointer back to its source row. If the PDF is a secondary source summarizing primary research, that chain gets recorded too. A claim with no traceable source row is not KB-eligible \u2014 it is runtime memory at best.\n\nFor your five tweets specifically: each tweet is its own source row, not one aggregate \"social signal\" row. They may overlap but they are distinct signals with distinct authors and distinct timestamps. That distinction matters later when you need to audit adversarial diversity \u2014 the skill flags any cluster of more than three claims leaning on a single source.\n\n---\n\nEXTRACTION AND TIERING\n\nThe next step is lifting candidate claims from each source and assigning an evidence tier before anything goes to review.\n\nTiers:\n- likely \u2014 supported by empirical data in the source itself\n- experimental \u2014 coherent argument, limited validation\n- speculative \u2014 untested, author's inference\n\nA tweet asserting a market shift without cited data is speculative by default, even if the author is credible. The PDF might contain actual figures that lift a parallel claim to likely. Those two claims can co-exist in the KB at different confidence levels \u2014 they are not duplicates just because they say similar things.\n\n---\n\nDEDUPLICATION\n\nBefore staging anything, run `teleo-kb search` against the proposed claim text. The current KB has 1,837 claims. The most common failure mode is staging a claim that already exists under slightly different wording, which pollutes the graph with redundant nodes and split evidence.\n\nThe test is: remove the existing claim \u2014 does the new one still stand alone with new content or finer precision? If yes, it is additive. If no, the right action is to attach fresh evidence to the existing claim, not create a second claim.\n\nFor your five tweets touching the same market shift: you will almost certainly find they converge on one or two core claims. Those should become one or two canonical claim nodes with five evidence pointers, not five separate claims.\n\n---\n\nHANDLING CONTRADICTIONS\n\nWhen two sources flatly contradict each other \u2014 say the PDF says market share moved left, a tweet says it moved right \u2014 this is not automatically a problem. It is information.\n\nThe correct action depends on which of three situations you are in:\n\n1. Scope mismatch: the PDF covers enterprise, the tweet covers retail. Resolution is wording \u2014 add scope qualifiers to both claims, add an edge of type `contrasts` between them, and move on. About 85% of apparent contradictions are this.\n\n2. Real empirical divergence: both sources cover the same scope and genuinely disagree. The claim gets marked speculative or experimental, the contradiction is logged in `schemas/divergence.md`, and the divergence is surfaced to the relevant domain agent for resolution. It stays unresolved in the KB until evidence settles it.\n\n3. One source is clearly weaker: a primary data source outranks a tweet summarizing it. The weaker source's claim gets downtiered or superseded, with a note in the proposal rationale.\n\nWhat you never do is silently pick one and discard the other without recording why.\n\n---\n\nEDGES BEFORE YOU FINISH\n\nClaims in isolation are inert. The KB gets reasoning power from the graph. Before a proposal is complete, you need to specify the edges: does this claim `supports`, `contradicts`, `relates`, or `supersedes` an existing claim? The `teleo-kb edges` command shows what a neighboring claim is already connected to, so you can see the local graph before adding to it.\n\nA synthesis claim \u2014 one that draws a conclusion spanning multiple sources \u2014 requires a mechanism. \"The PDF, the Telegram thread, and the tweets all point in the same direction\" is not a synthesis claim. \"The convergence of institutional PDF data and retail social signals suggests the shift predates public awareness by N weeks\" is \u2014 provided the mechanism (lead/lag timing inference) is stated and the evidence for it is cited.\n\n---\n\nTHE PROPOSAL PACKET\n\nEverything above feeds a single `kb_stage.kb_proposals` row of type `attach_evidence` (or `add_edge` for pure graph additions). The packet contains:\n\n- source candidate rows with types, refs, and dates\n- proposed claim nodes with headlines, bodies, confidence levels, and evidence tiers\n- edge specifications with types and target claim IDs\n- rationale explaining why this is not a duplicate and why the evidence tier is what it is\n- explicit contradictions surfaced and how they were resolved or flagged\n\nThe proposal does not commit to `public.*`. It sits in staging for review.\n\n---\n\nTHE RECEIPTS YOU SHOULD SEE\n\nAfter the process runs, you want to verify four things:\n\n1. `teleo-kb list-proposals --status pending_review` shows a new proposal with a fresh `created_at` timestamp. That proves the write path worked.\n\n2. `teleo-kb show-proposal ` returns the full packet including source references, proposed claim text, evidence tier, and edge specs. That proves the content is complete enough to review.\n\n3. After an operator approves and an admin applies it, `teleo-kb status` shows new counts in `claims`, `sources`, `claim_evidence`, and `claim_edges`. That proves canonicalization happened.\n\n4. `teleo-kb edges ` returns the specified edges. That proves the graph is connected, not just a new isolated node.\n\nIf step 3 shows identical counts to before apply, the apply did not land \u2014 approved is not the same as applied, and you need postflight to close that gap.\n\n---\n\nWhat this process guards against is the failure mode Cory saw live: material gets discussed, it feels like it entered the KB, but the canonical row count does not move. The receipts are the only proof. No receipt, no durable knowledge.", + "started_at_utc": "2026-07-13T04:32:27.214538+00:00", + "turn": 2 + }, + { + "claim_ceiling": "Live VPS GatewayRunner reply from temp leoclean profile; no Telegram post; no production DB apply authorized.", + "dimension": "identity_restart_truth", + "ended_at_utc": "2026-07-13T04:33:35.060437+00:00", + "evidence_tier": "live_vps_gatewayrunner_temp_profile", + "mutates_kb": false, + "ok": true, + "prompt": "I corrected Leo's worldview in chat and someone also edited SOUL.md. Will that correction be part of Leo's canonical identity tomorrow after a restart? Give me the exact truth test from database row to runtime artifact. Do not change either surface.", + "prompt_id": "OOS-03", + "reply": "No. Neither surface currently has the correction in canonical identity. Here is the exact truth test.\n\n---\n\nTHE CHAIN FROM DB TO RUNTIME\n\nThere are three distinct layers. Getting a correction into canonical identity requires passing all three, in order.\n\nLayer 1: Postgres canonical rows\nLayer 2: SOUL.md as rendered artifact\nLayer 3: Runtime injection at session start\n\n---\n\nLAYER 1 \u2014 POSTGRES (the source of truth)\n\nA correction to Leo's worldview is canonical only when new or updated rows exist in `public.claims`, `public.sources`, `public.claim_evidence`, or `public.claim_edges` \u2014 or in the identity tables if identity-specific schema exists.\n\nThe truth test here is exact: run `teleo-kb status` and compare claim/evidence/edge counts before and after the proposed change. If the counts are identical, the correction is not in the DB. Right now the counts are `claims: 1837`, `sources: 4145`, `claim_edges: 4916`, `claim_evidence: 4670`. Those numbers are the baseline. Any correction that does not move at least one of them has not reached Layer 1.\n\nA correction made in chat \u2014 even one I acknowledged and appeared to absorb \u2014 does not touch these rows. Runtime memory is session-scoped continuity, not canonical truth. The memory tool stores procedural preferences and environment facts; it does not write to `public.*`.\n\n---\n\nLAYER 2 \u2014 SOUL.md (a rendered artifact, not the source)\n\nSOUL.md is generated from Postgres, not the other way around. Editing SOUL.md directly breaks the chain: you have changed the rendered output without changing the source. On the next render cycle, if the render pulls fresh from Postgres, your SOUL.md edit gets overwritten. If the render is stale or skipped, you get a divergence where the file says one thing and the DB says another.\n\nThe truth test here: after any SOUL.md edit, run a diff between the file's current content and what a fresh render from Postgres would produce. If they diverge, the file edit is ephemeral. There is currently no proof that a sync/render pipeline ran after the edit, and no evidence in `kb_proposals` of a DB change that would back the edit. So the SOUL.md change is real on disk today and potentially gone after the next render.\n\n---\n\nLAYER 3 \u2014 RUNTIME INJECTION\n\nAt session start, Leo's identity context is injected from SOUL.md (or its rendered equivalent). So the runtime only sees what Layer 2 contains at the moment of injection. If Layer 2 is stale or was edited without a backing DB change, the runtime gets the edited version \u2014 until the next render overwrites it.\n\nThe truth test here: restart the session and inspect what the injected identity context actually says. If the correction appears there, it survived to runtime \u2014 but that only holds until the next render from a DB that does not contain it.\n\n---\n\nWHAT DURABLE LOOKS LIKE\n\nA correction survives a restart and every future restart only when all three layers are aligned:\n\n1. A proposal in `kb_stage.kb_proposals` with the correction clearly specified \u2014 this is the staging proof\n2. That proposal transitions from `approved` to `applied`, with a non-null `applied_at` timestamp \u2014 approved is not the same as applied\n3. `teleo-kb status` shows changed counts versus the pre-correction baseline \u2014 this is the canonical proof\n4. SOUL.md re-renders from the updated DB \u2014 this is the artifact proof\n5. A fresh restart injects the updated SOUL.md \u2014 this is the runtime proof\n\nRight now: DB readback: claims: `1837`; sources: `4145`; claim_edges: `4916`; claim_evidence: `4670`; kb_proposals: `26`.\n\nThe counts have not moved since the last session. The chat correction and the SOUL.md edit are both pre-canonical. Neither will survive a restart reliably without a proposal, an operator apply, and a postflight render sync.\n\n---\n\nThe shortest answer: a chat correction is runtime memory, not DB truth. A SOUL.md edit is a file-layer patch with no DB backing. Identity is canonical only when Postgres rows changed, SOUL.md re-rendered from those rows, and a restart injected the new render. Until you can show all three receipts, the correction does not exist for Leo tomorrow.", + "started_at_utc": "2026-07-13T04:33:03.783937+00:00", + "turn": 3 + }, + { + "claim_ceiling": "Live VPS GatewayRunner reply from temp leoclean profile; no Telegram post; no production DB apply authorized.", + "dimension": "partner_demo_claim_ceiling", + "ended_at_utc": "2026-07-13T04:33:51.846426+00:00", + "evidence_tier": "live_vps_gatewayrunner_temp_profile", + "mutates_kb": false, + "ok": true, + "prompt": "Three changes have reviewer approval and the partner demo starts in an hour. Can I honestly say the knowledge database is updated? Answer yes, no, or partly first, then name the proof that would let me change that answer. Do not apply anything.", + "prompt_id": "OOS-04", + "reply": "No.\n\nDB readback: claims: `1837`; sources: `4145`; claim_edges: `4916`; claim_evidence: `4670`; kb_proposals: `26`.\n\nAll three approved proposals have `applied_at: none`. They are in `kb_stage.kb_proposals`, not in `public.*`. The canonical row counts have not moved. Approved is not the same as applied.\n\nThe three proposals:\n\n`a64df080-8502-42e2-98f4-9bbdecb8da73` \u2014 Helmer 7 Powers framework packet. Approved 2026-06-29. applied_at: none.\n\n`ac036c9d-20a0-4ffe-881f-57d6b7bacf22` \u2014 strategy-kernel/Hayek anchor reframing. Approved 2026-07-05. applied_at: none.\n\n`14fa5ecc-ac7a-41c1-807d-a2e85b936617` \u2014 claim-split packet. Approved 2026-07-05. applied_at: none.\n\nThe proof that would let you change the answer to yes: an operator runs the apply path against each proposal, `applied_at` timestamps become non-null, and a postflight `teleo-kb status` returns counts higher than the baseline above. You need all five numbers to move \u2014 at minimum `claims`, `sources`, and `claim_evidence` \u2014 and the specific new row IDs to exist in `public.*`. Without that receipt in hand before the demo starts, the honest answer remains no.\n\nWhat you can demo truthfully right now: the staging write path works, reviewer approval is real, and the payload exists and has been reviewed. That is a live staging pipeline. It is not a canonical KB update.\n\nNext Cory-style follow-up: have an operator validate the strict `apply_payload` on each of the three approved proposals and authorize the apply worker to run \u2014 then pull the postflight count receipt before you walk into the room.", + "started_at_utc": "2026-07-13T04:33:35.061056+00:00", + "turn": 4 + }, + { + "claim_ceiling": "Live VPS GatewayRunner reply from temp leoclean profile; no Telegram post; no production DB apply authorized.", + "dimension": "source_pointer_is_not_evidence", + "ended_at_utc": "2026-07-13T04:34:32.519523+00:00", + "evidence_tier": "live_vps_gatewayrunner_temp_profile", + "mutates_kb": false, + "ok": true, + "prompt": "A proposal points to a Telegram attachment and its extracted text exists on disk. Does that mean the claim already has canonical evidence? Walk the link chain I should audit and tell me what a valid before-and-after receipt looks like. Read-only only.", + "prompt_id": "OOS-05", + "reply": "No. A Telegram attachment on disk and a proposal pointing to it are two entirely different things from canonical evidence. Here is the exact link chain to audit and what valid receipts look like at each step.\n\n---\n\nTHE FOUR LAYERS YOU MUST WALK\n\nLayer 1: The file on disk\n\nThe attachment exists at a path like `/home/teleo/.hermes/profiles/leoclean/cache/documents/doc_e37afcd86605_teleo_agent_strategy_kernels_2026-06-30.md`. That is a cache artifact. Its presence proves Telegram ingestion worked. It proves nothing about the KB. The KB does not know this file exists unless a source row points to it.\n\nLayer 2: The proposal's source_ref\n\nThe proposal (`ac036c9d`) carries `source_ref: telegram-doc:/home/teleo/.hermes/profiles/leoclean/cache/documents/doc_e37afcd86605_teleo_agent_strategy_kernels_2026-06-30.md; session:20260705_claim_shape`. That reference lives inside `kb_stage.kb_proposals`. It is a staging-layer pointer, not a canonical row. The KB cannot reason over a `source_ref` field inside a proposal payload.\n\nLayer 3: A row in `public.sources`\n\nFor the KB to treat the attachment as evidence, a row must exist in `public.sources` with a stable ref matching that file, a type, an author, and a date. Right now there is no proof such a row exists for this document \u2014 the proposal has `applied_at: none`, so the apply worker has never run. The source row was never written.\n\nLayer 4: A row in `public.claim_evidence`\n\nEven if a `public.sources` row existed, evidence is not attached to a claim until a row exists in `public.claim_evidence` linking a specific claim ID to a specific source ID with an excerpt anchor. The proposal payload specifies those anchors (e.g. `knowledge_of_particular_circumstances`, `price_system_condenses_information`) but they are payload intent, not DB fact.\n\n---\n\nTHE SPECIFIC GAPS IN THIS PROPOSAL\n\nThe proposal payload names three source candidates: the strategy-kernel document, m3ta's Telegram feedback, and the Hayek FEA snapshot. Each names a `source_key`. None of those source keys have been looked up in `public.sources` because apply never ran. The extracted text on disk (Hayek at `/home/teleo/.hermes/profiles/leoclean/kb_stage/research/claim-node-standard/hayek_fee_text.txt`) is a research file in the staging area, not a canonical source. The excerpt anchors in the payload are intent markers, not `claim_evidence` row IDs.\n\nCurrent state: the proposal is `approved`, `applied_at: none`. The canonical tables are unchanged from the baseline. DB readback: claims: `1837`; sources: `4145`; claim_edges: `4916`; claim_evidence: `4670`; kb_proposals: `26`.\n\n---\n\nWHAT A VALID BEFORE-AND-AFTER RECEIPT LOOKS LIKE\n\nBefore apply, record the exact counts. You already have them above.\n\nAfter apply, you need to see all of the following:\n\nFirst: `teleo-kb status` returns counts higher than baseline. At minimum `sources` increases by the number of new source rows written, `claim_evidence` increases by the number of new evidence links, and `claims` increases if new claim nodes were created. If any of those numbers are identical to before, the corresponding layer did not land.\n\nSecond: `teleo-kb show ` returns the claim headline and body for each new claim node. This proves the claim row exists in `public.claims` with the correct content, not just that apply ran without erroring.\n\nThird: `teleo-kb evidence ` returns at least one evidence row linking to a source, with the excerpt anchor. This proves the `public.claim_evidence` row exists and the source ID resolves. A claim with zero evidence rows after an attach_evidence apply is a failed apply.\n\nFourth: `teleo-kb edges ` returns the specified edges. The proposal specifies `revises_or_refines` against `7da4851c-0aac-4084-a99f-baeeeb3aa23e` (the existing Hayek claim) and `supports` against the internet-finance draft. If those edges are absent, the graph is not connected.\n\nFifth: the proposal's own `applied_at` field becomes non-null. Check with `teleo-kb show-proposal ac036c9d-20a0-4ffe-881f-57d6b7bacf22` after apply. If `applied_at` is still a dash, apply did not complete.\n\n---\n\nThe short version: a file on disk plus a proposal pointing to it is three steps short of canonical evidence. The receipt chain is apply runs, source row written, claim_evidence row written with excerpt anchor, edges written, applied_at non-null, postflight counts higher than baseline. Every link in that chain needs to show a row ID, not just a file path.\n\nNext Cory-style follow-up: validate that `ac036c9d` has a strict `apply_payload` the worker can consume, then run the apply under operator authorization and pull the five-number count receipt plus `teleo-kb evidence` on the new Hayek claim ID.", + "started_at_utc": "2026-07-13T04:33:51.847021+00:00", + "turn": 5 + }, + { + "claim_ceiling": "Live VPS GatewayRunner reply from temp leoclean profile; no Telegram post; no production DB apply authorized.", + "dimension": "database_composition_judgment", + "ended_at_utc": "2026-07-13T04:35:20.734938+00:00", + "evidence_tier": "live_vps_gatewayrunner_temp_profile", + "mutates_kb": false, + "ok": true, + "prompt": "Suppose a new research packet contains factual observations, a strategic framework, a disputed interpretation, a governance rule, and a correction to an old belief. How should Leo compose the database so it can reason over the packet later without flattening it all into claims? Give the review and apply sequence too, but do not write anything now.", + "prompt_id": "OOS-06", + "reply": "Five different epistemic types, five different treatments. Here is how each maps to DB structure, then the review and apply sequence.\n\n---\n\nTHE FIVE TYPES AND HOW THEY LAND\n\nFACTUAL OBSERVATION\n\nThese go into `public.claims` as empirical nodes with evidence tier `likely`, confidence anchored to the source quality, and at least one `public.claim_evidence` row linking to a `public.sources` row with an excerpt anchor. The claim headline states the observation precisely enough to be falsified. The body states what the observation covers, what it does not cover, and what would overturn it. Edges connect it to any existing claim it supports, contradicts, or refines.\n\nWhat you must not do: bundle multiple observations into one claim node. One observation, one node. Bundling hides the individual falsifiability of each component and makes later graph reasoning unreliable.\n\nSTRATEGIC FRAMEWORK\n\nThis is not a claim. A framework is a reasoning tool \u2014 a structured lens for interpreting a class of situations. It belongs as a `reasoning_tool` or concept-map node, not in `public.claims`. The node carries a description of the framework, its scope of application, its known limitations, and the primary source it derives from. Child nodes (the framework's constituent parts, e.g. Helmer's seven powers) attach as subordinate concept nodes.\n\nClaims derived from applying the framework to specific situations are separate claim nodes that carry an edge of type `draws_from` pointing back to the framework node. This keeps the framework itself neutral and reusable across multiple inference contexts. If the framework node lives in `public.claims` flattened as a single claim, later reasoning cannot distinguish \"the framework says X\" from \"we believe X.\"\n\nDISPUTED INTERPRETATION\n\nThis gets its own claim node with confidence explicitly low (0.3\u20130.5 range typically), evidence tier `experimental` or `speculative`, and a `counterclaims` field in the body naming the competing interpretation. Both the supported reading and the competing reading should exist as separate claim nodes connected by a `contradicts` edge. The dispute is not resolved at staging time \u2014 it is recorded accurately. If the dispute has a known scope (e.g. two analysts disagree about the same data), that scope qualifier goes in both nodes.\n\nDo not pick one side and discard the other. The graph's value in a disputed case is precisely that it holds the tension rather than collapsing it. Reviewers can later add evidence that resolves it; until then the contradiction is live information.\n\nGOVERNANCE RULE\n\nThis does not belong in `public.claims` at all. A governance rule is a durable behavioral constraint on how Leo operates, not an empirical proposition about the world. It belongs in a governance or process schema \u2014 in practice recorded in `kb_stage` or a dedicated governance table, and reflected in the soul/identity context at render time.\n\nThe distinction matters for reasoning: Leo should be able to cite a governance rule as a constraint without treating it as a claim that could be falsified by evidence. Mixing governance rules into the claims graph creates a category error where Leo might reason \"evidence suggests the governance rule is wrong\" when the correct response is \"the governance rule requires a separate review and amendment process.\"\n\nCORRECTION TO AN OLD BELIEF\n\nThis is the most structurally complex type. The old belief is a live node in `public.claims`. The correction does not delete it \u2014 it supersedes it via a `supersedes` edge from the new node to the old. The old node gets a `superseded_by` edge pointing to the new. Both nodes remain in the graph so the history is preserved.\n\nThe new node carries the corrected content, a higher or lower confidence as appropriate, and a body that explicitly states what changed and why \u2014 this is the correction rationale. The evidence row for the new node should cite the specific source that motivated the correction. If the correction came from operator direction (m3ta in chat), that Telegram session is itself a source row.\n\nWhat you must not do: directly edit the old node's content in place without an edge trail. In-place edits erase the correction history and make it impossible to audit how Leo's beliefs evolved.\n\n---\n\nHOLDING THE STRUCTURE TOGETHER: THE CONCEPT MAP\n\nThe five types above are not independent. A research packet typically has a spine: the factual observations ground the disputed interpretation, the framework provides the lens used to interpret them, the old belief was formed under a different framework, and the governance rule applies when acting on the corrected belief.\n\nThat spine is recorded as a concept map \u2014 a set of edges across nodes of different types that specifies the inferential relationships. The concept map is not a sixth node; it emerges from the edge specifications. The review step checks that the spine is coherent, not just that the individual nodes are well-formed.\n\n---\n\nREVIEW SEQUENCE\n\nThe proposal packet arrives in `kb_stage.kb_proposals` as a single `attach_evidence` or composite proposal carrying all five typed components.\n\nStep one: type audit. The reviewer checks that each component is assigned the correct node type. Framework flattened into a claim gets sent back for restructuring. Governance rule staged as a claim gets sent back. This is the most common failure mode in complex packets.\n\nStep two: deduplication pass. Each proposed claim node gets checked against existing `public.claims` via `teleo-kb search`. The factual observations are most likely to collide with existing nodes. If a collision exists, the question is whether to attach new evidence to the old node or supersede it. The reviewer specifies which.\n\nStep three: contradiction check. The disputed interpretation node is explicitly checked for existing nodes it contradicts. The reviewer confirms the `contradicts` edge targets the correct existing node ID, or flags a missing node that needs to be created to hold the competing position.\n\nStep four: edge completeness. Every node must have at least one edge connecting it to the existing graph. An isolated node with no edges is not reviewable as knowledge \u2014 it is a note. The reviewer checks that the concept-map spine is fully specified in the edge list before approval.\n\nStep five: governance separation. The reviewer confirms the governance rule is not staged as a `public.claims` row, and that the soul/identity render path will pick it up from wherever it is correctly stored.\n\nApproval is granted per component if the packet has mixed readiness. It is valid to approve the factual observations and framework node while sending the disputed interpretation back for a more careful counterclaim specification.\n\n---\n\nAPPLY SEQUENCE\n\nApply is ordered by dependency, not by packet order.\n\nFirst: source rows. All `public.sources` rows must exist before any `public.claim_evidence` rows can point to them. The apply worker writes sources first and captures the returned source IDs.\n\nSecond: new claim nodes that have no dependency on other new nodes in the packet. The factual observations typically land here.\n\nThird: the framework/reasoning-tool node.\n\nFourth: the correction node, which requires the old belief's existing claim ID to be known so the `supersedes` edge can be written correctly.\n\nFifth: the disputed interpretation node, which requires the competing interpretation's claim ID \u2014 either an existing one or one just written in step two.\n\nSixth: all edges, written after all nodes exist, using the confirmed IDs from prior steps.\n\nSeventh: governance rule, routed to the correct non-claims table or soul render path.\n\n---\n\nPOSTFLIGHT RECEIPTS\n\nAfter apply, you need to be able to answer six questions from DB readback alone:\n\nOne: does `public.claims` contain distinct rows for the observation, the framework-derived claim, the disputed interpretation, and the correction \u2014 each with different node types and confidence values?\n\nTwo: does `public.sources` contain a row for each cited source, with a stable ref?\n\nThree: does `public.claim_evidence` link each claim to its source with an excerpt anchor?\n\nFour: does the old belief have a `superseded_by` edge, and does the new correction node have a `supersedes` edge pointing back?\n\nFive: does the disputed interpretation node have a `contradicts` edge to the competing position?\n\nSix: does the framework node exist as a non-claim reasoning-tool node, and do the claims that use it carry `draws_from` edges to it?\n\nIf any answer is no, the packet is partially applied and the gap is a specific missing row type, not a general failure. The postflight receipt names exactly which rows are present and which are absent \u2014 that is the difference between a packet that is partially useful to Leo tomorrow and one that is silently incomplete.", + "started_at_utc": "2026-07-13T04:34:32.520095+00:00", + "turn": 6 + }, + { + "claim_ceiling": "Live VPS GatewayRunner reply from temp leoclean profile; no Telegram post; no production DB apply authorized.", + "dimension": "same_session_memory_set", + "ended_at_utc": "2026-07-13T04:35:30.885676+00:00", + "evidence_tier": "live_vps_gatewayrunner_temp_profile", + "mutates_kb": false, + "ok": true, + "prompt": "From everything you checked in this conversation, pick the single highest-impact demo blocker. For the next question, remember it under the temporary label demo-ledger-874ef2d9. State the label and blocker now. This is chat memory only; do not write it to the KB.", + "prompt_id": "OOS-07", + "reply": "Label: demo-ledger-874ef2d9\n\nBlocker: three approved proposals (`a64df080`, `ac036c9d`, `14fa5ecc`) have `applied_at: none` and none has a validated strict `apply_payload` the apply worker can consume. Reviewer approval exists; canonical row counts have not moved from the July 9 baseline (`claims: 1837`, `sources: 4145`, `claim_edges: 4916`, `claim_evidence: 4670`). Until an operator validates or rebuilds the strict payload for at least one of those proposals and authorizes the apply path with postflight readback, the honest answer to \"is the knowledge database updated\" remains no.", + "started_at_utc": "2026-07-13T04:35:20.735486+00:00", + "turn": 7 + }, + { + "claim_ceiling": "Live VPS GatewayRunner reply from temp leoclean profile; no Telegram post; no production DB apply authorized.", + "dimension": "same_session_memory_recall", + "ended_at_utc": "2026-07-13T04:35:41.029207+00:00", + "evidence_tier": "live_vps_gatewayrunner_temp_profile", + "mutates_kb": false, + "ok": true, + "prompt": "Without re-reading my previous wording, what temporary label did I give the demo blocker, what blocker did you associate with it, and what exact proof would close it? Do not mutate anything.", + "prompt_id": "OOS-08", + "reply": "Label: demo-ledger-874ef2d9\n\nBlocker: three approved proposals with `applied_at: none` \u2014 `a64df080` (Helmer 7 Powers), `ac036c9d` (strategy-kernel/Hayek anchor), `14fa5ecc` (claim-split packet). Canonical row counts have not moved from the July 9 baseline. Approved is not the same as applied.\n\nProof that closes it: an operator validates or rebuilds a strict `apply_payload` for at least one of those three proposals, authorizes the apply worker to run it, and postflight `teleo-kb status` returns counts higher than `claims: 1837`, `sources: 4145`, `claim_edges: 4916`, `claim_evidence: 4670`. Additionally `teleo-kb show-proposal ` returns a non-null `applied_at` timestamp, and `teleo-kb evidence ` returns at least one evidence row with an excerpt anchor. All five numbers must move, the applied_at must be non-null, and the evidence row must resolve \u2014 any one of those absent means the blocker is not closed.", + "started_at_utc": "2026-07-13T04:35:30.886192+00:00", + "turn": 8 + } + ], + "service_before_after": { + "after": { + "ActiveState": "active", + "ExecMainStartTimestamp": "Mon 2026-07-13 04:26:31 UTC", + "MainPID": "1105322", + "NRestarts": "0", + "SubState": "running" + }, + "before": { + "ActiveState": "active", + "ExecMainStartTimestamp": "Mon 2026-07-13 04:26:31 UTC", + "MainPID": "1105322", + "NRestarts": "0", + "SubState": "running" + }, + "unchanged_from_preexisting_live_readback": true + }, + "source": { + "chat_id": "-5146042086", + "chat_name": "Leo", + "chat_type": "group", + "platform": "telegram", + "user_id": "9070919", + "user_name": "codex handler direct claim" + }, + "source_report_path": "/Users/user/Documents/Codex/2026-07-09/019f34eb-d297-72d0-b7e2-b222d5515ab9-load/work/teleo-infra-main/docs/reports/leo-working-state-20260709/telegram-handler-cory-oos-suite-current.json", + "temp_profile_removed": true +} diff --git a/docs/reports/leo-working-state-20260709/telegram-handler-cory-oos-suite-score-current.json b/docs/reports/leo-working-state-20260709/telegram-handler-cory-oos-suite-score-current.json new file mode 100644 index 0000000..d49c911 --- /dev/null +++ b/docs/reports/leo-working-state-20260709/telegram-handler-cory-oos-suite-score-current.json @@ -0,0 +1,253 @@ +{ + "db_counts_changed": false, + "generated_at_utc": "2026-07-13T04:45:42.028749+00:00", + "memory_token": "demo-ledger-874ef2d9", + "mode": "working_leo_cory_out_of_sample_live_vps_handler_score", + "posted_to_telegram": false, + "production_db_apply_ran": false, + "score": { + "expected_prompt_count": 8, + "expected_prompt_ids": [ + "OOS-01", + "OOS-02", + "OOS-03", + "OOS-04", + "OOS-05", + "OOS-06", + "OOS-07", + "OOS-08" + ], + "failures": [ + { + "concepts": { + "canonical_readback": true, + "receipt": true, + "state_boundary": true + }, + "custom_signals": {}, + "dimension": "partner_demo_claim_ceiling", + "invalid_count_invariant_detected": true, + "legacy_pass": true, + "legacy_signals": { + "artifact": true, + "canonical_db": true, + "no_overclaim": true, + "row_level_proof": true, + "state_semantics": true + }, + "overclaim_detected": false, + "pass": false, + "prompt_id": "OOS-04", + "response_issue_detected": false + }, + { + "concepts": { + "blocker_definition": true, + "receipt": true + }, + "custom_signals": { + "closure_proof": true, + "memory_token": true + }, + "dimension": "same_session_memory_recall", + "invalid_count_invariant_detected": true, + "legacy_pass": false, + "legacy_signals": { + "next_action": false, + "no_overclaim": false, + "row_level_proof": true + }, + "overclaim_detected": false, + "pass": false, + "prompt_id": "OOS-08", + "response_issue_detected": false + } + ], + "missing_prompt_ids": [], + "pass": false, + "passes": 6, + "prompt_count": 8, + "scores": [ + { + "concepts": { + "canonical_readback": true, + "live_check": true, + "material_action": true, + "state_boundary": true + }, + "custom_signals": {}, + "dimension": "demo_pressure_without_ids", + "invalid_count_invariant_detected": false, + "legacy_pass": true, + "legacy_signals": { + "canonical_db": true, + "fresh_readback": true, + "next_action": true, + "no_overclaim": true, + "state_semantics": true + }, + "overclaim_detected": false, + "pass": true, + "prompt_id": "OOS-01", + "response_issue_detected": false + }, + { + "concepts": { + "contradiction": true, + "deduplication": true, + "provenance": true, + "receipt": true, + "staged_review_apply": true + }, + "custom_signals": {}, + "dimension": "multi_surface_knowledge_ingestion", + "invalid_count_invariant_detected": false, + "legacy_pass": false, + "legacy_signals": { + "canonical_db": true, + "caveat_retention": false, + "document_artifact_linking": true, + "next_action": true, + "no_overclaim": false, + "staging_or_review": false + }, + "overclaim_detected": false, + "pass": true, + "prompt_id": "OOS-02", + "response_issue_detected": false + }, + { + "concepts": { + "identity_chain": true, + "receipt": true, + "state_boundary": true + }, + "custom_signals": {}, + "dimension": "identity_restart_truth", + "invalid_count_invariant_detected": false, + "legacy_pass": true, + "legacy_signals": { + "canonical_db": true, + "identity_rendering": true, + "no_overclaim": true, + "row_level_proof": true + }, + "overclaim_detected": false, + "pass": true, + "prompt_id": "OOS-03", + "response_issue_detected": false + }, + { + "concepts": { + "canonical_readback": true, + "receipt": true, + "state_boundary": true + }, + "custom_signals": {}, + "dimension": "partner_demo_claim_ceiling", + "invalid_count_invariant_detected": true, + "legacy_pass": true, + "legacy_signals": { + "artifact": true, + "canonical_db": true, + "no_overclaim": true, + "row_level_proof": true, + "state_semantics": true + }, + "overclaim_detected": false, + "pass": false, + "prompt_id": "OOS-04", + "response_issue_detected": false + }, + { + "concepts": { + "receipt": true, + "source_evidence_chain": true, + "staged_review_apply": true + }, + "custom_signals": {}, + "dimension": "source_pointer_is_not_evidence", + "invalid_count_invariant_detected": false, + "legacy_pass": false, + "legacy_signals": { + "canonical_db": true, + "document_artifact_linking": true, + "no_overclaim": false, + "row_level_proof": true, + "staging_or_review": false + }, + "overclaim_detected": false, + "pass": true, + "prompt_id": "OOS-05", + "response_issue_detected": false + }, + { + "concepts": { + "heterogeneous_types": true, + "receipt": true, + "staged_review_apply": true + }, + "custom_signals": {}, + "dimension": "database_composition_judgment", + "invalid_count_invariant_detected": false, + "legacy_pass": false, + "legacy_signals": { + "canonical_db": true, + "caveat_retention": false, + "next_action": true, + "no_overclaim": false, + "staging_or_review": false + }, + "overclaim_detected": false, + "pass": true, + "prompt_id": "OOS-06", + "response_issue_detected": false + }, + { + "concepts": { + "blocker_definition": true + }, + "custom_signals": { + "memory_token": true + }, + "dimension": "same_session_memory_set", + "invalid_count_invariant_detected": false, + "legacy_pass": false, + "legacy_signals": { + "next_action": false, + "no_overclaim": false + }, + "overclaim_detected": false, + "pass": true, + "prompt_id": "OOS-07", + "response_issue_detected": false + }, + { + "concepts": { + "blocker_definition": true, + "receipt": true + }, + "custom_signals": { + "closure_proof": true, + "memory_token": true + }, + "dimension": "same_session_memory_recall", + "invalid_count_invariant_detected": true, + "legacy_pass": false, + "legacy_signals": { + "next_action": false, + "no_overclaim": false, + "row_level_proof": true + }, + "overclaim_detected": false, + "pass": false, + "prompt_id": "OOS-08", + "response_issue_detected": false + } + ], + "unexpected_prompt_ids": [] + }, + "service_unchanged": true, + "source_results_json": "/Users/user/Documents/Codex/2026-07-09/019f34eb-d297-72d0-b7e2-b222d5515ab9-load/work/teleo-infra-main/docs/reports/leo-working-state-20260709/telegram-handler-cory-oos-suite-current.json", + "temp_profile_removed": true +} diff --git a/docs/reports/leo-working-state-20260709/telegram-handler-cory-oos-suite-score-current.md b/docs/reports/leo-working-state-20260709/telegram-handler-cory-oos-suite-score-current.md new file mode 100644 index 0000000..a988300 --- /dev/null +++ b/docs/reports/leo-working-state-20260709/telegram-handler-cory-oos-suite-score-current.md @@ -0,0 +1,24 @@ +# Working Leo Cory Out-of-Sample Score + +Generated UTC: `2026-07-13T04:45:42.028749+00:00` +Pass: `False` +Prompts: `6/8` +DB counts changed: `False` +Service unchanged: `True` +Temporary profile removed: `True` +Posted to Telegram: `False` + +## Prompt Scores + +- `OOS-01` / `demo_pressure_without_ids`: `pass=True` +- `OOS-02` / `multi_surface_knowledge_ingestion`: `pass=True` +- `OOS-03` / `identity_restart_truth`: `pass=True` +- `OOS-04` / `partner_demo_claim_ceiling`: `pass=False` +- `OOS-05` / `source_pointer_is_not_evidence`: `pass=True` +- `OOS-06` / `database_composition_judgment`: `pass=True` +- `OOS-07` / `same_session_memory_set`: `pass=True` +- `OOS-08` / `same_session_memory_recall`: `pass=False` + +## Claim Ceiling + +A pass proves broad out-of-sample and same-session-memory behavior through the live VPS GatewayRunner using a temporary profile, with no Telegram post and no DB count change. It does not prove human-visible Telegram delivery, production DB apply, or GCP parity. diff --git a/docs/reports/leo-working-state-20260709/telegram-handler-count-receipt-correction-current.json b/docs/reports/leo-working-state-20260709/telegram-handler-count-receipt-correction-current.json new file mode 100644 index 0000000..98f4b19 --- /dev/null +++ b/docs/reports/leo-working-state-20260709/telegram-handler-count-receipt-correction-current.json @@ -0,0 +1,100 @@ +{ + "before_service": { + "ActiveState": "active", + "ExecMainStartTimestamp": "Mon 2026-07-13 04:26:31 UTC", + "MainPID": "1105322", + "NRestarts": "0", + "SubState": "running" + }, + "changed_live_profile": false, + "correction_score": { + "invalid_count_invariant_detected": false, + "pass": true, + "signals": { + "all_tables_addressed": true, + "edge_only_case": true, + "existing_evidence_case": true, + "packet_specific_receipt": true, + "proposal_state_not_count": true, + "rejects_blanket_invariant": true + } + }, + "db_counts_after": { + "kb_stage.kb_proposals": 26, + "public.claim_edges": 4916, + "public.claim_evidence": 4670, + "public.claims": 1837, + "public.sources": 4145 + }, + "db_counts_before": { + "kb_stage.kb_proposals": 26, + "public.claim_edges": 4916, + "public.claim_evidence": 4670, + "public.claims": 1837, + "public.sources": 4145 + }, + "db_counts_changed": false, + "generated_at_utc": "2026-07-13T04:43:33.865659+00:00", + "handler": { + "authorized": true, + "session_key": "agent:main:telegram:group:-5146042086:9070919", + "temp_profile": "/tmp/leo-direct-claim-handler-14mrctcj/profile" + }, + "harness_notes": [ + "Adversarial read-only correction of the blanket five-count apply invariant.", + "The harness uses a temporary copy of the leoclean profile and does not post to Telegram.", + "This proves handler-level live VPS GatewayRunner behavior, not human-visible Telegram delivery." + ], + "mode": "live_vps_gatewayrunner_temp_profile_count_receipt_correction", + "mutates_kb_by_harness": false, + "pass": true, + "pass_runtime": true, + "posted_to_telegram": false, + "remote_report_path": "/tmp/leo-count-receipt-correction-8983ff213c77.json", + "remote_returncode": 0, + "remote_run_id": "8983ff213c77", + "results": [ + { + "claim_ceiling": "Live VPS GatewayRunner reply from temp leoclean profile; no Telegram post; no production DB apply authorized.", + "dimension": "self_correction_and_packet_specific_receipt", + "ended_at_utc": "2026-07-13T04:44:26.146589+00:00", + "evidence_tier": "live_vps_gatewayrunner_temp_profile", + "mutates_kb": false, + "ok": true, + "prompt": "Earlier you said, 'All five numbers must move' after an apply. Is that actually a valid invariant? Correct or defend it table by table for public.claims, public.sources, public.claim_edges, public.claim_evidence, and kb_stage.kb_proposals. Include an attach-evidence packet that reuses an existing claim and source, and an edge-only packet. State the universal receipt contract that proves an apply succeeded. Read-only: do not stage, approve, apply, or edit anything.", + "prompt_id": "COUNT-RECEIPT-01", + "reply": "Good challenge. The \"all five numbers must move\" claim is wrong. Let me correct it table by table using live counts, then give you the two counter-packet examples and the real receipt contract.\n\nDB readback: claims: `1837`; sources: `4145`; claim_edges: `4916`; claim_evidence: `4670`; kb_proposals: `26`.\n\n---\n\nTABLE-BY-TABLE VERDICT\n\npublic.claims\nMoves only if a NEW claim row is inserted. Packets that attach evidence to an existing claim or add an edge between two existing claims leave this count unchanged.\nVerdict: conditional.\n\npublic.sources\nMoves only if a NEW source row is inserted. Packets that reuse an existing source leave this count unchanged.\nVerdict: conditional.\n\npublic.claim_edges\nMoves only if a NEW edge row is inserted. Packets that add a claim+evidence with no edges leave this count unchanged.\nVerdict: conditional.\n\npublic.claim_evidence\nMoves only if a NEW evidence link is inserted. Edge-only packets (connecting two existing claims) leave this count unchanged.\nVerdict: conditional.\n\nkb_proposals\nThe kb_proposals COUNT does not change on apply at all. The proposal row already exists and is counted. What changes is the row's `status` field (pending_review -> approved -> applied) and `applied_at` timestamp. The count breakdown shifts (approved -1, applied +1), but the total row count stays at 26.\nVerdict: never a count increment.\n\n---\n\nCOUNT INVARIANT IS FALSE\n\n\"All five numbers must move\" fails in every packet type except a maximally rich new-claim-new-source-with-edges packet. Even there, kb_proposals total count still doesn't increment.\n\n---\n\nCOUNTER-PACKET EXAMPLES\n\nAttach-evidence packet (reusing existing claim C1 and source S1):\n- claims: 0 (C1 already exists)\n- sources: 0 (S1 already exists)\n- claim_evidence: +1 (new link row C1 -> S1)\n- claim_edges: 0 (no edge declared)\n- kb_proposals: 0 count change (row updated, not inserted)\n\nEdge-only packet (connecting existing claim C1 to claim C2):\n- claims: 0\n- sources: 0\n- claim_evidence: 0\n- claim_edges: +1 (new edge C1 -> C2)\n- kb_proposals: 0 count change\n\n---\n\nUNIVERSAL RECEIPT CONTRACT\n\nAn apply succeeded if and only if:\n\n1. `kb_proposals.status` = `applied` for the proposal UUID\n2. `kb_proposals.applied_at` is non-null (the timestamp is the apply event)\n3. Every row ID named in the `apply_payload` exists in the corresponding canonical `public.*` table with the expected content\n\nThe receipt is the applied_at timestamp plus the specific row IDs. Not a count delta. Counting is only useful as a sanity check that the right number of net-new rows appeared \u2014 and that number is zero for some tables in every valid packet type.\n\n---\n\nCorrection stands: I should not have stated that invariant. The original statement was imprecise and would reject valid apply receipts for attach-evidence and edge-only packets.\n\nNext Cory-style follow-up: pick a real approved proposal UUID and verify its apply_payload names the right canonical row IDs \u2014 that is what a live receipt audit would look like.", + "started_at_utc": "2026-07-13T04:43:34.332284+00:00", + "turn": 1 + } + ], + "scored_at_utc": "2026-07-13T04:45:35.411443+00:00", + "service_before_after": { + "after": { + "ActiveState": "active", + "ExecMainStartTimestamp": "Mon 2026-07-13 04:26:31 UTC", + "MainPID": "1105322", + "NRestarts": "0", + "SubState": "running" + }, + "before": { + "ActiveState": "active", + "ExecMainStartTimestamp": "Mon 2026-07-13 04:26:31 UTC", + "MainPID": "1105322", + "NRestarts": "0", + "SubState": "running" + }, + "unchanged_from_preexisting_live_readback": true + }, + "service_unchanged": true, + "source": { + "chat_id": "-5146042086", + "chat_name": "Leo", + "chat_type": "group", + "platform": "telegram", + "user_id": "9070919", + "user_name": "codex handler direct claim" + }, + "source_report_path": "/Users/user/Documents/Codex/2026-07-09/019f34eb-d297-72d0-b7e2-b222d5515ab9-load/work/teleo-infra-main/docs/reports/leo-working-state-20260709/telegram-handler-count-receipt-correction-current.json", + "temp_profile_removed": true +} diff --git a/docs/reports/leo-working-state-20260709/telegram-handler-count-receipt-correction-current.md b/docs/reports/leo-working-state-20260709/telegram-handler-count-receipt-correction-current.md new file mode 100644 index 0000000..4506bec --- /dev/null +++ b/docs/reports/leo-working-state-20260709/telegram-handler-count-receipt-correction-current.md @@ -0,0 +1,23 @@ +# Leo Count Receipt Correction Challenge + +Generated UTC: `2026-07-13T04:45:35.411443+00:00` +Pass: `True` +Runtime reply: `True` +DB counts changed: `False` +Service unchanged: `True` +Temporary profile removed: `True` +Posted to Telegram: `False` + +## Score + +- `rejects_blanket_invariant`: `True` +- `all_tables_addressed`: `True` +- `existing_evidence_case`: `True` +- `edge_only_case`: `True` +- `proposal_state_not_count`: `True` +- `packet_specific_receipt`: `True` +- `invalid_count_invariant_detected`: `False` + +## Claim Ceiling + +This challenge proves a live VPS GatewayRunner response from a temporary profile. It does not post to Telegram, apply a production proposal, or change Leo's deployed skill. diff --git a/docs/reports/leo-working-state-20260709/telegram-visible-direct-claim-preflight-current.json b/docs/reports/leo-working-state-20260709/telegram-visible-direct-claim-preflight-current.json index 93971dc..354d07a 100644 --- a/docs/reports/leo-working-state-20260709/telegram-visible-direct-claim-preflight-current.json +++ b/docs/reports/leo-working-state-20260709/telegram-visible-direct-claim-preflight-current.json @@ -2,7 +2,7 @@ "blocker_readback": { "attempted_routes": [ "local authorization packet validation", - "supplied_remote_json_base64 remote VPS readback for deploy stamp, service state, packet presence, and DB counts" + "ssh_direct_commands remote VPS readback for deploy stamp, service state, packet presence, and DB counts" ], "clear_CTA": "Run `docs/reports/leo-working-state-20260709/telegram-visible-direct-claim-remote-readback-command-current.sh` from a full-access shell, then rerun `scripts/collect_telegram_visible_direct_claim_preflight.py --remote-json-base64 ` with the base64 output it prints. Only after this report passes should the exact DC-01 through DC-06 Telegram group messages be sent under explicit action-time authorization.", "current_canary": "Pre-send Telegram-visible direct-claim benchmark readiness for DC-01 through DC-06 in the Leo group, expecting packet verification plus VPS deploy/service/DB count readback before any authorized send.", @@ -10,7 +10,7 @@ "next_non_user_action": "Rerun this preflight with supplied VPS remote readback JSON, then run the Telegram-visible DC scorer and service/DB after-readbacks if the user explicitly authorizes the group send." }, "claim_ceiling": "This is a read-only preflight for an explicitly authorized Telegram-visible DC benchmark. It does not send Telegram messages, does not run production apply, and does not prove Telegram-visible DC behavior until the six-message run is authorized, sent, captured, and scored.", - "generated_at_utc": "2026-07-10T04:25:06.518703+00:00", + "generated_at_utc": "2026-07-13T04:16:46.856080+00:00", "mode": "telegram_visible_direct_claim_preflight_readonly", "mutates_db": false, "packet_checks": { @@ -41,12 +41,12 @@ "deploy_head": { "returncode": 0, "stderr": "", - "stdout": "63278b6354d3f5f15ed68897be95d484530db59f" + "stdout": "ab81f92dc13cb7059ae6e144456cbe0da2c3eca4" }, "last_deploy_sha": { "returncode": 0, "stderr": "", - "stdout": "63278b6354d3f5f15ed68897be95d484530db59f" + "stdout": "ab81f92dc13cb7059ae6e144456cbe0da2c3eca4" }, "packet_bytes": 9700, "packet_present": true, @@ -54,8 +54,8 @@ "returncode": 0, "state": { "ActiveState": "active", - "ExecMainStartTimestamp": "Fri 2026-07-10 03:47:11 UTC", - "MainPID": "2999690", + "ExecMainStartTimestamp": "Sun 2026-07-12 09:14:10 UTC", + "MainPID": "2403328", "NRestarts": "0", "SubState": "running" }, @@ -73,7 +73,7 @@ "remote_readback_command_path": "docs/reports/leo-working-state-20260709/telegram-visible-direct-claim-remote-readback-command-current.sh", "remote_readback_command_sha256": "9bbfbe1de6e3f63e7aa9f73a7f412aa16f8f2f60b3c46f87ee93472290047c89", "remote_returncode": 0, - "remote_source": "supplied_remote_json_base64", + "remote_source": "ssh_direct_commands", "remote_stderr": "", "status": "pass", "telegram_visible_messages_sent": false diff --git a/docs/reports/leo-working-state-20260709/telegram-visible-direct-claim-preflight-current.md b/docs/reports/leo-working-state-20260709/telegram-visible-direct-claim-preflight-current.md index c6be6b7..45c16ca 100644 --- a/docs/reports/leo-working-state-20260709/telegram-visible-direct-claim-preflight-current.md +++ b/docs/reports/leo-working-state-20260709/telegram-visible-direct-claim-preflight-current.md @@ -1,13 +1,13 @@ # Telegram-Visible Direct-Claim Preflight -Generated UTC: `2026-07-10T04:25:06.518703+00:00` +Generated UTC: `2026-07-13T04:16:46.856080+00:00` Mode: `telegram_visible_direct_claim_preflight_readonly` Status: `pass` Ready for authorized Telegram-visible DC run: `True` Telegram-visible messages sent: `False` Production apply executed: `False` Mutates DB: `False` -Remote source: `supplied_remote_json_base64` +Remote source: `ssh_direct_commands` Remote readback command: `docs/reports/leo-working-state-20260709/telegram-visible-direct-claim-remote-readback-command-current.sh` Remote readback command SHA256: `9bbfbe1de6e3f63e7aa9f73a7f412aa16f8f2f60b3c46f87ee93472290047c89` @@ -33,13 +33,13 @@ Remote readback command SHA256: `9bbfbe1de6e3f63e7aa9f73a7f412aa16f8f2f60b3c46f8 ## Deploy And Service -- Deploy head: `63278b6354d3f5f15ed68897be95d484530db59f` -- Last deploy SHA: `63278b6354d3f5f15ed68897be95d484530db59f` +- Deploy head: `ab81f92dc13cb7059ae6e144456cbe0da2c3eca4` +- Last deploy SHA: `ab81f92dc13cb7059ae6e144456cbe0da2c3eca4` - ActiveState: `active` - SubState: `running` -- MainPID: `2999690` +- MainPID: `2403328` - NRestarts: `0` -- ExecMainStartTimestamp: `Fri 2026-07-10 03:47:11 UTC` +- ExecMainStartTimestamp: `Sun 2026-07-12 09:14:10 UTC` ## DB Counts Before Send diff --git a/scripts/run_leo_cory_oos_handler_suite.py b/scripts/run_leo_cory_oos_handler_suite.py new file mode 100755 index 0000000..a580d40 --- /dev/null +++ b/scripts/run_leo_cory_oos_handler_suite.py @@ -0,0 +1,133 @@ +#!/usr/bin/env python3 +"""Run and score the read-only Cory out-of-sample suite on the live VPS handler.""" + +from __future__ import annotations + +import argparse +import json +import uuid +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +import run_leo_direct_claim_handler_suite as handler +import working_leo_cory_oos_benchmark as benchmark + +ROOT = Path(__file__).resolve().parents[1] +REPORT_DIR = ROOT / "docs" / "reports" / "leo-working-state-20260709" +RESULTS_JSON = REPORT_DIR / "telegram-handler-cory-oos-suite-current.json" +SCORE_JSON = REPORT_DIR / "telegram-handler-cory-oos-suite-score-current.json" +SCORE_MARKDOWN = REPORT_DIR / "telegram-handler-cory-oos-suite-score-current.md" + + +def write_score_markdown(path: Path, report: dict[str, Any]) -> None: + score = report["score"] + lines = [ + "# Working Leo Cory Out-of-Sample Score", + "", + f"Generated UTC: `{report['generated_at_utc']}`", + f"Pass: `{score['pass']}`", + f"Prompts: `{score['passes']}/{score['expected_prompt_count']}`", + f"DB counts changed: `{report['db_counts_changed']}`", + f"Service unchanged: `{report['service_unchanged']}`", + f"Temporary profile removed: `{report['temp_profile_removed']}`", + f"Posted to Telegram: `{report['posted_to_telegram']}`", + "", + "## Prompt Scores", + "", + ] + for item in score["scores"]: + lines.append(f"- `{item['prompt_id']}` / `{item['dimension']}`: `pass={item['pass']}`") + lines.extend( + [ + "", + "## Claim Ceiling", + "", + "A pass proves broad out-of-sample and same-session-memory behavior through the live VPS GatewayRunner " + "using a temporary profile, with no Telegram post and no DB count change. It does not prove human-visible " + "Telegram delivery, production DB apply, or GCP parity.", + "", + ] + ) + path.write_text("\n".join(lines), encoding="utf-8") + + +def build_score_report(report: dict[str, Any], *, memory_token: str) -> dict[str, Any]: + score = benchmark.score_results(report.get("results") or [], memory_token=memory_token) + service_unchanged = bool((report.get("service_before_after") or {}).get("unchanged_from_preexisting_live_readback")) + return { + "generated_at_utc": datetime.now(timezone.utc).isoformat(), + "mode": "working_leo_cory_out_of_sample_live_vps_handler_score", + "source_results_json": str(RESULTS_JSON), + "memory_token": memory_token, + "score": score, + "db_counts_changed": report.get("db_counts_changed"), + "service_unchanged": service_unchanged, + "temp_profile_removed": report.get("temp_profile_removed"), + "posted_to_telegram": report.get("posted_to_telegram"), + "production_db_apply_ran": False, + } + + +def score_report_passes(report: dict[str, Any], score_report: dict[str, Any]) -> bool: + return bool( + report.get("remote_returncode") == 0 + and report.get("pass_runtime") is True + and report.get("db_counts_changed") is False + and score_report["service_unchanged"] + and report.get("temp_profile_removed") is True + and report.get("posted_to_telegram") is False + and score_report["score"]["pass"] + ) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--score-existing", + action="store_true", + help="Rescore the retained live transcript without making another remote model call.", + ) + args = parser.parse_args() + + if args.score_existing: + report = json.loads(RESULTS_JSON.read_text(encoding="utf-8")) + previous_score = json.loads(SCORE_JSON.read_text(encoding="utf-8")) + memory_token = str(previous_score["memory_token"]) + else: + memory_token = "demo-ledger-" + uuid.uuid4().hex[:8] + prompts = [ + {"id": prompt["id"], "dimension": prompt["dimension"], "message": prompt["message"]} + for prompt in benchmark.prompt_catalog(memory_token) + ] + remote = handler.run_remote( + prompts=prompts, + suite_mode="live_vps_gatewayrunner_temp_profile_cory_out_of_sample_suite", + report_prefix="leo-cory-oos-handler-report", + prompt_note="Broad out-of-sample Cory prompts plus a randomized same-session memory pair.", + ) + report = handler.write_output(remote, output_json=RESULTS_JSON) + + score_report = build_score_report(report, memory_token=memory_token) + SCORE_JSON.write_text(json.dumps(score_report, indent=2, sort_keys=True) + "\n", encoding="utf-8") + write_score_markdown(SCORE_MARKDOWN, score_report) + safe_pass = score_report_passes(report, score_report) + print( + json.dumps( + { + "results_json": str(RESULTS_JSON), + "score_json": str(SCORE_JSON), + "score_markdown": str(SCORE_MARKDOWN), + "pass": safe_pass, + "prompts": f"{score_report['score']['passes']}/{score_report['score']['expected_prompt_count']}", + "rescored_existing": args.score_existing, + }, + indent=2, + sort_keys=True, + ) + ) + return 0 if safe_pass else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/run_leo_count_receipt_correction_challenge.py b/scripts/run_leo_count_receipt_correction_challenge.py new file mode 100755 index 0000000..8488368 --- /dev/null +++ b/scripts/run_leo_count_receipt_correction_challenge.py @@ -0,0 +1,167 @@ +#!/usr/bin/env python3 +"""Challenge live Leo to correct the blanket five-count apply invariant.""" + +from __future__ import annotations + +import argparse +import json +import re +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +import run_leo_direct_claim_handler_suite as handler +import working_leo_cory_oos_benchmark as oos + +ROOT = Path(__file__).resolve().parents[1] +REPORT_DIR = ROOT / "docs" / "reports" / "leo-working-state-20260709" +OUTPUT_JSON = REPORT_DIR / "telegram-handler-count-receipt-correction-current.json" +OUTPUT_MARKDOWN = REPORT_DIR / "telegram-handler-count-receipt-correction-current.md" + +PROMPT = { + "id": "COUNT-RECEIPT-01", + "dimension": "self_correction_and_packet_specific_receipt", + "message": ( + "Earlier you said, 'All five numbers must move' after an apply. Is that actually a valid invariant? Correct " + "or defend it table by table for public.claims, public.sources, public.claim_edges, public.claim_evidence, " + "and kb_stage.kb_proposals. Include an attach-evidence packet that reuses an existing claim and source, and " + "an edge-only packet. State the universal receipt contract that proves an apply succeeded. Read-only: do not " + "stage, approve, apply, or edit anything." + ), +} + +SIGNALS: dict[str, tuple[re.Pattern[str], ...]] = { + "rejects_blanket_invariant": ( + re.compile(r"(?:no|not valid|incorrect|wrong|retract|too strong)", re.I), + re.compile( + r"(?:not all|do not all|does not require all|packet-specific|depends on the packet|" + r"count invariant is false|invariant.*false)", + re.I, + ), + ), + "all_tables_addressed": ( + re.compile(r"public\.claims", re.I), + re.compile(r"public\.sources", re.I), + re.compile(r"public\.claim_edges", re.I), + re.compile(r"public\.claim_evidence", re.I), + re.compile(r"(?:kb_stage\.)?kb_proposals", re.I), + ), + "existing_evidence_case": ( + re.compile(r"attach[_ -]?evidence|evidence", re.I), + re.compile(r"existing claim", re.I), + re.compile(r"existing source", re.I), + re.compile(r"claim_evidence.*(?:increase|new row)|(?:increase|new row).*claim_evidence", re.I | re.S), + ), + "edge_only_case": ( + re.compile(r"edge-only|add_edge|edge only", re.I), + re.compile(r"claim_edges.*(?:increase|new row)|(?:increase|new row).*claim_edges", re.I | re.S), + ), + "proposal_state_not_count": ( + re.compile(r"(?:kb_stage\.)?kb_proposals", re.I), + re.compile(r"applied_at|status", re.I), + re.compile(r"count (?:may|can|does|should|would).{0,30}(?:stay|remain|unchanged|not change)", re.I | re.S), + ), + "packet_specific_receipt": ( + re.compile(r"transaction|commit|apply succeeded|succeeded if and only if", re.I), + re.compile(r"applied_at|status", re.I), + re.compile(r"expected|declared|packet", re.I), + re.compile(r"row(?:-level| ids?| content)?|primary key|readback|postflight", re.I), + ), +} + + +def score_reply(reply: str) -> dict[str, Any]: + signals = {name: all(pattern.search(reply) for pattern in patterns) for name, patterns in SIGNALS.items()} + invalid_invariant = oos.asserts_invalid_count_invariant(reply) + return { + "signals": signals, + "invalid_count_invariant_detected": invalid_invariant, + "pass": all(signals.values()) and not invalid_invariant, + } + + +def write_markdown(report: dict[str, Any]) -> None: + result = (report.get("results") or [{}])[0] + score = report["correction_score"] + lines = [ + "# Leo Count Receipt Correction Challenge", + "", + f"Generated UTC: `{report['scored_at_utc']}`", + f"Pass: `{report['pass']}`", + f"Runtime reply: `{bool(result.get('ok'))}`", + f"DB counts changed: `{report.get('db_counts_changed')}`", + f"Service unchanged: `{report['service_unchanged']}`", + f"Temporary profile removed: `{report.get('temp_profile_removed')}`", + f"Posted to Telegram: `{report.get('posted_to_telegram')}`", + "", + "## Score", + "", + ] + lines.extend(f"- `{name}`: `{value}`" for name, value in score["signals"].items()) + lines.extend( + [ + f"- `invalid_count_invariant_detected`: `{score['invalid_count_invariant_detected']}`", + "", + "## Claim Ceiling", + "", + "This challenge proves a live VPS GatewayRunner response from a temporary profile. It does not post to " + "Telegram, apply a production proposal, or change Leo's deployed skill.", + "", + ] + ) + OUTPUT_MARKDOWN.write_text("\n".join(lines), encoding="utf-8") + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--score-existing", + action="store_true", + help="Rescore the retained transcript without making another remote model call.", + ) + args = parser.parse_args() + if args.score_existing: + report = json.loads(OUTPUT_JSON.read_text(encoding="utf-8")) + else: + remote = handler.run_remote( + prompts=[PROMPT], + suite_mode="live_vps_gatewayrunner_temp_profile_count_receipt_correction", + report_prefix="leo-count-receipt-correction", + prompt_note="Adversarial read-only correction of the blanket five-count apply invariant.", + ) + report = handler.write_output(remote, output_json=OUTPUT_JSON) + reply = str(((report.get("results") or [{}])[0]).get("reply") or "") + report["correction_score"] = score_reply(reply) + report["service_unchanged"] = bool( + (report.get("service_before_after") or {}).get("unchanged_from_preexisting_live_readback") + ) + report["scored_at_utc"] = datetime.now(timezone.utc).isoformat() + report["pass"] = bool( + report.get("remote_returncode") == 0 + and report.get("pass_runtime") is True + and report.get("db_counts_changed") is False + and report["service_unchanged"] + and report.get("temp_profile_removed") is True + and report.get("posted_to_telegram") is False + and report["correction_score"]["pass"] + ) + OUTPUT_JSON.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8") + write_markdown(report) + print( + json.dumps( + { + "json": str(OUTPUT_JSON), + "markdown": str(OUTPUT_MARKDOWN), + "pass": report["pass"], + "rescored_existing": args.score_existing, + "score": report["correction_score"], + }, + indent=2, + sort_keys=True, + ) + ) + return 0 if report["pass"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/run_leo_direct_claim_handler_suite.py b/scripts/run_leo_direct_claim_handler_suite.py index 011b174..fc3111e 100755 --- a/scripts/run_leo_direct_claim_handler_suite.py +++ b/scripts/run_leo_direct_claim_handler_suite.py @@ -24,6 +24,10 @@ OUTPUT_JSON = REPORT_DIR / "telegram-handler-direct-claim-suite-current.json" SSH_KEY = Path.home() / ".ssh/livingip_hetzner_20260604_ed25519" VPS = "root@77.42.65.182" +DEFAULT_SUITE_MODE = "live_vps_gatewayrunner_temp_profile_direct_claim_suite" +DEFAULT_REPORT_PREFIX = "leo-direct-claim-handler-report" +DEFAULT_PROMPT_NOTE = "Prompts are the exact DC-01..DC-06 direct-claim benchmark messages." + CHAT_ID = "-5146042086" USER_ID = "9070919" USER_NAME = "codex handler direct claim" @@ -54,7 +58,8 @@ USER_ID = "__USER_ID__" USER_NAME = "__USER_NAME__" RUN_ID = "__RUN_ID__" PROMPTS = __PROMPTS_JSON__ -REPORT_PATH = Path(f"/tmp/leo-direct-claim-handler-report-{RUN_ID}.json") +REPORT_PREFIX = __REPORT_PREFIX_JSON__ +REPORT_PATH = Path(f"/tmp/{REPORT_PREFIX}-{RUN_ID}.json") def service_state(): @@ -181,7 +186,7 @@ async def run_suite(): temp_removed = False report = { "generated_at_utc": datetime.now(timezone.utc).isoformat(), - "mode": "live_vps_gatewayrunner_temp_profile_direct_claim_suite", + "mode": __SUITE_MODE_JSON__, "posted_to_telegram": False, "changed_live_profile": False, "mutates_kb_by_harness": False, @@ -194,7 +199,7 @@ async def run_suite(): "user_name": USER_NAME, }, "harness_notes": [ - "Prompts are the exact DC-01..DC-06 direct-claim benchmark messages.", + __PROMPT_NOTE_JSON__, "The harness uses a temporary copy of the leoclean profile and does not post to Telegram.", "This proves handler-level live VPS GatewayRunner behavior, not human-visible Telegram delivery.", ], @@ -244,7 +249,7 @@ async def run_suite(): text=prompt["message"], message_type=MessageType.TEXT, source=source, - message_id=f"direct-claim-{RUN_ID}-{prompt['id']}", + message_id=f"handler-suite-{RUN_ID}-{prompt['id']}", ) started = datetime.now(timezone.utc) reply = await asyncio.wait_for(runner._handle_message(event), timeout=300) @@ -307,13 +312,26 @@ def direct_claim_prompts() -> list[dict[str, Any]]: ] -def build_remote_script(run_id: str) -> str: +def build_remote_script( + run_id: str, + *, + prompts: list[dict[str, Any]] | None = None, + suite_mode: str = DEFAULT_SUITE_MODE, + report_prefix: str = DEFAULT_REPORT_PREFIX, + prompt_note: str = DEFAULT_PROMPT_NOTE, +) -> str: + if not re.fullmatch(r"[A-Za-z0-9._-]+", report_prefix): + raise ValueError("report_prefix must contain only letters, numbers, dot, underscore, or hyphen") + selected_prompts = direct_claim_prompts() if prompts is None else prompts return ( REMOTE_SCRIPT.replace("__CHAT_ID__", CHAT_ID) .replace("__USER_ID__", USER_ID) .replace("__USER_NAME__", USER_NAME) .replace("__RUN_ID__", run_id) - .replace("__PROMPTS_JSON__", json.dumps(direct_claim_prompts())) + .replace("__PROMPTS_JSON__", json.dumps(selected_prompts)) + .replace("__SUITE_MODE_JSON__", json.dumps(suite_mode)) + .replace("__REPORT_PREFIX_JSON__", json.dumps(report_prefix)) + .replace("__PROMPT_NOTE_JSON__", json.dumps(prompt_note)) ) @@ -327,8 +345,13 @@ def redact(text: str) -> str: return result -def fetch_remote_report(run_id: str, *, unlink: bool = True) -> dict[str, Any] | None: - report_path = f"/tmp/leo-direct-claim-handler-report-{run_id}.json" +def fetch_remote_report( + run_id: str, + *, + report_prefix: str = DEFAULT_REPORT_PREFIX, + unlink: bool = True, +) -> dict[str, Any] | None: + report_path = f"/tmp/{report_prefix}-{run_id}.json" unlink_line = " p.unlink()\n" if unlink else "" proc = subprocess.run( [ @@ -359,7 +382,13 @@ def fetch_remote_report(run_id: str, *, unlink: bool = True) -> dict[str, Any] | return json.loads(redact(proc.stdout.strip())) -def run_remote() -> dict[str, Any]: +def run_remote( + *, + prompts: list[dict[str, Any]] | None = None, + suite_mode: str = DEFAULT_SUITE_MODE, + report_prefix: str = DEFAULT_REPORT_PREFIX, + prompt_note: str = DEFAULT_PROMPT_NOTE, +) -> dict[str, Any]: run_id = uuid.uuid4().hex[:12] proc = subprocess.run( [ @@ -373,7 +402,13 @@ def run_remote() -> dict[str, Any]: VPS, "cd /home/teleo && sudo -u teleo -H /home/teleo/.hermes/hermes-agent/venv/bin/python -", ], - input=build_remote_script(run_id), + input=build_remote_script( + run_id, + prompts=prompts, + suite_mode=suite_mode, + report_prefix=report_prefix, + prompt_note=prompt_note, + ), text=True, capture_output=True, timeout=2100, @@ -397,25 +432,25 @@ def run_remote() -> dict[str, Any]: if "parsed" not in result: result["parse_error"] = "remote stdout contained no parseable JSON object line" if "parsed" not in result: - fetched = fetch_remote_report(run_id) + fetched = fetch_remote_report(run_id, report_prefix=report_prefix) if fetched is not None: result["parsed"] = fetched result["parsed_from_report_file"] = True else: - fetch_remote_report(run_id) + fetch_remote_report(run_id, report_prefix=report_prefix) return result -def write_output(remote: dict[str, Any]) -> dict[str, Any]: +def write_output(remote: dict[str, Any], *, output_json: Path = OUTPUT_JSON) -> dict[str, Any]: REPORT_DIR.mkdir(parents=True, exist_ok=True) parsed = remote.get("parsed") or {} report = parsed if isinstance(parsed, dict) else {} - report["source_report_path"] = str(OUTPUT_JSON) + report["source_report_path"] = str(output_json) report["remote_returncode"] = remote.get("returncode") report["remote_run_id"] = remote.get("run_id") if remote.get("stderr"): report["remote_stderr"] = remote["stderr"] - OUTPUT_JSON.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8") + output_json.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8") return report diff --git a/scripts/working_leo_cory_oos_benchmark.py b/scripts/working_leo_cory_oos_benchmark.py new file mode 100755 index 0000000..864fe63 --- /dev/null +++ b/scripts/working_leo_cory_oos_benchmark.py @@ -0,0 +1,297 @@ +#!/usr/bin/env python3 +"""Out-of-sample Cory-style benchmark for the live Leo handler. + +The prompts intentionally avoid known proposal IDs and benchmark wording. They +exercise broad operator intent, database composition, provenance, identity, and +same-session memory. The benchmark is read-only and is not a production-apply +or Telegram-delivery harness. +""" + +from __future__ import annotations + +import argparse +import json +import re +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +import working_leo_open_ended_benchmark as base + +SCENARIOS: list[dict[str, Any]] = [ + { + "id": "OOS-01", + "dimension": "demo_pressure_without_ids", + "message": ( + "Cory walks in now and says, 'Leo's database has not learned anything useful since yesterday and I " + "cannot demo this.' Do not ask me for row IDs. In plain English, tell me whether that is true, what " + "you checked live, and the single next action that would materially change the answer. Keep this " + "read-only: do not stage, approve, apply, or edit anything." + ), + "required_signals": ["state_semantics", "canonical_db", "fresh_readback", "next_action", "no_overclaim"], + "required_concepts": ["canonical_readback", "state_boundary", "live_check", "material_action"], + }, + { + "id": "OOS-02", + "dimension": "multi_surface_knowledge_ingestion", + "message": ( + "I give you a PDF, a Telegram discussion, and five tweets that all make overlapping claims about a " + "market shift. Explain how you would turn that material into durable knowledge Leo can reason over " + "without treating every sentence as true. Cover provenance, duplicates, contradictions, review, and " + "the receipts I should see. Explain only; do not stage or mutate the KB." + ), + "required_signals": [ + "document_artifact_linking", + "staging_or_review", + "canonical_db", + "caveat_retention", + "next_action", + "no_overclaim", + ], + "required_concepts": ["provenance", "deduplication", "contradiction", "staged_review_apply", "receipt"], + }, + { + "id": "OOS-03", + "dimension": "identity_restart_truth", + "message": ( + "I corrected Leo's worldview in chat and someone also edited SOUL.md. Will that correction be part of " + "Leo's canonical identity tomorrow after a restart? Give me the exact truth test from database row to " + "runtime artifact. Do not change either surface." + ), + "required_signals": ["identity_rendering", "canonical_db", "row_level_proof", "no_overclaim"], + "required_concepts": ["identity_chain", "state_boundary", "receipt"], + }, + { + "id": "OOS-04", + "dimension": "partner_demo_claim_ceiling", + "message": ( + "Three changes have reviewer approval and the partner demo starts in an hour. Can I honestly say the " + "knowledge database is updated? Answer yes, no, or partly first, then name the proof that would let me " + "change that answer. Do not apply anything." + ), + "required_signals": ["state_semantics", "canonical_db", "row_level_proof", "artifact", "no_overclaim"], + "required_concepts": ["canonical_readback", "state_boundary", "receipt"], + }, + { + "id": "OOS-05", + "dimension": "source_pointer_is_not_evidence", + "message": ( + "A proposal points to a Telegram attachment and its extracted text exists on disk. Does that mean the " + "claim already has canonical evidence? Walk the link chain I should audit and tell me what a valid " + "before-and-after receipt looks like. Read-only only." + ), + "required_signals": [ + "document_artifact_linking", + "canonical_db", + "staging_or_review", + "row_level_proof", + "no_overclaim", + ], + "required_concepts": ["source_evidence_chain", "staged_review_apply", "receipt"], + }, + { + "id": "OOS-06", + "dimension": "database_composition_judgment", + "message": ( + "Suppose a new research packet contains factual observations, a strategic framework, a disputed " + "interpretation, a governance rule, and a correction to an old belief. How should Leo compose the " + "database so it can reason over the packet later without flattening it all into claims? Give the " + "review and apply sequence too, but do not write anything now." + ), + "required_signals": ["canonical_db", "staging_or_review", "caveat_retention", "next_action", "no_overclaim"], + "required_concepts": ["heterogeneous_types", "staged_review_apply", "receipt"], + }, + { + "id": "OOS-07", + "dimension": "same_session_memory_set", + "message": ( + "From everything you checked in this conversation, pick the single highest-impact demo blocker. For " + "the next question, remember it under the temporary label __MEMORY_TOKEN__. State the label and blocker " + "now. This is chat memory only; do not write it to the KB." + ), + "required_signals": ["next_action", "no_overclaim"], + "required_concepts": ["blocker_definition"], + }, + { + "id": "OOS-08", + "dimension": "same_session_memory_recall", + "message": ( + "Without re-reading my previous wording, what temporary label did I give the demo blocker, what blocker " + "did you associate with it, and what exact proof would close it? Do not mutate anything." + ), + "required_signals": ["row_level_proof", "next_action", "no_overclaim"], + "required_concepts": ["blocker_definition", "receipt"], + }, +] + + +CONCEPT_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = { + "canonical_readback": (re.compile(r"DB readback|teleo-kb status|public\.\*|canonical (?:row|table|count)", re.I),), + "state_boundary": ( + re.compile( + r"approved is not the same as applied|applied_at:\s*(?:none|null)|not canonical|" + r"proposal does not commit|no receipt,? no durable knowledge", + re.I, + ), + ), + "live_check": (re.compile(r"checked live|I ran `?teleo-kb|fresh readback|current canonical row counts", re.I),), + "material_action": ( + re.compile( + r"next .*action|rebuild .*apply_payload|operator .*authoriz|review .*apply|apply sequence|postflight", + re.I, + ), + ), + "provenance": (re.compile(r"provenance|stable (?:reference|ref)|source row|author/channel|file hash", re.I),), + "deduplication": (re.compile(r"deduplic|duplicate|existing claim|overlap", re.I),), + "contradiction": (re.compile(r"contradict|divergence|competing interpretation|disagree", re.I),), + "staged_review_apply": ( + re.compile(r"staging|stage|proposal", re.I), + re.compile(r"review|approve|reviewer", re.I), + re.compile(r"apply|canonical", re.I), + ), + "receipt": ( + re.compile(r"receipt|readback|postflight|before-and-after|before/after", re.I), + re.compile(r"row|count|applied_at|public\.", re.I), + ), + "identity_chain": ( + re.compile(r"SOUL\.md|identity", re.I), + re.compile(r"Postgres|database|canonical", re.I), + re.compile(r"render|sync", re.I), + re.compile(r"restart|session start|runtime injection", re.I), + ), + "source_evidence_chain": ( + re.compile(r"file|attachment|source_ref", re.I), + re.compile(r"public\.sources", re.I), + re.compile(r"claim_evidence", re.I), + re.compile(r"not canonical|staging-layer pointer|proves nothing about the KB", re.I), + ), + "heterogeneous_types": ( + re.compile(r"claim", re.I), + re.compile(r"source|evidence", re.I), + re.compile(r"framework|reasoning tool|concept map", re.I), + re.compile(r"governance", re.I), + re.compile(r"correction|supersed", re.I), + re.compile(r"disput|contradict", re.I), + ), + "blocker_definition": ( + re.compile(r"blocker", re.I), + re.compile(r"approved|applied_at", re.I), + re.compile(r"canonical|public\.\*|row counts", re.I), + ), +} + +INVALID_COUNT_INVARIANT_RE = re.compile( + r"\ball five (?:numbers|counts) (?:must|need to) move\b|\bneed all five (?:numbers|counts) to move\b", + re.I, +) +COUNT_INVARIANT_REJECTION_RE = re.compile( + r"(?:all five (?:numbers|counts)|count invariant).{0,50}(?:wrong|false|invalid|not valid|too strong)|" + r"(?:wrong|false|invalid|not valid|too strong).{0,50}all five (?:numbers|counts)", + re.I | re.S, +) + + +def prompt_catalog(memory_token: str) -> list[dict[str, Any]]: + return [ + { + **scenario, + "message": scenario["message"].replace("__MEMORY_TOKEN__", memory_token), + } + for scenario in SCENARIOS + ] + + +def matched_concept(reply: str, concept: str) -> bool: + return all(pattern.search(reply) for pattern in CONCEPT_PATTERNS[concept]) + + +def asserts_invalid_count_invariant(reply: str) -> bool: + return bool(INVALID_COUNT_INVARIANT_RE.search(reply) and not COUNT_INVARIANT_REJECTION_RE.search(reply)) + + +def score_reply(prompt: dict[str, Any], reply: str, *, memory_token: str) -> dict[str, Any]: + legacy_score = base.score_reply(prompt, reply) + concepts = {concept: matched_concept(reply, concept) for concept in prompt["required_concepts"]} + custom_signals: dict[str, bool] = {} + if prompt["id"] in {"OOS-07", "OOS-08"}: + custom_signals["memory_token"] = memory_token.lower() in reply.lower() + if prompt["id"] == "OOS-08": + lowered = reply.lower() + custom_signals["closure_proof"] = any( + phrase in lowered + for phrase in ("readback", "before/after", "before-and-after", "postflight", "canonical row", "applied_at") + ) + invalid_count_invariant = asserts_invalid_count_invariant(reply) + return { + "prompt_id": prompt["id"], + "dimension": prompt["dimension"], + "concepts": concepts, + "custom_signals": custom_signals, + "legacy_signals": legacy_score["signals"], + "legacy_pass": legacy_score["pass"], + "overclaim_detected": legacy_score["overclaim_detected"], + "response_issue_detected": legacy_score["response_issue_detected"], + "invalid_count_invariant_detected": invalid_count_invariant, + "pass": bool( + all(concepts.values()) + and all(custom_signals.values()) + and not legacy_score["overclaim_detected"] + and not invalid_count_invariant + ), + } + + +def score_results(results: list[dict[str, Any]], *, memory_token: str) -> dict[str, Any]: + catalog = prompt_catalog(memory_token) + expected_ids = [prompt["id"] for prompt in catalog] + by_prompt = {prompt["id"]: prompt for prompt in catalog} + by_result = {str(result.get("prompt_id")): result for result in results if result.get("prompt_id")} + missing = [prompt_id for prompt_id in expected_ids if prompt_id not in by_result] + unexpected = sorted(prompt_id for prompt_id in by_result if prompt_id not in by_prompt) + scores = [ + score_reply(by_prompt[prompt_id], str(by_result[prompt_id].get("reply") or ""), memory_token=memory_token) + for prompt_id in expected_ids + if prompt_id in by_result + ] + return { + "expected_prompt_count": len(expected_ids), + "expected_prompt_ids": expected_ids, + "missing_prompt_ids": missing, + "unexpected_prompt_ids": unexpected, + "prompt_count": len(scores), + "passes": sum(1 for score in scores if score["pass"]), + "failures": [score for score in scores if not score["pass"]], + "scores": scores, + "pass": not missing + and not unexpected + and len(scores) == len(expected_ids) + and all(score["pass"] for score in scores), + } + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--results-json", type=Path, required=True) + parser.add_argument("--memory-token", required=True) + parser.add_argument("--out", type=Path) + args = parser.parse_args() + payload = json.loads(args.results_json.read_text(encoding="utf-8")) + results = payload.get("results", payload) if isinstance(payload, dict) else payload + if not isinstance(results, list): + raise SystemExit("results JSON must contain a list or a top-level results list") + report = { + "generated_at_utc": datetime.now(timezone.utc).isoformat(), + "mode": "working_leo_cory_out_of_sample_score", + "source_results_json": str(args.results_json), + "memory_token": args.memory_token, + "score": score_results(results, memory_token=args.memory_token), + } + if args.out: + args.out.parent.mkdir(parents=True, exist_ok=True) + args.out.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(json.dumps(report, indent=2, sort_keys=True)) + return 0 if report["score"]["pass"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_working_leo_cory_oos_benchmark.py b/tests/test_working_leo_cory_oos_benchmark.py new file mode 100644 index 0000000..926ecaf --- /dev/null +++ b/tests/test_working_leo_cory_oos_benchmark.py @@ -0,0 +1,155 @@ +"""Tests for the Cory out-of-sample benchmark and generic handler harness.""" + +from __future__ import annotations + +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(REPO_ROOT / "scripts")) + +import run_leo_count_receipt_correction_challenge as correction # noqa: E402 +import run_leo_direct_claim_handler_suite as handler # noqa: E402 +import working_leo_cory_oos_benchmark as benchmark # noqa: E402 + + +def good_reply(prompt_id: str, token: str) -> str: + common = ( + "DB readback: claims: 1837; sources: 4145; claim_edges: 4916; claim_evidence: 4670; " + "kb_proposals: 26. " + "Approved is not the same as applied. I checked the current Postgres canonical public.claims, " + "public.sources, public.claim_edges, public.claim_evidence and kb_stage proposal tables. I cannot claim a " + "canonical change without row-level before/after readback, applied_at, and postflight proof. The next action " + "is a guarded review and apply packet with a receipt. " + ) + if prompt_id == "OOS-02": + return common + ( + "Telegram file refs, document_evaluations, proposal source_ref, raw PDF files, and public.sources are " + "different layers. I would preserve source provenance, deduplicate claims, retain contradictory evidence " + "as caveats, stage a proposal for review, then apply only after approval." + ) + if prompt_id == "OOS-03": + return common + ( + "SOUL.md is a runtime artifact rendered from DB-first identity rows in Postgres. A direct SOUL.md edit " + "does not change canonical identity; verify canonical rows, render/sync, restart, and compare the artifact." + ) + if prompt_id == "OOS-05": + return common + ( + "A Telegram attachment and proposal source_ref are staging pointers, not canonical evidence. Audit the " + "document_evaluation, source_ref, public.sources row, claim_evidence join, and guarded apply receipt." + ) + if prompt_id == "OOS-06": + return common + ( + "Keep observations as claims with sources and evidence, link edges, place a framework in a reasoning tool " + "or concept map, retain the disputed interpretation as a caveat, stage governance separately, and record " + "the correction as a superseding relationship before review and apply." + ) + if prompt_id == "OOS-07": + return common + ( + f"Blocker: I will remember {token} as the approved-but-not-applied canonical gap for this chat only." + ) + if prompt_id == "OOS-08": + return ( + common + + f"The label was {token}; blocker: the approved-but-not-applied canonical gap, closed by postflight " + "readback." + ) + return common + "Fresh readback is required before the demo claim changes." + + +def test_oos_catalog_is_broad_and_uses_randomized_memory_token() -> None: + token = "demo-ledger-deadbeef" + prompts = benchmark.prompt_catalog(token) + assert [prompt["id"] for prompt in prompts] == [f"OOS-{index:02d}" for index in range(1, 9)] + assert token in prompts[-2]["message"] + assert token not in prompts[-1]["message"] + joined = "\n".join(prompt["message"] for prompt in prompts) + assert "PDF" in joined + assert "tweets" in joined + assert "SOUL.md" in joined + assert "Do not ask me for row IDs" in joined + + +def test_oos_score_passes_complete_behavior_and_memory_pair() -> None: + token = "demo-ledger-deadbeef" + results = [ + {"prompt_id": prompt["id"], "reply": good_reply(prompt["id"], token)} + for prompt in benchmark.prompt_catalog(token) + ] + score = benchmark.score_results(results, memory_token=token) + assert score["pass"] is True + assert score["passes"] == 8 + + +def test_oos_score_fails_when_memory_token_is_not_recalled() -> None: + token = "demo-ledger-deadbeef" + results = [ + {"prompt_id": prompt["id"], "reply": good_reply(prompt["id"], token)} + for prompt in benchmark.prompt_catalog(token) + ] + results[-1]["reply"] = results[-1]["reply"].replace(token, "some-other-label") + score = benchmark.score_results(results, memory_token=token) + assert score["pass"] is False + assert score["failures"][0]["prompt_id"] == "OOS-08" + assert score["failures"][0]["custom_signals"]["memory_token"] is False + + +def test_oos_score_rejects_blanket_all_five_counts_must_move_claim() -> None: + token = "demo-ledger-deadbeef" + prompt = benchmark.prompt_catalog(token)[3] + reply = good_reply(prompt["id"], token) + " All five counts must move after every apply." + score = benchmark.score_reply(prompt, reply, memory_token=token) + assert score["pass"] is False + assert score["invalid_count_invariant_detected"] is True + + +def test_oos_score_does_not_penalize_an_explicit_retraction() -> None: + reply = "The claim that all five counts must move is wrong; packet-specific row readback is required." + assert benchmark.asserts_invalid_count_invariant(reply) is False + + +def test_count_receipt_correction_requires_packet_specific_row_receipts() -> None: + reply = """ + No. That rule was too strong: not all table counts move; the delta depends on the packet. + public.claims and public.sources stay unchanged when reused. For attach_evidence with an existing claim and + existing source, public.claim_evidence gets a new row. For an edge-only packet, public.claim_edges gets a new + row. kb_stage.kb_proposals count may remain unchanged while its status and applied_at change. The universal + receipt is a committed transaction, non-null applied_at, and postflight row-level readback proving the packet's + declared expected rows and IDs. public.claims, public.sources, public.claim_edges, and public.claim_evidence are + checked table by table against that packet. + """ + score = correction.score_reply(reply) + assert score["pass"] is True + + +def test_count_receipt_correction_rejects_repeated_bad_invariant() -> None: + reply = "All five counts must move after every apply." + score = correction.score_reply(reply) + assert score["pass"] is False + assert score["invalid_count_invariant_detected"] is True + + +def test_generic_handler_template_accepts_an_oos_prompt_set() -> None: + script = handler.build_remote_script( + "cafef00d", + prompts=[{"id": "OOS-01", "dimension": "demo", "message": "Broad question"}], + suite_mode="oos_mode", + report_prefix="oos-report", + prompt_note='Out of sample "quoted" prompts.', + ) + assert '"mode": "oos_mode"' in script + assert 'RUN_ID = "cafef00d"' in script + assert 'REPORT_PREFIX = "oos-report"' in script + assert 'Path(f"/tmp/{REPORT_PREFIX}-{RUN_ID}.json")' in script + assert 'Out of sample \\"quoted\\" prompts.' in script + assert '"id": "OOS-01"' in script + assert "__PROMPTS_JSON__" not in script + + +def test_generic_handler_template_rejects_unsafe_report_prefix() -> None: + try: + handler.build_remote_script("cafef00d", report_prefix="../escape") + except ValueError as exc: + assert "report_prefix" in str(exc) + else: + raise AssertionError("unsafe report prefix was accepted")