From 08b3d79e0f2455f3395fce869644d7547487ad96 Mon Sep 17 00:00:00 2001 From: twentyOne2x Date: Fri, 10 Jul 2026 03:03:58 +0200 Subject: [PATCH] Tighten Leo direct-claim answer contract --- .../gcp/teleo-kb-bridge/SKILL.md | 38 +++++++++++++++++++ .../vps/teleo-kb-bridge/SKILL.md | 38 +++++++++++++++++++ scripts/working_leo_open_ended_benchmark.py | 30 ++++++++++----- tests/test_hermes_leoclean_skill_surfaces.py | 8 ++++ 4 files changed, 105 insertions(+), 9 deletions(-) diff --git a/hermes-agent/leoclean-skills/gcp/teleo-kb-bridge/SKILL.md b/hermes-agent/leoclean-skills/gcp/teleo-kb-bridge/SKILL.md index 4b290a4..8bfbad7 100644 --- a/hermes-agent/leoclean-skills/gcp/teleo-kb-bridge/SKILL.md +++ b/hermes-agent/leoclean-skills/gcp/teleo-kb-bridge/SKILL.md @@ -94,6 +94,44 @@ Use explicit no-overclaim wording when the canonical DB did not change: "I cannot claim canonical DB changed until `public.*` readback plus `applied_at`/postflight proof says it changed." +### Cory Direct-Claim Answer Contract + +For Cory-style no-context questions, keep the answer direct but include the +proof language below. These are behavioral examples, not feature changes. + +- "Did we actually update the KB?": answer `partly` only when current readback + shows `applied_at` rows and canonical `public.*` rows. Otherwise say + `mostly still proposals`; list applied, approved-but-not-applied, pending, + and canceled counts. +- "Is Helmer's 7 Powers in Leo now?": answer `no, not canonical` unless + `public.sources`, `public.claims`, evidence, edges, and any reasoning-tool + rows exist. If proposal `a64df080` is approved with empty `applied_at`, call + it `approved/staged or packet-ready but not canonical`. +- "Did the decision matrix approve this?": start with current/fresh schema + readback. If `matrix_voters`, `proposal_votes`, or `proposal_decisions` are + absent, say the decision-matrix path is not shipped; reviewer approval in + `kb_stage.kb_proposals` is not a matrix vote. +- "Are proposals stuck because documents point at the wrong source rows?": + do not answer as a single-cause `yes`. Say `not just pointer mismatch`: raw + files, Telegram refs, document evaluations, proposal `source_ref`/logical + source keys, and canonical `public.sources` rows are different layers. The + missing proof is a row-link audit plus guarded apply contract. +- "Can I demo Leo changes the KB?": separate demo tiers. A safe demo can show + a real staging write to `kb_stage.kb_proposals` and read it back. Canonical + mutation of `public.claims`, `public.sources`, `public.claim_evidence`, or + `public.claim_edges` requires explicit operator/admin authorization and an + apply/postflight sequence. +- "Did editing SOUL.md change canonical identity?": answer `no`. `SOUL.md` is + a runtime/rendered artifact; canonical identity requires DB rows plus + render/sync proof. Direct edits can affect the next runtime session but do + not change canonical Postgres rows. + +Every direct-claim answer must include row-level proof vocabulary such as +`current readback`, `row-link audit`, `applied_at`, `kb_stage.kb_proposals`, +`public.*`, and `postflight proof` where relevant. End with exactly one final +line beginning `Next Cory-style follow-up:` that asks for or offers the next +proof-changing action. + ## Telegram Rendering Make KB answers easy to scan in Telegram: diff --git a/hermes-agent/leoclean-skills/vps/teleo-kb-bridge/SKILL.md b/hermes-agent/leoclean-skills/vps/teleo-kb-bridge/SKILL.md index e673e27..f346964 100644 --- a/hermes-agent/leoclean-skills/vps/teleo-kb-bridge/SKILL.md +++ b/hermes-agent/leoclean-skills/vps/teleo-kb-bridge/SKILL.md @@ -86,6 +86,44 @@ Use explicit no-overclaim wording when the canonical DB did not change: "I cannot claim canonical DB changed until `public.*` readback plus `applied_at`/postflight proof says it changed." +### Cory Direct-Claim Answer Contract + +For Cory-style no-context questions, keep the answer direct but include the +proof language below. These are behavioral examples, not feature changes. + +- "Did we actually update the KB?": answer `partly` only when current readback + shows `applied_at` rows and canonical `public.*` rows. Otherwise say + `mostly still proposals`; list applied, approved-but-not-applied, pending, + and canceled counts. +- "Is Helmer's 7 Powers in Leo now?": answer `no, not canonical` unless + `public.sources`, `public.claims`, evidence, edges, and any reasoning-tool + rows exist. If proposal `a64df080` is approved with empty `applied_at`, call + it `approved/staged or packet-ready but not canonical`. +- "Did the decision matrix approve this?": start with current/fresh schema + readback. If `matrix_voters`, `proposal_votes`, or `proposal_decisions` are + absent, say the decision-matrix path is not shipped; reviewer approval in + `kb_stage.kb_proposals` is not a matrix vote. +- "Are proposals stuck because documents point at the wrong source rows?": + do not answer as a single-cause `yes`. Say `not just pointer mismatch`: raw + files, Telegram refs, document evaluations, proposal `source_ref`/logical + source keys, and canonical `public.sources` rows are different layers. The + missing proof is a row-link audit plus guarded apply contract. +- "Can I demo Leo changes the KB?": separate demo tiers. A safe demo can show + a real staging write to `kb_stage.kb_proposals` and read it back. Canonical + mutation of `public.claims`, `public.sources`, `public.claim_evidence`, or + `public.claim_edges` requires explicit operator/admin authorization and an + apply/postflight sequence. +- "Did editing SOUL.md change canonical identity?": answer `no`. `SOUL.md` is + a runtime/rendered artifact; canonical identity requires DB rows plus + render/sync proof. Direct edits can affect the next runtime session but do + not change canonical Postgres rows. + +Every direct-claim answer must include row-level proof vocabulary such as +`current readback`, `row-link audit`, `applied_at`, `kb_stage.kb_proposals`, +`public.*`, and `postflight proof` where relevant. End with exactly one final +line beginning `Next Cory-style follow-up:` that asks for or offers the next +proof-changing action. + ## Telegram Rendering Make KB answers easy to scan in Telegram: diff --git a/scripts/working_leo_open_ended_benchmark.py b/scripts/working_leo_open_ended_benchmark.py index d781582..570cd4c 100644 --- a/scripts/working_leo_open_ended_benchmark.py +++ b/scripts/working_leo_open_ended_benchmark.py @@ -363,7 +363,7 @@ SIGNAL_PATTERNS: dict[str, list[re.Pattern[str]]] = { r"approval is a human gate, not a write|approved with zero canonical effect|" r"applied_at:\s*NULL|no applied_at|NOTHING was applied|not applied|payload-only|" r"neither row has been promoted|canonical (?:layer|tables?|public\.\*) (?:is )?locked|" - r"reviewable proposal", + r"reviewable proposal|canonical apply step", re.I, ), ], @@ -371,21 +371,22 @@ SIGNAL_PATTERNS: dict[str, list[re.Pattern[str]]] = { re.compile(r"\bcanonical\b|\bpublic\.[a-z_]+|\bpublic\b|postgres|database rows?", re.I), re.compile( r"\bpublic\.\*|claim_edges|claim_evidence|claims|sources|reasoning_tools|" - r"personas|strategies|beliefs|strategy_nodes|strategy_node_anchors|canonical tables?\b", + r"personas|strategies|beliefs|strategy_nodes|strategy_node_anchors|canonical tables?|" + r"`?public\.\*`? equivalents\b|kb_stage|matrix_voters|proposal_decisions|proposal_votes", re.I, ), ], "row_level_proof": [ re.compile( - r"\brow[- ]level|before/after|table[- ]level|row ids?|counts?\b|" + r"\brow[- ]level|before/after|table[- ]level|row ids?|counts?|uuid|timestamp\b|" r"\bapplied_at\b|superseded_by:\s*NULL|public\.[a-z_]+ rows?|" r"\b[a-f0-9]{8}\b.*\b(?:approved|applied|pending|canceled)|" r"\b(?:approved|applied|pending|canceled)\b.*\b[a-f0-9]{8}\b", re.I, ), re.compile( - r"\bquery|read back|verify|postflight|preflight|canonical state|applied_at|superseded_by|" - r"canonical `?public\.\*`?|proposal ledger\b", + r"\bquery|read ?back|verify|postflight|preflight|canonical state|applied_at|superseded_by|" + r"canonical `?public\.\*`?|proposal ledger|verifiable readback|row appears|ledger\b", re.I, ), ], @@ -402,8 +403,12 @@ SIGNAL_PATTERNS: dict[str, list[re.Pattern[str]]] = { r"\bnot canonical\b|\bnot applied\b|\bdoes not exist in (?:canonical )?(?:Leo's )?KB\b|" r"\bonly in the proposal ledger\b|locked behind the apply tool|" r"\bcannot demo\b|\bnot possible\b|\bnot a decision[- ]matrix vote\b|" + r"\bnot .*decision[- ]matrix vote\b|" r"\bnot decision[- ]matrix approval\b|\bno auto[- ]apply path\b|\bschema gap\b|\bmissing apply tooling\b|" - r"\bapply tool (?:has not|hasn't) been run\b|\bruntime rendering\b", + r"\bapply tool (?:has not|hasn't) been run\b|\bruntime rendering\b|" + r"\bdb does not change\b|\bnothing in `?public\.\*`? has moved\b|" + r"\bsource rows? (?:themselves )?(?:do not|don't) exist\b|" + r"\bnothing has been inserted\b|\bcanonical apply step is the missing link\b", re.I, ), re.compile( @@ -416,8 +421,12 @@ SIGNAL_PATTERNS: dict[str, list[re.Pattern[str]]] = { r"\bnot in canonical\b|\bcanonical .* has(?:n't| not) (?:moved|changed)\b|" r"\bapplied_at empty\b|\bnot applied\b|\bnot possible\b|" r"\bnot a decision[- ]matrix vote\b|\blocked behind the apply tool\b|" + r"\bnot .*decision[- ]matrix vote\b|" r"\bnot decision[- ]matrix approval\b|\bno auto[- ]apply path\b|\bschema gap\b|\bmissing apply tooling\b|" - r"\bapply tool (?:has not|hasn't) been run\b|\bruntime rendering\b|\bnot canonical\b", + r"\bapply tool (?:has not|hasn't) been run\b|\bruntime rendering\b|\bnot canonical\b|" + r"\bdb does not change\b|\bnothing in `?public\.\*`? has moved\b|" + r"\bsource rows? (?:themselves )?(?:do not|don't) exist\b|" + r"\bnothing has been inserted\b|\bcanonical apply step is the missing link\b", re.I, ), ], @@ -456,7 +465,10 @@ SIGNAL_PATTERNS: dict[str, list[re.Pattern[str]]] = { ], "authorization": [ re.compile(r"\bauthori[sz]ation|approval|explicit|permission|allowed|not allowed|admin|operator\b", re.I), - re.compile(r"\bpublic update|post|capital|funds|sign|broadcast|admin review|operator pass|apply tool\b", re.I), + re.compile( + r"\bpublic update|post|capital|funds|sign|broadcast|admin review|operator pass|operator action|apply tool\b", + re.I, + ), ], "reversibility": [ re.compile(r"\breversib(?:le|ility)|rollback|undo|irreversible|spend|public\b", re.I), @@ -489,7 +501,7 @@ SIGNAL_PATTERNS: dict[str, list[re.Pattern[str]]] = { re.compile( r"\bI (?:would|should|will) (?:ask|query|check|run|show|stage|offer)|" r"\bshould I\b|\bdo you want\b|\bwant me to\b|\bthe next action is\b|\bnext admin action\b|" - r"\bproof-changing action\b|\bNext Cory-style follow-up:\b", + r"\bproof-changing action\b|\bNext Cory-style follow-up:", re.I, ), ], diff --git a/tests/test_hermes_leoclean_skill_surfaces.py b/tests/test_hermes_leoclean_skill_surfaces.py index f4b329b..728f91f 100644 --- a/tests/test_hermes_leoclean_skill_surfaces.py +++ b/tests/test_hermes_leoclean_skill_surfaces.py @@ -17,6 +17,10 @@ def test_gcp_kb_skill_uses_cloudsql_bridge_not_vps_docker() -> None: assert "Do not answer" in text or "do not infer matrix approval" in text assert "Next Cory-style follow-up" in text assert "Always include the final line label" in text + assert "Cory Direct-Claim Answer Contract" in text + assert "not just pointer mismatch" in text + assert "staging write to `kb_stage.kb_proposals`" in text + assert "canonical identity requires DB rows plus" in text assert "I cannot claim canonical DB changed" in text assert "explicit operator/admin authorization" in text assert "docker exec" not in text @@ -41,6 +45,10 @@ def test_vps_kb_skill_keeps_vps_scope_explicit() -> None: assert "approved/staged or packet-ready but not canonical" in text assert "Next Cory-style follow-up" in text assert "Always include the final line label" in text + assert "Cory Direct-Claim Answer Contract" in text + assert "not just pointer mismatch" in text + assert "staging write to `kb_stage.kb_proposals`" in text + assert "canonical identity requires DB rows plus" in text assert "I cannot claim canonical DB changed" in text assert "explicit operator/admin authorization" in text assert "Do not call an approved proposal \"implemented\"" in text