From ddb916ab7361dcabf4e0e793095a6e3dfa3610b5 Mon Sep 17 00:00:00 2001 From: twentyOne2x Date: Fri, 10 Jul 2026 02:45:01 +0200 Subject: [PATCH] Align Leo direct-claim benchmark semantics --- .../gcp/teleo-kb-bridge/SKILL.md | 4 ++ .../vps/teleo-kb-bridge/SKILL.md | 4 ++ scripts/working_leo_open_ended_benchmark.py | 62 ++++++++++++++----- tests/test_hermes_leoclean_skill_surfaces.py | 2 + .../test_working_leo_open_ended_benchmark.py | 37 +++++++++++ 5 files changed, 93 insertions(+), 16 deletions(-) diff --git a/hermes-agent/leoclean-skills/gcp/teleo-kb-bridge/SKILL.md b/hermes-agent/leoclean-skills/gcp/teleo-kb-bridge/SKILL.md index d7a106b..04503cb 100644 --- a/hermes-agent/leoclean-skills/gcp/teleo-kb-bridge/SKILL.md +++ b/hermes-agent/leoclean-skills/gcp/teleo-kb-bridge/SKILL.md @@ -82,6 +82,10 @@ the expected follow-up without needing to ask twice: 4. Next Cory-style follow-up: the one proof-changing or admin action that would change the answer. +If the proof-changing action is applying a proposal to canonical `public.*`, +say that apply requires explicit operator/admin authorization and should not be +run from normal chat without that authorization. + Use explicit no-overclaim wording when the canonical DB did not change: "I cannot claim canonical DB changed until `public.*` readback plus `applied_at`/postflight proof says it changed." diff --git a/hermes-agent/leoclean-skills/vps/teleo-kb-bridge/SKILL.md b/hermes-agent/leoclean-skills/vps/teleo-kb-bridge/SKILL.md index dcb5f8f..06c85aa 100644 --- a/hermes-agent/leoclean-skills/vps/teleo-kb-bridge/SKILL.md +++ b/hermes-agent/leoclean-skills/vps/teleo-kb-bridge/SKILL.md @@ -74,6 +74,10 @@ the expected follow-up without needing to ask twice: 4. Next Cory-style follow-up: the one proof-changing or admin action that would change the answer. +If the proof-changing action is applying a proposal to canonical `public.*`, +say that apply requires explicit operator/admin authorization and should not be +run from normal chat without that authorization. + Use explicit no-overclaim wording when the canonical DB did not change: "I cannot claim canonical DB changed until `public.*` readback plus `applied_at`/postflight proof says it changed." diff --git a/scripts/working_leo_open_ended_benchmark.py b/scripts/working_leo_open_ended_benchmark.py index a2ab9e9..628e285 100644 --- a/scripts/working_leo_open_ended_benchmark.py +++ b/scripts/working_leo_open_ended_benchmark.py @@ -356,31 +356,38 @@ CORY_DIRECT_CLAIM_FOLLOWUP_SCENARIOS: list[dict[str, Any]] = [ SIGNAL_PATTERNS: dict[str, list[re.Pattern[str]]] = { "state_semantics": [ - re.compile(r"\bpropos(?:ed|al)|pending[_ -]?review|approved|applied|not applied\b", re.I), + re.compile(r"\bpropos(?:ed|al)|pending[_ -]?review|approved|applied|not applied|staged|staging\b", re.I), re.compile( r"approved (?:is not|is not the same as|does not mean|!=|not the same as) applied|" r"not assume approval changed|approval changed .* canonical|approved[- ]not[- ]applied gap|" r"approval is a human gate, not a write|approved with zero canonical effect|" r"applied_at:\s*NULL|no applied_at|NOTHING was applied|not applied|payload-only|" - r"neither row has been promoted", + r"neither row has been promoted|canonical (?:layer|tables?|public\.\*) (?:is )?locked|" + r"reviewable proposal", re.I, ), ], "canonical_db": [ - re.compile(r"\bcanonical\b|\bpublic\.[a-z_]+|postgres|database rows?", re.I), + re.compile(r"\bcanonical\b|\bpublic\.[a-z_]+|\bpublic\b|postgres|database rows?", re.I), re.compile( r"\bpublic\.\*|claim_edges|claim_evidence|claims|sources|reasoning_tools|" - r"personas|strategies|beliefs|strategy_nodes|strategy_node_anchors\b", + r"personas|strategies|beliefs|strategy_nodes|strategy_node_anchors|canonical tables?\b", re.I, ), ], "row_level_proof": [ re.compile( r"\brow[- ]level|before/after|table[- ]level|row ids?|counts?\b|" - r"\bapplied_at:\s*NULL|superseded_by:\s*NULL|public\.[a-z_]+ rows?", + r"\bapplied_at\b|superseded_by:\s*NULL|public\.[a-z_]+ rows?|" + r"\b[a-f0-9]{8}\b.*\b(?:approved|applied|pending|canceled)|" + r"\b(?:approved|applied|pending|canceled)\b.*\b[a-f0-9]{8}\b", + re.I, + ), + re.compile( + r"\bquery|read back|verify|postflight|preflight|canonical state|applied_at|superseded_by|" + r"canonical `?public\.\*`?|proposal ledger\b", re.I, ), - re.compile(r"\bquery|read back|verify|postflight|preflight|canonical state|applied_at|superseded_by\b", re.I), ], "next_action": [ re.compile(r"\bnext\b|\bfirst\b|\bthen\b|\bwould check\b|\bneed to\b", re.I), @@ -391,7 +398,11 @@ SIGNAL_PATTERNS: dict[str, list[re.Pattern[str]]] = { r"\bnot (?:claim|assume|overclaim)|cannot (?:say|claim|overclaim)|do not know|without proof|if .* only\b|" r"\bonly ground truth\b|\bauthoritative test\b|candidate only|zero canonical effect|" r"\bno mutations? made\b|\bNOTHING was applied\b|\bout of scope\b|\bnot bundled\b|" - r"\bcannot (?:approve|publish|move|sign)|\bzero unilateral action\b", + r"\bcannot (?:approve|publish|move|sign)|\bzero unilateral action\b|" + r"\bnot canonical\b|\bnot applied\b|\bdoes not exist in (?:canonical )?(?:Leo's )?KB\b|" + r"\bonly in the proposal ledger\b|locked behind the apply tool|" + r"\bcannot demo\b|\bnot possible\b|\bnot a decision[- ]matrix vote\b|" + r"\bno auto[- ]apply path\b|\bschema gap\b|\bmissing apply tooling\b", re.I, ), re.compile( @@ -399,7 +410,12 @@ SIGNAL_PATTERNS: dict[str, list[re.Pattern[str]]] = { r"\bwithout\b.*\b(readback|proof|query|postflight|canonical)\b|" r"only ground truth.*canonical|canonical row.*authoritative test|readback confirms|zero canonical effect|" r"\bgovernance gap\b|\bno audit trail\b|\bwithout authorization\b|outside the PR-everything rule|" - r"\bno mutations? made\b|\bNOTHING was applied\b|\bpayload-only\b|\bout of scope\b|\bnot bundled\b", + r"\bno mutations? made\b|\bNOTHING was applied\b|\bpayload-only\b|\bout of scope\b|\bnot bundled\b|" + r"\buntil\b.*\bapplied_at\b|\bnot applied\b.*\bcanonical\b|" + r"\bnot in canonical\b|\bcanonical .* has(?:n't| not) (?:moved|changed)\b|" + r"\bapplied_at empty\b|\bnot applied\b|\bnot possible\b|" + r"\bnot a decision[- ]matrix vote\b|\blocked behind the apply tool\b|" + r"\bno auto[- ]apply path\b|\bschema gap\b|\bmissing apply tooling\b", re.I, ), ], @@ -407,7 +423,8 @@ SIGNAL_PATTERNS: dict[str, list[re.Pattern[str]]] = { re.compile(r"\bstage|proposal|review|approval|approved with changes\b", re.I), re.compile( r"\bapply\b.*\bguard|guarded\b.*\bapply|narrow\b.*\bapply\b|" - r"\bstaged separately|out of scope|proposal containing|source evidence required", + r"\bstaged separately|out of scope|proposal containing|source evidence required|" + r"\bmanual apply\b|operator tooling|admin tooling|admin review", re.I, ), ], @@ -424,12 +441,20 @@ SIGNAL_PATTERNS: dict[str, list[re.Pattern[str]]] = { re.compile(r"\bsource of truth|mirror|deploy|checkout|canonical repo\b", re.I), ], "fresh_readback": [ - re.compile(r"\bcheck|read back|inspect|git status|rev-parse|systemctl|service|no \.git|not a git repo\b", re.I), - re.compile(r"\bfresh|current|before claiming|last modified|latest mirrored|v0\.7\.0|MainPID\b", re.I), + re.compile( + r"\bcheck|read back|inspect|git status|rev-parse|systemctl|service|no \.git|not a git repo\b|" + r"\bschema\b|\btables?\b", + re.I, + ), + re.compile( + r"\bfresh|current|before claiming|last modified|latest mirrored|v0\.7\.0|MainPID|" + r"schema (?:absent|doesn't exist|does not exist)|tables?.*absent|do not exist", + re.I, + ), ], "authorization": [ - re.compile(r"\bauthori[sz]ation|approval|explicit|permission|allowed|not allowed\b", re.I), - re.compile(r"\bpublic update|post|capital|funds|sign|broadcast\b", re.I), + re.compile(r"\bauthori[sz]ation|approval|explicit|permission|allowed|not allowed|admin|operator\b", re.I), + re.compile(r"\bpublic update|post|capital|funds|sign|broadcast|admin review|operator pass|apply tool\b", re.I), ], "reversibility": [ re.compile(r"\breversib(?:le|ility)|rollback|undo|irreversible|spend|public\b", re.I), @@ -443,7 +468,11 @@ SIGNAL_PATTERNS: dict[str, list[re.Pattern[str]]] = { ], "decision_matrix": [ re.compile(r"\bdecision[- ]matrix|matrix_voters|proposal_votes|proposal_decisions|weighted voters?|tally\b", re.I), - re.compile(r"\bnot (?:yet )?shipped|schema lacks|tables? (?:do not|does not|don't) exist|check the schema|kb_stage\.kb_proposals\b", re.I), + re.compile( + r"\bnot (?:yet )?shipped|schema lacks|tables? (?:do not|does not|don't) exist|" + r"tables?.*absent|schema absent|doesn't exist yet|check the schema|kb_stage\.kb_proposals\b", + re.I, + ), ], "document_artifact_linking": [ re.compile(r"\btelegram_file_refs|document_evaluations|source_ref|public\.sources|raw PDFs?|document artifacts?\b", re.I), @@ -452,12 +481,13 @@ SIGNAL_PATTERNS: dict[str, list[re.Pattern[str]]] = { "cory_followup": [ re.compile( r"\bfollow[- ]?up|next (?:question|message|action|proof)|ask|confirm|choose|authorize|approval|" - r"show|run|audit|which\b", + r"show|run|audit|which|want me to\b", re.I, ), re.compile( r"\bI (?:would|should|will) (?:ask|query|check|run|show|stage|offer)|" - r"\bshould I\b|\bdo you want\b|\bthe next action is\b", + r"\bshould I\b|\bdo you want\b|\bwant me to\b|\bthe next action is\b|\bnext admin action\b|" + r"\bproof-changing action\b", re.I, ), ], diff --git a/tests/test_hermes_leoclean_skill_surfaces.py b/tests/test_hermes_leoclean_skill_surfaces.py index 5171c87..25e1d7b 100644 --- a/tests/test_hermes_leoclean_skill_surfaces.py +++ b/tests/test_hermes_leoclean_skill_surfaces.py @@ -17,6 +17,7 @@ def test_gcp_kb_skill_uses_cloudsql_bridge_not_vps_docker() -> None: assert "Do not answer" in text or "do not infer matrix approval" in text assert "Next Cory-style follow-up" in text assert "I cannot claim canonical DB changed" in text + assert "explicit operator/admin authorization" in text assert "docker exec" not in text assert "teleo-pg" not in text assert "on the VPS" not in text @@ -39,6 +40,7 @@ def test_vps_kb_skill_keeps_vps_scope_explicit() -> None: assert "approved/staged or packet-ready but not canonical" in text assert "Next Cory-style follow-up" in text assert "I cannot claim canonical DB changed" in text + assert "explicit operator/admin authorization" in text assert "Do not call an approved proposal \"implemented\"" in text assert "needs reviewer/operator apply tooling rather than inviting ad hoc SQL from chat" in squashed assert "Never end a normal Telegram answer by offering to run direct `INSERT`, `UPDATE`" in text diff --git a/tests/test_working_leo_open_ended_benchmark.py b/tests/test_working_leo_open_ended_benchmark.py index 7beda2e..5215243 100644 --- a/tests/test_working_leo_open_ended_benchmark.py +++ b/tests/test_working_leo_open_ended_benchmark.py @@ -172,6 +172,43 @@ def test_scores_direct_claim_followup_answer_as_pass(): assert bench.score_reply(prompt, reply)["pass"] is True +def test_scores_direct_claim_natural_db_split_answer_as_pass(): + prompt = next(p for p in bench.CORY_DIRECT_CLAIM_FOLLOWUP_SCENARIOS if p["id"] == "DC-01") + reply = ( + "Mostly still proposals. Applied to canonical public.* tables: 00957f6c applied 2026-07-09 and " + "f004bbb2 applied 2026-07-05. Approved but NOT applied: 14fa5ecc, ac036c9d, and a64df080 all have " + "applied_at empty. Everything else is pending_review or canceled. The substantive work is staged and " + "approved but not applied. Next admin action: run the apply tool or draft the review packet. Want me to " + "pull the full packet?" + ) + + assert bench.score_reply(prompt, reply)["pass"] is True + + +def test_scores_direct_claim_natural_helmer_answer_as_pass(): + prompt = next(p for p in bench.CORY_DIRECT_CLAIM_FOLLOWUP_SCENARIOS if p["id"] == "DC-02") + reply = ( + "No. 7 Powers is not in canonical public.*. Staged proposal a64df080 is approved, but applied_at is empty; " + "cea989fc is canceled. Proof-changing action: apply a64df080 through the admin review panel or apply tool. " + "Until applied_at is set and public.claims/public.claim_edges rows appear, it is only in the proposal ledger. " + "Want me to pull the full packet for review?" + ) + + assert bench.score_reply(prompt, reply)["pass"] is True + + +def test_scores_direct_claim_natural_matrix_absent_answer_as_pass(): + prompt = next(p for p in bench.CORY_DIRECT_CLAIM_FOLLOWUP_SCENARIOS if p["id"] == "DC-03") + reply = ( + "No. The decision-matrix schema doesn't exist yet: matrix_voters, proposal_decisions, and proposal_votes " + "tables are absent in both kb_stage and public. a64df080 has manual m3ta approval in kb_stage.kb_proposals, " + "not a matrix vote, and it is not applied to canonical public.*. Next action: use the admin apply tool or " + "pull the packet; want me to show it?" + ) + + assert bench.score_reply(prompt, reply)["pass"] is True + + def test_scores_approved_payload_only_answer_as_pass(): prompt = bench.OPEN_ENDED_PROMPTS[1] reply = (