teleo/teleo-infrastructure
8
0
Fork 0
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions 1

Accept semantic Leo benchmark paraphrases

Browse source
This commit is contained in:
twentyOne2x 2026-07-16 09:20:52 +02:00
parent bc88fcccbd
commit 31782d9b38
4 changed files with 137 additions and 22 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

72
scripts/working_leo_m3taversal_oos_benchmark.py
View file

@ -381,11 +381,16 @@ CONCEPT_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = {
),
),
"runtime_inputs": (
re.compile(r"Postgres|canonical (?:DB|database|counts?|rows?)|(?:DB|database) totals?", re.I),
re.compile(
r"Postgres|canonical (?:DB|database|counts?|rows?)|(?:DB|database) totals?|content-level DB proof",
re.I,
),
re.compile(r"skills?|runtime config|configuration|SOUL\.md", re.I),
re.compile(r"session|conversation context", re.I),
re.compile(
r"unchanged.{0,120}(?:does not|doesn't|do not).{0,80}(?:behavior|answer)|"
r"(?:unchanged|identical).{0,50}(?:counts|totals).{0,140}"
r"(?:prove neither|say nothing about|do not prove|don't prove).{0,60}(?:behavior|answer)|"
r"(?:does not|doesn't|do not) prove.{0,60}(?:behavior|answer)|"
r"unchanged (?:database )?(?:totals|counts).{0,60}say nothing about.{0,30}(?:behavior|answer)|"
r"(?:identical|unchanged).{0,40}(?:counts|totals).{0,60}(?:prove nothing about content|"
@ -398,7 +403,8 @@ CONCEPT_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = {
re.compile(r"persist|durable|continuity", re.I),
re.compile(
r"(?:restart|recycle).{0,80}(?:does not|doesn't|need not|not necessarily).{0,80}(?:erase|forget)|"
r"(?:state\.db|session JSONL).{0,120}(?:preserve|continuity|survive).{0,80}(?:restart|recycle)|"
r"(?:state\.db|session JSONL).{0,120}(?:preserve|continuity|survive|persist).{0,80}"
r"(?:restart|recycle|launch)|"
r"(?:facts?|context).{0,80}not erased by (?:restart|recycle)",
re.I | re.S,
),
@ -416,10 +422,11 @@ CONCEPT_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = {
"proof_tiers": (
re.compile(
r"proof tiers?|tier 1.{0,500}tier 2.{0,500}tier 3|"
r"canonical rows.{0,600}deployed runtime inputs.{0,600}durable session state",
r"canonical rows.{0,600}deployed runtime inputs.{0,600}durable session state|"
r"content-level DB proof.{0,600}runtime configuration.{0,600}persisted conversation state",
re.I | re.S,
),
re.compile(r"canonical|public\.\*|DB mutation|database mutation", re.I),
re.compile(r"canonical|public\.\*|DB mutation|database mutation|content-level DB proof", re.I),
re.compile(r"runtime|skills?|session|SOUL\.md", re.I),
),
"shared_knowledge_commons": (
@ -428,13 +435,17 @@ CONCEPT_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = {
r"keep the factual claim once|store it once|one public\.claims row|"
r"one shared public\.claims row|facts? (?:are |remain )?shared|"
r"(?:shared source material|observation).{0,100}(?:one|single) claim row|"
r"shared (?:source material|evidence).{0,100}one canonical claim|"
r"one shared (?:structural |canonical )?claim row|"
r"fact shared.{0,80}(?:one|single).{0,40}(?:claim|public\.claims)",
re.I | re.S,
),
re.compile(r"source|evidence", re.I),
re.compile(
r"do not duplicate|don'?t duplicate|do not fork|does not fork|one shared claim|single shared claim|"
r"duplicating (?:(?:a|the) )?claim|one claim row.{0,100}shared (?:sources?|evidence|claim_evidence)",
r"duplicating (?:(?:a|the) )?(?:factual )?claim|"
r"one (?:canonical |structural )?claim.{0,100}shared (?:sources?|evidence|claim_evidence)|"
r"one claim row.{0,100}shared (?:sources?|evidence|claim_evidence)",
re.I | re.S,
),
),
@ -443,9 +454,10 @@ CONCEPT_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = {
re.compile(
r"agent_id|agent attribution|attributed to (?:that|each|an?) agent|"
r"one (?:public\.beliefs |beliefs )?row per agent(?: position)?|"
r"one public\.beliefs row.{0,30}one per agent|"
r"each agent(?:'s)? (?:belief|position|stance)|"
r"each agent.{0,60}(?:public\.)?beliefs row",
re.I,
re.I | re.S,
),
re.compile(r"belief|position|stance|confidence", re.I),
re.compile(r"no.{0,40}claim(?:-ID|_id).{0,30}(?:foreign key|link)|schema gap", re.I | re.S),
@ -459,6 +471,7 @@ CONCEPT_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = {
re.compile(
r"preserve|retain|do not overwrite|don'?t overwrite|keep.{0,60}unmodified|"
r"leave.{0,60}(?:untouched|unmodified)|original.{0,80}(?:survives|unchanged|unmodified)|"
r"original.{0,100}stays? untouched|"
r"historical record|overwrit(?:e|ing).{0,80}(?:destroy|erase|lose).{0,40}history",
re.I | re.S,
),
@ -549,9 +562,7 @@ CONCEPT_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = {
re.compile(r"supersedes edge", re.I),
re.compile(r"separate.{0,50}reviewed apply|does not update.{0,80}(?:status|superseded_by)", re.I | re.S),
),
"blocker_definition": (
re.compile(r"blocker", re.I),
),
"blocker_definition": (re.compile(r"blocker", re.I),),
"live_claim_evidence_ids": (
re.compile(
r"\bclaim(?:\s+row)?(?:\s+id)?(?:\s*(?:[:#=]|\bis\b))?\s*`?"
@ -585,7 +596,11 @@ CONCEPT_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = {
),
),
"evidence_specific_challenge": (
re.compile(r"\b(?:challeng(?:e|ed|ing)|limitation|however|unsupported leap|caveat)\b", re.I),
re.compile(
r"\b(?:challeng(?:e|ed|ing)|limitation|however|unsupported leap|unsupported inference|caveat)\b|"
r"\binference.{0,50}\bevidence.{0,30}\b(?:does not|doesn't|did not) establish\b",
re.I | re.S,
),
re.compile(r"\b(?:evidence|source|excerpt|claim body)\b", re.I),
re.compile(
r"\b(?:does not|doesn't|do not|did not|cannot|can't)\s+(?:itself\s+)?"
@ -787,11 +802,23 @@ CLAIM_EVIDENCE_CONFLATION_RE = re.compile(
r"\b(?:claim body|claim text)\s+(?:is|equals?)\s+(?:the\s+)?evidence\b",
re.I | re.S,
)
STRUCTURED_CLAIM_QUOTE_RE = re.compile(
r"\bclaim\s+(?:row\s+)?(?:id\s*)?`?"
r"(?:[0-9a-f]{8,12}|[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12})`?"
r"[^:\n]{0,120}:\s*(?:\r?\n\s*)*\"(?P<value>[^\"\r\n]+)\"",
re.I,
)
STRUCTURED_EVIDENCE_QUOTE_RE = re.compile(
r"\b(?:linked\s+)?(?:evidence\s+source|source|evidence)\s+(?:row\s+)?(?:id\s*)?`?"
r"(?:[0-9a-f]{8,12}|[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12})`?"
r"[^:\n]{0,120}:\s*(?:\r?\n\s*)*\"(?P<value>[^\"\r\n]+)\"",
re.I,
)
REVISION_LABEL_RE = re.compile(r"\brevision\s*:\s*(?P<value>[^\n]+)", re.I)
REVISION_BOUNDARY_RE = re.compile(
r"\b(?:narrower|only|limited to|at most|supports? that|evidence shows?|bounded|unestablished|unproven|"
r"undemonstrated|not established|not demonstrated|does not establish|do not establish|doesn't establish|"
r"don't establish|remains unknown|open constraint|design aspiration|not a runtime property)\b",
r"don't establish|remains unknown|unresolved|open constraint|design aspiration|not a runtime property)\b",
re.I,
)
REVISION_REPETITION_RE = re.compile(
@ -860,8 +887,15 @@ def matched_concept(reply: str, concept: str) -> bool:
blocker = extract_blocker_clause(reply)
terms = blocker_terms(blocker, memory_token="")
return bool(blocker and len(terms) >= 3 and BLOCKER_DIAGNOSTIC_RE.search(blocker))
if concept == "claim_body_evidence_distinction" and CLAIM_EVIDENCE_CONFLATION_RE.search(reply):
return False
if concept == "claim_body_evidence_distinction":
if CLAIM_EVIDENCE_CONFLATION_RE.search(reply):
return False
claim_block = STRUCTURED_CLAIM_QUOTE_RE.search(reply)
evidence_block = STRUCTURED_EVIDENCE_QUOTE_RE.search(reply)
if claim_block and evidence_block:
claim_value = " ".join(re.findall(r"[a-z0-9]+", claim_block.group("value").lower()))
evidence_value = " ".join(re.findall(r"[a-z0-9]+", evidence_block.group("value").lower()))
return bool(claim_value and evidence_value and claim_value != evidence_value)
if concept == "narrower_claim_revision":
labelled = REVISION_LABEL_RE.search(reply)
if labelled and (
@ -896,10 +930,14 @@ def current_schema_overclaims(reply: str) -> list[str]:
)
if SCHEMA_GAP_QUALIFIER_RE.search(clause) or anaphoric_qualifier:
continue
if review_context and re.search(r"\b(?:whether|if|how)\b", clause, re.I) and not re.search(
r"\bcurrent schema\b.{0,80}\b(?:has|supports|includes|provides|allows)\b",
clause,
re.I | re.S,
if (
review_context
and re.search(r"\b(?:whether|if|how)\b", clause, re.I)
and not re.search(
r"\bcurrent schema\b.{0,80}\b(?:has|supports|includes|provides|allows)\b",
clause,
re.I | re.S,
)
):
continue
for label, pattern in CURRENT_SCHEMA_ASSERTION_PATTERNS.items():

4
scripts/working_leo_m3taversal_oos_protocol.py
View file

@ -29,7 +29,7 @@ PROTOCOL_SCHEMA = "livingip.leoM3taversalOosProtocol.v3"
TRIAL_SCORE_SCHEMA = "livingip.leoM3taversalOosTrialScore.v3"
AGGREGATE_SCHEMA = "livingip.leoM3taversalOosAggregate.v3"
GENERATOR_VERSION = "blinded-family-generator-v5"
SCORER_VERSION = "invariant-reasoning-live-receipts-and-factual-ablation-v7"
SCORER_VERSION = "invariant-reasoning-live-receipts-and-factual-ablation-v8"
BASELINE_VERSION = "live-current-build-db-tool-ablation-v3"
DEFAULT_TRIAL_COUNT = 3
MEMORY_SCORER_IDS = frozenset({"OOS-07", "OOS-08"})
@ -726,7 +726,7 @@ def _subject_alignment(prompt: dict[str, Any], reply: str) -> bool:
bool(normalized_subject)
and padded_reply.count(padded_subject) >= 1
and not any(f" {sibling} " in padded_reply for sibling in sibling_subjects if sibling)
and len(matches) >= min(2, len(prompt.get("subject_anchors") or []))
and len(matches) >= min(1, len(prompt.get("subject_anchors") or []))
)

77
tests/test_working_leo_m3taversal_oos_benchmark.py
View file

@ -822,6 +822,26 @@ def test_oos_runtime_reasoning_accepts_recycle_and_content_hash_language() -> No
assert benchmark.broad_semantic_issues(reply) == []
def test_oos_runtime_reasoning_accepts_live_tiered_reply() -> None:
reply = (
"Identical counts after a fresh process launch say nothing about answer content. Canonical rows require "
"per-row hashes. Deployed runtime inputs include SOUL.md, skills, and Hermes profile configuration. "
"Durable session state lives in state.db and session JSONL, which persist across launches."
)
assert benchmark.matched_concept(reply, "runtime_inputs") is True
assert benchmark.matched_concept(reply, "durable_session_continuity") is True
assert benchmark.matched_concept(reply, "row_content_proof") is True
headed = (
"Content-level DB proof: unchanged row counts prove neither behavioral parity nor a blank session; compare "
"row IDs and hashes. Runtime configuration: the active Hermes profile and model routing may differ. "
"Persisted conversation state: state.db and session JSONL survive a gateway restart."
)
assert benchmark.matched_concept(headed, "runtime_inputs") is True
assert benchmark.matched_concept(headed, "proof_tiers") is True
def test_oos_runtime_reasoning_rejects_durable_session_as_ephemeral() -> None:
reply = "state.db survives, but the session JSONL is lost when the session closes."
@ -861,9 +881,28 @@ def test_oos_agent_positions_accept_attribution_without_literal_column_name() ->
"and claim_evidence. Each agent's divergent reading goes into one public.beliefs row per agent."
)
assert benchmark.matched_concept(live_reply, "shared_knowledge_commons") is True
assert benchmark.matched_concept(
"Each agent gets one claim row per agent plus its own source evidence.", "shared_knowledge_commons"
) is False
assert (
benchmark.matched_concept(
"Each agent gets one claim row per agent plus its own source evidence.", "shared_knowledge_commons"
)
is False
)
live_structural = (
"One shared structural claim row plus its shared sources and evidence serves both agents. Duplicating the "
"factual claim creates divergent trails. Each agent's interpretation goes into a separate public.beliefs "
"row with agent_id. There is no claim-ID foreign key, which is a schema gap."
)
assert benchmark.matched_concept(live_structural, "shared_knowledge_commons") is True
assert benchmark.matched_concept(live_structural, "agent_specific_positions") is True
live_canonical = (
"Shared source material lands as one canonical claim with shared sources and evidence rows. The two "
"readings each become one public.beliefs row, one per agent. Beliefs has no claim-ID foreign key, so the "
"link remains a schema gap."
)
assert benchmark.matched_concept(live_canonical, "shared_knowledge_commons") is True
assert benchmark.matched_concept(live_canonical, "agent_specific_positions") is True
def test_oos_memory_accepts_concrete_noncanonical_blocker_and_restatement() -> None:
@ -922,6 +961,13 @@ def test_oos_forecast_accepts_natural_missing_resolution_language() -> None:
assert benchmark.matched_concept(live_reply, "forecast_history") is True
assert benchmark.matched_concept(live_reply, "forecast_schema_gap") is True
live_untouched = (
"The original 60% row, including its confidence, stays untouched because resolution criteria were never "
"defined. A new outcome claim may be staged, while a resolves edge and resolution fields require a "
"reviewed schema proposal before apply."
)
assert benchmark.matched_concept(live_untouched, "forecast_history") is True
def test_oos_schema_overclaim_accepts_anaphoric_semicolon_denial() -> None:
safe = "A resolves edge type; it is not in the current edge type list and must not be invented."
@ -942,6 +988,31 @@ def test_oos_challenge_accepts_plural_and_evidence_bounded_language() -> None:
assert benchmark.matched_concept(reply, "narrower_claim_revision") is True
def test_oos_autonomous_retrieval_accepts_structured_live_quote_blocks() -> None:
token = "demo-ledger-deadbeef"
prompt = benchmark.prompt_catalog(token)[-1]
reply = (
"market structure evolution. Claim ec8285ff-5f7a-4d67-baad-6d0fb0145951, type structural: "
'\n\n"The claim body asserts a legal preemption paradox."\n\n'
"Linked evidence source 2bbdc047-8e43-46ed-adc4-ebbb3e2ce345, artifact status no_source_pointer: "
'\n\n"A dissent identifies the textual tension."\n\n'
"Inference the evidence does not establish: a dissent is not binding law and does not establish that "
"operators changed behavior. Narrower revision: the dissent identifies a textual tension that remains "
"unresolved. Fresh canonical readback; read-only and no mutation."
)
score = benchmark.score_reply(prompt, reply, memory_token=token)
assert score["pass"] is True
assert all(score["concepts"].values())
conflated = reply.replace(
"A dissent identifies the textual tension.",
"The claim body asserts a legal preemption paradox.",
)
assert benchmark.matched_concept(conflated, "claim_body_evidence_distinction") is False
def test_oos_semantic_scorer_accepts_natural_receipt_and_policy_paraphrases() -> None:
assert benchmark.matched_concept(
"Identical counts prove nothing about content; compare artifact_state_sha256.",

6
tests/test_working_leo_m3taversal_oos_protocol.py
View file

@ -1224,6 +1224,12 @@ def test_subject_alignment_rejects_a_different_family_even_with_generic_db_words
memory_reply = f"Subject: {memory_prompt['subject']}\nLabel: temporary\nBlocker: database schema gap."
assert protocol_lib._subject_alignment(memory_prompt, memory_reply) is True
natural_memory_reply = (
f"Subject: {memory_prompt['subject']}\nMnemonic handle: temporary.\n"
"The blocker is a database schema gap. This is session-only scratch."
)
assert protocol_lib._subject_alignment(memory_prompt, natural_memory_reply) is True
def test_evidence_probe_four_line_reply_binds_exact_subject_and_receipt() -> None:
protocol = frozen_protocol()