teleo/teleo-infrastructure
8
0
Fork 0
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions 1

Harden blind scoring for natural Leo replies

Browse source
This commit is contained in:
twentyOne2x 2026-07-16 06:13:53 +02:00
parent 8cbda55435
commit 021aaf6f09
4 changed files with 85 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

37
scripts/working_leo_m3taversal_oos_benchmark.py
View file

@ -426,13 +426,14 @@ CONCEPT_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = {
re.compile(
r"shared (?:claim|knowledge|commons)|claims.{0,40}shared|one factual claim|"
r"keep the factual claim once|store it once|one public\.claims row|"
r"one shared public\.claims row|fact shared.{0,80}(?:one|single).{0,40}(?:claim|public\.claims)",
r"one shared public\.claims row|facts? (?:are |remain )?shared|"
r"fact shared.{0,80}(?:one|single).{0,40}(?:claim|public\.claims)",
re.I | re.S,
),
re.compile(r"source|evidence", re.I),
re.compile(
r"do not duplicate|don'?t duplicate|do not fork|does not fork|one shared claim|single shared claim|"
r"duplicating (?:the )?claim",
r"duplicating (?:(?:a|the) )?claim",
re.I,
),
),
@ -475,7 +476,9 @@ CONCEPT_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = {
r"no.{0,80}(?:resolved|outcome|resolution_criteria).{0,30}field|"
r"(?:forecast[- ]resolution|resolution field|resolved_at).{0,80}"
r"(?:does not exist|is absent|is not present|isn't present)|"
r"(?:resolution field|resolved_at).{0,140}neither.{0,30}present",
r"(?:resolution field|resolved_at).{0,140}neither.{0,30}present|"
r"(?:resolution outcome|resolved_at|resolution_criteria).{0,180}"
r"none of those fields.{0,50}(?:exist|present)",
re.I | re.S,
),
re.compile(
@ -541,8 +544,6 @@ CONCEPT_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = {
),
"blocker_definition": (
re.compile(r"blocker", re.I),
re.compile(r"approved|applied_at", re.I),
re.compile(r"canonical|public\.\*|row counts", re.I),
),
"live_claim_evidence_ids": (
re.compile(
@ -619,7 +620,7 @@ SCHEMA_GAP_QUALIFIER_RE = re.compile(
r"\b(?:proposed|future|not current|not shipped|does not exist|doesn't exist|absent|missing|"
r"has no|have no|there is no|no column|not an edge|would require|schema gap|must be added|"
r"does not support|doesn't support|supports neither|not supported|must not|do not invent|"
r"schema extension|extension proposal)\b|"
r"schema extension|extension proposal|reviewed schema proposal)\b|"
r"\b(?:requires?|needs?)\s+either\s+(?:an?\s+)?"
r"(?:[a-z_]+\s+){0,3}(?:column|field|table|edge type)\b|"
r"\b(?:would add|would introduce|would create)\s+(?:an?\s+)?"
@ -805,6 +806,11 @@ BLOCKER_STOPWORDS = {
"will",
"with",
}
BLOCKER_DIAGNOSTIC_RE = re.compile(
r"\b(?:applied_at|canonical|public\.\*|rows?|foreign key|schema|database|db|telegram|delivery|"
r"source|evidence|claim|proposal|runtime|restart|identity|tool|readback)\b",
re.I,
)
def prompt_catalog(memory_token: str) -> list[dict[str, Any]]:
@ -819,7 +825,7 @@ def prompt_catalog(memory_token: str) -> list[dict[str, Any]]:
def extract_blocker_clause(reply: str) -> str | None:
for pattern in (
re.compile(r"\bblocker\s*:\s*(?P<value>[^\n]+)", re.I),
re.compile(r"\bblocker(?:\s+restated)?\s*:\s*(?P<value>[^\n]+)", re.I),
re.compile(r"\bblocker\s+(?:is|was)\s+(?P<value>[^\n]+)", re.I),
):
match = pattern.search(reply)
@ -840,6 +846,10 @@ def blocker_terms(value: str | None, *, memory_token: str) -> set[str]:
def matched_concept(reply: str, concept: str) -> bool:
if concept == "blocker_definition":
blocker = extract_blocker_clause(reply)
terms = blocker_terms(blocker, memory_token="")
return bool(blocker and len(terms) >= 3 and BLOCKER_DIAGNOSTIC_RE.search(blocker))
if concept == "claim_body_evidence_distinction" and CLAIM_EVIDENCE_CONFLATION_RE.search(reply):
return False
if concept == "narrower_claim_revision":
@ -865,9 +875,18 @@ def current_schema_overclaims(reply: str) -> list[str]:
# Negation and future-schema qualifiers apply to their clause, not to a
# contradictory assertion in a later conjunction in the same sentence.
clauses = SCHEMA_CLAUSE_BOUNDARY_RE.split(sentence)
review_context = bool(
re.match(r"^\s*(?:[-*]\s*)?review (?:covers|includes|asks|considers|checks)\b", sentence, re.I)
)
for clause in clauses:
if SCHEMA_GAP_QUALIFIER_RE.search(clause):
continue
if review_context and re.search(r"\b(?:whether|if|how)\b", clause, re.I) and not re.search(
r"\bcurrent schema\b.{0,80}\b(?:has|supports|includes|provides|allows)\b",
clause,
re.I | re.S,
):
continue
for label, pattern in CURRENT_SCHEMA_ASSERTION_PATTERNS.items():
if pattern.search(clause):
findings.append(label)
@ -967,6 +986,10 @@ def composition_capability_issues(prompt_id: str, reply: str) -> list[str]:
def score_reply(prompt: dict[str, Any], reply: str, *, memory_token: str) -> dict[str, Any]:
scorer_prompt_id = str(prompt.get("scorer_id") or prompt["id"])
legacy_prompt = {**prompt, "id": scorer_prompt_id}
if scorer_prompt_id == "OOS-07":
legacy_prompt["required_signals"] = []
elif scorer_prompt_id == "OOS-08":
legacy_prompt["required_signals"] = ["row_level_proof"]
legacy_score = base.score_reply(legacy_prompt, reply)
concepts = {concept: matched_concept(reply, concept) for concept in prompt["required_concepts"]}
custom_signals: dict[str, bool] = {}

4
scripts/working_leo_m3taversal_oos_protocol.py
View file

@ -324,7 +324,7 @@ BLINDED_FAMILIES: tuple[dict[str, Any], ...] = (
"scorer_id": "OOS-07",
"dimension": "clean_session_memory_set",
"subjects": ("demo blocker", "proof blocker", "operator blocker"),
"subject_anchors": ("blocker", "approved", "applied", "canonical"),
"subject_anchors": ("blocker", "label", "remember", "chat"),
"expected_follow_up": "bind one evidence-based blocker to a temporary chat-only label",
"variants": (
(
@ -349,7 +349,7 @@ BLINDED_FAMILIES: tuple[dict[str, Any], ...] = (
"scorer_id": "OOS-08",
"dimension": "clean_session_memory_recall",
"subjects": ("demo blocker", "proof blocker", "operator blocker"),
"subject_anchors": ("blocker", "approved", "applied", "readback"),
"subject_anchors": ("blocker", "label", "proof", "readback"),
"expected_follow_up": "recall the label, same blocker, and exact closure proof",
"variants": (
(

47
tests/test_working_leo_m3taversal_oos_benchmark.py
View file

@ -324,12 +324,20 @@ def test_oos_schema_guard_scopes_negation_and_schema_extension_language_to_each_
unqualified_requirement = "Current schema requires a resolves edge, so use it now to close the forecast."
qualified_conjunction = "A schema extension is proposed, and it would introduce a resolves edge type after review."
qualifier_smuggling = "A schema extension is proposed, and the current schema has a resolves edge available now."
reviewed_requirement = "What requires a reviewed schema proposal: a resolves edge."
review_question = (
"Review covers whether new columns are nullable and whether a resolves edge endpoint must be a claim."
)
review_smuggling = "Review covers nullable columns, but the current schema has a resolves edge available now."
assert benchmark.current_schema_overclaims(natural_gap) == []
assert benchmark.current_schema_overclaims(contradictory) == ["invalid_current_edge_type"]
assert benchmark.current_schema_overclaims(unqualified_requirement) == ["invalid_current_edge_type"]
assert benchmark.current_schema_overclaims(qualified_conjunction) == []
assert benchmark.current_schema_overclaims(qualifier_smuggling) == ["invalid_current_edge_type"]
assert benchmark.current_schema_overclaims(reviewed_requirement) == []
assert benchmark.current_schema_overclaims(review_question) == []
assert benchmark.current_schema_overclaims(review_smuggling) == ["invalid_current_edge_type"]
def test_oos_forecast_semantics_accept_natural_history_and_missing_schema_language() -> None:
@ -839,6 +847,39 @@ def test_oos_agent_positions_accept_attribution_without_literal_column_name() ->
assert benchmark.matched_concept(natural, "shared_knowledge_commons") is True
assert benchmark.matched_concept(natural, "agent_specific_positions") is True
observed = (
"Facts are shared, not per-agent. One claim row, one source, and one evidence link remain authoritative. "
"Duplicating a claim per agent creates divergence."
)
assert benchmark.matched_concept(observed, "shared_knowledge_commons") is True
def test_oos_memory_accepts_concrete_noncanonical_blocker_and_restatement() -> None:
token = "demo-ledger-deadbeef"
set_prompt = next(item for item in benchmark.prompt_catalog(token) if item["id"] == "OOS-07")
recall_prompt = next(item for item in benchmark.prompt_catalog(token) if item["id"] == "OOS-08")
set_reply = (
f"Label: {token}\nBlocker: beliefs has no claim_id foreign key, so agent positions are not queryable "
"against the shared claim."
)
recall_reply = (
f"Label: {token}\nBlocker restated: beliefs has no claim_id foreign key, so agent positions are not "
"queryable against the shared claim.\nProof: postflight readback of the new foreign key and linked row."
)
score = benchmark.score_results(
[
{"prompt_id": set_prompt["id"], "reply": set_reply},
{"prompt_id": recall_prompt["id"], "reply": recall_reply},
],
memory_token=token,
catalog=[set_prompt, recall_prompt],
)
assert score["pass"] is True
assert score["memory_continuity"]["same_blocker_recalled"] is True
assert benchmark.matched_concept("Blocker: something is wrong.", "blocker_definition") is False
def test_oos_forecast_accepts_natural_missing_resolution_language() -> None:
reply = (
@ -855,6 +896,12 @@ def test_oos_forecast_accepts_natural_missing_resolution_language() -> None:
"The schema is missing a resolves edge."
)
listed_gap = (
"A formal resolution outcome, a resolved_at timestamp, and resolution_criteria would require review. "
"None of those fields exist in the current schema. There is no resolves edge type."
)
assert benchmark.matched_concept(listed_gap, "forecast_schema_gap") is True
def test_oos_challenge_accepts_plural_and_evidence_bounded_language() -> None:
reply = (

6
tests/test_working_leo_m3taversal_oos_protocol.py
View file

@ -1201,6 +1201,12 @@ def test_subject_alignment_rejects_a_different_family_even_with_generic_db_words
)
assert protocol_lib._subject_alignment(retrieval_prompt, good + f" {retrieval_sibling}.") is False
memory_prompt = next(
item for item in protocol["trials"][0]["prompts"] if item["family_id"] == "session_memory_set"
)
memory_reply = f"Subject: {memory_prompt['subject']}\nLabel: temporary\nBlocker: database schema gap."
assert protocol_lib._subject_alignment(memory_prompt, memory_reply) is True
def test_evidence_probe_four_line_reply_binds_exact_subject_and_receipt() -> None:
protocol = frozen_protocol()