teleo/teleo-infrastructure
8
0
Fork 0
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions 1

Model conditional no-DB receipt gaps

Browse source
This commit is contained in:
twentyOne2x 2026-07-15 06:23:15 +02:00
parent 8506dd9359
commit 93eddbcd4f
2 changed files with 32 additions and 10 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

26
scripts/working_leo_m3taversal_oos_protocol.py
View file

@ -66,6 +66,10 @@ ABLATION_EXECUTION_ALLOWED_MISSING = GROUNDED_EXECUTION_ALLOWED_MISSING | frozen
"database_retrieval_receipt",
}
)
# The generic manifest requires a tool result only when a prompt is classified
# as database-relevant. Under the declared no-DB ablation that conditional
# result may therefore be absent on some turns and inapplicable on others.
ABLATION_EXECUTION_OPTIONAL_MISSING = frozenset({"database_tool_results"})
NON_DB_CONTRACT_IDS = frozenset({"reply_budget"})
TRIAL_SCORE_ARTIFACT_FIELDS = frozenset(
{
@ -1128,13 +1132,15 @@ def _benchmark_execution_chain(
"""
mode = report.get("grounding_mode")
allowed_missing = (
GROUNDED_EXECUTION_ALLOWED_MISSING
if mode == "grounded"
else ABLATION_EXECUTION_ALLOWED_MISSING
if mode == "db_tool_ablated"
else frozenset()
)
if mode == "grounded":
allowed_missing_sets = (GROUNDED_EXECUTION_ALLOWED_MISSING,)
elif mode == "db_tool_ablated":
allowed_missing_sets = (
ABLATION_EXECUTION_ALLOWED_MISSING,
ABLATION_EXECUTION_ALLOWED_MISSING | ABLATION_EXECUTION_OPTIONAL_MISSING,
)
else:
allowed_missing_sets = (frozenset(),)
results = [item for item in report.get("results") or [] if isinstance(item, dict)]
summary = report.get("execution_manifest_summary") or {}
executed_behavior = report.get("executed_behavior_manifest") or {}
@ -1184,8 +1190,8 @@ def _benchmark_execution_chain(
and turn.get("prompt_sha256") == hashlib.sha256(str(result.get("prompt") or "").encode()).hexdigest(),
"reply_bound": turn.get("reply_sha256")
== hashlib.sha256(str(result.get("reply") or "").encode()).hexdigest(),
"declared_missing_exact": missing_set == allowed_missing
and attribution.get("status") == ("incomplete" if allowed_missing else "complete"),
"declared_missing_exact": missing_set in allowed_missing_sets
and attribution.get("status") == ("incomplete" if missing_set else "complete"),
"chain_bound": conversation.get("previous_execution_sha256") == previous_execution_sha256,
"session_bound": _valid_sha256(session.get("session_key_sha256"))
and session.get("source_platform") == "telegram"
@ -1262,7 +1268,7 @@ def _benchmark_execution_chain(
}
return {
"mode": mode,
"allowed_missing_bindings": sorted(allowed_missing),
"allowed_missing_binding_sets": [sorted(items) for items in allowed_missing_sets],
"local_state_checks": local_state_checks,
"turn_checks": turn_checks,
"checks": checks,

16
tests/test_working_leo_m3taversal_oos_protocol.py
View file

@ -731,6 +731,22 @@ def test_execution_chain_allows_only_declared_ablation_and_control_goal() -> Non
assert validation["pass"] is False
assert validation["turn_checks"][trial["prompts"][0]["id"]]["declared_missing_exact"] is False
ablated = fake_report(protocol, trial, grounded=False)
manifest = ablated["results"][-1]["execution_manifest"]
manifest["attribution"]["missing_required_bindings"].append("database_tool_results")
stable = {
key: value
for key, value in manifest.items()
if key not in {"generated_at_utc", "execution_sha256"}
}
manifest["execution_sha256"] = protocol_lib.execution_manifest_lib.canonical_sha256(stable)
validation = protocol_lib._benchmark_execution_chain(
ablated,
expected_harness_git_head=protocol["harness_git_head"],
)
assert validation["pass"] is True
assert validation["turn_checks"][trial["prompts"][-1]["id"]]["declared_missing_exact"] is True
drifted = fake_report(protocol, trial, grounded=True)
drifted["oos_harness_git_state"]["git_head"] = "f" * 40
validation = protocol_lib._benchmark_execution_chain(