Model conditional no-DB receipt gaps
This commit is contained in:
parent
8506dd9359
commit
93eddbcd4f
2 changed files with 32 additions and 10 deletions
|
|
@ -66,6 +66,10 @@ ABLATION_EXECUTION_ALLOWED_MISSING = GROUNDED_EXECUTION_ALLOWED_MISSING | frozen
|
||||||
"database_retrieval_receipt",
|
"database_retrieval_receipt",
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
# The generic manifest requires a tool result only when a prompt is classified
|
||||||
|
# as database-relevant. Under the declared no-DB ablation that conditional
|
||||||
|
# result may therefore be absent on some turns and inapplicable on others.
|
||||||
|
ABLATION_EXECUTION_OPTIONAL_MISSING = frozenset({"database_tool_results"})
|
||||||
NON_DB_CONTRACT_IDS = frozenset({"reply_budget"})
|
NON_DB_CONTRACT_IDS = frozenset({"reply_budget"})
|
||||||
TRIAL_SCORE_ARTIFACT_FIELDS = frozenset(
|
TRIAL_SCORE_ARTIFACT_FIELDS = frozenset(
|
||||||
{
|
{
|
||||||
|
|
@ -1128,13 +1132,15 @@ def _benchmark_execution_chain(
|
||||||
"""
|
"""
|
||||||
|
|
||||||
mode = report.get("grounding_mode")
|
mode = report.get("grounding_mode")
|
||||||
allowed_missing = (
|
if mode == "grounded":
|
||||||
GROUNDED_EXECUTION_ALLOWED_MISSING
|
allowed_missing_sets = (GROUNDED_EXECUTION_ALLOWED_MISSING,)
|
||||||
if mode == "grounded"
|
elif mode == "db_tool_ablated":
|
||||||
else ABLATION_EXECUTION_ALLOWED_MISSING
|
allowed_missing_sets = (
|
||||||
if mode == "db_tool_ablated"
|
ABLATION_EXECUTION_ALLOWED_MISSING,
|
||||||
else frozenset()
|
ABLATION_EXECUTION_ALLOWED_MISSING | ABLATION_EXECUTION_OPTIONAL_MISSING,
|
||||||
)
|
)
|
||||||
|
else:
|
||||||
|
allowed_missing_sets = (frozenset(),)
|
||||||
results = [item for item in report.get("results") or [] if isinstance(item, dict)]
|
results = [item for item in report.get("results") or [] if isinstance(item, dict)]
|
||||||
summary = report.get("execution_manifest_summary") or {}
|
summary = report.get("execution_manifest_summary") or {}
|
||||||
executed_behavior = report.get("executed_behavior_manifest") or {}
|
executed_behavior = report.get("executed_behavior_manifest") or {}
|
||||||
|
|
@ -1184,8 +1190,8 @@ def _benchmark_execution_chain(
|
||||||
and turn.get("prompt_sha256") == hashlib.sha256(str(result.get("prompt") or "").encode()).hexdigest(),
|
and turn.get("prompt_sha256") == hashlib.sha256(str(result.get("prompt") or "").encode()).hexdigest(),
|
||||||
"reply_bound": turn.get("reply_sha256")
|
"reply_bound": turn.get("reply_sha256")
|
||||||
== hashlib.sha256(str(result.get("reply") or "").encode()).hexdigest(),
|
== hashlib.sha256(str(result.get("reply") or "").encode()).hexdigest(),
|
||||||
"declared_missing_exact": missing_set == allowed_missing
|
"declared_missing_exact": missing_set in allowed_missing_sets
|
||||||
and attribution.get("status") == ("incomplete" if allowed_missing else "complete"),
|
and attribution.get("status") == ("incomplete" if missing_set else "complete"),
|
||||||
"chain_bound": conversation.get("previous_execution_sha256") == previous_execution_sha256,
|
"chain_bound": conversation.get("previous_execution_sha256") == previous_execution_sha256,
|
||||||
"session_bound": _valid_sha256(session.get("session_key_sha256"))
|
"session_bound": _valid_sha256(session.get("session_key_sha256"))
|
||||||
and session.get("source_platform") == "telegram"
|
and session.get("source_platform") == "telegram"
|
||||||
|
|
@ -1262,7 +1268,7 @@ def _benchmark_execution_chain(
|
||||||
}
|
}
|
||||||
return {
|
return {
|
||||||
"mode": mode,
|
"mode": mode,
|
||||||
"allowed_missing_bindings": sorted(allowed_missing),
|
"allowed_missing_binding_sets": [sorted(items) for items in allowed_missing_sets],
|
||||||
"local_state_checks": local_state_checks,
|
"local_state_checks": local_state_checks,
|
||||||
"turn_checks": turn_checks,
|
"turn_checks": turn_checks,
|
||||||
"checks": checks,
|
"checks": checks,
|
||||||
|
|
|
||||||
|
|
@ -731,6 +731,22 @@ def test_execution_chain_allows_only_declared_ablation_and_control_goal() -> Non
|
||||||
assert validation["pass"] is False
|
assert validation["pass"] is False
|
||||||
assert validation["turn_checks"][trial["prompts"][0]["id"]]["declared_missing_exact"] is False
|
assert validation["turn_checks"][trial["prompts"][0]["id"]]["declared_missing_exact"] is False
|
||||||
|
|
||||||
|
ablated = fake_report(protocol, trial, grounded=False)
|
||||||
|
manifest = ablated["results"][-1]["execution_manifest"]
|
||||||
|
manifest["attribution"]["missing_required_bindings"].append("database_tool_results")
|
||||||
|
stable = {
|
||||||
|
key: value
|
||||||
|
for key, value in manifest.items()
|
||||||
|
if key not in {"generated_at_utc", "execution_sha256"}
|
||||||
|
}
|
||||||
|
manifest["execution_sha256"] = protocol_lib.execution_manifest_lib.canonical_sha256(stable)
|
||||||
|
validation = protocol_lib._benchmark_execution_chain(
|
||||||
|
ablated,
|
||||||
|
expected_harness_git_head=protocol["harness_git_head"],
|
||||||
|
)
|
||||||
|
assert validation["pass"] is True
|
||||||
|
assert validation["turn_checks"][trial["prompts"][-1]["id"]]["declared_missing_exact"] is True
|
||||||
|
|
||||||
drifted = fake_report(protocol, trial, grounded=True)
|
drifted = fake_report(protocol, trial, grounded=True)
|
||||||
drifted["oos_harness_git_state"]["git_head"] = "f" * 40
|
drifted["oos_harness_git_state"]["git_head"] = "f" * 40
|
||||||
validation = protocol_lib._benchmark_execution_chain(
|
validation = protocol_lib._benchmark_execution_chain(
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue