From f461f98b4794f830a64f3726e56077694fb4319c Mon Sep 17 00:00:00 2001 From: twentyOne2x Date: Wed, 15 Jul 2026 10:33:27 +0200 Subject: [PATCH] fix natural causal objections and retain failure traces --- scripts/leo_agent_challenger_protocol.py | 32 ++++++++++-- scripts/run_leo_agent_challenger_loop.py | 28 ++++++++-- tests/test_leo_agent_challenger_protocol.py | 57 +++++++++++++++++++++ tests/test_run_leo_agent_challenger_loop.py | 13 +++++ 4 files changed, 123 insertions(+), 7 deletions(-) diff --git a/scripts/leo_agent_challenger_protocol.py b/scripts/leo_agent_challenger_protocol.py index a463c3b..75dd111 100644 --- a/scripts/leo_agent_challenger_protocol.py +++ b/scripts/leo_agent_challenger_protocol.py @@ -348,8 +348,8 @@ def evaluate_challenger_objection(candidate: str, message: str) -> dict[str, boo objection_prose = QUOTED_RE.sub(" ", message) mechanism = r"(?:mechanism|intervention|tooling|protocol)" outcome = ( - r"(?:adoption|business\s+outcome|downstream\s+outcome|latency|organizational\s+outcome|outcome|" - r"performance|productivity|result|retention|revenue)" + r"(?:adoption|business\s+outcome|downstream\s+outcome|failures?|latency|organizational\s+outcome|outcome|" + r"performance|underperformance|productivity|result|retention|revenue)" ) causal_verb = ( r"(?:caus(?:e|es)|driv(?:e|es)|improv(?:e|es)|increas(?:e|es)|lead(?:s)?\s+to|produc(?:e|es)|" @@ -364,6 +364,15 @@ def evaluate_challenger_objection(candidate: str, message: str) -> dict[str, boo r"(?:unsupported|unproven|unsubstantiated|without\s+causal\s+evidence|lacks?\s+causal\s+evidence|" r"missing\s+causal\s+evidence)" ) + evidence_denial = ( + r"(?:(?:does\s+not|doesn't|cannot|can't|fails?\s+to)(?:\s*,\s*by\s+itself\s*,)?\s+" + r"(?:establish|show|demonstrate|support|infer|prove|justify)(?:\s+(?:that|whether))?|" + r"(?:does\s+not|doesn't|cannot|can't)\s+provide\s+evidence\s+(?:for|of|that)|" + r"without\s+(?:showing|demonstrating|establishing|proving)(?:\s+that)?)" + ) + measurement_term = ( + r"(?:calibrat\w*|instrument\w*|measur\w*|methodolog\w*|operational\s+definition|rater\w*|timer\w*|timing)" + ) mechanism_relation = rf"\b{mechanism}\b(?:\s+[a-z-]+){{0,4}}\s+\b{causal_verb}\b.{{0,100}}\b{outcome}\b" jump_relation = ( rf"\b(?:jumps?|leaps?|moves?)\s+from\b.{{0,160}}\b{mechanism}\b.{{0,160}}\bto\b" @@ -384,10 +393,27 @@ def evaluate_challenger_objection(candidate: str, message: str) -> dict[str, boo rf"\b(?:data|evidence|observation|study)\b.{{0,160}}\b{denial}\b.{{0,120}}{mechanism_relation}", rf"{mechanism_relation}.{{0,120}}\b{causal_gap}\b", ) + denied_causal_relation = rf"\b{evidence_denial}\b.{{0,200}}\b{causal_verb}\b.{{0,100}}\b{outcome}\b" + causal_identification_gap = ( + r"\b(?:without|lacks?|missing)\b.{0,80}\b(?:detail\w*|specif\w*|identif\w*|isolat\w*|separat\w*)\b" + r".{0,120}\b(?:causal\s+(?:effect|pathway|relationship)|attribution|counterfactual|identification\s+strategy)\b" + ) quoted_candidate = bool(candidate and _quoted_from(message, candidate)) explicit_objection = bool(re.search(r"\bi\s+(?:object|disagree)\b", objection_prose, re.IGNORECASE)) causal_bridge = any(re.search(pattern, objection_prose, re.IGNORECASE) for pattern in bridge_patterns) - evidence_gap = any(re.search(pattern, objection_prose, re.IGNORECASE) for pattern in evidence_gap_patterns) + bound_denial = any( + not re.search( + rf"\b{measurement_term}\b", + objection_prose[max(0, match.start() - 100) : min(len(objection_prose), match.end() + 40)], + re.IGNORECASE, + ) + for match in re.finditer(denied_causal_relation, objection_prose, re.IGNORECASE) + ) + evidence_gap = ( + bound_denial + or any(re.search(pattern, objection_prose, re.IGNORECASE) for pattern in evidence_gap_patterns) + or bool(causal_bridge and re.search(causal_identification_gap, objection_prose, re.IGNORECASE)) + ) required_axis = causal_bridge and evidence_gap return { "quoted_candidate": quoted_candidate, diff --git a/scripts/run_leo_agent_challenger_loop.py b/scripts/run_leo_agent_challenger_loop.py index b385c2a..e2373fd 100644 --- a/scripts/run_leo_agent_challenger_loop.py +++ b/scripts/run_leo_agent_challenger_loop.py @@ -37,11 +37,11 @@ REMOTE_REPORT_FETCH_TIMEOUT_SECONDS = 45 LOCAL_RECOVERY_DIR = Path(tempfile.gettempdir()) / "teleo-agent-challenger-loop-recovery" RETENTION_PROJECTION_SCHEMA = "livingip.leoAgentChallengerRetentionProjection.v1" BEHAVIOR_RETENTION_SCHEMA = "livingip.leoBehaviorManifestRetentionProjection.v1" +RETENTION_ABSOLUTE_RUNTIME_PATH_RE = re.compile( + r"(? dict[str, Any]: if remote_report_path: redacted_fields.append("remote_report_path") + traceback_projection: dict[str, Any] = {} + traceback_tail = retained.get("traceback_tail") + if isinstance(traceback_tail, list): + raw_traceback_tail = [str(line) for line in traceback_tail] + redacted_traceback_tail: list[str] = [] + path_occurrence_count = 0 + for line in raw_traceback_tail: + redacted_line, count = RETENTION_ABSOLUTE_RUNTIME_PATH_RE.subn("[absolute-runtime-path-redacted]", line) + redacted_traceback_tail.append(redacted_line) + path_occurrence_count += count + retained["traceback_tail"] = redacted_traceback_tail + traceback_projection = { + "raw_traceback_tail_sha256": protocol.canonical_sha256(raw_traceback_tail), + "line_count": len(raw_traceback_tail), + "absolute_runtime_path_occurrence_count": path_occurrence_count, + } + if path_occurrence_count: + redacted_fields.append("traceback_tail[*].absolute_runtime_path") + transport = retained.get("execution_transport") if isinstance(transport, dict) and isinstance(transport.get("recovery"), dict): recovery = transport["recovery"] @@ -1552,6 +1571,7 @@ def sanitize_report_for_retention(report: dict[str, Any]) -> dict[str, Any]: "raw_remote_report_sha256": (recovery.get("recovered_report_sha256") if isinstance(recovery, dict) else None), "pre_projection_report_sha256": pre_projection_report_sha256, "behavior_manifests": behavior_projection, + "traceback_tail": traceback_projection, "redacted_fields": sorted(redacted_fields), "redacted_field_count": len(redacted_fields), "profile_realpath_hash_verified": profile_realpath_hash_verified, diff --git a/tests/test_leo_agent_challenger_protocol.py b/tests/test_leo_agent_challenger_protocol.py index 6edc8ba..5cd7ff9 100644 --- a/tests/test_leo_agent_challenger_protocol.py +++ b/tests/test_leo_agent_challenger_protocol.py @@ -4,6 +4,8 @@ import copy import json from pathlib import Path +import pytest + from scripts import leo_agent_challenger_protocol as protocol CLAIM_ID = "fd159490-280d-4ede-84ef-faa169cff766" @@ -504,6 +506,61 @@ def test_natural_association_does_not_establish_causation_objection_passes_axis( assert mechanism_gate["passes"] is True +@pytest.mark.parametrize( + "objection", + ( + ( + 'Leo, I object to Candidate A because "adoption failures attributable to coordination deficits" ' + "does not establish that coordination deficits are the cause of technological underperformance. " + "You've identified an association, but not a causal link." + ), + ( + 'Leo, I object to Candidate A because "adoption failures attributable to coordination deficits" ' + "does not establish that coordination deficits cause technological underperformance or failure. " + "The observation does not provide evidence for a causal link to the downstream outcome." + ), + ( + 'Leo, I object to Candidate A. Your claim that "adoption failures attributable to coordination deficits" ' + "does not provide evidence that these coordination deficits cause the adoption failures. You are asserting " + "a causal link without demonstrating that the former directly leads to the latter." + ), + ( + 'I object to Candidate A. The claim that "governance frameworks have consistently been enacted after" ' + "does not, by itself, establish that this temporal " + "sequence causes any specific downstream outcome or business impact. You have not provided evidence to " + "bridge the observed pattern and its consequence or establish a causal link." + ), + ( + 'I object to "Coordination handoff tooling reduces transfer latency" because this is a ' + "mechanism-to-business-outcome causal overreach. It asserts a primary proximate cause without detailing " + "the causal pathway or isolating the effect from other factors." + ), + ), +) +def test_live_natural_denied_causal_relation_passes_axis(objection: str) -> None: + candidate = ( + "Major AI governance frameworks have consistently been enacted after, not before, the capability classes " + "they address" + if "governance frameworks" in objection + else ( + "Coordination handoff tooling reduces transfer latency" + if "Coordination handoff tooling" in objection + else "adoption failures attributable to coordination deficits" + ) + ) + + gate = protocol.evaluate_challenger_objection(candidate, objection) + + assert gate == { + "quoted_candidate": True, + "explicit_objection": True, + "causal_bridge": True, + "evidence_gap": True, + "required_axis": True, + "passes": True, + } + + def test_rejected_c3_is_regenerated_and_only_accepted_attempt_enters_transcript() -> None: report = passing_report() c3 = next(turn for turn in report["turns"] if turn["turn_id"] == "C3") diff --git a/tests/test_run_leo_agent_challenger_loop.py b/tests/test_run_leo_agent_challenger_loop.py index 849ac47..2423649 100644 --- a/tests/test_run_leo_agent_challenger_loop.py +++ b/tests/test_run_leo_agent_challenger_loop.py @@ -183,6 +183,10 @@ def test_retention_projection_removes_runtime_paths_without_losing_hash_proof() original_contract = role["contract_sha256"] raw = { "remote_report_path": "/tmp/leo-agent-challenger-loop-test.json", + "traceback_tail": [ + ' File "/home/teleo/private/runtime.py", line 42, in exchange', + "RuntimeError: semantic generation failed", + ], "execution_transport": { "recovery": { "remote_path": "/tmp/leo-agent-challenger-loop-test.json", @@ -217,6 +221,15 @@ def test_retention_projection_removes_runtime_paths_without_losing_hash_proof() assert retained_role["contract_sha256"] == runner.protocol.compute_runtime_role_contract_sha256(retained_role) assert retained["retention_projection"]["original_runtime_role_contract_sha256"]["leo"] == (original_contract) assert retained["retention_projection"]["raw_remote_report_sha256"] == "5" * 64 + assert retained["traceback_tail"] == [ + ' File "[absolute-runtime-path-redacted]", line 42, in exchange', + "RuntimeError: semantic generation failed", + ] + assert retained["retention_projection"]["traceback_tail"] == { + "raw_traceback_tail_sha256": runner.protocol.canonical_sha256(raw["traceback_tail"]), + "line_count": 2, + "absolute_runtime_path_occurrence_count": 1, + } assert retained["retention_projection"]["profile_realpath_hash_verified"] == {"leo": True} assert runner.sanitize_report_for_retention(retained) == retained retained["remote_stderr"] = "traceback from /home/teleo/private/runtime.py"