fix natural causal objections and retain failure traces
This commit is contained in:
twentyOne2x
parent
e172207465
commit
f461f98b47
4 changed files with 123 additions and 7 deletions
|
|
@ -348,8 +348,8 @@ def evaluate_challenger_objection(candidate: str, message: str) -> dict[str, boo
|
|||
objection_prose = QUOTED_RE.sub(" ", message)
|
||||
mechanism = r"(?:mechanism|intervention|tooling|protocol)"
|
||||
outcome = (
|
||||
r"(?:adoption|business\s+outcome|downstream\s+outcome|latency|organizational\s+outcome|outcome|"
|
||||
r"performance|productivity|result|retention|revenue)"
|
||||
r"(?:adoption|business\s+outcome|downstream\s+outcome|failures?|latency|organizational\s+outcome|outcome|"
|
||||
r"performance|underperformance|productivity|result|retention|revenue)"
|
||||
)
|
||||
causal_verb = (
|
||||
r"(?:caus(?:e|es)|driv(?:e|es)|improv(?:e|es)|increas(?:e|es)|lead(?:s)?\s+to|produc(?:e|es)|"
|
||||
|
|
@ -364,6 +364,15 @@ def evaluate_challenger_objection(candidate: str, message: str) -> dict[str, boo
|
|||
r"(?:unsupported|unproven|unsubstantiated|without\s+causal\s+evidence|lacks?\s+causal\s+evidence|"
|
||||
r"missing\s+causal\s+evidence)"
|
||||
)
|
||||
evidence_denial = (
|
||||
r"(?:(?:does\s+not|doesn't|cannot|can't|fails?\s+to)(?:\s*,\s*by\s+itself\s*,)?\s+"
|
||||
r"(?:establish|show|demonstrate|support|infer|prove|justify)(?:\s+(?:that|whether))?|"
|
||||
r"(?:does\s+not|doesn't|cannot|can't)\s+provide\s+evidence\s+(?:for|of|that)|"
|
||||
r"without\s+(?:showing|demonstrating|establishing|proving)(?:\s+that)?)"
|
||||
)
|
||||
measurement_term = (
|
||||
r"(?:calibrat\w*|instrument\w*|measur\w*|methodolog\w*|operational\s+definition|rater\w*|timer\w*|timing)"
|
||||
)
|
||||
mechanism_relation = rf"\b{mechanism}\b(?:\s+[a-z-]+){{0,4}}\s+\b{causal_verb}\b.{{0,100}}\b{outcome}\b"
|
||||
jump_relation = (
|
||||
rf"\b(?:jumps?|leaps?|moves?)\s+from\b.{{0,160}}\b{mechanism}\b.{{0,160}}\bto\b"
|
||||
|
|
@ -384,10 +393,27 @@ def evaluate_challenger_objection(candidate: str, message: str) -> dict[str, boo
|
|||
rf"\b(?:data|evidence|observation|study)\b.{{0,160}}\b{denial}\b.{{0,120}}{mechanism_relation}",
|
||||
rf"{mechanism_relation}.{{0,120}}\b{causal_gap}\b",
|
||||
)
|
||||
denied_causal_relation = rf"\b{evidence_denial}\b.{{0,200}}\b{causal_verb}\b.{{0,100}}\b{outcome}\b"
|
||||
causal_identification_gap = (
|
||||
r"\b(?:without|lacks?|missing)\b.{0,80}\b(?:detail\w*|specif\w*|identif\w*|isolat\w*|separat\w*)\b"
|
||||
r".{0,120}\b(?:causal\s+(?:effect|pathway|relationship)|attribution|counterfactual|identification\s+strategy)\b"
|
||||
)
|
||||
quoted_candidate = bool(candidate and _quoted_from(message, candidate))
|
||||
explicit_objection = bool(re.search(r"\bi\s+(?:object|disagree)\b", objection_prose, re.IGNORECASE))
|
||||
causal_bridge = any(re.search(pattern, objection_prose, re.IGNORECASE) for pattern in bridge_patterns)
|
||||
evidence_gap = any(re.search(pattern, objection_prose, re.IGNORECASE) for pattern in evidence_gap_patterns)
|
||||
bound_denial = any(
|
||||
not re.search(
|
||||
rf"\b{measurement_term}\b",
|
||||
objection_prose[max(0, match.start() - 100) : min(len(objection_prose), match.end() + 40)],
|
||||
re.IGNORECASE,
|
||||
)
|
||||
for match in re.finditer(denied_causal_relation, objection_prose, re.IGNORECASE)
|
||||
)
|
||||
evidence_gap = (
|
||||
bound_denial
|
||||
or any(re.search(pattern, objection_prose, re.IGNORECASE) for pattern in evidence_gap_patterns)
|
||||
or bool(causal_bridge and re.search(causal_identification_gap, objection_prose, re.IGNORECASE))
|
||||
)
|
||||
required_axis = causal_bridge and evidence_gap
|
||||
return {
|
||||
"quoted_candidate": quoted_candidate,
|
||||
|
|
|
|||
|
|
@ -37,11 +37,11 @@ REMOTE_REPORT_FETCH_TIMEOUT_SECONDS = 45
|
|||
LOCAL_RECOVERY_DIR = Path(tempfile.gettempdir()) / "teleo-agent-challenger-loop-recovery"
|
||||
RETENTION_PROJECTION_SCHEMA = "livingip.leoAgentChallengerRetentionProjection.v1"
|
||||
BEHAVIOR_RETENTION_SCHEMA = "livingip.leoBehaviorManifestRetentionProjection.v1"
|
||||
RETENTION_ABSOLUTE_RUNTIME_PATH_RE = re.compile(
|
||||
r"(?<![A-Za-z0-9])/(?:Users|private|home|root|opt|tmp|var|etc)/[^\s\"']+"
|
||||
)
|
||||
RETENTION_FORBIDDEN_PATTERNS = (
|
||||
(
|
||||
"absolute runtime path",
|
||||
re.compile(r"(?<![A-Za-z0-9])/(?:Users|private|home|root|opt|tmp|var|etc)/[^\s\"']+"),
|
||||
),
|
||||
("absolute runtime path", RETENTION_ABSOLUTE_RUNTIME_PATH_RE),
|
||||
("PEM private key", re.compile(r"-----BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY-----")),
|
||||
("credentialed URL", re.compile(r"https?://[^/\s:@]+:[^/\s@]+@")),
|
||||
("JWT", re.compile(r"\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\b")),
|
||||
|
|
@ -1521,6 +1521,25 @@ def sanitize_report_for_retention(report: dict[str, Any]) -> dict[str, Any]:
|
|||
if remote_report_path:
|
||||
redacted_fields.append("remote_report_path")
|
||||
|
||||
traceback_projection: dict[str, Any] = {}
|
||||
traceback_tail = retained.get("traceback_tail")
|
||||
if isinstance(traceback_tail, list):
|
||||
raw_traceback_tail = [str(line) for line in traceback_tail]
|
||||
redacted_traceback_tail: list[str] = []
|
||||
path_occurrence_count = 0
|
||||
for line in raw_traceback_tail:
|
||||
redacted_line, count = RETENTION_ABSOLUTE_RUNTIME_PATH_RE.subn("[absolute-runtime-path-redacted]", line)
|
||||
redacted_traceback_tail.append(redacted_line)
|
||||
path_occurrence_count += count
|
||||
retained["traceback_tail"] = redacted_traceback_tail
|
||||
traceback_projection = {
|
||||
"raw_traceback_tail_sha256": protocol.canonical_sha256(raw_traceback_tail),
|
||||
"line_count": len(raw_traceback_tail),
|
||||
"absolute_runtime_path_occurrence_count": path_occurrence_count,
|
||||
}
|
||||
if path_occurrence_count:
|
||||
redacted_fields.append("traceback_tail[*].absolute_runtime_path")
|
||||
|
||||
transport = retained.get("execution_transport")
|
||||
if isinstance(transport, dict) and isinstance(transport.get("recovery"), dict):
|
||||
recovery = transport["recovery"]
|
||||
|
|
@ -1552,6 +1571,7 @@ def sanitize_report_for_retention(report: dict[str, Any]) -> dict[str, Any]:
|
|||
"raw_remote_report_sha256": (recovery.get("recovered_report_sha256") if isinstance(recovery, dict) else None),
|
||||
"pre_projection_report_sha256": pre_projection_report_sha256,
|
||||
"behavior_manifests": behavior_projection,
|
||||
"traceback_tail": traceback_projection,
|
||||
"redacted_fields": sorted(redacted_fields),
|
||||
"redacted_field_count": len(redacted_fields),
|
||||
"profile_realpath_hash_verified": profile_realpath_hash_verified,
|
||||
|
|
|
|||
|
|
@ -4,6 +4,8 @@ import copy
|
|||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from scripts import leo_agent_challenger_protocol as protocol
|
||||
|
||||
CLAIM_ID = "fd159490-280d-4ede-84ef-faa169cff766"
|
||||
|
|
@ -504,6 +506,61 @@ def test_natural_association_does_not_establish_causation_objection_passes_axis(
|
|||
assert mechanism_gate["passes"] is True
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"objection",
|
||||
(
|
||||
(
|
||||
'Leo, I object to Candidate A because "adoption failures attributable to coordination deficits" '
|
||||
"does not establish that coordination deficits are the cause of technological underperformance. "
|
||||
"You've identified an association, but not a causal link."
|
||||
),
|
||||
(
|
||||
'Leo, I object to Candidate A because "adoption failures attributable to coordination deficits" '
|
||||
"does not establish that coordination deficits cause technological underperformance or failure. "
|
||||
"The observation does not provide evidence for a causal link to the downstream outcome."
|
||||
),
|
||||
(
|
||||
'Leo, I object to Candidate A. Your claim that "adoption failures attributable to coordination deficits" '
|
||||
"does not provide evidence that these coordination deficits cause the adoption failures. You are asserting "
|
||||
"a causal link without demonstrating that the former directly leads to the latter."
|
||||
),
|
||||
(
|
||||
'I object to Candidate A. The claim that "governance frameworks have consistently been enacted after" '
|
||||
"does not, by itself, establish that this temporal "
|
||||
"sequence causes any specific downstream outcome or business impact. You have not provided evidence to "
|
||||
"bridge the observed pattern and its consequence or establish a causal link."
|
||||
),
|
||||
(
|
||||
'I object to "Coordination handoff tooling reduces transfer latency" because this is a '
|
||||
"mechanism-to-business-outcome causal overreach. It asserts a primary proximate cause without detailing "
|
||||
"the causal pathway or isolating the effect from other factors."
|
||||
),
|
||||
),
|
||||
)
|
||||
def test_live_natural_denied_causal_relation_passes_axis(objection: str) -> None:
|
||||
candidate = (
|
||||
"Major AI governance frameworks have consistently been enacted after, not before, the capability classes "
|
||||
"they address"
|
||||
if "governance frameworks" in objection
|
||||
else (
|
||||
"Coordination handoff tooling reduces transfer latency"
|
||||
if "Coordination handoff tooling" in objection
|
||||
else "adoption failures attributable to coordination deficits"
|
||||
)
|
||||
)
|
||||
|
||||
gate = protocol.evaluate_challenger_objection(candidate, objection)
|
||||
|
||||
assert gate == {
|
||||
"quoted_candidate": True,
|
||||
"explicit_objection": True,
|
||||
"causal_bridge": True,
|
||||
"evidence_gap": True,
|
||||
"required_axis": True,
|
||||
"passes": True,
|
||||
}
|
||||
|
||||
|
||||
def test_rejected_c3_is_regenerated_and_only_accepted_attempt_enters_transcript() -> None:
|
||||
report = passing_report()
|
||||
c3 = next(turn for turn in report["turns"] if turn["turn_id"] == "C3")
|
||||
|
|
|
|||
|
|
@ -183,6 +183,10 @@ def test_retention_projection_removes_runtime_paths_without_losing_hash_proof()
|
|||
original_contract = role["contract_sha256"]
|
||||
raw = {
|
||||
"remote_report_path": "/tmp/leo-agent-challenger-loop-test.json",
|
||||
"traceback_tail": [
|
||||
' File "/home/teleo/private/runtime.py", line 42, in exchange',
|
||||
"RuntimeError: semantic generation failed",
|
||||
],
|
||||
"execution_transport": {
|
||||
"recovery": {
|
||||
"remote_path": "/tmp/leo-agent-challenger-loop-test.json",
|
||||
|
|
@ -217,6 +221,15 @@ def test_retention_projection_removes_runtime_paths_without_losing_hash_proof()
|
|||
assert retained_role["contract_sha256"] == runner.protocol.compute_runtime_role_contract_sha256(retained_role)
|
||||
assert retained["retention_projection"]["original_runtime_role_contract_sha256"]["leo"] == (original_contract)
|
||||
assert retained["retention_projection"]["raw_remote_report_sha256"] == "5" * 64
|
||||
assert retained["traceback_tail"] == [
|
||||
' File "[absolute-runtime-path-redacted]", line 42, in exchange',
|
||||
"RuntimeError: semantic generation failed",
|
||||
]
|
||||
assert retained["retention_projection"]["traceback_tail"] == {
|
||||
"raw_traceback_tail_sha256": runner.protocol.canonical_sha256(raw["traceback_tail"]),
|
||||
"line_count": 2,
|
||||
"absolute_runtime_path_occurrence_count": 1,
|
||||
}
|
||||
assert retained["retention_projection"]["profile_realpath_hash_verified"] == {"leo": True}
|
||||
assert runner.sanitize_report_for_retention(retained) == retained
|
||||
retained["remote_stderr"] = "traceback from /home/teleo/private/runtime.py"
|
||||
|
|
|
|||
Loading…
Reference in a new issue