teleo-infrastructure/tests/test_leo_agent_challenger_protocol.py

from __future__ import annotations

import copy

from scripts import leo_agent_challenger_protocol as protocol

CLAIM_ID = "fd159490-280d-4ede-84ef-faa169cff766"


def _empty_conversation() -> dict:
    return {
        "message_count": 0,
        "messages_sha256": protocol.canonical_sha256([]),
        "last_message_role": None,
        "last_message_content_sha256": None,
    }


def _model_trace(prompt: str, reply: str, session_id: str, *, model: str) -> list[dict]:
    prompt_sha = protocol.text_sha256(prompt)
    return [
        {
            "event": "turn_pre_llm_call",
            "session_id_sha256": session_id,
            "prompt_sha256": prompt_sha,
        },
        {
            "event": "post_api_request",
            "session_id_sha256": session_id,
            "prompt_sha256": prompt_sha,
            "provider": "test-provider",
            "model": model,
            "response_model": model,
        },
        {
            "event": "turn_post_llm_call",
            "session_id_sha256": session_id,
            "prompt_sha256": prompt_sha,
            "raw_assistant_response_sha256": protocol.text_sha256(reply),
        },
    ]


def _response_trace(reply: str) -> list[dict]:
    return [
        {
            "content_sha256": protocol.text_sha256(reply),
            "content_bytes": len(reply.encode()),
            "tool_calls_sha256": protocol.canonical_sha256([]),
            "tool_call_count": 0,
        }
    ]


def _database_trace(*, required: bool) -> dict:
    calls = []
    if required:
        calls.append(
            {
                "database_invocations": [
                    {"executable": "kb_tool.py", "subcommand": "context", "access_mode": "read_only"}
                ],
                "result": {
                    "row_ids": [CLAIM_ID],
                    "source_ids": [],
                    "content_sha256": "d" * 64,
                    "nonempty": True,
                    "error_detected": False,
                },
            }
        )
    return {
        "database_tool_call_count": len(calls),
        "database_tool_completed_count": len(calls),
        "database_tool_call_proven": bool(calls),
        "database_retrieval_receipt_proven": bool(calls),
        "database_tool_calls_read_only": True,
        "calls": calls,
    }


def _turn(
    turn_id: str,
    actor: str,
    prompt: str,
    reply: str,
    session_id: str,
    *,
    before: dict,
    after_hash: str,
    after_count: int,
    database_required: bool = False,
) -> dict:
    turn = {
        "turn_id": turn_id,
        "actor": actor,
        "input_prompt": prompt,
        "visible_turn_ids": [],
        "message": reply,
        "started_at_utc": f"2026-07-15T00:00:0{turn_id[-1]}+00:00",
        "ended_at_utc": f"2026-07-15T00:00:1{turn_id[-1]}+00:00",
        "process_identity_sha256": ("c" if actor == "challenger" else "e") * 64,
        "handler_session_key_sha256": session_id,
        "model_call_trace": _model_trace(prompt, reply, session_id, model=f"{actor}-model"),
        "model_response_trace": _response_trace(reply),
        "database_context_trace": [],
        "database_tool_trace": _database_trace(required=database_required),
        "tool_audit": {
            "tool_call_count": 0,
            "calls": [],
            "forbidden_mutation_detected": False,
            "unknown_tool_call_detected": False,
        },
        "conversation_before": before,
        "conversation_after": {
            "message_count": after_count,
            "messages_sha256": after_hash,
            "last_message_role": "assistant",
            "last_message_content_sha256": protocol.text_sha256(reply),
        },
        "conversation_history_prefix_preserved": True,
    }
    if actor == "challenger":
        turn["stateless_input_rebuilt"] = True
    turn["turn_execution_sha256"] = protocol.compute_turn_execution_sha256(turn)
    return turn


def passing_report() -> dict:
    objective = protocol.DEFAULT_OBJECTIVE
    challenger_session = "c" * 64
    leo_session = "e" * 64
    turns: list[dict] = []

    c1_prompt = protocol.build_challenger_prompt(1, objective, turns)
    c1_reply = (
        "Which current coordination belief would most affect a founder's product decision while outrunning its "
        "evidence? Please cite the database row, confidence, and support, then offer a narrower review-only claim."
    )
    c1 = _turn(
        "C1",
        "challenger",
        c1_prompt,
        c1_reply,
        challenger_session,
        before=_empty_conversation(),
        after_hash="1" * 64,
        after_count=2,
    )
    c1["visible_turn_ids"] = []
    c1["turn_execution_sha256"] = protocol.compute_turn_execution_sha256(c1)
    turns.append(c1)

    l1_reply = (
        "Claim fd159490 concerns coordination bottlenecks. It has confidence 0.90 but has zero external evidence; "
        "the database support is only an internal assertion. A narrower claim would test handoff latency."
    )
    l1 = _turn(
        "L1",
        "leo",
        c1_reply,
        l1_reply,
        leo_session,
        before=_empty_conversation(),
        after_hash="2" * 64,
        after_count=2,
        database_required=True,
    )
    l1["visible_turn_ids"] = ["C1"]
    l1["turn_execution_sha256"] = protocol.compute_turn_execution_sha256(l1)
    turns.append(l1)

    c2_prompt = protocol.build_challenger_prompt(2, objective, turns)
    c2_reply = (
        'You say "confidence 0.90 but has zero external evidence"; which coordination premise is assumption rather '
        "than observed support, what would falsify it, and will you produce Candidate A: and Candidate B: for "
        "review only?"
    )
    c2 = _turn(
        "C2",
        "challenger",
        c2_prompt,
        c2_reply,
        challenger_session,
        before=_empty_conversation(),
        after_hash="3" * 64,
        after_count=2,
    )
    c2["visible_turn_ids"] = ["C1", "L1"]
    c2["turn_execution_sha256"] = protocol.compute_turn_execution_sha256(c2)
    turns.append(c2)

    l2_reply = (
        "Claim fd159490 assumes tooling produces coordination outcomes; observed support only describes handoffs.\n"
        "Candidate A: Coordination handoff tooling reduces transfer latency.\n"
        "Candidate B: Structured handoff records improve error detection.\n"
        "Falsifier: unchanged latency and errors in a controlled comparison. Both remain pending_review for human "
        "review and are not live knowledge; nothing applied."
    )
    l2 = _turn(
        "L2",
        "leo",
        c2_reply,
        l2_reply,
        leo_session,
        before=copy.deepcopy(l1["conversation_after"]),
        after_hash="4" * 64,
        after_count=4,
        database_required=True,
    )
    l2["visible_turn_ids"] = ["C2"]
    l2["turn_execution_sha256"] = protocol.compute_turn_execution_sha256(l2)
    turns.append(l2)

    c3_prompt = protocol.build_challenger_prompt(3, objective, turns)
    c3_reply = (
        'I object to "Coordination handoff tooling reduces transfer latency" because it jumps from a tooling '
        "mechanism to a business outcome without causal evidence. Revise it, label Revised candidate: and "
        "Addresses objection:, and name the study that would resolve the dispute."
    )
    c3 = _turn(
        "C3",
        "challenger",
        c3_prompt,
        c3_reply,
        challenger_session,
        before=_empty_conversation(),
        after_hash="5" * 64,
        after_count=2,
    )
    c3["visible_turn_ids"] = ["C1", "L1", "C2", "L2"]
    c3["turn_execution_sha256"] = protocol.compute_turn_execution_sha256(c3)
    turns.append(c3)

    l3_reply = (
        "Revised candidate: In teams using a defined handoff protocol, measured transfer latency is lower than in "
        "the same teams without that protocol.\n"
        "Addresses objection: This narrows the coordination handoff tooling mechanism to measured transfer latency "
        "and does not claim a broader business outcome.\n"
        "A randomized or quasi-experimental team study would resolve the causal dispute. The packet remains "
        "pending_review for human review; approved is not apply, and applied_at is null."
    )
    l3 = _turn(
        "L3",
        "leo",
        c3_reply,
        l3_reply,
        leo_session,
        before=copy.deepcopy(l2["conversation_after"]),
        after_hash="6" * 64,
        after_count=6,
    )
    l3["visible_turn_ids"] = ["C3"]
    l3["turn_execution_sha256"] = protocol.compute_turn_execution_sha256(l3)
    turns.append(l3)

    challenger_role = {
        "role": "challenger",
        "runtime_kind": "stateless_openrouter_agent",
        "process_pid": 101,
        "process_start_ticks": "1001",
        "process_identity_sha256": "c" * 64,
        "session_key_sha256": challenger_session,
        "profile_realpath": None,
        "profile_realpath_sha256": "a" * 64,
        "profile_manifest": {"kind": "no_writable_profile", "manifest_sha256": "b" * 64},
        "memory_seed_empty": True,
        "session_seed_empty": True,
        "state_seed_empty": True,
        "tools_enabled": False,
        "kb_plugin_present": False,
        "soul_sha256": protocol.text_sha256(protocol.CHALLENGER_SOUL),
        "input_boundary": "objective plus transcript",
    }
    challenger_role["contract_sha256"] = protocol.compute_runtime_role_contract_sha256(challenger_role)
    leo_role = {
        "role": "leo",
        "runtime_kind": "gatewayrunner_temp_profile_no_model_tools",
        "process_pid": 202,
        "process_start_ticks": "2002",
        "process_identity_sha256": "e" * 64,
        "session_key_sha256": leo_session,
        "profile_realpath": "/tmp/leo-profile",
        "profile_realpath_sha256": "f" * 64,
        "profile_manifest": {"kind": "bounded", "manifest_sha256": "9" * 64},
        "memory_seed_empty": True,
        "session_seed_empty": True,
        "state_seed_empty": True,
        "tools_enabled": False,
        "kb_plugin_present": True,
        "soul_sha256": "8" * 64,
        "input_boundary": "challenger messages plus read-only context",
    }
    leo_role["contract_sha256"] = protocol.compute_runtime_role_contract_sha256(leo_role)
    fingerprint = {
        "status": "ok",
        "fingerprint_sha256": "7" * 64,
        "table_rows_sha256": "6" * 64,
        "structure_sha256": "5" * 64,
        "table_count": 39,
        "total_rows": 52000,
    }
    report = {
        "schema": protocol.REPORT_SCHEMA,
        "run_id": "test-run",
        "objective": objective,
        "turns": turns,
        "isolation": {
            "distinct_processes": True,
            "writable_roots_disjoint": True,
            "distinct_profile_roots": True,
            "transcript_only_bridge": True,
            "roles": {"challenger": challenger_role, "leo": leo_role},
        },
        "db_fingerprint_before": copy.deepcopy(fingerprint),
        "db_fingerprint_after": copy.deepcopy(fingerprint),
        "db_fingerprint_unchanged": True,
        "posted_to_telegram": False,
        "mutates_kb_by_harness": False,
        "live_behavior_manifest_unchanged": True,
        "service_before_after": {"unchanged_from_preexisting_live_readback": True},
        "cleanup": {
            "challenger_profile_removed": True,
            "leo_profile_removed": True,
            "temp_root_removed": True,
            "all_workers_stopped": True,
            "orphan_worker_count": 0,
        },
        "safety_gate": {"status": "pass"},
    }
    report["proposal_packet"] = protocol.build_proposal_packet(report)
    return report


def test_valid_isolated_agent_challenger_loop_passes_with_all_negative_controls_caught() -> None:
    receipt = protocol.verify_report(passing_report(), source_report_sha256="4" * 64)

    assert receipt["status"] == "pass"
    assert all(receipt["checks"].values())
    assert len(receipt["negative_controls"]) == 4
    assert all(item["status"] == "caught" for item in receipt["negative_controls"])
    assert receipt["current_tier"] == "T2_runtime"


def test_same_handler_or_model_session_fails_separation() -> None:
    report = passing_report()
    report["isolation"]["roles"]["challenger"]["session_key_sha256"] = "e" * 64
    report["isolation"]["roles"]["challenger"]["contract_sha256"] = (
        protocol.compute_runtime_role_contract_sha256(report["isolation"]["roles"]["challenger"])
    )
    for turn in report["turns"]:
        if turn["actor"] == "challenger":
            for event in turn["model_call_trace"]:
                event["session_id_sha256"] = "e" * 64
            turn["turn_execution_sha256"] = protocol.compute_turn_execution_sha256(turn)

    receipt = protocol.verify_report(report, source_report_sha256="4" * 64)

    assert receipt["status"] == "fail"
    assert receipt["checks"]["distinct_handler_session_identities"] is False
    assert receipt["checks"]["distinct_model_session_identities"] is False


def test_repeated_execution_hashes_fail_even_with_valid_parent_transcript() -> None:
    report = passing_report()
    for turn in report["turns"]:
        turn["turn_execution_sha256"] = "a" * 64

    receipt = protocol.verify_report(report, source_report_sha256="4" * 64)

    assert receipt["checks"]["all_six_turns_are_real_unique_model_executions"] is False


def test_unbound_extra_row_reference_fails_instead_of_passing_on_intersection() -> None:
    report = passing_report()
    l1 = next(turn for turn in report["turns"] if turn["turn_id"] == "L1")
    l1["message"] += " Also inspect claim deadbeef."
    l1["turn_execution_sha256"] = protocol.compute_turn_execution_sha256(l1)

    receipt = protocol.verify_report(report, source_report_sha256="4" * 64)

    assert receipt["checks"]["leo_selected_row_reference_is_bound_to_tool_receipt"] is False


def test_agreement_write_claim_and_revision_theater_fail() -> None:
    report = passing_report()
    c1 = next(turn for turn in report["turns"] if turn["turn_id"] == "C1")
    c1["message"] = "I agree entirely; use claim deadbeef and apply it now."
    c1["turn_execution_sha256"] = protocol.compute_turn_execution_sha256(c1)
    packet = report["proposal_packet"]
    packet["revision_link"]["candidate_changed"] = False

    receipt = protocol.verify_report(report, source_report_sha256="4" * 64)

    assert receipt["checks"]["challenger_does_not_merely_agree"] is False
    assert receipt["checks"]["challenger_row_ids_come_only_from_visible_leo_transcript"] is False
    assert receipt["checks"]["no_positive_write_claim"] is False
    assert receipt["checks"]["review_only_proposal_packet_matches_transcript"] is False


def test_unknown_or_mutating_model_tool_fails_closed() -> None:
    report = passing_report()
    l2 = next(turn for turn in report["turns"] if turn["turn_id"] == "L2")
    l2["tool_audit"] = {
        "tool_call_count": 1,
        "calls": [{"tool_name": "terminal", "forbidden_mutation": True}],
        "forbidden_mutation_detected": True,
        "unknown_tool_call_detected": True,
    }
    l2["turn_execution_sha256"] = protocol.compute_turn_execution_sha256(l2)

    receipt = protocol.verify_report(report, source_report_sha256="4" * 64)

    assert receipt["checks"]["no_forbidden_or_unknown_tool_call"] is False