413 lines
15 KiB
Python
413 lines
15 KiB
Python
from __future__ import annotations
|
|
|
|
import copy
|
|
|
|
from scripts import leo_turn_execution_manifest as manifest
|
|
|
|
PROMPT = "Challenge the weak acquisition claim."
|
|
REPLY = "The current evidence is shallow; here is a narrower candidate claim."
|
|
SESSION_SHA = "6" * 64
|
|
EMPTY_TOOL_CALLS_SHA = manifest.canonical_sha256([])
|
|
|
|
|
|
def behavior_manifest() -> dict:
|
|
return {
|
|
"behavior_sha256": "a" * 64,
|
|
"hermes_runtime": {
|
|
"git_head": "b" * 40,
|
|
"source_tree": {"sha256": "c" * 64, "file_count": 123},
|
|
},
|
|
"teleo_infrastructure_runtime": {
|
|
"git_head": "d" * 40,
|
|
"source_tree": {"sha256": "e" * 64, "file_count": 42},
|
|
},
|
|
"components": {
|
|
"runtime_identity": {"content": {"sha256": "f" * 64}},
|
|
"procedural_skills": {"content": {"sha256": "1" * 64}},
|
|
},
|
|
}
|
|
|
|
|
|
def retrieval_receipt(prompt: str = PROMPT) -> dict:
|
|
return {
|
|
"schema": "livingip.teleoKbRetrievalReceipt.v1",
|
|
"query_sha256": manifest.text_sha256(prompt[:16_000]),
|
|
"semantic_context_sha256": "3" * 64,
|
|
"artifact_state_sha256": "4" * 64,
|
|
"receipt_sha256": "5" * 64,
|
|
"claim_ids": ["claim-b", "claim-a"],
|
|
"source_ids": ["source-a"],
|
|
"counts": {"claims": 2, "context_rows": 1},
|
|
"read_consistency": {
|
|
"status": "stable_wal_marker",
|
|
"attempts": 1,
|
|
"database": "teleo",
|
|
"database_user": "runtime",
|
|
"system_identifier": "system-1",
|
|
"wal_lsn_before": "0/123",
|
|
"wal_lsn_after": "0/123",
|
|
},
|
|
}
|
|
|
|
|
|
def conversation_after(reply: str = REPLY, *, message_count: int = 2, marker: str = "a") -> dict:
|
|
return {
|
|
"message_count": message_count,
|
|
"messages_sha256": marker * 64,
|
|
"last_message_role": "assistant",
|
|
"last_message_content_sha256": manifest.text_sha256(reply),
|
|
}
|
|
|
|
|
|
def empty_conversation() -> dict:
|
|
return {
|
|
"message_count": 0,
|
|
"messages_sha256": manifest.canonical_sha256([]),
|
|
"last_message_role": None,
|
|
"last_message_content_sha256": None,
|
|
}
|
|
|
|
|
|
def turn_result(
|
|
*,
|
|
prompt: str = PROMPT,
|
|
reply: str = REPLY,
|
|
before: dict | None = None,
|
|
after_marker: str = "a",
|
|
turn: int = 1,
|
|
) -> dict:
|
|
prompt_sha = manifest.text_sha256(prompt)
|
|
reply_sha = manifest.text_sha256(reply)
|
|
before_trace = copy.deepcopy(before or empty_conversation())
|
|
after_trace = conversation_after(
|
|
reply,
|
|
message_count=int(before_trace["message_count"]) + 2,
|
|
marker=after_marker,
|
|
)
|
|
return {
|
|
"turn": turn,
|
|
"prompt_id": f"OOS-{turn:02d}",
|
|
"prompt": prompt,
|
|
"reply": reply,
|
|
"started_at_utc": "2026-07-14T12:00:00+00:00",
|
|
"ended_at_utc": "2026-07-14T12:00:02+00:00",
|
|
"mutates_kb": False,
|
|
"database_context_trace": [
|
|
{
|
|
"event": "pre_llm_call",
|
|
"status": "ok",
|
|
"injected": True,
|
|
"query_sha256": prompt_sha,
|
|
"contract_ids": ["reply_budget", "source_link_audit"],
|
|
"contract_sha256": "7" * 64,
|
|
"retrieval_receipt": retrieval_receipt(prompt),
|
|
},
|
|
{
|
|
"event": "post_llm_call",
|
|
"status": "ok",
|
|
"validated": True,
|
|
"transformed": False,
|
|
"query_sha256": prompt_sha,
|
|
"response_sha256": reply_sha,
|
|
"delivered_response_sha256": reply_sha,
|
|
},
|
|
],
|
|
"database_tool_trace": {
|
|
"schema": "livingip.leoKbToolTrace.v1",
|
|
"database_tool_call_count": 0,
|
|
"database_tool_completed_count": 0,
|
|
"database_tool_calls_read_only": True,
|
|
"calls": [],
|
|
},
|
|
"model_call_trace": [
|
|
{
|
|
"event": "turn_pre_llm_call",
|
|
"session_id_sha256": SESSION_SHA,
|
|
"prompt_sha256": prompt_sha,
|
|
},
|
|
{
|
|
"event": "post_api_request",
|
|
"api_call_count": 1,
|
|
"api_mode": "openai_chat_completions",
|
|
"model": "anthropic/claude-sonnet-4-6",
|
|
"provider": "openrouter",
|
|
"response_model": "anthropic/claude-sonnet-4-6",
|
|
"finish_reason": "stop",
|
|
"message_count": 4,
|
|
"assistant_content_chars": len(reply),
|
|
"assistant_tool_call_count": 0,
|
|
"task_id_sha256": SESSION_SHA,
|
|
"session_id_sha256": SESSION_SHA,
|
|
"base_url_sha256": "8" * 64,
|
|
"usage": {"input_tokens": 100, "output_tokens": 20, "ignored": "secret-shaped"},
|
|
"provider_response_id": None,
|
|
"provider_response_id_status": "not_exposed_by_hermes_post_api_request_hook",
|
|
},
|
|
{
|
|
"event": "turn_post_llm_call",
|
|
"session_id_sha256": SESSION_SHA,
|
|
"prompt_sha256": prompt_sha,
|
|
"raw_assistant_response_sha256": reply_sha,
|
|
},
|
|
],
|
|
"model_response_trace": [
|
|
{
|
|
"content_sha256": reply_sha,
|
|
"content_bytes": len(reply.encode()),
|
|
"tool_calls_sha256": EMPTY_TOOL_CALLS_SHA,
|
|
"tool_call_count": 0,
|
|
}
|
|
],
|
|
"conversation_before": before_trace,
|
|
"conversation_after": after_trace,
|
|
"conversation_history_prefix_preserved": True,
|
|
}
|
|
|
|
|
|
def database_fingerprint() -> dict:
|
|
marker = {
|
|
"database": "teleo",
|
|
"database_user": "runtime",
|
|
"system_identifier": "system-1",
|
|
"wal_lsn": "0/123",
|
|
}
|
|
return {
|
|
"schema": "livingip.teleoCanonicalDatabaseFingerprint.v1",
|
|
"status": "ok",
|
|
"fingerprint_sha256": "9" * 64,
|
|
"table_count": 39,
|
|
"total_rows": 52167,
|
|
"table_rows_sha256": "a" * 64,
|
|
"structure_sha256": "b" * 64,
|
|
"read_consistency": {"status": "stable_wal_marker", "before": marker, "after": marker},
|
|
}
|
|
|
|
|
|
def suite_safety() -> dict:
|
|
return {
|
|
"remote_returncode": 0,
|
|
"pass_runtime": True,
|
|
"live_behavior_manifest_unchanged": True,
|
|
"temp_profile_removed": True,
|
|
"service_unchanged": True,
|
|
"db_fingerprint_unchanged": True,
|
|
"model_call_trace_all_bound": True,
|
|
}
|
|
|
|
|
|
def build(
|
|
result: dict | None = None,
|
|
*,
|
|
fingerprint: dict | None = None,
|
|
harness_source: dict | None = None,
|
|
previous_execution_sha256: str | None = None,
|
|
expected_previous_conversation: dict | None = None,
|
|
) -> dict:
|
|
selected_fingerprint = database_fingerprint() if fingerprint is None else fingerprint
|
|
return manifest.build_turn_manifest(
|
|
result=result or turn_result(),
|
|
behavior_manifest=behavior_manifest(),
|
|
session_key="agent:main:telegram:group:private",
|
|
source={"platform": "telegram", "chat_type": "group"},
|
|
suite_mode="live_vps_gatewayrunner_temp_profile",
|
|
remote_run_id="run-1",
|
|
db_counts_before={"public.claims": 1837},
|
|
db_counts_after={"public.claims": 1837},
|
|
db_counts_changed=False,
|
|
db_fingerprint_before=selected_fingerprint,
|
|
db_fingerprint_after=selected_fingerprint,
|
|
posted_to_telegram=False,
|
|
mutates_kb_by_harness=False,
|
|
harness_source=harness_source
|
|
or {"git_head": "8" * 40, "worktree_clean": True, "status_sha256": manifest.text_sha256("")},
|
|
suite_safety=suite_safety(),
|
|
previous_execution_sha256=previous_execution_sha256,
|
|
expected_previous_conversation=expected_previous_conversation,
|
|
)
|
|
|
|
|
|
def test_turn_manifest_binds_model_runtime_session_and_exact_database_read() -> None:
|
|
value = build()
|
|
|
|
assert value["attribution"]["status"] == "complete"
|
|
assert value["replay_claim"]["status"] == "content_addressed_attribution"
|
|
assert value["model_execution"]["prompt_bound"] is True
|
|
assert value["model_execution"]["raw_response_bound"] is True
|
|
assert value["model_execution"]["delivered_response_bound"] is True
|
|
assert value["model_execution"]["calls"][0]["response_model"] == "anthropic/claude-sonnet-4-6"
|
|
assert value["model_execution"]["calls"][0]["usage"] == {"input_tokens": 100, "output_tokens": 20}
|
|
assert value["canonical_database"]["binding_status"] == "retrieval_receipt_bound"
|
|
assert value["canonical_database"]["fingerprint_before"]["table_count"] == 39
|
|
assert value["canonical_database"]["fingerprint_unchanged"] is True
|
|
receipt = value["canonical_database"]["context_retrieval_receipts"][0]
|
|
assert receipt["read_consistency"]["system_identifier"] == "system-1"
|
|
assert receipt["read_consistency"]["wal_lsn_before"] == receipt["read_consistency"]["wal_lsn_after"]
|
|
assert value["delivery_and_safety"]["posted_to_telegram"] is False
|
|
assert value["delivery_and_safety"]["kb_mutation_by_harness"] is False
|
|
assert value["replay_claim"]["runtime_source_artifacts_embedded"] is False
|
|
assert manifest.validate_turn_manifest(value) == []
|
|
|
|
encoded = str(value)
|
|
assert PROMPT not in encoded
|
|
assert "narrower candidate claim" not in encoded
|
|
assert "agent:main:telegram" not in encoded
|
|
assert "secret-shaped" not in encoded
|
|
|
|
|
|
def test_turn_manifest_detects_missing_actual_model_call() -> None:
|
|
result = turn_result()
|
|
result["model_call_trace"] = []
|
|
|
|
value = build(result)
|
|
|
|
assert value["attribution"]["status"] == "incomplete"
|
|
assert "actual_model_call" in value["attribution"]["missing_required_bindings"]
|
|
assert value["replay_claim"]["runtime_and_turn_inputs_content_addressed"] is False
|
|
|
|
|
|
def test_turn_manifest_hash_detects_tampering() -> None:
|
|
value = build()
|
|
tampered = copy.deepcopy(value)
|
|
tampered["turn"]["reply_sha256"] = "0" * 64
|
|
|
|
assert manifest.validate_turn_manifest(tampered) == ["execution_sha256_mismatch"]
|
|
|
|
|
|
def test_database_question_requires_a_stable_full_database_fingerprint() -> None:
|
|
value = build(fingerprint={"status": "error"})
|
|
|
|
assert value["attribution"]["status"] == "incomplete"
|
|
assert "database_fingerprint_before" in value["attribution"]["missing_required_bindings"]
|
|
assert "database_fingerprint_after" in value["attribution"]["missing_required_bindings"]
|
|
assert value["canonical_database"]["fingerprint_unchanged"] is False
|
|
|
|
|
|
def test_failed_database_context_lookup_cannot_be_attributed_as_complete() -> None:
|
|
result = turn_result()
|
|
result["database_context_trace"][0] = {
|
|
"event": "pre_llm_call",
|
|
"status": "error",
|
|
"injected": True,
|
|
"query_sha256": manifest.text_sha256(PROMPT),
|
|
"error": "timeout",
|
|
}
|
|
result["database_context_trace"][1]["status"] = "error"
|
|
|
|
value = build(result)
|
|
|
|
assert value["attribution"]["status"] == "incomplete"
|
|
assert "database_retrieval_receipt" in value["attribution"]["missing_required_bindings"]
|
|
assert "database_context_response_binding" in value["attribution"]["missing_required_bindings"]
|
|
|
|
|
|
def test_schema_only_or_wrong_query_receipt_is_rejected() -> None:
|
|
result = turn_result()
|
|
result["database_context_trace"][0]["retrieval_receipt"] = {
|
|
"schema": "livingip.teleoKbRetrievalReceipt.v1",
|
|
"query_sha256": "0" * 64,
|
|
}
|
|
|
|
value = build(result)
|
|
|
|
assert value["canonical_database"]["context_retrieval_receipts"] == []
|
|
assert "database_retrieval_receipt" in value["attribution"]["missing_required_bindings"]
|
|
|
|
|
|
def test_delivered_reply_hash_mismatch_fails_closed() -> None:
|
|
result = turn_result()
|
|
result["database_context_trace"][1]["delivered_response_sha256"] = "0" * 64
|
|
|
|
value = build(result)
|
|
|
|
assert "database_context_response_binding" in value["attribution"]["missing_required_bindings"]
|
|
|
|
|
|
def test_valid_response_transform_binds_raw_and_delivered_hashes() -> None:
|
|
result = turn_result()
|
|
raw_reply = "A much longer raw answer that the response contract replaces."
|
|
raw_sha = manifest.text_sha256(raw_reply)
|
|
result["database_context_trace"][1]["response_sha256"] = raw_sha
|
|
result["database_context_trace"][1]["transformed"] = True
|
|
result["model_call_trace"][-1]["raw_assistant_response_sha256"] = raw_sha
|
|
|
|
value = build(result)
|
|
|
|
assert value["attribution"]["status"] == "complete"
|
|
assert value["model_execution"]["raw_response_bound"] is True
|
|
assert value["model_execution"]["delivered_response_bound"] is True
|
|
assert value["canonical_database"]["context_binding"]["post_transformed"] is True
|
|
|
|
|
|
def test_dirty_or_untracked_harness_cannot_produce_complete_manifest() -> None:
|
|
value = build(
|
|
harness_source={"git_head": "8" * 40, "worktree_clean": False, "status_sha256": "1" * 64}
|
|
)
|
|
|
|
assert "harness_worktree_clean" in value["attribution"]["missing_required_bindings"]
|
|
|
|
|
|
def test_write_capable_database_tool_trace_fails_closed() -> None:
|
|
result = turn_result()
|
|
result["database_tool_trace"] = {
|
|
"schema": "livingip.leoKbToolTrace.v1",
|
|
"database_tool_call_count": 1,
|
|
"database_tool_completed_count": 1,
|
|
"database_tool_calls_read_only": False,
|
|
"calls": [
|
|
{
|
|
"tool_call_id_sha256": "1" * 64,
|
|
"arguments_sha256": "2" * 64,
|
|
"database_invocations": [
|
|
{
|
|
"access_mode": "write",
|
|
"command_sha256": "3" * 64,
|
|
"executable": "teleo-kb",
|
|
"subcommand": "apply",
|
|
}
|
|
],
|
|
"result": {
|
|
"content_sha256": "4" * 64,
|
|
"content_bytes": 10,
|
|
"error_detected": False,
|
|
"nonempty": True,
|
|
"row_ids": [],
|
|
"sha256_values": [],
|
|
},
|
|
}
|
|
],
|
|
}
|
|
|
|
value = build(result)
|
|
|
|
assert "database_tools_read_only" in value["attribution"]["missing_required_bindings"]
|
|
|
|
|
|
def test_later_turn_binds_previous_execution_and_conversation_state() -> None:
|
|
first_result = turn_result()
|
|
first = build(first_result)
|
|
second_result = turn_result(
|
|
prompt="Now revise that candidate after my challenge.",
|
|
reply="The revision is narrower and still remains a proposal.",
|
|
before=first_result["conversation_after"],
|
|
after_marker="b",
|
|
turn=2,
|
|
)
|
|
|
|
second = build(
|
|
second_result,
|
|
previous_execution_sha256=first["execution_sha256"],
|
|
expected_previous_conversation=first_result["conversation_after"],
|
|
)
|
|
|
|
assert second["attribution"]["status"] == "complete"
|
|
conversation = second["session_boundary"]["conversation"]
|
|
assert conversation["previous_execution_sha256"] == first["execution_sha256"]
|
|
assert conversation["prior_turn_state_bound"] is True
|
|
|
|
second_result["conversation_before"]["messages_sha256"] = "0" * 64
|
|
broken = build(
|
|
second_result,
|
|
previous_execution_sha256=first["execution_sha256"],
|
|
expected_previous_conversation=first_result["conversation_after"],
|
|
)
|
|
assert "conversation_prior_turn_binding" in broken["attribution"]["missing_required_bindings"]
|