157 lines
5.1 KiB
Python
157 lines
5.1 KiB
Python
"""Tests for secret-safe Hermes knowledge-base tool attribution."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / "scripts"))
|
|
|
|
from leo_tool_trace import extract_kb_tool_trace # noqa: E402
|
|
|
|
|
|
def test_extracts_completed_read_only_context_call_and_receipt() -> None:
|
|
claim_id = "2a7ae257-d01d-46f4-b813-63f81bb9c7c7"
|
|
source_id = "15740795-1234-4abc-8def-1234567890ab"
|
|
semantic_sha = "8" * 64
|
|
artifact_sha = "4" * 64
|
|
messages = [
|
|
{
|
|
"role": "assistant",
|
|
"tool_calls": [
|
|
{
|
|
"id": "call-secret-looking-id",
|
|
"function": {
|
|
"name": "terminal",
|
|
"arguments": '{"command":"teleo-kb context \\\"AI sandbagging\\\""}',
|
|
},
|
|
}
|
|
],
|
|
},
|
|
{
|
|
"role": "tool",
|
|
"tool_call_id": "call-secret-looking-id",
|
|
"content": (
|
|
"# Teleo KB Context\n"
|
|
"- schema: `livingip.teleoKbRetrievalReceipt.v1`\n"
|
|
f"- semantic context SHA-256: `{semantic_sha}`\n"
|
|
f"- artifact state SHA-256: `{artifact_sha}`\n"
|
|
"- DB read consistency: `stable_wal_marker`; attempts: `1`\n"
|
|
f"claim_id: {claim_id}; source_id: {source_id}\n"
|
|
),
|
|
},
|
|
]
|
|
|
|
trace = extract_kb_tool_trace(messages)
|
|
|
|
assert trace["database_tool_call_proven"] is True
|
|
assert trace["database_retrieval_receipt_proven"] is True
|
|
assert trace["database_tool_calls_read_only"] is True
|
|
assert trace["access_modes"] == ["read_only"]
|
|
assert trace["calls"][0]["database_invocations"][0]["subcommand"] == "context"
|
|
assert trace["calls"][0]["result"]["row_ids"] == sorted([claim_id, source_id])
|
|
assert trace["calls"][0]["result"]["retrieval_receipt"] == {
|
|
"schema": "livingip.teleoKbRetrievalReceipt.v1",
|
|
"semantic_context_sha256": semantic_sha,
|
|
"artifact_state_sha256": artifact_sha,
|
|
"read_consistency_status": "stable_wal_marker",
|
|
}
|
|
|
|
|
|
def test_never_retains_raw_command_arguments_or_result() -> None:
|
|
secret = "password=do-not-retain"
|
|
messages = [
|
|
{
|
|
"role": "assistant",
|
|
"tool_calls": [
|
|
{
|
|
"id": "call-1",
|
|
"function": {
|
|
"name": "terminal",
|
|
"arguments": {"command": f"teleo-kb search demo; echo {secret}"},
|
|
},
|
|
}
|
|
],
|
|
},
|
|
{"role": "tool", "tool_call_id": "call-1", "content": f"result {secret}"},
|
|
]
|
|
|
|
trace = extract_kb_tool_trace(messages)
|
|
rendered = str(trace)
|
|
|
|
assert trace["database_tool_call_proven"] is True
|
|
assert secret not in rendered
|
|
assert "teleo-kb search demo" not in rendered
|
|
assert trace["raw_commands_retained"] is False
|
|
assert trace["raw_arguments_retained"] is False
|
|
assert trace["raw_results_retained"] is False
|
|
|
|
|
|
def test_staging_write_is_classified_and_not_called_read_only() -> None:
|
|
messages = [
|
|
{
|
|
"role": "assistant",
|
|
"tool_calls": [
|
|
{
|
|
"id": "call-write",
|
|
"function": {
|
|
"name": "terminal",
|
|
"arguments": '{"command":"teleo-kb propose-edge --from a --to b"}',
|
|
},
|
|
}
|
|
],
|
|
},
|
|
{"role": "tool", "tool_call_id": "call-write", "content": "pending_review"},
|
|
]
|
|
|
|
trace = extract_kb_tool_trace(messages)
|
|
|
|
assert trace["database_tool_call_proven"] is True
|
|
assert trace["database_tool_calls_read_only"] is False
|
|
assert trace["access_modes"] == ["staging_write"]
|
|
|
|
|
|
def test_unmatched_or_failed_result_does_not_prove_database_call() -> None:
|
|
unmatched = [
|
|
{
|
|
"role": "assistant",
|
|
"tool_calls": [
|
|
{
|
|
"id": "call-missing",
|
|
"function": {"name": "terminal", "arguments": '{"command":"teleo-kb show abc"}'},
|
|
}
|
|
],
|
|
}
|
|
]
|
|
failed = [
|
|
*unmatched,
|
|
{
|
|
"role": "tool",
|
|
"tool_call_id": "call-missing",
|
|
"content": "Traceback (most recent call last): connection failed",
|
|
}
|
|
]
|
|
|
|
assert extract_kb_tool_trace(unmatched)["database_tool_call_proven"] is False
|
|
assert extract_kb_tool_trace(failed)["database_tool_call_proven"] is False
|
|
|
|
|
|
def test_regular_terminal_call_is_ignored() -> None:
|
|
messages = [
|
|
{
|
|
"role": "assistant",
|
|
"tool_calls": [
|
|
{
|
|
"id": "call-ls",
|
|
"function": {"name": "terminal", "arguments": '{"command":"ls -la"}'},
|
|
}
|
|
],
|
|
},
|
|
{"role": "tool", "tool_call_id": "call-ls", "content": "total 8"},
|
|
]
|
|
|
|
trace = extract_kb_tool_trace(messages)
|
|
|
|
assert trace["database_tool_call_count"] == 0
|
|
assert trace["database_tool_call_proven"] is False
|