"""Tests for secret-safe Hermes knowledge-base tool attribution.""" from __future__ import annotations import json import sys from pathlib import Path ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / "scripts")) from leo_tool_trace import extract_kb_tool_trace, tool_arguments_sha256 # noqa: E402 def test_extracts_completed_read_only_context_call_and_receipt() -> None: claim_id = "2a7ae257-d01d-46f4-b813-63f81bb9c7c7" source_id = "15740795-1234-4abc-8def-1234567890ab" semantic_sha = "8" * 64 artifact_sha = "4" * 64 messages = [ { "role": "assistant", "tool_calls": [ { "id": "call-secret-looking-id", "function": { "name": "terminal", "arguments": '{"command":"teleo-kb context \\"AI sandbagging\\""}', }, } ], }, { "role": "tool", "tool_call_id": "call-secret-looking-id", "content": ( "# Teleo KB Context\n" "- schema: `livingip.teleoKbRetrievalReceipt.v1`\n" f"- semantic context SHA-256: `{semantic_sha}`\n" f"- artifact state SHA-256: `{artifact_sha}`\n" "- DB read consistency: `stable_wal_marker`; attempts: `1`\n" f"claim_id: {claim_id}; source_id: {source_id}\n" ), }, ] trace = extract_kb_tool_trace(messages) assert trace["database_tool_call_proven"] is True assert trace["database_retrieval_receipt_proven"] is True assert trace["database_tool_calls_read_only"] is True assert trace["access_modes"] == ["read_only"] assert trace["calls"][0]["database_invocations"][0]["subcommand"] == "context" assert trace["calls"][0]["result"]["row_ids"] == sorted([claim_id, source_id]) assert trace["calls"][0]["result"]["retrieval_receipt"] == { "schema": "livingip.teleoKbRetrievalReceipt.v1", "semantic_context_sha256": semantic_sha, "artifact_state_sha256": artifact_sha, "read_consistency_status": "stable_wal_marker", } def test_never_retains_raw_command_arguments_or_result() -> None: secret = "password=do-not-retain" messages = [ { "role": "assistant", "tool_calls": [ { "id": "call-1", "function": { "name": "terminal", "arguments": {"command": f"teleo-kb search demo; echo {secret}"}, }, } ], }, {"role": "tool", "tool_call_id": "call-1", "content": f"result {secret}"}, ] trace = extract_kb_tool_trace(messages) rendered = str(trace) assert trace["database_tool_call_proven"] is True assert secret not in rendered assert "teleo-kb search demo" not in rendered assert trace["raw_commands_retained"] is False assert trace["raw_arguments_retained"] is False assert trace["raw_results_retained"] is False def test_staging_write_is_classified_and_not_called_read_only() -> None: messages = [ { "role": "assistant", "tool_calls": [ { "id": "call-write", "function": { "name": "terminal", "arguments": '{"command":"teleo-kb propose-edge --from a --to b"}', }, } ], }, {"role": "tool", "tool_call_id": "call-write", "content": "pending_review"}, ] trace = extract_kb_tool_trace(messages) assert trace["database_tool_call_proven"] is True assert trace["database_tool_calls_read_only"] is False assert trace["access_modes"] == ["staging_write"] def test_unmatched_or_failed_result_does_not_prove_database_call() -> None: unmatched = [ { "role": "assistant", "tool_calls": [ { "id": "call-missing", "function": {"name": "terminal", "arguments": '{"command":"teleo-kb show abc"}'}, } ], } ] failed = [ *unmatched, { "role": "tool", "tool_call_id": "call-missing", "content": "Traceback (most recent call last): connection failed", }, ] assert extract_kb_tool_trace(unmatched)["database_tool_call_proven"] is False assert extract_kb_tool_trace(failed)["database_tool_call_proven"] is False def test_structured_nonzero_exit_does_not_prove_database_call() -> None: messages = [ { "role": "assistant", "tool_calls": [ { "id": "call-nonzero", "function": { "name": "terminal", "arguments": '{"command":"teleo-kb context broad-question"}', }, } ], }, { "role": "tool", "tool_call_id": "call-nonzero", "content": '{"output":"usage error","exit_code":2,"error":null}', }, ] trace = extract_kb_tool_trace(messages) assert trace["database_tool_call_proven"] is False assert trace["database_tool_completed_count"] == 0 assert trace["calls"][0]["result"]["error_detected"] is True assert trace["calls"][0]["result"]["exit_code"] == 2 assert trace["calls"][0]["result"]["error_category"] == "structured_tool_error" def test_structured_read_tool_is_attributed_without_retaining_raw_arguments() -> None: arguments = { "action": "context", "query": "Northstar status snapshot", "limit": 1, "context_limit": 1, "format": "markdown", } output = ( "# Teleo KB Context\n" "- schema: `livingip.teleoKbRetrievalReceipt.v1`\n" f"- semantic context SHA-256: `{'a' * 64}`\n" f"- artifact state SHA-256: `{'b' * 64}`\n" "- DB read consistency: `stable_wal_marker`; attempts: `1`\n" ) messages = [ { "role": "assistant", "tool_calls": [ { "id": "structured-read", "function": { "name": "teleo-kb", "arguments": json.dumps(arguments), }, } ], }, { "role": "tool", "tool_call_id": "structured-read", "content": json.dumps({"output": output, "exit_code": 0, "error": None, "error_category": None}), }, ] trace = extract_kb_tool_trace(messages) assert trace["database_tool_call_count"] == 1 assert trace["database_tool_completed_count"] == 1 assert trace["database_retrieval_receipt_proven"] is True assert trace["database_tool_calls_read_only"] is True invocation = trace["calls"][0]["database_invocations"][0] assert invocation["executable"] == "teleo-kb" assert invocation["subcommand"] == "context" assert trace["calls"][0]["arguments_sha256"] == invocation["command_sha256"] assert trace["calls"][0]["result"]["content_bytes"] == len(output.encode()) assert "Northstar status snapshot" not in str(trace) def test_empty_success_envelope_does_not_prove_a_database_call() -> None: arguments = {"action": "status", "format": "json"} messages = [ { "role": "assistant", "tool_calls": [ { "id": "empty-read", "function": {"name": "teleo-kb", "arguments": json.dumps(arguments)}, } ], }, { "role": "tool", "tool_call_id": "empty-read", "content": json.dumps({"output": "", "exit_code": 0, "error": None, "error_category": None}), }, ] trace = extract_kb_tool_trace(messages) assert trace["database_tool_call_count"] == 1 assert trace["database_tool_completed_count"] == 0 assert trace["database_tool_call_proven"] is False assert trace["database_retrieval_receipt_proven"] is False assert trace["calls"][0]["result"]["nonempty"] is False def test_tool_argument_hash_is_shared_for_unicode_arguments() -> None: arguments = {"action": "context", "query": "caf\u00e9", "limit": 1} trace = extract_kb_tool_trace( [ { "role": "assistant", "tool_calls": [ { "id": "unicode-read", "function": {"name": "teleo-kb", "arguments": json.dumps(arguments)}, } ], }, {"role": "tool", "tool_call_id": "unicode-read", "content": "ok"}, ] ) expected = tool_arguments_sha256(arguments) assert trace["calls"][0]["arguments_sha256"] == expected assert trace["calls"][0]["database_invocations"][0]["command_sha256"] == expected def test_safe_structured_error_category_is_retained_without_raw_error() -> None: messages = [ { "role": "assistant", "tool_calls": [ { "id": "structured-error", "function": { "name": "teleo-kb", "arguments": '{"action":"context","query":"secret query"}', }, } ], }, { "role": "tool", "tool_call_id": "structured-error", "content": json.dumps( { "output": "", "exit_code": 126, "error": "secret query was malformed", "error_category": "invalid_read_arguments", } ), }, ] trace = extract_kb_tool_trace(messages) result = trace["calls"][0]["result"] assert result["error_detected"] is True assert result["exit_code"] == 126 assert result["error_category"] == "invalid_read_arguments" assert "secret query" not in str(trace) def test_actual_staging_subcommands_and_unknown_subcommands_fail_read_only_classification() -> None: messages = [] commands = ( "teleo-kb propose-attachment-evaluation packet.json", "teleo-kb record-document-evaluation packet.json", "teleo-kb future-command --flag", ) for index, command in enumerate(commands): call_id = f"call-{index}" messages.extend( [ { "role": "assistant", "tool_calls": [ { "id": call_id, "function": {"name": "terminal", "arguments": json.dumps({"command": command})}, } ], }, {"role": "tool", "tool_call_id": call_id, "content": "completed"}, ] ) trace = extract_kb_tool_trace(messages) assert trace["database_tool_call_count"] == 3 assert trace["database_tool_calls_read_only"] is False assert trace["access_modes"] == ["staging_write", "unknown_write_risk"] def test_direct_kb_tool_global_flags_do_not_hide_mutating_or_unknown_subcommands() -> None: messages = [] commands = ( "python3 /profile/bin/kb_tool.py --format json --local propose-edge from supports to --rationale test", "python3 /profile/bin/kb_tool.py --local --format=json --db teleo --container teleo-pg future-command", "python3 /profile/bin/kb_tool.py --ssh=teleo@example --local --format markdown context topic", ) for index, command in enumerate(commands): call_id = f"direct-{index}" messages.extend( [ { "role": "assistant", "tool_calls": [ { "id": call_id, "function": {"name": "terminal", "arguments": json.dumps({"command": command})}, } ], }, {"role": "tool", "tool_call_id": call_id, "content": "completed"}, ] ) trace = extract_kb_tool_trace(messages) assert trace["database_tool_call_count"] == 3 assert trace["database_tool_calls_read_only"] is False assert trace["access_modes"] == ["read_only", "staging_write", "unknown_write_risk"] assert [call["database_invocations"][0]["subcommand"] for call in trace["calls"]] == [ "propose-edge", "future-command", "context", ] def test_regular_terminal_call_is_ignored() -> None: messages = [ { "role": "assistant", "tool_calls": [ { "id": "call-ls", "function": {"name": "terminal", "arguments": '{"command":"ls -la"}'}, } ], }, {"role": "tool", "tool_call_id": "call-ls", "content": "total 8"}, ] trace = extract_kb_tool_trace(messages) assert trace["database_tool_call_count"] == 0 assert trace["database_tool_call_proven"] is False