Merge pull request #146 from living-ip/codex/leo-unified-turn-manifest-20260714
Some checks are pending
CI / lint-and-test (push) Waiting to run
Some checks are pending
CI / lint-and-test (push) Waiting to run
Harden Leo DB reasoning receipts and add deterministic replay
This commit is contained in:
commit
b3aa184dd4
21 changed files with 4910 additions and 53 deletions
|
|
@ -103,7 +103,7 @@ fi
|
||||||
|
|
||||||
# Syntax check all Python files before copying
|
# Syntax check all Python files before copying
|
||||||
ERRORS=0
|
ERRORS=0
|
||||||
for f in lib/*.py *.py diagnostics/*.py telegram/*.py tests/*.py hermes-agent/leoclean-bin/*.py hermes-agent/leoclean-plugins/vps/*/*.py hermes-agent/patches/*.py scripts/compile_kb_source_packet.py scripts/kb_proposal_normalize.py scripts/prepare_kb_source_manifest.py scripts/leo_behavior_manifest.py; do
|
for f in lib/*.py *.py diagnostics/*.py telegram/*.py tests/*.py hermes-agent/leoclean-bin/*.py hermes-agent/leoclean-plugins/vps/*/*.py hermes-agent/patches/*.py scripts/compile_kb_source_packet.py scripts/kb_proposal_normalize.py scripts/prepare_kb_source_manifest.py scripts/leo_behavior_manifest.py scripts/leo_turn_execution_manifest.py; do
|
||||||
[ -f "$f" ] || continue
|
[ -f "$f" ] || continue
|
||||||
if ! python3 -c "import ast, sys; ast.parse(open(sys.argv[1]).read())" "$f" 2>&1; then
|
if ! python3 -c "import ast, sys; ast.parse(open(sys.argv[1]).read())" "$f" 2>&1; then
|
||||||
log "SYNTAX ERROR: $f"
|
log "SYNTAX ERROR: $f"
|
||||||
|
|
|
||||||
|
|
@ -48,7 +48,7 @@ echo ""
|
||||||
# Syntax check all Python files before deploying
|
# Syntax check all Python files before deploying
|
||||||
echo "=== Pre-deploy syntax check ==="
|
echo "=== Pre-deploy syntax check ==="
|
||||||
ERRORS=0
|
ERRORS=0
|
||||||
for f in "$REPO_ROOT/lib/"*.py "$REPO_ROOT/"*.py "$REPO_ROOT/diagnostics/"*.py "$REPO_ROOT/telegram/"*.py "$REPO_ROOT/hermes-agent/leoclean-bin/"*.py "$REPO_ROOT/hermes-agent/leoclean-plugins/vps/"*/*.py "$REPO_ROOT/hermes-agent/patches/"*.py "$REPO_ROOT/scripts/compile_kb_source_packet.py" "$REPO_ROOT/scripts/kb_proposal_normalize.py" "$REPO_ROOT/scripts/prepare_kb_source_manifest.py" "$REPO_ROOT/scripts/leo_behavior_manifest.py"; do
|
for f in "$REPO_ROOT/lib/"*.py "$REPO_ROOT/"*.py "$REPO_ROOT/diagnostics/"*.py "$REPO_ROOT/telegram/"*.py "$REPO_ROOT/hermes-agent/leoclean-bin/"*.py "$REPO_ROOT/hermes-agent/leoclean-plugins/vps/"*/*.py "$REPO_ROOT/hermes-agent/patches/"*.py "$REPO_ROOT/scripts/compile_kb_source_packet.py" "$REPO_ROOT/scripts/kb_proposal_normalize.py" "$REPO_ROOT/scripts/prepare_kb_source_manifest.py" "$REPO_ROOT/scripts/leo_behavior_manifest.py" "$REPO_ROOT/scripts/leo_turn_execution_manifest.py"; do
|
||||||
[ -f "$f" ] || continue
|
[ -f "$f" ] || continue
|
||||||
if ! python3 -c "import ast, sys; ast.parse(open(sys.argv[1]).read())" "$f" 2>/dev/null; then
|
if ! python3 -c "import ast, sys; ast.parse(open(sys.argv[1]).read())" "$f" 2>/dev/null; then
|
||||||
echo "SYNTAX ERROR: $f"
|
echo "SYNTAX ERROR: $f"
|
||||||
|
|
|
||||||
|
|
@ -141,9 +141,89 @@ Normal applies mark the SQL hash as `exact_executed_sql`. A later
|
||||||
pretends the current engine hash is historical proof of the originally executed
|
pretends the current engine hash is historical proof of the originally executed
|
||||||
program.
|
program.
|
||||||
|
|
||||||
This closes replay-receipt loss for new strict applies. It does not reconstruct
|
This closes replay-receipt loss for new strict applies. The receipt alone does
|
||||||
legacy freeform applies, and it does not yet execute the complete
|
not retain every column of the proposal ledger, so exact reconstruction also
|
||||||
genesis-plus-ledger blank-database replay.
|
needs the full approved proposal row, immutable approval snapshot, and final
|
||||||
|
applied proposal row.
|
||||||
|
|
||||||
|
## Genesis Plus Strict Ledger: Working Insert-Only Slice
|
||||||
|
|
||||||
|
`ops/run_local_genesis_ledger_rebuild.py` now executes the first exact
|
||||||
|
genesis-plus-ledger slice in one command:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
.venv/bin/python ops/run_local_genesis_ledger_rebuild.py \
|
||||||
|
--genesis-dump /private/path/genesis.dump \
|
||||||
|
--genesis-manifest /private/path/genesis-manifest.jsonl \
|
||||||
|
--ledger /private/path/ledger.json \
|
||||||
|
--ledger-sha256 "$LEDGER_SHA256" \
|
||||||
|
--output /tmp/genesis-ledger-rebuild-receipt.json
|
||||||
|
```
|
||||||
|
|
||||||
|
The v1 ledger pins the genesis dump and manifest, final parity manifest,
|
||||||
|
reconstruction/restore/guard/apply/replay/parity engines, and every ordered
|
||||||
|
private material file. Each material file contains one existing `kb_apply_replay_receipt`, the
|
||||||
|
exact full proposal row immediately before apply, its immutable
|
||||||
|
`kb_proposal_approvals` row, and the exact full proposal row after apply. These
|
||||||
|
files can contain claim text or source excerpts and must remain private.
|
||||||
|
|
||||||
|
The ledger shape is:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"artifact": "teleo_genesis_plus_ledger",
|
||||||
|
"contract_version": 1,
|
||||||
|
"engine": {
|
||||||
|
"reconstruction_command_sha256": "<sha256>",
|
||||||
|
"base_rebuild_engine_sha256": "<sha256>",
|
||||||
|
"apply_engine_sha256": "<sha256>",
|
||||||
|
"replay_receipt_engine_sha256": "<sha256>",
|
||||||
|
"guard_prerequisites_sha256": "<sha256>",
|
||||||
|
"parity_sql_sha256": "<sha256>"
|
||||||
|
},
|
||||||
|
"genesis": {
|
||||||
|
"dump_sha256": "<sha256>",
|
||||||
|
"parity_manifest_sha256": "<sha256>"
|
||||||
|
},
|
||||||
|
"entries": [{
|
||||||
|
"sequence": 1,
|
||||||
|
"material": "private/0001.json",
|
||||||
|
"sha256": "<material-file-sha256>",
|
||||||
|
"replay_material_sha256": "<receipt-replay-material-sha256>"
|
||||||
|
}],
|
||||||
|
"final_parity": {
|
||||||
|
"manifest": "final-manifest.jsonl",
|
||||||
|
"sha256": "<sha256>"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Each referenced material object has exact top-level keys
|
||||||
|
`artifact`, `contract_version`, `sequence`, `approved_proposal`,
|
||||||
|
`approval_snapshot`, `applied_proposal`, and `replay_receipt`. Proposal objects
|
||||||
|
must contain every current `kb_stage.kb_proposals` column; partial envelopes
|
||||||
|
are rejected.
|
||||||
|
|
||||||
|
The command verifies every hash before starting Docker, restores a tmpfs-backed
|
||||||
|
Postgres with `--network none`, reapplies the current guarded prerequisites,
|
||||||
|
and proves genesis parity. For each entry it seeds the receipt's exact row IDs
|
||||||
|
and timestamps in the disposable clone, executes the existing `kb_apply`
|
||||||
|
payload-bound guard and ledger transition, checks exact proposal and canonical
|
||||||
|
row readbacks, then verifies the complete final parity manifest. Its public
|
||||||
|
mode-`0600` receipt contains hashes, IDs, types, counts, parity summaries, and
|
||||||
|
cleanup proof, but no payloads, rows, SQL, source paths, or command errors.
|
||||||
|
|
||||||
|
The exact v1 claim ceiling is intentionally narrow:
|
||||||
|
|
||||||
|
- `add_edge`, `attach_evidence`, and `approve_claim` strict receipts execute;
|
||||||
|
- sequence gaps, hash drift, engine drift, duplicate proposals, and legacy or
|
||||||
|
freeform payloads fail before container start;
|
||||||
|
- `revise_strategy` fails closed because its current receipt omits the prior
|
||||||
|
strategies and nodes that the apply engine deactivates or retires;
|
||||||
|
- full proposal before/after rows are mandatory because the current receipt
|
||||||
|
envelope does not retain proposal origin fields or exact `updated_at`;
|
||||||
|
- this proves only an isolated local reconstruction. It does not touch or prove
|
||||||
|
VPS, GCP, Telegram, a live database, or blank-schema source recompilation.
|
||||||
|
|
||||||
The source compiler now turns one raw artifact, its strict UTF-8 extraction,
|
The source compiler now turns one raw artifact, its strict UTF-8 extraction,
|
||||||
and a reviewed extraction manifest into a deterministic, hash-bound
|
and a reviewed extraction manifest into a deterministic, hash-bound
|
||||||
|
|
@ -190,10 +270,11 @@ neither command applies canonical rows.
|
||||||
|
|
||||||
The existing full-data clone canary separately proves that a reviewed packet
|
The existing full-data clone canary separately proves that a reviewed packet
|
||||||
can create source, claim, evidence, and edge rows and affect later reasoning.
|
can create source, claim, evidence, and edge rows and affect later reasoning.
|
||||||
The remaining recompilation capability is to join genesis, accepted packets,
|
The remaining reconstruction work is to retain complete deltas for mutating
|
||||||
and their row-level receipts into a complete corpus runner. One-document
|
contracts such as `revise_strategy`, backfill or explicitly reject legacy
|
||||||
preparation, staging, and receipt capture do not yet prove a clean database can
|
freeform applies, and extend beyond genesis recovery to a blank-schema source
|
||||||
be rebuilt semantically from every source.
|
compiler. The insert-only ledger runner does not prove that every historical
|
||||||
|
canonical row can be rebuilt semantically from retained sources.
|
||||||
|
|
||||||
## Definition Of Working
|
## Definition Of Working
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,58 @@
|
||||||
|
{
|
||||||
|
"artifact": "leo_unified_turn_manifest_canary",
|
||||||
|
"attribution": {
|
||||||
|
"missing_required_bindings": [],
|
||||||
|
"status": "complete"
|
||||||
|
},
|
||||||
|
"claim_ceiling": {
|
||||||
|
"bitwise_identical_model_output_guaranteed": false,
|
||||||
|
"database_snapshot_reconstructible_from_this_receipt_alone": false,
|
||||||
|
"proven": "One no-send live VPS GatewayRunner turn is attributable to a clean harness commit, executed behavior manifest, actual model calls, session boundary, exact database retrieval receipts, every database tool-result hash, and an unchanged full canonical database content fingerprint.",
|
||||||
|
"provider_response_id_available": false,
|
||||||
|
"provider_response_id_reason": "Hermes post_api_request does not expose it."
|
||||||
|
},
|
||||||
|
"database": {
|
||||||
|
"fingerprint_sha256": "32a1f2c18f4b4c623443cbd4d9520982e9f291ec74ca63855cef51c88ac56f24",
|
||||||
|
"fingerprint_unchanged": true,
|
||||||
|
"read_consistency": "stable_wal_marker",
|
||||||
|
"system_identifier": "7649789040005668902",
|
||||||
|
"table_count": 39,
|
||||||
|
"tool_subcommands": [
|
||||||
|
"search",
|
||||||
|
"evidence",
|
||||||
|
"edges"
|
||||||
|
],
|
||||||
|
"total_rows": 52167,
|
||||||
|
"wal_lsn": "0/49BC4478"
|
||||||
|
},
|
||||||
|
"execution_sha256": "8fb8939e1f6dd935f9866eb7f756ab7c2640c3bf5f8477e17143c8dfcd42ff47",
|
||||||
|
"generated_at_utc": "2026-07-14T21:57:04.200366+00:00",
|
||||||
|
"harness_git_head": "cc46d66713f18ce880a921c7bc46546e5f7e7bf2",
|
||||||
|
"model_execution": {
|
||||||
|
"api_call_count": 5,
|
||||||
|
"api_mode": "chat_completions",
|
||||||
|
"configured_model": "anthropic/claude-sonnet-4-6",
|
||||||
|
"provider": "openrouter",
|
||||||
|
"response_model": "anthropic/claude-sonnet-4.6"
|
||||||
|
},
|
||||||
|
"outcome": {
|
||||||
|
"candidate_applied": false,
|
||||||
|
"evidence_and_inference_separated": true,
|
||||||
|
"evidence_and_edges_inspected": true,
|
||||||
|
"narrower_candidate_drafted": true,
|
||||||
|
"row_id_supplied_by_operator": false
|
||||||
|
},
|
||||||
|
"prompt_id": "OOS-MANIFEST-02",
|
||||||
|
"prompt_sha256": "8cb79d650d01865af5e1a3854f14f299c074be510f26f58230e43a8bfb379098",
|
||||||
|
"remote_run_id": "e4011ea6bb46",
|
||||||
|
"reply_sha256": "bd73b7ce153f789b15f6552de804c2d38de1aa23a9bdd60299a1802478c3217d",
|
||||||
|
"safety": {
|
||||||
|
"database_counts_changed": false,
|
||||||
|
"database_mutation_by_harness": false,
|
||||||
|
"live_behavior_manifest_unchanged": true,
|
||||||
|
"posted_to_telegram": false,
|
||||||
|
"temporary_profile_removed": true
|
||||||
|
},
|
||||||
|
"schema": "livingip.leoUnifiedTurnManifestCanaryReceipt.v1",
|
||||||
|
"status": "pass"
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,89 @@
|
||||||
|
{
|
||||||
|
"answer_excerpts": [
|
||||||
|
"The overconfident claim for a founder decision: \"designing coordination rules is categorically different from designing coordination outcomes as nine intellectual traditions independently confirm\" \u2014 confidence 0.9, id fd159490.\n\n- What the DB actually shows: two evidence rows. The \"grounds\" row is a cross-domain synthesis with artifact_verification: no_source_pointer \u2014 the source has no resolvable locator. The \"illustrates\" row is a verified internal MD file authored by the collective. No extern",
|
||||||
|
"Assumptions versus observed support in fd159490:\n\n- \"Independently confirm\" is assumed. Both evidence rows originate from one internal synthesis \u2014 a no_source_pointer grounds note and a self-authored MD. One agent reading nine traditions is not nine traditions independently arriving at the same conclusion.\n- \"Categorically different\" is assumed. Madison and Hayek illustrate that the distinction exists; neither tests whether outcome-design coordination fails at scale, which is what \"categorical\" ",
|
||||||
|
"Revised Proposal A \u2014 mechanism-level, not outcome-level:\n\n\"Process-rule design reduces coordination failures under uncertainty by preserving agent-level adaptation that outcome-target design forecloses, independent of whether the system persists longer.\"\n\nConfidence ceiling: 0.65. This tests the adaptive-response mechanism, not durability as a proxy. Durability is an outcome; whether agents can recombine within rules under novel conditions is the mechanism the original claim actually rests on.\n\n"
|
||||||
|
],
|
||||||
|
"checks": {
|
||||||
|
"all_turn_execution_manifests_complete": true,
|
||||||
|
"assumption_and_observed_support_separated": true,
|
||||||
|
"claim_selected_from_database_without_operator_id": true,
|
||||||
|
"exact_unseen_prompts_without_supplied_row_id": true,
|
||||||
|
"fail_closed_safety_gate_passed": true,
|
||||||
|
"missing_source_named": true,
|
||||||
|
"multiple_review_only_replacements_proposed": true,
|
||||||
|
"no_write_or_hidden_learning_claim": true,
|
||||||
|
"review_approval_apply_boundary_explicit": true,
|
||||||
|
"same_claim_body_and_evidence_reinspected_after_challenge": true,
|
||||||
|
"third_turn_used_conversation_instead_of_new_db_lookup": true,
|
||||||
|
"user_objection_changed_the_candidate": true,
|
||||||
|
"weak_support_identified": true
|
||||||
|
},
|
||||||
|
"claim_ceiling": {
|
||||||
|
"not_proven": [
|
||||||
|
"Telegram-visible message delivery",
|
||||||
|
"staging or canonical proposal mutation",
|
||||||
|
"agent-to-agent review",
|
||||||
|
"source ingestion or guarded apply",
|
||||||
|
"GCP runtime parity for this exact behavior commit and database state"
|
||||||
|
],
|
||||||
|
"proven": "One unseen three-turn live VPS handler session independently selected a canonical claim, inspected its body and evidence, survived two user challenges, revised candidate knowledge, named missing evidence, and preserved review-before-apply without changing the database."
|
||||||
|
},
|
||||||
|
"database": {
|
||||||
|
"fingerprint_sha256": "32a1f2c18f4b4c623443cbd4d9520982e9f291ec74ca63855cef51c88ac56f24",
|
||||||
|
"fingerprint_unchanged": true,
|
||||||
|
"table_count": 39,
|
||||||
|
"total_rows": 52167
|
||||||
|
},
|
||||||
|
"execution_sha256_chain": [
|
||||||
|
"67b3cb318a8b4521a54144e388bc1ace6abf35a7304bdca409c7bbff17990654",
|
||||||
|
"d039966e2b13414e4d2f3ccb1e503bc24652339a81ec54d4529cdad0118b2482",
|
||||||
|
"8f47aa9fd5aa9ae038a0bb4cec28868da982891d7238f7ccbb2142ac0d209437"
|
||||||
|
],
|
||||||
|
"harness_git_head": "0cc05aa2f4b296991d4260db6948eafaf0e00aa2",
|
||||||
|
"outcomes": {
|
||||||
|
"answers_from_current_database_without_row_ids": true,
|
||||||
|
"conversation_chain_and_runtime_are_attributable": true,
|
||||||
|
"iterates_with_the_user_and_revises_reasoning": true,
|
||||||
|
"names_the_evidence_needed_to_resolve_disagreement": true,
|
||||||
|
"preserves_human_review_and_guarded_apply": true,
|
||||||
|
"proposes_better_claims_without_making_them_live": true,
|
||||||
|
"survives_a_claim_body_and_evidence_challenge": true
|
||||||
|
},
|
||||||
|
"prompt_ids": [
|
||||||
|
"OOS-CHAIN-01",
|
||||||
|
"OOS-CHAIN-02",
|
||||||
|
"OOS-CHAIN-03"
|
||||||
|
],
|
||||||
|
"proof_tier": "live_vps_gatewayrunner_temp_profile_no_telegram_post_no_apply",
|
||||||
|
"remote_run_id": "dd647372817d",
|
||||||
|
"safety_gate": {
|
||||||
|
"checks": {
|
||||||
|
"all_tool_traces_read_only_and_bound": true,
|
||||||
|
"all_turn_manifests_complete": true,
|
||||||
|
"all_turns_declared_no_mutation": true,
|
||||||
|
"all_turns_returned_replies": true,
|
||||||
|
"database_counts_unchanged": true,
|
||||||
|
"database_fingerprint_unchanged": true,
|
||||||
|
"handler_authorized": true,
|
||||||
|
"harness_declared_no_kb_mutation": true,
|
||||||
|
"live_behavior_unchanged": true,
|
||||||
|
"model_trace_metadata_bound": true,
|
||||||
|
"no_runtime_error": true,
|
||||||
|
"no_telegram_post": true,
|
||||||
|
"remote_command_succeeded": true,
|
||||||
|
"results_nonempty": true,
|
||||||
|
"runtime_completed": true,
|
||||||
|
"service_unchanged": true,
|
||||||
|
"temporary_profile_removed": true
|
||||||
|
},
|
||||||
|
"failed_checks": [],
|
||||||
|
"status": "pass"
|
||||||
|
},
|
||||||
|
"schema": "livingip.leoUnseenReasoningChainReceipt.v1",
|
||||||
|
"selected_claim_ids": [
|
||||||
|
"fd159490-280d-4ede-84ef-faa169cff766"
|
||||||
|
],
|
||||||
|
"source_report_sha256": "163b9c369e3727b8cbe0bb6e210eefb9e4724fbfa720378652aebce529f879c3",
|
||||||
|
"status": "pass"
|
||||||
|
}
|
||||||
|
|
@ -581,10 +581,25 @@ def operational_contracts(
|
||||||
term in lowered
|
term in lowered
|
||||||
for term in ("proposal", "proposed", "pending", "approved", "applied", "staged", "reviewer approval")
|
for term in ("proposal", "proposed", "pending", "approved", "applied", "staged", "reviewer approval")
|
||||||
)
|
)
|
||||||
|
claim_reasoning_question = bool(
|
||||||
|
any(term in lowered for term in ("claim", "belief", "exact body"))
|
||||||
|
and any(
|
||||||
|
term in lowered
|
||||||
|
for term in (
|
||||||
|
"assumption",
|
||||||
|
"evidence",
|
||||||
|
"falsif",
|
||||||
|
"observed support",
|
||||||
|
"replacement",
|
||||||
|
"narrower",
|
||||||
|
"what supports",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
)
|
||||||
broad_kb_change_question = _has_unnegated_action(
|
broad_kb_change_question = _has_unnegated_action(
|
||||||
query, ("change", "changed", "update", "updated", "landed")
|
query, ("change", "changed", "update", "updated", "landed")
|
||||||
)
|
)
|
||||||
proposal_state_question = kb_scope and not forecast_resolution_question and (
|
proposal_state_question = kb_scope and not forecast_resolution_question and not claim_reasoning_question and (
|
||||||
proposal_lifecycle_question
|
proposal_lifecycle_question
|
||||||
or ("demo" not in lowered and broad_kb_change_question and not kb_implementation_question)
|
or ("demo" not in lowered and broad_kb_change_question and not kb_implementation_question)
|
||||||
)
|
)
|
||||||
|
|
|
||||||
|
|
@ -71,6 +71,41 @@ def _trace(record: dict[str, Any]) -> None:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def _retrieval_receipt_trace(value: Any) -> dict[str, Any] | None:
|
||||||
|
"""Retain the exact read receipt without retaining claim or source bodies."""
|
||||||
|
|
||||||
|
if not isinstance(value, dict) or value.get("schema") != "livingip.teleoKbRetrievalReceipt.v1":
|
||||||
|
return None
|
||||||
|
consistency = value.get("read_consistency") if isinstance(value.get("read_consistency"), dict) else {}
|
||||||
|
counts = value.get("counts") if isinstance(value.get("counts"), dict) else {}
|
||||||
|
safe = {
|
||||||
|
"schema": value.get("schema"),
|
||||||
|
"query_sha256": value.get("query_sha256"),
|
||||||
|
"semantic_context_sha256": value.get("semantic_context_sha256"),
|
||||||
|
"artifact_state_sha256": value.get("artifact_state_sha256"),
|
||||||
|
"claim_ids": sorted(str(item) for item in value.get("claim_ids") or []),
|
||||||
|
"source_ids": sorted(str(item) for item in value.get("source_ids") or []),
|
||||||
|
"counts": {
|
||||||
|
key: item
|
||||||
|
for key, item in sorted(counts.items())
|
||||||
|
if isinstance(key, str) and isinstance(item, int) and not isinstance(item, bool)
|
||||||
|
},
|
||||||
|
"read_consistency": {
|
||||||
|
"status": consistency.get("status"),
|
||||||
|
"attempts": consistency.get("attempts"),
|
||||||
|
"database": consistency.get("database"),
|
||||||
|
"database_user": consistency.get("database_user"),
|
||||||
|
"system_identifier": consistency.get("system_identifier"),
|
||||||
|
"wal_lsn_before": consistency.get("wal_lsn_before"),
|
||||||
|
"wal_lsn_after": consistency.get("wal_lsn_after"),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
safe["receipt_sha256"] = hashlib.sha256(
|
||||||
|
json.dumps(value, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
||||||
|
).hexdigest()
|
||||||
|
return safe
|
||||||
|
|
||||||
|
|
||||||
def _failure_context(reason: str) -> str:
|
def _failure_context(reason: str) -> str:
|
||||||
return (
|
return (
|
||||||
'<leo_current_runtime_contracts source="teleo-kb" status="unavailable">\n'
|
'<leo_current_runtime_contracts source="teleo-kb" status="unavailable">\n'
|
||||||
|
|
@ -246,6 +281,7 @@ def _load_database_snapshot(
|
||||||
compiled_response = payload.get("compiled_response")
|
compiled_response = payload.get("compiled_response")
|
||||||
if compiled_response is not None and not isinstance(compiled_response, str):
|
if compiled_response is not None and not isinstance(compiled_response, str):
|
||||||
raise ValueError("compiled_response_invalid")
|
raise ValueError("compiled_response_invalid")
|
||||||
|
retrieval_receipt = _retrieval_receipt_trace(payload.get("retrieval_receipt"))
|
||||||
except (json.JSONDecodeError, TypeError, ValueError) as exc:
|
except (json.JSONDecodeError, TypeError, ValueError) as exc:
|
||||||
record = base_record | {"status": "error", "error": str(exc), "injected": True}
|
record = base_record | {"status": "error", "error": str(exc), "injected": True}
|
||||||
_trace(record)
|
_trace(record)
|
||||||
|
|
@ -259,6 +295,8 @@ def _load_database_snapshot(
|
||||||
"contract_sha256": hashlib.sha256(contract_json.encode("utf-8")).hexdigest(),
|
"contract_sha256": hashlib.sha256(contract_json.encode("utf-8")).hexdigest(),
|
||||||
"compiled_response_available": bool(compiled_response),
|
"compiled_response_available": bool(compiled_response),
|
||||||
}
|
}
|
||||||
|
if retrieval_receipt is not None:
|
||||||
|
record["retrieval_receipt"] = retrieval_receipt
|
||||||
_trace(record)
|
_trace(record)
|
||||||
return {
|
return {
|
||||||
"status": "ok",
|
"status": "ok",
|
||||||
|
|
|
||||||
1284
ops/run_local_genesis_ledger_rebuild.py
Normal file
1284
ops/run_local_genesis_ledger_rebuild.py
Normal file
File diff suppressed because it is too large
Load diff
|
|
@ -23,6 +23,7 @@ import yaml
|
||||||
SCHEMA = "livingip.leoBehaviorManifest.v1"
|
SCHEMA = "livingip.leoBehaviorManifest.v1"
|
||||||
DEFAULT_PROFILE = Path("/home/teleo/.hermes/profiles/leoclean")
|
DEFAULT_PROFILE = Path("/home/teleo/.hermes/profiles/leoclean")
|
||||||
DEFAULT_HERMES_ROOT = Path("/home/teleo/.hermes/hermes-agent")
|
DEFAULT_HERMES_ROOT = Path("/home/teleo/.hermes/hermes-agent")
|
||||||
|
DEFAULT_DEPLOYMENT_ROOT = Path("/opt/teleo-eval/workspaces/deploy-infra")
|
||||||
|
|
||||||
|
|
||||||
def sha256_bytes(value: bytes) -> str:
|
def sha256_bytes(value: bytes) -> str:
|
||||||
|
|
@ -174,6 +175,76 @@ def git_head(repo: Path) -> str | None:
|
||||||
return value if completed.returncode == 0 and len(value) == 40 else None
|
return value if completed.returncode == 0 and len(value) == 40 else None
|
||||||
|
|
||||||
|
|
||||||
|
def git_state(repo: Path) -> dict[str, Any]:
|
||||||
|
head = git_head(repo)
|
||||||
|
try:
|
||||||
|
completed = subprocess.run(
|
||||||
|
["git", "-C", str(repo), "status", "--porcelain", "--untracked-files=all"],
|
||||||
|
check=False,
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
timeout=10,
|
||||||
|
env={**os.environ, "GIT_OPTIONAL_LOCKS": "0"},
|
||||||
|
)
|
||||||
|
except (OSError, subprocess.TimeoutExpired):
|
||||||
|
return {"git_head": head, "worktree_clean": None, "status_sha256": None}
|
||||||
|
status = completed.stdout if completed.returncode == 0 else ""
|
||||||
|
return {
|
||||||
|
"git_head": head,
|
||||||
|
"worktree_clean": completed.returncode == 0 and not status.strip(),
|
||||||
|
"status_sha256": sha256_bytes(status.encode("utf-8")) if completed.returncode == 0 else None,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def source_code_manifest(profile: Path, paths: tuple[Path, ...]) -> dict[str, Any]:
|
||||||
|
"""Hash executable source while excluding generated caches and environments."""
|
||||||
|
|
||||||
|
allowed_suffixes = {
|
||||||
|
".cfg",
|
||||||
|
".ini",
|
||||||
|
".j2",
|
||||||
|
".jinja",
|
||||||
|
".json",
|
||||||
|
".py",
|
||||||
|
".sh",
|
||||||
|
".sql",
|
||||||
|
".tmpl",
|
||||||
|
".toml",
|
||||||
|
".yaml",
|
||||||
|
".yml",
|
||||||
|
}
|
||||||
|
excluded_parts = {".git", ".pytest_cache", "__pycache__", "node_modules", "venv", ".venv"}
|
||||||
|
files: list[dict[str, Any]] = []
|
||||||
|
missing: list[str] = []
|
||||||
|
for requested in paths:
|
||||||
|
if not requested.exists() and not requested.is_symlink():
|
||||||
|
missing.append(_safe_relative(requested, profile))
|
||||||
|
continue
|
||||||
|
candidates = [requested] if requested.is_file() else sorted(requested.rglob("*"))
|
||||||
|
for path in candidates:
|
||||||
|
if not path.is_file() or excluded_parts & set(path.parts):
|
||||||
|
continue
|
||||||
|
stat = path.stat()
|
||||||
|
if path.suffix not in allowed_suffixes and not bool(stat.st_mode & 0o111):
|
||||||
|
continue
|
||||||
|
files.append(
|
||||||
|
{
|
||||||
|
"path": _safe_relative(path, profile),
|
||||||
|
"bytes": stat.st_size,
|
||||||
|
"mode": oct(stat.st_mode & 0o777),
|
||||||
|
"sha256": file_sha256(path),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
files.sort(key=lambda item: item["path"])
|
||||||
|
stable = {"files": files, "missing": sorted(missing)}
|
||||||
|
return {
|
||||||
|
**stable,
|
||||||
|
"file_count": len(files),
|
||||||
|
"total_bytes": sum(int(item["bytes"]) for item in files),
|
||||||
|
"sha256": canonical_sha256(stable),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
def component(
|
def component(
|
||||||
profile: Path,
|
profile: Path,
|
||||||
*,
|
*,
|
||||||
|
|
@ -190,9 +261,14 @@ def component(
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
def build_manifest(profile: Path = DEFAULT_PROFILE, hermes_root: Path = DEFAULT_HERMES_ROOT) -> dict[str, Any]:
|
def build_manifest(
|
||||||
|
profile: Path = DEFAULT_PROFILE,
|
||||||
|
hermes_root: Path = DEFAULT_HERMES_ROOT,
|
||||||
|
deployment_root: Path = DEFAULT_DEPLOYMENT_ROOT,
|
||||||
|
) -> dict[str, Any]:
|
||||||
profile = profile.resolve()
|
profile = profile.resolve()
|
||||||
hermes_root = hermes_root.resolve()
|
hermes_root = hermes_root.resolve()
|
||||||
|
deployment_root = deployment_root.resolve()
|
||||||
config = safe_model_config(profile / "config.yaml")
|
config = safe_model_config(profile / "config.yaml")
|
||||||
components = {
|
components = {
|
||||||
"profile_config": component(
|
"profile_config": component(
|
||||||
|
|
@ -267,9 +343,22 @@ def build_manifest(profile: Path = DEFAULT_PROFILE, hermes_root: Path = DEFAULT_
|
||||||
},
|
},
|
||||||
"hermes_runtime": {
|
"hermes_runtime": {
|
||||||
"git_head": git_head(hermes_root),
|
"git_head": git_head(hermes_root),
|
||||||
"source_tree": path_manifest(
|
"git_state": git_state(hermes_root),
|
||||||
profile, (hermes_root / "run_agent.py", hermes_root / "agent" / "prompt_builder.py")
|
"source_tree": source_code_manifest(
|
||||||
|
profile,
|
||||||
|
(
|
||||||
|
hermes_root / "run_agent.py",
|
||||||
|
hermes_root / "agent",
|
||||||
|
hermes_root / "gateway",
|
||||||
|
hermes_root / "hermes_cli",
|
||||||
|
hermes_root / "tools",
|
||||||
),
|
),
|
||||||
|
),
|
||||||
|
},
|
||||||
|
"teleo_infrastructure_runtime": {
|
||||||
|
"git_head": git_head(deployment_root),
|
||||||
|
"git_state": git_state(deployment_root),
|
||||||
|
"source_tree": source_code_manifest(profile, (deployment_root,)),
|
||||||
},
|
},
|
||||||
"components": components,
|
"components": components,
|
||||||
"canonical_database": {
|
"canonical_database": {
|
||||||
|
|
@ -282,6 +371,7 @@ def build_manifest(profile: Path = DEFAULT_PROFILE, hermes_root: Path = DEFAULT_
|
||||||
"generated_at_utc": datetime.now(timezone.utc).isoformat(),
|
"generated_at_utc": datetime.now(timezone.utc).isoformat(),
|
||||||
"profile_root": str(profile),
|
"profile_root": str(profile),
|
||||||
"hermes_root": str(hermes_root),
|
"hermes_root": str(hermes_root),
|
||||||
|
"deployment_root": str(deployment_root),
|
||||||
"behavior_sha256": canonical_sha256(stable),
|
"behavior_sha256": canonical_sha256(stable),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -290,9 +380,10 @@ def main() -> int:
|
||||||
parser = argparse.ArgumentParser(description=__doc__)
|
parser = argparse.ArgumentParser(description=__doc__)
|
||||||
parser.add_argument("--profile", type=Path, default=DEFAULT_PROFILE)
|
parser.add_argument("--profile", type=Path, default=DEFAULT_PROFILE)
|
||||||
parser.add_argument("--hermes-root", type=Path, default=DEFAULT_HERMES_ROOT)
|
parser.add_argument("--hermes-root", type=Path, default=DEFAULT_HERMES_ROOT)
|
||||||
|
parser.add_argument("--deployment-root", type=Path, default=DEFAULT_DEPLOYMENT_ROOT)
|
||||||
parser.add_argument("--output", type=Path)
|
parser.add_argument("--output", type=Path)
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
manifest = build_manifest(args.profile, args.hermes_root)
|
manifest = build_manifest(args.profile, args.hermes_root, args.deployment_root)
|
||||||
encoded = json.dumps(manifest, indent=2, sort_keys=True) + "\n"
|
encoded = json.dumps(manifest, indent=2, sort_keys=True) + "\n"
|
||||||
if args.output:
|
if args.output:
|
||||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
|
||||||
887
scripts/leo_turn_execution_manifest.py
Normal file
887
scripts/leo_turn_execution_manifest.py
Normal file
|
|
@ -0,0 +1,887 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Bind one tested Leo turn to its runtime, model call, session, and KB read.
|
||||||
|
|
||||||
|
The manifest is deliberately content-minimizing: prompts, replies, raw database
|
||||||
|
rows, tool arguments, provider URLs, and session identifiers are represented by
|
||||||
|
hashes. It proves attribution of a tested turn, not deterministic regeneration
|
||||||
|
of externally hosted model weights.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import re
|
||||||
|
import subprocess
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
SCHEMA = "livingip.leoTurnExecutionManifest.v1"
|
||||||
|
RETRIEVAL_RECEIPT_SCHEMA = "livingip.teleoKbRetrievalReceipt.v1"
|
||||||
|
DATABASE_PROMPT_TERMS = (
|
||||||
|
"knowledge base",
|
||||||
|
"database",
|
||||||
|
"postgres",
|
||||||
|
"claim",
|
||||||
|
"belief",
|
||||||
|
"evidence",
|
||||||
|
"source",
|
||||||
|
"proposal",
|
||||||
|
"canonical",
|
||||||
|
"approve",
|
||||||
|
"apply",
|
||||||
|
"document",
|
||||||
|
"identity",
|
||||||
|
"soul",
|
||||||
|
"strategy",
|
||||||
|
"row",
|
||||||
|
)
|
||||||
|
HEX_64 = re.compile(r"^[0-9a-f]{64}$")
|
||||||
|
HEX_40 = re.compile(r"^[0-9a-f]{40}$")
|
||||||
|
|
||||||
|
|
||||||
|
def canonical_sha256(value: Any) -> str:
|
||||||
|
encoded = json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=True).encode("utf-8")
|
||||||
|
return hashlib.sha256(encoded).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def text_sha256(value: str) -> str:
|
||||||
|
return hashlib.sha256(value.encode("utf-8")).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def git_state(repo: Path) -> dict[str, Any]:
|
||||||
|
try:
|
||||||
|
head = subprocess.run(
|
||||||
|
["git", "-C", str(repo), "rev-parse", "HEAD"],
|
||||||
|
check=False,
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
timeout=10,
|
||||||
|
)
|
||||||
|
status = subprocess.run(
|
||||||
|
["git", "-C", str(repo), "status", "--porcelain", "--untracked-files=all"],
|
||||||
|
check=False,
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
timeout=10,
|
||||||
|
)
|
||||||
|
except (OSError, subprocess.TimeoutExpired):
|
||||||
|
return {"git_head": None, "worktree_clean": None, "status_sha256": None}
|
||||||
|
git_head = head.stdout.strip() if head.returncode == 0 and len(head.stdout.strip()) == 40 else None
|
||||||
|
status_text = status.stdout if status.returncode == 0 else ""
|
||||||
|
return {
|
||||||
|
"git_head": git_head,
|
||||||
|
"worktree_clean": status.returncode == 0 and not status_text.strip(),
|
||||||
|
"status_sha256": text_sha256(status_text) if status.returncode == 0 else None,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _is_sha256(value: Any) -> bool:
|
||||||
|
return isinstance(value, str) and bool(HEX_64.fullmatch(value))
|
||||||
|
|
||||||
|
|
||||||
|
def _is_git_sha(value: Any) -> bool:
|
||||||
|
return isinstance(value, str) and bool(HEX_40.fullmatch(value))
|
||||||
|
|
||||||
|
|
||||||
|
def _non_negative_int(value: Any) -> bool:
|
||||||
|
return isinstance(value, int) and not isinstance(value, bool) and value >= 0
|
||||||
|
|
||||||
|
|
||||||
|
def _safe_usage(value: Any) -> dict[str, int | float | None]:
|
||||||
|
if not isinstance(value, dict):
|
||||||
|
return {}
|
||||||
|
safe: dict[str, int | float | None] = {}
|
||||||
|
for key in (
|
||||||
|
"input_tokens",
|
||||||
|
"output_tokens",
|
||||||
|
"prompt_tokens",
|
||||||
|
"completion_tokens",
|
||||||
|
"total_tokens",
|
||||||
|
"cache_read_tokens",
|
||||||
|
"cache_write_tokens",
|
||||||
|
"reasoning_tokens",
|
||||||
|
):
|
||||||
|
item = value.get(key)
|
||||||
|
if isinstance(item, (int, float)) and not isinstance(item, bool):
|
||||||
|
safe[key] = item
|
||||||
|
return safe
|
||||||
|
|
||||||
|
|
||||||
|
def _trace_events(result: dict[str, Any], event: str) -> list[dict[str, Any]]:
|
||||||
|
return [
|
||||||
|
item
|
||||||
|
for item in result.get("model_call_trace") or []
|
||||||
|
if isinstance(item, dict) and item.get("event") == event
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def _response_trace(result: dict[str, Any]) -> list[dict[str, Any]]:
|
||||||
|
responses = []
|
||||||
|
for raw in result.get("model_response_trace") or []:
|
||||||
|
if not isinstance(raw, dict):
|
||||||
|
continue
|
||||||
|
responses.append(
|
||||||
|
{
|
||||||
|
"content_sha256": raw.get("content_sha256"),
|
||||||
|
"content_bytes": raw.get("content_bytes"),
|
||||||
|
"tool_calls_sha256": raw.get("tool_calls_sha256"),
|
||||||
|
"tool_call_count": raw.get("tool_call_count"),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
return responses
|
||||||
|
|
||||||
|
|
||||||
|
def _model_calls(result: dict[str, Any]) -> list[dict[str, Any]]:
|
||||||
|
calls: list[dict[str, Any]] = []
|
||||||
|
responses = _response_trace(result)
|
||||||
|
for index, raw in enumerate(_trace_events(result, "post_api_request")):
|
||||||
|
response = responses[index] if index < len(responses) else {}
|
||||||
|
calls.append(
|
||||||
|
{
|
||||||
|
"api_call_count": raw.get("api_call_count"),
|
||||||
|
"api_mode": raw.get("api_mode"),
|
||||||
|
"model": raw.get("model"),
|
||||||
|
"provider": raw.get("provider"),
|
||||||
|
"response_model": raw.get("response_model"),
|
||||||
|
"finish_reason": raw.get("finish_reason"),
|
||||||
|
"message_count": raw.get("message_count"),
|
||||||
|
"assistant_content_chars": raw.get("assistant_content_chars"),
|
||||||
|
"assistant_tool_call_count": raw.get("assistant_tool_call_count"),
|
||||||
|
"task_id_sha256": raw.get("task_id_sha256"),
|
||||||
|
"session_id_sha256": raw.get("session_id_sha256"),
|
||||||
|
"base_url_sha256": raw.get("base_url_sha256"),
|
||||||
|
"usage": _safe_usage(raw.get("usage")),
|
||||||
|
"provider_response_id": raw.get("provider_response_id"),
|
||||||
|
"provider_response_id_status": raw.get("provider_response_id_status"),
|
||||||
|
"assistant_message": response,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
return calls
|
||||||
|
|
||||||
|
|
||||||
|
def _safe_retrieval_receipt(value: Any, *, expected_query_sha256: str) -> dict[str, Any] | None:
|
||||||
|
if not isinstance(value, dict) or value.get("schema") != RETRIEVAL_RECEIPT_SCHEMA:
|
||||||
|
return None
|
||||||
|
consistency = value.get("read_consistency") if isinstance(value.get("read_consistency"), dict) else {}
|
||||||
|
counts = value.get("counts") if isinstance(value.get("counts"), dict) else {}
|
||||||
|
required_hashes = (
|
||||||
|
value.get("query_sha256"),
|
||||||
|
value.get("semantic_context_sha256"),
|
||||||
|
value.get("artifact_state_sha256"),
|
||||||
|
value.get("receipt_sha256"),
|
||||||
|
)
|
||||||
|
stable_consistency = bool(
|
||||||
|
consistency.get("status") == "stable_wal_marker"
|
||||||
|
and consistency.get("database")
|
||||||
|
and consistency.get("database_user")
|
||||||
|
and consistency.get("system_identifier")
|
||||||
|
and consistency.get("wal_lsn_before")
|
||||||
|
and consistency.get("wal_lsn_before") == consistency.get("wal_lsn_after")
|
||||||
|
)
|
||||||
|
if (
|
||||||
|
not all(_is_sha256(item) for item in required_hashes)
|
||||||
|
or value.get("query_sha256") != expected_query_sha256
|
||||||
|
or not stable_consistency
|
||||||
|
):
|
||||||
|
return None
|
||||||
|
return {
|
||||||
|
"schema": value.get("schema"),
|
||||||
|
"query_sha256": value.get("query_sha256"),
|
||||||
|
"semantic_context_sha256": value.get("semantic_context_sha256"),
|
||||||
|
"artifact_state_sha256": value.get("artifact_state_sha256"),
|
||||||
|
"claim_ids_sha256": canonical_sha256(sorted(str(item) for item in value.get("claim_ids") or [])),
|
||||||
|
"source_ids_sha256": canonical_sha256(sorted(str(item) for item in value.get("source_ids") or [])),
|
||||||
|
"counts": {
|
||||||
|
key: item
|
||||||
|
for key, item in sorted(counts.items())
|
||||||
|
if isinstance(key, str) and isinstance(item, int) and not isinstance(item, bool)
|
||||||
|
},
|
||||||
|
"read_consistency": {
|
||||||
|
"status": consistency.get("status"),
|
||||||
|
"attempts": consistency.get("attempts"),
|
||||||
|
"database": consistency.get("database"),
|
||||||
|
"database_user": consistency.get("database_user"),
|
||||||
|
"system_identifier": consistency.get("system_identifier"),
|
||||||
|
"wal_lsn_before": consistency.get("wal_lsn_before"),
|
||||||
|
"wal_lsn_after": consistency.get("wal_lsn_after"),
|
||||||
|
},
|
||||||
|
"receipt_sha256": value.get("receipt_sha256"),
|
||||||
|
"trace_sha256": canonical_sha256(value),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _context_receipts(result: dict[str, Any], *, expected_query_sha256: str) -> list[dict[str, Any]]:
|
||||||
|
receipts = []
|
||||||
|
for record in result.get("database_context_trace") or []:
|
||||||
|
if (
|
||||||
|
not isinstance(record, dict)
|
||||||
|
or record.get("event") != "pre_llm_call"
|
||||||
|
or record.get("status") != "ok"
|
||||||
|
or record.get("injected") is not True
|
||||||
|
or record.get("query_sha256") != expected_query_sha256
|
||||||
|
):
|
||||||
|
continue
|
||||||
|
receipt = _safe_retrieval_receipt(
|
||||||
|
record.get("retrieval_receipt"), expected_query_sha256=expected_query_sha256
|
||||||
|
)
|
||||||
|
if receipt is not None:
|
||||||
|
receipts.append(receipt)
|
||||||
|
return receipts
|
||||||
|
|
||||||
|
|
||||||
|
def _tool_receipts(result: dict[str, Any]) -> list[dict[str, Any]]:
|
||||||
|
receipts = []
|
||||||
|
trace = result.get("database_tool_trace") or {}
|
||||||
|
for call in trace.get("calls") or []:
|
||||||
|
if not isinstance(call, dict):
|
||||||
|
continue
|
||||||
|
raw = ((call.get("result") or {}).get("retrieval_receipt") or {})
|
||||||
|
if (
|
||||||
|
not isinstance(raw, dict)
|
||||||
|
or raw.get("schema") != RETRIEVAL_RECEIPT_SCHEMA
|
||||||
|
or not _is_sha256(raw.get("semantic_context_sha256"))
|
||||||
|
or not _is_sha256(raw.get("artifact_state_sha256"))
|
||||||
|
or raw.get("read_consistency_status") != "stable_wal_marker"
|
||||||
|
):
|
||||||
|
continue
|
||||||
|
receipts.append(
|
||||||
|
{
|
||||||
|
"schema": raw.get("schema"),
|
||||||
|
"semantic_context_sha256": raw.get("semantic_context_sha256"),
|
||||||
|
"artifact_state_sha256": raw.get("artifact_state_sha256"),
|
||||||
|
"read_consistency": {"status": raw.get("read_consistency_status")},
|
||||||
|
"receipt_sha256": canonical_sha256(raw),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
return receipts
|
||||||
|
|
||||||
|
|
||||||
|
def _database_required(result: dict[str, Any], *, prompt: str) -> bool:
|
||||||
|
prompt_casefold = prompt.casefold()
|
||||||
|
if any(term in prompt_casefold for term in DATABASE_PROMPT_TERMS):
|
||||||
|
return True
|
||||||
|
for record in result.get("database_context_trace") or []:
|
||||||
|
if not isinstance(record, dict) or record.get("event") != "pre_llm_call":
|
||||||
|
continue
|
||||||
|
contract_ids = {str(item) for item in record.get("contract_ids") or []}
|
||||||
|
if contract_ids - {"reply_budget"}:
|
||||||
|
return True
|
||||||
|
trace = result.get("database_tool_trace") or {}
|
||||||
|
return bool(trace.get("database_tool_call_count"))
|
||||||
|
|
||||||
|
|
||||||
|
def _database_context_binding(
|
||||||
|
result: dict[str, Any], *, prompt_sha256: str, reply_sha256: str
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
records = [item for item in result.get("database_context_trace") or [] if isinstance(item, dict)]
|
||||||
|
pre_records = [item for item in records if item.get("event") == "pre_llm_call"]
|
||||||
|
post_records = [item for item in records if item.get("event") == "post_llm_call"]
|
||||||
|
pre = pre_records[0] if len(pre_records) == 1 else {}
|
||||||
|
post = post_records[0] if len(post_records) == 1 else {}
|
||||||
|
raw_response_sha256 = post.get("response_sha256")
|
||||||
|
delivered_response_sha256 = post.get("delivered_response_sha256")
|
||||||
|
return {
|
||||||
|
"pre_record_count": len(pre_records),
|
||||||
|
"post_record_count": len(post_records),
|
||||||
|
"query_sha256": pre.get("query_sha256"),
|
||||||
|
"contract_ids": sorted(str(item) for item in pre.get("contract_ids") or []),
|
||||||
|
"contract_sha256": pre.get("contract_sha256"),
|
||||||
|
"pre_status": pre.get("status"),
|
||||||
|
"pre_injected": pre.get("injected"),
|
||||||
|
"post_status": post.get("status"),
|
||||||
|
"post_validated": post.get("validated"),
|
||||||
|
"post_transformed": post.get("transformed"),
|
||||||
|
"raw_response_sha256": raw_response_sha256,
|
||||||
|
"delivered_response_sha256": delivered_response_sha256,
|
||||||
|
"query_bound": bool(
|
||||||
|
len(pre_records) == 1
|
||||||
|
and len(post_records) == 1
|
||||||
|
and pre.get("query_sha256") == prompt_sha256
|
||||||
|
and post.get("query_sha256") == prompt_sha256
|
||||||
|
),
|
||||||
|
"response_bound": bool(
|
||||||
|
_is_sha256(raw_response_sha256)
|
||||||
|
and delivered_response_sha256 == reply_sha256
|
||||||
|
and post.get("validated") is True
|
||||||
|
and post.get("status") == "ok"
|
||||||
|
),
|
||||||
|
"context_available": bool(
|
||||||
|
len(pre_records) == 1
|
||||||
|
and pre.get("status") == "ok"
|
||||||
|
and pre.get("injected") is True
|
||||||
|
and _is_sha256(pre.get("contract_sha256"))
|
||||||
|
),
|
||||||
|
"trace_sha256": canonical_sha256(records),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _database_tool_binding(result: dict[str, Any]) -> dict[str, Any]:
|
||||||
|
trace = result.get("database_tool_trace") or {}
|
||||||
|
calls = []
|
||||||
|
for call in trace.get("calls") or []:
|
||||||
|
if not isinstance(call, dict):
|
||||||
|
continue
|
||||||
|
call_result = call.get("result") if isinstance(call.get("result"), dict) else {}
|
||||||
|
invocations = [
|
||||||
|
{
|
||||||
|
"access_mode": invocation.get("access_mode"),
|
||||||
|
"command_sha256": invocation.get("command_sha256"),
|
||||||
|
"executable": invocation.get("executable"),
|
||||||
|
"subcommand": invocation.get("subcommand"),
|
||||||
|
}
|
||||||
|
for invocation in call.get("database_invocations") or []
|
||||||
|
if isinstance(invocation, dict)
|
||||||
|
]
|
||||||
|
calls.append(
|
||||||
|
{
|
||||||
|
"tool_call_id_sha256": call.get("tool_call_id_sha256"),
|
||||||
|
"arguments_sha256": call.get("arguments_sha256"),
|
||||||
|
"database_invocations": invocations,
|
||||||
|
"result_content_sha256": call_result.get("content_sha256"),
|
||||||
|
"result_content_bytes": call_result.get("content_bytes"),
|
||||||
|
"result_error_detected": call_result.get("error_detected"),
|
||||||
|
"result_nonempty": call_result.get("nonempty"),
|
||||||
|
"result_row_ids_sha256": canonical_sha256(sorted(call_result.get("row_ids") or [])),
|
||||||
|
"result_sha256_values_sha256": canonical_sha256(sorted(call_result.get("sha256_values") or [])),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
tool_call_count = trace.get("database_tool_call_count", 0)
|
||||||
|
completed_count = trace.get("database_tool_completed_count", 0)
|
||||||
|
all_results_bound = bool(
|
||||||
|
_non_negative_int(tool_call_count)
|
||||||
|
and _non_negative_int(completed_count)
|
||||||
|
and completed_count == tool_call_count
|
||||||
|
and len(calls) == tool_call_count
|
||||||
|
and all(
|
||||||
|
_is_sha256(call.get("tool_call_id_sha256"))
|
||||||
|
and _is_sha256(call.get("arguments_sha256"))
|
||||||
|
and _is_sha256(call.get("result_content_sha256"))
|
||||||
|
and _non_negative_int(call.get("result_content_bytes"))
|
||||||
|
and call.get("result_error_detected") is False
|
||||||
|
and call.get("result_nonempty") is True
|
||||||
|
for call in calls
|
||||||
|
)
|
||||||
|
)
|
||||||
|
all_calls_read_only = bool(
|
||||||
|
trace.get("database_tool_calls_read_only") is True
|
||||||
|
and all(
|
||||||
|
call.get("database_invocations")
|
||||||
|
and all(
|
||||||
|
invocation.get("access_mode") == "read_only"
|
||||||
|
and _is_sha256(invocation.get("command_sha256"))
|
||||||
|
for invocation in call.get("database_invocations") or []
|
||||||
|
)
|
||||||
|
for call in calls
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
"schema": trace.get("schema"),
|
||||||
|
"tool_call_count": tool_call_count,
|
||||||
|
"completed_count": completed_count,
|
||||||
|
"all_calls_read_only": all_calls_read_only if tool_call_count else True,
|
||||||
|
"all_results_content_bound": all_results_bound,
|
||||||
|
"calls": calls,
|
||||||
|
"trace_sha256": canonical_sha256(trace),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _database_fingerprint_complete(value: dict[str, Any]) -> bool:
|
||||||
|
consistency = value.get("read_consistency") if isinstance(value.get("read_consistency"), dict) else {}
|
||||||
|
before = consistency.get("before") if isinstance(consistency.get("before"), dict) else {}
|
||||||
|
after = consistency.get("after") if isinstance(consistency.get("after"), dict) else {}
|
||||||
|
return bool(
|
||||||
|
value.get("status") == "ok"
|
||||||
|
and _is_sha256(value.get("fingerprint_sha256"))
|
||||||
|
and _is_sha256(value.get("table_rows_sha256"))
|
||||||
|
and _is_sha256(value.get("structure_sha256"))
|
||||||
|
and isinstance(value.get("table_count"), int)
|
||||||
|
and value.get("table_count") > 0
|
||||||
|
and _non_negative_int(value.get("total_rows"))
|
||||||
|
and consistency.get("status") == "stable_wal_marker"
|
||||||
|
and before
|
||||||
|
and before == after
|
||||||
|
and before.get("database")
|
||||||
|
and before.get("database_user")
|
||||||
|
and before.get("system_identifier")
|
||||||
|
and before.get("wal_lsn")
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _source_tree_complete(runtime: Any) -> bool:
|
||||||
|
if not isinstance(runtime, dict):
|
||||||
|
return False
|
||||||
|
tree = runtime.get("source_tree") if isinstance(runtime.get("source_tree"), dict) else {}
|
||||||
|
return bool(
|
||||||
|
_is_git_sha(runtime.get("git_head"))
|
||||||
|
and _is_sha256(tree.get("sha256"))
|
||||||
|
and isinstance(tree.get("file_count"), int)
|
||||||
|
and tree.get("file_count") > 0
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _conversation_binding(
|
||||||
|
result: dict[str, Any],
|
||||||
|
*,
|
||||||
|
reply_sha256: str,
|
||||||
|
previous_execution_sha256: str | None,
|
||||||
|
expected_previous_conversation: dict[str, Any] | None,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
before = result.get("conversation_before") if isinstance(result.get("conversation_before"), dict) else {}
|
||||||
|
after = result.get("conversation_after") if isinstance(result.get("conversation_after"), dict) else {}
|
||||||
|
expected = expected_previous_conversation or {}
|
||||||
|
before_valid = bool(
|
||||||
|
_non_negative_int(before.get("message_count"))
|
||||||
|
and _is_sha256(before.get("messages_sha256"))
|
||||||
|
)
|
||||||
|
after_valid = bool(
|
||||||
|
_non_negative_int(after.get("message_count"))
|
||||||
|
and after.get("message_count", 0) > before.get("message_count", -1)
|
||||||
|
and _is_sha256(after.get("messages_sha256"))
|
||||||
|
and after.get("last_message_role") == "assistant"
|
||||||
|
and after.get("last_message_content_sha256") == reply_sha256
|
||||||
|
)
|
||||||
|
if previous_execution_sha256 is None:
|
||||||
|
prior_state_bound = before.get("message_count") == 0
|
||||||
|
else:
|
||||||
|
prior_state_bound = bool(
|
||||||
|
_is_sha256(previous_execution_sha256)
|
||||||
|
and before.get("message_count") == expected.get("message_count")
|
||||||
|
and before.get("messages_sha256") == expected.get("messages_sha256")
|
||||||
|
and before.get("last_message_role") == expected.get("last_message_role")
|
||||||
|
and before.get("last_message_content_sha256") == expected.get("last_message_content_sha256")
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
"before": before,
|
||||||
|
"after": after,
|
||||||
|
"history_prefix_preserved": result.get("conversation_history_prefix_preserved"),
|
||||||
|
"previous_execution_sha256": previous_execution_sha256,
|
||||||
|
"starts_from_empty_conversation": before.get("message_count") == 0,
|
||||||
|
"prior_turn_state_bound": prior_state_bound,
|
||||||
|
"conversation_hashes_valid": before_valid and after_valid,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _model_execution_binding(
|
||||||
|
result: dict[str, Any],
|
||||||
|
*,
|
||||||
|
prompt_sha256: str,
|
||||||
|
reply_sha256: str,
|
||||||
|
raw_response_sha256: str | None,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
calls = _model_calls(result)
|
||||||
|
responses = _response_trace(result)
|
||||||
|
pre_events = _trace_events(result, "turn_pre_llm_call")
|
||||||
|
post_events = _trace_events(result, "turn_post_llm_call")
|
||||||
|
pre = pre_events[0] if len(pre_events) == 1 else {}
|
||||||
|
post = post_events[0] if len(post_events) == 1 else {}
|
||||||
|
session_hashes = {
|
||||||
|
str(value)
|
||||||
|
for value in [
|
||||||
|
pre.get("session_id_sha256"),
|
||||||
|
post.get("session_id_sha256"),
|
||||||
|
*(call.get("session_id_sha256") for call in calls),
|
||||||
|
]
|
||||||
|
if value
|
||||||
|
}
|
||||||
|
call_sequence = [call.get("api_call_count") for call in calls]
|
||||||
|
response_hashes_valid = bool(
|
||||||
|
responses
|
||||||
|
and len(responses) == len(calls)
|
||||||
|
and all(
|
||||||
|
_is_sha256(response.get("content_sha256"))
|
||||||
|
and _is_sha256(response.get("tool_calls_sha256"))
|
||||||
|
and _non_negative_int(response.get("content_bytes"))
|
||||||
|
and _non_negative_int(response.get("tool_call_count"))
|
||||||
|
for response in responses
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
"call_count": len(calls),
|
||||||
|
"calls": calls,
|
||||||
|
"pre_llm_trace_count": len(pre_events),
|
||||||
|
"post_llm_trace_count": len(post_events),
|
||||||
|
"prompt_sha256": pre.get("prompt_sha256"),
|
||||||
|
"raw_assistant_response_sha256": post.get("raw_assistant_response_sha256"),
|
||||||
|
"session_id_sha256": next(iter(session_hashes)) if len(session_hashes) == 1 else None,
|
||||||
|
"prompt_bound": bool(
|
||||||
|
len(pre_events) == 1
|
||||||
|
and len(post_events) == 1
|
||||||
|
and pre.get("prompt_sha256") == prompt_sha256
|
||||||
|
and post.get("prompt_sha256") == prompt_sha256
|
||||||
|
),
|
||||||
|
"raw_response_bound": bool(
|
||||||
|
_is_sha256(raw_response_sha256)
|
||||||
|
and post.get("raw_assistant_response_sha256") == raw_response_sha256
|
||||||
|
),
|
||||||
|
"delivered_response_bound": bool(
|
||||||
|
response_hashes_valid and responses[-1].get("content_sha256") == reply_sha256
|
||||||
|
),
|
||||||
|
"response_trace_count_matches_api_calls": len(responses) == len(calls),
|
||||||
|
"api_call_sequence_valid": call_sequence == list(range(1, len(calls) + 1)),
|
||||||
|
"session_binding_valid": len(session_hashes) == 1 and all(_is_sha256(item) for item in session_hashes),
|
||||||
|
"response_hashes_valid": response_hashes_valid,
|
||||||
|
"externally_hosted_weights_hashable": False,
|
||||||
|
"provider_response_id_required_for_attribution": False,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _suite_safety_binding(value: dict[str, Any] | None) -> dict[str, Any]:
|
||||||
|
source = value or {}
|
||||||
|
return {
|
||||||
|
"remote_returncode": source.get("remote_returncode"),
|
||||||
|
"pass_runtime": source.get("pass_runtime"),
|
||||||
|
"live_behavior_manifest_unchanged": source.get("live_behavior_manifest_unchanged"),
|
||||||
|
"temp_profile_removed": source.get("temp_profile_removed"),
|
||||||
|
"service_unchanged": source.get("service_unchanged"),
|
||||||
|
"db_fingerprint_unchanged": source.get("db_fingerprint_unchanged"),
|
||||||
|
"model_call_trace_all_bound": source.get("model_call_trace_all_bound"),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _required_missing(
|
||||||
|
*,
|
||||||
|
behavior: dict[str, Any],
|
||||||
|
model_execution: dict[str, Any],
|
||||||
|
context_receipts: list[dict[str, Any]],
|
||||||
|
database_context_binding: dict[str, Any],
|
||||||
|
database_required: bool,
|
||||||
|
session_key: str,
|
||||||
|
prompt: str,
|
||||||
|
reply: str,
|
||||||
|
harness_source: dict[str, Any],
|
||||||
|
database_tool_binding: dict[str, Any],
|
||||||
|
db_fingerprint_before: dict[str, Any],
|
||||||
|
db_fingerprint_after: dict[str, Any],
|
||||||
|
db_counts_changed: bool | None,
|
||||||
|
posted_to_telegram: bool | None,
|
||||||
|
mutates_kb_by_harness: bool | None,
|
||||||
|
result_mutates_kb: bool | None,
|
||||||
|
conversation_binding: dict[str, Any],
|
||||||
|
suite_safety: dict[str, Any],
|
||||||
|
) -> list[str]:
|
||||||
|
missing = []
|
||||||
|
checks = {
|
||||||
|
"prompt": bool(prompt),
|
||||||
|
"reply": bool(reply),
|
||||||
|
"session_boundary": bool(session_key),
|
||||||
|
"behavior_sha256": _is_sha256(behavior.get("behavior_sha256")),
|
||||||
|
"hermes_runtime_content_addressed": _source_tree_complete(behavior.get("hermes_runtime")),
|
||||||
|
"teleo_infrastructure_runtime_content_addressed": _source_tree_complete(
|
||||||
|
behavior.get("teleo_infrastructure_runtime")
|
||||||
|
),
|
||||||
|
"harness_git_head": _is_git_sha(harness_source.get("git_head")),
|
||||||
|
"harness_worktree_clean": bool(
|
||||||
|
harness_source.get("worktree_clean") is True
|
||||||
|
and _is_sha256(harness_source.get("status_sha256"))
|
||||||
|
),
|
||||||
|
"actual_model_call": bool(
|
||||||
|
model_execution.get("calls")
|
||||||
|
and all(
|
||||||
|
call.get("model")
|
||||||
|
and call.get("provider")
|
||||||
|
and call.get("response_model")
|
||||||
|
and _is_sha256(call.get("task_id_sha256"))
|
||||||
|
and _is_sha256(call.get("session_id_sha256"))
|
||||||
|
and _is_sha256(call.get("base_url_sha256"))
|
||||||
|
for call in model_execution.get("calls") or []
|
||||||
|
)
|
||||||
|
),
|
||||||
|
"model_prompt_binding": model_execution.get("prompt_bound") is True,
|
||||||
|
"model_raw_response_binding": model_execution.get("raw_response_bound") is True,
|
||||||
|
"model_delivered_response_binding": model_execution.get("delivered_response_bound") is True,
|
||||||
|
"model_response_trace_count": model_execution.get("response_trace_count_matches_api_calls") is True,
|
||||||
|
"model_api_call_sequence": model_execution.get("api_call_sequence_valid") is True,
|
||||||
|
"model_session_binding": model_execution.get("session_binding_valid") is True,
|
||||||
|
"database_context_query_binding": database_context_binding.get("query_bound") is True,
|
||||||
|
"database_context_available": database_context_binding.get("context_available") is True,
|
||||||
|
"database_context_response_binding": database_context_binding.get("response_bound") is True,
|
||||||
|
"database_retrieval_receipt": not database_required or len(context_receipts) == 1,
|
||||||
|
"database_tool_results": not database_tool_binding.get("tool_call_count")
|
||||||
|
or bool(database_tool_binding.get("all_results_content_bound")),
|
||||||
|
"database_tools_read_only": database_tool_binding.get("all_calls_read_only") is True,
|
||||||
|
"database_fingerprint_before": _database_fingerprint_complete(db_fingerprint_before),
|
||||||
|
"database_fingerprint_after": _database_fingerprint_complete(db_fingerprint_after),
|
||||||
|
"database_fingerprint_unchanged": db_fingerprint_before.get("fingerprint_sha256")
|
||||||
|
== db_fingerprint_after.get("fingerprint_sha256"),
|
||||||
|
"database_counts_unchanged": db_counts_changed is False,
|
||||||
|
"telegram_not_posted": posted_to_telegram is False,
|
||||||
|
"harness_did_not_mutate_kb": mutates_kb_by_harness is False,
|
||||||
|
"turn_did_not_mutate_kb": result_mutates_kb is False,
|
||||||
|
"conversation_history_prefix": conversation_binding.get("history_prefix_preserved") is True,
|
||||||
|
"conversation_hashes": conversation_binding.get("conversation_hashes_valid") is True,
|
||||||
|
"conversation_prior_turn_binding": conversation_binding.get("prior_turn_state_bound") is True,
|
||||||
|
"suite_remote_success": suite_safety.get("remote_returncode") == 0,
|
||||||
|
"suite_runtime_success": suite_safety.get("pass_runtime") is True,
|
||||||
|
"suite_live_behavior_unchanged": suite_safety.get("live_behavior_manifest_unchanged") is True,
|
||||||
|
"suite_temp_profile_removed": suite_safety.get("temp_profile_removed") is True,
|
||||||
|
"suite_service_unchanged": suite_safety.get("service_unchanged") is True,
|
||||||
|
"suite_db_fingerprint_unchanged": suite_safety.get("db_fingerprint_unchanged") is True,
|
||||||
|
"suite_model_trace_bound": suite_safety.get("model_call_trace_all_bound") is True,
|
||||||
|
}
|
||||||
|
for label, present in checks.items():
|
||||||
|
if not present:
|
||||||
|
missing.append(label)
|
||||||
|
return missing
|
||||||
|
|
||||||
|
|
||||||
|
def build_turn_manifest(
|
||||||
|
*,
|
||||||
|
result: dict[str, Any],
|
||||||
|
behavior_manifest: dict[str, Any],
|
||||||
|
session_key: str,
|
||||||
|
source: dict[str, Any],
|
||||||
|
suite_mode: str,
|
||||||
|
remote_run_id: str | None,
|
||||||
|
db_counts_before: dict[str, Any] | None,
|
||||||
|
db_counts_after: dict[str, Any] | None,
|
||||||
|
db_counts_changed: bool | None,
|
||||||
|
db_fingerprint_before: dict[str, Any] | None,
|
||||||
|
db_fingerprint_after: dict[str, Any] | None,
|
||||||
|
posted_to_telegram: bool | None,
|
||||||
|
mutates_kb_by_harness: bool | None,
|
||||||
|
harness_source: dict[str, Any],
|
||||||
|
suite_safety: dict[str, Any] | None = None,
|
||||||
|
previous_execution_sha256: str | None = None,
|
||||||
|
expected_previous_conversation: dict[str, Any] | None = None,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
prompt = str(result.get("prompt") or "")
|
||||||
|
reply = str(result.get("reply") or "")
|
||||||
|
prompt_sha256 = text_sha256(prompt)
|
||||||
|
database_query_sha256 = text_sha256(prompt[:16_000])
|
||||||
|
reply_sha256 = text_sha256(reply)
|
||||||
|
database_context_binding = _database_context_binding(
|
||||||
|
result,
|
||||||
|
prompt_sha256=database_query_sha256,
|
||||||
|
reply_sha256=reply_sha256,
|
||||||
|
)
|
||||||
|
model_execution = _model_execution_binding(
|
||||||
|
result,
|
||||||
|
prompt_sha256=prompt_sha256,
|
||||||
|
reply_sha256=reply_sha256,
|
||||||
|
raw_response_sha256=database_context_binding.get("raw_response_sha256"),
|
||||||
|
)
|
||||||
|
context_receipts = _context_receipts(result, expected_query_sha256=database_query_sha256)
|
||||||
|
tool_receipts = _tool_receipts(result)
|
||||||
|
database_required = _database_required(result, prompt=prompt)
|
||||||
|
database_tool_binding = _database_tool_binding(result)
|
||||||
|
fingerprint_before = db_fingerprint_before or {}
|
||||||
|
fingerprint_after = db_fingerprint_after or {}
|
||||||
|
fingerprint_unchanged = bool(
|
||||||
|
fingerprint_before.get("fingerprint_sha256")
|
||||||
|
and fingerprint_after.get("fingerprint_sha256")
|
||||||
|
and fingerprint_before.get("fingerprint_sha256") == fingerprint_after.get("fingerprint_sha256")
|
||||||
|
)
|
||||||
|
conversation_binding = _conversation_binding(
|
||||||
|
result,
|
||||||
|
reply_sha256=reply_sha256,
|
||||||
|
previous_execution_sha256=previous_execution_sha256,
|
||||||
|
expected_previous_conversation=expected_previous_conversation,
|
||||||
|
)
|
||||||
|
safety_binding = _suite_safety_binding(suite_safety)
|
||||||
|
missing = _required_missing(
|
||||||
|
behavior=behavior_manifest,
|
||||||
|
model_execution=model_execution,
|
||||||
|
context_receipts=context_receipts,
|
||||||
|
database_context_binding=database_context_binding,
|
||||||
|
database_required=database_required,
|
||||||
|
session_key=session_key,
|
||||||
|
prompt=prompt,
|
||||||
|
reply=reply,
|
||||||
|
harness_source=harness_source,
|
||||||
|
database_tool_binding=database_tool_binding,
|
||||||
|
db_fingerprint_before=fingerprint_before,
|
||||||
|
db_fingerprint_after=fingerprint_after,
|
||||||
|
db_counts_changed=db_counts_changed,
|
||||||
|
posted_to_telegram=posted_to_telegram,
|
||||||
|
mutates_kb_by_harness=mutates_kb_by_harness,
|
||||||
|
result_mutates_kb=result.get("mutates_kb"),
|
||||||
|
conversation_binding=conversation_binding,
|
||||||
|
suite_safety=safety_binding,
|
||||||
|
)
|
||||||
|
stable = {
|
||||||
|
"schema": SCHEMA,
|
||||||
|
"turn": {
|
||||||
|
"prompt_id": result.get("prompt_id"),
|
||||||
|
"turn_index": result.get("turn"),
|
||||||
|
"remote_run_id": remote_run_id,
|
||||||
|
"started_at_utc": result.get("started_at_utc"),
|
||||||
|
"ended_at_utc": result.get("ended_at_utc"),
|
||||||
|
"prompt_sha256": prompt_sha256,
|
||||||
|
"database_query_sha256": database_query_sha256,
|
||||||
|
"reply_sha256": reply_sha256,
|
||||||
|
"prompt_bytes": len(prompt.encode("utf-8")),
|
||||||
|
"reply_bytes": len(reply.encode("utf-8")),
|
||||||
|
},
|
||||||
|
"session_boundary": {
|
||||||
|
"session_key_sha256": text_sha256(session_key) if session_key else None,
|
||||||
|
"source_platform": source.get("platform"),
|
||||||
|
"chat_type": source.get("chat_type"),
|
||||||
|
"fresh_temp_profile_for_suite": True,
|
||||||
|
"prior_dynamic_state_excluded_from_suite": True,
|
||||||
|
"conversation": conversation_binding,
|
||||||
|
},
|
||||||
|
"runtime": {
|
||||||
|
"behavior_sha256": behavior_manifest.get("behavior_sha256"),
|
||||||
|
"hermes_runtime": behavior_manifest.get("hermes_runtime"),
|
||||||
|
"teleo_infrastructure_runtime": behavior_manifest.get("teleo_infrastructure_runtime"),
|
||||||
|
"profile_component_hashes": {
|
||||||
|
name: (component.get("content") or {}).get("sha256")
|
||||||
|
for name, component in sorted((behavior_manifest.get("components") or {}).items())
|
||||||
|
if isinstance(component, dict)
|
||||||
|
},
|
||||||
|
"harness_source": harness_source,
|
||||||
|
},
|
||||||
|
"model_execution": model_execution,
|
||||||
|
"canonical_database": {
|
||||||
|
"required_for_turn": database_required,
|
||||||
|
"binding_status": "retrieval_receipt_bound"
|
||||||
|
if context_receipts
|
||||||
|
else "not_required"
|
||||||
|
if not database_required
|
||||||
|
else "missing",
|
||||||
|
"context_retrieval_receipts": context_receipts,
|
||||||
|
"explicit_tool_retrieval_receipts": tool_receipts,
|
||||||
|
"context_binding": database_context_binding,
|
||||||
|
"database_tool_binding": database_tool_binding,
|
||||||
|
"fingerprint_before": fingerprint_before,
|
||||||
|
"fingerprint_after": fingerprint_after,
|
||||||
|
"fingerprint_unchanged": fingerprint_unchanged,
|
||||||
|
"suite_counts_before_sha256": canonical_sha256(db_counts_before or {}),
|
||||||
|
"suite_counts_after_sha256": canonical_sha256(db_counts_after or {}),
|
||||||
|
"suite_counts_changed": db_counts_changed,
|
||||||
|
"full_database_content_fingerprint_included": bool(
|
||||||
|
fingerprint_before.get("fingerprint_sha256") and fingerprint_after.get("fingerprint_sha256")
|
||||||
|
),
|
||||||
|
"full_database_snapshot_artifact_included": False,
|
||||||
|
},
|
||||||
|
"delivery_and_safety": {
|
||||||
|
"suite_mode": suite_mode,
|
||||||
|
"posted_to_telegram": posted_to_telegram,
|
||||||
|
"kb_mutation_by_harness": mutates_kb_by_harness,
|
||||||
|
"turn_mutates_kb": result.get("mutates_kb"),
|
||||||
|
"suite_safety": safety_binding,
|
||||||
|
"raw_prompt_retained_in_manifest": False,
|
||||||
|
"raw_reply_retained_in_manifest": False,
|
||||||
|
"raw_database_rows_retained_in_manifest": False,
|
||||||
|
"raw_session_identifier_retained_in_manifest": False,
|
||||||
|
},
|
||||||
|
"attribution": {
|
||||||
|
"status": "complete" if not missing else "incomplete",
|
||||||
|
"missing_required_bindings": missing,
|
||||||
|
},
|
||||||
|
"replay_claim": {
|
||||||
|
"status": "content_addressed_attribution" if not missing else "incomplete_attribution",
|
||||||
|
"runtime_and_turn_inputs_content_addressed": not missing,
|
||||||
|
"runtime_source_artifacts_embedded": False,
|
||||||
|
"database_state_identified_by_content_fingerprint": bool(
|
||||||
|
not missing and fingerprint_before.get("fingerprint_sha256")
|
||||||
|
),
|
||||||
|
"database_snapshot_reconstructible_from_this_manifest_alone": False,
|
||||||
|
"external_model_state_reconstructible": False,
|
||||||
|
"bitwise_identical_model_output_guaranteed": False,
|
||||||
|
"reason": (
|
||||||
|
"The receipt binds the tested prompt, conversation chain, response hashes, source hashes, "
|
||||||
|
"model-call metadata, and database reads. It identifies those inputs; it does not embed the "
|
||||||
|
"runtime source, database snapshot, or hosted model weights needed to reconstruct them."
|
||||||
|
),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
**stable,
|
||||||
|
"generated_at_utc": datetime.now(UTC).isoformat(),
|
||||||
|
"execution_sha256": canonical_sha256(stable),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def validate_turn_manifest(manifest: dict[str, Any]) -> list[str]:
|
||||||
|
problems = []
|
||||||
|
if manifest.get("schema") != SCHEMA:
|
||||||
|
problems.append("schema_mismatch")
|
||||||
|
stable = {
|
||||||
|
key: value
|
||||||
|
for key, value in manifest.items()
|
||||||
|
if key not in {"generated_at_utc", "execution_sha256"}
|
||||||
|
}
|
||||||
|
if manifest.get("execution_sha256") != canonical_sha256(stable):
|
||||||
|
problems.append("execution_sha256_mismatch")
|
||||||
|
missing = (manifest.get("attribution") or {}).get("missing_required_bindings")
|
||||||
|
expected_status = "complete" if not missing else "incomplete"
|
||||||
|
if (manifest.get("attribution") or {}).get("status") != expected_status:
|
||||||
|
problems.append("attribution_status_mismatch")
|
||||||
|
return problems
|
||||||
|
|
||||||
|
|
||||||
|
def attach_execution_manifests(report: dict[str, Any], *, repo_root: Path) -> dict[str, Any]:
|
||||||
|
behavior = report.get("executed_behavior_manifest") or report.get("live_behavior_manifest_before") or {}
|
||||||
|
handler = report.get("handler") or {}
|
||||||
|
harness_source = git_state(repo_root)
|
||||||
|
results = [item for item in report.get("results") or [] if isinstance(item, dict)]
|
||||||
|
service = report.get("service_before_after") if isinstance(report.get("service_before_after"), dict) else {}
|
||||||
|
suite_safety = {
|
||||||
|
"remote_returncode": report.get("remote_returncode"),
|
||||||
|
"pass_runtime": report.get("pass_runtime"),
|
||||||
|
"live_behavior_manifest_unchanged": report.get("live_behavior_manifest_unchanged"),
|
||||||
|
"temp_profile_removed": report.get("temp_profile_removed"),
|
||||||
|
"service_unchanged": service.get("unchanged_from_preexisting_live_readback"),
|
||||||
|
"db_fingerprint_unchanged": report.get("db_fingerprint_unchanged"),
|
||||||
|
"model_call_trace_all_bound": report.get("model_call_trace_all_bound"),
|
||||||
|
}
|
||||||
|
complete = 0
|
||||||
|
previous_execution_sha256: str | None = None
|
||||||
|
expected_previous_conversation: dict[str, Any] | None = None
|
||||||
|
for result in results:
|
||||||
|
manifest = build_turn_manifest(
|
||||||
|
result=result,
|
||||||
|
behavior_manifest=behavior,
|
||||||
|
session_key=str(handler.get("session_key") or ""),
|
||||||
|
source=report.get("source") or {},
|
||||||
|
suite_mode=str(report.get("mode") or ""),
|
||||||
|
remote_run_id=report.get("remote_run_id"),
|
||||||
|
db_counts_before=report.get("db_counts_before"),
|
||||||
|
db_counts_after=report.get("db_counts_after"),
|
||||||
|
db_counts_changed=report.get("db_counts_changed"),
|
||||||
|
db_fingerprint_before=report.get("db_fingerprint_before"),
|
||||||
|
db_fingerprint_after=report.get("db_fingerprint_after"),
|
||||||
|
posted_to_telegram=report.get("posted_to_telegram"),
|
||||||
|
mutates_kb_by_harness=report.get("mutates_kb_by_harness"),
|
||||||
|
harness_source=harness_source,
|
||||||
|
suite_safety=suite_safety,
|
||||||
|
previous_execution_sha256=previous_execution_sha256,
|
||||||
|
expected_previous_conversation=expected_previous_conversation,
|
||||||
|
)
|
||||||
|
result["execution_manifest"] = manifest
|
||||||
|
if manifest["attribution"]["status"] == "complete" and not validate_turn_manifest(manifest):
|
||||||
|
complete += 1
|
||||||
|
previous_execution_sha256 = manifest.get("execution_sha256")
|
||||||
|
conversation = result.get("conversation_after")
|
||||||
|
expected_previous_conversation = conversation if isinstance(conversation, dict) else None
|
||||||
|
report["execution_manifest_summary"] = {
|
||||||
|
"schema": SCHEMA,
|
||||||
|
"turn_count": len(results),
|
||||||
|
"attribution_complete_count": complete,
|
||||||
|
"all_turns_attribution_complete": bool(results) and complete == len(results),
|
||||||
|
"harness_source": harness_source,
|
||||||
|
}
|
||||||
|
return report
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
parser = argparse.ArgumentParser(description=__doc__)
|
||||||
|
parser.add_argument("--suite-report", required=True, type=Path)
|
||||||
|
parser.add_argument("--repo-root", default=Path(__file__).resolve().parents[1], type=Path)
|
||||||
|
parser.add_argument("--output", required=True, type=Path)
|
||||||
|
args = parser.parse_args()
|
||||||
|
report = json.loads(args.suite_report.read_text(encoding="utf-8"))
|
||||||
|
attach_execution_manifests(report, repo_root=args.repo_root)
|
||||||
|
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
args.output.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||||
|
summary = report["execution_manifest_summary"]
|
||||||
|
print(json.dumps(summary, indent=2, sort_keys=True))
|
||||||
|
return 0 if summary["all_turns_attribution_complete"] else 1
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
|
|
@ -18,7 +18,15 @@ import uuid
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
import working_leo_open_ended_benchmark as benchmark
|
try:
|
||||||
|
import working_leo_open_ended_benchmark as benchmark
|
||||||
|
except ImportError: # pragma: no cover - imported as scripts.* in tests
|
||||||
|
from scripts import working_leo_open_ended_benchmark as benchmark
|
||||||
|
|
||||||
|
try:
|
||||||
|
from leo_turn_execution_manifest import attach_execution_manifests
|
||||||
|
except ImportError: # pragma: no cover - imported as scripts.* in tests
|
||||||
|
from scripts.leo_turn_execution_manifest import attach_execution_manifests
|
||||||
|
|
||||||
ROOT = Path(__file__).resolve().parents[1]
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
REPORT_DIR = ROOT / "docs" / "reports" / "leo-working-state-20260709"
|
REPORT_DIR = ROOT / "docs" / "reports" / "leo-working-state-20260709"
|
||||||
|
|
@ -34,15 +42,30 @@ CHAT_ID = "-5146042086"
|
||||||
USER_ID = "9070919"
|
USER_ID = "9070919"
|
||||||
USER_NAME = "codex handler direct claim"
|
USER_NAME = "codex handler direct claim"
|
||||||
|
|
||||||
SECRET_PATTERNS = [
|
JSON_SECRET_PATTERN = re.compile(
|
||||||
re.compile(r"\b\d{6,}:[A-Za-z0-9_-]{20,}\b"),
|
r'("(?:api[_-]?key|access[_-]?token|refresh[_-]?token|bot[_-]?token|token|secret|password)"\s*:\s*)'
|
||||||
re.compile(r"(Authorization: Bearer )([A-Za-z0-9._-]+)", re.IGNORECASE),
|
r'("(?:\\.|[^"\\])*"|null|[^\s,}\]]+)',
|
||||||
re.compile(r"((?:api[_-]?key|token|secret|password)[=:]\s*)([A-Za-z0-9._-]+)", re.IGNORECASE),
|
re.IGNORECASE,
|
||||||
]
|
)
|
||||||
|
TELEGRAM_TOKEN_PATTERN = re.compile(r"\b\d{6,}:[A-Za-z0-9_-]{20,}\b")
|
||||||
|
AUTHORIZATION_PATTERN = re.compile(r"(Authorization:\s*Bearer\s+)([^\s\"']+)", re.IGNORECASE)
|
||||||
|
ASSIGNMENT_SECRET_PATTERN = re.compile(
|
||||||
|
r"((?:api[_-]?key|access[_-]?token|refresh[_-]?token|bot[_-]?token|token|secret|password)"
|
||||||
|
r"\s*[=:]\s*)([^\s,;\"']+)",
|
||||||
|
re.IGNORECASE,
|
||||||
|
)
|
||||||
|
|
||||||
|
DB_CONTEXT_PLUGIN = (
|
||||||
|
ROOT / "hermes-agent" / "leoclean-plugins" / "vps" / "leo-db-context" / "__init__.py"
|
||||||
|
)
|
||||||
|
KB_TOOL = ROOT / "hermes-agent" / "leoclean-bin" / "kb_tool.py"
|
||||||
|
BEHAVIOR_MANIFEST = ROOT / "scripts" / "leo_behavior_manifest.py"
|
||||||
|
|
||||||
|
|
||||||
REMOTE_SCRIPT = r'''
|
REMOTE_SCRIPT = r'''
|
||||||
import asyncio
|
import asyncio
|
||||||
|
import copy
|
||||||
|
import hashlib
|
||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
import shutil
|
import shutil
|
||||||
|
|
@ -50,6 +73,7 @@ import subprocess
|
||||||
import sys
|
import sys
|
||||||
import tempfile
|
import tempfile
|
||||||
import traceback
|
import traceback
|
||||||
|
import types
|
||||||
from datetime import datetime, timezone
|
from datetime import datetime, timezone
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
|
|
@ -63,6 +87,114 @@ RUN_ID = "__RUN_ID__"
|
||||||
PROMPTS = __PROMPTS_JSON__
|
PROMPTS = __PROMPTS_JSON__
|
||||||
REPORT_PREFIX = __REPORT_PREFIX_JSON__
|
REPORT_PREFIX = __REPORT_PREFIX_JSON__
|
||||||
REPORT_PATH = Path(f"/tmp/{REPORT_PREFIX}-{RUN_ID}.json")
|
REPORT_PATH = Path(f"/tmp/{REPORT_PREFIX}-{RUN_ID}.json")
|
||||||
|
DB_CONTEXT_PLUGIN_SOURCE = __DB_CONTEXT_PLUGIN_SOURCE_JSON__
|
||||||
|
KB_TOOL_SOURCE = __KB_TOOL_SOURCE_JSON__
|
||||||
|
BEHAVIOR_MANIFEST_SOURCE = __BEHAVIOR_MANIFEST_SOURCE_JSON__
|
||||||
|
TURN_TRACE_PLUGIN_SOURCE = """\
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
|
||||||
|
def _safe_usage(value):
|
||||||
|
if not isinstance(value, dict):
|
||||||
|
return {}
|
||||||
|
safe = {}
|
||||||
|
for key in (
|
||||||
|
"input_tokens",
|
||||||
|
"output_tokens",
|
||||||
|
"prompt_tokens",
|
||||||
|
"completion_tokens",
|
||||||
|
"total_tokens",
|
||||||
|
"cache_read_tokens",
|
||||||
|
"cache_write_tokens",
|
||||||
|
"reasoning_tokens",
|
||||||
|
):
|
||||||
|
item = value.get(key)
|
||||||
|
if isinstance(item, (int, float)) and not isinstance(item, bool):
|
||||||
|
safe[key] = item
|
||||||
|
return safe
|
||||||
|
|
||||||
|
|
||||||
|
def _sha(value):
|
||||||
|
return hashlib.sha256(str(value or "").encode("utf-8")).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def _write(record):
|
||||||
|
raw_path = os.getenv("LEO_TURN_API_TRACE_PATH", "").strip()
|
||||||
|
if not raw_path:
|
||||||
|
return
|
||||||
|
path = Path(raw_path)
|
||||||
|
path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
with path.open("a", encoding="utf-8") as handle:
|
||||||
|
handle.write(json.dumps(record, sort_keys=True) + "\\n")
|
||||||
|
try:
|
||||||
|
path.chmod(0o600)
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def _pre_llm_call(**kwargs):
|
||||||
|
_write({
|
||||||
|
"generated_at_utc": datetime.now(timezone.utc).isoformat(),
|
||||||
|
"event": "turn_pre_llm_call",
|
||||||
|
"session_id_sha256": _sha(kwargs.get("session_id")),
|
||||||
|
"prompt_sha256": _sha(kwargs.get("user_message")),
|
||||||
|
})
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _post_api_request(**kwargs):
|
||||||
|
record = {
|
||||||
|
"generated_at_utc": datetime.now(timezone.utc).isoformat(),
|
||||||
|
"event": "post_api_request",
|
||||||
|
"task_id_sha256": _sha(kwargs.get("task_id")),
|
||||||
|
"session_id_sha256": _sha(kwargs.get("session_id")),
|
||||||
|
"model": kwargs.get("model"),
|
||||||
|
"provider": kwargs.get("provider"),
|
||||||
|
"base_url_sha256": _sha(kwargs.get("base_url")),
|
||||||
|
"api_mode": kwargs.get("api_mode"),
|
||||||
|
"api_call_count": kwargs.get("api_call_count"),
|
||||||
|
"finish_reason": kwargs.get("finish_reason"),
|
||||||
|
"message_count": kwargs.get("message_count"),
|
||||||
|
"response_model": kwargs.get("response_model"),
|
||||||
|
"usage": _safe_usage(kwargs.get("usage")),
|
||||||
|
"assistant_content_chars": kwargs.get("assistant_content_chars"),
|
||||||
|
"assistant_tool_call_count": kwargs.get("assistant_tool_call_count"),
|
||||||
|
"provider_response_id": None,
|
||||||
|
"provider_response_id_status": "not_exposed_by_hermes_post_api_request_hook",
|
||||||
|
}
|
||||||
|
_write(record)
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _post_llm_call(**kwargs):
|
||||||
|
_write({
|
||||||
|
"generated_at_utc": datetime.now(timezone.utc).isoformat(),
|
||||||
|
"event": "turn_post_llm_call",
|
||||||
|
"session_id_sha256": _sha(kwargs.get("session_id")),
|
||||||
|
"prompt_sha256": _sha(kwargs.get("user_message")),
|
||||||
|
"raw_assistant_response_sha256": _sha(kwargs.get("assistant_response")),
|
||||||
|
})
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def register(ctx):
|
||||||
|
ctx.register_hook("pre_llm_call", _pre_llm_call)
|
||||||
|
ctx.register_hook("post_api_request", _post_api_request)
|
||||||
|
ctx.register_hook("post_llm_call", _post_llm_call)
|
||||||
|
"""
|
||||||
|
TURN_TRACE_PLUGIN_MANIFEST = """\
|
||||||
|
name: leo-turn-execution-trace
|
||||||
|
version: "1.0.0"
|
||||||
|
description: Secret-safe model-call trace for no-send Leo harnesses.
|
||||||
|
provides_hooks:
|
||||||
|
- pre_llm_call
|
||||||
|
- post_api_request
|
||||||
|
- post_llm_call
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
def service_state():
|
def service_state():
|
||||||
|
|
@ -114,6 +246,127 @@ select jsonb_build_object(
|
||||||
return {"error": str(exc)}
|
return {"error": str(exc)}
|
||||||
|
|
||||||
|
|
||||||
|
def db_read_marker():
|
||||||
|
sql = """
|
||||||
|
select jsonb_build_object(
|
||||||
|
'database', current_database(),
|
||||||
|
'database_user', current_user,
|
||||||
|
'system_identifier', (select system_identifier::text from pg_control_system()),
|
||||||
|
'wal_lsn', pg_current_wal_lsn()::text
|
||||||
|
)::text;
|
||||||
|
"""
|
||||||
|
raw = subprocess.check_output(
|
||||||
|
["docker", "exec", "-i", "teleo-pg", "psql", "-U", "postgres", "-d", "teleo", "-At", "-q"],
|
||||||
|
input=sql,
|
||||||
|
text=True,
|
||||||
|
stderr=subprocess.STDOUT,
|
||||||
|
)
|
||||||
|
return json.loads(raw.strip())
|
||||||
|
|
||||||
|
|
||||||
|
def db_fingerprint():
|
||||||
|
sql_path = DEPLOY_ROOT / "ops" / "postgres_parity_manifest.sql"
|
||||||
|
before = db_read_marker()
|
||||||
|
raw = subprocess.check_output(
|
||||||
|
["docker", "exec", "-i", "teleo-pg", "psql", "-U", "postgres", "-d", "teleo", "-At", "-q"],
|
||||||
|
input=sql_path.read_text(encoding="utf-8"),
|
||||||
|
text=True,
|
||||||
|
stderr=subprocess.STDOUT,
|
||||||
|
timeout=180,
|
||||||
|
)
|
||||||
|
after = db_read_marker()
|
||||||
|
rows = []
|
||||||
|
for line in raw.splitlines():
|
||||||
|
candidate = line.strip()
|
||||||
|
if not candidate or candidate in {"BEGIN", "SET", "ROLLBACK"}:
|
||||||
|
continue
|
||||||
|
value = json.loads(candidate)
|
||||||
|
if isinstance(value, dict):
|
||||||
|
rows.append(value)
|
||||||
|
tables = {
|
||||||
|
f"{row['schema']}.{row['table']}": {
|
||||||
|
"row_count": int(row["row_count"]),
|
||||||
|
"rowset_md5": row["rowset_md5"],
|
||||||
|
}
|
||||||
|
for row in rows
|
||||||
|
if row.get("kind") == "table"
|
||||||
|
}
|
||||||
|
singleton = {
|
||||||
|
str(row["kind"]): row.get("items", [])
|
||||||
|
for row in rows
|
||||||
|
if row.get("kind")
|
||||||
|
in {
|
||||||
|
"schemas",
|
||||||
|
"extensions",
|
||||||
|
"application_roles",
|
||||||
|
"columns",
|
||||||
|
"constraints",
|
||||||
|
"indexes",
|
||||||
|
"sequences",
|
||||||
|
"views",
|
||||||
|
"functions",
|
||||||
|
"triggers",
|
||||||
|
"types",
|
||||||
|
"policies",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
identity_rows = [row for row in rows if row.get("kind") == "identity"]
|
||||||
|
if len(identity_rows) != 1 or not tables:
|
||||||
|
raise RuntimeError("canonical database fingerprint manifest is incomplete")
|
||||||
|
identity = identity_rows[0]
|
||||||
|
stable = {
|
||||||
|
"identity": {
|
||||||
|
key: identity.get(key)
|
||||||
|
for key in (
|
||||||
|
"database",
|
||||||
|
"server_version_num",
|
||||||
|
"server_encoding",
|
||||||
|
"database_collation",
|
||||||
|
"database_ctype",
|
||||||
|
"transaction_read_only",
|
||||||
|
)
|
||||||
|
},
|
||||||
|
"singleton_sha256": {
|
||||||
|
key: hashlib.sha256(
|
||||||
|
json.dumps(value, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
||||||
|
).hexdigest()
|
||||||
|
for key, value in sorted(singleton.items())
|
||||||
|
},
|
||||||
|
"tables": tables,
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
"schema": "livingip.teleoCanonicalDatabaseFingerprint.v1",
|
||||||
|
"status": "ok",
|
||||||
|
"fingerprint_sha256": hashlib.sha256(
|
||||||
|
json.dumps(stable, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
||||||
|
).hexdigest(),
|
||||||
|
"table_count": len(tables),
|
||||||
|
"total_rows": sum(item["row_count"] for item in tables.values()),
|
||||||
|
"table_rows_sha256": hashlib.sha256(
|
||||||
|
json.dumps(tables, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
||||||
|
).hexdigest(),
|
||||||
|
"structure_sha256": hashlib.sha256(
|
||||||
|
json.dumps(stable["singleton_sha256"], sort_keys=True, separators=(",", ":")).encode("utf-8")
|
||||||
|
).hexdigest(),
|
||||||
|
"read_consistency": {
|
||||||
|
"status": "stable_wal_marker" if before == after else "wal_changed_during_fingerprint",
|
||||||
|
"before": before,
|
||||||
|
"after": after,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def safe_db_fingerprint():
|
||||||
|
try:
|
||||||
|
return db_fingerprint()
|
||||||
|
except Exception as exc:
|
||||||
|
return {
|
||||||
|
"schema": "livingip.teleoCanonicalDatabaseFingerprint.v1",
|
||||||
|
"status": "error",
|
||||||
|
"error": type(exc).__name__,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
def copy_profile() -> Path:
|
def copy_profile() -> Path:
|
||||||
tmp_root = Path(tempfile.mkdtemp(prefix="leo-direct-claim-handler-"))
|
tmp_root = Path(tempfile.mkdtemp(prefix="leo-direct-claim-handler-"))
|
||||||
target = tmp_root / "profile"
|
target = tmp_root / "profile"
|
||||||
|
|
@ -155,6 +408,36 @@ def copy_profile() -> Path:
|
||||||
return target
|
return target
|
||||||
|
|
||||||
|
|
||||||
|
def install_harness_plugins(profile):
|
||||||
|
db_context = profile / "plugins" / "leo-db-context" / "__init__.py"
|
||||||
|
db_context.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
db_context.write_text(DB_CONTEXT_PLUGIN_SOURCE, encoding="utf-8")
|
||||||
|
|
||||||
|
kb_tool = profile / "bin" / "kb_tool.py"
|
||||||
|
kb_tool.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
kb_tool.write_text(KB_TOOL_SOURCE, encoding="utf-8")
|
||||||
|
kb_tool.chmod(0o700)
|
||||||
|
|
||||||
|
turn_trace = profile / "plugins" / "leo-turn-execution-trace"
|
||||||
|
turn_trace.mkdir(parents=True, exist_ok=True)
|
||||||
|
(turn_trace / "__init__.py").write_text(TURN_TRACE_PLUGIN_SOURCE, encoding="utf-8")
|
||||||
|
(turn_trace / "plugin.yaml").write_text(TURN_TRACE_PLUGIN_MANIFEST, encoding="utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
def read_jsonl(path):
|
||||||
|
if not path or not path.exists():
|
||||||
|
return []
|
||||||
|
records = []
|
||||||
|
for line in path.read_text(encoding="utf-8").splitlines():
|
||||||
|
try:
|
||||||
|
value = json.loads(line)
|
||||||
|
except json.JSONDecodeError:
|
||||||
|
continue
|
||||||
|
if isinstance(value, dict):
|
||||||
|
records.append(value)
|
||||||
|
return records
|
||||||
|
|
||||||
|
|
||||||
def remove_tree(path):
|
def remove_tree(path):
|
||||||
if not path or not path.exists():
|
if not path or not path.exists():
|
||||||
return True
|
return True
|
||||||
|
|
@ -187,22 +470,13 @@ def remove_tree(path):
|
||||||
def write_report_snapshot(report):
|
def write_report_snapshot(report):
|
||||||
try:
|
try:
|
||||||
REPORT_PATH.write_text(json.dumps(report, sort_keys=True), encoding="utf-8")
|
REPORT_PATH.write_text(json.dumps(report, sort_keys=True), encoding="utf-8")
|
||||||
|
REPORT_PATH.chmod(0o600)
|
||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
||||||
def read_database_context_trace(path):
|
def read_database_context_trace(path):
|
||||||
if not path or not path.exists():
|
return read_jsonl(path)
|
||||||
return []
|
|
||||||
records = []
|
|
||||||
for line in path.read_text(encoding="utf-8").splitlines():
|
|
||||||
try:
|
|
||||||
value = json.loads(line)
|
|
||||||
except json.JSONDecodeError:
|
|
||||||
continue
|
|
||||||
if isinstance(value, dict):
|
|
||||||
records.append(value)
|
|
||||||
return records
|
|
||||||
|
|
||||||
|
|
||||||
def current_agent_messages(runner, session_key):
|
def current_agent_messages(runner, session_key):
|
||||||
|
|
@ -217,19 +491,83 @@ def current_agent_messages(runner, session_key):
|
||||||
return []
|
return []
|
||||||
agent = cached[0]
|
agent = cached[0]
|
||||||
messages = getattr(agent, "_session_messages", None) or []
|
messages = getattr(agent, "_session_messages", None) or []
|
||||||
return [dict(message) for message in messages if isinstance(message, dict)]
|
return [copy.deepcopy(message) for message in messages if isinstance(message, dict)]
|
||||||
|
|
||||||
|
|
||||||
|
def conversation_trace(messages):
|
||||||
|
rows = []
|
||||||
|
for message in messages:
|
||||||
|
content = message.get("content")
|
||||||
|
content_text = content if isinstance(content, str) else json.dumps(content, sort_keys=True, default=str)
|
||||||
|
tool_calls = message.get("tool_calls") or []
|
||||||
|
rows.append({
|
||||||
|
"role": message.get("role"),
|
||||||
|
"content_sha256": hashlib.sha256(content_text.encode("utf-8")).hexdigest(),
|
||||||
|
"content_bytes": len(content_text.encode("utf-8")),
|
||||||
|
"tool_calls_sha256": hashlib.sha256(
|
||||||
|
json.dumps(tool_calls, sort_keys=True, separators=(",", ":"), default=str).encode("utf-8")
|
||||||
|
).hexdigest(),
|
||||||
|
"tool_call_count": len(tool_calls),
|
||||||
|
})
|
||||||
|
return {
|
||||||
|
"message_count": len(rows),
|
||||||
|
"messages_sha256": hashlib.sha256(
|
||||||
|
json.dumps(rows, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
||||||
|
).hexdigest(),
|
||||||
|
"last_message_role": rows[-1]["role"] if rows else None,
|
||||||
|
"last_message_content_sha256": rows[-1]["content_sha256"] if rows else None,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def model_response_trace(messages):
|
||||||
|
return [
|
||||||
|
{
|
||||||
|
"content_sha256": row["content_sha256"],
|
||||||
|
"content_bytes": row["content_bytes"],
|
||||||
|
"tool_calls_sha256": row["tool_calls_sha256"],
|
||||||
|
"tool_call_count": row["tool_call_count"],
|
||||||
|
}
|
||||||
|
for message, row in zip(messages, conversation_trace_rows(messages))
|
||||||
|
if message.get("role") == "assistant"
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def conversation_trace_rows(messages):
|
||||||
|
rows = []
|
||||||
|
for message in messages:
|
||||||
|
content = message.get("content")
|
||||||
|
content_text = content if isinstance(content, str) else json.dumps(content, sort_keys=True, default=str)
|
||||||
|
tool_calls = message.get("tool_calls") or []
|
||||||
|
rows.append({
|
||||||
|
"role": message.get("role"),
|
||||||
|
"content_sha256": hashlib.sha256(content_text.encode("utf-8")).hexdigest(),
|
||||||
|
"content_bytes": len(content_text.encode("utf-8")),
|
||||||
|
"tool_calls_sha256": hashlib.sha256(
|
||||||
|
json.dumps(tool_calls, sort_keys=True, separators=(",", ":"), default=str).encode("utf-8")
|
||||||
|
).hexdigest(),
|
||||||
|
"tool_call_count": len(tool_calls),
|
||||||
|
})
|
||||||
|
return rows
|
||||||
|
|
||||||
|
|
||||||
async def run_suite():
|
async def run_suite():
|
||||||
sys.path.insert(0, str(DEPLOY_ROOT / "scripts"))
|
sys.path.insert(0, str(DEPLOY_ROOT / "scripts"))
|
||||||
from leo_behavior_manifest import build_manifest as build_behavior_manifest
|
|
||||||
from leo_tool_trace import extract_kb_tool_trace
|
from leo_tool_trace import extract_kb_tool_trace
|
||||||
|
|
||||||
|
behavior_manifest_module = types.ModuleType("leo_behavior_manifest_harness")
|
||||||
|
behavior_manifest_module.__file__ = "<embedded leo_behavior_manifest.py>"
|
||||||
|
exec(compile(BEHAVIOR_MANIFEST_SOURCE, behavior_manifest_module.__file__, "exec"), behavior_manifest_module.__dict__)
|
||||||
|
|
||||||
|
def build_behavior_manifest(profile):
|
||||||
|
return behavior_manifest_module.build_manifest(profile, AGENT_ROOT, DEPLOY_ROOT)
|
||||||
|
|
||||||
before_service = service_state()
|
before_service = service_state()
|
||||||
before_counts = db_counts()
|
before_counts = db_counts()
|
||||||
behavior_before = build_behavior_manifest(LIVE_PROFILE, AGENT_ROOT)
|
before_fingerprint = safe_db_fingerprint()
|
||||||
|
behavior_before = build_behavior_manifest(LIVE_PROFILE)
|
||||||
temp_profile = None
|
temp_profile = None
|
||||||
context_trace_path = None
|
context_trace_path = None
|
||||||
|
model_trace_path = None
|
||||||
temp_removed = False
|
temp_removed = False
|
||||||
report = {
|
report = {
|
||||||
"generated_at_utc": datetime.now(timezone.utc).isoformat(),
|
"generated_at_utc": datetime.now(timezone.utc).isoformat(),
|
||||||
|
|
@ -264,16 +602,21 @@ async def run_suite():
|
||||||
"This proves handler-level live VPS GatewayRunner behavior, not human-visible Telegram delivery.",
|
"This proves handler-level live VPS GatewayRunner behavior, not human-visible Telegram delivery.",
|
||||||
],
|
],
|
||||||
"db_counts_before": before_counts,
|
"db_counts_before": before_counts,
|
||||||
|
"db_fingerprint_before": before_fingerprint,
|
||||||
"before_service": before_service,
|
"before_service": before_service,
|
||||||
"remote_report_path": str(REPORT_PATH),
|
"remote_report_path": str(REPORT_PATH),
|
||||||
}
|
}
|
||||||
write_report_snapshot(report)
|
write_report_snapshot(report)
|
||||||
try:
|
try:
|
||||||
temp_profile = copy_profile()
|
temp_profile = copy_profile()
|
||||||
|
install_harness_plugins(temp_profile)
|
||||||
os.environ["HERMES_HOME"] = str(temp_profile)
|
os.environ["HERMES_HOME"] = str(temp_profile)
|
||||||
os.environ["HOME"] = "/home/teleo"
|
os.environ["HOME"] = "/home/teleo"
|
||||||
context_trace_path = temp_profile / "state" / "leo-db-context-trace.jsonl"
|
context_trace_path = temp_profile / "state" / "leo-db-context-trace.jsonl"
|
||||||
|
model_trace_path = temp_profile / "state" / "leo-turn-api-trace.jsonl"
|
||||||
os.environ["LEO_DB_CONTEXT_TRACE_PATH"] = str(context_trace_path)
|
os.environ["LEO_DB_CONTEXT_TRACE_PATH"] = str(context_trace_path)
|
||||||
|
os.environ["LEO_TURN_API_TRACE_PATH"] = str(model_trace_path)
|
||||||
|
report["executed_behavior_manifest"] = build_behavior_manifest(temp_profile)
|
||||||
sys.path.insert(0, str(AGENT_ROOT))
|
sys.path.insert(0, str(AGENT_ROOT))
|
||||||
|
|
||||||
from gateway.config import Platform
|
from gateway.config import Platform
|
||||||
|
|
@ -308,6 +651,8 @@ async def run_suite():
|
||||||
results = []
|
results = []
|
||||||
for index, prompt in enumerate(PROMPTS, start=1):
|
for index, prompt in enumerate(PROMPTS, start=1):
|
||||||
trace_start = len(read_database_context_trace(context_trace_path))
|
trace_start = len(read_database_context_trace(context_trace_path))
|
||||||
|
model_trace_start = len(read_jsonl(model_trace_path))
|
||||||
|
messages_before = current_agent_messages(runner, session_key)
|
||||||
event = MessageEvent(
|
event = MessageEvent(
|
||||||
text=prompt["message"],
|
text=prompt["message"],
|
||||||
message_type=MessageType.TEXT,
|
message_type=MessageType.TEXT,
|
||||||
|
|
@ -317,7 +662,12 @@ async def run_suite():
|
||||||
started = datetime.now(timezone.utc)
|
started = datetime.now(timezone.utc)
|
||||||
reply = await asyncio.wait_for(runner._handle_message(event), timeout=300)
|
reply = await asyncio.wait_for(runner._handle_message(event), timeout=300)
|
||||||
ended = datetime.now(timezone.utc)
|
ended = datetime.now(timezone.utc)
|
||||||
database_tool_trace = extract_kb_tool_trace(current_agent_messages(runner, session_key))
|
messages_after = current_agent_messages(runner, session_key)
|
||||||
|
before_rows = conversation_trace_rows(messages_before)
|
||||||
|
after_rows = conversation_trace_rows(messages_after)
|
||||||
|
history_prefix_preserved = after_rows[: len(before_rows)] == before_rows
|
||||||
|
new_messages = messages_after[len(messages_before) :] if history_prefix_preserved else messages_after
|
||||||
|
database_tool_trace = extract_kb_tool_trace(new_messages)
|
||||||
results.append(
|
results.append(
|
||||||
{
|
{
|
||||||
"turn": index,
|
"turn": index,
|
||||||
|
|
@ -333,6 +683,11 @@ async def run_suite():
|
||||||
"claim_ceiling": "Live VPS GatewayRunner reply from temp leoclean profile; no Telegram post; no production DB apply authorized.",
|
"claim_ceiling": "Live VPS GatewayRunner reply from temp leoclean profile; no Telegram post; no production DB apply authorized.",
|
||||||
"database_context_trace": read_database_context_trace(context_trace_path)[trace_start:],
|
"database_context_trace": read_database_context_trace(context_trace_path)[trace_start:],
|
||||||
"database_tool_trace": database_tool_trace,
|
"database_tool_trace": database_tool_trace,
|
||||||
|
"model_call_trace": read_jsonl(model_trace_path)[model_trace_start:],
|
||||||
|
"model_response_trace": model_response_trace(new_messages),
|
||||||
|
"conversation_before": conversation_trace(messages_before),
|
||||||
|
"conversation_after": conversation_trace(messages_after),
|
||||||
|
"conversation_history_prefix_preserved": history_prefix_preserved,
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
report["results"] = results
|
report["results"] = results
|
||||||
|
|
@ -348,10 +703,18 @@ async def run_suite():
|
||||||
return report
|
return report
|
||||||
finally:
|
finally:
|
||||||
after_counts = db_counts()
|
after_counts = db_counts()
|
||||||
|
after_fingerprint = safe_db_fingerprint()
|
||||||
after_service = service_state()
|
after_service = service_state()
|
||||||
behavior_after = build_behavior_manifest(LIVE_PROFILE, AGENT_ROOT)
|
behavior_after = build_behavior_manifest(LIVE_PROFILE)
|
||||||
report["db_counts_after"] = after_counts
|
report["db_counts_after"] = after_counts
|
||||||
report["db_counts_changed"] = after_counts != before_counts
|
report["db_counts_changed"] = after_counts != before_counts
|
||||||
|
report["db_fingerprint_after"] = after_fingerprint
|
||||||
|
report["db_fingerprint_unchanged"] = bool(
|
||||||
|
before_fingerprint.get("status") == "ok"
|
||||||
|
and after_fingerprint.get("status") == "ok"
|
||||||
|
and before_fingerprint.get("fingerprint_sha256")
|
||||||
|
== after_fingerprint.get("fingerprint_sha256")
|
||||||
|
)
|
||||||
report["live_behavior_manifest_after"] = behavior_after
|
report["live_behavior_manifest_after"] = behavior_after
|
||||||
report["changed_live_profile"] = behavior_before["behavior_sha256"] != behavior_after["behavior_sha256"]
|
report["changed_live_profile"] = behavior_before["behavior_sha256"] != behavior_after["behavior_sha256"]
|
||||||
report["live_behavior_manifest_unchanged"] = not report["changed_live_profile"]
|
report["live_behavior_manifest_unchanged"] = not report["changed_live_profile"]
|
||||||
|
|
@ -361,11 +724,21 @@ async def run_suite():
|
||||||
"unchanged_from_preexisting_live_readback": before_service == after_service,
|
"unchanged_from_preexisting_live_readback": before_service == after_service,
|
||||||
}
|
}
|
||||||
database_context_trace = read_database_context_trace(context_trace_path)
|
database_context_trace = read_database_context_trace(context_trace_path)
|
||||||
|
model_call_trace = read_jsonl(model_trace_path)
|
||||||
|
api_model_trace = [item for item in model_call_trace if item.get("event") == "post_api_request"]
|
||||||
if temp_profile is not None:
|
if temp_profile is not None:
|
||||||
tmp_root = temp_profile.parent
|
tmp_root = temp_profile.parent
|
||||||
temp_removed = remove_tree(tmp_root)
|
temp_removed = remove_tree(tmp_root)
|
||||||
report["temp_profile_removed"] = temp_removed
|
report["temp_profile_removed"] = temp_removed
|
||||||
report["database_context_trace"] = database_context_trace
|
report["database_context_trace"] = database_context_trace
|
||||||
|
report["model_call_trace_count"] = len(api_model_trace)
|
||||||
|
report["model_call_trace_all_bound"] = bool(api_model_trace) and all(
|
||||||
|
item.get("event") == "post_api_request"
|
||||||
|
and item.get("model")
|
||||||
|
and item.get("provider")
|
||||||
|
and item.get("response_model")
|
||||||
|
for item in api_model_trace
|
||||||
|
)
|
||||||
context_injections = [
|
context_injections = [
|
||||||
item for item in database_context_trace if item.get("event", "pre_llm_call") == "pre_llm_call"
|
item for item in database_context_trace if item.get("event", "pre_llm_call") == "pre_llm_call"
|
||||||
]
|
]
|
||||||
|
|
@ -443,16 +816,17 @@ def build_remote_script(
|
||||||
.replace("__SUITE_MODE_JSON__", json.dumps(suite_mode))
|
.replace("__SUITE_MODE_JSON__", json.dumps(suite_mode))
|
||||||
.replace("__REPORT_PREFIX_JSON__", json.dumps(report_prefix))
|
.replace("__REPORT_PREFIX_JSON__", json.dumps(report_prefix))
|
||||||
.replace("__PROMPT_NOTE_JSON__", json.dumps(prompt_note))
|
.replace("__PROMPT_NOTE_JSON__", json.dumps(prompt_note))
|
||||||
|
.replace("__DB_CONTEXT_PLUGIN_SOURCE_JSON__", json.dumps(DB_CONTEXT_PLUGIN.read_text(encoding="utf-8")))
|
||||||
|
.replace("__KB_TOOL_SOURCE_JSON__", json.dumps(KB_TOOL.read_text(encoding="utf-8")))
|
||||||
|
.replace("__BEHAVIOR_MANIFEST_SOURCE_JSON__", json.dumps(BEHAVIOR_MANIFEST.read_text(encoding="utf-8")))
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def redact(text: str) -> str:
|
def redact(text: str) -> str:
|
||||||
result = text
|
result = JSON_SECRET_PATTERN.sub(lambda match: f'{match.group(1)}"[REDACTED]"', text)
|
||||||
for pattern in SECRET_PATTERNS:
|
result = AUTHORIZATION_PATTERN.sub(lambda match: f"{match.group(1)}[REDACTED]", result)
|
||||||
if pattern.groups >= 2:
|
result = ASSIGNMENT_SECRET_PATTERN.sub(lambda match: f"{match.group(1)}[REDACTED]", result)
|
||||||
result = pattern.sub(r"\1[REDACTED]", result)
|
result = TELEGRAM_TOKEN_PATTERN.sub("[REDACTED_TOKEN]", result)
|
||||||
else:
|
|
||||||
result = pattern.sub("[REDACTED_TOKEN]", result)
|
|
||||||
return result
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -536,7 +910,7 @@ def run_remote(
|
||||||
if not candidate.startswith("{"):
|
if not candidate.startswith("{"):
|
||||||
continue
|
continue
|
||||||
try:
|
try:
|
||||||
result["parsed"] = json.loads(candidate)
|
result["parsed"] = json.loads(redact(candidate))
|
||||||
break
|
break
|
||||||
except json.JSONDecodeError:
|
except json.JSONDecodeError:
|
||||||
continue
|
continue
|
||||||
|
|
@ -552,8 +926,83 @@ def run_remote(
|
||||||
return result
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def _tool_trace_is_safe(trace: Any) -> bool:
|
||||||
|
if not isinstance(trace, dict):
|
||||||
|
return False
|
||||||
|
count = trace.get("database_tool_call_count")
|
||||||
|
completed = trace.get("database_tool_completed_count")
|
||||||
|
calls = trace.get("calls") if isinstance(trace.get("calls"), list) else []
|
||||||
|
if not isinstance(count, int) or isinstance(count, bool) or count < 0:
|
||||||
|
return False
|
||||||
|
if not isinstance(completed, int) or isinstance(completed, bool) or completed != count:
|
||||||
|
return False
|
||||||
|
if len(calls) != count:
|
||||||
|
return False
|
||||||
|
if count and trace.get("database_tool_calls_read_only") is not True:
|
||||||
|
return False
|
||||||
|
for call in calls:
|
||||||
|
if not isinstance(call, dict):
|
||||||
|
return False
|
||||||
|
invocations = call.get("database_invocations")
|
||||||
|
result = call.get("result")
|
||||||
|
if not isinstance(invocations, list) or not invocations:
|
||||||
|
return False
|
||||||
|
if not all(isinstance(item, dict) and item.get("access_mode") == "read_only" for item in invocations):
|
||||||
|
return False
|
||||||
|
if (
|
||||||
|
not isinstance(result, dict)
|
||||||
|
or result.get("error_detected") is not False
|
||||||
|
or result.get("nonempty") is not True
|
||||||
|
or not re.fullmatch(r"[0-9a-f]{64}", str(result.get("content_sha256") or ""))
|
||||||
|
):
|
||||||
|
return False
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def build_safety_gate(report: dict[str, Any]) -> dict[str, Any]:
|
||||||
|
results = [item for item in report.get("results") or [] if isinstance(item, dict)]
|
||||||
|
handler = report.get("handler") if isinstance(report.get("handler"), dict) else {}
|
||||||
|
service = report.get("service_before_after") if isinstance(report.get("service_before_after"), dict) else {}
|
||||||
|
execution_summary = (
|
||||||
|
report.get("execution_manifest_summary")
|
||||||
|
if isinstance(report.get("execution_manifest_summary"), dict)
|
||||||
|
else {}
|
||||||
|
)
|
||||||
|
checks = {
|
||||||
|
"remote_command_succeeded": report.get("remote_returncode") == 0,
|
||||||
|
"runtime_completed": report.get("pass_runtime") is True,
|
||||||
|
"handler_authorized": handler.get("authorized") is True,
|
||||||
|
"results_nonempty": bool(results),
|
||||||
|
"all_turns_returned_replies": bool(results) and all(item.get("ok") is True for item in results),
|
||||||
|
"all_turns_declared_no_mutation": bool(results)
|
||||||
|
and all(item.get("mutates_kb") is False for item in results),
|
||||||
|
"all_tool_traces_read_only_and_bound": bool(results)
|
||||||
|
and all(_tool_trace_is_safe(item.get("database_tool_trace")) for item in results),
|
||||||
|
"no_telegram_post": report.get("posted_to_telegram") is False,
|
||||||
|
"harness_declared_no_kb_mutation": report.get("mutates_kb_by_harness") is False,
|
||||||
|
"database_counts_unchanged": report.get("db_counts_changed") is False,
|
||||||
|
"database_fingerprint_unchanged": report.get("db_fingerprint_unchanged") is True,
|
||||||
|
"live_behavior_unchanged": report.get("live_behavior_manifest_unchanged") is True,
|
||||||
|
"service_unchanged": service.get("unchanged_from_preexisting_live_readback") is True,
|
||||||
|
"temporary_profile_removed": report.get("temp_profile_removed") is True,
|
||||||
|
"model_trace_metadata_bound": report.get("model_call_trace_all_bound") is True,
|
||||||
|
"all_turn_manifests_complete": execution_summary.get("all_turns_attribution_complete") is True,
|
||||||
|
"no_runtime_error": not report.get("error") and not report.get("failure"),
|
||||||
|
}
|
||||||
|
failed = [name for name, passed in checks.items() if not passed]
|
||||||
|
return {
|
||||||
|
"status": "pass" if not failed else "fail",
|
||||||
|
"checks": checks,
|
||||||
|
"failed_checks": failed,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def report_passes_safety(report: dict[str, Any]) -> bool:
|
||||||
|
return build_safety_gate(report)["status"] == "pass"
|
||||||
|
|
||||||
|
|
||||||
def write_output(remote: dict[str, Any], *, output_json: Path = OUTPUT_JSON) -> dict[str, Any]:
|
def write_output(remote: dict[str, Any], *, output_json: Path = OUTPUT_JSON) -> dict[str, Any]:
|
||||||
REPORT_DIR.mkdir(parents=True, exist_ok=True)
|
output_json.parent.mkdir(parents=True, exist_ok=True)
|
||||||
parsed = remote.get("parsed") or {}
|
parsed = remote.get("parsed") or {}
|
||||||
report = parsed if isinstance(parsed, dict) else {}
|
report = parsed if isinstance(parsed, dict) else {}
|
||||||
report["source_report_path"] = str(output_json)
|
report["source_report_path"] = str(output_json)
|
||||||
|
|
@ -561,6 +1010,8 @@ def write_output(remote: dict[str, Any], *, output_json: Path = OUTPUT_JSON) ->
|
||||||
report["remote_run_id"] = remote.get("run_id")
|
report["remote_run_id"] = remote.get("run_id")
|
||||||
if remote.get("stderr"):
|
if remote.get("stderr"):
|
||||||
report["remote_stderr"] = remote["stderr"]
|
report["remote_stderr"] = remote["stderr"]
|
||||||
|
attach_execution_manifests(report, repo_root=ROOT)
|
||||||
|
report["safety_gate"] = build_safety_gate(report)
|
||||||
output_json.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
output_json.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||||
return report
|
return report
|
||||||
|
|
||||||
|
|
@ -568,8 +1019,17 @@ def write_output(remote: dict[str, Any], *, output_json: Path = OUTPUT_JSON) ->
|
||||||
def main() -> int:
|
def main() -> int:
|
||||||
remote = run_remote()
|
remote = run_remote()
|
||||||
report = write_output(remote)
|
report = write_output(remote)
|
||||||
print(json.dumps({"json": str(OUTPUT_JSON), "pass_runtime": report.get("pass_runtime")}, indent=2))
|
print(
|
||||||
return 0 if remote.get("returncode") == 0 and report.get("pass_runtime") else 1
|
json.dumps(
|
||||||
|
{
|
||||||
|
"json": str(OUTPUT_JSON),
|
||||||
|
"pass_runtime": report.get("pass_runtime"),
|
||||||
|
"safety_gate": report.get("safety_gate"),
|
||||||
|
},
|
||||||
|
indent=2,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return 0 if report_passes_safety(report) else 1
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
|
|
|
||||||
|
|
@ -133,6 +133,16 @@ def verify_report(
|
||||||
report.get("db_counts_changed") is False
|
report.get("db_counts_changed") is False
|
||||||
and report.get("db_counts_before") == report.get("db_counts_after")
|
and report.get("db_counts_before") == report.get("db_counts_after")
|
||||||
),
|
),
|
||||||
|
"canonical_content_fingerprint_unchanged": (
|
||||||
|
report.get("db_fingerprint_unchanged") is True
|
||||||
|
and (report.get("db_fingerprint_before") or {}).get("fingerprint_sha256")
|
||||||
|
== (report.get("db_fingerprint_after") or {}).get("fingerprint_sha256")
|
||||||
|
and bool((report.get("db_fingerprint_before") or {}).get("fingerprint_sha256"))
|
||||||
|
),
|
||||||
|
"execution_receipt_complete": (
|
||||||
|
(report.get("execution_manifest_summary") or {}).get("all_turns_attribution_complete") is True
|
||||||
|
and (report.get("safety_gate") or {}).get("status") == "pass"
|
||||||
|
),
|
||||||
"live_behavior_profile_unchanged": (
|
"live_behavior_profile_unchanged": (
|
||||||
report.get("changed_live_profile") is False
|
report.get("changed_live_profile") is False
|
||||||
and report.get("live_behavior_manifest_unchanged") is True
|
and report.get("live_behavior_manifest_unchanged") is True
|
||||||
|
|
@ -165,7 +175,10 @@ def verify_report(
|
||||||
"did_not_turn_the_test_conversation_into_hidden_training": (
|
"did_not_turn_the_test_conversation_into_hidden_training": (
|
||||||
checks["live_behavior_profile_unchanged"] and checks["temporary_profile_removed"]
|
checks["live_behavior_profile_unchanged"] and checks["temporary_profile_removed"]
|
||||||
),
|
),
|
||||||
"did_not_change_canonical_knowledge": checks["canonical_counts_unchanged"],
|
"did_not_change_canonical_knowledge": (
|
||||||
|
checks["canonical_counts_unchanged"]
|
||||||
|
and checks["canonical_content_fingerprint_unchanged"]
|
||||||
|
),
|
||||||
}
|
}
|
||||||
passed = all(checks.values()) and all(outcomes.values())
|
passed = all(checks.values()) and all(outcomes.values())
|
||||||
return {
|
return {
|
||||||
|
|
@ -194,6 +207,8 @@ def verify_report(
|
||||||
"state_readback": {
|
"state_readback": {
|
||||||
"database_counts_before": report.get("db_counts_before"),
|
"database_counts_before": report.get("db_counts_before"),
|
||||||
"database_counts_after": report.get("db_counts_after"),
|
"database_counts_after": report.get("db_counts_after"),
|
||||||
|
"database_fingerprint_before": report.get("db_fingerprint_before"),
|
||||||
|
"database_fingerprint_after": report.get("db_fingerprint_after"),
|
||||||
"behavior_sha256_before": behavior_before,
|
"behavior_sha256_before": behavior_before,
|
||||||
"behavior_sha256_after": behavior_after,
|
"behavior_sha256_after": behavior_after,
|
||||||
"service_before": service_before,
|
"service_before": service_before,
|
||||||
|
|
|
||||||
239
scripts/verify_leo_unseen_reasoning_chain.py
Normal file
239
scripts/verify_leo_unseen_reasoning_chain.py
Normal file
|
|
@ -0,0 +1,239 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Verify one unseen challenge-and-revision chain through live VPS Leo."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import re
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
SCHEMA = "livingip.leoUnseenReasoningChainReceipt.v1"
|
||||||
|
UUID_RE = re.compile(
|
||||||
|
r"\b[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}\b",
|
||||||
|
re.IGNORECASE,
|
||||||
|
)
|
||||||
|
PROMPTS = (
|
||||||
|
{
|
||||||
|
"id": "OOS-CHAIN-01",
|
||||||
|
"dimension": "independent_weak_claim_selection",
|
||||||
|
"message": (
|
||||||
|
"Looking at what Leo currently believes about collective intelligence or coordination, pick one claim "
|
||||||
|
"that would matter to a founder making a product decision but seems more confident than its support. "
|
||||||
|
"Do not ask me for an ID. Show what the database actually says and what supports it, then tell me the "
|
||||||
|
"first narrower claim you would test. Do not change anything yet."
|
||||||
|
),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "OOS-CHAIN-02",
|
||||||
|
"dimension": "challenge_body_evidence_and_candidates",
|
||||||
|
"message": (
|
||||||
|
"That still feels like you are polishing the same idea. Inspect the exact body and evidence you just "
|
||||||
|
"chose: what is assumption versus observed support, what would falsify it, and which replacement claim "
|
||||||
|
"or claims would you put into review? Keep them as proposals, not live knowledge."
|
||||||
|
),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "OOS-CHAIN-03",
|
||||||
|
"dimension": "user_objection_revision_and_apply_boundary",
|
||||||
|
"message": (
|
||||||
|
"I disagree with your first replacement because it jumps from a mechanism to a business outcome. "
|
||||||
|
"Revise it with that objection, tell me what source would resolve the disagreement, and show the "
|
||||||
|
"approval and apply boundary before it could become part of Leo."
|
||||||
|
),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _sha256_file(path: Path) -> str:
|
||||||
|
digest = hashlib.sha256()
|
||||||
|
with path.open("rb") as handle:
|
||||||
|
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||||
|
digest.update(chunk)
|
||||||
|
return digest.hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def _contains(value: str, *terms: str) -> bool:
|
||||||
|
lowered = value.casefold()
|
||||||
|
return all(term.casefold() in lowered for term in terms)
|
||||||
|
|
||||||
|
|
||||||
|
def _tool_facts(result: dict[str, Any]) -> tuple[set[str], set[str]]:
|
||||||
|
subcommands: set[str] = set()
|
||||||
|
row_ids: set[str] = set()
|
||||||
|
trace = result.get("database_tool_trace") if isinstance(result.get("database_tool_trace"), dict) else {}
|
||||||
|
for call in trace.get("calls") or []:
|
||||||
|
if not isinstance(call, dict):
|
||||||
|
continue
|
||||||
|
for invocation in call.get("database_invocations") or []:
|
||||||
|
if isinstance(invocation, dict) and invocation.get("subcommand"):
|
||||||
|
subcommands.add(str(invocation["subcommand"]))
|
||||||
|
call_result = call.get("result") if isinstance(call.get("result"), dict) else {}
|
||||||
|
row_ids.update(str(item) for item in call_result.get("row_ids") or [])
|
||||||
|
return subcommands, row_ids
|
||||||
|
|
||||||
|
|
||||||
|
def _execution_chain(results: list[dict[str, Any]]) -> tuple[bool, list[str]]:
|
||||||
|
hashes: list[str] = []
|
||||||
|
previous: str | None = None
|
||||||
|
valid = True
|
||||||
|
for index, result in enumerate(results):
|
||||||
|
execution = result.get("execution_manifest") if isinstance(result.get("execution_manifest"), dict) else {}
|
||||||
|
attribution = execution.get("attribution") if isinstance(execution.get("attribution"), dict) else {}
|
||||||
|
session = execution.get("session_boundary") if isinstance(execution.get("session_boundary"), dict) else {}
|
||||||
|
conversation = session.get("conversation") if isinstance(session.get("conversation"), dict) else {}
|
||||||
|
execution_sha256 = execution.get("execution_sha256")
|
||||||
|
valid = bool(
|
||||||
|
valid
|
||||||
|
and attribution.get("status") == "complete"
|
||||||
|
and isinstance(execution_sha256, str)
|
||||||
|
and len(execution_sha256) == 64
|
||||||
|
and conversation.get("previous_execution_sha256") == previous
|
||||||
|
and conversation.get("prior_turn_state_bound") is True
|
||||||
|
and conversation.get("history_prefix_preserved") is True
|
||||||
|
and (conversation.get("starts_from_empty_conversation") is (index == 0))
|
||||||
|
)
|
||||||
|
hashes.append(str(execution_sha256 or ""))
|
||||||
|
previous = str(execution_sha256 or "")
|
||||||
|
return valid, hashes
|
||||||
|
|
||||||
|
|
||||||
|
def verify_report(report: dict[str, Any], *, source_report_sha256: str) -> dict[str, Any]:
|
||||||
|
results = [item for item in report.get("results") or [] if isinstance(item, dict)]
|
||||||
|
by_id = {str(item.get("prompt_id") or ""): item for item in results}
|
||||||
|
ordered = [by_id.get(item["id"], {}) for item in PROMPTS]
|
||||||
|
replies = [str(item.get("reply") or "") for item in ordered]
|
||||||
|
first_commands, first_rows = _tool_facts(ordered[0])
|
||||||
|
second_commands, second_rows = _tool_facts(ordered[1])
|
||||||
|
third_commands, _ = _tool_facts(ordered[2])
|
||||||
|
selected_claims = sorted(
|
||||||
|
row_id
|
||||||
|
for row_id in first_rows & second_rows
|
||||||
|
if UUID_RE.fullmatch(row_id)
|
||||||
|
and row_id[:8].casefold() in replies[0].casefold()
|
||||||
|
and row_id[:8].casefold() in replies[1].casefold()
|
||||||
|
)
|
||||||
|
chain_valid, execution_hashes = _execution_chain(ordered)
|
||||||
|
fingerprint_before = report.get("db_fingerprint_before") or {}
|
||||||
|
fingerprint_after = report.get("db_fingerprint_after") or {}
|
||||||
|
|
||||||
|
checks = {
|
||||||
|
"exact_unseen_prompts_without_supplied_row_id": bool(
|
||||||
|
len(results) == 3
|
||||||
|
and all(by_id.get(item["id"], {}).get("prompt") == item["message"] for item in PROMPTS)
|
||||||
|
and not UUID_RE.search(PROMPTS[0]["message"])
|
||||||
|
),
|
||||||
|
"fail_closed_safety_gate_passed": (report.get("safety_gate") or {}).get("status") == "pass",
|
||||||
|
"all_turn_execution_manifests_complete": (
|
||||||
|
(report.get("execution_manifest_summary") or {}).get("all_turns_attribution_complete") is True
|
||||||
|
and chain_valid
|
||||||
|
),
|
||||||
|
"claim_selected_from_database_without_operator_id": bool(
|
||||||
|
selected_claims and first_commands & {"context", "search"}
|
||||||
|
),
|
||||||
|
"same_claim_body_and_evidence_reinspected_after_challenge": bool(
|
||||||
|
selected_claims and {"show", "evidence"} <= second_commands
|
||||||
|
),
|
||||||
|
"weak_support_identified": _contains(replies[0], "confidence 0.9", "no_source_pointer")
|
||||||
|
and any(term in replies[0].casefold() for term in ("internal", "no external primary source")),
|
||||||
|
"assumption_and_observed_support_separated": _contains(
|
||||||
|
replies[1], "assumptions versus observed support", "falsifier"
|
||||||
|
),
|
||||||
|
"multiple_review_only_replacements_proposed": bool(
|
||||||
|
_contains(replies[1], "proposal a", "proposal b")
|
||||||
|
and any(term in replies[1].casefold() for term in ("staged only", "nothing applied"))
|
||||||
|
),
|
||||||
|
"user_objection_changed_the_candidate": _contains(
|
||||||
|
replies[2], "revised proposal", "mechanism", "outcome"
|
||||||
|
),
|
||||||
|
"missing_source_named": _contains(replies[2], "ostrom", "governing the commons")
|
||||||
|
and any(term in replies[2].casefold() for term in ("primary", "per-case")),
|
||||||
|
"review_approval_apply_boundary_explicit": _contains(
|
||||||
|
replies[2], "pending_review", "human review", "approved", "apply", "applied_at"
|
||||||
|
),
|
||||||
|
"no_write_or_hidden_learning_claim": _contains(replies[2], "nothing staged or changed")
|
||||||
|
and report.get("db_counts_changed") is False
|
||||||
|
and report.get("db_fingerprint_unchanged") is True
|
||||||
|
and fingerprint_before.get("fingerprint_sha256") == fingerprint_after.get("fingerprint_sha256"),
|
||||||
|
"third_turn_used_conversation_instead_of_new_db_lookup": not third_commands
|
||||||
|
and ordered[2]
|
||||||
|
.get("execution_manifest", {})
|
||||||
|
.get("session_boundary", {})
|
||||||
|
.get("conversation", {})
|
||||||
|
.get("starts_from_empty_conversation")
|
||||||
|
is False,
|
||||||
|
}
|
||||||
|
outcomes = {
|
||||||
|
"answers_from_current_database_without_row_ids": checks[
|
||||||
|
"claim_selected_from_database_without_operator_id"
|
||||||
|
],
|
||||||
|
"survives_a_claim_body_and_evidence_challenge": checks[
|
||||||
|
"same_claim_body_and_evidence_reinspected_after_challenge"
|
||||||
|
]
|
||||||
|
and checks["assumption_and_observed_support_separated"],
|
||||||
|
"proposes_better_claims_without_making_them_live": checks[
|
||||||
|
"multiple_review_only_replacements_proposed"
|
||||||
|
]
|
||||||
|
and checks["no_write_or_hidden_learning_claim"],
|
||||||
|
"iterates_with_the_user_and_revises_reasoning": checks["user_objection_changed_the_candidate"],
|
||||||
|
"names_the_evidence_needed_to_resolve_disagreement": checks["missing_source_named"],
|
||||||
|
"preserves_human_review_and_guarded_apply": checks["review_approval_apply_boundary_explicit"],
|
||||||
|
"conversation_chain_and_runtime_are_attributable": checks["all_turn_execution_manifests_complete"],
|
||||||
|
}
|
||||||
|
passed = all(checks.values()) and all(outcomes.values())
|
||||||
|
return {
|
||||||
|
"schema": SCHEMA,
|
||||||
|
"status": "pass" if passed else "fail",
|
||||||
|
"proof_tier": "live_vps_gatewayrunner_temp_profile_no_telegram_post_no_apply",
|
||||||
|
"source_report_sha256": source_report_sha256,
|
||||||
|
"remote_run_id": report.get("remote_run_id"),
|
||||||
|
"harness_git_head": (report.get("execution_manifest_summary") or {})
|
||||||
|
.get("harness_source", {})
|
||||||
|
.get("git_head"),
|
||||||
|
"prompt_ids": [item["id"] for item in PROMPTS],
|
||||||
|
"selected_claim_ids": selected_claims,
|
||||||
|
"execution_sha256_chain": execution_hashes,
|
||||||
|
"checks": checks,
|
||||||
|
"outcomes": outcomes,
|
||||||
|
"answer_excerpts": [reply[:500] for reply in replies],
|
||||||
|
"database": {
|
||||||
|
"fingerprint_sha256": fingerprint_after.get("fingerprint_sha256"),
|
||||||
|
"fingerprint_unchanged": report.get("db_fingerprint_unchanged"),
|
||||||
|
"table_count": fingerprint_after.get("table_count"),
|
||||||
|
"total_rows": fingerprint_after.get("total_rows"),
|
||||||
|
},
|
||||||
|
"safety_gate": report.get("safety_gate"),
|
||||||
|
"claim_ceiling": {
|
||||||
|
"proven": (
|
||||||
|
"One unseen three-turn live VPS handler session independently selected a canonical claim, inspected "
|
||||||
|
"its body and evidence, survived two user challenges, revised candidate knowledge, named missing "
|
||||||
|
"evidence, and preserved review-before-apply without changing the database."
|
||||||
|
),
|
||||||
|
"not_proven": [
|
||||||
|
"Telegram-visible message delivery",
|
||||||
|
"staging or canonical proposal mutation",
|
||||||
|
"agent-to-agent review",
|
||||||
|
"source ingestion or guarded apply",
|
||||||
|
"GCP runtime parity for this exact behavior commit and database state",
|
||||||
|
],
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
parser = argparse.ArgumentParser(description=__doc__)
|
||||||
|
parser.add_argument("--report", required=True, type=Path)
|
||||||
|
parser.add_argument("--output", required=True, type=Path)
|
||||||
|
args = parser.parse_args()
|
||||||
|
report = json.loads(args.report.read_text(encoding="utf-8"))
|
||||||
|
receipt = verify_report(report, source_report_sha256=_sha256_file(args.report))
|
||||||
|
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
args.output.write_text(json.dumps(receipt, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||||
|
print(json.dumps({"output": str(args.output), "status": receipt["status"]}, sort_keys=True))
|
||||||
|
return 0 if receipt["status"] == "pass" else 1
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
|
|
@ -43,6 +43,24 @@ def test_plugin_injects_only_operational_contracts_and_writes_redacted_trace(tmp
|
||||||
{"id": "source_intake", "capability_tier": "build-local compiler only"},
|
{"id": "source_intake", "capability_tier": "build-local compiler only"},
|
||||||
],
|
],
|
||||||
"compiled_response": "must not be injected either",
|
"compiled_response": "must not be injected either",
|
||||||
|
"retrieval_receipt": {
|
||||||
|
"schema": "livingip.teleoKbRetrievalReceipt.v1",
|
||||||
|
"query_sha256": "1" * 64,
|
||||||
|
"semantic_context_sha256": "2" * 64,
|
||||||
|
"artifact_state_sha256": "3" * 64,
|
||||||
|
"claim_ids": ["claim-1"],
|
||||||
|
"source_ids": ["source-1"],
|
||||||
|
"counts": {"claims": 1, "sources": 1},
|
||||||
|
"read_consistency": {
|
||||||
|
"status": "stable_wal_marker",
|
||||||
|
"attempts": 1,
|
||||||
|
"database": "teleo",
|
||||||
|
"database_user": "runtime",
|
||||||
|
"system_identifier": "system-1",
|
||||||
|
"wal_lsn_before": "0/123",
|
||||||
|
"wal_lsn_after": "0/123",
|
||||||
|
},
|
||||||
|
},
|
||||||
}
|
}
|
||||||
calls = []
|
calls = []
|
||||||
|
|
||||||
|
|
@ -63,6 +81,9 @@ def test_plugin_injects_only_operational_contracts_and_writes_redacted_trace(tmp
|
||||||
assert record["status"] == "ok"
|
assert record["status"] == "ok"
|
||||||
assert record["injected"] is True
|
assert record["injected"] is True
|
||||||
assert record["contract_ids"] == ["reply_budget", "source_intake"]
|
assert record["contract_ids"] == ["reply_budget", "source_intake"]
|
||||||
|
assert record["retrieval_receipt"]["semantic_context_sha256"] == "2" * 64
|
||||||
|
assert record["retrieval_receipt"]["read_consistency"]["system_identifier"] == "system-1"
|
||||||
|
assert len(record["retrieval_receipt"]["receipt_sha256"]) == 64
|
||||||
command, kwargs = calls[0]
|
command, kwargs = calls[0]
|
||||||
assert command[2:5] == ["--local", "context", "operator secret-shaped but nonsecret message"]
|
assert command[2:5] == ["--local", "context", "operator secret-shaped but nonsecret message"]
|
||||||
assert command[-6:] == ["--limit", "0", "--context-limit", "0", "--format", "json"]
|
assert command[-6:] == ["--limit", "0", "--context-limit", "0", "--format", "json"]
|
||||||
|
|
@ -465,6 +486,7 @@ def test_plugin_fails_closed_on_missing_snapshot_or_invalid_compiler() -> None:
|
||||||
def test_handler_harness_retains_database_context_proof_fields() -> None:
|
def test_handler_harness_retains_database_context_proof_fields() -> None:
|
||||||
source = (ROOT / "scripts" / "run_leo_direct_claim_handler_suite.py").read_text(encoding="utf-8")
|
source = (ROOT / "scripts" / "run_leo_direct_claim_handler_suite.py").read_text(encoding="utf-8")
|
||||||
assert "LEO_DB_CONTEXT_TRACE_PATH" in source
|
assert "LEO_DB_CONTEXT_TRACE_PATH" in source
|
||||||
|
assert "KB_TOOL_SOURCE" in source
|
||||||
assert '"database_context_trace"' in source
|
assert '"database_context_trace"' in source
|
||||||
assert '"database_context_injection_count"' in source
|
assert '"database_context_injection_count"' in source
|
||||||
assert '"database_context_all_ok"' in source
|
assert '"database_context_all_ok"' in source
|
||||||
|
|
@ -475,3 +497,18 @@ def test_handler_harness_retains_database_context_proof_fields() -> None:
|
||||||
assert '"database_tool_call_proven"' in source
|
assert '"database_tool_call_proven"' in source
|
||||||
assert '"database_retrieval_receipt_proven"' in source
|
assert '"database_retrieval_receipt_proven"' in source
|
||||||
assert '"database_tool_calls_read_only"' in source
|
assert '"database_tool_calls_read_only"' in source
|
||||||
|
assert "LEO_TURN_API_TRACE_PATH" in source
|
||||||
|
assert '"model_call_trace"' in source
|
||||||
|
assert '"turn_pre_llm_call"' in source
|
||||||
|
assert '"turn_post_llm_call"' in source
|
||||||
|
assert '"model_response_trace"' in source
|
||||||
|
assert '"conversation_before"' in source
|
||||||
|
assert '"conversation_after"' in source
|
||||||
|
assert "copy.deepcopy(message)" in source
|
||||||
|
assert "after_rows[: len(before_rows)] == before_rows" in source
|
||||||
|
assert "attach_execution_manifests" in source
|
||||||
|
assert "report_passes_safety" in source
|
||||||
|
assert "json.loads(redact(candidate))" in source
|
||||||
|
assert '"db_fingerprint_before"' in source
|
||||||
|
assert '"db_fingerprint_after"' in source
|
||||||
|
assert '"db_fingerprint_unchanged"' in source
|
||||||
|
|
|
||||||
|
|
@ -846,6 +846,16 @@ def test_vps_bridge_direct_intents_reject_unrelated_and_resolve_overlaps() -> No
|
||||||
assert v3_ids & direct_ids == {"proposal_state_readback"}
|
assert v3_ids & direct_ids == {"proposal_state_readback"}
|
||||||
assert "source_link_audit" not in v3_ids
|
assert "source_link_audit" not in v3_ids
|
||||||
|
|
||||||
|
reasoning_followup_ids = {
|
||||||
|
item["id"]
|
||||||
|
for item in module.operational_contracts(
|
||||||
|
"Inspect the exact claim body and evidence: what is assumption versus observed support, what would "
|
||||||
|
"falsify it, and which replacement claims would you put into review? Keep them as proposals, not live "
|
||||||
|
"knowledge."
|
||||||
|
)
|
||||||
|
}
|
||||||
|
assert reasoning_followup_ids & direct_ids == set()
|
||||||
|
|
||||||
|
|
||||||
def test_vps_bridge_named_document_question_matches_exact_proposal_and_compiles_receipt(monkeypatch) -> None:
|
def test_vps_bridge_named_document_question_matches_exact_proposal_and_compiles_receipt(monkeypatch) -> None:
|
||||||
module = _load_module(BRIDGE_DIR / "kb_tool.py")
|
module = _load_module(BRIDGE_DIR / "kb_tool.py")
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,7 @@
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import json
|
import json
|
||||||
|
import subprocess
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
from scripts import leo_behavior_manifest as manifest
|
from scripts import leo_behavior_manifest as manifest
|
||||||
|
|
@ -16,6 +17,7 @@ def _write(path: Path, value: str) -> None:
|
||||||
def test_behavior_manifest_is_deterministic_and_never_emits_config_secrets(tmp_path: Path) -> None:
|
def test_behavior_manifest_is_deterministic_and_never_emits_config_secrets(tmp_path: Path) -> None:
|
||||||
profile = tmp_path / "profile"
|
profile = tmp_path / "profile"
|
||||||
hermes = tmp_path / "hermes"
|
hermes = tmp_path / "hermes"
|
||||||
|
deployment = tmp_path / "deployment"
|
||||||
_write(
|
_write(
|
||||||
profile / "config.yaml",
|
profile / "config.yaml",
|
||||||
"""model:
|
"""model:
|
||||||
|
|
@ -40,12 +42,15 @@ gateway:
|
||||||
_write(profile / "state.db", "runtime-state")
|
_write(profile / "state.db", "runtime-state")
|
||||||
_write(hermes / "run_agent.py", "# runtime\n")
|
_write(hermes / "run_agent.py", "# runtime\n")
|
||||||
_write(hermes / "agent" / "prompt_builder.py", "# prompts\n")
|
_write(hermes / "agent" / "prompt_builder.py", "# prompts\n")
|
||||||
|
_write(deployment / "scripts" / "leo_behavior_manifest.py", "# manifest\n")
|
||||||
|
_write(deployment / "scripts" / "leo_tool_trace.py", "# tool trace\n")
|
||||||
|
|
||||||
first = manifest.build_manifest(profile, hermes)
|
first = manifest.build_manifest(profile, hermes, deployment)
|
||||||
second = manifest.build_manifest(profile, hermes)
|
second = manifest.build_manifest(profile, hermes, deployment)
|
||||||
|
|
||||||
assert first["behavior_sha256"] == second["behavior_sha256"]
|
assert first["behavior_sha256"] == second["behavior_sha256"]
|
||||||
assert first["model_runtime"]["default_model"] == "anthropic/claude-sonnet-4-6"
|
assert first["model_runtime"]["default_model"] == "anthropic/claude-sonnet-4-6"
|
||||||
|
assert first["teleo_infrastructure_runtime"]["source_tree"]["file_count"] == 2
|
||||||
assert first["components"]["persistent_memory"]["content"]["file_count"] == 1
|
assert first["components"]["persistent_memory"]["content"]["file_count"] == 1
|
||||||
encoded = json.dumps(first)
|
encoded = json.dumps(first)
|
||||||
assert "super-secret-token" not in encoded
|
assert "super-secret-token" not in encoded
|
||||||
|
|
@ -56,14 +61,17 @@ gateway:
|
||||||
def test_behavior_manifest_changes_when_a_behavior_layer_changes(tmp_path: Path) -> None:
|
def test_behavior_manifest_changes_when_a_behavior_layer_changes(tmp_path: Path) -> None:
|
||||||
profile = tmp_path / "profile"
|
profile = tmp_path / "profile"
|
||||||
hermes = tmp_path / "hermes"
|
hermes = tmp_path / "hermes"
|
||||||
|
deployment = tmp_path / "deployment"
|
||||||
_write(profile / "config.yaml", "model:\n provider: openrouter\n default: model-a\n")
|
_write(profile / "config.yaml", "model:\n provider: openrouter\n default: model-a\n")
|
||||||
_write(profile / "SOUL.md", "identity-a\n")
|
_write(profile / "SOUL.md", "identity-a\n")
|
||||||
_write(hermes / "run_agent.py", "runtime\n")
|
_write(hermes / "run_agent.py", "runtime\n")
|
||||||
_write(hermes / "agent" / "prompt_builder.py", "prompts\n")
|
_write(hermes / "agent" / "prompt_builder.py", "prompts\n")
|
||||||
|
_write(deployment / "scripts" / "leo_behavior_manifest.py", "manifest\n")
|
||||||
|
_write(deployment / "scripts" / "leo_tool_trace.py", "tool trace\n")
|
||||||
|
|
||||||
before = manifest.build_manifest(profile, hermes)
|
before = manifest.build_manifest(profile, hermes, deployment)
|
||||||
_write(profile / "SOUL.md", "identity-b\n")
|
_write(profile / "SOUL.md", "identity-b\n")
|
||||||
after = manifest.build_manifest(profile, hermes)
|
after = manifest.build_manifest(profile, hermes, deployment)
|
||||||
|
|
||||||
assert before["behavior_sha256"] != after["behavior_sha256"]
|
assert before["behavior_sha256"] != after["behavior_sha256"]
|
||||||
assert (
|
assert (
|
||||||
|
|
@ -75,6 +83,7 @@ def test_behavior_manifest_changes_when_a_behavior_layer_changes(tmp_path: Path)
|
||||||
def test_behavior_manifest_hashes_content_behind_symlinks(tmp_path: Path) -> None:
|
def test_behavior_manifest_hashes_content_behind_symlinks(tmp_path: Path) -> None:
|
||||||
profile = tmp_path / "profile"
|
profile = tmp_path / "profile"
|
||||||
hermes = tmp_path / "hermes"
|
hermes = tmp_path / "hermes"
|
||||||
|
deployment = tmp_path / "deployment"
|
||||||
target = tmp_path / "deployed-skill.md"
|
target = tmp_path / "deployed-skill.md"
|
||||||
_write(profile / "config.yaml", "model:\n provider: test\n")
|
_write(profile / "config.yaml", "model:\n provider: test\n")
|
||||||
_write(profile / "SOUL.md", "identity\n")
|
_write(profile / "SOUL.md", "identity\n")
|
||||||
|
|
@ -83,10 +92,12 @@ def test_behavior_manifest_hashes_content_behind_symlinks(tmp_path: Path) -> Non
|
||||||
(profile / "skills" / "SKILL.md").symlink_to(target)
|
(profile / "skills" / "SKILL.md").symlink_to(target)
|
||||||
_write(hermes / "run_agent.py", "runtime\n")
|
_write(hermes / "run_agent.py", "runtime\n")
|
||||||
_write(hermes / "agent" / "prompt_builder.py", "prompts\n")
|
_write(hermes / "agent" / "prompt_builder.py", "prompts\n")
|
||||||
|
_write(deployment / "scripts" / "leo_behavior_manifest.py", "manifest\n")
|
||||||
|
_write(deployment / "scripts" / "leo_tool_trace.py", "tool trace\n")
|
||||||
|
|
||||||
before = manifest.build_manifest(profile, hermes)
|
before = manifest.build_manifest(profile, hermes, deployment)
|
||||||
_write(target, "version-b\n")
|
_write(target, "version-b\n")
|
||||||
after = manifest.build_manifest(profile, hermes)
|
after = manifest.build_manifest(profile, hermes, deployment)
|
||||||
|
|
||||||
assert before["behavior_sha256"] != after["behavior_sha256"]
|
assert before["behavior_sha256"] != after["behavior_sha256"]
|
||||||
symlinks = after["components"]["procedural_skills"]["content"]["symlinks"]
|
symlinks = after["components"]["procedural_skills"]["content"]["symlinks"]
|
||||||
|
|
@ -104,3 +115,44 @@ def test_live_handler_harness_excludes_prior_dynamic_state_and_compares_live_fin
|
||||||
assert "build_behavior_manifest" in source
|
assert "build_behavior_manifest" in source
|
||||||
assert 'report["changed_live_profile"] =' in source
|
assert 'report["changed_live_profile"] =' in source
|
||||||
assert '"changed_live_profile": False' not in source
|
assert '"changed_live_profile": False' not in source
|
||||||
|
|
||||||
|
|
||||||
|
def test_runtime_source_manifest_covers_nested_executable_sources(tmp_path: Path) -> None:
|
||||||
|
profile = tmp_path / "profile"
|
||||||
|
hermes = tmp_path / "hermes"
|
||||||
|
deployment = tmp_path / "deployment"
|
||||||
|
_write(profile / "config.yaml", "model:\n provider: test\n")
|
||||||
|
_write(profile / "SOUL.md", "identity\n")
|
||||||
|
_write(hermes / "run_agent.py", "runtime\n")
|
||||||
|
_write(hermes / "gateway" / "run.py", "gateway-a\n")
|
||||||
|
_write(hermes / "tools" / "terminal.py", "terminal-a\n")
|
||||||
|
_write(deployment / "scripts" / "leo_tool_trace.py", "trace-a\n")
|
||||||
|
_write(deployment / "ansible" / "roles" / "leo" / "tasks" / "main.yml", "---\n")
|
||||||
|
|
||||||
|
before = manifest.build_manifest(profile, hermes, deployment)
|
||||||
|
_write(deployment / "ops" / "new_runtime_helper.py", "helper-b\n")
|
||||||
|
after = manifest.build_manifest(profile, hermes, deployment)
|
||||||
|
|
||||||
|
assert before["hermes_runtime"]["source_tree"]["file_count"] == 3
|
||||||
|
assert before["teleo_infrastructure_runtime"]["source_tree"]["file_count"] == 2
|
||||||
|
assert after["teleo_infrastructure_runtime"]["source_tree"]["file_count"] == 3
|
||||||
|
assert (
|
||||||
|
before["teleo_infrastructure_runtime"]["source_tree"]["sha256"]
|
||||||
|
!= after["teleo_infrastructure_runtime"]["source_tree"]["sha256"]
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_git_state_marks_untracked_files_dirty(tmp_path: Path) -> None:
|
||||||
|
subprocess.run(["git", "init", "-q", str(tmp_path)], check=True)
|
||||||
|
subprocess.run(["git", "-C", str(tmp_path), "config", "user.email", "test@example.com"], check=True)
|
||||||
|
subprocess.run(["git", "-C", str(tmp_path), "config", "user.name", "Test"], check=True)
|
||||||
|
_write(tmp_path / "tracked.py", "tracked\n")
|
||||||
|
subprocess.run(["git", "-C", str(tmp_path), "add", "tracked.py"], check=True)
|
||||||
|
subprocess.run(["git", "-C", str(tmp_path), "commit", "-qm", "fixture"], check=True)
|
||||||
|
|
||||||
|
assert manifest.git_state(tmp_path)["worktree_clean"] is True
|
||||||
|
_write(tmp_path / "untracked.py", "untracked\n")
|
||||||
|
state = manifest.git_state(tmp_path)
|
||||||
|
|
||||||
|
assert state["worktree_clean"] is False
|
||||||
|
assert len(str(state["status_sha256"])) == 64
|
||||||
|
|
|
||||||
413
tests/test_leo_turn_execution_manifest.py
Normal file
413
tests/test_leo_turn_execution_manifest.py
Normal file
|
|
@ -0,0 +1,413 @@
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import copy
|
||||||
|
|
||||||
|
from scripts import leo_turn_execution_manifest as manifest
|
||||||
|
|
||||||
|
PROMPT = "Challenge the weak acquisition claim."
|
||||||
|
REPLY = "The current evidence is shallow; here is a narrower candidate claim."
|
||||||
|
SESSION_SHA = "6" * 64
|
||||||
|
EMPTY_TOOL_CALLS_SHA = manifest.canonical_sha256([])
|
||||||
|
|
||||||
|
|
||||||
|
def behavior_manifest() -> dict:
|
||||||
|
return {
|
||||||
|
"behavior_sha256": "a" * 64,
|
||||||
|
"hermes_runtime": {
|
||||||
|
"git_head": "b" * 40,
|
||||||
|
"source_tree": {"sha256": "c" * 64, "file_count": 123},
|
||||||
|
},
|
||||||
|
"teleo_infrastructure_runtime": {
|
||||||
|
"git_head": "d" * 40,
|
||||||
|
"source_tree": {"sha256": "e" * 64, "file_count": 42},
|
||||||
|
},
|
||||||
|
"components": {
|
||||||
|
"runtime_identity": {"content": {"sha256": "f" * 64}},
|
||||||
|
"procedural_skills": {"content": {"sha256": "1" * 64}},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def retrieval_receipt(prompt: str = PROMPT) -> dict:
|
||||||
|
return {
|
||||||
|
"schema": "livingip.teleoKbRetrievalReceipt.v1",
|
||||||
|
"query_sha256": manifest.text_sha256(prompt[:16_000]),
|
||||||
|
"semantic_context_sha256": "3" * 64,
|
||||||
|
"artifact_state_sha256": "4" * 64,
|
||||||
|
"receipt_sha256": "5" * 64,
|
||||||
|
"claim_ids": ["claim-b", "claim-a"],
|
||||||
|
"source_ids": ["source-a"],
|
||||||
|
"counts": {"claims": 2, "context_rows": 1},
|
||||||
|
"read_consistency": {
|
||||||
|
"status": "stable_wal_marker",
|
||||||
|
"attempts": 1,
|
||||||
|
"database": "teleo",
|
||||||
|
"database_user": "runtime",
|
||||||
|
"system_identifier": "system-1",
|
||||||
|
"wal_lsn_before": "0/123",
|
||||||
|
"wal_lsn_after": "0/123",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def conversation_after(reply: str = REPLY, *, message_count: int = 2, marker: str = "a") -> dict:
|
||||||
|
return {
|
||||||
|
"message_count": message_count,
|
||||||
|
"messages_sha256": marker * 64,
|
||||||
|
"last_message_role": "assistant",
|
||||||
|
"last_message_content_sha256": manifest.text_sha256(reply),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def empty_conversation() -> dict:
|
||||||
|
return {
|
||||||
|
"message_count": 0,
|
||||||
|
"messages_sha256": manifest.canonical_sha256([]),
|
||||||
|
"last_message_role": None,
|
||||||
|
"last_message_content_sha256": None,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def turn_result(
|
||||||
|
*,
|
||||||
|
prompt: str = PROMPT,
|
||||||
|
reply: str = REPLY,
|
||||||
|
before: dict | None = None,
|
||||||
|
after_marker: str = "a",
|
||||||
|
turn: int = 1,
|
||||||
|
) -> dict:
|
||||||
|
prompt_sha = manifest.text_sha256(prompt)
|
||||||
|
reply_sha = manifest.text_sha256(reply)
|
||||||
|
before_trace = copy.deepcopy(before or empty_conversation())
|
||||||
|
after_trace = conversation_after(
|
||||||
|
reply,
|
||||||
|
message_count=int(before_trace["message_count"]) + 2,
|
||||||
|
marker=after_marker,
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
"turn": turn,
|
||||||
|
"prompt_id": f"OOS-{turn:02d}",
|
||||||
|
"prompt": prompt,
|
||||||
|
"reply": reply,
|
||||||
|
"started_at_utc": "2026-07-14T12:00:00+00:00",
|
||||||
|
"ended_at_utc": "2026-07-14T12:00:02+00:00",
|
||||||
|
"mutates_kb": False,
|
||||||
|
"database_context_trace": [
|
||||||
|
{
|
||||||
|
"event": "pre_llm_call",
|
||||||
|
"status": "ok",
|
||||||
|
"injected": True,
|
||||||
|
"query_sha256": prompt_sha,
|
||||||
|
"contract_ids": ["reply_budget", "source_link_audit"],
|
||||||
|
"contract_sha256": "7" * 64,
|
||||||
|
"retrieval_receipt": retrieval_receipt(prompt),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"event": "post_llm_call",
|
||||||
|
"status": "ok",
|
||||||
|
"validated": True,
|
||||||
|
"transformed": False,
|
||||||
|
"query_sha256": prompt_sha,
|
||||||
|
"response_sha256": reply_sha,
|
||||||
|
"delivered_response_sha256": reply_sha,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
"database_tool_trace": {
|
||||||
|
"schema": "livingip.leoKbToolTrace.v1",
|
||||||
|
"database_tool_call_count": 0,
|
||||||
|
"database_tool_completed_count": 0,
|
||||||
|
"database_tool_calls_read_only": True,
|
||||||
|
"calls": [],
|
||||||
|
},
|
||||||
|
"model_call_trace": [
|
||||||
|
{
|
||||||
|
"event": "turn_pre_llm_call",
|
||||||
|
"session_id_sha256": SESSION_SHA,
|
||||||
|
"prompt_sha256": prompt_sha,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"event": "post_api_request",
|
||||||
|
"api_call_count": 1,
|
||||||
|
"api_mode": "openai_chat_completions",
|
||||||
|
"model": "anthropic/claude-sonnet-4-6",
|
||||||
|
"provider": "openrouter",
|
||||||
|
"response_model": "anthropic/claude-sonnet-4-6",
|
||||||
|
"finish_reason": "stop",
|
||||||
|
"message_count": 4,
|
||||||
|
"assistant_content_chars": len(reply),
|
||||||
|
"assistant_tool_call_count": 0,
|
||||||
|
"task_id_sha256": SESSION_SHA,
|
||||||
|
"session_id_sha256": SESSION_SHA,
|
||||||
|
"base_url_sha256": "8" * 64,
|
||||||
|
"usage": {"input_tokens": 100, "output_tokens": 20, "ignored": "secret-shaped"},
|
||||||
|
"provider_response_id": None,
|
||||||
|
"provider_response_id_status": "not_exposed_by_hermes_post_api_request_hook",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"event": "turn_post_llm_call",
|
||||||
|
"session_id_sha256": SESSION_SHA,
|
||||||
|
"prompt_sha256": prompt_sha,
|
||||||
|
"raw_assistant_response_sha256": reply_sha,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
"model_response_trace": [
|
||||||
|
{
|
||||||
|
"content_sha256": reply_sha,
|
||||||
|
"content_bytes": len(reply.encode()),
|
||||||
|
"tool_calls_sha256": EMPTY_TOOL_CALLS_SHA,
|
||||||
|
"tool_call_count": 0,
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"conversation_before": before_trace,
|
||||||
|
"conversation_after": after_trace,
|
||||||
|
"conversation_history_prefix_preserved": True,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def database_fingerprint() -> dict:
|
||||||
|
marker = {
|
||||||
|
"database": "teleo",
|
||||||
|
"database_user": "runtime",
|
||||||
|
"system_identifier": "system-1",
|
||||||
|
"wal_lsn": "0/123",
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
"schema": "livingip.teleoCanonicalDatabaseFingerprint.v1",
|
||||||
|
"status": "ok",
|
||||||
|
"fingerprint_sha256": "9" * 64,
|
||||||
|
"table_count": 39,
|
||||||
|
"total_rows": 52167,
|
||||||
|
"table_rows_sha256": "a" * 64,
|
||||||
|
"structure_sha256": "b" * 64,
|
||||||
|
"read_consistency": {"status": "stable_wal_marker", "before": marker, "after": marker},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def suite_safety() -> dict:
|
||||||
|
return {
|
||||||
|
"remote_returncode": 0,
|
||||||
|
"pass_runtime": True,
|
||||||
|
"live_behavior_manifest_unchanged": True,
|
||||||
|
"temp_profile_removed": True,
|
||||||
|
"service_unchanged": True,
|
||||||
|
"db_fingerprint_unchanged": True,
|
||||||
|
"model_call_trace_all_bound": True,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def build(
|
||||||
|
result: dict | None = None,
|
||||||
|
*,
|
||||||
|
fingerprint: dict | None = None,
|
||||||
|
harness_source: dict | None = None,
|
||||||
|
previous_execution_sha256: str | None = None,
|
||||||
|
expected_previous_conversation: dict | None = None,
|
||||||
|
) -> dict:
|
||||||
|
selected_fingerprint = database_fingerprint() if fingerprint is None else fingerprint
|
||||||
|
return manifest.build_turn_manifest(
|
||||||
|
result=result or turn_result(),
|
||||||
|
behavior_manifest=behavior_manifest(),
|
||||||
|
session_key="agent:main:telegram:group:private",
|
||||||
|
source={"platform": "telegram", "chat_type": "group"},
|
||||||
|
suite_mode="live_vps_gatewayrunner_temp_profile",
|
||||||
|
remote_run_id="run-1",
|
||||||
|
db_counts_before={"public.claims": 1837},
|
||||||
|
db_counts_after={"public.claims": 1837},
|
||||||
|
db_counts_changed=False,
|
||||||
|
db_fingerprint_before=selected_fingerprint,
|
||||||
|
db_fingerprint_after=selected_fingerprint,
|
||||||
|
posted_to_telegram=False,
|
||||||
|
mutates_kb_by_harness=False,
|
||||||
|
harness_source=harness_source
|
||||||
|
or {"git_head": "8" * 40, "worktree_clean": True, "status_sha256": manifest.text_sha256("")},
|
||||||
|
suite_safety=suite_safety(),
|
||||||
|
previous_execution_sha256=previous_execution_sha256,
|
||||||
|
expected_previous_conversation=expected_previous_conversation,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_turn_manifest_binds_model_runtime_session_and_exact_database_read() -> None:
|
||||||
|
value = build()
|
||||||
|
|
||||||
|
assert value["attribution"]["status"] == "complete"
|
||||||
|
assert value["replay_claim"]["status"] == "content_addressed_attribution"
|
||||||
|
assert value["model_execution"]["prompt_bound"] is True
|
||||||
|
assert value["model_execution"]["raw_response_bound"] is True
|
||||||
|
assert value["model_execution"]["delivered_response_bound"] is True
|
||||||
|
assert value["model_execution"]["calls"][0]["response_model"] == "anthropic/claude-sonnet-4-6"
|
||||||
|
assert value["model_execution"]["calls"][0]["usage"] == {"input_tokens": 100, "output_tokens": 20}
|
||||||
|
assert value["canonical_database"]["binding_status"] == "retrieval_receipt_bound"
|
||||||
|
assert value["canonical_database"]["fingerprint_before"]["table_count"] == 39
|
||||||
|
assert value["canonical_database"]["fingerprint_unchanged"] is True
|
||||||
|
receipt = value["canonical_database"]["context_retrieval_receipts"][0]
|
||||||
|
assert receipt["read_consistency"]["system_identifier"] == "system-1"
|
||||||
|
assert receipt["read_consistency"]["wal_lsn_before"] == receipt["read_consistency"]["wal_lsn_after"]
|
||||||
|
assert value["delivery_and_safety"]["posted_to_telegram"] is False
|
||||||
|
assert value["delivery_and_safety"]["kb_mutation_by_harness"] is False
|
||||||
|
assert value["replay_claim"]["runtime_source_artifacts_embedded"] is False
|
||||||
|
assert manifest.validate_turn_manifest(value) == []
|
||||||
|
|
||||||
|
encoded = str(value)
|
||||||
|
assert PROMPT not in encoded
|
||||||
|
assert "narrower candidate claim" not in encoded
|
||||||
|
assert "agent:main:telegram" not in encoded
|
||||||
|
assert "secret-shaped" not in encoded
|
||||||
|
|
||||||
|
|
||||||
|
def test_turn_manifest_detects_missing_actual_model_call() -> None:
|
||||||
|
result = turn_result()
|
||||||
|
result["model_call_trace"] = []
|
||||||
|
|
||||||
|
value = build(result)
|
||||||
|
|
||||||
|
assert value["attribution"]["status"] == "incomplete"
|
||||||
|
assert "actual_model_call" in value["attribution"]["missing_required_bindings"]
|
||||||
|
assert value["replay_claim"]["runtime_and_turn_inputs_content_addressed"] is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_turn_manifest_hash_detects_tampering() -> None:
|
||||||
|
value = build()
|
||||||
|
tampered = copy.deepcopy(value)
|
||||||
|
tampered["turn"]["reply_sha256"] = "0" * 64
|
||||||
|
|
||||||
|
assert manifest.validate_turn_manifest(tampered) == ["execution_sha256_mismatch"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_database_question_requires_a_stable_full_database_fingerprint() -> None:
|
||||||
|
value = build(fingerprint={"status": "error"})
|
||||||
|
|
||||||
|
assert value["attribution"]["status"] == "incomplete"
|
||||||
|
assert "database_fingerprint_before" in value["attribution"]["missing_required_bindings"]
|
||||||
|
assert "database_fingerprint_after" in value["attribution"]["missing_required_bindings"]
|
||||||
|
assert value["canonical_database"]["fingerprint_unchanged"] is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_failed_database_context_lookup_cannot_be_attributed_as_complete() -> None:
|
||||||
|
result = turn_result()
|
||||||
|
result["database_context_trace"][0] = {
|
||||||
|
"event": "pre_llm_call",
|
||||||
|
"status": "error",
|
||||||
|
"injected": True,
|
||||||
|
"query_sha256": manifest.text_sha256(PROMPT),
|
||||||
|
"error": "timeout",
|
||||||
|
}
|
||||||
|
result["database_context_trace"][1]["status"] = "error"
|
||||||
|
|
||||||
|
value = build(result)
|
||||||
|
|
||||||
|
assert value["attribution"]["status"] == "incomplete"
|
||||||
|
assert "database_retrieval_receipt" in value["attribution"]["missing_required_bindings"]
|
||||||
|
assert "database_context_response_binding" in value["attribution"]["missing_required_bindings"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_schema_only_or_wrong_query_receipt_is_rejected() -> None:
|
||||||
|
result = turn_result()
|
||||||
|
result["database_context_trace"][0]["retrieval_receipt"] = {
|
||||||
|
"schema": "livingip.teleoKbRetrievalReceipt.v1",
|
||||||
|
"query_sha256": "0" * 64,
|
||||||
|
}
|
||||||
|
|
||||||
|
value = build(result)
|
||||||
|
|
||||||
|
assert value["canonical_database"]["context_retrieval_receipts"] == []
|
||||||
|
assert "database_retrieval_receipt" in value["attribution"]["missing_required_bindings"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_delivered_reply_hash_mismatch_fails_closed() -> None:
|
||||||
|
result = turn_result()
|
||||||
|
result["database_context_trace"][1]["delivered_response_sha256"] = "0" * 64
|
||||||
|
|
||||||
|
value = build(result)
|
||||||
|
|
||||||
|
assert "database_context_response_binding" in value["attribution"]["missing_required_bindings"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_valid_response_transform_binds_raw_and_delivered_hashes() -> None:
|
||||||
|
result = turn_result()
|
||||||
|
raw_reply = "A much longer raw answer that the response contract replaces."
|
||||||
|
raw_sha = manifest.text_sha256(raw_reply)
|
||||||
|
result["database_context_trace"][1]["response_sha256"] = raw_sha
|
||||||
|
result["database_context_trace"][1]["transformed"] = True
|
||||||
|
result["model_call_trace"][-1]["raw_assistant_response_sha256"] = raw_sha
|
||||||
|
|
||||||
|
value = build(result)
|
||||||
|
|
||||||
|
assert value["attribution"]["status"] == "complete"
|
||||||
|
assert value["model_execution"]["raw_response_bound"] is True
|
||||||
|
assert value["model_execution"]["delivered_response_bound"] is True
|
||||||
|
assert value["canonical_database"]["context_binding"]["post_transformed"] is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_dirty_or_untracked_harness_cannot_produce_complete_manifest() -> None:
|
||||||
|
value = build(
|
||||||
|
harness_source={"git_head": "8" * 40, "worktree_clean": False, "status_sha256": "1" * 64}
|
||||||
|
)
|
||||||
|
|
||||||
|
assert "harness_worktree_clean" in value["attribution"]["missing_required_bindings"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_write_capable_database_tool_trace_fails_closed() -> None:
|
||||||
|
result = turn_result()
|
||||||
|
result["database_tool_trace"] = {
|
||||||
|
"schema": "livingip.leoKbToolTrace.v1",
|
||||||
|
"database_tool_call_count": 1,
|
||||||
|
"database_tool_completed_count": 1,
|
||||||
|
"database_tool_calls_read_only": False,
|
||||||
|
"calls": [
|
||||||
|
{
|
||||||
|
"tool_call_id_sha256": "1" * 64,
|
||||||
|
"arguments_sha256": "2" * 64,
|
||||||
|
"database_invocations": [
|
||||||
|
{
|
||||||
|
"access_mode": "write",
|
||||||
|
"command_sha256": "3" * 64,
|
||||||
|
"executable": "teleo-kb",
|
||||||
|
"subcommand": "apply",
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"result": {
|
||||||
|
"content_sha256": "4" * 64,
|
||||||
|
"content_bytes": 10,
|
||||||
|
"error_detected": False,
|
||||||
|
"nonempty": True,
|
||||||
|
"row_ids": [],
|
||||||
|
"sha256_values": [],
|
||||||
|
},
|
||||||
|
}
|
||||||
|
],
|
||||||
|
}
|
||||||
|
|
||||||
|
value = build(result)
|
||||||
|
|
||||||
|
assert "database_tools_read_only" in value["attribution"]["missing_required_bindings"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_later_turn_binds_previous_execution_and_conversation_state() -> None:
|
||||||
|
first_result = turn_result()
|
||||||
|
first = build(first_result)
|
||||||
|
second_result = turn_result(
|
||||||
|
prompt="Now revise that candidate after my challenge.",
|
||||||
|
reply="The revision is narrower and still remains a proposal.",
|
||||||
|
before=first_result["conversation_after"],
|
||||||
|
after_marker="b",
|
||||||
|
turn=2,
|
||||||
|
)
|
||||||
|
|
||||||
|
second = build(
|
||||||
|
second_result,
|
||||||
|
previous_execution_sha256=first["execution_sha256"],
|
||||||
|
expected_previous_conversation=first_result["conversation_after"],
|
||||||
|
)
|
||||||
|
|
||||||
|
assert second["attribution"]["status"] == "complete"
|
||||||
|
conversation = second["session_boundary"]["conversation"]
|
||||||
|
assert conversation["previous_execution_sha256"] == first["execution_sha256"]
|
||||||
|
assert conversation["prior_turn_state_bound"] is True
|
||||||
|
|
||||||
|
second_result["conversation_before"]["messages_sha256"] = "0" * 64
|
||||||
|
broken = build(
|
||||||
|
second_result,
|
||||||
|
previous_execution_sha256=first["execution_sha256"],
|
||||||
|
expected_previous_conversation=first_result["conversation_after"],
|
||||||
|
)
|
||||||
|
assert "conversation_prior_turn_binding" in broken["attribution"]["missing_required_bindings"]
|
||||||
114
tests/test_run_leo_direct_claim_handler_suite.py
Normal file
114
tests/test_run_leo_direct_claim_handler_suite.py
Normal file
|
|
@ -0,0 +1,114 @@
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import subprocess
|
||||||
|
|
||||||
|
from scripts import run_leo_direct_claim_handler_suite as suite
|
||||||
|
|
||||||
|
|
||||||
|
def safe_report() -> dict:
|
||||||
|
return {
|
||||||
|
"remote_returncode": 0,
|
||||||
|
"pass_runtime": True,
|
||||||
|
"handler": {"authorized": True},
|
||||||
|
"results": [
|
||||||
|
{
|
||||||
|
"ok": True,
|
||||||
|
"mutates_kb": False,
|
||||||
|
"database_tool_trace": {
|
||||||
|
"database_tool_call_count": 1,
|
||||||
|
"database_tool_completed_count": 1,
|
||||||
|
"database_tool_calls_read_only": True,
|
||||||
|
"calls": [
|
||||||
|
{
|
||||||
|
"database_invocations": [{"access_mode": "read_only"}],
|
||||||
|
"result": {
|
||||||
|
"content_sha256": "a" * 64,
|
||||||
|
"error_detected": False,
|
||||||
|
"nonempty": True,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
],
|
||||||
|
},
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"posted_to_telegram": False,
|
||||||
|
"mutates_kb_by_harness": False,
|
||||||
|
"db_counts_changed": False,
|
||||||
|
"db_fingerprint_unchanged": True,
|
||||||
|
"live_behavior_manifest_unchanged": True,
|
||||||
|
"service_before_after": {"unchanged_from_preexisting_live_readback": True},
|
||||||
|
"temp_profile_removed": True,
|
||||||
|
"model_call_trace_all_bound": True,
|
||||||
|
"execution_manifest_summary": {"all_turns_attribution_complete": True},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def test_redaction_preserves_json_and_removes_secret_values() -> None:
|
||||||
|
raw = json.dumps(
|
||||||
|
{
|
||||||
|
"bot_token": "123456789:abcdefghijklmnopqrstuvwxyz_ABCD",
|
||||||
|
"api_key": "sk-live-secret",
|
||||||
|
"nested": {"password": "hunter2", "access_token": "bearer-secret"},
|
||||||
|
"message": "Authorization: Bearer top-secret-value",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
redacted = suite.redact(raw)
|
||||||
|
parsed = json.loads(redacted)
|
||||||
|
|
||||||
|
assert parsed["bot_token"] == "[REDACTED]"
|
||||||
|
assert parsed["api_key"] == "[REDACTED]"
|
||||||
|
assert parsed["nested"]["password"] == "[REDACTED]"
|
||||||
|
assert parsed["nested"]["access_token"] == "[REDACTED]"
|
||||||
|
assert parsed["message"] == "Authorization: Bearer [REDACTED]"
|
||||||
|
assert "hunter2" not in redacted
|
||||||
|
assert "top-secret-value" not in redacted
|
||||||
|
|
||||||
|
|
||||||
|
def test_successful_stdout_is_redacted_before_json_parsing(monkeypatch) -> None:
|
||||||
|
secret = "123456789:abcdefghijklmnopqrstuvwxyz_ABCD"
|
||||||
|
calls = iter(
|
||||||
|
[
|
||||||
|
subprocess.CompletedProcess([], 0, stdout=json.dumps({"bot_token": secret}) + "\n", stderr=""),
|
||||||
|
subprocess.CompletedProcess([], 0, stdout="", stderr=""),
|
||||||
|
]
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(suite.subprocess, "run", lambda *args, **kwargs: next(calls))
|
||||||
|
|
||||||
|
result = suite.run_remote(prompts=[])
|
||||||
|
|
||||||
|
assert result["parsed"]["bot_token"] == "[REDACTED]"
|
||||||
|
assert secret not in json.dumps(result)
|
||||||
|
|
||||||
|
|
||||||
|
def test_safety_gate_passes_only_for_complete_no_send_no_write_run() -> None:
|
||||||
|
report = safe_report()
|
||||||
|
|
||||||
|
assert suite.report_passes_safety(report) is True
|
||||||
|
assert suite.build_safety_gate(report)["failed_checks"] == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_safety_gate_rejects_write_capable_tool_and_incomplete_receipt() -> None:
|
||||||
|
report = safe_report()
|
||||||
|
report["results"][0]["database_tool_trace"]["database_tool_calls_read_only"] = False
|
||||||
|
report["results"][0]["database_tool_trace"]["calls"][0]["database_invocations"][0][
|
||||||
|
"access_mode"
|
||||||
|
] = "write"
|
||||||
|
report["execution_manifest_summary"]["all_turns_attribution_complete"] = False
|
||||||
|
|
||||||
|
gate = suite.build_safety_gate(report)
|
||||||
|
|
||||||
|
assert gate["status"] == "fail"
|
||||||
|
assert "all_tool_traces_read_only_and_bound" in gate["failed_checks"]
|
||||||
|
assert "all_turn_manifests_complete" in gate["failed_checks"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_safety_gate_rejects_missing_no_send_declaration() -> None:
|
||||||
|
report = safe_report()
|
||||||
|
report.pop("posted_to_telegram")
|
||||||
|
|
||||||
|
gate = suite.build_safety_gate(report)
|
||||||
|
|
||||||
|
assert gate["status"] == "fail"
|
||||||
|
assert "no_telegram_post" in gate["failed_checks"]
|
||||||
833
tests/test_run_local_genesis_ledger_rebuild.py
Normal file
833
tests/test_run_local_genesis_ledger_rebuild.py
Normal file
|
|
@ -0,0 +1,833 @@
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import copy
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import shutil
|
||||||
|
import stat
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
import uuid
|
||||||
|
from collections.abc import Iterator
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from ops import run_local_genesis_ledger_rebuild as rebuild
|
||||||
|
|
||||||
|
CLAIM_A = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa"
|
||||||
|
CLAIM_B = "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb"
|
||||||
|
REVIEWER_ID = "11111111-1111-4111-8111-111111111111"
|
||||||
|
SERVICE_ID = "44444444-4444-4444-4444-444444444444"
|
||||||
|
PROPOSAL_ID = "99999999-9999-4999-8999-999999999999"
|
||||||
|
EDGE_ID = "eeeeeeee-eeee-4eee-8eee-eeeeeeeeeeee"
|
||||||
|
PRIVATE_MARKER = "PRIVATE-REPLAY-MATERIAL-MUST-NOT-LEAK"
|
||||||
|
|
||||||
|
|
||||||
|
def sha256_file(path: Path) -> str:
|
||||||
|
digest = hashlib.sha256()
|
||||||
|
digest.update(path.read_bytes())
|
||||||
|
return digest.hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def write_json(path: Path, value: object) -> Path:
|
||||||
|
path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||||
|
return path
|
||||||
|
|
||||||
|
|
||||||
|
def manifest_rows(database: str = "teleo_fixture") -> list[dict]:
|
||||||
|
role = {
|
||||||
|
"name": "kb_apply",
|
||||||
|
"can_login": True,
|
||||||
|
"superuser": False,
|
||||||
|
"create_db": False,
|
||||||
|
"create_role": False,
|
||||||
|
"inherit": False,
|
||||||
|
"replication": False,
|
||||||
|
"bypass_rls": False,
|
||||||
|
}
|
||||||
|
rows = [
|
||||||
|
{
|
||||||
|
"kind": "identity",
|
||||||
|
"database": database,
|
||||||
|
"server_version_num": 160010,
|
||||||
|
"transaction_read_only": "on",
|
||||||
|
"server_address": None,
|
||||||
|
"server_port": None,
|
||||||
|
"ssl": False,
|
||||||
|
"ssl_version": None,
|
||||||
|
"database_collation": "C",
|
||||||
|
"database_ctype": "C",
|
||||||
|
},
|
||||||
|
{"kind": "schemas", "items": ["kb_stage", "public"]},
|
||||||
|
{"kind": "extensions", "items": []},
|
||||||
|
{"kind": "application_roles", "items": [role]},
|
||||||
|
]
|
||||||
|
rows.extend(
|
||||||
|
{"kind": kind, "items": []}
|
||||||
|
for kind in (
|
||||||
|
"columns",
|
||||||
|
"constraints",
|
||||||
|
"indexes",
|
||||||
|
"sequences",
|
||||||
|
"views",
|
||||||
|
"functions",
|
||||||
|
"triggers",
|
||||||
|
"types",
|
||||||
|
"policies",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
rows.extend(
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"kind": "table",
|
||||||
|
"schema": "public",
|
||||||
|
"table": "claims",
|
||||||
|
"row_count": 2,
|
||||||
|
"rowset_md5": "fixture",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"kind": "performance",
|
||||||
|
"query": "count_claims",
|
||||||
|
"elapsed_ms": 1.0,
|
||||||
|
"observed_rows": 2,
|
||||||
|
},
|
||||||
|
]
|
||||||
|
)
|
||||||
|
return rows
|
||||||
|
|
||||||
|
|
||||||
|
def write_manifest(path: Path) -> Path:
|
||||||
|
path.write_text("".join(json.dumps(row) + "\n" for row in manifest_rows()), encoding="utf-8")
|
||||||
|
return path
|
||||||
|
|
||||||
|
|
||||||
|
def proposal_rows() -> tuple[dict, dict, dict]:
|
||||||
|
payload = {
|
||||||
|
"apply_payload": {
|
||||||
|
"from_claim": CLAIM_A,
|
||||||
|
"to_claim": CLAIM_B,
|
||||||
|
"edge_type": "supports",
|
||||||
|
"weight": 0.8,
|
||||||
|
},
|
||||||
|
"private_title": PRIVATE_MARKER,
|
||||||
|
}
|
||||||
|
approved = {
|
||||||
|
"id": PROPOSAL_ID,
|
||||||
|
"proposal_type": "add_edge",
|
||||||
|
"status": "approved",
|
||||||
|
"proposed_by_handle": "fixture-agent",
|
||||||
|
"proposed_by_agent_id": REVIEWER_ID,
|
||||||
|
"channel": "reconstruction_fixture",
|
||||||
|
"source_ref": "private://fixture/source",
|
||||||
|
"rationale": PRIVATE_MARKER,
|
||||||
|
"payload": payload,
|
||||||
|
"reviewed_by_handle": "m3ta",
|
||||||
|
"reviewed_by_agent_id": REVIEWER_ID,
|
||||||
|
"reviewed_at": "2026-07-14T01:00:00+00:00",
|
||||||
|
"review_note": "Reviewed exact strict fixture payload.",
|
||||||
|
"applied_by_handle": None,
|
||||||
|
"applied_by_agent_id": None,
|
||||||
|
"applied_at": None,
|
||||||
|
"created_at": "2026-07-14T00:59:00+00:00",
|
||||||
|
"updated_at": "2026-07-14T01:00:00+00:00",
|
||||||
|
}
|
||||||
|
applied = {
|
||||||
|
**approved,
|
||||||
|
"status": "applied",
|
||||||
|
"applied_by_handle": "kb-apply",
|
||||||
|
"applied_by_agent_id": SERVICE_ID,
|
||||||
|
"applied_at": "2026-07-14T01:01:00+00:00",
|
||||||
|
"updated_at": "2026-07-14T01:01:00.000001+00:00",
|
||||||
|
}
|
||||||
|
approval = {
|
||||||
|
"proposal_id": PROPOSAL_ID,
|
||||||
|
"proposal_type": "add_edge",
|
||||||
|
"payload": payload,
|
||||||
|
"reviewed_by_handle": "m3ta",
|
||||||
|
"reviewed_by_agent_id": REVIEWER_ID,
|
||||||
|
"reviewed_by_db_role": "kb_review",
|
||||||
|
"reviewed_at": "2026-07-14T01:00:00+00:00",
|
||||||
|
"review_note": "Reviewed exact strict fixture payload.",
|
||||||
|
}
|
||||||
|
return approved, applied, approval
|
||||||
|
|
||||||
|
|
||||||
|
def applied_envelope(applied: dict) -> dict:
|
||||||
|
fields = (
|
||||||
|
"id",
|
||||||
|
"proposal_type",
|
||||||
|
"status",
|
||||||
|
"payload",
|
||||||
|
"reviewed_by_handle",
|
||||||
|
"reviewed_by_agent_id",
|
||||||
|
"reviewed_at",
|
||||||
|
"review_note",
|
||||||
|
"applied_by_handle",
|
||||||
|
"applied_by_agent_id",
|
||||||
|
"applied_at",
|
||||||
|
)
|
||||||
|
return {field: applied[field] for field in fields}
|
||||||
|
|
||||||
|
|
||||||
|
def replay_material() -> dict:
|
||||||
|
approved, applied, approval = proposal_rows()
|
||||||
|
proposal = applied_envelope(applied)
|
||||||
|
edge = {
|
||||||
|
"id": EDGE_ID,
|
||||||
|
"from_claim": CLAIM_A,
|
||||||
|
"to_claim": CLAIM_B,
|
||||||
|
"edge_type": "supports",
|
||||||
|
"weight": 0.8,
|
||||||
|
"created_by": None,
|
||||||
|
"created_at": "2026-07-14T01:00:30+00:00",
|
||||||
|
}
|
||||||
|
receipt = rebuild.replay_receipt.build_replay_receipt(
|
||||||
|
proposal,
|
||||||
|
{"claim_edges": [edge]},
|
||||||
|
apply_sql_sha256=rebuild.replay_receipt.sha256_text("fixture original guarded apply SQL"),
|
||||||
|
apply_sql_source="exact_executed_sql",
|
||||||
|
exported_at_utc="2026-07-14T01:02:00+00:00",
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
"artifact": rebuild.MATERIAL_ARTIFACT,
|
||||||
|
"contract_version": rebuild.MATERIAL_CONTRACT_VERSION,
|
||||||
|
"sequence": 1,
|
||||||
|
"approved_proposal": approved,
|
||||||
|
"approval_snapshot": approval,
|
||||||
|
"applied_proposal": applied,
|
||||||
|
"replay_receipt": receipt,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def write_bundle(tmp_path: Path, *, material: dict | None = None) -> argparse.Namespace:
|
||||||
|
dump = tmp_path / "genesis.dump"
|
||||||
|
dump.write_bytes(b"PGDMPfixture")
|
||||||
|
genesis_manifest = write_manifest(tmp_path / "genesis-manifest.jsonl")
|
||||||
|
final_manifest = write_manifest(tmp_path / "final-manifest.jsonl")
|
||||||
|
material = copy.deepcopy(material or replay_material())
|
||||||
|
material_path = write_json(tmp_path / "entry-0001.json", material)
|
||||||
|
receipt_hash = material["replay_receipt"]["hashes"]["replay_material_sha256"]
|
||||||
|
ledger = {
|
||||||
|
"artifact": rebuild.LEDGER_ARTIFACT,
|
||||||
|
"contract_version": rebuild.LEDGER_CONTRACT_VERSION,
|
||||||
|
"engine": rebuild._engine_hashes(),
|
||||||
|
"genesis": {
|
||||||
|
"dump_sha256": sha256_file(dump),
|
||||||
|
"parity_manifest_sha256": sha256_file(genesis_manifest),
|
||||||
|
},
|
||||||
|
"entries": [
|
||||||
|
{
|
||||||
|
"sequence": 1,
|
||||||
|
"material": material_path.name,
|
||||||
|
"sha256": sha256_file(material_path),
|
||||||
|
"replay_material_sha256": receipt_hash,
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"final_parity": {
|
||||||
|
"manifest": final_manifest.name,
|
||||||
|
"sha256": sha256_file(final_manifest),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
ledger_path = write_json(tmp_path / "ledger.json", ledger)
|
||||||
|
return argparse.Namespace(
|
||||||
|
genesis_dump=dump,
|
||||||
|
genesis_manifest=genesis_manifest,
|
||||||
|
ledger=ledger_path,
|
||||||
|
ledger_sha256=sha256_file(ledger_path),
|
||||||
|
database="teleo",
|
||||||
|
image=rebuild.canonical_rebuild.DEFAULT_IMAGE,
|
||||||
|
container_prefix="teleo-genesis-ledger-test",
|
||||||
|
tmpfs_mb=256,
|
||||||
|
startup_timeout=30.0,
|
||||||
|
restore_timeout=60.0,
|
||||||
|
apply_timeout=60.0,
|
||||||
|
manifest_timeout=60.0,
|
||||||
|
command_timeout=20.0,
|
||||||
|
poll_interval=0.05,
|
||||||
|
max_target_query_ms=5000.0,
|
||||||
|
max_target_source_ratio=100.0,
|
||||||
|
docker_bin="docker",
|
||||||
|
output=tmp_path / "receipt.json",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def rewrite_material_and_ledger(args: argparse.Namespace, material: dict) -> None:
|
||||||
|
material_path = args.ledger.parent / "entry-0001.json"
|
||||||
|
write_json(material_path, material)
|
||||||
|
ledger = json.loads(args.ledger.read_text(encoding="utf-8"))
|
||||||
|
ledger["entries"][0]["sha256"] = sha256_file(material_path)
|
||||||
|
write_json(args.ledger, ledger)
|
||||||
|
args.ledger_sha256 = sha256_file(args.ledger)
|
||||||
|
|
||||||
|
|
||||||
|
def test_load_bundle_validates_every_pin_and_current_apply_engine(tmp_path: Path) -> None:
|
||||||
|
args = write_bundle(tmp_path)
|
||||||
|
|
||||||
|
bundle = rebuild.load_bundle(args)
|
||||||
|
|
||||||
|
assert bundle.genesis_dump_sha256 == sha256_file(args.genesis_dump)
|
||||||
|
assert bundle.genesis_manifest_sha256 == sha256_file(args.genesis_manifest)
|
||||||
|
assert bundle.final_manifest_sha256 == sha256_file(tmp_path / "final-manifest.jsonl")
|
||||||
|
assert bundle.engine_hashes == rebuild._engine_hashes()
|
||||||
|
assert len(bundle.entries) == 1
|
||||||
|
entry = bundle.entries[0]
|
||||||
|
assert entry.applied_proposal["proposal_type"] == "add_edge"
|
||||||
|
assert entry.current_apply_sql_sha256 == hashlib.sha256(entry.current_apply_sql.encode("utf-8")).hexdigest()
|
||||||
|
assert "kb_stage.assert_approved_proposal" in entry.current_apply_sql
|
||||||
|
|
||||||
|
|
||||||
|
def test_dump_hash_mismatch_fails_before_container_start(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
args = write_bundle(tmp_path)
|
||||||
|
args.genesis_dump.write_bytes(b"PGDMPtampered")
|
||||||
|
commands: list[list[str]] = []
|
||||||
|
|
||||||
|
def absent_docker(command: list[str], *, timeout: float):
|
||||||
|
commands.append(command)
|
||||||
|
if command[1:3] == ["rm", "--force"]:
|
||||||
|
return subprocess.CompletedProcess(command, 1, "", "No such container")
|
||||||
|
if command[1:3] == ["container", "inspect"]:
|
||||||
|
return subprocess.CompletedProcess(command, 1, "", "No such object")
|
||||||
|
raise AssertionError(f"unexpected command after failed preflight: {command}")
|
||||||
|
|
||||||
|
monkeypatch.setattr(rebuild.canonical_rebuild, "_run", absent_docker)
|
||||||
|
|
||||||
|
receipt = rebuild.run_reconstruction(args)
|
||||||
|
|
||||||
|
assert receipt["status"] == "fail"
|
||||||
|
assert receipt["error"]["code"] == "hash_pin_mismatch"
|
||||||
|
assert receipt["cleanup"]["container_absent"] is True
|
||||||
|
assert not any(len(command) > 1 and command[1] == "run" for command in commands)
|
||||||
|
|
||||||
|
|
||||||
|
def test_revise_strategy_receipt_fails_closed_before_receipt_rows_are_used(tmp_path: Path) -> None:
|
||||||
|
material = replay_material()
|
||||||
|
material["replay_receipt"]["proposal"]["proposal_type"] = "revise_strategy"
|
||||||
|
material["applied_proposal"]["proposal_type"] = "revise_strategy"
|
||||||
|
material["approved_proposal"]["proposal_type"] = "revise_strategy"
|
||||||
|
material["approval_snapshot"]["proposal_type"] = "revise_strategy"
|
||||||
|
args = write_bundle(tmp_path, material=material)
|
||||||
|
|
||||||
|
with pytest.raises(rebuild.ReconstructionError, match="prior strategy") as exc_info:
|
||||||
|
rebuild.load_bundle(args)
|
||||||
|
|
||||||
|
assert exc_info.value.code == "unsupported_mutating_receipt"
|
||||||
|
|
||||||
|
|
||||||
|
def test_legacy_freeform_receipt_fails_closed(tmp_path: Path) -> None:
|
||||||
|
material = replay_material()
|
||||||
|
material["replay_receipt"]["proposal"]["payload"] = {"legacy": PRIVATE_MARKER}
|
||||||
|
material["approved_proposal"]["payload"] = {"legacy": PRIVATE_MARKER}
|
||||||
|
material["applied_proposal"]["payload"] = {"legacy": PRIVATE_MARKER}
|
||||||
|
material["approval_snapshot"]["payload"] = {"legacy": PRIVATE_MARKER}
|
||||||
|
args = write_bundle(tmp_path, material=material)
|
||||||
|
|
||||||
|
with pytest.raises(rebuild.ReconstructionError) as exc_info:
|
||||||
|
rebuild.load_bundle(args)
|
||||||
|
|
||||||
|
assert exc_info.value.code == "invalid_replay_receipt"
|
||||||
|
assert PRIVATE_MARKER not in exc_info.value.safe_message
|
||||||
|
|
||||||
|
|
||||||
|
def test_proposal_metadata_outside_apply_transition_is_rejected(tmp_path: Path) -> None:
|
||||||
|
material = replay_material()
|
||||||
|
material["applied_proposal"]["rationale"] = "silently changed after review"
|
||||||
|
args = write_bundle(tmp_path)
|
||||||
|
rewrite_material_and_ledger(args, material)
|
||||||
|
|
||||||
|
with pytest.raises(rebuild.ReconstructionError) as exc_info:
|
||||||
|
rebuild.load_bundle(args)
|
||||||
|
|
||||||
|
assert exc_info.value.code == "material_mismatch"
|
||||||
|
assert "rationale" in exc_info.value.safe_message
|
||||||
|
|
||||||
|
|
||||||
|
def test_output_cannot_overwrite_a_fixed_guard_or_engine_file(tmp_path: Path) -> None:
|
||||||
|
args = write_bundle(tmp_path)
|
||||||
|
args.output = rebuild.GUARD_PREREQUISITES_PATH
|
||||||
|
|
||||||
|
with pytest.raises(rebuild.ReconstructionError) as exc_info:
|
||||||
|
rebuild.load_bundle(args)
|
||||||
|
|
||||||
|
assert exc_info.value.code == "unsafe_output_path"
|
||||||
|
|
||||||
|
|
||||||
|
def test_unknown_private_field_name_is_not_echoed_in_public_error(tmp_path: Path) -> None:
|
||||||
|
material = replay_material()
|
||||||
|
material[PRIVATE_MARKER] = "private value"
|
||||||
|
args = write_bundle(tmp_path, material=material)
|
||||||
|
|
||||||
|
with pytest.raises(rebuild.ReconstructionError) as exc_info:
|
||||||
|
rebuild.load_bundle(args)
|
||||||
|
|
||||||
|
assert exc_info.value.code == "invalid_contract"
|
||||||
|
assert PRIVATE_MARKER not in exc_info.value.safe_message
|
||||||
|
|
||||||
|
|
||||||
|
def test_seed_sql_preserves_exact_rows_and_uses_only_fixed_tables(tmp_path: Path) -> None:
|
||||||
|
entry = rebuild.load_bundle(write_bundle(tmp_path)).entries[0]
|
||||||
|
|
||||||
|
sql = rebuild.build_seed_sql(entry)
|
||||||
|
|
||||||
|
assert "jsonb_populate_record" in sql
|
||||||
|
assert "kb_stage.kb_proposals" in sql
|
||||||
|
assert "kb_stage.kb_proposal_approvals" in sql
|
||||||
|
assert "public.claim_edges" in sql
|
||||||
|
assert "on conflict do nothing" in sql
|
||||||
|
assert PRIVATE_MARKER in sql
|
||||||
|
assert "public.strategies" not in sql
|
||||||
|
|
||||||
|
|
||||||
|
def test_private_psql_failure_output_is_replaced_with_a_fingerprint(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
) -> None:
|
||||||
|
captured: dict[str, object] = {}
|
||||||
|
|
||||||
|
def fail(command: list[str], *, stdin: str, timeout: float):
|
||||||
|
captured.update(command=command, stdin=stdin, timeout=timeout)
|
||||||
|
return subprocess.CompletedProcess(command, 1, "", f"ERROR: {PRIVATE_MARKER}")
|
||||||
|
|
||||||
|
monkeypatch.setattr(rebuild, "_run_stdin", fail)
|
||||||
|
args = argparse.Namespace(docker_bin="docker", database="teleo")
|
||||||
|
|
||||||
|
with pytest.raises(rebuild.ReconstructionError) as exc_info:
|
||||||
|
rebuild._psql(
|
||||||
|
args,
|
||||||
|
"fixture-container",
|
||||||
|
f"select '{PRIVATE_MARKER}';",
|
||||||
|
role="postgres",
|
||||||
|
timeout=3.0,
|
||||||
|
code="fixture_failure",
|
||||||
|
action="private fixture SQL",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert PRIVATE_MARKER not in exc_info.value.safe_message
|
||||||
|
assert "diagnostic_sha256=" in exc_info.value.safe_message
|
||||||
|
assert PRIVATE_MARKER not in " ".join(captured["command"])
|
||||||
|
assert PRIVATE_MARKER in captured["stdin"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_entry_summary_is_secret_safe(tmp_path: Path) -> None:
|
||||||
|
entry = rebuild.load_bundle(write_bundle(tmp_path)).entries[0]
|
||||||
|
|
||||||
|
encoded = json.dumps(rebuild._entry_summary(entry), sort_keys=True)
|
||||||
|
|
||||||
|
assert PRIVATE_MARKER not in encoded
|
||||||
|
assert entry.applied_proposal["id"] in encoded
|
||||||
|
assert entry.replay_material_sha256 in encoded
|
||||||
|
|
||||||
|
|
||||||
|
BOOTSTRAP_SQL = f"""
|
||||||
|
create role kb_apply login noinherit;
|
||||||
|
create role kb_review login noinherit;
|
||||||
|
create schema kb_stage;
|
||||||
|
create type evidence_role as enum ('grounds', 'illustrates', 'contradicts');
|
||||||
|
create type edge_type as enum (
|
||||||
|
'supports', 'challenges', 'requires', 'relates', 'contradicts', 'supersedes',
|
||||||
|
'derives_from', 'cites', 'causes', 'constrains', 'accelerates'
|
||||||
|
);
|
||||||
|
|
||||||
|
create table public.agents (
|
||||||
|
id uuid primary key,
|
||||||
|
handle text not null unique,
|
||||||
|
kind text not null
|
||||||
|
);
|
||||||
|
create table public.claims (
|
||||||
|
id uuid primary key,
|
||||||
|
type text not null,
|
||||||
|
text text not null,
|
||||||
|
status text not null,
|
||||||
|
confidence numeric,
|
||||||
|
tags text[] not null default '{{}}',
|
||||||
|
created_by uuid references public.agents(id),
|
||||||
|
superseded_by uuid references public.claims(id),
|
||||||
|
created_at timestamptz not null default now(),
|
||||||
|
updated_at timestamptz not null default now()
|
||||||
|
);
|
||||||
|
create table public.sources (
|
||||||
|
id uuid primary key,
|
||||||
|
source_type text not null,
|
||||||
|
url text,
|
||||||
|
storage_path text,
|
||||||
|
excerpt text,
|
||||||
|
hash text not null,
|
||||||
|
created_by uuid references public.agents(id),
|
||||||
|
captured_at timestamptz,
|
||||||
|
created_at timestamptz not null default now()
|
||||||
|
);
|
||||||
|
create table public.claim_evidence (
|
||||||
|
id uuid primary key,
|
||||||
|
claim_id uuid not null references public.claims(id),
|
||||||
|
source_id uuid not null references public.sources(id),
|
||||||
|
role evidence_role not null,
|
||||||
|
weight numeric,
|
||||||
|
created_by uuid references public.agents(id),
|
||||||
|
created_at timestamptz not null default now()
|
||||||
|
);
|
||||||
|
create table public.claim_edges (
|
||||||
|
id uuid primary key,
|
||||||
|
from_claim uuid not null references public.claims(id),
|
||||||
|
to_claim uuid not null references public.claims(id),
|
||||||
|
edge_type edge_type not null,
|
||||||
|
weight numeric,
|
||||||
|
created_by uuid references public.agents(id),
|
||||||
|
created_at timestamptz not null default now()
|
||||||
|
);
|
||||||
|
create table public.reasoning_tools (
|
||||||
|
id uuid primary key,
|
||||||
|
agent_id uuid references public.agents(id),
|
||||||
|
name text not null,
|
||||||
|
description text not null,
|
||||||
|
category text,
|
||||||
|
created_at timestamptz not null default now()
|
||||||
|
);
|
||||||
|
create table public.strategies (
|
||||||
|
id uuid primary key,
|
||||||
|
agent_id uuid not null references public.agents(id),
|
||||||
|
diagnosis text,
|
||||||
|
guiding_policy text,
|
||||||
|
proximate_objectives jsonb,
|
||||||
|
version integer not null default 1,
|
||||||
|
active boolean not null default true,
|
||||||
|
created_at timestamptz not null default now()
|
||||||
|
);
|
||||||
|
create table public.strategy_nodes (
|
||||||
|
id uuid primary key,
|
||||||
|
agent_id uuid references public.agents(id),
|
||||||
|
node_type text,
|
||||||
|
title text,
|
||||||
|
body text,
|
||||||
|
rank integer,
|
||||||
|
status text not null default 'active',
|
||||||
|
horizon text,
|
||||||
|
measure text,
|
||||||
|
source_ref text,
|
||||||
|
metadata jsonb not null default '{{}}',
|
||||||
|
created_at timestamptz not null default now(),
|
||||||
|
updated_at timestamptz not null default now()
|
||||||
|
);
|
||||||
|
create table kb_stage.kb_proposals (
|
||||||
|
id uuid primary key,
|
||||||
|
proposal_type text not null,
|
||||||
|
status text not null,
|
||||||
|
proposed_by_handle text,
|
||||||
|
proposed_by_agent_id uuid references public.agents(id),
|
||||||
|
channel text,
|
||||||
|
source_ref text,
|
||||||
|
rationale text,
|
||||||
|
payload jsonb not null,
|
||||||
|
reviewed_by_handle text,
|
||||||
|
reviewed_by_agent_id uuid references public.agents(id),
|
||||||
|
reviewed_at timestamptz,
|
||||||
|
review_note text,
|
||||||
|
applied_by_handle text,
|
||||||
|
applied_by_agent_id uuid references public.agents(id),
|
||||||
|
applied_at timestamptz,
|
||||||
|
created_at timestamptz not null default now(),
|
||||||
|
updated_at timestamptz not null default now()
|
||||||
|
);
|
||||||
|
|
||||||
|
insert into public.agents (id, handle, kind) values
|
||||||
|
('{REVIEWER_ID}', 'm3ta', 'human');
|
||||||
|
insert into public.claims (id, type, text, status, confidence, tags, created_by) values
|
||||||
|
('{CLAIM_A}', 'structural', 'Fixture claim A', 'open', 0.8, array['fixture'], '{REVIEWER_ID}'),
|
||||||
|
('{CLAIM_B}', 'structural', 'Fixture claim B', 'open', 0.8, array['fixture'], '{REVIEWER_ID}');
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
def docker_psql(
|
||||||
|
container: str,
|
||||||
|
database: str,
|
||||||
|
sql: str,
|
||||||
|
*,
|
||||||
|
role: str = "postgres",
|
||||||
|
check: bool = True,
|
||||||
|
) -> subprocess.CompletedProcess[str]:
|
||||||
|
completed = subprocess.run(
|
||||||
|
[
|
||||||
|
"docker",
|
||||||
|
"exec",
|
||||||
|
"-i",
|
||||||
|
container,
|
||||||
|
"psql",
|
||||||
|
"-X",
|
||||||
|
"-U",
|
||||||
|
role,
|
||||||
|
"-h",
|
||||||
|
"127.0.0.1",
|
||||||
|
"-d",
|
||||||
|
database,
|
||||||
|
"-Atq",
|
||||||
|
"-v",
|
||||||
|
"ON_ERROR_STOP=1",
|
||||||
|
],
|
||||||
|
input=sql,
|
||||||
|
text=True,
|
||||||
|
capture_output=True,
|
||||||
|
check=False,
|
||||||
|
)
|
||||||
|
if check and completed.returncode != 0:
|
||||||
|
pytest.fail(
|
||||||
|
f"fixture psql failed ({completed.returncode})\nstdout:\n{completed.stdout}\nstderr:\n{completed.stderr}"
|
||||||
|
)
|
||||||
|
return completed
|
||||||
|
|
||||||
|
|
||||||
|
def wait_for_postgres(container: str, database: str) -> None:
|
||||||
|
for _ in range(120):
|
||||||
|
completed = docker_psql(container, database, "select 1;", check=False)
|
||||||
|
if completed.returncode == 0:
|
||||||
|
return
|
||||||
|
time.sleep(0.25)
|
||||||
|
pytest.fail("fixture PostgreSQL did not become ready")
|
||||||
|
|
||||||
|
|
||||||
|
def capture_manifest(container: str, database: str, destination: Path) -> None:
|
||||||
|
copy_result = subprocess.run(
|
||||||
|
["docker", "cp", str(rebuild.PARITY_SQL_PATH), f"{container}:/tmp/parity.sql"],
|
||||||
|
text=True,
|
||||||
|
capture_output=True,
|
||||||
|
check=False,
|
||||||
|
)
|
||||||
|
if copy_result.returncode != 0:
|
||||||
|
pytest.fail(f"could not copy parity SQL: {copy_result.stderr}")
|
||||||
|
completed = subprocess.run(
|
||||||
|
[
|
||||||
|
"docker",
|
||||||
|
"exec",
|
||||||
|
container,
|
||||||
|
"psql",
|
||||||
|
"-X",
|
||||||
|
"-U",
|
||||||
|
"postgres",
|
||||||
|
"-d",
|
||||||
|
database,
|
||||||
|
"-Atq",
|
||||||
|
"-v",
|
||||||
|
"ON_ERROR_STOP=1",
|
||||||
|
"-f",
|
||||||
|
"/tmp/parity.sql",
|
||||||
|
],
|
||||||
|
text=True,
|
||||||
|
capture_output=True,
|
||||||
|
check=False,
|
||||||
|
)
|
||||||
|
if completed.returncode != 0:
|
||||||
|
pytest.fail(f"could not capture parity manifest: {completed.stderr}")
|
||||||
|
destination.write_text(completed.stdout, encoding="utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(scope="module")
|
||||||
|
def source_postgres() -> Iterator[tuple[str, str]]:
|
||||||
|
if shutil.which("docker") is None:
|
||||||
|
pytest.skip("Docker is required for the genesis-plus-ledger lifecycle test")
|
||||||
|
container = f"genesis-ledger-source-{uuid.uuid4().hex[:12]}"
|
||||||
|
database = "teleo_fixture"
|
||||||
|
started = subprocess.run(
|
||||||
|
[
|
||||||
|
"docker",
|
||||||
|
"run",
|
||||||
|
"--rm",
|
||||||
|
"--detach",
|
||||||
|
"--name",
|
||||||
|
container,
|
||||||
|
"--env",
|
||||||
|
f"POSTGRES_DB={database}",
|
||||||
|
"--env",
|
||||||
|
"POSTGRES_HOST_AUTH_METHOD=trust",
|
||||||
|
rebuild.canonical_rebuild.DEFAULT_IMAGE,
|
||||||
|
],
|
||||||
|
text=True,
|
||||||
|
capture_output=True,
|
||||||
|
check=False,
|
||||||
|
)
|
||||||
|
if started.returncode != 0:
|
||||||
|
pytest.fail(f"could not start fixture PostgreSQL: {started.stderr}")
|
||||||
|
try:
|
||||||
|
wait_for_postgres(container, database)
|
||||||
|
docker_psql(container, database, BOOTSTRAP_SQL)
|
||||||
|
copy_result = subprocess.run(
|
||||||
|
[
|
||||||
|
"docker",
|
||||||
|
"cp",
|
||||||
|
str(rebuild.GUARD_PREREQUISITES_PATH),
|
||||||
|
f"{container}:/tmp/kb-apply-prereqs.sql",
|
||||||
|
],
|
||||||
|
text=True,
|
||||||
|
capture_output=True,
|
||||||
|
check=False,
|
||||||
|
)
|
||||||
|
if copy_result.returncode != 0:
|
||||||
|
pytest.fail(f"could not copy apply prerequisites: {copy_result.stderr}")
|
||||||
|
migration = subprocess.run(
|
||||||
|
[
|
||||||
|
"docker",
|
||||||
|
"exec",
|
||||||
|
container,
|
||||||
|
"psql",
|
||||||
|
"-X",
|
||||||
|
"-U",
|
||||||
|
"postgres",
|
||||||
|
"-d",
|
||||||
|
database,
|
||||||
|
"-Atq",
|
||||||
|
"-v",
|
||||||
|
"ON_ERROR_STOP=1",
|
||||||
|
"-f",
|
||||||
|
"/tmp/kb-apply-prereqs.sql",
|
||||||
|
],
|
||||||
|
text=True,
|
||||||
|
capture_output=True,
|
||||||
|
check=False,
|
||||||
|
)
|
||||||
|
if migration.returncode != 0:
|
||||||
|
pytest.fail(f"could not apply guarded prerequisites: {migration.stderr}")
|
||||||
|
yield container, database
|
||||||
|
finally:
|
||||||
|
subprocess.run(
|
||||||
|
["docker", "rm", "--force", container],
|
||||||
|
text=True,
|
||||||
|
capture_output=True,
|
||||||
|
check=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_live_genesis_plus_ledger_command_rebuilds_exactly_and_proves_cleanup(
|
||||||
|
tmp_path: Path,
|
||||||
|
source_postgres: tuple[str, str],
|
||||||
|
) -> None:
|
||||||
|
source_container, database = source_postgres
|
||||||
|
genesis_dump = tmp_path / "genesis.dump"
|
||||||
|
dump_result = subprocess.run(
|
||||||
|
[
|
||||||
|
"docker",
|
||||||
|
"exec",
|
||||||
|
source_container,
|
||||||
|
"pg_dump",
|
||||||
|
"-U",
|
||||||
|
"postgres",
|
||||||
|
"-d",
|
||||||
|
database,
|
||||||
|
"--format=custom",
|
||||||
|
"--file=/tmp/genesis.dump",
|
||||||
|
],
|
||||||
|
text=True,
|
||||||
|
capture_output=True,
|
||||||
|
check=False,
|
||||||
|
)
|
||||||
|
assert dump_result.returncode == 0, dump_result.stderr
|
||||||
|
copy_result = subprocess.run(
|
||||||
|
["docker", "cp", f"{source_container}:/tmp/genesis.dump", str(genesis_dump)],
|
||||||
|
text=True,
|
||||||
|
capture_output=True,
|
||||||
|
check=False,
|
||||||
|
)
|
||||||
|
assert copy_result.returncode == 0, copy_result.stderr
|
||||||
|
|
||||||
|
genesis_manifest = tmp_path / "genesis-manifest.jsonl"
|
||||||
|
capture_manifest(source_container, database, genesis_manifest)
|
||||||
|
|
||||||
|
material = replay_material()
|
||||||
|
material_path = write_json(tmp_path / "entry-0001.json", material)
|
||||||
|
entry = rebuild._validate_entry_material(
|
||||||
|
material,
|
||||||
|
expected_sequence=1,
|
||||||
|
expected_replay_hash=material["replay_receipt"]["hashes"]["replay_material_sha256"],
|
||||||
|
material_path=material_path,
|
||||||
|
material_sha256=sha256_file(material_path),
|
||||||
|
)
|
||||||
|
docker_psql(source_container, database, rebuild.build_seed_sql(entry))
|
||||||
|
docker_psql(
|
||||||
|
source_container,
|
||||||
|
database,
|
||||||
|
entry.current_apply_sql,
|
||||||
|
role="kb_apply",
|
||||||
|
)
|
||||||
|
docker_psql(
|
||||||
|
source_container,
|
||||||
|
database,
|
||||||
|
rebuild.build_applied_timestamp_normalization_sql(entry),
|
||||||
|
)
|
||||||
|
|
||||||
|
final_manifest = tmp_path / "final-manifest.jsonl"
|
||||||
|
capture_manifest(source_container, database, final_manifest)
|
||||||
|
ledger = {
|
||||||
|
"artifact": rebuild.LEDGER_ARTIFACT,
|
||||||
|
"contract_version": rebuild.LEDGER_CONTRACT_VERSION,
|
||||||
|
"engine": rebuild._engine_hashes(),
|
||||||
|
"genesis": {
|
||||||
|
"dump_sha256": sha256_file(genesis_dump),
|
||||||
|
"parity_manifest_sha256": sha256_file(genesis_manifest),
|
||||||
|
},
|
||||||
|
"entries": [
|
||||||
|
{
|
||||||
|
"sequence": 1,
|
||||||
|
"material": material_path.name,
|
||||||
|
"sha256": sha256_file(material_path),
|
||||||
|
"replay_material_sha256": entry.replay_material_sha256,
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"final_parity": {
|
||||||
|
"manifest": final_manifest.name,
|
||||||
|
"sha256": sha256_file(final_manifest),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
ledger_path = write_json(tmp_path / "ledger.json", ledger)
|
||||||
|
output = tmp_path / "reconstruction-receipt.json"
|
||||||
|
command = [
|
||||||
|
sys.executable,
|
||||||
|
str(Path(rebuild.__file__).resolve()),
|
||||||
|
"--genesis-dump",
|
||||||
|
str(genesis_dump),
|
||||||
|
"--genesis-manifest",
|
||||||
|
str(genesis_manifest),
|
||||||
|
"--ledger",
|
||||||
|
str(ledger_path),
|
||||||
|
"--ledger-sha256",
|
||||||
|
sha256_file(ledger_path),
|
||||||
|
"--database",
|
||||||
|
database,
|
||||||
|
"--container-prefix",
|
||||||
|
"teleo-genesis-ledger-live-test",
|
||||||
|
"--tmpfs-mb",
|
||||||
|
"256",
|
||||||
|
"--max-target-query-ms",
|
||||||
|
"5000",
|
||||||
|
"--max-target-source-ratio",
|
||||||
|
"100",
|
||||||
|
"--output",
|
||||||
|
str(output),
|
||||||
|
]
|
||||||
|
completed = subprocess.run(
|
||||||
|
command,
|
||||||
|
cwd=rebuild.REPO_ROOT,
|
||||||
|
text=True,
|
||||||
|
capture_output=True,
|
||||||
|
check=False,
|
||||||
|
timeout=180,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert completed.returncode == 0, completed.stderr + completed.stdout
|
||||||
|
receipt = json.loads(output.read_text(encoding="utf-8"))
|
||||||
|
assert receipt["status"] == "pass"
|
||||||
|
assert receipt["genesis_parity"]["status"] == "pass"
|
||||||
|
assert receipt["ledger"]["entries"][0]["status"] == "pass"
|
||||||
|
assert receipt["ledger"]["entries"][0]["guarded_apply_executed"] is True
|
||||||
|
assert receipt["ledger"]["entries"][0]["canonical_rows_exact"] is True
|
||||||
|
assert receipt["final_parity"]["status"] == "pass"
|
||||||
|
assert receipt["cleanup"]["container_absent"] is True
|
||||||
|
assert receipt["safety"]["network_access_available_to_container"] is False
|
||||||
|
assert receipt["safety"]["private_replay_material_emitted"] is False
|
||||||
|
assert PRIVATE_MARKER not in completed.stdout
|
||||||
|
assert PRIVATE_MARKER not in output.read_text(encoding="utf-8")
|
||||||
|
assert stat.S_IMODE(output.stat().st_mode) == 0o600
|
||||||
|
|
||||||
|
inspect = subprocess.run(
|
||||||
|
["docker", "container", "inspect", receipt["container"]["name"]],
|
||||||
|
text=True,
|
||||||
|
capture_output=True,
|
||||||
|
check=False,
|
||||||
|
)
|
||||||
|
assert inspect.returncode != 0
|
||||||
|
|
@ -46,6 +46,7 @@ def passing_report() -> dict:
|
||||||
"NRestarts": "0",
|
"NRestarts": "0",
|
||||||
}
|
}
|
||||||
counts = {"public.claims": 1, "public.sources": 2}
|
counts = {"public.claims": 1, "public.sources": 2}
|
||||||
|
fingerprint = {"fingerprint_sha256": "d" * 64}
|
||||||
behavior = {"behavior_sha256": "b" * 64}
|
behavior = {"behavior_sha256": "b" * 64}
|
||||||
return {
|
return {
|
||||||
"generated_at_utc": "2026-07-14T08:05:27+00:00",
|
"generated_at_utc": "2026-07-14T08:05:27+00:00",
|
||||||
|
|
@ -55,11 +56,16 @@ def passing_report() -> dict:
|
||||||
"db_counts_changed": False,
|
"db_counts_changed": False,
|
||||||
"db_counts_before": counts,
|
"db_counts_before": counts,
|
||||||
"db_counts_after": counts,
|
"db_counts_after": counts,
|
||||||
|
"db_fingerprint_before": fingerprint,
|
||||||
|
"db_fingerprint_after": fingerprint,
|
||||||
|
"db_fingerprint_unchanged": True,
|
||||||
"changed_live_profile": False,
|
"changed_live_profile": False,
|
||||||
"live_behavior_manifest_unchanged": True,
|
"live_behavior_manifest_unchanged": True,
|
||||||
"live_behavior_manifest_before": behavior,
|
"live_behavior_manifest_before": behavior,
|
||||||
"live_behavior_manifest_after": behavior,
|
"live_behavior_manifest_after": behavior,
|
||||||
"database_response_validation_all_ok": True,
|
"database_response_validation_all_ok": True,
|
||||||
|
"execution_manifest_summary": {"all_turns_attribution_complete": True},
|
||||||
|
"safety_gate": {"status": "pass"},
|
||||||
"temp_profile_removed": True,
|
"temp_profile_removed": True,
|
||||||
"service_before_after": {
|
"service_before_after": {
|
||||||
"before": service,
|
"before": service,
|
||||||
|
|
|
||||||
135
tests/test_verify_leo_unseen_reasoning_chain.py
Normal file
135
tests/test_verify_leo_unseen_reasoning_chain.py
Normal file
|
|
@ -0,0 +1,135 @@
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import copy
|
||||||
|
|
||||||
|
from scripts import verify_leo_unseen_reasoning_chain as verifier
|
||||||
|
|
||||||
|
CLAIM_ID = "fd159490-280d-4ede-84ef-faa169cff766"
|
||||||
|
|
||||||
|
|
||||||
|
def execution_manifest(execution_sha256: str, previous: str | None, *, first: bool) -> dict:
|
||||||
|
return {
|
||||||
|
"execution_sha256": execution_sha256,
|
||||||
|
"attribution": {"status": "complete"},
|
||||||
|
"session_boundary": {
|
||||||
|
"conversation": {
|
||||||
|
"previous_execution_sha256": previous,
|
||||||
|
"prior_turn_state_bound": True,
|
||||||
|
"history_prefix_preserved": True,
|
||||||
|
"starts_from_empty_conversation": first,
|
||||||
|
}
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def passing_report() -> dict:
|
||||||
|
replies = [
|
||||||
|
(
|
||||||
|
"The claim fd159490 has confidence 0.9. Its grounds row is no_source_pointer and the only other support "
|
||||||
|
"is an internal document with no external primary source. First narrower test: test one tradition."
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"Assumptions versus observed support: the categorical leap is assumed. Falsifier: one contrary case. "
|
||||||
|
"PROPOSAL A is empirical. PROPOSAL B is structural. Both are staged only; nothing applied. fd159490."
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"Revised Proposal A separates the mechanism from the outcome. Ostrom, Governing the Commons, is the "
|
||||||
|
"primary per-case source. Stage pending_review, then human review; approved is not apply, and applied_at "
|
||||||
|
"must be non-null. Nothing staged or changed here."
|
||||||
|
),
|
||||||
|
]
|
||||||
|
fingerprints = {
|
||||||
|
"fingerprint_sha256": "f" * 64,
|
||||||
|
"table_count": 39,
|
||||||
|
"total_rows": 52167,
|
||||||
|
}
|
||||||
|
first_sha = "a" * 64
|
||||||
|
second_sha = "b" * 64
|
||||||
|
third_sha = "c" * 64
|
||||||
|
return {
|
||||||
|
"remote_run_id": "run-1",
|
||||||
|
"db_counts_changed": False,
|
||||||
|
"db_fingerprint_unchanged": True,
|
||||||
|
"db_fingerprint_before": fingerprints,
|
||||||
|
"db_fingerprint_after": fingerprints,
|
||||||
|
"safety_gate": {"status": "pass"},
|
||||||
|
"execution_manifest_summary": {
|
||||||
|
"all_turns_attribution_complete": True,
|
||||||
|
"harness_source": {"git_head": "d" * 40},
|
||||||
|
},
|
||||||
|
"results": [
|
||||||
|
{
|
||||||
|
"prompt_id": verifier.PROMPTS[0]["id"],
|
||||||
|
"prompt": verifier.PROMPTS[0]["message"],
|
||||||
|
"reply": replies[0],
|
||||||
|
"database_tool_trace": {
|
||||||
|
"calls": [
|
||||||
|
{
|
||||||
|
"database_invocations": [{"subcommand": "context"}],
|
||||||
|
"result": {"row_ids": [CLAIM_ID]},
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"execution_manifest": execution_manifest(first_sha, None, first=True),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"prompt_id": verifier.PROMPTS[1]["id"],
|
||||||
|
"prompt": verifier.PROMPTS[1]["message"],
|
||||||
|
"reply": replies[1],
|
||||||
|
"database_tool_trace": {
|
||||||
|
"calls": [
|
||||||
|
{
|
||||||
|
"database_invocations": [
|
||||||
|
{"subcommand": "show"},
|
||||||
|
{"subcommand": "evidence"},
|
||||||
|
],
|
||||||
|
"result": {"row_ids": [CLAIM_ID]},
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"execution_manifest": execution_manifest(second_sha, first_sha, first=False),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"prompt_id": verifier.PROMPTS[2]["id"],
|
||||||
|
"prompt": verifier.PROMPTS[2]["message"],
|
||||||
|
"reply": replies[2],
|
||||||
|
"database_tool_trace": {"calls": []},
|
||||||
|
"execution_manifest": execution_manifest(third_sha, second_sha, first=False),
|
||||||
|
},
|
||||||
|
],
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def test_unseen_reasoning_chain_passes_only_with_db_challenge_revision_and_apply_boundary() -> None:
|
||||||
|
receipt = verifier.verify_report(passing_report(), source_report_sha256="e" * 64)
|
||||||
|
|
||||||
|
assert receipt["status"] == "pass"
|
||||||
|
assert all(receipt["checks"].values())
|
||||||
|
assert all(receipt["outcomes"].values())
|
||||||
|
assert receipt["selected_claim_ids"] == [CLAIM_ID]
|
||||||
|
assert receipt["execution_sha256_chain"] == ["a" * 64, "b" * 64, "c" * 64]
|
||||||
|
|
||||||
|
|
||||||
|
def test_unseen_reasoning_chain_rejects_a_challenge_that_does_not_reinspect_evidence() -> None:
|
||||||
|
report = copy.deepcopy(passing_report())
|
||||||
|
report["results"][1]["database_tool_trace"]["calls"][0]["database_invocations"] = [
|
||||||
|
{"subcommand": "show"}
|
||||||
|
]
|
||||||
|
|
||||||
|
receipt = verifier.verify_report(report, source_report_sha256="e" * 64)
|
||||||
|
|
||||||
|
assert receipt["status"] == "fail"
|
||||||
|
assert receipt["checks"]["same_claim_body_and_evidence_reinspected_after_challenge"] is False
|
||||||
|
assert receipt["outcomes"]["survives_a_claim_body_and_evidence_challenge"] is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_unseen_reasoning_chain_rejects_broken_conversation_parent_hash() -> None:
|
||||||
|
report = copy.deepcopy(passing_report())
|
||||||
|
report["results"][2]["execution_manifest"]["session_boundary"]["conversation"][
|
||||||
|
"previous_execution_sha256"
|
||||||
|
] = "0" * 64
|
||||||
|
|
||||||
|
receipt = verifier.verify_report(report, source_report_sha256="e" * 64)
|
||||||
|
|
||||||
|
assert receipt["status"] == "fail"
|
||||||
|
assert receipt["checks"]["all_turn_execution_manifests_complete"] is False
|
||||||
Loading…
Reference in a new issue