313 lines
14 KiB
Python
313 lines
14 KiB
Python
from __future__ import annotations
|
|
|
|
import copy
|
|
import hashlib
|
|
import json
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
REPO_ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(REPO_ROOT / "scripts"))
|
|
|
|
import run_leo_local_ingestion_proposal_canary as canary # noqa: E402
|
|
|
|
|
|
def _loaded(path: Path) -> tuple[dict, dict, dict, dict, dict]:
|
|
fixture, input_receipt = canary.load_fixture(path)
|
|
row_ids = canary.build_row_ids(input_receipt, fixture)
|
|
parent = canary.build_parent_packet(fixture, input_receipt, row_ids)
|
|
child = canary.normalized_stage.prepare_normalized_child(parent)
|
|
return fixture, input_receipt, row_ids, parent, child
|
|
|
|
|
|
def _copy_scenario(tmp_path: Path, scenario_path: Path) -> Path:
|
|
scenario = json.loads(scenario_path.read_text(encoding="utf-8"))
|
|
artifact_source = scenario_path.parent / scenario["artifact"]["path"]
|
|
artifact_target = tmp_path / artifact_source.name
|
|
artifact_target.write_bytes(artifact_source.read_bytes())
|
|
scenario_target = tmp_path / scenario_path.name
|
|
scenario_target.write_text(json.dumps(scenario), encoding="utf-8")
|
|
return scenario_target
|
|
|
|
|
|
def _synthetic_readback(fixture: dict, input_receipt: dict, row_ids: dict, child: dict) -> dict:
|
|
claim_rows = []
|
|
evidence_rows = []
|
|
segment_by_id = {segment["id"]: segment for segment in fixture["segments"]}
|
|
for claim in fixture["extraction"]["claims"]:
|
|
claim_id = row_ids["claim_extraction_ids"][claim["claim_key"]]
|
|
claim_rows.append(
|
|
{
|
|
"id": claim_id,
|
|
"source_capture_id": row_ids["source_capture_id"],
|
|
"claim_key": claim["claim_key"],
|
|
"body": claim["body"],
|
|
"metadata": {"extractor": input_receipt["extractor"]},
|
|
}
|
|
)
|
|
for index, evidence in enumerate(claim["evidence"]):
|
|
segment = segment_by_id[evidence["segment_id"]]
|
|
evidence_rows.append(
|
|
{
|
|
"id": row_ids["evidence_extraction_ids"][f"{claim['claim_key']}:{index}"],
|
|
"claim_extraction_id": claim_id,
|
|
"source_capture_id": row_ids["source_capture_id"],
|
|
"source_segment_id": segment["id"],
|
|
"source_locator": segment["locator"],
|
|
"source_content_sha256": segment["content_sha256"],
|
|
"body": evidence["excerpt"],
|
|
}
|
|
)
|
|
proposal = {
|
|
**child,
|
|
"reviewed_by_handle": None,
|
|
"reviewed_at": None,
|
|
"applied_by_handle": None,
|
|
"applied_at": None,
|
|
}
|
|
counts = input_receipt["counts"]
|
|
return {
|
|
"source_captures": [
|
|
{
|
|
"artifact_sha256": input_receipt["artifact_sha256"],
|
|
"replay_identity_sha256": input_receipt["replay_identity_sha256"],
|
|
}
|
|
],
|
|
"claim_extractions": claim_rows,
|
|
"evidence_extractions": evidence_rows,
|
|
"duplicate_assessments": [{} for _item in fixture["extraction"]["duplicates"]],
|
|
"conflict_assessments": [{} for _item in fixture["extraction"]["conflicts"]],
|
|
"staged_proposal": proposal,
|
|
"counts": {
|
|
"source_captures": 1,
|
|
"claim_extractions": counts["claim_candidates"],
|
|
"evidence_extractions": counts["evidence_candidates"],
|
|
"duplicate_assessments": counts["duplicate_judgments"],
|
|
"conflict_assessments": counts["conflict_relationships"],
|
|
"staged_proposals": 1,
|
|
},
|
|
}
|
|
|
|
|
|
@pytest.mark.parametrize("scenario_path", canary.DEFAULT_FIXTURES)
|
|
def test_source_artifact_is_separate_hash_bound_and_replayable(scenario_path: Path) -> None:
|
|
fixture, input_receipt, row_ids, parent, child = _loaded(scenario_path)
|
|
artifact_path = Path(input_receipt["artifact_path"])
|
|
|
|
assert input_receipt["artifact_sha256"] == hashlib.sha256(artifact_path.read_bytes()).hexdigest()
|
|
assert input_receipt["artifact_sha256"] != input_receipt["scenario_sha256"]
|
|
assert len(input_receipt["replay_identity_sha256"]) == 64
|
|
assert row_ids == canary.build_row_ids(input_receipt, fixture)
|
|
ingestion = child["payload"]["apply_payload"]["normalization_manifest"]["ingestion_manifest"]
|
|
assert ingestion["artifact_sha256"] == input_receipt["artifact_sha256"]
|
|
assert ingestion["extractor"] == input_receipt["extractor"]
|
|
assert ingestion["replay_identity_sha256"] == input_receipt["replay_identity_sha256"]
|
|
assert ingestion["row_ids"] == row_ids
|
|
assert parent["status"] == child["status"] == "pending_review"
|
|
|
|
|
|
def test_document_and_telegram_candidate_accounting_is_exact() -> None:
|
|
_doc, doc_input, _doc_ids, _doc_parent, doc_child = _loaded(canary.DEFAULT_DOCUMENT_FIXTURE)
|
|
telegram, telegram_input, _telegram_ids, _telegram_parent, telegram_child = _loaded(canary.DEFAULT_TELEGRAM_FIXTURE)
|
|
|
|
assert doc_input["counts"] == {
|
|
"source_segments": 3,
|
|
"claim_candidates": 1,
|
|
"evidence_candidates": 1,
|
|
"duplicate_judgments": 0,
|
|
"conflict_relationships": 0,
|
|
}
|
|
assert telegram_input["counts"] == {
|
|
"source_segments": 3,
|
|
"claim_candidates": 2,
|
|
"evidence_candidates": 3,
|
|
"duplicate_judgments": 1,
|
|
"conflict_relationships": 1,
|
|
}
|
|
doc_payload = doc_child["payload"]["apply_payload"]
|
|
telegram_payload = telegram_child["payload"]["apply_payload"]
|
|
assert {key: len(doc_payload[key]) for key in ("claims", "sources", "evidence", "edges")} == {
|
|
"claims": 1,
|
|
"sources": 2,
|
|
"evidence": 2,
|
|
"edges": 0,
|
|
}
|
|
assert {key: len(telegram_payload[key]) for key in ("claims", "sources", "evidence", "edges")} == {
|
|
"claims": 2,
|
|
"sources": 5,
|
|
"evidence": 5,
|
|
"edges": 1,
|
|
}
|
|
assessment = telegram_payload["normalization_manifest"]["dedupe_conflict_assessment"]
|
|
assert assessment["duplicates"] == telegram["extraction"]["duplicates"]
|
|
assert assessment["conflicts"] == telegram["extraction"]["conflicts"]
|
|
assert telegram_payload["edges"][0]["edge_type"] == "contradicts"
|
|
|
|
|
|
def test_telegram_duplicate_text_keeps_distinct_exact_message_provenance() -> None:
|
|
fixture, _input, _row_ids, _parent, _child = _loaded(canary.DEFAULT_TELEGRAM_FIXTURE)
|
|
first, _second, repeated = fixture["segments"]
|
|
|
|
assert first["body"] == repeated["body"]
|
|
assert first["text_sha256"] == repeated["text_sha256"]
|
|
assert first["content_sha256"] != repeated["content_sha256"]
|
|
assert first["locator"] == "telegram://chat/900001/message/7301"
|
|
assert repeated["locator"] == "telegram://chat/900001/message/7303"
|
|
duplicate = fixture["extraction"]["duplicates"][0]
|
|
assert duplicate["source_locator"] == repeated["locator"]
|
|
assert duplicate["source_json_pointer"] == "jsonl://line/3/message"
|
|
|
|
|
|
def test_fixture_validation_fails_closed_on_invented_evidence(tmp_path: Path) -> None:
|
|
scenario_path = _copy_scenario(tmp_path, canary.DEFAULT_TELEGRAM_FIXTURE)
|
|
scenario = json.loads(scenario_path.read_text(encoding="utf-8"))
|
|
scenario["extraction"]["claims"][0]["evidence"][0]["excerpt"] = "invented evidence"
|
|
scenario_path.write_text(json.dumps(scenario), encoding="utf-8")
|
|
|
|
with pytest.raises(canary.CanaryError, match="not an exact segment substring"):
|
|
canary.load_fixture(scenario_path)
|
|
|
|
|
|
def test_fixture_artifact_path_cannot_escape_scenario_directory(tmp_path: Path) -> None:
|
|
scenario = json.loads(canary.DEFAULT_DOCUMENT_FIXTURE.read_text(encoding="utf-8"))
|
|
scenario["artifact"]["path"] = "../outside.txt"
|
|
path = tmp_path / "scenario.json"
|
|
path.write_text(json.dumps(scenario), encoding="utf-8")
|
|
|
|
with pytest.raises(canary.CanaryError, match="remain inside"):
|
|
canary.load_fixture(path)
|
|
|
|
|
|
def test_capture_sql_escapes_quotes_and_backslashes(tmp_path: Path) -> None:
|
|
scenario_path = _copy_scenario(tmp_path, canary.DEFAULT_DOCUMENT_FIXTURE)
|
|
scenario = json.loads(scenario_path.read_text(encoding="utf-8"))
|
|
scenario["source"]["title"] = "O'Brien\\review"
|
|
scenario["source"]["metadata"]["note"] = "quoted ' value \\ path"
|
|
scenario_path.write_text(json.dumps(scenario), encoding="utf-8")
|
|
fixture, input_receipt, row_ids, _parent, _child = _loaded(scenario_path)
|
|
|
|
sql = canary.build_capture_sql(fixture, input_receipt, row_ids)
|
|
|
|
assert canary.ap.sql_literal(scenario["source"]["title"]) in sql
|
|
expected_metadata = {
|
|
**scenario["source"]["metadata"],
|
|
"extractor": input_receipt["extractor"],
|
|
}
|
|
assert canary.ap.sql_literal(json.dumps(expected_metadata, sort_keys=True)) in sql
|
|
|
|
|
|
def test_replay_identity_and_ids_change_with_extractor_version(tmp_path: Path) -> None:
|
|
scenario_path = _copy_scenario(tmp_path, canary.DEFAULT_DOCUMENT_FIXTURE)
|
|
fixture, original, original_ids, _parent, original_child = _loaded(scenario_path)
|
|
scenario = json.loads(scenario_path.read_text(encoding="utf-8"))
|
|
scenario["extractor"]["version"] = "3"
|
|
scenario_path.write_text(json.dumps(scenario), encoding="utf-8")
|
|
changed_fixture, changed, changed_ids, _changed_parent, changed_child = _loaded(scenario_path)
|
|
|
|
assert original["artifact_sha256"] == changed["artifact_sha256"]
|
|
assert original["replay_identity_sha256"] != changed["replay_identity_sha256"]
|
|
assert original_ids["source_capture_id"] == changed_ids["source_capture_id"]
|
|
assert original_ids["claim_extraction_ids"] != changed_ids["claim_extraction_ids"]
|
|
assert original_ids["parent_proposal_id"] != changed_ids["parent_proposal_id"]
|
|
assert original_child["payload_sha256"] != changed_child["payload_sha256"]
|
|
assert fixture["segments"] == changed_fixture["segments"]
|
|
|
|
|
|
def test_source_byte_tamper_changes_hashes_and_replay_ids(tmp_path: Path) -> None:
|
|
scenario_path = _copy_scenario(tmp_path, canary.DEFAULT_DOCUMENT_FIXTURE)
|
|
fixture, before, before_ids, _parent, _child = _loaded(scenario_path)
|
|
artifact_path = Path(before["artifact_path"])
|
|
artifact_path.write_text(
|
|
artifact_path.read_text(encoding="utf-8") + "\n\nA new source paragraph.", encoding="utf-8"
|
|
)
|
|
changed_fixture, after, after_ids, _changed_parent, _changed_child = _loaded(scenario_path)
|
|
|
|
assert before["artifact_sha256"] != after["artifact_sha256"]
|
|
assert before["source_content_sha256"] != after["source_content_sha256"]
|
|
assert before["replay_identity_sha256"] != after["replay_identity_sha256"]
|
|
assert before_ids["source_capture_id"] != after_ids["source_capture_id"]
|
|
assert before_ids["claim_extraction_ids"] != after_ids["claim_extraction_ids"]
|
|
assert len(changed_fixture["segments"]) == len(fixture["segments"]) + 1
|
|
|
|
|
|
def test_replay_properties_hold_across_many_version_labels(tmp_path: Path) -> None:
|
|
scenario_path = _copy_scenario(tmp_path, canary.DEFAULT_DOCUMENT_FIXTURE)
|
|
base = json.loads(scenario_path.read_text(encoding="utf-8"))
|
|
observed: set[str] = set()
|
|
for index in range(40):
|
|
scenario = copy.deepcopy(base)
|
|
scenario["extractor"]["version"] = f"property-{index}"
|
|
scenario_path.write_text(json.dumps(scenario), encoding="utf-8")
|
|
fixture_a, receipt_a = canary.load_fixture(scenario_path)
|
|
fixture_b, receipt_b = canary.load_fixture(scenario_path)
|
|
ids_a = canary.build_row_ids(receipt_a, fixture_a)
|
|
ids_b = canary.build_row_ids(receipt_b, fixture_b)
|
|
assert receipt_a == receipt_b
|
|
assert ids_a == ids_b
|
|
observed.add(receipt_a["replay_identity_sha256"])
|
|
assert len(observed) == 40
|
|
|
|
|
|
@pytest.mark.parametrize("scenario_path", canary.DEFAULT_FIXTURES)
|
|
def test_capture_and_stage_sql_do_not_mutate_canonical_tables(scenario_path: Path) -> None:
|
|
fixture, input_receipt, row_ids, parent, child = _loaded(scenario_path)
|
|
sql = "\n".join(
|
|
(
|
|
canary.build_capture_sql(fixture, input_receipt, row_ids),
|
|
canary.normalized_stage.build_stage_sql(child),
|
|
)
|
|
).lower()
|
|
|
|
assert "insert into public." not in sql
|
|
assert "update public." not in sql
|
|
assert "delete from public." not in sql
|
|
assert "insert into kb_canary.source_captures" in sql
|
|
assert "insert into kb_stage.kb_proposals" in sql
|
|
assert parent["payload"]["ingestion_manifest"]["row_ids"] == row_ids
|
|
|
|
|
|
def test_canonical_snapshot_covers_all_nonempty_canonical_tables() -> None:
|
|
schema_sql = canary.build_schema_sql().lower()
|
|
snapshot_sql = canary.build_canonical_snapshot_sql().lower()
|
|
|
|
for table in ("claims", "sources", "claim_evidence", "claim_edges"):
|
|
assert f"insert into public.{table}" in schema_sql
|
|
assert f"from public.{table}" in snapshot_sql
|
|
assert "order by" in snapshot_sql
|
|
assert "begin transaction read only" in snapshot_sql
|
|
assert canary.canonical_snapshot_complete({}) is False
|
|
assert (
|
|
canary.canonical_snapshot_complete(
|
|
{
|
|
"claims": [{"id": "1"}],
|
|
"sources": [{"id": "2"}],
|
|
"claim_evidence": [{"claim_id": "1"}],
|
|
"claim_edges": [{"from_claim": "1"}],
|
|
}
|
|
)
|
|
is True
|
|
)
|
|
|
|
|
|
def test_evaluation_fails_when_exact_evidence_locator_is_changed() -> None:
|
|
fixture, input_receipt, row_ids, _parent, child = _loaded(canary.DEFAULT_TELEGRAM_FIXTURE)
|
|
readback = _synthetic_readback(fixture, input_receipt, row_ids, child)
|
|
canonical = {
|
|
"claims": [{"id": "1"}],
|
|
"sources": [{"id": "2"}],
|
|
"claim_evidence": [{"claim_id": "1"}],
|
|
"claim_edges": [{"from_claim": "1"}],
|
|
}
|
|
|
|
checks = canary.evaluate_checks(fixture, input_receipt, row_ids, readback, canonical, copy.deepcopy(canonical))
|
|
assert all(checks.values())
|
|
broken = copy.deepcopy(readback)
|
|
broken["evidence_extractions"][0]["source_locator"] = "telegram://chat/900001/message/9999"
|
|
assert (
|
|
canary.evaluate_checks(fixture, input_receipt, row_ids, broken, canonical, copy.deepcopy(canonical))[
|
|
"all_evidence_has_exact_source_location"
|
|
]
|
|
is False
|
|
)
|