from __future__ import annotations import io import json import stat import sys import urllib.error from pathlib import Path from types import SimpleNamespace import pytest REPO_ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(REPO_ROOT / "scripts")) import prepare_kb_source_manifest as preparer # noqa: E402 SOURCE_TEXT = ( "Source preparation note.\n\n" "A staged proposal remains non-canonical until review and guarded apply both succeed.\n\n" "Every prepared claim must remain bound to an exact source quote.\n" ) CLAIM_QUOTE = "Every prepared claim must remain bound to an exact source quote." DUPLICATE_QUOTE = "A staged proposal remains non-canonical until review and guarded apply both succeed." CANDIDATE_ID = "11111111-1111-4111-8111-111111111111" def _args(tmp_path: Path, artifact: Path, context: Path) -> SimpleNamespace: return SimpleNamespace( artifact=artifact, text=None, kb_context=context, identity="document:source-preparation-test-v1", source_key="source_preparation_test_v1", source_type="article", title="Source preparation test", locator="artifact://source-preparation/test-v1", output_dir=tmp_path / "private-output", key_file=tmp_path / "openrouter-key", provider="openrouter", model="fixture/model", openrouter_url="https://openrouter.invalid/v1/chat/completions", timeout_seconds=30, max_budget_usd=0.10, ) def _inputs(tmp_path: Path) -> tuple[Path, Path]: artifact = tmp_path / "source.md" artifact.write_text(SOURCE_TEXT, encoding="utf-8") context = tmp_path / "context.json" context.write_text( json.dumps( { "database_search_query": "staged proposal exact source quote", "candidate_claims": [ { "id": CANDIDATE_ID, "text": "A staged proposal is not canonical before guarded apply.", "score": 3, "evidence_count": 2, "edge_count": 1, } ], } ), encoding="utf-8", ) (tmp_path / "openrouter-key").write_text("fixture-secret", encoding="utf-8") return artifact, context def _model_output(*, claims: bool = True, bad_reference: bool = False) -> str: claim_rows = [] if claims: quote_ref = "S99999" if bad_reference else "S00003" claim_rows = [ { "claim_key": "exact_quotes_bind_prepared_claims", "type": "structural", "text": "Prepared claims remain auditable when their evidence is bound to exact source quotes.", "quote_ref": quote_ref, "confidence": 0.7, "tags": ["provenance", "review"], "evidence": [{"quote_ref": quote_ref, "role": "grounds"}], } ] return json.dumps( { "schema": preparer.MODEL_OUTPUT_SCHEMA, "claims": claim_rows, "duplicates": [ { "claim_id": CANDIDATE_ID, "source_quote_ref": "S00002", "reason": "The canonical candidate already captures the review-before-apply argument.", } ], "extraction_notes": "One duplicate argument and one novel exact-quote provenance claim.", } ) def test_prepare_builds_private_compiler_validated_manifest_without_db_write( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: artifact, context = _inputs(tmp_path) monkeypatch.setattr( preparer, "call_openrouter", lambda *_args, **_kwargs: (_model_output(), {"prompt_tokens": 100, "completion_tokens": 50}), ) receipt = preparer.prepare(_args(tmp_path, artifact, context)) assert receipt["status"] == "prepared" assert receipt["database_write_performed"] is False assert receipt["canonical_apply_performed"] is False assert receipt["retrieval"]["canonical_candidate_ids"] == [CANDIDATE_ID] assert receipt["extraction"]["claim_count"] == 1 assert receipt["compiler"]["validated"] is True assert receipt["compiler"]["strict_child_proposal_id"] output_dir = Path(receipt["outputs"]["output_dir"]) manifest = json.loads(Path(receipt["outputs"]["manifest"]).read_text(encoding="utf-8")) assert manifest["schema"] == preparer.compiler.MANIFEST_SCHEMA_V2 assert manifest["extractor"]["version"] == preparer.EXTRACTOR_VERSION == "2" assert manifest["dedupe"]["duplicates"][0]["claim_id"] == CANDIDATE_ID assert manifest["claims"][0]["quote"] == CLAIM_QUOTE assert manifest["dedupe"]["duplicates"][0]["source_quote"] == DUPLICATE_QUOTE selected = receipt["extraction"]["selected_source_references"] assert [(row["kind"], row["reference"]) for row in selected] == [ ("claim", "S00003"), ("evidence", "S00003"), ("duplicate", "S00002"), ] assert all(row["char_end"] > row["char_start"] for row in selected) assert all(len(row["quote_sha256"]) == 64 for row in selected) assert receipt["replay"]["exact_selected_locations_validated"] is True assert receipt["extraction"]["evidence_count"] == 1 assert receipt["extraction"]["duplicate_judgment_count"] == 1 assert stat.S_IMODE(output_dir.stat().st_mode) == 0o700 for path in output_dir.iterdir(): assert stat.S_IMODE(path.stat().st_mode) == 0o600 def test_prepare_retries_once_when_model_invents_a_source_reference( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: artifact, context = _inputs(tmp_path) outputs = iter((_model_output(bad_reference=True), _model_output())) prompts: list[str] = [] def fake_call(prompt: str, **_kwargs): prompts.append(prompt) return next(outputs), {} monkeypatch.setattr(preparer, "call_openrouter", fake_call) receipt = preparer.prepare(_args(tmp_path, artifact, context)) assert receipt["status"] == "prepared" assert len(receipt["extraction"]["attempts"]) == 2 assert "previous extraction failed deterministic validation" in prompts[1] assert "must select one of the bracketed SOURCE fragment IDs" in prompts[1] def test_source_fragments_preserve_exact_unicode_and_punctuation() -> None: text = "Heading\n\nIdentity pins use an exact version - never a floating alias.\nCurly: \u201creviewed\u201d.\n" fragments = preparer.build_source_fragments(text) assert fragments == { "S00001": "Heading", "S00002": "Identity pins use an exact version - never a floating alias.", "S00003": "Curly: \u201creviewed\u201d.", } assert all(value in text for value in fragments.values()) def test_source_fragment_ids_are_dense_across_blank_lines() -> None: fragments = preparer.build_source_fragments("first\n\n\nsecond\n") assert fragments == {"S00001": "first", "S00002": "second"} def test_source_fragment_index_trims_indentation_but_preserves_exact_offsets() -> None: text = " indented Unicode: \u201creviewed\u201d \nnext\n" fragments, locations = preparer.build_source_fragment_index(text) assert fragments["S00001"] == "indented Unicode: \u201creviewed\u201d" selected = locations["S00001"] assert selected["line"] == 1 assert text[selected["char_start"] : selected["char_end"]] == fragments["S00001"] def test_bundle_location_validation_rejects_ambiguous_rebound() -> None: text = "same line\nsame line\n" fragments, locations = preparer.build_source_fragment_index(text) extraction = preparer.resolve_model_output( { "claims": [ { "claim_key": "second_occurrence", "type": "structural", "text": "The second occurrence was selected.", "quote_ref": "S00002", "confidence": 0.5, "tags": [], "evidence": [{"quote_ref": "S00002", "role": "grounds"}], } ], "duplicates": [], "extraction_notes": "Select the second repeated line.", }, fragments, locations, ) fake_bundle = { "quote_bindings": [ { "kind": "claim", "claim_key": "second_occurrence", "quote": "same line", "first_start": 0, "first_end": 9, }, { "kind": "evidence", "claim_key": "second_occurrence", "evidence_role": "grounds", "quote": "same line", "first_start": 0, "first_end": 9, }, ] } with pytest.raises(preparer.PreparationError, match="ambiguous repeated quote"): preparer.validate_bundle_source_locations(fake_bundle, extraction["selected_source_references"]) def test_prepare_returns_no_novel_claims_without_manifest_or_stageable_packet( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: artifact, context = _inputs(tmp_path) monkeypatch.setattr(preparer, "call_openrouter", lambda *_args, **_kwargs: (_model_output(claims=False), {})) receipt = preparer.prepare(_args(tmp_path, artifact, context)) assert receipt["status"] == "no_novel_claims" assert receipt["outputs"]["manifest"] is None assert receipt["compiler"]["validated"] is False assert "do not stage an empty proposal" in receipt["next_action"] def test_binary_artifact_requires_explicit_utf8_extraction(tmp_path: Path) -> None: artifact = tmp_path / "source.pdf" artifact.write_bytes(b"%PDF-1.7\x00binary") with pytest.raises(preparer.PreparationError, match=r"supply --text for \.pdf"): preparer.extract_utf8_text(artifact, None) def test_repo_native_jsonl_transcript_is_accepted_as_strict_utf8(tmp_path: Path) -> None: artifact = tmp_path / "telegram.jsonl" artifact.write_text('{"chat_id":1,"message_id":2,"message":"hello"}\n', encoding="utf-8") assert preparer.extract_utf8_text(artifact, None) == artifact.read_bytes() def test_openrouter_key_cannot_be_redirected_to_an_unapproved_endpoint(tmp_path: Path) -> None: with pytest.raises(preparer.PreparationError, match=r"must equal https://openrouter\.ai"): preparer.call_openrouter( "prompt", model="fixture/model", key_file=tmp_path / "private-file", url="https://example.invalid/collect", timeout_seconds=1, ) def test_claude_cli_uses_stdin_without_tools_or_session_persistence_and_retains_cost( monkeypatch: pytest.MonkeyPatch, ) -> None: prompt = "private source prompt" observed: dict[str, object] = {} def fake_run(command, **kwargs): observed["command"] = command observed.update(kwargs) return SimpleNamespace( returncode=0, stderr="", stdout=json.dumps( { "subtype": "success", "is_error": False, "result": _model_output(), "total_cost_usd": 0.0123, "usage": {"input_tokens": 321, "output_tokens": 123}, "modelUsage": {"claude-sonnet-test": {"costUSD": 0.0123}}, } ), ) monkeypatch.setattr(preparer.shutil, "which", lambda _name: "/usr/local/bin/claude") monkeypatch.setattr(preparer.subprocess, "run", fake_run) content, usage = preparer.call_claude_cli( prompt, model="sonnet", timeout_seconds=30, max_budget_usd=0.10, ) command = observed["command"] assert prompt not in command assert observed["input"] == prompt assert "--no-session-persistence" in command assert command[command.index("--tools") + 1] == "" assert command[command.index("--max-budget-usd") + 1] == "0.1" assert json.loads(content)["schema"] == preparer.MODEL_OUTPUT_SCHEMA assert usage["total_cost_usd"] == 0.0123 assert usage["session_persistence"] is False assert usage["tools_enabled"] is False def test_prepare_routes_claude_cli_and_records_provider_cost(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: artifact, context = _inputs(tmp_path) args = _args(tmp_path, artifact, context) args.provider = "claude-cli" args.model = "sonnet" monkeypatch.setattr( preparer, "call_openrouter", lambda *_args, **_kwargs: pytest.fail("OpenRouter must not be called for the Claude CLI provider"), ) monkeypatch.setattr( preparer, "call_claude_cli", lambda *_args, **_kwargs: (_model_output(), {"total_cost_usd": 0.0123}), ) receipt = preparer.prepare(args) assert receipt["extraction"]["provider"] == "claude-cli" assert receipt["extraction"]["attempts"][0]["usage"]["total_cost_usd"] == 0.0123 assert receipt["extraction"]["cost_control"] == { "max_total_budget_usd": 0.10, "reported_total_cost_usd": 0.0123, } assert receipt["replay"]["extractor"]["name"] == "claude-cli:sonnet" manifest = json.loads(Path(receipt["outputs"]["manifest"]).read_text(encoding="utf-8")) assert manifest["extractor"]["name"] == "claude-cli:sonnet" def test_claude_retry_budget_is_reduced_by_reported_first_attempt_cost( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: artifact, context = _inputs(tmp_path) args = _args(tmp_path, artifact, context) args.provider = "claude-cli" args.model = "sonnet" outputs = iter(((_model_output(bad_reference=True), 0.04), (_model_output(), 0.03))) budgets: list[float] = [] def fake_claude(_prompt: str, *, model: str, timeout_seconds: int, max_budget_usd: float): assert model == "sonnet" assert timeout_seconds == 30 budgets.append(max_budget_usd) output, cost = next(outputs) return output, {"total_cost_usd": cost} monkeypatch.setattr(preparer, "call_claude_cli", fake_claude) receipt = preparer.prepare(args) assert budgets == pytest.approx([0.10, 0.06]) assert receipt["extraction"]["cost_control"]["reported_total_cost_usd"] == pytest.approx(0.07) def test_provider_errors_redact_secret_and_private_detail(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: key_file = tmp_path / "key" key_file.write_text("fixture-secret", encoding="utf-8") error = urllib.error.HTTPError( preparer.DEFAULT_OPENROUTER_URL, 401, "unauthorized", {}, io.BytesIO(b"fixture-secret and private source quote"), ) def fail_urlopen(*_args, **_kwargs): raise error monkeypatch.setattr(preparer.urllib.request, "urlopen", fail_urlopen) with pytest.raises(preparer.PreparationError) as caught: preparer.call_openrouter( "private source quote", model="fixture/model", key_file=key_file, url=preparer.DEFAULT_OPENROUTER_URL, timeout_seconds=1, ) rendered = str(caught.value) assert "fixture-secret" not in rendered assert "private source quote" not in rendered assert "provider detail redacted" in rendered def test_cli_emits_only_public_status_for_private_preparation_receipt( tmp_path: Path, capsys: pytest.CaptureFixture, monkeypatch: pytest.MonkeyPatch ) -> None: sentinel = "PRIVATE-SOURCE-RECEIPT-SENTINEL" private_path = tmp_path / sentinel / "preparation-receipt.json" private_receipt = { "status": "prepared", "source": {"title": sentinel, "artifact_path": str(private_path)}, "retrieval": {"database_search_query": sentinel}, "outputs": {"receipt": str(private_path)}, } monkeypatch.setattr(preparer, "prepare", lambda _args: private_receipt) status = preparer.main( [ "--artifact", str(tmp_path / sentinel / "source.md"), "--kb-context", str(tmp_path / sentinel / "context.json"), "--identity", f"document:{sentinel}", "--source-key", "private_source", "--source-type", "article", "--title", sentinel, "--locator", f"artifact://{sentinel}", "--output-dir", str(private_path.parent), ] ) stdout = capsys.readouterr().out assert status == 0 assert json.loads(stdout) == { "schema": preparer.STATUS_SCHEMA, "status": "prepared", "database_write_performed": False, "canonical_apply_performed": False, "private_outputs_written": True, } assert sentinel not in stdout assert str(private_path) not in stdout def test_cli_redacts_private_preparation_failure_from_stdout( tmp_path: Path, capsys: pytest.CaptureFixture, monkeypatch: pytest.MonkeyPatch ) -> None: sentinel = "PRIVATE-PREPARATION-FAILURE-SENTINEL" def fail(_args): raise preparer.PreparationError(f"{sentinel}: {tmp_path / sentinel}") monkeypatch.setattr(preparer, "prepare", fail) status = preparer.main( [ "--artifact", str(tmp_path / sentinel / "source.md"), "--kb-context", str(tmp_path / sentinel / "context.json"), "--identity", f"document:{sentinel}", "--source-key", "private_source", "--source-type", "article", "--title", sentinel, "--locator", f"artifact://{sentinel}", "--output-dir", str(tmp_path / sentinel / "output"), ] ) stdout = capsys.readouterr().out public_status = json.loads(stdout) assert status == 2 assert public_status["reason"] == "source_preparation_failed" assert sentinel not in stdout assert str(tmp_path) not in stdout @pytest.mark.parametrize("fail_after_write", [1, 2, 3, 4]) def test_prepare_never_publishes_partially_staged_output_set( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, fail_after_write: int, ) -> None: artifact, context = _inputs(tmp_path) args = _args(tmp_path, artifact, context) monkeypatch.setattr( preparer, "call_openrouter", lambda *_args, **_kwargs: (_model_output(), {"prompt_tokens": 100, "completion_tokens": 50}), ) real_write = preparer._write_private write_count = 0 def faulted_write(path: Path, value: bytes) -> None: nonlocal write_count real_write(path, value) write_count += 1 if write_count == fail_after_write: raise OSError("staging fault detail must stay private") monkeypatch.setattr(preparer, "_write_private", faulted_write) with pytest.raises(OSError): preparer.prepare(args) assert not args.output_dir.exists() or not any(args.output_dir.iterdir()) assert not list(args.output_dir.parent.glob(f".{args.output_dir.name}.*.staging")) @pytest.mark.parametrize( ("fault_after_publication", "expected_published"), [(False, False), (True, True)], ) def test_prepare_atomic_publication_boundary_is_all_or_nothing( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, fault_after_publication: bool, expected_published: bool, ) -> None: artifact, context = _inputs(tmp_path) args = _args(tmp_path, artifact, context) monkeypatch.setattr( preparer, "call_openrouter", lambda *_args, **_kwargs: (_model_output(), {"prompt_tokens": 100, "completion_tokens": 50}), ) real_replace = preparer.os.replace def faulted_replace(source: Path, destination: Path) -> None: if Path(destination) != args.output_dir: real_replace(source, destination) return if fault_after_publication: real_replace(source, destination) raise OSError("publication fault detail must stay private") monkeypatch.setattr(preparer.os, "replace", faulted_replace) with pytest.raises(OSError): preparer.prepare(args) expected_state = { "extracted_text": expected_published, "extraction_result": expected_published, "manifest": expected_published, "receipt": expected_published, } assert preparer._private_output_state(args.output_dir) == expected_state assert not list(args.output_dir.parent.glob(f".{args.output_dir.name}.*.staging")) @pytest.mark.parametrize( "published_names", [ ("extracted-text.txt",), ("extracted-text.txt", "extraction-result.json"), ("extracted-text.txt", "extraction-result.json", "source-manifest.json"), ( "extracted-text.txt", "extraction-result.json", "source-manifest.json", "preparation-receipt.json", ), ], ) def test_cli_failure_reports_actual_state_after_each_private_publication_boundary( tmp_path: Path, capsys: pytest.CaptureFixture, monkeypatch: pytest.MonkeyPatch, published_names: tuple[str, ...], ) -> None: output_dir = tmp_path / "private-output" def fail(args) -> None: args.output_dir.mkdir(parents=True) for name in published_names: (args.output_dir / name).write_text("private", encoding="utf-8") raise preparer.PreparationError("private publication fault") monkeypatch.setattr(preparer, "prepare", fail) status = preparer.main( [ "--artifact", str(tmp_path / "source.md"), "--kb-context", str(tmp_path / "context.json"), "--identity", "document:private", "--source-key", "private_source", "--source-type", "article", "--title", "private", "--locator", "artifact://private", "--output-dir", str(output_dir), ] ) public_status = json.loads(capsys.readouterr().out) expected_state = { "extracted_text": "extracted-text.txt" in published_names, "extraction_result": "extraction-result.json" in published_names, "manifest": "source-manifest.json" in published_names, "receipt": "preparation-receipt.json" in published_names, } assert status == 2 assert public_status["private_output_files"] == expected_state assert public_status["private_outputs_written"] is expected_state["receipt"]