#!/usr/bin/env python3 """Compile one hash-bound source document into a pending-review KB proposal. The command is deliberately build-only. It validates a raw artifact, its UTF-8 text extraction, and an extraction manifest. V1/V2 reuse Leo's rich proposal normalizer; V3 additionally requires the exact extractor specification and builds the explicit epistemic payload. The command never connects to Postgres or executes the generated staging SQL. """ from __future__ import annotations import argparse import hashlib import json import os import re import sys import tempfile import uuid from datetime import datetime, timezone from pathlib import Path from typing import Any from urllib.parse import urlsplit HERE = Path(__file__).resolve().parent sys.path.insert(0, str(HERE)) import apply_proposal as ap # noqa: E402 import stage_normalized_proposal as stage # noqa: E402 MANIFEST_SCHEMA = "livingip.kbSourceExtractionManifest.v1" MANIFEST_SCHEMA_V2 = "livingip.kbSourceExtractionManifest.v2" MANIFEST_SCHEMAS = {MANIFEST_SCHEMA, MANIFEST_SCHEMA_V2} MANIFEST_SCHEMA_V3 = "livingip.kbSourceExtractionManifest.v3" BUNDLE_SCHEMA = "livingip.kbSourceProposalBundle.v1" BUNDLE_SCHEMA_V3 = "livingip.kbSourceProposalBundle.v3" STATUS_SCHEMA = "livingip.kbSourceCompilerStatus.v1" SHA256_RE = re.compile(r"^[0-9a-f]{64}$") KEY_RE = re.compile(r"^[a-z0-9][a-z0-9._:-]{0,127}$") URI_SCHEME_RE = re.compile(r"^[a-z][a-z0-9+.-]*:", re.IGNORECASE) ISO_TIMESTAMP_RE = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{1,6})?(?:Z|[+-]\d{2}:\d{2})$") UNSAFE_URI_SCHEMES = {"data", "javascript", "vbscript"} TOP_LEVEL_KEYS = { "schema", "artifact_sha256", "extracted_text_sha256", "extractor", "source", "claims", } TOP_LEVEL_KEYS_V2 = TOP_LEVEL_KEYS | {"dedupe"} EXTRACTOR_KEYS = {"name", "version"} SOURCE_KEYS = {"identity", "source_key", "source_type", "title", "locator"} CLAIM_KEYS = {"claim_key", "type", "text", "quote", "confidence", "tags", "evidence"} EVIDENCE_KEYS = {"quote", "role"} DEDUPE_KEYS = {"database_search_query", "candidate_claims", "duplicates"} DEDUPE_CANDIDATE_KEYS = {"id", "text", "score", "evidence_count", "edge_count"} DEDUPE_DUPLICATE_KEYS = {"claim_id", "source_quote", "reason"} TOP_LEVEL_KEYS_V3 = { "schema", "artifact_sha256", "extracted_text_sha256", "extractor", "agent_id", "source", "claims", } EXTRACTOR_KEYS_V3 = {"name", "version", "spec_sha256"} SOURCE_KEYS_V3 = { "identity", "source_key", "source_type", "title", "locator", "access_scope", "ingestion_origin", "provenance_status", "source_class", "auto_promotion_policy", "captured_at", } CLAIM_KEYS_V3 = { "claim_key", "type", "proposition", "body_markdown", "scope", "access_scope", "owner_agent_id", "status", "revision_criteria", "revision_kind", "supersedes_id", "tags", "evidence", "assessment", } EVIDENCE_KEYS_V3 = {"evidence_key", "quote", "polarity", "locator", "rationale"} LOCATOR_KEYS_V3 = {"unit", "start", "end"} ASSESSMENT_KEYS_V3 = { "support_tier", "challenge_tier", "overall_state", "rationale", "supersedes_assessment_id", } TRUTH_APT_V3_CLAIM_TYPES = {"empirical", "predictive", "causal"} ASSESSMENT_RELEVANT_POLARITIES = {"supports", "challenges"} class CompilerError(RuntimeError): """Raised when an input cannot safely become a reviewable proposal.""" def sha256_bytes(value: bytes) -> str: return hashlib.sha256(value).hexdigest() def canonical_json_bytes(value: Any) -> bytes: return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=True).encode("utf-8") def canonical_sha256(value: Any) -> str: return sha256_bytes(canonical_json_bytes(value)) def _load_duplicate_free_json(value: bytes, label: str) -> Any: try: text = value.decode("utf-8", errors="strict") except UnicodeDecodeError as exc: raise CompilerError(f"{label} must be valid UTF-8 JSON: {exc}") from exc def reject_duplicate_pairs(pairs: list[tuple[str, Any]]) -> dict[str, Any]: result: dict[str, Any] = {} for key, item in pairs: if key in result: raise CompilerError(f"{label} contains duplicate JSON object key: {key!r}") result[key] = item return result try: return json.loads(text, object_pairs_hook=reject_duplicate_pairs) except json.JSONDecodeError as exc: raise CompilerError(f"{label} must be valid UTF-8 JSON: {exc}") from exc def stable_uuid(label: str) -> str: return str(uuid.uuid5(uuid.NAMESPACE_URL, f"livingip:kb-source-compiler:v1:{label}")) def stable_uuid_v3(row_kind: str, packet_identity_sha256: str, semantic_key: Any) -> str: material = canonical_json_bytes( ["livingip", "kb-source-compiler", 3, packet_identity_sha256, row_kind, semantic_key] ).decode("ascii") return str(uuid.uuid5(uuid.NAMESPACE_URL, material)) def _require_object(value: Any, label: str) -> dict[str, Any]: if not isinstance(value, dict): raise CompilerError(f"{label} must be an object") return value def _require_exact_keys(value: dict[str, Any], allowed: set[str], label: str) -> None: unknown = sorted(set(value) - allowed) if unknown: raise CompilerError(f"{label} contains unsupported keys: {', '.join(unknown)}") def _require_exact_shape(value: dict[str, Any], required: set[str], label: str) -> None: _require_exact_keys(value, required, label) missing = sorted(required - set(value)) if missing: raise CompilerError(f"{label} is missing required keys: {', '.join(missing)}") def _require_nonempty_string(value: Any, label: str) -> str: if not isinstance(value, str) or not value.strip(): raise CompilerError(f"{label} must be a non-empty string") if value != value.strip(): raise CompilerError(f"{label} must not have leading or trailing whitespace") return value def _require_sha256(value: Any, label: str) -> str: text = _require_nonempty_string(value, label) if not SHA256_RE.fullmatch(text): raise CompilerError(f"{label} must be a lowercase SHA-256 digest") return text def _require_key(value: Any, label: str) -> str: text = _require_nonempty_string(value, label) if not KEY_RE.fullmatch(text): raise CompilerError(f"{label} must match {KEY_RE.pattern}") return text def _require_canonical_uuid(value: Any, label: str, *, nullable: bool = False) -> str | None: if value is None and nullable: return None if not isinstance(value, str): raise CompilerError(f"{label} must be a canonical lowercase hyphenated UUID") try: canonical = str(uuid.UUID(value)) except ValueError as exc: raise CompilerError(f"{label} must be a canonical lowercase hyphenated UUID") from exc if value != canonical: raise CompilerError(f"{label} must be a canonical lowercase hyphenated UUID") return canonical def _require_canonical_utc_timestamp(value: Any, label: str) -> str: text = _require_nonempty_string(value, label) if not ISO_TIMESTAMP_RE.fullmatch(text): raise CompilerError(f"{label} must be a timezone-aware ISO-8601 timestamp with at most 6 fractional digits") try: parsed = datetime.fromisoformat(text.replace("Z", "+00:00")) except ValueError as exc: raise CompilerError(f"{label} must be a valid ISO-8601 timestamp") from exc if parsed.tzinfo is None: raise CompilerError(f"{label} must include a timezone") return parsed.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%fZ") def _require_nullable_enum(value: Any, allowed: set[str], label: str) -> str | None: if value is None: return None if not isinstance(value, str) or value not in allowed: raise CompilerError(f"{label} must be null or one of {sorted(allowed)}") return value def _require_string_list(value: Any, label: str) -> list[str]: if not isinstance(value, list) or any( not isinstance(item, str) or not item.strip() or item != item.strip() for item in value ): raise CompilerError(f"{label} must be a list of non-empty strings without surrounding whitespace") if len(value) != len(set(value)): raise CompilerError(f"{label} contains duplicates") return sorted(value) def _require_stable_reference(value: Any, label: str) -> str: text = _require_nonempty_string(value, label) if any(character.isspace() for character in text) or not URI_SCHEME_RE.match(text): raise CompilerError(f"{label} must be a stable URI-like reference with a scheme") parsed = urlsplit(text) if not parsed.scheme or text.endswith(":"): raise CompilerError(f"{label} must identify a specific source, not only a scheme or chat label") scheme = parsed.scheme.lower() if scheme in UNSAFE_URI_SCHEMES: raise CompilerError(f"{label} uses an unsafe URI scheme: {scheme}") if scheme in {"http", "https"}: if not parsed.netloc: raise CompilerError(f"{label} HTTP reference must include a host") if parsed.username is not None or parsed.password is not None: raise CompilerError(f"{label} must not embed credentials") if scheme in {"telegram", "tg"} and not re.search(r"\d", f"{parsed.netloc}{parsed.path}"): raise CompilerError(f"{label} Telegram reference must include a stable chat or message identifier") return text def _read_required_file(path: Path, label: str) -> bytes: try: if not path.is_file(): raise CompilerError(f"{label} path is not a regular file: {path}") value = path.read_bytes() except OSError as exc: raise CompilerError(f"could not read {label}: {exc}") from exc if not value: raise CompilerError(f"{label} must not be empty") return value def _quote_binding(text: str, quote: Any, label: str) -> dict[str, Any]: exact = _require_nonempty_string(quote, label) start = text.find(exact) if start < 0: raise CompilerError(f"{label} is not an exact substring of the extracted text") return { "quote": exact, "quote_sha256": sha256_bytes(exact.encode("utf-8")), "first_start": start, "first_end": start + len(exact), "occurrence_count": text.count(exact), } def _require_nonnegative_integer(value: Any, label: str) -> int: if isinstance(value, bool) or not isinstance(value, int) or value < 0: raise CompilerError(f"{label} must be a non-negative integer") return value def _v3_quote_binding(text_bytes: bytes, quote: Any, locator_value: Any, label: str) -> dict[str, Any]: exact = _require_nonempty_string(quote, f"{label}.quote") locator = _require_object(locator_value, f"{label}.locator") _require_exact_shape(locator, LOCATOR_KEYS_V3, f"{label}.locator") if locator.get("unit") != "utf8_bytes": raise CompilerError(f"{label}.locator.unit must equal 'utf8_bytes'") start = _require_nonnegative_integer(locator.get("start"), f"{label}.locator.start") end = _require_nonnegative_integer(locator.get("end"), f"{label}.locator.end") quote_bytes = exact.encode("utf-8") if end <= start or end > len(text_bytes) or text_bytes[start:end] != quote_bytes: raise CompilerError(f"{label}.locator does not bind the exact quote bytes in the extracted text") return { "quote": exact, "quote_sha256": sha256_bytes(quote_bytes), "unit": "utf8_bytes", "start": start, "end": end, } def _validate_v3_assessment(value: Any, claim_type: str, evidence: list[dict[str, Any]], label: str) -> dict[str, Any]: assessment = _require_object(value, label) _require_exact_shape(assessment, ASSESSMENT_KEYS_V3, label) support_tier = _require_nullable_enum(assessment.get("support_tier"), ap.V3_SUPPORT_TIERS, f"{label}.support_tier") challenge_tier = _require_nullable_enum( assessment.get("challenge_tier"), ap.V3_SUPPORT_TIERS, f"{label}.challenge_tier" ) if claim_type in TRUTH_APT_V3_CLAIM_TYPES and (support_tier is None or challenge_tier is None): raise CompilerError(f"{label} requires support_tier and challenge_tier for a truth-apt claim") overall_state = assessment.get("overall_state") if overall_state not in ap.V3_ASSESSMENT_STATES: raise CompilerError(f"{label}.overall_state must be one of {sorted(ap.V3_ASSESSMENT_STATES)}") polarities = {row["polarity"] for row in evidence} if not polarities & ASSESSMENT_RELEVANT_POLARITIES: raise CompilerError(f"{label} requires at least one supports or challenges evidence row") if support_tier not in {None, "none"} and "supports" not in polarities: raise CompilerError(f"{label}.support_tier requires supports evidence") if challenge_tier not in {None, "none"} and "challenges" not in polarities: raise CompilerError(f"{label}.challenge_tier requires challenges evidence") if overall_state == "support_dominant" and "supports" not in polarities: raise CompilerError(f"{label}.overall_state=support_dominant requires supports evidence") if overall_state == "challenge_dominant" and "challenges" not in polarities: raise CompilerError(f"{label}.overall_state=challenge_dominant requires challenges evidence") if overall_state == "mixed" and not {"supports", "challenges"} <= polarities: raise CompilerError(f"{label}.overall_state=mixed requires supports and challenges evidence") if assessment.get("supersedes_assessment_id") is not None: raise CompilerError(f"{label}.supersedes_assessment_id must be null for a first assessment") return { "support_tier": support_tier, "challenge_tier": challenge_tier, "overall_state": overall_state, "rationale": _require_nonempty_string(assessment.get("rationale"), f"{label}.rationale"), "supersedes_assessment_id": None, } def _validate_manifest_v3( manifest: dict[str, Any], artifact_sha256: str, text_sha256: str, extractor_spec_sha256: str, text_bytes: bytes, ) -> tuple[dict[str, Any], list[dict[str, Any]]]: _require_exact_shape(manifest, TOP_LEVEL_KEYS_V3, "manifest") if manifest.get("schema") != MANIFEST_SCHEMA_V3: raise CompilerError(f"manifest.schema must equal {MANIFEST_SCHEMA_V3!r}") declared_artifact_sha256 = _require_sha256(manifest.get("artifact_sha256"), "manifest.artifact_sha256") declared_text_sha256 = _require_sha256(manifest.get("extracted_text_sha256"), "manifest.extracted_text_sha256") if declared_artifact_sha256 != artifact_sha256: raise CompilerError("artifact SHA-256 does not match manifest.artifact_sha256") if declared_text_sha256 != text_sha256: raise CompilerError("extracted text SHA-256 does not match manifest.extracted_text_sha256") extractor = _require_object(manifest.get("extractor"), "manifest.extractor") _require_exact_shape(extractor, EXTRACTOR_KEYS_V3, "manifest.extractor") declared_spec_sha256 = _require_sha256(extractor.get("spec_sha256"), "manifest.extractor.spec_sha256") if declared_spec_sha256 != extractor_spec_sha256: raise CompilerError("extractor specification SHA-256 does not match manifest.extractor.spec_sha256") validated_extractor = { "name": _require_nonempty_string(extractor.get("name"), "manifest.extractor.name"), "version": _require_nonempty_string(extractor.get("version"), "manifest.extractor.version"), "spec_sha256": extractor_spec_sha256, } agent_id = _require_canonical_uuid(manifest.get("agent_id"), "manifest.agent_id") source = _require_object(manifest.get("source"), "manifest.source") _require_exact_shape(source, SOURCE_KEYS_V3, "manifest.source") source_type = source.get("source_type") if source_type not in ap.SOURCE_TYPES: raise CompilerError(f"manifest.source.source_type must be one of {sorted(ap.SOURCE_TYPES)}") source_access_scope = source.get("access_scope") if source_access_scope not in ap.V3_ACCESS_SCOPES: raise CompilerError(f"manifest.source.access_scope must be one of {sorted(ap.V3_ACCESS_SCOPES)}") ingestion_origin = source.get("ingestion_origin") if ingestion_origin not in ap.V3_SOURCE_ORIGINS: raise CompilerError(f"manifest.source.ingestion_origin must be one of {sorted(ap.V3_SOURCE_ORIGINS)}") provenance_status = source.get("provenance_status") if provenance_status not in ap.V3_PROVENANCE_STATES: raise CompilerError(f"manifest.source.provenance_status must be one of {sorted(ap.V3_PROVENANCE_STATES)}") source_class = source.get("source_class") if source_class not in ap.V3_SOURCE_CLASSES: raise CompilerError(f"manifest.source.source_class must be one of {sorted(ap.V3_SOURCE_CLASSES)}") auto_promotion_policy = source.get("auto_promotion_policy") if auto_promotion_policy not in ap.V3_AUTO_PROMOTION_POLICIES: raise CompilerError( f"manifest.source.auto_promotion_policy must be one of {sorted(ap.V3_AUTO_PROMOTION_POLICIES)}" ) source_identity = _require_stable_reference(source.get("identity"), "manifest.source.identity") source_locator = _require_stable_reference(source.get("locator"), "manifest.source.locator") expected_source_identity = f"document:sha256:{artifact_sha256}" expected_source_locator = f"artifact://sha256/{artifact_sha256}" if source_identity != expected_source_identity: raise CompilerError( "manifest.source.identity must equal the content-addressed identity for the actual artifact bytes" ) if source_locator != expected_source_locator: raise CompilerError("manifest.source.locator must equal artifact://sha256/") validated_source = { "identity": source_identity, "source_key": _require_key(source.get("source_key"), "manifest.source.source_key"), "source_type": source_type, "title": _require_nonempty_string(source.get("title"), "manifest.source.title"), "locator": source_locator, "access_scope": source_access_scope, "ingestion_origin": ingestion_origin, "provenance_status": provenance_status, "source_class": source_class, "auto_promotion_policy": auto_promotion_policy, "captured_at": _require_canonical_utc_timestamp(source.get("captured_at"), "manifest.source.captured_at"), } raw_claims = manifest.get("claims") if not isinstance(raw_claims, list) or not raw_claims: raise CompilerError("manifest.claims must be a non-empty list of objects") if len(raw_claims) > ap.MAX_BUNDLE_ROWS["claims"]: raise CompilerError(f"manifest.claims exceeds {ap.MAX_BUNDLE_ROWS['claims']} rows") validated_claims: list[dict[str, Any]] = [] quote_bindings: list[dict[str, Any]] = [] seen_claim_keys: set[str] = set() evidence_count = 0 for claim_index, raw_claim in enumerate(raw_claims): label = f"manifest.claims[{claim_index}]" claim = _require_object(raw_claim, label) _require_exact_shape(claim, CLAIM_KEYS_V3, label) claim_key = _require_key(claim.get("claim_key"), f"{label}.claim_key") if claim_key in seen_claim_keys: raise CompilerError(f"duplicate claim_key: {claim_key}") seen_claim_keys.add(claim_key) claim_type = claim.get("type") if claim_type not in ap.V3_CLAIM_TYPES: raise CompilerError(f"{label}.type must be one of {sorted(ap.V3_CLAIM_TYPES)}") access_scope = claim.get("access_scope") if access_scope not in ap.V3_ACCESS_SCOPES: raise CompilerError(f"{label}.access_scope must be one of {sorted(ap.V3_ACCESS_SCOPES)}") owner_agent_id = _require_canonical_uuid(claim.get("owner_agent_id"), f"{label}.owner_agent_id", nullable=True) if access_scope == "agent_private" and owner_agent_id is None: raise CompilerError(f"{label}.owner_agent_id is required for agent_private claims") status = claim.get("status") if status not in {"active", "contested"}: raise CompilerError(f"{label}.status must be active or contested") if claim.get("revision_kind") != "original" or claim.get("supersedes_id") is not None: raise CompilerError(f"{label} must describe an original claim with supersedes_id=null") raw_evidence = claim.get("evidence") if not isinstance(raw_evidence, list) or not raw_evidence: raise CompilerError(f"{label}.evidence must be a non-empty list of exact locator-bound rows") evidence_count += len(raw_evidence) if evidence_count > ap.MAX_BUNDLE_ROWS["evidence"]: raise CompilerError(f"manifest evidence exceeds {ap.MAX_BUNDLE_ROWS['evidence']} rows") validated_evidence: list[dict[str, Any]] = [] seen_evidence_keys: set[str] = set() seen_polarities: set[str] = set() for evidence_index, raw_row in enumerate(raw_evidence): evidence_label = f"{label}.evidence[{evidence_index}]" row = _require_object(raw_row, evidence_label) _require_exact_shape(row, EVIDENCE_KEYS_V3, evidence_label) evidence_key = _require_key(row.get("evidence_key"), f"{evidence_label}.evidence_key") if evidence_key in seen_evidence_keys: raise CompilerError(f"{evidence_label}.evidence_key duplicates an earlier evidence row") seen_evidence_keys.add(evidence_key) polarity = row.get("polarity") if polarity not in ap.V3_EVIDENCE_POLARITIES: raise CompilerError(f"{evidence_label}.polarity must be one of {sorted(ap.V3_EVIDENCE_POLARITIES)}") if polarity in seen_polarities: raise CompilerError(f"{evidence_label}.polarity duplicates a claim-local source/polarity key") seen_polarities.add(polarity) binding = _v3_quote_binding(text_bytes, row.get("quote"), row.get("locator"), evidence_label) validated_evidence.append( { "evidence_key": evidence_key, "quote": binding["quote"], "polarity": polarity, "locator": { "unit": binding["unit"], "start": binding["start"], "end": binding["end"], }, "rationale": _require_nonempty_string(row.get("rationale"), f"{evidence_label}.rationale"), } ) quote_bindings.append( { "claim_key": claim_key, "evidence_key": evidence_key, "kind": "evidence", **binding, } ) validated_evidence.sort(key=lambda row: row["evidence_key"]) validated_claims.append( { "claim_key": claim_key, "type": claim_type, "proposition": _require_nonempty_string(claim.get("proposition"), f"{label}.proposition"), "body_markdown": _require_nonempty_string(claim.get("body_markdown"), f"{label}.body_markdown"), "scope": _require_nonempty_string(claim.get("scope"), f"{label}.scope"), "access_scope": access_scope, "owner_agent_id": owner_agent_id, "status": status, "revision_criteria": _require_nonempty_string( claim.get("revision_criteria"), f"{label}.revision_criteria" ), "revision_kind": "original", "supersedes_id": None, "tags": _require_string_list(claim.get("tags"), f"{label}.tags"), "evidence": validated_evidence, "assessment": _validate_v3_assessment( claim.get("assessment"), claim_type, validated_evidence, f"{label}.assessment" ), } ) validated_claims.sort(key=lambda row: row["claim_key"]) quote_bindings.sort(key=lambda row: (row["claim_key"], row["evidence_key"])) return ( { "schema": MANIFEST_SCHEMA_V3, "artifact_sha256": artifact_sha256, "extracted_text_sha256": text_sha256, "extractor": validated_extractor, "agent_id": agent_id, "source": validated_source, "claims": validated_claims, }, quote_bindings, ) def _validate_dedupe(value: Any, extracted_text: str) -> dict[str, Any]: dedupe = _require_object(value, "manifest.dedupe") _require_exact_keys(dedupe, DEDUPE_KEYS, "manifest.dedupe") query = _require_nonempty_string(dedupe.get("database_search_query"), "manifest.dedupe.database_search_query") raw_candidates = dedupe.get("candidate_claims") if not isinstance(raw_candidates, list) or len(raw_candidates) > 20: raise CompilerError("manifest.dedupe.candidate_claims must be a list with at most 20 rows") candidates: list[dict[str, Any]] = [] candidate_ids: set[str] = set() for index, raw_candidate in enumerate(raw_candidates): label = f"manifest.dedupe.candidate_claims[{index}]" candidate = _require_object(raw_candidate, label) _require_exact_keys(candidate, DEDUPE_CANDIDATE_KEYS, label) try: claim_id = str(uuid.UUID(_require_nonempty_string(candidate.get("id"), f"{label}.id"))) except ValueError as exc: raise CompilerError(f"{label}.id must be a full UUID") from exc if claim_id in candidate_ids: raise CompilerError(f"{label}.id duplicates an earlier canonical candidate") candidate_ids.add(claim_id) candidates.append( { "id": claim_id, "text": _require_nonempty_string(candidate.get("text"), f"{label}.text"), "score": _require_nonnegative_integer(candidate.get("score"), f"{label}.score"), "evidence_count": _require_nonnegative_integer( candidate.get("evidence_count"), f"{label}.evidence_count" ), "edge_count": _require_nonnegative_integer(candidate.get("edge_count"), f"{label}.edge_count"), } ) raw_duplicates = dedupe.get("duplicates") if not isinstance(raw_duplicates, list) or len(raw_duplicates) > len(candidates): raise CompilerError("manifest.dedupe.duplicates must not exceed the retrieved candidate count") duplicates: list[dict[str, Any]] = [] duplicate_ids: set[str] = set() for index, raw_duplicate in enumerate(raw_duplicates): label = f"manifest.dedupe.duplicates[{index}]" duplicate = _require_object(raw_duplicate, label) _require_exact_keys(duplicate, DEDUPE_DUPLICATE_KEYS, label) try: claim_id = str(uuid.UUID(_require_nonempty_string(duplicate.get("claim_id"), f"{label}.claim_id"))) except ValueError as exc: raise CompilerError(f"{label}.claim_id must be a full UUID") from exc if claim_id not in candidate_ids: raise CompilerError(f"{label}.claim_id was not present in the canonical retrieval candidates") if claim_id in duplicate_ids: raise CompilerError(f"{label}.claim_id duplicates an earlier duplicate judgment") duplicate_ids.add(claim_id) source_quote = _quote_binding(extracted_text, duplicate.get("source_quote"), f"{label}.source_quote") duplicates.append( { "claim_id": claim_id, "source_quote": source_quote["quote"], "reason": _require_nonempty_string(duplicate.get("reason"), f"{label}.reason"), } ) return { "database_search_query": query, "candidate_claims": candidates, "duplicates": duplicates, } def _validate_manifest( manifest: dict[str, Any], artifact_sha256: str, text_sha256: str, extracted_text: str ) -> tuple[dict[str, Any], list[dict[str, Any]]]: manifest_schema = manifest.get("schema") if manifest_schema not in MANIFEST_SCHEMAS: raise CompilerError(f"manifest.schema must be one of {sorted(MANIFEST_SCHEMAS)}") _require_exact_keys( manifest, TOP_LEVEL_KEYS_V2 if manifest_schema == MANIFEST_SCHEMA_V2 else TOP_LEVEL_KEYS, "manifest", ) declared_artifact_sha256 = _require_sha256(manifest.get("artifact_sha256"), "manifest.artifact_sha256") declared_text_sha256 = _require_sha256(manifest.get("extracted_text_sha256"), "manifest.extracted_text_sha256") if declared_artifact_sha256 != artifact_sha256: raise CompilerError("artifact SHA-256 does not match manifest.artifact_sha256") if declared_text_sha256 != text_sha256: raise CompilerError("extracted text SHA-256 does not match manifest.extracted_text_sha256") extractor = _require_object(manifest.get("extractor"), "manifest.extractor") _require_exact_keys(extractor, EXTRACTOR_KEYS, "manifest.extractor") extractor_name = _require_nonempty_string(extractor.get("name"), "manifest.extractor.name") extractor_version = _require_nonempty_string(extractor.get("version"), "manifest.extractor.version") source = _require_object(manifest.get("source"), "manifest.source") _require_exact_keys(source, SOURCE_KEYS, "manifest.source") source_identity = _require_stable_reference(source.get("identity"), "manifest.source.identity") source_locator = _require_stable_reference(source.get("locator"), "manifest.source.locator") source_key = _require_key(source.get("source_key"), "manifest.source.source_key") source_title = _require_nonempty_string(source.get("title"), "manifest.source.title") source_type = _require_nonempty_string(source.get("source_type"), "manifest.source.source_type") if source_type not in ap.SOURCE_TYPES: raise CompilerError(f"manifest.source.source_type must be one of {sorted(ap.SOURCE_TYPES)}") claims = manifest.get("claims") if not isinstance(claims, list) or not claims: raise CompilerError("manifest.claims must be a non-empty list of objects") if len(claims) > ap.MAX_BUNDLE_ROWS["claims"]: raise CompilerError(f"manifest.claims exceeds {ap.MAX_BUNDLE_ROWS['claims']} rows") validated_claims: list[dict[str, Any]] = [] quote_bindings: list[dict[str, Any]] = [] seen_claim_keys: set[str] = set() derived_source_keys = {source_key} for claim_index, raw_claim in enumerate(claims): label = f"manifest.claims[{claim_index}]" claim = _require_object(raw_claim, label) _require_exact_keys(claim, CLAIM_KEYS, label) claim_key = _require_key(claim.get("claim_key"), f"{label}.claim_key") if claim_key in seen_claim_keys: raise CompilerError(f"duplicate claim_key: {claim_key}") seen_claim_keys.add(claim_key) body_source_key = f"{claim_key}__proposal_body" if body_source_key in derived_source_keys: raise CompilerError(f"duplicate source_key after normalization: {body_source_key}") derived_source_keys.add(body_source_key) claim_type = _require_nonempty_string(claim.get("type"), f"{label}.type") if claim_type not in ap.CLAIM_TYPES: raise CompilerError(f"{label}.type must be one of {sorted(ap.CLAIM_TYPES)}") claim_text = _require_nonempty_string(claim.get("text"), f"{label}.text") claim_quote = _quote_binding(extracted_text, claim.get("quote"), f"{label}.quote") quote_bindings.append({"claim_key": claim_key, "kind": "claim", **claim_quote}) confidence = claim.get("confidence") if confidence is not None and ( isinstance(confidence, bool) or not isinstance(confidence, (int, float)) or not 0 <= confidence <= 1 ): raise CompilerError(f"{label}.confidence must be a number from 0 to 1 or null") tags = claim.get("tags", []) if not isinstance(tags, list) or any( not isinstance(tag, str) or not tag.strip() or tag != tag.strip() for tag in tags ): raise CompilerError(f"{label}.tags must be a list of non-empty strings") if len(tags) != len(set(tags)): raise CompilerError(f"{label}.tags contains duplicates") evidence = claim.get("evidence") if not isinstance(evidence, list) or not evidence: raise CompilerError(f"{label}.evidence must be a non-empty list of exact-quote objects") validated_evidence: list[dict[str, Any]] = [] seen_evidence: set[tuple[str, str]] = set() for evidence_index, raw_evidence in enumerate(evidence): evidence_label = f"{label}.evidence[{evidence_index}]" evidence_row = _require_object(raw_evidence, evidence_label) _require_exact_keys(evidence_row, EVIDENCE_KEYS, evidence_label) role = _require_nonempty_string(evidence_row.get("role"), f"{evidence_label}.role") if role not in ap.EVIDENCE_ROLES: raise CompilerError(f"{evidence_label}.role must be one of {sorted(ap.EVIDENCE_ROLES)}") binding = _quote_binding(extracted_text, evidence_row.get("quote"), f"{evidence_label}.quote") evidence_key = (binding["quote"], role) if evidence_key in seen_evidence: raise CompilerError(f"{evidence_label} duplicates an earlier evidence quote and role") seen_evidence.add(evidence_key) quote_bindings.append( { "claim_key": claim_key, "kind": "evidence", "evidence_role": role, **binding, } ) validated_evidence.append({"quote": binding["quote"], "role": role}) validated_claims.append( { "claim_key": claim_key, "type": claim_type, "text": claim_text, "quote": claim_quote["quote"], "confidence": confidence, "tags": tags, "evidence": validated_evidence, } ) validated_manifest = { "schema": manifest_schema, "artifact_sha256": artifact_sha256, "extracted_text_sha256": text_sha256, "extractor": {"name": extractor_name, "version": extractor_version}, "source": { "identity": source_identity, "source_key": source_key, "source_type": source_type, "title": source_title, "locator": source_locator, }, "claims": validated_claims, } if manifest_schema == MANIFEST_SCHEMA_V2: validated_manifest["dedupe"] = _validate_dedupe(manifest.get("dedupe"), extracted_text) return validated_manifest, quote_bindings def _locator_fields(locator: str) -> tuple[str | None, str | None]: scheme = urlsplit(locator).scheme.lower() if scheme in {"http", "https"}: return locator, None return None, locator def build_parent_proposal( manifest: dict[str, Any], manifest_sha256: str, quote_bindings: list[dict[str, Any]] ) -> dict[str, Any]: source = manifest["source"] parent_seed = ":".join((manifest["artifact_sha256"], manifest["extracted_text_sha256"], manifest_sha256)) parent_id = stable_uuid(f"parent:{parent_seed}") url, storage_path = _locator_fields(source["locator"]) excerpts: list[dict[str, Any]] = [] seen_quotes: set[str] = set() for binding in quote_bindings: quote = binding["quote"] if quote not in seen_quotes: seen_quotes.add(quote) excerpts.append({"text": quote}) claim_candidates = [] for claim in manifest["claims"]: claim_candidates.append( { "claim_key": claim["claim_key"], "type": claim["type"], "headline": claim["text"], "text": claim["text"], "body": claim["quote"], "confidence": claim["confidence"], "tags": claim["tags"], "evidence_tier": "hash_bound_exact_quote", "needs_research": False, "evidence": [{"source_key": source["source_key"], "role": item["role"]} for item in claim["evidence"]], "edges": [], } ) provenance_summary = ( f"identity={source['identity']}; locator={source['locator']}; " f"artifact_sha256={manifest['artifact_sha256']}; " f"extracted_text_sha256={manifest['extracted_text_sha256']}; " f"extractor={manifest['extractor']['name']}@{manifest['extractor']['version']}" ) dedupe = manifest.get("dedupe") or {} return { "id": parent_id, "proposal_type": "approve_claim", "status": "pending_review", "proposed_by_handle": "leo", "proposed_by_agent_id": None, "channel": "source_document_compiler", "source_ref": ( f"source-compiler:{manifest['artifact_sha256'][:16]}:" f"{manifest['extracted_text_sha256'][:16]}:{manifest_sha256[:16]}" ), "rationale": "Compile one hash-bound source artifact into exact-quote claim candidates for review.", "payload": { "source_document": { "manifest_schema": manifest["schema"], "manifest_sha256": manifest_sha256, "artifact_sha256": manifest["artifact_sha256"], "extracted_text_sha256": manifest["extracted_text_sha256"], "source_identity": source["identity"], "source_locator": source["locator"], "extractor": manifest["extractor"], }, "claim_candidates": claim_candidates, "source_candidates": [ { "source_key": source["source_key"], "source_type": source["source_type"], "title": source["title"], "url": url, "storage_path": storage_path, "source_quality": provenance_summary, "usage_limits": "Only the exact bound quotes may ground claims; human review is required.", "key_excerpts": excerpts, "content_sha256": manifest["artifact_sha256"], } ], "dedupe_conflict_assessment": { "database_search_query": dedupe.get("database_search_query") or " ".join(claim["text"] for claim in manifest["claims"]), "canonical_candidates": dedupe.get("candidate_claims", []), "potential_existing_claim_ids": [item["claim_id"] for item in dedupe.get("duplicates", [])], "conflicts": dedupe.get("duplicates", []), "review_required": True, }, }, } def _v3_identity_material(manifest: dict[str, Any], manifest_file_sha256: str) -> tuple[dict[str, Any], str, str]: candidate_content = { "agent_id": manifest["agent_id"], "source": manifest["source"], "claims": manifest["claims"], } candidate_content_sha256 = canonical_sha256(candidate_content) identity_material = { "artifact_sha256": manifest["artifact_sha256"], "extracted_text_sha256": manifest["extracted_text_sha256"], "extractor": manifest["extractor"], "manifest_file_sha256": manifest_file_sha256, "source_identity": manifest["source"]["identity"], "source_locator": manifest["source"]["locator"], "normalized_candidate_content_sha256": candidate_content_sha256, } return identity_material, candidate_content_sha256, canonical_sha256(identity_material) def _build_v3_parent_proposal( manifest: dict[str, Any], manifest_canonical_sha256: str, identity_material: dict[str, Any], candidate_content_sha256: str, packet_identity_sha256: str, ) -> dict[str, Any]: source = manifest["source"] parent_id = stable_uuid_v3("source_proposal", packet_identity_sha256, candidate_content_sha256) return { "id": parent_id, "proposal_type": "approve_claim", "status": "pending_review", "proposed_by_handle": "leo", "proposed_by_agent_id": manifest["agent_id"], "channel": "source_document_compiler_v3", "source_ref": f"source-compiler-v3:{packet_identity_sha256[:32]}", "rationale": "Compile one retained semantic extraction into a hash-bound V3 claim proposal for review.", "payload": { "source_document": { "manifest_schema": MANIFEST_SCHEMA_V3, "manifest_file_sha256": identity_material["manifest_file_sha256"], "manifest_canonical_sha256": manifest_canonical_sha256, "artifact_sha256": manifest["artifact_sha256"], "extracted_text_sha256": manifest["extracted_text_sha256"], "source_identity": source["identity"], "source_locator": source["locator"], "extractor": manifest["extractor"], }, "identity_material": identity_material, "normalized_candidate_content_sha256": candidate_content_sha256, "packet_identity_sha256": packet_identity_sha256, "semantic_extraction_is_retained_input": True, "model_training_claim": False, }, } def _v3_source_excerpt(manifest: dict[str, Any]) -> str: excerpts: list[str] = [] seen: set[str] = set() for claim in manifest["claims"]: for evidence in claim["evidence"]: quote = evidence["quote"] if quote not in seen: seen.add(quote) excerpts.append(quote) return "\n\n".join(excerpts) def _v3_locator_json(manifest: dict[str, Any], binding: dict[str, Any]) -> dict[str, Any]: return { "schema": "livingip.extractedTextQuoteLocator.v1", "source_identity": manifest["source"]["identity"], "source_locator": manifest["source"]["locator"], "artifact_sha256": manifest["artifact_sha256"], "extracted_text_sha256": manifest["extracted_text_sha256"], "extractor": manifest["extractor"], "unit": binding["unit"], "start": binding["start"], "end": binding["end"], "quote_sha256": binding["quote_sha256"], } def _ensure_distinct_v3_ids(values: list[str]) -> None: if len(values) != len(set(values)): raise CompilerError("compiled V3 packet contains duplicate deterministic ids") def _build_v3_strict_child( manifest: dict[str, Any], manifest_canonical_sha256: str, quote_bindings: list[dict[str, Any]], parent: dict[str, Any], identity_material: dict[str, Any], candidate_content_sha256: str, packet_identity_sha256: str, ) -> dict[str, Any]: source = manifest["source"] source_id = stable_uuid_v3("source", packet_identity_sha256, source["source_key"]) claim_ids = { claim["claim_key"]: stable_uuid_v3("claim", packet_identity_sha256, claim["claim_key"]) for claim in manifest["claims"] } binding_by_key = {(binding["claim_key"], binding["evidence_key"]): binding for binding in quote_bindings} canonical_url, storage_uri = _locator_fields(source["locator"]) source_row = { "id": source_id, "source_type": source["source_type"], "canonical_url": canonical_url, "storage_uri": storage_uri, "excerpt": _v3_source_excerpt(manifest), "content_hash": manifest["artifact_sha256"], "access_scope": source["access_scope"], "ingestion_origin": source["ingestion_origin"], "provenance_status": source["provenance_status"], "source_class": source["source_class"], "auto_promotion_policy": source["auto_promotion_policy"], "captured_at": source["captured_at"], "created_by": manifest["agent_id"], } claim_rows: list[dict[str, Any]] = [] evidence_rows: list[dict[str, Any]] = [] assessment_rows: list[dict[str, Any]] = [] for claim in manifest["claims"]: claim_id = claim_ids[claim["claim_key"]] claim_rows.append( { "id": claim_id, "owner_agent_id": claim["owner_agent_id"], "type": claim["type"], "proposition": claim["proposition"], "body_markdown": claim["body_markdown"], "scope": claim["scope"], "access_scope": claim["access_scope"], "status": claim["status"], "revision_criteria": claim["revision_criteria"], "revision_kind": "original", "supersedes_id": None, "tags": claim["tags"], "created_by": manifest["agent_id"], } ) for evidence in claim["evidence"]: binding = binding_by_key[(claim["claim_key"], evidence["evidence_key"])] evidence_rows.append( { "id": stable_uuid_v3( "evidence", packet_identity_sha256, [claim["claim_key"], evidence["evidence_key"]], ), "claim_id": claim_id, "source_id": source_id, "polarity": evidence["polarity"], "excerpt": evidence["quote"], "locator_json": _v3_locator_json(manifest, binding), "source_content_hash": manifest["artifact_sha256"], "rationale": evidence["rationale"], "created_by": manifest["agent_id"], } ) for claim in manifest["claims"]: claim_id = claim_ids[claim["claim_key"]] assessment = claim["assessment"] evidence_set_hash = ap._v3_evidence_set_hash(claim_id, evidence_rows) assessment_rows.append( { "id": stable_uuid_v3("assessment", packet_identity_sha256, claim["claim_key"]), "claim_id": claim_id, "support_tier": assessment["support_tier"], "challenge_tier": assessment["challenge_tier"], "overall_state": assessment["overall_state"], "rationale": assessment["rationale"], "evidence_set_hash": evidence_set_hash, "supersedes_assessment_id": None, } ) parent_packet_sha256 = canonical_sha256(parent) apply_payload = { "contract_version": 3, "source_proposal_id": parent["id"], "agent_id": manifest["agent_id"], "claims": claim_rows, "sources": [source_row], "evidence": evidence_rows, "edges": [], "assessments": assessment_rows, "reasoning_tools": [], "normalization_manifest": { "schema": "livingip.kbSourceProposalNormalization.v3", "semantic_extraction_statement": ( "The semantic extraction is a retained input; this is a reproducible learning input, " "not a model-training claim." ), "semantic_extraction_is_retained_input": True, "model_training_claim": False, "manifest_schema": MANIFEST_SCHEMA_V3, "manifest_file_sha256": identity_material["manifest_file_sha256"], "manifest_identity_binding": "sha256(exact duplicate-free UTF-8 JSON file bytes)", "manifest_canonical_sha256": manifest_canonical_sha256, "artifact_sha256": manifest["artifact_sha256"], "extracted_text_sha256": manifest["extracted_text_sha256"], "extractor": manifest["extractor"], "source_identity": source["identity"], "source_locator": source["locator"], "normalized_candidate_content_sha256": candidate_content_sha256, "identity_material": identity_material, "packet_identity_sha256": packet_identity_sha256, "parent_packet_sha256": parent_packet_sha256, "quote_bindings": quote_bindings, }, } payload = {"apply_payload": apply_payload} payload_sha256 = canonical_sha256(payload) child_id = stable_uuid_v3("pending_review_proposal", packet_identity_sha256, payload_sha256) _ensure_distinct_v3_ids( [ parent["id"], child_id, source_id, *[row["id"] for row in claim_rows], *[row["id"] for row in evidence_rows], *[row["id"] for row in assessment_rows], ] ) return { "id": child_id, "proposal_type": "approve_claim", "status": "pending_review", "proposed_by_handle": parent["proposed_by_handle"], "proposed_by_agent_id": manifest["agent_id"], "channel": parent["channel"], "source_ref": f"source-compiler-v3:{packet_identity_sha256[:32]}:{payload_sha256[:16]}", "rationale": f"Strict V3 source proposal compiled from retained packet {parent['id']}.", "payload": payload, "parent_packet_sha256": parent_packet_sha256, "payload_sha256": payload_sha256, } def _validate_v3_compiled_child( child: dict[str, Any], manifest: dict[str, Any], quote_bindings: list[dict[str, Any]] ) -> None: if child.get("status") != "pending_review" or child.get("proposal_type") != "approve_claim": raise CompilerError("V3 compilation did not produce one pending_review approve_claim proposal") payload = child.get("payload") apply_payload = payload.get("apply_payload") if isinstance(payload, dict) else None if not isinstance(apply_payload, dict) or apply_payload.get("contract_version") != 3: raise CompilerError("strict child has no schema-valid approve_claim V3 apply_payload") if apply_payload.get("reasoning_tools") != [] or apply_payload.get("edges") != []: raise CompilerError("V3 source compilation admits neither reasoning_tools nor inferred edges") if child.get("payload_sha256") != canonical_sha256(payload): raise CompilerError("strict child payload_sha256 does not bind the exact V3 payload") claims = apply_payload.get("claims") or [] sources = apply_payload.get("sources") or [] evidence = apply_payload.get("evidence") or [] assessments = apply_payload.get("assessments") or [] if len(claims) != len(manifest["claims"]) or len(sources) != 1: raise CompilerError("V3 strict child row counts differ from the extraction manifest") if len(evidence) != sum(len(claim["evidence"]) for claim in manifest["claims"]): raise CompilerError("V3 strict child evidence count differs from the extraction manifest") if len(assessments) != len(claims): raise CompilerError("V3 strict child requires exactly one assessment per claim") _ensure_distinct_v3_ids( [ child["id"], *[row.get("id") for row in sources], *[row.get("id") for row in claims], *[row.get("id") for row in evidence], *[row.get("id") for row in assessments], ] ) source_row = sources[0] if source_row.get("content_hash") != manifest["artifact_sha256"]: raise CompilerError("V3 source content_hash does not bind the original artifact") expected_url, expected_storage_uri = _locator_fields(manifest["source"]["locator"]) if source_row.get("canonical_url") != expected_url or source_row.get("storage_uri") != expected_storage_uri: raise CompilerError("V3 source row does not retain the validated content-addressed locator") if source_row.get("captured_at") != _require_canonical_utc_timestamp( source_row.get("captured_at"), "V3 source captured_at" ): raise CompilerError("V3 source captured_at is not in writer-canonical fixed-microsecond UTC Z form") binding_by_locator = {canonical_sha256(_v3_locator_json(manifest, binding)): binding for binding in quote_bindings} for row in evidence: binding = binding_by_locator.get(canonical_sha256(row.get("locator_json"))) if binding is None or row.get("excerpt") != binding["quote"]: raise CompilerError("V3 evidence does not retain an exact locator-bound quote") if row.get("source_content_hash") != manifest["artifact_sha256"]: raise CompilerError("V3 evidence source_content_hash does not bind the original artifact") assessment_by_claim = {row.get("claim_id"): row for row in assessments} if len(assessment_by_claim) != len(claims): raise CompilerError("V3 strict child contains duplicate assessment claim ids") for claim in claims: assessment = assessment_by_claim.get(claim.get("id")) if assessment is None: raise CompilerError("V3 strict child is missing a claim-local evidence assessment") if assessment.get("evidence_set_hash") != ap._v3_evidence_set_hash(claim["id"], evidence): raise CompilerError("V3 assessment evidence_set_hash does not match claim-local evidence") def _compile_v3_source_packet( artifact_bytes: bytes, text_bytes: bytes, manifest_bytes: bytes, extractor_spec_bytes: bytes, manifest_object: dict[str, Any], ) -> dict[str, Any]: artifact_sha256 = sha256_bytes(artifact_bytes) text_sha256 = sha256_bytes(text_bytes) extractor_spec_sha256 = sha256_bytes(extractor_spec_bytes) manifest_file_sha256 = sha256_bytes(manifest_bytes) manifest, quote_bindings = _validate_manifest_v3( manifest_object, artifact_sha256, text_sha256, extractor_spec_sha256, text_bytes, ) manifest_canonical_sha256 = canonical_sha256(manifest) identity_material, candidate_content_sha256, packet_identity_sha256 = _v3_identity_material( manifest, manifest_file_sha256 ) parent = _build_v3_parent_proposal( manifest, manifest_canonical_sha256, identity_material, candidate_content_sha256, packet_identity_sha256, ) child = _build_v3_strict_child( manifest, manifest_canonical_sha256, quote_bindings, parent, identity_material, candidate_content_sha256, packet_identity_sha256, ) _validate_v3_compiled_child(child, manifest, quote_bindings) stage_sql = stage.build_stage_sql(child) stable_bundle = { "schema": BUNDLE_SCHEMA_V3, "status": "pending_review", "build_only": True, "database_write_performed": False, "canonical_apply_performed": False, "inputs": { "artifact_bytes": len(artifact_bytes), "extracted_text_bytes": len(text_bytes), "manifest_bytes": len(manifest_bytes), "extractor_spec_bytes": len(extractor_spec_bytes), }, "validated_manifest": manifest, "quote_bindings": quote_bindings, "parent_proposal": parent, "strict_child_proposal": child, "stage_preview": { "target_table": "kb_stage.kb_proposals", "target_status": "pending_review", "executed": False, "stage_sql_sha256": sha256_bytes(stage_sql.encode("utf-8")), }, "compiler_contract": { "manifest_contract": MANIFEST_SCHEMA_V3, "output_contract": BUNDLE_SCHEMA_V3, "approve_claim_contract_version": 3, "strict_preflight": "scripts/compile_kb_source_packet.py::_validate_v3_compiled_child", "writer_apply_invoked": False, "writer_timestamp_contract": "fixed-microsecond UTC Z", "manifest_identity_binding": "sha256(exact duplicate-free UTF-8 JSON file bytes)", "extractor_spec_binding": "sha256(exact independently supplied artifact bytes)", "semantic_extraction_is_retained_input": True, "semantic_extraction_statement": ( "The semantic extraction is a retained input; this is the smallest reproducible learning input, " "not a model-training claim." ), "model_training_claim": False, "source_from_chat_labels": False, "database_stage_executed": False, }, "hashes": { "artifact_sha256": artifact_sha256, "extracted_text_sha256": text_sha256, "extractor_spec_sha256": extractor_spec_sha256, "manifest_file_sha256": manifest_file_sha256, "manifest_canonical_sha256": manifest_canonical_sha256, "normalized_candidate_content_sha256": candidate_content_sha256, "packet_identity_sha256": packet_identity_sha256, "parent_packet_sha256": child["parent_packet_sha256"], "strict_child_payload_sha256": child["payload_sha256"], }, } stable_bundle["hashes"]["bundle_content_sha256"] = canonical_sha256(stable_bundle) return stable_bundle def _validate_compiled_child( child: dict[str, Any], manifest: dict[str, Any], quote_bindings: list[dict[str, Any]] ) -> None: if child.get("status") != "pending_review" or child.get("proposal_type") != "approve_claim": raise CompilerError("normalization did not produce one pending_review approve_claim child") apply_payload = (child.get("payload") or {}).get("apply_payload") if not isinstance(apply_payload, dict) or apply_payload.get("contract_version") != 2: raise CompilerError("strict child has no schema-valid approve_claim v2 apply_payload") claims = apply_payload.get("claims") or [] sources = apply_payload.get("sources") or [] if len(claims) != len(manifest["claims"]): raise CompilerError("strict child claim count differs from the extraction manifest") if len({row.get("id") for row in claims}) != len(claims): raise CompilerError("strict child contains duplicate claim ids") if [row.get("text") for row in claims] != [row["text"] for row in manifest["claims"]]: raise CompilerError("strict child claim text differs from the extracted atomic propositions") if len({row.get("id") for row in sources}) != len(sources): raise CompilerError("strict child contains duplicate source ids") source = manifest["source"] artifact_sources = [ row for row in sources if row.get("hash") == manifest["artifact_sha256"] and (row.get("url") == source["locator"] or row.get("storage_path") == source["locator"]) ] if len(artifact_sources) != 1: raise CompilerError("strict child must contain exactly one hash-bound declared artifact source") try: excerpt_packet = json.loads(artifact_sources[0]["excerpt"]) except (TypeError, json.JSONDecodeError) as exc: raise CompilerError("declared source excerpt was truncated or is not reviewable JSON") from exc retained_excerpt = excerpt_packet.get("excerpt") if isinstance(excerpt_packet, dict) else None if not isinstance(retained_excerpt, str): raise CompilerError("declared source excerpt does not retain exact quote text") missing_quotes = sorted( {binding["quote"] for binding in quote_bindings if binding["quote"] not in retained_excerpt} ) if missing_quotes: raise CompilerError("one or more exact quotes exceed the current canonical source excerpt capacity") def compile_source_packet( artifact_path: Path, text_path: Path, manifest_path: Path, extractor_spec_path: Path | None = None, ) -> dict[str, Any]: artifact_bytes = _read_required_file(artifact_path, "artifact") text_bytes = _read_required_file(text_path, "extracted text") manifest_bytes = _read_required_file(manifest_path, "extraction manifest") try: extracted_text = text_bytes.decode("utf-8", errors="strict") except UnicodeDecodeError as exc: raise CompilerError(f"extracted text must be valid UTF-8: {exc}") from exc manifest_raw = _load_duplicate_free_json(manifest_bytes, "extraction manifest") manifest_object = _require_object(manifest_raw, "manifest") artifact_sha256 = sha256_bytes(artifact_bytes) text_sha256 = sha256_bytes(text_bytes) if manifest_object.get("schema") == MANIFEST_SCHEMA_V3: if extractor_spec_path is None: raise CompilerError("V3 requires an independently supplied extractor specification artifact") extractor_spec_bytes = _read_required_file(extractor_spec_path, "extractor specification") return _compile_v3_source_packet( artifact_bytes, text_bytes, manifest_bytes, extractor_spec_bytes, manifest_object, ) if extractor_spec_path is not None: raise CompilerError("extractor specification input is only valid for a V3 manifest") manifest, quote_bindings = _validate_manifest(manifest_object, artifact_sha256, text_sha256, extracted_text) manifest_canonical_sha256 = canonical_sha256(manifest) parent = build_parent_proposal(manifest, manifest_canonical_sha256, quote_bindings) try: child = stage.prepare_normalized_child(parent) except (ValueError, stage.bound.CheckpointError) as exc: raise CompilerError(f"strict child preflight rejected the source packet: {exc}") from exc _validate_compiled_child(child, manifest, quote_bindings) stage_sql = stage.build_stage_sql(child) stable_bundle = { "schema": BUNDLE_SCHEMA, "status": "pending_review", "build_only": True, "database_write_performed": False, "canonical_apply_performed": False, "inputs": { "artifact_bytes": len(artifact_bytes), "extracted_text_bytes": len(text_bytes), "manifest_bytes": len(manifest_bytes), }, "validated_manifest": manifest, "quote_bindings": quote_bindings, "parent_proposal": parent, "strict_child_proposal": child, "stage_preview": { "target_table": "kb_stage.kb_proposals", "executed": False, "stage_sql_sha256": sha256_bytes(stage_sql.encode("utf-8")), }, "compiler_contract": { "normalizer": "scripts/kb_proposal_normalize.py", "strict_preflight": "scripts/stage_normalized_proposal.py", "declared_artifact_source_count": 1, "source_from_chat_labels": False, }, "hashes": { "artifact_sha256": artifact_sha256, "extracted_text_sha256": text_sha256, "manifest_file_sha256": sha256_bytes(manifest_bytes), "manifest_canonical_sha256": manifest_canonical_sha256, "parent_packet_sha256": child["parent_packet_sha256"], "strict_child_payload_sha256": child["payload_sha256"], }, } stable_bundle["hashes"]["bundle_content_sha256"] = canonical_sha256(stable_bundle) return stable_bundle def parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--artifact", required=True, type=Path, help="raw source artifact; any binary format") parser.add_argument("--text", required=True, type=Path, help="strict UTF-8 extraction of the artifact") parser.add_argument("--manifest", required=True, type=Path, help="extraction manifest JSON") parser.add_argument( "--extractor-spec", type=Path, help="exact extractor specification artifact; required for V3 and rejected for V1/V2", ) parser.add_argument("--output", required=True, type=Path, help="private path for the compiled proposal bundle") return parser.parse_args(argv) def _write_private_output(path: Path, rendered: str) -> None: path.parent.mkdir(parents=True, exist_ok=True, mode=0o700) fd, raw_temp = tempfile.mkstemp(prefix=f".{path.name}.", suffix=".tmp", dir=path.parent) temp = Path(raw_temp) try: os.fchmod(fd, 0o600) with os.fdopen(fd, "w", encoding="utf-8") as handle: handle.write(rendered) handle.flush() os.fsync(handle.fileno()) os.replace(temp, path) os.chmod(path, 0o600) finally: if temp.exists(): temp.unlink() def _print_status(status: str, *, reason: str | None = None) -> None: payload: dict[str, Any] = { "schema": STATUS_SCHEMA, "status": status, "database_write_performed": False, "canonical_apply_performed": False, "private_bundle_written": status == "pending_review", } if reason is not None: payload["reason"] = reason print(json.dumps(payload, sort_keys=True)) def main(argv: list[str] | None = None) -> int: args = parse_args(argv) try: output_path = args.output.resolve() input_paths = {args.artifact.resolve(), args.text.resolve(), args.manifest.resolve()} if args.extractor_spec is not None: input_paths.add(args.extractor_spec.resolve()) if output_path in input_paths: raise CompilerError("--output must not overwrite an input file") bundle = compile_source_packet(args.artifact, args.text, args.manifest, args.extractor_spec) except (CompilerError, OSError, ValueError): _print_status("rejected", reason="input_validation_failed") return 2 rendered = json.dumps(bundle, indent=2, sort_keys=True, ensure_ascii=True) + "\n" try: _write_private_output(args.output, rendered) except OSError: _print_status("rejected", reason="output_write_failed") return 2 _print_status("pending_review") return 0 if __name__ == "__main__": raise SystemExit(main())