1443 lines
65 KiB
Python
1443 lines
65 KiB
Python
#!/usr/bin/env python3
|
|
"""Compile one hash-bound source document into a pending-review KB proposal.
|
|
|
|
The command is deliberately build-only. It validates a raw artifact, its UTF-8
|
|
text extraction, and an extraction manifest. V1/V2 reuse Leo's rich proposal
|
|
normalizer; V3 additionally requires the exact extractor specification and
|
|
builds the explicit epistemic payload. The command never connects to Postgres
|
|
or executes the generated staging SQL.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import re
|
|
import sys
|
|
import tempfile
|
|
import uuid
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from typing import Any
|
|
from urllib.parse import urlsplit
|
|
|
|
HERE = Path(__file__).resolve().parent
|
|
sys.path.insert(0, str(HERE))
|
|
|
|
import apply_proposal as ap # noqa: E402
|
|
import stage_normalized_proposal as stage # noqa: E402
|
|
|
|
MANIFEST_SCHEMA = "livingip.kbSourceExtractionManifest.v1"
|
|
MANIFEST_SCHEMA_V2 = "livingip.kbSourceExtractionManifest.v2"
|
|
MANIFEST_SCHEMAS = {MANIFEST_SCHEMA, MANIFEST_SCHEMA_V2}
|
|
MANIFEST_SCHEMA_V3 = "livingip.kbSourceExtractionManifest.v3"
|
|
BUNDLE_SCHEMA = "livingip.kbSourceProposalBundle.v1"
|
|
BUNDLE_SCHEMA_V3 = "livingip.kbSourceProposalBundle.v3"
|
|
STATUS_SCHEMA = "livingip.kbSourceCompilerStatus.v1"
|
|
SHA256_RE = re.compile(r"^[0-9a-f]{64}$")
|
|
KEY_RE = re.compile(r"^[a-z0-9][a-z0-9._:-]{0,127}$")
|
|
URI_SCHEME_RE = re.compile(r"^[a-z][a-z0-9+.-]*:", re.IGNORECASE)
|
|
ISO_TIMESTAMP_RE = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{1,6})?(?:Z|[+-]\d{2}:\d{2})$")
|
|
UNSAFE_URI_SCHEMES = {"data", "javascript", "vbscript"}
|
|
|
|
TOP_LEVEL_KEYS = {
|
|
"schema",
|
|
"artifact_sha256",
|
|
"extracted_text_sha256",
|
|
"extractor",
|
|
"source",
|
|
"claims",
|
|
}
|
|
TOP_LEVEL_KEYS_V2 = TOP_LEVEL_KEYS | {"dedupe"}
|
|
EXTRACTOR_KEYS = {"name", "version"}
|
|
SOURCE_KEYS = {"identity", "source_key", "source_type", "title", "locator"}
|
|
CLAIM_KEYS = {"claim_key", "type", "text", "quote", "confidence", "tags", "evidence"}
|
|
EVIDENCE_KEYS = {"quote", "role"}
|
|
DEDUPE_KEYS = {"database_search_query", "candidate_claims", "duplicates"}
|
|
DEDUPE_CANDIDATE_KEYS = {"id", "text", "score", "evidence_count", "edge_count"}
|
|
DEDUPE_DUPLICATE_KEYS = {"claim_id", "source_quote", "reason"}
|
|
|
|
TOP_LEVEL_KEYS_V3 = {
|
|
"schema",
|
|
"artifact_sha256",
|
|
"extracted_text_sha256",
|
|
"extractor",
|
|
"agent_id",
|
|
"source",
|
|
"claims",
|
|
}
|
|
EXTRACTOR_KEYS_V3 = {"name", "version", "spec_sha256"}
|
|
SOURCE_KEYS_V3 = {
|
|
"identity",
|
|
"source_key",
|
|
"source_type",
|
|
"title",
|
|
"locator",
|
|
"access_scope",
|
|
"ingestion_origin",
|
|
"provenance_status",
|
|
"source_class",
|
|
"auto_promotion_policy",
|
|
"captured_at",
|
|
}
|
|
CLAIM_KEYS_V3 = {
|
|
"claim_key",
|
|
"type",
|
|
"proposition",
|
|
"body_markdown",
|
|
"scope",
|
|
"access_scope",
|
|
"owner_agent_id",
|
|
"status",
|
|
"revision_criteria",
|
|
"revision_kind",
|
|
"supersedes_id",
|
|
"tags",
|
|
"evidence",
|
|
"assessment",
|
|
}
|
|
EVIDENCE_KEYS_V3 = {"evidence_key", "quote", "polarity", "locator", "rationale"}
|
|
LOCATOR_KEYS_V3 = {"unit", "start", "end"}
|
|
ASSESSMENT_KEYS_V3 = {
|
|
"support_tier",
|
|
"challenge_tier",
|
|
"overall_state",
|
|
"rationale",
|
|
"supersedes_assessment_id",
|
|
}
|
|
TRUTH_APT_V3_CLAIM_TYPES = {"empirical", "predictive", "causal"}
|
|
ASSESSMENT_RELEVANT_POLARITIES = {"supports", "challenges"}
|
|
|
|
|
|
class CompilerError(RuntimeError):
|
|
"""Raised when an input cannot safely become a reviewable proposal."""
|
|
|
|
|
|
def sha256_bytes(value: bytes) -> str:
|
|
return hashlib.sha256(value).hexdigest()
|
|
|
|
|
|
def canonical_json_bytes(value: Any) -> bytes:
|
|
return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=True).encode("utf-8")
|
|
|
|
|
|
def canonical_sha256(value: Any) -> str:
|
|
return sha256_bytes(canonical_json_bytes(value))
|
|
|
|
|
|
def _load_duplicate_free_json(value: bytes, label: str) -> Any:
|
|
try:
|
|
text = value.decode("utf-8", errors="strict")
|
|
except UnicodeDecodeError as exc:
|
|
raise CompilerError(f"{label} must be valid UTF-8 JSON: {exc}") from exc
|
|
|
|
def reject_duplicate_pairs(pairs: list[tuple[str, Any]]) -> dict[str, Any]:
|
|
result: dict[str, Any] = {}
|
|
for key, item in pairs:
|
|
if key in result:
|
|
raise CompilerError(f"{label} contains duplicate JSON object key: {key!r}")
|
|
result[key] = item
|
|
return result
|
|
|
|
try:
|
|
return json.loads(text, object_pairs_hook=reject_duplicate_pairs)
|
|
except json.JSONDecodeError as exc:
|
|
raise CompilerError(f"{label} must be valid UTF-8 JSON: {exc}") from exc
|
|
|
|
|
|
def stable_uuid(label: str) -> str:
|
|
return str(uuid.uuid5(uuid.NAMESPACE_URL, f"livingip:kb-source-compiler:v1:{label}"))
|
|
|
|
|
|
def stable_uuid_v3(row_kind: str, packet_identity_sha256: str, semantic_key: Any) -> str:
|
|
material = canonical_json_bytes(
|
|
["livingip", "kb-source-compiler", 3, packet_identity_sha256, row_kind, semantic_key]
|
|
).decode("ascii")
|
|
return str(uuid.uuid5(uuid.NAMESPACE_URL, material))
|
|
|
|
|
|
def _require_object(value: Any, label: str) -> dict[str, Any]:
|
|
if not isinstance(value, dict):
|
|
raise CompilerError(f"{label} must be an object")
|
|
return value
|
|
|
|
|
|
def _require_exact_keys(value: dict[str, Any], allowed: set[str], label: str) -> None:
|
|
unknown = sorted(set(value) - allowed)
|
|
if unknown:
|
|
raise CompilerError(f"{label} contains unsupported keys: {', '.join(unknown)}")
|
|
|
|
|
|
def _require_exact_shape(value: dict[str, Any], required: set[str], label: str) -> None:
|
|
_require_exact_keys(value, required, label)
|
|
missing = sorted(required - set(value))
|
|
if missing:
|
|
raise CompilerError(f"{label} is missing required keys: {', '.join(missing)}")
|
|
|
|
|
|
def _require_nonempty_string(value: Any, label: str) -> str:
|
|
if not isinstance(value, str) or not value.strip():
|
|
raise CompilerError(f"{label} must be a non-empty string")
|
|
if value != value.strip():
|
|
raise CompilerError(f"{label} must not have leading or trailing whitespace")
|
|
return value
|
|
|
|
|
|
def _require_sha256(value: Any, label: str) -> str:
|
|
text = _require_nonempty_string(value, label)
|
|
if not SHA256_RE.fullmatch(text):
|
|
raise CompilerError(f"{label} must be a lowercase SHA-256 digest")
|
|
return text
|
|
|
|
|
|
def _require_key(value: Any, label: str) -> str:
|
|
text = _require_nonempty_string(value, label)
|
|
if not KEY_RE.fullmatch(text):
|
|
raise CompilerError(f"{label} must match {KEY_RE.pattern}")
|
|
return text
|
|
|
|
|
|
def _require_canonical_uuid(value: Any, label: str, *, nullable: bool = False) -> str | None:
|
|
if value is None and nullable:
|
|
return None
|
|
if not isinstance(value, str):
|
|
raise CompilerError(f"{label} must be a canonical lowercase hyphenated UUID")
|
|
try:
|
|
canonical = str(uuid.UUID(value))
|
|
except ValueError as exc:
|
|
raise CompilerError(f"{label} must be a canonical lowercase hyphenated UUID") from exc
|
|
if value != canonical:
|
|
raise CompilerError(f"{label} must be a canonical lowercase hyphenated UUID")
|
|
return canonical
|
|
|
|
|
|
def _require_canonical_utc_timestamp(value: Any, label: str) -> str:
|
|
text = _require_nonempty_string(value, label)
|
|
if not ISO_TIMESTAMP_RE.fullmatch(text):
|
|
raise CompilerError(f"{label} must be a timezone-aware ISO-8601 timestamp with at most 6 fractional digits")
|
|
try:
|
|
parsed = datetime.fromisoformat(text.replace("Z", "+00:00"))
|
|
except ValueError as exc:
|
|
raise CompilerError(f"{label} must be a valid ISO-8601 timestamp") from exc
|
|
if parsed.tzinfo is None:
|
|
raise CompilerError(f"{label} must include a timezone")
|
|
return parsed.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%fZ")
|
|
|
|
|
|
def _require_nullable_enum(value: Any, allowed: set[str], label: str) -> str | None:
|
|
if value is None:
|
|
return None
|
|
if not isinstance(value, str) or value not in allowed:
|
|
raise CompilerError(f"{label} must be null or one of {sorted(allowed)}")
|
|
return value
|
|
|
|
|
|
def _require_string_list(value: Any, label: str) -> list[str]:
|
|
if not isinstance(value, list) or any(
|
|
not isinstance(item, str) or not item.strip() or item != item.strip() for item in value
|
|
):
|
|
raise CompilerError(f"{label} must be a list of non-empty strings without surrounding whitespace")
|
|
if len(value) != len(set(value)):
|
|
raise CompilerError(f"{label} contains duplicates")
|
|
return sorted(value)
|
|
|
|
|
|
def _require_stable_reference(value: Any, label: str) -> str:
|
|
text = _require_nonempty_string(value, label)
|
|
if any(character.isspace() for character in text) or not URI_SCHEME_RE.match(text):
|
|
raise CompilerError(f"{label} must be a stable URI-like reference with a scheme")
|
|
parsed = urlsplit(text)
|
|
if not parsed.scheme or text.endswith(":"):
|
|
raise CompilerError(f"{label} must identify a specific source, not only a scheme or chat label")
|
|
scheme = parsed.scheme.lower()
|
|
if scheme in UNSAFE_URI_SCHEMES:
|
|
raise CompilerError(f"{label} uses an unsafe URI scheme: {scheme}")
|
|
if scheme in {"http", "https"}:
|
|
if not parsed.netloc:
|
|
raise CompilerError(f"{label} HTTP reference must include a host")
|
|
if parsed.username is not None or parsed.password is not None:
|
|
raise CompilerError(f"{label} must not embed credentials")
|
|
if scheme in {"telegram", "tg"} and not re.search(r"\d", f"{parsed.netloc}{parsed.path}"):
|
|
raise CompilerError(f"{label} Telegram reference must include a stable chat or message identifier")
|
|
return text
|
|
|
|
|
|
def _read_required_file(path: Path, label: str) -> bytes:
|
|
try:
|
|
if not path.is_file():
|
|
raise CompilerError(f"{label} path is not a regular file: {path}")
|
|
value = path.read_bytes()
|
|
except OSError as exc:
|
|
raise CompilerError(f"could not read {label}: {exc}") from exc
|
|
if not value:
|
|
raise CompilerError(f"{label} must not be empty")
|
|
return value
|
|
|
|
|
|
def _quote_binding(text: str, quote: Any, label: str) -> dict[str, Any]:
|
|
exact = _require_nonempty_string(quote, label)
|
|
start = text.find(exact)
|
|
if start < 0:
|
|
raise CompilerError(f"{label} is not an exact substring of the extracted text")
|
|
return {
|
|
"quote": exact,
|
|
"quote_sha256": sha256_bytes(exact.encode("utf-8")),
|
|
"first_start": start,
|
|
"first_end": start + len(exact),
|
|
"occurrence_count": text.count(exact),
|
|
}
|
|
|
|
|
|
def _require_nonnegative_integer(value: Any, label: str) -> int:
|
|
if isinstance(value, bool) or not isinstance(value, int) or value < 0:
|
|
raise CompilerError(f"{label} must be a non-negative integer")
|
|
return value
|
|
|
|
|
|
def _v3_quote_binding(text_bytes: bytes, quote: Any, locator_value: Any, label: str) -> dict[str, Any]:
|
|
exact = _require_nonempty_string(quote, f"{label}.quote")
|
|
locator = _require_object(locator_value, f"{label}.locator")
|
|
_require_exact_shape(locator, LOCATOR_KEYS_V3, f"{label}.locator")
|
|
if locator.get("unit") != "utf8_bytes":
|
|
raise CompilerError(f"{label}.locator.unit must equal 'utf8_bytes'")
|
|
start = _require_nonnegative_integer(locator.get("start"), f"{label}.locator.start")
|
|
end = _require_nonnegative_integer(locator.get("end"), f"{label}.locator.end")
|
|
quote_bytes = exact.encode("utf-8")
|
|
if end <= start or end > len(text_bytes) or text_bytes[start:end] != quote_bytes:
|
|
raise CompilerError(f"{label}.locator does not bind the exact quote bytes in the extracted text")
|
|
return {
|
|
"quote": exact,
|
|
"quote_sha256": sha256_bytes(quote_bytes),
|
|
"unit": "utf8_bytes",
|
|
"start": start,
|
|
"end": end,
|
|
}
|
|
|
|
|
|
def _validate_v3_assessment(value: Any, claim_type: str, evidence: list[dict[str, Any]], label: str) -> dict[str, Any]:
|
|
assessment = _require_object(value, label)
|
|
_require_exact_shape(assessment, ASSESSMENT_KEYS_V3, label)
|
|
support_tier = _require_nullable_enum(assessment.get("support_tier"), ap.V3_SUPPORT_TIERS, f"{label}.support_tier")
|
|
challenge_tier = _require_nullable_enum(
|
|
assessment.get("challenge_tier"), ap.V3_SUPPORT_TIERS, f"{label}.challenge_tier"
|
|
)
|
|
if claim_type in TRUTH_APT_V3_CLAIM_TYPES and (support_tier is None or challenge_tier is None):
|
|
raise CompilerError(f"{label} requires support_tier and challenge_tier for a truth-apt claim")
|
|
overall_state = assessment.get("overall_state")
|
|
if overall_state not in ap.V3_ASSESSMENT_STATES:
|
|
raise CompilerError(f"{label}.overall_state must be one of {sorted(ap.V3_ASSESSMENT_STATES)}")
|
|
|
|
polarities = {row["polarity"] for row in evidence}
|
|
if not polarities & ASSESSMENT_RELEVANT_POLARITIES:
|
|
raise CompilerError(f"{label} requires at least one supports or challenges evidence row")
|
|
if support_tier not in {None, "none"} and "supports" not in polarities:
|
|
raise CompilerError(f"{label}.support_tier requires supports evidence")
|
|
if challenge_tier not in {None, "none"} and "challenges" not in polarities:
|
|
raise CompilerError(f"{label}.challenge_tier requires challenges evidence")
|
|
if overall_state == "support_dominant" and "supports" not in polarities:
|
|
raise CompilerError(f"{label}.overall_state=support_dominant requires supports evidence")
|
|
if overall_state == "challenge_dominant" and "challenges" not in polarities:
|
|
raise CompilerError(f"{label}.overall_state=challenge_dominant requires challenges evidence")
|
|
if overall_state == "mixed" and not {"supports", "challenges"} <= polarities:
|
|
raise CompilerError(f"{label}.overall_state=mixed requires supports and challenges evidence")
|
|
if assessment.get("supersedes_assessment_id") is not None:
|
|
raise CompilerError(f"{label}.supersedes_assessment_id must be null for a first assessment")
|
|
return {
|
|
"support_tier": support_tier,
|
|
"challenge_tier": challenge_tier,
|
|
"overall_state": overall_state,
|
|
"rationale": _require_nonempty_string(assessment.get("rationale"), f"{label}.rationale"),
|
|
"supersedes_assessment_id": None,
|
|
}
|
|
|
|
|
|
def _validate_manifest_v3(
|
|
manifest: dict[str, Any],
|
|
artifact_sha256: str,
|
|
text_sha256: str,
|
|
extractor_spec_sha256: str,
|
|
text_bytes: bytes,
|
|
) -> tuple[dict[str, Any], list[dict[str, Any]]]:
|
|
_require_exact_shape(manifest, TOP_LEVEL_KEYS_V3, "manifest")
|
|
if manifest.get("schema") != MANIFEST_SCHEMA_V3:
|
|
raise CompilerError(f"manifest.schema must equal {MANIFEST_SCHEMA_V3!r}")
|
|
|
|
declared_artifact_sha256 = _require_sha256(manifest.get("artifact_sha256"), "manifest.artifact_sha256")
|
|
declared_text_sha256 = _require_sha256(manifest.get("extracted_text_sha256"), "manifest.extracted_text_sha256")
|
|
if declared_artifact_sha256 != artifact_sha256:
|
|
raise CompilerError("artifact SHA-256 does not match manifest.artifact_sha256")
|
|
if declared_text_sha256 != text_sha256:
|
|
raise CompilerError("extracted text SHA-256 does not match manifest.extracted_text_sha256")
|
|
|
|
extractor = _require_object(manifest.get("extractor"), "manifest.extractor")
|
|
_require_exact_shape(extractor, EXTRACTOR_KEYS_V3, "manifest.extractor")
|
|
declared_spec_sha256 = _require_sha256(extractor.get("spec_sha256"), "manifest.extractor.spec_sha256")
|
|
if declared_spec_sha256 != extractor_spec_sha256:
|
|
raise CompilerError("extractor specification SHA-256 does not match manifest.extractor.spec_sha256")
|
|
validated_extractor = {
|
|
"name": _require_nonempty_string(extractor.get("name"), "manifest.extractor.name"),
|
|
"version": _require_nonempty_string(extractor.get("version"), "manifest.extractor.version"),
|
|
"spec_sha256": extractor_spec_sha256,
|
|
}
|
|
agent_id = _require_canonical_uuid(manifest.get("agent_id"), "manifest.agent_id")
|
|
|
|
source = _require_object(manifest.get("source"), "manifest.source")
|
|
_require_exact_shape(source, SOURCE_KEYS_V3, "manifest.source")
|
|
source_type = source.get("source_type")
|
|
if source_type not in ap.SOURCE_TYPES:
|
|
raise CompilerError(f"manifest.source.source_type must be one of {sorted(ap.SOURCE_TYPES)}")
|
|
source_access_scope = source.get("access_scope")
|
|
if source_access_scope not in ap.V3_ACCESS_SCOPES:
|
|
raise CompilerError(f"manifest.source.access_scope must be one of {sorted(ap.V3_ACCESS_SCOPES)}")
|
|
ingestion_origin = source.get("ingestion_origin")
|
|
if ingestion_origin not in ap.V3_SOURCE_ORIGINS:
|
|
raise CompilerError(f"manifest.source.ingestion_origin must be one of {sorted(ap.V3_SOURCE_ORIGINS)}")
|
|
provenance_status = source.get("provenance_status")
|
|
if provenance_status not in ap.V3_PROVENANCE_STATES:
|
|
raise CompilerError(f"manifest.source.provenance_status must be one of {sorted(ap.V3_PROVENANCE_STATES)}")
|
|
source_class = source.get("source_class")
|
|
if source_class not in ap.V3_SOURCE_CLASSES:
|
|
raise CompilerError(f"manifest.source.source_class must be one of {sorted(ap.V3_SOURCE_CLASSES)}")
|
|
auto_promotion_policy = source.get("auto_promotion_policy")
|
|
if auto_promotion_policy not in ap.V3_AUTO_PROMOTION_POLICIES:
|
|
raise CompilerError(
|
|
f"manifest.source.auto_promotion_policy must be one of {sorted(ap.V3_AUTO_PROMOTION_POLICIES)}"
|
|
)
|
|
source_identity = _require_stable_reference(source.get("identity"), "manifest.source.identity")
|
|
source_locator = _require_stable_reference(source.get("locator"), "manifest.source.locator")
|
|
expected_source_identity = f"document:sha256:{artifact_sha256}"
|
|
expected_source_locator = f"artifact://sha256/{artifact_sha256}"
|
|
if source_identity != expected_source_identity:
|
|
raise CompilerError(
|
|
"manifest.source.identity must equal the content-addressed identity for the actual artifact bytes"
|
|
)
|
|
if source_locator != expected_source_locator:
|
|
raise CompilerError("manifest.source.locator must equal artifact://sha256/<actual artifact SHA-256>")
|
|
validated_source = {
|
|
"identity": source_identity,
|
|
"source_key": _require_key(source.get("source_key"), "manifest.source.source_key"),
|
|
"source_type": source_type,
|
|
"title": _require_nonempty_string(source.get("title"), "manifest.source.title"),
|
|
"locator": source_locator,
|
|
"access_scope": source_access_scope,
|
|
"ingestion_origin": ingestion_origin,
|
|
"provenance_status": provenance_status,
|
|
"source_class": source_class,
|
|
"auto_promotion_policy": auto_promotion_policy,
|
|
"captured_at": _require_canonical_utc_timestamp(source.get("captured_at"), "manifest.source.captured_at"),
|
|
}
|
|
|
|
raw_claims = manifest.get("claims")
|
|
if not isinstance(raw_claims, list) or not raw_claims:
|
|
raise CompilerError("manifest.claims must be a non-empty list of objects")
|
|
if len(raw_claims) > ap.MAX_BUNDLE_ROWS["claims"]:
|
|
raise CompilerError(f"manifest.claims exceeds {ap.MAX_BUNDLE_ROWS['claims']} rows")
|
|
|
|
validated_claims: list[dict[str, Any]] = []
|
|
quote_bindings: list[dict[str, Any]] = []
|
|
seen_claim_keys: set[str] = set()
|
|
evidence_count = 0
|
|
for claim_index, raw_claim in enumerate(raw_claims):
|
|
label = f"manifest.claims[{claim_index}]"
|
|
claim = _require_object(raw_claim, label)
|
|
_require_exact_shape(claim, CLAIM_KEYS_V3, label)
|
|
claim_key = _require_key(claim.get("claim_key"), f"{label}.claim_key")
|
|
if claim_key in seen_claim_keys:
|
|
raise CompilerError(f"duplicate claim_key: {claim_key}")
|
|
seen_claim_keys.add(claim_key)
|
|
claim_type = claim.get("type")
|
|
if claim_type not in ap.V3_CLAIM_TYPES:
|
|
raise CompilerError(f"{label}.type must be one of {sorted(ap.V3_CLAIM_TYPES)}")
|
|
access_scope = claim.get("access_scope")
|
|
if access_scope not in ap.V3_ACCESS_SCOPES:
|
|
raise CompilerError(f"{label}.access_scope must be one of {sorted(ap.V3_ACCESS_SCOPES)}")
|
|
owner_agent_id = _require_canonical_uuid(claim.get("owner_agent_id"), f"{label}.owner_agent_id", nullable=True)
|
|
if access_scope == "agent_private" and owner_agent_id is None:
|
|
raise CompilerError(f"{label}.owner_agent_id is required for agent_private claims")
|
|
status = claim.get("status")
|
|
if status not in {"active", "contested"}:
|
|
raise CompilerError(f"{label}.status must be active or contested")
|
|
if claim.get("revision_kind") != "original" or claim.get("supersedes_id") is not None:
|
|
raise CompilerError(f"{label} must describe an original claim with supersedes_id=null")
|
|
|
|
raw_evidence = claim.get("evidence")
|
|
if not isinstance(raw_evidence, list) or not raw_evidence:
|
|
raise CompilerError(f"{label}.evidence must be a non-empty list of exact locator-bound rows")
|
|
evidence_count += len(raw_evidence)
|
|
if evidence_count > ap.MAX_BUNDLE_ROWS["evidence"]:
|
|
raise CompilerError(f"manifest evidence exceeds {ap.MAX_BUNDLE_ROWS['evidence']} rows")
|
|
validated_evidence: list[dict[str, Any]] = []
|
|
seen_evidence_keys: set[str] = set()
|
|
seen_polarities: set[str] = set()
|
|
for evidence_index, raw_row in enumerate(raw_evidence):
|
|
evidence_label = f"{label}.evidence[{evidence_index}]"
|
|
row = _require_object(raw_row, evidence_label)
|
|
_require_exact_shape(row, EVIDENCE_KEYS_V3, evidence_label)
|
|
evidence_key = _require_key(row.get("evidence_key"), f"{evidence_label}.evidence_key")
|
|
if evidence_key in seen_evidence_keys:
|
|
raise CompilerError(f"{evidence_label}.evidence_key duplicates an earlier evidence row")
|
|
seen_evidence_keys.add(evidence_key)
|
|
polarity = row.get("polarity")
|
|
if polarity not in ap.V3_EVIDENCE_POLARITIES:
|
|
raise CompilerError(f"{evidence_label}.polarity must be one of {sorted(ap.V3_EVIDENCE_POLARITIES)}")
|
|
if polarity in seen_polarities:
|
|
raise CompilerError(f"{evidence_label}.polarity duplicates a claim-local source/polarity key")
|
|
seen_polarities.add(polarity)
|
|
binding = _v3_quote_binding(text_bytes, row.get("quote"), row.get("locator"), evidence_label)
|
|
validated_evidence.append(
|
|
{
|
|
"evidence_key": evidence_key,
|
|
"quote": binding["quote"],
|
|
"polarity": polarity,
|
|
"locator": {
|
|
"unit": binding["unit"],
|
|
"start": binding["start"],
|
|
"end": binding["end"],
|
|
},
|
|
"rationale": _require_nonempty_string(row.get("rationale"), f"{evidence_label}.rationale"),
|
|
}
|
|
)
|
|
quote_bindings.append(
|
|
{
|
|
"claim_key": claim_key,
|
|
"evidence_key": evidence_key,
|
|
"kind": "evidence",
|
|
**binding,
|
|
}
|
|
)
|
|
validated_evidence.sort(key=lambda row: row["evidence_key"])
|
|
|
|
validated_claims.append(
|
|
{
|
|
"claim_key": claim_key,
|
|
"type": claim_type,
|
|
"proposition": _require_nonempty_string(claim.get("proposition"), f"{label}.proposition"),
|
|
"body_markdown": _require_nonempty_string(claim.get("body_markdown"), f"{label}.body_markdown"),
|
|
"scope": _require_nonempty_string(claim.get("scope"), f"{label}.scope"),
|
|
"access_scope": access_scope,
|
|
"owner_agent_id": owner_agent_id,
|
|
"status": status,
|
|
"revision_criteria": _require_nonempty_string(
|
|
claim.get("revision_criteria"), f"{label}.revision_criteria"
|
|
),
|
|
"revision_kind": "original",
|
|
"supersedes_id": None,
|
|
"tags": _require_string_list(claim.get("tags"), f"{label}.tags"),
|
|
"evidence": validated_evidence,
|
|
"assessment": _validate_v3_assessment(
|
|
claim.get("assessment"), claim_type, validated_evidence, f"{label}.assessment"
|
|
),
|
|
}
|
|
)
|
|
|
|
validated_claims.sort(key=lambda row: row["claim_key"])
|
|
quote_bindings.sort(key=lambda row: (row["claim_key"], row["evidence_key"]))
|
|
return (
|
|
{
|
|
"schema": MANIFEST_SCHEMA_V3,
|
|
"artifact_sha256": artifact_sha256,
|
|
"extracted_text_sha256": text_sha256,
|
|
"extractor": validated_extractor,
|
|
"agent_id": agent_id,
|
|
"source": validated_source,
|
|
"claims": validated_claims,
|
|
},
|
|
quote_bindings,
|
|
)
|
|
|
|
|
|
def _validate_dedupe(value: Any, extracted_text: str) -> dict[str, Any]:
|
|
dedupe = _require_object(value, "manifest.dedupe")
|
|
_require_exact_keys(dedupe, DEDUPE_KEYS, "manifest.dedupe")
|
|
query = _require_nonempty_string(dedupe.get("database_search_query"), "manifest.dedupe.database_search_query")
|
|
|
|
raw_candidates = dedupe.get("candidate_claims")
|
|
if not isinstance(raw_candidates, list) or len(raw_candidates) > 20:
|
|
raise CompilerError("manifest.dedupe.candidate_claims must be a list with at most 20 rows")
|
|
candidates: list[dict[str, Any]] = []
|
|
candidate_ids: set[str] = set()
|
|
for index, raw_candidate in enumerate(raw_candidates):
|
|
label = f"manifest.dedupe.candidate_claims[{index}]"
|
|
candidate = _require_object(raw_candidate, label)
|
|
_require_exact_keys(candidate, DEDUPE_CANDIDATE_KEYS, label)
|
|
try:
|
|
claim_id = str(uuid.UUID(_require_nonempty_string(candidate.get("id"), f"{label}.id")))
|
|
except ValueError as exc:
|
|
raise CompilerError(f"{label}.id must be a full UUID") from exc
|
|
if claim_id in candidate_ids:
|
|
raise CompilerError(f"{label}.id duplicates an earlier canonical candidate")
|
|
candidate_ids.add(claim_id)
|
|
candidates.append(
|
|
{
|
|
"id": claim_id,
|
|
"text": _require_nonempty_string(candidate.get("text"), f"{label}.text"),
|
|
"score": _require_nonnegative_integer(candidate.get("score"), f"{label}.score"),
|
|
"evidence_count": _require_nonnegative_integer(
|
|
candidate.get("evidence_count"), f"{label}.evidence_count"
|
|
),
|
|
"edge_count": _require_nonnegative_integer(candidate.get("edge_count"), f"{label}.edge_count"),
|
|
}
|
|
)
|
|
|
|
raw_duplicates = dedupe.get("duplicates")
|
|
if not isinstance(raw_duplicates, list) or len(raw_duplicates) > len(candidates):
|
|
raise CompilerError("manifest.dedupe.duplicates must not exceed the retrieved candidate count")
|
|
duplicates: list[dict[str, Any]] = []
|
|
duplicate_ids: set[str] = set()
|
|
for index, raw_duplicate in enumerate(raw_duplicates):
|
|
label = f"manifest.dedupe.duplicates[{index}]"
|
|
duplicate = _require_object(raw_duplicate, label)
|
|
_require_exact_keys(duplicate, DEDUPE_DUPLICATE_KEYS, label)
|
|
try:
|
|
claim_id = str(uuid.UUID(_require_nonempty_string(duplicate.get("claim_id"), f"{label}.claim_id")))
|
|
except ValueError as exc:
|
|
raise CompilerError(f"{label}.claim_id must be a full UUID") from exc
|
|
if claim_id not in candidate_ids:
|
|
raise CompilerError(f"{label}.claim_id was not present in the canonical retrieval candidates")
|
|
if claim_id in duplicate_ids:
|
|
raise CompilerError(f"{label}.claim_id duplicates an earlier duplicate judgment")
|
|
duplicate_ids.add(claim_id)
|
|
source_quote = _quote_binding(extracted_text, duplicate.get("source_quote"), f"{label}.source_quote")
|
|
duplicates.append(
|
|
{
|
|
"claim_id": claim_id,
|
|
"source_quote": source_quote["quote"],
|
|
"reason": _require_nonempty_string(duplicate.get("reason"), f"{label}.reason"),
|
|
}
|
|
)
|
|
|
|
return {
|
|
"database_search_query": query,
|
|
"candidate_claims": candidates,
|
|
"duplicates": duplicates,
|
|
}
|
|
|
|
|
|
def _validate_manifest(
|
|
manifest: dict[str, Any], artifact_sha256: str, text_sha256: str, extracted_text: str
|
|
) -> tuple[dict[str, Any], list[dict[str, Any]]]:
|
|
manifest_schema = manifest.get("schema")
|
|
if manifest_schema not in MANIFEST_SCHEMAS:
|
|
raise CompilerError(f"manifest.schema must be one of {sorted(MANIFEST_SCHEMAS)}")
|
|
_require_exact_keys(
|
|
manifest,
|
|
TOP_LEVEL_KEYS_V2 if manifest_schema == MANIFEST_SCHEMA_V2 else TOP_LEVEL_KEYS,
|
|
"manifest",
|
|
)
|
|
|
|
declared_artifact_sha256 = _require_sha256(manifest.get("artifact_sha256"), "manifest.artifact_sha256")
|
|
declared_text_sha256 = _require_sha256(manifest.get("extracted_text_sha256"), "manifest.extracted_text_sha256")
|
|
if declared_artifact_sha256 != artifact_sha256:
|
|
raise CompilerError("artifact SHA-256 does not match manifest.artifact_sha256")
|
|
if declared_text_sha256 != text_sha256:
|
|
raise CompilerError("extracted text SHA-256 does not match manifest.extracted_text_sha256")
|
|
|
|
extractor = _require_object(manifest.get("extractor"), "manifest.extractor")
|
|
_require_exact_keys(extractor, EXTRACTOR_KEYS, "manifest.extractor")
|
|
extractor_name = _require_nonempty_string(extractor.get("name"), "manifest.extractor.name")
|
|
extractor_version = _require_nonempty_string(extractor.get("version"), "manifest.extractor.version")
|
|
|
|
source = _require_object(manifest.get("source"), "manifest.source")
|
|
_require_exact_keys(source, SOURCE_KEYS, "manifest.source")
|
|
source_identity = _require_stable_reference(source.get("identity"), "manifest.source.identity")
|
|
source_locator = _require_stable_reference(source.get("locator"), "manifest.source.locator")
|
|
source_key = _require_key(source.get("source_key"), "manifest.source.source_key")
|
|
source_title = _require_nonempty_string(source.get("title"), "manifest.source.title")
|
|
source_type = _require_nonempty_string(source.get("source_type"), "manifest.source.source_type")
|
|
if source_type not in ap.SOURCE_TYPES:
|
|
raise CompilerError(f"manifest.source.source_type must be one of {sorted(ap.SOURCE_TYPES)}")
|
|
|
|
claims = manifest.get("claims")
|
|
if not isinstance(claims, list) or not claims:
|
|
raise CompilerError("manifest.claims must be a non-empty list of objects")
|
|
if len(claims) > ap.MAX_BUNDLE_ROWS["claims"]:
|
|
raise CompilerError(f"manifest.claims exceeds {ap.MAX_BUNDLE_ROWS['claims']} rows")
|
|
|
|
validated_claims: list[dict[str, Any]] = []
|
|
quote_bindings: list[dict[str, Any]] = []
|
|
seen_claim_keys: set[str] = set()
|
|
derived_source_keys = {source_key}
|
|
for claim_index, raw_claim in enumerate(claims):
|
|
label = f"manifest.claims[{claim_index}]"
|
|
claim = _require_object(raw_claim, label)
|
|
_require_exact_keys(claim, CLAIM_KEYS, label)
|
|
claim_key = _require_key(claim.get("claim_key"), f"{label}.claim_key")
|
|
if claim_key in seen_claim_keys:
|
|
raise CompilerError(f"duplicate claim_key: {claim_key}")
|
|
seen_claim_keys.add(claim_key)
|
|
body_source_key = f"{claim_key}__proposal_body"
|
|
if body_source_key in derived_source_keys:
|
|
raise CompilerError(f"duplicate source_key after normalization: {body_source_key}")
|
|
derived_source_keys.add(body_source_key)
|
|
|
|
claim_type = _require_nonempty_string(claim.get("type"), f"{label}.type")
|
|
if claim_type not in ap.CLAIM_TYPES:
|
|
raise CompilerError(f"{label}.type must be one of {sorted(ap.CLAIM_TYPES)}")
|
|
claim_text = _require_nonempty_string(claim.get("text"), f"{label}.text")
|
|
claim_quote = _quote_binding(extracted_text, claim.get("quote"), f"{label}.quote")
|
|
quote_bindings.append({"claim_key": claim_key, "kind": "claim", **claim_quote})
|
|
|
|
confidence = claim.get("confidence")
|
|
if confidence is not None and (
|
|
isinstance(confidence, bool) or not isinstance(confidence, (int, float)) or not 0 <= confidence <= 1
|
|
):
|
|
raise CompilerError(f"{label}.confidence must be a number from 0 to 1 or null")
|
|
tags = claim.get("tags", [])
|
|
if not isinstance(tags, list) or any(
|
|
not isinstance(tag, str) or not tag.strip() or tag != tag.strip() for tag in tags
|
|
):
|
|
raise CompilerError(f"{label}.tags must be a list of non-empty strings")
|
|
if len(tags) != len(set(tags)):
|
|
raise CompilerError(f"{label}.tags contains duplicates")
|
|
|
|
evidence = claim.get("evidence")
|
|
if not isinstance(evidence, list) or not evidence:
|
|
raise CompilerError(f"{label}.evidence must be a non-empty list of exact-quote objects")
|
|
validated_evidence: list[dict[str, Any]] = []
|
|
seen_evidence: set[tuple[str, str]] = set()
|
|
for evidence_index, raw_evidence in enumerate(evidence):
|
|
evidence_label = f"{label}.evidence[{evidence_index}]"
|
|
evidence_row = _require_object(raw_evidence, evidence_label)
|
|
_require_exact_keys(evidence_row, EVIDENCE_KEYS, evidence_label)
|
|
role = _require_nonempty_string(evidence_row.get("role"), f"{evidence_label}.role")
|
|
if role not in ap.EVIDENCE_ROLES:
|
|
raise CompilerError(f"{evidence_label}.role must be one of {sorted(ap.EVIDENCE_ROLES)}")
|
|
binding = _quote_binding(extracted_text, evidence_row.get("quote"), f"{evidence_label}.quote")
|
|
evidence_key = (binding["quote"], role)
|
|
if evidence_key in seen_evidence:
|
|
raise CompilerError(f"{evidence_label} duplicates an earlier evidence quote and role")
|
|
seen_evidence.add(evidence_key)
|
|
quote_bindings.append(
|
|
{
|
|
"claim_key": claim_key,
|
|
"kind": "evidence",
|
|
"evidence_role": role,
|
|
**binding,
|
|
}
|
|
)
|
|
validated_evidence.append({"quote": binding["quote"], "role": role})
|
|
|
|
validated_claims.append(
|
|
{
|
|
"claim_key": claim_key,
|
|
"type": claim_type,
|
|
"text": claim_text,
|
|
"quote": claim_quote["quote"],
|
|
"confidence": confidence,
|
|
"tags": tags,
|
|
"evidence": validated_evidence,
|
|
}
|
|
)
|
|
|
|
validated_manifest = {
|
|
"schema": manifest_schema,
|
|
"artifact_sha256": artifact_sha256,
|
|
"extracted_text_sha256": text_sha256,
|
|
"extractor": {"name": extractor_name, "version": extractor_version},
|
|
"source": {
|
|
"identity": source_identity,
|
|
"source_key": source_key,
|
|
"source_type": source_type,
|
|
"title": source_title,
|
|
"locator": source_locator,
|
|
},
|
|
"claims": validated_claims,
|
|
}
|
|
if manifest_schema == MANIFEST_SCHEMA_V2:
|
|
validated_manifest["dedupe"] = _validate_dedupe(manifest.get("dedupe"), extracted_text)
|
|
return validated_manifest, quote_bindings
|
|
|
|
|
|
def _locator_fields(locator: str) -> tuple[str | None, str | None]:
|
|
scheme = urlsplit(locator).scheme.lower()
|
|
if scheme in {"http", "https"}:
|
|
return locator, None
|
|
return None, locator
|
|
|
|
|
|
def build_parent_proposal(
|
|
manifest: dict[str, Any], manifest_sha256: str, quote_bindings: list[dict[str, Any]]
|
|
) -> dict[str, Any]:
|
|
source = manifest["source"]
|
|
parent_seed = ":".join((manifest["artifact_sha256"], manifest["extracted_text_sha256"], manifest_sha256))
|
|
parent_id = stable_uuid(f"parent:{parent_seed}")
|
|
url, storage_path = _locator_fields(source["locator"])
|
|
|
|
excerpts: list[dict[str, Any]] = []
|
|
seen_quotes: set[str] = set()
|
|
for binding in quote_bindings:
|
|
quote = binding["quote"]
|
|
if quote not in seen_quotes:
|
|
seen_quotes.add(quote)
|
|
excerpts.append({"text": quote})
|
|
|
|
claim_candidates = []
|
|
for claim in manifest["claims"]:
|
|
claim_candidates.append(
|
|
{
|
|
"claim_key": claim["claim_key"],
|
|
"type": claim["type"],
|
|
"headline": claim["text"],
|
|
"text": claim["text"],
|
|
"body": claim["quote"],
|
|
"confidence": claim["confidence"],
|
|
"tags": claim["tags"],
|
|
"evidence_tier": "hash_bound_exact_quote",
|
|
"needs_research": False,
|
|
"evidence": [{"source_key": source["source_key"], "role": item["role"]} for item in claim["evidence"]],
|
|
"edges": [],
|
|
}
|
|
)
|
|
|
|
provenance_summary = (
|
|
f"identity={source['identity']}; locator={source['locator']}; "
|
|
f"artifact_sha256={manifest['artifact_sha256']}; "
|
|
f"extracted_text_sha256={manifest['extracted_text_sha256']}; "
|
|
f"extractor={manifest['extractor']['name']}@{manifest['extractor']['version']}"
|
|
)
|
|
dedupe = manifest.get("dedupe") or {}
|
|
return {
|
|
"id": parent_id,
|
|
"proposal_type": "approve_claim",
|
|
"status": "pending_review",
|
|
"proposed_by_handle": "leo",
|
|
"proposed_by_agent_id": None,
|
|
"channel": "source_document_compiler",
|
|
"source_ref": (
|
|
f"source-compiler:{manifest['artifact_sha256'][:16]}:"
|
|
f"{manifest['extracted_text_sha256'][:16]}:{manifest_sha256[:16]}"
|
|
),
|
|
"rationale": "Compile one hash-bound source artifact into exact-quote claim candidates for review.",
|
|
"payload": {
|
|
"source_document": {
|
|
"manifest_schema": manifest["schema"],
|
|
"manifest_sha256": manifest_sha256,
|
|
"artifact_sha256": manifest["artifact_sha256"],
|
|
"extracted_text_sha256": manifest["extracted_text_sha256"],
|
|
"source_identity": source["identity"],
|
|
"source_locator": source["locator"],
|
|
"extractor": manifest["extractor"],
|
|
},
|
|
"claim_candidates": claim_candidates,
|
|
"source_candidates": [
|
|
{
|
|
"source_key": source["source_key"],
|
|
"source_type": source["source_type"],
|
|
"title": source["title"],
|
|
"url": url,
|
|
"storage_path": storage_path,
|
|
"source_quality": provenance_summary,
|
|
"usage_limits": "Only the exact bound quotes may ground claims; human review is required.",
|
|
"key_excerpts": excerpts,
|
|
"content_sha256": manifest["artifact_sha256"],
|
|
}
|
|
],
|
|
"dedupe_conflict_assessment": {
|
|
"database_search_query": dedupe.get("database_search_query")
|
|
or " ".join(claim["text"] for claim in manifest["claims"]),
|
|
"canonical_candidates": dedupe.get("candidate_claims", []),
|
|
"potential_existing_claim_ids": [item["claim_id"] for item in dedupe.get("duplicates", [])],
|
|
"conflicts": dedupe.get("duplicates", []),
|
|
"review_required": True,
|
|
},
|
|
},
|
|
}
|
|
|
|
|
|
def _v3_identity_material(manifest: dict[str, Any], manifest_file_sha256: str) -> tuple[dict[str, Any], str, str]:
|
|
candidate_content = {
|
|
"agent_id": manifest["agent_id"],
|
|
"source": manifest["source"],
|
|
"claims": manifest["claims"],
|
|
}
|
|
candidate_content_sha256 = canonical_sha256(candidate_content)
|
|
identity_material = {
|
|
"artifact_sha256": manifest["artifact_sha256"],
|
|
"extracted_text_sha256": manifest["extracted_text_sha256"],
|
|
"extractor": manifest["extractor"],
|
|
"manifest_file_sha256": manifest_file_sha256,
|
|
"source_identity": manifest["source"]["identity"],
|
|
"source_locator": manifest["source"]["locator"],
|
|
"normalized_candidate_content_sha256": candidate_content_sha256,
|
|
}
|
|
return identity_material, candidate_content_sha256, canonical_sha256(identity_material)
|
|
|
|
|
|
def _build_v3_parent_proposal(
|
|
manifest: dict[str, Any],
|
|
manifest_canonical_sha256: str,
|
|
identity_material: dict[str, Any],
|
|
candidate_content_sha256: str,
|
|
packet_identity_sha256: str,
|
|
) -> dict[str, Any]:
|
|
source = manifest["source"]
|
|
parent_id = stable_uuid_v3("source_proposal", packet_identity_sha256, candidate_content_sha256)
|
|
return {
|
|
"id": parent_id,
|
|
"proposal_type": "approve_claim",
|
|
"status": "pending_review",
|
|
"proposed_by_handle": "leo",
|
|
"proposed_by_agent_id": manifest["agent_id"],
|
|
"channel": "source_document_compiler_v3",
|
|
"source_ref": f"source-compiler-v3:{packet_identity_sha256[:32]}",
|
|
"rationale": "Compile one retained semantic extraction into a hash-bound V3 claim proposal for review.",
|
|
"payload": {
|
|
"source_document": {
|
|
"manifest_schema": MANIFEST_SCHEMA_V3,
|
|
"manifest_file_sha256": identity_material["manifest_file_sha256"],
|
|
"manifest_canonical_sha256": manifest_canonical_sha256,
|
|
"artifact_sha256": manifest["artifact_sha256"],
|
|
"extracted_text_sha256": manifest["extracted_text_sha256"],
|
|
"source_identity": source["identity"],
|
|
"source_locator": source["locator"],
|
|
"extractor": manifest["extractor"],
|
|
},
|
|
"identity_material": identity_material,
|
|
"normalized_candidate_content_sha256": candidate_content_sha256,
|
|
"packet_identity_sha256": packet_identity_sha256,
|
|
"semantic_extraction_is_retained_input": True,
|
|
"model_training_claim": False,
|
|
},
|
|
}
|
|
|
|
|
|
def _v3_source_excerpt(manifest: dict[str, Any]) -> str:
|
|
excerpts: list[str] = []
|
|
seen: set[str] = set()
|
|
for claim in manifest["claims"]:
|
|
for evidence in claim["evidence"]:
|
|
quote = evidence["quote"]
|
|
if quote not in seen:
|
|
seen.add(quote)
|
|
excerpts.append(quote)
|
|
return "\n\n".join(excerpts)
|
|
|
|
|
|
def _v3_locator_json(manifest: dict[str, Any], binding: dict[str, Any]) -> dict[str, Any]:
|
|
return {
|
|
"schema": "livingip.extractedTextQuoteLocator.v1",
|
|
"source_identity": manifest["source"]["identity"],
|
|
"source_locator": manifest["source"]["locator"],
|
|
"artifact_sha256": manifest["artifact_sha256"],
|
|
"extracted_text_sha256": manifest["extracted_text_sha256"],
|
|
"extractor": manifest["extractor"],
|
|
"unit": binding["unit"],
|
|
"start": binding["start"],
|
|
"end": binding["end"],
|
|
"quote_sha256": binding["quote_sha256"],
|
|
}
|
|
|
|
|
|
def _ensure_distinct_v3_ids(values: list[str]) -> None:
|
|
if len(values) != len(set(values)):
|
|
raise CompilerError("compiled V3 packet contains duplicate deterministic ids")
|
|
|
|
|
|
def _build_v3_strict_child(
|
|
manifest: dict[str, Any],
|
|
manifest_canonical_sha256: str,
|
|
quote_bindings: list[dict[str, Any]],
|
|
parent: dict[str, Any],
|
|
identity_material: dict[str, Any],
|
|
candidate_content_sha256: str,
|
|
packet_identity_sha256: str,
|
|
) -> dict[str, Any]:
|
|
source = manifest["source"]
|
|
source_id = stable_uuid_v3("source", packet_identity_sha256, source["source_key"])
|
|
claim_ids = {
|
|
claim["claim_key"]: stable_uuid_v3("claim", packet_identity_sha256, claim["claim_key"])
|
|
for claim in manifest["claims"]
|
|
}
|
|
binding_by_key = {(binding["claim_key"], binding["evidence_key"]): binding for binding in quote_bindings}
|
|
canonical_url, storage_uri = _locator_fields(source["locator"])
|
|
source_row = {
|
|
"id": source_id,
|
|
"source_type": source["source_type"],
|
|
"canonical_url": canonical_url,
|
|
"storage_uri": storage_uri,
|
|
"excerpt": _v3_source_excerpt(manifest),
|
|
"content_hash": manifest["artifact_sha256"],
|
|
"access_scope": source["access_scope"],
|
|
"ingestion_origin": source["ingestion_origin"],
|
|
"provenance_status": source["provenance_status"],
|
|
"source_class": source["source_class"],
|
|
"auto_promotion_policy": source["auto_promotion_policy"],
|
|
"captured_at": source["captured_at"],
|
|
"created_by": manifest["agent_id"],
|
|
}
|
|
|
|
claim_rows: list[dict[str, Any]] = []
|
|
evidence_rows: list[dict[str, Any]] = []
|
|
assessment_rows: list[dict[str, Any]] = []
|
|
for claim in manifest["claims"]:
|
|
claim_id = claim_ids[claim["claim_key"]]
|
|
claim_rows.append(
|
|
{
|
|
"id": claim_id,
|
|
"owner_agent_id": claim["owner_agent_id"],
|
|
"type": claim["type"],
|
|
"proposition": claim["proposition"],
|
|
"body_markdown": claim["body_markdown"],
|
|
"scope": claim["scope"],
|
|
"access_scope": claim["access_scope"],
|
|
"status": claim["status"],
|
|
"revision_criteria": claim["revision_criteria"],
|
|
"revision_kind": "original",
|
|
"supersedes_id": None,
|
|
"tags": claim["tags"],
|
|
"created_by": manifest["agent_id"],
|
|
}
|
|
)
|
|
for evidence in claim["evidence"]:
|
|
binding = binding_by_key[(claim["claim_key"], evidence["evidence_key"])]
|
|
evidence_rows.append(
|
|
{
|
|
"id": stable_uuid_v3(
|
|
"evidence",
|
|
packet_identity_sha256,
|
|
[claim["claim_key"], evidence["evidence_key"]],
|
|
),
|
|
"claim_id": claim_id,
|
|
"source_id": source_id,
|
|
"polarity": evidence["polarity"],
|
|
"excerpt": evidence["quote"],
|
|
"locator_json": _v3_locator_json(manifest, binding),
|
|
"source_content_hash": manifest["artifact_sha256"],
|
|
"rationale": evidence["rationale"],
|
|
"created_by": manifest["agent_id"],
|
|
}
|
|
)
|
|
|
|
for claim in manifest["claims"]:
|
|
claim_id = claim_ids[claim["claim_key"]]
|
|
assessment = claim["assessment"]
|
|
evidence_set_hash = ap._v3_evidence_set_hash(claim_id, evidence_rows)
|
|
assessment_rows.append(
|
|
{
|
|
"id": stable_uuid_v3("assessment", packet_identity_sha256, claim["claim_key"]),
|
|
"claim_id": claim_id,
|
|
"support_tier": assessment["support_tier"],
|
|
"challenge_tier": assessment["challenge_tier"],
|
|
"overall_state": assessment["overall_state"],
|
|
"rationale": assessment["rationale"],
|
|
"evidence_set_hash": evidence_set_hash,
|
|
"supersedes_assessment_id": None,
|
|
}
|
|
)
|
|
|
|
parent_packet_sha256 = canonical_sha256(parent)
|
|
apply_payload = {
|
|
"contract_version": 3,
|
|
"source_proposal_id": parent["id"],
|
|
"agent_id": manifest["agent_id"],
|
|
"claims": claim_rows,
|
|
"sources": [source_row],
|
|
"evidence": evidence_rows,
|
|
"edges": [],
|
|
"assessments": assessment_rows,
|
|
"reasoning_tools": [],
|
|
"normalization_manifest": {
|
|
"schema": "livingip.kbSourceProposalNormalization.v3",
|
|
"semantic_extraction_statement": (
|
|
"The semantic extraction is a retained input; this is a reproducible learning input, "
|
|
"not a model-training claim."
|
|
),
|
|
"semantic_extraction_is_retained_input": True,
|
|
"model_training_claim": False,
|
|
"manifest_schema": MANIFEST_SCHEMA_V3,
|
|
"manifest_file_sha256": identity_material["manifest_file_sha256"],
|
|
"manifest_identity_binding": "sha256(exact duplicate-free UTF-8 JSON file bytes)",
|
|
"manifest_canonical_sha256": manifest_canonical_sha256,
|
|
"artifact_sha256": manifest["artifact_sha256"],
|
|
"extracted_text_sha256": manifest["extracted_text_sha256"],
|
|
"extractor": manifest["extractor"],
|
|
"source_identity": source["identity"],
|
|
"source_locator": source["locator"],
|
|
"normalized_candidate_content_sha256": candidate_content_sha256,
|
|
"identity_material": identity_material,
|
|
"packet_identity_sha256": packet_identity_sha256,
|
|
"parent_packet_sha256": parent_packet_sha256,
|
|
"quote_bindings": quote_bindings,
|
|
},
|
|
}
|
|
payload = {"apply_payload": apply_payload}
|
|
payload_sha256 = canonical_sha256(payload)
|
|
child_id = stable_uuid_v3("pending_review_proposal", packet_identity_sha256, payload_sha256)
|
|
_ensure_distinct_v3_ids(
|
|
[
|
|
parent["id"],
|
|
child_id,
|
|
source_id,
|
|
*[row["id"] for row in claim_rows],
|
|
*[row["id"] for row in evidence_rows],
|
|
*[row["id"] for row in assessment_rows],
|
|
]
|
|
)
|
|
return {
|
|
"id": child_id,
|
|
"proposal_type": "approve_claim",
|
|
"status": "pending_review",
|
|
"proposed_by_handle": parent["proposed_by_handle"],
|
|
"proposed_by_agent_id": manifest["agent_id"],
|
|
"channel": parent["channel"],
|
|
"source_ref": f"source-compiler-v3:{packet_identity_sha256[:32]}:{payload_sha256[:16]}",
|
|
"rationale": f"Strict V3 source proposal compiled from retained packet {parent['id']}.",
|
|
"payload": payload,
|
|
"parent_packet_sha256": parent_packet_sha256,
|
|
"payload_sha256": payload_sha256,
|
|
}
|
|
|
|
|
|
def _validate_v3_compiled_child(
|
|
child: dict[str, Any], manifest: dict[str, Any], quote_bindings: list[dict[str, Any]]
|
|
) -> None:
|
|
if child.get("status") != "pending_review" or child.get("proposal_type") != "approve_claim":
|
|
raise CompilerError("V3 compilation did not produce one pending_review approve_claim proposal")
|
|
payload = child.get("payload")
|
|
apply_payload = payload.get("apply_payload") if isinstance(payload, dict) else None
|
|
if not isinstance(apply_payload, dict) or apply_payload.get("contract_version") != 3:
|
|
raise CompilerError("strict child has no schema-valid approve_claim V3 apply_payload")
|
|
if apply_payload.get("reasoning_tools") != [] or apply_payload.get("edges") != []:
|
|
raise CompilerError("V3 source compilation admits neither reasoning_tools nor inferred edges")
|
|
if child.get("payload_sha256") != canonical_sha256(payload):
|
|
raise CompilerError("strict child payload_sha256 does not bind the exact V3 payload")
|
|
|
|
claims = apply_payload.get("claims") or []
|
|
sources = apply_payload.get("sources") or []
|
|
evidence = apply_payload.get("evidence") or []
|
|
assessments = apply_payload.get("assessments") or []
|
|
if len(claims) != len(manifest["claims"]) or len(sources) != 1:
|
|
raise CompilerError("V3 strict child row counts differ from the extraction manifest")
|
|
if len(evidence) != sum(len(claim["evidence"]) for claim in manifest["claims"]):
|
|
raise CompilerError("V3 strict child evidence count differs from the extraction manifest")
|
|
if len(assessments) != len(claims):
|
|
raise CompilerError("V3 strict child requires exactly one assessment per claim")
|
|
_ensure_distinct_v3_ids(
|
|
[
|
|
child["id"],
|
|
*[row.get("id") for row in sources],
|
|
*[row.get("id") for row in claims],
|
|
*[row.get("id") for row in evidence],
|
|
*[row.get("id") for row in assessments],
|
|
]
|
|
)
|
|
|
|
source_row = sources[0]
|
|
if source_row.get("content_hash") != manifest["artifact_sha256"]:
|
|
raise CompilerError("V3 source content_hash does not bind the original artifact")
|
|
expected_url, expected_storage_uri = _locator_fields(manifest["source"]["locator"])
|
|
if source_row.get("canonical_url") != expected_url or source_row.get("storage_uri") != expected_storage_uri:
|
|
raise CompilerError("V3 source row does not retain the validated content-addressed locator")
|
|
if source_row.get("captured_at") != _require_canonical_utc_timestamp(
|
|
source_row.get("captured_at"), "V3 source captured_at"
|
|
):
|
|
raise CompilerError("V3 source captured_at is not in writer-canonical fixed-microsecond UTC Z form")
|
|
binding_by_locator = {canonical_sha256(_v3_locator_json(manifest, binding)): binding for binding in quote_bindings}
|
|
for row in evidence:
|
|
binding = binding_by_locator.get(canonical_sha256(row.get("locator_json")))
|
|
if binding is None or row.get("excerpt") != binding["quote"]:
|
|
raise CompilerError("V3 evidence does not retain an exact locator-bound quote")
|
|
if row.get("source_content_hash") != manifest["artifact_sha256"]:
|
|
raise CompilerError("V3 evidence source_content_hash does not bind the original artifact")
|
|
assessment_by_claim = {row.get("claim_id"): row for row in assessments}
|
|
if len(assessment_by_claim) != len(claims):
|
|
raise CompilerError("V3 strict child contains duplicate assessment claim ids")
|
|
for claim in claims:
|
|
assessment = assessment_by_claim.get(claim.get("id"))
|
|
if assessment is None:
|
|
raise CompilerError("V3 strict child is missing a claim-local evidence assessment")
|
|
if assessment.get("evidence_set_hash") != ap._v3_evidence_set_hash(claim["id"], evidence):
|
|
raise CompilerError("V3 assessment evidence_set_hash does not match claim-local evidence")
|
|
|
|
|
|
def _compile_v3_source_packet(
|
|
artifact_bytes: bytes,
|
|
text_bytes: bytes,
|
|
manifest_bytes: bytes,
|
|
extractor_spec_bytes: bytes,
|
|
manifest_object: dict[str, Any],
|
|
) -> dict[str, Any]:
|
|
artifact_sha256 = sha256_bytes(artifact_bytes)
|
|
text_sha256 = sha256_bytes(text_bytes)
|
|
extractor_spec_sha256 = sha256_bytes(extractor_spec_bytes)
|
|
manifest_file_sha256 = sha256_bytes(manifest_bytes)
|
|
manifest, quote_bindings = _validate_manifest_v3(
|
|
manifest_object,
|
|
artifact_sha256,
|
|
text_sha256,
|
|
extractor_spec_sha256,
|
|
text_bytes,
|
|
)
|
|
manifest_canonical_sha256 = canonical_sha256(manifest)
|
|
identity_material, candidate_content_sha256, packet_identity_sha256 = _v3_identity_material(
|
|
manifest, manifest_file_sha256
|
|
)
|
|
parent = _build_v3_parent_proposal(
|
|
manifest,
|
|
manifest_canonical_sha256,
|
|
identity_material,
|
|
candidate_content_sha256,
|
|
packet_identity_sha256,
|
|
)
|
|
child = _build_v3_strict_child(
|
|
manifest,
|
|
manifest_canonical_sha256,
|
|
quote_bindings,
|
|
parent,
|
|
identity_material,
|
|
candidate_content_sha256,
|
|
packet_identity_sha256,
|
|
)
|
|
_validate_v3_compiled_child(child, manifest, quote_bindings)
|
|
stage_sql = stage.build_stage_sql(child)
|
|
stable_bundle = {
|
|
"schema": BUNDLE_SCHEMA_V3,
|
|
"status": "pending_review",
|
|
"build_only": True,
|
|
"database_write_performed": False,
|
|
"canonical_apply_performed": False,
|
|
"inputs": {
|
|
"artifact_bytes": len(artifact_bytes),
|
|
"extracted_text_bytes": len(text_bytes),
|
|
"manifest_bytes": len(manifest_bytes),
|
|
"extractor_spec_bytes": len(extractor_spec_bytes),
|
|
},
|
|
"validated_manifest": manifest,
|
|
"quote_bindings": quote_bindings,
|
|
"parent_proposal": parent,
|
|
"strict_child_proposal": child,
|
|
"stage_preview": {
|
|
"target_table": "kb_stage.kb_proposals",
|
|
"target_status": "pending_review",
|
|
"executed": False,
|
|
"stage_sql_sha256": sha256_bytes(stage_sql.encode("utf-8")),
|
|
},
|
|
"compiler_contract": {
|
|
"manifest_contract": MANIFEST_SCHEMA_V3,
|
|
"output_contract": BUNDLE_SCHEMA_V3,
|
|
"approve_claim_contract_version": 3,
|
|
"strict_preflight": "scripts/compile_kb_source_packet.py::_validate_v3_compiled_child",
|
|
"writer_apply_invoked": False,
|
|
"writer_timestamp_contract": "fixed-microsecond UTC Z",
|
|
"manifest_identity_binding": "sha256(exact duplicate-free UTF-8 JSON file bytes)",
|
|
"extractor_spec_binding": "sha256(exact independently supplied artifact bytes)",
|
|
"semantic_extraction_is_retained_input": True,
|
|
"semantic_extraction_statement": (
|
|
"The semantic extraction is a retained input; this is the smallest reproducible learning input, "
|
|
"not a model-training claim."
|
|
),
|
|
"model_training_claim": False,
|
|
"source_from_chat_labels": False,
|
|
"database_stage_executed": False,
|
|
},
|
|
"hashes": {
|
|
"artifact_sha256": artifact_sha256,
|
|
"extracted_text_sha256": text_sha256,
|
|
"extractor_spec_sha256": extractor_spec_sha256,
|
|
"manifest_file_sha256": manifest_file_sha256,
|
|
"manifest_canonical_sha256": manifest_canonical_sha256,
|
|
"normalized_candidate_content_sha256": candidate_content_sha256,
|
|
"packet_identity_sha256": packet_identity_sha256,
|
|
"parent_packet_sha256": child["parent_packet_sha256"],
|
|
"strict_child_payload_sha256": child["payload_sha256"],
|
|
},
|
|
}
|
|
stable_bundle["hashes"]["bundle_content_sha256"] = canonical_sha256(stable_bundle)
|
|
return stable_bundle
|
|
|
|
|
|
def _validate_compiled_child(
|
|
child: dict[str, Any], manifest: dict[str, Any], quote_bindings: list[dict[str, Any]]
|
|
) -> None:
|
|
if child.get("status") != "pending_review" or child.get("proposal_type") != "approve_claim":
|
|
raise CompilerError("normalization did not produce one pending_review approve_claim child")
|
|
apply_payload = (child.get("payload") or {}).get("apply_payload")
|
|
if not isinstance(apply_payload, dict) or apply_payload.get("contract_version") != 2:
|
|
raise CompilerError("strict child has no schema-valid approve_claim v2 apply_payload")
|
|
|
|
claims = apply_payload.get("claims") or []
|
|
sources = apply_payload.get("sources") or []
|
|
if len(claims) != len(manifest["claims"]):
|
|
raise CompilerError("strict child claim count differs from the extraction manifest")
|
|
if len({row.get("id") for row in claims}) != len(claims):
|
|
raise CompilerError("strict child contains duplicate claim ids")
|
|
if [row.get("text") for row in claims] != [row["text"] for row in manifest["claims"]]:
|
|
raise CompilerError("strict child claim text differs from the extracted atomic propositions")
|
|
if len({row.get("id") for row in sources}) != len(sources):
|
|
raise CompilerError("strict child contains duplicate source ids")
|
|
|
|
source = manifest["source"]
|
|
artifact_sources = [
|
|
row
|
|
for row in sources
|
|
if row.get("hash") == manifest["artifact_sha256"]
|
|
and (row.get("url") == source["locator"] or row.get("storage_path") == source["locator"])
|
|
]
|
|
if len(artifact_sources) != 1:
|
|
raise CompilerError("strict child must contain exactly one hash-bound declared artifact source")
|
|
try:
|
|
excerpt_packet = json.loads(artifact_sources[0]["excerpt"])
|
|
except (TypeError, json.JSONDecodeError) as exc:
|
|
raise CompilerError("declared source excerpt was truncated or is not reviewable JSON") from exc
|
|
retained_excerpt = excerpt_packet.get("excerpt") if isinstance(excerpt_packet, dict) else None
|
|
if not isinstance(retained_excerpt, str):
|
|
raise CompilerError("declared source excerpt does not retain exact quote text")
|
|
missing_quotes = sorted(
|
|
{binding["quote"] for binding in quote_bindings if binding["quote"] not in retained_excerpt}
|
|
)
|
|
if missing_quotes:
|
|
raise CompilerError("one or more exact quotes exceed the current canonical source excerpt capacity")
|
|
|
|
|
|
def compile_source_packet(
|
|
artifact_path: Path,
|
|
text_path: Path,
|
|
manifest_path: Path,
|
|
extractor_spec_path: Path | None = None,
|
|
) -> dict[str, Any]:
|
|
artifact_bytes = _read_required_file(artifact_path, "artifact")
|
|
text_bytes = _read_required_file(text_path, "extracted text")
|
|
manifest_bytes = _read_required_file(manifest_path, "extraction manifest")
|
|
try:
|
|
extracted_text = text_bytes.decode("utf-8", errors="strict")
|
|
except UnicodeDecodeError as exc:
|
|
raise CompilerError(f"extracted text must be valid UTF-8: {exc}") from exc
|
|
manifest_raw = _load_duplicate_free_json(manifest_bytes, "extraction manifest")
|
|
manifest_object = _require_object(manifest_raw, "manifest")
|
|
|
|
artifact_sha256 = sha256_bytes(artifact_bytes)
|
|
text_sha256 = sha256_bytes(text_bytes)
|
|
if manifest_object.get("schema") == MANIFEST_SCHEMA_V3:
|
|
if extractor_spec_path is None:
|
|
raise CompilerError("V3 requires an independently supplied extractor specification artifact")
|
|
extractor_spec_bytes = _read_required_file(extractor_spec_path, "extractor specification")
|
|
return _compile_v3_source_packet(
|
|
artifact_bytes,
|
|
text_bytes,
|
|
manifest_bytes,
|
|
extractor_spec_bytes,
|
|
manifest_object,
|
|
)
|
|
if extractor_spec_path is not None:
|
|
raise CompilerError("extractor specification input is only valid for a V3 manifest")
|
|
manifest, quote_bindings = _validate_manifest(manifest_object, artifact_sha256, text_sha256, extracted_text)
|
|
manifest_canonical_sha256 = canonical_sha256(manifest)
|
|
parent = build_parent_proposal(manifest, manifest_canonical_sha256, quote_bindings)
|
|
try:
|
|
child = stage.prepare_normalized_child(parent)
|
|
except (ValueError, stage.bound.CheckpointError) as exc:
|
|
raise CompilerError(f"strict child preflight rejected the source packet: {exc}") from exc
|
|
_validate_compiled_child(child, manifest, quote_bindings)
|
|
|
|
stage_sql = stage.build_stage_sql(child)
|
|
stable_bundle = {
|
|
"schema": BUNDLE_SCHEMA,
|
|
"status": "pending_review",
|
|
"build_only": True,
|
|
"database_write_performed": False,
|
|
"canonical_apply_performed": False,
|
|
"inputs": {
|
|
"artifact_bytes": len(artifact_bytes),
|
|
"extracted_text_bytes": len(text_bytes),
|
|
"manifest_bytes": len(manifest_bytes),
|
|
},
|
|
"validated_manifest": manifest,
|
|
"quote_bindings": quote_bindings,
|
|
"parent_proposal": parent,
|
|
"strict_child_proposal": child,
|
|
"stage_preview": {
|
|
"target_table": "kb_stage.kb_proposals",
|
|
"executed": False,
|
|
"stage_sql_sha256": sha256_bytes(stage_sql.encode("utf-8")),
|
|
},
|
|
"compiler_contract": {
|
|
"normalizer": "scripts/kb_proposal_normalize.py",
|
|
"strict_preflight": "scripts/stage_normalized_proposal.py",
|
|
"declared_artifact_source_count": 1,
|
|
"source_from_chat_labels": False,
|
|
},
|
|
"hashes": {
|
|
"artifact_sha256": artifact_sha256,
|
|
"extracted_text_sha256": text_sha256,
|
|
"manifest_file_sha256": sha256_bytes(manifest_bytes),
|
|
"manifest_canonical_sha256": manifest_canonical_sha256,
|
|
"parent_packet_sha256": child["parent_packet_sha256"],
|
|
"strict_child_payload_sha256": child["payload_sha256"],
|
|
},
|
|
}
|
|
stable_bundle["hashes"]["bundle_content_sha256"] = canonical_sha256(stable_bundle)
|
|
return stable_bundle
|
|
|
|
|
|
def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--artifact", required=True, type=Path, help="raw source artifact; any binary format")
|
|
parser.add_argument("--text", required=True, type=Path, help="strict UTF-8 extraction of the artifact")
|
|
parser.add_argument("--manifest", required=True, type=Path, help="extraction manifest JSON")
|
|
parser.add_argument(
|
|
"--extractor-spec",
|
|
type=Path,
|
|
help="exact extractor specification artifact; required for V3 and rejected for V1/V2",
|
|
)
|
|
parser.add_argument("--output", required=True, type=Path, help="private path for the compiled proposal bundle")
|
|
return parser.parse_args(argv)
|
|
|
|
|
|
def _write_private_output(path: Path, rendered: str) -> None:
|
|
path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
|
|
fd, raw_temp = tempfile.mkstemp(prefix=f".{path.name}.", suffix=".tmp", dir=path.parent)
|
|
temp = Path(raw_temp)
|
|
try:
|
|
os.fchmod(fd, 0o600)
|
|
with os.fdopen(fd, "w", encoding="utf-8") as handle:
|
|
handle.write(rendered)
|
|
handle.flush()
|
|
os.fsync(handle.fileno())
|
|
os.replace(temp, path)
|
|
os.chmod(path, 0o600)
|
|
finally:
|
|
if temp.exists():
|
|
temp.unlink()
|
|
|
|
|
|
def _print_status(status: str, *, reason: str | None = None) -> None:
|
|
payload: dict[str, Any] = {
|
|
"schema": STATUS_SCHEMA,
|
|
"status": status,
|
|
"database_write_performed": False,
|
|
"canonical_apply_performed": False,
|
|
"private_bundle_written": status == "pending_review",
|
|
}
|
|
if reason is not None:
|
|
payload["reason"] = reason
|
|
print(json.dumps(payload, sort_keys=True))
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
args = parse_args(argv)
|
|
try:
|
|
output_path = args.output.resolve()
|
|
input_paths = {args.artifact.resolve(), args.text.resolve(), args.manifest.resolve()}
|
|
if args.extractor_spec is not None:
|
|
input_paths.add(args.extractor_spec.resolve())
|
|
if output_path in input_paths:
|
|
raise CompilerError("--output must not overwrite an input file")
|
|
bundle = compile_source_packet(args.artifact, args.text, args.manifest, args.extractor_spec)
|
|
except (CompilerError, OSError, ValueError):
|
|
_print_status("rejected", reason="input_validation_failed")
|
|
return 2
|
|
|
|
rendered = json.dumps(bundle, indent=2, sort_keys=True, ensure_ascii=True) + "\n"
|
|
try:
|
|
_write_private_output(args.output, rendered)
|
|
except OSError:
|
|
_print_status("rejected", reason="output_write_failed")
|
|
return 2
|
|
_print_status("pending_review")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|