#!/usr/bin/env python3 """Preview strict apply-payload normalization for staged KB proposals. Leo and reviewers can approve rich proposal intent before it is mechanically safe to apply. The apply worker intentionally accepts only strict contracts: ``add_edge``, ``attach_evidence``, ``revise_strategy``, and atomic ``approve_claim`` bundles that carry ``payload.apply_payload`` with canonical IDs and reviewed row content. This helper is read-only by default. It inspects a freeform proposal and reports: * strict child proposals that can be emitted because all canonical IDs exist in the proposal payload; * blocked fragments that still require claim/source creation, source dedup, or a schema decision before canonical apply is possible. """ from __future__ import annotations import argparse import hashlib import json import re import sys from collections import Counter from pathlib import Path from typing import Any HERE = Path(__file__).resolve().parent sys.path.insert(0, str(HERE)) import apply_proposal as ap # noqa: E402 import kb_proposal_review_packet as review_packet # noqa: E402 import kb_rich_proposal_creation_plan as rich_plan # noqa: E402 UUID_RE = re.compile( r"^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$" ) RICH_CANDIDATE_COLLECTIONS = ( ("claim_candidates", "claim_key", "claim"), ("proposed_claims", "claim_key", "claim"), ("source_candidates", "source_key", "source"), ) def _payload_dict(payload: dict[str, Any], key: str) -> dict[str, Any]: value = payload.get(key) return value if isinstance(value, dict) else {} def _payload_list(payload: dict[str, Any], key: str) -> list[Any]: value = payload.get(key) return value if isinstance(value, list) else [] def _is_uuid(value: Any) -> bool: return isinstance(value, str) and bool(UUID_RE.match(value)) def _first_uuid(fragment: dict[str, Any], keys: tuple[str, ...]) -> str | None: for key in keys: value = fragment.get(key) if _is_uuid(value): return value return None def _fingerprint(value: Any) -> str: raw = json.dumps(value, sort_keys=True, separators=(",", ":")) return hashlib.sha256(raw.encode("utf-8")).hexdigest()[:16] def _compact_fragment(fragment: Any) -> dict[str, Any]: if not isinstance(fragment, dict): return {"value_type": type(fragment).__name__, "fingerprint": _fingerprint(fragment)} keys = [ "claim_key", "source_key", "headline", "title", "edge_type", "target", "target_claim_id", "to_claim_id", "to_existing_claim", "from_claim_id", "from_claim", "claim_id", "source_id", ] compact = {key: fragment[key] for key in keys if key in fragment} compact["fingerprint"] = _fingerprint(fragment) return compact def _blocked(kind: str, reason: str, missing: list[str], fragment: Any) -> dict[str, Any]: return { "kind": kind, "reason": reason, "missing": missing, "fragment": _compact_fragment(fragment), } def _candidate_inventory(payload: dict[str, Any]) -> dict[str, Any]: """Validate rich candidate inputs before the planner can skip or dedupe them.""" inventory: dict[str, Any] = { "claim_items": [], "source_items": [], "collection_counts": {}, "blocked_fragments": [], } for collection, key_field, kind in RICH_CANDIDATE_COLLECTIONS: if collection not in payload: inventory["collection_counts"][collection] = 0 continue candidates = payload.get(collection) if not isinstance(candidates, list): block = _blocked( f"{kind}_candidate", f"payload.{collection} must be a list of objects", [key_field], candidates, ) block["location"] = f"payload.{collection}" inventory["blocked_fragments"].append(block) inventory["collection_counts"][collection] = None continue inventory["collection_counts"][collection] = len(candidates) for index, candidate in enumerate(candidates): location = f"payload.{collection}[{index}]" if not isinstance(candidate, dict): block = _blocked( f"{kind}_candidate", f"{location} must be an object", [key_field], candidate, ) block["location"] = location inventory["blocked_fragments"].append(block) continue raw_key = candidate.get(key_field) if not isinstance(raw_key, str) or not raw_key.strip(): block = _blocked( f"{kind}_candidate", f"{location} is missing a nonempty string {key_field}", [key_field], candidate, ) block["location"] = location inventory["blocked_fragments"].append(block) continue inventory[f"{kind}_items"].append( { "collection": collection, "index": index, "key": raw_key.strip(), "fragment": candidate, } ) for kind, key_field in (("claim", "claim_key"), ("source", "source_key")): by_key: dict[str, list[dict[str, Any]]] = {} for item in inventory[f"{kind}_items"]: by_key.setdefault(item["key"], []).append(item) for candidate_key, occurrences in by_key.items(): if len(occurrences) == 1: continue locations = [f"payload.{item['collection']}[{item['index']}]" for item in occurrences] block = _blocked( f"{kind}_candidate", f"duplicate {key_field} {candidate_key!r} at {', '.join(locations)}", [], [item["fragment"] for item in occurrences], ) block["locations"] = locations inventory["blocked_fragments"].append(block) return inventory def _planned_row_accounting( plan: dict[str, Any], inventory: dict[str, Any] ) -> tuple[dict[str, Any], list[str]]: """Reconcile every validated input candidate to exactly one planned row.""" rows = plan.get("planned_rows") if isinstance(plan.get("planned_rows"), dict) else {} generated = plan.get("generated_ids") if isinstance(plan.get("generated_ids"), dict) else {} claim_id_map = generated.get("claims") if isinstance(generated.get("claims"), dict) else {} source_id_map = generated.get("sources") if isinstance(generated.get("sources"), dict) else {} body_source_id_map = ( generated.get("claim_body_sources") if isinstance(generated.get("claim_body_sources"), dict) else {} ) claim_rows = rows.get("claims") if isinstance(rows.get("claims"), list) else [] source_rows = rows.get("sources") if isinstance(rows.get("sources"), list) else [] claim_keys = [item["key"] for item in inventory["claim_items"]] source_keys = [item["key"] for item in inventory["source_items"]] actual_claim_ids = [row.get("id") for row in claim_rows if isinstance(row, dict)] actual_source_ids = [row.get("id") for row in source_rows if isinstance(row, dict)] expected_claim_ids = [claim_id_map.get(key) for key in claim_keys] expected_candidate_source_ids = [source_id_map.get(key) for key in source_keys] expected_body_source_ids = [body_source_id_map.get(key) for key in claim_keys] candidate_source_id_set = {value for value in expected_candidate_source_ids if value is not None} body_source_id_set = {value for value in expected_body_source_ids if value is not None} planned_candidate_source_ids = [value for value in actual_source_ids if value in candidate_source_id_set] planned_body_source_ids = [value for value in actual_source_ids if value in body_source_id_set] accounting = { "input_collections": inventory["collection_counts"], "claim_candidates": { "input_count": len(claim_keys), "input_keys": claim_keys, "planned_row_count": len(claim_rows), "planned_row_ids": actual_claim_ids, }, "source_candidates": { "input_count": len(source_keys), "input_keys": source_keys, "expected_row_ids": expected_candidate_source_ids, "planned_row_count": len(planned_candidate_source_ids), "planned_row_ids": planned_candidate_source_ids, }, "generated_claim_body_sources": { "expected_count": len(claim_keys), "expected_row_ids": expected_body_source_ids, "planned_row_count": len(planned_body_source_ids), "planned_row_ids": planned_body_source_ids, }, "total_planned_source_rows": len(source_rows), } errors: list[str] = [] if set(claim_id_map) != set(claim_keys): errors.append("planned claim key set does not exactly match validated claim candidates") if set(source_id_map) != set(source_keys): errors.append("planned source key set does not exactly match validated source candidates") if set(body_source_id_map) != set(claim_keys): errors.append("planned claim-body source key set does not exactly match validated claim candidates") if any(value is None for value in expected_claim_ids): errors.append("one or more validated claim candidates has no generated claim id") if any(value is None for value in expected_candidate_source_ids): errors.append("one or more validated source candidates has no generated source id") if any(value is None for value in expected_body_source_ids): errors.append("one or more validated claim candidates has no generated claim-body source id") if Counter(actual_claim_ids) != Counter(expected_claim_ids): errors.append("planned claim rows do not account for every validated claim candidate exactly once") if Counter(actual_source_ids) != Counter(expected_candidate_source_ids + expected_body_source_ids): errors.append( "planned source rows do not account for every validated source candidate and generated claim body exactly once" ) return accounting, errors def _normalization_manifest(plan: dict[str, Any], accounting: dict[str, Any]) -> dict[str, Any]: rows = plan["planned_rows"] body_source_ids = set(plan["generated_ids"]["claim_body_sources"].values()) claim_type_mappings: list[dict[str, Any]] = [] source_type_mappings: list[dict[str, Any]] = [] for row in rows["claims"]: mapping = row.get("type_normalization") if mapping: claim_type_mappings.append( {"claim_key": row.get("claim_key"), "claim_id": row.get("id"), **dict(mapping)} ) for row in rows["sources"]: mapping = row.get("source_type_normalization") if mapping: source_type_mappings.append( { "source_key": row.get("source_key"), "source_id": row.get("id"), "generated_claim_body_source": row.get("id") in body_source_ids, **dict(mapping), } ) return { "candidate_accounting": accounting, "claim_type_mappings": claim_type_mappings, "source_type_mappings": source_type_mappings, } def _edge_candidate(fragment: dict[str, Any], default_from_claim: str | None = None) -> dict[str, Any] | None: edge_type = fragment.get("edge_type") from_claim = _first_uuid(fragment, ("from_claim", "from_claim_id", "from_claim_uuid")) or default_from_claim to_claim = _first_uuid( fragment, ( "to_claim", "to_claim_id", "target_claim_id", "target_claim", "to_existing_claim", "existing_claim_id", ), ) if from_claim and to_claim and edge_type: result: dict[str, Any] = {"from_claim": from_claim, "to_claim": to_claim, "edge_type": edge_type} if fragment.get("weight") is not None: result["weight"] = fragment.get("weight") return result return None def _evidence_candidate(fragment: dict[str, Any], default_claim_id: str | None = None) -> dict[str, Any] | None: claim_id = _first_uuid(fragment, ("claim_id", "target_claim_id", "to_claim_id")) or default_claim_id source_id = _first_uuid(fragment, ("source_id", "source_uuid", "canonical_source_id")) if claim_id and source_id: result: dict[str, Any] = {"claim_id": claim_id, "source_id": source_id} if fragment.get("role"): result["role"] = fragment.get("role") if fragment.get("weight") is not None: result["weight"] = fragment.get("weight") return result return None def _strict_child(kind: str, apply_payload: dict[str, Any], source_fragment: Any) -> dict[str, Any]: return { "proposal_type": kind, "payload": {"apply_payload": apply_payload}, "source_fragment_fingerprint": _fingerprint(source_fragment), } def _strict_approve_claim_child( proposal: dict[str, Any], plan: dict[str, Any], normalization_manifest: dict[str, Any] ) -> dict[str, Any] | None: """Convert a fully resolved rich graph into one atomic strict child.""" rows = plan["planned_rows"] if not rows["claims"]: return None if plan["blocked_fragments"] or plan["unresolved_design_decisions"]: return None agent_id = proposal.get("proposed_by_agent_id") apply_payload = { "contract_version": 2, "source_proposal_id": proposal.get("id"), "agent_id": agent_id, "claims": [ { key: row.get(key) for key in ("id", "type", "text", "status", "confidence", "tags", "created_by") } for row in rows["claims"] ], "sources": [ { key: row.get(key) for key in ("id", "source_type", "url", "storage_path", "excerpt", "hash", "created_by") } for row in rows["sources"] ], "evidence": [ { "claim_id": row.get("claim_id"), "source_id": row.get("source_id"), "role": row.get("role"), "weight": row.get("weight"), "created_by": agent_id, } for row in rows["claim_evidence"] ], "edges": [ { "from_claim": row.get("from_claim"), "to_claim": row.get("to_claim"), "edge_type": row.get("edge_type"), "weight": row.get("weight"), "created_by": agent_id, } for row in rows["claim_edges"] ], "reasoning_tools": [ { key: row.get(key) for key in ("id", "agent_id", "name", "description", "category") } for row in rows["reasoning_tools"] ], "normalization_manifest": normalization_manifest, } return _strict_child("approve_claim", apply_payload, proposal.get("payload") or {}) def _rich_plan_blockers(plan: dict[str, Any]) -> list[dict[str, Any]]: blocked: list[dict[str, Any]] = [] for fragment in plan.get("blocked_fragments") or []: reasons = fragment.get("reasons") if isinstance(fragment, dict) else None reason = "; ".join(str(item) for item in reasons) if isinstance(reasons, list) else "rich fragment is blocked" source_fragment = fragment.get("fragment") if isinstance(fragment, dict) else fragment kind = fragment.get("kind", "rich_fragment") if isinstance(fragment, dict) else "rich_fragment" blocked.append(_blocked(kind, reason, [], source_fragment)) for decision in plan.get("unresolved_design_decisions") or []: if isinstance(decision, dict): blocked.append( _blocked( "design_decision", str(decision.get("reason") or "rich proposal has an unresolved design decision"), [str(decision.get("decision") or "reviewed design decision")], decision, ) ) return blocked def _normalize_edges(payload: dict[str, Any]) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: strict: list[dict[str, Any]] = [] blocked: list[dict[str, Any]] = [] old_claim_id = _first_uuid(_payload_dict(payload, "old_claim"), ("id", "claim_id")) edge_sources: list[tuple[dict[str, Any], str | None]] = [] for edge in _payload_list(payload, "proposed_supersession_edges"): if isinstance(edge, dict): edge_sources.append((edge, None)) for candidate in _payload_list(payload, "claim_candidates"): if not isinstance(candidate, dict): continue candidate_claim_id = _first_uuid(candidate, ("claim_id", "id", "canonical_claim_id")) for edge in _payload_list(candidate, "edges"): if isinstance(edge, dict): edge_sources.append((edge, candidate_claim_id)) if candidate_claim_id and old_claim_id: edge_sources.append( ( { "from_claim": candidate_claim_id, "to_claim": old_claim_id, "edge_type": "supersedes_component_of", }, None, ) ) seen = set() for fragment, default_from_claim in edge_sources: candidate = _edge_candidate(fragment, default_from_claim) if candidate: key = json.dumps(candidate, sort_keys=True) if key not in seen: seen.add(key) strict.append(_strict_child("add_edge", candidate, fragment)) continue missing = [] if not fragment.get("edge_type"): missing.append("edge_type") if not (_first_uuid(fragment, ("from_claim", "from_claim_id", "from_claim_uuid")) or default_from_claim): missing.append("canonical from_claim") if not _first_uuid( fragment, ("to_claim", "to_claim_id", "target_claim_id", "target_claim", "to_existing_claim", "existing_claim_id"), ): missing.append("canonical to_claim") blocked.append(_blocked("edge", "edge fragment is not a strict add_edge contract", missing, fragment)) return strict, blocked def _normalize_evidence(payload: dict[str, Any]) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: evidence_rows: list[dict[str, Any]] = [] blocked: list[dict[str, Any]] = [] evidence_sources: list[tuple[dict[str, Any], str | None]] = [] for evidence in _payload_list(payload, "evidence"): if isinstance(evidence, dict): evidence_sources.append((evidence, None)) for evidence in _payload_list(payload, "evidence_candidates"): if isinstance(evidence, dict): evidence_sources.append((evidence, None)) for candidate in _payload_list(payload, "claim_candidates"): if not isinstance(candidate, dict): continue candidate_claim_id = _first_uuid(candidate, ("claim_id", "id", "canonical_claim_id")) for evidence in _payload_list(candidate, "evidence"): if isinstance(evidence, dict): evidence_sources.append((evidence, candidate_claim_id)) seen = set() for fragment, default_claim_id in evidence_sources: candidate = _evidence_candidate(fragment, default_claim_id) if candidate: key = json.dumps(candidate, sort_keys=True) if key not in seen: seen.add(key) evidence_rows.append(candidate) continue missing = [] if not (_first_uuid(fragment, ("claim_id", "target_claim_id", "to_claim_id")) or default_claim_id): missing.append("canonical claim_id") if not _first_uuid(fragment, ("source_id", "source_uuid", "canonical_source_id")): missing.append("canonical source_id") blocked.append(_blocked("evidence", "evidence fragment is not a strict attach_evidence contract", missing, fragment)) strict = [_strict_child("attach_evidence", {"evidence": evidence_rows}, evidence_rows)] if evidence_rows else [] return strict, blocked def _normalize_strategy(proposal: dict[str, Any], payload: dict[str, Any]) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: if proposal.get("proposal_type") != "revise_strategy": return [], [] strategy_payload = payload.get("apply_payload") or { key: payload.get(key) for key in ("agent_id", "strategy", "strategy_nodes") if key in payload } has_contract = all(strategy_payload.get(key) for key in ("agent_id", "strategy", "strategy_nodes")) if has_contract: return [_strict_child("revise_strategy", strategy_payload, payload)], [] return [], [ _blocked( "strategy", "strategy revision needs agent_id, strategy, and strategy_nodes", [key for key in ("agent_id", "strategy", "strategy_nodes") if not strategy_payload.get(key)], payload, ) ] def _dedupe_children(children: list[dict[str, Any]]) -> list[dict[str, Any]]: seen = set() out = [] for child in children: key = json.dumps({"proposal_type": child["proposal_type"], "payload": child["payload"]}, sort_keys=True) if key in seen: continue seen.add(key) out.append(child) return out def normalize_proposal( proposal: dict[str, Any], mappings: dict[str, Any] | None = None ) -> dict[str, Any]: payload = proposal.get("payload") or {} if not isinstance(payload, dict): payload = {} inventory = _candidate_inventory(payload) if inventory["blocked_fragments"]: return { "proposal_id": proposal.get("id"), "normalization_state": "blocked_lossy_rich_normalization", "strict_child_proposals": [], "blocked_fragments": inventory["blocked_fragments"], "blocked_count": len(inventory["blocked_fragments"]), "strict_child_count": 0, "next_normalization_action": ( "Repair every malformed, missing-key, or duplicate rich candidate, then rerun normalization." ), } if payload.get("apply_payload") and proposal.get("proposal_type") in ap.APPLYABLE_TYPES: return { "proposal_id": proposal.get("id"), "normalization_state": "already_strict_apply_payload", "strict_child_proposals": [], "blocked_fragments": [], "blocked_count": 0, "strict_child_count": 0, "next_normalization_action": "Proposal already carries a strict apply_payload; use dry-run/apply review.", } has_rich_candidates = bool(inventory["claim_items"] or inventory["source_items"]) if has_rich_candidates: plan = rich_plan.build_creation_plan(proposal, mappings) accounting, accounting_errors = _planned_row_accounting(plan, inventory) if not inventory["claim_items"]: accounting_errors.append( "rich source candidates cannot be emitted by approve_claim without at least one claim candidate" ) if accounting_errors: accounting_blocks = [ { "kind": "candidate_accounting", "reason": reason, "missing": [], "candidate_accounting": accounting, "fragment": {"fingerprint": _fingerprint(accounting)}, } for reason in accounting_errors ] return { "proposal_id": proposal.get("id"), "normalization_state": "blocked_lossy_rich_normalization", "strict_child_proposals": [], "blocked_fragments": accounting_blocks, "blocked_count": len(accounting_blocks), "strict_child_count": 0, "next_normalization_action": ( "Repair rich planner row accounting so every candidate maps to exactly one planned row, then rerun." ), } manifest = _normalization_manifest(plan, accounting) approve_claim_child = _strict_approve_claim_child(proposal, plan, manifest) if approve_claim_child: return { "proposal_id": proposal.get("id"), "normalization_state": "strict_children_ready", "strict_child_proposals": [approve_claim_child], "blocked_fragments": [], "blocked_count": 0, "strict_child_count": 1, "next_normalization_action": ( "Review and stage the atomic approve_claim child, then use dry-run/apply review." ), } blocked_fragments = _rich_plan_blockers(plan) if not blocked_fragments: blocked_fragments = [ { "kind": "rich_plan", "reason": "rich candidates were accounted for but no atomic strict child was produced", "missing": [], "candidate_accounting": accounting, "fragment": {"fingerprint": _fingerprint(accounting)}, } ] return { "proposal_id": proposal.get("id"), "normalization_state": "blocked_rich_normalization", "strict_child_proposals": [], "blocked_fragments": blocked_fragments, "blocked_count": len(blocked_fragments), "strict_child_count": 0, "next_normalization_action": ( "Resolve every blocked rich fragment or design decision, then rerun the atomic normalization." ), } strict_children: list[dict[str, Any]] = [] blocked_fragments: list[dict[str, Any]] = [] for strict, blocked in ( _normalize_edges(payload), _normalize_evidence(payload), _normalize_strategy(proposal, payload), ): strict_children.extend(strict) blocked_fragments.extend(blocked) strict_children = _dedupe_children(strict_children) if strict_children and not blocked_fragments: state = "strict_children_ready" next_action = "Review generated strict child proposal JSON, then stage as approved apply_payload proposals." elif strict_children: state = "partial_strict_children_ready" next_action = "Stage only the strict children after review; resolve blocked fragments separately." elif blocked_fragments: state = "blocked_missing_canonical_ids" next_action = "Create or identify canonical claim/source rows, then rerun normalization." else: state = "no_supported_apply_fragments" next_action = "Rewrite as a supported add_edge, attach_evidence, or revise_strategy apply contract." return { "proposal_id": proposal.get("id"), "normalization_state": state, "strict_child_proposals": strict_children, "blocked_fragments": blocked_fragments, "blocked_count": len(blocked_fragments), "strict_child_count": len(strict_children), "next_normalization_action": next_action, } def load_from_input(path: str) -> list[dict[str, Any]]: return review_packet.load_from_input(path) def load_from_db(args: argparse.Namespace) -> list[dict[str, Any]]: return review_packet.load_from_db(args) def parse_args(argv: list[str]) -> argparse.Namespace: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--input-json", help="normalize proposal JSON from a file") parser.add_argument("--proposal-id", help="load one live proposal from Postgres") parser.add_argument("--status", default="approved", help="load live proposals by status") parser.add_argument("--limit", type=int, default=20) parser.add_argument("--mapping-json", help="optional reviewed mapping overlay for rich claim graphs") parser.add_argument("--secrets-file", default=ap.DEFAULT_SECRETS_FILE) parser.add_argument("--container", default=ap.DEFAULT_CONTAINER) parser.add_argument("--db", default=ap.DEFAULT_DB) parser.add_argument("--host", default=ap.DEFAULT_HOST) parser.add_argument("--role", default=ap.DEFAULT_ROLE) parser.add_argument("--output", type=Path, help="optional JSON output path") return parser.parse_args(argv) def main(argv: list[str] | None = None) -> int: args = parse_args(sys.argv[1:] if argv is None else argv) proposals = load_from_input(args.input_json) if args.input_json else load_from_db(args) mappings = json.loads(Path(args.mapping_json).read_text(encoding="utf-8")) if args.mapping_json else None previews = [normalize_proposal(proposal, mappings) for proposal in proposals] rendered = json.dumps({"normalization_previews": previews}, indent=2, sort_keys=True) + "\n" if args.output: args.output.parent.mkdir(parents=True, exist_ok=True) args.output.write_text(rendered, encoding="utf-8") print(rendered, end="") return 0 if __name__ == "__main__": raise SystemExit(main())