558 lines
21 KiB
Python
558 lines
21 KiB
Python
"""Post-extraction validator — deterministic fixes and quality gate.
|
|
|
|
Runs AFTER LLM extraction, BEFORE git commit. Pure Python, $0 cost.
|
|
Catches the mechanical issues that account for 73% of eval rejections:
|
|
- Frontmatter schema violations (missing/invalid fields)
|
|
- Broken wiki links (strips brackets, keeps text)
|
|
- Date errors (wrong format, source date instead of today)
|
|
- Filename convention violations
|
|
- Title precision (too short, not a proposition)
|
|
- Duplicate detection against existing KB
|
|
|
|
Design principles (Leo):
|
|
- Mechanical rules belong in code, not prompts
|
|
- Fix what's fixable, reject what's not
|
|
- Never silently drop content — log everything
|
|
|
|
Epimetheus owns this module. Leo reviews changes.
|
|
"""
|
|
|
|
import json
|
|
import logging
|
|
import re
|
|
from datetime import date, datetime
|
|
from difflib import SequenceMatcher
|
|
from pathlib import Path
|
|
|
|
logger = logging.getLogger("pipeline.post_extract")
|
|
|
|
# ─── Constants ──────────────────────────────────────────────────────────────
|
|
|
|
VALID_DOMAINS = frozenset({
|
|
"internet-finance", "entertainment", "health", "ai-alignment",
|
|
"space-development", "grand-strategy", "mechanisms", "living-capital",
|
|
"living-agents", "teleohumanity", "critical-systems",
|
|
"collective-intelligence", "teleological-economics", "cultural-dynamics",
|
|
})
|
|
|
|
VALID_CONFIDENCE = frozenset({"proven", "likely", "experimental", "speculative"})
|
|
|
|
REQUIRED_CLAIM_FIELDS = ("type", "domain", "description", "confidence", "source", "created")
|
|
REQUIRED_ENTITY_FIELDS = ("type", "domain", "description")
|
|
|
|
WIKI_LINK_RE = re.compile(r"\[\[([^\]]+)\]\]")
|
|
|
|
# Minimum title word count for claims (Leo: titles must name specific mechanism)
|
|
MIN_TITLE_WORDS = 8
|
|
|
|
DEDUP_THRESHOLD = 0.85
|
|
|
|
|
|
# ─── YAML parsing ──────────────────────────────────────────────────────────
|
|
|
|
|
|
def parse_frontmatter(text: str) -> tuple[dict | None, str]:
|
|
"""Extract YAML frontmatter from markdown. Returns (frontmatter_dict, body)."""
|
|
if not text.startswith("---"):
|
|
return None, text
|
|
end = text.find("---", 3)
|
|
if end == -1:
|
|
return None, text
|
|
raw = text[3:end]
|
|
body = text[end + 3:].strip()
|
|
|
|
try:
|
|
import yaml
|
|
fm = yaml.safe_load(raw)
|
|
if not isinstance(fm, dict):
|
|
return None, body
|
|
for key, value in list(fm.items()):
|
|
if isinstance(value, date | datetime):
|
|
fm[key] = value.isoformat()
|
|
return fm, body
|
|
except ImportError:
|
|
pass
|
|
except Exception:
|
|
return None, body
|
|
|
|
# Fallback: simple key-value parser
|
|
fm = {}
|
|
for line in raw.strip().split("\n"):
|
|
line = line.strip()
|
|
if not line or line.startswith("#"):
|
|
continue
|
|
if ":" not in line:
|
|
continue
|
|
key, _, val = line.partition(":")
|
|
key = key.strip()
|
|
val = val.strip().strip('"').strip("'")
|
|
if val.lower() == "null" or val == "":
|
|
val = None
|
|
elif val.startswith("["):
|
|
val = [v.strip().strip('"').strip("'") for v in val.strip("[]").split(",") if v.strip()]
|
|
fm[key] = val
|
|
return fm if fm else None, body
|
|
|
|
|
|
# ─── Fixers (modify content, return fixed version) ─────────────────────────
|
|
|
|
|
|
def fix_frontmatter(content: str, domain: str, agent: str) -> tuple[str, list[str]]:
|
|
"""Fix common frontmatter issues. Returns (fixed_content, list_of_fixes_applied)."""
|
|
fixes = []
|
|
fm, body = parse_frontmatter(content)
|
|
if fm is None:
|
|
return content, ["unfixable:no_frontmatter"]
|
|
|
|
changed = False
|
|
ftype = fm.get("type", "claim")
|
|
|
|
# Fix 1: created = extraction date, always today. No parsing, no comparison.
|
|
# "created" means "when this was extracted," period. Source publication date
|
|
# belongs in a separate field if needed. (Ganymede review)
|
|
today_str = date.today().isoformat()
|
|
if ftype == "claim":
|
|
old_created = fm.get("created")
|
|
fm["created"] = today_str
|
|
if old_created != today_str:
|
|
fixes.append(f"set_created:{today_str}")
|
|
changed = True
|
|
|
|
# Fix 2: type field
|
|
if "type" not in fm:
|
|
fm["type"] = "claim"
|
|
fixes.append("added_type:claim")
|
|
changed = True
|
|
|
|
# Fix 3: domain field
|
|
if "domain" not in fm or fm["domain"] not in VALID_DOMAINS:
|
|
fm["domain"] = domain
|
|
fixes.append(f"fixed_domain:{fm.get('domain', 'missing')}->{domain}")
|
|
changed = True
|
|
|
|
# Fix 4: confidence field (claims only)
|
|
if ftype == "claim":
|
|
conf = fm.get("confidence")
|
|
if conf is None:
|
|
fm["confidence"] = "experimental"
|
|
fixes.append("added_confidence:experimental")
|
|
changed = True
|
|
elif conf not in VALID_CONFIDENCE:
|
|
fm["confidence"] = "experimental"
|
|
fixes.append(f"fixed_confidence:{conf}->experimental")
|
|
changed = True
|
|
|
|
# Fix 5: description field
|
|
if "description" not in fm or not fm["description"]:
|
|
# Try to derive from the first non-empty body line.
|
|
first_sentence = ""
|
|
for line in body.splitlines():
|
|
first_sentence = line.strip().lstrip("# ")
|
|
if first_sentence:
|
|
first_sentence = first_sentence.split(".")[0].strip()
|
|
break
|
|
if first_sentence and len(first_sentence) > 10:
|
|
fm["description"] = first_sentence[:200]
|
|
fixes.append("derived_description_from_body")
|
|
changed = True
|
|
|
|
# Fix 6: source field (claims only)
|
|
if ftype == "claim" and ("source" not in fm or not fm["source"]):
|
|
fm["source"] = f"extraction by {agent}"
|
|
fixes.append("added_default_source")
|
|
changed = True
|
|
|
|
if not changed:
|
|
return content, []
|
|
|
|
# Reconstruct frontmatter
|
|
return _rebuild_content(fm, body), fixes
|
|
|
|
|
|
def fix_wiki_links(content: str, existing_claims: set[str]) -> tuple[str, list[str]]:
|
|
"""Fix or strip broken wiki links. Resolves slug→space mismatches before stripping.
|
|
|
|
The LLM often generates wiki links as slugs (hyphens) but KB filenames use spaces.
|
|
Try normalizing hyphens→spaces before giving up and stripping brackets.
|
|
"""
|
|
fixes = []
|
|
# Build a lookup: normalized (lowercased, hyphens→spaces) → original stem
|
|
_normalized_lookup: dict[str, str] = {}
|
|
for stem in existing_claims:
|
|
_normalized_lookup[stem.lower().replace("-", " ")] = stem
|
|
|
|
def replace_broken(match):
|
|
link = match.group(1).strip()
|
|
if link in existing_claims:
|
|
return match.group(0) # Exact match — keep as-is
|
|
# Try normalizing slug to spaces
|
|
normalized = link.lower().replace("-", " ")
|
|
if normalized in _normalized_lookup:
|
|
resolved = _normalized_lookup[normalized]
|
|
fixes.append(f"resolved_wiki_link:{link[:40]}->{resolved[:40]}")
|
|
return f"[[{resolved}]]"
|
|
fixes.append(f"stripped_wiki_link:{link[:60]}")
|
|
return link # Keep text, remove brackets
|
|
|
|
fixed = WIKI_LINK_RE.sub(replace_broken, content)
|
|
return fixed, fixes
|
|
|
|
|
|
def fix_trailing_newline(content: str) -> tuple[str, list[str]]:
|
|
"""Ensure file ends with exactly one newline."""
|
|
if not content.endswith("\n"):
|
|
return content + "\n", ["added_trailing_newline"]
|
|
return content, []
|
|
|
|
|
|
def fix_h1_title_match(content: str, filename: str) -> tuple[str, list[str]]:
|
|
"""Ensure the content has an H1 title. Does NOT replace existing H1s.
|
|
|
|
The H1 title in the content is authoritative — the filename is derived from it
|
|
and may be truncated or slightly different. We only add a missing H1, never
|
|
overwrite an existing one.
|
|
"""
|
|
expected_title = Path(filename).stem.replace("-", " ")
|
|
fm, body = parse_frontmatter(content)
|
|
if fm is None:
|
|
return content, []
|
|
|
|
# Find existing H1
|
|
h1_match = re.search(r"^# (.+)$", body, re.MULTILINE)
|
|
if h1_match:
|
|
# H1 exists — leave it alone. The content's H1 is authoritative.
|
|
return content, []
|
|
elif body and not body.startswith("#"):
|
|
# No H1 at all — add one derived from filename
|
|
body = f"# {expected_title}\n\n{body}"
|
|
return _rebuild_content(fm, body), ["added_h1_title"]
|
|
|
|
return content, []
|
|
|
|
|
|
# ─── Validators (check without modifying, return issues) ──────────────────
|
|
|
|
|
|
def validate_claim(filename: str, content: str, existing_claims: set[str], agent: str | None = None) -> list[str]:
|
|
"""Validate a claim file. Returns list of issues (empty = pass)."""
|
|
issues = []
|
|
fm, body = parse_frontmatter(content)
|
|
|
|
if fm is None:
|
|
return ["no_frontmatter"]
|
|
|
|
ftype = fm.get("type", "claim")
|
|
|
|
# Schema check
|
|
required = REQUIRED_CLAIM_FIELDS if ftype == "claim" else REQUIRED_ENTITY_FIELDS
|
|
for field in required:
|
|
if field not in fm or fm[field] is None:
|
|
issues.append(f"missing_field:{field}")
|
|
|
|
# Domain check
|
|
domain = fm.get("domain")
|
|
if domain and domain not in VALID_DOMAINS:
|
|
issues.append(f"invalid_domain:{domain}")
|
|
|
|
# Confidence check (claims only)
|
|
if ftype == "claim":
|
|
conf = fm.get("confidence")
|
|
if conf and conf not in VALID_CONFIDENCE:
|
|
issues.append(f"invalid_confidence:{conf}")
|
|
|
|
# Title checks (claims only, not entities)
|
|
# Use H1 from body if available (authoritative), fall back to filename
|
|
if ftype in ("claim", "framework"):
|
|
h1_match = re.search(r"^# (.+)$", body, re.MULTILINE)
|
|
title = h1_match.group(1).strip() if h1_match else Path(filename).stem.replace("-", " ")
|
|
words = title.split()
|
|
# Always enforce minimum 4 words — a 2-3 word title is never specific
|
|
# enough to disagree with. (Ganymede review)
|
|
if len(words) < 4:
|
|
issues.append("title_too_few_words")
|
|
elif len(words) < 8:
|
|
# For 4-7 word titles, also require a verb/connective
|
|
has_verb = bool(re.search(
|
|
r"\b(is|are|was|were|will|would|can|could|should|must|has|have|had|"
|
|
r"does|did|do|may|might|shall|"
|
|
r"because|therefore|however|although|despite|since|through|by|"
|
|
r"when|where|while|if|unless|"
|
|
r"rather than|instead of|not just|more than|"
|
|
r"\w+(?:s|ed|ing|es|tes|ses|zes|ves|cts|pts|nts|rns))\b",
|
|
title, re.IGNORECASE,
|
|
))
|
|
if not has_verb:
|
|
issues.append("title_not_proposition")
|
|
|
|
# Description quality
|
|
desc = fm.get("description", "")
|
|
if isinstance(desc, str) and len(desc.strip()) < 10:
|
|
issues.append("description_too_short")
|
|
|
|
# Attribution check: extractor must be identified. (Leo: block extractor, warn sourcer)
|
|
if ftype == "claim":
|
|
from .attribution import validate_attribution
|
|
issues.extend(validate_attribution(fm, agent=agent))
|
|
|
|
# OPSEC check: flag claims containing dollar amounts + internal entity references.
|
|
# Rio's rule: never extract LivingIP/Teleo deal terms to public codex. (Ganymede review)
|
|
if ftype == "claim":
|
|
combined_text = (title + " " + desc + " " + body).lower()
|
|
has_dollar = bool(re.search(r"\$[\d,.]+[mkb]?\b", combined_text, re.IGNORECASE))
|
|
has_internal = bool(re.search(
|
|
r"\b(livingip|teleo|internal|deal terms?|valuation|equity percent)",
|
|
combined_text, re.IGNORECASE,
|
|
))
|
|
if has_dollar and has_internal:
|
|
issues.append("opsec_internal_deal_terms")
|
|
|
|
# Body substance check (claims only)
|
|
if ftype == "claim" and body:
|
|
# Strip the H1 title line and check remaining content
|
|
body_no_h1 = re.sub(r"^# .+\n*", "", body).strip()
|
|
# Remove "Relevant Notes" and "Topics" sections
|
|
body_content = re.split(r"\n---\n", body_no_h1)[0].strip()
|
|
if len(body_content) < 50:
|
|
issues.append("body_too_thin")
|
|
|
|
# Near-duplicate check (claims only, not entities)
|
|
if ftype != "entity":
|
|
title_lower = Path(filename).stem.replace("-", " ").lower()
|
|
title_words = set(title_lower.split()[:6])
|
|
for existing in existing_claims:
|
|
# Normalize existing stem: hyphens → spaces for consistent comparison
|
|
existing_normalized = existing.replace("-", " ").lower()
|
|
if len(title_words & set(existing_normalized.split()[:6])) < 2:
|
|
continue
|
|
ratio = SequenceMatcher(None, title_lower, existing_normalized).ratio()
|
|
if ratio >= DEDUP_THRESHOLD:
|
|
issues.append(f"near_duplicate:{existing[:80]}")
|
|
break # One is enough to flag
|
|
|
|
return issues
|
|
|
|
|
|
# ─── Main entry point ──────────────────────────────────────────────────────
|
|
|
|
|
|
def validate_and_fix_claims(
|
|
claims: list[dict],
|
|
domain: str,
|
|
agent: str,
|
|
existing_claims: set[str],
|
|
repo_root: str = ".",
|
|
) -> tuple[list[dict], list[dict], dict]:
|
|
"""Validate and fix extracted claims. Returns (kept_claims, rejected_claims, stats).
|
|
|
|
Each claim dict has: filename, domain, content
|
|
Returned claims have content fixed where possible.
|
|
|
|
Stats: {total, kept, fixed, rejected, fixes_applied: [...], rejections: [...]}
|
|
"""
|
|
kept = []
|
|
rejected = []
|
|
all_fixes = []
|
|
all_rejections = []
|
|
|
|
# Add intra-batch stems to existing claims (avoid false positive duplicates within same extraction)
|
|
batch_stems = {Path(c["filename"]).stem for c in claims}
|
|
existing_plus_batch = existing_claims | batch_stems
|
|
|
|
for claim in claims:
|
|
filename = claim.get("filename", "")
|
|
content = claim.get("content", "")
|
|
claim_domain = claim.get("domain", domain)
|
|
|
|
if not filename or not content:
|
|
rejected.append(claim)
|
|
all_rejections.append(f"{filename or '?'}:missing_filename_or_content")
|
|
continue
|
|
|
|
# Phase 1: Apply fixers
|
|
content, fixes1 = fix_frontmatter(content, claim_domain, agent)
|
|
content, fixes2 = fix_wiki_links(content, existing_plus_batch)
|
|
content, fixes3 = fix_trailing_newline(content)
|
|
content, fixes4 = fix_h1_title_match(content, filename)
|
|
|
|
fixes = fixes1 + fixes2 + fixes3 + fixes4
|
|
if fixes:
|
|
all_fixes.extend([f"{filename}:{f}" for f in fixes])
|
|
|
|
# Phase 2: Validate (after fixes)
|
|
issues = validate_claim(filename, content, existing_claims, agent=agent)
|
|
|
|
# Separate hard failures from warnings
|
|
hard_failures = [i for i in issues if not i.startswith("near_duplicate")]
|
|
warnings = [i for i in issues if i.startswith("near_duplicate")]
|
|
|
|
if hard_failures:
|
|
rejected.append({**claim, "content": content, "issues": hard_failures})
|
|
all_rejections.extend([f"{filename}:{i}" for i in hard_failures])
|
|
else:
|
|
if warnings:
|
|
all_fixes.extend([f"{filename}:WARN:{w}" for w in warnings])
|
|
kept.append({**claim, "content": content})
|
|
|
|
stats = {
|
|
"total": len(claims),
|
|
"kept": len(kept),
|
|
"fixed": len([f for f in all_fixes if ":WARN:" not in f]),
|
|
"rejected": len(rejected),
|
|
"fixes_applied": all_fixes,
|
|
"rejections": all_rejections,
|
|
}
|
|
|
|
logger.info(
|
|
"Post-extraction: %d/%d claims kept (%d fixed, %d rejected)",
|
|
stats["kept"], stats["total"], stats["fixed"], stats["rejected"],
|
|
)
|
|
|
|
return kept, rejected, stats
|
|
|
|
|
|
def validate_and_fix_entities(
|
|
entities: list[dict],
|
|
domain: str,
|
|
existing_claims: set[str],
|
|
) -> tuple[list[dict], list[dict], dict]:
|
|
"""Validate and fix extracted entities. Returns (kept, rejected, stats).
|
|
|
|
Lighter validation than claims — entities are factual records, not arguable propositions.
|
|
"""
|
|
kept = []
|
|
rejected = []
|
|
all_issues = []
|
|
|
|
for ent in entities:
|
|
filename = ent.get("filename", "")
|
|
content = ent.get("content", "")
|
|
action = ent.get("action", "create")
|
|
|
|
if not filename:
|
|
rejected.append(ent)
|
|
all_issues.append("missing_filename")
|
|
continue
|
|
|
|
issues = []
|
|
|
|
if action == "create" and content:
|
|
fm, _body = parse_frontmatter(content)
|
|
if fm is None:
|
|
issues.append("no_frontmatter")
|
|
else:
|
|
if fm.get("type") != "entity":
|
|
issues.append("wrong_type")
|
|
if "entity_type" not in fm:
|
|
issues.append("missing_entity_type")
|
|
if "domain" not in fm:
|
|
issues.append("missing_domain")
|
|
|
|
# decision_market specific checks
|
|
if fm.get("entity_type") == "decision_market":
|
|
for field in ("parent_entity", "platform", "category", "status"):
|
|
if field not in fm:
|
|
issues.append(f"dm_missing:{field}")
|
|
|
|
# Fix trailing newline
|
|
if content and not content.endswith("\n"):
|
|
ent["content"] = content + "\n"
|
|
|
|
elif action == "update":
|
|
timeline = ent.get("timeline_entry", "")
|
|
if not timeline:
|
|
issues.append("update_no_timeline")
|
|
|
|
if issues:
|
|
rejected.append({**ent, "issues": issues})
|
|
all_issues.extend([f"{filename}:{i}" for i in issues])
|
|
else:
|
|
kept.append(ent)
|
|
|
|
stats = {
|
|
"total": len(entities),
|
|
"kept": len(kept),
|
|
"rejected": len(rejected),
|
|
"issues": all_issues,
|
|
}
|
|
|
|
return kept, rejected, stats
|
|
|
|
|
|
def load_existing_claims_from_repo(repo_root: str) -> set[str]:
|
|
"""Build set of known claim/entity stems from the repo."""
|
|
claims: set[str] = set()
|
|
base = Path(repo_root)
|
|
for subdir in ["domains", "core", "foundations", "maps", "agents", "schemas", "entities"]:
|
|
full = base / subdir
|
|
if not full.is_dir():
|
|
continue
|
|
for f in full.rglob("*.md"):
|
|
claims.add(f.stem)
|
|
return claims
|
|
|
|
|
|
# ─── Helpers ────────────────────────────────────────────────────────────────
|
|
|
|
|
|
def _rebuild_content(fm: dict, body: str) -> str:
|
|
"""Rebuild markdown content from frontmatter dict and body."""
|
|
# Order frontmatter fields consistently
|
|
field_order = ["type", "entity_type", "name", "domain", "description",
|
|
"confidence", "source", "created", "status", "parent_entity",
|
|
"platform", "proposer", "proposal_url", "proposal_date",
|
|
"resolution_date", "category", "summary", "tracked_by",
|
|
"secondary_domains", "challenged_by"]
|
|
|
|
lines = ["---"]
|
|
written = set()
|
|
for field in field_order:
|
|
if field in fm and fm[field] is not None:
|
|
lines.append(_yaml_line(field, fm[field]))
|
|
written.add(field)
|
|
# Write remaining fields not in the order list
|
|
for key, val in fm.items():
|
|
if key not in written and val is not None:
|
|
lines.append(_yaml_line(key, val))
|
|
lines.append("---")
|
|
lines.append("")
|
|
lines.append(body)
|
|
|
|
content = "\n".join(lines)
|
|
if not content.endswith("\n"):
|
|
content += "\n"
|
|
return content
|
|
|
|
|
|
def _yaml_line(key: str, val) -> str:
|
|
"""Format a single YAML key-value line."""
|
|
if isinstance(val, dict):
|
|
# Nested YAML block (e.g. attribution with sub-keys)
|
|
lines = [f"{key}:"]
|
|
for sub_key, sub_val in val.items():
|
|
if isinstance(sub_val, list) and sub_val:
|
|
lines.append(f" {sub_key}:")
|
|
for item in sub_val:
|
|
if isinstance(item, dict):
|
|
first = True
|
|
for ik, iv in item.items():
|
|
prefix = " - " if first else " "
|
|
lines.append(f'{prefix}{ik}: "{iv}"')
|
|
first = False
|
|
else:
|
|
lines.append(f' - "{item}"')
|
|
else:
|
|
lines.append(f" {sub_key}: []")
|
|
return "\n".join(lines)
|
|
if isinstance(val, list):
|
|
return f"{key}: {json.dumps(val)}"
|
|
if isinstance(val, bool):
|
|
return f"{key}: {'true' if val else 'false'}"
|
|
if isinstance(val, (int, float)):
|
|
return f"{key}: {val}"
|
|
if isinstance(val, date):
|
|
return f"{key}: {val.isoformat()}"
|
|
# String — quote if it contains special chars
|
|
s = str(val)
|
|
if any(c in s for c in ":#{}[]|>&*!%@`"):
|
|
return f'{key}: "{s}"'
|
|
return f"{key}: {s}"
|