feat: atomic extract-and-connect + stale PR monitor + response audit
Some checks failed
CI / lint-and-test (pull_request) Has been cancelled
Some checks failed
CI / lint-and-test (pull_request) Has been cancelled
Atomic extract-and-connect (lib/connect.py): - After extraction writes claim files, each new claim is embedded via OpenRouter, searched against Qdrant, and top-5 neighbors (cosine > 0.55) are added as `related` edges in the claim's frontmatter - Edges written on NEW claim only — avoids merge conflicts - Cross-domain connections enabled, non-fatal on Qdrant failure - Wired into openrouter-extract-v2.py post-extraction step Stale PR monitor (lib/stale_pr.py): - Every watchdog cycle checks open extract/* PRs - If open >30 min AND 0 claim files → auto-close with comment - After 2 stale closures → marks source as extraction_failed - Wired into watchdog.py as check #6 Response audit system: - response_audit table (migration v8), persistent audit conn in bot.py - 90-day retention cleanup, tool_calls JSON column - Confidence tag stripping, systemd ReadWritePaths for pipeline.db Supporting infrastructure: - reweave.py: nightly edge reconnection for orphan claims - reconcile-sources.py: source status reconciliation - backfill-domains.py: domain classification backfill - ops/reconcile-source-status.sh: operational reconciliation script - Attribution improvements, post-extract enrichments, merge improvements Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
0457c49094
commit
5f554bc2de
17 changed files with 2784 additions and 50 deletions
193
backfill-domains.py
Normal file
193
backfill-domains.py
Normal file
|
|
@ -0,0 +1,193 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
# ONE-SHOT BACKFILL — do not cron. Idempotent.
|
||||||
|
"""Reclassify PRs with domain='general' or NULL using file paths from diffs.
|
||||||
|
|
||||||
|
The extraction prompt defaults to 'general' when it can't determine domain.
|
||||||
|
This script re-derives domains from actual file paths in merged PR diffs,
|
||||||
|
which are more reliable than extraction-time heuristics.
|
||||||
|
|
||||||
|
Usage:
|
||||||
|
python3 backfill-domains.py [--dry-run]
|
||||||
|
|
||||||
|
Pentagon-Agent: Epimetheus <0144398E-4ED3-4FE2-95A3-3D72E1ABF887>
|
||||||
|
"""
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import json
|
||||||
|
import re
|
||||||
|
import sqlite3
|
||||||
|
import subprocess
|
||||||
|
from collections import Counter
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
DB_PATH = "/opt/teleo-eval/pipeline/pipeline.db"
|
||||||
|
REPO_DIR = "/opt/teleo-eval/workspaces/main"
|
||||||
|
|
||||||
|
# Canonical domains — must match lib/domains.py DOMAIN_AGENT_MAP
|
||||||
|
VALID_DOMAINS = frozenset({
|
||||||
|
"internet-finance", "entertainment", "health", "ai-alignment",
|
||||||
|
"space-development", "mechanisms", "living-capital", "living-agents",
|
||||||
|
"teleohumanity", "grand-strategy", "critical-systems",
|
||||||
|
"collective-intelligence", "teleological-economics", "cultural-dynamics",
|
||||||
|
})
|
||||||
|
|
||||||
|
# Agent → primary domain (same as lib/domains.py)
|
||||||
|
AGENT_PRIMARY_DOMAIN = {
|
||||||
|
"rio": "internet-finance",
|
||||||
|
"clay": "entertainment",
|
||||||
|
"theseus": "ai-alignment",
|
||||||
|
"vida": "health",
|
||||||
|
"astra": "space-development",
|
||||||
|
"leo": "grand-strategy",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def detect_domain_from_paths(file_paths: list[str]) -> str | None:
|
||||||
|
"""Detect domain from file paths in a diff.
|
||||||
|
|
||||||
|
Checks domains/, entities/, core/, foundations/ directory structure.
|
||||||
|
Returns the most frequently referenced valid domain, or None.
|
||||||
|
"""
|
||||||
|
domain_counts: Counter = Counter()
|
||||||
|
for path in file_paths:
|
||||||
|
for prefix in ("domains/", "entities/"):
|
||||||
|
if path.startswith(prefix):
|
||||||
|
parts = path.split("/")
|
||||||
|
if len(parts) >= 2:
|
||||||
|
d = parts[1]
|
||||||
|
if d in VALID_DOMAINS:
|
||||||
|
domain_counts[d] += 1
|
||||||
|
break
|
||||||
|
else:
|
||||||
|
for prefix in ("core/", "foundations/"):
|
||||||
|
if path.startswith(prefix):
|
||||||
|
parts = path.split("/")
|
||||||
|
if len(parts) >= 2:
|
||||||
|
d = parts[1]
|
||||||
|
if d in VALID_DOMAINS:
|
||||||
|
domain_counts[d] += 1
|
||||||
|
break
|
||||||
|
|
||||||
|
if domain_counts:
|
||||||
|
return domain_counts.most_common(1)[0][0]
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def get_diff_files(pr_number: int, branch: str) -> list[str]:
|
||||||
|
"""Get list of changed file paths for a PR from git."""
|
||||||
|
try:
|
||||||
|
result = subprocess.run(
|
||||||
|
["git", "diff", "--name-only", f"origin/main...origin/{branch}"],
|
||||||
|
capture_output=True, text=True, timeout=10,
|
||||||
|
cwd=REPO_DIR,
|
||||||
|
)
|
||||||
|
if result.returncode == 0:
|
||||||
|
return [f.strip() for f in result.stdout.strip().split("\n") if f.strip()]
|
||||||
|
except (subprocess.TimeoutExpired, FileNotFoundError):
|
||||||
|
pass
|
||||||
|
|
||||||
|
# Fallback: try merge commit if branch is gone
|
||||||
|
try:
|
||||||
|
result = subprocess.run(
|
||||||
|
["git", "log", "--merges", f"--grep=#{pr_number}", "--format=%H", "-1"],
|
||||||
|
capture_output=True, text=True, timeout=10,
|
||||||
|
cwd=REPO_DIR,
|
||||||
|
)
|
||||||
|
if result.returncode == 0 and result.stdout.strip():
|
||||||
|
merge_sha = result.stdout.strip()
|
||||||
|
result2 = subprocess.run(
|
||||||
|
["git", "diff", "--name-only", f"{merge_sha}~1..{merge_sha}"],
|
||||||
|
capture_output=True, text=True, timeout=10,
|
||||||
|
cwd=REPO_DIR,
|
||||||
|
)
|
||||||
|
if result2.returncode == 0:
|
||||||
|
return [f.strip() for f in result2.stdout.strip().split("\n") if f.strip()]
|
||||||
|
except (subprocess.TimeoutExpired, FileNotFoundError):
|
||||||
|
pass
|
||||||
|
|
||||||
|
return []
|
||||||
|
|
||||||
|
|
||||||
|
def detect_domain_from_agent(agent: str | None) -> str | None:
|
||||||
|
"""Infer domain from agent's primary domain."""
|
||||||
|
if agent:
|
||||||
|
return AGENT_PRIMARY_DOMAIN.get(agent.lower())
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
parser = argparse.ArgumentParser(description="Backfill domain for 'general'/NULL PRs")
|
||||||
|
parser.add_argument("--dry-run", action="store_true", help="Print changes without applying")
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
conn = sqlite3.connect(DB_PATH)
|
||||||
|
conn.row_factory = sqlite3.Row
|
||||||
|
|
||||||
|
# Find PRs with missing or 'general' domain
|
||||||
|
rows = conn.execute(
|
||||||
|
"""SELECT number, branch, domain, agent FROM prs
|
||||||
|
WHERE status = 'merged'
|
||||||
|
AND (domain IS NULL OR domain = 'general')
|
||||||
|
ORDER BY number"""
|
||||||
|
).fetchall()
|
||||||
|
|
||||||
|
print(f"Found {len(rows)} merged PRs with domain=NULL or 'general'")
|
||||||
|
|
||||||
|
reclassified = 0
|
||||||
|
unchanged = 0
|
||||||
|
distribution: Counter = Counter()
|
||||||
|
log_entries = []
|
||||||
|
|
||||||
|
for row in rows:
|
||||||
|
pr_num = row["number"]
|
||||||
|
branch = row["branch"]
|
||||||
|
old_domain = row["domain"] or "NULL"
|
||||||
|
agent = row["agent"]
|
||||||
|
|
||||||
|
new_domain = None
|
||||||
|
|
||||||
|
# Strategy 1: File paths from diff
|
||||||
|
if branch:
|
||||||
|
files = get_diff_files(pr_num, branch)
|
||||||
|
new_domain = detect_domain_from_paths(files)
|
||||||
|
|
||||||
|
# Strategy 2: Agent's primary domain
|
||||||
|
if new_domain is None:
|
||||||
|
new_domain = detect_domain_from_agent(agent)
|
||||||
|
|
||||||
|
if new_domain and new_domain != old_domain:
|
||||||
|
log_entries.append(f"PR #{pr_num}: {old_domain} → {new_domain} (agent={agent}, branch={branch})")
|
||||||
|
distribution[new_domain] += 1
|
||||||
|
|
||||||
|
if not args.dry_run:
|
||||||
|
conn.execute(
|
||||||
|
"UPDATE prs SET domain = ? WHERE number = ?",
|
||||||
|
(new_domain, pr_num),
|
||||||
|
)
|
||||||
|
reclassified += 1
|
||||||
|
else:
|
||||||
|
unchanged += 1
|
||||||
|
|
||||||
|
if not args.dry_run and reclassified > 0:
|
||||||
|
conn.commit()
|
||||||
|
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
# Report
|
||||||
|
print(f"\nReclassified: {reclassified}")
|
||||||
|
print(f"Unchanged (still general): {unchanged}")
|
||||||
|
print(f"\nDistribution of reclassified PRs:")
|
||||||
|
for domain, count in distribution.most_common():
|
||||||
|
print(f" {domain}: {count}")
|
||||||
|
|
||||||
|
if log_entries:
|
||||||
|
print(f"\nDetailed log ({len(log_entries)} changes):")
|
||||||
|
for entry in log_entries:
|
||||||
|
print(f" {entry}")
|
||||||
|
|
||||||
|
if args.dry_run:
|
||||||
|
print("\n[DRY RUN — no changes applied]")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
|
|
@ -148,8 +148,8 @@ def embed_file(path: Path, api_key: str, dry_run: bool = False) -> bool:
|
||||||
file_type, domain, confidence, title = classify_file(fm, path)
|
file_type, domain, confidence, title = classify_file(fm, path)
|
||||||
rel_path = str(path.relative_to(REPO_DIR))
|
rel_path = str(path.relative_to(REPO_DIR))
|
||||||
|
|
||||||
# Build embed text: title + first ~2000 chars of body
|
# Build embed text: title + first ~6000 chars of body (model handles 8191 tokens)
|
||||||
embed_text_str = f"{title}\n\n{body[:2000]}" if body else title
|
embed_text_str = f"{title}\n\n{body[:6000]}" if body else title
|
||||||
|
|
||||||
if dry_run:
|
if dry_run:
|
||||||
print(f" [{file_type}] {rel_path}: {title[:60]}")
|
print(f" [{file_type}] {rel_path}: {title[:60]}")
|
||||||
|
|
|
||||||
|
|
@ -100,12 +100,15 @@ def parse_attribution_from_file(filepath: str) -> dict[str, list[dict]]:
|
||||||
# ─── Validate attribution ──────────────────────────────────────────────────
|
# ─── Validate attribution ──────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
def validate_attribution(fm: dict) -> list[str]:
|
def validate_attribution(fm: dict, agent: str | None = None) -> list[str]:
|
||||||
"""Validate attribution block in claim frontmatter.
|
"""Validate attribution block in claim frontmatter.
|
||||||
|
|
||||||
Returns list of issues. Block on missing extractor, warn on missing sourcer.
|
Returns list of issues. Block on missing extractor, warn on missing sourcer.
|
||||||
(Leo: extractor is always known, sourcer is best-effort.)
|
(Leo: extractor is always known, sourcer is best-effort.)
|
||||||
|
|
||||||
|
If agent is provided and extractor is missing, auto-fix by setting the
|
||||||
|
agent as extractor (same pattern as created-date auto-fix).
|
||||||
|
|
||||||
Only validates if an attribution block is explicitly present. Legacy claims
|
Only validates if an attribution block is explicitly present. Legacy claims
|
||||||
without attribution blocks are not blocked — they'll get attribution when
|
without attribution blocks are not blocked — they'll get attribution when
|
||||||
enriched. New claims from v2 extraction always have attribution.
|
enriched. New claims from v2 extraction always have attribution.
|
||||||
|
|
@ -123,7 +126,16 @@ def validate_attribution(fm: dict) -> list[str]:
|
||||||
attribution = parse_attribution(fm)
|
attribution = parse_attribution(fm)
|
||||||
|
|
||||||
if not attribution["extractor"]:
|
if not attribution["extractor"]:
|
||||||
issues.append("missing_attribution_extractor")
|
if agent:
|
||||||
|
# Auto-fix: set the processing agent as extractor
|
||||||
|
attr = fm.get("attribution")
|
||||||
|
if isinstance(attr, dict):
|
||||||
|
attr["extractor"] = [{"handle": agent}]
|
||||||
|
else:
|
||||||
|
fm["attribution"] = {"extractor": [{"handle": agent}]}
|
||||||
|
issues.append("fixed_missing_extractor")
|
||||||
|
else:
|
||||||
|
issues.append("missing_attribution_extractor")
|
||||||
|
|
||||||
return issues
|
return issues
|
||||||
|
|
||||||
|
|
|
||||||
202
lib/connect.py
Normal file
202
lib/connect.py
Normal file
|
|
@ -0,0 +1,202 @@
|
||||||
|
"""Atomic extract-and-connect — wire new claims to the KB at extraction time.
|
||||||
|
|
||||||
|
After extraction writes claim files to disk, this module:
|
||||||
|
1. Embeds each new claim (title + description + body snippet)
|
||||||
|
2. Searches Qdrant for semantically similar existing claims
|
||||||
|
3. Adds found neighbors as `related` edges on the NEW claim's frontmatter
|
||||||
|
|
||||||
|
Key design decision: edges are written on the NEW claim, not on existing claims.
|
||||||
|
Writing on existing claims would cause merge conflicts (same reason entities are
|
||||||
|
queued, not written on branches). When the PR merges, embed-on-merge adds the
|
||||||
|
new claim to Qdrant, and reweave can later add reciprocal edges on neighbors.
|
||||||
|
|
||||||
|
Cost: ~$0.0001 per claim (embedding only). No LLM classification — defaults to
|
||||||
|
"related". Reweave handles supports/challenges classification in a separate pass.
|
||||||
|
|
||||||
|
Owner: Epimetheus
|
||||||
|
"""
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
logger = logging.getLogger("pipeline.connect")
|
||||||
|
|
||||||
|
# Similarity threshold for auto-connecting (lower than reweave's 0.70 because
|
||||||
|
# we're using "related" not "supports/challenges" — less precision needed)
|
||||||
|
CONNECT_THRESHOLD = 0.55
|
||||||
|
CONNECT_MAX_NEIGHBORS = 5
|
||||||
|
|
||||||
|
# --- Import search functions ---
|
||||||
|
# This module is called from openrouter-extract-v2.py which may not have lib/ on path
|
||||||
|
# via the package, so handle both import paths.
|
||||||
|
try:
|
||||||
|
from .search import embed_query, search_qdrant
|
||||||
|
from .post_extract import parse_frontmatter, _rebuild_content
|
||||||
|
except ImportError:
|
||||||
|
sys.path.insert(0, os.path.dirname(__file__))
|
||||||
|
from search import embed_query, search_qdrant
|
||||||
|
from post_extract import parse_frontmatter, _rebuild_content
|
||||||
|
|
||||||
|
|
||||||
|
def _build_search_text(content: str) -> str:
|
||||||
|
"""Extract title + description + first 500 chars of body for embedding."""
|
||||||
|
fm, body = parse_frontmatter(content)
|
||||||
|
parts = []
|
||||||
|
if fm:
|
||||||
|
desc = fm.get("description", "")
|
||||||
|
if isinstance(desc, str) and desc:
|
||||||
|
parts.append(desc.strip('"').strip("'"))
|
||||||
|
# Get H1 title from body
|
||||||
|
h1_match = re.search(r"^# (.+)$", body, re.MULTILINE) if body else None
|
||||||
|
if h1_match:
|
||||||
|
parts.append(h1_match.group(1).strip())
|
||||||
|
# Add body snippet (skip H1 line)
|
||||||
|
if body:
|
||||||
|
body_text = re.sub(r"^# .+\n*", "", body).strip()
|
||||||
|
# Stop at "Relevant Notes" or "Topics" sections
|
||||||
|
body_text = re.split(r"\n---\n", body_text)[0].strip()
|
||||||
|
if body_text:
|
||||||
|
parts.append(body_text[:500])
|
||||||
|
return " ".join(parts)
|
||||||
|
|
||||||
|
|
||||||
|
def _add_related_edges(claim_path: str, neighbor_titles: list[str]) -> bool:
|
||||||
|
"""Add related edges to a claim's frontmatter. Returns True if modified."""
|
||||||
|
try:
|
||||||
|
with open(claim_path) as f:
|
||||||
|
content = f.read()
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning("Cannot read %s: %s", claim_path, e)
|
||||||
|
return False
|
||||||
|
|
||||||
|
fm, body = parse_frontmatter(content)
|
||||||
|
if fm is None:
|
||||||
|
return False
|
||||||
|
|
||||||
|
# Get existing related edges to avoid duplicates
|
||||||
|
existing = fm.get("related", [])
|
||||||
|
if isinstance(existing, str):
|
||||||
|
existing = [existing]
|
||||||
|
elif not isinstance(existing, list):
|
||||||
|
existing = []
|
||||||
|
|
||||||
|
existing_lower = {str(e).strip().lower() for e in existing}
|
||||||
|
|
||||||
|
# Add new edges
|
||||||
|
added = []
|
||||||
|
for title in neighbor_titles:
|
||||||
|
if title.strip().lower() not in existing_lower:
|
||||||
|
added.append(title)
|
||||||
|
existing_lower.add(title.strip().lower())
|
||||||
|
|
||||||
|
if not added:
|
||||||
|
return False
|
||||||
|
|
||||||
|
fm["related"] = existing + added
|
||||||
|
|
||||||
|
# Rebuild and write
|
||||||
|
new_content = _rebuild_content(fm, body)
|
||||||
|
with open(claim_path, "w") as f:
|
||||||
|
f.write(new_content)
|
||||||
|
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def connect_new_claims(
|
||||||
|
claim_paths: list[str],
|
||||||
|
domain: str | None = None,
|
||||||
|
threshold: float = CONNECT_THRESHOLD,
|
||||||
|
max_neighbors: int = CONNECT_MAX_NEIGHBORS,
|
||||||
|
) -> dict:
|
||||||
|
"""Connect newly-written claims to the existing KB via vector search.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
claim_paths: List of file paths to newly-written claim files.
|
||||||
|
domain: Optional domain filter for Qdrant search.
|
||||||
|
threshold: Minimum cosine similarity for connection.
|
||||||
|
max_neighbors: Maximum edges to add per claim.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
{
|
||||||
|
"total": int,
|
||||||
|
"connected": int,
|
||||||
|
"edges_added": int,
|
||||||
|
"skipped_embed_failed": int,
|
||||||
|
"skipped_no_neighbors": int,
|
||||||
|
"connections": [{"claim": str, "neighbors": [str]}],
|
||||||
|
}
|
||||||
|
"""
|
||||||
|
stats = {
|
||||||
|
"total": len(claim_paths),
|
||||||
|
"connected": 0,
|
||||||
|
"edges_added": 0,
|
||||||
|
"skipped_embed_failed": 0,
|
||||||
|
"skipped_no_neighbors": 0,
|
||||||
|
"connections": [],
|
||||||
|
}
|
||||||
|
|
||||||
|
for claim_path in claim_paths:
|
||||||
|
try:
|
||||||
|
with open(claim_path) as f:
|
||||||
|
content = f.read()
|
||||||
|
except Exception:
|
||||||
|
continue
|
||||||
|
|
||||||
|
# Build search text from claim content
|
||||||
|
search_text = _build_search_text(content)
|
||||||
|
if not search_text or len(search_text) < 20:
|
||||||
|
stats["skipped_no_neighbors"] += 1
|
||||||
|
continue
|
||||||
|
|
||||||
|
# Embed the claim
|
||||||
|
vector = embed_query(search_text)
|
||||||
|
if vector is None:
|
||||||
|
stats["skipped_embed_failed"] += 1
|
||||||
|
continue
|
||||||
|
|
||||||
|
# Search Qdrant for neighbors (exclude nothing — new claim isn't in Qdrant yet)
|
||||||
|
hits = search_qdrant(
|
||||||
|
vector,
|
||||||
|
limit=max_neighbors,
|
||||||
|
domain=None, # Cross-domain connections are valuable
|
||||||
|
score_threshold=threshold,
|
||||||
|
)
|
||||||
|
|
||||||
|
if not hits:
|
||||||
|
stats["skipped_no_neighbors"] += 1
|
||||||
|
continue
|
||||||
|
|
||||||
|
# Extract neighbor titles
|
||||||
|
neighbor_titles = []
|
||||||
|
for hit in hits:
|
||||||
|
payload = hit.get("payload", {})
|
||||||
|
title = payload.get("claim_title", "")
|
||||||
|
if title:
|
||||||
|
neighbor_titles.append(title)
|
||||||
|
|
||||||
|
if not neighbor_titles:
|
||||||
|
stats["skipped_no_neighbors"] += 1
|
||||||
|
continue
|
||||||
|
|
||||||
|
# Add edges to the new claim's frontmatter
|
||||||
|
if _add_related_edges(claim_path, neighbor_titles):
|
||||||
|
stats["connected"] += 1
|
||||||
|
stats["edges_added"] += len(neighbor_titles)
|
||||||
|
stats["connections"].append({
|
||||||
|
"claim": os.path.basename(claim_path),
|
||||||
|
"neighbors": neighbor_titles,
|
||||||
|
})
|
||||||
|
logger.info("Connected %s → %d neighbors", os.path.basename(claim_path), len(neighbor_titles))
|
||||||
|
else:
|
||||||
|
stats["skipped_no_neighbors"] += 1
|
||||||
|
|
||||||
|
logger.info(
|
||||||
|
"Extract-and-connect: %d/%d claims connected (%d edges added, %d embed failed, %d no neighbors)",
|
||||||
|
stats["connected"], stats["total"], stats["edges_added"],
|
||||||
|
stats["skipped_embed_failed"], stats["skipped_no_neighbors"],
|
||||||
|
)
|
||||||
|
|
||||||
|
return stats
|
||||||
211
lib/db.py
211
lib/db.py
|
|
@ -9,7 +9,7 @@ from . import config
|
||||||
|
|
||||||
logger = logging.getLogger("pipeline.db")
|
logger = logging.getLogger("pipeline.db")
|
||||||
|
|
||||||
SCHEMA_VERSION = 6
|
SCHEMA_VERSION = 9
|
||||||
|
|
||||||
SCHEMA_SQL = """
|
SCHEMA_SQL = """
|
||||||
CREATE TABLE IF NOT EXISTS schema_version (
|
CREATE TABLE IF NOT EXISTS schema_version (
|
||||||
|
|
@ -48,6 +48,7 @@ CREATE TABLE IF NOT EXISTS prs (
|
||||||
-- conflict: rebase failed or merge timed out — needs human intervention
|
-- conflict: rebase failed or merge timed out — needs human intervention
|
||||||
domain TEXT,
|
domain TEXT,
|
||||||
agent TEXT,
|
agent TEXT,
|
||||||
|
commit_type TEXT CHECK(commit_type IS NULL OR commit_type IN ('extract', 'research', 'entity', 'decision', 'reweave', 'fix', 'challenge', 'enrich', 'synthesize', 'unknown')),
|
||||||
tier TEXT,
|
tier TEXT,
|
||||||
-- LIGHT, STANDARD, DEEP
|
-- LIGHT, STANDARD, DEEP
|
||||||
tier0_pass INTEGER,
|
tier0_pass INTEGER,
|
||||||
|
|
@ -103,11 +104,52 @@ CREATE TABLE IF NOT EXISTS audit_log (
|
||||||
detail TEXT
|
detail TEXT
|
||||||
);
|
);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS response_audit (
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
timestamp TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
chat_id INTEGER,
|
||||||
|
user TEXT,
|
||||||
|
agent TEXT DEFAULT 'rio',
|
||||||
|
model TEXT,
|
||||||
|
query TEXT,
|
||||||
|
conversation_window TEXT,
|
||||||
|
-- JSON: prior N messages for context
|
||||||
|
-- NOTE: intentional duplication of transcript data for audit self-containment.
|
||||||
|
-- Transcripts live in /opt/teleo-eval/transcripts/ but audit rows need prompt
|
||||||
|
-- context inline for retrieval-quality diagnosis. Primary driver of row size —
|
||||||
|
-- target for cleanup when 90-day retention policy lands.
|
||||||
|
entities_matched TEXT,
|
||||||
|
-- JSON: [{name, path, score, used_in_response}]
|
||||||
|
claims_matched TEXT,
|
||||||
|
-- JSON: [{path, title, score, source, used_in_response}]
|
||||||
|
retrieval_layers_hit TEXT,
|
||||||
|
-- JSON: ["keyword","qdrant","graph"]
|
||||||
|
retrieval_gap TEXT,
|
||||||
|
-- What the KB was missing (if anything)
|
||||||
|
market_data TEXT,
|
||||||
|
-- JSON: injected token prices
|
||||||
|
research_context TEXT,
|
||||||
|
-- Haiku pre-pass results if any
|
||||||
|
kb_context_text TEXT,
|
||||||
|
-- Full context string sent to model
|
||||||
|
tool_calls TEXT,
|
||||||
|
-- JSON: ordered array [{tool, input, output, duration_ms, ts}]
|
||||||
|
raw_response TEXT,
|
||||||
|
display_response TEXT,
|
||||||
|
confidence_score REAL,
|
||||||
|
-- Model self-rated retrieval quality 0.0-1.0
|
||||||
|
response_time_ms INTEGER,
|
||||||
|
created_at TEXT DEFAULT (datetime('now'))
|
||||||
|
);
|
||||||
|
|
||||||
CREATE INDEX IF NOT EXISTS idx_sources_status ON sources(status);
|
CREATE INDEX IF NOT EXISTS idx_sources_status ON sources(status);
|
||||||
CREATE INDEX IF NOT EXISTS idx_prs_status ON prs(status);
|
CREATE INDEX IF NOT EXISTS idx_prs_status ON prs(status);
|
||||||
CREATE INDEX IF NOT EXISTS idx_prs_domain ON prs(domain);
|
CREATE INDEX IF NOT EXISTS idx_prs_domain ON prs(domain);
|
||||||
CREATE INDEX IF NOT EXISTS idx_costs_date ON costs(date);
|
CREATE INDEX IF NOT EXISTS idx_costs_date ON costs(date);
|
||||||
CREATE INDEX IF NOT EXISTS idx_audit_stage ON audit_log(stage);
|
CREATE INDEX IF NOT EXISTS idx_audit_stage ON audit_log(stage);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_response_audit_ts ON response_audit(timestamp);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_response_audit_agent ON response_audit(agent);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_response_audit_chat_ts ON response_audit(chat_id, timestamp);
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -140,6 +182,37 @@ def transaction(conn: sqlite3.Connection):
|
||||||
raise
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
# Branch prefix → (agent, commit_type) mapping.
|
||||||
|
# Single source of truth — used by merge.py at INSERT time and migration v7 backfill.
|
||||||
|
# Unknown prefixes → ('unknown', 'unknown') + warning log.
|
||||||
|
BRANCH_PREFIX_MAP = {
|
||||||
|
"extract": ("pipeline", "extract"),
|
||||||
|
"ingestion": ("pipeline", "extract"),
|
||||||
|
"epimetheus": ("epimetheus", "extract"),
|
||||||
|
"rio": ("rio", "research"),
|
||||||
|
"theseus": ("theseus", "research"),
|
||||||
|
"astra": ("astra", "research"),
|
||||||
|
"vida": ("vida", "research"),
|
||||||
|
"clay": ("clay", "research"),
|
||||||
|
"leo": ("leo", "entity"),
|
||||||
|
"reweave": ("pipeline", "reweave"),
|
||||||
|
"fix": ("pipeline", "fix"),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def classify_branch(branch: str) -> tuple[str, str]:
|
||||||
|
"""Derive (agent, commit_type) from branch prefix.
|
||||||
|
|
||||||
|
Returns ('unknown', 'unknown') and logs a warning for unrecognized prefixes.
|
||||||
|
"""
|
||||||
|
prefix = branch.split("/", 1)[0] if "/" in branch else branch
|
||||||
|
result = BRANCH_PREFIX_MAP.get(prefix)
|
||||||
|
if result is None:
|
||||||
|
logger.warning("Unknown branch prefix %r in branch %r — defaulting to ('unknown', 'unknown')", prefix, branch)
|
||||||
|
return ("unknown", "unknown")
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
def migrate(conn: sqlite3.Connection):
|
def migrate(conn: sqlite3.Connection):
|
||||||
"""Run schema migrations."""
|
"""Run schema migrations."""
|
||||||
conn.executescript(SCHEMA_SQL)
|
conn.executescript(SCHEMA_SQL)
|
||||||
|
|
@ -251,6 +324,121 @@ def migrate(conn: sqlite3.Connection):
|
||||||
""")
|
""")
|
||||||
logger.info("Migration v6: added metrics_snapshots table for analytics dashboard")
|
logger.info("Migration v6: added metrics_snapshots table for analytics dashboard")
|
||||||
|
|
||||||
|
if current < 7:
|
||||||
|
# Phase 7: agent attribution + commit_type for dashboard
|
||||||
|
# commit_type column + backfill agent/commit_type from branch prefix
|
||||||
|
try:
|
||||||
|
conn.execute("ALTER TABLE prs ADD COLUMN commit_type TEXT CHECK(commit_type IS NULL OR commit_type IN ('extract', 'research', 'entity', 'decision', 'reweave', 'fix', 'unknown'))")
|
||||||
|
except sqlite3.OperationalError:
|
||||||
|
pass # column already exists from CREATE TABLE
|
||||||
|
# Backfill agent and commit_type from branch prefix
|
||||||
|
rows = conn.execute("SELECT number, branch FROM prs WHERE branch IS NOT NULL").fetchall()
|
||||||
|
for row in rows:
|
||||||
|
agent, commit_type = classify_branch(row["branch"])
|
||||||
|
conn.execute(
|
||||||
|
"UPDATE prs SET agent = ?, commit_type = ? WHERE number = ? AND (agent IS NULL OR commit_type IS NULL)",
|
||||||
|
(agent, commit_type, row["number"]),
|
||||||
|
)
|
||||||
|
backfilled = len(rows)
|
||||||
|
logger.info("Migration v7: added commit_type column, backfilled %d PRs with agent/commit_type", backfilled)
|
||||||
|
|
||||||
|
if current < 8:
|
||||||
|
# Phase 8: response audit — full-chain visibility for agent response quality
|
||||||
|
# Captures: query → tool calls → retrieval → context → response → confidence
|
||||||
|
# Approved by Ganymede (architecture), Rio (agent needs), Rhea (ops)
|
||||||
|
conn.executescript("""
|
||||||
|
CREATE TABLE IF NOT EXISTS response_audit (
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
timestamp TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
chat_id INTEGER,
|
||||||
|
user TEXT,
|
||||||
|
agent TEXT DEFAULT 'rio',
|
||||||
|
model TEXT,
|
||||||
|
query TEXT,
|
||||||
|
conversation_window TEXT, -- intentional transcript duplication for audit self-containment
|
||||||
|
entities_matched TEXT,
|
||||||
|
claims_matched TEXT,
|
||||||
|
retrieval_layers_hit TEXT,
|
||||||
|
retrieval_gap TEXT,
|
||||||
|
market_data TEXT,
|
||||||
|
research_context TEXT,
|
||||||
|
kb_context_text TEXT,
|
||||||
|
tool_calls TEXT,
|
||||||
|
raw_response TEXT,
|
||||||
|
display_response TEXT,
|
||||||
|
confidence_score REAL,
|
||||||
|
response_time_ms INTEGER,
|
||||||
|
created_at TEXT DEFAULT (datetime('now'))
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_response_audit_ts ON response_audit(timestamp);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_response_audit_agent ON response_audit(agent);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_response_audit_chat_ts ON response_audit(chat_id, timestamp);
|
||||||
|
""")
|
||||||
|
logger.info("Migration v8: added response_audit table for agent response auditing")
|
||||||
|
|
||||||
|
if current < 9:
|
||||||
|
# Phase 9: rebuild prs table to expand CHECK constraint on commit_type.
|
||||||
|
# SQLite cannot ALTER CHECK constraints in-place — must rebuild table.
|
||||||
|
# Old constraint (v7): extract,research,entity,decision,reweave,fix,unknown
|
||||||
|
# New constraint: adds challenge,enrich,synthesize
|
||||||
|
# Also re-derive commit_type from branch prefix for rows with invalid/NULL values.
|
||||||
|
|
||||||
|
# Step 1: Get all column names from existing table
|
||||||
|
cols_info = conn.execute("PRAGMA table_info(prs)").fetchall()
|
||||||
|
col_names = [c["name"] for c in cols_info]
|
||||||
|
col_list = ", ".join(col_names)
|
||||||
|
|
||||||
|
# Step 2: Create new table with expanded CHECK constraint
|
||||||
|
conn.executescript(f"""
|
||||||
|
CREATE TABLE prs_new (
|
||||||
|
number INTEGER PRIMARY KEY,
|
||||||
|
source_path TEXT REFERENCES sources(path),
|
||||||
|
branch TEXT,
|
||||||
|
status TEXT NOT NULL DEFAULT 'open',
|
||||||
|
domain TEXT,
|
||||||
|
agent TEXT,
|
||||||
|
commit_type TEXT CHECK(commit_type IS NULL OR commit_type IN ('extract','research','entity','decision','reweave','fix','challenge','enrich','synthesize','unknown')),
|
||||||
|
tier TEXT,
|
||||||
|
tier0_pass INTEGER,
|
||||||
|
leo_verdict TEXT DEFAULT 'pending',
|
||||||
|
domain_verdict TEXT DEFAULT 'pending',
|
||||||
|
domain_agent TEXT,
|
||||||
|
domain_model TEXT,
|
||||||
|
priority TEXT,
|
||||||
|
origin TEXT DEFAULT 'pipeline',
|
||||||
|
transient_retries INTEGER DEFAULT 0,
|
||||||
|
substantive_retries INTEGER DEFAULT 0,
|
||||||
|
last_error TEXT,
|
||||||
|
last_attempt TEXT,
|
||||||
|
cost_usd REAL DEFAULT 0,
|
||||||
|
created_at TEXT DEFAULT (datetime('now')),
|
||||||
|
merged_at TEXT
|
||||||
|
);
|
||||||
|
INSERT INTO prs_new ({col_list}) SELECT {col_list} FROM prs;
|
||||||
|
DROP TABLE prs;
|
||||||
|
ALTER TABLE prs_new RENAME TO prs;
|
||||||
|
""")
|
||||||
|
logger.info("Migration v9: rebuilt prs table with expanded commit_type CHECK constraint")
|
||||||
|
|
||||||
|
# Step 3: Re-derive commit_type from branch prefix for invalid/NULL values
|
||||||
|
rows = conn.execute(
|
||||||
|
"""SELECT number, branch FROM prs
|
||||||
|
WHERE branch IS NOT NULL
|
||||||
|
AND (commit_type IS NULL
|
||||||
|
OR commit_type NOT IN ('extract','research','entity','decision','reweave','fix','challenge','enrich','synthesize','unknown'))"""
|
||||||
|
).fetchall()
|
||||||
|
fixed = 0
|
||||||
|
for row in rows:
|
||||||
|
agent, commit_type = classify_branch(row["branch"])
|
||||||
|
conn.execute(
|
||||||
|
"UPDATE prs SET agent = COALESCE(agent, ?), commit_type = ? WHERE number = ?",
|
||||||
|
(agent, commit_type, row["number"]),
|
||||||
|
)
|
||||||
|
fixed += 1
|
||||||
|
conn.commit()
|
||||||
|
logger.info("Migration v9: re-derived commit_type for %d PRs with invalid/NULL values", fixed)
|
||||||
|
|
||||||
if current < SCHEMA_VERSION:
|
if current < SCHEMA_VERSION:
|
||||||
conn.execute(
|
conn.execute(
|
||||||
"INSERT OR REPLACE INTO schema_version (version) VALUES (?)",
|
"INSERT OR REPLACE INTO schema_version (version) VALUES (?)",
|
||||||
|
|
@ -296,6 +484,27 @@ def append_priority_log(conn: sqlite3.Connection, path: str, stage: str, priorit
|
||||||
raise
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
def insert_response_audit(conn: sqlite3.Connection, **kwargs):
|
||||||
|
"""Insert a response audit record. All fields optional except query."""
|
||||||
|
cols = [
|
||||||
|
"timestamp", "chat_id", "user", "agent", "model", "query",
|
||||||
|
"conversation_window", "entities_matched", "claims_matched",
|
||||||
|
"retrieval_layers_hit", "retrieval_gap", "market_data",
|
||||||
|
"research_context", "kb_context_text", "tool_calls",
|
||||||
|
"raw_response", "display_response", "confidence_score",
|
||||||
|
"response_time_ms",
|
||||||
|
]
|
||||||
|
present = {k: v for k, v in kwargs.items() if k in cols and v is not None}
|
||||||
|
if not present:
|
||||||
|
return
|
||||||
|
col_names = ", ".join(present.keys())
|
||||||
|
placeholders = ", ".join("?" for _ in present)
|
||||||
|
conn.execute(
|
||||||
|
f"INSERT INTO response_audit ({col_names}) VALUES ({placeholders})",
|
||||||
|
tuple(present.values()),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def set_priority(conn: sqlite3.Connection, path: str, priority: str, reason: str = "human override"):
|
def set_priority(conn: sqlite3.Connection, path: str, priority: str, reason: str = "human override"):
|
||||||
"""Set a source's authoritative priority. Used for human overrides and initial triage."""
|
"""Set a source's authoritative priority. Used for human overrides and initial triage."""
|
||||||
conn.execute(
|
conn.execute(
|
||||||
|
|
|
||||||
|
|
@ -25,7 +25,7 @@ import re
|
||||||
from datetime import datetime, timezone
|
from datetime import datetime, timezone
|
||||||
|
|
||||||
from . import config, db
|
from . import config, db
|
||||||
from .domains import agent_for_domain, detect_domain_from_diff
|
from .domains import agent_for_domain, detect_domain_from_branch, detect_domain_from_diff
|
||||||
from .forgejo import api as forgejo_api
|
from .forgejo import api as forgejo_api
|
||||||
from .forgejo import get_agent_token, get_pr_diff, repo_path
|
from .forgejo import get_agent_token, get_pr_diff, repo_path
|
||||||
from .llm import run_batch_domain_review, run_domain_review, run_leo_review, triage_pr
|
from .llm import run_batch_domain_review, run_domain_review, run_leo_review, triage_pr
|
||||||
|
|
@ -556,13 +556,15 @@ async def evaluate_pr(conn, pr_number: int, tier: str = None) -> dict:
|
||||||
review_diff = diff
|
review_diff = diff
|
||||||
files = _extract_changed_files(diff)
|
files = _extract_changed_files(diff)
|
||||||
|
|
||||||
# Detect domain
|
# Detect domain — try diff paths first, then branch prefix, then 'general'
|
||||||
domain = detect_domain_from_diff(diff)
|
domain = detect_domain_from_diff(diff)
|
||||||
agent = agent_for_domain(domain)
|
if domain is None:
|
||||||
|
pr_row = conn.execute("SELECT branch FROM prs WHERE number = ?", (pr_number,)).fetchone()
|
||||||
# Default NULL domain to 'general' (archive-only PRs have no domain files)
|
if pr_row and pr_row["branch"]:
|
||||||
|
domain = detect_domain_from_branch(pr_row["branch"])
|
||||||
if domain is None:
|
if domain is None:
|
||||||
domain = "general"
|
domain = "general"
|
||||||
|
agent = agent_for_domain(domain)
|
||||||
|
|
||||||
# Update PR domain if not set
|
# Update PR domain if not set
|
||||||
conn.execute(
|
conn.execute(
|
||||||
|
|
@ -1272,7 +1274,7 @@ def _build_domain_batches(
|
||||||
individual.append(row)
|
individual.append(row)
|
||||||
continue
|
continue
|
||||||
|
|
||||||
domain = existing["domain"] if existing and existing["domain"] else "general"
|
domain = existing["domain"] if existing and existing["domain"] and existing["domain"] != "general" else "general"
|
||||||
domain_candidates.setdefault(domain, []).append(row)
|
domain_candidates.setdefault(domain, []).append(row)
|
||||||
|
|
||||||
# Build sized batches per domain
|
# Build sized batches per domain
|
||||||
|
|
|
||||||
143
lib/merge.py
143
lib/merge.py
|
|
@ -19,6 +19,7 @@ import shutil
|
||||||
from collections import defaultdict
|
from collections import defaultdict
|
||||||
|
|
||||||
from . import config, db
|
from . import config, db
|
||||||
|
from .db import classify_branch
|
||||||
from .domains import detect_domain_from_branch
|
from .domains import detect_domain_from_branch
|
||||||
from .forgejo import api as forgejo_api
|
from .forgejo import api as forgejo_api
|
||||||
|
|
||||||
|
|
@ -96,12 +97,13 @@ async def discover_external_prs(conn) -> int:
|
||||||
origin = "pipeline" if is_pipeline else "human"
|
origin = "pipeline" if is_pipeline else "human"
|
||||||
priority = "high" if origin == "human" else None
|
priority = "high" if origin == "human" else None
|
||||||
domain = None if not is_pipeline else detect_domain_from_branch(pr["head"]["ref"])
|
domain = None if not is_pipeline else detect_domain_from_branch(pr["head"]["ref"])
|
||||||
|
agent, commit_type = classify_branch(pr["head"]["ref"])
|
||||||
|
|
||||||
conn.execute(
|
conn.execute(
|
||||||
"""INSERT OR IGNORE INTO prs
|
"""INSERT OR IGNORE INTO prs
|
||||||
(number, branch, status, origin, priority, domain)
|
(number, branch, status, origin, priority, domain, agent, commit_type)
|
||||||
VALUES (?, ?, 'open', ?, ?, ?)""",
|
VALUES (?, ?, 'open', ?, ?, ?, ?, ?)""",
|
||||||
(pr["number"], pr["head"]["ref"], origin, priority, domain),
|
(pr["number"], pr["head"]["ref"], origin, priority, domain, agent, commit_type),
|
||||||
)
|
)
|
||||||
db.audit(
|
db.audit(
|
||||||
conn,
|
conn,
|
||||||
|
|
@ -409,37 +411,78 @@ async def _delete_remote_branch(branch: str):
|
||||||
# --- Contributor attribution ---
|
# --- Contributor attribution ---
|
||||||
|
|
||||||
|
|
||||||
def _classify_commit_type(diff: str) -> str:
|
def _is_knowledge_pr(diff: str) -> bool:
|
||||||
"""Classify a PR as 'knowledge' or 'pipeline' by files changed.
|
"""Check if a PR touches knowledge files (claims, decisions, core, foundations).
|
||||||
|
|
||||||
Knowledge: claims, decisions, core, foundations (full CI weight)
|
Knowledge PRs get full CI attribution weight.
|
||||||
Pipeline: inbox, entities, agents, archive (zero CI weight)
|
Pipeline-only PRs (inbox, entities, agents, archive) get zero CI weight.
|
||||||
|
|
||||||
Mixed PRs (knowledge + pipeline files) classify as 'knowledge' —
|
Mixed PRs count as knowledge — if a PR adds a claim, it gets attribution
|
||||||
if a PR adds a claim, it gets attribution even if it also moves
|
even if it also moves source files. Knowledge takes priority. (Ganymede review)
|
||||||
source files. Knowledge takes priority. (Ganymede review)
|
|
||||||
"""
|
"""
|
||||||
knowledge_prefixes = ("domains/", "core/", "foundations/", "decisions/")
|
knowledge_prefixes = ("domains/", "core/", "foundations/", "decisions/")
|
||||||
pipeline_prefixes = ("inbox/", "entities/", "agents/")
|
|
||||||
|
|
||||||
has_knowledge = False
|
|
||||||
for line in diff.split("\n"):
|
for line in diff.split("\n"):
|
||||||
if line.startswith("+++ b/") or line.startswith("--- a/"):
|
if line.startswith("+++ b/") or line.startswith("--- a/"):
|
||||||
path = line.split("/", 1)[1] if "/" in line else ""
|
path = line.split("/", 1)[1] if "/" in line else ""
|
||||||
if any(path.startswith(p) for p in knowledge_prefixes):
|
if any(path.startswith(p) for p in knowledge_prefixes):
|
||||||
has_knowledge = True
|
return True
|
||||||
break
|
|
||||||
|
|
||||||
return "knowledge" if has_knowledge else "pipeline"
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def _refine_commit_type(diff: str, branch_commit_type: str) -> str:
|
||||||
|
"""Refine commit_type from diff content when branch prefix is ambiguous.
|
||||||
|
|
||||||
|
Branch prefix gives initial classification (extract, research, entity, etc.).
|
||||||
|
For 'extract' branches, diff content can distinguish:
|
||||||
|
- challenge: adds challenged_by edges to existing claims
|
||||||
|
- enrich: modifies existing claim frontmatter without new files
|
||||||
|
- extract: creates new claim files (default for extract branches)
|
||||||
|
|
||||||
|
Only refines 'extract' type — other branch types (research, entity, reweave, fix)
|
||||||
|
are already specific enough.
|
||||||
|
"""
|
||||||
|
if branch_commit_type != "extract":
|
||||||
|
return branch_commit_type
|
||||||
|
|
||||||
|
new_files = 0
|
||||||
|
modified_files = 0
|
||||||
|
has_challenge_edge = False
|
||||||
|
|
||||||
|
in_diff_header = False
|
||||||
|
current_is_new = False
|
||||||
|
for line in diff.split("\n"):
|
||||||
|
if line.startswith("diff --git"):
|
||||||
|
in_diff_header = True
|
||||||
|
current_is_new = False
|
||||||
|
elif line.startswith("new file"):
|
||||||
|
current_is_new = True
|
||||||
|
elif line.startswith("+++ b/"):
|
||||||
|
path = line[6:]
|
||||||
|
if any(path.startswith(p) for p in ("domains/", "core/", "foundations/")):
|
||||||
|
if current_is_new:
|
||||||
|
new_files += 1
|
||||||
|
else:
|
||||||
|
modified_files += 1
|
||||||
|
in_diff_header = False
|
||||||
|
elif line.startswith("+") and not line.startswith("+++"):
|
||||||
|
if "challenged_by:" in line or "challenges:" in line:
|
||||||
|
has_challenge_edge = True
|
||||||
|
|
||||||
|
if has_challenge_edge and new_files == 0:
|
||||||
|
return "challenge"
|
||||||
|
if modified_files > 0 and new_files == 0:
|
||||||
|
return "enrich"
|
||||||
|
return "extract"
|
||||||
|
|
||||||
|
|
||||||
async def _record_contributor_attribution(conn, pr_number: int, branch: str):
|
async def _record_contributor_attribution(conn, pr_number: int, branch: str):
|
||||||
"""Record contributor attribution after a successful merge.
|
"""Record contributor attribution after a successful merge.
|
||||||
|
|
||||||
Parses git trailers and claim frontmatter to identify contributors
|
Parses git trailers and claim frontmatter to identify contributors
|
||||||
and their roles. Upserts into contributors table.
|
and their roles. Upserts into contributors table. Refines commit_type
|
||||||
Pipeline commits (inbox/, entities/, agents/) get commit_type='pipeline'
|
from diff content. Pipeline-only PRs (no knowledge files) are skipped.
|
||||||
and don't increment role counts.
|
|
||||||
"""
|
"""
|
||||||
import re as _re
|
import re as _re
|
||||||
from datetime import date as _date, datetime as _dt
|
from datetime import date as _date, datetime as _dt
|
||||||
|
|
@ -451,14 +494,19 @@ async def _record_contributor_attribution(conn, pr_number: int, branch: str):
|
||||||
if not diff:
|
if not diff:
|
||||||
return
|
return
|
||||||
|
|
||||||
# Classify commit type — pipeline commits don't count toward CI
|
# Pipeline-only PRs (inbox, entities, agents) don't count toward CI
|
||||||
commit_type = _classify_commit_type(diff)
|
if not _is_knowledge_pr(diff):
|
||||||
conn.execute("UPDATE prs SET commit_type = ? WHERE number = ?", (commit_type, pr_number))
|
logger.info("PR #%d: pipeline-only commit — skipping CI attribution", pr_number)
|
||||||
|
|
||||||
if commit_type == "pipeline":
|
|
||||||
logger.info("PR #%d: pipeline commit — skipping CI attribution", pr_number)
|
|
||||||
return
|
return
|
||||||
|
|
||||||
|
# Refine commit_type from diff content (branch prefix may be too broad)
|
||||||
|
row = conn.execute("SELECT commit_type FROM prs WHERE number = ?", (pr_number,)).fetchone()
|
||||||
|
branch_type = row["commit_type"] if row and row["commit_type"] else "extract"
|
||||||
|
refined_type = _refine_commit_type(diff, branch_type)
|
||||||
|
if refined_type != branch_type:
|
||||||
|
conn.execute("UPDATE prs SET commit_type = ? WHERE number = ?", (refined_type, pr_number))
|
||||||
|
logger.info("PR #%d: commit_type refined %s → %s", pr_number, branch_type, refined_type)
|
||||||
|
|
||||||
# Parse Pentagon-Agent trailer from branch commit messages
|
# Parse Pentagon-Agent trailer from branch commit messages
|
||||||
agents_found: set[str] = set()
|
agents_found: set[str] = set()
|
||||||
rc, log_output = await _git(
|
rc, log_output = await _git(
|
||||||
|
|
@ -612,17 +660,20 @@ def _update_source_frontmatter_status(path: str, new_status: str):
|
||||||
logger.warning("Failed to update source status in %s: %s", path, e)
|
logger.warning("Failed to update source status in %s: %s", path, e)
|
||||||
|
|
||||||
|
|
||||||
async def _embed_merged_claims(branch_sha: str):
|
async def _embed_merged_claims(main_sha: str, branch_sha: str):
|
||||||
"""Embed new/changed claim files from a merged PR into Qdrant.
|
"""Embed new/changed claim files from a merged PR into Qdrant.
|
||||||
|
|
||||||
Finds .md files changed between main~1 and the merged SHA, then calls
|
Diffs main_sha (pre-merge main HEAD) against branch_sha (merged branch tip)
|
||||||
embed-claims.py --file for each. Non-fatal — embedding failure does not
|
to find ALL changed files across the entire branch, not just the last commit.
|
||||||
block the merge pipeline.
|
Also deletes Qdrant vectors for files removed by the branch.
|
||||||
|
|
||||||
|
Non-fatal — embedding failure does not block the merge pipeline.
|
||||||
"""
|
"""
|
||||||
try:
|
try:
|
||||||
|
# --- Embed added/changed files ---
|
||||||
rc, diff_out = await _git(
|
rc, diff_out = await _git(
|
||||||
"diff", "--name-only", "--diff-filter=ACMR",
|
"diff", "--name-only", "--diff-filter=ACMR",
|
||||||
f"{branch_sha}~1", branch_sha,
|
main_sha, branch_sha,
|
||||||
cwd=str(config.MAIN_WORKTREE),
|
cwd=str(config.MAIN_WORKTREE),
|
||||||
timeout=10,
|
timeout=10,
|
||||||
)
|
)
|
||||||
|
|
@ -638,9 +689,6 @@ async def _embed_merged_claims(branch_sha: str):
|
||||||
and not f.split("/")[-1].startswith("_")
|
and not f.split("/")[-1].startswith("_")
|
||||||
]
|
]
|
||||||
|
|
||||||
if not md_files:
|
|
||||||
return
|
|
||||||
|
|
||||||
embedded = 0
|
embedded = 0
|
||||||
for fpath in md_files:
|
for fpath in md_files:
|
||||||
full_path = config.MAIN_WORKTREE / fpath
|
full_path = config.MAIN_WORKTREE / fpath
|
||||||
|
|
@ -659,6 +707,35 @@ async def _embed_merged_claims(branch_sha: str):
|
||||||
|
|
||||||
if embedded:
|
if embedded:
|
||||||
logger.info("embed: %d/%d files embedded into Qdrant", embedded, len(md_files))
|
logger.info("embed: %d/%d files embedded into Qdrant", embedded, len(md_files))
|
||||||
|
|
||||||
|
# --- Delete vectors for removed files (Ganymede: stale vector cleanup) ---
|
||||||
|
rc, del_out = await _git(
|
||||||
|
"diff", "--name-only", "--diff-filter=D",
|
||||||
|
main_sha, branch_sha,
|
||||||
|
cwd=str(config.MAIN_WORKTREE),
|
||||||
|
timeout=10,
|
||||||
|
)
|
||||||
|
if rc == 0 and del_out.strip():
|
||||||
|
deleted_files = [
|
||||||
|
f for f in del_out.strip().split("\n")
|
||||||
|
if f.endswith(".md")
|
||||||
|
and any(f.startswith(d) for d in embed_dirs)
|
||||||
|
]
|
||||||
|
if deleted_files:
|
||||||
|
import hashlib
|
||||||
|
point_ids = [hashlib.md5(f.encode()).hexdigest() for f in deleted_files]
|
||||||
|
try:
|
||||||
|
import urllib.request
|
||||||
|
req = urllib.request.Request(
|
||||||
|
"http://localhost:6333/collections/teleo-claims/points/delete",
|
||||||
|
data=json.dumps({"points": point_ids}).encode(),
|
||||||
|
headers={"Content-Type": "application/json"},
|
||||||
|
method="POST",
|
||||||
|
)
|
||||||
|
urllib.request.urlopen(req, timeout=10)
|
||||||
|
logger.info("embed: deleted %d stale vectors from Qdrant", len(point_ids))
|
||||||
|
except Exception:
|
||||||
|
logger.warning("embed: failed to delete stale vectors (non-fatal)")
|
||||||
except Exception:
|
except Exception:
|
||||||
logger.exception("embed: post-merge embedding failed (non-fatal)")
|
logger.exception("embed: post-merge embedding failed (non-fatal)")
|
||||||
|
|
||||||
|
|
@ -882,7 +959,7 @@ async def _merge_domain_queue(conn, domain: str) -> tuple[int, int]:
|
||||||
_archive_source_for_pr(branch, domain)
|
_archive_source_for_pr(branch, domain)
|
||||||
|
|
||||||
# Embed new/changed claims into Qdrant (non-fatal)
|
# Embed new/changed claims into Qdrant (non-fatal)
|
||||||
await _embed_merged_claims(branch_sha)
|
await _embed_merged_claims(main_sha, branch_sha)
|
||||||
|
|
||||||
# Delete remote branch immediately (Ganymede Q4)
|
# Delete remote branch immediately (Ganymede Q4)
|
||||||
await _delete_remote_branch(branch)
|
await _delete_remote_branch(branch)
|
||||||
|
|
|
||||||
|
|
@ -212,7 +212,7 @@ def fix_h1_title_match(content: str, filename: str) -> tuple[str, list[str]]:
|
||||||
# ─── Validators (check without modifying, return issues) ──────────────────
|
# ─── Validators (check without modifying, return issues) ──────────────────
|
||||||
|
|
||||||
|
|
||||||
def validate_claim(filename: str, content: str, existing_claims: set[str]) -> list[str]:
|
def validate_claim(filename: str, content: str, existing_claims: set[str], agent: str | None = None) -> list[str]:
|
||||||
"""Validate a claim file. Returns list of issues (empty = pass)."""
|
"""Validate a claim file. Returns list of issues (empty = pass)."""
|
||||||
issues = []
|
issues = []
|
||||||
fm, body = parse_frontmatter(content)
|
fm, body = parse_frontmatter(content)
|
||||||
|
|
@ -271,7 +271,7 @@ def validate_claim(filename: str, content: str, existing_claims: set[str]) -> li
|
||||||
# Attribution check: extractor must be identified. (Leo: block extractor, warn sourcer)
|
# Attribution check: extractor must be identified. (Leo: block extractor, warn sourcer)
|
||||||
if ftype == "claim":
|
if ftype == "claim":
|
||||||
from .attribution import validate_attribution
|
from .attribution import validate_attribution
|
||||||
issues.extend(validate_attribution(fm))
|
issues.extend(validate_attribution(fm, agent=agent))
|
||||||
|
|
||||||
# OPSEC check: flag claims containing dollar amounts + internal entity references.
|
# OPSEC check: flag claims containing dollar amounts + internal entity references.
|
||||||
# Rio's rule: never extract LivingIP/Teleo deal terms to public codex. (Ganymede review)
|
# Rio's rule: never extract LivingIP/Teleo deal terms to public codex. (Ganymede review)
|
||||||
|
|
@ -358,7 +358,7 @@ def validate_and_fix_claims(
|
||||||
all_fixes.extend([f"{filename}:{f}" for f in fixes])
|
all_fixes.extend([f"{filename}:{f}" for f in fixes])
|
||||||
|
|
||||||
# Phase 2: Validate (after fixes)
|
# Phase 2: Validate (after fixes)
|
||||||
issues = validate_claim(filename, content, existing_claims)
|
issues = validate_claim(filename, content, existing_claims, agent=agent)
|
||||||
|
|
||||||
# Separate hard failures from warnings
|
# Separate hard failures from warnings
|
||||||
hard_failures = [i for i in issues if not i.startswith("near_duplicate")]
|
hard_failures = [i for i in issues if not i.startswith("near_duplicate")]
|
||||||
|
|
@ -504,6 +504,24 @@ def _rebuild_content(fm: dict, body: str) -> str:
|
||||||
|
|
||||||
def _yaml_line(key: str, val) -> str:
|
def _yaml_line(key: str, val) -> str:
|
||||||
"""Format a single YAML key-value line."""
|
"""Format a single YAML key-value line."""
|
||||||
|
if isinstance(val, dict):
|
||||||
|
# Nested YAML block (e.g. attribution with sub-keys)
|
||||||
|
lines = [f"{key}:"]
|
||||||
|
for sub_key, sub_val in val.items():
|
||||||
|
if isinstance(sub_val, list) and sub_val:
|
||||||
|
lines.append(f" {sub_key}:")
|
||||||
|
for item in sub_val:
|
||||||
|
if isinstance(item, dict):
|
||||||
|
first = True
|
||||||
|
for ik, iv in item.items():
|
||||||
|
prefix = " - " if first else " "
|
||||||
|
lines.append(f'{prefix}{ik}: "{iv}"')
|
||||||
|
first = False
|
||||||
|
else:
|
||||||
|
lines.append(f' - "{item}"')
|
||||||
|
else:
|
||||||
|
lines.append(f" {sub_key}: []")
|
||||||
|
return "\n".join(lines)
|
||||||
if isinstance(val, list):
|
if isinstance(val, list):
|
||||||
return f"{key}: {json.dumps(val)}"
|
return f"{key}: {json.dumps(val)}"
|
||||||
if isinstance(val, bool):
|
if isinstance(val, bool):
|
||||||
|
|
|
||||||
220
lib/stale_pr.py
Normal file
220
lib/stale_pr.py
Normal file
|
|
@ -0,0 +1,220 @@
|
||||||
|
"""Stale PR monitor — auto-close extraction PRs that produced no claims.
|
||||||
|
|
||||||
|
Catches the failure mode where batch-extract creates a PR but extraction
|
||||||
|
produces only source-file updates (no actual claims). These PRs sit open
|
||||||
|
indefinitely, consuming merge queue bandwidth and confusing metrics.
|
||||||
|
|
||||||
|
Rules:
|
||||||
|
- PR branch starts with "extract/"
|
||||||
|
- PR is open for >30 minutes
|
||||||
|
- PR diff contains 0 files in domains/*/ or decisions/*/
|
||||||
|
→ Auto-close with comment, log to audit_log as stale_extraction_closed
|
||||||
|
|
||||||
|
- If same source branch has been stale-closed 2+ times
|
||||||
|
→ Mark source as extraction_failed in pipeline.db sources table
|
||||||
|
|
||||||
|
Called from the pipeline daemon (piggyback on validate_cycle interval)
|
||||||
|
or standalone via: python3 -m lib.stale_pr
|
||||||
|
|
||||||
|
Owner: Epimetheus
|
||||||
|
"""
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import sqlite3
|
||||||
|
import urllib.request
|
||||||
|
from datetime import datetime, timedelta, timezone
|
||||||
|
|
||||||
|
from . import config
|
||||||
|
|
||||||
|
logger = logging.getLogger("pipeline.stale_pr")
|
||||||
|
|
||||||
|
STALE_THRESHOLD_MINUTES = 30
|
||||||
|
MAX_STALE_FAILURES = 2 # After this many stale closures, mark source as failed
|
||||||
|
|
||||||
|
|
||||||
|
def _forgejo_api(method: str, path: str, body: dict | None = None) -> dict | list | None:
|
||||||
|
"""Call Forgejo API. Returns parsed JSON or None on failure."""
|
||||||
|
token_file = config.FORGEJO_TOKEN_FILE
|
||||||
|
if not token_file.exists():
|
||||||
|
logger.error("No Forgejo token at %s", token_file)
|
||||||
|
return None
|
||||||
|
token = token_file.read_text().strip()
|
||||||
|
|
||||||
|
url = f"{config.FORGEJO_URL}/api/v1/{path}"
|
||||||
|
data = json.dumps(body).encode() if body else None
|
||||||
|
req = urllib.request.Request(
|
||||||
|
url,
|
||||||
|
data=data,
|
||||||
|
headers={
|
||||||
|
"Authorization": f"token {token}",
|
||||||
|
"Content-Type": "application/json",
|
||||||
|
},
|
||||||
|
method=method,
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
with urllib.request.urlopen(req, timeout=15) as resp:
|
||||||
|
return json.loads(resp.read())
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning("Forgejo API %s %s failed: %s", method, path, e)
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _pr_has_claim_files(pr_number: int) -> bool:
|
||||||
|
"""Check if a PR's diff contains any files in domains/ or decisions/."""
|
||||||
|
diff_data = _forgejo_api("GET", f"repos/{config.FORGEJO_OWNER}/{config.FORGEJO_REPO}/pulls/{pr_number}/files")
|
||||||
|
if not diff_data or not isinstance(diff_data, list):
|
||||||
|
return False
|
||||||
|
|
||||||
|
for file_entry in diff_data:
|
||||||
|
filename = file_entry.get("filename", "")
|
||||||
|
if filename.startswith("domains/") or filename.startswith("decisions/"):
|
||||||
|
# Check it's a .md file, not a directory marker
|
||||||
|
if filename.endswith(".md"):
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def _close_pr(pr_number: int, reason: str) -> bool:
|
||||||
|
"""Close a PR with a comment explaining why."""
|
||||||
|
# Add comment
|
||||||
|
_forgejo_api("POST",
|
||||||
|
f"repos/{config.FORGEJO_OWNER}/{config.FORGEJO_REPO}/issues/{pr_number}/comments",
|
||||||
|
{"body": f"Auto-closed by stale PR monitor: {reason}\n\nPentagon-Agent: Epimetheus"},
|
||||||
|
)
|
||||||
|
# Close PR
|
||||||
|
result = _forgejo_api("PATCH",
|
||||||
|
f"repos/{config.FORGEJO_OWNER}/{config.FORGEJO_REPO}/pulls/{pr_number}",
|
||||||
|
{"state": "closed"},
|
||||||
|
)
|
||||||
|
return result is not None
|
||||||
|
|
||||||
|
|
||||||
|
def _log_audit(conn: sqlite3.Connection, pr_number: int, branch: str):
|
||||||
|
"""Log stale closure to audit_log."""
|
||||||
|
try:
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO audit_log (timestamp, stage, event, detail) VALUES (datetime('now'), ?, ?, ?)",
|
||||||
|
("monitor", "stale_extraction_closed", json.dumps({"pr": pr_number, "branch": branch})),
|
||||||
|
)
|
||||||
|
conn.commit()
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning("Audit log write failed: %s", e)
|
||||||
|
|
||||||
|
|
||||||
|
def _count_stale_closures(conn: sqlite3.Connection, branch: str) -> int:
|
||||||
|
"""Count how many times this branch has been stale-closed."""
|
||||||
|
try:
|
||||||
|
row = conn.execute(
|
||||||
|
"SELECT COUNT(*) FROM audit_log WHERE event = 'stale_extraction_closed' AND detail LIKE ?",
|
||||||
|
(f'%"branch": "{branch}"%',),
|
||||||
|
).fetchone()
|
||||||
|
return row[0] if row else 0
|
||||||
|
except Exception:
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
def _mark_source_failed(conn: sqlite3.Connection, branch: str):
|
||||||
|
"""Mark the source as extraction_failed after repeated stale closures."""
|
||||||
|
# Extract source name from branch: extract/source-name → source-name
|
||||||
|
source_name = branch.removeprefix("extract/")
|
||||||
|
try:
|
||||||
|
conn.execute(
|
||||||
|
"UPDATE sources SET status = 'extraction_failed', last_error = 'repeated_stale_extraction', updated_at = datetime('now') WHERE path LIKE ?",
|
||||||
|
(f"%{source_name}%",),
|
||||||
|
)
|
||||||
|
conn.commit()
|
||||||
|
logger.info("Marked source %s as extraction_failed (repeated stale closures)", source_name)
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning("Failed to mark source as failed: %s", e)
|
||||||
|
|
||||||
|
|
||||||
|
def check_stale_prs(conn: sqlite3.Connection) -> tuple[int, int]:
|
||||||
|
"""Check for and close stale extraction PRs.
|
||||||
|
|
||||||
|
Returns (closed_count, error_count).
|
||||||
|
"""
|
||||||
|
closed = 0
|
||||||
|
errors = 0
|
||||||
|
|
||||||
|
# Fetch all open PRs (paginated)
|
||||||
|
page = 1
|
||||||
|
all_prs = []
|
||||||
|
while True:
|
||||||
|
prs = _forgejo_api("GET",
|
||||||
|
f"repos/{config.FORGEJO_OWNER}/{config.FORGEJO_REPO}/pulls?state=open&limit=50&page={page}")
|
||||||
|
if not prs:
|
||||||
|
break
|
||||||
|
all_prs.extend(prs)
|
||||||
|
if len(prs) < 50:
|
||||||
|
break
|
||||||
|
page += 1
|
||||||
|
|
||||||
|
now = datetime.now(timezone.utc)
|
||||||
|
|
||||||
|
for pr in all_prs:
|
||||||
|
branch = pr.get("head", {}).get("ref", "")
|
||||||
|
if not branch.startswith("extract/"):
|
||||||
|
continue
|
||||||
|
|
||||||
|
# Check age
|
||||||
|
created_str = pr.get("created_at", "")
|
||||||
|
if not created_str:
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
# Forgejo returns ISO format with Z suffix
|
||||||
|
created = datetime.fromisoformat(created_str.replace("Z", "+00:00"))
|
||||||
|
except ValueError:
|
||||||
|
continue
|
||||||
|
|
||||||
|
age_minutes = (now - created).total_seconds() / 60
|
||||||
|
if age_minutes < STALE_THRESHOLD_MINUTES:
|
||||||
|
continue
|
||||||
|
|
||||||
|
pr_number = pr["number"]
|
||||||
|
|
||||||
|
# Check if PR has claim files
|
||||||
|
if _pr_has_claim_files(pr_number):
|
||||||
|
continue # PR has claims — not stale
|
||||||
|
|
||||||
|
# PR is stale — close it
|
||||||
|
logger.info("Stale PR #%d: branch=%s, age=%.0f min, no claim files — closing",
|
||||||
|
pr_number, branch, age_minutes)
|
||||||
|
|
||||||
|
if _close_pr(pr_number, f"No claim files after {int(age_minutes)} minutes. Branch: {branch}"):
|
||||||
|
closed += 1
|
||||||
|
_log_audit(conn, pr_number, branch)
|
||||||
|
|
||||||
|
# Check for repeated failures
|
||||||
|
failure_count = _count_stale_closures(conn, branch)
|
||||||
|
if failure_count >= MAX_STALE_FAILURES:
|
||||||
|
_mark_source_failed(conn, branch)
|
||||||
|
logger.warning("Source %s marked as extraction_failed after %d stale closures",
|
||||||
|
branch, failure_count)
|
||||||
|
else:
|
||||||
|
errors += 1
|
||||||
|
logger.warning("Failed to close stale PR #%d", pr_number)
|
||||||
|
|
||||||
|
if closed:
|
||||||
|
logger.info("Stale PR monitor: closed %d PRs", closed)
|
||||||
|
|
||||||
|
return closed, errors
|
||||||
|
|
||||||
|
|
||||||
|
# Allow standalone execution
|
||||||
|
if __name__ == "__main__":
|
||||||
|
import sys
|
||||||
|
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
|
||||||
|
|
||||||
|
db_path = config.DB_PATH
|
||||||
|
if not db_path.exists():
|
||||||
|
print(f"ERROR: Database not found at {db_path}", file=sys.stderr)
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
conn = sqlite3.connect(str(db_path))
|
||||||
|
conn.row_factory = sqlite3.Row
|
||||||
|
closed, errs = check_stale_prs(conn)
|
||||||
|
print(f"Stale PR monitor: {closed} closed, {errs} errors")
|
||||||
|
conn.close()
|
||||||
|
|
@ -19,6 +19,7 @@ import logging
|
||||||
from datetime import datetime, timezone
|
from datetime import datetime, timezone
|
||||||
|
|
||||||
from . import config, db
|
from . import config, db
|
||||||
|
from .stale_pr import check_stale_prs
|
||||||
|
|
||||||
logger = logging.getLogger("pipeline.watchdog")
|
logger = logging.getLogger("pipeline.watchdog")
|
||||||
|
|
||||||
|
|
@ -115,6 +116,26 @@ async def watchdog_check(conn) -> dict:
|
||||||
"action": "Check validate.py — may be the modified-file or wiki-link bug recurring",
|
"action": "Check validate.py — may be the modified-file or wiki-link bug recurring",
|
||||||
})
|
})
|
||||||
|
|
||||||
|
# 6. Stale extraction PRs: open >30 min with no claim files
|
||||||
|
try:
|
||||||
|
stale_closed, stale_errors = check_stale_prs(conn)
|
||||||
|
if stale_closed > 0:
|
||||||
|
issues.append({
|
||||||
|
"type": "stale_prs_closed",
|
||||||
|
"severity": "info",
|
||||||
|
"detail": f"Auto-closed {stale_closed} stale extraction PRs (no claims after {30} min)",
|
||||||
|
"action": "Check batch-extract logs for extraction failures",
|
||||||
|
})
|
||||||
|
if stale_errors > 0:
|
||||||
|
issues.append({
|
||||||
|
"type": "stale_pr_close_failed",
|
||||||
|
"severity": "warning",
|
||||||
|
"detail": f"Failed to close {stale_errors} stale PRs",
|
||||||
|
"action": "Check Forgejo API connectivity",
|
||||||
|
})
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning("Stale PR check failed: %s", e)
|
||||||
|
|
||||||
# Log issues
|
# Log issues
|
||||||
healthy = len(issues) == 0
|
healthy = len(issues) == 0
|
||||||
if not healthy:
|
if not healthy:
|
||||||
|
|
@ -124,7 +145,7 @@ async def watchdog_check(conn) -> dict:
|
||||||
else:
|
else:
|
||||||
logger.info("WATCHDOG: %s — %s", issue["type"], issue["detail"])
|
logger.info("WATCHDOG: %s — %s", issue["type"], issue["detail"])
|
||||||
|
|
||||||
return {"healthy": healthy, "issues": issues, "checks_run": 5}
|
return {"healthy": healthy, "issues": issues, "checks_run": 6}
|
||||||
|
|
||||||
|
|
||||||
async def watchdog_cycle(conn, max_workers=None) -> tuple[int, int]:
|
async def watchdog_cycle(conn, max_workers=None) -> tuple[int, int]:
|
||||||
|
|
|
||||||
|
|
@ -40,6 +40,7 @@ from lib.post_extract import (
|
||||||
validate_and_fix_claims,
|
validate_and_fix_claims,
|
||||||
validate_and_fix_entities,
|
validate_and_fix_entities,
|
||||||
)
|
)
|
||||||
|
from lib.connect import connect_new_claims
|
||||||
|
|
||||||
# ─── Source registration (Argus: pipeline funnel tracking) ─────────────────
|
# ─── Source registration (Argus: pipeline funnel tracking) ─────────────────
|
||||||
|
|
||||||
|
|
@ -455,6 +456,21 @@ def main():
|
||||||
written.append(filename)
|
written.append(filename)
|
||||||
print(f" Wrote: {claim_path}")
|
print(f" Wrote: {claim_path}")
|
||||||
|
|
||||||
|
# ── Atomic connect: wire new claims to existing KB via vector search ──
|
||||||
|
connect_stats = {"connected": 0, "edges_added": 0}
|
||||||
|
if written:
|
||||||
|
written_paths = [os.path.join(domain_dir, f) for f in written]
|
||||||
|
try:
|
||||||
|
connect_stats = connect_new_claims(written_paths, domain=domain)
|
||||||
|
if connect_stats["connected"] > 0:
|
||||||
|
print(f" Connected: {connect_stats['connected']}/{len(written)} claims → {connect_stats['edges_added']} edges")
|
||||||
|
for conn in connect_stats.get("connections", []):
|
||||||
|
print(f" {conn['claim']} → {', '.join(n[:40] for n in conn['neighbors'][:3])}")
|
||||||
|
if connect_stats.get("skipped_embed_failed"):
|
||||||
|
print(f" WARN: {connect_stats['skipped_embed_failed']} claims failed embedding (Qdrant unreachable?)")
|
||||||
|
except Exception as e:
|
||||||
|
print(f" WARN: Extract-and-connect failed (non-fatal): {e}", file=sys.stderr)
|
||||||
|
|
||||||
# ── Apply enrichments ──
|
# ── Apply enrichments ──
|
||||||
enriched = []
|
enriched = []
|
||||||
for enr in enrichments:
|
for enr in enrichments:
|
||||||
|
|
@ -616,6 +632,7 @@ def main():
|
||||||
print(f" Model: {args.model} ({p1_in} in / {p1_out} out)")
|
print(f" Model: {args.model} ({p1_in} in / {p1_out} out)")
|
||||||
print(f" Pass 2: Python validator ($0)")
|
print(f" Pass 2: Python validator ($0)")
|
||||||
print(f" Claims: {len(written)} written, {claim_stats['rejected']} rejected, {claim_stats['fixed']} auto-fixed")
|
print(f" Claims: {len(written)} written, {claim_stats['rejected']} rejected, {claim_stats['fixed']} auto-fixed")
|
||||||
|
print(f" Connected: {connect_stats.get('connected', 0)} claims → {connect_stats.get('edges_added', 0)} edges (Qdrant)")
|
||||||
print(f" Enrichments: {len(enriched)} applied")
|
print(f" Enrichments: {len(enriched)} applied")
|
||||||
if entities_enqueued:
|
if entities_enqueued:
|
||||||
print(f" Entities: {len(entities_enqueued)} enqueued (applied by batch on main)")
|
print(f" Entities: {len(entities_enqueued)} enqueued (applied by batch on main)")
|
||||||
|
|
|
||||||
115
ops/reconcile-source-status.sh
Executable file
115
ops/reconcile-source-status.sh
Executable file
|
|
@ -0,0 +1,115 @@
|
||||||
|
#!/bin/bash
|
||||||
|
# Reconcile source archive status: mark sources as processed if claims already exist
|
||||||
|
# Usage: ./reconcile-source-status.sh [--apply]
|
||||||
|
# Default: dry-run (preview only)
|
||||||
|
# --apply: actually modify files
|
||||||
|
|
||||||
|
CODEX_DIR="/Users/coryabdalla/Pentagon/teleo-codex"
|
||||||
|
ARCHIVE_DIR="$CODEX_DIR/inbox/archive"
|
||||||
|
DOMAINS_DIR="$CODEX_DIR/domains"
|
||||||
|
|
||||||
|
MODE="dry-run"
|
||||||
|
[[ "${1:-}" == "--apply" ]] && MODE="apply"
|
||||||
|
|
||||||
|
echo "=== Source Status Reconciliation ==="
|
||||||
|
echo "Mode: $MODE"
|
||||||
|
echo ""
|
||||||
|
|
||||||
|
matched=0
|
||||||
|
null_result=0
|
||||||
|
skipped=0
|
||||||
|
already_ok=0
|
||||||
|
|
||||||
|
while read -r src; do
|
||||||
|
# Only process unprocessed sources
|
||||||
|
status=$(grep "^status:" "$src" 2>/dev/null | head -1 | sed 's/^status: *//')
|
||||||
|
if [[ "$status" != "unprocessed" ]]; then
|
||||||
|
already_ok=$((already_ok + 1))
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
|
||||||
|
url=$(grep "^url:" "$src" 2>/dev/null | head -1 | sed 's/^url: *"*//;s/"*$//')
|
||||||
|
title=$(grep "^title:" "$src" 2>/dev/null | head -1 | sed 's/^title: *"*//;s/"*$//')
|
||||||
|
fname=$(basename "$src")
|
||||||
|
|
||||||
|
# Check 1: Is this a test/spam source?
|
||||||
|
is_test=false
|
||||||
|
if echo "$title" | grep -qiE "^(Futardio: )?test[ -]"; then
|
||||||
|
is_test=true
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Check 2: URL-based match — search for the unique URL identifier in claims
|
||||||
|
url_matched=false
|
||||||
|
if [[ -n "$url" ]]; then
|
||||||
|
# Extract the unique hash/slug from the URL (the long alphanumeric key)
|
||||||
|
url_key=$(echo "$url" | grep -oE '[A-Za-z0-9]{20,}' | tail -1 || true)
|
||||||
|
if [[ -n "$url_key" ]]; then
|
||||||
|
if grep -rq "$url_key" "$DOMAINS_DIR" 2>/dev/null; then
|
||||||
|
url_matched=true
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
# Also try the full URL domain+path
|
||||||
|
if ! $url_matched; then
|
||||||
|
# Try matching the last path segment
|
||||||
|
path_seg=$(echo "$url" | grep -oE '[^/]+$' || true)
|
||||||
|
if [[ -n "$path_seg" ]] && [[ ${#path_seg} -gt 10 ]]; then
|
||||||
|
if grep -rq "$path_seg" "$DOMAINS_DIR" 2>/dev/null; then
|
||||||
|
url_matched=true
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Check 3: Title match — search for a distinctive part of the title in claim source: fields
|
||||||
|
title_matched=false
|
||||||
|
if [[ -n "$title" ]]; then
|
||||||
|
# Strip "Futardio: " prefix and grab a distinctive portion
|
||||||
|
clean_title=$(echo "$title" | sed 's/^Futardio: //')
|
||||||
|
# Use first 30 chars as search key (enough to be distinctive)
|
||||||
|
title_key=$(echo "$clean_title" | cut -c1-30)
|
||||||
|
if [[ ${#title_key} -gt 8 ]]; then
|
||||||
|
if grep -rqi "$title_key" "$DOMAINS_DIR" 2>/dev/null; then
|
||||||
|
title_matched=true
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
if $is_test; then
|
||||||
|
echo " NULL-RESULT (test/spam): $fname"
|
||||||
|
null_result=$((null_result + 1))
|
||||||
|
if [[ "$MODE" == "apply" ]]; then
|
||||||
|
sed -i '' "s/^status: unprocessed/status: null-result/" "$src"
|
||||||
|
if ! grep -q "^processed_by:" "$src"; then
|
||||||
|
sed -i '' "/^status: null-result/a\\
|
||||||
|
processed_by: epimetheus-reconcile\\
|
||||||
|
processed_date: $(date +%Y-%m-%d)\\
|
||||||
|
notes: \"auto-reconciled: test/spam source\"" "$src"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
elif $url_matched || $title_matched; then
|
||||||
|
match_type=""
|
||||||
|
$url_matched && match_type="url" || true
|
||||||
|
$title_matched && match_type="${match_type:+$match_type+}title" || true
|
||||||
|
echo " PROCESSED ($match_type): $fname"
|
||||||
|
matched=$((matched + 1))
|
||||||
|
if [[ "$MODE" == "apply" ]]; then
|
||||||
|
sed -i '' "s/^status: unprocessed/status: processed/" "$src"
|
||||||
|
if ! grep -q "^processed_by:" "$src"; then
|
||||||
|
sed -i '' "/^status: processed/a\\
|
||||||
|
processed_by: epimetheus-reconcile\\
|
||||||
|
processed_date: $(date +%Y-%m-%d)\\
|
||||||
|
notes: \"auto-reconciled: claims found matching this source\"" "$src"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
skipped=$((skipped + 1))
|
||||||
|
fi
|
||||||
|
done < <(find "$ARCHIVE_DIR" -name "*.md" -type f)
|
||||||
|
|
||||||
|
echo ""
|
||||||
|
echo "=== Summary ==="
|
||||||
|
echo "Already correct status: $already_ok"
|
||||||
|
echo "Matched → processed: $matched"
|
||||||
|
echo "Test/spam → null-result: $null_result"
|
||||||
|
echo "Still unprocessed: $skipped"
|
||||||
|
echo "Total archive files: $(find "$ARCHIVE_DIR" -name '*.md' -type f 2>/dev/null | wc -l | tr -d ' ')"
|
||||||
450
reconcile-sources.py
Normal file
450
reconcile-sources.py
Normal file
|
|
@ -0,0 +1,450 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
"""
|
||||||
|
Reconcile archive source status and add bidirectional links.
|
||||||
|
|
||||||
|
Matches unprocessed archive sources to existing decisions, entities, and claims.
|
||||||
|
Updates status to 'processed' or 'null-result' and adds frontmatter links.
|
||||||
|
|
||||||
|
Linking pattern (Ganymede Option A — frontmatter only):
|
||||||
|
- Archive sources get `derived_items:` listing decision/entity paths
|
||||||
|
- Decisions/entities get `source_archive:` pointing to archive source path
|
||||||
|
- All paths relative to repo root
|
||||||
|
|
||||||
|
Usage:
|
||||||
|
python3 reconcile-sources.py [--apply] # default: dry-run
|
||||||
|
python3 reconcile-sources.py --apply # apply changes
|
||||||
|
"""
|
||||||
|
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
from urllib.parse import urlparse
|
||||||
|
from collections import defaultdict
|
||||||
|
|
||||||
|
REPO_ROOT = Path("/opt/teleo-eval/workspaces/main")
|
||||||
|
ARCHIVE_DIR = REPO_ROOT / "inbox" / "archive"
|
||||||
|
DECISIONS_DIR = REPO_ROOT / "decisions"
|
||||||
|
ENTITIES_DIR = REPO_ROOT / "entities"
|
||||||
|
DOMAINS_DIR = REPO_ROOT / "domains"
|
||||||
|
|
||||||
|
DRY_RUN = "--apply" not in sys.argv
|
||||||
|
|
||||||
|
# --- YAML frontmatter helpers ---
|
||||||
|
|
||||||
|
def read_frontmatter(filepath):
|
||||||
|
"""Read file, return (frontmatter_text, body_text, raw_content)."""
|
||||||
|
content = filepath.read_text(encoding="utf-8")
|
||||||
|
if not content.startswith("---"):
|
||||||
|
return None, content, content
|
||||||
|
end = content.find("\n---", 3)
|
||||||
|
if end == -1:
|
||||||
|
return None, content, content
|
||||||
|
fm = content[3:end].strip()
|
||||||
|
body = content[end + 4:] # skip \n---
|
||||||
|
return fm, body, content
|
||||||
|
|
||||||
|
|
||||||
|
def get_field(fm_text, field):
|
||||||
|
"""Get a single YAML field value from frontmatter text."""
|
||||||
|
if fm_text is None:
|
||||||
|
return None
|
||||||
|
m = re.search(rf'^{field}:\s*["\']?(.+?)["\']?\s*$', fm_text, re.MULTILINE)
|
||||||
|
return m.group(1) if m else None
|
||||||
|
|
||||||
|
|
||||||
|
def get_status(fm_text):
|
||||||
|
return get_field(fm_text, "status")
|
||||||
|
|
||||||
|
|
||||||
|
def get_url(fm_text):
|
||||||
|
return get_field(fm_text, "url")
|
||||||
|
|
||||||
|
|
||||||
|
def get_proposal_url(fm_text):
|
||||||
|
return get_field(fm_text, "proposal_url")
|
||||||
|
|
||||||
|
|
||||||
|
def get_title(fm_text):
|
||||||
|
return get_field(fm_text, "title")
|
||||||
|
|
||||||
|
|
||||||
|
def extract_hash_from_url(url):
|
||||||
|
"""Extract the proposal hash (last path segment) from a URL."""
|
||||||
|
if not url:
|
||||||
|
return None
|
||||||
|
parsed = urlparse(url.strip('"').strip("'"))
|
||||||
|
parts = [p for p in parsed.path.split("/") if p]
|
||||||
|
if parts:
|
||||||
|
last = parts[-1]
|
||||||
|
# Proposal hashes are base58-like, 32-50 chars
|
||||||
|
if len(last) >= 20 and re.match(r'^[A-Za-z0-9]+$', last):
|
||||||
|
return last
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def rel_path(filepath):
|
||||||
|
"""Get path relative to repo root."""
|
||||||
|
return str(filepath.relative_to(REPO_ROOT))
|
||||||
|
|
||||||
|
|
||||||
|
# --- Test/spam detection ---
|
||||||
|
|
||||||
|
TEST_PATTERNS = [
|
||||||
|
r'\btest\b', r'\btesting\b', r'\bmy-test\b', r'\bq\b$',
|
||||||
|
r'\ba-very-unique', r'\btext-mint', r'\bsample\b',
|
||||||
|
r'\basdf\b', r'\bfoo\b', r'\bbar\b', r'\bhello-world\b',
|
||||||
|
r'\bgrpc-indexer\b', r'\brocks{0,2}wd\b',
|
||||||
|
r'spending-limit', r'\btest-proposal\b',
|
||||||
|
r'\bdummy\b',
|
||||||
|
]
|
||||||
|
TEST_RE = re.compile('|'.join(TEST_PATTERNS), re.IGNORECASE)
|
||||||
|
|
||||||
|
# Title-based patterns
|
||||||
|
TEST_TITLE_PATTERNS = [
|
||||||
|
r'^test\b', r'^testing\b', r'^q$', r'^a$', r'^asdf',
|
||||||
|
r'^my test', r'^sample', r'^hello',
|
||||||
|
r'text mint ix', r'a very unique title',
|
||||||
|
r'testing spending limit', r'testing.*grpc',
|
||||||
|
r'my-test-proposal',
|
||||||
|
]
|
||||||
|
TEST_TITLE_RE = re.compile('|'.join(TEST_TITLE_PATTERNS), re.IGNORECASE)
|
||||||
|
|
||||||
|
|
||||||
|
def is_test_spam(filepath, fm_text):
|
||||||
|
"""Detect test/spam sources."""
|
||||||
|
name = filepath.stem
|
||||||
|
if TEST_RE.search(name):
|
||||||
|
return True
|
||||||
|
title = get_title(fm_text) or ""
|
||||||
|
if TEST_TITLE_RE.search(title):
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
# --- Build indexes ---
|
||||||
|
|
||||||
|
def build_decision_hash_index():
|
||||||
|
"""Map proposal hash → decision file path."""
|
||||||
|
index = {}
|
||||||
|
if not DECISIONS_DIR.exists():
|
||||||
|
return index
|
||||||
|
for f in DECISIONS_DIR.rglob("*.md"):
|
||||||
|
fm, _, _ = read_frontmatter(f)
|
||||||
|
url = get_proposal_url(fm)
|
||||||
|
h = extract_hash_from_url(url)
|
||||||
|
if h:
|
||||||
|
index[h] = f
|
||||||
|
return index
|
||||||
|
|
||||||
|
|
||||||
|
def build_entity_name_index():
|
||||||
|
"""Map normalized entity name → entity file path."""
|
||||||
|
index = {}
|
||||||
|
if not ENTITIES_DIR.exists():
|
||||||
|
return index
|
||||||
|
for f in ENTITIES_DIR.rglob("*.md"):
|
||||||
|
# Use filename as entity name
|
||||||
|
name = f.stem.lower().replace("-", " ").replace("_", " ")
|
||||||
|
index[name] = f
|
||||||
|
return index
|
||||||
|
|
||||||
|
|
||||||
|
def build_claim_source_index():
|
||||||
|
"""Map archive source slug → list of claim file paths (via wiki-links)."""
|
||||||
|
index = defaultdict(list)
|
||||||
|
if not DOMAINS_DIR.exists():
|
||||||
|
return index
|
||||||
|
for f in DOMAINS_DIR.rglob("*.md"):
|
||||||
|
try:
|
||||||
|
content = f.read_text(encoding="utf-8")
|
||||||
|
except Exception:
|
||||||
|
continue
|
||||||
|
# Find wiki-links to archive: [[inbox/archive/...]]
|
||||||
|
for m in re.finditer(r'\[\[inbox/archive/([^\]]+)\]\]', content):
|
||||||
|
slug = m.group(1)
|
||||||
|
index[slug].append(f)
|
||||||
|
return index
|
||||||
|
|
||||||
|
|
||||||
|
# --- Frontmatter modification ---
|
||||||
|
|
||||||
|
def add_frontmatter_field(filepath, field_name, field_value):
|
||||||
|
"""Add a YAML field to frontmatter. Returns modified content or None if already present."""
|
||||||
|
content = filepath.read_text(encoding="utf-8")
|
||||||
|
if not content.startswith("---"):
|
||||||
|
return None
|
||||||
|
|
||||||
|
end = content.find("\n---", 3)
|
||||||
|
if end == -1:
|
||||||
|
return None
|
||||||
|
|
||||||
|
fm = content[3:end]
|
||||||
|
|
||||||
|
# Check if field already exists
|
||||||
|
if re.search(rf'^{field_name}:', fm, re.MULTILINE):
|
||||||
|
return None # Already has this field
|
||||||
|
|
||||||
|
# Add before closing ---
|
||||||
|
if isinstance(field_value, list):
|
||||||
|
lines = f"\n{field_name}:"
|
||||||
|
for v in field_value:
|
||||||
|
lines += f'\n - "{v}"'
|
||||||
|
new_fm = fm.rstrip() + lines + "\n"
|
||||||
|
else:
|
||||||
|
new_fm = fm.rstrip() + f'\n{field_name}: "{field_value}"\n'
|
||||||
|
|
||||||
|
return "---" + new_fm + "---" + content[end + 4:]
|
||||||
|
|
||||||
|
|
||||||
|
def set_status(filepath, new_status):
|
||||||
|
"""Change status field in frontmatter."""
|
||||||
|
content = filepath.read_text(encoding="utf-8")
|
||||||
|
if not content.startswith("---"):
|
||||||
|
return None
|
||||||
|
# Replace status field
|
||||||
|
new_content = re.sub(
|
||||||
|
r'^(status:\s*).*$',
|
||||||
|
f'\\1{new_status}',
|
||||||
|
content,
|
||||||
|
count=1,
|
||||||
|
flags=re.MULTILINE
|
||||||
|
)
|
||||||
|
if new_content == content:
|
||||||
|
return None
|
||||||
|
return new_content
|
||||||
|
|
||||||
|
|
||||||
|
# --- Main reconciliation ---
|
||||||
|
|
||||||
|
def main():
|
||||||
|
print(f"{'DRY RUN' if DRY_RUN else 'APPLYING CHANGES'}")
|
||||||
|
print(f"Repo root: {REPO_ROOT}")
|
||||||
|
print()
|
||||||
|
|
||||||
|
# Build indexes
|
||||||
|
print("Building indexes...")
|
||||||
|
decision_hash_idx = build_decision_hash_index()
|
||||||
|
print(f" Decision hash index: {len(decision_hash_idx)} entries")
|
||||||
|
|
||||||
|
entity_name_idx = build_entity_name_index()
|
||||||
|
print(f" Entity name index: {len(entity_name_idx)} entries")
|
||||||
|
|
||||||
|
claim_source_idx = build_claim_source_index()
|
||||||
|
print(f" Claim source index: {len(claim_source_idx)} entries")
|
||||||
|
print()
|
||||||
|
|
||||||
|
# Find all unprocessed archive sources
|
||||||
|
unprocessed = []
|
||||||
|
for f in sorted(ARCHIVE_DIR.rglob("*.md")):
|
||||||
|
if ".extraction-debug" in str(f):
|
||||||
|
continue
|
||||||
|
fm, _, _ = read_frontmatter(f)
|
||||||
|
if get_status(fm) == "unprocessed":
|
||||||
|
unprocessed.append(f)
|
||||||
|
|
||||||
|
print(f"Found {len(unprocessed)} unprocessed sources")
|
||||||
|
print()
|
||||||
|
|
||||||
|
# Categorize and match
|
||||||
|
matched = [] # (source_path, [target_paths], match_type)
|
||||||
|
test_spam = []
|
||||||
|
futardio_unmatched = [] # futardio proposals with no KB output → null-result
|
||||||
|
genuine_backlog = [] # non-futardio sources still awaiting extraction → keep unprocessed
|
||||||
|
|
||||||
|
def is_futardio_source(filepath):
|
||||||
|
"""Check if file is a futardio/metadao governance proposal (not research)."""
|
||||||
|
name = filepath.name.lower()
|
||||||
|
return "futardio" in name
|
||||||
|
|
||||||
|
for src in unprocessed:
|
||||||
|
fm, _, _ = read_frontmatter(src)
|
||||||
|
|
||||||
|
# Check test/spam first
|
||||||
|
if is_test_spam(src, fm):
|
||||||
|
test_spam.append(src)
|
||||||
|
continue
|
||||||
|
|
||||||
|
targets = []
|
||||||
|
match_types = []
|
||||||
|
|
||||||
|
# Match 1: proposal hash → decision
|
||||||
|
url = get_url(fm)
|
||||||
|
src_hash = extract_hash_from_url(url)
|
||||||
|
if src_hash and src_hash in decision_hash_idx:
|
||||||
|
targets.append(decision_hash_idx[src_hash])
|
||||||
|
match_types.append("hash→decision")
|
||||||
|
|
||||||
|
# Match 2: wiki-links from claims
|
||||||
|
# Try multiple slug variants
|
||||||
|
src_rel = rel_path(src)
|
||||||
|
slug_no_ext = src_rel.replace("inbox/archive/", "").replace(".md", "")
|
||||||
|
# Also try just the filename without extension
|
||||||
|
slug_basename = src.stem
|
||||||
|
for slug in [slug_no_ext, slug_basename]:
|
||||||
|
if slug in claim_source_idx:
|
||||||
|
for claim_path in claim_source_idx[slug]:
|
||||||
|
if claim_path not in targets:
|
||||||
|
targets.append(claim_path)
|
||||||
|
match_types.append("wiki→claim")
|
||||||
|
|
||||||
|
# Match 3: entity name matching (for launches/fundraises)
|
||||||
|
title = get_title(fm) or ""
|
||||||
|
# Extract project name from title like "Futardio: ProjectName ..."
|
||||||
|
title_match = re.match(r'Futardio:\s*(.+?)(?:\s*[-—]|\s+Launch|\s+Fundraise|$)', title, re.IGNORECASE)
|
||||||
|
if title_match:
|
||||||
|
project_name = title_match.group(1).strip().lower().replace("-", " ")
|
||||||
|
if project_name in entity_name_idx:
|
||||||
|
entity_path = entity_name_idx[project_name]
|
||||||
|
if entity_path not in targets:
|
||||||
|
targets.append(entity_path)
|
||||||
|
match_types.append("name→entity")
|
||||||
|
|
||||||
|
if targets:
|
||||||
|
matched.append((src, targets, match_types))
|
||||||
|
elif is_futardio_source(src):
|
||||||
|
futardio_unmatched.append(src)
|
||||||
|
else:
|
||||||
|
genuine_backlog.append(src)
|
||||||
|
|
||||||
|
print(f"Results:")
|
||||||
|
print(f" Matched: {len(matched)}")
|
||||||
|
print(f" Test/spam: {len(test_spam)}")
|
||||||
|
print(f" Futardio unmatched (→ null-result): {len(futardio_unmatched)}")
|
||||||
|
print(f" Genuine backlog (kept unprocessed): {len(genuine_backlog)}")
|
||||||
|
print()
|
||||||
|
|
||||||
|
# Validate all link targets exist
|
||||||
|
broken_links = []
|
||||||
|
for src, targets, _ in matched:
|
||||||
|
for t in targets:
|
||||||
|
if isinstance(t, Path) and not t.exists():
|
||||||
|
broken_links.append((src, t))
|
||||||
|
|
||||||
|
if broken_links:
|
||||||
|
print(f"ERROR: {len(broken_links)} broken link targets!")
|
||||||
|
for src, target in broken_links:
|
||||||
|
print(f" {rel_path(src)} → {rel_path(target)}")
|
||||||
|
if not DRY_RUN:
|
||||||
|
print("Aborting — fix broken links first.")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
# Show match samples
|
||||||
|
print("Sample matches:")
|
||||||
|
for src, targets, types in matched[:5]:
|
||||||
|
print(f" {src.name}")
|
||||||
|
for t, mt in zip(targets, types):
|
||||||
|
print(f" → {rel_path(t)} ({mt})")
|
||||||
|
print()
|
||||||
|
|
||||||
|
# Show test/spam samples
|
||||||
|
if test_spam:
|
||||||
|
print(f"Test/spam samples ({len(test_spam)} total):")
|
||||||
|
for src in test_spam[:5]:
|
||||||
|
print(f" {src.name}")
|
||||||
|
print()
|
||||||
|
|
||||||
|
# Show futardio unmatched samples
|
||||||
|
if futardio_unmatched:
|
||||||
|
print(f"Futardio unmatched samples ({len(futardio_unmatched)} total):")
|
||||||
|
for src in futardio_unmatched[:10]:
|
||||||
|
print(f" {src.name}")
|
||||||
|
print()
|
||||||
|
|
||||||
|
# Show genuine backlog
|
||||||
|
if genuine_backlog:
|
||||||
|
print(f"Genuine backlog — kept unprocessed ({len(genuine_backlog)} total):")
|
||||||
|
from collections import Counter
|
||||||
|
backlog_domains = Counter()
|
||||||
|
for src in genuine_backlog:
|
||||||
|
parts = src.relative_to(ARCHIVE_DIR).parts
|
||||||
|
domain = parts[0] if len(parts) > 1 else "root"
|
||||||
|
backlog_domains[domain] += 1
|
||||||
|
for d, c in backlog_domains.most_common():
|
||||||
|
print(f" {d}: {c}")
|
||||||
|
print()
|
||||||
|
|
||||||
|
if DRY_RUN:
|
||||||
|
print("=== DRY RUN — no changes made. Use --apply to apply. ===")
|
||||||
|
return
|
||||||
|
|
||||||
|
# --- Apply changes ---
|
||||||
|
files_modified = 0
|
||||||
|
links_created = 0
|
||||||
|
|
||||||
|
# 1. Matched sources → processed + bidirectional links
|
||||||
|
for src, targets, _ in matched:
|
||||||
|
# Update source status
|
||||||
|
new_content = set_status(src, "processed")
|
||||||
|
if new_content:
|
||||||
|
# Also add derived_items
|
||||||
|
decision_entity_targets = [
|
||||||
|
rel_path(t) for t in targets
|
||||||
|
if isinstance(t, Path) and (
|
||||||
|
str(t).startswith(str(DECISIONS_DIR)) or
|
||||||
|
str(t).startswith(str(ENTITIES_DIR))
|
||||||
|
)
|
||||||
|
]
|
||||||
|
if decision_entity_targets:
|
||||||
|
# Add derived_items to the already-modified content
|
||||||
|
# Write status change first, then add field
|
||||||
|
src.write_text(new_content, encoding="utf-8")
|
||||||
|
linked = add_frontmatter_field(src, "derived_items", decision_entity_targets)
|
||||||
|
if linked:
|
||||||
|
src.write_text(linked, encoding="utf-8")
|
||||||
|
links_created += len(decision_entity_targets)
|
||||||
|
else:
|
||||||
|
src.write_text(new_content, encoding="utf-8")
|
||||||
|
files_modified += 1
|
||||||
|
|
||||||
|
# Add source_archive to decision/entity targets
|
||||||
|
src_rel = rel_path(src)
|
||||||
|
for t in targets:
|
||||||
|
if isinstance(t, Path) and (
|
||||||
|
str(t).startswith(str(DECISIONS_DIR)) or
|
||||||
|
str(t).startswith(str(ENTITIES_DIR))
|
||||||
|
):
|
||||||
|
linked = add_frontmatter_field(t, "source_archive", src_rel)
|
||||||
|
if linked:
|
||||||
|
t.write_text(linked, encoding="utf-8")
|
||||||
|
files_modified += 1
|
||||||
|
links_created += 1
|
||||||
|
|
||||||
|
# 2. Test/spam → null-result
|
||||||
|
for src in test_spam:
|
||||||
|
new_content = set_status(src, "null-result")
|
||||||
|
if new_content:
|
||||||
|
src.write_text(new_content, encoding="utf-8")
|
||||||
|
files_modified += 1
|
||||||
|
|
||||||
|
# 3. Futardio unmatched → null-result (no extraction output, won't be re-extracted)
|
||||||
|
for src in futardio_unmatched:
|
||||||
|
new_content = set_status(src, "null-result")
|
||||||
|
if new_content:
|
||||||
|
src.write_text(new_content, encoding="utf-8")
|
||||||
|
files_modified += 1
|
||||||
|
|
||||||
|
# 4. Genuine backlog → KEEP unprocessed (these are real extraction targets)
|
||||||
|
# No changes needed
|
||||||
|
|
||||||
|
print(f"\n=== APPLIED ===")
|
||||||
|
print(f"Files modified: {files_modified}")
|
||||||
|
print(f"Bidirectional links created: {links_created}")
|
||||||
|
print(f"Matched → processed: {len(matched)}")
|
||||||
|
print(f"Test/spam → null-result: {len(test_spam)}")
|
||||||
|
print(f"Futardio unmatched → null-result: {len(futardio_unmatched)}")
|
||||||
|
print(f"Genuine backlog → kept unprocessed: {len(genuine_backlog)}")
|
||||||
|
|
||||||
|
# Verify
|
||||||
|
remaining = 0
|
||||||
|
for f in ARCHIVE_DIR.rglob("*.md"):
|
||||||
|
if ".extraction-debug" in str(f):
|
||||||
|
continue
|
||||||
|
fm, _, _ = read_frontmatter(f)
|
||||||
|
if get_status(fm) == "unprocessed":
|
||||||
|
remaining += 1
|
||||||
|
print(f"\nRemaining unprocessed: {remaining}")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
901
reweave.py
Normal file
901
reweave.py
Normal file
|
|
@ -0,0 +1,901 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Orphan Reweave — connect isolated claims via vector similarity + Haiku classification.
|
||||||
|
|
||||||
|
Finds claims with zero incoming links (orphans), uses Qdrant to find semantically
|
||||||
|
similar neighbors, classifies the relationship with Haiku, and writes edges on the
|
||||||
|
neighbor's frontmatter pointing TO the orphan.
|
||||||
|
|
||||||
|
Usage:
|
||||||
|
python3 reweave.py --dry-run # Show what would be connected
|
||||||
|
python3 reweave.py --max-orphans 50 # Process up to 50 orphans
|
||||||
|
python3 reweave.py --threshold 0.72 # Override similarity floor
|
||||||
|
|
||||||
|
Design:
|
||||||
|
- Orphan = zero incoming links (no other claim's supports/challenges/related/depends_on points to it)
|
||||||
|
- Write edge on NEIGHBOR (not orphan) so orphan gains an incoming link
|
||||||
|
- Haiku classifies: supports | challenges | related (>=0.85 confidence for supports/challenges)
|
||||||
|
- reweave_edges parallel field for tooling-readable provenance
|
||||||
|
- Single PR per run for Leo review
|
||||||
|
|
||||||
|
Pentagon-Agent: Epimetheus <0144398e-4ed3-4fe2-95a3-3d72e1abf887>
|
||||||
|
"""
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import datetime
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
import urllib.request
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import yaml
|
||||||
|
|
||||||
|
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
|
||||||
|
logger = logging.getLogger("reweave")
|
||||||
|
|
||||||
|
# --- Config ---
|
||||||
|
REPO_DIR = Path(os.environ.get("REPO_DIR", "/opt/teleo-eval/workspaces/main"))
|
||||||
|
SECRETS_DIR = Path(os.environ.get("SECRETS_DIR", "/opt/teleo-eval/secrets"))
|
||||||
|
QDRANT_URL = os.environ.get("QDRANT_URL", "http://localhost:6333")
|
||||||
|
QDRANT_COLLECTION = os.environ.get("QDRANT_COLLECTION", "teleo-claims")
|
||||||
|
FORGEJO_URL = os.environ.get("FORGEJO_URL", "http://localhost:3000")
|
||||||
|
|
||||||
|
EMBED_DIRS = ["domains", "core", "foundations", "decisions", "entities"]
|
||||||
|
EDGE_FIELDS = ("supports", "challenges", "depends_on", "related")
|
||||||
|
WIKI_LINK_RE = re.compile(r"\[\[([^\]]+)\]\]")
|
||||||
|
|
||||||
|
# Thresholds (from calibration data — Mar 28)
|
||||||
|
DEFAULT_THRESHOLD = 0.70 # Elbow in score distribution
|
||||||
|
DEFAULT_MAX_ORPHANS = 50 # Keep PRs reviewable
|
||||||
|
DEFAULT_MAX_NEIGHBORS = 3 # Don't over-connect
|
||||||
|
HAIKU_CONFIDENCE_FLOOR = 0.85 # Below this → default to "related"
|
||||||
|
PER_FILE_EDGE_CAP = 10 # Max total reweave edges per neighbor file
|
||||||
|
|
||||||
|
# Domain processing order: diversity first, internet-finance last (Leo)
|
||||||
|
DOMAIN_PRIORITY = [
|
||||||
|
"ai-alignment", "health", "space-development", "entertainment",
|
||||||
|
"creative-industries", "collective-intelligence", "governance",
|
||||||
|
# internet-finance last — batch-imported futarchy cluster, lower cross-domain value
|
||||||
|
"internet-finance",
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
# ─── Orphan Detection ────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_frontmatter(path: Path) -> dict | None:
|
||||||
|
"""Parse YAML frontmatter from a markdown file. Returns dict or None."""
|
||||||
|
try:
|
||||||
|
text = path.read_text(errors="replace")
|
||||||
|
except Exception:
|
||||||
|
return None
|
||||||
|
if not text.startswith("---"):
|
||||||
|
return None
|
||||||
|
end = text.find("\n---", 3)
|
||||||
|
if end == -1:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
fm = yaml.safe_load(text[3:end])
|
||||||
|
return fm if isinstance(fm, dict) else None
|
||||||
|
except Exception:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _get_body(path: Path) -> str:
|
||||||
|
"""Get body text (after frontmatter) from a markdown file."""
|
||||||
|
try:
|
||||||
|
text = path.read_text(errors="replace")
|
||||||
|
except Exception:
|
||||||
|
return ""
|
||||||
|
if not text.startswith("---"):
|
||||||
|
return text
|
||||||
|
end = text.find("\n---", 3)
|
||||||
|
if end == -1:
|
||||||
|
return text
|
||||||
|
return text[end + 4:].strip()
|
||||||
|
|
||||||
|
|
||||||
|
def _get_edge_targets(path: Path) -> list[str]:
|
||||||
|
"""Extract all outgoing edge targets from a claim's frontmatter + wiki links."""
|
||||||
|
targets = []
|
||||||
|
fm = _parse_frontmatter(path)
|
||||||
|
if fm:
|
||||||
|
for field in EDGE_FIELDS:
|
||||||
|
val = fm.get(field)
|
||||||
|
if isinstance(val, list):
|
||||||
|
targets.extend(str(v).strip().lower() for v in val if v)
|
||||||
|
elif isinstance(val, str) and val.strip():
|
||||||
|
targets.append(val.strip().lower())
|
||||||
|
# Also check reweave_edges (from previous runs)
|
||||||
|
rw = fm.get("reweave_edges")
|
||||||
|
if isinstance(rw, list):
|
||||||
|
targets.extend(str(v).strip().lower() for v in rw if v)
|
||||||
|
|
||||||
|
# Wiki links in body
|
||||||
|
try:
|
||||||
|
text = path.read_text(errors="replace")
|
||||||
|
end = text.find("\n---", 3)
|
||||||
|
if end > 0:
|
||||||
|
body = text[end + 4:]
|
||||||
|
for link in WIKI_LINK_RE.findall(body):
|
||||||
|
targets.append(link.strip().lower())
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
return targets
|
||||||
|
|
||||||
|
|
||||||
|
def _claim_name_variants(path: Path, repo_root: Path = None) -> list[str]:
|
||||||
|
"""Generate name variants for a claim file (used for incoming link matching).
|
||||||
|
|
||||||
|
A claim at domains/ai-alignment/rlhf-reward-hacking.md could be referenced as:
|
||||||
|
- "rlhf-reward-hacking"
|
||||||
|
- "rlhf reward hacking"
|
||||||
|
- "RLHF reward hacking" (title case)
|
||||||
|
- The actual 'name' or 'title' from frontmatter
|
||||||
|
- "domains/ai-alignment/rlhf-reward-hacking" (relative path without .md)
|
||||||
|
"""
|
||||||
|
variants = set()
|
||||||
|
stem = path.stem
|
||||||
|
variants.add(stem.lower())
|
||||||
|
variants.add(stem.lower().replace("-", " "))
|
||||||
|
|
||||||
|
# Also match by relative path (Ganymede Q1: some edges use path references)
|
||||||
|
if repo_root:
|
||||||
|
try:
|
||||||
|
rel = str(path.relative_to(repo_root)).removesuffix(".md")
|
||||||
|
variants.add(rel.lower())
|
||||||
|
except ValueError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
fm = _parse_frontmatter(path)
|
||||||
|
if fm:
|
||||||
|
for key in ("name", "title"):
|
||||||
|
val = fm.get(key)
|
||||||
|
if isinstance(val, str) and val.strip():
|
||||||
|
variants.add(val.strip().lower())
|
||||||
|
|
||||||
|
return list(variants)
|
||||||
|
|
||||||
|
|
||||||
|
def find_all_claims(repo_root: Path) -> list[Path]:
|
||||||
|
"""Find all knowledge files (claim, framework, entity, decision) in the KB."""
|
||||||
|
claims = []
|
||||||
|
for d in EMBED_DIRS:
|
||||||
|
base = repo_root / d
|
||||||
|
if not base.is_dir():
|
||||||
|
continue
|
||||||
|
for md in base.rglob("*.md"):
|
||||||
|
if md.name.startswith("_"):
|
||||||
|
continue
|
||||||
|
fm = _parse_frontmatter(md)
|
||||||
|
if fm and fm.get("type") not in ("source", "musing", None):
|
||||||
|
claims.append(md)
|
||||||
|
return claims
|
||||||
|
|
||||||
|
|
||||||
|
def build_reverse_link_index(claims: list[Path]) -> dict[str, set[Path]]:
|
||||||
|
"""Build a reverse index: claim_name_variant → set of files that link TO it.
|
||||||
|
|
||||||
|
For each claim, extract all outgoing edges. For each target name, record
|
||||||
|
the source claim as an incoming link for that target.
|
||||||
|
"""
|
||||||
|
# name_variant → set of source paths that point to it
|
||||||
|
incoming: dict[str, set[Path]] = {}
|
||||||
|
|
||||||
|
for claim_path in claims:
|
||||||
|
targets = _get_edge_targets(claim_path)
|
||||||
|
for target in targets:
|
||||||
|
if target not in incoming:
|
||||||
|
incoming[target] = set()
|
||||||
|
incoming[target].add(claim_path)
|
||||||
|
|
||||||
|
return incoming
|
||||||
|
|
||||||
|
|
||||||
|
def find_orphans(claims: list[Path], incoming: dict[str, set[Path]],
|
||||||
|
repo_root: Path = None) -> list[Path]:
|
||||||
|
"""Find claims with zero incoming links."""
|
||||||
|
orphans = []
|
||||||
|
for claim_path in claims:
|
||||||
|
variants = _claim_name_variants(claim_path, repo_root)
|
||||||
|
has_incoming = any(
|
||||||
|
len(incoming.get(v, set()) - {claim_path}) > 0
|
||||||
|
for v in variants
|
||||||
|
)
|
||||||
|
if not has_incoming:
|
||||||
|
orphans.append(claim_path)
|
||||||
|
return orphans
|
||||||
|
|
||||||
|
|
||||||
|
def sort_orphans_by_domain(orphans: list[Path], repo_root: Path) -> list[Path]:
|
||||||
|
"""Sort orphans by domain priority (diversity first, internet-finance last)."""
|
||||||
|
def domain_key(path: Path) -> tuple[int, str]:
|
||||||
|
rel = path.relative_to(repo_root)
|
||||||
|
parts = rel.parts
|
||||||
|
domain = ""
|
||||||
|
if len(parts) >= 2 and parts[0] in ("domains", "entities", "decisions"):
|
||||||
|
domain = parts[1]
|
||||||
|
elif parts[0] == "foundations" and len(parts) >= 2:
|
||||||
|
domain = parts[1]
|
||||||
|
elif parts[0] == "core":
|
||||||
|
domain = "core"
|
||||||
|
|
||||||
|
try:
|
||||||
|
priority = DOMAIN_PRIORITY.index(domain)
|
||||||
|
except ValueError:
|
||||||
|
# Unknown domain goes before internet-finance but after known ones
|
||||||
|
priority = len(DOMAIN_PRIORITY) - 1
|
||||||
|
|
||||||
|
return (priority, path.stem)
|
||||||
|
|
||||||
|
return sorted(orphans, key=domain_key)
|
||||||
|
|
||||||
|
|
||||||
|
# ─── Qdrant Search ───────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def _get_api_key() -> str:
|
||||||
|
"""Load OpenRouter API key."""
|
||||||
|
key_file = SECRETS_DIR / "openrouter-key"
|
||||||
|
if key_file.exists():
|
||||||
|
return key_file.read_text().strip()
|
||||||
|
key = os.environ.get("OPENROUTER_API_KEY", "")
|
||||||
|
if key:
|
||||||
|
return key
|
||||||
|
logger.error("No OpenRouter API key found")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
|
||||||
|
def make_point_id(rel_path: str) -> str:
|
||||||
|
"""Deterministic point ID from repo-relative path (matches embed-claims.py)."""
|
||||||
|
return hashlib.md5(rel_path.encode()).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def get_vector_from_qdrant(rel_path: str) -> list[float] | None:
|
||||||
|
"""Retrieve a claim's existing vector from Qdrant by its point ID."""
|
||||||
|
point_id = make_point_id(rel_path)
|
||||||
|
body = json.dumps({"ids": [point_id], "with_vector": True}).encode()
|
||||||
|
req = urllib.request.Request(
|
||||||
|
f"{QDRANT_URL}/collections/{QDRANT_COLLECTION}/points",
|
||||||
|
data=body,
|
||||||
|
headers={"Content-Type": "application/json"},
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
with urllib.request.urlopen(req, timeout=10) as resp:
|
||||||
|
data = json.loads(resp.read())
|
||||||
|
points = data.get("result", [])
|
||||||
|
if points and points[0].get("vector"):
|
||||||
|
return points[0]["vector"]
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning("Qdrant point lookup failed for %s: %s", rel_path, e)
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def search_neighbors(vector: list[float], exclude_path: str,
|
||||||
|
threshold: float, limit: int) -> list[dict]:
|
||||||
|
"""Search Qdrant for nearest neighbors above threshold, excluding self."""
|
||||||
|
body = {
|
||||||
|
"vector": vector,
|
||||||
|
"limit": limit + 5, # over-fetch to account for self + filtered
|
||||||
|
"with_payload": True,
|
||||||
|
"score_threshold": threshold,
|
||||||
|
"filter": {
|
||||||
|
"must_not": [{"key": "claim_path", "match": {"value": exclude_path}}]
|
||||||
|
},
|
||||||
|
}
|
||||||
|
req = urllib.request.Request(
|
||||||
|
f"{QDRANT_URL}/collections/{QDRANT_COLLECTION}/points/search",
|
||||||
|
data=json.dumps(body).encode(),
|
||||||
|
headers={"Content-Type": "application/json"},
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
with urllib.request.urlopen(req, timeout=10) as resp:
|
||||||
|
data = json.loads(resp.read())
|
||||||
|
hits = data.get("result", [])
|
||||||
|
return hits[:limit]
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning("Qdrant search failed: %s", e)
|
||||||
|
return []
|
||||||
|
|
||||||
|
|
||||||
|
# ─── Haiku Edge Classification ───────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
CLASSIFY_PROMPT = """You are classifying the relationship between two knowledge claims.
|
||||||
|
|
||||||
|
CLAIM A (the orphan — needs to be connected):
|
||||||
|
Title: {orphan_title}
|
||||||
|
Body: {orphan_body}
|
||||||
|
|
||||||
|
CLAIM B (the neighbor — already connected in the knowledge graph):
|
||||||
|
Title: {neighbor_title}
|
||||||
|
Body: {neighbor_body}
|
||||||
|
|
||||||
|
What is the relationship FROM Claim B TO Claim A?
|
||||||
|
|
||||||
|
Options:
|
||||||
|
- "supports" — Claim B provides evidence, reasoning, or examples that strengthen Claim A
|
||||||
|
- "challenges" — Claim B contradicts, undermines, or provides counter-evidence to Claim A
|
||||||
|
- "related" — Claims are topically connected but neither supports nor challenges the other
|
||||||
|
|
||||||
|
Respond with EXACTLY this JSON format, nothing else:
|
||||||
|
{{"edge_type": "supports|challenges|related", "confidence": 0.0-1.0, "reason": "one sentence explanation"}}
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
def classify_edge(orphan_title: str, orphan_body: str,
|
||||||
|
neighbor_title: str, neighbor_body: str,
|
||||||
|
api_key: str) -> dict:
|
||||||
|
"""Use Haiku to classify the edge type between two claims.
|
||||||
|
|
||||||
|
Returns {"edge_type": str, "confidence": float, "reason": str}.
|
||||||
|
Falls back to "related" on any failure.
|
||||||
|
"""
|
||||||
|
default = {"edge_type": "related", "confidence": 0.5, "reason": "classification failed"}
|
||||||
|
|
||||||
|
prompt = CLASSIFY_PROMPT.format(
|
||||||
|
orphan_title=orphan_title,
|
||||||
|
orphan_body=orphan_body[:500],
|
||||||
|
neighbor_title=neighbor_title,
|
||||||
|
neighbor_body=neighbor_body[:500],
|
||||||
|
)
|
||||||
|
|
||||||
|
payload = json.dumps({
|
||||||
|
"model": "anthropic/claude-3.5-haiku",
|
||||||
|
"messages": [{"role": "user", "content": prompt}],
|
||||||
|
"max_tokens": 200,
|
||||||
|
"temperature": 0.1,
|
||||||
|
}).encode()
|
||||||
|
|
||||||
|
req = urllib.request.Request(
|
||||||
|
"https://openrouter.ai/api/v1/chat/completions",
|
||||||
|
data=payload,
|
||||||
|
headers={
|
||||||
|
"Authorization": f"Bearer {api_key}",
|
||||||
|
"Content-Type": "application/json",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
|
with urllib.request.urlopen(req, timeout=15) as resp:
|
||||||
|
data = json.loads(resp.read())
|
||||||
|
content = data["choices"][0]["message"]["content"].strip()
|
||||||
|
|
||||||
|
# Parse JSON from response (handle markdown code blocks)
|
||||||
|
if content.startswith("```"):
|
||||||
|
content = content.split("\n", 1)[-1].rsplit("```", 1)[0].strip()
|
||||||
|
|
||||||
|
result = json.loads(content)
|
||||||
|
edge_type = result.get("edge_type", "related")
|
||||||
|
confidence = float(result.get("confidence", 0.5))
|
||||||
|
|
||||||
|
# Enforce confidence floor for supports/challenges
|
||||||
|
if edge_type in ("supports", "challenges") and confidence < HAIKU_CONFIDENCE_FLOOR:
|
||||||
|
edge_type = "related"
|
||||||
|
|
||||||
|
return {
|
||||||
|
"edge_type": edge_type,
|
||||||
|
"confidence": confidence,
|
||||||
|
"reason": result.get("reason", ""),
|
||||||
|
}
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning("Haiku classification failed: %s", e)
|
||||||
|
return default
|
||||||
|
|
||||||
|
|
||||||
|
# ─── YAML Frontmatter Editing ────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def _count_reweave_edges(path: Path) -> int:
|
||||||
|
"""Count existing reweave_edges in a file's frontmatter."""
|
||||||
|
fm = _parse_frontmatter(path)
|
||||||
|
if not fm:
|
||||||
|
return 0
|
||||||
|
rw = fm.get("reweave_edges")
|
||||||
|
if isinstance(rw, list):
|
||||||
|
return len(rw)
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
def write_edge(neighbor_path: Path, orphan_title: str, edge_type: str,
|
||||||
|
date_str: str, dry_run: bool = False) -> bool:
|
||||||
|
"""Write a reweave edge on the neighbor's frontmatter.
|
||||||
|
|
||||||
|
Adds to both the edge_type list (related/supports/challenges) and
|
||||||
|
the parallel reweave_edges list for provenance tracking.
|
||||||
|
|
||||||
|
Uses ruamel.yaml for round-trip YAML preservation.
|
||||||
|
"""
|
||||||
|
# Check per-file cap
|
||||||
|
if _count_reweave_edges(neighbor_path) >= PER_FILE_EDGE_CAP:
|
||||||
|
logger.info(" Skip %s — per-file edge cap (%d) reached", neighbor_path.name, PER_FILE_EDGE_CAP)
|
||||||
|
return False
|
||||||
|
|
||||||
|
try:
|
||||||
|
text = neighbor_path.read_text(errors="replace")
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning(" Cannot read %s: %s", neighbor_path, e)
|
||||||
|
return False
|
||||||
|
|
||||||
|
if not text.startswith("---"):
|
||||||
|
logger.warning(" No frontmatter in %s", neighbor_path.name)
|
||||||
|
return False
|
||||||
|
|
||||||
|
end = text.find("\n---", 3)
|
||||||
|
if end == -1:
|
||||||
|
return False
|
||||||
|
|
||||||
|
fm_text = text[3:end]
|
||||||
|
body_text = text[end:] # includes the closing ---
|
||||||
|
|
||||||
|
# Try ruamel.yaml for round-trip editing
|
||||||
|
try:
|
||||||
|
from ruamel.yaml import YAML
|
||||||
|
ry = YAML()
|
||||||
|
ry.preserve_quotes = True
|
||||||
|
ry.width = 4096 # prevent line wrapping
|
||||||
|
|
||||||
|
import io
|
||||||
|
fm = ry.load(fm_text)
|
||||||
|
if not isinstance(fm, dict):
|
||||||
|
return False
|
||||||
|
|
||||||
|
# Add to edge_type list (related/supports/challenges)
|
||||||
|
# Clean value only — provenance tracked in reweave_edges (Ganymede: comment-in-string bug)
|
||||||
|
if edge_type not in fm:
|
||||||
|
fm[edge_type] = []
|
||||||
|
elif not isinstance(fm[edge_type], list):
|
||||||
|
fm[edge_type] = [fm[edge_type]]
|
||||||
|
|
||||||
|
# Check for duplicate
|
||||||
|
existing = [str(v).strip().lower() for v in fm[edge_type] if v]
|
||||||
|
if orphan_title.strip().lower() in existing:
|
||||||
|
logger.info(" Skip duplicate edge: %s → %s", neighbor_path.name, orphan_title)
|
||||||
|
return False
|
||||||
|
|
||||||
|
fm[edge_type].append(orphan_title)
|
||||||
|
|
||||||
|
# Add to reweave_edges with provenance (edge_type + date for audit trail)
|
||||||
|
if "reweave_edges" not in fm:
|
||||||
|
fm["reweave_edges"] = []
|
||||||
|
elif not isinstance(fm["reweave_edges"], list):
|
||||||
|
fm["reweave_edges"] = [fm["reweave_edges"]]
|
||||||
|
fm["reweave_edges"].append(f"{orphan_title}|{edge_type}|{date_str}")
|
||||||
|
|
||||||
|
# Serialize back
|
||||||
|
buf = io.StringIO()
|
||||||
|
ry.dump(fm, buf)
|
||||||
|
new_fm = buf.getvalue().rstrip("\n")
|
||||||
|
|
||||||
|
new_text = f"---\n{new_fm}{body_text}"
|
||||||
|
|
||||||
|
if not dry_run:
|
||||||
|
neighbor_path.write_text(new_text)
|
||||||
|
return True
|
||||||
|
|
||||||
|
except ImportError:
|
||||||
|
# Fallback: regex-based editing (no ruamel.yaml installed)
|
||||||
|
logger.info(" ruamel.yaml not available, using regex fallback")
|
||||||
|
return _write_edge_regex(neighbor_path, fm_text, body_text, orphan_title,
|
||||||
|
edge_type, date_str, dry_run)
|
||||||
|
|
||||||
|
|
||||||
|
def _write_edge_regex(neighbor_path: Path, fm_text: str, body_text: str,
|
||||||
|
orphan_title: str, edge_type: str, date_str: str,
|
||||||
|
dry_run: bool) -> bool:
|
||||||
|
"""Fallback: add edge via regex when ruamel.yaml is unavailable."""
|
||||||
|
# Check if edge_type field exists
|
||||||
|
field_re = re.compile(rf"^{edge_type}:\s*$", re.MULTILINE)
|
||||||
|
inline_re = re.compile(rf'^{edge_type}:\s*\[', re.MULTILINE)
|
||||||
|
|
||||||
|
entry_line = f' - "{orphan_title}"'
|
||||||
|
rw_line = f' - "{orphan_title}|{edge_type}|{date_str}"'
|
||||||
|
|
||||||
|
if field_re.search(fm_text):
|
||||||
|
# Multi-line list exists — find end of list, append
|
||||||
|
lines = fm_text.split("\n")
|
||||||
|
new_lines = []
|
||||||
|
in_field = False
|
||||||
|
inserted = False
|
||||||
|
for line in lines:
|
||||||
|
new_lines.append(line)
|
||||||
|
if re.match(rf"^{edge_type}:\s*$", line):
|
||||||
|
in_field = True
|
||||||
|
elif in_field and not line.startswith(" -"):
|
||||||
|
# End of list — insert before this line
|
||||||
|
new_lines.insert(-1, entry_line)
|
||||||
|
in_field = False
|
||||||
|
inserted = True
|
||||||
|
if in_field and not inserted:
|
||||||
|
# Field was last in frontmatter
|
||||||
|
new_lines.append(entry_line)
|
||||||
|
fm_text = "\n".join(new_lines)
|
||||||
|
|
||||||
|
elif inline_re.search(fm_text):
|
||||||
|
# Inline list — skip, too complex for regex
|
||||||
|
logger.warning(" Inline list format for %s in %s, skipping", edge_type, neighbor_path.name)
|
||||||
|
return False
|
||||||
|
else:
|
||||||
|
# Field doesn't exist — add at end of frontmatter
|
||||||
|
fm_text = fm_text.rstrip("\n") + f"\n{edge_type}:\n{entry_line}"
|
||||||
|
|
||||||
|
# Add reweave_edges field
|
||||||
|
if "reweave_edges:" in fm_text:
|
||||||
|
lines = fm_text.split("\n")
|
||||||
|
new_lines = []
|
||||||
|
in_rw = False
|
||||||
|
inserted_rw = False
|
||||||
|
for line in lines:
|
||||||
|
new_lines.append(line)
|
||||||
|
if re.match(r"^reweave_edges:\s*$", line):
|
||||||
|
in_rw = True
|
||||||
|
elif in_rw and not line.startswith(" -"):
|
||||||
|
new_lines.insert(-1, rw_line)
|
||||||
|
in_rw = False
|
||||||
|
inserted_rw = True
|
||||||
|
if in_rw and not inserted_rw:
|
||||||
|
new_lines.append(rw_line)
|
||||||
|
fm_text = "\n".join(new_lines)
|
||||||
|
else:
|
||||||
|
fm_text = fm_text.rstrip("\n") + f"\nreweave_edges:\n{rw_line}"
|
||||||
|
|
||||||
|
new_text = f"---\n{fm_text}{body_text}"
|
||||||
|
|
||||||
|
if not dry_run:
|
||||||
|
neighbor_path.write_text(new_text)
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
# ─── Git + PR ────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def create_branch(repo_root: Path, branch_name: str) -> bool:
|
||||||
|
"""Create and checkout a new branch."""
|
||||||
|
try:
|
||||||
|
subprocess.run(["git", "checkout", "-b", branch_name],
|
||||||
|
cwd=str(repo_root), check=True, capture_output=True)
|
||||||
|
return True
|
||||||
|
except subprocess.CalledProcessError as e:
|
||||||
|
logger.error("Failed to create branch %s: %s", branch_name, e.stderr.decode())
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def commit_and_push(repo_root: Path, branch_name: str, modified_files: list[Path],
|
||||||
|
orphan_count: int) -> bool:
|
||||||
|
"""Stage modified files, commit, and push."""
|
||||||
|
# Stage only modified files
|
||||||
|
for f in modified_files:
|
||||||
|
subprocess.run(["git", "add", str(f)], cwd=str(repo_root),
|
||||||
|
check=True, capture_output=True)
|
||||||
|
|
||||||
|
# Check if anything staged
|
||||||
|
result = subprocess.run(["git", "diff", "--cached", "--name-only"],
|
||||||
|
cwd=str(repo_root), capture_output=True, text=True)
|
||||||
|
if not result.stdout.strip():
|
||||||
|
logger.info("No files staged — nothing to commit")
|
||||||
|
return False
|
||||||
|
|
||||||
|
msg = (
|
||||||
|
f"reweave: connect {orphan_count} orphan claims via vector similarity\n\n"
|
||||||
|
f"Threshold: {DEFAULT_THRESHOLD}, Haiku classification, {len(modified_files)} files modified.\n\n"
|
||||||
|
f"Pentagon-Agent: Epimetheus <0144398e-4ed3-4fe2-95a3-3d72e1abf887>"
|
||||||
|
)
|
||||||
|
subprocess.run(["git", "commit", "-m", msg], cwd=str(repo_root),
|
||||||
|
check=True, capture_output=True)
|
||||||
|
|
||||||
|
# Push — inject token
|
||||||
|
token_file = SECRETS_DIR / "forgejo-admin-token"
|
||||||
|
if not token_file.exists():
|
||||||
|
logger.error("No Forgejo token found at %s", token_file)
|
||||||
|
return False
|
||||||
|
token = token_file.read_text().strip()
|
||||||
|
push_url = f"http://teleo:{token}@localhost:3000/teleo/teleo-codex.git"
|
||||||
|
|
||||||
|
subprocess.run(["git", "push", "-u", push_url, branch_name],
|
||||||
|
cwd=str(repo_root), check=True, capture_output=True)
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def create_pr(branch_name: str, orphan_count: int, summary_lines: list[str]) -> str | None:
|
||||||
|
"""Create a Forgejo PR for the reweave batch."""
|
||||||
|
token_file = SECRETS_DIR / "forgejo-admin-token"
|
||||||
|
if not token_file.exists():
|
||||||
|
return None
|
||||||
|
token = token_file.read_text().strip()
|
||||||
|
|
||||||
|
summary = "\n".join(f"- {line}" for line in summary_lines[:30])
|
||||||
|
body = (
|
||||||
|
f"## Orphan Reweave\n\n"
|
||||||
|
f"Connected **{orphan_count}** orphan claims to the knowledge graph "
|
||||||
|
f"via vector similarity (threshold {DEFAULT_THRESHOLD}) + Haiku edge classification.\n\n"
|
||||||
|
f"### Edges Added\n{summary}\n\n"
|
||||||
|
f"### Review Guide\n"
|
||||||
|
f"- Each edge has a `# reweave:YYYY-MM-DD` comment — strip after review\n"
|
||||||
|
f"- `reweave_edges` field tracks automated edges for tooling (graph_expand weights them 0.75x)\n"
|
||||||
|
f"- Upgrade `related` → `supports`/`challenges` where you have better judgment\n"
|
||||||
|
f"- Delete any edges that don't make sense\n\n"
|
||||||
|
f"Pentagon-Agent: Epimetheus"
|
||||||
|
)
|
||||||
|
|
||||||
|
payload = json.dumps({
|
||||||
|
"title": f"reweave: connect {orphan_count} orphan claims",
|
||||||
|
"body": body,
|
||||||
|
"head": branch_name,
|
||||||
|
"base": "main",
|
||||||
|
}).encode()
|
||||||
|
|
||||||
|
req = urllib.request.Request(
|
||||||
|
f"{FORGEJO_URL}/api/v1/repos/teleo/teleo-codex/pulls",
|
||||||
|
data=payload,
|
||||||
|
headers={
|
||||||
|
"Authorization": f"token {token}",
|
||||||
|
"Content-Type": "application/json",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
|
with urllib.request.urlopen(req, timeout=30) as resp:
|
||||||
|
data = json.loads(resp.read())
|
||||||
|
return data.get("html_url", "")
|
||||||
|
except Exception as e:
|
||||||
|
logger.error("PR creation failed: %s", e)
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
# ─── Worktree Lock ───────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
_lock_fd = None # Module-level to prevent GC and avoid function-attribute fragility
|
||||||
|
|
||||||
|
|
||||||
|
def acquire_lock(lock_path: Path, timeout: int = 30) -> bool:
|
||||||
|
"""Acquire file lock for worktree access. Returns True if acquired."""
|
||||||
|
global _lock_fd
|
||||||
|
import fcntl
|
||||||
|
try:
|
||||||
|
lock_path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
_lock_fd = open(lock_path, "w")
|
||||||
|
fcntl.flock(_lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||||
|
_lock_fd.write(f"reweave:{os.getpid()}\n")
|
||||||
|
_lock_fd.flush()
|
||||||
|
return True
|
||||||
|
except (IOError, OSError):
|
||||||
|
logger.warning("Could not acquire worktree lock at %s — another process has it", lock_path)
|
||||||
|
_lock_fd = None
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def release_lock(lock_path: Path):
|
||||||
|
"""Release worktree lock."""
|
||||||
|
global _lock_fd
|
||||||
|
import fcntl
|
||||||
|
fd = _lock_fd
|
||||||
|
_lock_fd = None
|
||||||
|
if fd:
|
||||||
|
try:
|
||||||
|
fcntl.flock(fd, fcntl.LOCK_UN)
|
||||||
|
fd.close()
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
try:
|
||||||
|
lock_path.unlink(missing_ok=True)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
# ─── Main ────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
global REPO_DIR, DEFAULT_THRESHOLD
|
||||||
|
|
||||||
|
parser = argparse.ArgumentParser(description="Orphan Reweave — connect isolated claims")
|
||||||
|
parser.add_argument("--dry-run", action="store_true",
|
||||||
|
help="Show what would be connected without modifying files")
|
||||||
|
parser.add_argument("--max-orphans", type=int, default=DEFAULT_MAX_ORPHANS,
|
||||||
|
help=f"Max orphans to process (default {DEFAULT_MAX_ORPHANS})")
|
||||||
|
parser.add_argument("--max-neighbors", type=int, default=DEFAULT_MAX_NEIGHBORS,
|
||||||
|
help=f"Max neighbors per orphan (default {DEFAULT_MAX_NEIGHBORS})")
|
||||||
|
parser.add_argument("--threshold", type=float, default=DEFAULT_THRESHOLD,
|
||||||
|
help=f"Minimum cosine similarity (default {DEFAULT_THRESHOLD})")
|
||||||
|
parser.add_argument("--repo-dir", type=str, default=None,
|
||||||
|
help="Override repo directory")
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
if args.repo_dir:
|
||||||
|
REPO_DIR = Path(args.repo_dir)
|
||||||
|
DEFAULT_THRESHOLD = args.threshold
|
||||||
|
|
||||||
|
date_str = datetime.date.today().isoformat()
|
||||||
|
branch_name = f"reweave/{date_str}"
|
||||||
|
|
||||||
|
logger.info("=== Orphan Reweave ===")
|
||||||
|
logger.info("Repo: %s", REPO_DIR)
|
||||||
|
logger.info("Threshold: %.2f, Max orphans: %d, Max neighbors: %d",
|
||||||
|
args.threshold, args.max_orphans, args.max_neighbors)
|
||||||
|
if args.dry_run:
|
||||||
|
logger.info("DRY RUN — no files will be modified")
|
||||||
|
|
||||||
|
# Step 1: Find all claims and build reverse-link index
|
||||||
|
logger.info("Step 1: Scanning KB for claims...")
|
||||||
|
claims = find_all_claims(REPO_DIR)
|
||||||
|
logger.info(" Found %d knowledge files", len(claims))
|
||||||
|
|
||||||
|
logger.info("Step 2: Building reverse-link index...")
|
||||||
|
incoming = build_reverse_link_index(claims)
|
||||||
|
|
||||||
|
logger.info("Step 3: Finding orphans...")
|
||||||
|
orphans = find_orphans(claims, incoming, REPO_DIR)
|
||||||
|
orphans = sort_orphans_by_domain(orphans, REPO_DIR)
|
||||||
|
logger.info(" Found %d orphans (%.1f%% of %d claims)",
|
||||||
|
len(orphans), 100 * len(orphans) / max(len(claims), 1), len(claims))
|
||||||
|
|
||||||
|
if not orphans:
|
||||||
|
logger.info("No orphans found — KB is fully connected!")
|
||||||
|
return
|
||||||
|
|
||||||
|
# Cap to max_orphans
|
||||||
|
batch = orphans[:args.max_orphans]
|
||||||
|
logger.info(" Processing batch of %d orphans", len(batch))
|
||||||
|
|
||||||
|
# Step 4: For each orphan, find neighbors and classify edges
|
||||||
|
api_key = _get_api_key()
|
||||||
|
edges_to_write: list[dict] = [] # {neighbor_path, orphan_title, edge_type, reason, score}
|
||||||
|
skipped_no_vector = 0
|
||||||
|
skipped_no_neighbors = 0
|
||||||
|
|
||||||
|
for i, orphan_path in enumerate(batch):
|
||||||
|
rel_path = str(orphan_path.relative_to(REPO_DIR))
|
||||||
|
fm = _parse_frontmatter(orphan_path)
|
||||||
|
orphan_title = fm.get("name", fm.get("title", orphan_path.stem.replace("-", " "))) if fm else orphan_path.stem
|
||||||
|
orphan_body = _get_body(orphan_path)
|
||||||
|
|
||||||
|
logger.info("[%d/%d] %s", i + 1, len(batch), orphan_title[:80])
|
||||||
|
|
||||||
|
# Get vector from Qdrant
|
||||||
|
vector = get_vector_from_qdrant(rel_path)
|
||||||
|
if not vector:
|
||||||
|
logger.info(" No vector in Qdrant — skipping (not embedded yet)")
|
||||||
|
skipped_no_vector += 1
|
||||||
|
continue
|
||||||
|
|
||||||
|
# Find neighbors
|
||||||
|
hits = search_neighbors(vector, rel_path, args.threshold, args.max_neighbors)
|
||||||
|
if not hits:
|
||||||
|
logger.info(" No neighbors above threshold %.2f", args.threshold)
|
||||||
|
skipped_no_neighbors += 1
|
||||||
|
continue
|
||||||
|
|
||||||
|
for hit in hits:
|
||||||
|
payload = hit.get("payload", {})
|
||||||
|
neighbor_rel = payload.get("claim_path", "")
|
||||||
|
neighbor_title = payload.get("claim_title", "")
|
||||||
|
score = hit.get("score", 0)
|
||||||
|
|
||||||
|
if not neighbor_rel:
|
||||||
|
continue
|
||||||
|
|
||||||
|
neighbor_path = REPO_DIR / neighbor_rel
|
||||||
|
if not neighbor_path.exists():
|
||||||
|
logger.info(" Neighbor %s not found on disk — skipping", neighbor_rel)
|
||||||
|
continue
|
||||||
|
|
||||||
|
neighbor_body = _get_body(neighbor_path)
|
||||||
|
|
||||||
|
# Classify with Haiku
|
||||||
|
result = classify_edge(orphan_title, orphan_body,
|
||||||
|
neighbor_title, neighbor_body, api_key)
|
||||||
|
edge_type = result["edge_type"]
|
||||||
|
confidence = result["confidence"]
|
||||||
|
reason = result["reason"]
|
||||||
|
|
||||||
|
logger.info(" → %s (%.3f) %s [%.2f]: %s",
|
||||||
|
neighbor_title[:50], score, edge_type, confidence, reason[:60])
|
||||||
|
|
||||||
|
edges_to_write.append({
|
||||||
|
"neighbor_path": neighbor_path,
|
||||||
|
"neighbor_rel": neighbor_rel,
|
||||||
|
"neighbor_title": neighbor_title,
|
||||||
|
"orphan_title": str(orphan_title),
|
||||||
|
"orphan_rel": rel_path,
|
||||||
|
"edge_type": edge_type,
|
||||||
|
"score": score,
|
||||||
|
"confidence": confidence,
|
||||||
|
"reason": reason,
|
||||||
|
})
|
||||||
|
|
||||||
|
# Rate limit courtesy
|
||||||
|
if not args.dry_run and i < len(batch) - 1:
|
||||||
|
time.sleep(0.3)
|
||||||
|
|
||||||
|
logger.info("\n=== Summary ===")
|
||||||
|
logger.info("Orphans processed: %d", len(batch))
|
||||||
|
logger.info("Edges to write: %d", len(edges_to_write))
|
||||||
|
logger.info("Skipped (no vector): %d", skipped_no_vector)
|
||||||
|
logger.info("Skipped (no neighbors): %d", skipped_no_neighbors)
|
||||||
|
|
||||||
|
if not edges_to_write:
|
||||||
|
logger.info("Nothing to write.")
|
||||||
|
return
|
||||||
|
|
||||||
|
if args.dry_run:
|
||||||
|
logger.info("\n=== Dry Run — Edges That Would Be Written ===")
|
||||||
|
for e in edges_to_write:
|
||||||
|
logger.info(" %s → [%s] → %s (score=%.3f, conf=%.2f)",
|
||||||
|
e["neighbor_title"][:40], e["edge_type"],
|
||||||
|
e["orphan_title"][:40], e["score"], e["confidence"])
|
||||||
|
return
|
||||||
|
|
||||||
|
# Step 5: Acquire lock, create branch, write edges, commit, push, create PR
|
||||||
|
lock_path = REPO_DIR.parent / ".main-worktree.lock"
|
||||||
|
if not acquire_lock(lock_path):
|
||||||
|
logger.error("Cannot acquire worktree lock — aborting")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
try:
|
||||||
|
# Create branch
|
||||||
|
if not create_branch(REPO_DIR, branch_name):
|
||||||
|
logger.error("Failed to create branch %s", branch_name)
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
# Write edges
|
||||||
|
modified_files = set()
|
||||||
|
written = 0
|
||||||
|
summary_lines = []
|
||||||
|
|
||||||
|
for e in edges_to_write:
|
||||||
|
ok = write_edge(
|
||||||
|
e["neighbor_path"], e["orphan_title"], e["edge_type"],
|
||||||
|
date_str, dry_run=False,
|
||||||
|
)
|
||||||
|
if ok:
|
||||||
|
modified_files.add(e["neighbor_path"])
|
||||||
|
written += 1
|
||||||
|
summary_lines.append(
|
||||||
|
f"`{e['neighbor_title'][:50]}` → [{e['edge_type']}] → "
|
||||||
|
f"`{e['orphan_title'][:50]}` (score={e['score']:.3f})"
|
||||||
|
)
|
||||||
|
|
||||||
|
logger.info("Wrote %d edges across %d files", written, len(modified_files))
|
||||||
|
|
||||||
|
if not modified_files:
|
||||||
|
logger.info("No edges written — cleaning up branch")
|
||||||
|
subprocess.run(["git", "checkout", "main"], cwd=str(REPO_DIR),
|
||||||
|
capture_output=True)
|
||||||
|
subprocess.run(["git", "branch", "-d", branch_name], cwd=str(REPO_DIR),
|
||||||
|
capture_output=True)
|
||||||
|
return
|
||||||
|
|
||||||
|
# Commit and push
|
||||||
|
orphan_count = len(set(e["orphan_title"] for e in edges_to_write if e["neighbor_path"] in modified_files))
|
||||||
|
if commit_and_push(REPO_DIR, branch_name, list(modified_files), orphan_count):
|
||||||
|
logger.info("Pushed branch %s", branch_name)
|
||||||
|
|
||||||
|
# Create PR
|
||||||
|
pr_url = create_pr(branch_name, orphan_count, summary_lines)
|
||||||
|
if pr_url:
|
||||||
|
logger.info("PR created: %s", pr_url)
|
||||||
|
else:
|
||||||
|
logger.warning("PR creation failed — branch is pushed, create manually")
|
||||||
|
else:
|
||||||
|
logger.error("Commit/push failed")
|
||||||
|
|
||||||
|
finally:
|
||||||
|
# Always return to main — even on exception (Ganymede: branch cleanup)
|
||||||
|
try:
|
||||||
|
subprocess.run(["git", "checkout", "main"], cwd=str(REPO_DIR),
|
||||||
|
capture_output=True)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
release_lock(lock_path)
|
||||||
|
|
||||||
|
logger.info("Done.")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
218
telegram/bot.py
218
telegram/bot.py
|
|
@ -41,6 +41,7 @@ from telegram.ext import (
|
||||||
)
|
)
|
||||||
|
|
||||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||||
|
import json as _json
|
||||||
from kb_retrieval import KBIndex, format_context_for_prompt, retrieve_context
|
from kb_retrieval import KBIndex, format_context_for_prompt, retrieve_context
|
||||||
from market_data import get_token_price, format_price_context
|
from market_data import get_token_price, format_price_context
|
||||||
from worktree_lock import main_worktree_lock
|
from worktree_lock import main_worktree_lock
|
||||||
|
|
@ -57,6 +58,10 @@ MAIN_WORKTREE = "/opt/teleo-eval/workspaces/main" # For git operations only
|
||||||
LEARNINGS_FILE = "/opt/teleo-eval/workspaces/main/agents/rio/learnings.md" # Agent memory (Option D)
|
LEARNINGS_FILE = "/opt/teleo-eval/workspaces/main/agents/rio/learnings.md" # Agent memory (Option D)
|
||||||
LOG_FILE = "/opt/teleo-eval/logs/telegram-bot.log"
|
LOG_FILE = "/opt/teleo-eval/logs/telegram-bot.log"
|
||||||
|
|
||||||
|
# Persistent audit connection — opened once at startup, reused for all writes
|
||||||
|
# (Ganymede + Rhea: no per-response sqlite3.connect / migrate)
|
||||||
|
_audit_conn: sqlite3.Connection | None = None
|
||||||
|
|
||||||
# Triage interval (seconds)
|
# Triage interval (seconds)
|
||||||
TRIAGE_INTERVAL = 900 # 15 minutes
|
TRIAGE_INTERVAL = 900 # 15 minutes
|
||||||
|
|
||||||
|
|
@ -828,6 +833,10 @@ async def handle_tagged(update: Update, context: ContextTypes.DEFAULT_TYPE):
|
||||||
|
|
||||||
logger.info("Tagged by @%s: %s", user.username if user else "unknown", text[:100])
|
logger.info("Tagged by @%s: %s", user.username if user else "unknown", text[:100])
|
||||||
|
|
||||||
|
# ─── Audit: init timing and tool call tracking ──────────────────
|
||||||
|
response_start = time.monotonic()
|
||||||
|
tool_calls = []
|
||||||
|
|
||||||
# Check for /research command — run search BEFORE Opus so results are in context
|
# Check for /research command — run search BEFORE Opus so results are in context
|
||||||
research_context = ""
|
research_context = ""
|
||||||
research_match = RESEARCH_PATTERN.search(text)
|
research_match = RESEARCH_PATTERN.search(text)
|
||||||
|
|
@ -885,6 +894,7 @@ async def handle_tagged(update: Update, context: ContextTypes.DEFAULT_TYPE):
|
||||||
logger.warning("Failed to fetch X link %s: %s", url, e)
|
logger.warning("Failed to fetch X link %s: %s", url, e)
|
||||||
|
|
||||||
# Haiku pre-pass: does this message need an X search? (Option A: two-pass)
|
# Haiku pre-pass: does this message need an X search? (Option A: two-pass)
|
||||||
|
t_haiku = time.monotonic()
|
||||||
if not research_context: # Skip if /research already ran
|
if not research_context: # Skip if /research already ran
|
||||||
try:
|
try:
|
||||||
haiku_prompt = (
|
haiku_prompt = (
|
||||||
|
|
@ -922,14 +932,94 @@ async def handle_tagged(update: Update, context: ContextTypes.DEFAULT_TYPE):
|
||||||
logger.warning("Haiku research archive failed: %s", e)
|
logger.warning("Haiku research archive failed: %s", e)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.warning("Haiku pre-pass failed: %s", e)
|
logger.warning("Haiku pre-pass failed: %s", e)
|
||||||
|
haiku_duration = int((time.monotonic() - t_haiku) * 1000)
|
||||||
|
if research_context:
|
||||||
|
tool_calls.append({
|
||||||
|
"tool": "haiku_prepass", "input": {"query": text[:200]},
|
||||||
|
"output": {"triggered": True, "result_length": len(research_context)},
|
||||||
|
"duration_ms": haiku_duration,
|
||||||
|
})
|
||||||
|
|
||||||
|
# ─── Query reformulation for follow-ups ────────────────────────
|
||||||
|
# Conversational follow-ups ("you're wrong", "tell me more") are unsearchable.
|
||||||
|
# Use Haiku to rewrite them into standalone queries using conversation context.
|
||||||
|
search_query_text = text # default: use raw message
|
||||||
|
user_key = (msg.chat_id, user.id if user else 0)
|
||||||
|
hist = conversation_history.get(user_key, [])
|
||||||
|
if hist:
|
||||||
|
# There's conversation history — check if this is a follow-up
|
||||||
|
try:
|
||||||
|
last_exchange = hist[-1]
|
||||||
|
recent_context = ""
|
||||||
|
if last_exchange.get("user"):
|
||||||
|
recent_context += f"User: {last_exchange['user'][:300]}\n"
|
||||||
|
if last_exchange.get("bot"):
|
||||||
|
recent_context += f"Bot: {last_exchange['bot'][:300]}\n"
|
||||||
|
reformulate_prompt = (
|
||||||
|
f"A user is in a conversation. Given the recent exchange and their new message, "
|
||||||
|
f"rewrite the new message as a STANDALONE search query that captures what they're "
|
||||||
|
f"actually asking about. The query should work for semantic search — specific topics, "
|
||||||
|
f"entities, and concepts.\n\n"
|
||||||
|
f"Recent exchange:\n{recent_context}\n"
|
||||||
|
f"New message: {text}\n\n"
|
||||||
|
f"If the message is already a clear standalone question or topic, return it unchanged.\n"
|
||||||
|
f"If it's a follow-up, correction, or reference to the conversation, rewrite it.\n\n"
|
||||||
|
f"Return ONLY the rewritten query, nothing else. Max 30 words."
|
||||||
|
)
|
||||||
|
reformulated = await call_openrouter("anthropic/claude-haiku-4.5", reformulate_prompt, max_tokens=80)
|
||||||
|
if reformulated and reformulated.strip() and len(reformulated.strip()) > 3:
|
||||||
|
search_query_text = reformulated.strip()
|
||||||
|
logger.info("Query reformulated: '%s' → '%s'", text[:60], search_query_text[:60])
|
||||||
|
tool_calls.append({
|
||||||
|
"tool": "query_reformulate", "input": {"original": text[:200], "history_turns": len(hist)},
|
||||||
|
"output": {"reformulated": search_query_text[:200]},
|
||||||
|
"duration_ms": 0, # included in haiku timing
|
||||||
|
})
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning("Query reformulation failed: %s", e)
|
||||||
|
# Fall through — use raw text
|
||||||
|
|
||||||
# Retrieve full KB context (entity resolution + claim search + agent positions)
|
# Retrieve full KB context (entity resolution + claim search + agent positions)
|
||||||
kb_ctx = retrieve_context(text, KB_READ_DIR, index=kb_index)
|
t_kb = time.monotonic()
|
||||||
|
kb_ctx = retrieve_context(search_query_text, KB_READ_DIR, index=kb_index)
|
||||||
kb_context_text = format_context_for_prompt(kb_ctx)
|
kb_context_text = format_context_for_prompt(kb_ctx)
|
||||||
|
kb_duration = int((time.monotonic() - t_kb) * 1000)
|
||||||
|
retrieval_layers = ["keyword"] if (kb_ctx and (kb_ctx.entities or kb_ctx.claims)) else []
|
||||||
|
tool_calls.append({
|
||||||
|
"tool": "retrieve_context",
|
||||||
|
"input": {"query": search_query_text[:200], "original_query": text[:200] if search_query_text != text else None},
|
||||||
|
"output": {"entities": len(kb_ctx.entities) if kb_ctx else 0,
|
||||||
|
"claims": len(kb_ctx.claims) if kb_ctx else 0},
|
||||||
|
"duration_ms": kb_duration,
|
||||||
|
})
|
||||||
|
|
||||||
|
# Layer 1+2: Qdrant vector search + graph expansion (semantic, complements keyword)
|
||||||
|
# Pass keyword-matched paths to exclude duplicates at Qdrant query level
|
||||||
|
# Normalize: KBIndex stores absolute paths, Qdrant stores repo-relative paths
|
||||||
|
keyword_paths = []
|
||||||
|
if kb_ctx and kb_ctx.claims:
|
||||||
|
for c in kb_ctx.claims:
|
||||||
|
p = c.path
|
||||||
|
if KB_READ_DIR and p.startswith(KB_READ_DIR):
|
||||||
|
p = p[len(KB_READ_DIR):].lstrip("/")
|
||||||
|
keyword_paths.append(p)
|
||||||
|
from kb_retrieval import retrieve_vector_context
|
||||||
|
vector_context, vector_meta = retrieve_vector_context(search_query_text, keyword_paths=keyword_paths)
|
||||||
|
if vector_context:
|
||||||
|
kb_context_text = kb_context_text + "\n\n" + vector_context
|
||||||
|
retrieval_layers.extend(vector_meta.get("layers_hit", []))
|
||||||
|
tool_calls.append({
|
||||||
|
"tool": "retrieve_qdrant_context", "input": {"query": text[:200]},
|
||||||
|
"output": {"direct_hits": len(vector_meta.get("direct_results", [])),
|
||||||
|
"expanded": len(vector_meta.get("expanded_results", []))},
|
||||||
|
"duration_ms": vector_meta.get("duration_ms", 0),
|
||||||
|
})
|
||||||
|
|
||||||
stats = get_db_stats()
|
stats = get_db_stats()
|
||||||
|
|
||||||
# Fetch live market data for any tokens mentioned (Rhea: market-data API)
|
# Fetch live market data for any tokens mentioned (Rhea: market-data API)
|
||||||
market_context = ""
|
market_context = ""
|
||||||
|
market_data_audit = {}
|
||||||
token_mentions = re.findall(r"\$([A-Z]{2,10})", text.upper())
|
token_mentions = re.findall(r"\$([A-Z]{2,10})", text.upper())
|
||||||
# Entity name → token mapping for natural language mentions
|
# Entity name → token mapping for natural language mentions
|
||||||
ENTITY_TOKEN_MAP = {
|
ENTITY_TOKEN_MAP = {
|
||||||
|
|
@ -945,6 +1035,7 @@ async def handle_tagged(update: Update, context: ContextTypes.DEFAULT_TYPE):
|
||||||
for tag in ent.tags:
|
for tag in ent.tags:
|
||||||
if tag.upper() in ENTITY_TOKEN_MAP.values():
|
if tag.upper() in ENTITY_TOKEN_MAP.values():
|
||||||
token_mentions.append(tag.upper())
|
token_mentions.append(tag.upper())
|
||||||
|
t_market = time.monotonic()
|
||||||
for token in set(token_mentions):
|
for token in set(token_mentions):
|
||||||
try:
|
try:
|
||||||
data = await get_token_price(token)
|
data = await get_token_price(token)
|
||||||
|
|
@ -952,8 +1043,16 @@ async def handle_tagged(update: Update, context: ContextTypes.DEFAULT_TYPE):
|
||||||
price_str = format_price_context(data, token)
|
price_str = format_price_context(data, token)
|
||||||
if price_str:
|
if price_str:
|
||||||
market_context += price_str + "\n"
|
market_context += price_str + "\n"
|
||||||
|
market_data_audit[token] = data
|
||||||
except Exception:
|
except Exception:
|
||||||
pass # Market data is supplementary — never block on failure
|
pass # Market data is supplementary — never block on failure
|
||||||
|
market_duration = int((time.monotonic() - t_market) * 1000)
|
||||||
|
if token_mentions:
|
||||||
|
tool_calls.append({
|
||||||
|
"tool": "market_data", "input": {"tickers": list(set(token_mentions))},
|
||||||
|
"output": market_data_audit,
|
||||||
|
"duration_ms": market_duration,
|
||||||
|
})
|
||||||
|
|
||||||
# Build Opus prompt — Rio's voice
|
# Build Opus prompt — Rio's voice
|
||||||
prompt = f"""You are Rio, the Teleo internet finance agent. Your Telegram handle is @FutAIrdBot — that IS you. Users tag @FutAIrdBot to reach you. Never say "I'm not FutAIrdBot." You are also @futaRdIO on X. You have deep knowledge about futarchy, prediction markets, token governance, and the MetaDAO ecosystem.
|
prompt = f"""You are Rio, the Teleo internet finance agent. Your Telegram handle is @FutAIrdBot — that IS you. Users tag @FutAIrdBot to reach you. Never say "I'm not FutAIrdBot." You are also @futaRdIO on X. You have deep knowledge about futarchy, prediction markets, token governance, and the MetaDAO ecosystem.
|
||||||
|
|
@ -1005,7 +1104,10 @@ IMPORTANT: Special tags you can append at the end of your response (after your m
|
||||||
When a user shares valuable source material (X posts, articles, data). Creates a source file in the ingestion pipeline, attributed to the user. Include the verbatim content — don't alter or summarize the user's contribution. Use this when someone drops a link or shares original analysis worth preserving.
|
When a user shares valuable source material (X posts, articles, data). Creates a source file in the ingestion pipeline, attributed to the user. Include the verbatim content — don't alter or summarize the user's contribution. Use this when someone drops a link or shares original analysis worth preserving.
|
||||||
|
|
||||||
4. CLAIM: [specific, disagreeable assertion]
|
4. CLAIM: [specific, disagreeable assertion]
|
||||||
When a user makes a specific claim with evidence that could enter the KB. Creates a draft claim file attributed to them. Only for genuine claims — not opinions or questions."""
|
When a user makes a specific claim with evidence that could enter the KB. Creates a draft claim file attributed to them. Only for genuine claims — not opinions or questions.
|
||||||
|
|
||||||
|
5. CONFIDENCE: [0.0-1.0]
|
||||||
|
ALWAYS include this tag. Rate how well the KB context above actually helped you answer this question. 1.0 = KB had exactly what was needed. 0.5 = KB had partial/tangential info. 0.0 = KB had nothing relevant, you answered from general knowledge. This is for internal audit only — never visible to users."""
|
||||||
|
|
||||||
# Call Opus
|
# Call Opus
|
||||||
response = await call_openrouter(RESPONSE_MODEL, prompt, max_tokens=1024)
|
response = await call_openrouter(RESPONSE_MODEL, prompt, max_tokens=1024)
|
||||||
|
|
@ -1054,6 +1156,90 @@ IMPORTANT: Special tags you can append at the end of your response (after your m
|
||||||
_create_inline_claim(claim_text.strip(), text, user, msg)
|
_create_inline_claim(claim_text.strip(), text, user, msg)
|
||||||
logger.info("Inline CLAIM drafted: %s", claim_text[:80])
|
logger.info("Inline CLAIM drafted: %s", claim_text[:80])
|
||||||
|
|
||||||
|
# CONFIDENCE: tag — model self-rated retrieval quality (audit only)
|
||||||
|
# Handles: "CONFIDENCE: 0.8", "CONFIDENCE: [0.8]", "Confidence: 0.8", case-insensitive
|
||||||
|
# Ganymede: must strip from display even if the model deviates from exact format
|
||||||
|
confidence_score = None
|
||||||
|
confidence_match = re.search(r'^CONFIDENCE:\s*\[?([\d.]+)\]?', response, re.MULTILINE | re.IGNORECASE)
|
||||||
|
if confidence_match:
|
||||||
|
try:
|
||||||
|
confidence_score = max(0.0, min(1.0, float(confidence_match.group(1))))
|
||||||
|
except ValueError:
|
||||||
|
pass
|
||||||
|
# Strip ANY line starting with CONFIDENCE (broad match — catches format deviations)
|
||||||
|
display_response = re.sub(r'\n?^CONFIDENCE\s*:.*$', '', display_response, flags=re.MULTILINE | re.IGNORECASE).rstrip()
|
||||||
|
|
||||||
|
# ─── Audit: write response_audit record ────────────────────────
|
||||||
|
response_time_ms = int((time.monotonic() - response_start) * 1000)
|
||||||
|
tool_calls.append({
|
||||||
|
"tool": "llm_call", "input": {"model": RESPONSE_MODEL},
|
||||||
|
"output": {"response_length": len(response), "tags_found": {
|
||||||
|
"learning": len(learning_lines) if learning_lines else 0,
|
||||||
|
"research": len(research_lines) if research_lines else 0,
|
||||||
|
"source": len(source_lines) if source_lines else 0,
|
||||||
|
"claim": len(claim_lines) if claim_lines else 0,
|
||||||
|
}},
|
||||||
|
"duration_ms": response_time_ms - sum(tc.get("duration_ms", 0) for tc in tool_calls),
|
||||||
|
})
|
||||||
|
|
||||||
|
# Build claims_matched with rank + source info (Rio: rank order matters)
|
||||||
|
claims_audit = []
|
||||||
|
for i, c in enumerate(kb_ctx.claims if kb_ctx else []):
|
||||||
|
claims_audit.append({"path": c.path, "title": c.title, "score": c.score,
|
||||||
|
"rank": i + 1, "source": "keyword"})
|
||||||
|
for r in vector_meta.get("direct_results", []):
|
||||||
|
claims_audit.append({"path": r["path"], "title": r["title"], "score": r["score"],
|
||||||
|
"rank": len(claims_audit) + 1, "source": "qdrant"})
|
||||||
|
for r in vector_meta.get("expanded_results", []):
|
||||||
|
claims_audit.append({"path": r["path"], "title": r["title"], "score": 0,
|
||||||
|
"rank": len(claims_audit) + 1, "source": "graph",
|
||||||
|
"edge_type": r.get("edge_type", "")})
|
||||||
|
|
||||||
|
# Detect retrieval gap (Rio: most valuable signal for KB improvement)
|
||||||
|
retrieval_gap = None
|
||||||
|
if not claims_audit and not (kb_ctx and kb_ctx.entities):
|
||||||
|
retrieval_gap = f"No KB matches for: {text[:200]}"
|
||||||
|
elif confidence_score is not None and confidence_score < 0.3:
|
||||||
|
retrieval_gap = f"Low confidence ({confidence_score}) — KB may lack coverage for: {text[:200]}"
|
||||||
|
|
||||||
|
# Conversation window (Ganymede + Rio: capture prior messages)
|
||||||
|
conv_window = None
|
||||||
|
if user:
|
||||||
|
hist = conversation_history.get((msg.chat_id, user.id), [])
|
||||||
|
if hist:
|
||||||
|
conv_window = _json.dumps(hist[-5:])
|
||||||
|
|
||||||
|
try:
|
||||||
|
from lib.db import insert_response_audit
|
||||||
|
insert_response_audit(
|
||||||
|
_audit_conn,
|
||||||
|
timestamp=datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S"),
|
||||||
|
chat_id=msg.chat_id,
|
||||||
|
user=f"@{user.username}" if user and user.username else "unknown",
|
||||||
|
agent="rio",
|
||||||
|
model=RESPONSE_MODEL,
|
||||||
|
query=text[:2000],
|
||||||
|
conversation_window=conv_window,
|
||||||
|
entities_matched=_json.dumps([{"name": e.name, "path": e.path}
|
||||||
|
for e in (kb_ctx.entities if kb_ctx else [])]),
|
||||||
|
claims_matched=_json.dumps(claims_audit),
|
||||||
|
retrieval_layers_hit=_json.dumps(list(set(retrieval_layers))),
|
||||||
|
retrieval_gap=retrieval_gap,
|
||||||
|
market_data=_json.dumps(market_data_audit) if market_data_audit else None,
|
||||||
|
research_context=research_context[:2000] if research_context else None,
|
||||||
|
kb_context_text=kb_context_text[:10000],
|
||||||
|
tool_calls=_json.dumps(tool_calls),
|
||||||
|
raw_response=response[:5000],
|
||||||
|
display_response=display_response[:5000],
|
||||||
|
confidence_score=confidence_score,
|
||||||
|
response_time_ms=response_time_ms,
|
||||||
|
)
|
||||||
|
_audit_conn.commit()
|
||||||
|
logger.info("Audit record written (confidence=%.2f, layers=%s, %d claims, %dms)",
|
||||||
|
confidence_score or 0, retrieval_layers, len(claims_audit), response_time_ms)
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning("Failed to write audit record: %s", e)
|
||||||
|
|
||||||
# Post response (without tag lines)
|
# Post response (without tag lines)
|
||||||
# Telegram has a 4096 char limit — split long messages
|
# Telegram has a 4096 char limit — split long messages
|
||||||
if len(display_response) <= 4096:
|
if len(display_response) <= 4096:
|
||||||
|
|
@ -1515,6 +1701,19 @@ def main():
|
||||||
|
|
||||||
logger.info("Starting Teleo Telegram bot (Rio)...")
|
logger.info("Starting Teleo Telegram bot (Rio)...")
|
||||||
|
|
||||||
|
# Initialize persistent audit connection (Ganymede + Rhea: once at startup, not per-response)
|
||||||
|
global _audit_conn
|
||||||
|
_audit_conn = sqlite3.connect(PIPELINE_DB, timeout=30)
|
||||||
|
_audit_conn.row_factory = sqlite3.Row
|
||||||
|
_audit_conn.execute("PRAGMA journal_mode=WAL")
|
||||||
|
_audit_conn.execute("PRAGMA busy_timeout=10000")
|
||||||
|
try:
|
||||||
|
from lib.db import migrate
|
||||||
|
migrate(_audit_conn)
|
||||||
|
logger.info("Audit DB connection initialized, schema migrated")
|
||||||
|
except Exception as e:
|
||||||
|
logger.error("Audit DB migration failed — audit writes will fail: %s", e)
|
||||||
|
|
||||||
# Build application
|
# Build application
|
||||||
app = Application.builder().token(token).build()
|
app = Application.builder().token(token).build()
|
||||||
|
|
||||||
|
|
@ -1557,6 +1756,21 @@ def main():
|
||||||
first=3600,
|
first=3600,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Audit retention cleanup — daily, 90-day window (Ganymede: match transcript policy)
|
||||||
|
async def _cleanup_audit(context=None):
|
||||||
|
try:
|
||||||
|
_audit_conn.execute("DELETE FROM response_audit WHERE timestamp < datetime('now', '-90 days')")
|
||||||
|
_audit_conn.commit()
|
||||||
|
logger.info("Audit retention cleanup complete")
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning("Audit cleanup failed: %s", e)
|
||||||
|
|
||||||
|
app.job_queue.run_repeating(
|
||||||
|
_cleanup_audit,
|
||||||
|
interval=86400, # daily
|
||||||
|
first=86400,
|
||||||
|
)
|
||||||
|
|
||||||
# Run
|
# Run
|
||||||
logger.info("Bot running. Triage interval: %ds, transcript dump: 1h", TRIAGE_INTERVAL)
|
logger.info("Bot running. Triage interval: %ds, transcript dump: 1h", TRIAGE_INTERVAL)
|
||||||
app.run_polling(drop_pending_updates=True)
|
app.run_polling(drop_pending_updates=True)
|
||||||
|
|
|
||||||
|
|
@ -86,6 +86,21 @@ class TestValidateAttribution:
|
||||||
issues = validate_attribution(fm)
|
issues = validate_attribution(fm)
|
||||||
assert "missing_attribution_extractor" in issues
|
assert "missing_attribution_extractor" in issues
|
||||||
|
|
||||||
|
def test_missing_extractor_auto_fix_with_agent(self):
|
||||||
|
"""When agent is provided, auto-fix missing extractor instead of blocking."""
|
||||||
|
fm = {"attribution": {"sourcer": [{"handle": "someone"}]}}
|
||||||
|
issues = validate_attribution(fm, agent="leo")
|
||||||
|
assert "fixed_missing_extractor" in issues
|
||||||
|
assert "missing_attribution_extractor" not in issues
|
||||||
|
# Verify the fix was applied in-place
|
||||||
|
assert fm["attribution"]["extractor"] == [{"handle": "leo"}]
|
||||||
|
|
||||||
|
def test_missing_extractor_no_agent_still_blocks(self):
|
||||||
|
"""Without agent context, missing extractor is still a hard failure."""
|
||||||
|
fm = {"attribution": {"sourcer": [{"handle": "someone"}]}}
|
||||||
|
issues = validate_attribution(fm, agent=None)
|
||||||
|
assert "missing_attribution_extractor" in issues
|
||||||
|
|
||||||
|
|
||||||
class TestBuildAttributionBlock:
|
class TestBuildAttributionBlock:
|
||||||
def test_basic_build(self):
|
def test_basic_build(self):
|
||||||
|
|
|
||||||
|
|
@ -544,3 +544,71 @@ description: "Test"
|
||||||
)
|
)
|
||||||
assert len(rejected) == 1
|
assert len(rejected) == 1
|
||||||
assert any("dm_missing" in i for i in stats["issues"])
|
assert any("dm_missing" in i for i in stats["issues"])
|
||||||
|
|
||||||
|
|
||||||
|
# ─── _yaml_line dict handling (attribution round-trip) ──────────────────
|
||||||
|
|
||||||
|
|
||||||
|
class TestYamlLineDict:
|
||||||
|
"""Verify _yaml_line produces valid YAML for nested dicts (attribution block)."""
|
||||||
|
|
||||||
|
def test_attribution_round_trip(self):
|
||||||
|
"""Attribution dict → _yaml_line → parse_frontmatter should survive."""
|
||||||
|
from lib.post_extract import _rebuild_content, parse_frontmatter
|
||||||
|
|
||||||
|
fm = {
|
||||||
|
"type": "claim",
|
||||||
|
"domain": "ai-alignment",
|
||||||
|
"description": "Test claim for round-trip",
|
||||||
|
"confidence": "experimental",
|
||||||
|
"source": "unit test",
|
||||||
|
"created": "2026-03-28",
|
||||||
|
"attribution": {
|
||||||
|
"extractor": [{"handle": "rio", "agent_id": "760F7FE7"}],
|
||||||
|
"sourcer": [{"handle": "someone", "context": "test source"}],
|
||||||
|
"challenger": [],
|
||||||
|
"synthesizer": [],
|
||||||
|
"reviewer": [],
|
||||||
|
},
|
||||||
|
}
|
||||||
|
body = "# Test claim for attribution round-trip\n\nBody text."
|
||||||
|
|
||||||
|
rebuilt = _rebuild_content(fm, body)
|
||||||
|
parsed_fm, parsed_body = parse_frontmatter(rebuilt)
|
||||||
|
|
||||||
|
assert parsed_fm is not None
|
||||||
|
# Attribution must survive as a dict, not a string
|
||||||
|
attr = parsed_fm.get("attribution")
|
||||||
|
assert isinstance(attr, dict), f"attribution is {type(attr)}, expected dict"
|
||||||
|
assert attr["extractor"][0]["handle"] == "rio"
|
||||||
|
assert attr["sourcer"][0]["handle"] == "someone"
|
||||||
|
|
||||||
|
def test_empty_attribution_roles(self):
|
||||||
|
"""Empty role lists should serialize as [] and survive round-trip."""
|
||||||
|
from lib.post_extract import _rebuild_content, parse_frontmatter
|
||||||
|
|
||||||
|
fm = {
|
||||||
|
"type": "claim",
|
||||||
|
"domain": "ai-alignment",
|
||||||
|
"description": "Test",
|
||||||
|
"confidence": "experimental",
|
||||||
|
"source": "test",
|
||||||
|
"created": "2026-03-28",
|
||||||
|
"attribution": {
|
||||||
|
"extractor": [{"handle": "leo"}],
|
||||||
|
"sourcer": [],
|
||||||
|
"challenger": [],
|
||||||
|
"synthesizer": [],
|
||||||
|
"reviewer": [],
|
||||||
|
},
|
||||||
|
}
|
||||||
|
body = "# Test claim with empty roles\n\nBody."
|
||||||
|
|
||||||
|
rebuilt = _rebuild_content(fm, body)
|
||||||
|
parsed_fm, _ = parse_frontmatter(rebuilt)
|
||||||
|
|
||||||
|
assert parsed_fm is not None
|
||||||
|
attr = parsed_fm.get("attribution")
|
||||||
|
assert isinstance(attr, dict)
|
||||||
|
assert attr["extractor"][0]["handle"] == "leo"
|
||||||
|
assert attr.get("sourcer") == [] or attr.get("sourcer") is None
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue