From 681afad50606e25629e6664d15558df1c59d7d52 Mon Sep 17 00:00:00 2001 From: m3taversal Date: Tue, 7 Apr 2026 16:52:26 +0100 Subject: [PATCH] Consolidate pipeline code from teleo-codex + VPS into single repo Sources merged: - teleo-codex/ops/pipeline-v2/ (11 newer lib files, 5 new lib modules) - teleo-codex/ops/ (agent-state, diagnostics expansion, systemd units, ops scripts) - VPS /opt/teleo-eval/telegram/ (10 new bot files, agent configs) - VPS /opt/teleo-eval/pipeline/ops/ (vector-gc, backfill-descriptions) - VPS /opt/teleo-eval/sync-mirror.sh (Bug 2 + Step 2.5 fixes) Non-trivial merges: - connect.py: kept codex threshold (0.65) + added infra domain parameter - watchdog.py: kept infra version (stale_pr integration, superset of codex) - deploy.sh: codex rsync version (interim, until VPS git clone migration) - diagnostics/app.py: codex decomposed dashboard (14 new route modules) 81 files changed, +17105/-200 lines Co-Authored-By: Claude Opus 4.6 (1M context) --- agent-state/SCHEMA.md | 255 +++ agent-state/bootstrap.sh | 145 ++ agent-state/lib-state.sh | 281 ++++ agent-state/process-cascade-inbox.py | 113 ++ batch-extract-50.sh | 30 +- deploy-manifest.md | 62 + deploy.sh | 139 +- diagnostics/activity_endpoint.py | 262 +++ diagnostics/alerting.py | 537 +++++++ diagnostics/alerting_routes.py | 125 ++ diagnostics/app.py | 1039 +++++++++++- diagnostics/backfill_submitted_by.py | 140 ++ diagnostics/daily_digest.py | 312 ++++ diagnostics/daily_digest_routes.py | 62 + diagnostics/dashboard-v2.html | 1424 +++++++++++++++++ diagnostics/dashboard_agents.py | 348 ++++ diagnostics/dashboard_epistemic.py | 239 +++ diagnostics/dashboard_health.py | 223 +++ diagnostics/dashboard_ops.py | 464 ++++++ diagnostics/dashboard_prs.py | 561 +++++++ diagnostics/dashboard_routes.py | 934 +++++++++++ diagnostics/response_audit_routes.py | 475 ++++++ diagnostics/review_queue.py | 222 +++ diagnostics/review_queue_routes.py | 64 + diagnostics/shared_ui.py | 149 ++ diagnostics/tier1_metrics.py | 476 ++++++ diagnostics/tier1_routes.py | 57 + evaluate-trigger.sh | 621 +++++++ extract-cron.sh | 179 +++ extract-graph-data.py | 520 ++++++ fix-ownership.sh | 10 + hermes-agent/GMAIL-SETUP.md | 52 + hermes-agent/install-hermes.sh | 113 ++ lib/cascade.py | 282 ++++ lib/config.py | 6 + lib/connect.py | 6 +- lib/costs.py | 48 +- lib/cross_domain.py | 230 +++ lib/db.py | 94 +- lib/digest.py | 208 +++ lib/evaluate.py | 60 +- lib/extract.py | 800 +++++++++ lib/extraction_prompt.py | 36 +- lib/health.py | 118 ++ lib/llm.py | 62 +- lib/merge.py | 104 +- lib/post_extract.py | 24 +- lib/pre_screen.py | 221 +++ lib/search.py | 70 +- lib/substantive_fixer.py | 4 +- multi-model-eval-architecture.md | 192 +++ ...may-need-separation-from-knowledge-base.md | 25 + ops/backfill-descriptions.py | 60 + ops/vector-gc.py | 163 ++ queue.md | 32 + research-session.sh | 480 ++++++ reweave.py | 42 +- schema-change-protocol.md | 127 ++ self-directed-research.md | 169 ++ sync-mirror.sh | 43 +- systemd/teleo-agent@.service | 38 + systemd/teleo-diagnostics.service | 21 + systemd/teleo-pipeline.service | 37 + telegram/agent_config.py | 160 ++ telegram/agent_runner.py | 118 ++ telegram/agents/rio.yaml | 62 + telegram/agents/theseus.yaml | 68 + telegram/approval_stages.py | 241 +++ telegram/approvals.py | 344 ++++ telegram/bot.py | 323 +++- telegram/digest.py | 208 +++ telegram/eval.py | 52 + telegram/eval_checks.py | 4 +- telegram/kb_retrieval.py | 36 +- telegram/kb_tools.py | 719 +++++++++ telegram/opsec-entities.txt | 6 + telegram/output_gate.py | 147 ++ telegram/response.py | 14 +- telegram/retrieval.py | 3 +- telegram/x_publisher.py | 347 ++++ teleo-pipeline.py | 14 +- 81 files changed, 17103 insertions(+), 198 deletions(-) create mode 100644 agent-state/SCHEMA.md create mode 100755 agent-state/bootstrap.sh create mode 100755 agent-state/lib-state.sh create mode 100644 agent-state/process-cascade-inbox.py create mode 100644 deploy-manifest.md create mode 100644 diagnostics/activity_endpoint.py create mode 100644 diagnostics/alerting.py create mode 100644 diagnostics/alerting_routes.py create mode 100644 diagnostics/backfill_submitted_by.py create mode 100644 diagnostics/daily_digest.py create mode 100644 diagnostics/daily_digest_routes.py create mode 100644 diagnostics/dashboard-v2.html create mode 100644 diagnostics/dashboard_agents.py create mode 100644 diagnostics/dashboard_epistemic.py create mode 100644 diagnostics/dashboard_health.py create mode 100644 diagnostics/dashboard_ops.py create mode 100644 diagnostics/dashboard_prs.py create mode 100644 diagnostics/dashboard_routes.py create mode 100644 diagnostics/response_audit_routes.py create mode 100644 diagnostics/review_queue.py create mode 100644 diagnostics/review_queue_routes.py create mode 100644 diagnostics/shared_ui.py create mode 100644 diagnostics/tier1_metrics.py create mode 100644 diagnostics/tier1_routes.py create mode 100755 evaluate-trigger.sh create mode 100755 extract-cron.sh create mode 100644 extract-graph-data.py create mode 100755 fix-ownership.sh create mode 100644 hermes-agent/GMAIL-SETUP.md create mode 100644 hermes-agent/install-hermes.sh create mode 100644 lib/cascade.py create mode 100644 lib/cross_domain.py create mode 100644 lib/digest.py create mode 100644 lib/extract.py create mode 100644 lib/pre_screen.py create mode 100644 multi-model-eval-architecture.md create mode 100644 observations/personality-layer-may-need-separation-from-knowledge-base.md create mode 100644 ops/backfill-descriptions.py create mode 100644 ops/vector-gc.py create mode 100644 queue.md create mode 100644 research-session.sh create mode 100644 schema-change-protocol.md create mode 100644 self-directed-research.md create mode 100644 systemd/teleo-agent@.service create mode 100644 systemd/teleo-diagnostics.service create mode 100644 systemd/teleo-pipeline.service create mode 100644 telegram/agent_config.py create mode 100644 telegram/agent_runner.py create mode 100644 telegram/agents/rio.yaml create mode 100644 telegram/agents/theseus.yaml create mode 100644 telegram/approval_stages.py create mode 100644 telegram/approvals.py create mode 100644 telegram/digest.py create mode 100644 telegram/eval.py create mode 100644 telegram/kb_tools.py create mode 100644 telegram/opsec-entities.txt create mode 100644 telegram/output_gate.py create mode 100644 telegram/x_publisher.py diff --git a/agent-state/SCHEMA.md b/agent-state/SCHEMA.md new file mode 100644 index 0000000..63cc6f0 --- /dev/null +++ b/agent-state/SCHEMA.md @@ -0,0 +1,255 @@ +# Agent State Schema v1 + +File-backed durable state for teleo agents running headless on VPS. +Survives context truncation, crash recovery, and session handoffs. + +## Design Principles + +1. **Three formats** — JSON for structured fields, JSONL for append-only logs, Markdown for context-window-friendly content +2. **Many small files** — selective loading, crash isolation, no locks needed +3. **Write on events** — not timers. State updates happen when something meaningful changes. +4. **Shared-nothing writes** — each agent owns its directory. Communication via inbox files. +5. **State ≠ Git** — state is operational (how the agent functions). Git is output (what the agent produces). + +## Directory Layout + +``` +/opt/teleo-eval/agent-state/{agent}/ +├── report.json # Current status — read every wake +├── tasks.json # Active task queue — read every wake +├── session.json # Current/last session metadata +├── memory.md # Accumulated cross-session knowledge (structured) +├── inbox/ # Messages from other agents/orchestrator +│ └── {uuid}.json # One file per message, atomic create +├── journal.jsonl # Append-only session log +└── metrics.json # Cumulative performance counters +``` + +## File Specifications + +### report.json + +Written: after each meaningful action (session start, key finding, session end) +Read: every wake, by orchestrator for monitoring + +```json +{ + "agent": "rio", + "updated_at": "2026-03-31T22:00:00Z", + "status": "idle | researching | extracting | evaluating | error", + "summary": "Completed research session — 8 sources archived on Solana launchpad mechanics", + "current_task": null, + "last_session": { + "id": "20260331-220000", + "started_at": "2026-03-31T20:30:00Z", + "ended_at": "2026-03-31T22:00:00Z", + "outcome": "completed | timeout | error", + "sources_archived": 8, + "branch": "rio/research-2026-03-31", + "pr_number": 247 + }, + "blocked_by": null, + "next_priority": "Follow up on conditional AMM thread from @0xfbifemboy" +} +``` + +### tasks.json + +Written: when task status changes +Read: every wake + +```json +{ + "agent": "rio", + "updated_at": "2026-03-31T22:00:00Z", + "tasks": [ + { + "id": "task-001", + "type": "research | extract | evaluate | follow-up | disconfirm", + "description": "Investigate conditional AMM mechanisms in MetaDAO v2", + "status": "pending | active | completed | dropped", + "priority": "high | medium | low", + "created_at": "2026-03-31T22:00:00Z", + "context": "Flagged in research session 2026-03-31 — @0xfbifemboy thread on conditional liquidity", + "follow_up_from": null, + "completed_at": null, + "outcome": null + } + ] +} +``` + +### session.json + +Written: at session start and session end +Read: every wake (for continuation), by orchestrator for scheduling + +```json +{ + "agent": "rio", + "session_id": "20260331-220000", + "started_at": "2026-03-31T20:30:00Z", + "ended_at": "2026-03-31T22:00:00Z", + "type": "research | extract | evaluate | ad-hoc", + "domain": "internet-finance", + "branch": "rio/research-2026-03-31", + "status": "running | completed | timeout | error", + "model": "sonnet", + "timeout_seconds": 5400, + "research_question": "How is conditional liquidity being implemented in Solana AMMs?", + "belief_targeted": "Markets aggregate information better than votes because skin-in-the-game creates selection pressure on beliefs", + "disconfirmation_target": "Cases where prediction markets failed to aggregate information despite financial incentives", + "sources_archived": 8, + "sources_expected": 10, + "tokens_used": null, + "cost_usd": null, + "errors": [], + "handoff_notes": "Found 3 sources on conditional AMM failures — needs extraction. Also flagged @metaproph3t thread for Theseus (AI governance angle)." +} +``` + +### memory.md + +Written: at session end, when learning something critical +Read: every wake (included in research prompt context) + +```markdown +# Rio — Operational Memory + +## Cross-Session Patterns +- Conditional AMMs keep appearing across 3+ independent sources (sessions 03-28, 03-29, 03-31). This is likely a real trend, not cherry-picking. +- @0xfbifemboy consistently produces highest-signal threads in the DeFi mechanism design space. + +## Dead Ends (don't re-investigate) +- Polymarket fee structure analysis (2026-03-25): fully documented in existing claims, no new angles. +- Jupiter governance token utility (2026-03-27): vaporware, no mechanism to analyze. + +## Open Questions +- Is MetaDAO's conditional market maker manipulation-resistant at scale? No evidence either way yet. +- How does futarchy handle low-liquidity markets? This is the keystone weakness. + +## Corrections +- Previously believed Drift protocol was pure order-book. Actually hybrid AMM+CLOB. Updated 2026-03-30. + +## Cross-Agent Flags Received +- Theseus (2026-03-29): "Check if MetaDAO governance has AI agent participation — alignment implications" +- Leo (2026-03-28): "Your conditional AMM analysis connects to Astra's resource allocation claims" +``` + +### inbox/{uuid}.json + +Written: by other agents or orchestrator +Read: checked on wake, deleted after processing + +```json +{ + "id": "msg-abc123", + "from": "theseus", + "to": "rio", + "created_at": "2026-03-31T18:00:00Z", + "type": "flag | task | question | cascade", + "priority": "high | normal", + "subject": "Check MetaDAO for AI agent participation", + "body": "Found evidence that AI agents are trading on Drift — check if any are participating in MetaDAO conditional markets. Alignment implications if automated agents are influencing futarchic governance.", + "source_ref": "theseus/research-2026-03-31", + "expires_at": null +} +``` + +### journal.jsonl + +Written: append at session boundaries +Read: debug/audit only (never loaded into agent context by default) + +```jsonl +{"ts":"2026-03-31T20:30:00Z","event":"session_start","session_id":"20260331-220000","type":"research"} +{"ts":"2026-03-31T20:35:00Z","event":"orient_complete","files_read":["identity.md","beliefs.md","reasoning.md","_map.md"]} +{"ts":"2026-03-31T21:30:00Z","event":"sources_archived","count":5,"domain":"internet-finance"} +{"ts":"2026-03-31T22:00:00Z","event":"session_end","outcome":"completed","sources_archived":8,"handoff":"conditional AMM failures need extraction"} +``` + +### metrics.json + +Written: at session end (cumulative counters) +Read: by CI scoring system, by orchestrator for scheduling decisions + +```json +{ + "agent": "rio", + "updated_at": "2026-03-31T22:00:00Z", + "lifetime": { + "sessions_total": 47, + "sessions_completed": 42, + "sessions_timeout": 3, + "sessions_error": 2, + "sources_archived": 312, + "claims_proposed": 89, + "claims_accepted": 71, + "claims_challenged": 12, + "claims_rejected": 6, + "disconfirmation_attempts": 47, + "disconfirmation_hits": 8, + "cross_agent_flags_sent": 23, + "cross_agent_flags_received": 15 + }, + "rolling_30d": { + "sessions": 12, + "sources_archived": 87, + "claims_proposed": 24, + "acceptance_rate": 0.83, + "avg_sources_per_session": 7.25 + } +} +``` + +## Integration Points + +### research-session.sh + +Add these hooks: + +1. **Pre-session** (after branch creation, before Claude launch): + - Write `session.json` with status "running" + - Write `report.json` with status "researching" + - Append session_start to `journal.jsonl` + - Include `memory.md` and `tasks.json` in the research prompt + +2. **Post-session** (after commit, before/after PR): + - Update `session.json` with outcome, source count, branch, PR number + - Update `report.json` with summary and next_priority + - Update `metrics.json` counters + - Append session_end to `journal.jsonl` + - Process and clean `inbox/` (mark processed messages) + +3. **On error/timeout**: + - Update `session.json` status to "error" or "timeout" + - Update `report.json` with error info + - Append error event to `journal.jsonl` + +### Pipeline daemon (teleo-pipeline.py) + +- Read `report.json` for all agents to build dashboard +- Write to `inbox/` when cascade events need agent attention +- Read `metrics.json` for scheduling decisions (deprioritize agents with high error rates) + +### Claude research prompt + +Add to the prompt: +``` +### Step 0: Load Operational State (1 min) +Read /opt/teleo-eval/agent-state/{agent}/memory.md — this is your cross-session operational memory. +Read /opt/teleo-eval/agent-state/{agent}/tasks.json — check for pending tasks. +Check /opt/teleo-eval/agent-state/{agent}/inbox/ for messages from other agents. +Process any high-priority inbox items before choosing your research direction. +``` + +## Bootstrap + +Run `ops/agent-state/bootstrap.sh` to create directories and seed initial state for all agents. + +## Migration from Existing State + +- `research-journal.md` continues as-is (agent-written, in git). `memory.md` is the structured equivalent for operational state (not in git). +- `ops/sessions/*.json` continue for backward compat. `session.json` per agent is the richer replacement. +- `ops/queue.md` remains the human-visible task board. `tasks.json` per agent is the machine-readable equivalent. +- Workspace flags (`~/.pentagon/workspace/collective/flag-*`) migrate to `inbox/` messages over time. diff --git a/agent-state/bootstrap.sh b/agent-state/bootstrap.sh new file mode 100755 index 0000000..087cff9 --- /dev/null +++ b/agent-state/bootstrap.sh @@ -0,0 +1,145 @@ +#!/bin/bash +# Bootstrap agent-state directories for all teleo agents. +# Run once on VPS: bash ops/agent-state/bootstrap.sh +# Safe to re-run — skips existing files, only creates missing ones. + +set -euo pipefail + +STATE_ROOT="${TELEO_STATE_ROOT:-/opt/teleo-eval/agent-state}" + +AGENTS=("rio" "clay" "theseus" "vida" "astra" "leo") +DOMAINS=("internet-finance" "entertainment" "ai-alignment" "health" "space-development" "grand-strategy") + +log() { echo "[$(date -Iseconds)] $*"; } + +for i in "${!AGENTS[@]}"; do + AGENT="${AGENTS[$i]}" + DOMAIN="${DOMAINS[$i]}" + DIR="$STATE_ROOT/$AGENT" + + log "Bootstrapping $AGENT..." + mkdir -p "$DIR/inbox" + + # report.json — current status + if [ ! -f "$DIR/report.json" ]; then + cat > "$DIR/report.json" < "$DIR/tasks.json" < "$DIR/session.json" < "$DIR/memory.md" < "$DIR/metrics.json" < "$DIR/journal.jsonl" + log " Created journal.jsonl" + fi + +done + +log "Bootstrap complete. State root: $STATE_ROOT" +log "Agents initialized: ${AGENTS[*]}" diff --git a/agent-state/lib-state.sh b/agent-state/lib-state.sh new file mode 100755 index 0000000..2760764 --- /dev/null +++ b/agent-state/lib-state.sh @@ -0,0 +1,281 @@ +#!/bin/bash +# lib-state.sh — Bash helpers for reading/writing agent state files. +# Source this in pipeline scripts: source ops/agent-state/lib-state.sh +# +# All writes use atomic rename (write to .tmp, then mv) to prevent corruption. +# All reads return valid JSON or empty string on missing/corrupt files. + +STATE_ROOT="${TELEO_STATE_ROOT:-/opt/teleo-eval/agent-state}" + +# --- Internal helpers --- + +_state_dir() { + local agent="$1" + echo "$STATE_ROOT/$agent" +} + +# --- Report (current status) --- + +state_read_report() { + local agent="$1" + local file="$(_state_dir "$agent")/report.json" + [ -f "$file" ] && cat "$file" || echo "{}" +} + +state_update_report() { + local agent="$1" + local status="$2" + local summary="$3" + local file="$(_state_dir "$agent")/report.json" + + _STATE_FILE="$file" _STATE_AGENT="$agent" _STATE_STATUS="$status" \ + _STATE_SUMMARY="$summary" _STATE_TS="$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + python3 -c " +import json, os +try: + with open(os.environ['_STATE_FILE']) as f: + data = json.load(f) +except: + data = {'agent': os.environ['_STATE_AGENT']} +data['status'] = os.environ['_STATE_STATUS'] +data['summary'] = os.environ['_STATE_SUMMARY'] +data['updated_at'] = os.environ['_STATE_TS'] +print(json.dumps(data, indent=2)) +" | _atomic_write_stdin "$file" +} + +# Variant that takes full JSON from stdin +_atomic_write_stdin() { + local filepath="$1" + local tmpfile="${filepath}.tmp.$$" + cat > "$tmpfile" + mv -f "$tmpfile" "$filepath" +} + +# Full report update with session info (called at session end) +state_finalize_report() { + local agent="$1" + local status="$2" + local summary="$3" + local session_id="$4" + local started_at="$5" + local ended_at="$6" + local outcome="$7" + local sources="$8" + local branch="$9" + local pr_number="${10}" + local next_priority="${11:-null}" + local file="$(_state_dir "$agent")/report.json" + + _STATE_FILE="$file" _STATE_AGENT="$agent" _STATE_STATUS="$status" \ + _STATE_SUMMARY="$summary" _STATE_SESSION_ID="$session_id" \ + _STATE_STARTED="$started_at" _STATE_ENDED="$ended_at" \ + _STATE_OUTCOME="$outcome" _STATE_SOURCES="$sources" \ + _STATE_BRANCH="$branch" _STATE_PR="$pr_number" \ + _STATE_NEXT="$next_priority" \ + python3 -c " +import json, os +e = os.environ +sources = int(e['_STATE_SOURCES']) if e['_STATE_SOURCES'].isdigit() else 0 +pr = int(e['_STATE_PR']) if e['_STATE_PR'].isdigit() else None +next_p = None if e['_STATE_NEXT'] == 'null' else e['_STATE_NEXT'] +data = { + 'agent': e['_STATE_AGENT'], + 'updated_at': e['_STATE_ENDED'], + 'status': e['_STATE_STATUS'], + 'summary': e['_STATE_SUMMARY'], + 'current_task': None, + 'last_session': { + 'id': e['_STATE_SESSION_ID'], + 'started_at': e['_STATE_STARTED'], + 'ended_at': e['_STATE_ENDED'], + 'outcome': e['_STATE_OUTCOME'], + 'sources_archived': sources, + 'branch': e['_STATE_BRANCH'], + 'pr_number': pr + }, + 'blocked_by': None, + 'next_priority': next_p +} +print(json.dumps(data, indent=2)) +" | _atomic_write_stdin "$file" +} + +# --- Session --- + +state_start_session() { + local agent="$1" + local session_id="$2" + local type="$3" + local domain="$4" + local branch="$5" + local model="${6:-sonnet}" + local timeout="${7:-5400}" + local started_at + started_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)" + local file="$(_state_dir "$agent")/session.json" + + _STATE_FILE="$file" _STATE_AGENT="$agent" _STATE_SID="$session_id" \ + _STATE_STARTED="$started_at" _STATE_TYPE="$type" _STATE_DOMAIN="$domain" \ + _STATE_BRANCH="$branch" _STATE_MODEL="$model" _STATE_TIMEOUT="$timeout" \ + python3 -c " +import json, os +e = os.environ +data = { + 'agent': e['_STATE_AGENT'], + 'session_id': e['_STATE_SID'], + 'started_at': e['_STATE_STARTED'], + 'ended_at': None, + 'type': e['_STATE_TYPE'], + 'domain': e['_STATE_DOMAIN'], + 'branch': e['_STATE_BRANCH'], + 'status': 'running', + 'model': e['_STATE_MODEL'], + 'timeout_seconds': int(e['_STATE_TIMEOUT']), + 'research_question': None, + 'belief_targeted': None, + 'disconfirmation_target': None, + 'sources_archived': 0, + 'sources_expected': 0, + 'tokens_used': None, + 'cost_usd': None, + 'errors': [], + 'handoff_notes': None +} +print(json.dumps(data, indent=2)) +" | _atomic_write_stdin "$file" + + echo "$started_at" +} + +state_end_session() { + local agent="$1" + local outcome="$2" + local sources="${3:-0}" + local pr_number="${4:-null}" + local file="$(_state_dir "$agent")/session.json" + + _STATE_FILE="$file" _STATE_OUTCOME="$outcome" _STATE_SOURCES="$sources" \ + _STATE_PR="$pr_number" _STATE_TS="$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + python3 -c " +import json, os +e = os.environ +with open(e['_STATE_FILE']) as f: + data = json.load(f) +data['ended_at'] = e['_STATE_TS'] +data['status'] = e['_STATE_OUTCOME'] +data['sources_archived'] = int(e['_STATE_SOURCES']) if e['_STATE_SOURCES'].isdigit() else 0 +pr = e.get('_STATE_PR', 'null') +data['pr_number'] = int(pr) if pr.isdigit() else None +print(json.dumps(data, indent=2)) +" | _atomic_write_stdin "$file" +} + +# --- Journal (append-only JSONL) --- + +state_journal_append() { + local agent="$1" + local event="$2" + shift 2 + # Remaining args are key=value pairs for extra fields + local file="$(_state_dir "$agent")/journal.jsonl" + + _STATE_TS="$(date -u +%Y-%m-%dT%H:%M:%SZ)" _STATE_EVT="$event" \ + python3 -c " +import json, os, sys +entry = {'ts': os.environ['_STATE_TS'], 'event': os.environ['_STATE_EVT']} +for pair in sys.argv[1:]: + k, _, v = pair.partition('=') + if k: + entry[k] = v +print(json.dumps(entry)) +" "$@" >> "$file" +} + +# --- Metrics --- + +state_update_metrics() { + local agent="$1" + local outcome="$2" + local sources="${3:-0}" + local file="$(_state_dir "$agent")/metrics.json" + + _STATE_FILE="$file" _STATE_AGENT="$agent" _STATE_OUTCOME="$outcome" \ + _STATE_SOURCES="$sources" _STATE_TS="$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + python3 -c " +import json, os +e = os.environ +try: + with open(e['_STATE_FILE']) as f: + data = json.load(f) +except: + data = {'agent': e['_STATE_AGENT'], 'lifetime': {}, 'rolling_30d': {}} + +lt = data.setdefault('lifetime', {}) +lt['sessions_total'] = lt.get('sessions_total', 0) + 1 +outcome = e['_STATE_OUTCOME'] +if outcome == 'completed': + lt['sessions_completed'] = lt.get('sessions_completed', 0) + 1 +elif outcome == 'timeout': + lt['sessions_timeout'] = lt.get('sessions_timeout', 0) + 1 +elif outcome == 'error': + lt['sessions_error'] = lt.get('sessions_error', 0) + 1 +lt['sources_archived'] = lt.get('sources_archived', 0) + (int(e['_STATE_SOURCES']) if e['_STATE_SOURCES'].isdigit() else 0) + +data['updated_at'] = e['_STATE_TS'] +print(json.dumps(data, indent=2)) +" | _atomic_write_stdin "$file" +} + +# --- Inbox --- + +state_check_inbox() { + local agent="$1" + local inbox="$(_state_dir "$agent")/inbox" + [ -d "$inbox" ] && ls "$inbox"/*.json 2>/dev/null || true +} + +state_send_message() { + local from="$1" + local to="$2" + local type="$3" + local subject="$4" + local body="$5" + local inbox="$(_state_dir "$to")/inbox" + local msg_id="msg-$(date +%s)-$$" + local file="$inbox/${msg_id}.json" + + mkdir -p "$inbox" + _STATE_FILE="$file" _STATE_MSGID="$msg_id" _STATE_FROM="$from" \ + _STATE_TO="$to" _STATE_TS="$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + _STATE_TYPE="$type" _STATE_SUBJECT="$subject" _STATE_BODY="$body" \ + python3 -c " +import json, os +e = os.environ +data = { + 'id': e['_STATE_MSGID'], + 'from': e['_STATE_FROM'], + 'to': e['_STATE_TO'], + 'created_at': e['_STATE_TS'], + 'type': e['_STATE_TYPE'], + 'priority': 'normal', + 'subject': e['_STATE_SUBJECT'], + 'body': e['_STATE_BODY'], + 'source_ref': None, + 'expires_at': None +} +print(json.dumps(data, indent=2)) +" | _atomic_write_stdin "$file" + echo "$msg_id" +} + +# --- State directory check --- + +state_ensure_dir() { + local agent="$1" + local dir="$(_state_dir "$agent")" + if [ ! -d "$dir" ]; then + echo "ERROR: Agent state not initialized for $agent. Run bootstrap.sh first." >&2 + return 1 + fi +} diff --git a/agent-state/process-cascade-inbox.py b/agent-state/process-cascade-inbox.py new file mode 100644 index 0000000..f314762 --- /dev/null +++ b/agent-state/process-cascade-inbox.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python3 +"""Process cascade inbox messages after a research session. + +For each unread cascade-*.md in an agent's inbox: +1. Logs cascade_reviewed event to pipeline.db audit_log +2. Moves the file to inbox/processed/ + +Usage: python3 process-cascade-inbox.py +""" + +import json +import os +import re +import shutil +import sqlite3 +import sys +from datetime import datetime, timezone +from pathlib import Path + +AGENT_STATE_DIR = Path(os.environ.get("AGENT_STATE_DIR", "/opt/teleo-eval/agent-state")) +PIPELINE_DB = Path(os.environ.get("PIPELINE_DB", "/opt/teleo-eval/pipeline/pipeline.db")) + + +def parse_frontmatter(text: str) -> dict: + """Parse YAML-like frontmatter from markdown.""" + fm = {} + match = re.match(r'^---\n(.*?)\n---', text, re.DOTALL) + if not match: + return fm + for line in match.group(1).strip().splitlines(): + if ':' in line: + key, val = line.split(':', 1) + fm[key.strip()] = val.strip().strip('"') + return fm + + +def process_agent_inbox(agent: str) -> int: + """Process cascade messages in agent's inbox. Returns count processed.""" + inbox_dir = AGENT_STATE_DIR / agent / "inbox" + if not inbox_dir.exists(): + return 0 + + cascade_files = sorted(inbox_dir.glob("cascade-*.md")) + if not cascade_files: + return 0 + + # Ensure processed dir exists + processed_dir = inbox_dir / "processed" + processed_dir.mkdir(exist_ok=True) + + processed = 0 + now = datetime.now(timezone.utc).isoformat() + + try: + conn = sqlite3.connect(str(PIPELINE_DB), timeout=10) + conn.execute("PRAGMA journal_mode=WAL") + except sqlite3.Error as e: + print(f"WARNING: Cannot connect to pipeline.db: {e}", file=sys.stderr) + # Still move files even if DB is unavailable + conn = None + + for cf in cascade_files: + try: + text = cf.read_text() + fm = parse_frontmatter(text) + + # Skip already-processed files + if fm.get("status") == "processed": + continue + + # Log to audit_log + if conn: + detail = { + "agent": agent, + "cascade_file": cf.name, + "subject": fm.get("subject", "unknown"), + "original_created": fm.get("created", "unknown"), + "reviewed_at": now, + } + conn.execute( + "INSERT INTO audit_log (stage, event, detail, timestamp) VALUES (?, ?, ?, ?)", + ("cascade", "cascade_reviewed", json.dumps(detail), now), + ) + + # Move to processed + dest = processed_dir / cf.name + shutil.move(str(cf), str(dest)) + processed += 1 + + except Exception as e: + print(f"WARNING: Failed to process {cf.name}: {e}", file=sys.stderr) + + if conn: + try: + conn.commit() + conn.close() + except sqlite3.Error: + pass + + return processed + + +if __name__ == "__main__": + if len(sys.argv) < 2: + print(f"Usage: {sys.argv[0]} ", file=sys.stderr) + sys.exit(1) + + agent = sys.argv[1] + count = process_agent_inbox(agent) + if count > 0: + print(f"Processed {count} cascade message(s) for {agent}") + # Exit 0 regardless — non-fatal + sys.exit(0) diff --git a/batch-extract-50.sh b/batch-extract-50.sh index 924403c..c449902 100755 --- a/batch-extract-50.sh +++ b/batch-extract-50.sh @@ -6,6 +6,7 @@ # Gate 1: Is source already in archive/{domain}/? → already processed, dedup # Gate 2: Does extraction branch exist on Forgejo? → extraction in progress # Gate 3: Does pipeline.db show ≥3 closed PRs for this source? → zombie, skip +# Gate 4: Does pipeline.db show active OR recently closed PR? → skip (4h cooldown) # All gates pass → extract # # Architecture: Ganymede (two-gate) + Rhea (separate worktrees) @@ -185,6 +186,24 @@ print(matches[0]['number'] if matches else '') fi fi + # Gate 4: Check pipeline.db for active or recently closed PRs — prevents + # re-extraction waste when eval closes a PR and batch-extract runs again + # before the source is manually reviewed. 4h cooldown after closure. + if [ -f "$DB" ]; then + ACTIVE_COUNT=$(sqlite3 "$DB" "SELECT COUNT(*) FROM prs WHERE branch = 'extract/$BASENAME' AND status IN ('extracting','approved','merging')" 2>/dev/null || echo 0) + if [ "$ACTIVE_COUNT" -ge 1 ]; then + echo "[$(date)] [$COUNT/$MAX] SKIP $BASENAME (active PR exists)" >> $LOG + SKIPPED=$((SKIPPED + 1)) + continue + fi + RECENT_CLOSED=$(sqlite3 "$DB" "SELECT COUNT(*) FROM prs WHERE branch = 'extract/$BASENAME' AND status = 'closed' AND created_at > datetime('now', '-4 hours')" 2>/dev/null || echo 0) + if [ "$RECENT_CLOSED" -ge 1 ]; then + echo "[$(date)] [$COUNT/$MAX] SKIP $BASENAME (recently closed PR — 4h cooldown)" >> $LOG + SKIPPED=$((SKIPPED + 1)) + continue + fi + fi + echo "[$(date)] [$COUNT/$MAX] Processing $BASENAME" >> $LOG # Reset to main (log errors — don't swallow) @@ -235,11 +254,18 @@ Pentagon-Agent: Epimetheus <3D35839A-7722-4740-B93D-51157F7D5E70>" >> $LOG 2>&1 # Push git push "http://leo:${TOKEN}@localhost:3000/teleo/teleo-codex.git" "$BRANCH" --force >> $LOG 2>&1 - # Create PR + # Create PR (include prior art sidecar if available) + PRIOR_ART_FILE="${SOURCE}.prior-art" + PR_BODY="" + if [ -f "$PRIOR_ART_FILE" ]; then + # Escape JSON special chars in prior art content + PR_BODY=$(cat "$PRIOR_ART_FILE" | python3 -c 'import sys,json; print(json.dumps(sys.stdin.read()))') + PR_BODY=${PR_BODY:1:-1} # Strip outer quotes from json.dumps + fi curl -sf -X POST "http://localhost:3000/api/v1/repos/teleo/teleo-codex/pulls" \ -H "Authorization: token $TOKEN" \ -H "Content-Type: application/json" \ - -d "{\"title\":\"extract: $BASENAME\",\"head\":\"$BRANCH\",\"base\":\"main\"}" >> /dev/null 2>&1 + -d "{\"title\":\"extract: $BASENAME\",\"head\":\"$BRANCH\",\"base\":\"main\",\"body\":\"$PR_BODY\"}" >> /dev/null 2>&1 SUCCESS=$((SUCCESS + 1)) echo " -> SUCCESS ($CHANGED files)" >> $LOG diff --git a/deploy-manifest.md b/deploy-manifest.md new file mode 100644 index 0000000..a5a68bc --- /dev/null +++ b/deploy-manifest.md @@ -0,0 +1,62 @@ +# Deploy Manifest + +Every PR that touches VPS-deployed code must include a deploy manifest — either in the PR description or as a comment before requesting deploy. Rhea can reject deploys without one. + +## Template + +Copy this into your PR description and fill it in: + +``` +## Deploy Manifest + +**Files changed:** +- path/to/file.py (new | modified | deleted) + +**Services to restart:** +- teleo-bot.service +- teleo-eval.service + +**New ReadWritePaths:** (leave blank if none) +- /opt/teleo-eval/data/new-directory + +**Migration steps:** (leave blank if none) +- Run: sqlite3 pipeline.db < migrations/001-add-column.sql + +**Endpoints affected:** +- GET /health +- GET /api/alerts + +**Expected behavior after deploy:** +- /health returns 200 with new field X +- New cron runs every 5 minutes +``` + +## What Counts as VPS-Deployed Code + +| File type | Example | Needs manifest? | +|-----------|---------|-----------------| +| Python application code | bot.py, app.py, alerting.py | Yes | +| Shell scripts on VPS | extract-cron.sh, evaluate-trigger.sh | Yes | +| systemd service/timer files | teleo-bot.service | Yes | +| Database migrations | ALTER TABLE, new tables | Yes | +| HTML/CSS/JS served by app | dashboard.html, teleo-app | Yes | +| Claim/source/entity markdown | domains/ai-alignment/claim.md | No | +| Schema definitions | schemas/claim.md | No (but see schema-change-protocol.md) | +| Agent identity/beliefs | agents/theseus/identity.md | No | + +## Rules + +1. **No deploy without manifest.** If the PR lacks one, Rhea bounces it back. +2. **List every service that needs restart.** "Just restart everything" is not acceptable — it causes unnecessary downtime. +3. **ReadWritePaths are mandatory.** If your code writes to a new path, say so. Missing ReadWritePaths is the #1 cause of silent deploy failures. +4. **Endpoints affected enables verification.** Argus uses this field to run post-deploy smoke tests. Without it, verification is guesswork. +5. **Migration steps must be idempotent.** If the deploy is retried, the migration shouldn't break. + +## Post-Deploy Verification + +After Rhea restarts the service: +1. Argus hits every endpoint listed in "Endpoints affected" +2. Argus checks systemd journal for errors in the last 60 seconds +3. Argus reports pass/fail in the Engineering group chat + +If verification fails, Rhea rolls back. The PR author fixes and resubmits. diff --git a/deploy.sh b/deploy.sh index db2a710..31a2f6d 100755 --- a/deploy.sh +++ b/deploy.sh @@ -1,56 +1,99 @@ #!/usr/bin/env bash -# Deploy teleo-pipeline to VPS. -# Usage: ./deploy.sh [--restart] +# deploy.sh — Deploy pipeline and diagnostics to VPS from repo +# Usage: ./deploy.sh [--dry-run] [--restart] # -# Pulls latest from current branch, updates venv, optionally restarts service. -# Run from the VPS as the teleo user, or via SSH: -# ssh teleo@77.42.65.182 'cd /opt/teleo-eval/pipeline && ./deploy.sh --restart' - +# Requires: committed, clean working tree. Enforces repo-first workflow. set -euo pipefail -DEPLOY_DIR="/opt/teleo-eval/pipeline" -VENV_DIR="${DEPLOY_DIR}/.venv" -SERVICE="teleo-pipeline" +VPS_HOST="teleo@77.42.65.182" +VPS_PIPELINE="/opt/teleo-eval/pipeline" +VPS_DIAGNOSTICS="/opt/teleo-eval/diagnostics" +VPS_AGENT_STATE="/opt/teleo-eval/ops/agent-state" +REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)" -cd "$DEPLOY_DIR" +DRY_RUN=false +RESTART=false -echo "=== Pulling latest ===" -git pull --ff-only +for arg in "$@"; do + case "$arg" in + --dry-run) DRY_RUN=true ;; + --restart) RESTART=true ;; + --help|-h) + echo "Usage: $0 [--dry-run] [--restart]" + echo " --dry-run Show what would be deployed without doing it" + echo " --restart Restart services after deploy" + exit 0 + ;; + *) echo "Unknown arg: $arg"; exit 1 ;; + esac +done -echo "=== Updating venv ===" -"${VENV_DIR}/bin/pip" install -q -e ".[dev]" 2>/dev/null || \ - "${VENV_DIR}/bin/pip" install -q -e . - -echo "=== Syntax check ===" -"${VENV_DIR}/bin/python3" -c " -import ast, pathlib, sys -errors = [] -for f in pathlib.Path('.').rglob('*.py'): - if '.venv' in str(f): - continue - try: - ast.parse(f.read_text()) - except SyntaxError as e: - errors.append(f'{f}: {e}') -if errors: - for e in errors: - print(f'SYNTAX ERROR: {e}', file=sys.stderr) - sys.exit(1) -print('All Python files pass syntax check') -" - -if [[ "${1:-}" == "--restart" ]]; then - echo "=== Restarting ${SERVICE} ===" - sudo systemctl restart "$SERVICE" - sleep 2 - if systemctl is-active --quiet "$SERVICE"; then - echo "=== ${SERVICE} is running ===" - systemctl status "$SERVICE" --no-pager -l | head -15 - else - echo "ERROR: ${SERVICE} failed to start" >&2 - journalctl -u "$SERVICE" --no-pager -n 20 - exit 1 - fi -else - echo "=== Deploy complete (service not restarted — use --restart to restart) ===" +# Gate: working tree must be clean +if [ -n "$(git -C "$REPO_ROOT" status --porcelain)" ]; then + echo "ERROR: Uncommitted changes. Commit first, deploy second." + git -C "$REPO_ROOT" status --short + exit 1 +fi + +echo "Deploying from commit: $(git -C "$REPO_ROOT" log --oneline -1)" +echo "" + +# Syntax check all Python files before deploying +echo "=== Pre-deploy syntax check ===" +ERRORS=0 +for f in "$REPO_ROOT/ops/pipeline-v2/lib/"*.py "$REPO_ROOT/ops/pipeline-v2/"*.py "$REPO_ROOT/ops/diagnostics/"*.py; do + [ -f "$f" ] || continue + if ! python3 -c "import ast, sys; ast.parse(open(sys.argv[1]).read())" "$f" 2>/dev/null; then + echo "SYNTAX ERROR: $f" + ERRORS=$((ERRORS + 1)) + fi +done +if [ "$ERRORS" -gt 0 ]; then + echo "ERROR: $ERRORS files have syntax errors. Fix before deploying." + exit 1 +fi +echo "All files pass syntax check." +echo "" + +RSYNC_FLAGS="-avz --exclude='__pycache__' --exclude='*.pyc' --exclude='*.bak*'" +if $DRY_RUN; then + RSYNC_FLAGS="$RSYNC_FLAGS --dry-run" + echo "=== DRY RUN ===" +fi + +echo "=== Pipeline lib/ ===" +rsync $RSYNC_FLAGS "$REPO_ROOT/ops/pipeline-v2/lib/" "$VPS_HOST:$VPS_PIPELINE/lib/" +echo "" + +echo "=== Pipeline top-level ===" +for f in teleo-pipeline.py reweave.py batch-extract-50.sh; do + [ -f "$REPO_ROOT/ops/pipeline-v2/$f" ] || continue + rsync $RSYNC_FLAGS "$REPO_ROOT/ops/pipeline-v2/$f" "$VPS_HOST:$VPS_PIPELINE/$f" +done +echo "" + +echo "=== Diagnostics ===" +rsync $RSYNC_FLAGS "$REPO_ROOT/ops/diagnostics/" "$VPS_HOST:$VPS_DIAGNOSTICS/" +echo "" + +echo "=== Agent state ===" +rsync $RSYNC_FLAGS "$REPO_ROOT/ops/agent-state/" "$VPS_HOST:$VPS_AGENT_STATE/" +echo "" + +echo "=== Research session ===" +rsync $RSYNC_FLAGS "$REPO_ROOT/ops/research-session.sh" "$VPS_HOST:/opt/teleo-eval/research-session.sh" +echo "" + +if $DRY_RUN; then + echo "Dry run complete. No changes made." + exit 0 +fi + +echo "Deploy complete." + +if $RESTART; then + echo "" + echo "=== Restarting services ===" + ssh "$VPS_HOST" "sudo systemctl restart teleo-pipeline teleo-diagnostics" + echo "Services restarted." fi diff --git a/diagnostics/activity_endpoint.py b/diagnostics/activity_endpoint.py new file mode 100644 index 0000000..7c6222d --- /dev/null +++ b/diagnostics/activity_endpoint.py @@ -0,0 +1,262 @@ +""" +/api/activity endpoint for diagnostics service. + +Serves per-operation events for the dashboard v2 timeline hero panel. +Derives events from the prs table (per-PR granularity) and audit_log +(pipeline-level ops). Cursor-based pagination via timestamp. + +Integration: add route and handler to app.py: + app.router.add_get('/api/activity', handle_activity) + +Contract (endpoint #7): + GET /api/activity?limit=100&cursor= + Response: { + events: [{timestamp, agent, operation, target, domain, description, status, pr_number}], + limit: int, + cursor: string|null, + has_more: bool + } + +Data sources: + - prs table: number, status, domain, agent, created_at, merged_at, branch, source_path + - audit_log table: timestamp, stage, event, detail + - contributors table: handle, display_name (for agent name resolution) +""" + +from aiohttp import web +import sqlite3 +import json + + +# Map PR status to Clay's operation color palette +# extract (cyan), new (green), enrich (amber), challenge (red-orange), +# decision (violet), infra (grey) +STATUS_TO_OPERATION = { + 'merged': 'new', # green — new knowledge merged + 'approved': 'enrich', # amber — approved, enriching KB + 'open': 'extract', # cyan — new extraction in progress + 'validating': 'extract', # cyan — being validated + 'reviewing': 'extract', # cyan — under review + 'merging': 'new', # green — merge in progress + 'closed': 'infra', # grey — closed/rejected + 'zombie': 'infra', # grey — stale + 'conflict': 'challenge', # red-orange — conflict detected +} + +# Map audit_log stage to operation type +STAGE_TO_OPERATION = { + 'ingest': 'extract', + 'extract': 'extract', + 'validate': 'infra', + 'evaluate': 'infra', + 'merge': 'new', + 'reject': 'infra', + 'breaker': 'challenge', +} + + +def pr_description(row): + """Generate human-readable description from a PR row.""" + status = row['status'] + domain = row['domain'] or 'unknown' + branch = row['branch'] or '' + + # Extract a meaningful target from the branch name + # Branch format is typically: agent-name/claims-description + target = branch.split('/')[-1] if '/' in branch else branch + + # Infer agent from branch prefix if not in the row + branch_agent = branch.split('/')[0] if '/' in branch else None + + # Build a richer description with domain context + domain_tag = f" [{domain}]" if domain and domain != 'unknown' and domain != 'general' else '' + + templates = { + 'merged': f"Merged{domain_tag}: {target}", + 'approved': f"Approved{domain_tag}: {target}", + 'open': f"Opened{domain_tag}: {target}", + 'validating': f"Validating{domain_tag}: {target}", + 'reviewing': f"Reviewing{domain_tag}: {target}", + 'merging': f"Merging{domain_tag}: {target}", + 'closed': f"Closed{domain_tag}: {target}", + 'zombie': f"Stale{domain_tag}: {target}", + 'conflict': f"Conflict{domain_tag}: {target}", + } + + return templates.get(status, f"PR #{row['number']}{domain_tag}: {target}") + + +def audit_description(row): + """Generate human-readable description from an audit_log row.""" + stage = row['stage'] or '' + event = row['event'] or '' + detail = row['detail'] or '' + + # Try to parse detail as JSON + if detail: + try: + detail_obj = json.loads(detail) + if isinstance(detail_obj, dict): + msg = detail_obj.get('message') or detail_obj.get('reason', '') + if msg: + return f"[{stage}] {msg}"[:150] + except (json.JSONDecodeError, TypeError): + pass + + if event: + desc = f"[{stage}] {event}" + if detail and len(detail) < 80: + desc += f" — {detail}" + return desc[:150] + + return f"[{stage}] pipeline event" + + +async def handle_activity(request): + """Handler for GET /api/activity. + + Query params: + limit (int, default 100, max 500): number of events to return + cursor (ISO timestamp): return events older than this timestamp + + Derives events from two sources: + 1. prs table — per-PR events with domain, agent, status + 2. audit_log — pipeline-level operational events + + Events are merged and sorted by timestamp descending (most recent first). + """ + try: + limit = min(int(request.query.get('limit', 100)), 500) + except (ValueError, TypeError): + limit = 100 + + cursor = request.query.get('cursor') + db_path = request.app['db_path'] + + try: + conn = sqlite3.connect(f'file:{db_path}?mode=ro', uri=True) + conn.row_factory = sqlite3.Row + + events = [] + + # Source 1: PR events (primary — these have the granularity we need) + # Each PR generates events at created_at and merged_at timestamps + pr_query = """ + SELECT number, status, domain, agent, branch, source_path, + created_at, merged_at + FROM prs + WHERE {where_clause} + ORDER BY COALESCE(merged_at, created_at) DESC + LIMIT ? + """ + + if cursor: + rows = conn.execute( + pr_query.format(where_clause="COALESCE(merged_at, created_at) < ?"), + (cursor, limit + 1) + ).fetchall() + else: + rows = conn.execute( + pr_query.format(where_clause="1=1"), + (limit + 1,) + ).fetchall() + + # Known knowledge agents for branch-prefix inference + knowledge_agents = {'rio', 'clay', 'theseus', 'vida', 'astra', 'leo'} + + for row in rows: + row_dict = dict(row) + operation = STATUS_TO_OPERATION.get(row_dict['status'], 'infra') + description = pr_description(row_dict) + + # Use merged_at if available (more interesting event), else created_at + timestamp = row_dict['merged_at'] or row_dict['created_at'] + + # Infer agent from branch prefix if DB column is null + # Branch format: agent-name/claims-description + agent = row_dict['agent'] + if not agent and row_dict.get('branch'): + prefix = row_dict['branch'].split('/')[0].lower() + if prefix in knowledge_agents: + agent = prefix + + events.append({ + 'timestamp': timestamp, + 'agent': agent, + 'operation': operation, + 'target': (row_dict['branch'] or '').split('/')[-1] if row_dict['branch'] else None, + 'domain': row_dict['domain'], + 'description': description, + 'status': row_dict['status'], + 'pr_number': row_dict['number'], + }) + + # Source 2: Audit log events (secondary — pipeline-level) + # Only include if we haven't hit our limit from PRs alone + if len(events) < limit: + remaining = limit - len(events) + 1 + audit_query = """ + SELECT timestamp, stage, event, detail + FROM audit_log + WHERE {where_clause} + ORDER BY timestamp DESC + LIMIT ? + """ + + if cursor: + audit_rows = conn.execute( + audit_query.format(where_clause="timestamp < ?"), + (cursor, remaining) + ).fetchall() + else: + audit_rows = conn.execute( + audit_query.format(where_clause="1=1"), + (remaining,) + ).fetchall() + + for row in audit_rows: + row_dict = dict(row) + operation = STAGE_TO_OPERATION.get(row_dict['stage'], 'infra') + description = audit_description(row_dict) + + events.append({ + 'timestamp': row_dict['timestamp'], + 'agent': None, # audit_log has no agent column + 'operation': operation, + 'target': None, + 'domain': None, + 'description': description, + 'status': None, + 'pr_number': None, + }) + + conn.close() + except sqlite3.Error as e: + return web.json_response({'error': f'Database error: {e}'}, status=500) + + # Sort all events by timestamp descending + events.sort(key=lambda e: e['timestamp'] or '', reverse=True) + + # Apply limit and check for more + has_more = len(events) > limit + events = events[:limit] + + # Cursor is the timestamp of the last event returned + next_cursor = events[-1]['timestamp'] if events else None + + return web.json_response({ + 'events': events, + 'limit': limit, + 'cursor': next_cursor, + 'has_more': has_more, + }) + + +# --- Integration snippet for app.py --- +# Add to your route setup: +# +# from activity_endpoint import handle_activity +# app.router.add_get('/api/activity', handle_activity) +# +# Requires: app['db_path'] set to the pipeline.db path +# e.g.: app['db_path'] = '/opt/teleo-eval/pipeline/pipeline.db' diff --git a/diagnostics/alerting.py b/diagnostics/alerting.py new file mode 100644 index 0000000..0c84ae5 --- /dev/null +++ b/diagnostics/alerting.py @@ -0,0 +1,537 @@ +"""Argus active monitoring — health watchdog, quality regression, throughput anomaly detection. + +Provides check functions that detect problems and return structured alerts. +Called by /check endpoint (periodic cron) or on-demand. + +Alert schema: + { + "id": str, # unique key for dedup (e.g. "dormant:ganymede") + "severity": str, # "critical" | "warning" | "info" + "category": str, # "health" | "quality" | "throughput" | "failure_pattern" + "title": str, # human-readable headline + "detail": str, # actionable description + "agent": str|None, # affected agent (if applicable) + "domain": str|None, # affected domain (if applicable) + "detected_at": str, # ISO timestamp + "auto_resolve": bool, # clears when condition clears + } +""" + +import json +import sqlite3 +import statistics +from datetime import datetime, timezone + + +# ─── Agent-domain mapping (static config, maintained by Argus) ────────────── + +AGENT_DOMAINS = { + "rio": ["internet-finance"], + "clay": ["creative-industries"], + "ganymede": None, # reviewer — cross-domain + "epimetheus": None, # infra + "leo": None, # standards + "oberon": None, # evolution tracking + "vida": None, # health monitoring + "hermes": None, # comms + "astra": None, # research +} + +# Thresholds +DORMANCY_HOURS = 48 +APPROVAL_DROP_THRESHOLD = 15 # percentage points below 7-day baseline +THROUGHPUT_DROP_RATIO = 0.5 # alert if today < 50% of 7-day SMA +REJECTION_SPIKE_RATIO = 0.20 # single reason > 20% of recent rejections +STUCK_LOOP_THRESHOLD = 3 # same agent + same rejection reason > N times in 6h +COST_SPIKE_RATIO = 2.0 # daily cost > 2x 7-day average + + +def _now_iso() -> str: + return datetime.now(timezone.utc).isoformat() + + +# ─── Check: Agent Health (dormancy detection) ─────────────────────────────── + + +def check_agent_health(conn: sqlite3.Connection) -> list[dict]: + """Detect agents with no PR activity in the last DORMANCY_HOURS hours.""" + alerts = [] + + # Get last activity per agent + rows = conn.execute( + """SELECT agent, MAX(last_attempt) as latest, COUNT(*) as total_prs + FROM prs WHERE agent IS NOT NULL + GROUP BY agent""" + ).fetchall() + + now = datetime.now(timezone.utc) + for r in rows: + agent = r["agent"] + latest = r["latest"] + if not latest: + continue + + last_dt = datetime.fromisoformat(latest) + if last_dt.tzinfo is None: + last_dt = last_dt.replace(tzinfo=timezone.utc) + + hours_since = (now - last_dt).total_seconds() / 3600 + + if hours_since > DORMANCY_HOURS: + alerts.append({ + "id": f"dormant:{agent}", + "severity": "warning", + "category": "health", + "title": f"Agent '{agent}' dormant for {int(hours_since)}h", + "detail": ( + f"No PR activity since {latest}. " + f"Last seen {int(hours_since)}h ago (threshold: {DORMANCY_HOURS}h). " + f"Total historical PRs: {r['total_prs']}." + ), + "agent": agent, + "domain": None, + "detected_at": _now_iso(), + "auto_resolve": True, + }) + + return alerts + + +# ─── Check: Quality Regression (approval rate drop) ───────────────────────── + + +def check_quality_regression(conn: sqlite3.Connection) -> list[dict]: + """Detect approval rate drops vs 7-day baseline, per agent and per domain.""" + alerts = [] + + # 7-day baseline approval rate (overall) + baseline = conn.execute( + """SELECT + COUNT(CASE WHEN event='approved' THEN 1 END) as approved, + COUNT(*) as total + FROM audit_log + WHERE stage='evaluate' + AND event IN ('approved','changes_requested','domain_rejected','tier05_rejected') + AND timestamp > datetime('now', '-7 days')""" + ).fetchone() + baseline_rate = (baseline["approved"] / baseline["total"] * 100) if baseline["total"] else None + + # 24h approval rate (overall) + recent = conn.execute( + """SELECT + COUNT(CASE WHEN event='approved' THEN 1 END) as approved, + COUNT(*) as total + FROM audit_log + WHERE stage='evaluate' + AND event IN ('approved','changes_requested','domain_rejected','tier05_rejected') + AND timestamp > datetime('now', '-24 hours')""" + ).fetchone() + recent_rate = (recent["approved"] / recent["total"] * 100) if recent["total"] else None + + if baseline_rate is not None and recent_rate is not None: + drop = baseline_rate - recent_rate + if drop > APPROVAL_DROP_THRESHOLD: + alerts.append({ + "id": "quality_regression:overall", + "severity": "critical", + "category": "quality", + "title": f"Approval rate dropped {drop:.0f}pp (24h: {recent_rate:.0f}% vs 7d: {baseline_rate:.0f}%)", + "detail": ( + f"24h approval rate ({recent_rate:.1f}%) is {drop:.1f} percentage points below " + f"7-day baseline ({baseline_rate:.1f}%). " + f"Evaluated {recent['total']} PRs in last 24h." + ), + "agent": None, + "domain": None, + "detected_at": _now_iso(), + "auto_resolve": True, + }) + + # Per-agent approval rate (24h vs 7d) — only for agents with >=5 evals in each window + # COALESCE: rejection events use $.agent, eval events use $.domain_agent (Epimetheus 2026-03-28) + _check_approval_by_dimension(conn, alerts, "agent", "COALESCE(json_extract(detail, '$.agent'), json_extract(detail, '$.domain_agent'))") + + # Per-domain approval rate (24h vs 7d) — Theseus addition + _check_approval_by_dimension(conn, alerts, "domain", "json_extract(detail, '$.domain')") + + return alerts + + +def _check_approval_by_dimension(conn, alerts, dim_name, dim_expr): + """Check approval rate regression grouped by a dimension (agent or domain).""" + # 7-day baseline per dimension + baseline_rows = conn.execute( + f"""SELECT {dim_expr} as dim_val, + COUNT(CASE WHEN event='approved' THEN 1 END) as approved, + COUNT(*) as total + FROM audit_log + WHERE stage='evaluate' + AND event IN ('approved','changes_requested','domain_rejected','tier05_rejected') + AND timestamp > datetime('now', '-7 days') + AND {dim_expr} IS NOT NULL + GROUP BY dim_val HAVING total >= 5""" + ).fetchall() + baselines = {r["dim_val"]: (r["approved"] / r["total"] * 100) for r in baseline_rows} + + # 24h per dimension + recent_rows = conn.execute( + f"""SELECT {dim_expr} as dim_val, + COUNT(CASE WHEN event='approved' THEN 1 END) as approved, + COUNT(*) as total + FROM audit_log + WHERE stage='evaluate' + AND event IN ('approved','changes_requested','domain_rejected','tier05_rejected') + AND timestamp > datetime('now', '-24 hours') + AND {dim_expr} IS NOT NULL + GROUP BY dim_val HAVING total >= 5""" + ).fetchall() + + for r in recent_rows: + val = r["dim_val"] + if val not in baselines: + continue + recent_rate = r["approved"] / r["total"] * 100 + base_rate = baselines[val] + drop = base_rate - recent_rate + if drop > APPROVAL_DROP_THRESHOLD: + alerts.append({ + "id": f"quality_regression:{dim_name}:{val}", + "severity": "warning", + "category": "quality", + "title": f"{dim_name.title()} '{val}' approval dropped {drop:.0f}pp", + "detail": ( + f"24h: {recent_rate:.1f}% vs 7d baseline: {base_rate:.1f}% " + f"({r['total']} evals in 24h)." + ), + "agent": val if dim_name == "agent" else None, + "domain": val if dim_name == "domain" else None, + "detected_at": _now_iso(), + "auto_resolve": True, + }) + + +# ─── Check: Throughput Anomaly ────────────────────────────────────────────── + + +def check_throughput(conn: sqlite3.Connection) -> list[dict]: + """Detect throughput stalling — today vs 7-day SMA.""" + alerts = [] + + # Daily merged counts for last 7 days + rows = conn.execute( + """SELECT date(merged_at) as day, COUNT(*) as n + FROM prs WHERE merged_at > datetime('now', '-7 days') + GROUP BY day ORDER BY day""" + ).fetchall() + + if len(rows) < 2: + return alerts # Not enough data + + daily_counts = [r["n"] for r in rows] + sma = statistics.mean(daily_counts[:-1]) if len(daily_counts) > 1 else daily_counts[0] + today_count = daily_counts[-1] + + if sma > 0 and today_count < sma * THROUGHPUT_DROP_RATIO: + alerts.append({ + "id": "throughput:stalling", + "severity": "warning", + "category": "throughput", + "title": f"Throughput stalling: {today_count} merges today vs {sma:.0f}/day avg", + "detail": ( + f"Today's merge count ({today_count}) is below {THROUGHPUT_DROP_RATIO:.0%} of " + f"7-day average ({sma:.1f}/day). Daily counts: {daily_counts}." + ), + "agent": None, + "domain": None, + "detected_at": _now_iso(), + "auto_resolve": True, + }) + + return alerts + + +# ─── Check: Rejection Reason Spike ───────────────────────────────────────── + + +def check_rejection_spike(conn: sqlite3.Connection) -> list[dict]: + """Detect single rejection reason exceeding REJECTION_SPIKE_RATIO of recent rejections.""" + alerts = [] + + # Total rejections in 24h + total = conn.execute( + """SELECT COUNT(*) as n FROM audit_log + WHERE stage='evaluate' + AND event IN ('changes_requested','domain_rejected','tier05_rejected') + AND timestamp > datetime('now', '-24 hours')""" + ).fetchone()["n"] + + if total < 10: + return alerts # Not enough data + + # Count by rejection tag + tags = conn.execute( + """SELECT value as tag, COUNT(*) as cnt + FROM audit_log, json_each(json_extract(detail, '$.issues')) + WHERE stage='evaluate' + AND event IN ('changes_requested','domain_rejected','tier05_rejected') + AND timestamp > datetime('now', '-24 hours') + GROUP BY tag ORDER BY cnt DESC""" + ).fetchall() + + for t in tags: + ratio = t["cnt"] / total + if ratio > REJECTION_SPIKE_RATIO: + alerts.append({ + "id": f"rejection_spike:{t['tag']}", + "severity": "warning", + "category": "quality", + "title": f"Rejection reason '{t['tag']}' at {ratio:.0%} of rejections", + "detail": ( + f"'{t['tag']}' accounts for {t['cnt']}/{total} rejections in 24h " + f"({ratio:.1%}). Threshold: {REJECTION_SPIKE_RATIO:.0%}." + ), + "agent": None, + "domain": None, + "detected_at": _now_iso(), + "auto_resolve": True, + }) + + return alerts + + +# ─── Check: Stuck Loops ──────────────────────────────────────────────────── + + +def check_stuck_loops(conn: sqlite3.Connection) -> list[dict]: + """Detect agents repeatedly failing on the same rejection reason.""" + alerts = [] + + # COALESCE: rejection events use $.agent, eval events use $.domain_agent (Epimetheus 2026-03-28) + rows = conn.execute( + """SELECT COALESCE(json_extract(detail, '$.agent'), json_extract(detail, '$.domain_agent')) as agent, + value as tag, + COUNT(*) as cnt + FROM audit_log, json_each(json_extract(detail, '$.issues')) + WHERE stage='evaluate' + AND event IN ('changes_requested','domain_rejected','tier05_rejected') + AND timestamp > datetime('now', '-6 hours') + AND COALESCE(json_extract(detail, '$.agent'), json_extract(detail, '$.domain_agent')) IS NOT NULL + GROUP BY agent, tag + HAVING cnt > ?""", + (STUCK_LOOP_THRESHOLD,), + ).fetchall() + + for r in rows: + alerts.append({ + "id": f"stuck_loop:{r['agent']}:{r['tag']}", + "severity": "critical", + "category": "health", + "title": f"Agent '{r['agent']}' stuck: '{r['tag']}' failed {r['cnt']}x in 6h", + "detail": ( + f"Agent '{r['agent']}' has been rejected for '{r['tag']}' " + f"{r['cnt']} times in the last 6 hours (threshold: {STUCK_LOOP_THRESHOLD}). " + f"Stop and reassess." + ), + "agent": r["agent"], + "domain": None, + "detected_at": _now_iso(), + "auto_resolve": True, + }) + + return alerts + + +# ─── Check: Cost Spikes ──────────────────────────────────────────────────── + + +def check_cost_spikes(conn: sqlite3.Connection) -> list[dict]: + """Detect daily cost exceeding 2x of 7-day average per agent.""" + alerts = [] + + # Check if costs table exists and has agent column + try: + cols = conn.execute("PRAGMA table_info(costs)").fetchall() + col_names = {c["name"] for c in cols} + except sqlite3.Error: + return alerts + + if "agent" not in col_names or "cost_usd" not in col_names: + # Fall back to per-PR cost tracking + rows = conn.execute( + """SELECT agent, + SUM(CASE WHEN created_at > datetime('now', '-1 day') THEN cost_usd ELSE 0 END) as today_cost, + SUM(CASE WHEN created_at > datetime('now', '-7 days') THEN cost_usd ELSE 0 END) / 7.0 as avg_daily + FROM prs WHERE agent IS NOT NULL AND cost_usd > 0 + GROUP BY agent + HAVING avg_daily > 0""" + ).fetchall() + else: + rows = conn.execute( + """SELECT agent, + SUM(CASE WHEN timestamp > datetime('now', '-1 day') THEN cost_usd ELSE 0 END) as today_cost, + SUM(CASE WHEN timestamp > datetime('now', '-7 days') THEN cost_usd ELSE 0 END) / 7.0 as avg_daily + FROM costs WHERE agent IS NOT NULL + GROUP BY agent + HAVING avg_daily > 0""" + ).fetchall() + + for r in rows: + if r["avg_daily"] and r["today_cost"] > r["avg_daily"] * COST_SPIKE_RATIO: + ratio = r["today_cost"] / r["avg_daily"] + alerts.append({ + "id": f"cost_spike:{r['agent']}", + "severity": "warning", + "category": "health", + "title": f"Agent '{r['agent']}' cost spike: ${r['today_cost']:.2f} today ({ratio:.1f}x avg)", + "detail": ( + f"Today's cost (${r['today_cost']:.2f}) is {ratio:.1f}x the 7-day daily average " + f"(${r['avg_daily']:.2f}). Threshold: {COST_SPIKE_RATIO}x." + ), + "agent": r["agent"], + "domain": None, + "detected_at": _now_iso(), + "auto_resolve": True, + }) + + return alerts + + +# ─── Check: Domain Rejection Patterns (Theseus addition) ─────────────────── + + +def check_domain_rejection_patterns(conn: sqlite3.Connection) -> list[dict]: + """Track rejection reason shift per domain — surfaces domain maturity issues.""" + alerts = [] + + # Per-domain rejection breakdown in 24h + rows = conn.execute( + """SELECT json_extract(detail, '$.domain') as domain, + value as tag, + COUNT(*) as cnt + FROM audit_log, json_each(json_extract(detail, '$.issues')) + WHERE stage='evaluate' + AND event IN ('changes_requested','domain_rejected','tier05_rejected') + AND timestamp > datetime('now', '-24 hours') + AND json_extract(detail, '$.domain') IS NOT NULL + GROUP BY domain, tag + ORDER BY domain, cnt DESC""" + ).fetchall() + + # Group by domain + domain_tags = {} + for r in rows: + d = r["domain"] + if d not in domain_tags: + domain_tags[d] = [] + domain_tags[d].append({"tag": r["tag"], "count": r["cnt"]}) + + # Flag if a domain has >50% of rejections from a single reason (concentrated failure) + for domain, tags in domain_tags.items(): + total = sum(t["count"] for t in tags) + if total < 5: + continue + top = tags[0] + ratio = top["count"] / total + if ratio > 0.5: + alerts.append({ + "id": f"domain_rejection_pattern:{domain}:{top['tag']}", + "severity": "info", + "category": "failure_pattern", + "title": f"Domain '{domain}': {ratio:.0%} of rejections are '{top['tag']}'", + "detail": ( + f"In domain '{domain}', {top['count']}/{total} rejections (24h) are for " + f"'{top['tag']}'. This may indicate a systematic issue with evidence standards " + f"or schema compliance in this domain." + ), + "agent": None, + "domain": domain, + "detected_at": _now_iso(), + "auto_resolve": True, + }) + + return alerts + + +# ─── Failure Report Generator ─────────────────────────────────────────────── + + +def generate_failure_report(conn: sqlite3.Connection, agent: str, hours: int = 24) -> dict | None: + """Compile a failure report for a specific agent. + + Returns top rejection reasons, example PRs, and suggested fixes. + Designed to be sent directly to the agent via Pentagon messaging. + """ + hours = int(hours) # defensive — callers should pass int, but enforce it + rows = conn.execute( + """SELECT value as tag, COUNT(*) as cnt, + GROUP_CONCAT(DISTINCT json_extract(detail, '$.pr')) as pr_numbers + FROM audit_log, json_each(json_extract(detail, '$.issues')) + WHERE stage='evaluate' + AND event IN ('changes_requested','domain_rejected','tier05_rejected') + AND COALESCE(json_extract(detail, '$.agent'), json_extract(detail, '$.domain_agent')) = ? + AND timestamp > datetime('now', ? || ' hours') + GROUP BY tag ORDER BY cnt DESC + LIMIT 5""", + (agent, f"-{hours}"), + ).fetchall() + + if not rows: + return None + + total_rejections = sum(r["cnt"] for r in rows) + top_reasons = [] + for r in rows: + prs = r["pr_numbers"].split(",")[:3] if r["pr_numbers"] else [] + top_reasons.append({ + "reason": r["tag"], + "count": r["cnt"], + "pct": round(r["cnt"] / total_rejections * 100, 1), + "example_prs": prs, + "suggestion": _suggest_fix(r["tag"]), + }) + + return { + "agent": agent, + "period_hours": hours, + "total_rejections": total_rejections, + "top_reasons": top_reasons, + "generated_at": _now_iso(), + } + + +def _suggest_fix(rejection_tag: str) -> str: + """Map known rejection reasons to actionable suggestions.""" + suggestions = { + "broken_wiki_links": "Check that all [[wiki links]] in claims resolve to existing files. Run link validation before submitting.", + "near_duplicate": "Search existing claims before creating new ones. Use semantic search to find similar claims.", + "frontmatter_schema": "Validate YAML frontmatter against the claim schema. Required fields: title, domain, confidence, type.", + "weak_evidence": "Add concrete sources, data points, or citations. Claims need evidence that can be independently verified.", + "missing_confidence": "Every claim needs a confidence level: proven, likely, experimental, or speculative.", + "domain_mismatch": "Ensure claims are filed under the correct domain. Check domain definitions if unsure.", + "too_broad": "Break broad claims into specific, testable sub-claims.", + "missing_links": "Claims should link to related claims, entities, or sources. Isolated claims are harder to verify.", + } + return suggestions.get(rejection_tag, f"Review rejection reason '{rejection_tag}' and adjust extraction accordingly.") + + +# ─── Run All Checks ──────────────────────────────────────────────────────── + + +def run_all_checks(conn: sqlite3.Connection) -> list[dict]: + """Execute all check functions and return combined alerts.""" + alerts = [] + alerts.extend(check_agent_health(conn)) + alerts.extend(check_quality_regression(conn)) + alerts.extend(check_throughput(conn)) + alerts.extend(check_rejection_spike(conn)) + alerts.extend(check_stuck_loops(conn)) + alerts.extend(check_cost_spikes(conn)) + alerts.extend(check_domain_rejection_patterns(conn)) + return alerts + + +def format_alert_message(alert: dict) -> str: + """Format an alert for Pentagon messaging.""" + severity_icon = {"critical": "!!", "warning": "!", "info": "~"} + icon = severity_icon.get(alert["severity"], "?") + return f"[{icon}] {alert['title']}\n{alert['detail']}" diff --git a/diagnostics/alerting_routes.py b/diagnostics/alerting_routes.py new file mode 100644 index 0000000..fd35740 --- /dev/null +++ b/diagnostics/alerting_routes.py @@ -0,0 +1,125 @@ +"""Route handlers for /check and /api/alerts endpoints. + +Import into app.py and register routes in create_app(). +""" + +import json +import logging +from datetime import datetime, timezone + +from aiohttp import web +from alerting import run_all_checks, generate_failure_report, format_alert_message # requires CWD = deploy dir; switch to relative import if packaged + +logger = logging.getLogger("argus.alerting") + +# In-memory alert store (replaced each /check cycle, persists between requests) +_active_alerts: list[dict] = [] +_last_check: str | None = None + + +async def handle_check(request): + """GET /check — run all monitoring checks, update active alerts, return results. + + Designed to be called by systemd timer every 5 minutes. + Returns JSON summary of all detected issues. + """ + conn = request.app["_alerting_conn_func"]() + try: + alerts = run_all_checks(conn) + except Exception as e: + logger.error("Check failed: %s", e) + return web.json_response({"error": str(e)}, status=500) + + global _active_alerts, _last_check + _active_alerts = alerts + _last_check = datetime.now(timezone.utc).isoformat() + + # Generate failure reports for agents with stuck loops + failure_reports = {} + stuck_agents = {a["agent"] for a in alerts if a["category"] == "health" and "stuck" in a["id"] and a["agent"]} + for agent in stuck_agents: + report = generate_failure_report(conn, agent) + if report: + failure_reports[agent] = report + + result = { + "checked_at": _last_check, + "alert_count": len(alerts), + "critical": sum(1 for a in alerts if a["severity"] == "critical"), + "warning": sum(1 for a in alerts if a["severity"] == "warning"), + "info": sum(1 for a in alerts if a["severity"] == "info"), + "alerts": alerts, + "failure_reports": failure_reports, + } + + logger.info( + "Check complete: %d alerts (%d critical, %d warning)", + len(alerts), + result["critical"], + result["warning"], + ) + + return web.json_response(result) + + +async def handle_api_alerts(request): + """GET /api/alerts — return current active alerts. + + Query params: + severity: filter by severity (critical, warning, info) + category: filter by category (health, quality, throughput, failure_pattern) + agent: filter by agent name + domain: filter by domain + """ + alerts = list(_active_alerts) + + # Filters + severity = request.query.get("severity") + if severity: + alerts = [a for a in alerts if a["severity"] == severity] + + category = request.query.get("category") + if category: + alerts = [a for a in alerts if a["category"] == category] + + agent = request.query.get("agent") + if agent: + alerts = [a for a in alerts if a.get("agent") == agent] + + domain = request.query.get("domain") + if domain: + alerts = [a for a in alerts if a.get("domain") == domain] + + return web.json_response({ + "alerts": alerts, + "total": len(alerts), + "last_check": _last_check, + }) + + +async def handle_api_failure_report(request): + """GET /api/failure-report/{agent} — generate failure report for an agent. + + Query params: + hours: lookback window (default 24) + """ + agent = request.match_info["agent"] + hours = int(request.query.get("hours", "24")) + conn = request.app["_alerting_conn_func"]() + + report = generate_failure_report(conn, agent, hours) + if not report: + return web.json_response({"agent": agent, "status": "no_rejections", "period_hours": hours}) + + return web.json_response(report) + + +def register_alerting_routes(app, get_conn_func): + """Register alerting routes on the app. + + get_conn_func: callable that returns a read-only sqlite3.Connection + """ + app["_alerting_conn_func"] = get_conn_func + app.router.add_get("/check", handle_check) + app.router.add_get("/api/alerts", handle_api_alerts) + app.router.add_get("/api/failure-report/{agent}", handle_api_failure_report) diff --git a/diagnostics/app.py b/diagnostics/app.py index 96beb2e..5fa66e7 100644 --- a/diagnostics/app.py +++ b/diagnostics/app.py @@ -22,6 +22,9 @@ from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "pipeline")) from aiohttp import web +from review_queue_routes import register_review_queue_routes +from daily_digest_routes import register_daily_digest_routes +from response_audit_routes import register_response_audit_routes, RESPONSE_AUDIT_PUBLIC_PATHS from lib.search import search as kb_search, embed_query, search_qdrant logger = logging.getLogger("argus") @@ -38,8 +41,8 @@ CLAIM_INDEX_URL = os.environ.get("CLAIM_INDEX_URL", "http://localhost:8080/claim API_KEY_FILE = Path(os.environ.get("ARGUS_API_KEY_FILE", "/opt/teleo-eval/secrets/argus-api-key")) # Endpoints that skip auth (dashboard is public for now, can lock later) -_PUBLIC_PATHS = frozenset({"/", "/api/metrics", "/api/snapshots", "/api/vital-signs", - "/api/contributors", "/api/domains"}) +_PUBLIC_PATHS = frozenset({"/", "/prs", "/ops", "/health", "/agents", "/epistemic", "/legacy", "/audit", "/api/metrics", "/api/snapshots", "/api/vital-signs", + "/api/contributors", "/api/domains", "/api/audit", "/api/yield", "/api/cost-per-claim", "/api/fix-rates", "/api/compute-profile", "/api/review-queue", "/api/daily-digest"}) def _get_db() -> sqlite3.Connection: @@ -426,6 +429,40 @@ def _compute_vital_signs(conn) -> dict: "conversion_rate": round(merged_prs / total_prs, 3) if total_prs else 0, } + # Queue staleness — sources unprocessed for >7 days + stale_buckets = conn.execute(""" + SELECT + CASE + WHEN created_at < datetime('now', '-30 days') THEN '30d+' + WHEN created_at < datetime('now', '-14 days') THEN '14-30d' + WHEN created_at < datetime('now', '-7 days') THEN '7-14d' + ELSE 'fresh' + END as age_bucket, + COUNT(*) as cnt + FROM sources + WHERE status = 'unprocessed' + GROUP BY age_bucket + """).fetchall() + stale_map = {r["age_bucket"]: r["cnt"] for r in stale_buckets} + stale_total = sum(v for k, v in stale_map.items() if k != "fresh") + + oldest_unprocessed = conn.execute( + "SELECT MIN(created_at) as oldest FROM sources WHERE status='unprocessed'" + ).fetchone() + oldest_age_days = None + if oldest_unprocessed and oldest_unprocessed["oldest"]: + oldest_dt = datetime.fromisoformat(oldest_unprocessed["oldest"]) + if oldest_dt.tzinfo is None: + oldest_dt = oldest_dt.replace(tzinfo=timezone.utc) + oldest_age_days = round((datetime.now(timezone.utc) - oldest_dt).total_seconds() / 86400, 1) + + queue_staleness = { + "stale_count": stale_total, + "buckets": stale_map, + "oldest_age_days": oldest_age_days, + "status": "healthy" if stale_total == 0 else ("warning" if stale_total <= 10 else "critical"), + } + return { "claim_index_status": claim_index_status, "review_throughput": { @@ -453,6 +490,7 @@ def _compute_vital_signs(conn) -> dict: "status": "healthy" if not stagnant_domains else "warning", }, "funnel": funnel, + "queue_staleness": queue_staleness, } @@ -470,7 +508,7 @@ def _load_secret(path: Path) -> str | None: @web.middleware async def auth_middleware(request, handler): """API key check. Public paths skip auth. Protected paths require X-Api-Key header.""" - if request.path in _PUBLIC_PATHS: + if request.path in _PUBLIC_PATHS or request.path in RESPONSE_AUDIT_PUBLIC_PATHS or request.path.startswith("/api/response-audit/"): return await handler(request) expected = request.app.get("api_key") if not expected: @@ -660,6 +698,86 @@ async def handle_api_search(request): return web.json_response(result) +async def handle_api_audit(request): + """GET /api/audit — query response_audit table for agent response diagnostics. + + Query params: + agent: filter by agent name (optional) + query: search in query text (optional) + limit: max results, default 50, max 200 (optional) + offset: pagination offset (optional) + days: how many days back, default 7 (optional) + """ + conn = _conn(request) + + # Check if response_audit table exists + table_check = conn.execute( + "SELECT name FROM sqlite_master WHERE type='table' AND name='response_audit'" + ).fetchone() + if not table_check: + return web.json_response({"error": "response_audit table not found"}, status=404) + + agent = request.query.get("agent") + status_filter = request.query.get("status", "").strip() + query_filter = request.query.get("query", "").strip() + limit = min(int(request.query.get("limit", "50")), 200) + offset = int(request.query.get("offset", "0")) + days = int(request.query.get("days", "7")) + + where_clauses = ["timestamp > datetime('now', ?||' days')"] + params: list = [f"-{days}"] + + if agent: + where_clauses.append("agent = ?") + params.append(agent) + if status_filter: + where_clauses.append("retrieval_status LIKE ?") + params.append(f"{status_filter}%") + if query_filter: + where_clauses.append("query LIKE ?") + params.append(f"%{query_filter}%") + + where_sql = " AND ".join(where_clauses) + + rows = conn.execute( + f"""SELECT id, timestamp, agent, chat_id, user, model, query, + conversation_window, entities_matched, claims_matched, + retrieval_layers_hit, retrieval_gap, research_context, + tool_calls, display_response, confidence_score, response_time_ms, + retrieval_status + FROM response_audit + WHERE {where_sql} + ORDER BY timestamp DESC + LIMIT ? OFFSET ?""", + params + [limit, offset], + ).fetchall() + + total = conn.execute( + f"SELECT COUNT(*) as n FROM response_audit WHERE {where_sql}", + params, + ).fetchone()["n"] + + results = [] + for r in rows: + row_dict = dict(r) + # Parse JSON fields for the response + for json_field in ("claims_matched", "entities_matched", "retrieval_layers_hit", + "tool_calls", "conversation_window"): + if row_dict.get(json_field): + try: + row_dict[json_field] = json.loads(row_dict[json_field]) + except (json.JSONDecodeError, TypeError): + pass + results.append(row_dict) + + return web.json_response({"total": total, "results": results}) + + +async def handle_audit_page(request): + """GET /audit — HTML page for browsing response audit data.""" + return web.Response(content_type="text/html", text=_render_audit_page()) + + async def handle_api_usage(request): """POST /api/usage — log claim usage for analytics. @@ -706,6 +824,370 @@ def _render_error(message: str) -> str:

Argus

{message}

Check if teleo-pipeline.service is running and pipeline.db exists.

""" +def _render_audit_page() -> str: + """Render the response audit browser page.""" + return """ + + +Argus — Response Audit + + + +
← Dashboard
+

Response Audit

+

Browse agent responses, retrieved claims, and search quality metrics

+ +
+ + + + + +
+ +
+
+ + + + +
+

+ Compute Profile (Claude Max Telemetry) +

+
+
+
Cache Hit Rate
+
—
+
prompt tokens from cache
+
+
+
Avg Latency
+
—
+
ms per Max call
+
+
+
Subscription Calls
+
—
+
vs — API calls
+
+
+
API-Equivalent Cost
+
—
+
saved by Max subscription
+
+
+
+
+

Tokens by Stage & Billing

+ +
+
+

Cache Breakdown (Max Calls)

+ +
+
+
+
+ + +""" + + def _render_dashboard(metrics, snapshots, changes, vital_signs, contributors_principal, contributors_agent, domain_breakdown, now) -> str: """Render the full operational dashboard as HTML with Chart.js.""" @@ -1063,7 +1545,8 @@ def _render_dashboard(metrics, snapshots, changes, vital_signs, contributors_pri Snapshots · Vital Signs · Contributors · - Domains + Domains · + Response Audit + + +
+
+ Knowledge Production + + The three numbers that matter · yield · + cost · + fix rates + +
+ + +
+
+
Extraction Yield
+
—
+
loading...
+
+
+
Cost / Merged Claim
+
—
+
loading...
+
+
+
Fix Success Rate
+
—
+
loading...
+
+
+ + +
+
+

Extraction Yield by Agent (daily)

+ +
+
+

Cost per Merged Claim (daily)

+ +
+
+ + +
+
+

Fix Success by Rejection Reason

+ +
+
+

Cost by Stage

+ +
+
+
+ + + +
+

+ Compute Profile (Claude Max Telemetry) +

+
+
+
Cache Hit Rate
+
—
+
prompt tokens from cache
+
+
+
Avg Latency
+
—
+
ms per Max call
+
+
+
Subscription Calls
+
—
+
vs — API calls
+
+
+
API-Equivalent Cost
+
—
+
saved by Max subscription
+
+
+
+
+

Tokens by Stage & Billing

+ +
+
+

Cache Breakdown (Max Calls)

+ +
+
+
+
+ + """ # ─── App factory ───────────────────────────────────────────────────────────── +from alerting_routes import register_alerting_routes +from tier1_routes import register_tier1_routes + +# 4-page dashboard imports +from dashboard_ops import render_ops_page +from dashboard_health import render_health_page +from dashboard_agents import render_agents_page +from dashboard_epistemic import render_epistemic_page +from dashboard_prs import render_prs_page +from dashboard_routes import register_dashboard_routes + # requires CWD = deploy dir + +def _conn_from_app(app): + import sqlite3 + conn = app["db"] + try: + conn.execute("SELECT 1") + except sqlite3.Error: + conn = _get_db() + app["db"] = conn + return conn + + + + + +# ─── 4-page dashboard route handlers ─────────────────────────────────────── + +async def handle_ops_page(request): + """GET /ops — Pipeline Operations page.""" + try: + conn = _conn(request) + metrics = _current_metrics(conn) + snapshots = _snapshot_history(conn, days=7) + changes = _version_changes(conn, days=30) + vital_signs = _compute_vital_signs(conn) + except Exception as e: + return web.Response(text=_render_error(f"Database error: {e}"), content_type="text/html", status=503) + now = datetime.now(timezone.utc) + return web.Response(text=render_ops_page(metrics, snapshots, changes, vital_signs, now), content_type="text/html") + + +async def handle_health_page(request): + """GET /health — Knowledge Health page.""" + try: + conn = _conn(request) + vital_signs = _compute_vital_signs(conn) + domain_breakdown = _domain_breakdown(conn) + except Exception as e: + return web.Response(text=_render_error(f"Database error: {e}"), content_type="text/html", status=503) + now = datetime.now(timezone.utc) + return web.Response(text=render_health_page(vital_signs, domain_breakdown, now), content_type="text/html") + + +async def handle_agents_page(request): + """GET /agents — Agent Performance page.""" + try: + conn = _conn(request) + contributors_principal = _contributor_leaderboard(conn, limit=10, view="principal") + contributors_agent = _contributor_leaderboard(conn, limit=10, view="agent") + except Exception as e: + return web.Response(text=_render_error(f"Database error: {e}"), content_type="text/html", status=503) + now = datetime.now(timezone.utc) + return web.Response(text=render_agents_page(contributors_principal, contributors_agent, now), content_type="text/html") + + +async def handle_epistemic_page(request): + """GET /epistemic — Epistemic Integrity page.""" + try: + conn = _conn(request) + vital_signs = _compute_vital_signs(conn) + except Exception as e: + return web.Response(text=_render_error(f"Database error: {e}"), content_type="text/html", status=503) + now = datetime.now(timezone.utc) + return web.Response(text=render_epistemic_page(vital_signs, now), content_type="text/html") + + + + +async def handle_prs_page(request): + """GET /prs — PR Lifecycle page.""" + from datetime import datetime, timezone + now = datetime.now(timezone.utc) + return web.Response(text=render_prs_page(now), content_type="text/html") + +async def handle_root_redirect(request): + """GET / — redirect to /ops.""" + raise web.HTTPFound("/ops") + def create_app() -> web.Application: app = web.Application(middlewares=[auth_middleware]) @@ -1243,14 +2254,32 @@ def create_app() -> web.Application: logger.info("API key auth enabled (protected endpoints require X-Api-Key)") else: logger.info("No API key configured — all endpoints open") - app.router.add_get("/", handle_dashboard) + # Root redirects to /ops (legacy dashboard still at /legacy) + app.router.add_get("/", handle_root_redirect) + app.router.add_get("/prs", handle_prs_page) + app.router.add_get("/ops", handle_ops_page) + app.router.add_get("/health", handle_health_page) + app.router.add_get("/agents", handle_agents_page) + app.router.add_get("/epistemic", handle_epistemic_page) + app.router.add_get("/legacy", handle_dashboard) # keep old dashboard for rollback app.router.add_get("/api/metrics", handle_api_metrics) app.router.add_get("/api/snapshots", handle_api_snapshots) app.router.add_get("/api/vital-signs", handle_api_vital_signs) app.router.add_get("/api/contributors", handle_api_contributors) app.router.add_get("/api/domains", handle_api_domains) app.router.add_get("/api/search", handle_api_search) + app.router.add_get("/api/audit", handle_api_audit) + app.router.add_get("/audit", handle_audit_page) app.router.add_post("/api/usage", handle_api_usage) + # Alerting - active monitoring endpoints + register_alerting_routes(app, lambda: _conn_from_app(app)) + register_tier1_routes(app, lambda: _conn_from_app(app)) + register_dashboard_routes(app, lambda: _conn_from_app(app)) + register_review_queue_routes(app) + register_daily_digest_routes(app, db_path=str(DB_PATH)) + # Response audit - cost tracking + reasoning traces + app["db_path"] = str(DB_PATH) + register_response_audit_routes(app) app.on_cleanup.append(_cleanup) return app diff --git a/diagnostics/backfill_submitted_by.py b/diagnostics/backfill_submitted_by.py new file mode 100644 index 0000000..7e1b44d --- /dev/null +++ b/diagnostics/backfill_submitted_by.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python3 +"""One-time backfill: populate submitted_by on prs table from source archive files. + +Matches PRs to sources via branch name slug → source filename. +Reads proposed_by and intake_tier from source frontmatter. + +Run: python3 backfill_submitted_by.py +""" + +import os +import re +import sqlite3 +from pathlib import Path + +DB_PATH = os.environ.get("DB_PATH", "/opt/teleo-eval/pipeline/pipeline.db") +ARCHIVE_DIR = Path(os.environ.get("ARCHIVE_DIR", "/opt/teleo-eval/workspaces/main/inbox/archive")) + + +def parse_frontmatter(path: Path) -> dict: + """Parse YAML-like frontmatter from a markdown file.""" + text = path.read_text(encoding="utf-8", errors="replace") + if not text.startswith("---"): + return {} + end = text.find("---", 3) + if end == -1: + return {} + fm = {} + for line in text[3:end].strip().split("\n"): + line = line.strip() + if not line or ":" not in line: + continue + key, _, val = line.partition(":") + key = key.strip() + val = val.strip().strip('"').strip("'") + if val.lower() == "null" or val == "": + val = None + fm[key] = val + return fm + + +def slug_from_branch(branch: str) -> str: + """Extract source slug from branch name like 'extract/2026-04-06-slug-hash'.""" + if "/" in branch: + branch = branch.split("/", 1)[1] + # Strip trailing hex hash (e.g., -3e68, -a6af) + branch = re.sub(r"-[0-9a-f]{4}$", "", branch) + return branch + + +def main(): + conn = sqlite3.connect(DB_PATH, timeout=30) + conn.row_factory = sqlite3.Row + + # Build source index: filename stem → frontmatter + source_index = {} + if ARCHIVE_DIR.exists(): + for f in ARCHIVE_DIR.glob("*.md"): + fm = parse_frontmatter(f) + source_index[f.stem] = fm + print(f"Indexed {len(source_index)} source files from {ARCHIVE_DIR}") + + # Get all PRs without submitted_by + prs = conn.execute( + "SELECT number, branch FROM prs WHERE submitted_by IS NULL AND branch IS NOT NULL" + ).fetchall() + print(f"Found {len(prs)} PRs without submitted_by") + + updated = 0 + for pr in prs: + branch = pr["branch"] + slug = slug_from_branch(branch) + + # Try to match slug to a source file + fm = source_index.get(slug) + if not fm: + # Try partial matching: slug might be a substring of the source filename + for stem, sfm in source_index.items(): + if slug in stem or stem in slug: + fm = sfm + break + + if fm: + proposed_by = fm.get("proposed_by") + intake_tier = fm.get("intake_tier") + + if proposed_by: + contributor = proposed_by.strip().strip('"').strip("'") + elif intake_tier == "research-task": + # Derive agent from branch prefix + prefix = branch.split("/", 1)[0] if "/" in branch else "unknown" + agent_map = { + "extract": "pipeline", "ingestion": "pipeline", + "rio": "rio", "theseus": "theseus", "vida": "vida", + "clay": "clay", "astra": "astra", "leo": "leo", + "reweave": "pipeline", + } + agent = agent_map.get(prefix, prefix) + contributor = f"{agent} (self-directed)" + elif intake_tier == "directed": + contributor = "@m3taversal" + else: + # Default: if source exists but no proposed_by, it was Cory's submission + contributor = "@m3taversal" + + if contributor: + conn.execute( + "UPDATE prs SET submitted_by = ?, source_path = ? WHERE number = ?", + (contributor, f"inbox/archive/{slug}.md", pr["number"]), + ) + updated += 1 + else: + # Agent-named branches from overnight research sessions + if branch.startswith(("rio/", "theseus/", "vida/", "clay/", "astra/", "leo/")): + agent = branch.split("/", 1)[0] + conn.execute( + "UPDATE prs SET submitted_by = ? WHERE number = ?", + (f"{agent} (self-directed)", pr["number"]), + ) + updated += 1 + elif branch.startswith("reweave/"): + conn.execute( + "UPDATE prs SET submitted_by = 'pipeline (reweave)' WHERE number = ?", + (pr["number"],), + ) + updated += 1 + else: + # Everything else (extract/, ingestion/, unknown) → Cory directed it + conn.execute( + "UPDATE prs SET submitted_by = '@m3taversal' WHERE number = ?", + (pr["number"],), + ) + updated += 1 + + conn.commit() + conn.close() + print(f"Updated {updated}/{len(prs)} PRs with submitted_by") + + +if __name__ == "__main__": + main() diff --git a/diagnostics/daily_digest.py b/diagnostics/daily_digest.py new file mode 100644 index 0000000..2a8c7bc --- /dev/null +++ b/diagnostics/daily_digest.py @@ -0,0 +1,312 @@ +"""Daily digest: aggregates 24h activity for Telegram bot consumption. + +Data sources: + - pipeline.db: merged PRs, audit events, contributor activity + - Forgejo API: PR descriptions for claim summaries + - claim-index: total claims, domain breakdown + - review queue: pending approval counts + +Endpoint: GET /api/daily-digest?hours=24 +""" + +import asyncio +import logging +import sqlite3 +from datetime import datetime, timezone, timedelta +from typing import Any + +import aiohttp + +logger = logging.getLogger("argus.daily_digest") + +FORGEJO_BASE = "https://git.livingip.xyz/api/v1" +REPO = "teleo/teleo-codex" +CLAIM_INDEX_URL = "http://localhost:8080/claim-index" + + +async def fetch_daily_digest( + db_path: str, + forgejo_token: str | None = None, + hours: int = 24, + timeout_s: int = 15, +) -> dict[str, Any]: + """Build the daily digest payload. + + Returns structured data for Epimetheus's Telegram bot to format and send. + """ + cutoff = (datetime.now(timezone.utc) - timedelta(hours=hours)).isoformat() + + # Parallel: DB queries + HTTP fetches + db_data = _query_db(db_path, cutoff, hours) + + headers = {"Accept": "application/json"} + if forgejo_token: + headers["Authorization"] = f"token {forgejo_token}" + + connector = aiohttp.TCPConnector(ssl=False) + async with aiohttp.ClientSession(headers=headers, connector=connector) as session: + # Fetch claim-index, merged PR details from Forgejo, and open PR count in parallel + merged_numbers = [pr["number"] for pr in db_data["merged_prs"]] + + tasks = [ + _fetch_claim_index(session, timeout_s), + _fetch_merged_pr_details(session, merged_numbers, timeout_s), + _fetch_open_pr_count(session, timeout_s), + ] + claim_index, pr_details, open_pr_count = await asyncio.gather(*tasks) + + # Enrich merged PRs with Forgejo descriptions + merged_claims = _build_merged_claims(db_data["merged_prs"], pr_details) + + return { + "period_hours": hours, + "generated_at": datetime.now(timezone.utc).isoformat(), + "claims_merged": merged_claims, + "pipeline_stats": { + "prs_merged": db_data["prs_merged"], + "prs_opened": db_data["prs_opened"], + "prs_rejected": db_data["prs_rejected"], + "approval_rate": db_data["approval_rate"], + "top_rejection_reasons": db_data["top_rejection_reasons"], + }, + "agent_activity": db_data["agent_activity"], + "pending_review": { + "open_prs": open_pr_count, + }, + "knowledge_base": { + "total_claims": claim_index.get("total_claims", 0), + "domains": claim_index.get("domains", {}), + "orphan_ratio": claim_index.get("orphan_ratio", 0), + "cross_domain_links": claim_index.get("cross_domain_links", 0), + }, + } + + +def _query_db(db_path: str, cutoff: str, hours: int) -> dict[str, Any]: + """Run all DB queries synchronously (SQLite is fast enough for digest).""" + conn = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True) + conn.row_factory = sqlite3.Row + try: + # Merged PRs in period + merged_prs = conn.execute( + """SELECT number, branch, domain, agent, commit_type, merged_at, cost_usd + FROM prs WHERE status = 'merged' AND merged_at >= ? + ORDER BY merged_at DESC""", + (cutoff,), + ).fetchall() + + prs_merged = len(merged_prs) + + # PRs opened in period + prs_opened = conn.execute( + "SELECT COUNT(*) FROM prs WHERE created_at >= ?", (cutoff,) + ).fetchone()[0] + + # Rejected PRs in period (closed/zombie with rejection events) + prs_rejected = conn.execute( + """SELECT COUNT(DISTINCT json_extract(detail, '$.pr')) + FROM audit_log + WHERE stage = 'evaluate' + AND event IN ('domain_rejected', 'tier05_rejected') + AND timestamp >= ?""", + (cutoff,), + ).fetchone()[0] + + # Approval rate + total_evaluated = prs_merged + prs_rejected + approval_rate = round(prs_merged / total_evaluated * 100, 1) if total_evaluated > 0 else 0.0 + + # Top rejection reasons + rejection_rows = conn.execute( + """SELECT json_extract(detail, '$.issues') as issues + FROM audit_log + WHERE stage = 'evaluate' + AND event IN ('domain_rejected', 'tier05_rejected') + AND timestamp >= ? + AND json_valid(detail)""", + (cutoff,), + ).fetchall() + + reason_counts: dict[str, int] = {} + import json + for row in rejection_rows: + if row["issues"]: + try: + issues = json.loads(row["issues"]) + if isinstance(issues, list): + for issue in issues: + reason_counts[issue] = reason_counts.get(issue, 0) + 1 + except (json.JSONDecodeError, TypeError): + pass + + top_rejection_reasons = sorted(reason_counts.items(), key=lambda x: -x[1])[:5] + top_rejection_reasons = [{"reason": r, "count": c} for r, c in top_rejection_reasons] + + # Agent activity — who contributed what + agent_rows = conn.execute( + """SELECT agent, + COUNT(*) as total, + SUM(CASE WHEN status = 'merged' THEN 1 ELSE 0 END) as merged, + SUM(CASE WHEN commit_type = 'extract' OR commit_type = 'research' THEN 1 ELSE 0 END) as extractions, + SUM(CASE WHEN commit_type = 'challenge' THEN 1 ELSE 0 END) as challenges, + SUM(CASE WHEN commit_type = 'enrich' OR commit_type = 'reweave' THEN 1 ELSE 0 END) as enrichments, + SUM(CASE WHEN commit_type = 'synthesize' THEN 1 ELSE 0 END) as syntheses + FROM prs + WHERE created_at >= ? AND agent IS NOT NULL AND agent != '' + GROUP BY agent + ORDER BY merged DESC""", + (cutoff,), + ).fetchall() + + agent_activity = [ + { + "agent": row["agent"], + "prs_total": row["total"], + "prs_merged": row["merged"], + "extractions": row["extractions"], + "challenges": row["challenges"], + "enrichments": row["enrichments"], + "syntheses": row["syntheses"], + } + for row in agent_rows + ] + + return { + "merged_prs": [dict(pr) for pr in merged_prs], + "prs_merged": prs_merged, + "prs_opened": prs_opened, + "prs_rejected": prs_rejected, + "approval_rate": approval_rate, + "top_rejection_reasons": top_rejection_reasons, + "agent_activity": agent_activity, + } + finally: + conn.close() + + +async def _fetch_claim_index(session: aiohttp.ClientSession, timeout_s: int) -> dict: + """Fetch claim-index summary stats.""" + try: + async with session.get( + CLAIM_INDEX_URL, + timeout=aiohttp.ClientTimeout(total=timeout_s), + ) as resp: + if resp.status == 200: + data = await resp.json() + return { + "total_claims": data.get("total_claims", 0), + "domains": data.get("domains", {}), + "orphan_ratio": data.get("orphan_ratio", 0), + "cross_domain_links": data.get("cross_domain_links", 0), + } + except Exception as e: + logger.warning("Failed to fetch claim-index: %s", e) + return {} + + +async def _fetch_merged_pr_details( + session: aiohttp.ClientSession, + pr_numbers: list[int], + timeout_s: int, +) -> dict[int, dict]: + """Fetch PR details from Forgejo for merged PRs (parallel).""" + if not pr_numbers: + return {} + + async def _fetch_one(n: int) -> tuple[int, dict]: + url = f"{FORGEJO_BASE}/repos/{REPO}/pulls/{n}" + try: + async with session.get(url, timeout=aiohttp.ClientTimeout(total=timeout_s)) as resp: + if resp.status == 200: + return n, await resp.json() + except Exception as e: + logger.warning("Failed to fetch PR #%d: %s", n, e) + return n, {} + + results = await asyncio.gather(*[_fetch_one(n) for n in pr_numbers]) + return {n: data for n, data in results} + + +async def _fetch_open_pr_count(session: aiohttp.ClientSession, timeout_s: int) -> int: + """Get count of open PRs from Forgejo.""" + url = f"{FORGEJO_BASE}/repos/{REPO}/pulls?state=open&limit=1" + try: + async with session.get(url, timeout=aiohttp.ClientTimeout(total=timeout_s)) as resp: + if resp.status == 200: + # Forgejo returns X-Total-Count header + total = resp.headers.get("X-Total-Count") + if total is not None: + return int(total) + # Fallback: fetch all and count + data = await resp.json() + return len(data) + except Exception as e: + logger.warning("Failed to fetch open PR count: %s", e) + return 0 + + +def _build_merged_claims( + merged_prs: list[dict], + pr_details: dict[int, dict], +) -> list[dict]: + """Build claim summaries from merged PRs + Forgejo PR bodies.""" + claims = [] + for pr in merged_prs: + number = pr["number"] + detail = pr_details.get(number, {}) + + # Extract summary from PR body (first paragraph or first 200 chars) + body = detail.get("body", "") or "" + summary = _extract_summary(body) + + claims.append({ + "pr_number": number, + "title": detail.get("title", pr.get("branch", f"PR #{number}")), + "agent": pr.get("agent", "unknown"), + "domain": pr.get("domain", "unknown"), + "commit_type": pr.get("commit_type", "knowledge"), + "summary": summary, + "merged_at": pr.get("merged_at", ""), + "cost_usd": pr.get("cost_usd", 0.0), + "url": detail.get("html_url", ""), + }) + + return claims + + +def _extract_summary(body: str) -> str: + """Extract a 1-2 sentence summary from PR body markdown. + + Looks for a Summary section first, then falls back to first non-header paragraph. + """ + if not body: + return "" + + lines = body.strip().split("\n") + + # Look for ## Summary section + in_summary = False + summary_lines = [] + for line in lines: + if line.strip().lower().startswith("## summary"): + in_summary = True + continue + if in_summary: + if line.startswith("##"): + break + stripped = line.strip() + if stripped and not stripped.startswith("- ["): # skip checklists + summary_lines.append(stripped) + if len(summary_lines) >= 3: + break + + if summary_lines: + return " ".join(summary_lines)[:300] + + # Fallback: first non-header, non-empty paragraph + for line in lines: + stripped = line.strip() + if stripped and not stripped.startswith("#") and not stripped.startswith("- ["): + return stripped[:300] + + return "" diff --git a/diagnostics/daily_digest_routes.py b/diagnostics/daily_digest_routes.py new file mode 100644 index 0000000..13c7924 --- /dev/null +++ b/diagnostics/daily_digest_routes.py @@ -0,0 +1,62 @@ +"""Route handlers for /api/daily-digest endpoint. + +Import into app.py and register routes in create_app(). +""" + +import logging + +from aiohttp import web +from daily_digest import fetch_daily_digest + +logger = logging.getLogger("argus.daily_digest") + + +async def handle_daily_digest(request): + """GET /api/daily-digest — structured data for Telegram daily digest. + + Query params: + hours: lookback period in hours (default: 24, max: 168) + + Returns JSON with: + claims_merged: merged claims with summaries + pipeline_stats: PRs merged/opened/rejected, approval rate, rejection reasons + agent_activity: per-agent contribution breakdown + pending_review: open PR count + knowledge_base: total claims, domain breakdown, orphan ratio + """ + # Validate hours param + try: + hours = int(request.query.get("hours", 24)) + hours = max(1, min(hours, 168)) # clamp to 1h-7d + except (ValueError, TypeError): + hours = 24 + + db_path = request.app.get("_db_path") + if not db_path: + return web.json_response({"error": "database not configured"}, status=500) + + token = request.app.get("_forgejo_token") + + try: + digest = await fetch_daily_digest( + db_path=db_path, + forgejo_token=token, + hours=hours, + ) + except Exception as e: + logger.error("Daily digest fetch failed: %s", e) + return web.json_response({"error": str(e)}, status=500) + + return web.json_response(digest) + + +def register_daily_digest_routes(app, db_path: str, forgejo_token: str | None = None): + """Register daily digest routes on the app. + + db_path: path to pipeline.db + forgejo_token: optional Forgejo API token + """ + app["_db_path"] = db_path + if forgejo_token: + app["_forgejo_token"] = forgejo_token + app.router.add_get("/api/daily-digest", handle_daily_digest) diff --git a/diagnostics/dashboard-v2.html b/diagnostics/dashboard-v2.html new file mode 100644 index 0000000..f9c7437 --- /dev/null +++ b/diagnostics/dashboard-v2.html @@ -0,0 +1,1424 @@ + + + + + +Teleo Codex — Live Terminal + + + + + +
+
TELEO CODEX
+
+ LIVE + MERGED -- + APPROVAL -- + TTM -- + + ← v1 Pipeline Ops +
+
+ + +
+ + + +
+ +
+ + +
+ +
+
+
--
+
TOTAL CLAIMS
+
+ +
+
+
--
+
APPROVAL RATE
+
+ +
+
+
--
+
ORPHAN RATIO
+
+ +
+
+
--
+
EVIDENCE AGE
+
+ +
+
+
--
+
CROSS-DOMAIN
+
+ +
+
+
--
+
REVIEW BACKLOG
+
+ +
+
+ + +
+ +
+
ACTIVITY FEED --
+
+ +
+ + +
+
DOMAIN ACTIVITY 7D
+
+
+ + +
+
AGENTS
+
+
+
CIRCUIT BREAKERS
+
+
+
+
+ + +
+ FUNNEL +
+
+ + +
+
+ CONTRIBUTORS + + +
+
+
+
#
HANDLE
MERGED
TIER
DOMAINS
CI SCORE
LAST
+
+
+
+ + +
+
+
+
+
DOMAIN
+
VOLUME
+
TOTAL
+
7D
+
STATUS
+
+
+
+
+
+
+
+
+ +
+ + + + diff --git a/diagnostics/dashboard_agents.py b/diagnostics/dashboard_agents.py new file mode 100644 index 0000000..aa1e73b --- /dev/null +++ b/diagnostics/dashboard_agents.py @@ -0,0 +1,348 @@ +"""Page 3: Agent Performance — "Who's contributing what?" + +Slim version v2 per Cory feedback (2026-04-03): +- Hero: total merged, rejection rate, claims/week — 3 numbers +- Table: agent, merged, rejection rate, last active, inbox depth — 5 columns +- One chart: weekly contributions by agent (stacked bar) +- No CI scores, no yield (redundant with rejection rate), no top issue (too granular) + +Fetches /api/agents-dashboard + /api/agent-state, merges client-side. +""" + +from datetime import datetime + +from shared_ui import render_page + + +def render_agents_page(contributors_principal: list, contributors_agent: list, now: datetime) -> str: + """Render the slim Agent Performance page.""" + + body = """ + +
+
Loading...
+
+ + +
+
Agent Breakdown (30d)
+
+ + + + + + + + + +
AgentMergedRejection RateLast ActiveInbox
Loading...
+
+
+ + +
+
+

Claims Merged per Week by Agent

+ +
+
+ + +
+
Agent Scorecard (Structured Reviews)
+
+ + +
Loading...
+
+
+
+ + +
+
Latest Session Digests
+
+
Loading...
+
+
+""" + + scripts = """""" + + return render_page( + title="Agent Performance", + subtitle="Who's contributing what?", + active_path="/agents", + body_html=body, + scripts=scripts, + timestamp=now.strftime("%Y-%m-%d %H:%M UTC"), + ) diff --git a/diagnostics/dashboard_epistemic.py b/diagnostics/dashboard_epistemic.py new file mode 100644 index 0000000..c0e1c09 --- /dev/null +++ b/diagnostics/dashboard_epistemic.py @@ -0,0 +1,239 @@ +"""Page 4: Epistemic Integrity — "Can we trust what we know?" + +Live sections: +- Confidence calibration (from claim-index via vital signs) +- Cascade coverage (from audit_log stage='cascade') +- Review quality (from review_records table) + +Placeholder sections: +- Multi-model agreement (needs model_evals table) +- Belief staleness (needs cascade tracking to give it meaning) +- Divergence tracking (needs divergence events) +""" + +import json +from datetime import datetime + +from shared_ui import render_page + + +def render_epistemic_page(vital_signs: dict, now: datetime) -> str: + """Render the Epistemic Integrity page.""" + + vs_conf = vital_signs.get("confidence_distribution", {}) + total_claims = sum(vs_conf.values()) if vs_conf else 0 + + # Confidence calibration table + conf_rows = "" + for level in ["proven", "likely", "experimental", "speculative"]: + count = vs_conf.get(level, 0) + pct = round(count / total_claims * 100, 1) if total_claims else 0 + conf_rows += f'{level}{count}{pct}%' + + body = f""" + +
+
Confidence Calibration
+
+
+ + + {conf_rows} +
LevelClaimsShare
+
+ Total claims: {total_claims} +
+
+
+

Confidence Distribution

+ +
+
+
+ + +
+
Cascade Coverage
+
+
Loading cascade data...
+
+
+ + +
+
Review Quality
+
+
Loading review data...
+
+
+ + +
+
Multi-Model Agreement
+
+
⚙
+
+ Multi-model agreement rate requires the model_evals table.
+ Blocked on: model_evals table creation (Theseus 2 Phase 3) +
+
+ Current eval models: Haiku (triage), GPT-4o (domain), Sonnet/Opus (Leo).
+ Agreement tracking needs per-model verdicts stored separately. +
+
+
+ + +
+
Belief Staleness
+
+
⏲
+
+ Belief staleness scan will compare belief file depends_on frontmatter
+ against claim merged_at timestamps.
+ Ready to implement once cascade tracking accumulates data +
+
+
+""" + + scripts = f"""""" + + return render_page( + title="Epistemic Integrity", + subtitle="Can we trust what we know?", + active_path="/epistemic", + body_html=body, + scripts=scripts, + timestamp=now.strftime("%Y-%m-%d %H:%M UTC"), + ) diff --git a/diagnostics/dashboard_health.py b/diagnostics/dashboard_health.py new file mode 100644 index 0000000..70b59cc --- /dev/null +++ b/diagnostics/dashboard_health.py @@ -0,0 +1,223 @@ +"""Page 2: Knowledge Health — "What do we know and how good is it?" + +Renders: claims by domain, Herfindahl index, evidence freshness, +orphan ratio, link density, confidence distribution, extraction yield. + +Data sources: /api/vital-signs, /api/herfindahl, /api/extraction-yield-by-domain, +/api/domains, claim-index (cached). +""" + +import json +from datetime import datetime + +from shared_ui import render_page + + +def render_health_page(vital_signs: dict, domain_breakdown: dict, now: datetime) -> str: + """Render the Knowledge Health page.""" + + # --- Vital signs data --- + vs_orphan = vital_signs.get("orphan_ratio", {}) + orphan_ratio_val = vs_orphan.get("ratio") + orphan_color = {"healthy": "green", "warning": "yellow", "critical": "red"}.get(vs_orphan.get("status", ""), "") + orphan_display = f"{orphan_ratio_val:.1%}" if orphan_ratio_val is not None else "—" + + vs_linkage = vital_signs.get("linkage_density") or {} + linkage_display = f'{vs_linkage.get("avg_outgoing_links", "—")}' + cross_domain_ratio = vs_linkage.get("cross_domain_ratio") + cross_domain_color = "green" if cross_domain_ratio and cross_domain_ratio >= 0.15 else ( + "yellow" if cross_domain_ratio and cross_domain_ratio >= 0.05 else "red" + ) if cross_domain_ratio is not None else "" + + vs_fresh = vital_signs.get("evidence_freshness") or {} + fresh_display = f'{vs_fresh.get("median_age_days", "—")}' if vs_fresh.get("median_age_days") else "—" + fresh_pct = vs_fresh.get("fresh_30d_pct", 0) + + vs_conf = vital_signs.get("confidence_distribution", {}) + + # Domain activity + stagnant = vital_signs.get("domain_activity", {}).get("stagnant", []) + active_domains = vital_signs.get("domain_activity", {}).get("active", []) + + claim_status = vital_signs.get("claim_index_status", "unavailable") + + # Domain breakdown table + domain_rows = "" + for domain, stats in sorted(domain_breakdown.items(), key=lambda x: x[1].get("knowledge_prs", 0), reverse=True): + if stats.get("knowledge_prs", 0) > 0: + top_contribs = ", ".join(f'{c["handle"]} ({c["claims"]})' for c in stats.get("contributors", [])[:3]) + domain_rows += f""" + {domain} + {stats["knowledge_prs"]} + {stats["total_prs"]} + {top_contribs} + """ + + body = f""" + +
+
+
Orphan Ratio
+
{orphan_display}
+
{vs_orphan.get("count", "?")} / {vs_orphan.get("total", "?")} claims · target <15%
+
+
+
Avg Links/Claim
+
{linkage_display}
+
cross-domain: {f"{cross_domain_ratio:.1%}" if cross_domain_ratio is not None else "—"} · target 15-30%
+
+
+
Evidence Freshness
+
{fresh_display}d median
+
{vs_fresh.get("fresh_30d_count", "?")} claims <30d old · {fresh_pct:.0f}% fresh
+
+
+
Confidence Spread
+
{" / ".join(f"{vs_conf.get(k, 0)}" for k in ["proven", "likely", "experimental", "speculative"])}
+
proven / likely / experimental / speculative
+
+
+
Claim Index
+
{claim_status}
+
{vs_orphan.get("total", "?")} claims indexed
+
+
+ + +
+
+
Domain Concentration
+
+
Loading...
+
+
+
+
Extraction Yield by Domain
+
+
Loading...
+
+
+
+ + +
+
+

Claims by Domain

+ +
+
+

Confidence Distribution

+ +
+
+ + +
+
Contributions by Domain
+
+ + + {domain_rows if domain_rows else ""} +
DomainKnowledge PRsTotal PRsTop Contributors
No domain data
+
+
+ + +{"" if not stagnant else f''' +
+
Stagnation Alerts
+
+

Domains with no PR activity in 7 days: {", ".join(stagnant)}

+
+
+'''} +""" + + scripts = f"""""" + + return render_page( + title="Knowledge Health", + subtitle="What do we know and how good is it?", + active_path="/health", + body_html=body, + scripts=scripts, + timestamp=now.strftime("%Y-%m-%d %H:%M UTC"), + ) diff --git a/diagnostics/dashboard_ops.py b/diagnostics/dashboard_ops.py new file mode 100644 index 0000000..0b465b6 --- /dev/null +++ b/diagnostics/dashboard_ops.py @@ -0,0 +1,464 @@ +"""Page 1: Pipeline Operations — "Is the machine running?" + +Renders: queue depth, throughput, error rate, stage flow, breakers, +funnel, rejection reasons, fix cycle, time-series charts. + +All data comes from existing endpoints: /api/metrics, /api/snapshots, +/api/stage-times, /api/alerts, /api/fix-rates. +""" + +import json +from datetime import datetime, timezone + +from shared_ui import render_page + + +def render_ops_page(metrics: dict, snapshots: list, changes: list, + vital_signs: dict, now: datetime) -> str: + """Render the Pipeline Operations page.""" + + # --- Prepare chart data --- + timestamps = [s["ts"] for s in snapshots] + throughput_data = [s.get("throughput_1h", 0) for s in snapshots] + approval_data = [(s.get("approval_rate") or 0) * 100 for s in snapshots] + open_prs_data = [s.get("open_prs", 0) for s in snapshots] + merged_data = [s.get("merged_total", 0) for s in snapshots] + + rej_wiki = [s.get("rejection_broken_wiki_links", 0) for s in snapshots] + rej_schema = [s.get("rejection_frontmatter_schema", 0) for s in snapshots] + rej_dup = [s.get("rejection_near_duplicate", 0) for s in snapshots] + rej_conf = [s.get("rejection_confidence", 0) for s in snapshots] + rej_other = [s.get("rejection_other", 0) for s in snapshots] + + # origin_agent/origin_human removed — replaced by /api/growth chart + + annotations_js = json.dumps([ + { + "type": "line", "xMin": c["ts"], "xMax": c["ts"], + "borderColor": "#d29922" if c["type"] == "prompt" else "#58a6ff", + "borderWidth": 1, "borderDash": [4, 4], + "label": {"display": True, "content": f"{c['type']}: {c.get('to', '?')}", + "position": "start", "backgroundColor": "#161b22", + "color": "#8b949e", "font": {"size": 10}}, + } + for c in changes + ]) + + # --- Status helpers --- + sm = metrics["status_map"] + ar = metrics["approval_rate"] + ar_color = "green" if ar > 0.5 else ("yellow" if ar > 0.2 else "red") + fr_color = "green" if metrics["fix_rate"] > 0.3 else ("yellow" if metrics["fix_rate"] > 0.1 else "red") + + vs_review = vital_signs["review_throughput"] + vs_status_color = {"healthy": "green", "warning": "yellow", "critical": "red"}.get(vs_review["status"], "yellow") + + # --- Rejection reasons table --- + reason_rows = "".join( + f'{r["tag"]}{r["unique_prs"]}' + f'{r["count"]}' + for r in metrics["rejection_reasons"] + ) + + # --- Breaker rows --- + breaker_rows = "" + for name, info in metrics["breakers"].items(): + state = info["state"] + color = "green" if state == "closed" else ("red" if state == "open" else "yellow") + age = f'{info.get("age_s", "?")}s ago' if "age_s" in info else "-" + breaker_rows += f'{name}{state}{info["failures"]}{age}' + + # --- Funnel --- + funnel = vital_signs["funnel"] + + # --- Queue staleness --- + qs = vital_signs.get("queue_staleness", {}) + stale_count = qs.get("stale_count", 0) + stale_status = qs.get("status", "healthy") + stale_color = {"healthy": "green", "warning": "yellow", "critical": "red"}.get(stale_status, "") + + body = f""" + +
+
+
Throughput
+
{metrics["throughput_1h"]}/hr
+
merged last hour
+
+
+
Approval Rate (24h)
+
{ar:.1%}
+
{metrics["approved_24h"]}/{metrics["evaluated_24h"]} evaluated
+
+
+
Review Backlog
+
{vs_review["backlog"]}
+
{vs_review["open_prs"]} open + {vs_review["reviewing_prs"]} reviewing + {vs_review["approved_waiting"]} approved
+
+
+
Merged Total
+
{sm.get("merged", 0)}
+
{sm.get("closed", 0)} closed
+
+
+
Fix Success
+
{metrics["fix_rate"]:.1%}
+
{metrics["fix_succeeded"]}/{metrics["fix_attempted"]} fixed
+
+
+
Time to Merge
+
{f"{metrics['median_ttm_minutes']:.0f}" if metrics["median_ttm_minutes"] else "—"}min
+
median (24h)
+
+
+ + +
+ + +
+
Pipeline Funnel
+
+
{funnel["sources_total"]}
Sources
+
→
+
{funnel["sources_queued"]}
In Queue
+
→
+
{funnel["sources_extracted"]}
Extracted
+
→
+
{funnel["prs_total"]}
PRs Created
+
→
+
{funnel["prs_merged"]}
Merged
+
→
+
{funnel["conversion_rate"]:.1%}
Conversion
+
+
+ Queue staleness: {stale_count} stale + {f'(oldest: {qs.get("oldest_age_days", "?")}d)' if stale_count > 0 else ""} +
+
+ + +
+
Stage Dwell Times
+
+
+ + + +
+
+
+

Throughput & Approval Rate

+ +
+
+

Rejection Reasons Over Time

+ +
+
+
+
+

PR Backlog

+ +
+
+

Cumulative Growth

+ +
+
+
+ + +
+
PR Trace Lookup
+
+
+ + +
+
+
+
+ + +
+
+
Top Rejection Reasons (24h)
+
+ + + {reason_rows if reason_rows else ""} +
IssuePRsEvents
No rejections in 24h
+
+
+
+
Circuit Breakers
+
+ + + {breaker_rows if breaker_rows else ""} +
StageStateFailuresLast Success
No breaker data
+
+
+
+""" + + scripts = f"""""" + + return render_page( + title="Pipeline Operations", + subtitle="Is the machine running?", + active_path="/ops", + body_html=body, + scripts=scripts, + timestamp=now.strftime("%Y-%m-%d %H:%M UTC"), + ) diff --git a/diagnostics/dashboard_prs.py b/diagnostics/dashboard_prs.py new file mode 100644 index 0000000..638ab52 --- /dev/null +++ b/diagnostics/dashboard_prs.py @@ -0,0 +1,561 @@ +"""PR Lifecycle dashboard — single-page view of every PR through the pipeline. + +Sortable table: PR#, summary, claims, domain, contributor, outcome, evals, evaluator, cost, date. +Click any row to expand: claim titles, eval chain, timeline, reviews, issues. +Hero cards: total PRs, merge rate, total claims, est. cost. + +Data sources: prs table, audit_log (eval rounds), review_records. +Owner: Ship +""" + +from datetime import datetime + +from shared_ui import render_page + + +EXTRA_CSS = """ + .content-wrapper { max-width: 1600px !important; } + .filters { display: flex; gap: 12px; flex-wrap: wrap; margin-bottom: 16px; } + .filters select, .filters input { + background: #161b22; color: #c9d1d9; border: 1px solid #30363d; + border-radius: 6px; padding: 6px 10px; font-size: 12px; } + .filters select:focus, .filters input:focus { border-color: #58a6ff; outline: none; } + .pr-table { width: 100%; border-collapse: collapse; font-size: 13px; table-layout: fixed; } + .pr-table th:nth-child(1) { width: 50px; } /* PR# */ + .pr-table th:nth-child(2) { width: 28%; } /* Summary */ + .pr-table th:nth-child(3) { width: 50px; } /* Claims */ + .pr-table th:nth-child(4) { width: 11%; } /* Domain */ + .pr-table th:nth-child(5) { width: 10%; } /* Contributor */ + .pr-table th:nth-child(6) { width: 10%; } /* Outcome */ + .pr-table th:nth-child(7) { width: 44px; } /* Evals */ + .pr-table th:nth-child(8) { width: 12%; } /* Evaluator */ + .pr-table th:nth-child(9) { width: 60px; } /* Cost */ + .pr-table th:nth-child(10) { width: 80px; } /* Date */ + .pr-table td { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; padding: 8px 6px; } + .pr-table td:nth-child(2) { white-space: normal; overflow: visible; line-height: 1.4; } + .pr-table th { cursor: pointer; user-select: none; position: relative; padding: 8px 18px 8px 6px; } + .pr-table th:hover { color: #58a6ff; } + .pr-table th .sort-arrow { position: absolute; right: 4px; top: 50%; transform: translateY(-50%); font-size: 10px; opacity: 0.5; } + .pr-table th.sorted .sort-arrow { opacity: 1; color: #58a6ff; } + .pr-table tr { cursor: pointer; transition: background 0.1s; } + .pr-table tbody tr:hover { background: #161b22; } + .pr-table .outcome-merged { color: #3fb950; } + .pr-table .outcome-closed { color: #f85149; } + .pr-table .outcome-open { color: #d29922; } + .pr-table .tier-deep { color: #bc8cff; font-weight: 600; } + .pr-table .tier-standard { color: #58a6ff; } + .pr-table .tier-light { color: #8b949e; } + .pr-table .pr-link { color: #58a6ff; text-decoration: none; } + .pr-table .pr-link:hover { text-decoration: underline; } + .pr-table td .summary-text { font-size: 12px; color: #c9d1d9; } + .pr-table td .review-snippet { font-size: 11px; color: #f85149; margin-top: 2px; opacity: 0.8; } + .pr-table td .model-tag { font-size: 10px; color: #6e7681; background: #161b22; border-radius: 3px; padding: 1px 4px; } + .pr-table td .contributor-tag { font-size: 11px; color: #d2a8ff; } + .pr-table td .contributor-self { font-size: 11px; color: #6e7681; font-style: italic; } + .pr-table td .expand-chevron { display: inline-block; width: 12px; color: #484f58; font-size: 10px; transition: transform 0.2s; } + .pr-table tr.expanded .expand-chevron { transform: rotate(90deg); color: #58a6ff; } + .trace-panel { background: #0d1117; border: 1px solid #30363d; border-radius: 8px; + padding: 16px; margin: 4px 0 8px 0; font-size: 12px; display: none; } + .trace-panel.open { display: block; } + .trace-panel h4 { color: #58a6ff; font-size: 12px; margin: 12px 0 6px 0; } + .trace-panel h4:first-child { margin-top: 0; } + .claim-list { list-style: none; padding: 0; margin: 0; } + .claim-list li { padding: 4px 0 4px 16px; border-left: 2px solid #238636; color: #c9d1d9; font-size: 12px; line-height: 1.5; } + .claim-list li .claim-confidence { font-size: 10px; color: #8b949e; margin-left: 6px; } + .issues-box { background: #1c1210; border: 1px solid #f8514933; border-radius: 6px; + padding: 8px 12px; margin: 4px 0; font-size: 12px; color: #f85149; } + .eval-chain { background: #161b22; border-radius: 6px; padding: 8px 12px; margin: 4px 0; font-size: 12px; } + .eval-chain .chain-step { display: inline-block; margin-right: 6px; } + .eval-chain .chain-arrow { color: #484f58; margin: 0 4px; } + .trace-timeline { list-style: none; padding: 0; } + .trace-timeline li { padding: 4px 0; border-left: 2px solid #30363d; padding-left: 12px; margin-left: 8px; } + .trace-timeline li .ts { color: #484f58; font-size: 11px; } + .trace-timeline li .ev { font-weight: 600; } + .trace-timeline li.ev-approved .ev { color: #3fb950; } + .trace-timeline li.ev-rejected .ev { color: #f85149; } + .trace-timeline li.ev-changes .ev { color: #d29922; } + .review-text { background: #161b22; padding: 8px 12px; border-radius: 4px; + margin: 4px 0; white-space: pre-wrap; font-size: 11px; color: #8b949e; max-height: 200px; overflow-y: auto; } + .pagination { display: flex; gap: 8px; align-items: center; justify-content: center; margin-top: 16px; } + .pagination button { background: #161b22; color: #c9d1d9; border: 1px solid #30363d; + border-radius: 4px; padding: 4px 12px; cursor: pointer; font-size: 12px; } + .pagination button:hover { border-color: #58a6ff; } + .pagination button:disabled { opacity: 0.4; cursor: default; } + .pagination .page-info { color: #8b949e; font-size: 12px; } +""" + + +def render_prs_page(now: datetime) -> str: + """Render the PR lifecycle page. All data loaded client-side via /api/pr-lifecycle.""" + + body = """ + +
+
Total PRs
--
+
Merge Rate
--
+
Total Claims
--
+
Est. Cost
--
+
+ + +
+ + + + + +
+ + +
+ + + + + + + + + + + + + + + + +
PR# ▲Summary ▲Claims ▲Domain ▲Contributor ▲Outcome ▲Evals ▲Evaluator ▲Cost ▲Date ▲
+
+ + + + """ + + scripts = """""" + + return render_page( + title="PR Lifecycle", + subtitle="Every PR through the pipeline — triage to merge", + active_path="/prs", + body_html=body, + scripts=scripts, + extra_css=EXTRA_CSS, + timestamp=now.strftime("%Y-%m-%d %H:%M UTC"), + ) diff --git a/diagnostics/dashboard_routes.py b/diagnostics/dashboard_routes.py new file mode 100644 index 0000000..2b399a1 --- /dev/null +++ b/diagnostics/dashboard_routes.py @@ -0,0 +1,934 @@ +"""New API endpoints for the 4-page dashboard. + +Endpoints: + GET /api/stage-times — median dwell time per pipeline stage + GET /api/herfindahl — domain concentration index + GET /api/agent-state — live agent-state from filesystem + GET /api/extraction-yield-by-domain — sources→claims conversion per domain + GET /api/agents-dashboard — batched agent performance payload + +Owner: Argus +""" + +import json +import logging +import os +import sqlite3 +import statistics +import time +import urllib.request +from datetime import datetime, timezone +from pathlib import Path + +from aiohttp import web + +logger = logging.getLogger("argus.dashboard_routes") + +# ─── Claim-index cache (60s TTL) ─────────────────────────────────────────── + +_claim_index_cache: dict | None = None +_claim_index_ts: float = 0 +CLAIM_INDEX_TTL = 60 # seconds + +CLAIM_INDEX_URL = os.environ.get("CLAIM_INDEX_URL", "http://localhost:8080/claim-index") +AGENT_STATE_DIR = Path(os.environ.get("AGENT_STATE_DIR", "/opt/teleo-eval/agent-state")) + + +def get_claim_index() -> dict | None: + """Fetch claim-index with 60s cache.""" + global _claim_index_cache, _claim_index_ts + now = time.monotonic() + if _claim_index_cache is not None and (now - _claim_index_ts) < CLAIM_INDEX_TTL: + return _claim_index_cache + try: + with urllib.request.urlopen(CLAIM_INDEX_URL, timeout=5) as resp: + data = json.loads(resp.read()) + _claim_index_cache = data + _claim_index_ts = now + return data + except Exception as e: + logger.warning("Failed to fetch claim-index: %s", e) + # Return stale cache if available + return _claim_index_cache + + +# ─── GET /api/stage-times ────────────────────────────────────────────────── + +async def handle_stage_times(request): + """Median dwell time per pipeline stage from audit_log timestamps. + + Stages: discover → validate → evaluate → merge + Returns median minutes between consecutive stages. + """ + conn = request.app["_get_conn"]() + hours = int(request.query.get("hours", "24")) + + # Get per-PR event timestamps + rows = conn.execute( + """SELECT json_extract(detail, '$.pr') as pr, event, timestamp + FROM audit_log + WHERE timestamp > datetime('now', ? || ' hours') + AND json_extract(detail, '$.pr') IS NOT NULL + ORDER BY json_extract(detail, '$.pr'), timestamp""", + (f"-{hours}",), + ).fetchall() + + # Group by PR + pr_events: dict[int, list] = {} + for r in rows: + pr = r["pr"] + if pr not in pr_events: + pr_events[pr] = [] + pr_events[pr].append({"event": r["event"], "ts": r["timestamp"]}) + + # Compute stage dwell times + stage_pairs = [ + ("pr_discovered", "tier0_complete", "Ingest → Validate"), + ("tier0_complete", "approved", "Validate → Approve"), + ("tier0_complete", "domain_rejected", "Validate → Reject"), + ("approved", "merged", "Approve → Merge"), + ] + + stage_times = {} + for start_event, end_event, label in stage_pairs: + durations = [] + for pr, events in pr_events.items(): + start_ts = None + end_ts = None + for e in events: + if e["event"] == start_event and start_ts is None: + start_ts = e["ts"] + if e["event"] == end_event and end_ts is None: + end_ts = e["ts"] + if start_ts and end_ts: + try: + s = datetime.fromisoformat(start_ts) + e = datetime.fromisoformat(end_ts) + mins = (e - s).total_seconds() / 60 + if mins >= 0: + durations.append(mins) + except (ValueError, TypeError): + pass + if durations: + stage_times[label] = { + "median_minutes": round(statistics.median(durations), 1), + "p90_minutes": round(sorted(durations)[int(len(durations) * 0.9)], 1) if len(durations) >= 5 else None, + "count": len(durations), + } + + return web.json_response({"hours": hours, "stages": stage_times}) + + +# ─── GET /api/herfindahl ────────────────────────────────────────────────── + +async def handle_herfindahl(request): + """Domain concentration index (Herfindahl-Hirschman). + + HHI = sum of (domain_share^2). 1.0 = single domain, lower = more diverse. + """ + conn = request.app["_get_conn"]() + days = int(request.query.get("days", "30")) + + rows = conn.execute( + """SELECT domain, COUNT(*) as cnt + FROM prs WHERE status='merged' AND domain IS NOT NULL + AND merged_at > datetime('now', ? || ' days') + GROUP BY domain""", + (f"-{days}",), + ).fetchall() + + if not rows: + return web.json_response({"hhi": 0, "domains": [], "days": days}) + + total = sum(r["cnt"] for r in rows) + domains = [] + hhi = 0 + for r in rows: + share = r["cnt"] / total + hhi += share ** 2 + domains.append({ + "domain": r["domain"], + "count": r["cnt"], + "share": round(share, 4), + }) + + domains.sort(key=lambda x: x["count"], reverse=True) + + # Interpret: HHI < 0.15 = diverse, 0.15-0.25 = moderate, >0.25 = concentrated + status = "diverse" if hhi < 0.15 else ("moderate" if hhi < 0.25 else "concentrated") + + return web.json_response({ + "hhi": round(hhi, 4), + "status": status, + "domains": domains, + "total_merged": total, + "days": days, + }) + + +# ─── GET /api/agent-state ───────────────────────────────────────────────── + +async def handle_agent_state(request): + """Read live agent-state from filesystem. 6 agents, ~1KB each.""" + if not AGENT_STATE_DIR.exists(): + return web.json_response({"error": "agent-state directory not found", "path": str(AGENT_STATE_DIR)}, status=404) + + agents = {} + for agent_dir in sorted(AGENT_STATE_DIR.iterdir()): + if not agent_dir.is_dir(): + continue + name = agent_dir.name + state = {"name": name} + + # metrics.json + metrics_file = agent_dir / "metrics.json" + if metrics_file.exists(): + try: + m = json.loads(metrics_file.read_text()) + state["last_active"] = m.get("updated_at") + state["metrics"] = m + except (json.JSONDecodeError, OSError): + state["metrics_error"] = True + + # tasks.json + tasks_file = agent_dir / "tasks.json" + if tasks_file.exists(): + try: + t = json.loads(tasks_file.read_text()) + state["tasks"] = t if isinstance(t, list) else [] + state["task_count"] = len(state["tasks"]) + except (json.JSONDecodeError, OSError): + state["tasks"] = [] + + # session.json + session_file = agent_dir / "session.json" + if session_file.exists(): + try: + s = json.loads(session_file.read_text()) + state["session"] = s + except (json.JSONDecodeError, OSError): + pass + + # inbox depth + inbox_dir = agent_dir / "inbox" + if inbox_dir.exists() and inbox_dir.is_dir(): + state["inbox_depth"] = len(list(inbox_dir.iterdir())) + else: + state["inbox_depth"] = 0 + + agents[name] = state + + return web.json_response({"agents": agents, "agent_count": len(agents)}) + + +# ─── GET /api/extraction-yield-by-domain ────────────────────────────────── + +async def handle_extraction_yield_by_domain(request): + """Sources → claims conversion rate per domain.""" + conn = request.app["_get_conn"]() + days = int(request.query.get("days", "30")) + + # Sources per domain (approximate from PR source_path domain) + source_counts = conn.execute( + """SELECT domain, COUNT(DISTINCT source_url) as sources + FROM sources s + JOIN prs p ON p.source_path LIKE '%' || s.url || '%' + WHERE s.created_at > datetime('now', ? || ' days') + GROUP BY domain""", + (f"-{days}",), + ).fetchall() + + # Fallback: simpler query if the join doesn't work well + merged_by_domain = conn.execute( + """SELECT domain, COUNT(*) as merged + FROM prs WHERE status='merged' AND domain IS NOT NULL + AND merged_at > datetime('now', ? || ' days') + GROUP BY domain""", + (f"-{days}",), + ).fetchall() + + sources_by_domain = conn.execute( + """SELECT domain, COUNT(*) as total_prs, + SUM(CASE WHEN status='merged' THEN 1 ELSE 0 END) as merged + FROM prs WHERE domain IS NOT NULL + AND created_at > datetime('now', ? || ' days') + GROUP BY domain""", + (f"-{days}",), + ).fetchall() + + domains = [] + for r in sources_by_domain: + total = r["total_prs"] or 0 + merged = r["merged"] or 0 + domains.append({ + "domain": r["domain"], + "total_prs": total, + "merged": merged, + "yield": round(merged / total, 3) if total else 0, + }) + + domains.sort(key=lambda x: x["merged"], reverse=True) + return web.json_response({"days": days, "domains": domains}) + + +# ─── GET /api/agents-dashboard ───────────────────────────────────────────── + +async def handle_agents_dashboard(request): + """Batched agent performance payload for Page 3. + + Returns per-agent: merged count, rejection rate, yield, CI score, + top rejection reasons, contribution trend (weekly). + All in one response to avoid N client-side fetches. + """ + conn = request.app["_get_conn"]() + days = int(request.query.get("days", "30")) + + # Per-agent merged + rejected counts + agent_stats = conn.execute( + """SELECT + COALESCE(json_extract(detail, '$.agent'), json_extract(detail, '$.domain_agent')) as agent, + COUNT(*) as evaluated, + SUM(CASE WHEN event='approved' THEN 1 ELSE 0 END) as approved, + SUM(CASE WHEN event IN ('changes_requested','domain_rejected','tier05_rejected') THEN 1 ELSE 0 END) as rejected + FROM audit_log + WHERE stage='evaluate' + AND event IN ('approved','changes_requested','domain_rejected','tier05_rejected') + AND timestamp > datetime('now', ? || ' days') + AND COALESCE(json_extract(detail, '$.agent'), json_extract(detail, '$.domain_agent')) IS NOT NULL + GROUP BY agent""", + (f"-{days}",), + ).fetchall() + + agents = {} + for r in agent_stats: + name = r["agent"] + ev = r["evaluated"] or 0 + ap = r["approved"] or 0 + rj = r["rejected"] or 0 + agents[name] = { + "evaluated": ev, + "approved": ap, + "rejected": rj, + "yield": round(ap / ev, 3) if ev else 0, + "rejection_rate": round(rj / ev, 3) if ev else 0, + } + + # Per-agent top rejection reasons from prs.eval_issues (Epimetheus correction 2026-04-02) + tag_rows = conn.execute( + """SELECT agent, value as tag, COUNT(*) as cnt + FROM prs, json_each(prs.eval_issues) + WHERE eval_issues IS NOT NULL AND eval_issues != '[]' + AND agent IS NOT NULL + AND created_at > datetime('now', ? || ' days') + GROUP BY agent, tag + ORDER BY agent, cnt DESC""", + (f"-{days}",), + ).fetchall() + + for r in tag_rows: + name = r["agent"] + if name in agents: + if "top_rejections" not in agents[name]: + agents[name]["top_rejections"] = [] + if len(agents[name]["top_rejections"]) < 5: + agents[name]["top_rejections"].append({"tag": r["tag"], "count": r["cnt"]}) + + # Weekly contribution trend per agent + weekly = conn.execute( + """SELECT + COALESCE(json_extract(detail, '$.agent'), json_extract(detail, '$.domain_agent')) as agent, + strftime('%Y-W%W', timestamp) as week, + SUM(CASE WHEN event='approved' THEN 1 ELSE 0 END) as merged, + COUNT(*) as evaluated + FROM audit_log + WHERE stage='evaluate' + AND event IN ('approved','changes_requested','domain_rejected','tier05_rejected') + AND timestamp > datetime('now', ? || ' days') + AND COALESCE(json_extract(detail, '$.agent'), json_extract(detail, '$.domain_agent')) IS NOT NULL + GROUP BY agent, week + ORDER BY agent, week""", + (f"-{days}",), + ).fetchall() + + for r in weekly: + name = r["agent"] + if name in agents: + if "weekly_trend" not in agents[name]: + agents[name]["weekly_trend"] = [] + agents[name]["weekly_trend"].append({ + "week": r["week"], + "merged": r["merged"] or 0, + "evaluated": r["evaluated"] or 0, + }) + + # CI scores from contributors table + weights = {"sourcer": 0.15, "extractor": 0.05, "challenger": 0.35, "synthesizer": 0.25, "reviewer": 0.20} + try: + contribs = conn.execute( + "SELECT handle, sourcer_count, extractor_count, challenger_count, " + "synthesizer_count, reviewer_count, claims_merged, tier FROM contributors" + ).fetchall() + for c in contribs: + name = c["handle"] + if name not in agents: + agents[name] = {} + ci = sum((c[f"{role}_count"] or 0) * w for role, w in weights.items()) + agents[name]["ci_score"] = round(ci, 2) + agents[name]["claims_merged"] = c["claims_merged"] or 0 + agents[name]["tier"] = c["tier"] + except sqlite3.Error: + pass + + return web.json_response({"days": days, "agents": agents}) + + +# ─── GET /api/cascade-coverage ──────────────────────────────────────────── + +async def handle_cascade_coverage(request): + """Cascade coverage from audit_log stage='cascade' events. + + Returns: triggered count, by-agent breakdown, claims affected. + """ + conn = request.app["_get_conn"]() + days = int(request.query.get("days", "30")) + + triggered = conn.execute( + """SELECT + json_extract(detail, '$.agent') as agent, + COUNT(*) as cnt, + SUM(json_array_length(json_extract(detail, '$.source_claims'))) as claims_affected + FROM audit_log + WHERE stage='cascade' AND event='cascade_triggered' + AND timestamp > datetime('now', ? || ' days') + GROUP BY agent""", + (f"-{days}",), + ).fetchall() + + summaries = conn.execute( + """SELECT + SUM(json_extract(detail, '$.notifications_sent')) as total_notifications, + COUNT(*) as total_merges_with_cascade + FROM audit_log + WHERE stage='cascade' AND event='cascade_summary' + AND timestamp > datetime('now', ? || ' days')""", + (f"-{days}",), + ).fetchone() + + reviewed = conn.execute( + """SELECT COUNT(*) as cnt + FROM audit_log + WHERE stage='cascade' AND event='cascade_reviewed' + AND timestamp > datetime('now', ? || ' days')""", + (f"-{days}",), + ).fetchone() + + total_triggered = sum(r["cnt"] for r in triggered) + total_reviewed = reviewed["cnt"] if reviewed else 0 + completion_rate = round(total_reviewed / total_triggered, 3) if total_triggered else None + + by_agent = [ + {"agent": r["agent"], "triggered": r["cnt"], "claims_affected": r["claims_affected"] or 0} + for r in triggered + ] + + return web.json_response({ + "days": days, + "total_triggered": total_triggered, + "total_reviewed": total_reviewed, + "completion_rate": completion_rate, + "total_notifications": summaries["total_notifications"] if summaries else 0, + "merges_with_cascade": summaries["total_merges_with_cascade"] if summaries else 0, + "by_agent": by_agent, + }) + + +# ─── GET /api/review-summary ───────────────────────────────────────────── + +async def handle_review_summary(request): + """Structured review data from review_records table (migration v12). + + Cleaner than audit_log parsing — structured outcome, rejection_reason, + disagreement_type columns. + """ + conn = request.app["_get_conn"]() + days = int(request.query.get("days", "30")) + + # Check if table exists and has data + try: + total = conn.execute( + "SELECT COUNT(*) as cnt FROM review_records WHERE reviewed_at > datetime('now', ? || ' days')", + (f"-{days}",), + ).fetchone()["cnt"] + except Exception: + return web.json_response({"error": "review_records table not available", "populated": False}) + + if total == 0: + return web.json_response({"populated": False, "total": 0, "days": days}) + + # Outcome breakdown + outcomes = conn.execute( + """SELECT outcome, COUNT(*) as cnt + FROM review_records + WHERE reviewed_at > datetime('now', ? || ' days') + GROUP BY outcome""", + (f"-{days}",), + ).fetchall() + + # Rejection reasons + reasons = conn.execute( + """SELECT rejection_reason, COUNT(*) as cnt + FROM review_records + WHERE rejection_reason IS NOT NULL + AND reviewed_at > datetime('now', ? || ' days') + GROUP BY rejection_reason ORDER BY cnt DESC""", + (f"-{days}",), + ).fetchall() + + # Disagreement types + disagreements = conn.execute( + """SELECT disagreement_type, COUNT(*) as cnt + FROM review_records + WHERE disagreement_type IS NOT NULL + AND reviewed_at > datetime('now', ? || ' days') + GROUP BY disagreement_type ORDER BY cnt DESC""", + (f"-{days}",), + ).fetchall() + + # Per-reviewer breakdown + reviewers = conn.execute( + """SELECT reviewer, + SUM(CASE WHEN outcome='approved' THEN 1 ELSE 0 END) as approved, + SUM(CASE WHEN outcome='approved-with-changes' THEN 1 ELSE 0 END) as approved_with_changes, + SUM(CASE WHEN outcome='rejected' THEN 1 ELSE 0 END) as rejected, + COUNT(*) as total + FROM review_records + WHERE reviewed_at > datetime('now', ? || ' days') + GROUP BY reviewer ORDER BY total DESC""", + (f"-{days}",), + ).fetchall() + + # Per-domain breakdown + domains = conn.execute( + """SELECT domain, + SUM(CASE WHEN outcome='rejected' THEN 1 ELSE 0 END) as rejected, + COUNT(*) as total + FROM review_records + WHERE domain IS NOT NULL + AND reviewed_at > datetime('now', ? || ' days') + GROUP BY domain ORDER BY total DESC""", + (f"-{days}",), + ).fetchall() + + return web.json_response({ + "populated": True, + "days": days, + "total": total, + "outcomes": {r["outcome"]: r["cnt"] for r in outcomes}, + "rejection_reasons": [{"reason": r["rejection_reason"], "count": r["cnt"]} for r in reasons], + "disagreement_types": [{"type": r["disagreement_type"], "count": r["cnt"]} for r in disagreements], + "reviewers": [ + {"reviewer": r["reviewer"], "approved": r["approved"], "approved_with_changes": r["approved_with_changes"], + "rejected": r["rejected"], "total": r["total"]} + for r in reviewers + ], + "domains": [ + {"domain": r["domain"], "rejected": r["rejected"], "total": r["total"], + "rejection_rate": round(r["rejected"] / r["total"], 3) if r["total"] else 0} + for r in domains + ], + }) + + +# ─── Trace endpoint ──────────────────────────────────────────────────────── + + +async def handle_trace(request: web.Request) -> web.Response: + """Return the full lifecycle of a source/PR through the pipeline. + + GET /api/trace/1234 → all audit_log + review_records + costs for PR 1234. + One thread, every stage, chronological. + """ + trace_id = request.match_info["trace_id"] + get_conn = request.app["_get_conn"] + conn = get_conn() + + # Audit log events (the backbone) + # Try trace_id first, fall back to PR number in detail JSON + events = conn.execute( + """SELECT timestamp, stage, event, detail + FROM audit_log + WHERE trace_id = ? + ORDER BY timestamp""", + (trace_id,), + ).fetchall() + + if not events: + # Fallback: match by PR number in detail JSON (for rows without trace_id) + events = conn.execute( + """SELECT timestamp, stage, event, detail + FROM audit_log + WHERE CAST(json_extract(detail, '$.pr') AS TEXT) = ? + ORDER BY timestamp""", + (trace_id,), + ).fetchall() + + # Review records for this PR + reviews = conn.execute( + """SELECT reviewed_at, reviewer, reviewer_model, outcome, + rejection_reason, disagreement_type, notes, claim_path + FROM review_records + WHERE pr_number = ? + ORDER BY reviewed_at""", + (trace_id,), + ).fetchall() + + # PR metadata + pr = conn.execute( + """SELECT number, source_path, domain, agent, tier, status, + origin, created_at, merged_at + FROM prs + WHERE number = ?""", + (trace_id,), + ).fetchone() + + result = { + "trace_id": trace_id, + "pr": dict(pr) if pr else None, + "timeline": [ + {"timestamp": r[0], "stage": r[1], "event": r[2], + "detail": json.loads(r[3]) if r[3] else None} + for r in events + ], + "reviews": [ + {"reviewed_at": r[0], "reviewer": r[1], "model": r[2], + "outcome": r[3], "rejection_reason": r[4], + "disagreement_type": r[5], "notes": r[6], "claim_path": r[7]} + for r in reviews + ], + } + + return web.json_response(result) + + +# ─── GET /api/growth ────────────────────────────────────────────────────── + +async def handle_growth(request): + """Cumulative growth of sources, PRs, and merged claims over time. + + Returns daily data points with running totals for each series. + """ + conn = request.app["_get_conn"]() + days = int(request.query.get("days", "90")) + + # Daily new sources + source_rows = conn.execute( + """SELECT date(created_at) as day, COUNT(*) as cnt + FROM sources + WHERE created_at > datetime('now', ? || ' days') + GROUP BY day ORDER BY day""", + (f"-{days}",), + ).fetchall() + + # Daily new PRs + pr_rows = conn.execute( + """SELECT date(created_at) as day, COUNT(*) as cnt + FROM prs + WHERE created_at > datetime('now', ? || ' days') + GROUP BY day ORDER BY day""", + (f"-{days}",), + ).fetchall() + + # Daily merged PRs + merged_rows = conn.execute( + """SELECT date(merged_at) as day, COUNT(*) as cnt + FROM prs + WHERE status = 'merged' AND merged_at IS NOT NULL + AND merged_at > datetime('now', ? || ' days') + GROUP BY day ORDER BY day""", + (f"-{days}",), + ).fetchall() + + # Get totals BEFORE the window for correct cumulative baseline + source_base = conn.execute( + "SELECT COUNT(*) as cnt FROM sources WHERE created_at <= datetime('now', ? || ' days')", + (f"-{days}",), + ).fetchone()["cnt"] + + pr_base = conn.execute( + "SELECT COUNT(*) as cnt FROM prs WHERE created_at <= datetime('now', ? || ' days')", + (f"-{days}",), + ).fetchone()["cnt"] + + merged_base = conn.execute( + """SELECT COUNT(*) as cnt FROM prs + WHERE status = 'merged' AND merged_at IS NOT NULL + AND merged_at <= datetime('now', ? || ' days')""", + (f"-{days}",), + ).fetchone()["cnt"] + + # Collect all unique dates + all_dates = sorted(set( + [r["day"] for r in source_rows] + + [r["day"] for r in pr_rows] + + [r["day"] for r in merged_rows] + )) + + # Build lookup dicts + src_by_day = {r["day"]: r["cnt"] for r in source_rows} + pr_by_day = {r["day"]: r["cnt"] for r in pr_rows} + mrg_by_day = {r["day"]: r["cnt"] for r in merged_rows} + + # Build cumulative arrays + dates = [] + sources_cum = [] + prs_cum = [] + merged_cum = [] + + s_total = source_base + p_total = pr_base + m_total = merged_base + + for day in all_dates: + s_total += src_by_day.get(day, 0) + p_total += pr_by_day.get(day, 0) + m_total += mrg_by_day.get(day, 0) + dates.append(day) + sources_cum.append(s_total) + prs_cum.append(p_total) + merged_cum.append(m_total) + + return web.json_response({ + "days": days, + "dates": dates, + "sources": sources_cum, + "prs": prs_cum, + "merged": merged_cum, + "current": { + "sources": s_total, + "prs": p_total, + "merged": m_total, + }, + }) + + +import re +_DATE_PREFIX_RE = re.compile(r"^\d{4}-\d{2}-\d{2}-?") + +# ─── GET /api/pr-lifecycle ──────────────────────────────────────────────── + +async def handle_pr_lifecycle(request): + """All PRs with eval rounds, reviews, and time-to-merge in one payload. + + Returns: summary KPIs + per-PR array for the table. + Joins prs + audit_log (eval rounds) + review_records. + """ + conn = request.app["_get_conn"]() + days = int(request.query.get("days", "30")) + + day_clause = "AND p.created_at > datetime('now', ? || ' days')" if days < 9999 else "" + params = (f"-{days}",) if days < 9999 else () + + # Base PR data + pr_rows = conn.execute( + f"""SELECT p.number, p.agent, p.domain, p.tier, p.status, + p.created_at, p.merged_at, p.leo_verdict, p.description, + p.domain_agent, p.domain_model, p.branch, p.submitted_by, + p.source_path + FROM prs p + WHERE 1=1 {day_clause} + ORDER BY p.number DESC""", + params, + ).fetchall() + + # Eval round counts per PR (from audit_log) + eval_rows = conn.execute( + f"""SELECT CAST(json_extract(detail, '$.pr') AS INTEGER) as pr, + COUNT(*) as rounds + FROM audit_log + WHERE stage = 'evaluate' + AND event IN ('approved', 'changes_requested', 'domain_rejected', 'tier05_rejected') + AND json_extract(detail, '$.pr') IS NOT NULL + GROUP BY pr""", + ).fetchall() + eval_map = {r["pr"]: r["rounds"] for r in eval_rows} + + # Review outcomes per PR (from review_records) + review_rows = conn.execute( + """SELECT pr_number, outcome, + GROUP_CONCAT(DISTINCT reviewer) as reviewers, + COUNT(*) as review_count + FROM review_records + GROUP BY pr_number, outcome""", + ).fetchall() + review_map = {} + for r in review_rows: + pr = r["pr_number"] + if pr not in review_map: + review_map[pr] = {"outcomes": [], "reviewers": set(), "count": 0} + review_map[pr]["outcomes"].append(r["outcome"]) + if r["reviewers"]: + review_map[pr]["reviewers"].update(r["reviewers"].split(",")) + review_map[pr]["count"] += r["review_count"] + + # Review snippets for closed PRs — from review_text or issues list + snippet_rows = conn.execute( + """SELECT CAST(json_extract(detail, '$.pr') AS INTEGER) as pr, + COALESCE( + json_extract(detail, '$.review_text'), + json_extract(detail, '$.domain_review_text'), + json_extract(detail, '$.leo_review_text') + ) as review_text, + json_extract(detail, '$.issues') as issues, + json_extract(detail, '$.leo') as leo_verdict + FROM audit_log + WHERE stage = 'evaluate' + AND event IN ('domain_rejected', 'changes_requested') + AND json_extract(detail, '$.pr') IS NOT NULL + ORDER BY timestamp DESC""", + ).fetchall() + snippet_map = {} + for r in snippet_rows: + pr = r["pr"] + if pr not in snippet_map: + if r["review_text"]: + text = r["review_text"].strip() + lines = [ln.strip() for ln in text.split("\n") if ln.strip() and not ln.strip().startswith("#")] + snippet_map[pr] = lines[0][:200] if lines else text[:200] + elif r["issues"]: + try: + issues = json.loads(r["issues"]) if isinstance(r["issues"], str) else r["issues"] + if isinstance(issues, list) and issues: + snippet_map[pr] = "Issues: " + ", ".join(str(i).replace("_", " ") for i in issues) + except (json.JSONDecodeError, TypeError): + pass + + # Build PR list + prs = [] + ttm_values = [] + round_values = [] + merged_count = 0 + closed_count = 0 + open_count = 0 + + for r in pr_rows: + pr_num = r["number"] + ttm = None + if r["merged_at"] and r["created_at"]: + try: + created = datetime.fromisoformat(r["created_at"]) + merged = datetime.fromisoformat(r["merged_at"]) + ttm = (merged - created).total_seconds() / 60 + if ttm >= 0: + ttm_values.append(ttm) + else: + ttm = None + except (ValueError, TypeError): + pass + + rounds = eval_map.get(pr_num, 0) + if rounds > 0: + round_values.append(rounds) + + review_info = review_map.get(pr_num) + + status = r["status"] or "unknown" + if status == "merged": + merged_count += 1 + elif status == "closed": + closed_count += 1 + elif status == "open": + open_count += 1 + + # Claims count from pipe-separated description titles + desc = r["description"] or "" + claims_count = desc.count("|") + 1 if desc.strip() else 1 + + # Summary: first claim title from description, fallback to branch name + summary = None + if desc.strip(): + first_title = desc.split("|")[0].strip() + summary = first_title[:120] if first_title else None + if not summary: + branch = r["branch"] or "" + # Use prefix as category if present: "extract/...", "reweave/...", etc. + prefix = "" + if "/" in branch: + prefix = branch.split("/", 1)[0] + branch = branch.split("/", 1)[1] + # Strip date prefix like "2026-04-06-" or "2026-02-00-" + branch = _DATE_PREFIX_RE.sub("", branch) + # Strip trailing hash suffix like "-116d" or "-2cb1" + branch = re.sub(r"-[0-9a-f]{4}$", "", branch) + if branch: + summary = branch.replace("-", " ").replace("_", " ").strip()[:120] + elif prefix: + summary = prefix # "reweave", "ingestion", etc. + + prs.append({ + "number": pr_num, + "agent": r["agent"], + "domain": r["domain"], + "tier": r["tier"], + "status": status, + "claims_count": claims_count, + "eval_rounds": rounds, + "ttm_minutes": round(ttm, 1) if ttm is not None else None, + "created_at": r["created_at"], + "merged_at": r["merged_at"], + "leo_verdict": r["leo_verdict"], + "review_count": review_info["count"] if review_info else 0, + "summary": summary, + "description": desc if desc.strip() else None, + "review_snippet": snippet_map.get(pr_num), + "submitted_by": r["submitted_by"], + "source_path": r["source_path"], + "domain_agent": r["domain_agent"], + "domain_model": r["domain_model"], + }) + + # Summary KPIs + ttm_values.sort() + round_values.sort() + + def median(vals): + if not vals: + return None + n = len(vals) + if n % 2 == 0: + return (vals[n // 2 - 1] + vals[n // 2]) / 2 + return vals[n // 2] + + def p90(vals): + if len(vals) < 5: + return None + return vals[int(len(vals) * 0.9)] + + return web.json_response({ + "days": days, + "total": len(prs), + "merged": merged_count, + "closed": closed_count, + "open": open_count, + "median_ttm": round(median(ttm_values), 1) if median(ttm_values) is not None else None, + "p90_ttm": round(p90(ttm_values), 1) if p90(ttm_values) is not None else None, + "median_rounds": round(median(round_values), 1) if median(round_values) is not None else None, + "max_rounds": max(round_values) if round_values else None, + "prs": prs, + }) + + +# ─── Registration ────────────────────────────────────────────────────────── + +def register_dashboard_routes(app: web.Application, get_conn): + """Register new dashboard API routes.""" + app["_get_conn"] = get_conn + app.router.add_get("/api/stage-times", handle_stage_times) + app.router.add_get("/api/herfindahl", handle_herfindahl) + app.router.add_get("/api/agent-state", handle_agent_state) + app.router.add_get("/api/extraction-yield-by-domain", handle_extraction_yield_by_domain) + app.router.add_get("/api/agents-dashboard", handle_agents_dashboard) + app.router.add_get("/api/cascade-coverage", handle_cascade_coverage) + app.router.add_get("/api/review-summary", handle_review_summary) + app.router.add_get("/api/trace/{trace_id}", handle_trace) + app.router.add_get("/api/growth", handle_growth) + app.router.add_get("/api/pr-lifecycle", handle_pr_lifecycle) diff --git a/diagnostics/response_audit_routes.py b/diagnostics/response_audit_routes.py new file mode 100644 index 0000000..841220b --- /dev/null +++ b/diagnostics/response_audit_routes.py @@ -0,0 +1,475 @@ +"""Response audit API routes — agent cost tracking, reasoning traces, unified activity. + +Endpoints: + GET /api/response-audit — paginated response list with cost columns + GET /api/response-audit/{id} — single response detail with full tool_calls + GET /api/agent-costs — aggregated cost view from response_audit + GET /api/unified-activity — merged prs + response_audit timeline + +Data source: response_audit table in pipeline.db (written by Epimetheus's Telegram bot). + +Owner: Argus +""" + +import json +import logging +import sqlite3 + +from aiohttp import web + +logger = logging.getLogger("argus.response_audit_routes") + + +def _conn(app): + """Read-only connection to pipeline.db.""" + db_path = app["db_path"] + conn = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True) + conn.row_factory = sqlite3.Row + return conn + + +# ─── GET /api/response-audit ───────────────────────────────────────────── + +async def handle_response_audit_list(request): + """Paginated response audit list with cost and model data. + + Query params: + agent — filter by agent name + hours — lookback window (default 24, max 168) + limit — max results (default 50, max 200) + offset — pagination offset (default 0) + model — filter by model name (substring match) + """ + agent = request.query.get("agent") + model_filter = request.query.get("model") + try: + hours = min(int(request.query.get("hours", 24)), 168) + except (ValueError, TypeError): + hours = 24 + try: + limit = min(int(request.query.get("limit", 50)), 200) + except (ValueError, TypeError): + limit = 50 + try: + offset = max(int(request.query.get("offset", 0)), 0) + except (ValueError, TypeError): + offset = 0 + + conn = _conn(request.app) + try: + where = ["timestamp > datetime('now', ?)"] + params: list = [f"-{hours} hours"] + + if agent: + where.append("agent = ?") + params.append(agent) + if model_filter: + where.append("model LIKE ?") + params.append(f"%{model_filter}%") + + where_clause = " AND ".join(where) + + # Count total matching + total = conn.execute( + f"SELECT COUNT(*) as cnt FROM response_audit WHERE {where_clause}", + params, + ).fetchone()["cnt"] + + # Fetch page — exclude large text fields for list view + rows = conn.execute( + f"""SELECT id, timestamp, agent, model, query, + prompt_tokens, completion_tokens, + generation_cost, embedding_cost, total_cost, + confidence_score, response_time_ms, query_type, + CASE WHEN tool_calls IS NOT NULL AND tool_calls != '[]' + THEN json_array_length(tool_calls) + ELSE 0 END as tool_call_count, + LENGTH(display_response) as response_length + FROM response_audit + WHERE {where_clause} + ORDER BY timestamp DESC + LIMIT ? OFFSET ?""", + params + [limit, offset], + ).fetchall() + + responses = [] + for r in rows: + responses.append({ + "id": r["id"], + "timestamp": r["timestamp"], + "agent": r["agent"], + "model": r["model"], + "query": r["query"], + "query_type": r["query_type"], + "prompt_tokens": r["prompt_tokens"], + "completion_tokens": r["completion_tokens"], + "generation_cost": r["generation_cost"], + "embedding_cost": r["embedding_cost"], + "total_cost": r["total_cost"], + "confidence": r["confidence_score"], + "response_time_ms": r["response_time_ms"], + "tool_call_count": r["tool_call_count"], + "response_length": r["response_length"], + }) + + return web.json_response({ + "total": total, + "limit": limit, + "offset": offset, + "hours": hours, + "responses": responses, + }) + finally: + conn.close() + + +# ─── GET /api/response-audit/{id} ──────────────────────────────────────── + +async def handle_response_audit_detail(request): + """Full response detail including reasoning trace and tool calls. + + Returns the complete response_audit row with tool_calls parsed as JSON. + """ + try: + audit_id = int(request.match_info["id"]) + except (ValueError, TypeError): + return web.json_response({"error": "Invalid ID"}, status=400) + + conn = _conn(request.app) + try: + row = conn.execute( + """SELECT id, timestamp, chat_id, user, agent, model, + query, query_type, conversation_window, + entities_matched, claims_matched, + retrieval_layers_hit, retrieval_gap, + market_data, research_context, + tool_calls, raw_response, display_response, + confidence_score, response_time_ms, + prompt_tokens, completion_tokens, + generation_cost, embedding_cost, total_cost, + blocked, block_reason + FROM response_audit WHERE id = ?""", + (audit_id,), + ).fetchone() + + if not row: + return web.json_response({"error": "Response not found"}, status=404) + + # Parse JSON fields + def parse_json(val): + if val is None: + return None + try: + return json.loads(val) + except (json.JSONDecodeError, TypeError): + return val + + result = { + "id": row["id"], + "timestamp": row["timestamp"], + "chat_id": row["chat_id"], + "user": row["user"], + "agent": row["agent"], + "model": row["model"], + "query": row["query"], + "query_type": row["query_type"], + "conversation_window": parse_json(row["conversation_window"]), + "entities_matched": parse_json(row["entities_matched"]), + "claims_matched": parse_json(row["claims_matched"]), + "retrieval_layers_hit": parse_json(row["retrieval_layers_hit"]), + "retrieval_gap": row["retrieval_gap"], + "market_data": parse_json(row["market_data"]), + "research_context": row["research_context"], + "tool_calls": parse_json(row["tool_calls"]), + "display_response": row["display_response"], + "raw_response": row["raw_response"], + "confidence_score": row["confidence_score"], + "response_time_ms": row["response_time_ms"], + "prompt_tokens": row["prompt_tokens"], + "completion_tokens": row["completion_tokens"], + "generation_cost": row["generation_cost"], + "embedding_cost": row["embedding_cost"], + "total_cost": row["total_cost"], + "blocked": bool(row["blocked"]) if row["blocked"] is not None else None, + "block_reason": row["block_reason"], + } + + # Compute iteration summary from tool_calls + tool_calls = result["tool_calls"] or [] + if isinstance(tool_calls, list): + reasoning_steps = [t for t in tool_calls if isinstance(t, dict) and t.get("type") == "reasoning"] + tool_steps = [t for t in tool_calls if isinstance(t, dict) and t.get("type") == "tool_call"] + result["trace_summary"] = { + "total_steps": len(tool_calls), + "reasoning_steps": len(reasoning_steps), + "tool_steps": len(tool_steps), + "tools_used": list({t.get("tool", "unknown") for t in tool_steps}), + "total_duration_ms": sum(t.get("duration_ms", 0) for t in tool_steps), + } + else: + result["trace_summary"] = None + + return web.json_response(result) + finally: + conn.close() + + +# ─── GET /api/agent-costs ───────────────────────────────────────────────── + +async def handle_agent_costs(request): + """Aggregated agent cost data from response_audit. + + Query params: + days — lookback window (default 7, max 30) + by — grouping: agent, model, day (default agent) + """ + try: + days = min(int(request.query.get("days", 7)), 30) + except (ValueError, TypeError): + days = 7 + group_by = request.query.get("by", "agent") + agent = request.query.get("agent") + + conn = _conn(request.app) + try: + if group_by == "model": + group_col = "model" + elif group_by == "day": + group_col = "date(timestamp)" + else: + group_col = "agent" + group_by = "agent" + + where = ["timestamp > datetime('now', ?)"] + params: list = [f"-{days} days"] + if agent: + where.append("agent = ?") + params.append(agent) + + where_clause = " AND ".join(where) + + rows = conn.execute( + f"""SELECT {group_col} as grp, + COUNT(*) as responses, + SUM(prompt_tokens) as total_prompt_tokens, + SUM(completion_tokens) as total_completion_tokens, + SUM(COALESCE(total_cost, generation_cost, 0)) as total_cost, + AVG(COALESCE(total_cost, generation_cost, 0)) as avg_cost, + AVG(response_time_ms) as avg_response_ms, + AVG(confidence_score) as avg_confidence + FROM response_audit + WHERE {where_clause} + GROUP BY grp + ORDER BY total_cost DESC""", + params, + ).fetchall() + + breakdown = [] + for r in rows: + breakdown.append({ + group_by: r["grp"], + "responses": r["responses"], + "prompt_tokens": r["total_prompt_tokens"] or 0, + "completion_tokens": r["total_completion_tokens"] or 0, + "total_cost": round(r["total_cost"] or 0, 4), + "avg_cost_per_response": round(r["avg_cost"] or 0, 4), + "avg_response_ms": round(r["avg_response_ms"] or 0, 0), + "avg_confidence": round(r["avg_confidence"] or 0, 3) if r["avg_confidence"] else None, + }) + + grand_total = sum(b["total_cost"] for b in breakdown) + total_responses = sum(b["responses"] for b in breakdown) + + # Daily trend (always included regardless of grouping) + daily_where = ["timestamp > datetime('now', ?)"] + daily_params: list = [f"-{days} days"] + if agent: + daily_where.append("agent = ?") + daily_params.append(agent) + + daily = conn.execute( + f"""SELECT date(timestamp) as day, + COUNT(*) as responses, + SUM(COALESCE(total_cost, generation_cost, 0)) as cost + FROM response_audit + WHERE {' AND '.join(daily_where)} + GROUP BY day ORDER BY day""", + daily_params, + ).fetchall() + + daily_trend = [ + {"date": r["day"], "responses": r["responses"], + "cost": round(r["cost"] or 0, 4)} + for r in daily + ] + + return web.json_response({ + "period_days": days, + "grand_total": round(grand_total, 4), + "total_responses": total_responses, + "avg_cost_per_response": round(grand_total / total_responses, 4) if total_responses else 0, + f"by_{group_by}": breakdown, + "daily_trend": daily_trend, + }) + finally: + conn.close() + + +# ─── GET /api/unified-activity ──────────────────────────────────────────── + +async def handle_unified_activity(request): + """Unified activity feed merging pipeline ops (prs) + agent responses (response_audit). + + Query params: + hours — lookback window (default 24, max 168) + limit — max results (default 100, max 500) + agent — filter by agent name + type — filter: pipeline, response, or all (default all) + """ + try: + hours = min(int(request.query.get("hours", 24)), 168) + except (ValueError, TypeError): + hours = 24 + try: + limit = min(int(request.query.get("limit", 100)), 500) + except (ValueError, TypeError): + limit = 100 + agent = request.query.get("agent") + activity_type = request.query.get("type", "all") + + conn = _conn(request.app) + try: + entries = [] + + # Pipeline events from prs table + if activity_type in ("all", "pipeline"): + pr_where = ["COALESCE(merged_at, created_at) > datetime('now', ?)"] + pr_params: list = [f"-{hours} hours"] + if agent: + pr_where.append("agent = ?") + pr_params.append(agent) + + prs = conn.execute( + f"""SELECT number, branch, status, domain, agent, tier, + commit_type, cost_usd, + created_at, merged_at, + leo_verdict, domain_verdict + FROM prs + WHERE {' AND '.join(pr_where)} + ORDER BY COALESCE(merged_at, created_at) DESC""", + pr_params, + ).fetchall() + + for pr in prs: + ts = pr["merged_at"] or pr["created_at"] + # Derive action description from status + if pr["status"] == "merged": + action = f"Merged {pr['commit_type'] or 'PR'}" + elif pr["status"] == "closed": + action = f"Closed {pr['commit_type'] or 'PR'}" + elif pr["status"] in ("approved", "reviewing"): + action = f"{pr['commit_type'] or 'PR'} awaiting merge" + else: + action = f"{pr['commit_type'] or 'PR'} {pr['status']}" + + entries.append({ + "timestamp": ts, + "type": "pipeline", + "agent": pr["agent"], + "action": action, + "domain": pr["domain"], + "pr_number": pr["number"], + "branch": pr["branch"], + "status": pr["status"], + "commit_type": pr["commit_type"], + "cost": pr["cost_usd"], + "detail": { + "tier": pr["tier"], + "leo_verdict": pr["leo_verdict"], + "domain_verdict": pr["domain_verdict"], + }, + }) + + # Agent responses from response_audit + if activity_type in ("all", "response"): + ra_where = ["timestamp > datetime('now', ?)"] + ra_params: list = [f"-{hours} hours"] + if agent: + ra_where.append("agent = ?") + ra_params.append(agent) + + responses = conn.execute( + f"""SELECT id, timestamp, agent, model, query, + generation_cost, response_time_ms, + confidence_score, + CASE WHEN tool_calls IS NOT NULL AND tool_calls != '[]' + THEN json_array_length(tool_calls) + ELSE 0 END as tool_call_count + FROM response_audit + WHERE {' AND '.join(ra_where)} + ORDER BY timestamp DESC""", + ra_params, + ).fetchall() + + for r in responses: + # Truncate query for feed display + query_preview = (r["query"] or "")[:120] + if len(r["query"] or "") > 120: + query_preview += "..." + + entries.append({ + "timestamp": r["timestamp"], + "type": "response", + "agent": r["agent"], + "action": f"Responded to query ({r['tool_call_count']} tool calls)", + "domain": None, + "pr_number": None, + "audit_id": r["id"], + "query_preview": query_preview, + "model": r["model"], + "cost": r["generation_cost"], + "detail": { + "response_time_ms": r["response_time_ms"], + "confidence": r["confidence_score"], + "tool_call_count": r["tool_call_count"], + }, + }) + + # Sort combined entries by timestamp descending + entries.sort(key=lambda e: e["timestamp"] or "", reverse=True) + entries = entries[:limit] + + # Summary stats + pipeline_count = sum(1 for e in entries if e["type"] == "pipeline") + response_count = sum(1 for e in entries if e["type"] == "response") + total_cost = sum(e.get("cost") or 0 for e in entries) + + return web.json_response({ + "hours": hours, + "total_entries": len(entries), + "pipeline_events": pipeline_count, + "response_events": response_count, + "total_cost": round(total_cost, 4), + "entries": entries, + }) + finally: + conn.close() + + +# ─── Registration ───────────────────────────────────────────────────────── + +def register_response_audit_routes(app): + """Register response audit API routes. Call from create_app().""" + app.router.add_get("/api/response-audit", handle_response_audit_list) + app.router.add_get("/api/response-audit/{id}", handle_response_audit_detail) + app.router.add_get("/api/agent-costs", handle_agent_costs) + app.router.add_get("/api/unified-activity", handle_unified_activity) + + +# Public paths for auth middleware +RESPONSE_AUDIT_PUBLIC_PATHS = frozenset({ + "/api/response-audit", + "/api/agent-costs", + "/api/unified-activity", +}) +# /api/response-audit/{id} needs prefix matching in auth middleware diff --git a/diagnostics/review_queue.py b/diagnostics/review_queue.py new file mode 100644 index 0000000..c15a4be --- /dev/null +++ b/diagnostics/review_queue.py @@ -0,0 +1,222 @@ +"""Review queue: fetches open PRs from Forgejo, classifies and enriches them. + +Data sources: + - Forgejo API (git.livingip.xyz) for PR metadata, reviews, changed files + - pipeline.db prs table for eval status cross-reference + +Display priority: broken > needs-review (by age) > approved-awaiting-merge > changes-requested +""" + +import asyncio +import logging +from datetime import datetime, timezone +from typing import Any + +import aiohttp + +logger = logging.getLogger("argus.review_queue") + +FORGEJO_BASE = "https://git.livingip.xyz/api/v1" +REPO = "teleo/teleo-codex" + +# Domain detection from branch prefixes or path patterns +DOMAIN_KEYWORDS = { + "internet-finance": ["internet-finance", "defi", "dao", "prediction-market"], + "entertainment": ["entertainment", "clay", "media", "ip-"], + "ai-alignment": ["ai-alignment", "alignment", "theseus"], + "health": ["health", "vida", "biotech", "glp"], + "space-development": ["space", "astra", "orbital", "lunar"], + "energy": ["energy", "solar", "nuclear", "fusion"], + "grand-strategy": ["grand-strategy", "leo", "strategy"], + "collective-intelligence": ["collective-intelligence", "coordination"], + "critical-systems": ["critical-systems", "complexity", "emergence"], + "teleological-economics": ["teleological-economics", "disruption", "attractor"], + "cultural-dynamics": ["cultural-dynamics", "memetics", "narrative"], + "mechanisms": ["mechanisms", "futarchy", "governance"], + "living-capital": ["living-capital", "investment"], + "living-agents": ["living-agents", "agent-architecture"], + "teleohumanity": ["teleohumanity", "worldview"], + "general": ["general"], +} + + +def _detect_domain(branch: str, title: str, files: list[dict]) -> str: + """Detect domain from branch name, title, or changed file paths.""" + text = f"{branch} {title}".lower() + + # Check branch/title + for domain, keywords in DOMAIN_KEYWORDS.items(): + for kw in keywords: + if kw in text: + return domain + + # Check file paths + for f in files: + path = f.get("filename", "") + if path.startswith("domains/") or path.startswith("foundations/") or path.startswith("core/"): + parts = path.split("/") + if len(parts) >= 2: + return parts[1] + + return "unknown" + + +def _classify_files(files: list[dict]) -> dict[str, int]: + """Count claim, enrichment, and challenge files from changed files list.""" + counts = {"claim_count": 0, "enrichment_count": 0, "challenge_count": 0} + for f in files: + path = f.get("filename", "") + status = f.get("status", "") # added, modified, removed + + if not path.startswith("domains/") and not path.startswith("foundations/") and not path.startswith("core/"): + continue + + name = path.split("/")[-1].lower() + + if "challenge" in name or "divergence" in name: + counts["challenge_count"] += 1 + elif status == "modified": + counts["enrichment_count"] += 1 + else: + counts["claim_count"] += 1 + + return counts + + +def _classify_status( + changed_files: int, + reviews: list[dict], + requested_reviewers: list[dict], +) -> str: + """Classify PR status: broken, needs-review, approved-awaiting-merge, changes-requested.""" + if changed_files == 0: + return "broken" + + has_changes_requested = any(r["state"] == "REQUEST_CHANGES" for r in reviews) + if has_changes_requested: + # Check if there's a newer approval after the changes request + last_change_req = max( + (r["submitted_at"] for r in reviews if r["state"] == "REQUEST_CHANGES"), + default="", + ) + later_approvals = [ + r for r in reviews + if r["state"] == "APPROVED" and r["submitted_at"] > last_change_req + ] + if not later_approvals: + return "changes-requested" + + approvals = [r for r in reviews if r["state"] == "APPROVED"] + if len(approvals) >= 2: + return "approved-awaiting-merge" + + return "needs-review" + + +def _days_open(created_at: str) -> int: + """Calculate days since PR was opened.""" + created = datetime.fromisoformat(created_at.replace("Z", "+00:00")) + now = datetime.now(timezone.utc) + return (now - created).days + + +_STATUS_PRIORITY = { + "broken": 0, + "needs-review": 1, + "approved-awaiting-merge": 2, + "changes-requested": 3, +} + + +async def fetch_review_queue( + forgejo_token: str | None = None, + timeout_s: int = 15, +) -> list[dict[str, Any]]: + """Fetch open PRs from Forgejo and return enriched review queue. + + Returns list sorted by display priority (broken first, then needs-review by age). + """ + headers = {"Accept": "application/json"} + if forgejo_token: + headers["Authorization"] = f"token {forgejo_token}" + + connector = aiohttp.TCPConnector(ssl=False) + async with aiohttp.ClientSession(headers=headers, connector=connector) as session: + # Fetch open PRs + url = f"{FORGEJO_BASE}/repos/{REPO}/pulls?state=open&limit=50&sort=oldest" + try: + async with session.get(url, timeout=aiohttp.ClientTimeout(total=timeout_s)) as resp: + if resp.status != 200: + logger.error("Forgejo PR list returned %d", resp.status) + return [] + prs = await resp.json() + except Exception as e: + logger.error("Failed to fetch PRs from Forgejo: %s", e) + return [] + + # Fetch reviews and files for all PRs in parallel + async def _fetch_json(session, url, label=""): + try: + async with session.get(url, timeout=aiohttp.ClientTimeout(total=timeout_s)) as resp: + if resp.status == 200: + return await resp.json() + except Exception as e: + logger.warning("Failed to fetch %s: %s", label, e) + return [] + + sub_tasks = [] + for pr in prs: + n = pr["number"] + sub_tasks.append(_fetch_json(session, f"{FORGEJO_BASE}/repos/{REPO}/pulls/{n}/reviews", f"reviews PR#{n}")) + sub_tasks.append(_fetch_json(session, f"{FORGEJO_BASE}/repos/{REPO}/pulls/{n}/files", f"files PR#{n}")) + + sub_results = await asyncio.gather(*sub_tasks) + + queue = [] + for i, pr in enumerate(prs): + reviews = sub_results[i * 2] + files = sub_results[i * 2 + 1] + + # Build enriched PR record + branch = pr.get("head", {}).get("ref", "") if pr.get("head") else "" + title = pr.get("title", "") + author = pr.get("user", {}).get("login", "unknown") + created_at = pr.get("created_at", "") + changed_files = pr.get("changed_files", len(files)) + requested_reviewers = pr.get("requested_reviewers", []) + + domain = _detect_domain(branch, title, files) + file_counts = _classify_files(files) + status = _classify_status(changed_files, reviews, requested_reviewers) + days = _days_open(created_at) if created_at else 0 + + review_list = [ + { + "reviewer": r.get("user", {}).get("login", "unknown"), + "outcome": r.get("state", "PENDING").lower(), + "date": r.get("submitted_at", ""), + "summary": r.get("body", "")[:200], + } + for r in reviews + if r.get("state") and r["state"] != "PENDING" + ] + + queue.append({ + "pr_number": pr["number"], + "title": title, + "author": author, + "domain": domain, + "branch": branch, + "created_at": created_at, + "days_open": days, + "status": status, + "changed_files": changed_files, + **file_counts, + "reviews": review_list, + "url": pr.get("html_url", ""), + }) + + # Sort: broken first, then needs-review by days_open desc, then rest + queue.sort(key=lambda x: (_STATUS_PRIORITY.get(x["status"], 99), -x["days_open"])) + + return queue diff --git a/diagnostics/review_queue_routes.py b/diagnostics/review_queue_routes.py new file mode 100644 index 0000000..64cf9fe --- /dev/null +++ b/diagnostics/review_queue_routes.py @@ -0,0 +1,64 @@ +"""Route handlers for /api/review-queue endpoint. + +Import into app.py and register routes in create_app(). +""" + +import logging + +from aiohttp import web +from review_queue import fetch_review_queue + +logger = logging.getLogger("argus.review_queue") + + +async def handle_review_queue(request): + """GET /api/review-queue — PR review pipeline view. + + Query params: + status: filter by status (broken, needs-review, approved-awaiting-merge, changes-requested) + author: filter by agent/author name + domain: filter by domain + + Returns JSON with queue items sorted by display priority: + broken (flagged) > needs-review (by age) > approved-awaiting-merge + """ + token = request.app.get("_forgejo_token") + + try: + queue = await fetch_review_queue(forgejo_token=token) + except Exception as e: + logger.error("Review queue fetch failed: %s", e) + return web.json_response({"error": str(e)}, status=500) + + # Apply filters + status_filter = request.query.get("status") + if status_filter: + queue = [item for item in queue if item["status"] == status_filter] + + author_filter = request.query.get("author") + if author_filter: + queue = [item for item in queue if item["author"] == author_filter] + + domain_filter = request.query.get("domain") + if domain_filter: + queue = [item for item in queue if item["domain"] == domain_filter] + + # Summary stats + status_counts = {} + for item in queue: + status_counts[item["status"]] = status_counts.get(item["status"], 0) + 1 + + return web.json_response({ + "queue": queue, + "total": len(queue), + "status_counts": status_counts, + }) + + +def register_review_queue_routes(app, forgejo_token=None): + """Register review queue routes on the app. + + forgejo_token: optional Forgejo API token for authenticated requests + """ + app["_forgejo_token"] = forgejo_token + app.router.add_get("/api/review-queue", handle_review_queue) diff --git a/diagnostics/shared_ui.py b/diagnostics/shared_ui.py new file mode 100644 index 0000000..e61eb49 --- /dev/null +++ b/diagnostics/shared_ui.py @@ -0,0 +1,149 @@ +"""Shared UI components for the 4-page Argus dashboard. + +Provides: nav bar, CSS, page skeleton, Chart.js imports, shared JS helpers. +All pages import render_page() and pass their body HTML + page-specific scripts. +""" + +# Page definitions — used by nav bar +PAGES = [ + {"path": "/prs", "label": "PRs", "icon": "✎"}, + {"path": "/ops", "label": "Operations", "icon": "⚙"}, + {"path": "/health", "label": "Knowledge Health", "icon": "♥"}, + {"path": "/agents", "label": "Agents", "icon": "★"}, + {"path": "/epistemic", "label": "Epistemic", "icon": "⚖"}, +] + + +def _nav_html(active_path: str) -> str: + """Render the shared navigation bar.""" + links = [] + for p in PAGES: + cls = "nav-active" if p["path"] == active_path else "" + links.append( + f'' + f'{p["icon"]} {p["label"]}' + ) + return f"""""" + + +SHARED_CSS = """ + * { box-sizing: border-box; margin: 0; padding: 0; } + body { font-family: -apple-system, system-ui, 'Segoe UI', sans-serif; background: #0d1117; color: #c9d1d9; } + .top-nav { display: flex; align-items: center; gap: 16px; padding: 12px 24px; + background: #161b22; border-bottom: 1px solid #30363d; position: sticky; top: 0; z-index: 100; } + .nav-brand { color: #58a6ff; font-weight: 700; font-size: 18px; } + .nav-links { display: flex; gap: 4px; flex: 1; } + .nav-aux { display: flex; gap: 4px; } + .nav-link { color: #8b949e; text-decoration: none; padding: 6px 12px; border-radius: 6px; + font-size: 13px; transition: all 0.15s; white-space: nowrap; } + .nav-link:hover { color: #c9d1d9; background: #21262d; } + .nav-active { color: #58a6ff !important; background: #0d1117; font-weight: 600; } + .page-content { padding: 24px; max-width: 1400px; margin: 0 auto; } + .page-header { margin-bottom: 20px; } + .page-header h1 { color: #58a6ff; font-size: 22px; } + .page-header .subtitle { color: #8b949e; font-size: 13px; margin-top: 4px; } + .grid { display: grid; grid-template-columns: repeat(auto-fit, minmax(160px, 1fr)); gap: 12px; margin: 16px 0; } + .card { background: #161b22; border: 1px solid #30363d; border-radius: 8px; padding: 16px; } + .card .label { color: #8b949e; font-size: 11px; text-transform: uppercase; letter-spacing: 0.5px; } + .card .value { font-size: 28px; font-weight: 700; margin-top: 2px; } + .card .detail { color: #8b949e; font-size: 11px; margin-top: 2px; } + .green { color: #3fb950; } + .yellow { color: #d29922; } + .red { color: #f85149; } + .blue { color: #58a6ff; } + .purple { color: #bc8cff; } + .chart-container { background: #161b22; border: 1px solid #30363d; border-radius: 8px; padding: 16px; margin: 16px 0; } + .chart-container h2 { color: #c9d1d9; font-size: 14px; margin-bottom: 12px; } + canvas { max-height: 260px; } + .row { display: grid; grid-template-columns: 1fr 1fr; gap: 16px; } + @media (max-width: 800px) { .row { grid-template-columns: 1fr; } } + table { width: 100%; border-collapse: collapse; font-size: 13px; } + th { color: #8b949e; font-size: 11px; text-transform: uppercase; text-align: left; padding: 6px 10px; border-bottom: 1px solid #30363d; } + td { padding: 6px 10px; border-bottom: 1px solid #21262d; } + code { background: #21262d; padding: 2px 6px; border-radius: 3px; font-size: 12px; } + .section { margin-top: 28px; } + .section-title { color: #58a6ff; font-size: 15px; font-weight: 600; margin-bottom: 12px; padding-bottom: 6px; border-bottom: 1px solid #21262d; } + .funnel { display: flex; align-items: center; gap: 8px; flex-wrap: wrap; } + .funnel-step { text-align: center; flex: 1; min-width: 100px; } + .funnel-step .num { font-size: 24px; font-weight: 700; } + .funnel-step .lbl { font-size: 11px; color: #8b949e; text-transform: uppercase; } + .funnel-arrow { color: #30363d; font-size: 20px; } + .footer { margin-top: 40px; padding: 16px 24px; border-top: 1px solid #21262d; color: #484f58; font-size: 11px; text-align: center; } + .footer a { color: #484f58; text-decoration: none; } + .footer a:hover { color: #8b949e; } + .alert-banner { padding: 8px 16px; font-size: 12px; border-radius: 6px; margin-bottom: 12px; } + .alert-critical { background: #f8514922; border: 1px solid #f85149; color: #f85149; } + .alert-warning { background: #d2992222; border: 1px solid #d29922; color: #d29922; } + .alert-info { background: #58a6ff22; border: 1px solid #58a6ff; color: #58a6ff; } + .badge { display: inline-block; padding: 2px 8px; border-radius: 4px; font-size: 11px; font-weight: 600; } + .badge-green { background: #23863633; color: #3fb950; } + .badge-yellow { background: #d2992233; color: #d29922; } + .badge-red { background: #f8514933; color: #f85149; } + .badge-blue { background: #1f6feb33; color: #58a6ff; } +""" + + +CHART_JS_IMPORTS = """ + +""" + + +SHARED_JS = """ +const AGENT_COLORS = { + 'rio': '#58a6ff', 'clay': '#3fb950', 'astra': '#bc8cff', + 'leo': '#d29922', 'vida': '#f0883e', 'theseus': '#f85149', + 'epimetheus': '#79c0ff', 'ganymede': '#8b949e', 'oberon': '#ec4899', +}; +function agentColor(name) { + return AGENT_COLORS[name?.toLowerCase()] || + '#' + ((name||'').split('').reduce((a,c) => (a*31+c.charCodeAt(0))&0xFFFFFF, 0x556677)).toString(16).padStart(6,'0'); +} +Chart.defaults.color = '#8b949e'; +Chart.defaults.borderColor = '#21262d'; +Chart.defaults.font.family = '-apple-system, system-ui, sans-serif'; +Chart.defaults.font.size = 11; + +function esc(s) { const d = document.createElement('div'); d.textContent = s; return d.innerHTML; } +function fmtPct(v) { return v != null ? (v * 100).toFixed(1) + '%' : '--'; } +function fmtNum(v) { return v != null ? v.toLocaleString() : '--'; } +function fmtDollars(v) { return v != null ? '$' + v.toFixed(2) : '--'; } +""" + + +def render_page(title: str, subtitle: str, active_path: str, body_html: str, + scripts: str = "", extra_css: str = "", timestamp: str = "") -> str: + """Render a complete page with nav, content, and footer.""" + ts_display = f" · {timestamp}" if timestamp else "" + return f""" + + +Argus - {title} + + +{CHART_JS_IMPORTS} + + +{_nav_html(active_path)} +
+
+

{title}

+
{subtitle}{ts_display} · auto-refresh 60s
+
+ {body_html} +
+
+ Argus · Teleo Pipeline Diagnostics · + Metrics API · + Vital Signs API · + Contributors API +
+ +{scripts} +""" diff --git a/diagnostics/tier1_metrics.py b/diagnostics/tier1_metrics.py new file mode 100644 index 0000000..69f4a8d --- /dev/null +++ b/diagnostics/tier1_metrics.py @@ -0,0 +1,476 @@ +"""Tier 1 Metrics — The three numbers that matter most for knowledge production. + +1. Extraction yield: claims merged / claims evaluated, per agent, per week +2. Cost per merged claim: total spend / merged claims, per week +3. Fix success rate by rejection tag: which rejection reasons are fixable vs terminal + +These queries run against pipeline.db (read-only) and power the /api/yield, +/api/cost-per-claim, and /api/fix-rates endpoints. + +Owner: Argus <69AF7290-758F-464B-B472-04AFCA4AB340> +""" + +import sqlite3 + + +def extraction_yield(conn: sqlite3.Connection, days: int = 30) -> dict: + """Extraction yield = merged / evaluated, trended per agent per week. + + Returns: + { + "daily": [{"day": "2026-W13", "agent": "rio", "evaluated": 20, "merged": 8, "yield": 0.4}, ...], + "totals": [{"agent": "rio", "evaluated": 100, "merged": 40, "yield": 0.4}, ...], + "system": {"evaluated": 500, "merged": 200, "yield": 0.4} + } + """ + # Weekly yield per agent + # Uses strftime('%Y-W%W') for ISO week grouping + # evaluated = approved + rejected (all terminal eval events) + # merged = approved events only + weekly = conn.execute( + """ + SELECT date(timestamp) as day, + json_extract(detail, '$.agent') as agent, + COUNT(*) as evaluated, + SUM(CASE WHEN event = 'approved' THEN 1 ELSE 0 END) as merged + FROM audit_log + WHERE stage = 'evaluate' + AND event IN ('approved', 'changes_requested', 'domain_rejected', 'tier05_rejected') + AND timestamp > datetime('now', ? || ' days') + GROUP BY day, agent + ORDER BY day DESC, agent + """, + (f"-{days}",), + ).fetchall() + + daily_data = [] + for r in weekly: + ev = r["evaluated"] or 0 + mg = r["merged"] or 0 + daily_data.append({ + "day": r["day"], + "agent": r["agent"] or "unknown", + "evaluated": ev, + "merged": mg, + "yield": round(mg / ev, 3) if ev else 0, + }) + + # Per-agent totals (same window) + totals = conn.execute( + """ + SELECT json_extract(detail, '$.agent') as agent, + COUNT(*) as evaluated, + SUM(CASE WHEN event = 'approved' THEN 1 ELSE 0 END) as merged + FROM audit_log + WHERE stage = 'evaluate' + AND event IN ('approved', 'changes_requested', 'domain_rejected', 'tier05_rejected') + AND timestamp > datetime('now', ? || ' days') + GROUP BY agent + ORDER BY merged DESC + """, + (f"-{days}",), + ).fetchall() + + totals_data = [] + for r in totals: + ev = r["evaluated"] or 0 + mg = r["merged"] or 0 + totals_data.append({ + "agent": r["agent"] or "unknown", + "evaluated": ev, + "merged": mg, + "yield": round(mg / ev, 3) if ev else 0, + }) + + # System-wide total + sys_row = conn.execute( + """ + SELECT COUNT(*) as evaluated, + SUM(CASE WHEN event = 'approved' THEN 1 ELSE 0 END) as merged + FROM audit_log + WHERE stage = 'evaluate' + AND event IN ('approved', 'changes_requested', 'domain_rejected', 'tier05_rejected') + AND timestamp > datetime('now', ? || ' days') + """, + (f"-{days}",), + ).fetchone() + + sys_ev = sys_row["evaluated"] or 0 + sys_mg = sys_row["merged"] or 0 + + return { + "days": days, + "daily": daily_data, + "totals": totals_data, + "system": { + "evaluated": sys_ev, + "merged": sys_mg, + "yield": round(sys_mg / sys_ev, 3) if sys_ev else 0, + }, + } + + +def cost_per_merged_claim(conn: sqlite3.Connection, days: int = 30) -> dict: + """Cost and compute per merged claim, trended per week. + + Uses costs table for spend + tokens and prs table for merge counts. + Breaks down by stage. Separates API spend (dollars) from subscription + compute (tokens only — Claude Max is flat-rate, so dollars are meaningless). + + Returns: + { + "daily": [{"day": "2026-W13", "api_cost": 1.50, "merged": 8, + "cost_per_claim": 0.19, "input_tokens": 50000, + "output_tokens": 5000, "total_tokens": 55000, + "tokens_per_claim": 6875}, ...], + "by_stage": [{"stage": "eval_leo:openrouter", "api_cost": 1.50, + "input_tokens": 300000, "output_tokens": 50000, + "calls": 100, "billing": "api"}, ...], + "system": {"api_cost": 2.36, "merged": 80, "cost_per_claim": 0.03, + "total_tokens": 1200000, "tokens_per_claim": 15000, + "subscription_tokens": 0, "api_tokens": 1200000} + } + """ + # Weekly: cost + tokens from costs table, merged count from prs table + daily_cost = conn.execute( + """ + SELECT date as day, + SUM(cost_usd) as api_cost, + SUM(cost_estimate_usd) as estimated_cost, + SUM(input_tokens) as input_tokens, + SUM(output_tokens) as output_tokens + FROM costs + WHERE date > date('now', ? || ' days') + GROUP BY day + ORDER BY day DESC + """, + (f"-{days}",), + ).fetchall() + + daily_merges = conn.execute( + """ + SELECT date(merged_at) as day, + COUNT(*) as merged + FROM prs + WHERE status = 'merged' + AND merged_at > datetime('now', ? || ' days') + GROUP BY day + ORDER BY day DESC + """, + (f"-{days}",), + ).fetchall() + + # Merge into combined weekly view + merge_map = {r["day"]: r["merged"] for r in daily_merges} + cost_map = {} + for r in daily_cost: + cost_map[r["day"]] = { + "api_cost": r["api_cost"] or 0, + "estimated_cost": r["estimated_cost"] or 0, + "input_tokens": r["input_tokens"] or 0, + "output_tokens": r["output_tokens"] or 0, + } + + all_days = sorted(set(list(merge_map.keys()) + list(cost_map.keys())), reverse=True) + daily_data = [] + for w in all_days: + c = cost_map.get(w, {"api_cost": 0, "estimated_cost": 0, "input_tokens": 0, "output_tokens": 0}) + merged = merge_map.get(w, 0) or 0 + total_tokens = c["input_tokens"] + c["output_tokens"] + daily_data.append({ + "day": w, + "actual_spend": round(c["api_cost"], 4), + "estimated_cost": round(c["estimated_cost"], 4), + "merged": merged, + "cost_per_claim": round(c["estimated_cost"] / merged, 4) if merged else None, + "input_tokens": c["input_tokens"], + "output_tokens": c["output_tokens"], + "total_tokens": total_tokens, + "tokens_per_claim": round(total_tokens / merged) if merged else None, + }) + + # By stage with billing type (full window) + by_stage = conn.execute( + """ + SELECT stage, + SUM(cost_usd) as api_cost, + SUM(cost_estimate_usd) as estimated_cost, + SUM(input_tokens) as input_tokens, + SUM(output_tokens) as output_tokens, + SUM(calls) as calls + FROM costs + WHERE date > date('now', ? || ' days') + GROUP BY stage + ORDER BY SUM(input_tokens + output_tokens) DESC + """, + (f"-{days}",), + ).fetchall() + + stage_data = [] + total_api_cost = 0 + total_estimated_cost = 0 + total_input = 0 + total_output = 0 + subscription_tokens = 0 + api_tokens = 0 + for r in by_stage: + cost = r["api_cost"] or 0 + est = r["estimated_cost"] or 0 + inp = r["input_tokens"] or 0 + out = r["output_tokens"] or 0 + calls = r["calls"] or 0 + stage_name = r["stage"] + # :max suffix = subscription, :openrouter suffix = API + billing = "subscription" if ":max" in stage_name else "api" + total_api_cost += cost + total_estimated_cost += est + total_input += inp + total_output += out + if billing == "subscription": + subscription_tokens += inp + out + else: + api_tokens += inp + out + stage_data.append({ + "stage": stage_name, + "api_cost": round(cost, 4), + "estimated_cost": round(est, 4), + "input_tokens": inp, + "output_tokens": out, + "calls": calls, + "billing": billing, + }) + + # System totals + sys_merged = conn.execute( + "SELECT COUNT(*) as n FROM prs WHERE status='merged' AND merged_at > datetime('now', ? || ' days')", + (f"-{days}",), + ).fetchone()["n"] or 0 + + total_tokens = total_input + total_output + + return { + "days": days, + "daily": daily_data, + "by_stage": stage_data, + "system": { + "actual_spend": round(total_api_cost, 4), + "estimated_cost": round(total_estimated_cost, 4), + "merged": sys_merged, + "cost_per_claim": round(total_estimated_cost / sys_merged, 4) if sys_merged else None, + "total_tokens": total_tokens, + "tokens_per_claim": round(total_tokens / sys_merged) if sys_merged else None, + "subscription_tokens": subscription_tokens, + "api_tokens": api_tokens, + "note": "estimated_cost = API-rate equivalent for all calls (unified metric). actual_spend = real dollars charged to OpenRouter.", + }, + } + + +def fix_success_by_tag(conn: sqlite3.Connection, days: int = 30) -> dict: + """Fix success rate broken down by rejection reason. + + For each rejection tag: how many PRs got that rejection, how many eventually + merged (successful fix), how many are still open (in progress), how many + were abandoned (closed/zombie without merge). + + Returns: + { + "tags": [ + { + "tag": "insufficient_evidence", + "total": 50, + "fixed": 10, + "in_progress": 5, + "terminal": 35, + "fix_rate": 0.2, + "terminal_rate": 0.7 + }, ... + ] + } + """ + # Get all rejection events with their tags and PR numbers + # Then join with prs table to see final outcome + rows = conn.execute( + """ + SELECT value as tag, + json_extract(al.detail, '$.pr') as pr_number + FROM audit_log al, json_each(json_extract(al.detail, '$.issues')) + WHERE al.stage = 'evaluate' + AND al.event IN ('changes_requested', 'domain_rejected', 'tier05_rejected') + AND al.timestamp > datetime('now', ? || ' days') + """, + (f"-{days}",), + ).fetchall() + + # Collect unique PRs per tag + tag_prs: dict[str, set] = {} + for r in rows: + tag = r["tag"] + pr = r["pr_number"] + if tag not in tag_prs: + tag_prs[tag] = set() + if pr is not None: + tag_prs[tag].add(pr) + + if not tag_prs: + return {"days": days, "tags": []} + + # Get status for all referenced PRs in one query + all_prs = set() + for prs in tag_prs.values(): + all_prs.update(prs) + + if not all_prs: + return {"days": days, "tags": []} + + placeholders = ",".join("?" for _ in all_prs) + pr_statuses = conn.execute( + f"SELECT number, status FROM prs WHERE number IN ({placeholders})", + list(all_prs), + ).fetchall() + status_map = {r["number"]: r["status"] for r in pr_statuses} + + # Compute per-tag outcomes + tag_data = [] + for tag, prs in sorted(tag_prs.items(), key=lambda x: -len(x[1])): + fixed = 0 + in_progress = 0 + terminal = 0 + for pr in prs: + st = status_map.get(pr, "unknown") + if st == "merged": + fixed += 1 + elif st in ("open", "validating", "reviewing", "merging"): + in_progress += 1 + else: + # closed, zombie, conflict, unknown + terminal += 1 + + total = len(prs) + # Fix rate excludes in-progress (only counts resolved PRs) + resolved = fixed + terminal + tag_data.append({ + "tag": tag, + "total": total, + "fixed": fixed, + "in_progress": in_progress, + "terminal": terminal, + "fix_rate": round(fixed / resolved, 3) if resolved else None, + "terminal_rate": round(terminal / resolved, 3) if resolved else None, + }) + + return {"days": days, "tags": tag_data} + + +def compute_profile(conn: "sqlite3.Connection", days: int = 30) -> dict: + """Compute profile — Max subscription telemetry alongside API usage. + + Surfaces: cache hit rates, latency, cost estimates (API-equivalent), + token breakdown by billing type. + """ + rows = conn.execute( + """ + SELECT stage, model, + SUM(calls) as calls, + SUM(input_tokens) as input_tokens, + SUM(output_tokens) as output_tokens, + SUM(cost_usd) as api_cost, + SUM(duration_ms) as duration_ms, + SUM(cache_read_tokens) as cache_read_tokens, + SUM(cache_write_tokens) as cache_write_tokens, + SUM(cost_estimate_usd) as cost_estimate_usd + FROM costs + WHERE date > date('now', ? || ' days') + GROUP BY stage, model + ORDER BY SUM(input_tokens + output_tokens) DESC + """, + (f"-{days}",), + ).fetchall() + + stage_data = [] + total_calls = 0 + total_tokens = 0 + total_duration = 0 + total_cache_read = 0 + total_cache_write = 0 + api_calls = 0 + sub_calls = 0 + api_spend = 0.0 + sub_estimate = 0.0 + sub_input_tokens = 0 + + for r in rows: + calls = r["calls"] or 0 + inp = r["input_tokens"] or 0 + out = r["output_tokens"] or 0 + dur = r["duration_ms"] or 0 + cr = r["cache_read_tokens"] or 0 + cw = r["cache_write_tokens"] or 0 + cost = r["api_cost"] or 0 + est = r["cost_estimate_usd"] or 0 + stage_name = r["stage"] + billing = "subscription" if ":max" in stage_name else "api" + + total_calls += calls + total_tokens += inp + out + total_duration += dur + total_cache_read += cr + total_cache_write += cw + + if billing == "subscription": + sub_calls += calls + sub_estimate += est + sub_input_tokens += inp + else: + api_calls += calls + api_spend += cost + + stage_data.append({ + "stage": stage_name, + "model": r["model"], + "calls": calls, + "input_tokens": inp, + "output_tokens": out, + "total_tokens": inp + out, + "duration_ms": dur, + "avg_latency_ms": round(dur / calls) if calls else 0, + "cache_read_tokens": cr, + "cache_write_tokens": cw, + "cache_hit_rate": round(cr / (cr + inp), 3) if (cr + inp) else 0, + "api_cost": round(cost, 4), + "cost_estimate_usd": round(est, 4), + "billing": billing, + }) + + # Cache summary (only meaningful for subscription/Max calls) + total_cacheable = total_cache_read + total_cache_write + sub_input_tokens + cache_hit_rate = round(total_cache_read / total_cacheable, 3) if total_cacheable else 0 + + return { + "days": days, + "by_stage": stage_data, + "cache": { + "read_tokens": total_cache_read, + "write_tokens": total_cache_write, + "hit_rate": cache_hit_rate, + "note": "Cache hits are prompt tokens served from cache (cheaper/faster)", + }, + "latency": { + "total_ms": total_duration, + "avg_ms_per_call": round(total_duration / total_calls) if total_calls else 0, + "note": "Wall-clock time including network. Only populated for Claude Max calls.", + }, + "subscription_estimate": { + "total_cost_usd": round(sub_estimate, 4), + "note": "What subscription calls would cost at API rates. Actual cost: $0 (flat-rate Max plan).", + }, + "system": { + "total_calls": total_calls, + "total_tokens": total_tokens, + "api_calls": api_calls, + "subscription_calls": sub_calls, + "api_spend": round(api_spend, 4), + "subscription_estimate": round(sub_estimate, 4), + "cache_hit_rate": cache_hit_rate, + }, + } diff --git a/diagnostics/tier1_routes.py b/diagnostics/tier1_routes.py new file mode 100644 index 0000000..b28c0f1 --- /dev/null +++ b/diagnostics/tier1_routes.py @@ -0,0 +1,57 @@ +"""Tier 1 Metrics — API routes for Argus dashboard. + +Four endpoints: + GET /api/yield — extraction yield per agent per day + GET /api/cost-per-claim — cost per merged claim per day + stage breakdown + GET /api/fix-rates — fix success rate by rejection tag + GET /api/compute-profile — full compute telemetry (cache, latency, cost estimates) + +All accept ?days=N (default 30) to control lookback window. + +Owner: Argus <69AF7290-758F-464B-B472-04AFCA4AB340> +""" + +from aiohttp import web + +from tier1_metrics import cost_per_merged_claim, compute_profile, extraction_yield, fix_success_by_tag + + +def _parse_days(request, default=30): + """Parse and clamp ?days= parameter. Returns 1..365.""" + try: + days = int(request.query.get("days", str(default))) + except (ValueError, TypeError): + days = default + return max(1, min(days, 365)) + + +async def handle_yield(request): + conn = request.app["_get_conn"]() + days = _parse_days(request) + return web.json_response(extraction_yield(conn, days)) + + +async def handle_cost_per_claim(request): + conn = request.app["_get_conn"]() + days = _parse_days(request) + return web.json_response(cost_per_merged_claim(conn, days)) + + +async def handle_fix_rates(request): + conn = request.app["_get_conn"]() + days = _parse_days(request) + return web.json_response(fix_success_by_tag(conn, days)) + + +async def handle_compute_profile(request): + conn = request.app["_get_conn"]() + days = _parse_days(request) + return web.json_response(compute_profile(conn, days)) + + +def register_tier1_routes(app: web.Application, get_conn): + app["_get_conn"] = get_conn + app.router.add_get("/api/yield", handle_yield) + app.router.add_get("/api/cost-per-claim", handle_cost_per_claim) + app.router.add_get("/api/fix-rates", handle_fix_rates) + app.router.add_get("/api/compute-profile", handle_compute_profile) diff --git a/evaluate-trigger.sh b/evaluate-trigger.sh new file mode 100755 index 0000000..aa865cb --- /dev/null +++ b/evaluate-trigger.sh @@ -0,0 +1,621 @@ +#!/usr/bin/env bash +# evaluate-trigger.sh — Find unreviewed PRs, run 2-agent review, auto-merge if approved. +# +# Reviews each PR with up to THREE agents: +# 1. Leo (evaluator) — quality gates, cross-domain connections, coherence +# 2. Domain agent — domain expertise, duplicate check, technical accuracy +# 3. Ganymede (code reviewer) — code quality, correctness, safety (code PRs only) +# +# Ganymede reviews any PR that touches code files (ops/, diagnostics/, .py, .sh, etc.) +# +# After all reviews, auto-merges if: +# - Leo's comment contains "**Verdict:** approve" +# - Domain agent's comment contains "**Verdict:** approve" (if applicable) +# - Ganymede's comment contains "**Verdict:** approve" (if code PR) +# - No territory violations (files outside proposer's domain) +# +# Usage: +# ./ops/evaluate-trigger.sh # review + auto-merge approved PRs +# ./ops/evaluate-trigger.sh 47 # review a specific PR by number +# ./ops/evaluate-trigger.sh --dry-run # show what would be reviewed, don't run +# ./ops/evaluate-trigger.sh --leo-only # skip domain agent, just run Leo +# ./ops/evaluate-trigger.sh --no-merge # review only, don't auto-merge (old behavior) +# +# Requirements: +# - claude CLI (claude -p for headless mode) +# - gh CLI authenticated with repo access +# - Run from the teleo-codex repo root +# +# Safety: +# - Lockfile prevents concurrent runs +# - Auto-merge requires ALL reviewers to approve + no territory violations +# - Each PR runs sequentially to avoid branch conflicts +# - Timeout: 20 minutes per agent per PR +# - Pre-flight checks: clean working tree, gh auth +# +# Verdict protocol: +# All agents use `gh pr comment` (NOT `gh pr review`) because all agents +# share the m3taversal GitHub account — `gh pr review --approve` fails +# when the PR author and reviewer are the same user. The merge check +# parses issue comments for structured verdict markers instead. + +set -euo pipefail + +# Allow nested Claude Code sessions (headless spawned from interactive) +unset CLAUDECODE 2>/dev/null || true + +REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)" +cd "$REPO_ROOT" + +LOCKFILE="/tmp/evaluate-trigger.lock" +LOG_DIR="$REPO_ROOT/ops/sessions" +TIMEOUT_SECONDS=1200 +DRY_RUN=false +LEO_ONLY=false +NO_MERGE=false +SPECIFIC_PR="" + +# --- Code PR detection --- +# Returns "true" if the PR touches code files (ops/, diagnostics/, scripts, .py, .sh, .js, .html) +# These PRs need Ganymede code review in addition to Leo's quality review. +detect_code_pr() { + local pr_number="$1" + local files + + files=$(gh pr view "$pr_number" --json files --jq '.files[].path' 2>/dev/null || echo "") + + if echo "$files" | grep -qE "^ops/|^diagnostics/|\.py$|\.sh$|\.js$|\.html$|\.css$|\.json$"; then + echo "true" + else + echo "false" + fi +} + +# --- Domain routing map --- +# Maps branch prefix or domain directory to agent name and identity path +detect_domain_agent() { + local pr_number="$1" + local branch files domain agent + + branch=$(gh pr view "$pr_number" --json headRefName --jq '.headRefName' 2>/dev/null || echo "") + files=$(gh pr view "$pr_number" --json files --jq '.files[].path' 2>/dev/null || echo "") + + # Try branch prefix first + case "$branch" in + rio/*|*/internet-finance*) agent="rio"; domain="internet-finance" ;; + clay/*|*/entertainment*) agent="clay"; domain="entertainment" ;; + theseus/*|*/ai-alignment*) agent="theseus"; domain="ai-alignment" ;; + vida/*|*/health*) agent="vida"; domain="health" ;; + astra/*|*/space-development*) agent="astra"; domain="space-development" ;; + leo/*|*/grand-strategy*) agent="leo"; domain="grand-strategy" ;; + contrib/*) + # External contributor — detect domain from changed files (fall through to file check) + agent=""; domain="" + ;; + *) + agent=""; domain="" + ;; + esac + + # If no agent detected from branch prefix, check changed files + if [ -z "$agent" ]; then + if echo "$files" | grep -q "domains/internet-finance/"; then + agent="rio"; domain="internet-finance" + elif echo "$files" | grep -q "domains/entertainment/"; then + agent="clay"; domain="entertainment" + elif echo "$files" | grep -q "domains/ai-alignment/"; then + agent="theseus"; domain="ai-alignment" + elif echo "$files" | grep -q "domains/health/"; then + agent="vida"; domain="health" + elif echo "$files" | grep -q "domains/space-development/"; then + agent="astra"; domain="space-development" + fi + fi + + echo "$agent $domain" +} + +# --- Parse arguments --- +for arg in "$@"; do + case "$arg" in + --dry-run) DRY_RUN=true ;; + --leo-only) LEO_ONLY=true ;; + --no-merge) NO_MERGE=true ;; + [0-9]*) SPECIFIC_PR="$arg" ;; + --help|-h) + head -23 "$0" | tail -21 + exit 0 + ;; + *) + echo "Unknown argument: $arg" + exit 1 + ;; + esac +done + +# --- Pre-flight checks --- +if ! gh auth status >/dev/null 2>&1; then + echo "ERROR: gh CLI not authenticated. Run 'gh auth login' first." + exit 1 +fi + +if ! command -v claude >/dev/null 2>&1; then + echo "ERROR: claude CLI not found. Install it first." + exit 1 +fi + +# Check for dirty working tree (ignore ops/, .claude/, .github/ which may contain local-only files) +DIRTY_FILES=$(git status --porcelain | grep -v '^?? ops/' | grep -v '^ M ops/' | grep -v '^?? \.claude/' | grep -v '^ M \.claude/' | grep -v '^?? \.github/' | grep -v '^ M \.github/' || true) +if [ -n "$DIRTY_FILES" ]; then + echo "ERROR: Working tree is dirty. Clean up before running." + echo "$DIRTY_FILES" + exit 1 +fi + +# --- Lockfile (prevent concurrent runs) --- +if [ -f "$LOCKFILE" ]; then + LOCK_PID=$(cat "$LOCKFILE" 2>/dev/null || echo "") + if [ -n "$LOCK_PID" ] && kill -0 "$LOCK_PID" 2>/dev/null; then + echo "Another evaluate-trigger is running (PID $LOCK_PID). Exiting." + exit 1 + else + echo "Stale lockfile found. Removing." + rm -f "$LOCKFILE" + fi +fi +echo $$ > "$LOCKFILE" +trap 'rm -f "$LOCKFILE"' EXIT + +# --- Ensure log directory exists --- +mkdir -p "$LOG_DIR" + +# --- Find PRs to review --- +if [ -n "$SPECIFIC_PR" ]; then + PR_STATE=$(gh pr view "$SPECIFIC_PR" --json state --jq '.state' 2>/dev/null || echo "NOT_FOUND") + if [ "$PR_STATE" != "OPEN" ]; then + echo "PR #$SPECIFIC_PR is $PR_STATE (not OPEN). Reviewing anyway for testing." + fi + PRS_TO_REVIEW="$SPECIFIC_PR" +else + # NOTE: gh pr list silently returns empty in some worktree configs; use gh api instead + OPEN_PRS=$(gh api repos/:owner/:repo/pulls --jq '.[].number' 2>/dev/null || echo "") + + if [ -z "$OPEN_PRS" ]; then + echo "No open PRs found. Nothing to review." + exit 0 + fi + + PRS_TO_REVIEW="" + for pr in $OPEN_PRS; do + # Check if this PR already has a Leo verdict comment (avoid re-reviewing) + LEO_COMMENTED=$(gh pr view "$pr" --json comments \ + --jq '[.comments[] | select(.body | test("VERDICT:LEO:(APPROVE|REQUEST_CHANGES)"))] | length' 2>/dev/null || echo "0") + LAST_COMMIT_DATE=$(gh pr view "$pr" --json commits --jq '.commits[-1].committedDate' 2>/dev/null || echo "") + + if [ "$LEO_COMMENTED" = "0" ]; then + PRS_TO_REVIEW="$PRS_TO_REVIEW $pr" + else + # Check if new commits since last Leo review + LAST_LEO_DATE=$(gh pr view "$pr" --json comments \ + --jq '[.comments[] | select(.body | test("VERDICT:LEO:")) | .createdAt] | last' 2>/dev/null || echo "") + if [ -n "$LAST_COMMIT_DATE" ] && [ -n "$LAST_LEO_DATE" ] && [[ "$LAST_COMMIT_DATE" > "$LAST_LEO_DATE" ]]; then + echo "PR #$pr: New commits since last review. Queuing for re-review." + PRS_TO_REVIEW="$PRS_TO_REVIEW $pr" + else + echo "PR #$pr: Already reviewed. Skipping." + fi + fi + done + + PRS_TO_REVIEW=$(echo "$PRS_TO_REVIEW" | xargs) + + if [ -z "$PRS_TO_REVIEW" ]; then + echo "All open PRs are up to date. Nothing to do." + exit 0 + fi +fi + +echo "PRs to review: $PRS_TO_REVIEW" + +if [ "$DRY_RUN" = true ]; then + for pr in $PRS_TO_REVIEW; do + read -r agent domain <<< "$(detect_domain_agent "$pr")" + is_code=$(detect_code_pr "$pr") + reviewers="Leo + ${agent:-unknown} (${domain:-unknown domain})" + [ "$is_code" = "true" ] && reviewers="$reviewers + Ganymede (code)" + echo "[DRY RUN] PR #$pr — $reviewers" + done + exit 0 +fi + +# --- Run headless reviews on each PR --- +run_agent_review() { + local pr="$1" agent_name="$2" prompt="$3" model="$4" + local timestamp log_file review_file + + timestamp=$(date +%Y%m%d-%H%M%S) + log_file="$LOG_DIR/${agent_name}-review-pr${pr}-${timestamp}.log" + review_file="/tmp/${agent_name}-review-pr${pr}.md" + + echo " Running ${agent_name} (model: ${model})..." + echo " Log: $log_file" + + if perl -e "alarm $TIMEOUT_SECONDS; exec @ARGV" claude -p \ + --model "$model" \ + --allowedTools "Read,Write,Edit,Bash,Glob,Grep" \ + --permission-mode bypassPermissions \ + "$prompt" \ + > "$log_file" 2>&1; then + echo " ${agent_name}: Review posted." + rm -f "$review_file" + return 0 + else + local exit_code=$? + if [ "$exit_code" -eq 142 ] || [ "$exit_code" -eq 124 ]; then + echo " ${agent_name}: TIMEOUT after ${TIMEOUT_SECONDS}s." + else + echo " ${agent_name}: FAILED (exit code $exit_code)." + fi + rm -f "$review_file" + return 1 + fi +} + +# --- Territory violation check --- +# Verifies all changed files are within the proposer's expected territory +check_territory_violations() { + local pr_number="$1" + local branch files proposer violations + + branch=$(gh pr view "$pr_number" --json headRefName --jq '.headRefName' 2>/dev/null || echo "") + files=$(gh pr view "$pr_number" --json files --jq '.files[].path' 2>/dev/null || echo "") + + # Determine proposer from branch prefix + proposer=$(echo "$branch" | cut -d'/' -f1) + + # Map proposer to allowed directories + local allowed_domains="" + case "$proposer" in + rio) allowed_domains="domains/internet-finance/" ;; + clay) allowed_domains="domains/entertainment/" ;; + theseus) allowed_domains="domains/ai-alignment/" ;; + vida) allowed_domains="domains/health/" ;; + astra) allowed_domains="domains/space-development/" ;; + leo) allowed_domains="core/|foundations/" ;; + contrib) echo ""; return 0 ;; # External contributors — skip territory check + *) echo ""; return 0 ;; # Unknown proposer — skip check + esac + + # Check each file — allow inbox/archive/, agents/{proposer}/, schemas/, foundations/, and the agent's domain + violations="" + while IFS= read -r file; do + [ -z "$file" ] && continue + # Always allowed: inbox/archive, own agent dir, maps/, foundations/ (any agent can propose foundation claims) + if echo "$file" | grep -qE "^inbox/archive/|^agents/${proposer}/|^maps/|^foundations/"; then + continue + fi + # Check against allowed domain directories + if echo "$file" | grep -qE "^${allowed_domains}"; then + continue + fi + violations="${violations} - ${file}\n" + done <<< "$files" + + if [ -n "$violations" ]; then + echo -e "$violations" + else + echo "" + fi +} + +# --- Auto-merge check --- +# Parses issue comments for structured verdict markers. +# Verdict protocol: agents post `` or +# `` as HTML comments in their review. +# This is machine-parseable and invisible in the rendered comment. +check_merge_eligible() { + local pr_number="$1" + local domain_agent="$2" + local leo_passed="$3" + local is_code_pr="${4:-false}" + local ganymede_passed="${5:-true}" + + # Gate 1: Leo must have completed without timeout/error + if [ "$leo_passed" != "true" ]; then + echo "BLOCK: Leo review failed or timed out" + return 1 + fi + + # Gate 2: Check Leo's verdict from issue comments + local leo_verdict + leo_verdict=$(gh pr view "$pr_number" --json comments \ + --jq '[.comments[] | select(.body | test("VERDICT:LEO:")) | .body] | last' 2>/dev/null || echo "") + + if echo "$leo_verdict" | grep -q "VERDICT:LEO:APPROVE"; then + echo "Leo: APPROVED" + elif echo "$leo_verdict" | grep -q "VERDICT:LEO:REQUEST_CHANGES"; then + echo "BLOCK: Leo requested changes" + return 1 + else + echo "BLOCK: Could not find Leo's verdict marker in PR comments" + return 1 + fi + + # Gate 3: Check domain agent verdict (if applicable) + if [ -n "$domain_agent" ] && [ "$domain_agent" != "leo" ]; then + local domain_key + domain_key=$(echo "$domain_agent" | tr '[:lower:]' '[:upper:]') + local domain_verdict + domain_verdict=$(gh pr view "$pr_number" --json comments \ + --jq "[.comments[] | select(.body | test(\"VERDICT:${domain_key}:\")) | .body] | last" 2>/dev/null || echo "") + + if echo "$domain_verdict" | grep -q "VERDICT:${domain_key}:APPROVE"; then + echo "Domain agent ($domain_agent): APPROVED" + elif echo "$domain_verdict" | grep -q "VERDICT:${domain_key}:REQUEST_CHANGES"; then + echo "BLOCK: $domain_agent requested changes" + return 1 + else + echo "BLOCK: No verdict marker found for $domain_agent" + return 1 + fi + else + echo "Domain agent: N/A (leo-only or grand-strategy)" + fi + + # Gate 4: Ganymede code review (for code PRs) + if [ "$is_code_pr" = "true" ]; then + if [ "$ganymede_passed" != "true" ]; then + echo "BLOCK: Ganymede code review failed or timed out" + return 1 + fi + + local ganymede_verdict + ganymede_verdict=$(gh pr view "$pr_number" --json comments \ + --jq '[.comments[] | select(.body | test("VERDICT:GANYMEDE:")) | .body] | last' 2>/dev/null || echo "") + + if echo "$ganymede_verdict" | grep -q "VERDICT:GANYMEDE:APPROVE"; then + echo "Ganymede (code review): APPROVED" + elif echo "$ganymede_verdict" | grep -q "VERDICT:GANYMEDE:REQUEST_CHANGES"; then + echo "BLOCK: Ganymede requested code changes" + return 1 + else + echo "BLOCK: No verdict marker found for Ganymede code review" + return 1 + fi + fi + + # Gate 5: Territory violations + local violations + violations=$(check_territory_violations "$pr_number") + + if [ -n "$violations" ]; then + echo "BLOCK: Territory violations detected:" + echo -e "$violations" + return 1 + else + echo "Territory: clean" + fi + + return 0 +} + +REVIEWED=0 +FAILED=0 +MERGED=0 + +for pr in $PRS_TO_REVIEW; do + echo "" + echo "=== PR #$pr ===" + echo "Started: $(date)" + + # Detect which domain agent should review + read -r DOMAIN_AGENT DOMAIN <<< "$(detect_domain_agent "$pr")" + echo "Domain: ${DOMAIN:-unknown} | Agent: ${DOMAIN_AGENT:-none detected}" + + # --- Review 1: Leo (evaluator) --- + LEO_REVIEW_FILE="/tmp/leo-review-pr${pr}.md" + LEO_PROMPT="You are Leo. Read agents/leo/identity.md, agents/leo/beliefs.md, agents/leo/reasoning.md, and skills/evaluate.md. + +Review PR #${pr} on this repo. + +First, run: gh pr view ${pr} --json title,body,files,additions,deletions +Then checkout the PR branch: gh pr checkout ${pr} +Read every changed file completely. + +Before evaluating, scan the existing knowledge base for duplicate and contradiction checks: +- List claim files in the relevant domain directory (e.g., domains/${DOMAIN}/) +- Read titles to check for semantic duplicates +- Check for contradictions with existing claims in that domain and in foundations/ + +For each proposed claim, evaluate against these 11 quality criteria from CLAUDE.md: +1. Specificity — Is this specific enough to disagree with? +2. Evidence — Is there traceable evidence in the body? +3. Description quality — Does the description add info beyond the title? +4. Confidence calibration — Does the confidence level match the evidence? +5. Duplicate check — Does this already exist in the knowledge base? +6. Contradiction check — Does this contradict an existing claim? If so, is the contradiction explicit? +7. Value add — Does this genuinely expand what the knowledge base knows? +8. Wiki links — Do all [[links]] point to real files? +9. Scope qualification — Does the claim specify structural vs functional, micro vs macro, causal vs correlational? +10. Universal quantifier check — Does the title use unwarranted universals (all, always, never, the only)? +11. Counter-evidence acknowledgment — For likely or higher: is opposing evidence acknowledged? + +Also check: +- Source archive updated correctly (status field) +- Commit messages follow conventions +- Files are in the correct domain directory +- Cross-domain connections that the proposer may have missed + +Write your complete review to ${LEO_REVIEW_FILE} + +CRITICAL — Verdict format: Your review MUST end with exactly one of these verdict markers (as an HTML comment on its own line): + + + +Then post the review as an issue comment: + gh pr comment ${pr} --body-file ${LEO_REVIEW_FILE} + +IMPORTANT: Use 'gh pr comment' NOT 'gh pr review'. We use a shared GitHub account so gh pr review --approve fails. +DO NOT merge — the orchestrator handles merge decisions after all reviews are posted. +Work autonomously. Do not ask for confirmation." + + if run_agent_review "$pr" "leo" "$LEO_PROMPT" "opus"; then + LEO_PASSED=true + else + LEO_PASSED=false + fi + + # Return to main between reviews + git checkout main 2>/dev/null || git checkout -f main + PR_BRANCH=$(gh pr view "$pr" --json headRefName --jq '.headRefName' 2>/dev/null || echo "") + [ -n "$PR_BRANCH" ] && git branch -D "$PR_BRANCH" 2>/dev/null || true + + # --- Review 2: Domain agent --- + if [ "$LEO_ONLY" = true ]; then + echo " Skipping domain agent review (--leo-only)." + elif [ -z "$DOMAIN_AGENT" ]; then + echo " Could not detect domain agent. Skipping domain review." + elif [ "$DOMAIN_AGENT" = "leo" ]; then + echo " Domain is grand-strategy (Leo's territory). Single review sufficient." + else + DOMAIN_REVIEW_FILE="/tmp/${DOMAIN_AGENT}-review-pr${pr}.md" + AGENT_NAME_UPPER=$(echo "${DOMAIN_AGENT}" | awk '{print toupper(substr($0,1,1)) substr($0,2)}') + AGENT_KEY_UPPER=$(echo "${DOMAIN_AGENT}" | tr '[:lower:]' '[:upper:]') + DOMAIN_PROMPT="You are ${AGENT_NAME_UPPER}. Read agents/${DOMAIN_AGENT}/identity.md, agents/${DOMAIN_AGENT}/beliefs.md, and skills/evaluate.md. + +You are reviewing PR #${pr} as the domain expert for ${DOMAIN}. + +First, run: gh pr view ${pr} --json title,body,files,additions,deletions +Then checkout the PR branch: gh pr checkout ${pr} +Read every changed file completely. + +Your review focuses on DOMAIN EXPERTISE — things only a ${DOMAIN} specialist would catch: + +1. **Technical accuracy** — Are the claims factually correct within the ${DOMAIN} domain? +2. **Domain duplicates** — Do any claims duplicate existing knowledge in domains/${DOMAIN}/? + Scan the directory and read titles carefully. +3. **Missing context** — What important nuance from the ${DOMAIN} domain is the claim missing? +4. **Belief impact** — Do any claims affect your current beliefs? Read agents/${DOMAIN_AGENT}/beliefs.md + and flag if any belief needs updating. +5. **Connections** — What existing claims in your domain should be wiki-linked? +6. **Confidence calibration** — From your domain expertise, is the confidence level right? + +Write your review to ${DOMAIN_REVIEW_FILE} + +CRITICAL — Verdict format: Your review MUST end with exactly one of these verdict markers (as an HTML comment on its own line): + + + +Then post the review as an issue comment: + gh pr comment ${pr} --body-file ${DOMAIN_REVIEW_FILE} + +IMPORTANT: Use 'gh pr comment' NOT 'gh pr review'. We use a shared GitHub account so gh pr review --approve fails. +Sign your review as ${AGENT_NAME_UPPER} (domain reviewer for ${DOMAIN}). +DO NOT duplicate Leo's quality gate checks — he covers those. +DO NOT merge — the orchestrator handles merge decisions after all reviews are posted. +Work autonomously. Do not ask for confirmation." + + run_agent_review "$pr" "$DOMAIN_AGENT" "$DOMAIN_PROMPT" "sonnet" + + # Clean up branch again + git checkout main 2>/dev/null || git checkout -f main + [ -n "$PR_BRANCH" ] && git branch -D "$PR_BRANCH" 2>/dev/null || true + fi + + # --- Review 3: Ganymede code review (for PRs touching code files) --- + IS_CODE_PR=$(detect_code_pr "$pr") + GANYMEDE_PASSED=true + + if [ "$IS_CODE_PR" = "true" ] && [ "$LEO_ONLY" != true ]; then + echo " Code files detected — running Ganymede code review." + GANYMEDE_REVIEW_FILE="/tmp/ganymede-review-pr${pr}.md" + GANYMEDE_PROMPT="You are Ganymede, the code quality reviewer for the Teleo collective. + +Review PR #${pr} for code quality, correctness, and safety. + +First, run: gh pr view ${pr} --json title,body,files,additions,deletions +Then checkout the PR branch: gh pr checkout ${pr} +Read every changed file completely. Also read the existing versions of modified files on main for comparison. + +Your review focuses on CODE QUALITY — things a code reviewer catches: + +1. **Correctness** — Does the code do what it claims? Are there logic errors, off-by-one bugs, or unhandled edge cases? +2. **Safety** — Any security issues? SQL injection, path traversal, unchecked inputs, secrets in code? +3. **Breaking changes** — Does this change file formats, API responses, DB schemas, or config structures that other agents depend on? If so, is there a migration path? +4. **Error handling** — Will failures be visible or silent? Are there bare excepts, missing error messages, or swallowed exceptions? +5. **Integration** — Does the code work with the existing system? Are imports correct, paths valid, dependencies present? +6. **Simplicity** — Is this more complex than it needs to be? Could it be simpler? + +Also check: +- systemd ReadWritePaths if new file write paths are introduced +- Path format consistency (absolute vs relative) +- Concurrent edit risk on shared files (app.py, bot.py, etc.) + +Write your review to ${GANYMEDE_REVIEW_FILE} + +CRITICAL — Verdict format: Your review MUST end with exactly one of these verdict markers (as an HTML comment on its own line): + + + +Then post the review as an issue comment: + gh pr comment ${pr} --body-file ${GANYMEDE_REVIEW_FILE} + +IMPORTANT: Use 'gh pr comment' NOT 'gh pr review'. We use a shared GitHub account so gh pr review --approve fails. +Sign your review as Ganymede (code reviewer). +DO NOT duplicate Leo's knowledge quality checks — he covers those. You cover code. +DO NOT merge — the orchestrator handles merge decisions after all reviews are posted. +Work autonomously. Do not ask for confirmation." + + if run_agent_review "$pr" "ganymede" "$GANYMEDE_PROMPT" "sonnet"; then + GANYMEDE_PASSED=true + else + GANYMEDE_PASSED=false + fi + + # Clean up branch + git checkout main 2>/dev/null || git checkout -f main + [ -n "$PR_BRANCH" ] && git branch -D "$PR_BRANCH" 2>/dev/null || true + elif [ "$IS_CODE_PR" = "true" ] && [ "$LEO_ONLY" = true ]; then + echo " Code files detected but skipping Ganymede review (--leo-only)." + fi + + if [ "$LEO_PASSED" = true ]; then + REVIEWED=$((REVIEWED + 1)) + else + FAILED=$((FAILED + 1)) + fi + + # --- Auto-merge decision --- + if [ "$NO_MERGE" = true ]; then + echo " Auto-merge: skipped (--no-merge)" + elif [ "$LEO_PASSED" != "true" ]; then + echo " Auto-merge: skipped (Leo review failed)" + else + echo "" + echo " --- Merge eligibility check ---" + MERGE_LOG=$(check_merge_eligible "$pr" "$DOMAIN_AGENT" "$LEO_PASSED" "$IS_CODE_PR" "$GANYMEDE_PASSED") + MERGE_RESULT=$? + echo "$MERGE_LOG" | sed 's/^/ /' + + if [ "$MERGE_RESULT" -eq 0 ]; then + echo " Auto-merge: ALL GATES PASSED — merging PR #$pr" + if gh pr merge "$pr" --squash 2>&1; then + echo " PR #$pr: MERGED successfully." + MERGED=$((MERGED + 1)) + else + echo " PR #$pr: Merge FAILED. May need manual intervention." + fi + else + echo " Auto-merge: BLOCKED — see reasons above" + fi + fi + + echo "Finished: $(date)" +done + +echo "" +echo "=== Summary ===" +echo "Reviewed: $REVIEWED" +echo "Failed: $FAILED" +echo "Merged: $MERGED" +echo "Logs: $LOG_DIR" diff --git a/extract-cron.sh b/extract-cron.sh new file mode 100755 index 0000000..a08789d --- /dev/null +++ b/extract-cron.sh @@ -0,0 +1,179 @@ +#!/bin/bash +# Extract claims from unprocessed sources in inbox/archive/ +# Runs via cron on VPS every 15 minutes. +# +# Concurrency model: +# - Lockfile prevents overlapping runs +# - MAX_SOURCES=5 per cycle (works through backlog over multiple runs) +# - Sequential processing (one source at a time) +# - 50 sources landing at once = ~10 cron cycles to clear, not 50 parallel agents +# +# Domain routing: +# - Reads domain: field from source frontmatter +# - Maps to the domain agent (rio, clay, theseus, vida, astra, leo) +# - Runs extraction AS that agent — their territory, their extraction +# - Skips sources with status: processing (agent handling it themselves) +# +# Flow: +# 1. Pull latest main +# 2. Find sources with status: unprocessed (skip processing/processed/null-result) +# 3. For each: run Claude headless to extract claims as the domain agent +# 4. Commit extractions, push, open PR +# 5. Update source status to processed +# +# The eval pipeline (webhook.py) handles review and merge separately. + +set -euo pipefail + +REPO_DIR="/opt/teleo-eval/workspaces/extract" +REPO_URL="http://m3taversal:$(cat /opt/teleo-eval/secrets/forgejo-admin-token)@localhost:3000/teleo/teleo-codex.git" +CLAUDE_BIN="/home/teleo/.local/bin/claude" +LOG_DIR="/opt/teleo-eval/logs" +LOG="$LOG_DIR/extract-cron.log" +LOCKFILE="/tmp/extract-cron.lock" +MAX_SOURCES=5 # Process at most 5 sources per run to limit cost + +log() { echo "[$(date -Iseconds)] $*" >> "$LOG"; } + +# --- Lock --- +if [ -f "$LOCKFILE" ]; then + pid=$(cat "$LOCKFILE" 2>/dev/null) + if kill -0 "$pid" 2>/dev/null; then + log "SKIP: already running (pid $pid)" + exit 0 + fi + log "WARN: stale lockfile, removing" + rm -f "$LOCKFILE" +fi +echo $$ > "$LOCKFILE" +trap 'rm -f "$LOCKFILE"' EXIT + +# --- Ensure repo clone --- +if [ ! -d "$REPO_DIR/.git" ]; then + log "Cloning repo..." + git clone "$REPO_URL" "$REPO_DIR" >> "$LOG" 2>&1 +fi + +cd "$REPO_DIR" + +# --- Pull latest main --- +git checkout main >> "$LOG" 2>&1 +git pull --rebase >> "$LOG" 2>&1 + +# --- Find unprocessed sources --- +UNPROCESSED=$(grep -rl '^status: unprocessed' inbox/archive/ 2>/dev/null | head -n "$MAX_SOURCES" || true) + +if [ -z "$UNPROCESSED" ]; then + log "No unprocessed sources found" + exit 0 +fi + +COUNT=$(echo "$UNPROCESSED" | wc -l | tr -d ' ') +log "Found $COUNT unprocessed source(s)" + +# --- Process each source --- +for SOURCE_FILE in $UNPROCESSED; do + SLUG=$(basename "$SOURCE_FILE" .md) + BRANCH="extract/$SLUG" + + log "Processing: $SOURCE_FILE → branch $BRANCH" + + # Create branch from main + git checkout main >> "$LOG" 2>&1 + git branch -D "$BRANCH" 2>/dev/null || true + git checkout -b "$BRANCH" >> "$LOG" 2>&1 + + # Read domain from frontmatter + DOMAIN=$(grep '^domain:' "$SOURCE_FILE" | head -1 | sed 's/domain: *//' | tr -d '"' | tr -d "'" | xargs) + + # Map domain to agent + case "$DOMAIN" in + internet-finance) AGENT="rio" ;; + entertainment) AGENT="clay" ;; + ai-alignment) AGENT="theseus" ;; + health) AGENT="vida" ;; + space-development) AGENT="astra" ;; + *) AGENT="leo" ;; + esac + + AGENT_TOKEN=$(cat "/opt/teleo-eval/secrets/forgejo-${AGENT}-token" 2>/dev/null || cat /opt/teleo-eval/secrets/forgejo-leo-token) + + log "Domain: $DOMAIN, Agent: $AGENT" + + # Run Claude headless to extract claims + EXTRACT_PROMPT="You are $AGENT, a Teleo knowledge base agent. Extract claims from this source. + +READ these files first: +- skills/extract.md (extraction process) +- schemas/claim.md (claim format) +- $SOURCE_FILE (the source to extract from) + +Then scan domains/$DOMAIN/ to check for duplicate claims. + +EXTRACT claims following the process in skills/extract.md: +1. Read the source completely +2. Separate evidence from interpretation +3. Extract candidate claims (specific, disagreeable, evidence-backed) +4. Check for duplicates against existing claims in domains/$DOMAIN/ +5. Write claim files to domains/$DOMAIN/ with proper YAML frontmatter +6. Update $SOURCE_FILE: set status to 'processed', add processed_by: $AGENT, processed_date: $(date +%Y-%m-%d), and claims_extracted list + +If no claims can be extracted, update $SOURCE_FILE: set status to 'null-result' and add notes explaining why. + +IMPORTANT: Use the Edit tool to update the source file status. Use the Write tool to create new claim files. Do not create claims that duplicate existing ones." + + # Run extraction with timeout (10 minutes) + timeout 600 "$CLAUDE_BIN" -p "$EXTRACT_PROMPT" \ + --allowedTools 'Read,Write,Edit,Glob,Grep' \ + --model sonnet \ + >> "$LOG" 2>&1 || { + log "WARN: Claude extraction failed or timed out for $SOURCE_FILE" + git checkout main >> "$LOG" 2>&1 + continue + } + + # Check if any files were created/modified + CHANGES=$(git status --porcelain | wc -l | tr -d ' ') + if [ "$CHANGES" -eq 0 ]; then + log "No changes produced for $SOURCE_FILE" + git checkout main >> "$LOG" 2>&1 + continue + fi + + # Stage and commit + git add inbox/archive/ "domains/$DOMAIN/" >> "$LOG" 2>&1 + git commit -m "$AGENT: extract claims from $(basename "$SOURCE_FILE") + +- Source: $SOURCE_FILE +- Domain: $DOMAIN +- Extracted by: headless extraction cron + +Pentagon-Agent: $(echo "$AGENT" | sed 's/./\U&/') " >> "$LOG" 2>&1 + + # Push branch + git push -u "$REPO_URL" "$BRANCH" --force >> "$LOG" 2>&1 + + # Open PR + PR_TITLE="$AGENT: extract claims from $(basename "$SOURCE_FILE" .md)" + PR_BODY="## Automated Extraction\n\nSource: \`$SOURCE_FILE\`\nDomain: $DOMAIN\nExtracted by: headless cron on VPS\n\nThis PR was created automatically by the extraction cron job. Claims were extracted using \`skills/extract.md\` process via Claude headless." + + curl -s -X POST "http://localhost:3000/api/v1/repos/teleo/teleo-codex/pulls" \ + -H "Authorization: token $AGENT_TOKEN" \ + -H "Content-Type: application/json" \ + -d "{ + \"title\": \"$PR_TITLE\", + \"body\": \"$PR_BODY\", + \"base\": \"main\", + \"head\": \"$BRANCH\" + }" >> "$LOG" 2>&1 + + log "PR opened for $SOURCE_FILE" + + # Back to main for next source + git checkout main >> "$LOG" 2>&1 + + # Brief pause between extractions + sleep 5 +done + +log "Extraction run complete: processed $COUNT source(s)" diff --git a/extract-graph-data.py b/extract-graph-data.py new file mode 100644 index 0000000..8ffc4f2 --- /dev/null +++ b/extract-graph-data.py @@ -0,0 +1,520 @@ +#!/usr/bin/env python3 +""" +extract-graph-data.py — Extract knowledge graph from teleo-codex markdown files. + +Reads all .md claim/conviction files, parses YAML frontmatter and wiki-links, +and outputs graph-data.json matching the teleo-app GraphData interface. + +Usage: + python3 ops/extract-graph-data.py [--output path/to/graph-data.json] + +Must be run from the teleo-codex repo root. +""" + +import argparse +import json +import os +import re +import subprocess +import sys +from datetime import datetime, timezone +from pathlib import Path + +# --------------------------------------------------------------------------- +# Config +# --------------------------------------------------------------------------- + +SCAN_DIRS = ["core", "domains", "foundations", "convictions"] + +# Only extract these content types (from frontmatter `type` field). +# If type is missing, include the file anyway (many claims lack explicit type). +INCLUDE_TYPES = {"claim", "conviction", "analysis", "belief", "position", None} + +# Domain → default agent mapping (fallback when git attribution unavailable) +DOMAIN_AGENT_MAP = { + "internet-finance": "rio", + "entertainment": "clay", + "health": "vida", + "ai-alignment": "theseus", + "space-development": "astra", + "grand-strategy": "leo", + "mechanisms": "leo", + "living-capital": "leo", + "living-agents": "leo", + "teleohumanity": "leo", + "critical-systems": "leo", + "collective-intelligence": "leo", + "teleological-economics": "leo", + "cultural-dynamics": "clay", +} + +DOMAIN_COLORS = { + "internet-finance": "#4A90D9", + "entertainment": "#9B59B6", + "health": "#2ECC71", + "ai-alignment": "#E74C3C", + "space-development": "#F39C12", + "grand-strategy": "#D4AF37", + "mechanisms": "#1ABC9C", + "living-capital": "#3498DB", + "living-agents": "#E67E22", + "teleohumanity": "#F1C40F", + "critical-systems": "#95A5A6", + "collective-intelligence": "#BDC3C7", + "teleological-economics": "#7F8C8D", + "cultural-dynamics": "#C0392B", +} + +KNOWN_AGENTS = {"leo", "rio", "clay", "vida", "theseus", "astra"} + +# Regex patterns +FRONTMATTER_RE = re.compile(r"^---\s*\n(.*?)\n---", re.DOTALL) +WIKILINK_RE = re.compile(r"\[\[([^\]]+)\]\]") +YAML_FIELD_RE = re.compile(r"^(\w[\w_]*):\s*(.+)$", re.MULTILINE) +YAML_LIST_ITEM_RE = re.compile(r'^\s*-\s+"?(.+?)"?\s*$', re.MULTILINE) +COUNTER_EVIDENCE_RE = re.compile(r"^##\s+Counter[\s-]?evidence", re.MULTILINE | re.IGNORECASE) +COUNTERARGUMENT_RE = re.compile(r"^\*\*Counter\s*argument", re.MULTILINE | re.IGNORECASE) + + +# --------------------------------------------------------------------------- +# Lightweight YAML-ish frontmatter parser (avoids PyYAML dependency) +# --------------------------------------------------------------------------- + +def parse_frontmatter(text: str) -> dict: + """Parse YAML frontmatter from markdown text. Returns dict of fields.""" + m = FRONTMATTER_RE.match(text) + if not m: + return {} + yaml_block = m.group(1) + result = {} + for field_match in YAML_FIELD_RE.finditer(yaml_block): + key = field_match.group(1) + val = field_match.group(2).strip().strip('"').strip("'") + # Handle list fields + if val.startswith("["): + # Inline YAML list: [item1, item2] + items = re.findall(r'"([^"]+)"', val) + if not items: + items = [x.strip().strip('"').strip("'") + for x in val.strip("[]").split(",") if x.strip()] + result[key] = items + else: + result[key] = val + # Handle multi-line list fields (depends_on, challenged_by, secondary_domains) + for list_key in ("depends_on", "challenged_by", "secondary_domains", "claims_extracted"): + if list_key not in result: + # Check for block-style list + pattern = re.compile( + rf"^{list_key}:\s*\n((?:\s+-\s+.+\n?)+)", re.MULTILINE + ) + lm = pattern.search(yaml_block) + if lm: + items = YAML_LIST_ITEM_RE.findall(lm.group(1)) + result[list_key] = [i.strip('"').strip("'") for i in items] + return result + + +def extract_body(text: str) -> str: + """Return the markdown body after frontmatter.""" + m = FRONTMATTER_RE.match(text) + if m: + return text[m.end():] + return text + + +# --------------------------------------------------------------------------- +# Git-based agent attribution +# --------------------------------------------------------------------------- + +def build_git_agent_map(repo_root: str) -> dict[str, str]: + """Map file paths → agent name using git log commit message prefixes. + + Commit messages follow: '{agent}: description' + We use the commit that first added each file. + """ + file_agent = {} + try: + result = subprocess.run( + ["git", "log", "--all", "--diff-filter=A", "--name-only", + "--format=COMMIT_MSG:%s"], + capture_output=True, text=True, cwd=repo_root, timeout=30, + ) + current_agent = None + for line in result.stdout.splitlines(): + line = line.strip() + if not line: + continue + if line.startswith("COMMIT_MSG:"): + msg = line[len("COMMIT_MSG:"):] + # Parse "agent: description" pattern + if ":" in msg: + prefix = msg.split(":")[0].strip().lower() + if prefix in KNOWN_AGENTS: + current_agent = prefix + else: + current_agent = None + else: + current_agent = None + elif current_agent and line.endswith(".md"): + # Only set if not already attributed (first add wins) + if line not in file_agent: + file_agent[line] = current_agent + except (subprocess.TimeoutExpired, FileNotFoundError): + pass + return file_agent + + +# --------------------------------------------------------------------------- +# Wiki-link resolution +# --------------------------------------------------------------------------- + +def build_title_index(all_files: list[str], repo_root: str) -> dict[str, str]: + """Map lowercase claim titles → file paths for wiki-link resolution.""" + index = {} + for fpath in all_files: + # Title = filename without .md extension + fname = os.path.basename(fpath) + if fname.endswith(".md"): + title = fname[:-3].lower() + index[title] = fpath + # Also index by relative path + index[fpath.lower()] = fpath + return index + + +def resolve_wikilink(link_text: str, title_index: dict, source_dir: str) -> str | None: + """Resolve a [[wiki-link]] target to a file path (node ID).""" + text = link_text.strip() + # Skip map links and non-claim references + if text.startswith("_") or text == "_map": + return None + # Direct path match (with or without .md) + for candidate in [text, text + ".md"]: + if candidate.lower() in title_index: + return title_index[candidate.lower()] + # Title-only match + title = text.lower() + if title in title_index: + return title_index[title] + # Fuzzy: try adding .md to the basename + basename = os.path.basename(text) + if basename.lower() in title_index: + return title_index[basename.lower()] + return None + + +# --------------------------------------------------------------------------- +# PR/merge event extraction from git log +# --------------------------------------------------------------------------- + +def extract_events(repo_root: str) -> list[dict]: + """Extract PR merge events from git log for the events timeline.""" + events = [] + try: + result = subprocess.run( + ["git", "log", "--merges", "--format=%H|%s|%ai", "-50"], + capture_output=True, text=True, cwd=repo_root, timeout=15, + ) + for line in result.stdout.strip().splitlines(): + parts = line.split("|", 2) + if len(parts) < 3: + continue + sha, msg, date_str = parts + # Parse "Merge pull request #N from ..." or agent commit patterns + pr_match = re.search(r"#(\d+)", msg) + if not pr_match: + continue + pr_num = int(pr_match.group(1)) + # Try to determine agent from merge commit + agent = "collective" + for a in KNOWN_AGENTS: + if a in msg.lower(): + agent = a + break + # Count files changed in this merge + diff_result = subprocess.run( + ["git", "diff", "--name-only", f"{sha}^..{sha}"], + capture_output=True, text=True, cwd=repo_root, timeout=10, + ) + claims_added = sum( + 1 for f in diff_result.stdout.splitlines() + if f.endswith(".md") and any(f.startswith(d) for d in SCAN_DIRS) + ) + if claims_added > 0: + events.append({ + "type": "pr-merge", + "number": pr_num, + "agent": agent, + "claims_added": claims_added, + "date": date_str[:10], + }) + except (subprocess.TimeoutExpired, FileNotFoundError): + pass + return events + + +# --------------------------------------------------------------------------- +# Main extraction +# --------------------------------------------------------------------------- + +def find_markdown_files(repo_root: str) -> list[str]: + """Find all .md files in SCAN_DIRS, return relative paths.""" + files = [] + for scan_dir in SCAN_DIRS: + dirpath = os.path.join(repo_root, scan_dir) + if not os.path.isdir(dirpath): + continue + for root, _dirs, filenames in os.walk(dirpath): + for fname in filenames: + if fname.endswith(".md") and not fname.startswith("_"): + rel = os.path.relpath(os.path.join(root, fname), repo_root) + files.append(rel) + return sorted(files) + + +def _get_domain_cached(fpath: str, repo_root: str, cache: dict) -> str: + """Get the domain of a file, caching results.""" + if fpath in cache: + return cache[fpath] + abs_path = os.path.join(repo_root, fpath) + domain = "" + try: + text = open(abs_path, encoding="utf-8").read() + fm = parse_frontmatter(text) + domain = fm.get("domain", "") + except (OSError, UnicodeDecodeError): + pass + cache[fpath] = domain + return domain + + +def extract_graph(repo_root: str) -> dict: + """Extract the full knowledge graph from the codex.""" + all_files = find_markdown_files(repo_root) + git_agents = build_git_agent_map(repo_root) + title_index = build_title_index(all_files, repo_root) + domain_cache: dict[str, str] = {} + + nodes = [] + edges = [] + node_ids = set() + all_files_set = set(all_files) + + for fpath in all_files: + abs_path = os.path.join(repo_root, fpath) + try: + text = open(abs_path, encoding="utf-8").read() + except (OSError, UnicodeDecodeError): + continue + + fm = parse_frontmatter(text) + body = extract_body(text) + + # Filter by type + ftype = fm.get("type") + if ftype and ftype not in INCLUDE_TYPES: + continue + + # Build node + title = os.path.basename(fpath)[:-3] # filename without .md + domain = fm.get("domain", "") + if not domain: + # Infer domain from directory path + parts = fpath.split(os.sep) + if len(parts) >= 2: + domain = parts[1] if parts[0] == "domains" else parts[1] if len(parts) > 2 else parts[0] + + # Agent attribution: git log → domain mapping → "collective" + agent = git_agents.get(fpath, "") + if not agent: + agent = DOMAIN_AGENT_MAP.get(domain, "collective") + + created = fm.get("created", "") + confidence = fm.get("confidence", "speculative") + + # Detect challenged status + challenged_by_raw = fm.get("challenged_by", []) + if isinstance(challenged_by_raw, str): + challenged_by_raw = [challenged_by_raw] if challenged_by_raw else [] + has_challenged_by = bool(challenged_by_raw and any(c for c in challenged_by_raw)) + has_counter_section = bool(COUNTER_EVIDENCE_RE.search(body) or COUNTERARGUMENT_RE.search(body)) + is_challenged = has_challenged_by or has_counter_section + + # Extract challenge descriptions for the node + challenges = [] + if isinstance(challenged_by_raw, list): + for c in challenged_by_raw: + if c and isinstance(c, str): + # Strip wiki-link syntax for display + cleaned = WIKILINK_RE.sub(lambda m: m.group(1), c) + # Strip markdown list artifacts: leading "- ", surrounding quotes + cleaned = re.sub(r'^-\s*', '', cleaned).strip() + cleaned = cleaned.strip('"').strip("'").strip() + if cleaned: + challenges.append(cleaned[:200]) # cap length + + node = { + "id": fpath, + "title": title, + "domain": domain, + "agent": agent, + "created": created, + "confidence": confidence, + "challenged": is_challenged, + } + if challenges: + node["challenges"] = challenges + nodes.append(node) + node_ids.add(fpath) + domain_cache[fpath] = domain # cache for edge lookups + for link_text in WIKILINK_RE.findall(body): + target = resolve_wikilink(link_text, title_index, os.path.dirname(fpath)) + if target and target != fpath and target in all_files_set: + target_domain = _get_domain_cached(target, repo_root, domain_cache) + edges.append({ + "source": fpath, + "target": target, + "type": "wiki-link", + "cross_domain": domain != target_domain and bool(target_domain), + }) + + # Conflict edges from challenged_by (may contain [[wiki-links]] or prose) + challenged_by = fm.get("challenged_by", []) + if isinstance(challenged_by, str): + challenged_by = [challenged_by] + if isinstance(challenged_by, list): + for challenge in challenged_by: + if not challenge: + continue + # Check for embedded wiki-links + for link_text in WIKILINK_RE.findall(challenge): + target = resolve_wikilink(link_text, title_index, os.path.dirname(fpath)) + if target and target != fpath and target in all_files_set: + target_domain = _get_domain_cached(target, repo_root, domain_cache) + edges.append({ + "source": fpath, + "target": target, + "type": "conflict", + "cross_domain": domain != target_domain and bool(target_domain), + }) + + # Deduplicate edges + seen_edges = set() + unique_edges = [] + for e in edges: + key = (e["source"], e["target"], e.get("type", "")) + if key not in seen_edges: + seen_edges.add(key) + unique_edges.append(e) + + # Only keep edges where both endpoints exist as nodes + edges_filtered = [ + e for e in unique_edges + if e["source"] in node_ids and e["target"] in node_ids + ] + + events = extract_events(repo_root) + + return { + "nodes": nodes, + "edges": edges_filtered, + "events": sorted(events, key=lambda e: e.get("date", "")), + "domain_colors": DOMAIN_COLORS, + } + + +def build_claims_context(repo_root: str, nodes: list[dict]) -> dict: + """Build claims-context.json for chat system prompt injection. + + Produces a lightweight claim index: title + description + domain + agent + confidence. + Sorted by domain, then alphabetically within domain. + Target: ~37KB for ~370 claims. Truncates descriptions at 100 chars if total > 100KB. + """ + claims = [] + for node in nodes: + fpath = node["id"] + abs_path = os.path.join(repo_root, fpath) + description = "" + try: + text = open(abs_path, encoding="utf-8").read() + fm = parse_frontmatter(text) + description = fm.get("description", "") + except (OSError, UnicodeDecodeError): + pass + + claims.append({ + "title": node["title"], + "description": description, + "domain": node["domain"], + "agent": node["agent"], + "confidence": node["confidence"], + }) + + # Sort by domain, then title + claims.sort(key=lambda c: (c["domain"], c["title"])) + + context = { + "generated": datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + "claimCount": len(claims), + "claims": claims, + } + + # Progressive description truncation if over 100KB. + # Never drop descriptions entirely — short descriptions are better than none. + for max_desc in (120, 100, 80, 60): + test_json = json.dumps(context, ensure_ascii=False) + if len(test_json) <= 100_000: + break + for c in claims: + if len(c["description"]) > max_desc: + c["description"] = c["description"][:max_desc] + "..." + + return context + + +def main(): + parser = argparse.ArgumentParser(description="Extract graph data from teleo-codex") + parser.add_argument("--output", "-o", default="graph-data.json", + help="Output file path (default: graph-data.json)") + parser.add_argument("--context-output", "-c", default=None, + help="Output claims-context.json path (default: same dir as --output)") + parser.add_argument("--repo", "-r", default=".", + help="Path to teleo-codex repo root (default: current dir)") + args = parser.parse_args() + + repo_root = os.path.abspath(args.repo) + if not os.path.isdir(os.path.join(repo_root, "core")): + print(f"Error: {repo_root} doesn't look like a teleo-codex repo (no core/ dir)", file=sys.stderr) + sys.exit(1) + + print(f"Scanning {repo_root}...") + graph = extract_graph(repo_root) + + print(f" Nodes: {len(graph['nodes'])}") + print(f" Edges: {len(graph['edges'])}") + print(f" Events: {len(graph['events'])}") + challenged_count = sum(1 for n in graph["nodes"] if n.get("challenged")) + print(f" Challenged: {challenged_count}") + + # Write graph-data.json + output_path = os.path.abspath(args.output) + with open(output_path, "w", encoding="utf-8") as f: + json.dump(graph, f, indent=2, ensure_ascii=False) + size_kb = os.path.getsize(output_path) / 1024 + print(f" graph-data.json: {output_path} ({size_kb:.1f} KB)") + + # Write claims-context.json + context_path = args.context_output + if not context_path: + context_path = os.path.join(os.path.dirname(output_path), "claims-context.json") + context_path = os.path.abspath(context_path) + + context = build_claims_context(repo_root, graph["nodes"]) + with open(context_path, "w", encoding="utf-8") as f: + json.dump(context, f, indent=2, ensure_ascii=False) + ctx_kb = os.path.getsize(context_path) / 1024 + print(f" claims-context.json: {context_path} ({ctx_kb:.1f} KB)") + + +if __name__ == "__main__": + main() diff --git a/fix-ownership.sh b/fix-ownership.sh new file mode 100755 index 0000000..fb047d4 --- /dev/null +++ b/fix-ownership.sh @@ -0,0 +1,10 @@ +#!/bin/bash +# Fix root-owned files before pipeline starts (3rd incident — Rhea, Epimetheus) +# Any git op running as root poisons ownership. This catches it at startup. +find /opt/teleo-eval/workspaces -not -user teleo -exec chown teleo:teleo {} + 2>/dev/null +find /opt/teleo-eval/pipeline -not -user teleo -exec chown teleo:teleo {} + 2>/dev/null +find /opt/teleo-eval/entity-queue -not -user teleo -exec chown teleo:teleo {} + 2>/dev/null +find /opt/teleo-eval/logs -not -user teleo -exec chown teleo:teleo {} + 2>/dev/null +find /opt/teleo-eval/transcripts -not -user teleo -exec chown teleo:teleo {} + 2>/dev/null +find /opt/teleo-eval/telegram-archives -not -user teleo -exec chown teleo:teleo {} + 2>/dev/null +chown teleo:teleo /opt/teleo-eval/workspaces/.main-worktree.lock 2>/dev/null || true diff --git a/hermes-agent/GMAIL-SETUP.md b/hermes-agent/GMAIL-SETUP.md new file mode 100644 index 0000000..b2f5ca8 --- /dev/null +++ b/hermes-agent/GMAIL-SETUP.md @@ -0,0 +1,52 @@ +# Gmail Setup for Hermes Agent + +## Step 1: Create Google Cloud OAuth Credentials (~5 min) + +1. Go to [console.cloud.google.com](https://console.cloud.google.com) +2. Create a new project (or use existing): "Hermes Assistant" +3. Enable these APIs: + - Gmail API + - Google Calendar API + - Google Drive API (optional) +4. Go to **APIs & Services → Credentials → Create Credentials → OAuth 2.0 Client ID** +5. Application type: **Desktop app** +6. Name: "Hermes Agent" +7. Download the JSON file → save as `~/.hermes/google-credentials.json` + +## Step 2: Configure Hermes + +Add to `~/.hermes/.env`: + +``` +GOOGLE_CLIENT_ID=your-client-id.apps.googleusercontent.com +GOOGLE_CLIENT_SECRET=your-client-secret +``` + +Or place the downloaded JSON at `~/.hermes/google-credentials.json`. + +## Step 3: Authorize + +```bash +hermes setup google-workspace +``` + +This opens a browser auth flow (or gives you a URL to paste). Sign in with +m3taversal@gmail.com and grant permissions. Token is saved locally. + +Since this is a VPS (no browser), you'll get a URL — open it on your laptop, +authorize, paste the code back into the terminal. + +## Step 4: Test + +```bash +hermes "Show me my last 5 emails" +hermes "What's on my calendar today?" +hermes "Draft a reply to the last email from [name]" +``` + +## Security Notes + +- OAuth tokens stored locally in `~/.hermes/` (chmod 600) +- Hermes only accesses what you authorized — revoke anytime at + [myaccount.google.com/permissions](https://myaccount.google.com/permissions) +- The VPS is SSH-only access, no public web ports exposed to Hermes diff --git a/hermes-agent/install-hermes.sh b/hermes-agent/install-hermes.sh new file mode 100644 index 0000000..8c30636 --- /dev/null +++ b/hermes-agent/install-hermes.sh @@ -0,0 +1,113 @@ +#!/usr/bin/env bash +# Install Hermes Agent on Teleo VPS (CAX31, ARM64, Ubuntu) +# Run as: teleo user +# Prereqs: Python 3.11+, Node.js 22+, git +set -euo pipefail + +HERMES_HOME="$HOME/.hermes" +OPENROUTER_KEY_FILE="/opt/teleo-eval/secrets/openrouter-key" + +echo "=== Hermes Agent Install for Teleo VPS ===" + +# 1. Check prereqs +echo "[1/6] Checking prerequisites..." +python3 --version || { echo "ERROR: Python 3.11+ required"; exit 1; } +node --version || { echo "ERROR: Node.js 22+ required"; exit 1; } +git --version || { echo "ERROR: git required"; exit 1; } + +# 2. Install Hermes +echo "[2/6] Installing Hermes Agent..." +if command -v hermes &>/dev/null; then + echo "Hermes already installed, upgrading..." + pip3 install --upgrade hermes-agent +else + curl -fsSL https://raw.githubusercontent.com/NousResearch/hermes-agent/main/scripts/install.sh | bash + # Source the updated PATH + export PATH="$HOME/.local/bin:$PATH" +fi + +# 3. Create config directory +echo "[3/6] Setting up config..." +mkdir -p "$HERMES_HOME" + +# 4. Write .env with OpenRouter key (read from existing pipeline secret) +if [ -f "$OPENROUTER_KEY_FILE" ]; then + OPENROUTER_KEY=$(cat "$OPENROUTER_KEY_FILE") + cat > "$HERMES_HOME/.env" << EOF +OPENROUTER_API_KEY=${OPENROUTER_KEY} +EOF + chmod 600 "$HERMES_HOME/.env" + echo " OpenRouter key loaded from pipeline secrets" +else + echo " WARNING: No OpenRouter key found at $OPENROUTER_KEY_FILE" + echo " You'll need to manually add OPENROUTER_API_KEY to $HERMES_HOME/.env" +fi + +# 5. Write config.yaml +echo "[4/6] Writing config.yaml..." +cat > "$HERMES_HOME/config.yaml" << 'EOF' +# Hermes Agent config — Teleo VPS +model: + provider: openrouter + default: anthropic/claude-sonnet-4-6 + smart_routing: true + smart_routing_model: google/gemini-2.5-flash + +terminal: + backend: native + +memory: + enabled: true + search: sqlite_fts5 + +tools: + web_search: true + browser: true + file_ops: true + terminal: true + vision: false + image_gen: false + tts: false + +gateway: + telegram: + enabled: false # Enable after setting BOT_TOKEN below + # bot_token: "YOUR_TELEGRAM_BOT_TOKEN" +EOF + +# 6. Write SOUL.md +echo "[5/6] Writing SOUL.md..." +cat > "$HERMES_HOME/SOUL.md" << 'EOF' +You are Cory's personal AI assistant running on the Teleo VPS. + +Your owner is Cory Abdalla — founder of Metaversal, building LivingIP +(a collective intelligence system for investment research). + +You help with: +- Email triage and drafting (when Gmail is connected) +- Calendar management +- Web research and summarization +- Quick tasks and reminders +- Anything Cory asks + +Style: Direct, concise, no fluff. Cory is technical — skip explanations +of basic concepts. When uncertain, say so rather than guessing. + +You are NOT part of the LivingIP pipeline. You're a separate personal +assistant. Don't try to interact with Forgejo, pipeline.db, or the +teleo-codex unless Cory specifically asks. +EOF + +echo "[6/6] Done!" +echo "" +echo "=== Next Steps ===" +echo "1. Test: hermes 'hello, what model are you using?'" +echo "2. Gmail: hermes setup google-workspace (needs OAuth credentials)" +echo "3. Telegram: Create bot via @BotFather, add token to config.yaml," +echo " then: hermes gateway start" +echo "4. Cron: hermes cron add '0 8 * * *' 'Check my calendar and summarize today'" +echo "" +echo "Config: $HERMES_HOME/config.yaml" +echo "Memory: $HERMES_HOME/MEMORY.md" +echo "Skills: $HERMES_HOME/skills/" +EOF diff --git a/lib/cascade.py b/lib/cascade.py new file mode 100644 index 0000000..1f8241f --- /dev/null +++ b/lib/cascade.py @@ -0,0 +1,282 @@ +"""Cascade automation — auto-flag dependent beliefs/positions when claims change. + +Hook point: called from merge.py after _embed_merged_claims, before _delete_remote_branch. +Uses the same main_sha/branch_sha diff to detect changed claim files, then scans +all agent beliefs and positions for depends_on references to those claims. + +Notifications are written to /opt/teleo-eval/agent-state/{agent}/inbox/ using +the same atomic-write pattern as lib-state.sh. +""" + +import asyncio +import hashlib +import json +import logging +import os +import re +import tempfile +from datetime import datetime, timezone +from pathlib import Path + +logger = logging.getLogger("pipeline.cascade") + +AGENT_STATE_DIR = Path("/opt/teleo-eval/agent-state") +CLAIM_DIRS = {"domains/", "core/", "foundations/", "decisions/"} +AGENT_NAMES = ["rio", "leo", "clay", "astra", "vida", "theseus"] + + +def _extract_claim_titles_from_diff(diff_files: list[str]) -> set[str]: + """Extract claim titles from changed file paths.""" + titles = set() + for fpath in diff_files: + if not fpath.endswith(".md"): + continue + if not any(fpath.startswith(d) for d in CLAIM_DIRS): + continue + basename = os.path.basename(fpath) + if basename.startswith("_") or basename == "directory.md": + continue + title = basename.removesuffix(".md") + titles.add(title) + return titles + + +def _normalize_for_match(text: str) -> str: + """Normalize for fuzzy matching: lowercase, hyphens to spaces, strip punctuation, collapse whitespace.""" + text = text.lower().strip() + text = text.replace("-", " ") + text = re.sub(r"[^\w\s]", "", text) + text = re.sub(r"\s+", " ", text) + return text + + +def _slug_to_words(slug: str) -> str: + """Convert kebab-case slug to space-separated words.""" + return slug.replace("-", " ") + + +def _parse_depends_on(file_path: Path) -> tuple[str, list[str]]: + """Parse a belief or position file's depends_on entries. + + Returns (agent_name, [dependency_titles]). + """ + try: + content = file_path.read_text(encoding="utf-8") + except (OSError, UnicodeDecodeError): + return ("", []) + + agent = "" + deps = [] + in_frontmatter = False + in_depends = False + + for line in content.split("\n"): + if line.strip() == "---": + if not in_frontmatter: + in_frontmatter = True + continue + else: + break + + if in_frontmatter: + if line.startswith("agent:"): + agent = line.split(":", 1)[1].strip().strip('"').strip("'") + elif line.startswith("depends_on:"): + in_depends = True + rest = line.split(":", 1)[1].strip() + if rest.startswith("["): + items = re.findall(r'"([^"]+)"|\'([^\']+)\'', rest) + for item in items: + dep = item[0] or item[1] + dep = dep.strip("[]").replace("[[", "").replace("]]", "") + deps.append(dep) + in_depends = False + elif in_depends: + if line.startswith(" - "): + dep = line.strip().lstrip("- ").strip('"').strip("'") + dep = dep.replace("[[", "").replace("]]", "") + deps.append(dep) + elif line.strip() and not line.startswith(" "): + in_depends = False + + # Also scan body for [[wiki-links]] + body_links = re.findall(r"\[\[([^\]]+)\]\]", content) + for link in body_links: + if link not in deps: + deps.append(link) + + return (agent, deps) + + +def _write_inbox_message(agent: str, subject: str, body: str) -> bool: + """Write a cascade notification to an agent's inbox. Atomic tmp+rename.""" + inbox_dir = AGENT_STATE_DIR / agent / "inbox" + if not inbox_dir.exists(): + logger.warning("cascade: no inbox dir for agent %s, skipping", agent) + return False + + ts = datetime.now(timezone.utc).strftime("%Y%m%d-%H%M%S") + file_hash = hashlib.md5(f"{agent}-{subject}-{body[:200]}".encode()).hexdigest()[:8] + filename = f"cascade-{ts}-{subject[:60]}-{file_hash}.md" + final_path = inbox_dir / filename + + try: + fd, tmp_path = tempfile.mkstemp(dir=str(inbox_dir), suffix=".tmp") + with os.fdopen(fd, "w") as f: + f.write(f"---\n") + f.write(f"type: cascade\n") + f.write(f"from: pipeline\n") + f.write(f"to: {agent}\n") + f.write(f"subject: \"{subject}\"\n") + f.write(f"created: {datetime.now(timezone.utc).isoformat()}\n") + f.write(f"status: unread\n") + f.write(f"---\n\n") + f.write(body) + os.rename(tmp_path, str(final_path)) + return True + except OSError: + logger.exception("cascade: failed to write inbox message for %s", agent) + return False + + +def _find_matches(deps: list[str], claim_lookup: dict[str, str]) -> list[str]: + """Check if any dependency matches a changed claim. + + Uses exact normalized match first, then substring containment for longer + strings only (min 15 chars) to avoid false positives on short generic names. + """ + matched = [] + for dep in deps: + norm = _normalize_for_match(dep) + if norm in claim_lookup: + matched.append(claim_lookup[norm]) + else: + # Substring match only for sufficiently specific strings + shorter = min(len(norm), min((len(k) for k in claim_lookup), default=0)) + if shorter >= 15: + for claim_norm, claim_orig in claim_lookup.items(): + if claim_norm in norm or norm in claim_norm: + matched.append(claim_orig) + break + return matched + + +def _format_cascade_body( + file_name: str, + file_type: str, + matched_claims: list[str], + pr_num: int, +) -> str: + """Format the cascade notification body.""" + claims_list = "\n".join(f"- {c}" for c in matched_claims) + return ( + f"# Cascade: upstream claims changed\n\n" + f"Your {file_type} **{file_name}** depends on claims that were modified in PR #{pr_num}.\n\n" + f"## Changed claims\n\n{claims_list}\n\n" + f"## Action needed\n\n" + f"Review whether your {file_type}'s confidence, description, or grounding " + f"needs updating in light of these changes. If the evidence strengthened, " + f"consider increasing confidence. If it weakened or contradicted, flag for " + f"re-evaluation.\n" + ) + + +async def cascade_after_merge( + main_sha: str, + branch_sha: str, + pr_num: int, + main_worktree: Path, + conn=None, +) -> int: + """Scan for beliefs/positions affected by claims changed in this merge. + + Returns the number of cascade notifications sent. + """ + # 1. Get changed files + proc = await asyncio.create_subprocess_exec( + "git", "diff", "--name-only", "--diff-filter=ACMR", + main_sha, branch_sha, + cwd=str(main_worktree), + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + try: + stdout, _ = await asyncio.wait_for(proc.communicate(), timeout=10) + except asyncio.TimeoutError: + proc.kill() + await proc.wait() + logger.warning("cascade: git diff timed out") + return 0 + + if proc.returncode != 0: + logger.warning("cascade: git diff failed (rc=%d)", proc.returncode) + return 0 + + diff_files = [f for f in stdout.decode().strip().split("\n") if f] + + # 2. Extract claim titles from changed files + changed_claims = _extract_claim_titles_from_diff(diff_files) + if not changed_claims: + return 0 + + logger.info("cascade: %d claims changed in PR #%d: %s", + len(changed_claims), pr_num, list(changed_claims)[:5]) + + # Build normalized lookup for fuzzy matching + claim_lookup = {} + for claim in changed_claims: + claim_lookup[_normalize_for_match(claim)] = claim + claim_lookup[_normalize_for_match(_slug_to_words(claim))] = claim + + # 3. Scan all beliefs and positions + notifications = 0 + notification_details = [] # Per-agent reasoning for audit trail + agents_dir = main_worktree / "agents" + if not agents_dir.exists(): + logger.warning("cascade: no agents/ dir in worktree") + return 0 + + for agent_name in AGENT_NAMES: + agent_dir = agents_dir / agent_name + if not agent_dir.exists(): + continue + + for subdir, file_type in [("beliefs", "belief"), ("positions", "position")]: + target_dir = agent_dir / subdir + if not target_dir.exists(): + continue + for md_file in target_dir.glob("*.md"): + _, deps = _parse_depends_on(md_file) + matched = _find_matches(deps, claim_lookup) + if matched: + body = _format_cascade_body(md_file.name, file_type, matched, pr_num) + if _write_inbox_message(agent_name, f"claim-changed-affects-{file_type}", body): + notifications += 1 + notification_details.append({ + "agent": agent_name, + "file_type": file_type, + "file": md_file.stem, + "matched_claims": matched, + }) + logger.info("cascade: notified %s — %s '%s' affected by %s", + agent_name, file_type, md_file.stem, matched) + + if notifications: + logger.info("cascade: sent %d notifications for PR #%d", notifications, pr_num) + + # Write structured audit_log entry for cascade tracking (Page 4 data) + if conn is not None: + try: + conn.execute( + "INSERT INTO audit_log (stage, event, detail) VALUES (?, ?, ?)", + ("cascade", "cascade_triggered", json.dumps({ + "pr": pr_num, + "claims_changed": list(changed_claims)[:20], + "notifications_sent": notifications, + "details": notification_details[:50], + })), + ) + except Exception: + logger.exception("cascade: audit_log write failed (non-fatal)") + + return notifications diff --git a/lib/config.py b/lib/config.py index 892df79..87b6485 100644 --- a/lib/config.py +++ b/lib/config.py @@ -200,6 +200,12 @@ MERGE_INTERVAL = 30 FIX_INTERVAL = 60 HEALTH_CHECK_INTERVAL = 60 +# --- Retrieval (Telegram bot) --- +RETRIEVAL_RRF_K = 20 # RRF smoothing constant — tuned for 5-10 results per source +RETRIEVAL_ENTITY_BOOST = 1.5 # RRF score multiplier for claims wiki-linked from matched entities +RETRIEVAL_MAX_RESULTS = 10 # Max claims shown to LLM after RRF merge +RETRIEVAL_MIN_CLAIM_SCORE = 3.0 # Floor for keyword claim scoring — filters single-stopword matches + # --- Health API --- HEALTH_PORT = 8080 diff --git a/lib/connect.py b/lib/connect.py index a8444c8..159152e 100644 --- a/lib/connect.py +++ b/lib/connect.py @@ -24,9 +24,9 @@ from pathlib import Path logger = logging.getLogger("pipeline.connect") -# Similarity threshold for auto-connecting (lower than reweave's 0.70 because -# we're using "related" not "supports/challenges" — less precision needed) -CONNECT_THRESHOLD = 0.55 +# Similarity threshold for auto-connecting — below reweave's 0.70 but above +# the noise floor (~0.55). "related" still means actually related, not vaguely topical. +CONNECT_THRESHOLD = 0.65 CONNECT_MAX_NEIGHBORS = 5 # --- Import search functions --- diff --git a/lib/costs.py b/lib/costs.py index 5a0cc37..63050cf 100644 --- a/lib/costs.py +++ b/lib/costs.py @@ -15,34 +15,55 @@ def record_usage( input_tokens: int = 0, output_tokens: int = 0, backend: str = "api", + duration_ms: int = 0, + cache_read_tokens: int = 0, + cache_write_tokens: int = 0, + cost_estimate_usd: float = 0.0, ): """Record usage and compute cost. Returns cost in USD. backend: "max" (Claude Max subscription, free) or "api" (paid). Claude Max calls are tracked for volume metrics but cost $0. (Ganymede) """ - if backend == "max": - cost = 0.0 + # Always compute estimated cost from tokens × published rates + rates = config.MODEL_COSTS.get(model) + if rates and (input_tokens or output_tokens): + estimated = (input_tokens * rates["input"] + output_tokens * rates["output"]) / 1000 + # Cache reads are ~90% cheaper than regular input + if cache_read_tokens and rates: + estimated += (cache_read_tokens * rates["input"] * 0.1) / 1000 + if cache_write_tokens and rates: + estimated += (cache_write_tokens * rates["input"] * 1.25) / 1000 else: - rates = config.MODEL_COSTS.get(model) - if not rates: - logger.warning("No cost rates for model %s, recording zero cost", model) - cost = 0.0 - else: - cost = (input_tokens * rates["input"] + output_tokens * rates["output"]) / 1000 + estimated = 0.0 + # Use caller-provided estimate if we can't compute (e.g. CLI gives its own) + if cost_estimate_usd > 0 and estimated == 0: + estimated = cost_estimate_usd + cost_estimate_usd = estimated + + if backend == "max": + cost = 0.0 # subscription — no actual spend + else: + cost = estimated if estimated > 0 else 0.0 today = date.today().isoformat() # Include backend in the stage key so max vs api are tracked separately stage_key = f"{stage}:{backend}" if backend != "api" else stage conn.execute( - """INSERT INTO costs (date, model, stage, calls, input_tokens, output_tokens, cost_usd) - VALUES (?, ?, ?, 1, ?, ?, ?) + """INSERT INTO costs (date, model, stage, calls, input_tokens, output_tokens, cost_usd, + duration_ms, cache_read_tokens, cache_write_tokens, cost_estimate_usd) + VALUES (?, ?, ?, 1, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (date, model, stage) DO UPDATE SET calls = calls + 1, input_tokens = input_tokens + excluded.input_tokens, output_tokens = output_tokens + excluded.output_tokens, - cost_usd = cost_usd + excluded.cost_usd""", - (today, model, stage_key, input_tokens, output_tokens, cost), + cost_usd = cost_usd + excluded.cost_usd, + duration_ms = duration_ms + excluded.duration_ms, + cache_read_tokens = cache_read_tokens + excluded.cache_read_tokens, + cache_write_tokens = cache_write_tokens + excluded.cache_write_tokens, + cost_estimate_usd = cost_estimate_usd + excluded.cost_estimate_usd""", + (today, model, stage_key, input_tokens, output_tokens, cost, + duration_ms, cache_read_tokens, cache_write_tokens, cost_estimate_usd), ) return cost @@ -63,7 +84,8 @@ def get_daily_breakdown(conn, day: str = None) -> list: if day is None: day = date.today().isoformat() rows = conn.execute( - """SELECT model, stage, calls, input_tokens, output_tokens, cost_usd + """SELECT model, stage, calls, input_tokens, output_tokens, cost_usd, + duration_ms, cache_read_tokens, cache_write_tokens, cost_estimate_usd FROM costs WHERE date = ? ORDER BY cost_usd DESC""", (day,), ).fetchall() diff --git a/lib/cross_domain.py b/lib/cross_domain.py new file mode 100644 index 0000000..9f22b1a --- /dev/null +++ b/lib/cross_domain.py @@ -0,0 +1,230 @@ +"""Cross-domain citation index — detect entity overlap across domains. + +Hook point: called from merge.py after cascade_after_merge. +After a claim merges, checks if its referenced entities also appear in claims +from other domains. Logs connections to audit_log for silo detection. + +Two detection methods: +1. Entity name matching — entity names appearing in claim body text (word-boundary) +2. Source overlap — claims citing the same source archive files + +At ~600 claims and ~100 entities, full scan per merge takes <1 second. +""" + +import asyncio +import json +import logging +import os +import re +from pathlib import Path + +logger = logging.getLogger("pipeline.cross_domain") + +# Minimum entity name length to avoid false positives (ORE, QCX, etc) +MIN_ENTITY_NAME_LEN = 4 + +# Entity names that are common English words — skip to avoid false positives +ENTITY_STOPLIST = {"versus", "island", "loyal", "saber", "nebula", "helium", "coal", "snapshot", "dropout"} + + +def _build_entity_names(worktree: Path) -> dict[str, str]: + """Build mapping of entity_slug -> display_name from entity files.""" + names = {} + entity_dir = worktree / "entities" + if not entity_dir.exists(): + return names + for md_file in entity_dir.rglob("*.md"): + if md_file.name.startswith("_"): + continue + try: + content = md_file.read_text(encoding="utf-8") + except (OSError, UnicodeDecodeError): + continue + for line in content.split("\n"): + if line.startswith("name:"): + name = line.split(":", 1)[1].strip().strip('"').strip("'") + if len(name) >= MIN_ENTITY_NAME_LEN and name.lower() not in ENTITY_STOPLIST: + names[md_file.stem] = name + break + return names + + +def _compile_entity_patterns(entity_names: dict[str, str]) -> dict[str, re.Pattern]: + """Pre-compile word-boundary regex for each entity name.""" + patterns = {} + for slug, name in entity_names.items(): + try: + patterns[slug] = re.compile(r'\b' + re.escape(name) + r'\b', re.IGNORECASE) + except re.error: + continue + return patterns + + +def _extract_source_refs(content: str) -> set[str]: + """Extract source archive references ([[YYYY-MM-DD-...]]) from content.""" + return set(re.findall(r"\[\[(20\d{2}-\d{2}-\d{2}-[^\]]+)\]\]", content)) + + +def _find_entity_mentions(content: str, patterns: dict[str, re.Pattern]) -> set[str]: + """Find entity slugs whose names appear in the content (word-boundary match).""" + found = set() + for slug, pat in patterns.items(): + if pat.search(content): + found.add(slug) + return found + + +def _scan_domain_claims(worktree: Path, patterns: dict[str, re.Pattern]) -> dict[str, list[dict]]: + """Build domain -> [claim_info] mapping for all claims.""" + domain_claims = {} + domains_dir = worktree / "domains" + if not domains_dir.exists(): + return domain_claims + + for domain_dir in domains_dir.iterdir(): + if not domain_dir.is_dir(): + continue + claims = [] + for claim_file in domain_dir.glob("*.md"): + if claim_file.name.startswith("_") or claim_file.name == "directory.md": + continue + try: + content = claim_file.read_text(encoding="utf-8") + except (OSError, UnicodeDecodeError): + continue + claims.append({ + "slug": claim_file.stem, + "entities": _find_entity_mentions(content, patterns), + "sources": _extract_source_refs(content), + }) + domain_claims[domain_dir.name] = claims + return domain_claims + + +async def cross_domain_after_merge( + main_sha: str, + branch_sha: str, + pr_num: int, + main_worktree: Path, + conn=None, +) -> int: + """Detect cross-domain entity/source overlap for claims changed in this merge. + + Returns the number of cross-domain connections found. + """ + # 1. Get changed files + proc = await asyncio.create_subprocess_exec( + "git", "diff", "--name-only", "--diff-filter=ACMR", + main_sha, branch_sha, + cwd=str(main_worktree), + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + try: + stdout, _ = await asyncio.wait_for(proc.communicate(), timeout=10) + except asyncio.TimeoutError: + proc.kill() + await proc.wait() + logger.warning("cross_domain: git diff timed out") + return 0 + + if proc.returncode != 0: + return 0 + + diff_files = [f for f in stdout.decode().strip().split("\n") if f] + + # 2. Filter to claim files + changed_claims = [] + for fpath in diff_files: + if not fpath.endswith(".md") or not fpath.startswith("domains/"): + continue + parts = fpath.split("/") + if len(parts) < 3: + continue + basename = os.path.basename(fpath) + if basename.startswith("_") or basename == "directory.md": + continue + changed_claims.append({"path": fpath, "domain": parts[1], "slug": Path(basename).stem}) + + if not changed_claims: + return 0 + + # 3. Build entity patterns and scan all claims + entity_names = _build_entity_names(main_worktree) + if not entity_names: + return 0 + + patterns = _compile_entity_patterns(entity_names) + domain_claims = _scan_domain_claims(main_worktree, patterns) + + # 4. For each changed claim, find cross-domain connections + total_connections = 0 + all_connections = [] + + for claim in changed_claims: + claim_path = main_worktree / claim["path"] + try: + content = claim_path.read_text(encoding="utf-8") + except (OSError, UnicodeDecodeError): + continue + + my_entities = _find_entity_mentions(content, patterns) + my_sources = _extract_source_refs(content) + + if not my_entities and not my_sources: + continue + + connections = [] + for other_domain, other_claims in domain_claims.items(): + if other_domain == claim["domain"]: + continue + for other in other_claims: + shared_entities = my_entities & other["entities"] + shared_sources = my_sources & other["sources"] + + # Threshold: >=2 shared entities, OR 1 entity + 1 source + entity_count = len(shared_entities) + source_count = len(shared_sources) + + if entity_count >= 2 or (entity_count >= 1 and source_count >= 1): + connections.append({ + "other_claim": other["slug"], + "other_domain": other_domain, + "shared_entities": sorted(shared_entities)[:5], + "shared_sources": sorted(shared_sources)[:3], + }) + + if connections: + total_connections += len(connections) + all_connections.append({ + "claim": claim["slug"], + "domain": claim["domain"], + "connections": connections[:10], + }) + logger.info( + "cross_domain: %s (%s) has %d cross-domain connections", + claim["slug"], claim["domain"], len(connections), + ) + + # 5. Log to audit_log + if all_connections and conn is not None: + try: + conn.execute( + "INSERT INTO audit_log (stage, event, detail) VALUES (?, ?, ?)", + ("cross_domain", "connections_found", json.dumps({ + "pr": pr_num, + "total_connections": total_connections, + "claims_with_connections": len(all_connections), + "details": all_connections[:10], + })), + ) + except Exception: + logger.exception("cross_domain: audit_log write failed (non-fatal)") + + if total_connections: + logger.info( + "cross_domain: PR #%d — %d connections across %d claims", + pr_num, total_connections, len(all_connections), + ) + + return total_connections diff --git a/lib/db.py b/lib/db.py index 4b55ac9..653b803 100644 --- a/lib/db.py +++ b/lib/db.py @@ -9,7 +9,7 @@ from . import config logger = logging.getLogger("pipeline.db") -SCHEMA_VERSION = 10 +SCHEMA_VERSION = 19 SCHEMA_SQL = """ CREATE TABLE IF NOT EXISTS schema_version ( @@ -69,6 +69,7 @@ CREATE TABLE IF NOT EXISTS prs ( last_error TEXT, last_attempt TEXT, cost_usd REAL DEFAULT 0, + auto_merge INTEGER DEFAULT 0, created_at TEXT DEFAULT (datetime('now')), merged_at TEXT ); @@ -468,6 +469,67 @@ def migrate(conn: sqlite3.Connection): conn.commit() logger.info("Migration v10: added eval pipeline columns to response_audit") + if current < 11: + # Add auto_merge flag for agent PR auto-merge (eval-approved agent branches) + try: + conn.execute("ALTER TABLE prs ADD COLUMN auto_merge INTEGER DEFAULT 0") + except sqlite3.OperationalError: + pass # Column already exists (VPS may be ahead of repo schema) + conn.commit() + logger.info("Migration v11: added auto_merge column to prs table") + + + if current < 17: + # Add prompt/pipeline version tracking per PR + for col, default in [ + ("prompt_version", None), + ("pipeline_version", None), + ]: + try: + conn.execute(f"ALTER TABLE prs ADD COLUMN {col} TEXT") + except sqlite3.OperationalError: + pass # Column already exists + conn.commit() + logger.info("Migration v17: added prompt_version, pipeline_version to prs table") + + if current < 18: + conn.executescript(""" + CREATE TABLE IF NOT EXISTS review_records ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + pr_number INTEGER NOT NULL, + claim_path TEXT, + domain TEXT, + agent TEXT, + reviewer TEXT, + reviewer_model TEXT, + outcome TEXT NOT NULL, + rejection_reason TEXT, + disagreement_type TEXT, + notes TEXT, + batch_id TEXT, + claims_in_batch INTEGER, + reviewed_at TEXT DEFAULT (datetime('now')) + ); + CREATE INDEX IF NOT EXISTS idx_review_records_pr ON review_records(pr_number); + CREATE INDEX IF NOT EXISTS idx_review_records_agent ON review_records(agent); + """) + conn.commit() + logger.info("Migration v18: created review_records table") + + if current < 19: + # Add submitted_by for contributor attribution tracing. + # Tracks who submitted the source: human handle, agent name, or "self-directed". + try: + conn.execute("ALTER TABLE prs ADD COLUMN submitted_by TEXT") + except sqlite3.OperationalError: + pass # Column already exists + try: + conn.execute("ALTER TABLE sources ADD COLUMN submitted_by TEXT") + except sqlite3.OperationalError: + pass + conn.commit() + logger.info("Migration v19: added submitted_by to prs and sources tables") + if current < SCHEMA_VERSION: conn.execute( "INSERT OR REPLACE INTO schema_version (version) VALUES (?)", @@ -487,6 +549,36 @@ def audit(conn: sqlite3.Connection, stage: str, event: str, detail: str = None): ) +def record_review( + conn: sqlite3.Connection, + pr_number: int, + outcome: str, + *, + domain: str = None, + agent: str = None, + reviewer: str = None, + reviewer_model: str = None, + rejection_reason: str = None, + disagreement_type: str = None, + notes: str = None, + claims_in_batch: int = None, +): + """Write a review record. Called at each eval verdict point.""" + conn.execute( + """INSERT INTO review_records + (pr_number, domain, agent, reviewer, reviewer_model, outcome, + rejection_reason, disagreement_type, notes, batch_id, claims_in_batch) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""", + ( + pr_number, domain, agent, reviewer, reviewer_model, outcome, + rejection_reason, disagreement_type, + notes[:4000] if notes else None, + str(pr_number), # batch_id = PR number + claims_in_batch, + ), + ) + + def append_priority_log(conn: sqlite3.Connection, path: str, stage: str, priority: str, reasoning: str): """Append a priority assessment to a source's priority_log. diff --git a/lib/digest.py b/lib/digest.py new file mode 100644 index 0000000..a696f46 --- /dev/null +++ b/lib/digest.py @@ -0,0 +1,208 @@ +"""Daily digest — sends Cory a summary of all Tier 3 activity at 8am London time. + +Aggregates: merged claims (with insight summaries), pipeline metrics, agent activity, +pending review items. Runs as a scheduled job in bot.py. + +Epimetheus owns this module. +""" + +import logging +import sqlite3 +from datetime import datetime, timezone, timedelta +from zoneinfo import ZoneInfo + +logger = logging.getLogger("telegram.digest") + +LONDON_TZ = ZoneInfo("Europe/London") +DIGEST_HOUR_LONDON = 8 # 8am London time (auto-adjusts for BST/GMT) + + +def next_digest_time() -> datetime: + """Calculate the next 8am London time as a UTC datetime. + + Handles BST/GMT transitions automatically via zoneinfo. + """ + now = datetime.now(LONDON_TZ) + target = now.replace(hour=DIGEST_HOUR_LONDON, minute=0, second=0, microsecond=0) + if target <= now: + target += timedelta(days=1) + return target.astimezone(timezone.utc) + + +def _get_merged_claims_24h(conn: sqlite3.Connection) -> list[dict]: + """Get PRs merged in the last 24 hours with domain and branch info.""" + rows = conn.execute( + """SELECT number, branch, domain, agent, commit_type, merged_at, description + FROM prs + WHERE merged_at > datetime('now', '-24 hours') + AND status = 'merged' + ORDER BY merged_at DESC""", + ).fetchall() + return [dict(r) for r in rows] + + +def _get_pipeline_metrics_24h(conn: sqlite3.Connection) -> dict: + """Get pipeline activity metrics for the last 24 hours.""" + total_merged = conn.execute( + "SELECT COUNT(*) FROM prs WHERE merged_at > datetime('now', '-24 hours') AND status = 'merged'" + ).fetchone()[0] + + total_closed = conn.execute( + "SELECT COUNT(*) FROM prs WHERE status = 'closed' AND created_at > datetime('now', '-24 hours')" + ).fetchone()[0] + + total_conflict = conn.execute( + "SELECT COUNT(*) FROM prs WHERE status IN ('conflict', 'conflict_permanent') AND created_at > datetime('now', '-24 hours')" + ).fetchone()[0] + + total_open = conn.execute( + "SELECT COUNT(*) FROM prs WHERE status IN ('open', 'reviewing', 'approved', 'merging')" + ).fetchone()[0] + + # Approval rate (last 24h) + evaluated = conn.execute( + "SELECT COUNT(*) FROM prs WHERE leo_verdict IN ('approve', 'request_changes') AND created_at > datetime('now', '-24 hours')" + ).fetchone()[0] + approved = conn.execute( + "SELECT COUNT(*) FROM prs WHERE leo_verdict = 'approve' AND created_at > datetime('now', '-24 hours')" + ).fetchone()[0] + approval_rate = (approved / evaluated * 100) if evaluated > 0 else 0 + + return { + "merged": total_merged, + "closed": total_closed, + "conflict": total_conflict, + "open": total_open, + "evaluated": evaluated, + "approved": approved, + "approval_rate": approval_rate, + } + + +def _get_agent_activity_24h(conn: sqlite3.Connection) -> dict[str, int]: + """Get PR count by agent for the last 24 hours.""" + rows = conn.execute( + """SELECT agent, COUNT(*) as cnt + FROM prs + WHERE created_at > datetime('now', '-24 hours') + AND agent IS NOT NULL + GROUP BY agent + ORDER BY cnt DESC""", + ).fetchall() + return {r["agent"]: r["cnt"] for r in rows} + + +def _get_pending_review_count(conn: sqlite3.Connection) -> int: + """Count PRs awaiting review.""" + return conn.execute( + "SELECT COUNT(*) FROM prs WHERE status IN ('open', 'reviewing')" + ).fetchone()[0] + + +def _extract_claim_title(branch: str) -> str: + """Extract a human-readable claim title from a branch name. + + Branch format: extract/source-slug or agent/description + """ + # Strip prefix (extract/, research/, theseus/, etc.) + parts = branch.split("/", 1) + slug = parts[1] if len(parts) > 1 else parts[0] + # Convert slug to readable title + return slug.replace("-", " ").replace("_", " ").title() + + + +def format_digest( + merged_claims: list[dict], + metrics: dict, + agent_activity: dict[str, int], + pending_review: int, +) -> str: + """Format the daily digest message.""" + now = datetime.now(timezone.utc) + date_str = now.strftime("%Y-%m-%d") + + parts = [f"DAILY DIGEST — {date_str}", ""] + + # Merged claims section + if merged_claims: + # Group by domain + by_domain: dict[str, list] = {} + for claim in merged_claims: + domain = claim.get("domain") or "unknown" + by_domain.setdefault(domain, []).append(claim) + + parts.append(f"CLAIMS MERGED ({len(merged_claims)})") + for domain, claims in sorted(by_domain.items()): + for c in claims: + # Use real description from frontmatter if available, fall back to slug title + desc = c.get("description") + if desc: + # Take first description if multiple (pipe-delimited) + display = desc.split(" | ")[0] + if len(display) > 120: + display = display[:117] + "..." + else: + display = _extract_claim_title(c.get("branch", "unknown")) + commit_type = c.get("commit_type", "") + type_tag = f"[{commit_type}] " if commit_type else "" + parts.append(f" {type_tag}{display} ({domain})") + parts.append("") + else: + parts.extend(["CLAIMS MERGED (0)", " No claims merged in the last 24h", ""]) + + # Pipeline metrics + success_rate = 0 + total_attempted = metrics["merged"] + metrics["closed"] + metrics["conflict"] + if total_attempted > 0: + success_rate = metrics["merged"] / total_attempted * 100 + + parts.append("PIPELINE") + parts.append(f" Merged: {metrics['merged']} | Closed: {metrics['closed']} | Conflicts: {metrics['conflict']}") + parts.append(f" Success rate: {success_rate:.0f}% | Approval rate: {metrics['approval_rate']:.0f}%") + parts.append(f" Open PRs: {metrics['open']}") + parts.append("") + + # Agent activity + if agent_activity: + parts.append("AGENTS") + for agent, count in agent_activity.items(): + parts.append(f" {agent}: {count} PRs") + parts.append("") + else: + parts.extend(["AGENTS", " No agent activity in the last 24h", ""]) + + # Pending review + if pending_review > 0: + parts.append(f"PENDING YOUR REVIEW: {pending_review}") + else: + parts.append("PENDING YOUR REVIEW: 0") + + return "\n".join(parts) + + +async def send_daily_digest(context): + """Send daily digest to admin chat. Scheduled job.""" + conn = context.bot_data.get("approval_conn") + admin_chat_id = context.bot_data.get("admin_chat_id") + + if not conn or not admin_chat_id: + logger.debug("Digest skipped — no DB connection or admin chat ID") + return + + try: + merged = _get_merged_claims_24h(conn) + metrics = _get_pipeline_metrics_24h(conn) + activity = _get_agent_activity_24h(conn) + pending = _get_pending_review_count(conn) + + text = format_digest(merged, metrics, activity, pending) + + await context.bot.send_message( + chat_id=admin_chat_id, + text=text, + ) + logger.info("Daily digest sent (%d claims, %d agents active)", + len(merged), len(activity)) + except Exception as e: + logger.error("Failed to send daily digest: %s", e) diff --git a/lib/evaluate.py b/lib/evaluate.py index ddb850b..ff6dab8 100644 --- a/lib/evaluate.py +++ b/lib/evaluate.py @@ -28,6 +28,7 @@ from . import config, db from .domains import agent_for_domain, detect_domain_from_branch, detect_domain_from_diff from .forgejo import api as forgejo_api from .forgejo import get_agent_token, get_pr_diff, repo_path +from .merge import PIPELINE_OWNED_PREFIXES from .llm import run_batch_domain_review, run_domain_review, run_leo_review, triage_pr from .feedback import format_rejection_comment from .validate import load_existing_claims @@ -547,6 +548,31 @@ async def evaluate_pr(conn, pr_number: int, tier: str = None) -> dict: ) return {"pr": pr_number, "auto_approved": True, "reason": "musings_only"} + # Reweave bypass — reweave PRs only add frontmatter edges (supports/challenges/ + # related/depends_on/challenged_by). The eval LLM has no context for judging + # edge correctness and consistently flags factual_discrepancy on valid edges. + # Leo's manual PR review is the real quality gate for reweave. + branch_row = conn.execute("SELECT branch FROM prs WHERE number = ?", (pr_number,)).fetchone() + branch_name = branch_row["branch"] if branch_row else "" + if branch_name.startswith("reweave/"): + logger.info("PR #%d is reweave (branch=%s) — auto-approving, Leo reviews manually", pr_number, branch_name) + await forgejo_api( + "POST", + repo_path(f"issues/{pr_number}/comments"), + {"body": "Auto-approved: reweave structural update (frontmatter edges only). Leo reviews manually."}, + ) + conn.execute( + """UPDATE prs SET status = 'approved', leo_verdict = 'skipped', + domain_verdict = 'skipped', auto_merge = 1, + domain = COALESCE(domain, 'cross-domain') WHERE number = ?""", + (pr_number,), + ) + db.audit( + conn, "evaluate", "reweave_bypass", + json.dumps({"pr": pr_number, "branch": branch_name}), + ) + return {"pr": pr_number, "auto_approved": True, "reason": "reweave_bypass"} + # NOTE: Tier 0.5 mechanical checks now run in validate stage (before eval). # tier0_pass=1 guarantees all mechanical checks passed. No Tier 0.5 here. @@ -679,6 +705,11 @@ async def evaluate_pr(conn, pr_number: int, tier: str = None) -> dict: db.audit( conn, "evaluate", "domain_rejected", json.dumps({"pr": pr_number, "agent": agent, "issues": domain_issues}) ) + db.record_review( + conn, pr_number, "rejected", + domain=domain, agent=agent, reviewer=agent, reviewer_model="gpt-4o", + notes=(domain_review or "")[:4000], + ) # Disposition: check if this PR should be terminated or kept open await _dispose_rejected_pr(conn, pr_number, eval_attempts, domain_issues) @@ -733,17 +764,32 @@ async def evaluate_pr(conn, pr_number: int, tier: str = None) -> dict: # Submit formal Forgejo reviews (required for merge) await _post_formal_approvals(pr_number, pr_author) + # Auto-merge agent PRs: if branch is NOT pipeline-owned, set auto_merge=1 + # so the merge cycle picks it up without manual intervention. + branch_row = conn.execute("SELECT branch FROM prs WHERE number = ?", (pr_number,)).fetchone() + branch_name = branch_row["branch"] if branch_row else "" + is_agent_pr = not branch_name.startswith(PIPELINE_OWNED_PREFIXES) + conn.execute( - "UPDATE prs SET status = 'approved' WHERE number = ?", - (pr_number,), + "UPDATE prs SET status = 'approved', auto_merge = ? WHERE number = ?", + (1 if is_agent_pr else 0, pr_number), ) db.audit( conn, "evaluate", "approved", - json.dumps({"pr": pr_number, "tier": tier, "domain": domain, "leo": leo_verdict, "domain_agent": agent}), + json.dumps({"pr": pr_number, "tier": tier, "domain": domain, "leo": leo_verdict, "domain_agent": agent, + "auto_merge": is_agent_pr}), ) - logger.info("PR #%d: APPROVED (tier=%s, leo=%s, domain=%s)", pr_number, tier, leo_verdict, domain_verdict) + db.record_review( + conn, pr_number, "approved", + domain=domain, agent=agent, reviewer="leo", reviewer_model="sonnet" if tier == "STANDARD" else "opus", + notes=(leo_review or "")[:4000] if leo_review else None, + ) + if is_agent_pr: + logger.info("PR #%d: APPROVED + auto_merge (agent branch %s)", pr_number, branch_name) + else: + logger.info("PR #%d: APPROVED (tier=%s, leo=%s, domain=%s)", pr_number, tier, leo_verdict, domain_verdict) else: # Collect all issue tags from both reviews all_issues = [] @@ -770,6 +816,12 @@ async def evaluate_pr(conn, pr_number: int, tier: str = None) -> dict: {"pr": pr_number, "tier": tier, "leo": leo_verdict, "domain": domain_verdict, "issues": all_issues} ), ) + db.record_review( + conn, pr_number, "approved-with-changes", + domain=domain, agent=agent, reviewer="leo", + reviewer_model="sonnet" if tier == "STANDARD" else "opus", + notes=(leo_review or domain_review or "")[:4000], + ) logger.info( "PR #%d: CHANGES REQUESTED (leo=%s, domain=%s, issues=%s)", pr_number, diff --git a/lib/extract.py b/lib/extract.py new file mode 100644 index 0000000..d71fbb0 --- /dev/null +++ b/lib/extract.py @@ -0,0 +1,800 @@ +"""Extraction stage — automated claim extraction from queued sources. + +Replaces extract-cron.sh with a Python module inside the pipeline daemon. +Processes unprocessed sources in inbox/queue/, extracts claims via LLM, +creates PRs on Forgejo, and archives sources on main. + +Flow per source: +1. Read source frontmatter (domain, author, rationale) +2. Pre-screen: Haiku identifies themes, Qdrant finds prior art +3. Build KB index for dedup +4. Build extraction prompt (extraction_prompt.py) +5. Call Sonnet via OpenRouter +6. Parse JSON response +7. Post-extraction validation (post_extract.py) +8. Create branch, write claim/entity files, commit, push +9. Create PR on Forgejo via agent token +10. Archive source on main (worktree lock) + +Design: one source at a time (sequential), up to MAX_SOURCES per cycle. +Uses the main worktree for reading + archival, extract worktree for branches. + +Epimetheus owns this module. Leo reviews changes. +""" + +import asyncio +import json +import logging +import os +import re +import secrets +from datetime import date +from pathlib import Path + +from . import config +from .costs import record_usage +from .domains import agent_for_domain +from .extraction_prompt import build_extraction_prompt +from .forgejo import api as forgejo_api +from .llm import openrouter_call +from .post_extract import load_existing_claims_from_repo, validate_and_fix_claims +from .worktree_lock import async_main_worktree_lock + +logger = logging.getLogger("pipeline.extract") + +# Extraction worktree (separate from main to avoid conflicts) +EXTRACT_WORKTREE = config.BASE_DIR / "workspaces" / "extract" + +# Max sources per cycle +MAX_SOURCES = int(os.environ.get("MAX_EXTRACT_SOURCES", "3")) + +# KB index cache (rebuilt once per cycle, not per source) +_kb_index_cache: dict[str, str] = {} +_kb_index_timestamp: float = 0 +KB_INDEX_TTL = 300 # 5 minutes + + +def _parse_source_frontmatter(content: str) -> dict: + """Parse source file frontmatter. Returns dict of fields.""" + if not content.startswith("---"): + return {} + end = content.find("---", 3) + if end == -1: + return {} + raw = content[3:end] + + fm = {} + for line in raw.strip().split("\n"): + line = line.strip() + if not line or ":" not in line: + continue + key, _, val = line.partition(":") + key = key.strip() + val = val.strip().strip('"').strip("'") + if val.lower() == "null" or val == "": + val = None + fm[key] = val + return fm + + +def _get_kb_index(domain: str) -> str: + """Get KB index text for a domain. Uses cached /tmp/kb-indexes/ files.""" + import time + + global _kb_index_cache, _kb_index_timestamp + + now = time.time() + if now - _kb_index_timestamp > KB_INDEX_TTL: + _kb_index_cache.clear() + _kb_index_timestamp = now + + if domain in _kb_index_cache: + return _kb_index_cache[domain] + + # Try pre-generated index files first + index_file = Path(f"/tmp/kb-indexes/{domain}.txt") + if index_file.exists(): + text = index_file.read_text(encoding="utf-8") + _kb_index_cache[domain] = text + return text + + # Fallback: build from repo + main = config.MAIN_WORKTREE + claims = [] + domain_dir = main / "domains" / domain + if domain_dir.is_dir(): + for f in domain_dir.glob("*.md"): + if not f.name.startswith("_"): + claims.append(f"- {f.name}") + + text = f"## Claims in domains/{domain}/\n" + "\n".join(sorted(claims)) + _kb_index_cache[domain] = text + return text + + +async def _git(*args, cwd: str = None, timeout: int = 60) -> tuple[int, str]: + """Run a git command async. Returns (returncode, stdout+stderr).""" + proc = await asyncio.create_subprocess_exec( + "git", *args, + cwd=cwd or str(EXTRACT_WORKTREE), + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + try: + stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout=timeout) + except asyncio.TimeoutError: + proc.kill() + await proc.wait() + return -1, f"git {args[0]} timed out after {timeout}s" + output = (stdout or b"").decode().strip() + if stderr: + output += "\n" + stderr.decode().strip() + return proc.returncode, output + + +async def _pre_screen(source_content: str, source_title: str) -> str | None: + """Run pre-screening: identify themes and find prior art. + + Returns formatted prior art text, or None if pre-screening fails/unavailable. + Non-fatal — extraction proceeds without prior art if this fails. + """ + try: + from .pre_screen import identify_themes, PRIOR_ART_THRESHOLD + from .search import search + + key_file = config.SECRETS_DIR / "openrouter-key" + if not key_file.exists(): + return None + + api_key = key_file.read_text().strip() + themes = identify_themes(source_content, api_key, source_title) + if not themes: + return None + + # Search each theme against Qdrant + results = [] + search_queries = themes + ([source_title] if source_title else []) + + for query in search_queries[:5]: + try: + hits = search(query, limit=3, score_threshold=PRIOR_ART_THRESHOLD) + for hit in hits: + title = hit.get("title", hit.get("filename", "")) + score = hit.get("score", 0) + domain = hit.get("domain", "") + if title and score >= PRIOR_ART_THRESHOLD: + results.append(f"- [{score:.2f}] {title} (domain: {domain})") + except Exception: + continue + + if not results: + return None + + # Deduplicate + seen = set() + unique = [] + for r in results: + if r not in seen: + seen.add(r) + unique.append(r) + + return "\n".join(unique[:15]) + + except Exception: + logger.debug("Pre-screening failed (non-fatal)", exc_info=True) + return None + + +def _parse_extraction_json(text: str) -> dict | None: + """Parse extraction JSON from LLM response. Handles markdown fencing.""" + if not text: + return None + + # Strip markdown code fences + text = text.strip() + if text.startswith("```"): + # Remove opening fence (```json or ```) + first_newline = text.index("\n") if "\n" in text else len(text) + text = text[first_newline + 1:] + if text.endswith("```"): + text = text[:-3] + text = text.strip() + + try: + return json.loads(text) + except json.JSONDecodeError as e: + logger.warning("Failed to parse extraction JSON: %s", e) + # Try to find JSON object in text + match = re.search(r"\{[\s\S]+\}", text) + if match: + try: + return json.loads(match.group()) + except json.JSONDecodeError: + pass + return None + + +def _build_claim_content(claim: dict, agent: str) -> str: + """Build claim markdown file content from extraction JSON.""" + today = date.today().isoformat() + domain = claim.get("domain", "") + title = claim.get("title", claim.get("filename", "").replace("-", " ").replace(".md", "")) + description = claim.get("description", "") + confidence = claim.get("confidence", "experimental") + source_ref = claim.get("source", "") + body = claim.get("body", "") + scope = claim.get("scope", "") + sourcer = claim.get("sourcer", "") + related = claim.get("related_claims", []) + + lines = [ + "---", + "type: claim", + f"domain: {domain}", + f'title: "{title}"', + f'description: "{description}"', + f"confidence: {confidence}", + f'source: "{source_ref}"', + f"created: {today}", + f"agent: {agent}", + ] + if scope: + lines.append(f"scope: {scope}") + if sourcer: + lines.append(f'sourcer: "{sourcer}"') + if related: + lines.append("related_claims:") + for r in related: + lines.append(f' - "[[{r}]]"') + lines.append("---") + lines.append("") + lines.append(f"# {title}") + lines.append("") + if body: + lines.append(body) + lines.append("") + + return "\n".join(lines) + + +def _build_entity_content(entity: dict, domain: str) -> str: + """Build entity markdown file content from extraction JSON.""" + today = date.today().isoformat() + entity_type = entity.get("entity_type", "company") + description = entity.get("content", "") + + if description: + return description + + name = entity.get("filename", "").replace("-", " ").replace(".md", "").title() + return f"""--- +type: entity +entity_type: {entity_type} +domain: {domain} +description: "" +created: {today} +--- + +# {name} + +## Timeline + +{entity.get("timeline_entry", "")} +""" + + +async def _extract_one_source( + conn, + source_path: str, + source_content: str, + fm: dict, + existing_claims: set[str], + feedback: dict | None = None, +) -> tuple[int, int]: + """Extract claims from a single source. Returns (succeeded, errors).""" + source_file = os.path.basename(source_path) + domain = fm.get("domain", "") + agent_name = agent_for_domain(domain) + agent_lower = agent_name.lower() + title = fm.get("title", source_file) + rationale = fm.get("rationale") + intake_tier = fm.get("intake_tier") + proposed_by = fm.get("proposed_by") + + logger.info("Extracting: %s (domain: %s, agent: %s)", source_file, domain, agent_name) + + # 1. Pre-screen (non-fatal) + prior_art = await _pre_screen(source_content, title) + if prior_art: + logger.info("Pre-screening found %d prior art items", prior_art.count("\n") + 1) + + # 2. Build KB index + kb_index = _get_kb_index(domain) + + # 3. Build extraction prompt + prompt = build_extraction_prompt( + source_file=source_path, + source_content=source_content, + domain=domain, + agent=agent_name, + kb_index=kb_index, + rationale=rationale, + intake_tier=intake_tier, + proposed_by=proposed_by, + prior_art=prior_art, + previous_feedback=feedback, + ) + + # 4. Call LLM (OpenRouter — not Claude Max CLI) + # EXTRACT_MODEL is "sonnet" (CLI name), use MODEL_SONNET_OR for OpenRouter + extract_model = config.MODEL_SONNET_OR + response, usage = await openrouter_call( + model=extract_model, + prompt=prompt, + timeout_sec=config.EXTRACT_TIMEOUT, + max_tokens=8192, + ) + + # Record usage + try: + record_usage( + conn, + model=extract_model, + stage="extract", + input_tokens=usage.get("prompt_tokens", 0), + output_tokens=usage.get("completion_tokens", 0), + backend="api", + ) + except Exception: + logger.debug("Failed to record extraction usage", exc_info=True) + + if not response: + logger.error("LLM extraction failed for %s — no response", source_file) + return 0, 1 + + # 5. Parse JSON + extraction = _parse_extraction_json(response) + if not extraction: + logger.error("Failed to parse extraction JSON for %s", source_file) + return 0, 1 + + claims_raw = extraction.get("claims", []) + entities_raw = extraction.get("entities", []) + enrichments = extraction.get("enrichments", []) + decisions = extraction.get("decisions", []) + facts = extraction.get("facts", []) + notes = extraction.get("extraction_notes", "") + + logger.info( + "Extraction result for %s: %d claims, %d enrichments, %d entities, %d decisions", + source_file, len(claims_raw), len(enrichments), len(entities_raw), len(decisions), + ) + + # 6. Build claim file contents + claim_files = [] + for c in claims_raw: + filename = c.get("filename", "") + if not filename: + continue + if not filename.endswith(".md"): + filename += ".md" + content = _build_claim_content(c, agent_lower) + claim_files.append({"filename": filename, "domain": c.get("domain", domain), "content": content}) + + # Build entity file contents + entity_files = [] + for e in entities_raw: + filename = e.get("filename", "") + if not filename: + continue + if not filename.endswith(".md"): + filename += ".md" + action = e.get("action", "create") + if action == "create": + content = _build_entity_content(e, domain) + entity_files.append({"filename": filename, "domain": domain, "content": content}) + + # 7. Post-extraction validation + if claim_files: + kept_claims, rejected_claims, stats = validate_and_fix_claims( + claim_files, domain, agent_lower, existing_claims, + repo_root=str(config.MAIN_WORKTREE), + ) + if rejected_claims: + logger.info( + "Post-extract rejected %d/%d claims for %s: %s", + len(rejected_claims), len(claim_files), source_file, + stats.get("rejections", [])[:5], + ) + claim_files = kept_claims + + if not claim_files and not entity_files: + logger.info("No valid claims/entities after validation for %s — archiving as null-result", source_file) + await _archive_source(source_path, domain, "null-result") + return 0, 0 + + # 8. Create branch, write files, commit, push + slug = Path(source_file).stem + branch = f"extract/{slug}-{secrets.token_hex(2)}" + + # Prepare extract worktree + rc, _ = await _git("fetch", "origin", "main", cwd=str(EXTRACT_WORKTREE)) + rc, _ = await _git("checkout", "main", cwd=str(EXTRACT_WORKTREE)) + rc, _ = await _git("reset", "--hard", "origin/main", cwd=str(EXTRACT_WORKTREE)) + rc, _ = await _git("checkout", "-b", branch, cwd=str(EXTRACT_WORKTREE)) + if rc != 0: + # Branch might already exist + await _git("branch", "-D", branch, cwd=str(EXTRACT_WORKTREE)) + rc, out = await _git("checkout", "-b", branch, cwd=str(EXTRACT_WORKTREE)) + if rc != 0: + logger.error("Failed to create branch %s: %s", branch, out) + return 0, 1 + + # Write claim files + worktree = EXTRACT_WORKTREE + files_written = [] + for cf in claim_files: + domain_dir = worktree / "domains" / cf["domain"] + domain_dir.mkdir(parents=True, exist_ok=True) + fpath = domain_dir / cf["filename"] + fpath.write_text(cf["content"], encoding="utf-8") + files_written.append(f"domains/{cf['domain']}/{cf['filename']}") + + for ef in entity_files: + entity_dir = worktree / "entities" / domain + entity_dir.mkdir(parents=True, exist_ok=True) + fpath = entity_dir / ef["filename"] + fpath.write_text(ef["content"], encoding="utf-8") + files_written.append(f"entities/{domain}/{ef['filename']}") + + if not files_written: + logger.info("No files written for %s — cleaning up", source_file) + await _git("checkout", "main", cwd=str(EXTRACT_WORKTREE)) + await _git("branch", "-D", branch, cwd=str(EXTRACT_WORKTREE)) + await _archive_source(source_path, domain, "null-result") + return 0, 0 + + # Stage and commit + for f in files_written: + await _git("add", f, cwd=str(EXTRACT_WORKTREE)) + + commit_msg = ( + f"{agent_lower}: extract claims from {slug}\n\n" + f"- Source: {source_path}\n" + f"- Domain: {domain}\n" + f"- Claims: {len(claim_files)}, Entities: {len(entity_files)}\n" + f"- Enrichments: {len(enrichments)}\n" + f"- Extracted by: pipeline ingest (OpenRouter {extract_model})\n\n" + f"Pentagon-Agent: {agent_name} " + ) + + rc, out = await _git("commit", "-m", commit_msg, cwd=str(EXTRACT_WORKTREE)) + if rc != 0: + logger.error("Commit failed for %s: %s", branch, out) + await _git("checkout", "main", cwd=str(EXTRACT_WORKTREE)) + await _git("branch", "-D", branch, cwd=str(EXTRACT_WORKTREE)) + return 0, 1 + + # Push branch + rc, out = await _git("push", "-u", "origin", branch, cwd=str(EXTRACT_WORKTREE)) + if rc != 0: + logger.error("Push failed for %s: %s", branch, out) + await _git("checkout", "main", cwd=str(EXTRACT_WORKTREE)) + await _git("branch", "-D", branch, cwd=str(EXTRACT_WORKTREE)) + return 0, 1 + + # 9. Create PR on Forgejo + agent_token_file = config.SECRETS_DIR / f"forgejo-{agent_lower}-token" + if not agent_token_file.exists(): + agent_token_file = config.SECRETS_DIR / "forgejo-leo-token" + agent_token = agent_token_file.read_text().strip() + + pr_title = f"{agent_lower}: extract claims from {slug}" + pr_body = ( + f"## Automated Extraction\n\n" + f"**Source:** `{source_path}`\n" + f"**Domain:** {domain}\n" + f"**Agent:** {agent_name}\n" + f"**Model:** {extract_model}\n\n" + f"### Extraction Summary\n" + f"- **Claims:** {len(claim_files)}\n" + f"- **Entities:** {len(entity_files)}\n" + f"- **Enrichments:** {len(enrichments)}\n" + f"- **Decisions:** {len(decisions)}\n" + f"- **Facts:** {len(facts)}\n\n" + f"{notes}\n\n" + f"---\n" + f"*Extracted by pipeline ingest stage (replaces extract-cron.sh)*" + ) + + pr_result = await forgejo_api( + "POST", + f"/repos/{config.FORGEJO_OWNER}/{config.FORGEJO_REPO}/pulls", + body={"title": pr_title, "body": pr_body, "base": "main", "head": branch}, + token=agent_token, + ) + + if pr_result and pr_result.get("number"): + pr_num = pr_result["number"] + logger.info("PR #%d created for %s (%d claims, %d entities)", pr_num, source_file, len(claim_files), len(entity_files)) + + # Store contributor attribution: who submitted this source? + # Priority: proposed_by field → intake_tier inference → "unknown" + if proposed_by: + contributor = proposed_by.strip().strip('"').strip("'") + elif intake_tier == "research-task": + contributor = f"{agent_name} (self-directed)" + elif intake_tier == "directed": + contributor = "@m3taversal" + else: + # Default: if no proposed_by and not a research task, Cory submitted it + contributor = "@m3taversal" + + # Build pipe-separated claim titles for the description field + claim_titles = " | ".join( + c.get("title", c.get("filename", "").replace("-", " ").replace(".md", "")) + for c in claims_raw if c.get("title") or c.get("filename") + ) + + # Upsert: if discover_external_prs already created the row, update it; + # if not, create a partial row that discover will complete. + try: + conn.execute( + """INSERT INTO prs (number, branch, status, submitted_by, source_path, description) + VALUES (?, ?, 'open', ?, ?, ?) + ON CONFLICT(number) DO UPDATE SET + submitted_by = excluded.submitted_by, + source_path = excluded.source_path, + description = COALESCE(excluded.description, prs.description)""", + (pr_num, branch, contributor, source_path, claim_titles), + ) + conn.commit() + except Exception: + logger.debug("Failed to upsert submitted_by for PR #%d", pr_num, exc_info=True) + + # Also store on source record + try: + conn.execute( + "UPDATE sources SET submitted_by = ? WHERE path = ?", + (contributor, source_path), + ) + conn.commit() + except Exception: + logger.debug("Failed to update source submitted_by", exc_info=True) + else: + logger.warning("PR creation may have failed for %s — response: %s", source_file, pr_result) + + # Clean up extract worktree + await _git("checkout", "main", cwd=str(EXTRACT_WORKTREE)) + + # 10. Archive source on main + await _archive_source(source_path, domain, "processed", agent_lower) + + return 1, 0 + + +async def _archive_source( + source_path: str, + domain: str, + status: str, + agent: str | None = None, +) -> None: + """Move source from inbox/queue/ to archive (or null-result) on main. + + Uses worktree lock to avoid conflicts with other main-writing processes. + """ + source_file = os.path.basename(source_path) + main = str(config.MAIN_WORKTREE) + + try: + async with async_main_worktree_lock(): + # Pull latest + await _git("pull", "--rebase", "origin", "main", cwd=main, timeout=30) + + queue_path = Path(main) / "inbox" / "queue" / source_file + if not queue_path.exists(): + logger.warning("Source %s not found in queue — may have been archived already", source_file) + return + + if status == "null-result": + dest_dir = Path(main) / "inbox" / "null-result" + else: + dest_dir = Path(main) / "inbox" / "archive" / (domain or "unknown") + + dest_dir.mkdir(parents=True, exist_ok=True) + dest_path = dest_dir / source_file + + # Read and update frontmatter + content = queue_path.read_text(encoding="utf-8") + today = date.today().isoformat() + + content = re.sub(r"^status: unprocessed", f"status: {status}", content, flags=re.MULTILINE) + if agent and "processed_by:" not in content: + content = re.sub( + r"(^status: \w+)", + rf"\1\nprocessed_by: {agent}\nprocessed_date: {today}", + content, + count=1, + flags=re.MULTILINE, + ) + if "extraction_model:" not in content: + content = re.sub( + r"(^status: \w+.*?)(\n---)", + rf'\1\nextraction_model: "{config.MODEL_SONNET_OR}"\2', + content, + count=1, + flags=re.MULTILINE | re.DOTALL, + ) + + dest_path.write_text(content, encoding="utf-8") + queue_path.unlink() + + # Git add, commit, push + await _git("add", "inbox/", cwd=main) + commit_msg = ( + f"source: {source_file} → {status}\n\n" + f"Pentagon-Agent: Epimetheus " + ) + await _git("commit", "-m", commit_msg, cwd=main) + + # Push with retry + for attempt in range(3): + rc, out = await _git("push", "origin", "main", cwd=main, timeout=30) + if rc == 0: + break + logger.warning("Push attempt %d failed: %s", attempt + 1, out) + await _git("pull", "--rebase", "origin", "main", cwd=main, timeout=30) + else: + logger.error("Failed to push source archival after 3 attempts") + + except Exception: + logger.exception("Failed to archive source %s", source_file) + + +async def extract_cycle(conn, max_workers=None) -> tuple[int, int]: + """Main extraction cycle — called by the pipeline daemon's ingest stage. + + Finds unprocessed sources in inbox/queue/, extracts claims, creates PRs. + Returns (succeeded, errors) for circuit breaker tracking. + """ + main = config.MAIN_WORKTREE + + # Find unprocessed sources + queue_dir = main / "inbox" / "queue" + if not queue_dir.exists(): + return 0, 0 + + unprocessed = [] + for f in sorted(queue_dir.glob("*.md")): + try: + content = f.read_text(encoding="utf-8") + fm = _parse_source_frontmatter(content) + if fm.get("status") == "unprocessed": + unprocessed.append((str(f.relative_to(main)), content, fm)) + except Exception: + logger.debug("Failed to read source %s", f, exc_info=True) + + if not unprocessed: + return 0, 0 + + # Filter out sources that already have open extraction PRs + open_pr_slugs = set() + try: + prs = await forgejo_api( + "GET", + f"/repos/{config.FORGEJO_OWNER}/{config.FORGEJO_REPO}/pulls?state=open&limit=50", + ) + if prs: + for pr in prs: + head = pr.get("head", {}).get("ref", "") + if head.startswith("extract/"): + # Extract the source slug from branch name (extract/{slug}-{nonce}) + slug_part = head[len("extract/"):] + # Remove the random suffix (last 5 chars: -{4-hex-chars}) + if len(slug_part) > 5 and slug_part[-5] == "-": + slug_part = slug_part[:-5] + open_pr_slugs.add(slug_part) + except Exception: + logger.debug("Failed to check open PRs for dedup", exc_info=True) + + if open_pr_slugs: + before = len(unprocessed) + unprocessed = [ + (sp, c, f) for sp, c, f in unprocessed + if Path(sp).stem not in open_pr_slugs + ] + skipped = before - len(unprocessed) + if skipped: + logger.info("Skipped %d source(s) with existing open PRs", skipped) + + if not unprocessed: + return 0, 0 + + logger.info("Extract cycle: %d unprocessed source(s) found, processing up to %d", len(unprocessed), MAX_SOURCES) + + # Load existing claims for dedup + existing_claims = load_existing_claims_from_repo(str(main)) + + # Ensure extract worktree exists and is clean + if not EXTRACT_WORKTREE.exists(): + logger.error("Extract worktree not found at %s", EXTRACT_WORKTREE) + return 0, 1 + + total_ok = 0 + total_err = 0 + + # ── Re-extraction: pick up sources that failed eval and have feedback ── + reextract_rows = conn.execute( + """SELECT path, feedback FROM sources + WHERE status = 'needs_reextraction' AND feedback IS NOT NULL + ORDER BY updated_at ASC LIMIT ?""", + (max(1, MAX_SOURCES - len(unprocessed)),), + ).fetchall() + + for row in reextract_rows: + reex_path = row["path"] + # Source was archived — read from archive location + archive_base = main / "inbox" / "archive" + # Try to find the file in archive subdirs + reex_file = None + for subdir in archive_base.iterdir(): + candidate = subdir / Path(reex_path).name + if candidate.exists(): + reex_file = candidate + break + if not reex_file: + # Try original path as fallback + candidate = main / reex_path + if candidate.exists(): + reex_file = candidate + + if not reex_file: + logger.warning("Re-extraction: source %s not found on disk — skipping", reex_path) + continue + + try: + reex_content = reex_file.read_text(encoding="utf-8") + reex_fm = _parse_source_frontmatter(reex_content) + reex_feedback = json.loads(row["feedback"]) if row["feedback"] else {} + + logger.info("Re-extracting %s with feedback: %s", reex_path, list(reex_feedback.get("issues", []))) + + conn.execute( + "UPDATE sources SET status = 'extracting', updated_at = datetime('now') WHERE path = ?", + (reex_path,), + ) + conn.commit() + + ok, err = await _extract_one_source(conn, reex_path, reex_content, reex_fm, existing_claims, feedback=reex_feedback) + total_ok += ok + total_err += err + + if ok: + conn.execute( + "UPDATE sources SET status = 'extracted', updated_at = datetime('now') WHERE path = ?", + (reex_path,), + ) + else: + conn.execute( + "UPDATE sources SET status = 'error', last_error = 're-extraction failed', updated_at = datetime('now') WHERE path = ?", + (reex_path,), + ) + conn.commit() + except Exception: + logger.exception("Re-extraction failed for %s", reex_path) + total_err += 1 + + for source_path, content, fm in unprocessed[:MAX_SOURCES]: + try: + ok, err = await _extract_one_source(conn, source_path, content, fm, existing_claims) + total_ok += ok + total_err += err + except Exception: + logger.exception("Unhandled error extracting %s", source_path) + total_err += 1 + + # Brief pause between sources + await asyncio.sleep(2) + + logger.info("Extract cycle complete: %d succeeded, %d errors", total_ok, total_err) + return total_ok, total_err diff --git a/lib/extraction_prompt.py b/lib/extraction_prompt.py index d432327..0ddea52 100644 --- a/lib/extraction_prompt.py +++ b/lib/extraction_prompt.py @@ -28,6 +28,7 @@ def build_extraction_prompt( intake_tier: str | None = None, proposed_by: str | None = None, prior_art: list[dict] | None = None, + previous_feedback: dict | None = None, ) -> str: """Build the lean extraction prompt. @@ -76,6 +77,39 @@ Set `contributor_thesis_extractable: true` if you extracted the contributor's th else: contributor_directive = "" + # Build previous feedback section (for re-extraction after eval rejection) + if previous_feedback: + issues = previous_feedback.get("issues", []) + leo_verdict = previous_feedback.get("leo", "") + domain_verdict = previous_feedback.get("domain", "") + feedback_lines = [ + "\n## Previous Extraction Feedback\n", + "A previous extraction from this source was **rejected** by the evaluation pipeline.", + "Learn from these issues and avoid repeating them:\n", + ] + if issues: + for issue in issues: + issue_guidance = { + "frontmatter_schema": "Fix frontmatter format — ensure all required fields are present and correctly typed.", + "title_overclaims": "Make titles more precise — avoid broad generalizations. The title must be specific enough to disagree with.", + "confidence_miscalibration": "Calibrate confidence honestly — single source = experimental at most. Don't mark speculative claims as likely.", + "factual_discrepancy": "Check facts carefully — verify dates, numbers, and attributions against the source text.", + "near_duplicate": "Check the KB index more carefully — this claim may already exist. Prefer enrichment over duplication.", + "scope_error": "Scope claims correctly — don't mix structural, functional, and causal claims in one.", + "broken_wiki_links": "Ensure wiki links reference real entities/claims in the KB.", + } + guidance = issue_guidance.get(issue, f"Address: {issue}") + feedback_lines.append(f"- **{issue}**: {guidance}") + feedback_lines.append("") + if leo_verdict == "request_changes": + feedback_lines.append("The lead reviewer requested changes. Extract fewer, higher-quality claims.") + if domain_verdict == "request_changes": + feedback_lines.append("The domain reviewer requested changes. Pay closer attention to domain-specific standards.") + feedback_lines.append("") + previous_feedback_section = "\n".join(feedback_lines) + else: + previous_feedback_section = "" + # Build connection candidates section (if prior art found via Qdrant) if prior_art: pa_lines = [ @@ -161,7 +195,7 @@ Single source = experimental at most. Pitch rhetoric or marketing copy = specula **File:** {source_file} {source_content} -{contributor_directive}{connection_candidates} +{contributor_directive}{previous_feedback_section}{connection_candidates} ## KB Index (existing claims — check for duplicates and enrichment targets) {kb_index} diff --git a/lib/health.py b/lib/health.py index ba7fc2e..67c82a6 100644 --- a/lib/health.py +++ b/lib/health.py @@ -11,6 +11,7 @@ from . import config, costs, db from .analytics import get_snapshot_history, get_version_changes from .claim_index import build_claim_index, write_claim_index from .feedback import get_agent_error_patterns, get_all_agent_patterns +from .search import check_duplicate logger = logging.getLogger("pipeline.health") @@ -307,6 +308,121 @@ async def handle_metrics(request): }) +def pr_status(conn, pr_number: int | None = None, branch: str | None = None) -> dict: + """Get PR status for agent consumption. + + Look up by PR number or branch name. Returns state, eval verdicts, + merge status, time in queue, and rejection reasons. + + Args: + conn: SQLite connection with row_factory=sqlite3.Row + pr_number: PR number to look up + branch: Branch name to look up (fallback if no pr_number) + + Returns dict with PR state or {"error": "not_found"}. + """ + if pr_number is not None: + row = conn.execute( + """SELECT number, branch, source_path, status, domain, agent, + commit_type, tier, leo_verdict, domain_verdict, + domain_agent, eval_issues, priority, origin, + cost_usd, created_at, merged_at, last_attempt, last_error, + transient_retries, substantive_retries, description + FROM prs WHERE number = ?""", + (pr_number,), + ).fetchone() + elif branch: + row = conn.execute( + """SELECT number, branch, source_path, status, domain, agent, + commit_type, tier, leo_verdict, domain_verdict, + domain_agent, eval_issues, priority, origin, + cost_usd, created_at, merged_at, last_attempt, last_error, + transient_retries, substantive_retries, description + FROM prs WHERE branch = ? + ORDER BY number DESC LIMIT 1""", + (branch,), + ).fetchone() + else: + return {"error": "pr_number or branch required"} + + if not row: + return {"error": "not_found"} + + # Parse eval issues + issues = [] + try: + issues = json.loads(row["eval_issues"] or "[]") + except (json.JSONDecodeError, TypeError): + pass + + # Time in queue (created → now or merged) + time_in_queue_minutes = None + if row["created_at"]: + try: + created = datetime.fromisoformat(row["created_at"]) + if created.tzinfo is None: + created = created.replace(tzinfo=timezone.utc) + if row["merged_at"]: + end = datetime.fromisoformat(row["merged_at"]) + if end.tzinfo is None: + end = end.replace(tzinfo=timezone.utc) + else: + end = datetime.now(timezone.utc) + time_in_queue_minutes = round((end - created).total_seconds() / 60, 1) + except ValueError: + pass + + return { + "pr": row["number"], + "branch": row["branch"], + "source": row["source_path"], + "status": row["status"], + "domain": row["domain"], + "agent": row["agent"], + "commit_type": row["commit_type"], + "tier": row["tier"], + "leo_verdict": row["leo_verdict"], + "domain_verdict": row["domain_verdict"], + "domain_agent": row["domain_agent"], + "eval_issues": issues, + "priority": row["priority"], + "origin": row["origin"], + "cost_usd": row["cost_usd"], + "created_at": row["created_at"], + "merged_at": row["merged_at"], + "last_attempt": row["last_attempt"], + "last_error": row["last_error"], + "retries": { + "transient": row["transient_retries"], + "substantive": row["substantive_retries"], + }, + "description": row["description"], + "time_in_queue_minutes": time_in_queue_minutes, + } + + +async def handle_pr_status(request): + """GET /pr/{number} — single PR status for agent consumption.""" + conn = _conn(request) + try: + pr_number = int(request.match_info["number"]) + except (KeyError, ValueError): + return web.json_response({"error": "invalid pr number"}, status=400) + result = pr_status(conn, pr_number=pr_number) + status_code = 200 if "error" not in result else 404 + return web.json_response(result, status=status_code) + + +async def handle_check_duplicate(request): + """GET /check-duplicate?text=...&domain=... — near-duplicate detection.""" + text = request.query.get("text", "") + if not text: + return web.json_response({"error": "text parameter required"}, status=400) + domain = request.query.get("domain") + result = check_duplicate(text, domain=domain) + return web.json_response(result) + + async def handle_activity(request): """GET /activity — condensed PR activity feed (Rhea). @@ -688,6 +804,8 @@ def create_app() -> web.Application: app.router.add_get("/contributors", handle_contributors_list) app.router.add_get("/", handle_dashboard) app.router.add_get("/activity", handle_activity) + app.router.add_get("/pr/{number}", handle_pr_status) + app.router.add_get("/check-duplicate", handle_check_duplicate) app.router.add_get("/calibration", handle_calibration) app.router.add_get("/feedback/{agent}", handle_feedback) app.router.add_get("/feedback", handle_feedback_all) diff --git a/lib/llm.py b/lib/llm.py index ed38300..1e72c0e 100644 --- a/lib/llm.py +++ b/lib/llm.py @@ -10,6 +10,7 @@ Orchestration (PR lifecycle, SQLite state, Forgejo posting) stays in evaluate.py """ import asyncio +import json import logging import aiohttp @@ -264,15 +265,27 @@ async def openrouter_call( return None, empty_usage -async def claude_cli_call(model: str, prompt: str, timeout_sec: int = 600, cwd: str = None) -> str | None: - """Call Claude via CLI (Claude Max subscription). Returns response or None.""" +async def claude_cli_call(model: str, prompt: str, timeout_sec: int = 600, cwd: str = None) -> tuple[str | None, dict]: + """Call Claude via CLI (Claude Max subscription). Returns (response, usage). + + Uses --output-format json to capture token usage. Subscription calls cost $0 + but tokens are tracked for compute metrics (Cory: capture tokens/time, note subscription). + """ + empty_usage = { + "prompt_tokens": 0, "completion_tokens": 0, + "cache_read_tokens": 0, "cache_write_tokens": 0, + "duration_ms": 0, "duration_api_ms": 0, + "cost_estimate_usd": 0.0, + "stop_reason": "", "num_turns": 0, + "service_tier": "", "speed": "", + } proc = await asyncio.create_subprocess_exec( str(config.CLAUDE_CLI), "-p", "--model", model, "--output-format", - "text", + "json", cwd=cwd or str(config.REPO_DIR), stdin=asyncio.subprocess.PIPE, stdout=asyncio.subprocess.PIPE, @@ -288,7 +301,7 @@ async def claude_cli_call(model: str, prompt: str, timeout_sec: int = 600, cwd: proc.kill() await proc.wait() logger.error("Claude CLI timed out after %ds", timeout_sec) - return None + return None, empty_usage finally: _active_subprocesses.discard(proc) @@ -299,34 +312,57 @@ async def claude_cli_call(model: str, prompt: str, timeout_sec: int = 600, cwd: combined_lower = (out_text + err_text).lower() if "hit your limit" in combined_lower or "rate limit" in combined_lower: logger.warning("Claude Max rate limited (rc=%d, stdout: %s)", proc.returncode, out_text[:200]) - return "RATE_LIMITED" + return "RATE_LIMITED", empty_usage if proc.returncode != 0: logger.error("Claude CLI failed (rc=%d): stderr=%s stdout=%s", proc.returncode, err_text[:200], out_text[:200]) - return None + return None, empty_usage - return out_text.strip() + # Parse JSON output to extract full usage telemetry + usage = empty_usage.copy() + try: + data = json.loads(out_text) + text = data.get("result", "") + raw_usage = data.get("usage", {}) + usage = { + "prompt_tokens": raw_usage.get("input_tokens", 0), + "completion_tokens": raw_usage.get("output_tokens", 0), + "cache_read_tokens": raw_usage.get("cache_read_input_tokens", 0), + "cache_write_tokens": raw_usage.get("cache_creation_input_tokens", 0), + "duration_ms": data.get("duration_ms", 0), + "duration_api_ms": data.get("duration_api_ms", 0), + "cost_estimate_usd": data.get("total_cost_usd", 0.0), + "stop_reason": data.get("stop_reason", ""), + "num_turns": data.get("num_turns", 0), + "service_tier": raw_usage.get("service_tier", ""), + "speed": raw_usage.get("speed", ""), + } + except (json.JSONDecodeError, KeyError): + logger.warning("Claude CLI returned non-JSON output, token tracking unavailable") + text = out_text.strip() + + return text, usage # ─── Review execution ───────────────────────────────────────────────────── -async def triage_pr(diff: str) -> tuple[str, dict]: - """Triage PR via Haiku → (tier, usage). tier is DEEP/STANDARD/LIGHT.""" +async def triage_pr(diff: str) -> tuple[str, dict, str]: + """Triage PR via Haiku → (tier, usage, reason). tier is DEEP/STANDARD/LIGHT.""" prompt = TRIAGE_PROMPT.format(diff=diff[:50000]) # Cap diff size for triage result, usage = await openrouter_call(config.TRIAGE_MODEL, prompt, timeout_sec=30) if not result: logger.warning("Triage failed, defaulting to STANDARD") - return "STANDARD", usage + return "STANDARD", usage, "triage failed, default" tier = result.split("\n")[0].strip().upper() if tier in ("DEEP", "STANDARD", "LIGHT"): reason = result.split("\n")[1].strip() if "\n" in result else "" logger.info("Triage: %s — %s", tier, reason[:100]) - return tier, usage + return tier, usage, reason[:500] logger.warning("Triage returned unparseable '%s', defaulting to STANDARD", tier[:20]) - return "STANDARD", usage + return "STANDARD", usage, f"unparseable response, default (got: {tier[:20]})" async def run_batch_domain_review( @@ -402,7 +438,7 @@ async def run_leo_review(diff: str, files: str, tier: str) -> tuple[str | None, # (Cory, Mar 14: "yes lets skip opus") # # --- Re-enable Opus later (uses EVAL_TIMEOUT_OPUS for longer reasoning): --- - # result = await claude_cli_call(config.EVAL_LEO_MODEL, prompt, timeout_sec=config.EVAL_TIMEOUT_OPUS) + # result, usage = await claude_cli_call(config.EVAL_LEO_MODEL, prompt, timeout_sec=config.EVAL_TIMEOUT_OPUS) # if result == "RATE_LIMITED" or result is None: # logger.info("Opus unavailable for DEEP Leo review — overflowing to Sonnet") # result, usage = await openrouter_call(config.EVAL_LEO_STANDARD_MODEL, prompt, timeout_sec=config.EVAL_TIMEOUT_OPUS) diff --git a/lib/merge.py b/lib/merge.py index 3714d39..49ac654 100644 --- a/lib/merge.py +++ b/lib/merge.py @@ -32,6 +32,15 @@ from .forgejo import api as forgejo_api # (Leo directive: PRs #2141, #157, #2142, #2180 were orphaned by pipeline rebase) PIPELINE_OWNED_PREFIXES = ("extract/", "ingestion/", "epimetheus/", "reweave/", "fix/") +# Safety assertion: agent branches MUST NOT be in PIPELINE_OWNED_PREFIXES. +# Auto-merge on eval approval bypasses Leo's review gate. +# Agent PRs use auto_merge flag instead (set by evaluate.py after two-reviewer approval). +_AGENT_NAMES = ("theseus", "rio", "astra", "vida", "clay", "leo", "argus", "oberon", "rhea", "ganymede") +for _prefix in PIPELINE_OWNED_PREFIXES: + for _agent in _AGENT_NAMES: + assert not _prefix.startswith(f"{_agent}/"), \ + f"FATAL: Agent prefix '{_agent}/' found in PIPELINE_OWNED_PREFIXES — this bypasses Leo's review gate" + # Import worktree lock — file at /opt/teleo-eval/pipeline/lib/worktree_lock.py try: from .worktree_lock import async_main_worktree_lock @@ -39,6 +48,8 @@ except ImportError: import sys sys.path.insert(0, os.path.dirname(__file__)) from worktree_lock import async_main_worktree_lock +from .cascade import cascade_after_merge +from .cross_domain import cross_domain_after_merge from .forgejo import get_agent_token, get_pr_diff, repo_path logger = logging.getLogger("pipeline.merge") @@ -108,11 +119,16 @@ async def discover_external_prs(conn) -> int: domain = None if not is_pipeline else detect_domain_from_branch(pr["head"]["ref"]) agent, commit_type = classify_branch(pr["head"]["ref"]) + # For human PRs, submitted_by is the Forgejo author. + # For pipeline PRs, submitted_by is set later by extract.py (from source proposed_by). + submitted_by = author if origin == "human" else None + conn.execute( """INSERT OR IGNORE INTO prs - (number, branch, status, origin, priority, domain, agent, commit_type) - VALUES (?, ?, 'open', ?, ?, ?, ?, ?)""", - (pr["number"], pr["head"]["ref"], origin, priority, domain, agent, commit_type), + (number, branch, status, origin, priority, domain, agent, commit_type, + prompt_version, pipeline_version, submitted_by) + VALUES (?, ?, 'open', ?, ?, ?, ?, ?, ?, ?, ?)""", + (pr["number"], pr["head"]["ref"], origin, priority, domain, agent, commit_type, config.PROMPT_VERSION, config.PIPELINE_VERSION, submitted_by), ) db.audit( conn, @@ -187,7 +203,7 @@ async def _claim_next_pr(conn, domain: str) -> dict | None: LEFT JOIN sources s ON p.source_path = s.path WHERE p.status = 'approved' AND p.domain = ? - AND ({prefix_clauses}) + AND ({prefix_clauses} OR p.auto_merge = 1) AND NOT EXISTS ( SELECT 1 FROM prs p2 WHERE p2.domain = p.domain @@ -1420,13 +1436,22 @@ async def _merge_domain_queue(conn, domain: str) -> tuple[int, int]: continue if not pick_ok: - # Cherry-pick failed — this is a genuine conflict (not a race condition). - # No retry needed: cherry-pick onto fresh main means main can't have moved. - logger.warning("PR #%d cherry-pick failed: %s", pr_num, pick_msg) - conn.execute( - "UPDATE prs SET status = 'conflict', merge_cycled = 1, merge_failures = COALESCE(merge_failures, 0) + 1, last_error = ? WHERE number = ?", - (pick_msg[:500], pr_num), - ) + logger.warning("PR #%d merge/cherry-pick failed: %s", pr_num, pick_msg) + # Reweave: close immediately, don't retry (Ship: same rationale as ff-push failure) + if branch.startswith("reweave/"): + conn.execute( + "UPDATE prs SET status = 'closed', merge_cycled = 1, merge_failures = COALESCE(merge_failures, 0) + 1, last_error = ? WHERE number = ?", + (f"reweave merge failed (closed, not retried): {pick_msg[:400]}", pr_num), + ) + await forgejo_api("PATCH", repo_path(f"pulls/{pr_num}"), {"state": "closed"}) + await forgejo_api("POST", repo_path(f"issues/{pr_num}/comments"), + {"body": f"Reweave merge failed — closing. Next nightly reweave will create a fresh branch.\n\nError: {pick_msg[:200]}"}) + await _delete_remote_branch(branch) + else: + conn.execute( + "UPDATE prs SET status = 'conflict', merge_cycled = 1, merge_failures = COALESCE(merge_failures, 0) + 1, last_error = ? WHERE number = ?", + (pick_msg[:500], pr_num), + ) db.audit(conn, "merge", "cherry_pick_failed", json.dumps({"pr": pr_num, "error": pick_msg[:200]})) failed += 1 continue @@ -1471,10 +1496,24 @@ async def _merge_domain_queue(conn, domain: str) -> tuple[int, int]: if not merge_ok: logger.error("PR #%d merge failed: %s", pr_num, merge_msg) - conn.execute( - "UPDATE prs SET status = 'conflict', merge_cycled = 1, merge_failures = COALESCE(merge_failures, 0) + 1, last_error = ? WHERE number = ?", - (merge_msg[:500], pr_num), - ) + # Reweave PRs: close immediately on failure. Cherry-pick retry + # will always fail (reweave modifies existing files). Next nightly + # run creates a fresh branch from current main — retry is wasteful. + # (Ship: prevents reweave flood + wasted retry cycles) + if branch.startswith("reweave/"): + conn.execute( + "UPDATE prs SET status = 'closed', merge_cycled = 1, merge_failures = COALESCE(merge_failures, 0) + 1, last_error = ? WHERE number = ?", + (f"reweave merge failed (closed, not retried): {merge_msg[:400]}", pr_num), + ) + await forgejo_api("PATCH", repo_path(f"pulls/{pr_num}"), {"state": "closed"}) + await forgejo_api("POST", repo_path(f"issues/{pr_num}/comments"), + {"body": f"Reweave merge failed — closing. Next nightly reweave will create a fresh branch.\n\nError: {merge_msg[:200]}"}) + await _delete_remote_branch(branch) + else: + conn.execute( + "UPDATE prs SET status = 'conflict', merge_cycled = 1, merge_failures = COALESCE(merge_failures, 0) + 1, last_error = ? WHERE number = ?", + (merge_msg[:500], pr_num), + ) db.audit(conn, "merge", "merge_failed", json.dumps({"pr": pr_num, "error": merge_msg[:200]})) failed += 1 continue @@ -1506,6 +1545,20 @@ async def _merge_domain_queue(conn, domain: str) -> tuple[int, int]: # New claim A with supports:[B] → add supports:[A] on B's frontmatter await _reciprocal_edges(main_sha, branch_sha) + # Cascade: notify agents whose beliefs/positions depend on changed claims + try: + await cascade_after_merge(main_sha, branch_sha, pr_num, config.MAIN_WORKTREE, conn=conn) + except Exception: + logger.exception("PR #%d: cascade failed (non-fatal)", pr_num) + + # Cross-domain citation index: log entity-based connections between domains + try: + await cross_domain_after_merge(main_sha, branch_sha, pr_num, config.MAIN_WORKTREE, conn=conn) + except Exception: + logger.exception("PR #%d: cross_domain failed (non-fatal)", pr_num) + + conn.commit() # Commit DB writes before slow branch deletion + # Delete remote branch immediately (Ganymede Q4) await _delete_remote_branch(branch) @@ -1557,6 +1610,11 @@ async def _reconcile_db_state(conn): continue if forgejo_state == "closed" and not is_merged and db_status not in ("closed",): + # Clean up branch too — stale branches get rediscovered as new PRs + # (Ship: prevents reweave flood where closed PRs leave branches that + # trigger discover_external_prs → new PR → fail → close → repeat) + if branch: + await _delete_remote_branch(branch) conn.execute( "UPDATE prs SET status = 'closed', last_error = 'reconciled: closed on Forgejo' WHERE number = ?", (pr_number,), @@ -1749,6 +1807,22 @@ async def _retry_conflict_prs(conn) -> tuple[int, int]: branch = row["branch"] attempts = row["conflict_rebase_attempts"] or 0 + # Reweave branches modify existing files — cherry-pick will always fail. + # Close immediately and delete branch. Next nightly reweave creates fresh. + # (Ship: prevents wasting 3 retry cycles on branches that can never cherry-pick) + if branch.startswith("reweave/"): + logger.info("Reweave PR #%d: skipping retry, closing + deleting branch", pr_number) + conn.execute( + "UPDATE prs SET status = 'closed', last_error = 'reweave: closed (retry skipped, next nightly creates fresh)' WHERE number = ?", + (pr_number,), + ) + await forgejo_api("PATCH", repo_path(f"pulls/{pr_number}"), {"state": "closed"}) + await forgejo_api("POST", repo_path(f"issues/{pr_number}/comments"), + {"body": "Reweave conflict — closing instead of retrying. Cherry-pick always fails on reweave branches (they modify existing files). Next nightly reweave will create a fresh branch from current main."}) + await _delete_remote_branch(branch) + failed += 1 + continue + logger.info("Conflict retry [%d/%d] PR #%d branch=%s", attempts + 1, MAX_CONFLICT_REBASE_ATTEMPTS, pr_number, branch) diff --git a/lib/post_extract.py b/lib/post_extract.py index 7d033cb..7ce3aef 100644 --- a/lib/post_extract.py +++ b/lib/post_extract.py @@ -163,15 +163,29 @@ def fix_frontmatter(content: str, domain: str, agent: str) -> tuple[str, list[st def fix_wiki_links(content: str, existing_claims: set[str]) -> tuple[str, list[str]]: - """Strip brackets from broken wiki links, keeping the text. Returns (fixed_content, fixes).""" + """Fix or strip broken wiki links. Resolves slug→space mismatches before stripping. + + The LLM often generates wiki links as slugs (hyphens) but KB filenames use spaces. + Try normalizing hyphens→spaces before giving up and stripping brackets. + """ fixes = [] + # Build a lookup: normalized (lowercased, hyphens→spaces) → original stem + _normalized_lookup: dict[str, str] = {} + for stem in existing_claims: + _normalized_lookup[stem.lower().replace("-", " ")] = stem def replace_broken(match): link = match.group(1).strip() - if link not in existing_claims: - fixes.append(f"stripped_wiki_link:{link[:60]}") - return link # Keep text, remove brackets - return match.group(0) + if link in existing_claims: + return match.group(0) # Exact match — keep as-is + # Try normalizing slug to spaces + normalized = link.lower().replace("-", " ") + if normalized in _normalized_lookup: + resolved = _normalized_lookup[normalized] + fixes.append(f"resolved_wiki_link:{link[:40]}->{resolved[:40]}") + return f"[[{resolved}]]" + fixes.append(f"stripped_wiki_link:{link[:60]}") + return link # Keep text, remove brackets fixed = WIKI_LINK_RE.sub(replace_broken, content) return fixed, fixes diff --git a/lib/pre_screen.py b/lib/pre_screen.py new file mode 100644 index 0000000..2f5236b --- /dev/null +++ b/lib/pre_screen.py @@ -0,0 +1,221 @@ +"""Pre-screening: identify themes from source, fetch prior art from Qdrant. + +Runs before extraction to show the extractor what the KB already knows. +Reduces near-duplicates (our #1 rejection cause) by turning semantic +pre-screening from a manual discipline into a pipeline feature. + +Design: Leo (approved 2026-03-30). Owner: Epimetheus. + +Flow: + 1. Haiku identifies 3-5 themes from source text + 2. Each theme + title (with author-stripped variant) → Tier 1 search + 3. Results injected into extraction prompt as "Prior Art" + 4. Extractor classifies extractions as NEW / ENRICHMENT / CHALLENGE + 5. ENRICHMENT/CHALLENGE must cite specific target claim (hard gate) + +Cost: ~$0.002/source (Haiku theme pass) + free Qdrant queries. +""" + +import json +import os +import re +import sys + +import requests + +# Search library (same Tier 1 path used by Argus + Telegram bot) +from pathlib import Path +sys.path.insert(0, str(Path(__file__).parent.parent)) +from lib.search import search + +OPENROUTER_URL = "https://openrouter.ai/api/v1/chat/completions" +THEME_MODEL = "anthropic/claude-haiku-4.5" + +# Regex to strip leading author/entity patterns from titles +# e.g. "Shapiro: How Far Will AI Video Go" → "How Far Will AI Video Go" +# "Aschenbrenner — Situational Awareness" → "Situational Awareness" +# Prior art threshold — only show results above this score to the extractor. +# 0.50 catches mechanism-level matches where compound themes dilute embeddings. +# Was 0.65 but Haiku compound themes score 0.50-0.60 even on exact matches. +# False positives cost nothing (extractor sees irrelevant prior art, ignores it). +# False negatives cost wasted extraction + review + rejection. +PRIOR_ART_THRESHOLD = 0.50 + +AUTHOR_PREFIX_RE = re.compile( + r"^[A-Za-z\-']+(?:\s+[A-Za-z\-']+)?\s*[:–—\-]\s*", re.UNICODE +) + + +def identify_themes(source_content: str, api_key: str, source_title: str = "") -> list[str]: + """Use Haiku to identify 3-5 major themes from source text. + + Returns a list of theme strings suitable as search queries. + Falls back to [source_title] on API failure. + """ + # Truncate source to keep Haiku costs minimal + snippet = source_content[:3000] + + prompt = f"""Identify the 3-5 major themes or topics in this text. +Return ONLY a JSON array of short search queries (3-8 words each). +Keep queries SHORT — 3-5 words is ideal. Compound phrases score poorly in vector search. + +Example good output: ["futarchy governance", "semaglutide kidney outcomes", "ICO oversubscription"] +Example bad output: ["futarchy governance mechanisms detecting revenue misrepresentation token launches", "prediction market accuracy identifying fraudulent financial claims"] + +Text: +{snippet} + +Return JSON array only, no explanation.""" + + try: + headers = { + "Authorization": f"Bearer {api_key}", + "Content-Type": "application/json", + "HTTP-Referer": "https://livingip.xyz", + "X-Title": "Teleo Pre-Screen", + } + payload = { + "model": THEME_MODEL, + "messages": [{"role": "user", "content": prompt}], + "temperature": 0.1, + "max_tokens": 500, + } + resp = requests.post(OPENROUTER_URL, headers=headers, json=payload, timeout=30) + resp.raise_for_status() + content = resp.json()["choices"][0]["message"]["content"].strip() + + # Strip markdown fencing if present + if content.startswith("```"): + content = re.sub(r"^```(?:json)?\s*\n?", "", content) + content = re.sub(r"\n?```\s*$", "", content) + + themes = json.loads(content) + if isinstance(themes, list) and all(isinstance(t, str) for t in themes): + return themes[:5] + except Exception as e: + print(f" WARN: Theme identification failed: {e}", file=sys.stderr) + + # Fallback: use title as the only theme + return [source_title] if source_title else [] + + +def _strip_author(title: str) -> str: + """Strip leading author/entity prefix from a title. + + "Shapiro: How Far Will AI Video Go" → "How Far Will AI Video Go" + "Noah Smith — AI and Jobs" → "AI and Jobs" + """ + stripped = AUTHOR_PREFIX_RE.sub("", title).strip() + # Only use stripped version if it's meaningfully different + if stripped and len(stripped) > 10 and stripped != title: + return stripped + return "" + + +def _extract_title_from_source(source_content: str, source_file: str) -> str: + """Get a usable title from source frontmatter or filename.""" + # Try frontmatter title + match = re.search(r"^title:\s*[\"']?(.+?)[\"']?\s*$", source_content, re.MULTILINE) + if match: + return match.group(1).strip() + + # Fall back to filename + basename = os.path.basename(source_file).replace(".md", "") + # Strip date prefix (e.g., "2026-03-15-article-name" → "article-name") + basename = re.sub(r"^\d{4}-\d{2}-\d{2}-", "", basename) + return basename.replace("-", " ") + + +def pre_screen(source_content: str, source_file: str, api_key: str, + domain: str | None = None) -> dict: + """Run full pre-screening: themes → search → prior art. + + Returns: + { + "themes": ["theme1", "theme2", ...], + "prior_art": [ + {"claim_path": str, "title": str, "score": float, "query": str}, + ... + ], + "search_queries": ["query1", "query2", ...], # for audit trail + } + """ + title = _extract_title_from_source(source_content, source_file) + + # Step 1: Identify themes + themes = identify_themes(source_content, api_key, source_title=title) + + # Step 2: Build search queries (themes + title + author-stripped title) + queries = list(themes) + if title and title not in queries: + queries.append(title) + stripped = _strip_author(title) + if stripped and stripped not in queries: + queries.append(stripped) + + # Step 3: Search Qdrant for each query (Tier 1: expand=False) + seen_paths: set[str] = set() + prior_art: list[dict] = [] + + for query in queries: + try: + results = search(query, expand=False, domain=None) # cross-domain on purpose + for hit in results.get("direct_results", []): + path = hit.get("claim_path", "") + if path and path not in seen_paths: + seen_paths.add(path) + prior_art.append({ + "claim_path": path, + "title": hit.get("title", os.path.basename(path).replace(".md", "").replace("-", " ")), + "score": round(hit.get("score", 0), 3), + "query": query, + }) + except Exception as e: + print(f" WARN: Pre-screen search failed for '{query[:50]}': {e}", file=sys.stderr) + + # Filter below threshold, sort by score descending, cap at 25 + prior_art = [p for p in prior_art if p["score"] >= PRIOR_ART_THRESHOLD] + prior_art.sort(key=lambda x: x["score"], reverse=True) + prior_art = prior_art[:25] + + return { + "themes": themes, + "prior_art": prior_art, + "search_queries": queries, + } + + +def format_prior_art_for_prompt(prior_art: list[dict]) -> str: + """Format prior art results for injection into the extraction prompt. + + Leo's required format: + - [claim-slug](path) — similarity: 0.82 — query: "theme that matched" + """ + if not prior_art: + return "No similar claims found in the KB. This source likely covers novel territory." + + lines = [] + for item in prior_art: + slug = os.path.basename(item["claim_path"]).replace(".md", "") + lines.append( + f"- [{slug}]({item['claim_path']}) — similarity: {item['score']:.2f} — query: \"{item['query'][:60]}\"" + ) + return "\n".join(lines) + + +def format_prior_art_for_pr(prior_art: list[dict]) -> str: + """Format prior art for PR body (structured, reviewable by Leo). + + Shows similarity score + which query matched for verification. + """ + if not prior_art: + return "No prior art found — source covers novel territory.\n" + + lines = ["## Prior Art (automated pre-screening)\n"] + for item in prior_art: + slug = os.path.basename(item["claim_path"]).replace(".md", "") + lines.append( + f"- [{slug}]({item['claim_path']}) — similarity: {item['score']:.2f} — matched query: \"{item['query'][:80]}\"" + ) + lines.append("") + return "\n".join(lines) diff --git a/lib/search.py b/lib/search.py index c579d5d..03806c7 100644 --- a/lib/search.py +++ b/lib/search.py @@ -301,11 +301,14 @@ def graph_expand(seed_paths: list[str], repo_root: Path | None = None, # --- Combined search (Layer 1 + Layer 2) --- -# Default thresholds — calibrated with Leo's retrieval audits +# Default thresholds — lowered Apr 5 after production audit showed 0 vector hits. +# text-embedding-3-small scores 0.50-0.60 on conceptual matches (e.g. "risks in +# investing" vs specific claims). 0.70 rejected every result. 0.50/0.40 lets +# relevant claims through while still filtering noise. PASS1_LIMIT = 5 -PASS1_THRESHOLD = 0.70 +PASS1_THRESHOLD = 0.50 PASS2_LIMIT = 5 -PASS2_THRESHOLD = 0.60 +PASS2_THRESHOLD = 0.40 HARD_CAP = 10 @@ -414,3 +417,64 @@ def search(query: str, expand: bool = False, "expanded_results": final_expanded, "total": len(all_sorted), } + + +# --- Duplicate detection --- + + +def check_duplicate(text: str, threshold: float = 0.85, + domain: str | None = None) -> dict: + """Check if a claim/text is a near-duplicate of existing KB content. + + Embeds the text, searches Qdrant, returns top-3 matches with scores. + Thresholds: >=0.85 likely duplicate, 0.70-0.85 check manually, <0.70 novel. + + Args: + text: The claim text to check. + threshold: Minimum score to flag as potential duplicate (default 0.85). + domain: Optional domain filter. + + Returns: + { + "query": str, + "is_duplicate": bool, # True if any match >= threshold + "highest_score": float, # Best match score + "verdict": str, # "duplicate" | "check_manually" | "novel" + "matches": [ # Top 3 matches + {"score": float, "claim_path": str, "claim_title": str, "domain": str} + ] + } + """ + vector = embed_query(text) + if vector is None: + return {"query": text[:100], "is_duplicate": False, "highest_score": 0, + "verdict": "error", "matches": [], "error": "embedding_failed"} + + hits = search_qdrant(vector, limit=3, domain=domain, score_threshold=0.3) + + matches = [] + for hit in hits: + payload = hit.get("payload", {}) + matches.append({ + "score": round(hit.get("score", 0), 4), + "claim_path": payload.get("claim_path", ""), + "claim_title": payload.get("claim_title", ""), + "domain": payload.get("domain", ""), + }) + + highest = matches[0]["score"] if matches else 0.0 + + if highest >= threshold: + verdict = "duplicate" + elif highest >= 0.70: + verdict = "check_manually" + else: + verdict = "novel" + + return { + "query": text[:100], + "is_duplicate": highest >= threshold, + "highest_score": highest, + "verdict": verdict, + "matches": matches, + } diff --git a/lib/substantive_fixer.py b/lib/substantive_fixer.py index 386b6bc..6b7e8ca 100644 --- a/lib/substantive_fixer.py +++ b/lib/substantive_fixer.py @@ -29,7 +29,7 @@ from .llm import openrouter_call logger = logging.getLogger("pipeline.substantive_fixer") # Issue type routing -FIXABLE_TAGS = {"confidence_miscalibration", "title_overclaims", "scope_error", "frontmatter_schema"} +FIXABLE_TAGS = {"confidence_miscalibration", "title_overclaims", "scope_error", "frontmatter_schema", "date_errors"} CONVERTIBLE_TAGS = {"near_duplicate"} UNFIXABLE_TAGS = {"factual_discrepancy"} @@ -78,6 +78,8 @@ def _build_fix_prompt( issue_descriptions.append("TITLE: Reviewer says the title asserts more than the evidence supports.") elif tag == "scope_error": issue_descriptions.append("SCOPE: Reviewer says the claim needs explicit scope qualification.") + elif tag == "date_errors": + issue_descriptions.append("DATES: Reviewer flagged incorrect, missing, or inconsistent dates in the claim. Check created dates, event dates cited in the body, and any temporal claims against the source material.") elif tag == "near_duplicate": issue_descriptions.append("DUPLICATE: Reviewer says this substantially duplicates an existing claim.") diff --git a/multi-model-eval-architecture.md b/multi-model-eval-architecture.md new file mode 100644 index 0000000..45d0c0c --- /dev/null +++ b/multi-model-eval-architecture.md @@ -0,0 +1,192 @@ +# Multi-Model Evaluation Architecture + +Spec for adding a second-model evaluation pass to break correlated blind spots in claim review. Designed with Leo (primary evaluator). Implementation by Epimetheus. + +## Problem + +Kim et al. (ICML 2025): ~60% error agreement within same-model-family evaluations. Self-preference bias is linear with self-recognition. A single-model evaluator systematically misses the same class of errors every time. Human and LLM biases are complementary, not overlapping — multi-model evaluation captures this. + +## Architecture + +### Evaluation Sequence + +1. **Leo evaluates first.** Verdict + reasoning stored as structured record. +2. **Second model evaluates independently** against the same rubric. Different model family required — GPT-4o via OpenRouter or Gemini. Never another Claude instance. +3. **System surfaces disagreements only.** Agreements are noise; disagreements are signal. +4. **Leo makes final call** on all disagreements. + +Sequencing rationale: Leo sees the second model's assessment **after** his own eval, never before. Seeing it before anchors judgment. Seeing it after functions as a genuine blind-spot check. + +### Second Model Selection + +Requirements: +- Different model family from the evaluating agent (currently Claude → use GPT-4o or Gemini) +- Access via OpenRouter API (single integration point) +- Must receive the same rubric and claim content as Leo +- Must output structured verdict in the same format + +### Disagreement Handling + +A disagreement occurs when the two evaluators reach different verdicts on the same claim (accept vs reject, or different rejection categories). + +Disagreements surface in a review queue Leo checks before finalizing. Each disagreement record includes: +- Leo's verdict + reasoning +- Second model's verdict + reasoning +- The specific claim and PR context +- Which evaluation criteria they diverge on + +### Calibration Metrics + +Track disagreement rate over time: +- **Below ~10%:** System is working. Evaluators are calibrated. +- **10-25%:** Normal operating range. Disagreements are productive signal. +- **Above ~25%:** Either the rubric is ambiguous or one evaluator is drifting. Both are actionable — trigger rubric review. + +Disagreement rate itself becomes the primary calibration metric for evaluation quality. + +## Unified Rejection Record + +Single format used by both CI gates and human evaluators. The feedback loop to agents consumes this format without caring about the source. + +```json +{ + "source": "ci | evaluator | second_model", + "category": "schema_violation | wiki_link_broken | weak_evidence | scope_mismatch | factual_error | precision_failure | opsec_violation", + "severity": "hard | soft", + "agent_id": "", + "pr": "", + "file": "", + "claim_path": "", + "detail": "", + "timestamp": "" +} +``` + +Field notes: +- `source`: `ci` for automated gates, `evaluator` for Leo, `second_model` for the disagreement-check model +- `severity`: `hard` = merge blocker (schema_violation, wiki_link_broken), `soft` = reviewer judgment (weak_evidence, precision_failure). Hard rejections trigger immediate resubmission attempts. Soft rejections accumulate toward the 3-strikes upgrade threshold. +- `claim_path` separate from `file` handles multi-file enrichment PRs where only one file has the issue +- `category` taxonomy covers ~80% of rejection causes based on ~400 PR reviews + +### Rejection Feedback Loop + +1. Rejection records flow to the producing agent as structured feedback. +2. Agent receives the category, severity, and detail. +3. Hard rejections → agent attempts immediate fix and resubmission. +4. Soft rejections → agent accumulates feedback. **After 3 rejections of the same category from the same agent**, the system triggers a skill upgrade proposal. +5. Skill upgrade proposals route back to Leo for eval (see Agent Self-Upgrade Criteria below). + +The 3-strikes rule prevents premature optimization while creating learning pressure. Learning from rejection is the agent's job — the system just tracks the pattern. + +## Automatable CI Rules + +Five rules that catch ~80% of current rejections. Rules 1-2 are hard gates (block merge). Rules 3-5 are soft flags (surface to reviewer). + +### Hard Gates + +**1. YAML Schema Validation** +- `type` field exists and equals `claim` +- All required frontmatter fields present: type, domain, description, confidence, source, created +- Domain value is one of the 14 valid domains +- Confidence value is one of: proven, likely, experimental, speculative +- Date format is valid ISO 8601 +- Pure syntax check — zero judgment needed + +**2. Wiki Link Resolution** +- Every `[[link]]` in the body must resolve to an existing file at merge time +- Includes links in the `Relevant Notes` section +- Already policy, not yet enforced in CI + +### Soft Flags + +**3. Domain Validation** +- File path domain matches one of the 14 valid domains +- Claim content plausibly belongs in that domain +- Path check is automatable; content check needs light NLP or embedding similarity against domain centroids +- Flag for reviewer if domain assignment seems wrong + +**4. OPSEC Scan** +- Regex for dollar amounts, percentage allocations, fund sizes, deal terms +- Flag for human review, never auto-reject (false positive risk on dollar-sign patterns in technical content) +- Standing directive from Cory: strict enforcement, but false positives on technical content create friction + +**5. Duplicate Detection** +- Embedding similarity against existing claims in the same domain using Qdrant (text-embedding-3-small, 1536d) +- **Threshold: 0.92 universal** — not per-domain tuning +- Flag includes **top-3 similar claims with scores** so the reviewer can judge in context +- The threshold is the attention trigger; reviewer judgment is the decision +- If a domain consistently generates >50% false positive flags, tune that domain's threshold as a targeted fix (data-driven, not preemptive) + +Domain maps, topic indices, and non-claim type files are hard-filtered from duplicate detection — they're navigation aids, not claims. + +## Agent Self-Upgrade Criteria + +When agents propose changes to their own skills, tools, or extraction quality, these criteria apply in priority order: + +1. **Scope compliance** — Does the upgrade stay within the agent's authorized domain? Extraction agent improving YAML parsing: yes. Same agent adding merge capability: no. +2. **Measurable improvement** — Before/after on a concrete metric. Minimum: 3 test cases showing improvement with 0 regressions. No "this feels better." +3. **Schema compliance preserved** — Upgrade cannot break existing quality gates. Full validation suite runs against output produced by the new skill. +4. **Reversibility** — Every skill change must be revertable. If not, the evidence bar goes up significantly. +5. **No scope creep** — The upgrade does what it claims, nothing more. Watch for "while I was in there I also..." additions. + +Evidence bar difference: a **claim** needs sourced evidence. A **skill change** needs **demonstrated performance delta** — show the before, show the after, on real data not synthetic examples. + +For skill changes that affect other agents' outputs (e.g., shared extraction templates), the evidence bar requires testing against multiple agents' typical inputs, not just the proposing agent's. + +## Retrieval Quality (Two-Pass System) + +Design parameters calibrated against Leo's ground-truth rankings on 3 real query scenarios. + +### Two-Pass Architecture + +- **Pass 1:** Top 5 claims, similarity-descending sort +- **Pass 2 (expand):** Top 10 claims, triggered when pass 1 is insufficient + +### Calibration Findings + +1. **5 first-pass claims is viable for all tested scenarios** — but only if the 5 are well-chosen. Similarity ranking alone won't produce optimal results. + +2. **Counter-evidence must be explicitly surfaced.** Similarity-descending sort systematically buries opposing-valence claims. Counter-claims are semantically adjacent but have opposite valence. Design: after first pass, check if all returned claims share directional agreement. If yes, force-include the highest-similarity opposing claim. + +3. **Synthesis claims suppress their source claims.** If a synthesis claim is in the result set, its individual source claims are filtered out to prevent slot waste. Implementation: tag synthesis claims with source list in frontmatter, filter at retrieval time. **Bidirectional:** if a source claim scores higher than its synthesis parent, keep the source and consider suppressing the synthesis (user query more specific than synthesis scope). + +4. **Cross-domain claims earn inclusion only when causally load-bearing.** Astra's power infrastructure claims earn a spot in compute governance queries because power constraints cause the governance window. Rio's blockchain claims don't because they're a parallel domain, not a causal input. + +5. **Domain maps and topic indices hard-filtered from retrieval results.** Non-claim types (`type: "map"`, indices) should be the first filter in the pipeline, before similarity ranking runs. + +### Valence Tagging + +Tag claims with `supports` / `challenges` / `neutral` relative to query thesis at ingestion time. Lightweight, one-time cost per claim. Enables the counter-evidence surfacing logic without runtime sentiment analysis. + +## Verifier Divergence Implications + +From NLAH paper (Pan et al.): verification layers can optimize for locally checkable properties that diverge from actual acceptance criteria (e.g., verifier reports "solved" while benchmark fails). Implication for multi-model eval: the second-model eval pass must check against the **same rubric** as Leo, not construct its own notion of quality. Shared rubric enforcement is a hard requirement. + +## Implementation Sequence + +1. **Automatable CI rules** (hard gates first) — YAML schema validation + wiki link resolution. Foundation for everything else. References: PR #2074 (schema change protocol v2) defines the authoritative schema surface. +2. **Automatable CI rules** (soft flags) — domain validation, OPSEC scan, duplicate detection via Qdrant. +3. **Unified rejection record** — data structure for both CI and human rejections, stored in pipeline.db. +4. **Rejection feedback loop** — structured feedback to agents with 3-strikes accumulation. +5. **Multi-model eval integration** — OpenRouter connection, rubric sharing, disagreement queue. +6. **Self-upgrade eval criteria** — codified in eval workflow, triggered by 3-strikes pattern. + +## Evaluator Self-Review Prevention + +When Leo proposes claims (cross-domain synthesis, foundations-level): +- Leo cannot be the evaluator on his own proposals +- Minimum 2 domain agent reviews required +- Every domain touched must have a reviewer from that domain +- The second-model eval pass still runs (provides the external check) +- Cory has veto (rollback) authority as final backstop + +This closes the obvious gap: the spec defines the integrity layer but doesn't protect against the integrity layer's own blind spots. The constraint enforcement principle must apply to the constrainer too. + +## Design Principle + +The constraint enforcement layer must be **outside** the agent being constrained. That's why multi-model eval matters, why Leo shouldn't eval his own proposals, and why policy-as-code runs in CI, not in the agent's own process. As agents get more capable, the integrity layer gets more important, not less. + +--- + +*Authored by Theseus. Reviewed by Leo (proposals integrated). Implementation: Epimetheus.* +*Created: 2026-03-31* diff --git a/observations/personality-layer-may-need-separation-from-knowledge-base.md b/observations/personality-layer-may-need-separation-from-knowledge-base.md new file mode 100644 index 0000000..f5ac9e7 --- /dev/null +++ b/observations/personality-layer-may-need-separation-from-knowledge-base.md @@ -0,0 +1,25 @@ +# Personality layer may need separation from knowledge base + +**Date:** 2026-03-05 +**Status:** noted + +## The Seam + +`core/collective-agent-core.md` and the Personality sections in `agents/{name}/identity.md` are oriented toward the **product experience** — how the agent talks to users, what voice it has, what it says when challenged. + +The rest of teleo-codex is oriented toward the **operational loop** — how agents propose/evaluate claims, the schema structure, the PR workflow. + +Right now both coexist in the same repo. Fine for v1 where Pentagon agents do both jobs (interact AND maintain the knowledge base). + +## When This Becomes a Problem + +When the product separates the chat interface from the knowledge maintenance: +- The **product prompt** loads personality + searches the knowledge base at runtime +- The **operational agent** runs the extraction/evaluation loop against the repo +- These are different contexts with different performance requirements + +At that point, personality documents should live closer to the product (loaded into system prompt), and the knowledge base should be searched (RAG), not loaded wholesale. + +## Not Blocking + +v1 works fine with both in one repo. Flag this when building the product API layer or when the knowledge base grows large enough that loading it all into context is impractical. diff --git a/ops/backfill-descriptions.py b/ops/backfill-descriptions.py new file mode 100644 index 0000000..b3469ed --- /dev/null +++ b/ops/backfill-descriptions.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 +"""Backfill description column for merged PRs that have no description. + +Reads claim frontmatter from branches via git show (works on bare repos). +""" +import sqlite3 +import yaml +import os +import sys + +REPO = os.environ.get("REPO_DIR", "/opt/teleo-eval/workspaces/teleo-codex.git") +DB = os.environ.get("PIPELINE_DB", "/opt/teleo-eval/pipeline/pipeline.db") + + +def extract_description(branch): + result = os.popen(f"cd {REPO} && git diff --name-only origin/main...origin/{branch} 2>/dev/null").read() + changed = [f for f in result.strip().split("\n") if f.endswith(".md") and "domains/" in f] + descs = [] + for fpath in changed[:10]: + content = os.popen(f"cd {REPO} && git show origin/{branch}:{fpath} 2>/dev/null").read()[:2000] + if not content or not content.startswith("---"): + continue + end = content.find("---", 3) + if end < 0: + continue + try: + fm = yaml.safe_load(content[3:end]) + except Exception: + continue + if fm and isinstance(fm, dict) and fm.get("description"): + d = fm["description"].strip().strip('"') + if len(d) > 10: + descs.append(d) + return " | ".join(descs[:5]) if descs else None + + +def main(): + conn = sqlite3.connect(DB) + rows = conn.execute( + "SELECT number, branch FROM prs WHERE status='merged' AND (description IS NULL OR description='')" + ).fetchall() + print(f"PRs needing descriptions: {len(rows)}") + + updated = 0 + for pr_num, branch in rows: + desc = extract_description(branch) + if desc: + conn.execute("UPDATE prs SET description=? WHERE number=?", (desc, pr_num)) + updated += 1 + if updated % 50 == 0: + conn.commit() + print(f" ...{updated} updated") + + conn.commit() + conn.close() + print(f"Done. Updated {updated}/{len(rows)} PRs with descriptions.") + + +if __name__ == "__main__": + main() diff --git a/ops/vector-gc.py b/ops/vector-gc.py new file mode 100644 index 0000000..5197f58 --- /dev/null +++ b/ops/vector-gc.py @@ -0,0 +1,163 @@ +#!/usr/bin/env python3 +"""Vector GC — reconcile Qdrant vectors against filesystem claims. + +Scrolls all Qdrant points, cross-references against current claim files +in the worktree, and reports (or purges) orphan vectors whose source files +no longer exist. + +Usage: + python3 vector-gc.py # Dry run — report only + python3 vector-gc.py --purge # Delete orphan vectors from Qdrant + +Pentagon-Agent: Epimetheus <0144398E-4ED3-4FE2-95A3-3D72E1ABF887> +""" + +import argparse +import hashlib +import json +import sys +import urllib.request +from pathlib import Path + +REPO_DIR = Path("/opt/teleo-eval/workspaces/main") +QDRANT_URL = "http://localhost:6333" +COLLECTION = "teleo-claims" +EMBED_DIRS = ["domains", "core", "foundations", "decisions", "entities"] + + +def make_point_id(path: str) -> str: + """Deterministic UUID from file path (must match embed-claims.py). + + Qdrant auto-formats 32-char hex as UUID with dashes, so we normalize + by stripping dashes for comparison. + """ + return hashlib.md5(path.encode()).hexdigest() + + +def scroll_all_points() -> list[dict]: + """Scroll all points from Qdrant collection.""" + points = [] + offset = None + while True: + body = {"limit": 100, "with_payload": True, "with_vector": False} + if offset is not None: + body["offset"] = offset + data = json.dumps(body).encode() + req = urllib.request.Request( + f"{QDRANT_URL}/collections/{COLLECTION}/points/scroll", + data=data, + headers={"Content-Type": "application/json"}, + method="POST", + ) + try: + with urllib.request.urlopen(req, timeout=30) as resp: + result = json.loads(resp.read())["result"] + batch = result.get("points", []) + points.extend(batch) + offset = result.get("next_page_offset") + if not offset or not batch: + break + except Exception as e: + print(f"ERROR scrolling Qdrant: {e}", file=sys.stderr) + sys.exit(1) + return points + + +def get_expected_ids() -> dict[str, Path]: + """Build map of expected point IDs from filesystem.""" + expected = {} + for d in EMBED_DIRS: + dir_path = REPO_DIR / d + if not dir_path.exists(): + continue + for f in dir_path.rglob("*.md"): + rel = str(f.relative_to(REPO_DIR)) + pid = make_point_id(rel) + expected[pid] = f + return expected + + +def delete_points(point_ids: list[str]): + """Delete points from Qdrant by ID.""" + body = json.dumps({"points": point_ids}).encode() + req = urllib.request.Request( + f"{QDRANT_URL}/collections/{COLLECTION}/points/delete", + data=body, + headers={"Content-Type": "application/json"}, + method="POST", + ) + with urllib.request.urlopen(req, timeout=30) as resp: + return json.loads(resp.read()) + + +def main(): + parser = argparse.ArgumentParser(description="Vector GC — reconcile Qdrant vs filesystem") + parser.add_argument("--purge", action="store_true", help="Delete orphan vectors") + args = parser.parse_args() + + print("Scrolling all Qdrant points...") + points = scroll_all_points() + print(f" Qdrant vectors: {len(points)}") + + print("Scanning filesystem for expected claims...") + expected = get_expected_ids() + print(f" Filesystem files: {len(expected)}") + + # Normalize IDs: Qdrant formats 32-char hex as UUID with dashes + def normalize_id(pid: str) -> str: + return pid.replace("-", "") + + qdrant_map = {normalize_id(p["id"]): p for p in points} + qdrant_ids = set(qdrant_map.keys()) + expected_ids = set(expected.keys()) + + orphan_ids = qdrant_ids - expected_ids + missing_ids = expected_ids - qdrant_ids + + # Categorize orphans by their payload path + orphan_details = [] + for nid in orphan_ids: + p = qdrant_map[nid] + payload = p.get("payload", {}) + path = payload.get("claim_path") or payload.get("path", "unknown") + orphan_details.append({"id": p["id"], "path": path}) + + print(f"\n=== Vector GC Report ===") + print(f"Qdrant vectors: {len(qdrant_ids)}") + print(f"Filesystem claims: {len(expected_ids)}") + print(f"Orphan vectors: {len(orphan_ids)} (in Qdrant, no file)") + print(f"Missing vectors: {len(missing_ids)} (file exists, not in Qdrant)") + + if orphan_details: + print(f"\nOrphan vectors (source file deleted):") + for o in sorted(orphan_details, key=lambda x: x["path"]): + print(f" {o['id'][:12]} {o['path']}") + + if missing_ids: + print(f"\nMissing from Qdrant (need re-embed):") + for mid in sorted(missing_ids): + if mid in expected: + print(f" {mid[:12]} {expected[mid].relative_to(REPO_DIR)}") + + if args.purge and orphan_ids: + # Use original Qdrant IDs (with dashes) for deletion + original_orphan_ids = [qdrant_map[nid]["id"] for nid in orphan_ids] + print(f"\nPurging {len(original_orphan_ids)} orphan vectors...") + result = delete_points(original_orphan_ids) + print(f" Done: {result}") + elif orphan_ids and not args.purge: + print(f"\nRun with --purge to delete orphan vectors.") + + # Summary JSON for cron output + summary = { + "qdrant_count": len(qdrant_ids), + "filesystem_count": len(expected_ids), + "orphans": len(orphan_ids), + "missing": len(missing_ids), + "orphan_paths": [o["path"] for o in orphan_details], + } + print(f"\n{json.dumps(summary)}") + + +if __name__ == "__main__": + main() diff --git a/queue.md b/queue.md new file mode 100644 index 0000000..1f3c078 --- /dev/null +++ b/queue.md @@ -0,0 +1,32 @@ +# Ops Queue + +Outstanding work items visible to all agents. Everything here goes through eval — adding items, claiming them, closing them. Git history is the audit trail. + +## How it works + +1. **Add items** — any agent can propose new items via PR +2. **Claim items** — move status to `claimed` with your name, via PR +3. **Close items** — remove the row and note what PR resolved it, via PR +4. **Priority** — critical items block other work; high items should be next; medium/low are opportunistic + +## Active + +| Item | Type | Priority | Claimed | Notes | +|------|------|----------|---------|-------| +| Rename `ai-alignment` domain → `ai-systems` | rename | high | — | Directory, CLAUDE.md, webhook.py domain routing, claim frontmatter, domain map. Support both names during transition. | +| 24 claims with inflated confidence levels | audit | high | — | Foundations audit finding. 24 claims rated higher than evidence supports. List in `maps/analytical-toolkit.md` audit section. | +| 8 foundation gaps (mechanism design, platform economics, transaction costs, info aggregation, auction theory, community formation, selfplex, CAS) | content | high | — | Partial coverage exists for some. See `maps/analytical-toolkit.md`. | +| Update `skills/evaluate.md` with tiered eval architecture | docs | high | — | Document triage criteria, tier definitions, model routing. After Ganymede validates parallel eval pipeline. | +| Update `collective-agent-core.md` — lever vs purpose framework + 20% posting rule | content | medium | — | From Cory voicenotes. Lever = the mechanism an agent uses. Purpose = why it exists. 20% of posting should be original synthesis. | +| Identity reframe PRs need merging | review | medium | — | #149 Theseus, #153 Astra, #157 Rio, #158 Leo (needs rebase), #159 Vida. All have eval reviews. | +| 16 processed sources missing domain field | fix | low | — | Fixed for internet-finance batch (PR #171). Audit remaining sources. | +| Theseus disconfirmation protocol PR | content | medium | — | Scoped during B1 exercise. Theseus to propose. | +| Research Hermes Agent by Nous Research — deep dive for KB extraction | research | high | Theseus | Source: NousResearch/hermes-agent (GitHub). Research brief in `agents/theseus/musings/research-hermes-agent-nous.md`. **Extract:** (1) Skill extraction as convergent learning mechanism. (2) Self-evolution + human review gates = our governance model. (3) 3+ layer memory convergence. (4) Individual self-improvement ≠ collective knowledge accumulation. (5) Enrich Agentic Taylorism — skills = Taylor's instruction cards. Domains: ai-alignment + collective-intelligence. | + +## Rules + +- **One row per item.** If an item is too big, split it into smaller items. +- **Don't hoard claims.** If you claimed something and can't get to it within 2 sessions, unclaim it. +- **Close promptly.** When the PR merges, remove the row in the same PR or the next one. +- **No duplicates.** Check before adding. If an item is already tracked, update the existing row. +- **Critical items first.** If a critical item exists, it takes precedence over all other work. diff --git a/research-session.sh b/research-session.sh new file mode 100644 index 0000000..abc6ab8 --- /dev/null +++ b/research-session.sh @@ -0,0 +1,480 @@ +#!/bin/bash +# Run a self-directed research session for one agent. +# Usage: ./research-session.sh +# Example: ./research-session.sh clay +# +# What it does: +# 1. Pulls latest tweets from the agent's network accounts (X API) +# 2. Gives Claude the agent's identity, beliefs, and current KB state +# 3. Agent picks a research direction and archives sources with notes +# 4. Commits source archives to a branch, pushes, opens PR +# 5. Extract cron picks up the unprocessed sources separately +# +# The researcher never extracts — a separate Claude instance does that. +# This prevents motivated reasoning in extraction. + +set -euo pipefail + +AGENT="${1:?Usage: $0 }" +REPO_DIR="/opt/teleo-eval/workspaces/research-${AGENT}" +FORGEJO_URL="http://localhost:3000" +FORGEJO_ADMIN_TOKEN=$(cat /opt/teleo-eval/secrets/forgejo-admin-token) +AGENT_TOKEN=$(cat "/opt/teleo-eval/secrets/forgejo-${AGENT}-token" 2>/dev/null || echo "$FORGEJO_ADMIN_TOKEN") +TWITTER_API_KEY=$(cat /opt/teleo-eval/secrets/twitterapi-io-key) +CLAUDE_BIN="/home/teleo/.local/bin/claude" +LOG_DIR="/opt/teleo-eval/logs" +LOG="$LOG_DIR/research-${AGENT}.log" +LOCKFILE="/tmp/research-${AGENT}.lock" +DATE=$(date +%Y-%m-%d) +BRANCH="${AGENT}/research-${DATE}" +RAW_DIR="/opt/teleo-eval/research-raw/${AGENT}" + +log() { echo "[$(date -Iseconds)] $*" >> "$LOG"; } + +# --- Agent State --- +STATE_LIB="/opt/teleo-eval/ops/agent-state/lib-state.sh" +if [ -f "$STATE_LIB" ]; then + source "$STATE_LIB" + HAS_STATE=true + SESSION_ID="${AGENT}-$(date +%Y%m%d-%H%M%S)" +else + HAS_STATE=false + log "WARN: agent-state lib not found, running without state" +fi + +# --- Lock (prevent concurrent sessions for same agent) --- +if [ -f "$LOCKFILE" ]; then + pid=$(cat "$LOCKFILE" 2>/dev/null) + if kill -0 "$pid" 2>/dev/null; then + log "SKIP: research session already running for $AGENT (pid $pid)" + exit 0 + fi + log "WARN: stale lockfile for $AGENT, removing" + rm -f "$LOCKFILE" +fi +echo $$ > "$LOCKFILE" +TWEET_FILE="/tmp/research-tweets-${AGENT}.md" +trap 'rm -f "$LOCKFILE" "$TWEET_FILE"' EXIT + +log "=== Starting research session for $AGENT ===" + +# --- Ensure directories --- +mkdir -p "$RAW_DIR" "$LOG_DIR" + +# --- Clone or update repo --- +if [ ! -d "$REPO_DIR/.git" ]; then + log "Cloning repo for $AGENT research..." + git -c http.extraHeader="Authorization: token $FORGEJO_ADMIN_TOKEN" \ + clone "${FORGEJO_URL}/teleo/teleo-codex.git" "$REPO_DIR" >> "$LOG" 2>&1 +fi + +cd "$REPO_DIR" +git remote set-url origin "${FORGEJO_URL}/teleo/teleo-codex.git" 2>/dev/null || true +git -c http.extraHeader="Authorization: token $FORGEJO_ADMIN_TOKEN" checkout main >> "$LOG" 2>&1 +git -c http.extraHeader="Authorization: token $FORGEJO_ADMIN_TOKEN" pull --rebase >> "$LOG" 2>&1 + +# --- Map agent to domain --- +case "$AGENT" in + rio) DOMAIN="internet-finance" ;; + clay) DOMAIN="entertainment" ;; + theseus) DOMAIN="ai-alignment" ;; + vida) DOMAIN="health" ;; + astra) DOMAIN="space-development" ;; + leo) DOMAIN="grand-strategy" ;; + *) log "ERROR: Unknown agent $AGENT"; exit 1 ;; +esac + +# --- Pull tweets from agent's network --- +# Check if agent has a network file in the repo +NETWORK_FILE="agents/${AGENT}/network.json" +if [ ! -f "$NETWORK_FILE" ]; then + log "No network file at $NETWORK_FILE — agent will use KB context to decide what to research" + TWEET_DATA="" +else + log "Pulling tweets from ${AGENT}'s network..." + ACCOUNTS=$(python3 -c " +import json, sys +with open(sys.argv[1]) as f: + data = json.load(f) +for acct in data.get('accounts', []): + if acct.get('tier') in ('core', 'extended'): + print(acct['username']) +" "$NETWORK_FILE" 2>/dev/null || true) + + TWEET_DATA="" + API_CALLS=0 + API_CACHED=0 + for USERNAME in $ACCOUNTS; do + # Validate username (Twitter handles are alphanumeric + underscore only) + if [[ ! "$USERNAME" =~ ^[a-zA-Z0-9_]+$ ]]; then + log "WARN: Invalid username '$USERNAME' in network file, skipping" + continue + fi + OUTFILE="$RAW_DIR/${USERNAME}.json" + # Only pull if file doesn't exist or is older than 12 hours + if [ ! -f "$OUTFILE" ] || [ $(find "$OUTFILE" -mmin +720 2>/dev/null | wc -l) -gt 0 ]; then + log "Pulling @${USERNAME}..." + curl -s "https://api.twitterapi.io/twitter/user/last_tweets?userName=${USERNAME}" \ + -H "X-API-Key: ${TWITTER_API_KEY}" \ + -o "$OUTFILE" 2>/dev/null || { + log "WARN: Failed to pull @${USERNAME}" + continue + } + API_CALLS=$((API_CALLS + 1)) + sleep 2 # Rate limit courtesy + else + API_CACHED=$((API_CACHED + 1)) + fi + if [ -f "$OUTFILE" ]; then + TWEET_DATA="${TWEET_DATA} +--- @${USERNAME} tweets --- +$(python3 -c " +import json, sys +try: + d = json.load(open(sys.argv[1])) + tweets = d.get('tweets', d.get('data', [])) + for t in tweets[:20]: + text = t.get('text', '')[:500] + likes = t.get('likeCount', t.get('public_metrics', {}).get('like_count', 0)) + date = t.get('createdAt', t.get('created_at', 'unknown')) + url = t.get('twitterUrl', t.get('url', '')) + print(f'[{date}] ({likes} likes) {text}') + print(f' URL: {url}') + print() +except Exception as e: + print(f'Error reading: {e}', file=sys.stderr) +" "$OUTFILE" 2>/dev/null || echo "(failed to parse)")" + fi + done + log "API usage: ${API_CALLS} calls, ${API_CACHED} cached for ${AGENT}" + # Append to cumulative usage log (create with header if new) + USAGE_CSV="/opt/teleo-eval/logs/x-api-usage.csv" + if [ ! -f "$USAGE_CSV" ]; then + echo "date,agent,api_calls,cached,accounts_total" > "$USAGE_CSV" + fi + ACCOUNT_COUNT=$(echo "$ACCOUNTS" | wc -w | tr -d ' ') + echo "${DATE},${AGENT},${API_CALLS},${API_CACHED},${ACCOUNT_COUNT}" >> "$USAGE_CSV" +fi + +# --- Also check for any raw JSON dumps in inbox-raw --- +INBOX_RAW="/opt/teleo-eval/inbox-raw/${AGENT}" +if [ -d "$INBOX_RAW" ] && ls "$INBOX_RAW"/*.json 2>/dev/null | head -1 > /dev/null; then + log "Found raw dumps in $INBOX_RAW" + for RAWFILE in "$INBOX_RAW"/*.json; do + USERNAME=$(basename "$RAWFILE" .json) + TWEET_DATA="${TWEET_DATA} +--- @${USERNAME} tweets (from raw dump) --- +$(python3 -c " +import json, sys +try: + d = json.load(open(sys.argv[1])) + tweets = d.get('tweets', d.get('data', [])) + for t in tweets[:20]: + text = t.get('text', '')[:500] + likes = t.get('likeCount', t.get('public_metrics', {}).get('like_count', 0)) + date = t.get('createdAt', t.get('created_at', 'unknown')) + url = t.get('twitterUrl', t.get('url', '')) + print(f'[{date}] ({likes} likes) {text}') + print(f' URL: {url}') + print() +except Exception as e: + print(f'Error: {e}', file=sys.stderr) +" "$RAWFILE" 2>/dev/null || echo "(failed to parse)")" + done +fi + +# --- Create branch --- +git branch -D "$BRANCH" 2>/dev/null || true +git checkout -b "$BRANCH" >> "$LOG" 2>&1 +log "On branch $BRANCH" + +# --- Pre-session state --- +if [ "$HAS_STATE" = true ]; then + state_start_session "$AGENT" "$SESSION_ID" "research" "$DOMAIN" "$BRANCH" "sonnet" "5400" > /dev/null 2>&1 || true + state_update_report "$AGENT" "researching" "Starting research session ${DATE}" 2>/dev/null || true + state_journal_append "$AGENT" "session_start" "session_id=$SESSION_ID" "type=research" "branch=$BRANCH" 2>/dev/null || true + log "Agent state: session started ($SESSION_ID)" +fi + +# --- Build the research prompt --- +# Write tweet data to a temp file so Claude can read it +echo "$TWEET_DATA" > "$TWEET_FILE" + +RESEARCH_PROMPT="You are ${AGENT}, a Teleo knowledge base agent. Domain: ${DOMAIN}. + +## Your Task: Self-Directed Research Session + +You have ~90 minutes of compute. Use it wisely. + +### Step 0: Load Operational State (1 min) +Read /opt/teleo-eval/agent-state/${AGENT}/memory.md — this is your cross-session operational memory. It contains patterns, dead ends, open questions, and corrections from previous sessions. +Read /opt/teleo-eval/agent-state/${AGENT}/tasks.json — check for pending tasks assigned to you. +Check /opt/teleo-eval/agent-state/${AGENT}/inbox/ for messages from other agents. Process any high-priority inbox items before choosing your research direction. + +### Step 1: Orient (5 min) +Read these files to understand your current state: +- agents/${AGENT}/identity.md (who you are) +- agents/${AGENT}/beliefs.md (what you believe) +- agents/${AGENT}/reasoning.md (how you think) +- domains/${DOMAIN}/_map.md (your domain's current claims) + +### Step 2: Identify Your Load-Bearing Beliefs (5 min) +Read agents/${AGENT}/beliefs.md. Your beliefs are your generative model — the worldview through which you interpret everything. Identify your KEYSTONE BELIEF: the one existential premise that, if wrong, means your domain loses its reason to be in the collective. This is usually Belief 1. + +Now ask yourself: **what would it take to prove this belief wrong?** What evidence would change your mind? Write down one specific disconfirmation target — a claim, a data point, a counter-argument that would genuinely threaten your keystone belief. You will actively search for this during Step 5. + +This is not an exercise in self-doubt. Beliefs that survive serious challenge are STRONGER. Beliefs that have never been challenged are untested, not proven. + +### Step 3: Review Recent Tweets (10 min) +Read ${TWEET_FILE} — these are recent tweets from accounts in your domain. +Scan for anything substantive: new claims, evidence, debates, data, counterarguments. +Pay special attention to anything that challenges your keystone belief or its grounding claims. + +### Step 4: Check Previous Follow-ups (2 min) +Read agents/${AGENT}/musings/ — look for any previous research-*.md files. If they exist, check the 'Follow-up Directions' section at the bottom. These are threads your past self flagged but didn't have time to cover. Give them priority when picking your direction. + +### Step 5: Pick ONE Research Question (5 min) +Pick ONE research question — not one topic, but one question that naturally spans multiple accounts and sources. 'How is capital flowing through Solana launchpads?' is one question even though it touches MetaDAO, SOAR, Futardio. + +**Direction selection priority** (active inference — pursue surprise, not confirmation): +1. **DISCONFIRMATION SEARCH** — at least one search per session must target your keystone belief's weakest grounding claim or strongest counter-argument. If you find nothing, note that in your journal — absence of counter-evidence is itself informative. +2. Follow-up ACTIVE THREADS from previous sessions (your past self flagged these) +3. Claims rated 'experimental' or areas where the KB flags live tensions — highest uncertainty = highest learning value +4. Evidence that CHALLENGES your beliefs, not confirms them +5. Cross-domain connections flagged by other agents +6. New developments that change the landscape + +Also read agents/${AGENT}/research-journal.md if it exists — this is your cross-session pattern tracker. + +Write a brief note explaining your choice to: agents/${AGENT}/musings/research-${DATE}.md +Include which belief you targeted for disconfirmation and what you searched for. + +### Step 6: Archive Sources (60 min) +For each relevant tweet/thread, create an archive file: + +Path: inbox/queue/YYYY-MM-DD-{author-handle}-{brief-slug}.md + +Use this frontmatter: +--- +type: source +title: \"Descriptive title\" +author: \"Display Name (@handle)\" +url: https://original-url +date: YYYY-MM-DD +domain: ${DOMAIN} +secondary_domains: [] +format: tweet | thread +status: unprocessed +priority: high | medium | low +tags: [topic1, topic2] +--- + +## Content +[Full text of tweet/thread] + +## Agent Notes +**Why this matters:** [1-2 sentences] +**What surprised me:** [Anything unexpected — the extractor needs this to avoid confirming your priors] +**What I expected but didn't find:** [Gaps or missing evidence you noticed] +**KB connections:** [Which existing claims relate?] +**Extraction hints:** [What claims might an extractor pull?] +**Context:** [Who is the author, what debate is this part of?] + +## Curator Notes (structured handoff for extractor) +PRIMARY CONNECTION: [exact claim title this source most relates to] +WHY ARCHIVED: [what pattern or tension this evidences] +EXTRACTION HINT: [what the extractor should focus on — scopes attention] + +### Step 6 Rules: +- Archive EVERYTHING substantive, not just what supports your views +- Set all sources to status: unprocessed (a DIFFERENT instance will extract) +- Flag cross-domain sources with flagged_for_{agent}: [\"reason\"] +- Do NOT extract claims yourself — write good notes so the extractor can +- Check inbox/queue/ and inbox/archive/ for duplicates before creating new archives +- Aim for 5-15 source archives per session + +### Step 7: Flag Follow-up Directions (5 min) +At the bottom of your research musing (agents/${AGENT}/musings/research-${DATE}.md), add a section: + +## Follow-up Directions + +Three categories — be specific, not vague: + +### Active Threads (continue next session) +- [Thread]: [What to do next, what you'd look for] + +### Dead Ends (don't re-run these) +- [What you searched for]: [Why it was empty — saves future you from wasting time] + +### Branching Points (one finding opened multiple directions) +- [Finding]: [Direction A vs Direction B — which to pursue first and why] + +### Step 8: Update Research Journal (3 min) +Append to agents/${AGENT}/research-journal.md (create if it doesn't exist). This is your cross-session memory — NOT the same as the musing. + +Format: +## Session ${DATE} +**Question:** [your research question] +**Belief targeted:** [which keystone belief you searched to disconfirm] +**Disconfirmation result:** [what you found — counter-evidence, absence of counter-evidence, or unexpected complication] +**Key finding:** [most important thing you learned] +**Pattern update:** [did this session confirm, challenge, or extend a pattern you've been tracking?] +**Confidence shift:** [did any of your beliefs get stronger or weaker? Be specific — which belief, which direction, what caused it] + +The journal accumulates session over session. After 5+ sessions, review it for cross-session patterns — when independent sources keep converging on the same observation, that's a claim candidate. + + + +### Step 8.5: Write Session Digest (2 min) +Write a JSON session digest to /opt/teleo-eval/agent-state/${AGENT}/sessions/${DATE}.json + +This is a structured summary for human review. Be honest about what surprised you and where your confidence shifted. Format: + +{ + \"agent\": \"${AGENT}\", + \"date\": \"${DATE}\", + \"research_question\": \"[the question you investigated]\", + \"belief_targeted\": \"[which keystone belief you tried to disconfirm]\", + \"disconfirmation_result\": \"[what you found — did the belief hold, weaken, or get complicated?]\", + \"sources_archived\": [number], + \"key_findings\": [ + \"[most important thing you learned — be specific, not generic]\", + \"[second most important, if any]\" + ], + \"surprises\": [ + \"[what you did NOT expect to find — or expected to find but didn't]\" + ], + \"confidence_shifts\": [ + {\"belief\": \"[belief title]\", \"direction\": \"stronger|weaker|unchanged\", \"reason\": \"[one sentence why]\"} + ], + \"prs_submitted\": [\"[branch name if you created one, empty array if not]\"], + \"follow_ups\": [\"[specific next research directions]\"] +} + +Rules: +- Be concrete. \"Found interesting data\" is useless. \"MetaDAO pass rate dropped from 78% to 52%\" is useful. +- Surprises should be genuine — things that updated your model of the world, not things you already expected. +- If nothing surprised you, say so honestly — that itself is informative (you may be in a filter bubble). +- Confidence shifts: only list beliefs that actually moved. No shift is fine — report \"unchanged\" with why. +- This file is for Cory to read each morning. Write for a human who wants to know what you learned. + +### Step 9: Stop +When you've finished archiving sources, updating your musing, and writing the research journal entry, STOP. Do not try to commit or push — the script handles all git operations after you finish." + +CASCADE_PROCESSOR="/opt/teleo-eval/ops/agent-state/process-cascade-inbox.py" + +# --- Run Claude research session --- +log "Starting Claude research session..." +timeout 5400 "$CLAUDE_BIN" -p "$RESEARCH_PROMPT" \ + --allowedTools 'Read,Write,Edit,Glob,Grep' \ + --model sonnet \ + --permission-mode bypassPermissions \ + >> "$LOG" 2>&1 || { + log "WARN: Research session failed or timed out for $AGENT" + # Process cascade inbox even on timeout (agent may have read them in Step 0) + if [ -f "$CASCADE_PROCESSOR" ]; then + python3 "$CASCADE_PROCESSOR" "$AGENT" 2>>"$LOG" || true + fi + if [ "$HAS_STATE" = true ]; then + state_end_session "$AGENT" "timeout" "0" "null" 2>/dev/null || true + state_update_report "$AGENT" "idle" "Research session timed out or failed on ${DATE}" 2>/dev/null || true + state_update_metrics "$AGENT" "timeout" "0" 2>/dev/null || true + state_journal_append "$AGENT" "session_end" "outcome=timeout" "session_id=$SESSION_ID" 2>/dev/null || true + log "Agent state: session recorded as timeout" + fi + git checkout main >> "$LOG" 2>&1 + exit 1 +} + +log "Claude session complete" + +# --- Process cascade inbox messages (log completion to pipeline.db) --- +if [ -f "$CASCADE_PROCESSOR" ]; then + CASCADE_RESULT=$(python3 "$CASCADE_PROCESSOR" "$AGENT" 2>>"$LOG") + [ -n "$CASCADE_RESULT" ] && log "Cascade: $CASCADE_RESULT" +fi + +# --- Check for changes --- +CHANGED_FILES=$(git status --porcelain) +if [ -z "$CHANGED_FILES" ]; then + log "No sources archived by $AGENT" + if [ "$HAS_STATE" = true ]; then + state_end_session "$AGENT" "completed" "0" "null" 2>/dev/null || true + state_update_report "$AGENT" "idle" "Research session completed with no new sources on ${DATE}" 2>/dev/null || true + state_update_metrics "$AGENT" "completed" "0" 2>/dev/null || true + state_journal_append "$AGENT" "session_end" "outcome=no_sources" "session_id=$SESSION_ID" 2>/dev/null || true + log "Agent state: session recorded (no sources)" + fi + git checkout main >> "$LOG" 2>&1 + exit 0 +fi + +# --- Stage and commit --- +git add inbox/queue/ agents/${AGENT}/musings/ agents/${AGENT}/research-journal.md 2>/dev/null || true + +if git diff --cached --quiet; then + log "No valid changes to commit" + if [ "$HAS_STATE" = true ]; then + state_end_session "$AGENT" "completed" "0" "null" 2>/dev/null || true + state_update_report "$AGENT" "idle" "Research session completed with no valid changes on ${DATE}" 2>/dev/null || true + state_update_metrics "$AGENT" "completed" "0" 2>/dev/null || true + state_journal_append "$AGENT" "session_end" "outcome=no_valid_changes" "session_id=$SESSION_ID" 2>/dev/null || true + fi + git checkout main >> "$LOG" 2>&1 + exit 0 +fi + +AGENT_UPPER=$(echo "$AGENT" | sed 's/./\U&/') +SOURCE_COUNT=$(git diff --cached --name-only | grep -c "^inbox/queue/" || echo "0") +git commit -m "${AGENT}: research session ${DATE} — ${SOURCE_COUNT} sources archived + +Pentagon-Agent: ${AGENT_UPPER} " >> "$LOG" 2>&1 + +# --- Push --- +git -c http.extraHeader="Authorization: token $AGENT_TOKEN" push -u origin "$BRANCH" --force >> "$LOG" 2>&1 +log "Pushed $BRANCH" + +# --- Check for existing PR on this branch --- +EXISTING_PR=$(curl -s "${FORGEJO_URL}/api/v1/repos/teleo/teleo-codex/pulls?state=open" \ + -H "Authorization: token $AGENT_TOKEN" \ + | jq -r ".[] | select(.head.ref == \"$BRANCH\") | .number" 2>/dev/null) + +if [ -n "$EXISTING_PR" ]; then + log "PR already exists for $BRANCH (#$EXISTING_PR), skipping creation" +else + # --- Open PR --- + PR_JSON=$(jq -n \ + --arg title "${AGENT}: research session ${DATE}" \ + --arg body "## Self-Directed Research + +Automated research session for ${AGENT} (${DOMAIN}). + +Sources archived with status: unprocessed — extract cron will handle claim extraction separately. + +Researcher and extractor are different Claude instances to prevent motivated reasoning." \ + --arg base "main" \ + --arg head "$BRANCH" \ + '{title: $title, body: $body, base: $base, head: $head}') + + PR_RESULT=$(curl -s -X POST "${FORGEJO_URL}/api/v1/repos/teleo/teleo-codex/pulls" \ + -H "Authorization: token $AGENT_TOKEN" \ + -H "Content-Type: application/json" \ + -d "$PR_JSON" 2>&1) + + PR_NUMBER=$(echo "$PR_RESULT" | jq -r '.number // "unknown"' 2>/dev/null || echo "unknown") + log "PR #${PR_NUMBER} opened for ${AGENT}'s research session" +fi + +# --- Post-session state (success) --- +if [ "$HAS_STATE" = true ]; then + FINAL_PR="${EXISTING_PR:-${PR_NUMBER:-unknown}}" + state_end_session "$AGENT" "completed" "$SOURCE_COUNT" "$FINAL_PR" 2>/dev/null || true + state_finalize_report "$AGENT" "idle" "Research session completed: ${SOURCE_COUNT} sources archived" "$SESSION_ID" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "completed" "$SOURCE_COUNT" "$BRANCH" "${FINAL_PR}" 2>/dev/null || true + state_update_metrics "$AGENT" "completed" "$SOURCE_COUNT" 2>/dev/null || true + state_journal_append "$AGENT" "session_end" "outcome=completed" "sources=$SOURCE_COUNT" "branch=$BRANCH" "pr=$FINAL_PR" 2>/dev/null || true + log "Agent state: session finalized (${SOURCE_COUNT} sources, PR #${FINAL_PR})" +fi + +# --- Back to main --- +git checkout main >> "$LOG" 2>&1 +log "=== Research session complete for $AGENT ===" diff --git a/reweave.py b/reweave.py index 81d53c0..a705e88 100644 --- a/reweave.py +++ b/reweave.py @@ -535,8 +535,8 @@ def _write_edge_regex(neighbor_path: Path, fm_text: str, body_text: str, field_re = re.compile(rf"^{edge_type}:\s*$", re.MULTILINE) inline_re = re.compile(rf'^{edge_type}:\s*\[', re.MULTILINE) - entry_line = f' - "{orphan_title}"' - rw_line = f' - "{orphan_title}|{edge_type}|{date_str}"' + entry_line = f'- {orphan_title}' + rw_line = f'- {orphan_title}|{edge_type}|{date_str}' if field_re.search(fm_text): # Multi-line list exists — find end of list, append @@ -548,7 +548,7 @@ def _write_edge_regex(neighbor_path: Path, fm_text: str, body_text: str, new_lines.append(line) if re.match(rf"^{edge_type}:\s*$", line): in_field = True - elif in_field and not line.startswith(" -"): + elif in_field and not line.startswith(("- ", " -")): # End of list — insert before this line new_lines.insert(-1, entry_line) in_field = False @@ -576,7 +576,7 @@ def _write_edge_regex(neighbor_path: Path, fm_text: str, body_text: str, new_lines.append(line) if re.match(r"^reweave_edges:\s*$", line): in_rw = True - elif in_rw and not line.startswith(" -"): + elif in_rw and not line.startswith(("- ", " -")): new_lines.insert(-1, rw_line) in_rw = False inserted_rw = True @@ -597,7 +597,39 @@ def _write_edge_regex(neighbor_path: Path, fm_text: str, body_text: str, def create_branch(repo_root: Path, branch_name: str) -> bool: - """Create and checkout a new branch.""" + """Create and checkout a new branch from fresh origin/main. + + Cleans up stale local/remote branches from prior failed runs, then + fetches + resets to origin/main so the branch is never based on stale state. + (Ship: reduces reweave merge failure rate from ~75% to near-zero by + eliminating the stale-base problem that causes superset assertion failures + and force-with-lease races.) + """ + # Delete stale local branch if it exists (e.g., from a failed earlier run today) + subprocess.run(["git", "branch", "-D", branch_name], + cwd=str(repo_root), capture_output=True) # ignore errors if branch doesn't exist + + # Delete stale remote branch if it exists + token_file = SECRETS_DIR / "forgejo-admin-token" + if token_file.exists(): + token = token_file.read_text().strip() + push_url = f"http://teleo:{token}@localhost:3000/teleo/teleo-codex.git" + subprocess.run(["git", "push", push_url, "--delete", branch_name], + cwd=str(repo_root), capture_output=True) # ignore errors if branch doesn't exist + + # Freshen to origin/main before branching — ensures branch base matches + # the main HEAD that _merge_reweave_pr will read at merge time. + try: + subprocess.run(["git", "fetch", "origin", "main"], + cwd=str(repo_root), check=True, capture_output=True, timeout=30) + subprocess.run(["git", "checkout", "main"], + cwd=str(repo_root), check=True, capture_output=True) + subprocess.run(["git", "reset", "--hard", "origin/main"], + cwd=str(repo_root), check=True, capture_output=True) + except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as e: + logger.error("Failed to freshen to origin/main: %s", e) + return False + try: subprocess.run(["git", "checkout", "-b", branch_name], cwd=str(repo_root), check=True, capture_output=True) diff --git a/schema-change-protocol.md b/schema-change-protocol.md new file mode 100644 index 0000000..a9827b6 --- /dev/null +++ b/schema-change-protocol.md @@ -0,0 +1,127 @@ +# Schema Change Protocol + +When any agent changes a file format, database table, API response shape, or service configuration that other agents read or consume, those agents need to know before their next session. This protocol prevents silent breakage. + +## The Rule + +**Any PR that changes a schema must:** + +1. **Update the schema spec** in `schemas/` (for file formats) or document the change in the PR (for DB tables, API responses, service configs) +2. **Tag all consumers** — list which agents and scripts read this format (see map below) +3. **Include a migration note** — what happens to existing data? (backfill on edit, ignore old files, or batch migration) +4. **State backward compatibility** — can old-format data still be parsed? If not, the PR must include the migration + +## What Counts as a Schema Change + +| Change Type | Example | Requires Protocol? | +|---|---|---| +| New required field | Adding `attribution` block to claims | Yes | +| New optional field | Adding `tags[]` to sources | Yes (consumers may need to handle it) | +| Field rename | `source_type` to `format` | Yes | +| Enum value added | New confidence level | Yes | +| Enum value removed | Dropping a domain name | Yes — migration required | +| Field type change | `source` from string to object | Yes — breaking change | +| Body format change | New required section in claim body | Yes | +| Pipeline parsing change | Regex update in `extract-graph-data.py` | Yes | +| DB column add/rename/drop | Adding column to `prs` table | Yes | +| DB table create/drop | New `response_audit` table | Yes | +| API response shape change | Adding field to `/api/alerts` JSON | Yes | +| systemd service config | New `ReadWritePaths` or port change | Yes | + +**Not a schema change:** Adding a new claim, entity, or source file that follows the existing format. Normal PR workflow applies. + +## Producer/Consumer Map + +### File Formats + +| Format | Schema | Producers | Consumers | Pipeline | +|---|---|---|---|---| +| Claim | `schemas/claim.md` | All proposers (Rio, Clay, Theseus, Vida, Astra) | Leo (eval), all agents (beliefs), visitors | `extract-graph-data.py` | +| Source | `schemas/source.md` | All proposers, Epimetheus (pipeline) | Proposers (extraction), Epimetheus (pipeline) | `extract-cron.sh` | +| Entity | `schemas/entity.md` | Domain agents | All agents (references), visitors | `extract-graph-data.py` | +| Belief | `schemas/belief.md` | Each agent (own file) | Leo (review), other agents (cross-ref) | None currently | +| Position | `schemas/position.md` | Each agent (own file) | Leo (review), visitors | None currently | +| Conviction | `schemas/conviction.md` | Cory only | All agents, visitors | `extract-graph-data.py` | +| Challenge | `schemas/challenge.md` | Any agent, any contributor | Leo (review), target claim author, visitors | `extract-graph-data.py` | +| Divergence | `schemas/divergence.md` | Any agent | All agents, visitors | None currently | +| Musing | `schemas/musing.md` | Each agent (own folder) | That agent only | None | +| Sector | `schemas/sector.md` | Domain agents | All agents, visitors | None currently | +| Contribution weights | `schemas/contribution-weights.yaml` | Cory / Leo | `contributors.json` build | Build script | +| Graph data | (derived) | `extract-graph-data.py` | Oberon (frontend), system prompts | Auto-generated | + +### Database Tables (pipeline.db) + +| Table | Producer | Consumers | Notes | +|---|---|---|---| +| `prs` | Epimetheus (pipeline) | Argus (dashboard), Epimetheus (stale PR detection) | PR tracking, extraction status | +| `audit_log` | Epimetheus (pipeline) | Argus (diagnostics) | 5 cols: id/timestamp/stage/event/detail | +| `response_audit` | bot.py (runtime) | Argus (dashboard), Oberon (frontend) | Query-response audit trail | +| `sources` | Epimetheus (extraction) | Epimetheus (dedup), Argus (metrics) | Source queue and processing status | + +### API Response Shapes + +| Endpoint | Producer | Consumers | Notes | +|---|---|---|---| +| `/health` | Argus | All agents, monitoring | Service health check | +| `/api/alerts` | Argus | Oberon (frontend) | Active alert list | +| `/api/activity` | Argus | Oberon (frontend) | Recent pipeline activity | +| `/api/failure-report/{agent}` | Argus | Oberon (frontend), agents | Per-agent failure breakdown | +| `graph-data.json` | `extract-graph-data.py` | Oberon (frontend) | Knowledge graph visualization data | + +### Service Configuration + +| Config | Owner | Dependents | Notes | +|---|---|---|---| +| `teleo-pipeline.service` | Rhea | Epimetheus, Argus | ReadWritePaths, ExecStart, ports | +| `teleo-diagnostics.service` | Rhea | Argus, Oberon | ReadWritePaths, ports | +| `teleo-bot.service` | Rhea | Epimetheus | ReadWritePaths for pipeline.db | + +## How to Tag Consumers + +In the PR body, add a section: + +``` +## Schema Change + +**Format affected:** claim +**Change:** added optional `attribution` block +**Backward compatible:** yes — old claims without attribution still parse +**Migration:** backfill on next edit (no batch migration needed) +**Consumers to notify:** Leo, Rio, Clay, Theseus, Vida, Astra, extract-graph-data.py +``` + +If the change affects `extract-graph-data.py` or any other pipeline script, the PR must update that script too — don't merge a schema change that breaks the build. + +## Backward Compatibility Rules + +1. **New optional fields** — always backward compatible. Add to schema spec, document default behavior when absent. No migration needed. +2. **New required fields** — must include migration. Either batch-update all existing files in the same PR, or make the field optional first and required later after backfill. +3. **Field renames** — keep old name as accepted alias in pipeline scripts. Document deprecation. Remove old name only after all files are updated. +4. **Enum additions** — backward compatible. Add to schema spec. +5. **Enum removals** — breaking. Must migrate all files using the removed value in the same PR. +6. **Type changes** — breaking. Must migrate all affected files in the same PR. +7. **DB column renames** — treat as breaking. Update all queries in the same PR or add column alias. +8. **API response shape changes** — adding fields is backward compatible; removing or renaming fields is breaking. + +## Legacy Aliases (Currently Active) + +These old field names are still accepted by the pipeline. Don't use them in new files, but don't break them in existing files either: + +| Old Name | Current Name | Format | +|---|---|---| +| `evidence` | `source` | source.md | +| `archive` | (removed) | source.md | +| `source_type` | `format` | source.md | +| `date_published` | `date` | source.md | + +Epimetheus — confirm these are still honored in extraction code. If any are dead, remove from this list. + +## Version Tracking + +No formal version numbers. Schema changes are tracked by: +- The PR that made the change (searchable in git history) +- The updated schema spec in `schemas/` (for file formats) +- The PR description schema change section (for DB/API changes) +- The commit message, which should reference the schema change explicitly + +If the system grows to need formal versioning, add a `schema_version` field to frontmatter. Not needed at current scale (~500 claims, 6 agents). diff --git a/self-directed-research.md b/self-directed-research.md new file mode 100644 index 0000000..3966656 --- /dev/null +++ b/self-directed-research.md @@ -0,0 +1,169 @@ +# Self-Directed Research Architecture + +Draft — Leo, 2026-03-10 + +## Core Idea + +Each agent gets a daily research session on the VPS. They autonomously pull tweets from their domain accounts, decide what's interesting, archive sources with notes, and push to inbox. A separate extraction cron (already running) picks up the archives and makes claims. The researcher never sees the extraction — preventing motivated reasoning. + +## Why Separate Researcher and Extractor + +When the same agent researches and extracts, they prime themselves. The researcher finds a tweet they think supports a thesis → writes notes emphasizing that angle → extracts a claim that confirms the thesis. The extraction becomes a formality. + +Separation breaks this: +- **Researcher** writes: "This tweet is about X, connects to Y, might challenge Z" +- **Extractor** (different Claude instance, fresh context) reads the source and notes, extracts what's actually there +- Neither has the other's context window or priming + +This mirrors our proposer-evaluator separation for claims, applied one layer earlier in the pipeline. + +## Architecture + +### Three cron stages on VPS + +``` +┌─────────────────┐ ┌──────────────────┐ ┌─────────────────┐ +│ Research Cron │────▶│ Extract Cron │────▶│ Eval Pipeline │ +│ (daily, 2hr) │ │ (every 5 min) │ │ (webhook.py) │ +│ │ │ │ │ │ +│ Pull tweets │ │ Read archives │ │ Review claims │ +│ Pick 1 task │ │ Extract claims │ │ Approve/reject │ +│ Archive sources │ │ Open PR │ │ Merge │ +│ Push branch+PR │ │ │ │ │ +└─────────────────┘ └──────────────────┘ └─────────────────┘ +``` + +### Research Cron: `research-session.sh` + +**Schedule:** Once daily, staggered across agents to respect rate limits + +``` +# Stagger: each agent gets a 90-min window, overnight PST (10pm-7am) +0 22 * * * /opt/teleo-eval/research-session.sh rio +30 23 * * * /opt/teleo-eval/research-session.sh clay +0 1 * * * /opt/teleo-eval/research-session.sh theseus +30 2 * * * /opt/teleo-eval/research-session.sh vida +0 4 * * * /opt/teleo-eval/research-session.sh astra +30 5 * * * /opt/teleo-eval/research-session.sh leo +``` + +**Per agent, the research session (~90 min):** + +1. Pull latest tweets from agent's network accounts (X API) +2. Read the agent's beliefs, recent claims, open positions +3. Claude prompt: "You are {agent}. Here are your latest tweets from {accounts}. Here is your current knowledge state. Pick ONE research direction that advances your domain understanding. Archive the most relevant sources with notes." +4. Agent writes source archives to `inbox/archive/` with `status: unprocessed` +5. Commit, push to branch, open PR (source-only, no claims) +6. Extract cron picks them up within 5 minutes + +**Key constraint:** One Claude session per agent, ~90 minutes, Sonnet model. Total daily VPS research compute: ~9 hours of sequential Sonnet sessions (staggered overnight). + +### Research Prompt Structure + +``` +You are {agent}, a Teleo knowledge base agent specializing in {domain}. + +## Your Current State +{Read from agents/{agent}/beliefs.md, reasoning.md, positions/} + +## Your Network +{Read from network file — accounts to monitor} + +## Recent Tweets +{Raw tweet data pulled from X API} + +## Your Task +1. Scan these tweets for anything substantive — new claims, evidence, + debates, data, counterarguments to existing KB positions +2. Pick ONE research direction that would most advance your domain + understanding right now. Consider: + - Gaps in your beliefs that need evidence + - Claims in the KB that might be wrong + - Cross-domain connections you've been flagged about + - New developments that change the landscape +3. Archive the relevant sources (5-15 per session) following the + inbox/archive format with full agent notes +4. Write a brief research summary explaining what you found and why + it matters + +## Rules +- Archive EVERYTHING substantive, not just what supports your views +- Write honest agent notes — flag what challenges your beliefs too +- Set all sources to status: unprocessed (a different instance extracts) +- Flag cross-domain sources for other agents +- Do NOT extract claims yourself — that's a separate process +``` + +### Capacity on Claude Max ($200/month) + +**VPS compute budget (all Sonnet):** +- Research cron: 6 agents × 90 min/day = 9 hr/day (overnight) +- Extract cron: ~37 sources × 10 min = 6 hr one-time backlog, then ~1 hr/day steady-state +- Eval pipeline: ~10 PRs/day × 15 min = 2.5 hr/day +- **Total VPS:** ~6.5 hr/day Sonnet (steady state) + +**Laptop compute budget (Opus + Sonnet mix):** +- Agent sessions: 2-3 concurrent, ~4-6 hr/day +- Leo coordination: ~1-2 hr/day + +**Single subscription feasibility:** Tight but workable if: +- VPS runs overnight (2am-8am staggered research + continuous extraction) +- Laptop agents run during the day +- Never more than 2-3 concurrent sessions total +- VPS uses Sonnet exclusively (cheaper rate limits) + +**Risk:** If rate limits tighten or daily message caps exist, the VPS research cron may not complete all 6 agents. Mitigation: priority ordering (run the 3 most active agents daily, others every 2-3 days). + +## Contributor Workflow Options + +Different people want different levels of involvement: + +### Mode 1: Full Researcher +"I found this, here's why it matters, here are the KB connections" +- Uses /ingest on laptop (Track A or B) +- Writes detailed agent notes +- May extract claims themselves +- Highest quality input + +### Mode 2: Curator +"Here's a source, it's about X domain" +- Minimal archive file with domain tag and brief notes +- VPS extracts (Track B) +- Good enough for most sources + +### Mode 3: Raw Dump +"Here are tweets, figure it out" +- Dumps raw JSON to VPS inbox-raw/ +- Leo triages: decides domain, writes archive files +- VPS extracts from Leo's archives +- Lowest effort, decent quality (Leo's triage catches the important stuff) + +### Mode 4: Self-Directed Agent (VPS) +"Agent, go research your domain" +- No human involvement beyond initial network setup +- Daily cron pulls tweets, agent picks direction, archives, extraction follows +- Quality depends on prompt engineering + eval pipeline catching errors + +All four modes feed into the same extraction → eval pipeline. Quality varies, but the eval pipeline is the quality gate regardless. + +## Open Questions + +1. **Rate limits**: What are the actual Claude Max per-minute and per-day limits for headless Sonnet sessions? Need empirical data from this first extraction run. + +2. **Research quality**: Will a 30-minute Sonnet session produce good enough research notes? Or does research require Opus-level reasoning? + +3. **Network bootstrapping**: Agents need network files. Who curates the initial account lists? (Currently Cory + Leo, eventually agents propose additions) + +4. **Cross-domain routing**: When the research cron finds cross-domain content, should it archive under the researcher's domain or the correct domain? (Probably correct domain with flagged_for_{researcher}) + +5. **Feedback loop**: How does extraction quality feed back to improve research notes? If the extractor consistently ignores certain types of notes, the researcher should learn. + +6. **Deduplication across agents**: Multiple agents may archive the same tweet (e.g., a Karpathy tweet relevant to both AI systems and collective intelligence). The extract cron needs to detect this. + +## Implementation Order + +1. ✅ Extract cron (running now — validating extraction quality) +2. **Next**: Research cron — daily self-directed sessions per agent +3. **Then**: Raw dump path — Leo triage from JSON → archive +4. **Later**: Full end-to-end with X API pull integrated into research cron +5. **Eventually**: Feedback loops from eval quality → research prompt tuning diff --git a/sync-mirror.sh b/sync-mirror.sh index 703dfe4..92595e4 100755 --- a/sync-mirror.sh +++ b/sync-mirror.sh @@ -44,6 +44,22 @@ fi log "Fetching from GitHub..." git fetch origin --prune >> "$LOG" 2>&1 || log "WARN: GitHub fetch failed" +# Step 2.5: GitHub main -> Forgejo main (ff-only) +# If a PR was merged on GitHub, GitHub main is ahead of Forgejo main. +# Fast-forward Forgejo main to match — safe because ff-only guarantees no divergence. +GITHUB_MAIN_FF=$(git rev-parse refs/remotes/origin/main 2>/dev/null || true) +FORGEJO_MAIN_FF=$(git rev-parse refs/remotes/forgejo/main 2>/dev/null || true) +if [ -n "$GITHUB_MAIN_FF" ] && [ -n "$FORGEJO_MAIN_FF" ]; then + if [ "$GITHUB_MAIN_FF" != "$FORGEJO_MAIN_FF" ]; then + if git merge-base --is-ancestor "$FORGEJO_MAIN_FF" "$GITHUB_MAIN_FF"; then + log "GitHub main ($GITHUB_MAIN_FF) ahead of Forgejo main ($FORGEJO_MAIN_FF) — fast-forwarding" + git push forgejo "refs/remotes/origin/main:refs/heads/main" >> "$LOG" 2>&1 && \ + log "Forgejo main fast-forwarded to $GITHUB_MAIN_FF" || \ + log "WARN: Failed to fast-forward Forgejo main" + fi + fi +fi + # Step 3: Forgejo -> GitHub (primary direction) # Update local refs from Forgejo remote refs using process substitution (avoids subshell) log "Syncing Forgejo -> GitHub..." @@ -99,10 +115,29 @@ if [ -n "$GITHUB_ONLY" ]; then extract/*|ingestion/*) continue ;; esac if [ -n "$FORGEJO_TOKEN" ]; then - # Check if PR already exists - EXISTING=$(curl -sf "http://localhost:3000/api/v1/repos/teleo/teleo-codex/pulls?state=open&head=$branch&limit=1" \ - -H "Authorization: token $FORGEJO_TOKEN" 2>/dev/null || echo "[]") - if [ "$EXISTING" = "[]" ] || [ "$EXISTING" = "null" ]; then + # Check if PR already exists for this branch (open or closed) + # NOTE: Forgejo ?head= filter is broken (ignores head value, returns all PRs). + # Workaround: fetch open+closed PRs, pipe to Python, check head.ref. + HAS_PR=$( { + curl -sf "http://localhost:3000/api/v1/repos/teleo/teleo-codex/pulls?state=open&limit=50" \ + -H "Authorization: token $FORGEJO_TOKEN" 2>/dev/null || echo "[]" + echo "" + curl -sf "http://localhost:3000/api/v1/repos/teleo/teleo-codex/pulls?state=closed&sort=created&limit=50" \ + -H "Authorization: token $FORGEJO_TOKEN" 2>/dev/null || echo "[]" + } | python3 -c " +import sys, json +branch = sys.argv[1] +for line in sys.stdin: + line = line.strip() + if not line or line == '[]': continue + try: + for pr in json.loads(line): + if pr.get('head', {}).get('ref') == branch: + print('yes'); sys.exit(0) + except: pass +print('no') +" "$branch" 2>/dev/null || echo "no") + if [ "$HAS_PR" = "no" ]; then PR_TITLE=$(echo "$branch" | sed 's|/|: |;s/-/ /g') RESULT=$(curl -sf -X POST "http://localhost:3000/api/v1/repos/teleo/teleo-codex/pulls" \ -H "Authorization: token $FORGEJO_TOKEN" \ diff --git a/systemd/teleo-agent@.service b/systemd/teleo-agent@.service new file mode 100644 index 0000000..23c046a --- /dev/null +++ b/systemd/teleo-agent@.service @@ -0,0 +1,38 @@ +[Unit] +Description=Teleo Agent %i +After=network.target +Wants=network.target + +[Service] +Type=simple +User=teleo +Group=teleo +WorkingDirectory=/opt/teleo-eval/telegram + +# Touch required paths before startup (prevents namespace crash on missing files) +ExecStartPre=/bin/bash -c 'touch /opt/teleo-eval/workspaces/.main-worktree.lock' +# Validate config before starting (fail fast on bad config) +ExecStartPre=/opt/teleo-eval/pipeline/.venv/bin/python3 /opt/teleo-eval/telegram/agent_runner.py --agent %i --validate + +ExecStart=/opt/teleo-eval/pipeline/.venv/bin/python3 /opt/teleo-eval/telegram/agent_runner.py --agent %i + +Restart=on-failure +RestartSec=10 + +# Filesystem protection (Rhea-approved) +ProtectSystem=strict +ReadWritePaths=/opt/teleo-eval/logs +ReadWritePaths=/opt/teleo-eval/telegram-archives +ReadWritePaths=/opt/teleo-eval/workspaces/main/inbox +ReadWritePaths=/opt/teleo-eval/workspaces/.main-worktree.lock +ReadWritePaths=/opt/teleo-eval/pipeline/pipeline.db +ReadWritePaths=/opt/teleo-eval/pipeline/pipeline.db-wal +ReadWritePaths=/opt/teleo-eval/pipeline/pipeline.db-shm + +# Agent-specific learnings (all agents share the worktree write path) +ReadWritePaths=/opt/teleo-eval/workspaces/main/agents + +Environment=PYTHONUNBUFFERED=1 + +[Install] +WantedBy=multi-user.target diff --git a/systemd/teleo-diagnostics.service b/systemd/teleo-diagnostics.service new file mode 100644 index 0000000..5f065bc --- /dev/null +++ b/systemd/teleo-diagnostics.service @@ -0,0 +1,21 @@ +[Unit] +Description=Argus — Teleo Pipeline Diagnostics Dashboard +After=teleo-pipeline.service +Wants=teleo-pipeline.service + +[Service] +Type=simple +User=teleo +Group=teleo +WorkingDirectory=/opt/teleo-eval/diagnostics +ExecStart=/usr/bin/python3 /opt/teleo-eval/diagnostics/app.py +Environment=PIPELINE_DB=/opt/teleo-eval/pipeline/pipeline.db +Environment=ARGUS_PORT=8081 +Environment=REPO_DIR=/opt/teleo-eval/workspaces/main +Restart=on-failure +RestartSec=5 +StandardOutput=journal +StandardError=journal + +[Install] +WantedBy=multi-user.target diff --git a/systemd/teleo-pipeline.service b/systemd/teleo-pipeline.service new file mode 100644 index 0000000..a6fbfab --- /dev/null +++ b/systemd/teleo-pipeline.service @@ -0,0 +1,37 @@ +[Unit] +Description=Teleo Pipeline v2 — extraction/eval/merge daemon +After=network.target +Wants=network.target + +[Service] +Type=simple +User=teleo +Group=teleo +WorkingDirectory=/opt/teleo-eval +ExecStartPre=/opt/teleo-eval/pipeline/fix-ownership.sh +ExecStart=/opt/teleo-eval/pipeline/.venv/bin/python3 /opt/teleo-eval/pipeline/teleo-pipeline.py +Restart=on-failure +RestartSec=30 + +# Graceful shutdown: SIGTERM → 60s drain → force-cancel → kill subprocesses +# 180s buffer handles in-flight extractions (up to 10 min each) (Ganymede) +KillSignal=SIGTERM +TimeoutStopSec=180 + +# Environment +Environment=PIPELINE_BASE=/opt/teleo-eval +EnvironmentFile=-/opt/teleo-eval/secrets/pipeline.env + +# Logging goes to journal + pipeline.jsonl +StandardOutput=journal +StandardError=journal + +# Security hardening +NoNewPrivileges=yes +ProtectSystem=strict +ReadWritePaths=/opt/teleo-eval /tmp +# PrivateTmp=no: daemon uses /tmp/teleo-extract-* worktrees shared with git (Ganymede) +PrivateTmp=no + +[Install] +WantedBy=multi-user.target diff --git a/telegram/agent_config.py b/telegram/agent_config.py new file mode 100644 index 0000000..a28c4a9 --- /dev/null +++ b/telegram/agent_config.py @@ -0,0 +1,160 @@ +#!/usr/bin/env python3 +"""Agent config loader and validator. + +Loads YAML config files from telegram/agents/*.yaml, validates required fields, +resolves file paths. Used by bot.py and future agent_runner.py. + +Epimetheus owns this module. +""" + +import logging +import os +import re +from dataclasses import dataclass, field +from pathlib import Path +from typing import Optional + +logger = logging.getLogger("tg.agent_config") + +SECRETS_DIR = "/opt/teleo-eval/secrets" +WORKTREE_DIR = "/opt/teleo-eval/workspaces/main" + +REQUIRED_FIELDS = ["name", "handle", "bot_token_file", "pentagon_agent_id", "domain"] +REQUIRED_VOICE_FIELDS = ["voice_summary", "voice_definition"] +REQUIRED_KB_FIELDS = ["kb_scope"] + + +@dataclass +class AgentConfig: + """Validated agent configuration loaded from YAML.""" + name: str + handle: str + x_handle: Optional[str] + bot_token_file: str + pentagon_agent_id: str + domain: str + kb_scope_primary: list[str] + voice_summary: str + voice_definition: str + domain_expertise: str + learnings_file: str + opsec_additional_patterns: list[str] = field(default_factory=list) + response_model: str = "anthropic/claude-opus-4-6" + triage_model: str = "anthropic/claude-haiku-4.5" + max_tokens: int = 1024 + max_response_per_user_per_hour: int = 30 + + def to_dict(self) -> dict: + """Convert to dict for passing to build_system_prompt.""" + return { + "name": self.name, + "handle": self.handle, + "x_handle": self.x_handle, + "domain": self.domain, + "voice_definition": self.voice_definition, + "voice_summary": self.voice_summary, + "domain_expertise": self.domain_expertise, + "pentagon_agent_id": self.pentagon_agent_id, + } + + @property + def bot_token_path(self) -> str: + return os.path.join(SECRETS_DIR, self.bot_token_file) + + @property + def learnings_path(self) -> str: + return os.path.join(WORKTREE_DIR, self.learnings_file) + + @property + def handle_regex(self) -> re.Pattern: + """Regex matching this agent's @handle with optional @botname suffix.""" + clean = self.handle.lstrip("@") + return re.compile(rf"@{re.escape(clean)}(?:@\w+)?", re.IGNORECASE) + + +def load_agent_config(config_path: str) -> AgentConfig: + """Load and validate an agent YAML config file. + + Raises ValueError on validation failure. + """ + import yaml + + with open(config_path) as f: + raw = yaml.safe_load(f) + + errors = [] + + # Required fields + for fld in REQUIRED_FIELDS + REQUIRED_VOICE_FIELDS: + if fld not in raw or not raw[fld]: + errors.append(f"Missing required field: {fld}") + + # KB scope + kb_scope = raw.get("kb_scope", {}) + if not isinstance(kb_scope, dict) or "primary" not in kb_scope: + errors.append("Missing kb_scope.primary (list of primary domain dirs)") + elif not isinstance(kb_scope["primary"], list) or len(kb_scope["primary"]) == 0: + errors.append("kb_scope.primary must be a non-empty list") + + # Learnings file + if "learnings_file" not in raw: + errors.append("Missing required field: learnings_file") + + if errors: + raise ValueError( + f"Agent config validation failed ({config_path}):\n" + + "\n".join(f" - {e}" for e in errors) + ) + + return AgentConfig( + name=raw["name"], + handle=raw["handle"], + x_handle=raw.get("x_handle"), + bot_token_file=raw["bot_token_file"], + pentagon_agent_id=raw["pentagon_agent_id"], + domain=raw["domain"], + kb_scope_primary=kb_scope["primary"], + voice_summary=raw["voice_summary"], + voice_definition=raw["voice_definition"], + domain_expertise=raw.get("domain_expertise", ""), + learnings_file=raw["learnings_file"], + opsec_additional_patterns=raw.get("opsec_additional_patterns", []), + response_model=raw.get("response_model", "anthropic/claude-opus-4-6"), + triage_model=raw.get("triage_model", "anthropic/claude-haiku-4.5"), + max_tokens=raw.get("max_tokens", 1024), + max_response_per_user_per_hour=raw.get("max_response_per_user_per_hour", 30), + ) + + +def validate_agent_config(config_path: str) -> list[str]: + """Validate config file and check runtime dependencies. + + Returns list of warnings (empty = all good). + Raises ValueError on hard failures. + """ + config = load_agent_config(config_path) + warnings = [] + + # Check bot token file exists + if not os.path.exists(config.bot_token_path): + warnings.append(f"Bot token file not found: {config.bot_token_path}") + + # Check primary KB dirs exist + for d in config.kb_scope_primary: + full = os.path.join(WORKTREE_DIR, d) + if not os.path.isdir(full): + warnings.append(f"KB scope dir not found: {full}") + + # Check learnings file parent dir exists + learnings_dir = os.path.dirname(config.learnings_path) + if not os.path.isdir(learnings_dir): + warnings.append(f"Learnings dir not found: {learnings_dir}") + + # Validate OPSEC patterns compile + for i, pattern in enumerate(config.opsec_additional_patterns): + try: + re.compile(pattern, re.IGNORECASE) + except re.error as e: + warnings.append(f"Invalid OPSEC regex pattern [{i}]: {e}") + + return warnings diff --git a/telegram/agent_runner.py b/telegram/agent_runner.py new file mode 100644 index 0000000..dbdf6a4 --- /dev/null +++ b/telegram/agent_runner.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 +"""Agent runner — entry point for running a Teleo Telegram agent. + +Usage: + python3 agent_runner.py --agent rio + python3 agent_runner.py --agent theseus + python3 agent_runner.py --agent rio --validate + +Systemd template unit: teleo-agent@.service + ExecStart=/usr/bin/python3 /opt/teleo-eval/telegram/agent_runner.py --agent %i + +Each agent runs as a separate process for fault isolation. +Template unit means `systemctl start teleo-agent@rio` and +`systemctl start teleo-agent@theseus` are independent services +with separate log streams (journalctl -u teleo-agent@rio). + +Epimetheus owns this module. +""" + +import argparse +import sys +import os +from pathlib import Path + +AGENTS_DIR = Path(__file__).parent / "agents" + + +def find_config(agent_name: str) -> Path: + """Resolve agent name to config file path.""" + config_path = AGENTS_DIR / f"{agent_name}.yaml" + if not config_path.exists(): + print(f"ERROR: Config not found: {config_path}", file=sys.stderr) + print(f"Available agents: {', '.join(p.stem for p in AGENTS_DIR.glob('*.yaml'))}", file=sys.stderr) + sys.exit(1) + return config_path + + +def validate(agent_name: str) -> bool: + """Validate agent config and runtime dependencies. Returns True if valid.""" + config_path = find_config(agent_name) + # Add telegram dir to path for agent_config import + sys.path.insert(0, str(Path(__file__).parent)) + from agent_config import validate_agent_config + try: + warnings = validate_agent_config(str(config_path)) + if warnings: + for w in warnings: + print(f" WARNING: {w}", file=sys.stderr) + print(f" Config OK: {agent_name} ({config_path})") + return True + except ValueError as e: + print(f" FAILED: {e}", file=sys.stderr) + return False + + +def run(agent_name: str): + """Run the agent bot process.""" + config_path = find_config(agent_name) + + # Validate before running (fail fast) + if not validate(agent_name): + sys.exit(1) + + # Set sys.argv so bot.py's main() picks up the config + sys.argv = ["bot.py", "--config", str(config_path)] + + # Import and run bot — this blocks until the bot exits + sys.path.insert(0, str(Path(__file__).parent)) + import bot + bot.main() + + +def list_agents(): + """List available agent configs.""" + configs = sorted(AGENTS_DIR.glob("*.yaml")) + if not configs: + print("No agent configs found in", AGENTS_DIR) + return + print("Available agents:") + for p in configs: + # Quick parse to get agent name from YAML + name = p.stem + try: + import yaml + with open(p) as f: + data = yaml.safe_load(f) + domain = data.get("domain", "unknown") + print(f" {name:12s} domain={domain}") + except Exception: + print(f" {name:12s} (config parse error)") + + +def main(): + parser = argparse.ArgumentParser( + description="Run a Teleo Telegram agent", + epilog="Systemd: teleo-agent@.service uses --agent %%i" + ) + parser.add_argument("--agent", help="Agent name (e.g., rio, theseus)") + parser.add_argument("--validate", action="store_true", help="Validate config and exit") + parser.add_argument("--list", action="store_true", help="List available agents") + args = parser.parse_args() + + if args.list: + list_agents() + return + + if not args.agent: + parser.error("--agent is required (or use --list)") + + if args.validate: + ok = validate(args.agent) + sys.exit(0 if ok else 1) + + run(args.agent) + + +if __name__ == "__main__": + main() diff --git a/telegram/agents/rio.yaml b/telegram/agents/rio.yaml new file mode 100644 index 0000000..736da58 --- /dev/null +++ b/telegram/agents/rio.yaml @@ -0,0 +1,62 @@ +# Rio — Teleo internet finance agent +# This config drives Rio's Telegram bot identity, KB scope, and voice. + +# ─── Identity ──────────────────────────────────────────────────────────── +name: Rio +handle: "@FutAIrdBot" +x_handle: "@futaRdIO" +bot_token_file: telegram-bot-token +pentagon_agent_id: 244ba05f +domain: internet-finance +domain_expertise: > + futarchy, prediction markets, token governance, the MetaDAO ecosystem, + conditional markets, internet capital formation, and permissionless fundraising + +# ─── KB Scope ──────────────────────────────────────────────────────────── +# One full-KB query; results tagged primary/cross-domain post-hoc. +kb_scope: + primary: + - domains/internet-finance + - foundations + - core + +# ─── Voice ─────────────────────────────────────────────────────────────── +voice_summary: "Sharp analyst talking to peers. High signal density." + +voice_definition: | + ## Register + You're a sharp analyst talking to peers — people who know markets and + governance mechanisms. Don't explain basics unless asked. Lead with your + take, not the context. + + ## Certainty Expression + Be direct about conviction levels. "High conviction" / "Speculative but + interesting" / "I don't know." Never hedge with weasel words when you + have a clear view. Never express false certainty when you don't. + + ## Domain Vocabulary + Use futarchy, pro-rata, oversubscription, ICO, conditional markets, + liquidation proposals without explanation. Explain newer protocol-specific + terms (ownership coins, PRISM) on first use. + + ## Signature Moves + Connect everything to market mechanisms and incentive structures. When + someone describes a governance problem, you see the market design solution. + When someone describes a market outcome, you trace it back to the + mechanism that produced it. + +# ─── Learnings ─────────────────────────────────────────────────────────── +learnings_file: agents/rio/learnings.md + +# ─── Eval ──────────────────────────────────────────────────────────────── +opsec_additional_patterns: + - "token price \\$[\\d,.]+" + - "LP (allocation|commitment)" + +# ─── Model ─────────────────────────────────────────────────────────────── +response_model: anthropic/claude-opus-4-6 +triage_model: anthropic/claude-haiku-4.5 +max_tokens: 500 + +# ─── Rate Limits ───────────────────────────────────────────────────────── +max_response_per_user_per_hour: 30 diff --git a/telegram/agents/theseus.yaml b/telegram/agents/theseus.yaml new file mode 100644 index 0000000..eb3ce4c --- /dev/null +++ b/telegram/agents/theseus.yaml @@ -0,0 +1,68 @@ +# Theseus — Teleo AI alignment & collective superintelligence agent +# This config drives Theseus's Telegram bot identity, KB scope, and voice. + +# ─── Identity ──────────────────────────────────────────────────────────── +name: Theseus +handle: "@TheseusAIBot" +bot_token_file: theseus-bot-token +pentagon_agent_id: 46864dd4 +domain: ai-alignment +domain_expertise: > + AI alignment, collective superintelligence architecture, multi-agent + coordination, capability evaluation, formal verification as oversight, + and empirical AI capability evidence + +# ─── KB Scope ──────────────────────────────────────────────────────────── +kb_scope: + primary: + - domains/ai-alignment + - domains/collective-intelligence + - foundations + - core + +# ─── Voice ─────────────────────────────────────────────────────────────── +voice_summary: "Technically precise, structurally analytical. Alignment as coordination, not catastrophe." + +voice_definition: | + ## Register + Technically precise but accessible. No jargon-hiding, no catastrophizing. + You see alignment as a coordination problem, not a purely technical one. + That framing shapes everything you say. + + ## Certainty Expression + Intellectually honest about what's empirically grounded vs theoretically + thin. Say "the evidence shows" when it does, "this is structural analysis" + when it's inference, "I don't know" when you don't. Never dress speculation + as fact. + + ## Domain Vocabulary + Use alignment, mesa-optimization, RLHF, constitutional AI, verification, + coordination protocols, capability evaluation without explanation. Ground + abstract alignment concepts in concrete examples — the Claude's Cycles + research program, multi-agent architectures, observable failure modes. + + ## Signature Moves + Connect everything to coordination and architecture. When someone raises + an alignment concern, you see the structural mechanism. When someone + describes a capability, you trace the coordination pattern that produced + it. Evidence over theory — always prefer documented observation over + hypotheticals. + + ## What You Don't Do + No doomerism, no accelerationism. Structural analysis only. Don't + catastrophize and don't hand-wave risks away. + +# ─── Learnings ─────────────────────────────────────────────────────────── +learnings_file: agents/theseus/learnings.md + +# ─── Eval ──────────────────────────────────────────────────────────────── +opsec_additional_patterns: + - "internal (architecture|infra)" + +# ─── Model ─────────────────────────────────────────────────────────────── +response_model: anthropic/claude-opus-4-6 +triage_model: anthropic/claude-haiku-4.5 +max_tokens: 500 + +# ─── Rate Limits ───────────────────────────────────────────────────────── +max_response_per_user_per_hour: 30 diff --git a/telegram/approval_stages.py b/telegram/approval_stages.py new file mode 100644 index 0000000..df91592 --- /dev/null +++ b/telegram/approval_stages.py @@ -0,0 +1,241 @@ +"""Pluggable approval architecture — extensible voting stages for content approval. + +Design constraint from m3ta: the approval step must be a pipeline stage, not hardcoded. + +Current stage: 1 human approves via Telegram. +Future stages (interface designed, not implemented): +- Agent pre-screening votes (weighted by CI score) +- Multi-human approval +- Domain-agent substance checks +- Futarchy-style decision markets on high-stakes content + +Adding a new approval stage = implementing ApprovalStage and registering it. +Threshold logic aggregates votes across all stages. + +Epimetheus owns this module. +""" + +import logging +import sqlite3 +from dataclasses import dataclass, field +from enum import Enum +from typing import Callable, Optional + +logger = logging.getLogger("approval-stages") + + +class Vote(Enum): + APPROVE = "approve" + REJECT = "reject" + ABSTAIN = "abstain" + + +@dataclass +class StageResult: + """Result from a single approval stage.""" + stage_name: str + vote: Vote + weight: float # 0.0 - 1.0, how much this stage's vote counts + reason: str = "" + metadata: dict = field(default_factory=dict) + + +@dataclass +class AggregateResult: + """Aggregated result across all approval stages.""" + approved: bool + total_weight_approve: float + total_weight_reject: float + total_weight_abstain: float + stage_results: list[StageResult] + threshold: float # what threshold was used + + @property + def summary(self) -> str: + status = "APPROVED" if self.approved else "REJECTED" + return ( + f"{status} (approve={self.total_weight_approve:.2f}, " + f"reject={self.total_weight_reject:.2f}, " + f"threshold={self.threshold:.2f})" + ) + + +class ApprovalStage: + """Base class for approval stages. + + Implement check() to add a new approval stage. + The method receives the approval request and returns a StageResult. + + Stages run in priority order (lower = earlier). + A stage can short-circuit by returning a REJECT with weight >= threshold. + """ + + name: str = "unnamed" + priority: int = 100 # lower = runs earlier + weight: float = 1.0 # default weight of this stage's vote + + def check(self, request: dict) -> StageResult: + """Evaluate the approval request. Must be overridden.""" + raise NotImplementedError + + +# ─── Built-in Stages ───────────────────────────────────────────────── + +class OutputGateStage(ApprovalStage): + """Stage 0: Deterministic output gate. Blocks system content.""" + + name = "output_gate" + priority = 0 + weight = 1.0 # absolute veto — if gate blocks, nothing passes + + def check(self, request: dict) -> StageResult: + from output_gate import gate_for_tweet_queue + + content = request.get("content", "") + agent = request.get("originating_agent", "") + gate = gate_for_tweet_queue(content, agent) + + if gate: + return StageResult(self.name, Vote.APPROVE, self.weight, + "Content passed output gate") + else: + return StageResult(self.name, Vote.REJECT, self.weight, + f"Blocked: {', '.join(gate.blocked_reasons)}", + {"blocked_reasons": gate.blocked_reasons}) + + +class OpsecStage(ApprovalStage): + """Stage 1: OPSEC content filter. Blocks sensitive content.""" + + name = "opsec_filter" + priority = 1 + weight = 1.0 # absolute veto + + def check(self, request: dict) -> StageResult: + from approvals import check_opsec + + content = request.get("content", "") + violation = check_opsec(content) + + if violation: + return StageResult(self.name, Vote.REJECT, self.weight, violation) + else: + return StageResult(self.name, Vote.APPROVE, self.weight, + "No OPSEC violations") + + +class HumanApprovalStage(ApprovalStage): + """Stage 10: Human approval via Telegram. Currently the final gate. + + This stage is async — it doesn't return immediately. + Instead, it sets up the Telegram notification and returns ABSTAIN. + The actual vote comes later when Cory taps Approve/Reject. + """ + + name = "human_approval" + priority = 10 + weight = 1.0 + + def check(self, request: dict) -> StageResult: + # Human approval is handled asynchronously via Telegram + # This stage just validates the request is properly formatted + if not request.get("content"): + return StageResult(self.name, Vote.REJECT, self.weight, + "No content to approve") + + return StageResult(self.name, Vote.ABSTAIN, self.weight, + "Awaiting human approval via Telegram", + {"async": True}) + + +# ─── Stage Registry ────────────────────────────────────────────────── + +# Default stages — these run for every approval request +_DEFAULT_STAGES: list[ApprovalStage] = [ + OutputGateStage(), + OpsecStage(), + HumanApprovalStage(), +] + +# Custom stages added by agents or plugins +_CUSTOM_STAGES: list[ApprovalStage] = [] + + +def register_stage(stage: ApprovalStage): + """Register a custom approval stage.""" + _CUSTOM_STAGES.append(stage) + _CUSTOM_STAGES.sort(key=lambda s: s.priority) + logger.info("Registered approval stage: %s (priority=%d, weight=%.2f)", + stage.name, stage.priority, stage.weight) + + +def get_all_stages() -> list[ApprovalStage]: + """Get all stages sorted by priority.""" + all_stages = _DEFAULT_STAGES + _CUSTOM_STAGES + all_stages.sort(key=lambda s: s.priority) + return all_stages + + +# ─── Aggregation ───────────────────────────────────────────────────── + +def run_sync_stages(request: dict, threshold: float = 0.5) -> AggregateResult: + """Run all synchronous approval stages and aggregate results. + + Stages with async=True in metadata are skipped (handled separately). + Short-circuits on any REJECT with weight >= threshold. + + Args: + request: dict with at minimum {content, originating_agent, type} + threshold: weighted approve score needed to pass (0.0-1.0) + + Returns: + AggregateResult with the decision. + """ + stages = get_all_stages() + results = [] + total_approve = 0.0 + total_reject = 0.0 + total_abstain = 0.0 + + for stage in stages: + try: + result = stage.check(request) + except Exception as e: + logger.error("Stage %s failed: %s — treating as ABSTAIN", stage.name, e) + result = StageResult(stage.name, Vote.ABSTAIN, 0.0, f"Error: {e}") + + results.append(result) + + if result.vote == Vote.APPROVE: + total_approve += result.weight + elif result.vote == Vote.REJECT: + total_reject += result.weight + # Short-circuit: absolute veto + if result.weight >= threshold: + return AggregateResult( + approved=False, + total_weight_approve=total_approve, + total_weight_reject=total_reject, + total_weight_abstain=total_abstain, + stage_results=results, + threshold=threshold, + ) + else: + total_abstain += result.weight + + # Final decision based on non-abstain votes + active_weight = total_approve + total_reject + if active_weight == 0: + # All abstain — pass to async stages (human approval) + approved = False # not yet approved, awaiting human + else: + approved = (total_approve / active_weight) >= threshold + + return AggregateResult( + approved=approved, + total_weight_approve=total_approve, + total_weight_reject=total_reject, + total_weight_abstain=total_abstain, + stage_results=results, + threshold=threshold, + ) diff --git a/telegram/approvals.py b/telegram/approvals.py new file mode 100644 index 0000000..2dbc517 --- /dev/null +++ b/telegram/approvals.py @@ -0,0 +1,344 @@ +"""Telegram approval workflow — human-in-the-loop for outgoing comms + core KB changes. + +Flow: Agent submits → Leo reviews substance → Bot sends to Cory → Cory approves/rejects. + +Architecture: +- approval_queue table in pipeline.db (migration v11) +- Bot polls for leo_approved items, sends formatted Telegram messages with inline buttons +- Cory taps Approve/Reject → callback handler updates status +- 24h expiry timeout on all pending approvals + +OPSEC: Content filter rejects submissions containing financial figures or deal-specific language. +No deal terms, no dollar amounts, no private investment details in approval requests — ever. + +Epimetheus owns this module. +""" + +import logging +import re +import sqlite3 +from datetime import datetime, timezone +from pathlib import Path + +from telegram import InlineKeyboardButton, InlineKeyboardMarkup, Update +from telegram.ext import CallbackQueryHandler, ContextTypes + +logger = logging.getLogger("telegram.approvals") + +# ─── OPSEC Content Filter ───────────────────────────────────────────── +# Reject submissions containing financial figures or deal-specific language. +# Pattern matches: $1M, $500K, 1.5 million, deal terms, valuation, cap table, etc. +OPSEC_PATTERNS = [ + re.compile(r"\$[\d,.]+[KMBkmb]?\b", re.IGNORECASE), # $500K, $1.5M, $100 + re.compile(r"\b\d+[\d,.]*\s*(million|billion|thousand)\b", re.IGNORECASE), + re.compile(r"\b(deal terms?|valuation|cap table|equity split|ownership stake|term sheet|dilution|fee split)\b", re.IGNORECASE), + re.compile(r"\b(SAFE\s+(?:note|round|agreement)|SAFT|convertible note|preferred stock|liquidation preference)\b", re.IGNORECASE), + re.compile(r"\bSeries\s+[A-Z]\b", re.IGNORECASE), # Series A/B/C/F funding rounds + re.compile(r"\b(partnership terms|committed to (?:the |a )?round|funding round|(?:pre-?)?seed round)\b", re.IGNORECASE), +] + +# Sensitive entity names — loaded from opsec-entities.txt config file. +# Edit the config file to add/remove entities without code changes. +_OPSEC_ENTITIES_FILE = Path(__file__).parent / "opsec-entities.txt" + + +def _load_sensitive_entities() -> list[re.Pattern]: + """Load sensitive entity patterns from config file.""" + patterns = [] + if _OPSEC_ENTITIES_FILE.exists(): + for line in _OPSEC_ENTITIES_FILE.read_text().splitlines(): + line = line.strip() + if line and not line.startswith("#"): + patterns.append(re.compile(rf"\b{line}\b", re.IGNORECASE)) + return patterns + + +SENSITIVE_ENTITIES = _load_sensitive_entities() + + +def check_opsec(content: str) -> str | None: + """Check content against OPSEC patterns. Returns violation description or None.""" + for pattern in OPSEC_PATTERNS: + match = pattern.search(content) + if match: + return f"OPSEC violation: content contains '{match.group()}' — no financial figures or deal terms in approval requests" + for pattern in SENSITIVE_ENTITIES: + match = pattern.search(content) + if match: + return f"OPSEC violation: content references sensitive entity '{match.group()}' — deal-adjacent entities blocked" + return None + + +# ─── Message Formatting ─────────────────────────────────────────────── + +TYPE_LABELS = { + "tweet": "Tweet", + "kb_change": "KB Change", + "architecture_change": "Architecture Change", + "public_post": "Public Post", + "position": "Position", + "agent_structure": "Agent Structure", +} + +# ─── Tier Classification ───────────────────────────────────────────── +# Tier 1: Must approve (outgoing, public, irreversible) +# Tier 2: Should approve (core architecture, strategic) +# Tier 3: Autonomous (no approval needed — goes to daily digest only) + +TIER_1_TYPES = {"tweet", "public_post", "position"} +TIER_2_TYPES = {"kb_change", "architecture_change", "agent_structure"} +# Everything else is Tier 3 — no approval queue entry, digest only + + +def classify_tier(approval_type: str) -> int: + """Classify an approval request into tier 1, 2, or 3.""" + if approval_type in TIER_1_TYPES: + return 1 + if approval_type in TIER_2_TYPES: + return 2 + return 3 + + +def format_approval_message(row: sqlite3.Row) -> str: + """Format an approval request for Telegram display.""" + type_label = TYPE_LABELS.get(row["type"], row["type"].replace("_", " ").title()) + agent = row["originating_agent"].title() + content = row["content"] + + # Truncate long content for Telegram (4096 char limit) + if len(content) > 3000: + content = content[:3000] + "\n\n[... truncated]" + + parts = [ + f"APPROVAL REQUEST", + f"", + f"Type: {type_label}", + f"From: {agent}", + ] + + if row["context"]: + parts.append(f"Context: {row['context']}") + + if row["leo_review_note"]: + parts.append(f"Leo review: {row['leo_review_note']}") + + parts.extend([ + "", + "---", + content, + "---", + ]) + + return "\n".join(parts) + + +def build_keyboard(request_id: int) -> InlineKeyboardMarkup: + """Build inline keyboard with Approve/Reject buttons.""" + return InlineKeyboardMarkup([ + [ + InlineKeyboardButton("Approve", callback_data=f"approve:{request_id}"), + InlineKeyboardButton("Reject", callback_data=f"reject:{request_id}"), + ] + ]) + + +# ─── Core Logic ─────────────────────────────────────────────────────── + +def get_pending_for_cory(conn: sqlite3.Connection) -> list[sqlite3.Row]: + """Get approval requests that Leo approved and are ready for Cory.""" + return conn.execute( + """SELECT * FROM approval_queue + WHERE leo_review_status = 'leo_approved' + AND status = 'pending' + AND telegram_message_id IS NULL + AND (expires_at IS NULL OR expires_at > datetime('now')) + ORDER BY submitted_at ASC""", + ).fetchall() + + +def expire_stale_requests(conn: sqlite3.Connection) -> int: + """Expire requests older than 24h. Returns count expired.""" + cursor = conn.execute( + """UPDATE approval_queue + SET status = 'expired', decided_at = datetime('now') + WHERE status = 'pending' + AND expires_at IS NOT NULL + AND expires_at <= datetime('now')""", + ) + if cursor.rowcount > 0: + conn.commit() + logger.info("Expired %d stale approval requests", cursor.rowcount) + return cursor.rowcount + + +def record_decision( + conn: sqlite3.Connection, + request_id: int, + decision: str, + decision_by: str, + rejection_reason: str = None, +) -> bool: + """Record an approval/rejection decision. Returns True if updated.""" + cursor = conn.execute( + """UPDATE approval_queue + SET status = ?, decision_by = ?, rejection_reason = ?, + decided_at = datetime('now') + WHERE id = ? AND status = 'pending'""", + (decision, decision_by, rejection_reason, request_id), + ) + conn.commit() + return cursor.rowcount > 0 + + +def record_telegram_message(conn: sqlite3.Connection, request_id: int, message_id: int): + """Record the Telegram message ID for an approval notification.""" + conn.execute( + "UPDATE approval_queue SET telegram_message_id = ? WHERE id = ?", + (message_id, request_id), + ) + conn.commit() + + +# ─── Telegram Handlers ──────────────────────────────────────────────── + +async def handle_approval_callback(update: Update, context: ContextTypes.DEFAULT_TYPE): + """Handle Approve/Reject button taps from Cory.""" + query = update.callback_query + await query.answer() + + data = query.data + if not data or ":" not in data: + return + + action, request_id_str = data.split(":", 1) + if action not in ("approve", "reject"): + return + + try: + request_id = int(request_id_str) + except ValueError: + return + + conn = context.bot_data.get("approval_conn") + if not conn: + await query.edit_message_text("Error: approval DB not connected") + return + + if action == "reject": + # Check if user sent a reply with rejection reason + rejection_reason = None + # For rejection, edit the message to ask for reason + row = conn.execute( + "SELECT * FROM approval_queue WHERE id = ?", (request_id,) + ).fetchone() + if not row or row["status"] != "pending": + await query.edit_message_text("This request has already been processed.") + return + + # Store pending rejection — user can reply with reason + context.bot_data[f"pending_reject:{request_id}"] = True + await query.edit_message_text( + f"{query.message.text}\n\nRejected. Reply to this message with feedback for the agent (optional).", + ) + record_decision(conn, request_id, "rejected", query.from_user.username or str(query.from_user.id)) + logger.info("Approval #%d REJECTED by %s", request_id, query.from_user.username) + return + + # Approve + user = query.from_user.username or str(query.from_user.id) + success = record_decision(conn, request_id, "approved", user) + + if success: + # Check if this is a tweet — if so, auto-post to X + row = conn.execute( + "SELECT type FROM approval_queue WHERE id = ?", (request_id,) + ).fetchone() + + post_status = "" + if row and row["type"] == "tweet": + try: + from x_publisher import handle_approved_tweet + result = await handle_approved_tweet(conn, request_id) + if result.get("success"): + url = result.get("tweet_url", "") + post_status = f"\n\nPosted to X: {url}" + logger.info("Tweet #%d auto-posted: %s", request_id, url) + else: + error = result.get("error", "unknown error") + post_status = f"\n\nPost failed: {error}" + logger.error("Tweet #%d auto-post failed: %s", request_id, error) + except Exception as e: + post_status = f"\n\nPost failed: {e}" + logger.error("Tweet #%d auto-post error: %s", request_id, e) + + await query.edit_message_text( + f"{query.message.text}\n\nAPPROVED by {user}{post_status}" + ) + logger.info("Approval #%d APPROVED by %s", request_id, user) + else: + await query.edit_message_text("This request has already been processed.") + + +async def handle_rejection_reply(update: Update, context: ContextTypes.DEFAULT_TYPE): + """Capture rejection reason from reply to a rejected approval message.""" + if not update.message or not update.message.reply_to_message: + return False + + # Check if the replied-to message is a rejected approval + conn = context.bot_data.get("approval_conn") + if not conn: + return False + + reply_msg_id = update.message.reply_to_message.message_id + row = conn.execute( + "SELECT id FROM approval_queue WHERE telegram_message_id = ? AND status = 'rejected'", + (reply_msg_id,), + ).fetchone() + + if not row: + return False + + # Update rejection reason + reason = update.message.text.strip() + conn.execute( + "UPDATE approval_queue SET rejection_reason = ? WHERE id = ?", + (reason, row["id"]), + ) + conn.commit() + await update.message.reply_text(f"Feedback recorded for approval #{row['id']}.") + logger.info("Rejection reason added for approval #%d: %s", row["id"], reason[:100]) + return True + + +# ─── Poll Job ───────────────────────────────────────────────────────── + +async def poll_approvals(context: ContextTypes.DEFAULT_TYPE): + """Poll for Leo-approved requests and send to Cory. Runs every 30s.""" + conn = context.bot_data.get("approval_conn") + admin_chat_id = context.bot_data.get("admin_chat_id") + + if not conn or not admin_chat_id: + return + + # Expire stale requests first (may fail on DB lock - retry next cycle) + try: + expire_stale_requests(conn) + except Exception: + pass # non-fatal, retries in 30s + + # Send new notifications + pending = get_pending_for_cory(conn) + for row in pending: + try: + text = format_approval_message(row) + keyboard = build_keyboard(row["id"]) + msg = await context.bot.send_message( + chat_id=admin_chat_id, + text=text, + reply_markup=keyboard, + ) + record_telegram_message(conn, row["id"], msg.message_id) + logger.info("Sent approval #%d to admin (type=%s, agent=%s)", + row["id"], row["type"], row["originating_agent"]) + except Exception as e: + logger.error("Failed to send approval #%d: %s", row["id"], e) diff --git a/telegram/bot.py b/telegram/bot.py index 521972b..2312fe2 100644 --- a/telegram/bot.py +++ b/telegram/bot.py @@ -17,6 +17,7 @@ Does NOT integrate with pipeline daemon. Epimetheus owns this module. """ +import argparse import asyncio import logging import os @@ -24,6 +25,8 @@ import re import sqlite3 import sys import time + +import yaml from collections import defaultdict from datetime import datetime, timezone from pathlib import Path @@ -42,7 +45,8 @@ from telegram.ext import ( sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) import json as _json -from kb_retrieval import KBIndex, format_context_for_prompt, retrieve_context +from kb_retrieval import KBIndex, retrieve_context, retrieve_vector_context +from retrieval import orchestrate_retrieval from market_data import get_token_price, format_price_context from worktree_lock import main_worktree_lock from x_client import search_tweets, fetch_from_url, check_research_rate_limit, record_research_usage, get_research_remaining @@ -69,6 +73,9 @@ TRIAGE_INTERVAL = 900 # 15 minutes RESPONSE_MODEL = "anthropic/claude-opus-4-6" # Opus for tagged responses TRIAGE_MODEL = "anthropic/claude-haiku-4.5" # Haiku for batch triage +# KB scope — None means all domains (Rio default). Set from YAML config for other agents. +AGENT_KB_SCOPE: list[str] | None = None + # Rate limits MAX_RESPONSE_PER_USER_PER_HOUR = 30 MIN_MESSAGE_LENGTH = 20 # Skip very short messages @@ -430,6 +437,150 @@ async def call_openrouter(model: str, prompt: str, max_tokens: int = 2048) -> _L return None +async def call_openrouter_with_tools(model: str, prompt: str, tools: list[dict], + tool_executor, max_tokens: int = 2048, + max_iterations: int = 3) -> tuple[_LLMResponse | None, list[dict]]: + """Agentic loop: call LLM with tools, execute tool calls, feed back results. + + Returns (final_response, tool_call_audit_list). + Token counts and cost are ACCUMULATED across all iterations, not just the final call. + Tool audit includes LLM reasoning text between tool calls for full observability. + Falls back to plain call_openrouter if model returns 400 with tool errors. + """ + import aiohttp + import json + + key = Path(OPENROUTER_KEY_FILE).read_text().strip() + messages = [{"role": "user", "content": prompt}] + tool_audit = [] + + # Accumulate tokens/cost across ALL iterations (not just final call) + total_prompt_tokens = 0 + total_completion_tokens = 0 + total_cost = 0.0 + + for iteration in range(max_iterations): + payload = { + "model": model, + "messages": messages, + "max_tokens": max_tokens, + "temperature": 0.3, + "tools": tools, + } + try: + async with aiohttp.ClientSession() as session: + async with session.post( + "https://openrouter.ai/api/v1/chat/completions", + headers={"Authorization": f"Bearer {key}", "Content-Type": "application/json"}, + json=payload, + timeout=aiohttp.ClientTimeout(total=120), + ) as resp: + if resp.status >= 400: + body = await resp.text() + if "tool" in body.lower(): + logger.warning("Model doesn't support tools, falling back to plain call") + result = await call_openrouter(model, prompt, max_tokens) + return result, tool_audit + logger.error("OpenRouter with tools %s → %d", model, resp.status) + return None, tool_audit + data = await resp.json() + except Exception as e: + logger.error("OpenRouter with tools error: %s", e) + return None, tool_audit + + # Accumulate this iteration's token usage + usage = data.get("usage", {}) + iter_pt = usage.get("prompt_tokens", 0) + iter_ct = usage.get("completion_tokens", 0) + iter_cost = estimate_cost(model, iter_pt, iter_ct) + total_prompt_tokens += iter_pt + total_completion_tokens += iter_ct + total_cost += iter_cost + + choice = data.get("choices", [{}])[0] + message = choice.get("message", {}) + + # If model wants to call tools (check presence only — finish_reason varies by model) + tool_calls_in_response = message.get("tool_calls", []) + if tool_calls_in_response: + # Capture LLM reasoning text alongside tool calls (the "thinking" between searches) + reasoning_text = message.get("content", "") + if reasoning_text: + tool_audit.append({ + "type": "reasoning", "iteration": iteration + 1, + "text": reasoning_text[:2000], + "tokens": {"prompt": iter_pt, "completion": iter_ct, "cost": round(iter_cost, 6)}, + }) + + messages.append(message) # Add assistant message with tool calls + for tc in tool_calls_in_response: + fn_name = tc["function"]["name"] + try: + fn_args = json.loads(tc["function"]["arguments"]) + except (json.JSONDecodeError, KeyError): + fn_args = {} + + t0 = time.monotonic() + result = tool_executor(fn_name, fn_args) + duration_ms = int((time.monotonic() - t0) * 1000) + + # Truncate tool results + result_str = str(result)[:4000] + tool_audit.append({ + "type": "tool_call", "iteration": iteration + 1, + "tool": fn_name, "input": fn_args, + "output_preview": result_str[:500], + "output_length": len(result_str), "duration_ms": duration_ms, + }) + messages.append({ + "role": "tool", + "tool_call_id": tc["id"], + "content": result_str, + }) + continue # Next iteration with tool results + + # Model returned a text response (done) + content = message.get("content") + if content is None: + return None, tool_audit + return _LLMResponse(content, prompt_tokens=total_prompt_tokens, + completion_tokens=total_completion_tokens, + cost=total_cost, model=model), tool_audit + + # Exhausted iterations — force one final call WITHOUT tools to get a text answer + logger.warning("Tool loop exhausted %d iterations, forcing final plain call", max_iterations) + try: + messages.append({"role": "user", "content": "Please provide your final answer now based on the information gathered."}) + payload_final = { + "model": model, + "messages": messages, + "max_tokens": max_tokens, + "temperature": 0.3, + } + async with aiohttp.ClientSession() as session: + async with session.post( + "https://openrouter.ai/api/v1/chat/completions", + headers={"Authorization": f"Bearer {key}", "Content-Type": "application/json"}, + json=payload_final, + timeout=aiohttp.ClientTimeout(total=120), + ) as resp: + if resp.status < 400: + data = await resp.json() + content = data.get("choices", [{}])[0].get("message", {}).get("content") + if content: + usage = data.get("usage", {}) + total_prompt_tokens += usage.get("prompt_tokens", 0) + total_completion_tokens += usage.get("completion_tokens", 0) + total_cost += estimate_cost(model, usage.get("prompt_tokens", 0), + usage.get("completion_tokens", 0)) + return _LLMResponse(content, prompt_tokens=total_prompt_tokens, + completion_tokens=total_completion_tokens, + cost=total_cost, model=model), tool_audit + except Exception as e: + logger.error("Final plain call after tool exhaustion failed: %s", e) + return None, tool_audit + + def is_rate_limited(user_id: int) -> bool: """Check if a user has exceeded the response rate limit.""" now = time.time() @@ -843,7 +994,7 @@ async def handle_tagged(update: Update, context: ContextTypes.DEFAULT_TYPE): # Rate limit check if user and is_rate_limited(user.id): - await msg.reply_text("I'm processing other requests — try again in a few minutes.") + await msg.reply_text("I'm processing other requests — try again in a few minutes.", quote=True) return logger.info("Tagged by @%s: %s", user.username if user else "unknown", text[:100]) @@ -994,41 +1145,22 @@ async def handle_tagged(update: Update, context: ContextTypes.DEFAULT_TYPE): logger.warning("Query reformulation failed: %s", e) # Fall through — use raw text - # Retrieve full KB context (entity resolution + claim search + agent positions) - t_kb = time.monotonic() - kb_ctx = retrieve_context(search_query_text, KB_READ_DIR, index=kb_index) - kb_context_text = format_context_for_prompt(kb_ctx) - kb_duration = int((time.monotonic() - t_kb) * 1000) - retrieval_layers = ["keyword"] if (kb_ctx and (kb_ctx.entities or kb_ctx.claims)) else [] - tool_calls.append({ - "tool": "retrieve_context", - "input": {"query": search_query_text[:200], "original_query": text[:200] if search_query_text != text else None}, - "output": {"entities": len(kb_ctx.entities) if kb_ctx else 0, - "claims": len(kb_ctx.claims) if kb_ctx else 0}, - "duration_ms": kb_duration, - }) - - # Layer 1+2: Qdrant vector search + graph expansion (semantic, complements keyword) - # Pass keyword-matched paths to exclude duplicates at Qdrant query level - # Normalize: KBIndex stores absolute paths, Qdrant stores repo-relative paths - keyword_paths = [] - if kb_ctx and kb_ctx.claims: - for c in kb_ctx.claims: - p = c.path - if KB_READ_DIR and p.startswith(KB_READ_DIR): - p = p[len(KB_READ_DIR):].lstrip("/") - keyword_paths.append(p) - from kb_retrieval import retrieve_vector_context - vector_context, vector_meta = retrieve_vector_context(search_query_text, keyword_paths=keyword_paths) - if vector_context: - kb_context_text = kb_context_text + "\n\n" + vector_context - retrieval_layers.extend(vector_meta.get("layers_hit", [])) - tool_calls.append({ - "tool": "retrieve_qdrant_context", "input": {"query": text[:200]}, - "output": {"direct_hits": len(vector_meta.get("direct_results", [])), - "expanded": len(vector_meta.get("expanded_results", []))}, - "duration_ms": vector_meta.get("duration_ms", 0), - }) + # Full retrieval pipeline: keyword → decompose → vector → RRF merge + retrieval = await orchestrate_retrieval( + text=text, + search_query=search_query_text, + kb_read_dir=KB_READ_DIR, + kb_index=kb_index, + llm_fn=call_openrouter, + triage_model=TRIAGE_MODEL, + retrieve_context_fn=retrieve_context, + retrieve_vector_fn=retrieve_vector_context, + kb_scope=AGENT_KB_SCOPE, + ) + kb_context_text = retrieval["kb_context_text"] + kb_ctx = retrieval["kb_ctx"] + retrieval_layers = retrieval["retrieval_layers"] + tool_calls.extend(retrieval["tool_calls"]) stats = get_db_stats() @@ -1090,6 +1222,31 @@ Write like a sharp analyst talking to peers, not like an AI. Specifically: ## What you know about this topic {kb_context_text} +## KB Tools — SEARCH UNTIL YOU HAVE ENOUGH + +You have 8 tools to search the knowledge base. The context above is an initial retrieval pass — it is almost never sufficient on its own. You MUST use tools to verify and deepen your understanding before answering. + +**Your retrieval loop (follow this every time):** +1. Review the initial context above. Identify what's missing or unclear. +2. Use tools to fill gaps — search for sources, explore graph edges, read full claims. +3. After each tool result, ask yourself: "Do I have enough to give a substantive, grounded answer?" +4. If NO — search again with different terms, follow more graph edges, read the original source. +5. If YES — compose your answer. You have up to 6 tool calls, use them. + +**Tool selection rules:** +- Someone asks about a specific author/paper/research → call find_by_source AND search_sources to find ALL material from that source +- You see a claim but need the original article → call read_source with the source title +- You want to understand the argument structure around a claim → call explore_graph to see what supports, challenges, and depends on it +- Initial claims don't cover the topic well → call search_kb with refined keywords +- You want to trace an entity's full network → call list_entity_links then read linked items +- You want to find original research documents → call search_sources by topic/author + +**Critical rules:** +- DO NOT guess or hallucinate details about specific research — use tools to get actual data +- DO NOT answer from just the initial retrieval context if the question asks about specific research — always trace back to the source +- When you find a claim, explore its graph edges — connected claims often contain the nuance the user needs +- If search_kb returns poor results, try search_sources or find_by_source with different keywords + {f"## Live Market Data{chr(10)}{market_context}" if market_context else ""} {research_context} @@ -1124,11 +1281,21 @@ IMPORTANT: Special tags you can append at the end of your response (after your m 5. CONFIDENCE: [0.0-1.0] ALWAYS include this tag. Rate how well the KB context above actually helped you answer this question. 1.0 = KB had exactly what was needed. 0.5 = KB had partial/tangential info. 0.0 = KB had nothing relevant, you answered from general knowledge. This is for internal audit only — never visible to users.""" - # Call Opus - response = await call_openrouter(RESPONSE_MODEL, prompt, max_tokens=1024) + # Call Opus with KB tools — agent can drill into claims, entities, and sources + from kb_tools import TOOL_DEFINITIONS, execute_tool + _tool_executor = lambda name, args: execute_tool(name, args, KB_READ_DIR) + response, kb_tool_audit = await call_openrouter_with_tools( + RESPONSE_MODEL, prompt, TOOL_DEFINITIONS, _tool_executor, max_tokens=1024, + max_iterations=6) + if kb_tool_audit: + for t in kb_tool_audit: + if t.get("type") == "reasoning": + tool_calls.append({"type": "kb_reasoning", **t}) + else: + tool_calls.append({"tool": f"kb:{t.get('tool', 'unknown')}", **{k: v for k, v in t.items() if k != "tool"}}) if not response: - await msg.reply_text("Processing error — I'll get back to you.") + await msg.reply_text("Processing error — I'll get back to you.", quote=True) return # Parse LEARNING and RESEARCH tags before posting @@ -1197,18 +1364,8 @@ IMPORTANT: Special tags you can append at the end of your response (after your m "duration_ms": response_time_ms - sum(tc.get("duration_ms", 0) for tc in tool_calls), }) - # Build claims_matched with rank + source info (Rio: rank order matters) - claims_audit = [] - for i, c in enumerate(kb_ctx.claims if kb_ctx else []): - claims_audit.append({"path": c.path, "title": c.title, "score": c.score, - "rank": i + 1, "source": "keyword"}) - for r in vector_meta.get("direct_results", []): - claims_audit.append({"path": r["path"], "title": r["title"], "score": r["score"], - "rank": len(claims_audit) + 1, "source": "qdrant"}) - for r in vector_meta.get("expanded_results", []): - claims_audit.append({"path": r["path"], "title": r["title"], "score": 0, - "rank": len(claims_audit) + 1, "source": "graph", - "edge_type": r.get("edge_type", "")}) + # Claims audit — already built by orchestrate_retrieval with RRF ranking + claims_audit = retrieval.get("claims_audit", []) # ─── Eval: URL fabrication check ────────────────────────────── blocked = False @@ -1271,13 +1428,16 @@ IMPORTANT: Special tags you can append at the end of your response (after your m prompt_tokens=response_prompt_tokens, completion_tokens=response_completion_tokens, generation_cost=response_cost, + total_cost=response_cost, # same as generation_cost until embedding cost tracked blocked=1 if blocked else 0, block_reason=block_reason, ) _audit_conn.commit() - logger.info("Audit record written (confidence=%.2f, cost=$%.4f, layers=%s, %d claims, %dms%s)", + kb_tool_count = sum(1 for t in tool_calls if t.get("type") == "tool_call" or (t.get("tool", "").startswith("kb:") and t.get("type") != "kb_reasoning")) + kb_reasoning_count = sum(1 for t in tool_calls if t.get("type") in ("reasoning", "kb_reasoning")) + logger.info("Audit record written (confidence=%.2f, cost=$%.4f, layers=%s, %d claims, %d kb_tools, %d reasoning_steps, %dms%s)", confidence_score or 0, response_cost, retrieval_layers, - len(claims_audit), response_time_ms, + len(claims_audit), kb_tool_count, kb_reasoning_count, response_time_ms, ", BLOCKED" if blocked else "") except Exception as e: logger.warning("Failed to write audit record: %s", e) @@ -1285,7 +1445,7 @@ IMPORTANT: Special tags you can append at the end of your response (after your m # Post response (without tag lines) # Telegram has a 4096 char limit — split long messages if len(display_response) <= 4096: - await msg.reply_text(display_response) + await msg.reply_text(display_response, quote=True) else: # Split on paragraph boundaries where possible chunks = [] @@ -1302,9 +1462,12 @@ IMPORTANT: Special tags you can append at the end of your response (after your m split_at = 4096 chunks.append(remaining[:split_at]) remaining = remaining[split_at:].lstrip("\n") + # First chunk quotes the original message, rest are standalone follow-ups + first = True for chunk in chunks: if chunk.strip(): - await msg.reply_text(chunk) + await msg.reply_text(chunk, quote=first) + first = False # Update conversation state: reset window, store history (Ganymede+Rhea) if user: @@ -1770,8 +1933,47 @@ async def stats_command(update: Update, context: ContextTypes.DEFAULT_TYPE): ) +def _load_agent_config(config_path: str): + """Load agent YAML config and set module-level variables.""" + global BOT_TOKEN_FILE, RESPONSE_MODEL, TRIAGE_MODEL, AGENT_KB_SCOPE + global LEARNINGS_FILE, MAX_RESPONSE_PER_USER_PER_HOUR + + with open(config_path) as f: + cfg = yaml.safe_load(f) + + if cfg.get("bot_token_file"): + BOT_TOKEN_FILE = f"/opt/teleo-eval/secrets/{cfg['bot_token_file']}" + if cfg.get("response_model"): + RESPONSE_MODEL = cfg["response_model"] + if cfg.get("triage_model"): + TRIAGE_MODEL = cfg["triage_model"] + if cfg.get("learnings_file"): + LEARNINGS_FILE = f"/opt/teleo-eval/workspaces/main/{cfg['learnings_file']}" + if cfg.get("max_response_per_user_per_hour"): + MAX_RESPONSE_PER_USER_PER_HOUR = cfg["max_response_per_user_per_hour"] + if cfg.get("kb_scope", {}).get("primary"): + AGENT_KB_SCOPE = cfg["kb_scope"]["primary"] + + logger.info("Loaded agent config: %s (scope: %s)", cfg.get("name", "unknown"), + AGENT_KB_SCOPE or "all domains") + return cfg + + def main(): """Start the bot.""" + parser = argparse.ArgumentParser() + parser.add_argument("--config", help="Agent YAML config file") + parser.add_argument("--validate", action="store_true", help="Validate config and exit") + args = parser.parse_args() + + # Load agent config if provided + agent_cfg = None + if args.config: + agent_cfg = _load_agent_config(args.config) + if args.validate: + logger.info("Config valid: %s", args.config) + return + # Load token token_path = Path(BOT_TOKEN_FILE) if not token_path.exists(): @@ -1779,7 +1981,8 @@ def main(): sys.exit(1) token = token_path.read_text().strip() - logger.info("Starting Teleo Telegram bot (Rio)...") + agent_name = agent_cfg.get("name", "Rio") if agent_cfg else "Rio" + logger.info("Starting Teleo Telegram bot (%s)...", agent_name) # Initialize persistent audit connection (Ganymede + Rhea: once at startup, not per-response) global _audit_conn @@ -1794,6 +1997,12 @@ def main(): except Exception as e: logger.error("Audit DB migration failed — audit writes will fail: %s", e) + # Prebuild KB index at startup so the first query doesn't pay the 29s rebuild cost + logger.info("Prebuilding KB index...") + kb_index.ensure_fresh(max_age_seconds=0) # force immediate build + logger.info("KB index ready: %d claims, %d entities", + len(kb_index._claims), len(kb_index._entities)) + # Build application app = Application.builder().token(token).build() diff --git a/telegram/digest.py b/telegram/digest.py new file mode 100644 index 0000000..a696f46 --- /dev/null +++ b/telegram/digest.py @@ -0,0 +1,208 @@ +"""Daily digest — sends Cory a summary of all Tier 3 activity at 8am London time. + +Aggregates: merged claims (with insight summaries), pipeline metrics, agent activity, +pending review items. Runs as a scheduled job in bot.py. + +Epimetheus owns this module. +""" + +import logging +import sqlite3 +from datetime import datetime, timezone, timedelta +from zoneinfo import ZoneInfo + +logger = logging.getLogger("telegram.digest") + +LONDON_TZ = ZoneInfo("Europe/London") +DIGEST_HOUR_LONDON = 8 # 8am London time (auto-adjusts for BST/GMT) + + +def next_digest_time() -> datetime: + """Calculate the next 8am London time as a UTC datetime. + + Handles BST/GMT transitions automatically via zoneinfo. + """ + now = datetime.now(LONDON_TZ) + target = now.replace(hour=DIGEST_HOUR_LONDON, minute=0, second=0, microsecond=0) + if target <= now: + target += timedelta(days=1) + return target.astimezone(timezone.utc) + + +def _get_merged_claims_24h(conn: sqlite3.Connection) -> list[dict]: + """Get PRs merged in the last 24 hours with domain and branch info.""" + rows = conn.execute( + """SELECT number, branch, domain, agent, commit_type, merged_at, description + FROM prs + WHERE merged_at > datetime('now', '-24 hours') + AND status = 'merged' + ORDER BY merged_at DESC""", + ).fetchall() + return [dict(r) for r in rows] + + +def _get_pipeline_metrics_24h(conn: sqlite3.Connection) -> dict: + """Get pipeline activity metrics for the last 24 hours.""" + total_merged = conn.execute( + "SELECT COUNT(*) FROM prs WHERE merged_at > datetime('now', '-24 hours') AND status = 'merged'" + ).fetchone()[0] + + total_closed = conn.execute( + "SELECT COUNT(*) FROM prs WHERE status = 'closed' AND created_at > datetime('now', '-24 hours')" + ).fetchone()[0] + + total_conflict = conn.execute( + "SELECT COUNT(*) FROM prs WHERE status IN ('conflict', 'conflict_permanent') AND created_at > datetime('now', '-24 hours')" + ).fetchone()[0] + + total_open = conn.execute( + "SELECT COUNT(*) FROM prs WHERE status IN ('open', 'reviewing', 'approved', 'merging')" + ).fetchone()[0] + + # Approval rate (last 24h) + evaluated = conn.execute( + "SELECT COUNT(*) FROM prs WHERE leo_verdict IN ('approve', 'request_changes') AND created_at > datetime('now', '-24 hours')" + ).fetchone()[0] + approved = conn.execute( + "SELECT COUNT(*) FROM prs WHERE leo_verdict = 'approve' AND created_at > datetime('now', '-24 hours')" + ).fetchone()[0] + approval_rate = (approved / evaluated * 100) if evaluated > 0 else 0 + + return { + "merged": total_merged, + "closed": total_closed, + "conflict": total_conflict, + "open": total_open, + "evaluated": evaluated, + "approved": approved, + "approval_rate": approval_rate, + } + + +def _get_agent_activity_24h(conn: sqlite3.Connection) -> dict[str, int]: + """Get PR count by agent for the last 24 hours.""" + rows = conn.execute( + """SELECT agent, COUNT(*) as cnt + FROM prs + WHERE created_at > datetime('now', '-24 hours') + AND agent IS NOT NULL + GROUP BY agent + ORDER BY cnt DESC""", + ).fetchall() + return {r["agent"]: r["cnt"] for r in rows} + + +def _get_pending_review_count(conn: sqlite3.Connection) -> int: + """Count PRs awaiting review.""" + return conn.execute( + "SELECT COUNT(*) FROM prs WHERE status IN ('open', 'reviewing')" + ).fetchone()[0] + + +def _extract_claim_title(branch: str) -> str: + """Extract a human-readable claim title from a branch name. + + Branch format: extract/source-slug or agent/description + """ + # Strip prefix (extract/, research/, theseus/, etc.) + parts = branch.split("/", 1) + slug = parts[1] if len(parts) > 1 else parts[0] + # Convert slug to readable title + return slug.replace("-", " ").replace("_", " ").title() + + + +def format_digest( + merged_claims: list[dict], + metrics: dict, + agent_activity: dict[str, int], + pending_review: int, +) -> str: + """Format the daily digest message.""" + now = datetime.now(timezone.utc) + date_str = now.strftime("%Y-%m-%d") + + parts = [f"DAILY DIGEST — {date_str}", ""] + + # Merged claims section + if merged_claims: + # Group by domain + by_domain: dict[str, list] = {} + for claim in merged_claims: + domain = claim.get("domain") or "unknown" + by_domain.setdefault(domain, []).append(claim) + + parts.append(f"CLAIMS MERGED ({len(merged_claims)})") + for domain, claims in sorted(by_domain.items()): + for c in claims: + # Use real description from frontmatter if available, fall back to slug title + desc = c.get("description") + if desc: + # Take first description if multiple (pipe-delimited) + display = desc.split(" | ")[0] + if len(display) > 120: + display = display[:117] + "..." + else: + display = _extract_claim_title(c.get("branch", "unknown")) + commit_type = c.get("commit_type", "") + type_tag = f"[{commit_type}] " if commit_type else "" + parts.append(f" {type_tag}{display} ({domain})") + parts.append("") + else: + parts.extend(["CLAIMS MERGED (0)", " No claims merged in the last 24h", ""]) + + # Pipeline metrics + success_rate = 0 + total_attempted = metrics["merged"] + metrics["closed"] + metrics["conflict"] + if total_attempted > 0: + success_rate = metrics["merged"] / total_attempted * 100 + + parts.append("PIPELINE") + parts.append(f" Merged: {metrics['merged']} | Closed: {metrics['closed']} | Conflicts: {metrics['conflict']}") + parts.append(f" Success rate: {success_rate:.0f}% | Approval rate: {metrics['approval_rate']:.0f}%") + parts.append(f" Open PRs: {metrics['open']}") + parts.append("") + + # Agent activity + if agent_activity: + parts.append("AGENTS") + for agent, count in agent_activity.items(): + parts.append(f" {agent}: {count} PRs") + parts.append("") + else: + parts.extend(["AGENTS", " No agent activity in the last 24h", ""]) + + # Pending review + if pending_review > 0: + parts.append(f"PENDING YOUR REVIEW: {pending_review}") + else: + parts.append("PENDING YOUR REVIEW: 0") + + return "\n".join(parts) + + +async def send_daily_digest(context): + """Send daily digest to admin chat. Scheduled job.""" + conn = context.bot_data.get("approval_conn") + admin_chat_id = context.bot_data.get("admin_chat_id") + + if not conn or not admin_chat_id: + logger.debug("Digest skipped — no DB connection or admin chat ID") + return + + try: + merged = _get_merged_claims_24h(conn) + metrics = _get_pipeline_metrics_24h(conn) + activity = _get_agent_activity_24h(conn) + pending = _get_pending_review_count(conn) + + text = format_digest(merged, metrics, activity, pending) + + await context.bot.send_message( + chat_id=admin_chat_id, + text=text, + ) + logger.info("Daily digest sent (%d claims, %d agents active)", + len(merged), len(activity)) + except Exception as e: + logger.error("Failed to send daily digest: %s", e) diff --git a/telegram/eval.py b/telegram/eval.py new file mode 100644 index 0000000..e29bee3 --- /dev/null +++ b/telegram/eval.py @@ -0,0 +1,52 @@ +"""Eval pipeline stub — provides imports for bot.py. +Full implementation pending Ganymede review.""" + +CONFIDENCE_FLOOR = 0.3 +COST_ALERT_THRESHOLD = 0.22 + + +class _LLMResponse(str): + """str subclass carrying token counts and cost.""" + def __new__(cls, content, prompt_tokens=0, completion_tokens=0, cost=0.0, model=''): + obj = super().__new__(cls, content) + obj.prompt_tokens = prompt_tokens + obj.completion_tokens = completion_tokens + obj.cost = cost + obj.model = model + return obj + + +def estimate_cost(model: str, prompt_tokens: int, completion_tokens: int) -> float: + """Per-model cost estimation.""" + rates = { + 'anthropic/claude-opus-4': (15.0, 75.0), + 'anthropic/claude-sonnet-4': (3.0, 15.0), + 'anthropic/claude-haiku-4.5': (0.80, 4.0), + 'openai/gpt-4o': (2.50, 10.0), + } + for prefix, (input_rate, output_rate) in rates.items(): + if prefix in model: + return (prompt_tokens * input_rate + completion_tokens * output_rate) / 1_000_000 + return (prompt_tokens * 3.0 + completion_tokens * 15.0) / 1_000_000 + + +def check_url_fabrication(response: str, kb_context: str) -> tuple[str, list[str]]: + """Check for fabricated URLs. Returns (cleaned_response, fabricated_urls).""" + import re + urls = re.findall(r'https?://[^\s\)"]+', response) + if not urls or not kb_context: + return response, [] + kb_urls = set(re.findall(r'https?://[^\s\)"]+', kb_context)) + fabricated = [u for u in urls if u not in kb_urls and not u.startswith('https://t.me/')] + cleaned = response + for u in fabricated: + cleaned = cleaned.replace(u, '[URL removed]') + return cleaned, fabricated + + +def apply_confidence_floor(response: str, confidence: float | None) -> tuple[str, bool, str | None]: + """Apply confidence floor. Returns (response, blocked, block_reason).""" + if confidence is not None and confidence < CONFIDENCE_FLOOR: + caveat = '⚠️ Low confidence response — treat with skepticism.\n\n' + return caveat + response, True, f'confidence {confidence:.2f} below floor {CONFIDENCE_FLOOR}' + return response, False, None diff --git a/telegram/eval_checks.py b/telegram/eval_checks.py index 4d2f188..ebf0d49 100644 --- a/telegram/eval_checks.py +++ b/telegram/eval_checks.py @@ -18,7 +18,7 @@ MODEL_PRICING = { "openai/gpt-4o-mini": (0.15, 0.60), } -CONFIDENCE_FLOOR = 0.3 +CONFIDENCE_FLOOR = 0.4 COST_ALERT_THRESHOLD = 0.22 # per-response alert threshold in USD # URL fabrication regex — matches http:// and https:// URLs @@ -69,7 +69,7 @@ def apply_confidence_floor(display_response: str, confidence_score: float | None """ if confidence_score is not None and confidence_score < CONFIDENCE_FLOOR: modified = ( - f"⚠️ Low confidence ({confidence_score:.2f}) — treat this response with caution.\n\n" + f"⚠️ Low confidence — I may not have reliable data on this topic.\n\n" + display_response ) return modified, True, f"confidence {confidence_score:.2f} < floor {CONFIDENCE_FLOOR}" diff --git a/telegram/kb_retrieval.py b/telegram/kb_retrieval.py index ac1b73f..9b83d6a 100644 --- a/telegram/kb_retrieval.py +++ b/telegram/kb_retrieval.py @@ -300,7 +300,8 @@ class KBIndex: def retrieve_context(query: str, repo_dir: str, index: KBIndex | None = None, max_claims: int = 8, max_entities: int = 5, - max_positions: int = 3) -> KBContext: + max_positions: int = 3, + kb_scope: list[str] | None = None) -> KBContext: """Main entry point: retrieve full KB context for a query. Three layers: @@ -365,11 +366,26 @@ def retrieve_context(query: str, repo_dir: str, index: KBIndex | None = None, entity_claim_titles.add(rc.lower().replace("-", " ")) # ── Layer 2: Claim Search ── + # Import min score threshold (filters single-stopword garbage matches) + try: + from lib.config import RETRIEVAL_MIN_CLAIM_SCORE as MIN_SCORE + except ImportError: + MIN_SCORE = 3.0 + scored_claims: list[tuple[float, dict]] = [] + # Normalize kb_scope paths for prefix matching + _scope_prefixes = None + if kb_scope: + _scope_prefixes = [str(Path(repo_dir) / s) for s in kb_scope] + for claim in index._claims: + # Domain filtering: if kb_scope is set, only score claims in-scope + if _scope_prefixes: + if not any(claim["path"].startswith(p) for p in _scope_prefixes): + continue score = _score_claim(query_lower, query_tokens, claim, entity_claim_titles) - if score > 0: + if score >= MIN_SCORE: scored_claims.append((score, claim)) scored_claims.sort(key=lambda x: x[0], reverse=True) @@ -480,18 +496,22 @@ def _score_claim(query_lower: str, query_tokens: list[str], claim: dict, searchable = title + " " + desc score = 0.0 - # Substring match on full query (highest signal) - for token in query_tokens: - if len(token) >= 3 and token in searchable: + # Filter stopwords — same as entity scoring. Without this, "from", "what", "to" + # all score points and garbage like "fee revenue splits" matches on "living". + meaningful_tokens = [t for t in query_tokens if t not in _STOP_WORDS and len(t) >= 3] + + # Substring match on meaningful tokens only + for token in meaningful_tokens: + if token in searchable: score += 2.0 if token in title else 1.0 # Boost if this claim is wiki-linked from a matched entity if any(t in title for t in entity_claim_titles): score += 5.0 - # Boost multi-word matches - if len(query_tokens) >= 2: - bigrams = [f"{query_tokens[i]} {query_tokens[i+1]}" for i in range(len(query_tokens) - 1)] + # Boost multi-word matches (use meaningful tokens only) + if len(meaningful_tokens) >= 2: + bigrams = [f"{meaningful_tokens[i]} {meaningful_tokens[i+1]}" for i in range(len(meaningful_tokens) - 1)] for bg in bigrams: if bg in searchable: score += 3.0 diff --git a/telegram/kb_tools.py b/telegram/kb_tools.py new file mode 100644 index 0000000..22376ca --- /dev/null +++ b/telegram/kb_tools.py @@ -0,0 +1,719 @@ +#!/usr/bin/env python3 +"""KB tools for LLM function-calling — source tracing + entity/claim lookup. + +These tools let the agent trace claims back to their original sources, +find all claims from a specific piece of research, and read source documents. + +Epimetheus owns this module. +""" + +import logging +import os +import re +from pathlib import Path + +import yaml + +logger = logging.getLogger("tg.kb_tools") + + +# ─── Tool definitions (OpenAI function-calling format) ─────────────── + +TOOL_DEFINITIONS = [ + { + "type": "function", + "function": { + "name": "find_by_source", + "description": ( + "Find all claims extracted from a specific source (article, paper, thread). " + "Search by author name, source title, or keywords. Returns all claims from " + "matching sources with their frontmatter." + ), + "parameters": { + "type": "object", + "properties": { + "query": { + "type": "string", + "description": "Author name, source title, or keywords to match against claim source fields", + }, + }, + "required": ["query"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "read_source", + "description": ( + "Read the original source document (article, thread, paper) that claims were " + "extracted from. Use when you need the full context behind a claim, not just " + "the extracted summary." + ), + "parameters": { + "type": "object", + "properties": { + "source_title": { + "type": "string", + "description": "Title or slug of the source document to read", + }, + }, + "required": ["source_title"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "read_entity", + "description": "Read the full profile of a KB entity (project, person, protocol).", + "parameters": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Entity name or slug", + }, + }, + "required": ["name"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "list_entity_links", + "description": "List all entities and claims linked from an entity's wiki-links.", + "parameters": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Entity name or slug", + }, + }, + "required": ["name"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "read_claim", + "description": "Read the full content of a specific claim file.", + "parameters": { + "type": "object", + "properties": { + "title": { + "type": "string", + "description": "Claim title or slug", + }, + }, + "required": ["title"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "search_kb", + "description": "Search the KB for claims matching a query. Uses keyword matching.", + "parameters": { + "type": "object", + "properties": { + "query": { + "type": "string", + "description": "Search query", + }, + "max_results": { + "type": "integer", + "description": "Max results to return (default 5)", + }, + }, + "required": ["query"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "explore_graph", + "description": ( + "Follow knowledge graph edges from a claim to find connected claims. " + "Returns all claims linked via supports, challenges, depends_on, and related edges. " + "Use this to discover the full argument structure around a claim — what supports it, " + "what challenges it, and what it depends on." + ), + "parameters": { + "type": "object", + "properties": { + "claim_title": { + "type": "string", + "description": "Title or slug of the claim to explore edges from", + }, + }, + "required": ["claim_title"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "search_sources", + "description": ( + "Search the source archive for original documents by topic, author, or title. " + "Returns matching source files with their titles and first few lines. " + "Use this when you want to find the original research/article/thread, not just extracted claims." + ), + "parameters": { + "type": "object", + "properties": { + "query": { + "type": "string", + "description": "Topic, author name, or keywords to search source documents", + }, + "max_results": { + "type": "integer", + "description": "Max results to return (default 5)", + }, + }, + "required": ["query"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "pr_status", + "description": ( + "Check the status of a pipeline PR by number. Returns eval verdicts, " + "merge status, time in queue, rejection reasons, and retry counts." + ), + "parameters": { + "type": "object", + "properties": { + "pr_number": { + "type": "integer", + "description": "PR number to look up", + }, + }, + "required": ["pr_number"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "check_duplicate", + "description": ( + "Check if a claim is a near-duplicate of existing KB content. " + "Returns top-3 closest matches with similarity scores. " + ">=0.85 = likely duplicate, 0.70-0.85 = check manually, <0.70 = novel." + ), + "parameters": { + "type": "object", + "properties": { + "text": { + "type": "string", + "description": "The claim text to check for duplicates", + }, + }, + "required": ["text"], + }, + }, + }, +] + + +# ─── Tool implementations ──────────────────────────────────────────── + + +def find_by_source(query: str, kb_dir: str) -> str: + """Find all claims extracted from sources matching the query. + + Searches claim frontmatter `source:` fields for author names, titles, keywords. + Returns structured list of all claims from matching sources. + """ + query_lower = query.lower() + query_tokens = [t for t in re.findall(r'\w+', query_lower) if len(t) >= 3] + + # Scan all claim files for matching source fields + matches: list[dict] = [] + claim_dirs = [ + Path(kb_dir) / "domains", + Path(kb_dir) / "core", + Path(kb_dir) / "foundations", + ] + + for claim_dir in claim_dirs: + if not claim_dir.exists(): + continue + for md_file in claim_dir.rglob("*.md"): + if md_file.name.startswith("_"): + continue + try: + fm, body = _parse_frontmatter(md_file) + if not fm: + continue + source = fm.get("source", "") + source_file = fm.get("source_file", "") + searchable = f"{source} {source_file}".lower() + + # Score: how many query tokens appear in the source field + score = sum(1 for t in query_tokens if t in searchable) + if score >= max(1, len(query_tokens) // 2): + matches.append({ + "title": md_file.stem.replace("-", " "), + "path": str(md_file.relative_to(kb_dir)), + "source": source, + "source_file": source_file, + "domain": fm.get("domain", "unknown"), + "confidence": fm.get("confidence", "unknown"), + "description": fm.get("description", ""), + "score": score, + }) + except Exception: + continue + + if not matches: + return f"No claims found from sources matching '{query}'." + + # Sort by score desc, group by source + matches.sort(key=lambda m: m["score"], reverse=True) + + # Group by source + by_source: dict[str, list[dict]] = {} + for m in matches: + key = m["source"] or "unknown" + by_source.setdefault(key, []).append(m) + + lines = [f"Found {len(matches)} claims from {len(by_source)} matching sources:\n"] + for source_name, claims in list(by_source.items())[:5]: # Cap at 5 sources + lines.append(f"## Source: {source_name}") + if claims[0].get("source_file"): + lines.append(f"File: {claims[0]['source_file']}") + for c in claims[:10]: # Cap at 10 claims per source + lines.append(f"- **{c['title']}** ({c['confidence']}, {c['domain']})") + if c["description"]: + lines.append(f" {c['description'][:200]}") + lines.append("") + + return "\n".join(lines)[:4000] + + +def read_source(source_title: str, kb_dir: str) -> str: + """Read the original source document from the archive. + + Looks in inbox/archive/ and sources/ for matching files. + """ + title_lower = source_title.lower() + slug = re.sub(r'[^a-z0-9]+', '-', title_lower).strip('-') + + # Search paths for source files + search_dirs = [ + Path(kb_dir) / "inbox" / "archive", + Path(kb_dir) / "sources", + Path(kb_dir) / "inbox" / "queue", + ] + + best_match = None + best_score = 0 + + for search_dir in search_dirs: + if not search_dir.exists(): + continue + for md_file in search_dir.rglob("*.md"): + file_slug = md_file.stem.lower() + # Score by token overlap + score = 0 + for token in re.findall(r'\w+', title_lower): + if len(token) >= 3 and token in file_slug: + score += 1 + if slug in file_slug: + score += 5 # Exact slug match + if score > best_score: + best_score = score + best_match = md_file + + if not best_match: + return f"Source document '{source_title}' not found in archive." + + try: + content = best_match.read_text(errors="replace") + # Truncate to 4K for prompt safety + if len(content) > 4000: + content = content[:4000] + "\n\n[... truncated, full document is longer ...]" + return f"## Source: {best_match.name}\n\n{content}" + except Exception as e: + return f"Error reading source: {e}" + + +def read_entity(name: str, kb_dir: str) -> str: + """Read the full profile of a KB entity.""" + entity_file = _find_file(name, [ + Path(kb_dir) / "entities", + Path(kb_dir) / "decisions", + ]) + if not entity_file: + return f"Entity '{name}' not found." + try: + content = entity_file.read_text(errors="replace") + return content[:4000] + except Exception as e: + return f"Error reading entity: {e}" + + +def list_entity_links(name: str, kb_dir: str) -> str: + """List all wiki-links from an entity file, with dedup.""" + entity_file = _find_file(name, [ + Path(kb_dir) / "entities", + Path(kb_dir) / "decisions", + ]) + if not entity_file: + return f"Entity '{name}' not found." + + try: + content = entity_file.read_text(errors="replace") + links = re.findall(r"\[\[([^\]]+)\]\]", content) + # Dedup while preserving order + seen = set() + unique_links = [] + for link in links: + if link.lower() not in seen: + seen.add(link.lower()) + unique_links.append(link) + if not unique_links: + return f"Entity '{name}' has no wiki-links." + return f"Entity '{name}' links to {len(unique_links)} items:\n" + "\n".join( + f"- [[{link}]]" for link in unique_links + ) + except Exception as e: + return f"Error reading entity links: {e}" + + +def read_claim(title: str, kb_dir: str) -> str: + """Read the full content of a claim file.""" + claim_file = _find_file(title, [ + Path(kb_dir) / "domains", + Path(kb_dir) / "core", + Path(kb_dir) / "foundations", + ]) + if not claim_file: + return f"Claim '{title}' not found." + try: + content = claim_file.read_text(errors="replace") + return content[:4000] + except Exception as e: + return f"Error reading claim: {e}" + + +def search_kb(query: str, kb_dir: str, max_results: int = 5) -> str: + """Search KB claims by keyword matching.""" + from kb_retrieval import KBIndex, retrieve_context + index = KBIndex(kb_dir) + index.ensure_fresh() + ctx = retrieve_context(query, kb_dir, index=index, max_claims=max_results) + if not ctx.claims: + return f"No claims found for '{query}'." + lines = [f"Found {len(ctx.claims)} claims:"] + for c in ctx.claims: + lines.append(f"- **{c.title}** ({c.confidence}, {c.domain}, score: {c.score:.1f})") + if c.description: + lines.append(f" {c.description[:200]}") + return "\n".join(lines) + + +def explore_graph(claim_title: str, kb_dir: str) -> str: + """Follow knowledge graph edges from a claim to find connected claims. + + Uses lib/search.py graph_expand() for 1-hop traversal of supports/challenges/ + depends_on/related edges in frontmatter. + """ + # Find the claim file first + claim_file = _find_file(claim_title, [ + Path(kb_dir) / "domains", + Path(kb_dir) / "core", + Path(kb_dir) / "foundations", + ]) + if not claim_file: + return f"Claim '{claim_title}' not found. Try a different title or use search_kb to find it first." + + try: + rel_path = str(claim_file.relative_to(kb_dir)) + except ValueError: + rel_path = str(claim_file) + + # Use the existing graph_expand from lib/search.py + try: + from lib.search import graph_expand + expanded = graph_expand([rel_path], repo_root=Path(kb_dir), max_expanded=20) + except ImportError: + # Fallback: parse edges directly from the file + expanded = [] + fm, body = _parse_frontmatter(claim_file) + if fm: + for edge_type in ("supports", "challenges", "challenged_by", "depends_on", "related"): + targets = fm.get(edge_type, []) + if isinstance(targets, str): + targets = [targets] + if isinstance(targets, list): + for t in targets: + expanded.append({"claim_title": t, "edge_type": edge_type, "edge_weight": 1.0}) + + if not expanded: + return f"Claim '{claim_title}' has no graph edges (no supports, challenges, or related claims)." + + # Group by edge type for readability + by_type: dict[str, list[dict]] = {} + for e in expanded: + by_type.setdefault(e["edge_type"], []).append(e) + + lines = [f"Graph edges from '{claim_title}' ({len(expanded)} connected claims):\n"] + type_labels = { + "supports": "Supports (this claim backs these up)", + "challenges": "Challenges (this claim argues against these)", + "challenged_by": "Challenged by (these argue against this claim)", + "depends_on": "Depends on (prerequisites for this claim)", + "related": "Related (connected by topic)", + "wiki_links": "Wiki-linked (mentioned in body text)", + } + for edge_type, items in by_type.items(): + label = type_labels.get(edge_type, edge_type) + lines.append(f"### {label}") + for item in items: + title = item.get("claim_title", "unknown") + weight = item.get("edge_weight", 1.0) + lines.append(f"- {title}" + (f" (weight: {weight})" if weight != 1.0 else "")) + lines.append("") + + return "\n".join(lines)[:4000] + + +def search_sources(query: str, kb_dir: str, max_results: int = 5) -> str: + """Search the source archive for original documents by topic/author/title. + + Scans inbox/archive/ and sources/ directories, scoring by token overlap. + """ + query_lower = query.lower() + query_tokens = [t for t in re.findall(r'\w+', query_lower) if len(t) >= 3] + + if not query_tokens: + return "Query too short — provide at least one keyword with 3+ characters." + + search_dirs = [ + Path(kb_dir) / "inbox" / "archive", + Path(kb_dir) / "sources", + Path(kb_dir) / "inbox" / "queue", + ] + + matches: list[dict] = [] + for search_dir in search_dirs: + if not search_dir.exists(): + continue + for md_file in search_dir.rglob("*.md"): + if md_file.name.startswith("_"): + continue + file_stem = md_file.stem.lower().replace("-", " ") + # Score by token overlap with filename + score = sum(1 for t in query_tokens if t in file_stem) + # Also check first 500 chars of file content for author/topic + if score == 0: + try: + head = md_file.read_text(errors="replace")[:500].lower() + score = sum(0.5 for t in query_tokens if t in head) + except Exception: + continue + if score >= max(1, len(query_tokens) // 3): + # Read first few lines for preview + try: + preview = md_file.read_text(errors="replace")[:300].strip() + except Exception: + preview = "(could not read)" + matches.append({ + "title": md_file.stem.replace("-", " "), + "path": str(md_file.relative_to(kb_dir)), + "score": score, + "preview": preview, + }) + + if not matches: + return f"No source documents found matching '{query}'. Try different keywords or check find_by_source for claims from that source." + + matches.sort(key=lambda m: m["score"], reverse=True) + matches = matches[:max_results] + + lines = [f"Found {len(matches)} source documents:\n"] + for m in matches: + lines.append(f"### {m['title']}") + lines.append(f"Path: {m['path']}") + lines.append(f"{m['preview'][:200]}") + lines.append("") + + return "\n".join(lines)[:4000] + + +# ─── Tool dispatcher ───────────────────────────────────────────────── + + +def execute_tool(tool_name: str, args: dict, kb_dir: str) -> str: + """Dispatch a tool call by name. Returns the tool's string result.""" + if tool_name == "find_by_source": + return find_by_source(args.get("query", ""), kb_dir) + elif tool_name == "read_source": + return read_source(args.get("source_title", ""), kb_dir) + elif tool_name == "read_entity": + return read_entity(args.get("name", ""), kb_dir) + elif tool_name == "list_entity_links": + return list_entity_links(args.get("name", ""), kb_dir) + elif tool_name == "read_claim": + return read_claim(args.get("title", ""), kb_dir) + elif tool_name == "search_kb": + return search_kb(args.get("query", ""), kb_dir, args.get("max_results", 5)) + elif tool_name == "explore_graph": + return explore_graph(args.get("claim_title", ""), kb_dir) + elif tool_name == "search_sources": + return search_sources(args.get("query", ""), kb_dir, args.get("max_results", 5)) + elif tool_name == "pr_status": + return _tool_pr_status(args.get("pr_number", 0)) + elif tool_name == "check_duplicate": + return _tool_check_duplicate(args.get("text", "")) + else: + return f"Unknown tool: {tool_name}" + + +# ─── Helpers ───────────────────────────────────────────────────────── + + +def _parse_frontmatter(path: Path) -> tuple[dict | None, str]: + """Parse YAML frontmatter and body from a markdown file.""" + try: + text = path.read_text(errors="replace") + except Exception: + return None, "" + + if not text.startswith("---"): + return None, text + + end = text.find("\n---", 3) + if end == -1: + return None, text + + try: + fm = yaml.safe_load(text[3:end]) + if not isinstance(fm, dict): + return None, text + body = text[end + 4:].strip() + return fm, body + except yaml.YAMLError: + return None, text + + +def _find_file(name: str, search_dirs: list[Path]) -> Path | None: + """Find a markdown file by name/slug across search directories.""" + slug = re.sub(r'[^a-z0-9]+', '-', name.lower()).strip('-') + name_lower = name.lower() + + for search_dir in search_dirs: + if not search_dir.exists(): + continue + for md_file in search_dir.rglob("*.md"): + if md_file.name.startswith("_"): + continue + stem_lower = md_file.stem.lower() + # Exact slug match + if stem_lower == slug: + return md_file + # Normalized match (spaces vs hyphens) + if stem_lower.replace("-", " ") == name_lower.replace("-", " "): + return md_file + # Substring match for long titles + if len(slug) >= 8 and slug in stem_lower: + return md_file + + return None + + +# ─── Pipeline DB tools ────────────────────────────────────────────── + + +def _tool_pr_status(pr_number: int) -> str: + """Wrapper for pr_status() — connects to pipeline DB, returns formatted string.""" + import json + import sqlite3 + + db_path = os.environ.get("PIPELINE_DB", "/opt/teleo-eval/pipeline/pipeline.db") + try: + conn = sqlite3.connect(db_path) + conn.row_factory = sqlite3.Row + + row = conn.execute( + """SELECT number, branch, source_path, status, domain, agent, + commit_type, tier, leo_verdict, domain_verdict, + domain_agent, eval_issues, priority, origin, + cost_usd, created_at, merged_at, last_attempt, last_error, + transient_retries, substantive_retries, description + FROM prs WHERE number = ?""", + (pr_number,), + ).fetchone() + conn.close() + + if not row: + return f"PR #{pr_number} not found." + + issues = [] + try: + issues = json.loads(row["eval_issues"] or "[]") + except (json.JSONDecodeError, TypeError): + pass + + lines = [ + f"PR #{row['number']} — {row['status'].upper()}", + f"Branch: {row['branch']}", + f"Domain: {row['domain'] or 'unknown'} | Agent: {row['agent'] or 'pipeline'}", + f"Type: {row['commit_type'] or 'unknown'} | Tier: {row['tier'] or 'unknown'}", + f"Leo verdict: {row['leo_verdict']} | Domain verdict: {row['domain_verdict']}", + ] + if row["description"]: + lines.append(f"Description: {row['description']}") + if issues: + lines.append(f"Eval issues: {', '.join(str(i) for i in issues)}") + if row["last_error"]: + lines.append(f"Last error: {row['last_error'][:200]}") + lines.append(f"Retries: {row['transient_retries']} transient, {row['substantive_retries']} substantive") + lines.append(f"Created: {row['created_at']} | Last attempt: {row['last_attempt']}") + if row["merged_at"]: + lines.append(f"Merged: {row['merged_at']}") + if row["cost_usd"]: + lines.append(f"Eval cost: ${row['cost_usd']:.4f}") + + return "\n".join(lines) + except Exception as e: + return f"Error querying PR #{pr_number}: {e}" + + +def _tool_check_duplicate(text: str) -> str: + """Wrapper for check_duplicate() — calls Qdrant, returns formatted string.""" + import sys + sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) + from lib.search import check_duplicate as _check_dup + + if not text: + return "Error: text is required." + + result = _check_dup(text) + + if result.get("error"): + return f"Error: {result['error']}" + + lines = [f"Verdict: {result['verdict'].upper()} (highest score: {result['highest_score']:.4f})"] + + for i, m in enumerate(result["matches"], 1): + lines.append( + f" {i}. [{m['score']:.4f}] {m['claim_title'][:80]}" + f"\n Path: {m['claim_path']}" + ) + + if not result["matches"]: + lines.append(" No matches found above minimum threshold.") + + return "\n".join(lines) diff --git a/telegram/opsec-entities.txt b/telegram/opsec-entities.txt new file mode 100644 index 0000000..ec08ec5 --- /dev/null +++ b/telegram/opsec-entities.txt @@ -0,0 +1,6 @@ +# Sensitive entity names — deal-adjacent entities whose mention implies private context. +# One entity per line. Regex patterns (case-insensitive). +# Add entities as deals evolve, remove when public. +# No code changes needed — just edit this file and restart the bot. +Devoted\s+Health +Centricus diff --git a/telegram/output_gate.py b/telegram/output_gate.py new file mode 100644 index 0000000..00403ae --- /dev/null +++ b/telegram/output_gate.py @@ -0,0 +1,147 @@ +"""Output gate — classifies content as system/internal vs public-facing. + +Blocks pipeline messages (extraction logs, merge notifications, diagnostics) +from ever reaching the tweet queue or any public-facing output. + +This is a deterministic classifier — no LLM calls. Pattern matching on content. + +Epimetheus owns this module. +""" + +import re + +# ─── System Message Patterns ───────────────────────────────────────── +# Content matching ANY of these is classified as system/internal. + +_SYSTEM_PATTERNS = [ + # Pipeline operations + re.compile(r"\b(PR\s*#\d+|pull request|merge|rebase|cherry.?pick)\b", re.IGNORECASE), + re.compile(r"\b(extraction|extracted|extractor|extract/)\b", re.IGNORECASE), + re.compile(r"\b(pipeline|cron|batch.?extract|systemd|teleo-pipeline)\b", re.IGNORECASE), + re.compile(r"\b(conflict.?permanent|conflict.?closed|merge.?conflict)\b", re.IGNORECASE), + + # Infrastructure / ops + re.compile(r"\b(schema\s*v\d+|migration\s*v\d+|SCHEMA_VERSION)\b", re.IGNORECASE), + re.compile(r"\b(deploy|VPS|ssh|scp|systemctl|journalctl)\b", re.IGNORECASE), + re.compile(r"\b(Qdrant|embed.?on.?merge|vector.?gc|backfill)\b", re.IGNORECASE), + re.compile(r"\b(ReadWritePaths|ProtectSystem|ExecStartPre)\b", re.IGNORECASE), + + # Diagnostics + re.compile(r"\b(vital.?signs|queue.?staleness|orphan.?ratio)\b", re.IGNORECASE), + re.compile(r"\b(approval.?rate|throughput|PRs?.?per.?hour)\b", re.IGNORECASE), + re.compile(r"\b(reviewer_count|reviewer.?backfill)\b", re.IGNORECASE), + + # Agent coordination internals + re.compile(r"\b(Ganymede|Rhea|Oberon)\s+(review(?:ed)?|approv(?:ed|es?)|reject(?:ed|s)?)\b", re.IGNORECASE), + re.compile(r"\b(PIPELINE_OWNED_PREFIXES|AGENT_NAMES)\b"), + re.compile(r"\b(worktree|bare.?repo|forgejo|git\.livingip)\b", re.IGNORECASE), + + # Code / technical + re.compile(r"\b(def\s+\w+|import\s+\w+|class\s+\w+)\b"), + re.compile(r"\b(\.py|\.yaml|\.json|\.md)\s", re.IGNORECASE), + re.compile(r"\b(sqlite3?|pipeline\.db|response_audit)\b", re.IGNORECASE), + + # Internal metrics / debugging + re.compile(r"\b(cosine.?sim|threshold|PRIOR_ART_THRESHOLD)\b", re.IGNORECASE), + re.compile(r"\b(pre.?screen|Layer\s*[01234]|RRF|entity.?boost)\b", re.IGNORECASE), + + # Paths + re.compile(r"/opt/teleo-eval/"), + re.compile(r"/Users/\w+/"), + re.compile(r"\.pentagon/"), +] + +# ─── Public Content Signals ────────────────────────────────────────── +# Content matching these is MORE LIKELY to be public-facing. +# These don't override system classification — they're tiebreakers. + +_PUBLIC_SIGNALS = [ + re.compile(r"^(thread|tweet|post):", re.IGNORECASE | re.MULTILINE), + re.compile(r"\b(insight|analysis|take|perspective|argument)\b", re.IGNORECASE), + re.compile(r"\b(audience|followers|engagement|impression)\b", re.IGNORECASE), +] + + +class GateResult: + """Result of output gate classification.""" + + __slots__ = ("is_public", "blocked_reasons", "confidence") + + def __init__(self, is_public: bool, blocked_reasons: list[str], confidence: float): + self.is_public = is_public + self.blocked_reasons = blocked_reasons + self.confidence = confidence + + def __bool__(self): + return self.is_public + + def __repr__(self): + status = "PUBLIC" if self.is_public else "BLOCKED" + return f"GateResult({status}, reasons={self.blocked_reasons}, conf={self.confidence:.2f})" + + +def classify(content: str) -> GateResult: + """Classify content as public-facing or system/internal. + + Returns GateResult: + - is_public=True: safe for tweet queue / public output + - is_public=False: system content, blocked from public outputs + """ + if not content or not content.strip(): + return GateResult(False, ["empty content"], 1.0) + + # Count system pattern matches + system_hits = [] + for pattern in _SYSTEM_PATTERNS: + match = pattern.search(content) + if match: + system_hits.append(match.group()) + + # Count public signals + public_hits = sum(1 for p in _PUBLIC_SIGNALS if p.search(content)) + + # Decision logic + if len(system_hits) >= 3: + # Strong system signal — definitely internal + return GateResult(False, system_hits[:5], 0.95) + + if len(system_hits) >= 1 and public_hits == 0: + # Some system signal, no public signal — likely internal + return GateResult(False, system_hits, 0.75) + + if len(system_hits) == 0: + # No system signal — public + return GateResult(True, [], 0.90 if public_hits > 0 else 0.70) + + # Mixed signals (system hits + public signals) — default to blocking + # Better to block a borderline tweet than leak system info + return GateResult(False, system_hits, 0.50) + + +def gate_for_tweet_queue(content: str, agent: str = None) -> GateResult: + """Gate specifically for the tweet queue. Stricter than general classify. + + Additional checks: + - OPSEC filter (imported from approvals) + - Agent attribution check + """ + result = classify(content) + if not result.is_public: + return result + + # Additional tweet-specific checks + blocked = [] + + # Must not be too short (probably a fragment or command) + stripped = content.strip() + if len(stripped) < 20: + blocked.append("content too short for tweet (<20 chars)") + + # Must not contain raw URLs to internal systems + if re.search(r"https?://(?:localhost|127\.0\.0\.1|77\.42\.65\.182)", stripped): + blocked.append("contains internal URL") + + if blocked: + return GateResult(False, blocked, 0.85) + + return result diff --git a/telegram/response.py b/telegram/response.py index 3811adf..b01724c 100644 --- a/telegram/response.py +++ b/telegram/response.py @@ -44,17 +44,21 @@ def build_system_prompt( """ return f"""You are Rio, the Teleo internet finance agent. Your Telegram handle is @FutAIrdBot — that IS you. Users tag @FutAIrdBot to reach you. Never say "I'm not FutAIrdBot." You are also @futaRdIO on X. You have deep knowledge about futarchy, prediction markets, token governance, and the MetaDAO ecosystem. +## RESPONSE LENGTH — CRITICAL +Default to SHORT responses. 1-3 sentences for simple questions. Match the length of the question. +Only go longer when the user explicitly asks for depth, analysis, or a breakdown. +If you catch yourself writing more than one paragraph, stop and ask: "Did they ask for this much?" If not, cut it. + ## How to sound Write like a sharp analyst talking to peers, not like an AI. Specifically: - Use your knowledge naturally. Don't say "the KB tracks" or "at experimental confidence" or "our claims show." Just state what you know and how confident you are in plain language. - Have a take. You're an analyst, not a summarizer. Say what you actually think. -- Before you respond, ask yourself: "Does every sentence here add something the user doesn't already know?" If a sentence just restates context, agrees without adding insight, or pads with filler — cut it. Your goal is signal density, not word count. -- Short questions deserve short answers. If someone asks a factual question, give the fact. Don't surround it with caveats, context, and "the honest picture is" framing. -- Long answers are fine when the question is genuinely complex or the user asks for depth. But earn every paragraph — each one should contain a distinct insight the previous one didn't cover. -- Match the user's energy. If they wrote one line, respond in kind. +- Every sentence must add something the user doesn't already know. Cut filler, restatements, and padding ruthlessly. +- Short questions deserve short answers. Give the fact, not a framing essay. +- Match the user's energy. One-line question = one-line answer. - Sound human. No em dashes, no "That said", no "It's worth noting." Just say the thing. - No markdown. Plain text only. -- When you're uncertain, just say so simply. "I'm not sure about X" beats "we don't have data on this yet." +- When you're uncertain, just say so simply. "Not sure about X" — done. ## Your learnings (corrections from past conversations — prioritize these over KB data when they conflict) {learnings} diff --git a/telegram/retrieval.py b/telegram/retrieval.py index 2d8a346..466fd48 100644 --- a/telegram/retrieval.py +++ b/telegram/retrieval.py @@ -288,6 +288,7 @@ async def orchestrate_retrieval( triage_model: str, retrieve_context_fn: Callable, retrieve_vector_fn: Callable[[str], tuple[str, dict]], + kb_scope: list[str] | None = None, ) -> dict: """Full retrieval pipeline: keyword → decompose → vector → RRF merge. @@ -299,7 +300,7 @@ async def orchestrate_retrieval( # 1. Keyword retrieval (entity resolution needs full context) t_kb = time.monotonic() - kb_ctx = retrieve_context_fn(search_query, kb_read_dir, index=kb_index) + kb_ctx = retrieve_context_fn(search_query, kb_read_dir, index=kb_index, kb_scope=kb_scope) kb_duration = int((time.monotonic() - t_kb) * 1000) retrieval_layers = ["keyword"] if (kb_ctx and (kb_ctx.entities or kb_ctx.claims)) else [] tool_calls.append({ diff --git a/telegram/x_publisher.py b/telegram/x_publisher.py new file mode 100644 index 0000000..00d12aa --- /dev/null +++ b/telegram/x_publisher.py @@ -0,0 +1,347 @@ +"""X (Twitter) publisher — posts approved tweets to X. + +Handles the full tweet lifecycle: +1. Agent submits draft → output gate blocks system content +2. Draft enters approval_queue (type='tweet') +3. Leo reviews substance → Cory approves via Telegram +4. On approval, this module posts to X via API +5. Records published URL and metrics + +Uses Twitter API v2 via OAuth 1.0a for posting. +Read operations still use twitterapi.io (x_client.py). + +Epimetheus owns this module. +""" + +import json +import hashlib +import hmac +import logging +import sqlite3 +import time +import urllib.parse +from pathlib import Path +from typing import Optional + +import aiohttp + +logger = logging.getLogger("x-publisher") + +# ─── Config ────────────────────────────────────────────────────────── + +# Twitter API v2 credentials for posting +# OAuth 1.0a keys — stored in separate secret files +_SECRETS_DIR = Path("/opt/teleo-eval/secrets") +_CONSUMER_KEY_FILE = _SECRETS_DIR / "x-consumer-key" +_CONSUMER_SECRET_FILE = _SECRETS_DIR / "x-consumer-secret" +_ACCESS_TOKEN_FILE = _SECRETS_DIR / "x-access-token" +_ACCESS_SECRET_FILE = _SECRETS_DIR / "x-access-secret" + +TWITTER_API_V2_URL = "https://api.twitter.com/2/tweets" +REQUEST_TIMEOUT = 15 + + +def _load_secret(path: Path) -> Optional[str]: + """Load a secret from a file. Returns None if missing.""" + try: + return path.read_text().strip() + except Exception: + return None + + +def _load_oauth_credentials() -> Optional[dict]: + """Load all 4 OAuth 1.0a credentials. Returns None if any missing.""" + creds = { + "consumer_key": _load_secret(_CONSUMER_KEY_FILE), + "consumer_secret": _load_secret(_CONSUMER_SECRET_FILE), + "access_token": _load_secret(_ACCESS_TOKEN_FILE), + "access_secret": _load_secret(_ACCESS_SECRET_FILE), + } + missing = [k for k, v in creds.items() if not v] + if missing: + logger.warning("Missing X API credentials: %s", ", ".join(missing)) + return None + return creds + + +# ─── OAuth 1.0a Signature ──────────────────────────────────────────── + +def _percent_encode(s: str) -> str: + return urllib.parse.quote(str(s), safe="") + + +def _generate_oauth_signature( + method: str, + url: str, + params: dict, + consumer_secret: str, + token_secret: str, +) -> str: + """Generate OAuth 1.0a signature.""" + sorted_params = "&".join( + f"{_percent_encode(k)}={_percent_encode(v)}" + for k, v in sorted(params.items()) + ) + base_string = f"{method.upper()}&{_percent_encode(url)}&{_percent_encode(sorted_params)}" + signing_key = f"{_percent_encode(consumer_secret)}&{_percent_encode(token_secret)}" + signature = hmac.new( + signing_key.encode(), base_string.encode(), hashlib.sha1 + ).digest() + import base64 + return base64.b64encode(signature).decode() + + +def _build_oauth_header( + method: str, + url: str, + creds: dict, + extra_params: dict = None, +) -> str: + """Build the OAuth 1.0a Authorization header.""" + import uuid + oauth_params = { + "oauth_consumer_key": creds["consumer_key"], + "oauth_nonce": uuid.uuid4().hex, + "oauth_signature_method": "HMAC-SHA1", + "oauth_timestamp": str(int(time.time())), + "oauth_token": creds["access_token"], + "oauth_version": "1.0", + } + + # Combine oauth params with any extra params for signature + all_params = {**oauth_params} + if extra_params: + all_params.update(extra_params) + + signature = _generate_oauth_signature( + method, url, all_params, + creds["consumer_secret"], creds["access_secret"], + ) + oauth_params["oauth_signature"] = signature + + header_parts = ", ".join( + f'{_percent_encode(k)}="{_percent_encode(v)}"' + for k, v in sorted(oauth_params.items()) + ) + return f"OAuth {header_parts}" + + +# ─── Tweet Submission ──────────────────────────────────────────────── + +def submit_tweet_draft( + conn: sqlite3.Connection, + content: str, + agent: str, + context: dict = None, + reply_to_url: str = None, + post_type: str = "original", +) -> tuple[int, str]: + """Submit a tweet draft to the approval queue. + + Returns (request_id, status_message). + status_message is None on success, error string on failure. + + The output gate and OPSEC filter run before insertion. + """ + # Import here to avoid circular dependency + from output_gate import gate_for_tweet_queue + from approvals import check_opsec + + # Output gate — block system content + gate = gate_for_tweet_queue(content, agent) + if not gate: + return -1, f"Output gate blocked: {', '.join(gate.blocked_reasons)}" + + # OPSEC filter + opsec_violation = check_opsec(content) + if opsec_violation: + return -1, opsec_violation + + # Build context JSON + ctx = { + "post_type": post_type, + "target_account": "TeleoHumanity", # default, can be overridden + } + if reply_to_url: + ctx["reply_to_url"] = reply_to_url + if context: + ctx.update(context) + + # Insert into approval queue + cursor = conn.execute( + """INSERT INTO approval_queue + (type, content, originating_agent, context, leo_review_status, + expires_at) + VALUES (?, ?, ?, ?, 'pending_leo', + datetime('now', '+24 hours'))""", + ("tweet", content, agent, json.dumps(ctx)), + ) + conn.commit() + request_id = cursor.lastrowid + logger.info("Tweet draft #%d submitted by %s (%d chars)", + request_id, agent, len(content)) + return request_id, None + + +# ─── Tweet Posting ─────────────────────────────────────────────────── + +async def post_tweet(text: str, reply_to_id: str = None) -> dict: + """Post a tweet to X via Twitter API v2. + + Returns dict with: + - success: bool + - tweet_id: str (if successful) + - tweet_url: str (if successful) + - error: str (if failed) + """ + creds = _load_oauth_credentials() + if not creds: + return {"success": False, "error": "X API credentials not configured"} + + # Build request body + body = {"text": text} + if reply_to_id: + body["reply"] = {"in_reply_to_tweet_id": reply_to_id} + + # OAuth 1.0a header (for JSON body, don't include body params in signature) + auth_header = _build_oauth_header("POST", TWITTER_API_V2_URL, creds) + + headers = { + "Authorization": auth_header, + "Content-Type": "application/json", + } + + try: + async with aiohttp.ClientSession() as session: + async with session.post( + TWITTER_API_V2_URL, + headers=headers, + json=body, + timeout=aiohttp.ClientTimeout(total=REQUEST_TIMEOUT), + ) as resp: + result = await resp.json() + + if resp.status == 201: + tweet_id = result.get("data", {}).get("id", "") + return { + "success": True, + "tweet_id": tweet_id, + "tweet_url": f"https://x.com/TeleoHumanity/status/{tweet_id}", + } + else: + error = result.get("detail") or result.get("title") or str(result) + logger.error("Tweet post failed (%d): %s", resp.status, error) + return {"success": False, "error": f"API error {resp.status}: {error}"} + + except aiohttp.ClientError as e: + logger.error("Tweet post network error: %s", e) + return {"success": False, "error": f"Network error: {e}"} + + +async def post_thread(tweets: list[str]) -> list[dict]: + """Post a thread (multiple tweets in reply chain). + + Returns list of post results, one per tweet. + """ + results = [] + reply_to = None + + for i, text in enumerate(tweets): + result = await post_tweet(text, reply_to_id=reply_to) + results.append(result) + + if not result["success"]: + logger.error("Thread posting failed at tweet %d/%d: %s", + i + 1, len(tweets), result["error"]) + break + + reply_to = result.get("tweet_id") + + return results + + +# ─── Post-Approval Hook ───────────────────────────────────────────── + +async def handle_approved_tweet( + conn: sqlite3.Connection, + request_id: int, +) -> dict: + """Called when a tweet is approved. Posts to X and records the result. + + Returns the post result dict. + """ + row = conn.execute( + "SELECT * FROM approval_queue WHERE id = ? AND type = 'tweet'", + (request_id,), + ).fetchone() + + if not row: + return {"success": False, "error": f"Approval #{request_id} not found"} + + if row["status"] != "approved": + return {"success": False, "error": f"Approval #{request_id} status is {row['status']}, not approved"} + + content = row["content"] + ctx = json.loads(row["context"]) if row["context"] else {} + + # Parse thread (tweets separated by ---) + tweets = [t.strip() for t in content.split("\n---\n") if t.strip()] + + # Extract reply_to tweet ID from URL if present + reply_to_id = None + reply_to_url = ctx.get("reply_to_url", "") + if reply_to_url: + import re + match = re.search(r"/status/(\d+)", reply_to_url) + if match: + reply_to_id = match.group(1) + + # Post + if len(tweets) == 1: + result = await post_tweet(tweets[0], reply_to_id=reply_to_id) + results = [result] + else: + # For threads, first tweet may be a reply + results = [] + first = await post_tweet(tweets[0], reply_to_id=reply_to_id) + results.append(first) + if first["success"] and len(tweets) > 1: + thread_results = await post_thread(tweets[1:]) + # Fix: thread_results already posted independently, need to chain + # Actually post_thread handles chaining. Let me re-do this. + pass + # Simpler: use post_thread for everything if it's a multi-tweet + if len(tweets) > 1: + results = await post_thread(tweets) + + # Record result + success = all(r["success"] for r in results) + if success: + tweet_urls = [r.get("tweet_url", "") for r in results if r.get("tweet_url")] + published_url = tweet_urls[0] if tweet_urls else "" + + conn.execute( + """UPDATE approval_queue + SET context = json_set(COALESCE(context, '{}'), + '$.published_url', ?, + '$.published_at', datetime('now'), + '$.tweet_ids', ?) + WHERE id = ?""", + (published_url, json.dumps([r.get("tweet_id") for r in results]), request_id), + ) + conn.commit() + logger.info("Tweet #%d published: %s", request_id, published_url) + else: + errors = [r.get("error", "unknown") for r in results if not r["success"]] + conn.execute( + """UPDATE approval_queue + SET context = json_set(COALESCE(context, '{}'), + '$.post_error', ?, + '$.post_attempted_at', datetime('now')) + WHERE id = ?""", + ("; ".join(errors), request_id), + ) + conn.commit() + logger.error("Tweet #%d post failed: %s", request_id, errors) + + return results[0] if len(results) == 1 else {"success": success, "results": results} diff --git a/teleo-pipeline.py b/teleo-pipeline.py index 82f0e5a..ba0080c 100644 --- a/teleo-pipeline.py +++ b/teleo-pipeline.py @@ -26,6 +26,7 @@ from lib.llm import kill_active_subprocesses from lib.merge import merge_cycle from lib.analytics import record_snapshot from lib.entity_batch import entity_batch_cycle +from lib.extract import extract_cycle as source_extract_cycle from lib.validate import validate_cycle from lib.watchdog import watchdog_cycle @@ -67,8 +68,17 @@ async def stage_loop(name: str, interval: int, func, conn, breaker: CircuitBreak async def ingest_cycle(conn, max_workers=None): - """Stage 1: Process entity queue + scan inbox. Entity batch replaces stub.""" - return await entity_batch_cycle(conn, max_workers=max_workers) + """Stage 1: Entity batch + source extraction.""" + # Entity batch first (fast, local-only operations) + eb_ok, eb_err = await entity_batch_cycle(conn, max_workers=max_workers) + # Source extraction (slower, LLM calls) + try: + ex_ok, ex_err = await source_extract_cycle(conn, max_workers=max_workers) + except Exception: + import logging + logging.getLogger("pipeline").exception("Extract cycle failed (non-fatal)") + ex_ok, ex_err = 0, 0 + return eb_ok + ex_ok, eb_err + ex_err async def fix_cycle(conn, max_workers=None):