"""Agent Vitality Diagnostics — data collection and schema. Records daily vitality snapshots per agent across 10 dimensions. Designed as the objective function for agent "aliveness" ranking. Owner: Ship (data collection) + Argus (storage, API, dashboard) Data sources: pipeline.db (read-only), claim-index API, agent-state filesystem, review_records Dimension keys (agreed with Leo 2026-04-08): knowledge_output, knowledge_quality, contributor_engagement, review_performance, spend_efficiency, autonomy, infrastructure_health, social_reach, capital, external_impact """ import json import logging import os import sqlite3 import urllib.request from datetime import datetime, timezone from pathlib import Path logger = logging.getLogger("vitality") # Known domain agents and their primary domains AGENT_DOMAINS = { "rio": ["internet-finance"], "theseus": ["collective-intelligence", "living-agents"], "astra": ["space-development", "energy", "manufacturing", "robotics"], "vida": ["health"], "clay": ["entertainment", "cultural-dynamics"], "leo": ["grand-strategy", "teleohumanity"], "hermes": [], # communications, no domain "rhea": [], # infrastructure ops, no domain "ganymede": [], # code review, no domain "epimetheus": [], # pipeline, no domain "oberon": [], # dashboard, no domain "argus": [], # diagnostics, no domain "ship": [], # engineering, no domain } # Agent file path prefixes — for matching claims by location, not just domain field. # Handles claims in core/ and foundations/ that may not have a standard domain field # in the claim-index (domain derived from directory path). AGENT_PATHS = { "rio": ["domains/internet-finance/"], "theseus": ["domains/ai-alignment/", "core/living-agents/", "core/collective-intelligence/", "foundations/collective-intelligence/"], "astra": ["domains/space-development/", "domains/energy/", "domains/manufacturing/", "domains/robotics/"], "vida": ["domains/health/"], "clay": ["domains/entertainment/", "foundations/cultural-dynamics/"], "leo": ["core/grand-strategy/", "core/teleohumanity/", "core/mechanisms/", "core/living-capital/", "foundations/teleological-economics/", "foundations/critical-systems/"], } ALL_AGENTS = list(AGENT_DOMAINS.keys()) # Agent-state directory (VPS filesystem) AGENT_STATE_DIR = Path(os.environ.get( "AGENT_STATE_DIR", "/opt/teleo-eval/agent-state" )) MIGRATION_SQL = """ CREATE TABLE IF NOT EXISTS vitality_snapshots ( id INTEGER PRIMARY KEY AUTOINCREMENT, agent_name TEXT NOT NULL, dimension TEXT NOT NULL, metric TEXT NOT NULL, value REAL NOT NULL DEFAULT 0, unit TEXT NOT NULL DEFAULT '', source TEXT, recorded_at TEXT NOT NULL DEFAULT (datetime('now')), UNIQUE(agent_name, dimension, metric, recorded_at) ); CREATE INDEX IF NOT EXISTS idx_vitality_agent_time ON vitality_snapshots(agent_name, recorded_at); CREATE INDEX IF NOT EXISTS idx_vitality_dimension ON vitality_snapshots(dimension, recorded_at); """ # Add source column if missing (idempotent upgrade from v1 schema) UPGRADE_SQL = """ ALTER TABLE vitality_snapshots ADD COLUMN source TEXT; """ def ensure_schema(db_path: str): """Create vitality_snapshots table if it doesn't exist.""" conn = sqlite3.connect(db_path, timeout=30) try: conn.executescript(MIGRATION_SQL) try: conn.execute(UPGRADE_SQL) except sqlite3.OperationalError: pass # column already exists conn.commit() logger.info("vitality_snapshots schema ensured") finally: conn.close() def _fetch_claim_index(url: str = "http://localhost:8080/claim-index") -> dict | None: """Fetch claim-index from pipeline health API.""" try: req = urllib.request.Request(url, headers={"Accept": "application/json"}) with urllib.request.urlopen(req, timeout=10) as resp: return json.loads(resp.read()) except Exception as e: logger.warning("claim-index fetch failed: %s", e) return None def _ro_conn(db_path: str) -> sqlite3.Connection: conn = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True, timeout=30) conn.row_factory = sqlite3.Row return conn # --------------------------------------------------------------------------- # Dimension 1: knowledge_output — "How much has this agent produced?" # --------------------------------------------------------------------------- def collect_knowledge_output(conn: sqlite3.Connection, agent: str) -> list[dict]: """Claims merged, domain count, PRs submitted.""" metrics = [] row = conn.execute( "SELECT COUNT(*) as cnt FROM prs WHERE agent = ? AND status = 'merged'", (agent,), ).fetchone() metrics.append({"metric": "claims_merged", "value": row["cnt"], "unit": "claims"}) row = conn.execute( "SELECT COUNT(DISTINCT domain) as cnt FROM prs " "WHERE agent = ? AND domain IS NOT NULL AND status = 'merged'", (agent,), ).fetchone() metrics.append({"metric": "domains_contributed", "value": row["cnt"], "unit": "domains"}) row = conn.execute( "SELECT COUNT(*) as cnt FROM prs WHERE agent = ? AND created_at > datetime('now', '-7 days')", (agent,), ).fetchone() metrics.append({"metric": "prs_7d", "value": row["cnt"], "unit": "PRs"}) return metrics # --------------------------------------------------------------------------- # Dimension 2: knowledge_quality — "How good is the output?" # --------------------------------------------------------------------------- def collect_knowledge_quality( conn: sqlite3.Connection, claim_index: dict | None, agent: str ) -> list[dict]: """Evidence density, challenge rate, cross-domain links, domain coverage.""" metrics = [] agent_domains = AGENT_DOMAINS.get(agent, []) # Challenge rate = challenge PRs / total PRs rows = conn.execute( "SELECT commit_type, COUNT(*) as cnt FROM prs " "WHERE agent = ? AND commit_type IS NOT NULL GROUP BY commit_type", (agent,), ).fetchall() total = sum(r["cnt"] for r in rows) type_counts = {r["commit_type"]: r["cnt"] for r in rows} challenge_rate = type_counts.get("challenge", 0) / total if total > 0 else 0 metrics.append({"metric": "challenge_rate", "value": round(challenge_rate, 4), "unit": "ratio"}) # Activity breadth (distinct commit types) metrics.append({"metric": "activity_breadth", "value": len(type_counts), "unit": "types"}) # Evidence density + cross-domain links from claim-index # Match by domain field OR file path prefix (catches core/, foundations/ claims) agent_paths = AGENT_PATHS.get(agent, []) if claim_index and (agent_domains or agent_paths): claims = claim_index.get("claims", []) agent_claims = [ c for c in claims if c.get("domain") in agent_domains or any(c.get("file", "").startswith(p) for p in agent_paths) ] total_claims = len(agent_claims) # Evidence density: claims with incoming links / total claims linked = sum(1 for c in agent_claims if c.get("incoming_count", 0) > 0) density = linked / total_claims if total_claims > 0 else 0 metrics.append({"metric": "evidence_density", "value": round(density, 4), "unit": "ratio"}) # Cross-domain links cross_domain = sum( 1 for c in agent_claims for link in c.get("outgoing_links", []) if any(d in link for d in claim_index.get("domains", {}).keys() if d not in agent_domains) ) metrics.append({"metric": "cross_domain_links", "value": cross_domain, "unit": "links"}) # Domain coverage: agent's claims / average domain size domains_data = claim_index.get("domains", {}) agent_claim_count = sum(domains_data.get(d, 0) for d in agent_domains) avg_domain_size = (sum(domains_data.values()) / len(domains_data)) if domains_data else 1 coverage = min(agent_claim_count / avg_domain_size, 1.0) if avg_domain_size > 0 else 0 metrics.append({"metric": "domain_coverage", "value": round(coverage, 4), "unit": "ratio"}) else: metrics.append({"metric": "evidence_density", "value": 0, "unit": "ratio"}) metrics.append({"metric": "cross_domain_links", "value": 0, "unit": "links"}) metrics.append({"metric": "domain_coverage", "value": 0, "unit": "ratio"}) return metrics # --------------------------------------------------------------------------- # Dimension 3: contributor_engagement — "Who contributes to this agent's domain?" # --------------------------------------------------------------------------- def collect_contributor_engagement(conn: sqlite3.Connection, agent: str) -> list[dict]: """Unique submitters to this agent's domain.""" row = conn.execute( "SELECT COUNT(DISTINCT submitted_by) as cnt FROM prs " "WHERE agent = ? AND submitted_by IS NOT NULL AND submitted_by != ''", (agent,), ).fetchone() return [ {"metric": "unique_submitters", "value": row["cnt"], "unit": "contributors"}, ] # --------------------------------------------------------------------------- # Dimension 4: review_performance — "How good is the evaluator feedback loop?" # --------------------------------------------------------------------------- def collect_review_performance(conn: sqlite3.Connection, agent: str) -> list[dict]: """Approval rate, rejection reasons from review_records.""" metrics = [] # Check if review_records table exists table_check = conn.execute( "SELECT name FROM sqlite_master WHERE type='table' AND name='review_records'" ).fetchone() if not table_check: return [ {"metric": "approval_rate", "value": 0, "unit": "ratio"}, {"metric": "total_reviews", "value": 0, "unit": "reviews"}, ] # Overall approval rate for this agent's claims (join through prs table) row = conn.execute( "SELECT COUNT(*) as total, " "SUM(CASE WHEN r.outcome = 'approved' THEN 1 ELSE 0 END) as approved, " "SUM(CASE WHEN r.outcome = 'approved-with-changes' THEN 1 ELSE 0 END) as with_changes, " "SUM(CASE WHEN r.outcome = 'rejected' THEN 1 ELSE 0 END) as rejected " "FROM review_records r " "JOIN prs p ON r.pr_number = p.pr_number " "WHERE LOWER(p.agent) = LOWER(?)", (agent,), ).fetchone() total = row["total"] or 0 approved = (row["approved"] or 0) + (row["with_changes"] or 0) rejected = row["rejected"] or 0 approval_rate = approved / total if total > 0 else 0 metrics.append({"metric": "total_reviews", "value": total, "unit": "reviews"}) metrics.append({"metric": "approval_rate", "value": round(approval_rate, 4), "unit": "ratio"}) metrics.append({"metric": "approved", "value": row["approved"] or 0, "unit": "reviews"}) metrics.append({"metric": "approved_with_changes", "value": row["with_changes"] or 0, "unit": "reviews"}) metrics.append({"metric": "rejected", "value": rejected, "unit": "reviews"}) # Top rejection reasons (last 30 days) reasons = conn.execute( "SELECT r.rejection_reason, COUNT(*) as cnt FROM review_records r " "JOIN prs p ON r.pr_number = p.pr_number " "WHERE LOWER(p.agent) = LOWER(?) AND r.outcome = 'rejected' " "AND r.rejection_reason IS NOT NULL " "AND r.review_date > datetime('now', '-30 days') " "GROUP BY r.rejection_reason ORDER BY cnt DESC", (agent,), ).fetchall() for r in reasons: metrics.append({ "metric": f"rejection_{r['rejection_reason']}", "value": r["cnt"], "unit": "rejections", }) return metrics # --------------------------------------------------------------------------- # Dimension 5: spend_efficiency — "What does it cost per merged claim?" # --------------------------------------------------------------------------- def collect_spend_efficiency(conn: sqlite3.Connection, agent: str) -> list[dict]: """Cost per merged claim, total spend, response costs.""" metrics = [] # Pipeline cost attributed to this agent (from prs.cost_usd) row = conn.execute( "SELECT COALESCE(SUM(cost_usd), 0) as cost, COUNT(*) as merged " "FROM prs WHERE agent = ? AND status = 'merged'", (agent,), ).fetchone() total_cost = row["cost"] or 0 merged = row["merged"] or 0 cost_per_claim = total_cost / merged if merged > 0 else 0 metrics.append({"metric": "total_pipeline_cost", "value": round(total_cost, 4), "unit": "USD"}) metrics.append({"metric": "cost_per_merged_claim", "value": round(cost_per_claim, 4), "unit": "USD"}) # Response audit costs (Telegram bot) — per-agent row = conn.execute( "SELECT COALESCE(SUM(generation_cost), 0) as cost, COUNT(*) as cnt " "FROM response_audit WHERE agent = ?", (agent,), ).fetchone() metrics.append({"metric": "response_cost_total", "value": round(row["cost"], 4), "unit": "USD"}) metrics.append({"metric": "total_responses", "value": row["cnt"], "unit": "responses"}) # 24h spend snapshot row = conn.execute( "SELECT COALESCE(SUM(generation_cost), 0) as cost " "FROM response_audit WHERE agent = ? AND timestamp > datetime('now', '-24 hours')", (agent,), ).fetchone() metrics.append({"metric": "response_cost_24h", "value": round(row["cost"], 4), "unit": "USD"}) return metrics # --------------------------------------------------------------------------- # Dimension 6: autonomy — "How independently does this agent act?" # --------------------------------------------------------------------------- def collect_autonomy(conn: sqlite3.Connection, agent: str) -> list[dict]: """Self-directed actions, active days.""" metrics = [] # Autonomous responses in last 24h row = conn.execute( "SELECT COUNT(*) as cnt FROM response_audit " "WHERE agent = ? AND timestamp > datetime('now', '-24 hours')", (agent,), ).fetchone() metrics.append({"metric": "autonomous_responses_24h", "value": row["cnt"], "unit": "actions"}) # Active days in last 7 row = conn.execute( "SELECT COUNT(DISTINCT date(created_at)) as days FROM prs " "WHERE agent = ? AND created_at > datetime('now', '-7 days')", (agent,), ).fetchone() metrics.append({"metric": "active_days_7d", "value": row["days"], "unit": "days"}) return metrics # --------------------------------------------------------------------------- # Dimension 7: infrastructure_health — "Is the agent's machinery working?" # --------------------------------------------------------------------------- def collect_infrastructure_health(conn: sqlite3.Connection, agent: str) -> list[dict]: """Circuit breakers, PR success rate, agent-state liveness.""" metrics = [] # Circuit breakers rows = conn.execute( "SELECT name, state FROM circuit_breakers WHERE name LIKE ?", (f"%{agent}%",), ).fetchall() open_breakers = sum(1 for r in rows if r["state"] != "closed") metrics.append({"metric": "open_circuit_breakers", "value": open_breakers, "unit": "breakers"}) # PR success rate last 7 days row = conn.execute( "SELECT COUNT(*) as total, " "SUM(CASE WHEN status='merged' THEN 1 ELSE 0 END) as merged " "FROM prs WHERE agent = ? AND created_at > datetime('now', '-7 days')", (agent,), ).fetchone() total = row["total"] rate = row["merged"] / total if total > 0 else 0 metrics.append({"metric": "merge_rate_7d", "value": round(rate, 4), "unit": "ratio"}) # Agent-state liveness (read metrics.json from filesystem) state_file = AGENT_STATE_DIR / agent / "metrics.json" if state_file.exists(): try: with open(state_file) as f: state = json.load(f) lifetime = state.get("lifetime", {}) metrics.append({ "metric": "sessions_total", "value": lifetime.get("sessions_total", 0), "unit": "sessions", }) metrics.append({ "metric": "sessions_timeout", "value": lifetime.get("sessions_timeout", 0), "unit": "sessions", }) metrics.append({ "metric": "sessions_error", "value": lifetime.get("sessions_error", 0), "unit": "sessions", }) except (json.JSONDecodeError, OSError) as e: logger.warning("Failed to read agent-state for %s: %s", agent, e) return metrics # --------------------------------------------------------------------------- # Dimensions 8-10: Stubs (no data sources yet) # --------------------------------------------------------------------------- def collect_social_reach(agent: str) -> list[dict]: """Social dimension: stub zeros until X API accounts are active.""" return [ {"metric": "followers", "value": 0, "unit": "followers"}, {"metric": "impressions_7d", "value": 0, "unit": "impressions"}, {"metric": "engagement_rate", "value": 0, "unit": "ratio"}, ] def collect_capital(agent: str) -> list[dict]: """Capital dimension: stub zeros until treasury/revenue tracking exists.""" return [ {"metric": "aum", "value": 0, "unit": "USD"}, {"metric": "treasury", "value": 0, "unit": "USD"}, ] def collect_external_impact(agent: str) -> list[dict]: """External impact dimension: stub zeros until manual tracking exists.""" return [ {"metric": "decisions_informed", "value": 0, "unit": "decisions"}, {"metric": "deals_sourced", "value": 0, "unit": "deals"}, ] # --------------------------------------------------------------------------- # Orchestration # --------------------------------------------------------------------------- DIMENSION_MAP = { "knowledge_output": lambda conn, ci, agent: collect_knowledge_output(conn, agent), "knowledge_quality": collect_knowledge_quality, "contributor_engagement": lambda conn, ci, agent: collect_contributor_engagement(conn, agent), "review_performance": lambda conn, ci, agent: collect_review_performance(conn, agent), "spend_efficiency": lambda conn, ci, agent: collect_spend_efficiency(conn, agent), "autonomy": lambda conn, ci, agent: collect_autonomy(conn, agent), "infrastructure_health": lambda conn, ci, agent: collect_infrastructure_health(conn, agent), "social_reach": lambda conn, ci, agent: collect_social_reach(agent), "capital": lambda conn, ci, agent: collect_capital(agent), "external_impact": lambda conn, ci, agent: collect_external_impact(agent), } def collect_all_for_agent( db_path: str, agent: str, claim_index_url: str = "http://localhost:8080/claim-index", ) -> dict: """Collect all 10 vitality dimensions for a single agent. Returns {dimension: [metrics]}. """ claim_index = _fetch_claim_index(claim_index_url) conn = _ro_conn(db_path) try: result = {} for dim_key, collector in DIMENSION_MAP.items(): try: result[dim_key] = collector(conn, claim_index, agent) except Exception as e: logger.error("collector %s failed for %s: %s", dim_key, agent, e) result[dim_key] = [] return result finally: conn.close() def collect_system_aggregate( db_path: str, claim_index_url: str = "http://localhost:8080/claim-index", ) -> dict: """System-level aggregate vitality metrics.""" claim_index = _fetch_claim_index(claim_index_url) conn = _ro_conn(db_path) try: metrics = {} # Knowledge totals total_claims = claim_index["total_claims"] if claim_index else 0 orphan_ratio = claim_index.get("orphan_ratio", 0) if claim_index else 0 domain_count = len(claim_index.get("domains", {})) if claim_index else 0 metrics["knowledge_output"] = [ {"metric": "total_claims", "value": total_claims, "unit": "claims"}, {"metric": "total_domains", "value": domain_count, "unit": "domains"}, {"metric": "orphan_ratio", "value": round(orphan_ratio, 4), "unit": "ratio"}, ] # Cross-domain citation rate if claim_index: claims = claim_index.get("claims", []) total_links = sum(c.get("outgoing_count", 0) for c in claims) cross_domain = 0 for c in claims: src_domain = c.get("domain") for link in c.get("outgoing_links", []): linked_claims = [ x for x in claims if x.get("stem") in link or x.get("file", "").endswith(link + ".md") ] for lc in linked_claims: if lc.get("domain") != src_domain: cross_domain += 1 metrics["knowledge_quality"] = [ {"metric": "cross_domain_citation_rate", "value": round(cross_domain / max(total_links, 1), 4), "unit": "ratio"}, ] # Pipeline throughput row = conn.execute( "SELECT COUNT(*) as merged FROM prs " "WHERE status='merged' AND merged_at > datetime('now', '-24 hours')" ).fetchone() row2 = conn.execute("SELECT COUNT(*) as total FROM sources").fetchone() row3 = conn.execute( "SELECT COUNT(*) as pending FROM prs " "WHERE status NOT IN ('merged','rejected','closed')" ).fetchone() metrics["infrastructure_health"] = [ {"metric": "prs_merged_24h", "value": row["merged"], "unit": "PRs/day"}, {"metric": "total_sources", "value": row2["total"], "unit": "sources"}, {"metric": "queue_depth", "value": row3["pending"], "unit": "PRs"}, ] # Total spend row = conn.execute( "SELECT COALESCE(SUM(cost_usd), 0) as cost " "FROM costs WHERE date > date('now', '-1 day')" ).fetchone() row2 = conn.execute( "SELECT COALESCE(SUM(generation_cost), 0) as cost FROM response_audit " "WHERE timestamp > datetime('now', '-24 hours')" ).fetchone() metrics["spend_efficiency"] = [ {"metric": "pipeline_cost_24h", "value": round(row["cost"], 4), "unit": "USD"}, {"metric": "response_cost_24h", "value": round(row2["cost"], 4), "unit": "USD"}, {"metric": "total_cost_24h", "value": round(row["cost"] + row2["cost"], 4), "unit": "USD"}, ] # Stubs metrics["social_reach"] = [{"metric": "total_followers", "value": 0, "unit": "followers"}] metrics["capital"] = [{"metric": "total_aum", "value": 0, "unit": "USD"}] return metrics finally: conn.close() def record_snapshot( db_path: str, claim_index_url: str = "http://localhost:8080/claim-index", ): """Run a full vitality snapshot — one row per agent per dimension per metric.""" now = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") rows = [] # Per-agent snapshots for agent in ALL_AGENTS: try: dimensions = collect_all_for_agent(db_path, agent, claim_index_url) for dim_name, metrics in dimensions.items(): collector_name = f"{dim_name}_collector" for m in metrics: rows.append(( agent, dim_name, m["metric"], m["value"], m["unit"], collector_name, now, )) except Exception as e: logger.error("vitality collection failed for %s: %s", agent, e) # System aggregate try: system = collect_system_aggregate(db_path, claim_index_url) for dim_name, metrics in system.items(): for m in metrics: rows.append(( "_system", dim_name, m["metric"], m["value"], m["unit"], "system_aggregate", now, )) except Exception as e: logger.error("vitality system aggregate failed: %s", e) # Write all rows ensure_schema(db_path) conn = sqlite3.connect(db_path, timeout=30) try: conn.executemany( "INSERT OR REPLACE INTO vitality_snapshots " "(agent_name, dimension, metric, value, unit, source, recorded_at) " "VALUES (?, ?, ?, ?, ?, ?, ?)", rows, ) conn.commit() logger.info( "vitality snapshot recorded: %d rows for %d agents + system", len(rows), len(ALL_AGENTS), ) return {"rows_written": len(rows), "agents": len(ALL_AGENTS), "recorded_at": now} finally: conn.close() if __name__ == "__main__": """CLI: python3 vitality.py [db_path] — runs a snapshot.""" import sys logging.basicConfig(level=logging.INFO) db = sys.argv[1] if len(sys.argv) > 1 else "/opt/teleo-eval/pipeline/pipeline.db" result = record_snapshot(db) print(json.dumps(result, indent=2))