Move vitality.py/vitality_routes.py from root diagnostics/ to ops/diagnostics/ (canonical location). Overwrite ops/diagnostics/alerting.py and alerting_routes.py with root versions (newer: SQL injection protection via _ALLOWED_DIM_EXPRS, proper error handling + conn.close). Remove root diagnostics/*.py — all code now in ops/diagnostics/. Include diff log documenting resolution of each multi-copy file. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
629 lines
25 KiB
Python
629 lines
25 KiB
Python
"""Agent Vitality Diagnostics — data collection and schema.
|
|
|
|
Records daily vitality snapshots per agent across 10 dimensions.
|
|
Designed as the objective function for agent "aliveness" ranking.
|
|
|
|
Owner: Ship (data collection) + Argus (storage, API, dashboard)
|
|
Data sources: pipeline.db (read-only), claim-index API, agent-state filesystem, review_records
|
|
|
|
Dimension keys (agreed with Leo 2026-04-08):
|
|
knowledge_output, knowledge_quality, contributor_engagement,
|
|
review_performance, spend_efficiency, autonomy,
|
|
infrastructure_health, social_reach, capital, external_impact
|
|
"""
|
|
|
|
import json
|
|
import logging
|
|
import os
|
|
import sqlite3
|
|
import urllib.request
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
|
|
logger = logging.getLogger("vitality")
|
|
|
|
# Known domain agents and their primary domains
|
|
AGENT_DOMAINS = {
|
|
"rio": ["internet-finance"],
|
|
"theseus": ["collective-intelligence", "living-agents"],
|
|
"astra": ["space-development", "energy", "manufacturing", "robotics"],
|
|
"vida": ["health"],
|
|
"clay": ["entertainment", "cultural-dynamics"],
|
|
"leo": ["grand-strategy", "teleohumanity"],
|
|
"hermes": [], # communications, no domain
|
|
"rhea": [], # infrastructure ops, no domain
|
|
"ganymede": [], # code review, no domain
|
|
"epimetheus": [], # pipeline, no domain
|
|
"oberon": [], # dashboard, no domain
|
|
"argus": [], # diagnostics, no domain
|
|
"ship": [], # engineering, no domain
|
|
}
|
|
|
|
# Agent file path prefixes — for matching claims by location, not just domain field.
|
|
# Handles claims in core/ and foundations/ that may not have a standard domain field
|
|
# in the claim-index (domain derived from directory path).
|
|
AGENT_PATHS = {
|
|
"rio": ["domains/internet-finance/"],
|
|
"theseus": ["domains/ai-alignment/", "core/living-agents/", "core/collective-intelligence/",
|
|
"foundations/collective-intelligence/"],
|
|
"astra": ["domains/space-development/", "domains/energy/",
|
|
"domains/manufacturing/", "domains/robotics/"],
|
|
"vida": ["domains/health/"],
|
|
"clay": ["domains/entertainment/", "foundations/cultural-dynamics/"],
|
|
"leo": ["core/grand-strategy/", "core/teleohumanity/", "core/mechanisms/",
|
|
"core/living-capital/", "foundations/teleological-economics/",
|
|
"foundations/critical-systems/"],
|
|
}
|
|
|
|
ALL_AGENTS = list(AGENT_DOMAINS.keys())
|
|
|
|
# Agent-state directory (VPS filesystem)
|
|
AGENT_STATE_DIR = Path(os.environ.get(
|
|
"AGENT_STATE_DIR", "/opt/teleo-eval/agent-state"
|
|
))
|
|
|
|
MIGRATION_SQL = """
|
|
CREATE TABLE IF NOT EXISTS vitality_snapshots (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
agent_name TEXT NOT NULL,
|
|
dimension TEXT NOT NULL,
|
|
metric TEXT NOT NULL,
|
|
value REAL NOT NULL DEFAULT 0,
|
|
unit TEXT NOT NULL DEFAULT '',
|
|
source TEXT,
|
|
recorded_at TEXT NOT NULL DEFAULT (datetime('now')),
|
|
UNIQUE(agent_name, dimension, metric, recorded_at)
|
|
);
|
|
CREATE INDEX IF NOT EXISTS idx_vitality_agent_time
|
|
ON vitality_snapshots(agent_name, recorded_at);
|
|
CREATE INDEX IF NOT EXISTS idx_vitality_dimension
|
|
ON vitality_snapshots(dimension, recorded_at);
|
|
"""
|
|
|
|
# Add source column if missing (idempotent upgrade from v1 schema)
|
|
UPGRADE_SQL = """
|
|
ALTER TABLE vitality_snapshots ADD COLUMN source TEXT;
|
|
"""
|
|
|
|
|
|
def ensure_schema(db_path: str):
|
|
"""Create vitality_snapshots table if it doesn't exist."""
|
|
conn = sqlite3.connect(db_path, timeout=30)
|
|
try:
|
|
conn.executescript(MIGRATION_SQL)
|
|
try:
|
|
conn.execute(UPGRADE_SQL)
|
|
except sqlite3.OperationalError:
|
|
pass # column already exists
|
|
conn.commit()
|
|
logger.info("vitality_snapshots schema ensured")
|
|
finally:
|
|
conn.close()
|
|
|
|
|
|
def _fetch_claim_index(url: str = "http://localhost:8080/claim-index") -> dict | None:
|
|
"""Fetch claim-index from pipeline health API."""
|
|
try:
|
|
req = urllib.request.Request(url, headers={"Accept": "application/json"})
|
|
with urllib.request.urlopen(req, timeout=10) as resp:
|
|
return json.loads(resp.read())
|
|
except Exception as e:
|
|
logger.warning("claim-index fetch failed: %s", e)
|
|
return None
|
|
|
|
|
|
def _ro_conn(db_path: str) -> sqlite3.Connection:
|
|
conn = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True, timeout=30)
|
|
conn.row_factory = sqlite3.Row
|
|
return conn
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Dimension 1: knowledge_output — "How much has this agent produced?"
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def collect_knowledge_output(conn: sqlite3.Connection, agent: str) -> list[dict]:
|
|
"""Claims merged, domain count, PRs submitted."""
|
|
metrics = []
|
|
|
|
row = conn.execute(
|
|
"SELECT COUNT(*) as cnt FROM prs WHERE agent = ? AND status = 'merged'",
|
|
(agent,),
|
|
).fetchone()
|
|
metrics.append({"metric": "claims_merged", "value": row["cnt"], "unit": "claims"})
|
|
|
|
row = conn.execute(
|
|
"SELECT COUNT(DISTINCT domain) as cnt FROM prs "
|
|
"WHERE agent = ? AND domain IS NOT NULL AND status = 'merged'",
|
|
(agent,),
|
|
).fetchone()
|
|
metrics.append({"metric": "domains_contributed", "value": row["cnt"], "unit": "domains"})
|
|
|
|
row = conn.execute(
|
|
"SELECT COUNT(*) as cnt FROM prs WHERE agent = ? AND created_at > datetime('now', '-7 days')",
|
|
(agent,),
|
|
).fetchone()
|
|
metrics.append({"metric": "prs_7d", "value": row["cnt"], "unit": "PRs"})
|
|
|
|
return metrics
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Dimension 2: knowledge_quality — "How good is the output?"
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def collect_knowledge_quality(
|
|
conn: sqlite3.Connection, claim_index: dict | None, agent: str
|
|
) -> list[dict]:
|
|
"""Evidence density, challenge rate, cross-domain links, domain coverage."""
|
|
metrics = []
|
|
agent_domains = AGENT_DOMAINS.get(agent, [])
|
|
|
|
# Challenge rate = challenge PRs / total PRs
|
|
rows = conn.execute(
|
|
"SELECT commit_type, COUNT(*) as cnt FROM prs "
|
|
"WHERE agent = ? AND commit_type IS NOT NULL GROUP BY commit_type",
|
|
(agent,),
|
|
).fetchall()
|
|
total = sum(r["cnt"] for r in rows)
|
|
type_counts = {r["commit_type"]: r["cnt"] for r in rows}
|
|
challenge_rate = type_counts.get("challenge", 0) / total if total > 0 else 0
|
|
metrics.append({"metric": "challenge_rate", "value": round(challenge_rate, 4), "unit": "ratio"})
|
|
|
|
# Activity breadth (distinct commit types)
|
|
metrics.append({"metric": "activity_breadth", "value": len(type_counts), "unit": "types"})
|
|
|
|
# Evidence density + cross-domain links from claim-index
|
|
# Match by domain field OR file path prefix (catches core/, foundations/ claims)
|
|
agent_paths = AGENT_PATHS.get(agent, [])
|
|
if claim_index and (agent_domains or agent_paths):
|
|
claims = claim_index.get("claims", [])
|
|
agent_claims = [
|
|
c for c in claims
|
|
if c.get("domain") in agent_domains
|
|
or any(c.get("file", "").startswith(p) for p in agent_paths)
|
|
]
|
|
total_claims = len(agent_claims)
|
|
|
|
# Evidence density: claims with incoming links / total claims
|
|
linked = sum(1 for c in agent_claims if c.get("incoming_count", 0) > 0)
|
|
density = linked / total_claims if total_claims > 0 else 0
|
|
metrics.append({"metric": "evidence_density", "value": round(density, 4), "unit": "ratio"})
|
|
|
|
# Cross-domain links
|
|
cross_domain = sum(
|
|
1 for c in agent_claims
|
|
for link in c.get("outgoing_links", [])
|
|
if any(d in link for d in claim_index.get("domains", {}).keys()
|
|
if d not in agent_domains)
|
|
)
|
|
metrics.append({"metric": "cross_domain_links", "value": cross_domain, "unit": "links"})
|
|
|
|
# Domain coverage: agent's claims / average domain size
|
|
domains_data = claim_index.get("domains", {})
|
|
agent_claim_count = sum(domains_data.get(d, 0) for d in agent_domains)
|
|
avg_domain_size = (sum(domains_data.values()) / len(domains_data)) if domains_data else 1
|
|
coverage = min(agent_claim_count / avg_domain_size, 1.0) if avg_domain_size > 0 else 0
|
|
metrics.append({"metric": "domain_coverage", "value": round(coverage, 4), "unit": "ratio"})
|
|
else:
|
|
metrics.append({"metric": "evidence_density", "value": 0, "unit": "ratio"})
|
|
metrics.append({"metric": "cross_domain_links", "value": 0, "unit": "links"})
|
|
metrics.append({"metric": "domain_coverage", "value": 0, "unit": "ratio"})
|
|
|
|
return metrics
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Dimension 3: contributor_engagement — "Who contributes to this agent's domain?"
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def collect_contributor_engagement(conn: sqlite3.Connection, agent: str) -> list[dict]:
|
|
"""Unique submitters to this agent's domain."""
|
|
row = conn.execute(
|
|
"SELECT COUNT(DISTINCT submitted_by) as cnt FROM prs "
|
|
"WHERE agent = ? AND submitted_by IS NOT NULL AND submitted_by != ''",
|
|
(agent,),
|
|
).fetchone()
|
|
return [
|
|
{"metric": "unique_submitters", "value": row["cnt"], "unit": "contributors"},
|
|
]
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Dimension 4: review_performance — "How good is the evaluator feedback loop?"
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def collect_review_performance(conn: sqlite3.Connection, agent: str) -> list[dict]:
|
|
"""Approval rate, rejection reasons from review_records."""
|
|
metrics = []
|
|
|
|
# Check if review_records table exists
|
|
table_check = conn.execute(
|
|
"SELECT name FROM sqlite_master WHERE type='table' AND name='review_records'"
|
|
).fetchone()
|
|
if not table_check:
|
|
return [
|
|
{"metric": "approval_rate", "value": 0, "unit": "ratio"},
|
|
{"metric": "total_reviews", "value": 0, "unit": "reviews"},
|
|
]
|
|
|
|
# Overall approval rate for this agent's claims (join through prs table)
|
|
row = conn.execute(
|
|
"SELECT COUNT(*) as total, "
|
|
"SUM(CASE WHEN r.outcome = 'approved' THEN 1 ELSE 0 END) as approved, "
|
|
"SUM(CASE WHEN r.outcome = 'approved-with-changes' THEN 1 ELSE 0 END) as with_changes, "
|
|
"SUM(CASE WHEN r.outcome = 'rejected' THEN 1 ELSE 0 END) as rejected "
|
|
"FROM review_records r "
|
|
"JOIN prs p ON r.pr_number = p.pr_number "
|
|
"WHERE LOWER(p.agent) = LOWER(?)",
|
|
(agent,),
|
|
).fetchone()
|
|
total = row["total"] or 0
|
|
approved = (row["approved"] or 0) + (row["with_changes"] or 0)
|
|
rejected = row["rejected"] or 0
|
|
approval_rate = approved / total if total > 0 else 0
|
|
|
|
metrics.append({"metric": "total_reviews", "value": total, "unit": "reviews"})
|
|
metrics.append({"metric": "approval_rate", "value": round(approval_rate, 4), "unit": "ratio"})
|
|
metrics.append({"metric": "approved", "value": row["approved"] or 0, "unit": "reviews"})
|
|
metrics.append({"metric": "approved_with_changes", "value": row["with_changes"] or 0, "unit": "reviews"})
|
|
metrics.append({"metric": "rejected", "value": rejected, "unit": "reviews"})
|
|
|
|
# Top rejection reasons (last 30 days)
|
|
reasons = conn.execute(
|
|
"SELECT r.rejection_reason, COUNT(*) as cnt FROM review_records r "
|
|
"JOIN prs p ON r.pr_number = p.pr_number "
|
|
"WHERE LOWER(p.agent) = LOWER(?) AND r.outcome = 'rejected' "
|
|
"AND r.rejection_reason IS NOT NULL "
|
|
"AND r.review_date > datetime('now', '-30 days') "
|
|
"GROUP BY r.rejection_reason ORDER BY cnt DESC",
|
|
(agent,),
|
|
).fetchall()
|
|
for r in reasons:
|
|
metrics.append({
|
|
"metric": f"rejection_{r['rejection_reason']}",
|
|
"value": r["cnt"],
|
|
"unit": "rejections",
|
|
})
|
|
|
|
return metrics
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Dimension 5: spend_efficiency — "What does it cost per merged claim?"
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def collect_spend_efficiency(conn: sqlite3.Connection, agent: str) -> list[dict]:
|
|
"""Cost per merged claim, total spend, response costs."""
|
|
metrics = []
|
|
|
|
# Pipeline cost attributed to this agent (from prs.cost_usd)
|
|
row = conn.execute(
|
|
"SELECT COALESCE(SUM(cost_usd), 0) as cost, COUNT(*) as merged "
|
|
"FROM prs WHERE agent = ? AND status = 'merged'",
|
|
(agent,),
|
|
).fetchone()
|
|
total_cost = row["cost"] or 0
|
|
merged = row["merged"] or 0
|
|
cost_per_claim = total_cost / merged if merged > 0 else 0
|
|
|
|
metrics.append({"metric": "total_pipeline_cost", "value": round(total_cost, 4), "unit": "USD"})
|
|
metrics.append({"metric": "cost_per_merged_claim", "value": round(cost_per_claim, 4), "unit": "USD"})
|
|
|
|
# Response audit costs (Telegram bot) — per-agent
|
|
row = conn.execute(
|
|
"SELECT COALESCE(SUM(generation_cost), 0) as cost, COUNT(*) as cnt "
|
|
"FROM response_audit WHERE agent = ?",
|
|
(agent,),
|
|
).fetchone()
|
|
metrics.append({"metric": "response_cost_total", "value": round(row["cost"], 4), "unit": "USD"})
|
|
metrics.append({"metric": "total_responses", "value": row["cnt"], "unit": "responses"})
|
|
|
|
# 24h spend snapshot
|
|
row = conn.execute(
|
|
"SELECT COALESCE(SUM(generation_cost), 0) as cost "
|
|
"FROM response_audit WHERE agent = ? AND timestamp > datetime('now', '-24 hours')",
|
|
(agent,),
|
|
).fetchone()
|
|
metrics.append({"metric": "response_cost_24h", "value": round(row["cost"], 4), "unit": "USD"})
|
|
|
|
return metrics
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Dimension 6: autonomy — "How independently does this agent act?"
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def collect_autonomy(conn: sqlite3.Connection, agent: str) -> list[dict]:
|
|
"""Self-directed actions, active days."""
|
|
metrics = []
|
|
|
|
# Autonomous responses in last 24h
|
|
row = conn.execute(
|
|
"SELECT COUNT(*) as cnt FROM response_audit "
|
|
"WHERE agent = ? AND timestamp > datetime('now', '-24 hours')",
|
|
(agent,),
|
|
).fetchone()
|
|
metrics.append({"metric": "autonomous_responses_24h", "value": row["cnt"], "unit": "actions"})
|
|
|
|
# Active days in last 7
|
|
row = conn.execute(
|
|
"SELECT COUNT(DISTINCT date(created_at)) as days FROM prs "
|
|
"WHERE agent = ? AND created_at > datetime('now', '-7 days')",
|
|
(agent,),
|
|
).fetchone()
|
|
metrics.append({"metric": "active_days_7d", "value": row["days"], "unit": "days"})
|
|
|
|
return metrics
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Dimension 7: infrastructure_health — "Is the agent's machinery working?"
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def collect_infrastructure_health(conn: sqlite3.Connection, agent: str) -> list[dict]:
|
|
"""Circuit breakers, PR success rate, agent-state liveness."""
|
|
metrics = []
|
|
|
|
# Circuit breakers
|
|
rows = conn.execute(
|
|
"SELECT name, state FROM circuit_breakers WHERE name LIKE ?",
|
|
(f"%{agent}%",),
|
|
).fetchall()
|
|
open_breakers = sum(1 for r in rows if r["state"] != "closed")
|
|
metrics.append({"metric": "open_circuit_breakers", "value": open_breakers, "unit": "breakers"})
|
|
|
|
# PR success rate last 7 days
|
|
row = conn.execute(
|
|
"SELECT COUNT(*) as total, "
|
|
"SUM(CASE WHEN status='merged' THEN 1 ELSE 0 END) as merged "
|
|
"FROM prs WHERE agent = ? AND created_at > datetime('now', '-7 days')",
|
|
(agent,),
|
|
).fetchone()
|
|
total = row["total"]
|
|
rate = row["merged"] / total if total > 0 else 0
|
|
metrics.append({"metric": "merge_rate_7d", "value": round(rate, 4), "unit": "ratio"})
|
|
|
|
# Agent-state liveness (read metrics.json from filesystem)
|
|
state_file = AGENT_STATE_DIR / agent / "metrics.json"
|
|
if state_file.exists():
|
|
try:
|
|
with open(state_file) as f:
|
|
state = json.load(f)
|
|
lifetime = state.get("lifetime", {})
|
|
metrics.append({
|
|
"metric": "sessions_total",
|
|
"value": lifetime.get("sessions_total", 0),
|
|
"unit": "sessions",
|
|
})
|
|
metrics.append({
|
|
"metric": "sessions_timeout",
|
|
"value": lifetime.get("sessions_timeout", 0),
|
|
"unit": "sessions",
|
|
})
|
|
metrics.append({
|
|
"metric": "sessions_error",
|
|
"value": lifetime.get("sessions_error", 0),
|
|
"unit": "sessions",
|
|
})
|
|
except (json.JSONDecodeError, OSError) as e:
|
|
logger.warning("Failed to read agent-state for %s: %s", agent, e)
|
|
|
|
return metrics
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Dimensions 8-10: Stubs (no data sources yet)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def collect_social_reach(agent: str) -> list[dict]:
|
|
"""Social dimension: stub zeros until X API accounts are active."""
|
|
return [
|
|
{"metric": "followers", "value": 0, "unit": "followers"},
|
|
{"metric": "impressions_7d", "value": 0, "unit": "impressions"},
|
|
{"metric": "engagement_rate", "value": 0, "unit": "ratio"},
|
|
]
|
|
|
|
|
|
def collect_capital(agent: str) -> list[dict]:
|
|
"""Capital dimension: stub zeros until treasury/revenue tracking exists."""
|
|
return [
|
|
{"metric": "aum", "value": 0, "unit": "USD"},
|
|
{"metric": "treasury", "value": 0, "unit": "USD"},
|
|
]
|
|
|
|
|
|
def collect_external_impact(agent: str) -> list[dict]:
|
|
"""External impact dimension: stub zeros until manual tracking exists."""
|
|
return [
|
|
{"metric": "decisions_informed", "value": 0, "unit": "decisions"},
|
|
{"metric": "deals_sourced", "value": 0, "unit": "deals"},
|
|
]
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Orchestration
|
|
# ---------------------------------------------------------------------------
|
|
|
|
DIMENSION_MAP = {
|
|
"knowledge_output": lambda conn, ci, agent: collect_knowledge_output(conn, agent),
|
|
"knowledge_quality": collect_knowledge_quality,
|
|
"contributor_engagement": lambda conn, ci, agent: collect_contributor_engagement(conn, agent),
|
|
"review_performance": lambda conn, ci, agent: collect_review_performance(conn, agent),
|
|
"spend_efficiency": lambda conn, ci, agent: collect_spend_efficiency(conn, agent),
|
|
"autonomy": lambda conn, ci, agent: collect_autonomy(conn, agent),
|
|
"infrastructure_health": lambda conn, ci, agent: collect_infrastructure_health(conn, agent),
|
|
"social_reach": lambda conn, ci, agent: collect_social_reach(agent),
|
|
"capital": lambda conn, ci, agent: collect_capital(agent),
|
|
"external_impact": lambda conn, ci, agent: collect_external_impact(agent),
|
|
}
|
|
|
|
|
|
def collect_all_for_agent(
|
|
db_path: str,
|
|
agent: str,
|
|
claim_index_url: str = "http://localhost:8080/claim-index",
|
|
) -> dict:
|
|
"""Collect all 10 vitality dimensions for a single agent.
|
|
Returns {dimension: [metrics]}.
|
|
"""
|
|
claim_index = _fetch_claim_index(claim_index_url)
|
|
conn = _ro_conn(db_path)
|
|
try:
|
|
result = {}
|
|
for dim_key, collector in DIMENSION_MAP.items():
|
|
try:
|
|
result[dim_key] = collector(conn, claim_index, agent)
|
|
except Exception as e:
|
|
logger.error("collector %s failed for %s: %s", dim_key, agent, e)
|
|
result[dim_key] = []
|
|
return result
|
|
finally:
|
|
conn.close()
|
|
|
|
|
|
def collect_system_aggregate(
|
|
db_path: str,
|
|
claim_index_url: str = "http://localhost:8080/claim-index",
|
|
) -> dict:
|
|
"""System-level aggregate vitality metrics."""
|
|
claim_index = _fetch_claim_index(claim_index_url)
|
|
conn = _ro_conn(db_path)
|
|
try:
|
|
metrics = {}
|
|
|
|
# Knowledge totals
|
|
total_claims = claim_index["total_claims"] if claim_index else 0
|
|
orphan_ratio = claim_index.get("orphan_ratio", 0) if claim_index else 0
|
|
domain_count = len(claim_index.get("domains", {})) if claim_index else 0
|
|
|
|
metrics["knowledge_output"] = [
|
|
{"metric": "total_claims", "value": total_claims, "unit": "claims"},
|
|
{"metric": "total_domains", "value": domain_count, "unit": "domains"},
|
|
{"metric": "orphan_ratio", "value": round(orphan_ratio, 4), "unit": "ratio"},
|
|
]
|
|
|
|
# Cross-domain citation rate
|
|
if claim_index:
|
|
claims = claim_index.get("claims", [])
|
|
total_links = sum(c.get("outgoing_count", 0) for c in claims)
|
|
cross_domain = 0
|
|
for c in claims:
|
|
src_domain = c.get("domain")
|
|
for link in c.get("outgoing_links", []):
|
|
linked_claims = [
|
|
x for x in claims
|
|
if x.get("stem") in link or x.get("file", "").endswith(link + ".md")
|
|
]
|
|
for lc in linked_claims:
|
|
if lc.get("domain") != src_domain:
|
|
cross_domain += 1
|
|
metrics["knowledge_quality"] = [
|
|
{"metric": "cross_domain_citation_rate",
|
|
"value": round(cross_domain / max(total_links, 1), 4),
|
|
"unit": "ratio"},
|
|
]
|
|
|
|
# Pipeline throughput
|
|
row = conn.execute(
|
|
"SELECT COUNT(*) as merged FROM prs "
|
|
"WHERE status='merged' AND merged_at > datetime('now', '-24 hours')"
|
|
).fetchone()
|
|
row2 = conn.execute("SELECT COUNT(*) as total FROM sources").fetchone()
|
|
row3 = conn.execute(
|
|
"SELECT COUNT(*) as pending FROM prs "
|
|
"WHERE status NOT IN ('merged','rejected','closed')"
|
|
).fetchone()
|
|
|
|
metrics["infrastructure_health"] = [
|
|
{"metric": "prs_merged_24h", "value": row["merged"], "unit": "PRs/day"},
|
|
{"metric": "total_sources", "value": row2["total"], "unit": "sources"},
|
|
{"metric": "queue_depth", "value": row3["pending"], "unit": "PRs"},
|
|
]
|
|
|
|
# Total spend
|
|
row = conn.execute(
|
|
"SELECT COALESCE(SUM(cost_usd), 0) as cost "
|
|
"FROM costs WHERE date > date('now', '-1 day')"
|
|
).fetchone()
|
|
row2 = conn.execute(
|
|
"SELECT COALESCE(SUM(generation_cost), 0) as cost FROM response_audit "
|
|
"WHERE timestamp > datetime('now', '-24 hours')"
|
|
).fetchone()
|
|
metrics["spend_efficiency"] = [
|
|
{"metric": "pipeline_cost_24h", "value": round(row["cost"], 4), "unit": "USD"},
|
|
{"metric": "response_cost_24h", "value": round(row2["cost"], 4), "unit": "USD"},
|
|
{"metric": "total_cost_24h",
|
|
"value": round(row["cost"] + row2["cost"], 4), "unit": "USD"},
|
|
]
|
|
|
|
# Stubs
|
|
metrics["social_reach"] = [{"metric": "total_followers", "value": 0, "unit": "followers"}]
|
|
metrics["capital"] = [{"metric": "total_aum", "value": 0, "unit": "USD"}]
|
|
|
|
return metrics
|
|
finally:
|
|
conn.close()
|
|
|
|
|
|
def record_snapshot(
|
|
db_path: str,
|
|
claim_index_url: str = "http://localhost:8080/claim-index",
|
|
):
|
|
"""Run a full vitality snapshot — one row per agent per dimension per metric."""
|
|
now = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
|
rows = []
|
|
|
|
# Per-agent snapshots
|
|
for agent in ALL_AGENTS:
|
|
try:
|
|
dimensions = collect_all_for_agent(db_path, agent, claim_index_url)
|
|
for dim_name, metrics in dimensions.items():
|
|
collector_name = f"{dim_name}_collector"
|
|
for m in metrics:
|
|
rows.append((
|
|
agent, dim_name, m["metric"], m["value"],
|
|
m["unit"], collector_name, now,
|
|
))
|
|
except Exception as e:
|
|
logger.error("vitality collection failed for %s: %s", agent, e)
|
|
|
|
# System aggregate
|
|
try:
|
|
system = collect_system_aggregate(db_path, claim_index_url)
|
|
for dim_name, metrics in system.items():
|
|
for m in metrics:
|
|
rows.append((
|
|
"_system", dim_name, m["metric"], m["value"],
|
|
m["unit"], "system_aggregate", now,
|
|
))
|
|
except Exception as e:
|
|
logger.error("vitality system aggregate failed: %s", e)
|
|
|
|
# Write all rows
|
|
ensure_schema(db_path)
|
|
conn = sqlite3.connect(db_path, timeout=30)
|
|
try:
|
|
conn.executemany(
|
|
"INSERT OR REPLACE INTO vitality_snapshots "
|
|
"(agent_name, dimension, metric, value, unit, source, recorded_at) "
|
|
"VALUES (?, ?, ?, ?, ?, ?, ?)",
|
|
rows,
|
|
)
|
|
conn.commit()
|
|
logger.info(
|
|
"vitality snapshot recorded: %d rows for %d agents + system",
|
|
len(rows), len(ALL_AGENTS),
|
|
)
|
|
return {"rows_written": len(rows), "agents": len(ALL_AGENTS), "recorded_at": now}
|
|
finally:
|
|
conn.close()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
"""CLI: python3 vitality.py [db_path] — runs a snapshot."""
|
|
import sys
|
|
logging.basicConfig(level=logging.INFO)
|
|
db = sys.argv[1] if len(sys.argv) > 1 else "/opt/teleo-eval/pipeline/pipeline.db"
|
|
result = record_snapshot(db)
|
|
print(json.dumps(result, indent=2))
|