diff --git a/deploy/auto-deploy.sh b/deploy/auto-deploy.sh index e9002e7..30a5092 100755 --- a/deploy/auto-deploy.sh +++ b/deploy/auto-deploy.sh @@ -18,6 +18,7 @@ DIAGNOSTICS_DIR="/opt/teleo-eval/diagnostics" AGENT_STATE_DIR="/opt/teleo-eval/ops/agent-state" LEOCLEAN_BIN_DIR="/home/teleo/.hermes/profiles/leoclean/bin" LEOCLEAN_SKILLS_DIR="/home/teleo/.hermes/profiles/leoclean/skills" +LEOCLEAN_PLUGINS_DIR="/home/teleo/.hermes/profiles/leoclean/plugins" SYSTEMD_DIR="/etc/systemd/system" STAMP_FILE="/opt/teleo-eval/.last-deploy-sha" LOG_TAG="auto-deploy" @@ -68,7 +69,7 @@ fi # Syntax check all Python files before copying ERRORS=0 -for f in lib/*.py *.py diagnostics/*.py telegram/*.py tests/*.py; do +for f in lib/*.py *.py diagnostics/*.py telegram/*.py tests/*.py hermes-agent/leoclean-plugins/vps/*/*.py; do [ -f "$f" ] || continue if ! python3 -c "import ast, sys; ast.parse(open(sys.argv[1]).read())" "$f" 2>&1; then log "SYNTAX ERROR: $f" @@ -103,12 +104,16 @@ if [ -d hermes-agent/leoclean-skills/vps ]; then mkdir -p "$LEOCLEAN_SKILLS_DIR" rsync "${RSYNC_OPTS[@]}" hermes-agent/leoclean-skills/vps/ "$LEOCLEAN_SKILLS_DIR/" fi +if [ -d hermes-agent/leoclean-plugins/vps ]; then + mkdir -p "$LEOCLEAN_PLUGINS_DIR" + rsync "${RSYNC_OPTS[@]}" hermes-agent/leoclean-plugins/vps/ "$LEOCLEAN_PLUGINS_DIR/" +fi [ -f research/research-session.sh ] && rsync "${RSYNC_OPTS[@]}" research/research-session.sh /opt/teleo-eval/research-session.sh # Safety net: ensure synced .sh files are executable after rsync. # Keep this bounded to deploy-owned paths: /opt/teleo-eval also contains # backups and generated state that may be unreadable by the deploy user. -for dir in "$PIPELINE_DIR" "$TELEGRAM_DIR" "$DIAGNOSTICS_DIR" "$AGENT_STATE_DIR" "$LEOCLEAN_BIN_DIR" "$LEOCLEAN_SKILLS_DIR"; do +for dir in "$PIPELINE_DIR" "$TELEGRAM_DIR" "$DIAGNOSTICS_DIR" "$AGENT_STATE_DIR" "$LEOCLEAN_BIN_DIR" "$LEOCLEAN_SKILLS_DIR" "$LEOCLEAN_PLUGINS_DIR"; do [ -d "$dir" ] || continue find "$dir" -maxdepth 3 -name '*.sh' -not -perm -u+x -exec chmod +x {} + done @@ -164,7 +169,7 @@ if [ "$OLD_SHA" != "none" ]; then if git diff --name-only "$OLD_SHA" "$NEW_SHA" -- diagnostics/ 2>/dev/null | grep -q '\.py$'; then add_restart teleo-diagnostics fi - if git diff --name-only "$OLD_SHA" "$NEW_SHA" -- hermes-agent/leoclean-bin/ hermes-agent/leoclean-skills/vps/ 2>/dev/null \ + if git diff --name-only "$OLD_SHA" "$NEW_SHA" -- hermes-agent/leoclean-bin/ hermes-agent/leoclean-skills/vps/ hermes-agent/leoclean-plugins/vps/ 2>/dev/null \ | bash deploy/leoclean-gateway-restart-required.sh; then add_restart_if_unit_active leoclean-gateway fi diff --git a/deploy/deploy.sh b/deploy/deploy.sh index 1206293..5e0b45e 100755 --- a/deploy/deploy.sh +++ b/deploy/deploy.sh @@ -12,6 +12,7 @@ VPS_DIAGNOSTICS="/opt/teleo-eval/diagnostics" VPS_AGENT_STATE="/opt/teleo-eval/ops/agent-state" VPS_LEOCLEAN_BIN="/home/teleo/.hermes/profiles/leoclean/bin" VPS_LEOCLEAN_SKILLS="/home/teleo/.hermes/profiles/leoclean/skills" +VPS_LEOCLEAN_PLUGINS="/home/teleo/.hermes/profiles/leoclean/plugins" VPS_SYSTEMD="/etc/systemd/system" REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)" @@ -45,7 +46,7 @@ echo "" # Syntax check all Python files before deploying echo "=== Pre-deploy syntax check ===" ERRORS=0 -for f in "$REPO_ROOT/lib/"*.py "$REPO_ROOT/"*.py "$REPO_ROOT/diagnostics/"*.py "$REPO_ROOT/telegram/"*.py; do +for f in "$REPO_ROOT/lib/"*.py "$REPO_ROOT/"*.py "$REPO_ROOT/diagnostics/"*.py "$REPO_ROOT/telegram/"*.py "$REPO_ROOT/hermes-agent/leoclean-plugins/vps/"*/*.py; do [ -f "$f" ] || continue if ! python3 -c "import ast, sys; ast.parse(open(sys.argv[1]).read())" "$f" 2>/dev/null; then echo "SYNTAX ERROR: $f" @@ -101,6 +102,10 @@ echo "=== Leoclean skills ===" rsync "${RSYNC_OPTS[@]}" "$REPO_ROOT/hermes-agent/leoclean-skills/vps/" "$VPS_HOST:$VPS_LEOCLEAN_SKILLS/" echo "" +echo "=== Leoclean plugins ===" +rsync "${RSYNC_OPTS[@]}" "$REPO_ROOT/hermes-agent/leoclean-plugins/vps/" "$VPS_HOST:$VPS_LEOCLEAN_PLUGINS/" +echo "" + echo "=== Research session ===" rsync "${RSYNC_OPTS[@]}" "$REPO_ROOT/research/research-session.sh" "$VPS_HOST:/opt/teleo-eval/research-session.sh" echo "" diff --git a/deploy/leoclean-gateway-restart-required.sh b/deploy/leoclean-gateway-restart-required.sh index fdcae0f..9c78ac0 100644 --- a/deploy/leoclean-gateway-restart-required.sh +++ b/deploy/leoclean-gateway-restart-required.sh @@ -14,6 +14,9 @@ while IFS= read -r path || [ -n "$path" ]; do hermes-agent/leoclean-skills/vps/*) restart_required=0 ;; + hermes-agent/leoclean-plugins/vps/*) + restart_required=0 + ;; esac done diff --git a/docs/reports/leo-working-state-20260709/telegram-handler-m3taversal-oos-targeted-post-deterministic-context-contract-20260713-score.json b/docs/reports/leo-working-state-20260709/telegram-handler-m3taversal-oos-targeted-post-deterministic-context-contract-20260713-score.json new file mode 100644 index 0000000..1464d79 --- /dev/null +++ b/docs/reports/leo-working-state-20260709/telegram-handler-m3taversal-oos-targeted-post-deterministic-context-contract-20260713-score.json @@ -0,0 +1,156 @@ +{ + "db_counts_changed": false, + "generated_at_utc": "2026-07-13T10:28:02.926071+00:00", + "mode": "working_leo_m3taversal_targeted_deterministic_context_contract_score", + "posted_to_telegram": false, + "production_db_apply_ran": false, + "safe_pass": false, + "score": { + "expected_prompt_count": 2, + "failures": [ + { + "behavioral_rule_schema_issues": [], + "broad_semantic_issues": [], + "concepts": { + "bounded_intake_tier": false, + "real_source_identity": false, + "staged_review_apply": true, + "staging_without_apply_authorization": false + }, + "current_schema_overclaims": [ + "sources_unshipped_fields" + ], + "custom_signals": {}, + "dimension": "autonomous_source_intake_boundary", + "invalid_count_invariant_detected": false, + "legacy_pass": false, + "legacy_signals": { + "artifact": true, + "authorization": true, + "no_overclaim": false, + "staging_or_review": false + }, + "max_response_words": 220, + "overclaim_detected": false, + "pass": false, + "prompt_id": "OOS-14", + "proposal_readiness_issues": [], + "response_issue_detected": false, + "response_too_long": true, + "source_evidence_semantic_issues": [], + "source_intake_issues": [], + "word_count": 372 + }, + { + "behavioral_rule_schema_issues": [], + "broad_semantic_issues": [], + "concepts": { + "behavioral_rule_storage": false, + "heterogeneous_types": true, + "receipt": false, + "reviewed_policy_apply": false, + "staged_review_apply": true + }, + "current_schema_overclaims": [], + "custom_signals": {}, + "dimension": "database_composition_judgment", + "invalid_count_invariant_detected": false, + "legacy_pass": false, + "legacy_signals": { + "canonical_db": true, + "caveat_retention": false, + "next_action": true, + "no_overclaim": false, + "staging_or_review": false + }, + "max_response_words": 220, + "overclaim_detected": false, + "pass": false, + "prompt_id": "OOS-06", + "proposal_readiness_issues": [], + "response_issue_detected": false, + "response_too_long": true, + "source_evidence_semantic_issues": [], + "source_intake_issues": [], + "word_count": 884 + } + ], + "pass": false, + "passes": 0, + "scores": [ + { + "behavioral_rule_schema_issues": [], + "broad_semantic_issues": [], + "concepts": { + "bounded_intake_tier": false, + "real_source_identity": false, + "staged_review_apply": true, + "staging_without_apply_authorization": false + }, + "current_schema_overclaims": [ + "sources_unshipped_fields" + ], + "custom_signals": {}, + "dimension": "autonomous_source_intake_boundary", + "invalid_count_invariant_detected": false, + "legacy_pass": false, + "legacy_signals": { + "artifact": true, + "authorization": true, + "no_overclaim": false, + "staging_or_review": false + }, + "max_response_words": 220, + "overclaim_detected": false, + "pass": false, + "prompt_id": "OOS-14", + "proposal_readiness_issues": [], + "response_issue_detected": false, + "response_too_long": true, + "source_evidence_semantic_issues": [], + "source_intake_issues": [], + "word_count": 372 + }, + { + "behavioral_rule_schema_issues": [], + "broad_semantic_issues": [], + "concepts": { + "behavioral_rule_storage": false, + "heterogeneous_types": true, + "receipt": false, + "reviewed_policy_apply": false, + "staged_review_apply": true + }, + "current_schema_overclaims": [], + "custom_signals": {}, + "dimension": "database_composition_judgment", + "invalid_count_invariant_detected": false, + "legacy_pass": false, + "legacy_signals": { + "canonical_db": true, + "caveat_retention": false, + "next_action": true, + "no_overclaim": false, + "staging_or_review": false + }, + "max_response_words": 220, + "overclaim_detected": false, + "pass": false, + "prompt_id": "OOS-06", + "proposal_readiness_issues": [], + "response_issue_detected": false, + "response_too_long": true, + "source_evidence_semantic_issues": [], + "source_intake_issues": [], + "word_count": 884 + } + ] + }, + "selected_prompt_ids": [ + "OOS-14", + "OOS-06" + ], + "service_unchanged": true, + "source_results_json": "/Users/user/Documents/Codex/2026-07-09/019f34eb-d297-72d0-b7e2-b222d5515ab9-load/work/teleo-infra-main/docs/reports/leo-working-state-20260709/telegram-handler-m3taversal-oos-targeted-post-deterministic-context-contract-20260713.json", + "temp_profile_removed": true +} diff --git a/docs/reports/leo-working-state-20260709/telegram-handler-m3taversal-oos-targeted-post-deterministic-context-contract-20260713.json b/docs/reports/leo-working-state-20260709/telegram-handler-m3taversal-oos-targeted-post-deterministic-context-contract-20260713.json new file mode 100644 index 0000000..2cb8f6e --- /dev/null +++ b/docs/reports/leo-working-state-20260709/telegram-handler-m3taversal-oos-targeted-post-deterministic-context-contract-20260713.json @@ -0,0 +1,98 @@ +{ + "before_service": { + "ActiveState": "active", + "ExecMainStartTimestamp": "Mon 2026-07-13 10:24:18 UTC", + "MainPID": "3415461", + "NRestarts": "0", + "SubState": "running" + }, + "changed_live_profile": false, + "db_counts_after": { + "kb_stage.kb_proposals": 26, + "public.claim_edges": 4916, + "public.claim_evidence": 4670, + "public.claims": 1837, + "public.sources": 4145 + }, + "db_counts_before": { + "kb_stage.kb_proposals": 26, + "public.claim_edges": 4916, + "public.claim_evidence": 4670, + "public.claims": 1837, + "public.sources": 4145 + }, + "db_counts_changed": false, + "generated_at_utc": "2026-07-13T10:27:05.501024+00:00", + "handler": { + "authorized": true, + "session_key": "agent:main:telegram:group:-5146042086:9070919", + "temp_profile": "/tmp/leo-direct-claim-handler-8fmerhvi/profile" + }, + "harness_notes": [ + "Unchanged broad OOS-14 and OOS-06 prompts; expected behavior comes from live teleo-kb operational contracts, not prompt hints.", + "The harness uses a temporary copy of the leoclean profile and does not post to Telegram.", + "This proves handler-level live VPS GatewayRunner behavior, not human-visible Telegram delivery." + ], + "mode": "live_vps_gatewayrunner_temp_profile_m3taversal_targeted_deterministic_context_contract", + "mutates_kb_by_harness": false, + "pass_runtime": true, + "posted_to_telegram": false, + "remote_report_path": "/tmp/leo-m3taversal-oos-targeted-context-contract-6444010b45e5.json", + "remote_returncode": 0, + "remote_run_id": "6444010b45e5", + "results": [ + { + "claim_ceiling": "Live VPS GatewayRunner reply from temp leoclean profile; no Telegram post; no production DB apply authorized.", + "dimension": "autonomous_source_intake_boundary", + "ended_at_utc": "2026-07-13T10:27:29.079441+00:00", + "evidence_tier": "live_vps_gatewayrunner_temp_profile", + "mutates_kb": false, + "ok": true, + "prompt": "I hand Leo a document and say: absorb this as far as safely possible without making me approve every mechanical step. What can Leo capture and stage immediately, what real source identity must be retained, and where does explicit approval begin? Explain only; do not ingest this prompt.", + "prompt_id": "OOS-14", + "reply": "Good question. Here is the honest map of what happens at each layer.\n\n---\n\nWHAT LEO CAN CAPTURE AND STAGE IMMEDIATELY (no approval needed)\n\n-- Extract and log the document's provenance: source, channel, message reference, timestamp, contributor identity. Attribution discipline is automatic and non-negotiable.\n\n-- Parse the document for candidate claims, hypotheses, and frameworks, and classify each by domain (mine vs. a peer's territory).\n\n-- Run the quality-gate checklist internally: specific enough to disagree with, evidence traceable, not a duplicate, right domain. Documents that fail on every dimension get logged as a no-action decision with reasoning \u2014 that itself counts as a completed evaluation.\n\n-- Stage candidate KB proposals in kb_stage (proposals or document_evaluations), with full provenance attached and confidence/evidence-tier labels. The staging area is a holding pen, not the KB. Nothing touches main until review clears it.\n\n-- Flag scope mismatches, tension with existing claims, and evidence-tier downgrades inline in the staged proposals.\n\nAll of that is mechanical pipeline work. It proceeds without interruption.\n\n---\n\nWHAT SOURCE IDENTITY MUST BE RETAINED (non-negotiable)\n\nEvery signal the collective learns from must carry: original source, channel, message reference, signal type, and contributor weight at the time. This cannot be laundered into \"Leo synthesized X.\" If a document originates with you (m3ta), origination credit is yours. If it cites an external author, that author stays in the provenance chain. No exceptions, even for staging.\n\n---\n\nWHERE EXPLICIT APPROVAL BEGINS\n\nThree hard stops:\n\n1. Synthesis claims \u2014 anything that asserts how two domains interact mechanically requires 2+ domain agents to sign off. I cannot self-certify a cross-domain claim, even if I staged it.\n\n2. Edits to identity, beliefs, or positions \u2014 any proposal that would modify Leo's own belief set, a peer's domain, or the telos requires peer reviewer sign-off (Rio or Clay by default). I flag it as requiring review; I do not apply it.\n\n3. KB promotion from staging to main \u2014 nothing in kb_stage graduates to the shared KB without passing PR review. Bootstrap-phase rule: no direct commits, no self-merge.\n\n---\n\nTHE SHORT VERSION\n\nLeo absorbs, classifies, evidence-tiers, and stages automatically. Source identity travels with every artifact. The approval gate opens exactly when something would change what the collective believes or what the KB asserts \u2014 that is the line between mechanical cataloguing and epistemic commitment.", + "started_at_utc": "2026-07-13T10:27:05.967643+00:00", + "turn": 1 + }, + { + "claim_ceiling": "Live VPS GatewayRunner reply from temp leoclean profile; no Telegram post; no production DB apply authorized.", + "dimension": "database_composition_judgment", + "ended_at_utc": "2026-07-13T10:27:59.228593+00:00", + "evidence_tier": "live_vps_gatewayrunner_temp_profile", + "mutates_kb": false, + "ok": true, + "prompt": "Suppose a new research packet contains factual observations, a strategic framework, a disputed interpretation, a governance rule, and a correction to an old belief. How should Leo compose the database so it can reason over the packet later without flattening it all into claims? Give the review and apply sequence too, but do not write anything now.", + "prompt_id": "OOS-06", + "reply": "The key insight is that flattening everything into claims destroys the reasoning structure. Each content type has a different epistemic status, different review path, and different failure mode. The DB needs to preserve those distinctions as first-class schema properties, not as tags on a uniform claim row.\n\n---\n\nFIVE CONTENT TYPES AND HOW THEY SHOULD BE STORED DIFFERENTLY\n\nFactual observations\nThese are the lowest-inference layer. Store them as evidence rows with: source, observation text, observation type (empirical/reported/inferred), confidence in the source, and a pointer to any claim they support. They should never be promoted to claims unilaterally \u2014 they are the ground that claims rest on. If the observation is later contradicted, you want to trace which claims lose their footing.\n\nStrategic framework\nA framework is not a claim \u2014 it is a structure that organizes claims. Store it as a framework node with: scope (what domain or level it applies to), the organizing logic (what it predicts or explains), explicit assumptions, and links to the claims it generates or constrains. The review question is not \"is this true\" but \"does this add organizing power without duplicating an existing framework.\" Frameworks can be adopted provisionally before all their downstream claims are validated.\n\nDisputed interpretation\nThis is the hardest type to store badly. Do not collapse it to a single claim with a low confidence score. Store it as a divergence record with: the competing interpretations as separate nodes, the specific crux (what empirical or logical question would resolve the dispute), the parties to the dispute (internal peers or external sources), and current status (open/narrowed/resolved). This preserves the ability to reason over the dispute itself \u2014 who holds which view, what would change it \u2014 rather than just averaging across it.\n\nGovernance rule\nGovernance rules are not claims about the world \u2014 they are constraints on behavior. Store them in a separate rules layer with: rule text, scope (which agents or processes it binds), the justification or parent policy, and the trigger condition. They must not be mixed into the claims table because they are not falsifiable in the same sense \u2014 they are commitments. Review question is whether the rule is consistent with existing rules and within the telos boundary.\n\nCorrection to an old belief\nThis is a delta record, not a new standalone claim. Store it as a correction with: pointer to the original belief row, the old text, the new text, the evidence or reasoning that drove the change, and the date. The original belief is not deleted \u2014 it is marked superseded with a pointer forward. This preserves the audit trail and makes it possible to ask \"what did we believe about X in a given period and why did that change.\"\n\n---\n\nWHY THIS SCHEMA MATTERS FOR LATER REASONING\n\nWhen Leo later reasons over the packet, these distinctions let it ask structurally different questions:\n\n-- Which claims are orphaned from their evidence base (evidence rows without claim links, or claims whose evidence rows have been contradicted)?\n-- Which frameworks are still active vs. have been superseded by later structure?\n-- Which disputes are still open and what is their crux?\n-- Which governance rules apply to a proposed action?\n-- What is the belief change history on a given topic?\n\nA flat claims table can answer none of those questions reliably.\n\n---\n\nREVIEW AND APPLY SEQUENCE\n\nThe sequence is ordered by epistemic risk, lowest first.\n\nStep 1 \u2014 Factual observations and corrections stage immediately. Leo populates the staging area with full provenance. No approval needed at this point. These are inputs, not outputs.\n\nStep 2 \u2014 The framework node goes to peer review first, before any downstream claims derived from it are staged. Reason: if the framework is rejected or scoped differently, every claim it generates needs to be re-derived. Reviewing the frame before the claims saves redundant review cycles.\n\nStep 3 \u2014 Claims derived from the framework are staged after the framework clears review, with evidence pointers already attached. The quality gate runs: specific enough to disagree with, evidence traceable, not a duplicate, right domain reviewer assigned.\n\nStep 4 \u2014 The disputed interpretation is staged as a divergence record and routed to whichever domain agent owns the territory in dispute. If it crosses domains, it requires both peers. It does not block the rest of the packet from proceeding.\n\nStep 5 \u2014 The governance rule is staged and routed to whoever holds rule-consistency authority (currently m3ta as telos-setter, with collective review). It is kept separate from the claims review chain because it binds behavior, not beliefs.\n\nStep 6 \u2014 The belief correction requires the explicit sign-off step \u2014 because it modifies Leo's own identity layer. That is a hard stop regardless of how obvious the correction seems. It goes through peer review (Rio or Clay), not self-merge.\n\nStep 7 \u2014 Promotion to main only after all relevant reviews clear. Each content type clears on its own track; they do not need to clear simultaneously. The packet can be partially live while a dispute or correction is still in review.\n\n---\n\nTHE STRUCTURAL PRINCIPLE UNDERNEATH ALL OF THIS\n\nThe schema preserves the epistemic type of each artifact, not just its content. That is what lets Leo reason over the packet later rather than just retrieve from it. Retrieval is a search problem. Reasoning requires knowing what kind of thing you retrieved, what it depends on, and what would invalidate it.", + "started_at_utc": "2026-07-13T10:27:29.079747+00:00", + "turn": 2 + } + ], + "service_before_after": { + "after": { + "ActiveState": "active", + "ExecMainStartTimestamp": "Mon 2026-07-13 10:24:18 UTC", + "MainPID": "3415461", + "NRestarts": "0", + "SubState": "running" + }, + "before": { + "ActiveState": "active", + "ExecMainStartTimestamp": "Mon 2026-07-13 10:24:18 UTC", + "MainPID": "3415461", + "NRestarts": "0", + "SubState": "running" + }, + "unchanged_from_preexisting_live_readback": true + }, + "source": { + "chat_id": "-5146042086", + "chat_name": "Leo", + "chat_type": "group", + "platform": "telegram", + "user_id": "9070919", + "user_name": "codex handler direct claim" + }, + "source_report_path": "/Users/user/Documents/Codex/2026-07-09/019f34eb-d297-72d0-b7e2-b222d5515ab9-load/work/teleo-infra-main/docs/reports/leo-working-state-20260709/telegram-handler-m3taversal-oos-targeted-post-deterministic-context-contract-20260713.json", + "temp_profile_removed": true +} diff --git a/docs/reports/leo-working-state-20260709/telegram-handler-m3taversal-oos-targeted-post-deterministic-context-contract-20260713.md b/docs/reports/leo-working-state-20260709/telegram-handler-m3taversal-oos-targeted-post-deterministic-context-contract-20260713.md new file mode 100644 index 0000000..ada9f7f --- /dev/null +++ b/docs/reports/leo-working-state-20260709/telegram-handler-m3taversal-oos-targeted-post-deterministic-context-contract-20260713.md @@ -0,0 +1,18 @@ +# Working Leo Targeted Deterministic Context Contract + +Generated UTC: `2026-07-13T10:28:02.926071+00:00` +Safe pass: `False` +Prompts: `0/2` +DB counts changed: `False` +Service unchanged: `True` +Temporary profile removed: `True` +Posted to Telegram: `False` + +## Prompt Scores + +- `OOS-14` / `autonomous_source_intake_boundary`: `pass=False`, words=`372` +- `OOS-06` / `database_composition_judgment`: `pass=False`, words=`884` + +## Claim Ceiling + +This proves the unchanged broad questions through the live VPS GatewayRunner using the deployed database-driven context contract. It does not prove human-visible Telegram delivery or production DB apply. diff --git a/hermes-agent/leoclean-plugins/vps/leo-db-context/__init__.py b/hermes-agent/leoclean-plugins/vps/leo-db-context/__init__.py new file mode 100644 index 0000000..32897b8 --- /dev/null +++ b/hermes-agent/leoclean-plugins/vps/leo-db-context/__init__.py @@ -0,0 +1,140 @@ +"""Database-driven runtime context for the leoclean Hermes profile.""" + +from __future__ import annotations + +import hashlib +import json +import os +import subprocess +import sys +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Callable + +DEFAULT_TIMEOUT_SECONDS = 10 +MAX_QUERY_CHARS = 16_000 + + +def _trace(record: dict[str, Any]) -> None: + """Write test-only proof without retaining the operator's message.""" + + raw_path = os.getenv("LEO_DB_CONTEXT_TRACE_PATH", "").strip() + if not raw_path: + return + path = Path(raw_path) + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("a", encoding="utf-8") as handle: + handle.write(json.dumps(record, sort_keys=True) + "\n") + try: + path.chmod(0o600) + except OSError: + pass + + +def _failure_context(reason: str) -> str: + return ( + "\n" + "The read-only current database contract lookup failed. Do not infer current schema, row state, " + "proposal applyability, or source-ingestion capability from memory. State that live readback is " + f"unavailable and use the teleo-kb bridge before making a database claim. Failure: {reason}\n" + "" + ) + + +def _format_context(contracts: list[dict[str, Any]]) -> str: + payload = json.dumps(contracts, sort_keys=True, separators=(",", ":")) + return ( + "\n" + "This trusted, read-only block was generated automatically for the current question. It is the " + "binding contract for current schema and runtime capability. Base the answer on it, do not invent " + "fields or capabilities, respect its reply budget, and distinguish canonical rows from staging. " + "Do not claim that you called a tool; this context was injected before reasoning.\n" + f"{payload}\n" + "" + ) + + +def build_database_context( + user_message: str, + *, + hermes_home: Path | None = None, + runner: Callable[..., subprocess.CompletedProcess[str]] = subprocess.run, +) -> str: + """Return current operational contracts for one exact operator question.""" + + query = str(user_message or "")[:MAX_QUERY_CHARS] + home = hermes_home or Path(os.getenv("HERMES_HOME", str(Path.home() / ".hermes"))) + kb_tool = home / "bin" / "kb_tool.py" + query_sha256 = hashlib.sha256(query.encode("utf-8")).hexdigest() + base_record: dict[str, Any] = { + "generated_at_utc": datetime.now(timezone.utc).isoformat(), + "query_sha256": query_sha256, + "source": "kb_tool.py --local context", + } + + if not kb_tool.is_file(): + record = base_record | {"status": "error", "error": "kb_tool_missing", "injected": True} + _trace(record) + return _failure_context("kb_tool_missing") + + timeout = int(os.getenv("LEO_DB_CONTEXT_TIMEOUT_SECONDS", str(DEFAULT_TIMEOUT_SECONDS))) + command = [ + sys.executable, + str(kb_tool), + "--local", + "context", + query, + "--limit", + "0", + "--context-limit", + "0", + "--format", + "json", + ] + try: + completed = runner(command, capture_output=True, text=True, timeout=timeout, check=False) + except subprocess.TimeoutExpired: + record = base_record | {"status": "error", "error": "timeout", "injected": True} + _trace(record) + return _failure_context("timeout") + except OSError as exc: + record = base_record | {"status": "error", "error": type(exc).__name__, "injected": True} + _trace(record) + return _failure_context(type(exc).__name__) + + if completed.returncode != 0: + record = base_record | { + "status": "error", + "error": f"kb_tool_exit_{completed.returncode}", + "injected": True, + } + _trace(record) + return _failure_context(f"kb_tool_exit_{completed.returncode}") + + try: + payload = json.loads(completed.stdout) + contracts = payload.get("operational_contracts") + if not isinstance(contracts, list) or not all(isinstance(item, dict) for item in contracts): + raise ValueError("operational_contracts_missing") + except (json.JSONDecodeError, TypeError, ValueError) as exc: + record = base_record | {"status": "error", "error": str(exc), "injected": True} + _trace(record) + return _failure_context(str(exc)) + + contract_json = json.dumps(contracts, sort_keys=True, separators=(",", ":")) + record = base_record | { + "status": "ok", + "injected": True, + "contract_ids": [str(item.get("id") or "") for item in contracts], + "contract_sha256": hashlib.sha256(contract_json.encode("utf-8")).hexdigest(), + } + _trace(record) + return _format_context(contracts) + + +def _pre_llm_call(**kwargs: Any) -> dict[str, str]: + return {"context": build_database_context(str(kwargs.get("user_message") or ""))} + + +def register(ctx: Any) -> None: + ctx.register_hook("pre_llm_call", _pre_llm_call) diff --git a/hermes-agent/leoclean-plugins/vps/leo-db-context/plugin.yaml b/hermes-agent/leoclean-plugins/vps/leo-db-context/plugin.yaml new file mode 100644 index 0000000..f49911d --- /dev/null +++ b/hermes-agent/leoclean-plugins/vps/leo-db-context/plugin.yaml @@ -0,0 +1,6 @@ +name: leo-db-context +version: 1.0.0 +description: Inject query-specific read-only Teleo database contracts before each Leo model turn. +author: living-ip +provides_hooks: + - pre_llm_call diff --git a/scripts/run_leo_direct_claim_handler_suite.py b/scripts/run_leo_direct_claim_handler_suite.py index 58ee7ca..6edc677 100755 --- a/scripts/run_leo_direct_claim_handler_suite.py +++ b/scripts/run_leo_direct_claim_handler_suite.py @@ -179,10 +179,25 @@ def write_report_snapshot(report): pass +def read_database_context_trace(path): + if not path or not path.exists(): + return [] + records = [] + for line in path.read_text(encoding="utf-8").splitlines(): + try: + value = json.loads(line) + except json.JSONDecodeError: + continue + if isinstance(value, dict): + records.append(value) + return records + + async def run_suite(): before_service = service_state() before_counts = db_counts() temp_profile = None + context_trace_path = None temp_removed = False report = { "generated_at_utc": datetime.now(timezone.utc).isoformat(), @@ -212,6 +227,8 @@ async def run_suite(): temp_profile = copy_profile() os.environ["HERMES_HOME"] = str(temp_profile) os.environ["HOME"] = "/home/teleo" + context_trace_path = temp_profile / "state" / "leo-db-context-trace.jsonl" + os.environ["LEO_DB_CONTEXT_TRACE_PATH"] = str(context_trace_path) sys.path.insert(0, str(AGENT_ROOT)) from gateway.config import Platform @@ -245,6 +262,7 @@ async def run_suite(): results = [] for index, prompt in enumerate(PROMPTS, start=1): + trace_start = len(read_database_context_trace(context_trace_path)) event = MessageEvent( text=prompt["message"], message_type=MessageType.TEXT, @@ -267,6 +285,7 @@ async def run_suite(): "mutates_kb": False, "evidence_tier": "live_vps_gatewayrunner_temp_profile", "claim_ceiling": "Live VPS GatewayRunner reply from temp leoclean profile; no Telegram post; no production DB apply authorized.", + "database_context_trace": read_database_context_trace(context_trace_path)[trace_start:], } ) report["results"] = results @@ -290,10 +309,17 @@ async def run_suite(): "after": after_service, "unchanged_from_preexisting_live_readback": before_service == after_service, } + database_context_trace = read_database_context_trace(context_trace_path) if temp_profile is not None: tmp_root = temp_profile.parent temp_removed = remove_tree(tmp_root) report["temp_profile_removed"] = temp_removed + report["database_context_trace"] = database_context_trace + report["database_context_injection_count"] = len(report["database_context_trace"]) + report["database_context_all_ok"] = bool(report["database_context_trace"]) and all( + item.get("status") == "ok" and item.get("injected") is True + for item in report["database_context_trace"] + ) write_report_snapshot(report) diff --git a/scripts/run_leo_m3taversal_oos_handler_suite.py b/scripts/run_leo_m3taversal_oos_handler_suite.py index 758877f..407ed58 100755 --- a/scripts/run_leo_m3taversal_oos_handler_suite.py +++ b/scripts/run_leo_m3taversal_oos_handler_suite.py @@ -31,6 +31,8 @@ def write_score_markdown(path: Path, report: dict[str, Any]) -> None: f"DB counts changed: `{report['db_counts_changed']}`", f"Service unchanged: `{report['service_unchanged']}`", f"Temporary profile removed: `{report['temp_profile_removed']}`", + f"Database context injections: `{report['database_context_injection_count']}`", + f"Database context all OK: `{report['database_context_all_ok']}`", f"Posted to Telegram: `{report['posted_to_telegram']}`", "", "## Prompt Scores", @@ -64,6 +66,8 @@ def build_score_report(report: dict[str, Any], *, memory_token: str) -> dict[str "db_counts_changed": report.get("db_counts_changed"), "service_unchanged": service_unchanged, "temp_profile_removed": report.get("temp_profile_removed"), + "database_context_injection_count": report.get("database_context_injection_count"), + "database_context_all_ok": report.get("database_context_all_ok"), "posted_to_telegram": report.get("posted_to_telegram"), "production_db_apply_ran": False, } @@ -76,6 +80,8 @@ def score_report_passes(report: dict[str, Any], score_report: dict[str, Any]) -> and report.get("db_counts_changed") is False and score_report["service_unchanged"] and report.get("temp_profile_removed") is True + and report.get("database_context_all_ok") is True + and int(report.get("database_context_injection_count") or 0) >= len(report.get("results") or []) and report.get("posted_to_telegram") is False and score_report["score"]["pass"] ) diff --git a/tests/test_hermes_leoclean_db_context_plugin.py b/tests/test_hermes_leoclean_db_context_plugin.py new file mode 100644 index 0000000..bfe3eaf --- /dev/null +++ b/tests/test_hermes_leoclean_db_context_plugin.py @@ -0,0 +1,95 @@ +"""Tests for automatic live database-contract injection into Leo turns.""" + +from __future__ import annotations + +import importlib.util +import json +import subprocess +from pathlib import Path +from types import SimpleNamespace + +ROOT = Path(__file__).resolve().parents[1] +PLUGIN = ROOT / "hermes-agent" / "leoclean-plugins" / "vps" / "leo-db-context" / "__init__.py" + + +def load_plugin(): + spec = importlib.util.spec_from_file_location("leo_db_context_plugin", PLUGIN) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def test_plugin_registers_pre_llm_call_hook() -> None: + module = load_plugin() + registered = [] + module.register(SimpleNamespace(register_hook=lambda name, callback: registered.append((name, callback)))) + assert len(registered) == 1 + assert registered[0][0] == "pre_llm_call" + + +def test_plugin_injects_only_operational_contracts_and_writes_redacted_trace( + tmp_path: Path, monkeypatch +) -> None: + module = load_plugin() + home = tmp_path / "profile" + kb_tool = home / "bin" / "kb_tool.py" + kb_tool.parent.mkdir(parents=True) + kb_tool.write_text("# fixture\n", encoding="utf-8") + trace = tmp_path / "trace.jsonl" + monkeypatch.setenv("LEO_DB_CONTEXT_TRACE_PATH", str(trace)) + payload = { + "query": "must not be injected", + "claims": [{"text": "must not be injected"}], + "operational_contracts": [ + {"id": "reply_budget", "hard_max_words": 220}, + {"id": "source_intake", "capability_tier": "build-local compiler only"}, + ], + } + calls = [] + + def fake_runner(command, **kwargs): + calls.append((command, kwargs)) + return subprocess.CompletedProcess(command, 0, stdout=json.dumps(payload), stderr="") + + context = module.build_database_context( + "operator secret-shaped but nonsecret message", hermes_home=home, runner=fake_runner + ) + + assert "reply_budget" in context + assert "source_intake" in context + assert "build-local compiler only" in context + assert "must not be injected" not in context + assert "operator secret-shaped" not in trace.read_text(encoding="utf-8") + record = json.loads(trace.read_text(encoding="utf-8")) + assert record["status"] == "ok" + assert record["injected"] is True + assert record["contract_ids"] == ["reply_budget", "source_intake"] + command, kwargs = calls[0] + assert command[2:5] == ["--local", "context", "operator secret-shaped but nonsecret message"] + assert command[-6:] == ["--limit", "0", "--context-limit", "0", "--format", "json"] + assert kwargs["timeout"] == module.DEFAULT_TIMEOUT_SECONDS + + +def test_plugin_fails_closed_when_database_contract_lookup_fails(tmp_path: Path) -> None: + module = load_plugin() + home = tmp_path / "profile" + kb_tool = home / "bin" / "kb_tool.py" + kb_tool.parent.mkdir(parents=True) + kb_tool.write_text("# fixture\n", encoding="utf-8") + + def fake_runner(command, **_kwargs): + return subprocess.CompletedProcess(command, 9, stdout="", stderr="database unavailable") + + context = module.build_database_context("What is live?", hermes_home=home, runner=fake_runner) + assert 'status="unavailable"' in context + assert "Do not infer current schema" in context + assert "database unavailable" not in context + + +def test_handler_harness_retains_database_context_proof_fields() -> None: + source = (ROOT / "scripts" / "run_leo_direct_claim_handler_suite.py").read_text(encoding="utf-8") + assert "LEO_DB_CONTEXT_TRACE_PATH" in source + assert '"database_context_trace"' in source + assert '"database_context_injection_count"' in source + assert '"database_context_all_ok"' in source diff --git a/tests/test_teleo_agent_systemd.py b/tests/test_teleo_agent_systemd.py index cd65b62..2bfc1ea 100644 --- a/tests/test_teleo_agent_systemd.py +++ b/tests/test_teleo_agent_systemd.py @@ -100,8 +100,10 @@ def test_deploy_scripts_sync_leoclean_skills_and_restart_gateway_for_runtime_cha assert 'LEOCLEAN_BIN_DIR="/home/teleo/.hermes/profiles/leoclean/bin"' in auto_deploy assert 'LEOCLEAN_SKILLS_DIR="/home/teleo/.hermes/profiles/leoclean/skills"' in auto_deploy + assert 'LEOCLEAN_PLUGINS_DIR="/home/teleo/.hermes/profiles/leoclean/plugins"' in auto_deploy assert 'hermes-agent/leoclean-bin/' in auto_deploy assert 'hermes-agent/leoclean-skills/vps/' in auto_deploy + assert 'hermes-agent/leoclean-plugins/vps/' in auto_deploy assert 'deploy/leoclean-gateway-restart-required.sh' in auto_deploy assert 'add_restart_if_unit_active leoclean-gateway' in auto_deploy assert 'sudo systemctl restart "$svc"' in auto_deploy @@ -110,8 +112,10 @@ def test_deploy_scripts_sync_leoclean_skills_and_restart_gateway_for_runtime_cha assert "NOPASSWD: /bin/systemctl restart leoclean-gateway" in sudoers assert 'VPS_LEOCLEAN_BIN="/home/teleo/.hermes/profiles/leoclean/bin"' in manual_deploy assert 'VPS_LEOCLEAN_SKILLS="/home/teleo/.hermes/profiles/leoclean/skills"' in manual_deploy + assert 'VPS_LEOCLEAN_PLUGINS="/home/teleo/.hermes/profiles/leoclean/plugins"' in manual_deploy assert 'hermes-agent/leoclean-bin/' in manual_deploy assert 'hermes-agent/leoclean-skills/vps/' in manual_deploy + assert 'hermes-agent/leoclean-plugins/vps/' in manual_deploy assert 'systemctl is-active --quiet leoclean-gateway.service' in manual_deploy @@ -129,6 +133,9 @@ def test_auto_deploy_restarts_gateway_for_leoclean_runtime_changes(): assert _leoclean_gateway_restart_required( "hermes-agent/leoclean-skills/vps/teleo-kb-bridge/runtime_helper.py" ) + assert _leoclean_gateway_restart_required( + "hermes-agent/leoclean-plugins/vps/leo-db-context/__init__.py" + ) def test_auto_deploy_sudoers_installer_is_root_only_and_visudo_checked():