from __future__ import annotations import json import re import subprocess from pathlib import Path ROOT = Path(__file__).resolve().parents[1] REPORT_DIR = ROOT / "docs" / "reports" / "leo-working-state-20260709" MANIFEST = REPORT_DIR / "skill-pack-manifest.json" def _skill(name: str) -> str: return (ROOT / ".agents" / "skills" / name / "SKILL.md").read_text(encoding="utf-8") def test_manifest_indexes_valid_repo_native_skills() -> None: manifest = json.loads(MANIFEST.read_text(encoding="utf-8")) assert manifest["status"] == "repo_native_path_validated" assert manifest["repo_skill_root"] == ".agents/skills" assert manifest["validation"] == "tests/test_repo_skill_pack.py" assert "37/37" in manifest["claim_ceiling"] assert "PRs #146 and #147 are merged" in manifest["claim_ceiling"] assert "PR #148 is open candidate evidence only" in manifest["claim_ceiling"] for entry in manifest["skills"]: path = ROOT / entry["path"] text = path.read_text(encoding="utf-8") assert path.is_file() assert text.startswith("---\n") assert f"name: {entry['name']}\n" in text.split("---\n", 2)[1] assert "description:" in text.split("---\n", 2)[1] for relative in manifest["reference_files"]: assert (ROOT / relative).is_file(), relative def test_manifest_covers_the_required_onboarding_surface() -> None: manifest = json.loads(MANIFEST.read_text(encoding="utf-8")) expected = { "company_product_context", "vps", "gcp", "hermes_runtime", "database_provenance", "ingestion", "proposal_apply", "reconstruction", "identity", "testing", "security", "secrets", "incident_recovery", } assert set(manifest["required_coverage"]) == expected skill_names = {entry["name"] for entry in manifest["skills"]} repo_skill_names = {path.parent.name for path in (ROOT / ".agents" / "skills").glob("*/SKILL.md")} assert skill_names == repo_skill_names for owners in manifest["required_coverage"].values(): assert owners assert set(owners) <= skill_names def test_path_validator_and_clean_installer() -> None: manifest = json.loads(MANIFEST.read_text(encoding="utf-8")) validator = ROOT / manifest["validator"] result = subprocess.run( [str(validator), "--root", str(ROOT)], check=False, capture_output=True, text=True, ) assert result.returncode == 0, result.stdout + result.stderr payload = json.loads(result.stdout) assert payload["status"] == "pass" assert payload["skill_count"] == len(manifest["skills"]) assert payload["coverage_area_count"] == len(manifest["required_coverage"]) assert payload["problems"] == [] def test_installer_copies_only_manifest_skills(tmp_path: Path) -> None: manifest = json.loads(MANIFEST.read_text(encoding="utf-8")) installer = ROOT / manifest["installer"] target = tmp_path / "skills" result = subprocess.run( [str(installer), "--root", str(ROOT), "--target", str(target)], check=False, capture_output=True, text=True, ) assert result.returncode == 0, result.stdout + result.stderr payload = json.loads(result.stdout) assert payload["status"] == "pass" assert payload["installed_skill_count"] == len(manifest["skills"]) assert {path.name for path in target.iterdir()} == {entry["name"] for entry in manifest["skills"]} assert all((target / entry["name"] / "SKILL.md").is_file() for entry in manifest["skills"]) def test_installer_rejects_manifest_path_traversal(tmp_path: Path) -> None: manifest = json.loads(MANIFEST.read_text(encoding="utf-8")) manifest["skills"][0]["name"] = "../escape" malicious_manifest = tmp_path / "manifest.json" malicious_manifest.write_text(json.dumps(manifest), encoding="utf-8") target = tmp_path / "target" / "skills" result = subprocess.run( [ str(ROOT / manifest["installer"]), "--root", str(ROOT), "--manifest", str(malicious_manifest), "--target", str(target), ], check=False, capture_output=True, text=True, ) assert result.returncode == 1 assert "invalid skill name" in result.stdout assert not (tmp_path / "target" / "escape").exists() assert not target.exists() def test_isolated_install_canary_reaches_t2_without_claiming_agent_reasoning(tmp_path: Path) -> None: manifest = json.loads(MANIFEST.read_text(encoding="utf-8")) canary = ROOT / manifest["isolated_install_canary"] output = tmp_path / "receipt.json" result = subprocess.run( [str(canary), "--root", str(ROOT), "--output", str(output)], check=False, capture_output=True, text=True, ) assert result.returncode == 0, result.stdout + result.stderr payload = json.loads(output.read_text(encoding="utf-8")) assert payload["status"] == "pass" assert payload["required_tier"] == "T2_runtime" assert payload["current_tier"] == "T2_runtime" assert payload["proof_scope"] == "deterministic_isolated_install_and_route_validation" assert payload["fresh_temporary_skill_root"] is True assert payload["ambient_skill_root_used"] is False assert payload["agent_reasoning_exercised"] is False assert payload["installed_skill_count"] == len(manifest["skills"]) assert all(payload["semantic_checks"].values()) assert payload["canary_exit_status"] == 0 assert payload["cleanup_readback"]["temporary_skill_root_removed"] is True assert payload["missing_to_reach_required_tier"] == [] def test_fresh_agent_receipt_rejects_stale_routes() -> None: manifest = json.loads(MANIFEST.read_text(encoding="utf-8")) receipt = json.loads((ROOT / manifest["fresh_agent_receipt"]).read_text(encoding="utf-8")) assert receipt["status"] == "pass" assert receipt["evidence_origin"]["fork_turns"] == "none" assert receipt["evidence_origin"]["memory_used"] is False assert all(receipt["stale_claims_rejected"].values()) assert receipt["validator"]["status"] == "pass" assert receipt["validator"]["skill_count"] == 16 assert receipt["repository_readback"]["unchanged"] is True assert receipt["cleanup_readback"]["temporary_skill_root_removed"] is True assert receipt["external_runtime_contacted"] is False assert receipt["production_mutation"] is False assert receipt["contains_secrets"] is False assert receipt["problems"] == [] def test_db_change_skill_tracks_guarded_v2_runtime_proof() -> None: text = _skill("teleo-kb-db-change-workflow") squashed = " ".join(text.split()) for required in ( "`approve_claim` v2", "kb_review", "kb_apply", "kb_gate_owner", "NOINHERIT", "37/37", "exact table deltas", "source hashes", "trusted operator-only canonical writer", "run_approve_claim_isolated_container_canary.sh", "all `13` checks", "`34/34`", "PR #86", ): assert required in text assert "Production was not applied" in squashed assert "Arbitrary production document/tweet ingestion is not proven" in squashed assert "three approved-but-unapplied legacy packets" in squashed def test_gcp_skill_uses_current_operator_access_and_parity_truth() -> None: text = _skill("teleo-gcp-parity-ops") squashed = " ".join(text.split()) for required in ( "ssh teleo-gcp-staging", "teleo-pgvector-standby", "PostgreSQL 16.14", "public IP disabled", "39/39", "52,167/52,167", "`148735`, `NRestarts=0`", "NRestarts=0", "state.db", "leoclean-cloudsql-memory-sync.service", "gcp-db-first-restore-current.json", "gcp-db-first-parity-current.json", "gcp-db-first-blind-claim-current.json", "gcp-db-first-cleanup-current.json", "gcp-db-first-live-deploy-restart-current.json", "18/18", "6/6", "PID `304036`", "fail closed", "invalid_target", "older staging copy", ): assert required in text assert "does not store a Google password" in squashed assert "firewall-source-dependent" in text assert "Do not call this path working until a live `status` run passes" in text assert "Do not describe a passing Hermes memory sync as canonical KB parity" in text assert "Do not call control-plane inventory" in text def test_vps_onboarding_and_m3taversal_skills_point_to_current_proof() -> None: vps = _skill("teleo-vps-runtime-ops") provenance = _skill("teleo-infra-provenance") onboarding = _skill("teleo-leo-onboarding") outcomes = _skill("working-leo-m3taversal-outcomes") assert "reasoning tools `17`" in vps assert "current source hashes" in vps assert "`teleo-auto-deploy.timer`" in vps assert "separate state changes" in vps assert "/opt/teleo-eval/.last-deploy-sha" in provenance assert "does not prove a gateway restart" in " ".join(provenance.split()) assert "gcp-cloud-sql-t3-live-readonly-current.md" in onboarding assert "gcp-db-first-working-leo-20260714.md" in onboarding assert "gcp-db-first-parity-current.json" in onboarding assert "gcp-db-first-blind-claim-current.json" in onboarding assert "gcp-db-first-cleanup-current.json" in onboarding assert "ssh -o BatchMode=yes teleo-gcp-staging true" in onboarding assert "37/37" in outcomes assert "Approved is not the same as applied" in outcomes assert "Address `@m3taversal` only as `m3taversal`, exactly" in outcomes assert "Next proof-changing follow-up:" in outcomes assert "working-leo-current-proof-20260712.md" in outcomes assert "Cory" not in outcomes def test_reconstruction_and_pr_state_are_routed_without_candidate_overclaim() -> None: reconstruction = _skill("teleo-reconstruction-recovery") onboarding = _skill("teleo-leo-onboarding") gcp = _skill("teleo-gcp-parity-ops") current_truth = (REPORT_DIR / "current-truth-index.md").read_text(encoding="utf-8") assert "ops/run_local_genesis_ledger_rebuild.py" in reconstruction assert "scripts/verify_leo_unseen_reasoning_chain.py" in reconstruction assert "Semantic source-to-blank-database recompilation remains incomplete" in reconstruction assert "PRs #146 and #147 are merged repository truth" in onboarding assert "PR #148 remains an open GCP" in onboarding assert "candidate evidence only" in gcp assert "PRs #146 and #147 are merged into canonical `main`" in current_truth assert "PR #148 remains open candidate evidence" in current_truth def test_active_onboarding_prose_uses_exact_m3taversal_identity() -> None: manifest = json.loads(MANIFEST.read_text(encoding="utf-8")) active_files = [ *[ROOT / relative for relative in manifest["scan_files"]], *[ROOT / entry["path"] for entry in manifest["skills"]], ] for path in active_files: text = path.read_text(encoding="utf-8") prose = re.sub(r"`[^`]*\/[^`]+\.(?:md|json|svg)`", "", text) assert re.search(r"\bcory(?:-style)?\b", prose, flags=re.IGNORECASE) is None, path