teleo-infrastructure/scripts/run_approve_claim_isolated_container_canary.sh

#!/usr/bin/env bash
set -euo pipefail

repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
source_container="${SOURCE_CONTAINER:-teleo-pg}"
source_db="${SOURCE_DB:-teleo}"
service="${LEO_SERVICE:-leoclean-gateway.service}"
image="${POSTGRES_CANARY_IMAGE:-postgres:16}"
normalization_json=""
output="${repo_root}/docs/reports/leo-working-state-20260709/approve-claim-clone-canary-current.json"

while [[ $# -gt 0 ]]; do
  case "$1" in
    --normalization-json)
      normalization_json="$2"
      shift 2
      ;;
    --output)
      output="$2"
      shift 2
      ;;
    --source-container)
      source_container="$2"
      shift 2
      ;;
    --source-db)
      source_db="$2"
      shift 2
      ;;
    --service)
      service="$2"
      shift 2
      ;;
    *)
      echo "unknown argument: $1" >&2
      exit 2
      ;;
  esac
done

container="working-leo-approve-claim-$RANDOM-$$"
workdir="/tmp/${container}"
apply_password="isolated-apply-${RANDOM}-${RANDOM}"
review_password="isolated-review-${RANDOM}-${RANDOM}"

cleanup() {
  docker rm -f "$container" >/dev/null 2>&1 || true
  rm -rf "$workdir"
}
trap cleanup EXIT

source_counts() {
  docker exec "$source_container" psql -U postgres -d "$source_db" -Atq \
    -v ON_ERROR_STOP=1 -c "begin transaction read only;
    select jsonb_build_object(
      'public.claims',(select count(*) from public.claims),
      'public.sources',(select count(*) from public.sources),
      'public.claim_evidence',(select count(*) from public.claim_evidence),
      'public.claim_edges',(select count(*) from public.claim_edges),
      'public.reasoning_tools',(select count(*) from public.reasoning_tools),
      'kb_stage.kb_proposals',(select count(*) from kb_stage.kb_proposals)
    )::text;
    rollback;"
}

service_state() {
  systemctl show "$service" --no-pager \
    -p ActiveState -p SubState -p MainPID -p NRestarts -p ExecMainStartTimestamp
}

mkdir -p "$workdir" "$(dirname "$output")"
source_counts_before="$(source_counts)"
service_before="$(service_state)"

docker run -d --name "$container" \
  -e POSTGRES_PASSWORD="isolated-postgres-${RANDOM}-${RANDOM}" \
  -e POSTGRES_DB=teleo \
  "$image" >/dev/null

for _ in $(seq 1 40); do
  if docker exec "$container" pg_isready -U postgres -d teleo >/dev/null 2>&1; then
    break
  fi
  sleep 0.25
done
docker exec "$container" pg_isready -U postgres -d teleo >/dev/null

docker exec "$source_container" pg_dump -U postgres -d "$source_db" \
  --schema-only --no-owner --no-privileges >"$workdir/schema.sql"
docker exec "$source_container" pg_dump -U postgres -d "$source_db" \
  --data-only --no-owner --no-privileges \
  --table=public.agents --table=public.claims >"$workdir/reference-data.sql"
docker cp "$workdir/schema.sql" "$container:/tmp/schema.sql"
docker cp "$workdir/reference-data.sql" "$container:/tmp/reference-data.sql"

docker exec -i "$container" psql -U postgres -d postgres -v ON_ERROR_STOP=1 -q <<SQL
create role kb_apply login noinherit nosuperuser nocreatedb nocreaterole
  noreplication nobypassrls password '${apply_password}';
create role kb_review login noinherit nosuperuser nocreatedb nocreaterole
  noreplication nobypassrls password '${review_password}';
create role kb_gate_owner nologin noinherit nosuperuser nocreatedb nocreaterole
  noreplication nobypassrls;
SQL
docker exec "$container" psql -U postgres -d teleo -v ON_ERROR_STOP=1 -q \
  -f /tmp/schema.sql
docker exec "$container" psql -U postgres -d teleo -v ON_ERROR_STOP=1 -q \
  -f /tmp/reference-data.sql
docker exec "$container" psql -U postgres -d teleo -v ON_ERROR_STOP=1 -q \
  -c "insert into public.agents (id, handle, kind)
      values ('99999999-9999-9999-9999-999999999999', 'm3ta', 'human')
      on conflict (handle) do nothing"

cat >"$workdir/kb-apply.env" <<EOF
KB_APPLY_PASSWORD=${apply_password}
EOF
cat >"$workdir/kb-review.env" <<EOF
KB_REVIEW_PASSWORD=${review_password}
EOF
chmod 600 "$workdir/kb-apply.env" "$workdir/kb-review.env"

normalization_args=()
if [[ -n "$normalization_json" ]]; then
  normalization_args=(--normalization-json "$normalization_json")
fi

set +e
python3 "$repo_root/scripts/run_approve_claim_clone_canary.py" \
  --container "$container" \
  --source-db teleo \
  --database-prefix teleo_approve_claim_isolated \
  --service "$service" \
  --approve-script "$repo_root/scripts/approve_proposal.py" \
  --apply-script "$repo_root/scripts/apply_proposal.py" \
  --prereqs "$repo_root/scripts/kb_apply_prereqs.sql" \
  --review-secrets-file "$workdir/kb-review.env" \
  --secrets-file "$workdir/kb-apply.env" \
  "${normalization_args[@]}" \
  --output "$output" \
  >"$workdir/stdout.json" 2>"$workdir/stderr.log"
canary_rc=$?
set -e

set +e
docker rm -f "$container" >/dev/null 2>&1
outer_container_remove_rc=$?
rm -rf "$workdir"
outer_workdir_remove_rc=$?
container_name_readback="$(
  docker container ls -a --filter "name=^/${container}$" --format '{{.Names}}' 2>&1
)"
container_name_readback_rc=$?
if [[ -e "$workdir" || -L "$workdir" ]]; then
  outer_workdir_lexists=true
else
  outer_workdir_lexists=false
fi
set -e
trap - EXIT

source_counts_after="$(source_counts)"
service_after="$(service_state)"

SOURCE_COUNTS_BEFORE="$source_counts_before" \
SOURCE_COUNTS_AFTER="$source_counts_after" \
SOURCE_CONTAINER="$source_container" \
SOURCE_DB="$source_db" \
SERVICE_BEFORE="$service_before" \
SERVICE_AFTER="$service_after" \
REPO_ROOT="$repo_root" \
OUTER_CONTAINER_REMOVE_RC="$outer_container_remove_rc" \
OUTER_WORKDIR_REMOVE_RC="$outer_workdir_remove_rc" \
CONTAINER_NAME_READBACK="$container_name_readback" \
CONTAINER_NAME_READBACK_RC="$container_name_readback_rc" \
OUTER_WORKDIR_LEXISTS="$outer_workdir_lexists" \
CANARY_RC="$canary_rc" \
python3 - "$output" <<'PY'
import hashlib
import json
import os
import sys
from pathlib import Path

path = Path(sys.argv[1])
if not path.is_file():
    raise SystemExit("inner canary did not produce a report")

data = json.loads(path.read_text(encoding="utf-8"))

def service(raw: str) -> dict[str, str]:
    return dict(line.split("=", 1) for line in raw.splitlines() if "=" in line)

def table_deltas(before: dict[str, int], after: dict[str, int]) -> dict[str, int]:
    if set(before) != set(after):
        raise SystemExit(
            "live source count keys changed during the canary: "
            f"before={sorted(before)} after={sorted(after)}"
        )
    return {key: after[key] - before[key] for key in sorted(before)}

def verify_source_binding(root: Path, records: list[dict[str, object]]) -> dict[str, object]:
    readbacks = []
    for record in records:
        relative_text = str(record.get("repo_relative_path", ""))
        relative = Path(relative_text)
        if not relative_text or relative.is_absolute() or ".." in relative.parts:
            raise SystemExit(f"unsafe repo-relative source path: {relative_text!r}")
        candidate = (root / relative).resolve()
        try:
            candidate.relative_to(root)
        except ValueError as exc:
            raise SystemExit(
                f"source path escaped repository root: {relative_text}"
            ) from exc
        if not candidate.is_file():
            readbacks.append(
                {
                    "repo_relative_path": relative.as_posix(),
                    "expected_sha256": record.get("sha256"),
                    "actual_sha256": None,
                    "matches": False,
                }
            )
            continue
        content = candidate.read_bytes()
        actual_sha256 = hashlib.sha256(content).hexdigest()
        readbacks.append(
            {
                "repo_relative_path": relative.as_posix(),
                "expected_sha256": record.get("sha256"),
                "actual_sha256": actual_sha256,
                "size_bytes": len(content),
                "matches": (
                    actual_sha256 == record.get("sha256")
                    and len(content) == record.get("size_bytes")
                ),
            }
        )
    return {
        "files": readbacks,
        "all_match": bool(readbacks) and all(row["matches"] for row in readbacks),
    }

def contains_ephemeral_path(value: object) -> bool:
    if isinstance(value, str):
        return "/tmp/" in value
    if isinstance(value, dict):
        return any(contains_ephemeral_path(item) for item in value.values())
    if isinstance(value, list):
        return any(contains_ephemeral_path(item) for item in value)
    return False

before_counts = json.loads(os.environ["SOURCE_COUNTS_BEFORE"])
after_counts = json.loads(os.environ["SOURCE_COUNTS_AFTER"])
live_table_deltas = table_deltas(before_counts, after_counts)
before_service = service(os.environ["SERVICE_BEFORE"])
after_service = service(os.environ["SERVICE_AFTER"])
service_stable = all(
    before_service.get(key) == after_service.get(key)
    for key in ("ActiveState", "SubState", "MainPID", "NRestarts")
)
container_absent = (
    int(os.environ["CONTAINER_NAME_READBACK_RC"]) == 0
    and not os.environ["CONTAINER_NAME_READBACK"].strip()
)
workdir_absent = os.environ["OUTER_WORKDIR_LEXISTS"] == "false"
source_binding = verify_source_binding(
    Path(os.environ["REPO_ROOT"]).resolve(),
    data.get("source_binding", {}).get("files", []),
)
data["source_binding"]["independent_post_cleanup_verification"] = source_binding
source_binding_ok = (
    data["source_binding"].get("unchanged_during_run") is True
    and source_binding["all_match"] is True
)
stale_ephemeral_paths_absent = not contains_ephemeral_path(data)
inner_runtime_pass = data.get("status") == "pass" and int(os.environ["CANARY_RC"]) == 0
outer_ok = (
    before_counts == after_counts
    and all(delta == 0 for delta in live_table_deltas.values())
    and service_stable
    and container_absent
    and workdir_absent
    and source_binding_ok
    and stale_ephemeral_paths_absent
)
data["required_tier"] = "T3_live_readonly"
data["required_tiers"] = ["T2_runtime", "T3_live_readonly"]
data["runtime_scope"] = "isolated_postgres_container_from_readonly_live_schema"
data["outer_isolation"] = {
    "source_container": os.environ.get("SOURCE_CONTAINER", "teleo-pg"),
    "source_database": os.environ.get("SOURCE_DB", "teleo"),
    "source_counts_before": before_counts,
    "source_counts_after": after_counts,
    "canonical_table_deltas": live_table_deltas,
    "source_counts_unchanged": before_counts == after_counts,
    "service_before": before_service,
    "service_after": after_service,
    "service_stable": service_stable,
    "cleanup_readback": {
        "container_remove_returncode": int(os.environ["OUTER_CONTAINER_REMOVE_RC"]),
        "container_name_query_returncode": int(
            os.environ["CONTAINER_NAME_READBACK_RC"]
        ),
        "container_name_query_stdout": os.environ["CONTAINER_NAME_READBACK"],
        "temporary_container_absent": container_absent,
        "workdir_remove_returncode": int(os.environ["OUTER_WORKDIR_REMOVE_RC"]),
        "workdir_lexists_after_cleanup": not workdir_absent,
        "temporary_workdir_absent": workdir_absent,
    },
}
data["checks"]["outer_live_counts_exactly_unchanged"] = before_counts == after_counts
data["checks"]["outer_live_table_deltas_exactly_zero"] = all(
    delta == 0 for delta in live_table_deltas.values()
)
data["checks"]["outer_service_stable"] = service_stable
data["checks"]["outer_container_absent_independent_readback"] = container_absent
data["checks"]["outer_workdir_absent_independent_readback"] = workdir_absent
data["checks"]["source_binding_independently_verified"] = source_binding_ok
data["checks"]["stale_ephemeral_paths_absent"] = stale_ephemeral_paths_absent
data["checks"]["outer_isolation_complete"] = outer_ok
if outer_ok and inner_runtime_pass:
    data["current_tier"] = "T3_live_readonly"
else:
    data["status"] = "fail"
    data["current_tier"] = "T2_runtime" if inner_runtime_pass else "T1_model"
path.write_text(json.dumps(data, indent=2, sort_keys=True) + "\n", encoding="utf-8")
raise SystemExit(0 if data["status"] == "pass" else 1)
PY