439 lines
16 KiB
Bash
Executable file
439 lines
16 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
|
source_container="${SOURCE_CONTAINER:-teleo-pg}"
|
|
source_db="${SOURCE_DB:-teleo}"
|
|
service="${LEO_SERVICE:-leoclean-gateway.service}"
|
|
image="${POSTGRES_CANARY_IMAGE:-postgres:16}"
|
|
source_tier="${SOURCE_TIER:-live-readonly}"
|
|
normalization_json=""
|
|
output="${repo_root}/docs/reports/leo-working-state-20260709/approve-claim-clone-canary-current.json"
|
|
|
|
while [[ $# -gt 0 ]]; do
|
|
case "$1" in
|
|
--normalization-json)
|
|
normalization_json="$2"
|
|
shift 2
|
|
;;
|
|
--output)
|
|
output="$2"
|
|
shift 2
|
|
;;
|
|
--source-container)
|
|
source_container="$2"
|
|
shift 2
|
|
;;
|
|
--source-db)
|
|
source_db="$2"
|
|
shift 2
|
|
;;
|
|
--service)
|
|
service="$2"
|
|
shift 2
|
|
;;
|
|
--source-tier)
|
|
source_tier="$2"
|
|
shift 2
|
|
;;
|
|
*)
|
|
echo "unknown argument: $1" >&2
|
|
exit 2
|
|
;;
|
|
esac
|
|
done
|
|
|
|
if [[ "$source_tier" != "isolated-local" && "$source_tier" != "live-readonly" ]]; then
|
|
echo "--source-tier must be isolated-local or live-readonly" >&2
|
|
exit 2
|
|
fi
|
|
|
|
source_network_mode="$(docker inspect -f '{{.HostConfig.NetworkMode}}' "$source_container")"
|
|
source_canary_label="$(
|
|
docker inspect -f '{{index .Config.Labels "livingip.canary"}}' "$source_container"
|
|
)"
|
|
if [[ "$source_tier" == "isolated-local" ]]; then
|
|
if [[ "$source_network_mode" != "none" || "$source_canary_label" != "working-leo-approved-source" ]]; then
|
|
echo "isolated-local source must be networkless and labeled livingip.canary=working-leo-approved-source" >&2
|
|
exit 2
|
|
fi
|
|
fi
|
|
|
|
container="working-leo-approve-claim-$RANDOM-$$"
|
|
workdir="/tmp/${container}"
|
|
apply_password="isolated-apply-${RANDOM}-${RANDOM}"
|
|
review_password="isolated-review-${RANDOM}-${RANDOM}"
|
|
|
|
cleanup() {
|
|
docker rm -f "$container" >/dev/null 2>&1 || true
|
|
rm -rf "$workdir"
|
|
}
|
|
trap cleanup EXIT
|
|
|
|
source_counts() {
|
|
docker exec "$source_container" psql -U postgres -d "$source_db" -Atq \
|
|
-v ON_ERROR_STOP=1 -c "begin transaction read only;
|
|
select jsonb_build_object(
|
|
'public.claims',(select count(*) from public.claims),
|
|
'public.sources',(select count(*) from public.sources),
|
|
'public.claim_evidence',(select count(*) from public.claim_evidence),
|
|
'public.claim_edges',(select count(*) from public.claim_edges),
|
|
'public.reasoning_tools',(select count(*) from public.reasoning_tools),
|
|
'kb_stage.kb_proposals',(select count(*) from kb_stage.kb_proposals)
|
|
)::text;
|
|
rollback;"
|
|
}
|
|
|
|
service_state() {
|
|
if ! command -v systemctl >/dev/null 2>&1; then
|
|
printf 'Available=false\nReason=systemctl_unavailable\n'
|
|
return 0
|
|
fi
|
|
local output
|
|
if output="$(systemctl show "$service" --no-pager \
|
|
-p ActiveState -p SubState -p MainPID -p NRestarts -p ExecMainStartTimestamp 2>&1)"; then
|
|
printf 'Available=true\n%s\n' "$output"
|
|
else
|
|
printf 'Available=false\nReason=service_readback_failed\n'
|
|
fi
|
|
}
|
|
|
|
mkdir -p "$workdir" "$(dirname "$output")"
|
|
source_counts_before="$(source_counts)"
|
|
service_before="$(service_state)"
|
|
|
|
docker run -d --name "$container" \
|
|
--network none \
|
|
--tmpfs /var/lib/postgresql/data:rw,nosuid,size=512m \
|
|
-e POSTGRES_PASSWORD="isolated-postgres-${RANDOM}-${RANDOM}" \
|
|
-e POSTGRES_DB=teleo \
|
|
"$image" >/dev/null
|
|
|
|
ready_streak=0
|
|
for _ in $(seq 1 80); do
|
|
if docker exec "$container" pg_isready -U postgres -d teleo >/dev/null 2>&1; then
|
|
ready_streak=$((ready_streak + 1))
|
|
if [[ "$ready_streak" -ge 2 ]]; then
|
|
break
|
|
fi
|
|
else
|
|
ready_streak=0
|
|
fi
|
|
sleep 0.25
|
|
done
|
|
if [[ "$ready_streak" -lt 2 ]]; then
|
|
echo "disposable Postgres did not become stably ready" >&2
|
|
exit 1
|
|
fi
|
|
container_network_mode="$(docker inspect -f '{{.HostConfig.NetworkMode}}' "$container")"
|
|
container_mounts_json="$(docker inspect -f '{{json .Mounts}}' "$container")"
|
|
|
|
docker exec "$source_container" pg_dump -U postgres -d "$source_db" \
|
|
--schema-only --no-owner --no-privileges >"$workdir/schema.sql"
|
|
docker exec "$source_container" pg_dump -U postgres -d "$source_db" \
|
|
--data-only --no-owner --no-privileges \
|
|
--table=public.agents --table=public.claims >"$workdir/reference-data.sql"
|
|
docker cp "$workdir/schema.sql" "$container:/tmp/schema.sql"
|
|
docker cp "$workdir/reference-data.sql" "$container:/tmp/reference-data.sql"
|
|
|
|
docker exec -i "$container" psql -U postgres -d postgres -v ON_ERROR_STOP=1 -q <<SQL
|
|
create role kb_apply login noinherit nosuperuser nocreatedb nocreaterole
|
|
noreplication nobypassrls password '${apply_password}';
|
|
create role kb_review login noinherit nosuperuser nocreatedb nocreaterole
|
|
noreplication nobypassrls password '${review_password}';
|
|
create role kb_gate_owner nologin noinherit nosuperuser nocreatedb nocreaterole
|
|
noreplication nobypassrls;
|
|
SQL
|
|
docker exec "$container" psql -U postgres -d teleo -v ON_ERROR_STOP=1 -q \
|
|
-f /tmp/schema.sql
|
|
docker exec "$container" psql -U postgres -d teleo -v ON_ERROR_STOP=1 -q \
|
|
-f /tmp/reference-data.sql
|
|
docker exec "$container" psql -U postgres -d teleo -v ON_ERROR_STOP=1 -q \
|
|
-c "insert into public.agents (id, handle, kind)
|
|
values ('99999999-9999-9999-9999-999999999999', 'm3ta', 'human')
|
|
on conflict (handle) do nothing"
|
|
|
|
cat >"$workdir/kb-apply.env" <<EOF
|
|
KB_APPLY_PASSWORD=${apply_password}
|
|
EOF
|
|
cat >"$workdir/kb-review.env" <<EOF
|
|
KB_REVIEW_PASSWORD=${review_password}
|
|
EOF
|
|
chmod 600 "$workdir/kb-apply.env" "$workdir/kb-review.env"
|
|
|
|
normalization_args=()
|
|
if [[ -n "$normalization_json" ]]; then
|
|
normalization_args=(--normalization-json "$normalization_json")
|
|
fi
|
|
|
|
set +e
|
|
python3 "$repo_root/scripts/run_approve_claim_clone_canary.py" \
|
|
--container "$container" \
|
|
--source-db teleo \
|
|
--database-prefix teleo_approve_claim_isolated \
|
|
--service "$service" \
|
|
--approve-script "$repo_root/scripts/approve_proposal.py" \
|
|
--apply-script "$repo_root/scripts/apply_proposal.py" \
|
|
--prereqs "$repo_root/scripts/kb_apply_prereqs.sql" \
|
|
--review-secrets-file "$workdir/kb-review.env" \
|
|
--secrets-file "$workdir/kb-apply.env" \
|
|
"${normalization_args[@]}" \
|
|
--output "$output" \
|
|
>"$workdir/stdout.json" 2>"$workdir/stderr.log"
|
|
canary_rc=$?
|
|
set -e
|
|
|
|
set +e
|
|
docker rm -f "$container" >/dev/null 2>&1
|
|
outer_container_remove_rc=$?
|
|
rm -rf "$workdir"
|
|
outer_workdir_remove_rc=$?
|
|
container_name_readback="$(
|
|
docker container ls -a --filter "name=^/${container}$" --format '{{.Names}}' 2>&1
|
|
)"
|
|
container_name_readback_rc=$?
|
|
if [[ -e "$workdir" || -L "$workdir" ]]; then
|
|
outer_workdir_lexists=true
|
|
else
|
|
outer_workdir_lexists=false
|
|
fi
|
|
set -e
|
|
trap - EXIT
|
|
|
|
source_counts_after="$(source_counts)"
|
|
service_after="$(service_state)"
|
|
|
|
SOURCE_COUNTS_BEFORE="$source_counts_before" \
|
|
SOURCE_COUNTS_AFTER="$source_counts_after" \
|
|
SOURCE_CONTAINER="$source_container" \
|
|
SOURCE_DB="$source_db" \
|
|
SERVICE_BEFORE="$service_before" \
|
|
SERVICE_AFTER="$service_after" \
|
|
REPO_ROOT="$repo_root" \
|
|
OUTER_CONTAINER_REMOVE_RC="$outer_container_remove_rc" \
|
|
OUTER_WORKDIR_REMOVE_RC="$outer_workdir_remove_rc" \
|
|
CONTAINER_NAME_READBACK="$container_name_readback" \
|
|
CONTAINER_NAME_READBACK_RC="$container_name_readback_rc" \
|
|
OUTER_WORKDIR_LEXISTS="$outer_workdir_lexists" \
|
|
CANARY_RC="$canary_rc" \
|
|
SOURCE_TIER="$source_tier" \
|
|
SOURCE_NETWORK_MODE="$source_network_mode" \
|
|
SOURCE_CANARY_LABEL="$source_canary_label" \
|
|
CANARY_NETWORK_MODE="$container_network_mode" \
|
|
CANARY_MOUNTS_JSON="$container_mounts_json" \
|
|
python3 - "$output" <<'PY'
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
path = Path(sys.argv[1])
|
|
if not path.is_file():
|
|
raise SystemExit("inner canary did not produce a report")
|
|
|
|
data = json.loads(path.read_text(encoding="utf-8"))
|
|
|
|
def service(raw: str) -> dict[str, str]:
|
|
return dict(line.split("=", 1) for line in raw.splitlines() if "=" in line)
|
|
|
|
def table_deltas(before: dict[str, int], after: dict[str, int]) -> dict[str, int]:
|
|
if set(before) != set(after):
|
|
raise SystemExit(
|
|
"live source count keys changed during the canary: "
|
|
f"before={sorted(before)} after={sorted(after)}"
|
|
)
|
|
return {key: after[key] - before[key] for key in sorted(before)}
|
|
|
|
def verify_source_binding(root: Path, records: list[dict[str, object]]) -> dict[str, object]:
|
|
readbacks = []
|
|
for record in records:
|
|
relative_text = str(record.get("repo_relative_path", ""))
|
|
relative = Path(relative_text)
|
|
if not relative_text or relative.is_absolute() or ".." in relative.parts:
|
|
raise SystemExit(f"unsafe repo-relative source path: {relative_text!r}")
|
|
candidate = (root / relative).resolve()
|
|
try:
|
|
candidate.relative_to(root)
|
|
except ValueError as exc:
|
|
raise SystemExit(
|
|
f"source path escaped repository root: {relative_text}"
|
|
) from exc
|
|
if not candidate.is_file():
|
|
readbacks.append(
|
|
{
|
|
"repo_relative_path": relative.as_posix(),
|
|
"expected_sha256": record.get("sha256"),
|
|
"actual_sha256": None,
|
|
"matches": False,
|
|
}
|
|
)
|
|
continue
|
|
content = candidate.read_bytes()
|
|
actual_sha256 = hashlib.sha256(content).hexdigest()
|
|
readbacks.append(
|
|
{
|
|
"repo_relative_path": relative.as_posix(),
|
|
"expected_sha256": record.get("sha256"),
|
|
"actual_sha256": actual_sha256,
|
|
"size_bytes": len(content),
|
|
"matches": (
|
|
actual_sha256 == record.get("sha256")
|
|
and len(content) == record.get("size_bytes")
|
|
),
|
|
}
|
|
)
|
|
return {
|
|
"files": readbacks,
|
|
"all_match": bool(readbacks) and all(row["matches"] for row in readbacks),
|
|
}
|
|
|
|
def contains_ephemeral_path(value: object) -> bool:
|
|
if isinstance(value, str):
|
|
return "/tmp/" in value
|
|
if isinstance(value, dict):
|
|
return any(contains_ephemeral_path(item) for item in value.values())
|
|
if isinstance(value, list):
|
|
return any(contains_ephemeral_path(item) for item in value)
|
|
return False
|
|
|
|
before_counts = json.loads(os.environ["SOURCE_COUNTS_BEFORE"])
|
|
after_counts = json.loads(os.environ["SOURCE_COUNTS_AFTER"])
|
|
live_table_deltas = table_deltas(before_counts, after_counts)
|
|
before_service = service(os.environ["SERVICE_BEFORE"])
|
|
after_service = service(os.environ["SERVICE_AFTER"])
|
|
source_tier = os.environ["SOURCE_TIER"]
|
|
source_network_mode = os.environ["SOURCE_NETWORK_MODE"]
|
|
source_canary_label = os.environ["SOURCE_CANARY_LABEL"]
|
|
container_network_mode = os.environ["CANARY_NETWORK_MODE"]
|
|
container_mounts = json.loads(os.environ["CANARY_MOUNTS_JSON"])
|
|
container_volume_mounts = [
|
|
mount for mount in container_mounts if mount.get("Type") == "volume"
|
|
]
|
|
container_isolated = (
|
|
container_network_mode == "none" and not container_volume_mounts
|
|
)
|
|
source_tier_enforced = (
|
|
source_tier != "isolated-local"
|
|
or (
|
|
source_network_mode == "none"
|
|
and source_canary_label == "working-leo-approved-source"
|
|
)
|
|
)
|
|
service_observable = (
|
|
before_service.get("Available") == "true"
|
|
and after_service.get("Available") == "true"
|
|
)
|
|
service_stable = (
|
|
all(
|
|
before_service.get(key) == after_service.get(key)
|
|
for key in ("ActiveState", "SubState", "MainPID", "NRestarts")
|
|
)
|
|
if service_observable
|
|
else None
|
|
)
|
|
service_gate_passes = (
|
|
service_stable is True
|
|
if source_tier == "live-readonly"
|
|
else service_stable is not False
|
|
)
|
|
container_absent = (
|
|
int(os.environ["CONTAINER_NAME_READBACK_RC"]) == 0
|
|
and not os.environ["CONTAINER_NAME_READBACK"].strip()
|
|
)
|
|
workdir_absent = os.environ["OUTER_WORKDIR_LEXISTS"] == "false"
|
|
source_binding = verify_source_binding(
|
|
Path(os.environ["REPO_ROOT"]).resolve(),
|
|
data.get("source_binding", {}).get("files", []),
|
|
)
|
|
data["source_binding"]["independent_post_cleanup_verification"] = source_binding
|
|
source_binding_ok = (
|
|
data["source_binding"].get("unchanged_during_run") is True
|
|
and source_binding["all_match"] is True
|
|
)
|
|
stale_ephemeral_paths_absent = not contains_ephemeral_path(data)
|
|
inner_runtime_pass = data.get("status") == "pass" and int(os.environ["CANARY_RC"]) == 0
|
|
outer_ok = (
|
|
before_counts == after_counts
|
|
and all(delta == 0 for delta in live_table_deltas.values())
|
|
and service_gate_passes
|
|
and container_absent
|
|
and workdir_absent
|
|
and source_binding_ok
|
|
and stale_ephemeral_paths_absent
|
|
and container_isolated
|
|
and source_tier_enforced
|
|
)
|
|
if source_tier == "isolated-local":
|
|
data["required_tier"] = "T2_runtime"
|
|
data["required_tiers"] = ["T2_runtime"]
|
|
data["runtime_scope"] = "isolated_postgres_container_from_readonly_local_fixture"
|
|
else:
|
|
data["required_tier"] = "T3_live_readonly"
|
|
data["required_tiers"] = ["T2_runtime", "T3_live_readonly"]
|
|
data["runtime_scope"] = "isolated_postgres_container_from_readonly_live_schema"
|
|
data["outer_isolation"] = {
|
|
"source_container": os.environ.get("SOURCE_CONTAINER", "teleo-pg"),
|
|
"source_database": os.environ.get("SOURCE_DB", "teleo"),
|
|
"source_counts_before": before_counts,
|
|
"source_counts_after": after_counts,
|
|
"canonical_table_deltas": live_table_deltas,
|
|
"source_counts_unchanged": before_counts == after_counts,
|
|
"service_before": before_service,
|
|
"service_after": after_service,
|
|
"service_required": source_tier == "live-readonly",
|
|
"service_observable": service_observable,
|
|
"service_stable": service_stable,
|
|
"service_gate_passes": service_gate_passes,
|
|
"source_boundary": {
|
|
"source_tier": source_tier,
|
|
"network_mode": source_network_mode,
|
|
"canary_label": source_canary_label,
|
|
"tier_enforced": source_tier_enforced,
|
|
},
|
|
"container_isolation": {
|
|
"network_mode": container_network_mode,
|
|
"volume_mounts": container_volume_mounts,
|
|
"network_disabled": container_network_mode == "none",
|
|
"no_docker_volumes": not container_volume_mounts,
|
|
},
|
|
"cleanup_readback": {
|
|
"container_remove_returncode": int(os.environ["OUTER_CONTAINER_REMOVE_RC"]),
|
|
"container_name_query_returncode": int(
|
|
os.environ["CONTAINER_NAME_READBACK_RC"]
|
|
),
|
|
"container_name_query_stdout": os.environ["CONTAINER_NAME_READBACK"],
|
|
"temporary_container_absent": container_absent,
|
|
"workdir_remove_returncode": int(os.environ["OUTER_WORKDIR_REMOVE_RC"]),
|
|
"workdir_lexists_after_cleanup": not workdir_absent,
|
|
"temporary_workdir_absent": workdir_absent,
|
|
},
|
|
}
|
|
data["checks"]["outer_live_counts_exactly_unchanged"] = before_counts == after_counts
|
|
data["checks"]["outer_live_table_deltas_exactly_zero"] = all(
|
|
delta == 0 for delta in live_table_deltas.values()
|
|
)
|
|
data["checks"]["outer_service_observed_if_required"] = (
|
|
source_tier != "live-readonly" or service_observable
|
|
)
|
|
data["checks"]["outer_service_stable_if_observed"] = service_gate_passes
|
|
data["checks"]["outer_container_absent_independent_readback"] = container_absent
|
|
data["checks"]["outer_workdir_absent_independent_readback"] = workdir_absent
|
|
data["checks"]["source_binding_independently_verified"] = source_binding_ok
|
|
data["checks"]["stale_ephemeral_paths_absent"] = stale_ephemeral_paths_absent
|
|
data["checks"]["outer_source_tier_enforced"] = source_tier_enforced
|
|
data["checks"]["outer_container_network_disabled"] = (
|
|
container_network_mode == "none"
|
|
)
|
|
data["checks"]["outer_container_has_no_docker_volumes"] = not container_volume_mounts
|
|
data["checks"]["outer_isolation_complete"] = outer_ok
|
|
if outer_ok and inner_runtime_pass:
|
|
data["current_tier"] = (
|
|
"T2_runtime" if source_tier == "isolated-local" else "T3_live_readonly"
|
|
)
|
|
else:
|
|
data["status"] = "fail"
|
|
data["current_tier"] = "T2_runtime" if inner_runtime_pass else "T1_model"
|
|
path.write_text(json.dumps(data, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
|
raise SystemExit(0 if data["status"] == "pass" else 1)
|
|
PY
|