#!/usr/bin/python3 -I
"""Execute one root-owned, fixed-target leoclean no-send IAP request."""

from __future__ import annotations

import fcntl
import hashlib
import io
import json
import os
import pwd
import re
import shutil
import stat
import subprocess
import sys
import tarfile
import tempfile
import time
import zipfile
from collections.abc import Callable, Mapping
from pathlib import Path
from typing import Any

SCHEMA = "livingip.leocleanNoSendIapRequest.v1"
RESULT_SCHEMA = "livingip.leocleanNoSendIapRemoteResult.v2"
FAILURE_RESULT_SCHEMA = "livingip.leocleanNoSendIapFailure.v2"
OPERATION_FAILURE_RESULT_SCHEMA = "livingip.leocleanNoSendIapOperationalFailure.v2"
PROJECT = "teleo-501523"
ZONE = "europe-west6-a"
INSTANCE = "teleo-staging-1"
SERVICE = "leoclean-gcp-nosend.service"
SERVICE_ACCOUNT = "sa-teleo-staging-vm@teleo-501523.iam.gserviceaccount.com"
IMAGE_REPOSITORY = "europe-west6-docker.pkg.dev/teleo-501523/teleo/leoclean-nosend-staging"
DOCKER = "/usr/bin/docker"
DOCKER_HOST = "unix:///var/run/docker.sock"
SERVICE_DOCKER_CONFIG = "/var/lib/livingip/leoclean-nosend/docker-config"
METADATA = "http://metadata.google.internal/computeMetadata/v1"
RUN_ROOT = Path("/var/lib/livingip/leoclean-nosend-iap/requests")
LOCK_PATH = Path("/run/leoclean-gcp-nosend-iap.lock")
UPLOAD_ROOT = Path("/tmp")
STAGE_ROOT = Path("/run/livingip-leoclean-nosend-iap-bootstrap")
REQUEST_ID = re.compile(r"^iap-[a-z0-9]{12,32}$")
HEX_40 = re.compile(r"^[0-9a-f]{40}$")
HEX_64 = re.compile(r"^[0-9a-f]{64}$")
IMAGE_REFERENCE = re.compile(rf"^{re.escape(IMAGE_REPOSITORY)}@sha256:([0-9a-f]{{64}})$")
OPERATIONS = ("install", "verify", "rollback")
MAX_ARCHIVE_BYTES = 8 * 1024 * 1024
MAX_MEMBER_BYTES = 2 * 1024 * 1024
MAX_MANIFEST_BYTES = 128 * 1024
MAX_COMMAND_OUTPUT_BYTES = 1024 * 1024
MAX_TOKEN_BYTES = 64 * 1024
POST_INSTALL_FAILURES = frozenset(
    {
        ("install_receipt", "install_receipt_invalid"),
        ("service_before_restart", "service_verification_failed"),
        ("database_before_restart", "database_verification_failed"),
        ("service_restart", "service_restart_failed"),
        ("service_after_restart", "service_verification_failed"),
        ("restart_identity", "restart_identity_mismatch"),
        ("database_after_restart", "database_verification_failed"),
    }
)
EXPECTED_MEMBERS = {
    "request-manifest.json",
    "release-artifact.zip",
    "release/release.json",
    f"release/{SERVICE}",
    "ops/gcp_leoclean_nosend_package.py",
    "ops/install_gcp_leoclean_nosend_release.py",
    "ops/leoclean_nosend_service_control.py",
    "ops/gcp_leoclean_nosend_iap_remote.py",
    "ops/run_gcp_leoclean_nosend_iap.py",
    "scripts/leo_behavior_manifest.py",
    "scripts/leo_identity_manifest.py",
}
RELEASE_ARTIFACT_MEMBERS = {
    "evidence/oci-smoke.json",
    "evidence/publish-outcome.json",
    "evidence/runtime-receipt.json",
    f"final/release-v3/{SERVICE}",
    "final/release-v3/release.json",
    "receipts/build-push-receipt.json",
}
SAFE_PATH = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"

Runner = Callable[..., subprocess.CompletedProcess[bytes]]


class RemoteError(RuntimeError):
    """A remote request failed closed without exposing command output."""


class RemoteOperationFailure(RemoteError):
    """A sanitized, request-bound failure that can be returned to the caller."""

    def __init__(self, result: Mapping[str, Any]) -> None:
        super().__init__("remote_operation_failed")
        self.result = dict(result)


class PostureRecoveryFailure(RemoteError):
    """A posture recovery failed after a bounded amount of service action."""

    def __init__(self, *, executed: bool) -> None:
        super().__init__("posture_recovery_failed")
        self.executed = executed


class ExplicitRollbackFailure(RemoteError):
    """An explicit rollback failed before or after entering its mutation phase."""

    def __init__(self, *, executed: bool) -> None:
        super().__init__("explicit_rollback_failed")
        self.executed = executed


def _validate_installer_transition(value: Any) -> dict[str, Any]:
    _require(
        isinstance(value, dict)
        and set(value)
        == {
            "outcome",
            "prior_posture",
            "prior_n_restarts",
            "final_posture",
            "service_action",
            "managed_files_replaced",
        },
        "installer transition was invalid",
    )
    outcome = value.get("outcome")
    prior_posture = value.get("prior_posture")
    prior_n_restarts = value.get("prior_n_restarts")
    service_action = value.get("service_action")
    replaced = value.get("managed_files_replaced")
    _require(
        isinstance(prior_n_restarts, int)
        and not isinstance(prior_n_restarts, bool)
        and prior_n_restarts >= 0
        and value.get("final_posture") == "active"
        and (
            (outcome, prior_posture, service_action, replaced)
            == ("verified_noop", "active", "none", False)
            or (outcome, prior_posture, service_action, replaced)
            == ("controlled_start", "inactive", "start", False)
            or (outcome, prior_posture, service_action, replaced)
            == ("installed", "absent", "restart", True)
            or (
                outcome == "replaced"
                and prior_posture in {"active", "inactive"}
                and service_action == "restart"
                and replaced is True
            )
        ),
        "installer transition was invalid",
    )
    return dict(value)


def _validate_install_execution_result(
    value: Any,
    *,
    release: Mapping[str, Any],
    installer_module: Any,
) -> dict[str, Any]:
    _require(
        isinstance(value, dict)
        and set(value)
        == {
            "schema",
            "status",
            "outcome",
            "release_sha256",
            "persisted_receipt",
            "transition",
            "verification",
        },
        "install execution result was invalid",
    )
    transition = _validate_installer_transition(value.get("transition"))
    persisted_receipt = value.get("persisted_receipt")
    outcome = value.get("outcome")
    _require(
        value.get("schema") == installer_module.INSTALL_EXECUTION_RESULT_SCHEMA
        and value.get("status") == "pass"
        and outcome == transition["outcome"]
        and value.get("release_sha256") == release["release_sha256"]
        and isinstance(persisted_receipt, dict)
        and set(persisted_receipt) == {"schema", "sha256"}
        and persisted_receipt.get("schema")
        in {
            installer_module.LEGACY_INSTALL_RECEIPT_SCHEMA,
            installer_module.INSTALL_RECEIPT_SCHEMA,
        }
        and isinstance(persisted_receipt.get("sha256"), str)
        and HEX_64.fullmatch(persisted_receipt["sha256"]) is not None
        and isinstance(value.get("verification"), dict)
        and (
            outcome in {"verified_noop", "controlled_start"}
            or (
                outcome in {"installed", "replaced"}
                and persisted_receipt["schema"] == installer_module.INSTALL_RECEIPT_SCHEMA
            )
        ),
        "install execution result was invalid",
    )
    return transition


def _rollback_summary(
    *,
    status: str,
    mode: str,
    executed: bool,
    restored_posture: str,
    prior_posture_preserved: bool,
) -> dict[str, Any]:
    value = {
        "status": status,
        "mode": mode,
        "executed": executed,
        "restored_posture": restored_posture,
        "prior_posture_preserved": prior_posture_preserved,
    }
    _require(
        (
            status == "pass"
            and (
                (
                    mode in {"installer_automatic", "explicit"}
                    and executed is True
                    and restored_posture in {"active", "inactive", "absent"}
                )
                or (
                    mode == "posture_recovery"
                    and restored_posture in {"active", "inactive"}
                )
            )
            and prior_posture_preserved is True
        )
        or (
            status == "not_run"
            and mode == "not_applicable"
            and executed is False
            and restored_posture == "unchanged"
            and prior_posture_preserved is True
        )
        or (
            status == "fail"
            and mode in {"explicit", "posture_recovery"}
            and restored_posture == "unknown"
            and prior_posture_preserved is False
        ),
        "rollback summary was invalid",
    )
    return value


def _require(condition: bool, message: str) -> None:
    if not condition:
        raise RemoteError(message)


def _sha256_bytes(value: bytes) -> str:
    return hashlib.sha256(value).hexdigest()


def _canonical_sha256(value: Mapping[str, Any]) -> str:
    try:
        encoded = json.dumps(
            value,
            allow_nan=False,
            ensure_ascii=True,
            separators=(",", ":"),
            sort_keys=True,
        ).encode("utf-8")
    except (TypeError, ValueError) as exc:
        raise RemoteError("request manifest could not be hashed") from exc
    return _sha256_bytes(encoded)


def _strict_json(raw: bytes, label: str, max_bytes: int) -> dict[str, Any]:
    _require(len(raw) <= max_bytes, f"{label} exceeded the safety limit")

    def reject_duplicates(pairs: list[tuple[str, Any]]) -> dict[str, Any]:
        value: dict[str, Any] = {}
        for key, item in pairs:
            if key in value:
                raise ValueError("duplicate key")
            value[key] = item
        return value

    try:
        value = json.loads(
            raw.decode("utf-8", "strict"),
            object_pairs_hook=reject_duplicates,
            parse_constant=lambda _value: (_ for _ in ()).throw(ValueError("constant")),
        )
    except (UnicodeError, ValueError, json.JSONDecodeError) as exc:
        raise RemoteError(f"{label} was invalid JSON") from exc
    _require(isinstance(value, dict), f"{label} was not a JSON object")
    return value


def _run(
    argv: list[str],
    *,
    runner: Runner,
    label: str,
    input_bytes: bytes | None = None,
    timeout: int = 120,
    require_success: bool = True,
) -> subprocess.CompletedProcess[bytes]:
    try:
        completed = runner(
            argv,
            check=False,
            stdin=subprocess.DEVNULL if input_bytes is None else None,
            input=input_bytes,
            capture_output=True,
            timeout=timeout,
            env={"PATH": SAFE_PATH},
        )
    except (OSError, subprocess.SubprocessError) as exc:
        raise RemoteError(f"{label} failed") from exc
    _require(
        len(completed.stdout) <= MAX_COMMAND_OUTPUT_BYTES and len(completed.stderr) <= MAX_COMMAND_OUTPUT_BYTES,
        f"{label} output exceeded the safety limit",
    )
    if require_success:
        _require(completed.returncode == 0, f"{label} failed")
    return completed


def _read_stable_stage(path: Path, expected_sha256: str, max_bytes: int) -> bytes:
    _require(path.parent.parent == STAGE_ROOT, "staged request path escaped its fixed root")
    flags = os.O_RDONLY
    if hasattr(os, "O_CLOEXEC"):
        flags |= os.O_CLOEXEC
    if hasattr(os, "O_NOFOLLOW"):
        flags |= os.O_NOFOLLOW
    try:
        descriptor = os.open(path, flags)
    except OSError as exc:
        raise RemoteError("request upload was missing or unsafe") from exc
    try:
        before = os.fstat(descriptor)
        _require(
            stat.S_ISREG(before.st_mode)
            and before.st_uid == 0
            and stat.S_IMODE(before.st_mode) == 0o600
            and before.st_nlink == 1
            and before.st_size <= max_bytes,
            "request upload posture was unsafe",
        )
        chunks: list[bytes] = []
        remaining = max_bytes + 1
        while remaining:
            chunk = os.read(descriptor, min(65536, remaining))
            if not chunk:
                break
            chunks.append(chunk)
            remaining -= len(chunk)
        content = b"".join(chunks)
        after = os.fstat(descriptor)
        named = path.stat(follow_symlinks=False)
        _require(
            len(content) == after.st_size
            and (before.st_dev, before.st_ino, before.st_mode, before.st_nlink, before.st_size, before.st_mtime_ns)
            == (after.st_dev, after.st_ino, after.st_mode, after.st_nlink, after.st_size, after.st_mtime_ns)
            and (after.st_dev, after.st_ino, after.st_mode, after.st_nlink)
            == (named.st_dev, named.st_ino, named.st_mode, named.st_nlink)
            and _sha256_bytes(content) == expected_sha256,
            "request upload changed or differed from its hash",
        )
        return content
    except OSError as exc:
        raise RemoteError("request upload could not be read") from exc
    finally:
        os.close(descriptor)


def _unlink_exact(path: Path, identity: tuple[int, int]) -> None:
    observed = path.stat(follow_symlinks=False)
    _require((observed.st_dev, observed.st_ino) == identity and stat.S_ISREG(observed.st_mode), "upload was replaced")
    path.unlink()


def _ensure_root_directory(
    path: Path,
    mode: int,
    *,
    expected_uid: int = 0,
    trusted_parent: Path = Path("/var/lib"),
) -> None:
    _require(path.is_absolute() and path.is_relative_to(trusted_parent), "request directory escaped its fixed root")
    missing: list[Path] = []
    cursor = path
    while not cursor.exists():
        _require(not cursor.is_symlink(), "request directory path was unsafe")
        missing.append(cursor)
        cursor = cursor.parent
    observed = cursor.stat(follow_symlinks=False)
    _require(stat.S_ISDIR(observed.st_mode), "request directory parent was unsafe")
    for directory in reversed(missing):
        directory.mkdir(mode=mode if directory == path else 0o755)
        os.chown(directory, expected_uid, os.getgid() if expected_uid != 0 else 0)
    observed = path.stat(follow_symlinks=False)
    _require(
        stat.S_ISDIR(observed.st_mode) and observed.st_uid == expected_uid and observed.st_mode & 0o022 == 0,
        "request directory was unsafe",
    )
    if path in missing:
        path.chmod(mode)
    cursor = path
    while True:
        observed = cursor.stat(follow_symlinks=False)
        _require(
            stat.S_ISDIR(observed.st_mode) and observed.st_uid == expected_uid and observed.st_mode & 0o022 == 0,
            "request directory ancestor was unsafe",
        )
        if cursor == trusted_parent:
            break
        cursor = cursor.parent


def _acquire_lock(path: Path = LOCK_PATH, *, expected_uid: int = 0) -> int:
    flags = os.O_RDWR | os.O_CREAT
    if hasattr(os, "O_CLOEXEC"):
        flags |= os.O_CLOEXEC
    if hasattr(os, "O_NOFOLLOW"):
        flags |= os.O_NOFOLLOW
    try:
        descriptor = os.open(path, flags, 0o600)
        observed = os.fstat(descriptor)
        named = path.stat(follow_symlinks=False)
        _require(
            stat.S_ISREG(observed.st_mode)
            and observed.st_uid == expected_uid
            and stat.S_IMODE(observed.st_mode) == 0o600
            and observed.st_nlink == 1
            and (observed.st_dev, observed.st_ino) == (named.st_dev, named.st_ino),
            "request lock was unsafe",
        )
        fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB)
        return descriptor
    except (OSError, RemoteError) as exc:
        if "descriptor" in locals():
            os.close(descriptor)
        raise RemoteError("another request is active or the request lock was unsafe") from exc


def _copy_into_request(path: Path, content: bytes, mode: int = 0o600) -> None:
    _require(mode in {0o600, 0o644, 0o755}, "request file mode was invalid")
    flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL
    if hasattr(os, "O_NOFOLLOW"):
        flags |= os.O_NOFOLLOW
    try:
        descriptor = os.open(path, flags, mode)
        with os.fdopen(descriptor, "wb") as handle:
            # receive_request() deliberately runs under umask 077. Apply the
            # manifest-bound mode explicitly so public release metadata and
            # executable helpers retain their reviewed 0644/0755 posture.
            os.fchmod(handle.fileno(), mode)
            handle.write(content)
            handle.flush()
            os.fsync(handle.fileno())
            observed = os.fstat(handle.fileno())
            named = path.stat(follow_symlinks=False)
            _require(
                stat.S_ISREG(observed.st_mode)
                and (observed.st_dev, observed.st_ino) == (named.st_dev, named.st_ino)
                and observed.st_uid == os.geteuid()
                and stat.S_IMODE(observed.st_mode) == mode
                and stat.S_IMODE(named.st_mode) == mode
                and observed.st_nlink == 1
                and observed.st_size == len(content),
                "request file posture differed from its manifest",
            )
    except OSError as exc:
        raise RemoteError("request file could not be secured") from exc


def _claim_request_directory(
    run_root: Path,
    request_id: str,
    *,
    expected_uid: int = 0,
    trusted_parent: Path = Path("/var/lib"),
) -> Path:
    _require(REQUEST_ID.fullmatch(request_id) is not None, "request ID was invalid")
    _ensure_root_directory(run_root, 0o700, expected_uid=expected_uid, trusted_parent=trusted_parent)
    request_dir = run_root / request_id
    _require(not request_dir.exists() and not request_dir.is_symlink(), "request ID was already used")
    request_dir.mkdir(mode=0o700)
    os.chown(request_dir, expected_uid, os.getgid() if expected_uid != 0 else 0)
    return request_dir


def _cleanup_upload_if_exact(path: Path, identity: tuple[int, int]) -> None:
    try:
        observed = path.stat(follow_symlinks=False)
        if stat.S_ISREG(observed.st_mode) and (observed.st_dev, observed.st_ino) == identity:
            path.unlink()
    except OSError:
        pass


def _validate_member_name(name: str) -> None:
    path = Path(name)
    _require(
        name
        and not name.startswith("/")
        and not name.endswith("/")
        and all(part not in {"", ".", ".."} for part in path.parts),
        "archive member path was unsafe",
    )


def extract_archive(archive_path: Path, destination: Path) -> dict[str, Any]:
    _require(archive_path.stat().st_size <= MAX_ARCHIVE_BYTES, "request archive exceeded the safety limit")
    try:
        with tarfile.open(archive_path, mode="r:gz") as archive:
            return _extract_open_archive(archive, destination)
    except (OSError, tarfile.TarError) as exc:
        raise RemoteError("request archive was invalid") from exc


def _extract_open_archive(archive: tarfile.TarFile, destination: Path) -> dict[str, Any]:
    members = archive.getmembers()
    names = [member.name for member in members]
    _require(len(names) == len(set(names)) and set(names) == EXPECTED_MEMBERS, "archive member set was not exact")
    for member in members:
        _validate_member_name(member.name)
        _require(
            member.isfile()
            and member.size <= MAX_MEMBER_BYTES
            and member.uid == 0
            and member.gid == 0
            and member.uname == "root"
            and member.gname == "root"
            and member.mtime == 0
            and stat.S_IMODE(member.mode) in {0o600, 0o644, 0o755},
            "archive member metadata was unsafe",
        )
    manifest_member = archive.getmember("request-manifest.json")
    _require(manifest_member.size <= MAX_MANIFEST_BYTES, "request manifest exceeded the safety limit")
    manifest_stream = archive.extractfile(manifest_member)
    _require(manifest_stream is not None, "request manifest was absent")
    manifest = _strict_json(manifest_stream.read(), "request manifest", MAX_MANIFEST_BYTES)
    files = manifest.get("files")
    _require(isinstance(files, list), "request manifest file list was invalid")
    file_map: dict[str, dict[str, Any]] = {}
    for item in files:
        _require(
            isinstance(item, dict) and set(item) == {"path", "sha256", "size", "mode"},
            "request manifest file record was invalid",
        )
        relative = item.get("path")
        _require(isinstance(relative, str) and relative not in file_map, "request manifest file path was invalid")
        file_map[relative] = item
    _require(set(file_map) == EXPECTED_MEMBERS - {"request-manifest.json"}, "request manifest files were not exact")
    for relative in sorted(file_map):
        member = archive.getmember(relative)
        stream = archive.extractfile(member)
        _require(stream is not None, "request member was absent")
        content = stream.read(MAX_MEMBER_BYTES + 1)
        record = file_map[relative]
        _require(
            len(content) == member.size == record.get("size")
            and stat.S_IMODE(member.mode) == record.get("mode")
            and _sha256_bytes(content) == record.get("sha256"),
            "request member differed from its manifest",
        )
        target = destination / relative
        target.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
        _require(target.resolve().is_relative_to(destination.resolve()), "request member escaped extraction root")
        _copy_into_request(target, content, stat.S_IMODE(member.mode))
    return manifest


def validate_manifest(
    manifest: Mapping[str, Any],
    *,
    operation: str,
    request_id: str,
    runner_revision: str,
    release_revision: str,
) -> None:
    _require(
        set(manifest)
        == {
            "schema",
            "operation",
            "request_id",
            "runner_revision",
            "release_revision",
            "target",
            "release",
            "installer_source",
            "release_artifact",
            "files",
            "manifest_sha256",
        },
        "request manifest fields were not exact",
    )
    _require(
        manifest.get("schema") == SCHEMA
        and manifest.get("operation") == operation
        and manifest.get("request_id") == request_id
        and manifest.get("runner_revision") == runner_revision
        and manifest.get("release_revision") == release_revision,
        "request manifest context differed",
    )
    _require(
        manifest.get("target")
        == {
            "project": PROJECT,
            "zone": ZONE,
            "instance": INSTANCE,
            "service": SERVICE,
            "service_account": SERVICE_ACCOUNT,
        },
        "request target differed",
    )
    stable = {key: value for key, value in manifest.items() if key != "manifest_sha256"}
    _require(manifest.get("manifest_sha256") == _canonical_sha256(stable), "request manifest hash differed")
    release = manifest.get("release")
    _require(
        isinstance(release, dict)
        and set(release) == {"release_sha256", "image_reference", "image_digest", "image_config_digest"}
        and HEX_64.fullmatch(str(release.get("release_sha256"))) is not None
        and IMAGE_REFERENCE.fullmatch(str(release.get("image_reference"))) is not None
        and re.fullmatch(r"sha256:[0-9a-f]{64}", str(release.get("image_digest"))) is not None
        and re.fullmatch(r"sha256:[0-9a-f]{64}", str(release.get("image_config_digest"))) is not None,
        "request release binding was invalid",
    )
    _require(
        release["image_reference"] == f"{IMAGE_REPOSITORY}@{release['image_digest']}",
        "request image digest binding differed",
    )
    artifact = manifest.get("release_artifact")
    _require(
        isinstance(artifact, dict)
        and set(artifact) == {"archive_sha256", "archive_size", "members"}
        and HEX_64.fullmatch(str(artifact.get("archive_sha256"))) is not None
        and isinstance(artifact.get("archive_size"), int)
        and 0 < artifact["archive_size"] <= MAX_ARCHIVE_BYTES
        and isinstance(artifact.get("members"), list),
        "release artifact binding was invalid",
    )


def validate_original_release_artifact(
    artifact_path: Path,
    *,
    manifest: Mapping[str, Any],
    payload_root: Path,
) -> None:
    raw = artifact_path.read_bytes()
    binding = manifest["release_artifact"]
    _require(
        len(raw) == binding["archive_size"] and _sha256_bytes(raw) == binding["archive_sha256"],
        "original release artifact archive differed",
    )
    try:
        archive = zipfile.ZipFile(io.BytesIO(raw), mode="r")
    except (OSError, zipfile.BadZipFile) as exc:
        raise RemoteError("original release artifact archive was invalid") from exc
    with archive:
        infos = archive.infolist()
        names = [info.filename for info in infos]
        _require(
            len(names) == len(set(names)) and set(names) == RELEASE_ARTIFACT_MEMBERS,
            "release artifact members differed",
        )
        observed: list[dict[str, Any]] = []
        payloads: dict[str, bytes] = {}
        for info in infos:
            unix_mode = (info.external_attr >> 16) & 0o177777
            _require(
                not info.is_dir()
                and info.file_size <= MAX_MEMBER_BYTES
                and (unix_mode == 0 or stat.S_ISREG(unix_mode)),
                "release artifact member was unsafe",
            )
            content = archive.read(info)
            payloads[info.filename] = content
            observed.append({"path": info.filename, "sha256": _sha256_bytes(content), "size": len(content)})
    _require(
        sorted(observed, key=lambda item: str(item["path"])) == binding["members"],
        "release artifact member receipt differed",
    )
    _require(
        payloads["final/release-v3/release.json"] == (payload_root / "release/release.json").read_bytes()
        and payloads[f"final/release-v3/{SERVICE}"] == (payload_root / f"release/{SERVICE}").read_bytes(),
        "release artifact differed from extracted release",
    )


def _metadata(path: str, *, runner: Runner) -> bytes:
    return _run(
        [
            "/usr/bin/curl",
            "--fail",
            "--silent",
            "--show-error",
            "--max-time",
            "5",
            "--header",
            "Metadata-Flavor: Google",
            f"{METADATA}/{path}",
        ],
        runner=runner,
        label="metadata read",
        timeout=10,
    ).stdout


def verify_service_account(*, runner: Runner) -> None:
    try:
        email = _metadata("instance/service-accounts/default/email", runner=runner).decode("ascii", "strict").strip()
        project = _metadata("project/project-id", runner=runner).decode("ascii", "strict").strip()
        instance = _metadata("instance/name", runner=runner).decode("ascii", "strict").strip()
        zone = _metadata("instance/zone", runner=runner).decode("ascii", "strict").strip()
    except UnicodeError as exc:
        raise RemoteError("VM target identity was invalid") from exc
    _require(
        email == SERVICE_ACCOUNT and project == PROJECT and instance == INSTANCE and zone.endswith(f"/zones/{ZONE}"),
        "VM target or service account identity differed",
    )


def _docker_prefix(config: Path) -> list[str]:
    return [DOCKER, f"--host={DOCKER_HOST}", f"--config={config}"]


def pull_immutable_image(reference: str, request_dir: Path, *, runner: Runner) -> None:
    _require(IMAGE_REFERENCE.fullmatch(reference) is not None, "image reference was not immutable")
    token_receipt = _strict_json(
        _metadata("instance/service-accounts/default/token", runner=runner),
        "metadata token",
        MAX_TOKEN_BYTES,
    )
    token = token_receipt.get("access_token")
    _require(
        isinstance(token, str)
        and 16 <= len(token) <= 8192
        and "\n" not in token
        and token_receipt.get("token_type") == "Bearer",
        "metadata token response was invalid",
    )
    config = Path(tempfile.mkdtemp(prefix="docker-auth-", dir=request_dir))
    config.chmod(0o700)
    try:
        _run(
            [
                *_docker_prefix(config),
                "login",
                "--username=oauth2accesstoken",
                "--password-stdin",
                "https://europe-west6-docker.pkg.dev",
            ],
            runner=runner,
            label="registry login",
            input_bytes=(token + "\n").encode("utf-8"),
        )
        _run(
            [*_docker_prefix(config), "image", "pull", reference],
            runner=runner,
            label="immutable image pull",
            timeout=300,
        )
    finally:
        try:
            _run(
                [*_docker_prefix(config), "logout", "https://europe-west6-docker.pkg.dev"],
                runner=runner,
                label="registry logout",
            )
        except RemoteError:
            pass
        shutil.rmtree(config, ignore_errors=True)
    _require(not config.exists(), "ephemeral Docker credentials were not removed")


def _load_contract_modules(payload_root: Path) -> tuple[Any, Any]:
    sys.path.insert(0, str(payload_root))
    try:
        from ops import gcp_leoclean_nosend_package as package
        from ops import install_gcp_leoclean_nosend_release as installer
    except Exception as exc:
        raise RemoteError("reviewed installer sources could not be loaded") from exc
    _require(
        package.PROJECT == PROJECT
        and package.ZONE == ZONE
        and package.INSTANCE == INSTANCE
        and package.SERVICE == SERVICE
        and package.SERVICE_ACCOUNT == SERVICE_ACCOUNT,
        "reviewed installer target differed",
    )
    return package, installer


def _verify_database(
    *,
    container_id: str,
    run_id: str,
    receipt_name: str,
    request_dir: Path,
    package_module: Any,
    runner: Runner,
) -> dict[str, Any]:
    completed = _run(
        [
            DOCKER,
            f"--host={DOCKER_HOST}",
            f"--config={SERVICE_DOCKER_CONFIG}",
            "container",
            "exec",
            "--user=0",
            container_id,
            "/opt/livingip/leoclean-nosend/venv/bin/python",
            "/opt/livingip/leoclean-nosend/entrypoint.py",
            "verify-database",
            run_id,
        ],
        runner=runner,
        label="container database verification",
        timeout=300,
    )
    receipt = _strict_json(completed.stdout, "database verification receipt", MAX_COMMAND_OUTPUT_BYTES)
    _require(
        receipt.get("schema") == "livingip.leocleanNoSendContainerDatabaseVerification.v1"
        and receipt.get("status") == "pass"
        and receipt.get("run_id") == run_id,
        "database verification receipt did not pass",
    )
    stable = {key: value for key, value in receipt.items() if key != "receipt_sha256"}
    _require(receipt.get("receipt_sha256") == package_module.canonical_sha256(stable), "database receipt hash differed")
    _require(re.fullmatch(r"[a-z][a-z0-9-]{1,62}\.json", receipt_name) is not None, "receipt name was invalid")
    receipt_path = request_dir / receipt_name
    _copy_into_request(
        receipt_path,
        (json.dumps(receipt, allow_nan=False, indent=2, sort_keys=True) + "\n").encode("utf-8"),
        0o600,
    )
    return {
        "schema": receipt["schema"],
        "status": "pass",
        "receipt_sha256": receipt["receipt_sha256"],
        "receipt_path": str(receipt_path),
        "proof": {
            "proposal_staging": "function_only",
            "proposal_transaction": "rolled_back",
            "durable_proposal_created": False,
            "canonical_direct_writes": "denied",
            "stage_direct_writes": "denied",
            "role_escalation": "denied",
        },
    }


def _execute_explicit_rollback(
    *,
    installer_module: Any,
    release_sha256: str,
    paths: Any,
    source_binding: Mapping[str, Any],
    runner: Runner,
) -> dict[str, Any]:
    """Call only the receipt-bound explicit rollback API; never model rollback as another install."""

    rollback = getattr(installer_module, "rollback_release", None)
    _require(callable(rollback), "reviewed installer lacked the explicit rollback interface")
    executed = False

    def contract_runner(argv: list[str]) -> subprocess.CompletedProcess[bytes]:
        return _run(
            argv,
            runner=runner,
            label="rollback command",
            require_success=False,
        )

    try:
        plan = rollback(
            release_sha256,
            paths=paths,
            source_binding=source_binding,
            runner=contract_runner,
            execute_rollback=False,
        )
        _require(
            isinstance(plan, dict)
            and plan.get("status") == "pass"
            and plan.get("mode") == "dry_run"
            and plan.get("expected_release_sha256") == release_sha256,
            "explicit rollback preflight did not pass",
        )
        executed = True
        result = rollback(
            release_sha256,
            paths=paths,
            source_binding=source_binding,
            runner=contract_runner,
            execute_rollback=True,
        )
        _require(isinstance(result, dict) and result.get("status") == "pass", "explicit rollback did not pass")
        return result
    except Exception as exc:
        raise ExplicitRollbackFailure(executed=executed) from exc


def _rollback_result(
    *,
    manifest: Mapping[str, Any],
    release: Mapping[str, Any],
    rollback: Mapping[str, Any],
    archive_sha256: str,
    helper_sha256: str,
) -> dict[str, Any]:
    posture = rollback.get("restored_service_posture")
    restored_release = rollback.get("restored_release_sha256")
    verification = rollback.get("verification")
    _require(
        rollback.get("schema") == "livingip.leocleanNoSendRollbackResult.v1"
        and rollback.get("status") == "pass"
        and rollback.get("mode") == "executed"
        and rollback.get("rolled_back_release_sha256") == release["release_sha256"]
        and posture in {"active", "inactive", "absent"},
        "explicit rollback result was invalid",
    )
    if posture == "active":
        _require(
            isinstance(restored_release, str)
            and HEX_64.fullmatch(restored_release) is not None
            and isinstance(verification, dict)
            and verification.get("active_state") == "active"
            and verification.get("sub_state") == "running"
            and isinstance(verification.get("container_id"), str)
            and HEX_64.fullmatch(verification["container_id"]) is not None,
            "active rollback result lacked exact restored verification",
        )
    elif posture == "inactive":
        _require(
            isinstance(restored_release, str)
            and HEX_64.fullmatch(restored_release) is not None
            and verification is None,
            "inactive rollback result was invalid",
        )
    else:
        _require(restored_release is None and verification is None, "absent rollback result was invalid")
    return {
        "schema": RESULT_SCHEMA,
        "status": "pass",
        "operation": "rollback",
        "request_id": manifest["request_id"],
        "runner_revision": manifest["runner_revision"],
        "release_revision": manifest["release_revision"],
        "release_sha256": release["release_sha256"],
        "image_digest": release["image"]["digest"],
        "image_config_digest": release["image"]["config_digest"],
        "release_artifact_sha256": manifest["release_artifact"]["archive_sha256"],
        "request_binding": {
            "archive_sha256": archive_sha256,
            "helper_sha256": helper_sha256,
            "manifest_sha256": manifest["manifest_sha256"],
        },
        "service": {
            "name": SERVICE,
            "restored_posture": posture,
            "restored_release_sha256": restored_release,
            "verification": verification,
        },
        "database": {"status": "not_run_after_explicit_rollback"},
        "authority": {
            "sending_enabled": False,
            "production_traffic_changed": False,
            "vps_touched": False,
            "canonical_proposal_approved": False,
        },
    }


def _verification_failure_result(
    *,
    manifest: Mapping[str, Any],
    release: Mapping[str, Any],
    failure: Exception,
    rollback: Mapping[str, Any],
    archive_sha256: str,
    helper_sha256: str,
    installer_module: Any,
) -> dict[str, Any]:
    failure_stage = getattr(failure, "failure_stage", None)
    failure_reason = getattr(failure, "failure_reason", None)
    _require(
        isinstance(failure_stage, str)
        and isinstance(failure_reason, str)
        and (failure_stage, failure_reason) in installer_module.VERIFICATION_FAILURES,
        "verification failure was not allowlisted",
    )
    _require(
        dict(rollback)
        == _rollback_summary(
            status=str(rollback.get("status")),
            mode=str(rollback.get("mode")),
            executed=rollback.get("executed") is True,
            restored_posture=str(rollback.get("restored_posture")),
            prior_posture_preserved=rollback.get("prior_posture_preserved") is True,
        ),
        "verification rollback summary was invalid",
    )
    return {
        "schema": FAILURE_RESULT_SCHEMA,
        "status": "fail",
        "operation": manifest["operation"],
        "request_id": manifest["request_id"],
        "runner_revision": manifest["runner_revision"],
        "release_revision": manifest["release_revision"],
        "release_sha256": release["release_sha256"],
        "image_digest": release["image"]["digest"],
        "image_config_digest": release["image"]["config_digest"],
        "release_artifact_sha256": manifest["release_artifact"]["archive_sha256"],
        "request_binding": {
            "archive_sha256": archive_sha256,
            "helper_sha256": helper_sha256,
            "manifest_sha256": manifest["manifest_sha256"],
        },
        "failure_stage": failure_stage,
        "failure_reason": failure_reason,
        "rollback": dict(rollback),
        "authority": {
            "sending_enabled": False,
            "production_traffic_changed": False,
            "vps_touched": False,
            "canonical_proposal_approved": False,
        },
    }


def _installer_failure_rollback(failure: Exception) -> dict[str, Any]:
    rollback_executed = getattr(failure, "rollback_executed", None)
    restored_posture = getattr(failure, "restored_posture", None)
    prior_posture_preserved = getattr(failure, "prior_posture_preserved", None)
    _require(
        isinstance(rollback_executed, bool)
        and restored_posture in {"active", "inactive", "absent"}
        and prior_posture_preserved is True,
        "installer failure lacked a proven recovery posture",
    )
    return _rollback_summary(
        status="pass" if rollback_executed else "not_run",
        mode="installer_automatic" if rollback_executed else "not_applicable",
        executed=rollback_executed,
        restored_posture=str(restored_posture) if rollback_executed else "unchanged",
        prior_posture_preserved=True,
    )


def _is_verification_failure(installer_module: Any, failure: Exception) -> bool:
    failure_type = getattr(installer_module, "VerificationError", None)
    return isinstance(failure_type, type) and isinstance(failure, failure_type)


def _post_install_failure_coordinates(
    installer_module: Any,
    failure: Exception,
    *,
    fallback_stage: str,
    fallback_reason: str,
) -> tuple[str, str]:
    if _is_verification_failure(installer_module, failure):
        failure_stage = getattr(failure, "failure_stage", None)
        failure_reason = getattr(failure, "failure_reason", None)
        _require(
            isinstance(failure_stage, str)
            and isinstance(failure_reason, str)
            and (failure_stage, failure_reason) in installer_module.VERIFICATION_FAILURES,
            "verification failure was not allowlisted",
        )
        return failure_stage, failure_reason
    _require(
        (fallback_stage, fallback_reason) in POST_INSTALL_FAILURES,
        "post-install failure was not allowlisted",
    )
    return fallback_stage, fallback_reason


def _rollback_proven_service(posture: str) -> dict[str, Any]:
    _require(posture in {"active", "inactive", "absent"}, "rollback posture was invalid")
    return {
        "status": "not_required",
        "service_posture": posture,
        "active_state": "not_observed",
        "sub_state": "not_observed",
        "main_pid": None,
        "n_restarts": None,
        "candidate_container": "not_observed",
        "service_action_executed": False,
    }


def _fail_closed_inert_service(
    *,
    installer_module: Any,
    release: Mapping[str, Any],
    paths: Any,
    runner: Runner,
) -> dict[str, Any]:
    service_action_executed = False

    def contract_runner(argv: list[str]) -> subprocess.CompletedProcess[bytes]:
        return _run(
            argv,
            runner=runner,
            label="fail-closed service command",
            require_success=False,
        )

    try:
        systemctl = getattr(installer_module, "_systemctl", None)
        stable_service_posture = getattr(installer_module, "_stable_service_posture", None)
        verify_running_release = getattr(installer_module, "verify_running_release", None)
        require_candidate_absent = getattr(installer_module, "_require_candidate_absent", None)
        _require(
            callable(systemctl)
            and callable(stable_service_posture)
            and callable(verify_running_release)
            and callable(require_candidate_absent),
            "reviewed installer lacked fail-closed service controls",
        )
        before = stable_service_posture(
            release,
            runner=contract_runner,
            paths=paths,
            sleeper=time.sleep,
        )
        _require(before.kind in {"active", "inactive"}, "managed service posture was not trusted")
        if before.kind == "active":
            verification = verify_running_release(
                release,
                runner=contract_runner,
                paths=paths,
                sleeper=time.sleep,
            )
            _require(
                verification.get("main_pid") == before.state.get("MainPID")
                and verification.get("invocation_id") == before.state.get("InvocationID")
                and verification.get("n_restarts") == before.state.get("NRestarts"),
                "running service identity changed before fail-closed stop",
            )
            service_action_executed = True
            systemctl("stop", contract_runner)
        after = stable_service_posture(
            release,
            runner=contract_runner,
            paths=paths,
            sleeper=time.sleep,
        )
        _require(
            after.kind == "inactive"
            and after.state.get("ActiveState") == "inactive"
            and after.state.get("SubState") == "dead"
            and after.state.get("MainPID") == 0
            and isinstance(after.state.get("NRestarts"), int)
            and not isinstance(after.state.get("NRestarts"), bool)
            and after.state["NRestarts"] >= 0,
            "service did not stop into a stable inert posture",
        )
        require_candidate_absent(
            runner=contract_runner,
            paths=paths,
            sleeper=time.sleep,
        )
        return {
            "status": "pass",
            "service_posture": "inactive",
            "active_state": "inactive",
            "sub_state": "dead",
            "main_pid": 0,
            "n_restarts": after.state["NRestarts"],
            "candidate_container": "absent",
            "service_action_executed": service_action_executed,
        }
    except Exception:
        return {
            "status": "fail",
            "service_posture": "unknown",
            "active_state": "unknown",
            "sub_state": "unknown",
            "main_pid": None,
            "n_restarts": None,
            "candidate_container": "unknown",
            "service_action_executed": service_action_executed,
        }


def _restore_transition_posture(
    *,
    installer_module: Any,
    release: Mapping[str, Any],
    transition: Mapping[str, Any],
    paths: Any,
    runner: Runner,
) -> dict[str, Any]:
    checked = _validate_installer_transition(transition)
    desired = str(checked["prior_posture"])
    _require(
        checked["outcome"] in {"verified_noop", "controlled_start"}
        and desired in {"active", "inactive"},
        "installer transition required release rollback",
    )

    def contract_runner(argv: list[str]) -> subprocess.CompletedProcess[bytes]:
        return _run(
            argv,
            runner=runner,
            label="prior posture recovery command",
            require_success=False,
        )

    systemctl = getattr(installer_module, "_systemctl", None)
    stable_service_posture = getattr(installer_module, "_stable_service_posture", None)
    verify_running_release = getattr(installer_module, "verify_running_release", None)
    require_candidate_absent = getattr(installer_module, "_require_candidate_absent", None)
    _require(
        callable(systemctl)
        and callable(stable_service_posture)
        and callable(verify_running_release)
        and callable(require_candidate_absent),
        "reviewed installer lacked posture recovery controls",
    )
    executed = False
    try:
        observed = stable_service_posture(
            release,
            runner=contract_runner,
            paths=paths,
            sleeper=time.sleep,
        )
        _require(observed.kind in {"active", "inactive"}, "managed service posture was not trusted")
        if desired == "inactive":
            if observed.kind == "active":
                verification = verify_running_release(
                    release,
                    runner=contract_runner,
                    paths=paths,
                    sleeper=time.sleep,
                )
                _require(
                    verification.get("main_pid") == observed.state.get("MainPID")
                    and verification.get("invocation_id") == observed.state.get("InvocationID")
                    and verification.get("n_restarts") == observed.state.get("NRestarts"),
                    "running service identity changed before posture recovery stop",
                )
                executed = True
                systemctl("stop", contract_runner)
            final = stable_service_posture(
                release,
                runner=contract_runner,
                paths=paths,
                sleeper=time.sleep,
            )
            _require(
                final.kind == "inactive"
                and final.state.get("NRestarts") == checked["prior_n_restarts"],
                "prior inactive posture was not restored",
            )
            require_candidate_absent(runner=contract_runner, paths=paths, sleeper=time.sleep)
        else:
            if observed.kind == "inactive":
                require_candidate_absent(runner=contract_runner, paths=paths, sleeper=time.sleep)
                executed = True
                systemctl("start", contract_runner)
            final = stable_service_posture(
                release,
                runner=contract_runner,
                paths=paths,
                sleeper=time.sleep,
            )
            _require(
                final.kind == "active"
                and final.state.get("NRestarts") == checked["prior_n_restarts"],
                "prior active posture was not restored",
            )
            verification = verify_running_release(
                release,
                runner=contract_runner,
                paths=paths,
                sleeper=time.sleep,
            )
            _require(
                verification.get("main_pid") == final.state.get("MainPID")
                and verification.get("invocation_id") == final.state.get("InvocationID")
                and verification.get("n_restarts") == checked["prior_n_restarts"],
                "restored active service identity was unstable",
            )
    except Exception as exc:
        raise PostureRecoveryFailure(executed=executed) from exc
    return _rollback_summary(
        status="pass",
        mode="posture_recovery",
        executed=executed,
        restored_posture=desired,
        prior_posture_preserved=True,
    )


def _operational_failure_result(
    *,
    manifest: Mapping[str, Any],
    release: Mapping[str, Any],
    failure_stage: str,
    failure_reason: str,
    rollback: Mapping[str, Any],
    fail_closed: Mapping[str, Any],
    archive_sha256: str,
    helper_sha256: str,
    installer_module: Any,
) -> dict[str, Any]:
    _require(
        manifest.get("operation") == "install"
        and set(rollback)
        == {"status", "mode", "executed", "restored_posture", "prior_posture_preserved"}
        and (
            (failure_stage, failure_reason) in POST_INSTALL_FAILURES
            or (failure_stage, failure_reason) in installer_module.VERIFICATION_FAILURES
        ),
        "post-install failure was not allowlisted",
    )
    rollback_passed = (
        rollback.get("status") == "pass"
        and rollback.get("mode") in {"explicit", "posture_recovery"}
        and isinstance(rollback.get("executed"), bool)
        and rollback.get("restored_posture") in {"active", "inactive", "absent"}
        and (
            rollback.get("mode") == "explicit"
            or rollback.get("restored_posture") in {"active", "inactive"}
        )
        and rollback.get("prior_posture_preserved") is True
    )
    rollback_failed = (
        rollback.get("status") == "fail"
        and rollback.get("mode") in {"explicit", "posture_recovery"}
        and isinstance(rollback.get("executed"), bool)
        and rollback.get("restored_posture") == "unknown"
        and rollback.get("prior_posture_preserved") is False
    )
    verification_pair = (failure_stage, failure_reason) in installer_module.VERIFICATION_FAILURES
    _require(
        (rollback_passed and not verification_pair) or rollback_failed,
        "post-install rollback summary was invalid",
    )
    if rollback_passed:
        _require(
            fail_closed == _rollback_proven_service(str(rollback["restored_posture"])),
            "post-install restored posture summary was invalid",
        )
    else:
        _require(
            fail_closed
            in (
                {
                    "status": "fail",
                    "service_posture": "unknown",
                    "active_state": "unknown",
                    "sub_state": "unknown",
                    "main_pid": None,
                    "n_restarts": None,
                    "candidate_container": "unknown",
                    "service_action_executed": fail_closed.get("service_action_executed"),
                },
                {
                    "status": "pass",
                    "service_posture": "inactive",
                    "active_state": "inactive",
                    "sub_state": "dead",
                    "main_pid": 0,
                    "n_restarts": fail_closed.get("n_restarts"),
                    "candidate_container": "absent",
                    "service_action_executed": fail_closed.get("service_action_executed"),
                },
            )
            and (
                fail_closed.get("status") != "pass"
                or (
                    isinstance(fail_closed.get("n_restarts"), int)
                    and not isinstance(fail_closed.get("n_restarts"), bool)
                    and fail_closed["n_restarts"] >= 0
                )
            )
            and isinstance(fail_closed.get("service_action_executed"), bool),
            "post-install fail-closed summary was invalid",
        )
    return {
        "schema": OPERATION_FAILURE_RESULT_SCHEMA,
        "status": "fail",
        "operation": manifest["operation"],
        "request_id": manifest["request_id"],
        "runner_revision": manifest["runner_revision"],
        "release_revision": manifest["release_revision"],
        "release_sha256": release["release_sha256"],
        "image_digest": release["image"]["digest"],
        "image_config_digest": release["image"]["config_digest"],
        "release_artifact_sha256": manifest["release_artifact"]["archive_sha256"],
        "request_binding": {
            "archive_sha256": archive_sha256,
            "helper_sha256": helper_sha256,
            "manifest_sha256": manifest["manifest_sha256"],
        },
        "failure_stage": failure_stage,
        "failure_reason": failure_reason,
        "rollback": dict(rollback),
        "fail_closed": dict(fail_closed),
        "authority": {
            "sending_enabled": False,
            "production_traffic_changed": False,
            "vps_touched": False,
            "canonical_proposal_approved": False,
        },
    }


def execute_request(
    manifest: Mapping[str, Any],
    payload_root: Path,
    request_dir: Path,
    archive_sha256: str,
    helper_sha256: str,
    *,
    runner: Runner = subprocess.run,
) -> dict[str, Any]:
    operation = str(manifest["operation"])
    release_binding = manifest["release"]
    validate_original_release_artifact(
        payload_root / "release-artifact.zip",
        manifest=manifest,
        payload_root=payload_root,
    )
    package_module, installer_module = _load_contract_modules(payload_root)
    bundle = payload_root / "release"
    release, unit = installer_module.validate_release_bundle(bundle)
    _require(
        release["release_sha256"] == release_binding["release_sha256"]
        and release["image"]["reference"] == release_binding["image_reference"]
        and release["image"]["digest"] == release_binding["image_digest"]
        and release["image"]["config_digest"] == release_binding["image_config_digest"]
        and unit == package_module.render_systemd_unit(release).encode("utf-8"),
        "extracted release differed from the request",
    )
    source_binding = manifest["installer_source"]
    installer_module._validate_source_binding(source_binding)
    _require(
        source_binding.get("revision") == manifest["runner_revision"],
        "installer source revision differed from runner revision",
    )
    _require(
        release["image_input"]["runtime"]["teleo_git_head"] == manifest["release_revision"],
        "release source revision differed from release revision",
    )
    paths = installer_module.InstallPaths.under(Path("/"))
    if operation == "install":
        pull_immutable_image(release["image"]["reference"], request_dir, runner=runner)
        try:
            install_receipt = installer_module.install_release(
                bundle,
                paths=paths,
                source_binding=source_binding,
                runner=lambda argv: _run(
                    argv,
                    runner=runner,
                    label="installer command",
                    require_success=False,
                ),
                execute_restart=True,
            )
        except Exception as exc:
            if not _is_verification_failure(installer_module, exc):
                raise
            raise RemoteOperationFailure(
                _verification_failure_result(
                    manifest=manifest,
                    release=release,
                    failure=exc,
                    rollback=_installer_failure_rollback(exc),
                    archive_sha256=archive_sha256,
                    helper_sha256=helper_sha256,
                    installer_module=installer_module,
                )
            ) from exc
    elif operation == "rollback":
        rollback = _execute_explicit_rollback(
            installer_module=installer_module,
            release_sha256=release["release_sha256"],
            paths=paths,
            source_binding=source_binding,
            runner=runner,
        )
        return _rollback_result(
            manifest=manifest,
            release=release,
            rollback=rollback,
            archive_sha256=archive_sha256,
            helper_sha256=helper_sha256,
        )

    def contract_runner(argv: list[str]) -> subprocess.CompletedProcess[bytes]:
        return _run(
            argv,
            runner=runner,
            label="release verification command",
            require_success=False,
        )

    if operation == "install":
        failure_stage = "install_receipt"
        failure_reason = "install_receipt_invalid"
        installer_transition: dict[str, Any] | None = None
        try:
            installer_transition = _validate_install_execution_result(
                install_receipt,
                release=release,
                installer_module=installer_module,
            )
            failure_stage = "service_before_restart"
            failure_reason = "service_verification_failed"
            before = installer_module.verify_running_release(
                release,
                runner=contract_runner,
                paths=paths,
            )
            failure_stage = "database_before_restart"
            failure_reason = "database_verification_failed"
            before_database = _verify_database(
                container_id=before["container_id"],
                run_id=f"{manifest['request_id']}-pre",
                receipt_name="database-before-restart.json",
                request_dir=request_dir,
                package_module=package_module,
                runner=runner,
            )
            failure_stage = "service_restart"
            failure_reason = "service_restart_failed"
            _run(
                [installer_module.SYSTEMCTL, "restart", SERVICE],
                runner=runner,
                label="fixed service restart",
            )
            failure_stage = "service_after_restart"
            failure_reason = "service_verification_failed"
            after = installer_module.verify_running_release(
                release,
                runner=contract_runner,
                paths=paths,
            )
            failure_stage = "restart_identity"
            failure_reason = "restart_identity_mismatch"
            _require(
                before["main_pid"] != after["main_pid"]
                and before["invocation_id"] != after["invocation_id"]
                and before["container_id"] != after["container_id"]
                and after["n_restarts"] >= before["n_restarts"],
                "post-install restart did not create a new exact service/container identity",
            )
            failure_stage = "database_after_restart"
            failure_reason = "database_verification_failed"
            after_database = _verify_database(
                container_id=after["container_id"],
                run_id=f"{manifest['request_id']}-post",
                receipt_name="database-after-restart.json",
                request_dir=request_dir,
                package_module=package_module,
                runner=runner,
            )
        except Exception as exc:
            bounded_stage, bounded_reason = _post_install_failure_coordinates(
                installer_module,
                exc,
                fallback_stage=failure_stage,
                fallback_reason=failure_reason,
            )
            if installer_transition is None:
                fail_closed = _fail_closed_inert_service(
                    installer_module=installer_module,
                    release=release,
                    paths=paths,
                    runner=runner,
                )
                raise RemoteOperationFailure(
                    _operational_failure_result(
                        manifest=manifest,
                        release=release,
                        failure_stage="install_receipt",
                        failure_reason="install_receipt_invalid",
                        rollback=_rollback_summary(
                            status="fail",
                            mode="posture_recovery",
                            executed=bool(fail_closed["service_action_executed"]),
                            restored_posture="unknown",
                            prior_posture_preserved=False,
                        ),
                        fail_closed=fail_closed,
                        archive_sha256=archive_sha256,
                        helper_sha256=helper_sha256,
                        installer_module=installer_module,
                    )
                ) from exc
            release_changing = installer_transition["outcome"] in {"installed", "replaced"}
            recovery_mode = "explicit" if release_changing else "posture_recovery"
            recovery_executed = False
            try:
                if release_changing:
                    try:
                        recovery = _execute_explicit_rollback(
                            installer_module=installer_module,
                            release_sha256=release["release_sha256"],
                            paths=paths,
                            source_binding=source_binding,
                            runner=runner,
                        )
                    except ExplicitRollbackFailure as rollback_exc:
                        recovery_executed = rollback_exc.executed
                        raise
                    recovery_executed = True
                    rollback_result = _rollback_result(
                        manifest=manifest,
                        release=release,
                        rollback=recovery,
                        archive_sha256=archive_sha256,
                        helper_sha256=helper_sha256,
                    )
                    rollback_summary = _rollback_summary(
                        status="pass",
                        mode="explicit",
                        executed=True,
                        restored_posture=str(rollback_result["service"]["restored_posture"]),
                        prior_posture_preserved=True,
                    )
                else:
                    try:
                        rollback_summary = _restore_transition_posture(
                            installer_module=installer_module,
                            release=release,
                            transition=installer_transition,
                            paths=paths,
                            runner=runner,
                        )
                    except PostureRecoveryFailure as posture_exc:
                        recovery_executed = posture_exc.executed
                        raise
                    recovery_executed = bool(rollback_summary["executed"])
            except Exception:
                fail_closed = _fail_closed_inert_service(
                    installer_module=installer_module,
                    release=release,
                    paths=paths,
                    runner=runner,
                )
                raise RemoteOperationFailure(
                    _operational_failure_result(
                        manifest=manifest,
                        release=release,
                        failure_stage=bounded_stage,
                        failure_reason=bounded_reason,
                        rollback=_rollback_summary(
                            status="fail",
                            mode=recovery_mode,
                            executed=recovery_executed,
                            restored_posture="unknown",
                            prior_posture_preserved=False,
                        ),
                        fail_closed=fail_closed,
                        archive_sha256=archive_sha256,
                        helper_sha256=helper_sha256,
                        installer_module=installer_module,
                    )
                ) from exc
            if _is_verification_failure(installer_module, exc):
                raise RemoteOperationFailure(
                    _verification_failure_result(
                        manifest=manifest,
                        release=release,
                        failure=exc,
                        rollback=rollback_summary,
                        archive_sha256=archive_sha256,
                        helper_sha256=helper_sha256,
                        installer_module=installer_module,
                    )
                ) from exc
            restored_posture = str(rollback_summary["restored_posture"])
            raise RemoteOperationFailure(
                _operational_failure_result(
                    manifest=manifest,
                    release=release,
                    failure_stage=bounded_stage,
                    failure_reason=bounded_reason,
                    rollback=rollback_summary,
                    fail_closed=_rollback_proven_service(restored_posture),
                    archive_sha256=archive_sha256,
                    helper_sha256=helper_sha256,
                    installer_module=installer_module,
                )
            ) from exc
        service = {
            "name": SERVICE,
            "installer_transition": installer_transition,
            "before_restart": before,
            "after_restart": after,
        }
        database = {
            "status": "pass",
            "before_restart": before_database,
            "after_restart": after_database,
        }
    else:
        try:
            verification = installer_module.verify_running_release(
                release,
                runner=contract_runner,
                paths=paths,
            )
        except Exception as exc:
            if not _is_verification_failure(installer_module, exc):
                raise
            raise RemoteOperationFailure(
                _verification_failure_result(
                    manifest=manifest,
                    release=release,
                    failure=exc,
                    rollback=_rollback_summary(
                        status="not_run",
                        mode="not_applicable",
                        executed=False,
                        restored_posture="unchanged",
                        prior_posture_preserved=True,
                    ),
                    archive_sha256=archive_sha256,
                    helper_sha256=helper_sha256,
                    installer_module=installer_module,
                )
            ) from exc
        database = _verify_database(
            container_id=verification["container_id"],
            run_id=f"{manifest['request_id']}-verify",
            receipt_name="database-verify.json",
            request_dir=request_dir,
            package_module=package_module,
            runner=runner,
        )
        service = {
            "name": SERVICE,
            "verification": verification,
        }
    return {
        "schema": RESULT_SCHEMA,
        "status": "pass",
        "operation": operation,
        "request_id": manifest["request_id"],
        "runner_revision": manifest["runner_revision"],
        "release_revision": manifest["release_revision"],
        "release_sha256": release["release_sha256"],
        "image_digest": release["image"]["digest"],
        "image_config_digest": release["image"]["config_digest"],
        "release_artifact_sha256": manifest["release_artifact"]["archive_sha256"],
        "request_binding": {
            "archive_sha256": archive_sha256,
            "helper_sha256": helper_sha256,
            "manifest_sha256": manifest["manifest_sha256"],
        },
        "service": service,
        "database": database,
        "authority": {
            "sending_enabled": False,
            "production_traffic_changed": False,
            "vps_touched": False,
            "canonical_proposal_approved": False,
        },
    }


def _execute_and_persist_result(
    manifest: Mapping[str, Any],
    payload_root: Path,
    request_dir: Path,
    archive_sha256: str,
    helper_sha256: str,
    *,
    runner: Runner,
) -> dict[str, Any]:
    try:
        result = execute_request(
            manifest,
            payload_root,
            request_dir,
            archive_sha256,
            helper_sha256,
            runner=runner,
        )
    except RemoteOperationFailure as exc:
        result = exc.result
    result_path = request_dir / "result.json"
    _copy_into_request(
        result_path,
        (json.dumps(result, allow_nan=False, indent=2, sort_keys=True) + "\n").encode("utf-8"),
        0o600,
    )
    return result


def receive_request(
    operation: str,
    request_id: str,
    runner_revision: str,
    release_revision: str,
    archive_sha256: str,
    helper_sha256: str,
    *,
    runner: Runner = subprocess.run,
    run_root: Path = RUN_ROOT,
    lock_path: Path = LOCK_PATH,
) -> dict[str, Any]:
    _require(os.geteuid() == 0, "remote execution required root")
    _require(operation in OPERATIONS, "operation was not allowlisted")
    _require(REQUEST_ID.fullmatch(request_id) is not None, "request ID was invalid")
    _require(HEX_40.fullmatch(runner_revision) is not None, "runner revision was invalid")
    _require(HEX_40.fullmatch(release_revision) is not None, "release revision was invalid")
    _require(HEX_64.fullmatch(archive_sha256) is not None, "archive hash was invalid")
    _require(HEX_64.fullmatch(helper_sha256) is not None, "helper hash was invalid")
    caller = os.environ.get("SUDO_USER", "")
    _require(caller and caller != "root", "remote execution lacked a distinct OS Login caller")
    try:
        pwd.getpwnam(caller)
    except KeyError as exc:
        raise RemoteError("OS Login caller did not resolve") from exc
    stage_dir = STAGE_ROOT / request_id
    archive_upload = stage_dir / "request.tar.gz"
    helper_upload = stage_dir / "remote-helper.py"
    archive_stat = archive_upload.stat(follow_symlinks=False)
    helper_stat = helper_upload.stat(follow_symlinks=False)
    archive_identity = (archive_stat.st_dev, archive_stat.st_ino)
    helper_identity = (helper_stat.st_dev, helper_stat.st_ino)
    try:
        stage_state = stage_dir.stat(follow_symlinks=False)
        _require(
            stat.S_ISDIR(stage_state.st_mode)
            and stage_state.st_uid == 0
            and stage_state.st_gid == 0
            and stat.S_IMODE(stage_state.st_mode) == 0o700,
            "root-owned bootstrap stage was unsafe",
        )
        helper_content = _read_stable_stage(helper_upload, helper_sha256, MAX_MEMBER_BYTES)
        archive_content = _read_stable_stage(archive_upload, archive_sha256, MAX_ARCHIVE_BYTES)
        _require(_sha256_bytes(Path(__file__).read_bytes()) == helper_sha256, "executing helper differed from request")
        lock_descriptor = _acquire_lock(lock_path)
        try:
            request_dir = _claim_request_directory(run_root, request_id)
            secured_archive = request_dir / "request.tar.gz"
            secured_helper = request_dir / "remote-helper.py"
            _copy_into_request(secured_archive, archive_content)
            _copy_into_request(secured_helper, helper_content)
            _unlink_exact(archive_upload, archive_identity)
            _unlink_exact(helper_upload, helper_identity)
            stage_dir.rmdir()
            payload_root = request_dir / "payload"
            payload_root.mkdir(mode=0o700)
            manifest = extract_archive(secured_archive, payload_root)
            validate_manifest(
                manifest,
                operation=operation,
                request_id=request_id,
                runner_revision=runner_revision,
                release_revision=release_revision,
            )
            verify_service_account(runner=runner)
            return _execute_and_persist_result(
                manifest,
                payload_root,
                request_dir,
                archive_sha256,
                helper_sha256,
                runner=runner,
            )
        finally:
            os.close(lock_descriptor)
    finally:
        _cleanup_upload_if_exact(archive_upload, archive_identity)
        _cleanup_upload_if_exact(helper_upload, helper_identity)


def main(argv: list[str] | None = None) -> int:
    arguments = argv if argv is not None else sys.argv[1:]
    try:
        os.umask(0o077)
        _require(len(arguments) == 6, "remote arguments were not exact")
        result = receive_request(*arguments)
        sys.stdout.write(json.dumps(result, allow_nan=False, separators=(",", ":"), sort_keys=True) + "\n")
        return 0 if result.get("status") == "pass" else 65
    except RemoteError as exc:
        error = str(exc)
        if error not in {"install_failed_rolled_back", "install_failed_rollback_incomplete"}:
            error = "remote_request_failed"
        sys.stderr.write(json.dumps({"error": error, "status": "fail"}, separators=(",", ":")) + "\n")
        return 65
    except Exception:
        sys.stderr.write('{"error":"remote_request_failed","status":"fail"}\n')
        return 65


if __name__ == "__main__":
    raise SystemExit(main())
