#!/usr/bin/python3 -I """Execute one root-owned, fixed-target leoclean no-send IAP request.""" from __future__ import annotations import fcntl import hashlib import io import json import os import pwd import re import shutil import stat import subprocess import sys import tarfile import tempfile import time import zipfile from collections.abc import Callable, Mapping from pathlib import Path from typing import Any SCHEMA = "livingip.leocleanNoSendIapRequest.v1" RESULT_SCHEMA = "livingip.leocleanNoSendIapRemoteResult.v2" FAILURE_RESULT_SCHEMA = "livingip.leocleanNoSendIapFailure.v2" OPERATION_FAILURE_RESULT_SCHEMA = "livingip.leocleanNoSendIapOperationalFailure.v2" PROJECT = "teleo-501523" ZONE = "europe-west6-a" INSTANCE = "teleo-staging-1" SERVICE = "leoclean-gcp-nosend.service" SERVICE_ACCOUNT = "sa-teleo-staging-vm@teleo-501523.iam.gserviceaccount.com" IMAGE_REPOSITORY = "europe-west6-docker.pkg.dev/teleo-501523/teleo/leoclean-nosend-staging" DOCKER = "/usr/bin/docker" DOCKER_HOST = "unix:///var/run/docker.sock" SERVICE_DOCKER_CONFIG = "/var/lib/livingip/leoclean-nosend/docker-config" METADATA = "http://metadata.google.internal/computeMetadata/v1" RUN_ROOT = Path("/var/lib/livingip/leoclean-nosend-iap/requests") LOCK_PATH = Path("/run/leoclean-gcp-nosend-iap.lock") UPLOAD_ROOT = Path("/tmp") STAGE_ROOT = Path("/run/livingip-leoclean-nosend-iap-bootstrap") REQUEST_ID = re.compile(r"^iap-[a-z0-9]{12,32}$") HEX_40 = re.compile(r"^[0-9a-f]{40}$") HEX_64 = re.compile(r"^[0-9a-f]{64}$") IMAGE_REFERENCE = re.compile(rf"^{re.escape(IMAGE_REPOSITORY)}@sha256:([0-9a-f]{{64}})$") OPERATIONS = ("install", "verify", "rollback") MAX_ARCHIVE_BYTES = 8 * 1024 * 1024 MAX_MEMBER_BYTES = 2 * 1024 * 1024 MAX_MANIFEST_BYTES = 128 * 1024 MAX_COMMAND_OUTPUT_BYTES = 1024 * 1024 MAX_TOKEN_BYTES = 64 * 1024 POST_INSTALL_FAILURES = frozenset( { ("install_receipt", "install_receipt_invalid"), ("service_before_restart", "service_verification_failed"), ("database_before_restart", "database_verification_failed"), ("service_restart", "service_restart_failed"), ("service_after_restart", "service_verification_failed"), ("restart_identity", "restart_identity_mismatch"), ("database_after_restart", "database_verification_failed"), } ) EXPECTED_MEMBERS = { "request-manifest.json", "release-artifact.zip", "release/release.json", f"release/{SERVICE}", "ops/gcp_leoclean_nosend_package.py", "ops/install_gcp_leoclean_nosend_release.py", "ops/leoclean_nosend_service_control.py", "ops/gcp_leoclean_nosend_iap_remote.py", "ops/run_gcp_leoclean_nosend_iap.py", "scripts/leo_behavior_manifest.py", "scripts/leo_identity_manifest.py", } RELEASE_ARTIFACT_MEMBERS = { "evidence/oci-smoke.json", "evidence/publish-outcome.json", "evidence/runtime-receipt.json", f"final/release-v3/{SERVICE}", "final/release-v3/release.json", "receipts/build-push-receipt.json", } SAFE_PATH = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" Runner = Callable[..., subprocess.CompletedProcess[bytes]] class RemoteError(RuntimeError): """A remote request failed closed without exposing command output.""" class RemoteOperationFailure(RemoteError): """A sanitized, request-bound failure that can be returned to the caller.""" def __init__(self, result: Mapping[str, Any]) -> None: super().__init__("remote_operation_failed") self.result = dict(result) class PostureRecoveryFailure(RemoteError): """A posture recovery failed after a bounded amount of service action.""" def __init__(self, *, executed: bool) -> None: super().__init__("posture_recovery_failed") self.executed = executed class ExplicitRollbackFailure(RemoteError): """An explicit rollback failed before or after entering its mutation phase.""" def __init__(self, *, executed: bool) -> None: super().__init__("explicit_rollback_failed") self.executed = executed def _validate_installer_transition(value: Any) -> dict[str, Any]: _require( isinstance(value, dict) and set(value) == { "outcome", "prior_posture", "prior_n_restarts", "final_posture", "service_action", "managed_files_replaced", }, "installer transition was invalid", ) outcome = value.get("outcome") prior_posture = value.get("prior_posture") prior_n_restarts = value.get("prior_n_restarts") service_action = value.get("service_action") replaced = value.get("managed_files_replaced") _require( isinstance(prior_n_restarts, int) and not isinstance(prior_n_restarts, bool) and prior_n_restarts >= 0 and value.get("final_posture") == "active" and ( (outcome, prior_posture, service_action, replaced) == ("verified_noop", "active", "none", False) or (outcome, prior_posture, service_action, replaced) == ("controlled_start", "inactive", "start", False) or (outcome, prior_posture, service_action, replaced) == ("installed", "absent", "restart", True) or ( outcome == "replaced" and prior_posture in {"active", "inactive"} and service_action == "restart" and replaced is True ) ), "installer transition was invalid", ) return dict(value) def _validate_install_execution_result( value: Any, *, release: Mapping[str, Any], installer_module: Any, ) -> dict[str, Any]: _require( isinstance(value, dict) and set(value) == { "schema", "status", "outcome", "release_sha256", "persisted_receipt", "transition", "verification", }, "install execution result was invalid", ) transition = _validate_installer_transition(value.get("transition")) persisted_receipt = value.get("persisted_receipt") outcome = value.get("outcome") _require( value.get("schema") == installer_module.INSTALL_EXECUTION_RESULT_SCHEMA and value.get("status") == "pass" and outcome == transition["outcome"] and value.get("release_sha256") == release["release_sha256"] and isinstance(persisted_receipt, dict) and set(persisted_receipt) == {"schema", "sha256"} and persisted_receipt.get("schema") in { installer_module.LEGACY_INSTALL_RECEIPT_SCHEMA, installer_module.INSTALL_RECEIPT_SCHEMA, } and isinstance(persisted_receipt.get("sha256"), str) and HEX_64.fullmatch(persisted_receipt["sha256"]) is not None and isinstance(value.get("verification"), dict) and ( outcome in {"verified_noop", "controlled_start"} or ( outcome in {"installed", "replaced"} and persisted_receipt["schema"] == installer_module.INSTALL_RECEIPT_SCHEMA ) ), "install execution result was invalid", ) return transition def _rollback_summary( *, status: str, mode: str, executed: bool, restored_posture: str, prior_posture_preserved: bool, ) -> dict[str, Any]: value = { "status": status, "mode": mode, "executed": executed, "restored_posture": restored_posture, "prior_posture_preserved": prior_posture_preserved, } _require( ( status == "pass" and ( ( mode in {"installer_automatic", "explicit"} and executed is True and restored_posture in {"active", "inactive", "absent"} ) or ( mode == "posture_recovery" and restored_posture in {"active", "inactive"} ) ) and prior_posture_preserved is True ) or ( status == "not_run" and mode == "not_applicable" and executed is False and restored_posture == "unchanged" and prior_posture_preserved is True ) or ( status == "fail" and mode in {"explicit", "posture_recovery"} and restored_posture == "unknown" and prior_posture_preserved is False ), "rollback summary was invalid", ) return value def _require(condition: bool, message: str) -> None: if not condition: raise RemoteError(message) def _sha256_bytes(value: bytes) -> str: return hashlib.sha256(value).hexdigest() def _canonical_sha256(value: Mapping[str, Any]) -> str: try: encoded = json.dumps( value, allow_nan=False, ensure_ascii=True, separators=(",", ":"), sort_keys=True, ).encode("utf-8") except (TypeError, ValueError) as exc: raise RemoteError("request manifest could not be hashed") from exc return _sha256_bytes(encoded) def _strict_json(raw: bytes, label: str, max_bytes: int) -> dict[str, Any]: _require(len(raw) <= max_bytes, f"{label} exceeded the safety limit") def reject_duplicates(pairs: list[tuple[str, Any]]) -> dict[str, Any]: value: dict[str, Any] = {} for key, item in pairs: if key in value: raise ValueError("duplicate key") value[key] = item return value try: value = json.loads( raw.decode("utf-8", "strict"), object_pairs_hook=reject_duplicates, parse_constant=lambda _value: (_ for _ in ()).throw(ValueError("constant")), ) except (UnicodeError, ValueError, json.JSONDecodeError) as exc: raise RemoteError(f"{label} was invalid JSON") from exc _require(isinstance(value, dict), f"{label} was not a JSON object") return value def _run( argv: list[str], *, runner: Runner, label: str, input_bytes: bytes | None = None, timeout: int = 120, require_success: bool = True, ) -> subprocess.CompletedProcess[bytes]: try: completed = runner( argv, check=False, stdin=subprocess.DEVNULL if input_bytes is None else None, input=input_bytes, capture_output=True, timeout=timeout, env={"PATH": SAFE_PATH}, ) except (OSError, subprocess.SubprocessError) as exc: raise RemoteError(f"{label} failed") from exc _require( len(completed.stdout) <= MAX_COMMAND_OUTPUT_BYTES and len(completed.stderr) <= MAX_COMMAND_OUTPUT_BYTES, f"{label} output exceeded the safety limit", ) if require_success: _require(completed.returncode == 0, f"{label} failed") return completed def _read_stable_stage(path: Path, expected_sha256: str, max_bytes: int) -> bytes: _require(path.parent.parent == STAGE_ROOT, "staged request path escaped its fixed root") flags = os.O_RDONLY if hasattr(os, "O_CLOEXEC"): flags |= os.O_CLOEXEC if hasattr(os, "O_NOFOLLOW"): flags |= os.O_NOFOLLOW try: descriptor = os.open(path, flags) except OSError as exc: raise RemoteError("request upload was missing or unsafe") from exc try: before = os.fstat(descriptor) _require( stat.S_ISREG(before.st_mode) and before.st_uid == 0 and stat.S_IMODE(before.st_mode) == 0o600 and before.st_nlink == 1 and before.st_size <= max_bytes, "request upload posture was unsafe", ) chunks: list[bytes] = [] remaining = max_bytes + 1 while remaining: chunk = os.read(descriptor, min(65536, remaining)) if not chunk: break chunks.append(chunk) remaining -= len(chunk) content = b"".join(chunks) after = os.fstat(descriptor) named = path.stat(follow_symlinks=False) _require( len(content) == after.st_size and (before.st_dev, before.st_ino, before.st_mode, before.st_nlink, before.st_size, before.st_mtime_ns) == (after.st_dev, after.st_ino, after.st_mode, after.st_nlink, after.st_size, after.st_mtime_ns) and (after.st_dev, after.st_ino, after.st_mode, after.st_nlink) == (named.st_dev, named.st_ino, named.st_mode, named.st_nlink) and _sha256_bytes(content) == expected_sha256, "request upload changed or differed from its hash", ) return content except OSError as exc: raise RemoteError("request upload could not be read") from exc finally: os.close(descriptor) def _unlink_exact(path: Path, identity: tuple[int, int]) -> None: observed = path.stat(follow_symlinks=False) _require((observed.st_dev, observed.st_ino) == identity and stat.S_ISREG(observed.st_mode), "upload was replaced") path.unlink() def _ensure_root_directory( path: Path, mode: int, *, expected_uid: int = 0, trusted_parent: Path = Path("/var/lib"), ) -> None: _require(path.is_absolute() and path.is_relative_to(trusted_parent), "request directory escaped its fixed root") missing: list[Path] = [] cursor = path while not cursor.exists(): _require(not cursor.is_symlink(), "request directory path was unsafe") missing.append(cursor) cursor = cursor.parent observed = cursor.stat(follow_symlinks=False) _require(stat.S_ISDIR(observed.st_mode), "request directory parent was unsafe") for directory in reversed(missing): directory.mkdir(mode=mode if directory == path else 0o755) os.chown(directory, expected_uid, os.getgid() if expected_uid != 0 else 0) observed = path.stat(follow_symlinks=False) _require( stat.S_ISDIR(observed.st_mode) and observed.st_uid == expected_uid and observed.st_mode & 0o022 == 0, "request directory was unsafe", ) if path in missing: path.chmod(mode) cursor = path while True: observed = cursor.stat(follow_symlinks=False) _require( stat.S_ISDIR(observed.st_mode) and observed.st_uid == expected_uid and observed.st_mode & 0o022 == 0, "request directory ancestor was unsafe", ) if cursor == trusted_parent: break cursor = cursor.parent def _acquire_lock(path: Path = LOCK_PATH, *, expected_uid: int = 0) -> int: flags = os.O_RDWR | os.O_CREAT if hasattr(os, "O_CLOEXEC"): flags |= os.O_CLOEXEC if hasattr(os, "O_NOFOLLOW"): flags |= os.O_NOFOLLOW try: descriptor = os.open(path, flags, 0o600) observed = os.fstat(descriptor) named = path.stat(follow_symlinks=False) _require( stat.S_ISREG(observed.st_mode) and observed.st_uid == expected_uid and stat.S_IMODE(observed.st_mode) == 0o600 and observed.st_nlink == 1 and (observed.st_dev, observed.st_ino) == (named.st_dev, named.st_ino), "request lock was unsafe", ) fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB) return descriptor except (OSError, RemoteError) as exc: if "descriptor" in locals(): os.close(descriptor) raise RemoteError("another request is active or the request lock was unsafe") from exc def _copy_into_request(path: Path, content: bytes, mode: int = 0o600) -> None: _require(mode in {0o600, 0o644, 0o755}, "request file mode was invalid") flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL if hasattr(os, "O_NOFOLLOW"): flags |= os.O_NOFOLLOW try: descriptor = os.open(path, flags, mode) with os.fdopen(descriptor, "wb") as handle: # receive_request() deliberately runs under umask 077. Apply the # manifest-bound mode explicitly so public release metadata and # executable helpers retain their reviewed 0644/0755 posture. os.fchmod(handle.fileno(), mode) handle.write(content) handle.flush() os.fsync(handle.fileno()) observed = os.fstat(handle.fileno()) named = path.stat(follow_symlinks=False) _require( stat.S_ISREG(observed.st_mode) and (observed.st_dev, observed.st_ino) == (named.st_dev, named.st_ino) and observed.st_uid == os.geteuid() and stat.S_IMODE(observed.st_mode) == mode and stat.S_IMODE(named.st_mode) == mode and observed.st_nlink == 1 and observed.st_size == len(content), "request file posture differed from its manifest", ) except OSError as exc: raise RemoteError("request file could not be secured") from exc def _claim_request_directory( run_root: Path, request_id: str, *, expected_uid: int = 0, trusted_parent: Path = Path("/var/lib"), ) -> Path: _require(REQUEST_ID.fullmatch(request_id) is not None, "request ID was invalid") _ensure_root_directory(run_root, 0o700, expected_uid=expected_uid, trusted_parent=trusted_parent) request_dir = run_root / request_id _require(not request_dir.exists() and not request_dir.is_symlink(), "request ID was already used") request_dir.mkdir(mode=0o700) os.chown(request_dir, expected_uid, os.getgid() if expected_uid != 0 else 0) return request_dir def _cleanup_upload_if_exact(path: Path, identity: tuple[int, int]) -> None: try: observed = path.stat(follow_symlinks=False) if stat.S_ISREG(observed.st_mode) and (observed.st_dev, observed.st_ino) == identity: path.unlink() except OSError: pass def _validate_member_name(name: str) -> None: path = Path(name) _require( name and not name.startswith("/") and not name.endswith("/") and all(part not in {"", ".", ".."} for part in path.parts), "archive member path was unsafe", ) def extract_archive(archive_path: Path, destination: Path) -> dict[str, Any]: _require(archive_path.stat().st_size <= MAX_ARCHIVE_BYTES, "request archive exceeded the safety limit") try: with tarfile.open(archive_path, mode="r:gz") as archive: return _extract_open_archive(archive, destination) except (OSError, tarfile.TarError) as exc: raise RemoteError("request archive was invalid") from exc def _extract_open_archive(archive: tarfile.TarFile, destination: Path) -> dict[str, Any]: members = archive.getmembers() names = [member.name for member in members] _require(len(names) == len(set(names)) and set(names) == EXPECTED_MEMBERS, "archive member set was not exact") for member in members: _validate_member_name(member.name) _require( member.isfile() and member.size <= MAX_MEMBER_BYTES and member.uid == 0 and member.gid == 0 and member.uname == "root" and member.gname == "root" and member.mtime == 0 and stat.S_IMODE(member.mode) in {0o600, 0o644, 0o755}, "archive member metadata was unsafe", ) manifest_member = archive.getmember("request-manifest.json") _require(manifest_member.size <= MAX_MANIFEST_BYTES, "request manifest exceeded the safety limit") manifest_stream = archive.extractfile(manifest_member) _require(manifest_stream is not None, "request manifest was absent") manifest = _strict_json(manifest_stream.read(), "request manifest", MAX_MANIFEST_BYTES) files = manifest.get("files") _require(isinstance(files, list), "request manifest file list was invalid") file_map: dict[str, dict[str, Any]] = {} for item in files: _require( isinstance(item, dict) and set(item) == {"path", "sha256", "size", "mode"}, "request manifest file record was invalid", ) relative = item.get("path") _require(isinstance(relative, str) and relative not in file_map, "request manifest file path was invalid") file_map[relative] = item _require(set(file_map) == EXPECTED_MEMBERS - {"request-manifest.json"}, "request manifest files were not exact") for relative in sorted(file_map): member = archive.getmember(relative) stream = archive.extractfile(member) _require(stream is not None, "request member was absent") content = stream.read(MAX_MEMBER_BYTES + 1) record = file_map[relative] _require( len(content) == member.size == record.get("size") and stat.S_IMODE(member.mode) == record.get("mode") and _sha256_bytes(content) == record.get("sha256"), "request member differed from its manifest", ) target = destination / relative target.parent.mkdir(parents=True, exist_ok=True, mode=0o700) _require(target.resolve().is_relative_to(destination.resolve()), "request member escaped extraction root") _copy_into_request(target, content, stat.S_IMODE(member.mode)) return manifest def validate_manifest( manifest: Mapping[str, Any], *, operation: str, request_id: str, runner_revision: str, release_revision: str, ) -> None: _require( set(manifest) == { "schema", "operation", "request_id", "runner_revision", "release_revision", "target", "release", "installer_source", "release_artifact", "files", "manifest_sha256", }, "request manifest fields were not exact", ) _require( manifest.get("schema") == SCHEMA and manifest.get("operation") == operation and manifest.get("request_id") == request_id and manifest.get("runner_revision") == runner_revision and manifest.get("release_revision") == release_revision, "request manifest context differed", ) _require( manifest.get("target") == { "project": PROJECT, "zone": ZONE, "instance": INSTANCE, "service": SERVICE, "service_account": SERVICE_ACCOUNT, }, "request target differed", ) stable = {key: value for key, value in manifest.items() if key != "manifest_sha256"} _require(manifest.get("manifest_sha256") == _canonical_sha256(stable), "request manifest hash differed") release = manifest.get("release") _require( isinstance(release, dict) and set(release) == {"release_sha256", "image_reference", "image_digest", "image_config_digest"} and HEX_64.fullmatch(str(release.get("release_sha256"))) is not None and IMAGE_REFERENCE.fullmatch(str(release.get("image_reference"))) is not None and re.fullmatch(r"sha256:[0-9a-f]{64}", str(release.get("image_digest"))) is not None and re.fullmatch(r"sha256:[0-9a-f]{64}", str(release.get("image_config_digest"))) is not None, "request release binding was invalid", ) _require( release["image_reference"] == f"{IMAGE_REPOSITORY}@{release['image_digest']}", "request image digest binding differed", ) artifact = manifest.get("release_artifact") _require( isinstance(artifact, dict) and set(artifact) == {"archive_sha256", "archive_size", "members"} and HEX_64.fullmatch(str(artifact.get("archive_sha256"))) is not None and isinstance(artifact.get("archive_size"), int) and 0 < artifact["archive_size"] <= MAX_ARCHIVE_BYTES and isinstance(artifact.get("members"), list), "release artifact binding was invalid", ) def validate_original_release_artifact( artifact_path: Path, *, manifest: Mapping[str, Any], payload_root: Path, ) -> None: raw = artifact_path.read_bytes() binding = manifest["release_artifact"] _require( len(raw) == binding["archive_size"] and _sha256_bytes(raw) == binding["archive_sha256"], "original release artifact archive differed", ) try: archive = zipfile.ZipFile(io.BytesIO(raw), mode="r") except (OSError, zipfile.BadZipFile) as exc: raise RemoteError("original release artifact archive was invalid") from exc with archive: infos = archive.infolist() names = [info.filename for info in infos] _require( len(names) == len(set(names)) and set(names) == RELEASE_ARTIFACT_MEMBERS, "release artifact members differed", ) observed: list[dict[str, Any]] = [] payloads: dict[str, bytes] = {} for info in infos: unix_mode = (info.external_attr >> 16) & 0o177777 _require( not info.is_dir() and info.file_size <= MAX_MEMBER_BYTES and (unix_mode == 0 or stat.S_ISREG(unix_mode)), "release artifact member was unsafe", ) content = archive.read(info) payloads[info.filename] = content observed.append({"path": info.filename, "sha256": _sha256_bytes(content), "size": len(content)}) _require( sorted(observed, key=lambda item: str(item["path"])) == binding["members"], "release artifact member receipt differed", ) _require( payloads["final/release-v3/release.json"] == (payload_root / "release/release.json").read_bytes() and payloads[f"final/release-v3/{SERVICE}"] == (payload_root / f"release/{SERVICE}").read_bytes(), "release artifact differed from extracted release", ) def _metadata(path: str, *, runner: Runner) -> bytes: return _run( [ "/usr/bin/curl", "--fail", "--silent", "--show-error", "--max-time", "5", "--header", "Metadata-Flavor: Google", f"{METADATA}/{path}", ], runner=runner, label="metadata read", timeout=10, ).stdout def verify_service_account(*, runner: Runner) -> None: try: email = _metadata("instance/service-accounts/default/email", runner=runner).decode("ascii", "strict").strip() project = _metadata("project/project-id", runner=runner).decode("ascii", "strict").strip() instance = _metadata("instance/name", runner=runner).decode("ascii", "strict").strip() zone = _metadata("instance/zone", runner=runner).decode("ascii", "strict").strip() except UnicodeError as exc: raise RemoteError("VM target identity was invalid") from exc _require( email == SERVICE_ACCOUNT and project == PROJECT and instance == INSTANCE and zone.endswith(f"/zones/{ZONE}"), "VM target or service account identity differed", ) def _docker_prefix(config: Path) -> list[str]: return [DOCKER, f"--host={DOCKER_HOST}", f"--config={config}"] def pull_immutable_image(reference: str, request_dir: Path, *, runner: Runner) -> None: _require(IMAGE_REFERENCE.fullmatch(reference) is not None, "image reference was not immutable") token_receipt = _strict_json( _metadata("instance/service-accounts/default/token", runner=runner), "metadata token", MAX_TOKEN_BYTES, ) token = token_receipt.get("access_token") _require( isinstance(token, str) and 16 <= len(token) <= 8192 and "\n" not in token and token_receipt.get("token_type") == "Bearer", "metadata token response was invalid", ) config = Path(tempfile.mkdtemp(prefix="docker-auth-", dir=request_dir)) config.chmod(0o700) try: _run( [ *_docker_prefix(config), "login", "--username=oauth2accesstoken", "--password-stdin", "https://europe-west6-docker.pkg.dev", ], runner=runner, label="registry login", input_bytes=(token + "\n").encode("utf-8"), ) _run( [*_docker_prefix(config), "image", "pull", reference], runner=runner, label="immutable image pull", timeout=300, ) finally: try: _run( [*_docker_prefix(config), "logout", "https://europe-west6-docker.pkg.dev"], runner=runner, label="registry logout", ) except RemoteError: pass shutil.rmtree(config, ignore_errors=True) _require(not config.exists(), "ephemeral Docker credentials were not removed") def _load_contract_modules(payload_root: Path) -> tuple[Any, Any]: sys.path.insert(0, str(payload_root)) try: from ops import gcp_leoclean_nosend_package as package from ops import install_gcp_leoclean_nosend_release as installer except Exception as exc: raise RemoteError("reviewed installer sources could not be loaded") from exc _require( package.PROJECT == PROJECT and package.ZONE == ZONE and package.INSTANCE == INSTANCE and package.SERVICE == SERVICE and package.SERVICE_ACCOUNT == SERVICE_ACCOUNT, "reviewed installer target differed", ) return package, installer def _verify_database( *, container_id: str, run_id: str, receipt_name: str, request_dir: Path, package_module: Any, runner: Runner, ) -> dict[str, Any]: completed = _run( [ DOCKER, f"--host={DOCKER_HOST}", f"--config={SERVICE_DOCKER_CONFIG}", "container", "exec", "--user=0", container_id, "/opt/livingip/leoclean-nosend/venv/bin/python", "/opt/livingip/leoclean-nosend/entrypoint.py", "verify-database", run_id, ], runner=runner, label="container database verification", timeout=300, ) receipt = _strict_json(completed.stdout, "database verification receipt", MAX_COMMAND_OUTPUT_BYTES) _require( receipt.get("schema") == "livingip.leocleanNoSendContainerDatabaseVerification.v1" and receipt.get("status") == "pass" and receipt.get("run_id") == run_id, "database verification receipt did not pass", ) stable = {key: value for key, value in receipt.items() if key != "receipt_sha256"} _require(receipt.get("receipt_sha256") == package_module.canonical_sha256(stable), "database receipt hash differed") _require(re.fullmatch(r"[a-z][a-z0-9-]{1,62}\.json", receipt_name) is not None, "receipt name was invalid") receipt_path = request_dir / receipt_name _copy_into_request( receipt_path, (json.dumps(receipt, allow_nan=False, indent=2, sort_keys=True) + "\n").encode("utf-8"), 0o600, ) return { "schema": receipt["schema"], "status": "pass", "receipt_sha256": receipt["receipt_sha256"], "receipt_path": str(receipt_path), "proof": { "proposal_staging": "function_only", "proposal_transaction": "rolled_back", "durable_proposal_created": False, "canonical_direct_writes": "denied", "stage_direct_writes": "denied", "role_escalation": "denied", }, } def _execute_explicit_rollback( *, installer_module: Any, release_sha256: str, paths: Any, source_binding: Mapping[str, Any], runner: Runner, ) -> dict[str, Any]: """Call only the receipt-bound explicit rollback API; never model rollback as another install.""" rollback = getattr(installer_module, "rollback_release", None) _require(callable(rollback), "reviewed installer lacked the explicit rollback interface") executed = False def contract_runner(argv: list[str]) -> subprocess.CompletedProcess[bytes]: return _run( argv, runner=runner, label="rollback command", require_success=False, ) try: plan = rollback( release_sha256, paths=paths, source_binding=source_binding, runner=contract_runner, execute_rollback=False, ) _require( isinstance(plan, dict) and plan.get("status") == "pass" and plan.get("mode") == "dry_run" and plan.get("expected_release_sha256") == release_sha256, "explicit rollback preflight did not pass", ) executed = True result = rollback( release_sha256, paths=paths, source_binding=source_binding, runner=contract_runner, execute_rollback=True, ) _require(isinstance(result, dict) and result.get("status") == "pass", "explicit rollback did not pass") return result except Exception as exc: raise ExplicitRollbackFailure(executed=executed) from exc def _rollback_result( *, manifest: Mapping[str, Any], release: Mapping[str, Any], rollback: Mapping[str, Any], archive_sha256: str, helper_sha256: str, ) -> dict[str, Any]: posture = rollback.get("restored_service_posture") restored_release = rollback.get("restored_release_sha256") verification = rollback.get("verification") _require( rollback.get("schema") == "livingip.leocleanNoSendRollbackResult.v1" and rollback.get("status") == "pass" and rollback.get("mode") == "executed" and rollback.get("rolled_back_release_sha256") == release["release_sha256"] and posture in {"active", "inactive", "absent"}, "explicit rollback result was invalid", ) if posture == "active": _require( isinstance(restored_release, str) and HEX_64.fullmatch(restored_release) is not None and isinstance(verification, dict) and verification.get("active_state") == "active" and verification.get("sub_state") == "running" and isinstance(verification.get("container_id"), str) and HEX_64.fullmatch(verification["container_id"]) is not None, "active rollback result lacked exact restored verification", ) elif posture == "inactive": _require( isinstance(restored_release, str) and HEX_64.fullmatch(restored_release) is not None and verification is None, "inactive rollback result was invalid", ) else: _require(restored_release is None and verification is None, "absent rollback result was invalid") return { "schema": RESULT_SCHEMA, "status": "pass", "operation": "rollback", "request_id": manifest["request_id"], "runner_revision": manifest["runner_revision"], "release_revision": manifest["release_revision"], "release_sha256": release["release_sha256"], "image_digest": release["image"]["digest"], "image_config_digest": release["image"]["config_digest"], "release_artifact_sha256": manifest["release_artifact"]["archive_sha256"], "request_binding": { "archive_sha256": archive_sha256, "helper_sha256": helper_sha256, "manifest_sha256": manifest["manifest_sha256"], }, "service": { "name": SERVICE, "restored_posture": posture, "restored_release_sha256": restored_release, "verification": verification, }, "database": {"status": "not_run_after_explicit_rollback"}, "authority": { "sending_enabled": False, "production_traffic_changed": False, "vps_touched": False, "canonical_proposal_approved": False, }, } def _verification_failure_result( *, manifest: Mapping[str, Any], release: Mapping[str, Any], failure: Exception, rollback: Mapping[str, Any], archive_sha256: str, helper_sha256: str, installer_module: Any, ) -> dict[str, Any]: failure_stage = getattr(failure, "failure_stage", None) failure_reason = getattr(failure, "failure_reason", None) _require( isinstance(failure_stage, str) and isinstance(failure_reason, str) and (failure_stage, failure_reason) in installer_module.VERIFICATION_FAILURES, "verification failure was not allowlisted", ) _require( dict(rollback) == _rollback_summary( status=str(rollback.get("status")), mode=str(rollback.get("mode")), executed=rollback.get("executed") is True, restored_posture=str(rollback.get("restored_posture")), prior_posture_preserved=rollback.get("prior_posture_preserved") is True, ), "verification rollback summary was invalid", ) return { "schema": FAILURE_RESULT_SCHEMA, "status": "fail", "operation": manifest["operation"], "request_id": manifest["request_id"], "runner_revision": manifest["runner_revision"], "release_revision": manifest["release_revision"], "release_sha256": release["release_sha256"], "image_digest": release["image"]["digest"], "image_config_digest": release["image"]["config_digest"], "release_artifact_sha256": manifest["release_artifact"]["archive_sha256"], "request_binding": { "archive_sha256": archive_sha256, "helper_sha256": helper_sha256, "manifest_sha256": manifest["manifest_sha256"], }, "failure_stage": failure_stage, "failure_reason": failure_reason, "rollback": dict(rollback), "authority": { "sending_enabled": False, "production_traffic_changed": False, "vps_touched": False, "canonical_proposal_approved": False, }, } def _installer_failure_rollback(failure: Exception) -> dict[str, Any]: rollback_executed = getattr(failure, "rollback_executed", None) restored_posture = getattr(failure, "restored_posture", None) prior_posture_preserved = getattr(failure, "prior_posture_preserved", None) _require( isinstance(rollback_executed, bool) and restored_posture in {"active", "inactive", "absent"} and prior_posture_preserved is True, "installer failure lacked a proven recovery posture", ) return _rollback_summary( status="pass" if rollback_executed else "not_run", mode="installer_automatic" if rollback_executed else "not_applicable", executed=rollback_executed, restored_posture=str(restored_posture) if rollback_executed else "unchanged", prior_posture_preserved=True, ) def _is_verification_failure(installer_module: Any, failure: Exception) -> bool: failure_type = getattr(installer_module, "VerificationError", None) return isinstance(failure_type, type) and isinstance(failure, failure_type) def _post_install_failure_coordinates( installer_module: Any, failure: Exception, *, fallback_stage: str, fallback_reason: str, ) -> tuple[str, str]: if _is_verification_failure(installer_module, failure): failure_stage = getattr(failure, "failure_stage", None) failure_reason = getattr(failure, "failure_reason", None) _require( isinstance(failure_stage, str) and isinstance(failure_reason, str) and (failure_stage, failure_reason) in installer_module.VERIFICATION_FAILURES, "verification failure was not allowlisted", ) return failure_stage, failure_reason _require( (fallback_stage, fallback_reason) in POST_INSTALL_FAILURES, "post-install failure was not allowlisted", ) return fallback_stage, fallback_reason def _rollback_proven_service(posture: str) -> dict[str, Any]: _require(posture in {"active", "inactive", "absent"}, "rollback posture was invalid") return { "status": "not_required", "service_posture": posture, "active_state": "not_observed", "sub_state": "not_observed", "main_pid": None, "n_restarts": None, "candidate_container": "not_observed", "service_action_executed": False, } def _fail_closed_inert_service( *, installer_module: Any, release: Mapping[str, Any], paths: Any, runner: Runner, ) -> dict[str, Any]: service_action_executed = False def contract_runner(argv: list[str]) -> subprocess.CompletedProcess[bytes]: return _run( argv, runner=runner, label="fail-closed service command", require_success=False, ) try: systemctl = getattr(installer_module, "_systemctl", None) stable_service_posture = getattr(installer_module, "_stable_service_posture", None) verify_running_release = getattr(installer_module, "verify_running_release", None) require_candidate_absent = getattr(installer_module, "_require_candidate_absent", None) _require( callable(systemctl) and callable(stable_service_posture) and callable(verify_running_release) and callable(require_candidate_absent), "reviewed installer lacked fail-closed service controls", ) before = stable_service_posture( release, runner=contract_runner, paths=paths, sleeper=time.sleep, ) _require(before.kind in {"active", "inactive"}, "managed service posture was not trusted") if before.kind == "active": verification = verify_running_release( release, runner=contract_runner, paths=paths, sleeper=time.sleep, ) _require( verification.get("main_pid") == before.state.get("MainPID") and verification.get("invocation_id") == before.state.get("InvocationID") and verification.get("n_restarts") == before.state.get("NRestarts"), "running service identity changed before fail-closed stop", ) service_action_executed = True systemctl("stop", contract_runner) after = stable_service_posture( release, runner=contract_runner, paths=paths, sleeper=time.sleep, ) _require( after.kind == "inactive" and after.state.get("ActiveState") == "inactive" and after.state.get("SubState") == "dead" and after.state.get("MainPID") == 0 and isinstance(after.state.get("NRestarts"), int) and not isinstance(after.state.get("NRestarts"), bool) and after.state["NRestarts"] >= 0, "service did not stop into a stable inert posture", ) require_candidate_absent( runner=contract_runner, paths=paths, sleeper=time.sleep, ) return { "status": "pass", "service_posture": "inactive", "active_state": "inactive", "sub_state": "dead", "main_pid": 0, "n_restarts": after.state["NRestarts"], "candidate_container": "absent", "service_action_executed": service_action_executed, } except Exception: return { "status": "fail", "service_posture": "unknown", "active_state": "unknown", "sub_state": "unknown", "main_pid": None, "n_restarts": None, "candidate_container": "unknown", "service_action_executed": service_action_executed, } def _restore_transition_posture( *, installer_module: Any, release: Mapping[str, Any], transition: Mapping[str, Any], paths: Any, runner: Runner, ) -> dict[str, Any]: checked = _validate_installer_transition(transition) desired = str(checked["prior_posture"]) _require( checked["outcome"] in {"verified_noop", "controlled_start"} and desired in {"active", "inactive"}, "installer transition required release rollback", ) def contract_runner(argv: list[str]) -> subprocess.CompletedProcess[bytes]: return _run( argv, runner=runner, label="prior posture recovery command", require_success=False, ) systemctl = getattr(installer_module, "_systemctl", None) stable_service_posture = getattr(installer_module, "_stable_service_posture", None) verify_running_release = getattr(installer_module, "verify_running_release", None) require_candidate_absent = getattr(installer_module, "_require_candidate_absent", None) _require( callable(systemctl) and callable(stable_service_posture) and callable(verify_running_release) and callable(require_candidate_absent), "reviewed installer lacked posture recovery controls", ) executed = False try: observed = stable_service_posture( release, runner=contract_runner, paths=paths, sleeper=time.sleep, ) _require(observed.kind in {"active", "inactive"}, "managed service posture was not trusted") if desired == "inactive": if observed.kind == "active": verification = verify_running_release( release, runner=contract_runner, paths=paths, sleeper=time.sleep, ) _require( verification.get("main_pid") == observed.state.get("MainPID") and verification.get("invocation_id") == observed.state.get("InvocationID") and verification.get("n_restarts") == observed.state.get("NRestarts"), "running service identity changed before posture recovery stop", ) executed = True systemctl("stop", contract_runner) final = stable_service_posture( release, runner=contract_runner, paths=paths, sleeper=time.sleep, ) _require( final.kind == "inactive" and final.state.get("NRestarts") == checked["prior_n_restarts"], "prior inactive posture was not restored", ) require_candidate_absent(runner=contract_runner, paths=paths, sleeper=time.sleep) else: if observed.kind == "inactive": require_candidate_absent(runner=contract_runner, paths=paths, sleeper=time.sleep) executed = True systemctl("start", contract_runner) final = stable_service_posture( release, runner=contract_runner, paths=paths, sleeper=time.sleep, ) _require( final.kind == "active" and final.state.get("NRestarts") == checked["prior_n_restarts"], "prior active posture was not restored", ) verification = verify_running_release( release, runner=contract_runner, paths=paths, sleeper=time.sleep, ) _require( verification.get("main_pid") == final.state.get("MainPID") and verification.get("invocation_id") == final.state.get("InvocationID") and verification.get("n_restarts") == checked["prior_n_restarts"], "restored active service identity was unstable", ) except Exception as exc: raise PostureRecoveryFailure(executed=executed) from exc return _rollback_summary( status="pass", mode="posture_recovery", executed=executed, restored_posture=desired, prior_posture_preserved=True, ) def _operational_failure_result( *, manifest: Mapping[str, Any], release: Mapping[str, Any], failure_stage: str, failure_reason: str, rollback: Mapping[str, Any], fail_closed: Mapping[str, Any], archive_sha256: str, helper_sha256: str, installer_module: Any, ) -> dict[str, Any]: _require( manifest.get("operation") == "install" and set(rollback) == {"status", "mode", "executed", "restored_posture", "prior_posture_preserved"} and ( (failure_stage, failure_reason) in POST_INSTALL_FAILURES or (failure_stage, failure_reason) in installer_module.VERIFICATION_FAILURES ), "post-install failure was not allowlisted", ) rollback_passed = ( rollback.get("status") == "pass" and rollback.get("mode") in {"explicit", "posture_recovery"} and isinstance(rollback.get("executed"), bool) and rollback.get("restored_posture") in {"active", "inactive", "absent"} and ( rollback.get("mode") == "explicit" or rollback.get("restored_posture") in {"active", "inactive"} ) and rollback.get("prior_posture_preserved") is True ) rollback_failed = ( rollback.get("status") == "fail" and rollback.get("mode") in {"explicit", "posture_recovery"} and isinstance(rollback.get("executed"), bool) and rollback.get("restored_posture") == "unknown" and rollback.get("prior_posture_preserved") is False ) verification_pair = (failure_stage, failure_reason) in installer_module.VERIFICATION_FAILURES _require( (rollback_passed and not verification_pair) or rollback_failed, "post-install rollback summary was invalid", ) if rollback_passed: _require( fail_closed == _rollback_proven_service(str(rollback["restored_posture"])), "post-install restored posture summary was invalid", ) else: _require( fail_closed in ( { "status": "fail", "service_posture": "unknown", "active_state": "unknown", "sub_state": "unknown", "main_pid": None, "n_restarts": None, "candidate_container": "unknown", "service_action_executed": fail_closed.get("service_action_executed"), }, { "status": "pass", "service_posture": "inactive", "active_state": "inactive", "sub_state": "dead", "main_pid": 0, "n_restarts": fail_closed.get("n_restarts"), "candidate_container": "absent", "service_action_executed": fail_closed.get("service_action_executed"), }, ) and ( fail_closed.get("status") != "pass" or ( isinstance(fail_closed.get("n_restarts"), int) and not isinstance(fail_closed.get("n_restarts"), bool) and fail_closed["n_restarts"] >= 0 ) ) and isinstance(fail_closed.get("service_action_executed"), bool), "post-install fail-closed summary was invalid", ) return { "schema": OPERATION_FAILURE_RESULT_SCHEMA, "status": "fail", "operation": manifest["operation"], "request_id": manifest["request_id"], "runner_revision": manifest["runner_revision"], "release_revision": manifest["release_revision"], "release_sha256": release["release_sha256"], "image_digest": release["image"]["digest"], "image_config_digest": release["image"]["config_digest"], "release_artifact_sha256": manifest["release_artifact"]["archive_sha256"], "request_binding": { "archive_sha256": archive_sha256, "helper_sha256": helper_sha256, "manifest_sha256": manifest["manifest_sha256"], }, "failure_stage": failure_stage, "failure_reason": failure_reason, "rollback": dict(rollback), "fail_closed": dict(fail_closed), "authority": { "sending_enabled": False, "production_traffic_changed": False, "vps_touched": False, "canonical_proposal_approved": False, }, } def execute_request( manifest: Mapping[str, Any], payload_root: Path, request_dir: Path, archive_sha256: str, helper_sha256: str, *, runner: Runner = subprocess.run, ) -> dict[str, Any]: operation = str(manifest["operation"]) release_binding = manifest["release"] validate_original_release_artifact( payload_root / "release-artifact.zip", manifest=manifest, payload_root=payload_root, ) package_module, installer_module = _load_contract_modules(payload_root) bundle = payload_root / "release" release, unit = installer_module.validate_release_bundle(bundle) _require( release["release_sha256"] == release_binding["release_sha256"] and release["image"]["reference"] == release_binding["image_reference"] and release["image"]["digest"] == release_binding["image_digest"] and release["image"]["config_digest"] == release_binding["image_config_digest"] and unit == package_module.render_systemd_unit(release).encode("utf-8"), "extracted release differed from the request", ) source_binding = manifest["installer_source"] installer_module._validate_source_binding(source_binding) _require( source_binding.get("revision") == manifest["runner_revision"], "installer source revision differed from runner revision", ) _require( release["image_input"]["runtime"]["teleo_git_head"] == manifest["release_revision"], "release source revision differed from release revision", ) paths = installer_module.InstallPaths.under(Path("/")) if operation == "install": pull_immutable_image(release["image"]["reference"], request_dir, runner=runner) try: install_receipt = installer_module.install_release( bundle, paths=paths, source_binding=source_binding, runner=lambda argv: _run( argv, runner=runner, label="installer command", require_success=False, ), execute_restart=True, ) except Exception as exc: if not _is_verification_failure(installer_module, exc): raise raise RemoteOperationFailure( _verification_failure_result( manifest=manifest, release=release, failure=exc, rollback=_installer_failure_rollback(exc), archive_sha256=archive_sha256, helper_sha256=helper_sha256, installer_module=installer_module, ) ) from exc elif operation == "rollback": rollback = _execute_explicit_rollback( installer_module=installer_module, release_sha256=release["release_sha256"], paths=paths, source_binding=source_binding, runner=runner, ) return _rollback_result( manifest=manifest, release=release, rollback=rollback, archive_sha256=archive_sha256, helper_sha256=helper_sha256, ) def contract_runner(argv: list[str]) -> subprocess.CompletedProcess[bytes]: return _run( argv, runner=runner, label="release verification command", require_success=False, ) if operation == "install": failure_stage = "install_receipt" failure_reason = "install_receipt_invalid" installer_transition: dict[str, Any] | None = None try: installer_transition = _validate_install_execution_result( install_receipt, release=release, installer_module=installer_module, ) failure_stage = "service_before_restart" failure_reason = "service_verification_failed" before = installer_module.verify_running_release( release, runner=contract_runner, paths=paths, ) failure_stage = "database_before_restart" failure_reason = "database_verification_failed" before_database = _verify_database( container_id=before["container_id"], run_id=f"{manifest['request_id']}-pre", receipt_name="database-before-restart.json", request_dir=request_dir, package_module=package_module, runner=runner, ) failure_stage = "service_restart" failure_reason = "service_restart_failed" _run( [installer_module.SYSTEMCTL, "restart", SERVICE], runner=runner, label="fixed service restart", ) failure_stage = "service_after_restart" failure_reason = "service_verification_failed" after = installer_module.verify_running_release( release, runner=contract_runner, paths=paths, ) failure_stage = "restart_identity" failure_reason = "restart_identity_mismatch" _require( before["main_pid"] != after["main_pid"] and before["invocation_id"] != after["invocation_id"] and before["container_id"] != after["container_id"] and after["n_restarts"] >= before["n_restarts"], "post-install restart did not create a new exact service/container identity", ) failure_stage = "database_after_restart" failure_reason = "database_verification_failed" after_database = _verify_database( container_id=after["container_id"], run_id=f"{manifest['request_id']}-post", receipt_name="database-after-restart.json", request_dir=request_dir, package_module=package_module, runner=runner, ) except Exception as exc: bounded_stage, bounded_reason = _post_install_failure_coordinates( installer_module, exc, fallback_stage=failure_stage, fallback_reason=failure_reason, ) if installer_transition is None: fail_closed = _fail_closed_inert_service( installer_module=installer_module, release=release, paths=paths, runner=runner, ) raise RemoteOperationFailure( _operational_failure_result( manifest=manifest, release=release, failure_stage="install_receipt", failure_reason="install_receipt_invalid", rollback=_rollback_summary( status="fail", mode="posture_recovery", executed=bool(fail_closed["service_action_executed"]), restored_posture="unknown", prior_posture_preserved=False, ), fail_closed=fail_closed, archive_sha256=archive_sha256, helper_sha256=helper_sha256, installer_module=installer_module, ) ) from exc release_changing = installer_transition["outcome"] in {"installed", "replaced"} recovery_mode = "explicit" if release_changing else "posture_recovery" recovery_executed = False try: if release_changing: try: recovery = _execute_explicit_rollback( installer_module=installer_module, release_sha256=release["release_sha256"], paths=paths, source_binding=source_binding, runner=runner, ) except ExplicitRollbackFailure as rollback_exc: recovery_executed = rollback_exc.executed raise recovery_executed = True rollback_result = _rollback_result( manifest=manifest, release=release, rollback=recovery, archive_sha256=archive_sha256, helper_sha256=helper_sha256, ) rollback_summary = _rollback_summary( status="pass", mode="explicit", executed=True, restored_posture=str(rollback_result["service"]["restored_posture"]), prior_posture_preserved=True, ) else: try: rollback_summary = _restore_transition_posture( installer_module=installer_module, release=release, transition=installer_transition, paths=paths, runner=runner, ) except PostureRecoveryFailure as posture_exc: recovery_executed = posture_exc.executed raise recovery_executed = bool(rollback_summary["executed"]) except Exception: fail_closed = _fail_closed_inert_service( installer_module=installer_module, release=release, paths=paths, runner=runner, ) raise RemoteOperationFailure( _operational_failure_result( manifest=manifest, release=release, failure_stage=bounded_stage, failure_reason=bounded_reason, rollback=_rollback_summary( status="fail", mode=recovery_mode, executed=recovery_executed, restored_posture="unknown", prior_posture_preserved=False, ), fail_closed=fail_closed, archive_sha256=archive_sha256, helper_sha256=helper_sha256, installer_module=installer_module, ) ) from exc if _is_verification_failure(installer_module, exc): raise RemoteOperationFailure( _verification_failure_result( manifest=manifest, release=release, failure=exc, rollback=rollback_summary, archive_sha256=archive_sha256, helper_sha256=helper_sha256, installer_module=installer_module, ) ) from exc restored_posture = str(rollback_summary["restored_posture"]) raise RemoteOperationFailure( _operational_failure_result( manifest=manifest, release=release, failure_stage=bounded_stage, failure_reason=bounded_reason, rollback=rollback_summary, fail_closed=_rollback_proven_service(restored_posture), archive_sha256=archive_sha256, helper_sha256=helper_sha256, installer_module=installer_module, ) ) from exc service = { "name": SERVICE, "installer_transition": installer_transition, "before_restart": before, "after_restart": after, } database = { "status": "pass", "before_restart": before_database, "after_restart": after_database, } else: try: verification = installer_module.verify_running_release( release, runner=contract_runner, paths=paths, ) except Exception as exc: if not _is_verification_failure(installer_module, exc): raise raise RemoteOperationFailure( _verification_failure_result( manifest=manifest, release=release, failure=exc, rollback=_rollback_summary( status="not_run", mode="not_applicable", executed=False, restored_posture="unchanged", prior_posture_preserved=True, ), archive_sha256=archive_sha256, helper_sha256=helper_sha256, installer_module=installer_module, ) ) from exc database = _verify_database( container_id=verification["container_id"], run_id=f"{manifest['request_id']}-verify", receipt_name="database-verify.json", request_dir=request_dir, package_module=package_module, runner=runner, ) service = { "name": SERVICE, "verification": verification, } return { "schema": RESULT_SCHEMA, "status": "pass", "operation": operation, "request_id": manifest["request_id"], "runner_revision": manifest["runner_revision"], "release_revision": manifest["release_revision"], "release_sha256": release["release_sha256"], "image_digest": release["image"]["digest"], "image_config_digest": release["image"]["config_digest"], "release_artifact_sha256": manifest["release_artifact"]["archive_sha256"], "request_binding": { "archive_sha256": archive_sha256, "helper_sha256": helper_sha256, "manifest_sha256": manifest["manifest_sha256"], }, "service": service, "database": database, "authority": { "sending_enabled": False, "production_traffic_changed": False, "vps_touched": False, "canonical_proposal_approved": False, }, } def _execute_and_persist_result( manifest: Mapping[str, Any], payload_root: Path, request_dir: Path, archive_sha256: str, helper_sha256: str, *, runner: Runner, ) -> dict[str, Any]: try: result = execute_request( manifest, payload_root, request_dir, archive_sha256, helper_sha256, runner=runner, ) except RemoteOperationFailure as exc: result = exc.result result_path = request_dir / "result.json" _copy_into_request( result_path, (json.dumps(result, allow_nan=False, indent=2, sort_keys=True) + "\n").encode("utf-8"), 0o600, ) return result def receive_request( operation: str, request_id: str, runner_revision: str, release_revision: str, archive_sha256: str, helper_sha256: str, *, runner: Runner = subprocess.run, run_root: Path = RUN_ROOT, lock_path: Path = LOCK_PATH, ) -> dict[str, Any]: _require(os.geteuid() == 0, "remote execution required root") _require(operation in OPERATIONS, "operation was not allowlisted") _require(REQUEST_ID.fullmatch(request_id) is not None, "request ID was invalid") _require(HEX_40.fullmatch(runner_revision) is not None, "runner revision was invalid") _require(HEX_40.fullmatch(release_revision) is not None, "release revision was invalid") _require(HEX_64.fullmatch(archive_sha256) is not None, "archive hash was invalid") _require(HEX_64.fullmatch(helper_sha256) is not None, "helper hash was invalid") caller = os.environ.get("SUDO_USER", "") _require(caller and caller != "root", "remote execution lacked a distinct OS Login caller") try: pwd.getpwnam(caller) except KeyError as exc: raise RemoteError("OS Login caller did not resolve") from exc stage_dir = STAGE_ROOT / request_id archive_upload = stage_dir / "request.tar.gz" helper_upload = stage_dir / "remote-helper.py" archive_stat = archive_upload.stat(follow_symlinks=False) helper_stat = helper_upload.stat(follow_symlinks=False) archive_identity = (archive_stat.st_dev, archive_stat.st_ino) helper_identity = (helper_stat.st_dev, helper_stat.st_ino) try: stage_state = stage_dir.stat(follow_symlinks=False) _require( stat.S_ISDIR(stage_state.st_mode) and stage_state.st_uid == 0 and stage_state.st_gid == 0 and stat.S_IMODE(stage_state.st_mode) == 0o700, "root-owned bootstrap stage was unsafe", ) helper_content = _read_stable_stage(helper_upload, helper_sha256, MAX_MEMBER_BYTES) archive_content = _read_stable_stage(archive_upload, archive_sha256, MAX_ARCHIVE_BYTES) _require(_sha256_bytes(Path(__file__).read_bytes()) == helper_sha256, "executing helper differed from request") lock_descriptor = _acquire_lock(lock_path) try: request_dir = _claim_request_directory(run_root, request_id) secured_archive = request_dir / "request.tar.gz" secured_helper = request_dir / "remote-helper.py" _copy_into_request(secured_archive, archive_content) _copy_into_request(secured_helper, helper_content) _unlink_exact(archive_upload, archive_identity) _unlink_exact(helper_upload, helper_identity) stage_dir.rmdir() payload_root = request_dir / "payload" payload_root.mkdir(mode=0o700) manifest = extract_archive(secured_archive, payload_root) validate_manifest( manifest, operation=operation, request_id=request_id, runner_revision=runner_revision, release_revision=release_revision, ) verify_service_account(runner=runner) return _execute_and_persist_result( manifest, payload_root, request_dir, archive_sha256, helper_sha256, runner=runner, ) finally: os.close(lock_descriptor) finally: _cleanup_upload_if_exact(archive_upload, archive_identity) _cleanup_upload_if_exact(helper_upload, helper_identity) def main(argv: list[str] | None = None) -> int: arguments = argv if argv is not None else sys.argv[1:] try: os.umask(0o077) _require(len(arguments) == 6, "remote arguments were not exact") result = receive_request(*arguments) sys.stdout.write(json.dumps(result, allow_nan=False, separators=(",", ":"), sort_keys=True) + "\n") return 0 if result.get("status") == "pass" else 65 except RemoteError as exc: error = str(exc) if error not in {"install_failed_rolled_back", "install_failed_rollback_incomplete"}: error = "remote_request_failed" sys.stderr.write(json.dumps({"error": error, "status": "fail"}, separators=(",", ":")) + "\n") return 65 except Exception: sys.stderr.write('{"error":"remote_request_failed","status":"fail"}\n') return 65 if __name__ == "__main__": raise SystemExit(main())