Merge pull request #239 from living-ip/fix/nosend-container-contract-receipt

Expose bounded container contract failures

ops/install_gcp_leoclean_nosend_release.py

@@ -121,7 +121,31 @@ VERIFICATION_FAILURES = frozenset(
         ("startup_identity", "container_id_invalid"),
         ("process_environment", "process_environment_mismatch"),
         ("supervising_process", "supervising_process_mismatch"),
-        ("container_startup", "container_contract_mismatch"),
+        ("container_inspection", "container_inspection_invalid"),
+        ("container_inspection", "container_config_invalid"),
+        ("container_inspection", "container_state_invalid"),
+        ("container_inspection", "container_host_config_invalid"),
+        ("container_identity", "container_id_mismatch"),
+        ("container_identity", "container_name_mismatch"),
+        ("container_image_identity", "container_image_id_mismatch"),
+        ("container_image_identity", "container_image_reference_mismatch"),
+        ("container_environment", "container_environment_mismatch"),
+        ("container_labels", "container_labels_invalid"),
+        ("container_labels", "container_release_label_mismatch"),
+        ("container_labels", "container_role_label_mismatch"),
+        ("container_security", "container_readonly_rootfs_mismatch"),
+        ("container_security", "container_network_mode_mismatch"),
+        ("container_security", "container_pids_limit_mismatch"),
+        ("container_security", "container_cap_drop_mismatch"),
+        ("container_security", "container_cap_add_mismatch"),
+        ("container_security", "container_security_options_mismatch"),
+        ("container_command", "container_path_mismatch"),
+        ("container_command", "container_args_mismatch"),
+        ("container_state", "container_running_flag_invalid"),
+        ("container_state", "container_pid_invalid"),
+        ("container_state", "container_exited_early"),
+        ("container_health", "container_health_missing"),
+        ("container_health", "container_health_status_invalid"),
         ("container_startup", "container_not_healthy"),
         ("image_identity", "image_contract_mismatch"),
         ("stable_container", "container_stability_mismatch"),
@@ -1051,48 +1075,148 @@ def _validate_starting_container(
 ) -> dict[str, Any] | None:
     config = value.get("Config")
     state = value.get("State")
-    health = state.get("Health") if isinstance(state, dict) else None
     host = value.get("HostConfig")
-    labels = config.get("Labels") if isinstance(config, dict) else None
     runtime = release["inspection"]["runtime_config"]
     container_id = value.get("Id")
-    _require(
-        container_id == expected_container_id
-        and HEX_64.fullmatch(expected_container_id) is not None
-        and value.get("Name") == f"/{package.CONTAINER_NAME}"
-        and value.get("Image") == release["image"]["config_digest"]
-        and isinstance(config, dict)
-        and config.get("Image") == release["image"]["reference"]
-        and config.get("Env") == runtime["environment"]
-        and isinstance(labels, dict)
-        and labels.get("livingip.release.sha256") == release["release_sha256"]
-        and labels.get("livingip.container.role") == "leoclean-gcp-nosend"
-        and isinstance(state, dict)
-        and isinstance(host, dict)
-        and host.get("ReadonlyRootfs") is True
-        and host.get("NetworkMode") == "bridge"
-        and host.get("PidsLimit") == 512
-        and sorted(host.get("CapDrop", [])) == ["ALL"]
-        and sorted(host.get("CapAdd", [])) == ["CHOWN", "SETGID", "SETPCAP", "SETUID"]
-        and host.get("SecurityOpt") == ["no-new-privileges:true"],
-        "running container differed from the release contract",
+    _verification_require(
+        container_id == expected_container_id and HEX_64.fullmatch(expected_container_id) is not None,
+        "container_identity",
+        "container_id_mismatch",
+    )
+    _verification_require(
+        value.get("Name") == f"/{package.CONTAINER_NAME}",
+        "container_identity",
+        "container_name_mismatch",
+    )
+    _verification_require(
+        value.get("Image") == release["image"]["config_digest"],
+        "container_image_identity",
+        "container_image_id_mismatch",
+    )
+    _verification_require(
+        isinstance(config, dict),
+        "container_inspection",
+        "container_config_invalid",
+    )
+    _verification_require(
+        config.get("Image") == release["image"]["reference"],
+        "container_image_identity",
+        "container_image_reference_mismatch",
+    )
+    _verification_require(
+        config.get("Env") == runtime["environment"],
+        "container_environment",
+        "container_environment_mismatch",
+    )
+    labels = config.get("Labels")
+    _verification_require(
+        isinstance(labels, dict),
+        "container_labels",
+        "container_labels_invalid",
+    )
+    _verification_require(
+        labels.get("livingip.release.sha256") == release["release_sha256"],
+        "container_labels",
+        "container_release_label_mismatch",
+    )
+    _verification_require(
+        labels.get("livingip.container.role") == "leoclean-gcp-nosend",
+        "container_labels",
+        "container_role_label_mismatch",
+    )
+    _verification_require(
+        isinstance(state, dict),
+        "container_inspection",
+        "container_state_invalid",
+    )
+    _verification_require(
+        isinstance(host, dict),
+        "container_inspection",
+        "container_host_config_invalid",
+    )
+    _verification_require(
+        host.get("ReadonlyRootfs") is True,
+        "container_security",
+        "container_readonly_rootfs_mismatch",
+    )
+    _verification_require(
+        host.get("NetworkMode") == "bridge",
+        "container_security",
+        "container_network_mode_mismatch",
+    )
+    _verification_require(
+        host.get("PidsLimit") == 512,
+        "container_security",
+        "container_pids_limit_mismatch",
+    )
+    _verification_require(
+        isinstance(host.get("CapDrop"), list)
+        and all(isinstance(item, str) for item in host["CapDrop"])
+        and sorted(host["CapDrop"]) == ["ALL"],
+        "container_security",
+        "container_cap_drop_mismatch",
+    )
+    _verification_require(
+        isinstance(host.get("CapAdd"), list)
+        and all(isinstance(item, str) for item in host["CapAdd"])
+        and sorted(host["CapAdd"]) == ["CHOWN", "SETGID", "SETPCAP", "SETUID"],
+        "container_security",
+        "container_cap_add_mismatch",
+    )
+    _verification_require(
+        host.get("SecurityOpt") == ["no-new-privileges:true"],
+        "container_security",
+        "container_security_options_mismatch",
     )
     expected_path = runtime["entrypoint"][0]
     expected_args = [*runtime["entrypoint"][1:], *runtime["cmd"]]
-    _require(
-        value.get("Path") == expected_path and value.get("Args") == expected_args,
-        "running container command differed from the release contract",
+    _verification_require(
+        value.get("Path") == expected_path,
+        "container_command",
+        "container_path_mismatch",
+    )
+    _verification_require(
+        value.get("Args") == expected_args,
+        "container_command",
+        "container_args_mismatch",
     )
     running = state.get("Running")
     pid = state.get("Pid")
     status = state.get("Status")
-    _require(isinstance(running, bool) and isinstance(pid, int), "container startup state was invalid")
+    _verification_require(
+        isinstance(running, bool),
+        "container_state",
+        "container_running_flag_invalid",
+    )
+    _verification_require(
+        isinstance(pid, int),
+        "container_state",
+        "container_pid_invalid",
+    )
     if running is False:
-        _require(status == "created" and pid == 0, "container exited before becoming healthy")
+        _verification_require(
+            status == "created" and pid == 0,
+            "container_state",
+            "container_exited_early",
+        )
         return None
-    _require(pid > 1 and isinstance(health, dict), "container startup state was invalid")
+    _verification_require(
+        pid > 1,
+        "container_state",
+        "container_pid_invalid",
+    )
+    health = state.get("Health")
+    _verification_require(
+        isinstance(health, dict),
+        "container_health",
+        "container_health_missing",
+    )
     health_status = health.get("Status")
-    _require(health_status in {"starting", "healthy"}, "container became unhealthy during startup")
+    _verification_require(
+        health_status in {"starting", "healthy"},
+        "container_health",
+        "container_health_status_invalid",
+    )
     if health_status == "starting":
         return None
     return {"container_id": container_id, "container_pid": state["Pid"], "health": "healthy"}
@@ -1195,8 +1319,8 @@ def verify_running_release(
                 ),
             )
         first_container = _verification_check(
-            "container_startup",
-            "container_contract_mismatch",
+            "container_inspection",
+            "container_inspection_invalid",
             lambda: _docker_inspect(
                 ["container", "inspect", package.CONTAINER_NAME],
                 runner,
@@ -1208,8 +1332,8 @@ def verify_running_release(
             sleeper(0.25)
             continue
         container = _verification_check(
-            "container_startup",
-            "container_contract_mismatch",
+            "container_inspection",
+            "container_inspection_invalid",
             lambda first_container=first_container, container_id=container_id: _validate_starting_container(
                 first_container,
                 release,

tests/test_gcp_leoclean_nosend_installer.py

@@ -255,8 +255,7 @@ class FakeHost:
         payload = "".join(
             f"{name}={value[name]}\n"
             for name in installer.SERVICE_PROPERTIES
-            if name not in self.omitted_properties
-            and (loaded or name not in self.not_found_omitted_properties)
+            if name not in self.omitted_properties and (loaded or name not in self.not_found_omitted_properties)
         ).encode()
         return subprocess.CompletedProcess(arguments, 0, stdout=payload, stderr=b"")
 
@@ -594,6 +593,241 @@ def test_process_environment_failure_has_bounded_stage_and_reason(tmp_path: Path
     assert caught.value.failure_reason == "process_environment_mismatch"
 
 
+@pytest.mark.parametrize(
+    ("path", "replacement", "failure_stage", "failure_reason"),
+    [
+        (("Id",), "f" * 64, "container_identity", "container_id_mismatch"),
+        (("Name",), "/unexpected", "container_identity", "container_name_mismatch"),
+        (
+            ("Image",),
+            f"sha256:{'f' * 64}",
+            "container_image_identity",
+            "container_image_id_mismatch",
+        ),
+        (
+            ("Config", "Image"),
+            "example.invalid/unexpected@sha256:" + "f" * 64,
+            "container_image_identity",
+            "container_image_reference_mismatch",
+        ),
+        (
+            ("Config", "Env"),
+            ["PATH=/unexpected"],
+            "container_environment",
+            "container_environment_mismatch",
+        ),
+        (("Config", "Labels"), [], "container_labels", "container_labels_invalid"),
+        (
+            ("Config", "Labels", "livingip.release.sha256"),
+            "f" * 64,
+            "container_labels",
+            "container_release_label_mismatch",
+        ),
+        (
+            ("Config", "Labels", "livingip.container.role"),
+            "unexpected",
+            "container_labels",
+            "container_role_label_mismatch",
+        ),
+        (("Config",), None, "container_inspection", "container_config_invalid"),
+        (("State",), None, "container_inspection", "container_state_invalid"),
+        (("HostConfig",), None, "container_inspection", "container_host_config_invalid"),
+        (
+            ("HostConfig", "ReadonlyRootfs"),
+            False,
+            "container_security",
+            "container_readonly_rootfs_mismatch",
+        ),
+        (
+            ("HostConfig", "NetworkMode"),
+            "host",
+            "container_security",
+            "container_network_mode_mismatch",
+        ),
+        (
+            ("HostConfig", "PidsLimit"),
+            513,
+            "container_security",
+            "container_pids_limit_mismatch",
+        ),
+        (
+            ("HostConfig", "CapDrop"),
+            [],
+            "container_security",
+            "container_cap_drop_mismatch",
+        ),
+        (
+            ("HostConfig", "CapAdd"),
+            [],
+            "container_security",
+            "container_cap_add_mismatch",
+        ),
+        (
+            ("HostConfig", "SecurityOpt"),
+            ["no-new-privileges=true"],
+            "container_security",
+            "container_security_options_mismatch",
+        ),
+        (("Path",), "/unexpected", "container_command", "container_path_mismatch"),
+        (("Args",), ["unexpected"], "container_command", "container_args_mismatch"),
+        (
+            ("State", "Running"),
+            "true",
+            "container_state",
+            "container_running_flag_invalid",
+        ),
+        (("State", "Pid"), "2001", "container_state", "container_pid_invalid"),
+        (("State", "Health"), None, "container_health", "container_health_missing"),
+        (
+            ("State", "Health", "Status"),
+            "unhealthy",
+            "container_health",
+            "container_health_status_invalid",
+        ),
+    ],
+)
+def test_container_contract_failures_have_field_specific_bounded_codes(
+    tmp_path: Path,
+    path: tuple[str, ...],
+    replacement: object,
+    failure_stage: str,
+    failure_reason: str,
+) -> None:
+    bundle, release, image = release_bundle(tmp_path, "2")
+    paths = install_paths(tmp_path)
+    host = FakeHost(paths)
+    host.add_release(release, image)
+    installer.install_release(
+        bundle,
+        paths=paths,
+        source_binding=source_binding(),
+        runner=host,
+        sleeper=lambda _seconds: None,
+        execute_restart=True,
+    )
+    assert host.container is not None
+    target = host.container
+    for key in path[:-1]:
+        nested = target[key]
+        assert isinstance(nested, dict)
+        target = nested
+    target[path[-1]] = copy.deepcopy(replacement)
+
+    with pytest.raises(installer.VerificationError) as caught:
+        installer.verify_running_release(
+            release,
+            runner=host,
+            paths=paths,
+            sleeper=lambda _seconds: None,
+        )
+
+    assert caught.value.failure_stage == failure_stage
+    assert caught.value.failure_reason == failure_reason
+
+
+def test_container_inspection_shape_has_bounded_code(tmp_path: Path) -> None:
+    bundle, release, image = release_bundle(tmp_path, "2")
+    paths = install_paths(tmp_path)
+    host = FakeHost(paths)
+    host.add_release(release, image)
+    installer.install_release(
+        bundle,
+        paths=paths,
+        source_binding=source_binding(),
+        runner=host,
+        sleeper=lambda _seconds: None,
+        execute_restart=True,
+    )
+
+    def runner(arguments: list[str]) -> subprocess.CompletedProcess[bytes]:
+        result = host(arguments)
+        if arguments[3:] == ["container", "inspect", package.CONTAINER_NAME] and result.returncode == 0:
+            return subprocess.CompletedProcess(arguments, 0, stdout=b"{}", stderr=b"")
+        return result
+
+    with pytest.raises(installer.VerificationError) as caught:
+        installer.verify_running_release(
+            release,
+            runner=runner,
+            paths=paths,
+            sleeper=lambda _seconds: None,
+        )
+
+    assert caught.value.failure_stage == "container_inspection"
+    assert caught.value.failure_reason == "container_inspection_invalid"
+
+
+def test_container_exit_before_health_has_bounded_code(tmp_path: Path) -> None:
+    bundle, release, image = release_bundle(tmp_path, "2")
+    paths = install_paths(tmp_path)
+    host = FakeHost(paths)
+    host.add_release(release, image)
+    installer.install_release(
+        bundle,
+        paths=paths,
+        source_binding=source_binding(),
+        runner=host,
+        sleeper=lambda _seconds: None,
+        execute_restart=True,
+    )
+    assert host.container is not None
+    state = host.container["State"]
+    assert isinstance(state, dict)
+    state.update({"Running": False, "Pid": 0, "Status": "exited"})
+
+    with pytest.raises(installer.VerificationError) as caught:
+        installer.verify_running_release(
+            release,
+            runner=host,
+            paths=paths,
+            sleeper=lambda _seconds: None,
+        )
+
+    assert caught.value.failure_stage == "container_state"
+    assert caught.value.failure_reason == "container_exited_early"
+
+
+def test_container_security_mismatch_rolls_back_without_exposing_observed_value(tmp_path: Path) -> None:
+    bundle, release, image = release_bundle(tmp_path, "2")
+    paths = install_paths(tmp_path)
+    host = FakeHost(paths)
+    host.add_release(release, image)
+
+    def runner(arguments: list[str]) -> subprocess.CompletedProcess[bytes]:
+        result = host(arguments)
+        if arguments[3:] == ["container", "inspect", package.CONTAINER_NAME] and result.returncode == 0:
+            observed = json.loads(result.stdout)
+            observed[0]["HostConfig"]["SecurityOpt"] = ["unexpected-sensitive-value"]
+            return subprocess.CompletedProcess(
+                arguments,
+                0,
+                stdout=json.dumps(observed).encode(),
+                stderr=b"",
+            )
+        return result
+
+    with pytest.raises(installer.VerificationError) as caught:
+        installer.install_release(
+            bundle,
+            paths=paths,
+            source_binding=source_binding(),
+            runner=runner,
+            sleeper=lambda _seconds: None,
+            execute_restart=True,
+        )
+
+    assert caught.value.failure_stage == "container_security"
+    assert caught.value.failure_reason == "container_security_options_mismatch"
+    assert "unexpected-sensitive-value" not in str(caught.value)
+    assert host.stop_calls == 1
+    assert not host.active
+    assert host.container is None
+    assert not paths.cidfile.exists()
+    assert not paths.unit.exists()
+    assert not paths.helper.exists()
+    assert not paths.state_dir.exists()
+
+
 def test_first_install_health_timeout_rolls_back_to_absent(tmp_path: Path) -> None:
     bundle, release, image = release_bundle(tmp_path, "2")
     paths = install_paths(tmp_path)
@@ -651,11 +885,7 @@ def test_verification_rebinds_stable_container_to_unchanged_cidfile(tmp_path: Pa
         nonlocal changed
         result = host(arguments)
         operation = arguments[3:]
-        if (
-            operation[:2] == ["container", "inspect"]
-            and operation[2] != package.CONTAINER_NAME
-            and not changed
-        ):
+        if operation[:2] == ["container", "inspect"] and operation[2] != package.CONTAINER_NAME and not changed:
             paths.cidfile.write_text("f" * 64, encoding="ascii")
             paths.cidfile.chmod(0o600)
             changed = True
@@ -1801,9 +2031,7 @@ def test_control_helper_waits_for_complete_cid_before_exact_cleanup(
                 return subprocess.CompletedProcess(arguments, 1, stdout=b"", stderr=error)
             return subprocess.CompletedProcess(arguments, 0, stdout=json.dumps([container]).encode(), stderr=b"")
         expected_mutation = (
-            ["container", "stop", "--time=30", container_id]
-            if running
-            else ["container", "rm", container_id]
+            ["container", "stop", "--time=30", container_id] if running else ["container", "rm", container_id]
         )
         if operation == expected_mutation:
             container = None
@@ -1833,15 +2061,9 @@ def test_control_helper_waits_for_complete_cid_before_exact_cleanup(
     assert result == {"action": "stop", "status": "pass", "container": "stopped"}
     assert completed
     assert not cidfile.exists()
-    mutations = [
-        call[3:]
-        for call in calls
-        if call[3:5] in (["container", "stop"], ["container", "rm"])
-    ]
+    mutations = [call[3:] for call in calls if call[3:5] in (["container", "stop"], ["container", "rm"])]
     assert mutations == [
-        ["container", "stop", "--time=30", container_id]
-        if running
-        else ["container", "rm", container_id]
+        ["container", "stop", "--time=30", container_id] if running else ["container", "rm", container_id]
     ]
 
 
@@ -1899,11 +2121,7 @@ def test_control_helper_reconciles_created_to_running_race_by_full_id(tmp_path:
 
     assert result == {"action": "stop", "status": "pass", "container": "stopped"}
     assert not cidfile.exists()
-    mutations = [
-        call[3:]
-        for call in calls
-        if call[3:5] in (["container", "stop"], ["container", "rm"])
-    ]
+    mutations = [call[3:] for call in calls if call[3:5] in (["container", "stop"], ["container", "rm"])]
     assert mutations == [
         ["container", "rm", container_id],
         ["container", "stop", "--time=30", container_id],