1363 lines
58 KiB
Python
Executable file
1363 lines
58 KiB
Python
Executable file
#!/usr/bin/env python3
|
|
"""Snapshot a local UTF-8 corpus for later, explicit semantic extraction.
|
|
|
|
Build mode writes only exact content-addressed source snapshots, syntactic
|
|
section locators, an extraction-work index, and a receipt. Verification mode is
|
|
read-only. Neither mode creates claims, proposals, admissions, or database
|
|
writes.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import codecs
|
|
import copy
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import re
|
|
import stat
|
|
import unicodedata
|
|
from collections.abc import Iterator, Mapping, Sequence
|
|
from dataclasses import dataclass
|
|
from pathlib import Path, PurePosixPath
|
|
from typing import Any
|
|
|
|
SOURCE_MANIFEST_SCHEMA = "livingip.localCorpusSourceManifest.v2"
|
|
EXTRACTION_WORK_INDEX_SCHEMA = "livingip.localCorpusExtractionWorkIndex.v1"
|
|
RECEIPT_SCHEMA = "livingip.localCorpusInventoryReceipt.v2"
|
|
STATUS_SCHEMA = "livingip.localCorpusInventoryStatus.v2"
|
|
|
|
SOURCE_MANIFEST_PATH = "source-manifest.json"
|
|
EXTRACTION_WORK_INDEX_PATH = "extraction-work-index.json"
|
|
RECEIPT_PATH = "receipt.json"
|
|
INCOMPLETE_MARKER = ".incomplete"
|
|
SNAPSHOT_ROOT = PurePosixPath("snapshots/sha256")
|
|
MODE_SOURCE_INVENTORY_ONLY = "source-inventory-only"
|
|
WORK_TYPE = "source_section_for_semantic_extraction"
|
|
|
|
DEFAULT_MAX_SOURCE_BYTES = 16 * 1024 * 1024
|
|
DEFAULT_MAX_TOTAL_BYTES = 1024 * 1024 * 1024
|
|
DEFAULT_MAX_SOURCE_COUNT = 20_000
|
|
DEFAULT_MAX_SECTIONS_PER_SOURCE = 10_000
|
|
DEFAULT_MAX_WORK_ITEMS = 250_000
|
|
|
|
HARD_MAX_SOURCE_BYTES = 256 * 1024 * 1024
|
|
HARD_MAX_TOTAL_BYTES = 16 * 1024 * 1024 * 1024
|
|
HARD_MAX_SOURCE_COUNT = 100_000
|
|
HARD_MAX_SECTIONS_PER_SOURCE = 100_000
|
|
HARD_MAX_WORK_ITEMS = 2_000_000
|
|
MAX_METADATA_FILE_BYTES = 512 * 1024 * 1024
|
|
COPY_CHUNK_BYTES = 1024 * 1024
|
|
|
|
ALLOWED_EXTENSIONS = frozenset(
|
|
{
|
|
".adoc",
|
|
".csv",
|
|
".htm",
|
|
".html",
|
|
".json",
|
|
".jsonl",
|
|
".markdown",
|
|
".md",
|
|
".rst",
|
|
".text",
|
|
".toml",
|
|
".tsv",
|
|
".txt",
|
|
".xml",
|
|
".yaml",
|
|
".yml",
|
|
}
|
|
)
|
|
DOCUMENT_EXTENSIONS = frozenset({".adoc", ".markdown", ".md", ".rst"})
|
|
MARKDOWN_HEADING_RE = re.compile(r"^(#{1,6})[ \t]+(.+?)[ \t]*#*[ \t]*$")
|
|
ASCIIDOC_HEADING_RE = re.compile(r"^(={1,6})[ \t]+(.+?)[ \t]*$")
|
|
RST_UNDERLINE_RE = re.compile(r'^[=\-~^"`:+*#<>_]{3,}$')
|
|
SHA256_RE = re.compile(r"^[0-9a-f]{64}$")
|
|
|
|
NO_WRITE_FLAGS = {
|
|
"semantic_claim_extraction_performed": False,
|
|
"database_write_performed": False,
|
|
"admission_performed": False,
|
|
}
|
|
|
|
|
|
class RebuildError(RuntimeError):
|
|
"""Fail-closed inventory error with a stable public code."""
|
|
|
|
def __init__(self, code: str, safe_message: str) -> None:
|
|
super().__init__(safe_message)
|
|
self.code = code
|
|
self.safe_message = safe_message
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class ResourceLimits:
|
|
max_source_bytes: int = DEFAULT_MAX_SOURCE_BYTES
|
|
max_total_bytes: int = DEFAULT_MAX_TOTAL_BYTES
|
|
max_source_count: int = DEFAULT_MAX_SOURCE_COUNT
|
|
max_sections_per_source: int = DEFAULT_MAX_SECTIONS_PER_SOURCE
|
|
max_work_items: int = DEFAULT_MAX_WORK_ITEMS
|
|
|
|
def as_dict(self) -> dict[str, int]:
|
|
return {
|
|
"max_sections_per_source": self.max_sections_per_source,
|
|
"max_source_bytes": self.max_source_bytes,
|
|
"max_source_count": self.max_source_count,
|
|
"max_total_bytes": self.max_total_bytes,
|
|
"max_work_items": self.max_work_items,
|
|
}
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class SourceEntry:
|
|
parent_fd: int
|
|
name: str
|
|
relative_path: str
|
|
extension: str
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class ReservedOutput:
|
|
path: Path
|
|
device: int
|
|
inode: int
|
|
|
|
|
|
def canonical_json_bytes(value: Any) -> bytes:
|
|
try:
|
|
rendered = json.dumps(
|
|
value,
|
|
allow_nan=False,
|
|
ensure_ascii=True,
|
|
separators=(",", ":"),
|
|
sort_keys=True,
|
|
)
|
|
except ValueError as exc:
|
|
raise RebuildError("invalid_json_value", "inventory metadata contains a non-finite JSON number") from exc
|
|
return rendered.encode("utf-8")
|
|
|
|
|
|
def canonical_sha256(value: Any) -> str:
|
|
return hashlib.sha256(canonical_json_bytes(value)).hexdigest()
|
|
|
|
|
|
def sha256_bytes(value: bytes) -> str:
|
|
return hashlib.sha256(value).hexdigest()
|
|
|
|
|
|
def sha256_file(path: Path, *, max_bytes: int | None = None) -> str:
|
|
digest = hashlib.sha256()
|
|
total = 0
|
|
flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0)
|
|
try:
|
|
descriptor = os.open(path, flags)
|
|
except OSError as exc:
|
|
raise RebuildError("bundle_file_unavailable", "a bundle file could not be opened safely") from exc
|
|
try:
|
|
metadata = os.fstat(descriptor)
|
|
if not stat.S_ISREG(metadata.st_mode):
|
|
raise RebuildError("unsafe_bundle_entry", "a bundle entry is not a regular file")
|
|
while True:
|
|
chunk = os.read(descriptor, COPY_CHUNK_BYTES)
|
|
if not chunk:
|
|
break
|
|
total += len(chunk)
|
|
if max_bytes is not None and total > max_bytes:
|
|
raise RebuildError("metadata_limit_exceeded", "a bundle metadata file exceeds the verifier limit")
|
|
digest.update(chunk)
|
|
finally:
|
|
os.close(descriptor)
|
|
return digest.hexdigest()
|
|
|
|
|
|
def rendered_json_bytes(value: Any) -> bytes:
|
|
try:
|
|
rendered = json.dumps(value, allow_nan=False, ensure_ascii=True, indent=2, sort_keys=True)
|
|
except ValueError as exc:
|
|
raise RebuildError("invalid_json_value", "inventory metadata contains a non-finite JSON number") from exc
|
|
return (rendered + "\n").encode("utf-8")
|
|
|
|
|
|
def _absolute_path(path: Path) -> Path:
|
|
expanded = path.expanduser()
|
|
return expanded if expanded.is_absolute() else Path.cwd() / expanded
|
|
|
|
|
|
def _assert_no_symlink_components(path: Path, *, label: str, allow_missing_leaf: bool = False) -> None:
|
|
absolute = _absolute_path(path)
|
|
current = Path(absolute.anchor)
|
|
parts = absolute.parts[1:]
|
|
for index, part in enumerate(parts):
|
|
current /= part
|
|
try:
|
|
metadata = current.lstat()
|
|
except FileNotFoundError:
|
|
if allow_missing_leaf and index == len(parts) - 1:
|
|
return
|
|
raise RebuildError("path_unavailable", f"{label} contains a missing component") from None
|
|
except OSError as exc:
|
|
raise RebuildError("path_unavailable", f"{label} could not be inspected") from exc
|
|
if stat.S_ISLNK(metadata.st_mode):
|
|
raise RebuildError("symlink_rejected", f"{label} contains a symlink component")
|
|
|
|
|
|
def _paths_overlap(left: Path, right: Path) -> bool:
|
|
return left == right or left in right.parents or right in left.parents
|
|
|
|
|
|
def _validate_roots(source_root: Path, output: Path) -> tuple[Path, Path]:
|
|
source_absolute = _absolute_path(source_root)
|
|
_assert_no_symlink_components(source_absolute, label="source root")
|
|
try:
|
|
source_metadata = source_absolute.lstat()
|
|
except OSError as exc:
|
|
raise RebuildError("invalid_source_root", "source root is unavailable") from exc
|
|
if not stat.S_ISDIR(source_metadata.st_mode):
|
|
raise RebuildError("invalid_source_root", "source root must be a non-symlink directory")
|
|
source_resolved = source_absolute.resolve(strict=True)
|
|
|
|
output_absolute = _absolute_path(output)
|
|
_assert_no_symlink_components(output_absolute, label="output path", allow_missing_leaf=True)
|
|
output_parent = output_absolute.parent
|
|
try:
|
|
parent_metadata = output_parent.lstat()
|
|
except OSError as exc:
|
|
raise RebuildError("invalid_output_parent", "output parent is unavailable") from exc
|
|
if not stat.S_ISDIR(parent_metadata.st_mode):
|
|
raise RebuildError("invalid_output_parent", "output parent must be a non-symlink directory")
|
|
output_resolved = output_parent.resolve(strict=True) / output_absolute.name
|
|
if _paths_overlap(source_resolved, output_resolved):
|
|
raise RebuildError("output_overlap", "source root and output directory must be disjoint")
|
|
return source_resolved, output_absolute
|
|
|
|
|
|
def _validate_bundle_root(bundle: Path) -> Path:
|
|
absolute = _absolute_path(bundle)
|
|
_assert_no_symlink_components(absolute, label="bundle root")
|
|
try:
|
|
metadata = absolute.lstat()
|
|
except OSError as exc:
|
|
raise RebuildError("invalid_bundle_root", "bundle root is unavailable") from exc
|
|
if not stat.S_ISDIR(metadata.st_mode):
|
|
raise RebuildError("invalid_bundle_root", "bundle root must be a non-symlink directory")
|
|
return absolute.resolve(strict=True)
|
|
|
|
|
|
def _validate_relative_path(relative_path: str) -> str:
|
|
try:
|
|
relative_path.encode("utf-8", errors="strict")
|
|
except UnicodeEncodeError as exc:
|
|
raise RebuildError("unsafe_relative_path", "source paths must be strict UTF-8") from exc
|
|
parsed = PurePosixPath(relative_path)
|
|
if parsed.is_absolute() or not relative_path or any(part in {"", ".", ".."} for part in parsed.parts):
|
|
raise RebuildError("unsafe_relative_path", "source paths must be normalized source-root-relative paths")
|
|
if any(unicodedata.category(character) == "Cc" for character in relative_path):
|
|
raise RebuildError("unsafe_relative_path", "source paths must not contain control characters")
|
|
return relative_path
|
|
|
|
|
|
def _normalized_path_key(relative_path: str) -> str:
|
|
return unicodedata.normalize("NFC", _validate_relative_path(relative_path)).casefold()
|
|
|
|
|
|
def _directory_flags() -> int:
|
|
return os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0)
|
|
|
|
|
|
def _iter_source_entries(source_root: Path) -> Iterator[SourceEntry]:
|
|
try:
|
|
root_fd = os.open(source_root, _directory_flags())
|
|
except OSError as exc:
|
|
raise RebuildError("source_inventory_failed", "source root could not be opened safely") from exc
|
|
|
|
def visit(directory_fd: int, prefix: PurePosixPath) -> Iterator[SourceEntry]:
|
|
try:
|
|
with os.scandir(directory_fd) as stream:
|
|
inspected: list[tuple[os.DirEntry[str], os.stat_result]] = []
|
|
for entry in stream:
|
|
try:
|
|
inspected.append((entry, entry.stat(follow_symlinks=False)))
|
|
except OSError as exc:
|
|
raise RebuildError(
|
|
"source_inventory_failed",
|
|
"a source entry could not be inspected",
|
|
) from exc
|
|
entries = sorted(
|
|
inspected,
|
|
key=lambda row: (
|
|
row[0].name.encode("utf-8", errors="surrogateescape")
|
|
+ (b"/" if stat.S_ISDIR(row[1].st_mode) else b"")
|
|
),
|
|
)
|
|
except OSError as exc:
|
|
raise RebuildError("source_inventory_failed", "source tree could not be inventoried") from exc
|
|
for entry, metadata in entries:
|
|
relative = (prefix / entry.name).as_posix()
|
|
_validate_relative_path(relative)
|
|
if stat.S_ISLNK(metadata.st_mode):
|
|
raise RebuildError("symlink_rejected", f"source entry is a symlink: {relative}")
|
|
if stat.S_ISDIR(metadata.st_mode):
|
|
try:
|
|
child_fd = os.open(entry.name, _directory_flags(), dir_fd=directory_fd)
|
|
except OSError as exc:
|
|
raise RebuildError(
|
|
"source_inventory_failed", "a source directory changed during inventory"
|
|
) from exc
|
|
try:
|
|
yield from visit(child_fd, prefix / entry.name)
|
|
finally:
|
|
os.close(child_fd)
|
|
continue
|
|
if not stat.S_ISREG(metadata.st_mode):
|
|
raise RebuildError("unsupported_source_entry", f"source entry is not a regular file: {relative}")
|
|
extension = PurePosixPath(relative).suffix.casefold()
|
|
if extension not in ALLOWED_EXTENSIONS:
|
|
raise RebuildError("unsupported_source_file", f"source extension is not allowed: {relative}")
|
|
yield SourceEntry(directory_fd, entry.name, relative, extension)
|
|
|
|
try:
|
|
yield from visit(root_fd, PurePosixPath())
|
|
finally:
|
|
os.close(root_fd)
|
|
|
|
|
|
def _validate_limits(limits: ResourceLimits) -> ResourceLimits:
|
|
hard_limits = {
|
|
"max_sections_per_source": HARD_MAX_SECTIONS_PER_SOURCE,
|
|
"max_source_bytes": HARD_MAX_SOURCE_BYTES,
|
|
"max_source_count": HARD_MAX_SOURCE_COUNT,
|
|
"max_total_bytes": HARD_MAX_TOTAL_BYTES,
|
|
"max_work_items": HARD_MAX_WORK_ITEMS,
|
|
}
|
|
for name, value in limits.as_dict().items():
|
|
if isinstance(value, bool) or not isinstance(value, int) or value <= 0:
|
|
raise RebuildError("invalid_resource_limit", f"{name} must be a positive integer")
|
|
if value > hard_limits[name]:
|
|
raise RebuildError("invalid_resource_limit", f"{name} exceeds its hard safety bound")
|
|
if limits.max_source_bytes > limits.max_total_bytes:
|
|
raise RebuildError("invalid_resource_limit", "max_source_bytes must not exceed max_total_bytes")
|
|
if limits.max_sections_per_source > limits.max_work_items:
|
|
raise RebuildError("invalid_resource_limit", "max_sections_per_source must not exceed max_work_items")
|
|
return limits
|
|
|
|
|
|
def _limits_from_mapping(value: Mapping[str, Any]) -> ResourceLimits:
|
|
expected = {
|
|
"max_sections_per_source",
|
|
"max_source_bytes",
|
|
"max_source_count",
|
|
"max_total_bytes",
|
|
"max_work_items",
|
|
}
|
|
if set(value) != expected:
|
|
raise RebuildError("invalid_bundle_schema", "receipt resource limits do not match the supported schema")
|
|
return _validate_limits(ResourceLimits(**{name: value[name] for name in expected}))
|
|
|
|
|
|
def _assert_output_binding(reserved: ReservedOutput) -> None:
|
|
try:
|
|
metadata = reserved.path.lstat()
|
|
except OSError as exc:
|
|
raise RebuildError("output_binding_changed", "reserved output directory is no longer available") from exc
|
|
if (
|
|
not stat.S_ISDIR(metadata.st_mode)
|
|
or stat.S_ISLNK(metadata.st_mode)
|
|
or metadata.st_dev != reserved.device
|
|
or metadata.st_ino != reserved.inode
|
|
):
|
|
raise RebuildError("output_binding_changed", "reserved output directory identity changed")
|
|
|
|
|
|
def _reserve_output(output: Path) -> ReservedOutput:
|
|
try:
|
|
os.mkdir(output, mode=0o700)
|
|
except FileExistsError as exc:
|
|
raise RebuildError("output_exists", "output directory must not already exist") from exc
|
|
except OSError as exc:
|
|
raise RebuildError("output_reservation_failed", "output directory could not be reserved") from exc
|
|
metadata = output.lstat()
|
|
reserved = ReservedOutput(output, metadata.st_dev, metadata.st_ino)
|
|
try:
|
|
_write_private_bytes(output / INCOMPLETE_MARKER, b"source inventory build incomplete\n")
|
|
except Exception:
|
|
_cleanup_reserved_output(reserved)
|
|
raise
|
|
return reserved
|
|
|
|
|
|
def _remove_tree_contents(directory_fd: int) -> None:
|
|
for name in os.listdir(directory_fd):
|
|
try:
|
|
metadata = os.stat(name, dir_fd=directory_fd, follow_symlinks=False)
|
|
except FileNotFoundError:
|
|
continue
|
|
if stat.S_ISDIR(metadata.st_mode) and not stat.S_ISLNK(metadata.st_mode):
|
|
child_fd = os.open(name, _directory_flags(), dir_fd=directory_fd)
|
|
try:
|
|
_remove_tree_contents(child_fd)
|
|
finally:
|
|
os.close(child_fd)
|
|
os.rmdir(name, dir_fd=directory_fd)
|
|
else:
|
|
os.unlink(name, dir_fd=directory_fd)
|
|
|
|
|
|
def _cleanup_reserved_output(reserved: ReservedOutput) -> None:
|
|
try:
|
|
_assert_output_binding(reserved)
|
|
directory_fd = os.open(reserved.path, _directory_flags())
|
|
except (OSError, RebuildError):
|
|
return
|
|
try:
|
|
opened = os.fstat(directory_fd)
|
|
if opened.st_dev != reserved.device or opened.st_ino != reserved.inode:
|
|
return
|
|
_remove_tree_contents(directory_fd)
|
|
except OSError:
|
|
return
|
|
finally:
|
|
os.close(directory_fd)
|
|
try:
|
|
_assert_output_binding(reserved)
|
|
os.rmdir(reserved.path)
|
|
except (OSError, RebuildError):
|
|
return
|
|
|
|
|
|
def _mkdir_private(path: Path) -> None:
|
|
try:
|
|
os.mkdir(path, mode=0o700)
|
|
except FileExistsError:
|
|
metadata = path.lstat()
|
|
if not stat.S_ISDIR(metadata.st_mode) or stat.S_ISLNK(metadata.st_mode):
|
|
raise RebuildError("unsafe_output_entry", "an output directory path is occupied unsafely") from None
|
|
|
|
|
|
def _write_private_bytes(path: Path, content: bytes) -> None:
|
|
flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0)
|
|
try:
|
|
descriptor = os.open(path, flags, 0o600)
|
|
except OSError as exc:
|
|
raise RebuildError("output_write_failed", "a bundle artifact could not be created exclusively") from exc
|
|
try:
|
|
view = memoryview(content)
|
|
while view:
|
|
written = os.write(descriptor, view)
|
|
if written <= 0:
|
|
raise RebuildError("output_write_failed", "a bundle artifact write did not complete")
|
|
view = view[written:]
|
|
os.fsync(descriptor)
|
|
finally:
|
|
os.close(descriptor)
|
|
|
|
|
|
def _write_private_json(path: Path, value: Any) -> None:
|
|
_write_private_bytes(path, rendered_json_bytes(value))
|
|
|
|
|
|
def _validate_text(text: str) -> str:
|
|
if not text.strip():
|
|
raise RebuildError("empty_source_file", "source file contains no non-whitespace UTF-8 text")
|
|
for character in text:
|
|
if unicodedata.category(character) == "Cc" and character not in {"\t", "\n", "\r"}:
|
|
raise RebuildError("binary_control_character", "source contains a disallowed control character")
|
|
for index, character in enumerate(text):
|
|
if character == "\r" and (index + 1 >= len(text) or text[index + 1] != "\n"):
|
|
raise RebuildError("unsupported_newline", "CR-only or lone-CR newlines are not supported")
|
|
return text
|
|
|
|
|
|
def _newline_style(text: str) -> str:
|
|
crlf = text.count("\r\n")
|
|
lf = text.count("\n") - crlf
|
|
if crlf and lf:
|
|
return "mixed-lf-crlf"
|
|
if crlf:
|
|
return "crlf"
|
|
return "lf" if lf else "none"
|
|
|
|
|
|
def _descriptor_signature(metadata: os.stat_result) -> tuple[int, int, int, int, int, int]:
|
|
return (
|
|
metadata.st_dev,
|
|
metadata.st_ino,
|
|
metadata.st_mode,
|
|
metadata.st_size,
|
|
metadata.st_mtime_ns,
|
|
metadata.st_ctime_ns,
|
|
)
|
|
|
|
|
|
def _copy_source_to_partial(
|
|
entry: SourceEntry,
|
|
partial_path: Path,
|
|
*,
|
|
limits: ResourceLimits,
|
|
total_before: int,
|
|
) -> tuple[str, int]:
|
|
flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0)
|
|
try:
|
|
source_fd = os.open(entry.name, flags, dir_fd=entry.parent_fd)
|
|
except OSError as exc:
|
|
raise RebuildError("source_open_failed", f"source could not be opened safely: {entry.relative_path}") from exc
|
|
output_fd: int | None = None
|
|
try:
|
|
before = os.fstat(source_fd)
|
|
if not stat.S_ISREG(before.st_mode):
|
|
raise RebuildError("source_changed", f"source is no longer a regular file: {entry.relative_path}")
|
|
if before.st_size > limits.max_source_bytes:
|
|
raise RebuildError("source_byte_limit_exceeded", f"source exceeds max_source_bytes: {entry.relative_path}")
|
|
if total_before + before.st_size > limits.max_total_bytes:
|
|
raise RebuildError("total_byte_limit_exceeded", "corpus exceeds max_total_bytes")
|
|
write_flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0)
|
|
output_fd = os.open(partial_path, write_flags, 0o600)
|
|
digest = hashlib.sha256()
|
|
decoder = codecs.getincrementaldecoder("utf-8")("strict")
|
|
copied = 0
|
|
try:
|
|
while True:
|
|
chunk = os.read(source_fd, COPY_CHUNK_BYTES)
|
|
if not chunk:
|
|
break
|
|
copied += len(chunk)
|
|
if copied > limits.max_source_bytes:
|
|
raise RebuildError(
|
|
"source_byte_limit_exceeded", f"source exceeds max_source_bytes: {entry.relative_path}"
|
|
)
|
|
if total_before + copied > limits.max_total_bytes:
|
|
raise RebuildError("total_byte_limit_exceeded", "corpus exceeds max_total_bytes")
|
|
decoder.decode(chunk)
|
|
digest.update(chunk)
|
|
view = memoryview(chunk)
|
|
while view:
|
|
written = os.write(output_fd, view)
|
|
if written <= 0:
|
|
raise RebuildError("output_write_failed", "source snapshot write did not complete")
|
|
view = view[written:]
|
|
decoder.decode(b"", final=True)
|
|
except UnicodeDecodeError as exc:
|
|
raise RebuildError("invalid_utf8_source", f"source is not strict UTF-8: {entry.relative_path}") from exc
|
|
os.fsync(output_fd)
|
|
after = os.fstat(source_fd)
|
|
if _descriptor_signature(before) != _descriptor_signature(after) or copied != after.st_size:
|
|
raise RebuildError(
|
|
"source_changed_during_read", f"source changed while being copied: {entry.relative_path}"
|
|
)
|
|
return digest.hexdigest(), copied
|
|
finally:
|
|
if output_fd is not None:
|
|
os.close(output_fd)
|
|
os.close(source_fd)
|
|
|
|
|
|
def _snapshot_path_for_hash(digest: str) -> PurePosixPath:
|
|
if not SHA256_RE.fullmatch(digest):
|
|
raise RebuildError("invalid_snapshot_hash", "snapshot SHA-256 is malformed")
|
|
return SNAPSHOT_ROOT / digest[:2] / f"{digest}.utf8"
|
|
|
|
|
|
def _install_snapshot(output: Path, partial_path: Path, digest: str, byte_count: int) -> tuple[Path, bool]:
|
|
relative = _snapshot_path_for_hash(digest)
|
|
prefix_dir = output / Path(relative.parent)
|
|
_mkdir_private(prefix_dir)
|
|
destination = output / Path(relative)
|
|
try:
|
|
os.link(partial_path, destination, follow_symlinks=False)
|
|
except FileExistsError:
|
|
metadata = destination.lstat()
|
|
if not stat.S_ISREG(metadata.st_mode) or stat.S_ISLNK(metadata.st_mode):
|
|
raise RebuildError("snapshot_collision", "snapshot hash path is occupied by a non-regular entry") from None
|
|
if metadata.st_size != byte_count or sha256_file(destination) != digest:
|
|
raise RebuildError("snapshot_collision", "snapshot hash path contains mismatched bytes") from None
|
|
deduplicated = True
|
|
except OSError as exc:
|
|
raise RebuildError("snapshot_publish_failed", "content-addressed snapshot could not be installed") from exc
|
|
else:
|
|
deduplicated = False
|
|
try:
|
|
partial_path.unlink()
|
|
except OSError as exc:
|
|
raise RebuildError("snapshot_cleanup_failed", "temporary snapshot could not be removed") from exc
|
|
return destination, deduplicated
|
|
|
|
|
|
def _read_snapshot_text(snapshot: Path, *, expected_sha256: str, expected_bytes: int) -> str:
|
|
flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0)
|
|
try:
|
|
descriptor = os.open(snapshot, flags)
|
|
except OSError as exc:
|
|
raise RebuildError("snapshot_read_failed", "source snapshot could not be reopened safely") from exc
|
|
try:
|
|
metadata = os.fstat(descriptor)
|
|
if not stat.S_ISREG(metadata.st_mode) or metadata.st_size != expected_bytes:
|
|
raise RebuildError("snapshot_mismatch", "source snapshot size changed")
|
|
raw = bytearray()
|
|
digest = hashlib.sha256()
|
|
while True:
|
|
chunk = os.read(descriptor, COPY_CHUNK_BYTES)
|
|
if not chunk:
|
|
break
|
|
digest.update(chunk)
|
|
raw.extend(chunk)
|
|
finally:
|
|
os.close(descriptor)
|
|
if digest.hexdigest() != expected_sha256:
|
|
raise RebuildError("snapshot_mismatch", "source snapshot hash changed")
|
|
try:
|
|
text = bytes(raw).decode("utf-8", errors="strict")
|
|
except UnicodeDecodeError as exc: # defensive: copy path already decoded incrementally
|
|
raise RebuildError("invalid_utf8_source", "source snapshot is not strict UTF-8") from exc
|
|
return _validate_text(text)
|
|
|
|
|
|
def _heading_rows(text: str, extension: str) -> list[tuple[int, str]]:
|
|
lines = text.splitlines(keepends=True)
|
|
rows: list[tuple[int, str]] = []
|
|
offset = 0
|
|
for index, line in enumerate(lines):
|
|
content = line.rstrip("\r\n")
|
|
match: re.Match[str] | None = None
|
|
if extension in {".markdown", ".md"}:
|
|
match = MARKDOWN_HEADING_RE.fullmatch(content)
|
|
elif extension == ".adoc":
|
|
match = ASCIIDOC_HEADING_RE.fullmatch(content)
|
|
if match is not None:
|
|
rows.append((offset, match.group(2).strip()))
|
|
elif extension == ".rst" and index + 1 < len(lines):
|
|
underline = lines[index + 1].rstrip("\r\n").strip()
|
|
title = content.strip()
|
|
if title and RST_UNDERLINE_RE.fullmatch(underline):
|
|
rows.append((offset, title))
|
|
offset += len(line)
|
|
return rows
|
|
|
|
|
|
def _position_map(text: str, positions: Sequence[int]) -> dict[int, tuple[int, int]]:
|
|
result: dict[int, tuple[int, int]] = {}
|
|
cursor = 0
|
|
byte_offset = 0
|
|
line = 1
|
|
for position in sorted(set(positions)):
|
|
fragment = text[cursor:position]
|
|
byte_offset += len(fragment.encode("utf-8"))
|
|
line += fragment.count("\n")
|
|
result[position] = (byte_offset, line)
|
|
cursor = position
|
|
return result
|
|
|
|
|
|
def _build_sections(text: str, extension: str, *, max_sections: int) -> list[dict[str, Any]]:
|
|
headings = _heading_rows(text, extension) if extension in DOCUMENT_EXTENSIONS else []
|
|
boundaries: list[int] = []
|
|
if headings and headings[0][0] > 0 and text[: headings[0][0]].strip():
|
|
boundaries.append(0)
|
|
boundaries.extend(position for position, _heading in headings)
|
|
if not boundaries:
|
|
boundaries = [0]
|
|
|
|
character_ranges: list[tuple[int, int]] = []
|
|
for index, raw_start in enumerate(boundaries):
|
|
raw_end = boundaries[index + 1] if index + 1 < len(boundaries) else len(text)
|
|
raw_segment = text[raw_start:raw_end]
|
|
stripped = raw_segment.strip()
|
|
if not stripped:
|
|
continue
|
|
leading = len(raw_segment) - len(raw_segment.lstrip())
|
|
start = raw_start + leading
|
|
end = start + len(stripped)
|
|
character_ranges.append((start, end))
|
|
if not character_ranges:
|
|
raise RebuildError("empty_source_file", "source file contains no non-whitespace UTF-8 text")
|
|
if len(character_ranges) > max_sections:
|
|
raise RebuildError("section_limit_exceeded", "source exceeds max_sections_per_source")
|
|
|
|
positions = [position for start, end in character_ranges for position in (start, end - 1)]
|
|
mapped = _position_map(text, positions)
|
|
sections: list[dict[str, Any]] = []
|
|
for ordinal, (start, end) in enumerate(character_ranges, start=1):
|
|
start_byte, line_start = mapped[start]
|
|
last_byte, line_end = mapped[end - 1]
|
|
encoded_last = text[end - 1].encode("utf-8")
|
|
end_byte = last_byte + len(encoded_last)
|
|
quote = text[start:end].encode("utf-8")
|
|
if start_byte + len(quote) != end_byte:
|
|
raise RebuildError("locator_construction_failed", "UTF-8 locator construction was inconsistent")
|
|
sections.append(
|
|
{
|
|
"line_end": line_end,
|
|
"line_start": line_start,
|
|
"quote_sha256": sha256_bytes(quote),
|
|
"section_ordinal": ordinal,
|
|
"utf8_byte_end": end_byte,
|
|
"utf8_byte_start": start_byte,
|
|
}
|
|
)
|
|
return sections
|
|
|
|
|
|
def _sectioning_strategy(extension: str) -> str:
|
|
if extension in {".markdown", ".md"}:
|
|
return "markdown_atx_heading_or_whole_source"
|
|
if extension == ".adoc":
|
|
return "asciidoc_heading_or_whole_source"
|
|
if extension == ".rst":
|
|
return "rst_underline_heading_or_whole_source"
|
|
return "whole_utf8_source"
|
|
|
|
|
|
def _work_item(source: Mapping[str, Any], locator: Mapping[str, Any]) -> dict[str, Any]:
|
|
identity = {
|
|
"artifact_sha256": source["artifact_sha256"],
|
|
"locator": dict(locator),
|
|
"relative_path": source["relative_path"],
|
|
"snapshot_path": source["snapshot_path"],
|
|
"work_type": WORK_TYPE,
|
|
}
|
|
return {
|
|
"artifact_sha256": source["artifact_sha256"],
|
|
"locator": dict(locator),
|
|
"relative_path": source["relative_path"],
|
|
"snapshot_path": source["snapshot_path"],
|
|
"work_item_id": canonical_sha256(identity),
|
|
"work_type": WORK_TYPE,
|
|
**NO_WRITE_FLAGS,
|
|
}
|
|
|
|
|
|
def _artifact_records(root: Path, paths: Sequence[str]) -> list[dict[str, Any]]:
|
|
records: list[dict[str, Any]] = []
|
|
for relative in sorted(paths, key=lambda value: value.encode("utf-8")):
|
|
validated = _validate_relative_path(relative)
|
|
path = root / Path(validated)
|
|
metadata = path.lstat()
|
|
if not stat.S_ISREG(metadata.st_mode) or stat.S_ISLNK(metadata.st_mode):
|
|
raise RebuildError("unsafe_bundle_entry", "bundle artifact is not a regular file")
|
|
records.append({"bytes": metadata.st_size, "path": validated, "sha256": sha256_file(path)})
|
|
return records
|
|
|
|
|
|
def _after_source_snapshot(_relative_path: str, _source_root: Path, _snapshot: Path) -> None:
|
|
"""Test seam after descriptor copy; production intentionally does nothing."""
|
|
|
|
|
|
def _after_output_reservation(_reserved: ReservedOutput) -> None:
|
|
"""Test seam after atomic reservation; production intentionally does nothing."""
|
|
|
|
|
|
def build_source_inventory_bundle(
|
|
source_root: Path,
|
|
output: Path,
|
|
*,
|
|
mode: str = MODE_SOURCE_INVENTORY_ONLY,
|
|
limits: ResourceLimits | None = None,
|
|
) -> dict[str, Any]:
|
|
if mode != MODE_SOURCE_INVENTORY_ONLY:
|
|
raise RebuildError("unsupported_mode", "this command supports source-inventory-only mode")
|
|
checked_limits = _validate_limits(limits or ResourceLimits())
|
|
source_resolved, output_absolute = _validate_roots(source_root, output)
|
|
reserved = _reserve_output(output_absolute)
|
|
try:
|
|
_after_output_reservation(reserved)
|
|
_assert_output_binding(reserved)
|
|
_mkdir_private(output_absolute / "snapshots")
|
|
_mkdir_private(output_absolute / "snapshots" / "sha256")
|
|
|
|
sources: list[dict[str, Any]] = []
|
|
work_items: list[dict[str, Any]] = []
|
|
seen_paths: set[str] = set()
|
|
snapshot_paths: set[str] = set()
|
|
total_bytes = 0
|
|
|
|
for ordinal, entry in enumerate(_iter_source_entries(source_resolved), start=1):
|
|
if ordinal > checked_limits.max_source_count:
|
|
raise RebuildError("source_count_limit_exceeded", "corpus exceeds max_source_count")
|
|
normalized = _normalized_path_key(entry.relative_path)
|
|
if normalized in seen_paths:
|
|
raise RebuildError(
|
|
"duplicate_relative_path",
|
|
"source tree contains duplicate case-folded or Unicode-normalized relative paths",
|
|
)
|
|
seen_paths.add(normalized)
|
|
|
|
partial = output_absolute / "snapshots" / f".partial-{ordinal:08d}"
|
|
digest, byte_count = _copy_source_to_partial(
|
|
entry,
|
|
partial,
|
|
limits=checked_limits,
|
|
total_before=total_bytes,
|
|
)
|
|
total_bytes += byte_count
|
|
snapshot, _deduplicated = _install_snapshot(output_absolute, partial, digest, byte_count)
|
|
snapshot_relative = snapshot.relative_to(output_absolute).as_posix()
|
|
snapshot_paths.add(snapshot_relative)
|
|
_after_source_snapshot(entry.relative_path, source_resolved, snapshot)
|
|
|
|
text = _read_snapshot_text(snapshot, expected_sha256=digest, expected_bytes=byte_count)
|
|
sections = _build_sections(
|
|
text,
|
|
entry.extension,
|
|
max_sections=checked_limits.max_sections_per_source,
|
|
)
|
|
if len(work_items) + len(sections) > checked_limits.max_work_items:
|
|
raise RebuildError("work_item_limit_exceeded", "corpus exceeds max_work_items")
|
|
source_row: dict[str, Any] = {
|
|
"artifact_sha256": digest,
|
|
"bytes": byte_count,
|
|
"content_validation": "strict_utf8_no_binary_controls",
|
|
"extension": entry.extension,
|
|
"format_syntax_validated": False,
|
|
"newline_style": _newline_style(text),
|
|
"relative_path": entry.relative_path,
|
|
"section_count": len(sections),
|
|
"sectioning_strategy": _sectioning_strategy(entry.extension),
|
|
"sections": sections,
|
|
"snapshot_path": snapshot_relative,
|
|
"source_path_binding": "source_root_relative_posix_path_plus_exact_snapshot_sha256",
|
|
}
|
|
sources.append(source_row)
|
|
work_items.extend(_work_item(source_row, locator) for locator in sections)
|
|
del text, sections
|
|
|
|
if not sources:
|
|
raise RebuildError("empty_source_root", "source root contains no supported source files")
|
|
_assert_output_binding(reserved)
|
|
|
|
source_manifest = {
|
|
"schema": SOURCE_MANIFEST_SCHEMA,
|
|
"mode": MODE_SOURCE_INVENTORY_ONLY,
|
|
**NO_WRITE_FLAGS,
|
|
"source_snapshot_semantics": {
|
|
"concurrent_directory_additions_or_removals_may_be_absent": True,
|
|
"directory_snapshot_atomic": False,
|
|
"guarantee": "each listed path is bound to the exact bytes copied from one stable file descriptor",
|
|
},
|
|
"resource_limits": checked_limits.as_dict(),
|
|
"source_count": len(sources),
|
|
"total_source_bytes": total_bytes,
|
|
"unique_snapshot_count": len(snapshot_paths),
|
|
"duplicate_content_source_count": len(sources) - len(snapshot_paths),
|
|
"unsupported_files_ignored": False,
|
|
"symlinks_followed": False,
|
|
"sources": sources,
|
|
}
|
|
source_manifest_file = output_absolute / SOURCE_MANIFEST_PATH
|
|
_write_private_json(source_manifest_file, source_manifest)
|
|
|
|
extraction_index = {
|
|
"schema": EXTRACTION_WORK_INDEX_SCHEMA,
|
|
"mode": MODE_SOURCE_INVENTORY_ONLY,
|
|
"work_type": WORK_TYPE,
|
|
**NO_WRITE_FLAGS,
|
|
"source_manifest": {
|
|
"path": SOURCE_MANIFEST_PATH,
|
|
"sha256": sha256_file(source_manifest_file),
|
|
},
|
|
"source_count": len(sources),
|
|
"work_item_count": len(work_items),
|
|
"items": work_items,
|
|
}
|
|
extraction_index_file = output_absolute / EXTRACTION_WORK_INDEX_PATH
|
|
_write_private_json(extraction_index_file, extraction_index)
|
|
del work_items
|
|
|
|
artifact_paths = [SOURCE_MANIFEST_PATH, EXTRACTION_WORK_INDEX_PATH, *snapshot_paths]
|
|
artifacts = _artifact_records(output_absolute, artifact_paths)
|
|
receipt = {
|
|
"schema": RECEIPT_SCHEMA,
|
|
"status": "pass",
|
|
"result": "source_inventory_bundle_ready",
|
|
"mode": MODE_SOURCE_INVENTORY_ONLY,
|
|
"deterministic": True,
|
|
**NO_WRITE_FLAGS,
|
|
"source_snapshot_atomic": False,
|
|
"source_count": len(sources),
|
|
"work_item_count": extraction_index["work_item_count"],
|
|
"total_source_bytes": total_bytes,
|
|
"unique_snapshot_count": len(snapshot_paths),
|
|
"resource_limits": checked_limits.as_dict(),
|
|
"outputs": {
|
|
"artifact_count": len(artifacts),
|
|
"artifacts": artifacts,
|
|
"extraction_work_index_path": EXTRACTION_WORK_INDEX_PATH,
|
|
"source_manifest_path": SOURCE_MANIFEST_PATH,
|
|
},
|
|
"hashes": {
|
|
"bundle_tree_sha256": canonical_sha256(artifacts),
|
|
"extraction_work_index_file_sha256": sha256_file(extraction_index_file),
|
|
"source_manifest_file_sha256": sha256_file(source_manifest_file),
|
|
},
|
|
}
|
|
receipt["hashes"]["receipt_content_sha256"] = canonical_sha256(receipt)
|
|
|
|
_assert_output_binding(reserved)
|
|
_write_private_json(output_absolute / RECEIPT_PATH, receipt)
|
|
_assert_output_binding(reserved)
|
|
(output_absolute / INCOMPLETE_MARKER).unlink()
|
|
_assert_output_binding(reserved)
|
|
verified = verify_bundle(output_absolute)
|
|
return {
|
|
"bundle_path": output_absolute,
|
|
"receipt": receipt,
|
|
"receipt_file_sha256": sha256_file(output_absolute / RECEIPT_PATH),
|
|
"verification": verified,
|
|
}
|
|
except Exception:
|
|
_cleanup_reserved_output(reserved)
|
|
raise
|
|
|
|
|
|
def _load_json(path: Path) -> dict[str, Any]:
|
|
try:
|
|
metadata = path.lstat()
|
|
except OSError as exc:
|
|
raise RebuildError("missing_bundle_artifact", "a required bundle artifact is missing") from exc
|
|
if not stat.S_ISREG(metadata.st_mode) or stat.S_ISLNK(metadata.st_mode):
|
|
raise RebuildError("unsafe_bundle_entry", "a required bundle artifact is not a regular file")
|
|
if metadata.st_size > MAX_METADATA_FILE_BYTES:
|
|
raise RebuildError("metadata_limit_exceeded", "a bundle metadata file exceeds the verifier limit")
|
|
flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0)
|
|
descriptor = os.open(path, flags)
|
|
try:
|
|
chunks: list[bytes] = []
|
|
total = 0
|
|
while True:
|
|
chunk = os.read(descriptor, COPY_CHUNK_BYTES)
|
|
if not chunk:
|
|
break
|
|
total += len(chunk)
|
|
if total > MAX_METADATA_FILE_BYTES:
|
|
raise RebuildError("metadata_limit_exceeded", "a bundle metadata file exceeds the verifier limit")
|
|
chunks.append(chunk)
|
|
finally:
|
|
os.close(descriptor)
|
|
|
|
def reject_duplicate_pairs(pairs: list[tuple[str, Any]]) -> dict[str, Any]:
|
|
value: dict[str, Any] = {}
|
|
for key, child in pairs:
|
|
if key in value:
|
|
raise RebuildError("duplicate_json_key", "a bundle metadata file contains a duplicate JSON key")
|
|
value[key] = child
|
|
return value
|
|
|
|
def reject_nonfinite_number(_value: str) -> None:
|
|
raise RebuildError("invalid_json_value", "a bundle metadata file contains a non-finite JSON number")
|
|
|
|
try:
|
|
value = json.loads(
|
|
b"".join(chunks).decode("utf-8", errors="strict"),
|
|
object_pairs_hook=reject_duplicate_pairs,
|
|
parse_constant=reject_nonfinite_number,
|
|
)
|
|
except RebuildError:
|
|
raise
|
|
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
|
|
raise RebuildError("invalid_bundle_json", "a bundle metadata file is not valid UTF-8 JSON") from exc
|
|
if not isinstance(value, dict):
|
|
raise RebuildError("invalid_bundle_schema", "a bundle metadata root must be an object")
|
|
return value
|
|
|
|
|
|
def _assert_no_write_flags(value: Mapping[str, Any], *, label: str) -> None:
|
|
for key, expected in NO_WRITE_FLAGS.items():
|
|
if value.get(key) is not expected:
|
|
raise RebuildError("unsafe_bundle_flags", f"{label} does not retain the required no-write flags")
|
|
|
|
|
|
def _assert_exact_keys(value: Mapping[str, Any], expected: set[str], *, label: str) -> None:
|
|
if set(value) != expected:
|
|
raise RebuildError("invalid_bundle_schema", f"{label} contains missing or unsupported fields")
|
|
|
|
|
|
def _actual_bundle_tree(root: Path) -> tuple[set[str], set[str]]:
|
|
files: set[str] = set()
|
|
directories: set[str] = set()
|
|
|
|
def visit(directory: Path, prefix: PurePosixPath) -> None:
|
|
try:
|
|
with os.scandir(directory) as stream:
|
|
entries = list(stream)
|
|
except OSError as exc:
|
|
raise RebuildError("bundle_scan_failed", "bundle tree could not be scanned") from exc
|
|
for entry in entries:
|
|
relative = (prefix / entry.name).as_posix()
|
|
_validate_relative_path(relative)
|
|
metadata = entry.stat(follow_symlinks=False)
|
|
if stat.S_ISLNK(metadata.st_mode):
|
|
raise RebuildError("unsafe_bundle_entry", "bundle contains a symlink")
|
|
if stat.S_ISDIR(metadata.st_mode):
|
|
directories.add(relative)
|
|
visit(Path(entry.path), prefix / entry.name)
|
|
elif stat.S_ISREG(metadata.st_mode):
|
|
files.add(relative)
|
|
else:
|
|
raise RebuildError("unsafe_bundle_entry", "bundle contains a special file")
|
|
|
|
visit(root, PurePosixPath())
|
|
return files, directories
|
|
|
|
|
|
def verify_bundle(bundle: Path) -> dict[str, Any]:
|
|
root = _validate_bundle_root(bundle)
|
|
if (root / INCOMPLETE_MARKER).exists() or (root / INCOMPLETE_MARKER).is_symlink():
|
|
raise RebuildError("incomplete_bundle", "bundle retains its incomplete marker")
|
|
|
|
source_manifest = _load_json(root / SOURCE_MANIFEST_PATH)
|
|
extraction_index = _load_json(root / EXTRACTION_WORK_INDEX_PATH)
|
|
receipt = _load_json(root / RECEIPT_PATH)
|
|
if source_manifest.get("schema") != SOURCE_MANIFEST_SCHEMA:
|
|
raise RebuildError("invalid_bundle_schema", "source manifest schema is unsupported")
|
|
if extraction_index.get("schema") != EXTRACTION_WORK_INDEX_SCHEMA:
|
|
raise RebuildError("invalid_bundle_schema", "extraction-work index schema is unsupported")
|
|
if receipt.get("schema") != RECEIPT_SCHEMA:
|
|
raise RebuildError("invalid_bundle_schema", "receipt schema is unsupported")
|
|
for label, value in (
|
|
("source manifest", source_manifest),
|
|
("extraction-work index", extraction_index),
|
|
("receipt", receipt),
|
|
):
|
|
if value.get("mode") != MODE_SOURCE_INVENTORY_ONLY:
|
|
raise RebuildError("invalid_bundle_schema", f"{label} mode is unsupported")
|
|
_assert_no_write_flags(value, label=label)
|
|
_assert_exact_keys(
|
|
source_manifest,
|
|
{
|
|
"admission_performed",
|
|
"database_write_performed",
|
|
"duplicate_content_source_count",
|
|
"mode",
|
|
"resource_limits",
|
|
"schema",
|
|
"semantic_claim_extraction_performed",
|
|
"source_count",
|
|
"source_snapshot_semantics",
|
|
"sources",
|
|
"symlinks_followed",
|
|
"total_source_bytes",
|
|
"unique_snapshot_count",
|
|
"unsupported_files_ignored",
|
|
},
|
|
label="source manifest",
|
|
)
|
|
_assert_exact_keys(
|
|
extraction_index,
|
|
{
|
|
"admission_performed",
|
|
"database_write_performed",
|
|
"items",
|
|
"mode",
|
|
"schema",
|
|
"semantic_claim_extraction_performed",
|
|
"source_count",
|
|
"source_manifest",
|
|
"work_item_count",
|
|
"work_type",
|
|
},
|
|
label="extraction-work index",
|
|
)
|
|
_assert_exact_keys(
|
|
receipt,
|
|
{
|
|
"admission_performed",
|
|
"database_write_performed",
|
|
"deterministic",
|
|
"hashes",
|
|
"mode",
|
|
"outputs",
|
|
"resource_limits",
|
|
"result",
|
|
"schema",
|
|
"semantic_claim_extraction_performed",
|
|
"source_count",
|
|
"source_snapshot_atomic",
|
|
"status",
|
|
"total_source_bytes",
|
|
"unique_snapshot_count",
|
|
"work_item_count",
|
|
},
|
|
label="receipt",
|
|
)
|
|
if receipt.get("status") != "pass" or receipt.get("result") != "source_inventory_bundle_ready":
|
|
raise RebuildError("invalid_bundle_schema", "receipt does not represent a completed inventory bundle")
|
|
if receipt.get("source_snapshot_atomic") is not False:
|
|
raise RebuildError("false_snapshot_claim", "receipt must not claim an atomic source-directory snapshot")
|
|
semantics = source_manifest.get("source_snapshot_semantics")
|
|
if semantics != {
|
|
"concurrent_directory_additions_or_removals_may_be_absent": True,
|
|
"directory_snapshot_atomic": False,
|
|
"guarantee": "each listed path is bound to the exact bytes copied from one stable file descriptor",
|
|
}:
|
|
raise RebuildError("false_snapshot_claim", "source manifest must bound concurrent-directory snapshot claims")
|
|
if receipt.get("deterministic") is not True:
|
|
raise RebuildError("invalid_bundle_schema", "receipt deterministic flag is unsupported")
|
|
if source_manifest.get("unsupported_files_ignored") is not False:
|
|
raise RebuildError("invalid_bundle_schema", "source manifest must not silently ignore unsupported files")
|
|
if source_manifest.get("symlinks_followed") is not False:
|
|
raise RebuildError("invalid_bundle_schema", "source manifest must not claim that symlinks were followed")
|
|
if extraction_index.get("work_type") != WORK_TYPE:
|
|
raise RebuildError("invalid_work_index", "extraction-work type is unsupported")
|
|
|
|
limits_value = receipt.get("resource_limits")
|
|
if not isinstance(limits_value, dict):
|
|
raise RebuildError("invalid_bundle_schema", "receipt resource limits are missing")
|
|
limits = _limits_from_mapping(limits_value)
|
|
if source_manifest.get("resource_limits") != limits.as_dict():
|
|
raise RebuildError("invalid_bundle_schema", "source manifest resource limits differ from the receipt")
|
|
|
|
sources = source_manifest.get("sources")
|
|
items = extraction_index.get("items")
|
|
if not isinstance(sources, list) or not isinstance(items, list):
|
|
raise RebuildError("invalid_bundle_schema", "source and work-item collections must be arrays")
|
|
if not sources or len(sources) > limits.max_source_count:
|
|
raise RebuildError("invalid_bundle_cardinality", "source count is empty or exceeds its limit")
|
|
if len(items) > limits.max_work_items:
|
|
raise RebuildError("invalid_bundle_cardinality", "work-item count exceeds its limit")
|
|
|
|
expected_items: list[dict[str, Any]] = []
|
|
snapshot_paths: set[str] = set()
|
|
path_keys: set[str] = set()
|
|
total_bytes = 0
|
|
previous_path: str | None = None
|
|
for source in sources:
|
|
if not isinstance(source, dict):
|
|
raise RebuildError("invalid_bundle_schema", "source rows must be objects")
|
|
_assert_exact_keys(
|
|
source,
|
|
{
|
|
"artifact_sha256",
|
|
"bytes",
|
|
"content_validation",
|
|
"extension",
|
|
"format_syntax_validated",
|
|
"newline_style",
|
|
"relative_path",
|
|
"section_count",
|
|
"sectioning_strategy",
|
|
"sections",
|
|
"snapshot_path",
|
|
"source_path_binding",
|
|
},
|
|
label="source row",
|
|
)
|
|
relative_path = source.get("relative_path")
|
|
extension = source.get("extension")
|
|
digest = source.get("artifact_sha256")
|
|
byte_count = source.get("bytes")
|
|
snapshot_relative = source.get("snapshot_path")
|
|
if not isinstance(relative_path, str) or not isinstance(extension, str):
|
|
raise RebuildError("invalid_bundle_schema", "source path or extension is malformed")
|
|
_validate_relative_path(relative_path)
|
|
if previous_path is not None and relative_path.encode("utf-8") <= previous_path.encode("utf-8"):
|
|
raise RebuildError("invalid_bundle_order", "source rows are not in deterministic path order")
|
|
previous_path = relative_path
|
|
path_key = _normalized_path_key(relative_path)
|
|
if path_key in path_keys:
|
|
raise RebuildError("duplicate_relative_path", "source rows contain a normalized path collision")
|
|
path_keys.add(path_key)
|
|
if extension != PurePosixPath(relative_path).suffix.casefold() or extension not in ALLOWED_EXTENSIONS:
|
|
raise RebuildError("invalid_bundle_schema", "source extension binding is invalid")
|
|
if not isinstance(digest, str) or not SHA256_RE.fullmatch(digest):
|
|
raise RebuildError("invalid_bundle_schema", "source hash is malformed")
|
|
if isinstance(byte_count, bool) or not isinstance(byte_count, int) or byte_count <= 0:
|
|
raise RebuildError("invalid_bundle_schema", "source byte count is malformed")
|
|
if byte_count > limits.max_source_bytes:
|
|
raise RebuildError("source_byte_limit_exceeded", "bundle source exceeds its recorded limit")
|
|
total_bytes += byte_count
|
|
if total_bytes > limits.max_total_bytes:
|
|
raise RebuildError("total_byte_limit_exceeded", "bundle exceeds its recorded total-byte limit")
|
|
if snapshot_relative != _snapshot_path_for_hash(digest).as_posix():
|
|
raise RebuildError("invalid_bundle_schema", "source snapshot path is not content addressed")
|
|
snapshot_paths.add(snapshot_relative)
|
|
snapshot = root / Path(snapshot_relative)
|
|
text = _read_snapshot_text(snapshot, expected_sha256=digest, expected_bytes=byte_count)
|
|
expected_sections = _build_sections(text, extension, max_sections=limits.max_sections_per_source)
|
|
if source.get("sections") != expected_sections or source.get("section_count") != len(expected_sections):
|
|
raise RebuildError("invalid_section_locator", "source section locators do not match snapshot bytes")
|
|
if source.get("newline_style") != _newline_style(text):
|
|
raise RebuildError("invalid_bundle_schema", "source newline style does not match snapshot bytes")
|
|
if source.get("content_validation") != "strict_utf8_no_binary_controls":
|
|
raise RebuildError("invalid_bundle_schema", "source content-validation label is unsupported")
|
|
if source.get("format_syntax_validated") is not False:
|
|
raise RebuildError("invalid_bundle_schema", "inventory must not claim format-level validation")
|
|
if source.get("sectioning_strategy") != _sectioning_strategy(extension):
|
|
raise RebuildError("invalid_bundle_schema", "source sectioning strategy is unsupported")
|
|
if source.get("source_path_binding") != "source_root_relative_posix_path_plus_exact_snapshot_sha256":
|
|
raise RebuildError("invalid_bundle_schema", "source path-binding label is unsupported")
|
|
expected_items.extend(_work_item(source, locator) for locator in expected_sections)
|
|
del text, expected_sections
|
|
|
|
if items != expected_items:
|
|
raise RebuildError("invalid_work_index", "extraction-work rows do not exactly match source section locators")
|
|
if any(item.get("work_type") != WORK_TYPE for item in items if isinstance(item, dict)):
|
|
raise RebuildError("invalid_work_index", "extraction-work type is unsupported")
|
|
if any(not isinstance(item, dict) for item in items):
|
|
raise RebuildError("invalid_work_index", "extraction-work rows must be objects")
|
|
if len({item["work_item_id"] for item in items}) != len(items):
|
|
raise RebuildError("invalid_work_index", "extraction-work IDs are not unique")
|
|
|
|
expected_counts = {
|
|
"source_count": len(sources),
|
|
"total_source_bytes": total_bytes,
|
|
"unique_snapshot_count": len(snapshot_paths),
|
|
}
|
|
for key, expected in expected_counts.items():
|
|
if source_manifest.get(key) != expected or receipt.get(key) != expected:
|
|
raise RebuildError("invalid_bundle_cardinality", f"{key} does not match verified content")
|
|
if source_manifest.get("duplicate_content_source_count") != len(sources) - len(snapshot_paths):
|
|
raise RebuildError("invalid_bundle_cardinality", "duplicate-content source count is incorrect")
|
|
if source_manifest.get("source_count") != extraction_index.get("source_count"):
|
|
raise RebuildError("invalid_bundle_cardinality", "source counts differ between bundle indexes")
|
|
if extraction_index.get("work_item_count") != len(items) or receipt.get("work_item_count") != len(items):
|
|
raise RebuildError("invalid_bundle_cardinality", "work-item counts differ from verified content")
|
|
manifest_binding = extraction_index.get("source_manifest")
|
|
if manifest_binding != {"path": SOURCE_MANIFEST_PATH, "sha256": sha256_file(root / SOURCE_MANIFEST_PATH)}:
|
|
raise RebuildError("invalid_bundle_hash", "extraction-work index does not bind the source manifest")
|
|
|
|
artifact_paths = [SOURCE_MANIFEST_PATH, EXTRACTION_WORK_INDEX_PATH, *snapshot_paths]
|
|
artifacts = _artifact_records(root, artifact_paths)
|
|
outputs = receipt.get("outputs")
|
|
hashes = receipt.get("hashes")
|
|
if not isinstance(outputs, dict) or not isinstance(hashes, dict):
|
|
raise RebuildError("invalid_bundle_schema", "receipt outputs or hashes are malformed")
|
|
_assert_exact_keys(
|
|
outputs,
|
|
{"artifact_count", "artifacts", "extraction_work_index_path", "source_manifest_path"},
|
|
label="receipt outputs",
|
|
)
|
|
_assert_exact_keys(
|
|
hashes,
|
|
{
|
|
"bundle_tree_sha256",
|
|
"extraction_work_index_file_sha256",
|
|
"receipt_content_sha256",
|
|
"source_manifest_file_sha256",
|
|
},
|
|
label="receipt hashes",
|
|
)
|
|
if outputs.get("artifacts") != artifacts or outputs.get("artifact_count") != len(artifacts):
|
|
raise RebuildError("invalid_bundle_hash", "receipt artifact inventory does not match bundle bytes")
|
|
if outputs.get("source_manifest_path") != SOURCE_MANIFEST_PATH:
|
|
raise RebuildError("invalid_bundle_schema", "receipt source-manifest path is incorrect")
|
|
if outputs.get("extraction_work_index_path") != EXTRACTION_WORK_INDEX_PATH:
|
|
raise RebuildError("invalid_bundle_schema", "receipt extraction-work path is incorrect")
|
|
expected_hashes = {
|
|
"bundle_tree_sha256": canonical_sha256(artifacts),
|
|
"extraction_work_index_file_sha256": sha256_file(root / EXTRACTION_WORK_INDEX_PATH),
|
|
"source_manifest_file_sha256": sha256_file(root / SOURCE_MANIFEST_PATH),
|
|
}
|
|
for key, expected in expected_hashes.items():
|
|
if hashes.get(key) != expected:
|
|
raise RebuildError("invalid_bundle_hash", f"receipt {key} does not match bundle bytes")
|
|
receipt_preimage = copy.deepcopy(receipt)
|
|
preimage_hashes = receipt_preimage.get("hashes")
|
|
if not isinstance(preimage_hashes, dict):
|
|
raise RebuildError("invalid_bundle_schema", "receipt hash object is malformed")
|
|
recorded_receipt_hash = preimage_hashes.pop("receipt_content_sha256", None)
|
|
if recorded_receipt_hash != canonical_sha256(receipt_preimage):
|
|
raise RebuildError("invalid_receipt_hash", "receipt content hash does not verify")
|
|
|
|
actual_files, actual_directories = _actual_bundle_tree(root)
|
|
expected_files = {RECEIPT_PATH, *artifact_paths}
|
|
if actual_files != expected_files:
|
|
raise RebuildError("unexpected_bundle_entry", "bundle contains missing or unexpected files")
|
|
expected_directories = {"snapshots", "snapshots/sha256"}
|
|
expected_directories.update(str(PurePosixPath(path).parent) for path in snapshot_paths)
|
|
if actual_directories != expected_directories:
|
|
raise RebuildError("unexpected_bundle_entry", "bundle contains missing or unexpected directories")
|
|
|
|
return {
|
|
"bundle_path": root,
|
|
"bundle_tree_sha256": expected_hashes["bundle_tree_sha256"],
|
|
"receipt_file_sha256": sha256_file(root / RECEIPT_PATH),
|
|
"source_count": len(sources),
|
|
"status": "pass",
|
|
"unique_snapshot_count": len(snapshot_paths),
|
|
"work_item_count": len(items),
|
|
}
|
|
|
|
|
|
def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--source-root", type=Path, help="directory of strict UTF-8 source files")
|
|
parser.add_argument("--output", type=Path, help="new bundle directory; must not already exist")
|
|
parser.add_argument("--mode", default=MODE_SOURCE_INVENTORY_ONLY, help="must be source-inventory-only")
|
|
parser.add_argument("--verify-bundle", type=Path, help="read-only validation of an existing bundle")
|
|
parser.add_argument("--max-source-bytes", type=int, default=DEFAULT_MAX_SOURCE_BYTES)
|
|
parser.add_argument("--max-total-bytes", type=int, default=DEFAULT_MAX_TOTAL_BYTES)
|
|
parser.add_argument("--max-source-count", type=int, default=DEFAULT_MAX_SOURCE_COUNT)
|
|
parser.add_argument("--max-sections-per-source", type=int, default=DEFAULT_MAX_SECTIONS_PER_SOURCE)
|
|
parser.add_argument("--max-work-items", type=int, default=DEFAULT_MAX_WORK_ITEMS)
|
|
return parser.parse_args(argv)
|
|
|
|
|
|
def _print_status(
|
|
status: str,
|
|
*,
|
|
operation: str,
|
|
reason: str | None = None,
|
|
message: str | None = None,
|
|
result: Mapping[str, Any] | None = None,
|
|
) -> None:
|
|
payload: dict[str, Any] = {
|
|
"schema": STATUS_SCHEMA,
|
|
"status": status,
|
|
"operation": operation,
|
|
"mode": MODE_SOURCE_INVENTORY_ONLY,
|
|
**NO_WRITE_FLAGS,
|
|
}
|
|
if reason is not None:
|
|
payload["reason"] = reason
|
|
if message is not None:
|
|
payload["message"] = message
|
|
if result is not None:
|
|
payload.update(
|
|
{
|
|
"bundle_path": str(result["bundle_path"]),
|
|
"bundle_tree_sha256": result["bundle_tree_sha256"],
|
|
"receipt_file_sha256": result["receipt_file_sha256"],
|
|
"source_count": result["source_count"],
|
|
"unique_snapshot_count": result["unique_snapshot_count"],
|
|
"work_item_count": result["work_item_count"],
|
|
}
|
|
)
|
|
print(json.dumps(payload, ensure_ascii=True, sort_keys=True))
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
args = parse_args(argv)
|
|
operation = "verify" if args.verify_bundle is not None else "build"
|
|
try:
|
|
if args.verify_bundle is not None:
|
|
if args.source_root is not None or args.output is not None:
|
|
raise RebuildError("invalid_arguments", "verify mode does not accept source-root or output")
|
|
result = verify_bundle(args.verify_bundle)
|
|
else:
|
|
if args.source_root is None or args.output is None:
|
|
raise RebuildError("invalid_arguments", "build mode requires source-root and output")
|
|
limits = ResourceLimits(
|
|
max_source_bytes=args.max_source_bytes,
|
|
max_total_bytes=args.max_total_bytes,
|
|
max_source_count=args.max_source_count,
|
|
max_sections_per_source=args.max_sections_per_source,
|
|
max_work_items=args.max_work_items,
|
|
)
|
|
built = build_source_inventory_bundle(args.source_root, args.output, mode=args.mode, limits=limits)
|
|
result = built["verification"]
|
|
except RebuildError as exc:
|
|
_print_status("rejected", operation=operation, reason=exc.code, message=exc.safe_message)
|
|
return 2
|
|
except OSError:
|
|
_print_status(
|
|
"rejected",
|
|
operation=operation,
|
|
reason="filesystem_operation_failed",
|
|
message="a fail-closed filesystem operation did not complete",
|
|
)
|
|
return 2
|
|
_print_status("pass", operation=operation, result=result)
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|