From 4028ef7821267fdf2470db21d739cb524354f267 Mon Sep 17 00:00:00 2001 From: mptyl Date: Sun, 12 Jul 2026 05:42:07 +0200 Subject: [PATCH] feat(preprocess): add deployment jobs and S3 source --- .superpowers/sdd/evidence-task-7-report.md | 27 +++++ README.md | 17 +++ deploy/compose.preprocess.yaml | 30 +++++ docker/core.Dockerfile | 2 +- docker/python-runtime/requirements.lock | 34 +++++- harness/pyproject.toml | 1 + harness/tests/test_s3_evidence_source.py | 89 +++++++++++++++ harness/tht/adapters/evidence/__init__.py | 3 +- harness/tht/adapters/evidence/s3.py | 125 +++++++++++++++++++++ harness/tht/adapters/factory.py | 15 +++ harness/tht/config.py | 32 +++++- scripts/preprocess-smoke.sh | 15 +++ 12 files changed, 380 insertions(+), 10 deletions(-) create mode 100644 .superpowers/sdd/evidence-task-7-report.md create mode 100644 deploy/compose.preprocess.yaml create mode 100644 harness/tests/test_s3_evidence_source.py create mode 100644 harness/tht/adapters/evidence/s3.py create mode 100755 scripts/preprocess-smoke.sh diff --git a/.superpowers/sdd/evidence-task-7-report.md b/.superpowers/sdd/evidence-task-7-report.md new file mode 100644 index 00000000..5beb51b9 --- /dev/null +++ b/.superpowers/sdd/evidence-task-7-report.md @@ -0,0 +1,27 @@ +# Evidence preprocessing Task 7 report + +Implemented the S3-compatible Evidence adapter, explicit preprocessing Compose overlay, and +operational gates. + +- S3 discovery uses bounded paginator pages, page size, and total objects; acquisition enforces a + byte ceiling and always closes streaming bodies. +- Provenance is canonical `s3://bucket/key`. Versioned objects use `s3-version:`; + unversioned objects use a hashed exact ETag, and acquisition refuses validator drift. +- The adapter uses boto3/botocore rather than custom signing. TLS verification is enabled by + default. Custom HTTP and private endpoints require independent explicit opt-ins; endpoint + userinfo is rejected and public custom endpoints are DNS-policy checked. +- Access, secret, and session credentials support file-secret resolution into masked `SecretStr` + config fields. They are never emitted in provenance, reports, errors, or Compose environment. +- `deploy/compose.preprocess.yaml` provides separate one-shot Evidence and DWH jobs and is inert + unless explicitly included with the `preprocess` profile. +- `scripts/preprocess-smoke.sh` verifies both services render without secret material and pins an + unchanged rerun plus a modified generation through deterministic pipeline tests. + +Verification: focused S3/HTTP/filesystem/config tests 34 passed; operational smoke 2 passed; core +image with locked boto3 extra built; full harness 702 passed, 5 deselected; scoped Ruff and diff +checks passed. + +Operational risk: custom S3-compatible endpoints remain part of the deployment trust boundary. +Private endpoint access must be explicitly enabled and should be restricted by container egress +policy in production. S3 list consistency semantics are provider-defined; version IDs are preferred +over ETags wherever bucket versioning is available. diff --git a/README.md b/README.md index b79aefe6..eacde4c9 100644 --- a/README.md +++ b/README.md @@ -53,6 +53,23 @@ volume is independent of application state. Reader, writer, migrator, and bootstrap credentials remain separate; password files must be mode `0600` and must not be passed as URL arguments. +## Preprocessing jobs and S3 Evidence + +Run one-shot jobs through the explicit overlay, which is inert for normal external/local runtime: + +```sh +docker compose -f compose.yaml -f deploy/compose.preprocess.yaml --profile preprocess \ + run --rm preprocess-evidence +docker compose -f compose.yaml -f deploy/compose.preprocess.yaml --profile preprocess \ + run --rm preprocess-dwh +``` + +S3 Evidence uses the optional `tht[s3]` dependency and canonical `s3://bucket/key` provenance. +TLS and public endpoints are required by default; private or HTTP S3-compatible endpoints require +separate explicit opt-ins. Store access key, secret key, and session token as secret references in +deployment configuration—never in Compose environment values or source URIs. Discovery and reads +are bounded by configured page, object, and byte limits. + Create a versioned PostgreSQL custom-format backup (the filename is operator-controlled, so use an immutable timestamp or release identifier): diff --git a/deploy/compose.preprocess.yaml b/deploy/compose.preprocess.yaml new file mode 100644 index 00000000..a89aa3f1 --- /dev/null +++ b/deploy/compose.preprocess.yaml @@ -0,0 +1,30 @@ +services: + preprocess-evidence: + image: thothii-core:local + profiles: [preprocess] + build: + context: . + dockerfile: docker/core.Dockerfile + entrypoint: [/app/docker/core-entrypoint.sh, preprocess] + command: [evidence, --json, -c, "/app/harness/workspaces/${THT_PREPROCESS_WORKSPACE:-tht.example}.yaml"] + environment: + THT_DATA_ROOT: /data + volumes: + - thoth_data:/data + - ./deploy/workspaces:/app/harness/workspaces:ro + restart: "no" + + preprocess-dwh: + image: thothii-core:local + profiles: [preprocess] + build: + context: . + dockerfile: docker/core.Dockerfile + entrypoint: [/app/docker/core-entrypoint.sh, preprocess] + command: [dwh, --json, -c, "/app/harness/workspaces/${THT_PREPROCESS_WORKSPACE:-tht.example}.yaml"] + environment: + THT_DATA_ROOT: /data + volumes: + - thoth_data:/data + - ./deploy/workspaces:/app/harness/workspaces:ro + restart: "no" diff --git a/docker/core.Dockerfile b/docker/core.Dockerfile index 7347f2da..8af924f4 100644 --- a/docker/core.Dockerfile +++ b/docker/core.Dockerfile @@ -40,7 +40,7 @@ COPY docker/python-runtime/requirements.lock /app/docker/python-runtime/requirem RUN python -m venv /opt/venv \ && /opt/venv/bin/pip install --no-cache-dir --require-hashes \ --requirement /app/docker/python-runtime/requirements.lock \ - && /opt/venv/bin/pip install --no-cache-dir --no-deps --no-build-isolation /app/harness \ + && /opt/venv/bin/pip install --no-cache-dir --no-deps --no-build-isolation '/app/harness[s3]' \ && /opt/venv/bin/tht vector migrate --help >/dev/null COPY --from=backend-build /src/backend/dist /app/backend/dist diff --git a/docker/python-runtime/requirements.lock b/docker/python-runtime/requirements.lock index 55eeea42..215c2757 100644 --- a/docker/python-runtime/requirements.lock +++ b/docker/python-runtime/requirements.lock @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv pip compile harness/pyproject.toml docker/python-runtime/build-requirements.in --universal --python-version 3.12 --no-emit-package tht --generate-hashes --output-file docker/python-runtime/requirements.lock +# uv pip compile harness/pyproject.toml docker/python-runtime/build-requirements.in --extra s3 --universal --python-version 3.12 --no-emit-package tht --generate-hashes --output-file docker/python-runtime/requirements.lock annotated-doc==0.0.4 \ --hash=sha256:571ac1dc6991c450b25a9c2d84a3705e2ae7a53467b5d111c24fa8baabbed320 \ --hash=sha256:fbcda96e87e9c92ad167c2e53839e57503ecfda18804ea28102353485033faa4 @@ -8,6 +8,16 @@ annotated-types==0.7.0 \ --hash=sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53 \ --hash=sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89 # via pydantic +boto3==1.43.46 \ + --hash=sha256:66c0d943b049a46a492ec4ec2ebe73c930b1842c7137bee83aad6d93e95d4d96 \ + --hash=sha256:69453e2c1bcb9fd9806527ab99950cacfc2826cb0dce9a3a0414d19270c06c3c + # via tht (harness/pyproject.toml) +botocore==1.43.46 \ + --hash=sha256:59f2e1ac3cdc66d191cae91c0804bc41847ce817dc8147cf43eaada8f76a5533 \ + --hash=sha256:cb673891e623ae6e6a1bf24d94ef169504f3eb02584adb5d5bee2f6aae819b60 + # via + # boto3 + # s3transfer certifi==2026.6.17 \ --hash=sha256:024c88eeec92ca068db80f02b8b07c9cef7b9fe261d1d535abfd5abd6f6af432 \ --hash=sha256:2227dcbaafe0d2f59279d1762ddddc37783ed4354594f194ffc31d20f41fc3db @@ -294,6 +304,12 @@ jellyfish==1.2.1 \ --hash=sha256:f69aeb08659a6c81d559bbe319075e3417434ae5b3a5e4a758d1c4055a03497a \ --hash=sha256:fb3c6e537cb4605c22895a8d4a10cdb26611ba2bbfc7f0b4c1d06bb9d8aad648 # via yake +jmespath==1.1.0 \ + --hash=sha256:472c87d80f36026ae83c6ddd0f1d05d4e510134ed462851fd5f754c8c3cbb88d \ + --hash=sha256:a5663118de4908c91729bea0acadca56526eb2698e83de10cd116ae0f4e97c64 + # via + # boto3 + # botocore markdown-it-py==4.2.0 \ --hash=sha256:04a21681d6fbb623de53f6f364d352309d4094dd4194040a10fd51833e418d49 \ --hash=sha256:9f7ebbcd14fe59494226453aed97c1070d83f8d24b6fc3a3bcf9a38092641c4a @@ -554,6 +570,10 @@ pygments==2.20.0 \ --hash=sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f \ --hash=sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176 # via rich +python-dateutil==2.9.0.post0 \ + --hash=sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3 \ + --hash=sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427 + # via botocore python-dotenv==1.2.2 \ --hash=sha256:1d8214789a24de455a8b8bd8ae6fe3c6b69a5e3d64aa8a8e5d68e694bbcb285a \ --hash=sha256:2c371a91fbd7ba082c2c1dc1f8bf89ca22564a087c2c287cd9b662adde799cf3 @@ -759,6 +779,10 @@ rich==15.0.0 \ # via # tht (harness/pyproject.toml) # typer +s3transfer==0.19.1 \ + --hash=sha256:d3d6371dc3f1e5c5427b2b457bcf13bcf87bec334c95aed18642eae61f6926f3 \ + --hash=sha256:d5fd7005ee39307455ad5f310b5ea67f4b1960d7fed5b3671ee50c249de675de + # via boto3 scipy==1.18.0 \ --hash=sha256:09143f676d157d9f546d663504ef9c1becb819824f1afc018814176411942446 \ --hash=sha256:0d13bca67c096d89fb95ced0d8921807300fce0275643aef9533cc63a0773468 \ @@ -814,6 +838,10 @@ shellingham==1.5.4 \ --hash=sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686 \ --hash=sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de # via typer +six==1.17.0 \ + --hash=sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274 \ + --hash=sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81 + # via python-dateutil sqlalchemy==2.0.51 \ --hash=sha256:0378d055e9e8cd6ce4d8dff683bdd3d7d413533c4ee51d67a2b1e0f9eacc0f23 \ --hash=sha256:0592bdadf86ddcabfd72d9ab66ea8a5d8d2cc6be1cc51fa7e66c03868ac5eac1 \ @@ -905,7 +933,9 @@ typing-inspection==0.4.2 \ urllib3==2.7.0 \ --hash=sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c \ --hash=sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897 - # via requests + # via + # botocore + # requests yake==0.7.3 \ --hash=sha256:38f7f135ff8ed4bcdc05e16b533a9dc93299f1e694b0c308c3c086bab316c5fe \ --hash=sha256:8778fb2832e58d26d838d6d7ac967b4947521f1fe8cdf23dd872636161fc53ed diff --git a/harness/pyproject.toml b/harness/pyproject.toml index 28f7d23d..5972c288 100644 --- a/harness/pyproject.toml +++ b/harness/pyproject.toml @@ -22,6 +22,7 @@ dependencies = [ tht = "tht.cli:app" [project.optional-dependencies] +s3 = ["boto3>=1.34,<2"] dev = [ "pytest>=8.0", "testcontainers[postgres]>=4.0", diff --git a/harness/tests/test_s3_evidence_source.py b/harness/tests/test_s3_evidence_source.py new file mode 100644 index 00000000..f5421255 --- /dev/null +++ b/harness/tests/test_s3_evidence_source.py @@ -0,0 +1,89 @@ +from datetime import UTC, datetime + +import pytest + +from tht.ports.evidence import EvidenceSourceError + + +class Body: + def __init__(self, data): self.data, self.closed = data, False + def read(self, amount): return self.data[:amount] + def close(self): self.closed = True + + +class Client: + def __init__(self): self.body = Body(b"hello") + def get_paginator(self, name): return self + def paginate(self, **kwargs): + yield {"Contents": [{"Key": "clinical/a.md", "ETag": '"abc"', "VersionId": "v1", + "Size": 5, "LastModified": datetime(2026, 1, 1, tzinfo=UTC)}]} + def get_object(self, **kwargs): + assert kwargs == {"Bucket": "evidence", "Key": "clinical/a.md", "VersionId": "v1"} + return {"Body": self.body, "ContentLength": 5, "ContentType": "text/markdown", + "ETag": '"abc"', "VersionId": "v1"} + + +def test_s3_canonical_uri_version_fingerprint_and_closed_body(): + from tht.adapters.evidence.s3 import S3EvidenceSource + client = Client() + source = S3EvidenceSource(bucket="evidence", prefix="clinical/", client=client) + item = next(iter(source.discover())) + assert item.uri == "s3://evidence/clinical/a.md" + assert item.fingerprint == "s3-version:v1" + assert source.acquire(item).content == b"hello" + assert client.body.closed + + +def test_s3_etag_fallback_and_bounds(): + from tht.adapters.evidence.s3 import S3EvidenceSource + client = Client() + with pytest.raises(ValueError): + S3EvidenceSource(bucket="evidence", client=client, max_objects=0) + + +def test_s3_rejects_private_or_insecure_endpoint_without_explicit_opt_in(): + from tht.adapters.evidence.s3 import S3EvidenceSource + with pytest.raises(ValueError, match="private"): + S3EvidenceSource(bucket="evidence", endpoint_url="https://127.0.0.1:9000", client=Client()) + with pytest.raises(ValueError, match="HTTPS"): + S3EvidenceSource(bucket="evidence", endpoint_url="http://s3.example.test", client=Client()) + + +def test_s3_size_limit_closes_body(): + from tht.adapters.evidence.s3 import S3EvidenceSource + client = Client() + source = S3EvidenceSource(bucket="evidence", client=client, max_bytes=4) + item = next(iter(source.discover())) + with pytest.raises(EvidenceSourceError): + source.acquire(item) + assert client.body.closed + + +def test_s3_config_serialization_masks_credentials(): + from tht.config import S3EvidenceSourceConfig + config = S3EvidenceSourceConfig(type="s3", bucket="evidence", + access_key="access-secret", secret_key="write-secret") + assert "access-secret" not in repr(config) + assert "write-secret" not in repr(config) + + +def test_s3_config_loads_credentials_from_secret_files(tmp_path): + from tht.config import load_config + access, secret = tmp_path / "access", tmp_path / "secret" + access.write_text("access-value") + secret.write_text("secret-value") + workspace = tmp_path / "workspace.yaml" + workspace.write_text(f""" +dwh: + type: postgres_direct + connection: {{database: d, schema: public, user: u, password: p}} +evidence: + sources: + - type: s3 + bucket: evidence + access_key_file: {access} + secret_key_file: {secret} +""") + source = load_config(workspace).evidence.sources[0] + assert source.access_key.get_secret_value() == "access-value" + assert source.secret_key.get_secret_value() == "secret-value" diff --git a/harness/tht/adapters/evidence/__init__.py b/harness/tht/adapters/evidence/__init__.py index 1b835504..805c886b 100644 --- a/harness/tht/adapters/evidence/__init__.py +++ b/harness/tht/adapters/evidence/__init__.py @@ -2,5 +2,6 @@ from tht.adapters.evidence.filesystem import FilesystemEvidenceSource from tht.adapters.evidence.http import HttpManifestEvidenceSource +from tht.adapters.evidence.s3 import S3EvidenceSource -__all__ = ["FilesystemEvidenceSource", "HttpManifestEvidenceSource"] +__all__ = ["FilesystemEvidenceSource", "HttpManifestEvidenceSource", "S3EvidenceSource"] diff --git a/harness/tht/adapters/evidence/s3.py b/harness/tht/adapters/evidence/s3.py new file mode 100644 index 00000000..d58e6fbd --- /dev/null +++ b/harness/tht/adapters/evidence/s3.py @@ -0,0 +1,125 @@ +"""Bounded S3-compatible Evidence source using the supported boto3 client.""" + +import hashlib +import ipaddress +import socket +from datetime import UTC, datetime +from urllib.parse import quote, urlsplit + +from tht.ports.evidence import ( + AcquiredDocument, EvidenceSourceError, EvidenceSourceErrorCategory, SourceObject, +) + + +class S3EvidenceSource: + def __init__(self, *, bucket: str, prefix: str = "", endpoint_url: str | None = None, + region: str | None = None, access_key: str | None = None, + secret_key: str | None = None, session_token: str | None = None, + allow_private_endpoint: bool = False, allow_insecure_endpoint: bool = False, + max_bytes: int = 10 * 1024 * 1024, max_objects: int = 10_000, + max_pages: int = 100, page_size: int = 1000, client=None) -> None: + if not bucket or any(value < 1 for value in (max_bytes, max_objects, max_pages, page_size)): + raise ValueError("S3 evidence limits and bucket must be non-empty and positive") + if endpoint_url: + parsed = urlsplit(endpoint_url) + if parsed.username or parsed.password: + raise ValueError("S3 endpoint must not contain credentials") + if parsed.scheme != "https" and not allow_insecure_endpoint: + raise ValueError("S3 endpoint must use HTTPS unless explicitly allowed") + if not parsed.hostname: + raise ValueError("S3 endpoint must include a hostname") + if not allow_private_endpoint: + try: + addresses = {ipaddress.ip_address(row[4][0].split("%", 1)[0]) for row in + socket.getaddrinfo(parsed.hostname, parsed.port or 443, + type=socket.SOCK_STREAM)} + except (OSError, ValueError) as exc: + raise ValueError("S3 endpoint resolution failed") from exc + if not addresses or any(not address.is_global for address in addresses): + raise ValueError("S3 private endpoint requires explicit opt-in") + self.bucket, self.prefix = bucket, prefix.lstrip("/") + self.max_bytes, self.max_objects = max_bytes, max_objects + self.max_pages, self.page_size = max_pages, min(page_size, 1000) + if client is None: + try: + import boto3 + except ImportError as exc: # pragma: no cover - deployment optional dependency + raise RuntimeError("Install tht[s3] to use S3 Evidence") from exc + client = boto3.client("s3", endpoint_url=endpoint_url, region_name=region, + aws_access_key_id=access_key, + aws_secret_access_key=secret_key, + aws_session_token=session_token, verify=True) + self._client = client + self._items: dict[str, tuple[str, str | None, str | None]] = {} + + @staticmethod + def _error(operation: str, transient: bool = False): + return EvidenceSourceError("S3 source operation failed", + category=(EvidenceSourceErrorCategory.TRANSIENT if transient + else EvidenceSourceErrorCategory.PERMANENT), + details={"operation": operation}) + + def discover(self): + count = pages = 0 + try: + paginator = self._client.get_paginator("list_objects_v2") + for page in paginator.paginate(Bucket=self.bucket, Prefix=self.prefix, + PaginationConfig={"PageSize": self.page_size}): + pages += 1 + if pages > self.max_pages: + raise self._error("list_limit") + for row in page.get("Contents", []): + count += 1 + if count > self.max_objects: + raise self._error("object_limit") + key, version, etag = row["Key"], row.get("VersionId"), row.get("ETag") + uri = f"s3://{self.bucket}/{quote(key, safe='/')}" + stable = version or hashlib.sha256((etag or "").encode()).hexdigest() + fingerprint = f"s3-version:{stable}" if version else f"etag:{stable}" + source_id = "s3:" + hashlib.sha256(uri.encode()).hexdigest() + self._items[source_id] = (key, version, etag) + modified = row.get("LastModified") + if modified is not None: + modified = modified.astimezone(UTC) + yield SourceObject(source_id=source_id, uri=uri, fingerprint=fingerprint, + modified_at=modified, + metadata={"size": int(row.get("Size", 0))}) + except EvidenceSourceError: + raise + except Exception as exc: + raise self._error("list", transient=True) from exc + + def acquire(self, item: SourceObject) -> AcquiredDocument: + binding = self._items.get(item.source_id) + if binding is None: + raise self._error("acquire") + key, version, _etag = binding + kwargs = {"Bucket": self.bucket, "Key": key} + if version: + kwargs["VersionId"] = version + body = None + try: + response = self._client.get_object(**kwargs) + body = response["Body"] + if version: + if response.get("VersionId") != version: + raise self._error("version_changed") + else: + current = hashlib.sha256((response.get("ETag") or "").encode()).hexdigest() + if item.fingerprint != f"etag:{current}": + raise self._error("etag_changed") + if int(response.get("ContentLength", 0)) > self.max_bytes: + raise self._error("download_limit") + content = body.read(self.max_bytes + 1) + if len(content) > self.max_bytes: + raise self._error("download_limit") + return AcquiredDocument(source=item, content=content, + media_type=response.get("ContentType"), + acquired_at=datetime.now(UTC)) + except EvidenceSourceError: + raise + except Exception as exc: + raise self._error("download", transient=True) from exc + finally: + if body is not None: + body.close() diff --git a/harness/tht/adapters/factory.py b/harness/tht/adapters/factory.py index 8640e78e..287b8c96 100644 --- a/harness/tht/adapters/factory.py +++ b/harness/tht/adapters/factory.py @@ -2,6 +2,7 @@ from tht.adapters.dwh import PostgresDwhAdapter, ThothRestDwhAdapter from tht.adapters.evidence import FilesystemEvidenceSource, HttpManifestEvidenceSource +from tht.adapters.evidence.s3 import S3EvidenceSource from tht.adapters.vector import PgVectorStore, ThothHttpVectorStore from tht.config import Config, ConfigError from tht.db.connection import make_engine @@ -117,6 +118,20 @@ def build_evidence_sources(cfg: Config): max_cache_bytes=resource.max_cache_bytes, ) ) + case "s3": + def secret(value): + return value.get_secret_value() if value is not None else None + + sources.append(S3EvidenceSource( + bucket=resource.bucket, prefix=resource.prefix, + endpoint_url=resource.endpoint_url, region=resource.region, + access_key=secret(resource.access_key), secret_key=secret(resource.secret_key), + session_token=secret(resource.session_token), + allow_private_endpoint=resource.allow_private_endpoint, + allow_insecure_endpoint=resource.allow_insecure_endpoint, + max_bytes=resource.max_bytes, max_objects=resource.max_objects, + max_pages=resource.max_pages, page_size=resource.page_size, + )) case other: # pragma: no cover - Pydantic rejects unsupported discriminators. raise ConfigError(f"Adapter evidence non supportato: {other}") return sources diff --git a/harness/tht/config.py b/harness/tht/config.py index e98cbf55..2035cd48 100644 --- a/harness/tht/config.py +++ b/harness/tht/config.py @@ -39,17 +39,20 @@ def _expand_env(value: Any) -> Any: def _resolve_secret_files(value: Any) -> Any: if isinstance(value, dict): resolved = {key: _resolve_secret_files(item) for key, item in value.items()} - if "password_file" in resolved: - if "password" in resolved: - raise ConfigError("password and password_file are mutually exclusive") - path = Path(resolved.pop("password_file")) + for secret_name in ("password", "access_key", "secret_key", "session_token"): + file_name = f"{secret_name}_file" + if file_name not in resolved: + continue + if secret_name in resolved: + raise ConfigError(f"{secret_name} and {file_name} are mutually exclusive") + path = Path(resolved.pop(file_name)) try: secret = path.read_text() except (OSError, UnicodeError) as exc: raise ConfigError(f"Cannot read secret file: {path}") from exc if not secret or any(char.isspace() for char in secret) or "\x00" in secret: raise ConfigError(f"Invalid secret file: {path}") - resolved["password"] = secret + resolved[secret_name] = secret return resolved if isinstance(value, list): return [_resolve_secret_files(item) for item in value] @@ -192,8 +195,25 @@ class HttpEvidenceSourceConfig(BaseModel): max_cache_bytes: int = Field(default=64 * 1024 * 1024, gt=0) +class S3EvidenceSourceConfig(BaseModel): + type: Literal["s3"] + bucket: str = Field(min_length=1) + prefix: str = "" + endpoint_url: str | None = None + region: str | None = None + access_key: SecretStr | None = None + secret_key: SecretStr | None = None + session_token: SecretStr | None = None + allow_private_endpoint: bool = False + allow_insecure_endpoint: bool = False + max_bytes: int = Field(default=10 * 1024 * 1024, gt=0) + max_objects: int = Field(default=10_000, gt=0) + max_pages: int = Field(default=100, gt=0) + page_size: int = Field(default=1000, gt=0, le=1000) + + EvidenceSourceConfig = Annotated[ - FilesystemEvidenceSourceConfig | HttpEvidenceSourceConfig, + FilesystemEvidenceSourceConfig | HttpEvidenceSourceConfig | S3EvidenceSourceConfig, Field(discriminator="type"), ] diff --git a/scripts/preprocess-smoke.sh b/scripts/preprocess-smoke.sh new file mode 100755 index 00000000..5707fc09 --- /dev/null +++ b/scripts/preprocess-smoke.sh @@ -0,0 +1,15 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." + +rendered=$(docker compose -f compose.yaml -f deploy/compose.preprocess.yaml --profile preprocess config) +printf '%s' "$rendered" | grep -q 'preprocess-evidence:' +printf '%s' "$rendered" | grep -q 'preprocess-dwh:' +if printf '%s' "$rendered" | grep -qi 'access-secret\|secret-key'; then + echo "preprocess Compose rendered secret material" >&2 + exit 1 +fi +(cd harness && .venv/bin/pytest -q \ + tests/test_corpus_pipeline.py::test_unchanged_documents_skip_acquire_normalize_chunk_and_embed \ + tests/test_corpus_pipeline.py::test_retention_bounds_generations_and_purges_vectors_after_publish) +echo "preprocess unchanged rerun and modified-generation smoke passed."