feat(preprocess): add deployment jobs and S3 source

.superpowers/sdd/evidence-task-7-report.md

@@ -0,0 +1,27 @@
+# Evidence preprocessing Task 7 report
+
+Implemented the S3-compatible Evidence adapter, explicit preprocessing Compose overlay, and
+operational gates.
+
+- S3 discovery uses bounded paginator pages, page size, and total objects; acquisition enforces a
+  byte ceiling and always closes streaming bodies.
+- Provenance is canonical `s3://bucket/key`. Versioned objects use `s3-version:<version>`;
+  unversioned objects use a hashed exact ETag, and acquisition refuses validator drift.
+- The adapter uses boto3/botocore rather than custom signing. TLS verification is enabled by
+  default. Custom HTTP and private endpoints require independent explicit opt-ins; endpoint
+  userinfo is rejected and public custom endpoints are DNS-policy checked.
+- Access, secret, and session credentials support file-secret resolution into masked `SecretStr`
+  config fields. They are never emitted in provenance, reports, errors, or Compose environment.
+- `deploy/compose.preprocess.yaml` provides separate one-shot Evidence and DWH jobs and is inert
+  unless explicitly included with the `preprocess` profile.
+- `scripts/preprocess-smoke.sh` verifies both services render without secret material and pins an
+  unchanged rerun plus a modified generation through deterministic pipeline tests.
+
+Verification: focused S3/HTTP/filesystem/config tests 34 passed; operational smoke 2 passed; core
+image with locked boto3 extra built; full harness 702 passed, 5 deselected; scoped Ruff and diff
+checks passed.
+
+Operational risk: custom S3-compatible endpoints remain part of the deployment trust boundary.
+Private endpoint access must be explicitly enabled and should be restricted by container egress
+policy in production. S3 list consistency semantics are provider-defined; version IDs are preferred
+over ETags wherever bucket versioning is available.

README.md

@@ -53,6 +53,23 @@ volume is independent of application state. Reader, writer,
 migrator, and bootstrap credentials remain separate; password files must be mode `0600` and
 must not be passed as URL arguments.
 
+## Preprocessing jobs and S3 Evidence
+
+Run one-shot jobs through the explicit overlay, which is inert for normal external/local runtime:
+
+```sh
+docker compose -f compose.yaml -f deploy/compose.preprocess.yaml --profile preprocess \
+  run --rm preprocess-evidence
+docker compose -f compose.yaml -f deploy/compose.preprocess.yaml --profile preprocess \
+  run --rm preprocess-dwh
+```
+
+S3 Evidence uses the optional `tht[s3]` dependency and canonical `s3://bucket/key` provenance.
+TLS and public endpoints are required by default; private or HTTP S3-compatible endpoints require
+separate explicit opt-ins. Store access key, secret key, and session token as secret references in
+deployment configuration—never in Compose environment values or source URIs. Discovery and reads
+are bounded by configured page, object, and byte limits.
+
 Create a versioned PostgreSQL custom-format backup (the filename is operator-controlled, so use
 an immutable timestamp or release identifier):
 

deploy/compose.preprocess.yaml

@@ -0,0 +1,30 @@
+services:
+  preprocess-evidence:
+    image: thothii-core:local
+    profiles: [preprocess]
+    build:
+      context: .
+      dockerfile: docker/core.Dockerfile
+    entrypoint: [/app/docker/core-entrypoint.sh, preprocess]
+    command: [evidence, --json, -c, "/app/harness/workspaces/${THT_PREPROCESS_WORKSPACE:-tht.example}.yaml"]
+    environment:
+      THT_DATA_ROOT: /data
+    volumes:
+      - thoth_data:/data
+      - ./deploy/workspaces:/app/harness/workspaces:ro
+    restart: "no"
+
+  preprocess-dwh:
+    image: thothii-core:local
+    profiles: [preprocess]
+    build:
+      context: .
+      dockerfile: docker/core.Dockerfile
+    entrypoint: [/app/docker/core-entrypoint.sh, preprocess]
+    command: [dwh, --json, -c, "/app/harness/workspaces/${THT_PREPROCESS_WORKSPACE:-tht.example}.yaml"]
+    environment:
+      THT_DATA_ROOT: /data
+    volumes:
+      - thoth_data:/data
+      - ./deploy/workspaces:/app/harness/workspaces:ro
+    restart: "no"

docker/core.Dockerfile

@@ -40,7 +40,7 @@ COPY docker/python-runtime/requirements.lock /app/docker/python-runtime/requirem
 RUN python -m venv /opt/venv \
     && /opt/venv/bin/pip install --no-cache-dir --require-hashes \
       --requirement /app/docker/python-runtime/requirements.lock \
-    && /opt/venv/bin/pip install --no-cache-dir --no-deps --no-build-isolation /app/harness \
+    && /opt/venv/bin/pip install --no-cache-dir --no-deps --no-build-isolation '/app/harness[s3]' \
     && /opt/venv/bin/tht vector migrate --help >/dev/null
 
 COPY --from=backend-build /src/backend/dist /app/backend/dist

docker/python-runtime/requirements.lock

@@ -1,5 +1,5 @@
 # This file was autogenerated by uv via the following command:
-#    uv pip compile harness/pyproject.toml docker/python-runtime/build-requirements.in --universal --python-version 3.12 --no-emit-package tht --generate-hashes --output-file docker/python-runtime/requirements.lock
+#    uv pip compile harness/pyproject.toml docker/python-runtime/build-requirements.in --extra s3 --universal --python-version 3.12 --no-emit-package tht --generate-hashes --output-file docker/python-runtime/requirements.lock
 annotated-doc==0.0.4 \
     --hash=sha256:571ac1dc6991c450b25a9c2d84a3705e2ae7a53467b5d111c24fa8baabbed320 \
     --hash=sha256:fbcda96e87e9c92ad167c2e53839e57503ecfda18804ea28102353485033faa4
@@ -8,6 +8,16 @@ annotated-types==0.7.0 \
     --hash=sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53 \
     --hash=sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89
     # via pydantic
+boto3==1.43.46 \
+    --hash=sha256:66c0d943b049a46a492ec4ec2ebe73c930b1842c7137bee83aad6d93e95d4d96 \
+    --hash=sha256:69453e2c1bcb9fd9806527ab99950cacfc2826cb0dce9a3a0414d19270c06c3c
+    # via tht (harness/pyproject.toml)
+botocore==1.43.46 \
+    --hash=sha256:59f2e1ac3cdc66d191cae91c0804bc41847ce817dc8147cf43eaada8f76a5533 \
+    --hash=sha256:cb673891e623ae6e6a1bf24d94ef169504f3eb02584adb5d5bee2f6aae819b60
+    # via
+    #   boto3
+    #   s3transfer
 certifi==2026.6.17 \
     --hash=sha256:024c88eeec92ca068db80f02b8b07c9cef7b9fe261d1d535abfd5abd6f6af432 \
     --hash=sha256:2227dcbaafe0d2f59279d1762ddddc37783ed4354594f194ffc31d20f41fc3db
@@ -294,6 +304,12 @@ jellyfish==1.2.1 \
     --hash=sha256:f69aeb08659a6c81d559bbe319075e3417434ae5b3a5e4a758d1c4055a03497a \
     --hash=sha256:fb3c6e537cb4605c22895a8d4a10cdb26611ba2bbfc7f0b4c1d06bb9d8aad648
     # via yake
+jmespath==1.1.0 \
+    --hash=sha256:472c87d80f36026ae83c6ddd0f1d05d4e510134ed462851fd5f754c8c3cbb88d \
+    --hash=sha256:a5663118de4908c91729bea0acadca56526eb2698e83de10cd116ae0f4e97c64
+    # via
+    #   boto3
+    #   botocore
 markdown-it-py==4.2.0 \
     --hash=sha256:04a21681d6fbb623de53f6f364d352309d4094dd4194040a10fd51833e418d49 \
     --hash=sha256:9f7ebbcd14fe59494226453aed97c1070d83f8d24b6fc3a3bcf9a38092641c4a
@@ -554,6 +570,10 @@ pygments==2.20.0 \
     --hash=sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f \
     --hash=sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176
     # via rich
+python-dateutil==2.9.0.post0 \
+    --hash=sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3 \
+    --hash=sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427
+    # via botocore
 python-dotenv==1.2.2 \
     --hash=sha256:1d8214789a24de455a8b8bd8ae6fe3c6b69a5e3d64aa8a8e5d68e694bbcb285a \
     --hash=sha256:2c371a91fbd7ba082c2c1dc1f8bf89ca22564a087c2c287cd9b662adde799cf3
@@ -759,6 +779,10 @@ rich==15.0.0 \
     # via
     #   tht (harness/pyproject.toml)
     #   typer
+s3transfer==0.19.1 \
+    --hash=sha256:d3d6371dc3f1e5c5427b2b457bcf13bcf87bec334c95aed18642eae61f6926f3 \
+    --hash=sha256:d5fd7005ee39307455ad5f310b5ea67f4b1960d7fed5b3671ee50c249de675de
+    # via boto3
 scipy==1.18.0 \
     --hash=sha256:09143f676d157d9f546d663504ef9c1becb819824f1afc018814176411942446 \
     --hash=sha256:0d13bca67c096d89fb95ced0d8921807300fce0275643aef9533cc63a0773468 \
@@ -814,6 +838,10 @@ shellingham==1.5.4 \
     --hash=sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686 \
     --hash=sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de
     # via typer
+six==1.17.0 \
+    --hash=sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274 \
+    --hash=sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81
+    # via python-dateutil
 sqlalchemy==2.0.51 \
     --hash=sha256:0378d055e9e8cd6ce4d8dff683bdd3d7d413533c4ee51d67a2b1e0f9eacc0f23 \
     --hash=sha256:0592bdadf86ddcabfd72d9ab66ea8a5d8d2cc6be1cc51fa7e66c03868ac5eac1 \
@@ -905,7 +933,9 @@ typing-inspection==0.4.2 \
 urllib3==2.7.0 \
     --hash=sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c \
     --hash=sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897
-    # via requests
+    # via
+    #   botocore
+    #   requests
 yake==0.7.3 \
     --hash=sha256:38f7f135ff8ed4bcdc05e16b533a9dc93299f1e694b0c308c3c086bab316c5fe \
     --hash=sha256:8778fb2832e58d26d838d6d7ac967b4947521f1fe8cdf23dd872636161fc53ed

harness/pyproject.toml

@@ -22,6 +22,7 @@ dependencies = [
 tht = "tht.cli:app"
 
 [project.optional-dependencies]
+s3 = ["boto3>=1.34,<2"]
 dev = [
     "pytest>=8.0",
     "testcontainers[postgres]>=4.0",

harness/tests/test_s3_evidence_source.py

@@ -0,0 +1,89 @@
+from datetime import UTC, datetime
+
+import pytest
+
+from tht.ports.evidence import EvidenceSourceError
+
+
+class Body:
+    def __init__(self, data): self.data, self.closed = data, False
+    def read(self, amount): return self.data[:amount]
+    def close(self): self.closed = True
+
+
+class Client:
+    def __init__(self): self.body = Body(b"hello")
+    def get_paginator(self, name): return self
+    def paginate(self, **kwargs):
+        yield {"Contents": [{"Key": "clinical/a.md", "ETag": '"abc"', "VersionId": "v1",
+                              "Size": 5, "LastModified": datetime(2026, 1, 1, tzinfo=UTC)}]}
+    def get_object(self, **kwargs):
+        assert kwargs == {"Bucket": "evidence", "Key": "clinical/a.md", "VersionId": "v1"}
+        return {"Body": self.body, "ContentLength": 5, "ContentType": "text/markdown",
+                "ETag": '"abc"', "VersionId": "v1"}
+
+
+def test_s3_canonical_uri_version_fingerprint_and_closed_body():
+    from tht.adapters.evidence.s3 import S3EvidenceSource
+    client = Client()
+    source = S3EvidenceSource(bucket="evidence", prefix="clinical/", client=client)
+    item = next(iter(source.discover()))
+    assert item.uri == "s3://evidence/clinical/a.md"
+    assert item.fingerprint == "s3-version:v1"
+    assert source.acquire(item).content == b"hello"
+    assert client.body.closed
+
+
+def test_s3_etag_fallback_and_bounds():
+    from tht.adapters.evidence.s3 import S3EvidenceSource
+    client = Client()
+    with pytest.raises(ValueError):
+        S3EvidenceSource(bucket="evidence", client=client, max_objects=0)
+
+
+def test_s3_rejects_private_or_insecure_endpoint_without_explicit_opt_in():
+    from tht.adapters.evidence.s3 import S3EvidenceSource
+    with pytest.raises(ValueError, match="private"):
+        S3EvidenceSource(bucket="evidence", endpoint_url="https://127.0.0.1:9000", client=Client())
+    with pytest.raises(ValueError, match="HTTPS"):
+        S3EvidenceSource(bucket="evidence", endpoint_url="http://s3.example.test", client=Client())
+
+
+def test_s3_size_limit_closes_body():
+    from tht.adapters.evidence.s3 import S3EvidenceSource
+    client = Client()
+    source = S3EvidenceSource(bucket="evidence", client=client, max_bytes=4)
+    item = next(iter(source.discover()))
+    with pytest.raises(EvidenceSourceError):
+        source.acquire(item)
+    assert client.body.closed
+
+
+def test_s3_config_serialization_masks_credentials():
+    from tht.config import S3EvidenceSourceConfig
+    config = S3EvidenceSourceConfig(type="s3", bucket="evidence",
+                                    access_key="access-secret", secret_key="write-secret")
+    assert "access-secret" not in repr(config)
+    assert "write-secret" not in repr(config)
+
+
+def test_s3_config_loads_credentials_from_secret_files(tmp_path):
+    from tht.config import load_config
+    access, secret = tmp_path / "access", tmp_path / "secret"
+    access.write_text("access-value")
+    secret.write_text("secret-value")
+    workspace = tmp_path / "workspace.yaml"
+    workspace.write_text(f"""
+dwh:
+  type: postgres_direct
+  connection: {{database: d, schema: public, user: u, password: p}}
+evidence:
+  sources:
+    - type: s3
+      bucket: evidence
+      access_key_file: {access}
+      secret_key_file: {secret}
+""")
+    source = load_config(workspace).evidence.sources[0]
+    assert source.access_key.get_secret_value() == "access-value"
+    assert source.secret_key.get_secret_value() == "secret-value"

harness/tht/adapters/evidence/__init__.py

@@ -2,5 +2,6 @@
 
 from tht.adapters.evidence.filesystem import FilesystemEvidenceSource
 from tht.adapters.evidence.http import HttpManifestEvidenceSource
+from tht.adapters.evidence.s3 import S3EvidenceSource
 
-__all__ = ["FilesystemEvidenceSource", "HttpManifestEvidenceSource"]
+__all__ = ["FilesystemEvidenceSource", "HttpManifestEvidenceSource", "S3EvidenceSource"]

harness/tht/adapters/evidence/s3.py

@@ -0,0 +1,125 @@
+"""Bounded S3-compatible Evidence source using the supported boto3 client."""
+
+import hashlib
+import ipaddress
+import socket
+from datetime import UTC, datetime
+from urllib.parse import quote, urlsplit
+
+from tht.ports.evidence import (
+    AcquiredDocument, EvidenceSourceError, EvidenceSourceErrorCategory, SourceObject,
+)
+
+
+class S3EvidenceSource:
+    def __init__(self, *, bucket: str, prefix: str = "", endpoint_url: str | None = None,
+                 region: str | None = None, access_key: str | None = None,
+                 secret_key: str | None = None, session_token: str | None = None,
+                 allow_private_endpoint: bool = False, allow_insecure_endpoint: bool = False,
+                 max_bytes: int = 10 * 1024 * 1024, max_objects: int = 10_000,
+                 max_pages: int = 100, page_size: int = 1000, client=None) -> None:
+        if not bucket or any(value < 1 for value in (max_bytes, max_objects, max_pages, page_size)):
+            raise ValueError("S3 evidence limits and bucket must be non-empty and positive")
+        if endpoint_url:
+            parsed = urlsplit(endpoint_url)
+            if parsed.username or parsed.password:
+                raise ValueError("S3 endpoint must not contain credentials")
+            if parsed.scheme != "https" and not allow_insecure_endpoint:
+                raise ValueError("S3 endpoint must use HTTPS unless explicitly allowed")
+            if not parsed.hostname:
+                raise ValueError("S3 endpoint must include a hostname")
+            if not allow_private_endpoint:
+                try:
+                    addresses = {ipaddress.ip_address(row[4][0].split("%", 1)[0]) for row in
+                                 socket.getaddrinfo(parsed.hostname, parsed.port or 443,
+                                                    type=socket.SOCK_STREAM)}
+                except (OSError, ValueError) as exc:
+                    raise ValueError("S3 endpoint resolution failed") from exc
+                if not addresses or any(not address.is_global for address in addresses):
+                    raise ValueError("S3 private endpoint requires explicit opt-in")
+        self.bucket, self.prefix = bucket, prefix.lstrip("/")
+        self.max_bytes, self.max_objects = max_bytes, max_objects
+        self.max_pages, self.page_size = max_pages, min(page_size, 1000)
+        if client is None:
+            try:
+                import boto3
+            except ImportError as exc:  # pragma: no cover - deployment optional dependency
+                raise RuntimeError("Install tht[s3] to use S3 Evidence") from exc
+            client = boto3.client("s3", endpoint_url=endpoint_url, region_name=region,
+                                  aws_access_key_id=access_key,
+                                  aws_secret_access_key=secret_key,
+                                  aws_session_token=session_token, verify=True)
+        self._client = client
+        self._items: dict[str, tuple[str, str | None, str | None]] = {}
+
+    @staticmethod
+    def _error(operation: str, transient: bool = False):
+        return EvidenceSourceError("S3 source operation failed",
+            category=(EvidenceSourceErrorCategory.TRANSIENT if transient
+                      else EvidenceSourceErrorCategory.PERMANENT),
+            details={"operation": operation})
+
+    def discover(self):
+        count = pages = 0
+        try:
+            paginator = self._client.get_paginator("list_objects_v2")
+            for page in paginator.paginate(Bucket=self.bucket, Prefix=self.prefix,
+                                           PaginationConfig={"PageSize": self.page_size}):
+                pages += 1
+                if pages > self.max_pages:
+                    raise self._error("list_limit")
+                for row in page.get("Contents", []):
+                    count += 1
+                    if count > self.max_objects:
+                        raise self._error("object_limit")
+                    key, version, etag = row["Key"], row.get("VersionId"), row.get("ETag")
+                    uri = f"s3://{self.bucket}/{quote(key, safe='/')}"
+                    stable = version or hashlib.sha256((etag or "").encode()).hexdigest()
+                    fingerprint = f"s3-version:{stable}" if version else f"etag:{stable}"
+                    source_id = "s3:" + hashlib.sha256(uri.encode()).hexdigest()
+                    self._items[source_id] = (key, version, etag)
+                    modified = row.get("LastModified")
+                    if modified is not None:
+                        modified = modified.astimezone(UTC)
+                    yield SourceObject(source_id=source_id, uri=uri, fingerprint=fingerprint,
+                                       modified_at=modified,
+                                       metadata={"size": int(row.get("Size", 0))})
+        except EvidenceSourceError:
+            raise
+        except Exception as exc:
+            raise self._error("list", transient=True) from exc
+
+    def acquire(self, item: SourceObject) -> AcquiredDocument:
+        binding = self._items.get(item.source_id)
+        if binding is None:
+            raise self._error("acquire")
+        key, version, _etag = binding
+        kwargs = {"Bucket": self.bucket, "Key": key}
+        if version:
+            kwargs["VersionId"] = version
+        body = None
+        try:
+            response = self._client.get_object(**kwargs)
+            body = response["Body"]
+            if version:
+                if response.get("VersionId") != version:
+                    raise self._error("version_changed")
+            else:
+                current = hashlib.sha256((response.get("ETag") or "").encode()).hexdigest()
+                if item.fingerprint != f"etag:{current}":
+                    raise self._error("etag_changed")
+            if int(response.get("ContentLength", 0)) > self.max_bytes:
+                raise self._error("download_limit")
+            content = body.read(self.max_bytes + 1)
+            if len(content) > self.max_bytes:
+                raise self._error("download_limit")
+            return AcquiredDocument(source=item, content=content,
+                                    media_type=response.get("ContentType"),
+                                    acquired_at=datetime.now(UTC))
+        except EvidenceSourceError:
+            raise
+        except Exception as exc:
+            raise self._error("download", transient=True) from exc
+        finally:
+            if body is not None:
+                body.close()

harness/tht/adapters/factory.py

@@ -2,6 +2,7 @@
 
 from tht.adapters.dwh import PostgresDwhAdapter, ThothRestDwhAdapter
 from tht.adapters.evidence import FilesystemEvidenceSource, HttpManifestEvidenceSource
+from tht.adapters.evidence.s3 import S3EvidenceSource
 from tht.adapters.vector import PgVectorStore, ThothHttpVectorStore
 from tht.config import Config, ConfigError
 from tht.db.connection import make_engine
@@ -117,6 +118,20 @@ def build_evidence_sources(cfg: Config):
                         max_cache_bytes=resource.max_cache_bytes,
                     )
                 )
+            case "s3":
+                def secret(value):
+                    return value.get_secret_value() if value is not None else None
+
+                sources.append(S3EvidenceSource(
+                    bucket=resource.bucket, prefix=resource.prefix,
+                    endpoint_url=resource.endpoint_url, region=resource.region,
+                    access_key=secret(resource.access_key), secret_key=secret(resource.secret_key),
+                    session_token=secret(resource.session_token),
+                    allow_private_endpoint=resource.allow_private_endpoint,
+                    allow_insecure_endpoint=resource.allow_insecure_endpoint,
+                    max_bytes=resource.max_bytes, max_objects=resource.max_objects,
+                    max_pages=resource.max_pages, page_size=resource.page_size,
+                ))
             case other:  # pragma: no cover - Pydantic rejects unsupported discriminators.
                 raise ConfigError(f"Adapter evidence non supportato: {other}")
     return sources

harness/tht/config.py

@@ -39,17 +39,20 @@ def _expand_env(value: Any) -> Any:
 def _resolve_secret_files(value: Any) -> Any:
     if isinstance(value, dict):
         resolved = {key: _resolve_secret_files(item) for key, item in value.items()}
-        if "password_file" in resolved:
-            if "password" in resolved:
-                raise ConfigError("password and password_file are mutually exclusive")
-            path = Path(resolved.pop("password_file"))
+        for secret_name in ("password", "access_key", "secret_key", "session_token"):
+            file_name = f"{secret_name}_file"
+            if file_name not in resolved:
+                continue
+            if secret_name in resolved:
+                raise ConfigError(f"{secret_name} and {file_name} are mutually exclusive")
+            path = Path(resolved.pop(file_name))
             try:
                 secret = path.read_text()
             except (OSError, UnicodeError) as exc:
                 raise ConfigError(f"Cannot read secret file: {path}") from exc
             if not secret or any(char.isspace() for char in secret) or "\x00" in secret:
                 raise ConfigError(f"Invalid secret file: {path}")
-            resolved["password"] = secret
+            resolved[secret_name] = secret
         return resolved
     if isinstance(value, list):
         return [_resolve_secret_files(item) for item in value]
@@ -192,8 +195,25 @@ class HttpEvidenceSourceConfig(BaseModel):
     max_cache_bytes: int = Field(default=64 * 1024 * 1024, gt=0)
 
 
+class S3EvidenceSourceConfig(BaseModel):
+    type: Literal["s3"]
+    bucket: str = Field(min_length=1)
+    prefix: str = ""
+    endpoint_url: str | None = None
+    region: str | None = None
+    access_key: SecretStr | None = None
+    secret_key: SecretStr | None = None
+    session_token: SecretStr | None = None
+    allow_private_endpoint: bool = False
+    allow_insecure_endpoint: bool = False
+    max_bytes: int = Field(default=10 * 1024 * 1024, gt=0)
+    max_objects: int = Field(default=10_000, gt=0)
+    max_pages: int = Field(default=100, gt=0)
+    page_size: int = Field(default=1000, gt=0, le=1000)
+
+
 EvidenceSourceConfig = Annotated[
-    FilesystemEvidenceSourceConfig | HttpEvidenceSourceConfig,
+    FilesystemEvidenceSourceConfig | HttpEvidenceSourceConfig | S3EvidenceSourceConfig,
     Field(discriminator="type"),
 ]
 

scripts/preprocess-smoke.sh

@@ -0,0 +1,15 @@
+#!/bin/sh
+set -eu
+cd "$(dirname "$0")/.."
+
+rendered=$(docker compose -f compose.yaml -f deploy/compose.preprocess.yaml --profile preprocess config)
+printf '%s' "$rendered" | grep -q 'preprocess-evidence:'
+printf '%s' "$rendered" | grep -q 'preprocess-dwh:'
+if printf '%s' "$rendered" | grep -qi 'access-secret\|secret-key'; then
+  echo "preprocess Compose rendered secret material" >&2
+  exit 1
+fi
+(cd harness && .venv/bin/pytest -q \
+  tests/test_corpus_pipeline.py::test_unchanged_documents_skip_acquire_normalize_chunk_and_embed \
+  tests/test_corpus_pipeline.py::test_retention_bounds_generations_and_purges_vectors_after_publish)
+echo "preprocess unchanged rerun and modified-generation smoke passed."