feat(preprocess): add deployment jobs and S3 source

harness/pyproject.toml

@@ -22,6 +22,7 @@ dependencies = [
 tht = "tht.cli:app"
 
 [project.optional-dependencies]
+s3 = ["boto3>=1.34,<2"]
 dev = [
     "pytest>=8.0",
     "testcontainers[postgres]>=4.0",

harness/tests/test_s3_evidence_source.py

@@ -0,0 +1,89 @@
+from datetime import UTC, datetime
+
+import pytest
+
+from tht.ports.evidence import EvidenceSourceError
+
+
+class Body:
+    def __init__(self, data): self.data, self.closed = data, False
+    def read(self, amount): return self.data[:amount]
+    def close(self): self.closed = True
+
+
+class Client:
+    def __init__(self): self.body = Body(b"hello")
+    def get_paginator(self, name): return self
+    def paginate(self, **kwargs):
+        yield {"Contents": [{"Key": "clinical/a.md", "ETag": '"abc"', "VersionId": "v1",
+                              "Size": 5, "LastModified": datetime(2026, 1, 1, tzinfo=UTC)}]}
+    def get_object(self, **kwargs):
+        assert kwargs == {"Bucket": "evidence", "Key": "clinical/a.md", "VersionId": "v1"}
+        return {"Body": self.body, "ContentLength": 5, "ContentType": "text/markdown",
+                "ETag": '"abc"', "VersionId": "v1"}
+
+
+def test_s3_canonical_uri_version_fingerprint_and_closed_body():
+    from tht.adapters.evidence.s3 import S3EvidenceSource
+    client = Client()
+    source = S3EvidenceSource(bucket="evidence", prefix="clinical/", client=client)
+    item = next(iter(source.discover()))
+    assert item.uri == "s3://evidence/clinical/a.md"
+    assert item.fingerprint == "s3-version:v1"
+    assert source.acquire(item).content == b"hello"
+    assert client.body.closed
+
+
+def test_s3_etag_fallback_and_bounds():
+    from tht.adapters.evidence.s3 import S3EvidenceSource
+    client = Client()
+    with pytest.raises(ValueError):
+        S3EvidenceSource(bucket="evidence", client=client, max_objects=0)
+
+
+def test_s3_rejects_private_or_insecure_endpoint_without_explicit_opt_in():
+    from tht.adapters.evidence.s3 import S3EvidenceSource
+    with pytest.raises(ValueError, match="private"):
+        S3EvidenceSource(bucket="evidence", endpoint_url="https://127.0.0.1:9000", client=Client())
+    with pytest.raises(ValueError, match="HTTPS"):
+        S3EvidenceSource(bucket="evidence", endpoint_url="http://s3.example.test", client=Client())
+
+
+def test_s3_size_limit_closes_body():
+    from tht.adapters.evidence.s3 import S3EvidenceSource
+    client = Client()
+    source = S3EvidenceSource(bucket="evidence", client=client, max_bytes=4)
+    item = next(iter(source.discover()))
+    with pytest.raises(EvidenceSourceError):
+        source.acquire(item)
+    assert client.body.closed
+
+
+def test_s3_config_serialization_masks_credentials():
+    from tht.config import S3EvidenceSourceConfig
+    config = S3EvidenceSourceConfig(type="s3", bucket="evidence",
+                                    access_key="access-secret", secret_key="write-secret")
+    assert "access-secret" not in repr(config)
+    assert "write-secret" not in repr(config)
+
+
+def test_s3_config_loads_credentials_from_secret_files(tmp_path):
+    from tht.config import load_config
+    access, secret = tmp_path / "access", tmp_path / "secret"
+    access.write_text("access-value")
+    secret.write_text("secret-value")
+    workspace = tmp_path / "workspace.yaml"
+    workspace.write_text(f"""
+dwh:
+  type: postgres_direct
+  connection: {{database: d, schema: public, user: u, password: p}}
+evidence:
+  sources:
+    - type: s3
+      bucket: evidence
+      access_key_file: {access}
+      secret_key_file: {secret}
+""")
+    source = load_config(workspace).evidence.sources[0]
+    assert source.access_key.get_secret_value() == "access-value"
+    assert source.secret_key.get_secret_value() == "secret-value"

harness/tht/adapters/evidence/__init__.py

@@ -2,5 +2,6 @@
 
 from tht.adapters.evidence.filesystem import FilesystemEvidenceSource
 from tht.adapters.evidence.http import HttpManifestEvidenceSource
+from tht.adapters.evidence.s3 import S3EvidenceSource
 
-__all__ = ["FilesystemEvidenceSource", "HttpManifestEvidenceSource"]
+__all__ = ["FilesystemEvidenceSource", "HttpManifestEvidenceSource", "S3EvidenceSource"]

harness/tht/adapters/evidence/s3.py

@@ -0,0 +1,125 @@
+"""Bounded S3-compatible Evidence source using the supported boto3 client."""
+
+import hashlib
+import ipaddress
+import socket
+from datetime import UTC, datetime
+from urllib.parse import quote, urlsplit
+
+from tht.ports.evidence import (
+    AcquiredDocument, EvidenceSourceError, EvidenceSourceErrorCategory, SourceObject,
+)
+
+
+class S3EvidenceSource:
+    def __init__(self, *, bucket: str, prefix: str = "", endpoint_url: str | None = None,
+                 region: str | None = None, access_key: str | None = None,
+                 secret_key: str | None = None, session_token: str | None = None,
+                 allow_private_endpoint: bool = False, allow_insecure_endpoint: bool = False,
+                 max_bytes: int = 10 * 1024 * 1024, max_objects: int = 10_000,
+                 max_pages: int = 100, page_size: int = 1000, client=None) -> None:
+        if not bucket or any(value < 1 for value in (max_bytes, max_objects, max_pages, page_size)):
+            raise ValueError("S3 evidence limits and bucket must be non-empty and positive")
+        if endpoint_url:
+            parsed = urlsplit(endpoint_url)
+            if parsed.username or parsed.password:
+                raise ValueError("S3 endpoint must not contain credentials")
+            if parsed.scheme != "https" and not allow_insecure_endpoint:
+                raise ValueError("S3 endpoint must use HTTPS unless explicitly allowed")
+            if not parsed.hostname:
+                raise ValueError("S3 endpoint must include a hostname")
+            if not allow_private_endpoint:
+                try:
+                    addresses = {ipaddress.ip_address(row[4][0].split("%", 1)[0]) for row in
+                                 socket.getaddrinfo(parsed.hostname, parsed.port or 443,
+                                                    type=socket.SOCK_STREAM)}
+                except (OSError, ValueError) as exc:
+                    raise ValueError("S3 endpoint resolution failed") from exc
+                if not addresses or any(not address.is_global for address in addresses):
+                    raise ValueError("S3 private endpoint requires explicit opt-in")
+        self.bucket, self.prefix = bucket, prefix.lstrip("/")
+        self.max_bytes, self.max_objects = max_bytes, max_objects
+        self.max_pages, self.page_size = max_pages, min(page_size, 1000)
+        if client is None:
+            try:
+                import boto3
+            except ImportError as exc:  # pragma: no cover - deployment optional dependency
+                raise RuntimeError("Install tht[s3] to use S3 Evidence") from exc
+            client = boto3.client("s3", endpoint_url=endpoint_url, region_name=region,
+                                  aws_access_key_id=access_key,
+                                  aws_secret_access_key=secret_key,
+                                  aws_session_token=session_token, verify=True)
+        self._client = client
+        self._items: dict[str, tuple[str, str | None, str | None]] = {}
+
+    @staticmethod
+    def _error(operation: str, transient: bool = False):
+        return EvidenceSourceError("S3 source operation failed",
+            category=(EvidenceSourceErrorCategory.TRANSIENT if transient
+                      else EvidenceSourceErrorCategory.PERMANENT),
+            details={"operation": operation})
+
+    def discover(self):
+        count = pages = 0
+        try:
+            paginator = self._client.get_paginator("list_objects_v2")
+            for page in paginator.paginate(Bucket=self.bucket, Prefix=self.prefix,
+                                           PaginationConfig={"PageSize": self.page_size}):
+                pages += 1
+                if pages > self.max_pages:
+                    raise self._error("list_limit")
+                for row in page.get("Contents", []):
+                    count += 1
+                    if count > self.max_objects:
+                        raise self._error("object_limit")
+                    key, version, etag = row["Key"], row.get("VersionId"), row.get("ETag")
+                    uri = f"s3://{self.bucket}/{quote(key, safe='/')}"
+                    stable = version or hashlib.sha256((etag or "").encode()).hexdigest()
+                    fingerprint = f"s3-version:{stable}" if version else f"etag:{stable}"
+                    source_id = "s3:" + hashlib.sha256(uri.encode()).hexdigest()
+                    self._items[source_id] = (key, version, etag)
+                    modified = row.get("LastModified")
+                    if modified is not None:
+                        modified = modified.astimezone(UTC)
+                    yield SourceObject(source_id=source_id, uri=uri, fingerprint=fingerprint,
+                                       modified_at=modified,
+                                       metadata={"size": int(row.get("Size", 0))})
+        except EvidenceSourceError:
+            raise
+        except Exception as exc:
+            raise self._error("list", transient=True) from exc
+
+    def acquire(self, item: SourceObject) -> AcquiredDocument:
+        binding = self._items.get(item.source_id)
+        if binding is None:
+            raise self._error("acquire")
+        key, version, _etag = binding
+        kwargs = {"Bucket": self.bucket, "Key": key}
+        if version:
+            kwargs["VersionId"] = version
+        body = None
+        try:
+            response = self._client.get_object(**kwargs)
+            body = response["Body"]
+            if version:
+                if response.get("VersionId") != version:
+                    raise self._error("version_changed")
+            else:
+                current = hashlib.sha256((response.get("ETag") or "").encode()).hexdigest()
+                if item.fingerprint != f"etag:{current}":
+                    raise self._error("etag_changed")
+            if int(response.get("ContentLength", 0)) > self.max_bytes:
+                raise self._error("download_limit")
+            content = body.read(self.max_bytes + 1)
+            if len(content) > self.max_bytes:
+                raise self._error("download_limit")
+            return AcquiredDocument(source=item, content=content,
+                                    media_type=response.get("ContentType"),
+                                    acquired_at=datetime.now(UTC))
+        except EvidenceSourceError:
+            raise
+        except Exception as exc:
+            raise self._error("download", transient=True) from exc
+        finally:
+            if body is not None:
+                body.close()

harness/tht/adapters/factory.py

@@ -2,6 +2,7 @@
 
 from tht.adapters.dwh import PostgresDwhAdapter, ThothRestDwhAdapter
 from tht.adapters.evidence import FilesystemEvidenceSource, HttpManifestEvidenceSource
+from tht.adapters.evidence.s3 import S3EvidenceSource
 from tht.adapters.vector import PgVectorStore, ThothHttpVectorStore
 from tht.config import Config, ConfigError
 from tht.db.connection import make_engine
@@ -117,6 +118,20 @@ def build_evidence_sources(cfg: Config):
                         max_cache_bytes=resource.max_cache_bytes,
                     )
                 )
+            case "s3":
+                def secret(value):
+                    return value.get_secret_value() if value is not None else None
+
+                sources.append(S3EvidenceSource(
+                    bucket=resource.bucket, prefix=resource.prefix,
+                    endpoint_url=resource.endpoint_url, region=resource.region,
+                    access_key=secret(resource.access_key), secret_key=secret(resource.secret_key),
+                    session_token=secret(resource.session_token),
+                    allow_private_endpoint=resource.allow_private_endpoint,
+                    allow_insecure_endpoint=resource.allow_insecure_endpoint,
+                    max_bytes=resource.max_bytes, max_objects=resource.max_objects,
+                    max_pages=resource.max_pages, page_size=resource.page_size,
+                ))
             case other:  # pragma: no cover - Pydantic rejects unsupported discriminators.
                 raise ConfigError(f"Adapter evidence non supportato: {other}")
     return sources

harness/tht/config.py

@@ -39,17 +39,20 @@ def _expand_env(value: Any) -> Any:
 def _resolve_secret_files(value: Any) -> Any:
     if isinstance(value, dict):
         resolved = {key: _resolve_secret_files(item) for key, item in value.items()}
-        if "password_file" in resolved:
-            if "password" in resolved:
-                raise ConfigError("password and password_file are mutually exclusive")
-            path = Path(resolved.pop("password_file"))
+        for secret_name in ("password", "access_key", "secret_key", "session_token"):
+            file_name = f"{secret_name}_file"
+            if file_name not in resolved:
+                continue
+            if secret_name in resolved:
+                raise ConfigError(f"{secret_name} and {file_name} are mutually exclusive")
+            path = Path(resolved.pop(file_name))
             try:
                 secret = path.read_text()
             except (OSError, UnicodeError) as exc:
                 raise ConfigError(f"Cannot read secret file: {path}") from exc
             if not secret or any(char.isspace() for char in secret) or "\x00" in secret:
                 raise ConfigError(f"Invalid secret file: {path}")
-            resolved["password"] = secret
+            resolved[secret_name] = secret
         return resolved
     if isinstance(value, list):
         return [_resolve_secret_files(item) for item in value]
@@ -192,8 +195,25 @@ class HttpEvidenceSourceConfig(BaseModel):
     max_cache_bytes: int = Field(default=64 * 1024 * 1024, gt=0)
 
 
+class S3EvidenceSourceConfig(BaseModel):
+    type: Literal["s3"]
+    bucket: str = Field(min_length=1)
+    prefix: str = ""
+    endpoint_url: str | None = None
+    region: str | None = None
+    access_key: SecretStr | None = None
+    secret_key: SecretStr | None = None
+    session_token: SecretStr | None = None
+    allow_private_endpoint: bool = False
+    allow_insecure_endpoint: bool = False
+    max_bytes: int = Field(default=10 * 1024 * 1024, gt=0)
+    max_objects: int = Field(default=10_000, gt=0)
+    max_pages: int = Field(default=100, gt=0)
+    page_size: int = Field(default=1000, gt=0, le=1000)
+
+
 EvidenceSourceConfig = Annotated[
-    FilesystemEvidenceSourceConfig | HttpEvidenceSourceConfig,
+    FilesystemEvidenceSourceConfig | HttpEvidenceSourceConfig | S3EvidenceSourceConfig,
     Field(discriminator="type"),
 ]