feat(preprocess): add deployment jobs and S3 source

This commit is contained in:
2026-07-12 05:42:07 +02:00
parent 6656a69630
commit 4028ef7821
12 changed files with 380 additions and 10 deletions
+15
View File
@@ -2,6 +2,7 @@
from tht.adapters.dwh import PostgresDwhAdapter, ThothRestDwhAdapter
from tht.adapters.evidence import FilesystemEvidenceSource, HttpManifestEvidenceSource
from tht.adapters.evidence.s3 import S3EvidenceSource
from tht.adapters.vector import PgVectorStore, ThothHttpVectorStore
from tht.config import Config, ConfigError
from tht.db.connection import make_engine
@@ -117,6 +118,20 @@ def build_evidence_sources(cfg: Config):
max_cache_bytes=resource.max_cache_bytes,
)
)
case "s3":
def secret(value):
return value.get_secret_value() if value is not None else None
sources.append(S3EvidenceSource(
bucket=resource.bucket, prefix=resource.prefix,
endpoint_url=resource.endpoint_url, region=resource.region,
access_key=secret(resource.access_key), secret_key=secret(resource.secret_key),
session_token=secret(resource.session_token),
allow_private_endpoint=resource.allow_private_endpoint,
allow_insecure_endpoint=resource.allow_insecure_endpoint,
max_bytes=resource.max_bytes, max_objects=resource.max_objects,
max_pages=resource.max_pages, page_size=resource.page_size,
))
case other: # pragma: no cover - Pydantic rejects unsupported discriminators.
raise ConfigError(f"Adapter evidence non supportato: {other}")
return sources