"""Evidence-owned bounded S3-compatible source using the supported boto3 client.""" import hashlib import ipaddress import re from datetime import UTC, datetime from urllib.parse import quote, urlsplit from tht.evidence.contracts import ( AcquiredDocument, EvidenceSourceError, EvidenceSourceErrorCategory, SourceObject, ) class S3EvidenceSource: def __init__(self, *, bucket: str, prefix: str = "", endpoint_url: str | None = None, region: str | None = None, access_key: str | None = None, secret_key: str | None = None, session_token: str | None = None, trusted_endpoint: bool = False, allow_private_endpoint: bool = False, allow_insecure_endpoint: bool = False, max_bytes: int = 10 * 1024 * 1024, max_objects: int = 10_000, max_pages: int = 100, page_size: int = 1000, client=None) -> None: bucket_valid = re.fullmatch(r"(?=.{3,63}$)(?!-)(?!.*\.\.)(?!.*\.-)(?!.*-\.)" r"[a-z0-9](?:[a-z0-9.-]*[a-z0-9])?", bucket) try: ipaddress.ip_address(bucket) bucket_is_ip = True except ValueError: bucket_is_ip = False if (not bucket_valid or bucket_is_ip or any(value < 1 for value in (max_bytes, max_objects, max_pages, page_size))): raise ValueError("S3 evidence limits and bucket must be non-empty and positive") if (prefix.startswith("/") or len(prefix.encode()) > 1024 or any(ord(char) < 32 or ord(char) == 127 for char in prefix)): raise ValueError("S3 prefix is invalid") if endpoint_url: parsed = urlsplit(endpoint_url) if parsed.username or parsed.password: raise ValueError("S3 endpoint must not contain credentials") if parsed.scheme not in {"http", "https"}: raise ValueError("S3 endpoint scheme must be exactly https or explicitly allowed http") if parsed.scheme == "http" and not allow_insecure_endpoint: raise ValueError("S3 endpoint must use HTTPS unless explicitly allowed") if not parsed.hostname: raise ValueError("S3 endpoint must include a hostname") if parsed.path not in {"", "/"} or parsed.query or parsed.fragment: raise ValueError("S3 custom endpoint must be an origin root without query/fragment") if not trusted_endpoint: raise ValueError("S3 custom endpoint requires explicit trusted_endpoint opt-in") try: literal = ipaddress.ip_address(parsed.hostname) except ValueError: literal = None if literal is not None and not literal.is_global and not allow_private_endpoint: raise ValueError("S3 private endpoint requires explicit opt-in") self.bucket, self.prefix = bucket, prefix self.max_bytes, self.max_objects = max_bytes, max_objects self.max_pages, self.page_size = max_pages, min(page_size, 1000) if client is None: try: import boto3 from botocore.config import Config as BotoConfig except ImportError as exc: # pragma: no cover - deployment optional dependency raise RuntimeError("Install tht[s3] to use S3 Evidence") from exc client = boto3.client("s3", endpoint_url=endpoint_url, region_name=region, aws_access_key_id=access_key, aws_secret_access_key=secret_key, aws_session_token=session_token, verify=True, config=BotoConfig(s3={"addressing_style": "path"})) self._client = client self._items: dict[str, tuple[SourceObject, str]] = {} @staticmethod def _error(operation: str, transient: bool = False): return EvidenceSourceError("S3 source operation failed", category=(EvidenceSourceErrorCategory.TRANSIENT if transient else EvidenceSourceErrorCategory.PERMANENT), details={"operation": operation}) def discover(self): count = pages = 0 try: token = None for _ in range(self.max_pages): params = {"Bucket": self.bucket, "Prefix": self.prefix, "MaxKeys": self.page_size} if token is not None: params["ContinuationToken"] = token page = self._client.list_objects_v2(**params) pages += 1 for row in page.get("Contents", []): count += 1 if count > self.max_objects: raise self._error("object_limit") key, etag = row.get("Key"), row.get("ETag") if (not isinstance(key, str) or not key or not key.startswith(self.prefix) or len(key.encode()) > 1024 or any(ord(char) < 32 or ord(char) == 127 for char in key)): raise self._error("invalid_key") if not isinstance(etag, str) or not etag or len(etag) > 1024: raise self._error("missing_validator") uri = f"s3://{self.bucket}/{quote(key, safe='/')}" fingerprint = f"etag:{hashlib.sha256(etag.encode()).hexdigest()}" source_id = "s3:" + hashlib.sha256(uri.encode()).hexdigest() modified = row.get("LastModified") if modified is not None: modified = modified.astimezone(UTC) item = SourceObject(source_id=source_id, uri=uri, fingerprint=fingerprint, modified_at=modified, metadata={"size": int(row.get("Size", 0))}) self._items[source_id] = (item, etag) yield item if not page.get("IsTruncated"): return token = page.get("NextContinuationToken") if not isinstance(token, str) or not token: raise self._error("list_continuation") raise self._error("list_limit") except EvidenceSourceError: raise except Exception as exc: raise self._error("list", transient=True) from exc def acquire(self, item: SourceObject) -> AcquiredDocument: binding = self._items.get(item.source_id) if binding is None or item != binding[0]: raise self._error("acquire") discovered, etag = binding key = discovered.uri.split(f"s3://{self.bucket}/", 1)[1] from urllib.parse import unquote key = unquote(key) kwargs = {"Bucket": self.bucket, "Key": key} body = None try: response = self._client.get_object(**kwargs) body = response["Body"] if response.get("ETag") != etag: raise self._error("etag_changed") if int(response.get("ContentLength", 0)) > self.max_bytes: raise self._error("download_limit") content = body.read(self.max_bytes + 1) if len(content) > self.max_bytes: raise self._error("download_limit") return AcquiredDocument(source=item, content=content, media_type=response.get("ContentType"), acquired_at=datetime.now(UTC)) except EvidenceSourceError: raise except Exception as exc: raise self._error("download", transient=True) from exc finally: if body is not None: body.close()