153 lines
7.7 KiB
Python
153 lines
7.7 KiB
Python
"""Evidence-owned bounded S3-compatible source using the supported boto3 client."""
|
|
|
|
import hashlib
|
|
import ipaddress
|
|
import re
|
|
from datetime import UTC, datetime
|
|
from urllib.parse import quote, urlsplit
|
|
|
|
from tht.evidence.contracts import (
|
|
AcquiredDocument, EvidenceSourceError, EvidenceSourceErrorCategory, SourceObject,
|
|
)
|
|
|
|
|
|
class S3EvidenceSource:
|
|
def __init__(self, *, bucket: str, prefix: str = "", endpoint_url: str | None = None,
|
|
region: str | None = None, access_key: str | None = None,
|
|
secret_key: str | None = None, session_token: str | None = None,
|
|
trusted_endpoint: bool = False,
|
|
allow_private_endpoint: bool = False, allow_insecure_endpoint: bool = False,
|
|
max_bytes: int = 10 * 1024 * 1024, max_objects: int = 10_000,
|
|
max_pages: int = 100, page_size: int = 1000, client=None) -> None:
|
|
bucket_valid = re.fullmatch(r"(?=.{3,63}$)(?!-)(?!.*\.\.)(?!.*\.-)(?!.*-\.)"
|
|
r"[a-z0-9](?:[a-z0-9.-]*[a-z0-9])?", bucket)
|
|
try:
|
|
ipaddress.ip_address(bucket)
|
|
bucket_is_ip = True
|
|
except ValueError:
|
|
bucket_is_ip = False
|
|
if (not bucket_valid or bucket_is_ip
|
|
or any(value < 1 for value in (max_bytes, max_objects, max_pages, page_size))):
|
|
raise ValueError("S3 evidence limits and bucket must be non-empty and positive")
|
|
if (prefix.startswith("/") or len(prefix.encode()) > 1024
|
|
or any(ord(char) < 32 or ord(char) == 127 for char in prefix)):
|
|
raise ValueError("S3 prefix is invalid")
|
|
if endpoint_url:
|
|
parsed = urlsplit(endpoint_url)
|
|
if parsed.username or parsed.password:
|
|
raise ValueError("S3 endpoint must not contain credentials")
|
|
if parsed.scheme not in {"http", "https"}:
|
|
raise ValueError("S3 endpoint scheme must be exactly https or explicitly allowed http")
|
|
if parsed.scheme == "http" and not allow_insecure_endpoint:
|
|
raise ValueError("S3 endpoint must use HTTPS unless explicitly allowed")
|
|
if not parsed.hostname:
|
|
raise ValueError("S3 endpoint must include a hostname")
|
|
if parsed.path not in {"", "/"} or parsed.query or parsed.fragment:
|
|
raise ValueError("S3 custom endpoint must be an origin root without query/fragment")
|
|
if not trusted_endpoint:
|
|
raise ValueError("S3 custom endpoint requires explicit trusted_endpoint opt-in")
|
|
try:
|
|
literal = ipaddress.ip_address(parsed.hostname)
|
|
except ValueError:
|
|
literal = None
|
|
if literal is not None and not literal.is_global and not allow_private_endpoint:
|
|
raise ValueError("S3 private endpoint requires explicit opt-in")
|
|
self.bucket, self.prefix = bucket, prefix
|
|
self.max_bytes, self.max_objects = max_bytes, max_objects
|
|
self.max_pages, self.page_size = max_pages, min(page_size, 1000)
|
|
if client is None:
|
|
try:
|
|
import boto3
|
|
from botocore.config import Config as BotoConfig
|
|
except ImportError as exc: # pragma: no cover - deployment optional dependency
|
|
raise RuntimeError("Install tht[s3] to use S3 Evidence") from exc
|
|
client = boto3.client("s3", endpoint_url=endpoint_url, region_name=region,
|
|
aws_access_key_id=access_key,
|
|
aws_secret_access_key=secret_key,
|
|
aws_session_token=session_token, verify=True,
|
|
config=BotoConfig(s3={"addressing_style": "path"}))
|
|
self._client = client
|
|
self._items: dict[str, tuple[SourceObject, str]] = {}
|
|
|
|
@staticmethod
|
|
def _error(operation: str, transient: bool = False):
|
|
return EvidenceSourceError("S3 source operation failed",
|
|
category=(EvidenceSourceErrorCategory.TRANSIENT if transient
|
|
else EvidenceSourceErrorCategory.PERMANENT),
|
|
details={"operation": operation})
|
|
|
|
def discover(self):
|
|
count = pages = 0
|
|
try:
|
|
token = None
|
|
for _ in range(self.max_pages):
|
|
params = {"Bucket": self.bucket, "Prefix": self.prefix,
|
|
"MaxKeys": self.page_size}
|
|
if token is not None:
|
|
params["ContinuationToken"] = token
|
|
page = self._client.list_objects_v2(**params)
|
|
pages += 1
|
|
for row in page.get("Contents", []):
|
|
count += 1
|
|
if count > self.max_objects:
|
|
raise self._error("object_limit")
|
|
key, etag = row.get("Key"), row.get("ETag")
|
|
if (not isinstance(key, str) or not key or not key.startswith(self.prefix)
|
|
or len(key.encode()) > 1024
|
|
or any(ord(char) < 32 or ord(char) == 127 for char in key)):
|
|
raise self._error("invalid_key")
|
|
if not isinstance(etag, str) or not etag or len(etag) > 1024:
|
|
raise self._error("missing_validator")
|
|
uri = f"s3://{self.bucket}/{quote(key, safe='/')}"
|
|
fingerprint = f"etag:{hashlib.sha256(etag.encode()).hexdigest()}"
|
|
source_id = "s3:" + hashlib.sha256(uri.encode()).hexdigest()
|
|
modified = row.get("LastModified")
|
|
if modified is not None:
|
|
modified = modified.astimezone(UTC)
|
|
item = SourceObject(source_id=source_id, uri=uri, fingerprint=fingerprint,
|
|
modified_at=modified,
|
|
metadata={"size": int(row.get("Size", 0))})
|
|
self._items[source_id] = (item, etag)
|
|
yield item
|
|
if not page.get("IsTruncated"):
|
|
return
|
|
token = page.get("NextContinuationToken")
|
|
if not isinstance(token, str) or not token:
|
|
raise self._error("list_continuation")
|
|
raise self._error("list_limit")
|
|
except EvidenceSourceError:
|
|
raise
|
|
except Exception as exc:
|
|
raise self._error("list", transient=True) from exc
|
|
|
|
def acquire(self, item: SourceObject) -> AcquiredDocument:
|
|
binding = self._items.get(item.source_id)
|
|
if binding is None or item != binding[0]:
|
|
raise self._error("acquire")
|
|
discovered, etag = binding
|
|
key = discovered.uri.split(f"s3://{self.bucket}/", 1)[1]
|
|
from urllib.parse import unquote
|
|
key = unquote(key)
|
|
kwargs = {"Bucket": self.bucket, "Key": key}
|
|
body = None
|
|
try:
|
|
response = self._client.get_object(**kwargs)
|
|
body = response["Body"]
|
|
if response.get("ETag") != etag:
|
|
raise self._error("etag_changed")
|
|
if int(response.get("ContentLength", 0)) > self.max_bytes:
|
|
raise self._error("download_limit")
|
|
content = body.read(self.max_bytes + 1)
|
|
if len(content) > self.max_bytes:
|
|
raise self._error("download_limit")
|
|
return AcquiredDocument(source=item, content=content,
|
|
media_type=response.get("ContentType"),
|
|
acquired_at=datetime.now(UTC))
|
|
except EvidenceSourceError:
|
|
raise
|
|
except Exception as exc:
|
|
raise self._error("download", transient=True) from exc
|
|
finally:
|
|
if body is not None:
|
|
body.close()
|