refactor(evidence): remove legacy Python layout (#31)
This commit is contained in:
@@ -0,0 +1,152 @@
|
||||
"""Evidence-owned bounded S3-compatible source using the supported boto3 client."""
|
||||
|
||||
import hashlib
|
||||
import ipaddress
|
||||
import re
|
||||
from datetime import UTC, datetime
|
||||
from urllib.parse import quote, urlsplit
|
||||
|
||||
from tht.evidence.contracts import (
|
||||
AcquiredDocument, EvidenceSourceError, EvidenceSourceErrorCategory, SourceObject,
|
||||
)
|
||||
|
||||
|
||||
class S3EvidenceSource:
|
||||
def __init__(self, *, bucket: str, prefix: str = "", endpoint_url: str | None = None,
|
||||
region: str | None = None, access_key: str | None = None,
|
||||
secret_key: str | None = None, session_token: str | None = None,
|
||||
trusted_endpoint: bool = False,
|
||||
allow_private_endpoint: bool = False, allow_insecure_endpoint: bool = False,
|
||||
max_bytes: int = 10 * 1024 * 1024, max_objects: int = 10_000,
|
||||
max_pages: int = 100, page_size: int = 1000, client=None) -> None:
|
||||
bucket_valid = re.fullmatch(r"(?=.{3,63}$)(?!-)(?!.*\.\.)(?!.*\.-)(?!.*-\.)"
|
||||
r"[a-z0-9](?:[a-z0-9.-]*[a-z0-9])?", bucket)
|
||||
try:
|
||||
ipaddress.ip_address(bucket)
|
||||
bucket_is_ip = True
|
||||
except ValueError:
|
||||
bucket_is_ip = False
|
||||
if (not bucket_valid or bucket_is_ip
|
||||
or any(value < 1 for value in (max_bytes, max_objects, max_pages, page_size))):
|
||||
raise ValueError("S3 evidence limits and bucket must be non-empty and positive")
|
||||
if (prefix.startswith("/") or len(prefix.encode()) > 1024
|
||||
or any(ord(char) < 32 or ord(char) == 127 for char in prefix)):
|
||||
raise ValueError("S3 prefix is invalid")
|
||||
if endpoint_url:
|
||||
parsed = urlsplit(endpoint_url)
|
||||
if parsed.username or parsed.password:
|
||||
raise ValueError("S3 endpoint must not contain credentials")
|
||||
if parsed.scheme not in {"http", "https"}:
|
||||
raise ValueError("S3 endpoint scheme must be exactly https or explicitly allowed http")
|
||||
if parsed.scheme == "http" and not allow_insecure_endpoint:
|
||||
raise ValueError("S3 endpoint must use HTTPS unless explicitly allowed")
|
||||
if not parsed.hostname:
|
||||
raise ValueError("S3 endpoint must include a hostname")
|
||||
if parsed.path not in {"", "/"} or parsed.query or parsed.fragment:
|
||||
raise ValueError("S3 custom endpoint must be an origin root without query/fragment")
|
||||
if not trusted_endpoint:
|
||||
raise ValueError("S3 custom endpoint requires explicit trusted_endpoint opt-in")
|
||||
try:
|
||||
literal = ipaddress.ip_address(parsed.hostname)
|
||||
except ValueError:
|
||||
literal = None
|
||||
if literal is not None and not literal.is_global and not allow_private_endpoint:
|
||||
raise ValueError("S3 private endpoint requires explicit opt-in")
|
||||
self.bucket, self.prefix = bucket, prefix
|
||||
self.max_bytes, self.max_objects = max_bytes, max_objects
|
||||
self.max_pages, self.page_size = max_pages, min(page_size, 1000)
|
||||
if client is None:
|
||||
try:
|
||||
import boto3
|
||||
from botocore.config import Config as BotoConfig
|
||||
except ImportError as exc: # pragma: no cover - deployment optional dependency
|
||||
raise RuntimeError("Install tht[s3] to use S3 Evidence") from exc
|
||||
client = boto3.client("s3", endpoint_url=endpoint_url, region_name=region,
|
||||
aws_access_key_id=access_key,
|
||||
aws_secret_access_key=secret_key,
|
||||
aws_session_token=session_token, verify=True,
|
||||
config=BotoConfig(s3={"addressing_style": "path"}))
|
||||
self._client = client
|
||||
self._items: dict[str, tuple[SourceObject, str]] = {}
|
||||
|
||||
@staticmethod
|
||||
def _error(operation: str, transient: bool = False):
|
||||
return EvidenceSourceError("S3 source operation failed",
|
||||
category=(EvidenceSourceErrorCategory.TRANSIENT if transient
|
||||
else EvidenceSourceErrorCategory.PERMANENT),
|
||||
details={"operation": operation})
|
||||
|
||||
def discover(self):
|
||||
count = pages = 0
|
||||
try:
|
||||
token = None
|
||||
for _ in range(self.max_pages):
|
||||
params = {"Bucket": self.bucket, "Prefix": self.prefix,
|
||||
"MaxKeys": self.page_size}
|
||||
if token is not None:
|
||||
params["ContinuationToken"] = token
|
||||
page = self._client.list_objects_v2(**params)
|
||||
pages += 1
|
||||
for row in page.get("Contents", []):
|
||||
count += 1
|
||||
if count > self.max_objects:
|
||||
raise self._error("object_limit")
|
||||
key, etag = row.get("Key"), row.get("ETag")
|
||||
if (not isinstance(key, str) or not key or not key.startswith(self.prefix)
|
||||
or len(key.encode()) > 1024
|
||||
or any(ord(char) < 32 or ord(char) == 127 for char in key)):
|
||||
raise self._error("invalid_key")
|
||||
if not isinstance(etag, str) or not etag or len(etag) > 1024:
|
||||
raise self._error("missing_validator")
|
||||
uri = f"s3://{self.bucket}/{quote(key, safe='/')}"
|
||||
fingerprint = f"etag:{hashlib.sha256(etag.encode()).hexdigest()}"
|
||||
source_id = "s3:" + hashlib.sha256(uri.encode()).hexdigest()
|
||||
modified = row.get("LastModified")
|
||||
if modified is not None:
|
||||
modified = modified.astimezone(UTC)
|
||||
item = SourceObject(source_id=source_id, uri=uri, fingerprint=fingerprint,
|
||||
modified_at=modified,
|
||||
metadata={"size": int(row.get("Size", 0))})
|
||||
self._items[source_id] = (item, etag)
|
||||
yield item
|
||||
if not page.get("IsTruncated"):
|
||||
return
|
||||
token = page.get("NextContinuationToken")
|
||||
if not isinstance(token, str) or not token:
|
||||
raise self._error("list_continuation")
|
||||
raise self._error("list_limit")
|
||||
except EvidenceSourceError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
raise self._error("list", transient=True) from exc
|
||||
|
||||
def acquire(self, item: SourceObject) -> AcquiredDocument:
|
||||
binding = self._items.get(item.source_id)
|
||||
if binding is None or item != binding[0]:
|
||||
raise self._error("acquire")
|
||||
discovered, etag = binding
|
||||
key = discovered.uri.split(f"s3://{self.bucket}/", 1)[1]
|
||||
from urllib.parse import unquote
|
||||
key = unquote(key)
|
||||
kwargs = {"Bucket": self.bucket, "Key": key}
|
||||
body = None
|
||||
try:
|
||||
response = self._client.get_object(**kwargs)
|
||||
body = response["Body"]
|
||||
if response.get("ETag") != etag:
|
||||
raise self._error("etag_changed")
|
||||
if int(response.get("ContentLength", 0)) > self.max_bytes:
|
||||
raise self._error("download_limit")
|
||||
content = body.read(self.max_bytes + 1)
|
||||
if len(content) > self.max_bytes:
|
||||
raise self._error("download_limit")
|
||||
return AcquiredDocument(source=item, content=content,
|
||||
media_type=response.get("ContentType"),
|
||||
acquired_at=datetime.now(UTC))
|
||||
except EvidenceSourceError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
raise self._error("download", transient=True) from exc
|
||||
finally:
|
||||
if body is not None:
|
||||
body.close()
|
||||
Reference in New Issue
Block a user