mptyl/ThothII
1
0
Fork 0
Code Issues 25 Pull Requests Actions Packages Projects Releases 1 Wiki Activity

fix(evidence): close S3 and smoke safety gaps

Browse Source
This commit is contained in:
marcopan
2026-07-12 06:09:15 +02:00
parent efcb0deb31
commit e6d44ba082
5 changed files with 148 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files
harness/tht/adapters/evidence/s3.py
+45 -26
View File
@@ -1,6 +1,7 @@
"""Bounded S3-compatible Evidence source using the supported boto3 client."""
import hashlib
import ipaddress
import re
from datetime import UTC, datetime
from urllib.parse import quote, urlsplit
@@ -18,10 +19,18 @@ class S3EvidenceSource:
allow_private_endpoint: bool = False, allow_insecure_endpoint: bool = False,
max_bytes: int = 10 * 1024 * 1024, max_objects: int = 10_000,
max_pages: int = 100, page_size: int = 1000, client=None) -> None:
if (not re.fullmatch(r"(?=.{3,63}$)(?!-)(?!.*\.\.)(?!.*\.-)(?!.*-\.)"
bucket_valid = re.fullmatch(r"(?=.{3,63}$)(?!-)(?!.*\.\.)(?!.*\.-)(?!.*-\.)"
r"[a-z0-9](?:[a-z0-9.-]*[a-z0-9])?", bucket)
try:
ipaddress.ip_address(bucket)
bucket_is_ip = True
except ValueError:
bucket_is_ip = False
if (not bucket_valid or bucket_is_ip
or any(value < 1 for value in (max_bytes, max_objects, max_pages, page_size))):
raise ValueError("S3 evidence limits and bucket must be non-empty and positive")
if prefix.startswith("/"):
raise ValueError("S3 prefix must not start with a slash")
if endpoint_url:
parsed = urlsplit(endpoint_url)
if parsed.username or parsed.password:
@@ -36,9 +45,13 @@ class S3EvidenceSource:
raise ValueError("S3 custom endpoint must be an origin root without query/fragment")
if not trusted_endpoint:
raise ValueError("S3 custom endpoint requires explicit trusted_endpoint opt-in")
if parsed.hostname in {"localhost", "127.0.0.1", "::1"} and not allow_private_endpoint:
try:
literal = ipaddress.ip_address(parsed.hostname)
except ValueError:
literal = None
if literal is not None and not literal.is_global and not allow_private_endpoint:
raise ValueError("S3 private endpoint requires explicit opt-in")
self.bucket, self.prefix = bucket, prefix.lstrip("/")
self.bucket, self.prefix = bucket, prefix
self.max_bytes, self.max_objects = max_bytes, max_objects
self.max_pages, self.page_size = max_pages, min(page_size, 1000)
if client is None:
@@ -53,7 +66,7 @@ class S3EvidenceSource:
aws_session_token=session_token, verify=True,
config=BotoConfig(s3={"addressing_style": "path"}))
self._client = client
self._items: dict[str, tuple[str, str | None, str | None]] = {}
self._items: dict[str, tuple[SourceObject, str]] = {}
@staticmethod
def _error(operation: str, transient: bool = False):
@@ -65,32 +78,42 @@ class S3EvidenceSource:
def discover(self):
count = pages = 0
try:
paginator = self._client.get_paginator("list_objects_v2")
for page in paginator.paginate(Bucket=self.bucket, Prefix=self.prefix,
PaginationConfig={"PageSize": self.page_size}):
token = None
for _ in range(self.max_pages):
params = {"Bucket": self.bucket, "Prefix": self.prefix,
"MaxKeys": self.page_size}
if token is not None:
params["ContinuationToken"] = token
page = self._client.list_objects_v2(**params)
pages += 1
if pages > self.max_pages:
raise self._error("list_limit")
for row in page.get("Contents", []):
count += 1
if count > self.max_objects:
raise self._error("object_limit")
key, etag = row.get("Key"), row.get("ETag")
if (not isinstance(key, str) or not key.startswith(self.prefix)
or len(key.encode()) > 1024 or any(ord(char) < 32 for char in key)):
if (not isinstance(key, str) or not key or not key.startswith(self.prefix)
or len(key.encode()) > 1024
or any(ord(char) < 32 or ord(char) == 127 for char in key)):
raise self._error("invalid_key")
if not isinstance(etag, str) or not etag or len(etag) > 1024:
raise self._error("missing_validator")
uri = f"s3://{self.bucket}/{quote(key, safe='/')}"
fingerprint = f"etag:{hashlib.sha256(etag.encode()).hexdigest()}"
source_id = "s3:" + hashlib.sha256(uri.encode()).hexdigest()
self._items[source_id] = (key, None, etag)
modified = row.get("LastModified")
if modified is not None:
modified = modified.astimezone(UTC)
yield SourceObject(source_id=source_id, uri=uri, fingerprint=fingerprint,
modified_at=modified,
metadata={"size": int(row.get("Size", 0))})
item = SourceObject(source_id=source_id, uri=uri, fingerprint=fingerprint,
modified_at=modified,
metadata={"size": int(row.get("Size", 0))})
self._items[source_id] = (item, etag)
yield item
if not page.get("IsTruncated"):
return
token = page.get("NextContinuationToken")
if not isinstance(token, str) or not token:
raise self._error("list_continuation")
raise self._error("list_limit")
except EvidenceSourceError:
raise
except Exception as exc:
@@ -98,23 +121,19 @@ class S3EvidenceSource:
def acquire(self, item: SourceObject) -> AcquiredDocument:
binding = self._items.get(item.source_id)
if binding is None:
if binding is None or item != binding[0]:
raise self._error("acquire")
key, version, _etag = binding
discovered, etag = binding
key = discovered.uri.split(f"s3://{self.bucket}/", 1)[1]
from urllib.parse import unquote
key = unquote(key)
kwargs = {"Bucket": self.bucket, "Key": key}
if version:
kwargs["VersionId"] = version
body = None
try:
response = self._client.get_object(**kwargs)
body = response["Body"]
if version:
if response.get("VersionId") != version:
raise self._error("version_changed")
else:
current = hashlib.sha256((response.get("ETag") or "").encode()).hexdigest()
if item.fingerprint != f"etag:{current}":
raise self._error("etag_changed")
if response.get("ETag") != etag:
raise self._error("etag_changed")
if int(response.get("ContentLength", 0)) > self.max_bytes:
raise self._error("download_limit")
content = body.read(self.max_bytes + 1)