203 lines
8.8 KiB
Python
203 lines
8.8 KiB
Python
"""Explicit-manifest HTTP Evidence source with bounded streaming reads."""
|
|
|
|
import hashlib
|
|
import ipaddress
|
|
import socket
|
|
from datetime import UTC, datetime
|
|
from email.utils import parsedate_to_datetime
|
|
from urllib.parse import urljoin, urlsplit
|
|
|
|
import requests
|
|
|
|
from tht.ports.evidence import (
|
|
AcquiredDocument,
|
|
EvidenceSourceError,
|
|
EvidenceSourceErrorCategory,
|
|
SourceObject,
|
|
canonical_provenance_uri,
|
|
)
|
|
|
|
|
|
class HttpManifestEvidenceSource:
|
|
def __init__(
|
|
self,
|
|
urls: list[str] | tuple[str, ...],
|
|
*,
|
|
connect_timeout: float = 5,
|
|
read_timeout: float = 30,
|
|
max_bytes: int = 10 * 1024 * 1024,
|
|
max_redirects: int = 5,
|
|
) -> None:
|
|
if not urls:
|
|
raise ValueError("HTTP evidence manifest must contain at least one URL")
|
|
if connect_timeout <= 0 or read_timeout <= 0 or max_bytes < 1 or max_redirects < 0:
|
|
raise ValueError("HTTP evidence limits must be positive")
|
|
self._transport_by_uri: dict[str, str] = {}
|
|
for url in urls:
|
|
parsed = urlsplit(url)
|
|
if parsed.scheme not in {"http", "https"} or not parsed.hostname:
|
|
raise ValueError("HTTP evidence URLs must use http or https")
|
|
if parsed.username is not None or parsed.password is not None:
|
|
raise ValueError("HTTP evidence URLs must not contain userinfo credentials")
|
|
provenance = canonical_provenance_uri(url)
|
|
if provenance in self._transport_by_uri:
|
|
raise ValueError("HTTP evidence manifest contains duplicate canonical provenance")
|
|
self._transport_by_uri[provenance] = url
|
|
self.connect_timeout = connect_timeout
|
|
self.read_timeout = read_timeout
|
|
self.max_bytes = max_bytes
|
|
self.max_redirects = max_redirects
|
|
self._session = requests.Session()
|
|
self._cache: dict[str, AcquiredDocument] = {}
|
|
|
|
def __repr__(self) -> str:
|
|
return f"HttpManifestEvidenceSource(objects={len(self._transport_by_uri)})"
|
|
|
|
@staticmethod
|
|
def _source_id(uri: str) -> str:
|
|
return f"http:{hashlib.sha256(uri.encode()).hexdigest()}"
|
|
|
|
@staticmethod
|
|
def _reject_private_redirect(url: str) -> None:
|
|
parsed = urlsplit(url)
|
|
if parsed.scheme not in {"http", "https"} or not parsed.hostname:
|
|
raise EvidenceSourceError(
|
|
"redirect uses unsupported destination",
|
|
category=EvidenceSourceErrorCategory.PERMANENT,
|
|
details={"operation": "redirect"},
|
|
)
|
|
try:
|
|
addresses = {row[4][0] for row in socket.getaddrinfo(parsed.hostname, parsed.port)}
|
|
except OSError as error:
|
|
raise EvidenceSourceError(
|
|
"redirect destination resolution failed",
|
|
category=EvidenceSourceErrorCategory.TRANSIENT,
|
|
details={"operation": "redirect_resolution"},
|
|
) from error
|
|
if any(not ipaddress.ip_address(address).is_global for address in addresses):
|
|
raise EvidenceSourceError(
|
|
"redirect to private destination is forbidden",
|
|
category=EvidenceSourceErrorCategory.PERMANENT,
|
|
details={"operation": "redirect"},
|
|
)
|
|
|
|
@staticmethod
|
|
def _status_category(status: int) -> EvidenceSourceErrorCategory:
|
|
if status in {408, 425, 429} or 500 <= status <= 599:
|
|
return EvidenceSourceErrorCategory.TRANSIENT
|
|
return EvidenceSourceErrorCategory.PERMANENT
|
|
|
|
def _download(self, transport_url: str, provenance: str) -> AcquiredDocument:
|
|
current = transport_url
|
|
try:
|
|
for redirect_count in range(self.max_redirects + 1):
|
|
response = self._session.get(
|
|
current,
|
|
stream=True,
|
|
allow_redirects=False,
|
|
timeout=(self.connect_timeout, self.read_timeout),
|
|
)
|
|
if response.is_redirect:
|
|
response.close()
|
|
if redirect_count == self.max_redirects:
|
|
raise EvidenceSourceError(
|
|
"too many redirects",
|
|
category=EvidenceSourceErrorCategory.PERMANENT,
|
|
details={"operation": "redirect"},
|
|
)
|
|
destination = urljoin(current, response.headers.get("Location", ""))
|
|
self._reject_private_redirect(destination)
|
|
current = destination
|
|
continue
|
|
if not 200 <= response.status_code <= 299:
|
|
status = response.status_code
|
|
response.close()
|
|
raise EvidenceSourceError(
|
|
"HTTP status failure",
|
|
category=self._status_category(status),
|
|
details={"operation": "download", "status": status},
|
|
)
|
|
length = response.headers.get("Content-Length")
|
|
if length is not None and int(length) > self.max_bytes:
|
|
response.close()
|
|
raise EvidenceSourceError(
|
|
"HTTP object exceeds configured limit",
|
|
category=EvidenceSourceErrorCategory.PERMANENT,
|
|
details={"operation": "download", "limit_bytes": self.max_bytes},
|
|
)
|
|
content = bytearray()
|
|
for chunk in response.iter_content(chunk_size=min(64 * 1024, self.max_bytes + 1)):
|
|
content.extend(chunk)
|
|
if len(content) > self.max_bytes:
|
|
response.close()
|
|
raise EvidenceSourceError(
|
|
"HTTP object exceeds configured limit",
|
|
category=EvidenceSourceErrorCategory.PERMANENT,
|
|
details={"operation": "download", "limit_bytes": self.max_bytes},
|
|
)
|
|
etag = response.headers.get("ETag")
|
|
last_modified = response.headers.get("Last-Modified")
|
|
media_type = response.headers.get("Content-Type", "").split(";", 1)[0] or None
|
|
response.close()
|
|
break
|
|
except EvidenceSourceError:
|
|
raise
|
|
except (requests.Timeout, requests.ConnectionError) as error:
|
|
raise EvidenceSourceError(
|
|
"HTTP transport unavailable",
|
|
category=EvidenceSourceErrorCategory.TRANSIENT,
|
|
details={"operation": "download"},
|
|
) from error
|
|
except (requests.RequestException, ValueError) as error:
|
|
raise EvidenceSourceError(
|
|
"HTTP acquisition failed",
|
|
category=EvidenceSourceErrorCategory.PERMANENT,
|
|
details={"operation": "download"},
|
|
) from error
|
|
|
|
modified_at = None
|
|
if etag:
|
|
# Entity tags are commonly quoted; the port's stable-value grammar is deliberately
|
|
# narrower, so preserve the opaque validator through a deterministic digest.
|
|
fingerprint = f"etag:{hashlib.sha256(etag.encode()).hexdigest()}"
|
|
elif last_modified:
|
|
try:
|
|
modified_at = parsedate_to_datetime(last_modified).astimezone(UTC)
|
|
fingerprint = f"last-modified:{int(modified_at.timestamp())}"
|
|
except (TypeError, ValueError, OverflowError):
|
|
fingerprint = f"sha256:{hashlib.sha256(content).hexdigest()}"
|
|
else:
|
|
fingerprint = f"sha256:{hashlib.sha256(content).hexdigest()}"
|
|
item = SourceObject(
|
|
source_id=self._source_id(provenance),
|
|
uri=provenance,
|
|
fingerprint=fingerprint,
|
|
modified_at=modified_at,
|
|
)
|
|
return AcquiredDocument(
|
|
source=item,
|
|
content=bytes(content),
|
|
media_type=media_type,
|
|
acquired_at=datetime.now(UTC),
|
|
)
|
|
|
|
def discover(self):
|
|
for provenance in sorted(self._transport_by_uri):
|
|
document = self._download(self._transport_by_uri[provenance], provenance)
|
|
self._cache[document.source.source_id] = document
|
|
yield document.source
|
|
|
|
def acquire(self, item: SourceObject) -> AcquiredDocument:
|
|
expected_id = self._source_id(item.uri)
|
|
transport = self._transport_by_uri.get(item.uri)
|
|
if transport is None or item.source_id != expected_id:
|
|
raise EvidenceSourceError(
|
|
"object does not belong to HTTP source",
|
|
category=EvidenceSourceErrorCategory.PERMANENT,
|
|
details={"operation": "acquire"},
|
|
)
|
|
cached = self._cache.get(item.source_id)
|
|
if cached is not None and cached.source.fingerprint == item.fingerprint:
|
|
return cached
|
|
return self._download(transport, item.uri)
|