"""Explicit-manifest HTTP Evidence source with bounded streaming reads.""" import hashlib import ipaddress import socket from datetime import UTC, datetime from email.utils import parsedate_to_datetime from urllib.parse import urljoin, urlsplit import requests from tht.ports.evidence import ( AcquiredDocument, EvidenceSourceError, EvidenceSourceErrorCategory, SourceObject, canonical_provenance_uri, ) class HttpManifestEvidenceSource: def __init__( self, urls: list[str] | tuple[str, ...], *, connect_timeout: float = 5, read_timeout: float = 30, max_bytes: int = 10 * 1024 * 1024, max_redirects: int = 5, ) -> None: if not urls: raise ValueError("HTTP evidence manifest must contain at least one URL") if connect_timeout <= 0 or read_timeout <= 0 or max_bytes < 1 or max_redirects < 0: raise ValueError("HTTP evidence limits must be positive") self._transport_by_uri: dict[str, str] = {} for url in urls: parsed = urlsplit(url) if parsed.scheme not in {"http", "https"} or not parsed.hostname: raise ValueError("HTTP evidence URLs must use http or https") if parsed.username is not None or parsed.password is not None: raise ValueError("HTTP evidence URLs must not contain userinfo credentials") provenance = canonical_provenance_uri(url) if provenance in self._transport_by_uri: raise ValueError("HTTP evidence manifest contains duplicate canonical provenance") self._transport_by_uri[provenance] = url self.connect_timeout = connect_timeout self.read_timeout = read_timeout self.max_bytes = max_bytes self.max_redirects = max_redirects self._session = requests.Session() self._cache: dict[str, AcquiredDocument] = {} def __repr__(self) -> str: return f"HttpManifestEvidenceSource(objects={len(self._transport_by_uri)})" @staticmethod def _source_id(uri: str) -> str: return f"http:{hashlib.sha256(uri.encode()).hexdigest()}" @staticmethod def _reject_private_redirect(url: str) -> None: parsed = urlsplit(url) if parsed.scheme not in {"http", "https"} or not parsed.hostname: raise EvidenceSourceError( "redirect uses unsupported destination", category=EvidenceSourceErrorCategory.PERMANENT, details={"operation": "redirect"}, ) try: addresses = {row[4][0] for row in socket.getaddrinfo(parsed.hostname, parsed.port)} except OSError as error: raise EvidenceSourceError( "redirect destination resolution failed", category=EvidenceSourceErrorCategory.TRANSIENT, details={"operation": "redirect_resolution"}, ) from error if any(not ipaddress.ip_address(address).is_global for address in addresses): raise EvidenceSourceError( "redirect to private destination is forbidden", category=EvidenceSourceErrorCategory.PERMANENT, details={"operation": "redirect"}, ) @staticmethod def _status_category(status: int) -> EvidenceSourceErrorCategory: if status in {408, 425, 429} or 500 <= status <= 599: return EvidenceSourceErrorCategory.TRANSIENT return EvidenceSourceErrorCategory.PERMANENT def _download(self, transport_url: str, provenance: str) -> AcquiredDocument: current = transport_url try: for redirect_count in range(self.max_redirects + 1): response = self._session.get( current, stream=True, allow_redirects=False, timeout=(self.connect_timeout, self.read_timeout), ) if response.is_redirect: response.close() if redirect_count == self.max_redirects: raise EvidenceSourceError( "too many redirects", category=EvidenceSourceErrorCategory.PERMANENT, details={"operation": "redirect"}, ) destination = urljoin(current, response.headers.get("Location", "")) self._reject_private_redirect(destination) current = destination continue if not 200 <= response.status_code <= 299: status = response.status_code response.close() raise EvidenceSourceError( "HTTP status failure", category=self._status_category(status), details={"operation": "download", "status": status}, ) length = response.headers.get("Content-Length") if length is not None and int(length) > self.max_bytes: response.close() raise EvidenceSourceError( "HTTP object exceeds configured limit", category=EvidenceSourceErrorCategory.PERMANENT, details={"operation": "download", "limit_bytes": self.max_bytes}, ) content = bytearray() for chunk in response.iter_content(chunk_size=min(64 * 1024, self.max_bytes + 1)): content.extend(chunk) if len(content) > self.max_bytes: response.close() raise EvidenceSourceError( "HTTP object exceeds configured limit", category=EvidenceSourceErrorCategory.PERMANENT, details={"operation": "download", "limit_bytes": self.max_bytes}, ) etag = response.headers.get("ETag") last_modified = response.headers.get("Last-Modified") media_type = response.headers.get("Content-Type", "").split(";", 1)[0] or None response.close() break except EvidenceSourceError: raise except (requests.Timeout, requests.ConnectionError) as error: raise EvidenceSourceError( "HTTP transport unavailable", category=EvidenceSourceErrorCategory.TRANSIENT, details={"operation": "download"}, ) from error except (requests.RequestException, ValueError) as error: raise EvidenceSourceError( "HTTP acquisition failed", category=EvidenceSourceErrorCategory.PERMANENT, details={"operation": "download"}, ) from error modified_at = None if etag: # Entity tags are commonly quoted; the port's stable-value grammar is deliberately # narrower, so preserve the opaque validator through a deterministic digest. fingerprint = f"etag:{hashlib.sha256(etag.encode()).hexdigest()}" elif last_modified: try: modified_at = parsedate_to_datetime(last_modified).astimezone(UTC) fingerprint = f"last-modified:{int(modified_at.timestamp())}" except (TypeError, ValueError, OverflowError): fingerprint = f"sha256:{hashlib.sha256(content).hexdigest()}" else: fingerprint = f"sha256:{hashlib.sha256(content).hexdigest()}" item = SourceObject( source_id=self._source_id(provenance), uri=provenance, fingerprint=fingerprint, modified_at=modified_at, ) return AcquiredDocument( source=item, content=bytes(content), media_type=media_type, acquired_at=datetime.now(UTC), ) def discover(self): for provenance in sorted(self._transport_by_uri): document = self._download(self._transport_by_uri[provenance], provenance) self._cache[document.source.source_id] = document yield document.source def acquire(self, item: SourceObject) -> AcquiredDocument: expected_id = self._source_id(item.uri) transport = self._transport_by_uri.get(item.uri) if transport is None or item.source_id != expected_id: raise EvidenceSourceError( "object does not belong to HTTP source", category=EvidenceSourceErrorCategory.PERMANENT, details={"operation": "acquire"}, ) cached = self._cache.get(item.source_id) if cached is not None and cached.source.fingerprint == item.fingerprint: return cached return self._download(transport, item.uri)