mptyl/ThothII
1
0
Fork 0
Code Issues 25 Pull Requests Actions Packages Projects Releases 1 Wiki Activity

fix(evidence): harden source acquisition

Browse Source
This commit is contained in:
marcopan
2026-07-12 03:32:18 +02:00
parent ffd683c587
commit 81ff1810d1
8 changed files with 523 additions and 183 deletions
Download Patch File Download Diff File Expand all files Collapse all files
harness/tests/test_http_evidence_source.py
+144 -5
View File
@@ -1,4 +1,5 @@
import threading
import socket
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
import pytest
@@ -8,12 +9,21 @@ from tht.ports.evidence import EvidenceSourceError
class Handler(BaseHTTPRequestHandler):
etag_requests = 0
etag_body_responses = 0
def do_GET(self):
if self.path.startswith("/etag"):
type(self).etag_requests += 1
if self.headers.get("If-None-Match") == '"abc"':
self.send_response(304)
self.end_headers()
return
self.send_response(200)
self.send_header("ETag", '"abc"')
self.send_header("Content-Type", "text/markdown")
self.end_headers()
type(self).etag_body_responses += 1
self.wfile.write(b"hello")
elif self.path == "/large":
self.send_response(200)
@@ -30,6 +40,12 @@ class Handler(BaseHTTPRequestHandler):
self.send_response(302)
self.send_header("Location", f"http://127.0.0.1:{self.server.server_port}/etag")
self.end_headers()
elif self.path == "/redirect-userinfo":
self.send_response(302)
self.send_header(
"Location", f"http://user:password@127.0.0.1:{self.server.server_port}/etag"
)
self.end_headers()
else:
self.send_response(200)
self.send_header("Last-Modified", "Wed, 21 Oct 2015 07:28:00 GMT")
@@ -42,6 +58,8 @@ class Handler(BaseHTTPRequestHandler):
@pytest.fixture
def server_url():
Handler.etag_requests = 0
Handler.etag_body_responses = 0
server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
thread = threading.Thread(target=server.serve_forever, daemon=True)
thread.start()
@@ -53,7 +71,9 @@ def server_url():
def test_http_uses_etag_and_strips_query_from_provenance(server_url):
source = HttpManifestEvidenceSource([f"{server_url}/etag?token=secret"])
source = HttpManifestEvidenceSource(
[f"{server_url}/etag?token=secret"], allow_private_hosts=True
)
item = next(iter(source.discover()))
assert item.fingerprint.startswith("etag:")
@@ -64,7 +84,9 @@ def test_http_uses_etag_and_strips_query_from_provenance(server_url):
def test_http_uses_last_modified_then_content_hash(server_url):
modified = next(iter(HttpManifestEvidenceSource([f"{server_url}/modified"]).discover()))
modified = next(iter(HttpManifestEvidenceSource(
[f"{server_url}/modified"], allow_private_hosts=True
).discover()))
assert modified.fingerprint.startswith("last-modified:")
class NoValidators(Handler):
@@ -78,7 +100,7 @@ def test_http_uses_last_modified_then_content_hash(server_url):
thread.start()
try:
item = next(iter(HttpManifestEvidenceSource(
[f"http://127.0.0.1:{server.server_port}/doc"]
[f"http://127.0.0.1:{server.server_port}/doc"], allow_private_hosts=True
).discover()))
assert item.fingerprint.startswith("sha256:")
finally:
@@ -89,14 +111,18 @@ def test_http_uses_last_modified_then_content_hash(server_url):
@pytest.mark.parametrize("path,retryable", [("/busy", True), ("/missing", False)])
def test_http_classifies_status_errors(server_url, path, retryable):
with pytest.raises(EvidenceSourceError) as caught:
list(HttpManifestEvidenceSource([server_url + path]).discover())
list(HttpManifestEvidenceSource(
[server_url + path], allow_private_hosts=True
).discover())
assert caught.value.retryable is retryable
assert server_url not in str(caught.value)
def test_http_rejects_oversize_and_private_redirect(server_url):
with pytest.raises(EvidenceSourceError) as large:
list(HttpManifestEvidenceSource([server_url + "/large"], max_bytes=10).discover())
list(HttpManifestEvidenceSource(
[server_url + "/large"], max_bytes=10, allow_private_hosts=True
).discover())
assert not large.value.retryable
with pytest.raises(EvidenceSourceError) as redirect:
@@ -107,3 +133,116 @@ def test_http_rejects_oversize_and_private_redirect(server_url):
def test_http_rejects_unsupported_manifest_scheme():
with pytest.raises(ValueError, match="http"):
HttpManifestEvidenceSource(["file:///tmp/secret"])
def test_http_conditional_discovery_reuses_cached_verified_bytes(server_url):
source = HttpManifestEvidenceSource([server_url + "/etag"], allow_private_hosts=True)
first = next(iter(source.discover()))
second = next(iter(source.discover()))
assert second == first
assert source.acquire(second).content == b"hello"
assert Handler.etag_requests == 3
assert Handler.etag_body_responses == 1
def test_http_rejects_mixed_public_private_dns_answers(monkeypatch):
monkeypatch.setattr(socket, "getaddrinfo", lambda *args, **kwargs: [
(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("93.184.216.34", 80)),
(socket.AF_INET6, socket.SOCK_STREAM, 6, "", ("::1", 80, 0, 0)),
])
with pytest.raises(EvidenceSourceError) as caught:
list(HttpManifestEvidenceSource(["http://example.test/doc"]).discover())
assert not caught.value.retryable
def test_http_rejects_userinfo_redirect(server_url):
with pytest.raises(EvidenceSourceError) as caught:
list(HttpManifestEvidenceSource(
[server_url + "/redirect-userinfo"], allow_private_hosts=True
).discover())
assert not caught.value.retryable
class FakeSocket:
def __init__(self, address):
self.address = address
def getpeername(self):
return (self.address, 443)
class FakeResponse:
status_code = 200
headers = {}
is_redirect = False
def __init__(self, *, peer="127.0.0.1", stream_error=None, location=None):
connection = type("Connection", (), {"sock": FakeSocket(peer)})()
self.raw = type("Raw", (), {"_connection": connection})()
self.stream_error = stream_error
self.closed = False
if location:
self.is_redirect = True
self.status_code = 302
self.headers = {"Location": location}
else:
self.is_redirect = False
self.status_code = 200
self.headers = {}
def iter_content(self, chunk_size):
if self.stream_error:
raise self.stream_error
yield b"ok"
def close(self):
self.closed = True
class FakeSession:
def __init__(self, response):
self.response = response
def get(self, *args, **kwargs):
return self.response
def test_http_rejects_public_to_private_rebind(monkeypatch):
monkeypatch.setattr(socket, "getaddrinfo", lambda *args, **kwargs: [
(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("93.184.216.34", 443)),
])
source = HttpManifestEvidenceSource(["https://example.test/doc"])
response = FakeResponse(peer="127.0.0.1")
source._session = FakeSession(response)
with pytest.raises(EvidenceSourceError):
list(source.discover())
assert response.closed
def test_http_rejects_public_redirect_to_private_destination(monkeypatch):
monkeypatch.setattr(socket, "getaddrinfo", lambda host, *args, **kwargs: [
(socket.AF_INET, socket.SOCK_STREAM, 6, "", (
"93.184.216.34" if host == "example.test" else "127.0.0.1", 443
)),
])
source = HttpManifestEvidenceSource(["https://example.test/doc"])
response = FakeResponse(
peer="93.184.216.34", location="https://private.test/secret"
)
source._session = FakeSession(response)
with pytest.raises(EvidenceSourceError) as caught:
list(source.discover())
assert not caught.value.retryable
assert response.closed
def test_http_closes_response_when_streaming_fails():
source = HttpManifestEvidenceSource(
["https://example.test/doc"], allow_private_hosts=True
)
response = FakeResponse(stream_error=socket.timeout("read timed out"))
source._session = FakeSession(response)
with pytest.raises(EvidenceSourceError):
list(source.discover())
assert response.closed