diff --git a/.superpowers/sdd/evidence-task-7-report.md b/.superpowers/sdd/evidence-task-7-report.md index 04f626b1..d2e2ab89 100644 --- a/.superpowers/sdd/evidence-task-7-report.md +++ b/.superpowers/sdd/evidence-task-7-report.md @@ -53,3 +53,12 @@ and absent validators fail closed. Acquisition accepts only the exact stored `So compares the response ETag with the stored discovery validator. The real smoke snapshots generation directory counts after every run and has an injected-failure cleanup mode; cleanup fails if Compose down fails or any owned container, volume, or network remains. + +The canonical smoke correction counts only root-level `corpus/gen-<32 hex>` directories. It exposed +that the durable job path still published an empty unchanged generation; the pipeline now returns +the existing ACTIVE generation without staging a directory when compatibility and all source +fingerprints are unchanged. The smoke therefore proves directory deltas `+1`, `+0`, `+1`. +Failure injection runs a real exit-97 command after resources exist and reaches the EXIT trap. +Cleanup aggregates Compose-down, residual container/volume/network, and temp-directory failures +while preserving the original failure status. S3 prefixes are validated before any client request +for leading slash, UTF-8 byte length, controls, and DEL. diff --git a/harness/tests/test_corpus_pipeline.py b/harness/tests/test_corpus_pipeline.py index fa9e2357..511c6fc0 100644 --- a/harness/tests/test_corpus_pipeline.py +++ b/harness/tests/test_corpus_pipeline.py @@ -431,6 +431,20 @@ def test_unchanged_documents_skip_acquire_normalize_chunk_and_embed(tmp_path): assert second_embedder.calls == [] +def test_unchanged_job_reuses_active_generation_without_new_directory(tmp_path): + source = Source([(item("one", "a"), "hello")]) + candidate = pipeline(tmp_path, source) + args = dict(workspace_id="demo", workspace_root=tmp_path, + config_fingerprint="sha256:" + "1" * 64, + input_fingerprint="sha256:" + "2" * 64) + first = candidate.run_as_job(**args) + count = len(candidate.store.list_generations()) + second = candidate.run_as_job(**args) + assert second.generation == first.generation + assert second.published is False + assert len(candidate.store.list_generations()) == count + + def test_removed_documents_are_marked_and_absent_from_new_manifest(tmp_path): one, two = item("one", "a"), item("two", "b") pipeline(tmp_path, Source([(one, "one"), (two, "two")])).run() diff --git a/harness/tests/test_s3_evidence_source.py b/harness/tests/test_s3_evidence_source.py index b23d7578..710b44fc 100644 --- a/harness/tests/test_s3_evidence_source.py +++ b/harness/tests/test_s3_evidence_source.py @@ -104,6 +104,15 @@ def test_s3_rejects_leading_slash_prefix_empty_and_control_keys(): list(S3EvidenceSource(bucket="evidence", prefix="clinical/", client=client).discover()) +@pytest.mark.parametrize("prefix", ["/bad", "x" * 1025, "bad\x00prefix", "bad\x7fprefix"]) +def test_s3_rejects_invalid_prefix_before_client_request(prefix): + from tht.adapters.evidence.s3 import S3EvidenceSource + client = Client() + with pytest.raises(ValueError, match="prefix"): + S3EvidenceSource(bucket="evidence", prefix=prefix, client=client) + assert client.list_calls == 0 + + def test_s3_hard_page_limit_never_requests_page_max_plus_one(): from tht.adapters.evidence.s3 import S3EvidenceSource client = Client() diff --git a/harness/tht/adapters/evidence/s3.py b/harness/tht/adapters/evidence/s3.py index 8e9ef4d5..e0a45fb7 100644 --- a/harness/tht/adapters/evidence/s3.py +++ b/harness/tht/adapters/evidence/s3.py @@ -29,8 +29,9 @@ class S3EvidenceSource: if (not bucket_valid or bucket_is_ip or any(value < 1 for value in (max_bytes, max_objects, max_pages, page_size))): raise ValueError("S3 evidence limits and bucket must be non-empty and positive") - if prefix.startswith("/"): - raise ValueError("S3 prefix must not start with a slash") + if (prefix.startswith("/") or len(prefix.encode()) > 1024 + or any(ord(char) < 32 or ord(char) == 127 for char in prefix)): + raise ValueError("S3 prefix is invalid") if endpoint_url: parsed = urlsplit(endpoint_url) if parsed.username or parsed.password: diff --git a/harness/tht/corpus/pipeline.py b/harness/tht/corpus/pipeline.py index c5a66649..2b860ff7 100644 --- a/harness/tht/corpus/pipeline.py +++ b/harness/tht/corpus/pipeline.py @@ -220,6 +220,16 @@ class CorpusPipeline: "dimensions": self.embedding_dimensions, "chunk_policy": asdict(self.chunk_policy), }) + previous = self.store.active_manifest() + if (not dry_run and resume_run_id is None and previous is not None + and previous.metadata.get("compatibility_fingerprint") == compatibility + and previous.metadata.get("fingerprints") == { + item.source_id: item.fingerprint for _, item in discovered + }): + return PipelineResult( + "succeeded", previous.manifest_id, False, (), + tuple(sorted(item.source_id for _, item in discovered)), (), previous, + ) spec = JobSpec( workspace_id=workspace_id, job_type="evidence", diff --git a/scripts/preprocess-smoke.sh b/scripts/preprocess-smoke.sh index 5377a0da..ca58403a 100755 --- a/scripts/preprocess-smoke.sh +++ b/scripts/preprocess-smoke.sh @@ -2,20 +2,50 @@ set -eu cd "$(dirname "$0")/.." -tmp=$(mktemp -d "${TMPDIR:-/tmp}/thoth-preprocess.XXXXXX") -project="thoth-preprocess-$$" +if [ "${1:-}" = "--cleanup-failure" ] && [ -z "${PREPROCESS_SMOKE_CHILD:-}" ]; then + child_project="thoth-preprocess-failure-$$" + child_tmp=$(mktemp -d "${TMPDIR:-/tmp}/thoth-preprocess-failure.XXXXXX") + set +e + PREPROCESS_SMOKE_CHILD=1 PREPROCESS_SMOKE_INJECT_FAILURE=1 \ + PREPROCESS_SMOKE_PROJECT="$child_project" PREPROCESS_SMOKE_TMP="$child_tmp" "$0" + child_status=$? + set -e + test "$child_status" -eq 97 + test ! -e "$child_tmp" + test -z "$(docker ps -aq --filter "label=com.docker.compose.project=$child_project")" + test -z "$(docker volume ls -q --filter "label=com.docker.compose.project=$child_project")" + test -z "$(docker network ls -q --filter "label=com.docker.compose.project=$child_project")" + echo "injected preprocessing failure preserved status and cleaned every owned resource." + exit 0 +fi + +tmp=${PREPROCESS_SMOKE_TMP:-$(mktemp -d "${TMPDIR:-/tmp}/thoth-preprocess.XXXXXX")} +project=${PREPROCESS_SMOKE_PROJECT:-thoth-preprocess-$$} compose="" cleanup() { - trap - EXIT HUP INT TERM + original_status=$? + cleanup_failed=0 + set +e if [ -n "$compose" ]; then $compose down --volumes >/dev/null + test "$?" -eq 0 || cleanup_failed=1 fi - test -z "$(docker ps -aq --filter "label=com.docker.compose.project=$project")" - test -z "$(docker volume ls -q --filter "label=com.docker.compose.project=$project")" - test -z "$(docker network ls -q --filter "label=com.docker.compose.project=$project")" + test -z "$(docker ps -aq --filter "label=com.docker.compose.project=$project")" || cleanup_failed=1 + test -z "$(docker volume ls -q --filter "label=com.docker.compose.project=$project")" || cleanup_failed=1 + test -z "$(docker network ls -q --filter "label=com.docker.compose.project=$project")" || cleanup_failed=1 rm -rf "$tmp" + test ! -e "$tmp" || cleanup_failed=1 + if [ "$original_status" -ne 0 ]; then + exit "$original_status" + fi + if [ "$cleanup_failed" -ne 0 ]; then + exit 1 + fi } -trap cleanup EXIT HUP INT TERM +trap cleanup EXIT +trap 'exit 129' HUP +trap 'exit 130' INT +trap 'exit 143' TERM mkdir -p "$tmp/source/evidence" printf '%s\n' '# Evidence' 'generation one' >"$tmp/source/evidence/a.md" for name in bootstrap migrator reader writer; do @@ -61,15 +91,12 @@ $compose build preprocess-evidence $compose up -d vector-db mock-embeddings $compose run --rm vector-reconcile >/dev/null $compose run --rm --no-deps vector-migrate >/dev/null -if [ "${1:-}" = "--cleanup-failure" ]; then - cleanup - test ! -e "$tmp" - echo "injected preprocessing failure cleanup passed." - exit 0 +if [ "${PREPROCESS_SMOKE_INJECT_FAILURE:-0}" = "1" ]; then + sh -c 'exit 97' fi generation_count() { - $compose run --rm --no-deps --entrypoint sh preprocess-evidence -c \ - 'find /data/workspaces/preprocess-evidence/corpus/generations -mindepth 1 -maxdepth 1 -type d 2>/dev/null | wc -l' + $compose run --rm --no-deps --entrypoint /opt/venv/bin/python preprocess-evidence -c \ + 'import pathlib,re; root=pathlib.Path("/data/workspaces/preprocess-evidence/corpus"); print(sum(1 for p in root.iterdir() if p.is_dir() and re.fullmatch(r"gen-[0-9a-f]{32}",p.name)) if root.exists() else 0)' } before=$(generation_count) first=$($compose run --rm preprocess-evidence) @@ -103,5 +130,3 @@ import json, sys assert json.loads(sys.argv[1])["generation"] == sys.argv[2].strip() PY echo "real Compose preprocessing unchanged rerun, mutation, DWH job, and ACTIVE publish passed." -cleanup -test ! -e "$tmp"