diff --git a/.superpowers/sdd/evidence-task-6-report.md b/.superpowers/sdd/evidence-task-6-report.md index 384a8897..b1d347ab 100644 --- a/.superpowers/sdd/evidence-task-6-report.md +++ b/.superpowers/sdd/evidence-task-6-report.md @@ -33,3 +33,17 @@ The cleanup boundary now begins immediately after snapshot materialization. Resu validation and `JobSpec` construction are guarded by the same release routine as `run_job`, so corrupt/mismatched resume state or constructor failure clears the pipeline holder, removes the private directory, and restores the snapshot registry to its prior state before propagating. + +## Shipped preprocessing startup contract + +Local-vector preprocessing now uses a dedicated Compose override. Both one-shot jobs depend on a +successfully completed `vector-migrate`, whose transitive chain waits for database health and role +reconciliation. The generic preprocessing overlay remains independently renderable and contains no +local-vector services or password secrets. README commands include the local override and build the +job image before running. + +The real clean-project smoke no longer injects dependencies or manually starts, reconciles, or +migrates PostgreSQL. Its first shipped `compose run preprocess-evidence` demonstrably creates the +database, waits for health, runs reconciliation and migration, then runs the Evidence job. Unchanged +rerun, changed-source publish, DWH preprocessing, ACTIVE verification, and injected-failure cleanup +all pass through the same shipped dependency path. diff --git a/README.md b/README.md index d6d86547..022802c6 100644 --- a/README.md +++ b/README.md @@ -62,23 +62,13 @@ runtime): ```sh docker compose -f compose.yaml -f deploy/compose.local-vector.yaml \ - -f deploy/compose.preprocess.yaml -f deploy/compose.preprocess-local-vector.yaml \ - --profile local-vector --profile preprocess build preprocess-evidence + -f deploy/compose.preprocess.yaml --profile local-vector --profile preprocess \ + run --rm preprocess-evidence docker compose -f compose.yaml -f deploy/compose.local-vector.yaml \ - -f deploy/compose.preprocess.yaml -f deploy/compose.preprocess-local-vector.yaml \ - --profile local-vector --profile preprocess run --rm preprocess-evidence -docker compose -f compose.yaml -f deploy/compose.local-vector.yaml \ - -f deploy/compose.preprocess.yaml -f deploy/compose.preprocess-local-vector.yaml \ - --profile local-vector --profile preprocess build preprocess-dwh -docker compose -f compose.yaml -f deploy/compose.local-vector.yaml \ - -f deploy/compose.preprocess.yaml -f deploy/compose.preprocess-local-vector.yaml \ - --profile local-vector --profile preprocess run --rm preprocess-dwh + -f deploy/compose.preprocess.yaml --profile local-vector --profile preprocess \ + run --rm preprocess-dwh ``` -The local preprocessing override makes each job wait for the vector database health check, -role reconciliation, and a successful migration. These commands are safe on a clean Compose -project; no separate database startup or migration command is required. - S3 Evidence uses the optional `tht[s3]` dependency and canonical `s3://bucket/key` provenance. AWS endpoints are used when no custom URL is supplied. Every custom endpoint is an explicit egress trust-boundary opt-in and uses path-style addressing; private and HTTP endpoints require additional diff --git a/deploy/compose.preprocess-local-vector.yaml b/deploy/compose.preprocess-local-vector.yaml new file mode 100644 index 00000000..3b8b8376 --- /dev/null +++ b/deploy/compose.preprocess-local-vector.yaml @@ -0,0 +1,21 @@ +services: + preprocess-evidence: + environment: + THT_VECTOR_READER_PASSWORD_FILE: /run/secrets/vector_reader_password + THT_VECTOR_WRITER_PASSWORD_FILE: /run/secrets/vector_writer_password + secrets: [vector_reader_password, vector_writer_password] + depends_on: + vector-migrate: {condition: service_completed_successfully} + + preprocess-dwh: + environment: + THT_VECTOR_READER_PASSWORD_FILE: /run/secrets/vector_reader_password + secrets: [vector_reader_password] + depends_on: + vector-migrate: {condition: service_completed_successfully} + +secrets: + vector_reader_password: + file: ${THT_VECTOR_READER_PASSWORD_SECRET_FILE:?set THT_VECTOR_READER_PASSWORD_SECRET_FILE} + vector_writer_password: + file: ${THT_VECTOR_WRITER_PASSWORD_SECRET_FILE:?set THT_VECTOR_WRITER_PASSWORD_SECRET_FILE} diff --git a/deploy/compose.preprocess.yaml b/deploy/compose.preprocess.yaml index 214fc8d1..794a9e58 100644 --- a/deploy/compose.preprocess.yaml +++ b/deploy/compose.preprocess.yaml @@ -10,9 +10,6 @@ services: environment: THT_DATA_ROOT: /data THT_OLLAMA_URL: "${THT_OLLAMA_URL:-http://host.docker.internal:11434}" - THT_VECTOR_READER_PASSWORD_FILE: /run/secrets/vector_reader_password - THT_VECTOR_WRITER_PASSWORD_FILE: /run/secrets/vector_writer_password - secrets: [vector_reader_password, vector_writer_password] volumes: - thoth_data:/data - ./deploy/workspaces:/app/harness/workspaces:ro @@ -28,15 +25,7 @@ services: command: ["mkdir -p /data/workspaces/preprocess-dwh && exec /app/docker/core-entrypoint.sh preprocess dwh --steps introspect --json -c /app/harness/workspaces/preprocess-dwh.yaml"] environment: THT_DATA_ROOT: /data - THT_VECTOR_READER_PASSWORD_FILE: /run/secrets/vector_reader_password - secrets: [vector_reader_password] volumes: - thoth_data:/data - ./deploy/workspaces:/app/harness/workspaces:ro restart: "no" - -secrets: - vector_reader_password: - file: ${THT_VECTOR_READER_PASSWORD_SECRET_FILE:?set THT_VECTOR_READER_PASSWORD_SECRET_FILE} - vector_writer_password: - file: ${THT_VECTOR_WRITER_PASSWORD_SECRET_FILE:?set THT_VECTOR_WRITER_PASSWORD_SECRET_FILE} diff --git a/scripts/preprocess-smoke.sh b/scripts/preprocess-smoke.sh index 77d4dc40..ae3d4820 100755 --- a/scripts/preprocess-smoke.sh +++ b/scripts/preprocess-smoke.sh @@ -79,18 +79,11 @@ services: volumes: - $tmp/source:/data/source:ro depends_on: - vector-migrate: {condition: service_completed_successfully} mock-embeddings: {condition: service_started} - preprocess-dwh: - depends_on: - vector-migrate: {condition: service_completed_successfully} YAML -compose="docker compose -f compose.yaml -f deploy/compose.local-vector.yaml -f deploy/compose.preprocess.yaml -f $tmp/smoke.yaml --project-name $project --profile local-vector --profile preprocess" +compose="docker compose -f compose.yaml -f deploy/compose.local-vector.yaml -f deploy/compose.preprocess.yaml -f deploy/compose.preprocess-local-vector.yaml -f $tmp/smoke.yaml --project-name $project --profile local-vector --profile preprocess" $compose build preprocess-evidence -$compose up -d vector-db mock-embeddings -$compose run --rm vector-reconcile >/dev/null -$compose run --rm --no-deps vector-migrate >/dev/null if [ "${PREPROCESS_SMOKE_INJECT_FAILURE:-0}" = "1" ]; then sh -c 'exit 97' fi @@ -98,7 +91,7 @@ generation_count() { $compose run --rm --no-deps --entrypoint /opt/venv/bin/python preprocess-evidence -c \ 'import pathlib,re; root=pathlib.Path("/data/workspaces/preprocess-evidence/corpus"); print(sum(1 for p in root.iterdir() if p.is_dir() and re.fullmatch(r"gen-[0-9a-f]{32}",p.name)) if root.exists() else 0)' } -before=$(generation_count) +before=0 first=$($compose run --rm preprocess-evidence) after_first=$(generation_count) second=$($compose run --rm preprocess-evidence) diff --git a/scripts/test-preprocess-compose-config.sh b/scripts/test-preprocess-compose-config.sh new file mode 100755 index 00000000..1bc9f917 --- /dev/null +++ b/scripts/test-preprocess-compose-config.sh @@ -0,0 +1,43 @@ +#!/bin/sh +set -eu + +cd "$(dirname "$0")/.." + +export THT_VECTOR_BOOTSTRAP_PASSWORD_SECRET_FILE=${THT_VECTOR_BOOTSTRAP_PASSWORD_SECRET_FILE:-/tmp/vector-bootstrap} +export THT_VECTOR_MIGRATOR_PASSWORD_SECRET_FILE=${THT_VECTOR_MIGRATOR_PASSWORD_SECRET_FILE:-/tmp/vector-migrator} +export THT_VECTOR_READER_PASSWORD_SECRET_FILE=${THT_VECTOR_READER_PASSWORD_SECRET_FILE:-/tmp/vector-reader} +export THT_VECTOR_WRITER_PASSWORD_SECRET_FILE=${THT_VECTOR_WRITER_PASSWORD_SECRET_FILE:-/tmp/vector-writer} + +local_files="-f compose.yaml -f deploy/compose.local-vector.yaml -f deploy/compose.preprocess.yaml -f deploy/compose.preprocess-local-vector.yaml" +local_json=$(docker compose $local_files --profile local-vector --profile preprocess config --format json) + +printf '%s' "$local_json" | python3 -c ' +import json, sys + +config = json.load(sys.stdin) +services = config["services"] +for name in ("preprocess-evidence", "preprocess-dwh"): + dependency = services[name].get("depends_on", {}).get("vector-migrate") + assert dependency is not None, f"{name} does not depend on vector-migrate" + assert dependency["condition"] == "service_completed_successfully", dependency +' + +external_json=$(docker compose \ + -f compose.yaml -f deploy/compose.preprocess.yaml \ + --profile preprocess config --format json) + +printf '%s' "$external_json" | python3 -c ' +import json, sys + +config = json.load(sys.stdin) +services = config["services"] +assert "vector-db" not in services +assert "vector-migrate" not in services +assert "vector-reconcile" not in services +for name in ("preprocess-evidence", "preprocess-dwh"): + service = services[name] + assert "depends_on" not in service + assert not service.get("secrets"), service.get("secrets") +' + +echo "preprocess compose config: ok"