fix(preprocess): enforce local vector startup chain

This commit is contained in:
2026-07-12 07:54:09 +02:00
parent e40a9d9a56
commit 72bc50ab2e
6 changed files with 84 additions and 34 deletions
+2 -9
View File
@@ -79,18 +79,11 @@ services:
volumes:
- $tmp/source:/data/source:ro
depends_on:
vector-migrate: {condition: service_completed_successfully}
mock-embeddings: {condition: service_started}
preprocess-dwh:
depends_on:
vector-migrate: {condition: service_completed_successfully}
YAML
compose="docker compose -f compose.yaml -f deploy/compose.local-vector.yaml -f deploy/compose.preprocess.yaml -f $tmp/smoke.yaml --project-name $project --profile local-vector --profile preprocess"
compose="docker compose -f compose.yaml -f deploy/compose.local-vector.yaml -f deploy/compose.preprocess.yaml -f deploy/compose.preprocess-local-vector.yaml -f $tmp/smoke.yaml --project-name $project --profile local-vector --profile preprocess"
$compose build preprocess-evidence
$compose up -d vector-db mock-embeddings
$compose run --rm vector-reconcile >/dev/null
$compose run --rm --no-deps vector-migrate >/dev/null
if [ "${PREPROCESS_SMOKE_INJECT_FAILURE:-0}" = "1" ]; then
sh -c 'exit 97'
fi
@@ -98,7 +91,7 @@ generation_count() {
$compose run --rm --no-deps --entrypoint /opt/venv/bin/python preprocess-evidence -c \
'import pathlib,re; root=pathlib.Path("/data/workspaces/preprocess-evidence/corpus"); print(sum(1 for p in root.iterdir() if p.is_dir() and re.fullmatch(r"gen-[0-9a-f]{32}",p.name)) if root.exists() else 0)'
}
before=$(generation_count)
before=0
first=$($compose run --rm preprocess-evidence)
after_first=$(generation_count)
second=$($compose run --rm preprocess-evidence)
+43
View File
@@ -0,0 +1,43 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
export THT_VECTOR_BOOTSTRAP_PASSWORD_SECRET_FILE=${THT_VECTOR_BOOTSTRAP_PASSWORD_SECRET_FILE:-/tmp/vector-bootstrap}
export THT_VECTOR_MIGRATOR_PASSWORD_SECRET_FILE=${THT_VECTOR_MIGRATOR_PASSWORD_SECRET_FILE:-/tmp/vector-migrator}
export THT_VECTOR_READER_PASSWORD_SECRET_FILE=${THT_VECTOR_READER_PASSWORD_SECRET_FILE:-/tmp/vector-reader}
export THT_VECTOR_WRITER_PASSWORD_SECRET_FILE=${THT_VECTOR_WRITER_PASSWORD_SECRET_FILE:-/tmp/vector-writer}
local_files="-f compose.yaml -f deploy/compose.local-vector.yaml -f deploy/compose.preprocess.yaml -f deploy/compose.preprocess-local-vector.yaml"
local_json=$(docker compose $local_files --profile local-vector --profile preprocess config --format json)
printf '%s' "$local_json" | python3 -c '
import json, sys
config = json.load(sys.stdin)
services = config["services"]
for name in ("preprocess-evidence", "preprocess-dwh"):
dependency = services[name].get("depends_on", {}).get("vector-migrate")
assert dependency is not None, f"{name} does not depend on vector-migrate"
assert dependency["condition"] == "service_completed_successfully", dependency
'
external_json=$(docker compose \
-f compose.yaml -f deploy/compose.preprocess.yaml \
--profile preprocess config --format json)
printf '%s' "$external_json" | python3 -c '
import json, sys
config = json.load(sys.stdin)
services = config["services"]
assert "vector-db" not in services
assert "vector-migrate" not in services
assert "vector-reconcile" not in services
for name in ("preprocess-evidence", "preprocess-dwh"):
service = services[name]
assert "depends_on" not in service
assert not service.get("secrets"), service.get("secrets")
'
echo "preprocess compose config: ok"