chore: commit remaining worktree changes
This commit is contained in:
@@ -1,160 +0,0 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
if [ "${1:-}" = "--cleanup-failure" ] && [ -z "${PREPROCESS_SMOKE_CHILD:-}" ]; then
|
||||
child_project="thoth-preprocess-failure-$$"
|
||||
child_tmp=$(mktemp -d "${TMPDIR:-/tmp}/thoth-preprocess-failure.XXXXXX")
|
||||
set +e
|
||||
PREPROCESS_SMOKE_CHILD=1 PREPROCESS_SMOKE_INJECT_FAILURE=1 \
|
||||
PREPROCESS_SMOKE_PROJECT="$child_project" PREPROCESS_SMOKE_TMP="$child_tmp" "$0"
|
||||
child_status=$?
|
||||
set -e
|
||||
test "$child_status" -eq 97
|
||||
test ! -e "$child_tmp"
|
||||
test -z "$(docker ps -aq --filter "label=com.docker.compose.project=$child_project")"
|
||||
test -z "$(docker volume ls -q --filter "label=com.docker.compose.project=$child_project")"
|
||||
test -z "$(docker network ls -q --filter "label=com.docker.compose.project=$child_project")"
|
||||
echo "injected preprocessing failure preserved status and cleaned every owned resource."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
tmp=${PREPROCESS_SMOKE_TMP:-$(mktemp -d "${TMPDIR:-/tmp}/thoth-preprocess.XXXXXX")}
|
||||
project=${PREPROCESS_SMOKE_PROJECT:-thoth-preprocess-$$}
|
||||
compose=""
|
||||
cleanup() {
|
||||
original_status=$?
|
||||
cleanup_failed=0
|
||||
set +e
|
||||
if [ -n "$compose" ]; then
|
||||
$compose down --volumes >/dev/null
|
||||
test "$?" -eq 0 || cleanup_failed=1
|
||||
fi
|
||||
test -z "$(docker ps -aq --filter "label=com.docker.compose.project=$project")" || cleanup_failed=1
|
||||
test -z "$(docker volume ls -q --filter "label=com.docker.compose.project=$project")" || cleanup_failed=1
|
||||
test -z "$(docker network ls -q --filter "label=com.docker.compose.project=$project")" || cleanup_failed=1
|
||||
rm -rf "$tmp"
|
||||
test ! -e "$tmp" || cleanup_failed=1
|
||||
if [ "$original_status" -ne 0 ]; then
|
||||
exit "$original_status"
|
||||
fi
|
||||
if [ "$cleanup_failed" -ne 0 ]; then
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
trap cleanup EXIT
|
||||
trap 'exit 129' HUP
|
||||
trap 'exit 130' INT
|
||||
trap 'exit 143' TERM
|
||||
|
||||
mkdir -p "$tmp/source/evidence"
|
||||
printf '%s\n' '# Evidence' 'generation one' >"$tmp/source/evidence/a.md"
|
||||
bundle="$tmp/thothii.secrets"
|
||||
printf '%s\n' 'THT_MODEL_API_KEY=smoke-model-key' >"$bundle"
|
||||
chmod 0600 "$bundle"
|
||||
printf '%s\n' '{}' >"$tmp/pi-auth.json"
|
||||
printf '%s' 'smoke-dwh-password' >"$tmp/dwh-password"
|
||||
chmod 0600 "$tmp/pi-auth.json" "$tmp/dwh-password"
|
||||
printf '%s\n' \
|
||||
'THT_WORKSPACE_GIT_REMOTE=https://git.example.invalid/platform/thoth-workspaces.git' \
|
||||
"PI_AUTH_FILE=$tmp/pi-auth.json" \
|
||||
"THT_SECRETS_FILE=$bundle" >"$tmp/operator.env"
|
||||
|
||||
cat >"$tmp/smoke.yaml" <<YAML
|
||||
services:
|
||||
embedding:
|
||||
image: thothii-core:local
|
||||
profiles: [preprocess]
|
||||
entrypoint: [/opt/venv/bin/python, -c]
|
||||
command:
|
||||
- |
|
||||
import json
|
||||
from http.server import BaseHTTPRequestHandler, HTTPServer
|
||||
class H(BaseHTTPRequestHandler):
|
||||
def do_POST(self):
|
||||
n = len(json.loads(self.rfile.read(int(self.headers["Content-Length"])))["input"])
|
||||
body = json.dumps({"embeddings": [[1.0] + [0.0] * 1023 for _ in range(n)]}).encode()
|
||||
self.send_response(200); self.send_header("Content-Length", str(len(body))); self.end_headers(); self.wfile.write(body)
|
||||
def do_GET(self):
|
||||
body = b'{"models":[{"name":"qwen3-embedding:0.6b"}]}'
|
||||
self.send_response(200); self.send_header("Content-Length", str(len(body))); self.end_headers(); self.wfile.write(body)
|
||||
def log_message(self, *args): pass
|
||||
HTTPServer(("0.0.0.0", 11434), H).serve_forever()
|
||||
embedding-model-init:
|
||||
profiles: [preprocess]
|
||||
image: busybox:1.37.0
|
||||
entrypoint: [sh, -ec]
|
||||
command: ["exit 0"]
|
||||
depends_on:
|
||||
embedding: {condition: service_started}
|
||||
dwh:
|
||||
image: postgres:16-alpine
|
||||
profiles: [preprocess]
|
||||
environment:
|
||||
POSTGRES_DB: warehouse
|
||||
POSTGRES_USER: thoth_reader
|
||||
POSTGRES_PASSWORD: smoke-dwh-password
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U thoth_reader -d warehouse"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 20
|
||||
start_period: 10s
|
||||
preprocess-evidence:
|
||||
volumes:
|
||||
- $tmp/source:/data/source:ro
|
||||
preprocess-dwh:
|
||||
environment:
|
||||
THT_PREPROCESS_DWH_HOST: dwh
|
||||
THT_PREPROCESS_DWH_PORT: "5432"
|
||||
THT_PREPROCESS_DWH_DATABASE: warehouse
|
||||
THT_PREPROCESS_DWH_SCHEMA: public
|
||||
THT_PREPROCESS_DWH_USER: thoth_reader
|
||||
THT_PREPROCESS_DWH_PASSWORD_FILE: /run/secrets/preprocess-dwh-password
|
||||
volumes:
|
||||
- $tmp/dwh-password:/run/secrets/preprocess-dwh-password:ro
|
||||
depends_on:
|
||||
dwh: {condition: service_healthy}
|
||||
YAML
|
||||
|
||||
compose="docker compose --env-file $tmp/operator.env -f compose.yaml -f deploy/compose.preprocess.yaml -f $tmp/smoke.yaml --project-name $project --profile preprocess"
|
||||
$compose build preprocess-evidence
|
||||
if [ "${PREPROCESS_SMOKE_INJECT_FAILURE:-0}" = "1" ]; then
|
||||
sh -c 'exit 97'
|
||||
fi
|
||||
generation_count() {
|
||||
$compose run --rm --no-deps --entrypoint /opt/venv/bin/python preprocess-evidence -c \
|
||||
'import pathlib,re; root=pathlib.Path("/data/workspaces/preprocess-evidence/corpus"); print(sum(1 for p in root.iterdir() if p.is_dir() and re.fullmatch(r"gen-[0-9a-f]{32}",p.name)) if root.exists() else 0)'
|
||||
}
|
||||
before=0
|
||||
first=$($compose run --rm preprocess-evidence)
|
||||
after_first=$(generation_count)
|
||||
second=$($compose run --rm preprocess-evidence)
|
||||
after_second=$(generation_count)
|
||||
printf '%s' 'generation two' >>"$tmp/source/evidence/a.md"
|
||||
third=$($compose run --rm preprocess-evidence)
|
||||
after_third=$(generation_count)
|
||||
dwh=$($compose run --rm preprocess-dwh)
|
||||
python3 - "$first" "$second" "$third" "$before" "$after_first" "$after_second" "$after_third" <<'PY'
|
||||
import json, sys
|
||||
a, b, c = map(json.loads, sys.argv[1:4])
|
||||
before, first_count, second_count, third_count = map(int, sys.argv[4:])
|
||||
assert len(a["changed"]) == 1 and not a["unchanged"]
|
||||
assert len(b["unchanged"]) == 1 and not b["changed"]
|
||||
assert len(c["changed"]) == 1 and c["generation"] != a["generation"]
|
||||
assert a["published"] and not b["published"] and c["published"]
|
||||
assert first_count == before + 1
|
||||
assert second_count == first_count
|
||||
assert third_count == second_count + 1
|
||||
PY
|
||||
python3 - "$dwh" <<'PY'
|
||||
import json, sys
|
||||
assert json.loads(sys.argv[1])["status"] == "succeeded"
|
||||
PY
|
||||
active=$($compose run --rm --no-deps --entrypoint sh preprocess-evidence -c \
|
||||
'cat /data/workspaces/preprocess-evidence/corpus/ACTIVE')
|
||||
python3 - "$third" "$active" <<'PY'
|
||||
import json, sys
|
||||
assert json.loads(sys.argv[1])["generation"] == sys.argv[2].strip()
|
||||
PY
|
||||
echo "real Compose preprocessing unchanged rerun, mutation, DWH job, and ACTIVE publish passed."
|
||||
@@ -17,7 +17,6 @@ targets=(
|
||||
scripts/build-local.sh
|
||||
scripts/build-local.ps1
|
||||
scripts/docker-smoke.sh
|
||||
scripts/preprocess-smoke.sh
|
||||
)
|
||||
|
||||
existing=()
|
||||
|
||||
@@ -14,8 +14,6 @@ new_fixture() {
|
||||
"$fixture/repository/deploy/workspaces" \
|
||||
"$fixture/repository/docker/smoke" \
|
||||
"$fixture/repository/docs/install" \
|
||||
"$fixture/repository/docs/superpowers/specs" \
|
||||
"$fixture/repository/docs/superpowers/plans" \
|
||||
"$fixture/repository/frontend" \
|
||||
"$fixture/repository/scripts"
|
||||
|
||||
@@ -31,11 +29,7 @@ new_fixture() {
|
||||
printf '%s\n' '# qdrant backup helper' >"$fixture/repository/scripts/vector-backup.sh"
|
||||
printf '%s\n' '# qdrant restore helper' >"$fixture/repository/scripts/vector-restore.sh"
|
||||
|
||||
# These are the three intentionally allowed categories from the Task 10 boundary.
|
||||
printf '%s\n' 'historical omics_portal and Chirone record' \
|
||||
>"$fixture/repository/docs/superpowers/plans/legacy.md"
|
||||
printf '%s\n' 'historical pgvector rollout note' \
|
||||
>"$fixture/repository/docs/superpowers/specs/history.md"
|
||||
# These two migration fixtures remain intentionally outside the active runtime surface.
|
||||
printf '%s\n' 'id: generic' >"$fixture/repository/deploy/workspaces/psd.yaml.example"
|
||||
printf '%s\n' '# migrate PSD sessions from /home/chirone' \
|
||||
>"$fixture/repository/docker/session-migrate.sh"
|
||||
|
||||
@@ -128,7 +128,7 @@ scan_category contract-test "$positive_contract" "${contract_test_files[@]}"
|
||||
contract_scan_files=()
|
||||
for file in "${contract_test_files[@]}"; do
|
||||
case "${file#scripts/}" in
|
||||
test-compose-secret-policy.sh|test-preprocess-compose-config.sh) continue ;;
|
||||
test-compose-secret-policy.sh) continue ;;
|
||||
esac
|
||||
contract_scan_files+=("$file")
|
||||
done
|
||||
|
||||
@@ -1,62 +0,0 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
tmp_bundle=$(mktemp)
|
||||
tmp_auth=$(mktemp)
|
||||
tmp_auth_config=$(mktemp -d)
|
||||
trap 'rm -f "$tmp_bundle" "$tmp_auth"; rm -rf "$tmp_auth_config"' EXIT HUP INT TERM
|
||||
printf '%s\n' 'THT_MODEL_API_KEY=test-model' >"$tmp_bundle"
|
||||
chmod 0600 "$tmp_bundle"
|
||||
printf '%s\n' '{}' >"$tmp_auth"
|
||||
chmod 0600 "$tmp_auth"
|
||||
chmod 0700 "$tmp_auth_config"
|
||||
printf '%s\n' 'mode: local' >"$tmp_auth_config/auth.yaml"
|
||||
chmod 0600 "$tmp_auth_config/auth.yaml"
|
||||
export THT_SECRETS_FILE="$tmp_bundle"
|
||||
export PI_AUTH_FILE="$tmp_auth"
|
||||
export THT_AUTH_CONFIG_ROOT="$tmp_auth_config"
|
||||
export THT_WORKSPACE_GIT_REMOTE=https://git.example.invalid/platform/thoth-workspaces.git
|
||||
|
||||
config_json=$(docker compose -f compose.yaml -f deploy/compose.preprocess.yaml --profile preprocess config --format json)
|
||||
|
||||
printf '%s' "$config_json" | python3 -c '
|
||||
import json, sys
|
||||
|
||||
config = json.load(sys.stdin)
|
||||
services = config["services"]
|
||||
assert "qdrant" in services
|
||||
assert "embedding" in services
|
||||
assert "embedding-model-init" in services
|
||||
assert "preprocess-evidence" in services
|
||||
assert "preprocess-dwh" in services
|
||||
for name in ("preprocess-evidence", "preprocess-dwh", "core"):
|
||||
service = services[name]
|
||||
assert any(item.get("target") == "thothii.secrets" for item in service.get("secrets", []) if isinstance(item, dict)), (name, service.get("secrets"))
|
||||
assert "THT_OLLAMA_URL" not in str(service)
|
||||
assert "THT_VECTOR_" not in str(service)
|
||||
assert services["preprocess-evidence"]["depends_on"]["qdrant"]["condition"] == "service_healthy"
|
||||
assert services["preprocess-evidence"]["depends_on"]["embedding-model-init"]["condition"] == "service_completed_successfully"
|
||||
assert "depends_on" not in services["preprocess-dwh"] or "vector-migrate" not in str(services["preprocess-dwh"]["depends_on"])
|
||||
'
|
||||
|
||||
python3 - <<'PY'
|
||||
from pathlib import Path
|
||||
|
||||
evidence = Path("deploy/workspaces/preprocess-evidence.yaml").read_text()
|
||||
dwh = Path("deploy/workspaces/preprocess-dwh.yaml").read_text()
|
||||
assert "type: qdrant" in evidence
|
||||
assert "base_url: http://qdrant:6333" in evidence
|
||||
assert "provider: ollama_internal" in evidence
|
||||
assert "base_url: http://embedding:11434" in evidence
|
||||
assert "qwen3-embedding:0.6b" in evidence
|
||||
assert "THT_VECTOR_" not in evidence
|
||||
assert "THT_OLLAMA_URL" not in evidence
|
||||
assert "pgvector" not in evidence
|
||||
assert "type: postgres_direct" in dwh
|
||||
assert "THT_PREPROCESS_DWH_HOST" in dwh
|
||||
print("preprocess workspace contract: ok")
|
||||
PY
|
||||
|
||||
echo "preprocess compose config: ok"
|
||||
@@ -417,7 +417,7 @@ done
|
||||
|
||||
# YAML that is not a top-level workspace descriptor is not a generic schema-version target.
|
||||
seed_fixture
|
||||
printf '%s\n' 'bundle_schema_version: 1' >"$fixture/deploy/workspaces/preprocess-dwh.yaml"
|
||||
printf '%s\n' 'bundle_schema_version: 1' >"$fixture/deploy/maintenance-job.yaml"
|
||||
commit_fixture unrelated-yaml-version
|
||||
expect_pass "unrelated YAML schema version"
|
||||
|
||||
@@ -469,13 +469,11 @@ write_fixture_descriptor \
|
||||
commit_fixture future-workspace-family
|
||||
expect_rejected "future workspace fixture family" "scripts/fixtures/workspace-registry-future.yaml"
|
||||
|
||||
# Only exact path+category policy literals are allowed; paths outside policy roots remain out of scope.
|
||||
# Only exact path+category policy literals are allowed.
|
||||
seed_fixture
|
||||
mkdir -p "$fixture/docs/superpowers/plans"
|
||||
printf '%s\n' 'Historical schema_version: 2 and migration_required.' >"$fixture/docs/superpowers/plans/history.md"
|
||||
printf '%s\n' 'migration_required migrate-legacy WorkspaceV2 revision.state' >"$fixture/scripts/test-verify-schema-v3-only.sh"
|
||||
commit_fixture exact-policy-allowlist
|
||||
expect_pass "exact self-test policy allowlist and historical docs"
|
||||
expect_pass "exact self-test policy allowlist"
|
||||
|
||||
# Diagnostic paths are shell-escaped so a newline cannot forge another log line.
|
||||
seed_fixture
|
||||
|
||||
@@ -234,7 +234,7 @@ while IFS= read -r -d '' path; do
|
||||
esac
|
||||
kind=""
|
||||
case "$path" in
|
||||
deploy/workspaces/preprocess-dwh.yaml|deploy/workspaces/preprocess-evidence.yaml|deploy/workspaces/server-sessions.yaml.example) ;;
|
||||
deploy/workspaces/server-sessions.yaml.example) ;;
|
||||
deploy/workspaces/*.yaml|deploy/workspaces/*.yml|deploy/workspaces/*.yaml.example|deploy/workspaces/*.yml.example|scripts/fixtures/workspace-registry-*.yaml|scripts/fixtures/workspace-registry-*.yml|scripts/fixtures/*/workspace-registry-*.yaml|scripts/fixtures/*/workspace-registry-*.yml) kind=workspace_descriptor ;;
|
||||
scripts/*.sh|scripts/*.ps1)
|
||||
case "$path" in scripts/verify-schema-v3-only.sh|scripts/test-verify-schema-v3-only.sh) ;; *) kind=deployment_script ;; esac
|
||||
|
||||
Reference in New Issue
Block a user