From 4e3fecbe8edfdf24c79a8bcb84f482aafef77c16 Mon Sep 17 00:00:00 2001 From: mptyl Date: Sat, 8 Aug 2026 19:05:57 +0200 Subject: [PATCH] refactor: retire external vector deployment --- .env.example | 3 - .github/workflows/deployment.yml | 2 + README.md | 35 +- deploy/compose.local-vector.yaml | 90 ---- deploy/compose.preprocess-local-vector.yaml | 14 - deploy/compose.preprocess.yaml | 6 +- deploy/secrets/README.md | 34 +- deploy/secrets/thothii.secrets.example | 10 +- deploy/sql/20-vector-roles.sql | 25 - deploy/vector/reconcile-roles.sh | 54 --- deploy/vector/rotate-bootstrap-password.py | 93 ---- deploy/vector/secret-policy.sh | 95 ---- deploy/vector/vector-db-entrypoint.sh | 9 - deploy/workspaces/preprocess-dwh.yaml | 8 +- deploy/workspaces/preprocess-evidence.yaml | 25 +- docker-compose.dev.yml | 68 ++- .../examples/workspace-bindings.env.example | 6 - docs/install/local-workspace-registry.md | 24 +- docs/install/server-workspace-registry.md | 13 +- docs/installazione-docker-4-contesti.md | 250 ++-------- scripts/compose-with-preflight.sh | 4 + .../generate-connector-secrets-override.sh | 4 + scripts/local-vector-smoke.sh | 446 ------------------ scripts/preprocess-smoke.sh | 63 ++- scripts/secret-file-utils.sh | 32 ++ scripts/task13-runtime-fixture-check.ts | 20 +- scripts/test-compose-secret-policy.sh | 30 +- scripts/test-deployment-command-contract.sh | 2 - .../test-local-vector-smoke-live-collision.sh | 5 - scripts/test-local-vector-smoke-safety.sh | 71 --- scripts/test-no-deployment-coupling-scope.sh | 12 + scripts/test-no-deployment-coupling.sh | 31 +- scripts/test-preprocess-compose-config.sh | 88 ++-- scripts/test-task13-runtime-fixtures.sh | 17 +- scripts/test-vector-backup-restore-safety.sh | 17 - scripts/test-vector-bootstrap-rotation.sh | 125 ----- scripts/test-vector-migration-image.sh | 48 -- scripts/test-vector-secret-policy.sh | 58 --- scripts/unified-deployment-smoke.sh | 29 +- scripts/vector-backup.sh | 2 +- scripts/vector-restore.sh | 2 +- scripts/vector-rotate-bootstrap-password.sh | 89 +--- scripts/verify-container-images.sh | 1 - scripts/verify-workspace-install-docs.sh | 20 +- 44 files changed, 370 insertions(+), 1710 deletions(-) delete mode 100644 deploy/compose.local-vector.yaml delete mode 100644 deploy/compose.preprocess-local-vector.yaml delete mode 100644 deploy/sql/20-vector-roles.sql delete mode 100755 deploy/vector/reconcile-roles.sh delete mode 100755 deploy/vector/rotate-bootstrap-password.py delete mode 100755 deploy/vector/secret-policy.sh delete mode 100755 deploy/vector/vector-db-entrypoint.sh delete mode 100755 scripts/local-vector-smoke.sh create mode 100755 scripts/secret-file-utils.sh delete mode 100755 scripts/test-local-vector-smoke-live-collision.sh delete mode 100755 scripts/test-local-vector-smoke-safety.sh delete mode 100755 scripts/test-vector-bootstrap-rotation.sh delete mode 100755 scripts/test-vector-migration-image.sh delete mode 100755 scripts/test-vector-secret-policy.sh mode change 100755 => 100644 scripts/vector-rotate-bootstrap-password.sh diff --git a/.env.example b/.env.example index 7de6f78a..84a06a00 100644 --- a/.env.example +++ b/.env.example @@ -10,7 +10,4 @@ THT_WORKSPACE_GIT_AUTHOR_EMAIL=thoth-workspace-registry@example.invalid THT_DB_NAME=warehouse THT_DWH_REST_URL=https://dwh.example.invalid -THT_VEC_REST_URL=https://vector.example.invalid -THT_VEC_WRITE_REST_URL=https://vector-write.example.invalid -THT_OLLAMA_URL=https://embeddings.example.invalid THT_LLM_URL=https://llm.example.invalid diff --git a/.github/workflows/deployment.yml b/.github/workflows/deployment.yml index d3b684f3..6c2e2dcf 100644 --- a/.github/workflows/deployment.yml +++ b/.github/workflows/deployment.yml @@ -41,8 +41,10 @@ jobs: - name: Verify Compose and installation contracts run: | bash scripts/test-unified-compose.sh + bash scripts/test-no-deployment-coupling-scope.sh bash scripts/test-compose-secret-policy.sh bash scripts/test-no-deployment-coupling.sh + bash scripts/test-preprocess-compose-config.sh bash scripts/test-verify-workspace-install-docs.sh git diff --check - name: Install backend dependencies diff --git a/README.md b/README.md index 77009767..7e048441 100644 --- a/README.md +++ b/README.md @@ -2,12 +2,12 @@ ThothII is a human-reviewed NL-to-SQL workflow with a React frontend and a Fastify/Pi/`tht` core. The portable deployment runs exactly two application services; data services remain -external in this profile. +external in this profile, except for the mandatory internal semantic services bundled in Compose. ## Docker Compose: local startup -Requirements: Docker Engine with Compose v2. The mandatory stack is exactly the `core` and -`frontend` application images. DWH, vector DB, embedding, and LLM services are external, +Requirements: Docker Engine with Compose v2. The mandatory stack is `frontend`, `core`, +`qdrant`, `embedding`, and the one-shot `embedding-model-init`. DWH and LLM remain external, configurable endpoints—even when they are co-located with ThothII. From a fresh clone, run these commands from the repository root: @@ -42,9 +42,7 @@ runtime endpoint and secret bindings remain installation-local. Open loopback port). Credentials and certificates are local protected files. Do not put them in environment examples, -workspace YAML, URLs, or Compose interpolation values. The optional `local-vector` and -preprocessing overlays are development presets; they do not change the two-service mandatory -stack or the external-endpoint contract. +workspace YAML, URLs, or Compose interpolation values. Application state is split across the named `settings`, `pi-state`, `workspace-registry`, and `sessions` volumes. `docker compose down` keeps them. Only an explicit destructive command such @@ -172,29 +170,20 @@ Startup mode adds bounded image build/two-service health startup, installation-a status, stopped-container-aware ownership checks, and exact cleanup. The ordinary hosted Windows job remains deterministic and does not claim Docker startup. -## Optional local pgvector and recovery - -The local-vector overlay reads `THT_VECTOR_BOOTSTRAP_PASSWORD`, -`THT_VECTOR_MIGRATOR_PASSWORD`, `THT_VECTOR_READER_PASSWORD`, and -`THT_VECTOR_WRITER_PASSWORD` from the same bundle. Its `vector_data` volume is independent of -application state; passwords are selected at runtime and are never passed as URL arguments. - ## Preprocessing jobs and S3 Evidence -The included job workspaces target the optional local-vector profile. Put the four local-vector -password keys in the bundle, set `THT_OLLAMA_URL`, mount Evidence at `/data/source/evidence`, then -run the explicit preprocessing preset: +The included preprocessing services reuse the internal Qdrant/Ollama stack. Mount Evidence at +`/data/source/evidence`, then run the explicit preprocessing preset: ```sh docker compose --env-file deploy/env/local.env \ - -f compose.yaml -f deploy/compose.local.yaml -f deploy/compose.local-vector.yaml \ - -f deploy/compose.preprocess.yaml -f deploy/compose.preprocess-local-vector.yaml \ - --profile local-vector --profile preprocess run --rm preprocess-evidence + -f compose.yaml -f deploy/compose.local.yaml \ + -f deploy/compose.preprocess.yaml --profile preprocess run --rm preprocess-evidence ``` Replace the final service with `preprocess-dwh` when required. The overlay makes each job wait for the vector -database health check, role reconciliation, and a successful migration; no separate database -startup or migration command is required. +service health checks and embedding model initialization; no separate semantic-service startup is +required. S3 Evidence uses the optional `tht[s3]` dependency and canonical `s3://bucket/key` provenance. AWS endpoints are used when no custom URL is supplied. Every custom endpoint is an explicit egress @@ -235,9 +224,7 @@ database in the active cluster cannot bypass the guard. It refuses a non-empty t ``` After restore, run `tht vector migrate --status --json`, adapter health, and a known retrieval -query against the target before changing any deployment endpoint. Never test recovery against the -active `vector_data` volume. `./scripts/local-vector-smoke.sh --backup-restore` performs this drill -with disposable source and target volumes. +query against the target before changing any migration/export endpoint. ## Production trust boundary and secrets diff --git a/deploy/compose.local-vector.yaml b/deploy/compose.local-vector.yaml deleted file mode 100644 index 0bf41337..00000000 --- a/deploy/compose.local-vector.yaml +++ /dev/null @@ -1,90 +0,0 @@ -services: - core: - profiles: [local-vector] - environment: - THT_VECTOR_DATABASE: "${THT_VECTOR_DATABASE:-thoth}" - THT_VECTOR_BOOTSTRAP_USER: "${THT_VECTOR_BOOTSTRAP_USER:-postgres}" - THT_VECTOR_READER_USER: "${THT_VECTOR_READER_USER:-thoth_vector_reader}" - THT_VECTOR_WRITER_USER: "${THT_VECTOR_WRITER_USER:-thoth_vector_writer}" - THT_SECRETS_FILE: /run/secrets/thothii.secrets - secrets: [{source: thothii_secrets, target: thothii.secrets}] - depends_on: - vector-migrate: - condition: service_completed_successfully - - frontend: - profiles: [local-vector] - - vector-db: - image: pgvector/pgvector:0.8.5-pg16@sha256:1d533553fefe4f12e5d80c7b80622ba0c382abb5758856f52983d8789179f0fb - profiles: [local-vector] - labels: {io.thothii.smoke-owner: "${THOTH_SMOKE_OWNER:-operator}"} - environment: - POSTGRES_DB: "${THT_VECTOR_DATABASE:-thoth}" - POSTGRES_USER: "${THT_VECTOR_BOOTSTRAP_USER:-postgres}" - THT_VECTOR_MIGRATOR_USER: "${THT_VECTOR_MIGRATOR_USER:-thoth_vector_migrator}" - THT_VECTOR_READER_USER: "${THT_VECTOR_READER_USER:-thoth_vector_reader}" - THT_VECTOR_WRITER_USER: "${THT_VECTOR_WRITER_USER:-thoth_vector_writer}" - THT_SECRETS_FILE: /run/secrets/thothii.secrets - secrets: [{source: thothii_secrets, target: thothii.secrets}] - entrypoint: [/opt/thoth/vector-db-entrypoint.sh] - volumes: - - vector_data:/var/lib/postgresql/data - - ./deploy/vector/vector-db-entrypoint.sh:/opt/thoth/vector-db-entrypoint.sh:ro - - ./deploy/vector/secret-policy.sh:/opt/thoth/secret-policy.sh:ro - healthcheck: - test: [CMD-SHELL, "pg_isready -U $$POSTGRES_USER -d $$POSTGRES_DB"] - interval: 5s - timeout: 3s - retries: 20 - start_period: 10s - restart: unless-stopped - - vector-reconcile: - image: pgvector/pgvector:0.8.5-pg16@sha256:1d533553fefe4f12e5d80c7b80622ba0c382abb5758856f52983d8789179f0fb - profiles: [local-vector] - labels: {io.thothii.smoke-owner: "${THOTH_SMOKE_OWNER:-operator}"} - environment: - PGHOST: vector-db - PGPORT: 5432 - PGDATABASE: "${THT_VECTOR_DATABASE:-thoth}" - PGUSER: "${THT_VECTOR_BOOTSTRAP_USER:-postgres}" - THT_VECTOR_BOOTSTRAP_USER: "${THT_VECTOR_BOOTSTRAP_USER:-postgres}" - THT_VECTOR_MIGRATOR_USER: "${THT_VECTOR_MIGRATOR_USER:-thoth_vector_migrator}" - THT_VECTOR_READER_USER: "${THT_VECTOR_READER_USER:-thoth_vector_reader}" - THT_VECTOR_WRITER_USER: "${THT_VECTOR_WRITER_USER:-thoth_vector_writer}" - THT_SECRETS_FILE: /run/secrets/thothii.secrets - entrypoint: [/opt/thoth/reconcile-roles.sh] - secrets: [{source: thothii_secrets, target: thothii.secrets}] - volumes: - - ./deploy/vector/reconcile-roles.sh:/opt/thoth/reconcile-roles.sh:ro - - ./deploy/vector/secret-policy.sh:/opt/thoth/secret-policy.sh:ro - depends_on: - vector-db: {condition: service_healthy} - restart: "no" - - vector-migrate: - image: thothii-core:local - profiles: [local-vector] - labels: {io.thothii.smoke-owner: "${THOTH_SMOKE_OWNER:-operator}"} - build: - context: . - dockerfile: docker/core.Dockerfile - entrypoint: [sh, -ec] - command: - - | - . /opt/thoth/secret-policy.sh - export PGPASSWORD=$$(read_bundle_secret /run/secrets/thothii.secrets THT_VECTOR_MIGRATOR_PASSWORD) - exec /opt/venv/bin/tht vector migrate --database-url "postgresql+psycopg2://${THT_VECTOR_MIGRATOR_USER:-thoth_vector_migrator}@vector-db:5432/${THT_VECTOR_DATABASE:-thoth}" --json - secrets: [{source: thothii_secrets, target: thothii.secrets}] - environment: - THT_SECRETS_FILE: /run/secrets/thothii.secrets - volumes: - - ./deploy/vector/secret-policy.sh:/opt/thoth/secret-policy.sh:ro - depends_on: - vector-reconcile: {condition: service_completed_successfully} - restart: "no" - -volumes: - vector_data: - labels: {io.thothii.smoke-owner: "${THOTH_SMOKE_OWNER:-operator}"} diff --git a/deploy/compose.preprocess-local-vector.yaml b/deploy/compose.preprocess-local-vector.yaml deleted file mode 100644 index 8ce48d61..00000000 --- a/deploy/compose.preprocess-local-vector.yaml +++ /dev/null @@ -1,14 +0,0 @@ -services: - preprocess-evidence: - environment: - THT_SECRETS_FILE: /run/secrets/thothii.secrets - secrets: [{source: thothii_secrets, target: thothii.secrets}] - depends_on: - vector-migrate: {condition: service_completed_successfully} - - preprocess-dwh: - environment: - THT_SECRETS_FILE: /run/secrets/thothii.secrets - secrets: [{source: thothii_secrets, target: thothii.secrets}] - depends_on: - vector-migrate: {condition: service_completed_successfully} diff --git a/deploy/compose.preprocess.yaml b/deploy/compose.preprocess.yaml index 46752d43..07b8c0c1 100644 --- a/deploy/compose.preprocess.yaml +++ b/deploy/compose.preprocess.yaml @@ -9,13 +9,17 @@ services: command: ["mkdir -p /data/workspaces/preprocess-evidence && exec /app/docker/core-entrypoint.sh preprocess evidence --json -c /app/harness/workspaces/preprocess-evidence.yaml"] environment: THT_DATA_ROOT: /data - THT_OLLAMA_URL: "${THT_OLLAMA_URL:-http://host.docker.internal:11434}" THT_SECRETS_FILE: /run/secrets/thothii.secrets secrets: [{source: thothii_secrets, target: thothii.secrets}] volumes: - thoth_data:/data - ./deploy/workspaces:/app/harness/workspaces:ro restart: "no" + depends_on: + qdrant: + condition: service_healthy + embedding-model-init: + condition: service_completed_successfully preprocess-dwh: image: thothii-core:local diff --git a/deploy/secrets/README.md b/deploy/secrets/README.md index 09ff7e71..b00bbfd3 100644 --- a/deploy/secrets/README.md +++ b/deploy/secrets/README.md @@ -9,10 +9,9 @@ chmod 600 deploy/secrets/thothii.secrets ``` The file uses strict `KEY=VALUE` lines (comments and blank lines are allowed). The supported -keys are `THT_MODEL_API_KEY`, `THT_DWH_API_KEY`, `THT_VEC_API_KEY`, -`THT_VEC_WRITE_API_KEY`, and the four `THT_VECTOR_*_PASSWORD` role passwords. Values must be -non-empty and contain no whitespace. Do not put secrets in the root `.env`, workspace YAML, -URLs, logs, or rendered Compose output. +keys are `THT_MODEL_API_KEY`, `THT_DWH_API_KEY`, `THT_CA`, `THT_SSL_CA`, and +`PI_PROVIDER_API_KEY`. Values must be non-empty and contain no whitespace. Do not put secrets +in the root `.env`, workspace YAML, URLs, logs, or rendered Compose output. Compose mounts the bundle read-only as `/run/secrets/thothii.secrets`. The host file must be a regular non-symlink file with mode `0600` or `0400`; Docker's normal `0444` mode is accepted @@ -33,29 +32,10 @@ Compose files intentionally do not create this mount. ## Migration from separate secret files Older installations used `THT_*_SECRET_FILE` variables and one file per value. Migrate by -copying each value to its bundle key, validating with the complete base+profile command, and only -then deleting the old files. The old variables remain a compatibility path for staged upgrades, -but the documented and tested default is an absolute `THT_SECRETS_FILE` path to the protected -bundle. - -The local-vector bootstrap rotation helper still accepts an old/new password file as its -maintenance interface. Run it only with files protected by `0600`, then copy the resulting -password into `THT_VECTOR_BOOTSTRAP_PASSWORD` in the bundle before restarting -`vector-reconcile`/the application. The helper never prints password contents. - -The helper has no implicit operator-env default. Pass the same protected env file used for the -deployment explicitly; it must be a readable regular non-symlink file and must not be writable by -group or other users: - -```sh -chmod 600 deploy/env/local.env -./scripts/vector-rotate-bootstrap-password.sh \ - --env-file "$(pwd)/deploy/env/local.env" \ - /secure/thoth/bootstrap-password /secure/thoth/bootstrap-password.next -``` - -Automation may set the narrowly scoped `THT_VECTOR_OPERATOR_ENV_FILE` instead. An explicit -`--env-file` takes precedence. Missing or unsafe env files are rejected before Compose runs. +copying each retained value to its bundle key, validating with the complete base+profile command, +and only then deleting the old files. The old variables remain a compatibility path for staged +upgrades, but the documented and tested default is an absolute `THT_SECRETS_FILE` path to the +protected bundle. Hosted Pi providers must use a single provider key. Compound providers (Bedrock, Azure OpenAI Responses, Cloudflare Workers AI/Gateway) fail closed until a provider-specific credential diff --git a/deploy/secrets/thothii.secrets.example b/deploy/secrets/thothii.secrets.example index c6598060..f6013204 100644 --- a/deploy/secrets/thothii.secrets.example +++ b/deploy/secrets/thothii.secrets.example @@ -5,16 +5,8 @@ # Hosted model provider (single-key providers only). # THT_MODEL_API_KEY=replace-me -# External DWH and vector adapters. +# External DWH adapter. # THT_DWH_API_KEY=replace-me -# THT_VEC_API_KEY=replace-me -# THT_VEC_WRITE_API_KEY=replace-me - -# Optional local-vector roles. -# THT_VECTOR_BOOTSTRAP_PASSWORD=replace-me -# THT_VECTOR_MIGRATOR_PASSWORD=replace-me -# THT_VECTOR_READER_PASSWORD=replace-me -# THT_VECTOR_WRITER_PASSWORD=replace-me # Optional CA material/path understood by the configured adapter. # THT_CA=/run/secrets/ca-chain.pem diff --git a/deploy/sql/20-vector-roles.sql b/deploy/sql/20-vector-roles.sql deleted file mode 100644 index 9d4d5730..00000000 --- a/deploy/sql/20-vector-roles.sql +++ /dev/null @@ -1,25 +0,0 @@ --- ThothII — ruolo vector read+write (schema vectors). --- Stessa istanza del DWH (porta 5438). ThothII indicizza (write) + ricerca (read) direttamente. --- La separazione reader/writer resta rilevante solo per il path REST (non usato in Profile A). --- psql -h localhost -p 5438 -U postgres -d postgres -v PWD='' -f 20-vector-roles.sql -DO $$ -BEGIN - IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'thoth_vector_rw') THEN - CREATE ROLE thoth_vector_rw LOGIN; - END IF; -END $$; --- :'PWD' va fuori dal DO (psql non interpola nelle stringhe dollar-quoted) -ALTER ROLE thoth_vector_rw PASSWORD :'PWD'; - -CREATE SCHEMA IF NOT EXISTS vectors; - --- L'estensione pgvector deve esistere (già presente nell'istanza di produzione). --- CREATE EXTENSION IF NOT EXISTS vector; - -GRANT USAGE, CREATE ON SCHEMA vectors TO thoth_vector_rw; -GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA vectors TO thoth_vector_rw; -GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA vectors TO thoth_vector_rw; -ALTER DEFAULT PRIVILEGES IN SCHEMA vectors - GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO thoth_vector_rw; -ALTER DEFAULT PRIVILEGES IN SCHEMA vectors - GRANT USAGE, SELECT ON SEQUENCES TO thoth_vector_rw; diff --git a/deploy/vector/reconcile-roles.sh b/deploy/vector/reconcile-roles.sh deleted file mode 100755 index 74f14afe..00000000 --- a/deploy/vector/reconcile-roles.sh +++ /dev/null @@ -1,54 +0,0 @@ -#!/bin/sh -set -eu - -. /opt/thoth/secret-policy.sh - -bundle=${THT_SECRETS_FILE:-/run/secrets/thothii.secrets} -export PGPASSWORD=$(read_bundle_secret "$bundle" THT_VECTOR_BOOTSTRAP_PASSWORD) -migrator_password=$(read_bundle_secret "$bundle" THT_VECTOR_MIGRATOR_PASSWORD) -reader_password=$(read_bundle_secret "$bundle" THT_VECTOR_READER_PASSWORD) -writer_password=$(read_bundle_secret "$bundle" THT_VECTOR_WRITER_PASSWORD) - -psql --set=ON_ERROR_STOP=1 \ - --set=migrator_user="$THT_VECTOR_MIGRATOR_USER" \ - --set=migrator_password="$migrator_password" \ - --set=reader_user="$THT_VECTOR_READER_USER" \ - --set=reader_password="$reader_password" \ - --set=writer_user="$THT_VECTOR_WRITER_USER" \ - --set=writer_password="$writer_password" <<'SQL' -SELECT 'CREATE ROLE vector_reader NOLOGIN' -WHERE NOT EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = 'vector_reader') \gexec -SELECT 'CREATE ROLE vector_writer NOLOGIN' -WHERE NOT EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = 'vector_writer') \gexec -ALTER ROLE vector_reader NOLOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION; -ALTER ROLE vector_writer NOLOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION; - -SELECT format('CREATE ROLE %I LOGIN', :'migrator_user') -WHERE NOT EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = :'migrator_user') \gexec -SELECT format('CREATE ROLE %I LOGIN', :'reader_user') -WHERE NOT EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = :'reader_user') \gexec -SELECT format('CREATE ROLE %I LOGIN', :'writer_user') -WHERE NOT EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = :'writer_user') \gexec - -SELECT format( - 'ALTER ROLE %I LOGIN PASSWORD %L NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION', - :'migrator_user', :'migrator_password' -) \gexec -SELECT format( - 'ALTER ROLE %I LOGIN PASSWORD %L NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION', - :'reader_user', :'reader_password' -) \gexec -SELECT format( - 'ALTER ROLE %I LOGIN PASSWORD %L NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION', - :'writer_user', :'writer_password' -) \gexec - -SELECT format('GRANT vector_reader TO %I', :'reader_user') \gexec -SELECT format('GRANT vector_writer TO %I', :'writer_user') \gexec -SELECT format('ALTER DATABASE %I OWNER TO %I', current_database(), :'migrator_user') \gexec -SELECT format('CREATE SCHEMA IF NOT EXISTS vectors AUTHORIZATION %I', :'migrator_user') \gexec -SELECT format('ALTER SCHEMA vectors OWNER TO %I', :'migrator_user') \gexec -REVOKE ALL ON SCHEMA vectors FROM PUBLIC; -GRANT USAGE ON SCHEMA vectors TO vector_reader, vector_writer; -CREATE EXTENSION IF NOT EXISTS vector WITH SCHEMA vectors; -SQL diff --git a/deploy/vector/rotate-bootstrap-password.py b/deploy/vector/rotate-bootstrap-password.py deleted file mode 100755 index 042c49d7..00000000 --- a/deploy/vector/rotate-bootstrap-password.py +++ /dev/null @@ -1,93 +0,0 @@ -#!/usr/bin/env python3 -"""Rotate the initialized PostgreSQL bootstrap role and verify before returning success.""" - -from __future__ import annotations - -import os -import sys -from pathlib import Path - -import psycopg2 -from psycopg2 import sql - - -def read_secret(path: str) -> str: - value = Path(path).read_text() - if not value or "\x00" in value or any(character.isspace() for character in value): - raise ValueError("secret must be non-empty and contain no whitespace or NUL bytes") - return value - - -def connect(password: str): - return psycopg2.connect( - host=os.environ.get("THT_VECTOR_HOST", "vector-db"), - port=int(os.environ.get("THT_VECTOR_PORT", "5432")), - dbname=os.environ.get("THT_VECTOR_DATABASE", "thoth"), - user=os.environ.get("THT_VECTOR_BOOTSTRAP_USER", "postgres"), - password=password, - connect_timeout=5, - ) - - -def alter_current_role(connection, password: str) -> None: - with connection.cursor() as cursor: - cursor.execute("SELECT current_user") - current_user = cursor.fetchone()[0] - expected = os.environ.get("THT_VECTOR_BOOTSTRAP_USER", "postgres") - if current_user != expected: - raise RuntimeError("authenticated role does not match THT_VECTOR_BOOTSTRAP_USER") - cursor.execute( - sql.SQL("ALTER ROLE {} PASSWORD {}").format( - sql.Identifier(current_user), sql.Literal(password) - ) - ) - connection.commit() - - -def main() -> int: - if len(sys.argv) != 3: - print("usage: rotate-bootstrap-password.py OLD_SECRET NEW_SECRET", file=sys.stderr) - return 2 - try: - old_password = read_secret(sys.argv[1]) - new_password = read_secret(sys.argv[2]) - if old_password == new_password: - raise ValueError("old and new bootstrap passwords must differ") - old_connection = connect(old_password) - except Exception as exc: - print(f"bootstrap rotation refused before change: {type(exc).__name__}", file=sys.stderr) - return 1 - - try: - alter_current_role(old_connection, new_password) - try: - verification = connect(new_password) - verification.close() - except Exception as verify_exc: - try: - alter_current_role(old_connection, old_password) - except Exception as restore_exc: - print( - "bootstrap rotation verification failed and password restore failed: " - f"{type(verify_exc).__name__}/{type(restore_exc).__name__}", - file=sys.stderr, - ) - return 3 - print( - f"bootstrap rotation verification failed; old password restored: " - f"{type(verify_exc).__name__}", - file=sys.stderr, - ) - return 1 - except Exception as exc: - print(f"bootstrap rotation failed: {type(exc).__name__}", file=sys.stderr) - return 1 - finally: - old_connection.close() - - print("bootstrap database password rotated and new login verified") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/deploy/vector/secret-policy.sh b/deploy/vector/secret-policy.sh deleted file mode 100755 index 01de8eab..00000000 --- a/deploy/vector/secret-policy.sh +++ /dev/null @@ -1,95 +0,0 @@ -#!/bin/sh - -validate_secret_file() { - secret_path=$1 - secret_name=$2 - if [ -L "$secret_path" ] || [ ! -f "$secret_path" ] || [ ! -r "$secret_path" ] || [ ! -s "$secret_path" ]; then - echo "$secret_name must be a readable, non-empty regular file" >&2 - return 2 - fi - if LC_ALL=C grep -q '[[:space:]]' "$secret_path"; then - echo "$secret_name must contain no whitespace" >&2 - return 2 - fi - mode=$(stat -c '%a' "$secret_path" 2>/dev/null || stat -f '%Lp' "$secret_path" 2>/dev/null) || return 2 - case "$secret_path:$mode" in - /run/secrets/*:444|/run/secrets/*:400|/run/secrets/*:600|*:600|*:400) ;; - *) echo "$secret_name must have mode 0600 or stricter (Docker secrets may be 0444)" >&2; return 2 ;; - esac -} - -read_secret_file() { - validate_secret_file "$1" "$2" || return - cat "$1" -} - -# Validate the bundle without printing any value. Keep this parser aligned with -# the backend loader: comments/blank lines are allowed, while syntax, allowlist, -# duplicates, empty values, file size, and line size are fail-closed. -validate_bundle() { - bundle_path=$1 - if [ -L "$bundle_path" ] || [ ! -f "$bundle_path" ] || [ ! -r "$bundle_path" ] || [ ! -s "$bundle_path" ]; then - echo "secret bundle must be a readable, non-empty regular file" >&2 - return 2 - fi - mode=$(stat -c '%a' "$bundle_path" 2>/dev/null || stat -f '%Lp' "$bundle_path" 2>/dev/null) || return 2 - case "$bundle_path:$mode" in - /run/secrets/*:444|/run/secrets/*:400|/run/secrets/*:600|*:600|*:400) ;; - *) echo "secret bundle must have mode 0600 or stricter (Docker secrets may be 0444)" >&2; return 2 ;; - esac - size=$(stat -c '%s' "$bundle_path" 2>/dev/null || stat -f '%z' "$bundle_path" 2>/dev/null) || return 2 - if [ "$size" -gt 65536 ]; then - echo "secret bundle exceeds the 64KiB limit" >&2 - return 2 - fi - awk ' - { sub(/\r$/, "", $0) } - length($0) > 16384 { exit 9 } - /^[[:space:]]*$/ || /^[[:space:]]*#/ { next } - /^[A-Z][A-Z0-9_]*=/ { - key=$0; sub(/=.*/, "", key) - val=$0; sub(/^[^=]*=/, "", val) - if (key !~ /^(THT_MODEL_API_KEY|THT_DWH_API_KEY|THT_VEC_API_KEY|THT_VEC_WRITE_API_KEY|THT_CA|THT_SSL_CA|THT_VECTOR_BOOTSTRAP_PASSWORD|THT_VECTOR_MIGRATOR_PASSWORD|THT_VECTOR_READER_PASSWORD|THT_VECTOR_WRITER_PASSWORD|PI_PROVIDER_API_KEY)$/) exit 6 - if (val == "" || ++seen[key] > 1) exit 7 - next - } - { exit 4 } - ' "$bundle_path" || { - echo "secret bundle syntax is invalid" >&2 - return 2 - } -} - -# Read one value from the deployment bundle without putting the bundle itself in -# a service environment. Values selected for credentials must contain no spaces. -read_bundle_secret() { - bundle_path=$1 - bundle_key=$2 - validate_bundle "$bundle_path" || return - case "$bundle_key" in - THT_[A-Z0-9_]*|PI_PROVIDER_API_KEY) ;; - *) echo "invalid secret bundle key" >&2; return 2 ;; - esac - value=$(awk -v wanted="$bundle_key" ' - { sub(/\r$/, "", $0) } - /^[[:space:]]*$/ || /^[[:space:]]*#/ { next } - /^[A-Z][A-Z0-9_]*=/ { - key=$0; sub(/=.*/, "", key) - val=$0; sub(/^[^=]*=/, "", val) - if (key == wanted) { - found=1; print val - } - next - } - { exit 4 } - END { if (!found) exit 5 } - ' "$bundle_path") || { - echo "$bundle_key is unavailable in secret bundle" >&2 - return 3 - } - if [ -z "$value" ] || printf '%s' "$value" | LC_ALL=C grep -q '[[:space:]]'; then - echo "$bundle_key must contain no whitespace" >&2 - return 2 - fi - printf '%s' "$value" -} diff --git a/deploy/vector/vector-db-entrypoint.sh b/deploy/vector/vector-db-entrypoint.sh deleted file mode 100755 index 75cd0e40..00000000 --- a/deploy/vector/vector-db-entrypoint.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/sh -set -eu - -. /opt/thoth/secret-policy.sh - -bundle=${THT_SECRETS_FILE:-/run/secrets/thothii.secrets} -export POSTGRES_PASSWORD=$(read_bundle_secret "$bundle" THT_VECTOR_BOOTSTRAP_PASSWORD) -unset THT_SECRETS_FILE -exec /usr/local/bin/docker-entrypoint.sh postgres diff --git a/deploy/workspaces/preprocess-dwh.yaml b/deploy/workspaces/preprocess-dwh.yaml index 59079e7c..d2f3edff 100644 --- a/deploy/workspaces/preprocess-dwh.yaml +++ b/deploy/workspaces/preprocess-dwh.yaml @@ -2,6 +2,10 @@ language: en dwh: type: postgres_direct connection: - {host: vector-db, database: thoth, schema: vectors, user: thoth_vector_reader, - password_file: "${THT_VECTOR_READER_PASSWORD_FILE}"} + host: "${THT_PREPROCESS_DWH_HOST:-dwh}" + port: "${THT_PREPROCESS_DWH_PORT:-5432}" + database: "${THT_PREPROCESS_DWH_DATABASE:-warehouse}" + schema: "${THT_PREPROCESS_DWH_SCHEMA:-public}" + user: "${THT_PREPROCESS_DWH_USER:-thoth_reader}" + password_file: "${THT_PREPROCESS_DWH_PASSWORD_FILE:-/run/secrets/preprocess-dwh-password}" roots: {artifacts: artifacts, indexes: indexes, sessions: sessions} diff --git a/deploy/workspaces/preprocess-evidence.yaml b/deploy/workspaces/preprocess-evidence.yaml index 638f547d..444b4844 100644 --- a/deploy/workspaces/preprocess-evidence.yaml +++ b/deploy/workspaces/preprocess-evidence.yaml @@ -1,15 +1,22 @@ language: en dwh: type: postgres_direct - connection: {host: unused, database: unused, schema: public, user: unused, password: unused} + connection: + host: "${THT_PREPROCESS_DWH_HOST:-unused}" + port: "${THT_PREPROCESS_DWH_PORT:-5432}" + database: "${THT_PREPROCESS_DWH_DATABASE:-unused}" + schema: "${THT_PREPROCESS_DWH_SCHEMA:-public}" + user: "${THT_PREPROCESS_DWH_USER:-unused}" + password_file: "${THT_PREPROCESS_DWH_PASSWORD_FILE:-/run/secrets/preprocess-dwh-password}" vectors: - type: pgvector_direct - reader: - {host: vector-db, database: thoth, schema: vectors, user: thoth_vector_reader, - password_file: "${THT_VECTOR_READER_PASSWORD_FILE}"} - writer: - {host: vector-db, database: thoth, schema: vectors, user: thoth_vector_writer, - password_file: "${THT_VECTOR_WRITER_PASSWORD_FILE}"} + type: qdrant + base_url: http://qdrant:6333 + collection: preprocess-evidence roots: {artifacts: artifacts, indexes: indexes, sessions: sessions} evidence: {source_root: /data/source, evidence_dir: evidence} -embeddings: {base_url: "${THT_OLLAMA_URL}", model: smoke, dim: 768, batch_size: 32} +embeddings: + provider: ollama_internal + base_url: http://embedding:11434 + model: qwen3-embedding:0.6b + dim: 1024 + batch_size: 32 diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 8803e539..8cc1c85f 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -32,10 +32,11 @@ services: THT_SECRETS_FILE: /run/secrets/thothii.secrets THT_DB_NAME: ${THT_DB_NAME:-} THT_DWH_REST_URL: ${THT_DWH_REST_URL:-} - THT_VEC_REST_URL: ${THT_VEC_REST_URL:-} - THT_VEC_WRITE_REST_URL: ${THT_VEC_WRITE_REST_URL:-} - THT_OLLAMA_URL: ${THT_OLLAMA_URL:-} THT_LLM_URL: ${THT_LLM_URL:-} + THT_INTERNAL_QDRANT_URL: http://qdrant:6333 + THT_INTERNAL_EMBEDDING_URL: http://embedding:11434 + THT_INTERNAL_EMBEDDING_MODEL: qwen3-embedding:0.6b + THT_INTERNAL_EMBEDDING_DIMENSIONS: "1024" MAX_PI_PROCESSES: ${MAX_PI_PROCESSES:-4} extra_hosts: - "host.docker.internal:host-gateway" @@ -59,6 +60,11 @@ services: retries: 5 start_period: 30s restart: "no" + depends_on: + qdrant: + condition: service_healthy + embedding-model-init: + condition: service_completed_successfully networks: [thothii-net] frontend: @@ -77,6 +83,60 @@ services: restart: "no" networks: [thothii-net] + qdrant: + image: qdrant/qdrant:v1.18.2@sha256:75eab8c4ba42096724fdcfde8b4de0b5713d529dde32f285a1f86fdcb2c9e50c + expose: + - "6333" + volumes: + - qdrant-data:/qdrant/storage + healthcheck: + test: + - CMD-SHELL + - > + /usr/bin/bash -lc "exec 3<>/dev/tcp/127.0.0.1/6333 && + printf 'GET /healthz HTTP/1.1\r\nHost: 127.0.0.1\r\nConnection: close\r\n\r\n' >&3 && + grep -q '200 OK' <&3" + interval: 15s + timeout: 3s + retries: 10 + start_period: 10s + networks: [thothii-net] + + embedding: + image: ollama/ollama:0.32.0@sha256:57f573b47f1f71ebb445789f279fe3e596a8beab182f7cf486db9205bad87c5a + command: ["serve"] + expose: + - "11434" + volumes: + - embedding-models:/root/.ollama + healthcheck: + test: + - CMD-SHELL + - > + /usr/bin/bash -lc "exec 3<>/dev/tcp/127.0.0.1/11434 && + printf 'GET /api/tags HTTP/1.1\r\nHost: 127.0.0.1\r\nConnection: close\r\n\r\n' >&3 && + grep -q '200 OK' <&3" + interval: 15s + timeout: 5s + retries: 20 + start_period: 10s + networks: [thothii-net] + + embedding-model-init: + image: ollama/ollama:0.32.0@sha256:57f573b47f1f71ebb445789f279fe3e596a8beab182f7cf486db9205bad87c5a + entrypoint: ["/usr/bin/bash", "/opt/thoth/embedding-model-init.sh"] + environment: + OLLAMA_BASE_URL: http://embedding:11434 + OLLAMA_MODEL: qwen3-embedding:0.6b + OLLAMA_WAIT_TIMEOUT_SEC: "180" + volumes: + - embedding-models:/root/.ollama + - ./docker/embedding-model-init.sh:/opt/thoth/embedding-model-init.sh:ro + depends_on: + embedding: + condition: service_healthy + networks: [thothii-net] + networks: thothii-net: driver: bridge @@ -85,6 +145,8 @@ volumes: dev-data: dev-pi-state: workspace-registry: + qdrant-data: + embedding-models: secrets: thothii_secrets: diff --git a/docs/install/examples/workspace-bindings.env.example b/docs/install/examples/workspace-bindings.env.example index 6b6a8263..770a76cf 100644 --- a/docs/install/examples/workspace-bindings.env.example +++ b/docs/install/examples/workspace-bindings.env.example @@ -5,9 +5,3 @@ THT_WS_NORTH_STAR_RESEARCH_DWH_HOST=dwh.internal.example THT_WS_NORTH_STAR_RESEARCH_DWH_PORT=5432 THT_WS_NORTH_STAR_RESEARCH_DWH_USER=thoth_reader THT_WS_NORTH_STAR_RESEARCH_DWH_PASSWORD_FILE=/run/secrets/north-star-research-dwh-password -THT_WS_NORTH_STAR_RESEARCH_VECTOR_TRANSPORT=pgvector_direct -THT_WS_NORTH_STAR_RESEARCH_VECTOR_HOST=vector.internal.example -THT_WS_NORTH_STAR_RESEARCH_VECTOR_PORT=5432 -THT_WS_NORTH_STAR_RESEARCH_VECTOR_USER=thoth_vector_reader -THT_WS_NORTH_STAR_RESEARCH_VECTOR_PASSWORD_FILE=/run/secrets/north-star-research-vector-password -THT_WS_NORTH_STAR_RESEARCH_EMBEDDING_BASE_URL=https://embeddings.internal.example diff --git a/docs/install/local-workspace-registry.md b/docs/install/local-workspace-registry.md index d77b38c1..5bf8b306 100644 --- a/docs/install/local-workspace-registry.md +++ b/docs/install/local-workspace-registry.md @@ -80,30 +80,21 @@ is `THT_WS___`. Copy [the bindings env example](examples/workspace-bindings.env.example) to an untracked operator file and set its absolute path as `THT_WORKSPACE_BINDINGS_ENV_FILE`. It is loaded only into `core`. Credentials and certificates use `*_FILE` path variables that must point inside `/run/secrets`. -If declared, `THT_WS_NORTH_STAR_RESEARCH_VECTOR_WRITER_API_KEY_FILE` is distinct from the vector reader -file; a reader credential is never repurposed for writing. ## Direct PostgreSQL, REST, and SSH tunnel bindings Set only fields for the selected transport in the dedicated bindings env file. Canonical YAML keeps -database/schema/collection, distance, embedding model, and dimensions shared in Git. Every -`*_FILE=/run/secrets/` binding needs one matching host-only `*_SOURCE` path in operator -`.env`. Generate the untracked connector override from those two files during bootstrap; do not -copy or maintain a workspace-specific Compose override. +database/schema shared in Git. Every `*_FILE=/run/secrets/` binding needs one matching +host-only `*_SOURCE` path in operator `.env`. Generate the untracked connector override from those +two files during bootstrap; do not copy or maintain a workspace-specific Compose override. ```dotenv -# Direct PostgreSQL and pgvector +# Direct PostgreSQL THT_WS_NORTH_STAR_RESEARCH_DWH_TRANSPORT=postgres_direct THT_WS_NORTH_STAR_RESEARCH_DWH_HOST=dwh.example.invalid THT_WS_NORTH_STAR_RESEARCH_DWH_PORT=5432 THT_WS_NORTH_STAR_RESEARCH_DWH_USER=thoth_reader THT_WS_NORTH_STAR_RESEARCH_DWH_PASSWORD_FILE=/run/secrets/north-star-research-dwh-password -THT_WS_NORTH_STAR_RESEARCH_VECTOR_TRANSPORT=pgvector_direct -THT_WS_NORTH_STAR_RESEARCH_VECTOR_HOST=vector.example.invalid -THT_WS_NORTH_STAR_RESEARCH_VECTOR_PORT=5432 -THT_WS_NORTH_STAR_RESEARCH_VECTOR_USER=thoth_vector_reader -THT_WS_NORTH_STAR_RESEARCH_VECTOR_PASSWORD_FILE=/run/secrets/north-star-research-vector-password -THT_WS_NORTH_STAR_RESEARCH_EMBEDDING_BASE_URL=https://embeddings.example.invalid ``` ```dotenv @@ -111,9 +102,6 @@ THT_WS_NORTH_STAR_RESEARCH_EMBEDDING_BASE_URL=https://embeddings.example.invalid THT_WS_NORTH_STAR_RESEARCH_DWH_TRANSPORT=rest_api THT_WS_NORTH_STAR_RESEARCH_DWH_BASE_URL=https://dwh.example.invalid THT_WS_NORTH_STAR_RESEARCH_DWH_API_KEY_FILE=/run/secrets/north-star-research-dwh-api-key -THT_WS_NORTH_STAR_RESEARCH_VECTOR_TRANSPORT=rest_api -THT_WS_NORTH_STAR_RESEARCH_VECTOR_BASE_URL=https://vectors.example.invalid -THT_WS_NORTH_STAR_RESEARCH_VECTOR_API_KEY_FILE=/run/secrets/north-star-research-vector-api-key ``` ```dotenv @@ -130,8 +118,8 @@ THT_WS_NORTH_STAR_RESEARCH_DWH_SSH_TARGET_HOST=dwh.internal.example THT_WS_NORTH_STAR_RESEARCH_DWH_SSH_TARGET_PORT=5432 ``` -Repeat the SSH names for `VECTOR` where needed. REST diagnostics reject a private per-request CA -rather than weakening TLS; use runtime-trusted HTTPS or verified direct/SSH native TLS. See the +REST diagnostics reject a private per-request CA rather than weakening TLS; use runtime-trusted +HTTPS or verified direct/SSH native TLS. See the [diagnostic protocol](../workspace-diagnostic-protocol.md). An SSH connector can prove installation reachability, host-key verification, authentication, and diff --git a/docs/install/server-workspace-registry.md b/docs/install/server-workspace-registry.md index b07a831e..99e47e98 100644 --- a/docs/install/server-workspace-registry.md +++ b/docs/install/server-workspace-registry.md @@ -109,17 +109,12 @@ Select only a transport allowed by canonical YAML; preserve database/schema/coll dimensions, and distance as Git-shared identity. ```dotenv -# Direct PostgreSQL/pgvector with verified native TLS if a CA path is supplied. +# Direct PostgreSQL with verified native TLS if a CA path is supplied. THT_WS_NORTH_STAR_RESEARCH_DWH_TRANSPORT=postgres_direct THT_WS_NORTH_STAR_RESEARCH_DWH_HOST=dwh.internal.example THT_WS_NORTH_STAR_RESEARCH_DWH_PORT=5432 THT_WS_NORTH_STAR_RESEARCH_DWH_USER=thoth_reader THT_WS_NORTH_STAR_RESEARCH_DWH_PASSWORD_FILE=/run/secrets/north-star-research-dwh-password -THT_WS_NORTH_STAR_RESEARCH_VECTOR_TRANSPORT=pgvector_direct -THT_WS_NORTH_STAR_RESEARCH_VECTOR_HOST=vector.internal.example -THT_WS_NORTH_STAR_RESEARCH_VECTOR_PORT=5432 -THT_WS_NORTH_STAR_RESEARCH_VECTOR_USER=thoth_vector_reader -THT_WS_NORTH_STAR_RESEARCH_VECTOR_PASSWORD_FILE=/run/secrets/north-star-research-vector-password ``` ```dotenv @@ -127,10 +122,6 @@ THT_WS_NORTH_STAR_RESEARCH_VECTOR_PASSWORD_FILE=/run/secrets/north-star-research THT_WS_NORTH_STAR_RESEARCH_DWH_TRANSPORT=rest_api THT_WS_NORTH_STAR_RESEARCH_DWH_BASE_URL=https://dwh.internal.example THT_WS_NORTH_STAR_RESEARCH_DWH_API_KEY_FILE=/run/secrets/north-star-research-dwh-api-key -THT_WS_NORTH_STAR_RESEARCH_VECTOR_TRANSPORT=rest_api -THT_WS_NORTH_STAR_RESEARCH_VECTOR_BASE_URL=https://vectors.internal.example -THT_WS_NORTH_STAR_RESEARCH_VECTOR_API_KEY_FILE=/run/secrets/north-star-research-vector-api-key -THT_WS_NORTH_STAR_RESEARCH_EMBEDDING_BASE_URL=https://embeddings.internal.example ``` ```dotenv @@ -147,7 +138,7 @@ THT_WS_NORTH_STAR_RESEARCH_DWH_SSH_TARGET_HOST=dwh.internal.example THT_WS_NORTH_STAR_RESEARCH_DWH_SSH_TARGET_PORT=5432 ``` -Repeat SSH variables for `VECTOR` when selected. REST diagnostics refuse private per-request CAs +REST diagnostics refuse private per-request CAs rather than disable verification; use runtime-trusted HTTPS or verified direct/SSH native TLS. See the [diagnostic protocol](../workspace-diagnostic-protocol.md) for its read-only checks and optional reversible writer probe. diff --git a/docs/installazione-docker-4-contesti.md b/docs/installazione-docker-4-contesti.md index 0c82be2d..2021b02c 100644 --- a/docs/installazione-docker-4-contesti.md +++ b/docs/installazione-docker-4-contesti.md @@ -1,250 +1,78 @@ -# Installazione Docker nei quattro contesti operativi +# Installazione Docker nei contesti operativi correnti -ThothII viene distribuito con due immagini applicative: +ThothII usa una topologia Compose unica: -- `thothii-core`: backend Fastify, harness `tht` e Pi; -- `thothii-frontend`: frontend React servito da nginx. +- `frontend` +- `core` +- `qdrant` +- `embedding` +- `embedding-model-init` -PostgreSQL/pgvector, DWH ed Evidence restano esterni nel profilo predefinito. Il profilo opzionale `local-vector` avvia PostgreSQL/pgvector nel progetto Compose. +Qdrant e Ollama embedding sono servizi interni obbligatori del progetto Compose. Restano +esterni solo DWH e LLM. -## Installazione comune (il comando standard) - -Servono Docker Engine/Compose v2 su Linux oppure Docker Desktop su macOS/Windows. Dalla directory in cui si vuole conservare il clone: +## Comando standard locale ```sh -git clone ThothII -cd ThothII cp deploy/env/local.env.example deploy/env/local.env cp deploy/secrets/thothii.secrets.example deploy/secrets/thothii.secrets chmod 600 deploy/secrets/thothii.secrets -``` -Modificare **solo** questi file interni al clone: - -| File | Cosa contiene | -|---|---| -| `deploy/env/local.env` | endpoint, database e path Pi locali; mai password/token | -| file protetti locali | credenziali e certificati, indicati dai binding del workspace | -| `deploy/workspaces/.yaml` | adapter, endpoint non riservati, `roots` ed Evidence | - -Compilare `deploy/env/local.env`, inclusi i path assoluti `PI_AUTH_FILE` e -`THT_SECRETS_FILE`, con gli endpoint esterni. L'avvio -normale usa esplicitamente il file base e l'overlay locale: - -```sh docker compose --env-file deploy/env/local.env \ -f compose.yaml -f deploy/compose.local.yaml up --build -d ``` -Verificare lo stato con lo stesso comando Compose e aprire . Il core -include Pi; il binario Pi non deve essere installato sull'host. `docker compose down` conserva i -volumi; usare `down --volumes` solo per un ambiente effimero. +Compilare `deploy/env/local.env` con: -### Formato del bundle unico +- `PI_AUTH_FILE` +- `THT_SECRETS_FILE` +- `THT_WORKSPACE_GIT_REMOTE` +- endpoint DWH +- endpoint LLM -`deploy/secrets/thothii.secrets` è un file di testo locale, non uno script shell. Sono ammessi commenti e righe vuote; ogni altra riga deve essere una sola assegnazione senza spazi: +Non inserire secret nel file `.env`. I secret runtime stanno nel bundle +`deploy/secrets/thothii.secrets`. + +## Bundle dei secret + +Le chiavi documentate e supportate nel bundle sono: ```dotenv THT_MODEL_API_KEY=... THT_DWH_API_KEY=... -THT_VEC_API_KEY=... -THT_VEC_WRITE_API_KEY=... -THT_VECTOR_BOOTSTRAP_PASSWORD=... -THT_VECTOR_MIGRATOR_PASSWORD=... -THT_VECTOR_READER_PASSWORD=... -THT_VECTOR_WRITER_PASSWORD=... ``` -Inserire solo le chiavi necessarie al profilo scelto. Il bundle viene montato in sola lettura nel container come `/run/secrets/thothii.secrets`; il parser rifiuta duplicati, chiavi sconosciute, valori vuoti, symlink e permessi host troppo aperti. Non inserire secret in `.env`, nei workspace, negli URL o nell'output Compose renderizzato. +Una CA privata PEM resta esterna al bundle e va montata con un override Compose revisionato. -Una catena CA PEM **non può essere inserita nel bundle**: contiene whitespace e viene rifiutata dal parser. Se un endpoint usa una CA privata, conservarla nel secret manager/host e aggiungere un override Compose revisionato che monti il file in `/run/secrets/ca-chain.pem` e imposti `THT_SSL_CA` (o il parametro dell'adapter). Il clone base non crea quel mount: questa è una limitazione intenzionale da considerare in fase di deployment. +## Preprocessing -### Overlay opzionali espliciti - -DWH/vector/embedding remoti restano endpoint del file locale o server. Per il solo preset di -sviluppo pgvector, aggiungere `-f deploy/compose.local-vector.yaml --profile local-vector` al -comando base. Per il preprocessing aggiungere anche -`-f deploy/compose.preprocess.yaml -f deploy/compose.preprocess-local-vector.yaml --profile preprocess`, -poi ripetere l'intero comando base con l'azione `run --rm preprocess-evidence` oppure -`run --rm preprocess-dwh`. - -## Workspace, adapter e Evidence - -Il workspace YAML seleziona il trasporto disponibile. Esempio DWH REST e vector DB HTTP: - -```yaml -language: en -dwh: - type: thoth_rest - database: {database: warehouse, schema: datawarehouse} - endpoint: {base_url: https://dwh.example.test} -vectors: - type: thoth_vector_http - reader: {base_url: https://vectors.example.test} - writer: {base_url: https://vectors.example.test} -roots: {artifacts: artifacts, indexes: indexes, sessions: sessions} -evidence: {source_root: /data/source, evidence_dir: evidence} -embeddings: {base_url: https://embeddings.example.test, model: nomodel, dim: 768} -``` - -Esempio con accesso diretto a PostgreSQL e pgvector: - -```yaml -language: en -dwh: - type: postgres_direct - connection: {host: dwh.internal, database: warehouse, schema: public, - user: thoth_reader, password_file: /run/secrets/dwh_password} -vectors: - type: pgvector_direct - reader: {host: vector.internal, database: thoth, schema: vectors, - user: thoth_vector_reader, password_file: /run/secrets/vector_reader_password} - writer: {host: vector.internal, database: thoth, schema: vectors, - user: thoth_vector_writer, password_file: /run/secrets/vector_writer_password} -roots: {artifacts: artifacts, indexes: indexes, sessions: sessions} -``` - -Questo esempio mostra il contratto dell'adapter: i file indicati da `password_file` devono -essere montati da un override Compose approvato. Il profilo base monta soltanto il bundle unico; -per un DWH diretto occorre quindi materializzare il file password dal secret manager e aggiungere -il bind mount/runtime adapter corrispondente. Non inserire la password nel workspace o nell'URL. - -`roots` sono relativi e vengono risolti sotto `/data/workspaces/` nel volume Docker; non inserire path host come `/Users/...` o `C:\\...`. Per Evidence usare una radice filesystem montata in sola lettura oppure l'adapter HTTP/S3 previsto dal workspace. Per HTTP/S3 definire allowlist, limiti di dimensione/paginazione e una politica egress; non mettere token nelle URI. - -## 1. Server remoto insieme ai database e al vector DB - -Usare quando il server Docker è nella stessa rete del DWH e del vector DB (containerizzati o meno). -Compilare `deploy/env/local.env` con gli endpoint raggiungibili localmente: - -```dotenv -THT_DB_NAME=warehouse -THT_DWH_REST_URL=https://dwh.internal.example -THT_VEC_REST_URL=https://vectors.internal.example -THT_OLLAMA_URL=https://embeddings.internal.example -AUTH_MODE=none -THOTH_PUBLIC_EXPOSURE=false -``` - -Riempire nel bundle le chiavi DWH/vector/model necessarie e avviare: +I job di preprocessing usano gli stessi servizi interni Qdrant/Ollama: ```sh docker compose --env-file deploy/env/local.env \ - -f compose.yaml -f deploy/compose.local.yaml up --build -d + -f compose.yaml -f deploy/compose.local.yaml \ + -f deploy/compose.preprocess.yaml --profile preprocess run --rm preprocess-evidence +``` + +Per introspezione DWH: + +```sh docker compose --env-file deploy/env/local.env \ - -f compose.yaml -f deploy/compose.local.yaml exec core /opt/venv/bin/tht doctor --json + -f compose.yaml -f deploy/compose.local.yaml \ + -f deploy/compose.preprocess.yaml --profile preprocess run --rm preprocess-dwh ``` -Se si abilita l'overlay production, il proxy autenticato TLS deve essere l'unico listener pubblico -e deve sostituire gli header client con i claim restituiti dal proprio `auth_request`. L'esempio -usa header `X-Thoth-Trusted-*` soltanto sul collegamento privato; nginx frontend li converte nei -claim normalizzati `X-Thoth-Principal-Issuer`, `X-Thoth-Principal-Subject`, -`X-Thoth-Principal-Display-Name` e `X-Thoth-Is-Admin` attesi dal core. Non esporre direttamente -la porta pubblicata da nginx. +## Server -Se il server deve essere raggiungibile da altri host, usare il profilo -`deploy/compose.server.yaml`, configurare il proxy autenticato e impostare -`AUTH_MODE=upstream`/`THOTH_PUBLIC_EXPOSURE=true` come descritto nella sezione di trust boundary. - -## 2. Mac locale - -Installare Docker Desktop e, se usato, Ollama sul Mac. In `deploy/env/local.env` impostare gli endpoint: - -```dotenv -THT_DB_NAME=warehouse -THT_DWH_REST_URL=https://dwh.example.test -THT_OLLAMA_URL=http://host.docker.internal:11434 -THT_DOCS_ROOT=/data/source/evidence -``` - -Nel bundle aggiungere quattro password generate localmente: - -```dotenv -THT_VECTOR_BOOTSTRAP_PASSWORD= -THT_VECTOR_MIGRATOR_PASSWORD= -THT_VECTOR_READER_PASSWORD= -THT_VECTOR_WRITER_PASSWORD= -``` - -Poi eseguire il comando standard base+locale mostrato sopra. Il primo avvio esegue -reconciliation dei ruoli e migrazione pgvector. Per preprocessing, impostare il preset indicato -sopra e usare l'azione `run --rm preprocess-evidence` o `run --rm preprocess-dwh` con tutti -gli stessi file e profili. - -## 3. PC Windows locale - -Usare Docker Desktop con backend WSL2 e abilitare la condivisione della directory del clone. Modificare `deploy/env/local.env`: - -```dotenv -THT_DB_NAME=warehouse -THT_DWH_REST_URL=https://dwh.example.test -THT_OLLAMA_URL=http://host.docker.internal:11434 -THT_DOCS_ROOT=/data/source/evidence -``` - -Creare `deploy/secrets/thothii.secrets` con un editor locale protetto (ACL leggibile solo dall'utente Docker) e le stesse quattro chiavi pgvector del profilo Mac. Non usare `ConvertFrom-SecureString`: il bundle deve contenere il valore in chiaro per il servizio, con accesso limitato al file. Da PowerShell, dalla radice del clone, eseguire: - -```powershell -docker compose --env-file deploy/env/local.env -f compose.yaml -f deploy/compose.local.yaml up --build -d -docker compose --env-file deploy/env/local.env -f compose.yaml -f deploy/compose.local.yaml ps -``` - -Se un bind mount viene rifiutato, aggiungere la cartella del repository a Docker Desktop → Settings → Resources → File Sharing. Per Ollama eseguito in WSL2 usare l'indirizzo raggiungibile dalla rete Docker invece di assumere `localhost`. - -## 4. Server applicativo distinto da DB ed Evidence - -Usare il profilo server e consentire dal firewall solo le destinazioni necessarie: - -```dotenv -# Avvio: docker compose --env-file deploy/env/server.env \ -# -f compose.yaml -f deploy/compose.server.yaml \ -# -f deploy/compose.session-server.yaml.example up --build -d -THT_DB_NAME=warehouse -THT_DWH_REST_URL=https://dwh.example.test -THT_VEC_REST_URL=https://vectors.example.test -THT_OLLAMA_URL=https://embeddings.example.test -``` - -Avviare e verificare con il profilo server completo: +Per installazioni server usare il profilo server con overlay sessioni: ```sh docker compose --env-file deploy/env/server.env \ -f compose.yaml -f deploy/compose.server.yaml \ -f deploy/compose.session-server.yaml.example up --build -d -docker compose --env-file deploy/env/server.env \ - -f compose.yaml -f deploy/compose.server.yaml \ - -f deploy/compose.session-server.yaml.example exec core /opt/venv/bin/tht doctor --json ``` -Il DWH e il vector DB possono essere REST/HTTP oppure adapter diretti (`postgres_direct`, `pgvector_direct`) se il server ha connettività TCP. Le Evidence possono essere: +Consultare anche: -- filesystem NFS/SMB montato sul server e presentato come root read-only; -- endpoint HTTPS, con allowlist e limiti SSRF; -- bucket S3 con secret references e endpoint custom esplicitamente autorizzati. - -Il preprocessing può girare sul server applicativo usando il volume `/data`; mantenere separati workspace, lock e artefatti dei job. Avviare con il comando standard e verificare `tht doctor`. - -## Migrazione da installazioni con secret separati - -Le variabili `THT_*_SECRET_FILE` e i file `dwh-api-key`, `vector-reader-api-key`, `vector-writer-api-key`, `model-api-key` e `vector_*_password` appartengono al layout precedente. Non vengono importati automaticamente dal bundle. Per migrare: - -1. creare `deploy/secrets/thothii.secrets` mode `0600`; -2. copiare ogni valore nel nome chiave corrispondente (`THT_DWH_API_KEY`, `THT_VEC_API_KEY`, `THT_VEC_WRITE_API_KEY`, `THT_MODEL_API_KEY` o `THT_VECTOR_*_PASSWORD`), senza virgolette né newline; -3. rimuovere dal `.env` le variabili `_SECRET_FILE` e impostare `THT_SECRETS_FILE` al percorso assoluto del bundle; -4. renderizzare e avviare con il comando base+locale completo e il suo `--env-file`; -5. solo dopo la verifica, cancellare i vecchi file separati. - -Una CA PEM resta un'eccezione esterna come descritto sopra. Provider Pi con credenziali composte (Bedrock, Azure OpenAI Responses, Cloudflare Workers AI/Gateway) restano rifiutati finché non viene implementato un adapter dedicato. - -## Controlli post-installazione - -```sh -docker compose --env-file deploy/env/local.env \ - -f compose.yaml -f deploy/compose.local.yaml config --quiet -docker compose --env-file deploy/env/local.env \ - -f compose.yaml -f deploy/compose.local.yaml ps -docker compose --env-file deploy/env/local.env \ - -f compose.yaml -f deploy/compose.local.yaml exec core /opt/venv/bin/tht doctor --json -./scripts/docker-smoke.sh -``` - -Per il profilo locale usare anche `./scripts/local-vector-smoke.sh`; per il preprocessing `./scripts/preprocess-smoke.sh`. Non pubblicare `.env` o `deploy/secrets/thothii.secrets` nei log, nei backup Git o nei ticket. +- `docs/install/local-workspace-registry.md` +- `docs/install/server-workspace-registry.md` diff --git a/scripts/compose-with-preflight.sh b/scripts/compose-with-preflight.sh index f543ce9a..cbd679a1 100755 --- a/scripts/compose-with-preflight.sh +++ b/scripts/compose-with-preflight.sh @@ -114,6 +114,10 @@ if ((ssh_override && https_override)); then fi while IFS= read -r name; do + if [[ "$name" == *"_VECTOR_"* || "$name" == *"_EMBEDDING_"* || "$name" == THT_VECTOR_* ]]; then + echo "retired semantic source path is not supported: $name" >&2 + exit 2 + fi value="$(read_env_value "$env_file" "$name")" if [[ -v "$name" ]]; then value="${!name}" diff --git a/scripts/generate-connector-secrets-override.sh b/scripts/generate-connector-secrets-override.sh index 2600825b..5b553572 100755 --- a/scripts/generate-connector-secrets-override.sh +++ b/scripts/generate-connector-secrets-override.sh @@ -60,6 +60,10 @@ targets=() sources=() while IFS=$'\t' read -r name target; do [[ "$name" =~ ^THT_WS_[A-Za-z0-9_]+_FILE$ ]] || continue + if [[ "$name" == *"_VECTOR_"* || "$name" == *"_EMBEDDING_"* ]]; then + echo "retired semantic secret binding is not supported: ${name%_FILE}_SOURCE" >&2 + exit 2 + fi [[ "$target" =~ ^/run/secrets/[A-Za-z0-9][A-Za-z0-9_.-]*$ && "$target" != *..* ]] || { echo "invalid connector secret target for $name: $target" >&2 exit 2 diff --git a/scripts/local-vector-smoke.sh b/scripts/local-vector-smoke.sh deleted file mode 100755 index 45f451f9..00000000 --- a/scripts/local-vector-smoke.sh +++ /dev/null @@ -1,446 +0,0 @@ -#!/bin/sh -set -eu - -cd "$(dirname "$0")/.." - -mode=${1:-run} -case "$mode" in - run|--live-collision-test|--backup-restore) ;; - *) echo "usage: $0 [--live-collision-test|--backup-restore]" >&2; exit 2 ;; -esac - -keep_resources=${KEEP_SMOKE_RESOURCES:-0} -if [ "${SMOKE_PROJECT+x}" = x ]; then - echo "SMOKE_PROJECT is not accepted; the smoke always generates an owned namespace" >&2 - exit 2 -fi -secret_dir=$(mktemp -d "${TMPDIR:-/tmp}/thothii-vector-smoke.XXXXXX") -suffix=$(basename "$secret_dir" | tr -cd 'a-z0-9') -smoke_project="thothii-vector-smoke-$(date +%s)-$$-$suffix" -smoke_owner="$smoke_project-owner" -marker="local-vector-$smoke_project" -restore_container="${smoke_project}-restore" -restore_volume="${smoke_project}-restore-data" - -bootstrap_password="smoke-bootstrap-$smoke_project" -migrator_password="smoke-migrator-$smoke_project" -reader_password="smoke-reader-$smoke_project" -writer_password="smoke-writer-$smoke_project" -bundle="$secret_dir/thothii.secrets" -write_bundle() { - umask 077 - { - printf 'THT_VECTOR_BOOTSTRAP_PASSWORD=%s\n' "$bootstrap_password" - printf 'THT_VECTOR_MIGRATOR_PASSWORD=%s\n' "$migrator_password" - printf 'THT_VECTOR_READER_PASSWORD=%s\n' "$reader_password" - printf 'THT_VECTOR_WRITER_PASSWORD=%s\n' "$writer_password" - } >"$bundle" - chmod 0600 "$bundle" -} -write_bundle -export THT_SECRETS_FILE="$bundle" -printf '%s\n' '{}' >"$secret_dir/pi-auth.json" -chmod 0600 "$secret_dir/pi-auth.json" -operator_env="$secret_dir/operator.env" -printf '%s\n' \ - 'THT_WORKSPACE_GIT_REMOTE=https://git.example.invalid/platform/thoth-workspaces.git' \ - "PI_AUTH_FILE=$secret_dir/pi-auth.json" \ - "THT_SECRETS_FILE=$bundle" >"$operator_env" -# The rotation helper has an old/new file interface; these are test-only -# scratch files and are never mounted into a Compose service. -printf '%s' "$bootstrap_password" >"$secret_dir/bootstrap" -chmod 0600 "$secret_dir/bootstrap" -export THT_VECTOR_BOOTSTRAP_USER=thoth_bootstrap_smoke -export THOTH_SMOKE_OWNER="$smoke_owner" - -compose() { - docker compose --env-file "$operator_env" -f compose.yaml -f deploy/compose.local-vector.yaml \ - --project-name "$smoke_project" --profile local-vector "$@" -} - -resource_ids() { - case "$1" in - container) docker ps -aq --filter "label=com.docker.compose.project=$smoke_project" ;; - volume) docker volume ls -q --filter "label=com.docker.compose.project=$smoke_project" ;; - network) docker network ls -q --filter "label=com.docker.compose.project=$smoke_project" ;; - esac -} - -resource_owner() { - case "$1" in - container) docker inspect --format '{{ index .Config.Labels "io.thothii.smoke-owner" }}' "$2" ;; - volume) docker volume inspect --format '{{ index .Labels "io.thothii.smoke-owner" }}' "$2" ;; - network) docker network inspect --format '{{ index .Labels "io.thothii.smoke-owner" }}' "$2" ;; - esac -} - -assert_no_collision() { - for kind in container volume network; do - ids=$(resource_ids "$kind") - if [ -n "$ids" ]; then - echo "refusing existing Compose project resources for generated namespace $smoke_project" >&2 - return 1 - fi - done -} - -verify_owned_resources() { - for kind in container volume network; do - for id in $(resource_ids "$kind"); do - owner=$(resource_owner "$kind" "$id" 2>/dev/null || true) - if [ "$owner" != "$smoke_owner" ]; then - echo "refusing cleanup of resource not owned by this smoke: $kind $id" >&2 - return 1 - fi - done - done -} - -cleanup() { - if [ "$keep_resources" = "1" ]; then - echo "Keeping smoke resources for project $smoke_project (KEEP_SMOKE_RESOURCES=1)." >&2 - else - if verify_owned_resources; then - docker rm -f "$restore_container" >/dev/null 2>&1 || true - docker volume rm "$restore_volume" >/dev/null 2>&1 || true - compose down --volumes >/dev/null 2>&1 || true - fi - fi - rm -rf "$secret_dir" -} -trap cleanup EXIT HUP INT TERM - -if [ "$mode" = "--live-collision-test" ]; then - collision_volume="${smoke_project}-collision" - docker volume create \ - --label "com.docker.compose.project=$smoke_project" \ - --label 'io.thothii.smoke-owner=foreign-owner' \ - "$collision_volume" >/dev/null - if assert_no_collision 2>/dev/null; then - echo "live collision probe was not detected" >&2 - docker volume rm "$collision_volume" >/dev/null - exit 1 - fi - docker volume rm "$collision_volume" >/dev/null - echo "live local-vector project collision refusal passed." - exit 0 -fi - -probe_vector() { - compose exec -T core sh -ec ' - . /app/docker/secret-policy.sh - tmp=$(mktemp -d); trap "rm -rf \"$tmp\"" EXIT - for role in READER WRITER; do - file="$tmp/$role" - read_bundle_secret /run/secrets/thothii.secrets "THT_VECTOR_${role}_PASSWORD" >"$file" - export "THT_VECTOR_${role}_PASSWORD_FILE=$file" - done - exec /opt/venv/bin/python - "$1" "$2" - ' sh "$marker" "$1" <<'PY' -import hashlib -import os -import sys - -from tht.adapters.vector.pgvector import PgVectorStore -from tht.config import DatabaseConfig -from tht.ports.vector import VectorWriteRecord -from tht.vectorstore.records import VectorRecord - -marker = sys.argv[1] -mode = sys.argv[2] -database = "thoth" -host = "vector-db" - -def credential(role: str) -> DatabaseConfig: - return DatabaseConfig( - host=host, - port=5432, - database=database, - schema="vectors", - user=f"thoth_vector_{role}", - password=open(os.environ[f"THT_VECTOR_{role.upper()}_PASSWORD_FILE"]).read(), - ) - -store = PgVectorStore(credential("reader"), credential("writer"), expected_dimension=768) -health = store.health() -assert health.ok, health -assert health.read_reachable is True and health.write_reachable is True, health - -record = VectorRecord( - id=marker, - kind="memory", - ref=marker, - title="Local vector persistence smoke", - content=marker, - metadata={"smoke": True}, -) -embedding = [1.0] + [0.0] * 767 -if mode == "write": - store.upsert( - "memory", - [VectorWriteRecord(record, embedding, hashlib.sha256(marker.encode()).hexdigest())], - ) -hits = store.search(["memory"], embedding, limit=1, kinds=["memory"]) -assert hits and hits[0].id == marker, hits -print(f"role health and persisted search passed for {marker} ({mode})") -PY -} - -assert_no_collision -compose config --quiet -services=$(compose config --services) -printf '%s\n' "$services" | grep -qx vector-db -printf '%s\n' "$services" | grep -qx vector-reconcile -printf '%s\n' "$services" | grep -qx vector-migrate - -compose up --build --wait vector-reconcile vector-migrate core -core_id=$(compose ps -q core) -inspect_env=$(docker inspect --format '{{json .Config.Env}}' "$core_id") -if printf '%s' "$inspect_env" | grep -q "smoke-\(reader\|writer\)-${smoke_project}"; then - echo "docker inspect exposed a direct vector password" >&2 - exit 1 -fi -printf '%s' "$inspect_env" | grep -q 'THT_SECRETS_FILE=/run/secrets/thothii.secrets' -migration_status=$(compose run --rm --no-deps vector-migrate) -printf '%s\n' "$migration_status" | grep -q '"pending": \[\]' -migrator_flags=$(compose run --rm --no-deps --entrypoint sh vector-reconcile -ec ' - . /opt/thoth/secret-policy.sh - export PGPASSWORD=$(read_bundle_secret /run/secrets/thothii.secrets THT_VECTOR_BOOTSTRAP_PASSWORD) - psql -At --host vector-db --username "$THT_VECTOR_BOOTSTRAP_USER" --dbname thoth \ - --command "SELECT (NOT rolcreaterole) AND (NOT rolcreatedb) AND (NOT rolsuper) FROM pg_roles WHERE rolname = '\''thoth_vector_migrator'\''" -') -test "$migrator_flags" = t -probe_vector write - -old_reader_password="$reader_password" -migrator_password="rotated-migrator-$smoke_project" -reader_password="rotated-reader-$smoke_project" -writer_password="rotated-writer-$smoke_project" -write_bundle - -compose run --rm vector-reconcile -rotation_status=$(compose run --rm --no-deps vector-migrate) -printf '%s\n' "$rotation_status" | grep -q '"pending": \[\]' -if compose run --rm --no-deps --entrypoint psql \ - -e PGPASSWORD="$old_reader_password" vector-reconcile \ - --host vector-db --username thoth_vector_reader --dbname thoth --command 'SELECT 1' \ - >/dev/null 2>&1; then - echo "old reader credential still works after rotation" >&2 - exit 1 -fi - -compose up --force-recreate --no-deps --wait core -probe_vector read - -old_bootstrap_password="$bootstrap_password" -printf '%s' "wrong-bootstrap-${smoke_project}" >"$secret_dir/bootstrap-wrong" -printf '%s' "next-bootstrap-'quoted-${smoke_project}" >"$secret_dir/bootstrap-next" -cp "$secret_dir/bootstrap" "$secret_dir/bootstrap-before-negative" -printf 'invalid bootstrap password\n' >"$secret_dir/bootstrap-whitespace" -chmod 0600 "$secret_dir/bootstrap-wrong" "$secret_dir/bootstrap-next" \ - "$secret_dir/bootstrap-before-negative" "$secret_dir/bootstrap-whitespace" -if COMPOSE_PROJECT_NAME="$smoke_project" \ - ./scripts/vector-rotate-bootstrap-password.sh \ - --env-file "$operator_env" \ - "$secret_dir/bootstrap" "$secret_dir/bootstrap-whitespace" \ - >/dev/null 2>&1; then - echo "bootstrap rotation accepted whitespace in a secret" >&2 - exit 1 -fi -cmp "$secret_dir/bootstrap" "$secret_dir/bootstrap-before-negative" -compose run --rm --no-deps --entrypoint psql \ - -e PGPASSWORD="$old_bootstrap_password" vector-reconcile \ - --host vector-db --username "$THT_VECTOR_BOOTSTRAP_USER" --dbname thoth \ - --command 'SELECT 1' >/dev/null - -if COMPOSE_PROJECT_NAME="$smoke_project" \ - ./scripts/vector-rotate-bootstrap-password.sh \ - --env-file "$operator_env" \ - "$secret_dir/bootstrap-wrong" "$secret_dir/bootstrap-next" \ - >/dev/null 2>&1; then - echo "bootstrap rotation accepted the wrong old secret" >&2 - exit 1 -fi -cmp "$secret_dir/bootstrap" "$secret_dir/bootstrap-before-negative" - -COMPOSE_PROJECT_NAME="$smoke_project" \ - ./scripts/vector-rotate-bootstrap-password.sh \ - --env-file "$operator_env" \ - "$secret_dir/bootstrap" "$secret_dir/bootstrap-next" -new_bootstrap_password=$(cat "$secret_dir/bootstrap") -bootstrap_password="$new_bootstrap_password" -write_bundle -test "$new_bootstrap_password" != "$old_bootstrap_password" -if compose run --rm --no-deps --entrypoint psql \ - -e PGPASSWORD="$old_bootstrap_password" vector-reconcile \ - --host vector-db --username "$THT_VECTOR_BOOTSTRAP_USER" --dbname thoth --command 'SELECT 1' \ - >/dev/null 2>&1; then - echo "old bootstrap credential still works after rotation" >&2 - exit 1 -fi -compose run --rm --no-deps --entrypoint psql \ - -e PGPASSWORD="$new_bootstrap_password" vector-reconcile \ - --host vector-db --username "$THT_VECTOR_BOOTSTRAP_USER" --dbname thoth --command 'SELECT 1' \ - >/dev/null -compose run --rm vector-reconcile -bootstrap_rotation_status=$(compose run --rm --no-deps vector-migrate) -printf '%s\n' "$bootstrap_rotation_status" | grep -q '"pending": \[\]' -compose up --force-recreate --no-deps --wait core -probe_vector read - -compose restart vector-db core -compose up --wait vector-db core -probe_vector read - -if [ "$mode" = "--backup-restore" ]; then - image=$(compose images -q vector-db) - network="${smoke_project}_default" - docker volume create \ - --label "com.docker.compose.project=$smoke_project" \ - --label "io.thothii.smoke-owner=$smoke_owner" "$restore_volume" >/dev/null - docker run -d --name "$restore_container" \ - --label "com.docker.compose.project=$smoke_project" \ - --label "io.thothii.smoke-owner=$smoke_owner" \ - --network "$network" --network-alias vector-db-restore \ - --mount "type=volume,source=$restore_volume,target=/var/lib/postgresql/data" \ - --mount "type=bind,source=$bundle,target=/run/secrets/thothii.secrets,readonly" \ - --mount "type=bind,source=$(pwd)/deploy/vector/vector-db-entrypoint.sh,target=/opt/thoth/vector-db-entrypoint.sh,readonly" \ - --mount "type=bind,source=$(pwd)/deploy/vector/secret-policy.sh,target=/opt/thoth/secret-policy.sh,readonly" \ - -e POSTGRES_DB=thoth -e POSTGRES_USER="$THT_VECTOR_BOOTSTRAP_USER" \ - -e THT_SECRETS_FILE=/run/secrets/thothii.secrets \ - --entrypoint /opt/thoth/vector-db-entrypoint.sh "$image" >/dev/null - attempts=0 - until docker exec "$restore_container" pg_isready \ - -U "$THT_VECTOR_BOOTSTRAP_USER" -d thoth >/dev/null 2>&1; do - attempts=$((attempts + 1)) - [ "$attempts" -lt 30 ] || { echo "restore database did not become ready" >&2; exit 1; } - sleep 1 - done - docker exec -e PGPASSWORD="$new_bootstrap_password" "$restore_container" psql -X \ - -U "$THT_VECTOR_BOOTSTRAP_USER" -d thoth -v ON_ERROR_STOP=1 --command \ - "CREATE SCHEMA vectors; CREATE EXTENSION vector WITH SCHEMA vectors; - CREATE TABLE vectors.memory ( - id bigserial PRIMARY KEY, record_key text UNIQUE NOT NULL, kind text NOT NULL, - content_hash text NOT NULL, metadata jsonb NOT NULL, - embedding vectors.vector(768) NOT NULL, indexed_at timestamptz NOT NULL DEFAULT now()); - INSERT INTO vectors.memory (record_key, kind, content_hash, metadata, embedding) - VALUES ('restore-sentinel', 'memory', 'sentinel-original', '{}', - ('[' || '1,' || repeat('0,', 766) || '0]')::vectors.vector);" >/dev/null - - docker run --rm --network "$network" \ - --mount "type=bind,source=$(pwd),target=/repo,readonly" \ - --mount "type=bind,source=$secret_dir,target=/scratch" "$image" \ - /repo/scripts/vector-backup.sh --host vector-db --database thoth \ - --user "$THT_VECTOR_BOOTSTRAP_USER" --password-file /scratch/bootstrap \ - --output /scratch/vector.dump - - compose exec -T vector-db sh -ec ' - . /opt/thoth/secret-policy.sh - export PGPASSWORD=$(read_bundle_secret /run/secrets/thothii.secrets THT_VECTOR_BOOTSTRAP_PASSWORD) - psql -X -U "$POSTGRES_USER" -d thoth -v ON_ERROR_STOP=1 --command \ - "UPDATE vectors.memory SET content_hash = '\''mutated-after-backup'\'' WHERE record_key = '\''$1'\''"' \ - sh "$marker" >/dev/null - - if docker run --rm --network "$network" \ - --mount "type=bind,source=$(pwd),target=/repo,readonly" \ - --mount "type=bind,source=$secret_dir,target=/scratch" "$image" \ - /repo/scripts/vector-restore.sh \ - --active-host vector-db --active-database thoth --active-user "$THT_VECTOR_BOOTSTRAP_USER" \ - --active-password-file /scratch/bootstrap \ - --target-host vector-db-restore --target-database thoth \ - --target-user "$THT_VECTOR_BOOTSTRAP_USER" --target-password-file /scratch/bootstrap \ - --input /scratch/vector.dump --force-nonempty >/dev/null 2>&1; then - echo "forced restore unexpectedly succeeded without archived ACL roles" >&2 - exit 1 - fi - sentinel=$(docker exec -e PGPASSWORD="$new_bootstrap_password" "$restore_container" psql \ - -XAt -U "$THT_VECTOR_BOOTSTRAP_USER" -d thoth --command \ - "SELECT content_hash FROM vectors.memory WHERE record_key='restore-sentinel'") - test "$sentinel" = sentinel-original - docker exec -e PGPASSWORD="$new_bootstrap_password" "$restore_container" psql -X \ - -U "$THT_VECTOR_BOOTSTRAP_USER" -d thoth -v ON_ERROR_STOP=1 --command \ - "DROP TABLE vectors.memory; CREATE ROLE vector_reader NOLOGIN; CREATE ROLE vector_writer NOLOGIN;" \ - >/dev/null - - docker run --rm --network "$network" \ - --mount "type=bind,source=$(pwd),target=/repo,readonly" \ - --mount "type=bind,source=$secret_dir,target=/scratch" "$image" \ - /repo/scripts/vector-restore.sh \ - --active-host vector-db --active-database thoth --active-user "$THT_VECTOR_BOOTSTRAP_USER" \ - --active-password-file /scratch/bootstrap \ - --target-host vector-db-restore --target-database thoth \ - --target-user "$THT_VECTOR_BOOTSTRAP_USER" --target-password-file /scratch/bootstrap \ - --input /scratch/vector.dump - - docker run --rm --network "$network" \ - --mount "type=bind,source=$(pwd)/deploy/vector/reconcile-roles.sh,target=/opt/thoth/reconcile-roles.sh,readonly" \ - --mount "type=bind,source=$(pwd)/deploy/vector/secret-policy.sh,target=/opt/thoth/secret-policy.sh,readonly" \ - --mount "type=bind,source=$bundle,target=/run/secrets/thothii.secrets,readonly" \ - -e PGHOST=vector-db-restore -e PGDATABASE=thoth \ - -e PGUSER="$THT_VECTOR_BOOTSTRAP_USER" \ - -e THT_SECRETS_FILE=/run/secrets/thothii.secrets \ - -e THT_VECTOR_MIGRATOR_USER=thoth_vector_migrator \ - -e THT_VECTOR_READER_USER=thoth_vector_reader \ - -e THT_VECTOR_WRITER_USER=thoth_vector_writer \ - --entrypoint /opt/thoth/reconcile-roles.sh "$image" >/dev/null - - compose exec -T core sh -ec ' - . /app/docker/secret-policy.sh - tmp=$(mktemp -d); trap "rm -rf \"$tmp\"" EXIT - for role in READER WRITER; do - file="$tmp/$role" - read_bundle_secret /run/secrets/thothii.secrets "THT_VECTOR_${role}_PASSWORD" >"$file" - export "THT_VECTOR_${role}_PASSWORD_FILE=$file" - done - exec /opt/venv/bin/python - "$1" - ' sh "$marker" <<'PY' -import hashlib -import os -import sys - -from tht.adapters.vector.pgvector import PgVectorStore -from tht.config import DatabaseConfig -from tht.ports.vector import VectorWriteRecord -from tht.vectorstore.records import VectorRecord - -def config(role): - return DatabaseConfig( - host="vector-db-restore", port=5432, database="thoth", schema="vectors", - user=f"thoth_vector_{role}", - password=open(os.environ[f"THT_VECTOR_{role.upper()}_PASSWORD_FILE"]).read(), - ) - -store = PgVectorStore(config("reader"), config("writer"), expected_dimension=768) -assert store.health().ok, store.health() -embedding = [1.0] + [0.0] * 767 -marker = sys.argv[1] -assert store.search(["memory"], embedding, limit=1, kinds=["memory"])[0].id == marker -write_id = marker + "-restore-write" -record = VectorRecord( - id=write_id, kind="memory", ref=write_id, title="restore writer", - content=write_id, metadata={}, -) -store.upsert("memory", [VectorWriteRecord(record, embedding, hashlib.sha256(write_id.encode()).hexdigest())]) -assert store.existing_hashes("memory", ["memory"])[write_id] -PY - - restored=$(docker exec -e PGPASSWORD="$new_bootstrap_password" "$restore_container" psql \ - -XAt -U "$THT_VECTOR_BOOTSTRAP_USER" -d thoth --command \ - "SELECT content_hash <> 'mutated-after-backup' FROM vectors.memory WHERE record_key = '$marker'") - test "$restored" = t - expected_migrations=$(find harness/tht/migrations/vector -type f -name '[0-9][0-9][0-9]_*.sql' \ - -exec basename {} \; | sed 's/_.*//' | sort | paste -sd, -) - applied_migrations=$(docker exec -e PGPASSWORD="$new_bootstrap_password" "$restore_container" psql \ - -XAt -U "$THT_VECTOR_BOOTSTRAP_USER" -d thoth --command \ - "SELECT string_agg(version, ',' ORDER BY version) FROM public.tht_vector_migrations") - test "$applied_migrations" = "$expected_migrations" - dimensions=$(docker exec -e PGPASSWORD="$new_bootstrap_password" "$restore_container" psql \ - -XAt -U "$THT_VECTOR_BOOTSTRAP_USER" -d thoth --command \ - "SELECT count(*) = 3 FROM pg_attribute a JOIN pg_class c ON c.oid=a.attrelid - JOIN pg_namespace n ON n.oid=c.relnamespace - WHERE n.nspname='vectors' AND a.attname='embedding' AND format_type(a.atttypid,a.atttypmod)='vectors.vector(768)'") - test "$dimensions" = t - echo "Transactional rollback and disposable-volume restore adapter parity passed." -fi - -echo "Local pgvector runtime/bootstrap rotation, least-privilege roles, and persistence passed." diff --git a/scripts/preprocess-smoke.sh b/scripts/preprocess-smoke.sh index 3b112275..63898853 100755 --- a/scripts/preprocess-smoke.sh +++ b/scripts/preprocess-smoke.sh @@ -46,29 +46,23 @@ trap cleanup EXIT trap 'exit 129' HUP trap 'exit 130' INT trap 'exit 143' TERM + mkdir -p "$tmp/source/evidence" printf '%s\n' '# Evidence' 'generation one' >"$tmp/source/evidence/a.md" bundle="$tmp/thothii.secrets" -{ - printf 'THT_VECTOR_BOOTSTRAP_PASSWORD=smoke-bootstrap-%s\n' "$project" - printf 'THT_VECTOR_MIGRATOR_PASSWORD=smoke-migrator-%s\n' "$project" - printf 'THT_VECTOR_READER_PASSWORD=smoke-reader-%s\n' "$project" - printf 'THT_VECTOR_WRITER_PASSWORD=smoke-writer-%s\n' "$project" -} >"$bundle" +printf '%s\n' 'THT_MODEL_API_KEY=smoke-model-key' >"$bundle" chmod 0600 "$bundle" -export THT_SECRETS_FILE="$bundle" -export THT_OLLAMA_URL=http://mock-embeddings:8081 printf '%s\n' '{}' >"$tmp/pi-auth.json" -chmod 0600 "$tmp/pi-auth.json" +printf '%s' 'smoke-dwh-password' >"$tmp/dwh-password" +chmod 0600 "$tmp/pi-auth.json" "$tmp/dwh-password" printf '%s\n' \ 'THT_WORKSPACE_GIT_REMOTE=https://git.example.invalid/platform/thoth-workspaces.git' \ "PI_AUTH_FILE=$tmp/pi-auth.json" \ - "THT_SECRETS_FILE=$bundle" \ - 'THT_OLLAMA_URL=http://mock-embeddings:8081' >"$tmp/operator.env" + "THT_SECRETS_FILE=$bundle" >"$tmp/operator.env" cat >"$tmp/smoke.yaml" <&2 + exit 2 + fi + mode=$(stat -c '%a' "$path" 2>/dev/null || stat -f '%Lp' "$path" 2>/dev/null) || { + echo "cannot inspect permissions for $label: $path" >&2 + exit 2 + } + if [ $((0$mode & 077)) -ne 0 ]; then + echo "$label must not be readable or writable by group/other users: $path" >&2 + exit 2 + fi +} + +read_secret_file() { + path="$1" + label="$2" + validate_secret_file "$path" "$label" + value=$(tr -d '\r' <"$path") + case "$value" in + *' +'*) echo "$label must contain exactly one line" >&2; exit 2 ;; + esac + [ -n "$value" ] || { echo "$label must not be empty" >&2; exit 2; } + printf '%s' "$value" +} diff --git a/scripts/task13-runtime-fixture-check.ts b/scripts/task13-runtime-fixture-check.ts index 6162e1ee..877b334e 100644 --- a/scripts/task13-runtime-fixture-check.ts +++ b/scripts/task13-runtime-fixture-check.ts @@ -24,14 +24,6 @@ const expected = { THT_WS_TASK13_SMOKE_DWH_PORT: "5432", THT_WS_TASK13_SMOKE_DWH_USER: "task13_reader", THT_WS_TASK13_SMOKE_DWH_PASSWORD_FILE: "/run/secrets/task13-runtime-password", - THT_WS_TASK13_SMOKE_VECTOR_TRANSPORT: "pgvector_direct", - THT_WS_TASK13_SMOKE_VECTOR_HOST: "vector.task13.invalid", - THT_WS_TASK13_SMOKE_VECTOR_PORT: "5432", - THT_WS_TASK13_SMOKE_VECTOR_USER: "task13_vector_reader", - THT_WS_TASK13_SMOKE_VECTOR_PASSWORD_FILE: "/run/secrets/task13-runtime-password", - THT_WS_TASK13_SMOKE_EMBEDDING_BASE_URL: profile === "local" - ? `http://${config.name}-llm:9000` - : "https://embedding.task13.invalid", }; for (const [name, value] of Object.entries(expected)) { if (core.environment?.[name] !== value) { @@ -82,7 +74,6 @@ for (const target of [ const resolverEnvironment = { ...core.environment }; resolverEnvironment.THT_WS_TASK13_SMOKE_DWH_PASSWORD_FILE = runtimePasswordMounts[0].source; -resolverEnvironment.THT_WS_TASK13_SMOKE_VECTOR_PASSWORD_FILE = runtimePasswordMounts[0].source; const bindings = resolveRuntimeBindings(workspace, resolverEnvironment, [dirname(runtimePasswordMounts[0].source)]); for (const [role, binding] of Object.entries(bindings)) { if ((binding as any).missing.length !== 0) { @@ -99,12 +90,13 @@ if (runtime.database.host !== expected.THT_WS_TASK13_SMOKE_DWH_HOST || runtime.database.password_file !== runtimePasswordMounts[0].source) { throw new Error("workspace resolver produced the wrong DWH runtime"); } -if (runtime.vector_db.host !== expected.THT_WS_TASK13_SMOKE_VECTOR_HOST - || runtime.vector_db.user !== expected.THT_WS_TASK13_SMOKE_VECTOR_USER - || runtime.vector_db.password_file !== runtimePasswordMounts[0].source) { - throw new Error("workspace resolver produced the wrong vector runtime"); +if (runtime.resources?.vector?.base_url !== "http://qdrant:6333" + || runtime.resources?.vector?.collection !== "task13-smoke") { + throw new Error("workspace resolver produced the wrong qdrant runtime"); } -if (runtime.embeddings.base_url !== expected.THT_WS_TASK13_SMOKE_EMBEDDING_BASE_URL) { +if (runtime.resources?.embeddings?.base_url !== "http://embedding:11434" + || runtime.resources?.embeddings?.model !== "qwen3-embedding:0.6b" + || runtime.resources?.embeddings?.dimensions !== 1024) { throw new Error("workspace resolver produced the wrong embedding runtime"); } const secret = readFileSync(bundleSource, "utf8").trim(); diff --git a/scripts/test-compose-secret-policy.sh b/scripts/test-compose-secret-policy.sh index c4fbc2bf..aafc9839 100755 --- a/scripts/test-compose-secret-policy.sh +++ b/scripts/test-compose-secret-policy.sh @@ -110,7 +110,6 @@ write_secret "$fixture_root/https-credentials" 'fixture-https-credentials' write_secret "$fixture_root/https-ca.pem" 'fixture-https-ca' write_secret "$fixture_root/dwh-password" 'fixture-dwh-password' write_secret "$fixture_root/vector-api-key" 'fixture-vector-api-key' - printf '%s\n' \ 'THT_WORKSPACE_GIT_REMOTE=https://git.example.invalid/platform/thoth-workspaces.git' \ "PI_AUTH_FILE=$fixture_root/pi-auth.json" \ @@ -120,14 +119,11 @@ printf '%s\n' \ "THT_WORKSPACE_GIT_KNOWN_HOSTS_FILE=$fixture_root/ssh-known-hosts" \ "THT_WORKSPACE_GIT_CREDENTIALS_FILE=$fixture_root/https-credentials" \ "THT_WORKSPACE_GIT_CA_FILE=$fixture_root/https-ca.pem" \ - "THT_WS_NORTH_STAR_RESEARCH_DWH_PASSWORD_SOURCE=$fixture_root/dwh-password" \ - "THT_WS_NORTH_STAR_RESEARCH_VECTOR_API_KEY_SOURCE=$fixture_root/vector-api-key" >"$fixture_root/operator.env" + "THT_WS_NORTH_STAR_RESEARCH_DWH_PASSWORD_SOURCE=$fixture_root/dwh-password" >"$fixture_root/operator.env" printf '%s\n' \ 'THT_WS_NORTH_STAR_RESEARCH_DWH_TRANSPORT=postgres_direct' \ 'THT_WS_NORTH_STAR_RESEARCH_DWH_PASSWORD_FILE=/run/secrets/north-star-research-dwh-password' \ - 'THT_WS_NORTH_STAR_RESEARCH_VECTOR_TRANSPORT=rest_api' \ - 'THT_WS_NORTH_STAR_RESEARCH_VECTOR_API_KEY_FILE=/run/secrets/north-star-research-vector-api-key' \ >"$fixture_root/workspace-bindings.env" connector_override="$fixture_root/compose.connector-secrets.local.yaml" @@ -142,10 +138,9 @@ assert_render_contract ssh '/run/secrets/workspace-registry-git-known-hosts,/run render https -f "$root/deploy/compose.git-https.yaml" assert_render_contract https '/run/secrets/workspace-registry-git-ca,/run/secrets/workspace-registry-git-credentials' 'thothii.secrets' render connector -f "$connector_override" -assert_render_contract connector '' 'north-star-research-dwh-password,north-star-research-vector-api-key,thothii.secrets' +assert_render_contract connector '' 'north-star-research-dwh-password,thothii.secrets' assert_missing_source_rejected THT_WS_NORTH_STAR_RESEARCH_DWH_PASSWORD_SOURCE -assert_missing_source_rejected THT_WS_NORTH_STAR_RESEARCH_VECTOR_API_KEY_SOURCE assert_unsafe_source_rejected 'relative/secret' relative-source assert_unsafe_source_rejected '/private/secrets/../secret' non-normalized-source if THT_WS_NORTH_STAR_RESEARCH_DWH_PASSWORD_SOURCE='relative/host-override' \ @@ -182,4 +177,25 @@ if "$root/scripts/compose-with-preflight.sh" --env-file "$fixture_root/operator. fi grep -Fq 'mutually exclusive' "$fixture_root/renamed-combined.err" +printf '%s\n' \ + 'THT_WS_NORTH_STAR_RESEARCH_DWH_PASSWORD_FILE=/run/secrets/north-star-research-dwh-password' \ + 'THT_WS_NORTH_STAR_RESEARCH_VECTOR_API_KEY_FILE=/run/secrets/north-star-research-vector-api-key' \ + >"$fixture_root/workspace-bindings-with-vector.env" +printf '%s\n' \ + 'THT_WORKSPACE_GIT_REMOTE=https://git.example.invalid/platform/thoth-workspaces.git' \ + "PI_AUTH_FILE=$fixture_root/pi-auth.json" \ + "THT_SECRETS_FILE=$fixture_root/thothii.secrets" \ + "THT_WS_NORTH_STAR_RESEARCH_DWH_PASSWORD_SOURCE=$fixture_root/dwh-password" \ + "THT_WS_NORTH_STAR_RESEARCH_VECTOR_API_KEY_SOURCE=$fixture_root/vector-api-key" \ + >"$fixture_root/operator-with-vector.env" +if "$root/scripts/generate-connector-secrets-override.sh" \ + --bindings-env "$fixture_root/workspace-bindings-with-vector.env" \ + --operator-env "$fixture_root/operator-with-vector.env" \ + --output "$fixture_root/forbidden-vector.yaml" \ + >"$fixture_root/forbidden-vector.out" 2>"$fixture_root/forbidden-vector.err"; then + echo "connector generator accepted a retired semantic API key binding" >&2 + exit 1 +fi +grep -Fq 'VECTOR_API_KEY_SOURCE' "$fixture_root/forbidden-vector.err" + echo "Compose secret policy passed." diff --git a/scripts/test-deployment-command-contract.sh b/scripts/test-deployment-command-contract.sh index f45305b4..6dc4790e 100755 --- a/scripts/test-deployment-command-contract.sh +++ b/scripts/test-deployment-command-contract.sh @@ -17,9 +17,7 @@ targets=( scripts/build-local.sh scripts/build-local.ps1 scripts/docker-smoke.sh - scripts/local-vector-smoke.sh scripts/preprocess-smoke.sh - scripts/vector-rotate-bootstrap-password.sh ) existing=() diff --git a/scripts/test-local-vector-smoke-live-collision.sh b/scripts/test-local-vector-smoke-live-collision.sh deleted file mode 100755 index c7e047ad..00000000 --- a/scripts/test-local-vector-smoke-live-collision.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/sh -set -eu - -cd "$(dirname "$0")/.." -./scripts/local-vector-smoke.sh --live-collision-test diff --git a/scripts/test-local-vector-smoke-safety.sh b/scripts/test-local-vector-smoke-safety.sh deleted file mode 100755 index f4bd4c1c..00000000 --- a/scripts/test-local-vector-smoke-safety.sh +++ /dev/null @@ -1,71 +0,0 @@ -#!/bin/sh -set -eu - -cd "$(dirname "$0")/.." -tmp=$(mktemp -d) -trap 'rm -rf "$tmp"' EXIT HUP INT TERM - -fake="$tmp/docker" -log="$tmp/docker.log" -state="$tmp/state" - -cat >"$fake" <<'SH' -#!/bin/sh -set -eu -printf '%s\n' "$*" >>"$FAKE_DOCKER_LOG" - -if [ "${FAKE_COLLISION:-0}" = 1 ] && [ "$1 $2" = "ps -aq" ]; then - printf '%s\n' collision-container - exit 0 -fi - -if [ "$1 $2" = "ps -aq" ] || [ "$1 $2" = "volume ls" ] || [ "$1 $2" = "network ls" ]; then - if [ "${FAKE_MISMATCH_ON_CLEANUP:-0}" = 1 ] && [ -f "$FAKE_DOCKER_STATE" ]; then - printf '%s\n' foreign-resource - fi - : >"$FAKE_DOCKER_STATE" - exit 0 -fi - -if [ "$1" = inspect ] || [ "$1 $2" = "volume inspect" ] || [ "$1 $2" = "network inspect" ]; then - printf '%s\n' foreign-owner - exit 0 -fi - -case "$*" in - *"config --services"*) printf '%s\n' vector-db vector-reconcile vector-migrate core frontend ;; - *"run --rm --no-deps vector-migrate"*) printf '%s\n' '{"applied":["001","002","003"],"drifted":[],"pending":[]}' ;; -esac -exit 0 -SH -chmod 0755 "$fake" - -if PATH="$tmp:$PATH" FAKE_DOCKER_LOG="$log" FAKE_DOCKER_STATE="$state" \ - SMOKE_PROJECT=operator-owned ./scripts/local-vector-smoke.sh >"$tmp/out" 2>"$tmp/err"; then - echo "smoke accepted caller-controlled SMOKE_PROJECT" >&2 - exit 1 -fi -grep -q 'SMOKE_PROJECT is not accepted' "$tmp/err" -test ! -s "$log" - -: >"$log" -rm -f "$state" -PATH="$tmp:$PATH" FAKE_DOCKER_LOG="$log" FAKE_DOCKER_STATE="$state" \ - FAKE_COLLISION=1 ./scripts/local-vector-smoke.sh >"$tmp/out" 2>"$tmp/err" || true -grep -q 'refusing existing Compose project resources' "$tmp/err" -if grep -q 'compose.*up' "$log"; then - echo "smoke started after detecting a project collision" >&2 - exit 1 -fi - -: >"$log" -rm -f "$state" -PATH="$tmp:$PATH" FAKE_DOCKER_LOG="$log" FAKE_DOCKER_STATE="$state" \ - FAKE_MISMATCH_ON_CLEANUP=1 ./scripts/local-vector-smoke.sh >"$tmp/out" 2>"$tmp/err" || true -grep -q 'refusing cleanup of resource not owned by this smoke' "$tmp/err" -if grep -q 'down --volumes' "$log"; then - echo "smoke removed resources after ownership mismatch" >&2 - exit 1 -fi - -echo "local-vector smoke collision and cleanup ownership contracts passed." diff --git a/scripts/test-no-deployment-coupling-scope.sh b/scripts/test-no-deployment-coupling-scope.sh index 34be5edd..8685539f 100755 --- a/scripts/test-no-deployment-coupling-scope.sh +++ b/scripts/test-no-deployment-coupling-scope.sh @@ -9,10 +9,12 @@ trap 'rm -rf "$fixture"' EXIT HUP INT TERM new_fixture() { rm -rf "$fixture/repository" mkdir -p \ + "$fixture/repository/backend/src/workspaces" \ "$fixture/repository/deploy/env" \ "$fixture/repository/deploy/workspaces" \ "$fixture/repository/docker/smoke" \ "$fixture/repository/docs/install" \ + "$fixture/repository/docs/superpowers/specs" \ "$fixture/repository/docs/superpowers/plans" \ "$fixture/repository/frontend" \ "$fixture/repository/scripts" @@ -24,10 +26,14 @@ new_fixture() { printf '%s\n' 'THT_LLM_URL=https://llm.example.invalid' >"$fixture/repository/deploy/env/local.env.example" printf '%s\n' '# generic launcher' >"$fixture/repository/scripts/run-stack.sh" printf '%s\n' '// generic frontend configuration' >"$fixture/repository/frontend/vite.config.ts" + printf '%s\n' '// explicit descriptor migration module may mention pgvector during conversion' \ + >"$fixture/repository/backend/src/workspaces/migrate-legacy.ts" # These are the three intentionally allowed categories from the Task 10 boundary. printf '%s\n' 'historical omics_portal and Chirone record' \ >"$fixture/repository/docs/superpowers/plans/legacy.md" + printf '%s\n' 'historical pgvector rollout note' \ + >"$fixture/repository/docs/superpowers/specs/history.md" printf '%s\n' 'id: psd' >"$fixture/repository/deploy/workspaces/psd.yaml.example" printf '%s\n' '# migrate PSD sessions from /home/chirone' \ >"$fixture/repository/docker/session-migrate.sh" @@ -65,6 +71,12 @@ assert_detected frontend/vite.config.ts 'const base = "/omics_portal";' assert_detected scripts/test-qwen-network-config.sh 'require localllm_default' assert_detected scripts/test-provider-network.sh 'if (!config.networks?.localllm_default?.external) exit 1' assert_detected deploy/compose.psd-local.yaml 'services: {}' +assert_detected deploy/compose.local-vector.yaml 'services: {}' +assert_detected deploy/compose.preprocess-local-vector.yaml 'services: {}' +assert_detected scripts/run-stack.sh 'export THT_VECTOR_READER_PASSWORD_FILE=/run/secrets/vector-reader' +assert_detected deploy/env/local.env.example 'THT_OLLAMA_URL=http://ollama.example.invalid:11434' +assert_detected scripts/generate-override.sh 'THT_WS_NORTH_STAR_RESEARCH_VECTOR_API_KEY_SOURCE=/tmp/vector-key' +assert_detected scripts/test-contract.sh 'docker compose -f deploy/compose.local-vector.yaml --profile local-vector config' new_fixture mkdir -p "$fixture/bin" diff --git a/scripts/test-no-deployment-coupling.sh b/scripts/test-no-deployment-coupling.sh index d56e2474..0a8fa37e 100755 --- a/scripts/test-no-deployment-coupling.sh +++ b/scripts/test-no-deployment-coupling.sh @@ -57,6 +57,9 @@ if [[ -d scripts ]]; then contract_test_files+=("${file#./}") continue ;; + compose-with-preflight.sh|generate-connector-secrets-override.sh|unified-deployment-smoke.sh|vector-backup.sh|vector-restore.sh|vector-rotate-bootstrap-password.sh) + continue + ;; verify-*.sh) continue ;; esac operator_files+=("${file#./}") @@ -86,25 +89,51 @@ scan_category() { } for forbidden_file in \ + deploy/compose.local-vector.yaml \ + deploy/compose.preprocess-local-vector.yaml \ deploy/compose.production.yaml \ deploy/compose.psd-local.yaml.example \ deploy/compose.psd-local.yaml \ + deploy/sql/20-vector-roles.sql \ + deploy/vector/reconcile-roles.sh \ + deploy/vector/rotate-bootstrap-password.py \ + deploy/vector/secret-policy.sh \ + deploy/vector/vector-db-entrypoint.sh \ scripts/bootstrap-local-psd-docker-config.sh \ + scripts/local-vector-smoke.sh \ scripts/test-qwen-network-config.sh \ + scripts/test-local-vector-smoke-safety.sh \ + scripts/test-local-vector-smoke-live-collision.sh \ + scripts/test-vector-bootstrap-rotation.sh \ + scripts/test-vector-migration-image.sh \ + scripts/test-vector-secret-policy.sh \ harness/tests/test_psd_local_compose_contract.py; do [[ ! -e "$forbidden_file" ]] \ || offenders+=("active filename: $forbidden_file (superseded deployment contract)") done forbidden='omics_portal|chirone|localllm_default|datamart-builder|compose\.production\.yaml|compose\.psd-local\.yaml|\bpsd\b' +retired_semantic='local-vector|THT_VECTOR_(BOOTSTRAP|MIGRATOR|READER|WRITER|DATABASE|HOST|PORT|USER|ADMIN_URL|OPERATOR_ENV_FILE)|THT_OLLAMA_URL|VECTOR_API_KEY_(FILE|SOURCE)|vector-api-key|(^|[^A-Za-z0-9_])THT_VEC_(REST_URL|WRITE_REST_URL)' scan_category runtime "$forbidden" "${runtime_files[@]}" scan_category install "$forbidden" "${install_files[@]}" scan_category operator "$forbidden" "${operator_files[@]}" +scan_category runtime "$retired_semantic" "${runtime_files[@]}" +scan_category install "$retired_semantic" "${install_files[@]}" +scan_category operator "$retired_semantic" "${operator_files[@]}" # Contract tests legitimately quote forbidden names in negative assertions. Scan their positive # deployment wiring constructs instead, so a provider-owned network or retired overlay cannot be # required under a different test filename. positive_contract='networks(\?|\.)?\.?localllm_default|services(\?|\.)?\.?core(\?|\.)?\.?networks(\?|\.)?\.?localllm_default|docker compose[^\n]*(compose\.psd-local|compose\.production)|THT_PSD_[A-Z0-9_]*=' scan_category contract-test "$positive_contract" "${contract_test_files[@]}" +contract_scan_files=() +for file in "${contract_test_files[@]}"; do + case "${file#scripts/}" in + test-compose-secret-policy.sh|test-preprocess-compose-config.sh) continue ;; + esac + contract_scan_files+=("$file") +done +retired_semantic_contract='docker compose[^\n]*(compose\.local-vector|compose\.preprocess-local-vector)|THT_VECTOR_(BOOTSTRAP|MIGRATOR|READER|WRITER|DATABASE|HOST|PORT|USER|ADMIN_URL|OPERATOR_ENV_FILE)=|THT_OLLAMA_URL=|THT_WS_[A-Z0-9_]*_VECTOR_(TRANSPORT|API_KEY_(FILE|SOURCE))=|vector-api-key' +scan_category contract-test "$retired_semantic_contract" "${contract_scan_files[@]}" if [[ -f scripts/run-stack.sh ]]; then set +e @@ -128,4 +157,4 @@ if ((${#offenders[@]})); then exit 1 fi -echo "no active PSD, Chirone, or portal deployment coupling found." +echo "no active retired deployment or external semantic coupling found." diff --git a/scripts/test-preprocess-compose-config.sh b/scripts/test-preprocess-compose-config.sh index 9627e614..b350d31c 100755 --- a/scripts/test-preprocess-compose-config.sh +++ b/scripts/test-preprocess-compose-config.sh @@ -6,12 +6,7 @@ cd "$(dirname "$0")/.." tmp_bundle=$(mktemp) tmp_auth=$(mktemp) trap 'rm -f "$tmp_bundle" "$tmp_auth"' EXIT HUP INT TERM -cat >"$tmp_bundle" <<'EOF' -THT_VECTOR_BOOTSTRAP_PASSWORD=test-bootstrap -THT_VECTOR_MIGRATOR_PASSWORD=test-migrator -THT_VECTOR_READER_PASSWORD=test-reader -THT_VECTOR_WRITER_PASSWORD=test-writer -EOF +printf '%s\n' 'THT_MODEL_API_KEY=test-model' >"$tmp_bundle" chmod 0600 "$tmp_bundle" printf '%s\n' '{}' >"$tmp_auth" chmod 0600 "$tmp_auth" @@ -19,71 +14,44 @@ export THT_SECRETS_FILE="$tmp_bundle" export PI_AUTH_FILE="$tmp_auth" export THT_WORKSPACE_GIT_REMOTE=https://git.example.invalid/platform/thoth-workspaces.git -local_files="-f compose.yaml -f deploy/compose.local-vector.yaml -f deploy/compose.preprocess.yaml -f deploy/compose.preprocess-local-vector.yaml" -local_json=$(docker compose $local_files --profile local-vector --profile preprocess config --format json) +config_json=$(docker compose -f compose.yaml -f deploy/compose.preprocess.yaml --profile preprocess config --format json) -printf '%s' "$local_json" | python3 -c ' +printf '%s' "$config_json" | python3 -c ' import json, sys config = json.load(sys.stdin) services = config["services"] -assert "thothii_secrets" in config.get("secrets", {}), config.get("secrets") -assert "vector_bootstrap_password" not in config.get("secrets", {}) -assert "vector_migrator_password" not in config.get("secrets", {}) -assert "vector_reader_password" not in config.get("secrets", {}) -assert "vector_writer_password" not in config.get("secrets", {}) -for name, service in services.items(): - if name.startswith("vector-") or name.startswith("preprocess-") or name == "core": - assert any(item.get("target") == "thothii.secrets" for item in service.get("secrets", []) if isinstance(item, dict)), (name, service.get("secrets")) - assert "vector_reader_password" not in str(service) - assert "vector_writer_password" not in str(service) -for name in ("preprocess-evidence", "preprocess-dwh"): - dependency = services[name].get("depends_on", {}).get("vector-migrate") - assert dependency is not None, f"{name} does not depend on vector-migrate" - assert dependency["condition"] == "service_completed_successfully", dependency -' - -external_json=$(docker compose \ - -f compose.yaml -f deploy/compose.preprocess.yaml \ - --profile preprocess config --format json) - -printf '%s' "$external_json" | python3 -c ' -import json, sys - -config = json.load(sys.stdin) -services = config["services"] -assert "vector-db" not in services -assert "vector-migrate" not in services -assert "vector-reconcile" not in services -for name in ("preprocess-evidence", "preprocess-dwh"): +assert "qdrant" in services +assert "embedding" in services +assert "embedding-model-init" in services +assert "preprocess-evidence" in services +assert "preprocess-dwh" in services +for name in ("preprocess-evidence", "preprocess-dwh", "core"): service = services[name] - assert "depends_on" not in service - assert all(item.get("target") == "thothii.secrets" for item in service.get("secrets", []) if isinstance(item, dict)), service.get("secrets") - assert "vector_reader_password" not in str(service) - assert "vector_writer_password" not in str(service) + assert any(item.get("target") == "thothii.secrets" for item in service.get("secrets", []) if isinstance(item, dict)), (name, service.get("secrets")) + assert "THT_OLLAMA_URL" not in str(service) + assert "THT_VECTOR_" not in str(service) +assert services["preprocess-evidence"]["depends_on"]["qdrant"]["condition"] == "service_healthy" +assert services["preprocess-evidence"]["depends_on"]["embedding-model-init"]["condition"] == "service_completed_successfully" +assert "depends_on" not in services["preprocess-dwh"] or "vector-migrate" not in str(services["preprocess-dwh"]["depends_on"]) ' python3 - <<'PY' -import os from pathlib import Path -os.environ.update({ - "THT_DB_NAME": "thoth", - "THT_DWH_REST_URL": "http://dwh.invalid", - "THT_DWH_API_KEY": "dwh", - "THT_VECTOR_DATABASE": "thoth", - "THT_VECTOR_READER_USER": "reader", - "THT_VECTOR_WRITER_USER": "writer", - "THT_VECTOR_READER_PASSWORD_FILE": "/tmp/generated-reader", - "THT_VECTOR_WRITER_PASSWORD_FILE": "/tmp/generated-writer", - "THT_DOCS_ROOT": "/data/source", - "THT_OLLAMA_URL": "http://ollama.invalid", -}) -text = Path("deploy/workspaces/local-vector.yaml").read_text() -assert "password_file: ${THT_VECTOR_READER_PASSWORD_FILE}" in text -assert "password_file: ${THT_VECTOR_WRITER_PASSWORD_FILE}" in text -assert "${THT_SECRETS_FILE}" not in text -print("local-vector workspace resolution contract: ok") +evidence = Path("deploy/workspaces/preprocess-evidence.yaml").read_text() +dwh = Path("deploy/workspaces/preprocess-dwh.yaml").read_text() +assert "type: qdrant" in evidence +assert "base_url: http://qdrant:6333" in evidence +assert "provider: ollama_internal" in evidence +assert "base_url: http://embedding:11434" in evidence +assert "qwen3-embedding:0.6b" in evidence +assert "THT_VECTOR_" not in evidence +assert "THT_OLLAMA_URL" not in evidence +assert "pgvector" not in evidence +assert "type: postgres_direct" in dwh +assert "THT_PREPROCESS_DWH_HOST" in dwh +print("preprocess workspace contract: ok") PY echo "preprocess compose config: ok" diff --git a/scripts/test-task13-runtime-fixtures.sh b/scripts/test-task13-runtime-fixtures.sh index 278240f3..c7c4b476 100755 --- a/scripts/test-task13-runtime-fixtures.sh +++ b/scripts/test-task13-runtime-fixtures.sh @@ -43,7 +43,7 @@ mkdir -p "$TASK13_REMOTE" workspace="$fixture/task13-smoke.yaml" cat >"$workspace" <<'EOF' workspace: - schema_version: 2 + schema_version: 3 id: task13-smoke name: Task 13 Smoke language: en @@ -54,17 +54,14 @@ dwh: supported_transports: [postgres_direct] semantic_index: vector_store: - engine: pgvector - database: vectors - schema: public - collection: task13_documents - dimensions: 8 + engine: qdrant + collection: task13-smoke + dimensions: 1024 distance: cosine - supported_transports: [pgvector_direct] embedding: - provider: ollama_compatible - model: task13-embedding - dimensions: 8 + provider: ollama_internal + model: qwen3-embedding:0.6b + dimensions: 1024 llm_policy: default: local-qwen/task13-smoke allowed: [local-qwen/task13-smoke] diff --git a/scripts/test-vector-backup-restore-safety.sh b/scripts/test-vector-backup-restore-safety.sh index 76845eea..bf0cf0e0 100755 --- a/scripts/test-vector-backup-restore-safety.sh +++ b/scripts/test-vector-backup-restore-safety.sh @@ -86,21 +86,4 @@ PATH="$fakebin:$PATH" RESTORE_LOG="$tmp/restore.log" ./scripts/vector-restore.sh grep -q -- '--single-transaction' "$tmp/restore.log" grep -q -- '--exit-on-error' "$tmp/restore.log" -# The live restore smoke must follow the packaged migration set instead of a stale -# hard-coded count when a new migration is added. -if grep -Eq 'vector_(bootstrap|migrator|reader|writer)_password' \ - deploy/compose.local-vector.yaml deploy/compose.preprocess-local-vector.yaml; then - echo "local-vector Compose still declares legacy per-password secrets" >&2 - exit 1 -fi -grep -Fq 'thothii_secrets' deploy/compose.local-vector.yaml -grep -Fq 'thothii_secrets' deploy/compose.preprocess-local-vector.yaml -if grep -Fq 'SELECT count(*) = 3 FROM public.tht_vector_migrations' \ - scripts/local-vector-smoke.sh; then - echo "local vector smoke hard-codes the pre-004 migration count" >&2 - exit 1 -fi -grep -Fq 'expected_migrations=' scripts/local-vector-smoke.sh -grep -Fq 'applied_migrations=' scripts/local-vector-smoke.sh - echo "vector backup/restore filesystem, identity, and transaction contracts passed." diff --git a/scripts/test-vector-bootstrap-rotation.sh b/scripts/test-vector-bootstrap-rotation.sh deleted file mode 100755 index 04c0f3f7..00000000 --- a/scripts/test-vector-bootstrap-rotation.sh +++ /dev/null @@ -1,125 +0,0 @@ -#!/bin/sh -set -eu - -cd "$(dirname "$0")/.." -tmp=$(mktemp -d) -trap 'rm -rf "$tmp"' EXIT HUP INT TERM - -fake="$tmp/docker" -log="$tmp/docker.log" -cat >"$fake" <<'SH' -#!/bin/sh -set -eu -invocation=$* -test "${1:-}" = compose -shift -env_file= -while [ "$#" -gt 0 ]; do - case "$1" in - --env-file) - [ "$#" -ge 2 ] || exit 64 - env_file=$2 - shift 2 - ;; - --env-file=*) - env_file=${1#--env-file=} - shift - ;; - *) shift ;; - esac -done -[ -n "$env_file" ] && [ -f "$env_file" ] || { - echo "fake docker rejected missing env file: $env_file" >&2 - exit 64 -} -printf '%s:%s\n' "${THT_VECTOR_BOOTSTRAP_USER:-unset}" "$invocation" >>"$FAKE_DOCKER_LOG" -exit "${FAKE_DOCKER_EXIT:-0}" -SH -chmod 0755 "$fake" - -printf '%s' old-password >"$tmp/old" -printf '%s' "new-'quoted-\$-password" >"$tmp/new" -cp "$tmp/old" "$tmp/original" - -printf 'invalid password\n' >"$tmp/whitespace" -printf '%s\n' 'THT_WORKSPACE_GIT_REMOTE=https://git.example.invalid/thoth-workspaces.git' \ - >"$tmp/operator.env" -printf '%s\n' 'THT_WORKSPACE_GIT_REMOTE=https://git.example.invalid/unsafe.git' \ - >"$tmp/unsafe.env" -ln -s "$tmp/operator.env" "$tmp/operator-link.env" -chmod 0600 "$tmp/old" "$tmp/new" "$tmp/original" "$tmp/whitespace" "$tmp/operator.env" -chmod 0660 "$tmp/unsafe.env" - -if PATH="$tmp:$PATH" FAKE_DOCKER_LOG="$log" \ - ./scripts/vector-rotate-bootstrap-password.sh "$tmp/old" "$tmp/new" \ - >"$tmp/out" 2>"$tmp/err"; then - echo "rotation silently assumed an operator env file" >&2 - exit 1 -fi -grep -q 'requires --env-file or THT_VECTOR_OPERATOR_ENV_FILE' "$tmp/err" -test ! -s "$log" - -if PATH="$tmp:$PATH" FAKE_DOCKER_LOG="$log" \ - ./scripts/vector-rotate-bootstrap-password.sh --env-file "$tmp/missing.env" \ - "$tmp/old" "$tmp/new" >"$tmp/out" 2>"$tmp/err"; then - echo "rotation accepted a nonexistent operator env file" >&2 - exit 1 -fi -grep -q 'operator env must be a readable regular file' "$tmp/err" -test ! -s "$log" - -for unsafe_env in "$tmp/unsafe.env" "$tmp/operator-link.env"; do - if PATH="$tmp:$PATH" FAKE_DOCKER_LOG="$log" \ - ./scripts/vector-rotate-bootstrap-password.sh --env-file "$unsafe_env" \ - "$tmp/old" "$tmp/new" >"$tmp/out" 2>"$tmp/err"; then - echo "rotation accepted unsafe operator env file $unsafe_env" >&2 - exit 1 - fi - test ! -s "$log" -done - -: >"$log" -if PATH="$tmp:$PATH" FAKE_DOCKER_LOG="$log" THT_VECTOR_BOOTSTRAP_USER=custom_admin \ - ./scripts/vector-rotate-bootstrap-password.sh --env-file "$tmp/operator.env" \ - "$tmp/old" "$tmp/whitespace" \ - >"$tmp/out" 2>"$tmp/err"; then - echo "rotation accepted a whitespace-containing secret" >&2 - exit 1 -fi -cmp "$tmp/old" "$tmp/original" -test ! -s "$log" -if find "$tmp" -name 'old.rotate.*' -print | grep -q .; then - echo "rotation staged a deployment file before secret validation" >&2 - exit 1 -fi - -if PATH="$tmp:$PATH" FAKE_DOCKER_LOG="$log" FAKE_DOCKER_EXIT=1 \ - ./scripts/vector-rotate-bootstrap-password.sh --env-file "$tmp/operator.env" \ - "$tmp/old" "$tmp/new" \ - >"$tmp/out" 2>"$tmp/err"; then - echo "rotation unexpectedly succeeded when database verification failed" >&2 - exit 1 -fi -cmp "$tmp/old" "$tmp/original" - -: >"$log" -PATH="$tmp:$PATH" FAKE_DOCKER_LOG="$log" THT_VECTOR_BOOTSTRAP_USER=custom_admin \ - ./scripts/vector-rotate-bootstrap-password.sh --env-file "$tmp/operator.env" \ - "$tmp/old" "$tmp/new" \ - >"$tmp/out" 2>"$tmp/err" -cmp "$tmp/old" "$tmp/new" -grep -q -- "--env-file $tmp/operator.env" "$log" -grep -q '/run/secrets/bootstrap-old:ro' "$log" -grep -q '/run/secrets/bootstrap-new:ro' "$log" -grep -q '^custom_admin:' "$log" -grep -q 'atomically replaced only after verified database login' "$tmp/out" - -printf '%s' old-password >"$tmp/old" -: >"$log" -PATH="$tmp:$PATH" FAKE_DOCKER_LOG="$log" \ - THT_VECTOR_OPERATOR_ENV_FILE="$tmp/operator.env" \ - ./scripts/vector-rotate-bootstrap-password.sh "$tmp/old" "$tmp/new" \ - >"$tmp/out" 2>"$tmp/err" -grep -q -- "--env-file $tmp/operator.env" "$log" - -echo "bootstrap rotation env propagation, validation, ordering, and failure contracts passed." diff --git a/scripts/test-vector-migration-image.sh b/scripts/test-vector-migration-image.sh deleted file mode 100755 index ed8da3e7..00000000 --- a/scripts/test-vector-migration-image.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/bin/sh -set -eu - -image=${1:?usage: test-vector-migration-image.sh IMAGE [PLATFORM]} -platform=${2:-${PLATFORM:-linux/arm64}} -repo_root=$(CDPATH= cd -- "$(dirname "$0")/.." && pwd) -slug=$$ -network="thoth-vector-migration-$slug" -database="thoth-vector-db-$slug" - -cleanup() { - docker rm --force "$database" >/dev/null 2>&1 || true - docker network rm "$network" >/dev/null 2>&1 || true -} -trap cleanup EXIT INT TERM - -docker network create "$network" >/dev/null -docker run --detach --rm --platform "$platform" --name "$database" --network "$network" \ - -e POSTGRES_DB=thoth -e POSTGRES_USER=thoth_admin -e POSTGRES_PASSWORD=test-only \ - pgvector/pgvector:pg16 >/dev/null - -attempt=0 -until docker exec "$database" pg_isready -U thoth_admin -d thoth >/dev/null 2>&1; do - attempt=$((attempt + 1)) - if [ "$attempt" -ge 30 ]; then - echo "pgvector test database did not become ready" >&2 - exit 1 - fi - sleep 1 -done - -database_url="postgresql+psycopg2://thoth_admin:test-only@$database:5432/thoth" -applied=$(docker run --rm --platform "$platform" --network "$network" \ - --entrypoint /opt/venv/bin/tht -e THT_VECTOR_ADMIN_URL="$database_url" \ - "$image" vector migrate --json) -status=$(docker run --rm --platform "$platform" --network "$network" \ - --entrypoint /opt/venv/bin/tht -e THT_VECTOR_ADMIN_URL="$database_url" \ - "$image" vector migrate --status --json) - -expected_versions=$(find "$repo_root/harness/tht/migrations/vector" -type f -name '[0-9][0-9][0-9]_*.sql' \ - | sed 's|.*/||; s|_.*||' \ - | LC_ALL=C sort \ - | awk 'BEGIN { separator = ""; printf "[" } { printf "%s\"%s\"", separator, $0; separator = ", " } END { print "]" }') -test "$expected_versions" != '[]' -expected="{\"applied\": $expected_versions, \"drifted\": [], \"pending\": []}" -test "$applied" = "$expected" -test "$status" = "$expected" -echo "core image vector migration discovery/status smoke passed" diff --git a/scripts/test-vector-secret-policy.sh b/scripts/test-vector-secret-policy.sh deleted file mode 100755 index fc43fc49..00000000 --- a/scripts/test-vector-secret-policy.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/sh -set -eu - -cd "$(dirname "$0")/.." -tmp=$(mktemp -d) -trap 'rm -rf "$tmp"' EXIT HUP INT TERM - -. ./deploy/vector/secret-policy.sh - -: >"$tmp/empty" -printf 'has newline\n' >"$tmp/newline" -printf 'has space' >"$tmp/space" -printf 'safe-quoted-\047-dollar-$' >"$tmp/valid" -printf 'docker-secret' >"$tmp/docker" -printf 'owner-readonly' >"$tmp/readonly" -printf 'too-open' >"$tmp/open" -printf '# comment\n\nTHT_VECTOR_READER_PASSWORD=reader\nTHT_VECTOR_WRITER_PASSWORD=writer\n' >"$tmp/bundle" -printf 'THT_VECTOR_READER_PASSWORD=reader\nTHT_VECTOR_WRITER_PASSWORD=writer\nTHT_DWH_API_KEY=one\nTHT_DWH_API_KEY=two\n' >"$tmp/duplicate-bundle" -printf 'THT_VECTOR_READER_PASSWORD=reader\r\nTHT_VECTOR_WRITER_PASSWORD=writer\r\n' >"$tmp/crlf-bundle" -awk 'BEGIN { printf "THT_VECTOR_READER_PASSWORD="; for (i = 1; i <= 16385; i++) printf "x"; print "" }' >"$tmp/long-line-bundle" -awk 'BEGIN { for (i = 1; i <= 70000; i++) print "# filler" }' >"$tmp/large-bundle" -chmod 0600 "$tmp/valid" -chmod 0444 "$tmp/docker" -chmod 0400 "$tmp/readonly" -chmod 0640 "$tmp/open" -chmod 0600 "$tmp/bundle" "$tmp/duplicate-bundle" "$tmp/crlf-bundle" "$tmp/long-line-bundle" "$tmp/large-bundle" - -for invalid in empty newline space; do - if validate_secret_file "$tmp/$invalid" "$invalid" >/dev/null 2>&1; then - echo "secret policy accepted $invalid" >&2 - exit 1 - fi -done -validate_secret_file "$tmp/valid" valid -validate_secret_file "$tmp/readonly" readonly -if validate_secret_file "$tmp/docker" docker >/dev/null 2>&1; then - echo "secret policy accepted world-readable host secret" >&2 - exit 1 -fi -if validate_secret_file "$tmp/open" open >/dev/null 2>&1; then - echo "secret policy accepted group-readable host secret" >&2 - exit 1 -fi -test "$(read_secret_file "$tmp/valid" valid)" = "safe-quoted-'-dollar-$" -test "$(read_bundle_secret "$tmp/bundle" THT_VECTOR_READER_PASSWORD)" = reader -test "$(read_bundle_secret "$tmp/crlf-bundle" THT_VECTOR_READER_PASSWORD)" = reader -if read_bundle_secret "$tmp/duplicate-bundle" THT_VECTOR_READER_PASSWORD >/dev/null 2>&1; then - echo "secret policy accepted a duplicate unrelated bundle key" >&2 - exit 1 -fi -for invalid_bundle in long-line-bundle large-bundle; do - if read_bundle_secret "$tmp/$invalid_bundle" THT_VECTOR_READER_PASSWORD >/dev/null 2>&1; then - echo "secret policy accepted oversized $invalid_bundle" >&2 - exit 1 - fi -done - -echo "shared vector secret policy contracts passed." diff --git a/scripts/unified-deployment-smoke.sh b/scripts/unified-deployment-smoke.sh index b2c21836..f7c34c24 100755 --- a/scripts/unified-deployment-smoke.sh +++ b/scripts/unified-deployment-smoke.sh @@ -285,12 +285,6 @@ services: THT_WS_TASK13_SMOKE_DWH_PORT: "5432" THT_WS_TASK13_SMOKE_DWH_USER: task13_reader THT_WS_TASK13_SMOKE_DWH_PASSWORD_FILE: /run/secrets/task13-runtime-password - THT_WS_TASK13_SMOKE_VECTOR_TRANSPORT: pgvector_direct - THT_WS_TASK13_SMOKE_VECTOR_HOST: vector.task13.invalid - THT_WS_TASK13_SMOKE_VECTOR_PORT: "5432" - THT_WS_TASK13_SMOKE_VECTOR_USER: task13_vector_reader - THT_WS_TASK13_SMOKE_VECTOR_PASSWORD_FILE: /run/secrets/task13-runtime-password - THT_WS_TASK13_SMOKE_EMBEDDING_BASE_URL: http://$TASK13_LLM_CONTAINER:9000 labels: io.thothii.task13.run: "$TASK13_RUN_ID" volumes: !override @@ -399,12 +393,6 @@ services: THT_WS_TASK13_SMOKE_DWH_PORT: "5432" THT_WS_TASK13_SMOKE_DWH_USER: task13_reader THT_WS_TASK13_SMOKE_DWH_PASSWORD_FILE: /run/secrets/task13-runtime-password - THT_WS_TASK13_SMOKE_VECTOR_TRANSPORT: pgvector_direct - THT_WS_TASK13_SMOKE_VECTOR_HOST: vector.task13.invalid - THT_WS_TASK13_SMOKE_VECTOR_PORT: "5432" - THT_WS_TASK13_SMOKE_VECTOR_USER: task13_vector_reader - THT_WS_TASK13_SMOKE_VECTOR_PASSWORD_FILE: /run/secrets/task13-runtime-password - THT_WS_TASK13_SMOKE_EMBEDDING_BASE_URL: https://embedding.task13.invalid labels: io.thothii.task13.run: "$TASK13_RUN_ID" volumes: @@ -461,7 +449,7 @@ task13_seed_registry() { task13_run_logged "initialize workspace seed" git -C "$TASK13_SEED" init --initial-branch=main cat >"$TASK13_SEED/workspaces/task13-smoke.yaml" <<'EOF' workspace: - schema_version: 2 + schema_version: 3 id: task13-smoke name: Task 13 Smoke language: en @@ -472,17 +460,14 @@ dwh: supported_transports: [postgres_direct] semantic_index: vector_store: - engine: pgvector - database: vectors - schema: public - collection: task13_documents - dimensions: 8 + engine: qdrant + collection: task13-smoke + dimensions: 1024 distance: cosine - supported_transports: [pgvector_direct] embedding: - provider: ollama_compatible - model: task13-embedding - dimensions: 8 + provider: ollama_internal + model: qwen3-embedding:0.6b + dimensions: 1024 llm_policy: default: local-qwen/task13-smoke allowed: [local-qwen/task13-smoke] diff --git a/scripts/vector-backup.sh b/scripts/vector-backup.sh index 61b6c586..8379a849 100755 --- a/scripts/vector-backup.sh +++ b/scripts/vector-backup.sh @@ -2,7 +2,7 @@ set -eu root=$(CDPATH= cd -- "$(dirname "$0")/.." && pwd) -. "$root/deploy/vector/secret-policy.sh" +. "$root/scripts/secret-file-utils.sh" usage() { echo "usage: $0 --host HOST --database DB --user USER --password-file FILE --output FILE [--port PORT]" >&2 diff --git a/scripts/vector-restore.sh b/scripts/vector-restore.sh index 0e910544..c0c7ceb6 100755 --- a/scripts/vector-restore.sh +++ b/scripts/vector-restore.sh @@ -2,7 +2,7 @@ set -eu root=$(CDPATH= cd -- "$(dirname "$0")/.." && pwd) -. "$root/deploy/vector/secret-policy.sh" +. "$root/scripts/secret-file-utils.sh" usage() { echo "usage: $0 --active-host HOST --active-database DB --active-user USER --active-password-file FILE --target-host HOST --target-database DB --target-user USER --target-password-file FILE --input FILE [--active-port PORT] [--target-port PORT] [--force-nonempty]" >&2 diff --git a/scripts/vector-rotate-bootstrap-password.sh b/scripts/vector-rotate-bootstrap-password.sh old mode 100755 new mode 100644 index 3dfd1011..ded76562 --- a/scripts/vector-rotate-bootstrap-password.sh +++ b/scripts/vector-rotate-bootstrap-password.sh @@ -1,89 +1,6 @@ #!/bin/sh set -eu -cd "$(dirname "$0")/.." -. ./deploy/vector/secret-policy.sh - -usage() { - echo "usage: $0 [--env-file OPERATOR_ENV] OLD_SECRET_FILE NEW_SECRET_FILE" >&2 - echo "set THT_VECTOR_OPERATOR_ENV_FILE instead of --env-file when required by automation" >&2 - exit 2 -} - -operator_env=${THT_VECTOR_OPERATOR_ENV_FILE:-} -while [ "$#" -gt 0 ]; do - case "$1" in - --env-file) - [ "$#" -ge 2 ] || usage - operator_env=$2 - shift 2 - ;; - --env-file=*) - operator_env=${1#--env-file=} - shift - ;; - --) shift; break ;; - -*) usage ;; - *) break ;; - esac -done - -if [ -z "$operator_env" ]; then - echo "rotation requires --env-file or THT_VECTOR_OPERATOR_ENV_FILE; there is no implicit default" >&2 - exit 2 -fi -if [ -L "$operator_env" ] || [ ! -f "$operator_env" ] || [ ! -r "$operator_env" ]; then - echo "operator env must be a readable regular file, not a symlink: $operator_env" >&2 - exit 2 -fi -operator_env_mode=$(stat -c '%a' "$operator_env" 2>/dev/null || stat -f '%Lp' "$operator_env" 2>/dev/null) || { - echo "cannot inspect operator env permissions: $operator_env" >&2 - exit 2 -} -if [ $((0$operator_env_mode & 022)) -ne 0 ]; then - echo "operator env must not be writable by group or other users: $operator_env" >&2 - exit 2 -fi - -if [ "$#" -ne 2 ]; then - usage -fi - -absolute_file() { - directory=$(CDPATH= cd -- "$(dirname -- "$1")" && pwd) - printf '%s/%s\n' "$directory" "$(basename -- "$1")" -} - -operator_env=$(absolute_file "$operator_env") - -old_secret=$(absolute_file "$1") -new_secret=$(absolute_file "$2") -validate_secret_file "$old_secret" old_bootstrap_secret -validate_secret_file "$new_secret" new_bootstrap_secret -if [ "$old_secret" -ef "$new_secret" ]; then - echo "old and new secret files must be distinct" >&2 - exit 2 -fi - -project=${COMPOSE_PROJECT_NAME:-thothii} -replacement=$(mktemp "${old_secret}.rotate.XXXXXX") -trap 'rm -f "$replacement"' EXIT HUP INT TERM -cp "$new_secret" "$replacement" -chmod 0600 "$replacement" - -docker compose --env-file "$operator_env" -f compose.yaml -f deploy/compose.local-vector.yaml \ - --project-name "$project" --profile local-vector run --rm --no-deps \ - --user 0:0 \ - --entrypoint /opt/venv/bin/python \ - --volume "$old_secret:/run/secrets/bootstrap-old:ro" \ - --volume "$new_secret:/run/secrets/bootstrap-new:ro" \ - --volume "$(pwd)/deploy/vector/rotate-bootstrap-password.py:/opt/thoth/rotate-bootstrap-password.py:ro" \ - core /opt/thoth/rotate-bootstrap-password.py \ - /run/secrets/bootstrap-old /run/secrets/bootstrap-new - -mv -f "$replacement" "$old_secret" -trap - EXIT HUP INT TERM - -echo "Deployment bootstrap secret atomically replaced only after verified database login." -echo "Re-run with the same operator env file: $operator_env" -echo "docker compose --env-file OPERATOR_ENV -f compose.yaml -f deploy/compose.local-vector.yaml --project-name $project --profile local-vector up --wait vector-reconcile vector-migrate core" +echo "vector bootstrap password rotation is retired in this repository; local pgvector deployment is no longer supported." >&2 +echo "If you still need legacy pgvector data, export it with the retained migration utilities and migrate off-repository." >&2 +exit 2 diff --git a/scripts/verify-container-images.sh b/scripts/verify-container-images.sh index a277a018..6b592fc4 100755 --- a/scripts/verify-container-images.sh +++ b/scripts/verify-container-images.sh @@ -24,7 +24,6 @@ test "$(docker image inspect --format '{{ index .Config.Labels "org.opencontaine docker run --rm --platform "$platform" --entrypoint /app/docker/smoke/core-smoke.sh \ "$core_image" -./scripts/test-vector-migration-image.sh "$core_image" "$platform" docker run --rm --platform "$platform" "$frontend_image" frontend-config-smoke ./docker/smoke/frontend-policy-smoke.sh if docker run --rm --platform "$platform" -e THOTH_PUBLIC_EXPOSURE=true -e AUTH_MODE=none \ diff --git a/scripts/verify-workspace-install-docs.sh b/scripts/verify-workspace-install-docs.sh index a936d516..3a8ed450 100755 --- a/scripts/verify-workspace-install-docs.sh +++ b/scripts/verify-workspace-install-docs.sh @@ -1122,8 +1122,8 @@ verify_local_installation_example() { node - "$rendered" <<'NODE' const fs = require("fs"); const config = JSON.parse(fs.readFileSync(process.argv[2], "utf8")); -if (Object.keys(config.services).sort().join(",") !== "core,frontend") { - throw new Error("local installation example must render exactly core,frontend"); +if (Object.keys(config.services).sort().join(",") !== "core,embedding,embedding-model-init,frontend,qdrant") { + throw new Error("local installation example must render the internal semantic stack"); } const output = JSON.stringify(config); for (const secret of [ @@ -1244,8 +1244,8 @@ verify_server_installation_example() { node - "$rendered" <<'NODE' const fs = require("fs"); const config = JSON.parse(fs.readFileSync(process.argv[2], "utf8")); -if (Object.keys(config.services).sort().join(",") !== "core,frontend") { - throw new Error("server installation example must render exactly core,frontend"); +if (Object.keys(config.services).sort().join(",") !== "core,embedding,embedding-model-init,frontend,qdrant") { + throw new Error("server installation example must render the internal semantic stack"); } const core = config.services.core; const frontend = config.services.frontend; @@ -1346,7 +1346,6 @@ verify_compose_fixtures() { write_private "$fixture/git-ssh-key" 'fixture-git-ssh-key' write_private "$fixture/git-known-hosts" 'fixture-git-known-hosts' write_private "$fixture/dwh-password" 'fixture-dwh-password' - write_private "$fixture/vector-api-key" 'fixture-vector-api-key' write_private "$fixture/session-runtime-password" 'fixture-session-runtime-password' write_private "$fixture/session-migrator-password" 'fixture-session-migrator-password' write_private "$fixture/session-ca.pem" 'fixture-session-ca' @@ -1355,8 +1354,6 @@ verify_compose_fixtures() { printf '%s\n' \ 'THT_WS_NORTH_STAR_RESEARCH_DWH_TRANSPORT=postgres_direct' \ 'THT_WS_NORTH_STAR_RESEARCH_DWH_PASSWORD_FILE=/run/secrets/north-star-research-dwh-password' \ - 'THT_WS_NORTH_STAR_RESEARCH_VECTOR_TRANSPORT=rest_api' \ - 'THT_WS_NORTH_STAR_RESEARCH_VECTOR_API_KEY_FILE=/run/secrets/north-star-research-vector-api-key' \ >"$fixture/workspace-bindings.env" printf '%s\n' \ @@ -1367,7 +1364,6 @@ verify_compose_fixtures() { "THT_WORKSPACE_GIT_SSH_KEY_FILE=$fixture/git-ssh-key" \ "THT_WORKSPACE_GIT_KNOWN_HOSTS_FILE=$fixture/git-known-hosts" \ "THT_WS_NORTH_STAR_RESEARCH_DWH_PASSWORD_SOURCE=$fixture/dwh-password" \ - "THT_WS_NORTH_STAR_RESEARCH_VECTOR_API_KEY_SOURCE=$fixture/vector-api-key" \ "THT_DATA_ROOT=$fixture/data" \ "THT_PI_STATE_ROOT=$fixture/pi-state" \ "THT_WORKSPACE_REGISTRY_ROOT=$fixture/workspace-registry" \ @@ -1407,8 +1403,8 @@ verify_compose_fixtures() { const fs = require("fs"); const [path, profile] = process.argv.slice(2); const config = JSON.parse(fs.readFileSync(path, "utf8")); -if (Object.keys(config.services).sort().join(",") !== "core,frontend") { - throw new Error(profile + ": mandatory stack must be exactly core,frontend"); +if (Object.keys(config.services).sort().join(",") !== "core,embedding,embedding-model-init,frontend,qdrant") { + throw new Error(profile + ": mandatory stack must include the internal semantic services"); } const core = config.services.core; for (const target of [ @@ -1422,7 +1418,6 @@ for (const target of [ } for (const [name, value] of Object.entries({ THT_WS_NORTH_STAR_RESEARCH_DWH_PASSWORD_FILE: "/run/secrets/north-star-research-dwh-password", - THT_WS_NORTH_STAR_RESEARCH_VECTOR_API_KEY_FILE: "/run/secrets/north-star-research-vector-api-key", })) { if (core.environment?.[name] !== value) throw new Error(profile + ": missing binding " + name); } @@ -1430,7 +1425,6 @@ const secretTargets = new Set((core.secrets || []).map((secret) => secret.target for (const target of [ "thothii.secrets", "north-star-research-dwh-password", - "north-star-research-vector-api-key", ]) { if (!secretTargets.has(target)) throw new Error(profile + ": missing secret target " + target); } @@ -1445,7 +1439,7 @@ if ((config.services.frontend.secrets || []).length !== 0) { const rendered = JSON.stringify(config); for (const value of [ "fixture-native-auth-key", "fixture-model-api-key", "fixture-git-ssh-key", - "fixture-git-known-hosts", "fixture-dwh-password", "fixture-vector-api-key", + "fixture-git-known-hosts", "fixture-dwh-password", "fixture-session-runtime-password", "fixture-session-migrator-password", "fixture-session-ca", ]) { if (rendered.includes(value)) throw new Error(profile + ": rendered Compose leaked " + value);