From a6dbe1d023d1a90b6f528a26fd034f2f5c476565 Mon Sep 17 00:00:00 2001 From: mptyl Date: Sat, 8 Aug 2026 19:27:05 +0200 Subject: [PATCH] fix: retire active pgvector artifacts --- README.md | 43 ++-- backend/src/app.ts | 7 +- backend/src/workspaces/diagnostics.ts | 232 +++++++++++++++++- backend/test/workspaces-diagnostics.test.ts | 89 +++++++ deploy/workspaces/example.yaml | 26 +- deploy/workspaces/local-vector.yaml | 48 ---- deploy/workspaces/psd.yaml.example | 34 ++- .../workspaces/server-sessions.yaml.example | 15 +- docs/install/local-workspace-registry.md | 19 +- docs/install/server-workspace-registry.md | 19 +- scripts/test-canonical-install-compose.sh | 5 +- scripts/test-no-deployment-coupling-scope.sh | 10 +- scripts/test-no-deployment-coupling.sh | 8 +- scripts/test-vector-backup-restore-safety.sh | 149 ++++++----- scripts/vector-backup.sh | 81 ++++-- scripts/vector-restore.sh | 137 ++++++----- 16 files changed, 630 insertions(+), 292 deletions(-) delete mode 100644 deploy/workspaces/local-vector.yaml diff --git a/README.md b/README.md index 7e048441..9848db26 100644 --- a/README.md +++ b/README.md @@ -141,7 +141,7 @@ an independent 32-minute outer timeout and does not retry a failed command. Current release status (2026-08-05): clean-root render/setup and the production runtime-binding resolver contracts are green. The server fixture supplies all four private trusted claims, including exact non-admin value `0`, and a focused test proves nginx normalization produces the -accepted non-admin backend principal. Canonical schema-v2 registry descriptors now pass through +accepted non-admin backend principal. Canonical schema-v3 registry descriptors now pass through one backend-owned, secret-safe runtime handoff for inventory and session execution; canonical identity and durable session/artifact/index roots are retained. The fresh update-only smoke passed bad-candidate mutation, automatic `rolled_back` compensation, exact prior-image restoration, @@ -181,7 +181,7 @@ docker compose --env-file deploy/env/local.env \ -f deploy/compose.preprocess.yaml --profile preprocess run --rm preprocess-evidence ``` -Replace the final service with `preprocess-dwh` when required. The overlay makes each job wait for the vector +Replace the final service with `preprocess-dwh` when required. The overlay makes each job wait for the internal Qdrant service health checks and embedding model initialization; no separate semantic-service startup is required. @@ -194,37 +194,36 @@ egress policy. Store access key, secret key, and session token as secret referen deployment configuration—never in Compose environment values or source URIs. Discovery and reads are bounded by configured page, object, and byte limits. -Create a versioned PostgreSQL custom-format backup (the filename is operator-controlled, so use -an immutable timestamp or release identifier): +Create a versioned Qdrant volume backup for one exact Compose project (the filename is +operator-controlled, so use an immutable timestamp or release identifier): ```sh ./scripts/vector-backup.sh \ - --host 127.0.0.1 --port 5432 --database thoth --user thoth_backup \ - --password-file /secure/thoth/vector-backup-password \ - --output /secure/backups/thoth-vectors-2026-07-12.dump + --project-name thothii \ + --output /secure/backups/thoth-qdrant-2026-08-08.tar ``` -The dump contains the three allowlisted `vectors` tables, their data and ACLs, plus the -`public.tht_vector_migrations` ledger. Login roles and passwords are deliberately not copied: -provision/reconcile the approved role names on the target first, and install the `vector` -extension in its `vectors` schema. The target must otherwise contain no vector tables or ledger. +The script resolves exactly one Docker volume with the labels +`com.docker.compose.project=` and `com.docker.compose.volume=qdrant-data`, stops the +`qdrant` service if it is running, archives that volume's persistent contents, then restores the +prior service state. It never performs global Docker cleanup and refuses to overwrite an existing +archive path. -Restore always names both the currently active source and a target on a physically distinct -PostgreSQL cluster. The script compares PostgreSQL system identity, so host aliases or a different -database in the active cluster cannot bypass the guard. It refuses a non-empty target unless -`--force-nonempty` is explicit, and the clean restore is one transaction: +Restore targets that same exact project-scoped `qdrant-data` volume. Because restore replaces the +persistent Qdrant data in place, it requires an explicit confirmation that exactly repeats the +Compose project name: ```sh ./scripts/vector-restore.sh \ - --active-host vector-db --active-database thoth --active-user thoth_backup \ - --active-password-file /secure/thoth/vector-active-password \ - --target-host vector-db-restore --target-database thoth --target-user thoth_restore \ - --target-password-file /secure/thoth/vector-restore-password \ - --input /secure/backups/thoth-vectors-2026-07-12.dump + --project-name thothii \ + --input /secure/backups/thoth-qdrant-2026-08-08.tar \ + --confirm-project thothii ``` -After restore, run `tht vector migrate --status --json`, adapter health, and a known retrieval -query against the target before changing any migration/export endpoint. +The restore script stops `qdrant`, validates the exact labeled target, stages the current volume +contents for rollback, extracts the requested archive into the volume, and then returns the +service to its prior running state. After restore, run the backend health checks and a known +retrieval query before reopening write traffic. ## Production trust boundary and secrets diff --git a/backend/src/app.ts b/backend/src/app.ts index bdead472..003a2626 100644 --- a/backend/src/app.ts +++ b/backend/src/app.ts @@ -69,7 +69,12 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc const hub = deps?.hub ?? new SseHub(); const workspaceRegistry = deps?.workspaceRegistry ?? new WorkspaceRegistry(config.workspaceRegistry); const workspaceDiagnoser = deps?.workspaceDiagnoser - ?? createProductionWorkspaceDiagnoser(config.workspaceDiagnosticTimeoutMs); + ?? createProductionWorkspaceDiagnoser(config.workspaceDiagnosticTimeoutMs, undefined, { + internalQdrantUrl: config.internalQdrantUrl, + internalEmbeddingUrl: config.internalEmbeddingUrl, + internalEmbeddingModel: config.internalEmbeddingModel, + internalEmbeddingDimensions: config.internalEmbeddingDimensions, + }); const workspaceRuntimeSupport = deps?.workspaceRuntimeSupport ?? ((workspace: WorkspaceDescriptor) => ( supportsSessionRuntime(resolveRuntimeBindings( workspace, diff --git a/backend/src/workspaces/diagnostics.ts b/backend/src/workspaces/diagnostics.ts index 70e16ae9..97387bd4 100644 --- a/backend/src/workspaces/diagnostics.ts +++ b/backend/src/workspaces/diagnostics.ts @@ -16,6 +16,7 @@ import { type WorkspaceDescriptor, } from "./schema.js"; import type { WorkspaceErrorCode } from "./types.js"; +import type { SemanticRuntimeConfig } from "./runtime-renderer.js"; export interface Diagnostic { level: "error" | "warning" | "info"; @@ -395,6 +396,26 @@ export function createConcreteDiagnosticAdapters( } }, async inspectVector(request) { + if (request.transport === "rest_api" && request.baseUrl && request.diagnostic === undefined) { + const response = await fetch(new URL(`/collections/${request.collection}`, `${request.baseUrl}/`).toString(), { + method: "GET", + signal: request.signal, + redirect: "error", + }); + const payload = await response.json().catch(() => undefined) as { + result?: { config?: { params?: { vectors?: { size?: unknown; distance?: unknown } } } }; + } | undefined; + const size = payload?.result?.config?.params?.vectors?.size; + const distance = payload?.result?.config?.params?.vectors?.distance; + if (!response.ok || !Number.isInteger(size) || typeof distance !== "string") { + throw new Error("vector metadata adapter is unavailable"); + } + return { + collection: request.collection, + dimensions: size as number, + distance: distance.toLowerCase() as VectorDiagnosticResult["distance"], + }; + } if (request.transport === "pgvector_direct" || request.transport === "ssh_tunnel") { const resource = request.resource; if (!request.host || !request.port || !request.user || !request.credentialFile @@ -440,6 +461,21 @@ export function createConcreteDiagnosticAdapters( }; }, async probeEmbedding(request) { + if (!request.diagnostic && !request.tlsCaFile) { + const response = await fetch(new URL("/api/embed", `${request.baseUrl}/`).toString(), { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ model: request.model, input: "diagnostic" }), + signal: request.signal, + redirect: "error", + }); + const payload = await response.json().catch(() => undefined) as { + embeddings?: unknown[]; + } | undefined; + const vector = Array.isArray(payload?.embeddings) ? payload?.embeddings[0] : undefined; + if (!response.ok || !Array.isArray(vector)) throw new Error("embedding probe failed"); + return { available: true, dimensions: vector.length }; + } if (!request.diagnostic || request.tlsCaFile) throw new Error("embedding probe failed"); const response = await fetch(resolveDiagnosticUrl(request.baseUrl, request.diagnostic.path).toString(), { method: request.diagnostic.method, @@ -492,8 +528,31 @@ export function createConcreteDiagnosticAdapters( export function createProductionWorkspaceDiagnoser( timeoutMs: number, adapters: DiagnosticAdapters = createConcreteDiagnosticAdapters(), + semanticRuntime: SemanticRuntimeConfig = { + internalQdrantUrl: "http://qdrant:6333", + internalEmbeddingUrl: "http://embedding:11434", + internalEmbeddingModel: "qwen3-embedding:0.6b", + internalEmbeddingDimensions: 1024, + }, ) { - return createWorkspaceDiagnoser(adapters, { timeoutMs }); + const legacyDiagnoser = createWorkspaceDiagnoser(adapters, { timeoutMs }); + return async ( + workspace: WorkspaceDescriptor, + bindings: RuntimeBindings, + options: { writeProbe: boolean }, + ): Promise => { + const descriptor = validateWorkspaceDescriptor(workspace); + if (descriptor.workspace.schema_version !== 3) { + return await legacyDiagnoser(descriptor, bindings, options); + } + return await diagnoseSchemaV3Workspace( + descriptor as Extract, + bindings, + adapters, + timeoutMs, + semanticRuntime, + ); + }; } function boundedTimeout(value: number | undefined, fallback: number): number { @@ -526,6 +585,22 @@ function hasRequiredConnectorChecks(result: ConnectorDiagnosticResult, resource: return result.resolved && result.tlsVerified && result.authenticated && sameResource(resource, result.resource); } +function hasMatchingVectorMetadata( + actual: VectorDiagnosticResult, + expected: { collection: string; dimensions: number; distance: "cosine" | "l2" | "inner_product" }, +): boolean { + return actual.collection === expected.collection + && actual.dimensions === expected.dimensions + && actual.distance === expected.distance; +} + +function hasMatchingEmbeddingMetadata( + actual: EmbeddingDiagnosticResult, + expected: { dimensions: number }, +): boolean { + return actual.available && actual.dimensions === expected.dimensions; +} + function diagnosticError(code: WorkspaceErrorCode, field?: string): Diagnostic { return { level: "error", @@ -542,7 +617,7 @@ function diagnosticError(code: WorkspaceErrorCode, field?: string): Diagnostic { } function bindingName( - workspace: WorkspaceV2, + workspace: WorkspaceDescriptor, role: "DWH" | "VECTOR" | "VECTOR_WRITER" | "EMBEDDING", suffix: string, ): string { @@ -558,6 +633,159 @@ function numericBinding(binding: Record, name: string): number | return Number.isInteger(value) && value > 0 && value <= 65_535 ? value : undefined; } +async function diagnoseSchemaV3Workspace( + descriptor: Extract, + bindings: RuntimeBindings, + adapters: DiagnosticAdapters, + timeoutMs: number, + semanticRuntime: SemanticRuntimeConfig, +): Promise { + const diagnostics = [...bindings.dwh.missing] + .sort() + .map((field) => diagnosticError("binding_missing", field)); + if (diagnostics.length > 0) { + return { activatable: false, diagnostics }; + } + + const dwhTimeout = boundedTimeout(descriptor.dwh.timeout_ms, timeoutMs); + const vectorTimeout = timeoutMs; + const embeddingTimeout = timeoutMs; + let activatable = true; + + const dwhValues = bindings.dwh.values; + const dwhField = (suffix: string) => bindingName(descriptor, "DWH", suffix); + const dwhResource = { database: descriptor.dwh.database, schema: descriptor.dwh.schema }; + let dwhRequest: ConnectorDiagnosticRequest | SshTunnelRequest | undefined; + if (bindings.dwh.transport === "rest_api") { + const diagnostic = descriptor.diagnostics?.dwh_rest; + const baseUrl = dwhValues[dwhField("BASE_URL")]; + if (diagnostic && baseUrl) { + const credentialFile = diagnostic.auth === "none" ? undefined : dwhValues[dwhField("API_KEY_FILE")]; + if (diagnostic.auth === "none" || credentialFile !== undefined) { + dwhRequest = { + role: "dwh", + transport: "rest_api", + baseUrl, + credentialFile, + tlsCaFile: dwhValues[dwhField("TLS_CA_FILE")], + resource: dwhResource, + timeoutMs: dwhTimeout, + signal: new AbortController().signal, + diagnostic, + }; + } + } + } else if (bindings.dwh.transport === "postgres_direct") { + const host = dwhValues[dwhField("HOST")]; + const port = numericBinding(dwhValues, dwhField("PORT")); + const user = dwhValues[dwhField("USER")]; + const credentialFile = dwhValues[dwhField("PASSWORD_FILE")]; + if (host && port && user && credentialFile) { + dwhRequest = { + role: "dwh", + transport: "postgres_direct", + host, + port, + user, + credentialFile, + tlsCaFile: dwhValues[dwhField("TLS_CA_FILE")], + resource: dwhResource, + timeoutMs: dwhTimeout, + signal: new AbortController().signal, + }; + } + } else { + const sshHost = dwhValues[dwhField("SSH_HOST")]; + const sshPort = numericBinding(dwhValues, dwhField("SSH_PORT")); + const sshUser = dwhValues[dwhField("SSH_USER")]; + const privateKeyFile = dwhValues[dwhField("SSH_PRIVATE_KEY_FILE")]; + const knownHostsFile = dwhValues[dwhField("SSH_KNOWN_HOSTS_FILE")]; + const targetHost = dwhValues[dwhField("SSH_TARGET_HOST")]; + const targetPort = numericBinding(dwhValues, dwhField("SSH_TARGET_PORT")); + if (sshHost && sshPort && sshUser && privateKeyFile && knownHostsFile && targetHost && targetPort) { + dwhRequest = { + sshHost, + sshPort, + sshUser, + privateKeyFile, + knownHostsFile, + targetHost, + targetPort, + localHost: "127.0.0.1", + localPort: 0, + timeoutMs: dwhTimeout, + signal: new AbortController().signal, + }; + } + } + + if (!dwhRequest || "sshHost" in dwhRequest) { + diagnostics.push(diagnosticError("workspace_not_activatable")); + return { activatable: false, diagnostics }; + } + + try { + const dwhResult = await withTimeout(dwhTimeout, (signal) => adapters.probeConnector({ + ...dwhRequest, + signal, + timeoutMs: dwhTimeout, + })); + if (!hasRequiredConnectorChecks(dwhResult, dwhRequest.resource)) { + diagnostics.push(diagnosticError("connector_unavailable")); + activatable = false; + } + } catch { + diagnostics.push(diagnosticError("connector_unavailable")); + activatable = false; + } + + try { + const vector = await withTimeout(vectorTimeout, (signal) => adapters.inspectVector({ + transport: "rest_api", + baseUrl: semanticRuntime.internalQdrantUrl, + collection: descriptor.semantic_index.vector_store.collection, + dimensions: descriptor.semantic_index.vector_store.dimensions, + distance: descriptor.semantic_index.vector_store.distance, + timeoutMs: vectorTimeout, + signal, + })); + if (!hasMatchingVectorMetadata(vector, descriptor.semantic_index.vector_store)) { + diagnostics.push(diagnosticError("semantic_index_incompatible")); + activatable = false; + } + } catch { + diagnostics.push(diagnosticError("connector_unavailable")); + activatable = false; + } + + try { + const embedding = await withTimeout(embeddingTimeout, (signal) => adapters.probeEmbedding({ + baseUrl: semanticRuntime.internalEmbeddingUrl, + model: semanticRuntime.internalEmbeddingModel, + timeoutMs: embeddingTimeout, + signal, + })); + if ( + semanticRuntime.internalEmbeddingModel !== descriptor.semantic_index.embedding.model + || semanticRuntime.internalEmbeddingDimensions !== descriptor.semantic_index.embedding.dimensions + || !hasMatchingEmbeddingMetadata(embedding, { + dimensions: descriptor.semantic_index.embedding.dimensions, + }) + ) { + diagnostics.push(diagnosticError("semantic_index_incompatible")); + activatable = false; + } + } catch { + diagnostics.push(diagnosticError("connector_unavailable")); + activatable = false; + } + + return { + activatable, + diagnostics: diagnostics.length > 0 ? diagnostics : [{ level: "info", code: "binding_ok", message: "Installation bindings and diagnostics succeeded." }], + }; +} + function diagnosticsForMissingBindings( workspace: WorkspaceV2, bindings: RuntimeBindings, diff --git a/backend/test/workspaces-diagnostics.test.ts b/backend/test/workspaces-diagnostics.test.ts index 43f3eaf6..f2cec1ff 100644 --- a/backend/test/workspaces-diagnostics.test.ts +++ b/backend/test/workspaces-diagnostics.test.ts @@ -160,6 +160,38 @@ const writerBindings: RuntimeBindings = { }, }; +const workspaceV3 = parseWorkspaceYaml(`workspace: + schema_version: 3 + id: psd-clinical + name: Policlinico San Donato + language: it +dwh: + engine: postgres + database: warehouse + schema: datawarehouse + timeout_ms: 8000 + supported_transports: [postgres_direct, rest_api] +semantic_index: + vector_store: + engine: qdrant + collection: psd-clinical + dimensions: 1024 + distance: cosine + embedding: + provider: ollama_internal + model: qwen3-embedding:0.6b + dimensions: 1024 +llm_policy: + allowed: [zai/glm-5.2] +`); + +const bindingsV3: RuntimeBindings = { + dwh: bindings.dwh, + vector: { transport: "rest_api", missing: [], values: {} }, + vectorWriter: { transport: "rest_api", missing: [], values: {} }, + embedding: { transport: "rest_api", missing: [], values: {} }, +}; + function successfulAdapters(overrides: Partial = {}): DiagnosticAdapters { return { probeConnector: vi.fn(async (request) => ({ @@ -843,6 +875,63 @@ test("constructs the production diagnoser with the configured timeout and inject expect(adapters.probeEmbedding).toHaveBeenCalledWith(expect.objectContaining({ timeoutMs: 1234 })); }); +test("diagnoses a schema-v3 workspace through internal Qdrant and embedding config without workspace semantic bindings", async () => { + const adapters = successfulAdapters({ + inspectVector: vi.fn(async () => ({ + collection: "psd-clinical", + dimensions: 1024, + distance: "cosine", + })), + probeEmbedding: vi.fn(async () => ({ available: true, dimensions: 1024 })), + }); + + const result = await createProductionWorkspaceDiagnoser(1234, adapters, { + internalQdrantUrl: "http://qdrant:6333", + internalEmbeddingUrl: "http://embedding:11434", + internalEmbeddingModel: "qwen3-embedding:0.6b", + internalEmbeddingDimensions: 1024, + })(workspaceV3, bindingsV3, { writeProbe: false }); + + expect(result.activatable).toBe(true); + expect(adapters.inspectVector).toHaveBeenCalledWith(expect.objectContaining({ + transport: "rest_api", + baseUrl: "http://qdrant:6333", + collection: "psd-clinical", + dimensions: 1024, + distance: "cosine", + timeoutMs: 1234, + })); + expect(adapters.probeEmbedding).toHaveBeenCalledWith(expect.objectContaining({ + baseUrl: "http://embedding:11434", + model: "qwen3-embedding:0.6b", + timeoutMs: 1234, + })); + expect(adapters.probeConnector).toHaveBeenCalledTimes(1); +}); + +test("fails closed for schema-v3 when internal semantic diagnostics do not match descriptor identity", async () => { + const adapters = successfulAdapters({ + inspectVector: vi.fn(async () => ({ + collection: "wrong-collection", + dimensions: 1024, + distance: "cosine", + })), + probeEmbedding: vi.fn(async () => ({ available: true, dimensions: 1024 })), + }); + + const result = await createProductionWorkspaceDiagnoser(1234, adapters, { + internalQdrantUrl: "http://qdrant:6333", + internalEmbeddingUrl: "http://embedding:11434", + internalEmbeddingModel: "qwen3-embedding:0.6b", + internalEmbeddingDimensions: 1024, + })(workspaceV3, bindingsV3, { writeProbe: false }); + + expect(result.activatable).toBe(false); + expect(result.diagnostics).toContainEqual(expect.objectContaining({ + code: "semantic_index_incompatible", + })); +}); + test("retries bounded cleanup after a write-probe removal times out", async () => { const adapters = successfulAdapters({ removeDiagnosticRecord: vi.fn(() => new Promise(() => undefined)), diff --git a/deploy/workspaces/example.yaml b/deploy/workspaces/example.yaml index 10c1a186..a8306e65 100644 --- a/deploy/workspaces/example.yaml +++ b/deploy/workspaces/example.yaml @@ -35,22 +35,16 @@ evidence: source_root: ${THT_DOCS_ROOT} evidence_dir: evidence -embeddings: - base_url: ${THT_OLLAMA_URL} - model: nomic-embed-text-v2-moe - dim: 768 - batch_size: 32 - -vectors: - type: thoth_vector_http - reader: - base_url: ${THT_VEC_REST_URL} - api_key: ${THT_VEC_API_KEY} - ssl_ca: ${THT_SSL_CA} - writer: - base_url: ${THT_VEC_REST_URL} - api_key: ${THT_VEC_WRITE_API_KEY} - ssl_ca: ${THT_SSL_CA} +resources: + vector: + engine: qdrant + base_url: http://qdrant:6333 + collection: example + embeddings: + provider: ollama_internal + base_url: http://embedding:11434 + model: qwen3-embedding:0.6b + dimensions: 1024 vector: max_chunk_chars: 4000 diff --git a/deploy/workspaces/local-vector.yaml b/deploy/workspaces/local-vector.yaml deleted file mode 100644 index b8a46556..00000000 --- a/deploy/workspaces/local-vector.yaml +++ /dev/null @@ -1,48 +0,0 @@ -language: en - -dwh: - type: thoth_rest - database: - database: ${THT_DB_NAME} - schema: datawarehouse - endpoint: - base_url: ${THT_DWH_REST_URL} - api_key: ${THT_DWH_API_KEY} - -vectors: - type: pgvector_direct - reader: - host: vector-db - port: 5432 - database: ${THT_VECTOR_DATABASE} - schema: vectors - user: ${THT_VECTOR_READER_USER} - password_file: ${THT_VECTOR_READER_PASSWORD_FILE} - writer: - host: vector-db - port: 5432 - database: ${THT_VECTOR_DATABASE} - schema: vectors - user: ${THT_VECTOR_WRITER_USER} - password_file: ${THT_VECTOR_WRITER_PASSWORD_FILE} - -roots: - artifacts: artifacts - indexes: indexes - sessions: sessions - -evidence: - source_root: ${THT_DOCS_ROOT} - evidence_dir: evidence - -embeddings: - base_url: ${THT_OLLAMA_URL} - model: nomic-embed-text-v2-moe - dim: 768 - batch_size: 32 - -execution: - allow: [cte_test, explain, preview, aggregate, export] - max_preview_rows: 10 - max_export_rows: 100000 - statement_timeout_ms: 30000 diff --git a/deploy/workspaces/psd.yaml.example b/deploy/workspaces/psd.yaml.example index 4b9e52f3..ded7c9ee 100644 --- a/deploy/workspaces/psd.yaml.example +++ b/deploy/workspaces/psd.yaml.example @@ -1,4 +1,4 @@ -language: it +language: en dwh: type: thoth_rest @@ -10,31 +10,25 @@ dwh: api_key: ${THT_DWH_API_KEY} ssl_ca: ${THT_SSL_CA} -vectors: - type: thoth_vector_http - reader: - base_url: ${THT_VEC_REST_URL} - api_key: ${THT_VEC_API_KEY} - ssl_ca: ${THT_SSL_CA} - writer: - base_url: ${THT_VEC_WRITE_REST_URL} - api_key: ${THT_VEC_WRITE_API_KEY} - ssl_ca: ${THT_SSL_CA} - roots: - artifacts: /data/workspaces/psd/runtime-v2/artifacts - indexes: /data/workspaces/psd/runtime-v2/indexes - sessions: /data/workspaces/psd/sessions + artifacts: /data/workspaces/generic/artifacts + indexes: /data/workspaces/generic/indexes + sessions: /data/workspaces/generic/sessions evidence: source_root: ${THT_DOCS_ROOT} evidence_dir: evidence -embeddings: - base_url: ${THT_OLLAMA_URL} - model: nomic-embed-text-v2-moe - dim: 768 - batch_size: 32 +resources: + vector: + engine: qdrant + base_url: http://qdrant:6333 + collection: generic + embeddings: + provider: ollama_internal + base_url: http://embedding:11434 + model: qwen3-embedding:0.6b + dimensions: 1024 execution: allow: [cte_test, explain, preview, aggregate, export] diff --git a/deploy/workspaces/server-sessions.yaml.example b/deploy/workspaces/server-sessions.yaml.example index 69402679..616c11fa 100644 --- a/deploy/workspaces/server-sessions.yaml.example +++ b/deploy/workspaces/server-sessions.yaml.example @@ -29,8 +29,13 @@ roots: indexes: indexes sessions: sessions -embeddings: - base_url: ${THT_OLLAMA_URL} - model: nomic-embed-text-v2-moe - dim: 768 - batch_size: 32 +resources: + vector: + engine: qdrant + base_url: http://qdrant:6333 + collection: server-sessions + embeddings: + provider: ollama_internal + base_url: http://embedding:11434 + model: qwen3-embedding:0.6b + dimensions: 1024 diff --git a/docs/install/local-workspace-registry.md b/docs/install/local-workspace-registry.md index 5bf8b306..c111dfde 100644 --- a/docs/install/local-workspace-registry.md +++ b/docs/install/local-workspace-registry.md @@ -5,9 +5,10 @@ Git-backed workspace source of truth, installation-local connector bindings, and the [Pi management manual](pi-management.md) for provider configuration and image recovery. This guide runs a single-user ThothII registry on Docker Desktop (macOS or Windows) or a local -Linux Docker Engine. It is intentionally loopback-only. Git is shared; the checkout, connector -bindings, credentials, and session data are local. Never put credentials in workspace YAML, Git, -browser drafts, diagnostics, or `.env.example`. +Linux Docker Engine. It is intentionally loopback-only. Git is shared; the checkout, DWH +bindings, credentials, and session data are local, while internal Qdrant/Ollama ship in the +Compose stack. Never put credentials in workspace YAML, Git, browser drafts, diagnostics, or +`.env.example`. ## Prerequisites @@ -60,7 +61,7 @@ and branch are non-secret; every `*_FILE` is a local path whose content never en | Location | Contains | Never contains | | --- | --- | --- | -| Git workspace repository | schema v2 YAML, generated binding names, LLM policy, model/index identity | installation hostnames, keys, passwords, certificates, SSH keys | +| Git workspace repository | schema v3 YAML, generated binding names, LLM policy, and semantic-index identity | installation hostnames, keys, passwords, certificates, SSH keys | | local `.env` | remote, branch, installation ID, `THT_WORKSPACE_BINDINGS_ENV_FILE`, and secret source paths | secret contents or `THT_WS_*` values | | workspace bindings env file | only `THT_WS_*` transport, endpoint, user, and `/run/secrets/...` path bindings | secret contents or unrelated application settings | | local secret directory | Git credentials/key, known hosts, CA, connector secret files | a copied registry checkout | @@ -168,13 +169,13 @@ curl --fail --silent http://127.0.0.1:8787/workspaces The first status request clones, validates all descriptors, and atomically activates a snapshot. Use `POST /workspace-registry/pull` to fetch later revisions. Run workspace diagnostics only after -required bindings are mounted. The optional writer probe uses a distinct writer file and removes -its uniquely named temporary record; ordinary diagnostics are read-only. +required DWH bindings are mounted. Schema-v3 diagnostics probe the internal Qdrant/Ollama +services through backend config; ordinary diagnostics are read-only. To migrate an existing legacy descriptor, create/clone an empty private remote, set the absolute -`THT_SOURCE_ROOT`, transform with absolute paths, review the schema-v1 result, explicitly add -vector database/schema and the complete schema-v2 contract, then commit/push. The transformer -never imports `${ENV}` values or secrets. +`THT_SOURCE_ROOT`, transform with absolute paths, review the schema-v1 result, explicitly produce +the reviewed schema-v3 contract, then commit/push. The transformer never imports `${ENV}` values +or secrets. ```sh THT_SOURCE_ROOT=/absolute/path/to/ThothII diff --git a/docs/install/server-workspace-registry.md b/docs/install/server-workspace-registry.md index 99e47e98..b6fb4e6b 100644 --- a/docs/install/server-workspace-registry.md +++ b/docs/install/server-workspace-registry.md @@ -32,7 +32,7 @@ targets with runtime ownership without copying secret or tracked file contents i state. Rerun it after a restore and before Compose or `thothctl` startup; it is idempotent and does not overwrite existing targets. -Permit outbound TCP only to approved Git/Gitea, DWH, vector, embedding, and bastion endpoints. +Permit outbound TCP only to approved Git/Gitea, DWH, Qdrant, embedding, and bastion endpoints. Allow inbound traffic only from the reverse proxy/Docker network. Do not give the runtime service account Gitea administration, database-superuser rights, or a shell in the Git host. @@ -40,7 +40,7 @@ account Gitea administration, database-superuser rights, or a shell in the Git h Create a private Gitea (or compatible Git) repository such as `platform/thoth-workspaces`. Protect `main` according to the release policy and grant the ThothII publisher only the intended repository -scope. Commit canonical schema-v2 descriptors and generated `.md`/`.env.example` artifacts only; +scope. Commit canonical schema-v3 descriptors and generated `.md`/`.env.example` artifacts only; do not commit installation bindings or secret material. For SSH, create a least-privilege deploy key, record Gitea's host key in managed known-hosts, and @@ -49,7 +49,7 @@ machine credential in the secret manager and mount the Gitea/private CA separate Gitea admin credential in the application. Bootstrap an empty remote from a temporary review clone: migrate legacy descriptors, review their -schema-v2 identity and generated artifacts, commit, and push `main`. The running server is not an +schema-v3 identity and generated artifacts, commit, and push `main`. The running server is not an authoring environment for migration. ## Git credentials, CA, SSH key, and known-hosts mounts @@ -78,8 +78,8 @@ rendered Compose output. ## Shared Git values, local bindings, and secret files -Git describes workspace schema, immutable ID, DWH/vector identity, semantic-index dimensions and -distance, embedding contract, and LLM policy. The installation supplies remote/branch/installation +Git describes workspace schema, immutable ID, DWH identity, semantic-index dimensions and +distance, internal embedding contract, and LLM policy. The installation supplies remote/branch/installation ID and one absolute `THT_WORKSPACE_BINDINGS_ENV_FILE` containing only `THT_WS_*` transport, endpoint, user, and `/run/secrets/...` path bindings. The base Compose loads that file only into `core`. Secret contents are only in host files, never the values stored in Git or browser-local @@ -95,8 +95,9 @@ The runtime registry layout is persistent and must be backed up together: ``` Variable names derive from the immutable ID: `north-star-research` becomes `NORTH_STAR_RESEARCH`, producing -`THT_WS_NORTH_STAR_RESEARCH_DWH_PASSWORD_FILE`. A declared vector writer uses the distinct -`THT_WS_NORTH_STAR_RESEARCH_VECTOR_WRITER_API_KEY_FILE`; a reader file is never a writer substitute. +`THT_WS_NORTH_STAR_RESEARCH_DWH_PASSWORD_FILE`. Keep the bindings file limited to DWH transport, +endpoint, user, and secret-path values; internal semantic services are supplied by Compose and do +not require workspace-local vector or embedding bindings. Copy [the bindings env example](examples/workspace-bindings.env.example) to the protected operator directory. Every path-valued `*_FILE` entry needs an absolute host-only `*_SOURCE` path. Generate the untracked connector override from those files during bootstrap; do not copy or maintain a @@ -216,8 +217,8 @@ For upgrades, record active status/head, finish active work, use the documented proxy traffic. For legacy descriptor migration, use a temporary review clone and the legacy transformer with absolute paths. -Its schema-v1 output is `migration_required`; explicitly supply vector database/schema, collection -identity, diagnostics, and the reviewed v2 contract before commit. Never import `${ENV}` values or +Its schema-v1 output is `migration_required`; explicitly supply collection identity, diagnostics, +and the reviewed v3 contract before commit. Never import `${ENV}` values or copy secret files. After valid bootstrap, Git outage retains the active snapshot with `degraded: true`. Repair diff --git a/scripts/test-canonical-install-compose.sh b/scripts/test-canonical-install-compose.sh index e44cd1ac..462524e9 100755 --- a/scripts/test-canonical-install-compose.sh +++ b/scripts/test-canonical-install-compose.sh @@ -61,8 +61,9 @@ for profile in local server; do const fs = require("fs"); const [path, profile] = process.argv.slice(2); const config = JSON.parse(fs.readFileSync(path, "utf8")); -if (Object.keys(config.services).sort().join(",") !== "core,frontend") { - throw new Error(profile + ": install stack must be exactly core,frontend"); +const expected = "core,embedding,embedding-model-init,frontend,qdrant"; +if (Object.keys(config.services).sort().join(",") !== expected) { + throw new Error(profile + ": install stack must be exactly " + expected); } if (!config.services.core.secrets?.some((secret) => secret.target === "thothii.secrets")) { throw new Error(profile + ": install stack lacks the runtime secret bundle"); diff --git a/scripts/test-no-deployment-coupling-scope.sh b/scripts/test-no-deployment-coupling-scope.sh index 8685539f..c2b4d1f6 100755 --- a/scripts/test-no-deployment-coupling-scope.sh +++ b/scripts/test-no-deployment-coupling-scope.sh @@ -28,13 +28,17 @@ new_fixture() { printf '%s\n' '// generic frontend configuration' >"$fixture/repository/frontend/vite.config.ts" printf '%s\n' '// explicit descriptor migration module may mention pgvector during conversion' \ >"$fixture/repository/backend/src/workspaces/migrate-legacy.ts" + printf '%s\n' 'language: en' 'vectors: { type: qdrant, base_url: http://qdrant:6333, collection: demo }' \ + >"$fixture/repository/deploy/workspaces/example.yaml" + printf '%s\n' '# qdrant backup helper' >"$fixture/repository/scripts/vector-backup.sh" + printf '%s\n' '# qdrant restore helper' >"$fixture/repository/scripts/vector-restore.sh" # These are the three intentionally allowed categories from the Task 10 boundary. printf '%s\n' 'historical omics_portal and Chirone record' \ >"$fixture/repository/docs/superpowers/plans/legacy.md" printf '%s\n' 'historical pgvector rollout note' \ >"$fixture/repository/docs/superpowers/specs/history.md" - printf '%s\n' 'id: psd' >"$fixture/repository/deploy/workspaces/psd.yaml.example" + printf '%s\n' 'id: generic' >"$fixture/repository/deploy/workspaces/psd.yaml.example" printf '%s\n' '# migrate PSD sessions from /home/chirone' \ >"$fixture/repository/docker/session-migrate.sh" } @@ -77,6 +81,10 @@ assert_detected scripts/run-stack.sh 'export THT_VECTOR_READER_PASSWORD_FILE=/ru assert_detected deploy/env/local.env.example 'THT_OLLAMA_URL=http://ollama.example.invalid:11434' assert_detected scripts/generate-override.sh 'THT_WS_NORTH_STAR_RESEARCH_VECTOR_API_KEY_SOURCE=/tmp/vector-key' assert_detected scripts/test-contract.sh 'docker compose -f deploy/compose.local-vector.yaml --profile local-vector config' +assert_detected deploy/workspaces/local-vector.yaml 'vectors: { type: pgvector_direct }' +assert_detected deploy/workspaces/example.yaml 'embeddings: { base_url: ${THT_OLLAMA_URL} }' +assert_detected scripts/vector-backup.sh 'pg_dump --format=custom' +assert_detected scripts/vector-restore.sh 'pg_restore --single-transaction' new_fixture mkdir -p "$fixture/bin" diff --git a/scripts/test-no-deployment-coupling.sh b/scripts/test-no-deployment-coupling.sh index 0a8fa37e..1521a9af 100755 --- a/scripts/test-no-deployment-coupling.sh +++ b/scripts/test-no-deployment-coupling.sh @@ -28,7 +28,7 @@ for file in .dockerignore compose.yaml docker-compose.dev.yml frontend/vite.conf done if [[ -d deploy ]]; then while IFS= read -r -d '' file; do runtime_files+=("${file#./}"); done < <( - find deploy -type f ! -path 'deploy/workspaces/*' -print0 + find deploy -type f -print0 ) fi if [[ -d docker ]]; then @@ -57,7 +57,7 @@ if [[ -d scripts ]]; then contract_test_files+=("${file#./}") continue ;; - compose-with-preflight.sh|generate-connector-secrets-override.sh|unified-deployment-smoke.sh|vector-backup.sh|vector-restore.sh|vector-rotate-bootstrap-password.sh) + compose-with-preflight.sh|generate-connector-secrets-override.sh|unified-deployment-smoke.sh|vector-rotate-bootstrap-password.sh) continue ;; verify-*.sh) continue ;; @@ -113,7 +113,7 @@ for forbidden_file in \ done forbidden='omics_portal|chirone|localllm_default|datamart-builder|compose\.production\.yaml|compose\.psd-local\.yaml|\bpsd\b' -retired_semantic='local-vector|THT_VECTOR_(BOOTSTRAP|MIGRATOR|READER|WRITER|DATABASE|HOST|PORT|USER|ADMIN_URL|OPERATOR_ENV_FILE)|THT_OLLAMA_URL|VECTOR_API_KEY_(FILE|SOURCE)|vector-api-key|(^|[^A-Za-z0-9_])THT_VEC_(REST_URL|WRITE_REST_URL)' +retired_semantic='local-vector|pgvector(_direct)?|pg_(dump|restore)|THT_VECTOR_([A-Z0-9_]+)|THT_OLLAMA_URL|THT_WS_[A-Z0-9_]*_(VECTOR|EMBEDDING)_[A-Z0-9_]+|VECTOR_API_KEY_(FILE|SOURCE)|EMBEDDING_API_KEY_(FILE|SOURCE)|vector-api-key|(^|[^A-Za-z0-9_])THT_VEC_(REST_URL|WRITE_REST_URL)|thoth_vector_http' scan_category runtime "$forbidden" "${runtime_files[@]}" scan_category install "$forbidden" "${install_files[@]}" scan_category operator "$forbidden" "${operator_files[@]}" @@ -132,7 +132,7 @@ for file in "${contract_test_files[@]}"; do esac contract_scan_files+=("$file") done -retired_semantic_contract='docker compose[^\n]*(compose\.local-vector|compose\.preprocess-local-vector)|THT_VECTOR_(BOOTSTRAP|MIGRATOR|READER|WRITER|DATABASE|HOST|PORT|USER|ADMIN_URL|OPERATOR_ENV_FILE)=|THT_OLLAMA_URL=|THT_WS_[A-Z0-9_]*_VECTOR_(TRANSPORT|API_KEY_(FILE|SOURCE))=|vector-api-key' +retired_semantic_contract='docker compose[^\n]*(compose\.local-vector|compose\.preprocess-local-vector)|THT_VECTOR_([A-Z0-9_]+)=|THT_OLLAMA_URL=|THT_WS_[A-Z0-9_]*_(VECTOR|EMBEDDING)_[A-Z0-9_]+=|vector-api-key|pgvector|pg_(dump|restore)' scan_category contract-test "$retired_semantic_contract" "${contract_scan_files[@]}" if [[ -f scripts/run-stack.sh ]]; then diff --git a/scripts/test-vector-backup-restore-safety.sh b/scripts/test-vector-backup-restore-safety.sh index bf0cf0e0..1351ed3d 100755 --- a/scripts/test-vector-backup-restore-safety.sh +++ b/scripts/test-vector-backup-restore-safety.sh @@ -6,84 +6,103 @@ tmp=$(mktemp -d) trap 'rm -rf "$tmp"' EXIT HUP INT TERM fakebin="$tmp/bin" mkdir "$fakebin" -printf '%s' secret >"$tmp/password" -chmod 0600 "$tmp/password" -cat >"$fakebin/pg_dump" <<'SH' +project="thoth-task8" +volume_name="${project}_qdrant-data" +mountpoint="$tmp/docker-volumes/$volume_name/_data" +mkdir -p "$mountpoint/collections/demo" +printf '%s' before-backup >"$mountpoint/collections/demo/state.json" + +cat >"$fakebin/docker" <<'SH' #!/bin/sh set -eu -for arg in "$@"; do case "$arg" in --file=*) output=${arg#--file=} ;; esac; done -printf 'custom dump' >"$output" -if [ -n "${RACE_OUTPUT:-}" ]; then - printf 'concurrent owner' >"$RACE_OUTPUT" +log_file=${DOCKER_LOG:?} +printf '%s\n' "$*" >>"$log_file" + +if [ "$1" = volume ] && [ "$2" = ls ]; then + if [ "${VOLUME_LS_OUTPUT:-}" = multiple ]; then + printf '%s\n%s\n' "${PROJECT_NAME}_qdrant-data" "${PROJECT_NAME}_qdrant-data-copy" + exit 0 + fi + printf '%s\n' "${PROJECT_NAME}_qdrant-data" + exit 0 fi + +if [ "$1" = volume ] && [ "$2" = inspect ]; then + printf '%s\n' "${MOUNTPOINT:?}" + exit 0 +fi + +if [ "$1" = compose ] && [ "$2" = --project-name ]; then + case "$4" in + ps) + if [ "${QDRANT_RUNNING:-1}" = 1 ]; then + printf '%s\n' qdrant-container + fi + exit 0 + ;; + stop) + exit 0 + ;; + start) + exit 0 + ;; + esac +fi + +exit 0 SH -chmod 0755 "$fakebin/pg_dump" +chmod 0755 "$fakebin/docker" -victim="$tmp/victim" -output="$tmp/vector.dump" -printf 'sentinel' >"$victim" -ln -s "$victim" "$output.partial" -PATH="$fakebin:$PATH" ./scripts/vector-backup.sh --host source --database thoth --user admin \ - --password-file "$tmp/password" --output "$output" >/dev/null -test "$(cat "$victim")" = sentinel -test "$(cat "$output")" = 'custom dump' -test -L "$output.partial" +backup_output="$tmp/qdrant-backup.tar" +docker_log="$tmp/docker.log" +PATH="$fakebin:$PATH" DOCKER_LOG="$docker_log" PROJECT_NAME="$project" MOUNTPOINT="$mountpoint" \ + ./scripts/vector-backup.sh --project-name "$project" --output "$backup_output" >/dev/null +test -s "$backup_output" +tar -tf "$backup_output" | grep -q '^./collections/demo/state.json$' +grep -q "volume ls --filter label=com.docker.compose.project=$project --filter label=com.docker.compose.volume=qdrant-data" "$docker_log" +grep -q "compose --project-name $project ps --status running -q qdrant" "$docker_log" +grep -q "compose --project-name $project stop qdrant" "$docker_log" +grep -q "compose --project-name $project start qdrant" "$docker_log" -race_output="$tmp/raced.dump" -if PATH="$fakebin:$PATH" RACE_OUTPUT="$race_output" ./scripts/vector-backup.sh \ - --host source --database thoth --user admin --password-file "$tmp/password" \ - --output "$race_output" >"$tmp/race.out" 2>"$tmp/race.err"; then - echo "backup replaced a destination created concurrently" >&2 +existing="$tmp/existing.tar" +printf '%s' sentinel >"$existing" +if PATH="$fakebin:$PATH" DOCKER_LOG="$tmp/existing.log" PROJECT_NAME="$project" MOUNTPOINT="$mountpoint" \ + ./scripts/vector-backup.sh --project-name "$project" --output "$existing" >"$tmp/existing.out" 2>"$tmp/existing.err"; then + echo "backup overwrote an existing archive" >&2 exit 1 fi -test "$(cat "$race_output")" = 'concurrent owner' -if find "$tmp" -name '.raced.dump.tmp.*' -print | grep -q .; then - echo "backup left its owned temporary archive after publication failure" >&2 +test "$(cat "$existing")" = sentinel + +if PATH="$fakebin:$PATH" DOCKER_LOG="$tmp/ambiguous.log" PROJECT_NAME="$project" MOUNTPOINT="$mountpoint" VOLUME_LS_OUTPUT=multiple \ + ./scripts/vector-backup.sh --project-name "$project" --output "$tmp/ambiguous.tar" >"$tmp/ambiguous.out" 2>"$tmp/ambiguous.err"; then + echo "backup accepted an ambiguous qdrant-data target" >&2 exit 1 fi +grep -q 'exactly one qdrant-data volume' "$tmp/ambiguous.err" -cat >"$fakebin/psql" <<'SH' -#!/bin/sh -set -eu -case "$*" in - *pg_control_system*) - echo same-cluster ;; - *) echo 0 ;; -esac -SH -cat >"$fakebin/pg_restore" <<'SH' -#!/bin/sh -printf '%s\n' "$*" >"$RESTORE_LOG" -SH -chmod 0755 "$fakebin/psql" "$fakebin/pg_restore" -printf 'archive' >"$tmp/input" -if PATH="$fakebin:$PATH" RESTORE_LOG="$tmp/restore.log" ./scripts/vector-restore.sh \ - --active-host source --active-database active --active-user admin \ - --active-password-file "$tmp/password" --target-host target --target-database restore \ - --target-user admin --target-password-file "$tmp/password" --input "$tmp/input" \ - >"$tmp/out" 2>"$tmp/err"; then - echo "restore accepted a target on the active PostgreSQL cluster" >&2 +restore_input="$tmp/restore.tar" +restore_source="$tmp/restore-source" +mkdir -p "$restore_source/collections/demo" +printf '%s' restored >"$restore_source/collections/demo/state.json" +tar -C "$restore_source" -cf "$restore_input" . + +if PATH="$fakebin:$PATH" DOCKER_LOG="$tmp/restore-refuse.log" PROJECT_NAME="$project" MOUNTPOINT="$mountpoint" \ + ./scripts/vector-restore.sh --project-name "$project" --input "$restore_input" --confirm-project wrong-project \ + >"$tmp/restore-refuse.out" 2>"$tmp/restore-refuse.err"; then + echo "restore skipped explicit project confirmation" >&2 exit 1 fi -grep -q 'same PostgreSQL cluster' "$tmp/err" -test ! -e "$tmp/restore.log" +grep -q 'confirmation must match --project-name exactly' "$tmp/restore-refuse.err" +test "$(cat "$mountpoint/collections/demo/state.json")" = before-backup -cat >"$fakebin/psql" <<'SH' -#!/bin/sh -set -eu -case "$*" in - *pg_control_system*) - case "$*" in *--host=source*) echo same-cluster ;; *) echo other-cluster ;; esac ;; - *) echo 0 ;; -esac -SH -chmod 0755 "$fakebin/psql" -PATH="$fakebin:$PATH" RESTORE_LOG="$tmp/restore.log" ./scripts/vector-restore.sh \ - --active-host source --active-database active --active-user admin \ - --active-password-file "$tmp/password" --target-host target --target-database restore \ - --target-user admin --target-password-file "$tmp/password" --input "$tmp/input" >/dev/null -grep -q -- '--single-transaction' "$tmp/restore.log" -grep -q -- '--exit-on-error' "$tmp/restore.log" +printf '%s' modified-live >"$mountpoint/collections/demo/state.json" +restore_log="$tmp/restore-ok.log" +PATH="$fakebin:$PATH" DOCKER_LOG="$restore_log" PROJECT_NAME="$project" MOUNTPOINT="$mountpoint" \ + ./scripts/vector-restore.sh --project-name "$project" --input "$restore_input" --confirm-project "$project" >/dev/null +test "$(cat "$mountpoint/collections/demo/state.json")" = restored +grep -q "compose --project-name $project stop qdrant" "$restore_log" +grep -q "compose --project-name $project start qdrant" "$restore_log" +grep -q "volume inspect --format {{ .Mountpoint }} $volume_name" "$restore_log" -echo "vector backup/restore filesystem, identity, and transaction contracts passed." +echo "qdrant backup/restore target resolution, refusal, and service-state contracts passed." diff --git a/scripts/vector-backup.sh b/scripts/vector-backup.sh index 8379a849..a449215d 100755 --- a/scripts/vector-backup.sh +++ b/scripts/vector-backup.sh @@ -1,53 +1,80 @@ #!/bin/sh set -eu -root=$(CDPATH= cd -- "$(dirname "$0")/.." && pwd) -. "$root/scripts/secret-file-utils.sh" - usage() { - echo "usage: $0 --host HOST --database DB --user USER --password-file FILE --output FILE [--port PORT]" >&2 + echo "usage: $0 --project-name NAME --output FILE" >&2 exit 2 } -host= database= user= password_file= output= port=5432 +project_name= +output= while [ "$#" -gt 0 ]; do case "$1" in - --host) host=${2-}; shift 2 ;; - --port) port=${2-}; shift 2 ;; - --database) database=${2-}; shift 2 ;; - --user) user=${2-}; shift 2 ;; - --password-file) password_file=${2-}; shift 2 ;; + --project-name) project_name=${2-}; shift 2 ;; --output) output=${2-}; shift 2 ;; *) usage ;; esac done -[ -n "$host" ] && [ -n "$database" ] && [ -n "$user" ] || usage -[ -n "$password_file" ] && [ -n "$output" ] || usage -validate_secret_file "$password_file" "backup password file" + +[ -n "$project_name" ] && [ -n "$output" ] || usage [ ! -e "$output" ] || { echo "refusing to overwrite existing backup: $output" >&2; exit 2; } + output_dir=$(dirname "$output") output_name=$(basename "$output") [ -d "$output_dir" ] || { echo "backup destination directory does not exist" >&2; exit 2; } -password=$(read_secret_file "$password_file" "backup password file") +resolve_volume() { + names=$(docker volume ls \ + --filter "label=com.docker.compose.project=$project_name" \ + --filter "label=com.docker.compose.volume=qdrant-data" \ + --format '{{.Name}}') + count=$(printf '%s\n' "$names" | sed '/^$/d' | wc -l | tr -d ' ') + [ "$count" -eq 1 ] || { + echo "expected exactly one qdrant-data volume for compose project $project_name" >&2 + exit 2 + } + printf '%s\n' "$names" | sed -n '/./{p;q;}' +} + +resolve_mountpoint() { + mountpoint=$(docker volume inspect --format '{{ .Mountpoint }}' "$1") + [ -n "$mountpoint" ] || { echo "docker did not return a qdrant-data mountpoint" >&2; exit 2; } + case "$mountpoint" in + /*) ;; + *) echo "qdrant-data mountpoint is not absolute: $mountpoint" >&2; exit 2 ;; + esac + [ -d "$mountpoint" ] || { echo "qdrant-data mountpoint is not a directory: $mountpoint" >&2; exit 2; } + printf '%s\n' "$mountpoint" +} + +volume_name=$(resolve_volume) +mountpoint=$(resolve_mountpoint "$volume_name") +running_container=$(docker compose --project-name "$project_name" ps --status running -q qdrant) +restart_qdrant=0 +cleanup() { + status=$? + if [ "${temporary_output:-}" ] && [ -e "${temporary_output:-}" ]; then + rm -f "$temporary_output" + fi + if [ "$restart_qdrant" -eq 1 ]; then + docker compose --project-name "$project_name" start qdrant >/dev/null + fi + exit "$status" +} +trap cleanup EXIT HUP INT TERM + +if [ -n "$running_container" ]; then + docker compose --project-name "$project_name" stop qdrant >/dev/null + restart_qdrant=1 +fi umask 077 -passfile=$(mktemp "${TMPDIR:-/tmp}/thoth-vector-pgpass.XXXXXX") temporary_output=$(mktemp "$output_dir/.${output_name}.tmp.XXXXXX") -cleanup() { rm -f "$passfile" "$temporary_output"; } -trap cleanup EXIT HUP INT TERM -escaped=$(printf '%s' "$password" | sed 's/\\/\\\\/g; s/:/\\:/g') -printf '%s:%s:%s:%s:%s\n' "$host" "$port" "$database" "$user" "$escaped" >"$passfile" -chmod 0600 "$passfile" - -PGPASSFILE=$passfile pg_dump \ - --host="$host" --port="$port" --username="$user" --dbname="$database" \ - --format=custom --compress=9 \ - --table=vectors.schema_records --table=vectors.evidence --table=vectors.memory \ - --table=public.tht_vector_migrations --file="$temporary_output" +tar -C "$mountpoint" -cf "$temporary_output" . if ! ln "$temporary_output" "$output"; then echo "refusing to replace backup destination created concurrently: $output" >&2 exit 2 fi rm -f "$temporary_output" -echo "Vector backup written: $output" +temporary_output= +echo "Qdrant backup written from $volume_name to $output" diff --git a/scripts/vector-restore.sh b/scripts/vector-restore.sh index c0c7ceb6..5faa51d3 100755 --- a/scripts/vector-restore.sh +++ b/scripts/vector-restore.sh @@ -1,80 +1,95 @@ #!/bin/sh set -eu -root=$(CDPATH= cd -- "$(dirname "$0")/.." && pwd) -. "$root/scripts/secret-file-utils.sh" - usage() { - echo "usage: $0 --active-host HOST --active-database DB --active-user USER --active-password-file FILE --target-host HOST --target-database DB --target-user USER --target-password-file FILE --input FILE [--active-port PORT] [--target-port PORT] [--force-nonempty]" >&2 + echo "usage: $0 --project-name NAME --input FILE --confirm-project NAME" >&2 exit 2 } -active_host= active_database= active_user= active_password_file= active_port=5432 -target_host= target_database= target_user= target_password_file= target_port=5432 -input= force=0 +project_name= +input= +confirm_project= while [ "$#" -gt 0 ]; do case "$1" in - --active-host) active_host=${2-}; shift 2 ;; - --active-port) active_port=${2-}; shift 2 ;; - --active-database) active_database=${2-}; shift 2 ;; - --active-user) active_user=${2-}; shift 2 ;; - --active-password-file) active_password_file=${2-}; shift 2 ;; - --target-host) target_host=${2-}; shift 2 ;; - --target-port) target_port=${2-}; shift 2 ;; - --target-database) target_database=${2-}; shift 2 ;; - --target-user) target_user=${2-}; shift 2 ;; - --target-password-file) target_password_file=${2-}; shift 2 ;; + --project-name) project_name=${2-}; shift 2 ;; --input) input=${2-}; shift 2 ;; - --force-nonempty) force=1; shift ;; + --confirm-project) confirm_project=${2-}; shift 2 ;; *) usage ;; esac done -for value in "$active_host" "$active_database" "$active_user" "$active_password_file" \ - "$target_host" "$target_database" "$target_user" "$target_password_file" "$input"; do - [ -n "$value" ] || usage -done + +[ -n "$project_name" ] && [ -n "$input" ] && [ -n "$confirm_project" ] || usage +[ "$confirm_project" = "$project_name" ] || { + echo "restore confirmation must match --project-name exactly" >&2 + exit 2 +} [ -r "$input" ] || { echo "backup input is not readable" >&2; exit 2; } -validate_secret_file "$active_password_file" "active source password file" -validate_secret_file "$target_password_file" "target password file" -umask 077 -active_pass=$(mktemp "${TMPDIR:-/tmp}/thoth-vector-active-pgpass.XXXXXX") -target_pass=$(mktemp "${TMPDIR:-/tmp}/thoth-vector-target-pgpass.XXXXXX") -cleanup() { rm -f "$active_pass" "$target_pass"; } +resolve_volume() { + names=$(docker volume ls \ + --filter "label=com.docker.compose.project=$project_name" \ + --filter "label=com.docker.compose.volume=qdrant-data" \ + --format '{{.Name}}') + count=$(printf '%s\n' "$names" | sed '/^$/d' | wc -l | tr -d ' ') + [ "$count" -eq 1 ] || { + echo "expected exactly one qdrant-data volume for compose project $project_name" >&2 + exit 2 + } + printf '%s\n' "$names" | sed -n '/./{p;q;}' +} + +resolve_mountpoint() { + mountpoint=$(docker volume inspect --format '{{ .Mountpoint }}' "$1") + [ -n "$mountpoint" ] || { echo "docker did not return a qdrant-data mountpoint" >&2; exit 2; } + case "$mountpoint" in + /*) ;; + *) echo "qdrant-data mountpoint is not absolute: $mountpoint" >&2; exit 2 ;; + esac + [ -d "$mountpoint" ] || { echo "qdrant-data mountpoint is not a directory: $mountpoint" >&2; exit 2; } + printf '%s\n' "$mountpoint" +} + +volume_name=$(resolve_volume) +mountpoint=$(resolve_mountpoint "$volume_name") +running_container=$(docker compose --project-name "$project_name" ps --status running -q qdrant) +restart_qdrant=0 +staging_dir= +extract_dir= + +cleanup() { + status=$? + if [ "$status" -ne 0 ] && [ -n "${staging_dir:-}" ] && [ -d "${staging_dir:-}" ]; then + find "$mountpoint" -mindepth 1 -maxdepth 1 -exec rm -rf -- {} + + find "$staging_dir" -mindepth 1 -maxdepth 1 -exec mv {} "$mountpoint"/ \; + fi + if [ -n "${staging_dir:-}" ] && [ -d "${staging_dir:-}" ]; then + rm -rf "$staging_dir" + fi + if [ -n "${extract_dir:-}" ] && [ -d "${extract_dir:-}" ]; then + rm -rf "$extract_dir" + fi + if [ "$restart_qdrant" -eq 1 ]; then + docker compose --project-name "$project_name" start qdrant >/dev/null + fi + exit "$status" +} trap cleanup EXIT HUP INT TERM -make_passfile() { - secret=$(read_secret_file "$5" "database password file") - escaped=$(printf '%s' "$secret" | sed 's/\\/\\\\/g; s/:/\\:/g') - printf '%s:%s:%s:%s:%s\n' "$1" "$2" "$3" "$4" "$escaped" >"$6" - chmod 0600 "$6" -} -make_passfile "$active_host" "$active_port" "$active_database" "$active_user" \ - "$active_password_file" "$active_pass" -make_passfile "$target_host" "$target_port" "$target_database" "$target_user" \ - "$target_password_file" "$target_pass" -identity_sql="SELECT system_identifier::text FROM pg_control_system()" -active_identity=$(PGPASSFILE=$active_pass psql -XAt --host="$active_host" --port="$active_port" \ - --username="$active_user" --dbname="$active_database" --command="$identity_sql") -target_identity=$(PGPASSFILE=$target_pass psql -XAt --host="$target_host" --port="$target_port" \ - --username="$target_user" --dbname="$target_database" --command="$identity_sql") -[ "$active_identity" != "$target_identity" ] || { - echo "refusing restore: active source and target are on the same PostgreSQL cluster" >&2 - exit 2 -} - -object_count=$(PGPASSFILE=$target_pass psql -XAt --host="$target_host" --port="$target_port" \ - --username="$target_user" --dbname="$target_database" --command=" - SELECT count(*) FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace - WHERE (n.nspname='vectors' OR (n.nspname='public' AND c.relname='tht_vector_migrations')) - AND c.relkind IN ('r','p','S','v','m');") -if [ "$object_count" != 0 ] && [ "$force" != 1 ]; then - echo "refusing restore into non-empty target; use --force-nonempty explicitly" >&2 - exit 2 +if [ -n "$running_container" ]; then + docker compose --project-name "$project_name" stop qdrant >/dev/null + restart_qdrant=1 fi -PGPASSFILE=$target_pass pg_restore --exit-on-error --single-transaction \ - --clean --if-exists --no-owner \ - --host="$target_host" --port="$target_port" --username="$target_user" \ - --dbname="$target_database" "$input" -echo "Vector restore completed into explicit target $target_host:$target_port/$target_database" +parent_dir=$(dirname "$mountpoint") +staging_dir=$(mktemp -d "$parent_dir/.qdrant-restore-staging.XXXXXX") +extract_dir=$(mktemp -d "${TMPDIR:-/tmp}/qdrant-restore.XXXXXX") +tar -C "$extract_dir" -xf "$input" + +find "$mountpoint" -mindepth 1 -maxdepth 1 -exec mv {} "$staging_dir"/ \; +find "$extract_dir" -mindepth 1 -maxdepth 1 -exec mv {} "$mountpoint"/ \; + +rm -rf "$staging_dir" +staging_dir= +rm -rf "$extract_dir" +extract_dir= +echo "Qdrant restore completed into $volume_name for compose project $project_name"