diff --git a/backend/src/workspaces/migrate-legacy.ts b/backend/src/workspaces/migrate-legacy.ts deleted file mode 100644 index 3830322e..00000000 --- a/backend/src/workspaces/migrate-legacy.ts +++ /dev/null @@ -1,213 +0,0 @@ -import { lstat, mkdir, readFile, rename, rm, writeFile } from "node:fs/promises"; -import { basename, dirname, isAbsolute, join, resolve } from "node:path"; -import { fileURLToPath } from "node:url"; -import { parseAllDocuments, stringify } from "yaml"; -import { parseWorkspaceYaml, validateOperationalWorkspace, type WorkspaceDescriptor, type WorkspaceV3 } from "./schema.js"; - -export interface LegacyMigrationResult { - source: string; - workspace: WorkspaceV3; -} - -export interface LegacyMigrationOptions { - /** Immutable repository identifier, normally derived from the input filename by the CLI. */ - id: string; - /** Required Qdrant collection name for the migrated schema-v3 descriptor. */ - collection: string; -} - -type LegacyRecord = Record; - -const workspaceId = /^[a-z][a-z0-9-]{2,62}$/; -const identifier = /^[A-Za-z_][A-Za-z0-9_]*$/; - -function record(value: unknown): LegacyRecord | undefined { - return value !== null && typeof value === "object" && !Array.isArray(value) - ? value as LegacyRecord - : undefined; -} - -function literalIdentifier(value: unknown): string | undefined { - return typeof value === "string" && identifier.test(value) ? value : undefined; -} - -function literalText(value: unknown): string | undefined { - return typeof value === "string" && value.trim() === value && value.length > 0 && !value.includes("${") - ? value - : undefined; -} - -function literalPort(value: unknown): number | undefined { - if (typeof value === "number" && Number.isInteger(value) && value > 0 && value <= 65_535) return value; - return undefined; -} - -function titleFor(id: string): string { - return id.split("-").map((word) => word[0].toUpperCase() + word.slice(1)).join(" "); -} - -function sourceDocument(source: string): LegacyRecord { - const documents = parseAllDocuments(source, { uniqueKeys: true }); - if (documents.length !== 1 || documents[0].errors.length > 0) { - throw new Error("legacy workspace YAML must contain exactly one valid document"); - } - const parsed = record(documents[0].toJSON()); - if (!parsed) throw new Error("legacy workspace YAML must contain an object"); - return parsed; -} - -function dwhFrom(source: LegacyRecord): { section: LegacyRecord; transport: "postgres_direct" | "rest_api" } { - const dwh = record(source.dwh); - const database = record(source.database); - if (dwh) { - const type = literalText(dwh.type); - return { section: record(dwh.connection) ?? record(dwh.database) ?? dwh, transport: type === "postgres_direct" ? "postgres_direct" : "rest_api" }; - } - if (database) { - return { section: database, transport: literalText(database.transport) === "direct" ? "postgres_direct" : "rest_api" }; - } - return { section: {}, transport: "rest_api" }; -} - -function vectorFrom(source: LegacyRecord): { - section: LegacyRecord; transport: "pgvector_direct" | "rest_api"; writer: boolean; -} { - const vectors = record(source.vectors); - if (vectors) { - const type = literalText(vectors.type); - const direct = record(vectors.direct); - return { - section: type === "pgvector_direct" ? record(vectors.connection) ?? record(vectors.reader) ?? direct ?? {} : direct ?? {}, - transport: type === "pgvector_direct" ? "pgvector_direct" : "rest_api", - writer: record(vectors.writer) !== undefined, - }; - } - const vectorDb = record(source.vector_db); - return { section: vectorDb ?? {}, transport: "pgvector_direct", writer: record(source.vector_write_rest) !== undefined }; -} - -/** - * Converts a legacy runtime descriptor into a schema-v3 registry descriptor. - * Runtime YAMLs mix shared metadata with `${ENV}` bindings and omit internal semantic identity, - * so the operator must explicitly choose the target Qdrant collection during migration. - */ -export function migrateLegacyWorkspace(source: string, options: LegacyMigrationOptions): LegacyMigrationResult { - if (!workspaceId.test(options.id)) throw new Error("legacy workspace ID is invalid"); - const collection = typeof options.collection === "string" && workspaceId.test(options.collection) - ? options.collection - : undefined; - if (collection === undefined) throw new Error("legacy migration requires an explicit target collection"); - const legacy = sourceDocument(source); - const language = legacy.language === "it" ? "it" : "en"; - const { section: dwh, transport: dwhTransport } = dwhFrom(legacy); - const { section: vector } = vectorFrom(legacy); - const embedding = record(legacy.embeddings) ?? {}; - const dwhDatabase = literalIdentifier(dwh.database) ?? "legacy_dwh"; - const dwhSchema = literalIdentifier(dwh.schema) ?? "public"; - void vector; - void embedding; - const workspace = validateOperationalWorkspace({ - workspace: { - schema_version: 3, - id: options.id, - name: titleFor(options.id), - language, - }, - dwh: { - engine: "postgres", - database: dwhDatabase, - schema: dwhSchema, - supported_transports: [dwhTransport], - ...(literalPort(dwh.port) === undefined ? {} : { port: literalPort(dwh.port) }), - }, - semantic_index: { - vector_store: { - engine: "qdrant", - collection, - dimensions: 1024, - distance: "cosine", - }, - embedding: { - provider: "ollama_internal", - model: "qwen3-embedding:0.6b", - dimensions: 1024, - }, - }, - llm_policy: { allowed: ["zai/glm-5.2"] }, - }); - const rendered = stringify(workspace, { lineWidth: 0, sortMapEntries: true }); - return { source: rendered, workspace }; -} - -function destinationFor(repositoryRoot: string, id: string): string { - if (!isAbsolute(repositoryRoot)) throw new Error("migration output root must be absolute"); - if (!workspaceId.test(id)) throw new Error("legacy workspace ID is invalid"); - return join(repositoryRoot, "workspaces", `${id}.yaml`); -} - -/** Safely adds a migrated descriptor without replacing a previous operator-reviewed migration. */ -export async function writeMigratedWorkspace(result: LegacyMigrationResult, repositoryRoot: string): Promise { - const destination = destinationFor(repositoryRoot, result.workspace.workspace.id); - const directory = dirname(destination); - await mkdir(directory, { recursive: true, mode: 0o700 }); - try { - await lstat(destination); - throw new Error("migrated workspace already exists"); - } catch (error) { - if (!(error instanceof Error) || !("code" in error) || error.code !== "ENOENT") throw error; - } - const temporary = join(directory, `.${result.workspace.workspace.id}.${process.pid}.${Date.now()}.tmp`); - try { - await writeFile(temporary, result.source, { encoding: "utf8", mode: 0o600, flag: "wx" }); - await rename(temporary, destination); - } catch (error) { - await rm(temporary, { force: true }); - throw error; - } - return destination; -} - -function parseCliArguments(argv: readonly string[]): { input: string; output: string; id?: string; collection: string } { - if (argv.length !== 6 && argv.length !== 8) { - throw new Error("usage: migrate-legacy --input --output --collection [--id ]"); - } - const options = new Map(); - for (let index = 0; index < argv.length; index += 2) { - const flag = argv[index]; - const value = argv[index + 1]; - if ((flag !== "--input" && flag !== "--output" && flag !== "--id" && flag !== "--collection") || value === undefined || options.has(flag)) { - throw new Error("usage: migrate-legacy --input --output --collection [--id ]"); - } - options.set(flag, value); - } - const input = options.get("--input"); - const output = options.get("--output"); - const id = options.get("--id"); - const collection = options.get("--collection"); - if (input === undefined || output === undefined || collection === undefined) { - throw new Error("usage: migrate-legacy --input --output --collection [--id ]"); - } - if (!isAbsolute(input) || !isAbsolute(output)) { - throw new Error("migration input and output paths must be absolute"); - } - if (id !== undefined && !workspaceId.test(id)) throw new Error("legacy workspace ID is invalid"); - if (!workspaceId.test(collection)) { - throw new Error("legacy migration requires an explicit target collection"); - } - return { input, output, id, collection }; -} - -export async function main(argv = process.argv.slice(2)): Promise { - const { input, output, id: explicitId, collection } = parseCliArguments(argv); - const id = explicitId ?? basename(input, ".yaml"); - const result = migrateLegacyWorkspace(await readFile(input, "utf8"), { id, collection }); - const destination = await writeMigratedWorkspace(result, output); - process.stdout.write(`${destination}\n`); -} - -if (process.argv[1] !== undefined && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { - main().catch((error: unknown) => { - process.stderr.write(`${error instanceof Error ? error.message : "migration failed"}\n`); - process.exitCode = 1; - }); -} diff --git a/backend/src/workspaces/migrate-v2-qdrant.ts b/backend/src/workspaces/migrate-v2-qdrant.ts deleted file mode 100644 index 99b39c72..00000000 --- a/backend/src/workspaces/migrate-v2-qdrant.ts +++ /dev/null @@ -1,58 +0,0 @@ -import { - validateOperationalWorkspace, - type CanonicalDiagnostics, - type DwhTransport, - type WorkspaceV3, -} from "./schema.js"; - -/** Legacy input exists only at the migration boundary and is never an accepted runtime descriptor. */ -interface WorkspaceV2MigrationInput { - workspace: { - schema_version: 2; - id: string; - name: string; - description?: string; - language: "en" | "it"; - }; - dwh: { - engine: "postgres"; - database: string; - schema: string; - port?: number; - timeout_ms?: number; - supported_transports: DwhTransport[]; - }; - semantic_index: unknown; - llm_policy: { - default?: `${string}/${string}`; - allowed: `${string}/${string}`[]; - }; - diagnostics?: CanonicalDiagnostics; -} - -export function migrateWorkspaceV2ToV3( - legacy: WorkspaceV2MigrationInput, - collection: string, -): WorkspaceV3 { - return validateOperationalWorkspace({ - workspace: { ...legacy.workspace, schema_version: 3 }, - dwh: legacy.dwh, - semantic_index: { - vector_store: { - engine: "qdrant", - collection, - dimensions: 1024, - distance: "cosine", - }, - embedding: { - provider: "ollama_internal", - model: "qwen3-embedding:0.6b", - dimensions: 1024, - }, - }, - llm_policy: legacy.llm_policy, - ...(legacy.diagnostics?.dwh_rest - ? { diagnostics: { dwh_rest: legacy.diagnostics.dwh_rest } } - : {}), - }); -} diff --git a/backend/test/workspace-registry-deployment.test.ts b/backend/test/workspace-registry-deployment.test.ts new file mode 100644 index 00000000..e1d7f5a0 --- /dev/null +++ b/backend/test/workspace-registry-deployment.test.ts @@ -0,0 +1,43 @@ +import { execFileSync } from "node:child_process"; +import { existsSync, readFileSync } from "node:fs"; +import { expect, test } from "vitest"; + +test("declares a durable isolated registry volume and only read-only Git credential mounts", () => { + const compose = readFileSync(new URL("../../compose.yaml", import.meta.url), "utf8"); + const development = readFileSync(new URL("../../docker-compose.dev.yml", import.meta.url), "utf8"); + const gitHttps = readFileSync(new URL("../../deploy/compose.git-https.yaml", import.meta.url), "utf8"); + const gitSsh = readFileSync(new URL("../../deploy/compose.git-ssh.yaml", import.meta.url), "utf8"); + const dockerfile = readFileSync(new URL("../../docker/core.Dockerfile", import.meta.url), "utf8"); + const smoke = readFileSync(new URL("../../scripts/workspace-registry-smoke.sh", import.meta.url), "utf8"); + + for (const source of [compose, development]) { + expect(source).toContain("THT_WORKSPACE_REGISTRY_ROOT: /data/workspace-registry"); + expect(source).toContain("THT_WORKSPACE_GIT_REMOTE: ${THT_WORKSPACE_GIT_REMOTE:?set THT_WORKSPACE_GIT_REMOTE}"); + expect(source).toContain("workspace-registry:/data/workspace-registry"); + } + expect(compose).not.toMatch(/workspace-registry-git-(?:credentials|ca|ssh-key|known-hosts):ro/); + expect(gitHttps).toMatch(/workspace-registry-git-credentials:ro/); + expect(gitHttps).toMatch(/workspace-registry-git-ca:ro/); + expect(gitSsh).toMatch(/workspace-registry-git-ssh-key:ro/); + expect(gitSsh).toMatch(/workspace-registry-git-known-hosts:ro/); + expect(dockerfile).toMatch(/mkdir -p[^\n]*\/data\/workspace-registry/); + expect(dockerfile).toMatch(/chown -R thoth:thoth \/home\/thoth\/\.pi \/data/); + expect(smoke).toContain('core_remote="/fixtures/offline.git"'); + expect(smoke).toContain('"degraded":true'); + expect(smoke).toContain('core_remote="/fixtures/remote.git"'); +}); + +test("workspace registry smoke image cleanup is scoped to the per-run image identity", () => { + const output = execFileSync("bash", ["scripts/workspace-registry-smoke.sh"], { + cwd: new URL("../..", import.meta.url), + env: { ...process.env, WORKSPACE_REGISTRY_SMOKE_SELF_TEST: "image-cleanup-identity" }, + encoding: "utf8", + }); + + expect(output).toContain("workspace registry smoke image cleanup identity self-test passed"); +}); + +test("workspace migration source modules are absent from the live backend boundary", () => { + expect(existsSync(new URL("../src/workspaces/migrate-legacy.ts", import.meta.url))).toBe(false); + expect(existsSync(new URL("../src/workspaces/migrate-v2-qdrant.ts", import.meta.url))).toBe(false); +}); diff --git a/backend/test/workspaces-migrate-legacy.test.ts b/backend/test/workspaces-migrate-legacy.test.ts deleted file mode 100644 index 71d05ac1..00000000 --- a/backend/test/workspaces-migrate-legacy.test.ts +++ /dev/null @@ -1,114 +0,0 @@ -import { execFileSync } from "node:child_process"; -import { existsSync, readFileSync, rmSync, writeFileSync } from "node:fs"; -import { mkdtemp } from "node:fs/promises"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; -import { afterEach, expect, test } from "vitest"; -import { - main, - migrateLegacyWorkspace, - writeMigratedWorkspace, -} from "../src/workspaces/migrate-legacy.js"; -import * as workspaceSchema from "../src/workspaces/schema.js"; -import { parseWorkspaceYaml } from "../src/workspaces/schema.js"; - -const temporaryRoots: string[] = []; - -afterEach(() => { - temporaryRoots.splice(0).forEach((root) => rmSync(root, { recursive: true, force: true })); -}); - -function readFixture(name: string): string { - return readFileSync(new URL(`../../harness/workspaces/${name}`, import.meta.url), "utf8"); -} - -test("migrates the current local PSD descriptor without copying secret values", () => { - const result = migrateLegacyWorkspace(readFixture("local.yaml"), { id: "local", collection: "local" }); - - expect(result.workspace.workspace).toMatchObject({ id: "local", schema_version: 3, language: "it" }); - expect(result.workspace).not.toHaveProperty("evidence"); - expect(JSON.stringify(result)).not.toMatch(/password:|api_key:|\$\{THT_/i); -}); - -test("migrates a legacy descriptor only with an explicit target collection into schema v3", () => { - const result = migrateLegacyWorkspace(readFixture("tht.example.yaml"), { id: "example", collection: "shared" }); - const isOperationalWorkspace = (workspaceSchema as { isOperationalWorkspace?: unknown }).isOperationalWorkspace; - - expect(result.workspace.workspace.schema_version).toBe(3); - expect(parseWorkspaceYaml(result.source).workspace.schema_version).toBe(3); - expect(isOperationalWorkspace).toBeTypeOf("function"); - expect((isOperationalWorkspace as (workspace: ReturnType) => boolean)( - parseWorkspaceYaml(result.source), - )).toBe(true); - expect(parseWorkspaceYaml(result.source)).toMatchObject({ - semantic_index: { vector_store: { engine: "qdrant", collection: "shared" } }, - }); -}); - -test("requires an explicit target collection for legacy migration", () => { - expect(() => migrateLegacyWorkspace(readFixture("local.yaml"), { id: "local" } as never)).toThrow( - /collection/i, - ); -}); - -test("writes versioned repository artifacts atomically without replacing a prior migration", async () => { - const root = await mkdtemp(join(tmpdir(), "thoth-workspace-migrate-")); - temporaryRoots.push(root); - const migration = migrateLegacyWorkspace(readFixture("local.yaml"), { id: "local", collection: "local" }); - - const destination = await writeMigratedWorkspace(migration, root); - - expect(destination).toBe(join(root, "workspaces", "local.yaml")); - expect(parseWorkspaceYaml(readFileSync(destination, "utf8"))).toMatchObject({ workspace: { id: "local" } }); - await expect(writeMigratedWorkspace(migration, root)).rejects.toThrow(/already exists/i); - expect(existsSync(destination)).toBe(true); -}); - -test("CLI accepts an explicit valid ID when a legacy filename contains dots", async () => { - const root = await mkdtemp(join(tmpdir(), "thoth-workspace-migrate-")); - temporaryRoots.push(root); - const input = join(root, "psd.clinical.yaml"); - writeFileSync(input, readFixture("local.yaml")); - - await main(["--input", input, "--output", root, "--id", "psd-clinical", "--collection", "psd-clinical"]); - - const destination = join(root, "workspaces", "psd-clinical.yaml"); - expect(parseWorkspaceYaml(readFileSync(destination, "utf8"))).toMatchObject({ - workspace: { id: "psd-clinical", schema_version: 3 }, - }); -}); - -test("declares a durable isolated registry volume and only read-only Git credential mounts", () => { - const compose = readFileSync(new URL("../../compose.yaml", import.meta.url), "utf8"); - const development = readFileSync(new URL("../../docker-compose.dev.yml", import.meta.url), "utf8"); - const gitHttps = readFileSync(new URL("../../deploy/compose.git-https.yaml", import.meta.url), "utf8"); - const gitSsh = readFileSync(new URL("../../deploy/compose.git-ssh.yaml", import.meta.url), "utf8"); - const dockerfile = readFileSync(new URL("../../docker/core.Dockerfile", import.meta.url), "utf8"); - const smoke = readFileSync(new URL("../../scripts/workspace-registry-smoke.sh", import.meta.url), "utf8"); - - for (const source of [compose, development]) { - expect(source).toContain("THT_WORKSPACE_REGISTRY_ROOT: /data/workspace-registry"); - expect(source).toContain("THT_WORKSPACE_GIT_REMOTE: ${THT_WORKSPACE_GIT_REMOTE:?set THT_WORKSPACE_GIT_REMOTE}"); - expect(source).toContain("workspace-registry:/data/workspace-registry"); - } - expect(compose).not.toMatch(/workspace-registry-git-(?:credentials|ca|ssh-key|known-hosts):ro/); - expect(gitHttps).toMatch(/workspace-registry-git-credentials:ro/); - expect(gitHttps).toMatch(/workspace-registry-git-ca:ro/); - expect(gitSsh).toMatch(/workspace-registry-git-ssh-key:ro/); - expect(gitSsh).toMatch(/workspace-registry-git-known-hosts:ro/); - expect(dockerfile).toMatch(/mkdir -p[^\n]*\/data\/workspace-registry/); - expect(dockerfile).toMatch(/chown -R thoth:thoth \/home\/thoth\/\.pi \/data/); - expect(smoke).toContain('core_remote="/fixtures/offline.git"'); - expect(smoke).toContain('"degraded":true'); - expect(smoke).toContain('core_remote="/fixtures/remote.git"'); -}); - -test("workspace registry smoke image cleanup is scoped to the per-run image identity", () => { - const output = execFileSync("bash", ["scripts/workspace-registry-smoke.sh"], { - cwd: new URL("../..", import.meta.url), - env: { ...process.env, WORKSPACE_REGISTRY_SMOKE_SELF_TEST: "image-cleanup-identity" }, - encoding: "utf8", - }); - - expect(output).toContain("workspace registry smoke image cleanup identity self-test passed"); -}); diff --git a/backend/test/workspaces-migrate-v2-qdrant.test.ts b/backend/test/workspaces-migrate-v2-qdrant.test.ts deleted file mode 100644 index ec6412a5..00000000 --- a/backend/test/workspaces-migrate-v2-qdrant.test.ts +++ /dev/null @@ -1,102 +0,0 @@ -import { expect, test } from "vitest"; -import { migrateWorkspaceV2ToV3 } from "../src/workspaces/migrate-v2-qdrant.js"; -import { parseWorkspaceYaml } from "../src/workspaces/schema.js"; - -const workspaceV2Yaml = `workspace: - schema_version: 2 - id: psd-clinical - name: Policlinico San Donato - description: Clinical data warehouse workspace - language: it -dwh: - engine: postgres - database: postgres - schema: datawarehouse - supported_transports: - - postgres_direct - - rest_api -semantic_index: - vector_store: - engine: pgvector - database: postgres - schema: vectors - collection: clinical_documents - dimensions: 768 - distance: inner_product - supported_transports: - - pgvector_direct - - rest_api - embedding: - provider: openai_compatible - model: text-embedding-3-large - dimensions: 768 -llm_policy: - default: zai/glm-5.2 - allowed: - - zai/glm-5.2 -diagnostics: - dwh_rest: - method: POST - path: /rpc/dwh - auth: bearer - response: - database: database - schema: schema - vector_rest: - metadata: - method: GET - path: /vector - auth: bearer - response: - collection: collection - dimensions: dimensions - distance: distance - embedding: - method: GET - path: /models - auth: none - response: - model: model - dimensions: dimensions -`; - -test("migrates a schema v2 workspace to the internal qdrant schema v3 shape", () => { - const legacy = parseWorkspaceYaml(workspaceV2Yaml); - - const migrated = migrateWorkspaceV2ToV3(legacy, "psd-clinical"); - - expect(migrated).not.toHaveProperty("evidence"); - expect(migrated).toMatchObject({ - workspace: { schema_version: 3, id: "psd-clinical", name: "Policlinico San Donato" }, - dwh: legacy.dwh, - semantic_index: { - vector_store: { - engine: "qdrant", - collection: "psd-clinical", - dimensions: 1024, - distance: "cosine", - }, - embedding: { - provider: "ollama_internal", - model: "qwen3-embedding:0.6b", - dimensions: 1024, - }, - }, - llm_policy: legacy.llm_policy, - diagnostics: { - dwh_rest: legacy.diagnostics?.dwh_rest, - }, - }); -}); - -test("drops vector and embedding diagnostics and transports during v2 to v3 migration", () => { - const legacy = parseWorkspaceYaml(workspaceV2Yaml); - - const migrated = migrateWorkspaceV2ToV3(legacy, "psd-clinical"); - - expect(migrated.diagnostics).toEqual({ - dwh_rest: legacy.diagnostics?.dwh_rest, - }); - expect(migrated.semantic_index.vector_store).not.toHaveProperty("supported_transports"); - expect(migrated.semantic_index.embedding).not.toHaveProperty("timeout_ms"); -});