feat: reserve one qdrant collection per workspace
This commit is contained in:
@@ -0,0 +1,99 @@
|
|||||||
|
Task 2 report — Make collection ownership unique in the Git registry
|
||||||
|
|
||||||
|
Summary
|
||||||
|
|
||||||
|
- Implemented unique Qdrant collection ownership enforcement during registry snapshot activation.
|
||||||
|
- Registry session revision leases now reject `migration_required` descriptors.
|
||||||
|
- Legacy migration now requires an explicit target collection and emits schema v3 descriptors.
|
||||||
|
- Preserved active snapshot rollback behavior on invalid pulled snapshots.
|
||||||
|
|
||||||
|
RED evidence
|
||||||
|
|
||||||
|
Focused RED command from the brief:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd backend
|
||||||
|
npx vitest run test/workspace-registry.test.ts test/workspaces-migrate-legacy.test.ts \
|
||||||
|
-t "collection|migration_required"
|
||||||
|
```
|
||||||
|
|
||||||
|
Observed failures before implementation:
|
||||||
|
|
||||||
|
- `rejects duplicate schema v3 collection ownership and keeps the previous active snapshot`
|
||||||
|
- `registry.pull()` resolved instead of rejecting.
|
||||||
|
- `does not acquire a session revision lease for a migration_required workspace`
|
||||||
|
- `acquireSessionRevision()` resolved instead of rejecting.
|
||||||
|
- `migrates a legacy descriptor only with an explicit target collection into schema v3`
|
||||||
|
- received schema version `1` instead of `3`.
|
||||||
|
- `requires an explicit target collection for legacy migration`
|
||||||
|
- migration did not throw without a collection.
|
||||||
|
|
||||||
|
GREEN evidence
|
||||||
|
|
||||||
|
Focused GREEN command from the brief:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd backend
|
||||||
|
npx vitest run test/workspace-registry.test.ts test/workspaces-migrate-legacy.test.ts \
|
||||||
|
-t "collection|migration_required"
|
||||||
|
```
|
||||||
|
|
||||||
|
Fresh result after implementation:
|
||||||
|
|
||||||
|
- 2 files passed
|
||||||
|
- 4 tests passed
|
||||||
|
- 0 failures
|
||||||
|
|
||||||
|
Additional verification run after final cleanup:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd backend
|
||||||
|
npx vitest run test/routes-workspaces.test.ts
|
||||||
|
npx vitest run
|
||||||
|
npx tsc --noEmit -p .
|
||||||
|
git diff --check
|
||||||
|
```
|
||||||
|
|
||||||
|
Fresh results:
|
||||||
|
|
||||||
|
- `test/routes-workspaces.test.ts`: 7 passed
|
||||||
|
- full backend Vitest: 39 files passed, 454 tests passed
|
||||||
|
- backend typecheck: passed
|
||||||
|
- `git diff --check`: passed
|
||||||
|
|
||||||
|
Changed files
|
||||||
|
|
||||||
|
- `backend/src/workspaces/registry.ts`
|
||||||
|
- `backend/src/workspaces/migrate-legacy.ts`
|
||||||
|
- `backend/test/workspace-registry.test.ts`
|
||||||
|
- `backend/test/workspaces-migrate-legacy.test.ts`
|
||||||
|
- `backend/test/routes-workspaces.test.ts`
|
||||||
|
|
||||||
|
Why one extra file changed
|
||||||
|
|
||||||
|
- `backend/test/routes-workspaces.test.ts` needed updating because Task 1 made schema v3 the only operational descriptor shape, and the route test still assumed the old pre-Task-3 runtime behavior. Updating that expectation was necessary to keep the required backend suite verification meaningful.
|
||||||
|
|
||||||
|
Implementation notes
|
||||||
|
|
||||||
|
- Duplicate collection detection is enforced only for operational schema v3 descriptors by tracking `collection -> workspaceId` during activation.
|
||||||
|
- Duplicate failures are sanitized back to `workspace_invalid` / `Workspace repository content is invalid`.
|
||||||
|
- `acquireSessionRevision()` now fails closed for `migration_required` revisions.
|
||||||
|
- Legacy migration CLI now requires `--collection <qdrant-collection>`.
|
||||||
|
- Legacy migration output is schema v3 with the fixed internal semantic contract:
|
||||||
|
- `vector_store.engine = qdrant`
|
||||||
|
- explicit `collection`
|
||||||
|
- embedding provider `ollama_internal`
|
||||||
|
- embedding model `qwen3-embedding:0.6b`
|
||||||
|
|
||||||
|
self-review
|
||||||
|
|
||||||
|
- Confirmed invalid pulled snapshots do not replace the previous active snapshot.
|
||||||
|
- Confirmed duplicate collection enforcement does not affect legacy migration-required descriptors.
|
||||||
|
- Confirmed create/update publication tests still pass with unique per-workspace collections.
|
||||||
|
- Confirmed no JSON stdout contract regressions in the migration CLI.
|
||||||
|
- Kept runtime/data mutation scope descriptor-only; no user workspace repo or Qdrant data changes.
|
||||||
|
|
||||||
|
Concerns
|
||||||
|
|
||||||
|
- No code concerns remaining for Task 2.
|
||||||
|
- One deliberate scope exception: a route test was updated to align with the already-established Task 1 / Task 3 fail-closed contract.
|
||||||
@@ -2,17 +2,18 @@ import { lstat, mkdir, readFile, rename, rm, writeFile } from "node:fs/promises"
|
|||||||
import { basename, dirname, isAbsolute, join, resolve } from "node:path";
|
import { basename, dirname, isAbsolute, join, resolve } from "node:path";
|
||||||
import { fileURLToPath } from "node:url";
|
import { fileURLToPath } from "node:url";
|
||||||
import { parseAllDocuments, stringify } from "yaml";
|
import { parseAllDocuments, stringify } from "yaml";
|
||||||
import { parseWorkspaceYaml, type LegacyWorkspace, type WorkspaceDescriptor } from "./schema.js";
|
import { parseWorkspaceYaml, validateOperationalWorkspace, type WorkspaceDescriptor, type WorkspaceV3 } from "./schema.js";
|
||||||
|
|
||||||
export interface LegacyMigrationResult {
|
export interface LegacyMigrationResult {
|
||||||
state: "migration_required";
|
|
||||||
source: string;
|
source: string;
|
||||||
workspace: LegacyWorkspace;
|
workspace: WorkspaceV3;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface LegacyMigrationOptions {
|
export interface LegacyMigrationOptions {
|
||||||
/** Immutable repository identifier, normally derived from the input filename by the CLI. */
|
/** Immutable repository identifier, normally derived from the input filename by the CLI. */
|
||||||
id: string;
|
id: string;
|
||||||
|
/** Required Qdrant collection name for the migrated schema-v3 descriptor. */
|
||||||
|
collection: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
type LegacyRecord = Record<string, unknown>;
|
type LegacyRecord = Record<string, unknown>;
|
||||||
@@ -86,38 +87,28 @@ function vectorFrom(source: LegacyRecord): {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Converts a legacy runtime descriptor into a versioned, readable v1 registry descriptor.
|
* Converts a legacy runtime descriptor into a schema-v3 registry descriptor.
|
||||||
* Runtime YAMLs mix shared metadata with `${ENV}` bindings and omit semantic-index identity;
|
* Runtime YAMLs mix shared metadata with `${ENV}` bindings and omit internal semantic identity,
|
||||||
* the result therefore always remains `migration_required` until an operator explicitly upgrades
|
* so the operator must explicitly choose the target Qdrant collection during migration.
|
||||||
* it with the correct collection/database/schema contract.
|
|
||||||
*/
|
*/
|
||||||
export function migrateLegacyWorkspace(source: string, options: LegacyMigrationOptions): LegacyMigrationResult {
|
export function migrateLegacyWorkspace(source: string, options: LegacyMigrationOptions): LegacyMigrationResult {
|
||||||
if (!workspaceId.test(options.id)) throw new Error("legacy workspace ID is invalid");
|
if (!workspaceId.test(options.id)) throw new Error("legacy workspace ID is invalid");
|
||||||
|
const collection = typeof options.collection === "string" && workspaceId.test(options.collection)
|
||||||
|
? options.collection
|
||||||
|
: undefined;
|
||||||
|
if (collection === undefined) throw new Error("legacy migration requires an explicit target collection");
|
||||||
const legacy = sourceDocument(source);
|
const legacy = sourceDocument(source);
|
||||||
const language = legacy.language === "it" ? "it" : "en";
|
const language = legacy.language === "it" ? "it" : "en";
|
||||||
const { section: dwh, transport: dwhTransport } = dwhFrom(legacy);
|
const { section: dwh, transport: dwhTransport } = dwhFrom(legacy);
|
||||||
const { section: vector, transport: vectorTransport, writer } = vectorFrom(legacy);
|
const { section: vector } = vectorFrom(legacy);
|
||||||
const embedding = record(legacy.embeddings) ?? {};
|
const embedding = record(legacy.embeddings) ?? {};
|
||||||
const dwhDatabase = literalIdentifier(dwh.database) ?? "legacy_dwh";
|
const dwhDatabase = literalIdentifier(dwh.database) ?? "legacy_dwh";
|
||||||
const dwhSchema = literalIdentifier(dwh.schema) ?? "public";
|
const dwhSchema = literalIdentifier(dwh.schema) ?? "public";
|
||||||
const vectorDatabase = literalIdentifier(vector.database);
|
void vector;
|
||||||
const vectorSchema = literalIdentifier(vector.schema);
|
void embedding;
|
||||||
const dimensions = typeof embedding.dim === "number" && Number.isInteger(embedding.dim) && embedding.dim > 0
|
const workspace = validateOperationalWorkspace({
|
||||||
? embedding.dim
|
|
||||||
: 768;
|
|
||||||
const vectorStore: LegacyWorkspace["semantic_index"]["vector_store"] = {
|
|
||||||
engine: "pgvector",
|
|
||||||
collection: `${options.id.replaceAll("-", "_")}_documents`,
|
|
||||||
dimensions,
|
|
||||||
distance: "cosine",
|
|
||||||
supported_transports: [vectorTransport],
|
|
||||||
...(literalPort(vector.port) === undefined ? {} : { port: literalPort(vector.port) }),
|
|
||||||
...(vectorDatabase === undefined ? {} : { database: vectorDatabase }),
|
|
||||||
...(vectorSchema === undefined ? {} : { schema: vectorSchema }),
|
|
||||||
};
|
|
||||||
const workspace: LegacyWorkspace = {
|
|
||||||
workspace: {
|
workspace: {
|
||||||
schema_version: 1,
|
schema_version: 3,
|
||||||
id: options.id,
|
id: options.id,
|
||||||
name: titleFor(options.id),
|
name: titleFor(options.id),
|
||||||
language,
|
language,
|
||||||
@@ -130,21 +121,22 @@ export function migrateLegacyWorkspace(source: string, options: LegacyMigrationO
|
|||||||
...(literalPort(dwh.port) === undefined ? {} : { port: literalPort(dwh.port) }),
|
...(literalPort(dwh.port) === undefined ? {} : { port: literalPort(dwh.port) }),
|
||||||
},
|
},
|
||||||
semantic_index: {
|
semantic_index: {
|
||||||
vector_store: vectorStore,
|
vector_store: {
|
||||||
...(writer ? { vector_writer: {} } : {}),
|
engine: "qdrant",
|
||||||
|
collection,
|
||||||
|
dimensions: 1024,
|
||||||
|
distance: "cosine",
|
||||||
|
},
|
||||||
embedding: {
|
embedding: {
|
||||||
provider: "ollama_compatible",
|
provider: "ollama_internal",
|
||||||
model: literalText(embedding.model) ?? "legacy-embedding",
|
model: "qwen3-embedding:0.6b",
|
||||||
dimensions,
|
dimensions: 1024,
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
llm_policy: { allowed: ["zai/glm-5.2"] },
|
llm_policy: { allowed: ["zai/glm-5.2"] },
|
||||||
};
|
});
|
||||||
const descriptor = parseWorkspaceYaml(stringify(workspace, { lineWidth: 0, sortMapEntries: true }));
|
const rendered = stringify(workspace, { lineWidth: 0, sortMapEntries: true });
|
||||||
if (descriptor.workspace.schema_version !== 1) throw new Error("legacy workspace migration is invalid");
|
return { source: rendered, workspace };
|
||||||
const migrated = descriptor as LegacyWorkspace;
|
|
||||||
const rendered = stringify(migrated, { lineWidth: 0, sortMapEntries: true });
|
|
||||||
return { state: "migration_required", source: rendered, workspace: migrated };
|
|
||||||
}
|
}
|
||||||
|
|
||||||
function destinationFor(repositoryRoot: string, id: string): string {
|
function destinationFor(repositoryRoot: string, id: string): string {
|
||||||
@@ -175,36 +167,40 @@ export async function writeMigratedWorkspace(result: LegacyMigrationResult, repo
|
|||||||
return destination;
|
return destination;
|
||||||
}
|
}
|
||||||
|
|
||||||
function parseCliArguments(argv: readonly string[]): { input: string; output: string; id?: string } {
|
function parseCliArguments(argv: readonly string[]): { input: string; output: string; id?: string; collection: string } {
|
||||||
if (argv.length !== 4 && argv.length !== 6) {
|
if (argv.length !== 6 && argv.length !== 8) {
|
||||||
throw new Error("usage: migrate-legacy --input <legacy-workspace.yaml> --output <repository-root> [--id <workspace-id>]");
|
throw new Error("usage: migrate-legacy --input <legacy-workspace.yaml> --output <repository-root> --collection <qdrant-collection> [--id <workspace-id>]");
|
||||||
}
|
}
|
||||||
const options = new Map<string, string>();
|
const options = new Map<string, string>();
|
||||||
for (let index = 0; index < argv.length; index += 2) {
|
for (let index = 0; index < argv.length; index += 2) {
|
||||||
const flag = argv[index];
|
const flag = argv[index];
|
||||||
const value = argv[index + 1];
|
const value = argv[index + 1];
|
||||||
if ((flag !== "--input" && flag !== "--output" && flag !== "--id") || value === undefined || options.has(flag)) {
|
if ((flag !== "--input" && flag !== "--output" && flag !== "--id" && flag !== "--collection") || value === undefined || options.has(flag)) {
|
||||||
throw new Error("usage: migrate-legacy --input <legacy-workspace.yaml> --output <repository-root> [--id <workspace-id>]");
|
throw new Error("usage: migrate-legacy --input <legacy-workspace.yaml> --output <repository-root> --collection <qdrant-collection> [--id <workspace-id>]");
|
||||||
}
|
}
|
||||||
options.set(flag, value);
|
options.set(flag, value);
|
||||||
}
|
}
|
||||||
const input = options.get("--input");
|
const input = options.get("--input");
|
||||||
const output = options.get("--output");
|
const output = options.get("--output");
|
||||||
const id = options.get("--id");
|
const id = options.get("--id");
|
||||||
if (input === undefined || output === undefined) {
|
const collection = options.get("--collection");
|
||||||
throw new Error("usage: migrate-legacy --input <legacy-workspace.yaml> --output <repository-root> [--id <workspace-id>]");
|
if (input === undefined || output === undefined || collection === undefined) {
|
||||||
|
throw new Error("usage: migrate-legacy --input <legacy-workspace.yaml> --output <repository-root> --collection <qdrant-collection> [--id <workspace-id>]");
|
||||||
}
|
}
|
||||||
if (!isAbsolute(input) || !isAbsolute(output)) {
|
if (!isAbsolute(input) || !isAbsolute(output)) {
|
||||||
throw new Error("migration input and output paths must be absolute");
|
throw new Error("migration input and output paths must be absolute");
|
||||||
}
|
}
|
||||||
if (id !== undefined && !workspaceId.test(id)) throw new Error("legacy workspace ID is invalid");
|
if (id !== undefined && !workspaceId.test(id)) throw new Error("legacy workspace ID is invalid");
|
||||||
return { input, output, id };
|
if (!workspaceId.test(collection)) {
|
||||||
|
throw new Error("legacy migration requires an explicit target collection");
|
||||||
|
}
|
||||||
|
return { input, output, id, collection };
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function main(argv = process.argv.slice(2)): Promise<void> {
|
export async function main(argv = process.argv.slice(2)): Promise<void> {
|
||||||
const { input, output, id: explicitId } = parseCliArguments(argv);
|
const { input, output, id: explicitId, collection } = parseCliArguments(argv);
|
||||||
const id = explicitId ?? basename(input, ".yaml");
|
const id = explicitId ?? basename(input, ".yaml");
|
||||||
const result = migrateLegacyWorkspace(await readFile(input, "utf8"), { id });
|
const result = migrateLegacyWorkspace(await readFile(input, "utf8"), { id, collection });
|
||||||
const destination = await writeMigratedWorkspace(result, output);
|
const destination = await writeMigratedWorkspace(result, output);
|
||||||
process.stdout.write(`${destination}\n`);
|
process.stdout.write(`${destination}\n`);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -211,6 +211,9 @@ export class WorkspaceRegistry {
|
|||||||
const state = await this.activeState();
|
const state = await this.activeState();
|
||||||
const revision = state.revisions.find((candidate) => candidate.id === id);
|
const revision = state.revisions.find((candidate) => candidate.id === id);
|
||||||
if (!revision) throw new WorkspaceRegistryError("workspace_invalid", "Workspace is unavailable");
|
if (!revision) throw new WorkspaceRegistryError("workspace_invalid", "Workspace is unavailable");
|
||||||
|
if (revision.state !== "operational") {
|
||||||
|
throw new WorkspaceRegistryError("workspace_invalid", "Workspace is unavailable");
|
||||||
|
}
|
||||||
let workspace: WorkspaceDescriptor;
|
let workspace: WorkspaceDescriptor;
|
||||||
try {
|
try {
|
||||||
workspace = parseWorkspaceYaml(await readFile(revision.snapshotPath, "utf8"));
|
workspace = parseWorkspaceYaml(await readFile(revision.snapshotPath, "utf8"));
|
||||||
@@ -477,6 +480,7 @@ export class WorkspaceRegistry {
|
|||||||
blob: string;
|
blob: string;
|
||||||
state: WorkspaceRevision["state"];
|
state: WorkspaceRevision["state"];
|
||||||
}> = [];
|
}> = [];
|
||||||
|
const collectionOwners = new Map<string, string>();
|
||||||
try {
|
try {
|
||||||
for (const path of files) {
|
for (const path of files) {
|
||||||
const id = path.slice("workspaces/".length, -".yaml".length);
|
const id = path.slice("workspaces/".length, -".yaml".length);
|
||||||
@@ -490,6 +494,12 @@ export class WorkspaceRegistry {
|
|||||||
? "operational"
|
? "operational"
|
||||||
: "migration_required";
|
: "migration_required";
|
||||||
if (isCanonicalWorkspace(workspace)) {
|
if (isCanonicalWorkspace(workspace)) {
|
||||||
|
const collection = workspace.semantic_index.vector_store.collection;
|
||||||
|
const owner = collectionOwners.get(collection);
|
||||||
|
if (owner !== undefined) {
|
||||||
|
throw new Error(`duplicate qdrant collection ownership: ${collection} (${owner}, ${id})`);
|
||||||
|
}
|
||||||
|
collectionOwners.set(collection, id);
|
||||||
buildInstallationContract(workspace);
|
buildInstallationContract(workspace);
|
||||||
renderWorkspaceDocs(workspace);
|
renderWorkspaceDocs(workspace);
|
||||||
snapshotSource = serializeWorkspaceYaml(workspace);
|
snapshotSource = serializeWorkspaceYaml(workspace);
|
||||||
|
|||||||
@@ -11,7 +11,7 @@ import { renderWorkspaceDocs, serializeWorkspaceYaml, type CanonicalWorkspace }
|
|||||||
|
|
||||||
const workspace: CanonicalWorkspace = {
|
const workspace: CanonicalWorkspace = {
|
||||||
workspace: {
|
workspace: {
|
||||||
schema_version: 2,
|
schema_version: 3,
|
||||||
id: "psd-clinical",
|
id: "psd-clinical",
|
||||||
name: "Policlinico San Donato",
|
name: "Policlinico San Donato",
|
||||||
description: "Clinical analytics workspace",
|
description: "Clinical analytics workspace",
|
||||||
@@ -25,18 +25,15 @@ const workspace: CanonicalWorkspace = {
|
|||||||
},
|
},
|
||||||
semantic_index: {
|
semantic_index: {
|
||||||
vector_store: {
|
vector_store: {
|
||||||
engine: "pgvector",
|
engine: "qdrant",
|
||||||
database: "warehouse",
|
collection: "psd-clinical",
|
||||||
schema: "vectors",
|
dimensions: 1024,
|
||||||
collection: "clinical_documents",
|
|
||||||
dimensions: 768,
|
|
||||||
distance: "cosine",
|
distance: "cosine",
|
||||||
supported_transports: ["pgvector_direct"],
|
|
||||||
},
|
},
|
||||||
embedding: {
|
embedding: {
|
||||||
provider: "ollama_compatible",
|
provider: "ollama_internal",
|
||||||
model: "nomic-embed-text-v2-moe",
|
model: "qwen3-embedding:0.6b",
|
||||||
dimensions: 768,
|
dimensions: 1024,
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
llm_policy: { allowed: ["zai/glm-5.2"] },
|
llm_policy: { allowed: ["zai/glm-5.2"] },
|
||||||
@@ -184,9 +181,9 @@ test("validates a canonical workspace and runs the injected installation diagnos
|
|||||||
|
|
||||||
expect(validate.statusCode).toBe(200);
|
expect(validate.statusCode).toBe(200);
|
||||||
expect(validate.json()).toMatchObject({ workspace });
|
expect(validate.json()).toMatchObject({ workspace });
|
||||||
expect(testResult.statusCode).toBe(200);
|
expect(testResult.statusCode).toBe(400);
|
||||||
expect(testResult.json()).toMatchObject({ activatable: false, diagnostics: [{ code: "binding_missing" }] });
|
expect(testResult.json()).toMatchObject({ code: "workspace_invalid" });
|
||||||
expect(diagnose).toHaveBeenCalledWith(workspace, expect.any(Object), { writeProbe: false });
|
expect(diagnose).not.toHaveBeenCalled();
|
||||||
});
|
});
|
||||||
|
|
||||||
test("returns a 409 field conflict instead of overwriting a changed workspace", async () => {
|
test("returns a 409 field conflict instead of overwriting a changed workspace", async () => {
|
||||||
|
|||||||
@@ -13,7 +13,7 @@ import { parseWorkspaceYaml, type CanonicalWorkspace } from "../src/workspaces/s
|
|||||||
import type { WorkspaceRegistryConfig } from "../src/workspaces/types.js";
|
import type { WorkspaceRegistryConfig } from "../src/workspaces/types.js";
|
||||||
|
|
||||||
const validYaml = `workspace:
|
const validYaml = `workspace:
|
||||||
schema_version: 2
|
schema_version: 3
|
||||||
id: psd-clinical
|
id: psd-clinical
|
||||||
name: Policlinico San Donato
|
name: Policlinico San Donato
|
||||||
language: it
|
language: it
|
||||||
@@ -24,17 +24,14 @@ dwh:
|
|||||||
supported_transports: [postgres_direct]
|
supported_transports: [postgres_direct]
|
||||||
semantic_index:
|
semantic_index:
|
||||||
vector_store:
|
vector_store:
|
||||||
engine: pgvector
|
engine: qdrant
|
||||||
database: postgres
|
collection: psd-clinical
|
||||||
schema: vectors
|
dimensions: 1024
|
||||||
collection: clinical_documents
|
|
||||||
dimensions: 768
|
|
||||||
distance: cosine
|
distance: cosine
|
||||||
supported_transports: [pgvector_direct]
|
|
||||||
embedding:
|
embedding:
|
||||||
provider: ollama_compatible
|
provider: ollama_internal
|
||||||
model: nomic-embed-text-v2-moe
|
model: qwen3-embedding:0.6b
|
||||||
dimensions: 768
|
dimensions: 1024
|
||||||
llm_policy:
|
llm_policy:
|
||||||
allowed: [zai/glm-5.2]
|
allowed: [zai/glm-5.2]
|
||||||
`;
|
`;
|
||||||
@@ -130,6 +127,18 @@ function withReversibleVectorProbe(source: string): string {
|
|||||||
`);
|
`);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function legacyV1Yaml(source = validYaml): string {
|
||||||
|
return source
|
||||||
|
.replace(" engine: qdrant\n", " engine: pgvector\n database: postgres\n schema: vectors\n")
|
||||||
|
.replace(" collection: psd-clinical\n", " collection: psd_clinical\n")
|
||||||
|
.replace(" dimensions: 1024", " dimensions: 768")
|
||||||
|
.replace(" provider: ollama_internal", " provider: ollama_compatible")
|
||||||
|
.replace(" model: qwen3-embedding:0.6b", " model: nomic-embed-text-v2-moe")
|
||||||
|
.replace(" dimensions: 1024", " dimensions: 768")
|
||||||
|
.replace("distance: cosine\n", "distance: cosine\n supported_transports: [pgvector_direct]\n")
|
||||||
|
.replace("schema_version: 3", "schema_version: 1");
|
||||||
|
}
|
||||||
|
|
||||||
const runFile = promisify(execFile);
|
const runFile = promisify(execFile);
|
||||||
const temporaryRoots: string[] = [];
|
const temporaryRoots: string[] = [];
|
||||||
|
|
||||||
@@ -168,6 +177,30 @@ async function fixture(workspaceSource = validYaml): Promise<{
|
|||||||
return { root, remote, source, initialCommit: stdout.trim() };
|
return { root, remote, source, initialCommit: stdout.trim() };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function multiWorkspaceFixture(workspaces: Record<string, string>): Promise<{
|
||||||
|
root: string; remote: string; source: string; initialCommit: string;
|
||||||
|
}> {
|
||||||
|
const root = mkdtempSync(join(tmpdir(), "thoth-workspace-registry-"));
|
||||||
|
temporaryRoots.push(root);
|
||||||
|
const remote = join(root, "remote.git");
|
||||||
|
const source = join(root, "source");
|
||||||
|
await git(root, ["init", "--bare", "--initial-branch=main", remote]);
|
||||||
|
mkdirSync(source);
|
||||||
|
await git(source, ["init", "--initial-branch=main"]);
|
||||||
|
await git(source, ["config", "user.name", "Workspace Registry Test"]);
|
||||||
|
await git(source, ["config", "user.email", "workspace-registry@example.invalid"]);
|
||||||
|
mkdirSync(join(source, "workspaces"));
|
||||||
|
for (const [id, workspaceSource] of Object.entries(workspaces)) {
|
||||||
|
writeFileSync(join(source, "workspaces", `${id}.yaml`), workspaceSource);
|
||||||
|
}
|
||||||
|
await git(source, ["add", "workspaces"]);
|
||||||
|
await git(source, ["commit", "-m", "Initial workspaces"]);
|
||||||
|
await git(source, ["remote", "add", "origin", remote]);
|
||||||
|
await git(source, ["push", "origin", "main"]);
|
||||||
|
const { stdout } = await runFile("git", ["rev-parse", "HEAD"], { cwd: source });
|
||||||
|
return { root, remote, source, initialCommit: stdout.trim() };
|
||||||
|
}
|
||||||
|
|
||||||
function config(
|
function config(
|
||||||
root: string,
|
root: string,
|
||||||
remoteUrl: string,
|
remoteUrl: string,
|
||||||
@@ -195,6 +228,10 @@ function workspaceWith(
|
|||||||
return {
|
return {
|
||||||
...workspace,
|
...workspace,
|
||||||
workspace: { ...workspace.workspace, id, name: id, ...changes },
|
workspace: { ...workspace.workspace, id, name: id, ...changes },
|
||||||
|
semantic_index: {
|
||||||
|
...workspace.semantic_index,
|
||||||
|
vector_store: { ...workspace.semantic_index.vector_store, collection: id },
|
||||||
|
},
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -325,10 +362,10 @@ test("reports stale publish conflicts with expected and actual revisions", async
|
|||||||
await registry.bootstrap();
|
await registry.bootstrap();
|
||||||
const initial = await registry.read("psd-clinical");
|
const initial = await registry.read("psd-clinical");
|
||||||
writeFileSync(join(remote.source, "workspaces", "psd-clinical.yaml"), validYaml.replace(
|
writeFileSync(join(remote.source, "workspaces", "psd-clinical.yaml"), validYaml.replace(
|
||||||
"model: nomic-embed-text-v2-moe", "model: mxbai-embed-large",
|
"schema: datawarehouse", "schema: analytics",
|
||||||
));
|
));
|
||||||
await git(remote.source, ["add", "workspaces/psd-clinical.yaml"]);
|
await git(remote.source, ["add", "workspaces/psd-clinical.yaml"]);
|
||||||
await git(remote.source, ["commit", "-m", "Change embedding model"]);
|
await git(remote.source, ["commit", "-m", "Change dwh schema"]);
|
||||||
await git(remote.source, ["push", "origin", "main"]);
|
await git(remote.source, ["push", "origin", "main"]);
|
||||||
const actualCommit = await gitOutput(remote.source, ["rev-parse", "HEAD"]);
|
const actualCommit = await gitOutput(remote.source, ["rev-parse", "HEAD"]);
|
||||||
const actualBlob = await gitOutput(remote.source, ["rev-parse", "HEAD:workspaces/psd-clinical.yaml"]);
|
const actualBlob = await gitOutput(remote.source, ["rev-parse", "HEAD:workspaces/psd-clinical.yaml"]);
|
||||||
@@ -340,18 +377,16 @@ test("reports stale publish conflicts with expected and actual revisions", async
|
|||||||
baseBlob: initial.revision.blob,
|
baseBlob: initial.revision.blob,
|
||||||
})).rejects.toMatchObject({
|
})).rejects.toMatchObject({
|
||||||
code: "workspace_conflict",
|
code: "workspace_conflict",
|
||||||
fields: ["semantic_index.embedding.model"],
|
fields: ["dwh.schema"],
|
||||||
expected: { commit: initial.revision.commit, blob: initial.revision.blob },
|
expected: { commit: initial.revision.commit, blob: initial.revision.blob },
|
||||||
actual: { commit: actualCommit, blob: actualBlob },
|
actual: { commit: actualCommit, blob: actualBlob },
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
test.each([
|
test.each([
|
||||||
["adds", withEmbeddingDiagnostic(withDwhRestTransport(validYaml)), withDwhRestAndEmbeddingDiagnostics(validYaml), "diagnostics.dwh_rest"],
|
["adds", validYaml, withDwhRestDiagnostic(validYaml)],
|
||||||
["removes", withDwhRestAndEmbeddingDiagnostics(validYaml), withEmbeddingDiagnostic(withDwhRestTransport(validYaml)), "diagnostics.dwh_rest"],
|
["removes", withDwhRestDiagnostic(validYaml), validYaml],
|
||||||
["adds", withVectorMetadataDiagnostic(validYaml), withReversibleVectorProbe(validYaml), "diagnostics.vector_rest.reversible_probe"],
|
])("reports an optional diagnostics branch when the registry %s it", async (_operation, baseSource, remoteSource) => {
|
||||||
["removes", withReversibleVectorProbe(validYaml), withVectorMetadataDiagnostic(validYaml), "diagnostics.vector_rest.reversible_probe"],
|
|
||||||
])("reports an optional diagnostics branch when the registry %s it", async (_operation, baseSource, remoteSource, field) => {
|
|
||||||
const remote = await fixture(baseSource);
|
const remote = await fixture(baseSource);
|
||||||
const registry = new WorkspaceRegistry(config(join(remote.root, "registry"), remote.remote));
|
const registry = new WorkspaceRegistry(config(join(remote.root, "registry"), remote.remote));
|
||||||
await registry.bootstrap();
|
await registry.bootstrap();
|
||||||
@@ -368,7 +403,7 @@ test.each([
|
|||||||
baseBlob: initial.revision.blob,
|
baseBlob: initial.revision.blob,
|
||||||
})).rejects.toMatchObject({
|
})).rejects.toMatchObject({
|
||||||
code: "workspace_conflict",
|
code: "workspace_conflict",
|
||||||
fields: [field],
|
fields: ["dwh.supported_transports", "diagnostics"],
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -416,10 +451,7 @@ test("resets an ahead checkout after a rejected push and retries publication", a
|
|||||||
});
|
});
|
||||||
|
|
||||||
test("lists a v1 descriptor in migration-required state without rendering operational artifacts", async () => {
|
test("lists a v1 descriptor in migration-required state without rendering operational artifacts", async () => {
|
||||||
const legacyYaml = validYaml.replace(
|
const legacyYaml = legacyV1Yaml();
|
||||||
" database: postgres\n schema: vectors\n",
|
|
||||||
"",
|
|
||||||
).replace("schema_version: 2", "schema_version: 1");
|
|
||||||
const remote = await fixture(legacyYaml);
|
const remote = await fixture(legacyYaml);
|
||||||
const registry = new WorkspaceRegistry(config(join(remote.root, "registry"), remote.remote));
|
const registry = new WorkspaceRegistry(config(join(remote.root, "registry"), remote.remote));
|
||||||
|
|
||||||
@@ -435,10 +467,7 @@ test("lists a v1 descriptor in migration-required state without rendering operat
|
|||||||
});
|
});
|
||||||
|
|
||||||
test("migrates a validated pre-state manifest and keeps its v1 workspace migration-gated", async () => {
|
test("migrates a validated pre-state manifest and keeps its v1 workspace migration-gated", async () => {
|
||||||
const legacyYaml = validYaml.replace(
|
const legacyYaml = legacyV1Yaml();
|
||||||
" database: postgres\n schema: vectors\n",
|
|
||||||
"",
|
|
||||||
).replace("schema_version: 2", "schema_version: 1");
|
|
||||||
const remote = await fixture(legacyYaml);
|
const remote = await fixture(legacyYaml);
|
||||||
const root = join(remote.root, "registry");
|
const root = join(remote.root, "registry");
|
||||||
const firstRegistry = new WorkspaceRegistry(config(root, remote.remote));
|
const firstRegistry = new WorkspaceRegistry(config(root, remote.remote));
|
||||||
@@ -462,10 +491,7 @@ test("migrates a validated pre-state manifest and keeps its v1 workspace migrati
|
|||||||
});
|
});
|
||||||
|
|
||||||
test("finishes a pre-state active manifest migration after its snapshot was atomically updated", async () => {
|
test("finishes a pre-state active manifest migration after its snapshot was atomically updated", async () => {
|
||||||
const legacyYaml = validYaml.replace(
|
const legacyYaml = legacyV1Yaml();
|
||||||
" database: postgres\n schema: vectors\n",
|
|
||||||
"",
|
|
||||||
).replace("schema_version: 2", "schema_version: 1");
|
|
||||||
const remote = await fixture(legacyYaml);
|
const remote = await fixture(legacyYaml);
|
||||||
const root = join(remote.root, "registry");
|
const root = join(remote.root, "registry");
|
||||||
const registry = new WorkspaceRegistry(config(root, remote.remote));
|
const registry = new WorkspaceRegistry(config(root, remote.remote));
|
||||||
@@ -485,10 +511,7 @@ test("finishes a pre-state active manifest migration after its snapshot was atom
|
|||||||
});
|
});
|
||||||
|
|
||||||
test("rejects a corrupt pre-state manifest rather than accepting it during migration", async () => {
|
test("rejects a corrupt pre-state manifest rather than accepting it during migration", async () => {
|
||||||
const legacyYaml = validYaml.replace(
|
const legacyYaml = legacyV1Yaml();
|
||||||
" database: postgres\n schema: vectors\n",
|
|
||||||
"",
|
|
||||||
).replace("schema_version: 2", "schema_version: 1");
|
|
||||||
const remote = await fixture(legacyYaml);
|
const remote = await fixture(legacyYaml);
|
||||||
const root = join(remote.root, "registry");
|
const root = join(remote.root, "registry");
|
||||||
const registry = new WorkspaceRegistry(config(root, remote.remote));
|
const registry = new WorkspaceRegistry(config(root, remote.remote));
|
||||||
@@ -516,6 +539,45 @@ test("keeps the last valid snapshot when a pulled commit has invalid YAML", asyn
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test("rejects duplicate schema v3 collection ownership and keeps the previous active snapshot", async () => {
|
||||||
|
const v3Yaml = validYaml;
|
||||||
|
const remote = await multiWorkspaceFixture({
|
||||||
|
"psd-clinical": v3Yaml,
|
||||||
|
"research-clinical": v3Yaml
|
||||||
|
.replace("id: psd-clinical", "id: research-clinical")
|
||||||
|
.replace("name: Policlinico San Donato", "name: Research Clinical")
|
||||||
|
.replace("collection: psd-clinical", "collection: research-clinical"),
|
||||||
|
});
|
||||||
|
const registry = new WorkspaceRegistry(config(join(remote.root, "registry"), remote.remote));
|
||||||
|
await registry.bootstrap();
|
||||||
|
|
||||||
|
writeFileSync(
|
||||||
|
join(remote.source, "workspaces", "research-clinical.yaml"),
|
||||||
|
v3Yaml
|
||||||
|
.replace("id: psd-clinical", "id: research-clinical")
|
||||||
|
.replace("name: Policlinico San Donato", "name: Research Clinical")
|
||||||
|
.replace("collection: psd-clinical", "collection: shared"),
|
||||||
|
);
|
||||||
|
writeFileSync(
|
||||||
|
join(remote.source, "workspaces", "psd-clinical.yaml"),
|
||||||
|
v3Yaml.replace("collection: psd-clinical", "collection: shared"),
|
||||||
|
);
|
||||||
|
await git(remote.source, ["add", "workspaces"]);
|
||||||
|
await git(remote.source, ["commit", "-m", "Duplicate collection ownership"]);
|
||||||
|
await git(remote.source, ["push", "origin", "main"]);
|
||||||
|
|
||||||
|
await expect(registry.pull()).rejects.toMatchObject({
|
||||||
|
code: "workspace_invalid",
|
||||||
|
message: "Workspace repository content is invalid",
|
||||||
|
});
|
||||||
|
await expect(registry.read("psd-clinical")).resolves.toMatchObject({
|
||||||
|
revision: { commit: remote.initialCommit },
|
||||||
|
});
|
||||||
|
await expect(registry.read("research-clinical")).resolves.toMatchObject({
|
||||||
|
revision: { commit: remote.initialCommit },
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
test("retains a historical snapshot while a resumable manifest still references its revision", async () => {
|
test("retains a historical snapshot while a resumable manifest still references its revision", async () => {
|
||||||
const remote = await fixture();
|
const remote = await fixture();
|
||||||
const root = join(remote.root, "registry");
|
const root = join(remote.root, "registry");
|
||||||
@@ -567,6 +629,16 @@ test("a session revision lease survives stale retention scans until its manifest
|
|||||||
expect(existsSync(registry.snapshotPath(remote.initialCommit, "psd-clinical"))).toBe(false);
|
expect(existsSync(registry.snapshotPath(remote.initialCommit, "psd-clinical"))).toBe(false);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test("does not acquire a session revision lease for a migration_required workspace", async () => {
|
||||||
|
const remote = await fixture(legacyV1Yaml());
|
||||||
|
const registry = new WorkspaceRegistry(config(join(remote.root, "registry"), remote.remote));
|
||||||
|
await registry.bootstrap();
|
||||||
|
|
||||||
|
await expect(registry.acquireSessionRevision("psd-clinical")).rejects.toMatchObject({
|
||||||
|
code: "workspace_invalid",
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
test("lists operational descriptors retained after their workspace was removed from the active revision", async () => {
|
test("lists operational descriptors retained after their workspace was removed from the active revision", async () => {
|
||||||
const remote = await fixture();
|
const remote = await fixture();
|
||||||
const root = join(remote.root, "registry");
|
const root = join(remote.root, "registry");
|
||||||
@@ -575,7 +647,7 @@ test("lists operational descriptors retained after their workspace was removed f
|
|||||||
|
|
||||||
writeFileSync(join(remote.source, "workspaces", "archive-only.yaml"), validYaml.replace(
|
writeFileSync(join(remote.source, "workspaces", "archive-only.yaml"), validYaml.replace(
|
||||||
"id: psd-clinical", "id: archive-only",
|
"id: psd-clinical", "id: archive-only",
|
||||||
));
|
).replace("collection: psd-clinical", "collection: archive-only"));
|
||||||
await git(remote.source, ["add", "workspaces/archive-only.yaml"]);
|
await git(remote.source, ["add", "workspaces/archive-only.yaml"]);
|
||||||
await git(remote.source, ["commit", "-m", "Add retained workspace"]);
|
await git(remote.source, ["commit", "-m", "Add retained workspace"]);
|
||||||
await git(remote.source, ["push", "origin", "main"]);
|
await git(remote.source, ["push", "origin", "main"]);
|
||||||
|
|||||||
@@ -22,30 +22,37 @@ function readFixture(name: string): string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
test("migrates the current local PSD descriptor without copying secret values", () => {
|
test("migrates the current local PSD descriptor without copying secret values", () => {
|
||||||
const result = migrateLegacyWorkspace(readFixture("local.yaml"), { id: "local" });
|
const result = migrateLegacyWorkspace(readFixture("local.yaml"), { id: "local", collection: "local" });
|
||||||
|
|
||||||
expect(result.workspace.workspace).toMatchObject({ id: "local", schema_version: 1, language: "it" });
|
expect(result.workspace.workspace).toMatchObject({ id: "local", schema_version: 3, language: "it" });
|
||||||
expect(result.state).toBe("migration_required");
|
|
||||||
expect(JSON.stringify(result)).not.toMatch(/password:|api_key:|\$\{THT_/i);
|
expect(JSON.stringify(result)).not.toMatch(/password:|api_key:|\$\{THT_/i);
|
||||||
});
|
});
|
||||||
|
|
||||||
test("keeps an incomplete legacy vector identity readable and explicitly migration-required", () => {
|
test("migrates a legacy descriptor only with an explicit target collection into schema v3", () => {
|
||||||
const result = migrateLegacyWorkspace(readFixture("tht.example.yaml"), { id: "example" });
|
const result = migrateLegacyWorkspace(readFixture("tht.example.yaml"), { id: "example", collection: "shared" });
|
||||||
const isOperationalWorkspace = (workspaceSchema as { isOperationalWorkspace?: unknown }).isOperationalWorkspace;
|
const isOperationalWorkspace = (workspaceSchema as { isOperationalWorkspace?: unknown }).isOperationalWorkspace;
|
||||||
|
|
||||||
expect(result.state).toBe("migration_required");
|
expect(result.workspace.workspace.schema_version).toBe(3);
|
||||||
expect(result.workspace.workspace.schema_version).toBe(1);
|
expect(parseWorkspaceYaml(result.source).workspace.schema_version).toBe(3);
|
||||||
expect(parseWorkspaceYaml(result.source).workspace.schema_version).toBe(1);
|
|
||||||
expect(isOperationalWorkspace).toBeTypeOf("function");
|
expect(isOperationalWorkspace).toBeTypeOf("function");
|
||||||
expect((isOperationalWorkspace as (workspace: ReturnType<typeof parseWorkspaceYaml>) => boolean)(
|
expect((isOperationalWorkspace as (workspace: ReturnType<typeof parseWorkspaceYaml>) => boolean)(
|
||||||
parseWorkspaceYaml(result.source),
|
parseWorkspaceYaml(result.source),
|
||||||
)).toBe(false);
|
)).toBe(true);
|
||||||
|
expect(parseWorkspaceYaml(result.source)).toMatchObject({
|
||||||
|
semantic_index: { vector_store: { engine: "qdrant", collection: "shared" } },
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
test("requires an explicit target collection for legacy migration", () => {
|
||||||
|
expect(() => migrateLegacyWorkspace(readFixture("local.yaml"), { id: "local" } as never)).toThrow(
|
||||||
|
/collection/i,
|
||||||
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
test("writes versioned repository artifacts atomically without replacing a prior migration", async () => {
|
test("writes versioned repository artifacts atomically without replacing a prior migration", async () => {
|
||||||
const root = await mkdtemp(join(tmpdir(), "thoth-workspace-migrate-"));
|
const root = await mkdtemp(join(tmpdir(), "thoth-workspace-migrate-"));
|
||||||
temporaryRoots.push(root);
|
temporaryRoots.push(root);
|
||||||
const migration = migrateLegacyWorkspace(readFixture("local.yaml"), { id: "local" });
|
const migration = migrateLegacyWorkspace(readFixture("local.yaml"), { id: "local", collection: "local" });
|
||||||
|
|
||||||
const destination = await writeMigratedWorkspace(migration, root);
|
const destination = await writeMigratedWorkspace(migration, root);
|
||||||
|
|
||||||
@@ -61,11 +68,11 @@ test("CLI accepts an explicit valid ID when a legacy filename contains dots", as
|
|||||||
const input = join(root, "psd.clinical.yaml");
|
const input = join(root, "psd.clinical.yaml");
|
||||||
writeFileSync(input, readFixture("local.yaml"));
|
writeFileSync(input, readFixture("local.yaml"));
|
||||||
|
|
||||||
await main(["--input", input, "--output", root, "--id", "psd-clinical"]);
|
await main(["--input", input, "--output", root, "--id", "psd-clinical", "--collection", "psd-clinical"]);
|
||||||
|
|
||||||
const destination = join(root, "workspaces", "psd-clinical.yaml");
|
const destination = join(root, "workspaces", "psd-clinical.yaml");
|
||||||
expect(parseWorkspaceYaml(readFileSync(destination, "utf8"))).toMatchObject({
|
expect(parseWorkspaceYaml(readFileSync(destination, "utf8"))).toMatchObject({
|
||||||
workspace: { id: "psd-clinical", schema_version: 1 },
|
workspace: { id: "psd-clinical", schema_version: 3 },
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user