mptyl/ThothII
1
0
Fork 0
Code Issues 25 Pull Requests Actions Packages Projects Releases 1 Wiki Activity

feat: reserve one qdrant collection per workspace

Browse Source
This commit is contained in:
marcopan
2026-08-08 17:03:51 +02:00
parent ba1d7b0e78
commit 76bc94d5da
6 changed files with 288 additions and 107 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/src/workspaces/migrate-legacy.ts
+42 -46
View File
@@ -2,17 +2,18 @@ import { lstat, mkdir, readFile, rename, rm, writeFile } from "node:fs/promises"
import { basename, dirname, isAbsolute, join, resolve } from "node:path";
import { fileURLToPath } from "node:url";
import { parseAllDocuments, stringify } from "yaml";
import { parseWorkspaceYaml, type LegacyWorkspace, type WorkspaceDescriptor } from "./schema.js";
import { parseWorkspaceYaml, validateOperationalWorkspace, type WorkspaceDescriptor, type WorkspaceV3 } from "./schema.js";
export interface LegacyMigrationResult {
state: "migration_required";
source: string;
workspace: LegacyWorkspace;
workspace: WorkspaceV3;
}
export interface LegacyMigrationOptions {
/** Immutable repository identifier, normally derived from the input filename by the CLI. */
id: string;
/** Required Qdrant collection name for the migrated schema-v3 descriptor. */
collection: string;
}
type LegacyRecord = Record<string, unknown>;
@@ -86,38 +87,28 @@ function vectorFrom(source: LegacyRecord): {
}
/**
* Converts a legacy runtime descriptor into a versioned, readable v1 registry descriptor.
* Runtime YAMLs mix shared metadata with `${ENV}` bindings and omit semantic-index identity;
* the result therefore always remains `migration_required` until an operator explicitly upgrades
* it with the correct collection/database/schema contract.
* Converts a legacy runtime descriptor into a schema-v3 registry descriptor.
* Runtime YAMLs mix shared metadata with `${ENV}` bindings and omit internal semantic identity,
* so the operator must explicitly choose the target Qdrant collection during migration.
*/
export function migrateLegacyWorkspace(source: string, options: LegacyMigrationOptions): LegacyMigrationResult {
if (!workspaceId.test(options.id)) throw new Error("legacy workspace ID is invalid");
const collection = typeof options.collection === "string" && workspaceId.test(options.collection)
? options.collection
: undefined;
if (collection === undefined) throw new Error("legacy migration requires an explicit target collection");
const legacy = sourceDocument(source);
const language = legacy.language === "it" ? "it" : "en";
const { section: dwh, transport: dwhTransport } = dwhFrom(legacy);
const { section: vector, transport: vectorTransport, writer } = vectorFrom(legacy);
const { section: vector } = vectorFrom(legacy);
const embedding = record(legacy.embeddings) ?? {};
const dwhDatabase = literalIdentifier(dwh.database) ?? "legacy_dwh";
const dwhSchema = literalIdentifier(dwh.schema) ?? "public";
const vectorDatabase = literalIdentifier(vector.database);
const vectorSchema = literalIdentifier(vector.schema);
const dimensions = typeof embedding.dim === "number" && Number.isInteger(embedding.dim) && embedding.dim > 0
? embedding.dim
: 768;
const vectorStore: LegacyWorkspace["semantic_index"]["vector_store"] = {
engine: "pgvector",
collection: `${options.id.replaceAll("-", "_")}_documents`,
dimensions,
distance: "cosine",
supported_transports: [vectorTransport],
...(literalPort(vector.port) === undefined ? {} : { port: literalPort(vector.port) }),
...(vectorDatabase === undefined ? {} : { database: vectorDatabase }),
...(vectorSchema === undefined ? {} : { schema: vectorSchema }),
};
const workspace: LegacyWorkspace = {
void vector;
void embedding;
const workspace = validateOperationalWorkspace({
workspace: {
schema_version: 1,
schema_version: 3,
id: options.id,
name: titleFor(options.id),
language,
@@ -130,21 +121,22 @@ export function migrateLegacyWorkspace(source: string, options: LegacyMigrationO
...(literalPort(dwh.port) === undefined ? {} : { port: literalPort(dwh.port) }),
},
semantic_index: {
vector_store: vectorStore,
...(writer ? { vector_writer: {} } : {}),
vector_store: {
engine: "qdrant",
collection,
dimensions: 1024,
distance: "cosine",
},
embedding: {
provider: "ollama_compatible",
model: literalText(embedding.model) ?? "legacy-embedding",
dimensions,
provider: "ollama_internal",
model: "qwen3-embedding:0.6b",
dimensions: 1024,
},
},
llm_policy: { allowed: ["zai/glm-5.2"] },
};
const descriptor = parseWorkspaceYaml(stringify(workspace, { lineWidth: 0, sortMapEntries: true }));
if (descriptor.workspace.schema_version !== 1) throw new Error("legacy workspace migration is invalid");
const migrated = descriptor as LegacyWorkspace;
const rendered = stringify(migrated, { lineWidth: 0, sortMapEntries: true });
return { state: "migration_required", source: rendered, workspace: migrated };
});
const rendered = stringify(workspace, { lineWidth: 0, sortMapEntries: true });
return { source: rendered, workspace };
}
function destinationFor(repositoryRoot: string, id: string): string {
@@ -175,36 +167,40 @@ export async function writeMigratedWorkspace(result: LegacyMigrationResult, repo
return destination;
}
function parseCliArguments(argv: readonly string[]): { input: string; output: string; id?: string } {
if (argv.length !== 4 && argv.length !== 6) {
throw new Error("usage: migrate-legacy --input <legacy-workspace.yaml> --output <repository-root> [--id <workspace-id>]");
function parseCliArguments(argv: readonly string[]): { input: string; output: string; id?: string; collection: string } {
if (argv.length !== 6 && argv.length !== 8) {
throw new Error("usage: migrate-legacy --input <legacy-workspace.yaml> --output <repository-root> --collection <qdrant-collection> [--id <workspace-id>]");
}
const options = new Map<string, string>();
for (let index = 0; index < argv.length; index += 2) {
const flag = argv[index];
const value = argv[index + 1];
if ((flag !== "--input" && flag !== "--output" && flag !== "--id") || value === undefined || options.has(flag)) {
throw new Error("usage: migrate-legacy --input <legacy-workspace.yaml> --output <repository-root> [--id <workspace-id>]");
if ((flag !== "--input" && flag !== "--output" && flag !== "--id" && flag !== "--collection") || value === undefined || options.has(flag)) {
throw new Error("usage: migrate-legacy --input <legacy-workspace.yaml> --output <repository-root> --collection <qdrant-collection> [--id <workspace-id>]");
}
options.set(flag, value);
}
const input = options.get("--input");
const output = options.get("--output");
const id = options.get("--id");
if (input === undefined || output === undefined) {
throw new Error("usage: migrate-legacy --input <legacy-workspace.yaml> --output <repository-root> [--id <workspace-id>]");
const collection = options.get("--collection");
if (input === undefined || output === undefined || collection === undefined) {
throw new Error("usage: migrate-legacy --input <legacy-workspace.yaml> --output <repository-root> --collection <qdrant-collection> [--id <workspace-id>]");
}
if (!isAbsolute(input) || !isAbsolute(output)) {
throw new Error("migration input and output paths must be absolute");
}
if (id !== undefined && !workspaceId.test(id)) throw new Error("legacy workspace ID is invalid");
return { input, output, id };
if (!workspaceId.test(collection)) {
throw new Error("legacy migration requires an explicit target collection");
}
return { input, output, id, collection };
}
export async function main(argv = process.argv.slice(2)): Promise<void> {
const { input, output, id: explicitId } = parseCliArguments(argv);
const { input, output, id: explicitId, collection } = parseCliArguments(argv);
const id = explicitId ?? basename(input, ".yaml");
const result = migrateLegacyWorkspace(await readFile(input, "utf8"), { id });
const result = migrateLegacyWorkspace(await readFile(input, "utf8"), { id, collection });
const destination = await writeMigratedWorkspace(result, output);
process.stdout.write(`${destination}\n`);
}
backend/src/workspaces/registry.ts
+10
View File
@@ -211,6 +211,9 @@ export class WorkspaceRegistry {
const state = await this.activeState();
const revision = state.revisions.find((candidate) => candidate.id === id);
if (!revision) throw new WorkspaceRegistryError("workspace_invalid", "Workspace is unavailable");
if (revision.state !== "operational") {
throw new WorkspaceRegistryError("workspace_invalid", "Workspace is unavailable");
}
let workspace: WorkspaceDescriptor;
try {
workspace = parseWorkspaceYaml(await readFile(revision.snapshotPath, "utf8"));
@@ -477,6 +480,7 @@ export class WorkspaceRegistry {
blob: string;
state: WorkspaceRevision["state"];
}> = [];
const collectionOwners = new Map<string, string>();
try {
for (const path of files) {
const id = path.slice("workspaces/".length, -".yaml".length);
@@ -490,6 +494,12 @@ export class WorkspaceRegistry {
? "operational"
: "migration_required";
if (isCanonicalWorkspace(workspace)) {
const collection = workspace.semantic_index.vector_store.collection;
const owner = collectionOwners.get(collection);
if (owner !== undefined) {
throw new Error(`duplicate qdrant collection ownership: ${collection} (${owner}, ${id})`);
}
collectionOwners.set(collection, id);
buildInstallationContract(workspace);
renderWorkspaceDocs(workspace);
snapshotSource = serializeWorkspaceYaml(workspace);