mptyl/ThothII
1
0
Fork 0
Code Issues 25 Pull Requests Actions Packages Projects Releases 1 Wiki Activity

feat: P4 qdrant collection lifecycle (self-heal + guarded rebuild)

Browse Source 
- shared TS collection manager: self-heal creates missing collection (1024/cosine)
  and missing keyword payload indexes; never mutates incompatible contracts
  (semantic_index_incompatible); async index visibility polled with bounded deadline
- session admission (qdrantEnsure) uses the manager in self-heal mode; operator path
  keeps require_existing semantics
- runtime lease exposes semanticQdrantUrl to the operator
- operator commands vector-inspect/vector-rebuild with exact confirmation guards
- thothctl workspace vector inspect|rebuild (Go) with --collection/--confirm/--destroy
- p4 acceptance runner: real Qdrant (v1.18.2) lifecycle checks, 11/11 PASS
- docs: CLI contract, manual walkthrough P4 (PENDING), PROJECT_STATE
This commit is contained in:
marcopan
2026-08-12 20:00:14 +02:00
parent 230a876314
commit e056c19e62
20 changed files with 1041 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/test/qdrant-collection.test.ts
+88
View File
@@ -0,0 +1,88 @@
import { expect, test } from "vitest";
import { QDRANT_REQUIRED_INDEXES, reconcileCollection } from "../src/workspaces/qdrant-collection.js";
function fakeRequest(info: any | undefined, { create = true, index = true } = {}) {
let current = info;
let created = false;
return async (url: string, init?: any) => {
if (init?.method === "PUT" && /\/index$/.test(url)) {
if (!index) return { status: 409, ok: false, json: async () => ({}) } as any;
// Simulate the index being created: the collection becomes fully compatible.
current = compatible();
return { status: 200, ok: true, json: async () => ({}) } as any;
}
if (init?.method === "PUT") {
if (!create) return { status: 409, ok: false, json: async () => ({}) } as any;
created = true;
current = compatible();
return { status: 200, ok: true, json: async () => ({}) } as any;
}
if (current === undefined) return { status: 404, ok: false, json: async () => ({}) } as any;
return { status: 200, ok: true, json: async () => ({ result: current }) } as any;
};
}
const payloadSchema = Object.fromEntries(QDRANT_REQUIRED_INDEXES.map((f) => [f, { data_type: "keyword" }]));
const compatible = (size = 1024, distance = "Cosine", schema = payloadSchema) => ({
config: { params: { vectors: { size, distance } } },
payload_schema: schema,
});
test("self-heal creates a missing compatible collection", async () => {
const r = await reconcileCollection({
baseUrl: "http://qdrant:6333", collection: "c", dimensions: 1024, distance: "cosine",
mode: "self_heal", request: fakeRequest(undefined),
});
expect(r.ok).toBe(true);
});
test("require_existing refuses a missing collection", async () => {
const r = await reconcileCollection({
baseUrl: "http://qdrant:6333", collection: "c", dimensions: 1024, distance: "cosine",
mode: "require_existing", request: fakeRequest(undefined),
});
expect(r).toEqual({ ok: false, code: "semantic_index_incompatible" });
});
test("self-heal adds missing keyword indexes", async () => {
const schema = { ...payloadSchema };
delete schema["workspace_revision"];
const r = await reconcileCollection({
baseUrl: "http://qdrant:6333", collection: "c", dimensions: 1024, distance: "cosine",
mode: "self_heal", request: fakeRequest(compatible(1024, "Cosine", schema)),
});
expect(r).toMatchObject({ ok: true, state: "repaired" });
});
test("refuses incompatible dimensions or distance without mutating", async () => {
for (const info of [compatible(768, "Cosine"), compatible(1024, "Dot")]) {
const r = await reconcileCollection({
baseUrl: "http://qdrant:6333", collection: "c", dimensions: 1024, distance: "cosine",
mode: "self_heal", request: fakeRequest(info),
});
expect(r).toEqual({ ok: false, code: "semantic_index_incompatible" });
}
});
test("self-heal creates with the Qdrant-valid distance enum", async () => {
let createdBody: any;
const base = fakeRequest(undefined);
const request = async (url: string, init?: any) => {
if (init?.method === "PUT" && !/\/index$/.test(url)) createdBody = JSON.parse(String(init.body));
return base(url, init);
};
const r = await reconcileCollection({
baseUrl: "http://qdrant:6333", collection: "c", dimensions: 1024, distance: "cosine",
mode: "self_heal", request,
});
expect(r.ok).toBe(true);
expect(createdBody.vectors.distance).toBe("Cosine");
});
test("ready compatible collection passes", async () => {
const r = await reconcileCollection({
baseUrl: "http://qdrant:6333", collection: "c", dimensions: 1024, distance: "cosine",
mode: "require_existing", request: fakeRequest(compatible()),
});
expect(r).toMatchObject({ ok: true, state: "ready" });
});
backend/test/routes-sessions.test.ts
+1 -1
View File
@@ -2476,7 +2476,7 @@ test("POST /sessions proceeds when ollamaEnsure succeeds", async () => {
});
const res = await app.inject({ method: "POST", url: "/sessions", payload: { question: "q" } });
expect(res.json()).toEqual({ id: "s1" });
expect(qdrantEnsure).toHaveBeenCalledWith(operationalWorkspace("psd"), 60);
expect(qdrantEnsure).toHaveBeenCalledWith(operationalWorkspace("psd"), 60, "self_heal");
expect(ensureWs).toContain(`/snapshots/${"e".repeat(40)}/psd.yaml`);
});
backend/test/tht-qdrant-readiness.test.ts
+1 -1
View File
@@ -58,7 +58,7 @@ function collection(overrides: Record<string, unknown> = {}) {
test("Qdrant readiness uses only the internal URL and accepts the exact collection contract", async () => {
const request = vi.fn(async () => response(200, collection()));
await expect(runner(request).qdrantEnsure(workspace, 3)).resolves.toEqual({ ok: true });
await expect(runner(request).qdrantEnsure(workspace, 3)).resolves.toEqual({ ok: true, state: "ready" });
expect(request).toHaveBeenCalledOnce();
expect(request.mock.calls[0][0]).toBe("http://qdrant:6333/collections/psd");
expect(request.mock.calls[0][1]).toMatchObject({ method: "GET", signal: expect.any(AbortSignal) });
backend/test/workspace-maintenance.test.ts
+29
View File
@@ -83,3 +83,32 @@ test("raw exception text is redacted from stderr and stdout remains within the p
expect(captured.stderr.join("")).not.toContain("secret.example.invalid");
expect(captured.stderr.join("")).not.toContain("SELECT *");
});
test("vector-inspect and vector-rebuild dispatch to the service with the exact envelope", async () => {
const service = {
vectorInspect: vi.fn(async () => ok("vector inspect")),
vectorRebuild: vi.fn(async () => ok("vector rebuild")),
} as any;
const inspectIo = io(JSON.stringify({ schemaVersion: 1, workspaceId: "psd-clinical" }));
expect(await runWorkspaceMaintenanceCli(["node", "workspace-maintenance", "vector-inspect"], service, inspectIo)).toBe(0);
expect(service.vectorInspect).toHaveBeenCalledWith({ workspaceId: "psd-clinical" });
expect(JSON.parse(inspectIo.stdout.join(""))).toMatchObject({ operation: "vector inspect", code: "ok" });
const rebuildIo = io(JSON.stringify({ schemaVersion: 1, workspaceId: "psd-clinical", collection: "psd-clinical", confirm: "psd-clinical", destroy: true }));
expect(await runWorkspaceMaintenanceCli(["node", "workspace-maintenance", "vector-rebuild"], service, rebuildIo)).toBe(0);
expect(service.vectorRebuild).toHaveBeenCalledWith({
workspaceId: "psd-clinical",
collection: "psd-clinical",
confirm: "psd-clinical",
destroy: true,
});
});
test("vector-rebuild without exact confirmation is refused by the service", async () => {
const service = {
vectorRebuild: vi.fn(async () => ({ ...ok("vector rebuild"), status: "failed", code: "semantic_index_incompatible" as const })),
} as any;
const rebuildIo = io(JSON.stringify({ schemaVersion: 1, workspaceId: "psd-clinical", collection: "other", confirm: "other", destroy: true }));
expect(await runWorkspaceMaintenanceCli(["node", "workspace-maintenance", "vector-rebuild"], service, rebuildIo)).toBe(1);
});