ThothII/backend/src/workspaces/qdrant-collection.ts

export const QDRANT_REQUIRED_INDEXES = Object.freeze([
  "content_hash", "document_id", "kind", "record_key",
  "record_kind", "vector_generation", "workspace_id", "workspace_revision",
]);

export type CollectionMode = "self_heal" | "require_existing" | "evidence_maintenance";

export interface CollectionCheck {
  ok: boolean;
  code?: "semantic_index_incompatible" | "workspace_not_activatable";
  state?: "ready" | "created" | "repaired" | "upgraded";
}

export interface ReconcileCollectionOptions {
  baseUrl: string;
  collection: string;
  dimensions: number;
  distance: string;
  mode: CollectionMode;
  request?: typeof fetch;
  signal?: AbortSignal;
}

function qdrantDistance(distance: string): string {
  return distance.length === 0 ? distance : distance.charAt(0).toUpperCase() + distance.slice(1);
}

function qdrantUrl(baseUrl: string, path: string): string {
  return new URL(path, baseUrl).toString();
}

async function collectionInfo(opts: ReconcileCollectionOptions, request: typeof fetch): Promise<unknown | undefined> {
  const res = await request(qdrantUrl(opts.baseUrl, `/collections/${encodeURIComponent(opts.collection)}`), { method: "GET", signal: opts.signal });
  if (res.status === 404) return undefined;
  if (!res.ok) throw new Error("qdrant collection check failed");
  return (await res.json() as any)?.result;
}

function vectorCompatibility(info: any, opts: ReconcileCollectionOptions): boolean {
  const vectors = info?.config?.params?.vectors;
  return Boolean(vectors && vectors.size === opts.dimensions && typeof vectors.distance === "string"
    && vectors.distance.toLowerCase() === opts.distance);
}

async function missingIndexes(opts: ReconcileCollectionOptions, info: any): Promise<string[]> {
  const payloadSchema = info?.payload_schema;
  if (!payloadSchema || typeof payloadSchema !== "object") return [...QDRANT_REQUIRED_INDEXES];
  return QDRANT_REQUIRED_INDEXES.filter((field) => payloadSchema[field]?.data_type !== "keyword");
}

async function createCollection(opts: ReconcileCollectionOptions, request: typeof fetch): Promise<void> {
  const res = await request(qdrantUrl(opts.baseUrl, `/collections/${encodeURIComponent(opts.collection)}`), {
    method: "PUT",
    headers: { "content-type": "application/json" },
    body: JSON.stringify({
      vectors: { size: opts.dimensions, distance: qdrantDistance(opts.distance) },
      ...(opts.mode === "evidence_maintenance" ? { sparse_vectors: { bm25: { modifier: "idf" } } } : {}),
    }),
    signal: opts.signal,
  });
  if (!res.ok && res.status !== 409) throw new Error("qdrant collection creation failed");
}

type EvidenceSparseCompatibility = "compatible" | "upgradeable" | "incompatible";

function evidenceSparseCompatibility(info: any): EvidenceSparseCompatibility {
  const sparseVectors = info?.config?.params?.sparse_vectors;
  if (sparseVectors === undefined) return "upgradeable";
  if (!sparseVectors || typeof sparseVectors !== "object" || Array.isArray(sparseVectors)) {
    return "incompatible";
  }
  const bm25 = sparseVectors.bm25;
  if (bm25 === undefined) return "upgradeable";
  return typeof bm25 === "object" && bm25 !== null && !Array.isArray(bm25)
    && typeof bm25.modifier === "string" && bm25.modifier.toLowerCase() === "idf"
    ? "compatible" : "incompatible";
}

async function createBm25Vector(opts: ReconcileCollectionOptions, request: typeof fetch): Promise<void> {
  const res = await request(
    qdrantUrl(opts.baseUrl, `/collections/${encodeURIComponent(opts.collection)}/vectors/bm25`),
    {
      method: "PUT",
      headers: { "content-type": "application/json" },
      body: JSON.stringify({ sparse: { modifier: "idf" } }),
      signal: opts.signal,
    },
  );
  if (!res.ok && res.status !== 409) throw new Error("qdrant BM25 vector creation failed");
}

async function createIndex(opts: ReconcileCollectionOptions, field: string, request: typeof fetch): Promise<void> {
  const res = await request(qdrantUrl(opts.baseUrl, `/collections/${encodeURIComponent(opts.collection)}/index`), {
    method: "PUT",
    headers: { "content-type": "application/json" },
    body: JSON.stringify({ field_name: field, field_schema: "keyword" }),
    signal: opts.signal,
  });
  if (!res.ok && res.status !== 409) throw new Error("qdrant index creation failed");
}

/** Reconcile a Qdrant collection. Only Evidence maintenance may add the BM25 sparse vector;
 *  session admission remains limited to the existing dense/index self-heal behavior. */
export async function reconcileCollection(opts: ReconcileCollectionOptions): Promise<CollectionCheck> {
  const request = opts.request ?? fetch;
  const allowsMutation = opts.mode === "self_heal" || opts.mode === "evidence_maintenance";
  let info = await collectionInfo(opts, request);
  if (info === undefined) {
    if (!allowsMutation) return { ok: false, code: "semantic_index_incompatible" };
    await createCollection(opts, request);
    // Tolerate an already-compatible concurrent creator: re-read the final state.
    info = await collectionInfo(opts, request);
    if (info === undefined) return { ok: false, code: "workspace_not_activatable" };
  }
  if (!vectorCompatibility(info, opts)) {
    return { ok: false, code: "semantic_index_incompatible" };
  }
  let bm25Added = false;
  if (opts.mode === "evidence_maintenance") {
    const sparse = evidenceSparseCompatibility(info);
    if (sparse === "incompatible") return { ok: false, code: "semantic_index_incompatible" };
    if (sparse === "upgradeable") {
      await createBm25Vector(opts, request);
      info = await collectionInfo(opts, request);
      if (!vectorCompatibility(info, opts) || evidenceSparseCompatibility(info) !== "compatible") {
        return { ok: false, code: "semantic_index_incompatible" };
      }
      bm25Added = true;
    }
  }
  const missing = await missingIndexes(opts, info);
  if (missing.length > 0) {
    if (!allowsMutation) return { ok: false, code: "semantic_index_incompatible" };
    for (const field of missing) await createIndex(opts, field, request);
    // Qdrant payload indexes become visible asynchronously: poll until the
    // contract is complete or a bounded deadline passes (fail closed).
    const deadline = Date.now() + 15000;
    let current: any = info;
    while (Date.now() < deadline) {
      current = await collectionInfo(opts, request);
      if (!vectorCompatibility(current, opts)) break;
      if ((await missingIndexes(opts, current)).length === 0) {
        return { ok: true, state: "repaired" };
      }
      await new Promise((resolve) => setTimeout(resolve, 500));
    }
    return { ok: false, code: "semantic_index_incompatible" };
  }
  return { ok: true, state: bm25Added ? "upgraded" : "ready" };
}