feat(evidence): add Qdrant BM25 vector in place

backend/src/workspaces/evidence/preprocessing.ts

@@ -14,6 +14,7 @@ export interface EvidencePreprocessingDependencies {
   runStage(argv: string[]): Promise<Record<string, unknown>>;
   persistJob(): void;
   semanticPreflight(): Promise<{ ok: true } | { ok: false; code: SemanticFailureCode }>;
+  evidencePreflight(): Promise<{ ok: true } | { ok: false; code: SemanticFailureCode }>;
   requireRunId(value: unknown): string;
   numberRecord(value: unknown): Record<string, number> | undefined;
 }
@@ -138,7 +139,7 @@ export async function preprocessEvidence(
   }
   const policy = evidencePolicy(request.evidence, request.httpPrivateHostAllowlist);
   if (policy) return policy;
-  const semantic = await deps.semanticPreflight();
+  const semantic = await deps.evidencePreflight();
   if (!semantic.ok) {
     return { status: "failed", code: semantic.code, runId: request.job.runId };
   }

backend/src/workspaces/preprocessing-service.ts

@@ -73,6 +73,9 @@ export interface WorkspacePreprocessingServiceDeps {
   semanticPreflight(workspace: WorkspaceDescriptor): Promise<
     { ok: true } | { ok: false; code: "workspace_not_activatable" | "semantic_index_incompatible" }
   >;
+  evidencePreflight(workspace: WorkspaceDescriptor): Promise<
+    { ok: true } | { ok: false; code: "workspace_not_activatable" | "semantic_index_incompatible" }
+  >;
   httpPrivateHostAllowlist?: readonly string[];
 }
 
@@ -410,6 +413,7 @@ export class WorkspacePreprocessingService {
       runStage: async (argv) => await this.runJsonStage(scope.runtime, argv),
       persistJob: () => this.state(scope.runtime.workspaceId).writeJob(scope.job),
       semanticPreflight: async () => await this.deps.semanticPreflight(scope.runtime.workspace),
+      evidencePreflight: async () => await this.deps.evidencePreflight(scope.runtime.workspace),
       requireRunId: (value) => this.requireRunId(value),
       numberRecord: (value) => this.numberRecord(value),
     };

backend/src/workspaces/qdrant-collection.ts

@@ -3,12 +3,12 @@ export const QDRANT_REQUIRED_INDEXES = Object.freeze([
   "record_kind", "vector_generation", "workspace_id", "workspace_revision",
 ]);
 
-export type CollectionMode = "self_heal" | "require_existing";
+export type CollectionMode = "self_heal" | "require_existing" | "evidence_maintenance";
 
 export interface CollectionCheck {
   ok: boolean;
   code?: "semantic_index_incompatible" | "workspace_not_activatable";
-  state?: "ready" | "created" | "repaired";
+  state?: "ready" | "created" | "repaired" | "upgraded";
 }
 
 export interface ReconcileCollectionOptions {
@@ -52,12 +52,43 @@ async function createCollection(opts: ReconcileCollectionOptions, request: typeo
   const res = await request(qdrantUrl(opts.baseUrl, `/collections/${encodeURIComponent(opts.collection)}`), {
     method: "PUT",
     headers: { "content-type": "application/json" },
-    body: JSON.stringify({ vectors: { size: opts.dimensions, distance: qdrantDistance(opts.distance) } }),
+    body: JSON.stringify({
+      vectors: { size: opts.dimensions, distance: qdrantDistance(opts.distance) },
+      ...(opts.mode === "evidence_maintenance" ? { sparse_vectors: { bm25: { modifier: "idf" } } } : {}),
+    }),
     signal: opts.signal,
   });
   if (!res.ok && res.status !== 409) throw new Error("qdrant collection creation failed");
 }
 
+type EvidenceSparseCompatibility = "compatible" | "upgradeable" | "incompatible";
+
+function evidenceSparseCompatibility(info: any): EvidenceSparseCompatibility {
+  const sparseVectors = info?.config?.params?.sparse_vectors;
+  if (sparseVectors === undefined) return "upgradeable";
+  if (!sparseVectors || typeof sparseVectors !== "object" || Array.isArray(sparseVectors)) {
+    return "incompatible";
+  }
+  const bm25 = sparseVectors.bm25;
+  if (bm25 === undefined) return "upgradeable";
+  return typeof bm25 === "object" && bm25 !== null && !Array.isArray(bm25)
+    && typeof bm25.modifier === "string" && bm25.modifier.toLowerCase() === "idf"
+    ? "compatible" : "incompatible";
+}
+
+async function createBm25Vector(opts: ReconcileCollectionOptions, request: typeof fetch): Promise<void> {
+  const res = await request(
+    qdrantUrl(opts.baseUrl, `/collections/${encodeURIComponent(opts.collection)}/vectors/bm25`),
+    {
+      method: "PUT",
+      headers: { "content-type": "application/json" },
+      body: JSON.stringify({ sparse: { modifier: "idf" } }),
+      signal: opts.signal,
+    },
+  );
+  if (!res.ok && res.status !== 409) throw new Error("qdrant BM25 vector creation failed");
+}
+
 async function createIndex(opts: ReconcileCollectionOptions, field: string, request: typeof fetch): Promise<void> {
   const res = await request(qdrantUrl(opts.baseUrl, `/collections/${encodeURIComponent(opts.collection)}/index`), {
     method: "PUT",
@@ -68,13 +99,14 @@ async function createIndex(opts: ReconcileCollectionOptions, field: string, requ
   if (!res.ok && res.status !== 409) throw new Error("qdrant index creation failed");
 }
 
-/** Reconcile a Qdrant collection: self-heal creates missing collections/indexes; require_existing
- *  only validates and refuses incompatible contracts (never mutates). */
+/** Reconcile a Qdrant collection. Only Evidence maintenance may add the BM25 sparse vector;
+ *  session admission remains limited to the existing dense/index self-heal behavior. */
 export async function reconcileCollection(opts: ReconcileCollectionOptions): Promise<CollectionCheck> {
   const request = opts.request ?? fetch;
+  const allowsMutation = opts.mode === "self_heal" || opts.mode === "evidence_maintenance";
   let info = await collectionInfo(opts, request);
   if (info === undefined) {
-    if (opts.mode !== "self_heal") return { ok: false, code: "semantic_index_incompatible" };
+    if (!allowsMutation) return { ok: false, code: "semantic_index_incompatible" };
     await createCollection(opts, request);
     // Tolerate an already-compatible concurrent creator: re-read the final state.
     info = await collectionInfo(opts, request);
@@ -83,9 +115,22 @@ export async function reconcileCollection(opts: ReconcileCollectionOptions): Pro
   if (!vectorCompatibility(info, opts)) {
     return { ok: false, code: "semantic_index_incompatible" };
   }
+  let bm25Added = false;
+  if (opts.mode === "evidence_maintenance") {
+    const sparse = evidenceSparseCompatibility(info);
+    if (sparse === "incompatible") return { ok: false, code: "semantic_index_incompatible" };
+    if (sparse === "upgradeable") {
+      await createBm25Vector(opts, request);
+      info = await collectionInfo(opts, request);
+      if (!vectorCompatibility(info, opts) || evidenceSparseCompatibility(info) !== "compatible") {
+        return { ok: false, code: "semantic_index_incompatible" };
+      }
+      bm25Added = true;
+    }
+  }
   const missing = await missingIndexes(opts, info);
   if (missing.length > 0) {
-    if (opts.mode !== "self_heal") return { ok: false, code: "semantic_index_incompatible" };
+    if (!allowsMutation) return { ok: false, code: "semantic_index_incompatible" };
     for (const field of missing) await createIndex(opts, field, request);
     // Qdrant payload indexes become visible asynchronously: poll until the
     // contract is complete or a bounded deadline passes (fail closed).
@@ -101,5 +146,5 @@ export async function reconcileCollection(opts: ReconcileCollectionOptions): Pro
     }
     return { ok: false, code: "semantic_index_incompatible" };
   }
-  return { ok: true, state: "ready" };
+  return { ok: true, state: bm25Added ? "upgraded" : "ready" };
 }