feat: complete catalog-driven preprocessing

backend/src/workspace-maintenance.ts

@@ -1,15 +1,14 @@
 import { spawn } from "node:child_process";
 import { closeSync, constants as fsConstants, openSync } from "node:fs";
-import { readdir, readFile } from "node:fs/promises";
 import { join } from "node:path";
-import { parse } from "yaml";
-import { loadConfig } from "./config.js";
+import { loadConfig, type AppConfig } from "./config.js";
 import { ThtRunner } from "./tht/tht-runner.js";
 import { WorkspaceRegistry } from "./workspaces/registry.js";
 import { publishDeterministicRuntimeConfigLease, renderActiveWorkspaceRuntime } from "./workspaces/runtime-config-lease.js";
 import { WorkspaceSecretStore } from "./workspaces/secret-store.js";
 import { WorkspacePreprocessingService, type WorkspaceOperationResult } from "./workspaces/preprocessing-service.js";
-import type { SessionInventoryRow } from "./workspaces/preprocessing-state.js";
+import { createCatalogRepository } from "./catalog/repository.js";
+import type { CatalogRepository } from "./catalog/types.js";
 
 export interface WorkspaceMaintenanceIo {
   stdin: string;
@@ -19,7 +18,7 @@ export interface WorkspaceMaintenanceIo {
   writeStderr(value: string): void;
 }
 
-type Command = "inspect" | "preprocess-dwh" | "schema-suggest-fks" | "schema-check" | "schema-accept" | "index-schema" | "preprocess-evidence" | "preprocess-run" | "vector-inspect" | "vector-rebuild";
+type Command = "inspect" | "preprocess-run" | "preprocess-clear";
 
 function failureResult(
   operation: string,
@@ -64,15 +63,8 @@ function parseRequest(command: string, stdin: string): Record<string, unknown> {
   }
   const allowedByCommand: Record<string, readonly string[]> = {
     inspect: ["schemaVersion", "workspaceId"],
-    "preprocess-dwh": ["schemaVersion", "workspaceId", "resumeRunId"],
-    "schema-suggest-fks": ["schemaVersion", "workspaceId", "fromSql", "assume", "resumeRunId"],
-    "schema-check": ["schemaVersion", "workspaceId", "annotationsYaml", "reviewedCandidatesDigest"],
-    "schema-accept": ["schemaVersion", "workspaceId", "runId", "yes"],
-    "index-schema": ["schemaVersion", "workspaceId", "resumeRunId"],
-    "preprocess-evidence": ["schemaVersion", "workspaceId", "dryRun", "resumeRunId"],
-    "preprocess-run": ["schemaVersion", "workspaceId", "resumeRunId"],
-    "vector-inspect": ["schemaVersion", "workspaceId"],
-    "vector-rebuild": ["schemaVersion", "workspaceId", "collection", "confirm", "destroy"],
+    "preprocess-run": ["schemaVersion", "workspaceId"],
+    "preprocess-clear": ["schemaVersion", "workspaceId"],
   };
   const allowed = allowedByCommand[command];
   if (!allowed) throw new Error("unknown command");
@@ -91,55 +83,12 @@ async function dispatch(command: Command, service: WorkspacePreprocessingService
   switch (command) {
     case "inspect":
       return await service.inspect({ workspaceId: request.workspaceId as string });
-    case "preprocess-dwh":
-      return await service.preprocessDwh({
-        workspaceId: request.workspaceId as string,
-        resumeRunId: request.resumeRunId as string | undefined,
-      });
-    case "schema-suggest-fks":
-      return await service.suggestFks({
-        workspaceId: request.workspaceId as string,
-        fromSql: request.fromSql as any,
-        assume: request.assume as any,
-        resumeRunId: request.resumeRunId as string | undefined,
-      });
-    case "schema-check":
-      return await service.checkSchema({
-        workspaceId: request.workspaceId as string,
-        annotationsYaml: request.annotationsYaml as string | undefined,
-        reviewedCandidatesDigest: request.reviewedCandidatesDigest as string | undefined,
-      });
-    case "schema-accept":
-      return await service.acceptSchema({
-        workspaceId: request.workspaceId as string,
-        runId: request.runId as string,
-        yes: request.yes === true,
-      });
-    case "index-schema":
-      return await service.indexSchema({
-        workspaceId: request.workspaceId as string,
-        resumeRunId: request.resumeRunId as string | undefined,
-      });
-    case "preprocess-evidence":
-      return await service.preprocessEvidence({
-        workspaceId: request.workspaceId as string,
-        dryRun: request.dryRun as boolean | undefined,
-        resumeRunId: request.resumeRunId as string | undefined,
-      });
     case "preprocess-run":
       return await service.run({
         workspaceId: request.workspaceId as string,
-        resumeRunId: request.resumeRunId as string | undefined,
-      });
-    case "vector-inspect":
-      return await service.vectorInspect({ workspaceId: request.workspaceId as string });
-    case "vector-rebuild":
-      return await service.vectorRebuild({
-        workspaceId: request.workspaceId as string,
-        collection: request.collection as string | undefined,
-        confirm: request.confirm as string | undefined,
-        destroy: request.destroy === true,
       });
+    case "preprocess-clear":
+      return await service.clear({ workspaceId: request.workspaceId as string });
   }
 }
 
@@ -178,48 +127,31 @@ export async function runWorkspaceMaintenanceCli(
       || message === "unexpected request field"
       || message === "invalid workspace id";
     return command in {
-      inspect: true, "preprocess-dwh": true, "schema-suggest-fks": true, "schema-check": true,
-      "schema-accept": true,
-      "index-schema": true, "preprocess-evidence": true, "preprocess-run": true,
+      inspect: true, "preprocess-run": true, "preprocess-clear": true,
     } ? (requestError ? 2 : 1) : 2;
   }
 }
 
-async function readSessionInventory(dataRoot: string, workspaceId: string): Promise<readonly SessionInventoryRow[]> {
-  const directory = join(dataRoot, "sessions", workspaceId, "sessions");
-  try {
-    const entries = await readdir(directory, { withFileTypes: true });
-    const rows: SessionInventoryRow[] = [];
-    for (const entry of entries) {
-      if (!entry.isDirectory() || entry.isSymbolicLink()) continue;
-      try {
-        const source = await readFile(join(directory, entry.name, "session_manifest.yaml"), "utf8");
-        const manifest = parse(source) as Record<string, unknown>;
-        rows.push({
-          id: entry.name,
-          status: typeof manifest.status === "string" ? manifest.status : "open",
-          archived: manifest.archived === true,
-          workspaceRevision: typeof manifest.workspace_revision === "string" ? manifest.workspace_revision : null,
-        });
-      } catch {
-        // fail closed at mutation time by ignoring unreadable manifests from the resumable scan
-      }
-    }
-    return rows;
-  } catch {
-    return [];
-  }
+export interface ProductionWorkspacePreprocessingDeps {
+  config?: AppConfig;
+  catalogRepository?: CatalogRepository;
+  registry?: WorkspaceRegistry;
+  workspaceSecretStore?: WorkspaceSecretStore;
+  runner?: ThtRunner;
 }
 
-function createProductionService(): WorkspacePreprocessingService {
-  const config = loadConfig(process.env, { surface: "workspace-maintenance" });
-  const registry = new WorkspaceRegistry(config.workspaceRegistry);
-  const workspaceSecretStore = new WorkspaceSecretStore({
+export function createProductionWorkspacePreprocessingService(
+  deps: ProductionWorkspacePreprocessingDeps = {},
+): WorkspacePreprocessingService {
+  const config = deps.config ?? loadConfig(process.env, { surface: "workspace-maintenance" });
+  const catalogRepository = deps.catalogRepository ?? createCatalogRepository(config.catalogDatabase);
+  const registry = deps.registry ?? new WorkspaceRegistry(config.workspaceRegistry);
+  const workspaceSecretStore = deps.workspaceSecretStore ?? new WorkspaceSecretStore({
     root: config.workspaceSecretStoreRoot,
     runtimeRoot: config.workspaceSecretRuntimeRoot,
     installationId: config.workspaceRegistry.installationId,
   });
-  const runner = new ThtRunner({
+  const runner = deps.runner ?? new ThtRunner({
     thtBin: config.thtBin,
     harnessDir: config.harnessDir,
     configPath: process.env.THT_CONFIG ?? "config/tht.yaml",
@@ -232,16 +164,19 @@ function createProductionService(): WorkspacePreprocessingService {
     semanticRuntime: {
       internalQdrantUrl: config.internalQdrantUrl,
       internalEmbeddingUrl: config.internalEmbeddingUrl,
+      internalEmbeddingId: config.internalEmbeddingId,
       internalEmbeddingModel: config.internalEmbeddingModel,
       internalEmbeddingDimensions: config.internalEmbeddingDimensions,
     },
   });
   return new WorkspacePreprocessingService({
     dataRoot: config.dataRoot ?? "/data",
-    embeddingDimensions: config.internalEmbeddingDimensions,
     httpPrivateHostAllowlist: (process.env.THT_EVIDENCE_PRIVATE_HOST_ALLOWLIST ?? "")
       .split(",").map((value) => value.trim()).filter((value) => value.length > 0),
+    catalogRepository,
     acquireActiveRuntime: async (workspaceId) => {
+      const catalogDatabase = await catalogRepository.getByWorkspace(workspaceId);
+      if (!catalogDatabase) throw new Error("workspace database is not configured in the Catalog");
       const active = await renderActiveWorkspaceRuntime({
         workspaceId,
         registry,
@@ -251,6 +186,7 @@ function createProductionService(): WorkspacePreprocessingService {
         dataRoot: config.dataRoot ?? "/data",
         secretRoots: config.workspaceRegistry.secretRoots,
         workspaceSecretStore,
+        catalogDatabase,
         semanticRuntime: {
           internalQdrantUrl: config.internalQdrantUrl,
           internalEmbeddingUrl: config.internalEmbeddingUrl,
@@ -258,38 +194,55 @@ function createProductionService(): WorkspacePreprocessingService {
           internalEmbeddingDimensions: config.internalEmbeddingDimensions,
         },
       });
-      const configLease = await publishDeterministicRuntimeConfigLease({
-        workspaceId,
-        registry,
-        registryConfig: config.workspaceRegistry,
-        harnessDir: config.harnessDir,
-        configPath: process.env.THT_CONFIG ?? "config/tht.yaml",
-        dataRoot: config.dataRoot ?? "/data",
-        secretRoots: config.workspaceRegistry.secretRoots,
-        semanticRuntime: {
-          internalQdrantUrl: config.internalQdrantUrl,
-          internalEmbeddingUrl: config.internalEmbeddingUrl,
-          internalEmbeddingModel: config.internalEmbeddingModel,
-          internalEmbeddingDimensions: config.internalEmbeddingDimensions,
-        },
-        workspaceSecretStore,
-      });
-      return {
-        workspace: active.workspace,
-        workspaceId: active.workspaceId,
-        workspaceRevision: active.workspaceRevision,
-        descriptorBlob: active.descriptorBlob,
-        catalogBlob: active.catalogBlob,
-        configLease,
-      };
+      try {
+        const configLease = await publishDeterministicRuntimeConfigLease({
+          workspaceId,
+          registry,
+          registryConfig: config.workspaceRegistry,
+          harnessDir: config.harnessDir,
+          configPath: process.env.THT_CONFIG ?? "config/tht.yaml",
+          dataRoot: config.dataRoot ?? "/data",
+          secretRoots: config.workspaceRegistry.secretRoots,
+          semanticRuntime: {
+            internalQdrantUrl: config.internalQdrantUrl,
+            internalEmbeddingUrl: config.internalEmbeddingUrl,
+            internalEmbeddingModel: config.internalEmbeddingModel,
+            internalEmbeddingDimensions: config.internalEmbeddingDimensions,
+          },
+          workspaceSecretStore,
+          catalogDatabase,
+        });
+        return {
+          workspace: active.workspace,
+          workspaceId: active.workspaceId,
+          workspaceRevision: active.workspaceRevision,
+          descriptorBlob: active.descriptorBlob,
+          catalogBlob: active.catalogBlob,
+          configLease,
+        };
+      } finally {
+        active.releaseSecrets();
+      }
     },
     runChild: async ({ argv, configPath }) => {
       const configFd = openSync(configPath, fsConstants.O_RDONLY | fsConstants.O_NOFOLLOW);
       try {
         return await new Promise((resolve) => {
+          const childEnvironment = { ...process.env };
+          for (const name of [
+            "THT_CATALOG_DATABASE_URL",
+            "THT_CATALOG_DB_HOST",
+            "THT_CATALOG_DB_PORT",
+            "THT_CATALOG_DB_NAME",
+            "THT_CATALOG_RUNTIME_USER",
+            "THT_CATALOG_RUNTIME_PASSWORD_FILE",
+            "THT_CATALOG_MIGRATOR_DATABASE_URL",
+            "THT_CATALOG_MIGRATOR_USER",
+            "THT_CATALOG_MIGRATOR_PASSWORD_FILE",
+          ]) delete childEnvironment[name];
           const child = spawn(config.thtBin, argv, {
             cwd: config.harnessDir,
-            env: { ...process.env, ...(config.dataRoot ? { THT_DATA_ROOT: config.dataRoot } : {}) },
+            env: { ...childEnvironment, ...(config.dataRoot ? { THT_DATA_ROOT: config.dataRoot } : {}) },
             stdio: ["ignore", "pipe", "pipe", configFd],
           });
           let stdout = "";
@@ -303,9 +256,8 @@ function createProductionService(): WorkspacePreprocessingService {
         closeSync(configFd);
       }
     },
-    listSessions: async (workspaceId) => await readSessionInventory(config.dataRoot ?? "/data", workspaceId),
     semanticPreflight: async (workspace) => {
-      const result = await runner.qdrantEnsure(workspace, 30);
+      const result = await runner.qdrantEnsure(workspace, 30, "self_heal");
       return result.ok ? { ok: true as const } : { ok: false as const, code: result.code ?? "workspace_not_activatable" };
     },
     evidencePreflight: async (workspace) => {
@@ -330,7 +282,11 @@ if (process.argv[1] && import.meta.url === new URL(`file://${process.argv[1]}`).
     writeStdout: (value) => { stdout.push(value); },
     writeStderr: (value) => { stderr.push(value); },
   };
-  const exitCode = await runWorkspaceMaintenanceCli(process.argv, createProductionService(), io);
+  const exitCode = await runWorkspaceMaintenanceCli(
+    process.argv,
+    createProductionWorkspacePreprocessingService(),
+    io,
+  );
   process.stdout.write(stdout.join(""));
   if (stderr.length > 0) process.stderr.write(stderr.join("").slice(0, 64 * 1024));
   process.exit(exitCode);