feat: add AI catalog description generation

backend/src/app.ts

@@ -2,7 +2,7 @@ import Fastify, { type FastifyInstance, type FastifyRequest } from "fastify";
 import cors from "@fastify/cors";
 import cookie from "@fastify/cookie";
 import rateLimit from "@fastify/rate-limit";
-import { join } from "node:path";
+import { dirname, isAbsolute, join } from "node:path";
 import { tmpdir } from "node:os";
 import type { AppConfig } from "./config.js";
 import { ThtRunner } from "./tht/tht-runner.js";
@@ -48,6 +48,19 @@ import { catalogTableRoutes } from "./routes/catalog-tables.js";
 import { ConcreteCatalogSchemaIntrospector, type CatalogSchemaIntrospector } from "./catalog/schema-introspector.js";
 import { CatalogSyncWorker } from "./catalog/sync-worker.js";
 import { catalogSchemaRoutes } from "./routes/catalog-schema.js";
+import {
+  loadMetadataGenerationModels,
+  type MetadataGenerationModels,
+} from "./catalog/metadata-generation-models.js";
+import { metadataGenerationModelRoutes } from "./routes/metadata-generation-models.js";
+import { catalogDescriptionConsolidationRoutes } from "./routes/catalog-description-consolidation.js";
+import { PythonModelCompleter, type ModelCompleter } from "./catalog/model-completer.js";
+import { DescriptionGenerationWorker } from "./catalog/description-generation-worker.js";
+import {
+  PostgresDescriptionSourceSampler,
+  type DescriptionSourceSampler,
+} from "./catalog/description-source-sampler.js";
+import { catalogDescriptionGenerationRoutes } from "./routes/catalog-description-generation.js";
 
 export interface BuildAppDeps {
   thtRunner?: ThtRunner;
@@ -67,6 +80,9 @@ export interface BuildAppDeps {
   catalogSchemaIntrospector?: CatalogSchemaIntrospector;
   catalogSyncWorker?: CatalogSyncWorker;
   catalogOperationCoordinator?: CatalogOperationCoordinator;
+  metadataGenerationModels?: MetadataGenerationModels;
+  modelCompleter?: ModelCompleter;
+  descriptionSourceSampler?: DescriptionSourceSampler;
   workspaceRuntimeSupport?: (workspace: WorkspaceDescriptor) => boolean;
   maintenanceBarrier?: MaintenanceBarrier;
   piManagement?: PiManagementService;
@@ -141,10 +157,28 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc
   const workspaceRegistry = deps?.workspaceRegistry ?? new WorkspaceRegistry(config.workspaceRegistry);
   const catalogRepository = deps?.catalogRepository ?? createCatalogRepository(config.catalogDatabase);
   const catalogOperationCoordinator = deps?.catalogOperationCoordinator ?? new CatalogOperationCoordinator();
+  const metadataGenerationModels = deps?.metadataGenerationModels ?? loadMetadataGenerationModels({
+    installationFile: config.installationConfigFile,
+    secretsFile: config.secretsFile,
+  });
+  const modelCompleter = deps?.modelCompleter ?? new PythonModelCompleter({
+    pythonExecutable: isAbsolute(config.thtBin) ? join(dirname(config.thtBin), "python") : "python3",
+    cwd: config.harnessDir,
+  });
   const catalogPostgresAccess = deps?.catalogPostgresAccess ?? new ConcreteCatalogPostgresAccess(
     workspaceSecretStore,
     { connectTimeoutMs: config.workspaceDiagnosticTimeoutMs },
   );
+  const descriptionSourceSampler = deps?.descriptionSourceSampler
+    ?? new PostgresDescriptionSourceSampler(catalogPostgresAccess);
+  const descriptionGenerationWorker = new DescriptionGenerationWorker(
+    catalogRepository,
+    workspaceRegistry,
+    metadataGenerationModels,
+    modelCompleter,
+    catalogOperationCoordinator,
+    descriptionSourceSampler,
+  );
   const catalogService = deps?.catalogService ?? new CatalogService(
     catalogRepository,
     workspaceRegistry,
@@ -166,10 +200,12 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc
     config.catalogSyncTimeoutMs,
   );
   app.addHook("onReady", async () => { await catalogSyncWorker.initialize(); });
+  app.addHook("onReady", async () => { await descriptionGenerationWorker.initialize(); });
   if (!deps?.catalogRepository && catalogRepository.close) {
     app.addHook("onClose", async () => { await catalogRepository.close?.(); });
   }
   app.addHook("onClose", async () => { await catalogSyncWorker.stop(); });
+  app.addHook("onClose", async () => { await descriptionGenerationWorker.stop(); });
   const workspaceDiagnoser = deps?.workspaceDiagnoser
     ?? createProductionWorkspaceDiagnoser(config.workspaceDiagnosticTimeoutMs, undefined, {
       internalQdrantUrl: config.internalQdrantUrl,
@@ -384,12 +420,25 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc
     secretStore: workspaceSecretStore,
   });
   catalogDatabaseRoutes(app, { repository: catalogRepository, service: catalogService, operations: catalogOperationCoordinator });
-  catalogTableRoutes(app, { repository: catalogRepository, service: catalogTableService });
+  catalogTableRoutes(app, {
+    repository: catalogRepository,
+    service: catalogTableService,
+    operations: catalogOperationCoordinator,
+  });
   catalogSchemaRoutes(app, {
     repository: catalogRepository,
     worker: catalogSyncWorker,
     operations: catalogOperationCoordinator,
   });
+  catalogDescriptionConsolidationRoutes(app, {
+    repository: catalogRepository,
+    operations: catalogOperationCoordinator,
+  });
+  metadataGenerationModelRoutes(app, metadataGenerationModels);
+  catalogDescriptionGenerationRoutes(app, {
+    repository: catalogRepository,
+    worker: descriptionGenerationWorker,
+  });
   settingsRoutes(app, { cfg: config, listModels, getSettings });
   piManagementRoutes(app, { service: piManagement });