feat: classify sensitive columns locally

backend/src/config.ts

@@ -33,6 +33,12 @@ export interface AppConfig {
   secretsFile?: string;
   installationConfigFile?: string;
   modelCatalogFile?: string;
+  sensitivityNer?: {
+    pythonExecutable: string;
+    modelPath: string;
+    workerScript?: string;
+    threads: number;
+  };
   piAuthFile?: string;
   secretFiles: Readonly<Record<string, string | undefined>>;
   modelApiKeyFile?: string;
@@ -354,6 +360,35 @@ export function loadConfig(
     || modelCatalogFile.includes("\0")
     || !path.isAbsolute(modelCatalogFile)
   )) throw new Error("runtime model catalog configuration is invalid");
+  const sensitivityNerModelPath = env.THT_SENSITIVITY_NER_MODEL_PATH;
+  const sensitivityNerPython = env.THT_SENSITIVITY_NER_PYTHON;
+  const sensitivityNerWorker = env.THT_SENSITIVITY_NER_WORKER;
+  for (const [value, label] of [
+    [sensitivityNerModelPath, "model path"],
+    [sensitivityNerPython, "Python executable"],
+    [sensitivityNerWorker, "worker path"],
+  ] as const) {
+    if (value !== undefined && (
+      value.length === 0 || value.trim() !== value || value.includes("\0") || !path.isAbsolute(value)
+    )) throw new Error(`sensitivity NER ${label} configuration is invalid`);
+  }
+  if (sensitivityNerModelPath === undefined && (
+    sensitivityNerPython !== undefined
+    || sensitivityNerWorker !== undefined
+    || env.THT_SENSITIVITY_NER_THREADS !== undefined
+  )) throw new Error("sensitivity NER settings require a model path");
+  const sensitivityNerThreads = Number(env.THT_SENSITIVITY_NER_THREADS ?? 2);
+  if (!Number.isSafeInteger(sensitivityNerThreads) || sensitivityNerThreads < 1 || sensitivityNerThreads > 8) {
+    throw new Error("sensitivity NER thread configuration is invalid");
+  }
+  const sensitivityNer = sensitivityNerModelPath === undefined
+    ? undefined
+    : {
+      modelPath: sensitivityNerModelPath,
+      pythonExecutable: sensitivityNerPython ?? "/opt/sensitivity-ner/bin/python",
+      ...(sensitivityNerWorker ? { workerScript: sensitivityNerWorker } : {}),
+      threads: sensitivityNerThreads,
+    };
   const piAuthFile = env.THT_PI_AUTH_FILE;
   if (piAuthFile !== undefined && (
     piAuthFile.trim() !== piAuthFile || piAuthFile.length === 0 || piAuthFile.includes("\0")
@@ -442,6 +477,7 @@ export function loadConfig(
     secretsFile,
     installationConfigFile,
     modelCatalogFile,
+    sensitivityNer,
     piAuthFile,
     secretFiles,
     modelApiKeyFile,