Codex
82e2c91f42
Publish documentation / publish (push) Successful in 1m27s
Add PostgreSQL-backed memory, editable evidence with source review and activation, and human-approved archive repairs across the harness, API, and UI. Include migrations, deployment support, regression coverage, and validation documentation. Refresh permissions from validated session roles so existing administrator logins can access newly deployed archive management features.
308 lines
13 KiB
TypeScript
308 lines
13 KiB
TypeScript
import { spawn } from "node:child_process";
|
|
import { closeSync, constants as fsConstants, openSync } from "node:fs";
|
|
import { join } from "node:path";
|
|
import { loadConfig, type AppConfig } from "./config.js";
|
|
import { ThtRunner } from "./tht/tht-runner.js";
|
|
import { WorkspaceRegistry } from "./workspaces/registry.js";
|
|
import { publishDeterministicRuntimeConfigLease, renderActiveWorkspaceRuntime } from "./workspaces/runtime-config-lease.js";
|
|
import { WorkspaceSecretStore } from "./workspaces/secret-store.js";
|
|
import { WorkspacePreprocessingService, type WorkspaceOperationResult } from "./workspaces/preprocessing-service.js";
|
|
import { createCatalogRepository } from "./catalog/repository.js";
|
|
import type { CatalogRepository } from "./catalog/types.js";
|
|
|
|
export interface WorkspaceMaintenanceIo {
|
|
stdin: string;
|
|
stdout: string[];
|
|
stderr: string[];
|
|
writeStdout(value: string): void;
|
|
writeStderr(value: string): void;
|
|
}
|
|
|
|
type Command = "inspect" | "preprocess-run" | "preprocess-clear" | "evidence-consolidate" | "evidence-refresh" | "evidence-decide";
|
|
|
|
function failureResult(
|
|
operation: string,
|
|
workspaceId = "",
|
|
code: WorkspaceOperationResult["code"] = "workspace_not_activatable",
|
|
): WorkspaceOperationResult {
|
|
return {
|
|
schemaVersion: 1,
|
|
status: "failed",
|
|
code,
|
|
workspaceId,
|
|
workspaceRevision: "",
|
|
descriptorBlob: "",
|
|
operation,
|
|
completedStages: [],
|
|
};
|
|
}
|
|
|
|
const STATE_ERROR_CODES: Record<string, WorkspaceOperationResult["code"]> = {
|
|
preprocessing_resume_mismatch: "preprocessing_resume_mismatch",
|
|
preprocessing_conflict: "preprocessing_conflict",
|
|
effective_config_mismatch: "effective_config_mismatch",
|
|
};
|
|
|
|
function boundedJson(result: WorkspaceOperationResult): string {
|
|
const encoded = JSON.stringify(result);
|
|
if (Buffer.byteLength(encoded, "utf8") > 1024 * 1024) {
|
|
return JSON.stringify(failureResult(result.operation || "unknown", result.workspaceId));
|
|
}
|
|
return encoded;
|
|
}
|
|
|
|
function sanitizeStderr(_error: unknown): string {
|
|
// Never return raw exception text: it may embed endpoints, tokens, or SQL.
|
|
return "workspace maintenance failed\n";
|
|
}
|
|
|
|
function parseRequest(command: string, stdin: string): Record<string, unknown> {
|
|
const parsed = JSON.parse(stdin) as Record<string, unknown>;
|
|
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed) || parsed.schemaVersion !== 1) {
|
|
throw new Error("invalid request");
|
|
}
|
|
const allowedByCommand: Record<string, readonly string[]> = {
|
|
inspect: ["schemaVersion", "workspaceId"],
|
|
"preprocess-run": ["schemaVersion", "workspaceId"],
|
|
"preprocess-clear": ["schemaVersion", "workspaceId"],
|
|
"evidence-consolidate": ["schemaVersion", "workspaceId"],
|
|
"evidence-refresh": ["schemaVersion", "workspaceId"],
|
|
"evidence-decide": ["schemaVersion", "workspaceId", "sourceId", "revision", "decision"],
|
|
};
|
|
const allowed = allowedByCommand[command];
|
|
if (!allowed) throw new Error("unknown command");
|
|
if (typeof parsed.workspaceId !== "string") throw new Error("invalid workspace id");
|
|
for (const key of Object.keys(parsed)) if (!allowed.includes(key)) throw new Error("unexpected request field");
|
|
return parsed;
|
|
}
|
|
|
|
function exitCodeFor(result: WorkspaceOperationResult): number {
|
|
if (["succeeded", "unchanged", "dry_run"].includes(result.status)) return 0;
|
|
if (result.status === "blocked") return 3;
|
|
return 1;
|
|
}
|
|
|
|
async function dispatch(command: Command, service: WorkspacePreprocessingService, request: Record<string, unknown>): Promise<WorkspaceOperationResult> {
|
|
switch (command) {
|
|
case "evidence-refresh":
|
|
return await service.evidenceSources({ workspaceId: request.workspaceId as string, action: "refresh" });
|
|
case "evidence-decide":
|
|
if (typeof request.sourceId !== "string" || !/^[a-f0-9]{64}$/.test(request.sourceId)
|
|
|| typeof request.revision !== "string" || !/^[a-f0-9]{64}$/.test(request.revision)
|
|
|| (request.decision !== "keep" && request.decision !== "replace")) throw new Error("invalid request");
|
|
return await service.evidenceSources({ workspaceId: request.workspaceId as string, action: "decide",
|
|
sourceId: request.sourceId, revision: request.revision, decision: request.decision });
|
|
case "evidence-consolidate":
|
|
return await service.consolidateEvidence({ workspaceId: request.workspaceId as string });
|
|
case "inspect":
|
|
return await service.inspect({ workspaceId: request.workspaceId as string });
|
|
case "preprocess-run":
|
|
return await service.run({
|
|
workspaceId: request.workspaceId as string,
|
|
});
|
|
case "preprocess-clear":
|
|
return await service.clear({ workspaceId: request.workspaceId as string });
|
|
}
|
|
}
|
|
|
|
export async function runWorkspaceMaintenanceCli(
|
|
argv: readonly string[],
|
|
service: WorkspacePreprocessingService,
|
|
io: WorkspaceMaintenanceIo,
|
|
): Promise<number> {
|
|
const command = argv[2];
|
|
if (!command) {
|
|
const result = failureResult("unknown");
|
|
io.writeStdout(boundedJson(result));
|
|
return 2;
|
|
}
|
|
try {
|
|
const request = parseRequest(command, io.stdin);
|
|
const result = await dispatch(command as Command, service, request);
|
|
io.writeStdout(boundedJson(result));
|
|
return exitCodeFor(result);
|
|
} catch (error) {
|
|
const failureCode = error instanceof Error
|
|
&& "code" in error
|
|
&& typeof (error as { code?: unknown }).code === "string"
|
|
&& (error as { code: string }).code in STATE_ERROR_CODES
|
|
? STATE_ERROR_CODES[(error as { code: string }).code]
|
|
: "workspace_not_activatable";
|
|
const result = failureResult(command, (() => {
|
|
try { return JSON.parse(io.stdin).workspaceId ?? ""; } catch { return ""; }
|
|
})(), failureCode);
|
|
io.writeStdout(boundedJson(result));
|
|
io.writeStderr(sanitizeStderr(error));
|
|
const message = String((error as Error).message ?? "");
|
|
const requestError = error instanceof SyntaxError
|
|
|| message === "invalid request"
|
|
|| message === "unknown command"
|
|
|| message === "unexpected request field"
|
|
|| message === "invalid workspace id";
|
|
return command in {
|
|
inspect: true, "preprocess-run": true, "preprocess-clear": true, "evidence-consolidate": true,
|
|
"evidence-refresh": true, "evidence-decide": true,
|
|
} ? (requestError ? 2 : 1) : 2;
|
|
}
|
|
}
|
|
|
|
export interface ProductionWorkspacePreprocessingDeps {
|
|
config?: AppConfig;
|
|
catalogRepository?: CatalogRepository;
|
|
registry?: WorkspaceRegistry;
|
|
workspaceSecretStore?: WorkspaceSecretStore;
|
|
runner?: ThtRunner;
|
|
}
|
|
|
|
export function createProductionWorkspacePreprocessingService(
|
|
deps: ProductionWorkspacePreprocessingDeps = {},
|
|
): WorkspacePreprocessingService {
|
|
const config = deps.config ?? loadConfig(process.env, { surface: "workspace-maintenance" });
|
|
const catalogRepository = deps.catalogRepository ?? createCatalogRepository(config.catalogDatabase);
|
|
const registry = deps.registry ?? new WorkspaceRegistry(config.workspaceRegistry);
|
|
const workspaceSecretStore = deps.workspaceSecretStore ?? new WorkspaceSecretStore({
|
|
root: config.workspaceSecretStoreRoot,
|
|
runtimeRoot: config.workspaceSecretRuntimeRoot,
|
|
installationId: config.workspaceRegistry.installationId,
|
|
});
|
|
const runner = deps.runner ?? new ThtRunner({
|
|
thtBin: config.thtBin,
|
|
harnessDir: config.harnessDir,
|
|
configPath: process.env.THT_CONFIG ?? "config/tht.yaml",
|
|
dataRoot: config.dataRoot,
|
|
runtimeSnapshotRoot: join(config.workspaceRegistry.root, "snapshots", "runtime"),
|
|
secretRoots: config.workspaceRegistry.secretRoots,
|
|
secretsFile: config.secretsFile,
|
|
secretFiles: config.secretFiles,
|
|
workspaceSecretStore,
|
|
semanticRuntime: {
|
|
internalQdrantUrl: config.internalQdrantUrl,
|
|
internalEmbeddingUrl: config.internalEmbeddingUrl,
|
|
internalEmbeddingId: config.internalEmbeddingId,
|
|
internalEmbeddingModel: config.internalEmbeddingModel,
|
|
internalEmbeddingDimensions: config.internalEmbeddingDimensions,
|
|
},
|
|
});
|
|
return new WorkspacePreprocessingService({
|
|
dataRoot: config.dataRoot ?? "/data",
|
|
httpPrivateHostAllowlist: (process.env.THT_EVIDENCE_PRIVATE_HOST_ALLOWLIST ?? "")
|
|
.split(",").map((value) => value.trim()).filter((value) => value.length > 0),
|
|
catalogRepository,
|
|
acquireActiveRuntime: async (workspaceId) => {
|
|
const catalogDatabase = await catalogRepository.getByWorkspace(workspaceId);
|
|
if (!catalogDatabase) throw new Error("workspace database is not configured in the Catalog");
|
|
const active = await renderActiveWorkspaceRuntime({
|
|
workspaceId,
|
|
registry,
|
|
registryConfig: config.workspaceRegistry,
|
|
harnessDir: config.harnessDir,
|
|
configPath: process.env.THT_CONFIG ?? "config/tht.yaml",
|
|
dataRoot: config.dataRoot ?? "/data",
|
|
secretRoots: config.workspaceRegistry.secretRoots,
|
|
workspaceSecretStore,
|
|
catalogDatabase,
|
|
semanticRuntime: {
|
|
internalQdrantUrl: config.internalQdrantUrl,
|
|
internalEmbeddingUrl: config.internalEmbeddingUrl,
|
|
internalEmbeddingModel: config.internalEmbeddingModel,
|
|
internalEmbeddingDimensions: config.internalEmbeddingDimensions,
|
|
},
|
|
});
|
|
try {
|
|
const configLease = await publishDeterministicRuntimeConfigLease({
|
|
workspaceId,
|
|
registry,
|
|
registryConfig: config.workspaceRegistry,
|
|
harnessDir: config.harnessDir,
|
|
configPath: process.env.THT_CONFIG ?? "config/tht.yaml",
|
|
dataRoot: config.dataRoot ?? "/data",
|
|
secretRoots: config.workspaceRegistry.secretRoots,
|
|
semanticRuntime: {
|
|
internalQdrantUrl: config.internalQdrantUrl,
|
|
internalEmbeddingUrl: config.internalEmbeddingUrl,
|
|
internalEmbeddingModel: config.internalEmbeddingModel,
|
|
internalEmbeddingDimensions: config.internalEmbeddingDimensions,
|
|
},
|
|
workspaceSecretStore,
|
|
catalogDatabase,
|
|
});
|
|
return {
|
|
workspace: active.workspace,
|
|
workspaceId: active.workspaceId,
|
|
workspaceRevision: active.workspaceRevision,
|
|
descriptorBlob: active.descriptorBlob,
|
|
catalogBlob: active.catalogBlob,
|
|
configLease,
|
|
};
|
|
} finally {
|
|
active.releaseSecrets();
|
|
}
|
|
},
|
|
runChild: async ({ argv, configPath }) => {
|
|
const configFd = openSync(configPath, fsConstants.O_RDONLY | fsConstants.O_NOFOLLOW);
|
|
try {
|
|
return await new Promise((resolve) => {
|
|
const childEnvironment = { ...process.env };
|
|
for (const name of [
|
|
"THT_CATALOG_DATABASE_URL",
|
|
"THT_CATALOG_DB_HOST",
|
|
"THT_CATALOG_DB_PORT",
|
|
"THT_CATALOG_DB_NAME",
|
|
"THT_CATALOG_RUNTIME_USER",
|
|
"THT_CATALOG_RUNTIME_PASSWORD_FILE",
|
|
"THT_CATALOG_MIGRATOR_DATABASE_URL",
|
|
"THT_CATALOG_MIGRATOR_USER",
|
|
"THT_CATALOG_MIGRATOR_PASSWORD_FILE",
|
|
]) delete childEnvironment[name];
|
|
const child = spawn(config.thtBin, argv, {
|
|
cwd: config.harnessDir,
|
|
env: { ...childEnvironment, ...(config.dataRoot ? { THT_DATA_ROOT: config.dataRoot } : {}) },
|
|
stdio: ["ignore", "pipe", "pipe", configFd],
|
|
});
|
|
let stdout = "";
|
|
let stderr = "";
|
|
child.stdout?.on("data", (chunk: Buffer) => { stdout += chunk.toString("utf8"); });
|
|
child.stderr?.on("data", (chunk: Buffer) => { stderr += chunk.toString("utf8"); });
|
|
child.on("close", (code) => resolve({ exitCode: code ?? 0, stdout, stderr: stderr.slice(0, 64 * 1024) }));
|
|
child.on("error", (error) => resolve({ exitCode: 1, stdout, stderr: String(error.message).slice(0, 4096) }));
|
|
});
|
|
} finally {
|
|
closeSync(configFd);
|
|
}
|
|
},
|
|
semanticPreflight: async (workspace) => {
|
|
const result = await runner.qdrantEnsure(workspace, 30, "self_heal");
|
|
return result.ok ? { ok: true as const } : { ok: false as const, code: result.code ?? "workspace_not_activatable" };
|
|
},
|
|
evidencePreflight: async (workspace) => {
|
|
const result = await runner.qdrantEnsure(workspace, 30, "evidence_maintenance");
|
|
return result.ok ? { ok: true as const } : { ok: false as const, code: result.code ?? "workspace_not_activatable" };
|
|
},
|
|
});
|
|
}
|
|
|
|
if (process.argv[1] && import.meta.url === new URL(`file://${process.argv[1]}`).href) {
|
|
const stdout: string[] = [];
|
|
const stderr: string[] = [];
|
|
const io: WorkspaceMaintenanceIo = {
|
|
stdin: await new Promise<string>((resolve) => {
|
|
let input = "";
|
|
process.stdin.setEncoding("utf8");
|
|
process.stdin.on("data", (chunk) => { input += chunk; });
|
|
process.stdin.on("end", () => resolve(input));
|
|
}),
|
|
stdout,
|
|
stderr,
|
|
writeStdout: (value) => { stdout.push(value); },
|
|
writeStderr: (value) => { stderr.push(value); },
|
|
};
|
|
const exitCode = await runWorkspaceMaintenanceCli(
|
|
process.argv,
|
|
createProductionWorkspacePreprocessingService(),
|
|
io,
|
|
);
|
|
process.stdout.write(stdout.join(""));
|
|
if (stderr.length > 0) process.stderr.write(stderr.join("").slice(0, 64 * 1024));
|
|
process.exit(exitCode);
|
|
}
|