feat: implement memory and evidence administration with guided repairs
Publish documentation / publish (push) Successful in 1m27s
Publish documentation / publish (push) Successful in 1m27s
Add PostgreSQL-backed memory, editable evidence with source review and activation, and human-approved archive repairs across the harness, API, and UI. Include migrations, deployment support, regression coverage, and validation documentation. Refresh permissions from validated session roles so existing administrator logins can access newly deployed archive management features.
This commit is contained in:
+18
-1
@@ -7,6 +7,7 @@ import { fileURLToPath } from "node:url";
|
||||
import { tmpdir } from "node:os";
|
||||
import type { AppConfig } from "./config.js";
|
||||
import { ThtRunner } from "./tht/tht-runner.js";
|
||||
import { createMemoryCleanup } from "./catalog/memory-cleanup.js";
|
||||
import { PiProcessManager } from "./pi/pi-process-manager.js";
|
||||
import { SseHub } from "./sse/sse-hub.js";
|
||||
import { authenticateSession, captureAuthConfigSnapshot, configuredOrigin } from "./auth/auth.js";
|
||||
@@ -80,6 +81,8 @@ import { createProductionWorkspacePreprocessingService } from "./workspace-maint
|
||||
import type { WorkspacePreprocessingService } from "./workspaces/preprocessing-service.js";
|
||||
import { PreprocessingStateStore } from "./workspaces/preprocessing-state.js";
|
||||
import { workspacePreprocessingRoutes } from "./routes/workspace-preprocessing.js";
|
||||
import { memoryRoutes } from "./routes/memory.js";
|
||||
import { evidenceRoutes } from "./routes/evidence.js";
|
||||
|
||||
export interface BuildAppDeps {
|
||||
thtRunner?: ThtRunner;
|
||||
@@ -93,7 +96,7 @@ export interface BuildAppDeps {
|
||||
workspaceDiagnoser?: WorkspaceDiagnoser;
|
||||
workspaceDatabaseTester?: WorkspaceDatabaseTester;
|
||||
workspaceSecretStore?: WorkspaceSecretStore;
|
||||
workspacePreprocessingService?: Pick<WorkspacePreprocessingService, "run" | "clear">;
|
||||
workspacePreprocessingService?: Pick<WorkspacePreprocessingService, "run" | "clear"> & Partial<Pick<WorkspacePreprocessingService, "consolidateEvidence" | "evidenceSources">>;
|
||||
catalogRepository?: CatalogRepository;
|
||||
catalogService?: CatalogService;
|
||||
catalogPostgresAccess?: CatalogPostgresAccess;
|
||||
@@ -265,6 +268,11 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc
|
||||
catalogSchemaIntrospector,
|
||||
catalogOperationCoordinator,
|
||||
config.catalogSyncTimeoutMs,
|
||||
createMemoryCleanup(tht as ThtRunner, {
|
||||
internalQdrantUrl: config.internalQdrantUrl, internalEmbeddingUrl: config.internalEmbeddingUrl,
|
||||
internalEmbeddingId: config.internalEmbeddingId, internalEmbeddingModel: config.internalEmbeddingModel,
|
||||
internalEmbeddingDimensions: config.internalEmbeddingDimensions,
|
||||
}),
|
||||
);
|
||||
app.addHook("onReady", async () => { await catalogSyncWorker.initialize(); });
|
||||
app.addHook("onReady", async () => { await descriptionGenerationWorker.initialize(); });
|
||||
@@ -493,7 +501,16 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc
|
||||
return maintenanceBarrier.status();
|
||||
});
|
||||
sqlRoutes(app, { tht: tht as ThtRunner, getSettings, workspaceRegistry });
|
||||
memoryRoutes(app, { runner: tht as ThtRunner, registry: workspaceRegistry, runtime: {
|
||||
internalQdrantUrl: config.internalQdrantUrl,
|
||||
internalEmbeddingUrl: config.internalEmbeddingUrl,
|
||||
internalEmbeddingModel: config.internalEmbeddingModel,
|
||||
internalEmbeddingDimensions: config.internalEmbeddingDimensions,
|
||||
} });
|
||||
metaRoutes(app, { harnessDir: config.harnessDir, modelCatalog: runtimeModelCatalog });
|
||||
evidenceRoutes(app, { runner: tht as ThtRunner, registry: workspaceRegistry,
|
||||
registryRoot: config.workspaceRegistry.root, hostRegistryRoot: config.evidenceHostRegistryRoot,
|
||||
service: workspacePreprocessingService });
|
||||
workspaceRoutes(app, {
|
||||
registry: workspaceRegistry,
|
||||
config: config.workspaceRegistry,
|
||||
|
||||
@@ -119,7 +119,9 @@ export function authenticateSession(deps: AuthDependencies): preHandlerHookHandl
|
||||
subject: session.subject,
|
||||
...(session.displayName === undefined ? {} : { displayName: session.displayName }),
|
||||
roles: session.roles,
|
||||
permissions: session.permissions,
|
||||
// Sessions can outlive a deployment that changes the role permission catalog.
|
||||
// resolve() has already checked validity, including current local user roles.
|
||||
permissions: rolesToPermissions(session.roles),
|
||||
isAdmin: session.roles.includes("admin"),
|
||||
};
|
||||
if (STATE_CHANGING_METHODS.has(request.method)) {
|
||||
|
||||
@@ -38,7 +38,7 @@ const MAX_MAPPED_GROUPS = 128;
|
||||
const ROLES = ["user", "admin"] as const;
|
||||
export const PERMISSION_CATALOG: readonly Permission[] = [
|
||||
"session.use", "session.read_all", "session.manage_all", "settings.manage",
|
||||
"workspace.manage", "workspace.secrets.manage", "database.manage", "pi.manage", "auth.diagnostics.read",
|
||||
"workspace.manage", "workspace.secrets.manage", "database.manage", "memory.manage", "evidence.manage", "pi.manage", "auth.diagnostics.read",
|
||||
];
|
||||
|
||||
const invalid = (): Error => new Error("authentication configuration is invalid");
|
||||
|
||||
@@ -33,7 +33,7 @@ const EMPTY_HKDF_SALT = Buffer.alloc(0);
|
||||
const ROLES = ["user", "admin"] as const;
|
||||
const PERMISSIONS = [
|
||||
"session.use", "session.read_all", "session.manage_all", "settings.manage",
|
||||
"workspace.manage", "workspace.secrets.manage", "database.manage", "pi.manage", "auth.diagnostics.read",
|
||||
"workspace.manage", "workspace.secrets.manage", "database.manage", "memory.manage", "evidence.manage", "pi.manage", "auth.diagnostics.read",
|
||||
] as const satisfies readonly Permission[];
|
||||
|
||||
const invalid = (): Error => new Error("auth_session_store_invalid");
|
||||
|
||||
@@ -5,7 +5,7 @@ export type Role = "user" | "admin";
|
||||
export type Permission =
|
||||
| "session.use" | "session.read_all" | "session.manage_all"
|
||||
| "settings.manage" | "workspace.manage" | "workspace.secrets.manage"
|
||||
| "database.manage" | "pi.manage" | "auth.diagnostics.read";
|
||||
| "database.manage" | "memory.manage" | "evidence.manage" | "pi.manage" | "auth.diagnostics.read";
|
||||
|
||||
export interface AuthenticationSessionConfig {
|
||||
regularTtlSeconds: number;
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
import type { ThtRunner } from "../tht/tht-runner.js";
|
||||
import type { SemanticRuntimeConfig } from "../workspaces/runtime-renderer.js";
|
||||
import type { CatalogSyncRun, WorkspaceDatabase } from "./types.js";
|
||||
|
||||
/** Internal continuation of an applied physical sync, using the harness Memory boundary. */
|
||||
export function createMemoryCleanup(runner: Pick<ThtRunner, "withPrincipal">, runtime: SemanticRuntimeConfig) {
|
||||
return async (database: WorkspaceDatabase, run: CatalogSyncRun): Promise<number> => {
|
||||
if (run.phase !== "memory_cleanup" || !run.plannedDiff) throw new Error("Physical cleanup is not committed");
|
||||
const result = await runner.withPrincipal({ issuer: "installation", subject: "catalog-sync",
|
||||
roles: ["admin"], permissions: ["memory.manage"], isAdmin: true,
|
||||
}).runWithRuntimeSnapshot(["memory", "admin", "--workspace", database.workspaceId], JSON.stringify({
|
||||
action: "cleanup", runtime, request: { sync_id: run.id, database: database.databaseName,
|
||||
schema_name: database.schema, removed_tables: run.plannedDiff.deletedTables,
|
||||
removed_columns: run.plannedDiff.deletedColumns.map(column => ({ table: column.tableName, column: column.columnName })),
|
||||
},
|
||||
}));
|
||||
const payload = JSON.parse(result.stdout);
|
||||
if (result.code !== 0 || payload.indexed !== true || !Number.isInteger(payload.deleted) || payload.deleted < 0) {
|
||||
throw new Error("Memory cleanup is incomplete");
|
||||
}
|
||||
return payload.deleted;
|
||||
};
|
||||
}
|
||||
@@ -926,6 +926,7 @@ export class MemoryCatalogRepository implements CatalogRepository {
|
||||
scope: CatalogSyncScope,
|
||||
tableIds: readonly string[],
|
||||
snapshot: ObservedSchemaSnapshot,
|
||||
syncRunId?: string,
|
||||
): Promise<CatalogSyncCounts | undefined> {
|
||||
const database = this.records.get(databaseId);
|
||||
if (!database || database.version !== expectedDatabaseVersion) return undefined;
|
||||
@@ -1141,6 +1142,7 @@ export class MemoryCatalogRepository implements CatalogRepository {
|
||||
if (scope === "all") {
|
||||
this.records.set(databaseId, { ...database, schemaSyncedVersion: expectedDatabaseVersion, schemaSyncedAt: now });
|
||||
}
|
||||
if (syncRunId) await this.updateSyncRun(syncRunId, { phase: "memory_cleanup" });
|
||||
return {
|
||||
tables: (await this.listTables(databaseId)).length,
|
||||
columns: [...this.columns.values()].filter((column) => this.tables.get(column.tableId)?.databaseId === databaseId).length,
|
||||
@@ -1254,7 +1256,7 @@ export class MemoryCatalogRepository implements CatalogRepository {
|
||||
for (const run of this.syncRuns.values()) {
|
||||
if (["queued", "running", "awaiting_confirmation", "applying"].includes(run.state)) {
|
||||
await this.updateSyncRun(run.id, {
|
||||
state: "interrupted", phase: "completed", finishedAt: new Date().toISOString(),
|
||||
state: "interrupted", phase: run.phase === "memory_cleanup" ? "memory_cleanup" : "completed", finishedAt: new Date().toISOString(),
|
||||
errorCode: "SYNC_INTERRUPTED", errorMessage: "Synchronization was interrupted by a service restart",
|
||||
});
|
||||
}
|
||||
|
||||
@@ -1604,6 +1604,7 @@ export class KyselyCatalogRepository implements CatalogRepository {
|
||||
scope: CatalogSyncScope,
|
||||
tableIds: readonly string[],
|
||||
snapshot: ObservedSchemaSnapshot,
|
||||
syncRunId?: string,
|
||||
): Promise<CatalogSyncCounts | undefined> {
|
||||
return await this.db.transaction().execute(async (trx) => {
|
||||
const database = await trx.selectFrom("workspaceDatabases").select("version")
|
||||
@@ -1779,6 +1780,10 @@ export class KyselyCatalogRepository implements CatalogRepository {
|
||||
schemaSyncedAt: now,
|
||||
}).where("id", "=", databaseId).execute();
|
||||
}
|
||||
if (syncRunId) {
|
||||
await trx.updateTable("catalogSyncRuns").set({ phase: "memory_cleanup" })
|
||||
.where("id", "=", syncRunId).where("databaseId", "=", databaseId).execute();
|
||||
}
|
||||
return {
|
||||
tables: snapshot.tables.length,
|
||||
columns: scope === "tables" ? undefined : snapshot.columns.length,
|
||||
@@ -1882,7 +1887,7 @@ export class KyselyCatalogRepository implements CatalogRepository {
|
||||
|
||||
async interruptActiveSyncRuns(): Promise<void> {
|
||||
await this.db.updateTable("catalogSyncRuns").set({
|
||||
state: "interrupted", phase: "completed", errorCode: "worker_restarted",
|
||||
state: "interrupted", phase: sql`case when phase='memory_cleanup' then phase else 'completed' end`, errorCode: "worker_restarted",
|
||||
errorMessage: "Synchronization was interrupted by a backend restart.",
|
||||
finishedAt: sql`now()`, updatedAt: sql`now()`,
|
||||
leaseOwner: null, leaseExpiresAt: null,
|
||||
|
||||
@@ -58,6 +58,8 @@ export class CatalogSyncWorker {
|
||||
private readonly introspector: CatalogSchemaIntrospector,
|
||||
private readonly operations: CatalogOperationCoordinator,
|
||||
private readonly timeoutMs: number,
|
||||
private readonly cleanupMemory: (database: WorkspaceDatabase, run: CatalogSyncRun) => Promise<number>
|
||||
= async () => 0,
|
||||
) {}
|
||||
|
||||
async initialize(): Promise<void> {
|
||||
@@ -67,6 +69,9 @@ export class CatalogSyncWorker {
|
||||
}
|
||||
|
||||
async start(database: WorkspaceDatabase, scope: CatalogSyncScope, tableIds: readonly string[]): Promise<CatalogSyncRun> {
|
||||
if ((await this.repository.listSyncRuns(database.id, 100)).some(run => run.phase === "memory_cleanup")) {
|
||||
throw new CatalogConflictError("Retry the pending Memory cleanup before starting another synchronization");
|
||||
}
|
||||
const uniqueTableIds = [...new Set(tableIds)];
|
||||
if (scope === "columns") {
|
||||
const tables = await Promise.all(uniqueTableIds.map((tableId) => this.repository.getTable(database.id, tableId)));
|
||||
@@ -109,7 +114,7 @@ export class CatalogSyncWorker {
|
||||
async cancel(runId: string): Promise<CatalogSyncRun | undefined> {
|
||||
const run = await this.repository.getSyncRun(runId);
|
||||
if (!run) return undefined;
|
||||
if (run.state === "applying" || TERMINAL_STATES.has(run.state)) return run;
|
||||
if (run.phase === "memory_cleanup" || run.state === "applying" || TERMINAL_STATES.has(run.state)) return run;
|
||||
await this.repository.requestSyncRunCancellation(runId);
|
||||
this.controllers.get(runId)?.abort();
|
||||
if (run.state === "queued" || run.state === "awaiting_confirmation") {
|
||||
@@ -138,6 +143,16 @@ export class CatalogSyncWorker {
|
||||
}
|
||||
const database = await this.repository.get(previous.databaseId);
|
||||
if (!database) return undefined;
|
||||
if (previous.phase === "memory_cleanup") {
|
||||
const release = this.operations.reserve(database.id);
|
||||
try {
|
||||
const queued = await this.repository.updateSyncRun(runId, { state: "queued", cancelRequested: false,
|
||||
errorCode: null, errorMessage: null, finishedAt: null, leaseOwner: null, leaseExpiresAt: null });
|
||||
this.reservations.set(runId, release);
|
||||
this.launch(runId);
|
||||
return queued;
|
||||
} catch (error) { release(); throw error; }
|
||||
}
|
||||
return await this.start(database, previous.scope, previous.tableIds);
|
||||
}
|
||||
|
||||
@@ -179,6 +194,10 @@ export class CatalogSyncWorker {
|
||||
if (!database || database.version !== claimed.requestedDatabaseVersion) {
|
||||
throw new CatalogConflictError("Database binding changed before synchronization started");
|
||||
}
|
||||
if (claimed.phase === "memory_cleanup") {
|
||||
await this.finishMemoryCleanup(database, claimed);
|
||||
return;
|
||||
}
|
||||
const progress: CatalogSchemaScanProgress = async (phase, counts) => {
|
||||
await this.checkCancelled(runId);
|
||||
await this.repository.updateSyncRun(runId, {
|
||||
@@ -230,36 +249,30 @@ export class CatalogSyncWorker {
|
||||
claimed.scope,
|
||||
claimed.tableIds,
|
||||
snapshot,
|
||||
runId,
|
||||
);
|
||||
if (!applied) throw new CatalogConflictError("Database binding changed before schema changes were applied");
|
||||
await this.repository.updateSyncRun(runId, {
|
||||
state: "succeeded",
|
||||
phase: "completed",
|
||||
counts: applied,
|
||||
finishedAt: new Date().toISOString(),
|
||||
observedSnapshot: null,
|
||||
plannedDiff: null,
|
||||
confirmationToken: null,
|
||||
heartbeatAt: new Date().toISOString(),
|
||||
leaseOwner: null,
|
||||
leaseExpiresAt: null,
|
||||
});
|
||||
await this.repository.appendSyncEvent(runId, "info", "succeeded", "Synchronization completed.", { ...applied });
|
||||
this.release(runId);
|
||||
const cleanupRun = await this.repository.updateSyncRun(runId, { counts: applied });
|
||||
if (!cleanupRun) throw new Error("Synchronization run disappeared");
|
||||
await this.finishMemoryCleanup(database, cleanupRun);
|
||||
/* Completion is recorded only after the durable Memory cleanup succeeds. */
|
||||
return;
|
||||
} catch (error) {
|
||||
const current = await this.repository.getSyncRun(runId);
|
||||
const cancelled = !timedOut && (error instanceof SyncCancelledError || controller.signal.aborted || current?.cancelRequested);
|
||||
const failure = timedOut
|
||||
const failure = current?.phase === "memory_cleanup"
|
||||
? { code: "memory_cleanup_pending", message: "Catalog synchronized. Memory cleanup is pending; retry this synchronization to complete it." }
|
||||
: timedOut
|
||||
? { code: "schema_sync_timed_out", message: "Schema synchronization timed out." }
|
||||
: safeFailure(error);
|
||||
await this.repository.updateSyncRun(runId, {
|
||||
state: cancelled ? "cancelled" : "failed",
|
||||
phase: "completed",
|
||||
phase: current?.phase === "memory_cleanup" ? "memory_cleanup" : "completed",
|
||||
errorCode: cancelled ? null : failure.code,
|
||||
errorMessage: cancelled ? null : failure.message,
|
||||
finishedAt: new Date().toISOString(),
|
||||
observedSnapshot: null,
|
||||
plannedDiff: null,
|
||||
observedSnapshot: current?.phase === "memory_cleanup" ? current.observedSnapshot : null,
|
||||
plannedDiff: current?.phase === "memory_cleanup" ? current.plannedDiff : null,
|
||||
confirmationToken: null,
|
||||
leaseOwner: null,
|
||||
leaseExpiresAt: null,
|
||||
@@ -278,6 +291,20 @@ export class CatalogSyncWorker {
|
||||
}
|
||||
}
|
||||
|
||||
private async finishMemoryCleanup(database: WorkspaceDatabase, run: CatalogSyncRun): Promise<void> {
|
||||
if (!run.plannedDiff || run.phase !== "memory_cleanup") throw new Error("Missing committed cleanup context");
|
||||
await this.repository.appendSyncEvent(run.id, "info", "memory_cleanup", "Removing Memory cards with deleted physical dependencies.");
|
||||
const memoryDeleted = await this.cleanupMemory(database, run);
|
||||
const counts = { ...run.counts, memoryDeleted };
|
||||
await this.repository.updateSyncRun(run.id, {
|
||||
state: "succeeded", phase: "completed", counts, finishedAt: new Date().toISOString(),
|
||||
observedSnapshot: null, plannedDiff: null, confirmationToken: null,
|
||||
heartbeatAt: new Date().toISOString(), leaseOwner: null, leaseExpiresAt: null,
|
||||
});
|
||||
await this.repository.appendSyncEvent(run.id, "info", "succeeded", "Synchronization completed.", counts);
|
||||
this.release(run.id);
|
||||
}
|
||||
|
||||
private assertCapability(scope: CatalogSyncScope, snapshot: ObservedSchemaSnapshot): void {
|
||||
const required = scope === "all" ? ["tables", "columns", "relationships"] as const : [scope] as const;
|
||||
for (const name of required) {
|
||||
|
||||
@@ -366,7 +366,7 @@ export type CatalogSyncState =
|
||||
export type CatalogSyncPhase =
|
||||
| "queued" | "connecting" | "scanning_tables" | "scanning_columns"
|
||||
| "scanning_relationships" | "planning" | "awaiting_confirmation"
|
||||
| "applying" | "completed";
|
||||
| "applying" | "memory_cleanup" | "completed";
|
||||
|
||||
export interface CatalogSchemaDiff {
|
||||
deletedTables: string[];
|
||||
@@ -375,6 +375,7 @@ export interface CatalogSchemaDiff {
|
||||
}
|
||||
|
||||
export interface CatalogSyncCounts {
|
||||
memoryDeleted?: number;
|
||||
tables?: number;
|
||||
columns?: number;
|
||||
relationships?: number;
|
||||
@@ -587,6 +588,7 @@ export interface CatalogRepository {
|
||||
scope: CatalogSyncScope,
|
||||
tableIds: readonly string[],
|
||||
snapshot: ObservedSchemaSnapshot,
|
||||
syncRunId?: string,
|
||||
): Promise<CatalogSyncCounts | undefined>;
|
||||
createSyncRun(
|
||||
databaseId: string,
|
||||
|
||||
@@ -32,6 +32,7 @@ export interface AppConfig {
|
||||
piManagementTimeoutMs: number;
|
||||
secretsFile?: string;
|
||||
installationConfigFile?: string;
|
||||
evidenceHostRegistryRoot?: string;
|
||||
modelCatalogFile?: string;
|
||||
sensitivityNer?: {
|
||||
pythonExecutable: string;
|
||||
@@ -476,6 +477,7 @@ export function loadConfig(
|
||||
piManagementTimeoutMs: piManagementTimeout(env.PI_MANAGEMENT_TIMEOUT_MS),
|
||||
secretsFile,
|
||||
installationConfigFile,
|
||||
evidenceHostRegistryRoot: env.THT_EVIDENCE_HOST_REGISTRY_ROOT || undefined,
|
||||
modelCatalogFile,
|
||||
sensitivityNer,
|
||||
piAuthFile,
|
||||
|
||||
@@ -0,0 +1,90 @@
|
||||
import path from "node:path";
|
||||
import type { FastifyInstance } from "fastify";
|
||||
import { z } from "zod";
|
||||
import { requirePermission, isPrincipalContext } from "../auth/authorization.js";
|
||||
import type { ThtRunner } from "../tht/tht-runner.js";
|
||||
import type { WorkspaceRegistry } from "../workspaces/registry.js";
|
||||
import type { WorkspacePreprocessingService } from "../workspaces/preprocessing-service.js";
|
||||
|
||||
const params = z.object({ workspaceId: z.string().regex(/^[a-z][a-z0-9-]{2,62}$/),
|
||||
evidenceId: z.string().regex(/^evidence:[a-z0-9]+(?:-[a-z0-9]+)*$/).optional() });
|
||||
const query = z.object({
|
||||
q: z.string().max(1000).optional(), kind: z.enum(["domain", "glossary", "enum", "example", "mapping", "normalization", "formula", "reference"]).optional(),
|
||||
purpose: z.enum(["disambiguation", "rewriting", "schema_linking", "sql_generation"]).optional(),
|
||||
status: z.enum(["new", "modified", "active", "removed", "review_required", "legacy", "invalid"]).optional(),
|
||||
concept: z.string().max(300).optional(), table: z.string().max(300).optional(), column: z.string().max(300).optional(),
|
||||
source: z.string().max(300).optional(), language: z.string().max(30).optional(),
|
||||
sort: z.enum(["title", "id", "kind", "status"]).optional(), direction: z.enum(["asc", "desc"]).optional(),
|
||||
page: z.coerce.number().int().positive().optional(), page_size: z.coerce.number().int().min(1).max(100).optional(),
|
||||
}).strict();
|
||||
const quote = (value: string) => `'${value.replaceAll("'", "'\\''")}'`;
|
||||
|
||||
export function evidenceRoutes(app: FastifyInstance, deps: {
|
||||
runner: Pick<ThtRunner, "withPrincipal">; registry: Pick<WorkspaceRegistry, "list">;
|
||||
registryRoot: string; hostRegistryRoot?: string;
|
||||
service: Partial<Pick<WorkspacePreprocessingService, "consolidateEvidence" | "evidenceSources">>;
|
||||
}) {
|
||||
app.post("/workspaces/:workspaceId/evidence/sources", async (request, reply) => {
|
||||
const principal = requirePermission(request, reply, "evidence.manage");
|
||||
if (!isPrincipalContext(principal)) return reply;
|
||||
try {
|
||||
const { workspaceId } = params.parse(request.params);
|
||||
const body = z.discriminatedUnion("action", [
|
||||
z.object({ action: z.literal("refresh") }).strict(),
|
||||
z.object({ action: z.literal("decide"), sourceId: z.string().regex(/^[a-f0-9]{64}$/),
|
||||
revision: z.string().regex(/^[a-f0-9]{64}$/), decision: z.enum(["keep", "replace"]) }).strict(),
|
||||
]).parse(request.body);
|
||||
if (!(await deps.registry.list()).some(w => w.id === workspaceId)) return reply.code(404).send({ code: "workspace_invalid" });
|
||||
if (!deps.service.evidenceSources) throw new Error("Evidence service unavailable");
|
||||
return await deps.service.evidenceSources({ workspaceId, ...body, actor: principal.subject });
|
||||
} catch (error) {
|
||||
return reply.code(error instanceof z.ZodError ? 400 : 503).send({ code: "evidence_unavailable",
|
||||
message: error instanceof z.ZodError ? "Invalid source request." : "Evidence source service is unavailable." });
|
||||
}
|
||||
});
|
||||
app.get<{ Params: { workspaceId: string; evidenceId?: string } }>("/workspaces/:workspaceId/evidence", read);
|
||||
app.get<{ Params: { workspaceId: string; evidenceId?: string } }>("/workspaces/:workspaceId/evidence/:evidenceId", read);
|
||||
async function read(request: import("fastify").FastifyRequest, reply: import("fastify").FastifyReply) {
|
||||
const principal = requirePermission(request, reply, "evidence.manage");
|
||||
if (!isPrincipalContext(principal)) return reply;
|
||||
try {
|
||||
const { workspaceId, evidenceId } = params.parse(request.params);
|
||||
const filters = query.parse(request.query);
|
||||
if (!(await deps.registry.list()).some(w => w.id === workspaceId)) return reply.code(404).send({ code: "workspace_invalid" });
|
||||
const root = path.join(deps.registryRoot, "repo", workspaceId);
|
||||
const result = await deps.runner.withPrincipal(principal).runWithRuntimeSnapshot(
|
||||
["evidence", "admin", "--workspace", workspaceId],
|
||||
JSON.stringify({ root, query: { ...filters, ...(evidenceId ? { id: evidenceId } : {}) } }),
|
||||
);
|
||||
const payload = JSON.parse(result.stdout);
|
||||
if (result.code !== 0) return reply.code(503).send(payload);
|
||||
if (evidenceId && !payload.item) return reply.code(404).send({ code: "evidence_not_found", message: "Evidence was not found." });
|
||||
const host = deps.hostRegistryRoot;
|
||||
const hostPath = host && (path.isAbsolute(host) || path.win32.isAbsolute(host))
|
||||
? (path.win32.isAbsolute(host) && !path.isAbsolute(host) ? path.win32 : path).join(host, "repo") : null;
|
||||
const hostJoin = hostPath && path.win32.isAbsolute(hostPath) && !path.isAbsolute(hostPath) ? path.win32.join : path.join;
|
||||
return { ...payload, location: { repository: hostPath, workspace: hostPath ? hostJoin(hostPath, workspaceId) : null,
|
||||
runtime_workspace: root, host: "Installation host", command: `tht workspace evidence consolidate --workspace ${workspaceId}`,
|
||||
git_commands: hostPath ? [
|
||||
`git -C ${quote(hostPath)} status --short -- ${quote(workspaceId + "/evidence")}`,
|
||||
`git -C ${quote(hostPath)} diff -- ${quote(workspaceId + "/evidence")}`,
|
||||
`git -C ${quote(hostPath)} add -A -- ${quote(workspaceId + "/evidence")}`,
|
||||
`git -C ${quote(hostPath)} commit --only -m 'Curate Evidence' -- ${quote(workspaceId + "/evidence")}`,
|
||||
`git -C ${quote(hostPath)} push`,
|
||||
] : [] } };
|
||||
} catch (error) {
|
||||
return reply.code(error instanceof z.ZodError ? 400 : 503).send({ code: "evidence_unavailable",
|
||||
message: error instanceof z.ZodError ? "Evidence filters are invalid." : "Evidence archive is unavailable." });
|
||||
}
|
||||
}
|
||||
app.post("/workspaces/:workspaceId/evidence/consolidate", async (request, reply) => {
|
||||
const principal = requirePermission(request, reply, "evidence.manage");
|
||||
if (!isPrincipalContext(principal)) return reply;
|
||||
try {
|
||||
const { workspaceId } = params.parse(request.params);
|
||||
if (!(await deps.registry.list()).some(w => w.id === workspaceId)) return reply.code(404).send({ code: "workspace_invalid" });
|
||||
if (!deps.service.consolidateEvidence) throw new Error("Evidence service unavailable");
|
||||
return await deps.service.consolidateEvidence({ workspaceId });
|
||||
} catch { return reply.code(503).send({ code: "evidence_unavailable", message: "Evidence consolidation is unavailable." }); }
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
import type { FastifyInstance, FastifyReply, FastifyRequest } from "fastify";
|
||||
import { z } from "zod";
|
||||
import { isPrincipalContext, requirePermission } from "../auth/authorization.js";
|
||||
import type { ThtRunner } from "../tht/tht-runner.js";
|
||||
import type { WorkspaceRegistry } from "../workspaces/registry.js";
|
||||
import type { SemanticRuntimeConfig } from "../workspaces/runtime-renderer.js";
|
||||
|
||||
const paramsSchema = z.object({
|
||||
workspaceId: z.string().regex(/^[a-z][a-z0-9_-]{0,63}$/),
|
||||
cardId: z.string().regex(/^mem-[0-9a-f-]{36}$/).optional(),
|
||||
});
|
||||
const family = z.enum(["domain_clarification", "sql_rule", "solved_question", "explained_error"]);
|
||||
const cardSchema = z.object({
|
||||
family, subject: z.string().trim().min(1).max(1000),
|
||||
detail: z.string().max(50000).default(""), scope: z.string().trim().min(1).max(10000),
|
||||
rationale: z.string().max(10000).default(""), question: z.string().max(10000).default(""),
|
||||
sql: z.string().max(100000).default(""),
|
||||
concepts: z.array(z.string().trim().min(1).max(200)).max(100).default([]),
|
||||
dependencies: z.array(z.object({
|
||||
database: z.string().trim().min(1).max(200), schema_name: z.string().max(200).default(""),
|
||||
table: z.string().max(200).default(""), column: z.string().max(200).default(""),
|
||||
}).strict()).max(200).default([]),
|
||||
links: z.array(z.object({
|
||||
target_id: z.string().min(1).max(100), meaning: z.string().trim().min(1).max(1000),
|
||||
}).strict()).max(200).default([]),
|
||||
}).strict();
|
||||
const querySchema = z.object({
|
||||
q: z.string().max(1000).optional(), family: family.optional(),
|
||||
concept: z.string().max(200).optional(), database: z.string().max(200).optional(),
|
||||
table: z.string().max(200).optional(), column: z.string().max(200).optional(),
|
||||
origin: z.enum(["manual", "workflow"]).optional(),
|
||||
updated_after: z.iso.datetime({ offset: true }).optional(),
|
||||
updated_before: z.iso.datetime({ offset: true }).optional(),
|
||||
page: z.coerce.number().int().positive().optional(),
|
||||
page_size: z.coerce.number().int().min(1).max(100).optional(),
|
||||
sort: z.enum(["updated_at", "created_at", "subject", "family"]).optional(),
|
||||
direction: z.enum(["asc", "desc"]).optional(),
|
||||
}).strict();
|
||||
|
||||
export function memoryRoutes(app: FastifyInstance, deps: {
|
||||
runner: Pick<ThtRunner, "withPrincipal">;
|
||||
registry: Pick<WorkspaceRegistry, "list" | "read">;
|
||||
runtime: SemanticRuntimeConfig;
|
||||
}) {
|
||||
const invoke = (action: string) => async (request: FastifyRequest, reply: FastifyReply) => {
|
||||
const principal = requirePermission(request, reply, "memory.manage");
|
||||
if (!isPrincipalContext(principal)) return reply;
|
||||
try {
|
||||
const { workspaceId, cardId } = paramsSchema.parse(request.params);
|
||||
const input = action === "list" ? querySchema.parse(request.query)
|
||||
: action === "create" || action === "update"
|
||||
? { card: cardSchema.parse(request.body), ...(cardId ? { id: cardId } : {}) }
|
||||
: cardId ? { id: cardId } : {};
|
||||
// Registry identity is enough: administrative browsing must not require a DWH binding.
|
||||
const workspaces = await deps.registry.list();
|
||||
if (!workspaces.some(workspace => workspace.id === workspaceId)) {
|
||||
return reply.code(404).send({ code: "workspace_invalid", message: "Workspace was not found." });
|
||||
}
|
||||
const { workspace } = await deps.registry.read(workspaceId);
|
||||
const result = await deps.runner.withPrincipal(principal).runWithRuntimeSnapshot(
|
||||
["memory", "admin", "--workspace", workspaceId],
|
||||
JSON.stringify({ action, request: input, runtime: {
|
||||
...deps.runtime, memoryLanguage: workspace.workspace.language,
|
||||
} }),
|
||||
);
|
||||
let payload;
|
||||
try { payload = JSON.parse(result.stdout); } catch {
|
||||
return reply.code(503).send({ code: "memory_unavailable", message: "Memory archive is unavailable." });
|
||||
}
|
||||
if (result.code !== 0) {
|
||||
const codes: Record<string, number> = {
|
||||
memory_invalid: 400, memory_forbidden: 403, memory_not_found: 404,
|
||||
memory_conflict: 409, memory_unavailable: 503,
|
||||
};
|
||||
const code = typeof payload?.code === "string" && payload.code in codes
|
||||
? payload.code : "memory_unavailable";
|
||||
return reply.code(codes[code]).send({ code, message: "Memory operation could not be completed." });
|
||||
}
|
||||
return reply.code(action === "create" ? 201 : 200).send(payload);
|
||||
} catch (error) {
|
||||
if (error instanceof z.ZodError) {
|
||||
return reply.code(400).send({ code: "memory_invalid", message: "Memory request is invalid." });
|
||||
}
|
||||
return reply.code(503).send({ code: "memory_unavailable", message: "Memory archive is unavailable." });
|
||||
}
|
||||
};
|
||||
app.get("/workspaces/:workspaceId/memory", invoke("list"));
|
||||
app.post("/workspaces/:workspaceId/memory", invoke("create"));
|
||||
app.get("/workspaces/:workspaceId/memory/pending", invoke("pending"));
|
||||
app.get("/workspaces/:workspaceId/memory/:cardId", invoke("show"));
|
||||
app.put("/workspaces/:workspaceId/memory/:cardId", invoke("update"));
|
||||
app.delete("/workspaces/:workspaceId/memory/:cardId", invoke("delete"));
|
||||
app.post("/workspaces/:workspaceId/memory/:cardId/retry", invoke("retry"));
|
||||
}
|
||||
@@ -626,6 +626,23 @@ export function sessionRoutes(
|
||||
} catch (error) { return lifecycleFailure(reply, error); }
|
||||
const rt = d.mgr.get(id);
|
||||
if (!rt) return reply.code(404).send({ error: "sessione non attiva" });
|
||||
const pending = rt.bridge.pendingWidget?.() as any;
|
||||
const response = (req.body as any)?.ui_response;
|
||||
if (pending?.widget === "archive-repair" && response?.id === pending.id && !response.control) {
|
||||
const choices = response.choices;
|
||||
if (!Array.isArray(choices) || choices.length !== 1)
|
||||
return reply.code(400).send({ error: "Select one archive repair choice" });
|
||||
if (choices[0] !== "continue" && choices[0] !== "reject") {
|
||||
const option = pending.repair?.options?.find((item: any) => item.id === choices[0]);
|
||||
if (!option || !["memory", "evidence"].includes(option.archive))
|
||||
return reply.code(400).send({ error: "Unknown archive repair choice" });
|
||||
const permission = option.archive === "memory" ? "memory.manage" : "evidence.manage";
|
||||
if (!principal.isAdmin || !hasPermission(principal, permission))
|
||||
return reply.code(403).send({ error: "Archive corrections require an administrator" });
|
||||
if (rt.ownerKey !== `${principal.issuer}\0${principal.subject}`)
|
||||
return reply.code(409).send({ error: "Resume the session with your account before correcting an archive" });
|
||||
}
|
||||
}
|
||||
if (!rt.bridge.respond((req.body as any).ui_response)) {
|
||||
return reply.code(409).send({ error: "risposta non corrispondente al gate in attesa" });
|
||||
}
|
||||
|
||||
@@ -18,7 +18,7 @@ export interface WorkspaceMaintenanceIo {
|
||||
writeStderr(value: string): void;
|
||||
}
|
||||
|
||||
type Command = "inspect" | "preprocess-run" | "preprocess-clear";
|
||||
type Command = "inspect" | "preprocess-run" | "preprocess-clear" | "evidence-consolidate" | "evidence-refresh" | "evidence-decide";
|
||||
|
||||
function failureResult(
|
||||
operation: string,
|
||||
@@ -65,6 +65,9 @@ function parseRequest(command: string, stdin: string): Record<string, unknown> {
|
||||
inspect: ["schemaVersion", "workspaceId"],
|
||||
"preprocess-run": ["schemaVersion", "workspaceId"],
|
||||
"preprocess-clear": ["schemaVersion", "workspaceId"],
|
||||
"evidence-consolidate": ["schemaVersion", "workspaceId"],
|
||||
"evidence-refresh": ["schemaVersion", "workspaceId"],
|
||||
"evidence-decide": ["schemaVersion", "workspaceId", "sourceId", "revision", "decision"],
|
||||
};
|
||||
const allowed = allowedByCommand[command];
|
||||
if (!allowed) throw new Error("unknown command");
|
||||
@@ -81,6 +84,16 @@ function exitCodeFor(result: WorkspaceOperationResult): number {
|
||||
|
||||
async function dispatch(command: Command, service: WorkspacePreprocessingService, request: Record<string, unknown>): Promise<WorkspaceOperationResult> {
|
||||
switch (command) {
|
||||
case "evidence-refresh":
|
||||
return await service.evidenceSources({ workspaceId: request.workspaceId as string, action: "refresh" });
|
||||
case "evidence-decide":
|
||||
if (typeof request.sourceId !== "string" || !/^[a-f0-9]{64}$/.test(request.sourceId)
|
||||
|| typeof request.revision !== "string" || !/^[a-f0-9]{64}$/.test(request.revision)
|
||||
|| (request.decision !== "keep" && request.decision !== "replace")) throw new Error("invalid request");
|
||||
return await service.evidenceSources({ workspaceId: request.workspaceId as string, action: "decide",
|
||||
sourceId: request.sourceId, revision: request.revision, decision: request.decision });
|
||||
case "evidence-consolidate":
|
||||
return await service.consolidateEvidence({ workspaceId: request.workspaceId as string });
|
||||
case "inspect":
|
||||
return await service.inspect({ workspaceId: request.workspaceId as string });
|
||||
case "preprocess-run":
|
||||
@@ -127,7 +140,8 @@ export async function runWorkspaceMaintenanceCli(
|
||||
|| message === "unexpected request field"
|
||||
|| message === "invalid workspace id";
|
||||
return command in {
|
||||
inspect: true, "preprocess-run": true, "preprocess-clear": true,
|
||||
inspect: true, "preprocess-run": true, "preprocess-clear": true, "evidence-consolidate": true,
|
||||
"evidence-refresh": true, "evidence-decide": true,
|
||||
} ? (requestError ? 2 : 1) : 2;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -22,6 +22,7 @@ export interface EvidencePreprocessingRequest {
|
||||
evidence: EvidenceConfig;
|
||||
job: EvidenceJobState;
|
||||
dryRun?: boolean;
|
||||
consolidate?: boolean;
|
||||
httpPrivateHostAllowlist?: readonly string[];
|
||||
}
|
||||
|
||||
@@ -56,7 +57,7 @@ function isPrivateHost(hostname: string): boolean {
|
||||
return hostname.endsWith(".internal");
|
||||
}
|
||||
|
||||
function evidencePolicy(
|
||||
export function evidencePolicy(
|
||||
evidence: EvidenceConfig,
|
||||
httpPrivateHostAllowlist?: readonly string[],
|
||||
): EvidencePreprocessingOutcome | undefined {
|
||||
@@ -95,13 +96,14 @@ function jobResult(job: EvidenceJobState): Pick<
|
||||
};
|
||||
}
|
||||
|
||||
async function runEvidenceStage(
|
||||
export async function runEvidenceStage(
|
||||
request: EvidencePreprocessingRequest,
|
||||
deps: EvidencePreprocessingDependencies,
|
||||
): Promise<EvidencePreprocessingOutcome> {
|
||||
const payload = await deps.runStage([
|
||||
"preprocess",
|
||||
"evidence",
|
||||
...(request.consolidate ? ["--consolidate"] : []),
|
||||
...(request.dryRun ? ["--dry-run"] : []),
|
||||
...(request.job.childRuns.evidence
|
||||
? ["--resume", request.job.childRuns.evidence]
|
||||
|
||||
@@ -3,6 +3,8 @@ import { renameSync, rmSync, writeFileSync, mkdirSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import {
|
||||
continueEvidencePreprocessing,
|
||||
evidencePolicy,
|
||||
runEvidenceStage,
|
||||
type EvidencePreprocessingDependencies,
|
||||
type EvidencePreprocessingOutcome,
|
||||
} from "./evidence/preprocessing.js";
|
||||
@@ -108,6 +110,72 @@ function baseResult(
|
||||
export class WorkspacePreprocessingService {
|
||||
constructor(private readonly deps: WorkspacePreprocessingServiceDeps) {}
|
||||
|
||||
async evidenceSources(options: { workspaceId: string; action: "refresh" | "decide";
|
||||
sourceId?: string; revision?: string; decision?: "keep" | "replace"; actor?: string }): Promise<WorkspaceOperationResult> {
|
||||
const runtime = await this.deps.acquireActiveRuntime(options.workspaceId);
|
||||
try {
|
||||
if (options.action === "refresh") {
|
||||
const refused = evidencePolicy(runtime.workspace.evidence, this.deps.httpPrivateHostAllowlist);
|
||||
if (refused) return baseResult(runtime, "evidence refresh", "failed", refused.code);
|
||||
}
|
||||
const argv = ["evidence", "sources", options.action, "--json", "-c", "/dev/fd/3"];
|
||||
if (options.action === "decide") {
|
||||
if (!/^[a-f0-9]{64}$/.test(options.sourceId ?? "") || !/^[a-f0-9]{64}$/.test(options.revision ?? "")
|
||||
|| !["keep", "replace"].includes(options.decision ?? "")) throw new Error("Invalid source decision");
|
||||
const preflight = await this.deps.evidencePreflight(runtime.workspace);
|
||||
if (!preflight.ok) return baseResult(runtime, "evidence sources", "failed", preflight.code);
|
||||
argv.push("--source-id", options.sourceId!, "--revision", options.revision!, "--decision", options.decision!);
|
||||
}
|
||||
argv.push("--actor", options.actor ?? "installation operator");
|
||||
const result = await this.deps.runChild({ argv, configPath: runtime.configLease.path });
|
||||
const payload = JSON.parse(result.stdout);
|
||||
if (result.exitCode !== 0 || payload.status !== "succeeded") throw new Error(
|
||||
typeof payload.error === "string" ? payload.error.slice(0, 1500) : "Evidence source operation failed");
|
||||
return baseResult(runtime, `evidence ${options.action}`, "succeeded", "ok", {
|
||||
counts: this.numberRecord(payload.counts),
|
||||
warnings: [options.action === "refresh" ? "Source comparisons are ready in Evidence management. Active content is unchanged."
|
||||
: "Source decision activated locally. Commit and push the Evidence tree manually."],
|
||||
});
|
||||
} catch (error) {
|
||||
return baseResult(runtime, `evidence ${options.action}`, "failed", "evidence_materialization_required", {
|
||||
warnings: [error instanceof Error ? error.message : "Evidence source operation failed"],
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async consolidateEvidence(options: { workspaceId: string }): Promise<WorkspaceOperationResult> {
|
||||
const runtime = await this.deps.acquireActiveRuntime(options.workspaceId);
|
||||
const preflight = await this.deps.evidencePreflight(runtime.workspace);
|
||||
if (!preflight.ok) return baseResult(runtime, "evidence consolidate", "failed", preflight.code);
|
||||
// Evidence has its own durable archive/corpus jobs. Do not alter Catalog readiness
|
||||
// or the full preprocessing job when publishing this one component.
|
||||
try {
|
||||
const outcome = await runEvidenceStage({ evidence: runtime.workspace.evidence,
|
||||
consolidate: true, job: { runId: randomBytes(16).toString("hex"), completedStages: [], childRuns: {} },
|
||||
}, {
|
||||
runStage: async argv => {
|
||||
const result = await this.deps.runChild({ argv, configPath: runtime.configLease.path });
|
||||
const payload = JSON.parse(result.stdout);
|
||||
if (result.exitCode !== 0 || payload.status !== "succeeded") {
|
||||
return Promise.reject(new Error(typeof payload.error === "string" ? payload.error.slice(0, 1500) : "Evidence consolidation failed. Retry the command."));
|
||||
}
|
||||
return payload;
|
||||
},
|
||||
persistJob: () => undefined,
|
||||
evidencePreflight: async () => preflight,
|
||||
requireRunId: value => this.requireRunId(value),
|
||||
numberRecord: value => this.numberRecord(value),
|
||||
});
|
||||
return baseResult(runtime, "evidence consolidate", "succeeded", "ok", {
|
||||
...outcome, warnings: ["Evidence is active locally. Catalog and Schema readiness are unchanged. Commit and push the Evidence files manually."],
|
||||
});
|
||||
} catch (error) {
|
||||
return baseResult(runtime, "evidence consolidate", "failed", "evidence_materialization_required", {
|
||||
warnings: [error instanceof Error ? error.message : "Evidence consolidation failed. Retry the command."],
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async inspect(options: { workspaceId: string }): Promise<WorkspaceOperationResult> {
|
||||
try {
|
||||
const runtime = await this.deps.acquireActiveRuntime(options.workspaceId);
|
||||
|
||||
@@ -497,7 +497,10 @@ export async function publishDeterministicRuntimeConfigLease(options: {
|
||||
rendered.workspaceRevision,
|
||||
renderedConfigObject,
|
||||
);
|
||||
const identitySuffix = inputFingerprintValue.slice(7, 23);
|
||||
// The Catalog input identity intentionally excludes representation-only changes.
|
||||
// A lease also identifies its bytes, so a new renderer never collides with an
|
||||
// immutable config produced by an earlier release for the same Catalog inputs.
|
||||
const identitySuffix = sha256(inputFingerprintValue + "\n" + publishedConfig).slice(7, 23);
|
||||
|
||||
const preprocessingRoot = ensureTrustedDirectory(join(
|
||||
options.dataRoot,
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { basename, join } from "node:path";
|
||||
import { basename, dirname, join } from "node:path";
|
||||
import { stringify } from "yaml";
|
||||
import { buildInstallationContract } from "./contracts.js";
|
||||
import { validateWorkspaceDescriptor, type WorkspaceDescriptor } from "./schema.js";
|
||||
@@ -179,6 +179,7 @@ function renderEvidence(
|
||||
return {
|
||||
evidence: {
|
||||
...(workspace.evidence.schema_version === 2 ? { schema_version: 2 } : {}),
|
||||
local_archive_root: join(dirname(dirname(context.revisionContentRoot)), "repo", context.workspaceId),
|
||||
sources: [renderedSource],
|
||||
},
|
||||
vector: {
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
import Fastify from "fastify";
|
||||
import { expect, test, vi } from "vitest";
|
||||
import { sessionRoutes } from "../src/routes/sessions.js";
|
||||
import type { PrincipalContext } from "../src/auth/principal.js";
|
||||
|
||||
test.each([
|
||||
[false, "m", 403], [false, "e", 403], [false, "reject", 204],
|
||||
[true, "m", 204], [true, "e", 204], [true, "forged", 400],
|
||||
] as const)("archive repair response enforces the responding principal (%s, %s)", async (admin, choice, status) => {
|
||||
const app = Fastify();
|
||||
const principal: PrincipalContext = { issuer: "test", subject: "reviewer", isAdmin: admin,
|
||||
roles: admin ? ["admin"] : ["user"],
|
||||
permissions: admin ? ["session.use", "memory.manage", "evidence.manage"] : ["session.use"] };
|
||||
app.addHook("preHandler", async request => { request.principal = principal; });
|
||||
const respond = vi.fn(() => true);
|
||||
sessionRoutes(app, {
|
||||
tht: {}, mgr: { get: () => ({ ownerKey: "test\0reviewer", bridge: {
|
||||
respond, pendingWidget: () => ({ id: "gate", widget: "archive-repair", repair: { options: [
|
||||
{ id: "m", archive: "memory" }, { id: "e", archive: "evidence" },
|
||||
] } }),
|
||||
} }) },
|
||||
} as unknown as Parameters<typeof sessionRoutes>[1]);
|
||||
try {
|
||||
const result = await app.inject({ method: "POST", url: "/sessions/s/response",
|
||||
payload: { ui_response: { id: "gate", choices: [choice] } } });
|
||||
expect(result.statusCode).toBe(status);
|
||||
expect(respond).toHaveBeenCalledTimes(status === 204 ? 1 : 0);
|
||||
} finally { await app.close(); }
|
||||
});
|
||||
|
||||
test("an administrator cannot attribute a repair to another runtime's principal", async () => {
|
||||
const app = Fastify();
|
||||
app.addHook("preHandler", async request => { request.principal = {
|
||||
issuer: "test", subject: "second-admin", isAdmin: true, roles: ["admin"],
|
||||
permissions: ["session.use", "memory.manage"],
|
||||
}; });
|
||||
const respond = vi.fn();
|
||||
sessionRoutes(app, { tht: {}, mgr: { get: () => ({ ownerKey: "test\0first-admin", bridge: {
|
||||
respond, pendingWidget: () => ({ id: "gate", widget: "archive-repair",
|
||||
repair: { options: [{ id: "m", archive: "memory" }] } }),
|
||||
} }) } } as unknown as Parameters<typeof sessionRoutes>[1]);
|
||||
try {
|
||||
const result = await app.inject({ method: "POST", url: "/sessions/s/response",
|
||||
payload: { ui_response: { id: "gate", choices: ["m"] } } });
|
||||
expect(result.statusCode).toBe(409);
|
||||
expect(respond).not.toHaveBeenCalled();
|
||||
} finally { await app.close(); }
|
||||
});
|
||||
@@ -188,6 +188,8 @@ test("roles collapse duplicates and admin contains all administrative permission
|
||||
"workspace.manage",
|
||||
"workspace.secrets.manage",
|
||||
"database.manage",
|
||||
"memory.manage",
|
||||
"evidence.manage",
|
||||
"pi.manage",
|
||||
"auth.diagnostics.read",
|
||||
]);
|
||||
|
||||
@@ -130,7 +130,7 @@ test("local login sets a non-persistent opaque session cookie and exposes only a
|
||||
roles: ["admin"],
|
||||
permissions: [
|
||||
"session.use", "session.read_all", "session.manage_all", "settings.manage",
|
||||
"workspace.manage", "workspace.secrets.manage", "database.manage", "pi.manage", "auth.diagnostics.read",
|
||||
"workspace.manage", "workspace.secrets.manage", "database.manage", "memory.manage", "evidence.manage", "pi.manage", "auth.diagnostics.read",
|
||||
],
|
||||
isAdmin: true,
|
||||
csrfToken: expect.stringMatching(/^[A-Za-z0-9_-]{43}$/),
|
||||
|
||||
@@ -7,6 +7,8 @@ import { join } from "node:path";
|
||||
import { expandLocalHome, localPrincipal, upstreamPrincipal } from "../src/auth/principal.js";
|
||||
import { buildApp } from "../src/app.js";
|
||||
import { loadConfig } from "../src/config.js";
|
||||
import { rolesToPermissions } from "../src/auth/config.js";
|
||||
import type { Permission, Role } from "../src/auth/types.js";
|
||||
|
||||
test("server smoke rejects retired trusted claims under OIDC authentication", () => {
|
||||
const smoke = readFileSync("../scripts/unified-deployment-smoke.sh", "utf8");
|
||||
@@ -32,7 +34,7 @@ test("local mode resolves a stable local principal", async () => {
|
||||
roles: ["admin"],
|
||||
permissions: [
|
||||
"session.use", "session.read_all", "session.manage_all", "settings.manage",
|
||||
"workspace.manage", "workspace.secrets.manage", "database.manage", "pi.manage", "auth.diagnostics.read",
|
||||
"workspace.manage", "workspace.secrets.manage", "database.manage", "memory.manage", "evidence.manage", "pi.manage", "auth.diagnostics.read",
|
||||
],
|
||||
isAdmin: true,
|
||||
});
|
||||
@@ -74,7 +76,7 @@ test("upstream mode accepts only normalized proxy principal headers", async () =
|
||||
issuer: "portal", subject: "42", displayName: "Alice", roles: ["user", "admin"],
|
||||
permissions: [
|
||||
"session.use", "session.read_all", "session.manage_all", "settings.manage",
|
||||
"workspace.manage", "workspace.secrets.manage", "database.manage", "pi.manage", "auth.diagnostics.read",
|
||||
"workspace.manage", "workspace.secrets.manage", "database.manage", "memory.manage", "evidence.manage", "pi.manage", "auth.diagnostics.read",
|
||||
],
|
||||
isAdmin: true,
|
||||
});
|
||||
@@ -206,15 +208,33 @@ test("the session boundary rejects tht maintenance headers outside exact loopbac
|
||||
})).statusCode).toBe(503);
|
||||
});
|
||||
|
||||
test("the session boundary touches a valid cookie session through the bounded Task 7 store operation", async () => {
|
||||
test.each<{
|
||||
name: string;
|
||||
roles: Role[];
|
||||
storedPermissions: Permission[];
|
||||
}>([
|
||||
{ name: "current user", roles: ["user"], storedPermissions: ["session.use"] },
|
||||
{
|
||||
name: "administrator signed in before archive permissions existed",
|
||||
roles: ["admin"],
|
||||
storedPermissions: rolesToPermissions(["admin"]).filter(
|
||||
(permission) => permission !== "memory.manage" && permission !== "evidence.manage",
|
||||
),
|
||||
},
|
||||
{
|
||||
name: "user with obsolete administrative permissions",
|
||||
roles: ["user"],
|
||||
storedPermissions: ["session.use", "memory.manage", "evidence.manage"],
|
||||
},
|
||||
])("the session boundary derives current permissions and touches a valid cookie: $name", async ({ roles, storedPermissions }) => {
|
||||
const sessions = {
|
||||
resolve: vi.fn(async () => ({
|
||||
version: 1,
|
||||
issuer: "local",
|
||||
subject: "user-1",
|
||||
method: "local",
|
||||
roles: ["user"],
|
||||
permissions: ["session.use"],
|
||||
roles,
|
||||
permissions: storedPermissions,
|
||||
userAuthRevision: 1,
|
||||
authConfigRevision: "b".repeat(64),
|
||||
remembered: false,
|
||||
@@ -249,7 +269,13 @@ test("the session boundary touches a valid cookie session through the bounded Ta
|
||||
app.get("/private", async (request) => getPrincipal(request));
|
||||
|
||||
const token = "z".repeat(43);
|
||||
expect((await app.inject({ method: "GET", url: "/private", headers: { cookie: `thothii_session=${token}` } })).statusCode).toBe(200);
|
||||
const response = await app.inject({ method: "GET", url: "/private", headers: { cookie: `thothii_session=${token}` } });
|
||||
expect(response.statusCode).toBe(200);
|
||||
expect(response.json()).toMatchObject({
|
||||
roles,
|
||||
permissions: rolesToPermissions(roles),
|
||||
isAdmin: roles.includes("admin"),
|
||||
});
|
||||
expect(sessions.touch).toHaveBeenCalledWith(token);
|
||||
});
|
||||
|
||||
|
||||
@@ -124,8 +124,17 @@ test.skipIf(!dockerAvailable)("PostgreSQL migration enforces one database per wo
|
||||
],
|
||||
relationships: [],
|
||||
};
|
||||
expect(await repository.applySchemaSync(created.id, 1, "columns", [], fullColumnsSnapshot))
|
||||
const memoryCleanupRun = await repository.createSyncRun(created.id, "columns", [], 1);
|
||||
await repository.updateSyncRun(memoryCleanupRun.id, { state: "applying", phase: "applying" });
|
||||
await expect(repository.applySchemaSync(created.id, 999, "columns", [], fullColumnsSnapshot, memoryCleanupRun.id))
|
||||
.resolves.toBeUndefined();
|
||||
expect((await repository.getSyncRun(memoryCleanupRun.id))?.phase).toBe("applying");
|
||||
expect(await repository.applySchemaSync(created.id, 1, "columns", [], fullColumnsSnapshot, memoryCleanupRun.id))
|
||||
.toMatchObject({ created: 4, deleted: 0 });
|
||||
expect((await repository.getSyncRun(memoryCleanupRun.id))?.phase).toBe("memory_cleanup");
|
||||
await repository.interruptActiveSyncRuns();
|
||||
expect(await repository.getSyncRun(memoryCleanupRun.id))
|
||||
.toMatchObject({ state: "interrupted", phase: "memory_cleanup" });
|
||||
expect((await repository.listColumns(created.id, patients.id)).map((column) => ({
|
||||
name: column.name,
|
||||
sensitive: column.sensitive,
|
||||
@@ -233,6 +242,7 @@ test.skipIf(!dockerAvailable)("PostgreSQL migration enforces one database per wo
|
||||
});
|
||||
expect(await repository.listSyncRuns(created.id)).toEqual([
|
||||
expect.objectContaining({ id: syncRun.id, tableIds: [patients.id] }),
|
||||
expect.objectContaining({ id: memoryCleanupRun.id, phase: "memory_cleanup" }),
|
||||
]);
|
||||
|
||||
const lockedDatabase = await repository.create({
|
||||
|
||||
@@ -117,6 +117,7 @@ async function setup(env: Record<string, string> = {}) {
|
||||
});
|
||||
const introspector: CatalogSchemaIntrospector = { scan };
|
||||
const operations = new CatalogOperationCoordinator();
|
||||
const cleanup = vi.fn(async () => ({ code: 0, stdout: JSON.stringify({ indexed: true, deleted: 0 }), stderr: "" }));
|
||||
const registry = {
|
||||
list: vi.fn(async () => [revision]),
|
||||
listCatalog: vi.fn(async () => [{ id: "psd-clinical", name: "Policlinico San Donato", configurationState: "ready", revision }]),
|
||||
@@ -124,7 +125,7 @@ async function setup(env: Record<string, string> = {}) {
|
||||
readPinned: vi.fn(async () => ({ workspace, workspaceConfigPath: revision.snapshotPath })),
|
||||
} as unknown as WorkspaceRegistry;
|
||||
const app = buildApp(loadConfig({ THT_HARNESS_DIR: "/missing", NODE_ENV: "test", ...env }), {
|
||||
thtRunner: {} as never,
|
||||
thtRunner: { withPrincipal: () => ({ runWithRuntimeSnapshot: cleanup }) } as never,
|
||||
workspaceRegistry: registry,
|
||||
workspaceSecretStore: new WorkspaceSecretStore({ root: secretRoot, runtimeRoot, installationId: "test" }),
|
||||
catalogRepository: repository,
|
||||
@@ -133,7 +134,7 @@ async function setup(env: Record<string, string> = {}) {
|
||||
workspaceDiagnoser: vi.fn(),
|
||||
});
|
||||
return {
|
||||
app, repository, database: (await repository.get(created.id))!, scan, operations,
|
||||
app, repository, database: (await repository.get(created.id))!, scan, operations, cleanup,
|
||||
setObserved(next: ObservedSchemaSnapshot) { observed = next; },
|
||||
};
|
||||
}
|
||||
@@ -614,6 +615,45 @@ test("waits for confirmation and rescans before applying destructive changes", a
|
||||
expect(scan).toHaveBeenCalledTimes(3);
|
||||
});
|
||||
|
||||
test("retries committed Memory cleanup with the original removals without rescanning", async () => {
|
||||
const { app, repository, database, scan, setObserved, cleanup } = await setup();
|
||||
await seedCatalog(repository, database);
|
||||
const observed = snapshot();
|
||||
observed.tables = observed.tables.filter(t => t.name !== "visits");
|
||||
observed.columns = observed.columns.filter(c => c.tableName !== "visits");
|
||||
observed.relationships = [];
|
||||
setObserved(observed);
|
||||
cleanup.mockResolvedValueOnce({ code: 0, stdout: JSON.stringify({ indexed: false, deleted: 2 }), stderr: "" });
|
||||
const started = await app.inject({ method: "POST", url: `/catalog/databases/${database.id}/sync-runs`,
|
||||
payload: { version: database.version, scope: "all", tableIds: [] } });
|
||||
const waiting = await waitFor(repository, started.json().id, "awaiting_confirmation");
|
||||
expect(cleanup).not.toHaveBeenCalled();
|
||||
await app.inject({ method: "POST", url: `/catalog/sync-runs/${waiting.id}/confirm`,
|
||||
payload: { confirmationToken: waiting.confirmationToken } });
|
||||
const failed = await waitFor(repository, waiting.id, "failed");
|
||||
expect(failed).toMatchObject({ phase: "memory_cleanup", errorCode: "memory_cleanup_pending",
|
||||
plannedDiff: { deletedTables: ["visits"] } });
|
||||
expect((await repository.listTables(database.id)).map(t => t.name)).toEqual(["patients"]);
|
||||
const callsBeforeRetry = scan.mock.calls.length;
|
||||
const blocked = await app.inject({ method: "POST", url: `/catalog/databases/${database.id}/sync-runs`,
|
||||
payload: { version: database.version, scope: "all", tableIds: [] } });
|
||||
expect(blocked.statusCode).toBe(409);
|
||||
// Simulate startup recovery and an unreachable DWH after schema application.
|
||||
await repository.interruptActiveSyncRuns();
|
||||
scan.mockRejectedValue(new Error("DWH unavailable"));
|
||||
cleanup.mockResolvedValue({ code: 0, stdout: JSON.stringify({ indexed: true, deleted: 2 }), stderr: "" });
|
||||
const retried = await app.inject({ method: "POST", url: `/catalog/sync-runs/${waiting.id}/retry` });
|
||||
expect(retried.statusCode).toBe(202);
|
||||
const completed = await waitFor(repository, waiting.id, "succeeded");
|
||||
expect(completed.counts).toMatchObject({ memoryDeleted: 2 });
|
||||
expect(scan.mock.calls.length).toBe(callsBeforeRetry);
|
||||
expect(cleanup).toHaveBeenCalledTimes(2);
|
||||
expect(cleanup.mock.calls[1]).toEqual(cleanup.mock.calls[0]);
|
||||
const request = JSON.parse((cleanup.mock.calls[0] as unknown as string[])[1]!).request;
|
||||
expect(request).toMatchObject({ sync_id: waiting.id, database: "warehouse",
|
||||
schema_name: "datawarehouse", removed_tables: ["visits"] });
|
||||
});
|
||||
|
||||
test("deletes every catalog table for multiple selected databases and cascades dependent metadata", async () => {
|
||||
const { app, repository, database } = await setup();
|
||||
const second = await repository.create({
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
import Fastify from "fastify";
|
||||
import { afterEach, expect, test, vi } from "vitest";
|
||||
import { evidenceRoutes } from "../src/routes/evidence.js";
|
||||
import type { ThtRunner } from "../src/tht/tht-runner.js";
|
||||
import type { PrincipalContext } from "../src/auth/principal.js";
|
||||
|
||||
const apps: ReturnType<typeof Fastify>[] = [];
|
||||
afterEach(async () => { await Promise.all(apps.splice(0).map(app => app.close())); });
|
||||
function setup(admin = true) {
|
||||
const app = Fastify(); apps.push(app);
|
||||
const principal: PrincipalContext = { issuer: "test", subject: "curator", roles: [admin ? "admin" : "user"],
|
||||
permissions: admin ? ["evidence.manage"] : ["session.use"], isAdmin: admin };
|
||||
app.addHook("preHandler", async request => { request.principal = principal; });
|
||||
const run = vi.fn(async () => ({ code: 0, stdout: JSON.stringify({ items: [], total: 0 }), stderr: "" }));
|
||||
const withPrincipal = vi.fn(() => ({ runWithRuntimeSnapshot: run }) as unknown as ThtRunner);
|
||||
const consolidateEvidence = vi.fn();
|
||||
const evidenceSources = vi.fn().mockResolvedValue({status: "succeeded"});
|
||||
evidenceRoutes(app, { runner: { withPrincipal }, registryRoot: "/data/registry", hostRegistryRoot: "/srv/Thoth workspaces",
|
||||
registry: { list: async () => [{ id: "sales", commit: "a".repeat(40), blob: "b".repeat(40), snapshotPath: "/missing" }] },
|
||||
service: { consolidateEvidence, evidenceSources } });
|
||||
return { app, run, withPrincipal, principal, consolidateEvidence, evidenceSources };
|
||||
}
|
||||
test("browses complete local files with host paths, without a database runtime", async () => {
|
||||
const { app, run, withPrincipal, principal } = setup();
|
||||
const response = await app.inject("/workspaces/sales/evidence?kind=domain&concept=orders&page=2");
|
||||
expect(response.statusCode).toBe(200);
|
||||
expect(withPrincipal).toHaveBeenCalledWith(principal);
|
||||
const [argv, snapshot] = run.mock.calls[0] as unknown as [string[], string];
|
||||
expect(argv).toEqual(["evidence", "admin", "--workspace", "sales"]);
|
||||
expect(JSON.parse(snapshot)).toEqual({ root: "/data/registry/repo/sales", query: { kind: "domain", concept: "orders", page: 2 } });
|
||||
expect(response.json().location.workspace).toBe("/srv/Thoth workspaces/repo/sales");
|
||||
expect(response.json().location.git_commands[0]).toContain("git -C '/srv/Thoth workspaces/repo'");
|
||||
});
|
||||
|
||||
test("source actions require admin, bind the actor, and reject cross-workspace or arbitrary inputs", async () => {
|
||||
const denied = setup(false);
|
||||
expect((await denied.app.inject({method: "POST", url: "/workspaces/sales/evidence/sources", payload: {action: "refresh"}})).statusCode).toBe(403);
|
||||
expect(denied.evidenceSources).not.toHaveBeenCalled();
|
||||
const allowed = setup();
|
||||
expect((await allowed.app.inject({method: "POST", url: "/workspaces/sales/evidence/sources", payload: {action: "refresh"}})).statusCode).toBe(200);
|
||||
expect(allowed.evidenceSources).toHaveBeenCalledWith({workspaceId: "sales", action: "refresh", actor: "curator"});
|
||||
const choice = {action: "decide", sourceId: "a".repeat(64), revision: "b".repeat(64), decision: "keep"};
|
||||
expect((await allowed.app.inject({method: "POST", url: "/workspaces/sales/evidence/sources", payload: choice})).statusCode).toBe(200);
|
||||
expect(allowed.evidenceSources).toHaveBeenLastCalledWith({...choice, workspaceId: "sales", actor: "curator"});
|
||||
for (const payload of [{action: "refresh", url: "http://localhost"}, {...choice, actor: "forged"}, {...choice, revision: "../other"}]) {
|
||||
expect((await allowed.app.inject({method: "POST", url: "/workspaces/sales/evidence/sources", payload})).statusCode).toBe(400);
|
||||
}
|
||||
expect((await allowed.app.inject({method: "POST", url: "/workspaces/other/evidence/sources", payload: choice})).statusCode).toBe(404);
|
||||
expect(allowed.evidenceSources).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
test.each(["/", "/evidence:rule", "/consolidate"])("refuses non-admin access %s", async suffix => {
|
||||
const { app, run, consolidateEvidence } = setup(false);
|
||||
const response = await app.inject({ method: suffix === "/consolidate" ? "POST" : "GET", url: `/workspaces/sales/evidence${suffix === "/" ? "" : suffix}` });
|
||||
expect(response.statusCode).toBe(403); expect(run).not.toHaveBeenCalled(); expect(consolidateEvidence).not.toHaveBeenCalled();
|
||||
});
|
||||
test("rejects workspace escape, unknown workspace and unsupported filters", async () => {
|
||||
const { app, run } = setup();
|
||||
expect((await app.inject("/workspaces/foreign/evidence")).statusCode).toBe(404);
|
||||
expect((await app.inject("/workspaces/sales/evidence?root=/private")).statusCode).toBe(400);
|
||||
expect((await app.inject("/workspaces/sales/evidence?page_size=101")).statusCode).toBe(400);
|
||||
expect(run).not.toHaveBeenCalled();
|
||||
});
|
||||
test("a missing unit is 404 and consolidation preserves the partial outcome", async () => {
|
||||
const { app, consolidateEvidence } = setup();
|
||||
expect((await app.inject("/workspaces/sales/evidence/evidence:missing")).statusCode).toBe(404);
|
||||
consolidateEvidence.mockResolvedValue({ status: "failed", warnings: ["Saved; retry indexing."] });
|
||||
const response = await app.inject({ method: "POST", url: "/workspaces/sales/evidence/consolidate" });
|
||||
expect(response.json()).toEqual({ status: "failed", warnings: ["Saved; retry indexing."] });
|
||||
});
|
||||
@@ -0,0 +1,81 @@
|
||||
import Fastify from "fastify";
|
||||
import { afterEach, expect, test, vi } from "vitest";
|
||||
import { memoryRoutes } from "../src/routes/memory.js";
|
||||
import { DEFAULT_SEMANTIC_RUNTIME } from "../src/workspaces/runtime-renderer.js";
|
||||
import type { ThtRunner } from "../src/tht/tht-runner.js";
|
||||
import type { PrincipalContext } from "../src/auth/principal.js";
|
||||
|
||||
const apps: ReturnType<typeof Fastify>[] = [];
|
||||
afterEach(async () => { await Promise.all(apps.splice(0).map(app => app.close())); });
|
||||
const id = "mem-11111111-1111-4111-8111-111111111111";
|
||||
const input = { family: "domain_clarification", subject: "Order", scope: "Sales" };
|
||||
|
||||
function setup(admin = true) {
|
||||
const app = Fastify(); apps.push(app);
|
||||
const principal: PrincipalContext = { issuer: "test", subject: "operator", roles: [admin ? "admin" : "user"],
|
||||
permissions: admin ? ["memory.manage"] : ["session.use"], isAdmin: admin };
|
||||
app.addHook("preHandler", async request => { request.principal = principal; });
|
||||
const run = vi.fn(async () => ({ code: 0, stdout: JSON.stringify({ items: [], total: 0 }), stderr: "" }));
|
||||
const bound = { runWithRuntimeSnapshot: run } as unknown as ThtRunner;
|
||||
const withPrincipal = vi.fn(() => bound);
|
||||
memoryRoutes(app, { runner: { withPrincipal }, runtime: DEFAULT_SEMANTIC_RUNTIME,
|
||||
registry: {
|
||||
list: async () => [{ id: "sales", commit: "a".repeat(40), blob: "b".repeat(40), snapshotPath: "/missing" }],
|
||||
read: vi.fn().mockResolvedValue({ workspace: { workspace: { language: "it" } } }),
|
||||
} });
|
||||
return { app, run, withPrincipal, principal };
|
||||
}
|
||||
|
||||
test("list binds principal and workspace without requiring a DWH runtime", async () => {
|
||||
const { app, run, withPrincipal, principal } = setup();
|
||||
const response = await app.inject("/workspaces/sales/memory?q=Order&page=2&column=id");
|
||||
expect(response.statusCode).toBe(200);
|
||||
expect(withPrincipal).toHaveBeenCalledWith(principal);
|
||||
const [args, snapshot] = run.mock.calls[0] as unknown as [string[], string];
|
||||
expect(args).toEqual(["memory", "admin", "--workspace", "sales"]);
|
||||
expect(JSON.parse(snapshot)).toMatchObject({ action: "list", request: { q: "Order", page: 2, column: "id" } });
|
||||
expect(JSON.parse(snapshot).runtime.memoryLanguage).toBe("it");
|
||||
});
|
||||
|
||||
test.each([
|
||||
["GET", ""], ["POST", ""], ["GET", "/pending"], ["GET", `/${id}`],
|
||||
["PUT", `/${id}`], ["DELETE", `/${id}`], ["POST", `/${id}/retry`],
|
||||
] as const)("non-admin cannot %s %s", async (method, suffix) => {
|
||||
const { app, run } = setup(false);
|
||||
const response = await app.inject({ method, url: `/workspaces/sales/memory${suffix}`,
|
||||
...(method === "PUT" || method === "POST" && suffix === "" ? { payload: input } : {}) });
|
||||
expect(response.statusCode).toBe(403); expect(run).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
test("unknown workspace and invalid input do not invoke the harness", async () => {
|
||||
const { app, run } = setup();
|
||||
expect((await app.inject("/workspaces/missing/memory")).statusCode).toBe(404);
|
||||
expect((await app.inject("/workspaces/sales/memory?page_size=101")).statusCode).toBe(400);
|
||||
expect((await app.inject({ method: "POST", url: "/workspaces/sales/memory",
|
||||
payload: { ...input, workspace_id: "foreign" } })).statusCode).toBe(400);
|
||||
expect(run).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
test("create preserves saved-but-unindexed outcome, update carries complete related changes", async () => {
|
||||
const { app, run } = setup();
|
||||
run.mockResolvedValue({ code: 0, stdout: JSON.stringify({ id, saved: true, indexed: false }), stderr: "" });
|
||||
const result = await app.inject({ method: "POST", url: "/workspaces/sales/memory", payload: input });
|
||||
expect(result.statusCode).toBe(201); expect(result.json()).toEqual({ id, saved: true, indexed: false });
|
||||
await app.inject({ method: "PUT", url: `/workspaces/sales/memory/${id}`, payload: {
|
||||
...input, links: [{ target_id: id, meaning: "Related" }], dependencies: [{ database: "dwh" }],
|
||||
} });
|
||||
const [, snapshot] = run.mock.calls[1] as unknown as [string[], string];
|
||||
expect(JSON.parse(snapshot)).toMatchObject({ action: "update", request: { id, card: {
|
||||
links: [{ target_id: id, meaning: "Related" }], dependencies: [{ database: "dwh" }],
|
||||
} } });
|
||||
});
|
||||
|
||||
test.each([["memory_not_found", 404], ["memory_conflict", 409], ["memory_unavailable", 503]])(
|
||||
"maps %s without leaking stderr", async (code, status) => {
|
||||
const { app, run } = setup();
|
||||
run.mockResolvedValue({ code: 1, stdout: JSON.stringify({ code, message: "sensitive detail" }), stderr: "secret" });
|
||||
const result = await app.inject("/workspaces/sales/memory");
|
||||
expect(result.statusCode).toBe(status); expect(result.json().code).toBe(code);
|
||||
expect(result.body).not.toMatch(/secret|sensitive/);
|
||||
},
|
||||
);
|
||||
@@ -10,6 +10,28 @@ import {
|
||||
|
||||
const roots: string[] = [];
|
||||
|
||||
test("Evidence consolidation runs only its stage and preserves blocked Catalog readiness", async () => {
|
||||
const catalog = repository();
|
||||
const runChild = vi.fn(async () => ({ exitCode: 0, stdout: JSON.stringify({ status: "succeeded", counts: { documents: 2 } }), stderr: "" }));
|
||||
const result = await service({ repository: catalog, runChild }).consolidateEvidence({ workspaceId: "catalog-workspace" });
|
||||
expect(result.status).toBe("succeeded");
|
||||
expect(runChild).toHaveBeenCalledOnce();
|
||||
expect(runChild.mock.calls[0]?.[0]).toMatchObject({ argv: ["preprocess", "evidence", "--consolidate", "--json", "-c", "/dev/fd/3"] });
|
||||
expect(catalog.beginPreprocessing).not.toHaveBeenCalled();
|
||||
expect(catalog.finishPreprocessing).not.toHaveBeenCalled();
|
||||
expect(catalog.clearPreprocessing).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
test("Evidence index failure reports retry without changing Catalog state", async () => {
|
||||
const catalog = repository();
|
||||
const runChild = vi.fn(async () => ({ exitCode: 1, stdout: JSON.stringify({ status: "failed", saved: true, error: "Saved; retry indexing." }), stderr: "private provider endpoint" }));
|
||||
const result = await service({ repository: catalog, runChild }).consolidateEvidence({ workspaceId: "catalog-workspace" });
|
||||
expect(result.status).toBe("failed");
|
||||
expect(result.warnings).toEqual(["Saved; retry indexing."]);
|
||||
expect(catalog.finishPreprocessing).not.toHaveBeenCalled();
|
||||
expect(JSON.stringify(result)).not.toContain("private provider");
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
roots.splice(0).forEach((root) => rmSync(root, { recursive: true, force: true }));
|
||||
});
|
||||
@@ -254,6 +276,24 @@ test("semantic preflight failure is persisted before returning", async () => {
|
||||
);
|
||||
});
|
||||
|
||||
test.each(["refresh", "decide"] as const)("Evidence %s calls only the source worker and preserves Catalog readiness", async action => {
|
||||
const catalog = repository();
|
||||
const runChild = vi.fn(async () => ({exitCode: 0, stdout: JSON.stringify({status: "succeeded", counts: {changed: 1}}), stderr: ""}));
|
||||
const result = await service({repository: catalog, runChild}).evidenceSources({workspaceId: "catalog-workspace", action,
|
||||
...(action === "decide" ? {sourceId: "a".repeat(64), revision: "b".repeat(64), decision: "replace" as const} : {}), actor: "Curator"});
|
||||
expect(result.status).toBe("succeeded");
|
||||
expect(runChild).toHaveBeenCalledWith({configPath: expect.any(String), argv: expect.arrayContaining(["evidence", "sources", action, "-c", "/dev/fd/3", "--actor", "Curator"])});
|
||||
expect(catalog.beginPreprocessing).not.toHaveBeenCalled();
|
||||
expect(catalog.finishPreprocessing).not.toHaveBeenCalled();
|
||||
expect(catalog.clearPreprocessing).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
test("failed source activation reports the saved decision for retry", async () => {
|
||||
const result = await service({runChild: async () => ({exitCode: 1, stdout: JSON.stringify({status: "failed", saved: true, error: "Decision saved; retry indexing"}), stderr: "secret"})})
|
||||
.evidenceSources({workspaceId: "catalog-workspace", action: "decide", sourceId: "a".repeat(64), revision: "b".repeat(64), decision: "keep"});
|
||||
expect(result).toMatchObject({status: "failed", warnings: ["Decision saved; retry indexing"]});
|
||||
});
|
||||
|
||||
test("clear invalidates Catalog readiness before clearing only derived worker data", async () => {
|
||||
const catalog = repository();
|
||||
const runChild = vi.fn(async () => ({
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { execFile } from "node:child_process";
|
||||
import { createHash } from "node:crypto";
|
||||
import {
|
||||
existsSync,
|
||||
mkdtempSync,
|
||||
@@ -202,7 +203,7 @@ test("deterministic operator leases are keyed by logical identity and stable acr
|
||||
workspaceSecretStore: f.workspaceSecretStore,
|
||||
});
|
||||
|
||||
const suffix = first.inputFingerprint.slice(7, 23);
|
||||
const suffix = createHash("sha256").update(first.inputFingerprint + "\n" + readFileSync(first.path, "utf8")).digest("hex").slice(0, 16);
|
||||
expect(second.path).toBe(first.path);
|
||||
expect(first.path).toBe(join(
|
||||
f.dataRoot,
|
||||
|
||||
@@ -242,6 +242,7 @@ test("separate runtime leases hand off byte-identical revision Evidence configs
|
||||
source_identity: "workspace://psd-clinical",
|
||||
});
|
||||
expect(parse(firstYaml).evidence).toEqual({
|
||||
local_archive_root: join(f.registryConfig.root, "repo", "psd-clinical"),
|
||||
sources: [{
|
||||
type: "filesystem",
|
||||
root: expectedRoot,
|
||||
|
||||
@@ -192,6 +192,7 @@ test("renders filesystem Evidence below the immutable revision content root with
|
||||
expect(rendered.runtime_identity.workspace_revision).toBe(evidenceRevision);
|
||||
expect(rendered.evidence).toEqual({
|
||||
schema_version: 2,
|
||||
local_archive_root: "/srv/registry/repo/psd-clinical",
|
||||
sources: [{
|
||||
type: "filesystem",
|
||||
root: `/srv/registry/snapshots/${evidenceRevision}/psd-clinical/evidence`,
|
||||
@@ -203,7 +204,7 @@ test("renders filesystem Evidence below the immutable revision content root with
|
||||
max_chunk_chars: 4_000,
|
||||
retain_published_generations: 3,
|
||||
});
|
||||
expect(yaml).not.toContain("/srv/registry/repo");
|
||||
expect(rendered.evidence.sources[0].root).not.toContain("/srv/registry/repo");
|
||||
});
|
||||
|
||||
test("renders public HTTP Evidence with exact fractional-second timeouts and every policy limit", () => {
|
||||
@@ -223,6 +224,7 @@ test("renders public HTTP Evidence with exact fractional-second timeouts and eve
|
||||
}));
|
||||
|
||||
expect(rendered.evidence).toEqual({
|
||||
local_archive_root: "/srv/registry/repo/psd-clinical",
|
||||
sources: [{
|
||||
type: "http",
|
||||
urls: ["https://evidence.example.test/guide.md"],
|
||||
|
||||
Reference in New Issue
Block a user