mptyl/ThothII
1
0
Fork 0
Code Issues 25 Pull Requests Actions Packages Projects Releases 1 Wiki Activity

feat: refine metadata catalog workflows

Browse Source
This commit is contained in:
Codex
2026-09-02 11:38:47 +02:00
parent 076c9742c5
commit 4531746038
48 changed files with 1345 additions and 493 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/test/catalog-description-generation-routes.test.ts
+90 -19
View File
@@ -218,6 +218,12 @@ test("suggests sensitive flags from structural metadata without persisting them"
runId: responseBody.run.id,
sequence: 2,
level: "info",
message: "Classified 1 of 1 columns.",
},
{
runId: responseBody.run.id,
sequence: 3,
level: "info",
message: "Sensitive-field suggestion generation completed for 1 column.",
},
]);
@@ -671,6 +677,7 @@ test("generates one selected Catalog Column from a single JSON code fence", asyn
errorSummary: null,
});
expect(Object.keys(start.json()).sort()).toEqual([
"cacheReadTokens",
"createdAt",
"databaseId",
"errorSummary",
@@ -678,9 +685,11 @@ test("generates one selected Catalog Column from a single JSON code fence", asyn
"finishedAt",
"generated",
"id",
"inputTokens",
"language",
"modelId",
"nonGeneratable",
"outputTokens",
"processed",
"scope",
"startedAt",
@@ -1272,7 +1281,7 @@ test("Stop aborts source sampling before any model request", async () => {
test("an isolated exhausted technical batch failure allows completion with errors", async () => {
const modelCompleter: ModelCompleter = {
complete: vi.fn(async (request) => {
if (vi.mocked(modelCompleter.complete).mock.calls.length === 1) {
if (vi.mocked(modelCompleter.complete).mock.calls.length <= 2) {
throw new ModelCompletionProviderError();
}
const context = JSON.parse(request.messages[1]!.content.split("\n").slice(1).join("\n"));
@@ -1320,7 +1329,7 @@ test("an isolated exhausted technical batch failure allows completion with error
failed: 10,
errorSummary: "Description generation completed with errors.",
});
expect(modelCompleter.complete).toHaveBeenCalledTimes(2);
expect(modelCompleter.complete).toHaveBeenCalledTimes(3);
const updated = new Map(
(await repository.listColumns(database.id, table.id)).map((column) => [column.id, column]),
);
@@ -1346,10 +1355,10 @@ test("success resets the technical-failure streak and the third later failure st
const modelCompleter: ModelCompleter = {
complete: vi.fn(async (request) => {
const call = vi.mocked(modelCompleter.complete).mock.calls.length;
if ([1, 2, 4, 5, 6].includes(call)) {
if ([1, 2, 4, 5, 6, 7, 8, 9].includes(call)) {
throw Object.assign(new ModelCompletionProviderError(), { message: sensitiveDiagnostic });
}
if (call > 6) throw new Error("a later batch must not start");
if (call > 9) throw new Error("a later batch must not start");
const context = JSON.parse(request.messages[1]!.content.split("\n").slice(1).join("\n"));
return JSON.stringify({
results: context.targets.map((target: { targetId: string }) => ({
@@ -1389,25 +1398,25 @@ test("success resets the technical-failure streak and the third later failure st
expect(run).toMatchObject({
status: "failed",
total: 61,
processed: 60,
processed: 50,
generated: 10,
nonGeneratable: 0,
failed: 50,
failed: 40,
errorSummary: "Description generation stopped after three consecutive technical batch failures.",
});
expect(modelCompleter.complete).toHaveBeenCalledTimes(6);
expect(modelCompleter.complete).toHaveBeenCalledTimes(9);
const requests = vi.mocked(modelCompleter.complete).mock.calls.map(([request]) => request);
expect(new Set(requests.map((request) => request.signal)).size).toBe(1);
expect(new Set(requests.map((request) => request.model.id))).toEqual(new Set([configuredModel.id]));
const updated = new Map(
(await repository.listColumns(database.id, table.id)).map((column) => [column.id, column]),
);
targetIds.slice(20, 30).forEach((targetId) => {
targetIds.slice(10, 20).forEach((targetId) => {
expect(updated.get(targetId)?.generatedDescription).toBe("Successful reset batch.");
});
expect(updated.get(targetIds[60]!)?.generatedDescription).toBeNull();
expect(updated.get(targetIds[50]!)?.generatedDescription).toBeNull();
const events = await repository.listDescriptionGenerationEvents(run.id);
expect(events.filter((event) => event.message.includes("model provider request failed"))).toHaveLength(5);
expect(events.filter((event) => event.message.includes("model provider request failed"))).toHaveLength(8);
expect(events.at(-1)).toEqual(expect.objectContaining({
level: "error",
message: "Description generation stopped after three consecutive technical batch failures.",
@@ -1660,6 +1669,7 @@ test("Description Generation history is newest-first, bounded, and exposes only
expect(response.statusCode).toBe(200);
expect(response.json().map((run: { id: string }) => run.id)).toEqual(ids.slice(1).reverse());
expect(Object.keys(response.json()[0]).sort()).toEqual([
"cacheReadTokens",
"createdAt",
"databaseId",
"errorSummary",
@@ -1667,9 +1677,11 @@ test("Description Generation history is newest-first, bounded, and exposes only
"finishedAt",
"generated",
"id",
"inputTokens",
"language",
"modelId",
"nonGeneratable",
"outputTokens",
"processed",
"scope",
"startedAt",
@@ -1945,7 +1957,7 @@ test("retains completed batch writes when a later batch response is malformed",
});
const { run } = await waitForTerminalRun(app, start.json().id);
expect(modelCompleter.complete).toHaveBeenCalledTimes(2);
expect(modelCompleter.complete).toHaveBeenCalledTimes(3);
expect(run).toMatchObject({
status: "completed_with_errors",
total: 11,
@@ -1972,9 +1984,14 @@ test("retains completed batch writes when a later batch response is malformed",
expect(events.slice(2, 12).map((event) => event.message)).toEqual(
orderedIds.slice(0, 10).map((targetId) => `Generated description for Catalog Column ${targetId}.`),
);
expect(events.find((event) => event.level === "warning" && event.message.includes("Retrying batch"))).toEqual(
expect.objectContaining({
message: "The model response did not match the required schema. Retrying batch (attempt 2 of 2).",
}),
);
expect(events.find((event) => event.level === "error")).toEqual(expect.objectContaining({
level: "error",
message: `The model response was invalid. Affected Catalog Column target: ${orderedIds[10]}.`,
message: `The model response did not match the required schema. Affected Catalog Column target: ${orderedIds[10]}.`,
}));
} finally {
await app.close();
@@ -2186,7 +2203,7 @@ test("Generate Missing skips prior partial results and includes null, empty, and
const metadata = JSON.parse(request.messages[1]!.content.split("\n").slice(1).join("\n"));
if (mode === "partial") {
partialCall += 1;
if (partialCall === 2) throw new ModelCompletionProviderError();
if (partialCall === 2 || partialCall === 3) throw new ModelCompletionProviderError();
return JSON.stringify({
results: metadata.targets.map((target: { targetId: string }) => ({
targetId: target.targetId,
@@ -2512,6 +2529,58 @@ test("localizes valid non-generatable Catalog Column results in English", async
}
});
test("retries invalid JSON once and completes the batch when the second response is valid", async () => {
const modelCompleter: ModelCompleter = {
complete: vi.fn(async (request) => {
if (vi.mocked(modelCompleter.complete).mock.calls.length === 1) {
return { content: "not-json", usage: { input: 11, cacheRead: 3, output: 2 } };
}
const context = JSON.parse(request.messages[1]!.content.split("\n").slice(1).join("\n"));
return {
content: JSON.stringify({
results: context.targets.map((target: { targetId: string }) => ({
targetId: target.targetId,
outcome: "generated",
description: "Generated after the application retry.",
})),
}),
usage: { input: 7, cacheRead: 1, output: 5 },
};
}),
};
const { app, repository, database, table, column } = await setup(modelCompleter);
try {
const start = await app.inject({
method: "POST",
url: `/catalog/databases/${database.id}/description-generation-runs`,
payload: { modelId: configuredModel.id, scope: "selected_columns", targetIds: [column.id] },
});
const { run } = await waitForTerminalRun(app, start.json().id);
expect(run).toMatchObject({
status: "completed",
processed: 1,
generated: 1,
failed: 0,
inputTokens: 18,
cacheReadTokens: 4,
outputTokens: 7,
});
expect(modelCompleter.complete).toHaveBeenCalledTimes(2);
expect(await repository.getColumn(database.id, table.id, column.id)).toMatchObject({
generatedDescription: "Generated after the application retry.",
});
expect(await repository.listDescriptionGenerationEvents(run.id)).toEqual(expect.arrayContaining([
expect.objectContaining({
level: "warning",
message: "The model response was not valid JSON. Retrying batch (attempt 2 of 2).",
}),
]));
} finally {
await app.close();
}
});
test("fails safely when the provider fails and redacts provider diagnostics", async () => {
const sensitiveDiagnostic = "test-provider-secret private prompt raw provider payload";
const modelCompleter: ModelCompleter = {
@@ -2528,6 +2597,7 @@ test("fails safely when the provider fails and redacts provider diagnostics", as
});
const { run } = await waitForTerminalRun(app, start.json().id);
expect(modelCompleter.complete).toHaveBeenCalledTimes(2);
expect(run).toMatchObject({
status: "completed_with_errors",
processed: 1,
@@ -2565,7 +2635,7 @@ test.each([
["duplicate mappings", (targetIds: readonly string[]) => ({ results: [
{ targetId: targetIds[0], outcome: "generated", description: "First valid value" },
{ targetId: targetIds[0], outcome: "non_generatable" },
] })],
] }), "The model response was missing one or more requested targets."],
["unknown mappings", (targetIds: readonly string[]) => ({ results: [
{ targetId: targetIds[0], outcome: "non_generatable" },
{
@@ -2573,15 +2643,15 @@ test.each([
outcome: "generated",
description: "Unknown target value",
},
] })],
] }), "The model response was missing one or more requested targets."],
["missing mappings", (targetIds: readonly string[]) => ({ results: [
{ targetId: targetIds[0], outcome: "generated", description: "Only one result" },
] })],
] }), "The model response was missing one or more requested targets."],
["malformed mappings", (targetIds: readonly string[]) => ({ results: [
{ targetId: targetIds[0], outcome: "generated", description: "First valid value" },
{ targetId: targetIds[1], outcome: "generated", description: " " },
] })],
] as const)("rejects %s without applying any result from the batch", async (_name, responseFor) => {
] }), "The model response did not match the required schema."],
] as const)("rejects %s without applying any result from the batch", async (_name, responseFor, failureMessage) => {
let selectedColumnIds: string[] = [];
const modelCompleter: ModelCompleter = {
complete: vi.fn(async () => JSON.stringify(responseFor(selectedColumnIds))),
@@ -2625,6 +2695,7 @@ test.each([
});
const { run } = await waitForTerminalRun(app, start.json().id);
expect(modelCompleter.complete).toHaveBeenCalledTimes(2);
expect(run).toMatchObject({
status: "completed_with_errors",
processed: 2,
@@ -2644,7 +2715,7 @@ test.each([
expect((await repository.listDescriptionGenerationEvents(run.id)).find((event) => event.level === "error")).toEqual(
expect.objectContaining({
level: "error",
message: `The model response was invalid. Affected Catalog Column targets: ${selectedColumnIds.join(", ")}.`,
message: `${failureMessage} Affected Catalog Column targets: ${selectedColumnIds.join(", ")}.`,
}),
);
} finally {