feat: complete catalog-driven preprocessing
Publish documentation / publish (push) Successful in 2m12s

This commit is contained in:
Codex
2026-09-06 17:49:35 +02:00
parent 8707ae1d46
commit cffa60772e
141 changed files with 5898 additions and 3015 deletions
@@ -59,10 +59,16 @@ const completionResponseSchema = z.object({
class InvalidModelJsonError extends Error {}
class InvalidModelSchemaError extends Error {}
class MissingModelTargetsError extends Error {}
interface DescriptionGenerationFailureTarget {
id: string;
reference: string;
}
class DescriptionGenerationBatchError extends Error {
constructor(
readonly failure: unknown,
readonly failedTargets: readonly { id: string; label: "Catalog Column" | "Catalog Table" }[],
readonly failedTargets: readonly DescriptionGenerationFailureTarget[],
) {
super("description generation batch failed");
}
@@ -138,7 +144,7 @@ type ParsedOutcome = z.infer<typeof outcomeSchema>;
interface DescriptionGenerationPlan {
columnTargets: SelectedColumnTarget[];
tableIds: string[];
tableTargets: Array<{ id: string; name: string }>;
}
interface DescriptionGenerationCounters {
@@ -164,10 +170,17 @@ function persistedCounters(counters: DescriptionGenerationCounters) {
};
}
function failureTarget(target: SelectedTarget) {
function targetReference(target: SelectedTarget): string {
return target.kind === "column"
? { id: target.column.id, label: "Catalog Column" as const }
: { id: target.table.id, label: "Catalog Table" as const };
? `Column ${JSON.stringify(`${target.table.name}.${target.column.name}`)}`
: `Table ${JSON.stringify(target.table.name)}`;
}
function failureTarget(target: SelectedTarget): DescriptionGenerationFailureTarget {
return {
id: target.kind === "column" ? target.column.id : target.table.id,
reference: targetReference(target),
};
}
const NON_GENERATABLE_DESCRIPTION: Record<DescriptionGenerationRun["language"], string> = {
@@ -681,7 +694,7 @@ function safeFailure(error: unknown): string {
function batchFailureEvent(error: DescriptionGenerationBatchError): string {
const summary = safeFailure(error);
return error.failedTargets.length > 0
? `${summary} Affected ${error.failedTargets[0]!.label} target${error.failedTargets.length === 1 ? "" : "s"}: ${error.failedTargets.map((target) => target.id).join(", ")}.`
? `${summary} Affected target${error.failedTargets.length === 1 ? "" : "s"}: ${error.failedTargets.map((target) => target.reference).join(", ")}.`
: summary;
}
@@ -794,7 +807,7 @@ export class DescriptionGenerationWorker {
scope === "selected_columns" ? "column" : "table",
);
}
const total = plan.columnTargets.length + plan.tableIds.length;
const total = plan.columnTargets.length + plan.tableTargets.length;
if (total === 0 && (scope === "all" || scope === "missing")) {
throw new DescriptionGenerationNoEligibleTargetsError(scope);
}
@@ -934,12 +947,18 @@ export class DescriptionGenerationWorker {
};
await this.processTargets(run, database, plan.columnTargets, model, counters, signal);
throwIfCancelled(signal);
if (plan.tableIds.length > 0) {
const tableTargets = await this.resolveTableTargets(run.databaseId, plan.tableIds);
if (plan.tableTargets.length > 0) {
const tableTargets = await this.resolveTableTargets(
run.databaseId,
plan.tableTargets.map((target) => target.id),
);
if (!tableTargets) {
throw new DescriptionGenerationBatchError(
new Error("selected tables changed during generation"),
plan.tableIds.map((id) => ({ id, label: "Catalog Table" })),
plan.tableTargets.map((target) => ({
id: target.id,
reference: `Table ${JSON.stringify(target.name)}`,
})),
);
}
await this.processTargets(run, database, tableTargets, model, counters, signal);
@@ -1093,8 +1112,8 @@ export class DescriptionGenerationWorker {
run.id,
"info",
outcome.outcome === "generated"
? `Generated description for ${target.kind === "column" ? "Catalog Column" : "Catalog Table"} ${targetId}.`
: `Stored non-generatable result for ${target.kind === "column" ? "Catalog Column" : "Catalog Table"} ${targetId}.`,
? `Generated description for ${targetReference(target)}.`
: `Stored non-generatable result for ${targetReference(target)}.`,
);
}
}
@@ -1188,16 +1207,22 @@ export class DescriptionGenerationWorker {
}
return {
columnTargets,
tableIds: tables
tableTargets: tables
.filter((table) => scope === "all" || !table.generatedDescription?.trim())
.map((table) => table.id),
.map((table) => ({ id: table.id, name: table.name })),
};
}
if (scope === "selected_tables") {
const tableById = new Map(tables.map((table) => [table.id, table]));
const selected = targetIds.map((tableId) => tableById.get(tableId));
if (selected.some((table) => table === undefined)) return undefined;
return { columnTargets: [], tableIds: [...targetIds] };
return {
columnTargets: [],
tableTargets: (selected as CatalogTable[]).map((table) => ({
id: table.id,
name: table.name,
})),
};
}
const byId = new Map<string, SelectedColumnTarget>();
@@ -1209,7 +1234,7 @@ export class DescriptionGenerationWorker {
const targets = targetIds.map((columnId) => byId.get(columnId));
return targets.some((target) => target === undefined)
? undefined
: { columnTargets: targets as SelectedColumnTarget[], tableIds: [] };
: { columnTargets: targets as SelectedColumnTarget[], tableTargets: [] };
}
private async resolveTableTargets(