feat: complete catalog-driven preprocessing
Publish documentation / publish (push) Successful in 2m12s
Publish documentation / publish (push) Successful in 2m12s
This commit is contained in:
@@ -59,10 +59,16 @@ const completionResponseSchema = z.object({
|
||||
class InvalidModelJsonError extends Error {}
|
||||
class InvalidModelSchemaError extends Error {}
|
||||
class MissingModelTargetsError extends Error {}
|
||||
|
||||
interface DescriptionGenerationFailureTarget {
|
||||
id: string;
|
||||
reference: string;
|
||||
}
|
||||
|
||||
class DescriptionGenerationBatchError extends Error {
|
||||
constructor(
|
||||
readonly failure: unknown,
|
||||
readonly failedTargets: readonly { id: string; label: "Catalog Column" | "Catalog Table" }[],
|
||||
readonly failedTargets: readonly DescriptionGenerationFailureTarget[],
|
||||
) {
|
||||
super("description generation batch failed");
|
||||
}
|
||||
@@ -138,7 +144,7 @@ type ParsedOutcome = z.infer<typeof outcomeSchema>;
|
||||
|
||||
interface DescriptionGenerationPlan {
|
||||
columnTargets: SelectedColumnTarget[];
|
||||
tableIds: string[];
|
||||
tableTargets: Array<{ id: string; name: string }>;
|
||||
}
|
||||
|
||||
interface DescriptionGenerationCounters {
|
||||
@@ -164,10 +170,17 @@ function persistedCounters(counters: DescriptionGenerationCounters) {
|
||||
};
|
||||
}
|
||||
|
||||
function failureTarget(target: SelectedTarget) {
|
||||
function targetReference(target: SelectedTarget): string {
|
||||
return target.kind === "column"
|
||||
? { id: target.column.id, label: "Catalog Column" as const }
|
||||
: { id: target.table.id, label: "Catalog Table" as const };
|
||||
? `Column ${JSON.stringify(`${target.table.name}.${target.column.name}`)}`
|
||||
: `Table ${JSON.stringify(target.table.name)}`;
|
||||
}
|
||||
|
||||
function failureTarget(target: SelectedTarget): DescriptionGenerationFailureTarget {
|
||||
return {
|
||||
id: target.kind === "column" ? target.column.id : target.table.id,
|
||||
reference: targetReference(target),
|
||||
};
|
||||
}
|
||||
|
||||
const NON_GENERATABLE_DESCRIPTION: Record<DescriptionGenerationRun["language"], string> = {
|
||||
@@ -681,7 +694,7 @@ function safeFailure(error: unknown): string {
|
||||
function batchFailureEvent(error: DescriptionGenerationBatchError): string {
|
||||
const summary = safeFailure(error);
|
||||
return error.failedTargets.length > 0
|
||||
? `${summary} Affected ${error.failedTargets[0]!.label} target${error.failedTargets.length === 1 ? "" : "s"}: ${error.failedTargets.map((target) => target.id).join(", ")}.`
|
||||
? `${summary} Affected target${error.failedTargets.length === 1 ? "" : "s"}: ${error.failedTargets.map((target) => target.reference).join(", ")}.`
|
||||
: summary;
|
||||
}
|
||||
|
||||
@@ -794,7 +807,7 @@ export class DescriptionGenerationWorker {
|
||||
scope === "selected_columns" ? "column" : "table",
|
||||
);
|
||||
}
|
||||
const total = plan.columnTargets.length + plan.tableIds.length;
|
||||
const total = plan.columnTargets.length + plan.tableTargets.length;
|
||||
if (total === 0 && (scope === "all" || scope === "missing")) {
|
||||
throw new DescriptionGenerationNoEligibleTargetsError(scope);
|
||||
}
|
||||
@@ -934,12 +947,18 @@ export class DescriptionGenerationWorker {
|
||||
};
|
||||
await this.processTargets(run, database, plan.columnTargets, model, counters, signal);
|
||||
throwIfCancelled(signal);
|
||||
if (plan.tableIds.length > 0) {
|
||||
const tableTargets = await this.resolveTableTargets(run.databaseId, plan.tableIds);
|
||||
if (plan.tableTargets.length > 0) {
|
||||
const tableTargets = await this.resolveTableTargets(
|
||||
run.databaseId,
|
||||
plan.tableTargets.map((target) => target.id),
|
||||
);
|
||||
if (!tableTargets) {
|
||||
throw new DescriptionGenerationBatchError(
|
||||
new Error("selected tables changed during generation"),
|
||||
plan.tableIds.map((id) => ({ id, label: "Catalog Table" })),
|
||||
plan.tableTargets.map((target) => ({
|
||||
id: target.id,
|
||||
reference: `Table ${JSON.stringify(target.name)}`,
|
||||
})),
|
||||
);
|
||||
}
|
||||
await this.processTargets(run, database, tableTargets, model, counters, signal);
|
||||
@@ -1093,8 +1112,8 @@ export class DescriptionGenerationWorker {
|
||||
run.id,
|
||||
"info",
|
||||
outcome.outcome === "generated"
|
||||
? `Generated description for ${target.kind === "column" ? "Catalog Column" : "Catalog Table"} ${targetId}.`
|
||||
: `Stored non-generatable result for ${target.kind === "column" ? "Catalog Column" : "Catalog Table"} ${targetId}.`,
|
||||
? `Generated description for ${targetReference(target)}.`
|
||||
: `Stored non-generatable result for ${targetReference(target)}.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1188,16 +1207,22 @@ export class DescriptionGenerationWorker {
|
||||
}
|
||||
return {
|
||||
columnTargets,
|
||||
tableIds: tables
|
||||
tableTargets: tables
|
||||
.filter((table) => scope === "all" || !table.generatedDescription?.trim())
|
||||
.map((table) => table.id),
|
||||
.map((table) => ({ id: table.id, name: table.name })),
|
||||
};
|
||||
}
|
||||
if (scope === "selected_tables") {
|
||||
const tableById = new Map(tables.map((table) => [table.id, table]));
|
||||
const selected = targetIds.map((tableId) => tableById.get(tableId));
|
||||
if (selected.some((table) => table === undefined)) return undefined;
|
||||
return { columnTargets: [], tableIds: [...targetIds] };
|
||||
return {
|
||||
columnTargets: [],
|
||||
tableTargets: (selected as CatalogTable[]).map((table) => ({
|
||||
id: table.id,
|
||||
name: table.name,
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
||||
const byId = new Map<string, SelectedColumnTarget>();
|
||||
@@ -1209,7 +1234,7 @@ export class DescriptionGenerationWorker {
|
||||
const targets = targetIds.map((columnId) => byId.get(columnId));
|
||||
return targets.some((target) => target === undefined)
|
||||
? undefined
|
||||
: { columnTargets: targets as SelectedColumnTarget[], tableIds: [] };
|
||||
: { columnTargets: targets as SelectedColumnTarget[], tableTargets: [] };
|
||||
}
|
||||
|
||||
private async resolveTableTargets(
|
||||
|
||||
Reference in New Issue
Block a user