mptyl/ThothII
1
0
Fork 0
Code Issues 25 Pull Requests Actions Packages Projects Releases 1 Wiki Activity

feat(cli): prepare and validate application documents offline

Browse Source
This commit is contained in:
Codex
2026-09-28 16:33:07 +02:00
parent 64e6b9664a
commit b9c3369e7b
20 changed files with 1164 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/src/catalog/bootstrap-cli.ts
+21
View File
@@ -0,0 +1,21 @@
import { readFileSync, lstatSync } from "node:fs";
import { parseAllDocuments } from "yaml";
import { decode, DocumentError, runWorkspaceDocuments } from "../workspaces/documents.js";
import { validateDatabaseBootstrap } from "./bootstrap-documents.js";
/** Internal sibling protocol: only references cross back to Go, never secret contents. */
export function runBootstrapValidation(args: string[]): { status: number; output: string } {
try {
if (args.length !== 5 || args[0] !== "--directory" || args[2] !== "--bootstrap" || args[4] !== "--json") throw new Error("usage");
const checked = runWorkspaceDocuments(["validate", "--directory", args[1], "--json"]);
if (checked.status !== 0) return checked;
const info = lstatSync(args[3]);
if (!info.isFile() || info.isSymbolicLink() || info.size > 1024 * 1024) throw new Error("file");
const validated = decode(readFileSync(args[3], "utf8"), "database-bootstrap.yaml", (source) =>
validateDatabaseBootstrap(parseAllDocuments(source)[0].toJSON(), args[1]), "database bootstrap schema v1 and the Catalog binding contract");
return { status: 0, output: JSON.stringify({ schema_version: 1, ok: true, secret_files: validated.secretFiles, warnings: validated.warnings, issues: [] }) };
} catch (error) {
const issue = error instanceof DocumentError ? error.issue : { document: "database-bootstrap.yaml", field: "$", code: "bootstrap_invalid", correction: "Supply one complete Catalog database configuration per workspace in a readable local bootstrap document." };
return { status: 1, output: JSON.stringify({ schema_version: 1, ok: false, issues: [issue] }) };
}
}
backend/src/catalog/bootstrap-documents.ts
+47
View File
@@ -0,0 +1,47 @@
import { readFileSync } from "node:fs";
import { join } from "node:path";
import { z } from "zod";
import { databaseConfigurationSchema } from "./configuration-schema.js";
import { parseWorkspaceCatalogYaml } from "../workspaces/catalog.js";
import { parseWorkspaceYaml } from "../workspaces/schema.js";
import { discoverWorkspaceSecretRequirements } from "../workspaces/secret-requirements.js";
const reference = z.string().min(1).max(4096);
const database = databaseConfigurationSchema.extend({
secretFiles: z.object({ password: reference.optional(), apiKey: reference.optional(), sshPrivateKey: reference.optional(), sshPrivateKeyPassphrase: reference.optional(), sshKnownHosts: reference.optional(), tlsCa: reference.optional() }).strict(),
evidenceSecretFiles: z.object({ "evidence.signed_urls": reference.optional(), "evidence.access_key": reference.optional(), "evidence.secret_key": reference.optional(), "evidence.session_token": reference.optional() }).strict().optional(),
});
const bootstrap = z.object({ schemaVersion: z.literal(1), databases: z.array(database).min(1).max(1000) }).strict();
export interface BootstrapReference { field: string; path: string }
/** Offline bootstrap boundary: runtime Catalog owns the resulting bindings after import. */
export function validateDatabaseBootstrap(value: unknown, workspaceRoot: string): { secretFiles: BootstrapReference[]; warnings: string[] } {
const document = bootstrap.parse(value);
const catalog = parseWorkspaceCatalogYaml(readFileSync(join(workspaceRoot, "thoth-workspaces.yaml"), "utf8"));
const expected = new Set(catalog.workspaces.map((entry) => entry.id));
const seen = new Set<string>();
const secretFiles: BootstrapReference[] = [];
const warnings: string[] = [];
const issue = (path: (string | number)[], message: string): never => { throw new z.ZodError([{ code: "custom", path, message }]); };
document.databases.forEach((entry, index) => {
if (!expected.has(entry.workspaceId) || seen.has(entry.workspaceId)) issue(["databases", index, "workspaceId"], "Declare each catalog workspace exactly once.");
seen.add(entry.workspaceId);
const required = entry.binding.transport === "rest_api"
? entry.binding.restAuth === "none" ? [] : ["apiKey"] as const
: entry.binding.transport === "ssh_tunnel" ? ["password", "sshPrivateKey", "sshKnownHosts"] as const : ["password"] as const;
for (const name of required) {
if (!entry.secretFiles[name]) issue(["databases", index, "secretFiles", name], "Supply a protected file reference for this transport.");
}
if (entry.binding.transport === "ssh_tunnel") warnings.push(`databases.${index}:ssh_tunnel supports Catalog diagnostics, not NL-to-SQL sessions; choose direct or REST for practice.`);
for (const [name, path] of Object.entries(entry.secretFiles)) secretFiles.push({ field: `databases.${index}.secretFiles.${name}`, path });
const workspace = parseWorkspaceYaml(readFileSync(join(workspaceRoot, entry.workspaceId, "workspace.yaml"), "utf8"));
const requirements = discoverWorkspaceSecretRequirements(workspace, {});
for (const requirement of requirements.filter((item) => item.connector === "evidence" && item.required)) {
if (!(entry.evidenceSecretFiles as Record<string, string> | undefined)?.[requirement.id]) issue(["databases", index, "evidenceSecretFiles"], "Supply the configured Evidence authentication file references.");
}
for (const [name, path] of Object.entries(entry.evidenceSecretFiles ?? {})) secretFiles.push({ field: `databases.${index}.evidenceSecretFiles.${name}`, path });
});
if (seen.size !== expected.size) issue(["databases"], "Add a database binding for every catalog workspace.");
return { secretFiles, warnings };
}
backend/src/catalog/configuration-schema.ts
+44
View File
@@ -0,0 +1,44 @@
import { z } from "zod";
import { parseCredentialFreeHttpUrl } from "../auth/url-policy.js";
import { DATABASE_TRANSPORTS } from "./types.js";
const workspaceIdSchema = z.string().regex(/^[a-z][a-z0-9-]{2,62}$/);
const identifier = z.string().trim().min(1).max(128).regex(/^[A-Za-z_][A-Za-z0-9_$-]*$/);
const nonEmpty = z.string().trim().min(1).max(512);
const port = z.number().int().min(1).max(65_535);
const optionalText = nonEmpty.optional();
const sshHost = z.string().trim().min(1).max(255).regex(/^[A-Za-z0-9_.:\[\]-]+$/).optional();
const sshUsername = z.string().trim().min(1).max(128).regex(/^[A-Za-z0-9._-]+$/).optional();
const bindingSchema = z.object({
transport: z.enum(DATABASE_TRANSPORTS),
host: optionalText,
port: port.optional(),
username: optionalText,
baseUrl: z.string().max(2048)
.refine((value) => parseCredentialFreeHttpUrl(value) !== undefined)
.optional(),
restPath: z.string().regex(/^\/(?!\/)[^?#\\\u0000-\u001f]*$/).max(512).optional(),
restAuth: z.enum(["none", "bearer", "x-api-key"]).optional(),
tlsServername: optionalText,
sshHost,
sshPort: port.optional(),
sshUsername,
sshTargetHost: sshHost,
sshTargetPort: port.optional(),
}).strict().superRefine((binding, context) => {
const required = binding.transport === "postgres_direct"
? ["host", "port", "username"] as const
: binding.transport === "rest_api"
? ["baseUrl", "restPath", "restAuth"] as const
: ["username", "sshHost", "sshPort", "sshUsername", "sshTargetHost", "sshTargetPort"] as const;
for (const field of required) {
if (binding[field] === undefined) context.addIssue({ code: "custom", path: [field], message: "Required" });
}
});
export const databaseConfigurationSchema = z.object({
workspaceId: workspaceIdSchema,
engine: z.literal("postgres"),
databaseName: identifier,
schema: identifier,
binding: bindingSchema,
}).strict();
backend/src/routes/catalog-databases.ts
+1 -42
View File
@@ -1,60 +1,19 @@
import type { FastifyInstance, FastifyReply, FastifyRequest } from "fastify";
import { z } from "zod";
import { isPrincipalContext, requirePermission } from "../auth/authorization.js";
import { parseCredentialFreeHttpUrl } from "../auth/url-policy.js";
import { databaseConfigurationSchema as configSchema } from "../catalog/configuration-schema.js";
import { CatalogService, type CatalogSecretName } from "../catalog/service.js";
import { WorkspaceRegistryError } from "../workspaces/git-repository.js";
import {
CatalogConflictError,
CatalogOperationInProgressError,
CatalogUnavailableError,
DATABASE_TRANSPORTS,
type CatalogRepository,
type DatabaseConfigurationInput,
} from "../catalog/types.js";
import type { CatalogOperationCoordinator } from "../catalog/operation-coordinator.js";
const idSchema = z.uuid();
const workspaceIdSchema = z.string().regex(/^[a-z][a-z0-9-]{2,62}$/);
const identifier = z.string().trim().min(1).max(128).regex(/^[A-Za-z_][A-Za-z0-9_$-]*$/);
const nonEmpty = z.string().trim().min(1).max(512);
const port = z.number().int().min(1).max(65_535);
const optionalText = nonEmpty.optional();
const sshHost = z.string().trim().min(1).max(255).regex(/^[A-Za-z0-9_.:\[\]-]+$/).optional();
const sshUsername = z.string().trim().min(1).max(128).regex(/^[A-Za-z0-9._-]+$/).optional();
const bindingSchema = z.object({
transport: z.enum(DATABASE_TRANSPORTS),
host: optionalText,
port: port.optional(),
username: optionalText,
baseUrl: z.string().max(2048)
.refine((value) => parseCredentialFreeHttpUrl(value) !== undefined)
.optional(),
restPath: z.string().regex(/^\/(?!\/)[^?#\\\u0000-\u001f]*$/).max(512).optional(),
restAuth: z.enum(["none", "bearer", "x-api-key"]).optional(),
tlsServername: optionalText,
sshHost,
sshPort: port.optional(),
sshUsername,
sshTargetHost: sshHost,
sshTargetPort: port.optional(),
}).strict().superRefine((binding, context) => {
const required = binding.transport === "postgres_direct"
? ["host", "port", "username"] as const
: binding.transport === "rest_api"
? ["baseUrl", "restPath", "restAuth"] as const
: ["username", "sshHost", "sshPort", "sshUsername", "sshTargetHost", "sshTargetPort"] as const;
for (const field of required) {
if (binding[field] === undefined) context.addIssue({ code: "custom", path: [field], message: "Required" });
}
});
const configSchema = z.object({
workspaceId: workspaceIdSchema,
engine: z.literal("postgres"),
databaseName: identifier,
schema: identifier,
binding: bindingSchema,
}).strict();
const updateSchema = configSchema.extend({ version: z.number().int().positive() });
const secretNames = [
"password",
backend/src/workspace-documents-cli.ts
+3 -1
View File
@@ -1,6 +1,8 @@
/** Compiled with its runtime for the host CLI: no installation, Docker or host Node required. */
import { runWorkspaceDocuments } from "./workspaces/documents.js";
import { runBootstrapValidation } from "./catalog/bootstrap-cli.js";
const result = runWorkspaceDocuments(process.argv.slice(2));
const args = process.argv.slice(2);
const result = args[0] === "bootstrap" ? runBootstrapValidation(args.slice(1)) : runWorkspaceDocuments(args);
console.log(result.output);
process.exitCode = result.status;
backend/src/workspaces/documents.ts
+3 -3
View File
@@ -17,7 +17,7 @@ interface Report {
const MAX_DOCUMENT_BYTES = 1024 * 1024;
const usage = "tht workspace prepare --directory NEW_PATH --id ID --name NAME [--language en|it] [--json]\ntht workspace validate --directory PATH [--json]";
class DocumentError extends Error {
export class DocumentError extends Error {
constructor(readonly issue: Issue) { super(issue.correction); }
}
function fail(document: string, field: string, code: string, correction: string): never {
@@ -25,7 +25,7 @@ function fail(document: string, field: string, code: string, correction: string)
}
/** Never include parser messages or submitted values: YAML and Zod errors can contain secrets. */
function decode<T>(source: string, document: string, parser: (text: string) => T): T {
export function decode<T>(source: string, document: string, parser: (text: string) => T, contract = "workspace schema v4 or catalog schema v1"): T {
try {
const documents = parseAllDocuments(source, { uniqueKeys: true });
if (documents.length !== 1) fail(document, "$", "yaml_documents", "Keep exactly one YAML document in this file.");
@@ -43,7 +43,7 @@ function decode<T>(source: string, document: string, parser: (text: string) => T
// Strict schemas produce paths containing schema-defined keys and array indices only.
fail(document, issue.path.join(".") || "$", "schema_invalid",
issue.code === "unrecognized_keys" ? "Remove fields not defined by the current workspace/catalog contract."
: "Correct this field using workspace schema v4 or catalog schema v1; check type, required value, uniqueness and allowed values.");
: `Correct this field using ${contract}; check type, required value, uniqueness and allowed values.`);
}
fail(document, "$", "schema_invalid", "Use an authored workspace v4 descriptor; remove database configuration and keep it in the PostgreSQL Metadata Catalog.");
}