mptyl/ThothII
1
0
Fork 0
Code Issues 25 Pull Requests Actions Packages Projects Releases 1 Wiki Activity

feat(cli): prepare and validate application documents offline

Browse Source
This commit is contained in:
Codex
2026-09-28 16:33:07 +02:00
parent 64e6b9664a
commit b9c3369e7b
20 changed files with 1164 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/src/catalog/bootstrap-cli.ts
+21
View File
@@ -0,0 +1,21 @@
import { readFileSync, lstatSync } from "node:fs";
import { parseAllDocuments } from "yaml";
import { decode, DocumentError, runWorkspaceDocuments } from "../workspaces/documents.js";
import { validateDatabaseBootstrap } from "./bootstrap-documents.js";
/** Internal sibling protocol: only references cross back to Go, never secret contents. */
export function runBootstrapValidation(args: string[]): { status: number; output: string } {
try {
if (args.length !== 5 || args[0] !== "--directory" || args[2] !== "--bootstrap" || args[4] !== "--json") throw new Error("usage");
const checked = runWorkspaceDocuments(["validate", "--directory", args[1], "--json"]);
if (checked.status !== 0) return checked;
const info = lstatSync(args[3]);
if (!info.isFile() || info.isSymbolicLink() || info.size > 1024 * 1024) throw new Error("file");
const validated = decode(readFileSync(args[3], "utf8"), "database-bootstrap.yaml", (source) =>
validateDatabaseBootstrap(parseAllDocuments(source)[0].toJSON(), args[1]), "database bootstrap schema v1 and the Catalog binding contract");
return { status: 0, output: JSON.stringify({ schema_version: 1, ok: true, secret_files: validated.secretFiles, warnings: validated.warnings, issues: [] }) };
} catch (error) {
const issue = error instanceof DocumentError ? error.issue : { document: "database-bootstrap.yaml", field: "$", code: "bootstrap_invalid", correction: "Supply one complete Catalog database configuration per workspace in a readable local bootstrap document." };
return { status: 1, output: JSON.stringify({ schema_version: 1, ok: false, issues: [issue] }) };
}
}
backend/src/catalog/bootstrap-documents.ts
+47
View File
@@ -0,0 +1,47 @@
import { readFileSync } from "node:fs";
import { join } from "node:path";
import { z } from "zod";
import { databaseConfigurationSchema } from "./configuration-schema.js";
import { parseWorkspaceCatalogYaml } from "../workspaces/catalog.js";
import { parseWorkspaceYaml } from "../workspaces/schema.js";
import { discoverWorkspaceSecretRequirements } from "../workspaces/secret-requirements.js";
const reference = z.string().min(1).max(4096);
const database = databaseConfigurationSchema.extend({
secretFiles: z.object({ password: reference.optional(), apiKey: reference.optional(), sshPrivateKey: reference.optional(), sshPrivateKeyPassphrase: reference.optional(), sshKnownHosts: reference.optional(), tlsCa: reference.optional() }).strict(),
evidenceSecretFiles: z.object({ "evidence.signed_urls": reference.optional(), "evidence.access_key": reference.optional(), "evidence.secret_key": reference.optional(), "evidence.session_token": reference.optional() }).strict().optional(),
});
const bootstrap = z.object({ schemaVersion: z.literal(1), databases: z.array(database).min(1).max(1000) }).strict();
export interface BootstrapReference { field: string; path: string }
/** Offline bootstrap boundary: runtime Catalog owns the resulting bindings after import. */
export function validateDatabaseBootstrap(value: unknown, workspaceRoot: string): { secretFiles: BootstrapReference[]; warnings: string[] } {
const document = bootstrap.parse(value);
const catalog = parseWorkspaceCatalogYaml(readFileSync(join(workspaceRoot, "thoth-workspaces.yaml"), "utf8"));
const expected = new Set(catalog.workspaces.map((entry) => entry.id));
const seen = new Set<string>();
const secretFiles: BootstrapReference[] = [];
const warnings: string[] = [];
const issue = (path: (string | number)[], message: string): never => { throw new z.ZodError([{ code: "custom", path, message }]); };
document.databases.forEach((entry, index) => {
if (!expected.has(entry.workspaceId) || seen.has(entry.workspaceId)) issue(["databases", index, "workspaceId"], "Declare each catalog workspace exactly once.");
seen.add(entry.workspaceId);
const required = entry.binding.transport === "rest_api"
? entry.binding.restAuth === "none" ? [] : ["apiKey"] as const
: entry.binding.transport === "ssh_tunnel" ? ["password", "sshPrivateKey", "sshKnownHosts"] as const : ["password"] as const;
for (const name of required) {
if (!entry.secretFiles[name]) issue(["databases", index, "secretFiles", name], "Supply a protected file reference for this transport.");
}
if (entry.binding.transport === "ssh_tunnel") warnings.push(`databases.${index}:ssh_tunnel supports Catalog diagnostics, not NL-to-SQL sessions; choose direct or REST for practice.`);
for (const [name, path] of Object.entries(entry.secretFiles)) secretFiles.push({ field: `databases.${index}.secretFiles.${name}`, path });
const workspace = parseWorkspaceYaml(readFileSync(join(workspaceRoot, entry.workspaceId, "workspace.yaml"), "utf8"));
const requirements = discoverWorkspaceSecretRequirements(workspace, {});
for (const requirement of requirements.filter((item) => item.connector === "evidence" && item.required)) {
if (!(entry.evidenceSecretFiles as Record<string, string> | undefined)?.[requirement.id]) issue(["databases", index, "evidenceSecretFiles"], "Supply the configured Evidence authentication file references.");
}
for (const [name, path] of Object.entries(entry.evidenceSecretFiles ?? {})) secretFiles.push({ field: `databases.${index}.evidenceSecretFiles.${name}`, path });
});
if (seen.size !== expected.size) issue(["databases"], "Add a database binding for every catalog workspace.");
return { secretFiles, warnings };
}
backend/src/catalog/configuration-schema.ts
+44
View File
@@ -0,0 +1,44 @@
import { z } from "zod";
import { parseCredentialFreeHttpUrl } from "../auth/url-policy.js";
import { DATABASE_TRANSPORTS } from "./types.js";
const workspaceIdSchema = z.string().regex(/^[a-z][a-z0-9-]{2,62}$/);
const identifier = z.string().trim().min(1).max(128).regex(/^[A-Za-z_][A-Za-z0-9_$-]*$/);
const nonEmpty = z.string().trim().min(1).max(512);
const port = z.number().int().min(1).max(65_535);
const optionalText = nonEmpty.optional();
const sshHost = z.string().trim().min(1).max(255).regex(/^[A-Za-z0-9_.:\[\]-]+$/).optional();
const sshUsername = z.string().trim().min(1).max(128).regex(/^[A-Za-z0-9._-]+$/).optional();
const bindingSchema = z.object({
transport: z.enum(DATABASE_TRANSPORTS),
host: optionalText,
port: port.optional(),
username: optionalText,
baseUrl: z.string().max(2048)
.refine((value) => parseCredentialFreeHttpUrl(value) !== undefined)
.optional(),
restPath: z.string().regex(/^\/(?!\/)[^?#\\\u0000-\u001f]*$/).max(512).optional(),
restAuth: z.enum(["none", "bearer", "x-api-key"]).optional(),
tlsServername: optionalText,
sshHost,
sshPort: port.optional(),
sshUsername,
sshTargetHost: sshHost,
sshTargetPort: port.optional(),
}).strict().superRefine((binding, context) => {
const required = binding.transport === "postgres_direct"
? ["host", "port", "username"] as const
: binding.transport === "rest_api"
? ["baseUrl", "restPath", "restAuth"] as const
: ["username", "sshHost", "sshPort", "sshUsername", "sshTargetHost", "sshTargetPort"] as const;
for (const field of required) {
if (binding[field] === undefined) context.addIssue({ code: "custom", path: [field], message: "Required" });
}
});
export const databaseConfigurationSchema = z.object({
workspaceId: workspaceIdSchema,
engine: z.literal("postgres"),
databaseName: identifier,
schema: identifier,
binding: bindingSchema,
}).strict();
backend/src/routes/catalog-databases.ts
+1 -42
View File
@@ -1,60 +1,19 @@
import type { FastifyInstance, FastifyReply, FastifyRequest } from "fastify";
import { z } from "zod";
import { isPrincipalContext, requirePermission } from "../auth/authorization.js";
import { parseCredentialFreeHttpUrl } from "../auth/url-policy.js";
import { databaseConfigurationSchema as configSchema } from "../catalog/configuration-schema.js";
import { CatalogService, type CatalogSecretName } from "../catalog/service.js";
import { WorkspaceRegistryError } from "../workspaces/git-repository.js";
import {
CatalogConflictError,
CatalogOperationInProgressError,
CatalogUnavailableError,
DATABASE_TRANSPORTS,
type CatalogRepository,
type DatabaseConfigurationInput,
} from "../catalog/types.js";
import type { CatalogOperationCoordinator } from "../catalog/operation-coordinator.js";
const idSchema = z.uuid();
const workspaceIdSchema = z.string().regex(/^[a-z][a-z0-9-]{2,62}$/);
const identifier = z.string().trim().min(1).max(128).regex(/^[A-Za-z_][A-Za-z0-9_$-]*$/);
const nonEmpty = z.string().trim().min(1).max(512);
const port = z.number().int().min(1).max(65_535);
const optionalText = nonEmpty.optional();
const sshHost = z.string().trim().min(1).max(255).regex(/^[A-Za-z0-9_.:\[\]-]+$/).optional();
const sshUsername = z.string().trim().min(1).max(128).regex(/^[A-Za-z0-9._-]+$/).optional();
const bindingSchema = z.object({
transport: z.enum(DATABASE_TRANSPORTS),
host: optionalText,
port: port.optional(),
username: optionalText,
baseUrl: z.string().max(2048)
.refine((value) => parseCredentialFreeHttpUrl(value) !== undefined)
.optional(),
restPath: z.string().regex(/^\/(?!\/)[^?#\\\u0000-\u001f]*$/).max(512).optional(),
restAuth: z.enum(["none", "bearer", "x-api-key"]).optional(),
tlsServername: optionalText,
sshHost,
sshPort: port.optional(),
sshUsername,
sshTargetHost: sshHost,
sshTargetPort: port.optional(),
}).strict().superRefine((binding, context) => {
const required = binding.transport === "postgres_direct"
? ["host", "port", "username"] as const
: binding.transport === "rest_api"
? ["baseUrl", "restPath", "restAuth"] as const
: ["username", "sshHost", "sshPort", "sshUsername", "sshTargetHost", "sshTargetPort"] as const;
for (const field of required) {
if (binding[field] === undefined) context.addIssue({ code: "custom", path: [field], message: "Required" });
}
});
const configSchema = z.object({
workspaceId: workspaceIdSchema,
engine: z.literal("postgres"),
databaseName: identifier,
schema: identifier,
binding: bindingSchema,
}).strict();
const updateSchema = configSchema.extend({ version: z.number().int().positive() });
const secretNames = [
"password",
backend/src/workspace-documents-cli.ts
+3 -1
View File
@@ -1,6 +1,8 @@
/** Compiled with its runtime for the host CLI: no installation, Docker or host Node required. */
import { runWorkspaceDocuments } from "./workspaces/documents.js";
import { runBootstrapValidation } from "./catalog/bootstrap-cli.js";
const result = runWorkspaceDocuments(process.argv.slice(2));
const args = process.argv.slice(2);
const result = args[0] === "bootstrap" ? runBootstrapValidation(args.slice(1)) : runWorkspaceDocuments(args);
console.log(result.output);
process.exitCode = result.status;
backend/src/workspaces/documents.ts
+3 -3
View File
@@ -17,7 +17,7 @@ interface Report {
const MAX_DOCUMENT_BYTES = 1024 * 1024;
const usage = "tht workspace prepare --directory NEW_PATH --id ID --name NAME [--language en|it] [--json]\ntht workspace validate --directory PATH [--json]";
class DocumentError extends Error {
export class DocumentError extends Error {
constructor(readonly issue: Issue) { super(issue.correction); }
}
function fail(document: string, field: string, code: string, correction: string): never {
@@ -25,7 +25,7 @@ function fail(document: string, field: string, code: string, correction: string)
}
/** Never include parser messages or submitted values: YAML and Zod errors can contain secrets. */
function decode<T>(source: string, document: string, parser: (text: string) => T): T {
export function decode<T>(source: string, document: string, parser: (text: string) => T, contract = "workspace schema v4 or catalog schema v1"): T {
try {
const documents = parseAllDocuments(source, { uniqueKeys: true });
if (documents.length !== 1) fail(document, "$", "yaml_documents", "Keep exactly one YAML document in this file.");
@@ -43,7 +43,7 @@ function decode<T>(source: string, document: string, parser: (text: string) => T
// Strict schemas produce paths containing schema-defined keys and array indices only.
fail(document, issue.path.join(".") || "$", "schema_invalid",
issue.code === "unrecognized_keys" ? "Remove fields not defined by the current workspace/catalog contract."
: "Correct this field using workspace schema v4 or catalog schema v1; check type, required value, uniqueness and allowed values.");
: `Correct this field using ${contract}; check type, required value, uniqueness and allowed values.`);
}
fail(document, "$", "schema_invalid", "Use an authored workspace v4 descriptor; remove database configuration and keep it in the PostgreSQL Metadata Catalog.");
}
backend/test/database-bootstrap-documents.test.ts
+49
View File
@@ -0,0 +1,49 @@
import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from "node:fs";
import { join } from "node:path";
import { tmpdir } from "node:os";
import { afterEach, expect, test } from "vitest";
import { validateDatabaseBootstrap } from "../src/catalog/bootstrap-documents.js";
import { runBootstrapValidation } from "../src/catalog/bootstrap-cli.js";
const roots: string[] = [];
afterEach(() => roots.splice(0).forEach((root) => rmSync(root, { recursive: true, force: true })));
function fixture() {
const root = mkdtempSync(join(tmpdir(), "bootstrap-documents-")); roots.push(root);
mkdirSync(join(root, "practice"));
writeFileSync(join(root, "thoth-workspaces.yaml"), "schema_version: 1\nworkspaces: [{id: practice, name: Practice}]\n");
writeFileSync(join(root, "practice/workspace.yaml"), "workspace: {schema_version: 4, id: practice, name: Practice, language: en}\n");
return root;
}
const entry = { workspaceId: "practice", engine: "postgres", databaseName: "practice", schema: "public", binding: { transport: "postgres_direct", host: "db.internal", port: 5432, username: "reader" }, secretFiles: { password: "/private/operator/db-password" } };
test("bootstrap reuses Catalog configuration and requires one complete binding per workspace", () => {
const root = fixture();
const valid = validateDatabaseBootstrap({ schemaVersion: 1, databases: [entry] }, root);
expect(valid.secretFiles).toContainEqual({ field: "databases.0.secretFiles.password", path: "/private/operator/db-password" });
for (const databases of [[], [entry, entry], [{ ...entry, workspaceId: "unknown" }], [{ ...entry, secretFiles: {} }], [{ ...entry, engine: "mysql" }], [{ ...entry, binding: { ...entry.binding, port: 70000 } }]]) {
expect(() => validateDatabaseBootstrap({ schemaVersion: 1, databases }, root)).toThrow();
}
});
test("REST authentication, SSH credentials and optional Evidence use their declared contracts", () => {
const root = fixture();
const rest = { ...entry, binding: { transport: "rest_api", baseUrl: "https://data.internal", restPath: "/query", restAuth: "none" }, secretFiles: {} };
expect(validateDatabaseBootstrap({ schemaVersion: 1, databases: [rest] }, root).secretFiles).toEqual([]);
expect(() => validateDatabaseBootstrap({ schemaVersion: 1, databases: [{ ...rest, binding: { ...rest.binding, restAuth: "bearer" } }] }, root)).toThrow();
const ssh = { ...entry, binding: { transport: "ssh_tunnel", username: "reader", sshHost: "bastion", sshPort: 22, sshUsername: "tunnel", sshTargetHost: "database", sshTargetPort: 5432 }, secretFiles: { password: "/password", sshPrivateKey: "/key", sshKnownHosts: "/hosts" } };
expect(validateDatabaseBootstrap({ schemaVersion: 1, databases: [ssh] }, root).warnings[0]).toContain("not NL-to-SQL");
writeFileSync(join(root, "practice/workspace.yaml"), "workspace: {schema_version: 4, id: practice, name: Practice, language: en}\nevidence:\n source:\n type: http\n uris: [https://docs.internal/manual.md]\n authentication: signed_urls_file\n");
expect(() => validateDatabaseBootstrap({ schemaVersion: 1, databases: [entry] }, root)).toThrow();
expect(validateDatabaseBootstrap({ schemaVersion: 1, databases: [{ ...entry, evidenceSecretFiles: { "evidence.signed_urls": "/urls.json" } }] }, root).secretFiles).toContainEqual({ field: "databases.0.evidenceSecretFiles.evidence.signed_urls", path: "/urls.json" });
});
test("bootstrap CLI never echoes arbitrary keys from submitted secret references", () => {
const root = fixture();
const path = join(root, "bootstrap.yaml");
for (const field of ["secretFiles", "evidenceSecretFiles"]) {
writeFileSync(path, JSON.stringify({ schemaVersion: 1, databases: [{ ...entry, [field]: { PRIVATE_CREDENTIAL_SENTINEL: "/path" } }] }));
const result = runBootstrapValidation(["--directory", root, "--bootstrap", path, "--json"]);
expect(result.status).toBe(1);
expect(result.output).not.toContain("PRIVATE_CREDENTIAL_SENTINEL");
}
});
backend/test/installation-documents-cli.test.ts
+67
View File
@@ -0,0 +1,67 @@
import { spawnSync } from "node:child_process";
import { chmodSync, mkdtempSync, readFileSync, realpathSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { rootCertificates } from "node:tls";
import { afterEach, expect, test } from "vitest";
const binary = process.env.THT_INSTALLATION_TEST_CLI;
const roots: string[] = [];
afterEach(() => roots.splice(0).forEach((path) => rmSync(path, { recursive: true, force: true })));
function run(...args: string[]) {
return spawnSync(binary!, [...args, "--json"], { encoding: "utf8", env: { ...process.env, PATH: "" }, input: "" });
}
test.skipIf(!binary)("operator prepares, completes and repeatedly validates before any stack exists", () => {
const root = realpathSync(mkdtempSync(join(tmpdir(), "application-documents-"))); roots.push(root);
const workspace = join(root, "workspaces"), directory = join(root, "installation");
expect(run("workspace", "prepare", "--directory", workspace, "--id", "practice", "--name", "Practice").status).toBe(0);
expect(run("installation", "prepare", "--directory", directory).status).toBe(0);
const installation = join(directory, "thothii-installation.yaml");
const validate = () => run("--installation", installation, "installation", "validate", "--workspaces", workspace);
expect(validate().status).toBe(1); // Visible placeholders cannot be approved.
expect(run("installation", "credentials", "--directory", directory).status).toBe(0);
for (const name of ["thothii-installation.yaml", "operator.env", "database-bootstrap.yaml"]) {
const path = join(directory, name);
writeFileSync(path, readFileSync(path, "utf8").replaceAll("CHANGE_ME", "practice"));
}
for (const [name, contents] of Object.entries({ "secrets.env": "OPENAI_API_KEY=PRIVATE_PROVIDER_VALUE\n", "database-password": "PRIVATE_DATABASE_VALUE", "git-credentials": "", "git-ca.pem": rootCertificates[0] })) {
writeFileSync(join(directory, "secrets", name), contents, { mode: 0o600 });
}
const before = readFileSync(installation, "utf8");
for (let index = 0; index < 2; index++) {
const checked = validate();
expect(checked.stderr).toBe("");
expect(checked.stdout).not.toContain("PRIVATE_");
expect(checked.status, checked.stdout).toBe(0);
expect(JSON.parse(checked.stdout).deferred_checks).toContain("release-assets");
}
expect(readFileSync(installation, "utf8")).toBe(before);
writeFileSync(installation, before.replace("interaction: openai/gpt-4.1-mini", "interaction: openai/nonexistent"));
expect(validate().status).toBe(1);
writeFileSync(installation, before);
const authFile = join(directory, "secrets/pi-auth.json");
writeFileSync(authFile, "not-json");
expect(validate().status).toBe(1);
writeFileSync(installation, before.replace("{mode: secret_env, apiKeyEnv: OPENAI_API_KEY}", "{mode: pi_auth}"));
writeFileSync(authFile, JSON.stringify({ openai: { unrelated: true } }));
expect(validate().status).toBe(1);
writeFileSync(authFile, JSON.stringify({ " OpenAI ": { type: "api_key", key: "PRIVATE_PI_KEY" } }));
expect(validate().status).toBe(1);
writeFileSync(authFile, JSON.stringify({ openai: { type: "api_key", key: "PRIVATE_PI_KEY" } }));
expect(validate().status).toBe(0);
writeFileSync(installation, before);
writeFileSync(authFile, "{}");
const bootstrap = join(directory, "database-bootstrap.yaml");
const originalBootstrap = readFileSync(bootstrap, "utf8");
writeFileSync(join(workspace, "practice", "private-password"), "PRIVATE_DATABASE_VALUE", { mode: 0o600 });
writeFileSync(bootstrap, originalBootstrap.replace(join(directory, "secrets/database-password"), join(workspace, "practice/private-password")));
expect(validate().status).toBe(1);
writeFileSync(bootstrap, originalBootstrap);
chmodSync(join(directory, "secrets/database-password"), 0o644);
expect(validate().status).toBe(1);
chmodSync(join(directory, "secrets/database-password"), 0o600);
const env = join(directory, "operator.env");
writeFileSync(env, readFileSync(env, "utf8") + "THOTH_HTTP_PORT=8081\n");
expect(validate().status).toBe(1);
}, 15_000);