From b9c3369e7b565164b2fb867893cca84de260f446 Mon Sep 17 00:00:00 2001 From: Codex Date: Mon, 28 Sep 2026 16:33:07 +0200 Subject: [PATCH] feat(cli): prepare and validate application documents offline --- PROJECT_STATE.md | 6 +- backend/src/catalog/bootstrap-cli.ts | 21 ++ backend/src/catalog/bootstrap-documents.ts | 47 +++ backend/src/catalog/configuration-schema.ts | 44 +++ backend/src/routes/catalog-databases.ts | 43 +-- backend/src/workspace-documents-cli.ts | 4 +- backend/src/workspaces/documents.ts | 6 +- .../test/database-bootstrap-documents.test.ts | 49 ++++ .../test/installation-documents-cli.test.ts | 67 +++++ docs/install/standalone-manual-en.md | 88 ++++++ docs/install/standalone-manual-it.md | 93 ++++++ ...026-09-28-installation-ticket-breakdown.md | 13 +- tools/tht/README.md | 34 +++ tools/tht/cmd/tht/installation_documents.go | 129 +++++++++ .../cmd/tht/installation_documents_test.go | 96 ++++++ tools/tht/cmd/tht/main.go | 9 + tools/tht/internal/config/installation.go | 23 +- tools/tht/internal/preparation/credentials.go | 73 +++++ tools/tht/internal/preparation/documents.go | 94 ++++++ tools/tht/internal/preparation/validation.go | 274 ++++++++++++++++++ 20 files changed, 1164 insertions(+), 49 deletions(-) create mode 100644 backend/src/catalog/bootstrap-cli.ts create mode 100644 backend/src/catalog/bootstrap-documents.ts create mode 100644 backend/src/catalog/configuration-schema.ts create mode 100644 backend/test/database-bootstrap-documents.test.ts create mode 100644 backend/test/installation-documents-cli.test.ts create mode 100644 tools/tht/cmd/tht/installation_documents.go create mode 100644 tools/tht/cmd/tht/installation_documents_test.go create mode 100644 tools/tht/internal/preparation/credentials.go create mode 100644 tools/tht/internal/preparation/documents.go create mode 100644 tools/tht/internal/preparation/validation.go diff --git a/PROJECT_STATE.md b/PROJECT_STATE.md index cd7bf512..ee8ca7c9 100644 --- a/PROJECT_STATE.md +++ b/PROJECT_STATE.md @@ -9,7 +9,11 @@ and invariants are in [AGENTS.md](AGENTS.md); prior snapshots remain in Git. and `tht workspace validate` through a native two-executable bundle. Build and test instructions are in [the host CLI guide](tools/tht/README.md). Local validation reuses runtime workspace/catalog parsers and explicitly defers runtime Evidence, - database binding and readiness checks. The remainder of the installation tickets, + database binding and readiness checks. Ticket #44 adds `tht installation prepare`, + explicit `installation credentials`, and `installation validate --workspaces PATH` + for protected application documents, canonical model settings and schema-v1 + database bootstrap inputs. These commands do not start services or import Catalog + bindings. The remainder of the installation tickets, Docker Hub publication and example databases remain pending. - React supports full/embedded rendering independently of local/OIDC/upstream auth, diff --git a/backend/src/catalog/bootstrap-cli.ts b/backend/src/catalog/bootstrap-cli.ts new file mode 100644 index 00000000..db4c3564 --- /dev/null +++ b/backend/src/catalog/bootstrap-cli.ts @@ -0,0 +1,21 @@ +import { readFileSync, lstatSync } from "node:fs"; +import { parseAllDocuments } from "yaml"; +import { decode, DocumentError, runWorkspaceDocuments } from "../workspaces/documents.js"; +import { validateDatabaseBootstrap } from "./bootstrap-documents.js"; + +/** Internal sibling protocol: only references cross back to Go, never secret contents. */ +export function runBootstrapValidation(args: string[]): { status: number; output: string } { + try { + if (args.length !== 5 || args[0] !== "--directory" || args[2] !== "--bootstrap" || args[4] !== "--json") throw new Error("usage"); + const checked = runWorkspaceDocuments(["validate", "--directory", args[1], "--json"]); + if (checked.status !== 0) return checked; + const info = lstatSync(args[3]); + if (!info.isFile() || info.isSymbolicLink() || info.size > 1024 * 1024) throw new Error("file"); + const validated = decode(readFileSync(args[3], "utf8"), "database-bootstrap.yaml", (source) => + validateDatabaseBootstrap(parseAllDocuments(source)[0].toJSON(), args[1]), "database bootstrap schema v1 and the Catalog binding contract"); + return { status: 0, output: JSON.stringify({ schema_version: 1, ok: true, secret_files: validated.secretFiles, warnings: validated.warnings, issues: [] }) }; + } catch (error) { + const issue = error instanceof DocumentError ? error.issue : { document: "database-bootstrap.yaml", field: "$", code: "bootstrap_invalid", correction: "Supply one complete Catalog database configuration per workspace in a readable local bootstrap document." }; + return { status: 1, output: JSON.stringify({ schema_version: 1, ok: false, issues: [issue] }) }; + } +} diff --git a/backend/src/catalog/bootstrap-documents.ts b/backend/src/catalog/bootstrap-documents.ts new file mode 100644 index 00000000..58842d3c --- /dev/null +++ b/backend/src/catalog/bootstrap-documents.ts @@ -0,0 +1,47 @@ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { z } from "zod"; +import { databaseConfigurationSchema } from "./configuration-schema.js"; +import { parseWorkspaceCatalogYaml } from "../workspaces/catalog.js"; +import { parseWorkspaceYaml } from "../workspaces/schema.js"; +import { discoverWorkspaceSecretRequirements } from "../workspaces/secret-requirements.js"; + +const reference = z.string().min(1).max(4096); +const database = databaseConfigurationSchema.extend({ + secretFiles: z.object({ password: reference.optional(), apiKey: reference.optional(), sshPrivateKey: reference.optional(), sshPrivateKeyPassphrase: reference.optional(), sshKnownHosts: reference.optional(), tlsCa: reference.optional() }).strict(), + evidenceSecretFiles: z.object({ "evidence.signed_urls": reference.optional(), "evidence.access_key": reference.optional(), "evidence.secret_key": reference.optional(), "evidence.session_token": reference.optional() }).strict().optional(), +}); +const bootstrap = z.object({ schemaVersion: z.literal(1), databases: z.array(database).min(1).max(1000) }).strict(); + +export interface BootstrapReference { field: string; path: string } + +/** Offline bootstrap boundary: runtime Catalog owns the resulting bindings after import. */ +export function validateDatabaseBootstrap(value: unknown, workspaceRoot: string): { secretFiles: BootstrapReference[]; warnings: string[] } { + const document = bootstrap.parse(value); + const catalog = parseWorkspaceCatalogYaml(readFileSync(join(workspaceRoot, "thoth-workspaces.yaml"), "utf8")); + const expected = new Set(catalog.workspaces.map((entry) => entry.id)); + const seen = new Set(); + const secretFiles: BootstrapReference[] = []; + const warnings: string[] = []; + const issue = (path: (string | number)[], message: string): never => { throw new z.ZodError([{ code: "custom", path, message }]); }; + document.databases.forEach((entry, index) => { + if (!expected.has(entry.workspaceId) || seen.has(entry.workspaceId)) issue(["databases", index, "workspaceId"], "Declare each catalog workspace exactly once."); + seen.add(entry.workspaceId); + const required = entry.binding.transport === "rest_api" + ? entry.binding.restAuth === "none" ? [] : ["apiKey"] as const + : entry.binding.transport === "ssh_tunnel" ? ["password", "sshPrivateKey", "sshKnownHosts"] as const : ["password"] as const; + for (const name of required) { + if (!entry.secretFiles[name]) issue(["databases", index, "secretFiles", name], "Supply a protected file reference for this transport."); + } + if (entry.binding.transport === "ssh_tunnel") warnings.push(`databases.${index}:ssh_tunnel supports Catalog diagnostics, not NL-to-SQL sessions; choose direct or REST for practice.`); + for (const [name, path] of Object.entries(entry.secretFiles)) secretFiles.push({ field: `databases.${index}.secretFiles.${name}`, path }); + const workspace = parseWorkspaceYaml(readFileSync(join(workspaceRoot, entry.workspaceId, "workspace.yaml"), "utf8")); + const requirements = discoverWorkspaceSecretRequirements(workspace, {}); + for (const requirement of requirements.filter((item) => item.connector === "evidence" && item.required)) { + if (!(entry.evidenceSecretFiles as Record | undefined)?.[requirement.id]) issue(["databases", index, "evidenceSecretFiles"], "Supply the configured Evidence authentication file references."); + } + for (const [name, path] of Object.entries(entry.evidenceSecretFiles ?? {})) secretFiles.push({ field: `databases.${index}.evidenceSecretFiles.${name}`, path }); + }); + if (seen.size !== expected.size) issue(["databases"], "Add a database binding for every catalog workspace."); + return { secretFiles, warnings }; +} diff --git a/backend/src/catalog/configuration-schema.ts b/backend/src/catalog/configuration-schema.ts new file mode 100644 index 00000000..ac63dd49 --- /dev/null +++ b/backend/src/catalog/configuration-schema.ts @@ -0,0 +1,44 @@ +import { z } from "zod"; +import { parseCredentialFreeHttpUrl } from "../auth/url-policy.js"; +import { DATABASE_TRANSPORTS } from "./types.js"; + +const workspaceIdSchema = z.string().regex(/^[a-z][a-z0-9-]{2,62}$/); +const identifier = z.string().trim().min(1).max(128).regex(/^[A-Za-z_][A-Za-z0-9_$-]*$/); +const nonEmpty = z.string().trim().min(1).max(512); +const port = z.number().int().min(1).max(65_535); +const optionalText = nonEmpty.optional(); +const sshHost = z.string().trim().min(1).max(255).regex(/^[A-Za-z0-9_.:\[\]-]+$/).optional(); +const sshUsername = z.string().trim().min(1).max(128).regex(/^[A-Za-z0-9._-]+$/).optional(); +const bindingSchema = z.object({ + transport: z.enum(DATABASE_TRANSPORTS), + host: optionalText, + port: port.optional(), + username: optionalText, + baseUrl: z.string().max(2048) + .refine((value) => parseCredentialFreeHttpUrl(value) !== undefined) + .optional(), + restPath: z.string().regex(/^\/(?!\/)[^?#\\\u0000-\u001f]*$/).max(512).optional(), + restAuth: z.enum(["none", "bearer", "x-api-key"]).optional(), + tlsServername: optionalText, + sshHost, + sshPort: port.optional(), + sshUsername, + sshTargetHost: sshHost, + sshTargetPort: port.optional(), +}).strict().superRefine((binding, context) => { + const required = binding.transport === "postgres_direct" + ? ["host", "port", "username"] as const + : binding.transport === "rest_api" + ? ["baseUrl", "restPath", "restAuth"] as const + : ["username", "sshHost", "sshPort", "sshUsername", "sshTargetHost", "sshTargetPort"] as const; + for (const field of required) { + if (binding[field] === undefined) context.addIssue({ code: "custom", path: [field], message: "Required" }); + } +}); +export const databaseConfigurationSchema = z.object({ + workspaceId: workspaceIdSchema, + engine: z.literal("postgres"), + databaseName: identifier, + schema: identifier, + binding: bindingSchema, +}).strict(); diff --git a/backend/src/routes/catalog-databases.ts b/backend/src/routes/catalog-databases.ts index b00002a6..a34f2674 100644 --- a/backend/src/routes/catalog-databases.ts +++ b/backend/src/routes/catalog-databases.ts @@ -1,60 +1,19 @@ import type { FastifyInstance, FastifyReply, FastifyRequest } from "fastify"; import { z } from "zod"; import { isPrincipalContext, requirePermission } from "../auth/authorization.js"; -import { parseCredentialFreeHttpUrl } from "../auth/url-policy.js"; +import { databaseConfigurationSchema as configSchema } from "../catalog/configuration-schema.js"; import { CatalogService, type CatalogSecretName } from "../catalog/service.js"; import { WorkspaceRegistryError } from "../workspaces/git-repository.js"; import { CatalogConflictError, CatalogOperationInProgressError, CatalogUnavailableError, - DATABASE_TRANSPORTS, type CatalogRepository, type DatabaseConfigurationInput, } from "../catalog/types.js"; import type { CatalogOperationCoordinator } from "../catalog/operation-coordinator.js"; const idSchema = z.uuid(); -const workspaceIdSchema = z.string().regex(/^[a-z][a-z0-9-]{2,62}$/); -const identifier = z.string().trim().min(1).max(128).regex(/^[A-Za-z_][A-Za-z0-9_$-]*$/); -const nonEmpty = z.string().trim().min(1).max(512); -const port = z.number().int().min(1).max(65_535); -const optionalText = nonEmpty.optional(); -const sshHost = z.string().trim().min(1).max(255).regex(/^[A-Za-z0-9_.:\[\]-]+$/).optional(); -const sshUsername = z.string().trim().min(1).max(128).regex(/^[A-Za-z0-9._-]+$/).optional(); -const bindingSchema = z.object({ - transport: z.enum(DATABASE_TRANSPORTS), - host: optionalText, - port: port.optional(), - username: optionalText, - baseUrl: z.string().max(2048) - .refine((value) => parseCredentialFreeHttpUrl(value) !== undefined) - .optional(), - restPath: z.string().regex(/^\/(?!\/)[^?#\\\u0000-\u001f]*$/).max(512).optional(), - restAuth: z.enum(["none", "bearer", "x-api-key"]).optional(), - tlsServername: optionalText, - sshHost, - sshPort: port.optional(), - sshUsername, - sshTargetHost: sshHost, - sshTargetPort: port.optional(), -}).strict().superRefine((binding, context) => { - const required = binding.transport === "postgres_direct" - ? ["host", "port", "username"] as const - : binding.transport === "rest_api" - ? ["baseUrl", "restPath", "restAuth"] as const - : ["username", "sshHost", "sshPort", "sshUsername", "sshTargetHost", "sshTargetPort"] as const; - for (const field of required) { - if (binding[field] === undefined) context.addIssue({ code: "custom", path: [field], message: "Required" }); - } -}); -const configSchema = z.object({ - workspaceId: workspaceIdSchema, - engine: z.literal("postgres"), - databaseName: identifier, - schema: identifier, - binding: bindingSchema, -}).strict(); const updateSchema = configSchema.extend({ version: z.number().int().positive() }); const secretNames = [ "password", diff --git a/backend/src/workspace-documents-cli.ts b/backend/src/workspace-documents-cli.ts index 88362155..bbf4a468 100644 --- a/backend/src/workspace-documents-cli.ts +++ b/backend/src/workspace-documents-cli.ts @@ -1,6 +1,8 @@ /** Compiled with its runtime for the host CLI: no installation, Docker or host Node required. */ import { runWorkspaceDocuments } from "./workspaces/documents.js"; +import { runBootstrapValidation } from "./catalog/bootstrap-cli.js"; -const result = runWorkspaceDocuments(process.argv.slice(2)); +const args = process.argv.slice(2); +const result = args[0] === "bootstrap" ? runBootstrapValidation(args.slice(1)) : runWorkspaceDocuments(args); console.log(result.output); process.exitCode = result.status; diff --git a/backend/src/workspaces/documents.ts b/backend/src/workspaces/documents.ts index 1e4fb92c..f51dd676 100644 --- a/backend/src/workspaces/documents.ts +++ b/backend/src/workspaces/documents.ts @@ -17,7 +17,7 @@ interface Report { const MAX_DOCUMENT_BYTES = 1024 * 1024; const usage = "tht workspace prepare --directory NEW_PATH --id ID --name NAME [--language en|it] [--json]\ntht workspace validate --directory PATH [--json]"; -class DocumentError extends Error { +export class DocumentError extends Error { constructor(readonly issue: Issue) { super(issue.correction); } } function fail(document: string, field: string, code: string, correction: string): never { @@ -25,7 +25,7 @@ function fail(document: string, field: string, code: string, correction: string) } /** Never include parser messages or submitted values: YAML and Zod errors can contain secrets. */ -function decode(source: string, document: string, parser: (text: string) => T): T { +export function decode(source: string, document: string, parser: (text: string) => T, contract = "workspace schema v4 or catalog schema v1"): T { try { const documents = parseAllDocuments(source, { uniqueKeys: true }); if (documents.length !== 1) fail(document, "$", "yaml_documents", "Keep exactly one YAML document in this file."); @@ -43,7 +43,7 @@ function decode(source: string, document: string, parser: (text: string) => T // Strict schemas produce paths containing schema-defined keys and array indices only. fail(document, issue.path.join(".") || "$", "schema_invalid", issue.code === "unrecognized_keys" ? "Remove fields not defined by the current workspace/catalog contract." - : "Correct this field using workspace schema v4 or catalog schema v1; check type, required value, uniqueness and allowed values."); + : `Correct this field using ${contract}; check type, required value, uniqueness and allowed values.`); } fail(document, "$", "schema_invalid", "Use an authored workspace v4 descriptor; remove database configuration and keep it in the PostgreSQL Metadata Catalog."); } diff --git a/backend/test/database-bootstrap-documents.test.ts b/backend/test/database-bootstrap-documents.test.ts new file mode 100644 index 00000000..f5b840d4 --- /dev/null +++ b/backend/test/database-bootstrap-documents.test.ts @@ -0,0 +1,49 @@ +import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { afterEach, expect, test } from "vitest"; +import { validateDatabaseBootstrap } from "../src/catalog/bootstrap-documents.js"; +import { runBootstrapValidation } from "../src/catalog/bootstrap-cli.js"; + +const roots: string[] = []; +afterEach(() => roots.splice(0).forEach((root) => rmSync(root, { recursive: true, force: true }))); +function fixture() { + const root = mkdtempSync(join(tmpdir(), "bootstrap-documents-")); roots.push(root); + mkdirSync(join(root, "practice")); + writeFileSync(join(root, "thoth-workspaces.yaml"), "schema_version: 1\nworkspaces: [{id: practice, name: Practice}]\n"); + writeFileSync(join(root, "practice/workspace.yaml"), "workspace: {schema_version: 4, id: practice, name: Practice, language: en}\n"); + return root; +} +const entry = { workspaceId: "practice", engine: "postgres", databaseName: "practice", schema: "public", binding: { transport: "postgres_direct", host: "db.internal", port: 5432, username: "reader" }, secretFiles: { password: "/private/operator/db-password" } }; + +test("bootstrap reuses Catalog configuration and requires one complete binding per workspace", () => { + const root = fixture(); + const valid = validateDatabaseBootstrap({ schemaVersion: 1, databases: [entry] }, root); + expect(valid.secretFiles).toContainEqual({ field: "databases.0.secretFiles.password", path: "/private/operator/db-password" }); + for (const databases of [[], [entry, entry], [{ ...entry, workspaceId: "unknown" }], [{ ...entry, secretFiles: {} }], [{ ...entry, engine: "mysql" }], [{ ...entry, binding: { ...entry.binding, port: 70000 } }]]) { + expect(() => validateDatabaseBootstrap({ schemaVersion: 1, databases }, root)).toThrow(); + } +}); + +test("REST authentication, SSH credentials and optional Evidence use their declared contracts", () => { + const root = fixture(); + const rest = { ...entry, binding: { transport: "rest_api", baseUrl: "https://data.internal", restPath: "/query", restAuth: "none" }, secretFiles: {} }; + expect(validateDatabaseBootstrap({ schemaVersion: 1, databases: [rest] }, root).secretFiles).toEqual([]); + expect(() => validateDatabaseBootstrap({ schemaVersion: 1, databases: [{ ...rest, binding: { ...rest.binding, restAuth: "bearer" } }] }, root)).toThrow(); + const ssh = { ...entry, binding: { transport: "ssh_tunnel", username: "reader", sshHost: "bastion", sshPort: 22, sshUsername: "tunnel", sshTargetHost: "database", sshTargetPort: 5432 }, secretFiles: { password: "/password", sshPrivateKey: "/key", sshKnownHosts: "/hosts" } }; + expect(validateDatabaseBootstrap({ schemaVersion: 1, databases: [ssh] }, root).warnings[0]).toContain("not NL-to-SQL"); + writeFileSync(join(root, "practice/workspace.yaml"), "workspace: {schema_version: 4, id: practice, name: Practice, language: en}\nevidence:\n source:\n type: http\n uris: [https://docs.internal/manual.md]\n authentication: signed_urls_file\n"); + expect(() => validateDatabaseBootstrap({ schemaVersion: 1, databases: [entry] }, root)).toThrow(); + expect(validateDatabaseBootstrap({ schemaVersion: 1, databases: [{ ...entry, evidenceSecretFiles: { "evidence.signed_urls": "/urls.json" } }] }, root).secretFiles).toContainEqual({ field: "databases.0.evidenceSecretFiles.evidence.signed_urls", path: "/urls.json" }); +}); + +test("bootstrap CLI never echoes arbitrary keys from submitted secret references", () => { + const root = fixture(); + const path = join(root, "bootstrap.yaml"); + for (const field of ["secretFiles", "evidenceSecretFiles"]) { + writeFileSync(path, JSON.stringify({ schemaVersion: 1, databases: [{ ...entry, [field]: { PRIVATE_CREDENTIAL_SENTINEL: "/path" } }] })); + const result = runBootstrapValidation(["--directory", root, "--bootstrap", path, "--json"]); + expect(result.status).toBe(1); + expect(result.output).not.toContain("PRIVATE_CREDENTIAL_SENTINEL"); + } +}); diff --git a/backend/test/installation-documents-cli.test.ts b/backend/test/installation-documents-cli.test.ts new file mode 100644 index 00000000..a85845d6 --- /dev/null +++ b/backend/test/installation-documents-cli.test.ts @@ -0,0 +1,67 @@ +import { spawnSync } from "node:child_process"; +import { chmodSync, mkdtempSync, readFileSync, realpathSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { rootCertificates } from "node:tls"; +import { afterEach, expect, test } from "vitest"; + +const binary = process.env.THT_INSTALLATION_TEST_CLI; +const roots: string[] = []; +afterEach(() => roots.splice(0).forEach((path) => rmSync(path, { recursive: true, force: true }))); +function run(...args: string[]) { + return spawnSync(binary!, [...args, "--json"], { encoding: "utf8", env: { ...process.env, PATH: "" }, input: "" }); +} + +test.skipIf(!binary)("operator prepares, completes and repeatedly validates before any stack exists", () => { + const root = realpathSync(mkdtempSync(join(tmpdir(), "application-documents-"))); roots.push(root); + const workspace = join(root, "workspaces"), directory = join(root, "installation"); + expect(run("workspace", "prepare", "--directory", workspace, "--id", "practice", "--name", "Practice").status).toBe(0); + expect(run("installation", "prepare", "--directory", directory).status).toBe(0); + const installation = join(directory, "thothii-installation.yaml"); + const validate = () => run("--installation", installation, "installation", "validate", "--workspaces", workspace); + expect(validate().status).toBe(1); // Visible placeholders cannot be approved. + expect(run("installation", "credentials", "--directory", directory).status).toBe(0); + for (const name of ["thothii-installation.yaml", "operator.env", "database-bootstrap.yaml"]) { + const path = join(directory, name); + writeFileSync(path, readFileSync(path, "utf8").replaceAll("CHANGE_ME", "practice")); + } + for (const [name, contents] of Object.entries({ "secrets.env": "OPENAI_API_KEY=PRIVATE_PROVIDER_VALUE\n", "database-password": "PRIVATE_DATABASE_VALUE", "git-credentials": "", "git-ca.pem": rootCertificates[0] })) { + writeFileSync(join(directory, "secrets", name), contents, { mode: 0o600 }); + } + const before = readFileSync(installation, "utf8"); + for (let index = 0; index < 2; index++) { + const checked = validate(); + expect(checked.stderr).toBe(""); + expect(checked.stdout).not.toContain("PRIVATE_"); + expect(checked.status, checked.stdout).toBe(0); + expect(JSON.parse(checked.stdout).deferred_checks).toContain("release-assets"); + } + expect(readFileSync(installation, "utf8")).toBe(before); + writeFileSync(installation, before.replace("interaction: openai/gpt-4.1-mini", "interaction: openai/nonexistent")); + expect(validate().status).toBe(1); + writeFileSync(installation, before); + const authFile = join(directory, "secrets/pi-auth.json"); + writeFileSync(authFile, "not-json"); + expect(validate().status).toBe(1); + writeFileSync(installation, before.replace("{mode: secret_env, apiKeyEnv: OPENAI_API_KEY}", "{mode: pi_auth}")); + writeFileSync(authFile, JSON.stringify({ openai: { unrelated: true } })); + expect(validate().status).toBe(1); + writeFileSync(authFile, JSON.stringify({ " OpenAI ": { type: "api_key", key: "PRIVATE_PI_KEY" } })); + expect(validate().status).toBe(1); + writeFileSync(authFile, JSON.stringify({ openai: { type: "api_key", key: "PRIVATE_PI_KEY" } })); + expect(validate().status).toBe(0); + writeFileSync(installation, before); + writeFileSync(authFile, "{}"); + const bootstrap = join(directory, "database-bootstrap.yaml"); + const originalBootstrap = readFileSync(bootstrap, "utf8"); + writeFileSync(join(workspace, "practice", "private-password"), "PRIVATE_DATABASE_VALUE", { mode: 0o600 }); + writeFileSync(bootstrap, originalBootstrap.replace(join(directory, "secrets/database-password"), join(workspace, "practice/private-password"))); + expect(validate().status).toBe(1); + writeFileSync(bootstrap, originalBootstrap); + chmodSync(join(directory, "secrets/database-password"), 0o644); + expect(validate().status).toBe(1); + chmodSync(join(directory, "secrets/database-password"), 0o600); + const env = join(directory, "operator.env"); + writeFileSync(env, readFileSync(env, "utf8") + "THOTH_HTTP_PORT=8081\n"); + expect(validate().status).toBe(1); +}, 15_000); diff --git a/docs/install/standalone-manual-en.md b/docs/install/standalone-manual-en.md index e8f6d412..c11f1b12 100644 --- a/docs/install/standalone-manual-en.md +++ b/docs/install/standalone-manual-en.md @@ -64,6 +64,94 @@ success does not certify semantic truth, connectivity or readiness. The Git revi activated later must contain the checked documents; this command does not publish uncommitted files or empty directories. +## Prepare and validate application documents + +After validating workspaces, create a local directory **outside their repository**: + +```sh +tht installation prepare --directory ./my-installation +``` + +This creates private, commented `thothii-installation.yaml`, `operator.env`, +`database-bootstrap.yaml` and `README.md`. The destination must be new and its parent +must exist. It starts no services and does not implicitly generate passwords. + +1. Choose models and providers in the descriptor. The template proposes + `openai/gpt-4.1-mini` for interaction and `ollama/qwen3-embedding:0.6b` with 1024 + dimensions for embedding. Edit these before setup. `modelCatalog.defaults.interaction` + must support sessions and metadata generation when the latter is configured. + The template omits optional metadata generation. See + [model configuration](../general/pi-configuration.md) for custom providers. +2. Replace the Git remote in both the descriptor and `operator.env`; keep branch and + transport consistent. Paths are absolute and machine-local. `operator.env` accepts + one literal `KEY=value` assignment per line, without duplicate keys or shell + interpolation. Credentials belong in referenced protected files. +3. Complete `database-bootstrap.yaml` with exactly one entry per workspace. A complete + direct-connection example is: + + ```yaml + schemaVersion: 1 + databases: + - workspaceId: practice + engine: postgres + databaseName: sales + schema: public + binding: + transport: postgres_direct + host: db.intranet + port: 5432 + username: thoth_reader + secretFiles: + password: /private/path/my-installation/secrets/database-password + ``` + + Use a read-only DWH account. `rest_api` requires `baseUrl`, `restPath`, `restAuth` + (`none`, `bearer`, `x-api-key`) and `secretFiles.apiKey` when authenticated. + `ssh_tunnel` requires `username`, `sshHost`, `sshPort`, `sshUsername`, + `sshTargetHost`, `sshTargetPort` and files `password`, `sshPrivateKey`, + `sshKnownHosts`; it supports Catalog diagnostics, not NL-to-SQL sessions. + `tlsCa` and `sshPrivateKeyPassphrase` are optional. Signed HTTP Evidence needs + `evidenceSecretFiles` with key `evidence.signed_urls`; static S3 credentials need + `evidence.access_key`, `evidence.secret_key` and optional `evidence.session_token`. + All values are private file paths. Workspace descriptors remain schema v4; + this bootstrap input is not a second runtime Catalog. +4. Explicitly generate technical credentials in the standard layout: + + ```sh + tht installation credentials --directory ./my-installation + ``` + + This creates separate random Catalog runtime/migrator and administrator passwords, + `auth/auth.yaml`, `auth/users.yaml`, a `secrets/secrets.env` template and + `secrets/pi-auth.json`. Existing files are retained; invalid ones stop the command. + The initial administrator is `admin`; its password stays in private + `secrets/admin-password` and is never printed. The default is local authentication + at `http://localhost:8080`: review and edit `auth/auth.yaml` before validation. + This increment does not validate offline OIDC bootstrap for the existing path. +5. Fill the provider key in `secrets/secrets.env` and create the DWH password file. + For Git HTTPS supply the referenced credentials and CA files; empty credentials + are allowed for a public remote, and the CA file must be available. For SSH supply + a key and known_hosts and select the matching descriptor override. `pi_auth` + providers require prepared Pi credentials. Keep every secret outside workspace + Git with installer-only access (0600 on Unix, equivalent Windows ACLs). +6. Validate and repeat after each correction: + + ```sh + tht --installation /absolute/path/my-installation/thothii-installation.yaml installation validate --workspaces /absolute/path/my-workspaces --json + ``` + + The default bootstrap is beside the descriptor; `--bootstrap PATH` selects another. + Validation changes no documents, generates no projections, uses no network and + writes no database. It rejects placeholders, inconsistencies, missing/non-private + files and secrets inside workspace Git. Reports identify document, field and + correction without secret values. Exit statuses: 0 local success, 1 corrections + needed, 2 invalid arguments. + +Standard release Compose assets may still be absent at this stage; custom overrides +must already exist. Release assets, external connectivity, Catalog import and runtime +readiness remain explicit deferred checks. Success prepares the next preflight; +it neither skips those checks nor establishes a completed installation. + ## Before you start: the two repositories There are two separate repositories: diff --git a/docs/install/standalone-manual-it.md b/docs/install/standalone-manual-it.md index c97fc0c3..6398219f 100644 --- a/docs/install/standalone-manual-it.md +++ b/docs/install/standalone-manual-it.md @@ -65,6 +65,99 @@ Il successo locale non certifica verità semantica, connettività o readiness. L revisione Git attivata in seguito deve contenere i documenti verificati; il comando non pubblica file non committati o directory vuote. +## Predisporre e verificare i documenti applicativi + +Dopo la verifica dei workspace, creare una cartella locale **esterna al loro repository**: + +```sh +tht installation prepare --directory ./mia-installazione +``` + +Il comando crea file privati commentati: `thothii-installation.yaml`, `operator.env`, +`database-bootstrap.yaml` e `README.md`. La cartella deve essere nuova e il padre +deve esistere. Non avvia servizi e non genera implicitamente password. + +1. Nel descrittore, scegliere modelli e provider. Il template propone + `openai/gpt-4.1-mini` per l'interazione e `ollama/qwen3-embedding:0.6b` con 1024 + dimensioni per l'embedding. Sono valori modificabili, non una selezione richiesta + durante il setup. `modelCatalog.defaults.interaction` deve essere utilizzabile + nelle sessioni e anche nella generazione metadati, se quest'ultima è configurata. + Il template omette la generazione metadati, che è facoltativa. Consultare la + [configurazione dei modelli](../general/pi-configuration.md) per provider personalizzati. +2. Sostituire il remoto Git sia nel descrittore sia in `operator.env`; mantenere + coerenti branch e trasporto. I percorsi sono assoluti e riferiti a questa macchina. + `operator.env` accetta una sola assegnazione letterale `KEY=value` per riga, senza + duplicati o interpolazioni shell. Le credenziali restano nei file referenziati. +3. Compilare `database-bootstrap.yaml`: una voce per ciascun workspace, senza + duplicati. Questo esempio mostra il contratto completo di un collegamento diretto: + + ```yaml + schemaVersion: 1 + databases: + - workspaceId: pratica + engine: postgres + databaseName: vendite + schema: public + binding: + transport: postgres_direct + host: db.intranet + port: 5432 + username: thoth_reader + secretFiles: + password: /percorso/privato/mia-installazione/secrets/database-password + ``` + + Usare le credenziali di un utente DWH in sola lettura. Per `rest_api`, il binding + richiede `baseUrl`, `restPath` e `restAuth` (`none`, `bearer`, `x-api-key`); quando + serve autenticazione, aggiungere `secretFiles.apiKey`. `ssh_tunnel` richiede + `username`, `sshHost`, `sshPort`, `sshUsername`, `sshTargetHost`, `sshTargetPort` e + i file `password`, `sshPrivateKey`, `sshKnownHosts`; abilita diagnostica Catalog, + non sessioni NL→SQL. Sono facoltativi `tlsCa` e `sshPrivateKeyPassphrase`. + Le Evidence HTTP firmate richiedono `evidenceSecretFiles` con chiave + `evidence.signed_urls`; S3 con credenziali statiche richiede `evidence.access_key` + e `evidence.secret_key`, con `evidence.session_token` facoltativo. Tutti i valori + sono percorsi di file privati. I workspace rimangono nello schema v4: il bootstrap + è un input iniziale, non un secondo Catalog runtime. +4. Generare esplicitamente le credenziali tecniche nel layout standard: + + ```sh + tht installation credentials --directory ./mia-installazione + ``` + + Sono creati password casuali separate per Catalog runtime/migrator e amministratore, + il relativo `auth/auth.yaml` con `auth/users.yaml`, un template `secrets/secrets.env` + e `secrets/pi-auth.json`. I file esistenti vengono conservati; se non validi, il + comando si ferma. L'amministratore iniziale è `admin`, la password è nel file + privato `secrets/admin-password` e non viene stampata. Il default è autenticazione + locale con URL pubblico `http://localhost:8080`: verificare e, se necessario, + modificare `auth/auth.yaml` prima del controllo. Questo incremento non valida il + bootstrap OIDC del percorso esistente. +5. Inserire la chiave del provider in `secrets/secrets.env` e creare il file della + password DWH. Per HTTPS Git fornire i file referenziati per credenziali e CA: + credenziali vuote sono ammesse per un remoto pubblico, la CA deve essere disponibile. + Per SSH fornire chiave e known_hosts, scegliendo il relativo override nel descrittore. + I provider `pi_auth` richiedono credenziali Pi già preparate. Conservare tutti + questi file fuori dal repository workspace e proteggere l'accesso al solo utente + installatore (0600 su Unix, ACL equivalenti su Windows). Non committarli. +6. Verificare e ripetere dopo ogni correzione: + + ```sh + tht --installation /percorso/assoluto/mia-installazione/thothii-installation.yaml installation validate --workspaces /percorso/assoluto/miei-workspace --json + ``` + + Il bootstrap viene cercato accanto al descrittore; `--bootstrap PERCORSO` ne + seleziona uno diverso. Il controllo non cambia documenti, non genera proiezioni, + non usa la rete e non scrive database. Rifiuta placeholder, incoerenze, file + mancanti/non privati e segreti situati nel repository workspace. Il report indica + documento, campo e correzione senza valori riservati. Exit status: 0 successo + locale, 1 correzioni necessarie, 2 argomenti errati. + +Gli asset Compose standard del rilascio possono ancora mancare in questa fase; +gli override personalizzati devono già esistere. Il report distingue i controlli +differiti: asset del rilascio, connettività esterna, import Catalog e readiness. +Un esito positivo prepara il successivo preflight: non autorizza a saltare tali +controlli e non equivale a un'installazione completata. + ## Prima di iniziare: i due repository Servono due repository distinti: diff --git a/docs/plans/2026-09-28-installation-ticket-breakdown.md b/docs/plans/2026-09-28-installation-ticket-breakdown.md index c20fc25b..1a6a0101 100644 --- a/docs/plans/2026-09-28-installation-ticket-breakdown.md +++ b/docs/plans/2026-09-28-installation-ticket-breakdown.md @@ -24,7 +24,18 @@ saltato, 1.417 test passati e 40 saltati; typecheck e build rigorosa documentazi passati. Suite Go: tutti i pacchetti passati, salvo un primo errore intermittente nel test di concorrenza authstorage; quel test passa in tre ripetizioni e il pacchetto completo passa nella verifica isolata. Le revisioni Standards e Spec non lasciano -finding aperti. Il prossimo incremento sequenziale è T02/#44. +finding aperti. + +T02/#44 implementato sullo stesso branch: `tht installation prepare`, generazione +esplicita delle credenziali tecniche e `tht installation validate` preparano e +controllano documenti privati, modelli, autenticazione e bootstrap dei binding, +riusando gli schemi runtime senza avviare servizi. Guide IT/EN aggiornate. +Verifica backend completa su Node 24.16: 111 file passati, uno saltato, 1.420 test +passati e 40 saltati; le regressioni successive della revisione passano nella suite +mirata (quattro test, incluso il percorso con binari nativi e `PATH` vuoto). +Typecheck, build rigorosa documentazione e pacchetti Go passati; il pacchetto CLI +è stato ripetuto dopo la correzione rilevata in revisione. Nessun finding residuo +delle revisioni Standards/Spec. Il prossimo incremento sequenziale è T03/#45. | Ticket | Issue Gitea | Dipendenze dirette | | --- | --- | --- | diff --git a/tools/tht/README.md b/tools/tht/README.md index 14fecacf..c5bd52a3 100644 --- a/tools/tht/README.md +++ b/tools/tht/README.md @@ -41,6 +41,40 @@ THT_WORKSPACE_TEST_CLI=/absolute/path/dist/workspace-tools/darwin-arm64/tht \ npx vitest run test/workspace-documents-cli.test.ts ``` +## Application document preparation (issue #44) + +Before installation, use `tht installation prepare --directory NEW_PATH`, then +`tht installation credentials --directory PATH` to explicitly create technical +credentials in the default protected layout. The latter preserves existing files. +Edit the documents and external credentials, then repeat: + +```sh +tht --installation /absolute/path/thothii-installation.yaml \ + installation validate --workspaces /absolute/path/workspaces --json +``` + +`database-bootstrap.yaml` beside the descriptor is a schemaVersion-1 bootstrap +input, not a runtime Catalog. Its database entries use the exact Catalog API +configuration schema plus private `secretFiles` and optional `evidenceSecretFiles` +references. `--bootstrap` selects another document. The helper validates workspace +membership, uniqueness, supported transports and required credential references; +Go checks protected files, the canonical Installation Model Catalog, environment +and local administrator. No process contacts a service or creates projections. +Only missing standard release Compose assets are deferred by `config.LoadPrepared`; +normal runtime `config.Load` retains all existing checks. Custom overrides must exist. + +The native integration test is opt-in because it requires the built pair: + +```sh +THT_INSTALLATION_TEST_CLI=/absolute/path/dist/workspace-tools/darwin-arm64/tht \ + npx vitest run test/installation-documents-cli.test.ts +``` + +Run it from `backend/`, following the bundle build above. It supplies prepared +fixtures and removes host tools from the subprocess PATH. Platform acceptance and +real external credentials remain separate gates. See the IT/EN guides for the +complete parameter collection procedure, default `admin` account and local-auth scope. + ## Shell configuration The schema-v2 `thothii-installation.yaml` accepts this optional section: diff --git a/tools/tht/cmd/tht/installation_documents.go b/tools/tht/cmd/tht/installation_documents.go new file mode 100644 index 00000000..fa47163e --- /dev/null +++ b/tools/tht/cmd/tht/installation_documents.go @@ -0,0 +1,129 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "path/filepath" + "slices" + "strings" + + "github.com/aritmolab/thothii/tools/tht/internal/preparation" +) + +func installationDocumentsCommand(ctx context.Context, installationPath string, args []string, stdout io.Writer) int { + report := preparation.NewReport() + status := 0 + options := map[string]string{} + valid := len(args) > 0 + for index := 1; index < len(args); index++ { + key := args[index] + if _, duplicate := options[key]; duplicate { + valid = false + break + } + if key == "--json" { + options[key] = "true" + continue + } + if (key != "--directory" && key != "--workspaces" && key != "--bootstrap") || index+1 == len(args) { + valid = false + break + } + options[key] = args[index+1] + index++ + } + if !valid || (args[0] == "validate" && (installationPath == "" || options["--workspaces"] == "" || options["--directory"] != "")) || (args[0] != "validate" && (options["--directory"] == "" || options["--workspaces"] != "" || options["--bootstrap"] != "")) { + report.Add("CLI", "$", "usage", "Use installation prepare|credentials --directory PATH [--json], or tht --installation ABSOLUTE_PATH installation validate --workspaces PATH [--bootstrap PATH] [--json].") + status = 2 + } else if args[0] == "validate" { + installation, checkedReport := preparation.Validate(installationPath) + report = checkedReport + if report.OK { + bootstrap := options["--bootstrap"] + if bootstrap == "" { + bootstrap = filepath.Join(filepath.Dir(installationPath), "database-bootstrap.yaml") + } + bootstrap, _ = filepath.Abs(bootstrap) + workspace, _ := filepath.Abs(options["--workspaces"]) + if resolved, err := filepath.EvalSymlinks(workspace); err == nil { + workspace = resolved + } + outsideWorkspace := func(path, document, field string) bool { + relative, err := filepath.Rel(workspace, path) + if err == nil && relative != ".." && !strings.HasPrefix(relative, ".."+string(filepath.Separator)) { + report.Add(document, field, "installation_file_in_workspace", "Keep installation documents and credential files outside the shared workspace repository.") + return false + } + return true + } + for _, path := range []string{installationPath, installation.EnvFile, installation.AuthenticationDirectory(), bootstrap} { + outsideWorkspace(path, "thothii-installation.yaml", "local-files") + } + files, _ := installation.SecretFiles() + for _, path := range files { + outsideWorkspace(path, "operator.env", "protected-file-reference") + } + if preparation.CheckYAML(bootstrap, "database-bootstrap.yaml", &report) { + var output, discarded bytes.Buffer + code := workspaceDocumentsCommand(ctx, []string{"bootstrap", "--directory", workspace, "--bootstrap", bootstrap, "--json"}, &output, &discarded) + var checked struct { + OK bool `json:"ok"` + Issues []preparation.Issue `json:"issues"` + Warnings []string `json:"warnings"` + SecretFiles []struct { + Field string `json:"field"` + Path string `json:"path"` + } `json:"secret_files"` + } + if json.Unmarshal(output.Bytes(), &checked) != nil { + report.Add("CLI", "$", "validator_unavailable", "Reinstall the matching tht and workspace helper pair.") + } else if code != 0 || !checked.OK { + report.Issues = append(report.Issues, checked.Issues...) + if len(checked.Issues) == 0 { + report.Add("database-bootstrap.yaml", "$", "bootstrap_invalid", "Correct workspace and binding documents, then validate again.") + } + } else { + report.Warnings = append(report.Warnings, checked.Warnings...) + for _, file := range checked.SecretFiles { + if outsideWorkspace(file.Path, "database-bootstrap.yaml", file.Field) { + preparation.CheckSecret(file.Path, "database-bootstrap.yaml", file.Field, false, &report) + } + } + } + } + } + } else { + directory, err := filepath.Abs(options["--directory"]) + if err == nil { + if args[0] == "prepare" { + err = preparation.Prepare(directory) + } else { + err = preparation.Credentials(ctx, directory) + } + } + if err != nil { + report.Add("preparation", "$", "preparation_refused", err.Error()) + } + } + report.OK = len(report.Issues) == 0 + if !report.OK && status == 0 { + status = 1 + } + if slices.Contains(args, "--json") { + _ = json.NewEncoder(stdout).Encode(report) + } else { + if report.OK { + fmt.Fprintln(stdout, "Document operation completed. Local validation does not establish runtime readiness.") + } + for _, issue := range report.Issues { + fmt.Fprintf(stdout, "%s [%s] %s: %s\n", issue.Document, issue.Field, issue.Code, issue.Correction) + } + for _, warning := range report.Warnings { + fmt.Fprintln(stdout, warning) + } + } + return status +} diff --git a/tools/tht/cmd/tht/installation_documents_test.go b/tools/tht/cmd/tht/installation_documents_test.go new file mode 100644 index 00000000..abb91040 --- /dev/null +++ b/tools/tht/cmd/tht/installation_documents_test.go @@ -0,0 +1,96 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "os" + "path/filepath" + "runtime" + "testing" +) + +func TestInstallationPrepareDocumentsBeforeRuntime(t *testing.T) { + t.Setenv("PATH", "") + root, err := filepath.EvalSymlinks(t.TempDir()) + if err != nil { + t.Fatal(err) + } + destination := filepath.Join(root, "installation") + var stdout, stderr bytes.Buffer + status := run(context.Background(), []string{"installation", "prepare", "--directory", destination, "--json"}, &stdout, &stderr) + if status != 0 { + t.Fatalf("status=%d stdout=%s stderr=%s", status, &stdout, &stderr) + } + var report map[string]any + if json.Unmarshal(stdout.Bytes(), &report) != nil || report["ok"] != true { + t.Fatalf("report=%s", &stdout) + } + for _, name := range []string{"thothii-installation.yaml", "operator.env", "database-bootstrap.yaml", "README.md"} { + if _, err := os.Stat(filepath.Join(destination, name)); err != nil { + t.Fatal(err) + } + } + before, _ := os.ReadFile(filepath.Join(destination, "thothii-installation.yaml")) + stdout.Reset() + stderr.Reset() + if run(context.Background(), []string{"installation", "prepare", "--directory", destination, "--json"}, &stdout, &stderr) == 0 { + t.Fatal("overwrote existing documents") + } + after, _ := os.ReadFile(filepath.Join(destination, "thothii-installation.yaml")) + if !bytes.Equal(before, after) { + t.Fatal("existing document changed") + } +} + +func TestInstallationCredentialsAreExplicitPrivateAndNeverReplaced(t *testing.T) { + t.Setenv("PATH", "") + root, _ := filepath.EvalSymlinks(t.TempDir()) + destination := filepath.Join(root, "installation") + var stdout, stderr bytes.Buffer + if run(context.Background(), []string{"installation", "prepare", "--directory", destination}, &stdout, &stderr) != 0 { + t.Fatal(&stdout, &stderr) + } + secret := filepath.Join(destination, "secrets", "catalog-runtime-password") + if _, err := os.Stat(secret); !os.IsNotExist(err) { + t.Fatal("prepare generated a secret implicitly") + } + stdout.Reset() + stderr.Reset() + if run(context.Background(), []string{"installation", "credentials", "--directory", destination, "--json"}, &stdout, &stderr) != 0 { + t.Fatal(&stdout, &stderr) + } + before, err := os.ReadFile(secret) + if err != nil || len(bytes.TrimSpace(before)) < 32 { + t.Fatal("missing strong technical credential", err) + } + if bytes.Contains(stdout.Bytes(), bytes.TrimSpace(before)) || bytes.Contains(stderr.Bytes(), bytes.TrimSpace(before)) { + t.Fatal("secret leaked") + } + info, _ := os.Stat(secret) + if runtime.GOOS != "windows" && info.Mode().Perm()&0o077 != 0 { + t.Fatal("credential is not private") + } + stdout.Reset() + stderr.Reset() + if run(context.Background(), []string{"installation", "credentials", "--directory", destination, "--json"}, &stdout, &stderr) != 0 { + t.Fatal(&stdout, &stderr) + } + after, _ := os.ReadFile(secret) + if !bytes.Equal(before, after) { + t.Fatal("credential was replaced") + } +} + +func TestInstallationValidateRejectsWrongEnvironmentFieldType(t *testing.T) { + root, _ := filepath.EvalSymlinks(t.TempDir()) + path := filepath.Join(root, "thothii-installation.yaml") + if err := os.WriteFile(path, []byte("schemaVersion: 2\nenvFile: []\n"), 0o600); err != nil { + t.Fatal(err) + } + var stdout, stderr bytes.Buffer + status := run(context.Background(), []string{"--installation", path, "installation", "validate", "--workspaces", root, "--json"}, &stdout, &stderr) + if status != 1 { + t.Fatalf("invalid field accepted: %d %s", status, &stdout) + } +} diff --git a/tools/tht/cmd/tht/main.go b/tools/tht/cmd/tht/main.go index ade2d40a..090f7d6b 100644 --- a/tools/tht/cmd/tht/main.go +++ b/tools/tht/cmd/tht/main.go @@ -43,6 +43,12 @@ Commands: setup [--complete|--configure-only] [--installation-id ID] [--profile local|server] [--shell-mode full|embedded] [--shell-default-locale BCP47-TAG] [--shell-adapter omics-portal] Create, validate, and optionally complete the local installation. + installation prepare --directory NEW_PATH [--json] + Create commented installation, environment and database bootstrap templates. + installation credentials --directory PATH [--json] + Explicitly generate protected technical credentials before setup. + installation validate --workspaces PATH [--bootstrap PATH] [--json] + Check prepared application documents; requires --installation. installation migrate --output PATH --session-default PROVIDER/MODEL --embedding-id PROVIDER/MODEL --embedding-dimensions N Create a review-only schema-v2 candidate from all three legacy model sources. @@ -138,6 +144,9 @@ func run(ctx context.Context, args []string, stdout, stderr io.Writer) int { return versionCommand(commandArgs, stdout, stderr) } if command == "installation" { + if len(commandArgs) > 0 && (commandArgs[0] == "prepare" || commandArgs[0] == "credentials" || commandArgs[0] == "validate") { + return installationDocumentsCommand(ctx, installationPath, commandArgs, stdout) + } if len(commandArgs) > 0 && commandArgs[0] == "generate" { return installationGenerationCommand(installationPath, commandArgs[1:], stdout, stderr) } diff --git a/tools/tht/internal/config/installation.go b/tools/tht/internal/config/installation.go index a942861a..d7eabd47 100644 --- a/tools/tht/internal/config/installation.go +++ b/tools/tht/internal/config/installation.go @@ -98,6 +98,16 @@ type Installation struct { // Load reads and validates an installation descriptor at an absolute path. func Load(path string) (Installation, error) { + return load(path, false) +} + +// LoadPrepared applies the same authored configuration rules before release assets exist. +// Only standard distribution-owned Compose assets are deferred, never custom overrides. +func LoadPrepared(path string) (Installation, error) { + return load(path, true) +} + +func load(path string, prepared bool) (Installation, error) { if !filepath.IsAbs(path) { return Installation{}, fmt.Errorf("installation path must be absolute") } @@ -143,6 +153,11 @@ func Load(path string) (Installation, error) { if err := requireDirectory(raw.ProjectDirectory, "projectDirectory"); err != nil { return Installation{}, err } + if prepared { + if err := requireCanonicalDirectory(raw.ProjectDirectory); err != nil { + return Installation{}, errors.New("projectDirectory must be canonical and accessible") + } + } for _, legacyProjection := range []string{ filepath.Join(raw.ProjectDirectory, "deploy", "pi", "models.json"), filepath.Join(raw.ProjectDirectory, "deploy", "pi", "settings.json"), @@ -195,7 +210,8 @@ func Load(path string) (Installation, error) { return Installation{}, errors.New("authentication.configDirectory must match THT_AUTH_CONFIG_ROOT") } for _, override := range raw.Overrides { - if err := requireRegularFile(override, "override"); err != nil { + standardAsset := override == filepath.Join(installation.ProjectDirectory, "deploy", "compose.git-https.yaml") || override == filepath.Join(installation.ProjectDirectory, "deploy", "compose.git-ssh.yaml") + if err := requireRegularFile(override, "override"); err != nil && !(prepared && standardAsset && errors.Is(statPathError(override), os.ErrNotExist)) { return Installation{}, err } installation.Overrides = append(installation.Overrides, filepath.Clean(override)) @@ -209,6 +225,9 @@ func Load(path string) (Installation, error) { } } for _, composeFile := range installation.ComposeFiles()[:2] { + if prepared && errors.Is(statPathError(composeFile), os.ErrNotExist) { + continue + } if err := requireRegularFile(composeFile, "Compose file"); err != nil { return Installation{}, err } @@ -226,6 +245,8 @@ func Load(path string) (Installation, error) { return installation, nil } +func statPathError(path string) error { _, err := os.Lstat(path); return err } + var metadataSecretBundleKeyPattern = regexp.MustCompile(`^[A-Z][A-Z0-9_]{0,127}$`) var metadataAPIKeyEnvironments = map[string]struct{}{ "THT_MODEL_API_KEY": {}, diff --git a/tools/tht/internal/preparation/credentials.go b/tools/tht/internal/preparation/credentials.go new file mode 100644 index 00000000..e1db1e97 --- /dev/null +++ b/tools/tht/internal/preparation/credentials.go @@ -0,0 +1,73 @@ +package preparation + +import ( + "context" + "crypto/rand" + "encoding/hex" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + + "github.com/aritmolab/thothii/tools/tht/internal/authconfig" + "github.com/aritmolab/thothii/tools/tht/internal/config" + "github.com/aritmolab/thothii/tools/tht/internal/safeio" +) + +// Credentials creates installation-owned secrets only. External credentials are supplied by +// the operator. Existing files are checked and retained so interrupted preparation can resume. +func Credentials(ctx context.Context, directory string) error { + if err := safeio.ValidatePrivateDirectory(directory); err != nil { + return fmt.Errorf("use an existing private preparation directory") + } + if _, err := safeio.ReadCanonicalPrivateRegular(filepath.Join(directory, "thothii-installation.yaml"), 1<<20); err != nil { + return fmt.Errorf("prepare installation documents first") + } + secrets := filepath.Join(directory, "secrets") + if err := safeio.EnsurePrivateDirectory(secrets); err != nil { + return fmt.Errorf("secrets directory must be private and operator-owned") + } + for _, name := range []string{"catalog-runtime-password", "catalog-migrator-password", "admin-password"} { + path := filepath.Join(secrets, name) + if _, err := os.Lstat(path); err == nil { + value, err := safeio.ReadCanonicalPrivateRegular(path, 64<<10) + if err != nil || len(strings.TrimSpace(string(value))) < 32 { + return fmt.Errorf("existing technical credentials must be private, readable and at least 32 characters; no file was replaced") + } + continue + } else if !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("cannot inspect technical credentials") + } + value := make([]byte, 32) + if _, err := rand.Read(value); err != nil { + return fmt.Errorf("cannot generate random credentials") + } + if err := safeio.WriteCanonicalNewPrivateFile(path, []byte(hex.EncodeToString(value)+"\n"), 0o600); err != nil { + return fmt.Errorf("cannot create credential; existing files are retained") + } + } + for name, contents := range map[string]string{"secrets.env": "# Supply the provider credential; never commit this file.\nOPENAI_API_KEY=CHANGE_ME\n", "pi-auth.json": "{}\n"} { + path := filepath.Join(secrets, name) + if _, err := os.Lstat(path); errors.Is(err, os.ErrNotExist) { + if err := safeio.WriteCanonicalNewPrivateFile(path, []byte(contents), 0o600); err != nil { + return fmt.Errorf("cannot create external credential template") + } + } else if _, err := safeio.ReadCanonicalPrivateRegular(path, 64<<10); err != nil { + return fmt.Errorf("existing credential template is not private or readable") + } + } + authDirectory := filepath.Join(directory, "auth") + if _, err := os.Lstat(filepath.Join(authDirectory, "auth.yaml")); err == nil { + if _, _, err := authconfig.Load(authDirectory); err != nil { + return fmt.Errorf("existing authentication documents are invalid; repair them explicitly") + } + return nil + } + installation := config.Installation{Authentication: config.Authentication{ConfigDirectory: authDirectory}} + if authconfig.Run(ctx, installation, []string{"configure", "--mode", "local", "--public-url", "http://localhost:8080", "--admin-user", "admin", "--password-file", filepath.Join(secrets, "admin-password")}, strings.NewReader(""), io.Discard, io.Discard) != 0 { + return fmt.Errorf("cannot prepare local administrator; inspect protected auth files and retry without replacing existing credentials") + } + return nil +} diff --git a/tools/tht/internal/preparation/documents.go b/tools/tht/internal/preparation/documents.go new file mode 100644 index 00000000..6a01e6ea --- /dev/null +++ b/tools/tht/internal/preparation/documents.go @@ -0,0 +1,94 @@ +// Package preparation owns installation-local documents before any runtime exists. +package preparation + +import ( + "fmt" + "path/filepath" + "strconv" + "strings" + + "github.com/aritmolab/thothii/tools/tht/internal/safeio" +) + +func Prepare(directory string) error { + if err := safeio.ValidateCanonicalPath(directory); err != nil { + return fmt.Errorf("choose an absolute canonical destination") + } + exists, err := safeio.PreflightPrivateDirectory(directory) + if err != nil || exists { + return fmt.Errorf("choose a new private directory with an existing parent; existing documents are never replaced") + } + if err := safeio.EnsurePrivateDirectory(directory); err != nil { + return fmt.Errorf("cannot create private preparation directory") + } + root := func(name string) string { return strconv.Quote(filepath.Join(directory, name)) } + installation := fmt.Sprintf(`# Installation schema v2; replace CHANGE_ME before validation. +# Paths refer to local preparation files, never to workspace Git. +schemaVersion: 2 +profile: local +projectDirectory: %s +envFile: %s +shell: {mode: full, defaultLocale: en} +workspaceRepository: + remote: https://CHANGE_ME/workspaces.git + branch: main + access: https +authentication: + configDirectory: %s +modelCatalog: + defaults: {interaction: openai/gpt-4.1-mini} + embedding: {id: 'ollama/qwen3-embedding:0.6b', dimensions: 1024} + providers: + openai: + authentication: {mode: secret_env, apiKeyEnv: OPENAI_API_KEY} + session: {mode: pi_builtin} + models: + gpt-4.1-mini: {session: {}} +# Metadata generation is optional: omitted here. Configure its eligibility in this catalog. +# This Compose asset will come from the release; no application checkout is required here. +overrides: [%s] +`, strconv.Quote(directory), root("operator.env"), root("auth"), root("deploy/compose.git-https.yaml")) + environment := "# Non-secret paths and parameters; no interpolation or duplicate keys.\n" + for _, entry := range [][2]string{ + {"COMPOSE_PROJECT_NAME", "thothii-local"}, {"THT_WORKSPACE_INSTALLATION_ID", "local"}, + {"THT_WORKSPACE_GIT_REMOTE", "https://CHANGE_ME/workspaces.git"}, {"THT_WORKSPACE_GIT_BRANCH", "main"}, + {"THT_INSTALLATION_CONFIG_SOURCE", filepath.Join(directory, "thothii-installation.yaml")}, + {"THT_AUTH_CONFIG_ROOT", filepath.Join(directory, "auth")}, + {"THT_SECRETS_FILE", filepath.Join(directory, "secrets", "secrets.env")}, + {"PI_AUTH_FILE", filepath.Join(directory, "secrets", "pi-auth.json")}, + {"THT_WORKSPACE_GIT_CREDENTIALS_FILE", filepath.Join(directory, "secrets", "git-credentials")}, + {"THT_WORKSPACE_GIT_CA_FILE", filepath.Join(directory, "secrets", "git-ca.pem")}, + {"THT_CATALOG_RUNTIME_PASSWORD_SOURCE", filepath.Join(directory, "secrets", "catalog-runtime-password")}, + {"THT_CATALOG_MIGRATOR_PASSWORD_SOURCE", filepath.Join(directory, "secrets", "catalog-migrator-password")}, + {"THOTH_HTTP_PORT", "8080"}, {"THOTH_CORE_HTTP_PORT", "8787"}, {"MAX_PI_PROCESSES", "4"}, + } { + environment += entry[0] + "=" + strconv.Quote(entry[1]) + "\n" + } + bootstrap := fmt.Sprintf(`# Bootstrap input only; PostgreSQL Metadata Catalog remains the runtime authority. +schemaVersion: 1 +databases: + - workspaceId: CHANGE_ME + engine: postgres + databaseName: CHANGE_ME + schema: public + binding: + transport: postgres_direct + host: CHANGE_ME + port: 5432 + username: CHANGE_ME + secretFiles: + password: %s +`, root("secrets/database-password")) + documents := map[string]string{ + ".gitignore": "*\n", + "thothii-installation.yaml": installation, "operator.env": environment, + "database-bootstrap.yaml": bootstrap, + "README.md": "# Preparation / Preparazione\n\nReplace every CHANGE_ME / Sostituire ogni CHANGE_ME. Keep all files outside workspace Git / Tenere tutti i file fuori dal Git dei workspace.\n\n1. Edit installation models, workspace remote and operator.env consistently. / Modificare modelli, remoto e operator.env in modo coerente.\n2. Add one database bootstrap entry for every workspace; use read-only DWH credentials in protected files. / Una voce database per workspace, credenziali DWH in sola lettura in file protetti.\n3. Run tht installation credentials --directory PATH before setup; fill provider/database secrets yourself. / Generare credenziali tecniche prima del setup; compilare i segreti esterni.\n4. Repeat tht --installation PATH/thothii-installation.yaml installation validate --workspaces WORKSPACES. / Correggere e ripetere.\n\nNo services, network calls or database writes / Nessun servizio, chiamata di rete o scrittura database.\n", + } + for _, name := range []string{".gitignore", "thothii-installation.yaml", "operator.env", "database-bootstrap.yaml", "README.md"} { + if err := safeio.WriteCanonicalNewPrivateFile(filepath.Join(directory, name), []byte(strings.TrimSpace(documents[name])+"\n"), 0o600); err != nil { + return fmt.Errorf("cannot create preparation files; inspect the new directory and retry in a new destination") + } + } + return nil +} diff --git a/tools/tht/internal/preparation/validation.go b/tools/tht/internal/preparation/validation.go new file mode 100644 index 00000000..299715ff --- /dev/null +++ b/tools/tht/internal/preparation/validation.go @@ -0,0 +1,274 @@ +package preparation + +import ( + "bytes" + "encoding/json" + "io" + "regexp" + "strconv" + "strings" + + "github.com/aritmolab/thothii/tools/tht/internal/authconfig" + "github.com/aritmolab/thothii/tools/tht/internal/config" + "github.com/aritmolab/thothii/tools/tht/internal/safeio" + "gopkg.in/yaml.v3" +) + +type Issue struct { + Document string `json:"document"` + Field string `json:"field"` + Code string `json:"code"` + Correction string `json:"correction"` +} +type Report struct { + SchemaVersion int `json:"schema_version"` + Scope string `json:"scope"` + OK bool `json:"ok"` + Issues []Issue `json:"issues"` + Warnings []string `json:"warnings"` + DeferredChecks []string `json:"deferred_checks"` +} + +func NewReport() Report { + return Report{SchemaVersion: 1, Scope: "application-documents", Issues: []Issue{}, Warnings: []string{}, DeferredChecks: []string{"release-assets", "external-connectivity", "catalog-import", "runtime-readiness"}} +} +func (r *Report) Add(document, field, code, correction string) { + r.OK = false + r.Issues = append(r.Issues, Issue{document, field, code, correction}) +} +func placeholder(value string) bool { + upper := strings.ToUpper(value) + return strings.Contains(upper, "CHANGE_ME") || strings.Contains(upper, "REPLACE_ME") || strings.Contains(upper, "YOUR_API_KEY") || strings.Contains(upper, "") +} + +// CheckYAML refuses unresolved placeholders and ambiguous authored YAML without returning values. +func CheckYAML(path, document string, report *Report) bool { + content, err := safeio.ReadCanonicalPrivateRegular(path, 1<<20) + if err != nil { + report.Add(document, "$", "file_unavailable", "Use a readable, private regular document at a canonical absolute path (no links).") + return false + } + decoder := yaml.NewDecoder(bytes.NewReader(content)) + var node, extra yaml.Node + if err := decoder.Decode(&node); err != nil || decoder.Decode(&extra) != io.EOF { + report.Add(document, "$", "yaml_invalid", "Keep one well-formed YAML document.") + return false + } + var walk func(*yaml.Node) bool + walk = func(n *yaml.Node) bool { + if n.Kind == yaml.AliasNode || n.Tag == "!!merge" { + return false + } + if n.Kind == yaml.ScalarNode && placeholder(n.Value) { + return false + } + if n.Kind == yaml.MappingNode { + seen := map[string]bool{} + for index := 0; index < len(n.Content); index += 2 { + key := n.Content[index] + if key.Kind != yaml.ScalarNode || seen[key.Value] { + return false + } + seen[key.Value] = true + } + } + for _, child := range n.Content { + if !walk(child) { + return false + } + } + return true + } + if !walk(&node) { + report.Add(document, "$", "incomplete_or_ambiguous", "Replace CHANGE_ME/REPLACE_ME placeholders, remove duplicate keys, aliases and YAML merge keys.") + return false + } + return true +} + +var environmentKey = regexp.MustCompile(`^[A-Z][A-Z0-9_]*$`) + +func checkEnvironment(path string, report *Report) bool { + contents, err := safeio.ReadCanonicalPrivateRegular(path, 1<<20) + if err != nil { + report.Add("operator.env", "$", "environment_unavailable", "Provide a private readable environment file.") + return false + } + seen := map[string]bool{} + for _, raw := range strings.Split(string(contents), "\n") { + line := strings.TrimSpace(raw) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + key, value, found := strings.Cut(line, "=") + if !found || !environmentKey.MatchString(key) || seen[key] || placeholder(value) || strings.Contains(value, "$") { + report.Add("operator.env", "$", "environment_invalid", "Use one literal KEY=value per line, unique uppercase keys, and replace placeholders; shell interpolation is not accepted.") + return false + } + if strings.HasSuffix(key, "_PASSWORD") || strings.HasSuffix(key, "_API_KEY") { + report.Add("operator.env", "$", "inline_secret", "Move credential values to protected files and keep only file references in operator.env.") + return false + } + seen[key] = true + } + return true +} + +func CheckSecret(path, document, field string, allowEmpty bool, report *Report) bool { + contents, err := safeio.ReadCanonicalPrivateRegular(path, 64<<10) + if err != nil || (!allowEmpty && len(bytes.TrimSpace(contents)) == 0) || placeholder(string(contents)) { + report.Add(document, field, "secret_unavailable", "Supply a non-placeholder private readable regular secret file (owner-only permissions, no links); do not put its contents in YAML or logs.") + return false + } + return true +} + +func Validate(path string) (config.Installation, Report) { + report := NewReport() + if !CheckYAML(path, "thothii-installation.yaml", &report) { + return config.Installation{}, report + } + // Read only the location before the canonical loader checks every field. + contents, _ := safeio.ReadCanonicalPrivateRegular(path, 1<<20) + var location struct { + EnvFile string `yaml:"envFile"` + } + if yaml.Unmarshal(contents, &location) != nil { + report.Add("thothii-installation.yaml", "envFile", "schema_invalid", "Set envFile to one absolute path string, then correct the remaining descriptor fields.") + return config.Installation{}, report + } + if !checkEnvironment(location.EnvFile, &report) { + return config.Installation{}, report + } + installation, err := config.LoadPrepared(path) + if err != nil { + field, correction := "$", "Check installation schema v2, absolute paths, workspace remote/branch and matching environment values. Custom overrides and Git credential/CA files must exist." + if strings.Contains(err.Error(), "modelCatalog") { + field, correction = "modelCatalog", "Check interaction default eligibility, provider endpoints/authentication, embedding id/dimensions, and the private provider key bundle." + } + if strings.Contains(err.Error(), "authentication") { + field, correction = "authentication", "Match the authentication directory to operator.env and prepare its protected files." + } + report.Add("thothii-installation.yaml", field, "configuration_invalid", correction) + return config.Installation{}, report + } + value := func(key string) string { result, _ := installation.EnvironmentValue(key); return result } + for _, key := range []string{"COMPOSE_PROJECT_NAME", "THT_WORKSPACE_INSTALLATION_ID", "THT_INSTALLATION_CONFIG_SOURCE", "THT_SECRETS_FILE", "PI_AUTH_FILE", "THT_CATALOG_RUNTIME_PASSWORD_SOURCE", "THT_CATALOG_MIGRATOR_PASSWORD_SOURCE"} { + if value(key) == "" { + report.Add("operator.env", key, "required", "Supply this installation parameter or protected file reference before setup.") + } + } + if value("THT_INSTALLATION_CONFIG_SOURCE") != path { + report.Add("operator.env", "THT_INSTALLATION_CONFIG_SOURCE", "path_mismatch", "Point to the exact installation descriptor being validated.") + } + for _, key := range []string{"THOTH_HTTP_PORT", "THOTH_CORE_HTTP_PORT", "MAX_PI_PROCESSES"} { + number, err := strconv.Atoi(value(key)) + if err != nil || number < 1 || number > 65535 { + report.Add("operator.env", key, "invalid_number", "Supply a positive integer; HTTP ports must be within 1..65535.") + } + } + if value("THOTH_HTTP_PORT") == value("THOTH_CORE_HTTP_PORT") { + report.Add("operator.env", "THOTH_CORE_HTTP_PORT", "port_collision", "Choose different frontend and core host ports.") + } + files, err := installation.SecretFiles() + if err != nil { + report.Add("operator.env", "$", "secret_references_invalid", "Use canonical absolute paths for all _FILE and _SOURCE references.") + } + for _, file := range files { + allowEmpty := file == value("THT_WORKSPACE_GIT_CREDENTIALS_FILE") + CheckSecret(file, "operator.env", "protected-file-reference", allowEmpty, &report) + } + auth, users, err := authconfig.Load(installation.AuthenticationDirectory()) + if err != nil { + report.Add("auth/auth.yaml", "$", "authentication_invalid", "Prepare and validate local authentication documents before setup, including the initial administrator.") + } else if auth.Mode == "local" { + admin := false + for _, user := range users.Users { + for _, role := range user.Roles { + if user.Enabled && role == authconfig.RoleAdmin { + admin = true + } + } + } + if !admin { + report.Add("auth/users.yaml", "users", "administrator_missing", "Enable at least one administrator before setup.") + } + } else { + report.Add("auth/auth.yaml", "mode", "unsupported_bootstrap", "This preparation increment supports local administrative authentication; use the documented existing OIDC preparation path until its offline bootstrap validation is available.") + } + checkPiCredentials(value("PI_AUTH_FILE"), installation.ModelCatalog, &report) + report.OK = len(report.Issues) == 0 + return installation, report +} + +// Pi stores api_key or oauth records. Require literal prepared material here; model +// environment references belong to the catalog's secret_env path, not host process state. +func checkPiCredentials(path string, catalog config.ModelCatalog, report *Report) { + contents, err := safeio.ReadCanonicalPrivateRegular(path, 64<<10) + var credentials map[string]map[string]any + invalid := err != nil || json.Unmarshal(contents, &credentials) != nil || credentials == nil + literal := func(value any) bool { + text, ok := value.(string) + return ok && strings.TrimSpace(text) != "" && !strings.HasPrefix(text, "!") && !strings.Contains(text, "$") && !placeholder(text) + } + var declarative func(any) bool + declarative = func(value any) bool { + switch v := value.(type) { + case string: + return !strings.HasPrefix(v, "!") + case []any: + for _, item := range v { + if !declarative(item) { + return false + } + } + case map[string]any: + for _, item := range v { + if !declarative(item) { + return false + } + } + } + return true + } + seen := map[string]bool{} + for provider, record := range credentials { + name := strings.ToLower(strings.TrimSpace(provider)) + if name == "" || provider != name || seen[name] || !declarative(record) { + invalid = true + } + seen[name] = true + switch record["type"] { + case "api_key": + if !literal(record["key"]) { + invalid = true + } + if env, present := record["env"]; present { + values, ok := env.(map[string]any) + if !ok { + invalid = true + } + for _, value := range values { + if _, ok := value.(string); !ok { + invalid = true + } + } + } + case "oauth": + expires, ok := record["expires"].(float64) + if !literal(record["access"]) || !literal(record["refresh"]) || !ok || expires <= 0 { + invalid = true + } + default: + invalid = true + } + } + for id, provider := range catalog.Providers { + if provider.Authentication.Mode == "pi_auth" && !seen[id] { + invalid = true + } + } + if invalid { + report.Add("pi-auth.json", "providers", "provider_auth_invalid", "Provide a JSON object with literal api_key/type+key or oauth/type+access+refresh+expires records for selected Pi providers; use catalog secret_env for environment-based keys. Commands and unresolved references are not accepted.") + } +}