feat(cli): prepare and validate application documents offline

This commit is contained in:
Codex
2026-09-28 16:33:07 +02:00
parent 64e6b9664a
commit b9c3369e7b
20 changed files with 1164 additions and 49 deletions
+5 -1
View File
@@ -9,7 +9,11 @@ and invariants are in [AGENTS.md](AGENTS.md); prior snapshots remain in Git.
and `tht workspace validate` through a native two-executable bundle. Build and
test instructions are in [the host CLI guide](tools/tht/README.md). Local validation
reuses runtime workspace/catalog parsers and explicitly defers runtime Evidence,
database binding and readiness checks. The remainder of the installation tickets,
database binding and readiness checks. Ticket #44 adds `tht installation prepare`,
explicit `installation credentials`, and `installation validate --workspaces PATH`
for protected application documents, canonical model settings and schema-v1
database bootstrap inputs. These commands do not start services or import Catalog
bindings. The remainder of the installation tickets,
Docker Hub publication and example databases remain pending.
- React supports full/embedded rendering independently of local/OIDC/upstream auth,
+21
View File
@@ -0,0 +1,21 @@
import { readFileSync, lstatSync } from "node:fs";
import { parseAllDocuments } from "yaml";
import { decode, DocumentError, runWorkspaceDocuments } from "../workspaces/documents.js";
import { validateDatabaseBootstrap } from "./bootstrap-documents.js";
/** Internal sibling protocol: only references cross back to Go, never secret contents. */
export function runBootstrapValidation(args: string[]): { status: number; output: string } {
try {
if (args.length !== 5 || args[0] !== "--directory" || args[2] !== "--bootstrap" || args[4] !== "--json") throw new Error("usage");
const checked = runWorkspaceDocuments(["validate", "--directory", args[1], "--json"]);
if (checked.status !== 0) return checked;
const info = lstatSync(args[3]);
if (!info.isFile() || info.isSymbolicLink() || info.size > 1024 * 1024) throw new Error("file");
const validated = decode(readFileSync(args[3], "utf8"), "database-bootstrap.yaml", (source) =>
validateDatabaseBootstrap(parseAllDocuments(source)[0].toJSON(), args[1]), "database bootstrap schema v1 and the Catalog binding contract");
return { status: 0, output: JSON.stringify({ schema_version: 1, ok: true, secret_files: validated.secretFiles, warnings: validated.warnings, issues: [] }) };
} catch (error) {
const issue = error instanceof DocumentError ? error.issue : { document: "database-bootstrap.yaml", field: "$", code: "bootstrap_invalid", correction: "Supply one complete Catalog database configuration per workspace in a readable local bootstrap document." };
return { status: 1, output: JSON.stringify({ schema_version: 1, ok: false, issues: [issue] }) };
}
}
@@ -0,0 +1,47 @@
import { readFileSync } from "node:fs";
import { join } from "node:path";
import { z } from "zod";
import { databaseConfigurationSchema } from "./configuration-schema.js";
import { parseWorkspaceCatalogYaml } from "../workspaces/catalog.js";
import { parseWorkspaceYaml } from "../workspaces/schema.js";
import { discoverWorkspaceSecretRequirements } from "../workspaces/secret-requirements.js";
const reference = z.string().min(1).max(4096);
const database = databaseConfigurationSchema.extend({
secretFiles: z.object({ password: reference.optional(), apiKey: reference.optional(), sshPrivateKey: reference.optional(), sshPrivateKeyPassphrase: reference.optional(), sshKnownHosts: reference.optional(), tlsCa: reference.optional() }).strict(),
evidenceSecretFiles: z.object({ "evidence.signed_urls": reference.optional(), "evidence.access_key": reference.optional(), "evidence.secret_key": reference.optional(), "evidence.session_token": reference.optional() }).strict().optional(),
});
const bootstrap = z.object({ schemaVersion: z.literal(1), databases: z.array(database).min(1).max(1000) }).strict();
export interface BootstrapReference { field: string; path: string }
/** Offline bootstrap boundary: runtime Catalog owns the resulting bindings after import. */
export function validateDatabaseBootstrap(value: unknown, workspaceRoot: string): { secretFiles: BootstrapReference[]; warnings: string[] } {
const document = bootstrap.parse(value);
const catalog = parseWorkspaceCatalogYaml(readFileSync(join(workspaceRoot, "thoth-workspaces.yaml"), "utf8"));
const expected = new Set(catalog.workspaces.map((entry) => entry.id));
const seen = new Set<string>();
const secretFiles: BootstrapReference[] = [];
const warnings: string[] = [];
const issue = (path: (string | number)[], message: string): never => { throw new z.ZodError([{ code: "custom", path, message }]); };
document.databases.forEach((entry, index) => {
if (!expected.has(entry.workspaceId) || seen.has(entry.workspaceId)) issue(["databases", index, "workspaceId"], "Declare each catalog workspace exactly once.");
seen.add(entry.workspaceId);
const required = entry.binding.transport === "rest_api"
? entry.binding.restAuth === "none" ? [] : ["apiKey"] as const
: entry.binding.transport === "ssh_tunnel" ? ["password", "sshPrivateKey", "sshKnownHosts"] as const : ["password"] as const;
for (const name of required) {
if (!entry.secretFiles[name]) issue(["databases", index, "secretFiles", name], "Supply a protected file reference for this transport.");
}
if (entry.binding.transport === "ssh_tunnel") warnings.push(`databases.${index}:ssh_tunnel supports Catalog diagnostics, not NL-to-SQL sessions; choose direct or REST for practice.`);
for (const [name, path] of Object.entries(entry.secretFiles)) secretFiles.push({ field: `databases.${index}.secretFiles.${name}`, path });
const workspace = parseWorkspaceYaml(readFileSync(join(workspaceRoot, entry.workspaceId, "workspace.yaml"), "utf8"));
const requirements = discoverWorkspaceSecretRequirements(workspace, {});
for (const requirement of requirements.filter((item) => item.connector === "evidence" && item.required)) {
if (!(entry.evidenceSecretFiles as Record<string, string> | undefined)?.[requirement.id]) issue(["databases", index, "evidenceSecretFiles"], "Supply the configured Evidence authentication file references.");
}
for (const [name, path] of Object.entries(entry.evidenceSecretFiles ?? {})) secretFiles.push({ field: `databases.${index}.evidenceSecretFiles.${name}`, path });
});
if (seen.size !== expected.size) issue(["databases"], "Add a database binding for every catalog workspace.");
return { secretFiles, warnings };
}
@@ -0,0 +1,44 @@
import { z } from "zod";
import { parseCredentialFreeHttpUrl } from "../auth/url-policy.js";
import { DATABASE_TRANSPORTS } from "./types.js";
const workspaceIdSchema = z.string().regex(/^[a-z][a-z0-9-]{2,62}$/);
const identifier = z.string().trim().min(1).max(128).regex(/^[A-Za-z_][A-Za-z0-9_$-]*$/);
const nonEmpty = z.string().trim().min(1).max(512);
const port = z.number().int().min(1).max(65_535);
const optionalText = nonEmpty.optional();
const sshHost = z.string().trim().min(1).max(255).regex(/^[A-Za-z0-9_.:\[\]-]+$/).optional();
const sshUsername = z.string().trim().min(1).max(128).regex(/^[A-Za-z0-9._-]+$/).optional();
const bindingSchema = z.object({
transport: z.enum(DATABASE_TRANSPORTS),
host: optionalText,
port: port.optional(),
username: optionalText,
baseUrl: z.string().max(2048)
.refine((value) => parseCredentialFreeHttpUrl(value) !== undefined)
.optional(),
restPath: z.string().regex(/^\/(?!\/)[^?#\\\u0000-\u001f]*$/).max(512).optional(),
restAuth: z.enum(["none", "bearer", "x-api-key"]).optional(),
tlsServername: optionalText,
sshHost,
sshPort: port.optional(),
sshUsername,
sshTargetHost: sshHost,
sshTargetPort: port.optional(),
}).strict().superRefine((binding, context) => {
const required = binding.transport === "postgres_direct"
? ["host", "port", "username"] as const
: binding.transport === "rest_api"
? ["baseUrl", "restPath", "restAuth"] as const
: ["username", "sshHost", "sshPort", "sshUsername", "sshTargetHost", "sshTargetPort"] as const;
for (const field of required) {
if (binding[field] === undefined) context.addIssue({ code: "custom", path: [field], message: "Required" });
}
});
export const databaseConfigurationSchema = z.object({
workspaceId: workspaceIdSchema,
engine: z.literal("postgres"),
databaseName: identifier,
schema: identifier,
binding: bindingSchema,
}).strict();
+1 -42
View File
@@ -1,60 +1,19 @@
import type { FastifyInstance, FastifyReply, FastifyRequest } from "fastify";
import { z } from "zod";
import { isPrincipalContext, requirePermission } from "../auth/authorization.js";
import { parseCredentialFreeHttpUrl } from "../auth/url-policy.js";
import { databaseConfigurationSchema as configSchema } from "../catalog/configuration-schema.js";
import { CatalogService, type CatalogSecretName } from "../catalog/service.js";
import { WorkspaceRegistryError } from "../workspaces/git-repository.js";
import {
CatalogConflictError,
CatalogOperationInProgressError,
CatalogUnavailableError,
DATABASE_TRANSPORTS,
type CatalogRepository,
type DatabaseConfigurationInput,
} from "../catalog/types.js";
import type { CatalogOperationCoordinator } from "../catalog/operation-coordinator.js";
const idSchema = z.uuid();
const workspaceIdSchema = z.string().regex(/^[a-z][a-z0-9-]{2,62}$/);
const identifier = z.string().trim().min(1).max(128).regex(/^[A-Za-z_][A-Za-z0-9_$-]*$/);
const nonEmpty = z.string().trim().min(1).max(512);
const port = z.number().int().min(1).max(65_535);
const optionalText = nonEmpty.optional();
const sshHost = z.string().trim().min(1).max(255).regex(/^[A-Za-z0-9_.:\[\]-]+$/).optional();
const sshUsername = z.string().trim().min(1).max(128).regex(/^[A-Za-z0-9._-]+$/).optional();
const bindingSchema = z.object({
transport: z.enum(DATABASE_TRANSPORTS),
host: optionalText,
port: port.optional(),
username: optionalText,
baseUrl: z.string().max(2048)
.refine((value) => parseCredentialFreeHttpUrl(value) !== undefined)
.optional(),
restPath: z.string().regex(/^\/(?!\/)[^?#\\\u0000-\u001f]*$/).max(512).optional(),
restAuth: z.enum(["none", "bearer", "x-api-key"]).optional(),
tlsServername: optionalText,
sshHost,
sshPort: port.optional(),
sshUsername,
sshTargetHost: sshHost,
sshTargetPort: port.optional(),
}).strict().superRefine((binding, context) => {
const required = binding.transport === "postgres_direct"
? ["host", "port", "username"] as const
: binding.transport === "rest_api"
? ["baseUrl", "restPath", "restAuth"] as const
: ["username", "sshHost", "sshPort", "sshUsername", "sshTargetHost", "sshTargetPort"] as const;
for (const field of required) {
if (binding[field] === undefined) context.addIssue({ code: "custom", path: [field], message: "Required" });
}
});
const configSchema = z.object({
workspaceId: workspaceIdSchema,
engine: z.literal("postgres"),
databaseName: identifier,
schema: identifier,
binding: bindingSchema,
}).strict();
const updateSchema = configSchema.extend({ version: z.number().int().positive() });
const secretNames = [
"password",
+3 -1
View File
@@ -1,6 +1,8 @@
/** Compiled with its runtime for the host CLI: no installation, Docker or host Node required. */
import { runWorkspaceDocuments } from "./workspaces/documents.js";
import { runBootstrapValidation } from "./catalog/bootstrap-cli.js";
const result = runWorkspaceDocuments(process.argv.slice(2));
const args = process.argv.slice(2);
const result = args[0] === "bootstrap" ? runBootstrapValidation(args.slice(1)) : runWorkspaceDocuments(args);
console.log(result.output);
process.exitCode = result.status;
+3 -3
View File
@@ -17,7 +17,7 @@ interface Report {
const MAX_DOCUMENT_BYTES = 1024 * 1024;
const usage = "tht workspace prepare --directory NEW_PATH --id ID --name NAME [--language en|it] [--json]\ntht workspace validate --directory PATH [--json]";
class DocumentError extends Error {
export class DocumentError extends Error {
constructor(readonly issue: Issue) { super(issue.correction); }
}
function fail(document: string, field: string, code: string, correction: string): never {
@@ -25,7 +25,7 @@ function fail(document: string, field: string, code: string, correction: string)
}
/** Never include parser messages or submitted values: YAML and Zod errors can contain secrets. */
function decode<T>(source: string, document: string, parser: (text: string) => T): T {
export function decode<T>(source: string, document: string, parser: (text: string) => T, contract = "workspace schema v4 or catalog schema v1"): T {
try {
const documents = parseAllDocuments(source, { uniqueKeys: true });
if (documents.length !== 1) fail(document, "$", "yaml_documents", "Keep exactly one YAML document in this file.");
@@ -43,7 +43,7 @@ function decode<T>(source: string, document: string, parser: (text: string) => T
// Strict schemas produce paths containing schema-defined keys and array indices only.
fail(document, issue.path.join(".") || "$", "schema_invalid",
issue.code === "unrecognized_keys" ? "Remove fields not defined by the current workspace/catalog contract."
: "Correct this field using workspace schema v4 or catalog schema v1; check type, required value, uniqueness and allowed values.");
: `Correct this field using ${contract}; check type, required value, uniqueness and allowed values.`);
}
fail(document, "$", "schema_invalid", "Use an authored workspace v4 descriptor; remove database configuration and keep it in the PostgreSQL Metadata Catalog.");
}
@@ -0,0 +1,49 @@
import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from "node:fs";
import { join } from "node:path";
import { tmpdir } from "node:os";
import { afterEach, expect, test } from "vitest";
import { validateDatabaseBootstrap } from "../src/catalog/bootstrap-documents.js";
import { runBootstrapValidation } from "../src/catalog/bootstrap-cli.js";
const roots: string[] = [];
afterEach(() => roots.splice(0).forEach((root) => rmSync(root, { recursive: true, force: true })));
function fixture() {
const root = mkdtempSync(join(tmpdir(), "bootstrap-documents-")); roots.push(root);
mkdirSync(join(root, "practice"));
writeFileSync(join(root, "thoth-workspaces.yaml"), "schema_version: 1\nworkspaces: [{id: practice, name: Practice}]\n");
writeFileSync(join(root, "practice/workspace.yaml"), "workspace: {schema_version: 4, id: practice, name: Practice, language: en}\n");
return root;
}
const entry = { workspaceId: "practice", engine: "postgres", databaseName: "practice", schema: "public", binding: { transport: "postgres_direct", host: "db.internal", port: 5432, username: "reader" }, secretFiles: { password: "/private/operator/db-password" } };
test("bootstrap reuses Catalog configuration and requires one complete binding per workspace", () => {
const root = fixture();
const valid = validateDatabaseBootstrap({ schemaVersion: 1, databases: [entry] }, root);
expect(valid.secretFiles).toContainEqual({ field: "databases.0.secretFiles.password", path: "/private/operator/db-password" });
for (const databases of [[], [entry, entry], [{ ...entry, workspaceId: "unknown" }], [{ ...entry, secretFiles: {} }], [{ ...entry, engine: "mysql" }], [{ ...entry, binding: { ...entry.binding, port: 70000 } }]]) {
expect(() => validateDatabaseBootstrap({ schemaVersion: 1, databases }, root)).toThrow();
}
});
test("REST authentication, SSH credentials and optional Evidence use their declared contracts", () => {
const root = fixture();
const rest = { ...entry, binding: { transport: "rest_api", baseUrl: "https://data.internal", restPath: "/query", restAuth: "none" }, secretFiles: {} };
expect(validateDatabaseBootstrap({ schemaVersion: 1, databases: [rest] }, root).secretFiles).toEqual([]);
expect(() => validateDatabaseBootstrap({ schemaVersion: 1, databases: [{ ...rest, binding: { ...rest.binding, restAuth: "bearer" } }] }, root)).toThrow();
const ssh = { ...entry, binding: { transport: "ssh_tunnel", username: "reader", sshHost: "bastion", sshPort: 22, sshUsername: "tunnel", sshTargetHost: "database", sshTargetPort: 5432 }, secretFiles: { password: "/password", sshPrivateKey: "/key", sshKnownHosts: "/hosts" } };
expect(validateDatabaseBootstrap({ schemaVersion: 1, databases: [ssh] }, root).warnings[0]).toContain("not NL-to-SQL");
writeFileSync(join(root, "practice/workspace.yaml"), "workspace: {schema_version: 4, id: practice, name: Practice, language: en}\nevidence:\n source:\n type: http\n uris: [https://docs.internal/manual.md]\n authentication: signed_urls_file\n");
expect(() => validateDatabaseBootstrap({ schemaVersion: 1, databases: [entry] }, root)).toThrow();
expect(validateDatabaseBootstrap({ schemaVersion: 1, databases: [{ ...entry, evidenceSecretFiles: { "evidence.signed_urls": "/urls.json" } }] }, root).secretFiles).toContainEqual({ field: "databases.0.evidenceSecretFiles.evidence.signed_urls", path: "/urls.json" });
});
test("bootstrap CLI never echoes arbitrary keys from submitted secret references", () => {
const root = fixture();
const path = join(root, "bootstrap.yaml");
for (const field of ["secretFiles", "evidenceSecretFiles"]) {
writeFileSync(path, JSON.stringify({ schemaVersion: 1, databases: [{ ...entry, [field]: { PRIVATE_CREDENTIAL_SENTINEL: "/path" } }] }));
const result = runBootstrapValidation(["--directory", root, "--bootstrap", path, "--json"]);
expect(result.status).toBe(1);
expect(result.output).not.toContain("PRIVATE_CREDENTIAL_SENTINEL");
}
});
@@ -0,0 +1,67 @@
import { spawnSync } from "node:child_process";
import { chmodSync, mkdtempSync, readFileSync, realpathSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { rootCertificates } from "node:tls";
import { afterEach, expect, test } from "vitest";
const binary = process.env.THT_INSTALLATION_TEST_CLI;
const roots: string[] = [];
afterEach(() => roots.splice(0).forEach((path) => rmSync(path, { recursive: true, force: true })));
function run(...args: string[]) {
return spawnSync(binary!, [...args, "--json"], { encoding: "utf8", env: { ...process.env, PATH: "" }, input: "" });
}
test.skipIf(!binary)("operator prepares, completes and repeatedly validates before any stack exists", () => {
const root = realpathSync(mkdtempSync(join(tmpdir(), "application-documents-"))); roots.push(root);
const workspace = join(root, "workspaces"), directory = join(root, "installation");
expect(run("workspace", "prepare", "--directory", workspace, "--id", "practice", "--name", "Practice").status).toBe(0);
expect(run("installation", "prepare", "--directory", directory).status).toBe(0);
const installation = join(directory, "thothii-installation.yaml");
const validate = () => run("--installation", installation, "installation", "validate", "--workspaces", workspace);
expect(validate().status).toBe(1); // Visible placeholders cannot be approved.
expect(run("installation", "credentials", "--directory", directory).status).toBe(0);
for (const name of ["thothii-installation.yaml", "operator.env", "database-bootstrap.yaml"]) {
const path = join(directory, name);
writeFileSync(path, readFileSync(path, "utf8").replaceAll("CHANGE_ME", "practice"));
}
for (const [name, contents] of Object.entries({ "secrets.env": "OPENAI_API_KEY=PRIVATE_PROVIDER_VALUE\n", "database-password": "PRIVATE_DATABASE_VALUE", "git-credentials": "", "git-ca.pem": rootCertificates[0] })) {
writeFileSync(join(directory, "secrets", name), contents, { mode: 0o600 });
}
const before = readFileSync(installation, "utf8");
for (let index = 0; index < 2; index++) {
const checked = validate();
expect(checked.stderr).toBe("");
expect(checked.stdout).not.toContain("PRIVATE_");
expect(checked.status, checked.stdout).toBe(0);
expect(JSON.parse(checked.stdout).deferred_checks).toContain("release-assets");
}
expect(readFileSync(installation, "utf8")).toBe(before);
writeFileSync(installation, before.replace("interaction: openai/gpt-4.1-mini", "interaction: openai/nonexistent"));
expect(validate().status).toBe(1);
writeFileSync(installation, before);
const authFile = join(directory, "secrets/pi-auth.json");
writeFileSync(authFile, "not-json");
expect(validate().status).toBe(1);
writeFileSync(installation, before.replace("{mode: secret_env, apiKeyEnv: OPENAI_API_KEY}", "{mode: pi_auth}"));
writeFileSync(authFile, JSON.stringify({ openai: { unrelated: true } }));
expect(validate().status).toBe(1);
writeFileSync(authFile, JSON.stringify({ " OpenAI ": { type: "api_key", key: "PRIVATE_PI_KEY" } }));
expect(validate().status).toBe(1);
writeFileSync(authFile, JSON.stringify({ openai: { type: "api_key", key: "PRIVATE_PI_KEY" } }));
expect(validate().status).toBe(0);
writeFileSync(installation, before);
writeFileSync(authFile, "{}");
const bootstrap = join(directory, "database-bootstrap.yaml");
const originalBootstrap = readFileSync(bootstrap, "utf8");
writeFileSync(join(workspace, "practice", "private-password"), "PRIVATE_DATABASE_VALUE", { mode: 0o600 });
writeFileSync(bootstrap, originalBootstrap.replace(join(directory, "secrets/database-password"), join(workspace, "practice/private-password")));
expect(validate().status).toBe(1);
writeFileSync(bootstrap, originalBootstrap);
chmodSync(join(directory, "secrets/database-password"), 0o644);
expect(validate().status).toBe(1);
chmodSync(join(directory, "secrets/database-password"), 0o600);
const env = join(directory, "operator.env");
writeFileSync(env, readFileSync(env, "utf8") + "THOTH_HTTP_PORT=8081\n");
expect(validate().status).toBe(1);
}, 15_000);
+88
View File
@@ -64,6 +64,94 @@ success does not certify semantic truth, connectivity or readiness. The Git revi
activated later must contain the checked documents; this command does not publish
uncommitted files or empty directories.
## Prepare and validate application documents
After validating workspaces, create a local directory **outside their repository**:
```sh
tht installation prepare --directory ./my-installation
```
This creates private, commented `thothii-installation.yaml`, `operator.env`,
`database-bootstrap.yaml` and `README.md`. The destination must be new and its parent
must exist. It starts no services and does not implicitly generate passwords.
1. Choose models and providers in the descriptor. The template proposes
`openai/gpt-4.1-mini` for interaction and `ollama/qwen3-embedding:0.6b` with 1024
dimensions for embedding. Edit these before setup. `modelCatalog.defaults.interaction`
must support sessions and metadata generation when the latter is configured.
The template omits optional metadata generation. See
[model configuration](../general/pi-configuration.md) for custom providers.
2. Replace the Git remote in both the descriptor and `operator.env`; keep branch and
transport consistent. Paths are absolute and machine-local. `operator.env` accepts
one literal `KEY=value` assignment per line, without duplicate keys or shell
interpolation. Credentials belong in referenced protected files.
3. Complete `database-bootstrap.yaml` with exactly one entry per workspace. A complete
direct-connection example is:
```yaml
schemaVersion: 1
databases:
- workspaceId: practice
engine: postgres
databaseName: sales
schema: public
binding:
transport: postgres_direct
host: db.intranet
port: 5432
username: thoth_reader
secretFiles:
password: /private/path/my-installation/secrets/database-password
```
Use a read-only DWH account. `rest_api` requires `baseUrl`, `restPath`, `restAuth`
(`none`, `bearer`, `x-api-key`) and `secretFiles.apiKey` when authenticated.
`ssh_tunnel` requires `username`, `sshHost`, `sshPort`, `sshUsername`,
`sshTargetHost`, `sshTargetPort` and files `password`, `sshPrivateKey`,
`sshKnownHosts`; it supports Catalog diagnostics, not NL-to-SQL sessions.
`tlsCa` and `sshPrivateKeyPassphrase` are optional. Signed HTTP Evidence needs
`evidenceSecretFiles` with key `evidence.signed_urls`; static S3 credentials need
`evidence.access_key`, `evidence.secret_key` and optional `evidence.session_token`.
All values are private file paths. Workspace descriptors remain schema v4;
this bootstrap input is not a second runtime Catalog.
4. Explicitly generate technical credentials in the standard layout:
```sh
tht installation credentials --directory ./my-installation
```
This creates separate random Catalog runtime/migrator and administrator passwords,
`auth/auth.yaml`, `auth/users.yaml`, a `secrets/secrets.env` template and
`secrets/pi-auth.json`. Existing files are retained; invalid ones stop the command.
The initial administrator is `admin`; its password stays in private
`secrets/admin-password` and is never printed. The default is local authentication
at `http://localhost:8080`: review and edit `auth/auth.yaml` before validation.
This increment does not validate offline OIDC bootstrap for the existing path.
5. Fill the provider key in `secrets/secrets.env` and create the DWH password file.
For Git HTTPS supply the referenced credentials and CA files; empty credentials
are allowed for a public remote, and the CA file must be available. For SSH supply
a key and known_hosts and select the matching descriptor override. `pi_auth`
providers require prepared Pi credentials. Keep every secret outside workspace
Git with installer-only access (0600 on Unix, equivalent Windows ACLs).
6. Validate and repeat after each correction:
```sh
tht --installation /absolute/path/my-installation/thothii-installation.yaml installation validate --workspaces /absolute/path/my-workspaces --json
```
The default bootstrap is beside the descriptor; `--bootstrap PATH` selects another.
Validation changes no documents, generates no projections, uses no network and
writes no database. It rejects placeholders, inconsistencies, missing/non-private
files and secrets inside workspace Git. Reports identify document, field and
correction without secret values. Exit statuses: 0 local success, 1 corrections
needed, 2 invalid arguments.
Standard release Compose assets may still be absent at this stage; custom overrides
must already exist. Release assets, external connectivity, Catalog import and runtime
readiness remain explicit deferred checks. Success prepares the next preflight;
it neither skips those checks nor establishes a completed installation.
## Before you start: the two repositories
There are two separate repositories:
+93
View File
@@ -65,6 +65,99 @@ Il successo locale non certifica verità semantica, connettività o readiness. L
revisione Git attivata in seguito deve contenere i documenti verificati; il comando
non pubblica file non committati o directory vuote.
## Predisporre e verificare i documenti applicativi
Dopo la verifica dei workspace, creare una cartella locale **esterna al loro repository**:
```sh
tht installation prepare --directory ./mia-installazione
```
Il comando crea file privati commentati: `thothii-installation.yaml`, `operator.env`,
`database-bootstrap.yaml` e `README.md`. La cartella deve essere nuova e il padre
deve esistere. Non avvia servizi e non genera implicitamente password.
1. Nel descrittore, scegliere modelli e provider. Il template propone
`openai/gpt-4.1-mini` per l'interazione e `ollama/qwen3-embedding:0.6b` con 1024
dimensioni per l'embedding. Sono valori modificabili, non una selezione richiesta
durante il setup. `modelCatalog.defaults.interaction` deve essere utilizzabile
nelle sessioni e anche nella generazione metadati, se quest'ultima è configurata.
Il template omette la generazione metadati, che è facoltativa. Consultare la
[configurazione dei modelli](../general/pi-configuration.md) per provider personalizzati.
2. Sostituire il remoto Git sia nel descrittore sia in `operator.env`; mantenere
coerenti branch e trasporto. I percorsi sono assoluti e riferiti a questa macchina.
`operator.env` accetta una sola assegnazione letterale `KEY=value` per riga, senza
duplicati o interpolazioni shell. Le credenziali restano nei file referenziati.
3. Compilare `database-bootstrap.yaml`: una voce per ciascun workspace, senza
duplicati. Questo esempio mostra il contratto completo di un collegamento diretto:
```yaml
schemaVersion: 1
databases:
- workspaceId: pratica
engine: postgres
databaseName: vendite
schema: public
binding:
transport: postgres_direct
host: db.intranet
port: 5432
username: thoth_reader
secretFiles:
password: /percorso/privato/mia-installazione/secrets/database-password
```
Usare le credenziali di un utente DWH in sola lettura. Per `rest_api`, il binding
richiede `baseUrl`, `restPath` e `restAuth` (`none`, `bearer`, `x-api-key`); quando
serve autenticazione, aggiungere `secretFiles.apiKey`. `ssh_tunnel` richiede
`username`, `sshHost`, `sshPort`, `sshUsername`, `sshTargetHost`, `sshTargetPort` e
i file `password`, `sshPrivateKey`, `sshKnownHosts`; abilita diagnostica Catalog,
non sessioni NL→SQL. Sono facoltativi `tlsCa` e `sshPrivateKeyPassphrase`.
Le Evidence HTTP firmate richiedono `evidenceSecretFiles` con chiave
`evidence.signed_urls`; S3 con credenziali statiche richiede `evidence.access_key`
e `evidence.secret_key`, con `evidence.session_token` facoltativo. Tutti i valori
sono percorsi di file privati. I workspace rimangono nello schema v4: il bootstrap
è un input iniziale, non un secondo Catalog runtime.
4. Generare esplicitamente le credenziali tecniche nel layout standard:
```sh
tht installation credentials --directory ./mia-installazione
```
Sono creati password casuali separate per Catalog runtime/migrator e amministratore,
il relativo `auth/auth.yaml` con `auth/users.yaml`, un template `secrets/secrets.env`
e `secrets/pi-auth.json`. I file esistenti vengono conservati; se non validi, il
comando si ferma. L'amministratore iniziale è `admin`, la password è nel file
privato `secrets/admin-password` e non viene stampata. Il default è autenticazione
locale con URL pubblico `http://localhost:8080`: verificare e, se necessario,
modificare `auth/auth.yaml` prima del controllo. Questo incremento non valida il
bootstrap OIDC del percorso esistente.
5. Inserire la chiave del provider in `secrets/secrets.env` e creare il file della
password DWH. Per HTTPS Git fornire i file referenziati per credenziali e CA:
credenziali vuote sono ammesse per un remoto pubblico, la CA deve essere disponibile.
Per SSH fornire chiave e known_hosts, scegliendo il relativo override nel descrittore.
I provider `pi_auth` richiedono credenziali Pi già preparate. Conservare tutti
questi file fuori dal repository workspace e proteggere l'accesso al solo utente
installatore (0600 su Unix, ACL equivalenti su Windows). Non committarli.
6. Verificare e ripetere dopo ogni correzione:
```sh
tht --installation /percorso/assoluto/mia-installazione/thothii-installation.yaml installation validate --workspaces /percorso/assoluto/miei-workspace --json
```
Il bootstrap viene cercato accanto al descrittore; `--bootstrap PERCORSO` ne
seleziona uno diverso. Il controllo non cambia documenti, non genera proiezioni,
non usa la rete e non scrive database. Rifiuta placeholder, incoerenze, file
mancanti/non privati e segreti situati nel repository workspace. Il report indica
documento, campo e correzione senza valori riservati. Exit status: 0 successo
locale, 1 correzioni necessarie, 2 argomenti errati.
Gli asset Compose standard del rilascio possono ancora mancare in questa fase;
gli override personalizzati devono già esistere. Il report distingue i controlli
differiti: asset del rilascio, connettività esterna, import Catalog e readiness.
Un esito positivo prepara il successivo preflight: non autorizza a saltare tali
controlli e non equivale a un'installazione completata.
## Prima di iniziare: i due repository
Servono due repository distinti:
@@ -24,7 +24,18 @@ saltato, 1.417 test passati e 40 saltati; typecheck e build rigorosa documentazi
passati. Suite Go: tutti i pacchetti passati, salvo un primo errore intermittente
nel test di concorrenza authstorage; quel test passa in tre ripetizioni e il pacchetto
completo passa nella verifica isolata. Le revisioni Standards e Spec non lasciano
finding aperti. Il prossimo incremento sequenziale è T02/#44.
finding aperti.
T02/#44 implementato sullo stesso branch: `tht installation prepare`, generazione
esplicita delle credenziali tecniche e `tht installation validate` preparano e
controllano documenti privati, modelli, autenticazione e bootstrap dei binding,
riusando gli schemi runtime senza avviare servizi. Guide IT/EN aggiornate.
Verifica backend completa su Node 24.16: 111 file passati, uno saltato, 1.420 test
passati e 40 saltati; le regressioni successive della revisione passano nella suite
mirata (quattro test, incluso il percorso con binari nativi e `PATH` vuoto).
Typecheck, build rigorosa documentazione e pacchetti Go passati; il pacchetto CLI
è stato ripetuto dopo la correzione rilevata in revisione. Nessun finding residuo
delle revisioni Standards/Spec. Il prossimo incremento sequenziale è T03/#45.
| Ticket | Issue Gitea | Dipendenze dirette |
| --- | --- | --- |
+34
View File
@@ -41,6 +41,40 @@ THT_WORKSPACE_TEST_CLI=/absolute/path/dist/workspace-tools/darwin-arm64/tht \
npx vitest run test/workspace-documents-cli.test.ts
```
## Application document preparation (issue #44)
Before installation, use `tht installation prepare --directory NEW_PATH`, then
`tht installation credentials --directory PATH` to explicitly create technical
credentials in the default protected layout. The latter preserves existing files.
Edit the documents and external credentials, then repeat:
```sh
tht --installation /absolute/path/thothii-installation.yaml \
installation validate --workspaces /absolute/path/workspaces --json
```
`database-bootstrap.yaml` beside the descriptor is a schemaVersion-1 bootstrap
input, not a runtime Catalog. Its database entries use the exact Catalog API
configuration schema plus private `secretFiles` and optional `evidenceSecretFiles`
references. `--bootstrap` selects another document. The helper validates workspace
membership, uniqueness, supported transports and required credential references;
Go checks protected files, the canonical Installation Model Catalog, environment
and local administrator. No process contacts a service or creates projections.
Only missing standard release Compose assets are deferred by `config.LoadPrepared`;
normal runtime `config.Load` retains all existing checks. Custom overrides must exist.
The native integration test is opt-in because it requires the built pair:
```sh
THT_INSTALLATION_TEST_CLI=/absolute/path/dist/workspace-tools/darwin-arm64/tht \
npx vitest run test/installation-documents-cli.test.ts
```
Run it from `backend/`, following the bundle build above. It supplies prepared
fixtures and removes host tools from the subprocess PATH. Platform acceptance and
real external credentials remain separate gates. See the IT/EN guides for the
complete parameter collection procedure, default `admin` account and local-auth scope.
## Shell configuration
The schema-v2 `thothii-installation.yaml` accepts this optional section:
+129
View File
@@ -0,0 +1,129 @@
package main
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"path/filepath"
"slices"
"strings"
"github.com/aritmolab/thothii/tools/tht/internal/preparation"
)
func installationDocumentsCommand(ctx context.Context, installationPath string, args []string, stdout io.Writer) int {
report := preparation.NewReport()
status := 0
options := map[string]string{}
valid := len(args) > 0
for index := 1; index < len(args); index++ {
key := args[index]
if _, duplicate := options[key]; duplicate {
valid = false
break
}
if key == "--json" {
options[key] = "true"
continue
}
if (key != "--directory" && key != "--workspaces" && key != "--bootstrap") || index+1 == len(args) {
valid = false
break
}
options[key] = args[index+1]
index++
}
if !valid || (args[0] == "validate" && (installationPath == "" || options["--workspaces"] == "" || options["--directory"] != "")) || (args[0] != "validate" && (options["--directory"] == "" || options["--workspaces"] != "" || options["--bootstrap"] != "")) {
report.Add("CLI", "$", "usage", "Use installation prepare|credentials --directory PATH [--json], or tht --installation ABSOLUTE_PATH installation validate --workspaces PATH [--bootstrap PATH] [--json].")
status = 2
} else if args[0] == "validate" {
installation, checkedReport := preparation.Validate(installationPath)
report = checkedReport
if report.OK {
bootstrap := options["--bootstrap"]
if bootstrap == "" {
bootstrap = filepath.Join(filepath.Dir(installationPath), "database-bootstrap.yaml")
}
bootstrap, _ = filepath.Abs(bootstrap)
workspace, _ := filepath.Abs(options["--workspaces"])
if resolved, err := filepath.EvalSymlinks(workspace); err == nil {
workspace = resolved
}
outsideWorkspace := func(path, document, field string) bool {
relative, err := filepath.Rel(workspace, path)
if err == nil && relative != ".." && !strings.HasPrefix(relative, ".."+string(filepath.Separator)) {
report.Add(document, field, "installation_file_in_workspace", "Keep installation documents and credential files outside the shared workspace repository.")
return false
}
return true
}
for _, path := range []string{installationPath, installation.EnvFile, installation.AuthenticationDirectory(), bootstrap} {
outsideWorkspace(path, "thothii-installation.yaml", "local-files")
}
files, _ := installation.SecretFiles()
for _, path := range files {
outsideWorkspace(path, "operator.env", "protected-file-reference")
}
if preparation.CheckYAML(bootstrap, "database-bootstrap.yaml", &report) {
var output, discarded bytes.Buffer
code := workspaceDocumentsCommand(ctx, []string{"bootstrap", "--directory", workspace, "--bootstrap", bootstrap, "--json"}, &output, &discarded)
var checked struct {
OK bool `json:"ok"`
Issues []preparation.Issue `json:"issues"`
Warnings []string `json:"warnings"`
SecretFiles []struct {
Field string `json:"field"`
Path string `json:"path"`
} `json:"secret_files"`
}
if json.Unmarshal(output.Bytes(), &checked) != nil {
report.Add("CLI", "$", "validator_unavailable", "Reinstall the matching tht and workspace helper pair.")
} else if code != 0 || !checked.OK {
report.Issues = append(report.Issues, checked.Issues...)
if len(checked.Issues) == 0 {
report.Add("database-bootstrap.yaml", "$", "bootstrap_invalid", "Correct workspace and binding documents, then validate again.")
}
} else {
report.Warnings = append(report.Warnings, checked.Warnings...)
for _, file := range checked.SecretFiles {
if outsideWorkspace(file.Path, "database-bootstrap.yaml", file.Field) {
preparation.CheckSecret(file.Path, "database-bootstrap.yaml", file.Field, false, &report)
}
}
}
}
}
} else {
directory, err := filepath.Abs(options["--directory"])
if err == nil {
if args[0] == "prepare" {
err = preparation.Prepare(directory)
} else {
err = preparation.Credentials(ctx, directory)
}
}
if err != nil {
report.Add("preparation", "$", "preparation_refused", err.Error())
}
}
report.OK = len(report.Issues) == 0
if !report.OK && status == 0 {
status = 1
}
if slices.Contains(args, "--json") {
_ = json.NewEncoder(stdout).Encode(report)
} else {
if report.OK {
fmt.Fprintln(stdout, "Document operation completed. Local validation does not establish runtime readiness.")
}
for _, issue := range report.Issues {
fmt.Fprintf(stdout, "%s [%s] %s: %s\n", issue.Document, issue.Field, issue.Code, issue.Correction)
}
for _, warning := range report.Warnings {
fmt.Fprintln(stdout, warning)
}
}
return status
}
@@ -0,0 +1,96 @@
package main
import (
"bytes"
"context"
"encoding/json"
"os"
"path/filepath"
"runtime"
"testing"
)
func TestInstallationPrepareDocumentsBeforeRuntime(t *testing.T) {
t.Setenv("PATH", "")
root, err := filepath.EvalSymlinks(t.TempDir())
if err != nil {
t.Fatal(err)
}
destination := filepath.Join(root, "installation")
var stdout, stderr bytes.Buffer
status := run(context.Background(), []string{"installation", "prepare", "--directory", destination, "--json"}, &stdout, &stderr)
if status != 0 {
t.Fatalf("status=%d stdout=%s stderr=%s", status, &stdout, &stderr)
}
var report map[string]any
if json.Unmarshal(stdout.Bytes(), &report) != nil || report["ok"] != true {
t.Fatalf("report=%s", &stdout)
}
for _, name := range []string{"thothii-installation.yaml", "operator.env", "database-bootstrap.yaml", "README.md"} {
if _, err := os.Stat(filepath.Join(destination, name)); err != nil {
t.Fatal(err)
}
}
before, _ := os.ReadFile(filepath.Join(destination, "thothii-installation.yaml"))
stdout.Reset()
stderr.Reset()
if run(context.Background(), []string{"installation", "prepare", "--directory", destination, "--json"}, &stdout, &stderr) == 0 {
t.Fatal("overwrote existing documents")
}
after, _ := os.ReadFile(filepath.Join(destination, "thothii-installation.yaml"))
if !bytes.Equal(before, after) {
t.Fatal("existing document changed")
}
}
func TestInstallationCredentialsAreExplicitPrivateAndNeverReplaced(t *testing.T) {
t.Setenv("PATH", "")
root, _ := filepath.EvalSymlinks(t.TempDir())
destination := filepath.Join(root, "installation")
var stdout, stderr bytes.Buffer
if run(context.Background(), []string{"installation", "prepare", "--directory", destination}, &stdout, &stderr) != 0 {
t.Fatal(&stdout, &stderr)
}
secret := filepath.Join(destination, "secrets", "catalog-runtime-password")
if _, err := os.Stat(secret); !os.IsNotExist(err) {
t.Fatal("prepare generated a secret implicitly")
}
stdout.Reset()
stderr.Reset()
if run(context.Background(), []string{"installation", "credentials", "--directory", destination, "--json"}, &stdout, &stderr) != 0 {
t.Fatal(&stdout, &stderr)
}
before, err := os.ReadFile(secret)
if err != nil || len(bytes.TrimSpace(before)) < 32 {
t.Fatal("missing strong technical credential", err)
}
if bytes.Contains(stdout.Bytes(), bytes.TrimSpace(before)) || bytes.Contains(stderr.Bytes(), bytes.TrimSpace(before)) {
t.Fatal("secret leaked")
}
info, _ := os.Stat(secret)
if runtime.GOOS != "windows" && info.Mode().Perm()&0o077 != 0 {
t.Fatal("credential is not private")
}
stdout.Reset()
stderr.Reset()
if run(context.Background(), []string{"installation", "credentials", "--directory", destination, "--json"}, &stdout, &stderr) != 0 {
t.Fatal(&stdout, &stderr)
}
after, _ := os.ReadFile(secret)
if !bytes.Equal(before, after) {
t.Fatal("credential was replaced")
}
}
func TestInstallationValidateRejectsWrongEnvironmentFieldType(t *testing.T) {
root, _ := filepath.EvalSymlinks(t.TempDir())
path := filepath.Join(root, "thothii-installation.yaml")
if err := os.WriteFile(path, []byte("schemaVersion: 2\nenvFile: []\n"), 0o600); err != nil {
t.Fatal(err)
}
var stdout, stderr bytes.Buffer
status := run(context.Background(), []string{"--installation", path, "installation", "validate", "--workspaces", root, "--json"}, &stdout, &stderr)
if status != 1 {
t.Fatalf("invalid field accepted: %d %s", status, &stdout)
}
}
+9
View File
@@ -43,6 +43,12 @@ Commands:
setup [--complete|--configure-only] [--installation-id ID] [--profile local|server]
[--shell-mode full|embedded] [--shell-default-locale BCP47-TAG] [--shell-adapter omics-portal]
Create, validate, and optionally complete the local installation.
installation prepare --directory NEW_PATH [--json]
Create commented installation, environment and database bootstrap templates.
installation credentials --directory PATH [--json]
Explicitly generate protected technical credentials before setup.
installation validate --workspaces PATH [--bootstrap PATH] [--json]
Check prepared application documents; requires --installation.
installation migrate --output PATH --session-default PROVIDER/MODEL
--embedding-id PROVIDER/MODEL --embedding-dimensions N
Create a review-only schema-v2 candidate from all three legacy model sources.
@@ -138,6 +144,9 @@ func run(ctx context.Context, args []string, stdout, stderr io.Writer) int {
return versionCommand(commandArgs, stdout, stderr)
}
if command == "installation" {
if len(commandArgs) > 0 && (commandArgs[0] == "prepare" || commandArgs[0] == "credentials" || commandArgs[0] == "validate") {
return installationDocumentsCommand(ctx, installationPath, commandArgs, stdout)
}
if len(commandArgs) > 0 && commandArgs[0] == "generate" {
return installationGenerationCommand(installationPath, commandArgs[1:], stdout, stderr)
}
+22 -1
View File
@@ -98,6 +98,16 @@ type Installation struct {
// Load reads and validates an installation descriptor at an absolute path.
func Load(path string) (Installation, error) {
return load(path, false)
}
// LoadPrepared applies the same authored configuration rules before release assets exist.
// Only standard distribution-owned Compose assets are deferred, never custom overrides.
func LoadPrepared(path string) (Installation, error) {
return load(path, true)
}
func load(path string, prepared bool) (Installation, error) {
if !filepath.IsAbs(path) {
return Installation{}, fmt.Errorf("installation path must be absolute")
}
@@ -143,6 +153,11 @@ func Load(path string) (Installation, error) {
if err := requireDirectory(raw.ProjectDirectory, "projectDirectory"); err != nil {
return Installation{}, err
}
if prepared {
if err := requireCanonicalDirectory(raw.ProjectDirectory); err != nil {
return Installation{}, errors.New("projectDirectory must be canonical and accessible")
}
}
for _, legacyProjection := range []string{
filepath.Join(raw.ProjectDirectory, "deploy", "pi", "models.json"),
filepath.Join(raw.ProjectDirectory, "deploy", "pi", "settings.json"),
@@ -195,7 +210,8 @@ func Load(path string) (Installation, error) {
return Installation{}, errors.New("authentication.configDirectory must match THT_AUTH_CONFIG_ROOT")
}
for _, override := range raw.Overrides {
if err := requireRegularFile(override, "override"); err != nil {
standardAsset := override == filepath.Join(installation.ProjectDirectory, "deploy", "compose.git-https.yaml") || override == filepath.Join(installation.ProjectDirectory, "deploy", "compose.git-ssh.yaml")
if err := requireRegularFile(override, "override"); err != nil && !(prepared && standardAsset && errors.Is(statPathError(override), os.ErrNotExist)) {
return Installation{}, err
}
installation.Overrides = append(installation.Overrides, filepath.Clean(override))
@@ -209,6 +225,9 @@ func Load(path string) (Installation, error) {
}
}
for _, composeFile := range installation.ComposeFiles()[:2] {
if prepared && errors.Is(statPathError(composeFile), os.ErrNotExist) {
continue
}
if err := requireRegularFile(composeFile, "Compose file"); err != nil {
return Installation{}, err
}
@@ -226,6 +245,8 @@ func Load(path string) (Installation, error) {
return installation, nil
}
func statPathError(path string) error { _, err := os.Lstat(path); return err }
var metadataSecretBundleKeyPattern = regexp.MustCompile(`^[A-Z][A-Z0-9_]{0,127}$`)
var metadataAPIKeyEnvironments = map[string]struct{}{
"THT_MODEL_API_KEY": {},
@@ -0,0 +1,73 @@
package preparation
import (
"context"
"crypto/rand"
"encoding/hex"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"github.com/aritmolab/thothii/tools/tht/internal/authconfig"
"github.com/aritmolab/thothii/tools/tht/internal/config"
"github.com/aritmolab/thothii/tools/tht/internal/safeio"
)
// Credentials creates installation-owned secrets only. External credentials are supplied by
// the operator. Existing files are checked and retained so interrupted preparation can resume.
func Credentials(ctx context.Context, directory string) error {
if err := safeio.ValidatePrivateDirectory(directory); err != nil {
return fmt.Errorf("use an existing private preparation directory")
}
if _, err := safeio.ReadCanonicalPrivateRegular(filepath.Join(directory, "thothii-installation.yaml"), 1<<20); err != nil {
return fmt.Errorf("prepare installation documents first")
}
secrets := filepath.Join(directory, "secrets")
if err := safeio.EnsurePrivateDirectory(secrets); err != nil {
return fmt.Errorf("secrets directory must be private and operator-owned")
}
for _, name := range []string{"catalog-runtime-password", "catalog-migrator-password", "admin-password"} {
path := filepath.Join(secrets, name)
if _, err := os.Lstat(path); err == nil {
value, err := safeio.ReadCanonicalPrivateRegular(path, 64<<10)
if err != nil || len(strings.TrimSpace(string(value))) < 32 {
return fmt.Errorf("existing technical credentials must be private, readable and at least 32 characters; no file was replaced")
}
continue
} else if !errors.Is(err, os.ErrNotExist) {
return fmt.Errorf("cannot inspect technical credentials")
}
value := make([]byte, 32)
if _, err := rand.Read(value); err != nil {
return fmt.Errorf("cannot generate random credentials")
}
if err := safeio.WriteCanonicalNewPrivateFile(path, []byte(hex.EncodeToString(value)+"\n"), 0o600); err != nil {
return fmt.Errorf("cannot create credential; existing files are retained")
}
}
for name, contents := range map[string]string{"secrets.env": "# Supply the provider credential; never commit this file.\nOPENAI_API_KEY=CHANGE_ME\n", "pi-auth.json": "{}\n"} {
path := filepath.Join(secrets, name)
if _, err := os.Lstat(path); errors.Is(err, os.ErrNotExist) {
if err := safeio.WriteCanonicalNewPrivateFile(path, []byte(contents), 0o600); err != nil {
return fmt.Errorf("cannot create external credential template")
}
} else if _, err := safeio.ReadCanonicalPrivateRegular(path, 64<<10); err != nil {
return fmt.Errorf("existing credential template is not private or readable")
}
}
authDirectory := filepath.Join(directory, "auth")
if _, err := os.Lstat(filepath.Join(authDirectory, "auth.yaml")); err == nil {
if _, _, err := authconfig.Load(authDirectory); err != nil {
return fmt.Errorf("existing authentication documents are invalid; repair them explicitly")
}
return nil
}
installation := config.Installation{Authentication: config.Authentication{ConfigDirectory: authDirectory}}
if authconfig.Run(ctx, installation, []string{"configure", "--mode", "local", "--public-url", "http://localhost:8080", "--admin-user", "admin", "--password-file", filepath.Join(secrets, "admin-password")}, strings.NewReader(""), io.Discard, io.Discard) != 0 {
return fmt.Errorf("cannot prepare local administrator; inspect protected auth files and retry without replacing existing credentials")
}
return nil
}
@@ -0,0 +1,94 @@
// Package preparation owns installation-local documents before any runtime exists.
package preparation
import (
"fmt"
"path/filepath"
"strconv"
"strings"
"github.com/aritmolab/thothii/tools/tht/internal/safeio"
)
func Prepare(directory string) error {
if err := safeio.ValidateCanonicalPath(directory); err != nil {
return fmt.Errorf("choose an absolute canonical destination")
}
exists, err := safeio.PreflightPrivateDirectory(directory)
if err != nil || exists {
return fmt.Errorf("choose a new private directory with an existing parent; existing documents are never replaced")
}
if err := safeio.EnsurePrivateDirectory(directory); err != nil {
return fmt.Errorf("cannot create private preparation directory")
}
root := func(name string) string { return strconv.Quote(filepath.Join(directory, name)) }
installation := fmt.Sprintf(`# Installation schema v2; replace CHANGE_ME before validation.
# Paths refer to local preparation files, never to workspace Git.
schemaVersion: 2
profile: local
projectDirectory: %s
envFile: %s
shell: {mode: full, defaultLocale: en}
workspaceRepository:
remote: https://CHANGE_ME/workspaces.git
branch: main
access: https
authentication:
configDirectory: %s
modelCatalog:
defaults: {interaction: openai/gpt-4.1-mini}
embedding: {id: 'ollama/qwen3-embedding:0.6b', dimensions: 1024}
providers:
openai:
authentication: {mode: secret_env, apiKeyEnv: OPENAI_API_KEY}
session: {mode: pi_builtin}
models:
gpt-4.1-mini: {session: {}}
# Metadata generation is optional: omitted here. Configure its eligibility in this catalog.
# This Compose asset will come from the release; no application checkout is required here.
overrides: [%s]
`, strconv.Quote(directory), root("operator.env"), root("auth"), root("deploy/compose.git-https.yaml"))
environment := "# Non-secret paths and parameters; no interpolation or duplicate keys.\n"
for _, entry := range [][2]string{
{"COMPOSE_PROJECT_NAME", "thothii-local"}, {"THT_WORKSPACE_INSTALLATION_ID", "local"},
{"THT_WORKSPACE_GIT_REMOTE", "https://CHANGE_ME/workspaces.git"}, {"THT_WORKSPACE_GIT_BRANCH", "main"},
{"THT_INSTALLATION_CONFIG_SOURCE", filepath.Join(directory, "thothii-installation.yaml")},
{"THT_AUTH_CONFIG_ROOT", filepath.Join(directory, "auth")},
{"THT_SECRETS_FILE", filepath.Join(directory, "secrets", "secrets.env")},
{"PI_AUTH_FILE", filepath.Join(directory, "secrets", "pi-auth.json")},
{"THT_WORKSPACE_GIT_CREDENTIALS_FILE", filepath.Join(directory, "secrets", "git-credentials")},
{"THT_WORKSPACE_GIT_CA_FILE", filepath.Join(directory, "secrets", "git-ca.pem")},
{"THT_CATALOG_RUNTIME_PASSWORD_SOURCE", filepath.Join(directory, "secrets", "catalog-runtime-password")},
{"THT_CATALOG_MIGRATOR_PASSWORD_SOURCE", filepath.Join(directory, "secrets", "catalog-migrator-password")},
{"THOTH_HTTP_PORT", "8080"}, {"THOTH_CORE_HTTP_PORT", "8787"}, {"MAX_PI_PROCESSES", "4"},
} {
environment += entry[0] + "=" + strconv.Quote(entry[1]) + "\n"
}
bootstrap := fmt.Sprintf(`# Bootstrap input only; PostgreSQL Metadata Catalog remains the runtime authority.
schemaVersion: 1
databases:
- workspaceId: CHANGE_ME
engine: postgres
databaseName: CHANGE_ME
schema: public
binding:
transport: postgres_direct
host: CHANGE_ME
port: 5432
username: CHANGE_ME
secretFiles:
password: %s
`, root("secrets/database-password"))
documents := map[string]string{
".gitignore": "*\n",
"thothii-installation.yaml": installation, "operator.env": environment,
"database-bootstrap.yaml": bootstrap,
"README.md": "# Preparation / Preparazione\n\nReplace every CHANGE_ME / Sostituire ogni CHANGE_ME. Keep all files outside workspace Git / Tenere tutti i file fuori dal Git dei workspace.\n\n1. Edit installation models, workspace remote and operator.env consistently. / Modificare modelli, remoto e operator.env in modo coerente.\n2. Add one database bootstrap entry for every workspace; use read-only DWH credentials in protected files. / Una voce database per workspace, credenziali DWH in sola lettura in file protetti.\n3. Run tht installation credentials --directory PATH before setup; fill provider/database secrets yourself. / Generare credenziali tecniche prima del setup; compilare i segreti esterni.\n4. Repeat tht --installation PATH/thothii-installation.yaml installation validate --workspaces WORKSPACES. / Correggere e ripetere.\n\nNo services, network calls or database writes / Nessun servizio, chiamata di rete o scrittura database.\n",
}
for _, name := range []string{".gitignore", "thothii-installation.yaml", "operator.env", "database-bootstrap.yaml", "README.md"} {
if err := safeio.WriteCanonicalNewPrivateFile(filepath.Join(directory, name), []byte(strings.TrimSpace(documents[name])+"\n"), 0o600); err != nil {
return fmt.Errorf("cannot create preparation files; inspect the new directory and retry in a new destination")
}
}
return nil
}
@@ -0,0 +1,274 @@
package preparation
import (
"bytes"
"encoding/json"
"io"
"regexp"
"strconv"
"strings"
"github.com/aritmolab/thothii/tools/tht/internal/authconfig"
"github.com/aritmolab/thothii/tools/tht/internal/config"
"github.com/aritmolab/thothii/tools/tht/internal/safeio"
"gopkg.in/yaml.v3"
)
type Issue struct {
Document string `json:"document"`
Field string `json:"field"`
Code string `json:"code"`
Correction string `json:"correction"`
}
type Report struct {
SchemaVersion int `json:"schema_version"`
Scope string `json:"scope"`
OK bool `json:"ok"`
Issues []Issue `json:"issues"`
Warnings []string `json:"warnings"`
DeferredChecks []string `json:"deferred_checks"`
}
func NewReport() Report {
return Report{SchemaVersion: 1, Scope: "application-documents", Issues: []Issue{}, Warnings: []string{}, DeferredChecks: []string{"release-assets", "external-connectivity", "catalog-import", "runtime-readiness"}}
}
func (r *Report) Add(document, field, code, correction string) {
r.OK = false
r.Issues = append(r.Issues, Issue{document, field, code, correction})
}
func placeholder(value string) bool {
upper := strings.ToUpper(value)
return strings.Contains(upper, "CHANGE_ME") || strings.Contains(upper, "REPLACE_ME") || strings.Contains(upper, "YOUR_API_KEY") || strings.Contains(upper, "<PASSWORD>")
}
// CheckYAML refuses unresolved placeholders and ambiguous authored YAML without returning values.
func CheckYAML(path, document string, report *Report) bool {
content, err := safeio.ReadCanonicalPrivateRegular(path, 1<<20)
if err != nil {
report.Add(document, "$", "file_unavailable", "Use a readable, private regular document at a canonical absolute path (no links).")
return false
}
decoder := yaml.NewDecoder(bytes.NewReader(content))
var node, extra yaml.Node
if err := decoder.Decode(&node); err != nil || decoder.Decode(&extra) != io.EOF {
report.Add(document, "$", "yaml_invalid", "Keep one well-formed YAML document.")
return false
}
var walk func(*yaml.Node) bool
walk = func(n *yaml.Node) bool {
if n.Kind == yaml.AliasNode || n.Tag == "!!merge" {
return false
}
if n.Kind == yaml.ScalarNode && placeholder(n.Value) {
return false
}
if n.Kind == yaml.MappingNode {
seen := map[string]bool{}
for index := 0; index < len(n.Content); index += 2 {
key := n.Content[index]
if key.Kind != yaml.ScalarNode || seen[key.Value] {
return false
}
seen[key.Value] = true
}
}
for _, child := range n.Content {
if !walk(child) {
return false
}
}
return true
}
if !walk(&node) {
report.Add(document, "$", "incomplete_or_ambiguous", "Replace CHANGE_ME/REPLACE_ME placeholders, remove duplicate keys, aliases and YAML merge keys.")
return false
}
return true
}
var environmentKey = regexp.MustCompile(`^[A-Z][A-Z0-9_]*$`)
func checkEnvironment(path string, report *Report) bool {
contents, err := safeio.ReadCanonicalPrivateRegular(path, 1<<20)
if err != nil {
report.Add("operator.env", "$", "environment_unavailable", "Provide a private readable environment file.")
return false
}
seen := map[string]bool{}
for _, raw := range strings.Split(string(contents), "\n") {
line := strings.TrimSpace(raw)
if line == "" || strings.HasPrefix(line, "#") {
continue
}
key, value, found := strings.Cut(line, "=")
if !found || !environmentKey.MatchString(key) || seen[key] || placeholder(value) || strings.Contains(value, "$") {
report.Add("operator.env", "$", "environment_invalid", "Use one literal KEY=value per line, unique uppercase keys, and replace placeholders; shell interpolation is not accepted.")
return false
}
if strings.HasSuffix(key, "_PASSWORD") || strings.HasSuffix(key, "_API_KEY") {
report.Add("operator.env", "$", "inline_secret", "Move credential values to protected files and keep only file references in operator.env.")
return false
}
seen[key] = true
}
return true
}
func CheckSecret(path, document, field string, allowEmpty bool, report *Report) bool {
contents, err := safeio.ReadCanonicalPrivateRegular(path, 64<<10)
if err != nil || (!allowEmpty && len(bytes.TrimSpace(contents)) == 0) || placeholder(string(contents)) {
report.Add(document, field, "secret_unavailable", "Supply a non-placeholder private readable regular secret file (owner-only permissions, no links); do not put its contents in YAML or logs.")
return false
}
return true
}
func Validate(path string) (config.Installation, Report) {
report := NewReport()
if !CheckYAML(path, "thothii-installation.yaml", &report) {
return config.Installation{}, report
}
// Read only the location before the canonical loader checks every field.
contents, _ := safeio.ReadCanonicalPrivateRegular(path, 1<<20)
var location struct {
EnvFile string `yaml:"envFile"`
}
if yaml.Unmarshal(contents, &location) != nil {
report.Add("thothii-installation.yaml", "envFile", "schema_invalid", "Set envFile to one absolute path string, then correct the remaining descriptor fields.")
return config.Installation{}, report
}
if !checkEnvironment(location.EnvFile, &report) {
return config.Installation{}, report
}
installation, err := config.LoadPrepared(path)
if err != nil {
field, correction := "$", "Check installation schema v2, absolute paths, workspace remote/branch and matching environment values. Custom overrides and Git credential/CA files must exist."
if strings.Contains(err.Error(), "modelCatalog") {
field, correction = "modelCatalog", "Check interaction default eligibility, provider endpoints/authentication, embedding id/dimensions, and the private provider key bundle."
}
if strings.Contains(err.Error(), "authentication") {
field, correction = "authentication", "Match the authentication directory to operator.env and prepare its protected files."
}
report.Add("thothii-installation.yaml", field, "configuration_invalid", correction)
return config.Installation{}, report
}
value := func(key string) string { result, _ := installation.EnvironmentValue(key); return result }
for _, key := range []string{"COMPOSE_PROJECT_NAME", "THT_WORKSPACE_INSTALLATION_ID", "THT_INSTALLATION_CONFIG_SOURCE", "THT_SECRETS_FILE", "PI_AUTH_FILE", "THT_CATALOG_RUNTIME_PASSWORD_SOURCE", "THT_CATALOG_MIGRATOR_PASSWORD_SOURCE"} {
if value(key) == "" {
report.Add("operator.env", key, "required", "Supply this installation parameter or protected file reference before setup.")
}
}
if value("THT_INSTALLATION_CONFIG_SOURCE") != path {
report.Add("operator.env", "THT_INSTALLATION_CONFIG_SOURCE", "path_mismatch", "Point to the exact installation descriptor being validated.")
}
for _, key := range []string{"THOTH_HTTP_PORT", "THOTH_CORE_HTTP_PORT", "MAX_PI_PROCESSES"} {
number, err := strconv.Atoi(value(key))
if err != nil || number < 1 || number > 65535 {
report.Add("operator.env", key, "invalid_number", "Supply a positive integer; HTTP ports must be within 1..65535.")
}
}
if value("THOTH_HTTP_PORT") == value("THOTH_CORE_HTTP_PORT") {
report.Add("operator.env", "THOTH_CORE_HTTP_PORT", "port_collision", "Choose different frontend and core host ports.")
}
files, err := installation.SecretFiles()
if err != nil {
report.Add("operator.env", "$", "secret_references_invalid", "Use canonical absolute paths for all _FILE and _SOURCE references.")
}
for _, file := range files {
allowEmpty := file == value("THT_WORKSPACE_GIT_CREDENTIALS_FILE")
CheckSecret(file, "operator.env", "protected-file-reference", allowEmpty, &report)
}
auth, users, err := authconfig.Load(installation.AuthenticationDirectory())
if err != nil {
report.Add("auth/auth.yaml", "$", "authentication_invalid", "Prepare and validate local authentication documents before setup, including the initial administrator.")
} else if auth.Mode == "local" {
admin := false
for _, user := range users.Users {
for _, role := range user.Roles {
if user.Enabled && role == authconfig.RoleAdmin {
admin = true
}
}
}
if !admin {
report.Add("auth/users.yaml", "users", "administrator_missing", "Enable at least one administrator before setup.")
}
} else {
report.Add("auth/auth.yaml", "mode", "unsupported_bootstrap", "This preparation increment supports local administrative authentication; use the documented existing OIDC preparation path until its offline bootstrap validation is available.")
}
checkPiCredentials(value("PI_AUTH_FILE"), installation.ModelCatalog, &report)
report.OK = len(report.Issues) == 0
return installation, report
}
// Pi stores api_key or oauth records. Require literal prepared material here; model
// environment references belong to the catalog's secret_env path, not host process state.
func checkPiCredentials(path string, catalog config.ModelCatalog, report *Report) {
contents, err := safeio.ReadCanonicalPrivateRegular(path, 64<<10)
var credentials map[string]map[string]any
invalid := err != nil || json.Unmarshal(contents, &credentials) != nil || credentials == nil
literal := func(value any) bool {
text, ok := value.(string)
return ok && strings.TrimSpace(text) != "" && !strings.HasPrefix(text, "!") && !strings.Contains(text, "$") && !placeholder(text)
}
var declarative func(any) bool
declarative = func(value any) bool {
switch v := value.(type) {
case string:
return !strings.HasPrefix(v, "!")
case []any:
for _, item := range v {
if !declarative(item) {
return false
}
}
case map[string]any:
for _, item := range v {
if !declarative(item) {
return false
}
}
}
return true
}
seen := map[string]bool{}
for provider, record := range credentials {
name := strings.ToLower(strings.TrimSpace(provider))
if name == "" || provider != name || seen[name] || !declarative(record) {
invalid = true
}
seen[name] = true
switch record["type"] {
case "api_key":
if !literal(record["key"]) {
invalid = true
}
if env, present := record["env"]; present {
values, ok := env.(map[string]any)
if !ok {
invalid = true
}
for _, value := range values {
if _, ok := value.(string); !ok {
invalid = true
}
}
}
case "oauth":
expires, ok := record["expires"].(float64)
if !literal(record["access"]) || !literal(record["refresh"]) || !ok || expires <= 0 {
invalid = true
}
default:
invalid = true
}
}
for id, provider := range catalog.Providers {
if provider.Authentication.Mode == "pi_auth" && !seen[id] {
invalid = true
}
}
if invalid {
report.Add("pi-auth.json", "providers", "provider_auth_invalid", "Provide a JSON object with literal api_key/type+key or oauth/type+access+refresh+expires records for selected Pi providers; use catalog secret_env for environment-based keys. Commands and unresolved references are not accepted.")
}
}