feat(cli): prepare and validate application documents offline
This commit is contained in:
+5
-1
@@ -9,7 +9,11 @@ and invariants are in [AGENTS.md](AGENTS.md); prior snapshots remain in Git.
|
||||
and `tht workspace validate` through a native two-executable bundle. Build and
|
||||
test instructions are in [the host CLI guide](tools/tht/README.md). Local validation
|
||||
reuses runtime workspace/catalog parsers and explicitly defers runtime Evidence,
|
||||
database binding and readiness checks. The remainder of the installation tickets,
|
||||
database binding and readiness checks. Ticket #44 adds `tht installation prepare`,
|
||||
explicit `installation credentials`, and `installation validate --workspaces PATH`
|
||||
for protected application documents, canonical model settings and schema-v1
|
||||
database bootstrap inputs. These commands do not start services or import Catalog
|
||||
bindings. The remainder of the installation tickets,
|
||||
Docker Hub publication and example databases remain pending.
|
||||
|
||||
- React supports full/embedded rendering independently of local/OIDC/upstream auth,
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
import { readFileSync, lstatSync } from "node:fs";
|
||||
import { parseAllDocuments } from "yaml";
|
||||
import { decode, DocumentError, runWorkspaceDocuments } from "../workspaces/documents.js";
|
||||
import { validateDatabaseBootstrap } from "./bootstrap-documents.js";
|
||||
|
||||
/** Internal sibling protocol: only references cross back to Go, never secret contents. */
|
||||
export function runBootstrapValidation(args: string[]): { status: number; output: string } {
|
||||
try {
|
||||
if (args.length !== 5 || args[0] !== "--directory" || args[2] !== "--bootstrap" || args[4] !== "--json") throw new Error("usage");
|
||||
const checked = runWorkspaceDocuments(["validate", "--directory", args[1], "--json"]);
|
||||
if (checked.status !== 0) return checked;
|
||||
const info = lstatSync(args[3]);
|
||||
if (!info.isFile() || info.isSymbolicLink() || info.size > 1024 * 1024) throw new Error("file");
|
||||
const validated = decode(readFileSync(args[3], "utf8"), "database-bootstrap.yaml", (source) =>
|
||||
validateDatabaseBootstrap(parseAllDocuments(source)[0].toJSON(), args[1]), "database bootstrap schema v1 and the Catalog binding contract");
|
||||
return { status: 0, output: JSON.stringify({ schema_version: 1, ok: true, secret_files: validated.secretFiles, warnings: validated.warnings, issues: [] }) };
|
||||
} catch (error) {
|
||||
const issue = error instanceof DocumentError ? error.issue : { document: "database-bootstrap.yaml", field: "$", code: "bootstrap_invalid", correction: "Supply one complete Catalog database configuration per workspace in a readable local bootstrap document." };
|
||||
return { status: 1, output: JSON.stringify({ schema_version: 1, ok: false, issues: [issue] }) };
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
import { readFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { z } from "zod";
|
||||
import { databaseConfigurationSchema } from "./configuration-schema.js";
|
||||
import { parseWorkspaceCatalogYaml } from "../workspaces/catalog.js";
|
||||
import { parseWorkspaceYaml } from "../workspaces/schema.js";
|
||||
import { discoverWorkspaceSecretRequirements } from "../workspaces/secret-requirements.js";
|
||||
|
||||
const reference = z.string().min(1).max(4096);
|
||||
const database = databaseConfigurationSchema.extend({
|
||||
secretFiles: z.object({ password: reference.optional(), apiKey: reference.optional(), sshPrivateKey: reference.optional(), sshPrivateKeyPassphrase: reference.optional(), sshKnownHosts: reference.optional(), tlsCa: reference.optional() }).strict(),
|
||||
evidenceSecretFiles: z.object({ "evidence.signed_urls": reference.optional(), "evidence.access_key": reference.optional(), "evidence.secret_key": reference.optional(), "evidence.session_token": reference.optional() }).strict().optional(),
|
||||
});
|
||||
const bootstrap = z.object({ schemaVersion: z.literal(1), databases: z.array(database).min(1).max(1000) }).strict();
|
||||
|
||||
export interface BootstrapReference { field: string; path: string }
|
||||
|
||||
/** Offline bootstrap boundary: runtime Catalog owns the resulting bindings after import. */
|
||||
export function validateDatabaseBootstrap(value: unknown, workspaceRoot: string): { secretFiles: BootstrapReference[]; warnings: string[] } {
|
||||
const document = bootstrap.parse(value);
|
||||
const catalog = parseWorkspaceCatalogYaml(readFileSync(join(workspaceRoot, "thoth-workspaces.yaml"), "utf8"));
|
||||
const expected = new Set(catalog.workspaces.map((entry) => entry.id));
|
||||
const seen = new Set<string>();
|
||||
const secretFiles: BootstrapReference[] = [];
|
||||
const warnings: string[] = [];
|
||||
const issue = (path: (string | number)[], message: string): never => { throw new z.ZodError([{ code: "custom", path, message }]); };
|
||||
document.databases.forEach((entry, index) => {
|
||||
if (!expected.has(entry.workspaceId) || seen.has(entry.workspaceId)) issue(["databases", index, "workspaceId"], "Declare each catalog workspace exactly once.");
|
||||
seen.add(entry.workspaceId);
|
||||
const required = entry.binding.transport === "rest_api"
|
||||
? entry.binding.restAuth === "none" ? [] : ["apiKey"] as const
|
||||
: entry.binding.transport === "ssh_tunnel" ? ["password", "sshPrivateKey", "sshKnownHosts"] as const : ["password"] as const;
|
||||
for (const name of required) {
|
||||
if (!entry.secretFiles[name]) issue(["databases", index, "secretFiles", name], "Supply a protected file reference for this transport.");
|
||||
}
|
||||
if (entry.binding.transport === "ssh_tunnel") warnings.push(`databases.${index}:ssh_tunnel supports Catalog diagnostics, not NL-to-SQL sessions; choose direct or REST for practice.`);
|
||||
for (const [name, path] of Object.entries(entry.secretFiles)) secretFiles.push({ field: `databases.${index}.secretFiles.${name}`, path });
|
||||
const workspace = parseWorkspaceYaml(readFileSync(join(workspaceRoot, entry.workspaceId, "workspace.yaml"), "utf8"));
|
||||
const requirements = discoverWorkspaceSecretRequirements(workspace, {});
|
||||
for (const requirement of requirements.filter((item) => item.connector === "evidence" && item.required)) {
|
||||
if (!(entry.evidenceSecretFiles as Record<string, string> | undefined)?.[requirement.id]) issue(["databases", index, "evidenceSecretFiles"], "Supply the configured Evidence authentication file references.");
|
||||
}
|
||||
for (const [name, path] of Object.entries(entry.evidenceSecretFiles ?? {})) secretFiles.push({ field: `databases.${index}.evidenceSecretFiles.${name}`, path });
|
||||
});
|
||||
if (seen.size !== expected.size) issue(["databases"], "Add a database binding for every catalog workspace.");
|
||||
return { secretFiles, warnings };
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
import { z } from "zod";
|
||||
import { parseCredentialFreeHttpUrl } from "../auth/url-policy.js";
|
||||
import { DATABASE_TRANSPORTS } from "./types.js";
|
||||
|
||||
const workspaceIdSchema = z.string().regex(/^[a-z][a-z0-9-]{2,62}$/);
|
||||
const identifier = z.string().trim().min(1).max(128).regex(/^[A-Za-z_][A-Za-z0-9_$-]*$/);
|
||||
const nonEmpty = z.string().trim().min(1).max(512);
|
||||
const port = z.number().int().min(1).max(65_535);
|
||||
const optionalText = nonEmpty.optional();
|
||||
const sshHost = z.string().trim().min(1).max(255).regex(/^[A-Za-z0-9_.:\[\]-]+$/).optional();
|
||||
const sshUsername = z.string().trim().min(1).max(128).regex(/^[A-Za-z0-9._-]+$/).optional();
|
||||
const bindingSchema = z.object({
|
||||
transport: z.enum(DATABASE_TRANSPORTS),
|
||||
host: optionalText,
|
||||
port: port.optional(),
|
||||
username: optionalText,
|
||||
baseUrl: z.string().max(2048)
|
||||
.refine((value) => parseCredentialFreeHttpUrl(value) !== undefined)
|
||||
.optional(),
|
||||
restPath: z.string().regex(/^\/(?!\/)[^?#\\\u0000-\u001f]*$/).max(512).optional(),
|
||||
restAuth: z.enum(["none", "bearer", "x-api-key"]).optional(),
|
||||
tlsServername: optionalText,
|
||||
sshHost,
|
||||
sshPort: port.optional(),
|
||||
sshUsername,
|
||||
sshTargetHost: sshHost,
|
||||
sshTargetPort: port.optional(),
|
||||
}).strict().superRefine((binding, context) => {
|
||||
const required = binding.transport === "postgres_direct"
|
||||
? ["host", "port", "username"] as const
|
||||
: binding.transport === "rest_api"
|
||||
? ["baseUrl", "restPath", "restAuth"] as const
|
||||
: ["username", "sshHost", "sshPort", "sshUsername", "sshTargetHost", "sshTargetPort"] as const;
|
||||
for (const field of required) {
|
||||
if (binding[field] === undefined) context.addIssue({ code: "custom", path: [field], message: "Required" });
|
||||
}
|
||||
});
|
||||
export const databaseConfigurationSchema = z.object({
|
||||
workspaceId: workspaceIdSchema,
|
||||
engine: z.literal("postgres"),
|
||||
databaseName: identifier,
|
||||
schema: identifier,
|
||||
binding: bindingSchema,
|
||||
}).strict();
|
||||
@@ -1,60 +1,19 @@
|
||||
import type { FastifyInstance, FastifyReply, FastifyRequest } from "fastify";
|
||||
import { z } from "zod";
|
||||
import { isPrincipalContext, requirePermission } from "../auth/authorization.js";
|
||||
import { parseCredentialFreeHttpUrl } from "../auth/url-policy.js";
|
||||
import { databaseConfigurationSchema as configSchema } from "../catalog/configuration-schema.js";
|
||||
import { CatalogService, type CatalogSecretName } from "../catalog/service.js";
|
||||
import { WorkspaceRegistryError } from "../workspaces/git-repository.js";
|
||||
import {
|
||||
CatalogConflictError,
|
||||
CatalogOperationInProgressError,
|
||||
CatalogUnavailableError,
|
||||
DATABASE_TRANSPORTS,
|
||||
type CatalogRepository,
|
||||
type DatabaseConfigurationInput,
|
||||
} from "../catalog/types.js";
|
||||
import type { CatalogOperationCoordinator } from "../catalog/operation-coordinator.js";
|
||||
|
||||
const idSchema = z.uuid();
|
||||
const workspaceIdSchema = z.string().regex(/^[a-z][a-z0-9-]{2,62}$/);
|
||||
const identifier = z.string().trim().min(1).max(128).regex(/^[A-Za-z_][A-Za-z0-9_$-]*$/);
|
||||
const nonEmpty = z.string().trim().min(1).max(512);
|
||||
const port = z.number().int().min(1).max(65_535);
|
||||
const optionalText = nonEmpty.optional();
|
||||
const sshHost = z.string().trim().min(1).max(255).regex(/^[A-Za-z0-9_.:\[\]-]+$/).optional();
|
||||
const sshUsername = z.string().trim().min(1).max(128).regex(/^[A-Za-z0-9._-]+$/).optional();
|
||||
const bindingSchema = z.object({
|
||||
transport: z.enum(DATABASE_TRANSPORTS),
|
||||
host: optionalText,
|
||||
port: port.optional(),
|
||||
username: optionalText,
|
||||
baseUrl: z.string().max(2048)
|
||||
.refine((value) => parseCredentialFreeHttpUrl(value) !== undefined)
|
||||
.optional(),
|
||||
restPath: z.string().regex(/^\/(?!\/)[^?#\\\u0000-\u001f]*$/).max(512).optional(),
|
||||
restAuth: z.enum(["none", "bearer", "x-api-key"]).optional(),
|
||||
tlsServername: optionalText,
|
||||
sshHost,
|
||||
sshPort: port.optional(),
|
||||
sshUsername,
|
||||
sshTargetHost: sshHost,
|
||||
sshTargetPort: port.optional(),
|
||||
}).strict().superRefine((binding, context) => {
|
||||
const required = binding.transport === "postgres_direct"
|
||||
? ["host", "port", "username"] as const
|
||||
: binding.transport === "rest_api"
|
||||
? ["baseUrl", "restPath", "restAuth"] as const
|
||||
: ["username", "sshHost", "sshPort", "sshUsername", "sshTargetHost", "sshTargetPort"] as const;
|
||||
for (const field of required) {
|
||||
if (binding[field] === undefined) context.addIssue({ code: "custom", path: [field], message: "Required" });
|
||||
}
|
||||
});
|
||||
const configSchema = z.object({
|
||||
workspaceId: workspaceIdSchema,
|
||||
engine: z.literal("postgres"),
|
||||
databaseName: identifier,
|
||||
schema: identifier,
|
||||
binding: bindingSchema,
|
||||
}).strict();
|
||||
const updateSchema = configSchema.extend({ version: z.number().int().positive() });
|
||||
const secretNames = [
|
||||
"password",
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
/** Compiled with its runtime for the host CLI: no installation, Docker or host Node required. */
|
||||
import { runWorkspaceDocuments } from "./workspaces/documents.js";
|
||||
import { runBootstrapValidation } from "./catalog/bootstrap-cli.js";
|
||||
|
||||
const result = runWorkspaceDocuments(process.argv.slice(2));
|
||||
const args = process.argv.slice(2);
|
||||
const result = args[0] === "bootstrap" ? runBootstrapValidation(args.slice(1)) : runWorkspaceDocuments(args);
|
||||
console.log(result.output);
|
||||
process.exitCode = result.status;
|
||||
|
||||
@@ -17,7 +17,7 @@ interface Report {
|
||||
const MAX_DOCUMENT_BYTES = 1024 * 1024;
|
||||
const usage = "tht workspace prepare --directory NEW_PATH --id ID --name NAME [--language en|it] [--json]\ntht workspace validate --directory PATH [--json]";
|
||||
|
||||
class DocumentError extends Error {
|
||||
export class DocumentError extends Error {
|
||||
constructor(readonly issue: Issue) { super(issue.correction); }
|
||||
}
|
||||
function fail(document: string, field: string, code: string, correction: string): never {
|
||||
@@ -25,7 +25,7 @@ function fail(document: string, field: string, code: string, correction: string)
|
||||
}
|
||||
|
||||
/** Never include parser messages or submitted values: YAML and Zod errors can contain secrets. */
|
||||
function decode<T>(source: string, document: string, parser: (text: string) => T): T {
|
||||
export function decode<T>(source: string, document: string, parser: (text: string) => T, contract = "workspace schema v4 or catalog schema v1"): T {
|
||||
try {
|
||||
const documents = parseAllDocuments(source, { uniqueKeys: true });
|
||||
if (documents.length !== 1) fail(document, "$", "yaml_documents", "Keep exactly one YAML document in this file.");
|
||||
@@ -43,7 +43,7 @@ function decode<T>(source: string, document: string, parser: (text: string) => T
|
||||
// Strict schemas produce paths containing schema-defined keys and array indices only.
|
||||
fail(document, issue.path.join(".") || "$", "schema_invalid",
|
||||
issue.code === "unrecognized_keys" ? "Remove fields not defined by the current workspace/catalog contract."
|
||||
: "Correct this field using workspace schema v4 or catalog schema v1; check type, required value, uniqueness and allowed values.");
|
||||
: `Correct this field using ${contract}; check type, required value, uniqueness and allowed values.`);
|
||||
}
|
||||
fail(document, "$", "schema_invalid", "Use an authored workspace v4 descriptor; remove database configuration and keep it in the PostgreSQL Metadata Catalog.");
|
||||
}
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
import { afterEach, expect, test } from "vitest";
|
||||
import { validateDatabaseBootstrap } from "../src/catalog/bootstrap-documents.js";
|
||||
import { runBootstrapValidation } from "../src/catalog/bootstrap-cli.js";
|
||||
|
||||
const roots: string[] = [];
|
||||
afterEach(() => roots.splice(0).forEach((root) => rmSync(root, { recursive: true, force: true })));
|
||||
function fixture() {
|
||||
const root = mkdtempSync(join(tmpdir(), "bootstrap-documents-")); roots.push(root);
|
||||
mkdirSync(join(root, "practice"));
|
||||
writeFileSync(join(root, "thoth-workspaces.yaml"), "schema_version: 1\nworkspaces: [{id: practice, name: Practice}]\n");
|
||||
writeFileSync(join(root, "practice/workspace.yaml"), "workspace: {schema_version: 4, id: practice, name: Practice, language: en}\n");
|
||||
return root;
|
||||
}
|
||||
const entry = { workspaceId: "practice", engine: "postgres", databaseName: "practice", schema: "public", binding: { transport: "postgres_direct", host: "db.internal", port: 5432, username: "reader" }, secretFiles: { password: "/private/operator/db-password" } };
|
||||
|
||||
test("bootstrap reuses Catalog configuration and requires one complete binding per workspace", () => {
|
||||
const root = fixture();
|
||||
const valid = validateDatabaseBootstrap({ schemaVersion: 1, databases: [entry] }, root);
|
||||
expect(valid.secretFiles).toContainEqual({ field: "databases.0.secretFiles.password", path: "/private/operator/db-password" });
|
||||
for (const databases of [[], [entry, entry], [{ ...entry, workspaceId: "unknown" }], [{ ...entry, secretFiles: {} }], [{ ...entry, engine: "mysql" }], [{ ...entry, binding: { ...entry.binding, port: 70000 } }]]) {
|
||||
expect(() => validateDatabaseBootstrap({ schemaVersion: 1, databases }, root)).toThrow();
|
||||
}
|
||||
});
|
||||
|
||||
test("REST authentication, SSH credentials and optional Evidence use their declared contracts", () => {
|
||||
const root = fixture();
|
||||
const rest = { ...entry, binding: { transport: "rest_api", baseUrl: "https://data.internal", restPath: "/query", restAuth: "none" }, secretFiles: {} };
|
||||
expect(validateDatabaseBootstrap({ schemaVersion: 1, databases: [rest] }, root).secretFiles).toEqual([]);
|
||||
expect(() => validateDatabaseBootstrap({ schemaVersion: 1, databases: [{ ...rest, binding: { ...rest.binding, restAuth: "bearer" } }] }, root)).toThrow();
|
||||
const ssh = { ...entry, binding: { transport: "ssh_tunnel", username: "reader", sshHost: "bastion", sshPort: 22, sshUsername: "tunnel", sshTargetHost: "database", sshTargetPort: 5432 }, secretFiles: { password: "/password", sshPrivateKey: "/key", sshKnownHosts: "/hosts" } };
|
||||
expect(validateDatabaseBootstrap({ schemaVersion: 1, databases: [ssh] }, root).warnings[0]).toContain("not NL-to-SQL");
|
||||
writeFileSync(join(root, "practice/workspace.yaml"), "workspace: {schema_version: 4, id: practice, name: Practice, language: en}\nevidence:\n source:\n type: http\n uris: [https://docs.internal/manual.md]\n authentication: signed_urls_file\n");
|
||||
expect(() => validateDatabaseBootstrap({ schemaVersion: 1, databases: [entry] }, root)).toThrow();
|
||||
expect(validateDatabaseBootstrap({ schemaVersion: 1, databases: [{ ...entry, evidenceSecretFiles: { "evidence.signed_urls": "/urls.json" } }] }, root).secretFiles).toContainEqual({ field: "databases.0.evidenceSecretFiles.evidence.signed_urls", path: "/urls.json" });
|
||||
});
|
||||
|
||||
test("bootstrap CLI never echoes arbitrary keys from submitted secret references", () => {
|
||||
const root = fixture();
|
||||
const path = join(root, "bootstrap.yaml");
|
||||
for (const field of ["secretFiles", "evidenceSecretFiles"]) {
|
||||
writeFileSync(path, JSON.stringify({ schemaVersion: 1, databases: [{ ...entry, [field]: { PRIVATE_CREDENTIAL_SENTINEL: "/path" } }] }));
|
||||
const result = runBootstrapValidation(["--directory", root, "--bootstrap", path, "--json"]);
|
||||
expect(result.status).toBe(1);
|
||||
expect(result.output).not.toContain("PRIVATE_CREDENTIAL_SENTINEL");
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,67 @@
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { chmodSync, mkdtempSync, readFileSync, realpathSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { rootCertificates } from "node:tls";
|
||||
import { afterEach, expect, test } from "vitest";
|
||||
|
||||
const binary = process.env.THT_INSTALLATION_TEST_CLI;
|
||||
const roots: string[] = [];
|
||||
afterEach(() => roots.splice(0).forEach((path) => rmSync(path, { recursive: true, force: true })));
|
||||
function run(...args: string[]) {
|
||||
return spawnSync(binary!, [...args, "--json"], { encoding: "utf8", env: { ...process.env, PATH: "" }, input: "" });
|
||||
}
|
||||
|
||||
test.skipIf(!binary)("operator prepares, completes and repeatedly validates before any stack exists", () => {
|
||||
const root = realpathSync(mkdtempSync(join(tmpdir(), "application-documents-"))); roots.push(root);
|
||||
const workspace = join(root, "workspaces"), directory = join(root, "installation");
|
||||
expect(run("workspace", "prepare", "--directory", workspace, "--id", "practice", "--name", "Practice").status).toBe(0);
|
||||
expect(run("installation", "prepare", "--directory", directory).status).toBe(0);
|
||||
const installation = join(directory, "thothii-installation.yaml");
|
||||
const validate = () => run("--installation", installation, "installation", "validate", "--workspaces", workspace);
|
||||
expect(validate().status).toBe(1); // Visible placeholders cannot be approved.
|
||||
expect(run("installation", "credentials", "--directory", directory).status).toBe(0);
|
||||
for (const name of ["thothii-installation.yaml", "operator.env", "database-bootstrap.yaml"]) {
|
||||
const path = join(directory, name);
|
||||
writeFileSync(path, readFileSync(path, "utf8").replaceAll("CHANGE_ME", "practice"));
|
||||
}
|
||||
for (const [name, contents] of Object.entries({ "secrets.env": "OPENAI_API_KEY=PRIVATE_PROVIDER_VALUE\n", "database-password": "PRIVATE_DATABASE_VALUE", "git-credentials": "", "git-ca.pem": rootCertificates[0] })) {
|
||||
writeFileSync(join(directory, "secrets", name), contents, { mode: 0o600 });
|
||||
}
|
||||
const before = readFileSync(installation, "utf8");
|
||||
for (let index = 0; index < 2; index++) {
|
||||
const checked = validate();
|
||||
expect(checked.stderr).toBe("");
|
||||
expect(checked.stdout).not.toContain("PRIVATE_");
|
||||
expect(checked.status, checked.stdout).toBe(0);
|
||||
expect(JSON.parse(checked.stdout).deferred_checks).toContain("release-assets");
|
||||
}
|
||||
expect(readFileSync(installation, "utf8")).toBe(before);
|
||||
writeFileSync(installation, before.replace("interaction: openai/gpt-4.1-mini", "interaction: openai/nonexistent"));
|
||||
expect(validate().status).toBe(1);
|
||||
writeFileSync(installation, before);
|
||||
const authFile = join(directory, "secrets/pi-auth.json");
|
||||
writeFileSync(authFile, "not-json");
|
||||
expect(validate().status).toBe(1);
|
||||
writeFileSync(installation, before.replace("{mode: secret_env, apiKeyEnv: OPENAI_API_KEY}", "{mode: pi_auth}"));
|
||||
writeFileSync(authFile, JSON.stringify({ openai: { unrelated: true } }));
|
||||
expect(validate().status).toBe(1);
|
||||
writeFileSync(authFile, JSON.stringify({ " OpenAI ": { type: "api_key", key: "PRIVATE_PI_KEY" } }));
|
||||
expect(validate().status).toBe(1);
|
||||
writeFileSync(authFile, JSON.stringify({ openai: { type: "api_key", key: "PRIVATE_PI_KEY" } }));
|
||||
expect(validate().status).toBe(0);
|
||||
writeFileSync(installation, before);
|
||||
writeFileSync(authFile, "{}");
|
||||
const bootstrap = join(directory, "database-bootstrap.yaml");
|
||||
const originalBootstrap = readFileSync(bootstrap, "utf8");
|
||||
writeFileSync(join(workspace, "practice", "private-password"), "PRIVATE_DATABASE_VALUE", { mode: 0o600 });
|
||||
writeFileSync(bootstrap, originalBootstrap.replace(join(directory, "secrets/database-password"), join(workspace, "practice/private-password")));
|
||||
expect(validate().status).toBe(1);
|
||||
writeFileSync(bootstrap, originalBootstrap);
|
||||
chmodSync(join(directory, "secrets/database-password"), 0o644);
|
||||
expect(validate().status).toBe(1);
|
||||
chmodSync(join(directory, "secrets/database-password"), 0o600);
|
||||
const env = join(directory, "operator.env");
|
||||
writeFileSync(env, readFileSync(env, "utf8") + "THOTH_HTTP_PORT=8081\n");
|
||||
expect(validate().status).toBe(1);
|
||||
}, 15_000);
|
||||
@@ -64,6 +64,94 @@ success does not certify semantic truth, connectivity or readiness. The Git revi
|
||||
activated later must contain the checked documents; this command does not publish
|
||||
uncommitted files or empty directories.
|
||||
|
||||
## Prepare and validate application documents
|
||||
|
||||
After validating workspaces, create a local directory **outside their repository**:
|
||||
|
||||
```sh
|
||||
tht installation prepare --directory ./my-installation
|
||||
```
|
||||
|
||||
This creates private, commented `thothii-installation.yaml`, `operator.env`,
|
||||
`database-bootstrap.yaml` and `README.md`. The destination must be new and its parent
|
||||
must exist. It starts no services and does not implicitly generate passwords.
|
||||
|
||||
1. Choose models and providers in the descriptor. The template proposes
|
||||
`openai/gpt-4.1-mini` for interaction and `ollama/qwen3-embedding:0.6b` with 1024
|
||||
dimensions for embedding. Edit these before setup. `modelCatalog.defaults.interaction`
|
||||
must support sessions and metadata generation when the latter is configured.
|
||||
The template omits optional metadata generation. See
|
||||
[model configuration](../general/pi-configuration.md) for custom providers.
|
||||
2. Replace the Git remote in both the descriptor and `operator.env`; keep branch and
|
||||
transport consistent. Paths are absolute and machine-local. `operator.env` accepts
|
||||
one literal `KEY=value` assignment per line, without duplicate keys or shell
|
||||
interpolation. Credentials belong in referenced protected files.
|
||||
3. Complete `database-bootstrap.yaml` with exactly one entry per workspace. A complete
|
||||
direct-connection example is:
|
||||
|
||||
```yaml
|
||||
schemaVersion: 1
|
||||
databases:
|
||||
- workspaceId: practice
|
||||
engine: postgres
|
||||
databaseName: sales
|
||||
schema: public
|
||||
binding:
|
||||
transport: postgres_direct
|
||||
host: db.intranet
|
||||
port: 5432
|
||||
username: thoth_reader
|
||||
secretFiles:
|
||||
password: /private/path/my-installation/secrets/database-password
|
||||
```
|
||||
|
||||
Use a read-only DWH account. `rest_api` requires `baseUrl`, `restPath`, `restAuth`
|
||||
(`none`, `bearer`, `x-api-key`) and `secretFiles.apiKey` when authenticated.
|
||||
`ssh_tunnel` requires `username`, `sshHost`, `sshPort`, `sshUsername`,
|
||||
`sshTargetHost`, `sshTargetPort` and files `password`, `sshPrivateKey`,
|
||||
`sshKnownHosts`; it supports Catalog diagnostics, not NL-to-SQL sessions.
|
||||
`tlsCa` and `sshPrivateKeyPassphrase` are optional. Signed HTTP Evidence needs
|
||||
`evidenceSecretFiles` with key `evidence.signed_urls`; static S3 credentials need
|
||||
`evidence.access_key`, `evidence.secret_key` and optional `evidence.session_token`.
|
||||
All values are private file paths. Workspace descriptors remain schema v4;
|
||||
this bootstrap input is not a second runtime Catalog.
|
||||
4. Explicitly generate technical credentials in the standard layout:
|
||||
|
||||
```sh
|
||||
tht installation credentials --directory ./my-installation
|
||||
```
|
||||
|
||||
This creates separate random Catalog runtime/migrator and administrator passwords,
|
||||
`auth/auth.yaml`, `auth/users.yaml`, a `secrets/secrets.env` template and
|
||||
`secrets/pi-auth.json`. Existing files are retained; invalid ones stop the command.
|
||||
The initial administrator is `admin`; its password stays in private
|
||||
`secrets/admin-password` and is never printed. The default is local authentication
|
||||
at `http://localhost:8080`: review and edit `auth/auth.yaml` before validation.
|
||||
This increment does not validate offline OIDC bootstrap for the existing path.
|
||||
5. Fill the provider key in `secrets/secrets.env` and create the DWH password file.
|
||||
For Git HTTPS supply the referenced credentials and CA files; empty credentials
|
||||
are allowed for a public remote, and the CA file must be available. For SSH supply
|
||||
a key and known_hosts and select the matching descriptor override. `pi_auth`
|
||||
providers require prepared Pi credentials. Keep every secret outside workspace
|
||||
Git with installer-only access (0600 on Unix, equivalent Windows ACLs).
|
||||
6. Validate and repeat after each correction:
|
||||
|
||||
```sh
|
||||
tht --installation /absolute/path/my-installation/thothii-installation.yaml installation validate --workspaces /absolute/path/my-workspaces --json
|
||||
```
|
||||
|
||||
The default bootstrap is beside the descriptor; `--bootstrap PATH` selects another.
|
||||
Validation changes no documents, generates no projections, uses no network and
|
||||
writes no database. It rejects placeholders, inconsistencies, missing/non-private
|
||||
files and secrets inside workspace Git. Reports identify document, field and
|
||||
correction without secret values. Exit statuses: 0 local success, 1 corrections
|
||||
needed, 2 invalid arguments.
|
||||
|
||||
Standard release Compose assets may still be absent at this stage; custom overrides
|
||||
must already exist. Release assets, external connectivity, Catalog import and runtime
|
||||
readiness remain explicit deferred checks. Success prepares the next preflight;
|
||||
it neither skips those checks nor establishes a completed installation.
|
||||
|
||||
## Before you start: the two repositories
|
||||
|
||||
There are two separate repositories:
|
||||
|
||||
@@ -65,6 +65,99 @@ Il successo locale non certifica verità semantica, connettività o readiness. L
|
||||
revisione Git attivata in seguito deve contenere i documenti verificati; il comando
|
||||
non pubblica file non committati o directory vuote.
|
||||
|
||||
## Predisporre e verificare i documenti applicativi
|
||||
|
||||
Dopo la verifica dei workspace, creare una cartella locale **esterna al loro repository**:
|
||||
|
||||
```sh
|
||||
tht installation prepare --directory ./mia-installazione
|
||||
```
|
||||
|
||||
Il comando crea file privati commentati: `thothii-installation.yaml`, `operator.env`,
|
||||
`database-bootstrap.yaml` e `README.md`. La cartella deve essere nuova e il padre
|
||||
deve esistere. Non avvia servizi e non genera implicitamente password.
|
||||
|
||||
1. Nel descrittore, scegliere modelli e provider. Il template propone
|
||||
`openai/gpt-4.1-mini` per l'interazione e `ollama/qwen3-embedding:0.6b` con 1024
|
||||
dimensioni per l'embedding. Sono valori modificabili, non una selezione richiesta
|
||||
durante il setup. `modelCatalog.defaults.interaction` deve essere utilizzabile
|
||||
nelle sessioni e anche nella generazione metadati, se quest'ultima è configurata.
|
||||
Il template omette la generazione metadati, che è facoltativa. Consultare la
|
||||
[configurazione dei modelli](../general/pi-configuration.md) per provider personalizzati.
|
||||
2. Sostituire il remoto Git sia nel descrittore sia in `operator.env`; mantenere
|
||||
coerenti branch e trasporto. I percorsi sono assoluti e riferiti a questa macchina.
|
||||
`operator.env` accetta una sola assegnazione letterale `KEY=value` per riga, senza
|
||||
duplicati o interpolazioni shell. Le credenziali restano nei file referenziati.
|
||||
3. Compilare `database-bootstrap.yaml`: una voce per ciascun workspace, senza
|
||||
duplicati. Questo esempio mostra il contratto completo di un collegamento diretto:
|
||||
|
||||
```yaml
|
||||
schemaVersion: 1
|
||||
databases:
|
||||
- workspaceId: pratica
|
||||
engine: postgres
|
||||
databaseName: vendite
|
||||
schema: public
|
||||
binding:
|
||||
transport: postgres_direct
|
||||
host: db.intranet
|
||||
port: 5432
|
||||
username: thoth_reader
|
||||
secretFiles:
|
||||
password: /percorso/privato/mia-installazione/secrets/database-password
|
||||
```
|
||||
|
||||
Usare le credenziali di un utente DWH in sola lettura. Per `rest_api`, il binding
|
||||
richiede `baseUrl`, `restPath` e `restAuth` (`none`, `bearer`, `x-api-key`); quando
|
||||
serve autenticazione, aggiungere `secretFiles.apiKey`. `ssh_tunnel` richiede
|
||||
`username`, `sshHost`, `sshPort`, `sshUsername`, `sshTargetHost`, `sshTargetPort` e
|
||||
i file `password`, `sshPrivateKey`, `sshKnownHosts`; abilita diagnostica Catalog,
|
||||
non sessioni NL→SQL. Sono facoltativi `tlsCa` e `sshPrivateKeyPassphrase`.
|
||||
Le Evidence HTTP firmate richiedono `evidenceSecretFiles` con chiave
|
||||
`evidence.signed_urls`; S3 con credenziali statiche richiede `evidence.access_key`
|
||||
e `evidence.secret_key`, con `evidence.session_token` facoltativo. Tutti i valori
|
||||
sono percorsi di file privati. I workspace rimangono nello schema v4: il bootstrap
|
||||
è un input iniziale, non un secondo Catalog runtime.
|
||||
4. Generare esplicitamente le credenziali tecniche nel layout standard:
|
||||
|
||||
```sh
|
||||
tht installation credentials --directory ./mia-installazione
|
||||
```
|
||||
|
||||
Sono creati password casuali separate per Catalog runtime/migrator e amministratore,
|
||||
il relativo `auth/auth.yaml` con `auth/users.yaml`, un template `secrets/secrets.env`
|
||||
e `secrets/pi-auth.json`. I file esistenti vengono conservati; se non validi, il
|
||||
comando si ferma. L'amministratore iniziale è `admin`, la password è nel file
|
||||
privato `secrets/admin-password` e non viene stampata. Il default è autenticazione
|
||||
locale con URL pubblico `http://localhost:8080`: verificare e, se necessario,
|
||||
modificare `auth/auth.yaml` prima del controllo. Questo incremento non valida il
|
||||
bootstrap OIDC del percorso esistente.
|
||||
5. Inserire la chiave del provider in `secrets/secrets.env` e creare il file della
|
||||
password DWH. Per HTTPS Git fornire i file referenziati per credenziali e CA:
|
||||
credenziali vuote sono ammesse per un remoto pubblico, la CA deve essere disponibile.
|
||||
Per SSH fornire chiave e known_hosts, scegliendo il relativo override nel descrittore.
|
||||
I provider `pi_auth` richiedono credenziali Pi già preparate. Conservare tutti
|
||||
questi file fuori dal repository workspace e proteggere l'accesso al solo utente
|
||||
installatore (0600 su Unix, ACL equivalenti su Windows). Non committarli.
|
||||
6. Verificare e ripetere dopo ogni correzione:
|
||||
|
||||
```sh
|
||||
tht --installation /percorso/assoluto/mia-installazione/thothii-installation.yaml installation validate --workspaces /percorso/assoluto/miei-workspace --json
|
||||
```
|
||||
|
||||
Il bootstrap viene cercato accanto al descrittore; `--bootstrap PERCORSO` ne
|
||||
seleziona uno diverso. Il controllo non cambia documenti, non genera proiezioni,
|
||||
non usa la rete e non scrive database. Rifiuta placeholder, incoerenze, file
|
||||
mancanti/non privati e segreti situati nel repository workspace. Il report indica
|
||||
documento, campo e correzione senza valori riservati. Exit status: 0 successo
|
||||
locale, 1 correzioni necessarie, 2 argomenti errati.
|
||||
|
||||
Gli asset Compose standard del rilascio possono ancora mancare in questa fase;
|
||||
gli override personalizzati devono già esistere. Il report distingue i controlli
|
||||
differiti: asset del rilascio, connettività esterna, import Catalog e readiness.
|
||||
Un esito positivo prepara il successivo preflight: non autorizza a saltare tali
|
||||
controlli e non equivale a un'installazione completata.
|
||||
|
||||
## Prima di iniziare: i due repository
|
||||
|
||||
Servono due repository distinti:
|
||||
|
||||
@@ -24,7 +24,18 @@ saltato, 1.417 test passati e 40 saltati; typecheck e build rigorosa documentazi
|
||||
passati. Suite Go: tutti i pacchetti passati, salvo un primo errore intermittente
|
||||
nel test di concorrenza authstorage; quel test passa in tre ripetizioni e il pacchetto
|
||||
completo passa nella verifica isolata. Le revisioni Standards e Spec non lasciano
|
||||
finding aperti. Il prossimo incremento sequenziale è T02/#44.
|
||||
finding aperti.
|
||||
|
||||
T02/#44 implementato sullo stesso branch: `tht installation prepare`, generazione
|
||||
esplicita delle credenziali tecniche e `tht installation validate` preparano e
|
||||
controllano documenti privati, modelli, autenticazione e bootstrap dei binding,
|
||||
riusando gli schemi runtime senza avviare servizi. Guide IT/EN aggiornate.
|
||||
Verifica backend completa su Node 24.16: 111 file passati, uno saltato, 1.420 test
|
||||
passati e 40 saltati; le regressioni successive della revisione passano nella suite
|
||||
mirata (quattro test, incluso il percorso con binari nativi e `PATH` vuoto).
|
||||
Typecheck, build rigorosa documentazione e pacchetti Go passati; il pacchetto CLI
|
||||
è stato ripetuto dopo la correzione rilevata in revisione. Nessun finding residuo
|
||||
delle revisioni Standards/Spec. Il prossimo incremento sequenziale è T03/#45.
|
||||
|
||||
| Ticket | Issue Gitea | Dipendenze dirette |
|
||||
| --- | --- | --- |
|
||||
|
||||
@@ -41,6 +41,40 @@ THT_WORKSPACE_TEST_CLI=/absolute/path/dist/workspace-tools/darwin-arm64/tht \
|
||||
npx vitest run test/workspace-documents-cli.test.ts
|
||||
```
|
||||
|
||||
## Application document preparation (issue #44)
|
||||
|
||||
Before installation, use `tht installation prepare --directory NEW_PATH`, then
|
||||
`tht installation credentials --directory PATH` to explicitly create technical
|
||||
credentials in the default protected layout. The latter preserves existing files.
|
||||
Edit the documents and external credentials, then repeat:
|
||||
|
||||
```sh
|
||||
tht --installation /absolute/path/thothii-installation.yaml \
|
||||
installation validate --workspaces /absolute/path/workspaces --json
|
||||
```
|
||||
|
||||
`database-bootstrap.yaml` beside the descriptor is a schemaVersion-1 bootstrap
|
||||
input, not a runtime Catalog. Its database entries use the exact Catalog API
|
||||
configuration schema plus private `secretFiles` and optional `evidenceSecretFiles`
|
||||
references. `--bootstrap` selects another document. The helper validates workspace
|
||||
membership, uniqueness, supported transports and required credential references;
|
||||
Go checks protected files, the canonical Installation Model Catalog, environment
|
||||
and local administrator. No process contacts a service or creates projections.
|
||||
Only missing standard release Compose assets are deferred by `config.LoadPrepared`;
|
||||
normal runtime `config.Load` retains all existing checks. Custom overrides must exist.
|
||||
|
||||
The native integration test is opt-in because it requires the built pair:
|
||||
|
||||
```sh
|
||||
THT_INSTALLATION_TEST_CLI=/absolute/path/dist/workspace-tools/darwin-arm64/tht \
|
||||
npx vitest run test/installation-documents-cli.test.ts
|
||||
```
|
||||
|
||||
Run it from `backend/`, following the bundle build above. It supplies prepared
|
||||
fixtures and removes host tools from the subprocess PATH. Platform acceptance and
|
||||
real external credentials remain separate gates. See the IT/EN guides for the
|
||||
complete parameter collection procedure, default `admin` account and local-auth scope.
|
||||
|
||||
## Shell configuration
|
||||
|
||||
The schema-v2 `thothii-installation.yaml` accepts this optional section:
|
||||
|
||||
@@ -0,0 +1,129 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"github.com/aritmolab/thothii/tools/tht/internal/preparation"
|
||||
)
|
||||
|
||||
func installationDocumentsCommand(ctx context.Context, installationPath string, args []string, stdout io.Writer) int {
|
||||
report := preparation.NewReport()
|
||||
status := 0
|
||||
options := map[string]string{}
|
||||
valid := len(args) > 0
|
||||
for index := 1; index < len(args); index++ {
|
||||
key := args[index]
|
||||
if _, duplicate := options[key]; duplicate {
|
||||
valid = false
|
||||
break
|
||||
}
|
||||
if key == "--json" {
|
||||
options[key] = "true"
|
||||
continue
|
||||
}
|
||||
if (key != "--directory" && key != "--workspaces" && key != "--bootstrap") || index+1 == len(args) {
|
||||
valid = false
|
||||
break
|
||||
}
|
||||
options[key] = args[index+1]
|
||||
index++
|
||||
}
|
||||
if !valid || (args[0] == "validate" && (installationPath == "" || options["--workspaces"] == "" || options["--directory"] != "")) || (args[0] != "validate" && (options["--directory"] == "" || options["--workspaces"] != "" || options["--bootstrap"] != "")) {
|
||||
report.Add("CLI", "$", "usage", "Use installation prepare|credentials --directory PATH [--json], or tht --installation ABSOLUTE_PATH installation validate --workspaces PATH [--bootstrap PATH] [--json].")
|
||||
status = 2
|
||||
} else if args[0] == "validate" {
|
||||
installation, checkedReport := preparation.Validate(installationPath)
|
||||
report = checkedReport
|
||||
if report.OK {
|
||||
bootstrap := options["--bootstrap"]
|
||||
if bootstrap == "" {
|
||||
bootstrap = filepath.Join(filepath.Dir(installationPath), "database-bootstrap.yaml")
|
||||
}
|
||||
bootstrap, _ = filepath.Abs(bootstrap)
|
||||
workspace, _ := filepath.Abs(options["--workspaces"])
|
||||
if resolved, err := filepath.EvalSymlinks(workspace); err == nil {
|
||||
workspace = resolved
|
||||
}
|
||||
outsideWorkspace := func(path, document, field string) bool {
|
||||
relative, err := filepath.Rel(workspace, path)
|
||||
if err == nil && relative != ".." && !strings.HasPrefix(relative, ".."+string(filepath.Separator)) {
|
||||
report.Add(document, field, "installation_file_in_workspace", "Keep installation documents and credential files outside the shared workspace repository.")
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
for _, path := range []string{installationPath, installation.EnvFile, installation.AuthenticationDirectory(), bootstrap} {
|
||||
outsideWorkspace(path, "thothii-installation.yaml", "local-files")
|
||||
}
|
||||
files, _ := installation.SecretFiles()
|
||||
for _, path := range files {
|
||||
outsideWorkspace(path, "operator.env", "protected-file-reference")
|
||||
}
|
||||
if preparation.CheckYAML(bootstrap, "database-bootstrap.yaml", &report) {
|
||||
var output, discarded bytes.Buffer
|
||||
code := workspaceDocumentsCommand(ctx, []string{"bootstrap", "--directory", workspace, "--bootstrap", bootstrap, "--json"}, &output, &discarded)
|
||||
var checked struct {
|
||||
OK bool `json:"ok"`
|
||||
Issues []preparation.Issue `json:"issues"`
|
||||
Warnings []string `json:"warnings"`
|
||||
SecretFiles []struct {
|
||||
Field string `json:"field"`
|
||||
Path string `json:"path"`
|
||||
} `json:"secret_files"`
|
||||
}
|
||||
if json.Unmarshal(output.Bytes(), &checked) != nil {
|
||||
report.Add("CLI", "$", "validator_unavailable", "Reinstall the matching tht and workspace helper pair.")
|
||||
} else if code != 0 || !checked.OK {
|
||||
report.Issues = append(report.Issues, checked.Issues...)
|
||||
if len(checked.Issues) == 0 {
|
||||
report.Add("database-bootstrap.yaml", "$", "bootstrap_invalid", "Correct workspace and binding documents, then validate again.")
|
||||
}
|
||||
} else {
|
||||
report.Warnings = append(report.Warnings, checked.Warnings...)
|
||||
for _, file := range checked.SecretFiles {
|
||||
if outsideWorkspace(file.Path, "database-bootstrap.yaml", file.Field) {
|
||||
preparation.CheckSecret(file.Path, "database-bootstrap.yaml", file.Field, false, &report)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
directory, err := filepath.Abs(options["--directory"])
|
||||
if err == nil {
|
||||
if args[0] == "prepare" {
|
||||
err = preparation.Prepare(directory)
|
||||
} else {
|
||||
err = preparation.Credentials(ctx, directory)
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
report.Add("preparation", "$", "preparation_refused", err.Error())
|
||||
}
|
||||
}
|
||||
report.OK = len(report.Issues) == 0
|
||||
if !report.OK && status == 0 {
|
||||
status = 1
|
||||
}
|
||||
if slices.Contains(args, "--json") {
|
||||
_ = json.NewEncoder(stdout).Encode(report)
|
||||
} else {
|
||||
if report.OK {
|
||||
fmt.Fprintln(stdout, "Document operation completed. Local validation does not establish runtime readiness.")
|
||||
}
|
||||
for _, issue := range report.Issues {
|
||||
fmt.Fprintf(stdout, "%s [%s] %s: %s\n", issue.Document, issue.Field, issue.Code, issue.Correction)
|
||||
}
|
||||
for _, warning := range report.Warnings {
|
||||
fmt.Fprintln(stdout, warning)
|
||||
}
|
||||
}
|
||||
return status
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestInstallationPrepareDocumentsBeforeRuntime(t *testing.T) {
|
||||
t.Setenv("PATH", "")
|
||||
root, err := filepath.EvalSymlinks(t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
destination := filepath.Join(root, "installation")
|
||||
var stdout, stderr bytes.Buffer
|
||||
status := run(context.Background(), []string{"installation", "prepare", "--directory", destination, "--json"}, &stdout, &stderr)
|
||||
if status != 0 {
|
||||
t.Fatalf("status=%d stdout=%s stderr=%s", status, &stdout, &stderr)
|
||||
}
|
||||
var report map[string]any
|
||||
if json.Unmarshal(stdout.Bytes(), &report) != nil || report["ok"] != true {
|
||||
t.Fatalf("report=%s", &stdout)
|
||||
}
|
||||
for _, name := range []string{"thothii-installation.yaml", "operator.env", "database-bootstrap.yaml", "README.md"} {
|
||||
if _, err := os.Stat(filepath.Join(destination, name)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
before, _ := os.ReadFile(filepath.Join(destination, "thothii-installation.yaml"))
|
||||
stdout.Reset()
|
||||
stderr.Reset()
|
||||
if run(context.Background(), []string{"installation", "prepare", "--directory", destination, "--json"}, &stdout, &stderr) == 0 {
|
||||
t.Fatal("overwrote existing documents")
|
||||
}
|
||||
after, _ := os.ReadFile(filepath.Join(destination, "thothii-installation.yaml"))
|
||||
if !bytes.Equal(before, after) {
|
||||
t.Fatal("existing document changed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestInstallationCredentialsAreExplicitPrivateAndNeverReplaced(t *testing.T) {
|
||||
t.Setenv("PATH", "")
|
||||
root, _ := filepath.EvalSymlinks(t.TempDir())
|
||||
destination := filepath.Join(root, "installation")
|
||||
var stdout, stderr bytes.Buffer
|
||||
if run(context.Background(), []string{"installation", "prepare", "--directory", destination}, &stdout, &stderr) != 0 {
|
||||
t.Fatal(&stdout, &stderr)
|
||||
}
|
||||
secret := filepath.Join(destination, "secrets", "catalog-runtime-password")
|
||||
if _, err := os.Stat(secret); !os.IsNotExist(err) {
|
||||
t.Fatal("prepare generated a secret implicitly")
|
||||
}
|
||||
stdout.Reset()
|
||||
stderr.Reset()
|
||||
if run(context.Background(), []string{"installation", "credentials", "--directory", destination, "--json"}, &stdout, &stderr) != 0 {
|
||||
t.Fatal(&stdout, &stderr)
|
||||
}
|
||||
before, err := os.ReadFile(secret)
|
||||
if err != nil || len(bytes.TrimSpace(before)) < 32 {
|
||||
t.Fatal("missing strong technical credential", err)
|
||||
}
|
||||
if bytes.Contains(stdout.Bytes(), bytes.TrimSpace(before)) || bytes.Contains(stderr.Bytes(), bytes.TrimSpace(before)) {
|
||||
t.Fatal("secret leaked")
|
||||
}
|
||||
info, _ := os.Stat(secret)
|
||||
if runtime.GOOS != "windows" && info.Mode().Perm()&0o077 != 0 {
|
||||
t.Fatal("credential is not private")
|
||||
}
|
||||
stdout.Reset()
|
||||
stderr.Reset()
|
||||
if run(context.Background(), []string{"installation", "credentials", "--directory", destination, "--json"}, &stdout, &stderr) != 0 {
|
||||
t.Fatal(&stdout, &stderr)
|
||||
}
|
||||
after, _ := os.ReadFile(secret)
|
||||
if !bytes.Equal(before, after) {
|
||||
t.Fatal("credential was replaced")
|
||||
}
|
||||
}
|
||||
|
||||
func TestInstallationValidateRejectsWrongEnvironmentFieldType(t *testing.T) {
|
||||
root, _ := filepath.EvalSymlinks(t.TempDir())
|
||||
path := filepath.Join(root, "thothii-installation.yaml")
|
||||
if err := os.WriteFile(path, []byte("schemaVersion: 2\nenvFile: []\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var stdout, stderr bytes.Buffer
|
||||
status := run(context.Background(), []string{"--installation", path, "installation", "validate", "--workspaces", root, "--json"}, &stdout, &stderr)
|
||||
if status != 1 {
|
||||
t.Fatalf("invalid field accepted: %d %s", status, &stdout)
|
||||
}
|
||||
}
|
||||
@@ -43,6 +43,12 @@ Commands:
|
||||
setup [--complete|--configure-only] [--installation-id ID] [--profile local|server]
|
||||
[--shell-mode full|embedded] [--shell-default-locale BCP47-TAG] [--shell-adapter omics-portal]
|
||||
Create, validate, and optionally complete the local installation.
|
||||
installation prepare --directory NEW_PATH [--json]
|
||||
Create commented installation, environment and database bootstrap templates.
|
||||
installation credentials --directory PATH [--json]
|
||||
Explicitly generate protected technical credentials before setup.
|
||||
installation validate --workspaces PATH [--bootstrap PATH] [--json]
|
||||
Check prepared application documents; requires --installation.
|
||||
installation migrate --output PATH --session-default PROVIDER/MODEL
|
||||
--embedding-id PROVIDER/MODEL --embedding-dimensions N
|
||||
Create a review-only schema-v2 candidate from all three legacy model sources.
|
||||
@@ -138,6 +144,9 @@ func run(ctx context.Context, args []string, stdout, stderr io.Writer) int {
|
||||
return versionCommand(commandArgs, stdout, stderr)
|
||||
}
|
||||
if command == "installation" {
|
||||
if len(commandArgs) > 0 && (commandArgs[0] == "prepare" || commandArgs[0] == "credentials" || commandArgs[0] == "validate") {
|
||||
return installationDocumentsCommand(ctx, installationPath, commandArgs, stdout)
|
||||
}
|
||||
if len(commandArgs) > 0 && commandArgs[0] == "generate" {
|
||||
return installationGenerationCommand(installationPath, commandArgs[1:], stdout, stderr)
|
||||
}
|
||||
|
||||
@@ -98,6 +98,16 @@ type Installation struct {
|
||||
|
||||
// Load reads and validates an installation descriptor at an absolute path.
|
||||
func Load(path string) (Installation, error) {
|
||||
return load(path, false)
|
||||
}
|
||||
|
||||
// LoadPrepared applies the same authored configuration rules before release assets exist.
|
||||
// Only standard distribution-owned Compose assets are deferred, never custom overrides.
|
||||
func LoadPrepared(path string) (Installation, error) {
|
||||
return load(path, true)
|
||||
}
|
||||
|
||||
func load(path string, prepared bool) (Installation, error) {
|
||||
if !filepath.IsAbs(path) {
|
||||
return Installation{}, fmt.Errorf("installation path must be absolute")
|
||||
}
|
||||
@@ -143,6 +153,11 @@ func Load(path string) (Installation, error) {
|
||||
if err := requireDirectory(raw.ProjectDirectory, "projectDirectory"); err != nil {
|
||||
return Installation{}, err
|
||||
}
|
||||
if prepared {
|
||||
if err := requireCanonicalDirectory(raw.ProjectDirectory); err != nil {
|
||||
return Installation{}, errors.New("projectDirectory must be canonical and accessible")
|
||||
}
|
||||
}
|
||||
for _, legacyProjection := range []string{
|
||||
filepath.Join(raw.ProjectDirectory, "deploy", "pi", "models.json"),
|
||||
filepath.Join(raw.ProjectDirectory, "deploy", "pi", "settings.json"),
|
||||
@@ -195,7 +210,8 @@ func Load(path string) (Installation, error) {
|
||||
return Installation{}, errors.New("authentication.configDirectory must match THT_AUTH_CONFIG_ROOT")
|
||||
}
|
||||
for _, override := range raw.Overrides {
|
||||
if err := requireRegularFile(override, "override"); err != nil {
|
||||
standardAsset := override == filepath.Join(installation.ProjectDirectory, "deploy", "compose.git-https.yaml") || override == filepath.Join(installation.ProjectDirectory, "deploy", "compose.git-ssh.yaml")
|
||||
if err := requireRegularFile(override, "override"); err != nil && !(prepared && standardAsset && errors.Is(statPathError(override), os.ErrNotExist)) {
|
||||
return Installation{}, err
|
||||
}
|
||||
installation.Overrides = append(installation.Overrides, filepath.Clean(override))
|
||||
@@ -209,6 +225,9 @@ func Load(path string) (Installation, error) {
|
||||
}
|
||||
}
|
||||
for _, composeFile := range installation.ComposeFiles()[:2] {
|
||||
if prepared && errors.Is(statPathError(composeFile), os.ErrNotExist) {
|
||||
continue
|
||||
}
|
||||
if err := requireRegularFile(composeFile, "Compose file"); err != nil {
|
||||
return Installation{}, err
|
||||
}
|
||||
@@ -226,6 +245,8 @@ func Load(path string) (Installation, error) {
|
||||
return installation, nil
|
||||
}
|
||||
|
||||
func statPathError(path string) error { _, err := os.Lstat(path); return err }
|
||||
|
||||
var metadataSecretBundleKeyPattern = regexp.MustCompile(`^[A-Z][A-Z0-9_]{0,127}$`)
|
||||
var metadataAPIKeyEnvironments = map[string]struct{}{
|
||||
"THT_MODEL_API_KEY": {},
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
package preparation
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/aritmolab/thothii/tools/tht/internal/authconfig"
|
||||
"github.com/aritmolab/thothii/tools/tht/internal/config"
|
||||
"github.com/aritmolab/thothii/tools/tht/internal/safeio"
|
||||
)
|
||||
|
||||
// Credentials creates installation-owned secrets only. External credentials are supplied by
|
||||
// the operator. Existing files are checked and retained so interrupted preparation can resume.
|
||||
func Credentials(ctx context.Context, directory string) error {
|
||||
if err := safeio.ValidatePrivateDirectory(directory); err != nil {
|
||||
return fmt.Errorf("use an existing private preparation directory")
|
||||
}
|
||||
if _, err := safeio.ReadCanonicalPrivateRegular(filepath.Join(directory, "thothii-installation.yaml"), 1<<20); err != nil {
|
||||
return fmt.Errorf("prepare installation documents first")
|
||||
}
|
||||
secrets := filepath.Join(directory, "secrets")
|
||||
if err := safeio.EnsurePrivateDirectory(secrets); err != nil {
|
||||
return fmt.Errorf("secrets directory must be private and operator-owned")
|
||||
}
|
||||
for _, name := range []string{"catalog-runtime-password", "catalog-migrator-password", "admin-password"} {
|
||||
path := filepath.Join(secrets, name)
|
||||
if _, err := os.Lstat(path); err == nil {
|
||||
value, err := safeio.ReadCanonicalPrivateRegular(path, 64<<10)
|
||||
if err != nil || len(strings.TrimSpace(string(value))) < 32 {
|
||||
return fmt.Errorf("existing technical credentials must be private, readable and at least 32 characters; no file was replaced")
|
||||
}
|
||||
continue
|
||||
} else if !errors.Is(err, os.ErrNotExist) {
|
||||
return fmt.Errorf("cannot inspect technical credentials")
|
||||
}
|
||||
value := make([]byte, 32)
|
||||
if _, err := rand.Read(value); err != nil {
|
||||
return fmt.Errorf("cannot generate random credentials")
|
||||
}
|
||||
if err := safeio.WriteCanonicalNewPrivateFile(path, []byte(hex.EncodeToString(value)+"\n"), 0o600); err != nil {
|
||||
return fmt.Errorf("cannot create credential; existing files are retained")
|
||||
}
|
||||
}
|
||||
for name, contents := range map[string]string{"secrets.env": "# Supply the provider credential; never commit this file.\nOPENAI_API_KEY=CHANGE_ME\n", "pi-auth.json": "{}\n"} {
|
||||
path := filepath.Join(secrets, name)
|
||||
if _, err := os.Lstat(path); errors.Is(err, os.ErrNotExist) {
|
||||
if err := safeio.WriteCanonicalNewPrivateFile(path, []byte(contents), 0o600); err != nil {
|
||||
return fmt.Errorf("cannot create external credential template")
|
||||
}
|
||||
} else if _, err := safeio.ReadCanonicalPrivateRegular(path, 64<<10); err != nil {
|
||||
return fmt.Errorf("existing credential template is not private or readable")
|
||||
}
|
||||
}
|
||||
authDirectory := filepath.Join(directory, "auth")
|
||||
if _, err := os.Lstat(filepath.Join(authDirectory, "auth.yaml")); err == nil {
|
||||
if _, _, err := authconfig.Load(authDirectory); err != nil {
|
||||
return fmt.Errorf("existing authentication documents are invalid; repair them explicitly")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
installation := config.Installation{Authentication: config.Authentication{ConfigDirectory: authDirectory}}
|
||||
if authconfig.Run(ctx, installation, []string{"configure", "--mode", "local", "--public-url", "http://localhost:8080", "--admin-user", "admin", "--password-file", filepath.Join(secrets, "admin-password")}, strings.NewReader(""), io.Discard, io.Discard) != 0 {
|
||||
return fmt.Errorf("cannot prepare local administrator; inspect protected auth files and retry without replacing existing credentials")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
// Package preparation owns installation-local documents before any runtime exists.
|
||||
package preparation
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/aritmolab/thothii/tools/tht/internal/safeio"
|
||||
)
|
||||
|
||||
func Prepare(directory string) error {
|
||||
if err := safeio.ValidateCanonicalPath(directory); err != nil {
|
||||
return fmt.Errorf("choose an absolute canonical destination")
|
||||
}
|
||||
exists, err := safeio.PreflightPrivateDirectory(directory)
|
||||
if err != nil || exists {
|
||||
return fmt.Errorf("choose a new private directory with an existing parent; existing documents are never replaced")
|
||||
}
|
||||
if err := safeio.EnsurePrivateDirectory(directory); err != nil {
|
||||
return fmt.Errorf("cannot create private preparation directory")
|
||||
}
|
||||
root := func(name string) string { return strconv.Quote(filepath.Join(directory, name)) }
|
||||
installation := fmt.Sprintf(`# Installation schema v2; replace CHANGE_ME before validation.
|
||||
# Paths refer to local preparation files, never to workspace Git.
|
||||
schemaVersion: 2
|
||||
profile: local
|
||||
projectDirectory: %s
|
||||
envFile: %s
|
||||
shell: {mode: full, defaultLocale: en}
|
||||
workspaceRepository:
|
||||
remote: https://CHANGE_ME/workspaces.git
|
||||
branch: main
|
||||
access: https
|
||||
authentication:
|
||||
configDirectory: %s
|
||||
modelCatalog:
|
||||
defaults: {interaction: openai/gpt-4.1-mini}
|
||||
embedding: {id: 'ollama/qwen3-embedding:0.6b', dimensions: 1024}
|
||||
providers:
|
||||
openai:
|
||||
authentication: {mode: secret_env, apiKeyEnv: OPENAI_API_KEY}
|
||||
session: {mode: pi_builtin}
|
||||
models:
|
||||
gpt-4.1-mini: {session: {}}
|
||||
# Metadata generation is optional: omitted here. Configure its eligibility in this catalog.
|
||||
# This Compose asset will come from the release; no application checkout is required here.
|
||||
overrides: [%s]
|
||||
`, strconv.Quote(directory), root("operator.env"), root("auth"), root("deploy/compose.git-https.yaml"))
|
||||
environment := "# Non-secret paths and parameters; no interpolation or duplicate keys.\n"
|
||||
for _, entry := range [][2]string{
|
||||
{"COMPOSE_PROJECT_NAME", "thothii-local"}, {"THT_WORKSPACE_INSTALLATION_ID", "local"},
|
||||
{"THT_WORKSPACE_GIT_REMOTE", "https://CHANGE_ME/workspaces.git"}, {"THT_WORKSPACE_GIT_BRANCH", "main"},
|
||||
{"THT_INSTALLATION_CONFIG_SOURCE", filepath.Join(directory, "thothii-installation.yaml")},
|
||||
{"THT_AUTH_CONFIG_ROOT", filepath.Join(directory, "auth")},
|
||||
{"THT_SECRETS_FILE", filepath.Join(directory, "secrets", "secrets.env")},
|
||||
{"PI_AUTH_FILE", filepath.Join(directory, "secrets", "pi-auth.json")},
|
||||
{"THT_WORKSPACE_GIT_CREDENTIALS_FILE", filepath.Join(directory, "secrets", "git-credentials")},
|
||||
{"THT_WORKSPACE_GIT_CA_FILE", filepath.Join(directory, "secrets", "git-ca.pem")},
|
||||
{"THT_CATALOG_RUNTIME_PASSWORD_SOURCE", filepath.Join(directory, "secrets", "catalog-runtime-password")},
|
||||
{"THT_CATALOG_MIGRATOR_PASSWORD_SOURCE", filepath.Join(directory, "secrets", "catalog-migrator-password")},
|
||||
{"THOTH_HTTP_PORT", "8080"}, {"THOTH_CORE_HTTP_PORT", "8787"}, {"MAX_PI_PROCESSES", "4"},
|
||||
} {
|
||||
environment += entry[0] + "=" + strconv.Quote(entry[1]) + "\n"
|
||||
}
|
||||
bootstrap := fmt.Sprintf(`# Bootstrap input only; PostgreSQL Metadata Catalog remains the runtime authority.
|
||||
schemaVersion: 1
|
||||
databases:
|
||||
- workspaceId: CHANGE_ME
|
||||
engine: postgres
|
||||
databaseName: CHANGE_ME
|
||||
schema: public
|
||||
binding:
|
||||
transport: postgres_direct
|
||||
host: CHANGE_ME
|
||||
port: 5432
|
||||
username: CHANGE_ME
|
||||
secretFiles:
|
||||
password: %s
|
||||
`, root("secrets/database-password"))
|
||||
documents := map[string]string{
|
||||
".gitignore": "*\n",
|
||||
"thothii-installation.yaml": installation, "operator.env": environment,
|
||||
"database-bootstrap.yaml": bootstrap,
|
||||
"README.md": "# Preparation / Preparazione\n\nReplace every CHANGE_ME / Sostituire ogni CHANGE_ME. Keep all files outside workspace Git / Tenere tutti i file fuori dal Git dei workspace.\n\n1. Edit installation models, workspace remote and operator.env consistently. / Modificare modelli, remoto e operator.env in modo coerente.\n2. Add one database bootstrap entry for every workspace; use read-only DWH credentials in protected files. / Una voce database per workspace, credenziali DWH in sola lettura in file protetti.\n3. Run tht installation credentials --directory PATH before setup; fill provider/database secrets yourself. / Generare credenziali tecniche prima del setup; compilare i segreti esterni.\n4. Repeat tht --installation PATH/thothii-installation.yaml installation validate --workspaces WORKSPACES. / Correggere e ripetere.\n\nNo services, network calls or database writes / Nessun servizio, chiamata di rete o scrittura database.\n",
|
||||
}
|
||||
for _, name := range []string{".gitignore", "thothii-installation.yaml", "operator.env", "database-bootstrap.yaml", "README.md"} {
|
||||
if err := safeio.WriteCanonicalNewPrivateFile(filepath.Join(directory, name), []byte(strings.TrimSpace(documents[name])+"\n"), 0o600); err != nil {
|
||||
return fmt.Errorf("cannot create preparation files; inspect the new directory and retry in a new destination")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,274 @@
|
||||
package preparation
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/aritmolab/thothii/tools/tht/internal/authconfig"
|
||||
"github.com/aritmolab/thothii/tools/tht/internal/config"
|
||||
"github.com/aritmolab/thothii/tools/tht/internal/safeio"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
type Issue struct {
|
||||
Document string `json:"document"`
|
||||
Field string `json:"field"`
|
||||
Code string `json:"code"`
|
||||
Correction string `json:"correction"`
|
||||
}
|
||||
type Report struct {
|
||||
SchemaVersion int `json:"schema_version"`
|
||||
Scope string `json:"scope"`
|
||||
OK bool `json:"ok"`
|
||||
Issues []Issue `json:"issues"`
|
||||
Warnings []string `json:"warnings"`
|
||||
DeferredChecks []string `json:"deferred_checks"`
|
||||
}
|
||||
|
||||
func NewReport() Report {
|
||||
return Report{SchemaVersion: 1, Scope: "application-documents", Issues: []Issue{}, Warnings: []string{}, DeferredChecks: []string{"release-assets", "external-connectivity", "catalog-import", "runtime-readiness"}}
|
||||
}
|
||||
func (r *Report) Add(document, field, code, correction string) {
|
||||
r.OK = false
|
||||
r.Issues = append(r.Issues, Issue{document, field, code, correction})
|
||||
}
|
||||
func placeholder(value string) bool {
|
||||
upper := strings.ToUpper(value)
|
||||
return strings.Contains(upper, "CHANGE_ME") || strings.Contains(upper, "REPLACE_ME") || strings.Contains(upper, "YOUR_API_KEY") || strings.Contains(upper, "<PASSWORD>")
|
||||
}
|
||||
|
||||
// CheckYAML refuses unresolved placeholders and ambiguous authored YAML without returning values.
|
||||
func CheckYAML(path, document string, report *Report) bool {
|
||||
content, err := safeio.ReadCanonicalPrivateRegular(path, 1<<20)
|
||||
if err != nil {
|
||||
report.Add(document, "$", "file_unavailable", "Use a readable, private regular document at a canonical absolute path (no links).")
|
||||
return false
|
||||
}
|
||||
decoder := yaml.NewDecoder(bytes.NewReader(content))
|
||||
var node, extra yaml.Node
|
||||
if err := decoder.Decode(&node); err != nil || decoder.Decode(&extra) != io.EOF {
|
||||
report.Add(document, "$", "yaml_invalid", "Keep one well-formed YAML document.")
|
||||
return false
|
||||
}
|
||||
var walk func(*yaml.Node) bool
|
||||
walk = func(n *yaml.Node) bool {
|
||||
if n.Kind == yaml.AliasNode || n.Tag == "!!merge" {
|
||||
return false
|
||||
}
|
||||
if n.Kind == yaml.ScalarNode && placeholder(n.Value) {
|
||||
return false
|
||||
}
|
||||
if n.Kind == yaml.MappingNode {
|
||||
seen := map[string]bool{}
|
||||
for index := 0; index < len(n.Content); index += 2 {
|
||||
key := n.Content[index]
|
||||
if key.Kind != yaml.ScalarNode || seen[key.Value] {
|
||||
return false
|
||||
}
|
||||
seen[key.Value] = true
|
||||
}
|
||||
}
|
||||
for _, child := range n.Content {
|
||||
if !walk(child) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
if !walk(&node) {
|
||||
report.Add(document, "$", "incomplete_or_ambiguous", "Replace CHANGE_ME/REPLACE_ME placeholders, remove duplicate keys, aliases and YAML merge keys.")
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
var environmentKey = regexp.MustCompile(`^[A-Z][A-Z0-9_]*$`)
|
||||
|
||||
func checkEnvironment(path string, report *Report) bool {
|
||||
contents, err := safeio.ReadCanonicalPrivateRegular(path, 1<<20)
|
||||
if err != nil {
|
||||
report.Add("operator.env", "$", "environment_unavailable", "Provide a private readable environment file.")
|
||||
return false
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for _, raw := range strings.Split(string(contents), "\n") {
|
||||
line := strings.TrimSpace(raw)
|
||||
if line == "" || strings.HasPrefix(line, "#") {
|
||||
continue
|
||||
}
|
||||
key, value, found := strings.Cut(line, "=")
|
||||
if !found || !environmentKey.MatchString(key) || seen[key] || placeholder(value) || strings.Contains(value, "$") {
|
||||
report.Add("operator.env", "$", "environment_invalid", "Use one literal KEY=value per line, unique uppercase keys, and replace placeholders; shell interpolation is not accepted.")
|
||||
return false
|
||||
}
|
||||
if strings.HasSuffix(key, "_PASSWORD") || strings.HasSuffix(key, "_API_KEY") {
|
||||
report.Add("operator.env", "$", "inline_secret", "Move credential values to protected files and keep only file references in operator.env.")
|
||||
return false
|
||||
}
|
||||
seen[key] = true
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func CheckSecret(path, document, field string, allowEmpty bool, report *Report) bool {
|
||||
contents, err := safeio.ReadCanonicalPrivateRegular(path, 64<<10)
|
||||
if err != nil || (!allowEmpty && len(bytes.TrimSpace(contents)) == 0) || placeholder(string(contents)) {
|
||||
report.Add(document, field, "secret_unavailable", "Supply a non-placeholder private readable regular secret file (owner-only permissions, no links); do not put its contents in YAML or logs.")
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func Validate(path string) (config.Installation, Report) {
|
||||
report := NewReport()
|
||||
if !CheckYAML(path, "thothii-installation.yaml", &report) {
|
||||
return config.Installation{}, report
|
||||
}
|
||||
// Read only the location before the canonical loader checks every field.
|
||||
contents, _ := safeio.ReadCanonicalPrivateRegular(path, 1<<20)
|
||||
var location struct {
|
||||
EnvFile string `yaml:"envFile"`
|
||||
}
|
||||
if yaml.Unmarshal(contents, &location) != nil {
|
||||
report.Add("thothii-installation.yaml", "envFile", "schema_invalid", "Set envFile to one absolute path string, then correct the remaining descriptor fields.")
|
||||
return config.Installation{}, report
|
||||
}
|
||||
if !checkEnvironment(location.EnvFile, &report) {
|
||||
return config.Installation{}, report
|
||||
}
|
||||
installation, err := config.LoadPrepared(path)
|
||||
if err != nil {
|
||||
field, correction := "$", "Check installation schema v2, absolute paths, workspace remote/branch and matching environment values. Custom overrides and Git credential/CA files must exist."
|
||||
if strings.Contains(err.Error(), "modelCatalog") {
|
||||
field, correction = "modelCatalog", "Check interaction default eligibility, provider endpoints/authentication, embedding id/dimensions, and the private provider key bundle."
|
||||
}
|
||||
if strings.Contains(err.Error(), "authentication") {
|
||||
field, correction = "authentication", "Match the authentication directory to operator.env and prepare its protected files."
|
||||
}
|
||||
report.Add("thothii-installation.yaml", field, "configuration_invalid", correction)
|
||||
return config.Installation{}, report
|
||||
}
|
||||
value := func(key string) string { result, _ := installation.EnvironmentValue(key); return result }
|
||||
for _, key := range []string{"COMPOSE_PROJECT_NAME", "THT_WORKSPACE_INSTALLATION_ID", "THT_INSTALLATION_CONFIG_SOURCE", "THT_SECRETS_FILE", "PI_AUTH_FILE", "THT_CATALOG_RUNTIME_PASSWORD_SOURCE", "THT_CATALOG_MIGRATOR_PASSWORD_SOURCE"} {
|
||||
if value(key) == "" {
|
||||
report.Add("operator.env", key, "required", "Supply this installation parameter or protected file reference before setup.")
|
||||
}
|
||||
}
|
||||
if value("THT_INSTALLATION_CONFIG_SOURCE") != path {
|
||||
report.Add("operator.env", "THT_INSTALLATION_CONFIG_SOURCE", "path_mismatch", "Point to the exact installation descriptor being validated.")
|
||||
}
|
||||
for _, key := range []string{"THOTH_HTTP_PORT", "THOTH_CORE_HTTP_PORT", "MAX_PI_PROCESSES"} {
|
||||
number, err := strconv.Atoi(value(key))
|
||||
if err != nil || number < 1 || number > 65535 {
|
||||
report.Add("operator.env", key, "invalid_number", "Supply a positive integer; HTTP ports must be within 1..65535.")
|
||||
}
|
||||
}
|
||||
if value("THOTH_HTTP_PORT") == value("THOTH_CORE_HTTP_PORT") {
|
||||
report.Add("operator.env", "THOTH_CORE_HTTP_PORT", "port_collision", "Choose different frontend and core host ports.")
|
||||
}
|
||||
files, err := installation.SecretFiles()
|
||||
if err != nil {
|
||||
report.Add("operator.env", "$", "secret_references_invalid", "Use canonical absolute paths for all _FILE and _SOURCE references.")
|
||||
}
|
||||
for _, file := range files {
|
||||
allowEmpty := file == value("THT_WORKSPACE_GIT_CREDENTIALS_FILE")
|
||||
CheckSecret(file, "operator.env", "protected-file-reference", allowEmpty, &report)
|
||||
}
|
||||
auth, users, err := authconfig.Load(installation.AuthenticationDirectory())
|
||||
if err != nil {
|
||||
report.Add("auth/auth.yaml", "$", "authentication_invalid", "Prepare and validate local authentication documents before setup, including the initial administrator.")
|
||||
} else if auth.Mode == "local" {
|
||||
admin := false
|
||||
for _, user := range users.Users {
|
||||
for _, role := range user.Roles {
|
||||
if user.Enabled && role == authconfig.RoleAdmin {
|
||||
admin = true
|
||||
}
|
||||
}
|
||||
}
|
||||
if !admin {
|
||||
report.Add("auth/users.yaml", "users", "administrator_missing", "Enable at least one administrator before setup.")
|
||||
}
|
||||
} else {
|
||||
report.Add("auth/auth.yaml", "mode", "unsupported_bootstrap", "This preparation increment supports local administrative authentication; use the documented existing OIDC preparation path until its offline bootstrap validation is available.")
|
||||
}
|
||||
checkPiCredentials(value("PI_AUTH_FILE"), installation.ModelCatalog, &report)
|
||||
report.OK = len(report.Issues) == 0
|
||||
return installation, report
|
||||
}
|
||||
|
||||
// Pi stores api_key or oauth records. Require literal prepared material here; model
|
||||
// environment references belong to the catalog's secret_env path, not host process state.
|
||||
func checkPiCredentials(path string, catalog config.ModelCatalog, report *Report) {
|
||||
contents, err := safeio.ReadCanonicalPrivateRegular(path, 64<<10)
|
||||
var credentials map[string]map[string]any
|
||||
invalid := err != nil || json.Unmarshal(contents, &credentials) != nil || credentials == nil
|
||||
literal := func(value any) bool {
|
||||
text, ok := value.(string)
|
||||
return ok && strings.TrimSpace(text) != "" && !strings.HasPrefix(text, "!") && !strings.Contains(text, "$") && !placeholder(text)
|
||||
}
|
||||
var declarative func(any) bool
|
||||
declarative = func(value any) bool {
|
||||
switch v := value.(type) {
|
||||
case string:
|
||||
return !strings.HasPrefix(v, "!")
|
||||
case []any:
|
||||
for _, item := range v {
|
||||
if !declarative(item) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
case map[string]any:
|
||||
for _, item := range v {
|
||||
if !declarative(item) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for provider, record := range credentials {
|
||||
name := strings.ToLower(strings.TrimSpace(provider))
|
||||
if name == "" || provider != name || seen[name] || !declarative(record) {
|
||||
invalid = true
|
||||
}
|
||||
seen[name] = true
|
||||
switch record["type"] {
|
||||
case "api_key":
|
||||
if !literal(record["key"]) {
|
||||
invalid = true
|
||||
}
|
||||
if env, present := record["env"]; present {
|
||||
values, ok := env.(map[string]any)
|
||||
if !ok {
|
||||
invalid = true
|
||||
}
|
||||
for _, value := range values {
|
||||
if _, ok := value.(string); !ok {
|
||||
invalid = true
|
||||
}
|
||||
}
|
||||
}
|
||||
case "oauth":
|
||||
expires, ok := record["expires"].(float64)
|
||||
if !literal(record["access"]) || !literal(record["refresh"]) || !ok || expires <= 0 {
|
||||
invalid = true
|
||||
}
|
||||
default:
|
||||
invalid = true
|
||||
}
|
||||
}
|
||||
for id, provider := range catalog.Providers {
|
||||
if provider.Authentication.Mode == "pi_auth" && !seen[id] {
|
||||
invalid = true
|
||||
}
|
||||
}
|
||||
if invalid {
|
||||
report.Add("pi-auth.json", "providers", "provider_auth_invalid", "Provide a JSON object with literal api_key/type+key or oauth/type+access+refresh+expires records for selected Pi providers; use catalog secret_env for environment-based keys. Commands and unresolved references are not accepted.")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user