diff --git a/README.md b/README.md index ac3739c4..19a67857 100644 --- a/README.md +++ b/README.md @@ -34,11 +34,12 @@ for the executed consolidation and the inventory of historical sources retained ## Docker Compose and installation -For a fresh installation, follow the complete manual procedure in +For a fresh installation, follow the guided terminal procedure in [Italian](docs/install/standalone-manual-it.md) or -[English](docs/install/standalone-manual-en.md). Configure protected files first; -then run the documented build, explicit migrations and startup commands with the -same installation descriptor and Compose project. There is no installer or launcher. +[English](docs/install/standalone-manual-en.md). The single +`tht setup --complete` command validates protected files, builds the images, runs +Catalog migration, starts the stack and imports the configured workspace repository. +There is no graphical installer or native launcher. The mandatory stack includes frontend, core, PostgreSQL catalog, Qdrant, Ollama and the embedding initializer. DWH and LLM endpoints remain external dependencies. diff --git a/backend/src/operator-command.ts b/backend/src/operator-command.ts index bfaf3945..3891d6aa 100644 --- a/backend/src/operator-command.ts +++ b/backend/src/operator-command.ts @@ -16,12 +16,16 @@ import { loadSettings } from "./settings/settings-store.js"; import { ThtRunner, type SessionRow } from "./tht/tht-runner.js"; import { WorkspaceRegistry } from "./workspaces/registry.js"; import { WorkspaceSecretStore } from "./workspaces/secret-store.js"; +import { createProductionWorkspaceDiagnoser } from "./workspaces/diagnostics.js"; +import { resolveCatalogRuntimeBinding } from "./catalog/runtime-binding.js"; +import { CatalogService } from "./catalog/service.js"; +import { validateOperationalWorkspace } from "./workspaces/schema.js"; import { createCatalogRepository } from "./catalog/repository.js"; import type { CatalogRepository } from "./catalog/types.js"; type OperatorAction = "maintenance-activate" | "maintenance-deactivate" | "maintenance-status" | "session-inventory" | "workflow-doctor" | "workspace-integrity" - | "pi-test" | "effective-settings"; + | "workspace-pull" | "workspace-test" | "pi-test" | "effective-settings"; const lifecyclePrincipal: PrincipalContext = { issuer: "tht-operator-command", @@ -119,6 +123,119 @@ async function workspaceIntegrity(config: AppConfig): Promise<{ return { ready: true, ...integrity }; } +async function workspacePull(config: AppConfig): Promise<{ + ready: boolean; + status: "succeeded" | "degraded"; + branch: string; + head?: string; + degraded: boolean; +}> { + const status = await new WorkspaceRegistry(config.workspaceRegistry).pull(); + return { + ready: !status.degraded, + status: status.degraded ? "degraded" : "succeeded", + branch: status.branch, + ...(status.head ? { head: status.head } : {}), + degraded: status.degraded, + }; +} + +interface WorkspaceTestReport { + id: string; + status: "ready" | "failed"; + database: "reachable" | "not_configured" | "failed"; + diagnostics: string[]; +} + +async function workspaceTest(config: AppConfig): Promise<{ + ready: boolean; + workspaces: WorkspaceTestReport[]; +}> { + if (!config.catalogDatabase) throw new Error("Catalog database is not configured"); + const registry = new WorkspaceRegistry(config.workspaceRegistry); + const revisions = await registry.list(); + const repository = createCatalogRepository(config.catalogDatabase); + try { + const secretStore = new WorkspaceSecretStore({ + root: config.workspaceSecretStoreRoot, + runtimeRoot: config.workspaceSecretRuntimeRoot, + installationId: config.workspaceRegistry.installationId, + }); + const catalogService = new CatalogService( + repository, + registry, + secretStore, + config.workspaceRegistry.secretRoots, + config.workspaceDiagnosticTimeoutMs, + ); + const diagnose = createProductionWorkspaceDiagnoser(config.workspaceDiagnosticTimeoutMs, undefined, { + internalQdrantUrl: config.internalQdrantUrl, + internalEmbeddingUrl: config.internalEmbeddingUrl, + internalEmbeddingId: config.internalEmbeddingId, + internalEmbeddingModel: config.internalEmbeddingModel, + internalEmbeddingDimensions: config.internalEmbeddingDimensions, + }); + const databases = await repository.list(); + const reports: WorkspaceTestReport[] = []; + for (const revision of revisions) { + const diagnostics: string[] = []; + let workspace: ReturnType; + try { + workspace = validateOperationalWorkspace((await registry.read(revision.id)).workspace); + } catch { + reports.push({ id: revision.id, status: "failed", database: "not_configured", diagnostics: ["workspace_invalid"] }); + continue; + } + const database = databases.find((candidate) => candidate.workspaceId === revision.id); + if (!database) { + reports.push({ id: revision.id, status: "failed", database: "not_configured", diagnostics: ["database_binding_missing"] }); + continue; + } + let tested; + try { + tested = await catalogService.test(database); + } catch { + reports.push({ id: revision.id, status: "failed", database: "failed", diagnostics: ["database_unreachable"] }); + continue; + } + if (!tested || tested.connectionStatus !== "reachable") { + reports.push({ id: revision.id, status: "failed", database: "failed", diagnostics: ["database_unreachable"] }); + continue; + } + let lease: ReturnType; + try { + lease = resolveCatalogRuntimeBinding({ + workspace, + database: tested, + environment: process.env, + secretRoots: config.workspaceRegistry.secretRoots, + secretStore, + }); + } catch { + reports.push({ id: revision.id, status: "failed", database: "reachable", diagnostics: ["binding_missing"] }); + continue; + } + try { + // CatalogService.test() above is the authoritative database probe and records its + // outcome. The remaining diagnoser pass checks Evidence and internal semantic services; + // skipping its legacy DWH probe avoids requiring a second response-shape contract for a + // REST health endpoint. + const result = await diagnose(lease.workspace, lease.bindings, { writeProbe: false, skipDwh: true }); + diagnostics.push(...result.diagnostics.map((diagnostic) => diagnostic.code)); + const ready = result.activatable; + reports.push({ id: revision.id, status: ready ? "ready" : "failed", database: "reachable", diagnostics }); + } catch { + reports.push({ id: revision.id, status: "failed", database: "reachable", diagnostics: ["connector_unavailable"] }); + } finally { + lease.release(); + } + } + return { ready: reports.length > 0 && reports.every((report) => report.status === "ready"), workspaces: reports }; + } finally { + await repository.close?.(); + } +} + export async function runOperatorAction( action: OperatorAction, config: AppConfig, @@ -132,6 +249,8 @@ export async function runOperatorAction( if (action === "session-inventory") return await sessionInventory(config); if (action === "workflow-doctor") return await workflowDiagnostics(config); if (action === "workspace-integrity") return await workspaceIntegrity(config); + if (action === "workspace-pull") return await workspacePull(config); + if (action === "workspace-test") return await workspaceTest(config); const modelCatalog = loadRuntimeModelCatalog(config.modelCatalogFile); if (action === "effective-settings") { return effectiveSettings(config, loadSettings(config), modelCatalog); @@ -146,6 +265,7 @@ async function main(): Promise { if (!action || ![ "maintenance-activate", "maintenance-deactivate", "maintenance-status", "session-inventory", "workflow-doctor", "workspace-integrity", "pi-test", "effective-settings", + "workspace-pull", "workspace-test", ].includes(action)) throw new Error("invalid operator action"); const result = await runOperatorAction(action, loadConfig(process.env)); process.stdout.write(`${JSON.stringify(result)}\n`); diff --git a/backend/test/operator-command.test.ts b/backend/test/operator-command.test.ts index 1a7cfa72..8a371bc8 100644 --- a/backend/test/operator-command.test.ts +++ b/backend/test/operator-command.test.ts @@ -36,6 +36,10 @@ vi.mock("../src/tht/tht-runner.js", () => ({ vi.mock("../src/workspaces/registry.js", () => ({ WorkspaceRegistry: class { + async pull() { + return { branch: "main", head: "a".repeat(40), ahead: 0, behind: 0, degraded: false }; + } + async listRetainedSnapshots() { return [{ snapshotPath: "/data/workspace-registry/snapshots/revision/workspace.yaml" }]; } @@ -98,3 +102,13 @@ test("workflow doctor gives schema-v4 runtime rendering a live Catalog repositor expect(fakes.releaseWorkspaceRuntime).toHaveBeenCalledOnce(); expect(fakes.catalogRepository.close).toHaveBeenCalledOnce(); }); + +test("workspace pull exposes only safe Git status", async () => { + await expect(runOperatorAction("workspace-pull", config)).resolves.toEqual({ + ready: true, + status: "succeeded", + branch: "main", + head: "a".repeat(40), + degraded: false, + }); +}); diff --git a/deploy/secrets/README.md b/deploy/secrets/README.md index 88fab80f..52081e4c 100644 --- a/deploy/secrets/README.md +++ b/deploy/secrets/README.md @@ -8,6 +8,11 @@ cp deploy/secrets/thothii.secrets.example deploy/secrets/thothii.secrets chmod 600 deploy/secrets/thothii.secrets ``` +For a normal local installation, `tht setup --complete` creates the active bundle at +`deploy/local/secrets/thothii.secrets` and creates the two Catalog password files beside it. The +generated `deploy/local/operator.env` contains only absolute paths to those files; never copy +secret values into `operator.env`. + The file uses strict `KEY=VALUE` lines (comments and blank lines are allowed). The supported installation keys are `THT_MODEL_API_KEY`, `THT_DWH_API_KEY`, `THT_CA`, `THT_SSL_CA`, `THT_OIDC_CLIENT_SECRET`, and `THT_AUTHENTIK_API_TOKEN`. Installation Model Catalog providers may @@ -29,6 +34,11 @@ Session and metadata-generation runtimes read only the provider key named by credential reference, not their execution lifecycle. Pi-owned authentication remains available only to session-only built-in providers through `authentication.mode: pi_auth`. +Workspace database credentials are intentionally not part of this global bundle. Configure each +workspace's database binding, password/token, tunnel key and CA in Database Management; the +installation stores those values in its encrypted workspace secret store. The workspace Git +repository may declare database identity and Evidence, but must never contain these credentials. + Do not add vector or embedding endpoint credentials to the bundle. Active operator manuals use internal Qdrant and Ollama services, so vector/embedding runtime endpoint secrets are not part of the supported installation contract. diff --git a/docs/install/first-start.md b/docs/install/first-start.md index 2b30b413..562c784d 100644 --- a/docs/install/first-start.md +++ b/docs/install/first-start.md @@ -1,64 +1,63 @@ # Install and first start -Use one complete procedure for a fresh installation: +Use the guided procedure for a fresh installation: -- [Italian manual installation](standalone-manual-it.md) -- [English manual installation](standalone-manual-en.md) +- [Italian guided installation](standalone-manual-it.md) +- [English guided installation](standalone-manual-en.md) -Both cover macOS, Windows through Ubuntu WSL2, and Linux. They use a Gitea clone, -protected local configuration and manual terminal commands, without an application -installer or launcher. See their verification matrix for tests still pending. +The procedure covers Windows through Ubuntu WSL2, macOS, and Linux including an Omarchy/Arch-like +host. It uses the application clone, one installation secret bundle, protected repository +credentials when needed, and a terminal command. No host Node.js, Python or Pi installation is +required. ## What must be ready -You need Docker with Compose, the host operator command `tht`, access to the workspace -repository, and the credentials and network routes for the configured DWH and model -providers. Pi runs inside the application runtime; no host Pi installation is needed. +You need Docker with Compose v2, Git, Bash, curl, OpenSSL and shasum. You also need access to the +workspace repository and the values supplied by its owner: repository URL/branch, DWH endpoint, +database/schema, transport, credentials or certificates, Evidence credentials when applicable, +and LLM provider/API-key information. -The stack includes `frontend`, `core`, `catalog-db`, `qdrant`, `embedding`, plus the -one-shot `embedding-model-init` and `catalog-migrate` services. DWH and generative-model -endpoints remain separate installation settings. +The workspace repository and the THothII application repository are different. A workspace +descriptor may declare Evidence, but database passwords and installation bindings are stored in +the installation Catalog, not in Git. -Secrets, certificates, Pi authentication and endpoint bindings are protected local files. -Do not commit them or copy the configuration of another machine unchanged. +## One guided command -## Follow the ordered procedure +After cloning THothII, checking prerequisites and installing tht, run: -The bilingual guides provide the exact commands for: +~~~ +tht setup --complete --profile local --shell-mode full --shell-default-locale en +~~~ -1. Cloning the selected revision and checking prerequisites. -2. Bootstrapping the native host command. -3. Preparing catalog passwords and using - `tht setup --profile local --shell-mode full --shell-default-locale en --configure-only`. -4. Completing model, authentication and workspace credentials. -5. Generating configuration, building images and explicitly running `catalog-migrate`. -6. Starting the installation and checking health and readiness. - -Do not run setup alone as a substitute for that sequence. Migrations are not an -implicit effect of backend startup or `tht start`. Do not mix this installation's -descriptor/project with a different low-level Compose environment. +The first run creates protected placeholders under deploy/local/secrets/. Fill the required +credential files and rerun the same command. The command validates the local files and paths, +renders Compose, builds the images, starts catalog-db, runs catalog-migrate, starts the full +stack, and pulls/activates the workspace repository. Evidence source files declared by the +workspace are imported during activation. For an already configured installation: -```sh +~~~ tht --installation /absolute/path/thothii-installation.yaml status tht --installation /absolute/path/thothii-installation.yaml doctor --json -``` +tht --installation /absolute/path/thothii-installation.yaml workspace test --json +~~~ -`/health` checks application-process readiness. Doctor also checks configuration, -workspace, workflow and Pi prerequisites; a healthy web page alone does not prove -that a real database question can complete. +doctor --json is the non-destructive general core test. workspace test also probes the configured +database, Evidence, Qdrant and embedding service for every active workspace. It requires the +workspace database to have been configured in Database Management first. -## After startup +## Installer-only completion -Prepare [workspaces](../operations/workspaces.md), configure a database in -[Database Management](../operations/database-management.md), and complete the functional -checks in the installation guide before using real data. +The installer must still decide which LLMs and API keys are approved, configure and test each +workspace database, synchronize its schema, generate and consolidate descriptions, create Qdrant +entries, review naming-based FK suggestions alongside schema FKs, and load the approved +relationships. The final declaration of completeness requires green doctor and workspace test +results plus one real natural-language question completed through final SQL. -See [display mode and language](shell-and-language.md), [local authentication](authentication-local.md), -[OIDC](authentication-oidc.md) and [model configuration](../general/pi-configuration.md) -for later changes. Embedded portal integration is separate from a fresh standalone setup. +Migrations are part of setup --complete. Do not mix this installation’s descriptor or volumes with +a different Compose environment. Preserve the descriptor, credentials, Catalog and persistent +volumes; do not use docker compose down --volumes as a routine stop. -Use the installation's normal `tht start`, `tht stop` and diagnostic commands. -Preserve its descriptor, credentials, database and persistent volumes; do not use -`down --volumes` as a routine stop or upgrade. +See the guides for the Windows/macOS/Linux prerequisite matrix, workspace repository explanation, +secret layout and Gate A/Gate B acceptance checks. diff --git a/docs/install/standalone-manual-en.md b/docs/install/standalone-manual-en.md index b09586c9..312bb201 100644 --- a/docs/install/standalone-manual-en.md +++ b/docs/install/standalone-manual-en.md @@ -1,324 +1,256 @@ -# Manual standalone installation +# Guided standalone installation [Versione italiana](standalone-manual-it.md) -This is the verification procedure for preparing THothII as a standalone application in `full` -mode on macOS, Windows, and Linux. +This is the fresh-machine installation procedure for THothII. THothII receives a natural-language +question, queries an enterprise database read-only, and guides the user through SQL review. The +core, PostgreSQL catalog, Qdrant, embedding service, and Pi run in Docker; Node.js, Python, and Pi +are not required on the host. -In this document, “standalone” means that the user does not need to install Node.js, Python or Pi -on the host: the application services and local semantic -services run through Docker. DWH and LLM providers remain external endpoints configured by the -installation; this is not an offline package. +## Before you start: the two repositories -This path uses no graphical installer, native launcher, or Docker Hub image. It starts from a Gitea -clone and uses explicit terminal commands. Publishing pre-built images is a later step. +There are two separate repositories: -## Verification matrix +1. the application repository cloned by the user: + https://git.tylconsulting.it/mptyl/ThothII.git; +2. the workspace repository supplied by the curator/installer. It is not the THothII repository + and must not be cloned inside the application directory. -| System | Recommended terminal | Runtime | Test architecture | -| --- | --- | --- | --- | -| macOS supported by the installed Docker Desktop version | Bash in Terminal | Docker Desktop | Apple Silicon (`arm64`) | -| Windows 11 | Ubuntu inside WSL2 | Docker Desktop with WSL2 integration | x64 (`amd64`) | -| Ubuntu Linux 22.04 or 24.04 | Bash | Docker Engine + Compose v2 | x64 (`amd64`) | +The workspace repository normally contains: -Intel macOS is excluded from the first verification campaign. ARM Linux may be tested when the -machine’s Docker runtime reports `arm64`, but it is not part of the minimum matrix. +~~~ +thoth-workspaces.yaml +/workspace.yaml +/evidence/** # when Evidence is declared +~~~ -Documentation and Mac prerequisite checks have passed. Complete fresh installations on all three -systems remain pending; this matrix describes the tests to perform, not completed certification. +workspace.yaml contains workspace identity, language and optional Evidence source. By design it does +not contain database passwords. Database identity, transport (PostgreSQL, REST, or tunnel), user, +password, token and certificates are installation-local settings stored encrypted by the Catalog. +This prevents credentials from being committed to the workspace repository. -## Before you start +## 0. Machine prerequisites -You need: +### Windows -- access to the THothII Gitea repository and the workspace Git repository; -- Git; -- Docker Desktop on macOS and Windows, or Docker Engine with the Compose v2 plugin on Linux; -- Bash, `curl`, OpenSSL and `shasum` (Ubuntu package: `libdigest-sha-perl`); -- enough disk space to build the images and download the embedding model; -- the DWH and LLM endpoints, plus the credentials required by the installation. +- Windows 10/11 with Docker Desktop running and the WSL2 backend enabled. +- Ubuntu in WSL2, with Docker Desktop integration enabled for that distribution. +- Git, Bash, curl, OpenSSL and shasum inside WSL2. +- Do not install Node.js, Python or Pi on the host for this procedure. -On Linux, the current user must be able to run Docker. If the system requires `sudo`, add the user -to the Docker group according to local policy and open a new session before continuing. +If WSL2 is not installed, use the company procedure or, in PowerShell: -On Windows, run every command in this guide from Ubuntu under WSL2. In Docker Desktop, enable WSL2 -integration for that distribution. Clone the project inside the WSL2 Linux filesystem, for example -under `~/src`, rather than under `/mnt/c`: this avoids slow builds and path/line-ending issues. Pi -does not need to be installed on the host. +~~~ +wsl --install -d Ubuntu +~~~ -Check the runtime before or immediately after cloning: +Run all commands inside Ubuntu WSL2, in a Linux directory such as $HOME/src, not under /mnt/c. +scripts/install-tht.ps1 exists for advanced native PowerShell scenarios; use WSL2 for the +reproducible test. -```sh -docker version +### macOS + +- Docker Desktop installed and running, with several GB free for images and the embedding model. +- Git, Bash, curl, OpenSSL and shasum. +- Intel and Apple Silicon Macs are supported when Docker Desktop supports the architecture + reported by the Docker server. +- Do not install Node.js, Python or Pi on the host for this procedure. + +### Linux, including Omarchy + +- Git, Bash, curl, OpenSSL and shasum. +- Docker Engine and the Docker Compose v2 plugin. On Omarchy, check first: + +~~~ +command -v docker docker compose version +docker info +~~~ + +If Docker is missing, install Docker and Compose using the distribution-approved package procedure, +then start the service. On an Arch-like distribution the typical route is: + +~~~ +sudo pacman -S docker docker-compose +sudo systemctl enable --now docker +sudo usermod -aG docker "$USER" +~~~ + +After adding the group, open a new session and repeat docker info. Node.js, Python and Pi are not +needed on the host: they are in the Docker images. + +On every system run: + +~~~ +bash scripts/check-standalone-prerequisites.sh docker version --format '{{.Server.Arch}}' -``` +~~~ -The last command must return `amd64`, `x86_64`, `arm64`, or `aarch64`. +The architecture must be amd64, x86_64, arm64, or aarch64. You also need access to the +application Gitea repository, the workspace repository URL/branch and credentials, container +reachability to DWH/LLM endpoints, and the credentials, tokens or certificates associated with +the databases. -## 1. Clone a project revision +## 1. What to clone -Use the project repository on Gitea: +Clone only the application: -```sh +~~~ mkdir -p "$HOME/src" cd "$HOME/src" git clone https://git.tylconsulting.it/mptyl/ThothII.git cd ThothII git rev-parse --short HEAD -``` +~~~ -For an SSH clone, when the key is already authorized on Gitea: +Record the revision. tht setup --complete downloads the workspace repository into a persistent +Docker volume using the URL, branch and transport supplied during setup. -```sh -git clone git@git.tylconsulting.it:mptyl/ThothII.git -``` - -Record the hash printed by `git rev-parse` for a repeatable test. In a later campaign, use the -maintainer-approved revision/tag rather than implicitly following a mutable `main` branch. - -## 2. Check prerequisites and install the operator command +## 2. Install the terminal command From the clone root: -```sh +~~~ bash scripts/check-standalone-prerequisites.sh -export PATH="$HOME/.local/bin:$PATH" +mkdir -p "$HOME/.local/bin" THT_INSTALL_DIRECTORY="$HOME/.local/bin" bash scripts/install-tht.sh +export PATH="$HOME/.local/bin:$PATH" tht version -``` +~~~ -`install-tht.sh` bootstraps only the native `tht` operator command; it does not install a desktop -version of THothII. It uses the repository’s Docker builder, installs the binary for the current -terminal environment, and installs it in the user directory. Persist `$HOME/.local/bin` in your -shell PATH for new terminals too. An existing `tht` in this directory will be updated. +tht is the only native component to install. It builds the binary with Docker and orchestrates +Compose; it is not a second application runtime. -On Windows, run these commands inside WSL2. The installed `tht` binary is the Linux binary inside -WSL2; the application runtime remains Docker Desktop. Do not use `scripts/install-tht.ps1` as the -primary path for this test. +## 3. Prepare a few secrets and run complete setup -## 3. Configure and start the local installation +The first execution creates protected placeholders under deploy/local/secrets/ and stops if a +required credential is missing. Fill in the requested files and rerun the same command; compatible +configuration files are reused. -Run the remaining blocks in one Bash session from the physical clone root (`pwd -P`). -First create two distinct catalog passwords, preserving any existing files: +~~~ +tht setup --complete --profile local --shell-mode full --shell-default-locale en +~~~ -```bash -umask 077 -mkdir -p deploy/local/secrets -for name in catalog-runtime-password catalog-migrator-password; do - target="deploy/local/secrets/$name" - if [ ! -e "$target" ]; then - (set -C; openssl rand -hex 32 > "$target") || exit 1 - fi -done -export THT_CATALOG_RUNTIME_PASSWORD_SOURCE="$(pwd -P)/deploy/local/secrets/catalog-runtime-password" -export THT_CATALOG_MIGRATOR_PASSWORD_SOURCE="$(pwd -P)/deploy/local/secrets/catalog-migrator-password" -``` +The setup asks only for information the computer cannot know: -Do not regenerate passwords for an initialized catalog. Configure without starting services: - -```sh -tht setup --profile local --shell-mode full --shell-default-locale en --configure-only -``` - -Answer the prompts as follows: - -| Prompt | Value or rule | +| Request | What to provide | | --- | --- | -| Installation ID | `local`, unless one clone hosts multiple installations | -| Deployment profile | `local` | -| DWH API endpoint | An `http(s)` URL without user, password, query, or fragment; may be empty for a smoke-only test | -| LLM API endpoint | An `http(s)` URL without credentials; may be empty for a smoke-only test | -| Workspace repository URL | The workspace repository URL, not the THothII source clone | -| Workspace branch | Normally `main` | -| Workspace access | `ssh` with a deploy key, or `https` with a protected credential file | -| File paths | Accept the default paths under `deploy/local/secrets/` for the first test | -| Secret templates | Answer `yes` when protected files do not exist yet | -| Authentication | Configure the local login required by the installation; never put passwords on a command line | +| Workspace repository | Data/configuration repository URL, not ThothII.git | +| Branch | normally main | +| Access | ssh with key and known_hosts, or https with credential file and CA | +| DWH/LLM URL | endpoint without a token in the URL | +| Local login | initial user and password requested by the prompt | -The generated configuration is local and ignored by Git: +The setup generates random Catalog passwords and writes their paths, never their values, to +operator.env. It runs docker compose config, builds images, starts the Catalog, runs +catalog-migrate, starts the stack, and pulls the workspace repository. The pull also activates +declared Evidence; at minimum source files present in the workspace are materialized locally. -```text -deploy/local/thothii-installation.yaml -deploy/local/operator.env -deploy/local/auth/ -deploy/local/secrets/ -``` +### The file the user fills in -Edit secrets only in protected local files; never commit them. `deploy/env/local.env.example` is a tracked reference; the -generated path `deploy/local/operator.env` is the active path for this installation. +The main file is: -### Complete protected files +~~~ +deploy/local/secrets/thothii.secrets +~~~ -If setup created blank templates, enter the values with a local editor: +Add only NAME=VALUE lines needed by modelCatalog and installation adapters, such as an LLM API key +(DEEPSEEK_API_KEY, OPENAI_API_KEY, or the key declared by the catalog) and, when applicable, +THT_DWH_API_KEY. Allowed names are documented in deploy/secrets/README.md. Never put tokens in +URLs, the repository, or copied shell commands. -```sh -chmod 600 deploy/local/secrets/* -"${EDITOR:-vi}" deploy/local/secrets/thothii.secrets -``` +Two distinctions prevent common errors: -The bundle must contain only `KEY=VALUE` lines for credentials actually used by `modelCatalog`. The -allowed names and credential boundary are documented in the local file -`deploy/secrets/README.md`. Do not put tokens in URLs, the YAML -descriptor, the Git repository, or commands copied into the shell. +- when the catalog uses pi_auth, the LLM token belongs in the Pi pi-auth.json file created by + setup; {} is only a placeholder and does not enable a model; +- workspace database credentials (PostgreSQL password, REST API token, tunnel SSH key, + known_hosts, CA) do not belong in the workspace repository. Enter them per workspace in + Database Management, which stores them encrypted in the Catalog. The workspace declares + database/schema and transport; the installer must obtain the actual values from the database owner. -For SSH workspace access, also provide the private key and `known_hosts` file requested by setup. -For HTTPS access, provide the Git credential file and any required CA. Both must remain protected -and outside version control. +A private workspace repository also needs the Git files required by its transport: an SSH key and +known_hosts, or an HTTPS credential file and CA. These are transport files, not a second bundle to +commit. To minimize manual files, use SSH with an already-authorized deploy key. -Before starting, complete these additional configuration steps: +## 4. Automatic checks and terminal tests -1. Add `THT_CATALOG_RUNTIME_PASSWORD_SOURCE` and `THT_CATALOG_MIGRATOR_PASSWORD_SOURCE` to - `deploy/local/operator.env`, with the same absolute paths exported above. Setup does not persist - these two variables. Store paths, not passwords. -2. Replace the descriptor's generic `modelCatalog` with the approved provider/model configuration. - The generated defaults do not replicate the existing Mac. See [Pi/model configuration](../general/pi-configuration.md) - and the local example `deploy/psd/thothii-installation.yaml.example`. -3. Populate the keys referenced by `authentication.apiKeyEnv` in `thothii.secrets`. Providers using - `pi_auth` need valid credentials at `PI_AUTH_FILE`; the `{}` template is not authentication. -4. Complete the workspace Git files. SSH requires an authorized deploy key and verified known-hosts; - HTTPS requires the credential file and CA bundle expected by the overlay. Blank templates cannot - provide repository access. +Setup verifies files, permissions, descriptor, Compose, Docker, authentication, services, Pi and +the workspace. After startup, run these commands at any time: -After editing generated configuration, do not rerun setup: it rejects different existing content. -Generate the projections and run the explicit migration below. Use `THT_GIT_ACCESS=https` if that -was selected during setup. This block targets the fresh `local` descriptor with only the Git overlay; -custom installations must include their extra descriptor overlays in the same order. +~~~ +INSTALLATION="$PWD/deploy/local/thothii-installation.yaml" +tht --installation "$INSTALLATION" doctor --json +tht --installation "$INSTALLATION" workspace pull --json +tht --installation "$INSTALLATION" workspace test --json +~~~ -```bash -INSTALLATION="$(pwd -P)/deploy/local/thothii-installation.yaml" -tht --installation "$INSTALLATION" installation generate -THT_PROJECT="thothii-$(printf '%s' "$INSTALLATION" | shasum -a 256 | cut -c 1-12)" -THT_GIT_ACCESS=ssh -compose=( - docker compose --project-name "$THT_PROJECT" --project-directory "$(pwd -P)" - --env-file "$(pwd -P)/deploy/local/operator.env" - -f compose.yaml -f deploy/compose.local.yaml - -f "deploy/compose.git-$THT_GIT_ACCESS.yaml" - -f deploy/local/generated/compose.models.yaml -) -"${compose[@]}" config --quiet -"${compose[@]}" build core frontend -"${compose[@]}" up -d catalog-db -"${compose[@]}" run --rm catalog-migrate -tht --installation "$INSTALLATION" start -``` +workspace test checks, for every active workspace, database binding and credentials, Evidence, +Qdrant, and the embedding service. It exits non-zero when the database binding is missing or a +connection is unusable. Before running it, the installer must configure the database in Database +Management: the workspace repository cannot contain the password by itself. -Stop if a command fails. The project name matches the hash used by `tht`, preserving volume -identity. `catalog-migrate` applies Catalog and Memory migrations; `tht start` does not run it -automatically. Initial embedding-model download may take time. Use this installation-specific -sequence, not `run-stack.sh` with a different environment/project name. +doctor --json is the repeatable, non-destructive core verification. The final functional test must +also open http://127.0.0.1:8080, sign in, and complete a real question through final SQL. -## 4. Verify the installation +## Activities only the installer can complete -The descriptor generated for the default ID is: +The procedure automates bootstrap, but it cannot invent enterprise decisions or authorizations. +The installer must complete and record: -```sh -INSTALLATION="$(pwd -P)/deploy/local/thothii-installation.yaml" -test -f "$INSTALLATION" -bash scripts/verify-standalone-install.sh "$INSTALLATION" -``` +1. usable LLMs, the modelCatalog, and linked API keys; then run tht pi test and tht doctor; +2. database configuration, connection test, schema synchronization, and description generation; +3. human consolidation of generated descriptions; +4. Qdrant semantic entries through workspace preprocess run; +5. naming-based FK suggestions as a complement to schema FKs, human review, and loading approved + relationships into Qdrant; +6. recurring tht doctor --json and tht workspace test --json checks; +7. one real question completed successfully without connection or model errors. -The verifier is read-only: it runs `tht doctor --json` and `tht status` without restarting the -stack, regenerating configuration, or printing secret contents. +Configuration is complete only when all applicable activities are done, decisions are recorded, and +the two terminal tests are green. The core is usable only after the real question, not merely +because the frontend answers /health. -### Gate A — platform smoke test on all three computers +## Gate A and Gate B -Record the following for each machine: +### Gate A — platform -```sh +~~~ uname -a docker version --format '{{.Server.Version}} {{.Server.Arch}}' tht version bash scripts/check-standalone-prerequisites.sh bash scripts/verify-standalone-install.sh "$INSTALLATION" -``` +~~~ -The gate passes when the clone is intact, Docker and Compose are reachable, `tht doctor` is OK, the -stack is running, and the frontend responds at the default local URL `http://127.0.0.1:8080`. -Doctor also checks workspace and Pi: record their failures separately rather than labeling every -failure as a platform problem. Check HTTP readiness with: +### Gate B — usability -```sh -curl --fail --silent --show-error http://127.0.0.1:8080/health -``` - -### Gate B — functional verification - -Run this on at least one machine with available endpoints and credentials: - -First follow [Workspace operations](../operations/workspaces.md) to import/prepare the workspace -and configure the Database and local binding. The source clone does not transfer catalog data, -secrets or sessions from the Mac. Connect any VPN required by DWH/model endpoints and verify -that their names are reachable from containers too. - -1. open `http://127.0.0.1:8080`; -2. sign in with the configured local account; -3. verify that the configured workspace is readable; -4. start a real question and complete the review gates through final SQL; -5. stop and restart the installation, then run `verify-standalone-install.sh` again. - -A Gate B failure involving DWH, the LLM provider, workspace Git, or credentials does not by itself -prove a Docker portability problem: record the failed endpoint or component separately. - -## Daily lifecycle - -Use the explicit descriptor when more than one installation may be discoverable: - -```sh -INSTALLATION="$PWD/deploy/local/thothii-installation.yaml" - -tht --installation "$INSTALLATION" status -tht --installation "$INSTALLATION" start -tht --installation "$INSTALLATION" start --build -tht --installation "$INSTALLATION" logs +~~~ tht --installation "$INSTALLATION" doctor --json -tht --installation "$INSTALLATION" stop -``` +tht --installation "$INSTALLATION" workspace test --json +curl --fail --silent --show-error http://127.0.0.1:8080/health +~~~ -Use `start --build` after source changes or to rebuild images from the current checkout. `stop` -preserves volumes, sessions, the catalog, Pi state, Qdrant data, and the embedding model. Do not -use `docker compose down --volumes` during a normal test: it is destructive and removes local data. -For upgrades requiring migrations, follow the release runbook before starting the new application. +Then run a real question and stop/restart with tht stop and tht start. Do not use docker compose +down --volumes: it deletes the Catalog, sessions, Qdrant data and the embedding model. ## Quick diagnosis -| Symptom | Check | +| Symptom | Action | | --- | --- | -| `Docker Engine is not reachable` | start Docker Desktop or the Docker service and rerun `docker info` | -| Windows sees Docker but Bash fails | run the guide inside Ubuntu WSL2 and enable that distribution in Docker Desktop | -| `tht: command not found` | open a new shell and check `command -v tht`; rerun the bootstrap if needed | -| line-ending or executable-script errors | use a clone in the WSL2/Linux filesystem and rerun `bash scripts/...` | -| unsupported architecture | check `docker version --format '{{.Server.Arch}}'`; the test requires `amd64` or `arm64` | -| missing descriptor or env file | use `deploy/local/...` generated by `tht setup`, not an arbitrary copied file | -| healthy stack but workflow failure | check external URLs, the credential bundle, workspace Git, and authentication separately | -| data appears missing | check that `down --volumes` was not used; `stop` does not remove volumes | - -## Acceptance checklist - -- [ ] The clone comes from the expected Gitea repository and the revision is recorded. -- [ ] Docker Desktop/Engine and Compose v2 are available. -- [ ] The runtime reports an allowed architecture. -- [ ] `tht` was built from the repository and responds to `tht version`. -- [ ] Setup uses `profile: local`, `shell.mode: full`, and `shell.defaultLocale: en`. -- [ ] The descriptor, `operator.env`, authentication, and secrets exist only under `deploy/local/`. -- [ ] No secret appears in Git, URLs, public YAML, or recorded commands. -- [ ] Gate A passes on Apple Silicon macOS, x64 Windows WSL2, and x64 Linux. -- [ ] Gate B runs on at least one machine with DWH and LLM available. -- [ ] Stop/start and final verification complete without deleting volumes. - -## Out of scope for this release - -The following remain future work: - -- publishing pre-built images on Docker Hub; -- reducing prompts through a dedicated non-interactive configuration; -- creating DMG, MSI/EXE, AppImage, or other native installers; -- providing an offline runtime or bundling a local DWH/LLM into the application. +| Docker Engine is not reachable | start Docker Desktop or systemctl and repeat docker info | +| Omarchy cannot find docker | install Docker/Compose, enable the service and open a new session | +| Windows sees Docker but Bash fails | use Ubuntu WSL2 and enable its Docker Desktop integration | +| workspace pull fails | check URL, branch, key/credential file and known_hosts from the container | +| workspace test reports a missing binding | configure database, token/password and CA in Database Management | +| Pi is not ready | fill pi-auth.json or the key declared by modelCatalog, then run tht pi test | ## Related documents - [Install and first start](first-start.md) -- [Shell and localization](shell-and-language.md) - [Workspace operations](../operations/workspaces.md) -- `deploy/secrets/README.md` (runtime secrets) +- [Database Management](../operations/database-management.md) +- [Model configuration](../general/pi-configuration.md) +- deploy/secrets/README.md + +Publishing images on Docker Hub and native DMG/MSI/AppImage installers remain later work: this +procedure starts from the Gitea clone and does not require pre-published Docker Hub images. diff --git a/docs/install/standalone-manual-it.md b/docs/install/standalone-manual-it.md index 173adff1..244d8b71 100644 --- a/docs/install/standalone-manual-it.md +++ b/docs/install/standalone-manual-it.md @@ -1,329 +1,262 @@ -# Installazione manuale standalone +# Installazione standalone guidata [English version](standalone-manual-en.md) -Questa è la procedura di prova per predisporre THothII come applicazione autonoma in modalità -`full` su macOS, Windows e Linux. +Questa è la procedura per installare THothII da zero. THothII riceve una domanda in linguaggio +naturale, interroga in sola lettura un database aziendale e accompagna l’utente nella revisione +della SQL risultante. Il core, il catalogo PostgreSQL, Qdrant, il servizio di embedding e Pi vengono +eseguiti in Docker; sul computer non servono Node.js, Python o Pi. -In questo documento “autonoma” significa che l’utente non deve installare Node.js, Python o Pi -sull'host: i servizi applicativi e i servizi semantici -locali vengono eseguiti con Docker. DWH e provider LLM restano endpoint esterni configurati -dall’installazione; questa procedura non è un pacchetto offline. +## Prima di iniziare: i due repository -Il percorso non usa installer grafici, launcher nativi o immagini Docker Hub. Si parte da un clone -Gitea e si usano comandi espliciti da terminale. La pubblicazione di immagini pre-costruite è una -fase successiva. +Servono due repository distinti: -## Matrice di verifica +1. il repository dell’applicazione, che l’utente clona: + https://git.tylconsulting.it/mptyl/ThothII.git; +2. il repository dei workspace, indicato dal curatore/installatore. Non è il repository di + THothII e non va clonato manualmente nella directory dell’applicazione. -| Sistema | Terminale raccomandato | Runtime | Architettura della prova | -| --- | --- | --- | --- | -| macOS supportato dalla versione Docker Desktop installata | Bash nel Terminale | Docker Desktop | Apple Silicon (`arm64`) | -| Windows 11 | Ubuntu dentro WSL2 | Docker Desktop con integrazione WSL2 | x64 (`amd64`) | -| Linux Ubuntu 22.04 o 24.04 | Bash | Docker Engine + Compose v2 | x64 (`amd64`) | +Il repository workspace contiene il catalogo e una directory per ogni workspace, normalmente: -Intel macOS non fa parte della prima campagna di verifica. ARM Linux può essere provato quando il -runtime Docker della macchina restituisce `arm64`, ma non è un requisito della matrice minima. +~~~ +thoth-workspaces.yaml +/workspace.yaml +/evidence/** # se il workspace dichiara Evidence +~~~ -Le verifiche documentali e dei prerequisiti Mac sono passate. Le installazioni complete da zero -sui tre sistemi restano da eseguire: la matrice indica le prove previste, non una certificazione. +Il file workspace.yaml descrive identità, lingua e, opzionalmente, la sorgente Evidence. Per scelta +architetturale non contiene password del database. L’identità del database, il trasporto +(PostgreSQL, REST o tunnel), username, password, token e certificati sono configurazione locale +dell’installazione, conservata cifrata dal Catalog. Questo evita di committare credenziali nel +repository workspace. -## Cosa serve prima di iniziare +## 0. Prerequisiti della macchina -Servono: +### Windows -- accesso al repository Gitea di THothII e al repository Git dei workspace; -- Git; -- Docker Desktop su macOS e Windows, oppure Docker Engine con il plugin Compose v2 su Linux; -- Bash, `curl`, OpenSSL e `shasum` (su Ubuntu, pacchetto `libdigest-sha-perl`); -- spazio disco sufficiente per compilare le immagini e scaricare il modello di embedding; -- gli endpoint DWH e LLM, più le credenziali che l’installazione deve usare. +- Windows 10/11 con Docker Desktop avviato e backend WSL2 abilitato. +- Ubuntu in WSL2, con integrazione Docker Desktop abilitata per quella distribuzione. +- Git, Bash, curl, OpenSSL e shasum nella distribuzione WSL2. +- Non installare Node.js, Python o Pi sull’host per questa procedura. -Su Linux l’utente corrente deve poter eseguire Docker. Se il sistema richiede `sudo`, aggiungere -l’utente al gruppo Docker secondo la policy locale e aprire una nuova sessione prima di continuare. +In PowerShell, se WSL2 non esiste ancora, usare la procedura aziendale oppure: -Su Windows usare Ubuntu in WSL2 per tutti i comandi di questa guida. In Docker Desktop attivare -l’integrazione WSL2 per quella distribuzione. Clonare il progetto nel filesystem Linux di WSL2, -per esempio sotto `~/src`, e non sotto `/mnt/c`: si evitano rallentamenti e problemi di permessi o -line ending. Non è necessario installare Pi sull’host. +~~~ +wsl --install -d Ubuntu +~~~ -Verificare il runtime prima del clone o subito dopo: +Eseguire poi tutti i comandi dentro Ubuntu WSL2, in una directory Linux come $HOME/src, non sotto +/mnt/c. Il percorso nativo scripts/install-tht.ps1 esiste per scenari PowerShell avanzati; per la +prova riproducibile usare WSL2. -```sh -docker version +### macOS + +- Docker Desktop installato, avviato e con alcuni GB liberi per immagini e modello di embedding. +- Git, Bash, curl, OpenSSL e shasum. +- Sono supportati Mac Intel e Apple Silicon se Docker Desktop supporta l’architettura restituita + dal Docker server. +- Non installare Node.js, Python o Pi sull’host per questa procedura. + +### Linux, incluso Omarchy + +- Git, Bash, curl, OpenSSL e shasum. +- Docker Engine e il plugin Docker Compose v2. Su Omarchy verificare prima: + +~~~ +command -v docker docker compose version +docker info +~~~ + +Se Docker manca, installare Docker e Compose con il gestore pacchetti/procedura approvata dalla +distribuzione, poi avviare il servizio. Su una distribuzione Arch-like il percorso tipico è: + +~~~ +sudo pacman -S docker docker-compose +sudo systemctl enable --now docker +sudo usermod -aG docker "$USER" +~~~ + +Dopo l’aggiunta al gruppo aprire una nuova sessione e ripetere docker info. Non installare Node.js, +Python o Pi sull’host: sono dentro le immagini Docker. + +Su tutti i sistemi il controllo finale è: + +~~~ +bash scripts/check-standalone-prerequisites.sh docker version --format '{{.Server.Arch}}' -``` +~~~ -L’ultima istruzione deve restituire `amd64`, `x86_64`, `arm64` o `aarch64`. +L’architettura deve essere amd64, x86_64, arm64 o aarch64. Servono inoltre accesso al repository +Gitea dell’applicazione, URL/branch e credenziali del repository workspace, raggiungibilità dal +container degli endpoint DWH/LLM e le credenziali, token o certificati associati ai database. -## 1. Clonare una revisione del progetto +## 1. Cosa clonare -Usare il repository di progetto su Gitea: +Clonare solo l’applicazione: -```sh +~~~ mkdir -p "$HOME/src" cd "$HOME/src" git clone https://git.tylconsulting.it/mptyl/ThothII.git cd ThothII git rev-parse --short HEAD -``` +~~~ -Per un clone SSH usare, se la chiave è già autorizzata su Gitea: +Annotare la revisione. Il repository workspace verrà scaricato da tht setup --complete dentro un +volume Docker persistente, usando URL, branch e trasporto indicati durante il setup. -```sh -git clone git@git.tylconsulting.it:mptyl/ThothII.git -``` - -Per una prova ripetibile annotare l’hash stampato da `git rev-parse`. In una campagna successiva -usare la revisione/tag approvata dal maintainer invece di seguire implicitamente una `main` che -può cambiare. - -## 2. Verificare i prerequisiti e installare il comando operatore +## 2. Installare il comando terminale Dal root del clone: -```sh +~~~ bash scripts/check-standalone-prerequisites.sh -export PATH="$HOME/.local/bin:$PATH" +mkdir -p "$HOME/.local/bin" THT_INSTALL_DIRECTORY="$HOME/.local/bin" bash scripts/install-tht.sh +export PATH="$HOME/.local/bin:$PATH" tht version -``` +~~~ -`install-tht.sh` è un bootstrap del solo comando operatore nativo `tht`; non installa una versione -desktop di THothII. Usa il builder Docker del repository, installa il binario adatto all’ambiente -del terminale e lo installa nella directory utente. Aggiungere `$HOME/.local/bin` al PATH della -shell anche per i terminali successivi. Un `tht` già presente in quella directory viene aggiornato. +Il comando tht è l’unico componente nativo da installare. Costruisce il binario con Docker e +orchestra Compose; non è un secondo runtime dell’applicazione. -Su Windows, eseguire questi comandi dentro WSL2. Il binario `tht` installato è quello Linux di WSL2; -il runtime dell’applicazione rimane Docker Desktop. Non usare `scripts/install-tht.ps1` come percorso -principale di questa prova. +## 3. Preparare pochi segreti e avviare il setup completo -## 3. Configurare e avviare l’installazione locale +La prima esecuzione crea i placeholder protetti sotto deploy/local/secrets/ e si ferma se manca +una credenziale necessaria. Compilare i file indicati e rilanciare lo stesso comando: i file di +configurazione già compatibili vengono riutilizzati. -Eseguire i blocchi successivi nella stessa sessione Bash dal root fisico del clone (`pwd -P`). -Creare prima le due password distinte del catalogo, conservando eventuali file già esistenti: +~~~ +tht setup --complete --profile local --shell-mode full --shell-default-locale en +~~~ -```bash -umask 077 -mkdir -p deploy/local/secrets -for name in catalog-runtime-password catalog-migrator-password; do - target="deploy/local/secrets/$name" - if [ ! -e "$target" ]; then - (set -C; openssl rand -hex 32 > "$target") || exit 1 - fi -done -export THT_CATALOG_RUNTIME_PASSWORD_SOURCE="$(pwd -P)/deploy/local/secrets/catalog-runtime-password" -export THT_CATALOG_MIGRATOR_PASSWORD_SOURCE="$(pwd -P)/deploy/local/secrets/catalog-migrator-password" -``` +Durante il setup servono solo le informazioni operative che il computer non può conoscere: -Non rigenerare le password di un catalogo già inizializzato. Configurare senza avviare i servizi: - -```sh -tht setup --profile local --shell-mode full --shell-default-locale en --configure-only -``` - -Rispondere ai prompt nel seguente modo: - -| Prompt | Valore o regola | +| Richiesta | Cosa inserire | | --- | --- | -| Installation ID | `local`, salvo necessità di più installazioni nello stesso clone | -| Deployment profile | `local` | -| DWH API endpoint | URL `http(s)` senza user, password, query o fragment; può restare vuoto per il solo smoke test | -| LLM API endpoint | URL `http(s)` senza credenziali; può restare vuoto per il solo smoke test | -| Workspace repository URL | URL del repository dei workspace, non il clone sorgente di THothII | -| Workspace branch | normalmente `main` | -| Workspace access | `ssh` se si usa una chiave deploy; altrimenti `https` con credential file protetto | -| Percorsi dei file | accettare i percorsi predefiniti sotto `deploy/local/secrets/` nella prima prova | -| Secret templates | rispondere `yes` quando i file protetti non esistono ancora | -| Autenticazione | configurare il login locale richiesto dall’installazione; non inserire password in una riga di comando | +| Repository workspace | URL del repository dati/configurazione, non ThothII.git | +| Branch | normalmente main | +| Accesso | ssh con chiave e known_hosts, oppure https con credential file e CA | +| DWH/LLM URL | endpoint senza token nella URL | +| Login locale | utente e password iniziale richiesti dal prompt | -La configurazione generata è locale e ignorata da Git: +Il setup crea automaticamente le password casuali del Catalog e le scrive in operator.env come +percorsi, non come valori. Esegue docker compose config, costruisce le immagini, avvia il Catalog, +esegue catalog-migrate, avvia lo stack e importa il repository workspace. L’import attiva anche +l’Evidence dichiarata: almeno i file source presenti nel workspace vengono materializzati nel +registro locale. -```text -deploy/local/thothii-installation.yaml -deploy/local/operator.env -deploy/local/auth/ -deploy/local/secrets/ -``` +### Il file da compilare -Modificare i segreti solo nei file locali protetti; non committarli. Il file `deploy/env/local.env.example` è un riferimento -tracciato; il percorso generato da `tht setup`, `deploy/local/operator.env`, è quello da usare per -questa installazione. +Il file principale è: -### Completare i file protetti +~~~ +deploy/local/secrets/thothii.secrets +~~~ -Se il setup ha creato template vuoti, inserire i valori con un editor locale: +Inserire solo righe NOME=VALORE necessarie al modelCatalog e agli adapter, per esempio una API key +LLM (DEEPSEEK_API_KEY, OPENAI_API_KEY o quella dichiarata dal catalogo) ed eventualmente +THT_DWH_API_KEY. I nomi ammessi sono documentati in deploy/secrets/README.md. Non mettere token +nelle URL, nel repository o nei comandi. -```sh -chmod 600 deploy/local/secrets/* -"${EDITOR:-vi}" deploy/local/secrets/thothii.secrets -``` +Due precisazioni evitano gli errori più comuni: -Il bundle deve contenere solo righe `KEY=VALUE` per le credenziali effettivamente usate dal -`modelCatalog`. I nomi ammessi e il confine delle credenziali sono descritti nel file locale -`deploy/secrets/README.md`. Non mettere token nelle URL, nel -descriptor YAML, nel repository Git o nei comandi copiati nella shell. +- se il catalogo usa pi_auth, il token LLM va nel file Pi pi-auth.json creato dal setup; {} è solo + un placeholder e non abilita alcun modello; +- le credenziali specifiche di un database workspace (password PostgreSQL, API token REST, chiave + SSH del tunnel, known_hosts, CA) non vanno nel repository workspace: si inseriscono per workspace + in Database Management, che le conserva nel Catalog cifrato. Il workspace indica database/schema + e trasporto; l’installatore deve ottenere dal proprietario il valore corretto. -Per accesso workspace SSH, predisporre anche la chiave privata e il file `known_hosts` indicati dal -setup. Per accesso HTTPS, predisporre il credential file Git e l’eventuale CA. Entrambi devono -restare protetti e fuori dal controllo versione. +Per un repository workspace privato sono inoltre indispensabili i file Git richiesti dal trasporto: +una chiave SSH e known_hosts, oppure credential file HTTPS e CA. Sono file di trasporto, non un +secondo bundle da committare. Per ridurre i file da compilare, usare SSH con una deploy key già +autorizzata. -Prima dell'avvio completare anche questi passaggi: +## 4. Controlli automatici e test da terminale -1. Aggiungere `THT_CATALOG_RUNTIME_PASSWORD_SOURCE` e `THT_CATALOG_MIGRATOR_PASSWORD_SOURCE` a - `deploy/local/operator.env`, con gli stessi percorsi assoluti esportati sopra. Il setup non salva - queste due variabili. Inserire i percorsi, non le password. -2. Sostituire il `modelCatalog` generico nel descriptor con la configurazione provider/modelli - approvata. I default generati non replicano il Mac esistente. Vedere [configurazione Pi/modelli](../general/pi-configuration.md) - e l'esempio locale `deploy/psd/thothii-installation.yaml.example`. -3. Inserire in `thothii.secrets` le chiavi referenziate da `authentication.apiKeyEnv`. I provider - `pi_auth` richiedono credenziali valide nel file `PI_AUTH_FILE`; il template `{}` non autentica. -4. Completare i file Git del workspace. SSH richiede una chiave deploy autorizzata e known-hosts - verificato; HTTPS richiede il credential file e il bundle CA previsti dall'overlay. I template - vuoti non consentono l'accesso al repository. +Il setup verifica file, permessi, descriptor, Compose, Docker, autenticazione, servizi, Pi e +workspace. Dopo l’avvio usare questi comandi in qualunque momento: -Dopo aver modificato la configurazione generata, non rilanciare setup: rifiuta file esistenti con -contenuti diversi. Generare le proiezioni ed eseguire la migrazione esplicita indicata sotto. -Impostare `THT_GIT_ACCESS=https` se scelto nel setup. Il blocco usa il nuovo descriptor `local` -con il solo overlay Git; per installazioni personalizzate includere gli overlay aggiuntivi -nello stesso ordine del descriptor. +~~~ +INSTALLATION="$PWD/deploy/local/thothii-installation.yaml" +tht --installation "$INSTALLATION" doctor --json +tht --installation "$INSTALLATION" workspace pull --json +tht --installation "$INSTALLATION" workspace test --json +~~~ -```bash -INSTALLATION="$(pwd -P)/deploy/local/thothii-installation.yaml" -tht --installation "$INSTALLATION" installation generate -THT_PROJECT="thothii-$(printf '%s' "$INSTALLATION" | shasum -a 256 | cut -c 1-12)" -THT_GIT_ACCESS=ssh -compose=( - docker compose --project-name "$THT_PROJECT" --project-directory "$(pwd -P)" - --env-file "$(pwd -P)/deploy/local/operator.env" - -f compose.yaml -f deploy/compose.local.yaml - -f "deploy/compose.git-$THT_GIT_ACCESS.yaml" - -f deploy/local/generated/compose.models.yaml -) -"${compose[@]}" config --quiet -"${compose[@]}" build core frontend -"${compose[@]}" up -d catalog-db -"${compose[@]}" run --rm catalog-migrate -tht --installation "$INSTALLATION" start -``` +workspace test prova, per ogni workspace attivo, il binding del database e le credenziali, Evidence, +Qdrant e il servizio embedding. Restituisce exit code diverso da zero se manca il binding del +database o una connessione non è utilizzabile. Prima di eseguirlo l’installatore deve aver +configurato il database in Database Management: il workspace repository da solo non può contenere +la password. -Fermarsi se un comando fallisce. Il nome progetto coincide con l'hash usato da `tht`, preservando -l'identità dei volumi. `catalog-migrate` applica le migrazioni Catalog e Memory; `tht start` non -lo esegue automaticamente. Il primo download del modello embedding può richiedere tempo. -Usare questa sequenza legata all'installazione, non `run-stack.sh` con env/progetto diversi. +Per una verifica generale del core, doctor --json è il test ripetibile e non distruttivo. Il test +funzionale finale deve inoltre aprire http://127.0.0.1:8080, autenticarsi e completare una domanda +reale fino alla SQL finale. -## 4. Verificare l’installazione +## Attività che può svolgere solo l’installatore -Il descriptor generato per l’ID predefinito è: +La procedura automatizza il bootstrap, non può inventare decisioni o autorizzazioni aziendali. +L’installatore deve completare e registrare: -```sh -INSTALLATION="$(pwd -P)/deploy/local/thothii-installation.yaml" -test -f "$INSTALLATION" -bash scripts/verify-standalone-install.sh "$INSTALLATION" -``` +1. quali LLM sono utilizzabili, il relativo modelCatalog e le API key collegate; poi eseguire + tht pi test e tht doctor; +2. per ogni database: configurazione, test connessione, sincronizzazione dello schema e + generazione delle descrizioni; +3. consolidamento umano delle descrizioni generate; +4. generazione delle entry semantiche in Qdrant tramite workspace preprocess run; +5. generazione delle FK suggerite dal naming, come complemento alle FK lette dallo schema, revisione + umana delle proposte e caricamento delle relazioni approvate in Qdrant; +6. verifica periodica con tht doctor --json e tht workspace test --json; +7. una domanda reale completata con successo, senza errori di connessione o modello. -Il verificatore è read-only: esegue `tht doctor --json` e `tht status`, senza ristartare lo stack, -rigenerare la configurazione o stampare il contenuto dei segreti. +La configurazione è dichiarata completa solo quando tutti i punti applicabili sono stati eseguiti, +le decisioni sono state registrate e i due test terminali sono verdi. Il core è dichiarato usabile +solo dopo la domanda reale, non perché il frontend risponde a /health. -### Gate A — smoke di piattaforma, su tutti e tre i computer +## Gate A e Gate B -Registrare per ogni macchina: +### Gate A — piattaforma -```sh +~~~ uname -a docker version --format '{{.Server.Version}} {{.Server.Arch}}' tht version bash scripts/check-standalone-prerequisites.sh bash scripts/verify-standalone-install.sh "$INSTALLATION" -``` +~~~ -Il gate passa quando il clone è integro, Docker e Compose sono raggiungibili, `tht doctor` è OK, -lo stack è avviato e il frontend risponde sulla porta locale predefinita `http://127.0.0.1:8080`. -Il doctor controlla anche workspace e Pi: registrare separatamente i loro errori senza attribuire -ogni fallimento alla piattaforma. Verificare la disponibilità HTTP con: +### Gate B — usabilità -```sh -curl --fail --silent --show-error http://127.0.0.1:8080/health -``` - -### Gate B — verifica funzionale - -Eseguire almeno su una macchina con endpoint e credenziali disponibili: - -Seguire prima [Workspace operations](../operations/workspaces.md) per importare/preparare il -workspace e configurare Database e binding locale. Il clone sorgente non trasferisce catalogo, -segreti o sessioni del Mac. Connettere l'eventuale VPN richiesta da DWH/modelli e verificare -che i relativi nomi siano raggiungibili anche dai container. - -1. aprire `http://127.0.0.1:8080`; -2. autenticarsi con l’account locale configurato; -3. verificare che il workspace configurato sia leggibile; -4. avviare una domanda reale e completare i gate di revisione fino alla SQL finale; -5. fermare e riavviare l’installazione, poi ripetere `verify-standalone-install.sh`. - -Un fallimento del Gate B per DWH, provider LLM, Git workspace o credenziali non dimostra da solo un -problema di portabilità Docker: registrare separatamente l’endpoint o il componente fallito. - -## Ciclo di vita quotidiano - -Usare il descriptor esplicito quando più installazioni possono essere scoperte: - -```sh -INSTALLATION="$PWD/deploy/local/thothii-installation.yaml" - -tht --installation "$INSTALLATION" status -tht --installation "$INSTALLATION" start -tht --installation "$INSTALLATION" start --build -tht --installation "$INSTALLATION" logs +~~~ tht --installation "$INSTALLATION" doctor --json -tht --installation "$INSTALLATION" stop -``` +tht --installation "$INSTALLATION" workspace test --json +curl --fail --silent --show-error http://127.0.0.1:8080/health +~~~ -`start --build` è necessario dopo una modifica al codice o per ricostruire le immagini dal clone -corrente. `stop` conserva volumi, sessioni, catalogo, stato Pi, dati Qdrant e modello embedding. -Non usare `docker compose down --volumes` durante una prova normale: è un’operazione distruttiva -che cancella i dati locali. -Per aggiornamenti che richiedono migrazioni, seguire il runbook della release prima dell'avvio. +Poi eseguire una domanda reale e fermare/riavviare con tht stop e tht start. Non usare docker +compose down --volumes: cancella Catalog, sessioni, Qdrant e il modello embedding. ## Diagnosi rapida -| Sintomo | Controllo | +| Sintomo | Azione | | --- | --- | -| `Docker Engine is not reachable` | avviare Docker Desktop oppure il servizio Docker e ripetere `docker info` | -| Windows vede Docker ma Bash fallisce | eseguire la guida dentro Ubuntu WSL2 e abilitare l’integrazione della distribuzione in Docker Desktop | -| `tht: command not found` | aprire una nuova shell e verificare `command -v tht`; se necessario ripetere il bootstrap | -| line ending o script non eseguibile | usare un clone nel filesystem WSL2/Linux e rieseguire `bash scripts/...` | -| architettura non supportata | verificare `docker version --format '{{.Server.Arch}}'`; la prova richiede `amd64` o `arm64` | -| descriptor o env file mancanti | usare il percorso `deploy/local/...` generato da `tht setup`, non un file copiato casualmente | -| stack sano ma workflow fallisce | controllare separatamente URL, credential bundle, workspace Git e autenticazione | -| dati apparentemente persi | verificare che non sia stato usato `down --volumes`; `stop` non rimuove i volumi | - -## Checklist di accettazione - -- [ ] Il clone proviene dal repository Gitea atteso e la revisione è stata annotata. -- [ ] Docker Desktop/Engine e Compose v2 sono disponibili. -- [ ] Il runtime restituisce un’architettura ammessa. -- [ ] `tht` è stato costruito dal repository e risponde a `tht version`. -- [ ] Il setup usa `profile: local`, `shell.mode: full` e `shell.defaultLocale: en`. -- [ ] Descriptor, `operator.env`, autenticazione e segreti sono presenti solo in `deploy/local/`. -- [ ] Nessun segreto compare in Git, URL, YAML pubblico o comandi registrati. -- [ ] Gate A superato su macOS Apple Silicon, Windows WSL2/x64 e Linux x64. -- [ ] Gate B eseguito almeno su una macchina con DWH e LLM disponibili. -- [ ] Stop/start e verifica finale completati senza cancellare i volumi. - -## Fuori perimetro di questa release - -Restano attività successive: - -- pubblicare immagini pre-costruite su Docker Hub; -- ridurre ulteriormente i prompt tramite una configurazione non interattiva dedicata; -- creare pacchetti DMG, MSI/EXE, AppImage o altri installer nativi; -- fornire un runtime offline o un DWH/LLM locale incluso nell’applicazione. +| Docker Engine is not reachable | avviare Docker Desktop o systemctl e ripetere docker info | +| Omarchy non trova docker | installare Docker/Compose, abilitare il servizio e riaprire la sessione | +| Windows vede Docker ma Bash fallisce | usare Ubuntu WSL2 e abilitarne l’integrazione in Docker Desktop | +| pull workspace fallisce | controllare URL, branch, chiave/credential file e known_hosts dal container | +| workspace test segnala binding mancante | configurare database, token/password e CA in Database Management | +| Pi non è pronto | compilare pi-auth.json o la chiave dichiarata dal modelCatalog, poi eseguire tht pi test | ## Documenti collegati -- [Install and first start](first-start.md) -- [Shell and localization](shell-and-language.md) -- [Workspace operations](../operations/workspaces.md) -- `deploy/secrets/README.md` (runtime secrets) +- [Installazione e primo avvio](first-start.md) +- [Operazioni sui workspace](../operations/workspaces.md) +- [Database Management](../operations/database-management.md) +- [Configurazione dei modelli](../general/pi-configuration.md) +- deploy/secrets/README.md + +La pubblicazione di immagini su Docker Hub e gli installer nativi DMG/MSI/AppImage restano attività +successive: questa procedura parte dal clone Gitea e non richiede immagini Docker Hub pre-pubblicate. diff --git a/docs/operations/shell-and-localization.md b/docs/operations/shell-and-localization.md index 4ed21e65..1504840b 100644 --- a/docs/operations/shell-and-localization.md +++ b/docs/operations/shell-and-localization.md @@ -54,7 +54,7 @@ Omics. Le preferenze non modificano il descrittore installato. Per una nuova installazione autonoma, selezionare esplicitamente full: ```bash -tht setup --profile local --shell-mode full --shell-default-locale en +tht setup --complete --profile local --shell-mode full --shell-default-locale en ``` Il setup senza opzioni shell conserva per compatibilità il default embedded. diff --git a/scripts/verify-workspace-install-docs.sh b/scripts/verify-workspace-install-docs.sh index 5bdc19f0..c2ef99e5 100755 --- a/scripts/verify-workspace-install-docs.sh +++ b/scripts/verify-workspace-install-docs.sh @@ -79,7 +79,7 @@ verify_standalone_installation_guides() { 'git clone https://git.tylconsulting.it/mptyl/ThothII.git' \ 'scripts/check-standalone-prerequisites.sh' \ 'scripts/install-tht.sh' \ - 'tht setup --profile local --shell-mode full --shell-default-locale en' \ + 'tht setup --complete --profile local --shell-mode full --shell-default-locale en' \ 'scripts/verify-standalone-install.sh' \ 'Docker Hub' \ 'Gate A' \ @@ -99,8 +99,7 @@ verify_install_and_workspace_guides() { require_file "$workspace" for text in \ - 'tht setup --profile local' \ - '--configure-only' \ + 'tht setup --complete --profile local' \ 'catalog-migrate' \ 'tht --installation /absolute/path/thothii-installation.yaml doctor --json'; do require_text "$install" "$text" diff --git a/tools/tht/cmd/tht/main.go b/tools/tht/cmd/tht/main.go index 0c506618..a1db7ba9 100644 --- a/tools/tht/cmd/tht/main.go +++ b/tools/tht/cmd/tht/main.go @@ -40,9 +40,9 @@ When --installation is omitted, tht uses THOTHII_INSTALLATION or discovers one v descriptor in the current project tree. Commands: - setup [--configure-only] [--installation-id ID] [--profile local|server] + setup [--complete|--configure-only] [--installation-id ID] [--profile local|server] [--shell-mode full|embedded] [--shell-default-locale BCP47-TAG] [--shell-adapter omics-portal] - Create or validate the local non-secret installation configuration. + Create, validate, and optionally complete the local installation. installation migrate --output PATH --session-default PROVIDER/MODEL --embedding-id PROVIDER/MODEL --embedding-dimensions N Create a review-only schema-v2 candidate from all three legacy model sources. @@ -86,6 +86,10 @@ Commands: Verify a terminal installation, remove stale lifecycle files, and clear maintenance. pi logs Show the latest 200 sanitized core log lines (bounded; no follow mode). workspace inspect --workspace ID [--json] + workspace pull [--json] + Pull and activate the configured workspace repository. + workspace test [--json] + Test configured database, Evidence, Qdrant, and embedding connectivity. workspace evidence consolidate --workspace ID [--json] workspace evidence refresh --workspace ID [--json] workspace evidence decide --workspace ID --source-id SHA --revision SHA --decision keep|replace [--json] @@ -400,6 +404,11 @@ func parseSetupArgs(args []string) (setup.Request, error) { flag := args[0] args = args[1:] switch flag { + case "--complete": + if request.Complete { + return setup.Request{}, errors.New("--complete may be supplied once") + } + request.Complete = true case "--configure-only": if request.ConfigureOnly { return setup.Request{}, errors.New("--configure-only may be supplied once") @@ -484,6 +493,9 @@ func parseSetupArgs(args []string) (setup.Request, error) { *target = value } } + if request.Complete && request.ConfigureOnly { + return setup.Request{}, errors.New("--complete and --configure-only cannot be combined") + } return request, nil } @@ -499,6 +511,9 @@ func writeRemovalTargets(outputWriter io.Writer, project string, targets []serve } func workspaceCommand(ctx context.Context, installation config.Installation, runner compose.Runner, args []string, secretValues []string, stdout, stderr io.Writer) int { + if len(args) > 0 && (args[0] == "pull" || args[0] == "test") { + return workspaceOperatorCommand(ctx, installation, runner, args, secretValues, stdout, stderr) + } request, err := workspaceops.Parse(args) if err != nil { return commandUsageError(stderr, err.Error()) @@ -527,6 +542,51 @@ func workspaceCommand(ctx context.Context, installation config.Installation, run } } +func workspaceOperatorCommand(ctx context.Context, installation config.Installation, runner compose.Runner, args []string, secretValues []string, stdout, stderr io.Writer) int { + action := "workspace-" + args[0] + jsonMode := false + for _, arg := range args[1:] { + if arg != "--json" || jsonMode { + return commandUsageError(stderr, "workspace pull/test accepts only --json") + } + jsonMode = true + } + result, err := runner.Run(ctx, installation.ComposeArgs("exec", "-T", "core", "node", "dist/operator-command.js", action), nil) + if err != nil { + return writeResult(result, err, secretValues, stdout, stderr) + } + var payload struct { + Ready bool `json:"ready"` + Status string `json:"status"` + } + if err := json.Unmarshal([]byte(result.Stdout), &payload); err != nil { + fmt.Fprintln(stderr, "tht: workspace operator returned invalid JSON") + return 1 + } + if jsonMode { + fmt.Fprintln(stdout, output.Sanitize(result.Stdout, secretValues)) + } else { + fmt.Fprintf(stdout, "workspace %s: %s\n", args[0], output.Sanitize(workspaceOperatorSummary(payload), secretValues)) + } + if !payload.Ready { + return 1 + } + return 0 +} + +func workspaceOperatorSummary(payload struct { + Ready bool `json:"ready"` + Status string `json:"status"` +}) string { + if payload.Status != "" { + return payload.Status + } + if payload.Ready { + return "ready" + } + return "failed" +} + func workspaceFailure(stderr io.Writer, err error, secretValues []string) int { message := output.Sanitize(err.Error(), secretValues) var operationErr *workspaceops.OperationError diff --git a/tools/tht/internal/setup/files.go b/tools/tht/internal/setup/files.go index 8e521d3e..6efd0ac0 100644 --- a/tools/tht/internal/setup/files.go +++ b/tools/tht/internal/setup/files.go @@ -3,6 +3,8 @@ package setup import ( "bufio" "bytes" + "crypto/rand" + "encoding/hex" "errors" "fmt" "io" @@ -13,6 +15,7 @@ import ( "sort" "strconv" "strings" + "unicode" "github.com/aritmolab/thothii/tools/tht/internal/config" "github.com/aritmolab/thothii/tools/tht/internal/safeio" @@ -26,6 +29,7 @@ const ( ) var installationIDPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9_-]*$`) +var secretBundleKeyPattern = regexp.MustCompile(`^[A-Z][A-Z0-9_]{0,127}$`) // atomicWriteNewFile is a seam for failure testing. Its implementation never replaces an existing // file and leaves no final target until all content is synced. @@ -44,6 +48,7 @@ type answers struct { secretsFile, piAuthFile string gitCredentialsFile, gitCAFile string gitSSHKeyFile, gitKnownHostsFile string + complete bool createSecretTemplates bool } @@ -116,6 +121,11 @@ func EnsureFiles(request Request, input io.Reader, output io.Writer) (FilesResul if err := validateOrCreateSecretFiles(values, output); err != nil { return FilesResult{}, err } + if values.complete { + if err := validateCompleteProtectedFiles(values); err != nil { + return FilesResult{}, err + } + } created := make([]string, 0, 2) cleanup := func() { @@ -163,6 +173,27 @@ func collectAnswers(request Request, input io.Reader, output io.Writer, root str value := answersFromRequest(request) value.installationID = firstNonEmpty(request.InstallationID, os.Getenv("THT_SETUP_INSTALLATION_ID"), "local") value.profile = firstNonEmpty(request.Profile, os.Getenv("THT_SETUP_PROFILE"), "local") + value.complete = request.Complete + if request.Complete { + // The complete path has one predictable protected directory. The user only fills the + // bundle and any repository credential that is genuinely required; catalog passwords + // are generated below and never appear in the questionnaire. + directory := filepath.Join(root, "deploy", value.installationID, "secrets") + value.workspaceBranch = firstNonEmpty(value.workspaceBranch, "main") + value.secretsFile = firstNonEmpty(value.secretsFile, filepath.Join(directory, "thothii.secrets")) + value.piAuthFile = firstNonEmpty(value.piAuthFile, filepath.Join(directory, "pi-auth.json")) + if request.NonInteractive { + value.workspaceAccess = firstNonEmpty(value.workspaceAccess, accessForRemote(value.workspaceRemote)) + if value.workspaceAccess == "ssh" { + value.gitSSHKeyFile = firstNonEmpty(value.gitSSHKeyFile, filepath.Join(directory, "workspace-git-key")) + value.gitKnownHostsFile = firstNonEmpty(value.gitKnownHostsFile, filepath.Join(directory, "workspace-git-known-hosts")) + } else { + value.gitCredentialsFile = firstNonEmpty(value.gitCredentialsFile, filepath.Join(directory, "workspace-git-credentials")) + value.gitCAFile = firstNonEmpty(value.gitCAFile, filepath.Join(directory, "workspace-git-ca.pem")) + } + } + value.createSecretTemplates = true + } if request.NonInteractive { return requireNonInteractiveAnswers(value) } @@ -190,6 +221,18 @@ func collectAnswers(request Request, input io.Reader, output io.Writer, root str return answers{}, err } directory := filepath.Join(root, "deploy", value.installationID, "secrets") + if request.Complete { + value.secretsFile = firstNonEmpty(value.secretsFile, filepath.Join(directory, "thothii.secrets")) + value.piAuthFile = firstNonEmpty(value.piAuthFile, filepath.Join(directory, "pi-auth.json")) + if value.workspaceAccess == "ssh" { + value.gitSSHKeyFile = firstNonEmpty(value.gitSSHKeyFile, filepath.Join(directory, "workspace-git-key")) + value.gitKnownHostsFile = firstNonEmpty(value.gitKnownHostsFile, filepath.Join(directory, "workspace-git-known-hosts")) + } else { + value.gitCredentialsFile = firstNonEmpty(value.gitCredentialsFile, filepath.Join(directory, "workspace-git-credentials")) + value.gitCAFile = firstNonEmpty(value.gitCAFile, filepath.Join(directory, "workspace-git-ca.pem")) + } + return value, nil + } if value.secretsFile, err = prompt(scanner, output, "Secret file location", firstNonEmpty(value.secretsFile, filepath.Join(directory, "thothii.secrets"))); err != nil { return answers{}, err } @@ -216,11 +259,15 @@ func collectAnswers(request Request, input io.Reader, output io.Writer, root str return answers{}, missingErr } if len(missing) > 0 { - answer, promptErr := prompt(scanner, output, "Create blank secret-file templates for the missing locations? Type yes to confirm", "no") - if promptErr != nil { - return answers{}, promptErr + if request.Complete { + value.createSecretTemplates = true + } else { + answer, promptErr := prompt(scanner, output, "Create blank secret-file templates for the missing locations? Type yes to confirm", "no") + if promptErr != nil { + return answers{}, promptErr + } + value.createSecretTemplates = strings.EqualFold(answer, "yes") } - value.createSecretTemplates = strings.EqualFold(answer, "yes") } return value, nil } @@ -379,6 +426,13 @@ func render(root, descriptorPath string, value answers) ([]byte, []byte, error) if value.llmURL != "" { lines = append(lines, "THT_LLM_URL="+dotenvValue(value.llmURL)) } + if value.complete { + passwordDirectory := filepath.Dir(value.secretsFile) + lines = append(lines, + "THT_CATALOG_RUNTIME_PASSWORD_SOURCE="+dotenvValue(filepath.Join(passwordDirectory, "catalog-runtime-password")), + "THT_CATALOG_MIGRATOR_PASSWORD_SOURCE="+dotenvValue(filepath.Join(passwordDirectory, "catalog-migrator-password")), + ) + } if value.profile == "server" { installationDirectory := filepath.Dir(descriptorPath) lines = append(lines, @@ -474,10 +528,7 @@ func validateOrCreateSecretFiles(value answers, output io.Writer) error { if err != nil { return err } - if len(missing) == 0 { - return nil - } - if !value.createSecretTemplates { + if len(missing) > 0 && !value.createSecretTemplates { return fmt.Errorf("secret files are missing: %s; create them yourself or explicitly confirm blank secret-file templates", strings.Join(missing, ", ")) } for _, path := range missing { @@ -489,6 +540,104 @@ func validateOrCreateSecretFiles(value answers, output io.Writer) error { } fmt.Fprintf(output, "Created blank secret-file template: %s\n", path) } + if value.complete { + for _, path := range catalogPasswordPaths(value) { + exists, err := inspectExistingSecretFile(path) + if err != nil { + return err + } + if exists { + continue + } + contents, err := generatedCatalogPassword() + if err != nil { + return fmt.Errorf("generate catalog password: %w", err) + } + if err := atomicWriteNewFile(path, contents, 0o600); err != nil { + return fmt.Errorf("create catalog password %s: %w", path, err) + } + fmt.Fprintf(output, "Created generated catalog password file: %s\n", path) + } + } + return nil +} + +func catalogPasswordPaths(value answers) []string { + directory := filepath.Dir(value.secretsFile) + return []string{ + filepath.Join(directory, "catalog-runtime-password"), + filepath.Join(directory, "catalog-migrator-password"), + } +} + +func generatedCatalogPassword() ([]byte, error) { + value := make([]byte, 32) + if _, err := rand.Read(value); err != nil { + return nil, errors.New("secure random source is unavailable") + } + return []byte(hex.EncodeToString(value) + "\n"), nil +} + +func validateCompleteProtectedFiles(value answers) error { + if err := validateSecretBundle(value.secretsFile); err != nil { + return err + } + // The generated catalog deliberately uses Pi's built-in provider. A syntactically empty + // auth store would let Docker start only to fail at the first provider check, so catch it + // before any image is built. Other model providers can be selected later in the descriptor. + contents, err := safeio.ReadCanonicalRegular(value.piAuthFile, maxSecretBytes) + if err != nil || strings.TrimSpace(string(contents)) == "" || strings.TrimSpace(string(contents)) == "{}" { + return fmt.Errorf("complete setup requires usable Pi credentials in %s", value.piAuthFile) + } + if value.workspaceAccess == "ssh" { + for name, path := range map[string]string{ + "workspace Git SSH key": value.gitSSHKeyFile, + "workspace Git known-hosts": value.gitKnownHostsFile, + } { + contents, readErr := safeio.ReadCanonicalRegular(path, maxSecretBytes) + if readErr != nil || strings.TrimSpace(string(contents)) == "" { + return fmt.Errorf("complete setup requires usable %s in %s", name, path) + } + } + } else { + for name, path := range map[string]string{ + "workspace Git credentials": value.gitCredentialsFile, + "workspace Git CA": value.gitCAFile, + } { + contents, readErr := safeio.ReadCanonicalRegular(path, maxSecretBytes) + if readErr != nil || strings.TrimSpace(string(contents)) == "" { + return fmt.Errorf("complete setup requires usable %s in %s", name, path) + } + } + } + return nil +} + +func validateSecretBundle(path string) error { + contents, err := safeio.ReadCanonicalRegular(path, maxSecretBytes) + if err != nil { + return fmt.Errorf("complete setup cannot read the secret bundle %s", path) + } + seen := make(map[string]struct{}) + for lineNumber, raw := range strings.Split(string(contents), "\n") { + line := strings.TrimSuffix(raw, "\r") + trimmed := strings.TrimSpace(line) + if trimmed == "" || strings.HasPrefix(trimmed, "#") { + continue + } + key, secret, found := strings.Cut(line, "=") + invalid := !found || !secretBundleKeyPattern.MatchString(key) || strings.TrimSpace(key) != key || + secret == "" || strings.TrimSpace(secret) != secret || + strings.Contains(strings.ToLower(secret), "replace-me") || + strings.IndexFunc(secret, unicode.IsSpace) >= 0 + if invalid { + return fmt.Errorf("complete setup found an invalid secret bundle entry at line %d", lineNumber+1) + } + if _, duplicate := seen[key]; duplicate { + return fmt.Errorf("complete setup found a duplicate secret bundle key %s", key) + } + seen[key] = struct{}{} + } return nil } diff --git a/tools/tht/internal/setup/files_test.go b/tools/tht/internal/setup/files_test.go index 2f98bd68..65fd0930 100644 --- a/tools/tht/internal/setup/files_test.go +++ b/tools/tht/internal/setup/files_test.go @@ -207,6 +207,50 @@ func TestEnsureFilesRequiresExplicitNonInteractiveAnswers(t *testing.T) { } } +func TestCompleteSetupCreatesProtectedPlaceholdersThenRequiresUsableCredentials(t *testing.T) { + root := newProject(t, "complete setup") + for name, value := range map[string]string{ + "THT_SETUP_WORKSPACE_REMOTE": "git@git.example.invalid:team/workspaces.git", + "THT_SETUP_WORKSPACE_BRANCH": "main", + "THT_SETUP_WORKSPACE_ACCESS": "ssh", + } { + t.Setenv(name, value) + } + request := Request{ProjectRoot: root, InstallationID: "local", Profile: "local", Complete: true, NonInteractive: true} + if _, err := EnsureFiles(request, strings.NewReader(""), ioDiscard{}); err == nil || !strings.Contains(err.Error(), "usable Pi credentials") { + t.Fatalf("first complete setup error = %v, want the placeholder guidance", err) + } + secretRoot := filepath.Join(root, "deploy", "local", "secrets") + for path, contents := range map[string]string{ + filepath.Join(secretRoot, "pi-auth.json"): "{\"deepseek\":{\"apiKey\":\"configured\"}}\n", + filepath.Join(secretRoot, "workspace-git-key"): "private-key\n", + filepath.Join(secretRoot, "workspace-git-known-hosts"): "git.example.invalid ssh-ed25519 AAAA\n", + } { + if err := os.WriteFile(path, []byte(contents), 0o600); err != nil { + t.Fatal(err) + } + } + result, err := EnsureFiles(request, strings.NewReader(""), ioDiscard{}) + if err != nil { + t.Fatal(err) + } + environment, err := os.ReadFile(result.EnvironmentPath) + if err != nil { + t.Fatal(err) + } + for _, name := range []string{"THT_CATALOG_RUNTIME_PASSWORD_SOURCE", "THT_CATALOG_MIGRATOR_PASSWORD_SOURCE"} { + if !strings.Contains(string(environment), name+"=") { + t.Fatalf("complete environment misses %s: %s", name, environment) + } + } + for _, name := range []string{"catalog-runtime-password", "catalog-migrator-password"} { + contents, readErr := os.ReadFile(filepath.Join(secretRoot, name)) + if readErr != nil || len(strings.TrimSpace(string(contents))) < 32 { + t.Fatalf("generated catalog password %s is unavailable or too short", name) + } + } +} + func TestEnsureFilesIncludesServerStorageLocations(t *testing.T) { requireProjectedServerTestHost(t) root := newProject(t, "server profile") diff --git a/tools/tht/internal/setup/request.go b/tools/tht/internal/setup/request.go index 13f6b833..b90cb3c2 100644 --- a/tools/tht/internal/setup/request.go +++ b/tools/tht/internal/setup/request.go @@ -1,12 +1,15 @@ // Package setup creates the local, non-secret configuration selected by tht setup. package setup -// Request contains the stable setup-file inputs. Task 5 will use ConfigureOnly when it adds -// Compose validation and lifecycle orchestration. +// Request contains the stable setup-file inputs. type Request struct { ProjectRoot string InstallationID string Profile string + // Complete runs the installation-only steps that are safe to automate: catalog migration, + // stack startup, and the initial workspace pull. It intentionally does not invent database + // bindings or credentials that belong to the installation operator. + Complete bool ConfigureOnly bool NonInteractive bool Answers Answers diff --git a/tools/tht/internal/setup/run.go b/tools/tht/internal/setup/run.go index 67d99f44..ee214897 100644 --- a/tools/tht/internal/setup/run.go +++ b/tools/tht/internal/setup/run.go @@ -3,6 +3,7 @@ package setup import ( "context" + "encoding/json" "errors" "fmt" "io" @@ -35,7 +36,8 @@ type Result struct { } // Run validates the host, creates or validates non-secret configuration, and by default builds, -// starts, and verifies the current checkout. ConfigureOnly stops after Compose rendering. +// starts, and verifies the current checkout. Complete additionally migrates the Catalog and +// imports the configured workspace repository. ConfigureOnly stops after Compose rendering. func Run(ctx context.Context, runner compose.Runner, request Request, input io.Reader, output io.Writer) (Result, error) { if runner == nil { return Result{}, errors.New("setup requires a Docker command runner") @@ -71,13 +73,33 @@ func Run(ctx context.Context, runner compose.Runner, request Request, input io.R fmt.Fprintf(output, "Configuration is ready: %s\n", result.DescriptorPath) return result, nil } - if err := service.Start(ctx, installation, runner, true); err != nil { + if request.Complete { + if err := runCompose(ctx, runner, installation, "build"); err != nil { + return Result{}, fmt.Errorf("setup image build: %w", err) + } + if err := runCompose(ctx, runner, installation, "up", "--detach", "catalog-db"); err != nil { + return Result{}, fmt.Errorf("setup Catalog database start: %w", err) + } + if err := runCompose(ctx, runner, installation, + "--profile", "catalog-maintenance", "run", "--rm", "catalog-migrate"); err != nil { + return Result{}, fmt.Errorf("setup Catalog migration: %w", err) + } + } + if err := service.Start(ctx, installation, runner, !request.Complete); err != nil { if strings.Contains(err.Error(), "image build") { return Result{}, fmt.Errorf("setup %w", err) } return Result{}, withStartupRecovery(fmt.Errorf("setup %w", err), recoveryService(err)) } result.Built, result.Started, result.Healthy = true, true, true + if request.Complete { + if err := runOperator(ctx, runner, installation, "workspace-pull"); err != nil { + return Result{}, withStartupRecovery(fmt.Errorf("setup workspace import: %w", err), "core") + } + if err := runOperator(ctx, runner, installation, "pi-test"); err != nil { + return Result{}, withStartupRecovery(fmt.Errorf("setup LLM credential test: %w", err), "core") + } + } report, err := doctor.Run(ctx, installation, runner) if err != nil { return Result{}, withStartupRecovery(fmt.Errorf("setup doctor: %w", err), "core") @@ -85,6 +107,9 @@ func Run(ctx context.Context, runner compose.Runner, request Request, input io.R if !report.OK { return Result{}, withStartupRecovery(errors.New("setup doctor reported failed checks"), "core") } + if request.Complete { + fmt.Fprintln(output, "Workspace repository pulled and activated; run 'tht workspace test' after configuring each workspace database.") + } fmt.Fprintf(output, "ThothII is ready at %s\nInstallation descriptor: %s\nNext: tht status\n", frontendURL(installation), result.DescriptorPath) return result, nil } @@ -219,6 +244,25 @@ func runCompose(ctx context.Context, runner compose.Runner, installation config. return nil } +func runOperator(ctx context.Context, runner compose.Runner, installation config.Installation, action string) error { + result, err := runner.Run(ctx, installation.ComposeArgs( + "exec", "-T", "core", "node", "dist/operator-command.js", action, + ), nil) + if err != nil { + if result.ExitCode != 0 { + return fmt.Errorf("Docker exited with status %d", result.ExitCode) + } + return err + } + var payload struct { + Ready *bool `json:"ready"` + } + if err := json.Unmarshal([]byte(result.Stdout), &payload); err != nil || payload.Ready == nil || !*payload.Ready { + return fmt.Errorf("operator action %s reported failure", action) + } + return nil +} + func composeFailure(result compose.Result, cause error) error { if result.ExitCode != 0 { return fmt.Errorf("Docker exited with status %d", result.ExitCode) diff --git a/tools/tht/internal/setup/run_test.go b/tools/tht/internal/setup/run_test.go index 0df79581..1d6f61f9 100644 --- a/tools/tht/internal/setup/run_test.go +++ b/tools/tht/internal/setup/run_test.go @@ -68,6 +68,34 @@ func TestRunConfigureOnlyStopsAfterRenderedConfiguration(t *testing.T) { } } +func TestRunCompleteMigratesCatalogPullsWorkspaceAndTestsLLM(t *testing.T) { + root, request := setupRunFixture(t, false) + request.Complete = true + secretRoot := filepath.Join(root, "deploy", "ci", "secrets") + for path, contents := range map[string]string{ + filepath.Join(secretRoot, "pi-auth.json"): "{\"deepseek\":{\"apiKey\":\"configured\"}}\n", + filepath.Join(secretRoot, "workspace-git-key"): "private-key\n", + filepath.Join(secretRoot, "workspace-git-known-hosts"): "git.example.invalid ssh-ed25519 AAAA\n", + } { + if err := os.WriteFile(path, []byte(contents), 0o600); err != nil { + t.Fatal(err) + } + } + runner := &setupRunner{health: []string{healthyServicesJSON}} + if _, err := Run(context.Background(), runner, request, strings.NewReader(""), io.Discard); err != nil { + t.Fatal(err) + } + want := []string{ + "docker engine", "docker compose", "architecture", "compose config", "compose build", + "catalog db", "catalog migrate", "compose up", "health", "workspace pull", "pi doctor", + "doctor docker", "doctor compose", "compose config", "doctor config", "health", + "authentication", "core HTTP", "frontend HTTP", "workspace registry", "workflow doctor", "pi doctor", + } + if got := collapseStages(runner.stages); strings.Join(got, " | ") != strings.Join(want, " | ") { + t.Fatalf("complete setup stages = %v, want %v", got, want) + } +} + func TestRunConfiguresAndStaticallyValidatesLocalAuthBeforeComposeRender(t *testing.T) { projectRoot, request := setupRunFixture(t, true) passwordFile := filepath.Join(projectRoot, "initial-admin-password") @@ -421,6 +449,10 @@ func setupStage(args []string) (string, compose.Result) { return "architecture", compose.Result{Stdout: "arm64\n"} case strings.HasSuffix(joined, " config --quiet"): return "compose config", compose.Result{} + case strings.HasSuffix(joined, " up --detach catalog-db"): + return "catalog db", compose.Result{} + case strings.HasSuffix(joined, " --profile catalog-maintenance run --rm catalog-migrate"): + return "catalog migrate", compose.Result{} case strings.HasSuffix(joined, " build"): return "compose build", compose.Result{} case strings.HasSuffix(joined, " up --detach --remove-orphans"): @@ -433,6 +465,8 @@ func setupStage(args []string) (string, compose.Result) { return "authentication", compose.Result{Stdout: `{"ready":true,"mode":"oidc","checks":[{"level":"info","code":"auth_ready","message":"Authentication is ready."}]}`} case strings.Contains(joined, "exec -T core node dist/operator-command.js workflow-doctor"): return "workflow doctor", compose.Result{Stdout: `{"ready":true,"workspaces":1}`} + case strings.Contains(joined, "operator-command.js workspace-pull"): + return "workspace pull", compose.Result{Stdout: `{"ready":true,"status":"succeeded"}`} case strings.Contains(joined, "exec -T core curl -fsS --max-time 5 http://127.0.0.1:8787/health"): return "core HTTP", compose.Result{} case strings.Contains(joined, "exec -T frontend wget -q -T 5 -O /dev/null http://127.0.0.1:8080/"):