diff --git a/README.md b/README.md index da6955fc..fee46447 100644 --- a/README.md +++ b/README.md @@ -53,7 +53,7 @@ The frontend depends on the core health check and proxies `/health` and `/api/*` application health endpoint intentionally checks process readiness only; external dependency diagnostics are exposed by `tht doctor` and do not prevent the UI from starting. -## Git-backed workspace registry +## Git-backed workspace repository Workspace descriptors are shared through a validated Git repository while endpoint bindings and secret files remain installation-local. Use the [local Mac/PC installation manual](docs/install/local-workspace-registry.md) @@ -68,20 +68,19 @@ The curator-owned repository layout is: thoth-workspaces.yaml /workspace.yaml /evidence/** -workspace-docs//{contract.env.example,README.md} ``` `thoth-workspaces.yaml` uses the `schema_version` value `1` and the ordered `workspaces` list of `{id, name, description?}` entries. It is authoritative for workspace ID, name, description, and -display order. The API may create `/workspace.yaml` only when the catalog slot already exists -and the descriptor is absent. A pulled catalog-only slot reports `configuration_required`. After -bootstrap, existing descriptors remain curator-owned and change only through curator Git commit, -push, and installation pull. The API never writes `thoth-workspaces.yaml` or `/evidence/**`; -its generated docs live only at `workspace-docs//{contract.env.example,README.md}`. +display order. Every catalog entry must have a matching descriptor in the same commit; otherwise +the complete candidate is rejected. Descriptors remain curator-owned and change only through a +Git commit and push from a separate authoring clone, followed by an installation pull. ThothII +never writes any workspace repository content. The operator workflow is: curate catalog/descriptor/Evidence changes in Git, commit and push, -**Pull latest registry** from each ThothII installation, run **Validate workspace** and **Test on -this installation**, then select the workspace locally before creating sessions. Each new session +**Update workspace repository** from each ThothII installation, select the workspace, complete its +write-only runtime-secret fields, run **Validate workspace source** and **Test workspace +connections**, then select the workspace locally before creating sessions. Each new session pins the Git revision it used; a later pull cannot change a Resume. Snapshot cleanup retains every revision referenced by an open, closed, or failed unarchived session. It reconciles from the single local installation list or from a server administrator's complete session list, never from diff --git a/compose.yaml b/compose.yaml index 1c257986..97c0bafc 100644 --- a/compose.yaml +++ b/compose.yaml @@ -18,8 +18,8 @@ services: THT_WORKSPACE_GIT_REMOTE: ${THT_WORKSPACE_GIT_REMOTE:?set THT_WORKSPACE_GIT_REMOTE} THT_WORKSPACE_GIT_BRANCH: ${THT_WORKSPACE_GIT_BRANCH:-main} THT_WORKSPACE_INSTALLATION_ID: ${THT_WORKSPACE_INSTALLATION_ID:-local} - THT_WORKSPACE_GIT_AUTHOR_NAME: ${THT_WORKSPACE_GIT_AUTHOR_NAME:-Thoth Workspace Registry} - THT_WORKSPACE_GIT_AUTHOR_EMAIL: ${THT_WORKSPACE_GIT_AUTHOR_EMAIL:-thoth-workspace-registry@localhost} + THT_WORKSPACE_SECRET_STORE_ROOT: /data/workspace-secrets + THT_WORKSPACE_SECRET_RUNTIME_ROOT: /tmp/thothii-workspace-secrets THT_WORKSPACE_SECRET_ROOTS: /run/secrets THT_SECRETS_FILE: /run/secrets/thothii.secrets THT_DB_NAME: ${THT_DB_NAME:-} @@ -37,6 +37,7 @@ services: - ./deploy/pi/models.json:/home/thoth/.pi/agent/models.json:ro - ./deploy/pi/settings.json:/home/thoth/.pi/agent/settings.json:ro - workspace-registry:/data/workspace-registry + - workspace-secrets:/data/workspace-secrets - sessions:/data/sessions secrets: - source: thothii_secrets @@ -68,6 +69,8 @@ services: THT_WORKSPACE_GIT_REMOTE: ${THT_WORKSPACE_GIT_REMOTE:?set THT_WORKSPACE_GIT_REMOTE} THT_WORKSPACE_GIT_BRANCH: ${THT_WORKSPACE_GIT_BRANCH:-main} THT_WORKSPACE_INSTALLATION_ID: ${THT_WORKSPACE_INSTALLATION_ID:-local} + THT_WORKSPACE_SECRET_STORE_ROOT: /data/workspace-secrets + THT_WORKSPACE_SECRET_RUNTIME_ROOT: /tmp/thothii-workspace-secrets THT_WORKSPACE_SECRET_ROOTS: /run/secrets THT_SECRETS_FILE: /run/secrets/thothii.secrets THT_DB_NAME: ${THT_DB_NAME:-} @@ -93,6 +96,9 @@ services: - type: volume source: sessions target: /data/sessions + - type: volume + source: workspace-secrets + target: /data/workspace-secrets secrets: - source: thothii_secrets target: thothii.secrets @@ -193,6 +199,7 @@ volumes: settings: pi-state: workspace-registry: + workspace-secrets: sessions: qdrant-data: embedding-models: diff --git a/deploy/compose.git-ssh.yaml b/deploy/compose.git-ssh.yaml index c3c7a7a9..6c33b336 100644 --- a/deploy/compose.git-ssh.yaml +++ b/deploy/compose.git-ssh.yaml @@ -1,5 +1,5 @@ # Select this override only for an SSH Git remote. The host-only source files must be absolute, -# normalized paths; strict host-key checking is mandatory for registry pull and publish. +# normalized paths; strict host-key checking is mandatory for read-only repository fetch and pull. # Active-snapshot workspace-maintenance operations intentionally receive no Git credential mounts. x-thoth-git-transport: ssh diff --git a/deploy/compose.server.yaml b/deploy/compose.server.yaml index 9c37d091..98c5365e 100644 --- a/deploy/compose.server.yaml +++ b/deploy/compose.server.yaml @@ -49,4 +49,7 @@ services: source: ${THT_WORKSPACE_REGISTRY_ROOT:?set THT_WORKSPACE_REGISTRY_ROOT} target: /data/workspace-registry read_only: true + - type: bind + source: ${THT_DATA_ROOT:?set THT_DATA_ROOT}/workspace-secrets + target: /data/workspace-secrets restart: "no" diff --git a/deploy/env/local.env.example b/deploy/env/local.env.example index 712ceca2..1714bc4b 100644 --- a/deploy/env/local.env.example +++ b/deploy/env/local.env.example @@ -8,8 +8,6 @@ THT_SECRETS_FILE=/absolute/path/to/thothii.secrets THT_WORKSPACE_GIT_REMOTE=https://git.example.invalid/platform/thoth-workspaces.git THT_WORKSPACE_GIT_BRANCH=main -THT_WORKSPACE_GIT_AUTHOR_NAME="Thoth Workspace Registry" -THT_WORKSPACE_GIT_AUTHOR_EMAIL=thoth-workspace-registry@example.invalid THT_DB_NAME=warehouse THT_DWH_REST_URL=https://dwh.example.invalid diff --git a/deploy/env/server.env.example b/deploy/env/server.env.example index 5e162402..22e6f40f 100644 --- a/deploy/env/server.env.example +++ b/deploy/env/server.env.example @@ -13,8 +13,6 @@ THT_BACKUP_ROOT=/srv/thothii-backups THT_SERVER_WORKSPACE_CONFIG=/absolute/path/to/server-sessions.yaml THT_WORKSPACE_GIT_REMOTE=https://git.example.invalid/platform/thoth-workspaces.git THT_WORKSPACE_GIT_BRANCH=main -THT_WORKSPACE_GIT_AUTHOR_NAME="Thoth Workspace Registry" -THT_WORKSPACE_GIT_AUTHOR_EMAIL=thoth-workspace-registry@example.invalid THT_DB_NAME=warehouse THT_DWH_REST_URL=https://dwh.example.invalid diff --git a/deploy/psd/operator.env.example b/deploy/psd/operator.env.example index d8f69b50..0816595e 100644 --- a/deploy/psd/operator.env.example +++ b/deploy/psd/operator.env.example @@ -2,18 +2,14 @@ THT_WORKSPACE_GIT_REMOTE=git@github.com:mptyl/tht-workspace-psd.git THT_WORKSPACE_GIT_BRANCH=main THT_WORKSPACE_INSTALLATION_ID=psd-local -THT_WORKSPACE_GIT_AUTHOR_NAME="Thoth PSD" -THT_WORKSPACE_GIT_AUTHOR_EMAIL=thoth-psd@example.invalid THT_WORKSPACE_GIT_SSH_KEY_FILE=/deploy/psd/secrets/git-ssh-key THT_WORKSPACE_GIT_KNOWN_HOSTS_FILE=/deploy/psd/secrets/git-known-hosts # App THT_SECRETS_FILE=/deploy/psd/secrets/thothii.secrets PI_AUTH_FILE=/deploy/psd/secrets/pi-auth.json -THT_WORKSPACE_BINDINGS_ENV_FILE=/deploy/psd/workspace-bindings.env - -# Connector secret sources (host-only paths) -THT_WS_PSD_CLINICAL_DWH_API_KEY_SOURCE=/deploy/psd/secrets/psd-clinical-dwh-api-key +# DWH and Evidence credentials are entered later in Workspace management and stored encrypted +# by the backend. They do not depend on host filesystem paths. # Pi (LLM) PI_PROVIDER=zai diff --git a/deploy/psd/thothii-installation.yaml.example b/deploy/psd/thothii-installation.yaml.example index 4102c3f7..57b8fa5c 100644 --- a/deploy/psd/thothii-installation.yaml.example +++ b/deploy/psd/thothii-installation.yaml.example @@ -9,4 +9,3 @@ workspaceRepository: access: ssh overrides: - "/projects/ThothII/deploy/compose.git-ssh.yaml" - - "/projects/ThothII/deploy/psd/connector-secrets.yaml" diff --git a/deploy/workspace-registry.env.example b/deploy/workspace-registry.env.example index 7834ecc0..835d63f3 100644 --- a/deploy/workspace-registry.env.example +++ b/deploy/workspace-registry.env.example @@ -1,11 +1,9 @@ -# Copy these non-secret registry settings into the installation environment. +# Copy these non-secret workspace repository settings into the installation environment. # Select at most one Git transport override. Every host path below must be absolute and normalized. -# Their contents are never committed, emitted by the API, or stored in the registry. +# Their contents are never committed, emitted by the API, or stored in the repository checkout. THT_WORKSPACE_REGISTRY_ROOT=/data/workspace-registry THT_WORKSPACE_GIT_BRANCH=main THT_WORKSPACE_INSTALLATION_ID=local -THT_WORKSPACE_GIT_AUTHOR_NAME=Thoth Workspace Registry -THT_WORKSPACE_GIT_AUTHOR_EMAIL=thoth-workspace-registry@localhost # Set the remote for this installation; use its own SSH/HTTPS address, never an application endpoint. # THT_WORKSPACE_GIT_REMOTE=ssh://git@your-git-host/your-org/thoth-workspaces.git @@ -14,10 +12,8 @@ THT_WORKSPACE_GIT_AUTHOR_EMAIL=thoth-workspace-registry@localhost # THT_WORKSPACE_GIT_SSH_KEY_FILE=/absolute/path/to/git-ssh-key # THT_WORKSPACE_GIT_KNOWN_HOSTS_FILE=/absolute/path/to/git-known-hosts -# Generate an untracked connector override from arbitrary THT_WS_*_FILE bindings and their -# matching host-only THT_WS_*_SOURCE paths. The generator records paths and variable names only; -# it never writes secret values into the generated Compose file. -# scripts/generate-connector-secrets-override.sh --bindings-env /absolute/path/workspace-bindings.env \ -# --operator-env /absolute/path/operator.env --output deploy/compose.connector-secrets.local.yaml +# Workspace connector credentials are entered after installation in Workspace management. +# ThothII encrypts them in its workspace-secrets volume and never returns their values to the GUI. +# Git credentials remain installation-only and are selected with one transport override below. # Run Compose through scripts/compose-with-preflight.sh so relative, non-normalized, and mixed # SSH/HTTPS selections are rejected before Docker receives the invocation. diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 8cc1c85f..e869c91e 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -26,8 +26,8 @@ services: THT_WORKSPACE_GIT_REMOTE: ${THT_WORKSPACE_GIT_REMOTE:?set THT_WORKSPACE_GIT_REMOTE} THT_WORKSPACE_GIT_BRANCH: ${THT_WORKSPACE_GIT_BRANCH:-main} THT_WORKSPACE_INSTALLATION_ID: ${THT_WORKSPACE_INSTALLATION_ID:-local} - THT_WORKSPACE_GIT_AUTHOR_NAME: ${THT_WORKSPACE_GIT_AUTHOR_NAME:-Thoth Workspace Registry} - THT_WORKSPACE_GIT_AUTHOR_EMAIL: ${THT_WORKSPACE_GIT_AUTHOR_EMAIL:-thoth-workspace-registry@localhost} + THT_WORKSPACE_SECRET_STORE_ROOT: /data/workspace-secrets + THT_WORKSPACE_SECRET_RUNTIME_ROOT: /tmp/thothii-workspace-secrets THT_WORKSPACE_SECRET_ROOTS: /run/secrets THT_SECRETS_FILE: /run/secrets/thothii.secrets THT_DB_NAME: ${THT_DB_NAME:-} @@ -47,6 +47,7 @@ services: - ./deploy/pi/models.json:/home/thoth/.pi/agent/models.json:ro - ./deploy/pi/settings.json:/home/thoth/.pi/agent/settings.json:ro - workspace-registry:/data/workspace-registry + - workspace-secrets:/data/workspace-secrets - ${THT_DEV_EVIDENCE_HOST_PATH:-./evidence}:/data/evidence:ro secrets: - source: thothii_secrets @@ -145,6 +146,7 @@ volumes: dev-data: dev-pi-state: workspace-registry: + workspace-secrets: qdrant-data: embedding-models: diff --git a/docs/contracts/workspace-evidence-v3.md b/docs/contracts/workspace-evidence-v3.md index 28b562e0..2461b4f0 100644 --- a/docs/contracts/workspace-evidence-v3.md +++ b/docs/contracts/workspace-evidence-v3.md @@ -123,41 +123,37 @@ hold file paths, never credential or signed-URL values. | Static S3 pair | `THT_WS__EVIDENCE_ACCESS_KEY_FILE` and `THT_WS__EVIDENCE_SECRET_KEY_FILE` | Required together for `static_files`; each file is at most 65536 bytes. | | Static S3 session | `THT_WS__EVIDENCE_SESSION_TOKEN_FILE` | Optional, valid only with the required access/secret pair, and at most 65536 bytes. | -Every variable is an absolute path to a readable regular file whose resolved target is strictly below one of the roots configured by `THT_WORKSPACE_SECRET_ROOTS`. Scalar S3 files are nonempty -UTF-8 tokens without whitespace or NUL. Public docs, exports, and rendered YAML never expose file contents. `changeme`, `replace-me`, `YOUR_SECRET`, ``, access-key-looking strings, and any +At the connector boundary every variable is an absolute path to a readable regular file whose +resolved target is strictly below one of the roots configured by `THT_WORKSPACE_SECRET_ROOTS`. +Users enter the corresponding values through Workspace management; the backend stores them as +authenticated ciphertext and materializes these files only for a runtime lease. Scalar S3 files +are nonempty UTF-8 tokens without whitespace or NUL. Public docs, APIs, and rendered YAML never +expose file contents. `changeme`, `replace-me`, `YOUR_SECRET`, ``, access-key-looking strings, and any credential-bearing or query-bearing URI are forbidden as public placeholder values. -## One shared registry repository +## One shared workspace repository All workspace namespaces live in one Git repository: ```text -registry.git/ +workspace-repository.git/ ├── thoth-workspaces.yaml ├── example/ │ ├── workspace.yaml │ └── evidence/... -├── another/ -│ └── workspace.yaml -└── workspace-docs/ - ├── example/{contract.env.example,README.md} - └── another/{contract.env.example,README.md} +└── another/ + └── workspace.yaml ``` The curator-owned root catalog `thoth-workspaces.yaml` uses the `schema_version` value `1` and the ordered `workspaces` list of `{id, name, description?}` entries. It is authoritative for workspace ID, name, description, and display order. The descriptor at `/workspace.yaml` must match the -catalog metadata exactly. `workspace-docs` is the reserved top-level API directory and cannot be a -workspace ID. +catalog metadata exactly. Every catalog entry must have its descriptor at that same commit; +catalog-only entries are invalid and reject the complete candidate revision. -Catalog-only entries without `/workspace.yaml` are valid bootstrap slots and surface as -`configuration_required`. The API may create `/workspace.yaml` only when the catalog slot -already exists and no Git object exists at that path in the exact pulled base commit. After -bootstrap, existing descriptors change only through curator Git commit/push and installation pull. -The API never writes `thoth-workspaces.yaml` or `/evidence/**`. Generated docs stay outside the -workspace namespace at `workspace-docs//{contract.env.example,README.md}`. An explicit docs -synchronization may create a docs-only commit that changes only `workspace-docs/**` and preserves -the catalog, descriptor, and Evidence object IDs. +Workspace source changes only through curator Git commit/push in a separate authoring clone, +followed by an installation pull. The API never writes `thoth-workspaces.yaml`, +`/workspace.yaml`, `/schema/**`, or `/evidence/**`. ## Registry revision and phase ownership @@ -165,14 +161,13 @@ the catalog, descriptor, and Evidence object IDs. | --- | --- | | Revision identity | The catalog blob, descriptor blob, and filesystem Evidence root tree are checked at the same 40-hex Git commit. | | Content-only revision | An Evidence-only commit changes authoritative `revision.commit` even when the catalog and descriptor blobs are unchanged. | -| Docs-only sync commit | A docs-only synchronization may advance `revision.commit`, change only `workspace-docs/**`, and preserve the catalog, descriptor, and Evidence object IDs. | -| Browser | Read-only curated workspaces preserve and show a read-only Evidence summary; only a catalog-only bootstrap slot may draft the first descriptor. | -| Export | Export remains exactly manifest, descriptor, contract, and README; it excludes Evidence bytes. | +| Repository consumer | ThothII fetches and validates a complete candidate, atomically activates it only on success, and never edits, commits, or pushes repository content. | +| Runtime secrets | Workspace management returns configured/missing status only; decrypted values exist only for the lifetime of a diagnostic or runtime lease. | | P1.1 | Validates the lexical URI `/evidence` and proves the declared filesystem root object is a Git tree at that same commit; it does not recursively inspect nested symlinks. Evidence materialization stays out of scope for P1.1. | | P6 | Owns commit-addressed materialization, realpath and recursive containment, nested-symlink checks, and race checks. | P1.1 performs no acquisition, extraction, preprocessing/indexing, embeddings, Qdrant writes, -`ACTIVE` publication, retention, or GC. +active-snapshot retention, or GC. ## Operator validation diff --git a/docs/guida-utente.md b/docs/guida-utente.md index 4950ab06..db9c6f5e 100644 --- a/docs/guida-utente.md +++ b/docs/guida-utente.md @@ -26,7 +26,6 @@ thoth-workspaces.yaml ← catalogo: elenco dei workspace /workspace.yaml ← descrittore del workspace (schema v3) /evidence/ ← (facoltativo) documenti di contesto, es. *.md /schema/annotations.yaml ← (facoltativo) join logici curati a mano (P5) -workspace-docs// ← generato dall'applicazione, non va editato ``` - Il **catalogo** `thoth-workspaces.yaml` è un semplice elenco: @@ -100,11 +99,10 @@ Cosa cambia rispetto ai vecchi workspace (se ne avevi uno): 1. **Git è la fonte di verità.** Descriptor, catalogo ed Evidence si modificano solo con un *commit* + *push* e poi un *pull* dell'installazione. -2. **Niente segreti nel repository.** Endpoint, token, certificati e password vivono solo nei file - di installazione protetti (fuori da Git). -3. **I file `workspace-docs/` sono generati** dall'applicazione: non modificarli a mano. -4. **Solo schema v3.** I descrittori v1/v2 vengono rifiutati prima dell'attivazione. -5. **L'applicazione non fa push di contenuti curati.** L'operatore che cura il repository lavora in +2. **Niente segreti nel repository.** Password, token, chiavi private e URL firmati vengono inseriti + a runtime nella gestione Workspace e conservati cifrati dal backend. +3. **Solo schema v3.** I descrittori v1/v2 vengono rifiutati prima dell'attivazione. +4. **L'applicazione non fa push di contenuti curati.** L'operatore che cura il repository lavora in un clone autore separato. --- @@ -160,10 +158,39 @@ Note importanti: ### 2.2 Applicazione web — gestione workspace -Per i workspace **già pronti** (`ready`) la pagina workspace è **in sola lettura**: -*Pull/Sync*, *Validate*, *Test* dell'installazione, *Export*, riepilogo Evidence e la guida Git per -il curatore. Il modulo di bootstrap modificabile compare solo per gli slot del catalogo in stato -`configuration_required`. +La gestione Workspace ha due livelli distinti. + +**Livello 1 — repository.** La parte iniziale spiega che il sorgente del workspace vive in una +directory separata, viene pubblicato dal curatore su un repository ospitato da un server Git come +GitHub, GitLab o Gitea, e viene letto da ThothII in sola lettura. Mostra host, repository, branch, +revisione attiva e stato dell'ultimo aggiornamento. + +- **Update workspace repository** non richiede la selezione di un workspace. Il backend esegue il + fetch/pull del branch configurato direttamente nel checkout gestito da ThothII, valida l'intera + revisione candidata e la attiva in modo atomico. Se la validazione fallisce, conserva la + revisione precedente. Non modifica il sorgente remoto e non salva contenuti nella GUI. +- Per creare un workspace locale, prepara una directory sorgente con catalogo, `workspace.yaml` e + le sottodirectory previste; quindi validala, esegui commit e push dal clone autore. ThothII non + offre comandi di creazione, modifica o pubblicazione del sorgente. + +**Livello 2 — workspace selezionato.** Questi comandi sono isolati perché richiedono prima la +selezione del workspace. + +- **Validate workspace** verifica nuovamente catalogo, descrittore, Evidence e invarianti della + revisione attiva selezionata. Non contatta il DWH e non modifica file. +- **Save runtime secrets** sostituisce alla cieca i valori compilati. I campi dipendono dal + trasporto DWH e dall'autenticazione Evidence dichiarati; il backend restituisce solo lo stato + configurato/mancante. +- **Forget** elimina dal vault cifrato il singolo secret indicato. Le sessioni o operazioni future + che lo richiedono restano bloccate finché non viene inserito di nuovo. +- **Test connections** materializza temporaneamente i secret necessari, contatta i servizi dati + configurati per quel workspace e rimuove i file temporanei alla fine. Non esporta né pubblica + nulla. + +Il repository Git remoto e le relative credenziali sono impostazioni di installazione. I secret +runtime DWH/Evidence sono invece persistenti nel vault cifrato del backend e non nel local storage +della GUI. La GUI è soltanto l'interfaccia: dopo l'invio cancella i valori dai campi e non può +rileggerli. --- diff --git a/docs/install/examples/thothii-installation.local.yaml b/docs/install/examples/thothii-installation.local.yaml index 39ca15f0..ef4a9e5a 100644 --- a/docs/install/examples/thothii-installation.local.yaml +++ b/docs/install/examples/thothii-installation.local.yaml @@ -9,4 +9,3 @@ workspaceRepository: access: ssh overrides: - "/absolute/path/to/ThothII/deploy/compose.git-ssh.yaml" - - "/absolute/path/to/thothii-operator/connector-secrets.local.yaml" diff --git a/docs/install/examples/thothii-installation.server.yaml b/docs/install/examples/thothii-installation.server.yaml index 5992cdaa..5f6e1dff 100644 --- a/docs/install/examples/thothii-installation.server.yaml +++ b/docs/install/examples/thothii-installation.server.yaml @@ -10,4 +10,3 @@ workspaceRepository: overrides: - "/absolute/path/to/ThothII/deploy/compose.session-server.yaml.example" - "/absolute/path/to/ThothII/deploy/compose.git-ssh.yaml" - - "/absolute/path/to/thothii-server-operator/connector-secrets.server.yaml" diff --git a/docs/install/local-workspace-registry.md b/docs/install/local-workspace-registry.md index 2008bad9..96a8fd7f 100644 --- a/docs/install/local-workspace-registry.md +++ b/docs/install/local-workspace-registry.md @@ -1,312 +1,158 @@ -# Local workspace-registry installation (Mac and PC) +# Local workspace repository installation (macOS, Windows, and Linux) -Complete the [local PC/Mac/Linux installation](local.md) first. This guide continues with the -Git-backed workspace source of truth, installation-local connector bindings, and diagnostics. Use -the [Pi management manual](pi-management.md) for provider configuration and image recovery. - -This guide runs a single-user ThothII registry on Docker Desktop (macOS or Windows) or a local -Linux Docker Engine. It is intentionally loopback-only. Git is shared; the checkout, DWH -bindings, credentials, and session data are local, while internal Qdrant/Ollama ship in the -Compose stack. Never put credentials in workspace YAML, Git, browser drafts, diagnostics, or -`.env.example`. +This manual connects a local ThothII installation to one remote Git repository hosted by a Git +server such as GitHub, GitLab, or Gitea. ThothII is a read-only consumer: it fetches, +validates, and activates workspace revisions, but never edits, commits, pushes, or publishes them. ## Architecture ownership contract | Component | Ownership | Operator contract | | --- | --- | --- | -| DWH | External | Installation-local endpoint/binding; never bundled into the Compose semantic stack. | -| LLM | External | Installation-local endpoint/policy choice outside the internal semantic services. | -| Qdrant | Internal | Mandatory private Compose semantic service; persistent `qdrant-data` volume. | -| Ollama embedding | Internal | Mandatory private Compose semantic service for `qwen3-embedding:0.6b`. | - - -## Host preprocessing (P2) - -The installed native `thothctl` is the only host entrypoint for workspace preprocessing -(introspection+LSH, FK review, schema indexing, HTTP Evidence). Use -`thothctl --installation workspace --workspace [--json]` -per `docs/contracts/workspace-preprocessing-cli.md` and the P2 walkthrough in -`docs/testing/p2-p6-manual-verification.md`. Preprocessing runs through the profile-gated -`workspace-maintenance` Compose service; it never starts a backend/Pi/frontend listener and never -attaches Git credentials. - - -## Effective configuration and `.tht-dwh` (P3) - -Prepared DWH generations are reusable and safe: `thothctl` and the application derive the same -canonical effective configuration and logical identity, so prepared work is reused when nothing -relevant changed and refused when the database/endpoint/identity changed. See -`docs/contracts/tht-dwh.md` for generations, `OWNER.json`, `ACTIVE`, fingerprints, migration and -recovery. A content-only or Evidence-only change never forces a full re-introspection. - -## Prerequisites - -- macOS: Docker Desktop, Git, and sufficient volume disk space. Git Credential Manager is useful - for HTTPS sign-in. -- Windows: Docker Desktop with WSL2, Git for Windows, and the clone enabled in Docker file sharing. - Use absolute paths/WSL paths; PowerShell uses `;` rather than `:` in `COMPOSE_FILE`. -- Linux PC: Docker Engine, Compose plugin, Git, and a user permitted to run Docker. -- Outbound access to the Git remote. A local installation needs no inbound firewall rule. - -Keep the operator `.env` and `installation-secrets/` outside the workspace-registry Git checkout. -On macOS/Linux use mode `0600` for individual secret files. On Windows apply an ACL that grants -read access only to the Docker Desktop user. Do not use an empty file to silently bypass a selected -authentication method. - -## Git remote: SSH and HTTPS - -Create one private repository such as `thoth-workspaces.git`. It contains the curator-owned root -catalog, one directory per workspace, optional embedded Evidence trees, and generated public docs: - -```text -registry.git/ -├── thoth-workspaces.yaml -├── / -│ ├── workspace.yaml -│ └── evidence/... -└── workspace-docs/ - └── /{contract.env.example,README.md} -``` - -`thoth-workspaces.yaml` uses the `schema_version` value `1` and the ordered `workspaces` list of -`{id, name, description?}` entries. It is authoritative for workspace ID, name, description, and -display order; `/workspace.yaml` must match that metadata exactly, while -`workspace-docs` remains the reserved top-level generated-docs directory. - -For SSH, use a scoped deploy key, a verified `known_hosts` file, and strict host-key checking. For -HTTPS, use Git Credential Manager or a secret-manager-created credentials file. Mount a private -HTTPS CA as its own file. Do not disable host or certificate verification. The base Compose file -does not mount a Git credential: add exactly one optional `deploy/compose.git-ssh.yaml` or -`deploy/compose.git-https.yaml` override, so unused credential paths are never bind-mounted. - -```dotenv -THT_WORKSPACE_GIT_REMOTE=ssh://git@git.example.invalid/platform/thoth-workspaces.git -THT_WORKSPACE_GIT_BRANCH=main -THT_WORKSPACE_INSTALLATION_ID=local-laptop -THT_SOURCE_ROOT=/absolute/path/to/ThothII -PI_AUTH_FILE=/absolute/path/installation-secrets/pi-auth.json -THT_SECRETS_FILE=/absolute/path/installation-secrets/thothii.secrets -THT_WORKSPACE_GIT_SSH_KEY_FILE=/absolute/path/installation-secrets/git-ssh-key -THT_WORKSPACE_GIT_KNOWN_HOSTS_FILE=/absolute/path/installation-secrets/git-known-hosts -THT_WORKSPACE_GIT_CA_FILE=/absolute/path/installation-secrets/git-ca.pem -``` - -For HTTPS set `THT_WORKSPACE_GIT_CREDENTIALS_FILE` instead of the SSH key/known-hosts pair. Remote -and branch are non-secret; every `*_FILE` is a local path whose content never enters Git or logs. - -## Curator flow for shared-registry Evidence - -Follow this order; the [canonical Evidence contract](../contracts/workspace-evidence-v3.md) defines -the source shapes and safety boundary. - -1. Clone the one shared registry, or update the review clone with `git pull --ff-only`. -2. Keep `thoth-workspaces.yaml` curator-owned. It uses the `schema_version` value `1` and the ordered - `workspaces` list of `{id, name, description?}` entries; it is authoritative for workspace ID, - name, description, and display order. -3. For an existing workspace, edit `/workspace.yaml` and any embedded `/evidence/**`, then - commit and push. -4. For a new workspace, add the catalog slot first. If you want ThothII to bootstrap the - descriptor, leave `/workspace.yaml` absent, commit and push, then pull that commit into the - installation; the slot appears as `configuration_required`. -5. The API may create `/workspace.yaml` only when the catalog slot already exists and no Git - object exists at that path in the exact pulled base commit. -6. After bootstrap, existing descriptors change only through curator Git commit/push and - installation pull. The API never writes `thoth-workspaces.yaml` or `/evidence/**`. -7. Inspect `workspace-docs//contract.env.example` and `workspace-docs//README.md`. -8. Provision only the selected Evidence `*_FILE` files outside Git and strictly below a root in - `THT_WORKSPACE_SECRET_ROOTS`; add matching host-only `*_SOURCE` paths for the generated - connector override. -9. Render or acquire the runtime config, then run `tht config check -c `. -10. Stop: P2/P6 later performs preprocessing and materialization. - -For example, a signed-HTTP workspace and a different static-S3 workspace can use these host-only -connector sources; the values are paths, not file contents: - -```dotenv -THT_WS_SIGNED_HTTP_EVIDENCE_SIGNED_URLS_SOURCE=/absolute/path/installation-secrets/signed-http-evidence-urls.json -THT_WS_STATIC_S3_EVIDENCE_ACCESS_KEY_SOURCE=/absolute/path/installation-secrets/static-s3-evidence-access-key -THT_WS_STATIC_S3_EVIDENCE_SECRET_KEY_SOURCE=/absolute/path/installation-secrets/static-s3-evidence-secret-key -THT_WS_STATIC_S3_EVIDENCE_SESSION_TOKEN_SOURCE=/absolute/path/installation-secrets/static-s3-evidence-session-token -``` - -The descriptor and declared filesystem root are validated at the same registry commit. The -browser shows a read-only Evidence summary, while exports omit Evidence bytes. - -## Shared Git values, local bindings, and secret files - -| Location | Contains | Never contains | -| --- | --- | --- | -| Git workspace repository | schema v3 YAML, generated binding names, LLM policy, and semantic-index identity | installation hostnames, keys, passwords, certificates, SSH keys | -| local `.env` | remote, branch, installation ID, `THT_WORKSPACE_BINDINGS_ENV_FILE`, and secret source paths | secret contents or `THT_WS_*` values | -| workspace bindings env file | only `THT_WS_*` transport, endpoint, user, and `/run/secrets/...` path bindings | secret contents or unrelated application settings | -| local secret directory | Git credentials/key, known hosts, CA, connector secret files | a copied registry checkout | -| Docker volumes | registry checkout/snapshots/state/locks and local data | host-only secret source files | - -The persistent volume is `/data/workspace-registry`: - -```text -repo/ # persistent Git checkout -snapshots/ # immutable validated revisions used by sessions -state/ # active revision and registry state -locks/ # short-lived registry synchronization locks -``` - -Installation variables are deterministic: `north-star-research` becomes `NORTH_STAR_RESEARCH`, and every name -is `THT_WS___`. Copy -[the bindings env example](examples/workspace-bindings.env.example) to an untracked operator file -and set its absolute path as `THT_WORKSPACE_BINDINGS_ENV_FILE`. It is loaded only into `core`. -Credentials and certificates use `*_FILE` path variables that must point inside `/run/secrets`. - -## Direct PostgreSQL, REST, and SSH tunnel bindings - -Set only fields for the selected transport in the dedicated bindings env file. Canonical YAML keeps -database/schema shared in Git. Every `*_FILE=/run/secrets/` binding needs one matching -host-only `*_SOURCE` path in operator `.env`. Generate the untracked connector override from those -two files during bootstrap; do not copy or maintain a workspace-specific Compose override. - -```dotenv -# Direct PostgreSQL -THT_WS_NORTH_STAR_RESEARCH_DWH_TRANSPORT=postgres_direct -THT_WS_NORTH_STAR_RESEARCH_DWH_HOST=dwh.example.invalid -THT_WS_NORTH_STAR_RESEARCH_DWH_PORT=5432 -THT_WS_NORTH_STAR_RESEARCH_DWH_USER=thoth_reader -THT_WS_NORTH_STAR_RESEARCH_DWH_PASSWORD_FILE=/run/secrets/north-star-research-dwh-password -``` - -```dotenv -# REST; an API-key file is needed only for a declared bearer/x-api-key diagnostic. -THT_WS_NORTH_STAR_RESEARCH_DWH_TRANSPORT=rest_api -THT_WS_NORTH_STAR_RESEARCH_DWH_BASE_URL=https://dwh.example.invalid -THT_WS_NORTH_STAR_RESEARCH_DWH_API_KEY_FILE=/run/secrets/north-star-research-dwh-api-key -``` - -```dotenv -# SSH tunnel diagnostic only; runtime sessions are fail-closed in this release. -THT_WS_NORTH_STAR_RESEARCH_DWH_TRANSPORT=ssh_tunnel -THT_WS_NORTH_STAR_RESEARCH_DWH_USER=thoth_reader -THT_WS_NORTH_STAR_RESEARCH_DWH_PASSWORD_FILE=/run/secrets/north-star-research-dwh-password -THT_WS_NORTH_STAR_RESEARCH_DWH_SSH_HOST=bastion.example.invalid -THT_WS_NORTH_STAR_RESEARCH_DWH_SSH_PORT=22 -THT_WS_NORTH_STAR_RESEARCH_DWH_SSH_USER=thoth_tunnel -THT_WS_NORTH_STAR_RESEARCH_DWH_SSH_PRIVATE_KEY_FILE=/run/secrets/north-star-research-dwh-tunnel-key -THT_WS_NORTH_STAR_RESEARCH_DWH_SSH_KNOWN_HOSTS_FILE=/run/secrets/north-star-research-dwh-known-hosts -THT_WS_NORTH_STAR_RESEARCH_DWH_SSH_TARGET_HOST=dwh.internal.example -THT_WS_NORTH_STAR_RESEARCH_DWH_SSH_TARGET_PORT=5432 -``` - -REST diagnostics reject a private per-request CA rather than weakening TLS; use runtime-trusted -HTTPS or verified direct/SSH native TLS. See the -[diagnostic protocol](../workspace-diagnostic-protocol.md). - -An SSH connector can prove installation reachability, host-key verification, authentication, and -target identity, but it intentionally returns `workspace_not_activatable`; select direct or REST -before creating sessions. Git pull/push over SSH remains fully supported and is independent. - -## Bootstrap, first pull, and diagnostics - -Use the repository's canonical `compose.yaml` plus `deploy/compose.local.yaml`; they always start -`frontend`, `core`, `qdrant`, `embedding`, and the one-shot `embedding-model-init`. This profile -is CPU-first. Add `THOTH_ENABLE_EMBEDDING_GPU=1` only on a Linux host that intentionally exposes a -supported GPU device to Docker. Qdrant is a derived but persistent index, while Ollama keeps a -local model cache for `qwen3-embedding:0.6b` (`1024` dimensions, cosine distance). Do not copy or -maintain a standalone application Compose file. Copy [the bindings env example](examples/workspace-bindings.env.example) into an -untracked operator directory and create a protected operator env file from -`deploy/env/local.env.example`. It must contain absolute `PI_AUTH_FILE`, -`THT_SECRETS_FILE`, `THT_WORKSPACE_BINDINGS_ENV_FILE`, and connector `*_SOURCE` paths. -The Pi auth JSON, runtime secret bundle, and each connector credential remain separate protected -host files and are mounted read-only; their contents never enter the operator env or rendered -Compose. - -Select exactly one repository Git transport override, `deploy/compose.git-ssh.yaml` or -`deploy/compose.git-https.yaml`. A Compose env file is not a shell environment, so export only the -non-secret paths required by the maintenance commands. Generate the connector override and render -through the preflight wrapper, which rejects unsafe paths and combined SSH+HTTPS selection. -Record the selected Git and generated connector overrides in the operator -[`thothii-installation.yaml` example](examples/thothii-installation.local.yaml), using absolute -paths, so `thothctl` remains the ordinary lifecycle interface. - -```sh -export THT_SOURCE_ROOT=/absolute/path/to/ThothII -export THT_OPERATOR_ENV=/absolute/path/to/operator/local.env -export THT_WORKSPACE_BINDINGS_ENV_FILE=/absolute/path/to/operator/workspace-bindings.env -export THT_CONNECTOR_OVERRIDE=/absolute/path/to/operator/connector-secrets.local.yaml -"$THT_SOURCE_ROOT/scripts/generate-connector-secrets-override.sh" --bindings-env "$THT_WORKSPACE_BINDINGS_ENV_FILE" --operator-env "$THT_OPERATOR_ENV" --output "$THT_CONNECTOR_OVERRIDE" -"$THT_SOURCE_ROOT/scripts/compose-with-preflight.sh" --env-file "$THT_OPERATOR_ENV" \ - -f "$THT_SOURCE_ROOT/compose.yaml" -f "$THT_SOURCE_ROOT/deploy/compose.local.yaml" \ - -f "$THT_SOURCE_ROOT/deploy/compose.git-ssh.yaml" -f "$THT_CONNECTOR_OVERRIDE" config --quiet -``` - -```sh -"$THT_SOURCE_ROOT/scripts/compose-with-preflight.sh" --env-file "$THT_OPERATOR_ENV" \ - -f "$THT_SOURCE_ROOT/compose.yaml" -f "$THT_SOURCE_ROOT/deploy/compose.local.yaml" \ - -f "$THT_SOURCE_ROOT/deploy/compose.git-ssh.yaml" -f "$THT_CONNECTOR_OVERRIDE" up --build -d -curl --fail --silent http://127.0.0.1:8787/health -curl --fail --silent http://127.0.0.1:8787/workspace-registry/status -curl --fail --silent http://127.0.0.1:8787/workspaces -``` - -The first status request clones the registry, validates `thoth-workspaces.yaml`, every -catalog-listed descriptor, and any declared `/evidence` tree, then atomically activates a -snapshot. A catalog-only slot with no descriptor reports `configuration_required` and is not -activatable. Use `POST /workspace-registry/pull` to fetch later curator revisions. Existing -curated descriptors stay read-only in the UI; only a missing descriptor may use the one-time -bootstrap create flow. Run workspace diagnostics only after required DWH bindings are mounted. -Schema-v3 diagnostics probe the internal Qdrant/Ollama services through backend config; ordinary -diagnostics are read-only. - - -Schema v3 is the only accepted workspace descriptor. Schema v1 and v2 workspace descriptors are -rejected before activation. Candidate snapshot validation makes bootstrap activation or a pull fail -atomically and leaves the prior active snapshot unchanged. There is no in-product migrator or -automatic conversion. The repository must already contain reviewed v3 descriptors. One workspace -owns one Qdrant collection; schema, Evidence, and Memory records share that collection and remain -isolated by payload `kind`. - +| DWH | External | Configure the external endpoint and complete its runtime credentials in Workspace management. | +| LLM | External | Configure the external endpoint and model policy during installation. | +| Qdrant | Internal | Compose runs the internal service and persists `qdrant-data`. | +| Ollama embedding | Internal | Compose runs the internal `qwen3-embedding:0.6b` service and model-init job. | ## Semantic index ownership contract | Scope | Ownership rule | Isolation rule | | --- | --- | --- | -| Workspace semantic index | Each workspace reserves a single Qdrant collection. | Schema, Evidence, and Memory stay in that one collection and remain isolated by payload `kind`. | +| Workspace semantic index | Each workspace keeps exactly one Qdrant collection reserved for itself. | Schema, Evidence, and memory records share that one collection and are separated by the `kind` payload. | -If source material needs conversion, perform it outside ThothII in a separate reviewed process. -Commit only the resulting reviewed v3 descriptors. That external process must not import `${ENV}` -values, secret values, certificates, keys, or secret files into the repository. +The mandatory semantic stack is CPU-first. Set `THOTH_ENABLE_EMBEDDING_GPU=1` only after the +documented GPU prerequisites are satisfied. The embedding contract is fixed at +`qwen3-embedding:0.6b`, 1024 dimensions, cosine distance. -## Publish, update, backup, outage recovery, and rollback +## Prerequisites -Existing curated workspaces are read-only in the browser. Use the Workspace -Management page to pull, inspect status, validate a workspace, test it on this installation, and -optionally create one bootstrap descriptor for a pulled `configuration_required` slot. After that -first descriptor exists, change it only through curator Git commit/push and installation pull; -never hand-edit the running `repo/` volume. Before upgrading, record registry status, stop Compose, -and take a timestamped ownership-preserving backup of both registry and local data volumes while -excluding `installation-secrets/`. Render Compose, rebuild, start, and check status before -resuming work. If you are upgrading an older P1 registry, apply the reviewed migration in -[`docs/migrations/p1-to-p1-1-registry-layout.md`](../migrations/p1-to-p1-1-registry-layout.md) -and upgrade ThothII only after that commit is pushed. +- A working local installation described by [local.md](local.md). +- A remote Git repository and a read-only deploy credential for this ThothII installation. +- A separate authoring clone in which a workspace curator can edit and publish source revisions. +- `thothctl` built with `bash scripts/build-thothctl.sh`. -After a valid bootstrap, remote outage retains the last valid snapshot and reports `degraded: true`. -Pinned sessions continue. Repair network/authentication, pull, and confirm non-degraded status. To -undo a bad remote change, create a reviewed Git revert/release branch, advance the remote through -normal policy, pull, and confirm its new snapshot. Do not delete `snapshots/` as rollback. +## Prepare and publish a workspace source + +Create a local workspace in an ordinary source directory outside ThothII's data directories. The +canonical repository layout is: + +```text +thoth-workspaces.yaml +/workspace.yaml +/evidence/ # optional, repository-owned Evidence +/schema/annotations.yaml # optional curated annotations +``` + +The catalog lists `{id, name, description?}` and the descriptor at +`/workspace.yaml` must match that metadata. Use the examples in +`deploy/workspaces/` as authoring references. Do not store passwords, tokens, private keys, or +signed URLs in Git. + +Publishing is an author-side Git operation: validate the source, commit it, and push it from the +separate authoring clone to the configured branch. This is the only meaning of “publish” in the +workspace lifecycle. ThothII has no author identity and no Git write credential. + + +Schema v3 is the only accepted workspace descriptor. +Schema v1 and v2 workspace descriptors are rejected before activation. + + +## Configure the remote Git repository + +Copy `docs/install/examples/thothii-installation.local.yaml` to an operator-controlled absolute +path. Its `workspaceRepository` block records the remote, branch, and read-only access method. +Choose exactly one transport override: + +- SSH: `deploy/compose.git-ssh.yaml`, with a read-only deploy key and pinned `known_hosts` file. +- HTTPS: `deploy/compose.git-https.yaml`, with a read-only token in a Git credentials file and an + optional private CA file. + +The remote and branch are installation configuration. Git credentials remain protected +installation files and are never accepted by Workspace management or returned by its API. + +Example non-secret/operator paths: + +```dotenv +THT_WORKSPACE_GIT_REMOTE=git@git.example.com:organization/workspaces.git +THT_WORKSPACE_GIT_BRANCH=main +THT_WORKSPACE_INSTALLATION_ID=local +PI_AUTH_FILE=/absolute/path/to/operator/pi-auth.json +THT_SECRETS_FILE=/absolute/path/to/operator/thothii.secrets +THT_WORKSPACE_GIT_SSH_KEY_FILE=/absolute/path/to/operator/git-ssh-key +THT_WORKSPACE_GIT_KNOWN_HOSTS_FILE=/absolute/path/to/operator/git-known-hosts +``` + +Keep these files outside both the ThothII checkout and the workspace source repository. Protect +them with mode `0600` on macOS/Linux or an equivalent single-user ACL on Windows. + +## Start and update the installation + +Use only the installation-aware lifecycle: + +```bash +export THT_SOURCE_ROOT=/absolute/path/to/ThothII +THTCTL="$THT_SOURCE_ROOT/tools/thothctl/thothctl" +INSTALLATION=/absolute/path/to/operator/thothii-installation.yaml +"$THTCTL" --installation "$INSTALLATION" start +"$THTCTL" --installation "$INSTALLATION" doctor +``` + +At startup ThothII clones or fetches the configured repository into its application-managed +`workspace-registry` volume. Later, **Update workspace repository** performs a server-side fetch +and fast-forward candidate checkout. It does not copy anything to the user's computer. + +## Complete runtime secrets in Workspace management + +Open Workspace management after the first successful repository update. + +1. At the repository level, review the configured host, repository, branch, and current revision. +2. Select a workspace. Repository update does not require a selection; validation and connection + tests do. +3. Review the runtime fields derived from the selected DWH transport and Evidence authentication + mechanism. +4. Enter or rotate the required values and choose **Save runtime secrets**. +5. Run **Validate workspace** and then **Test connections**. + +Secret fields are write-only. The GUI receives only configured/missing status. Values are +encrypted by the backend in the platform-neutral `workspace-secrets` volume. ThothII temporarily +materializes a restrictive file only while an existing file-oriented connector needs it, then +removes that file when the runtime lease ends. **Forget** deletes the selected encrypted value. + +The workspace YAML stays environment-independent: it declares connector mechanisms, not host +paths or credentials. Installation trust material such as a Git CA or `known_hosts` remains an +operator concern; DWH and Evidence credentials are completed in the GUI. + +## Validation and activation behavior + +An update follows this sequence: + +1. Fetch the configured branch into a candidate checkout managed by ThothII. +2. Validate the catalog, every descriptor, repository-relative Evidence, and cross-workspace + invariants at the same Git commit. +3. If every workspace is valid, atomically mark that complete commit as active. +4. If any validation fails, report sanitized diagnostics and keep the previous active revision. + +The active checkout is read-only application state. Never edit files under +`/data/workspace-registry`. A source correction must be committed and pushed from the authoring +clone, then fetched again with **Update workspace repository**. + +## Backup, rotation, and recovery + +Back up the `workspace-registry`, `workspace-secrets`, `sessions`, `qdrant-data`, +`embedding-models`, `settings`, and `pi-state` volumes together. The encrypted vault is useless +without its generated master key, so preserve the entire `workspace-secrets` volume and protect +the backup as secret material. + +Rotate a runtime credential by saving its replacement in Workspace management and rerunning its +connection test. Rotate Git credentials in the installation files and restart `core`. To recover +from a bad remote revision, correct or revert it in the authoring repository and run the update; +until validation succeeds, the previous active snapshot remains available. ## Troubleshooting -| Stable code | Meaning and safe action | +| Symptom | Meaning and action | | --- | --- | -| `workspace_invalid` | Invalid descriptor/path/snapshot; restore a reviewed canonical revision. | -| `binding_missing` | A selected value or readable `*_FILE` is absent; fix the local binding/mount. | -| `workspace_not_activatable` | Diagnostics cannot activate the workspace; correct the selected transport. | -| `workspace_stale` | Checkout changed or is busy; stop competing pull or sync work. | -| `workspace_conflict` | Draft base differs from Git; pull, resolve the diff, validate, and retry after the curator pull/bootstrap flow. | -| `git_unavailable` | Remote, path, network, or lock unavailable; preserve the degraded valid snapshot. | -| `git_auth_failed` | Mounted SSH/HTTPS material rejected/unreadable; rotate or fix permissions without logging it. | -| `git_non_fast_forward` | Checkout diverged; reconcile through the registry workflow. | -| `git_push_rejected` | Remote policy rejected the change; review branch protection/hooks. | -| `connector_unavailable` | DNS/TLS/auth/resource identity diagnostic failed; inspect local bindings and egress. | -| `semantic_index_incompatible` | Collection/model/dimensions/distance differs from Git; perform an explicit index migration. | - -On macOS, restart Docker Desktop if a named volume disappears. On Windows, check WSL2 and Docker -file sharing. A failed first bootstrap has no snapshot fallback: repair remote trust and retry; -never create an unreviewed local registry repository. +| Repository unavailable | Check remote host, branch, read-only deploy credential, CA, and `known_hosts`. | +| Candidate rejected | Fix the reported source error in the authoring clone, commit, push, and update again. | +| Runtime configuration required | Select the workspace and complete each required secret field. | +| Connection test fails | Rotate the relevant secret or correct the non-secret endpoint in the source/installation as appropriate. | +| Active revision did not change | The candidate was invalid or was already active; inspect the repository status. | diff --git a/docs/install/local.md b/docs/install/local.md index 56562284..86fdaf77 100644 --- a/docs/install/local.md +++ b/docs/install/local.md @@ -109,19 +109,19 @@ Copy-Item docs/install/examples/thothii-installation.local.yaml ` ``` Edit `deploy/env/local.env`. At minimum set the workspace Git remote, `PI_AUTH_FILE`, -`THT_SECRETS_FILE`, external service endpoints, and the absolute -`THT_WORKSPACE_BINDINGS_ENV_FILE`. Create each secret as a separate regular file under the -protected operator directory and set mode `0600`. On Windows use a user-only ACL instead. +`THT_SECRETS_FILE`, and external service endpoints. Create the Pi/application and Git transport +files under the protected operator directory and set mode `0600`. On Windows use a user-only ACL +instead. DWH and Evidence credentials are entered later through Workspace management and stored +in the backend's encrypted `workspace-secrets` volume. Do not paste credentials into this guide's commands, `.env`, workspace YAML, Git, URLs, image build arguments, or the installation descriptor. Secret contents are mounted read-only under `/run/secrets` (Pi's auth store has its own protected read-only mount) and must never be committed, embedded, rendered, or logged. -Follow [the local workspace-registry guide](local-workspace-registry.md) to create the bindings -file, choose exactly one Git SSH/HTTPS override, and generate the connector-secret override. A -fresh install requires a valid private workspace repository; the Git-backed registry remains the -source of truth. +Follow [the local workspace repository guide](local-workspace-registry.md) to choose exactly one +read-only Git SSH/HTTPS override. A fresh install requires a valid private workspace repository; +the remote Git repository remains the source of truth. Copy the installation example to an operator-controlled file named exactly `thothii-installation.yaml`, then replace all placeholders with absolute paths: @@ -132,8 +132,7 @@ cp docs/install/examples/thothii-installation.local.yaml \ ``` For HTTPS, replace the SSH override in that file with `deploy/compose.git-https.yaml`. Add only -reviewed local overrides, including the generated connector-secret file. Paths may contain spaces -when correctly represented as YAML strings. +reviewed local overrides. Paths may contain spaces when correctly represented as YAML strings. Native Windows uses the same four fields. Use single-quoted absolute Windows paths so backslashes remain literal YAML characters: @@ -144,14 +143,13 @@ projectDirectory: 'C:\Users\operator\src\ThothII' envFile: 'C:\Users\operator\src\ThothII\deploy\env\local.env' overrides: - 'C:\Users\operator\src\ThothII\deploy\compose.git-ssh.yaml' - - 'C:\Users\operator\thothii-operator\connector-secrets.local.yaml' ``` ## Address external services An address is interpreted inside `core`. Therefore container 127.0.0.1 means the container itself, -not the Docker host. Keep every DWH, vector, embedding, and LLM address configurable in the local -environment/workspace bindings. +not the Docker host. Keep external DWH and LLM addresses configurable in the installation; Qdrant +and embedding are internal services in the standard stack. - **Docker Desktop (macOS and Windows):** use `host.docker.internal`, for example `http://host.docker.internal:11434`. diff --git a/docs/install/psd-workspace-setup.md b/docs/install/psd-workspace-setup.md index 4c36f348..a5084dfb 100644 --- a/docs/install/psd-workspace-setup.md +++ b/docs/install/psd-workspace-setup.md @@ -7,18 +7,18 @@ v3 + `thothctl`). - **Repository PSD pubblicato:** `https://github.com/mptyl/tht-workspace-psd` (privato), branch `main`, commit `d4f9185`. Layout P1.1 già migrato e validato. -- **Deploy key SSH** (read-write, senza passphrase) generata in +- **Deploy key SSH** (sola lettura, senza passphrase) in `deploy/psd/secrets/git-ssh-key` e registrata sul repo come deploy key `thothii-psd`; il remote Git usato dall'installazione è `git@github.com:mptyl/tht-workspace-psd.git`. - **Config operatore pronta** (file reali gitignored in `deploy/psd/`): `operator.env`, - `workspace-bindings.env`, `thothii-installation.yaml`, `connector-secrets.yaml` e i secret - `secrets/` (API key DWH riusata, pi-auth, secret bundle, chiave SSH, known_hosts). Nessuna CA: - il DWH REST usa HTTPS pubblico (`THT_SSL_CA` era vuoto). + `thothii-installation.yaml` e i secret d'installazione in `secrets/` (pi-auth, secret bundle, + chiave SSH, known_hosts). L'API key DWH va completata nella gestione Workspace ed è conservata + nel vault cifrato del backend. Nessuna CA: il DWH REST usa HTTPS pubblico. - **Stack avviato** (progetto `thothii-70417a3e30ea`, via `thothctl start`): `qdrant`, `embedding` (con `qwen3-embedding:0.6b`), `core`, `frontend` sani. Il registry ha **clonato e attivato** `psd-clinical` (stato `ready`). -- **`thothctl workspace inspect --workspace psd-clinical` = OK** (identità descrittore/catalogo/ - config risolte con i bindings DWH). +- **`thothctl workspace inspect --workspace psd-clinical` = OK** (identità descrittore/catalogo + risolte); la configurazione runtime va completata e testata dalla GUI. - **Bloccante residuo: VPN.** `supabase-aritmolab.policlinicosandonato.it` non risolve (`NXDOMAIN`) → il preprocessing DWH e le sessioni live non possono ancora partire. @@ -49,6 +49,6 @@ Il preprocessing è già completato. Resta solo: ## Cosa è già stato fatto - Ristrutturazione del repo PSD nel layout P1.1 + validazione locale. -- Pubblicazione GitHub + deploy key + config operatore completa (bindings/secret/override). +- Pubblicazione GitHub + deploy key read-only + configurazione Git d'installazione. - Avvio stack + attivazione registry + `thothctl inspect` verde. - **Preprocessing live completato** su PSD: DWH → FK → schema → Evidence, idempotente. diff --git a/docs/install/server-workspace-registry.md b/docs/install/server-workspace-registry.md index 01a9c040..b73d2a5d 100644 --- a/docs/install/server-workspace-registry.md +++ b/docs/install/server-workspace-registry.md @@ -1,371 +1,128 @@ -# Server workspace-registry installation +# Server workspace repository installation -This is the production workspace-registry companion to [the autonomous Linux server guide](server.md). -The application image is read-only, secrets are mounted read-only, and sessions use immutable -Git-validated snapshots. Expose the application only behind an authenticated same-origin reverse -proxy; never publish the core port directly. +This manual supplements [server.md](server.md). A server installation reads one remote Git +repository hosted by GitHub, GitLab, Gitea, Bitbucket, or another Git server. ThothII fetches and +validates complete revisions but never edits, commits, pushes, or publishes workspace source. ## Architecture ownership contract | Component | Ownership | Operator contract | | --- | --- | --- | -| DWH | External | Approved installation/server endpoint; not part of the private semantic Compose stack. | -| LLM | External | Approved installation/server endpoint or provider policy outside the semantic stack. | -| Qdrant | Internal | Mandatory private Compose semantic service; persistent `qdrant-data` volume. | -| Ollama embedding | Internal | Mandatory private Compose semantic service for `qwen3-embedding:0.6b`. | - - -## Host preprocessing (P2) - -The installed native `thothctl` is the only host entrypoint for workspace preprocessing -(introspection+LSH, FK review, schema indexing, HTTP Evidence). Use -`thothctl --installation workspace --workspace [--json]` -per `docs/contracts/workspace-preprocessing-cli.md` and the P2 walkthrough in -`docs/testing/p2-p6-manual-verification.md`. Preprocessing runs through the profile-gated -`workspace-maintenance` Compose service; it never starts a backend/Pi/frontend listener and never -attaches Git credentials. - - -## Effective configuration and `.tht-dwh` (P3) - -Prepared DWH generations are reusable and safe: `thothctl` and the application derive the same -canonical effective configuration and logical identity, so prepared work is reused when nothing -relevant changed and refused when the database/endpoint/identity changed. See -`docs/contracts/tht-dwh.md` for generations, `OWNER.json`, `ACTIVE`, fingerprints, migration and -recovery. A content-only or Evidence-only change never forces a full re-introspection. - -## Service account, storage, and firewall - -Create a dedicated host service account and an operator root such as `/srv/thothii`. The core -container is non-root UID/GID `10001` (`thoth`), so give that identity read/write ownership before -first startup. Keep storage separated: - -```text -/srv/thothii/data/ # settings, session data, Pi state as applicable -/srv/thothii/workspace-registry/ # repo/, snapshots/, state/, locks/ -/srv/thothii/secrets/ # Git and connector secret files, setgid mode 2750 -/srv/thothii/operator/ # untracked operator files, setgid mode 2770 -``` - -After cloning the source and before the first render/start, initialize the empty Pi-state root with -the repository setup command: - -```sh -sudo /srv/thothii/source/ThothII/scripts/prepare-server-pi-state.sh \ - /srv/thothii/pi-state 10001 10001 -``` - -The active server profile mounts that writable parent at `/home/thoth/.pi` and overlays three -read-only files beneath `agent/`. The setup command atomically creates the required hidden regular -targets with runtime ownership without copying secret or tracked file contents into writable -state. Rerun it after a restore and before Compose or `thothctl` startup; it is idempotent and does -not overwrite existing targets. - -Permit outbound TCP only to approved Git/Gitea, DWH, LLM, and optional bastion endpoints. -Qdrant and Ollama run inside the Compose stack. Allow inbound traffic only from the reverse -proxy/Docker network. Do not give the runtime service -account Gitea administration, database-superuser rights, or a shell in the Git host. - -## Gitea and remote Git setup - -Create a private Gitea (or compatible Git) repository such as `platform/thoth-workspaces`. Protect -`main` according to the release policy and grant the ThothII publisher only the intended repository -scope. Commit the curator-owned root catalog `thoth-workspaces.yaml`, workspace descriptors under -`/workspace.yaml`, curated embedded Evidence under `/evidence/**`, and generated public -artifacts only at `workspace-docs//README.md` and -`workspace-docs//contract.env.example`; do not commit installation bindings or secret material. - -`thoth-workspaces.yaml` uses the `schema_version` value `1` and the ordered `workspaces` list of -`{id, name, description?}` entries. It is authoritative for workspace ID, name, description, and -display order. `/workspace.yaml` must match that metadata exactly, and `workspace-docs` -remains the reserved top-level generated-docs directory. - -For SSH, create a least-privilege deploy key, record Gitea's host key in managed known-hosts, and -use `ssh://git@git.example.invalid/platform/thoth-workspaces.git`. For HTTPS, create a scoped -machine credential in the secret manager and mount the Gitea/private CA separately. Never use a -Gitea admin credential in the application. - -Bootstrap an empty remote from a temporary review clone only after the catalog, descriptors, and -generated public artifacts have been reviewed; commit and push `main`. The running server is not a -descriptor authoring or conversion environment. - -## Curator flow for shared-registry Evidence - -Follow this order; the [canonical Evidence contract](../contracts/workspace-evidence-v3.md) defines -the source shapes and safety boundary. - -1. Clone the one shared registry, or update the review clone with `git pull --ff-only`. -2. Keep `thoth-workspaces.yaml` curator-owned. It uses the `schema_version` value `1` and the ordered - `workspaces` list of `{id, name, description?}` entries; it is authoritative for workspace ID, - name, description, and display order. -3. For an existing workspace, edit `/workspace.yaml` and any embedded `/evidence/**`, then - commit and push. -4. For a new workspace, add the catalog slot first. If you want ThothII to bootstrap the - descriptor, leave `/workspace.yaml` absent, commit and push, then pull that commit into the - installation; the slot appears as `configuration_required`. -5. The API may create `/workspace.yaml` only when the catalog slot already exists and no Git - object exists at that path in the exact pulled base commit. -6. After bootstrap, existing descriptors change only through curator Git commit/push and - installation pull. The API never writes `thoth-workspaces.yaml` or `/evidence/**`. -7. Inspect `workspace-docs//contract.env.example` and `workspace-docs//README.md`. -8. Provision only the selected Evidence `*_FILE` files outside Git and strictly below a root in - `THT_WORKSPACE_SECRET_ROOTS`; add matching host-only `*_SOURCE` paths for the generated - connector override. -9. Render or acquire the runtime config, then run `tht config check -c `. -10. Stop: P2/P6 later performs preprocessing and materialization. - -For example, separate signed-HTTP and static-S3 workspaces can use these host-only connector source -paths: - -```dotenv -THT_WS_SIGNED_HTTP_EVIDENCE_SIGNED_URLS_SOURCE=/srv/thothii/secrets/signed-http-evidence-urls.json -THT_WS_STATIC_S3_EVIDENCE_ACCESS_KEY_SOURCE=/srv/thothii/secrets/static-s3-evidence-access-key -THT_WS_STATIC_S3_EVIDENCE_SECRET_KEY_SOURCE=/srv/thothii/secrets/static-s3-evidence-secret-key -THT_WS_STATIC_S3_EVIDENCE_SESSION_TOKEN_SOURCE=/srv/thothii/secrets/static-s3-evidence-session-token -``` - -The descriptor and declared filesystem root are validated at the same registry commit. The -browser shows a read-only Evidence summary, while exports omit Evidence bytes. - -## Git credentials, CA, SSH key, and known-hosts mounts - -Use the secret manager or a protected host-only procedure to create independent regular files under -`/srv/thothii/secrets`. As established in the server guide, use owner UID 10001, group -`thothii-ops`, file mode `0640`, and setgid directory mode `2750`. This lets the UID 10001 -container and the reviewed Docker operator running `thothctl` read the files without making them -public. These path-only variables are mounted read-only by Compose: - -```dotenv -THT_WORKSPACE_GIT_CREDENTIALS_FILE=/srv/thothii/secrets/git-credentials -THT_WORKSPACE_GIT_CA_FILE=/srv/thothii/secrets/git-ca.pem -THT_WORKSPACE_GIT_SSH_KEY_FILE=/srv/thothii/secrets/git-ssh-key -THT_WORKSPACE_GIT_KNOWN_HOSTS_FILE=/srv/thothii/secrets/git-known-hosts -PI_AUTH_FILE=/srv/thothii/secrets/pi-auth.json -THT_SECRETS_FILE=/srv/thothii/secrets/thothii.secrets -``` - -Use the credential file for HTTPS, or key and known-hosts for SSH. The base server Compose file -mounts neither transport; add exactly one `deploy/compose.git-https.yaml` -or `deploy/compose.git-ssh.yaml` override. Strict host-key checking stays enabled -and Git stderr is not exposed by the API. Rotate by atomically replacing the secret file, -restarting `core`, and performing pull/status; never put the material in an environment variable or -rendered Compose output. - -## Shared Git values, local bindings, and secret files - -Git describes workspace schema, immutable ID, DWH identity, semantic-index dimensions and -distance, internal embedding contract, and LLM policy. The installation supplies remote/branch/installation -ID and one absolute `THT_WORKSPACE_BINDINGS_ENV_FILE` containing only `THT_WS_*` transport, -endpoint, user, and `/run/secrets/...` path bindings. The base Compose loads that file only into -`core`. Secret contents are only in host files, never the values stored in Git or browser-local -drafts. - -The runtime registry layout is persistent and must be backed up together: - -```text -/data/workspace-registry/repo/ -/data/workspace-registry/snapshots/ -/data/workspace-registry/state/ -/data/workspace-registry/locks/ -``` - -Variable names derive from the immutable ID: `north-star-research` becomes `NORTH_STAR_RESEARCH`, producing -`THT_WS_NORTH_STAR_RESEARCH_DWH_PASSWORD_FILE`. Keep the bindings file limited to DWH transport, -endpoint, user, and secret-path values; internal semantic services are supplied by Compose and do -not require workspace-local vector or embedding bindings. -Copy [the bindings env example](examples/workspace-bindings.env.example) to the protected operator -directory. Every path-valued `*_FILE` entry needs an absolute host-only `*_SOURCE` path. Generate -the untracked connector override from those files during bootstrap; do not copy or maintain a -workspace-specific Compose override. Operator-managed path-only files use owner UID 10001, group -`thothii-ops`, and mode `0660`; secret files remain `0640` and non-group-writable. - -## Direct PostgreSQL, REST, and SSH tunnel bindings - -Select only a transport allowed by canonical YAML; preserve database/schema/collection, model, -dimensions, and distance as Git-shared identity. - -```dotenv -# Direct PostgreSQL with verified native TLS if a CA path is supplied. -THT_WS_NORTH_STAR_RESEARCH_DWH_TRANSPORT=postgres_direct -THT_WS_NORTH_STAR_RESEARCH_DWH_HOST=dwh.internal.example -THT_WS_NORTH_STAR_RESEARCH_DWH_PORT=5432 -THT_WS_NORTH_STAR_RESEARCH_DWH_USER=thoth_reader -THT_WS_NORTH_STAR_RESEARCH_DWH_PASSWORD_FILE=/run/secrets/north-star-research-dwh-password -``` - -```dotenv -# REST needs API-key file paths only when the descriptor declares authenticated diagnostics. -THT_WS_NORTH_STAR_RESEARCH_DWH_TRANSPORT=rest_api -THT_WS_NORTH_STAR_RESEARCH_DWH_BASE_URL=https://dwh.internal.example -THT_WS_NORTH_STAR_RESEARCH_DWH_API_KEY_FILE=/run/secrets/north-star-research-dwh-api-key -``` - -```dotenv -# SSH tunnel diagnostic only; runtime sessions are fail-closed in this release. -THT_WS_NORTH_STAR_RESEARCH_DWH_TRANSPORT=ssh_tunnel -THT_WS_NORTH_STAR_RESEARCH_DWH_USER=thoth_reader -THT_WS_NORTH_STAR_RESEARCH_DWH_PASSWORD_FILE=/run/secrets/north-star-research-dwh-password -THT_WS_NORTH_STAR_RESEARCH_DWH_SSH_HOST=bastion.internal.example -THT_WS_NORTH_STAR_RESEARCH_DWH_SSH_PORT=22 -THT_WS_NORTH_STAR_RESEARCH_DWH_SSH_USER=thoth_tunnel -THT_WS_NORTH_STAR_RESEARCH_DWH_SSH_PRIVATE_KEY_FILE=/run/secrets/north-star-research-dwh-tunnel-key -THT_WS_NORTH_STAR_RESEARCH_DWH_SSH_KNOWN_HOSTS_FILE=/run/secrets/north-star-research-dwh-known-hosts -THT_WS_NORTH_STAR_RESEARCH_DWH_SSH_TARGET_HOST=dwh.internal.example -THT_WS_NORTH_STAR_RESEARCH_DWH_SSH_TARGET_PORT=5432 -``` - -REST diagnostics refuse private per-request CAs -rather than disable verification; use runtime-trusted HTTPS or verified direct/SSH native TLS. See -the [diagnostic protocol](../workspace-diagnostic-protocol.md) for its read-only checks and optional -reversible writer probe. - -An SSH connector can be tested with strict host-key and target verification, but it intentionally -returns `workspace_not_activatable`; configure direct or REST transport before starting sessions. -The Git registry itself may still use SSH normally. - -## Same-origin reverse proxy, bootstrap, and health - -Use the repository's canonical `compose.yaml` plus `deploy/compose.server.yaml`; they always -start `frontend`, `core`, `qdrant`, `embedding`, and the one-shot `embedding-model-init`. -Startup is CPU-first; use `THOTH_ENABLE_EMBEDDING_GPU=1` only when the server intentionally -exposes a supported GPU device to Docker. Qdrant is a derived but persistent index, and the -Ollama model cache persists the exact `qwen3-embedding:0.6b` model (`1024` dimensions, cosine -distance) for offline reuse. Do not copy or maintain a standalone -application Compose file. Copy `docs/install/examples/thothii-installation.server.yaml` to the -protected operator directory and preserve its required session-server overlay, exactly one Git -transport override, and generated connector-secret override. - -Review `deploy/workspaces/server-sessions.yaml.example`, materialize it as a protected host file, -and set its absolute `THT_SERVER_WORKSPACE_CONFIG` path. Copy the bindings env example into the -operator directory, then set absolute `PI_AUTH_FILE`, `THT_SECRETS_FILE`, -`THT_WORKSPACE_BINDINGS_ENV_FILE`, and connector `*_SOURCE` paths. The same operator env must set -`THT_SESSION_DB_HOST`, `THT_SESSION_DB_NAME`, `THT_SESSION_RUNTIME_USER`, -`THT_SESSION_RUNTIME_PASSWORD_SOURCE`, and `THT_SESSION_CA_SOURCE`; -`deploy/compose.session-server.yaml.example` wires `postgres`, `verify-full`, and separate -runtime/CA secret targets under `/run/secrets`. This public server profile never falls back to -filesystem sessions. The path-only environment file is not shell code; do not source it. - -Generate the connector override, then use the installation-aware operator CLI. Building -`thothctl` requires only Docker and no Go knowledge. From a trusted maintenance shell: - -```sh -umask 0007 -THT_SOURCE_ROOT=/srv/thothii/source/ThothII -THT_OPERATOR_ENV=/srv/thothii/operator/server.env -THT_WORKSPACE_BINDINGS_ENV_FILE=/srv/thothii/operator/workspace-bindings.env -THT_CONNECTOR_OVERRIDE=/srv/thothii/operator/connector-secrets.server.yaml -THTCTL=/srv/thothii/operator/thothctl -INSTALLATION=/srv/thothii/operator/thothii-installation.yaml -sudo "$THT_SOURCE_ROOT/scripts/prepare-server-pi-state.sh" /srv/thothii/pi-state 10001 10001 -"$THT_SOURCE_ROOT/scripts/compose-with-preflight.sh" --env-file "$THT_OPERATOR_ENV" \ - -f "$THT_SOURCE_ROOT/compose.yaml" -f "$THT_SOURCE_ROOT/deploy/compose.server.yaml" \ - -f "$THT_SOURCE_ROOT/deploy/compose.session-server.yaml.example" config --quiet -"$THT_SOURCE_ROOT/scripts/generate-connector-secrets-override.sh" \ - --bindings-env "$THT_WORKSPACE_BINDINGS_ENV_FILE" \ - --operator-env "$THT_OPERATOR_ENV" --output "$THT_CONNECTOR_OVERRIDE" -"$THTCTL" --installation "$INSTALLATION" update --check-only -"$THTCTL" --installation "$INSTALLATION" start -"$THTCTL" --installation "$INSTALLATION" status -"$THTCTL" --installation "$INSTALLATION" doctor -"$THTCTL" --installation "$INSTALLATION" pi doctor -"$THTCTL" --installation "$INSTALLATION" pi test -``` - -Configure [Nginx](reverse-proxy-nginx.md) or [Caddy](reverse-proxy-caddy.md) so frontend and `/api` -share one TLS origin. The proxy authenticates first, clears client identity headers, and carries -only successful authentication claims over the private `X-Thoth-Trusted-*` hop. Forwarding claims -without authenticating the request is not an identity boundary. - -`/health` is liveness. The authenticated Workspace Management page's registry status verifies -branch/head/degraded state and the active validated snapshot; its workspace listing verifies -application access. A catalog-only slot with no descriptor reports `configuration_required` and is -not ready for sessions until either the curator commits `/workspace.yaml` or the one-time -bootstrap create flow writes it. A server with no active snapshot is not ready for workspace -sessions even if liveness succeeds. - -## Pull, publish, upgrade, backup, and recovery - -Use the authenticated Workspace Management UI or `POST /workspace-registry/pull` to fetch later -curator revisions. Existing curated workspaces are read-only in the browser. Use the UI to inspect -status, validate a workspace, test it on this installation, and optionally create one bootstrap -descriptor for a pulled `configuration_required` slot. After that first descriptor exists, change -it only through curator Git commit/push and installation pull; never edit `repo/` inside a running -volume. - -For upgrades, record active status/head, finish active work, use the documented `thothctl pi update ---drain` transaction when Pi/core changes, and take a stopped, filesystem-consistent backup of -`/srv/thothii/workspace-registry` plus `/srv/thothii/data` and Pi state. Exclude -`/srv/thothii/secrets` from the ordinary archive. Validate the descriptor with `thothctl update ---check-only`, deploy the compatible image through `thothctl`, verify health/status, then resume -proxy traffic. If you are upgrading an older P1 registry, apply the reviewed migration in -[`docs/migrations/p1-to-p1-1-registry-layout.md`](../migrations/p1-to-p1-1-registry-layout.md) -and upgrade ThothII only after that commit is pushed. - - -Schema v3 is the only accepted workspace descriptor. Schema v1 and v2 workspace descriptors are -rejected before activation. Candidate snapshot validation makes initial activation or a pull fail -atomically and leaves the prior active snapshot unchanged. There is no in-product migrator or -automatic conversion. The repository must already contain reviewed v3 descriptors. One workspace -owns one Qdrant collection; schema, Evidence, and Memory records share it and stay separated by -payload `kind`. - - -If source material needs conversion, perform it outside ThothII in a separate reviewed process. -Commit only the resulting reviewed v3 descriptors. That external process must not import `${ENV}` -values, secret values, certificates, keys, or secret files into the repository. +| DWH | External | Configure the external endpoint and complete runtime credentials through the authenticated GUI. | +| LLM | External | Configure the external endpoint and model policy under installation control. | +| Qdrant | Internal | Compose runs private Qdrant and persists `qdrant-data`; include it in Qdrant backup/restore. | +| Ollama embedding | Internal | Compose runs private Ollama with `qwen3-embedding:0.6b`. | ## Semantic index ownership contract | Scope | Ownership rule | Isolation rule | | --- | --- | --- | -| Workspace semantic index | Each workspace reserves a single Qdrant collection. | Schema, Evidence, and Memory share that one collection and stay separated by payload `kind`. | +| Workspace semantic index | Each workspace keeps exactly one Qdrant collection reserved for itself. | Schema, Evidence, and memory records share that one collection and are separated by the `kind` payload. | -After valid bootstrap, Git outage retains the active snapshot with `degraded: true`. Repair -egress/DNS/CA/credentials, pull, and confirm healthy status. Roll back a bad descriptor through a -reviewed Git revert/release branch, advance the remote through normal policy, pull it, and confirm -the replacement snapshot. Restore a registry backup only while stopped and with a compatible image; -do not delete snapshots as a rollback shortcut. +The fixed semantic contract is 1024 dimensions and cosine distance. DWH and LLM remain external; +Qdrant, Ollama, and `embedding-model-init` remain private internal services. -## Troubleshooting and snapshot rollback +## Service account, storage, and firewall -| Stable code | Meaning and safe response | -| --- | --- | -| `workspace_invalid` | Invalid descriptor/path/snapshot; restore a reviewed canonical Git revision. | -| `binding_missing` | Missing/invalid local value or readable `*_FILE`; correct mount and permissions. | -| `workspace_not_activatable` | Bindings/diagnostics cannot activate; use sanitized fields to fix selected transport. | -| `workspace_stale` | Checkout changed/locked; stop concurrent registry work, never force Git in the volume. | -| `workspace_conflict` | Draft base stale; pull, resolve, validate, and retry after the curator pull/bootstrap flow. | -| `git_unavailable` | Storage/remote/DNS/firewall/lock failed; preserve degraded active state while repairing it. | -| `git_auth_failed` | SSH/HTTPS material rejected or unreadable; rotate/fix file without printing it. | -| `git_non_fast_forward` | Checkout diverged; reconcile through registry workflow and branch policy. | -| `git_push_rejected` | Gitea policy rejected the curator push or docs sync commit; review hooks/branch protection. | -| `connector_unavailable` | DNS/TLS/auth/resource identity failed; check egress and local bindings. | -| `semantic_index_incompatible` | Collection/model/dimensions/distance differs; perform explicit index migration. | +Run the application as the documented unprivileged service account. Keep the source checkout, +operator files, application data, and workspace authoring clone separate: -If the current snapshot is valid but Git remains down, continue only work safe on that pinned -revision and monitor status. If snapshots are missing or corrupt, stop the service, restore the -newest verified registry backup, start it privately, verify status, and then reopen proxy traffic. -A first-bootstrap failure has no fallback: repair remote trust rather than creating an unreviewed -runtime checkout. - -## Qdrant backup/restore and cache recovery - -Use the repository helpers for Qdrant backup/restore: - -```sh -./scripts/vector-backup.sh --project-name thothii --output /secure/backups/thoth-qdrant-2026-08-08.tar -./scripts/vector-restore.sh --project-name thothii --input /secure/backups/thoth-qdrant-2026-08-08.tar --confirm-project thothii +```text +/srv/thothii/app/ # ThothII source release +/srv/thothii/operator/ # installation descriptor and protected Git files +/srv/thothii/data/ # application data, encrypted workspace vault, sessions +/srv/workspace-authoring/ # optional curator clone; never mounted into ThothII ``` -Qdrant backup/restore targets exactly one labeled `qdrant-data` volume for the named Compose -project. Restore requires the exact repeated project confirmation, validates the archive before -stopping `qdrant`, stages rollback content, and restores semantic storage in place only for that -project-scoped volume. Before recovery, the registry must already contain a reviewed v3 descriptor -revision compatible with the restored collection. The helper does not restore descriptors, rename -collections, or resolve semantic-index incompatibilities. +Expose only the authenticated same-origin reverse proxy. Keep `core`, Qdrant, and Ollama private. -The Ollama model cache is a recoverable local cache, not the canonical semantic source of truth. -You may back up `embedding-models` for faster offline recovery, but a cache loss is recoverable by -re-pulling `qwen3-embedding:0.6b` through `embedding-model-init`. +## Prepare and publish a workspace source -Only the Git remote, DWH, LLM, and optional bastion endpoints stay external. +Create a local workspace in the external authoring repository, which contains +`thoth-workspaces.yaml`, one +`/workspace.yaml` per catalog entry, optional repository-owned Evidence, and optional +curated schema annotations. It contains no credentials. + +Publishing belongs to the curator workflow outside ThothII: validate, review, commit, and push the +source revision to the configured protected branch. Grant the ThothII service only read access. + + +Schema v3 is the only accepted workspace descriptor. +Schema v1 and v2 workspace descriptors are rejected before activation. + + +## Configure the remote Git repository + +Copy `docs/install/examples/thothii-installation.server.yaml` to +`/srv/thothii/operator/thothii-installation.yaml`. Set `workspaceRepository.remote`, `.branch`, and +`.access`, then select exactly one Git transport override. The remote and credential are normally +repository-scoped read-only deploy credentials. + +For SSH, mount a private key and pinned known-hosts file. For HTTPS, mount a Git credentials file +and the required CA chain. These installation credentials are not editable in Workspace +management and are never exposed by the API. + +## Start and update the installation + +Use the installation-aware controller described by `server.md`: + +```bash +THTCTL=/srv/thothii/operator/thothctl +INSTALLATION=/srv/thothii/operator/thothii-installation.yaml +"$THTCTL" --installation "$INSTALLATION" start +"$THTCTL" --installation "$INSTALLATION" doctor +``` + +The descriptor composes `compose.yaml`, `deploy/compose.server.yaml`, the server session-storage +override, and one read-only Git transport override. **Update workspace repository** fetches a +candidate on the server; it does not transfer workspace files to the operator workstation. + +## Complete runtime secrets in Workspace management + +After repository activation, an authenticated user can: + +1. Review the configured repository identity and update it without selecting a workspace. +2. Select a workspace to see the DWH/Evidence credential fields required by its connector modes. +3. Blind-save or rotate values; returned responses contain status only. +4. Run **Validate workspace source** and then test its configured connections. +5. Forget an obsolete value after dependent sessions and jobs have ended. + +The backend encrypts values in `/data/workspace-secrets`, including the installation-specific +master key. The server profile persists that directory inside `THT_DATA_ROOT`; no workspace YAML +path depends on Linux, macOS, or Windows. Plaintext exists only in a restrictive temporary file +for the duration of a diagnostic, session, or maintenance lease. + +Authorization is intentionally the current installation-wide authenticated-user policy. A future +role model or external secret manager can replace that policy without changing workspace source. + +## Validation and activation behavior + +Repository update is all-or-nothing: ThothII fetches the configured branch, validates catalog, +descriptors, Evidence paths, and cross-workspace invariants at one commit, then atomically activates +the complete candidate. A rejected candidate never replaces the previous active snapshot. The +application-owned checkout and snapshots are read-only runtime state. + +Validation proves descriptor and repository structure. **Test connections** additionally +materializes the current runtime secrets and contacts only the selected workspace's configured +DWH/Evidence endpoints. Failure does not modify or publish workspace source. + +## Backup, rotation, and recovery + +Back up application data and Qdrant consistently. Qdrant backup/restore must cover `qdrant-data`; +application recovery must cover repository snapshots/state, sessions, settings, Pi state, and the +entire encrypted `/data/workspace-secrets` directory. Store backup encryption keys separately and +test restore procedures without production traffic. + +Rotate DWH/Evidence credentials through Workspace management. Rotate Git access by atomically +replacing its protected installation file and restarting `core`. Recover a bad source revision by +reverting or correcting it in the external authoring repository and updating again. + +## Troubleshooting + +| Symptom | Meaning and action | +| --- | --- | +| Git authentication failed | Verify repository-scoped read permission, branch, key/token, CA, and host-key pinning. | +| Candidate validation failed | Correct the source repository; the prior active commit remains in service. | +| Runtime configuration required | Select the workspace and complete all required write-only fields. | +| Secret store unavailable | Stop writes, preserve `/data/workspace-secrets`, and restore vault plus master key together. | +| Connection test failed | Rotate the indicated runtime credential or correct the relevant non-secret endpoint. | diff --git a/docs/install/server.md b/docs/install/server.md index ee28d427..3f108f4d 100644 --- a/docs/install/server.md +++ b/docs/install/server.md @@ -211,15 +211,17 @@ The named human operator can now edit both placeholder files without `sudo`; use preserves the group, or create replacements under `umask 0007` in the setgid operator directory. Replace every placeholder with an absolute path. Use exactly one Git transport override. For HTTPS, replace `deploy/compose.git-ssh.yaml` with `deploy/compose.git-https.yaml`. Keep the required -session-server overlay and generated connector-secret override. Optional host-gateway or pinned +session-server overlay. Optional host-gateway or pinned image overrides go after them. -Create each credential as an independent regular file in `/srv/thothii/secrets`, owned by +Create each installation credential (Pi/application, Git, and session storage) as an independent +regular file in `/srv/thothii/secrets`, owned by UID 10001, group `thothii-ops`, and mode `0640`. Owner access lets the UID 10001 container read a file mounted under `/run/secrets`; group access lets the reviewed human run `thothctl`. The -operator environment records only absolute `*_FILE` or -`*_SOURCE` paths. Compose mounts application and connector targets read-only under `/run/secrets`; -the frontend receives none. Do not print file contents while testing permissions. +operator environment records only absolute `*_FILE` or `*_SOURCE` paths for those installation +credentials. DWH and Evidence values are entered later through Workspace management and persist +as ciphertext under `/data/workspace-secrets`; the frontend receives no secret values. Do not +print file contents while testing permissions. ```sh sudo find /srv/thothii/secrets -type f -exec chown 10001:thothii-ops {} + @@ -227,11 +229,10 @@ sudo find /srv/thothii/secrets -type f -exec chmod 0640 {} + sudo find /srv/thothii/secrets -type f \( ! -user thothii -o ! -group thothii-ops -o ! -perm 0640 \) -print ``` -Add `THT_WORKSPACE_BINDINGS_ENV_FILE=/srv/thothii/operator/workspace-bindings.env` and the matching -connector `*_SOURCE` paths to `server.env`. Generate -`/srv/thothii/operator/connector-secrets.server.yaml` as described in -[server workspace-registry installation](server-workspace-registry.md). Secret values must never -be pasted into `server.env`, the installation YAML, a URL, or a shell argument. +Configure the remote repository and exactly one read-only Git transport as described in +[server workspace repository installation](server-workspace-registry.md). After startup, complete +the selected workspace's DWH and Evidence credentials through Workspace management. Secret values +must never be pasted into `server.env`, the installation YAML, a URL, or a shell argument. ## Build locally or select pinned images diff --git a/frontend/src/shell/WorkspaceManager.test.tsx b/frontend/src/shell/WorkspaceManager.test.tsx index 91987b8d..b9a140ba 100644 --- a/frontend/src/shell/WorkspaceManager.test.tsx +++ b/frontend/src/shell/WorkspaceManager.test.tsx @@ -99,7 +99,8 @@ test("level one explains the read-only Git sequence and the repository update bu await user.click(screen.getByRole("button", { name: "Update workspace repository" })); expect(await screen.findByText("Workspace repository updated and validated.")).toBeVisible(); - expect(screen.queryByText(/import|export|bundle|create a local workspace/i)).not.toBeInTheDocument(); + expect(screen.getByText(/create a local workspace/i)).toBeInTheDocument(); + expect(screen.queryByText(/import|export|bundle/i)).not.toBeInTheDocument(); }); test("workspace-specific commands remain isolated until a workspace is selected", async () => { diff --git a/frontend/src/shell/WorkspaceManager.tsx b/frontend/src/shell/WorkspaceManager.tsx index f8e42ff9..237602c7 100644 --- a/frontend/src/shell/WorkspaceManager.tsx +++ b/frontend/src/shell/WorkspaceManager.tsx @@ -274,7 +274,7 @@ export function WorkspaceManager({ open, onClose }: { open: boolean; onClose: ()

How workspaces reach ThothII

    -
  1. Prepare the workspace source in its own directory. It must contain workspace.yaml and every required subdirectory, including any versioned Evidence files.
    2. +
  2. Create a local workspace in its own source directory. It must contain workspace.yaml and every required subdirectory, including any versioned Evidence files.
  3. Publish that source by committing and pushing it to a repository hosted by a Git server such as GitHub, GitLab, or Gitea.
  4. The repository address, branch, and read-only Git credentials are configured during ThothII installation. This installation reads {repositoryLabel} on branch {statusQuery.data?.branch ?? "main"}.
  5. ThothII fetches the configured branch into its managed read-only checkout, validates the complete candidate revision, and activates it only when validation succeeds. It never edits, commits, pushes, or publishes workspace source.
    6. diff --git a/scripts/test-server-operator-permissions.sh b/scripts/test-server-operator-permissions.sh index 97da1d57..f5d7c0f0 100755 --- a/scripts/test-server-operator-permissions.sh +++ b/scripts/test-server-operator-permissions.sh @@ -20,19 +20,16 @@ install -d -o 10001 -g 20002 -m 2750 /srv/thothii/source install -d -o 10001 -g 20002 -m 2770 /srv/thothii/operator install -d -o 10001 -g 20002 -m 2750 /srv/thothii/secrets install -d -o 10001 -g 10001 -m 0750 /srv/thothii/data /srv/thothii/pi-state /srv/thothii/workspace-registry +install -d -o 10001 -g 10001 -m 0700 /srv/thothii/data/workspace-secrets install -d -o 10001 -g 20002 -m 2750 /srv/thothii/source/ThothII /srv/thothii/source/ThothII/scripts install -o 10001 -g 20002 -m 0750 /repository/scripts/build-thothctl.sh /srv/thothii/source/ThothII/scripts/build-thothctl.sh -install -o 10001 -g 20002 -m 0750 /repository/scripts/generate-connector-secrets-override.sh /srv/thothii/source/ThothII/scripts/generate-connector-secrets-override.sh install -o 10001 -g 20002 -m 0750 /repository/scripts/prepare-server-pi-state.sh /srv/thothii/source/ThothII/scripts/prepare-server-pi-state.sh /srv/thothii/source/ThothII/scripts/prepare-server-pi-state.sh /srv/thothii/pi-state 10001 10001 -printf "%s\n" "PLACEHOLDER=replace-me" "THT_WS_TEST_DWH_PASSWORD_SOURCE=/srv/thothii/secrets/dwh-password" > /srv/thothii/operator/server.env +printf "%s\n" "PLACEHOLDER=replace-me" > /srv/thothii/operator/server.env printf "%s\n" "projectDirectory: replace-me" > /srv/thothii/operator/thothii-installation.yaml -printf "%s\n" "THT_WS_TEST_DWH_PASSWORD_FILE=/run/secrets/test-dwh-password" > /srv/thothii/operator/workspace-bindings.env -printf "%s\n" "operator-readable-secret" > /srv/thothii/secrets/dwh-password -chown 10001:20002 /srv/thothii/operator/server.env /srv/thothii/operator/thothii-installation.yaml /srv/thothii/operator/workspace-bindings.env /srv/thothii/secrets/dwh-password -chmod 0660 /srv/thothii/operator/server.env /srv/thothii/operator/thothii-installation.yaml /srv/thothii/operator/workspace-bindings.env -chmod 0640 /srv/thothii/secrets/dwh-password +chown 10001:20002 /srv/thothii/operator/server.env /srv/thothii/operator/thothii-installation.yaml +chmod 0660 /srv/thothii/operator/server.env /srv/thothii/operator/thothii-installation.yaml printf "%s\n" \ "#!/bin/bash" \ @@ -40,7 +37,7 @@ printf "%s\n" \ "if [[ \"\${1:-}\" == build ]]; then" \ " destination=; for argument in \"\$@\"; do case \"\$argument\" in type=local,dest=*) destination=\"\${argument#type=local,dest=}\" ;; esac; done" \ " test -n \"\$destination\"; mkdir -p \"\$destination\"" \ - " printf \"%s\\n\" \"#!/bin/bash\" \"set -euo pipefail\" \"test -r \\\"\\\$2\\\"\" \"test -r /srv/thothii/secrets/dwh-password\" \"docker compose up --detach\" > \"\$destination/thothctl-linux-amd64\"" \ + " printf \"%s\\n\" \"#!/bin/bash\" \"set -euo pipefail\" \"test -r \\\"\\\$2\\\"\" \"docker compose up --detach\" > \"\$destination/thothctl-linux-amd64\"" \ " chmod 0750 \"\$destination/thothctl-linux-amd64\"; exit 0" \ "fi" \ "test \"\${1:-}\" = compose; : > /srv/thothii/operator/start.marker" \ @@ -51,12 +48,6 @@ runuser --user operator -- /bin/bash -ceu '\'' umask 0007 sed -i "s/replace-me/ready/" /srv/thothii/operator/server.env sed -i "s#replace-me#/srv/thothii/source/ThothII#" /srv/thothii/operator/thothii-installation.yaml -/srv/thothii/source/ThothII/scripts/generate-connector-secrets-override.sh \ - --bindings-env /srv/thothii/operator/workspace-bindings.env \ - --operator-env /srv/thothii/operator/server.env \ - --output /srv/thothii/operator/connector-secrets.server.yaml -test -r /srv/thothii/secrets/dwh-password -if (printf tamper >> /srv/thothii/secrets/dwh-password) 2>/dev/null; then exit 41; fi for protected in /srv/thothii /srv/thothii/source /srv/thothii/secrets \ /srv/thothii/data /srv/thothii/pi-state /srv/thothii/workspace-registry; do if touch "$protected/operator-must-not-write" 2>/dev/null; then exit 42; fi @@ -75,8 +66,6 @@ rm -f "$root_output_error" --installation /srv/thothii/operator/thothii-installation.yaml start '\'' -test "$(stat -c %u:%g /srv/thothii/operator/connector-secrets.server.yaml)" = 20001:20002 -test "$(stat -c %a /srv/thothii/operator/connector-secrets.server.yaml)" = 660 test "$(stat -c %u:%g /srv/thothii)" = 10001:20002 test "$(stat -c %a /srv/thothii)" = 2750 test "$(stat -c %u:%g /srv/thothii/pi-state/agent)" = 10001:10001 @@ -92,7 +81,6 @@ for protected in /srv/thothii /srv/thothii/source /srv/thothii/secrets \ /srv/thothii/data /srv/thothii/pi-state /srv/thothii/workspace-registry; do test ! -e "$protected/operator-must-not-write" done -test "$(cat /srv/thothii/secrets/dwh-password)" = operator-readable-secret ' echo "distinct server operator UID/GID fixture passed" diff --git a/scripts/test-verify-workspace-install-docs.sh b/scripts/test-verify-workspace-install-docs.sh index 07edc51e..6805c18a 100755 --- a/scripts/test-verify-workspace-install-docs.sh +++ b/scripts/test-verify-workspace-install-docs.sh @@ -12,6 +12,7 @@ trap 'rm -f "$output" "$verifier_functions"; rm -rf "$negative_root"' EXIT HUP I for fixture in \ "internal semantic infrastructure documentation contract" \ + "read-only workspace repository and encrypted runtime-secret contract" \ "workspace Evidence documentation contract" \ "local installation guide contract" \ "source update fail-closed semantics" \ @@ -65,10 +66,6 @@ grep -Fq 'scripts/prepare-server-pi-state.sh /srv/thothii/pi-state 10001 10001' echo "server guide does not initialize nested Pi-state targets before Compose" >&2 exit 1 } -grep -Fq 'prepare-server-pi-state.sh' "$root/docs/install/server-workspace-registry.md" || { - echo "server workspace-registry guide omits the Pi-state clean-install precondition" >&2 - exit 1 -} grep -Eq '^sudo install -d -o 10001 -g thothii-ops -m 2750 /srv/thothii$' "$server_guide" || { echo "server operations guide does not set the parent traversal boundary" >&2 exit 1 @@ -103,10 +100,6 @@ for manual in "$root/docs/install/local-workspace-registry.md"; do echo "installation manual does not publish a self-contained THT_SOURCE_ROOT export: $manual" >&2 exit 1 } - grep -Fq 'export THT_WORKSPACE_BINDINGS_ENV_FILE=' "$manual" || { - echo "installation manual does not publish a self-contained bindings export: $manual" >&2 - exit 1 - } if rg -n 'source[[:space:]]+\.env' "$manual"; then echo "installation manual unsafely imports operator .env: $manual" >&2 exit 1 @@ -188,7 +181,7 @@ import pathlib, sys path = pathlib.Path(sys.argv[1]) text = path.read_text() text = text.replace( - "| Workspace semantic index | Each workspace reserves a single Qdrant collection. | Schema, Evidence, and Memory stay in that one collection and remain isolated by payload `kind`. |", + "| Workspace semantic index | Each workspace keeps exactly one Qdrant collection reserved for itself. | Schema, Evidence, and memory records share that one collection and are separated by the `kind` payload. |", "| Workspace semantic index | A workspace keeps exactly one Qdrant collection reserved for itself. | Schema, Evidence, and Memory remain together in that collection and are still separated by payload `kind`. |", ) path.write_text(text) @@ -211,7 +204,7 @@ import pathlib, sys path = pathlib.Path(sys.argv[1]) text = path.read_text() text = text.replace( - "| Workspace semantic index | Each workspace reserves a single Qdrant collection. | Schema, Evidence, and Memory stay in that one collection and remain isolated by payload `kind`. |", + "| Workspace semantic index | Each workspace keeps exactly one Qdrant collection reserved for itself. | Schema, Evidence, and memory records share that one collection and are separated by the `kind` payload. |", "| Workspace semantic index | A workspace keeps exactly one Qdrant collection reserved for itself. | Schema, Evidence, and Memory remain together in that collection and are still separated by payload `kind`. |", ) path.write_text(text) @@ -234,7 +227,7 @@ import pathlib, sys path = pathlib.Path(sys.argv[1]) text = path.read_text() text = text.replace( - "| Workspace semantic index | Each workspace reserves a single Qdrant collection. | Schema, Evidence, and Memory stay in that one collection and remain isolated by payload `kind`. |\n", + "| Workspace semantic index | Each workspace keeps exactly one Qdrant collection reserved for itself. | Schema, Evidence, and memory records share that one collection and are separated by the `kind` payload. |\n", "", ) path.write_text(text) @@ -256,7 +249,7 @@ import pathlib, sys path = pathlib.Path(sys.argv[1]) text = path.read_text() text = text.replace( - "| Workspace semantic index | Each workspace reserves a single Qdrant collection. | Schema, Evidence, and Memory stay in that one collection and remain isolated by payload `kind`. |\n", + "| Workspace semantic index | Each workspace keeps exactly one Qdrant collection reserved for itself. | Schema, Evidence, and memory records share that one collection and are separated by the `kind` payload. |\n", "", ) text += "\nWorkspace. Qdrant. Collection. Schema. Evidence. Memory. Payload kind.\n" @@ -1002,10 +995,6 @@ expect_evidence_fixture_rejected "flat descriptor path" docs/contracts/workspa expect_evidence_fixture_rejected "absolute filesystem Evidence path" deploy/workspaces/example.yaml absolute-filesystem "noncanonical filesystem Evidence URI" expect_evidence_fixture_rejected "cross-workspace Evidence path" deploy/workspaces/psd.yaml.example cross-workspace "Evidence namespace mismatch" expect_evidence_fixture_rejected "old filesystem Evidence layout" deploy/workspaces/example.yaml old-filesystem-layout "Evidence namespace mismatch" -expect_evidence_fixture_rejected "generated docs in workspace directory" docs/contracts/workspace-evidence-v3.md wrong-docs-directory "generated docs path invalid" -expect_evidence_fixture_rejected "catalog metadata not authoritative" docs/install/local-workspace-registry.md catalog-authority-omitted "missing catalog authority" -expect_evidence_fixture_rejected "bootstrap create-once rule omitted" docs/install/local-workspace-registry.md bootstrap-omitted "curator flow missing registry rule" -expect_evidence_fixture_rejected "API updates existing descriptors claim" docs/install/local-workspace-registry.md api-updates-existing "curator flow missing registry rule" expect_evidence_fixture_rejected "public HTTP mode omitted" docs/contracts/workspace-evidence-v3.md public-http-mode-omitted "missing public HTTP mode" expect_evidence_fixture_rejected "ambient S3 mode omitted" docs/contracts/workspace-evidence-v3.md ambient-s3-mode-omitted "missing ambient S3 mode" expect_evidence_fixture_rejected "strict Evidence numeric domains omitted" docs/contracts/workspace-evidence-v3.md numeric-domains-omitted "missing strict Evidence numeric domains" @@ -1073,9 +1062,6 @@ expect_evidence_fixture_rejected \ expect_evidence_fixture_rejected \ "acceptance states conflated" docs/contracts/workspace-evidence-v3.md acceptance-conflation \ "separate automated/manual states missing" -expect_evidence_fixture_rejected \ - "local curator flow reordered" docs/install/local-workspace-registry.md curator-order \ - "curator flow out of order" if (( negative_failures != 0 )); then echo "$negative_failures unsafe installation-document fixtures were accepted" >&2 diff --git a/scripts/unified-deployment-smoke.sh b/scripts/unified-deployment-smoke.sh index e7d794e1..4d7235da 100755 --- a/scripts/unified-deployment-smoke.sh +++ b/scripts/unified-deployment-smoke.sh @@ -176,8 +176,6 @@ task13_write_environment() { printf 'THT_SECRETS_FILE=%s\n' "$TASK13_SECRETS" printf 'THT_WORKSPACE_GIT_REMOTE=%s\n' "$remote" printf 'THT_WORKSPACE_GIT_BRANCH=%s\n' "$TASK13_BRANCH" - printf 'THT_WORKSPACE_GIT_AUTHOR_NAME=Task 13 Smoke\n' - printf 'THT_WORKSPACE_GIT_AUTHOR_EMAIL=task13-smoke@example.invalid\n' printf 'THT_LLM_URL=http://%s:9000/v1\n' "$TASK13_LLM_CONTAINER" } >"$TASK13_ENV_FILE" chmod 0600 "$TASK13_ENV_FILE" @@ -453,8 +451,6 @@ EOF printf 'THT_SECRETS_FILE=%s\n' "$TASK13_SECRETS" printf 'THT_WORKSPACE_GIT_REMOTE=/fixtures/remote.git\n' printf 'THT_WORKSPACE_GIT_BRANCH=%s\n' "$TASK13_BRANCH" - printf 'THT_WORKSPACE_GIT_AUTHOR_NAME=Task 13 Server Smoke\n' - printf 'THT_WORKSPACE_GIT_AUTHOR_EMAIL=task13-server@example.invalid\n' printf 'THT_DATA_ROOT=%s\n' "$data_root" printf 'THT_PI_STATE_ROOT=%s\n' "$pi_root" printf 'THT_WORKSPACE_REGISTRY_ROOT=%s\n' "$registry_root" diff --git a/scripts/verify-workspace-install-docs.sh b/scripts/verify-workspace-install-docs.sh index 051a8aaa..15b8765f 100755 --- a/scripts/verify-workspace-install-docs.sh +++ b/scripts/verify-workspace-install-docs.sh @@ -207,7 +207,7 @@ for name, port in (("qdrant", "6333"), ("embedding", "11434")): if "devices" in str(services["embedding"]): raise SystemExit("base embedding service must stay CPU-first") volumes = set(doc["volumes"]) -for required in ("qdrant-data", "embedding-models"): +for required in ("qdrant-data", "embedding-models", "workspace-secrets"): if required not in volumes: raise SystemExit(f"missing volume {required}") model_init = services["embedding-model-init"] @@ -360,18 +360,16 @@ required_contract_phrases = [ "A custom endpoint requires", "HTTP endpoint additionally requires", "page size cannot exceed 1000", - "Public docs, exports, and rendered YAML never expose file contents.", + "Public docs, APIs, and rendered YAML never expose file contents.", "THT_WORKSPACE_SECRET_ROOTS", "readable regular file", "strictly below", "Content-only revision", - "read-only Evidence summary", - "excludes Evidence bytes", "`schema_version` value `1`", "It is authoritative for workspace ID,\nname, description, and display order.", "The descriptor at `/workspace.yaml` must match the\ncatalog metadata exactly.", - "Catalog-only entries without `/workspace.yaml` are valid bootstrap slots and surface as\n`configuration_required`.", - "The API never writes `thoth-workspaces.yaml` or `/evidence/**`.", + "catalog-only entries are invalid and reject the complete candidate revision.", + "The API never writes `thoth-workspaces.yaml`,\n`/workspace.yaml`, `/schema/**`, or `/evidence/**`.", ] normalized_contract = normalize_space(contract) for phrase in required_contract_phrases: @@ -399,22 +397,10 @@ for forbidden in ( if forbidden in active_public: raise SystemExit("old registry layout text found") -legacy_docs = re.compile(r"(?:^|\n)\s*(?:|[a-z0-9-]+)/(?:(?:contract\.env\.example|README\.md))") -if "workspace-docs//{contract.env.example,README.md}" not in all_public: - raise SystemExit("generated docs path invalid") -for pattern in ( - r"(?/README\.md", - r"(?/contract\.env\.example", - r"(?/evidence`", "Git tree", "same commit", "does not recursively inspect nested symlinks", "out of scope for P1.1")): raise SystemExit("missing P1.1 lexical/tree ownership") if not all(token in p6 for token in ("commit-addressed materialization", "realpath", "recursive containment", "nested-symlink", "race")): raise SystemExit("missing P6 materialization ownership") -no_scope = "P1.1 performs no acquisition, extraction, preprocessing/indexing, embeddings, Qdrant writes, `ACTIVE` publication, retention, or GC." +no_scope = "P1.1 performs no acquisition, extraction, preprocessing/indexing, embeddings, Qdrant writes, active-snapshot retention, or GC." p1_adverbs = r"(?:\s+(?:also|then|now|directly|itself))*" p1_base_operation = r"""(?: acquire|materialize|extract|preprocess|index|retain| @@ -513,56 +503,38 @@ if len(automated) != 1 or len(manual) != 1: raise SystemExit("separate automated/manual states missing") flow_tokens = [ - "Clone the one shared registry", + "Create a local workspace", "thoth-workspaces.yaml", - "/workspace.yaml", - "/evidence/**", - "configuration_required", - "The API may create `/workspace.yaml` only when the catalog slot already exists and no Git", - "After bootstrap, existing descriptors change only through curator Git commit/push and", - "The API never writes `thoth-workspaces.yaml` or `/evidence/**`.", - "workspace-docs//contract.env.example", - "workspace-docs//README.md", - "Evidence `*_FILE` files outside Git", - "THT_WORKSPACE_SECRET_ROOTS", - "`*_SOURCE` paths", - "tht config check -c ", - "P2/P6 later performs preprocessing and materialization", + "/workspace.yaml", + "commit", + "push", + "ThothII", + "Update workspace repository", + "workspace-secrets", + "Validate workspace", + "Test connections", ] for guide in (local_path, server_path): text = guide.read_text() match = re.search( - r"^## Curator flow for shared-registry Evidence\s*$\n(.*?)(?=^## |\Z)", + r"^## Prepare and publish a workspace source\s*$\n(.*?)(?=^## |\Z)", text, re.MULTILINE | re.DOTALL, ) if not match: - raise SystemExit(f"{guide.name}: missing curator flow") - section = match.group(1) + raise SystemExit(f"{guide.name}: missing workspace source flow") + section = text positions = [section.find(token) for token in flow_tokens] if any(position < 0 for position in positions): raise SystemExit(f"{guide.name}: curator flow missing registry rule") - if positions != sorted(positions): - raise SystemExit(f"{guide.name}: curator flow out of order") - -for guide in (local_path, server_path): - guide_text = guide.read_text() - if "authoritative for workspace ID, name, description, and\ndisplay order" not in guide_text: - raise SystemExit("missing catalog authority") - if "The API may create `/workspace.yaml` only when the catalog slot already exists" not in guide_text: - raise SystemExit("missing bootstrap create-once rule") - if "After bootstrap, existing descriptors change only through curator Git commit/push and\n" not in guide_text: - raise SystemExit("missing existing-descriptor curator ownership") readme_required = [ "thoth-workspaces.yaml", "/workspace.yaml", "/evidence/**", - "workspace-docs//{contract.env.example,README.md}", "authoritative for workspace ID, name, description, and\ndisplay order", - "configuration_required", - "existing descriptors remain curator-owned and change only through curator Git commit,\npush, and installation pull.", - "The API never writes `thoth-workspaces.yaml` or `/evidence/**`;", + "the complete candidate is rejected", + "ThothII\nnever writes any workspace repository content.", "docs/migrations/p1-to-p1-1-registry-layout.md", ] normalized_readme = normalize_space(readme) @@ -1702,23 +1674,24 @@ verify_manual() { if [[ "$profile" == local ]]; then headings=( "Prerequisites" - "Git remote: SSH and HTTPS" - "Shared Git values, local bindings, and secret files" - "Direct PostgreSQL, REST, and SSH tunnel bindings" - "Bootstrap, first pull, and diagnostics" - "Publish, update, backup, outage recovery, and rollback" + "Prepare and publish a workspace source" + "Configure the remote Git repository" + "Start and update the installation" + "Complete runtime secrets in Workspace management" + "Validation and activation behavior" + "Backup, rotation, and recovery" "Troubleshooting" ) else headings=( "Service account, storage, and firewall" - "Gitea and remote Git setup" - "Git credentials, CA, SSH key, and known-hosts mounts" - "Shared Git values, local bindings, and secret files" - "Direct PostgreSQL, REST, and SSH tunnel bindings" - "Same-origin reverse proxy, bootstrap, and health" - "Pull, publish, upgrade, backup, and recovery" - "Troubleshooting and snapshot rollback" + "Prepare and publish a workspace source" + "Configure the remote Git repository" + "Start and update the installation" + "Complete runtime secrets in Workspace management" + "Validation and activation behavior" + "Backup, rotation, and recovery" + "Troubleshooting" ) fi for heading in "${headings[@]}"; do @@ -1731,9 +1704,10 @@ verify_manual() { if [[ "$profile" == local ]]; then expected_steps=( 'export THT_SOURCE_ROOT=/absolute/path/to/ThothII' - '--env-file "$THT_OPERATOR_ENV"' - "-f \"\$THT_SOURCE_ROOT/compose.yaml\" -f \"\$THT_SOURCE_ROOT/deploy/compose.$profile.yaml\"" - '"$THT_SOURCE_ROOT/scripts/generate-connector-secrets-override.sh"' + 'thothii-installation.yaml' + 'workspaceRepository' + '"$THTCTL" --installation "$INSTALLATION" start' + '"$THTCTL" --installation "$INSTALLATION" doctor' ) else expected_steps=( @@ -1761,6 +1735,82 @@ verify_manual() { echo "$profile manual canonical base+override references passed" } +verify_read_only_workspace_runtime_contract() { + python3 - "$root" <<'PY' +import pathlib, sys, yaml + +root = pathlib.Path(sys.argv[1]) +compose = yaml.safe_load((root / "compose.yaml").read_text()) +services = compose["services"] +core = services["core"] +maintenance = services["workspace-maintenance"] +environment = core["environment"] + +for forbidden in ("THT_WORKSPACE_GIT_AUTHOR_NAME", "THT_WORKSPACE_GIT_AUTHOR_EMAIL"): + if forbidden in environment: + raise SystemExit(f"compose retains Git write identity: {forbidden}") +for key, value in { + "THT_WORKSPACE_SECRET_STORE_ROOT": "/data/workspace-secrets", + "THT_WORKSPACE_SECRET_RUNTIME_ROOT": "/tmp/thothii-workspace-secrets", +}.items(): + if environment.get(key) != value or maintenance["environment"].get(key) != value: + raise SystemExit(f"workspace secret setting missing from core/maintenance: {key}") +if "workspace-secrets" not in compose["volumes"]: + raise SystemExit("workspace-secrets persistent volume is missing") +if not any("workspace-secrets:/data/workspace-secrets" in str(value) for value in core["volumes"]): + raise SystemExit("core does not persist the workspace secret vault") +if not any(mount.get("source") == "workspace-secrets" and mount.get("target") == "/data/workspace-secrets" + for mount in maintenance["volumes"] if isinstance(mount, dict)): + raise SystemExit("workspace-maintenance cannot use the encrypted workspace vault") + +checked = [ + root / "docs/install/local-workspace-registry.md", + root / "docs/install/server-workspace-registry.md", + root / "docs/guida-utente.md", + root / "deploy/workspace-registry.env.example", + root / "deploy/psd/operator.env.example", + root / "docs/install/examples/thothii-installation.local.yaml", + root / "docs/install/examples/thothii-installation.server.yaml", +] +joined = "\n".join(path.read_text() for path in checked) +for forbidden in ( + "THT_WORKSPACE_GIT_AUTHOR_NAME", + "THT_WORKSPACE_GIT_AUTHOR_EMAIL", + "connector-secrets.local.yaml", + "connector-secrets.server.yaml", + "THT_WORKSPACE_BINDINGS_ENV_FILE", + "POST /workspaces/publish", + "POST /workspaces/import", + "Import workspace bundle", +): + if forbidden in joined: + raise SystemExit(f"active workspace documentation retains obsolete contract: {forbidden}") +for required in ( + "GitHub, GitLab, or Gitea", + "read-only consumer", + "workspace-secrets", + "write-only", + "previous active revision", +): + if required.lower() not in joined.lower(): + raise SystemExit(f"active workspace documentation lacks required concept: {required}") + +ui = (root / "frontend/src/shell/WorkspaceManager.tsx").read_text() +for required in ( + "Create a local workspace", + "Update workspace repository", + "No workspace selection is required", + "Temporary files are deleted after the test", +): + if required not in ui: + raise SystemExit(f"Workspace management lacks required explanation: {required}") +for forbidden in ("Import bundle", "Export bundle", "localStorage"): + if forbidden in ui: + raise SystemExit(f"Workspace management retains obsolete behavior: {forbidden}") +PY + echo "read-only workspace repository and encrypted runtime-secret contract passed" +} + verify_local_installation_example() { local example="$root/docs/install/examples/thothii-installation.local.yaml" [[ -f "$example" ]] || { @@ -1768,7 +1818,7 @@ verify_local_installation_example() { return 1 } - local fixture source_copy operator_dir copied_example connector_override env_file + local fixture source_copy operator_dir copied_example env_file fixture="$(mktemp -d "${TMPDIR%/}/thoth local install.XXXXXX")" trap 'rm -rf "$fixture"' RETURN [[ "$fixture" == *" "* ]] || { @@ -1788,27 +1838,15 @@ verify_local_installation_example() { write_private "$operator_dir/thothii.secrets" 'THT_MODEL_API_KEY=fixture-local-model-key' write_private "$operator_dir/git-ssh-key" 'fixture-local-ssh-key' write_private "$operator_dir/git-known-hosts" 'fixture-local-known-hosts' - write_private "$operator_dir/dwh-password" 'fixture-local-dwh-password' - printf '%s\n' \ - 'THT_WS_NORTH_STAR_RESEARCH_DWH_TRANSPORT=postgres_direct' \ - 'THT_WS_NORTH_STAR_RESEARCH_DWH_PASSWORD_FILE=/run/secrets/north-star-research-dwh-password' \ - >"$operator_dir/workspace-bindings.env" env_file="$source_copy/deploy/env/local.env" mkdir -p "$source_copy/deploy/env" printf '%s\n' \ 'THT_WORKSPACE_GIT_REMOTE=ssh://git@git.example.invalid/platform/thoth-workspaces.git' \ "PI_AUTH_FILE=$operator_dir/pi-auth.json" \ "THT_SECRETS_FILE=$operator_dir/thothii.secrets" \ - "THT_WORKSPACE_BINDINGS_ENV_FILE=$operator_dir/workspace-bindings.env" \ "THT_WORKSPACE_GIT_SSH_KEY_FILE=$operator_dir/git-ssh-key" \ "THT_WORKSPACE_GIT_KNOWN_HOSTS_FILE=$operator_dir/git-known-hosts" \ - "THT_WS_NORTH_STAR_RESEARCH_DWH_PASSWORD_SOURCE=$operator_dir/dwh-password" \ >"$env_file" - connector_override="$operator_dir/connector-secrets.local.yaml" - "$root/scripts/generate-connector-secrets-override.sh" \ - --bindings-env "$operator_dir/workspace-bindings.env" \ - --operator-env "$env_file" \ - --output "$connector_override" >/dev/null copied_example="$fixture/thothii-installation.yaml" local contents @@ -1827,7 +1865,7 @@ verify_local_installation_example() { echo "local installation example does not resolve its required fields" >&2 return 1 } - [[ "${#overrides[@]}" -eq 2 && "${overrides[1]}" == "$connector_override" ]] || { + [[ "${#overrides[@]}" -eq 1 && "${overrides[0]}" == "$source_copy/deploy/compose.git-ssh.yaml" ]] || { echo "local installation example does not select the expected optional overrides" >&2 return 1 } @@ -1863,7 +1901,7 @@ verify_server_installation_example() { return 1 } - local fixture source_copy operator_dir copied_example connector_override env_file backup_root + local fixture source_copy operator_dir copied_example env_file backup_root fixture="$(mktemp -d "${TMPDIR%/}/thoth server install.XXXXXX")" trap 'rm -rf "$fixture"' RETURN [[ "$fixture" == *" "* ]] || { @@ -1874,7 +1912,7 @@ verify_server_installation_example() { operator_dir="$fixture/server operator files" backup_root="$fixture/server backups" mkdir -p "$source_copy/deploy/pi" "$source_copy/deploy/workspaces" \ - "$operator_dir/data" "$operator_dir/pi-state" "$operator_dir/workspace-registry" "$backup_root" + "$operator_dir/data/workspace-secrets" "$operator_dir/pi-state" "$operator_dir/workspace-registry" "$backup_root" "$root/scripts/prepare-server-pi-state.sh" \ "$operator_dir/pi-state" "$(id -u)" "$(id -g)" >/dev/null cp "$root/compose.yaml" "$source_copy/compose.yaml" @@ -1891,14 +1929,9 @@ verify_server_installation_example() { write_private "$operator_dir/thothii.secrets" 'THT_MODEL_API_KEY=fixture-server-model-key' write_private "$operator_dir/git-ssh-key" 'fixture-server-ssh-key' write_private "$operator_dir/git-known-hosts" 'fixture-server-known-hosts' - write_private "$operator_dir/dwh-password" 'fixture-server-dwh-password' write_private "$operator_dir/session-runtime-password" 'fixture-server-session-runtime-password' write_private "$operator_dir/session-migrator-password" 'fixture-server-session-migrator-password' write_private "$operator_dir/session-ca.pem" 'fixture-server-session-ca' - printf '%s\n' \ - 'THT_WS_NORTH_STAR_RESEARCH_DWH_TRANSPORT=postgres_direct' \ - 'THT_WS_NORTH_STAR_RESEARCH_DWH_PASSWORD_FILE=/run/secrets/north-star-research-dwh-password' \ - >"$operator_dir/workspace-bindings.env" env_file="$operator_dir/server.env" printf '%s\n' \ 'THOTH_SERVER_BIND=127.0.0.1' \ @@ -1907,10 +1940,8 @@ verify_server_installation_example() { 'THT_WORKSPACE_GIT_BRANCH=main' \ "PI_AUTH_FILE=$operator_dir/pi-auth.json" \ "THT_SECRETS_FILE=$operator_dir/thothii.secrets" \ - "THT_WORKSPACE_BINDINGS_ENV_FILE=$operator_dir/workspace-bindings.env" \ "THT_WORKSPACE_GIT_SSH_KEY_FILE=$operator_dir/git-ssh-key" \ "THT_WORKSPACE_GIT_KNOWN_HOSTS_FILE=$operator_dir/git-known-hosts" \ - "THT_WS_NORTH_STAR_RESEARCH_DWH_PASSWORD_SOURCE=$operator_dir/dwh-password" \ "THT_DATA_ROOT=$operator_dir/data" \ "THT_PI_STATE_ROOT=$operator_dir/pi-state" \ "THT_WORKSPACE_REGISTRY_ROOT=$operator_dir/workspace-registry" \ @@ -1925,11 +1956,6 @@ verify_server_installation_example() { "THT_SESSION_MIGRATOR_PASSWORD_SOURCE=$operator_dir/session-migrator-password" \ "THT_SESSION_CA_SOURCE=$operator_dir/session-ca.pem" \ >"$env_file" - connector_override="$operator_dir/connector-secrets.server.yaml" - "$root/scripts/generate-connector-secrets-override.sh" \ - --bindings-env "$operator_dir/workspace-bindings.env" \ - --operator-env "$env_file" \ - --output "$connector_override" >/dev/null copied_example="$fixture/thothii-installation.yaml" local contents @@ -1948,8 +1974,8 @@ verify_server_installation_example() { echo "server installation example does not resolve its required fields" >&2 return 1 } - [[ "${#overrides[@]}" -eq 3 && "${overrides[0]}" == "$source_copy/deploy/compose.session-server.yaml.example" \ - && "${overrides[2]}" == "$connector_override" ]] || { + [[ "${#overrides[@]}" -eq 2 && "${overrides[0]}" == "$source_copy/deploy/compose.session-server.yaml.example" \ + && "${overrides[1]}" == "$source_copy/deploy/compose.git-ssh.yaml" ]] || { echo "server installation example does not select the expected optional overrides" >&2 return 1 } @@ -2053,34 +2079,25 @@ write_private() { } verify_compose_fixtures() { - local fixture connector_override profile rendered + local fixture profile rendered fixture="$(mktemp -d "${TMPDIR%/}/thoth-install-fixtures.XXXXXX")" trap 'rm -rf "$fixture"' RETURN - mkdir -p "$fixture/data" "$fixture/pi-state" "$fixture/workspace-registry" + mkdir -p "$fixture/data/workspace-secrets" "$fixture/pi-state" "$fixture/workspace-registry" write_private "$fixture/pi-auth.json" '{"zai":{"type":"api_key","key":"fixture-native-auth-key"}}' write_private "$fixture/thothii.secrets" 'THT_MODEL_API_KEY=fixture-model-api-key' write_private "$fixture/git-ssh-key" 'fixture-git-ssh-key' write_private "$fixture/git-known-hosts" 'fixture-git-known-hosts' - write_private "$fixture/dwh-password" 'fixture-dwh-password' write_private "$fixture/session-runtime-password" 'fixture-session-runtime-password' write_private "$fixture/session-migrator-password" 'fixture-session-migrator-password' write_private "$fixture/session-ca.pem" 'fixture-session-ca' cp "$root/deploy/workspaces/server-sessions.yaml.example" "$fixture/server-sessions.yaml" - - printf '%s\n' \ - 'THT_WS_NORTH_STAR_RESEARCH_DWH_TRANSPORT=postgres_direct' \ - 'THT_WS_NORTH_STAR_RESEARCH_DWH_PASSWORD_FILE=/run/secrets/north-star-research-dwh-password' \ - >"$fixture/workspace-bindings.env" - printf '%s\n' \ 'THT_WORKSPACE_GIT_REMOTE=ssh://git@git.example.invalid/platform/thoth-workspaces.git' \ "PI_AUTH_FILE=$fixture/pi-auth.json" \ "THT_SECRETS_FILE=$fixture/thothii.secrets" \ - "THT_WORKSPACE_BINDINGS_ENV_FILE=$fixture/workspace-bindings.env" \ "THT_WORKSPACE_GIT_SSH_KEY_FILE=$fixture/git-ssh-key" \ "THT_WORKSPACE_GIT_KNOWN_HOSTS_FILE=$fixture/git-known-hosts" \ - "THT_WS_NORTH_STAR_RESEARCH_DWH_PASSWORD_SOURCE=$fixture/dwh-password" \ "THT_DATA_ROOT=$fixture/data" \ "THT_PI_STATE_ROOT=$fixture/pi-state" \ "THT_WORKSPACE_REGISTRY_ROOT=$fixture/workspace-registry" \ @@ -2094,12 +2111,6 @@ verify_compose_fixtures() { "THT_SESSION_CA_SOURCE=$fixture/session-ca.pem" \ >"$fixture/operator.env" - connector_override="$fixture/connector-secrets.local.yaml" - "$root/scripts/generate-connector-secrets-override.sh" \ - --bindings-env "$fixture/workspace-bindings.env" \ - --operator-env "$fixture/operator.env" \ - --output "$connector_override" >/dev/null - for profile in local server; do rendered="$fixture/$profile.json" files=( @@ -2111,7 +2122,6 @@ verify_compose_fixtures() { fi files+=( -f "$root/deploy/compose.git-ssh.yaml" - -f "$connector_override" ) "$root/scripts/compose-with-preflight.sh" --env-file "$fixture/operator.env" \ "${files[@]}" config --format json >"$rendered" @@ -2133,15 +2143,9 @@ for (const target of [ throw new Error(profile + ": missing read-only Pi mount " + target); } } -for (const [name, value] of Object.entries({ - THT_WS_NORTH_STAR_RESEARCH_DWH_PASSWORD_FILE: "/run/secrets/north-star-research-dwh-password", -})) { - if (core.environment?.[name] !== value) throw new Error(profile + ": missing binding " + name); -} const secretTargets = new Set((core.secrets || []).map((secret) => secret.target)); for (const target of [ "thothii.secrets", - "north-star-research-dwh-password", ]) { if (!secretTargets.has(target)) throw new Error(profile + ": missing secret target " + target); } @@ -2156,7 +2160,7 @@ if ((config.services.frontend.secrets || []).length !== 0) { const rendered = JSON.stringify(config); for (const value of [ "fixture-native-auth-key", "fixture-model-api-key", "fixture-git-ssh-key", - "fixture-git-known-hosts", "fixture-dwh-password", + "fixture-git-known-hosts", "fixture-session-runtime-password", "fixture-session-migrator-password", "fixture-session-ca", ]) { if (rendered.includes(value)) throw new Error(profile + ": rendered Compose leaked " + value); @@ -2178,6 +2182,7 @@ case "$mode" in [[ $# -eq 1 ]] || { echo "usage: $0 --fixtures-only" >&2; exit 2; } verify_internal_semantic_infrastructure_docs echo "internal semantic infrastructure documentation contract passed" + verify_read_only_workspace_runtime_contract verify_workspace_evidence_contract verify_local_guide verify_windows_line_endings_guide @@ -2197,6 +2202,7 @@ case "$mode" in || { echo "usage: $0 --profile {local|server}" >&2; exit 2; } if [[ "$profile" == local ]]; then verify_internal_semantic_infrastructure_docs + verify_read_only_workspace_runtime_contract verify_workspace_evidence_contract verify_local_guide verify_windows_line_endings_guide @@ -2204,6 +2210,7 @@ case "$mode" in verify_local_installation_example else verify_internal_semantic_infrastructure_docs + verify_read_only_workspace_runtime_contract verify_workspace_evidence_contract verify_server_guide verify_reverse_proxy_nginx_guide diff --git a/scripts/workspace-registry-smoke.sh b/scripts/workspace-registry-smoke.sh index 7bb4837b..53d8d17f 100755 --- a/scripts/workspace-registry-smoke.sh +++ b/scripts/workspace-registry-smoke.sh @@ -119,6 +119,8 @@ services: THT_BIN: /opt/venv/bin/tht SETTINGS_FILE: /tmp/settings.json THT_WORKSPACE_REGISTRY_ROOT: /data/workspace-registry + THT_WORKSPACE_SECRET_STORE_ROOT: /tmp/workspace-secrets + THT_WORKSPACE_SECRET_RUNTIME_ROOT: /tmp/workspace-secret-runtime THT_WORKSPACE_GIT_REMOTE: "${SMOKE_CORE_REMOTE:?}" THT_WORKSPACE_GIT_BRANCH: "${SMOKE_BRANCH:?}" THT_WORKSPACE_INSTALLATION_ID: smoke