Compare commits

...
4 Commits
Author SHA1 Message Date
Codex ec0421e9fd feat: publish verified installation images and native release bundles 2026-09-28 17:46:09 +02:00
Codex 55f3569e55 feat(cli): validate prerequisites and seal installation plans 2026-09-28 17:03:25 +02:00
Codex b9c3369e7b feat(cli): prepare and validate application documents offline 2026-09-28 16:33:07 +02:00
Codex 64e6b9664a feat(cli): prepare and validate workspace documents offline
Reuse the runtime catalog and workspace parsers in a standalone helper paired with tht. Add document templates, safe diagnostics, local Evidence checks, native bundle builds, shared CLI fixtures and IT/EN preparation guides. Record the approved document-first specification and ticket breakdown. Refs #43.
2026-09-28 15:35:25 +02:00
62 changed files with 6292 additions and 48 deletions
+11
View File
@@ -638,6 +638,17 @@ procedura non implica che DWH o provider LLM siano locali o disponibili offline.
installazione manuale: verifica dell'host, generazione della configurazione, predisposizione
delle credenziali protette e avvio dei servizi. Non è un installer dell'applicazione.
**Installation preparation** — La predisposizione dei documenti che descrivono i workspace
e i parametri dell'installazione, prima di applicarli. Permette all'operatore di raccogliere
e correggere le informazioni senza avviare l'applicazione.
**Installation validation** — La verifica ripetibile della completezza e coerenza dei
documenti e delle precondizioni di un'installazione. Distingue ciò che è stato verificato
da ciò che richiede un'applicazione già avviata.
**Installation execution** — L'applicazione dei documenti verificati per predisporre e
avviare ThothII. Non raccoglie nuovi parametri dall'operatore durante l'esecuzione.
**Platform acceptance** — La verifica che una Manual standalone installation possa essere
predisposta e avviata su una specifica combinazione di sistema operativo, architettura e runtime,
distinta dalla verifica funzionale del collegamento a DWH e provider LLM.
+15 -1
View File
@@ -1,10 +1,24 @@
# Project state
Updated: 2026-09-15. This is a current snapshot, not a release diary. Stable commands
Updated: 2026-09-28. This is a current snapshot, not a release diary. Stable commands
and invariants are in [AGENTS.md](AGENTS.md); prior snapshots remain in Git.
## Current contracts
- Document-first installation ticket #43 provides offline `tht workspace prepare`
and `tht workspace validate` through a native two-executable bundle. Build and
test instructions are in [the host CLI guide](tools/tht/README.md). Local validation
reuses runtime workspace/catalog parsers and explicitly defers runtime Evidence,
database binding and readiness checks. Ticket #44 adds `tht installation prepare`,
explicit `installation credentials`, and `installation validate --workspaces PATH`
for protected application documents, canonical model settings and schema-v1
database bootstrap inputs. These commands do not start services or import Catalog
bindings. Ticket #45 adds host `installation preflight` and `installation plan`
with release/image checks, canonical external diagnostics and private input seals;
see [the preflight reference](docs/install/installation-preflight.md).
The remainder of the installation tickets,
Docker Hub publication and example databases remain pending.
- React supports full/embedded rendering independently of local/OIDC/upstream auth,
with EN/IT UI and immutable session interaction language. See
[application shell](docs/architecture/application-shell.md) and
+609
View File
@@ -6,11 +6,13 @@
"": {
"name": "thothii-backend",
"dependencies": {
"@aws-sdk/client-s3": "3.1141.0",
"@fastify/cookie": "11.1.2",
"@fastify/cors": "^11.2.0",
"@fastify/rate-limit": "11.2.0",
"@types/pg": "^8.20.3",
"fastify": "^5.0.0",
"ipaddr.js": "2.4.0",
"kysely": "^0.29.5",
"libphonenumber-js": "1.13.12",
"openid-client": "6.8.5",
@@ -23,11 +25,320 @@
"@testcontainers/postgresql": "^12.1.0",
"@types/node": "24.13.3",
"@types/validator": "13.15.10",
"bun": "1.4.2",
"tsx": "^4.19.0",
"typescript": "^5.6.0",
"vitest": "^2.1.0"
}
},
"node_modules/@aws-sdk/checksums": {
"version": "3.1001.1",
"resolved": "https://registry.npmjs.org/@aws-sdk/checksums/-/checksums-3.1001.1.tgz",
"integrity": "sha512-x12Q17KYlJAd3nKf8LV5LV0vt8sh8/6YfQLGPtrGnQf/tW4jqxPGq5GPpuVitpQYM3eUR4XB7CbxZf751NMbLw==",
"license": "Apache-2.0",
"dependencies": {
"@aws-sdk/core": "^3.978.1",
"@aws-sdk/types": "^3.974.6",
"@smithy/core": "^3.35.0",
"@smithy/types": "^4.19.0",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=20.0.0"
}
},
"node_modules/@aws-sdk/client-s3": {
"version": "3.1141.0",
"resolved": "https://registry.npmjs.org/@aws-sdk/client-s3/-/client-s3-3.1141.0.tgz",
"integrity": "sha512-uOVH37xGLenAdJkCPCin/JJG2PgWrFcSsDnQ9+C9Zq8N9Oalo5ol4xmn5fG28iWAlA/b/9boQZgHbMh+UsIhcg==",
"license": "Apache-2.0",
"dependencies": {
"@aws-sdk/checksums": "^3.1001.1",
"@aws-sdk/core": "^3.978.1",
"@aws-sdk/credential-provider-node": "^3.972.84",
"@aws-sdk/middleware-sdk-s3": "^3.972.77",
"@aws-sdk/signature-v4-multi-region": "^3.996.47",
"@aws-sdk/types": "^3.974.6",
"@smithy/core": "^3.35.0",
"@smithy/fetch-http-handler": "^5.8.0",
"@smithy/node-http-handler": "^4.12.1",
"@smithy/types": "^4.19.0",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=20.0.0"
}
},
"node_modules/@aws-sdk/core": {
"version": "3.978.1",
"resolved": "https://registry.npmjs.org/@aws-sdk/core/-/core-3.978.1.tgz",
"integrity": "sha512-LbY9aGsEiznDWmUc30Nwv3aIX/+dbwTx8KfS0yOC3NPYMO+O91e6jkT1azf34FwjOndq8/Q+RcVVZz5xnerwdg==",
"license": "Apache-2.0",
"dependencies": {
"@aws-sdk/types": "^3.974.6",
"@aws-sdk/xml-builder": "^3.972.41",
"@aws/lambda-invoke-store": "^0.3.0",
"@smithy/core": "^3.35.0",
"@smithy/signature-v4": "^5.7.3",
"@smithy/types": "^4.19.0",
"bowser": "^2.11.0",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=20.0.0"
}
},
"node_modules/@aws-sdk/credential-provider-env": {
"version": "3.972.72",
"resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.972.72.tgz",
"integrity": "sha512-xTKO/FWJPozTIXbozVnVGoNBhaGba8TBcx+KyUjRVeOlXE+dUc7GTR1cLvu0uTdIdmemzaFbqqCshXeZA1fZew==",
"license": "Apache-2.0",
"dependencies": {
"@aws-sdk/core": "^3.978.1",
"@aws-sdk/types": "^3.974.6",
"@smithy/core": "^3.35.0",
"@smithy/types": "^4.19.0",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=20.0.0"
}
},
"node_modules/@aws-sdk/credential-provider-http": {
"version": "3.972.74",
"resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-http/-/credential-provider-http-3.972.74.tgz",
"integrity": "sha512-u91E/hT8f4d1xy0Jl7VG4nVKJ3lxbrZkoBTeSVoJdWBiSEUMwMS/9+e0H/aJVQV//Lt5wuzP+E69v4aRSsNTmw==",
"license": "Apache-2.0",
"dependencies": {
"@aws-sdk/core": "^3.978.1",
"@aws-sdk/types": "^3.974.6",
"@smithy/core": "^3.35.0",
"@smithy/fetch-http-handler": "^5.8.0",
"@smithy/node-http-handler": "^4.12.1",
"@smithy/types": "^4.19.0",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=20.0.0"
}
},
"node_modules/@aws-sdk/credential-provider-ini": {
"version": "3.973.17",
"resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.973.17.tgz",
"integrity": "sha512-ged4KXdBkvIC81bLvNHHuQKdKak/VXhQTR1NWYTTqW0474nlmsxy9O/vlgTIohDDWH3xpBdtVMZRyjb+DnocDA==",
"license": "Apache-2.0",
"dependencies": {
"@aws-sdk/core": "^3.978.1",
"@aws-sdk/credential-provider-env": "^3.972.72",
"@aws-sdk/credential-provider-http": "^3.972.74",
"@aws-sdk/credential-provider-login": "^3.972.79",
"@aws-sdk/credential-provider-process": "^3.972.72",
"@aws-sdk/credential-provider-sso": "^3.973.16",
"@aws-sdk/credential-provider-web-identity": "^3.972.78",
"@aws-sdk/nested-clients": "^3.997.46",
"@aws-sdk/types": "^3.974.6",
"@smithy/core": "^3.35.0",
"@smithy/credential-provider-imds": "^4.5.2",
"@smithy/types": "^4.19.0",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=20.0.0"
}
},
"node_modules/@aws-sdk/credential-provider-login": {
"version": "3.972.79",
"resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-login/-/credential-provider-login-3.972.79.tgz",
"integrity": "sha512-L+Z85anONJd8MaiuraO4wRxATCdEejBZ3K3eymzWI5JPXa9sOS9CkIm72PBKqXKX+Z9p9NGMX5AIMXm0LEflgw==",
"license": "Apache-2.0",
"dependencies": {
"@aws-sdk/core": "^3.978.1",
"@aws-sdk/nested-clients": "^3.997.46",
"@aws-sdk/types": "^3.974.6",
"@smithy/core": "^3.35.0",
"@smithy/types": "^4.19.0",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=20.0.0"
}
},
"node_modules/@aws-sdk/credential-provider-node": {
"version": "3.972.84",
"resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.972.84.tgz",
"integrity": "sha512-oHt854odINVwzwsh+c5x69j0ajm4DbqqqVJ+O1ECsCIZeMDAbzFpXItaqP7UZstJj/ATdTk/KFSH0LaNAgV+kA==",
"license": "Apache-2.0",
"dependencies": {
"@aws-sdk/credential-provider-env": "^3.972.72",
"@aws-sdk/credential-provider-http": "^3.972.74",
"@aws-sdk/credential-provider-ini": "^3.973.17",
"@aws-sdk/credential-provider-process": "^3.972.72",
"@aws-sdk/credential-provider-sso": "^3.973.16",
"@aws-sdk/credential-provider-web-identity": "^3.972.78",
"@aws-sdk/types": "^3.974.6",
"@smithy/core": "^3.35.0",
"@smithy/credential-provider-imds": "^4.5.2",
"@smithy/types": "^4.19.0",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=20.0.0"
}
},
"node_modules/@aws-sdk/credential-provider-process": {
"version": "3.972.72",
"resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.972.72.tgz",
"integrity": "sha512-rLIp2xbMjX/k9/od7APpqq1ZgXXnV0pOL1Th3ZsL8Wu0TRtBsDTVS8iPqcfRFcHakFxPvR04OSTv2ka2qOb/2A==",
"license": "Apache-2.0",
"dependencies": {
"@aws-sdk/core": "^3.978.1",
"@aws-sdk/types": "^3.974.6",
"@smithy/core": "^3.35.0",
"@smithy/types": "^4.19.0",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=20.0.0"
}
},
"node_modules/@aws-sdk/credential-provider-sso": {
"version": "3.973.16",
"resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.973.16.tgz",
"integrity": "sha512-IGihaJfFZYacJJr/odqILCoK7W/mvrZ7cuK7ECn3sAu4vLC6u0V8bS7mCGbdugJ8Aum2tnvqmx0F2MRFp2rn9g==",
"license": "Apache-2.0",
"dependencies": {
"@aws-sdk/core": "^3.978.1",
"@aws-sdk/nested-clients": "^3.997.46",
"@aws-sdk/token-providers": "3.1138.0",
"@aws-sdk/types": "^3.974.6",
"@smithy/core": "^3.35.0",
"@smithy/types": "^4.19.0",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=20.0.0"
}
},
"node_modules/@aws-sdk/credential-provider-web-identity": {
"version": "3.972.78",
"resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.972.78.tgz",
"integrity": "sha512-/y9WvNtlcPBGLR0qc1a+9J/xtYZfVczvLUOuXaVWylzttH7ewsxwHtjmiJSolNrVSDorIxHGHMU61CbonRkmwA==",
"license": "Apache-2.0",
"dependencies": {
"@aws-sdk/core": "^3.978.1",
"@aws-sdk/nested-clients": "^3.997.46",
"@aws-sdk/types": "^3.974.6",
"@smithy/core": "^3.35.0",
"@smithy/types": "^4.19.0",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=20.0.0"
}
},
"node_modules/@aws-sdk/middleware-sdk-s3": {
"version": "3.972.77",
"resolved": "https://registry.npmjs.org/@aws-sdk/middleware-sdk-s3/-/middleware-sdk-s3-3.972.77.tgz",
"integrity": "sha512-E7W2UOeUoc+lg3uIfR/dM7ZwusHwhBQrKMnlkRv4EXRR+C0YtV1pg25xC7GdZIhXH+NAMgZPCbE7o5to2cjFiw==",
"license": "Apache-2.0",
"dependencies": {
"@aws-sdk/core": "^3.978.1",
"@aws-sdk/signature-v4-multi-region": "^3.996.47",
"@aws-sdk/types": "^3.974.6",
"@smithy/core": "^3.35.0",
"@smithy/types": "^4.19.0",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=20.0.0"
}
},
"node_modules/@aws-sdk/nested-clients": {
"version": "3.997.46",
"resolved": "https://registry.npmjs.org/@aws-sdk/nested-clients/-/nested-clients-3.997.46.tgz",
"integrity": "sha512-oRxtBcka/JGHGs9l9p9IVajGoTP8vTPmoAzdHGy4Qcy9P5vPnDf6nhIeM/COQNY9k/OahImTRaLkHftoXvfcmQ==",
"license": "Apache-2.0",
"dependencies": {
"@aws-sdk/core": "^3.978.1",
"@aws-sdk/signature-v4-multi-region": "^3.996.47",
"@aws-sdk/types": "^3.974.6",
"@smithy/core": "^3.35.0",
"@smithy/fetch-http-handler": "^5.8.0",
"@smithy/node-http-handler": "^4.12.1",
"@smithy/types": "^4.19.0",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=20.0.0"
}
},
"node_modules/@aws-sdk/signature-v4-multi-region": {
"version": "3.996.47",
"resolved": "https://registry.npmjs.org/@aws-sdk/signature-v4-multi-region/-/signature-v4-multi-region-3.996.47.tgz",
"integrity": "sha512-Zk08macMvQTHzQJCLJVkOlviVoqwYMrpXv4lmLN7b7sAbiMoOK7Go0NYdR5UeF+MW8LIbRmwrNy9u/5VvX1U5g==",
"license": "Apache-2.0",
"dependencies": {
"@aws-sdk/types": "^3.974.6",
"@smithy/signature-v4": "^5.7.3",
"@smithy/types": "^4.19.0",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=20.0.0"
}
},
"node_modules/@aws-sdk/token-providers": {
"version": "3.1138.0",
"resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.1138.0.tgz",
"integrity": "sha512-GpyAr0DD63YOEmYFM6Df+gJuIgC92MMTiBK4FTKfxii5MJ9ge20epR7LyroulscYlG89J+ZB2ivFDPjvfQhzdw==",
"license": "Apache-2.0",
"dependencies": {
"@aws-sdk/core": "^3.978.1",
"@aws-sdk/nested-clients": "^3.997.46",
"@aws-sdk/types": "^3.974.6",
"@smithy/core": "^3.35.0",
"@smithy/types": "^4.19.0",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=20.0.0"
}
},
"node_modules/@aws-sdk/types": {
"version": "3.974.6",
"resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.974.6.tgz",
"integrity": "sha512-v/clNZzZnDxGyvpHMOGpJKVXFAExJzUNAAjaWGdcx8QAcXLGwTaOkw33p5SHAi0YAioK32xB3hWwOekRVfmfKg==",
"license": "Apache-2.0",
"dependencies": {
"@smithy/types": "^4.19.0",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=20.0.0"
}
},
"node_modules/@aws-sdk/xml-builder": {
"version": "3.972.41",
"resolved": "https://registry.npmjs.org/@aws-sdk/xml-builder/-/xml-builder-3.972.41.tgz",
"integrity": "sha512-ctjVSyCMegrWfXlx6VqzSBFI6UqmQ5ZlnfMhdLIiWmhoH8UAQxSCP5N3OpG7X3k4LnS7ou74C4mt20+bfTW2aQ==",
"license": "Apache-2.0",
"dependencies": {
"@smithy/types": "^4.19.0",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=20.0.0"
}
},
"node_modules/@aws/lambda-invoke-store": {
"version": "0.3.0",
"resolved": "https://registry.npmjs.org/@aws/lambda-invoke-store/-/lambda-invoke-store-0.3.0.tgz",
"integrity": "sha512-sl4Bm6yiMNYrZKkqqDFWN0UfnWhlS8ivKxrYl+6t0gCLrqr8y3B2IqZZbFRkfaVVp7C/baApyh71P+LeE1A2sQ==",
"license": "Apache-2.0",
"engines": {
"node": ">=18.0.0"
}
},
"node_modules/@balena/dockerignore": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/@balena/dockerignore/-/dockerignore-1.0.2.tgz",
@@ -802,6 +1113,174 @@
"node": ">=8"
}
},
"node_modules/@oven/bun-darwin-aarch64": {
"version": "1.4.2",
"resolved": "https://registry.npmjs.org/@oven/bun-darwin-aarch64/-/bun-darwin-aarch64-1.4.2.tgz",
"integrity": "sha512-MXdZkP1featqxZ+/VTXWG1BVjM4OGBehVY2Q88EeUj/7L0UMeCGItmyPYTN+wxvlGJ6F66JEtzsw+GvQWewnag==",
"cpu": [
"arm64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
"darwin"
]
},
"node_modules/@oven/bun-darwin-x64": {
"version": "1.4.2",
"resolved": "https://registry.npmjs.org/@oven/bun-darwin-x64/-/bun-darwin-x64-1.4.2.tgz",
"integrity": "sha512-gZTxZuLjkUhAWjTETu3tw0WhsEdNkJ64daj60ybhPf835a2yollV3yTkK9JozvzKPx4TRFzLSl8C+U525pxVbw==",
"cpu": [
"x64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
"darwin"
]
},
"node_modules/@oven/bun-freebsd-aarch64": {
"version": "1.4.2",
"resolved": "https://registry.npmjs.org/@oven/bun-freebsd-aarch64/-/bun-freebsd-aarch64-1.4.2.tgz",
"integrity": "sha512-SMNItMw1Z8QeeQVKnw8jA7xQNkeXdP+OPgin4Wi/QTx/B8RHHLnuZfqmFy7NtVeT2NF0kKYppW4WWd2CCYZjhQ==",
"cpu": [
"arm64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
"freebsd"
]
},
"node_modules/@oven/bun-freebsd-x64": {
"version": "1.4.2",
"resolved": "https://registry.npmjs.org/@oven/bun-freebsd-x64/-/bun-freebsd-x64-1.4.2.tgz",
"integrity": "sha512-THbPKXhO54N0DpFRKZNDZpQ7dpbX0bWASuARckAUS9wRtFIHsiY+uULXJvxJGo2YD1YewvXQ4G8Fj7XT5oBCiw==",
"cpu": [
"x64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
"freebsd"
]
},
"node_modules/@oven/bun-linux-aarch64": {
"version": "1.4.2",
"resolved": "https://registry.npmjs.org/@oven/bun-linux-aarch64/-/bun-linux-aarch64-1.4.2.tgz",
"integrity": "sha512-3BBP9ovJ2RGHFH6Ae1CAtxNtG1+YY6GD6rmYbsUosoAk9+OEl6zeDQ/k4fBkc6dYOJCtWnx8hUxzNzQATSmvYQ==",
"cpu": [
"arm64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
"linux"
]
},
"node_modules/@oven/bun-linux-aarch64-android": {
"version": "1.4.2",
"resolved": "https://registry.npmjs.org/@oven/bun-linux-aarch64-android/-/bun-linux-aarch64-android-1.4.2.tgz",
"integrity": "sha512-3mZKO2rhsNgbAUtAHC1UKUlF2zTxFraDZT/Elv8wzyH0fJL9h+Iv3TgB9lO63w89PRn3eFe+NRA1bhVgikKNPQ==",
"cpu": [
"arm64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
"android"
]
},
"node_modules/@oven/bun-linux-aarch64-musl": {
"version": "1.4.2",
"resolved": "https://registry.npmjs.org/@oven/bun-linux-aarch64-musl/-/bun-linux-aarch64-musl-1.4.2.tgz",
"integrity": "sha512-+Sm6y+lSiSFBOtXmnekp5Q6n1tUKlyv71FCPWBc61Cgb14T5eBs8SN/nh4MUCOKzONkI3O+as3MGUgikS4aCBQ==",
"cpu": [
"arm64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
"linux"
]
},
"node_modules/@oven/bun-linux-x64": {
"version": "1.4.2",
"resolved": "https://registry.npmjs.org/@oven/bun-linux-x64/-/bun-linux-x64-1.4.2.tgz",
"integrity": "sha512-9/E/UXOTpSo3YsV5g+FhtTd/qTpiWoKuxS12cqtuYA1ssu9fRAoPQnipFgGyck3tWO63iUdxBiygq+kELFawng==",
"cpu": [
"x64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
"linux"
]
},
"node_modules/@oven/bun-linux-x64-android": {
"version": "1.4.2",
"resolved": "https://registry.npmjs.org/@oven/bun-linux-x64-android/-/bun-linux-x64-android-1.4.2.tgz",
"integrity": "sha512-6HC5tzcC79113n2IHCTJMWv+HsQImv4ZFEK2XpYLxY6HbT8tM4cUM2Zv1bHZBQsS3jv/zYBamDJ1UX7If0d5tw==",
"cpu": [
"x64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
"android"
]
},
"node_modules/@oven/bun-linux-x64-musl": {
"version": "1.4.2",
"resolved": "https://registry.npmjs.org/@oven/bun-linux-x64-musl/-/bun-linux-x64-musl-1.4.2.tgz",
"integrity": "sha512-vVTKUg1bnPhRP/Hp73jIVoFh2vPFNYEqYX0ERKfZBOQEEHitNAeukZzzuUDZS0SoDCIpuWUGSpd/CDMbjdR+Uw==",
"cpu": [
"x64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
"linux"
]
},
"node_modules/@oven/bun-windows-aarch64": {
"version": "1.4.2",
"resolved": "https://registry.npmjs.org/@oven/bun-windows-aarch64/-/bun-windows-aarch64-1.4.2.tgz",
"integrity": "sha512-8EJ1ST7339WJE3poPW5nBgVW/lWf9HBz4W27ZUNhburKmcBLOByPyE6DP9fHD8FQGm5c+ilUN2hX1mrW0jxq9Q==",
"cpu": [
"arm64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
"win32"
]
},
"node_modules/@oven/bun-windows-x64": {
"version": "1.4.2",
"resolved": "https://registry.npmjs.org/@oven/bun-windows-x64/-/bun-windows-x64-1.4.2.tgz",
"integrity": "sha512-+bN6OuVld/9diT/RLSXSW7JE6CvNE3gL9XsAEjULi1nUsXd6DNO6GuA9jNdNb3r8PdJFnYHr5aypNV1Oj3Rd9g==",
"cpu": [
"x64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
"win32"
]
},
"node_modules/@pinojs/redact": {
"version": "0.4.0",
"resolved": "https://registry.npmjs.org/@pinojs/redact/-/redact-0.4.0.tgz",
@@ -1235,6 +1714,87 @@
"win32"
]
},
"node_modules/@smithy/core": {
"version": "3.35.0",
"resolved": "https://registry.npmjs.org/@smithy/core/-/core-3.35.0.tgz",
"integrity": "sha512-zRMhfkByhT2snNdr1si24vJitU6Cr9ix2MikUfWmkAgp4jrNP0GcKSP5YvwQ+TlI8AZXER5QOGJn3JsVtSD9/A==",
"license": "Apache-2.0",
"dependencies": {
"@smithy/types": "^4.19.0",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=18.0.0"
}
},
"node_modules/@smithy/credential-provider-imds": {
"version": "4.5.2",
"resolved": "https://registry.npmjs.org/@smithy/credential-provider-imds/-/credential-provider-imds-4.5.2.tgz",
"integrity": "sha512-A9uSdn72ozbRUSit0eib0TW7nXuNPlaeM0zcGkJ+nE6tFcSDbnmtwoxbTCFBukVQcszDAyvsd7+rTduPTXpygg==",
"license": "Apache-2.0",
"dependencies": {
"@smithy/core": "^3.33.2",
"@smithy/types": "^4.17.2",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=18.0.0"
}
},
"node_modules/@smithy/fetch-http-handler": {
"version": "5.8.0",
"resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-5.8.0.tgz",
"integrity": "sha512-ycSJu3tFAQ4v04CBB0agqFMVsSQ1iG3yw+SpgxRqKfaURpQD4CZ8Wn0zPMmSnOuTpTh65Vz+EA0rMrw089wvkA==",
"license": "Apache-2.0",
"dependencies": {
"@smithy/core": "^3.33.3",
"@smithy/types": "^4.18.0",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=18.0.0"
}
},
"node_modules/@smithy/node-http-handler": {
"version": "4.12.1",
"resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-4.12.1.tgz",
"integrity": "sha512-ThMkboGeONWXAelq9FvGsuJC4rOi+qyC4/zhUF58xYpxUg5sQKx2VXZYJmtNjr4dSuBJ1HeJXETQILCz3wOHvw==",
"license": "Apache-2.0",
"dependencies": {
"@smithy/core": "^3.33.3",
"@smithy/types": "^4.18.0",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=18.0.0"
}
},
"node_modules/@smithy/signature-v4": {
"version": "5.7.4",
"resolved": "https://registry.npmjs.org/@smithy/signature-v4/-/signature-v4-5.7.4.tgz",
"integrity": "sha512-tHy0K0VtqNd5Y7Y41h0a0Lhh0L1GzC08dTWg0F7vRJWFtTENg7IZikf3wQkanYIRdb7ngoIPMTmqgUi401fEeQ==",
"license": "Apache-2.0",
"dependencies": {
"@smithy/core": "^3.35.0",
"@smithy/types": "^4.19.0",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=18.0.0"
}
},
"node_modules/@smithy/types": {
"version": "4.19.0",
"resolved": "https://registry.npmjs.org/@smithy/types/-/types-4.19.0.tgz",
"integrity": "sha512-r7jh49VJxGerfAcTQA6gXcKc+98zOp/tqRwzYjgOE+iSQsP6cEU1hq2QzbuipmP68QtYdY9wKEhiCQZIzHgZ4Q==",
"license": "Apache-2.0",
"dependencies": {
"tslib": "^2.6.2"
},
"engines": {
"node": ">=18.0.0"
}
},
"node_modules/@testcontainers/postgresql": {
"version": "12.1.0",
"resolved": "https://registry.npmjs.org/@testcontainers/postgresql/-/postgresql-12.1.0.tgz",
@@ -1821,6 +2381,12 @@
"node": ">= 6"
}
},
"node_modules/bowser": {
"version": "2.14.1",
"resolved": "https://registry.npmjs.org/bowser/-/bowser-2.14.1.tgz",
"integrity": "sha512-tzPjzCxygAKWFOJP011oxFHs57HzIhOEracIgAePE4pqB3LikALKnSzUyU4MGs9/iCEUuHlAJTjTc5M+u7YEGg==",
"license": "MIT"
},
"node_modules/brace-expansion": {
"version": "2.1.4",
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.1.4.tgz",
@@ -1876,6 +2442,43 @@
"node": ">=10.0.0"
}
},
"node_modules/bun": {
"version": "1.4.2",
"resolved": "https://registry.npmjs.org/bun/-/bun-1.4.2.tgz",
"integrity": "sha512-TrSXo6HJfIEaczpb3kjX82I2pL47vK1QUNmHRCUdz9IzaOwa9lzOXSWwu2l18YHE3sNfGRapVLd4nNm+22vVVA==",
"cpu": [
"arm64",
"x64"
],
"dev": true,
"hasInstallScript": true,
"license": "MIT",
"os": [
"darwin",
"linux",
"android",
"freebsd",
"win32"
],
"bin": {
"bun": "bin/bun.exe",
"bunx": "bin/bunx.exe"
},
"optionalDependencies": {
"@oven/bun-darwin-aarch64": "1.4.2",
"@oven/bun-darwin-x64": "1.4.2",
"@oven/bun-freebsd-aarch64": "1.4.2",
"@oven/bun-freebsd-x64": "1.4.2",
"@oven/bun-linux-aarch64": "1.4.2",
"@oven/bun-linux-aarch64-android": "1.4.2",
"@oven/bun-linux-aarch64-musl": "1.4.2",
"@oven/bun-linux-x64": "1.4.2",
"@oven/bun-linux-x64-android": "1.4.2",
"@oven/bun-linux-x64-musl": "1.4.2",
"@oven/bun-windows-aarch64": "1.4.2",
"@oven/bun-windows-x64": "1.4.2"
}
},
"node_modules/byline": {
"version": "5.0.0",
"resolved": "https://registry.npmjs.org/byline/-/byline-5.0.0.tgz",
@@ -4114,6 +4717,12 @@
"node": ">=20"
}
},
"node_modules/tslib": {
"version": "2.8.1",
"resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz",
"integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==",
"license": "0BSD"
},
"node_modules/tsx": {
"version": "4.22.4",
"resolved": "https://registry.npmjs.org/tsx/-/tsx-4.22.4.tgz",
+4
View File
@@ -3,6 +3,7 @@
"private": true,
"type": "module",
"scripts": {
"build:workspace-tools": "node scripts/build-workspace-tools.mjs",
"dev": "tsx watch src/server.ts",
"prebuild": "node scripts/clean-dist.mjs",
"build": "tsc -p tsconfig.json",
@@ -14,11 +15,13 @@
"test:schema-v3-verifier": "npm run test:schema-v4-verifier"
},
"dependencies": {
"@aws-sdk/client-s3": "3.1141.0",
"@fastify/cookie": "11.1.2",
"@fastify/cors": "^11.2.0",
"@fastify/rate-limit": "11.2.0",
"@types/pg": "^8.20.3",
"fastify": "^5.0.0",
"ipaddr.js": "2.4.0",
"kysely": "^0.29.5",
"libphonenumber-js": "1.13.12",
"openid-client": "6.8.5",
@@ -31,6 +34,7 @@
"@testcontainers/postgresql": "^12.1.0",
"@types/node": "24.13.3",
"@types/validator": "13.15.10",
"bun": "1.4.2",
"tsx": "^4.19.0",
"typescript": "^5.6.0",
"vitest": "^2.1.0"
+49
View File
@@ -0,0 +1,49 @@
// Maintainer-only build. The resulting two-binary bundle needs no extra host runtime.
import { spawnSync } from "node:child_process";
import { createHash } from "node:crypto";
import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
import { dirname, join, resolve } from "node:path";
import { fileURLToPath } from "node:url";
const repository = process.env.THT_BUILD_SOURCE_ROOT ? resolve(process.env.THT_BUILD_SOURCE_ROOT) : resolve(dirname(fileURLToPath(import.meta.url)), "../..");
const backend = join(repository, "backend");
const native = `${process.platform === "win32" ? "windows" : process.platform}-${process.arch === "x64" ? "amd64" : process.arch}`;
const targets = {
"windows-amd64": ["windows", "amd64", "bun-windows-x64"],
"darwin-amd64": ["darwin", "amd64", "bun-darwin-x64"],
"darwin-arm64": ["darwin", "arm64", "bun-darwin-arm64"],
"linux-amd64": ["linux", "amd64", "bun-linux-x64"],
"linux-arm64": ["linux", "arm64", "bun-linux-arm64"],
};
const requested = process.argv.slice(2);
const selected = requested.length === 1 && requested[0] === "--all" ? Object.keys(targets) : requested.length ? requested : [native];
if (selected.some((target) => !targets[target])) {
console.error(`Usage: npm run build:workspace-tools -- [${Object.keys(targets).join("|")}|--all]`);
process.exit(2);
}
function run(command, args, cwd = backend, env = process.env) {
const result = spawnSync(command, args, { cwd, env, stdio: "inherit" });
if (result.error || result.status !== 0) throw new Error(`Build failed: ${command}`);
}
const revision = spawnSync("git", ["rev-parse", "HEAD"], { cwd: repository, encoding: "utf8" });
if (revision.status !== 0) throw new Error("Cannot read build revision");
const commit = revision.stdout.trim();
const buildTime = process.env.THT_BUILD_TIME ?? new Date().toISOString();
const releaseVersion = process.env.THT_BUILD_VERSION ?? "0.0.0-dev";
if (!/^[0-9]+\.[0-9]+\.[0-9]+(?:-[A-Za-z0-9.-]+)?$/.test(releaseVersion) || !Number.isFinite(Date.parse(buildTime))) throw new Error("Invalid build identity");
const module = "github.com/aritmolab/thothii/tools/tht/internal/version";
const bunPackage = JSON.parse(readFileSync(join(backend, "node_modules", "bun", "package.json"), "utf8"));
const bun = join(backend, "node_modules", "bun", bunPackage.bin.bun);
for (const target of selected) {
const [os, arch, bunTarget] = targets[target];
const output = join(repository, "dist", "workspace-tools", target);
mkdirSync(output, { recursive: true });
const extension = os === "windows" ? ".exe" : "";
const names = [`tht${extension}`, `tht-workspace-documents${extension}`];
run(bun, ["build", "src/workspace-documents-cli.ts", "--compile", `--target=${bunTarget}`, "--outfile", join(output, names[1])]);
run("go", ["build", "-trimpath", "-ldflags", `-s -w -X ${module}.semanticVersion=${releaseVersion} -X ${module}.commit=${commit} -X ${module}.buildTime=${buildTime}`, "-o", join(output, names[0]), "./cmd/tht"], join(repository, "tools", "tht"), { ...process.env, CGO_ENABLED: "0", GOOS: os, GOARCH: arch });
const hashes = names.map((name) => `${createHash("sha256").update(readFileSync(join(output, name))).digest("hex")} ${name}\n`).join("");
writeFileSync(join(output, "SHA256SUMS"), hashes);
writeFileSync(join(output, "build.json"), JSON.stringify({ version: releaseVersion, commit, buildTime, target, bun: JSON.parse(readFileSync(join(backend, "package.json"), "utf8")).devDependencies.bun }, null, 2) + "\n");
console.log(output);
}
+195
View File
@@ -0,0 +1,195 @@
#!/usr/bin/env node
// Maintainer-only producer. Consumers download the resulting native bundle.
import { spawn } from "node:child_process";
import { mkdirSync, mkdtempSync, readFileSync, writeFileSync, existsSync, realpathSync, renameSync, rmSync, statSync, copyFileSync, chmodSync, openSync, closeSync, readdirSync } from "node:fs";
import { tmpdir } from "node:os";
import { basename, dirname, join, resolve } from "node:path";
import { pathToFileURL } from "node:url";
import { parse } from "yaml";
import { prepareBundle, sha256 } from "./release-bundle.mjs";
import { dockerCredentials, dockerHub } from "./release-registry.mjs";
import { giteaHosting } from "./release-hosting.mjs";
import { publishVerifiedRelease } from "./release-publication.mjs";
const repositoryRoot = resolve(import.meta.dirname, "../..");
export function optionsFromArgs(args) {
const values = {};
for (let i = 0; i < args.length; i += 2) {
if (!['--revision', '--version', '--namespace', '--platforms', '--output', '--repository'].includes(args[i]) || !args[i + 1] || values[args[i]]) throw new Error("Usage: --revision REF --version VERSION --namespace DOCKER_HUB_NAMESPACE --platforms linux/amd64 --output NEW_OR_MATCHING_DIRECTORY [--repository HTTPS_GITEA_REPO]");
values[args[i]] = args[i + 1];
}
if (!values['--revision'] || values['--revision'].startsWith('-') || !/^[0-9]+\.[0-9]+\.[0-9]+(?:-[A-Za-z0-9.-]+)?$/.test(values['--version'] ?? '') || !/^[a-z0-9][a-z0-9_-]{1,38}$/.test(values['--namespace'] ?? '') || !values['--output']) throw new Error("Supply an explicit source revision, semantic release version, Docker Hub namespace and output directory.");
const platforms = (values['--platforms'] ?? '').split(',');
if (!platforms.length || new Set(platforms).size !== platforms.length || platforms.some((p) => !['linux/amd64', 'linux/arm64'].includes(p))) throw new Error("Select explicit Linux image platforms; begin with linux/amd64 for Windows/WSL2 and Omarchy.");
const repository = values['--repository'] ?? 'https://git.tylconsulting.it/mptyl/ThothII';
const url = new URL(repository);
if (url.protocol !== 'https:' || url.username || url.password || url.search || url.hash || !/^\/[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/.test(url.pathname)) throw new Error("Use a credential-free HTTPS Gitea owner/repository URL.");
return { revision: values['--revision'], version: values['--version'], namespace: values['--namespace'], platforms: platforms.sort(), output: resolve(values['--output']), repository };
}
export function commandRunner(logs) {
let sequence = 0;
return async (command, args, { cwd = repositoryRoot, input = '', env = process.env, quiet = false, timeout = 120_000 } = {}) => {
const log = join(logs, `${++sequence}-${basename(command)}.log`);
return new Promise((accept, reject) => {
if (process.platform === 'win32') { reject(new Error('Run the release producer on Linux, WSL2 or macOS.')); return; }
const child = spawn(command, args, { cwd, env, detached: true, stdio: ['pipe', 'pipe', 'pipe'] });
const stdout = [], stderr = []; let size = 0, overflow = false, settled = false, reapTimer;
const terminate = () => {
if (overflow) return;
overflow = true;
try { process.kill(-child.pid, 'SIGKILL'); } catch { child.kill('SIGKILL'); }
// An escaped descendant must never retain our pipes indefinitely.
reapTimer = setTimeout(() => { child.stdout.destroy(); child.stderr.destroy(); finish(-1); }, 500);
};
const timer = setTimeout(terminate, timeout);
const collect = (chunks) => (data) => { size += data.length; if (size > 64 * 2 ** 20) terminate(); else chunks.push(data); };
child.stdout.on('data', collect(stdout)); child.stderr.on('data', collect(stderr));
child.stdin.on('error', () => {}); child.stdin.end(input);
child.on('error', () => { settled = true; clearTimeout(timer); clearTimeout(reapTimer); reject(new Error(`Required maintainer command ${basename(command)} is unavailable.`)); });
function finish(code) {
if (settled) return;
settled = true;
clearTimeout(timer); clearTimeout(reapTimer);
if (!quiet) writeFileSync(log, Buffer.concat([...stdout, ...stderr]), { mode: 0o600 });
if (code !== 0 || overflow) reject(new Error(`${basename(command)} failed${quiet ? '.' : `; inspect private log ${log}`}`));
else accept(Buffer.concat(stdout).toString());
}
child.on('close', finish);
});
};
}
function acquireOutput(output) {
if (!existsSync(output)) mkdirSync(output, { mode: 0o700 });
if (realpathSync(output) !== output || !statSync(output).isDirectory() || (statSync(output).mode & 0o077)) throw new Error("Use a canonical owner-only output directory.");
if (!existsSync(join(output, 'publication-state.json')) && readdirSync(output).length) throw new Error("Choose an empty output directory or the matching previous publication directory.");
const lock = join(output, '.publisher.lock');
if (existsSync(lock)) {
const pid = Number(readFileSync(lock, 'utf8'));
if (!Number.isInteger(pid) || pid <= 0) throw new Error("Inspect the incomplete publisher lock before retrying.");
let alive = true;
try { process.kill(pid, 0); } catch (error) { if (error.code === 'ESRCH') alive = false; }
if (alive) throw new Error("Another publisher owns this output directory.");
rmSync(lock);
}
const fd = openSync(lock, 'wx', 0o600); writeFileSync(fd, String(process.pid)); closeSync(fd);
return () => rmSync(lock, { force: true });
}
export async function publishInstallation(options) {
const unlock = acquireOutput(options.output);
const logs = join(options.output, 'logs'); mkdirSync(logs, { recursive: true, mode: 0o700 });
const run = commandRunner(logs);
let worktree, temporary;
try {
const revision = (await run('git', ['rev-parse', '--verify', `${options.revision}^{commit}`], { quiet: true })).trim();
if (!/^[a-f0-9]{40}$/.test(revision)) throw new Error("Source revision is not a commit.");
const identity = { revision, version: options.version, namespace: options.namespace, platforms: options.platforms, repository: options.repository };
const statePath = join(options.output, 'publication-state.json');
let state = { identity };
if (existsSync(statePath)) {
state = JSON.parse(readFileSync(statePath, 'utf8'));
if (JSON.stringify(state.identity) !== JSON.stringify(identity)) throw new Error("Output directory belongs to a different release; choose a new directory.");
}
const save = () => { const temp = statePath + '.tmp'; writeFileSync(temp, JSON.stringify(state, null, 2) + '\n', { mode: 0o600 }); renameSync(temp, statePath); };
save();
console.log(`Release ${identity.version}: ${identity.namespace}, ${identity.platforms.join(',')}, source ${revision}`);
await run('docker', ['info', '--format', '{{.OSType}}']);
await run('docker', ['buildx', 'version']);
const registry = dockerHub(await dockerCredentials(run));
const hosting = await giteaHosting({ run, repository: options.repository, identity, body: `Installer prerelease for ${identity.platforms.join(', ')}.\n\nImages: docker.io/${identity.namespace}/thothii-core:${identity.version} and docker.io/${identity.namespace}/thothii-frontend:${identity.version}.\n\nDownload the native operator bundle and SHA256SUMS.txt below. This release supplies images, document validation and preflight; complete non-interactive setup and Windows/Omarchy/macOS acceptance are subsequent tickets. No example databases, credentials or user workspace data are included.` });
async function prepare() {
if (state.assets) {
const names = [...identity.platforms.map((platform) => `thothii-${identity.version}-${platform.replace('/', '-')}.tar.gz`), 'SHA256SUMS.txt'];
if (state.assets.length !== names.length) throw new Error("Cached artifact set is incomplete.");
for (const asset of state.assets) if (!names.includes(asset.name) || asset.path !== join(options.output, asset.name) || !existsSync(asset.path) || sha256(readFileSync(asset.path)) !== asset.sha256) throw new Error("Previously built release artifact changed; do not overwrite an immutable version.");
for (const [platform, images] of Object.entries(state.images)) for (const reference of Object.values(images)) {
const [repository, digest] = reference.replace(/^docker.io\//, '').split('@');
await registry.inspect(repository, digest, platform, { anonymous: true });
}
return state.assets;
}
temporary = realpathSync(mkdtempSync(join(tmpdir(), 'thothii-release-')));
worktree = join(temporary, 'source');
await run('git', ['worktree', 'add', '--detach', worktree, revision]);
const sourceCompose = parse(readFileSync(join(worktree, 'compose.yaml'), 'utf8'));
for (const role of ['core', 'frontend']) {
console.log(`Preparing public repository ${identity.namespace}/thothii-${role}`);
await registry.ensurePublic(identity.namespace, `thothii-${role}`);
let existing = null;
for (const platform of identity.platforms) {
const image = await registry.inspect(`${identity.namespace}/thothii-${role}`, identity.version, platform, { allowMissing: true });
if (image && (image.labels['org.opencontainers.image.revision'] !== revision || image.labels['org.opencontainers.image.version'] !== identity.version)) throw new Error("Image tag already belongs to another immutable build; select a new release version.");
existing = existing || image;
}
if (!existing) {
console.log(`Building and publishing ${role} (${identity.platforms.join(', ')})`);
await run('docker', ['buildx', 'build', '--platform', identity.platforms.join(','), '--file', `docker/${role}.Dockerfile`, '--tag', `docker.io/${identity.namespace}/thothii-${role}:${identity.version}`, '--build-arg', `IMAGE_VERSION=${identity.version}`, '--label', `org.opencontainers.image.revision=${revision}`, '--label', `org.opencontainers.image.source=${identity.repository}`, '--provenance=false', '--sbom=false', '--push', '.'], { cwd: worktree, timeout: 45 * 60_000 });
}
}
state.images = {};
for (const platform of identity.platforms) {
const images = {};
for (const role of ['core', 'frontend']) images[role] = (await registry.inspect(`${identity.namespace}/thothii-${role}`, identity.version, platform, { anonymous: true })).reference;
for (const [role, service] of [['catalog', 'catalog-db'], ['qdrant', 'qdrant'], ['embedding', 'embedding']]) {
const [named, digest] = sourceCompose.services[service].image.split('@');
let repository = named.replace(/:[^/:]+$/, '').replace(/^docker.io\//, '');
if (!repository.includes('/')) repository = 'library/' + repository;
images[role] = (await registry.inspect(repository, digest, platform, { anonymous: true })).reference;
}
state.images[platform] = images;
}
save();
console.log('Building native operator bundles from the selected source');
await run('npm', ['ci'], { cwd: join(worktree, 'backend'), timeout: 10 * 60_000 });
const sourceTime = (await run('git', ['show', '-s', '--format=%cI', revision], { quiet: true })).trim();
const targets = identity.platforms.map((platform) => platform.replace('/', '-'));
await run('node', [join(repositoryRoot, 'backend/scripts/build-workspace-tools.mjs'), ...targets], { cwd: join(worktree, 'backend'), env: { ...process.env, THT_BUILD_SOURCE_ROOT: worktree, THT_BUILD_VERSION: identity.version, THT_BUILD_TIME: sourceTime }, timeout: 10 * 60_000 });
const assets = [];
for (const platform of identity.platforms) {
const target = platform.replace('/', '-');
const name = `thothii-${identity.version}-${target}`;
const bundle = join(options.output, name);
if (existsSync(bundle)) rmSync(bundle, { recursive: true }); // owned staging, never an installed runtime
mkdirSync(bundle);
prepareBundle({ source: worktree, destination: bundle, platform, version: identity.version, revision, images: state.images[platform] });
mkdirSync(join(bundle, 'bin'));
for (const executable of ['tht', 'tht-workspace-documents']) {
copyFileSync(join(worktree, 'dist/workspace-tools', target, executable), join(bundle, 'bin', executable));
chmodSync(join(bundle, 'bin', executable), 0o755);
}
// Resolve source-independent resource/config shape without any operator credentials.
await run('docker', ['compose', '-f', join(bundle, 'compose.yaml'), '-f', join(bundle, 'deploy/compose.local.yaml'), 'config', '--no-interpolate', '--no-env-resolution', '--format', 'json']);
const archive = join(options.output, name + '.tar.gz');
await run('tar', ['-czf', archive, '-C', options.output, name]);
assets.push({ name: basename(archive), path: archive, bytes: statSync(archive).size, sha256: sha256(readFileSync(archive)) });
}
const sums = join(options.output, 'SHA256SUMS.txt');
writeFileSync(sums, assets.map((asset) => `${asset.sha256} ${asset.name}\n`).join(''));
assets.push({ name: 'SHA256SUMS.txt', path: sums, bytes: statSync(sums).size, sha256: sha256(readFileSync(sums)) });
// An empty Docker configuration proves the consumer can pull without publisher credentials.
const publicConfig = join(temporary, 'public-docker'); mkdirSync(publicConfig);
for (const [platform, images] of Object.entries(state.images)) {
for (const reference of Object.values(images)) {
console.log(`Verifying anonymous pull ${reference.split('@')[0]} (${platform})`);
await run('docker', ['--config', publicConfig, 'pull', '--platform', platform, reference], { timeout: 20 * 60_000 });
}
console.log(`Smoke checking published images (${platform})`);
await run('docker', ['run', '--rm', '--platform', platform, '--network', 'none', '--entrypoint', '/bin/sh', images.core, '-ec', 'test "$(pi --version)" = "$PI_VERSION"; tht --help >/dev/null; test -f /app/backend/dist/catalog/migrate.js; test -x /app/docker/workspace-maintenance-entrypoint.sh; test -f /app/docker/catalog-migrate.sh; test ! -e /run/secrets/thothii.secrets'], { timeout: 5 * 60_000 });
await run('docker', ['run', '--rm', '--platform', platform, '--network', 'none', '--entrypoint', '/usr/local/bin/frontend-config-smoke', images.frontend], { timeout: 60_000 });
}
state.assets = assets; save();
return assets;
}
const published = await publishVerifiedRelease({ hosting, prepare });
state.releaseURL = published.html_url; state.complete = true; save();
console.log(`Published and verified: ${published.html_url}`);
return published;
} finally {
if (worktree && existsSync(worktree)) await run('git', ['worktree', 'remove', '--force', worktree]).catch(() => {});
if (temporary && !existsSync(worktree ?? '')) rmSync(temporary, { recursive: true, force: true });
unlock();
}
}
if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) {
try { await publishInstallation(optionsFromArgs(process.argv.slice(2))); }
catch (error) { console.error(error instanceof SyntaxError ? 'Invalid release metadata; no secret values are printed.' : error.message); process.exitCode = 1; }
}
+48
View File
@@ -0,0 +1,48 @@
import { createHash } from "node:crypto";
import { mkdirSync, readFileSync, writeFileSync, lstatSync } from "node:fs";
import { dirname, join } from "node:path";
import { parse, stringify } from "yaml";
export const serviceRoles = Object.freeze({ core: "core", frontend: "frontend", "catalog-db": "catalog", "catalog-migrate": "core", "workspace-maintenance": "core", qdrant: "qdrant", embedding: "embedding", "embedding-model-init": "embedding" });
export const sha256 = (data) => createHash("sha256").update(data).digest("hex");
/** Only distribution assets enter the bundle: never a checkout, environment file or workspace. */
export function prepareBundle({ source, destination, platform, version, revision, images }) {
const compose = parse(readFileSync(join(source, "compose.yaml"), "utf8"));
if (Object.keys(compose.services).sort().join() !== Object.keys(serviceRoles).sort().join()) throw new Error("Release service contract changed; review packaging before publishing.");
for (const [name, role] of Object.entries(serviceRoles)) {
if (!/^docker\.io\/[a-z0-9][a-z0-9._/-]*@sha256:[a-f0-9]{64}$/.test(images[role] ?? "")) throw new Error("Release requires immutable Docker Hub image references.");
delete compose.services[name].build;
delete compose.services[name].pull_policy;
compose.services[name].image = images[role];
compose.services[name].platform = platform;
}
const files = {};
const write = (name, data) => {
mkdirSync(dirname(join(destination, name)), { recursive: true });
writeFileSync(join(destination, name), data);
files[name] = sha256(data);
};
write("compose.yaml", stringify(compose));
for (const name of ["deploy/compose.local.yaml", "deploy/compose.git-https.yaml", "deploy/compose.git-ssh.yaml", "docker/catalog-db-init.sql", "docker/embedding-model-init.sh"]) {
const path = join(source, name);
if (!lstatSync(path).isFile()) throw new Error("Release asset must be a regular tracked file.");
write(name, readFileSync(path));
}
// Any future source bind mount must be deliberately added to the asset allowlist.
for (const service of Object.values(compose.services)) {
for (const volume of service.volumes ?? []) {
const sourcePath = typeof volume === "string" ? volume.split(":")[0] : volume.type === "bind" ? volume.source : undefined;
if (sourcePath?.startsWith(".") && !files[sourcePath.replace(/^\.\//, "")]) throw new Error("Source bind mount is missing from the release bundle.");
}
}
const manifest = { schema_version: 1, version, revision, validator_protocol: 1,
requirements: { cpus: 2, memory_bytes: 4 * 2 ** 30, disk_bytes: 10 * 2 ** 30 },
components: ["pi", "catalog-migrations", "workspace-maintenance"],
images: Object.fromEntries(Object.entries(images).map(([role, reference]) => [role, { [platform]: reference }])),
files, compose: ["compose.yaml", "deploy/compose.local.yaml"] };
writeFileSync(join(destination, "release-manifest.json"), JSON.stringify(manifest, null, 2) + "\n");
writeFileSync(join(destination, "README.md"), `# ThothII ${version} — ${platform}\n\nSource / Sorgente: ${revision}\n\nThis prerelease provides images and document/preflight tools. Non-interactive execution and real-host acceptance are separate follow-up tickets; this is not a certified complete installation.\nQuesta prerelease fornisce immagini e strumenti di preparazione/preflight. Esecuzione non interattiva e collaudi reali sono incrementi successivi: non è ancora un'installazione completa certificata.\n\nUse bin/tht and its sibling bin/tht-workspace-documents together; no Node, Python, Bun or application checkout is required on the consumer host.\nWindows: use the Linux amd64 bundle inside Ubuntu WSL2, not a native Windows shell.\n\n1. bin/tht workspace prepare --directory NEW_WORKSPACE --id practice --name Practice\n2. bin/tht workspace validate --directory WORKSPACE\n3. bin/tht installation prepare --directory NEW_PRIVATE_INSTALLATION\n4. bin/tht installation preflight --directory INSTALLATION --release ABSOLUTE_RELEASE_DIR/release-manifest.json\n5. Complete the commented documents, generate technical credentials explicitly, then run installation validate and installation plan with --installation ABSOLUTE_INSTALLATION_FILE.\n\nImages are pinned by digest; runtime credentials and user workspaces are never bundled.\nConsult the accompanying IT/EN guides for prepared documents and mandatory runtime checks.\n`);
for (const [name, target] of [["standalone-manual-it.md", "GUIDE-IT.md"], ["standalone-manual-en.md", "GUIDE-EN.md"], ["installation-preflight.md", "PREFLIGHT.md"]]) writeFileSync(join(destination, target), readFileSync(join(source, "docs/install", name)));
return manifest;
}
+35
View File
@@ -0,0 +1,35 @@
import { test } from "node:test";
import assert from "node:assert/strict";
import { mkdtempSync, readFileSync, rmSync, existsSync } from "node:fs";
import { tmpdir } from "node:os";
import { resolve, join } from "node:path";
import { parse } from "yaml";
import { prepareBundle } from "./release-bundle.mjs";
test("consumer bundle pins every service and carries all source bind resources", () => {
const output = mkdtempSync(join(tmpdir(), "thoth-release-test-"));
const images = Object.fromEntries(["core", "frontend", "catalog", "qdrant", "embedding"].map((role) => [role, `docker.io/tylconsulting/${role}@sha256:${"a".repeat(64)}`]));
try {
prepareBundle({ source: resolve(import.meta.dirname, "../.."), destination: output, platform: "linux/amd64", version: "0.1.0-install-preview.1", revision: "b".repeat(40), images });
const manifest = JSON.parse(readFileSync(join(output, "release-manifest.json")));
const compose = parse(readFileSync(join(output, "compose.yaml"), "utf8"));
assert.equal(compose.services.core.image, images.core);
assert.equal(compose.services["catalog-migrate"].image, images.core);
assert.equal(compose.services["workspace-maintenance"].image, images.core);
assert.equal(compose.services["embedding-model-init"].image, images.embedding);
for (const service of Object.values(compose.services)) {
assert.equal(service.build, undefined);
assert.equal(service.pull_policy, undefined);
assert.equal(service.platform, "linux/amd64");
for (const volume of service.volumes ?? []) {
if (typeof volume === "string" && volume.startsWith("./")) assert.ok(existsSync(join(output, volume.split(":")[0])));
}
}
assert.equal(manifest.validator_protocol, 1);
assert.deepEqual(manifest.compose, ["compose.yaml", "deploy/compose.local.yaml"]);
assert.ok(manifest.files["docker/catalog-db-init.sql"]);
assert.ok(manifest.files["docker/embedding-model-init.sh"]);
assert.ok(!existsSync(join(output, "backend")));
assert.ok(!existsSync(join(output, "harness")));
} finally { rmSync(output, { recursive: true, force: true }); }
});
+69
View File
@@ -0,0 +1,69 @@
import { readFileSync } from "node:fs";
import { boundedFetch, readLimited } from "./release-registry.mjs";
import { sha256 } from "./release-bundle.mjs";
export async function giteaHosting({ run, repository, identity, body }) {
const remote = new URL(repository);
const credential = await run("git", ["credential", "fill"], { input: `protocol=https\nhost=${remote.host}\n\n`, quiet: true, env: { ...process.env, GIT_TERMINAL_PROMPT: "0" } });
const fields = Object.fromEntries(credential.trim().split("\n").map((line) => { const at = line.indexOf("="); return [line.slice(0, at), line.slice(at + 1)]; }));
if (!fields.username || !fields.password) throw new Error("Gitea publishing credentials are unavailable in the Git credential store.");
const authorization = `Basic ${Buffer.from(`${fields.username}:${fields.password}`).toString("base64")}`;
const api = `${remote.origin}/api/v1/repos${remote.pathname.replace(/\.git$/, "")}`;
const marker = `<!-- thothii-release:${JSON.stringify(identity)} -->`;
const tag = `installation-v${identity.version}`;
async function request(path, options = {}, allowMissing = false) {
const response = await boundedFetch(api + path, { ...options, headers: { Authorization: authorization, ...options.headers } }, 120_000);
if (allowMissing && response.status === 404) return null;
if (!response.ok) throw new Error(`Gitea release operation failed (HTTP ${response.status}).`);
const bytes = await readLimited(response, 4 * 2 ** 20);
try { return JSON.parse(bytes.toString()); } catch { throw new Error("Gitea returned invalid release metadata."); }
}
const json = (method, value) => ({ method, headers: { "Content-Type": "application/json" }, body: JSON.stringify(value) });
async function verifyTag(required = false) {
const existing = await request(`/tags/${encodeURIComponent(tag)}`, {}, true);
if ((!existing && required) || (existing && existing.commit?.sha !== identity.revision)) throw new Error("Release Git tag does not match the requested source revision.");
}
async function assets(release) { return request(`/releases/${release.id}/assets`); }
async function verifyAsset(asset, expected, publicRead) {
const url = new URL(asset.browser_download_url);
if (url.origin !== remote.origin) throw new Error("Unexpected release asset origin.");
const response = await boundedFetch(url, { headers: publicRead ? {} : { Authorization: authorization } }, 120_000);
if (!response.ok || sha256(await readLimited(response, expected.bytes + 1)) !== expected.sha256) throw new Error("Published release asset does not match its verified checksum.");
}
return {
async open() {
const info = await request("");
if (info.private || !info.permissions?.push) throw new Error("Release hosting must be a public Gitea repository with publication rights.");
await verifyTag();
const existing = await request(`/releases/tags/${encodeURIComponent(tag)}`, {}, true);
if (existing) {
if (!existing.body?.includes(marker)) throw new Error("Release version already belongs to another source or publication identity; it will not be overwritten.");
return existing;
}
return request("/releases", json("POST", { tag_name: tag, target_commitish: identity.revision, name: `ThothII ${identity.version}`, body: `${body}\n\n${marker}`, draft: true, prerelease: true }));
},
async upload(release, asset) {
const matching = (await assets(release)).filter((item) => item.name === asset.name);
if (matching.length > 1) throw new Error("Ambiguous release assets; no published files were replaced.");
if (matching.length === 1) { await verifyAsset(matching[0], asset, false); return; }
const data = readFileSync(asset.path);
if (sha256(data) !== asset.sha256) throw new Error("Local release asset changed before upload.");
const form = new FormData(); form.append("attachment", new Blob([data]), asset.name);
await request(`/releases/${release.id}/assets?name=${encodeURIComponent(asset.name)}`, { method: "POST", body: form });
},
async verify(release, expected, publicRead) {
await verifyTag(publicRead);
const uploaded = await assets(release);
if (uploaded.length !== expected.length) throw new Error("Release asset set is incomplete or contains unexpected files.");
for (const asset of expected) {
const matching = uploaded.filter((item) => item.name === asset.name);
if (matching.length !== 1) throw new Error("Release asset missing or ambiguous.");
await verifyAsset(matching[0], asset, publicRead);
}
},
async publish(release) {
await verifyTag();
return request(`/releases/${release.id}`, json("PATCH", { draft: false }));
},
};
}
+43
View File
@@ -0,0 +1,43 @@
import { test } from 'node:test';
import assert from 'node:assert/strict';
import { giteaHosting } from './release-hosting.mjs';
test('release hosting refuses a pre-existing Git tag on another commit', async () => {
const fetch = globalThis.fetch;
const revision = 'a'.repeat(40);
let writes = 0;
globalThis.fetch = async (url, options) => {
if (options.method) { writes++; return Response.json({ id: 1, draft: true }); }
if (String(url).includes('/releases/tags/')) return new Response('', { status: 404 });
if (String(url).includes('/tags/installation-v')) return Response.json({ commit: { sha: 'b'.repeat(40) } });
return Response.json({ private: false, permissions: { push: true } });
};
try {
const hosting = await giteaHosting({ run: async () => 'username=test\npassword=test\n', repository: 'https://example.test/owner/repo', identity: { revision, version: '1.0.0' }, body: '' });
await assert.rejects(hosting.open(), /tag.*revision/i);
assert.equal(writes, 0);
} finally { globalThis.fetch = fetch; }
});
test('matching Git tag is accepted and verified again before publishing', async () => {
const fetch = globalThis.fetch;
const identity = { revision: 'a'.repeat(40), version: '1.0.0' };
let sha = identity.revision;
let writes = 0;
globalThis.fetch = async (url, options) => {
if (options.method) { writes++; return Response.json({ id: 1, draft: false }); }
if (String(url).includes('/releases/tags/')) return Response.json({ id: 1, draft: true, body: `<!-- thothii-release:${JSON.stringify(identity)} -->` });
if (String(url).includes('/tags/')) return Response.json({ commit: { sha } });
if (String(url).endsWith('/assets')) return Response.json([]);
return Response.json({ private: false, permissions: { push: true } });
};
try {
const hosting = await giteaHosting({ run: async () => 'username=test\npassword=test\n', repository: 'https://example.test/owner/repo', identity, body: '' });
const release = await hosting.open();
await hosting.verify(release, [], false);
sha = 'b'.repeat(40);
await assert.rejects(hosting.verify(release, [], false), /tag.*revision/i);
await assert.rejects(hosting.publish(release), /tag.*revision/i);
assert.equal(writes, 0);
} finally { globalThis.fetch = fetch; }
});
+18
View File
@@ -0,0 +1,18 @@
import { test } from 'node:test';
import assert from 'node:assert/strict';
import { mkdtempSync, rmSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { commandRunner } from './publish-installation.mjs';
test('publisher timeout terminates descendants that retain output pipes', async () => {
const logs = mkdtempSync(join(tmpdir(), 'release-process-test-'));
try {
const started = Date.now();
await assert.rejects(commandRunner(logs)(process.execPath, ['-e', `
require('child_process').spawn(process.execPath, ['-e', 'setTimeout(() => {}, 2500)'], {stdio: 'inherit'});
setTimeout(() => {}, 2500);
`], { timeout: 250, quiet: true }));
assert.ok(Date.now() - started < 1800, 'timeout must not wait for the descendant to exit naturally');
} finally { rmSync(logs, { recursive: true, force: true }); }
});
+14
View File
@@ -0,0 +1,14 @@
/** Drafts are the publication boundary. A partial build/upload is never a consumer release. */
export async function publishVerifiedRelease({ hosting, prepare }) {
const release = await hosting.open();
const assets = await prepare();
if (!release.draft) {
await hosting.verify(release, assets, true);
return release;
}
for (const asset of assets) await hosting.upload(release, asset);
await hosting.verify(release, assets, false);
const published = await hosting.publish(release);
await hosting.verify(published, assets, true);
return published;
}
@@ -0,0 +1,34 @@
import { test } from "node:test";
import assert from "node:assert/strict";
import { publishVerifiedRelease } from "./release-publication.mjs";
test("an interrupted preparation stays draft and retry publishes only after every artifact verifies", async () => {
const events = [];
let failing = true;
const hosting = {
open: async () => ({ id: 7, draft: true }),
upload: async (_draft, asset) => events.push(`upload:${asset.name}`),
verify: async (_draft, assets, publicRead) => events.push(`verify:${publicRead}:${assets.length}`),
publish: async () => { events.push("publish"); return { html_url: "https://example.test/release" }; },
};
const prepare = async () => {
if (failing) throw new Error("frontend build unavailable");
return [{ name: "bundle.tar.gz" }, { name: "SHA256SUMS" }];
};
await assert.rejects(publishVerifiedRelease({ hosting, prepare }), /frontend/);
assert.deepEqual(events, []);
failing = false;
await publishVerifiedRelease({ hosting, prepare });
assert.deepEqual(events, ["upload:bundle.tar.gz", "upload:SHA256SUMS", "verify:false:2", "publish", "verify:true:2"]);
});
test("a published version is verified without replacing any asset", async () => {
const events = [];
await publishVerifiedRelease({ prepare: async () => [{ name: "bundle.tar.gz" }], hosting: {
open: async () => ({ id: 7, draft: false }),
upload: async () => { throw new Error("overwrote a published version"); },
publish: async () => { throw new Error("republished a version"); },
verify: async (_release, _assets, publicRead) => events.push(publicRead),
} });
assert.deepEqual(events, [true]);
});
+86
View File
@@ -0,0 +1,86 @@
import { readFileSync } from "node:fs";
import { homedir } from "node:os";
import { join } from "node:path";
import { sha256 } from "./release-bundle.mjs";
const accept = "application/vnd.oci.image.index.v1+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.docker.distribution.manifest.v2+json";
export async function boundedFetch(url, options = {}, timeout = 30_000) {
try { return await fetch(url, { ...options, redirect: options.redirect ?? "error", signal: AbortSignal.timeout(timeout) }); }
catch { throw new Error("Release network request failed; retry with the same output directory."); }
}
export async function readLimited(response, maximum) {
const chunks = []; let size = 0;
for await (const chunk of response.body) { size += chunk.length; if (size > maximum) throw new Error("Release response exceeded its bound."); chunks.push(chunk); }
return Buffer.concat(chunks);
}
async function jsonResponse(response, description) {
if (!response.ok) throw new Error(`${description} refused (HTTP ${response.status}).`);
const bytes = await readLimited(response, 4 * 2 ** 20);
try { return { bytes, value: JSON.parse(bytes.toString()) }; }
catch { throw new Error("Release service returned invalid metadata."); }
}
export async function dockerCredentials(run) {
const config = JSON.parse(readFileSync(join(process.env.DOCKER_CONFIG || join(homedir(), ".docker"), "config.json"), "utf8"));
const server = "https://index.docker.io/v1/";
const helper = config.credHelpers?.[server] || config.credsStore;
if (!helper || !/^[A-Za-z0-9._-]+$/.test(helper)) throw new Error("Use docker login with an OS credential store before publishing.");
const auth = JSON.parse(await run(`docker-credential-${helper}`, ["get"], { input: server + "\n", quiet: true }));
if (!auth.Username || !auth.Secret) throw new Error("Docker Hub login is unavailable.");
return auth;
}
export function dockerHub(auth) {
async function token(repository, anonymous) {
const url = new URL("https://auth.docker.io/token");
url.searchParams.set("service", "registry.docker.io");
url.searchParams.set("scope", `repository:${repository}:pull`);
const response = await boundedFetch(url, { headers: anonymous ? {} : { Authorization: `Basic ${Buffer.from(`${auth.Username}:${auth.Secret}`).toString("base64")}` } });
return (await jsonResponse(response, "Registry authentication")).value.token;
}
async function registryJSON(repository, route, anonymous, allowMissing = false) {
const bearer = await token(repository, anonymous);
let response = await boundedFetch(`https://registry-1.docker.io/v2/${repository}/${route}`, { redirect: "manual", headers: { Accept: accept, Authorization: `Bearer ${bearer}` } });
if ([302, 307].includes(response.status) && route.startsWith("blobs/")) {
const target = new URL(response.headers.get("location"));
if (target.protocol !== "https:" || target.username || target.password) throw new Error("Invalid registry blob redirect.");
// Signed blob URLs are fetched without forwarding registry credentials.
response = await boundedFetch(target);
}
if (allowMissing && response.status === 404) return null;
const { bytes, value } = await jsonResponse(response, "Registry read");
const digest = `sha256:${sha256(bytes)}`;
const advertised = response.headers.get("docker-content-digest");
if (advertised && advertised !== digest) throw new Error("Registry content digest mismatch.");
return { value, digest };
}
return {
async ensurePublic(namespace, name) {
const response = await boundedFetch("https://hub.docker.com/v2/auth/token", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ identifier: auth.Username, secret: auth.Secret }) });
const bearer = (await jsonResponse(response, "Docker Hub authentication")).value.access_token;
const headers = { Authorization: `Bearer ${bearer}`, "Content-Type": "application/json" };
const path = `https://hub.docker.com/v2/namespaces/${namespace}/repositories`;
let existing = await boundedFetch(`${path}/${name}`, { headers });
if (existing.status === 404) {
existing = await boundedFetch(path, { method: "POST", headers, body: JSON.stringify({ namespace, name, registry: "docker.io", is_private: false, description: `ThothII ${name.endsWith("core") ? "core with embedded Pi" : "standalone frontend"}` }) });
}
const data = (await jsonResponse(existing, "Public repository preparation")).value;
if (data.is_private !== false) throw new Error("Selected Docker Hub repository is private; make this release repository public before retrying.");
},
async inspect(repository, reference, platform, { anonymous = false, allowMissing = false } = {}) {
let image = await registryJSON(repository, `manifests/${reference}`, anonymous, allowMissing);
if (!image) return null;
if (reference.startsWith("sha256:") && image.digest !== reference) throw new Error("Requested image digest does not match registry content.");
if (image.value.manifests) {
const match = image.value.manifests.find((entry) => `${entry.platform?.os}/${entry.platform?.architecture}` === platform);
if (!match || !/^sha256:[a-f0-9]{64}$/.test(match.digest)) throw new Error("Image does not contain the requested platform.");
image = await registryJSON(repository, `manifests/${match.digest}`, anonymous);
if (image.digest !== match.digest) throw new Error("Image index digest mismatch.");
}
if (reference.startsWith("sha256:") && !image.value.config) throw new Error("Image metadata is incomplete.");
const configDigest = image.value.config?.digest;
if (!/^sha256:[a-f0-9]{64}$/.test(configDigest ?? "")) throw new Error("Image configuration digest is invalid.");
const config = await registryJSON(repository, `blobs/${configDigest}`, anonymous);
if (config.digest !== configDigest || `${config.value.os}/${config.value.architecture}` !== platform) throw new Error("Image configuration or platform mismatch.");
return { reference: `docker.io/${repository}@${image.digest}`, labels: config.value.config?.Labels ?? {} };
},
};
}
+28
View File
@@ -0,0 +1,28 @@
import { test } from "node:test";
import assert from "node:assert/strict";
import { dockerHub } from "./release-registry.mjs";
import { sha256 } from "./release-bundle.mjs";
test("registry verifies pinned config/platform and does not forward auth to blob storage", async () => {
const original = globalThis.fetch;
const config = JSON.stringify({ os: "linux", architecture: "amd64", config: { Labels: { "org.opencontainers.image.revision": "b".repeat(40) } } });
const configDigest = "sha256:" + sha256(config);
const manifest = JSON.stringify({ schemaVersion: 2, config: { digest: configDigest } });
const imageDigest = "sha256:" + sha256(manifest);
const auth = [];
globalThis.fetch = async (target, options) => {
const url = new URL(target);
if (url.hostname === "auth.docker.io") return new Response(JSON.stringify({ token: "registry-token" }));
if (url.hostname === "blob.example.test") { auth.push(options.headers?.Authorization); return new Response(config); }
if (url.pathname.includes("/blobs/")) return new Response(null, { status: 307, headers: { location: "https://blob.example.test/config" } });
return new Response(manifest, { headers: { "docker-content-digest": imageDigest } });
};
try {
const registry = dockerHub({ Username: "publisher", Secret: "PRIVATE_TOKEN" });
const image = await registry.inspect("example/core", imageDigest, "linux/amd64", { anonymous: true });
assert.equal(image.reference, `docker.io/example/core@${imageDigest}`);
assert.deepEqual(auth, [undefined]);
await assert.rejects(registry.inspect("example/core", imageDigest, "linux/arm64"), /platform mismatch/);
await assert.rejects(registry.inspect("example/core", "sha256:" + "0".repeat(64), "linux/amd64"), /Requested image digest/);
} finally { globalThis.fetch = original; }
});
+21
View File
@@ -0,0 +1,21 @@
import { readFileSync, lstatSync } from "node:fs";
import { parseAllDocuments } from "yaml";
import { decode, DocumentError, runWorkspaceDocuments } from "../workspaces/documents.js";
import { validateDatabaseBootstrap } from "./bootstrap-documents.js";
/** Internal sibling protocol: only references cross back to Go, never secret contents. */
export function runBootstrapValidation(args: string[]): { status: number; output: string } {
try {
if (args.length !== 5 || args[0] !== "--directory" || args[2] !== "--bootstrap" || args[4] !== "--json") throw new Error("usage");
const checked = runWorkspaceDocuments(["validate", "--directory", args[1], "--json"]);
if (checked.status !== 0) return checked;
const info = lstatSync(args[3]);
if (!info.isFile() || info.isSymbolicLink() || info.size > 1024 * 1024) throw new Error("file");
const validated = decode(readFileSync(args[3], "utf8"), "database-bootstrap.yaml", (source) =>
validateDatabaseBootstrap(parseAllDocuments(source)[0].toJSON(), args[1]), "database bootstrap schema v1 and the Catalog binding contract");
return { status: 0, output: JSON.stringify({ schema_version: 1, ok: true, secret_files: validated.secretFiles, warnings: validated.warnings, issues: [] }) };
} catch (error) {
const issue = error instanceof DocumentError ? error.issue : { document: "database-bootstrap.yaml", field: "$", code: "bootstrap_invalid", correction: "Supply one complete Catalog database configuration per workspace in a readable local bootstrap document." };
return { status: 1, output: JSON.stringify({ schema_version: 1, ok: false, issues: [issue] }) };
}
}
@@ -0,0 +1,49 @@
import { readFileSync } from "node:fs";
import { join } from "node:path";
import { z } from "zod";
import { databaseConfigurationSchema } from "./configuration-schema.js";
import { parseWorkspaceCatalogYaml } from "../workspaces/catalog.js";
import { parseWorkspaceYaml } from "../workspaces/schema.js";
import { discoverWorkspaceSecretRequirements } from "../workspaces/secret-requirements.js";
const reference = z.string().min(1).max(4096);
const database = databaseConfigurationSchema.extend({
secretFiles: z.object({ password: reference.optional(), apiKey: reference.optional(), sshPrivateKey: reference.optional(), sshPrivateKeyPassphrase: reference.optional(), sshKnownHosts: reference.optional(), tlsCa: reference.optional() }).strict(),
evidenceSecretFiles: z.object({ "evidence.signed_urls": reference.optional(), "evidence.access_key": reference.optional(), "evidence.secret_key": reference.optional(), "evidence.session_token": reference.optional() }).strict().optional(),
});
const bootstrap = z.object({ schemaVersion: z.literal(1), databases: z.array(database).min(1).max(1000) }).strict();
export const parseDatabaseBootstrap = (value: unknown) => bootstrap.parse(value);
export interface BootstrapReference { field: string; path: string }
/** Offline bootstrap boundary: runtime Catalog owns the resulting bindings after import. */
export function validateDatabaseBootstrap(value: unknown, workspaceRoot: string): { secretFiles: BootstrapReference[]; warnings: string[] } {
const document = bootstrap.parse(value);
const catalog = parseWorkspaceCatalogYaml(readFileSync(join(workspaceRoot, "thoth-workspaces.yaml"), "utf8"));
const expected = new Set(catalog.workspaces.map((entry) => entry.id));
const seen = new Set<string>();
const secretFiles: BootstrapReference[] = [];
const warnings: string[] = [];
const issue = (path: (string | number)[], message: string): never => { throw new z.ZodError([{ code: "custom", path, message }]); };
document.databases.forEach((entry, index) => {
if (!expected.has(entry.workspaceId) || seen.has(entry.workspaceId)) issue(["databases", index, "workspaceId"], "Declare each catalog workspace exactly once.");
seen.add(entry.workspaceId);
const required = entry.binding.transport === "rest_api"
? entry.binding.restAuth === "none" ? [] : ["apiKey"] as const
: entry.binding.transport === "ssh_tunnel" ? ["password", "sshPrivateKey", "sshKnownHosts"] as const : ["password"] as const;
for (const name of required) {
if (!entry.secretFiles[name]) issue(["databases", index, "secretFiles", name], "Supply a protected file reference for this transport.");
}
if (entry.binding.transport === "ssh_tunnel") warnings.push(`databases.${index}:ssh_tunnel supports Catalog diagnostics, not NL-to-SQL sessions; choose direct or REST for practice.`);
for (const [name, path] of Object.entries(entry.secretFiles)) secretFiles.push({ field: `databases.${index}.secretFiles.${name}`, path });
const workspace = parseWorkspaceYaml(readFileSync(join(workspaceRoot, entry.workspaceId, "workspace.yaml"), "utf8"));
const requirements = discoverWorkspaceSecretRequirements(workspace, {});
for (const requirement of requirements.filter((item) => item.connector === "evidence" && item.required)) {
if (!(entry.evidenceSecretFiles as Record<string, string> | undefined)?.[requirement.id]) issue(["databases", index, "evidenceSecretFiles"], "Supply the configured Evidence authentication file references.");
}
for (const [name, path] of Object.entries(entry.evidenceSecretFiles ?? {})) secretFiles.push({ field: `databases.${index}.evidenceSecretFiles.${name}`, path });
});
if (seen.size !== expected.size) issue(["databases"], "Add a database binding for every catalog workspace.");
return { secretFiles, warnings };
}
@@ -0,0 +1,47 @@
import { readFileSync } from "node:fs";
import { join } from "node:path";
import { S3Client, ListObjectsV2Command } from "@aws-sdk/client-s3";
import { parseWorkspaceYaml } from "../workspaces/schema.js";
import { evidencePolicy } from "../workspaces/evidence/preprocessing.js";
import type { parseDatabaseBootstrap } from "./bootstrap-documents.js";
import { probePublicEvidenceUrl, publicEvidenceAgent } from "./evidence-probe-http.js";
type Entry = ReturnType<typeof parseDatabaseBootstrap>["databases"][number];
/** Read-only availability probes; domain correctness and materialization remain runtime gates. */
export async function probeEvidence(entry: Entry, root: string): Promise<void> {
const evidence = parseWorkspaceYaml(readFileSync(join(root, entry.workspaceId, "workspace.yaml"), "utf8")).evidence;
if (!evidence || evidence.source.type === "filesystem") return;
if (evidencePolicy(evidence)) throw new Error("Evidence egress policy refused");
const secret = (name: keyof NonNullable<Entry["evidenceSecretFiles"]>) => {
const path = entry.evidenceSecretFiles?.[name];
if (!path) throw new Error("Evidence credential missing");
return readFileSync(path, "utf8").trim();
};
const source = evidence.source;
if (source.type === "http") {
const urls: unknown = source.authentication === "signed_urls_file"
? JSON.parse(secret("evidence.signed_urls")) : source.uris;
if (!Array.isArray(urls) || urls.length !== source.uris.length || urls.length > 1000) throw new Error("Invalid signed URLs");
for (const [index, value] of urls.entries()) {
if (typeof value !== "string") throw new Error("Invalid signed URL");
const url = new URL(value);
const provenance = new URL(source.uris[index]);
// Signed queries may authorize the same identity, never a different host/path.
if (url.origin !== provenance.origin || url.pathname !== provenance.pathname || url.username || url.password || url.hash) throw new Error("Invalid signed URL identity");
await probePublicEvidenceUrl(url);
}
return;
}
// The shared runtime policy currently permits trusted AWS endpoints with explicit file credentials.
const location = new URL(source.uri);
const client = new S3Client({
region: source.region ?? "us-east-1", maxAttempts: 1,
requestHandler: { httpsAgent: publicEvidenceAgent(), connectionTimeout: 5_000, requestTimeout: 5_000 },
credentials: { accessKeyId: secret("evidence.access_key"), secretAccessKey: secret("evidence.secret_key"),
...(entry.evidenceSecretFiles?.["evidence.session_token"] ? { sessionToken: secret("evidence.session_token") } : {}) },
});
try {
await client.send(new ListObjectsV2Command({ Bucket: location.hostname, Prefix: decodeURIComponent(location.pathname.slice(1)), MaxKeys: 1 }), { abortSignal: AbortSignal.timeout(5_000) });
} finally { client.destroy(); }
}
+53
View File
@@ -0,0 +1,53 @@
import { readFileSync } from "node:fs";
import { parse } from "yaml";
import { parseDatabaseBootstrap } from "./bootstrap-documents.js";
import { runBootstrapValidation } from "./bootstrap-cli.js";
import { createConcreteDiagnosticAdapters, type DiagnosticAdapters } from "../workspaces/diagnostics.js";
import { probeEvidence } from "./bootstrap-evidence-probes.js";
interface ProbeCheck { id: string; outcome: "passed" | "error"; field: string; action: string }
/** Uses the same read-only, authenticated connector diagnostics as the Catalog. */
export async function probeBootstrapDependencies(value: unknown, adapters: DiagnosticAdapters = createConcreteDiagnosticAdapters(), workspaceRoot?: string) {
const document = parseDatabaseBootstrap(value);
const checks: ProbeCheck[] = [];
for (const [index, entry] of document.databases.entries()) {
let outcome: ProbeCheck["outcome"] = "passed";
const controller = new AbortController();
const timer = setTimeout(() => controller.abort(), 5_000);
try {
if (entry.binding.transport === "ssh_tunnel") throw new Error("session transport unavailable");
await adapters.probeConnector({
role: "dwh", transport: entry.binding.transport,
host: entry.binding.host, port: entry.binding.port, user: entry.binding.username,
baseUrl: entry.binding.baseUrl,
credentialFile: entry.binding.transport === "rest_api" ? entry.secretFiles.apiKey : entry.secretFiles.password,
tlsCaFile: entry.secretFiles.tlsCa, tlsServername: entry.binding.tlsServername,
resource: { database: entry.databaseName, schema: entry.schema },
timeoutMs: 5_000, signal: controller.signal,
diagnostic: { method: "GET", path: entry.binding.restPath ?? "/health", auth: entry.binding.restAuth ?? "bearer" },
});
} catch { outcome = "error"; } finally { clearTimeout(timer); }
checks.push({ id: `database-${index}`, outcome, field: `database-bootstrap.databases.${index}`,
action: entry.binding.transport === "ssh_tunnel"
? "Choose postgres_direct or rest_api for NL-to-SQL practice; SSH diagnostics alone cannot establish session readiness."
: "Require an authenticated read-only connection and access to the configured database/schema; correct endpoint, permissions or protected credentials." });
if (workspaceRoot) {
let evidenceOutcome: ProbeCheck["outcome"] = "passed";
try { await probeEvidence(entry, workspaceRoot); } catch { evidenceOutcome = "error"; }
checks.push({ id: `evidence-${index}`, outcome: evidenceOutcome, field: `workspaces.${index}.evidence`, action: "Require readable local Evidence or authenticated bounded HTTP/S3 access under the canonical egress policy; domain meaning is verified during practice." });
}
}
return { schema_version: 1, ok: checks.every((check) => check.outcome === "passed"), checks };
}
export async function runBootstrapProbes(args: string[]) {
const validation = runBootstrapValidation(args);
if (validation.status !== 0) return validation;
try {
const report = await probeBootstrapDependencies(parse(readFileSync(args[3], "utf8")), undefined, args[1]);
return { status: report.ok ? 0 : 1, output: JSON.stringify(report) };
} catch {
return { status: 1, output: JSON.stringify({ schema_version: 1, ok: false, checks: [{ id: "database-probes", outcome: "error", field: "database-bootstrap", action: "Revalidate prepared documents and protected credential references." }] }) };
}
}
@@ -0,0 +1,44 @@
import { z } from "zod";
import { parseCredentialFreeHttpUrl } from "../auth/url-policy.js";
import { DATABASE_TRANSPORTS } from "./types.js";
const workspaceIdSchema = z.string().regex(/^[a-z][a-z0-9-]{2,62}$/);
const identifier = z.string().trim().min(1).max(128).regex(/^[A-Za-z_][A-Za-z0-9_$-]*$/);
const nonEmpty = z.string().trim().min(1).max(512);
const port = z.number().int().min(1).max(65_535);
const optionalText = nonEmpty.optional();
const sshHost = z.string().trim().min(1).max(255).regex(/^[A-Za-z0-9_.:\[\]-]+$/).optional();
const sshUsername = z.string().trim().min(1).max(128).regex(/^[A-Za-z0-9._-]+$/).optional();
const bindingSchema = z.object({
transport: z.enum(DATABASE_TRANSPORTS),
host: optionalText,
port: port.optional(),
username: optionalText,
baseUrl: z.string().max(2048)
.refine((value) => parseCredentialFreeHttpUrl(value) !== undefined)
.optional(),
restPath: z.string().regex(/^\/(?!\/)[^?#\\\u0000-\u001f]*$/).max(512).optional(),
restAuth: z.enum(["none", "bearer", "x-api-key"]).optional(),
tlsServername: optionalText,
sshHost,
sshPort: port.optional(),
sshUsername,
sshTargetHost: sshHost,
sshTargetPort: port.optional(),
}).strict().superRefine((binding, context) => {
const required = binding.transport === "postgres_direct"
? ["host", "port", "username"] as const
: binding.transport === "rest_api"
? ["baseUrl", "restPath", "restAuth"] as const
: ["username", "sshHost", "sshPort", "sshUsername", "sshTargetHost", "sshTargetPort"] as const;
for (const field of required) {
if (binding[field] === undefined) context.addIssue({ code: "custom", path: [field], message: "Required" });
}
});
export const databaseConfigurationSchema = z.object({
workspaceId: workspaceIdSchema,
engine: z.literal("postgres"),
databaseName: identifier,
schema: identifier,
binding: bindingSchema,
}).strict();
@@ -0,0 +1,57 @@
import { lookup } from "node:dns/promises";
import { request as httpRequest } from "node:http";
import { request as httpsRequest, Agent } from "node:https";
import type { LookupFunction } from "node:net";
import ipaddr from "ipaddr.js";
const refused = () => new Error("Evidence network policy refused");
function normalizedPublicAddress(value: string): string {
const address = ipaddr.process(value);
if (address.range() !== "unicast") throw refused();
return address.toString();
}
/** Reject the entire DNS answer set, then pin the connection to that verified set. */
export async function resolvePublicEvidenceHost(hostname: string) {
const values = await lookup(hostname.replace(/^\[|\]$/g, ""), { all: true });
if (!values.length) throw refused();
values.forEach((value) => normalizedPublicAddress(value.address));
return values;
}
const publicLookup: LookupFunction = (hostname, options, callback) => {
void resolvePublicEvidenceHost(hostname).then((values) => {
if (options.all) callback(null, values);
else callback(null, values[0].address, values[0].family);
}, () => callback(refused(), "", 0));
};
// Node's direct agent does not inherit HTTP proxy environment or ambient credentials.
export const publicEvidenceAgent = () => new Agent({ lookup: publicLookup });
export async function probePublicEvidenceUrl(url: URL): Promise<void> {
if (!['http:', 'https:'].includes(url.protocol)) throw refused();
const signal = AbortSignal.timeout(5_000);
const values = await Promise.race([
resolvePublicEvidenceHost(url.hostname),
new Promise<never>((_, reject) => signal.addEventListener("abort", () => reject(refused()), { once: true })),
]);
signal.throwIfAborted();
const allowed = new Set(values.map((value) => normalizedPublicAddress(value.address)));
const pinned: LookupFunction = (_hostname, options, callback) => {
if (options.all) callback(null, values);
else callback(null, values[0].address, values[0].family);
};
await new Promise<void>((resolve, reject) => {
const request = (url.protocol === "https:" ? httpsRequest : httpRequest)(url, {
method: "GET", lookup: pinned, signal, agent: false,
}, (response) => {
try {
const peer = response.socket.remoteAddress;
if (!peer || !allowed.has(normalizedPublicAddress(peer)) || !response.statusCode || response.statusCode < 200 || response.statusCode >= 300) throw refused();
resolve();
} catch { reject(refused()); } finally { response.destroy(); }
});
request.on("error", () => reject(refused()));
request.end();
});
}
+1 -42
View File
@@ -1,60 +1,19 @@
import type { FastifyInstance, FastifyReply, FastifyRequest } from "fastify";
import { z } from "zod";
import { isPrincipalContext, requirePermission } from "../auth/authorization.js";
import { parseCredentialFreeHttpUrl } from "../auth/url-policy.js";
import { databaseConfigurationSchema as configSchema } from "../catalog/configuration-schema.js";
import { CatalogService, type CatalogSecretName } from "../catalog/service.js";
import { WorkspaceRegistryError } from "../workspaces/git-repository.js";
import {
CatalogConflictError,
CatalogOperationInProgressError,
CatalogUnavailableError,
DATABASE_TRANSPORTS,
type CatalogRepository,
type DatabaseConfigurationInput,
} from "../catalog/types.js";
import type { CatalogOperationCoordinator } from "../catalog/operation-coordinator.js";
const idSchema = z.uuid();
const workspaceIdSchema = z.string().regex(/^[a-z][a-z0-9-]{2,62}$/);
const identifier = z.string().trim().min(1).max(128).regex(/^[A-Za-z_][A-Za-z0-9_$-]*$/);
const nonEmpty = z.string().trim().min(1).max(512);
const port = z.number().int().min(1).max(65_535);
const optionalText = nonEmpty.optional();
const sshHost = z.string().trim().min(1).max(255).regex(/^[A-Za-z0-9_.:\[\]-]+$/).optional();
const sshUsername = z.string().trim().min(1).max(128).regex(/^[A-Za-z0-9._-]+$/).optional();
const bindingSchema = z.object({
transport: z.enum(DATABASE_TRANSPORTS),
host: optionalText,
port: port.optional(),
username: optionalText,
baseUrl: z.string().max(2048)
.refine((value) => parseCredentialFreeHttpUrl(value) !== undefined)
.optional(),
restPath: z.string().regex(/^\/(?!\/)[^?#\\\u0000-\u001f]*$/).max(512).optional(),
restAuth: z.enum(["none", "bearer", "x-api-key"]).optional(),
tlsServername: optionalText,
sshHost,
sshPort: port.optional(),
sshUsername,
sshTargetHost: sshHost,
sshTargetPort: port.optional(),
}).strict().superRefine((binding, context) => {
const required = binding.transport === "postgres_direct"
? ["host", "port", "username"] as const
: binding.transport === "rest_api"
? ["baseUrl", "restPath", "restAuth"] as const
: ["username", "sshHost", "sshPort", "sshUsername", "sshTargetHost", "sshTargetPort"] as const;
for (const field of required) {
if (binding[field] === undefined) context.addIssue({ code: "custom", path: [field], message: "Required" });
}
});
const configSchema = z.object({
workspaceId: workspaceIdSchema,
engine: z.literal("postgres"),
databaseName: identifier,
schema: identifier,
binding: bindingSchema,
}).strict();
const updateSchema = configSchema.extend({ version: z.number().int().positive() });
const secretNames = [
"password",
+10
View File
@@ -0,0 +1,10 @@
/** Compiled with its runtime for the host CLI: no installation, Docker or host Node required. */
import { runWorkspaceDocuments } from "./workspaces/documents.js";
import { runBootstrapValidation } from "./catalog/bootstrap-cli.js";
import { runBootstrapProbes } from "./catalog/bootstrap-probes.js";
const args = process.argv.slice(2);
const result = args[0] === "probe" ? await runBootstrapProbes(args.slice(1))
: args[0] === "bootstrap" ? runBootstrapValidation(args.slice(1)) : runWorkspaceDocuments(args);
console.log(result.output);
process.exitCode = result.status;
+3 -3
View File
@@ -44,8 +44,8 @@ const catalogSchema = z.object({
});
});
function safeCatalogError(): Error {
return new Error("Workspace catalog is invalid");
function safeCatalogError(cause?: unknown): Error {
return new Error("Workspace catalog is invalid", { cause });
}
export function parseWorkspaceCatalogYaml(source: string): WorkspaceCatalog {
@@ -57,7 +57,7 @@ export function parseWorkspaceCatalogYaml(source: string): WorkspaceCatalog {
return catalogSchema.parse(document.toJSON()) as WorkspaceCatalog;
} catch (error) {
if (error instanceof Error && error.message === "Workspace catalog is invalid") throw error;
throw safeCatalogError();
throw safeCatalogError(error);
}
}
+222
View File
@@ -0,0 +1,222 @@
import { closeSync, lstatSync, mkdirSync, openSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { join, resolve } from "node:path";
import { parseAllDocuments, stringify } from "yaml";
import { ZodError } from "zod";
import { CATALOG_PATH, parseWorkspaceCatalogYaml, assertCatalogMatchesDescriptor } from "./catalog.js";
import { parseWorkspaceYaml, type WorkspaceDescriptor } from "./schema.js";
interface Issue { document: string; field: string; code: string; correction: string; line?: number }
interface Report {
schema_version: 1;
scope: "local-documents";
ok: boolean;
workspaces: { id: string; evidence: "absent" | "local-files" | "remote-deferred" }[];
issues: Issue[];
deferred_checks: string[];
}
const MAX_DOCUMENT_BYTES = 1024 * 1024;
const usage = "tht workspace prepare --directory NEW_PATH --id ID --name NAME [--language en|it] [--json]\ntht workspace validate --directory PATH [--json]";
export class DocumentError extends Error {
constructor(readonly issue: Issue) { super(issue.correction); }
}
function fail(document: string, field: string, code: string, correction: string): never {
throw new DocumentError({ document, field, code, correction });
}
/** Never include parser messages or submitted values: YAML and Zod errors can contain secrets. */
export function decode<T>(source: string, document: string, parser: (text: string) => T, contract = "workspace schema v4 or catalog schema v1"): T {
try {
const documents = parseAllDocuments(source, { uniqueKeys: true });
if (documents.length !== 1) fail(document, "$", "yaml_documents", "Keep exactly one YAML document in this file.");
const problem = [...documents[0].errors, ...documents[0].warnings][0];
if (problem) {
throw new DocumentError({ document, field: "$", code: "yaml_syntax", line: problem.linePos?.[0].line,
correction: "Correct YAML syntax, remove duplicate keys and unsupported tags at the indicated line." });
}
return parser(source);
} catch (error) {
if (error instanceof DocumentError) throw error;
const cause = error instanceof Error && error.cause instanceof ZodError ? error.cause : error;
if (cause instanceof ZodError) {
const issue = cause.issues[0];
// Strict schemas produce paths containing schema-defined keys and array indices only.
fail(document, issue.path.join(".") || "$", "schema_invalid",
issue.code === "unrecognized_keys" ? "Remove fields not defined by the current workspace/catalog contract."
: `Correct this field using ${contract}; check type, required value, uniqueness and allowed values.`);
}
fail(document, "$", "schema_invalid", "Use an authored workspace v4 descriptor; remove database configuration and keep it in the PostgreSQL Metadata Catalog.");
}
}
function stat(root: string, document: string, directory: boolean) {
let info;
try { info = lstatSync(join(root, document)); }
catch { fail(document, "$", "missing_reference", "Create the referenced local file or directory and grant read access."); }
if (info.isSymbolicLink() || (directory ? !info.isDirectory() : !info.isFile())) {
fail(document, "$", "unsafe_reference", "Use a regular local file or directory, without symbolic links or special files.");
}
return info;
}
function readDocument(root: string, document: string): string {
if (stat(root, document, false).size > MAX_DOCUMENT_BYTES) {
fail(document, "$", "document_too_large", "Keep YAML documents below the 1 MiB local validation limit.");
}
try { return readFileSync(join(root, document), "utf8"); }
catch { fail(document, "$", "unreadable", "Grant read access to this document and retry validation."); }
}
function directories(root: string, document: string) {
stat(root, document, true);
try { return readdirSync(join(root, document), { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name)); }
catch { fail(document, "$", "unreadable", "Grant read and traversal access to this directory and retry validation."); }
}
function inspectEvidence(root: string, descriptor: WorkspaceDescriptor): "absent" | "local-files" | "remote-deferred" {
const evidence = descriptor.evidence;
if (!evidence) return "absent";
if (evidence.source.type !== "filesystem") return "remote-deferred";
const base = evidence.source.uri;
stat(root, base, true);
if (evidence.schema_version === 2) stat(root, `${base}/curated`, true);
const patterns = evidence.source.patterns ?? ["**/*.md"];
const literals = patterns.filter((pattern) => !/[?*\[]/.test(pattern));
for (const pattern of literals) {
const parts = pattern.split("/");
for (let index = 1; index < parts.length; index++) stat(root, `${base}/${parts.slice(0, index).join("/")}`, true);
stat(root, `${base}/${pattern}`, false);
}
// Inventory without following links; curated-unit interpretation remains a runtime check.
const pending = [base];
let count = 0;
while (pending.length) {
const current = pending.pop()!;
for (const entry of directories(root, current)) {
if (++count > 100_000) fail(base, "evidence.source", "inventory_limit", "Reduce the Evidence tree below 100,000 entries before local validation.");
const path = `${current}/${entry.name}`;
if (entry.isDirectory()) pending.push(path);
else {
const info = stat(root, path, false);
const relative = path.slice(base.length + 1);
// Only apply content checks to selections whose meaning is unambiguous locally.
// Arbitrary globs are expanded by the canonical Python adapter after startup.
const curated = evidence.schema_version === 2 && relative.startsWith("curated/") && relative.endsWith(".md");
const selected = curated || literals.includes(relative) || (patterns.includes("**/*.md") && relative.endsWith(".md"));
if (selected && info.size > evidence.source.max_bytes) fail(path, "evidence.source.max_bytes", "evidence_too_large", "Reduce the source file or increase the declared max_bytes limit deliberately.");
try { closeSync(openSync(join(root, path), "r")); }
catch { fail(path, "$", "unreadable", "Grant read access to this Evidence file and retry validation."); }
if (curated) {
const text = readDocument(root, path);
const frontmatter = text.match(/^---\r?\n([\s\S]*?)\r?\n---(?:\r?\n|$)/);
if (!frontmatter) fail(path, "$", "evidence_frontmatter", "Add a YAML frontmatter block delimited by --- to the curated Markdown unit; follow the Curated Evidence contract.");
decode(frontmatter[1], path, (source) => parseAllDocuments(source)[0].toJSON());
}
}
}
}
return "local-files";
}
function validate(root: string, report: Report): void {
directories(root, ".");
const catalog = decode(readDocument(root, CATALOG_PATH), CATALOG_PATH, parseWorkspaceCatalogYaml);
const ids = new Set(catalog.workspaces.map((entry) => entry.id));
for (const entry of directories(root, ".")) {
if (entry.name === ".git") continue;
if (entry.isSymbolicLink()) fail(".", "$", "unsafe_reference", "Replace repository-root symbolic links with regular files or directories.");
if (entry.isDirectory() && entry.name !== "workspace-docs" && !ids.has(entry.name)) {
fail(".", "workspaces", "unlisted_directory", "Every root directory except workspace-docs must match a catalog workspace id; remove or register the extra directory.");
}
if (entry.name === "workspace-docs") {
for (const docs of directories(root, "workspace-docs")) {
if (!ids.has(docs.name)) fail("workspace-docs", "$", "unlisted_documentation", "Keep documentation only for workspace ids listed in the catalog.");
for (const file of directories(root, `workspace-docs/${docs.name}`)) {
const path = `workspace-docs/${docs.name}/${file.name}`;
if (!["README.md", "contract.env.example"].includes(file.name)) fail(`workspace-docs/${docs.name}`, "$", "unsupported_documentation", "Keep only README.md and contract.env.example in workspace-docs/<id>. Place Evidence inside the workspace directory.");
stat(root, path, false);
}
}
}
}
for (const entry of catalog.workspaces) {
const document = `${entry.id}/workspace.yaml`;
try {
stat(root, entry.id, true);
const descriptor = decode(readDocument(root, document), document, parseWorkspaceYaml);
try { assertCatalogMatchesDescriptor(entry, descriptor); }
catch { fail(document, "workspace", "catalog_mismatch", "Make id, name and description identical in the catalog, descriptor and workspace directory name."); }
const evidence = inspectEvidence(root, descriptor);
report.workspaces.push({ id: entry.id, evidence });
if (evidence !== "absent") report.deferred_checks.push(`${entry.id}:evidence-source-selection`, `${entry.id}:evidence-content-provenance-and-indexing`);
if (evidence === "remote-deferred") report.deferred_checks.push(`${entry.id}:remote-evidence-access`);
} catch (error) {
if (error instanceof DocumentError) report.issues.push(error.issue);
else throw error;
}
}
}
function prepare(root: string, options: Map<string, string>): void {
const id = options.get("--id");
const name = options.get("--name");
const language = options.get("--language") ?? "en";
const catalog = stringify({ schema_version: 1, workspaces: [{ id, name }] });
const descriptor = stringify({ workspace: { schema_version: 4, id, name, language } });
decode(catalog, CATALOG_PATH, parseWorkspaceCatalogYaml);
decode(descriptor, "workspace.yaml", parseWorkspaceYaml);
try { mkdirSync(root); }
catch (error) {
if ((error as NodeJS.ErrnoException).code === "EEXIST") fail(".", "--directory", "destination_exists", "Choose a new directory; preparation never overwrites an existing directory or its documents.");
fail(".", "--directory", "destination_unavailable", "Create the parent directory and grant write access, then choose a new destination.");
}
try {
mkdirSync(join(root, id!));
mkdirSync(join(root, "workspace-docs", id!), { recursive: true });
writeFileSync(join(root, CATALOG_PATH), "# Index of workspace identities. Keep metadata identical to each descriptor.\n" + catalog, { flag: "wx" });
writeFileSync(join(root, id!, "workspace.yaml"), "# Authored workspace v4: optional Evidence; database binding belongs to the Metadata Catalog.\n" + descriptor, { flag: "wx" });
writeFileSync(join(root, "workspace-docs", id!, "README.md"),
"# Workspace documents / Documenti workspace\n\n" +
"EN: Edit thoth-workspaces.yaml and <id>/workspace.yaml together. Evidence is optional and absent by default. Add reviewed source material under <id>/evidence only when configured. Database connections and schema belong to the installation Metadata Catalog. No example databases are downloaded.\n\n" +
"IT: Modificare insieme thoth-workspaces.yaml e <id>/workspace.yaml. Le Evidence sono facoltative e inizialmente assenti. Inserire materiale verificato in <id>/evidence solo quando configurato. Connessioni e schema dei database appartengono al Metadata Catalog dell'installazione. Nessun database di esempio viene scaricato.\n\n" +
"Repeat / Ripetere: `tht workspace validate --directory <repository>`. Local success does not establish runtime readiness or semantic truth / Il successo locale non certifica readiness o verità semantica.\n", { flag: "wx" });
} catch {
rmSync(root, { recursive: true, force: true });
fail(".", "$", "prepare_failed", "Preparation could not write the documents; check disk space and permissions, then retry with a new directory.");
}
}
export function runWorkspaceDocuments(args: string[]): { status: number; output: string } {
const report: Report = { schema_version: 1, scope: "local-documents", ok: false, workspaces: [], issues: [], deferred_checks: ["catalog-database-binding", "database-connectivity", "runtime-preprocessing"] };
const json = args.includes("--json");
let status = 1;
try {
const [command, ...rest] = args;
if ((command === "prepare" || command === "validate") && rest.length === 1 && rest[0] === "--help") return { status: 0, output: usage };
const allowed = command === "prepare" ? ["--directory", "--id", "--name", "--language"] : command === "validate" ? ["--directory"] : [];
const options = new Map<string, string>();
const seen = new Set<string>();
for (let index = 0; index < rest.length; index++) {
const key = rest[index];
if (seen.has(key)) fail("CLI", "$", "usage", usage);
seen.add(key);
if (key === "--json") continue;
if (!allowed.includes(key) || !rest[index + 1] || rest[index + 1].startsWith("--")) fail("CLI", "$", "usage", usage);
options.set(key, rest[++index]);
}
if (!allowed.length || !options.has("--directory") || (command === "prepare" && (!options.has("--id") || !options.has("--name")))) fail("CLI", "$", "usage", usage);
const root = resolve(options.get("--directory")!);
if (command === "prepare") prepare(root, options);
validate(root, report);
report.ok = report.issues.length === 0;
status = report.ok ? 0 : 1;
} catch (error) {
report.issues.push(error instanceof DocumentError ? error.issue : { document: ".", field: "$", code: "io_error", correction: "Check local permissions and regular files, then retry; no services were started." });
if (report.issues[0].code === "usage") status = 2;
}
const output = json ? JSON.stringify(report) : [
report.ok ? "Workspace documents pass local validation." : "Workspace documents require corrections.",
...report.issues.map((issue) => `${issue.document}${issue.line ? `:${issue.line}` : ""} [${issue.field}] ${issue.code}: ${issue.correction}`),
...report.workspaces.map((entry) => `${entry.id}: Evidence ${entry.evidence}`),
`Deferred until runtime: ${report.deferred_checks.join(", ")}.`,
].join("\n");
return { status, output };
}
@@ -0,0 +1,49 @@
import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from "node:fs";
import { join } from "node:path";
import { tmpdir } from "node:os";
import { afterEach, expect, test } from "vitest";
import { validateDatabaseBootstrap } from "../src/catalog/bootstrap-documents.js";
import { runBootstrapValidation } from "../src/catalog/bootstrap-cli.js";
const roots: string[] = [];
afterEach(() => roots.splice(0).forEach((root) => rmSync(root, { recursive: true, force: true })));
function fixture() {
const root = mkdtempSync(join(tmpdir(), "bootstrap-documents-")); roots.push(root);
mkdirSync(join(root, "practice"));
writeFileSync(join(root, "thoth-workspaces.yaml"), "schema_version: 1\nworkspaces: [{id: practice, name: Practice}]\n");
writeFileSync(join(root, "practice/workspace.yaml"), "workspace: {schema_version: 4, id: practice, name: Practice, language: en}\n");
return root;
}
const entry = { workspaceId: "practice", engine: "postgres", databaseName: "practice", schema: "public", binding: { transport: "postgres_direct", host: "db.internal", port: 5432, username: "reader" }, secretFiles: { password: "/private/operator/db-password" } };
test("bootstrap reuses Catalog configuration and requires one complete binding per workspace", () => {
const root = fixture();
const valid = validateDatabaseBootstrap({ schemaVersion: 1, databases: [entry] }, root);
expect(valid.secretFiles).toContainEqual({ field: "databases.0.secretFiles.password", path: "/private/operator/db-password" });
for (const databases of [[], [entry, entry], [{ ...entry, workspaceId: "unknown" }], [{ ...entry, secretFiles: {} }], [{ ...entry, engine: "mysql" }], [{ ...entry, binding: { ...entry.binding, port: 70000 } }]]) {
expect(() => validateDatabaseBootstrap({ schemaVersion: 1, databases }, root)).toThrow();
}
});
test("REST authentication, SSH credentials and optional Evidence use their declared contracts", () => {
const root = fixture();
const rest = { ...entry, binding: { transport: "rest_api", baseUrl: "https://data.internal", restPath: "/query", restAuth: "none" }, secretFiles: {} };
expect(validateDatabaseBootstrap({ schemaVersion: 1, databases: [rest] }, root).secretFiles).toEqual([]);
expect(() => validateDatabaseBootstrap({ schemaVersion: 1, databases: [{ ...rest, binding: { ...rest.binding, restAuth: "bearer" } }] }, root)).toThrow();
const ssh = { ...entry, binding: { transport: "ssh_tunnel", username: "reader", sshHost: "bastion", sshPort: 22, sshUsername: "tunnel", sshTargetHost: "database", sshTargetPort: 5432 }, secretFiles: { password: "/password", sshPrivateKey: "/key", sshKnownHosts: "/hosts" } };
expect(validateDatabaseBootstrap({ schemaVersion: 1, databases: [ssh] }, root).warnings[0]).toContain("not NL-to-SQL");
writeFileSync(join(root, "practice/workspace.yaml"), "workspace: {schema_version: 4, id: practice, name: Practice, language: en}\nevidence:\n source:\n type: http\n uris: [https://docs.internal/manual.md]\n authentication: signed_urls_file\n");
expect(() => validateDatabaseBootstrap({ schemaVersion: 1, databases: [entry] }, root)).toThrow();
expect(validateDatabaseBootstrap({ schemaVersion: 1, databases: [{ ...entry, evidenceSecretFiles: { "evidence.signed_urls": "/urls.json" } }] }, root).secretFiles).toContainEqual({ field: "databases.0.evidenceSecretFiles.evidence.signed_urls", path: "/urls.json" });
});
test("bootstrap CLI never echoes arbitrary keys from submitted secret references", () => {
const root = fixture();
const path = join(root, "bootstrap.yaml");
for (const field of ["secretFiles", "evidenceSecretFiles"]) {
writeFileSync(path, JSON.stringify({ schemaVersion: 1, databases: [{ ...entry, [field]: { PRIVATE_CREDENTIAL_SENTINEL: "/path" } }] }));
const result = runBootstrapValidation(["--directory", root, "--bootstrap", path, "--json"]);
expect(result.status).toBe(1);
expect(result.output).not.toContain("PRIVATE_CREDENTIAL_SENTINEL");
}
});
@@ -0,0 +1,67 @@
import { spawnSync } from "node:child_process";
import { chmodSync, mkdtempSync, readFileSync, realpathSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { rootCertificates } from "node:tls";
import { afterEach, expect, test } from "vitest";
const binary = process.env.THT_INSTALLATION_TEST_CLI;
const roots: string[] = [];
afterEach(() => roots.splice(0).forEach((path) => rmSync(path, { recursive: true, force: true })));
function run(...args: string[]) {
return spawnSync(binary!, [...args, "--json"], { encoding: "utf8", env: { ...process.env, PATH: "" }, input: "" });
}
test.skipIf(!binary)("operator prepares, completes and repeatedly validates before any stack exists", () => {
const root = realpathSync(mkdtempSync(join(tmpdir(), "application-documents-"))); roots.push(root);
const workspace = join(root, "workspaces"), directory = join(root, "installation");
expect(run("workspace", "prepare", "--directory", workspace, "--id", "practice", "--name", "Practice").status).toBe(0);
expect(run("installation", "prepare", "--directory", directory).status).toBe(0);
const installation = join(directory, "thothii-installation.yaml");
const validate = () => run("--installation", installation, "installation", "validate", "--workspaces", workspace);
expect(validate().status).toBe(1); // Visible placeholders cannot be approved.
expect(run("installation", "credentials", "--directory", directory).status).toBe(0);
for (const name of ["thothii-installation.yaml", "operator.env", "database-bootstrap.yaml"]) {
const path = join(directory, name);
writeFileSync(path, readFileSync(path, "utf8").replaceAll("CHANGE_ME", "practice"));
}
for (const [name, contents] of Object.entries({ "secrets.env": "OPENAI_API_KEY=PRIVATE_PROVIDER_VALUE\n", "database-password": "PRIVATE_DATABASE_VALUE", "git-credentials": "", "git-ca.pem": rootCertificates[0] })) {
writeFileSync(join(directory, "secrets", name), contents, { mode: 0o600 });
}
const before = readFileSync(installation, "utf8");
for (let index = 0; index < 2; index++) {
const checked = validate();
expect(checked.stderr).toBe("");
expect(checked.stdout).not.toContain("PRIVATE_");
expect(checked.status, checked.stdout).toBe(0);
expect(JSON.parse(checked.stdout).deferred_checks).toContain("release-assets");
}
expect(readFileSync(installation, "utf8")).toBe(before);
writeFileSync(installation, before.replace("interaction: openai/gpt-4.1-mini", "interaction: openai/nonexistent"));
expect(validate().status).toBe(1);
writeFileSync(installation, before);
const authFile = join(directory, "secrets/pi-auth.json");
writeFileSync(authFile, "not-json");
expect(validate().status).toBe(1);
writeFileSync(installation, before.replace("{mode: secret_env, apiKeyEnv: OPENAI_API_KEY}", "{mode: pi_auth}"));
writeFileSync(authFile, JSON.stringify({ openai: { unrelated: true } }));
expect(validate().status).toBe(1);
writeFileSync(authFile, JSON.stringify({ " OpenAI ": { type: "api_key", key: "PRIVATE_PI_KEY" } }));
expect(validate().status).toBe(1);
writeFileSync(authFile, JSON.stringify({ openai: { type: "api_key", key: "PRIVATE_PI_KEY" } }));
expect(validate().status).toBe(0);
writeFileSync(installation, before);
writeFileSync(authFile, "{}");
const bootstrap = join(directory, "database-bootstrap.yaml");
const originalBootstrap = readFileSync(bootstrap, "utf8");
writeFileSync(join(workspace, "practice", "private-password"), "PRIVATE_DATABASE_VALUE", { mode: 0o600 });
writeFileSync(bootstrap, originalBootstrap.replace(join(directory, "secrets/database-password"), join(workspace, "practice/private-password")));
expect(validate().status).toBe(1);
writeFileSync(bootstrap, originalBootstrap);
chmodSync(join(directory, "secrets/database-password"), 0o644);
expect(validate().status).toBe(1);
chmodSync(join(directory, "secrets/database-password"), 0o600);
const env = join(directory, "operator.env");
writeFileSync(env, readFileSync(env, "utf8") + "THOTH_HTTP_PORT=8081\n");
expect(validate().status).toBe(1);
}, 15_000);
@@ -0,0 +1,43 @@
import { createServer } from "node:http";
import { mkdtempSync, writeFileSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { expect, it } from "vitest";
import { probeBootstrapDependencies } from "../src/catalog/bootstrap-probes.js";
import { probePublicEvidenceUrl } from "../src/catalog/evidence-probe-http.js";
it("refuses loopback literals and DNS answers before sending an Evidence GET", async () => {
for (const hostname of ["[::1]", "127.0.0.1", "localhost", "[::ffff:127.0.0.1]"]) {
await expect(probePublicEvidenceUrl(new URL(`http://${hostname}/private`))).rejects.toThrow("Evidence network policy refused");
}
});
it("authenticates a bounded read-only REST probe and blocks unavailable credentials/services", async () => {
const directory = mkdtempSync(join(tmpdir(), "tht-probe-"));
const secret = join(directory, "key");
writeFileSync(secret, "PRIVATE_SENTINEL", { mode: 0o600 });
let status = 200;
const requests: string[] = [];
const server = createServer((req, res) => {
requests.push(`${req.method} ${req.url}`);
res.writeHead(req.headers.authorization === "Bearer PRIVATE_SENTINEL" ? status : 401);
res.end("PRIVATE_SERVER_RESPONSE");
});
await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
const address = server.address() as { port: number };
const document = { schemaVersion: 1, databases: [{ workspaceId: "demo", engine: "postgres", databaseName: "demo", schema: "public", binding: { transport: "rest_api", baseUrl: `http://127.0.0.1:${address.port}`, restPath: "/health", restAuth: "bearer" }, secretFiles: { apiKey: secret } }] };
try {
expect((await probeBootstrapDependencies(document)).ok).toBe(true);
status = 503;
const failed = await probeBootstrapDependencies(document);
expect(failed.ok).toBe(false);
expect(JSON.stringify(failed)).not.toContain("PRIVATE");
status = 200;
writeFileSync(secret, "rotated-but-invalid");
expect((await probeBootstrapDependencies(document)).ok).toBe(false);
expect(requests).toEqual(["GET /health", "GET /health", "GET /health"]);
} finally {
await new Promise<void>((resolve) => server.close(() => resolve()));
rmSync(directory, { recursive: true });
}
});
@@ -0,0 +1,119 @@
import { spawnSync } from "node:child_process";
import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync, symlinkSync } from "node:fs";
import { tmpdir } from "node:os";
import { join, resolve } from "node:path";
import { afterEach, expect, test } from "vitest";
import { parseWorkspaceCatalogYaml } from "../src/workspaces/catalog.js";
import { parseWorkspaceYaml } from "../src/workspaces/schema.js";
const roots: string[] = [];
function directory() {
const root = mkdtempSync(join(tmpdir(), "tht-documents-"));
roots.push(root);
return join(root, "workspaces");
}
afterEach(() => roots.splice(0).forEach((root) => rmSync(root, { recursive: true, force: true })));
function cli(...args: string[]) {
const packaged = process.env.THT_WORKSPACE_TEST_CLI;
const result = spawnSync(packaged ?? process.execPath, [...(packaged ? ["workspace"] : ["--import", "tsx", resolve("src/workspace-documents-cli.ts")]), ...args, "--json"], {
encoding: "utf8", env: { ...process.env, PATH: "" },
});
return { ...result, report: result.stdout.trim() ? JSON.parse(result.stdout) : null };
}
const descriptor = "workspace:\n schema_version: 4\n id: practice\n name: Practice\n language: en\n";
const catalog = "schema_version: 1\nworkspaces:\n - id: practice\n name: Practice\n";
const corpus = [
{ name: "minimal", descriptor, catalog, valid: true },
{ name: "optional Evidence", descriptor: descriptor + "evidence:\n source:\n type: http\n uris: [https://example.com/manual.md]\n", catalog, valid: true },
{ name: "duplicate YAML key", descriptor: descriptor + " id: practice\n", catalog, valid: false },
{ name: "ambiguous document", descriptor: descriptor + "---\n" + descriptor, catalog, valid: false },
{ name: "unknown workspace field", descriptor: descriptor + " secret: VERY_SECRET_VALUE\n", catalog, valid: false },
{ name: "database binding in authored descriptor", descriptor: descriptor + "dwh: {password: VERY_SECRET_VALUE}\n", catalog, valid: false },
{ name: "duplicate catalog id", descriptor, catalog: catalog + " - id: practice\n name: Practice\n", valid: false },
{ name: "unknown catalog field", descriptor, catalog: catalog + "secret: VERY_SECRET_VALUE\n", valid: false },
{ name: "invalid catalog YAML", descriptor, catalog: catalog + "schema_version: 1\n", valid: false },
{ name: "unsafe Evidence URI", descriptor: descriptor + "evidence:\n source:\n type: http\n uris: [https://user:VERY_SECRET_VALUE@example.com/file]\n", catalog, valid: false },
];
test.each(corpus)("CLI and runtime agree: $name", (fixture) => {
const root = directory();
mkdirSync(join(root, "practice"), { recursive: true });
writeFileSync(join(root, "thoth-workspaces.yaml"), fixture.catalog);
writeFileSync(join(root, "practice/workspace.yaml"), fixture.descriptor);
let accepted = true;
try { parseWorkspaceCatalogYaml(fixture.catalog); parseWorkspaceYaml(fixture.descriptor); }
catch { accepted = false; }
expect(accepted).toBe(fixture.valid);
const checked = cli("validate", "--directory", root);
expect(checked.status).toBe(fixture.valid ? 0 : 1);
expect(checked.report.ok).toBe(accepted);
expect(checked.stdout + checked.stderr).not.toContain("VERY_SECRET_VALUE");
if (!fixture.valid) {
expect(checked.report.issues[0].document).not.toBe(".");
expect(checked.report.issues[0].correction.length).toBeGreaterThan(10);
}
});
test("prepare refuses existing directories and invalid options without changing documents", () => {
const root = directory();
expect(cli("prepare", "--directory", root, "--id", "practice", "--name", "Practice").status).toBe(0);
const before = readFileSync(join(root, "thoth-workspaces.yaml"), "utf8");
expect(cli("prepare", "--directory", root, "--id", "other", "--name", "Other").report.issues[0].code).toBe("destination_exists");
expect(readFileSync(join(root, "thoth-workspaces.yaml"), "utf8")).toBe(before);
expect(cli("validate", "--directory", root, "--typo", "VERY_SECRET_VALUE").status).toBe(2);
});
test("validation rejects directory mismatches, missing references and symlinks without following them", () => {
const root = directory();
cli("prepare", "--directory", root, "--id", "practice", "--name", "Practice");
mkdirSync(join(root, "unlisted"));
expect(cli("validate", "--directory", root).report.issues.some((i: {code: string}) => i.code === "unlisted_directory")).toBe(true);
rmSync(join(root, "unlisted"), { recursive: true });
writeFileSync(join(root, "practice/workspace.yaml"), descriptor.replace("name: Practice", "name: Different"));
expect(cli("validate", "--directory", root).report.issues[0].code).toBe("catalog_mismatch");
writeFileSync(join(root, "practice/workspace.yaml"), descriptor + "evidence:\n source:\n type: filesystem\n uri: practice/evidence\n");
expect(cli("validate", "--directory", root).report.issues[0].code).toBe("missing_reference");
symlinkSync(roots[roots.length - 1], join(root, "practice/evidence"), "dir");
expect(cli("validate", "--directory", root).report.issues[0].code).toBe("unsafe_reference");
});
test("local Evidence checks references and YAML frontmatter, and explicitly defers canonical content validation", () => {
const root = directory();
cli("prepare", "--directory", root, "--id", "practice", "--name", "Practice");
writeFileSync(join(root, "practice/workspace.yaml"), descriptor + "evidence:\n schema_version: 2\n source:\n type: filesystem\n uri: practice/evidence\n patterns: ['curated/**/*.md']\n");
mkdirSync(join(root, "practice/evidence/curated/domain"), { recursive: true });
const evidence = join(root, "practice/evidence/curated/domain/rule.md");
writeFileSync(evidence, "---\nschema_version: 4\nid: evidence:rule\nkind: domain\nlanguage: en\npurposes: [sql_generation]\n---\n# Rule\n\n## Rule\nUse the order number.\n");
const checked = cli("validate", "--directory", root);
expect(checked.status).toBe(0);
expect(checked.report.workspaces).toEqual([{ id: "practice", evidence: "local-files" }]);
expect(checked.report.deferred_checks).toContain("practice:evidence-content-provenance-and-indexing");
writeFileSync(evidence, "---\nid: first\nid: VERY_SECRET_VALUE\n---\n# Rule\n");
const invalid = cli("validate", "--directory", root);
expect(invalid.status).toBe(1);
expect(invalid.report.issues[0].document).toBe("practice/evidence/curated/domain/rule.md");
expect(invalid.stdout).not.toContain("VERY_SECRET_VALUE");
writeFileSync(join(root, "practice/workspace.yaml"), descriptor + "evidence:\n source:\n type: filesystem\n uri: practice/evidence\n patterns: [missing.md]\n");
expect(cli("validate", "--directory", root).report.issues[0].code).toBe("missing_reference");
});
test("prepare creates documents accepted by the runtime and validate is repeatable without installation or services", () => {
const root = directory();
const prepared = cli("prepare", "--directory", root, "--id", "practice", "--name", "Practice");
expect(prepared.stderr).toBe("");
expect(prepared.status).toBe(0);
expect(prepared.report.ok).toBe(true);
const catalog = readFileSync(join(root, "thoth-workspaces.yaml"), "utf8");
const descriptor = readFileSync(join(root, "practice/workspace.yaml"), "utf8");
expect(parseWorkspaceCatalogYaml(catalog).workspaces[0].id).toBe("practice");
expect(parseWorkspaceYaml(descriptor).evidence).toBeUndefined();
for (let index = 0; index < 2; index++) {
const checked = cli("validate", "--directory", root);
expect(checked.status).toBe(0);
expect(checked.report.workspaces).toEqual([{ id: "practice", evidence: "absent" }]);
}
expect(readFileSync(join(root, "thoth-workspaces.yaml"), "utf8")).toBe(catalog);
expect(readFileSync(join(root, "practice/workspace.yaml"), "utf8")).toBe(descriptor);
});
+91
View File
@@ -0,0 +1,91 @@
# Installation preflight and release manifest
The native operator protocol is version 1. `installation preflight` is the independent
step-3 host check. `installation plan` repeats document validation, performs step-5
live checks and writes an owner-only JSON plan plus a separate owner-only `.key` file.
Neither command creates containers, imports bindings, modifies a database or invokes
model generation. Exit codes are 0 (checks passed), 1 (blocking error), and 2 (usage).
JSON stdout contains only the report. Checks carry stable `id`, `outcome`, `field`
and `action`; outcomes are `passed`, `error`, `warning`, `deferred-to-runtime`.
## Release manifest schema 1
The manifest is a local JSON file shipped with the verified operator/release bundle.
Required keys:
| Key | Contract |
| --- | --- |
| `schema_version` | `1` |
| `version` | Semantic release version, optionally prerelease |
| `revision` | 40 lowercase hexadecimal Git commit characters |
| `validator_protocol` | `1`; incompatible consumers refuse the manifest |
| `requirements` | `cpus`, `memory_bytes`, `disk_bytes`; at least 2 CPUs, 4 GiB Docker memory and 10 GiB installation filesystem space |
| `components` | Includes `pi`, `catalog-migrations`, `workspace-maintenance` |
| `images` | Exactly `core`, `frontend`, `catalog`, `qdrant`, `embedding`; each maps `linux/amd64` and/or `linux/arm64` to a `docker.io/...@sha256:...` **single-platform image digest** |
| `files` | Relative packaged resource paths to SHA-256; no traversal, links or absolute paths; maximum 256 files, 32 MiB per resource |
| `compose` | Ordered relative Compose file paths present in `files` for this release configuration |
Include the selected `deploy/compose.git-https.yaml` or `deploy/compose.git-ssh.yaml`
transport overlay in `files`. Standard transport overlays are resolved from the
release while absent in the installation directory; existing authored overrides
remain input files. Compose must resolve all eight services: core, frontend,
catalog-db, catalog-migrate, workspace-maintenance, qdrant, embedding and
embedding-model-init. The two maintenance services share the core digest; embedding
initialization shares the embedding digest. Source builds and undeclared services
are rejected in this prebuilt path. The explicit source path is a separate ticket.
`docker manifest inspect --verbose` checks each selected immutable image and its
platform without pulling layers. `docker compose config --format json` checks the
effective service configuration. Compose receives only Docker connection/trust,
proxy and executable-discovery host variables; application parameters come from
the prepared environment file. Raw Docker output is never copied into reports.
Each Docker command has a 15-second bound. No release is currently certified merely
because controlled manifest tests pass: publication and real pull acceptance belong
to the publication/execution tickets.
## External checks and bounds
- Git HTTPS: authenticated `GET /info/refs?service=git-upload-pack`, configured CA,
no redirects, selected branch advertised, 1 MiB response and 5-second bound.
Git SSH uses its prepared key/known-hosts and `git-upload-pack --advertise-refs`
with the same response/time bounds; no checkout or push occurs.
- PostgreSQL: the existing Catalog diagnostic adapter authenticates and reads
`current_database()` plus schema `USAGE`; no user tables are modified.
- REST database transport: existing Catalog diagnostic `GET` with the configured
bearer/API-key header and status validation. SSH database bindings cannot pass
this NL-to-SQL installation plan because runtime sessions do not support them.
- Evidence: local paths were already validated. HTTP performs bounded GET requests
and cancels response bodies; signed URL identities must match authored provenance.
S3 performs one `ListObjectsV2` request with `MaxKeys: 1`, no retries, explicit
file credentials and the canonical Evidence egress policy. These metadata requests
can incur normal remote-service request charges; they do not run LLM generation.
Each external request has a 5-second bound; the native database/Evidence helper
has a 60-second aggregate bound. Correct unavailable services before repeating.
- Explicit model endpoints: DNS/TCP/TLS origin reachability, without generating
tokens. Built-in endpoint resolution, provider authentication and model smoke
operations use the bundled Pi SDK at runtime; the report never claims those
operations have already passed.
No unreachable configured external dependency is converted to a deferred success.
Runtime obligations have explicit identities: `container-network`,
`catalog-initialization`, `pi-operation`, `local-embedding`,
`workspace-preprocessing`, `workspace-readiness`. These must be discharged by the
execution/readiness tickets before final success. Host disk inspection cannot prove
Docker Desktop VM free storage; its separate storage warning remains explicit.
## Input identity and freshness
The plan records normalized installation configuration, release digests, validator
build identity, verified input paths, local workspace content and its Git HEAD when
available (otherwise a content snapshot). Files are bounded to 32 MiB each and
256 MiB total; workspace/auth trees to 10,000 entries, without links or special files.
Credential contents are never serialized. An HMAC covers the private inputs and
plan using a separate random 32-byte owner-only key; no public unkeyed secret hash
is generated. Credential rotation invalidates the plan. Added/deleted/changed
workspace files invalidate it as well. Changes during the checks abort publication.
Plan output never overwrites an existing plan or key.
Execution and resumption must call `VerifyPlanInputs` **and repeat live checks and
credential reads before mutations**. A valid seal alone does not certify current
network availability, Docker state or runtime readiness. Keep both plan files
private and outside the workspace repository; they are installation-local artifacts.
+94
View File
@@ -0,0 +1,94 @@
# Pubblicare immagini e pacchetto operatore / Publish images and operator bundle
## Italiano
Questo comando è riservato al manutentore. L'utente finale scarica il pacchetto e
le immagini già compilati. La prima prerelease riguarda **Linux amd64**, utilizzato
da Ubuntu WSL2 su Windows e successivamente da Omarchy; non certifica ancora il
percorso completo di installazione o i collaudi manuali.
Prerequisiti del manutentore: Git, Node 24/npm, Go indicato in `tools/tht/go.mod`,
Docker con Buildx e capacità di eseguire Linux amd64, `tar`. Bun viene installato
dal lock npm e serve soltanto per compilare il pacchetto. Il commit da pubblicare
deve essere già disponibile sul repository Gitea pubblico.
1. Eseguire `docker login` sul computer di pubblicazione con un account autorizzato
a creare repository pubblici e pubblicare immagini nel namespace scelto.
Usare un credential store Docker: il token non va passato sulla riga di comando,
scritto nel repository o copiato nel pacchetto.
2. Predisporre nel credential helper Git l'accesso al repository Gitea con diritto
di creare rilasci e allegati. Il comando riusa quelle credenziali senza stamparle.
3. Installare le dipendenze del produttore con `cd backend && npm ci`, poi eseguire:
```bash
node scripts/publish-installation.mjs \
--revision COMMIT_GIA_PUBBLICATO \
--version 0.1.0-install-preview.1 \
--namespace tylconsulting \
--platforms linux/amd64 \
--output /percorso/privato/rilascio-0.1.0-install-preview.1
```
La directory di output deve essere nuova o vuota e avere un genitore esistente.
Diventa privata e contiene stato di ripresa, log del produttore, archivi e checksum.
Non è una directory di installazione. Il produttore usa un worktree temporaneo al
commit richiesto, così modifiche locali, segreti e workspace non entrano nelle build.
Il comando prepara `tylconsulting/thothii-core` e `tylconsulting/thothii-frontend`
come repository pubblici, costruisce e pubblica le immagini versionate, risolve i
digest di tutte le immagini e crea gli archivi del comando nativo con Compose e
risorse di inizializzazione. Catalog migration e workspace maintenance usano lo
stesso digest core. PostgreSQL, Qdrant e Ollama restano immagini upstream.
Prima di rendere pubblico il rilascio Gitea, vengono verificati pull anonimi delle
immagini, smoke test senza rete di core/frontend, checksum e download degli allegati.
Una build o un upload incompleto lascia il rilascio **in bozza**. Per riprovare,
rieseguire lo stesso comando con gli stessi parametri e la stessa directory.
Immagini già presenti devono appartenere allo stesso commit/versione; allegati
esistenti devono avere lo stesso checksum. Il produttore non sostituisce versioni
pubblicate con contenuti diversi. Conservare la directory fino al completamento.
Il risultato pubblico comprende `thothii-VERSION-linux-amd64.tar.gz` e
`SHA256SUMS.txt`. L'archivio contiene `bin/tht`, il validatore affiancato,
`release-manifest.json`, Compose, SQL/script di inizializzazione e guide. Non
contiene credenziali, dati dei workspace o database di esempio. La verifica su
questa macchina di pubblicazione non sostituisce il successivo collaudo Windows.
## English
This is a maintainer command. Consumers download precompiled images and the native
operator bundle. The first prerelease targets **Linux amd64** for Ubuntu WSL2 and
later Omarchy; full installation and real-host acceptance remain separate work.
The maintainer needs Git, Node 24/npm, the Go toolchain from `tools/tht/go.mod`,
Docker Buildx with Linux amd64 execution support, and `tar`. The npm lock supplies
Bun for producer builds only. Push the selected source commit to the public Gitea
repository before publication. Use Docker's credential store for `docker login`
and Git's credential helper for Gitea release/attachment permissions. Never pass
tokens as command arguments or include them in a checkout or archive.
From `backend`, run `npm ci`, then the command above with an explicit revision,
version, namespace, platforms and a new private output directory. A temporary Git
worktree isolates the selected commit. The producer publishes core/frontend to
Docker Hub and retains PostgreSQL, Qdrant and Ollama upstream. All runtime and
maintenance services use resolved immutable platform digests.
The Gitea release stays a draft until images, anonymous pulls, network-isolated
smoke checks and uploaded bundle checksums pass. An interrupted run can be retried
with the same arguments and output directory. Existing images must match the
source/version, and existing attachments must match their checksum; published
versions are not overwritten. Producer logs and retry state stay local.
Download the matching `.tar.gz` and `SHA256SUMS.txt` from the public Gitea prerelease,
verify the archive checksum, then extract it. Keep the two executables together.
The bundle needs no application checkout, Node, Bun, Python or compiler on the
consumer host. It carries the manifest, Compose and initialization assets, but no
installation credentials, workspace data or example databases. Linux arm64 can be
selected explicitly for later release work; it does not imply macOS acceptance.
Developer regression checks:
```bash
cd backend
node --test scripts/release-*.test.mjs
```
+185
View File
@@ -7,6 +7,191 @@ question, queries an enterprise database read-only, and guides the user through
core, PostgreSQL catalog, Qdrant, embedding service, and Pi run in Docker; Node.js, Python, and Pi
are not required on the host.
## Prepare and validate workspace documents before starting the stack
The first two steps of the new flow work without Docker, Node, Python, Pi or an
installation descriptor. Use the platform bundle with **both** `tht` and
`tht-workspace-documents` in the same directory (`.exe` on Windows). Put that directory
on `PATH`, or invoke the absolute executable path. Older packages containing only
`tht` do not provide this capability. Maintainers can currently build the bundle;
publishing assets and Docker Hub images belongs to a later delivery step. The rest
of this guide still describes the existing installation path.
1. Choose a new directory outside the application checkout, with an existing parent:
```sh
tht workspace prepare --directory ./my-workspaces --id practice --name "Practice" --language en
```
This creates `thoth-workspaces.yaml`, `practice/workspace.yaml` and
`workspace-docs/practice/README.md`. Existing destinations, even empty ones, are
refused. No services start, Git is not initialized and no remote is contacted.
Example databases remain a deferred subproject. For a curator-supplied repository,
use a separate local copy and go straight to step 3; read access to the origin is
sufficient.
2. Edit the catalog (schema v1) and workspace descriptor (schema v4) at your own pace.
Keep `id`, `name` and optional `description` identical in both; the id must match
the directory name. Root directories must match catalog entries, except
`workspace-docs` and the local `.git` directory. Database connections, schema and
credentials belong to the installation Metadata Catalog. Evidence is optional
and initially absent.
3. Validate, correct the reported document/field, and repeat:
```sh
tht workspace validate --directory ./my-workspaces
tht workspace validate --directory ./my-workspaces --json
```
PowerShell uses the same arguments, for example
`C:\ThothII\bin\tht.exe workspace validate --directory C:\ThothII\my-workspaces`.
Validation changes no files. It rejects multiple/malformed YAML documents,
duplicate keys/ids, unknown fields, catalog/directory/descriptor mismatches,
missing local references and symbolic links. Fix the first error in each document
and repeat to reveal any subsequent errors.
Evidence `absent` is valid. Filesystem checks cover directories, accessibility and
literal references; standard Markdown selections also check declared size limits.
Evidence v2 requires `curated/` and syntactically valid YAML frontmatter. Local limits
are 1 MiB per document read and 100,000 entries per Evidence tree. Arbitrary patterns,
the complete curated-unit contract, provenance, HTTP/S3 access and indexing remain
explicit runtime checks. See the [Evidence guide](../evidence.md).
JSON includes `schema_version`, `scope: local-documents`, `ok`, `workspaces`, `issues`
and `deferred_checks`. Issues identify document, field, code, correction and YAML
line where available, without printing document values. Exit statuses: `0` local
success, `1` documents/access/bundle need correction, `2` invalid arguments. Local
success does not certify semantic truth, connectivity or readiness. The Git revision
activated later must contain the checked documents; this command does not publish
uncommitted files or empty directories.
## Prepare and validate application documents
After validating workspaces, create a local directory **outside their repository**:
```sh
tht installation prepare --directory ./my-installation
```
This creates private, commented `thothii-installation.yaml`, `operator.env`,
`database-bootstrap.yaml` and `README.md`. The destination must be new and its parent
must exist. It starts no services and does not implicitly generate passwords.
1. Choose models and providers in the descriptor. The template proposes
`openai/gpt-4.1-mini` for interaction and `ollama/qwen3-embedding:0.6b` with 1024
dimensions for embedding. Edit these before setup. `modelCatalog.defaults.interaction`
must support sessions and metadata generation when the latter is configured.
The template omits optional metadata generation. See
[model configuration](../general/pi-configuration.md) for custom providers.
2. Replace the Git remote in both the descriptor and `operator.env`; keep branch and
transport consistent. Paths are absolute and machine-local. `operator.env` accepts
one literal `KEY=value` assignment per line, without duplicate keys or shell
interpolation. Credentials belong in referenced protected files.
3. Complete `database-bootstrap.yaml` with exactly one entry per workspace. A complete
direct-connection example is:
```yaml
schemaVersion: 1
databases:
- workspaceId: practice
engine: postgres
databaseName: sales
schema: public
binding:
transport: postgres_direct
host: db.intranet
port: 5432
username: thoth_reader
secretFiles:
password: /private/path/my-installation/secrets/database-password
```
Use a read-only DWH account. `rest_api` requires `baseUrl`, `restPath`, `restAuth`
(`none`, `bearer`, `x-api-key`) and `secretFiles.apiKey` when authenticated.
`ssh_tunnel` requires `username`, `sshHost`, `sshPort`, `sshUsername`,
`sshTargetHost`, `sshTargetPort` and files `password`, `sshPrivateKey`,
`sshKnownHosts`; it supports Catalog diagnostics, not NL-to-SQL sessions.
`tlsCa` and `sshPrivateKeyPassphrase` are optional. Signed HTTP Evidence needs
`evidenceSecretFiles` with key `evidence.signed_urls`; static S3 credentials need
`evidence.access_key`, `evidence.secret_key` and optional `evidence.session_token`.
All values are private file paths. Workspace descriptors remain schema v4;
this bootstrap input is not a second runtime Catalog.
4. Explicitly generate technical credentials in the standard layout:
```sh
tht installation credentials --directory ./my-installation
```
This creates separate random Catalog runtime/migrator and administrator passwords,
`auth/auth.yaml`, `auth/users.yaml`, a `secrets/secrets.env` template and
`secrets/pi-auth.json`. Existing files are retained; invalid ones stop the command.
The initial administrator is `admin`; its password stays in private
`secrets/admin-password` and is never printed. The default is local authentication
at `http://localhost:8080`: review and edit `auth/auth.yaml` before validation.
This increment does not validate offline OIDC bootstrap for the existing path.
5. Fill the provider key in `secrets/secrets.env` and create the DWH password file.
For Git HTTPS supply the referenced credentials and CA files; empty credentials
are allowed for a public remote, and the CA file must be available. For SSH supply
a key and known_hosts and select the matching descriptor override. `pi_auth`
providers require prepared Pi credentials. Keep every secret outside workspace
Git with installer-only access (0600 on Unix, equivalent Windows ACLs).
6. Validate and repeat after each correction:
```sh
tht --installation /absolute/path/my-installation/thothii-installation.yaml installation validate --workspaces /absolute/path/my-workspaces --json
```
The default bootstrap is beside the descriptor; `--bootstrap PATH` selects another.
Validation changes no documents, generates no projections, uses no network and
writes no database. It rejects placeholders, inconsistencies, missing/non-private
files and secrets inside workspace Git. Reports identify document, field and
correction without secret values. Exit statuses: 0 local success, 1 corrections
needed, 2 invalid arguments.
Standard release Compose assets may still be absent at this stage; custom overrides
must already exist. Release assets, external connectivity, Catalog import and runtime
readiness remain explicit deferred checks. Success prepares the next preflight;
it neither skips those checks nor establishes a completed installation.
## Check prerequisites and produce the plan
At step 3, before completing all application parameters, check the machine and
the private installation directory already prepared:
```bash
tht installation preflight --directory /path/installation --json
```
This requires a reachable Linux Docker daemon, Compose 2.24 or newer, at least
2 CPUs, 4 GiB allocated to Docker and 10 GiB free on the installation filesystem.
A release may require more resources. On Windows run the Linux executable in
Ubuntu WSL2 with Docker Desktop integration; Pi is bundled in the core image.
At step 5, after `installation validate`, select the published release manifest
with its downloaded bundle resources and produce a new plan:
```bash
tht --installation /path/installation/thothii-installation.yaml installation plan \
--workspaces /path/workspaces \
--release /path/release/release-manifest.json \
--output /path/installation/installation-plan.json --json
```
This repeats document checks, verifies image digests and Compose, Git, available
external databases and Evidence, then saves the plan and its separate private
`.key` file. It does not execute setup. Missing images and unavailable existing
dependencies block the plan. Actual Docker Hub publication remains the next ticket;
an invented manifest cannot bypass publication.
Correct `error` outcomes and read `warning` outcomes. `deferred-to-runtime` entries
are mandatory checks after startup, not readiness already achieved. After changing
documents or rotating credentials, produce a new plan; existing files are never
overwritten. Keep both plan files outside workspace Git. External probes perform
bounded database authentication/schema reads, Git/HTTP/S3 reads and explicit model
endpoint reachability checks. They invoke no LLM generation; HTTP/S3 requests may
incur ordinary service request charges. See the
[preflight reference](installation-preflight.md) for limits, the manifest
format and runtime obligations.
## Before you start: the two repositories
There are two separate repositories:
+192
View File
@@ -7,6 +7,198 @@ naturale, interroga in sola lettura un database aziendale e accompagna l’utent
della SQL risultante. Il core, il catalogo PostgreSQL, Qdrant, il servizio di embedding e Pi vengono
eseguiti in Docker; sul computer non servono Node.js, Python o Pi.
## Preparazione e verifica dei workspace prima dello stack
I primi due passi del nuovo percorso funzionano senza Docker, Node, Python, Pi o
un file di installazione. Usare il bundle della propria piattaforma con **entrambi**
gli eseguibili `tht` e `tht-workspace-documents` nella stessa cartella (`.exe` su
Windows). Aggiungere la cartella al `PATH`, oppure usare il percorso completo.
Il vecchio pacchetto con il solo `tht` non contiene questa capacità. Il bundle è
attualmente producibile dal manutentore; pubblicazione degli asset e immagini
Docker Hub appartengono a una fase successiva. Il resto della guida descrive ancora
il percorso di installazione esistente.
1. Scegliere una cartella nuova, esterna all'applicazione, con padre già esistente:
```sh
tht workspace prepare --directory ./miei-workspace --id pratica --name "Pratica" --language it
```
Sono creati `thoth-workspaces.yaml`, `pratica/workspace.yaml` e
`workspace-docs/pratica/README.md`. Una destinazione esistente, anche vuota, viene
rifiutata. Nessun servizio viene avviato; Git non viene inizializzato, nessun remoto
viene contattato. I database di esempio restano un sottoprogetto differito. Per
un repository fornito dal curatore, usare una copia locale separata e passare al
punto 3: è sufficiente accesso in lettura all'origine.
2. Modificare con calma catalogo (schema v1) e descrittore workspace (schema v4).
Mantenere uguali `id`, `name` e l'eventuale `description` nei due file; l'id deve
coincidere con la cartella. Ogni cartella alla radice deve corrispondere a un
workspace elencato, salvo `workspace-docs` e la directory locale `.git`.
Connessioni, schema e credenziali dei database appartengono al Metadata Catalog
dell'installazione. Le Evidence sono facoltative e inizialmente assenti.
3. Verificare, correggere il documento/campo indicato e ripetere:
```sh
tht workspace validate --directory ./miei-workspace
tht workspace validate --directory ./miei-workspace --json
```
Su PowerShell usare gli stessi argomenti, ad esempio
`C:\ThothII\bin\tht.exe workspace validate --directory C:\ThothII\miei-workspace`.
Il controllo non modifica file. Rifiuta YAML multipli o malformati, chiavi/id
duplicati, campi sconosciuti, incoerenze tra catalogo, cartelle e descrittori,
riferimenti locali mancanti e link simbolici. Correggere il primo errore del
documento e ripetere per vedere eventuali errori successivi.
Evidence `absent` è valido. Per filesystem si verificano directory, accessibilità e
riferimenti letterali; per le selezioni Markdown standard anche i limiti dichiarati.
Evidence v2 richiede `curated/` e frontmatter con sintassi YAML valida. I limiti locali
sono 1 MiB per documento letto e 100.000 elementi per albero Evidence. Pattern
arbitrari, contratto completo delle unità curate, provenienza, accesso HTTP/S3 e
indicizzazione restano controlli runtime espliciti. Vedere la [guida Evidence](../evidence.md).
Il JSON espone `schema_version`, `scope: local-documents`, `ok`, `workspaces`, `issues`
e `deferred_checks`. I problemi riportano documento, campo, codice, correzione e,
quando disponibile, riga YAML, senza stampare i valori del documento. Exit status:
`0` successo locale, `1` documenti/accesso/bundle da correggere, `2` argomenti errati.
Il successo locale non certifica verità semantica, connettività o readiness. La
revisione Git attivata in seguito deve contenere i documenti verificati; il comando
non pubblica file non committati o directory vuote.
## Predisporre e verificare i documenti applicativi
Dopo la verifica dei workspace, creare una cartella locale **esterna al loro repository**:
```sh
tht installation prepare --directory ./mia-installazione
```
Il comando crea file privati commentati: `thothii-installation.yaml`, `operator.env`,
`database-bootstrap.yaml` e `README.md`. La cartella deve essere nuova e il padre
deve esistere. Non avvia servizi e non genera implicitamente password.
1. Nel descrittore, scegliere modelli e provider. Il template propone
`openai/gpt-4.1-mini` per l'interazione e `ollama/qwen3-embedding:0.6b` con 1024
dimensioni per l'embedding. Sono valori modificabili, non una selezione richiesta
durante il setup. `modelCatalog.defaults.interaction` deve essere utilizzabile
nelle sessioni e anche nella generazione metadati, se quest'ultima è configurata.
Il template omette la generazione metadati, che è facoltativa. Consultare la
[configurazione dei modelli](../general/pi-configuration.md) per provider personalizzati.
2. Sostituire il remoto Git sia nel descrittore sia in `operator.env`; mantenere
coerenti branch e trasporto. I percorsi sono assoluti e riferiti a questa macchina.
`operator.env` accetta una sola assegnazione letterale `KEY=value` per riga, senza
duplicati o interpolazioni shell. Le credenziali restano nei file referenziati.
3. Compilare `database-bootstrap.yaml`: una voce per ciascun workspace, senza
duplicati. Questo esempio mostra il contratto completo di un collegamento diretto:
```yaml
schemaVersion: 1
databases:
- workspaceId: pratica
engine: postgres
databaseName: vendite
schema: public
binding:
transport: postgres_direct
host: db.intranet
port: 5432
username: thoth_reader
secretFiles:
password: /percorso/privato/mia-installazione/secrets/database-password
```
Usare le credenziali di un utente DWH in sola lettura. Per `rest_api`, il binding
richiede `baseUrl`, `restPath` e `restAuth` (`none`, `bearer`, `x-api-key`); quando
serve autenticazione, aggiungere `secretFiles.apiKey`. `ssh_tunnel` richiede
`username`, `sshHost`, `sshPort`, `sshUsername`, `sshTargetHost`, `sshTargetPort` e
i file `password`, `sshPrivateKey`, `sshKnownHosts`; abilita diagnostica Catalog,
non sessioni NL→SQL. Sono facoltativi `tlsCa` e `sshPrivateKeyPassphrase`.
Le Evidence HTTP firmate richiedono `evidenceSecretFiles` con chiave
`evidence.signed_urls`; S3 con credenziali statiche richiede `evidence.access_key`
e `evidence.secret_key`, con `evidence.session_token` facoltativo. Tutti i valori
sono percorsi di file privati. I workspace rimangono nello schema v4: il bootstrap
è un input iniziale, non un secondo Catalog runtime.
4. Generare esplicitamente le credenziali tecniche nel layout standard:
```sh
tht installation credentials --directory ./mia-installazione
```
Sono creati password casuali separate per Catalog runtime/migrator e amministratore,
il relativo `auth/auth.yaml` con `auth/users.yaml`, un template `secrets/secrets.env`
e `secrets/pi-auth.json`. I file esistenti vengono conservati; se non validi, il
comando si ferma. L'amministratore iniziale è `admin`, la password è nel file
privato `secrets/admin-password` e non viene stampata. Il default è autenticazione
locale con URL pubblico `http://localhost:8080`: verificare e, se necessario,
modificare `auth/auth.yaml` prima del controllo. Questo incremento non valida il
bootstrap OIDC del percorso esistente.
5. Inserire la chiave del provider in `secrets/secrets.env` e creare il file della
password DWH. Per HTTPS Git fornire i file referenziati per credenziali e CA:
credenziali vuote sono ammesse per un remoto pubblico, la CA deve essere disponibile.
Per SSH fornire chiave e known_hosts, scegliendo il relativo override nel descrittore.
I provider `pi_auth` richiedono credenziali Pi già preparate. Conservare tutti
questi file fuori dal repository workspace e proteggere l'accesso al solo utente
installatore (0600 su Unix, ACL equivalenti su Windows). Non committarli.
6. Verificare e ripetere dopo ogni correzione:
```sh
tht --installation /percorso/assoluto/mia-installazione/thothii-installation.yaml installation validate --workspaces /percorso/assoluto/miei-workspace --json
```
Il bootstrap viene cercato accanto al descrittore; `--bootstrap PERCORSO` ne
seleziona uno diverso. Il controllo non cambia documenti, non genera proiezioni,
non usa la rete e non scrive database. Rifiuta placeholder, incoerenze, file
mancanti/non privati e segreti situati nel repository workspace. Il report indica
documento, campo e correzione senza valori riservati. Exit status: 0 successo
locale, 1 correzioni necessarie, 2 argomenti errati.
Gli asset Compose standard del rilascio possono ancora mancare in questa fase;
gli override personalizzati devono già esistere. Il report distingue i controlli
differiti: asset del rilascio, connettività esterna, import Catalog e readiness.
Un esito positivo prepara il successivo preflight: non autorizza a saltare tali
controlli e non equivale a un'installazione completata.
## Verificare le precondizioni e produrre il piano
Al passo 3, prima di completare tutti i parametri applicativi, controllare la
macchina e la directory privata già preparata:
```bash
tht installation preflight --directory /percorso/installazione --json
```
Servono Docker Linux raggiungibile, Compose 2.24 o successivo, almeno 2 CPU,
4 GiB assegnati a Docker e 10 GiB liberi nella directory di installazione. Il
rilascio può richiedere risorse maggiori. Su Windows eseguire il binario Linux
in Ubuntu WSL2 con integrazione Docker Desktop; Pi è incluso nell'immagine core.
Al passo 5, dopo `installation validate`, selezionare il manifest del rilascio
pubblicato, con le risorse del bundle già scaricate, e produrre un piano nuovo:
```bash
tht --installation /percorso/installazione/thothii-installation.yaml installation plan \
--workspaces /percorso/workspaces \
--release /percorso/rilascio/release-manifest.json \
--output /percorso/installazione/installation-plan.json --json
```
Il comando ripete i controlli dei documenti, verifica immagini/digest e Compose,
Git, database ed Evidence esterne disponibili, poi salva il piano e il suo file
privato `.key`. Non esegue il setup. Le immagini assenti e le dipendenze esterne
irraggiungibili bloccano il piano. Al momento la pubblicazione reale Docker Hub
è ancora il ticket successivo: un manifest inventato non permette di aggirarla.
Correggere gli esiti `error`; leggere gli `warning`. Gli esiti
`deferred-to-runtime` identificano controlli obbligatori dopo l'avvio, non una
readiness già ottenuta. Un piano valido non sostituisce questi gate. Dopo una
correzione o rotazione di credenziali produrre un nuovo piano; i file esistenti
non vengono sovrascritti. Conservare piano e chiave fuori dal Git dei workspace.
Le prove esterne sono letture limitate: autenticazione/schema del database,
letture Git e HTTP/S3, raggiungibilità degli endpoint modello espliciti. Nessuna
generazione LLM viene invocata; le richieste HTTP/S3 possono avere i normali costi
del servizio. Limiti, manifest e obblighi sono nel
[riferimento di preflight](installation-preflight.md).
## Prima di iniziare: i due repository
Servono due repository distinti:
@@ -3,6 +3,14 @@
**Stato:** design concordato con grill-with-docs il 2026-09-14. Questo worktree definisce e rende
verificabile il percorso manuale; non introduce pacchetti nativi.
**Aggiornamento:** il [PRD del 27 settembre](2026-09-27-guided-installation-prd.md)
rende le immagini precompilate su Docker Hub il percorso ordinario e include la
loro pubblicazione nel progetto. Il percorso con build da sorgente documentato qui
resta un'alternativa; il precedente rinvio di Docker Hub è superato.
La revisione del 28 settembre dello stesso PRD sostituisce inoltre il setup
interattivo con preparazione dei documenti, verifiche ripetibili ed esecuzione
senza richiesta di parametri.
## Obiettivo
Permettere di predisporre una copia di THothII su macOS, Windows e Linux partendo dal clone del
@@ -0,0 +1,386 @@
# PRD — Installazione di ThothII da documenti verificati
Creato: 2026-09-27. Revisione: 2026-09-28. Stato: requisiti e chiarimenti R1/R2
approvati; `grill-with-docs` e `/to-spec` conclusi. Specifica pubblicata su Gitea
con etichetta `ready-for-agent`; implementazione non iniziata.
Branch: `codex/guided-standalone-install`.
La [specifica derivata](2026-09-28-document-first-installation-spec.md) raccoglie
user story, decisioni implementative e collaudi. Il piano di test è stato confermato
dall'utente e la specifica è pubblicata nell'issue
[Spec: installazione da documenti verificati e distribuzione Docker Hub](https://git.tylconsulting.it/mptyl/ThothII/issues/42).
La [scomposizione approvata](2026-09-28-installation-ticket-breakdown.md) è pubblicata
nelle issue #43–#54 con dipendenze native. Il prossimo passaggio è `/implement`
sui ticket senza blocchi, inizialmente validazione workspace e rilascio Docker Hub.
Questo documento consolida le decisioni della
[ripresa del progetto](2026-09-27-guided-installation-resumption.md) e aggiorna il
[piano standalone del 14 settembre](2026-09-14-manual-standalone-installation.md)
per l'esperienza guidata. Conserva architettura, protezione dei segreti e contratti
del prodotto; modifica il percorso operativo e l'ordine dei collaudi. La successiva
precisazione dell'utente rende la distribuzione di immagini precompilate su Docker Hub
parte della prima versione del percorso ordinario, superando il precedente rinvio.
La revisione del 28 settembre sostituisce la raccolta interattiva dei parametri:
si preparano prima i documenti, li si verifica anche più volte, poi si esegue il setup.
Prevale sulle precedenti decisioni I1/I2 dove consentivano configurazioni obbligatorie
rinviate o richieste durante l'installazione.
## Obiettivo e destinatario
Una persona capace di installare Docker, clonare un repository e fornire le proprie
credenziali deve poter predisporre con calma i documenti necessari e rendere
utilizzabile almeno un workspace, senza conoscere l'architettura interna.
Template commentati, esempi compilati e documentazione passo per passo spiegano
cosa inserire nei file YAML e nei file protetti `.env` o equivalenti.
DWH e provider LLM possono essere esterni; non si promette un funzionamento offline.
La CLI offre preparazione dei template, verifiche ripetibili e applicazione dei
documenti già verificati. Il setup non raccoglie parametri, non apre questionari
e non completa silenziosamente documenti incompleti. L'utente corregge i documenti
prima di eseguirlo. Le pagine amministrative rimangono disponibili per l'uso e la
manutenzione successivi, senza diventare una scorciatoia per rinviare parametri
obbligatori dell'installazione.
Il percorso predefinito scarica da Docker Hub le immagini applicative già compilate:
il computer dell'utente svolge configurazione, inizializzazione dei servizi e dei
database, avvio e verifiche. L'installazione da sorgente resta disponibile come
scelta esplicita alternativa. Nessuna compilazione dell'applicazione, neppure
all'interno di un container locale, è richiesta dal percorso ordinario.
## Decisioni approvate
| Area | Comportamento richiesto |
| --- | --- |
| Distribuzione predefinita | Creare, collaudare e pubblicare su Docker Hub le immagini applicative precompilate; il setup le scarica e le avvia senza build locale. |
| Alternativa da sorgente | Conservare un percorso esplicito di build dai sorgenti, documentato e verificato, con configurazione e funzionalità equivalenti. |
| Due traguardi | Mostrare separatamente piattaforma installata e workspace pronto. |
| Preparazione anticipata | Repository workspace e documenti applicativi predisposti prima dell'esecuzione; template, esempi e guida alla compilazione. |
| Setup senza domande | Consuma documenti già verificati, mostra avanzamento ed errori e non chiede valori mancanti. |
| Modelli | Provider, modelli, usi, endpoint e riferimenti alle credenziali descritti nei documenti prima del setup; configurazioni di esempio supportate e commentate. |
| Embedding | Configurazione locale precompilata come percorso ordinario. |
| Ripresa | Correggere i documenti, ripetere le verifiche e riprendere l'esecuzione senza questionari né perdita del lavoro completato. |
| Workspace | Preparare prima repository e descriptor conformi; dichiarare separatamente i parametri di collegamento ai database secondo i contratti ThothII. |
| Contenuti | Riutilizzare descrizioni/Evidence curate; nessuna generazione AI implicita per completare un documento. Eventuali attività di curation restano esplicite. |
| Evidence | Opzionali nel contratto; se configurate devono essere valide e utilizzabili. |
| Collaudi | Prima Windows, poi Omarchy su PC Intel, infine macOS, in tre tappe distinte. |
## Rilascio e distribuzione delle immagini
La creazione e pubblicazione delle immagini appartengono al processo di rilascio
del progetto. Il sottoprogetto installazione comprende quindi anche una procedura
riproducibile per costruire, verificare e pubblicare queste immagini su Docker Hub;
non basta aggiungere un'opzione di pull senza fornire immagini utilizzabili.
**Stato iniziale dichiarato dall'utente il 28 settembre:** le immagini applicative
ThothII su Docker Hub non sono disponibili. Prima di collaudare l'installazione
precompilata su un PC Windows o Linux occorre produrle, pubblicarle e verificarne
il download. Questa dipendenza è bloccante per quel collaudo, non viene aggirata
usando immagini costruite soltanto nella cache della macchina di test.
Il progetto deve fornire un comando di produzione e pubblicazione, mantenuto nel
repository e documentato per il manutentore. Il comando riceve revisione/versione,
namespace Docker Hub e architetture, costruisce e verifica le immagini proprietarie,
le pubblica e produce il manifest di rilascio con digest e artefatti compatibili.
Il nome e la sintassi saranno fissati nella specifica; il comando non è ancora
implementato. Le credenziali di pubblicazione appartengono al manutentore e non
entrano nei documenti dell'utente che installerà ThothII.
La sequenza di rilascio è: produzione e controlli, pubblicazione su Docker Hub,
verifica del pull del rilascio pubblicato, quindi collaudo della procedura sui
sistemi destinatari. Questa fase del manutentore precede i sei passi dell'utente.
Il pacchetto distribuito deve comprendere tutto ciò che serve all'installazione:
immagini applicative, manifest Compose/configurazione di avvio, migrazioni e risorse
di inizializzazione, oltre al comando operatore o al suo bootstrap. Il numero delle
immagini segue i servizi dell'architettura corrente: non è richiesto accorpare tutto
in un singolo container. I servizi di terze parti mantengono immagini compatibili
con lo stack, senza ricostruirli sul computer dell'utente.
Nell'architettura corrente le immagini proprietarie da pubblicare sono `core` e
`frontend`; PostgreSQL, Qdrant e Ollama usano immagini upstream. Le attività
`catalog-migrate` e `workspace-maintenance` riusano la stessa immagine release di
`core`. Il pacchetto di installazione include anche le risorse oggi montate dal
checkout, fra cui `docker/catalog-db-init.sql` e `docker/embedding-model-init.sh`.
Il download del modello embedding, la creazione dei volumi/database e le migrazioni
restano attività di inizializzazione locale, distinte dalla compilazione.
Ogni rilascio identifica una versione coerente di immagini, CLI e configurazione;
il manifest registra riferimenti verificabili, inclusi i digest delle immagini.
Il setup riporta la versione installata e non combina automaticamente componenti
incompatibili tramite tag mobili. Nomi e namespace Docker Hub saranno definiti
nella specifica di pubblicazione; non sono presunti già esistenti.
Il percorso ordinario deve poter partire dal pacchetto di rilascio senza richiedere
il checkout dei sorgenti applicativi o strumenti di compilazione. Anche l'eventuale
CLI nativa deve essere distribuita già compilata per gli host supportati; nascondere
una compilazione di `tht` nel bootstrap non soddisfa il requisito.
L'installazione pubblica non richiede credenziali di pubblicazione Docker Hub.
Le immagini non includono credenziali dell'installazione, dati personali, workspace
dell'operatore o i database di esempio preinstallati. Configurazione e dati
persistenti vengono creati localmente nei percorsi e volumi dell'installazione.
Il supporto iniziale Windows/WSL2 e Omarchy richiede immagini Linux amd64; per la
tappa macOS Apple Silicon servono immagini Linux arm64 e un bootstrap host adeguato.
La pubblicazione dichiara solo le architetture effettivamente verificate, rispettando
l'ordine dei collaudi concordato.
La modalità sorgente costruisce gli stessi componenti a partire da una revisione
esplicita, documenta i prerequisiti aggiuntivi e usa gli stessi contratti di
configurazione, persistenza e migrazione. Un errore di download da Docker Hub non
deve attivarla automaticamente: l'utente può correggere il problema e riprovare,
oppure scegliere consapevolmente l'alternativa da sorgente.
## Percorso dell'utente
Prima dei sei passi sono disponibili guida, template e strumenti di verifica già
compilati. Git e gli strumenti minimi necessari per acquisire i documenti sono
esplicitati nella guida; la verifica completa dell'host rimane al passo 3. I primi
controlli documentali non devono richiedere l'avvio di ThothII o dei suoi container.
### 1. Preparazione del repository dei workspace
L'utente prepara una copia del repository predefinito con Financial, European
Football e F1, oppure un repository ad hoc a partire da un template documentato.
Il repository predefinito contiene definizioni, documentazione, Evidence e riferimenti
ai pacchetti dati; il clone non equivale ad aver già creato i database PostgreSQL.
La disponibilità effettiva del percorso predefinito dipende dal sottoprogetto esempi.
Si preservano le scelte già approvate sulla copia autonoma o sull'accesso in sola
lettura all'originale; workspace ed Evidence locali restano modificabili. La
preparazione non richiede diritti di push al repository pubblico e non sostituisce
un repository già configurato senza una scelta esplicita.
### 2. Verifica dei documenti dei workspace
Un comando dedicato verifica sintassi YAML, versione/schema ThothII, campi richiesti,
tipi, identificatori, unicità, corrispondenza fra catalogo e directory, descriptor
referenziati, percorsi e file Evidence dove configurati. La verifica è ripetibile
sui file locali prima che Docker o ThothII siano in esecuzione.
La verifica sostanziale copre ciò che è dimostrabile dai documenti: coerenza dei
riferimenti, esistenza e leggibilità dei contenuti, conformità delle Evidence e
assenza di contraddizioni rilevabili. Non pretende di certificare automaticamente
la verità delle regole di dominio o interrogare database non ancora creati.
Ogni errore identifica documento, campo e, quando disponibile, riga, con indicazione
della correzione. L'utente modifica i documenti e ripete il controllo. Le Evidence
restano opzionali: assenza dichiarata ed Evidence configurate ma invalide sono
condizioni diverse. I controlli incrociati che richiedono i parametri applicativi
vengono completati al passo 5.
### 3. Verifica delle precondizioni
Verificare sistema/architettura, Docker e Compose, accesso al daemon, risorse e spazio
richiesti, percorsi e permessi, porte previste, accesso ai servizi esterni e al registry
per quanto valutabile. Su Windows verificare Ubuntu WSL2 e integrazione Docker
Desktop. I requisiti dipendenti da valori scelti al passo 4 sono ricontrollati al 5.
Distinguere componenti necessari sull'host da componenti inclusi nelle immagini:
Pi appartiene a `core`, non è un prerequisito da installare separatamente sul PC.
Presenza e versione di Pi sono verificate nel rilascio; il funzionamento effettivo
nel container è verificato al passo 6. Lo stesso principio vale per le dipendenze
applicative già incluse. Go, Python, Node e compilatori non sono prerequisiti host
del percorso precompilato.
### 4. Preparazione dei parametri applicativi
L'utente compila i documenti locali usando template commentati ed esempi: descriptor
di installazione, configurazione dei modelli e riferimenti ai file protetti `.env`
o equivalenti. Sono espliciti campi obbligatori, opzionali, default e condizioni in
cui un parametro serve. La preparazione può svolgersi in più sessioni senza avviare
il setup o i servizi applicativi.
I documenti definiscono versione del rilascio, percorsi/volumi, profilo e accesso,
repository/workspace selezionati, provider/modelli per i rispettivi usi, endpoint,
embedding, parametri di collegamento ai database e riferimenti ai segreti. La
configurazione dei modelli deriva dall'Installation Model Catalog, senza un secondo
catalogo del setup. L'embedding locale ha un esempio precompilato.
Il descriptor workspace v4 continua a contenere identità ed Evidence: non vi si
inseriscono campi database o modelli estranei al contratto. I binding database sono
predisposti in un input locale separato, da specificare, e applicati al Metadata
Catalog durante il setup mediante i suoi servizi; non diventano una seconda autorità
runtime. La configurazione server/Omics rimane fuori dal perimetro.
I segreti non entrano nel repository pubblico, nei log o nei rapporti di verifica.
Per le credenziali tecniche interne un comando preparatorio può generare file
protetti prima delle verifiche, senza questionario né richiesta durante il setup.
Le selezioni degli esempi e le eventuali operazioni facoltative sono dichiarate
prima dell'esecuzione. Le pagine amministrative restano disponibili dopo l'avvio
per modifiche e curation, non per raccogliere valori obbligatori dimenticati.
### 5. Verifica dei parametri applicativi
Un comando ripetibile controlla completezza e correttezza dei documenti, sintassi
e compatibilità dei valori, riferimenti ai segreti, modelli/usi/default, collegamenti
workspace/database, configurazione Compose, disponibilità del rilascio nel registry
e compatibilità delle architetture. Completa i controlli incrociati dei passi 2 e 3.
Quando fattibile senza creare lo stack, verifica raggiungibilità, autenticazione
e compatibilità delle dipendenze esterne con operazioni circoscritte; la documentazione
spiega le prove effettuate, inclusi eventuali accessi a provider a consumo.
Non cambia dati applicativi né esegue migrazioni o importazioni.
Il rapporto distingue `superato`, `errore`, `avviso` e `non ancora verificabile`,
senza trasformare assenza di verifica in successo. I controlli che richiedono lo
stack sono elencati prima e obbligatori al passo 6, secondo R2 approvata.
Un errore formale o una configurazione obbligatoria mancante blocca l'esecuzione.
Il risultato identifica i documenti e la revisione del repository esaminati. Una
modifica successiva invalida i controlli dipendenti: il setup non deve applicare
file diversi da quelli verificati senza ricontrollarli. Segreti e loro valori
non vengono copiati nel rapporto.
### 6. Setup esecutivo
Il setup ricontrolla l'ammissibilità del piano verificato, scarica le immagini del
rilascio da Docker Hub, crea reti/volumi/container e inizializza i database applicativi.
Esegue migrazioni, applicazione della configurazione e dei binding, sincronizzazione
e preparazione necessaria dei workspace secondo i contratti esistenti. Quando
disponibili e selezionati, crea e carica i database di esempio dal relativo pacchetto.
Non pone domande su provider, modelli, percorsi, credenziali o altri parametri.
Se trova un valore mancante o incoerente, si ferma con una diagnosi e rimanda alla
correzione dei documenti e alla loro verifica; non apre un wizard di riparazione.
Le conferme dei contratti di dominio non vengono aggirate: la specifica deve
distinguere operazioni predisponibili nel piano da attività umane residue senza
reinserire la raccolta dei parametri durante l'installazione.
Esegue i controlli disponibili solo a runtime: salute dei servizi, Pi, modello
embedding effettivamente caricato, collegamenti dalla rete dei container, Catalog
e preparazione dei workspace. Mostra separatamente piattaforma avviata e workspace
pronto, più eventuali verifiche funzionali ancora da svolgere. Il collaudo completo
include una domanda reale con revisione umana, senza SQL target di benchmark.
## Documentazione di accompagnamento
Le guide italiana e inglese seguono esattamente i sei passi. Per ciascuno indicano
input, file da predisporre, template ed esempio compilato, comando di verifica,
esito atteso, errori comuni e passaggio successivo. Un elenco dei documenti e dei
segreti necessari consente di raccogliere le informazioni in anticipo.
Le istruzioni distinguono manutentore del rilascio e utente installatore, host e
container, controlli locali e runtime. Il percorso sorgente è esplicitamente
alternativo; un download fallito non ne provoca l'attivazione automatica.
## Avanzamento, errori e ripresa
La procedura conserva l'installazione di riferimento, gli input verificati, i
passaggi completati, l'ultimo errore e il prossimo passo, senza conservare segreti
nello stato di avanzamento. Alla ripresa verifica lo stato reale: una vecchia spunta
non prova che servizio, credenziale o indice siano ancora validi.
Correggere un endpoint o una credenziale non impone di rifare tutto il setup. La
specifica deve definire quali verifiche dipendenti vanno ripetute, preservando
configurazioni, workspace, sessioni e contenuti non coinvolti nella modifica.
Le modifiche manuali vengono riconosciute e spiegate, non cancellate implicitamente.
La ripresa riguarda le tappe della procedura: non promette una ripresa interna
di operazioni che non la supportano, come il preprocessing corrente. In questi
casi il passaggio viene rieseguito in modo coerente con il suo contratto.
Un fallimento riporta fase, causa comprensibile, correzione consigliata e modalità
di ripresa. Distinguere problemi dell'host, dei servizi locali e delle dipendenze
esterne. Un provider o DWH indisponibile non annulla una verifica valida della
piattaforma, ma impedisce di dichiarare il percorso complessivo pronto.
## Criteri di accettazione
| Scenario | Esito verificabile |
| --- | --- |
| Rilascio Docker Hub | Immagini costruite e pubblicate con versione, digest e architetture dichiarate; avvio verificato usando quanto effettivamente scaricato dal registry. |
| Installazione precompilata | Su host senza toolchain e senza sorgenti applicativi, il setup scarica le immagini e completa configurazione/avvio senza compilare né invocare build locali. |
| Risorse di avvio | Compose, migrazioni e inizializzazione non dipendono da file presenti soltanto in un checkout dei sorgenti. |
| Download fallito | Errore comprensibile e ripetibile; nessuna build da sorgente avviata implicitamente. |
| Modalità sorgente | Scelta esplicita ancora funzionante, con gli stessi contratti di configurazione e persistenza del rilascio precompilato. |
| Preparazione documentale | Template e guida consentono di predisporre tutti i YAML e file protetti necessari prima dell'esecuzione. |
| Ordine dei passi | Repository, verifica workspace, precondizioni, parametri applicativi, verifica parametri, setup esecutivo. |
| Verifica workspace senza stack | Il passo 2 funziona senza Docker attivo o applicazione installata e senza toolchain host aggiuntive. |
| Controlli ripetibili | Ripetere i controlli non crea container, non migra DB e non modifica i documenti dell'utente. |
| Completezza prima del setup | Un campo obbligatorio mancante blocca l'esecuzione, indicando file/campo e correzione; nessuna domanda interattiva. |
| Input modificati | I controlli dipendenti sono invalidati o ripetuti; nessuna applicazione di input diversi da quelli verificati. |
| Setup non interattivo | Con documenti completi il passo 6 termina senza richiedere input da terminale; un errore non apre un questionario. |
| Verifiche sostanziali | Il rapporto distingue prove eseguite da controlli non ancora possibili; non certifica verità di dominio o servizi non verificati. |
| Modello configurato | Credenziali/endpoint verificati e selezione valida per gli usi richiesti. |
| Credenziale errata | Diagnosi senza esporre il segreto, correzione nel file protetto e nuova verifica prima della ripresa. |
| Interruzione e riavvio | La procedura ricontrolla gli input e riprende dall'avanzamento verificato senza nuove domande sui parametri. |
| Contenuti esistenti | Descrizioni curate ed Evidence locali preservate; nessuna generazione o sovrascrittura silenziosa. |
| Evidence assenti | Nessun blocco dovuto alla sola assenza quando non sono configurate. |
| Workspace pronto | Connessione, schema e preparazione necessaria verificati; domanda reale completata con revisione umana. |
| Riesecuzione | Nessuna duplicazione, azzeramento di dati o perdita delle personalizzazioni. |
| Stop/start | Accesso e workspace utilizzabile persistono; i problemi esterni sono segnalati separatamente. |
| Segreti | Assenti da log, riepiloghi, file pubblici e stato di avanzamento. |
Il dettaglio dei controlli e i test automatici devono rispettare le API e i contratti
attuali; i test di portabilità e il collaudo con servizi reali restano distinti.
La prima tappa usa Windows x64/WSL2, la seconda Linux Omarchy x64, la terza macOS
Apple Silicon, coerentemente con le architetture già previste dal progetto.
Gli esiti di una tappa non valgono come collaudo delle successive.
## Sottoprogetto degli esempi
L'implementazione rimane rinviata, come confermato con R1. I requisiti sono conservati sul branch
`codex/benchmark-examples`, commit `08a5db55`. Il setup non offre come disponibili
CLI, database o modalità di copia del repository non ancora implementati.
Il percorso richiesto ora parte dal repository predefinito oppure da quello ad hoc.
Il primo collaudo utilizza un repository ad hoc; il percorso predefinito completo
arriverà con il sottoprogetto esempi. Quando disponibili e selezionati nei documenti,
creazione e caricamento dei database avvengono nell'installazione
dell'utente a partire dai pacchetti dati verificati, senza compilare l'applicazione
o incorporare quei database nelle immagini. Il percorso ad hoc può essere collaudato
con workspace/database disponibili. La disponibilità di immagini Docker Hub è
invece un prerequisito esplicito di ogni collaudo del percorso precompilato.
## Confini della specifica successiva
La verifica del codice ha individuato questi punti di integrazione concreti:
- `compose.yaml` costruisce oggi `core` e `frontend`; i servizi di manutenzione
usano l'immagine core locale. `setup --complete` esegue una build
(`tools/tht/internal/setup/run.go:77`) e `scripts/install-tht.sh:84` compila la CLI
tramite Docker se non riceve un artefatto già costruito. Il percorso ordinario
deve sostituire entrambi i comportamenti con artefatti del rilascio. La CI
`.github/workflows/container-multiarch.yml` verifica già entrambe le architetture
Linux; occorre aggiungere la pubblicazione Docker Hub e la verifica dei pacchetti
effettivamente distribuiti.
- I parser workspace (`backend/src/workspaces/catalog.ts:52`, `schema.ts:382`)
verificano i documenti senza dipendenze dal runtime, ma non sono oggi un comando
preinstallazione nativo. Devono essere resi disponibili negli strumenti
precompilati preservando gli stessi contratti, senza richiedere Node sul PC.
- La configurazione database è applicata al descriptor runtime dal Catalog
(`backend/src/catalog/runtime-binding.ts:33`). Serve un input locale preparatorio
e un percorso di applicazione al Catalog, senza cambiare il significato del
workspace v4 o creare due autorità persistenti per i binding.
- `config.Load` contiene verifiche riutilizzabili su YAML e file ambiente
(`tools/tht/internal/config/installation.go:100`); il `doctor` corrente combina
controlli documentali e runtime (`tools/tht/internal/doctor/report.go:97`).
La specifica deve separarli per rendere disponibili i gate dei passi 2, 3 e 5.
- L'ammissione della sessione controlla modello, preprocessing e servizi necessari
(`backend/src/routes/sessions.ts:438`); il preprocessing richiede un database
associato e una sincronizzazione corrente. Le descrizioni generate dall'AI
non costituiscono un requisito generale: possono essere già disponibili
descrizioni curate o commenti della sorgente. Riferimento:
[contratto di preprocessing](../contracts/workspace-preprocessing-cli.md).
La specifica tecnica dovrà definire build/pubblicazione Docker Hub, pacchetto di
rilascio e bootstrap precompilato, selezione esplicita del percorso sorgente,
template e documenti locali, validatori senza runtime, applicazione dei parametri
al Catalog, esecuzione senza questionari, stato/ripresa e verifiche dei due traguardi.
Nomi dei comandi di preparazione/verifica/pubblicazione e formato dello stato sono
dettagli da progettare, non funzionalità esistenti attestate da questo PRD.
## Chiarimenti approvati del 28 settembre
| ID | Scelta | Decisione approvata |
| --- | --- | --- |
| R1 | Disponibilità dei tre esempi al primo rilascio | Conservare il rinvio del sottoprogetto e collaudare prima il percorso con repository ad hoc; introdurre il percorso predefinito completo quando gli esempi sono disponibili. |
| R2 | Controlli impossibili prima della creazione dello stack | Bloccare gli errori rilevabili prima; elencare i controlli runtime non ancora eseguibili e renderli obbligatori al passo 6, senza contarli come superati preventivamente. |
Approvazione: «ok per le tue proposte. dopodichè procedi con to-spec».
Il principio dei sei passi resta invariato.
Non rientrano in questo lavoro un nuovo installer grafico, la riscrittura delle
superfici amministrative, il supporto multi-repository, la distribuzione dei
database di esempio o modifiche al deployment server/Omics.
@@ -0,0 +1,155 @@
# Ripresa dell'installazione guidata
Stato al 28 settembre: revisione documentale in sei passi richiesta dall'utente;
raccolta interattiva dei parametri durante il setup superata. R1/R2 confermate,
`grill-with-docs` e `/to-spec` conclusi, piano di test confermato dall'utente.
Il riferimento consolidato è il
[PRD dell'installazione guidata](2026-09-27-guided-installation-prd.md).
La [specifica derivata](2026-09-28-document-first-installation-spec.md) è pubblicata
su Gitea come
[Spec: installazione da documenti verificati e distribuzione Docker Hub](https://git.tylconsulting.it/mptyl/ThothII/issues/42),
con etichetta `ready-for-agent`. `/to-tickets` è completato: la
[scomposizione approvata](2026-09-28-installation-ticket-breakdown.md) collega le
issue #43–#54, con dipendenze native verificate e parent invariata.
Prossimo passo: `/implement` su #43 o #46, inizialmente senza blocchi.
Nessun codice applicativo è stato implementato in questi passaggi.
## Base di lavoro
Branch `codex/guided-standalone-install`, commit iniziale `67ee5262`.
Il branch conserva il setup guidato e i controlli workspace incompleti, separati dal
rilascio server. Il piano del 14 settembre descrive il precedente percorso manuale;
le decisioni qui registrate aggiornano l'obiettivo del lavoro ripreso.
## Decisioni confermate il 27 settembre 2026
- Destinatario: una persona capace di installare Docker, clonare un repository e
compilare i valori richiesti, senza conoscere l'architettura di ThothII.
- Risultato: almeno un workspace utilizzabile per una domanda reale, attraverso due
traguardi espliciti: piattaforma installata e workspace pronto. La procedura guida
le decisioni umane necessarie e può essere ripresa senza ricominciare.
- Collaudo in tre tappe distinte e ordinate: prima Windows, poi Linux Omarchy su PC
Intel, infine macOS. Non attribuire a una piattaforma gli esiti ottenuti su un'altra.
- Distribuzione ordinaria tramite immagini applicative precompilate pubblicate su
Docker Hub; build e pubblicazione fanno parte del progetto. L'installazione locale
configura e inizializza lo stack senza compilare; l'alternativa da sorgente resta
esplicitamente disponibile. Anche il comando host deve essere fornito precompilato
nel percorso ordinario. La creazione/caricamento degli esempi resta un'integrazione
prevista, con l'implementazione del relativo sottoprogetto ancora rinviata.
## Vincoli già documentati
Il precedente piano prevedeva clone Gitea e build locale; la decisione successiva
li mantiene come alternativa al percorso precompilato Docker Hub. Restano Docker
Compose e, su Windows, Ubuntu WSL2 con integrazione Docker Desktop.
Il workspace descriptor contiene identità ed
Evidence; configurazione database e metadati appartengono al Metadata Catalog.
L'Installation Model Catalog appartiene all'installazione.
## Revisione del 28 settembre — prevale sul percorso interattivo
La preparazione e le verifiche precedono l'esecuzione, nell'ordine richiesto:
1. Preparare il repository workspace predefinito con i tre esempi oppure uno ad hoc.
2. Verificare formalmente e, per quanto possibile, sostanzialmente i documenti workspace.
3. Verificare le precondizioni dell'host e dei componenti previsti.
4. Predisporre i documenti locali YAML/.env con i parametri dell'applicazione.
5. Verificarne completezza, correttezza e coerenza con i workspace.
6. Eseguire il setup dai documenti verificati, scaricando le immagini Docker Hub
e creando lo stack, senza domande sui parametri.
Template ed esempi commentati e guide IT/EN accompagnano ogni fase. Le verifiche
devono essere ripetibili prima di creare i container. Pi è incluso in `core`, non
deve essere installato sull'host; i suoi controlli runtime avvengono dopo l'avvio.
I binding database restano di competenza del Catalog, con un input locale
preparatorio distinto dai descriptor workspace v4.
L'utente conferma che le immagini Docker Hub oggi non esistono: occorre un comando
di produzione/pubblicazione per il manutentore, poi verificare il pull degli
artefatti pubblicati prima di collaudare l'installazione precompilata sui PC.
La preparazione del rilascio non è un compito dell'utente installatore.
R1/R2 confermate: esempi ancora rinviati e primo collaudo con repository ad hoc;
controlli runtime elencati prima ed eseguiti obbligatoriamente dopo la creazione
dello stack. Nessun controllo non eseguito conta come superato.
## Aspetti da tradurre nella specifica tecnica
- Quali template e documenti locali preparare, verificare e applicare, senza
raccogliere parametri durante l'esecuzione.
- Come distinguere validazione documentale, controlli preventivi esterni e controlli
runtime, mantenendo i contratti delle superfici amministrative esistenti.
- Criteri dettagliati di verifica, ripresa dopo errori e accettazione per ogni tappa.
## Sottoprogetto esempi: requisiti definiti, implementazione rinviata
Su richiesta dell'utente l'implementazione dei database di esempio viene rinviata;
la definizione dei requisiti dell'installazione prosegue indipendentemente.
Branch dedicato: `codex/benchmark-examples`, creato da `67ee5262`, con documentazione
consolidata nel commit `08a5db55`. Il PRD approvato è
`docs/plans/2026-09-27-example-databases-prd.md` su quel branch.
Il PRD conserva D1–D8: Financial, European Football e F1, tre workspace separati,
dati PostgreSQL e schema commentato, Evidence curate e domande di accompagnamento
senza SQL target, contenuti italiano/inglese. CLI scaricabile dal repository pubblico
degli esempi Gitea gestito da TYL Consulting, collegato dal repository pubblico
ThothII. Selezione di uno, due o tre esempi dopo il setup, oppure come ultimo passo
facoltativo dello stesso setup; pacchetti PostgreSQL già verificati ove redistribuibili.
La copia del repository potrà essere indipendente, senza storia e collegamenti Git
all'originale, oppure scaricata con accesso al repository pubblico in sola lettura.
Workspace ed Evidence locali restano modificabili. Preparazione e verifiche sono
responsabilità del progetto; all'utente vengono sottoposte solo ambiguità non
risolvibili dalle fonti. Chi installa dovrà trovare gli esempi pronti all'uso.
Il caricatore, il supporto alla cartella `examples/` e alla copia autonoma sono da
implementare. Finché il sottoprogetto è rinviato, il setup non deve offrirli come
funzionalità disponibili. Il requisito di un workspace utilizzabile resta valido:
per il collaudo si dovrà usare un workspace/database realmente disponibile.
## Verifiche da completare
Riesecuzione e fallimenti intermedi del setup; requisiti HTTPS per il repository
workspace; test dedicati dei nuovi comandi; distinzione fra stato della piattaforma
e Workspace Readiness; collaudo reale completo secondo l'ordine concordato.
## Riscontro sul setup corrente
- CLI guidata e pagine amministrative esistenti sono già il percorso previsto dal
piano del 14 settembre; non è richiesto un nuovo installer grafico.
- `setup --complete` prepara autenticazione e modelli, build, migrazioni Catalog,
avvio, workspace pull, test Pi e doctor. Non configura binding DB, sincronizzazione
dei metadati e preprocessing (`tools/tht/internal/setup/run.go:60`). Il messaggio
finale corrente «ThothII is ready» deve essere allineato al traguardo verificato.
- I modelli sono oggi preimpostati, senza selettore del provider nel Request
(`tools/tht/internal/setup/files.go:449`).
- La riesecuzione rifiuta file di configurazione esistenti con contenuto diverso;
non equivale ancora a una ripresa guidata delle tappe tecniche e umane
(`tools/tht/internal/setup/files.go:107`). La specifica deve prevedere stato delle
tappe e gestione delle correzioni, senza sovrascrivere personalizzazioni.
## Round installazione I1–I3 del 27 settembre — storico superato dove indicato
La revisione del 28 settembre sopra prevale su I1/I2: nessun rinvio di parametri
obbligatori al setup e nessuna raccolta tramite questionario. Il testo seguente
conserva il contesto della decisione precedente e non è il comportamento richiesto
per la nuova procedura. I3 resta valido per riuso dei contenuti e curation esplicita.
L'utente conferma «tutto come da te suggerito», dopo il chiarimento sulla CLI locale
interattiva: domande condizionate alle risposte, configurazioni precompilate,
credenziali protette, verifica delle connessioni, possibilità di rinviare una
configurazione e ripresa senza ricominciare. La CLI guida alle pagine amministrative
esistenti per il workspace e ne verifica il completamento.
| ID | Decisione | Scelta approvata |
| --- | --- | --- |
| I1 | Primo avvio senza repository/workspace disponibile | Consentire di completare il solo traguardo «piattaforma installata» e riprendere in seguito la configurazione del workspace. Il percorso complessivo resta incompleto fino al primo workspace utilizzabile e alla domanda reale. Nessuna dipendenza dalla futura disponibilità degli esempi. |
| I2 | Scelta dei modelli durante il setup | Selezione guidata di provider e modello tra configurazioni supportate/precompilate, chiedendo le credenziali necessarie; percorso avanzato per configurazioni personalizzate. Embedding locale preconfigurato come scelta iniziale. |
| I3 | Preparazione di descrizioni ed Evidence | Riutilizzare i contenuti già curati. Proporre la generazione AI delle descrizioni mancanti come scelta esplicita, con revisione umana, invece di avviarla automaticamente. Guidare alle pagine amministrative necessarie e registrare il punto di ripresa. |
Le Evidence sono opzionali nel contratto del workspace: non introdurre un obbligo
generale di crearle per completare l'installazione. La specifica deve rispettare
i controlli di Workspace Readiness su connessione, schema e indicizzazione,
distinguendo assenza lecita di Evidence da configurazione incompleta o incoerente.
Per le decisioni correnti e i criteri di accettazione fa fede il PRD revisionato
al 28 settembre, senza attestare che siano già implementati.
@@ -0,0 +1,387 @@
# Spec: installation from validated documents and Docker Hub releases
## Problem Statement
An operator who understands Docker and can edit a documented configuration should
not have to discover ThothII's architecture while answering an installation wizard.
The operator needs time to prepare workspace and application documents, validate
them repeatedly, and correct errors before applying changes to the machine.
The current setup combines document generation, local builds, startup and runtime
diagnostics. Its success message can precede an actually usable workspace. It does
not provide the complete preinstallation validation and resumable, non-interactive
execution required by this workflow.
The ordinary installation must consume published, prebuilt application images.
As reported by the project owner on 28 September 2026, ThothII images are not yet
available on Docker Hub. Publishing and verifying a real release is therefore a
prerequisite for testing the consumer installation, rather than a later enhancement.
## Solution
Deliver a documented six-step workflow, in this exact order:
1. Prepare a workspace repository: a project-specific repository initially, or the
default examples repository when that separately deferred project is available.
2. Validate workspace documents formally and substantively to the extent possible
from their contents and available references.
3. Verify host and distribution prerequisites.
4. Prepare application parameters in installation-local YAML and protected
environment/secret documents, using commented templates and complete examples.
5. Validate application parameters and their consistency with the selected workspaces.
6. Execute the validated installation without asking configuration questions: pull
the release images, create and initialize the stack, apply declared configuration,
and run the required runtime checks.
Before these consumer steps can be tested against Docker Hub, a maintainer command
must build, check and publish the release and verify that its artifacts can be pulled.
Local source builds remain an explicit alternative, with the same configuration and
persistence contracts. A failed pull never silently switches to source compilation.
Checks that cannot run before container or database creation are enumerated as
deferred obligations and must run after startup. Errors detectable beforehand block
execution. Platform acceptance, Workspace Readiness and functional acceptance are
reported separately. A real question with human review completes functional acceptance.
## User Stories
1. As an installer, I want a six-step guide, so that I can understand the whole process before changing my machine.
2. As an installer, I want commented templates and completed examples, so that I can prepare documents without knowing internal component names.
3. As an installer, I want to pause document preparation, so that I can obtain missing information without restarting an installer.
4. As an installer, I want to prepare a project-specific workspace repository, so that I can use my own database before the example databases are delivered.
5. As an installer, I want the future default repository to be clearly distinguished from available features, so that I am not directed to unavailable examples.
6. As an installer, I want read-only access to the source workspace repository, so that consuming a workspace does not require publication rights.
7. As a workspace author, I want local document validation before Docker starts, so that syntax and contract errors are inexpensive to correct.
8. As a workspace author, I want duplicate identifiers, unsupported fields and inconsistent references reported, so that formally valid YAML does not hide an invalid workspace.
9. As a workspace author, I want errors to identify the document and field, so that I know precisely what to edit.
10. As a workspace author, I want optional Evidence distinguished from invalid configured Evidence, so that an intentionally absent corpus does not block installation.
11. As a workspace author, I want semantic validation limits stated honestly, so that successful validation is not mistaken for certification of domain knowledge.
12. As an installer, I want host prerequisites checked explicitly, so that Docker, architecture, permissions and resource problems are detected before setup.
13. As a Windows installer, I want a verified WSL2 and Docker Desktop path, so that I can follow one supported installation procedure.
14. As an installer, I want Pi supplied with the application image, so that I do not have to install an unnecessary host dependency.
15. As an installer, I want application models, endpoints and credentials prepared before execution, so that I can consult colleagues or provider documentation at my own pace.
16. As an installer, I want one authored model catalog, so that provider and embedding configuration do not disagree across components.
17. As an installer, I want database bindings prepared locally and separately from workspace definitions, so that environment-specific details do not leak into shared workspace repositories.
18. As an installer, I want generated internal credentials prepared in protected files, so that I need not invent technical passwords during execution.
19. As an installer, I want my secrets excluded from logs and validation reports, so that diagnostic output is safe to inspect and share.
20. As an installer, I want repeatable application validation, so that I can correct configuration without creating containers or changing databases.
21. As an installer, I want available external connections checked before startup, so that preventable endpoint or authentication errors are found early.
22. As an installer, I want non-executable checks listed explicitly, so that I know what still needs to be proved after startup.
23. As an installer, I want validation tied to the documents and release I selected, so that execution cannot silently apply different inputs.
24. As an installer, I want setup to run with closed standard input, so that it cannot unexpectedly ask me for a parameter.
25. As an installer, I want missing values to stop setup with a useful diagnosis, so that I can correct the document and validate again.
26. As an installer, I want prebuilt images downloaded from Docker Hub, so that I need no application source checkout or compiler.
27. As an installer, I want the operator tool supplied precompiled, so that its bootstrap does not hide a local build.
28. As an installer, I want release components to be compatible and identifiable, so that my installation is reproducible.
29. As an installer, I want registry failures reported without automatic compilation, so that the selected installation mode remains predictable.
30. As an installer, I want configuration, credentials and data kept outside application images, so that my installation remains local and persistent.
31. As an installer, I want migrations and database binding initialization handled explicitly, so that a running container is not mistaken for an initialized application.
32. As an installer, I want existing curated descriptions and Evidence preserved, so that rerunning setup cannot overwrite human work.
33. As an installer, I want runtime checks performed from the actual container environment, so that host connectivity is not confused with application connectivity.
34. As an installer, I want completed and failed execution stages recorded, so that I can resume after interruption without duplicating data.
35. As an installer, I want configuration corrections to invalidate dependent checks, so that resuming does not trust stale results.
36. As an administrator, I want Catalog changes to retain their existing confirmation and concurrency rules, so that installation automation does not bypass domain safeguards.
37. As an installer, I want separate platform and workspace status, so that I know whether I can already ask a real question.
38. As an installer, I want stop/start behavior checked, so that a successful first run is not the only working state.
39. As a maintainer, I want an explicit release build-and-publish command, so that consumer installation can use actual Docker Hub artifacts.
40. As a maintainer, I want versioned images and release metadata, so that published artifacts can be traced to a source revision.
41. As a maintainer, I want publication credentials isolated from consumer configuration, so that installers need no write access to Docker Hub.
42. As a maintainer, I want the published artifacts tested by pulling them, so that an unpublished local image cannot satisfy release acceptance.
43. As a source-build user, I want an explicit supported build path, so that source customization remains possible.
44. As a maintainer, I want Windows, Omarchy and macOS acceptance recorded separately and in order, so that support claims reflect tests actually performed.
45. As an Italian or English reader, I want matching step-by-step guides, so that language choice does not change the installation contract.
## Implementation Decisions
### Boundaries and authority
- Keep the existing standalone architecture and operator CLI. The application runs
through Compose; the operator CLI owns preparation, validation and execution of
the installation, rather than introducing another installer or backend workflow.
- Preserve Workspace schema v4 and the workspace catalog as the authority for
workspace identity and optional Evidence. Formal validation uses the same rules
as runtime loading, including strict YAML interpretation, duplicate detection,
directory/index consistency and prohibited fields.
- Preserve the Installation Model Catalog as the sole authored source for provider,
model eligibility, defaults and embedding facts. Session and embedding configuration
must be complete before ordinary execution. Metadata generation remains optional
when its catalog configuration is omitted consistently.
- Preserve the PostgreSQL Metadata Catalog as the authority for Workspace Database
identity, schema, metadata and active Database Binding. Respect the existing
one-database-per-workspace boundary. Prepared binding documents are bootstrap
inputs, not a second runtime database catalog.
- Preserve installation-local secret storage, reference-based binding credentials,
editable Evidence authority, read-only DWH access and the separation of reference
preprocessing from Memory. Do not rewrite model projections or runtime snapshots
as independent authored configuration.
### Preparation documents and command surfaces
- Expose distinct operator operations for template preparation, workspace validation,
host checks, application validation, execution, status and resumption. Their exact
CLI spelling can be finalized with the implementation tickets; they must remain
separately invocable and scriptable. Setup execution is never a parameter wizard.
- Template preparation creates explicitly requested sample documents or protected
credential files without starting application services. It never silently replaces
existing user documents. Placeholders are visibly incomplete and cannot pass the
required-field checks.
- Supply a versioned, installation-local database bootstrap document describing the
workspace identity, engine, physical database/schema, transport and endpoint
configuration, and references to secrets. Validate against existing Catalog
capabilities and selected workspace identities. No credentials belong in the
shared workspace descriptors or public repository.
- Binding imports use authenticated, authorized Catalog services and their version
checks. On first execution they create the declared bindings and install secrets
through the existing store. On rerun, equivalent values are a no-op; conflicting
existing administrative changes stop with a reconciliation report. The bootstrap
input does not continuously overwrite a mutable Catalog.
- Initial local administrative authentication and its protected bootstrap material
are prepared before execution. Execution cannot depend on a person answering a
login wizard. Reuse supported operator authentication boundaries and required
permissions, without introducing a privileged unauthenticated installation API.
- Supply a precompiled validation capability with the operator distribution. The
workspace validation step must work without Docker, ThothII, Node or an application
source checkout. Reuse canonical validation rules; if a new packaging boundary is
necessary, prove equivalence with a shared set of valid and invalid documents.
### Validation contract
- Workspace validation covers syntax, strict schema, identity, catalog/descriptor
relationships and locally available Evidence references. It does not claim to
establish the truth of domain rules, database contents or services that do not exist.
- Host checks distinguish host prerequisites from bundled application dependencies.
Pi is checked as a release component and subsequently in the running core image,
never required as a separate host installation.
- Application validation checks required configuration, compatible release and host,
model usages/defaults, protected secret references, database transport capabilities,
workspace links, effective Compose configuration and accessible remote dependencies.
A transport that cannot serve NL-to-SQL sessions cannot pass workspace-readiness
validation merely because it can perform administrative diagnostics.
- Reports expose a stable outcome, check identifier, affected logical input/field,
explanation and next action. The public outcomes are passed, error, warning and
deferred-to-runtime. Human-readable output is accompanied by pristine structured
output for automation; exit status distinguishes success from blocking failure.
- Validation is read-only with respect to user documents, application state and
target databases. Explicit report output is allowed. Remote checks are bounded,
documented and non-mutating; any provider usage incurred by a configured smoke
test is disclosed before invocation, not requested interactively during setup.
- Missing or invalid required configuration, missing release artifacts, unsupported
architecture and failures of available required dependencies block execution.
Unreachable existing external services are errors, not automatically reclassified
as deferred. Only checks intrinsically dependent on the not-yet-created local
stack qualify for the accepted deferred category.
- Maintain an explicit obligation list for deferred checks: container-network
connectivity, Catalog initialization, Pi operation, local embedding availability,
preprocessing and relevant workspace runtime readiness. Each obligation has a
defined runtime check; there is no successful final state while a required
obligation remains unverified or failed.
- Bind the execution plan to normalized non-secret configuration, repository revision
and verified content, selected release digests and validator version. Re-read
protected credentials when checking or executing; do not expose their values or
unkeyed secret-derived fingerprints in reports. Re-run credential checks where
freshness cannot be established safely.
- Revalidate changed dependencies and live prerequisites at execution or resume.
A previously successful report is not blanket authorization to apply changed files
or evidence of current network availability.
### Release production and distribution
- Provide a maintainer command accepting source revision, release version, Docker Hub
namespace and target architectures. It performs preflight checks, reproducible
builds, artifact checks, publication and output of a coherent release manifest.
This command is a deliverable of this feature, not an undocumented manual prerequisite.
- Publish the existing core and frontend application images. Catalog migration and
workspace maintenance use the same released core image. Keep PostgreSQL, Qdrant
and Ollama as compatible upstream images; preserve the existing service boundaries.
- Package the operator executable, validation capability, Compose definitions,
initialization resources and migration support required by the release. No runtime
mount may require a resource that exists only in an application source checkout.
- Pin the release identity and resolved image digests. A published release manifest
binds compatible images and operator/configuration versions. Do not overwrite an
already published immutable release version or declare a partially published
image set installable. An interrupted publication can retry without advertising
an incomplete consumer release.
- Keep publishing credentials outside consumer bundles and logs. Public consumers
pull without publishing rights. Application images contain application software,
not installation secrets, user workspace data or prepopulated example databases.
- The ordinary bootstrap downloads a precompiled operator and release artifacts.
It must not compile the CLI through Docker as a hidden fallback. Windows WSL2
uses the Linux executable; macOS uses an appropriate host executable.
- Deliver and accept Linux amd64 images for Windows/WSL2 and Omarchy first. Add and
accept Linux arm64 for the macOS Apple Silicon stage. Multiarchitecture build
results do not by themselves prove host installation acceptance.
- A maintainer smoke test pulls the published artifacts by their release references.
Consumer acceptance runs must not succeed because of an unpushed locally built
image. Confirm core, frontend and maintenance references all resolve to the release.
- Retain explicit source mode with the same configuration, validation and persistence
rules. It is not the default and is never an automatic recovery action for a pull
failure. Registry recovery is a retry of the selected released artifacts.
### Non-interactive execution and recovery
- Execute only a complete, currently validated plan with no blocking errors. The
ordinary sequence pulls the release, prepares runtime projections and isolated
installation storage, starts required services, applies migrations, registers the
workspace source and imports the prepared Catalog bindings before dependent work.
- Apply configuration and migrations through existing service boundaries. Hold an
installation execution lock to prevent concurrent runs from racing over the same
state, containers or bootstrap imports.
- Reuse durable Catalog Sync Runs and their freshness/locking rules. Fresh additive
synchronization can proceed under the existing contract. A destructive diff or
another domain-required human decision stops at an explicit awaiting-review state;
the operator reviews through the existing administration surface and subsequently
resumes. This is a domain decision, not permission to collect missing setup parameters
or add an automatic confirmation bypass.
- Reuse existing curated metadata and Evidence. Do not generate AI descriptions or
new domain rules as an installation side effect. Optional generation remains a
separate explicit administrative action. Required preprocessing can use supported
source comments or curated descriptions without mandatory AI generation.
- Persist a bounded execution journal with installation identity, plan identity,
stage outcomes, released component versions, deferred-check outcomes and recovery
guidance. Do not persist secret values or raw exception output. Write progress
atomically and check actual state on resume.
- A repeated execution of the same completed plan must not recreate bindings,
duplicate data, clear Memory, overwrite Evidence or erase sessions. Reconcile
already completed stages with their actual persistent state before proceeding.
- After a document correction, revalidate and repeat only affected checks/stages.
Do not infer that an interrupted migration or import failed before inspecting its
durable result. Preprocessing, which has no internal resume contract, may need to
rerun as a whole; report that honestly.
- Never implement recovery by deleting all volumes or reverting user data. Report
the failing stage and safe next action. Failed or interrupted execution is not
advertised as a complete installation.
- Report platform state, Workspace Readiness and functional acceptance separately.
Runtime checks exercise the released Pi, embedding and actual container transport.
Final acceptance includes a real human-reviewed question and stop/start persistence.
The workflow's human review is not replaced by unattended benchmark evaluation.
### Documentation and delivery boundaries
- Keep Italian and English guides aligned with the six steps. Each step states its
inputs, documents, examples, verification operation, expected output and common
corrections. Provide an advance checklist of information and credentials to collect.
- Separate maintainer publication instructions, consumer prebuilt installation and
source-build instructions. State precisely which components are host prerequisites
and which are shipped inside the release.
- First acceptance uses an available project-specific repository and database.
The Financial, European Football and F1 project stays deferred. Do not expose its
unavailable loader, repository-copy support or data bundles as usable features.
- Preserve the future integration boundary: examples will be selected in documents
and loaded locally after application initialization, without rebuilding the
application or embedding the datasets into its Docker images.
## Testing Decisions
### Confirmed test boundaries
Use the public operator workflow as the principal test boundary: prepared documents
in, stable reports/exit statuses and observable installation outcomes out. Exercise
validation and execution through this boundary while replacing external command
execution and remote services with controllable test counterparts. Keep focused
contract tests at existing workspace parsing and Catalog boundaries where they
prevent divergent schemas or authority rules. Add real release and host acceptance
tests for behavior that simulated external services cannot establish.
The project owner confirmed this testing boundary on 28 September 2026, completing
the `/to-spec` checkpoint. The product decisions, six-step workflow and testing
scope are approved for specification publication and subsequent ticket decomposition.
### Existing testing practice to extend
- Operator setup tests already use temporary installation fixtures and a replaceable
command runner to cover sequencing, startup failures, error preservation and recovery
messages. Extend that boundary to document validation, prebuilt execution and resumption.
- Installation configuration tests cover strict schemas, model catalog rules and
incompatible existing files. Extend them with complete/incomplete preparation
documents, protected references and cross-document consistency.
- Workspace tests cover strict catalog/descriptor parsing, immutable Git revisions,
runtime handoff and secret handling. Reuse their document fixtures and validity
rules to demonstrate equivalence of the preinstallation validator.
- Catalog and preprocessing tests already cover durable runs, locks, binding freshness,
failure recording and preservation of authoritative state. Reuse these boundaries
to verify bootstrap import and resumption without bypassing the domain contracts.
- Existing multiarchitecture image checks provide a starting point for released
artifact verification. They do not replace pulling the published artifacts or
testing supported host environments.
### Required behavioral coverage
1. Validate workspace documents with Docker absent and no application runtime;
reject malformed YAML, duplicate keys/identifiers, unsupported fields, broken
references and invalid configured Evidence with actionable locations.
2. Repeated document/host/application checks do not create containers, change source
documents, import data or migrate databases. Only explicitly requested reports
may be written by validation.
3. Complete application documents pass; placeholders, missing required models,
invalid binding transport and unreadable secret references block execution.
4. Existing external-service failures remain errors. Checks genuinely dependent on
newly created local services are listed as deferred and cannot disappear from
final acceptance.
5. Execute with standard input closed. Valid inputs need no responses; missing
values yield an error without waiting for input or prompting for a replacement.
6. Changing documents, workspace revision or release after validation invalidates
dependent results. Credential changes are caught without leaking secret material.
7. A clean prebuilt consumer installation performs pulls and initialization, never
application or operator compilation; absent images fail without a source fallback.
8. Published manifests resolve all required images, platform variants and maintenance
components coherently. Simulated publication interruption does not advertise a
partial release; a real smoke test exercises artifacts pulled from Docker Hub.
9. Prepared database bindings become Catalog state once, use the existing protected
secret store and remain unchanged on equivalent reruns. Administrative divergence
is reported rather than silently overwritten.
10. Interruption after a durable operation but before journal completion resumes by
inspecting state, without duplicating that operation. Concurrent setup runs cannot
mutate the same installation simultaneously.
11. Destructive Catalog synchronization requires its existing review and fresh source
checks. The setup's non-interactive nature does not auto-approve a destructive diff.
12. Existing descriptions, Evidence, sessions and Memory survive validation, rerun,
configuration correction and stop/start. Optional AI generation is not triggered.
13. Runtime Pi, embedding and connectivity checks are executed from the installed
release, and platform success cannot mask failed Workspace Readiness.
14. Source mode remains functional and explicit with equivalent configuration
contracts. A source-mode pass cannot close prebuilt distribution acceptance.
15. Execute real consumer acceptance first on Windows x64/WSL2, then Omarchy x64,
then macOS Apple Silicon. Record each environment, release identity, stage
outcomes, a real reviewed question and a stop/start check separately.
Good tests assert observable contracts, preserved data and required side effects,
not private helper calls or incidental internal ordering. Use real isolated Catalog
instances where transaction and lock behavior matters. Mock external provider
failures for repeatable tests, while keeping actual DWH/model acceptance separate.
## Out of Scope
- Implementing or publishing the three example databases, their curated contents,
example CLI, auxiliary repository layout or autonomous repository-copy mode.
- A new graphical installer, a conversational parameter wizard, or collecting
required parameters in administration pages after an incomplete setup.
- Installing Pi, Python, Node or an application build toolchain on ordinary consumer hosts.
- Changes to server/Omics deployment, upstream authentication or the application's
existing human-in-the-loop workflow.
- Multiple workspace repositories per installation or multiple databases per workspace.
- Automatic release upgrades, destructive reset/uninstall, whole-volume rollback
or backup-policy redesign. Interrupted initial setup recovery remains in scope.
- Automatic semantic certification, generated Evidence without sources, benchmark
SQL targets or automated accuracy scoring.
- Publishing Docker images or executing installations as part of this specification
authoring task. These are implementation and release deliverables described above.
## Further Notes
The project owner approved the six-step revision and both final clarifications on
28 September 2026: examples stay deferred, and runtime-only checks are explicit
post-start obligations. Earlier interactive-wizard and incomplete-configuration
installation proposals are superseded where they conflict with this specification.
This specification follows the existing decisions on PostgreSQL metadata authority,
installation-local bindings, secret references, durable schema synchronization and
the Installation Model Catalog. It does not change those architectural authorities.
Publication of a usable Docker Hub release is a blocking dependency of consumer
prebuilt-installation acceptance. Availability of the deferred examples is not.
Docker Hub namespace, publishing credentials and concrete release versions are
maintainer release inputs, not values to invent or embed into user templates.
After publication to Gitea, `/to-tickets` will split this specification into small
end-to-end increments with explicit blockers. Implementation has not started in
this task, and publishing the specification does not attest that a release exists.
@@ -0,0 +1,420 @@
# Scomposizione della specifica di installazione
Data: 2026-09-28. Stato: scomposizione approvata dall'utente («approvo»);
pubblicazione `/to-tickets` completata. Verificati testi, etichetta `ready-for-agent`
e dipendenze native delle issue #43–#54; specifica parent invariata.
Parent: [Spec: installazione da documenti verificati e distribuzione Docker Hub](https://git.tylconsulting.it/mptyl/ThothII/issues/42).
Gli identificatori T01–T12 restano riferimenti della scomposizione; le issue reali
sono elencate sotto. Ogni issue usa `ready-for-agent` e dipendenze native Gitea.
La parent non viene modificata né chiusa.
## Issue pubblicate
Avanzamento locale, 2026-09-28: T01/#43 implementato sul branch
`codex/guided-standalone-install`. Disponibili `tht workspace prepare` e
`tht workspace validate`, con helper autonomo che riusa i parser runtime. Guide
IT/EN aggiornate; esempi e pubblicazione degli artefatti restano differiti ai ticket
previsti. Nessuna chiusura o modifica della parent effettuata.
Verifica: 14 test della CLI passano sia da sorgenti sia con il bundle nativo macOS
arm64 e `PATH` vuoto; artefatti Windows amd64/Linux amd64 cross-compilati, senza
attribuire loro un collaudo host. Backend su Node 24.16: 109 file passati, un file
saltato, 1.417 test passati e 40 saltati; typecheck e build rigorosa documentazione
passati. Suite Go: tutti i pacchetti passati, salvo un primo errore intermittente
nel test di concorrenza authstorage; quel test passa in tre ripetizioni e il pacchetto
completo passa nella verifica isolata. Le revisioni Standards e Spec non lasciano
finding aperti.
T02/#44 implementato sullo stesso branch: `tht installation prepare`, generazione
esplicita delle credenziali tecniche e `tht installation validate` preparano e
controllano documenti privati, modelli, autenticazione e bootstrap dei binding,
riusando gli schemi runtime senza avviare servizi. Guide IT/EN aggiornate.
Verifica backend completa su Node 24.16: 111 file passati, uno saltato, 1.420 test
passati e 40 saltati; le regressioni successive della revisione passano nella suite
mirata (quattro test, incluso il percorso con binari nativi e `PATH` vuoto).
Typecheck, build rigorosa documentazione e pacchetti Go passati; il pacchetto CLI
è stato ripetuto dopo la correzione rilevata in revisione. Nessun finding residuo
delle revisioni Standards/Spec.
T03/#45 implementato: `installation preflight` verifica l'host al passo 3 e
`installation plan` ripete i documenti, verifica rilascio/Compose e dipendenze
esterne, poi sigilla un piano privato legato agli input. Restano espliciti gli
obblighi runtime; nessun container viene creato. Il contratto del manifest è nel
[riferimento pubblico di preflight](../install/installation-preflight.md).
Verifica completa: tutti i pacchetti Go passati, inclusa l'integrazione con la
coppia nativa, Docker controllato e servizi Git HTTPS/database REST locali;
backend Node 24.16 con 112 file passati, uno saltato, 1.423 test passati e 40 saltati.
Typecheck e documentazione rigorosa passati. Bundle macOS arm64, Linux amd64 e
Windows amd64 ricompilati; solo macOS è stato eseguito qui, senza attribuire un
collaudo host alle compilazioni incrociate. Le revisioni Standards/Spec non lasciano
finding aperti dopo le regressioni su rete Evidence, collocazione del piano,
piattaforma Compose e comparsa di override locali. Nessuna pubblicazione reale
effettuata: il prossimo incremento è T04/#46, con namespace e accessi del manutentore.
| Ticket | Issue Gitea | Dipendenze dirette |
| --- | --- | --- |
| T01 | [Preparare e validare un repository workspace senza stack](https://git.tylconsulting.it/mptyl/ThothII/issues/43) | Nessuna |
| T02 | [Preparare e validare i documenti applicativi](https://git.tylconsulting.it/mptyl/ThothII/issues/44) | #43 |
| T03 | [Verificare precondizioni e produrre il piano eseguibile](https://git.tylconsulting.it/mptyl/ThothII/issues/45) | #44 |
| T04 | [Produrre e pubblicare un rilascio Docker Hub installabile](https://git.tylconsulting.it/mptyl/ThothII/issues/46) | Nessuna |
| T05 | [Installare la piattaforma dal rilascio senza domande](https://git.tylconsulting.it/mptyl/ThothII/issues/47) | #45, #46 |
| T06 | [Applicare i binding preparati al Catalog](https://git.tylconsulting.it/mptyl/ThothII/issues/48) | #47 |
| T07 | [Portare il workspace alla readiness con controlli runtime](https://git.tylconsulting.it/mptyl/ThothII/issues/49) | #48 |
| T08 | [Conservare il percorso esplicito da sorgente](https://git.tylconsulting.it/mptyl/ThothII/issues/50) | #47 |
| T09 | [Riprendere dopo correzioni e interruzioni senza perdere stato](https://git.tylconsulting.it/mptyl/ThothII/issues/51) | #49, #50 |
| T10 | [Collaudare l'installazione pubblicata su Windows/WSL2](https://git.tylconsulting.it/mptyl/ThothII/issues/52) | #51 |
| T11 | [Collaudare l'installazione su Omarchy](https://git.tylconsulting.it/mptyl/ThothII/issues/53) | #52 |
| T12 | [Pubblicare e collaudare il percorso macOS Apple Silicon](https://git.tylconsulting.it/mptyl/ThothII/issues/54) | #53 |
Ogni ticket comprende verifiche del comportamento e aggiornamenti pertinenti delle
guide IT/EN. Le dipendenze elencate sono dirette; non si ripetono quelle transitive.
Si riusano il runner dell'operatore e i servizi di dominio esistenti; gli adattamenti
necessari sono inclusi nella prima funzionalità che li usa. Non emerge una necessità
di refactoring trasversale da pubblicare come lavoro orizzontale separato.
| Ticket | Titolo | Bloccato da | Risultato dimostrabile |
| --- | --- | --- | --- |
| T01 | Preparare e validare un repository workspace senza stack | Nessuno | Template e controllo locale conformi ai contratti, senza Docker attivo. |
| T02 | Preparare e validare i documenti applicativi | T01 | Parametri, modelli e binding completi verificati senza avviare servizi. |
| T03 | Verificare precondizioni e produrre il piano eseguibile | T02 | Rapporto con errori bloccanti e obblighi runtime, legato agli input. |
| T04 | Produrre e pubblicare un rilascio Docker Hub installabile | Nessuno | Comando manutentore e pacchetto pubblico verificato tramite pull. |
| T05 | Installare la piattaforma dal rilascio senza domande | T03, T04 | Pull, inizializzazione e avvio da documenti, con stato e ripresa delle fasi. |
| T06 | Applicare i binding preparati al Catalog | T05 | Database dei workspace configurati senza questionari o duplicazioni. |
| T07 | Portare il workspace alla readiness con controlli runtime | T06 | Schema, preprocessing e servizi verificati senza aggirare la revisione umana. |
| T08 | Conservare il percorso esplicito da sorgente | T05 | Stessi input e contratti, con build scelta esplicitamente. |
| T09 | Riprendere dopo correzioni e interruzioni senza perdere stato | T07, T08 | Recupero dell'intero percorso, compresi binding, sync e preprocessing. |
| T10 | Collaudare l'installazione pubblicata su Windows/WSL2 | T09 | Prima accettazione reale senza sorgenti o compilatori. |
| T11 | Collaudare l'installazione su Omarchy | T10 | Seconda accettazione reale su Linux x64, distinta da Windows. |
| T12 | Pubblicare e collaudare il percorso macOS Apple Silicon | T11 | Terza accettazione con immagini arm64 e comando host compatibile. |
## T01 — Preparare e validare un repository workspace senza stack
### Parent
[Spec: installazione da documenti verificati e distribuzione Docker Hub](https://git.tylconsulting.it/mptyl/ThothII/issues/42).
### What to build
Un autore prepara un repository ad hoc usando un template documentato e verifica
i documenti localmente prima di installare ThothII. Il validatore è fornito come
capacità eseguibile senza Node, Docker attivo o checkout dei sorgenti applicativi.
Riusa i contratti del runtime, senza creare uno schema workspace alternativo.
### Acceptance criteria
- [ ] Il template distingue catalogo workspace, descriptor ed Evidence opzionali e non contiene funzionalità degli esempi ancora indisponibili.
- [ ] Preparare un template non avvia servizi, non sovrascrive documenti esistenti e non richiede accesso in scrittura al repository originale.
- [ ] Il controllo respinge YAML ambiguo o invalido, chiavi duplicate, identificatori duplicati, campi estranei, incoerenze catalogo/directory e riferimenti locali mancanti.
- [ ] Le Evidence configurate sono verificate per ciò che è controllabile localmente; assenza lecita e invalidità sono distinte, senza certificare il significato delle regole di dominio.
- [ ] Gli esiti identificano documento/campo e correzione; output strutturato e codici di uscita sono verificabili senza esporre segreti.
- [ ] Una raccolta condivisa di casi validi/invalidi prova equivalenza con i parser runtime, e il controllo passa senza Docker e senza runtime host aggiuntivi.
- [ ] Guide IT/EN mostrano preparazione, correzione e ripetizione del passo 2.
### Blocked by
None (can start immediately).
## T02 — Preparare e validare i documenti applicativi
### Parent
[Spec: installazione da documenti verificati e distribuzione Docker Hub](https://git.tylconsulting.it/mptyl/ThothII/issues/42).
### What to build
L'operatore compila template locali per installazione, modelli, binding database e
segreti, e ne verifica completezza e coerenza con i workspace già verificati.
Non viene avviata l'applicazione e nessun valore viene richiesto dal futuro setup.
### Acceptance criteria
- [ ] Template commentati ed esempi completi spiegano obblighi, default e riferimenti ai documenti protetti; i placeholder non superano la validazione.
- [ ] Modelli e embedding rispettano l'Installation Model Catalog; generazione metadati opzionale e default sono coerenti con i contratti esistenti.
- [ ] Un input bootstrap locale versionato descrive Workspace Database e Database Binding con riferimenti ai segreti; i descriptor workspace restano conformi allo schema v4.
- [ ] Validazione incrociata di workspace, binding, engine/trasporto, modelli, percorsi e file ambiente, senza migrare o interrogare in scrittura alcun database.
- [ ] La generazione esplicita delle credenziali tecniche produce file protetti prima del setup, senza sovrascritture o segreti nei log/rapporti.
- [ ] I test coprono input completi, mancanti, incompatibili e segreti illeggibili; i documenti dell'utente rimangono invariati durante le verifiche.
- [ ] Guide IT/EN consentono di raccogliere e preparare tutte le informazioni con calma prima dell'esecuzione.
### Blocked by
- T01 — Preparare e validare un repository workspace senza stack.
## T03 — Verificare precondizioni e produrre il piano eseguibile
### Parent
[Spec: installazione da documenti verificati e distribuzione Docker Hub](https://git.tylconsulting.it/mptyl/ThothII/issues/42).
### What to build
L'operatore verifica host e dipendenze esterne disponibili, quindi ottiene un piano
eseguibile riferito ai documenti e al rilascio scelti. Il rapporto distingue errori,
avvisi e controlli necessariamente rinviati al runtime, senza creare lo stack.
### Acceptance criteria
- [ ] I controlli host sono invocabili al passo 3; quelli dipendenti dai parametri finali sono completati o ripetuti al passo 5.
- [ ] Sono verificati Docker/Compose, architettura, WSL2 quando pertinente, percorsi/permessi, risorse e disponibilità del rilascio e dei suoi componenti nel registry.
- [ ] Le prove sulle dipendenze esterne disponibili sono circoscritte e documentate; un servizio esistente irraggiungibile non viene promosso a semplice controllo differito.
- [ ] Pi non è richiesto sull'host; le dipendenze incluse nelle immagini sono riconosciute nel manifest e associate a controlli runtime precisi.
- [ ] Il piano registra input non segreti, revisione/contenuti workspace, release e versione del validatore; nessun segreto o fingerprint pubblico non protetto di segreti.
- [ ] Ogni controllo differito ha un'identità e un'obbligazione runtime; valori obbligatori mancanti o immagini assenti bloccano il piano.
- [ ] Prove con manifest e servizi controllati coprono cambiamento degli input, credenziali, errori di rete e architetture; nessuna creazione di container o mutazione di dati.
- [ ] Guide IT/EN spiegano rapporto, errori e verifiche ancora da eseguire. La prova con il rilascio reale verrà completata dal ticket di esecuzione, dopo T04.
### Blocked by
- T02 — Preparare e validare i documenti applicativi.
## T04 — Produrre e pubblicare un rilascio Docker Hub installabile
### Parent
[Spec: installazione da documenti verificati e distribuzione Docker Hub](https://git.tylconsulting.it/mptyl/ThothII/issues/42).
### What to build
Il manutentore esegue un comando riproducibile che costruisce, verifica e pubblica
core/frontend su Docker Hub, insieme al pacchetto operatore compatibile, e dimostra
che il rilascio pubblicato è scaricabile. La pubblicazione delle immagini oggi
mancanti è un risultato concreto del ticket, non un prerequisito lasciato a mano.
### Acceptance criteria
- [ ] Il comando riceve revisione, versione, namespace e architetture e mantiene fuori da bundle/log le credenziali di pubblicazione.
- [ ] Pubblica core/frontend Linux amd64 per la prima tappa; Catalog migration e workspace maintenance risolvono alla stessa immagine core del rilascio.
- [ ] Il bundle contiene comando host precompilato, Compose, inizializzazione e risorse di migrazione, senza dipendenze da checkout sorgente durante l'avvio.
- [ ] Il processo può includere la capacità di validazione preinstallazione prodotta da T01 nelle revisioni che la contengono; non serve duplicarne l'implementazione per questo ticket.
- [ ] Il manifest lega versione/revisione e digest compatibili; una pubblicazione parziale non viene esposta come rilascio completo e una versione immutabile non viene sovrascritta.
- [ ] Viene pubblicato un rilascio reale e viene verificato il pull degli artefatti pubblicati, senza affidarsi a immagini presenti soltanto nella cache di build.
- [ ] Test automatici verificano orchestrazione, fallimenti e retry senza richiedere una pubblicazione reale a ogni test; la prova reale del ticket resta distinta e registrata.
- [ ] Documentazione manutentore IT/EN e istruzioni del bundle distinguono pubblicazione, consumo e futura estensione arm64. Namespace e accessi effettivi sono input del manutentore, non valori inventati.
### Blocked by
None (can start immediately).
## T05 — Installare la piattaforma dal rilascio senza domande
### Parent
[Spec: installazione da documenti verificati e distribuzione Docker Hub](https://git.tylconsulting.it/mptyl/ThothII/issues/42).
### What to build
L'operatore applica un piano verificato, scarica gli artefatti pubblicati e ottiene
una piattaforma inizializzata e accessibile senza compilazione o domande. Lo stato
registrato permette di ritentare le fasi di piattaforma interrotte; non viene ancora
dichiarato pronto un workspace privo delle successive verifiche Catalog.
### Acceptance criteria
- [ ] Avvio da bundle rilasciato e operatore precompilato, senza checkout applicativo o toolchain; la revisione del bundle include i validatori e i comandi effettivamente utilizzati.
- [ ] Il piano viene ricontrollato rispetto a input, release e prerequisiti vivi prima delle mutazioni; un piano mancante o incoerente viene rifiutato.
- [ ] Con standard input chiuso il setup esegue pull, configurazione runtime, reti/volumi/container, inizializzazione Catalog/Memory e migrazioni senza richiedere parametri.
- [ ] L'accesso amministrativo iniziale deriva da materiale protetto preparato prima; non si introduce un endpoint privilegiato senza autenticazione.
- [ ] Un lock impedisce esecuzioni concorrenti; il journal atomico registra le fasi senza segreti e consente di verificare lo stato reale prima di ripetere una fase interrotta.
- [ ] Errori di pull non causano build locali; errori di configurazione rimandano ai documenti e alla nuova verifica, senza prompt di riparazione.
- [ ] La piattaforma accessibile è distinta dalla Workspace Readiness ancora da verificare; nessun messaggio finale prematuro di piena utilizzabilità.
- [ ] Test del runner e prova con artefatti pubblicati coprono successo, stdin chiuso, interruzioni e ripetizione senza cancellare volumi o dati. Guide IT/EN documentano il risultato parziale corretto.
### Blocked by
- T03 — Verificare precondizioni e produrre il piano eseguibile.
- T04 — Produrre e pubblicare un rilascio Docker Hub installabile.
## T06 — Applicare i binding preparati al Catalog
### Parent
[Spec: installazione da documenti verificati e distribuzione Docker Hub](https://git.tylconsulting.it/mptyl/ThothII/issues/42).
### What to build
Il setup rende operativa la configurazione database predisposta nei documenti:
registra il repository, crea i Workspace Database e le Database Binding nel Catalog,
installa i riferimenti segreti e verifica le connessioni. Il risultato è un binding
utilizzabile senza una compilazione manuale dei parametri nell'interfaccia web.
### Acceptance criteria
- [ ] Identità e revisioni dei workspace corrispondono al piano; il consumo del repository non richiede push e non sostituisce implicitamente una sorgente esistente.
- [ ] Creazione e modifica dei binding utilizzano servizi autorizzati, controlli di versione e secret store esistenti; nessuna seconda autorità runtime nei documenti bootstrap.
- [ ] La stessa configurazione applicata due volte non duplica record, credenziali o binding.
- [ ] Una modifica amministrativa incompatibile produce un rapporto di riconciliazione invece di essere sovrascritta dai file preparatori.
- [ ] La connessione viene controllata dall'ambiente applicativo; un esito positivo ottenuto dall'host non basta a dichiararla utilizzabile dai container.
- [ ] Un'interruzione dopo il salvataggio ma prima dell'aggiornamento del journal viene riconosciuta alla ripresa, senza duplicazioni o perdita di segreti.
- [ ] Test di contratto e integrazione Catalog coprono autorizzazioni, concorrenza, versioni e rerun; guide IT/EN illustrano diagnosi e riconciliazione.
### Blocked by
- T05 — Installare la piattaforma dal rilascio senza domande.
## T07 — Portare il workspace alla readiness con controlli runtime
### Parent
[Spec: installazione da documenti verificati e distribuzione Docker Hub](https://git.tylconsulting.it/mptyl/ThothII/issues/42).
### What to build
Da un binding applicato, il percorso completa sincronizzazione dello schema e
preparazione necessaria e rende visibili i risultati dei controlli runtime.
Un workspace è pronto solo quando tutti i requisiti applicabili sono verificati;
le decisioni umane già previste dai contratti rimangono esplicite.
### Acceptance criteria
- [ ] La sincronizzazione usa i Catalog Sync Runs durabili con lock, freschezza e transazioni esistenti; non introduce una seconda implementazione.
- [ ] Diff distruttive fermano il percorso in attesa della revisione di dominio esistente; ripresa successiva senza auto-conferme né domande sui parametri di setup.
- [ ] Preprocessing e consolidamento riusano descrizioni/commenti ed Evidence curate; nessuna generazione AI implicita, nessuna cancellazione di Memory o sovrascrittura di curation.
- [ ] Assenza lecita di Evidence non blocca; Evidence configurate ma invalide e indici necessari non pronti restano blocchi reali.
- [ ] Pi, modello embedding, trasporto DWH e altri obblighi differiti sono eseguiti a runtime e rendicontati; nessun obbligo scompare o viene considerato superato senza prova.
- [ ] Stato piattaforma, Workspace Readiness e collaudo funzionale sono distinti; una domanda reale con revisione rimane la prova funzionale, senza SQL target.
- [ ] Test di servizio e integrazione dimostrano esiti, conservazione dati e ripresa dei run; guide IT/EN spiegano le eventuali revisioni umane residue.
### Blocked by
- T06 — Applicare i binding preparati al Catalog.
## T08 — Conservare il percorso esplicito da sorgente
### Parent
[Spec: installazione da documenti verificati e distribuzione Docker Hub](https://git.tylconsulting.it/mptyl/ThothII/issues/42).
### What to build
Un operatore sceglie esplicitamente la build da una revisione sorgente e usa gli
stessi documenti, validatori, identità d'installazione e servizi del percorso
precompilato. L'alternativa resta praticabile mentre il default diventa Docker Hub.
### Acceptance criteria
- [ ] Modalità sorgente e prerequisiti aggiuntivi sono espliciti; nessun errore del registry la attiva automaticamente.
- [ ] I componenti costruiti sono equivalenti nei contratti di configurazione, migrazione e persistenza; non esistono implementazioni parallele dei binding o della readiness.
- [ ] Il piano identifica modalità e revisione e invalida i controlli dipendenti quando cambiano.
- [ ] L'esecuzione rimane non interattiva e usa journal/lock comuni; i segreti non entrano nelle immagini di sviluppo.
- [ ] Una prova automatizzata dimostra build e avvio espliciti e il mancato fallback da pull; una prova sorgente non chiude l'accettazione del rilascio precompilato.
- [ ] Guide IT/EN separano il percorso avanzato da quello ordinario e rendono visibili i prerequisiti aggiuntivi.
### Blocked by
- T05 — Installare la piattaforma dal rilascio senza domande.
## T09 — Riprendere dopo correzioni e interruzioni senza perdere stato
### Parent
[Spec: installazione da documenti verificati e distribuzione Docker Hub](https://git.tylconsulting.it/mptyl/ThothII/issues/42).
### What to build
L'operatore corregge un endpoint, una credenziale o un altro documento dopo un errore
e riprende l'intero percorso con verifiche aggiornate. Questo ticket completa il
recupero fra stadi e modalità, oltre ai retry locali già consegnati dai singoli ticket.
### Acceptance criteria
- [ ] Cambiamenti ai documenti, ai contenuti/revisioni workspace o al rilascio invalidano le sole verifiche/fasi dipendenti; le altre vengono riconciliate con lo stato reale.
- [ ] La rotazione di una credenziale viene rilevata senza esporla o pubblicarne fingerprint non protetti; si ripetono le prove necessarie.
- [ ] Ripresa dopo interruzione nei confini fra pull, inizializzazione, importazione Catalog, sync e preprocessing non duplica operazioni già persistite.
- [ ] Il preprocessing interrotto è rieseguito secondo il contratto esistente, senza promettere resume interno; un run in attesa di decisione umana conserva tale stato.
- [ ] Interruzioni, errori e concorrenza non corrompono il journal né attivano reset di volumi; diagnosi e stato rimangono privi di segreti.
- [ ] Test del percorso pubblico, con guasti controllati e integrazione dove conta la persistenza, dimostrano conservazione di sessioni, Evidence, descrizioni e Memory in entrambe le modalità.
- [ ] Le guide IT/EN presentano scenari di correzione/ripresa senza suggerire la cancellazione dei dati come normale rimedio.
### Blocked by
- T07 — Portare il workspace alla readiness con controlli runtime.
- T08 — Conservare il percorso esplicito da sorgente.
## T10 — Collaudare l'installazione pubblicata su Windows/WSL2
### Parent
[Spec: installazione da documenti verificati e distribuzione Docker Hub](https://git.tylconsulting.it/mptyl/ThothII/issues/42).
### What to build
Dimostrare il percorso completo su un PC Windows x64 con Ubuntu WSL2 e Docker
Desktop usando il rilascio realmente pubblicato, repository ad hoc e documenti
predisposti. Il collaudo include le correzioni necessarie a rendere utilizzabile
la prima piattaforma e un rapporto riproducibile.
### Acceptance criteria
- [ ] Un rilascio della revisione integrata viene pubblicato tramite T04 e consumato tramite pull; le immagini costruite soltanto localmente non soddisfano la prova.
- [ ] Il consumer non dispone di sorgenti applicativi o toolchain necessarie a compilare; operatore e validatori sono quelli precompilati nel bundle.
- [ ] Tutti i sei passi sono percorsi nell'ordine documentato, con almeno una correzione documentale e una ripresa dopo errore, senza domande durante il setup.
- [ ] Primo workspace ad hoc realmente utilizzabile, una domanda con revisione umana e stop/start con stato preservato; nessun uso presunto degli esempi rinviati.
- [ ] Rapporto con host/runtime, revisione, digest e risultati distinti di piattaforma/workspace/funzione, senza segreti; problemi esterni non sono nascosti.
- [ ] Guide IT/EN sono verificate rispetto ai comandi e agli esiti reali; eventuale assenza di host o credenziali necessarie lascia il collaudo incompleto, non simulato come riuscito.
### Blocked by
- T09 — Riprendere dopo correzioni e interruzioni senza perdere stato.
## T11 — Collaudare l'installazione su Omarchy
### Parent
[Spec: installazione da documenti verificati e distribuzione Docker Hub](https://git.tylconsulting.it/mptyl/ThothII/issues/42).
### What to build
Dopo la tappa Windows, ripetere e rendere funzionante il percorso su Linux Omarchy
x64, producendo un'evidenza di accettazione propria e mantenendo il comportamento
documentale e non interattivo già consegnato.
### Acceptance criteria
- [ ] Rilascio pubblico compatibile scaricato da Docker Hub e comando host precompilato; nessuna compilazione nel percorso ordinario.
- [ ] Prerequisiti, permessi, percorsi e rete di Omarchy sono verificati su un host reale, senza trasferire automaticamente l'esito Windows.
- [ ] Sei passi, input invalido/corretto, ripresa, workspace ad hoc, domanda reale e stop/start superano il collaudo.
- [ ] Ogni correzione di portabilità include la relativa verifica e non introduce una divergenza dei contratti rispetto al percorso Windows.
- [ ] Rapporto separato con versioni/digest e guide IT/EN coerenti; senza un host disponibile il gate rimane aperto.
### Blocked by
- T10 — Collaudare l'installazione pubblicata su Windows/WSL2.
## T12 — Pubblicare e collaudare il percorso macOS Apple Silicon
### Parent
[Spec: installazione da documenti verificati e distribuzione Docker Hub](https://git.tylconsulting.it/mptyl/ThothII/issues/42).
### What to build
Dopo Omarchy, pubblicare e verificare il set di artefatti compatibile con macOS
Apple Silicon, incluse immagini Linux arm64 e comando nativo, e chiudere la terza
tappa di accettazione su un Mac reale.
### Acceptance criteria
- [ ] Il comando di rilascio pubblica immagini arm64 e bundle host compatibile, con manifest/digest coerenti e senza dichiarare supporto prima del collaudo.
- [ ] Il consumer usa il rilascio pubblico e non compila; gli script e le risorse di inizializzazione sono presenti nel bundle.
- [ ] Tutti i sei passi, correzione/ripresa, workspace ad hoc, domanda reale e stop/start sono verificati sul Mac.
- [ ] Le eventuali correzioni conservano compatibilità e contratti delle tappe precedenti; le prove multiarch di build non sostituiscono il collaudo host.
- [ ] Rapporto macOS separato e guide IT/EN finalizzate per le tre piattaforme; nessun risultato sintetico viene presentato come prova reale.
### Blocked by
- T11 — Collaudare l'installazione su Omarchy.
## Verifiche della scomposizione
- I primi ticket lavorabili sono T01 e T04.
- T03 usa manifest e servizi controllati per i propri contratti; non aspetta la
pubblicazione reale. T05 è il primo punto che richiede insieme validazione e
artefatti realmente pubblicati.
- T06 e T08 possono procedere in parallelo dopo T05. T09 riunisce i percorsi per
verificare correzioni e ripresa dell'intera installazione.
- I tre gate host sono sequenziali per scelta esplicita dell'utente, non per una
dipendenza architetturale inventata.
- Gli esempi restano esclusi. Il comando di pubblicazione Docker Hub e almeno una
pubblicazione reale sono inclusi, non demandati a un futuro progetto.
- Nessun aggiornamento o chiusura della parent è previsto dalla pubblicazione.
+4
View File
@@ -56,6 +56,8 @@ exclude_docs: |
!/install/first-start.md
!/install/standalone-manual-it.md
!/install/standalone-manual-en.md
!/install/installation-preflight.md
!/install/publishing-images.md
!/install/shell-and-language.md
!/install/authentication-local.md
!/install/authentication-oidc.md
@@ -109,6 +111,8 @@ nav:
- Start here: install/first-start.md
- Mac, Windows, Linux — Italiano: install/standalone-manual-it.md
- Mac, Windows, Linux — English: install/standalone-manual-en.md
- Preflight and release manifest: install/installation-preflight.md
- Publishing images and bundles: install/publishing-images.md
- Display mode and language: install/shell-and-language.md
- Local authentication: install/authentication-local.md
- OIDC authentication: install/authentication-oidc.md
+97
View File
@@ -1,5 +1,102 @@
# Native installation CLI
## Offline workspace documents (issue #43)
`tht workspace prepare --directory NEW_PATH --id ID --name NAME [--language en|it]`
creates an ad hoc workspace repository template. `tht workspace validate --directory
PATH [--json]` checks the working documents without an installation descriptor or
services. Both commands invoke the packaged sibling `tht-workspace-documents`, which
compiles the canonical backend YAML/Zod parsers with its runtime. Neither Node nor
Bun is required on the user's computer. Validation never writes documents or Git
configuration. See the [IT](../../docs/install/standalone-manual-it.md) and
[EN](../../docs/install/standalone-manual-en.md) guides for the correction loop and
explicit runtime checks that local validation cannot satisfy.
Maintainer build (Go from `go.mod`, Node/npm for build only):
```sh
cd backend
npm ci
npm run build:workspace-tools
# Cross-compile the complete platform pairs:
npm run build:workspace-tools -- --all
```
The output is `dist/workspace-tools/<os>-<arch>/` containing `tht[.exe]`,
`tht-workspace-documents[.exe]`, `SHA256SUMS` and `build.json`. Individual targets:
`windows-amd64`, `linux-amd64`, `linux-arm64`, `darwin-amd64`, `darwin-arm64`.
Bun is pinned in `backend/package-lock.json`; compilation embeds the runtime, and
cross-compilation may download the selected Bun target. Deliver the complete pair
from one build, verify checksums and keep the executables together. The previous
`build-tht.sh` / `install-tht.sh` single-binary path remains for existing operator
commands; it does not package this helper. Release publication is tracked separately
in issue #46; building a Windows/Linux artifact does not establish acceptance there.
Run the same public CLI fixture corpus against the native pair, with subprocess
`PATH` deliberately empty:
```sh
cd backend
THT_WORKSPACE_TEST_CLI=/absolute/path/dist/workspace-tools/darwin-arm64/tht \
npx vitest run test/workspace-documents-cli.test.ts
```
## Application document preparation (issue #44)
Before installation, use `tht installation prepare --directory NEW_PATH`, then
`tht installation credentials --directory PATH` to explicitly create technical
credentials in the default protected layout. The latter preserves existing files.
Edit the documents and external credentials, then repeat:
```sh
tht --installation /absolute/path/thothii-installation.yaml \
installation validate --workspaces /absolute/path/workspaces --json
```
`database-bootstrap.yaml` beside the descriptor is a schemaVersion-1 bootstrap
input, not a runtime Catalog. Its database entries use the exact Catalog API
configuration schema plus private `secretFiles` and optional `evidenceSecretFiles`
references. `--bootstrap` selects another document. The helper validates workspace
membership, uniqueness, supported transports and required credential references;
Go checks protected files, the canonical Installation Model Catalog, environment
and local administrator. No process contacts a service or creates projections.
Only missing standard release Compose assets are deferred by `config.LoadPrepared`;
normal runtime `config.Load` retains all existing checks. Custom overrides must exist.
The native integration test is opt-in because it requires the built pair:
```sh
THT_INSTALLATION_TEST_CLI=/absolute/path/dist/workspace-tools/darwin-arm64/tht \
npx vitest run test/installation-documents-cli.test.ts
```
Run it from `backend/`, following the bundle build above. It supplies prepared
fixtures and removes host tools from the subprocess PATH. Platform acceptance and
real external credentials remain separate gates. See the IT/EN guides for the
complete parameter collection procedure, default `admin` account and local-auth scope.
## Host preflight and installation plan (issue #45)
For the maintainer release producer and public native archives, see
[publishing images and bundles](../../docs/install/publishing-images.md).
`tht installation preflight --directory PATH [--release MANIFEST] [--json]` checks
the prepared directory and host without creating a stack. After completing the
documents, use `tht --installation ABS_PATH installation plan --workspaces PATH
--release MANIFEST --output NEW_PLAN [--bootstrap PATH] [--json]` for the full plan.
The manifest, bounded external reads, private input seal and mandatory runtime
obligations are specified in the
[preflight reference](../../docs/install/installation-preflight.md).
The native end-to-end test supplies a controlled Docker executable and real local
HTTPS Git/HTTP database services. No application container is created:
```sh
cd tools/tht
THT_INSTALLATION_TEST_CLI=/absolute/path/dist/workspace-tools/darwin-arm64/tht \
go test ./cmd/tht -run TestNativeInstallationPlanBeforeContainersExist -count=1
```
## Shell configuration
The schema-v2 `thothii-installation.yaml` accepts this optional section:
+129
View File
@@ -0,0 +1,129 @@
package main
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"path/filepath"
"slices"
"strings"
"github.com/aritmolab/thothii/tools/tht/internal/preparation"
)
func installationDocumentsCommand(ctx context.Context, installationPath string, args []string, stdout io.Writer) int {
report := preparation.NewReport()
status := 0
options := map[string]string{}
valid := len(args) > 0
for index := 1; index < len(args); index++ {
key := args[index]
if _, duplicate := options[key]; duplicate {
valid = false
break
}
if key == "--json" {
options[key] = "true"
continue
}
if (key != "--directory" && key != "--workspaces" && key != "--bootstrap") || index+1 == len(args) {
valid = false
break
}
options[key] = args[index+1]
index++
}
if !valid || (args[0] == "validate" && (installationPath == "" || options["--workspaces"] == "" || options["--directory"] != "")) || (args[0] != "validate" && (options["--directory"] == "" || options["--workspaces"] != "" || options["--bootstrap"] != "")) {
report.Add("CLI", "$", "usage", "Use installation prepare|credentials --directory PATH [--json], or tht --installation ABSOLUTE_PATH installation validate --workspaces PATH [--bootstrap PATH] [--json].")
status = 2
} else if args[0] == "validate" {
installation, checkedReport := preparation.Validate(installationPath)
report = checkedReport
if report.OK {
bootstrap := options["--bootstrap"]
if bootstrap == "" {
bootstrap = filepath.Join(filepath.Dir(installationPath), "database-bootstrap.yaml")
}
bootstrap, _ = filepath.Abs(bootstrap)
workspace, _ := filepath.Abs(options["--workspaces"])
if resolved, err := filepath.EvalSymlinks(workspace); err == nil {
workspace = resolved
}
outsideWorkspace := func(path, document, field string) bool {
relative, err := filepath.Rel(workspace, path)
if err == nil && relative != ".." && !strings.HasPrefix(relative, ".."+string(filepath.Separator)) {
report.Add(document, field, "installation_file_in_workspace", "Keep installation documents and credential files outside the shared workspace repository.")
return false
}
return true
}
for _, path := range []string{installationPath, installation.EnvFile, installation.AuthenticationDirectory(), bootstrap} {
outsideWorkspace(path, "thothii-installation.yaml", "local-files")
}
files, _ := installation.SecretFiles()
for _, path := range files {
outsideWorkspace(path, "operator.env", "protected-file-reference")
}
if preparation.CheckYAML(bootstrap, "database-bootstrap.yaml", &report) {
var output, discarded bytes.Buffer
code := workspaceDocumentsCommand(ctx, []string{"bootstrap", "--directory", workspace, "--bootstrap", bootstrap, "--json"}, &output, &discarded)
var checked struct {
OK bool `json:"ok"`
Issues []preparation.Issue `json:"issues"`
Warnings []string `json:"warnings"`
SecretFiles []struct {
Field string `json:"field"`
Path string `json:"path"`
} `json:"secret_files"`
}
if json.Unmarshal(output.Bytes(), &checked) != nil {
report.Add("CLI", "$", "validator_unavailable", "Reinstall the matching tht and workspace helper pair.")
} else if code != 0 || !checked.OK {
report.Issues = append(report.Issues, checked.Issues...)
if len(checked.Issues) == 0 {
report.Add("database-bootstrap.yaml", "$", "bootstrap_invalid", "Correct workspace and binding documents, then validate again.")
}
} else {
report.Warnings = append(report.Warnings, checked.Warnings...)
for _, file := range checked.SecretFiles {
if outsideWorkspace(file.Path, "database-bootstrap.yaml", file.Field) {
preparation.CheckSecret(file.Path, "database-bootstrap.yaml", file.Field, false, &report)
}
}
}
}
}
} else {
directory, err := filepath.Abs(options["--directory"])
if err == nil {
if args[0] == "prepare" {
err = preparation.Prepare(directory)
} else {
err = preparation.Credentials(ctx, directory)
}
}
if err != nil {
report.Add("preparation", "$", "preparation_refused", err.Error())
}
}
report.OK = len(report.Issues) == 0
if !report.OK && status == 0 {
status = 1
}
if slices.Contains(args, "--json") {
_ = json.NewEncoder(stdout).Encode(report)
} else {
if report.OK {
fmt.Fprintln(stdout, "Document operation completed. Local validation does not establish runtime readiness.")
}
for _, issue := range report.Issues {
fmt.Fprintf(stdout, "%s [%s] %s: %s\n", issue.Document, issue.Field, issue.Code, issue.Correction)
}
for _, warning := range report.Warnings {
fmt.Fprintln(stdout, warning)
}
}
return status
}
@@ -0,0 +1,96 @@
package main
import (
"bytes"
"context"
"encoding/json"
"os"
"path/filepath"
"runtime"
"testing"
)
func TestInstallationPrepareDocumentsBeforeRuntime(t *testing.T) {
t.Setenv("PATH", "")
root, err := filepath.EvalSymlinks(t.TempDir())
if err != nil {
t.Fatal(err)
}
destination := filepath.Join(root, "installation")
var stdout, stderr bytes.Buffer
status := run(context.Background(), []string{"installation", "prepare", "--directory", destination, "--json"}, &stdout, &stderr)
if status != 0 {
t.Fatalf("status=%d stdout=%s stderr=%s", status, &stdout, &stderr)
}
var report map[string]any
if json.Unmarshal(stdout.Bytes(), &report) != nil || report["ok"] != true {
t.Fatalf("report=%s", &stdout)
}
for _, name := range []string{"thothii-installation.yaml", "operator.env", "database-bootstrap.yaml", "README.md"} {
if _, err := os.Stat(filepath.Join(destination, name)); err != nil {
t.Fatal(err)
}
}
before, _ := os.ReadFile(filepath.Join(destination, "thothii-installation.yaml"))
stdout.Reset()
stderr.Reset()
if run(context.Background(), []string{"installation", "prepare", "--directory", destination, "--json"}, &stdout, &stderr) == 0 {
t.Fatal("overwrote existing documents")
}
after, _ := os.ReadFile(filepath.Join(destination, "thothii-installation.yaml"))
if !bytes.Equal(before, after) {
t.Fatal("existing document changed")
}
}
func TestInstallationCredentialsAreExplicitPrivateAndNeverReplaced(t *testing.T) {
t.Setenv("PATH", "")
root, _ := filepath.EvalSymlinks(t.TempDir())
destination := filepath.Join(root, "installation")
var stdout, stderr bytes.Buffer
if run(context.Background(), []string{"installation", "prepare", "--directory", destination}, &stdout, &stderr) != 0 {
t.Fatal(&stdout, &stderr)
}
secret := filepath.Join(destination, "secrets", "catalog-runtime-password")
if _, err := os.Stat(secret); !os.IsNotExist(err) {
t.Fatal("prepare generated a secret implicitly")
}
stdout.Reset()
stderr.Reset()
if run(context.Background(), []string{"installation", "credentials", "--directory", destination, "--json"}, &stdout, &stderr) != 0 {
t.Fatal(&stdout, &stderr)
}
before, err := os.ReadFile(secret)
if err != nil || len(bytes.TrimSpace(before)) < 32 {
t.Fatal("missing strong technical credential", err)
}
if bytes.Contains(stdout.Bytes(), bytes.TrimSpace(before)) || bytes.Contains(stderr.Bytes(), bytes.TrimSpace(before)) {
t.Fatal("secret leaked")
}
info, _ := os.Stat(secret)
if runtime.GOOS != "windows" && info.Mode().Perm()&0o077 != 0 {
t.Fatal("credential is not private")
}
stdout.Reset()
stderr.Reset()
if run(context.Background(), []string{"installation", "credentials", "--directory", destination, "--json"}, &stdout, &stderr) != 0 {
t.Fatal(&stdout, &stderr)
}
after, _ := os.ReadFile(secret)
if !bytes.Equal(before, after) {
t.Fatal("credential was replaced")
}
}
func TestInstallationValidateRejectsWrongEnvironmentFieldType(t *testing.T) {
root, _ := filepath.EvalSymlinks(t.TempDir())
path := filepath.Join(root, "thothii-installation.yaml")
if err := os.WriteFile(path, []byte("schemaVersion: 2\nenvFile: []\n"), 0o600); err != nil {
t.Fatal(err)
}
var stdout, stderr bytes.Buffer
status := run(context.Background(), []string{"--installation", path, "installation", "validate", "--workspaces", root, "--json"}, &stdout, &stderr)
if status != 1 {
t.Fatalf("invalid field accepted: %d %s", status, &stdout)
}
}
+192
View File
@@ -0,0 +1,192 @@
package main
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"os"
"path/filepath"
"slices"
"strings"
"time"
"github.com/aritmolab/thothii/tools/tht/internal/compose"
"github.com/aritmolab/thothii/tools/tht/internal/config"
"github.com/aritmolab/thothii/tools/tht/internal/preflight"
"github.com/aritmolab/thothii/tools/tht/internal/preparation"
"github.com/aritmolab/thothii/tools/tht/internal/safeio"
"github.com/aritmolab/thothii/tools/tht/internal/version"
)
func installationPreflightCommand(ctx context.Context, installationPath string, args []string, stdout io.Writer) int {
report := preflight.NewReport()
options := map[string]string{}
usage := false
for index := 1; index < len(args); index++ {
key := args[index]
if _, exists := options[key]; exists {
usage = true
break
}
if key == "--json" {
options[key] = "true"
continue
}
if !slices.Contains([]string{"--directory", "--workspaces", "--bootstrap", "--release", "--output"}, key) || index+1 == len(args) {
usage = true
break
}
options[key] = args[index+1]
index++
}
planning := len(args) > 0 && args[0] == "plan"
if usage || len(args) == 0 || (!planning && args[0] != "preflight") || (planning && (installationPath == "" || options["--workspaces"] == "" || options["--release"] == "" || options["--output"] == "" || options["--directory"] != "")) || (!planning && (options["--directory"] == "" || options["--workspaces"] != "" || options["--bootstrap"] != "" || options["--output"] != "")) {
report.Add("usage", "error", "CLI", "Use installation preflight --directory PATH [--release MANIFEST] [--json], or --installation ABS_PATH installation plan --workspaces PATH --release MANIFEST --output NEW_PLAN [--bootstrap PATH] [--json].")
writePreflight(stdout, report, slices.Contains(args, "--json"))
return 2
}
for key, value := range options {
if key != "--json" {
absolute, err := filepath.Abs(value)
if err != nil {
report.Add("path", "error", "CLI", "Supply canonical absolute paths.")
} else {
options[key] = absolute
}
}
}
var installation config.Installation
if planning {
root, err := filepath.EvalSymlinks(options["--workspaces"])
if err == nil {
relative, err := filepath.Rel(root, options["--output"])
if err == nil && relative != ".." && !strings.HasPrefix(relative, ".."+string(filepath.Separator)) {
report.Add("plan-in-workspace", "error", "output", "Keep the private plan and its key outside the shared workspace repository.")
}
}
}
if planning {
bootstrap := options["--bootstrap"]
if bootstrap == "" {
bootstrap = filepath.Join(filepath.Dir(installationPath), "database-bootstrap.yaml")
options["--bootstrap"] = bootstrap
}
var output bytes.Buffer
code := installationDocumentsCommand(ctx, installationPath, []string{"validate", "--workspaces", options["--workspaces"], "--bootstrap", bootstrap, "--json"}, &output)
var documents preparation.Report
if code != 0 || json.Unmarshal(output.Bytes(), &documents) != nil || !documents.OK {
report.Add("application-documents", "error", "documents", "Run installation validate and correct every reported issue before planning.")
} else {
var err error
installation, err = config.LoadPrepared(installationPath)
if err != nil {
report.Add("application-documents", "error", "documents", "Prepared inputs changed; repeat validation.")
}
options["--directory"] = installation.ProjectDirectory
}
}
if report.OK {
if exists, err := safeio.PreflightPrivateDirectory(options["--directory"]); err != nil || !exists {
report.Add("installation-directory", "error", "projectDirectory", "Use an existing private canonical installation directory.")
}
}
minimum := preflight.DefaultRequirements()
var inputs []string
var absent []string
var revision string
var unchanged func() bool
var manifest preflight.Manifest
if options["--release"] != "" {
var err error
manifest, err = preflight.LoadManifest(options["--release"])
if err != nil {
report.Add("release-manifest", "error", "release", "Use a complete supported manifest with all packaged file digests verified; publication must precede planning.")
} else {
minimum = manifest.Requirements
}
}
if report.OK && planning {
var err error
inputs, revision, err = preflight.CollectInputs(installation, options["--workspaces"], options["--bootstrap"], options["--release"])
if err == nil {
absent = preflight.AbsentOverrides(installation)
unchanged, err = preflight.CaptureInputs(inputs, options["--workspaces"], absent...)
}
if err != nil {
report.Add("input-snapshot", "error", "documents", "Keep input trees bounded, readable and free from links; repeat document validation.")
} else {
var discarded bytes.Buffer
if installationDocumentsCommand(ctx, installationPath, []string{"validate", "--workspaces", options["--workspaces"], "--bootstrap", options["--bootstrap"], "--json"}, &discarded) != 0 {
report.Add("changed-documents", "error", "documents", "Documents changed during validation; correct and repeat.")
}
}
}
host, hostErr := preflight.InspectHost(options["--directory"])
if hostErr != nil {
report.Add("host-filesystem", "error", "projectDirectory", "Allow filesystem capacity inspection at the canonical installation path.")
}
// Docker connection/trust and executable discovery remain host-owned. Compose parameters
// are exclusively read from the authored env file, not inherited shell overrides.
dockerEnvironment := []string{}
for _, entry := range os.Environ() {
key, _, _ := strings.Cut(entry, "=")
if slices.Contains([]string{"PATH", "HOME", "USERPROFILE", "SystemRoot", "SYSTEMROOT", "TEMP", "TMP", "TMPDIR", "DOCKER_HOST", "DOCKER_CONTEXT", "DOCKER_CONFIG", "DOCKER_TLS_VERIFY", "DOCKER_CERT_PATH", "HTTP_PROXY", "HTTPS_PROXY", "NO_PROXY", "http_proxy", "https_proxy", "no_proxy"}, key) {
dockerEnvironment = append(dockerEnvironment, entry)
}
}
runner := compose.NewRunnerWithEnvironment("", dockerEnvironment)
if report.OK {
report.Merge(preflight.CheckHost(ctx, runner, host, minimum))
}
platform := "linux/" + host.Arch
if report.OK && options["--release"] != "" {
report.Merge(preflight.CheckImages(ctx, runner, manifest, platform))
}
if report.OK && planning {
report.Merge(preflight.CheckCompose(ctx, runner, installation, manifest, options["--release"], platform))
report.Merge(preflight.CheckExternal(ctx, installation))
var output, discarded bytes.Buffer
bound, cancel := context.WithTimeout(ctx, 60*time.Second)
code := workspaceDocumentsCommand(bound, []string{"probe", "--directory", options["--workspaces"], "--bootstrap", options["--bootstrap"], "--json"}, &output, &discarded)
cancel()
var probes preflight.Report
if json.Unmarshal(output.Bytes(), &probes) != nil || len(probes.Checks) == 0 {
report.Add("database-probes", "error", "database-bootstrap", "Restore the matching validation helper and rerun bounded external dependency checks.")
} else {
report.Merge(probes)
}
if code != 0 && report.OK {
report.Add("database-probes", "error", "database-bootstrap", "A required external dependency is unavailable; correct it before planning.")
}
if report.OK {
if unchanged == nil || !unchanged() {
report.Add("changed-inputs", "error", "documents", "An input or credential changed during checks; repeat planning with stable prepared files.")
} else {
preflight.AddRuntimeObligations(&report)
plan := preflight.Plan{SchemaVersion: 1, ValidatorProtocol: preflight.Protocol, Validator: version.Current(), Installation: installation, Release: manifest, Platform: platform, WorkspaceDirectory: options["--workspaces"], WorkspaceRevision: revision, Inputs: inputs, AbsentInputs: absent, Report: report}
if err := preflight.WritePlan(options["--output"], &plan, unchanged); err != nil {
report.Add("plan-output", "error", "output", "Choose a new filename in a private canonical directory; existing plans and key files are never overwritten.")
}
}
}
}
if !planning {
report.Add("prepared-inputs", "warning", "documents", "Host preflight alone is not an executable plan; complete application validation and release selection at step 5.")
}
writePreflight(stdout, report, options["--json"] != "")
if report.OK {
return 0
}
return 1
}
func writePreflight(stdout io.Writer, report preflight.Report, structured bool) {
if structured {
_ = json.NewEncoder(stdout).Encode(report)
return
}
for _, check := range report.Checks {
fmt.Fprintf(stdout, "%s [%s] %s: %s\n", check.Outcome, check.ID, check.Field, check.Action)
}
}
@@ -0,0 +1,138 @@
package main
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"encoding/pem"
"fmt"
"net/http"
"net/http/httptest"
"os"
"os/exec"
"path/filepath"
"runtime"
"strings"
"testing"
"time"
"github.com/aritmolab/thothii/tools/tht/internal/preflight"
)
func TestNativeInstallationPlanBeforeContainersExist(t *testing.T) {
binary := os.Getenv("THT_INSTALLATION_TEST_CLI")
if binary == "" || runtime.GOOS == "windows" {
t.Skip("set THT_INSTALLATION_TEST_CLI to the compiled sibling bundle")
}
root, _ := filepath.EvalSymlinks(t.TempDir())
_ = os.Chmod(root, 0o700)
workspace := filepath.Join(root, "workspaces")
installation := filepath.Join(root, "installation")
release := filepath.Join(root, "release")
_ = os.Mkdir(release, 0o700)
command := func(args ...string) (int, string) {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
cmd := exec.CommandContext(ctx, binary, append(args, "--json")...)
cmd.Env = append(os.Environ(), "PATH="+root)
data, err := cmd.CombinedOutput()
if err != nil {
return 1, string(data)
}
return 0, string(data)
}
for _, args := range [][]string{{"workspace", "prepare", "--directory", workspace, "--id", "practice", "--name", "Practice"}, {"installation", "prepare", "--directory", installation}, {"installation", "credentials", "--directory", installation}} {
if code, text := command(args...); code != 0 {
t.Fatal(text)
}
}
git := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method != "GET" {
t.Error("Git mutation")
}
fmt.Fprintln(w, "0044"+strings.Repeat("b", 40)+" refs/heads/main")
}))
defer git.Close()
database := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method != "GET" {
t.Error("DWH mutation")
}
if r.Header.Get("Authorization") != "Bearer PRIVATE_DATABASE_VALUE" {
w.WriteHeader(401)
}
}))
defer database.Close()
write := func(path string, data []byte) {
t.Helper()
if err := os.WriteFile(path, data, 0o600); err != nil {
t.Fatal(err)
}
}
descriptor := filepath.Join(installation, "thothii-installation.yaml")
for _, name := range []string{"thothii-installation.yaml", "operator.env"} {
path := filepath.Join(installation, name)
data, _ := os.ReadFile(path)
write(path, []byte(strings.ReplaceAll(string(data), "https://CHANGE_ME/workspaces.git", git.URL+"/workspaces.git")))
}
for name, data := range map[string][]byte{"secrets.env": []byte("OPENAI_API_KEY=PRIVATE_PROVIDER_VALUE\n"), "database-password": []byte("PRIVATE_DATABASE_VALUE"), "git-credentials": {}, "git-ca.pem": pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: git.Certificate().Raw})} {
write(filepath.Join(installation, "secrets", name), data)
}
write(filepath.Join(installation, "database-bootstrap.yaml"), []byte(fmt.Sprintf("schemaVersion: 1\ndatabases:\n - workspaceId: practice\n engine: postgres\n databaseName: practice\n schema: public\n binding: {transport: rest_api, baseUrl: %q, restPath: /health, restAuth: bearer}\n secretFiles: {apiKey: %q}\n", database.URL, filepath.Join(installation, "secrets/database-password"))))
manifest := preflight.Manifest{SchemaVersion: 1, Version: "1.0.0", Revision: strings.Repeat("b", 40), ValidatorProtocol: 1, Requirements: preflight.DefaultRequirements(), Components: []string{"pi", "catalog-migrations", "workspace-maintenance"}, Images: map[string]map[string]string{}, Files: map[string]string{}, Compose: []string{"compose.yaml"}}
platform := "linux/" + runtime.GOARCH
services := map[string]map[string]string{}
for _, role := range []string{"core", "frontend", "catalog", "qdrant", "embedding"} {
manifest.Images[role] = map[string]string{platform: "docker.io/example/" + role + "@sha256:" + strings.Repeat("a", 64)}
}
for service, role := range map[string]string{"core": "core", "frontend": "frontend", "catalog-db": "catalog", "catalog-migrate": "core", "workspace-maintenance": "core", "qdrant": "qdrant", "embedding": "embedding", "embedding-model-init": "embedding"} {
services[service] = map[string]string{"image": manifest.Images[role][platform]}
}
_ = os.Mkdir(filepath.Join(release, "deploy"), 0o700)
for name, contents := range map[string]string{"compose.yaml": "services: {}\n", "deploy/compose.git-https.yaml": "services: {}\n"} {
write(filepath.Join(release, filepath.FromSlash(name)), []byte(contents))
sum := sha256.Sum256([]byte(contents))
manifest.Files[name] = hex.EncodeToString(sum[:])
}
manifestPath := filepath.Join(release, "release-manifest.json")
manifestJSON, _ := json.Marshal(manifest)
write(manifestPath, manifestJSON)
effective, _ := json.Marshal(map[string]any{"services": services})
script := fmt.Sprintf("#!/bin/sh\ncase \"$1\" in\ninfo) printf '%%s\\n' '{\"OSType\":\"linux\",\"Architecture\":\"%s\",\"NCPU\":4,\"MemTotal\":17179869184}';;\ncompose) if [ \"$2\" = version ]; then printf '2.39.0\\n'; else printf '%%s\\n' '%s'; fi;;\nmanifest) printf '%%s\\n' '{\"Descriptor\":{\"digest\":\"sha256:%s\",\"platform\":{\"os\":\"linux\",\"architecture\":\"%s\"}}}';;\n*) exit 1;;\nesac\n", runtime.GOARCH, effective, strings.Repeat("a", 64), runtime.GOARCH)
write(filepath.Join(root, "docker"), []byte(script))
_ = os.Chmod(filepath.Join(root, "docker"), 0o700)
planPath := filepath.Join(installation, "plan.json")
if code, text := command("--installation", descriptor, "installation", "validate", "--workspaces", workspace); code != 0 {
t.Fatalf("documents: %s", text)
}
args := []string{"--installation", descriptor, "installation", "plan", "--workspaces", workspace, "--release", manifestPath, "--output", planPath}
args[len(args)-1] = filepath.Join(workspace, "forbidden-plan.json")
if code, _ := command(args...); code == 0 {
t.Fatal("plan written inside workspace")
}
if _, err := os.Stat(args[len(args)-1]); !os.IsNotExist(err) {
t.Fatal("private plan published inside workspace")
}
args[len(args)-1] = planPath
code, text := command(args...)
if code != 0 {
t.Fatalf("plan failed: %s", text)
}
if strings.Contains(text, "PRIVATE_") {
t.Fatal("secret in report")
}
if err := preflight.VerifyPlanInputs(planPath); err != nil {
t.Fatal(err)
}
write(filepath.Join(installation, "secrets/database-password"), []byte("rotated-invalid"))
if preflight.VerifyPlanInputs(planPath) == nil {
t.Fatal("rotation did not invalidate plan")
}
args[len(args)-1] = filepath.Join(installation, "second-plan.json")
if code, text := command(args...); code == 0 || strings.Contains(text, "PRIVATE_") {
t.Fatalf("invalid credential plan: %s", text)
}
if _, err := os.Stat(args[len(args)-1]); !os.IsNotExist(err) {
t.Fatal("failed checks published plan")
}
}
+23
View File
@@ -43,6 +43,16 @@ Commands:
setup [--complete|--configure-only] [--installation-id ID] [--profile local|server]
[--shell-mode full|embedded] [--shell-default-locale BCP47-TAG] [--shell-adapter omics-portal]
Create, validate, and optionally complete the local installation.
installation prepare --directory NEW_PATH [--json]
Create commented installation, environment and database bootstrap templates.
installation credentials --directory PATH [--json]
Explicitly generate protected technical credentials before setup.
installation validate --workspaces PATH [--bootstrap PATH] [--json]
Check prepared application documents; requires --installation.
installation preflight --directory PATH [--release MANIFEST] [--json]
Check host prerequisites and optionally the published release, without a stack.
installation plan --workspaces PATH --release MANIFEST --output NEW_PLAN [--bootstrap PATH] [--json]
Validate documents and live dependencies, then seal a private plan; requires --installation.
installation migrate --output PATH --session-default PROVIDER/MODEL
--embedding-id PROVIDER/MODEL --embedding-dimensions N
Create a review-only schema-v2 candidate from all three legacy model sources.
@@ -85,6 +95,10 @@ Commands:
pi maintenance recover --yes
Verify a terminal installation, remove stale lifecycle files, and clear maintenance.
pi logs Show the latest 200 sanitized core log lines (bounded; no follow mode).
workspace prepare --directory NEW_PATH --id ID --name NAME [--language en|it] [--json]
Prepare workspace documents locally, before installation.
workspace validate --directory PATH [--json]
Validate local workspace documents without starting services.
workspace inspect --workspace ID [--json]
workspace pull [--json]
Pull and activate the configured workspace repository.
@@ -134,6 +148,12 @@ func run(ctx context.Context, args []string, stdout, stderr io.Writer) int {
return versionCommand(commandArgs, stdout, stderr)
}
if command == "installation" {
if len(commandArgs) > 0 && (commandArgs[0] == "preflight" || commandArgs[0] == "plan") {
return installationPreflightCommand(ctx, installationPath, commandArgs, stdout)
}
if len(commandArgs) > 0 && (commandArgs[0] == "prepare" || commandArgs[0] == "credentials" || commandArgs[0] == "validate") {
return installationDocumentsCommand(ctx, installationPath, commandArgs, stdout)
}
if len(commandArgs) > 0 && commandArgs[0] == "generate" {
return installationGenerationCommand(installationPath, commandArgs[1:], stdout, stderr)
}
@@ -141,6 +161,9 @@ func run(ctx context.Context, args []string, stdout, stderr io.Writer) int {
}
return commandUsageError(stderr, fmt.Sprintf("unknown command %q", command))
}
if command == "workspace" && len(commandArgs) > 0 && (commandArgs[0] == "prepare" || commandArgs[0] == "validate") {
return workspaceDocumentsCommand(ctx, commandArgs, stdout, stderr)
}
workingDirectory, err := os.Getwd()
if err != nil {
fmt.Fprintf(stderr, "tht: current directory is unavailable: %s\n", output.Sanitize(err.Error(), nil))
+49
View File
@@ -0,0 +1,49 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"os/exec"
"path/filepath"
"runtime"
"slices"
)
// Resolve only the packaged sibling, never an executable from the workspace or PATH.
func workspaceDocumentsCommand(ctx context.Context, args []string, stdout, stderr io.Writer) int {
executable, err := os.Executable()
if err == nil {
executable, err = filepath.EvalSymlinks(executable)
}
if err == nil {
name := "tht-workspace-documents"
if runtime.GOOS == "windows" {
name += ".exe"
}
command := exec.CommandContext(ctx, filepath.Join(filepath.Dir(executable), name), args...)
command.Stdout, command.Stderr = stdout, stderr
err = command.Run()
if err == nil {
return 0
}
var exitError *exec.ExitError
if errors.As(err, &exitError) && exitError.ExitCode() >= 0 {
return exitError.ExitCode()
}
}
const correction = "Install tht and tht-workspace-documents from the same platform bundle in the same directory, then retry."
if slices.Contains(args, "--json") {
_ = json.NewEncoder(stdout).Encode(map[string]any{
"schema_version": 1, "scope": "local-documents", "ok": false,
"workspaces": []any{}, "deferred_checks": []string{},
"issues": []map[string]string{{"document": "CLI", "field": "$", "code": "workspace_helper_unavailable", "correction": correction}},
})
} else {
fmt.Fprintln(stderr, correction)
}
return 1
}
@@ -0,0 +1,29 @@
package main
import (
"bytes"
"context"
"encoding/json"
"testing"
)
func TestWorkspaceDocumentsNeedsPackagedHelperNotInstallation(t *testing.T) {
t.Setenv("PATH", "")
t.Setenv("THOTHII_INSTALLATION", "/nonexistent/installation.yaml")
for _, action := range []string{"prepare", "validate"} {
var stdout, stderr bytes.Buffer
status := run(context.Background(), []string{"workspace", action, "--directory", t.TempDir(), "--json"}, &stdout, &stderr)
var report struct {
OK bool `json:"ok"`
Issues []struct {
Code string `json:"code"`
} `json:"issues"`
}
if err := json.Unmarshal(stdout.Bytes(), &report); err != nil {
t.Fatalf("missing structured helper error: status=%d stdout=%s stderr=%s", status, &stdout, &stderr)
}
if status != 1 || report.OK || len(report.Issues) != 1 || report.Issues[0].Code != "workspace_helper_unavailable" || stderr.Len() != 0 {
t.Fatalf("unexpected missing-helper result: status=%d stdout=%s stderr=%s", status, &stdout, &stderr)
}
}
}
+14 -1
View File
@@ -59,7 +59,17 @@ type boundedStreamingRunner interface {
// execRunner executes the Docker CLI. It never invokes a shell.
type execRunner struct {
binary string
binary string
environment []string
}
// NewRunnerWithEnvironment uses an explicit process environment, for document-owned Compose
// configuration that must not inherit unrelated operator shell parameter overrides.
func NewRunnerWithEnvironment(binary string, environment []string) Runner {
if binary == "" {
binary = "docker"
}
return execRunner{binary: binary, environment: append([]string{}, environment...)}
}
// NewRunner returns a runner for binary. An empty binary selects docker from PATH.
@@ -106,6 +116,9 @@ func (r execRunner) runBounded(ctx context.Context, args []string, stdin io.Read
}
}
command := exec.Command(r.binary, preparedArgs...)
if r.environment != nil {
command.Env = r.environment
}
configureProcess(command)
command.Stdin = stdin
overflow := make(chan struct{}, 1)
+22 -1
View File
@@ -98,6 +98,16 @@ type Installation struct {
// Load reads and validates an installation descriptor at an absolute path.
func Load(path string) (Installation, error) {
return load(path, false)
}
// LoadPrepared applies the same authored configuration rules before release assets exist.
// Only standard distribution-owned Compose assets are deferred, never custom overrides.
func LoadPrepared(path string) (Installation, error) {
return load(path, true)
}
func load(path string, prepared bool) (Installation, error) {
if !filepath.IsAbs(path) {
return Installation{}, fmt.Errorf("installation path must be absolute")
}
@@ -143,6 +153,11 @@ func Load(path string) (Installation, error) {
if err := requireDirectory(raw.ProjectDirectory, "projectDirectory"); err != nil {
return Installation{}, err
}
if prepared {
if err := requireCanonicalDirectory(raw.ProjectDirectory); err != nil {
return Installation{}, errors.New("projectDirectory must be canonical and accessible")
}
}
for _, legacyProjection := range []string{
filepath.Join(raw.ProjectDirectory, "deploy", "pi", "models.json"),
filepath.Join(raw.ProjectDirectory, "deploy", "pi", "settings.json"),
@@ -195,7 +210,8 @@ func Load(path string) (Installation, error) {
return Installation{}, errors.New("authentication.configDirectory must match THT_AUTH_CONFIG_ROOT")
}
for _, override := range raw.Overrides {
if err := requireRegularFile(override, "override"); err != nil {
standardAsset := override == filepath.Join(installation.ProjectDirectory, "deploy", "compose.git-https.yaml") || override == filepath.Join(installation.ProjectDirectory, "deploy", "compose.git-ssh.yaml")
if err := requireRegularFile(override, "override"); err != nil && !(prepared && standardAsset && errors.Is(statPathError(override), os.ErrNotExist)) {
return Installation{}, err
}
installation.Overrides = append(installation.Overrides, filepath.Clean(override))
@@ -209,6 +225,9 @@ func Load(path string) (Installation, error) {
}
}
for _, composeFile := range installation.ComposeFiles()[:2] {
if prepared && errors.Is(statPathError(composeFile), os.ErrNotExist) {
continue
}
if err := requireRegularFile(composeFile, "Compose file"); err != nil {
return Installation{}, err
}
@@ -226,6 +245,8 @@ func Load(path string) (Installation, error) {
return installation, nil
}
func statPathError(path string) error { _, err := os.Lstat(path); return err }
var metadataSecretBundleKeyPattern = regexp.MustCompile(`^[A-Z][A-Z0-9_]{0,127}$`)
var metadataAPIKeyEnvironments = map[string]struct{}{
"THT_MODEL_API_KEY": {},
+122
View File
@@ -0,0 +1,122 @@
// Package preflight checks a prepared installation without creating application state.
package preflight
import (
"context"
"encoding/json"
"io"
"os"
"runtime"
"strconv"
"strings"
"time"
"github.com/aritmolab/thothii/tools/tht/internal/compose"
)
const Protocol = 1
type Check struct {
ID string `json:"id"`
Outcome string `json:"outcome"`
Field string `json:"field"`
Action string `json:"action"`
}
type Report struct {
SchemaVersion int `json:"schema_version"`
OK bool `json:"ok"`
Checks []Check `json:"checks"`
}
func NewReport() Report { return Report{SchemaVersion: 1, OK: true, Checks: []Check{}} }
func (r *Report) Add(id, outcome, field, action string) {
r.Checks = append(r.Checks, Check{id, outcome, field, action})
if outcome == "error" {
r.OK = false
}
}
func (r *Report) Merge(other Report) {
r.Checks = append(r.Checks, other.Checks...)
r.OK = r.OK && other.OK
}
func (r Report) JSON() string { data, _ := json.Marshal(r); return string(data) }
type Runner interface {
Run(context.Context, []string, io.Reader) (compose.Result, error)
}
func docker(ctx context.Context, runner Runner, args ...string) (compose.Result, error) {
bound, cancel := context.WithTimeout(ctx, 15*time.Second)
defer cancel()
return runner.Run(bound, args, nil)
}
type Requirements struct {
CPUs int `json:"cpus"`
MemoryBytes uint64 `json:"memory_bytes"`
DiskBytes uint64 `json:"disk_bytes"`
}
func DefaultRequirements() Requirements { return Requirements{2, 4 << 30, 10 << 30} }
type Host struct {
OS string
Arch string
Kernel string
Distribution string
FreeBytes uint64
}
func InspectHost(directory string) (Host, error) {
h := Host{OS: runtime.GOOS, Arch: runtime.GOARCH}
kernel, _ := os.ReadFile("/proc/sys/kernel/osrelease")
h.Kernel = strings.ToLower(string(kernel))
distribution, _ := os.ReadFile("/etc/os-release")
h.Distribution = strings.ToLower(string(distribution))
var err error
h.FreeBytes, err = freeBytes(directory)
return h, err
}
func CheckHost(ctx context.Context, runner Runner, host Host, minimum Requirements) Report {
r := NewReport()
check := func(id string, ok bool, action string) {
outcome := "passed"
if !ok {
outcome = "error"
}
r.Add(id, outcome, "host", action)
}
check("host-platform", (host.OS == "linux" || host.OS == "darwin") && (host.Arch == "amd64" || host.Arch == "arm64"), "Use the Linux executable in Ubuntu WSL2/Omarchy, or the matching macOS executable; native Windows is not the installation path.")
if strings.Contains(strings.ToLower(host.Kernel), "microsoft") {
check("wsl2", strings.Contains(strings.ToLower(host.Kernel), "wsl2") && strings.Contains(host.Distribution, "ubuntu"), "Use Ubuntu WSL2 and enable Docker Desktop integration for that distribution.")
}
info, err := docker(ctx, runner, "info", "--format", "{{json .}}")
var parsed struct {
OSType string
Architecture string
NCPU int
MemTotal uint64
}
good := err == nil && json.Unmarshal([]byte(info.Stdout), &parsed) == nil
check("docker-daemon", good && parsed.OSType == "linux", "Start a reachable Linux Docker daemon for this user.")
arch := parsed.Architecture
if arch == "x86_64" {
arch = "amd64"
}
if arch == "aarch64" {
arch = "arm64"
}
check("docker-architecture", good && arch == host.Arch, "Use a Linux Docker daemon matching the host architecture; emulation is not certified.")
check("docker-resources", good && parsed.NCPU >= minimum.CPUs && parsed.MemTotal >= minimum.MemoryBytes, "Allocate at least the release CPU and memory minimum to Docker.")
check("installation-disk", host.FreeBytes >= minimum.DiskBytes && host.FreeBytes > 0, "Provide the release minimum free space on the installation filesystem.")
result, err := docker(ctx, runner, "compose", "version", "--short")
parts := strings.Split(strings.TrimPrefix(strings.TrimSpace(result.Stdout), "v"), ".")
major, _ := strconv.Atoi(parts[0])
minor := 0
if len(parts) > 1 {
minor, _ = strconv.Atoi(parts[1])
}
check("docker-compose", err == nil && (major > 2 || (major == 2 && minor >= 24)), "Install Docker Compose v2.24 or newer.")
r.Add("daemon-storage", "warning", "host", "Host disk capacity does not measure a Docker Desktop VM disk; reserve equivalent Docker storage and verify volume allocation during setup.")
return r
}
+60
View File
@@ -0,0 +1,60 @@
package preflight
import (
"context"
"encoding/json"
"os"
"path/filepath"
"github.com/aritmolab/thothii/tools/tht/internal/config"
)
// CheckCompose resolves the distributed release plus authored overrides without starting services.
func CheckCompose(ctx context.Context, runner Runner, installation config.Installation, m Manifest, manifestPath, platform string) Report {
r := NewReport()
args := []string{"compose", "--project-directory", installation.ProjectDirectory, "--env-file", installation.EnvFile}
for _, name := range m.Compose {
args = append(args, "-f", filepath.Join(filepath.Dir(manifestPath), filepath.FromSlash(name)))
}
for _, path := range installation.Overrides {
if _, err := os.Stat(path); os.IsNotExist(err) {
// Only the two well-known distribution-owned transport overlays can be relocated.
name := "deploy/" + filepath.Base(path)
if path != filepath.Join(installation.ProjectDirectory, filepath.FromSlash(name)) || (name != "deploy/compose.git-https.yaml" && name != "deploy/compose.git-ssh.yaml") || m.Files[name] == "" {
r.Add("compose-assets", "error", "overrides", "Include the selected Git transport overlay in the verified release.")
return r
}
path = filepath.Join(filepath.Dir(manifestPath), filepath.FromSlash(name))
}
args = append(args, "-f", path)
}
args = append(args, "config", "--format", "json")
result, err := docker(ctx, runner, args...)
var effective struct {
Services map[string]struct {
Image string `json:"image"`
Build any `json:"build"`
Platform string `json:"platform"`
} `json:"services"`
}
if err != nil || json.Unmarshal([]byte(result.Stdout), &effective) != nil {
r.Add("compose-configuration", "error", "operator.env", "Correct the effective Compose configuration using the release assets and prepared environment; no raw output is logged.")
return r
}
roles := map[string]string{"core": "core", "catalog-migrate": "core", "workspace-maintenance": "core", "frontend": "frontend", "catalog-db": "catalog", "qdrant": "qdrant", "embedding": "embedding", "embedding-model-init": "embedding"}
for service, role := range roles {
entry, exists := effective.Services[service]
if !exists || entry.Build != nil || entry.Image != m.Images[role][platform] || (entry.Platform != "" && entry.Platform != platform) {
r.Add("compose-image-"+service, "error", "release.compose", "Each runtime and maintenance service must use its released immutable image without a source build.")
}
}
for service := range effective.Services {
if _, ok := roles[service]; !ok {
r.Add("compose-service", "error", "release.compose", "Additional services need an explicit release contract before execution.")
}
}
if r.OK {
r.Add("compose-configuration", "passed", "release.compose", "Effective Compose configuration uses the complete released image set.")
}
return r
}
+13
View File
@@ -0,0 +1,13 @@
//go:build !windows
package preflight
import "golang.org/x/sys/unix"
func freeBytes(path string) (uint64, error) {
var stat unix.Statfs_t
if err := unix.Statfs(path, &stat); err != nil {
return 0, err
}
return uint64(stat.Bavail) * uint64(stat.Bsize), nil
}
@@ -0,0 +1,15 @@
//go:build windows
package preflight
import "golang.org/x/sys/windows"
func freeBytes(path string) (uint64, error) {
p, err := windows.UTF16PtrFromString(path)
if err != nil {
return 0, err
}
var available uint64
err = windows.GetDiskFreeSpaceEx(p, &available, nil, nil)
return available, err
}
+216
View File
@@ -0,0 +1,216 @@
package preflight
import (
"context"
"crypto/tls"
"crypto/x509"
"errors"
"io"
"net"
"net/http"
"net/url"
"strconv"
"strings"
"time"
"github.com/aritmolab/thothii/tools/tht/internal/config"
"github.com/aritmolab/thothii/tools/tht/internal/safeio"
"golang.org/x/crypto/ssh"
"golang.org/x/crypto/ssh/knownhosts"
)
func CheckExternal(ctx context.Context, installation config.Installation) Report {
r := NewReport()
value := func(name string) string { result, _ := installation.EnvironmentValue(name); return result }
err := checkGit(ctx, installation, value)
outcome := "passed"
if err != nil {
outcome = "error"
}
r.Add("workspace-remote", outcome, "workspaceRepository", "Require authenticated read access to the configured Git remote and branch using the prepared trust/credential files.")
for name, provider := range installation.ModelCatalog.Providers {
if provider.Endpoint == nil {
r.Add("provider-"+name, "warning", "modelCatalog.providers", "Built-in provider endpoint resolution belongs to the bundled Pi SDK. The required Pi runtime smoke check verifies model availability and credentials; preflight makes no billable generation requests.")
continue
}
parsed, err := url.Parse(provider.Endpoint.BaseURL)
if err == nil {
err = probeOrigin(ctx, parsed)
}
outcome := "passed"
if err != nil {
outcome = "error"
}
r.Add("provider-"+name, outcome, "modelCatalog.providers."+name+".endpoint", "Require DNS/TCP/TLS reachability of the configured provider origin. Credential/model eligibility still requires the bundled Pi runtime smoke check; no generation request is sent here.")
}
return r
}
func probeOrigin(ctx context.Context, target *url.URL) error {
bound, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
port := target.Port()
if port == "" {
port = "443"
if target.Scheme == "http" {
port = "80"
}
}
address := net.JoinHostPort(target.Hostname(), port)
var connection net.Conn
var err error
if target.Scheme == "https" {
dialer := tls.Dialer{NetDialer: &net.Dialer{Timeout: 5 * time.Second}, Config: &tls.Config{MinVersion: tls.VersionTLS12, ServerName: target.Hostname()}}
connection, err = dialer.DialContext(bound, "tcp", address)
} else {
connection, err = (&net.Dialer{Timeout: 5 * time.Second}).DialContext(bound, "tcp", address)
}
if err == nil {
connection.Close()
}
return err
}
func checkGit(ctx context.Context, installation config.Installation, value func(string) string) error {
remote := installation.WorkspaceRepository.Remote
if installation.WorkspaceRepository.Access == "ssh" && strings.HasPrefix(remote, "git@") {
host, path, found := strings.Cut(strings.TrimPrefix(remote, "git@"), ":")
if !found {
return errors.New("invalid Git remote")
}
return checkSSHGit(ctx, &url.URL{Scheme: "ssh", User: url.User("git"), Host: host, Path: path}, installation.WorkspaceRepository.Branch, value)
}
u, err := url.Parse(remote)
if err != nil {
return errors.New("remote unavailable")
}
if installation.WorkspaceRepository.Access == "ssh" {
return checkSSHGit(ctx, u, installation.WorkspaceRepository.Branch, value)
}
ca, err := safeio.ReadCanonicalPrivateRegular(value("THT_WORKSPACE_GIT_CA_FILE"), 64<<10)
if err != nil {
return err
}
pool, err := x509.SystemCertPool()
if err != nil {
pool = x509.NewCertPool()
}
if !pool.AppendCertsFromPEM(ca) {
return errors.New("invalid Git trust")
}
credentials, err := safeio.ReadCanonicalPrivateRegular(value("THT_WORKSPACE_GIT_CREDENTIALS_FILE"), 64<<10)
if err != nil {
return err
}
var user, password string
for _, line := range strings.Split(string(credentials), "\n") {
if strings.TrimSpace(line) == "" {
continue
}
credential, err := url.Parse(strings.TrimSpace(line))
if err != nil || credential.User == nil {
return errors.New("invalid Git credentials")
}
if credential.Scheme == u.Scheme && credential.Host == u.Host && (credential.Path == "" || credential.Path == u.Path) {
user = credential.User.Username()
password, _ = credential.User.Password()
}
}
u.Path = strings.TrimSuffix(u.Path, "/") + "/info/refs"
u.RawQuery = "service=git-upload-pack"
bound, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
request, err := http.NewRequestWithContext(bound, http.MethodGet, u.String(), nil)
if err != nil {
return err
}
if user != "" {
request.SetBasicAuth(user, password)
}
transport := &http.Transport{TLSClientConfig: &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12}, Proxy: http.ProxyFromEnvironment}
defer transport.CloseIdleConnections()
client := http.Client{Transport: transport, Timeout: 5 * time.Second, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }}
response, err := client.Do(request)
if err != nil {
return err
}
defer response.Body.Close()
if response.StatusCode != http.StatusOK {
return errors.New("Git remote refused")
}
data, err := io.ReadAll(io.LimitReader(response.Body, (1<<20)+1))
if err != nil || len(data) > 1<<20 || !advertisesBranch(data, installation.WorkspaceRepository.Branch) {
return errors.New("Git branch unavailable")
}
return nil
}
func advertisesBranch(data []byte, branch string) bool {
for _, line := range strings.Split(string(data), "\n") {
line = strings.SplitN(line, "\x00", 2)[0]
if strings.HasSuffix(line, " refs/heads/"+branch) {
return true
}
}
return false
}
func checkSSHGit(ctx context.Context, target *url.URL, branch string, value func(string) string) error {
// Git paths and branch names are already validated by the canonical installation loader.
if target.Scheme != "ssh" || target.User == nil || strings.ContainsAny(target.Path, "'\r\n\x00") {
return errors.New("use canonical ssh:// remote")
}
key, err := safeio.ReadCanonicalPrivateRegular(value("THT_WORKSPACE_GIT_SSH_KEY_FILE"), 64<<10)
if err != nil {
return err
}
signer, err := ssh.ParsePrivateKey(key)
if err != nil {
return err
}
hostKey, err := knownhosts.New(value("THT_WORKSPACE_GIT_KNOWN_HOSTS_FILE"))
if err != nil {
return err
}
port := target.Port()
if port == "" {
port = "22"
}
if _, err := strconv.Atoi(port); err != nil {
return err
}
address := net.JoinHostPort(target.Hostname(), port)
bound, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
connection, err := (&net.Dialer{Timeout: 5 * time.Second}).DialContext(bound, "tcp", address)
if err != nil {
return err
}
defer connection.Close()
deadline, _ := bound.Deadline()
_ = connection.SetDeadline(deadline)
clientConnection, channels, requests, err := ssh.NewClientConn(connection, address, &ssh.ClientConfig{User: target.User.Username(), Auth: []ssh.AuthMethod{ssh.PublicKeys(signer)}, HostKeyCallback: hostKey, Timeout: 5 * time.Second})
if err != nil {
return err
}
client := ssh.NewClient(clientConnection, channels, requests)
defer client.Close()
session, err := client.NewSession()
if err != nil {
return err
}
defer session.Close()
stdout, err := session.StdoutPipe()
if err != nil {
return err
}
stdin, err := session.StdinPipe()
if err != nil {
return err
}
if err = session.Start("git-upload-pack --advertise-refs '" + target.Path + "'"); err != nil {
return err
}
_ = stdin.Close()
data, err := io.ReadAll(io.LimitReader(stdout, (1<<20)+1))
if err != nil || len(data) > 1<<20 || !advertisesBranch(data, branch) {
return errors.New("Git branch unavailable")
}
return nil
}
@@ -0,0 +1,62 @@
package preflight
import (
"context"
"encoding/pem"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
"github.com/aritmolab/thothii/tools/tht/internal/config"
)
func TestGitProbeAuthenticatesAndRejectsWrongBranchAndCredentials(t *testing.T) {
status := 200
server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
user, password, ok := r.BasicAuth()
if !ok || user != "reader" || password != "PRIVATE_SENTINEL" {
w.WriteHeader(401)
return
}
if r.Method != "GET" || r.URL.Path != "/workspaces.git/info/refs" || r.URL.RawQuery != "service=git-upload-pack" {
t.Error("unexpected Git mutation/request")
w.WriteHeader(400)
return
}
w.WriteHeader(status)
_, _ = w.Write([]byte("0044" + strings.Repeat("a", 40) + " refs/heads/main\n"))
}))
defer server.Close()
root, _ := filepath.EvalSymlinks(t.TempDir())
ca := filepath.Join(root, "ca.pem")
credentials := filepath.Join(root, "credentials")
_ = os.WriteFile(ca, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: server.Certificate().Raw}), 0o600)
_ = os.WriteFile(credentials, []byte(strings.Replace(server.URL, "https://", "https://reader:PRIVATE_SENTINEL@", 1)), 0o600)
installation := config.Installation{WorkspaceRepository: config.WorkspaceRepository{Remote: server.URL + "/workspaces.git", Branch: "main", Access: "https"}}
value := func(name string) string {
if name == "THT_WORKSPACE_GIT_CA_FILE" {
return ca
}
return credentials
}
if err := checkGit(context.Background(), installation, value); err != nil {
t.Fatal(err)
}
installation.WorkspaceRepository.Branch = "missing"
if checkGit(context.Background(), installation, value) == nil {
t.Fatal("missing branch accepted")
}
installation.WorkspaceRepository.Branch = "main"
status = 503
if checkGit(context.Background(), installation, value) == nil {
t.Fatal("unavailable existing Git accepted")
}
status = 200
_ = os.WriteFile(credentials, []byte(strings.Replace(server.URL, "https://", "https://reader:rotated@", 1)), 0o600)
if checkGit(context.Background(), installation, value) == nil {
t.Fatal("bad credential accepted")
}
}
+287
View File
@@ -0,0 +1,287 @@
package preflight
import (
"crypto/hmac"
"crypto/rand"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"io/fs"
"os"
"path/filepath"
"slices"
"strings"
"github.com/aritmolab/thothii/tools/tht/internal/config"
"github.com/aritmolab/thothii/tools/tht/internal/safeio"
"github.com/aritmolab/thothii/tools/tht/internal/version"
"gopkg.in/yaml.v3"
)
type Plan struct {
SchemaVersion int `json:"schema_version"`
ValidatorProtocol int `json:"validator_protocol"`
Validator version.Info `json:"validator"`
Installation config.Installation `json:"installation"`
Release Manifest `json:"release"`
Platform string `json:"platform"`
WorkspaceDirectory string `json:"workspace_directory"`
WorkspaceRevision string `json:"workspace_revision"`
Inputs []string `json:"inputs"`
AbsentInputs []string `json:"absent_inputs"`
InputSeal string `json:"input_seal"`
Report Report `json:"report"`
}
func invalidPlan() error {
return errors.New("plan inputs changed or protected plan files are unavailable; repeat validation and produce a new plan")
}
// CaptureInputs gives a value-free freshness guard around live probes and document validation.
func CaptureInputs(paths []string, workspace string, absent ...string) (func() bool, error) {
key := make([]byte, 32)
if _, err := rand.Read(key); err != nil {
return nil, invalidPlan()
}
probe := Plan{Inputs: paths, AbsentInputs: absent}
before, err := seal(probe, key)
if err != nil {
return nil, err
}
files, err := treeFiles(workspace, true)
if err != nil {
return nil, err
}
return func() bool {
after, err := seal(probe, key)
current, treeErr := treeFiles(workspace, true)
return err == nil && treeErr == nil && hmac.Equal([]byte(before), []byte(after)) && slices.Equal(files, current)
}, nil
}
// A separate owner-only random key prevents public/offline guessing of low-entropy secrets.
// Live prerequisites must still be rechecked immediately before any execution or resumption.
func seal(plan Plan, key []byte) (string, error) {
for _, path := range plan.AbsentInputs {
if _, err := os.Lstat(path); !errors.Is(err, os.ErrNotExist) {
return "", invalidPlan()
}
}
mac := hmac.New(sha256.New, key)
plan.InputSeal = ""
data, err := json.Marshal(plan)
if err != nil {
return "", invalidPlan()
}
mac.Write(data)
var total int
for _, path := range plan.Inputs {
contents, err := safeio.ReadCanonicalRegular(path, 32<<20)
if err != nil {
return "", invalidPlan()
}
total += len(contents)
if total > 256<<20 {
return "", invalidPlan()
}
length, _ := json.Marshal([]any{path, len(contents)})
mac.Write(length)
mac.Write(contents)
}
return hex.EncodeToString(mac.Sum(nil)), nil
}
func AbsentOverrides(installation config.Installation) []string {
paths := []string{}
for _, path := range installation.Overrides {
if _, err := os.Lstat(path); errors.Is(err, os.ErrNotExist) {
paths = append(paths, path)
}
}
return paths
}
func WritePlan(path string, plan *Plan, guards ...func() bool) error {
if !plan.Report.OK || plan.ValidatorProtocol != Protocol || !filepath.IsAbs(path) {
return invalidPlan()
}
if exists, err := safeio.PreflightPrivateDirectory(filepath.Dir(path)); err != nil || !exists {
return invalidPlan()
}
for _, target := range []string{path, path + ".key"} {
if _, err := os.Lstat(target); !errors.Is(err, os.ErrNotExist) {
return invalidPlan()
}
}
key := make([]byte, 32)
if _, err := rand.Read(key); err != nil {
return invalidPlan()
}
var err error
plan.InputSeal, err = seal(*plan, key)
if err != nil {
return err
}
for _, guard := range guards {
if guard == nil || !guard() {
return invalidPlan()
}
}
data, _ := json.MarshalIndent(plan, "", " ")
if err := safeio.WriteCanonicalNewPrivateFile(path+".key", key, 0o600); err != nil {
return invalidPlan()
}
if err := safeio.WriteCanonicalNewPrivateFile(path, append(data, '\n'), 0o600); err != nil {
_ = safeio.RemoveCanonicalPrivateRegular(path + ".key")
return invalidPlan()
}
return nil
}
func VerifyPlanInputs(path string) error {
data, err := safeio.ReadCanonicalPrivateRegular(path, 4<<20)
if err != nil {
return invalidPlan()
}
key, err := safeio.ReadCanonicalPrivateRegular(path+".key", 32)
if err != nil || len(key) != 32 {
return invalidPlan()
}
var plan Plan
if json.Unmarshal(data, &plan) != nil || plan.SchemaVersion != 1 || plan.ValidatorProtocol != Protocol || !plan.Report.OK || len(plan.Inputs) == 0 || len(plan.Inputs) > 10000 {
return invalidPlan()
}
actual, err := seal(plan, key)
if err != nil || !hmac.Equal([]byte(actual), []byte(plan.InputSeal)) {
return invalidPlan()
}
// Detect added or removed workspace files as well as changes to known file bytes.
if plan.WorkspaceDirectory != "" {
files, err := treeFiles(plan.WorkspaceDirectory, true)
if err != nil {
return invalidPlan()
}
for _, file := range files {
if !slices.Contains(plan.Inputs, file) {
return invalidPlan()
}
}
}
return nil
}
func treeFiles(root string, skipGit bool) ([]string, error) {
paths := []string{}
count := 0
err := filepath.WalkDir(root, func(path string, entry fs.DirEntry, err error) error {
if err != nil {
return invalidPlan()
}
count++
if count > 10000 {
return invalidPlan()
}
if skipGit && entry.Name() == ".git" {
if entry.IsDir() {
return filepath.SkipDir
}
return nil
}
if entry.IsDir() {
return nil
}
if !entry.Type().IsRegular() {
return invalidPlan()
}
paths = append(paths, path)
return nil
})
return paths, err
}
// CollectInputs fingerprints exact prepared contents, with normalized configuration in Plan.
// Referenced credentials are sealed, never copied. Git object stores are excluded.
func CollectInputs(installation config.Installation, workspace, bootstrap, manifest string) ([]string, string, error) {
paths := []string{installation.Path, installation.EnvFile, bootstrap, manifest}
for _, root := range []string{workspace, installation.AuthenticationDirectory()} {
files, err := treeFiles(root, root == workspace)
if err != nil {
return nil, "", err
}
paths = append(paths, files...)
}
secrets, err := installation.SecretFiles()
if err != nil {
return nil, "", invalidPlan()
}
for _, path := range secrets {
paths = append(paths, path)
}
data, err := safeio.ReadCanonicalPrivateRegular(bootstrap, 1<<20)
if err != nil {
return nil, "", invalidPlan()
}
var bindings struct {
Databases []struct {
SecretFiles map[string]string `yaml:"secretFiles"`
EvidenceSecretFiles map[string]string `yaml:"evidenceSecretFiles"`
} `yaml:"databases"`
}
if yaml.Unmarshal(data, &bindings) != nil {
return nil, "", invalidPlan()
}
for _, entry := range bindings.Databases {
for _, values := range []map[string]string{entry.SecretFiles, entry.EvidenceSecretFiles} {
for _, path := range values {
paths = append(paths, path)
}
}
}
m, err := LoadManifest(manifest)
if err != nil {
return nil, "", err
}
for name := range m.Files {
paths = append(paths, filepath.Join(filepath.Dir(manifest), filepath.FromSlash(name)))
}
for _, path := range installation.Overrides {
if _, err := os.Lstat(path); err == nil {
paths = append(paths, path)
}
}
revision := "content-snapshot"
head := filepath.Join(workspace, ".git", "HEAD")
if data, err := safeio.ReadCanonicalRegular(head, 1024); err == nil {
paths = append(paths, head)
value := strings.TrimSpace(string(data))
if strings.HasPrefix(value, "ref: refs/") {
ref := strings.TrimPrefix(value, "ref: ")
if safeRelative(ref) {
path := filepath.Join(workspace, ".git", filepath.FromSlash(ref))
if data, err := safeio.ReadCanonicalRegular(path, 1024); err == nil {
paths = append(paths, path)
value = strings.TrimSpace(string(data))
}
}
}
if len(value) == 40 {
if _, err := hex.DecodeString(value); err == nil {
revision = value
}
}
}
slices.Sort(paths)
paths = slices.Compact(paths)
return paths, revision, nil
}
func AddRuntimeObligations(report *Report) {
for _, check := range []Check{
{"container-network", "deferred-to-runtime", "bindings", "Repeat authenticated DWH and external connectivity checks from the core network."},
{"catalog-initialization", "deferred-to-runtime", "catalog", "Apply migrations and verify Catalog health plus prepared binding import."},
{"pi-operation", "deferred-to-runtime", "release.components.pi", "Verify the bundled Pi version and authenticated provider/model smoke operation inside core; do not install Pi on the host."},
{"local-embedding", "deferred-to-runtime", "modelCatalog.embedding", "Initialize the local embedding model and verify returned vector dimensions."},
{"workspace-preprocessing", "deferred-to-runtime", "workspaces", "Sync the exact verified workspace contents, materialize Evidence, preprocess and verify collections."},
{"workspace-readiness", "deferred-to-runtime", "workspaces", "Complete required administrative and human review gates before claiming final readiness."},
} {
report.Checks = append(report.Checks, check)
}
}
+68
View File
@@ -0,0 +1,68 @@
package preflight
import (
"github.com/aritmolab/thothii/tools/tht/internal/safeio"
"os"
"path/filepath"
"strings"
"testing"
)
func TestPlanBindsInputsAndDetectsCredentialRotationWithoutPublicSecretHashes(t *testing.T) {
root, _ := filepath.EvalSymlinks(t.TempDir())
if err := safeio.ProtectPrivateDirectory(root); err != nil {
t.Fatal(err)
}
input := filepath.Join(root, "descriptor.yaml")
secret := filepath.Join(root, "credential")
for path, data := range map[string]string{input: "schemaVersion: 2\n", secret: "PRIVATE_SENTINEL"} {
if err := safeio.WriteCanonicalNewPrivateFile(path, []byte(data), 0o600); err != nil {
t.Fatal(err)
}
}
plan := Plan{SchemaVersion: 1, ValidatorProtocol: Protocol, Inputs: []string{input, secret}, WorkspaceRevision: "content-snapshot", Report: NewReport()}
output := filepath.Join(root, "plan.json")
if err := WritePlan(output, &plan); err != nil {
t.Fatal(err)
}
data, _ := os.ReadFile(output)
if strings.Contains(string(data), "PRIVATE_SENTINEL") {
t.Fatal("secret in plan")
}
if err := VerifyPlanInputs(output); err != nil {
t.Fatal(err)
}
if err := WritePlan(output, &plan); err == nil {
t.Fatal("existing plan replaced")
}
if err := os.WriteFile(secret, []byte("rotated"), 0o600); err != nil {
t.Fatal(err)
}
if err := VerifyPlanInputs(output); err == nil {
t.Fatal("credential rotation did not invalidate plan")
}
if err := os.WriteFile(secret, []byte("PRIVATE_SENTINEL"), 0o600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(input, []byte("schemaVersion: 3\n"), 0o600); err != nil {
t.Fatal(err)
}
if err := VerifyPlanInputs(output); err == nil {
t.Fatal("document change did not invalidate plan")
}
if err := os.WriteFile(input, []byte("schemaVersion: 2\n"), 0o600); err != nil {
t.Fatal(err)
}
absent := filepath.Join(root, "transport-override.yaml")
plan.AbsentInputs = []string{absent}
second := filepath.Join(root, "second-plan.json")
if err := WritePlan(second, &plan); err != nil {
t.Fatal(err)
}
if err := safeio.WriteCanonicalNewPrivateFile(absent, []byte("services: {}\n"), 0o600); err != nil {
t.Fatal(err)
}
if VerifyPlanInputs(second) == nil {
t.Fatal("newly appearing override did not invalidate plan")
}
}
@@ -0,0 +1,118 @@
package preflight
import (
"context"
"encoding/json"
"errors"
"io"
"strings"
"testing"
"github.com/aritmolab/thothii/tools/tht/internal/compose"
"github.com/aritmolab/thothii/tools/tht/internal/config"
)
type fakeDocker struct {
calls [][]string
fail string
effective string
}
func (f *fakeDocker) Run(_ context.Context, args []string, _ io.Reader) (compose.Result, error) {
f.calls = append(f.calls, args)
if strings.Contains(strings.Join(args, " "), f.fail) && f.fail != "" {
return compose.Result{Stderr: "PRIVATE_SENTINEL"}, errors.New("PRIVATE_SENTINEL")
}
switch args[0] {
case "info":
return compose.Result{Stdout: `{"OSType":"linux","Architecture":"x86_64","NCPU":4,"MemTotal":17179869184}`}, nil
case "compose":
if args[len(args)-1] == "json" {
if f.effective != "" {
return compose.Result{Stdout: f.effective}, nil
}
return compose.Result{Stdout: `{"services":{"core":{"image":"example/core:latest"}}}`}, nil
}
return compose.Result{Stdout: "2.39.0"}, nil
case "manifest":
return compose.Result{Stdout: `{"Descriptor":{"digest":"sha256:` + strings.Repeat("a", 64) + `","platform":{"os":"linux","architecture":"amd64"}}}`}, nil
}
return compose.Result{}, errors.New("unexpected command")
}
func TestComposeRejectsIncompleteMutableServiceSet(t *testing.T) {
r := CheckCompose(context.Background(), &fakeDocker{}, config.Installation{ProjectDirectory: "/private", EnvFile: "/private/operator.env"}, Manifest{Compose: []string{"compose.yaml"}}, "/release/manifest.json", "linux/amd64")
if r.OK {
t.Fatal("unreleased service set accepted")
}
}
func TestComposeRejectsPlatformOverrideAgainstSelectedImage(t *testing.T) {
m := Manifest{Images: map[string]map[string]string{}, Compose: []string{"compose.yaml"}}
services := map[string]map[string]string{}
for service, role := range map[string]string{"core": "core", "frontend": "frontend", "catalog-db": "catalog", "catalog-migrate": "core", "workspace-maintenance": "core", "qdrant": "qdrant", "embedding": "embedding", "embedding-model-init": "embedding"} {
m.Images[role] = map[string]string{"linux/amd64": "docker.io/example/" + role + "@sha256:" + strings.Repeat("a", 64)}
services[service] = map[string]string{"image": m.Images[role]["linux/amd64"]}
}
services["core"]["platform"] = "linux/arm64"
data, _ := json.Marshal(map[string]any{"services": services})
r := CheckCompose(context.Background(), &fakeDocker{effective: string(data)}, config.Installation{}, m, "/release/manifest.json", "linux/amd64")
if r.OK {
t.Fatal("incompatible Compose platform accepted")
}
}
func TestHostChecksAreReadOnlyAndRejectUnavailableDocker(t *testing.T) {
f := &fakeDocker{}
host := Host{OS: "linux", Arch: "amd64", Kernel: "6.6-microsoft-standard-WSL2", Distribution: "ubuntu", FreeBytes: 30 << 30}
r := CheckHost(context.Background(), f, host, Requirements{CPUs: 2, MemoryBytes: 4 << 30, DiskBytes: 10 << 30})
if !r.OK {
t.Fatalf("host rejected: %+v", r)
}
for _, args := range f.calls {
if args[0] != "info" && !(args[0] == "compose" && args[1] == "version") {
t.Fatalf("mutating call: %v", args)
}
}
f.fail = "info"
r = CheckHost(context.Background(), f, host, Requirements{CPUs: 2, MemoryBytes: 4 << 30, DiskBytes: 10 << 30})
if r.OK || strings.Contains(r.JSON(), "PRIVATE_SENTINEL") {
t.Fatalf("unsafe success/report: %s", r.JSON())
}
host.Kernel = "4.4-microsoft"
if CheckHost(context.Background(), &fakeDocker{}, host, Requirements{CPUs: 2, MemoryBytes: 4 << 30, DiskBytes: 10 << 30}).OK {
t.Fatal("WSL1 accepted")
}
host.OS = "windows"
if CheckHost(context.Background(), &fakeDocker{}, host, Requirements{}).OK {
t.Fatal("native Windows accepted instead of WSL2")
}
}
func TestReleaseChecksEveryImmutableImageAndPlatform(t *testing.T) {
m := Manifest{SchemaVersion: 1, Version: "1.0.0", Revision: strings.Repeat("b", 40), ValidatorProtocol: 1, Requirements: Requirements{CPUs: 2, MemoryBytes: 4 << 30, DiskBytes: 10 << 30}, Components: []string{"pi", "catalog-migrations", "workspace-maintenance"}, Images: map[string]map[string]string{}}
for _, service := range []string{"core", "frontend", "catalog", "qdrant", "embedding"} {
m.Images[service] = map[string]string{"linux/amd64": "docker.io/example/" + service + "@sha256:" + strings.Repeat("a", 64)}
}
m.Files = map[string]string{"deploy/compose.yaml": strings.Repeat("c", 64)}
m.Compose = []string{"deploy/compose.yaml"}
if err := m.Validate(); err != nil {
t.Fatal(err)
}
f := &fakeDocker{}
r := CheckImages(context.Background(), f, m, "linux/amd64")
if !r.OK || len(f.calls) != 5 {
t.Fatalf("images not checked: %s calls=%d", r.JSON(), len(f.calls))
}
if CheckImages(context.Background(), f, m, "linux/arm64").OK {
t.Fatal("unsupported release architecture accepted")
}
f.fail = "frontend"
if CheckImages(context.Background(), f, m, "linux/amd64").OK {
t.Fatal("missing image accepted")
}
m.Images["core"]["linux/amd64"] = "example/core:latest"
if m.Validate() == nil {
t.Fatal("mutable tag accepted")
}
}
+131
View File
@@ -0,0 +1,131 @@
package preflight
import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"io"
"path/filepath"
"regexp"
"slices"
"strings"
"github.com/aritmolab/thothii/tools/tht/internal/safeio"
)
// Manifest is the publication/consumer contract. Each platform maps to a single-image digest,
// not a mutable tag or multi-platform index. Maintenance roles use Images["core"].
type Manifest struct {
SchemaVersion int `json:"schema_version"`
Version string `json:"version"`
Revision string `json:"revision"`
ValidatorProtocol int `json:"validator_protocol"`
Requirements Requirements `json:"requirements"`
Components []string `json:"components"`
Images map[string]map[string]string `json:"images"`
Files map[string]string `json:"files"`
Compose []string `json:"compose"`
}
var hex256 = regexp.MustCompile(`^[a-f0-9]{64}$`)
var imageReference = regexp.MustCompile(`^docker\.io/[a-z0-9][a-z0-9._/-]*@sha256:[a-f0-9]{64}$`)
func invalidRelease() error {
return errors.New("release manifest or packaged files are incomplete, incompatible or invalid")
}
func (m Manifest) Validate() error {
if m.SchemaVersion != 1 || m.ValidatorProtocol != Protocol || !regexp.MustCompile(`^[0-9]+\.[0-9]+\.[0-9]+(?:-[A-Za-z0-9.-]+)?$`).MatchString(m.Version) || !regexp.MustCompile(`^[a-f0-9]{40}$`).MatchString(m.Revision) {
return invalidRelease()
}
if m.Requirements.CPUs < 2 || m.Requirements.MemoryBytes < 4<<30 || m.Requirements.DiskBytes < 10<<30 {
return invalidRelease()
}
for _, component := range []string{"pi", "catalog-migrations", "workspace-maintenance"} {
if !slices.Contains(m.Components, component) {
return invalidRelease()
}
}
if len(m.Images) != 5 || len(m.Files) == 0 || len(m.Files) > 256 || len(m.Compose) == 0 || len(m.Compose) > 8 {
return invalidRelease()
}
for _, service := range []string{"core", "frontend", "catalog", "qdrant", "embedding"} {
if len(m.Images[service]) == 0 {
return invalidRelease()
}
for platform, ref := range m.Images[service] {
if (platform != "linux/amd64" && platform != "linux/arm64") || !imageReference.MatchString(ref) {
return invalidRelease()
}
}
}
for name, digest := range m.Files {
if !safeRelative(name) || !hex256.MatchString(digest) {
return invalidRelease()
}
}
for _, name := range m.Compose {
if _, ok := m.Files[name]; !ok {
return invalidRelease()
}
}
return nil
}
func safeRelative(name string) bool {
return name != "" && !strings.Contains(name, "\\") && !strings.Contains(name, ":") && !strings.HasPrefix(name, "/") && filepath.ToSlash(filepath.Clean(name)) == name && name != ".." && !strings.HasPrefix(name, "../") && name != "."
}
func LoadManifest(path string) (Manifest, error) {
var m Manifest
contents, err := safeio.ReadCanonicalRegular(path, 1<<20)
if err != nil {
return m, invalidRelease()
}
decoder := json.NewDecoder(bytes.NewReader(contents))
decoder.DisallowUnknownFields()
if decoder.Decode(&m) != nil || decoder.Decode(new(any)) != io.EOF {
return Manifest{}, invalidRelease()
}
if err = m.Validate(); err != nil {
return Manifest{}, err
}
for name, digest := range m.Files {
contents, err := safeio.ReadCanonicalRegular(filepath.Join(filepath.Dir(path), filepath.FromSlash(name)), 32<<20)
if err != nil {
return Manifest{}, invalidRelease()
}
sum := sha256.Sum256(contents)
if hex.EncodeToString(sum[:]) != digest {
return Manifest{}, invalidRelease()
}
}
return m, nil
}
func CheckImages(ctx context.Context, runner Runner, m Manifest, platform string) Report {
r := NewReport()
for _, service := range []string{"core", "frontend", "catalog", "qdrant", "embedding"} {
ref := m.Images[service][platform]
if ref == "" {
r.Add("image-"+service, "error", "release.images."+service, "Publish the selected Linux architecture before producing an executable plan.")
continue
}
result, err := docker(ctx, runner, "manifest", "inspect", "--verbose", ref)
var manifest struct {
Descriptor struct {
Digest string `json:"digest"`
Platform struct {
OS string `json:"os"`
Architecture string `json:"architecture"`
} `json:"platform"`
} `json:"Descriptor"`
}
good := err == nil && json.Unmarshal([]byte(result.Stdout), &manifest) == nil && manifest.Descriptor.Platform.OS+"/"+manifest.Descriptor.Platform.Architecture == platform && strings.HasSuffix(ref, "@"+manifest.Descriptor.Digest) && manifest.Descriptor.Digest != ""
outcome := "passed"
if !good {
outcome = "error"
}
r.Add("image-"+service, outcome, "release.images."+service, "Require the pinned digest to be publicly readable in Docker Hub for the selected Linux architecture.")
}
return r
}
@@ -0,0 +1,73 @@
package preparation
import (
"context"
"crypto/rand"
"encoding/hex"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"github.com/aritmolab/thothii/tools/tht/internal/authconfig"
"github.com/aritmolab/thothii/tools/tht/internal/config"
"github.com/aritmolab/thothii/tools/tht/internal/safeio"
)
// Credentials creates installation-owned secrets only. External credentials are supplied by
// the operator. Existing files are checked and retained so interrupted preparation can resume.
func Credentials(ctx context.Context, directory string) error {
if err := safeio.ValidatePrivateDirectory(directory); err != nil {
return fmt.Errorf("use an existing private preparation directory")
}
if _, err := safeio.ReadCanonicalPrivateRegular(filepath.Join(directory, "thothii-installation.yaml"), 1<<20); err != nil {
return fmt.Errorf("prepare installation documents first")
}
secrets := filepath.Join(directory, "secrets")
if err := safeio.EnsurePrivateDirectory(secrets); err != nil {
return fmt.Errorf("secrets directory must be private and operator-owned")
}
for _, name := range []string{"catalog-runtime-password", "catalog-migrator-password", "admin-password"} {
path := filepath.Join(secrets, name)
if _, err := os.Lstat(path); err == nil {
value, err := safeio.ReadCanonicalPrivateRegular(path, 64<<10)
if err != nil || len(strings.TrimSpace(string(value))) < 32 {
return fmt.Errorf("existing technical credentials must be private, readable and at least 32 characters; no file was replaced")
}
continue
} else if !errors.Is(err, os.ErrNotExist) {
return fmt.Errorf("cannot inspect technical credentials")
}
value := make([]byte, 32)
if _, err := rand.Read(value); err != nil {
return fmt.Errorf("cannot generate random credentials")
}
if err := safeio.WriteCanonicalNewPrivateFile(path, []byte(hex.EncodeToString(value)+"\n"), 0o600); err != nil {
return fmt.Errorf("cannot create credential; existing files are retained")
}
}
for name, contents := range map[string]string{"secrets.env": "# Supply the provider credential; never commit this file.\nOPENAI_API_KEY=CHANGE_ME\n", "pi-auth.json": "{}\n"} {
path := filepath.Join(secrets, name)
if _, err := os.Lstat(path); errors.Is(err, os.ErrNotExist) {
if err := safeio.WriteCanonicalNewPrivateFile(path, []byte(contents), 0o600); err != nil {
return fmt.Errorf("cannot create external credential template")
}
} else if _, err := safeio.ReadCanonicalPrivateRegular(path, 64<<10); err != nil {
return fmt.Errorf("existing credential template is not private or readable")
}
}
authDirectory := filepath.Join(directory, "auth")
if _, err := os.Lstat(filepath.Join(authDirectory, "auth.yaml")); err == nil {
if _, _, err := authconfig.Load(authDirectory); err != nil {
return fmt.Errorf("existing authentication documents are invalid; repair them explicitly")
}
return nil
}
installation := config.Installation{Authentication: config.Authentication{ConfigDirectory: authDirectory}}
if authconfig.Run(ctx, installation, []string{"configure", "--mode", "local", "--public-url", "http://localhost:8080", "--admin-user", "admin", "--password-file", filepath.Join(secrets, "admin-password")}, strings.NewReader(""), io.Discard, io.Discard) != 0 {
return fmt.Errorf("cannot prepare local administrator; inspect protected auth files and retry without replacing existing credentials")
}
return nil
}
@@ -0,0 +1,94 @@
// Package preparation owns installation-local documents before any runtime exists.
package preparation
import (
"fmt"
"path/filepath"
"strconv"
"strings"
"github.com/aritmolab/thothii/tools/tht/internal/safeio"
)
func Prepare(directory string) error {
if err := safeio.ValidateCanonicalPath(directory); err != nil {
return fmt.Errorf("choose an absolute canonical destination")
}
exists, err := safeio.PreflightPrivateDirectory(directory)
if err != nil || exists {
return fmt.Errorf("choose a new private directory with an existing parent; existing documents are never replaced")
}
if err := safeio.EnsurePrivateDirectory(directory); err != nil {
return fmt.Errorf("cannot create private preparation directory")
}
root := func(name string) string { return strconv.Quote(filepath.Join(directory, name)) }
installation := fmt.Sprintf(`# Installation schema v2; replace CHANGE_ME before validation.
# Paths refer to local preparation files, never to workspace Git.
schemaVersion: 2
profile: local
projectDirectory: %s
envFile: %s
shell: {mode: full, defaultLocale: en}
workspaceRepository:
remote: https://CHANGE_ME/workspaces.git
branch: main
access: https
authentication:
configDirectory: %s
modelCatalog:
defaults: {interaction: openai/gpt-4.1-mini}
embedding: {id: 'ollama/qwen3-embedding:0.6b', dimensions: 1024}
providers:
openai:
authentication: {mode: secret_env, apiKeyEnv: OPENAI_API_KEY}
session: {mode: pi_builtin}
models:
gpt-4.1-mini: {session: {}}
# Metadata generation is optional: omitted here. Configure its eligibility in this catalog.
# This Compose asset will come from the release; no application checkout is required here.
overrides: [%s]
`, strconv.Quote(directory), root("operator.env"), root("auth"), root("deploy/compose.git-https.yaml"))
environment := "# Non-secret paths and parameters; no interpolation or duplicate keys.\n"
for _, entry := range [][2]string{
{"COMPOSE_PROJECT_NAME", "thothii-local"}, {"THT_WORKSPACE_INSTALLATION_ID", "local"},
{"THT_WORKSPACE_GIT_REMOTE", "https://CHANGE_ME/workspaces.git"}, {"THT_WORKSPACE_GIT_BRANCH", "main"},
{"THT_INSTALLATION_CONFIG_SOURCE", filepath.Join(directory, "thothii-installation.yaml")},
{"THT_AUTH_CONFIG_ROOT", filepath.Join(directory, "auth")},
{"THT_SECRETS_FILE", filepath.Join(directory, "secrets", "secrets.env")},
{"PI_AUTH_FILE", filepath.Join(directory, "secrets", "pi-auth.json")},
{"THT_WORKSPACE_GIT_CREDENTIALS_FILE", filepath.Join(directory, "secrets", "git-credentials")},
{"THT_WORKSPACE_GIT_CA_FILE", filepath.Join(directory, "secrets", "git-ca.pem")},
{"THT_CATALOG_RUNTIME_PASSWORD_SOURCE", filepath.Join(directory, "secrets", "catalog-runtime-password")},
{"THT_CATALOG_MIGRATOR_PASSWORD_SOURCE", filepath.Join(directory, "secrets", "catalog-migrator-password")},
{"THOTH_HTTP_PORT", "8080"}, {"THOTH_CORE_HTTP_PORT", "8787"}, {"MAX_PI_PROCESSES", "4"},
} {
environment += entry[0] + "=" + strconv.Quote(entry[1]) + "\n"
}
bootstrap := fmt.Sprintf(`# Bootstrap input only; PostgreSQL Metadata Catalog remains the runtime authority.
schemaVersion: 1
databases:
- workspaceId: CHANGE_ME
engine: postgres
databaseName: CHANGE_ME
schema: public
binding:
transport: postgres_direct
host: CHANGE_ME
port: 5432
username: CHANGE_ME
secretFiles:
password: %s
`, root("secrets/database-password"))
documents := map[string]string{
".gitignore": "*\n",
"thothii-installation.yaml": installation, "operator.env": environment,
"database-bootstrap.yaml": bootstrap,
"README.md": "# Preparation / Preparazione\n\nReplace every CHANGE_ME / Sostituire ogni CHANGE_ME. Keep all files outside workspace Git / Tenere tutti i file fuori dal Git dei workspace.\n\n1. Edit installation models, workspace remote and operator.env consistently. / Modificare modelli, remoto e operator.env in modo coerente.\n2. Add one database bootstrap entry for every workspace; use read-only DWH credentials in protected files. / Una voce database per workspace, credenziali DWH in sola lettura in file protetti.\n3. Run tht installation credentials --directory PATH before setup; fill provider/database secrets yourself. / Generare credenziali tecniche prima del setup; compilare i segreti esterni.\n4. Repeat tht --installation PATH/thothii-installation.yaml installation validate --workspaces WORKSPACES. / Correggere e ripetere.\n\nNo services, network calls or database writes / Nessun servizio, chiamata di rete o scrittura database.\n",
}
for _, name := range []string{".gitignore", "thothii-installation.yaml", "operator.env", "database-bootstrap.yaml", "README.md"} {
if err := safeio.WriteCanonicalNewPrivateFile(filepath.Join(directory, name), []byte(strings.TrimSpace(documents[name])+"\n"), 0o600); err != nil {
return fmt.Errorf("cannot create preparation files; inspect the new directory and retry in a new destination")
}
}
return nil
}
@@ -0,0 +1,279 @@
package preparation
import (
"bytes"
"encoding/json"
"io"
"regexp"
"strconv"
"strings"
"github.com/aritmolab/thothii/tools/tht/internal/authconfig"
"github.com/aritmolab/thothii/tools/tht/internal/config"
"github.com/aritmolab/thothii/tools/tht/internal/safeio"
"gopkg.in/yaml.v3"
)
type Issue struct {
Document string `json:"document"`
Field string `json:"field"`
Code string `json:"code"`
Correction string `json:"correction"`
}
type Report struct {
SchemaVersion int `json:"schema_version"`
Scope string `json:"scope"`
OK bool `json:"ok"`
Issues []Issue `json:"issues"`
Warnings []string `json:"warnings"`
DeferredChecks []string `json:"deferred_checks"`
}
func NewReport() Report {
return Report{SchemaVersion: 1, Scope: "application-documents", Issues: []Issue{}, Warnings: []string{}, DeferredChecks: []string{"release-assets", "external-connectivity", "catalog-import", "runtime-readiness"}}
}
func (r *Report) Add(document, field, code, correction string) {
r.OK = false
r.Issues = append(r.Issues, Issue{document, field, code, correction})
}
func placeholder(value string) bool {
upper := strings.ToUpper(value)
return strings.Contains(upper, "CHANGE_ME") || strings.Contains(upper, "REPLACE_ME") || strings.Contains(upper, "YOUR_API_KEY") || strings.Contains(upper, "<PASSWORD>")
}
// CheckYAML refuses unresolved placeholders and ambiguous authored YAML without returning values.
func CheckYAML(path, document string, report *Report) bool {
content, err := safeio.ReadCanonicalPrivateRegular(path, 1<<20)
if err != nil {
report.Add(document, "$", "file_unavailable", "Use a readable, private regular document at a canonical absolute path (no links).")
return false
}
decoder := yaml.NewDecoder(bytes.NewReader(content))
var node, extra yaml.Node
if err := decoder.Decode(&node); err != nil || decoder.Decode(&extra) != io.EOF {
report.Add(document, "$", "yaml_invalid", "Keep one well-formed YAML document.")
return false
}
var walk func(*yaml.Node) bool
walk = func(n *yaml.Node) bool {
if n.Kind == yaml.AliasNode || n.Tag == "!!merge" {
return false
}
if n.Kind == yaml.ScalarNode && placeholder(n.Value) {
return false
}
if n.Kind == yaml.MappingNode {
seen := map[string]bool{}
for index := 0; index < len(n.Content); index += 2 {
key := n.Content[index]
if key.Kind != yaml.ScalarNode || seen[key.Value] {
return false
}
seen[key.Value] = true
}
}
for _, child := range n.Content {
if !walk(child) {
return false
}
}
return true
}
if !walk(&node) {
report.Add(document, "$", "incomplete_or_ambiguous", "Replace CHANGE_ME/REPLACE_ME placeholders, remove duplicate keys, aliases and YAML merge keys.")
return false
}
return true
}
var environmentKey = regexp.MustCompile(`^[A-Z][A-Z0-9_]*$`)
func checkEnvironment(path string, report *Report) bool {
contents, err := safeio.ReadCanonicalPrivateRegular(path, 1<<20)
if err != nil {
report.Add("operator.env", "$", "environment_unavailable", "Provide a private readable environment file.")
return false
}
seen := map[string]bool{}
for _, raw := range strings.Split(string(contents), "\n") {
line := strings.TrimSpace(raw)
if line == "" || strings.HasPrefix(line, "#") {
continue
}
key, value, found := strings.Cut(line, "=")
if !found || !environmentKey.MatchString(key) || seen[key] || placeholder(value) || strings.Contains(value, "$") {
report.Add("operator.env", "$", "environment_invalid", "Use one literal KEY=value per line, unique uppercase keys, and replace placeholders; shell interpolation is not accepted.")
return false
}
if strings.HasSuffix(key, "_PASSWORD") || strings.HasSuffix(key, "_API_KEY") {
report.Add("operator.env", "$", "inline_secret", "Move credential values to protected files and keep only file references in operator.env.")
return false
}
seen[key] = true
}
return true
}
func CheckSecret(path, document, field string, allowEmpty bool, report *Report) bool {
contents, err := safeio.ReadCanonicalPrivateRegular(path, 64<<10)
if err != nil || (!allowEmpty && len(bytes.TrimSpace(contents)) == 0) || placeholder(string(contents)) {
report.Add(document, field, "secret_unavailable", "Supply a non-placeholder private readable regular secret file (owner-only permissions, no links); do not put its contents in YAML or logs.")
return false
}
return true
}
func Validate(path string) (config.Installation, Report) {
report := NewReport()
if !CheckYAML(path, "thothii-installation.yaml", &report) {
return config.Installation{}, report
}
// Read only the location before the canonical loader checks every field.
contents, _ := safeio.ReadCanonicalPrivateRegular(path, 1<<20)
var location struct {
EnvFile string `yaml:"envFile"`
}
if yaml.Unmarshal(contents, &location) != nil {
report.Add("thothii-installation.yaml", "envFile", "schema_invalid", "Set envFile to one absolute path string, then correct the remaining descriptor fields.")
return config.Installation{}, report
}
if !checkEnvironment(location.EnvFile, &report) {
return config.Installation{}, report
}
installation, err := config.LoadPrepared(path)
if err != nil {
field, correction := "$", "Check installation schema v2, absolute paths, workspace remote/branch and matching environment values. Custom overrides and Git credential/CA files must exist."
if strings.Contains(err.Error(), "modelCatalog") {
field, correction = "modelCatalog", "Check interaction default eligibility, provider endpoints/authentication, embedding id/dimensions, and the private provider key bundle."
}
if strings.Contains(err.Error(), "authentication") {
field, correction = "authentication", "Match the authentication directory to operator.env and prepare its protected files."
}
report.Add("thothii-installation.yaml", field, "configuration_invalid", correction)
return config.Installation{}, report
}
value := func(key string) string { result, _ := installation.EnvironmentValue(key); return result }
for _, key := range []string{"COMPOSE_PROJECT_NAME", "THT_WORKSPACE_INSTALLATION_ID", "THT_INSTALLATION_CONFIG_SOURCE", "THT_SECRETS_FILE", "PI_AUTH_FILE", "THT_CATALOG_RUNTIME_PASSWORD_SOURCE", "THT_CATALOG_MIGRATOR_PASSWORD_SOURCE"} {
if value(key) == "" {
report.Add("operator.env", key, "required", "Supply this installation parameter or protected file reference before setup.")
}
}
if value("THT_INSTALLATION_CONFIG_SOURCE") != path {
report.Add("operator.env", "THT_INSTALLATION_CONFIG_SOURCE", "path_mismatch", "Point to the exact installation descriptor being validated.")
}
for _, key := range []string{"THOTH_HTTP_PORT", "THOTH_CORE_HTTP_PORT", "MAX_PI_PROCESSES"} {
number, err := strconv.Atoi(value(key))
if err != nil || number < 1 || number > 65535 {
report.Add("operator.env", key, "invalid_number", "Supply a positive integer; HTTP ports must be within 1..65535.")
}
}
if value("THOTH_HTTP_PORT") == value("THOTH_CORE_HTTP_PORT") {
report.Add("operator.env", "THOTH_CORE_HTTP_PORT", "port_collision", "Choose different frontend and core host ports.")
}
files, err := installation.SecretFiles()
if err != nil {
report.Add("operator.env", "$", "secret_references_invalid", "Use canonical absolute paths for all _FILE and _SOURCE references.")
}
for _, file := range files {
// The descriptor is a _SOURCE reference too, but its YAML values were already checked.
// Template instructions in comments are not unresolved credential placeholders.
if file == path {
continue
}
allowEmpty := file == value("THT_WORKSPACE_GIT_CREDENTIALS_FILE")
CheckSecret(file, "operator.env", "protected-file-reference", allowEmpty, &report)
}
auth, users, err := authconfig.Load(installation.AuthenticationDirectory())
if err != nil {
report.Add("auth/auth.yaml", "$", "authentication_invalid", "Prepare and validate local authentication documents before setup, including the initial administrator.")
} else if auth.Mode == "local" {
admin := false
for _, user := range users.Users {
for _, role := range user.Roles {
if user.Enabled && role == authconfig.RoleAdmin {
admin = true
}
}
}
if !admin {
report.Add("auth/users.yaml", "users", "administrator_missing", "Enable at least one administrator before setup.")
}
} else {
report.Add("auth/auth.yaml", "mode", "unsupported_bootstrap", "This preparation increment supports local administrative authentication; use the documented existing OIDC preparation path until its offline bootstrap validation is available.")
}
checkPiCredentials(value("PI_AUTH_FILE"), installation.ModelCatalog, &report)
report.OK = len(report.Issues) == 0
return installation, report
}
// Pi stores api_key or oauth records. Require literal prepared material here; model
// environment references belong to the catalog's secret_env path, not host process state.
func checkPiCredentials(path string, catalog config.ModelCatalog, report *Report) {
contents, err := safeio.ReadCanonicalPrivateRegular(path, 64<<10)
var credentials map[string]map[string]any
invalid := err != nil || json.Unmarshal(contents, &credentials) != nil || credentials == nil
literal := func(value any) bool {
text, ok := value.(string)
return ok && strings.TrimSpace(text) != "" && !strings.HasPrefix(text, "!") && !strings.Contains(text, "$") && !placeholder(text)
}
var declarative func(any) bool
declarative = func(value any) bool {
switch v := value.(type) {
case string:
return !strings.HasPrefix(v, "!")
case []any:
for _, item := range v {
if !declarative(item) {
return false
}
}
case map[string]any:
for _, item := range v {
if !declarative(item) {
return false
}
}
}
return true
}
seen := map[string]bool{}
for provider, record := range credentials {
name := strings.ToLower(strings.TrimSpace(provider))
if name == "" || provider != name || seen[name] || !declarative(record) {
invalid = true
}
seen[name] = true
switch record["type"] {
case "api_key":
if !literal(record["key"]) {
invalid = true
}
if env, present := record["env"]; present {
values, ok := env.(map[string]any)
if !ok {
invalid = true
}
for _, value := range values {
if _, ok := value.(string); !ok {
invalid = true
}
}
}
case "oauth":
expires, ok := record["expires"].(float64)
if !literal(record["access"]) || !literal(record["refresh"]) || !ok || expires <= 0 {
invalid = true
}
default:
invalid = true
}
}
for id, provider := range catalog.Providers {
if provider.Authentication.Mode == "pi_auth" && !seen[id] {
invalid = true
}
}
if invalid {
report.Add("pi-auth.json", "providers", "provider_auth_invalid", "Provide a JSON object with literal api_key/type+key or oauth/type+access+refresh+expires records for selected Pi providers; use catalog secret_env for environment-based keys. Commands and unresolved references are not accepted.")
}
}