diff --git a/.gitea/workflows/publish-docs.yml b/.gitea/workflows/publish-docs.yml index df0451c6..d9cc6605 100644 --- a/.gitea/workflows/publish-docs.yml +++ b/.gitea/workflows/publish-docs.yml @@ -7,6 +7,9 @@ on: paths: - "docs/**" - "mkdocs.yml" + - "scripts/build-docs.sh" + - "scripts/verify-public-docs.py" + - "scripts/test-verify-public-docs.py" - "docs/requirements.txt" - ".gitea/workflows/publish-docs.yml" workflow_dispatch: @@ -34,14 +37,18 @@ jobs: with: python-version: "3.x" cache: pip - cache-dependency-path: docs/requirements.txt + cache-dependency-path: docs/requirements.lock - name: Install MkDocs dependencies - run: python -m pip install -r docs/requirements.txt + run: python -m pip install -r docs/requirements.lock + + - name: Test public documentation boundary + run: python scripts/test-verify-public-docs.py - name: Build documentation - # Some documented source files intentionally live outside docs/. - run: mkdocs build + run: | + mkdocs build --strict + python scripts/verify-public-docs.py - name: Publish generated site to the pages branch working-directory: site diff --git a/PROJECT_STATE.md b/PROJECT_STATE.md index 061bc0af..c7b4640a 100644 --- a/PROJECT_STATE.md +++ b/PROJECT_STATE.md @@ -1,6 +1,6 @@ # ThothII — Project State -Last updated: 2026-09-14. +Last updated: 2026-09-15. This file is the short operational snapshot. Stable commands and the architecture mental model live in `AGENTS.md`; current design and runtime contracts live under `docs/architecture/`, @@ -12,6 +12,17 @@ Authentik, internal catalog/embedding services, and the PSD workspace repository `docs/operations/server-upgrade-gitea-workspace-v2.md`. Treat its operator gates and rollback requirements as mandatory; do not replace the running server stack in place. +## Public documentation boundary — 2026-09-15 + +MkDocs now publishes only its explicit public-manual allowlist: 20 product, installation, +usage and administration pages, plus approved assets. Internal developer/project documents +remain in Git but are excluded from generated pages and search. The build and publication +workflow enforce this boundary with `scripts/verify-public-docs.py` and negative fixtures. +The critical review and proposed (not executed) source-file retirements are in +`docs/maintenance/2026-09-15-documentation-cleanup.md`. It also records pre-existing stale +paths later in this snapshot and obsolete authentication/DWH documentation verifiers; +those require a separate reference/test cleanup, not restoration of superseded guides. + ## Session review layout deployed — 2026-09-14 Session-only dialogs now use the visible app bounds: artifact/column review grows diff --git a/README.md b/README.md index eef0be9c..2763763a 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,13 @@ For the clone-based manual standalone installation test on macOS, Windows, and L [Italian procedure](docs/install/standalone-manual-it.md) or the [English procedure](docs/install/standalone-manual-en.md). +The [public manual](https://git.tylconsulting.it/thothii-docs/) covers the product, +installation, use and administration. Developer architecture, contracts, ADRs, tests, +plans and release records remain in this repository but are excluded from MkDocs +pages and search. This is an editorial boundary, not an access restriction on the +public repository. See the [documentation cleanup review](docs/maintenance/2026-09-15-documentation-cleanup.md) +for the proposed consolidation; no historical source documents have been deleted. + ## Docker Compose: local startup Requirements: Docker Engine with Compose v2. The mandatory stack is `frontend`, `core`, diff --git a/docs/evidence.md b/docs/evidence.md index 5c47124c..8d060c1f 100644 --- a/docs/evidence.md +++ b/docs/evidence.md @@ -4,7 +4,7 @@ This page describes the complete workspace Evidence lifecycle: where original ma ## Editable local Evidence -E1 adds [Curated Evidence v4 and a persistent local archive](contracts/curated-evidence-v4.md). +E1 adds [Curated Evidence v4 and a persistent local archive](https://git.tylconsulting.it/mptyl/ThothII/src/branch/main/docs/contracts/curated-evidence-v4.md). The visible title and payload fields are authoritative Markdown. New manual units need no external source; consolidation records their curator and distinguishes later corrections from original documentary provenance. The core archive API creates immutable candidates @@ -14,12 +14,12 @@ E2 adds **Administration → Evidence management**, actual host file paths, comp browsing and filtering, and the installed `tht workspace evidence consolidate --workspace ` command. Edit files externally, consolidate to activate them, then review and run Git manually. Runtime consumes only the active local snapshot. -The [v4 contract](contracts/curated-evidence-v4.md#administration-and-installed-command) +The [v4 contract](https://git.tylconsulting.it/mptyl/ThothII/src/branch/main/docs/contracts/curated-evidence-v4.md#administration-and-installed-command) describes host mounting, first conversion, failure recovery and Clear behavior. The local PSD preview has 35 converted and indexed units. E3 adds explicit import/refresh from local drafts and configured HTTP/S3 sources, durable current/proposed comparisons, and keep/replace decisions with activation and retry. See -[Import drafts and refresh sources](contracts/curated-evidence-v4.md#import-drafts-and-refresh-sources). +[Import drafts and refresh sources](https://git.tylconsulting.it/mptyl/ThothII/src/branch/main/docs/contracts/curated-evidence-v4.md#import-drafts-and-refresh-sources). Workflow gate corrections remain the subsequent shared increment, X1. ## Existing repository publication path @@ -92,7 +92,7 @@ Join orders using the order number, financial year and company. Use `tht evidence migrate ` for deterministic legacy conversion. The conversion preserves typed content and initializes an archive baseline; it does not -activate the local corpus. See the [v4 contract](contracts/curated-evidence-v4.md) for +activate the local corpus. See the [v4 contract](https://git.tylconsulting.it/mptyl/ThothII/src/branch/main/docs/contracts/curated-evidence-v4.md) for all eight kinds, provenance, file layout and the E1/E2 boundary. ## Legacy v3 representation @@ -262,5 +262,5 @@ Formulas use a format distinct from document Evidence. A formula proposed during ## Contract references -- [Workspace Evidence v3 contract](contracts/workspace-evidence-v3.md) -- [Preprocessing CLI contract](contracts/workspace-preprocessing-cli.md) +- [Workspace Evidence v3 contract](https://git.tylconsulting.it/mptyl/ThothII/src/branch/main/docs/contracts/workspace-evidence-v3.md) +- [Preprocessing CLI contract](https://git.tylconsulting.it/mptyl/ThothII/src/branch/main/docs/contracts/workspace-preprocessing-cli.md) diff --git a/docs/guida-utente.md b/docs/guida-utente.md index c2714339..b032b057 100644 --- a/docs/guida-utente.md +++ b/docs/guida-utente.md @@ -90,4 +90,4 @@ happen from the workflow’s point of view. session does not become Evidence automatically: a curator must review and publish it in Git. - For login and access recovery, use [local authentication](install/authentication-local.md) or [OIDC authentication](install/authentication-oidc.md) for full, or contact the - portal administrator for [embedded/upstream access](install/authentication-upstream.md). + portal administrator for [embedded/upstream access](install/shell-and-language.md). diff --git a/docs/index.md b/docs/index.md index eca41e32..150c5fcc 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,35 +1,33 @@ # ThothII documentation -ThothII is a human-reviewed datamart builder. It turns a question into validated SQL through an -eight-phase workflow: the model proposes; a reviewer makes the decisions that are persisted. +ThothII turns a natural-language question into reviewed SQL. The model proposes; +a human reviewer decides which interpretations, sources and results to accept. -Start with the path that matches the work you need to do: +This is the public product manual. Start with the task you need to perform: | I need to… | Start here | | --- | --- | -| Install or operate one instance | [Install and first start](install/first-start.md) | -| Clone and manually install on macOS, Windows, or Linux | [Italian procedure](install/standalone-manual-it.md) · [English procedure](install/standalone-manual-en.md) | -| Upgrade the server and integrate Omics Portal | [Codex server handoff](operations/server-codex-handoff.md) | -| Choose full/embedded and configure the shell | [Shell and localization](operations/shell-and-localization.md) | -| Reuse an authenticated server portal without a second login | [Upstream authentication](install/authentication-upstream.md) | -| Understand how the two renderings share the same application | [Rendering architecture](architecture/application-shell.md) | -| Validate login, logout and portal integration before release | [Acceptance matrix](testing/authentication-manual-acceptance.md) | -| Add, update, or prepare a workspace | [Workspace operations](operations/workspaces.md) | -| Ask a question and review the SQL workflow | [User guide](guida-utente.md) | -| Configure and refresh an authoritative database catalog | [Database management](operations/database-management.md) | -| Author and publish domain Evidence | [Evidence](evidence.md) | -| Understand boundaries and persistence | [Architecture overview](architecture/overview.md) | +| Understand the product and its boundaries | [What ThothII does](product-overview.md) | +| Install on Mac, Windows through WSL2, or Linux | [Italian procedure](install/standalone-manual-it.md) · [English procedure](install/standalone-manual-en.md) | +| Choose display mode and language | [Display mode and language](install/shell-and-language.md) | +| Configure login | [Local accounts](install/authentication-local.md) · [OIDC](install/authentication-oidc.md) | +| Configure model providers | [Model configuration](general/pi-configuration.md) | +| Prepare a domain workspace | [Workspaces](operations/workspaces.md) | +| Configure databases and reviewed descriptions | [Database management](operations/database-management.md) | +| Ask a question and review SQL | [User guide](guida-utente.md) · [Workflow](skills.md) | +| Maintain domain knowledge | [Evidence](evidence.md) · [Memory](usage/memory.md) | -## How the documentation is organised +## Scope of this manual -- **Install and operate** documents host-side setup, authentication, lifecycle, workspaces, and - Pi administration. -- **Use ThothII** documents the two application paths: reviewed NL→SQL sessions and administrative - database management. -- **Architecture and contracts** explain why the system behaves as it does and define the - machine-facing boundaries. Consult them when integrating or changing an implementation; they - are not a substitute for an operator runbook. +The manual covers the product, installation, use and administration. Architecture, +code contracts, design decisions, implementation plans, test reports and site-specific +deployment handoffs are developer/project material maintained in the repository; +they are not pages of this site and are not included in its search index. -The host-side `tht` command is the operator interface. The Python `tht` inside `harness/` is a -different, internal workflow CLI invoked by the core. In examples, use an absolute installation -descriptor path whenever discovery is not unambiguous. +The installation procedures distinguish checked documentation from platform and +functional tests that still require execution. A clone does not transfer another +installation's credentials, data or network access. + +The host-side `tht` command operates an installation. The Python workflow CLI inside +the runtime is a separate internal interface; do not substitute its commands for +the host installation procedure. diff --git a/docs/install/authentication-local.md b/docs/install/authentication-local.md index 8dcfd589..0e1d5cc3 100644 --- a/docs/install/authentication-local.md +++ b/docs/install/authentication-local.md @@ -3,7 +3,7 @@ Use local mode for a standalone PC or Mac, with `shell.mode: full` and `shell.defaultLocale: en` in the installation descriptor. Presentation and authentication are independent: selecting full does not create accounts. Omics -embedded instead uses the [upstream guide](authentication-upstream.md), not local users. +embedded instead uses the [upstream guide](shell-and-language.md), not local users. Configure local authentication through `tht`; passwords are entered at an echo-free prompt or read from a protected `--password-file`, never from a command argument. diff --git a/docs/install/authentication-oidc.md b/docs/install/authentication-oidc.md index 84d42a46..e5d7e6db 100644 --- a/docs/install/authentication-oidc.md +++ b/docs/install/authentication-oidc.md @@ -2,7 +2,7 @@ Use this guide for **ThothII's own login**, normally `shell.mode: full` on an autonomous server. It is not the integration procedure for an already logged-in -Omics user. That deployment uses [embedded/upstream](authentication-upstream.md), +Omics user. That deployment uses [embedded/upstream](shell-and-language.md), even when Omics's identity provider is Authentik. OIDC mode supports a standards-based provider. The browser flow is generic: Authorization Code, @@ -104,7 +104,7 @@ Any authentication failure prevents activation according to the static or live s relevant diagnostic surface. The complete closed diagnostic-code union and exact role-to-permission expansion are in the -[authentication architecture](../architecture/authentication.md). +[authentication architecture](https://git.tylconsulting.it/mptyl/ThothII/src/branch/main/docs/architecture/authentication.md). ## Browser login and logout @@ -114,4 +114,4 @@ password prompt, but this remains a distinct ThothII login/session, unlike Omics upstream. Full's name menu logs out of ThothII only. It does not revoke the provider session or log out other applications, so a subsequent login can return immediately through SSO. No provider token is placed in the UI adapter or browser -storage. See the [manual acceptance matrix](../testing/authentication-manual-acceptance.md). +storage. See the [manual acceptance matrix](https://git.tylconsulting.it/mptyl/ThothII/src/branch/main/docs/testing/authentication-manual-acceptance.md). diff --git a/docs/install/authentik.md b/docs/install/authentik.md index 48a3c30b..b2c944e0 100644 --- a/docs/install/authentik.md +++ b/docs/install/authentik.md @@ -8,7 +8,7 @@ For **embedded in Omics**, retain Omics's existing Authentik authentication and configure ThothII as upstream. Omics verifies `datamart_builder.access` and administrator status and the proxy supplies the identity; no additional ThothII OIDC client, login or local user is required for that path. Follow the -[portal integration guide](authentication-upstream.md). +[portal integration guide](shell-and-language.md). ```mermaid sequenceDiagram diff --git a/docs/install/first-start.md b/docs/install/first-start.md index 6ce3b037..2b30b413 100644 --- a/docs/install/first-start.md +++ b/docs/install/first-start.md @@ -1,98 +1,64 @@ # Install and first start -For the ordered clone-to-start procedure on Mac, Windows WSL2 and Linux, use the -[Italian manual guide](standalone-manual-it.md) or [English manual guide](standalone-manual-en.md), -including protected credentials and the explicit initial catalog migration. +Use one complete procedure for a fresh installation: -This is the supported local installation path. It creates an installation-local configuration and -starts the Compose stack; it does not create a workspace repository or a database catalog entry. +- [Italian manual installation](standalone-manual-it.md) +- [English manual installation](standalone-manual-en.md) -## Prerequisites and boundaries +Both cover macOS, Windows through Ubuntu WSL2, and Linux. They use a Gitea clone, +protected local configuration and manual terminal commands, without an application +installer or launcher. See their verification matrix for tests still pending. -Install Docker Engine with Compose v2, plus the host `tht` command. On macOS or Linux, install the -host command from the repository with `./scripts/install-tht.sh`; Windows uses -`./scripts/install-tht.ps1`. The installer builds or verifies the native command and checks that -`tht` is resolvable on `PATH`. +## What must be ready -The stack contains `frontend`, `core`, `catalog-db`, `qdrant`, `embedding`, and the one-shot -`embedding-model-init` and `catalog-migrate` services. DWH and model-provider endpoints are -external installation settings. Pi runs inside `core`; do not install a host Pi executable for -the application runtime. +You need Docker with Compose, the host operator command `tht`, access to the workspace +repository, and the credentials and network routes for the configured DWH and model +providers. Pi runs inside the application runtime; no host Pi installation is needed. -Secrets, certificates, Pi authentication, and workspace endpoint bindings are protected -installation-local files. Never put them in a workspace descriptor, an env file intended for -version control, a URL, or a command line. +The stack includes `frontend`, `core`, `catalog-db`, `qdrant`, `embedding`, plus the +one-shot `embedding-model-init` and `catalog-migrate` services. DWH and generative-model +endpoints remain separate installation settings. -## Create the local installation +Secrets, certificates, Pi authentication and endpoint bindings are protected local files. +Do not commit them or copy the configuration of another machine unchanged. -From the repository root, start the interactive setup and select the local profile: +## Follow the ordered procedure + +The bilingual guides provide the exact commands for: + +1. Cloning the selected revision and checking prerequisites. +2. Bootstrapping the native host command. +3. Preparing catalog passwords and using + `tht setup --profile local --shell-mode full --shell-default-locale en --configure-only`. +4. Completing model, authentication and workspace credentials. +5. Generating configuration, building images and explicitly running `catalog-migrate`. +6. Starting the installation and checking health and readiness. + +Do not run setup alone as a substitute for that sequence. Migrations are not an +implicit effect of backend startup or `tht start`. Do not mix this installation's +descriptor/project with a different low-level Compose environment. + +For an already configured installation: ```sh -tht setup --profile local --shell-mode full --shell-default-locale en -``` - -It writes the selected non-secret descriptor below `deploy//`, the associated -operator env file, and can create protected secret templates. Keep the descriptor path: pass it -to commands as `--installation /absolute/path/thothii-installation.yaml` when more than one -installation can be discovered. - -The explicit shell options are important: compatibility defaults without them -are embedded/en/omics-portal, which expects an Omics document. The Mac's standalone -installation must use full, with English as its initial locale. Existing browser -language preferences take precedence over that initial value. Full does not -configure authentication; setup separately bootstraps local login. - -For a server portal, use the [embedded/upstream procedure](authentication-upstream.md) -instead of creating a second ThothII login. For a standalone server, use full -with [direct OIDC](authentication-oidc.md). Shell mode does not follow `profile` -automatically. See [configuration and regeneration](../operations/shell-and-localization.md) -before modifying an existing installation. - -If the descriptor is prepared manually instead, begin with -[`thothii-installation.local.yaml`](examples/thothii-installation.local.yaml), set mode `0600` or -`0400`, and ensure `THT_INSTALLATION_CONFIG_SOURCE` in the selected env file points to that exact -file. Create the secret bundle from `deploy/secrets/thothii.secrets.example`, protect it, and set -the file locations and external endpoints in the env file. The required settings include: - -- `PI_AUTH_FILE`, `THT_SECRETS_FILE`, and the catalog password source files; -- `THT_INSTALLATION_CONFIG_SOURCE` and the workspace Git remote/branch; -- the DWH and model-provider endpoints; and -- `THT_AUTH_CONFIG_ROOT` for local authentication or the OIDC configuration selected during setup. - -For the supported secret names and the metadata-generation model credential boundary, see the -`deploy/secrets/README.md` file in the installation checkout. It is intentionally not published -as a documentation page because it describes a protected local-file contract. - -## Start and verify - -For the normal local path, use the launcher: - -```sh -./scripts/run-stack.sh -``` - -It builds `core`, starts `catalog-db`, runs `catalog-migrate`, then keeps the base plus local -Compose profile in the foreground. Database migrations are deliberately not a hidden backend -startup action. Open `http://127.0.0.1:8080` unless `THOTH_HTTP_PORT` was changed. - -In another terminal, verify the installation without changing it: - -```sh -tht --installation /absolute/path/thothii-installation.yaml doctor --json tht --installation /absolute/path/thothii-installation.yaml status +tht --installation /absolute/path/thothii-installation.yaml doctor --json ``` -`/health` verifies application-process readiness; `tht doctor` is the diagnostic surface for -Compose, configuration, workspace, workflow, and Pi prerequisites. +`/health` checks application-process readiness. Doctor also checks configuration, +workspace, workflow and Pi prerequisites; a healthy web page alone does not prove +that a real database question can complete. -## Routine lifecycle and next steps +## After startup -Use `tht start [--build]`, `tht stop`, `tht logs`, and `tht doctor` rather than composing ad-hoc -container commands. Named volumes retain settings, Pi state, workspace registry, sessions, -Qdrant data, and embedding models across `docker compose down`; removing them requires the -explicit destructive `--volumes` form. +Prepare [workspaces](../operations/workspaces.md), configure a database in +[Database Management](../operations/database-management.md), and complete the functional +checks in the installation guide before using real data. -After the stack is healthy, configure authentication if setup did not do so, then continue with -[Workspace operations](../operations/workspaces.md). For server profile, reverse proxy, backups, -and recovery, use the deployment program and its manual gates; the server profile is not a -drop-in replacement for the local command above. +See [display mode and language](shell-and-language.md), [local authentication](authentication-local.md), +[OIDC](authentication-oidc.md) and [model configuration](../general/pi-configuration.md) +for later changes. Embedded portal integration is separate from a fresh standalone setup. + +Use the installation's normal `tht start`, `tht stop` and diagnostic commands. +Preserve its descriptor, credentials, database and persistent volumes; do not use +`down --volumes` as a routine stop or upgrade. diff --git a/docs/install/shell-and-language.md b/docs/install/shell-and-language.md new file mode 100644 index 00000000..f100bd78 --- /dev/null +++ b/docs/install/shell-and-language.md @@ -0,0 +1,70 @@ +# Display mode and language + +ThothII can run with its own application header (**full**) or inside an integrated +portal (**embedded**). Display mode and authentication are separate choices. + +| Installation | Display | Authentication | +| --- | --- | --- | +| Standalone local instance | `full` | Local ThothII account | +| Standalone server | `full` | Local accounts or configured OIDC provider | +| Integrated portal | `embedded` | Identity verified by the portal's trusted server proxy | + +## Standalone setup + +Follow the complete [Italian](standalone-manual-it.md) or +[English](standalone-manual-en.md) installation procedure. It explicitly selects +`--shell-mode full --shell-default-locale en` and separates configuration, credentials, +initial migrations and startup. Do not skip those steps by running setup alone. + +The authored installation descriptor contains: + +```yaml +shell: + mode: full + defaultLocale: en +``` + +Use `it` for an Italian initial interface. Existing browser language preferences can +override that initial value. Full mode remembers language and theme in the browser. + +For an existing installation, preserve the current descriptor and edit only the intended +settings; do not rerun setup to overwrite it. With a current host `tht` binary: + +```sh +tht --installation /absolute/path/thothii-installation.yaml installation generate +tht --installation /absolute/path/thothii-installation.yaml start +tht --installation /absolute/path/thothii-installation.yaml status +tht --installation /absolute/path/thothii-installation.yaml doctor --json +``` + +Generation updates derived configuration; it does not start services. `start` applies +the installation's normal lifecycle and is not guaranteed to touch only the frontend. +Follow the deployment's maintenance procedure and retain its network, authentication and +model settings. Do not edit generated files or remove persistent volumes. + +## Authentication and embedded deployments + +Full mode does not configure login by itself. See [local authentication](authentication-local.md) +or [OIDC](authentication-oidc.md), with [Authentik](authentik.md) as a provider option. + +Embedded mode requires a compatible portal integration, not just a descriptor toggle. +The portal owns login/logout and supplies a server-verified identity. A presentation +adapter does not authenticate users. The core must not be reachable by a route that +bypasses the trusted proxy. Do not add a second OIDC login to an already authenticated +upstream deployment. + +Portal implementation details belong to the +[developer integration reference in the repository](https://git.tylconsulting.it/mptyl/ThothII/src/branch/main/docs/install/authentication-upstream.md), +not to the standalone installation procedure. + +## Three different languages + +- **Interface language** controls labels, forms and application messages. +- **Session interaction language** is captured when a session is created. Resuming it + retains that language even if the interface language changes later. +- **Workspace language** concerns domain content and retrieval; switching the interface + does not translate Evidence, SQL, identifiers or database values. + +In embedded mode the interface follows the portal's language and theme. A portal language +change may reload the page. Saved session artifacts remain available, but resuming work +is explicit; a reload does not by itself request a new model generation. diff --git a/docs/install/standalone-manual-en.md b/docs/install/standalone-manual-en.md index 9241e6bb..b09586c9 100644 --- a/docs/install/standalone-manual-en.md +++ b/docs/install/standalone-manual-en.md @@ -319,6 +319,6 @@ The following remain future work: ## Related documents - [Install and first start](first-start.md) -- [Shell and localization](../operations/shell-and-localization.md) +- [Shell and localization](shell-and-language.md) - [Workspace operations](../operations/workspaces.md) - `deploy/secrets/README.md` (runtime secrets) diff --git a/docs/install/standalone-manual-it.md b/docs/install/standalone-manual-it.md index c3375c97..173adff1 100644 --- a/docs/install/standalone-manual-it.md +++ b/docs/install/standalone-manual-it.md @@ -324,6 +324,6 @@ Restano attività successive: ## Documenti collegati - [Install and first start](first-start.md) -- [Shell and localization](../operations/shell-and-localization.md) +- [Shell and localization](shell-and-language.md) - [Workspace operations](../operations/workspaces.md) - `deploy/secrets/README.md` (runtime secrets) diff --git a/docs/maintenance/2026-09-15-documentation-cleanup.md b/docs/maintenance/2026-09-15-documentation-cleanup.md new file mode 100644 index 00000000..55820add --- /dev/null +++ b/docs/maintenance/2026-09-15-documentation-cleanup.md @@ -0,0 +1,307 @@ +# Revisione e proposta di bonifica della documentazione + +Data: 15 settembre 2026. Baseline: `6a4634dcf1b1df4ad29510a4da371245abd5c666`. +Ambito: 121 Markdown e 23 altri file già presenti in `docs/`, navigazione, collegamenti +locali, build e verificatori documentali. Questa è una proposta di manutenzione interna, +non una pagina del manuale pubblico e non un'autorizzazione a cancellare i file elencati. + +## Esito e modifica applicata + +Il sito confondeva quattro funzioni: manuale del prodotto, riferimento del codice, +progettazione e diario delle consegne. Il problema non era soltanto la lunghezza della nav: +le pagine non elencate potevano comunque essere generate e indicizzate. + +La configurazione ora pubblica soltanto 20 pagine e cinque asset esplicitamente ammessi. +Il resto è escluso da HTML, file copiati e indice di ricerca. Le nuove pagine pubbliche +sono `product-overview.md`, `install/shell-and-language.md` e `usage/memory.md`; +i documenti tecnici da cui sono state ricavate restano intatti nel repository. +Home e `install/first-start.md` sono state riscritte per dare un ingresso univoco. +Non è stato cancellato alcun documento della baseline. + +`exclude_docs` nega per default la pubblicazione: aggiungere un file a `docs/` non basta +più a metterlo online. La build e la pipeline verificano corrispondenza tra nav ed +eccezioni, pagine generate, asset e ricerca. I riferimenti tecnici ancora necessari +al lettore rinviano esplicitamente ai sorgenti su Gitea, non a pagine interne del sito. + +**Fuori dal manuale non significa privato.** Il repository ThothII è pubblico: questi +file e la cronologia restano leggibili su Gitea. Per riservatezza reale occorrerebbe una +decisione separata su repository/accessi e sulla storia già pubblicata. Questa bonifica +non modifica autorizzazioni, altri repository o l'installazione applicativa. + +## Tre destinazioni, con regole diverse + +| Destinazione | Contenuto | Regola | +| --- | --- | --- | +| Manuale pubblico | Prodotto, uso, installazione, configurazione, amministrazione | Descrivere ciò che il lettore può fare oggi; distinguere prerequisiti e limiti verificati. | +| Riferimento developer | Architettura, contratti, ADR, test ripetibili, integrazione | Conservare nel repository, fuori da MkDocs; un'autorità per ciascun contratto. | +| Lavoro temporaneo o storico | Piani attuati, prompt di ripresa, survey, report di singola consegna | Estrarre decisioni, difetti aperti e rollback ancora necessari; poi ritirare dal working tree con Git come archivio. | + +Un file datato non è automaticamente obsoleto. Un contratto con `v3` nel nome non è +automaticamente sostituito da un descriptor v4: sono versioni di oggetti diversi. +Un ADR superato conserva il motivo della decisione e il collegamento al successore. + +## Rilievi prioritari, con motivazione + +### 1. Percorsi di installazione concorrenti — corretto per il pubblico + +`install/first-start.md` proponeva ancora setup senza `--configure-only` e avvio con +`run-stack.sh`, mentre le guide manuali separano segreti, migrazione e avvio per lo +stesso descriptor/progetto. Ora è un punto d'ingresso alle due procedure complete, +non una terza ricetta. `installazione-docker-4-contesti.md` rimane interno: consolidarne +i dettagli ancora esclusivi in una procedura developer di distribuzione, poi ritirarlo. +Anche il percorso rapido nel README va riallineato prima di eliminare quei dettagli. + +### 2. Ricerca architetturale presentata come stato corrente — ritirare dopo estrazione + +`research/2026-08-23-thothii-metadata-publication-qdrant-seams.md` dichiara ancora il +passaggio catalog-to-core «deferred» e descrive `annotations.yaml` come input corrente. +ADR 0016 e gli attuali contratti registrano invece PostgreSQL come autorità per i +consumatori del core. Non usarlo per implementare nuovi comportamenti. + +`research/2026-08-23-postgresql-catalog-deployment-constraints.md` dichiara esplicitamente +di essere parzialmente superato da ADR 0004. L'inventario legacy ThothAI del 23 agosto +è esplicitamente storico. Estrarre soltanto vincoli non già presenti in ADR/contratti; +poi eliminare i tre file dal working tree, conservandoli nella storia Git. + +### 3. Decisioni correnti distribuite tra troppi piani — consolidare prima di eliminare + +Per Memory/Evidence convivono piani di famiglia, M1, amministrazione comune, revisione +di semplificazione e sette resoconti M1–M3/E1–E3/X1. Conservare gli invarianti nei +contratti e gli scenari ripetibili in `testing/`; trasferire i gate non chiusi in issue. +Solo dopo si possono ritirare piani e validazioni di incremento. + +`adr/0019-author-evidence-in-app-with-automatic-activation.md` ha un nome che suggerisce +editor in-app e attivazione automatica, ma il titolo descrive editor esterni e +consolidamento manuale. Contiene anche «Implementation is pending»: non è un affidabile +indicatore dello stato di tutte le parti della release. Conservare numero/decisione, +correggere metadati e rimandi e distinguere eventuali estensioni ancora pendenti. + +### 4. Report operativi e piani con stato non aggiornato — non cancellare in blocco + +Il report `2026-09-12-ui-visual-review-delivery.md` apre con «non integrata in main»; +le integrazioni successive sono documentate altrove. I piani full-shell dicono ancora +«deploy server non eseguito», mentre esistono report di rilascio server del 14 settembre. +Ciò prova che lo stato è distribuito, non che tutti i collaudi siano conclusi. + +Consolidare report UI del 12–13 settembre in un record di rilascio per versione. +Non ritirare i report del 14 settembre né i runbook di upgrade finché rollback, +accettazione interattiva e finestra di osservazione non risultano chiusi dall'operatore. +`server-handoff-260906-preprocessing-complete.md` va confrontato con il runbook corrente +`server-codex-handoff.md`: estrarre eventuali passaggi di preprocessing esclusivi prima +di ritirare la consegna datata. + +### 5. Ricerca su strumenti personali estranea al manuale — candidata all'eliminazione + +I sei confronti CLI/editor/provider del 7 settembre (Antigravity, DeepSeek/CyberArk, +OMP/Codex/ZCode, Pi/Oh My Pi, quota Z.ai e Zed/ACP) non spiegano un contratto di ThothII. +Prezzi, quote e confronti non sono stati riverificati in questa revisione. Proposta: +toglierli dal repository del prodotto, mantenendoli in Git o trasferendoli, su scelta +del proprietario, a una raccolta personale. Stessa destinazione proposta per +`research/text-to-sql-products.md`, che è una ricognizione di mercato. + +Non applicare automaticamente questa regola alla ricerca ERD, relationship e +classificatore sensibile: prima verificare se documenta decisioni ancora aperte. + +### 6. Controlli e snapshot già disallineati — debito da correggere + +Due verificatori falliscono su file già assenti nella baseline, non per l'esclusione +da MkDocs introdotta qui: + +- `scripts/auth-docs-smoke.sh`: manca `docs/install/local.md`. +- `scripts/verify-dwh-auth-docs.sh`: manca `docs/operations/psd-dwh-auth-rollout.md`. + +Le relative suite di fixture conservano la vecchia struttura. Non ricreare documenti +obsoleti per far passare i test: aggiornare i controlli ai contratti attuali, mantenendo +le verifiche su segreti, modalità di autenticazione e TLS. Questo riallineamento è +proposto, non eseguito in questa bonifica editoriale. + +`PROJECT_STATE.md` cita nove percorsi `docs/*.md` non più esistenti, fra cui il programma +server PSD del 20 agosto, l'accettazione Evidence del 25 agosto e il rollout dwh-auth. +Inoltre accumula consegne anziché restare uno snapshot breve. Riscriverlo come stato +attuale, gate aperti e rimandi esistenti. I normali link Markdown relativi dei documenti +esistenti non presentano target mancanti nella verifica eseguita: i riferimenti rotti +qui citati sono anche percorsi in backtick o imposti dagli script. + +## Proposta operativa in ordine + +1. **Separazione pubblica:** applicata; niente cancellazioni di sorgenti. +2. **Riallineare le autorità:** snapshot, verificatori, ADR 0019, README e stato delle + consegne; spostare gate aperti in issue senza marcarli completati. +3. **Ritiro a basso rischio, previa approvazione:** confronti personali, ricerca già + esplicitamente superata, prompt di ripresa dopo assorbimento nel PRD, output visuali + rigenerabili. Controllare prima i riferimenti in tutto il repository. +4. **Consolidamento:** piani completati e report incrementali; non perdere criteri di + accettazione, problemi aperti, provenienza delle immagini e rollback ancora validi. +5. **Riorganizzazione eventuale:** `docs/` per il pubblico, `developer-docs/` per + architettura/contratti/ADR/testing, `project-notes/` per lavoro in corso. È un secondo + intervento: aggiornare insieme tutti i link in README, AGENTS, CONTEXT, script e codice. + +Per ogni ritiro usare un commit dedicato con motivazione e documento sostitutivo. +Non creare una cartella `archive/` piena di copie obsolete: la cronologia Git è già +l'archivio, salvo una necessità operativa o di conservazione esplicita. + +## Inventario e destinazione proposta + +L'inventario seguente distingue l'azione proposta dalla sola esclusione già applicata +al sito. «Consolidare» e «ritirare» non indicano cancellazioni eseguite. + + +| File (relativo a `docs/`) | Destinazione | Azione proposta | +| --- | --- | --- | +| `adr/0001-postgres-metadata-catalog.md` | Developer | Conservare decisione architetturale. | +| `adr/0002-workspace-database-secret-references.md` | Developer | Conservare decisione architetturale. | +| `adr/0003-installation-local-database-bindings.md` | Developer | Conservare decisione architetturale. | +| `adr/0004-fastify-kysely-metadata-catalog.md` | Developer | Conservare decisione architetturale. | +| `adr/0005-hard-delete-catalog-tables-during-synchronization.md` | Developer | Conservare ADR superato e catena dei successori; non eliminarlo. | +| `adr/0006-separate-physical-and-logical-relationships.md` | Developer | Conservare decisione architetturale. | +| `adr/0007-durable-authoritative-schema-synchronization.md` | Developer | Conservare decisione architetturale. | +| `adr/0008-allow-manual-catalog-metadata-cleanup.md` | Developer | Conservare decisione architetturale. | +| `adr/0009-use-one-sequential-description-generation-run.md` | Developer | Conservare decisione architetturale. | +| `adr/0010-allow-bounded-real-source-samples-for-description-generation.md` | Developer | Conservare decisione architetturale. | +| `adr/0011-gate-source-samples-with-a-sensitive-data-flag.md` | Developer | Conservare decisione architetturale. | +| `adr/0012-use-the-catalog-as-the-logical-relationship-authority.md` | Developer | Conservare decisione architetturale. | +| `adr/0013-use-one-installation-model-catalog-with-runtime-projections.md` | Developer | Conservare decisione architetturale. | +| `adr/0014-assess-sensitive-columns-locally-from-source-content.md` | Developer | Conservare ADR superato e catena dei successori; non eliminarlo. | +| `adr/0015-use-progressive-sampling-for-sensitive-columns.md` | Developer | Conservare decisione architetturale. | +| `adr/0016-use-postgres-metadata-for-all-core-consumers.md` | Developer | Conservare decisione architetturale. | +| `adr/0017-separate-reference-vectors-from-runtime-memory.md` | Developer | Conservare decisione architetturale. | +| `adr/0018-use-postgres-for-memory-and-qdrant-for-retrieval.md` | Developer | Conservare decisione architetturale. | +| `adr/0019-author-evidence-in-app-with-automatic-activation.md` | Developer | Conservare; allineare nome/stato alla decisione manuale effettiva. | +| `adr/0020-unify-administration-pages-and-use-namespaced-routes.md` | Developer | Conservare decisione architetturale. | +| `adr/0021-separate-shell-modes-and-replaceable-portal-adapter.md` | Developer | Conservare decisione architetturale. | +| `adr/0022-separate-ui-locale-from-session-interaction-language.md` | Developer | Conservare decisione architetturale. | +| `agents/domain.md` | Developer | Conservare regole di contribuzione/agenti, fuori dal sito. | +| `agents/issue-tracker.md` | Developer | Conservare regole di contribuzione/agenti, fuori dal sito. | +| `agents/triage-labels.md` | Developer | Conservare regole di contribuzione/agenti, fuori dal sito. | +| `architecture/application-shell.md` | Developer | Conservare riferimento corrente; distinguere il prodotto dalla struttura del codice. | +| `architecture/authentication.md` | Developer | Conservare riferimento corrente; distinguere il prodotto dalla struttura del codice. | +| `architecture/components.md` | Developer | Conservare riferimento corrente; distinguere il prodotto dalla struttura del codice. | +| `architecture/overview.md` | Developer | Conservare riferimento corrente; distinguere il prodotto dalla struttura del codice. | +| `architecture/thothii-core-sequence.html` | Developer/build | Conservare sorgente/diagramma tecnico; non pubblicare nel manuale. | +| `architecture/thothii-core-sequence.visual-check.1440x900.dark.png` | Transitorio | Ritirare output QA rigenerabili dopo conferma che non siano baseline uniche. | +| `architecture/thothii-core-sequence.visual-check.1440x900.light.png` | Transitorio | Ritirare output QA rigenerabili dopo conferma che non siano baseline uniche. | +| `architecture/thothii-core-sequence.visual-check.2048x1320.dark.png` | Transitorio | Ritirare output QA rigenerabili dopo conferma che non siano baseline uniche. | +| `architecture/thothii-core-sequence.visual-check.2048x1320.light.png` | Transitorio | Ritirare output QA rigenerabili dopo conferma che non siano baseline uniche. | +| `architecture/thothii-core-sequence.visual-check.html` | Transitorio | Ritirare output QA rigenerabili dopo conferma che non siano baseline uniche. | +| `architecture/thothii-core-sequence.visual-check.json` | Transitorio | Ritirare output QA rigenerabili dopo conferma che non siano baseline uniche. | +| `architecture/thothii-core.sequence.json` | Developer/build | Conservare sorgente/diagramma tecnico; non pubblicare nel manuale. | +| `architecture/thothii-runtime.architecture.json` | Developer/build | Conservare sorgente/diagramma tecnico; non pubblicare nel manuale. | +| `architecture/thothii-runtime.html` | Developer/build | Conservare sorgente/diagramma tecnico; non pubblicare nel manuale. | +| `architecture/thothii-runtime.visual-check.1440x900.dark.png` | Transitorio | Ritirare output QA rigenerabili dopo conferma che non siano baseline uniche. | +| `architecture/thothii-runtime.visual-check.1440x900.light.png` | Transitorio | Ritirare output QA rigenerabili dopo conferma che non siano baseline uniche. | +| `architecture/thothii-runtime.visual-check.2048x1320.dark.png` | Transitorio | Ritirare output QA rigenerabili dopo conferma che non siano baseline uniche. | +| `architecture/thothii-runtime.visual-check.2048x1320.light.png` | Transitorio | Ritirare output QA rigenerabili dopo conferma che non siano baseline uniche. | +| `architecture/thothii-runtime.visual-check.html` | Transitorio | Ritirare output QA rigenerabili dopo conferma che non siano baseline uniche. | +| `architecture/thothii-runtime.visual-check.json` | Transitorio | Ritirare output QA rigenerabili dopo conferma che non siano baseline uniche. | +| `contracts/archive-repair.md` | Developer | Conservare contratto e versioni semantiche; aggiornare insieme a codice/test. | +| `contracts/catalog-schema-snapshot.md` | Developer | Conservare contratto e versioni semantiche; aggiornare insieme a codice/test. | +| `contracts/curated-evidence-v4.md` | Developer | Conservare contratto e versioni semantiche; aggiornare insieme a codice/test. | +| `contracts/portal-shell-adapter-v1.md` | Developer | Conservare contratto e versioni semantiche; aggiornare insieme a codice/test. | +| `contracts/tht-dwh.md` | Developer | Conservare contratto e versioni semantiche; aggiornare insieme a codice/test. | +| `contracts/workspace-evidence-v3.md` | Developer | Conservare contratto e versioni semantiche; aggiornare insieme a codice/test. | +| `contracts/workspace-preprocessing-cli.md` | Developer | Conservare contratto e versioni semantiche; aggiornare insieme a codice/test. | +| `disambiguazione-iniziale.md` | Developer | Conservare invarianti di gate/ledger; il percorso utente è in skills.md. | +| `evidence.md` | Pubblico | Conservare; istruzioni di prodotto/operatore. | +| `general/pi-configuration.md` | Pubblico | Conservare; istruzioni di prodotto/operatore. | +| `gestione-memory.md` | Developer | Conservare contratto di Memory; istruzioni pubbliche in usage/memory.md. | +| `guida-utente.md` | Pubblico | Conservare; istruzioni di prodotto/operatore. | +| `index.md` | Pubblico | Conservare; istruzioni di prodotto/operatore. | +| `install/authentication-local.md` | Pubblico | Conservare; istruzioni di prodotto/operatore. | +| `install/authentication-oidc.md` | Pubblico | Conservare; istruzioni di prodotto/operatore. | +| `install/authentication-upstream.md` | Developer/integratore | Conservare integrazione e confine di fiducia; guida pubblica separata. | +| `install/authentik.md` | Pubblico | Conservare; istruzioni di prodotto/operatore. | +| `install/dwh-auth-client-enrollment.md` | Pubblico | Conservare; istruzioni di prodotto/operatore. | +| `install/dwh-auth-server.md` | Pubblico | Conservare; istruzioni di prodotto/operatore. | +| `install/dwh-auth-tls.md` | Pubblico | Conservare; istruzioni di prodotto/operatore. | +| `install/examples/thothii-installation.local.yaml` | Asset pubblico | Conservare asset o esempio operativo. | +| `install/examples/thothii-installation.server.yaml` | Asset pubblico | Conservare asset o esempio operativo. | +| `install/examples/workspace-bindings.env.example` | Asset pubblico | Conservare asset o esempio operativo. | +| `install/first-start.md` | Pubblico | Conservare; istruzioni di prodotto/operatore. | +| `install/standalone-manual-en.md` | Pubblico | Conservare; istruzioni di prodotto/operatore. | +| `install/standalone-manual-it.md` | Pubblico | Conservare; istruzioni di prodotto/operatore. | +| `installazione-docker-4-contesti.md` | Misto/transitorio | Estrarre dettagli server unici, poi ritirare la ricetta duplicata. | +| `javascripts/layout-init.js` | Asset pubblico | Conservare asset o esempio operativo. | +| `operations/database-management.md` | Pubblico | Conservare; istruzioni di prodotto/operatore. | +| `operations/docker-refresh.md` | Operazioni interne | Conservare finché descrive il contesto locale attivo; poi consolidare il lifecycle. | +| `operations/sensitivity-analysis.md` | Pubblico | Conservare; istruzioni di prodotto/operatore. | +| `operations/server-codex-handoff.md` | Operazioni interne | Conservare runbook finché migrazione e rollback sono operativamente necessari. | +| `operations/server-handoff-260906-preprocessing-complete.md` | Operazioni/transitorio | Confrontare con server-codex-handoff, assorbire passaggi unici, poi ritirare. | +| `operations/server-upgrade-gitea-workspace-v2.md` | Operazioni interne | Conservare runbook finché migrazione e rollback sono operativamente necessari. | +| `operations/shell-and-localization.md` | Developer/integratore | Conservare integrazione e confine di fiducia; guida pubblica separata. | +| `operations/workspaces.md` | Pubblico | Conservare; istruzioni di prodotto/operatore. | +| `plans/2026-08-26-metadata-catalog-from-thothai.md` | Progetto/transitorio | Consolidare invarianti in contratti e gate in issue/test; poi ritirare piano o resoconto. | +| `plans/2026-08-28-ai-catalog-description-generation-spec.md` | Progetto/transitorio | Consolidare invarianti in contratti e gate in issue/test; poi ritirare piano o resoconto. | +| `plans/2026-08-28-ai-catalog-description-generation.md` | Progetto/transitorio | Consolidare invarianti in contratti e gate in issue/test; poi ritirare piano o resoconto. | +| `plans/2026-09-02-installation-model-catalog.md` | Progetto/transitorio | Consolidare invarianti in contratti e gate in issue/test; poi ritirare piano o resoconto. | +| `plans/2026-09-08-evidence-management.md` | Progetto/transitorio | Consolidare invarianti in contratti e gate in issue/test; poi ritirare piano o resoconto. | +| `plans/2026-09-08-memory-evidence-administration.md` | Progetto/transitorio | Consolidare invarianti in contratti e gate in issue/test; poi ritirare piano o resoconto. | +| `plans/2026-09-08-memory-evidence-simplification-review.md` | Progetto/transitorio | Consolidare invarianti in contratti e gate in issue/test; poi ritirare piano o resoconto. | +| `plans/2026-09-08-memory-m1-spec.md` | Progetto/transitorio | Consolidare invarianti in contratti e gate in issue/test; poi ritirare piano o resoconto. | +| `plans/2026-09-08-memory-m1-validation.md` | Progetto/transitorio | Consolidare invarianti in contratti e gate in issue/test; poi ritirare piano o resoconto. | +| `plans/2026-09-08-memory-management.md` | Progetto/transitorio | Consolidare invarianti in contratti e gate in issue/test; poi ritirare piano o resoconto. | +| `plans/2026-09-08-security-hardening-prd.md` | Progetto attivo | Conservare PRD aperto; risolvere decisioni/issue prima del ritiro. | +| `plans/2026-09-08-security-hardening-resume-prompt.md` | Transitorio | Ritirare dopo trasferimento di istruzioni e questioni aperte nel PRD/issue. | +| `plans/2026-09-09-archive-repair-x1-validation.md` | Progetto/transitorio | Consolidare invarianti in contratti e gate in issue/test; poi ritirare piano o resoconto. | +| `plans/2026-09-09-evidence-e1-validation.md` | Progetto/transitorio | Consolidare invarianti in contratti e gate in issue/test; poi ritirare piano o resoconto. | +| `plans/2026-09-09-evidence-e2-validation.md` | Progetto/transitorio | Consolidare invarianti in contratti e gate in issue/test; poi ritirare piano o resoconto. | +| `plans/2026-09-09-evidence-e3-validation.md` | Progetto/transitorio | Consolidare invarianti in contratti e gate in issue/test; poi ritirare piano o resoconto. | +| `plans/2026-09-09-memory-m2-validation.md` | Progetto/transitorio | Consolidare invarianti in contratti e gate in issue/test; poi ritirare piano o resoconto. | +| `plans/2026-09-09-memory-m3-validation.md` | Progetto/transitorio | Consolidare invarianti in contratti e gate in issue/test; poi ritirare piano o resoconto. | +| `plans/2026-09-10-administration-pages-spec.md` | Progetto/transitorio | Consolidare invarianti in contratti e gate in issue/test; poi ritirare piano o resoconto. | +| `plans/2026-09-13-full-shell-and-portal-integration.md` | Progetto/transitorio | Consolidare invarianti in contratti e gate in issue/test; poi ritirare piano o resoconto. | +| `plans/2026-09-13-full-shell-spec.md` | Progetto/transitorio | Consolidare invarianti in contratti e gate in issue/test; poi ritirare piano o resoconto. | +| `plans/2026-09-14-manual-standalone-installation.md` | Progetto attivo | Conservare fino alla verifica su tre piattaforme; poi assorbire gate e ritirare. | +| `plans/administration-pages/01-navigation.md` | Progetto/transitorio | Consolidare invarianti in contratti e gate in issue/test; poi ritirare piano o resoconto. | +| `plans/administration-pages/02-workspace-readiness.md` | Progetto/transitorio | Consolidare invarianti in contratti e gate in issue/test; poi ritirare piano o resoconto. | +| `plans/administration-pages/03-workbench-family.md` | Progetto/transitorio | Consolidare invarianti in contratti e gate in issue/test; poi ritirare piano o resoconto. | +| `plans/administration-pages/04-embedded-acceptance.md` | Progetto/transitorio | Consolidare invarianti in contratti e gate in issue/test; poi ritirare piano o resoconto. | +| `reports/2026-09-02-psd-sensitivity-shadow.md` | Developer/benchmark | Conservare risultati aggregati come evidenza versionata, non garanzia corrente. | +| `reports/2026-09-02-sensitivity-ner-license-inventory.md` | Developer/release | Conservare provenienza licenze della release; rigenerare quando cambiano le dipendenze. | +| `reports/2026-09-03-psd-progressive-sensitivity-shadow.md` | Developer/benchmark | Conservare risultati aggregati come evidenza versionata, non garanzia corrente. | +| `reports/2026-09-12-admin-issues-28-31.md` | Progetto/transitorio | Consolidare in record di release; ritirare solo dopo trasferimento gate/rollback utili. | +| `reports/2026-09-12-context-shelf-a-implementation.md` | Progetto/transitorio | Consolidare in record di release; ritirare solo dopo trasferimento gate/rollback utili. | +| `reports/2026-09-12-ui-survey-and-graphic-revision-plan.md` | Progetto/transitorio | Consolidare in record di release; ritirare solo dopo trasferimento gate/rollback utili. | +| `reports/2026-09-12-ui-visual-review-delivery.md` | Progetto/transitorio | Consolidare in record di release; ritirare solo dopo trasferimento gate/rollback utili. | +| `reports/2026-09-12-unified-interaction-model.md` | Progetto/transitorio | Consolidare in record di release; ritirare solo dopo trasferimento gate/rollback utili. | +| `reports/2026-09-13-full-shell-implementation.md` | Progetto/transitorio | Consolidare in record di release; ritirare solo dopo trasferimento gate/rollback utili. | +| `reports/2026-09-13-header-layout-refinements.md` | Progetto/transitorio | Consolidare in record di release; ritirare solo dopo trasferimento gate/rollback utili. | +| `reports/2026-09-13-knowledge-reading.md` | Progetto/transitorio | Consolidare in record di release; ritirare solo dopo trasferimento gate/rollback utili. | +| `reports/2026-09-13-knowledge-typography.md` | Progetto/transitorio | Consolidare in record di release; ritirare solo dopo trasferimento gate/rollback utili. | +| `reports/2026-09-13-shell-simplification-review.md` | Progetto/transitorio | Consolidare in record di release; ritirare solo dopo trasferimento gate/rollback utili. | +| `reports/2026-09-13-visual-shell-integration.md` | Progetto/transitorio | Consolidare in record di release; ritirare solo dopo trasferimento gate/rollback utili. | +| `reports/2026-09-14-session-dialogs-release.md` | Operazioni/release | Conservare finché accettazione, osservazione e rollback non sono chiusi. | +| `reports/2026-09-14-session-layout-memory-fix.md` | Operazioni/release | Conservare finché accettazione, osservazione e rollback non sono chiusi. | +| `requirements.lock` | Developer/build | Conservare dipendenze e lock; esclusi dal sito. | +| `requirements.txt` | Developer/build | Conservare dipendenze e lock; esclusi dal sito. | +| `research/2026-08-23-legacy-thothai-metadata-capabilities.md` | Storico/superato | Estrarre vincoli non assorbiti da ADR/contratti, poi ritirare. | +| `research/2026-08-23-postgresql-catalog-deployment-constraints.md` | Storico/superato | Estrarre vincoli non assorbiti da ADR/contratti, poi ritirare. | +| `research/2026-08-23-thothii-metadata-publication-qdrant-seams.md` | Storico/superato | Estrarre vincoli non assorbiti da ADR/contratti, poi ritirare. | +| `research/2026-08-31-browser-erd-library-options.md` | Developer/ricerca | Verificare decisioni ancora aperte; assorbire conclusioni in issue/ADR, poi ritirare. | +| `research/2026-08-31-relationship-management-thothai-to-thothii.md` | Developer/ricerca | Verificare decisioni ancora aperte; assorbire conclusioni in issue/ADR, poi ritirare. | +| `research/2026-09-02-local-sensitive-column-classifier-libraries.md` | Developer/ricerca | Verificare decisioni ancora aperte; assorbire conclusioni in issue/ADR, poi ritirare. | +| `research/2026-09-07-antigravity-gemini-flash-value.md` | Fuori prodotto | Proposta di ritiro dal working tree; Git o raccolta personale su scelta del proprietario. | +| `research/2026-09-07-deepseek-harness-terminal.md` | Fuori prodotto | Proposta di ritiro dal working tree; Git o raccolta personale su scelta del proprietario. | +| `research/2026-09-07-omp-codex-zcode.md` | Fuori prodotto | Proposta di ritiro dal working tree; Git o raccolta personale su scelta del proprietario. | +| `research/2026-09-07-pi-config-vs-oh-my-pi.md` | Fuori prodotto | Proposta di ritiro dal working tree; Git o raccolta personale su scelta del proprietario. | +| `research/2026-09-07-zai-coding-plan-cli-quota.md` | Fuori prodotto | Proposta di ritiro dal working tree; Git o raccolta personale su scelta del proprietario. | +| `research/2026-09-07-zed-acp-omp-pi.md` | Fuori prodotto | Proposta di ritiro dal working tree; Git o raccolta personale su scelta del proprietario. | +| `research/text-to-sql-products.md` | Fuori prodotto | Proposta di ritiro dal working tree; Git o raccolta personale su scelta del proprietario. | +| `skills.md` | Pubblico | Conservare; istruzioni di prodotto/operatore. | +| `stylesheets/extra.css` | Asset pubblico | Conservare asset o esempio operativo. | +| `testing/2026-08-29-ai-catalog-description-generation-acceptance.md` | Developer/QA | Conservare criteri ripetibili e gate; separare risultati di singola release. | +| `testing/2026-08-31-metadata-privacy-description-test-plan.md` | Developer/QA | Conservare scenari; ridurre duplicazioni e distinguere test proposti da eseguiti. | +| `testing/authentication-manual-acceptance.md` | Developer/QA | Conservare criteri ripetibili e gate; separare risultati di singola release. | +| `testing/evidence-lifecycle-test-plan.md` | Developer/QA | Conservare criteri ripetibili e gate; separare risultati di singola release. | + + +## Verifica della modifica + +- Build MkDocs con `--strict` e dipendenze bloccate: superata. +- Verifica confine pubblico: 20 pagine, nessun file interno generato o indicizzato. +- Otto fixture positive/negative del confine pubblico: superate. +- Suite corrente installazione/workspace: superata. +- Due verificatori legacy: fallimenti preesistenti descritti sopra; non dichiarati verdi. +- Nessuna certificazione di nuova installazione o nuovo collaudo applicativo. + +La pubblicazione remota va verificata separatamente dalla build locale: il ramo sorgente +`main`, il ramo generato `pages` e il sito servito non sono la stessa prova. diff --git a/docs/operations/database-management.md b/docs/operations/database-management.md index 144501b2..1986c970 100644 --- a/docs/operations/database-management.md +++ b/docs/operations/database-management.md @@ -67,7 +67,7 @@ SSH uses a private key, optional key passphrase, mandatory `known_hosts`, and op TLS CA/server name. REST prefers `POST /rpc/schema_snapshot`; when it is absent, the catalog may use the same strict v1 snapshot through one read-only `POST /rpc/run_query`. An unavailable capability, malformed snapshot, or connector error applies no catalog changes. See the -[schema snapshot contract](../contracts/catalog-schema-snapshot.md). +[schema snapshot contract](https://git.tylconsulting.it/mptyl/ThothII/src/branch/main/docs/contracts/catalog-schema-snapshot.md). ## Synchronize authoritative schema metadata @@ -176,6 +176,6 @@ atomic operation. It skips empty generated descriptions, reports aggregate copie counts, and retains the generated text. Because this can replace reviewed descriptions, the interface requires explicit confirmation before applying it. -The decisions behind this surface are [ADRs 0001–0011](../adr/0001-postgres-metadata-catalog.md) +The decisions behind this surface are [ADRs 0001–0011](https://git.tylconsulting.it/mptyl/ThothII/src/branch/main/docs/adr/0001-postgres-metadata-catalog.md) and the detailed acceptance record is -[AI catalog description generation acceptance](../testing/2026-08-29-ai-catalog-description-generation-acceptance.md). +[AI catalog description generation acceptance](https://git.tylconsulting.it/mptyl/ThothII/src/branch/main/docs/testing/2026-08-29-ai-catalog-description-generation-acceptance.md). diff --git a/docs/operations/sensitivity-analysis.md b/docs/operations/sensitivity-analysis.md index ef027e04..2e140617 100644 --- a/docs/operations/sensitivity-analysis.md +++ b/docs/operations/sensitivity-analysis.md @@ -118,12 +118,12 @@ other CPU workloads. Run the first evaluation in shadow mode: read the source with its existing read-only role, do not save proposed flags, and report only aggregate counts, rule IDs, coverage, and timings. Never copy matched values into test output. Use a separately approved, labeled Italian corpus to calculate -precision and recall; raw PSD values must remain inside the authorized environment. +precision and recall; source values must remain inside the authorized environment. Inside the configured core runtime, the non-mutating command is: ```bash -npm run sensitivity:shadow -- psd-clinical +npm run sensitivity:shadow -- ``` It reads catalog metadata and source values but emits one aggregate JSON object with no database, @@ -139,12 +139,7 @@ Enabling NER by default requires all of these gates: If a gate fails, leave NER disabled. The deterministic policy remains available and produces the binary draft from its scan coverage; no content is sent to an internal or external LLM. -The first aggregate PSD shadow comparison is recorded in -[`2026-09-02-psd-sensitivity-shadow.md`](../reports/2026-09-02-psd-sensitivity-shadow.md). On the -local CPU runner, NER found additional entities but reduced total coverage under the superseded -global deadline. The v2 benchmark removed that confounder: CPU NER added 18 sensitive proposals and -increased the warm analysis time from 50.1 to 61.3 seconds. It remains opt-in until a labeled Italian -evaluation establishes that the additional findings justify their false-positive rate and cost. -The deterministic progressive PSD run is recorded in -[`2026-09-03-psd-progressive-sensitivity-shadow.md`](../reports/2026-09-03-psd-progressive-sensitivity-shadow.md): -both deterministic and CPU-NER profiles assessed all 2,275 columns with zero `unknown` decisions. +Benchmarks from a particular installation are not a guarantee for another database or machine. +NER remains opt-in until an approved evaluation establishes that additional findings justify +their false-positive rate and operational cost. Keep benchmark and release records with the +installation's technical evidence, separate from this operator procedure. diff --git a/docs/operations/workspaces.md b/docs/operations/workspaces.md index 18d7d45c..ab63fca9 100644 --- a/docs/operations/workspaces.md +++ b/docs/operations/workspaces.md @@ -73,9 +73,9 @@ only that run's safe stage, error code, and finish time; use `docker compose log corresponding service log. The contract gives exact validation, exit code, and JSON rules in -[Workspace preprocessing CLI](../contracts/workspace-preprocessing-cli.md). For Evidence source +[Workspace preprocessing CLI](https://git.tylconsulting.it/mptyl/ThothII/src/branch/main/docs/contracts/workspace-preprocessing-cli.md). For Evidence source forms and the schema-v4 descriptor contract, see -[Workspace Evidence v3](../contracts/workspace-evidence-v3.md). +[Workspace Evidence v3](https://git.tylconsulting.it/mptyl/ThothII/src/branch/main/docs/contracts/workspace-evidence-v3.md). ## Transport and revision rules diff --git a/docs/product-overview.md b/docs/product-overview.md new file mode 100644 index 00000000..546d6547 --- /dev/null +++ b/docs/product-overview.md @@ -0,0 +1,43 @@ +# What ThothII does + +ThothII helps turn a natural-language question into reviewed SQL. It is intended for +people who know the meaning of their data and need to make the assumptions behind a +query explicit. The model proposes; a human reviewer approves, corrects or rejects. + +## From a question to a reusable result + +The [eight-phase workflow](skills.md) clarifies the question, consults reusable knowledge, +selects relevant Evidence and database objects, prepares a query plan, and produces SQL +for review. Approved knowledge can be saved for later questions. + +- **Workspace:** the domain context and its Evidence configuration. +- **Database catalog:** tables, columns, relationships and reviewed descriptions. +- **Evidence:** domain sources and rules with provenance that a reviewer can inspect. +- **Memory:** reusable clarifications, SQL rules, solved questions and explained errors. +- **Session:** the saved artifacts and decisions for one question, not a permanent chat transcript. + +Resuming a session uses its saved state. Model output is a proposal, not a guarantee of +correctness: review the scope, assumptions, sources and SQL before relying on the result. + +## What an installation includes + +The Docker stack includes the web application, its runtime, a PostgreSQL metadata/Memory +catalog, Qdrant and the embedding service. A browser is the normal user interface; the +host `tht` command is used to configure and operate the installation. + +The data warehouse and generative model endpoints are configured separately. Docker +does not supply their credentials, network access or domain data. A standalone installation +is therefore self-hosted, but is not automatically offline or independent of those services. +Review data-access permissions and the model-provider configuration before using real data. + +## Choose your next step + +- Install on Mac, Windows through WSL2, or Linux: [Italian](install/standalone-manual-it.md) + or [English](install/standalone-manual-en.md) manual procedure. +- Configure [display mode and language](install/shell-and-language.md), + [authentication](install/authentication-local.md) and [models](general/pi-configuration.md). +- Prepare [workspaces](operations/workspaces.md) and [databases](operations/database-management.md). +- Start a reviewed question using the [user guide](guida-utente.md). + +The installation guides state the platform tests still to complete. Availability of a +procedure is not a certification that every target machine has been tested. diff --git a/docs/skills.md b/docs/skills.md index 5724a994..7d4b815f 100644 --- a/docs/skills.md +++ b/docs/skills.md @@ -62,8 +62,9 @@ to the reviewer. A correction can reopen the CTE plan without discarding decisio ## F8: Memory promotion -At the end of the session, the system proposes reusable clarifications. The reviewer decides which -ones to promote to the global registry, and the session is then finalized. +At the end of the session, the system proposes reusable Memory cards, including the approved +solved question. The reviewer selects and edits the additions or updates to retain in the +workspace's Memory archive, or declines them all. The session is then finalized. ## Available gates diff --git a/docs/usage/memory.md b/docs/usage/memory.md new file mode 100644 index 00000000..22e7fb93 --- /dev/null +++ b/docs/usage/memory.md @@ -0,0 +1,50 @@ +# Use and administer Memory + +Memory stores reusable knowledge for a workspace. It is separate from Evidence sources +and from the saved documents of an individual session. + +| Card family | Purpose | +| --- | --- | +| Domain clarification | Define a term or interpretation, with its scope. | +| SQL rule | Explain how to construct SQL and why. | +| Solved question | Retain an approved question and SQL as a consultative example. | +| Explained error | Record a correction and its rationale. | + +## Browse and edit + +Open **Administration → Memory management** and select the workspace. Administration +requires the appropriate permissions. You can search, filter, open a card's complete +content, edit it, manage its dependencies and links, or explicitly confirm deletion. + +Browsing and editing require the PostgreSQL archive but do not require an active session +or a working DWH or search index. Cards retain their identity and origin when edited; +there is no editorial revision history. Deleting a card also removes its links and +dependencies, not the other cards or the workspace's Evidence. + +Links connect cards in the same workspace. Record scope and physical dependencies +carefully so knowledge is not applied to unrelated databases, tables or columns. + +## Saved content and search availability + +Saving a card and updating its search index are different outcomes. **Saved, index update +incomplete** means the content is already in the archive but is not ready for recall. +Use **Pending index updates** to retry. Do not create a duplicate card to work around an +indexing failure. A restart does not discard the pending operation. + +Qdrant is a rebuildable search projection, not the authoritative archive. Rebuilding it +does not recover deleted cards from old sessions or vector payloads. Back up the +authoritative installation data before maintenance. + +After a successful physical schema synchronization, cards dependent on removed database +objects can be deleted. Workspace-wide cards and unrelated dependencies remain. Review +the synchronization result and any pending cleanup before starting another synchronization. + +## Memory in a reviewed session + +The workflow proposes relevant clarifications, rules and examples. A search result is +not approval: the reviewer decides whether it applies. At the final Memory review, +select the additions or updates worth retaining, edit their content and scope, or decline +them all. Finalizing a session does not silently approve every proposed card. + +Approved SQL is read-only at that final review. To change the solution, return to SQL +review. See the [workflow guide](../skills.md) and [user guide](../guida-utente.md). diff --git a/mkdocs.yml b/mkdocs.yml index 828696a7..a26cd32c 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -1,5 +1,5 @@ site_name: ThothII Docs -site_description: Functional, technical, and operational documentation for ThothII +site_description: Understand, install, and use ThothII site_url: https://git.tylconsulting.it/thothii-docs/ docs_dir: docs site_dir: site @@ -42,92 +42,58 @@ markdown_extensions: format: !!python/name:mermaid2.fence_mermaid - pymdownx.tabbed: alternate_style: true +# Public manual: deny by default, including search and copied assets. +# Internal developer documents remain in Git, not in the generated site. +# Keep these exceptions aligned with nav; scripts/verify-public-docs.py checks it. +exclude_docs: | + * + !/index.md + !/product-overview.md + !/skills.md + !/guida-utente.md + !/install/first-start.md + !/install/standalone-manual-it.md + !/install/standalone-manual-en.md + !/install/shell-and-language.md + !/install/authentication-local.md + !/install/authentication-oidc.md + !/install/authentik.md + !/general/pi-configuration.md + !/operations/workspaces.md + !/operations/database-management.md + !/operations/sensitivity-analysis.md + !/evidence.md + !/usage/memory.md + !/install/dwh-auth-server.md + !/install/dwh-auth-client-enrollment.md + !/install/dwh-auth-tls.md + !/stylesheets/extra.css + !/javascripts/layout-init.js + !/install/examples/thothii-installation.local.yaml + !/install/examples/thothii-installation.server.yaml + !/install/examples/workspace-bindings.env.example nav: - Home: index.md -- Install and operate: - - Install and first start: install/first-start.md - - Manual standalone installation (Italian): install/standalone-manual-it.md - - Manual standalone installation (English): install/standalone-manual-en.md - - Server upgrade from legacy release: operations/server-upgrade-gitea-workspace-v2.md - - Preprocessing-complete server handoff: operations/server-handoff-260906-preprocessing-complete.md - - Local authentication: install/authentication-local.md - - Generic OIDC: install/authentication-oidc.md - - Portal upstream authentication: install/authentication-upstream.md - - Authentik: install/authentik.md - - Workspace operations: operations/workspaces.md - - Shell and localization: operations/shell-and-localization.md - - Codex server handoff: operations/server-codex-handoff.md - - Full shell verification: reports/2026-09-13-full-shell-implementation.md - - Local sensitivity analysis: operations/sensitivity-analysis.md - - Pi model configuration: general/pi-configuration.md - - Docker installation contexts: installazione-docker-4-contesti.md -- Use ThothII: +- Understand ThothII: + - Product overview: product-overview.md + - Reviewed workflow: skills.md - User guide: guida-utente.md - - Database management: operations/database-management.md - - Evidence authoring and publication: evidence.md - - Workflow references: - - Initial disambiguation: disambiguazione-iniziale.md - - Memory management: gestione-memory.md - - Operating skills: skills.md -- Architecture: - - Overview: architecture/overview.md - - Components, modules, and flows: architecture/components.md - - Authentication and authorization: architecture/authentication.md - - Full and embedded rendering: architecture/application-shell.md -- Contracts and integration: - - Portal Shell Adapter v1: contracts/portal-shell-adapter-v1.md - - Catalog schema snapshot RPC: contracts/catalog-schema-snapshot.md - - Workspace preprocessing CLI: contracts/workspace-preprocessing-cli.md - - tht–DWH contract: contracts/tht-dwh.md - - Workspace Evidence contract: contracts/workspace-evidence-v3.md - - Editable Curated Evidence v4: contracts/curated-evidence-v4.md - - Session archive corrections: contracts/archive-repair.md - - DWH REST server: install/dwh-auth-server.md - - DWH client enrollment: install/dwh-auth-client-enrollment.md - - DWH REST TLS: install/dwh-auth-tls.md -- Decisions and acceptance: - - Shell and authentication acceptance: testing/authentication-manual-acceptance.md - - Architecture decisions: - - 0001 Metadata catalog: adr/0001-postgres-metadata-catalog.md - - 0002 Workspace database secrets: adr/0002-workspace-database-secret-references.md - - 0003 Installation-local bindings: adr/0003-installation-local-database-bindings.md - - 0004 Fastify/Kysely catalog: adr/0004-fastify-kysely-metadata-catalog.md - - 0005 Table deletion during synchronization: adr/0005-hard-delete-catalog-tables-during-synchronization.md - - 0006 Physical and logical relationships: adr/0006-separate-physical-and-logical-relationships.md - - 0007 Authoritative schema synchronization: adr/0007-durable-authoritative-schema-synchronization.md - - 0008 Manual catalog cleanup: adr/0008-allow-manual-catalog-metadata-cleanup.md - - 0009 Sequential description generation: adr/0009-use-one-sequential-description-generation-run.md - - 0010 Bounded source samples: adr/0010-allow-bounded-real-source-samples-for-description-generation.md - - 0011 Sensitive data flag: adr/0011-gate-source-samples-with-a-sensitive-data-flag.md - - 0012 Effective relationship authority: adr/0012-use-the-catalog-as-the-logical-relationship-authority.md - - 0013 Installation model catalog: adr/0013-use-one-installation-model-catalog-with-runtime-projections.md - - 0014 Local sensitive-column assessment: adr/0014-assess-sensitive-columns-locally-from-source-content.md - - 0015 Progressive sensitive-column sampling: adr/0015-use-progressive-sampling-for-sensitive-columns.md - - 0016 PostgreSQL metadata for all core consumers: adr/0016-use-postgres-metadata-for-all-core-consumers.md - - 0017 Separate reference vectors from runtime memory: adr/0017-separate-reference-vectors-from-runtime-memory.md - - 0018 PostgreSQL Memory authority and Qdrant retrieval: adr/0018-use-postgres-for-memory-and-qdrant-for-retrieval.md - - 0019 Evidence editing and manual consolidation: adr/0019-author-evidence-in-app-with-automatic-activation.md - - 0021 Shell modes and replaceable portal adapter: adr/0021-separate-shell-modes-and-replaceable-portal-adapter.md - - 0022 UI locale and session interaction language: adr/0022-separate-ui-locale-from-session-interaction-language.md - - AI catalog description acceptance: testing/2026-08-29-ai-catalog-description-generation-acceptance.md - - Sensitivity NER license inventory: reports/2026-09-02-sensitivity-ner-license-inventory.md - - PSD sensitivity shadow evaluation: reports/2026-09-02-psd-sensitivity-shadow.md - - PSD progressive sensitivity shadow evaluation: reports/2026-09-03-psd-progressive-sensitivity-shadow.md - - Design records: - - Security hardening PRD (draft): plans/2026-09-08-security-hardening-prd.md - - Security hardening session prompt: plans/2026-09-08-security-hardening-resume-prompt.md - - Memory and Evidence administration: plans/2026-09-08-memory-evidence-administration.md - - Memory management project: plans/2026-09-08-memory-management.md - - Memory M1 specification: plans/2026-09-08-memory-m1-spec.md - - Memory M2 implementation and validation: plans/2026-09-09-memory-m2-validation.md - - Evidence management project: plans/2026-09-08-evidence-management.md - - Evidence E1 validation: plans/2026-09-09-evidence-e1-validation.md - - Evidence E2 validation: plans/2026-09-09-evidence-e2-validation.md - - Evidence E3 validation: plans/2026-09-09-evidence-e3-validation.md - - Archive repair X1 validation: plans/2026-09-09-archive-repair-x1-validation.md - - Memory and Evidence simplification review: plans/2026-09-08-memory-evidence-simplification-review.md - - Metadata catalog design: plans/2026-08-26-metadata-catalog-from-thothai.md - - Description generation design: plans/2026-08-28-ai-catalog-description-generation.md - - Description generation specification: plans/2026-08-28-ai-catalog-description-generation-spec.md - - Full shell and portal integration: plans/2026-09-13-full-shell-and-portal-integration.md - - Full shell specification: plans/2026-09-13-full-shell-spec.md +- Install: + - Start here: install/first-start.md + - Mac, Windows, Linux — Italiano: install/standalone-manual-it.md + - Mac, Windows, Linux — English: install/standalone-manual-en.md + - Display mode and language: install/shell-and-language.md + - Local authentication: install/authentication-local.md + - OIDC authentication: install/authentication-oidc.md + - Authentik: install/authentik.md + - Model configuration: general/pi-configuration.md +- Administer: + - Workspaces: operations/workspaces.md + - Databases and descriptions: operations/database-management.md + - Sensitive columns: operations/sensitivity-analysis.md + - Evidence: evidence.md + - Memory: usage/memory.md +- Connect a REST DWH: + - Server: install/dwh-auth-server.md + - Client enrollment: install/dwh-auth-client-enrollment.md + - TLS: install/dwh-auth-tls.md diff --git a/scripts/build-docs.sh b/scripts/build-docs.sh index d6050375..64b8b4c3 100755 --- a/scripts/build-docs.sh +++ b/scripts/build-docs.sh @@ -3,9 +3,16 @@ set -euo pipefail root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P) +uv run \ + --quiet \ + --isolated \ + --no-env-file \ + --with-requirements "$root/docs/requirements.lock" \ + mkdocs build --strict --config-file "$root/mkdocs.yml" + exec uv run \ --quiet \ --isolated \ --no-env-file \ --with-requirements "$root/docs/requirements.lock" \ - mkdocs build --strict --config-file "$root/mkdocs.yml" + python "$root/scripts/verify-public-docs.py" --root "$root" diff --git a/scripts/test-verify-public-docs.py b/scripts/test-verify-public-docs.py new file mode 100644 index 00000000..4abfb179 --- /dev/null +++ b/scripts/test-verify-public-docs.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python3 +"""Negative fixtures for public documentation publication checks.""" + +import importlib.util +import json +from pathlib import Path +import tempfile +import unittest + + +spec = importlib.util.spec_from_file_location( + "public_docs", Path(__file__).with_name("verify-public-docs.py") +) +verifier = importlib.util.module_from_spec(spec) +spec.loader.exec_module(verifier) + + +class PublicationBoundaryTest(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory(prefix="thoth-public-docs-") + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + self.site = self.root / "site" + self.config = "nav:\n- Home: index.md\nexclude_docs: |\n *\n" + for source in ["index.md", *sorted(verifier.PUBLIC_ASSETS)]: + self.config += f" !/{source}\n" + self.write(self.root / "docs" / source, "# Public") + self.write(self.site / verifier.output_path(source), "public") + self.write(self.root / "mkdocs.yml", self.config) + self.search = {"docs": [{"location": "", "text": "public"}]} + self.write_search() + self.write(self.root / "docs/plans/private.md", "# Internal") + self.write(self.root / "docs/reports/screenshot.png", "internal asset") + + @staticmethod + def write(path, content): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content) + + def write_search(self): + self.write(self.site / "search/search_index.json", json.dumps(self.search)) + + def test_public_only(self): + self.assertEqual(verifier.check(self.root, self.site), 1) + + def test_internal_nav(self): + self.write(self.root / "mkdocs.yml", self.config.replace( + "- Home: index.md", "- Home: index.md\n- Internal: plans/private.md" + )) + with self.assertRaisesRegex(ValueError, "internal page"): + verifier.check(self.root, self.site) + + def test_unlisted_exception(self): + self.write(self.root / "mkdocs.yml", self.config + " !/plans/private.md\n") + with self.assertRaisesRegex(ValueError, "exceptions must match"): + verifier.check(self.root, self.site) + + def test_missing_deny_default(self): + self.write(self.root / "mkdocs.yml", self.config.replace(" *\n", "")) + with self.assertRaisesRegex(ValueError, "deny-by-default"): + verifier.check(self.root, self.site) + + def test_leaked_internal_page(self): + self.write(self.site / "plans/private/index.html", "internal") + with self.assertRaisesRegex(ValueError, "internal file leaked"): + verifier.check(self.root, self.site) + + def test_leaked_internal_asset(self): + self.write(self.site / "reports/screenshot.png", "internal") + with self.assertRaisesRegex(ValueError, "internal file leaked"): + verifier.check(self.root, self.site) + + def test_internal_search_entry(self): + self.search["docs"].append({"location": "plans/private/#internal", "text": "internal"}) + self.write_search() + with self.assertRaisesRegex(ValueError, "non-public page in search"): + verifier.check(self.root, self.site) + + def test_missing_public_page(self): + (self.site / "index.html").unlink() + with self.assertRaisesRegex(ValueError, "missing generated public"): + verifier.check(self.root, self.site) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/test-verify-workspace-install-docs.sh b/scripts/test-verify-workspace-install-docs.sh index d4be8964..a9807b62 100755 --- a/scripts/test-verify-workspace-install-docs.sh +++ b/scripts/test-verify-workspace-install-docs.sh @@ -34,9 +34,7 @@ mkdir -p "$fixture/docs" "$fixture/scripts" cp -R "$root/docs/install" "$fixture/docs/install" cp -R "$root/docs/operations" "$fixture/docs/operations" cp "$root/docs/guida-utente.md" "$fixture/docs/guida-utente.md" -mkdir -p "$fixture/docs/architecture" "$fixture/docs/contracts" -cp "$root/docs/architecture/overview.md" "$fixture/docs/architecture/overview.md" -cp "$root/docs/contracts/catalog-schema-snapshot.md" "$fixture/docs/contracts/catalog-schema-snapshot.md" +cp "$root/docs/product-overview.md" "$fixture/docs/product-overview.md" cp "$root/mkdocs.yml" "$fixture/mkdocs.yml" cp "$root/compose.yaml" "$fixture/compose.yaml" cp "$root/scripts/run-stack.sh" "$fixture/scripts/run-stack.sh" diff --git a/scripts/verify-public-docs.py b/scripts/verify-public-docs.py new file mode 100644 index 00000000..1f6830bb --- /dev/null +++ b/scripts/verify-public-docs.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python3 +"""Verify the public manual boundary, including generated pages and search.""" + +import argparse +import json +from pathlib import Path +from urllib.parse import unquote, urlsplit + +import yaml + + +PUBLIC_ASSETS = { + "stylesheets/extra.css", + "javascripts/layout-init.js", + "install/examples/thothii-installation.local.yaml", + "install/examples/thothii-installation.server.yaml", + "install/examples/workspace-bindings.env.example", +} +INTERNAL_DIRS = { + "adr", "agents", "architecture", "contracts", "maintenance", "plans", + "reports", "research", "testing", +} +INTERNAL_PAGES = { + "disambiguazione-iniziale.md", "gestione-memory.md", "installazione-docker-4-contesti.md", + "install/authentication-upstream.md", "operations/docker-refresh.md", + "operations/shell-and-localization.md", +} + + +def nav_pages(value): + if isinstance(value, str): + yield value + elif isinstance(value, list): + for item in value: + yield from nav_pages(item) + elif isinstance(value, dict): + for item in value.values(): + yield from nav_pages(item) + + +def output_path(source): + path = Path(source) + if path.suffix != ".md": + return path + return path.with_suffix("") / "index.html" if path.name != "index.md" else path.with_suffix(".html") + + +def check(root, site): + # BaseLoader reads configuration without executing custom Python YAML tags. + config = yaml.load((root / "mkdocs.yml").read_text(), Loader=yaml.BaseLoader) + pages = list(nav_pages(config["nav"])) + if len(pages) != len(set(pages)): + raise ValueError("duplicate public navigation page") + for page in pages: + path = Path(page) + if path.is_absolute() or ".." in path.parts or path.suffix != ".md": + raise ValueError(f"invalid public page: {page}") + if (path.parts[0] in INTERNAL_DIRS or page in INTERNAL_PAGES + or page.startswith("operations/server-")): + raise ValueError(f"internal page in public navigation: {page}") + if not (root / "docs" / page).is_file(): + raise ValueError(f"missing public source: {page}") + + rules = config.get("exclude_docs", "").splitlines() + if not rules or rules[0] != "*" or any(not r.startswith("!/") for r in rules[1:]): + raise ValueError("public docs must use deny-by-default exclusions and explicit exceptions") + published = [r[2:] for r in rules[1:]] + if len(published) != len(set(published)) or set(published) != set(pages) | PUBLIC_ASSETS: + raise ValueError("publication exceptions must match nav pages and approved assets exactly") + + for source in published: + if not (root / "docs" / source).is_file(): + raise ValueError(f"missing public source: {source}") + if not (site / output_path(source)).is_file(): + raise ValueError(f"missing generated public file: {source}") + for source in (root / "docs").rglob("*"): + if not source.is_file(): + continue + relative = source.relative_to(root / "docs").as_posix() + if relative not in published and (site / output_path(relative)).exists(): + raise ValueError(f"internal file leaked into site: {relative}") + + expected_urls = { + "" if page == "index.md" else str(output_path(page).parent).replace("\\", "/") + "/" + for page in pages + } + search = json.loads((site / "search/search_index.json").read_text()) + indexed = set() + for entry in search["docs"]: + location = unquote(urlsplit(entry["location"]).path) + if location not in expected_urls: + raise ValueError(f"non-public page in search index: {location}") + indexed.add(location) + if indexed != expected_urls: + raise ValueError("search index does not cover exactly the public pages") + return len(pages) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--root", type=Path, default=Path(__file__).resolve().parents[1]) + parser.add_argument("--site-dir", type=Path) + args = parser.parse_args() + root = args.root.resolve() + try: + count = check(root, args.site_dir or root / "site") + except (ValueError, KeyError, OSError, yaml.YAMLError) as error: + raise SystemExit(f"public docs verification failed: {error}") from error + print(f"Public documentation boundary passed: {count} pages; internal files and search excluded") diff --git a/scripts/verify-workspace-install-docs.sh b/scripts/verify-workspace-install-docs.sh index 601a38ce..5bdc19f0 100755 --- a/scripts/verify-workspace-install-docs.sh +++ b/scripts/verify-workspace-install-docs.sh @@ -64,8 +64,8 @@ verify_navigation() { operations/workspaces.md \ operations/database-management.md \ guida-utente.md \ - architecture/overview.md \ - contracts/catalog-schema-snapshot.md; do + product-overview.md \ + install/shell-and-language.md; do require_file "docs/$path" require_text mkdocs.yml "$path" done @@ -100,7 +100,7 @@ verify_install_and_workspace_guides() { for text in \ 'tht setup --profile local' \ - './scripts/run-stack.sh' \ + '--configure-only' \ 'catalog-migrate' \ 'tht --installation /absolute/path/thothii-installation.yaml doctor --json'; do require_text "$install" "$text"