feat: complete evidence restructuring worktree

This commit is contained in:
Codex
2026-08-26 11:39:02 +02:00
parent a54d4769dd
commit 38f02cfd08
56 changed files with 1981 additions and 1801 deletions
+4
View File
@@ -25,6 +25,10 @@ review gates and keeps the live transcript in memory. See
The evidence restructuring and PSD migration completed real acceptance on 2026-08-25.
- The curated PSD revision contains 35 approved Evidence units and 60 review items.
- The PSD authoring clone currently has all 35 units migrated locally to Curated unit schema v2:
short YAML metadata plus a typed, human-readable Markdown body. The changes remain pending a
curator commit/publication. `tht evidence migrate <workspace-root>` performs the deterministic
v1-to-v2 rewrite without model calls.
- The accepted snapshot is
`psd-clinical-675990d90eae51da6f2bd51b1ae2609f245772ef-snapshot`.
- The active generation is `gen:f968b3bd7a553dbfef3cf47093698f2bc7f95f11`.
+30
View File
@@ -56,6 +56,34 @@ export function validateDeclarativePiConfig(raw: string): void {
assertDeclarativePiConfig(parsePiConfigJson(raw));
}
/** Return the selected provider's declarative apiKey value without knowing provider IDs in code. */
export function configuredPiProviderApiKey(
raw: string | undefined,
provider: string | undefined,
): string | undefined {
if (raw === undefined || provider === undefined) return undefined;
const parsed = parsePiConfigJson(raw);
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
throw new PiManagedConfigError();
}
const providers = (parsed as { providers?: unknown }).providers;
if (!providers || typeof providers !== "object" || Array.isArray(providers)) {
throw new PiManagedConfigError();
}
const entry = Object.entries(providers as Record<string, unknown>)
.find(([id]) => id.trim().toLowerCase() === provider.trim().toLowerCase());
if (!entry) return undefined;
const config = entry[1];
if (!config || typeof config !== "object" || Array.isArray(config)) {
throw new PiManagedConfigError();
}
assertDeclarativePiConfig(config);
const apiKey = (config as { apiKey?: unknown }).apiKey;
if (apiKey === undefined) return undefined;
if (typeof apiKey !== "string" || apiKey.length === 0) throw new PiManagedConfigError();
return apiKey;
}
function configuredPiAgentDir(): string {
return resolve(process.env.PI_CODING_AGENT_DIR ?? join(homedir(), ".pi", "agent"));
}
@@ -102,6 +130,7 @@ export function readConfiguredPiAgentFile(
export interface PiRuntimeAgentSnapshot {
agentDir: string;
sessionDir: string;
models?: string;
cleanup: () => void;
}
@@ -152,6 +181,7 @@ export function createPiRuntimeAgentSnapshot(): PiRuntimeAgentSnapshot {
return {
agentDir: snapshotDir,
sessionDir: process.env.PI_CODING_AGENT_SESSION_DIR || join(sourceAgentDir, "sessions"),
models,
cleanup: () => {
if (cleaned) return;
cleaned = true;
+6
View File
@@ -9,8 +9,10 @@ import {
} from "../settings/settings-store.js";
import type { PiModel } from "./list-models.js";
import {
configuredPiProviderApiKey,
PI_MANAGED_CONFIG_ERROR_MESSAGE,
isPiManagedConfigError,
readConfiguredPiAgentFile,
} from "./managed-config.js";
import { createPiProviderSmoke, type PiProviderSmoke } from "./provider-smoke.js";
import { loadPiAuthProviders } from "./auth-providers.js";
@@ -119,6 +121,10 @@ export function createPiManagement(config: AppConfig, deps: PiManagementDeps): P
authProviders: loadPiAuthProviders(),
resolveCredentialValue: () => secretValue(config, "THT_MODEL_API_KEY"),
credentialFile: config.modelApiKeyFile,
configuredApiKey: configuredPiProviderApiKey(
readConfiguredPiAgentFile("models.json", true),
provider,
),
});
} catch {
return "missing";
+5 -1
View File
@@ -7,7 +7,10 @@ import { buildPiChildEnv, canonicalPiProvider } from "./provider-credentials.js"
import { loadPiAuthProviders } from "./auth-providers.js";
import { secretValue } from "../config/secret-bundle.js";
import { clearPrincipalEnvironment, principalEnvironment, type PrincipalContext } from "../auth/principal.js";
import { createPiRuntimeAgentSnapshot } from "./managed-config.js";
import {
configuredPiProviderApiKey,
createPiRuntimeAgentSnapshot,
} from "./managed-config.js";
export interface SessionRuntime {
rpc: RpcClient;
@@ -81,6 +84,7 @@ export class PiProcessManager {
authProviders: this.loadAuthProviders(agent.agentDir),
credentialValue: secretValue(this.cfg, "THT_MODEL_API_KEY"),
credentialFile: this.cfg.modelApiKeyFile,
configuredApiKey: configuredPiProviderApiKey(agent.models, provider),
additions: { THT_SESSION: sessionId, THT_AUTHOR: author },
});
env.PI_CODING_AGENT_DIR = agent.agentDir;
+23 -7
View File
@@ -42,9 +42,14 @@ const PROVIDER_KEY_ENV: Readonly<Record<string, string>> = {
const COMPOUND_PROVIDERS = new Set([
"amazon-bedrock", "azure-openai-responses", "cloudflare-ai-gateway", "cloudflare-workers-ai",
]);
const LOCAL_PROVIDERS = new Set([
"ollama", "lmstudio", "local", "aritmolab", "local-qwen", "faux",
]);
function configuredCredentialEnv(apiKey: string | undefined): string | null | undefined {
if (apiKey === undefined) return undefined;
if (!apiKey.startsWith("$")) return null;
const matched = /^\$(?:\{([A-Z][A-Z0-9_]*(?:API_KEY|TOKEN))\}|([A-Z][A-Z0-9_]*(?:API_KEY|TOKEN)))$/.exec(apiKey);
if (!matched) throw new Error("model provider credential is unavailable");
return matched[1] ?? matched[2];
}
export function canonicalPiProvider(provider: string | undefined): string | undefined {
const value = provider?.trim().toLowerCase();
@@ -106,6 +111,8 @@ export function buildPiChildEnv(opts: {
credentialFile?: string;
additions?: NodeJS.ProcessEnv;
credentialValue?: string;
/** Exact apiKey declaration from the selected provider in models.json. */
configuredApiKey?: string;
fsOps?: CredentialFsOps;
/**
* Providers pi can authenticate from its own auth store. For these, the single
@@ -147,17 +154,22 @@ export function buildPiChildEnv(opts: {
delete env.THT_SSL_CA_FILE;
for (const name of PI_0803_CREDENTIAL_ENV_NAMES) delete env[name];
const provider = canonicalPiProvider(opts.provider);
const configuredEnv = configuredCredentialEnv(opts.configuredApiKey);
if (configuredEnv) delete env[configuredEnv];
if (provider && COMPOUND_PROVIDERS.has(provider)) {
throw new Error(
"compound credential bundles are unsupported by THT_MODEL_API_KEY_FILE; "
+ "dedicated provider configuration is required",
);
}
if (provider && !LOCAL_PROVIDERS.has(provider)) {
if (provider) {
// pi self-authenticates this provider from its own auth store; injecting the
// single managed key here would force one provider's key onto another.
if (opts.authProviders?.has(provider)) return env;
const envName = PROVIDER_KEY_ENV[provider];
// A literal apiKey is entirely owned by models.json (commonly a non-secret
// placeholder for a local OpenAI-compatible endpoint) and needs no managed key.
if (configuredEnv === null) return env;
const envName = configuredEnv ?? PROVIDER_KEY_ENV[provider];
if (!envName || (!opts.credentialFile && opts.credentialValue === undefined)) {
throw new Error("model provider credential is unavailable");
}
@@ -169,7 +181,7 @@ export function buildPiChildEnv(opts: {
}
else if (opts.credentialFile) env[envName] = readCredential(opts.credentialFile, opts.fsOps ?? realFs);
else throw new Error("model provider credential is unavailable");
} else if (opts.credentialFile && !provider) {
} else if (opts.credentialFile) {
throw new Error("model provider credential is unavailable");
}
return env;
@@ -181,11 +193,14 @@ export function piProviderCredentialStatus(opts: {
credentialFile?: string;
resolveCredentialValue?: () => string | undefined;
authProviders?: ReadonlySet<string>;
configuredApiKey?: string;
fsOps?: CredentialFsOps;
}): PiCredentialStatus {
const provider = canonicalPiProvider(opts.provider);
if (!provider || LOCAL_PROVIDERS.has(provider)) return "missing";
if (!provider) return "missing";
if (opts.authProviders?.has(provider)) return "present";
const configuredEnv = configuredCredentialEnv(opts.configuredApiKey);
if (configuredEnv === null) return "missing";
try {
buildPiChildEnv({
ambient: {},
@@ -193,6 +208,7 @@ export function piProviderCredentialStatus(opts: {
credentialFile: opts.credentialFile,
credentialValue: opts.resolveCredentialValue?.(),
authProviders: opts.authProviders,
configuredApiKey: opts.configuredApiKey,
fsOps: opts.fsOps,
});
return "present";
+5 -3
View File
@@ -11,6 +11,7 @@ import { buildPiChildEnv, canonicalPiProvider } from "./provider-credentials.js"
import type { PiReasoning } from "./management.js";
import {
PiManagedConfigError,
configuredPiProviderApiKey,
isPiManagedConfigError,
parsePiConfigJson,
readConfiguredPiAgentFile,
@@ -65,11 +66,15 @@ export function createPiProviderSmoke(
const canonicalProvider = canonicalPiProvider(provider);
if (!canonicalProvider || timeoutMs <= 0) throw providerFailure();
const configuredAuthProviders = authProviders();
const configuredModels = options.readModelsStore
? options.readModelsStore()
: readConfiguredPiAgentFile("models.json", true);
const env = buildPiChildEnv({
provider: canonicalProvider,
authProviders: configuredAuthProviders,
credentialValue: secretValue(config, "THT_MODEL_API_KEY"),
credentialFile: config.modelApiKeyFile,
configuredApiKey: configuredPiProviderApiKey(configuredModels, canonicalProvider),
});
clearPrincipalEnvironment(env);
delete env.THT_DATA_ROOT;
@@ -90,9 +95,6 @@ export function createPiProviderSmoke(
);
writeDeclarativeAgentConfig(join(isolatedAgentDir, "auth.json"), authStore);
}
const configuredModels = options.readModelsStore
? options.readModelsStore()
: readConfiguredPiAgentFile("models.json", true);
if (configuredModels !== undefined) {
const modelsStore = selectedProviderModelsStore(
configuredModels,
+28
View File
@@ -0,0 +1,28 @@
import { expect, test } from "vitest";
import {
PI_MANAGED_CONFIG_ERROR_MESSAGE,
configuredPiProviderApiKey,
} from "../src/pi/managed-config.js";
test("provider credential declarations are selected from models.json by provider ID", () => {
const raw = JSON.stringify({
providers: {
hosted: { apiKey: "$HOSTED_API_KEY", models: [{ id: "one" }] },
local: { apiKey: "local", models: [{ id: "two" }] },
},
});
expect(configuredPiProviderApiKey(raw, "HOSTED")).toBe("$HOSTED_API_KEY");
expect(configuredPiProviderApiKey(raw, "local")).toBe("local");
expect(configuredPiProviderApiKey(raw, "missing")).toBeUndefined();
});
test.each([
JSON.stringify({ providers: [] }),
JSON.stringify({ providers: { local: "invalid" } }),
JSON.stringify({ providers: { local: { apiKey: 42 } } }),
JSON.stringify({ providers: { local: { apiKey: "!must-not-run" } } }),
])("invalid declarative provider credential configuration fails closed", (raw) => {
expect(() => configuredPiProviderApiKey(raw, "local"))
.toThrow(PI_MANAGED_CONFIG_ERROR_MESSAGE);
});
+17 -3
View File
@@ -22,7 +22,7 @@ const SAFE_AUTH = '{"deepseek":{"type":"api_key","key":"safe-token"}}\n';
const SAFE_MODELS = [
"{",
' "providers": {',
' "local-qwen": {"baseUrl":"http://model.invalid/v1","models":[{"id":"qwen"}]}',
' "local-qwen": {"baseUrl":"http://model.invalid/v1","apiKey":"local","models":[{"id":"qwen"}]}',
" }",
"}",
"",
@@ -670,9 +670,22 @@ test.each([["OpenAI", "openai"], ["gemini", "google"]])(
},
);
test.each(["ollama", "local-qwen"])(
"local provider %s spawns without a model key and scrubs ambient credentials",
test.each(["installation-local", "private-compatible"])(
"provider %s configured with a literal apiKey spawns without a managed key",
async (provider) => {
const root = mkdtempSync(path.join(tmpdir(), "tht-local-provider-"));
const agentDir = path.join(root, "agent");
mkdirSync(agentDir, { mode: 0o700 });
writeFileSync(path.join(agentDir, "models.json"), JSON.stringify({
providers: {
[provider]: {
baseUrl: "http://model.invalid/v1",
apiKey: "local",
models: [{ id: "model" }],
},
},
}), { mode: 0o600 });
vi.stubEnv("PI_CODING_AGENT_DIR", agentDir);
vi.stubEnv("PI_PROVIDER_API_KEY", "ambient-secret");
vi.stubEnv("THT_MODEL_API_KEY_FILE", "/ambient/secret-path");
vi.stubEnv("OPENAI_API_KEY", "unselected-provider-secret");
@@ -690,6 +703,7 @@ test.each(["ollama", "local-qwen"])(
} finally {
mgr.teardown(`local-session-${provider}`);
vi.unstubAllEnvs();
rmSync(root, { recursive: true, force: true });
}
},
);
+27 -4
View File
@@ -117,20 +117,41 @@ test("single-key providers scrub ambient compound companions before injecting th
expect(env).not.toHaveProperty("CLOUDFLARE_GATEWAY_ID");
});
test("local-qwen is an explicit local provider and needs no generic key", () => {
test("a provider with a literal apiKey in models.json needs no code-level provider exception", () => {
const env = buildPiChildEnv({
ambient: {
PI_PROVIDER_API_KEY: "must-not-leak",
OPENAI_API_KEY: "must-not-leak",
THT_MODEL_API_KEY_FILE: "/must/not/leak",
},
provider: "local-qwen",
provider: "installation-local",
configuredApiKey: "local",
});
expect(env).not.toHaveProperty("PI_PROVIDER_API_KEY");
expect(env).not.toHaveProperty("OPENAI_API_KEY");
expect(env).not.toHaveProperty("THT_MODEL_API_KEY_FILE");
});
test.each(["$PRIVATE_PROVIDER_API_KEY", "${PRIVATE_PROVIDER_API_KEY}"])(
"a custom provider credential target is derived from models.json: %s",
(configuredApiKey) => {
const env = buildPiChildEnv({
ambient: { PRIVATE_PROVIDER_API_KEY: "stale" },
provider: "private-provider",
configuredApiKey,
credentialValue: "selected-secret",
});
expect(env.PRIVATE_PROVIDER_API_KEY).toBe("selected-secret");
},
);
test("a custom provider cannot redirect a managed credential into a process-control variable", () => {
expect(() => buildPiChildEnv({
ambient: {}, provider: "private-provider", configuredApiKey: "$PATH",
credentialValue: "selected-secret",
})).toThrow("model provider credential is unavailable");
});
test("credential status reports only present or missing without treating local providers as credentialed", () => {
expect(piProviderCredentialStatus({
provider: "deepseek",
@@ -139,7 +160,8 @@ test("credential status reports only present or missing without treating local p
})).toBe("present");
expect(piProviderCredentialStatus({ provider: "deepseek" })).toBe("missing");
expect(piProviderCredentialStatus({
provider: "local-qwen",
provider: "installation-local",
configuredApiKey: "local",
resolveCredentialValue: () => "must-not-be-returned",
})).toBe("missing");
});
@@ -159,7 +181,8 @@ test("credential status never resolves the generic secret for auth-store or loca
resolveCredentialValue: unreadableSecret,
})).toBe("present");
expect(piProviderCredentialStatus({
provider: "local-qwen",
provider: "installation-local",
configuredApiKey: "local",
resolveCredentialValue: unreadableSecret,
})).toBe("missing");
expect(secretReads).toBe(0);
+28
View File
@@ -13,6 +13,34 @@
"maxTokens": 131072
}
]
},
"local-qwen": {
"name": "Local Qwen",
"baseUrl": "https://ml-aritmolab.policlinicosandonato.it/v1",
"api": "openai-completions",
"apiKey": "local",
"models": [
{
"id": "qwen3.6-35b-a3b",
"name": "Qwen3.6 35B A3B",
"reasoning": false,
"input": ["text"],
"cost": {
"input": 0,
"output": 0,
"cacheRead": 0,
"cacheWrite": 0
},
"contextWindow": 131072,
"maxTokens": 16384,
"compat": {
"supportsDeveloperRole": false,
"supportsReasoningEffort": false,
"supportsStore": false,
"maxTokensField": "max_tokens"
}
}
]
}
}
}
+1 -1
View File
@@ -4,6 +4,6 @@
"zai/glm-5.3",
"deepseek/deepseek-v4-flash",
"deepseek/deepseek-v4-pro",
"aritmolab/qwen3.6-35b-a3b"
"local-qwen/qwen3.6-35b-a3b"
]
}
+30 -30
View File
@@ -1,10 +1,10 @@
# Componenti, moduli e flussi
# Components, modules, and flows
Questa pagina completa la [panoramica dell'architettura](overview.md) con la struttura dei moduli e i flussi che attraversano ThothII. I diagrammi descrivono il codice corrente, non un'architettura futura.
This page complements the [architecture overview](overview.md) with the module structure and flows through ThothII. The diagrams describe the current code, not a future architecture.
## Moduli e dipendenze
## Modules and dependencies
Il frontend comunica con il backend tramite REST e SSE. Il backend non possiede la persistenza delle sessioni: avvia Pi, invoca la CLI `tht` e inoltra gli eventi. L'harness contiene il workflow, la CLI Python e gli adattatori verso DWH e vector store.
The frontend communicates with the backend through REST and SSE. The backend does not own session persistence: it starts Pi, invokes the `tht` CLI, and forwards events. The harness contains the workflow, the Python CLI, and adapters for the DWH and vector store.
```mermaid
flowchart LR
@@ -14,30 +14,30 @@ flowchart LR
PI --> EXT["harness/.pi/extensions/\ntht-gate.js"]
EXT --> SKILL["harness/.pi/skills/\ntht-sessione"]
EXT --> THT
THT --> FS["Sessioni e artefatti\nworkspace repository"]
THT --> FS["Sessions and artifacts\nworkspace repository"]
THT --> DWH["DWH\nread-only"]
THT --> VDB["Qdrant / vector store"]
BE --> CFG["settings.json\nworkspace registry"]
FE -.->|renderizza widget| EXT
FE -.->|renders widgets| EXT
```
Dipendenze principali:
| Modulo | Dipende da | Responsabilità |
| Module | Depends on | Responsibility |
| --- | --- | --- |
| `frontend/` | API REST e SSE del backend | UI, widget di gate e transcript in memoria |
| `backend/src/` | Pi, `tht`, configurazione e workspace registry | Trasporto, lifecycle delle sessioni e API |
| `harness/.pi/` | Pi e `tht phase` | Orchestrazione del workflow e gate human-in-the-loop |
| `harness/tht/` | filesystem, DWH e vector store | Persistenza, CLI, evidence, schema e preprocessing |
| workspace repository | `source/`, `curated/`, manifest e artefatti | Sorgente versionata delle evidence e output di sessione |
| `frontend/` | Backend REST and SSE APIs | UI, gate widgets, and in-memory transcript |
| `backend/src/` | Pi, `tht`, configuration, and workspace registry | Transport, session lifecycle, and APIs |
| `harness/.pi/` | Pi and `tht phase` | Workflow orchestration and human-in-the-loop gates |
| `harness/tht/` | Filesystem, DWH, and vector store | Persistence, CLI, Evidence, schema, and preprocessing |
| workspace repository | `source/`, `curated/`, manifest, and artifacts | Versioned Evidence source and session output |
## Sequenza di una sessione
## Session sequence
Il percorso principale parte da una domanda dell'utente e termina con un evento SSE. Le decisioni del revisore rientrano nello stesso canale e vengono persistite dall'harness.
The main path starts with a user question and ends with an SSE event. Reviewer decisions use the same channel and are persisted by the harness.
```mermaid
sequenceDiagram
actor U as Utente o revisore
actor U as User or reviewer
participant FE as Frontend
participant BE as Backend
participant PI as Pi RPC
@@ -45,24 +45,24 @@ sequenceDiagram
participant WS as Workspace
participant DWH as DWH
U->>FE: Invia domanda o decisione di gate
U->>FE: Send question or gate decision
FE->>BE: POST session / risposta widget
BE->>PI: RPC input o prompt di resume
PI->>THT: phase/session/evidence commands
THT->>WS: Legge e scrive artefatti di fase
THT->>DWH: Introspezione o query read-only
DWH-->>THT: Schema, risultati o diagnostica
THT-->>PI: JSON e stato della fase
PI-->>BE: Eventi RPC e widget descriptor
THT->>WS: Read and write phase artifacts
THT->>DWH: Introspection or read-only query
DWH-->>THT: Schema, results, or diagnostics
THT-->>PI: JSON and phase state
PI-->>BE: RPC events and widget descriptor
BE-->>FE: SSE text_delta, info, ui_request
FE-->>U: Testo, artefatto o richiesta di revisione
FE-->>U: Text, artifact, or review request
```
Il backend usa `ThtRunner` per i subprocess della CLI, `PiProcessManager` per un processo Pi per sessione, `SessionBridge` per adattare gli eventi RPC e `SseHub` per distribuirli ai client.
The backend uses `ThtRunner` for CLI subprocesses, `PiProcessManager` for one Pi process per session, `SessionBridge` to adapt RPC events, and `SseHub` to distribute them to clients.
## Classi principali del backend
## Main backend classes
Il diagramma mostra le classi che compongono il ponte tra browser, Pi e `tht`. Le route Fastify ricevono le richieste e delegano a questi servizi.
The diagram shows the classes that form the bridge between the browser, Pi, and `tht`. Fastify routes receive requests and delegate to these services.
```mermaid
classDiagram
@@ -114,9 +114,9 @@ classDiagram
SessionBridge --> SseHub
```
## Moduli Python della CLI `tht`
## Python modules in the `tht` CLI
La CLI è composta da comandi Typer e da moduli di dominio. `cli/` traduce gli argomenti in operazioni; `evidence/`, `session/`, `db/`, `adapters/` e gli altri package contengono la logica applicativa.
The CLI consists of Typer commands and domain modules. `cli/` turns arguments into operations; `evidence/`, `session/`, `db/`, `adapters/`, and the other packages contain the application logic.
```mermaid
flowchart TB
@@ -137,11 +137,11 @@ flowchart TB
PHASE --> LEDGER["decisions.py\nreview_decisions"]
```
Il comando di operatore `tht` in `tools/tht/` è distinto dalla CLI Python dell'harness. Il primo gestisce installazione, lifecycle, autenticazione e workspace; il secondo esegue il workflow e le operazioni sui dati.
The operator command `tht` in `tools/tht/` is separate from the harness Python CLI. The former handles installation, lifecycle, authentication, and workspaces; the latter runs the workflow and data operations.
## Workflow a otto fasi e gate
## Eight-phase workflow and gates
La fonte di verità è `harness/workflow.yaml`. La fase corrente si calcola dal decision ledger, non da un campo aggiornato manualmente.
The source of truth is `harness/workflow.yaml`. The current phase is computed from the decision ledger, not from a manually updated field.
```mermaid
flowchart LR
+49 -50
View File
@@ -1,12 +1,11 @@
# Panoramica dell'architettura
# Architecture overview
> Per il dettaglio dei moduli e dei flussi vedi
> [Componenti, moduli e flussi](components.md).
> For details about modules and flows, see [Components, modules, and flows](components.md).
ThothII è un **datamart builder human-in-the-loop**: trasforma una domanda in linguaggio naturale in SQL validato (ed eventualmente un datamart dbt) attraverso un **workflow deterministico a 8 fasi NL→SQL**, in cui il modello *propone* e un revisore umano *decide* ai gate.
ThothII is a **human-in-the-loop datamart builder**. It turns a natural-language question into validated SQL, and optionally a dbt datamart, through a **deterministic eight-phase NL-to-SQL workflow** in which the model *proposes* and a human reviewer *decides* at gates.
L'autenticazione di produzione usa local oppure OIDC generico; il solo CLI operatore è tht.
Per sessioni, ruoli, gruppi, diagnostica e ripristino vedere la [documentazione autenticazione](authentication.md).
Production authentication uses local authentication or generic OIDC. `tht` is the only operator CLI.
For sessions, roles, groups, diagnostics, and recovery, see the [authentication documentation](authentication.md).
```mermaid
flowchart LR
@@ -20,71 +19,71 @@ flowchart LR
THT --> FE
```
## I tre progetti indipendenti
## The three independent projects
```
frontend (React/SSE) → backend (Fastify) → pi --mode rpc → tht/harness → DWH (sola lettura)
frontend (React/SSE) → backend (Fastify) → pi --mode rpc → tht/harness → DWH (read-only)
```
| Layer | Stack | Ruolo |
|---|---|---|
| **harness/** | Python (`tht` CLI) + Pi gate extension (JS) | Possiede il workflow e **tutta** la persistenza |
| **backend/** | Fastify + TypeScript | Ponte sottile senza database proprio |
| **frontend/** | React 18 + Vite | UI che renderizza i widget di gate e ricostruisce il transcript live dallo stream SSE |
| **harness/** | Python (`tht` CLI) + Pi gate extension (JS) | Owns the workflow and **all** persistence |
| **backend/** | Fastify + TypeScript | Thin bridge with no database of its own |
| **frontend/** | React 18 + Vite | UI that renders gate widgets and rebuilds the live transcript from the SSE stream |
## L'harness possiede il workflow
## The harness owns the workflow
`tht` (Python) è una CLI deterministica; `harness/.pi/extensions/tht-gate.js` è un'estensione Pi che guida il workflow a 8 fasi. La fonte di verità unica del workflow è `harness/workflow.yaml`; le regole di orchestrazione che il modello deve seguire sono in `harness/.pi/skills/tht-sessione/SKILL.md`. La "fase corrente" **non è memorizzata**: viene calcolata piegando il decision ledger (`harness/tht/phase.py`) — va letta prima di ragionare sulla logica di fase.
`tht` (Python) is a deterministic CLI. `harness/.pi/extensions/tht-gate.js` is a Pi extension that guides the eight-phase workflow. `harness/workflow.yaml` is the single source of workflow truth, and `harness/.pi/skills/tht-sessione/SKILL.md` contains the orchestration rules the model must follow. The "current phase" is **not stored**. It is computed by folding the decision ledger (`harness/tht/phase.py`), which must be read before reasoning about phase logic.
## Persistenza = documenti di fase, non chat
## Persistence means phase documents, not chat
Una sessione è una directory sotto `sessions/` (path definito dal workspace): `session_manifest.yaml` + artefatti per fase (`question.md`, `schema_linking.json`, `sql_final.sql`, …) + `review_decisions.jsonl`. Il contratto (SKILL.md): *"lo stato persistito è la verità — ciò che non è registrato non è accaduto"*. Non esiste uno store di transcript verbatim. Un processo Pi ripreso ricostruisce il contesto da `tht session show <id>` + gli artefatti su disco.
A session is a directory under `sessions/` (the workspace defines the path): `session_manifest.yaml`, phase artifacts (`question.md`, `schema_linking.json`, `sql_final.sql`, and others), and `review_decisions.jsonl`. The contract says: *"persisted state is the truth; what is not recorded did not happen"*. There is no verbatim transcript store. A resumed Pi process rebuilds context from `tht session show <id>` and the artifacts on disk.
## Il backend è un ponte sottile senza database
## The backend is a thin bridge with no database
- `ThtRunner` esegue subcommand `tht` in shell
- `PiProcessManager` esegue un processo Pi figlio per sessione e fa da bridge al suo stream RPC
- `SessionBridge` mappa eventi RPC di Pi → eventi client (`ui_request` / `text_delta` / `info`)
- `SseHub` distribuisce questi eventi via SSE al browser
- `ThtRunner` runs `tht` subcommands in a shell.
- `PiProcessManager` runs one Pi child process per session and bridges its RPC stream.
- `SessionBridge` maps Pi RPC events to client events (`ui_request` / `text_delta` / `info`).
- `SseHub` distributes these events to the browser over SSE.
Le impostazioni applicative vivono in un file JSON (`backend/data/settings.json`), non in un database.
Application settings live in a JSON file (`backend/data/settings.json`), not in a database.
## Contratto del gate human-in-the-loop
## Human-in-the-loop gate contract
Il modello propone; un revisore umano decide ai gate tramite widget:
The model proposes; a human reviewer decides at gates through widgets:
- **`reviewer_select`** — scelta singola: un'opzione con `decision` payload auto-conferma/persiste direttamente; un'opzione senza payload chiede soltanto
- **`reviewer_decide`** — multiselect: ogni scelta È una decisione
- **`reviewer_confirm`** — gate su artefatto/fase
- **`reviewer_select`**: single choice. An option with a `decision` payload confirms and persists directly; an option without a payload only asks.
- **`reviewer_decide`**: multiselect. Each choice is a decision.
- **`reviewer_confirm`**: artifact or phase gate.
Il frontend renderizza questi widget-descriptor (registro in `src/widgets/`); il transcript live viene ricostruito in memoria dallo stream SSE (`src/store/sessionStore.ts`) — **non è persistito**.
The frontend renders these widget descriptors (registry in `src/widgets/`). It rebuilds the live transcript in memory from the SSE stream (`src/store/sessionStore.ts`); it is **not persisted**.
## Evidence curata e immutabile
## Curated and immutable Evidence
Il repository del workspace è il confine di pubblicazione: il curatore prepara `evidence/source/`,
revisa le unità in `evidence/curated/`, valida e fa merge. Per `evidence.schema_version: 2` il
runtime materializza l'intero albero `evidence/` dal commit Git esatto, ma il renderer consegna al
preprocessing esattamente `curated/**/*.md` dalla root immutabile della revisione. Sorgenti, manifest
ed evaluation restano disponibili solo per tracciabilità. Il runtime non modifica, stagea, committa
o pubblica il repository di authoring.
The workspace repository is the publication boundary. The curator prepares `evidence/source/`,
reviews units in `evidence/curated/`, validates them, and merges them. With `evidence.schema_version: 2`,
the runtime materializes the full `evidence/` tree from the exact Git commit, but the renderer passes
only `curated/**/*.md` from the immutable revision root to preprocessing. Sources, manifests, and
evaluation data remain available for traceability. The runtime never modifies, stages, commits, or
publishes the authoring repository.
Prima dell'indicizzazione, il corpus curato della revisione pinnata viene validato. La collezione
Qdrant condivisa conserva il vettore dense senza nome di Schema e Memory; il preprocessing Evidence
può aggiungere soltanto in modo additivo il vettore sparse `bm25` con `idf`, senza eliminare,
rinominare o ricreare la collezione. `workspace preprocess evidence` e la parte Evidence di
`workspace preprocess run` sono le sole operazioni pubbliche che effettuano questo upgrade.
Before indexing, the curated corpus from the pinned revision is validated. The shared Qdrant
collection keeps the unnamed dense vector used by Schema and Memory. Evidence preprocessing may
add only the sparse `bm25` vector with `idf`, without deleting, renaming, or recreating the collection.
`workspace preprocess evidence` and the Evidence part of `workspace preprocess run` are the only public
operations that perform this upgrade.
## Punti di attenzione ricorrenti
## Recurring points of attention
- `tht -c`/`--config` è un'opzione **per-comando**: deve seguire il subcommand, mai precederlo (`ThtRunner.buildArgv` lo impone).
- L'output `--json` deve essere JSON puro su stdout — è un contratto machine-readable.
- Le stringhe UI sono in inglese; il *contenuto* dei documenti resta nella lingua del workspace, perché è il dato reale — solo chrome e label sono in inglese.
- Ogni workspace imposta il target DWH e le proprie directory operative. I segreti restano nei file protetti dell'installazione e non nel repository del workspace.
- Le impostazioni sono globali (`backend/data/settings.json`: workspace/provider/modello/thinking); il form di nuova sessione richiede solo la domanda.
- **Resume**: una sessione riprendibile rientra all'ultima fase incompleta. Il backend rifiuta il resume con 409 se `finalized` o `archived`; `PiProcessManager.spawnFor` deve inviare `/riprendi-sessione <id>` (resume) vs `/nuova-domanda` (nuova) — il prompt sbagliato trasforma silenziosamente un resume in una nuova domanda.
- `tht -c`/`--config` is a **per-command** option. It must follow the subcommand, never precede it (`ThtRunner.buildArgv` enforces this).
- `--json` output must be plain JSON on stdout. It is a machine-readable contract.
- UI strings are in English. Document *content* stays in the workspace language because it is the actual data; only chrome and labels are in English.
- Each workspace defines its DWH target and working directories. Secrets remain in protected installation files, not in the workspace repository.
- Settings are global (`backend/data/settings.json`: workspace/provider/model/thinking); the new-session form asks only for the question.
- **Resume**: a resumable session returns to its last incomplete phase. The backend rejects resume with 409 when `finalized` or `archived`; `PiProcessManager.spawnFor` must send `/riprendi-sessione <id>` for resume and `/nuova-domanda` for a new session. The wrong prompt silently turns a resume into a new question.
## Come si lancia lo stack
## Starting the stack
Lo stack locale si avvia con `./scripts/run-stack.sh`, dopo aver creato
`deploy/env/local.env` da `deploy/env/local.env.example`. Il core Compose include Pi; DWH,
vector DB, embedding e LLM sono endpoint esterni configurati nel file locale.
Start the local stack with `./scripts/run-stack.sh` after creating
`deploy/env/local.env` from `deploy/env/local.env.example`. The Compose core includes Pi; DWH,
the vector database, embeddings, and the LLM are external endpoints configured in the local file.
+13
View File
@@ -47,6 +47,19 @@ evidence/
`source/`, the manifest, the evaluation set, and other support files are materialized for
traceability but never acquired by v2 runtime preprocessing.
### Curated unit representation
The `schema_version` inside each `curated/**/*.md` file is distinct from the workspace descriptor
version above. Unit schema v1 stores the complete typed unit in YAML frontmatter and remains
readable for compatibility. Unit schema v2 keeps short metadata in frontmatter and stores the
typed payload, supporting excerpts, and review items in a deterministic Markdown body.
V2 bodies use headings, paragraphs, code lists, enum tables, fenced SQL, and blockquotes according
to the Evidence kind. Invisible `tht:` comments delimit typed fields. Parsers must reject missing,
duplicate, unknown, or unstructured body content; they must never silently ignore it. Newly
prepared units use v2. `tht evidence migrate <workspace-root>` upgrades existing v1 units locally
without a model call, commit, publication, or semantic change.
### Example: filesystem
```yaml
+135 -135
View File
@@ -1,10 +1,10 @@
# Disambiguazione nelle prime fasi del workflow
# Disambiguation in the early workflow phases
La disambiguazione è il processo con cui Thoth trasforma una domanda naturale ambigua in un significato verificato dal reviewer prima di costruire lo schema-linking e il SQL.
Disambiguation turns an ambiguous natural-language question into a meaning that the reviewer verifies before schema linking and SQL generation begin.
Il principio architetturale è **human-in-the-middle**: il modello propone interpretazioni motivate, il reviewer decide, il gate persiste la decisione nel ledger. Il modello non può scegliere autonomamente un significato solo perché è quello semanticamente più vicino.
The architectural principle is **human-in-the-middle**: the model proposes reasoned interpretations, the reviewer decides, and the gate persists the decision in the ledger. The model cannot choose a meaning on its own simply because it is the closest semantic match.
La procedura è definita nella skill canonica [tht-sessione](../harness/.pi/skills/tht-sessione/SKILL.md), soprattutto nelle sezioni F1 e F2, ed è applicata dai widget in [tht-gate.js](../harness/.pi/extensions/tht-gate.js).
The canonical [tht-sessione](../harness/.pi/skills/tht-sessione/SKILL.md) skill defines the procedure, especially its F1 and F2 sections. The widgets in [tht-gate.js](../harness/.pi/extensions/tht-gate.js) enforce it.
```mermaid
stateDiagram-v2
@@ -23,105 +23,105 @@ stateDiagram-v2
ACCEPTED --> [*]
```
## Dove avviene la disambiguazione
## Where disambiguation happens
La disambiguazione iniziale attraversa quattro passaggi distinti:
Initial disambiguation has four distinct steps:
```text
F1 Chiarimento → significato della domanda
F2 Memory → eventuale conoscenza già chiarita e riusabile
F3 Riscrittura → domanda esplicita e non ambigua
F4 Schema-linking → traduzione del significato in tabelle, colonne e join
F1 Clarification → meaning of the question
F2 Memory → previously clarified knowledge that may be reused
F3 Rewriting → explicit, unambiguous question
F4 Schema linking → translating meaning into tables, columns, and joins
```
Questi passaggi non sono intercambiabili:
These steps are not interchangeable:
- F1 stabilisce cosa significa la domanda;
- F2 propone conoscenza preesistente, senza applicarla automaticamente;
- F3 rende esplicito il significato concordato;
- F4 sceglie gli oggetti tecnici necessari per quel significato.
- F1 establishes what the question means.
- F2 proposes existing knowledge without applying it automatically.
- F3 makes the agreed meaning explicit.
- F4 selects the technical objects needed for that meaning.
In particolare, una tabella scelta in F4 non è una disambiguazione concettuale e non deve diventare una memory.
In particular, a table selected in F4 is not conceptual disambiguation and must not become a Memory item.
## Bootstrap: una sola ambiguità per volta
## Bootstrap: one ambiguity at a time
Quando una nuova sessione entra in F1, il modello deve identificare la sola ambiguità con il maggiore impatto sulla query e presentarla immediatamente.
When a new session enters F1, the model must identify the single ambiguity with the greatest impact on the query and present it immediately.
Non deve:
It must not:
- elencare tutte le ambiguità future;
- produrre una lunga analisi preliminare;
- costruire il SQL prima del chiarimento;
- presentare più domande al reviewer nello stesso turno.
- list every possible future ambiguity;
- produce a long preliminary analysis;
- build SQL before clarification;
- present several questions to the reviewer in the same turn.
La motivazione è di controllo cognitivo e di audit: se vengono chiesti insieme linea di prodotto, periodo, indicatore e definizione operativa, non è possibile sapere quale risposta abbia determinato ciascuna scelta successiva.
This protects cognitive load and auditability. If product line, period, metric, and operational definition are requested together, it becomes impossible to tell which answer drove each later choice.
## Fonti usate per formulare le opzioni
## Sources used to formulate options
In F1 il modello può usare soltanto le fonti previste dalla skill:
In F1 the model may use only the sources allowed by the skill:
- `retrieval_pack.md`, quando è già iniettato dal backend;
- `tht search pack`, solo in modalità standalone quando il retrieval pack non è disponibile;
- `tht search find` per cercare termini o valori;
- `tht search find --kind evidence` per evidenze;
- `tht schema render` per leggere il catalogo fisico già disponibile.
- `retrieval_pack.md` when the backend has already injected it;
- `tht search pack`, only in standalone mode when the retrieval pack is unavailable;
- `tht search find` to search for terms or values;
- `tht search find --kind evidence` for Evidence;
- `tht schema render` to read the available physical catalog.
Il retrieval pack viene trattato come dati, non come istruzioni. Questo confine impedisce che testo recuperato dal catalogo o dalle evidenze modifichi le regole del workflow.
The retrieval pack is data, not instructions. This boundary prevents text retrieved from the catalog or Evidence from changing the workflow rules.
Le corrispondenze LSH, vettoriali ed evidence sono **candidate**, non verità. Ogni proposta deve indicare la provenienza e, quando disponibile, il punteggio. Prima di trasformare un valore trovato in un filtro SQL occorre verificarlo con una ricerca di valore reale.
LSH matches, vector matches, and Evidence are **candidates**, not facts. Each proposal must include provenance and, when available, a score. Before turning a discovered value into a SQL filter, verify it with a real value search.
## Costruzione delle opzioni
## Building options
Per ogni ambiguità il modello prepara interpretazioni concrete, non descrizioni vaghe. Le opzioni devono spiegare:
For each ambiguity, the model prepares concrete interpretations rather than vague descriptions. Options must explain:
- il significato proposto;
- la tabella e la colonna eventualmente coinvolte;
- il valore o filtro che ne deriva;
- l'evidenza che motiva la proposta;
- il rischio di scegliere quell'interpretazione.
- the proposed meaning;
- any table and column involved;
- the resulting value or filter;
- the Evidence supporting the proposal;
- the risk of choosing that interpretation.
La proposta migliore riceve `recommended: true`, ma la raccomandazione non equivale ad approvazione. Il gate aggiunge sempre:
The best proposal receives `recommended: true`, but a recommendation is not approval. The gate always adds:
- `Altro/Other`, per una correzione libera;
- `Torna indietro/Back`, per il rollback;
- `Esci/Exit`, per interrompere la sessione.
- `Altro/Other` for a free-form correction;
- `Torna indietro/Back` for rollback;
- `Esci/Exit` to stop the session.
L'opzione “accetta la proposta” deve essere esplicita: il reviewer non deve essere costretto a confermare implicitamente una scelta preselezionata.
The "accept proposal" option must be explicit. The reviewer must not be forced to confirm a preselected choice implicitly.
## Scelta tra `reviewer_select` e `reviewer_decide`
## Choosing between `reviewer_select` and `reviewer_decide`
La forma del problema determina il widget.
The shape of the problem determines the widget.
### Interpretazioni mutuamente esclusive
### Mutually exclusive interpretations
Quando esattamente una sola interpretazione può essere corretta si usa `reviewer_select`.
Use `reviewer_select` when exactly one interpretation can be correct.
Esempi:
Examples:
- “ablazione” significa una procedura transcatetere oppure qualcos'altro;
- “anno” significa anno solare oppure anno fiscale;
- “biciclette attive” significa modelli a catalogo oppure unità presenti nella produzione corrente.
- "ablation" means a catheter procedure or something else;
- "year" means calendar year or fiscal year;
- "active bicycles" means catalog models or units in current production.
Ogni opzione concreta contiene una decisione `concept_clarified`. La scelta del reviewer è già la conferma e viene persistita direttamente: non serve un secondo `reviewer_decide`.
Each concrete option contains a `concept_clarified` decision. The reviewer's choice is also the confirmation and is persisted directly. A second `reviewer_decide` is not needed.
### Più risposte contemporaneamente valide
### Several answers can be valid at once
Quando più interpretazioni possono essere vere nello stesso tempo si usa `reviewer_decide`, che visualizza un multiselect.
Use `reviewer_decide`, which displays a multiselect, when several interpretations can be true at the same time.
Esempi:
Examples:
- la domanda comprende più popolazioni valide;
- sono possibili più codici di procedura;
- devono essere considerate più finestre temporali;
- più condizioni sono indipendentemente applicabili.
- the question includes several valid populations;
- several procedure codes are possible;
- several time windows must be considered;
- several conditions apply independently.
Usare `reviewer_select` in questi casi sarebbe fuorviante perché obbligherebbe il reviewer a sceglierne una sola.
Using `reviewer_select` in these cases would mislead the reviewer by forcing a single choice.
In F1 le scelte multiple producono più decisioni `concept_clarified`, mentre la fase viene chiusa in seguito con il gate di fase.
In F1, multiple choices produce several `concept_clarified` decisions. The phase is closed later through the phase gate.
## Persistenza delle decisioni
## Persisting decisions
La scelta del reviewer non rimane soltanto nella UI. Il gate registra nel ledger:
The reviewer's choice does not remain only in the UI. The gate records this in the ledger:
```text
type = concept_clarified
@@ -130,124 +130,124 @@ detail = definizione o regola operativa
rationale = motivazione, evidenza e/o testo del reviewer
```
La regola “una decisione, un comando” impedisce al modello di scrivere direttamente il ledger con shell, `tht decision add` o `tht phase advance`. Il gate è l'unico punto autorizzato a trasformare il widget in stato persistito.
The "one decision, one command" rule prevents the model from writing to the ledger through the shell, `tht decision add`, or `tht phase advance`. The gate is the only component allowed to turn a widget interaction into persisted state.
La decisione è quindi riutilizzabile come memory solo dopo la promozione esplicita di F8. Anche in quel caso viene conservato il contesto originale e non viene trasferita la scelta delle tabelle.
The decision can therefore be reused as Memory only after explicit promotion in F8. The original context is preserved, and table choices are not transferred.
## Gestione di “Altro” e testo libero
## Handling `Altro` and free text
`Altro/Other` non è una scelta neutra e non può essere ignorato.
`Altro/Other` is not a neutral choice and cannot be ignored.
Quando il reviewer inserisce testo libero, il modello deve:
When the reviewer enters free text, the model must:
1. interpretare il testo nel contesto della domanda;
2. incorporarlo nella proposta successiva;
3. registrare le parole del reviewer nel `rationale`;
4. chiedere nuovamente se il testo resta ambiguo.
1. interpret the text in the context of the question;
2. include it in the next proposal;
3. record the reviewer's words in `rationale`;
4. ask again if the text remains ambiguous.
Non è ammesso tornare automaticamente alla prima opzione consigliata o scegliere in silenzio una semantica plausibile.
The system must not automatically return to the first recommended option or silently choose a plausible meaning.
Questo comportamento consente di distinguere una correzione umana da una semplice deselezione e mantiene l'audit leggibile.
This distinguishes a human correction from a simple deselection and keeps the audit readable.
## Ambiguità non risolta
## Unresolved ambiguity
Un'ambiguità non può sparire perché il modello non sa risolverla. Deve essere resa esplicita con un'opzione del tipo:
An ambiguity cannot disappear because the model does not know how to resolve it. Make it explicit with an option such as:
```text
Lasciare aperta l'ambiguità
Leave the ambiguity open
```
L'opzione deve spiegare:
The option must explain:
- quale parte della query resta indeterminata;
- quale rischio introduce;
- quale effetto può avere su filtri, conteggi o join.
- which part of the query remains undetermined;
- what risk this introduces;
- how it may affect filters, counts, or joins.
Il reviewer può quindi accettare consapevolmente il rischio oppure chiedere ulteriori ricerche.
The reviewer can then accept the risk knowingly or request more research.
## Chiusura di F1
## Closing F1
Ogni singolo widget può registrare uno o più chiarimenti, ma non chiude automaticamente F1. Quando il chiarimento è completo il modello presenta `reviewer_confirm kind:"phase"`.
Each widget can record one or more clarifications, but it does not close F1 automatically. When clarification is complete, the model presents `reviewer_confirm kind:"phase"`.
Il riepilogo di chiusura deve contenere l'intero insieme dei chiarimenti della fase, non solo l'ultimo. Il gate aggiunge inoltre le decisioni registrate dal ledger, evitando che il modello debba ricopiarle manualmente.
The closing summary must contain every clarification from the phase, not only the latest one. The gate also adds the decisions recorded in the ledger, so the model does not have to copy them by hand.
La chiusura F1 avanza a F2. La domanda non viene ancora riscritta: `question_rewritten` appartiene a F3.
Closing F1 advances to F2. The question is not rewritten yet: `question_rewritten` belongs to F3.
## F2: memory come supporto alla disambiguazione
## F2: Memory as disambiguation support
F2 non sostituisce il chiarimento umano. Cerca memory concettuali già promosse:
F2 does not replace human clarification. It searches for previously promoted conceptual Memory:
```text
tht memory search "<domanda>" --session <id> --json
```
Il risultato viene proposto in una checklist unica. Sono ammesse solo memory `concept_clarified`; le decisioni su tabelle, colonne o SQL non sono trasferibili.
The result is presented as one checklist. Only `concept_clarified` Memory is allowed; decisions about tables, columns, or SQL cannot be transferred.
Se il reviewer applica una memory:
If the reviewer applies a Memory item:
- viene registrato un nuovo `concept_clarified` nella sessione corrente;
- il `rationale` cita l'id `mem-XXXX` della fonte;
- la scelta viene comunque contestualizzata nella domanda corrente.
- a new `concept_clarified` is recorded in the current session;
- `rationale` cites the source's `mem-XXXX` ID;
- the choice is still placed in the context of the current question.
Se il reviewer deseleziona una memory, essa non viene applicata ora, ma non viene cancellata globalmente e può essere riproposta dopo una riapertura di F2.
If the reviewer deselects a Memory item, it is not applied now. It is not deleted globally and may be proposed again after F2 is reopened.
## F3: rendere esplicito il risultato
## F3: make the result explicit
F3 trasforma i chiarimenti accettati in una domanda riscritta:
F3 turns accepted clarifications into a rewritten question with:
- popolazione espressa con termini del modello dati;
- condizioni separate e numerate;
- concetti ambigui sostituiti dalle definizioni concordate;
- output atteso esplicito;
- assunzioni dichiarate.
- the population expressed using data-model terms;
- separate, numbered conditions;
- ambiguous concepts replaced by the agreed definitions;
- an explicit expected output;
- stated assumptions.
La riscrittura non introduce nuove scelte implicite. Se emerge un'ambiguità sostanziale, il percorso corretto è riaprire F1, non “aggiustare” il significato dentro F3 o dentro il SQL.
Rewriting must not introduce new implicit choices. If a material ambiguity appears, reopen F1 rather than "fixing" the meaning in F3 or SQL.
## F4: disambiguazione tecnica dello schema
## F4: technical schema disambiguation
Solo dopo F3 la disambiguazione semantica viene tradotta in oggetti tecnici.
Only after F3 is the semantic meaning translated into technical objects.
Il modello propone:
The model proposes:
- tabelle da promuovere o escludere;
- colonne candidate;
- colonne di output;
- join necessari.
- tables to include or exclude;
- candidate columns;
- output columns;
- required joins.
Il reviewer cura le tabelle e le colonne con `reviewer_schema_linking`. I join vengono trattati in una revisione separata `reviewer_decide` join-only.
The reviewer curates tables and columns with `reviewer_schema_linking`. Joins are handled in a separate join-only `reviewer_decide` review.
Questa separazione è importante: una tabella può essere corretta per una domanda e totalmente irrilevante per un'altra. Per questo le decisioni F4 sono locali alla sessione e non diventano memory.
This separation matters. A table can be correct for one question and completely irrelevant to another. F4 decisions are therefore local to the session and do not become Memory.
## Riapertura e rollback
## Reopening and rollback
Se il reviewer usa “Torna indietro”, la sessione riprende dalla fase indicata esaminando gli artefatti ancora validi.
When the reviewer uses "Torna indietro", the session resumes from the selected phase and examines the artifacts that are still valid.
Gli artefatti oltre la fase riaperta vengono invalidati da `tht phase reopen`; quelli precedenti non vanno rigenerati senza motivo. `effective_decisions()` esclude le decisioni stale, così un chiarimento superato non può alimentare una nuova promozione memory o una nuova sintesi SQL.
Artifacts after the reopened phase are invalidated by `tht phase reopen`; earlier artifacts should not be regenerated without a reason. `effective_decisions()` excludes stale decisions, so an outdated clarification cannot feed a new Memory promotion or SQL synthesis.
## Invarianti di sicurezza e qualità
## Security and quality invariants
La disambiguazione è affidabile perché la stessa regola è applicata su più livelli:
Disambiguation is reliable because the same rule is enforced at several levels:
1. la skill prescrive una sola ambiguità per volta;
2. il gate offre widget vincolati e controlli `Altro/Back/Exit`;
3. il ledger registra le decisioni e il rationale;
4. i prerequisiti impediscono di saltare fasi;
5. F4 separa concetti da schema-linking;
6. le memory accettano solo `concept_clarified`;
7. rollback ed `effective_decisions()` escludono stato obsoleto.
1. the skill requires one ambiguity at a time;
2. the gate provides constrained widgets and `Altro/Back/Exit` controls;
3. the ledger records decisions and rationale;
4. prerequisites prevent phases from being skipped;
5. F4 separates concepts from schema linking;
6. Memory accepts only `concept_clarified`;
7. rollback and `effective_decisions()` exclude obsolete state.
Il risultato è una catena verificabile:
The result is a verifiable chain:
```text
termine ambiguo
→ evidenza e candidate interpretations
→ scelta esplicita del reviewer
→ concept_clarified nel ledger
→ domanda riscritta
→ schema-linking locale
→ piano CTE e SQL
ambiguous term
→ Evidence and candidate interpretations
→ explicit reviewer choice
→ concept_clarified in the ledger
→ rewritten question
→ local schema linking
→ CTE plan and SQL
```
## Riferimenti
## References
- [Gestione delle memory](gestione-memory.md)
- [Memory management](gestione-memory.md)
+118 -97
View File
@@ -1,36 +1,36 @@
# Evidence: sorgenti, preparazione e revisione
# Evidence: sources, preparation, and review
Questa pagina descrive il ciclo completo delle Evidence di workspace: dove si trova il materiale originale, come si producono le unità curate, quando diventano disponibili al runtime e quali responsabilità hanno autore e revisore.
This page describes the complete workspace Evidence lifecycle: where original material lives, how curated units are produced, when they become available at runtime, and what the author and reviewer are responsible for.
## Regola di pubblicazione
## Publication rule
Il workspace repository è la sorgente versionata. ThothII lo legge, lo valida e pubblica una generazione atomica. Non modifica, committa o pusha il repository dell'autore.
The workspace repository is the versioned source. ThothII reads it, validates it, and publishes an atomic generation. It does not modify, commit, or push the author's repository.
Una Evidence diventa utilizzabile dal workflow solo quando:
Evidence becomes available to the workflow only when:
1. il materiale originale è presente in `source/`;
2. l'unità derivata è presente in `curated/`;
3. il manifest collega unità, sorgente e hash;
4. la validazione non produce errori o review item irrisolti;
5. la pipeline di preprocessing costruisce una generazione indicizzata e la attiva.
1. the original material is in `source/`;
2. the derived unit is in `curated/`;
3. the manifest links the unit, source, and hash;
4. validation finds no errors or unresolved review items;
5. preprocessing builds and activates an indexed generation.
Una proposta generata durante una sessione non è automaticamente Evidence pubblicata. Il modello può proporre una formula o una spiegazione, ma un curatore deve importarla, revisionarla e pubblicarla nel repository prima che un'altra sessione possa recuperarla.
A proposal generated during a session is not automatically published Evidence. The model may propose a formula or explanation, but a curator must import, review, and publish it in the repository before another session can retrieve it.
## Dove deve stare il primo sorgente
## Where the original source belongs
Per il filesystem Evidence v2, il primo sorgente autorevole deve stare nella directory `source/` del repository di workspace. `curated/` contiene il risultato revisionato e indicizzato, non il materiale originale.
For filesystem Evidence v2, the authoritative original source must be in the `source/` directory of the workspace repository. `curated/` contains the reviewed and indexed result, not the original material.
```text
<workspace-repository>/
├── source/ # materiale originale, preservato
│ └── <dominio>/<file>.md
├── curated/ # Evidence Units revisionate
│ └── <dominio>/<unit>.md
├── manifest.yaml # legami, hash e metadati della preparazione
└── example/ # esempi e materiale di supporto
│ └── <domain>/<file>.md
├── curated/ # reviewed Evidence Units
│ └── <domain>/<unit>.md
├── manifest.yaml # preparation links, hashes, and metadata
└── example/ # examples and supporting material
```
Il descriptor del workspace deve dichiarare `evidence.schema_version: 2` e, per una sorgente filesystem, usare esattamente:
The workspace descriptor must declare `evidence.schema_version: 2` and use exactly this configuration for a filesystem source:
```yaml
evidence:
@@ -42,141 +42,162 @@ evidence:
- "curated/**/*.md"
```
La configurazione storica può esporre `source_root`, per esempio `${THT_DOCS_ROOT}` o `/data`. Per la struttura v2 il pattern runtime deve selezionare solo `curated/**/*.md`. Non bisogna indicizzare direttamente `source/`, mescolare `source/` e `curated/`, usare glob più ampi o includere file non Markdown.
The legacy configuration may expose `source_root`, such as `${THT_DOCS_ROOT}` or `/data`. For the v2 structure, the runtime pattern must select only `curated/**/*.md`. Do not index `source/` directly, mix `source/` and `curated/`, use broader globs, or include non-Markdown files.
HTTP e S3 sono adapter distinti. Non usano la struttura filesystem `source/` e `curated/`, ma devono comunque fornire una provenienza stabile, senza credenziali negli URI e con il contratto specifico dell'adapter.
HTTP and S3 are separate adapters. They do not use the filesystem structure `source/` and `curated/`, but they must still provide stable provenance, without credentials in URIs, under the adapter-specific contract.
## Come deve essere fatta un'unità curata
## What a curated unit must contain
Le unità Markdown lette dal loader storico della CLI hanno frontmatter YAML. I campi minimi sono `id` e `title`; `tier`, `status`, `sources`, `tables`, `concepts` descrivono il contesto dell'unità.
Canonical Curated Evidence v2 keeps short machine metadata in YAML frontmatter and renders the
reviewable content as real Markdown. The body layout is deterministic for each Evidence kind:
prose uses sections and paragraphs, identifiers use code lists, enum values use tables, formulas
use fenced SQL, supporting excerpts use blockquotes, and unresolved review items use dedicated
blocks.
```markdown
---
id: evidence:autonomia-batteria
title: Autonomia nominale della batteria
tier: structural
status: reviewed
sources:
- source/domain/bicycle.md
tables:
- bicycle_model
concepts:
- concept:battery-range
schema_version: 2
id: evidence:fascia-pediatrica
title: Fascia pediatrica
kind: domain
purposes:
- disambiguation
language: it
provenance:
source_file: source/domain/paziente.md
source_sha256: sha256:0000000000000000000000000000000000000000000000000000000000000000
---
Definizione verificata dell'autonomia nominale per modello di bicicletta elettrica.
# Fascia pediatrica
La regola deve essere abbastanza atomica da poter essere citata senza ricostruire
un intero capitolo. Il testo deve distinguere definizione, condizioni e limiti.
## Regola
La fascia pediatrica comprende i pazienti con età inferiore a 18 anni.
## Estratti di supporto
> I pazienti sotto i 18 anni sono pediatrici.
```
Le unità curate devono essere atomiche, leggibili da un secondo revisore e sostenute dal sorgente. I riferimenti di provenienza devono permettere di risalire al file originale e alla porzione che supporta l'affermazione. Non inserire segreti, token, password o credenziali nei metadati o negli URI.
The actual files also contain invisible `tht:` comments delimiting typed fields. Curators edit the
visible Markdown between those markers; removing or duplicating markers makes validation fail
closed instead of silently ignoring content. V1 files containing only frontmatter remain readable
for compatibility, but newly prepared units use v2.
La forma canonica moderna conserva anche il tipo di Evidence, la provenienza, gli estratti di supporto, `source_file` e `source_sha256`. Il contratto canonico rifiuta campi sconosciuti, metadati mutabili e URI con credenziali. Gli identificatori devono restare stabili anche quando cambia il tipo di unità.
Curated units must be atomic, readable by a second reviewer, and supported by the source.
Provenance references must lead back to the original file and the passage that supports the claim.
Do not put secrets, tokens, passwords, or credentials in metadata or URIs.
## Preparazione: dal sorgente alla generazione attiva
The modern canonical form also stores the Evidence kind, provenance, supporting excerpts, `source_file`, and `source_sha256`. The canonical contract rejects unknown fields, mutable metadata, and URIs containing credentials. Identifiers must remain stable even when a unit's kind changes.
## Preparation: from source to active generation
```mermaid
flowchart TD
SRC["source/DOMINIO/*.md\nmateriale originale"] --> PREP["tht evidence prepare\npreparazione candidata"]
PREP --> CAND["curated/DOMINIO/*.md\nunità proposte o aggiornate"]
CAND --> VAL["tht evidence validate\ncontrolli di struttura e legami"]
VAL -->|errori o review item| FIX["Correzioni dell'autore\ne revisione"]
SRC["source/DOMAIN/*.md\noriginal material"] --> PREP["tht evidence prepare\ncandidate preparation"]
PREP --> CAND["curated/DOMAIN/*.md\nproposed or updated units"]
CAND --> VAL["tht evidence validate\nstructure and link checks"]
VAL -->|errors or review items| FIX["Author corrections\nand review"]
FIX --> PREP
VAL -->|publishable| COMMIT["Commit del repository\nauthoring clone"]
COMMIT --> ING["tht preprocess evidence\nnormalizzazione e chunking"]
ING --> BM25["Indice BM25"]
ING --> VEC["Embedding e vector store"]
BM25 --> GEN["Generazione candidata"]
COMMIT --> ING["tht preprocess evidence\nnormalization and chunking"]
ING --> BM25["BM25 index"]
ING --> VEC["Embeddings and vector store"]
BM25 --> GEN["Candidate generation"]
VEC --> GEN
GEN --> ACT["Generazione attiva"]
ACT --> RUNTIME["Ricerca Evidence nel workflow"]
GEN --> ACT["Active generation"]
ACT --> RUNTIME["Evidence retrieval in the workflow"]
```
La preparazione può ristrutturare sorgenti cambiate, ma non pubblica da sola. `prepare` produce una proposta e può indicare il documento coinvolto in caso di errore. `validate` non scrive né pubblica. La pubblicazione della revisione è un'azione del curatore. Il runtime legge una revisione completa e validata, poi la pipeline crea una generazione versionata. L'attivazione è atomica: una generazione precedente resta disponibile secondo la policy di retention.
Preparation can restructure changed sources, but it does not publish by itself. `prepare` produces a proposal and can identify the document involved in an error. `validate` does not write or publish. The curator publishes the revision. The runtime reads a complete, validated revision, then the pipeline creates a versioned generation. Activation is atomic, and a previous generation remains available under the retention policy.
La ricerca runtime usa il recupero ibrido. Il ramo denso usa gli embedding, il ramo BM25 usa la ricerca lessicale e la fusione deterministica ordina i risultati. L'unità pubblicata conserva la provenienza, che il modello deve citare quando usa l'evidence.
Runtime retrieval is hybrid. The dense branch uses embeddings, the BM25 branch uses lexical search, and deterministic fusion orders the results. The published unit keeps its provenance, which the model must cite when it uses the Evidence.
## Responsabilità del creatore
## Author responsibilities
Il creatore prepara il materiale e rende verificabile ogni unità. In pratica deve:
The author prepares the material and makes every unit verifiable. The author must:
- mettere il materiale originale in `source/`, senza sovrascriverne il significato durante la curatela;
- suddividere il contenuto in unità atomiche, una regola o definizione per unità quando possibile;
- assegnare un identificatore stabile e un titolo comprensibile;
- indicare la provenienza, le tabelle e i concetti coinvolti quando sono noti;
- mantenere il testo nella lingua del workspace;
- separare fatti, regole, esempi, formule e limiti;
- riportare gli estratti che sostengono l'unità, senza estendere la conclusione oltre il sorgente;
- eseguire `tht evidence prepare` e `tht evidence validate`;
- risolvere ogni errore e ogni review item prima di proporre il commit;
- fornire al revisore il contesto necessario, inclusi i cambiamenti nel sorgente e il motivo di eventuali rinomini o ritiri.
- put the original material in `source/` without changing its meaning during curation;
- split the content into atomic units, with one rule or definition per unit when possible;
- assign a stable identifier and a clear title;
- provide provenance, tables, and related concepts when known;
- keep the text in the workspace language;
- separate facts, rules, examples, formulas, and limits;
- include excerpts that support the unit without extending the conclusion beyond the source;
- run `tht evidence prepare` and `tht evidence validate`;
- resolve every error and review item before proposing a commit;
- give the reviewer the necessary context, including source changes and the reason for any rename or retirement.
Il creatore non deve:
The author must not:
- scrivere direttamente nel corpus attivo di produzione;
- trattare una proposta del modello come fatto verificato;
- eliminare un'unità solo perché non è più supportata senza registrare il ritiro o il relink;
- inserire credenziali nei metadati, nei file o negli URL di provenienza;
- modificare manualmente manifest, hash o generazioni per far passare la validazione.
- write directly to the active production corpus;
- treat a model proposal as a verified fact;
- delete an unsupported unit without recording its retirement or relink;
- put credentials in metadata, files, or provenance URLs;
- manually change manifests, hashes, or generations to make validation pass.
## Responsabilità del revisore
## Reviewer responsibilities
Il revisore non approva la forma del testo soltanto perché è chiara. Verifica il rapporto tra sorgente, unità e uso previsto. Per ogni unità deve controllare:
The reviewer does not approve text merely because it is clear. The reviewer checks the relationship between source, unit, and intended use. For each unit, the reviewer must check:
1. che il sorgente indicato esista nella revisione esaminata;
2. che l'estratto sostenga davvero l'affermazione;
3. che l'unità non unisca regole incompatibili o concetti indipendenti;
4. che tabelle, colonne e concetti siano identificati correttamente;
5. che l'identificatore sia stabile e non duplichi un'altra unità;
6. che il testo distingua definizione, condizione, eccezione ed esempio;
7. che non contenga informazioni sensibili o dettagli non presenti nel sorgente;
8. che la valutazione del retrieval copra le query rilevanti e non nasconda risultati vuoti.
1. the cited source exists in the reviewed revision;
2. the excerpt actually supports the claim;
3. the unit does not combine incompatible rules or independent concepts;
4. tables, columns, and concepts are identified correctly;
5. the identifier is stable and does not duplicate another unit;
6. the text distinguishes the definition, condition, exception, and example;
7. it contains no sensitive information or details absent from the source;
8. retrieval evaluation covers relevant queries and does not hide empty results.
Il revisore può approvare, chiedere modifiche, rigettare, ritirare o riallacciare un'unità a un nuovo sorgente. Un ritiro deve essere esplicito. Un relink deve indicare il nuovo file e deve lasciare una traccia verificabile della decisione. L'approvazione non comporta la pubblicazione immediata: il repository deve passare la validazione e la generazione deve superare la valutazione prima dell'attivazione.
The reviewer can approve, request changes, reject, retire, or relink a unit to a new source. Retirement must be explicit. A relink must name the new file and leave a verifiable record of the decision. Approval does not publish immediately: the repository must pass validation and the generation must pass evaluation before activation.
## Comandi disponibili
## Available commands
I comandi di authoring operano sul repository di workspace e non pubblicano direttamente.
Authoring commands operate on the workspace repository and do not publish directly.
```bash
# Prepara le sorgenti cambiate. Non committa e non pubblica.
# Prepare changed sources. Does not commit or publish.
tht evidence prepare <workspace-root>
# Rielabora tutte le sorgenti con la pipeline installata.
# Reprocess all sources with the installed pipeline.
tht evidence prepare <workspace-root> --upgrade
# Valida struttura, manifest, legami e review item.
# Rewrite legacy v1 units as readable v2 Markdown without model calls.
tht evidence migrate <workspace-root>
# Validate structure, manifest, links, and review items.
tht evidence validate <workspace-root>
# Restituisce JSON per CI o strumenti automatici.
# Return JSON for CI or automated tools.
tht evidence validate <workspace-root> --json
# Valuta il retrieval su una generazione o sulla generazione attiva.
# Evaluate retrieval on a generation or the active generation.
tht evidence evaluate <workspace-root> --config <workspace-config>
tht evidence evaluate <workspace-root> --config <workspace-config> --generation <id> --json
# Risolve una unità senza pubblicare: ritiro oppure nuovo collegamento al sorgente.
# Resolve a unit without publishing: retire it or link it to a new source.
tht evidence resolve <workspace-root> evidence:<id> --retire
tht evidence resolve <workspace-root> evidence:<id> --source source/domain/nuovo.md
# Materializza e indicizza una generazione versionata.
# Materialize and index a versioned generation.
tht preprocess evidence --config <workspace-config>
# Esecuzione a secco e ripresa di un job, quando supportate dalla configurazione.
# Dry run and resume a job when supported by the configuration.
tht preprocess evidence --config <workspace-config> --dry-run
tht preprocess evidence --config <workspace-config> --resume <run-id>
```
`evidence prepare`, `evidence validate` e `evidence resolve` richiedono il path del repository. `preprocess evidence` usa invece la configurazione del workspace, perché deve conoscere embedding, vector store, policy di retention e artifact directory.
`evidence prepare`, `evidence migrate`, `evidence validate`, and `evidence resolve` require the
repository path. `preprocess evidence` uses the workspace configuration because it needs the
embedding, vector store, retention policy, and artifact directory.
I codici di uscita sono parte del contratto operativo: `evidence validate` usa `0` quando il corpus è pubblicabile, `1` per errori di validazione e `3` quando restano solo elementi da revisionare o unità orfane. Con `--json`, stdout deve contenere solo JSON valido.
Exit codes are part of the operating contract: `evidence validate` returns `0` when the corpus is publishable, `1` for validation errors, and `3` when only review items or orphaned units remain. With `--json`, stdout must contain valid JSON only.
## Formule e proposte di sessione
## Formulas and session proposals
Le formule hanno un formato distinto dalle Evidence documentali. Una formula proposta durante una sessione può essere citata nella proposta corrente, ma non entra nel corpus runtime, non riceve un ID `evidence:` utilizzabile e non scrive nel repository. Per diventare pubblicata deve seguire lo stesso percorso di importazione, revisione e preprocessing delle altre unità.
Formulas use a format distinct from document Evidence. A formula proposed during a session may be cited in the current proposal, but it does not enter the runtime corpus, receive a usable `evidence:` ID, or write to the repository. To become published, it must follow the same import, review, and preprocessing path as other units.
## Riferimenti contrattuali
## Contract references
- [Contratto Workspace Evidence v3](contracts/workspace-evidence-v3.md)
- [Contratto della CLI di preprocessing](contracts/workspace-preprocessing-cli.md)
- [Workspace Evidence v3 contract](contracts/workspace-evidence-v3.md)
- [Preprocessing CLI contract](contracts/workspace-preprocessing-cli.md)
+104 -100
View File
@@ -1,10 +1,10 @@
# Configurazione locale dei modelli Pi
# Local Pi model configuration
Pi, l'agente che orchestra il workflow NL→SQL, risolve i modelli integrati e i provider
OpenAI-compatible dichiarati nel catalogo locale. ThothII applica una regola più stretta di Pi:
un provider o un modello custom non può essere registrato da codice in `harness/.pi/extensions/`.
Endpoint, protocollo, compatibilità e identificatori dei modelli appartengono esclusivamente ai file
locali `deploy/pi/models.json` e `deploy/pi/settings.json`.
Pi orchestrates the NL-to-SQL workflow and resolves built-in models and OpenAI-compatible providers
declared in the local catalog. ThothII applies a stricter rule than Pi: code in
`harness/.pi/extensions/` cannot register a provider or custom model. Endpoints, protocols,
compatibility settings, and model identifiers belong exclusively in the local files
`deploy/pi/models.json` and `deploy/pi/settings.json`.
> **ThothII operator note:** ThothII runs Pi only in Docker Compose. Paths under
> `~/.pi/agent/` in this document describe Pi's container-side behavior. Operators edit
@@ -12,62 +12,66 @@ locali `deploy/pi/models.json` e `deploy/pi/settings.json`.
> the protected host credential file selected by `PI_AUTH_FILE`; they do not edit files inside
> the running container.
## Credenziali nel backend container
## Credentials in the backend container
In produzione configurare una sola sorgente generica: il bundle `THT_SECRETS_FILE` con la voce
`THT_MODEL_API_KEY` (predefinito della distribuzione unificata), oppure
`THT_MODEL_API_KEY_FILE` come secret file assoluto. Non mettere il valore della chiave in `.env`.
`PiProcessManager` rilegge e valida la sorgente per ogni processo, normalizza il provider
selezionato e passa al solo child Pi la variabile nativa appropriata
(`ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, `GEMINI_API_KEY`, `ZAI_API_KEY`, ecc.). Il percorso generico,
le chiavi di provider non selezionati e il vecchio `PI_PROVIDER_API_KEY` vengono rimossi dall'ambiente
del child. Per un provider custom, il backend deriva il nome della variabile dal campo dichiarativo
`apiKey` di `models.json`; un valore letterale indica che il catalogo è autosufficiente. Non esistono
eccezioni per nomi di provider compilate nel codice. Un secret mancante o non sicuro fallisce prima
dello spawn con errore sanitizzato.
In production, configure one generic source: the `THT_SECRETS_FILE` bundle with the
`THT_MODEL_API_KEY` entry, which is the default for the unified distribution, or
`THT_MODEL_API_KEY_FILE` as an absolute secret-file path. Do not put the key value in `.env`.
For each process, `PiProcessManager` rereads and validates the source, normalizes the selected
provider, and passes only the appropriate native variable to the Pi child
(`ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, `GEMINI_API_KEY`, `ZAI_API_KEY`, and so on). It removes the
generic path, keys for unselected providers, and the old `PI_PROVIDER_API_KEY` from the child
environment. For a custom provider, the backend derives the variable name from the declarative
`apiKey` field in `models.json`; a literal value means the catalog is self-contained. Provider-name
exceptions are not compiled into the code. A missing or insecure secret fails before spawn with a
sanitized error.
La sorgente generica supporta soltanto provider con una singola chiave: `ant-ling`, `anthropic`,
`cerebras`, `deepseek`, `fireworks`, `github-copilot`, `google` (anche tramite alias `gemini`),
`google-vertex` in modalità API key, `groq`, `huggingface`, `kimi-coding`, `minimax`, `minimax-cn`,
The generic source supports only single-key providers: `ant-ling`, `anthropic`,
`cerebras`, `deepseek`, `fireworks`, `github-copilot`, `google` (including the `gemini` alias),
`google-vertex` in API-key mode, `groq`, `huggingface`, `kimi-coding`, `minimax`, `minimax-cn`,
`mistral`, `moonshotai`, `moonshotai-cn`, `nvidia`, `openai`, `opencode`, `opencode-go`,
`openrouter`, `together`, `vercel-ai-gateway`, `xai`, i quattro provider `xiaomi*`, `zai` e
`openrouter`, `together`, `vercel-ai-gateway`, `xai`, the four `xiaomi*` providers, `zai`, and
`zai-coding-cn`.
I provider composti `amazon-bedrock`, `azure-openai-responses`, `cloudflare-workers-ai` e
`cloudflare-ai-gateway` non sono rappresentabili da un solo file. La selezione fallisce prima
dello spawn (anche durante l'elenco modelli); tutte le credenziali ambientali AWS, Azure e
Cloudflare restano comunque rimosse. Servirà una futura configurazione dedicata per provider per
supportare questi bundle senza ambiguità.
The composite providers `amazon-bedrock`, `azure-openai-responses`, `cloudflare-workers-ai`, and
`cloudflare-ai-gateway` cannot be represented by one file. Selection fails before spawn, including
when models are listed. AWS, Azure, and Cloudflare environment credentials are still removed. A
future provider-specific configuration will be needed to support these bundles unambiguously.
## Le fonti di un modello
## Model sources
### 1. Built-in (compilato dentro Pi)
### 1. Built-in (compiled into Pi)
Pi viene distribuito con un elenco di modelli già noti (`models.generated.js` dentro il pacchetto `@earendil-works/pi-ai`): Anthropic, OpenAI, Google, e anche provider di terze parti con API pubblica ben nota come DeepSeek. Per questi **non serve alcuna configurazione**: bastano le credenziali (env var o `pi auth`).
Pi ships with a list of known models (`models.generated.js` in the `@earendil-works/pi-ai` package):
Anthropic, OpenAI, Google, and third-party providers with well-known public APIs such as DeepSeek.
These require **no configuration** beyond credentials, provided through an environment variable or
`pi auth`.
`deepseek/deepseek-v4-pro` è così: è nella build di Pi perché `api.deepseek.com` è un'API pubblica documentata, non un endpoint interno.
`deepseek/deepseek-v4-pro` is one of these models. It is included in Pi because `api.deepseek.com`
is a documented public API, not an internal endpoint.
### 2. `models.json` a livello utente (`~/.pi/agent/models.json`)
### 2. User-level `models.json` (`~/.pi/agent/models.json`)
Per un endpoint **OpenAI-compatible** che non è tra i built-in — ma che non richiede nessuna logica di trasporto speciale — basta *dichiararlo*: baseUrl, apiKey, lista modelli. Questo file esiste **solo a livello utente**: non c'è un equivalente project-level (un `./.pi/models.json` non viene letto).
For an **OpenAI-compatible** endpoint that is not built in but needs no special transport logic,
declare it with its baseUrl, apiKey, and model list. This file exists **only at user level**; there is
no project-level equivalent, and `./.pi/models.json` is not read.
Nelle installazioni gestite da ThothII il file deve essere interamente dichiarativo. ThothII
rifiuta ricorsivamente qualsiasi valore JSON che inizi con `!`, anche dentro `headers`, `models`,
`modelOverrides`, `compat`, array o campi non ancora conosciuti. Pi 0.80.3 tratterebbe quel prefisso
come un comando shell al momento della richiesta; questa forma non è ammessa dall'elenco gestito
dei modelli. L'errore restituito è fisso e non include comando, percorso o
secret.
In ThothII-managed installations, the file must be entirely declarative. ThothII recursively
rejects any JSON value that starts with `!`, including values inside `headers`, `models`,
`modelOverrides`, `compat`, arrays, or fields it does not yet know. Pi 0.80.3 would treat that
prefix as a shell command when making a request, so managed model catalogs do not allow it. The
returned error is fixed and contains no command, path, or secret.
Per i secret usare un riferimento ambiente come `"$ZAI_API_KEY"` o `"${ZAI_API_KEY}"`. Il backend
può popolare la variabile nativa del solo provider selezionato leggendo
`THT_MODEL_API_KEY_FILE`, oppure dal bundle `THT_SECRETS_FILE` (`THT_MODEL_API_KEY`); in alternativa
le credenziali possono arrivare dal file protetto montato con `PI_AUTH_FILE`, omettendo `apiKey` da
`models.json`. Non esiste una sintassi di riferimento diretto a un secret file dentro
`models.json`: con le sorgenti `THT_MODEL_*` il file viene letto da ThothII e trasformato nella
variabile ambiente del child Pi; `PI_AUTH_FILE` viene invece montato come archivio credenziali Pi
protetto. Per un punto esclamativo letterale iniziale, la sintassi Pi dichiarativa è `$!`, non `!`.
For secrets, use an environment reference such as `"$ZAI_API_KEY"` or `"${ZAI_API_KEY}"`. The
backend can populate the native variable for the selected provider by reading
`THT_MODEL_API_KEY_FILE`, or from the `THT_SECRETS_FILE` bundle (`THT_MODEL_API_KEY`). Credentials
can also come from the protected file mounted through `PI_AUTH_FILE`, with `apiKey` omitted from
`models.json`. There is no direct secret-file reference syntax in `models.json`. With `THT_MODEL_*`
sources, ThothII reads the file and turns it into the Pi child's environment variable; `PI_AUTH_FILE`
is mounted instead as a protected Pi credential store. For a literal leading exclamation mark, Pi's
declarative syntax is `$!`, not `!`.
Esempio reale in uso su questa macchina — GLM (provider `zai`):
Real example in use on this machine: GLM (provider `zai`):
```json
{
@@ -90,113 +94,113 @@ Esempio reale in uso su questa macchina — GLM (provider `zai`):
}
```
Essendo a livello utente, GLM è visibile da **qualsiasi progetto**.
Because it is user-level, GLM is visible to **every project**.
### Provider da estensione: non ammessi in ThothII
### Extension providers: not allowed in ThothII
Pi supporta tecnicamente provider registrati da estensioni JavaScript, ma ThothII non usa questa
possibilità. Le estensioni di progetto sono riservate al workflow e ai gate; non devono contenere
`registerProvider(...)`. Un endpoint che non può essere descritto dal catalogo OpenAI-compatible
non è un provider supportato da questa installazione finché il contratto dichiarativo non viene
esteso in modo generico.
Pi technically supports providers registered by JavaScript extensions, but ThothII does not use
that capability. Project extensions are reserved for the workflow and gates; they must not contain
`registerProvider(...)`. An endpoint that cannot be described by the OpenAI-compatible catalog is
not supported by this installation until the declarative contract is extended generically.
## Tabella riassuntiva (stato attuale di questa macchina)
## Summary table (current machine state)
| Modello | Livello | Perché | Visibilità |
| Model | Level | Why | Visibility |
|---|---|---|---|
| `deepseek/deepseek-v4-pro` | Built-in Pi | API pubblica nota, già nella build | Tutti i progetti |
| `zai/glm-5.3` | `deploy/pi/models.json` | Endpoint OpenAI-compatible custom | Installazione ThothII |
| `local-qwen/qwen3.6-35b-a3b` | `deploy/pi/models.json` | Endpoint OpenAI-compatible configurato localmente | Installazione ThothII |
| `deepseek/deepseek-v4-pro` | Built-in Pi | Known public API, already in the build | All projects |
| `zai/glm-5.3` | `deploy/pi/models.json` | Custom OpenAI-compatible endpoint | ThothII installation |
| `local-qwen/qwen3.6-35b-a3b` | `deploy/pi/models.json` | Locally configured OpenAI-compatible endpoint | ThothII installation |
## Come scegliere il livello giusto per un nuovo modello
## Choosing the right level for a new model
1. **L'endpoint è un'API pubblica già nota a Pi?** → abilita l'identificatore esatto in `deploy/pi/settings.json`.
2. **È OpenAI-compatible ma non built-in?** → dichiaralo in `deploy/pi/models.json`, poi abilitalo in `deploy/pi/settings.json`.
3. **Richiede codice di trasporto specifico del provider?** → non aggiungere un'estensione specifica; il provider non è supportato finché manca una capacità dichiarativa generica.
1. **Is the endpoint a public API already known to Pi?** Enable the exact identifier in `deploy/pi/settings.json`.
2. **Is it OpenAI-compatible but not built in?** Declare it in `deploy/pi/models.json`, then enable it in `deploy/pi/settings.json`.
3. **Does it require provider-specific transport code?** Do not add a provider-specific extension. The provider is unsupported until a generic declarative capability exists.
---
## Ambito utente vs. progetto — riepilogo generale
## User versus project scope: general summary
Oltre ai modelli, Pi carica altre risorse da due alberi paralleli: `~/.pi/agent/` (utente) e `<cwd>/.pi/` (progetto, risolto in base alla directory da cui viene lanciato `pi`).
In addition to models, Pi loads other resources from two parallel trees: `~/.pi/agent/` (user) and
`<cwd>/.pi/` (project, resolved from the directory where `pi` is launched).
| File/Directory | Livello utente | Livello progetto | Auto-discovery | Precedenza |
|---|---|---|---|---|
| `models.json` | `~/.pi/agent/models.json` | non supportato | no | solo utente |
| `settings.json` | `~/.pi/agent/settings.json` | `./.pi/settings.json` | no | progetto sovrascrive utente |
| `extensions/` | `~/.pi/agent/extensions/` | `./.pi/extensions/` | sì (`.ts`/`.js`) | uniti (progetto + utente) |
| `prompts/` | `~/.pi/agent/prompts/` | `./.pi/prompts/` | sì (`.md`) | uniti |
| `themes/` | `~/.pi/agent/themes/` | `./.pi/themes/` | sì (`.json`) | progetto preferito |
| `skills/` | `~/.pi/agent/skills/` | `./.pi/skills/` | sì (`.md`) | uniti |
| `models.json` | `~/.pi/agent/models.json` | not supported | no | user only |
| `settings.json` | `~/.pi/agent/settings.json` | `./.pi/settings.json` | no | project overrides user |
| `extensions/` | `~/.pi/agent/extensions/` | `./.pi/extensions/` | yes (`.ts`/`.js`) | merged (project + user) |
| `prompts/` | `~/.pi/agent/prompts/` | `./.pi/prompts/` | yes (`.md`) | merged |
| `themes/` | `~/.pi/agent/themes/` | `./.pi/themes/` | yes (`.json`) | project preferred |
| `skills/` | `~/.pi/agent/skills/` | `./.pi/skills/` | yes (`.md`) | merged |
### Esempio reale: ThothII
### Real example: ThothII
```
harness/.pi/
├── extensions/
│ ├── tht-gate.js ← gate human-in-the-loop
│ ├── tht-gate.js # human-in-the-loop gate
│ └── gate/
│ ├── core/ ← enforcement e utility condivise del gate
│ ├── disambiguation/ ← policy F1/F3
│ └── memory/ ← policy F2/F8
├── settings.json ← (opzionale) override delle impostazioni utente
│ ├── core/ # shared gate enforcement and utilities
│ ├── disambiguation/ # F1/F3 policy
│ └── memory/ # F2/F8 policy
├── settings.json # optional override of user settings
└── themes/
└── thothii-mono.json ← tema del progetto
└── thothii-mono.json # project theme
```
### Comportamento rispetto alla cwd
### Behavior relative to cwd
La directory da cui lanci `pi` determina quali estensioni del workflow vengono trovate, ma non
quali modelli ThothII rende disponibili: il catalogo è montato nel Pi agent directory del container.
The directory from which you launch `pi` determines which workflow extensions are found, but not
which models ThothII makes available. The catalog is mounted in the container's Pi agent directory.
```bash
# Da harness/ — carica il gate di progetto e il catalogo locale montato
# From harness/: load the project gate and mounted local catalog
cd /path/to/ThothII/harness
pi --model local-qwen/qwen3.6-35b-a3b "..."
```
Il backend di ThothII lancia sempre Pi con `cwd: harnessDir` per il workflow; la disponibilità del
modello continua a dipendere soltanto da `models.json`, `settings.json` e dalle credenziali locali.
The ThothII backend always launches Pi with `cwd: harnessDir` for the workflow. Model availability
continues to depend only on `models.json`, `settings.json`, and local credentials.
---
## Trappola nell'auto-discovery: `.mjs` viene ignorato
## Auto-discovery trap: `.mjs` is ignored
Il pattern di auto-discovery delle estensioni in Pi è **`/\.(ts|js)$/`** — non include `.mjs`.
Pi's extension auto-discovery pattern is **`/\.(ts|js)$/`**; it does not include `.mjs`.
```
.pi/extensions/
├── my-extension.js ✅ auto-caricata
├── my-extension.ts ✅ auto-caricata
├── my-extension.mjs ❌ ignorata silenziosamente (non combacia col pattern)
└── shared-module.mjs ✅ va bene per moduli helper (deliberatamente non caricato come estensione)
├── my-extension.js ✅ auto-loaded
├── my-extension.ts ✅ auto-loaded
├── my-extension.mjs ❌ silently ignored (does not match the pattern)
└── shared-module.mjs ✅ suitable for helper modules (deliberately not loaded as an extension)
```
Se serve un modulo condiviso importato da un'estensione, usa `.mjs` proprio per evitare che venga trattato come estensione a sé.
If an extension imports a shared module, use `.mjs` so Pi does not treat it as an extension on its own.
---
## Verifica
### Elenco modelli
### List models
```bash
pi --list-models
```
Mostra i built-in e i modelli dichiarati in `models.json`.
This shows built-in models and models declared in `models.json`.
### Verifica il catalogo usato dall'applicazione
### Check the catalog used by the application
```bash
cd harness # o la cwd rilevante per il progetto
cd harness # or the project's relevant cwd
pi --mode rpc
# poi: {"type": "get_available_models", "id": "1"}
```
La risposta RPC deve includere soltanto modelli built-in o dichiarati nel catalogo locale.
The RPC response must include only built-in models or models declared in the local catalog.
---
## Troubleshooting
| Problema | Causa | Soluzione |
| Problem | Cause | Solution |
|---|---|---|
| `Model "X/Y" not found` | Provider/modello assente da `models.json` oppure identificatore assente da `enabledModels` | Correggi i due file locali e ricarica Pi |
| Impostazioni di progetto non applicate | `settings.json` di progetto ha errori di sintassi, o si sta lanciando `pi` dalla cwd sbagliata | Valida il JSON, controlla la cwd |
| `Model "X/Y" not found` | Provider/model missing from `models.json` or identifier missing from `enabledModels` | Fix the two local files and reload Pi |
| Project settings not applied | Project `settings.json` has a syntax error, or `pi` is launched from the wrong cwd | Validate the JSON and check the cwd |
+126 -127
View File
@@ -1,10 +1,10 @@
# Gestione delle memory
# Memory management
Questo documento descrive l'organizzazione attuale delle memory nel workflow ThothII: modello concettuale, ciclo di vita, persistenza, ricerca semantica, gate di revisione, visualizzazione delle sessioni e principali limiti tecnici.
This document describes how Memory currently works in ThothII: its conceptual model, lifecycle, persistence, semantic search, review gates, session display, and main technical limits.
## Sintesi architetturale
## Architectural summary
Una memory è conoscenza di dominio riutilizzabile tra domande. Non è una copia dello schema-linking di una singola domanda.
A Memory item is domain knowledge that can be reused across questions. It is not a copy of one question's schema linking.
```mermaid
flowchart TB
@@ -17,49 +17,49 @@ flowchart TB
```
```text
F1: chiarimento di un concetto
F1: clarify a concept
│
▼
decisione concept_clarified nel ledger della sessione
concept_clarified decision in the session ledger
│
▼
F8: il reviewer decide se promuoverla
F8: reviewer decides whether to promote it
│
├── registro globale registry.jsonl
└── indice semantico Qdrant
├── global registry registry.jsonl
└── Qdrant semantic index
│
▼
F2 di una sessione futura
ricerca e proposta al reviewer
F2 in a future session
search and proposal to the reviewer
```
L'invariante principale è `REUSABLE_TYPES = {"concept_clarified"}`: le sole memory generabili, salvabili, ricercabili e proponibili sono i concetti chiariti. Le decisioni `table_promoted`, `table_excluded`, `column_promoted` e analoghe restano decisioni locali alla domanda.
The main invariant is `REUSABLE_TYPES = {"concept_clarified"}`: only clarified concepts can be generated, saved, searched, or proposed as Memory. Decisions such as `table_promoted`, `table_excluded`, and `column_promoted` remain local to the question.
## I tre livelli della gestione
## The three management levels
| Livello | Contenuto | Funzione |
| Level | Content | Function |
| --- | --- | --- |
| Ledger della sessione | `concept_clarified`, `memory_promoted`, `memory_promotion_declined` | Audit e stato della singola sessione |
| Registro globale | Record `mem-XXXX` in `registry.jsonl` | Archivio canonico attuale delle memory |
| Indice Qdrant | Embedding e metadati derivati dal registro | Ricerca semantica |
| Session ledger | `concept_clarified`, `memory_promoted`, `memory_promotion_declined` | Audit and state for one session |
| Global registry | `mem-XXXX` records in `registry.jsonl` | Current canonical Memory archive |
| Qdrant index | Embeddings and metadata derived from the registry | Semantic search |
Il ledger contiene la provenienza e le decisioni umane. Il record globale contiene il testo riutilizzabile. L'indice vettoriale è una proiezione per la ricerca, non il posto in cui il workflow registra direttamente le decisioni.
The ledger contains provenance and human decisions. The global record contains reusable text. The vector index is a search projection, not the place where the workflow records decisions directly.
## Che cosa può diventare una memory
## What can become Memory
Durante F1 il workflow registra i chiarimenti come decisioni `concept_clarified`. Un chiarimento può esprimere:
During F1, the workflow records clarifications as `concept_clarified` decisions. A clarification can express:
- definizioni di concetti produttivi o organizzativi;
- criteri di inclusione ed esclusione di una linea di prodotto;
- formule e metodi di calcolo;
- interpretazioni temporali;
- mapping verso tabelle e colonne specifiche;
- significato di flag, codici o indicatori.
- definitions of production or organizational concepts;
- inclusion and exclusion criteria for a product line;
- formulas and calculation methods;
- interpretations of time periods;
- mappings to specific tables and columns;
- the meaning of flags, codes, or indicators.
Il modello `MemoryRecord` contiene:
The `MemoryRecord` model contains:
- `id`, ad esempio `mem-0001`;
- timestamp, sessione e sequenza della decisione originale;
- `id`, such as `mem-0001`;
- the timestamp, session, and sequence of the original decision;
- `type`;
- `subject`;
- `detail`;
@@ -67,179 +67,178 @@ Il modello `MemoryRecord` contiene:
- `question_context`;
- `tables` e `concepts`.
Per le nuove memory il tipo è sempre `concept_clarified` e `tables` viene inizializzato vuoto. Una tabella o una colonna può essere citata dentro la spiegazione come mapping tecnico; non può essere il concetto autonomo della memory.
For new Memory items, the type is always `concept_clarified` and `tables` starts empty. A table or column may appear in the explanation as a technical mapping, but it cannot be the Memory item's standalone concept.
Esempio valido:
Valid example:
> Per ablazione si intende una procedura con `ablazione_transcatetere = TRUE`, conteggiata con `COUNT(DISTINCT cod_paz)` per anno.
> Ablation means a procedure with `ablazione_transcatetere = TRUE`, counted with `COUNT(DISTINCT cod_paz)` by year.
Esempi non validi:
Invalid examples:
- `fact_cardioversione` come memory approvata;
- `dim_time` come memory rifiutata;
- una decisione “includi questa tabella” salvata per domande future.
- `fact_cardioversione` as approved Memory;
- `dim_time` as rejected Memory;
- an "include this table" decision saved for future questions.
La regola è documentata anche nella skill del workflow, in [harness/.pi/skills/tht-sessione/SKILL.md](../harness/.pi/skills/tht-sessione/SKILL.md:217).
The workflow skill also documents this rule in [harness/.pi/skills/tht-sessione/SKILL.md](../harness/.pi/skills/tht-sessione/SKILL.md:217).
## Promozione alla fine della sessione: F8
## Promotion at the end of the session: F8
Alla fine del workflow il gate `reviewer_memory_promote` esegue una preview deterministica:
At the end of the workflow, the `reviewer_memory_promote` gate runs a deterministic preview:
```text
tht memory promote --session <id> --preview --json
```
La preview:
The preview:
1. legge le decisioni effettive della sessione;
2. considera solo `concept_clarified`;
3. scarta le decisioni già promosse;
4. scarta le sequenze già rifiutate in F8;
5. deduplica contenuti equivalenti;
6. propone al massimo cinque candidati.
1. reads the session's effective decisions;
2. considers only `concept_clarified`;
3. discards decisions already promoted;
4. discards sequences already declined in F8;
5. deduplicates equivalent content;
6. proposes at most five candidates.
Il codice applica il filtro e la deduplica in
[harness/tht/memory/core.py](../harness/tht/memory/core.py); il gate applica un
ulteriore filtro difensivo in
The code applies filtering and deduplication in
[harness/tht/memory/core.py](../harness/tht/memory/core.py); the gate applies an additional defensive filter in
[harness/.pi/extensions/gate/memory/index.js](../harness/.pi/extensions/gate/memory/index.js).
Il reviewer vede un'unica checklist, preselezionata. Per ogni candidato:
The reviewer sees one preselected checklist. For each candidate:
- selezionato: viene eseguito `tht memory save-one` e poi viene registrato `memory_promoted`;
- deselezionato: viene registrato `memory_promotion_declined`;
- nessun candidato: F8 si chiude automaticamente.
- selected: `tht memory save-one` runs, followed by a `memory_promoted` record;
- deselected: records `memory_promotion_declined`;
- no candidates: F8 closes automatically.
Il marker `memory_promoted` usa `detail: seq:N`, cioè un riferimento alla decisione `concept_clarified` originale. Il flusso è in [harness/.pi/extensions/tht-gate.js](../harness/.pi/extensions/tht-gate.js:1691).
The `memory_promoted` marker uses `detail: seq:N`, a reference to the original `concept_clarified` decision. The flow is in [harness/.pi/extensions/tht-gate.js](../harness/.pi/extensions/tht-gate.js:1691).
La promozione non viene eseguita in F2 e il modello non può inventare candidati F8. I comandi diretti di promozione sono inoltre protetti dal gate anti-bypass.
Promotion does not run in F2, and the model cannot invent F8 candidates. Direct promotion commands are also protected by the anti-bypass gate.
## Persistenza globale
## Global persistence
### Registro JSONL
### JSONL registry
Il registro attuale è:
The current registry is:
```text
<artifacts>/memory/registry.jsonl
```
La scrittura viene fatta tramite file temporaneo e `os.replace`, quindi la sostituzione del registro è atomica. L'idempotenza della promozione è basata sulla coppia `session_id + decision_seq`: la stessa decisione della stessa sessione non genera due record globali.
The registry is written through a temporary file and `os.replace`, so replacement is atomic. Promotion is idempotent on the `session_id + decision_seq` pair: the same decision from the same session cannot create two global records.
### Qdrant
Dopo la promozione, `save-one` costruisce un solo `VectorRecord` e lo invia all'indice Qdrant. Il testo indicizzato include:
After promotion, `save-one` builds one `VectorRecord` and sends it to the Qdrant index. The indexed text includes:
- tipo e soggetto;
- dettaglio;
- motivazione;
- domanda di contesto;
- eventuali concetti e mapping.
- type and subject;
- detail;
- rationale;
- question context;
- any concepts and mappings.
Il record vettoriale usa l'id `memory:mem-XXXX`, mentre i metadati conservano `subject`, `detail`, `rationale`, `tables`, `concepts` e il discriminante `kind`. L'hash SHA-256 del contenuto impedisce di ricalcolare embedding e upsert quando il testo non è cambiato.
The vector record uses the ID `memory:mem-XXXX`. Its metadata stores `subject`, `detail`, `rationale`, `tables`, `concepts`, and the `kind` discriminator. The content's SHA-256 hash prevents embedding and upsert work when the text has not changed.
Il comportamento è implementato in
This behavior is implemented in
[harness/tht/memory/core.py](../harness/tht/memory/core.py).
### Fonte canonica attuale
### Current canonical source
Oggi il registro JSONL è ancora la fonte canonica applicativa e Qdrant resta un indice derivato ma persistente. Il workflow non registra direttamente le decisioni nel vector DB: usa Qdrant come proiezione interrogabile del registro e del ledger effettivo.
The JSONL registry remains the application's canonical source, while Qdrant is a derived but persistent index. The workflow does not record decisions directly in the vector database. It uses Qdrant as a searchable projection of the registry and effective ledger.
## Riutilizzo in F2
## Reuse in F2
In una sessione futura F2 esegue:
In a future session, F2 runs:
```text
tht memory search "<domanda>" --session <id> --json
tht memory search "<question>" --session <id> --json
```
Il comando:
The command:
1. crea l'embedding della domanda;
2. cerca nel vector store solo record `kind=memory`;
3. risolve ogni hit nel registro JSONL tramite il suo `ref`;
4. scarta record assenti dal registro;
5. scarta qualsiasi tipo diverso da `concept_clarified`;
6. esclude le memory già decise nella sessione corrente;
7. restituisce i risultati ordinati per similarità.
1. creates an embedding for the question;
2. searches the vector store for records with `kind=memory` only;
3. resolves each hit in the JSONL registry through its `ref`;
4. discards records missing from the registry;
5. discards every type other than `concept_clarified`;
6. excludes Memory already decided in the current session;
7. returns results ordered by similarity.
Le memory non vengono mai applicate automaticamente. Il modello deve presentarle in un'unica scelta `reviewer_decide`:
Memory is never applied automatically. The model must present it in one `reviewer_decide` choice:
- una memory selezionata viene registrata come nuovo `concept_clarified` nella sessione corrente;
- il rationale deve citare l'id originale `mem-XXXX`;
- una memory deselezionata significa “non applicarla ora”, non “cancellarla globalmente”.
- a selected Memory item is recorded as a new `concept_clarified` in the current session;
- the rationale must cite the original `mem-XXXX` ID;
- a deselected Memory item means "do not apply it now", not "delete it globally".
Se F2 viene riaperta, una memory deselezionata può quindi essere proposta di nuovo. `memory_rejected` resta supportato per decisioni e sessioni legacy, ma non rappresenta il normale comportamento della deselezione F2 attuale.
If F2 is reopened, a deselected Memory item can be proposed again. `memory_rejected` remains supported for legacy decisions and sessions, but it is not the normal behavior for current F2 deselection.
## Ledger effettivo, rollback e riaperture
## Effective ledger, rollback, and reopening
Il ledger è append-only. Riaperture e ritrattazioni non cancellano le righe precedenti; cambiano però quali decisioni sono effettive.
The ledger is append-only. Reopening and withdrawing decisions do not delete earlier rows, but they change which decisions are effective.
Gli helper memory usano `effective_decisions()` per:
Memory helpers use `effective_decisions()` to:
- escludere decisioni ritirate;
- ignorare decisioni appartenenti a fasi diventate stale dopo un rollback;
- impedire la promozione di chiarimenti non più validi.
- exclude withdrawn decisions;
- ignore decisions from phases that became stale after a rollback;
- prevent promotion of clarifications that are no longer valid.
La vista effettiva è definita in [harness/tht/phase.py](../harness/tht/phase.py:82).
The effective view is defined in [harness/tht/phase.py](../harness/tht/phase.py:82).
## Visualizzazione nel riepilogo della sessione
## Display in the session summary
La sezione “Memories” del riepilogo viene proiettata a runtime dal ledger della sessione; non è una copia diretta del registro globale.
The "Memories" section of the summary is projected at runtime from the session ledger; it is not a direct copy of the global registry.
La proiezione:
The projection:
- mostra prima le memory approved;
- mostra poi le memory declined;
- risolve `seq:N` verso il `concept_clarified` originale;
- nasconde marker il cui record originale non è `concept_clarified`;
- nasconde soggetti che corrispondono a tabelle dello schema-linking;
- nasconde soggetti autonomi con forma `fact_*` o `dim_*`;
- mantiene invece i riferimenti a tabelle e campi quando fanno parte della spiegazione concettuale.
- shows approved Memory first;
- then shows declined Memory;
- resolves `seq:N` to the original `concept_clarified`;
- hides markers whose original record is not `concept_clarified`;
- hides subjects that match schema-linking tables;
- hides standalone subjects shaped like `fact_*` or `dim_*`;
- keeps table and field references when they are part of the conceptual explanation.
La logica è in [harness/tht/session/store.py](../harness/tht/session/store.py:237). Il rendering frontend usa una lista strutturata e tratta `subject`, `detail` e `rationale` come Markdown, evitando di mostrare il markdown grezzo.
The logic is in [harness/tht/session/store.py](../harness/tht/session/store.py:237). The frontend renders a structured list and treats `subject`, `detail`, and `rationale` as Markdown instead of showing raw Markdown.
La trasformazione avviene in lettura: anche le sessioni storiche vengono organizzate con il layout e i filtri correnti senza riscrivere gli artefatti originali.
The transformation happens on read. Historical sessions use the current layout and filters without rewriting their original artifacts.
## Invarianti applicate
## Enforced invariants
Le protezioni sono distribuite su più confini:
The protections are distributed across several boundaries:
1. `REUSABLE_TYPES` nel core Python;
2. filtro del comando `memory search`;
3. filtro della preview F8;
4. filtro e deduplica nel gate Pi;
5. esclusione delle table-memory nella proiezione UI.
1. `REUSABLE_TYPES` in the Python core;
2. the `memory search` command filter;
3. the F8 preview filter;
4. filtering and deduplication in the Pi gate;
5. exclusion of table Memory from the UI projection.
Questo evita che una singola modifica al prompt o a un solo componente reintroduca le tabelle come memory.
This prevents one prompt or component change from reintroducing tables as Memory.
## Limiti e rischi residui
## Remaining limits and risks
### Registro e indice non sono una singola transazione
### The registry and index are not one transaction
Il salvataggio segue sostanzialmente questa sequenza:
Saving broadly follows this sequence:
```text
registro JSONL → Qdrant → marker memory_promoted nel ledger
JSONL registry → Qdrant → memory_promoted marker in the ledger
```
Se Qdrant non è disponibile, il registro può contenere una memory non ancora ricercabile; il comando segnala che sarà necessario reindicizzare.
If Qdrant is unavailable, the registry can contain Memory that is not yet searchable. The command reports that reindexing is required.
Se il marker del ledger fallisce dopo il salvataggio nel vector DB, la memory può risultare globalmente presente ma senza audit completo nella sessione. Il gate restituisce un comando di recupero manuale.
If the ledger marker fails after the vector database save, the Memory item can exist globally without a complete session audit. The gate returns a manual recovery command.
### Limite di cinque candidati
### Five-candidate limit
F8 propone al massimo cinque memory. Se una sessione produce più di cinque concetti validi, gli elementi eccedenti non vengono mostrati e la sessione può essere finalizzata senza promuoverli.
F8 proposes at most five Memory items. If a session produces more than five valid concepts, the extra items are not shown and the session can be finalized without promoting them.
### Deduplica non globale
### Deduplication is not global
La deduplica impedisce duplicati nella stessa proposta e l'idempotenza impedisce di ripromuovere la stessa decisione. Non esiste però una fusione globale di due memory semanticamente simili provenienti da sessioni diverse.
Deduplication prevents duplicates within one proposal, and idempotency prevents the same decision from being promoted twice. There is no global merge of semantically similar Memory items from different sessions.
### Vecchi record fisici
### Old physical records
Vecchi record `table_promoted` o `table_excluded` possono ancora esistere in artefatti o indici storici. Il codice attuale li rende non riutilizzabili filtrandoli per tipo e non li mostra nella proiezione delle sessioni. La loro eventuale rimozione fisica dal vector DB resta un'attività di bonifica separata.
Old `table_promoted` or `table_excluded` records may still exist in historical artifacts or indexes. The current code makes them unusable by filtering by type and does not show them in session projections. Physically removing them from the vector database remains a separate cleanup task.
## Valutazione finale
## Final assessment
La gestione attuale è coerente con il requisito funzionale: una memory è una conoscenza concettuale riutilizzabile, non una scelta di schema-linking.
The current implementation matches the functional requirement: Memory is reusable conceptual knowledge, not a schema-linking choice.
La parte più solida è la difesa multilivello del tipo `concept_clarified`. Il principale debito tecnico riguarda invece la convivenza del registro JSONL con Qdrant e l'assenza di una transazione unica tra archivio globale, indice semantico e ledger della sessione.
The strongest part is the multilayer protection of the `concept_clarified` type. The main technical debt is the coexistence of the JSONL registry and Qdrant, with no single transaction spanning the global archive, semantic index, and session ledger.
+136 -136
View File
@@ -1,64 +1,64 @@
# ThothII — Guida utente
# ThothII user guide
Per login, **Remember me**, ruoli, invalidazione delle sessioni, gruppi OIDC e ripristino, vedere
la [guida autenticazione locale](install/authentication-local.md) e la [guida OIDC generica](install/authentication-oidc.md).
For login, **Remember me**, roles, session invalidation, OIDC groups, and recovery, see the
[local authentication guide](install/authentication-local.md) and the [generic OIDC guide](install/authentication-oidc.md).
Questa guida accompagna passo-passo chi deve **preparare** il repository dei workspace, **usare
gli strumenti** ThothII per quel repository e **usare l'applicazione** per fare domande in
linguaggio naturale e ottenere SQL validato. Usa parole semplici ed esempi; i dettagli tecnici
restano nei contratti citati in fondo.
This guide walks you through **preparing** the workspace repository, **using ThothII's tools** for
that repository, and **using the application** to ask natural-language questions and obtain
validated SQL. It uses plain language and examples. The contracts listed at the end contain the
technical details.
> **Che cos'è ThothII.** È un *datamart builder* con revisione umana: tu scrivi una domanda in
> linguaggio naturale, un modello propone via via i passaggi (chiarimenti, schema, CTE, SQL) e un
> **revisore umano decide** a ogni passaggio chiave. Il risultato finale è SQL validato pronto da
> eseguire sul data warehouse.
> **What is ThothII?** It is a *datamart builder* with human review. You write a natural-language
> question, the model proposes each step in turn (clarifications, schema, CTEs, and SQL), and a
> **human reviewer decides** at every important step. The final result is validated SQL ready to
> run on the data warehouse.
---
## Parte 1 — Preparare il repository dei workspace su Git
## Part 1: prepare the workspace repository on Git
### 1.1 La struttura
### 1.1 Structure
Il repository dei workspace è un **repository Git** che descrive *quali dati* sono disponibili e
*come raggiungerli*. Non contiene i dati e **non contiene segreti** (password, token, certificati).
The workspace repository is a **Git repository** that describes *which data* is available and
*how to reach it*. It contains neither the data nor **secrets** such as passwords, tokens, or certificates.
Un repository valido contiene:
A valid repository contains:
```text
thoth-workspaces.yaml ← catalogo: elenco dei workspace
<id-workspace>/workspace.yaml ← descrittore del workspace (schema v3)
<id-workspace>/evidence/ ← (facoltativo) documenti di contesto, es. *.md
<id-workspace>/schema/annotations.yaml ← (facoltativo) join logici curati a mano (P5)
thoth-workspaces.yaml # catalog: list of workspaces
<workspace-id>/workspace.yaml # workspace descriptor (schema v3)
<workspace-id>/evidence/ # optional context documents, such as *.md
<workspace-id>/schema/annotations.yaml # optional manually curated logical joins (P5)
```
- Il **catalogo** `thoth-workspaces.yaml` è un semplice elenco:
- The **catalog** `thoth-workspaces.yaml` is a simple list:
```yaml
schema_version: 1
workspaces:
- id: acme-ebikes
name: ACME Limited
description: DWH della produzione di biciclette elettriche
description: DWH for electric bicycle production
```
- L'**id** deve essere minuscolo, senza spazi, es. `acme-ebikes` (`[a-z][a-z0-9-]{2,62}`).
- Il **descrittore** `<id>/workspace.yaml` è lo schema v3. È l'unica descrizione valida.
- The **ID** must be lowercase, contain no spaces, and follow `acme-ebikes` (`[a-z][a-z0-9-]{2,62}`).
- The **descriptor** `<id>/workspace.yaml` uses schema v3. It is the only valid description.
### 1.2 Esempio di descrittore (ACME Limited)
### 1.2 Descriptor example (ACME Limited)
```yaml
workspace:
schema_version: 3
id: acme-ebikes
name: ACME Limited
description: DWH industriale — produzione di biciclette elettriche
language: it # le descrizioni/evidence sono in italiano
description: Industrial DWH for electric bicycle production
language: en # descriptions and Evidence are in English
dwh:
engine: postgres
database: postgres
schema: datawarehouse
supported_transports: [rest_api] # accesso tramite API REST (PostgREST)
supported_transports: [rest_api] # access through the REST API (PostgREST)
semantic_index:
vector_store:
@@ -84,197 +84,197 @@ diagnostics:
evidence:
source:
type: filesystem
uri: acme-ebikes/evidence # percorso dentro il repository
uri: acme-ebikes/evidence # path inside the repository
policy:
max_chunk_chars: 4000
retain_published_generations: 3
```
Cosa cambia rispetto ai vecchi workspace (se ne avevi uno):
Changes from older workspaces:
- il database si raggiunge solo con **REST** o **Postgres diretto** (`rest_api` /
`postgres_direct`); il tunnel SSH resta disabilitato;
- l'indice semantico è **interno** (Qdrant + `qwen3-embedding:0.6b`, 1024 dimensioni, cosine);
- l'Evidence **filesystem** sta dentro il repository (`<id>/evidence`) e viene materializzata dal
commit Git fissato (P6); è supportata anche l'Evidence HTTP.
- the database is reached only through **REST** or **direct Postgres** (`rest_api` /
`postgres_direct`); the SSH tunnel remains disabled;
- the semantic index is **internal** (Qdrant plus `qwen3-embedding:0.6b`, 1024 dimensions, cosine);
- **filesystem** Evidence lives in the repository (`<id>/evidence`) and is materialized from the
pinned Git commit (P6). HTTP Evidence is also supported.
### 1.3 Regole da rispettare
### 1.3 Rules
1. **Git è la fonte di verità.** Descriptor, catalogo ed Evidence si modificano solo con un
*commit* + *push* e poi un *pull* dell'installazione.
2. **Niente segreti nel repository.** Password, token, chiavi private e URL firmati vengono inseriti
a runtime nella gestione Workspace e conservati cifrati dal backend.
3. **Solo schema v3.** I descrittori v1/v2 vengono rifiutati prima dell'attivazione.
4. **L'applicazione non fa push di contenuti curati.** L'operatore che cura il repository lavora in
un clone autore separato.
1. **Git is the source of truth.** Change the descriptor, catalog, and Evidence only through a
*commit* and *push*, followed by an installation *pull*.
2. **No secrets in the repository.** Add passwords, tokens, private keys, and signed URLs at
runtime through Workspace management; the backend stores them encrypted.
3. **Schema v3 only.** Reject v1 and v2 descriptors before activation.
4. **The application does not push curated content.** The repository curator works in a separate
authoring clone.
---
## Parte 2 — Usare gli strumenti ThothII per il repository
## Part 2: use ThothII's repository tools
Ci sono **due** strumenti: l'**applicazione web** (gestione workspace) e la **CLI `tht`**
(preprocessing/operator). L'installazione completa è descritta nei manuali
There are **two** tools: the **web application** (workspace management) and the **`tht` CLI**
(preprocessing and operations). The complete installation is described in
`docs/install/local-workspace-registry.md` (macOS/Windows/Linux) e
`docs/install/server-workspace-registry.md`.
### 2.1 `tht` — comandi principali
### 2.1 `tht`: main commands
`tht` si invoca sempre con `--installation <percorso>/thothii-installation.yaml`. I comandi
utili, nell'ordine tipico:
Always invoke `tht` with `--installation <path>/thothii-installation.yaml`. The usual commands are:
```bash
# 1) vedere lo stato di un workspace (revisione e identità)
# 1) inspect workspace state (revision and identity)
tht --installation <install> workspace inspect --workspace <id> --json
# 2) introspezione del DWH (genera physical.yaml + LSH)
# 2) inspect the DWH (generates physical.yaml and LSH)
tht --installation <install> workspace preprocess dwh --workspace <id> --json
# 3) suggerire le join (FK) da SQL già approvato
# 3) suggest joins (FKs) from approved SQL
tht --installation <install> workspace schema suggest-fks --workspace <id> --from-sql <query>.sql --output <candidati>.yaml --json
# 4) dopo la revisione: pubblicare gli FK curati in Git e accettarli
# 4) after review, publish curated FKs in Git and accept them
tht --installation <install> workspace schema accept --workspace <id> --run <run-id> --yes --json
# 5) indicizzare lo schema (Qdrant)
# 5) index the schema (Qdrant)
tht --installation <install> workspace index-schema --workspace <id> --json
# 6) preprocessing dell'Evidence
# 6) preprocess Evidence
tht --installation <install> workspace preprocess evidence --workspace <id> --json
# 7) catena completa (DWH → FK → schema → Evidence)
# 7) complete chain (DWH → FK → schema → Evidence)
tht --installation <install> workspace preprocess run --workspace <id> --json
# 8) ispezione/ricostruzione della collection Qdrant (solo manutenzione)
# 8) inspect or rebuild the Qdrant collection (maintenance only)
tht --installation <install> workspace vector inspect --workspace <id> --json
tht --installation <install> workspace vector rebuild --workspace <id> --collection <nome> --confirm <nome> --destroy
```
Note importanti:
Important notes:
- **`--json` produce solo JSON su stdout** (contratto macchina): usalo negli script.
- **`preprocess run` si ferma per la revisione umana** quando ci sono nuove join proposte: esce con
`manual_review_required`. Dopo la revisione si riparte con `schema accept ... --yes` e
- **`--json` writes JSON only to stdout** (machine contract); use it in scripts.
- **`preprocess run` stops for human review** when it finds new proposed joins: it exits with
`manual_review_required`. After review, continue with `schema accept ... --yes` and
`preprocess run --resume <run-id>`.
- **Un file Evidence filesystem viene materializzato dal commit Git fissato** (niente checkout
mobile); symlink, percorsi pericolosi e alberi troppo grandi vengono rifiutati.
- **Il CLI non scrive mai nel repository** (nessun push di contenuti curati).
- **A filesystem Evidence file is materialized from the pinned Git commit** (there is no moving
checkout); symlinks, unsafe paths, and trees that are too large are rejected.
- **The CLI never writes to the repository** (it never pushes curated content).
### 2.2 Applicazione web — gestione workspace
### 2.2 Web application: workspace management
La gestione Workspace ha due livelli distinti.
Workspace management has two distinct levels.
**Livello 1 — repository.** La parte iniziale spiega che il sorgente del workspace vive in una
directory separata, viene pubblicato dal curatore su un repository ospitato da un server Git come
GitHub, GitLab o Gitea, e viene letto da ThothII in sola lettura. Mostra host, repository, branch,
revisione attiva e stato dell'ultimo aggiornamento.
**Level 1: repository.** The first section shows that the workspace source lives in a separate
directory, is published by the curator to a repository hosted on a Git server such as GitHub,
GitLab, or Gitea, and is read by ThothII in read-only mode. It shows the host, repository, branch,
active revision, and status of the last update.
- **Update workspace repository** non richiede la selezione di un workspace. Il backend esegue il
fetch/pull del branch configurato direttamente nel checkout gestito da ThothII, valida l'intera
revisione candidata e la attiva in modo atomico. Se la validazione fallisce, conserva la
revisione precedente. Non modifica il sorgente remoto e non salva contenuti nella GUI.
- Per creare un workspace locale, prepara una directory sorgente con catalogo, `workspace.yaml` e
le sottodirectory previste; quindi validala, esegui commit e push dal clone autore. ThothII non
offre comandi di creazione, modifica o pubblicazione del sorgente.
* **Update workspace repository** does not require a workspace to be selected. The backend fetches
or pulls the configured branch into ThothII's managed checkout, validates the entire candidate
revision, and activates it atomically. If validation fails, it keeps the previous revision. It
does not modify the remote source or save content from the GUI.
* To create a local workspace, prepare a source directory with the catalog, `workspace.yaml`, and
the expected subdirectories. Validate it, then commit and push from the authoring clone.
ThothII provides no commands to create, edit, or publish the source.
**Livello 2 — workspace selezionato.** Questi comandi sono isolati perché richiedono prima la
selezione del workspace.
**Level 2: selected workspace.** These commands are separate because they require a workspace to be selected first.
These commands are separate because they require a workspace to be selected first.
- **Validate workspace source** verifica nuovamente catalogo, descrittore, Evidence e invarianti della
revisione attiva selezionata. Non contatta il DWH e non modifica file.
- **Save entered secrets** sostituisce alla cieca i valori compilati. I campi dipendono dal
trasporto DWH e dall'autenticazione Evidence dichiarati; il backend restituisce solo lo stato
configurato/mancante.
- **Forget stored value** elimina dal vault cifrato il singolo secret indicato. Le sessioni o operazioni future
che lo richiedono restano bloccate finché non viene inserito di nuovo.
Il repository Git remoto e le relative credenziali sono impostazioni di installazione. I secret
runtime DWH/Evidence sono invece persistenti nel vault cifrato del backend e non nel local storage
della GUI. La GUI è soltanto l'interfaccia: dopo l'invio cancella i valori dai campi e non può
rileggerli.
- **Validate workspace source** checks the catalog, descriptor, Evidence, and invariants of the
selected active revision again. It does not contact the DWH or modify files.
- **Save entered secrets** replaces the entered values without displaying them. The fields depend
on the declared DWH transport and Evidence authentication. The backend returns only configured
or missing status.
* **Forget stored value** removes the selected secret from the encrypted vault. Future sessions or
operations that need it remain blocked until it is entered again.
The remote Git repository and its credentials are installation settings. Runtime DWH and Evidence
secrets persist in the backend's encrypted vault, not in the GUI's local storage. The GUI is only
the interface: after submission it clears the field values and cannot read them back.
---
## Parte 3 — Usare l'applicazione ThothII di base
## Part 3: use the ThothII application
### 3.1 Nuova sessione
### 3.1 New session
Apri l'applicazione e usa il modulo **New session**: inserisci solo la **domanda** in linguaggio
naturale (workspace, modello e provider sono impostazioni globali già configurate).
Open the application and use **New session**. Enter only the **natural-language question**;
workspace, model, and provider are already configured as global settings.
Esempio di domanda:
Example question:
> «Elenca le biciclette elettriche completate nell'ultimo anno, con modello, numero di telaio e
> data di completamento.»
> "List the electric bicycles completed in the last year, with model, frame number, and completion
> date."
### 3.2 Il workflow a 8 fasi e i gate
### 3.2 The eight-phase workflow and gates
La domanda attraversa **8 fasi**. Tu vedi i documenti intermedi e decidi nei punti chiave:
The question passes through **eight phases**. You see the intermediate documents and decide at the important points:
1. **F1 chiarimento** — se serve, il modello chiede di togliere ambiguità;
2. **F2 memoria** — recupera le memory riutilizzabili;
3. **F3 riscrittura** — riscrive e approva la domanda;
4. **F4 schema-linking** — propone tabelle e colonne collegate;
5. **F5 sintesi** — riassume lo schema scelto;
6. **F6 CTE** — costruisce i CTE;
7. **F7 SQL finale** — produce `sql_final.sql`;
8. **F8 datamart** — esecuzione/export (dbt, CSV, Excel).
1. **F1 clarification**: the model removes ambiguity when needed;
2. **F2 Memory**: retrieves reusable Memory;
3. **F3 rewriting**: rewrites and approves the question;
4. **F4 schema linking**: proposes related tables and columns;
5. **F5 summary**: summarizes the selected schema;
6. **F6 CTE**: builds the CTEs;
7. **F7 final SQL**: produces `sql_final.sql`;
8. **F8 datamart**: execution or export (dbt, CSV, Excel).
I **gate di revisione** appaiono come widget: scegli un'opzione singola, seleziona più voci, o
conferma un artefatto/fase. Il modello *propone*, il revisore *decide*. Il lato destro mostra gli
artefatti (schema-linking, CTE, SQL); il pannello Model activity mostra domanda/ragionamento.
**Review gates** appear as widgets: choose one option, select several items, or confirm an
artifact or phase. The model *proposes* and the reviewer *decides*. The right side shows artifacts
(schema linking, CTEs, and SQL); the Model activity panel shows the question and reasoning.
### 3.3 Sessioni
### 3.3 Sessions
Le sessioni sono elencate nella barra laterale con id, domanda, data e autore. Una sessione
**riprende** dall'ultima fase incompleta ricostruendo lo stato dai documenti salvati su disco
(`session_manifest.yaml` + artefatti di fase + `review_decisions.jsonl`). Lo stato salvato **è** la
verità: ciò che non è registrato non è avvenuto.
Sessions appear in the sidebar with their ID, question, date, and author. A session
**resumes** from its last incomplete phase by rebuilding state from documents saved on disk
(`session_manifest.yaml`, phase artifacts, and `review_decisions.jsonl`). Saved state **is** the
truth: what is not recorded did not happen.
---
## Esempio pratico completo — ACME Limited
## Complete example: ACME Limited
### Passo 0 — repository
### Step 0: repository
Crea il repository Git del workspace (es. `tht-workspace-acme`):
Create the workspace Git repository, for example `tht-workspace-acme`:
```text
thoth-workspaces.yaml # catalogo con acme-ebikes
acme-ebikes/workspace.yaml # descrittore v3 (vedi §1.2)
acme-ebikes/evidence/ # i documenti .md di contesto curati
acme-ebikes/schema/annotations.yaml # (quando ci sono join curate)
thoth-workspaces.yaml # catalog containing acme-ebikes
acme-ebikes/workspace.yaml # v3 descriptor (see §1.2)
acme-ebikes/evidence/ # curated context .md documents
acme-ebikes/schema/annotations.yaml # when curated joins exist
```
Pubblica una nuova revisione Git. Nell'installazione, l'applicazione acquisisce e **attiva** il workspace:
valida lo schema v3, materializza l'Evidence dalla revisione fissata e prepara la collection Qdrant
(1024/cosine + indici).
Publish a new Git revision. In the installation, the application fetches and **activates** the
workspace, validates schema v3, materializes Evidence from the pinned revision, and prepares the
Qdrant collection (1024/cosine plus indexes).
### Passo 1 — preprocessing
### Step 1: preprocessing
```bash
tht --installation ~/thothii-installation.yaml workspace preprocess dwh --workspace acme-ebikes --json
tht --installation ~/thothii-installation.yaml workspace preprocess run --workspace acme-ebikes --json
```
Se il run si ferma per le join (`manual_review_required`):
If the run stops for joins (`manual_review_required`):
```bash
# il curatore rivede i candidati e pubblica acme-ebikes/schema/annotations.yaml, poi:
# the curator reviews the candidates and publishes acme-ebikes/schema/annotations.yaml, then:
tht --installation ~/thothii-installation.yaml workspace schema accept --workspace acme-ebikes --run RUN_ID --yes --json
tht --installation ~/thothii-installation.yaml workspace preprocess run --workspace acme-ebikes --resume RUN_ID --json
```
### Passo 2 — la domanda
### Step 2: the question
Nell'applicazione seleziona il workspace `acme-ebikes` e crea una sessione con la domanda. Segui
le fasi e conferma ai gate: il modello proporrà lo schema-linking (tabelle/colonne del DWH
`datawarehouse`), i CTE e infine l'SQL finale, che potrai copiare/visualizzare ed eseguire.
In the application, select the `acme-ebikes` workspace and create a session with the question.
Follow the phases and confirm the gates. The model will propose schema linking (tables and columns
from the `datawarehouse` DWH), CTEs, and finally the SQL, which you can view, copy, and run.
---
## Dove trovare i dettagli tecnici
## Where to find technical details
Per l'accesso DWH REST, la chiave è per installazione e vale solo per `rest_api`; `postgres_direct` e `ssh_tunnel` non usano questa chiave. Vedere [guida server DWH](install/dwh-auth-server.md), [enrollment client](install/dwh-auth-client-enrollment.md) e [TLS](install/dwh-auth-tls.md).
For DWH REST access, the key is installation-specific and applies only to `rest_api`; `postgres_direct`
and `ssh_tunnel` do not use it. See the [DWH server guide](install/dwh-auth-server.md), [client
enrollment guide](install/dwh-auth-client-enrollment.md), and [TLS guide](install/dwh-auth-tls.md).
- Contratto CLI: `docs/contracts/workspace-preprocessing-cli.md`
- Contratto `.tht-dwh`: `docs/contracts/tht-dwh.md`
- CLI contract: `docs/contracts/workspace-preprocessing-cli.md`
- `.tht-dwh` contract: `docs/contracts/tht-dwh.md`
- Evidence v3: `docs/contracts/workspace-evidence-v3.md`
+11 -12
View File
@@ -1,21 +1,20 @@
# ThothII — Documentazione
# ThothII documentation
Benvenuto nella documentazione di ThothII, il datamart builder human-in-the-loop che trasforma domande in linguaggio naturale in SQL validato attraverso un workflow a 8 fasi orchestrato da Pi.
ThothII is a human-in-the-loop datamart builder. It turns natural-language questions into validated SQL through an eight-phase workflow orchestrated by Pi.
La documentazione è divisa in due aree:
The documentation is divided into two areas:
## ThothII (Documentazione Tecnica)
## ThothII technical documentation
Come funziona il sistema: architettura, workflow, contratti operativi, Evidence e gestione delle Memory. Parte da qui: [Panoramica dell'architettura](architecture/overview.md).
This section explains the system architecture, workflow, operating contracts, Evidence, and Memory. Start with the [architecture overview](architecture/overview.md).
Per autenticazione locale, OIDC generico e Authentik: [documentazione autenticazione](architecture/authentication.md).
For local authentication, generic OIDC, and Authentik, see the [authentication documentation](architecture/authentication.md).
Per installare l'applicazione in Docker nei quattro contesti operativi, usando il file env,
`compose.yaml`, l'overlay locale/server e il bundle di secret montato:
[Installazione Docker nei quattro contesti](installazione-docker-4-contesti.md).
To install the application in Docker across the four operating contexts, using the env file,
`compose.yaml`, the local/server overlay, and the mounted secret bundle, see [Docker installation in the four operating contexts](installazione-docker-4-contesti.md).
Per il DWH REST con una chiave revocabile per installazione: [guida server](install/dwh-auth-server.md), [enrollment client](install/dwh-auth-client-enrollment.md) e [TLS](install/dwh-auth-tls.md). Il componente resta separato dallo stack Compose ThothII.
For DWH REST with an installation-specific revocable key, see the [server guide](install/dwh-auth-server.md), [client enrollment guide](install/dwh-auth-client-enrollment.md), and [TLS guide](install/dwh-auth-tls.md). This component remains separate from the ThothII Compose stack.
## Considerazioni Generali
## General topics
Note operative e di configurazione che non sono specifiche del dominio ThothII — ad esempio come Pi risolve i modelli a livello integrato, utente e progetto. Parte da qui: [Configurazione dei modelli in Pi](general/pi-configuration.md).
Operating and configuration notes that are not specific to the ThothII domain, such as how Pi resolves built-in, user-level, and project-level models. Start with [Pi model configuration](general/pi-configuration.md).
+18 -19
View File
@@ -1,7 +1,7 @@
# Configurazione del provider Authentik
# Authentik provider configuration
Il protocollo browser di ThothII è OIDC generico. Authentik fornisce il catalogo gruppi e il
provider di identità senza introdurre un percorso di login proprietario.
ThothII uses generic OIDC in the browser. Authentik provides the identity provider and group
catalog without adding a proprietary login flow.
```mermaid
sequenceDiagram
@@ -17,26 +17,25 @@ sequenceDiagram
ThothII-->>Browser: Opaque session
```
## Provider OIDC
## OIDC provider
1. Creare applicazione e provider OAuth2/OIDC.
2. Registrare esattamente `PUBLIC_URL/api/auth/oidc/callback`.
3. Abilitare gli scope `openid`, `profile` ed `email`.
4. Configurare un claim diretto `groups` come array di stringhe.
1. Create an OAuth2/OIDC application and provider.
2. Register exactly `PUBLIC_URL/api/auth/oidc/callback`.
3. Enable the `openid`, `profile`, and `email` scopes.
4. Configure a direct `groups` claim as an array of strings.
## Catalogo gruppi
## Group catalog
Creare un account di servizio dedicato con sola lettura dei gruppi. Conservare il token nel
bundle protetto come `THT_AUTHENTIK_API_TOKEN`.
Create a dedicated service account with read-only access to groups. Store its token in the
protected bundle as `THT_AUTHENTIK_API_TOKEN`.
Mappare in `auth.yaml` i nomi esatti dei gruppi aziendali ai ruoli ThothII `user` e `admin`.
Gruppi non mappati vengono ignorati; un gruppo configurato ma assente genera un errore chiuso.
Map the exact enterprise group names to the ThothII `user` and `admin` roles in `auth.yaml`.
Unmapped groups are ignored. A configured group that does not exist produces a closed error.
## Diagnostica
## Diagnostics
`tht auth check` controlla discovery, issuer, JWKS, accesso al catalogo e presenza dei gruppi
configurati. L'opzione `--interactive` aggiunge la verifica dell'identità tramite device flow,
quando il provider la supporta.
`tht auth check` checks discovery, the issuer, JWKS, catalog access, and the configured groups.
The `--interactive` option also verifies identity through device flow when the provider supports it.
Ruotare separatamente secret OIDC e token del catalogo gruppi. Nessuno dei due deve comparire in
YAML, cronologia shell, log o output diagnostico.
Rotate the OIDC secret and group-catalog token separately. Neither may appear in YAML, shell
history, logs, or diagnostic output.
+16 -18
View File
@@ -1,7 +1,7 @@
# Enrollment client per DWH REST
# DWH REST client enrollment
La credenziale `dwh-auth` appartiene a una installazione ThothII e serve soltanto quando il
workspace usa il trasporto `rest_api`.
The `dwh-auth` credential belongs to one ThothII installation and is needed only when the
workspace uses the `rest_api` transport.
| Trasporto | Materiale richiesto |
| --- | --- |
@@ -9,13 +9,13 @@ workspace usa il trasporto `rest_api`.
| `postgres_direct` | Credenziali PostgreSQL e configurazione TLS PostgreSQL |
| `ssh_tunnel` | Credenziali PostgreSQL e materiale SSH |
## Consegna e conservazione
## Delivery and storage
Ricevere chiave e CA attraverso canali protetti separati. Conservare la chiave nel vault
dell'installazione o in un file regolare accessibile soltanto all'account autorizzato. Non
inserirla in Git, file YAML, argomenti, log o schermate condivise.
Receive the key and CA through separate protected channels. Store the key in the installation
vault or in a regular file accessible only to the authorized account. Do not put it in Git, YAML
files, arguments, logs, or shared screens.
## Configurazione ACME Limited
## ACME Limited configuration
Esempio di binding headless per il workspace `acme-ebikes`:
@@ -26,16 +26,14 @@ THT_WS_ACME_EBIKES_DWH_API_KEY_FILE=/run/secrets/acme-ebikes-dwh-api-key
THT_WS_ACME_EBIKES_DWH_TLS_CA_FILE=/run/secrets/acme-ebikes-dwh-ca.pem
```
Il suffisso del workspace deriva dall'ID immutabile trasformando i trattini in underscore e
usando lettere maiuscole. `API_KEY_FILE` contiene il percorso del file montato, non il valore
della chiave.
The workspace suffix comes from the immutable ID, with hyphens changed to underscores and letters
converted to uppercase. `API_KEY_FILE` contains the mounted file path, not the key value.
## Rotazione e revoca
## Rotation and revocation
Durante la rotazione, ricevere la nuova generazione, aggiornare il vault o il file montato e
confermare la connettività sulla route innocua `/rpc/ping`. Solo dopo questa conferma il
responsabile del server revoca la generazione precedente.
During rotation, receive the new generation, update the vault or mounted file, and confirm
connectivity through the harmless `/rpc/ping` route. The server owner revokes the previous
generation only after this confirmation.
Un `401` indica una chiave assente, sconosciuta, scaduta o revocata. Un `503` indica che il
servizio di autorizzazione o il registro non sono disponibili. In entrambi i casi non aggirare
REST e non ridurre la verifica TLS.
A `401` means the key is missing, unknown, expired, or revoked. A `503` means the authorization
service or registry is unavailable. In either case, do not bypass REST or weaken TLS verification.
+20 -22
View File
@@ -1,8 +1,8 @@
# `dwh-auth`: guida server
# `dwh-auth`: server guide
`dwh-auth` protegge la route REST `/dwh/` con una chiave distinta per ogni installazione
ThothII. Il componente gira come servizio Linux separato, non legge i dati del DWH e non si
collega direttamente a PostgreSQL.
`dwh-auth` protects the REST `/dwh/` route with a separate key for each ThothII installation.
It runs as a separate Linux service, does not read DWH data, and does not connect directly to
PostgreSQL.
```mermaid
flowchart LR
@@ -12,14 +12,14 @@ flowchart LR
AUTH -->|"authorized"| REST["DWH REST"]
```
## Confini di sicurezza
## Security boundaries
- Una chiave identifica un'installazione, non una persona.
- Chiavi e backup restano in file protetti e non entrano in Git, log, argomenti o JSON pubblico.
- Il registro conserva digest e metadati, mai la chiave in chiaro.
- La route REST deve essere esposta esclusivamente tramite TLS verificato.
- A key identifies an installation, not a person.
- Keys and backups stay in protected files and never enter Git, logs, arguments, or public JSON.
- The registry stores digests and metadata, never the key in plaintext.
- The REST route must be exposed only through verified TLS.
## Installazione
## Installation
Il servizio usa questi percorsi:
@@ -31,11 +31,10 @@ Il servizio usa questi percorsi:
| Socket | `/run/dwh-auth/verify.sock` |
| Consegne protette | `/root/dwh-auth-provision/` |
Installare binario e unit con owner `root`, creare l'utente di servizio `dwh-auth`, quindi
abilitare l'unità con `systemctl enable --now dwh-auth`. Il socket deve essere accessibile al
gruppo usato da Nginx.
Install the binary and unit with `root` ownership, create the `dwh-auth` service user, and enable
the unit with `systemctl enable --now dwh-auth`. The socket must be accessible to Nginx's group.
## Creazione e revoca delle chiavi
## Creating and revoking keys
Esempio per l'installazione ACME Limited:
@@ -46,9 +45,8 @@ sudo dwh-auth --registry-root /var/lib/dwh-auth key create \
--output /root/dwh-auth-provision/acme-factory-primary.key
```
Consegnare il file attraverso un vault aziendale o un canale autenticato. Per la rotazione,
creare una nuova chiave, distribuirla, aggiornare il client e revocare la precedente usando il
suo ID pubblico:
Deliver the file through an enterprise vault or an authenticated channel. To rotate a key, create
a new one, distribute it, update the client, and revoke the old one using its public ID:
```bash
sudo dwh-auth --registry-root /var/lib/dwh-auth key revoke \
@@ -56,10 +54,10 @@ sudo dwh-auth --registry-root /var/lib/dwh-auth key revoke \
--reason scheduled-rotation
```
La revoca è definitiva. Conservare backup cifrati del registro prima di ogni mutazione.
Revocation is permanent. Keep encrypted registry backups before every mutation.
## Integrazione Nginx
## Nginx integration
Nginx inoltra la chiave al socket di `dwh-auth`. Solo una risposta autorizzata permette il
passaggio verso il DWH REST; chiavi assenti, sconosciute, scadute o revocate ricevono `401`,
mentre indisponibilità del servizio o del registro producono `503`.
Nginx forwards the key to the `dwh-auth` socket. Only an authorized response allows the request
to reach DWH REST. Missing, unknown, expired, or revoked keys receive `401`; an unavailable
service or registry produces `503`.
+18 -20
View File
@@ -1,13 +1,13 @@
# TLS per DWH REST
# TLS for DWH REST
La chiave DWH è accettabile solo sopra TLS verificato. Errori di autorizzazione o disponibilità
non autorizzano mai a disabilitare la verifica del certificato.
The DWH key may be used only over verified TLS. Authorization or availability errors never justify
disabling certificate verification.
## CA privata
## Private CA
Quando il DWH REST usa una CA aziendale, consegnare il certificato separatamente dalla chiave
API. La CA non è una credenziale, ma la sua integrità è un confine di sicurezza: deve restare
fuori da Git e non essere scrivibile da utenti non autorizzati.
When DWH REST uses an enterprise CA, deliver the certificate separately from the API key. The CA
is not a credential, but its integrity is part of the security boundary. Keep it out of Git and
make it unwritable by unauthorized users.
Esempio ACME Limited:
@@ -15,26 +15,24 @@ Esempio ACME Limited:
THT_WS_ACME_EBIKES_DWH_TLS_CA_FILE=/run/secrets/acme-ebikes-dwh-ca.pem
```
## Fingerprint fuori banda
## Out-of-band fingerprint
Calcolare il fingerprint del file ricevuto e confrontarlo attraverso un canale indipendente:
Calculate the fingerprint of the received file and compare it through an independent channel:
```bash
openssl x509 -noout -fingerprint -sha256 \
-in /absolute/protected/acme-ebikes-dwh-ca.pem
```
Il SAN del certificato deve includere il nome esatto usato dal binding, per esempio
`dwh.acme.example`.
The certificate SAN must include the exact name used by the binding, such as `dwh.acme.example`.
## Rinnovo
## Renewal
1. Preparare certificato e chain nuovi.
2. Confermare SAN e fingerprint fuori banda.
3. Distribuire la nuova CA ai client mantenendo temporaneamente la precedente.
4. Aggiornare il binding e confermare la connettività con TLS normale.
5. Installare il certificato server.
6. Ritirare il trust precedente dopo la finestra concordata.
1. Prepare the new certificate and chain.
2. Confirm the SAN and fingerprint out of band.
3. Distribute the new CA to clients while temporarily keeping the old one.
4. Update the binding and confirm connectivity with normal TLS.
5. Install the server certificate.
6. Remove the old trust after the agreed window.
Non usare `curl -k`, non disabilitare TLS e non incorporare certificati o fingerprint completi
nei documenti condivisi.
Do not use `curl -k`, disable TLS, or embed complete certificates or fingerprints in shared documents.
+24 -23
View File
@@ -1,6 +1,6 @@
# Installazione Docker nei contesti operativi correnti
# Docker installation in the current operating contexts
ThothII usa una topologia Compose unica:
ThothII uses one Compose topology:
- `frontend`
- `core`
@@ -8,8 +8,9 @@ ThothII usa una topologia Compose unica:
- `embedding`
- `embedding-model-init`
Qdrant e Ollama embedding sono servizi interni obbligatori del progetto Compose. Restano esterni solo DWH e LLM. Il modello fissato è `qwen3-embedding:0.6b` con 1024 dimensioni e distanza
coseno; `embedding-model-init` lo prepara prima dell'avvio di `core`.
Qdrant and Ollama embedding are required internal Compose services. Only DWH and the LLM remain
external. The fixed model is `qwen3-embedding:0.6b` with 1024 dimensions and cosine distance;
`embedding-model-init` prepares it before `core` starts.
```mermaid
flowchart TB
@@ -25,14 +26,14 @@ flowchart TB
BUNDLE --> SERVICES["Frontend, core, vector, embedding"]
```
## Contratto sintetico di ownership
## Short ownership contract
| Componente | Ownership | Contratto operativo |
| --- | --- | --- |
| DWH | Esterno | Endpoint esterno configurato dall'installazione. |
| LLM | Esterno | Endpoint o policy esterna all'infrastruttura semantica interna. |
| Qdrant | Interno | Servizio Compose interno obbligatorio con volume persistente `qdrant-data`. |
| Ollama embedding | Interno | Servizio Compose interno obbligatorio per `qwen3-embedding:0.6b`. |
| DWH | External | External endpoint configured by the installation. |
| LLM | External | Endpoint or policy outside the internal semantic infrastructure. |
| Qdrant | Internal | Required internal Compose service with persistent `qdrant-data` volume. |
| Ollama embedding | Internal | Required internal Compose service for `qwen3-embedding:0.6b`. |
## Comando standard locale
@@ -45,44 +46,44 @@ docker compose --env-file deploy/env/local.env \
-f compose.yaml -f deploy/compose.local.yaml up --build -d
```
Compilare `deploy/env/local.env` con:
Set these values in `deploy/env/local.env`:
- `PI_AUTH_FILE`
- `THT_SECRETS_FILE`
- `THT_WORKSPACE_GIT_REMOTE`
- endpoint DWH
- endpoint LLM
- DWH endpoint
- LLM endpoint
Non inserire secret nel file `.env`. I secret runtime stanno nel bundle
`deploy/secrets/thothii.secrets`.
Do not put secrets in `.env`. Runtime secrets belong in the
`deploy/secrets/thothii.secrets` bundle.
## Bundle dei secret
## Secret bundle
Le chiavi documentate e supportate nel bundle sono:
The documented and supported bundle keys are:
```dotenv
THT_MODEL_API_KEY=...
THT_DWH_API_KEY=...
```
Una CA privata PEM resta esterna al bundle e va montata con un override Compose revisionato.
A private PEM CA remains outside the bundle and must be mounted through a reviewed Compose override.
## Preprocessing
Il preprocessing passa dal CLI host nativo e dal descrittore dell'installazione:
Preprocessing runs through the native host CLI and the installation descriptor:
```sh
tht --installation /percorso/assoluto/thothii-installation.yaml workspace preprocess evidence
tht --installation /percorso/assoluto/thothii-installation.yaml workspace preprocess dwh
```
Il CLI esegue il servizio profile-gated `workspace-maintenance`. I dettagli sono nel
[contratto del preprocessing](contracts/workspace-preprocessing-cli.md) e nella guida
[Evidence](evidence.md).
The CLI runs the profile-gated `workspace-maintenance` service. See the
[preprocessing contract](contracts/workspace-preprocessing-cli.md) and the
[Evidence guide](evidence.md) for details.
## Server
Per installazioni server usare il profilo server con overlay sessioni:
For server installations, use the server profile with the session overlay:
```sh
docker compose --env-file deploy/env/server.env \
@@ -90,7 +91,7 @@ docker compose --env-file deploy/env/server.env \
-f deploy/compose.session-server.yaml.example up --build -d
```
Consultare anche:
Also see:
- `docs/install/local-workspace-registry.md`
- `docs/install/server-workspace-registry.md`
+48 -50
View File
@@ -1,82 +1,80 @@
# Workflow operativo di ThothII
# ThothII operating workflow
ThothII guida ogni domanda attraverso otto fasi. Il modello propone i passaggi, il revisore
decide nei gate e il sistema registra artefatti e decisioni persistenti.
ThothII guides every question through eight phases. The model proposes the work, the reviewer
makes decisions at gates, and the system records persistent artifacts and decisions.
```mermaid
flowchart LR
Q["Domanda"] --> F1["F1 Chiarimento"]
Q["Question"] --> F1["F1 Clarification"]
F1 --> F2["F2 Memory"]
F2 --> F3["F3 Riscrittura"]
F2 --> F3["F3 Rewriting"]
F3 --> F4["F4 Evidence"]
F4 --> F5["F5 Schema"]
F5 --> F6["F6 Piano CTE"]
F5 --> F6["F6 CTE plan"]
F6 --> F7["F7 SQL"]
F7 --> F8["F8 Promozione"]
F8 --> DONE["Sessione finalizzata"]
F7 --> F8["F8 Promotion"]
F8 --> DONE["Finalized session"]
```
## Principi del workflow
## Workflow principles
- Il modello propone; il revisore approva, corregge o rifiuta.
- Ogni decisione rilevante viene registrata nel ledger della sessione.
- Lo stato persistito è la fonte di verità.
- Una fase può avanzare soltanto quando i suoi artefatti e gate sono completi.
- La ripresa ricostruisce il contesto dagli artefatti persistiti, non dalla conversazione.
- The model proposes; the reviewer approves, corrects, or rejects.
- The session ledger records every material decision.
- Persisted state is the source of truth.
- A phase advances only when its artifacts and gates are complete.
- Resume rebuilds context from persisted artifacts, not from the conversation.
## F1 — Chiarimento
## F1: clarification
Il sistema identifica l'ambiguità con maggiore impatto sul significato della domanda e presenta
una sola decisione per volta. Interpretazioni mutuamente esclusive usano una scelta singola;
risposte contemporaneamente valide usano una scelta multipla.
The system identifies the ambiguity most likely to change the meaning of the question and presents
one decision at a time. Mutually exclusive interpretations use a single choice; multiple valid
answers use a multiple choice.
## F2 — Memory
## F2: Memory
Le Memory compatibili con la domanda vengono proposte al revisore. Quelle selezionate entrano
nel contesto della sessione corrente; quelle non selezionate restano disponibili per domande
future.
The system proposes Memory items that fit the question. Selected items enter the current session
context; unselected items remain available for future questions.
## F3 — Riscrittura
## F3: rewriting
La domanda viene riscritta in forma esplicita usando i chiarimenti approvati. Il revisore verifica
la domanda risultante e le assunzioni prima di proseguire.
The system rewrites the question explicitly using the approved clarifications. The reviewer checks
the resulting question and its assumptions before continuing.
## F4 — Evidence
## F4: Evidence
Il sistema recupera Evidence dal corpus attivo e presenta citazioni e provenienza. Il revisore
decide quali elementi sono pertinenti alla domanda.
The system retrieves Evidence from the active corpus and presents citations and provenance. The
reviewer decides which items apply to the question.
## F5 — Schema
## F5: schema
Tabelle, colonne, relazioni e filtri vengono collegati al significato approvato della domanda.
Il riepilogo chiude la fase quando domanda, assunzioni ed elementi del DWH sono coerenti.
Tables, columns, relationships, and filters are linked to the approved meaning of the question.
The summary closes the phase when the question, assumptions, and DWH elements are consistent.
## F6 — Piano CTE
## F6: CTE plan
La query viene scomposta in CTE nominati con scopo, dipendenze, tabelle, filtri e colonne di
output. Ogni passaggio viene presentato al revisore prima della produzione dell'SQL finale.
The query is broken into named CTEs with a purpose, dependencies, tables, filters, and output
columns. The reviewer sees each step before the system produces the final SQL.
## F7 — SQL finale
## F7: final SQL
Il sistema produce `sql_final.sql`, ne controlla la coerenza con il piano approvato e presenta
l'artefatto al revisore. Una correzione può riaprire il piano CTE senza perdere le decisioni
ancora valide.
The system produces `sql_final.sql`, checks it against the approved plan, and presents the artifact
to the reviewer. A correction can reopen the CTE plan without discarding decisions that remain valid.
## F8 — Promozione delle Memory
## F8: Memory promotion
Alla fine della sessione il sistema propone i chiarimenti riutilizzabili. Il revisore decide
quali promuovere nel registro globale; la sessione viene quindi finalizzata.
At the end of the session, the system proposes reusable clarifications. The reviewer decides which
ones to promote to the global registry, and the session is then finalized.
## Gate disponibili
## Available gates
| Gate | Uso |
| Gate | Use |
| --- | --- |
| Scelta singola | Una sola interpretazione può essere valida |
| Scelta multipla | Più elementi possono essere validi contemporaneamente |
| Conferma artefatto | Approvazione di un documento o risultato della fase |
| Conferma fase | Chiusura esplicita di una fase |
| Single choice | Only one interpretation can be valid |
| Multiple choice | Several items can be valid at the same time |
| Artifact confirmation | Approval of a document or phase result |
| Phase confirmation | Explicitly closes a phase |
## Ripresa e riapertura
## Resume and reopening
Una sessione ripresa rientra nell'ultima fase incompleta. Una riapertura invalida soltanto le
decisioni e gli artefatti che dipendono dal punto modificato; il resto del lavoro rimane valido.
A resumed session returns to its last incomplete phase. Reopening invalidates only the decisions
and artifacts that depend on the changed point; the rest of the work remains valid.
+31
View File
@@ -50,6 +50,37 @@ test("accepts the immutable workspace revision contract", async () => {
await expect(getWorkspace("psd-clinical")).resolves.toEqual({ workspace, revision });
});
test("accepts the evidence schema version materialized by the backend", async () => {
const persistedWorkspace = {
...workspace,
evidence: {
schema_version: 1,
source: {
type: "filesystem",
uri: "psd-clinical/evidence",
patterns: ["**/*.md"],
max_bytes: 10 * 1024 * 1024,
},
policy: { max_chunk_chars: 5000, retain_published_generations: 3 },
},
};
server.use(http.get("/api/workspaces/psd-clinical", () => HttpResponse.json({
workspace: persistedWorkspace,
revision,
})));
await expect(getWorkspace("psd-clinical")).resolves.toEqual({
workspace: {
...persistedWorkspace,
evidence: {
source: persistedWorkspace.evidence.source,
policy: persistedWorkspace.evidence.policy,
},
},
revision,
});
});
test("decodes runtime requirements but rejects any secret value returned by the server", async () => {
server.use(http.get(
"/api/workspaces/psd-clinical/runtime-configuration",
@@ -159,6 +159,7 @@ test("opens Workspace management from the right sidebar without interrupting the
server.use(
http.get("/api/workspace-registry/status", () =>
HttpResponse.json({ branch: "main", ahead: 0, behind: 0, degraded: false })),
http.get("/api/health/dwh", () => HttpResponse.json({ ok: false })),
);
renderShell();
@@ -168,6 +169,21 @@ test("opens Workspace management from the right sidebar without interrupting the
expect(screen.getByTestId("app-shell")).toHaveAttribute("data-activity-layout", "closed");
});
test("does not block the shell when the DWH is unavailable at startup", async () => {
let healthChecks = 0;
server.use(
http.get("/api/health/dwh", () => {
healthChecks += 1;
return HttpResponse.json({ ok: false });
}),
);
renderShell();
expect(await screen.findByRole("button", { name: "Workspace management" })).toBeVisible();
expect(screen.queryByRole("heading", { name: "Connection unavailable" })).not.toBeInTheDocument();
expect(healthChecks).toBe(0);
});
test("marks the composer as awaiting input for a pending freetext gate", () => {
useSessionStore.setState({
-36
View File
@@ -23,7 +23,6 @@ import { toast } from "sonner";
import {
closeSession, listSessions, resumeSession, getSession,
renameSession, setSessionGroup, archiveSession, unarchiveSession, deleteSession, prewarmRuntime,
checkDwhHealth,
} from "../api/sessions";
import { logout as logoutUser } from "../api/auth";
import {
@@ -129,23 +128,8 @@ export function AppShell({ canLogout }: AppShellProps) {
const [stopConfirm, setStopConfirm] = useState(false);
const [collapsedGroups, setCollapsedGroups] = useState<Record<string, boolean>>({});
const [renameGroupTarget, setRenameGroupTarget] = useState<string | null>(null);
const [dwhDown, setDwhDown] = useState(false);
const [dwhChecking, setDwhChecking] = useState(true);
const [dwhCheckEpoch, setDwhCheckEpoch] = useState(0);
const operationEpochRef = useRef(0);
useEffect(() => () => { operationEpochRef.current += 1; }, []);
useEffect(() => {
let cancelled = false;
const operation = captureAuthOperation({ disposalEpoch: operationEpochRef.current });
if (!operation) return () => { cancelled = true; };
setDwhChecking(true);
checkDwhHealth().then((r) => {
if (cancelled || !isAuthOperationCurrent(operation, { disposalEpoch: operationEpochRef.current })) return;
setDwhDown(!r.ok);
setDwhChecking(false);
});
return () => { cancelled = true; };
}, [dwhCheckEpoch]);
const groups = useMemo(
() => [...new Set(sessions.map((s) => s.group).filter((g): g is string => !!g))].sort(),
@@ -958,26 +942,6 @@ export function AppShell({ canLogout }: AppShellProps) {
}}
/>
)}
{dwhDown && (
<Dialog open onOpenChange={() => {}}>
<DialogContent showCloseButton={false} className="sm:max-w-md">
<DialogHeader>
<DialogTitle>Connection unavailable</DialogTitle>
<DialogDescription>
The database is unreachable. Check the VPN connection and try again.
</DialogDescription>
</DialogHeader>
<DialogFooter>
<Button
disabled={dwhChecking}
onClick={() => setDwhCheckEpoch((e) => e + 1)}
>
{dwhChecking ? "Checking…" : "Retry"}
</Button>
</DialogFooter>
</DialogContent>
</Dialog>
)}
</div>
);
}
+1 -1
View File
@@ -197,7 +197,7 @@ function PiInstructionSteps({ details }: { details: PiPlatformDetails }) {
</li>
<li>
<h4 className="font-semibold text-foreground">Edit the provider catalog</h4>
<p className="mt-1 text-muted-foreground">Edit <code>{details.modelsPath}</code> only when adding or correcting a provider/model definition. The provider catalog is an address book/map of the services Pi can call: each entry supplies the API endpoint and format, and lists the model identifiers offered there. It does not enable a model and never contains credentials.</p>
<p className="mt-1 text-muted-foreground">Edit <code>{details.modelsPath}</code> only when adding or correcting a provider/model definition. The provider catalog is an address book/map of the services Pi can call: each entry supplies the API endpoint and format, and lists the model identifiers offered there. It does not enable a model and never contains credentials. Provider integrations must remain declarative; do not add model-provider code under <code>harness/.pi/extensions/</code>.</p>
<dl className="mt-2 grid grid-cols-[auto_1fr] gap-x-2 gap-y-1 text-muted-foreground">
<dt className="font-mono text-foreground">baseUrl </dt><dd>is the provider API endpoint.</dd>
<dt className="font-mono text-foreground">api </dt><dd>selects the provider API format.</dd>
+8 -2
View File
@@ -313,7 +313,11 @@ function copyS3Evidence(value: unknown): EvidenceSource | undefined {
}
function copyEvidence(value: unknown, id: string): WorkspaceEvidence | undefined {
const source = exactRecord(value, ["source", "policy"]);
// The backend's canonical YAML parser materializes the evidence schema version
// in responses, even though the browser draft does not need to retain it.
const source = exactRecord(value, ["schema_version", "source", "policy"]);
const schemaVersion = source?.schema_version;
if (!source) return undefined;
if (!source) return undefined;
const type = record(source.source)?.type;
const evidenceSource = type === "filesystem"
@@ -324,7 +328,9 @@ function copyEvidence(value: unknown, id: string): WorkspaceEvidence | undefined
? copyS3Evidence(source.source)
: undefined;
const policy = copyEvidencePolicy(source.policy);
return evidenceSource && policy ? { source: evidenceSource, policy } : undefined;
return (schemaVersion === 1 || schemaVersion === 2) && evidenceSource && policy
? { source: evidenceSource, policy }
: undefined;
}
function oneOf<T extends string>(value: unknown, choices: readonly T[]): T | undefined {
@@ -1,96 +0,0 @@
import { createAssistantMessageEventStream, streamSimpleOpenAICompletions } from "@earendil-works/pi-ai";
function convertThinkingBlocks(message) {
return {
...message,
content: (message.content ?? []).map((block) => {
if (block?.type !== "thinking") return block;
return { type: "text", text: block.thinking ?? "" };
}),
};
}
function convertEvent(event) {
if (event.type === "thinking_start") {
return { type: "text_start", contentIndex: event.contentIndex, partial: convertThinkingBlocks(event.partial) };
}
if (event.type === "thinking_delta") {
return { type: "text_delta", contentIndex: event.contentIndex, delta: event.delta, partial: convertThinkingBlocks(event.partial) };
}
if (event.type === "thinking_end") {
return { type: "text_end", contentIndex: event.contentIndex, content: event.content, partial: convertThinkingBlocks(event.partial) };
}
if (event.type === "done") {
return { ...event, message: convertThinkingBlocks(event.message) };
}
if (event.type === "error") {
return { ...event, error: convertThinkingBlocks(event.error) };
}
if (event.partial) {
return { ...event, partial: convertThinkingBlocks(event.partial) };
}
return event;
}
function streamAritmolab(model, context, options) {
const source = streamSimpleOpenAICompletions(model, context, options);
const stream = createAssistantMessageEventStream();
(async () => {
try {
for await (const event of source) {
stream.push(convertEvent(event));
}
} catch (error) {
const message = {
role: "assistant",
content: [],
api: model.api,
provider: model.provider,
model: model.id,
usage: {
input: 0,
output: 0,
cacheRead: 0,
cacheWrite: 0,
totalTokens: 0,
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
},
stopReason: options?.signal?.aborted ? "aborted" : "error",
errorMessage: error instanceof Error ? error.message : String(error),
timestamp: Date.now(),
};
stream.push({ type: "error", reason: message.stopReason, error: message });
stream.end();
}
})();
return stream;
}
export default function (pi) {
pi.registerProvider("aritmolab", {
name: "AritmoLab",
baseUrl: "https://ml-aritmolab.policlinicosandonato.it/v1",
apiKey: "aritmolab",
api: "openai-completions",
streamSimple: streamAritmolab,
models: [
{
id: "qwen3.6-35b-a3b",
name: "AritmoLab Qwen3.6 35B A3B",
reasoning: false,
input: ["text"],
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
contextWindow: 131072,
maxTokens: 16384,
compat: {
supportsDeveloperRole: false,
supportsReasoningEffort: false,
supportsStore: false,
maxTokensField: "max_tokens",
},
},
],
});
}
@@ -1,6 +1,5 @@
const test = require("node:test");
const assert = require("node:assert");
const crypto = require("node:crypto");
const cp = require("node:child_process");
const fs = require("node:fs");
const { createRequire } = require("node:module");
@@ -125,17 +124,26 @@ function stripDescriptions(value, insideProperties = false) {
return value;
}
test("the injected skill stays byte-identical during extraction", () => {
test("before_agent_start injects the canonical skill byte-for-byte", async () => {
const harnessRoot = path.resolve(__dirname, "..", "..", "..", "..");
const digest = (relativePath) => crypto
.createHash("sha256")
.update(fs.readFileSync(path.join(harnessRoot, relativePath)))
.digest("hex");
assert.equal(
digest(path.join(".pi", "skills", "tht-sessione", "SKILL.md")),
"626a794071c095a4f20fffabb3bab901f05c101590adbdc58e45adfae56f3219",
const canonicalSkill = fs.readFileSync(
path.join(harnessRoot, ".pi", "skills", "tht-sessione", "SKILL.md"),
"utf8",
);
const { pi } = createFakePi();
installGate(pi);
await pi.emit("input", { source: "rpc", text: '/nuova-domanda "x"' });
const injected = await pi.emit("before_agent_start", { systemPrompt: "BASE" });
const systemPrompt = injected?.systemPrompt ?? "";
const opening = "<tht-sessione-skill>\n";
const closing = "\n</tht-sessione-skill>";
const start = systemPrompt.indexOf(opening);
const end = systemPrompt.indexOf(closing, start + opening.length);
assert.notEqual(start, -1);
assert.notEqual(end, -1);
assert.equal(systemPrompt.slice(start + opening.length, end), canonicalSkill);
});
test("F1 persists a concrete clarification directly from the select widget", async () => {
+1 -1
View File
@@ -43,7 +43,7 @@ no network.
**Coverage (honest):**
- Python logic-pure: `workflow.yaml` loading, `effective_decisions`,
`teardown_to_phase`, `aggregate_lsh_multi` (on fake hits),
`formula_store` read/write, `decision_retracted`, `save_one_memory`, the
`decision_retracted`, `save_one_memory`, the
rationale-capture contract, the session-coherence smoke, CLI `phase meta --json`.
- **Gate builder functions** (pure, in JS, tested in JS): the widget-descriptor
builders produce the correct JSON given params. Tested in-language (`node --test`),
+1
View File
@@ -11,6 +11,7 @@
"decision add-batch",
"decision add-join-set",
"evidence evaluate",
"evidence migrate",
"evidence prepare",
"evidence resolve",
"evidence validate",
+1 -1
View File
@@ -32,7 +32,7 @@ def test_typer_tree_matches_the_approved_command_surface():
approved = _approved_surface()
expected = set(approved["maintained"]) | set(approved["enhanced"])
assert len(approved["maintained"]) == 60
assert len(approved["maintained"]) == 61
assert len(approved["enhanced"]) == 8
assert len(approved["erased"]) == 14
assert not (expected & set(approved["erased"]))
+1 -1
View File
@@ -121,7 +121,7 @@ def pipeline(tmp_path, source, *, embedder=None, vectors=None, model="model-a",
def test_pipeline_embeds_validated_curated_evidence_as_semantic_fragments(tmp_path):
evidence = CuratedEvidence.model_validate(
{
"schema_version": 1,
"schema_version": 2,
"id": "evidence:fascia-pediatrica",
"title": "Fascia pediatrica",
"kind": "formula",
+35
View File
@@ -1,4 +1,5 @@
import hashlib
import subprocess
import threading
import unicodedata
@@ -15,6 +16,7 @@ from tht.evidence import (
dump_manifest,
load_curated_tree,
load_manifest,
migrate_workspace_evidence,
prepare_workspace_evidence,
validate_workspace_evidence,
)
@@ -346,9 +348,41 @@ def test_prepare_changed_source_uses_one_model_call_and_applies_a_valid_batch(tm
assert report.created == ()
assert len(restructurer.requests) == 1
assert restructurer.requests[0].previous_units[0].id == "evidence:fascia-pediatrica"
curated_path = tmp_path / "evidence" / "curated" / "domain" / "fascia-pediatrica.md"
curated = load_curated_tree(tmp_path / "evidence" / "curated")[0]
assert curated.schema_version == 2
assert "# Fascia pediatrica\n" in curated_path.read_text(encoding="utf-8")
assert validate_workspace_evidence(tmp_path).publishable is True
def test_migrate_workspace_evidence_rewrites_v1_units_without_a_model_call(tmp_path):
source_text = "I pazienti sotto i 18 anni sono pediatrici."
_write_workspace(tmp_path, _evidence(source_text), source_text)
report = migrate_workspace_evidence(tmp_path, git_status=lambda _: ())
curated_path = tmp_path / "evidence" / "curated" / "domain" / "fascia-pediatrica.md"
migrated = load_curated_tree(tmp_path / "evidence" / "curated")[0]
assert report.migrated == ("evidence:fascia-pediatrica",)
assert report.unchanged == ()
assert migrated.schema_version == 2
assert migrated.payload.rule == "La fascia pediatrica comprende i minori."
assert "## Regola\n\nLa fascia pediatrica comprende i minori." in curated_path.read_text(
encoding="utf-8",
)
assert report.findings == ()
def test_migrate_workspace_evidence_rejects_dirty_curated_files_in_a_nested_workspace(tmp_path):
subprocess.run(["git", "init", "--quiet", str(tmp_path)], check=True)
workspace_root = tmp_path / "psd-clinical"
source_text = "I pazienti sotto i 18 anni sono pediatrici."
_write_workspace(workspace_root, _evidence(source_text), source_text)
with pytest.raises(EvidencePreparationError, match="authoring_worktree_dirty"):
migrate_workspace_evidence(workspace_root)
def test_prepare_can_issue_independent_source_calls_concurrently(tmp_path):
source_text = "I pazienti sotto i 18 anni sono pediatrici."
_write_workspace(tmp_path, _evidence(source_text), source_text)
@@ -512,6 +546,7 @@ def test_prepare_marks_an_omitted_prior_unit_for_human_review(tmp_path):
"supporting_excerpt_missing", "unresolved_review_item",
]
retained = load_curated_tree(tmp_path / "evidence" / "curated")[0]
assert retained.schema_version == 2
assert retained.review_items[0].code == "source_no_longer_supports_unit"
+148
View File
@@ -206,6 +206,154 @@ def _formula_evidence() -> CuratedEvidence:
})
def _domain_evidence_v2() -> CuratedEvidence:
return CuratedEvidence.model_validate({
**COMMON,
"schema_version": 2,
"title": "Dominio Ablazione",
"kind": "domain",
"payload": {
"rule": (
"Il dominio Ablazione rappresenta la procedura transcatetere.\n\n"
"La fact centrale è `clinical.fact_ablazione`."
),
},
})
def test_v2_curated_markdown_renders_domain_content_in_the_markdown_body(tmp_path):
evidence = _domain_evidence_v2()
path = tmp_path / "curated" / "domain" / "dominio-ablazione.md"
text = dump_curated_markdown(evidence)
frontmatter = text.split("---\n", 2)[1]
parsed = parse_curated_markdown(text, path=path)
assert "domain:" not in frontmatter
assert "supporting_excerpts:" not in frontmatter
assert "review_items:" not in frontmatter
assert "# Dominio Ablazione\n" in text
assert "## Regola\n\nIl dominio Ablazione" in text
assert "## Estratti di supporto\n\n> I pazienti sotto i 18 anni sono pediatrici." in text
assert parsed == evidence
@pytest.mark.parametrize(("kind", "payload", "rendered"), [
("glossary", {
"definition": "Un paziente con età inferiore a 18 anni.",
"synonyms": ["minore"],
"variants": ["pediatrico"],
}, "## Sinonimi\n\n- minore"),
("enum", {
"column": "clinical.episode.discharge_status",
"values": {"D": "dimesso", "T": "trasferito | altra struttura"},
}, "| `D` | dimesso |"),
("example", {
"question": "Come riconosco un paziente pediatrico?",
"interpretation": "Applicare la formula della fascia pediatrica.",
}, "## Domanda\n\nCome riconosco un paziente pediatrico?"),
("mapping", {
"concept": "fascia pediatrica",
"tables": ["clinical.patient"],
"columns": ["clinical.patient.birth_date"],
}, "## Tabelle\n\n- `clinical.patient`"),
("normalization", {
"input": "PEDS",
"output": "pediatrico",
"rule": "Converte il codice abbreviato nella forma canonica.",
}, "## Output\n\npediatrico"),
("formula", {
"concept": "fascia pediatrica",
"columns": ["clinical.patient.birth_date"],
"sql": "CASE WHEN age < 18 THEN 'pediatrica' ELSE 'adulta' END",
}, "```sql\nCASE WHEN age < 18"),
("reference", {
"url": "https://example.test/linea-guida",
"label": "Linea guida",
"description": "Criteri clinici di riferimento.",
}, "## URL\n\n<https://example.test/linea-guida>"),
])
def test_v2_curated_markdown_renders_and_round_trips_every_typed_payload(
tmp_path, kind, payload, rendered,
):
evidence = CuratedEvidence.model_validate({
**COMMON,
"schema_version": 2,
"kind": kind,
"payload": payload,
})
path = tmp_path / "curated" / kind / "fascia-pediatrica.md"
text = dump_curated_markdown(evidence)
assert rendered in text
assert parse_curated_markdown(text, path=path) == evidence
def test_v2_curated_markdown_renders_review_items_as_readable_blocks(tmp_path):
evidence = CuratedEvidence.model_validate({
**COMMON,
"schema_version": 2,
"kind": "domain",
"review_items": [{
"code": "ambiguous_source_statement",
"message": "Il sorgente non chiarisce la data di riferimento.",
"field": "domain.rule",
}],
"payload": {"rule": "L'età è calcolata alla data di ricovero."},
})
path = tmp_path / "curated" / "domain" / "fascia-pediatrica.md"
text = dump_curated_markdown(evidence)
assert "## Elementi da rivedere" in text
assert "### `ambiguous_source_statement`" in text
assert "Il sorgente non chiarisce la data di riferimento." in text
assert "**Campo:** `domain.rule`" in text
assert parse_curated_markdown(text, path=path) == evidence
@pytest.mark.parametrize("legacy_field", [
"review_items: []\n",
"payload:\n rule: should-not-be-ignored\n",
])
def test_v2_curated_markdown_rejects_body_owned_fields_in_frontmatter(legacy_field):
text = dump_curated_markdown(_domain_evidence_v2()).replace(
"kind: domain\n",
f"kind: domain\n{legacy_field}",
)
with pytest.raises(ValueError, match="frontmatter"):
parse_curated_markdown(text)
def test_v2_curated_markdown_rejects_unstructured_body_content():
text = dump_curated_markdown(_domain_evidence_v2()) + "should-not-be-ignored\n"
with pytest.raises(ValueError, match="unstructured"):
parse_curated_markdown(text)
def test_v2_curated_markdown_round_trips_an_empty_enum_as_an_explicit_empty_state(tmp_path):
evidence = CuratedEvidence.model_validate({
**COMMON,
"schema_version": 2,
"kind": "enum",
"payload": {
"column": "clinical.episode.discharge_status",
"values": {},
},
})
text = dump_curated_markdown(evidence)
assert "Nessun elemento" in text
assert parse_curated_markdown(
text,
path=tmp_path / "curated" / "enum" / "discharge-status.md",
) == evidence
def test_curated_markdown_round_trip_uses_the_kind_specific_key(tmp_path):
evidence = _formula_evidence()
path = tmp_path / "curated" / "formula" / "fascia-pediatrica.md"
+30
View File
@@ -21,10 +21,40 @@ def test_evidence_authoring_commands_are_distinct_from_runtime_preprocessing():
assert result.exit_code == 0
assert "prepare" in result.output
assert "migrate" in result.output
assert "resolve" in result.output
assert "validate" in result.output
def test_evidence_migrate_json_is_pristine(monkeypatch, tmp_path):
from tht.cli import evidence_cmd
from tht.evidence import EvidenceMigrationReport
monkeypatch.setattr(evidence_cmd, "_canonical_worktree", lambda root: root)
monkeypatch.setattr(
evidence_cmd,
"migrate_workspace_evidence",
lambda *args, **kwargs: EvidenceMigrationReport(
migrated=("evidence:fascia-pediatrica",),
unchanged=("evidence:fascia-adulta",),
findings=(),
),
)
result = CliRunner().invoke(app, ["evidence", "migrate", str(tmp_path), "--json"])
assert result.exit_code == 0
assert result.stderr == ""
assert json.loads(result.stdout) == {
"findings": [],
"migrated": ["evidence:fascia-pediatrica"],
"operation": "evidence_migrate",
"schemaVersion": 1,
"status": "migrated",
"unchanged": ["evidence:fascia-adulta"],
}
def test_evidence_prepare_failure_identifies_the_source_file(monkeypatch, tmp_path):
from tht.cli import evidence_cmd
from tht.evidence import EvidencePreparationError
@@ -1,155 +0,0 @@
"""Migration boundary: legacy formulas become curated evidence or session proposals."""
import hashlib
from tht.evidence import formula_store
from tht.evidence.authoring import normalize_source_text
from tht.evidence.canonical import CuratedEvidence
from tht.evidence.formula_store import ConceptFormula
def test_reviewed_formula_migration_has_deterministic_provenance_hash():
formula = ConceptFormula(
concept="fascia pediatrica",
columns=["clinical.patient.birth_date"],
sql="CASE WHEN age < 18 THEN 'pediatrica' ELSE 'adulta' END",
status="reviewed",
sources=["Regola clinica approvata dal gruppo pediatrico."],
)
original_source = """---
concept: fascia pediatrica
columns: [clinical.patient.birth_date]
status: reviewed
sources:
- Regola clinica approvata dal gruppo pediatrico.
---
CASE WHEN age < 18 THEN 'pediatrica' ELSE 'adulta' END
"""
first = formula_store.legacy_formula_to_curated(
formula,
legacy_path="formulas/fascia-pediatrica-1.sql.md",
source_content=original_source,
)
second = formula_store.legacy_formula_to_curated(
formula,
legacy_path="formulas/fascia-pediatrica-1.sql.md",
source_content=original_source,
)
assert isinstance(first, CuratedEvidence)
assert isinstance(second, CuratedEvidence)
assert first.provenance.source_sha256 == second.provenance.source_sha256
expected = hashlib.sha256(normalize_source_text(original_source).encode("utf-8")).hexdigest()
assert first.provenance.source_sha256 == f"sha256:{expected}"
def test_reviewed_formula_without_original_source_fails_closed():
formula = ConceptFormula(
concept="fascia pediatrica",
columns=["clinical.patient.birth_date"],
sql="CASE WHEN age < 18 THEN 'pediatrica' ELSE 'adulta' END",
status="reviewed",
sources=["Regola clinica approvata dal gruppo pediatrico."],
)
outcome = formula_store.legacy_formula_to_curated(
formula, legacy_path="formulas/fascia-pediatrica-1.sql.md",
)
assert outcome.code == "legacy_formula_requires_manual_review"
assert outcome.problems == ("original_source_required",)
def test_reviewed_formula_without_verified_supporting_excerpts_fails_closed():
formula = ConceptFormula(
concept="fascia pediatrica",
columns=["clinical.patient.birth_date"],
sql="CASE WHEN age < 18 THEN 'pediatrica' ELSE 'adulta' END",
status="reviewed",
sources=["Nota non presente nel sorgente originale."],
)
original_source = """---
concept: fascia pediatrica
columns: [clinical.patient.birth_date]
status: reviewed
sources:
- >-
Nota non presente nel
sorgente originale.
---
CASE WHEN age < 18 THEN 'pediatrica' ELSE 'adulta' END
"""
outcome = formula_store.legacy_formula_to_curated(
formula,
legacy_path="formulas/fascia-pediatrica-1.sql.md",
source_content=original_source,
)
assert outcome.code == "legacy_formula_requires_manual_review"
assert outcome.problems == ("supporting_excerpt_unverified",)
def test_reviewed_formula_without_provenance_notes_fails_closed():
formula = ConceptFormula(
concept="fascia pediatrica",
columns=["clinical.patient.birth_date"],
sql="CASE WHEN age < 18 THEN 'pediatrica' ELSE 'adulta' END",
status="reviewed",
sources=[],
)
outcome = formula_store.legacy_formula_to_curated(
formula,
legacy_path="formulas/fascia-pediatrica-1.sql.md",
source_content=formula.dump(),
)
assert outcome.code == "legacy_formula_requires_manual_review"
assert outcome.problems == ("supporting_excerpts_required",)
def test_reviewed_formula_must_match_the_original_source_record():
formula = ConceptFormula(
concept="fascia pediatrica",
columns=["clinical.patient.birth_date"],
sql="CASE WHEN age < 18 THEN 'pediatrica' ELSE 'adulta' END",
status="reviewed",
sources=["Regola clinica approvata dal gruppo pediatrico."],
)
different_source = ConceptFormula(
concept=formula.concept,
columns=formula.columns,
sql="CASE WHEN age < 16 THEN 'pediatrica' ELSE 'adulta' END",
status=formula.status,
sources=formula.sources,
).dump()
outcome = formula_store.legacy_formula_to_curated(
formula,
legacy_path="formulas/fascia-pediatrica-1.sql.md",
source_content=different_source,
)
assert outcome.code == "legacy_formula_requires_manual_review"
assert outcome.problems == ("original_source_mismatch",)
def test_incompatible_reviewed_legacy_formula_fails_closed_with_the_original_record():
formula = ConceptFormula(
concept="ablazione",
columns=["testo"],
sql="SELECT 2",
status="reviewed",
sources=["legacy manual"],
)
outcome = formula_store.legacy_formula_to_curated(
formula, legacy_path="formulas/ablazione-2.sql.md",
)
assert outcome.code == "legacy_formula_requires_manual_review"
assert outcome.legacy_path == "formulas/ablazione-2.sql.md"
assert outcome.formula == formula
+12 -145
View File
@@ -1,161 +1,22 @@
"""L1: SQL formula evidence -- concept->formula units (spec D14b).
A concept (e.g. 'fascia pediatrica') maps to a SQL formula (CASE WHEN ...) that
derives it from physical columns. These are reusable, reviewable units: the gate
surfaces a candidate formula, the reviewer approves or rejects it (recorded via
concept_formula_approved / concept_formula_rejected), and approved formulas are
part of the schema-linking artifact. The store is frontmatter-YAML + SQL body.
"""
"""L1: session-local Formula proposals and their reviewer decisions."""
from datetime import UTC, datetime
from tht.decisions import DecisionRecord
from tht.evidence import formula_store
from tht.evidence.formula_store import ConceptFormula, retrieve_formula, save_formula
from tht.evidence.session import project_session
from tht.session.models import SchemaLinking
def test_formula_retrieval_by_concept(tmp_path):
f = ConceptFormula(
concept="fascia pediatrica",
columns=["data_nascita"],
sql="CASE WHEN age < 18 THEN 'pediatrica' ELSE 'adulto' END",
status="reviewed",
sources=["src1"],
)
save_formula(tmp_path, f)
results = retrieve_formula(tmp_path, "fascia pediatrica")
assert len(results) == 1
assert results[0].sql.startswith("CASE WHEN")
assert results[0].concept == "fascia pediatrica"
assert results[0].status == "reviewed"
assert "data_nascita" in results[0].columns
def test_save_and_reload_roundtrip_preserves_sql_body(tmp_path):
f = ConceptFormula(
concept="fascia pediatrica",
columns=["data_nascita"],
sql="CASE\n WHEN x THEN 1\nEND",
status="draft",
sources=[],
)
path = save_formula(tmp_path, f)
assert path.exists()
# the file is frontmatter YAML + SQL body
text = path.read_text()
assert text.startswith("---")
assert "concept: fascia pediatrica" in text
assert "CASE" in text # SQL body preserved
def test_retrieve_multiple_formulas_for_concept(tmp_path):
# two competing formulas for the same concept (different sources/status)
save_formula(tmp_path, ConceptFormula(concept="ablazione", columns=["flag"],
sql="SELECT 1", status="draft", sources=["a"]))
save_formula(tmp_path, ConceptFormula(concept="ablazione", columns=["testo"],
sql="SELECT 2", status="reviewed", sources=["b"]))
results = retrieve_formula(tmp_path, "ablazione")
assert len(results) == 2
statuses = {r.status for r in results}
assert statuses == {"draft", "reviewed"}
def test_retrieve_empty_when_no_match(tmp_path):
save_formula(tmp_path, ConceptFormula(concept="altro", columns=["c"],
sql="SELECT 1", status="reviewed", sources=[]))
assert retrieve_formula(tmp_path, "inesistente") == []
def test_retrieve_empty_on_missing_dir(tmp_path):
# no formulas dir at all -> empty list, not error
assert retrieve_formula(tmp_path / "nope", "anything") == []
def test_concept_formula_decision_types_exist():
import typing
from tht.decisions import DecisionType
args = typing.get_args(DecisionType)
assert "concept_formula_approved" in args
assert "concept_formula_rejected" in args
def test_concept_formula_default_status(tmp_path):
# status has a sensible default so an author can write a draft quickly
f = ConceptFormula(concept="x", columns=["c"], sql="SELECT 1")
assert f.status == "draft" # not yet reviewed
assert f.sources == []
def test_reviewed_legacy_formula_becomes_curated_formula_with_stable_provenance():
formula = ConceptFormula(
concept="fascia pediatrica",
columns=["clinical.patient.birth_date"],
sql="CASE WHEN age < 18 THEN 'pediatrica' ELSE 'adulta' END",
status="reviewed",
sources=["Regola clinica approvata dal gruppo pediatrico."],
)
migrated = formula_store.legacy_formula_to_curated(
formula,
legacy_path="formulas/fascia-pediatrica-1.sql.md",
source_content=formula.dump(),
)
assert migrated is not None
assert migrated.id.startswith("evidence:fascia-pediatrica-")
assert migrated.title == "Fascia pediatrica"
assert migrated.kind == "formula"
assert migrated.payload.concept == formula.concept
assert migrated.payload.columns == ("clinical.patient.birth_date",)
assert migrated.payload.sql == formula.sql
assert migrated.provenance.source_file == "source/formulas/fascia-pediatrica-1.sql.md"
assert migrated.provenance.supporting_excerpts == tuple(formula.sources)
assert migrated.review_items == ()
assert formula_store.legacy_formula_to_curated(
formula,
legacy_path="formulas/fascia-pediatrica-1.sql.md",
source_content=formula.dump(),
).id == migrated.id
def test_reviewed_legacy_formulas_with_the_same_concept_keep_distinct_path_identities():
first = ConceptFormula(
concept="fascia pediatrica",
columns=["clinical.patient.birth_date"],
sql="CASE WHEN age < 18 THEN 'pediatrica' ELSE 'adulta' END",
status="reviewed",
sources=["Regola legacy revisionata."],
)
second = ConceptFormula(
concept="fascia pediatrica",
columns=["clinical.patient.birth_date"],
sql="CASE WHEN age < 16 THEN 'pediatrica' ELSE 'adulta' END",
status="reviewed",
sources=["Regola legacy revisionata."],
)
first_migration = formula_store.legacy_formula_to_curated(
first,
legacy_path="formulas/fascia-pediatrica-1.sql.md",
source_content=first.dump(),
)
second_migration = formula_store.legacy_formula_to_curated(
second,
legacy_path="formulas/fascia-pediatrica-2.sql.md",
source_content=second.dump(),
)
assert first_migration is not None
assert second_migration is not None
assert first_migration.id != second_migration.id
assert first_migration.id.startswith("evidence:fascia-pediatrica-")
assert second_migration.id.startswith("evidence:fascia-pediatrica-")
def test_session_formula_proposal_is_versioned_and_is_not_published_evidence(tmp_path):
linking = SchemaLinking(
question="Conta i pazienti pediatrici",
@@ -199,12 +60,18 @@ def test_session_formula_proposals_require_an_unretracted_positive_f4_decision(t
)
decisions = [
DecisionRecord(
seq=3, ts=datetime(2026, 8, 25, tzinfo=UTC), type="concept_formula_approved",
subject="phase:4", detail="approvata",
seq=3,
ts=datetime(2026, 8, 25, tzinfo=UTC),
type="concept_formula_approved",
subject="phase:4",
detail="approvata",
),
DecisionRecord(
seq=4, ts=datetime(2026, 8, 25, tzinfo=UTC), type="concept_formula_rejected",
subject="phase:4", detail="rifiutata",
seq=4,
ts=datetime(2026, 8, 25, tzinfo=UTC),
type="concept_formula_rejected",
subject="phase:4",
detail="rifiutata",
),
]
+24 -60
View File
@@ -1,63 +1,14 @@
"""D14b wiring: status=auto, search_formulas, and evidence loader skips formula files.
"""L1: Formula Evidence search uses only the typed, published Evidence surface."""
Completes the formula layer beyond the store: the `auto` status the spec requires
(§4.7.2), the concept-substring retrieval that `tht search find --kind formula` uses,
and the guarantee that load_evidence_dir does NOT choke on *.sql.md formula files when
they live under the evidence root.
"""
import json
from types import SimpleNamespace
from typer.testing import CliRunner
from tht.cli import app, search_cmd
from tht.evidence import formula_store
from tht.evidence.formula_store import ConceptFormula, save_formula, search_formulas
from tht.evidence.model import EvidenceDoc, load_evidence_dir
from tht.evidence.search import EvidenceSearchOutcome
def test_status_auto_is_valid():
f = ConceptFormula(concept="x", sql="SELECT 1", status="auto")
assert f.status == "auto"
# round-trips through parse/dump
assert ConceptFormula.parse(f.dump()).status == "auto"
def test_search_formulas_substring_case_insensitive(tmp_path):
save_formula(tmp_path, ConceptFormula(concept="fascia pediatrica", sql="SELECT 1"))
save_formula(tmp_path, ConceptFormula(concept="indice di Charlson", sql="SELECT 2"))
hits = search_formulas(tmp_path, "PEDIATRICA")
assert len(hits) == 1
assert hits[0].concept == "fascia pediatrica"
assert search_formulas(tmp_path, "charlson")[0].concept == "indice di Charlson"
def test_load_evidence_dir_skips_formula_files(tmp_path):
# a real evidence doc + a formula file under the same root
(tmp_path / "ev1.md").write_text(
"---\nid: ev1\ntitle: T\n---\nbody text\n"
)
save_formula(tmp_path, ConceptFormula(concept="ablazione", sql="SELECT 1"))
docs = load_evidence_dir(tmp_path)
ids = [d.id for d in docs]
assert ids == ["ev1"] # the .sql.md formula file is skipped, no crash
assert all(isinstance(d, EvidenceDoc) for d in docs)
def test_unreviewed_legacy_formulas_cannot_become_curated_evidence():
for status in ("auto", "draft"):
formula = ConceptFormula(
concept="fascia pediatrica",
columns=["clinical.patient.birth_date"],
sql="CASE WHEN age < 18 THEN 'pediatrica' ELSE 'adulta' END",
status=status,
)
assert formula_store.legacy_formula_to_curated(
formula, legacy_path="formulas/pediatric-1.sql.md",
) is None
def test_formula_search_uses_typed_evidence_with_a_formula_constraint():
class Searcher:
vector_generation = "gen:" + "a" * 32
@@ -68,8 +19,11 @@ def test_formula_search_uses_typed_evidence_with_a_formula_constraint():
def search(self, embedding, **kwargs):
self.calls.append((embedding, kwargs))
return [SimpleNamespace(
id="fragment:formula", similarity=0.9, content="formula excerpt",
title="Fascia pediatrica", metadata={
id="fragment:formula",
similarity=0.9,
content="formula excerpt",
title="Fascia pediatrica",
metadata={
"evidence_id": "evidence:fascia-pediatrica",
"evidence_kind": "formula",
"document_id": "doc:formula",
@@ -89,16 +43,26 @@ def test_formula_search_uses_typed_evidence_with_a_formula_constraint():
)
assert outcome.status == "available"
assert [result.evidence_id for result in outcome.results] == ["evidence:fascia-pediatrica"]
assert [result.evidence_id for result in outcome.results] == [
"evidence:fascia-pediatrica",
]
assert searcher.calls[0][1]["metadata_filter"]["required_kinds"] == ["formula"]
def test_formula_search_json_is_pristine_while_human_output_warns_about_legacy_store(monkeypatch):
monkeypatch.setattr(search_cmd, "_load_config_or_exit", lambda _path: SimpleNamespace(embeddings=object()))
def test_formula_search_json_is_pristine_and_human_output_uses_only_published_evidence(
monkeypatch,
):
monkeypatch.setattr(
search_cmd,
"_load_config_or_exit",
lambda _path: SimpleNamespace(embeddings=object()),
)
monkeypatch.setattr(search_cmd, "workspace_id_for_config", lambda _cfg, _path: "workspace-a")
monkeypatch.setattr(search_cmd, "search_formula_evidence", lambda *args, **kwargs: EvidenceSearchOutcome(
"available", "gen:" + "a" * 32,
))
monkeypatch.setattr(
search_cmd,
"search_formula_evidence",
lambda *args, **kwargs: EvidenceSearchOutcome("available", "gen:" + "a" * 32),
)
monkeypatch.setattr("tht.cli.vector_cmd.require_vector_cfg", lambda _cfg: None)
monkeypatch.setattr("tht.cli.vector_cmd.open_searcher", lambda _cfg: object())
monkeypatch.setattr("tht.cli.vector_cmd.make_embedder", lambda _cfg: object())
@@ -114,6 +78,6 @@ def test_formula_search_json_is_pristine_while_human_output_warns_about_legacy_s
assert json_result.exit_code == 0
assert json.loads(json_result.stdout) == []
assert "ATTENZIONE" not in json_result.stdout
assert human_result.exit_code == 0
assert "ATTENZIONE" in human_result.output
assert "ATTENZIONE" not in human_result.output
assert "Nessuna formula" in human_result.output
+33
View File
@@ -14,6 +14,7 @@ from tht.cli.config_cmd import CONFIG_OPT
from tht.evidence import (
EvidencePreparationError,
PiEvidenceRestructurer,
migrate_workspace_evidence,
prepare_workspace_evidence,
resolve_workspace_evidence,
validate_workspace_evidence,
@@ -174,6 +175,38 @@ def validate_cmd(
raise typer.Exit(code=1)
@evidence_app.command("migrate")
def migrate_cmd(
workspace_root: Path,
json_output: Annotated[bool, typer.Option("--json", help="Write machine JSON to stdout.")] = False,
) -> None:
"""Rewrite legacy Curated units as readable Markdown without model calls."""
root = _canonical_worktree(workspace_root)
try:
report = migrate_workspace_evidence(root)
except EvidencePreparationError as error:
_emit({
"schemaVersion": 1,
"operation": "evidence_migrate",
"status": "failed",
"code": error.code,
}, json_output)
raise typer.Exit(code=1) from error
_emit({
"schemaVersion": 1,
"operation": "evidence_migrate",
"status": "migrated",
"migrated": list(report.migrated),
"unchanged": list(report.unchanged),
"findings": _findings_payload(report.findings),
}, json_output)
if report.findings:
if all(finding.code in {"orphaned_unit", "unresolved_review_item"}
for finding in report.findings):
raise typer.Exit(code=3)
raise typer.Exit(code=1)
@evidence_app.command("evaluate")
def evaluate_cmd(
workspace_root: Path,
-6
View File
@@ -196,12 +196,6 @@ def search_cmd(
for result in outcome.results
], ensure_ascii=False, indent=2))
return
typer.secho(
"ATTENZIONE: lo store formule legacy non viene più consultato; "
"sono disponibili solo Formula Evidence pubblicate.",
fg=typer.colors.YELLOW,
err=True,
)
if not outcome.results:
typer.secho(f"Nessuna formula per '{keyword}'.", fg=typer.colors.YELLOW)
return
+2 -1
View File
@@ -52,7 +52,8 @@ DecisionType = Literal[
"value_grounded",
# D14b: formula di concetto approvata/rifiutata dal reviewer. subject = "phase:4",
# detail = il concetto (es. "fascia pediatrica"), rationale = la/e colonna/e o il motivo.
# retrieve_formula restituisce i candidati; queste decisioni registrano la scelta.
# La ricerca nelle Formula Evidence pubblicate restituisce i candidati; queste decisioni
# registrano la scelta della proposta nella sessione.
"concept_formula_approved",
"concept_formula_rejected",
]
+4
View File
@@ -3,6 +3,7 @@
from tht.evidence.acquisition import acquire, discover
from tht.evidence.authoring import (
EvidenceManifest,
EvidenceMigrationReport,
EvidencePreparationError,
EvidencePreparationReport,
EvidenceResolutionReport,
@@ -14,6 +15,7 @@ from tht.evidence.authoring import (
ValidationReport,
dump_manifest,
load_manifest,
migrate_workspace_evidence,
prepare_workspace_evidence,
resolve_workspace_evidence,
validate_workspace_evidence,
@@ -60,6 +62,7 @@ __all__ = [
"CuratedEvidence",
"EvidenceEmbedder",
"EvidenceManifest",
"EvidenceMigrationReport",
"EvidencePreparationError",
"EvidencePreparationReport",
"EvidenceQueryEmbedder",
@@ -89,6 +92,7 @@ __all__ = [
"dump_manifest",
"load_curated_tree",
"load_manifest",
"migrate_workspace_evidence",
"normalize_aware_datetime",
"parse_curated_markdown",
"prepare_workspace_evidence",
+77 -3
View File
@@ -293,6 +293,15 @@ class EvidencePreparationReport:
model_calls: int
@dataclass(frozen=True)
class EvidenceMigrationReport:
"""Result of a deterministic Curated Evidence presentation-format migration."""
migrated: tuple[str, ...]
unchanged: tuple[str, ...]
findings: tuple[ValidationFinding, ...]
@dataclass(frozen=True)
class EvidenceResolutionReport:
"""The result of one curator-directed Evidence resolution."""
@@ -680,6 +689,48 @@ def prepare_workspace_evidence(
)
def migrate_workspace_evidence(
workspace_root: Path,
*,
git_status: Callable[[Path], tuple[str, ...]] | None = None,
) -> EvidenceMigrationReport:
"""Rewrite v1 Curated units as readable v2 Markdown without changing semantics."""
workspace_root = workspace_root.resolve()
evidence_root = workspace_root / "evidence"
_reject_dirty_authoring_state(workspace_root, git_status or _git_status)
try:
manifest = load_manifest(evidence_root / "manifest.yaml")
documents = load_curated_tree(evidence_root / "curated")
except (OSError, ValidationError, ValueError) as error:
raise EvidencePreparationError("authoring_state_invalid") from error
documents_by_id = {document.id: document for document in documents}
if len(documents_by_id) != len(documents):
raise EvidencePreparationError("duplicate_evidence_id")
migrated = tuple(sorted(
document.id for document in documents if document.schema_version == 1
))
unchanged = tuple(sorted(
document.id for document in documents if document.schema_version == 2
))
if not migrated:
return EvidenceMigrationReport(
migrated=(),
unchanged=unchanged,
findings=validate_workspace_evidence(workspace_root).findings,
)
upgraded = {
evidence_id: document.model_copy(update={"schema_version": 2})
for evidence_id, document in documents_by_id.items()
}
findings = _stage_and_apply_authoring_tree(workspace_root, upgraded, manifest)
return EvidenceMigrationReport(
migrated=migrated,
unchanged=unchanged,
findings=findings,
)
def resolve_workspace_evidence(
workspace_root: Path,
evidence_id: str,
@@ -827,7 +878,7 @@ def _load_resolution_source(evidence_root: Path, source_file: str) -> str:
def _git_status(workspace_root: Path) -> tuple[str, ...]:
result = subprocess.run(
["git", "status", "--porcelain"],
["git", "status", "--porcelain", "--untracked-files=all"],
cwd=workspace_root,
check=False,
capture_output=True,
@@ -835,7 +886,25 @@ def _git_status(workspace_root: Path) -> tuple[str, ...]:
)
if result.returncode != 0:
raise EvidencePreparationError("canonical_git_worktree_required")
return tuple(line for line in result.stdout.splitlines() if line)
prefix_result = subprocess.run(
["git", "rev-parse", "--show-prefix"],
cwd=workspace_root,
check=False,
capture_output=True,
text=True,
)
if prefix_result.returncode != 0:
raise EvidencePreparationError("canonical_git_worktree_required")
prefix = prefix_result.stdout.strip()
entries: list[str] = []
for line in result.stdout.splitlines():
if not line:
continue
path = line[3:] if len(line) > 3 else line
if prefix and path.startswith(prefix):
line = line[:3] + path.removeprefix(prefix)
entries.append(line)
return tuple(entries)
def _reject_dirty_authoring_state(
@@ -934,7 +1003,11 @@ def _candidate_to_evidence(
source_file: str,
source_hash: str,
) -> CuratedEvidence:
data = candidate.model_dump(mode="json", exclude={"existing_id", "supporting_excerpts"})
data = candidate.model_dump(
mode="json",
exclude={"schema_version", "existing_id", "supporting_excerpts"},
)
data["schema_version"] = 2
data["id"] = evidence_id
data["provenance"] = {
"source_file": source_file,
@@ -955,6 +1028,7 @@ def _unsupported_unit(
message="The current source no longer supports this Evidence unit.",
),)
return evidence.model_copy(update={
"schema_version": 2,
"provenance": evidence.provenance.model_copy(update={
"source_file": source_file,
"source_sha256": source_hash,
+447 -6
View File
@@ -226,7 +226,7 @@ _EVIDENCE_ID = re.compile(r"^evidence:[a-z0-9]+(?:-[a-z0-9]+)*$")
class CuratedEvidence(StrictModel):
schema_version: Literal[1]
schema_version: Literal[1, 2]
id: str
title: str
kind: EvidenceKind
@@ -247,6 +247,439 @@ class CuratedEvidence(StrictModel):
return self
_V2_LABELS = {
"en": {
"column": "Column",
"columns": "Columns",
"concept": "Concept",
"definition": "Definition",
"description": "Description",
"empty": "No items",
"input": "Input",
"interpretation": "Interpretation",
"label": "Label",
"output": "Output",
"question": "Question",
"review_items": "Review items",
"field": "Field",
"rule": "Rule",
"sql": "SQL",
"supporting_excerpts": "Supporting excerpts",
"synonyms": "Synonyms",
"tables": "Tables",
"url": "URL",
"value": "Value",
"values": "Values",
"meaning": "Meaning",
"variants": "Variants",
},
"it": {
"column": "Colonna",
"columns": "Colonne",
"concept": "Concetto",
"definition": "Definizione",
"description": "Descrizione",
"empty": "Nessun elemento",
"input": "Input",
"interpretation": "Interpretazione",
"label": "Etichetta",
"output": "Output",
"question": "Domanda",
"review_items": "Elementi da rivedere",
"field": "Campo",
"rule": "Regola",
"sql": "SQL",
"supporting_excerpts": "Estratti di supporto",
"synonyms": "Sinonimi",
"tables": "Tabelle",
"url": "URL",
"value": "Valore",
"values": "Valori",
"meaning": "Significato",
"variants": "Varianti",
},
}
_V2_FIELD = re.compile(
r"<!-- tht:field:([a-z_]+) -->\n(.*?)\n<!-- /tht:field:\1 -->",
re.DOTALL,
)
_V2_EXCERPT_SEPARATOR = "<!-- tht:excerpt-separator -->"
_V2_EMPTY_LIST = "<!-- tht:empty-list -->"
_V2_REVIEW_SEPARATOR = "<!-- tht:review-separator -->"
_V2_REVIEW_FIELD = "<!-- tht:review-field -->"
def _v2_labels(language: str) -> dict[str, str]:
return _V2_LABELS["it" if language.lower().startswith("it") else "en"]
def _render_v2_field(name: str, label: str, content: str) -> str:
closing_marker = f"<!-- /tht:field:{name} -->"
if "<!-- tht:field:" in content or "<!-- /tht:field:" in content:
raise ValueError(f"curated evidence {name} contains a reserved marker")
return (
f"<!-- tht:field:{name} -->\n"
f"## {label}\n\n"
f"{content}\n"
f"{closing_marker}"
)
def _render_v2_excerpt(value: str) -> str:
if _V2_EXCERPT_SEPARATOR in value:
raise ValueError("supporting excerpt contains a reserved marker")
quoted = "\n".join(">" if not line else f"> {line}" for line in value.split("\n"))
return quoted
def _render_v2_list(
values: tuple[str, ...], *, code: bool = False, empty_label: str = "No items",
) -> str:
if not values:
return f"{_V2_EMPTY_LIST}\n_{empty_label}._"
if any("\n" in value for value in values):
raise ValueError("curated evidence list values must be single-line")
if code and any("`" in value for value in values):
raise ValueError("curated evidence code values must not contain backticks")
return "\n".join(f"- `{value}`" if code else f"- {value}" for value in values)
def _parse_v2_list(
value: str, *, code: bool = False, empty_label: str = "No items",
) -> tuple[str, ...]:
if value == f"{_V2_EMPTY_LIST}\n_{empty_label}._":
return ()
parsed: list[str] = []
for line in value.split("\n"):
if not line.startswith("- "):
raise ValueError("curated evidence list is malformed")
item = line[2:]
if code:
if len(item) < 2 or not item.startswith("`") or not item.endswith("`"):
raise ValueError("curated evidence code list is malformed")
item = item[1:-1]
parsed.append(item)
return tuple(parsed)
def _escape_v2_table_value(value: str) -> str:
return value.replace("\\", "\\\\").replace("|", "\\|").replace("\n", "\\n")
def _unescape_v2_table_value(value: str) -> str:
output: list[str] = []
index = 0
while index < len(value):
if value[index] != "\\":
output.append(value[index])
index += 1
continue
if index + 1 >= len(value):
raise ValueError("curated evidence table escape is malformed")
escaped = value[index + 1]
if escaped not in {"\\", "|", "n"}:
raise ValueError("curated evidence table escape is malformed")
output.append("\n" if escaped == "n" else escaped)
index += 2
return "".join(output)
def _render_v2_values(values: dict[str, str], labels: dict[str, str]) -> str:
if any("`" in value for value in values):
raise ValueError("curated evidence enum values must not contain backticks")
rows = [
f"| {labels['value']} | {labels['meaning']} |",
"| --- | --- |",
]
if not values:
rows.append(f"| _{labels['empty']}._ | |")
return "\n".join(rows)
rows.extend(
f"| `{_escape_v2_table_value(value)}` | {_escape_v2_table_value(meaning)} |"
for value, meaning in sorted(values.items())
)
return "\n".join(rows)
def _parse_v2_values(value: str, labels: dict[str, str]) -> dict[str, str]:
lines = value.split("\n")
if (
len(lines) < 3
or lines[0] != f"| {labels['value']} | {labels['meaning']} |"
or lines[1] != "| --- | --- |"
):
raise ValueError("curated evidence values table is malformed")
if lines[2:] == [f"| _{labels['empty']}._ | |"]:
return {}
parsed: dict[str, str] = {}
for line in lines[2:]:
if not line.startswith("| ") or not line.endswith(" |"):
raise ValueError("curated evidence values table is malformed")
cells = re.split(r"(?<!\\)\s\|\s", line[2:-2], maxsplit=1)
if len(cells) != 2 or not cells[0].startswith("`") or not cells[0].endswith("`"):
raise ValueError("curated evidence values table is malformed")
key = _unescape_v2_table_value(cells[0][1:-1])
if key in parsed:
raise ValueError("curated evidence enum value appears more than once")
parsed[key] = _unescape_v2_table_value(cells[1])
return parsed
def _render_v2_payload(value: CuratedEvidence, labels: dict[str, str]) -> list[str]:
payload = value.payload
if value.kind == "glossary":
fields = [_render_v2_field("definition", labels["definition"], payload.definition)]
if payload.synonyms:
fields.append(_render_v2_field(
"synonyms", labels["synonyms"], _render_v2_list(payload.synonyms),
))
if payload.variants:
fields.append(_render_v2_field(
"variants", labels["variants"], _render_v2_list(payload.variants),
))
return fields
if value.kind == "domain":
return [_render_v2_field("rule", labels["rule"], payload.rule)]
if value.kind == "enum":
return [
_render_v2_field("column", labels["column"], f"`{payload.column}`"),
_render_v2_field("values", labels["values"], _render_v2_values(payload.values, labels)),
]
if value.kind == "example":
return [
_render_v2_field("question", labels["question"], payload.question),
_render_v2_field("interpretation", labels["interpretation"], payload.interpretation),
]
if value.kind == "mapping":
return [
_render_v2_field("concept", labels["concept"], payload.concept),
_render_v2_field("tables", labels["tables"], _render_v2_list(
payload.tables, code=True, empty_label=labels["empty"],
)),
_render_v2_field("columns", labels["columns"], _render_v2_list(
payload.columns, code=True, empty_label=labels["empty"],
)),
]
if value.kind == "normalization":
return [
_render_v2_field("input", labels["input"], payload.input),
_render_v2_field("output", labels["output"], payload.output),
_render_v2_field("rule", labels["rule"], payload.rule),
]
if value.kind == "formula":
if "```" in payload.sql:
raise ValueError("curated evidence SQL contains a reserved Markdown fence")
return [
_render_v2_field("concept", labels["concept"], payload.concept),
_render_v2_field("columns", labels["columns"], _render_v2_list(
payload.columns, code=True, empty_label=labels["empty"],
)),
_render_v2_field("sql", labels["sql"], f"```sql\n{payload.sql}\n```"),
]
if value.kind == "reference":
return [
_render_v2_field("label", labels["label"], payload.label),
_render_v2_field("url", labels["url"], f"<{payload.url}>"),
_render_v2_field("description", labels["description"], payload.description),
]
raise ValueError(f"unsupported curated evidence kind {value.kind}")
def _render_v2_review_items(value: CuratedEvidence, labels: dict[str, str]) -> str:
rendered: list[str] = []
for item in value.review_items:
if "`" in item.code or (item.field is not None and "`" in item.field):
raise ValueError("curated evidence review identifiers must not contain backticks")
if _V2_REVIEW_SEPARATOR in item.message or _V2_REVIEW_FIELD in item.message:
raise ValueError("curated evidence review message contains a reserved marker")
block = f"### `{item.code}`\n\n{item.message}"
if item.field is not None:
block += f"\n\n{_V2_REVIEW_FIELD}\n**{labels['field']}:** `{item.field}`"
rendered.append(block)
return f"\n{_V2_REVIEW_SEPARATOR}\n".join(rendered)
def _render_v2_body(value: CuratedEvidence) -> str:
if "\n" in value.title:
raise ValueError("curated evidence title must be single-line in v2")
labels = _v2_labels(value.language)
fields = [
*_render_v2_payload(value, labels),
_render_v2_field(
"supporting_excerpts",
labels["supporting_excerpts"],
f"\n{_V2_EXCERPT_SEPARATOR}\n".join(
_render_v2_excerpt(excerpt)
for excerpt in value.provenance.supporting_excerpts
),
),
]
if value.review_items:
fields.append(_render_v2_field(
"review_items",
labels["review_items"],
_render_v2_review_items(value, labels),
))
return f"# {value.title}\n\n" + "\n\n".join(fields) + "\n"
def _parse_v2_field_content(name: str, block: str) -> str:
try:
heading, content = block.split("\n\n", 1)
except ValueError as error:
raise ValueError(f"curated evidence field {name} is malformed") from error
if not heading.startswith("## ") or not content:
raise ValueError(f"curated evidence field {name} is malformed")
return content
def _parse_v2_excerpts(block: str) -> tuple[str, ...]:
excerpts: list[str] = []
for raw_excerpt in block.split(f"\n{_V2_EXCERPT_SEPARATOR}\n"):
lines = raw_excerpt.split("\n")
if any(line != ">" and not line.startswith("> ") for line in lines):
raise ValueError("curated evidence supporting excerpt is malformed")
excerpts.append("\n".join(line[2:] if line.startswith("> ") else "" for line in lines))
if not excerpts:
raise ValueError("curated evidence supporting excerpts are malformed")
return tuple(excerpts)
def _parse_inline_code(value: str, name: str) -> str:
if len(value) < 2 or not value.startswith("`") or not value.endswith("`"):
raise ValueError(f"curated evidence field {name} must be inline code")
return value[1:-1]
def _parse_v2_payload(
kind: str, fields: dict[str, str], labels: dict[str, str],
) -> tuple[dict, set[str]]:
if kind == "glossary":
expected = {"definition"}
payload: dict = {"definition": fields.get("definition")}
for name in ("synonyms", "variants"):
if name in fields:
expected.add(name)
payload[name] = _parse_v2_list(fields[name], empty_label=labels["empty"])
else:
payload[name] = ()
return payload, expected
if kind == "domain":
return {"rule": fields.get("rule")}, {"rule"}
if kind == "enum":
return {
"column": _parse_inline_code(fields.get("column", ""), "column"),
"values": _parse_v2_values(fields.get("values", ""), labels),
}, {"column", "values"}
if kind == "example":
return {
"question": fields.get("question"),
"interpretation": fields.get("interpretation"),
}, {"question", "interpretation"}
if kind == "mapping":
return {
"concept": fields.get("concept"),
"tables": _parse_v2_list(
fields.get("tables", ""), code=True, empty_label=labels["empty"],
),
"columns": _parse_v2_list(
fields.get("columns", ""), code=True, empty_label=labels["empty"],
),
}, {"concept", "tables", "columns"}
if kind == "normalization":
return {
"input": fields.get("input"),
"output": fields.get("output"),
"rule": fields.get("rule"),
}, {"input", "output", "rule"}
if kind == "formula":
sql = fields.get("sql", "")
if not sql.startswith("```sql\n") or not sql.endswith("\n```"):
raise ValueError("curated evidence SQL block is malformed")
return {
"concept": fields.get("concept"),
"columns": _parse_v2_list(
fields.get("columns", ""), code=True, empty_label=labels["empty"],
),
"sql": sql.removeprefix("```sql\n").removesuffix("\n```"),
}, {"concept", "columns", "sql"}
if kind == "reference":
url = fields.get("url", "")
if not url.startswith("<") or not url.endswith(">"):
raise ValueError("curated evidence reference URL is malformed")
return {
"label": fields.get("label"),
"url": url[1:-1],
"description": fields.get("description"),
}, {"label", "url", "description"}
raise ValueError("curated evidence body kind is unsupported")
def _parse_v2_review_items(value: str) -> tuple[ReviewItem, ...]:
items: list[ReviewItem] = []
for raw_item in value.split(f"\n{_V2_REVIEW_SEPARATOR}\n"):
try:
heading, detail = raw_item.split("\n\n", 1)
except ValueError as error:
raise ValueError("curated evidence review item is malformed") from error
if not heading.startswith("### `") or not heading.endswith("`"):
raise ValueError("curated evidence review item code is malformed")
code = heading.removeprefix("### `").removesuffix("`")
field = None
marker = f"\n\n{_V2_REVIEW_FIELD}\n"
if marker in detail:
message, rendered_field = detail.split(marker, 1)
match = re.fullmatch(r"\*\*[^*]+:\*\* `([^`]+)`", rendered_field)
if match is None:
raise ValueError("curated evidence review item field is malformed")
field = match.group(1)
else:
message = detail
if not code or not message:
raise ValueError("curated evidence review item is malformed")
items.append(ReviewItem(code=code, message=message, field=field))
return tuple(items)
def _parse_v2_body(data: dict, body: str) -> dict:
kind = data.get("kind")
body_owned = {"payload", "review_items"}
if isinstance(kind, str):
body_owned.add(kind)
if body_owned.intersection(data):
raise ValueError("curated evidence v2 frontmatter contains body-owned fields")
title = data.get("title")
if not isinstance(title, str) or not body.startswith(f"# {title}\n"):
raise ValueError("curated evidence body title must match its metadata")
fields: dict[str, str] = {}
for match in _V2_FIELD.finditer(body):
name = match.group(1)
if name in fields:
raise ValueError(f"curated evidence field {name} appears more than once")
fields[name] = _parse_v2_field_content(name, match.group(2))
skeleton = _V2_FIELD.sub("", body).strip()
if skeleton != f"# {title}":
raise ValueError("curated evidence body contains unstructured content")
labels = _v2_labels(str(data.get("language", "")))
payload, payload_fields = _parse_v2_payload(kind, fields, labels)
common_fields = {"supporting_excerpts"}
if "review_items" in fields:
common_fields.add("review_items")
if set(fields) != payload_fields | common_fields:
raise ValueError("curated evidence body fields do not match its kind")
provenance = data.get("provenance")
if not isinstance(provenance, dict) or "supporting_excerpts" in provenance:
raise ValueError("curated evidence v2 provenance is malformed")
provenance["supporting_excerpts"] = _parse_v2_excerpts(fields["supporting_excerpts"])
data["review_items"] = (
_parse_v2_review_items(fields["review_items"])
if "review_items" in fields
else []
)
data["payload"] = payload
return data
def parse_curated_markdown(text: str, *, path: Path | None = None) -> CuratedEvidence:
"""Parse the canonical frontmatter representation of one Curated Evidence unit."""
if not text.startswith("---\n"):
@@ -260,11 +693,14 @@ def parse_curated_markdown(text: str, *, path: Path | None = None) -> CuratedEvi
data = dict(raw)
except (TypeError, ValueError) as error:
raise ValueError("curated evidence frontmatter must be a mapping") from error
if body.strip():
raise ValueError("curated evidence must not contain an ignored body")
kind = data.get("kind")
if "payload" not in data and kind in _PAYLOAD_TYPE_BY_KIND:
data["payload"] = data.pop(kind, None)
if data.get("schema_version") == 2:
data = _parse_v2_body(data, body)
else:
if body.strip():
raise ValueError("curated evidence must not contain an ignored body")
kind = data.get("kind")
if "payload" not in data and kind in _PAYLOAD_TYPE_BY_KIND:
data["payload"] = data.pop(kind, None)
evidence = CuratedEvidence.model_validate(data)
if path is not None:
_validate_kind_directory(path, evidence.kind)
@@ -273,6 +709,11 @@ def parse_curated_markdown(text: str, *, path: Path | None = None) -> CuratedEvi
def dump_curated_markdown(value: CuratedEvidence) -> str:
"""Render canonical frontmatter with a human-readable kind-specific payload key."""
if value.schema_version == 2:
data = value.model_dump(mode="json", exclude={"payload", "review_items"})
data["provenance"].pop("supporting_excerpts")
frontmatter = yaml.safe_dump(data, allow_unicode=True, sort_keys=False)
return f"---\n{frontmatter}---\n{_render_v2_body(value)}"
data = value.model_dump(mode="json", exclude={"payload"})
data[value.kind] = value.payload.model_dump(mode="json")
frontmatter = yaml.safe_dump(data, allow_unicode=True, sort_keys=False)
-202
View File
@@ -1,202 +0,0 @@
"""Legacy ConceptFormula reader and one-way migration into Curated Evidence.
The ``formulas/*.sql.md`` store is retained only for the migration window. Runtime
lookup uses typed, published ``kind=formula`` Evidence instead. A session reviewer
may still approve a formula locally; that is a proposal, not publication.
"""
from __future__ import annotations
import hashlib
import re
import unicodedata
from dataclasses import dataclass
from pathlib import Path
from typing import Literal
import yaml
from pydantic import BaseModel, ValidationError
from tht.evidence.canonical import CuratedEvidence
FORMULAS_SUBDIR = "formulas"
_SUFFIX_RE = re.compile(r"^(.*?)-(\d+)\.sql\.md$")
class ConceptFormula(BaseModel):
concept: str
columns: list[str] = []
sql: str
# auto = sintetizzata dal modello (non ancora rivista); draft = bozza umana;
# reviewed = approvata da un revisore. (spec §4.7.2: status auto/draft/reviewed)
status: Literal["auto", "draft", "reviewed"] = "draft"
sources: list[str] = []
@property
def _slug(self) -> str:
"""ASCII slug for the filename (matches textutil.slugify shape)."""
import unicodedata
text = unicodedata.normalize("NFKD", self.concept).encode("ascii", "ignore").decode()
return re.sub(r"[^a-z0-9_]+", "-", text.lower()).strip("-") or "formula"
def dump(self) -> str:
meta = self.model_dump(exclude={"sql"}, mode="json")
fm = yaml.safe_dump(meta, sort_keys=False, allow_unicode=True)
return f"---\n{fm}---\n{self.sql}\n"
@classmethod
def parse(cls, text: str) -> ConceptFormula:
if not text.startswith("---\n"):
raise ValueError("frontmatter mancante (atteso '---\\n' iniziale)")
try:
_, fm, body = text.split("---\n", 2)
except ValueError as e:
raise ValueError("frontmatter malformato") from e
meta = yaml.safe_load(fm)
if not isinstance(meta, dict):
raise TypeError("frontmatter non valido")
return cls.model_validate({**meta, "sql": body.strip("\n")})
@dataclass(frozen=True)
class LegacyFormulaMigrationFailure:
"""A reviewed legacy formula that must be resolved manually before publication."""
code: Literal["legacy_formula_requires_manual_review"]
legacy_path: str
formula: ConceptFormula
problems: tuple[str, ...]
def _next_path(root: Path, slug: str) -> Path:
"""First free <slug>-<n>.sql.md path under root (n starts at 1)."""
root.mkdir(parents=True, exist_ok=True)
existing = sorted(root.glob(f"{slug}-*.sql.md"))
n = 0
for p in existing:
m = _SUFFIX_RE.match(p.name)
if m:
n = max(n, int(m.group(2)))
return root / f"{slug}-{n + 1}.sql.md"
def save_formula(root: Path | str, formula: ConceptFormula) -> Path:
"""Persist a single concept->formula unit under <root>/formulas/. Returns the
written path. Append-only: each save writes a new file (so competing drafts and
reviewed versions coexist until a curator prunes)."""
root = Path(root)
formulas_dir = root / FORMULAS_SUBDIR
path = _next_path(formulas_dir, formula._slug)
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(formula.dump())
return path
def _load_all(root: Path) -> list[ConceptFormula]:
formulas_dir = root / FORMULAS_SUBDIR
if not formulas_dir.is_dir():
return []
out: list[ConceptFormula] = []
for f in sorted(formulas_dir.glob("*.sql.md")):
try:
out.append(ConceptFormula.parse(f.read_text()))
except ValueError:
continue # malformed file: skip, don't crash retrieval
return out
def retrieve_formula(root: Path | str, concept: str) -> list[ConceptFormula]:
"""All formulas matching `concept` exactly under <root>/formulas/. Empty list if
none (or if the dir is absent). Multiple results mean competing drafts/versions for
the same concept -- the caller (gate) lets the reviewer pick."""
return [f for f in _load_all(Path(root)) if f.concept == concept]
def search_formulas(root: Path | str, query: str) -> list[ConceptFormula]:
"""Read legacy formulas for migration tooling only (case-insensitive concept match)."""
q = query.strip().lower()
return [f for f in _load_all(Path(root)) if q in f.concept.lower()]
def legacy_formula_to_curated(
formula: ConceptFormula,
*,
legacy_path: str,
source_content: str | None = None,
) -> CuratedEvidence | LegacyFormulaMigrationFailure | None:
"""Convert one reviewed legacy formula into its deterministic curated counterpart.
Drafts and model-generated formulas have no global publication status. Their
caller must project them as session-local Formula proposals instead.
"""
if formula.status != "reviewed":
return None
problems: list[str] = []
normalized_source: str | None = None
if source_content is None:
problems.append("original_source_required")
else:
from tht.evidence.authoring import normalize_source_text
normalized_source = normalize_source_text(source_content)
try:
original_formula = ConceptFormula.parse(source_content)
except (TypeError, ValidationError, ValueError, yaml.YAMLError):
problems.append("original_source_invalid")
else:
if original_formula != formula:
problems.append("original_source_mismatch")
source_notes = tuple(formula.sources)
if not source_notes:
problems.append("supporting_excerpts_required")
elif normalized_source is not None and any(
unicodedata.normalize("NFC", note.replace("\r\n", "\n").replace("\r", "\n"))
not in normalized_source
for note in source_notes
):
problems.append("supporting_excerpt_unverified")
if problems:
return LegacyFormulaMigrationFailure(
code="legacy_formula_requires_manual_review",
legacy_path=legacy_path,
formula=formula,
problems=tuple(sorted(problems)),
)
assert normalized_source is not None
source_sha256 = hashlib.sha256(normalized_source.encode("utf-8")).hexdigest()
source_file = legacy_path if legacy_path.startswith("source/") else f"source/{legacy_path}"
# The legacy path is the immutable identity of this unit during migration. Keeping
# its full digest avoids a duplicate public ID when the same concept has reviewed
# competing formulas, while retaining the readable concept slug as the prefix.
legacy_identity = hashlib.sha256(legacy_path.encode("utf-8")).hexdigest()
try:
return CuratedEvidence.model_validate({
"schema_version": 1,
"id": f"evidence:{formula._slug}-{legacy_identity}",
"title": formula.concept[:1].upper() + formula.concept[1:],
"kind": "formula",
"purposes": ["schema_linking", "sql_generation"],
"applies_to": {"concepts": [formula.concept], "columns": formula.columns},
"language": "it",
"provenance": {
"source_file": source_file,
"source_sha256": f"sha256:{source_sha256}",
"supporting_excerpts": source_notes,
},
"review_items": [],
"payload": {
"concept": formula.concept,
"columns": formula.columns,
"sql": formula.sql,
},
})
except ValidationError as error:
return LegacyFormulaMigrationFailure(
code="legacy_formula_requires_manual_review",
legacy_path=legacy_path,
formula=formula,
problems=tuple(sorted(
".".join(str(part) for part in issue["loc"])
for issue in error.errors()
)),
)
-4
View File
@@ -58,9 +58,5 @@ def load_evidence_dir(root: Path) -> list[EvidenceDoc]:
for f in sorted(root.rglob("*.md")):
if f.name.upper().startswith("README"):
continue
# I file formula (concept->SQL, frontmatter diverso) vivono sotto formulas/ con
# estensione .sql.md: non sono EvidenceDoc, li gestisce formula_store (D14b).
if f.name.endswith(".sql.md"):
continue
docs.append(EvidenceDoc.parse(f.read_text(), path=f))
return docs
+15 -2
View File
@@ -1,6 +1,19 @@
from typing import Protocol
from pydantic import BaseModel
from tht.vectorstore.store import VectorStore
from tht.vectorstore.store import VectorHit
class SemanticSearcher(Protocol):
def search(
self,
embedding: list[float],
*,
top_n: int,
kinds: list[str] | None,
query_text: str | None = None,
) -> list[VectorHit]: ...
class SearchResult(BaseModel):
@@ -92,7 +105,7 @@ def combined_search(
keyword: str,
*,
lsh_hits: list[tuple[str, str, str, float]] | None,
store: VectorStore,
store: SemanticSearcher,
embedder,
top: int,
rrf_k: int,
+1 -155
View File
@@ -1,20 +1,11 @@
import hashlib
import json
from dataclasses import dataclass
from sqlalchemy import Engine, text
from tht.vectorstore.records import VectorRecord
def content_hash(content: str) -> str:
return hashlib.sha256(content.encode()).hexdigest()
def _to_vector_literal(vec: list[float]) -> str:
return "[" + ",".join(f"{x:.8f}" for x in vec) + "]"
@dataclass
class SyncStats:
added: int = 0
@@ -35,10 +26,7 @@ class VectorHit:
def hit_from_metadata(similarity: float, metadata: dict | None) -> VectorHit:
"""Ricostruisce un VectorHit dal solo `metadata` (più la similarity). È l'unico modo
disponibile leggendo via REST (`search_similar` ritorna id/similarity/metadata), e viene
usato anche dalla lettura diretta per avere un'unica logica. Tollerante: usa default sui
campi assenti (es. metadata estranei della tabella fake remota)."""
"""Build a transport-neutral hit from the metadata returned by a vector adapter."""
md = metadata or {}
return VectorHit(
id=md.get("record_key", ""),
@@ -49,145 +37,3 @@ def hit_from_metadata(similarity: float, metadata: dict | None) -> VectorHit:
metadata=md,
similarity=float(similarity),
)
class VectorStore:
"""Legacy table-scoped vector store retained for compatibility fixtures.
New operational semantic storage is handled by the Qdrant adapter. This class preserves
the older SQL-table contract used by historical tests and migration checks: `id`
(BIGSERIAL), `embedding vector(N)`, and `metadata jsonb`; the extra columns
(`record_key`, `kind`, `content_hash`) serve only the loader and are not exposed by REST.
"""
def __init__(
self, engine: Engine, schema: str = "vectors", table: str = "records", dim: int = 768
):
self.engine = engine
self.schema = schema
self.dim = dim
self._table = f"{schema}.{table}"
def init_schema(self) -> None:
with self.engine.begin() as conn:
conn.execute(text("CREATE EXTENSION IF NOT EXISTS vector"))
conn.execute(text(f"CREATE SCHEMA IF NOT EXISTS {self.schema}"))
conn.execute(text(f"""
CREATE TABLE IF NOT EXISTS {self._table} (
id bigserial PRIMARY KEY,
record_key text UNIQUE NOT NULL,
kind text NOT NULL,
content_hash text NOT NULL,
metadata jsonb NOT NULL DEFAULT '{{}}',
embedding vector({self.dim}) NOT NULL,
indexed_at timestamptz NOT NULL DEFAULT now()
)
"""))
conn.execute(text(
f"CREATE INDEX IF NOT EXISTS {self._idx('embedding')} ON {self._table} "
f"USING hnsw (embedding vector_cosine_ops) WITH (m = 16, ef_construction = 200)"
))
conn.execute(text(
f"CREATE INDEX IF NOT EXISTS {self._idx('kind')} ON {self._table} (kind)"
))
# GRANT al ruolo di sola lettura della REST, solo se esiste (assente in test/locale).
conn.execute(text(f"""
DO $$ BEGIN
IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'vector_reader') THEN
EXECUTE 'GRANT SELECT ON {self._table} TO vector_reader';
END IF;
END $$;
"""))
def _idx(self, suffix: str) -> str:
return f"{self._table.replace('.', '_')}_{suffix}_idx"
def clear(self) -> None:
with self.engine.begin() as conn:
conn.execute(text(f"DELETE FROM {self._table}"))
def existing_hashes(self, kinds: set[str]) -> dict[str, str]:
q = text(
f"SELECT record_key, content_hash FROM {self._table} WHERE kind = ANY(:kinds)"
)
with self.engine.connect() as conn:
return dict(conn.execute(q, {"kinds": list(kinds)}).fetchall())
def sync(self, records: list[VectorRecord], embedder, kinds: set[str]) -> SyncStats:
"""Allinea l'indice ai record correnti (per i kind dati): embedda solo il nuovo
o il modificato, elimina cio' che non esiste piu'."""
stats = SyncStats()
existing = self.existing_hashes(kinds)
current_ids = {r.id for r in records}
to_embed: list[VectorRecord] = []
for r in records:
h = content_hash(r.content)
if r.id not in existing:
to_embed.append(r)
stats.added += 1
elif existing[r.id] != h:
to_embed.append(r)
stats.updated += 1
else:
stats.unchanged += 1
vectors = embedder.embed_documents([r.content for r in to_embed]) if to_embed else []
upsert = text(f"""
INSERT INTO {self._table}
(record_key, kind, content_hash, metadata, embedding)
VALUES
(:record_key, :kind, :content_hash, CAST(:metadata AS jsonb),
CAST(:embedding AS vector))
ON CONFLICT (record_key) DO UPDATE SET
kind = EXCLUDED.kind, content_hash = EXCLUDED.content_hash,
metadata = EXCLUDED.metadata, embedding = EXCLUDED.embedding,
indexed_at = now()
""")
stale = [i for i in existing if i not in current_ids]
with self.engine.begin() as conn:
for r, vec in zip(to_embed, vectors):
conn.execute(upsert, {
"record_key": r.id, "kind": r.kind,
"content_hash": content_hash(r.content),
"metadata": json.dumps(_pack_metadata(r)),
"embedding": _to_vector_literal(vec),
})
if stale:
conn.execute(
text(f"DELETE FROM {self._table} WHERE record_key = ANY(:ids)"),
{"ids": stale},
)
stats.deleted = len(stale)
return stats
def search(
self, query_vec: list[float], top_n: int = 10, kinds: list[str] | None = None
) -> list[VectorHit]:
where = "WHERE kind = ANY(:kinds)" if kinds else ""
q = text(f"""
SELECT metadata, 1 - (embedding <=> CAST(:q AS vector)) AS similarity
FROM {self._table}
{where}
ORDER BY embedding <=> CAST(:q AS vector)
LIMIT :top_n
""")
params: dict = {"q": _to_vector_literal(query_vec), "top_n": top_n}
if kinds:
params["kinds"] = kinds
with self.engine.connect() as conn:
rows = conn.execute(q, params).fetchall()
return [hit_from_metadata(r.similarity, r.metadata) for r in rows]
def _pack_metadata(r: VectorRecord) -> dict:
"""Impacchetta nel `metadata` (unica colonna letta via REST) tutta la semantica Thoth."""
return {
"kind": r.kind,
"ref": r.ref,
"record_key": r.id,
"title": r.title,
"content": r.content,
**r.metadata,
}
+22 -22
View File
@@ -1,5 +1,5 @@
site_name: ThothII Docs
site_description: Documentazione funzionale, tecnica e operativa di ThothII
site_description: Functional, technical, and operational documentation for ThothII
site_url: https://git.tylconsulting.it/thothii-docs/
docs_dir: docs
site_dir: site
@@ -44,26 +44,26 @@ markdown_extensions:
alternate_style: true
nav:
- Home: index.md
- Guida utente: guida-utente.md
- DWH REST per installazione:
- Server dwh-auth: install/dwh-auth-server.md
- Enrollment client DWH: install/dwh-auth-client-enrollment.md
- TLS DWH REST: install/dwh-auth-tls.md
- Contratti e CLI:
- CLI workspace preprocessing: contracts/workspace-preprocessing-cli.md
- Contratto tht–DWH: contracts/tht-dwh.md
- Contratto Evidence workspace v3: contracts/workspace-evidence-v3.md
- ThothII (Documentazione Tecnica):
- Panoramica Architettura: architecture/overview.md
- Componenti, moduli e flussi: architecture/components.md
- User guide: guida-utente.md
- DWH REST per installation:
- dwh-auth server: install/dwh-auth-server.md
- DWH client enrollment: install/dwh-auth-client-enrollment.md
- DWH REST TLS: install/dwh-auth-tls.md
- Contracts and CLI:
- Workspace preprocessing CLI: contracts/workspace-preprocessing-cli.md
- tht–DWH contract: contracts/tht-dwh.md
- Workspace Evidence v3 contract: contracts/workspace-evidence-v3.md
- ThothII technical documentation:
- Architecture overview: architecture/overview.md
- Components, modules, and flows: architecture/components.md
- Evidence: evidence.md
- Autenticazione: architecture/authentication.md
- Installazione autenticazione locale: install/authentication-local.md
- OIDC generico: install/authentication-oidc.md
- Authentication: architecture/authentication.md
- Local authentication installation: install/authentication-local.md
- Generic OIDC: install/authentication-oidc.md
- Authentik: install/authentik.md
- Installazione Docker (4 contesti): installazione-docker-4-contesti.md
- Gestione delle memory: gestione-memory.md
- Skill operative: skills.md
- Disambiguazione iniziale: disambiguazione-iniziale.md
- Considerazioni Generali:
- Configurazione dei modelli in Pi: general/pi-configuration.md
- Docker installation (4 contexts): installazione-docker-4-contesti.md
- Memory management: gestione-memory.md
- Operating skills: skills.md
- Initial disambiguation: disambiguazione-iniziale.md
- General topics:
- Pi model configuration: general/pi-configuration.md
-32
View File
@@ -1,32 +0,0 @@
#!/bin/sh
set -eu
validate_secret_file() {
path="$1"
label="$2"
if [ -L "$path" ] || [ ! -f "$path" ] || [ ! -r "$path" ]; then
echo "$label must be a readable regular file, not a symlink: $path" >&2
exit 2
fi
mode=$(stat -c '%a' "$path" 2>/dev/null || stat -f '%Lp' "$path" 2>/dev/null) || {
echo "cannot inspect permissions for $label: $path" >&2
exit 2
}
if [ $((0$mode & 077)) -ne 0 ]; then
echo "$label must not be readable or writable by group/other users: $path" >&2
exit 2
fi
}
read_secret_file() {
path="$1"
label="$2"
validate_secret_file "$path" "$label"
value=$(tr -d '\r' <"$path")
case "$value" in
*'
'*) echo "$label must contain exactly one line" >&2; exit 2 ;;
esac
[ -n "$value" ] || { echo "$label must not be empty" >&2; exit 2; }
printf '%s' "$value"
}
+13 -2
View File
@@ -58,13 +58,24 @@ from pathlib import Path
settings = json.loads(Path("deploy/pi/settings.json").read_text())
assert settings["enabledModels"] == [
"zai/glm-5.2",
"zai/glm-5.3",
"deepseek/deepseek-v4-flash",
"deepseek/deepseek-v4-pro",
"aritmolab/qwen3.6-35b-a3b",
"local-qwen/qwen3.6-35b-a3b",
]
models = json.loads(Path("deploy/pi/models.json").read_text())
qwen = models["providers"]["local-qwen"]
assert qwen["api"] == "openai-completions"
assert qwen["models"][0]["id"] == "qwen3.6-35b-a3b"
assert qwen["models"][0]["compat"]["maxTokensField"] == "max_tokens"
assert "aritmolab" not in models["providers"]
PY
if grep -R -n 'registerProvider' harness/.pi/extensions; then
echo "Pi model providers must be declared in deploy/pi/models.json, not in code" >&2
exit 1
fi
grep -q '^ARG PI_VERSION=0.80.3$' docker/core.Dockerfile
grep -q '^PI_AUTH_FILE=/absolute/path/to/pi-auth.json$' deploy/env/local.env.example
grep -q '^PI_AUTH_FILE=/absolute/path/to/pi-auth.json$' deploy/env/server.env.example