diff --git a/PROJECT_STATE.md b/PROJECT_STATE.md index 226d80b6..61f0a55b 100644 --- a/PROJECT_STATE.md +++ b/PROJECT_STATE.md @@ -226,6 +226,13 @@ - **Engine:** `evidencePolicy` no longer stops filesystem sources (`evidence_materialization_required` retired); `preprocess evidence`/`preprocess run` operate on the materialized root. Evidence Qdrant records remain revision-scoped; corpus ACTIVE is revision-qualified. HTTP/S3 Evidence is unchanged. +- **Curated-only runtime contract (Evidence schema v2):** the new authoring layout preserves the + complete commit-addressed `evidence/` tree (`source/`, `curated/`, manifest and evaluation files), + while the rendered filesystem acquisition default is only `curated/**/*.md`. Version 2 rejects + source or mixed source/curated runtime patterns; legacy Evidence version 1 retains its explicit + safe-pattern compatibility. The curator validates before merge and the runtime validates the + pinned curated corpus before indexing. The existing unnamed dense vector remains intact while + Evidence may add `bm25`/`idf` additively; no runtime operation writes the authoring repository. - **Retention:** materialized roots live inside the commit-addressed snapshot directory, so they are retained while pinned and removed by the existing snapshot retention scan when unreferenced. - **Key files:** `backend/src/workspaces/evidence-materialization.ts` (+test), diff --git a/backend/src/workspaces/schema.ts b/backend/src/workspaces/schema.ts index 666fae3e..9ba9d75d 100644 --- a/backend/src/workspaces/schema.ts +++ b/backend/src/workspaces/schema.ts @@ -115,6 +115,7 @@ export type EvidenceSource = }; export interface WorkspaceEvidence { + schema_version: 1 | 2; source: EvidenceSource; policy: EvidencePolicy; } @@ -246,10 +247,10 @@ const evidencePattern = z.string().refine(isSafeEvidencePattern, { const filesystemEvidenceSourceSchema = z.object({ type: z.literal("filesystem"), uri: z.string(), - patterns: z.array(evidencePattern).min(1).default(["**/*.md"]), + patterns: z.array(evidencePattern).min(1).optional(), max_bytes: positiveSafeInteger.default(10 * 1024 * 1024), }).strict().superRefine((source, context) => { - if (new Set(source.patterns).size !== source.patterns.length) { + if (source.patterns !== undefined && new Set(source.patterns).size !== source.patterns.length) { context.addIssue({ code: "custom", path: ["patterns"], message: "evidence patterns must not repeat" }); } }); @@ -323,12 +324,39 @@ const evidencePolicySchema = z.object({ retain_published_generations: positiveSafeInteger.default(3), }).strict(); const workspaceEvidenceSchema = z.object({ + schema_version: z.union([z.literal(1), z.literal(2)]).default(1), source: evidenceSourceSchema, policy: evidencePolicySchema.default({ max_chunk_chars: 4_000, retain_published_generations: 3, }), -}).strict(); +}).strict().superRefine((evidence, context) => { + if (evidence.schema_version !== 2 || evidence.source.type !== "filesystem") return; + const patterns = evidence.source.patterns ?? ["curated/**/*.md"]; + const selectsSource = patterns.some((pattern) => pattern === "source" || pattern.startsWith("source/")); + const selectsCurated = patterns.some((pattern) => pattern === "curated" || pattern.startsWith("curated/")); + if (selectsSource && selectsCurated) { + context.addIssue({ + code: "custom", + path: ["source", "patterns"], + message: "schema-versioned filesystem Evidence patterns cannot span source and curated", + }); + } else if (!patterns.every((pattern) => pattern === "curated" || pattern.startsWith("curated/"))) { + context.addIssue({ + code: "custom", + path: ["source", "patterns"], + message: "schema-versioned filesystem Evidence patterns must acquire curated documents only", + }); + } +}).transform((evidence) => ({ + ...evidence, + source: evidence.source.type !== "filesystem" || evidence.source.patterns !== undefined + ? evidence.source + : { + ...evidence.source, + patterns: evidence.schema_version === 2 ? ["curated/**/*.md"] : ["**/*.md"], + }, +})); function unique(values: readonly T[], context: z.RefinementCtx, path: PropertyKey[]) { if (new Set(values).size !== values.length) { diff --git a/backend/test/workspace-runtime-renderer.test.ts b/backend/test/workspace-runtime-renderer.test.ts index 8a151f38..28b80145 100644 --- a/backend/test/workspace-runtime-renderer.test.ts +++ b/backend/test/workspace-runtime-renderer.test.ts @@ -139,8 +139,14 @@ function evidenceSecretFile(name: string, contents: string): string { return path; } -function evidenceWorkspace(source: Record, policy?: Record) { - return parseWorkspaceYaml(`${canonicalEvidenceWorkspace}\nevidence:\n source: ${JSON.stringify(source)}${ +function evidenceWorkspace( + source: Record, + policy?: Record, + evidenceSchemaVersion?: number, +) { + return parseWorkspaceYaml(`${canonicalEvidenceWorkspace}\nevidence:${ + evidenceSchemaVersion === undefined ? "" : `\n schema_version: ${evidenceSchemaVersion}` + }\n source: ${JSON.stringify(source)}${ policy === undefined ? "" : `\n policy: ${JSON.stringify(policy)}` }\n`); } @@ -180,9 +186,10 @@ function evidenceRender( source: Record, evidenceBinding: RuntimeBindings["evidence"] = { missing: [], values: {} }, policy?: Record, + evidenceSchemaVersion?: number, ) { return renderRuntimeConfig( - evidenceWorkspace(source, policy), + evidenceWorkspace(source, policy, evidenceSchemaVersion), { ...directBindings, evidence: evidenceBinding }, paths, evidenceContext, @@ -195,7 +202,7 @@ test("renders filesystem Evidence below the immutable revision content root with const yaml = evidenceRender({ type: "filesystem", uri: "psd-clinical/evidence", - }); + }, undefined, undefined, 2); const rendered = parse(yaml); expect(rendered.runtime_identity.workspace_revision).toBe(evidenceRevision); @@ -203,7 +210,7 @@ test("renders filesystem Evidence below the immutable revision content root with sources: [{ type: "filesystem", root: `/srv/registry/snapshots/${evidenceRevision}/psd-clinical/evidence`, - patterns: ["**/*.md"], + patterns: ["curated/**/*.md"], max_bytes: 10_485_760, }], }); diff --git a/backend/test/workspaces-schema.test.ts b/backend/test/workspaces-schema.test.ts index 3e93669e..5f8f70ef 100644 --- a/backend/test/workspaces-schema.test.ts +++ b/backend/test/workspaces-schema.test.ts @@ -390,6 +390,58 @@ test("applies filesystem and policy defaults to the canonical descriptor", () => }); }); +test("defaults schema-versioned filesystem Evidence to curated documents only", () => { + const parsed = validateWorkspaceDescriptor({ + ...withEvidence({ + type: "filesystem", + uri: "psd-clinical/evidence", + }), + evidence: { + schema_version: 2, + source: { type: "filesystem", uri: "psd-clinical/evidence" }, + }, + }); + + expect(parsed.evidence).toMatchObject({ + schema_version: 2, + source: { patterns: ["curated/**/*.md"] }, + }); +}); + +test("rejects a schema-versioned Evidence layout that mixes source and curated runtime patterns", () => { + expectSafeEvidenceError({ + ...withEvidence({ + type: "filesystem", + uri: "psd-clinical/evidence", + }), + evidence: { + schema_version: 2, + source: { + type: "filesystem", + uri: "psd-clinical/evidence", + patterns: ["source/**/*.md", "curated/**/*.md"], + }, + }, + }, /source.*curated|curated.*source/i); +}); + +test("rejects a schema-versioned Evidence layout that acquires source documents at runtime", () => { + expectSafeEvidenceError({ + ...withEvidence({ + type: "filesystem", + uri: "psd-clinical/evidence", + }), + evidence: { + schema_version: 2, + source: { + type: "filesystem", + uri: "psd-clinical/evidence", + patterns: ["source/**/*.md"], + }, + }, + }, /curated/i); +}); + test("keeps evidence optional on schema v3", () => { expect(validateWorkspaceDescriptor(validWorkspaceObject())).not.toHaveProperty("evidence"); }); diff --git a/docs/architecture/overview.md b/docs/architecture/overview.md index 3d8c9f6e..7af73289 100644 --- a/docs/architecture/overview.md +++ b/docs/architecture/overview.md @@ -46,6 +46,21 @@ Il modello propone; un revisore umano decide ai gate tramite widget: Il frontend renderizza questi widget-descriptor (registro in `src/widgets/`); il transcript live viene ricostruito in memoria dallo stream SSE (`src/store/sessionStore.ts`) — **non è persistito**. +## Evidence curata e immutabile + +Il repository del workspace è il confine di pubblicazione: il curatore prepara `evidence/source/`, +revisa le unità in `evidence/curated/`, valida e fa merge. Per `evidence.schema_version: 2` il +runtime materializza l'intero albero `evidence/` dal commit Git esatto, ma il renderer consegna al +preprocessing soltanto `curated/**/*.md` dalla root immutabile della revisione. Sorgenti, manifest +ed evaluation restano disponibili solo per tracciabilità. Il runtime non modifica, stagea, committa +o pubblica il repository di authoring. + +Prima dell'indicizzazione, il corpus curato della revisione pinnata viene validato. La collezione +Qdrant condivisa conserva il vettore dense senza nome di Schema e Memory; il preprocessing Evidence +può aggiungere soltanto in modo additivo il vettore sparse `bm25` con `idf`, senza eliminare, +rinominare o ricreare la collezione. `workspace preprocess evidence` e la parte Evidence di +`workspace preprocess run` sono le sole operazioni pubbliche che effettuano questo upgrade. + ## Punti di attenzione ricorrenti - `tht -c`/`--config` è un'opzione **per-comando**: deve seguire il subcommand, mai precederlo (`ThtRunner.buildArgv` lo impone). diff --git a/docs/contracts/workspace-evidence-v3.md b/docs/contracts/workspace-evidence-v3.md index 2461b4f0..b2f38ec9 100644 --- a/docs/contracts/workspace-evidence-v3.md +++ b/docs/contracts/workspace-evidence-v3.md @@ -8,18 +8,40 @@ source variant and the policy reject unknown keys. ## Filesystem source A filesystem source uses the exact URI `/evidence`. `patterns` is a nonempty list of -unique, normalized relative POSIX globs. Its defaults are `patterns: ["**/*.md"]` and +unique, normalized relative POSIX globs. The Evidence-local `schema_version` defaults to `1` for +compatibility, where an omitted filesystem pattern defaults to `patterns: ["**/*.md"]` and `max_bytes: 10485760`. +`evidence.schema_version: 2` declares the source/curated authoring layout. Its omitted filesystem +pattern defaults to `patterns: ["curated/**/*.md"]`; every explicit v2 filesystem pattern must also +remain below `curated/`. A v2 descriptor that selects `source/`, or spans both `source/` and +`curated/`, is rejected. Explicit safe legacy filesystem patterns remain supported under Evidence +version 1. HTTP and S3 sources do not use filesystem layout patterns and retain their existing +contracts. + +The v2 authoring tree is: + +```text +evidence/ +├── source/ # preserved original material +├── curated/ # reviewed Evidence Units indexed at runtime +├── manifest.yaml +└── evaluation.yaml +``` + +`source/`, the manifest, the evaluation set, and other support files are materialized for +traceability but never acquired by v2 runtime preprocessing. + ### Example: filesystem ```yaml evidence: + schema_version: 2 source: type: filesystem uri: example/evidence patterns: - - "**/*.md" + - "curated/**/*.md" max_bytes: 10485760 policy: max_chunk_chars: 4000 @@ -152,8 +174,10 @@ catalog metadata exactly. Every catalog entry must have its descriptor at that s catalog-only entries are invalid and reject the complete candidate revision. Workspace source changes only through curator Git commit/push in a separate authoring clone, -followed by an installation pull. The API never writes `thoth-workspaces.yaml`, -`/workspace.yaml`, `/schema/**`, or `/evidence/**`. +followed by an installation pull. Curator validation occurs before merge; activation and +preprocessing consume only the merged, pinned commit. The API and runtime never write +`thoth-workspaces.yaml`, `/workspace.yaml`, `/schema/**`, or `/evidence/**` in the +authoring repository. ## Registry revision and phase ownership @@ -164,7 +188,7 @@ followed by an installation pull. The API never writes `thoth-workspaces.yaml`, | Repository consumer | ThothII fetches and validates a complete candidate, atomically activates it only on success, and never edits, commits, or pushes repository content. | | Runtime secrets | Workspace management returns configured/missing status only; decrypted values exist only for the lifetime of a diagnostic or runtime lease. | | P1.1 | Validates the lexical URI `/evidence` and proves the declared filesystem root object is a Git tree at that same commit; it does not recursively inspect nested symlinks. Evidence materialization stays out of scope for P1.1. | -| P6 | Owns commit-addressed materialization, realpath and recursive containment, nested-symlink checks, and race checks. | +| P6 | Owns commit-addressed materialization of the complete Evidence tree, realpath and recursive containment, nested-symlink checks, and race checks. | P1.1 performs no acquisition, extraction, preprocessing/indexing, embeddings, Qdrant writes, active-snapshot retention, or GC. diff --git a/docs/contracts/workspace-preprocessing-cli.md b/docs/contracts/workspace-preprocessing-cli.md index 38de00cf..14204a76 100644 --- a/docs/contracts/workspace-preprocessing-cli.md +++ b/docs/contracts/workspace-preprocessing-cli.md @@ -106,7 +106,13 @@ tht --installation /thothii-installation.yaml workspace vector rebuild before any bytes are written and no partial root is published. - `preprocess evidence` and `preprocess run` operate directly on the materialized root; the temporary `evidence_materialization_required` stop is retired (the code remains only for - pre-P6 compatibility). HTTP/S3 Evidence is unchanged. + pre-P6 compatibility). For `evidence.schema_version: 2`, runtime acquisition receives only + `curated/**/*.md`; `source/` and support files remain in the materialized tree for traceability. + HTTP/S3 Evidence is unchanged. +- The curator validates Evidence before merge. Preprocessing validates the pinned curated corpus + again before it constructs a candidate generation, so an invalid revision is never indexed. +- The runtime writes only its immutable materialized snapshot and derived index state. It never + writes, stages, commits, or pushes the workspace authoring repository. - Materialized roots are retained with their commit-addressed snapshot directory and removed only when the revision becomes unreferenced.