From 7356d6794bb9d0480394bd645a165fbc57e6136e Mon Sep 17 00:00:00 2001 From: mptyl Date: Tue, 11 Aug 2026 14:22:28 +0200 Subject: [PATCH] docs: add P1.1 workspace directory plan --- backend/src/workspaces/catalog.ts | 75 + backend/test/workspaces-catalog.test.ts | 82 ++ ...08-11-p1-1-workspace-directory-registry.md | 1250 +++++++++++++++++ ...1-1-workspace-directory-registry-design.md | 255 ++++ 4 files changed, 1662 insertions(+) create mode 100644 backend/src/workspaces/catalog.ts create mode 100644 backend/test/workspaces-catalog.test.ts create mode 100644 docs/superpowers/plans/2026-08-11-p1-1-workspace-directory-registry.md create mode 100644 docs/superpowers/specs/2026-08-11-p1-1-workspace-directory-registry-design.md diff --git a/backend/src/workspaces/catalog.ts b/backend/src/workspaces/catalog.ts new file mode 100644 index 00000000..95b09cac --- /dev/null +++ b/backend/src/workspaces/catalog.ts @@ -0,0 +1,75 @@ +import { parseAllDocuments } from "yaml"; +import { z } from "zod"; +import type { WorkspaceDescriptor } from "./schema.js"; + +export const CATALOG_PATH = "thoth-workspaces.yaml"; + +export interface WorkspaceCatalogEntry { + id: string; + name: string; + description?: string; +} + +export interface WorkspaceCatalog { + schema_version: 1; + workspaces: WorkspaceCatalogEntry[]; +} + +const workspaceId = z.string().trim().regex(/^[a-z][a-z0-9-]{2,62}$/, { + message: "workspace id must match ^[a-z][a-z0-9-]{2,62}$", +}).refine((value) => value !== "workspace-docs", { + message: "workspace id is reserved", +}); + +const catalogEntry = z.object({ + id: workspaceId, + name: z.string().trim().min(1), + description: z.string().trim().min(1).optional(), +}).strict(); + +const catalogSchema = z.object({ + schema_version: z.literal(1), + workspaces: z.array(catalogEntry), +}).strict().superRefine((catalog, context) => { + const seen = new Set(); + catalog.workspaces.forEach((entry, index) => { + if (seen.has(entry.id)) { + context.addIssue({ + code: "custom", + path: ["workspaces", index, "id"], + message: "workspace id is duplicated in the catalog", + }); + } + seen.add(entry.id); + }); +}); + +function safeCatalogError(): Error { + return new Error("Workspace catalog is invalid"); +} + +export function parseWorkspaceCatalogYaml(source: string): WorkspaceCatalog { + try { + const documents = parseAllDocuments(source, { uniqueKeys: true }); + if (documents.length !== 1) throw safeCatalogError(); + const document = documents[0]; + if (document.errors.length > 0 || document.warnings.length > 0) throw safeCatalogError(); + return catalogSchema.parse(document.toJSON()) as WorkspaceCatalog; + } catch (error) { + if (error instanceof Error && error.message === "Workspace catalog is invalid") throw error; + throw safeCatalogError(); + } +} + +export function assertCatalogMatchesDescriptor( + entry: WorkspaceCatalogEntry, + workspace: WorkspaceDescriptor, +): void { + if ( + entry.id !== workspace.workspace.id + || entry.name !== workspace.workspace.name + || entry.description !== workspace.workspace.description + ) { + throw new Error("Workspace catalog metadata does not match descriptor"); + } +} diff --git a/backend/test/workspaces-catalog.test.ts b/backend/test/workspaces-catalog.test.ts new file mode 100644 index 00000000..23b71a66 --- /dev/null +++ b/backend/test/workspaces-catalog.test.ts @@ -0,0 +1,82 @@ +import { expect, test } from "vitest"; +import { parseWorkspaceYaml } from "../src/workspaces/schema.js"; +import { + CATALOG_PATH, + assertCatalogMatchesDescriptor, + parseWorkspaceCatalogYaml, +} from "../src/workspaces/catalog.js"; + +const descriptor = parseWorkspaceYaml(`workspace: + schema_version: 3 + id: psd + name: Policlinico San Donato + description: Clinical warehouse + language: it +dwh: + engine: postgres + database: postgres + schema: datawarehouse + supported_transports: [rest_api] +semantic_index: + vector_store: { engine: qdrant, collection: psd, dimensions: 1024, distance: cosine } + embedding: { provider: ollama_internal, model: qwen3-embedding:0.6b, dimensions: 1024 } +llm_policy: { allowed: [zai/glm-5.2] } +`); + +test("parses the strict ordered root catalog", () => { + expect(CATALOG_PATH).toBe("thoth-workspaces.yaml"); + expect(parseWorkspaceCatalogYaml(`schema_version: 1 +workspaces: + - id: psd + name: Policlinico San Donato + description: Clinical warehouse + - id: research + name: Research +`)).toEqual({ + schema_version: 1, + workspaces: [ + { id: "psd", name: "Policlinico San Donato", description: "Clinical warehouse" }, + { id: "research", name: "Research" }, + ], + }); +}); + +test("preserves optional description absence and trims metadata", () => { + expect(parseWorkspaceCatalogYaml(`schema_version: 1 +workspaces: + - id: psd + name: " PSD " +`)).toEqual({ schema_version: 1, workspaces: [{ id: "psd", name: "PSD" }] }); +}); + +test.each([ + ["duplicate IDs", `schema_version: 1\nworkspaces: [{id: psd, name: One}, {id: psd, name: Two}]`], + ["invalid ID", `schema_version: 1\nworkspaces: [{id: PSD, name: One}]`], + ["reserved ID", `schema_version: 1\nworkspaces: [{id: workspace-docs, name: One}]`], + ["unknown key", `schema_version: 1\nworkspaces: [{id: psd, name: One, secret: CANARY}]`], + ["duplicate YAML key", `schema_version: 1\nworkspaces:\n - id: psd\n id: research\n name: One`], + ["multiple documents", `schema_version: 1\nworkspaces: []\n---\nschema_version: 1\nworkspaces: []`], +])("rejects %s without exposing unsafe input", (_name, source) => { + expect(() => parseWorkspaceCatalogYaml(source)).toThrow(/catalog|workspace|id|YAML/i); + try { parseWorkspaceCatalogYaml(source); } catch (error) { + expect(String(error)).not.toContain("CANARY"); + } +}); + +test("rejects catalog metadata that differs from its descriptor", () => { + expect(() => assertCatalogMatchesDescriptor( + { id: "psd", name: "Other", description: "Clinical warehouse" }, descriptor, + )).toThrow(/catalog|metadata/i); + expect(() => assertCatalogMatchesDescriptor( + { id: "psd", name: "Policlinico San Donato" }, descriptor, + )).toThrow(/catalog|metadata/i); + expect(() => assertCatalogMatchesDescriptor( + { id: "other", name: "Policlinico San Donato", description: "Clinical warehouse" }, descriptor, + )).toThrow(/catalog|metadata/i); +}); + +test("accepts exact catalog metadata", () => { + expect(() => assertCatalogMatchesDescriptor( + { id: "psd", name: "Policlinico San Donato", description: "Clinical warehouse" }, descriptor, + )).not.toThrow(); +}); diff --git a/docs/superpowers/plans/2026-08-11-p1-1-workspace-directory-registry.md b/docs/superpowers/plans/2026-08-11-p1-1-workspace-directory-registry.md new file mode 100644 index 00000000..da66b5a3 --- /dev/null +++ b/docs/superpowers/plans/2026-08-11-p1-1-workspace-directory-registry.md @@ -0,0 +1,1250 @@ +# P1.1 Workspace-Directory Git Registry Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Use superpowers:test-driven-development for every behavior change and superpowers:verification-before-completion before any completion claim. + +**Goal:** Replace P1's split/flat Git source layout with an authoritative root catalog and one self-contained directory per workspace, while making descriptor publication bootstrap-only and all later descriptor changes curator-owned through Git. + +**Architecture:** `thoth-workspaces.yaml` becomes the strict curator-owned catalog; descriptors move to `/workspace.yaml`, embedded Evidence moves to `/evidence`, and generated docs remain under `workspace-docs/`. The API may create a descriptor only when its catalog slot exists and the descriptor Git object is absent at the exact base commit; existing descriptors and curated content are read-only to the API. Internal immutable snapshot paths stay flat to preserve ThtRunner/session compatibility. P1.1 gets new automated and manual acceptance evidence; accepted P1 evidence remains historical and untouched. + +**Tech Stack:** TypeScript 5, Zod 4, YAML, Fastify 5, Git CLI with fixed argv, React 18, Vitest, Node.js 22, Bash, Python harness `tht config check`. + +**Companion design draft:** `docs/superpowers/specs/2026-08-11-p1-1-workspace-directory-registry-design.md` + +--- + +## P1.1 completion contract + +P1.1 is complete only when all of the following are true: + +1. The only accepted source-repository layout is: + + ```text + thoth-workspaces.yaml + /workspace.yaml + /evidence/** # optional; required only for filesystem Evidence + workspace-docs//{contract.env.example,README.md} + ``` + +2. The strict root catalog is curator-owned and authoritative for ID, name, description, and display order. Catalog-only entries are valid bootstrap slots with `configuration_required`; orphan descriptors/directories and catalog/descriptor metadata mismatches invalidate the candidate atomically. +3. Schema v3 remains the only descriptor schema. For filesystem Evidence the only URI is `/evidence`; HTTP/S3 and absent-Evidence behavior stay as delivered by P1. +4. The API creates `/workspace.yaml` only when no Git object exists there at the exact base commit. A present empty, malformed, symlink, submodule, tree, or valid descriptor is never replaced or deleted. Existing update/delete payloads fail with safe `workspace_curator_owned` semantics. +5. Curator-pushed descriptor/catalog/Evidence changes become active only after strict pull validation. The API never stages, writes, cleans, or pushes catalog or curated content. +6. Generated docs remain API-owned under `workspace-docs/`. Explicit registry synchronization may produce one deterministic docs-only commit; it must preserve catalog, descriptor, and Evidence object IDs and activate only the final validated commit. +7. Internal snapshots remain `//.yaml`; session revision pins, retention leases, runtime acquisition, export, and resume retain their current contract. +8. Existing workspace UI is read-only for repository-backed descriptors. Only a catalog slot with no descriptor offers an editable bootstrap draft; stale old drafts cannot update/delete. Pull/sync, validate, installation test, export, and Evidence summary remain available. +9. A new clean-state P1.1 automated run and a separate P1.1 manual walkthrough prove the complete process. Accepted retained P1 artifacts remain immutable historical evidence and are not relabelled as P1.1. Old P1 process commands are not required to execute successfully against the superseding P1.1 repository contract. +10. No preprocessing, materialization, FK synchronization, Qdrant write, embedding, ACTIVE publication, GC, or P2–P6 plan edit is performed. + +## Explicit decisions frozen by this plan + +- Root catalog name: `thoth-workspaces.yaml`. +- Workspace source directory: `/`. +- Descriptor filename: `workspace.yaml`. +- Catalog schema: `schema_version: 1`, ordered `workspaces` list with strict `{id,name,description?}` entries. +- Catalog-only entries are allowed and listable as `configuration_required`. +- Descriptor metadata remains present for self-contained snapshots/exports and must exactly equal catalog metadata. +- "Empty" means **absent Git object**, not zero bytes. +- Old repository layouts are rejected; there is no dual reader or automatic remote migration. +- Internal snapshot filenames do not change. +- P2–P6 documents are inventoried but not edited until after owner manual acceptance of P1.1. + +--- + +### Task 1: Freeze the P1.1 design, catalog schema, and strict parser + +**Files:** +- Add: `docs/superpowers/specs/2026-08-11-p1-1-workspace-directory-registry-design.md` +- Add: `backend/src/workspaces/catalog.ts` +- Add: `backend/test/workspaces-catalog.test.ts` +- Modify: `backend/src/workspaces/types.ts` + +**Step 1: Write failing catalog parser tests** + +Define the exact public shape: + +```ts +export interface WorkspaceCatalogEntry { + id: string; + name: string; + description?: string; +} + +export interface WorkspaceCatalog { + schema_version: 1; + workspaces: WorkspaceCatalogEntry[]; +} +``` + +Test: + +- one and many ordered entries parse without reordering; +- optional description presence is preserved (missing is not silently converted to an empty string); +- Unicode names/descriptions and the descriptor's existing trim/nonblank semantics are preserved without adding a new length limit; +- duplicate IDs, invalid/reserved IDs (including `workspace-docs`), blank names, unknown keys, duplicate YAML keys, aliases, tags, multiple documents, non-mappings, and malformed YAML are rejected with `workspace_invalid`; +- safe errors include a stable catalog field location but never rejected canary text; +- `assertCatalogMatchesDescriptor(entry, descriptor)` accepts exact ID/name/optional-description equality and rejects every mismatch safely; +- serialization, if exposed, is deterministic and does not become an API write path. + +**Step 2: Run the focused test and verify RED** + +```bash +cd backend +npx vitest run test/workspaces-catalog.test.ts +``` + +Expected: FAIL because `catalog.ts` does not exist. + +**Step 3: Implement the strict parser** + +Use `yaml.parseAllDocuments` with the same safe-document rules as `parseWorkspaceYaml` and a strict Zod schema. Keep catalog parsing separate from descriptor parsing. Export only: + +```ts +export const CATALOG_PATH = "thoth-workspaces.yaml"; +export function parseWorkspaceCatalogYaml(source: string): WorkspaceCatalog; +export function assertCatalogMatchesDescriptor( + entry: WorkspaceCatalogEntry, + workspace: WorkspaceDescriptor, +): void; +``` + +Do not project catalog metadata into a descriptor and do not read the filesystem from this module. + +Add any new stable error code only when needed by later tasks; catalog syntax/matching errors remain `workspace_invalid`. + +**Step 4: Run tests and typecheck** + +```bash +cd backend +npx vitest run test/workspaces-catalog.test.ts test/workspaces-schema.test.ts +npx tsc --noEmit -p . +``` + +Expected: PASS. + +**Step 5: Commit** + +```bash +git add \ + docs/superpowers/specs/2026-08-11-p1-1-workspace-directory-registry-design.md \ + backend/src/workspaces/catalog.ts \ + backend/src/workspaces/types.ts \ + backend/test/workspaces-catalog.test.ts +git commit -m "feat: define P1.1 workspace catalog contract" +``` + +--- + +### Task 2: Enforce the nested repository paths and curator/API ownership boundary + +**Files:** +- Modify: `backend/src/workspaces/git-repository.ts` +- Modify: `backend/test/workspaces-git-repository.test.ts` + +**Step 1: Write failing low-level Git tests** + +Create a real bare-repository fixture with: + +```text +thoth-workspaces.yaml +research/workspace.yaml +research/evidence/guide.md +workspace-docs/research/README.md +workspace-docs/research/contract.env.example +``` + +Test fixed-argv helpers that: + +- read exactly `thoth-workspaces.yaml` as a regular Git blob at HEAD/revision; +- discover only `/workspace.yaml` descriptors, without treating nested Evidence files as descriptors; +- reject flat `workspaces/.yaml`, `workspace-content/**`, the reserved `workspace-docs` ID, unlisted top-level workspace directories, traversal, alternate descriptor names, symlink descriptor, tree-at-descriptor, submodule/gitlink, and malformed IDs; +- read/resolve the exact descriptor blob at `/workspace.yaml`; +- accept only `/evidence` as the filesystem Evidence root and require a Git tree at the exact revision; +- keep nested symlink checks deferred to P6 while still rejecting a symlink at the declared root; +- prove catalog and `/evidence/**` are not API-writable/stageable paths; +- allow only a create-only descriptor path and generated docs in API publication helpers; +- refuse an exclusive descriptor create if any filesystem/Git object already occupies the path; +- journal each exact API-owned path before mutation (object type/mode/blob/bytes or explicit absence); +- restore overwritten/deleted generated docs to their exact pre-operation objects and remove only absent-before files on failure, never by running a directory-wide `git clean`; +- cover failed bootstrap with pre-existing stale docs plus failed docs update/deletion, and preserve all curator object IDs across cleanup; +- use argv arrays only and do not invoke Git filters, hooks, shell interpolation, or helper-bearing repository state. + +**Step 2: Run the focused test and verify RED** + +```bash +cd backend +npx vitest run test/workspaces-git-repository.test.ts +``` + +Expected: old flat-path assertions fail and required helpers are missing. + +**Step 3: Implement narrow path helpers** + +Introduce named guards such as: + +```ts +function workspaceDescriptorPath(id: string): string { + return `${safeWorkspaceId(id)}/workspace.yaml`; +} + +function evidenceRootPath(id: string): string { + return `${safeWorkspaceId(id)}/evidence`; +} +``` + +Add read-only helpers for catalog/descriptor object type and descriptor discovery. Replace broad `writeRegistryFile` use for descriptors with an explicit exclusive creation primitive. Keep generated-doc writes separate and exact. + +`restoreFailedPublication` must consume the bounded per-path journal from the current operation. It restores tracked generated docs from the prior blob/mode and deletes only paths proven absent before the operation. Remove any directory-wide `git clean` that can descend into curated workspace content. + +**Step 4: Run tests and typecheck** + +```bash +cd backend +npx vitest run test/workspaces-git-repository.test.ts +npx tsc --noEmit -p . +``` + +Expected: PASS. + +**Step 5: Commit** + +```bash +git add backend/src/workspaces/git-repository.ts backend/test/workspaces-git-repository.test.ts +git commit -m "refactor: enforce P1.1 registry path ownership" +``` + +--- + +### Task 3: Make activation catalog-driven while preserving internal snapshots + +**Files:** +- Modify: `backend/src/workspaces/registry.ts` +- Modify: `backend/test/workspace-registry.test.ts` +- Modify: `backend/src/workspaces/types.ts` + +**Step 1: Write failing activation/state tests** + +Cover real Git candidates for: + +- valid catalog plus one/many matching descriptors activates in catalog order; +- catalog-only entry activates as `configuration_required` without a `WorkspaceRevision` and without allowing session/read/test/export; +- missing catalog, malformed catalog, duplicate catalog ID, orphan descriptor/directory, metadata mismatch, wrong descriptor path, duplicate Qdrant collection, invalid descriptor, or unsafe Evidence root leaves the prior active snapshot unchanged; +- a present empty/comments-only descriptor fails activation and is never converted into a bootstrap slot; +- removing a descriptor while leaving its catalog entry produces `configuration_required`, while retained historical revisions/session pins remain readable; +- removing catalog entry and its workspace directory together removes it from the active catalog but retains historical pinned snapshots; +- removing a catalog entry while leaving its workspace directory/descriptor is invalid; +- catalog order controls summaries independently from Git path order; +- offline fallback restores the last complete catalog and ready revisions; +- internal snapshot paths remain exactly `//.yaml`; +- the immutable snapshot binds a canonical catalog copy/digest plus canonical descriptor/docs, but contains no Evidence bytes; +- pre-P1.1 historical internal snapshots needed by already retained session pins remain readable, without accepting old source-repository layout for new activation. + +**Step 2: Run the registry suite and verify RED** + +```bash +cd backend +npx vitest run test/workspace-registry.test.ts +``` + +Expected: nested repository and catalog-only cases fail. + +**Step 3: Refactor candidate parsing from activation** + +Introduce an internal candidate model, for example: + +```ts +interface WorkspaceCatalogRecord { + entry: WorkspaceCatalogEntry; + state: "ready" | "configuration_required"; + revision?: WorkspaceRevision; +} + +interface RegistryCandidate { + commit: string; + catalog: WorkspaceCatalog; + ready: Array<{ + entry: WorkspaceCatalogEntry; + workspace: WorkspaceDescriptor; + descriptorPath: string; + blob: string; + }>; + missing: WorkspaceCatalogEntry[]; +} +``` + +Separate: + +1. `readCandidate(commit)` — read/validate Git objects without mutating active state; +2. `stageSnapshot(candidate)` — write immutable local snapshot bytes; +3. `activateCandidate(candidate)` — atomically publish active state only after all checks/synchronization succeed. + +Keep `WorkspaceRevision` and internal flat snapshot filenames unchanged. Persist enough canonical catalog data in the immutable snapshot to list catalog-only entries during offline fallback. Do not force `WorkspaceRevision` to represent a missing descriptor. + +Expose a catalog-aware method for routes, while preserving `list()`/retained-revision methods used by sessions: + +```ts +listCatalog(): Promise; +``` + +**Step 4: Run focused cross-boundary tests** + +```bash +cd backend +npx vitest run \ + test/workspace-registry.test.ts \ + test/routes-sessions.test.ts \ + test/workspace-runtime-handoff.test.ts +npx tsc --noEmit -p . +``` + +Expected: PASS; ready workspaces remain session-activatable and missing descriptors do not. + +**Step 5: Commit** + +```bash +git add backend/src/workspaces/registry.ts backend/src/workspaces/types.ts backend/test/workspace-registry.test.ts +git commit -m "feat: activate workspaces from the root catalog" +``` + +--- + +### Task 4: Implement bootstrap-only descriptor publication and deterministic docs synchronization + +**Files:** +- Modify: `backend/src/workspaces/registry.ts` +- Modify: `backend/src/workspaces/git-repository.ts` +- Modify: `backend/src/workspaces/types.ts` +- Modify: `backend/test/workspace-registry.test.ts` + +**Step 1: Write failing create-only publication tests** + +Prove: + +- catalog entry exists + descriptor absent + exact base commit + matching metadata + valid Evidence context → API creates descriptor and generated docs once; +- catalog bytes/blob and every Evidence tree/blob are unchanged by bootstrap; +- a descriptor path containing zero bytes, invalid YAML, a symlink/blob/tree/gitlink, or valid YAML is considered present and is not overwritten; +- update and delete requests return `workspace_curator_owned`, perform no write/stage/commit, and preserve HEAD/object IDs; +- create for an unknown catalog ID or mismatched name/description fails without mutation; +- stale base and a race in which a curator creates the descriptor first fail safely; +- failed commit/push replays the per-path journal, including stale pre-existing generated docs, and leaves curator paths untouched; +- a curator modifies catalog metadata and the descriptor together, pushes, and pull activates the exact curator bytes without reserializing the descriptor in Git; +- a content-only Evidence commit changes the active workspace commit even when descriptor/catalog blobs are unchanged; +- a curator descriptor-only commit changes the descriptor blob and active revision without any API descriptor write; +- explicit pull computes generated docs and, when stale, produces at most one docs-only follow-up commit; +- that docs-only commit changes only `workspace-docs/**`, preserves catalog/descriptor/Evidence object IDs, and becomes the active revision; +- startup/status activation never pushes; before explicit sync, local snapshots/exports contain freshly derived docs even if committed `workspace-docs` are stale; +- no-op synchronization makes no commit; +- a docs push race/rejection keeps the prior active snapshot and restores a clean checkout; +- generated docs are removed only when the catalog/descriptor state no longer owns them, never by directory-wide cleanup. + +**Step 2: Run the focused tests and verify RED** + +```bash +cd backend +npx vitest run test/workspace-registry.test.ts -t "bootstrap|curator|generated docs|content-only" +``` + +Expected: FAIL against create/update/delete publication. + +**Step 3: Narrow the public mutation contract** + +Add: + +```ts +export type BootstrapWorkspaceRequest = { + action: "create"; + workspace: CanonicalWorkspace; + baseCommit: string; +}; +``` + +Keep legacy request parsing only long enough to return the stable refusal; do not keep update/delete implementation branches. Add `workspace_curator_owned` to `WorkspaceErrorCode` and map it to HTTP 409. + +`publishBootstrap` must: + +1. pull and read a candidate without activating it; +2. compare the exact requested base; +3. locate the authoritative catalog slot; +4. verify descriptor absence and metadata equality; +5. verify contextual filesystem Evidence at that base; +6. exclusively create the descriptor plus deterministic docs; +7. commit/push fixed paths with fixed argv; +8. read/validate the resulting candidate; +9. activate only after complete success. + +Refactor explicit pull to reconcile docs as defined in the design. GET/status/bootstrap paths remain read-only with respect to the remote. + +**Step 4: Run focused tests, typecheck, and build** + +```bash +cd backend +npx vitest run test/workspace-registry.test.ts test/workspaces-git-repository.test.ts +npx tsc --noEmit -p . +npm run build +``` + +Expected: PASS. + +**Step 5: Commit** + +```bash +git add \ + backend/src/workspaces/registry.ts \ + backend/src/workspaces/git-repository.ts \ + backend/src/workspaces/types.ts \ + backend/test/workspace-registry.test.ts +git commit -m "feat: make workspace publication bootstrap-only" +``` + +--- + +### Task 5: Update workspace routes and machine contracts + +**Files:** +- Modify: `backend/src/routes/workspaces.ts` +- Modify: `backend/test/routes-workspaces.test.ts` +- Modify: `backend/test/routes-sessions.test.ts` + +**Step 1: Write failing real-route tests** + +Using the real local bare-repository fixture, assert: + +- `GET /workspaces` returns root-catalog order/metadata and explicit `ready` vs `configuration_required` state; +- summary `file` is exactly `/workspace.yaml`; summary omits `language` because a catalog-only slot has none, while ready detail/bootstrap drafts retain descriptor language; +- catalog-only entries have no revision and no descriptor body; +- `GET /workspaces/:id`, diagnostic, export, and session creation for a catalog-only entry return safe `workspace_not_activatable` and never start Pi; +- `POST /workspaces/validate` remains context-free and says nothing about catalog publication eligibility; +- create for one matching catalog slot succeeds once; +- second create, update, and delete produce HTTP 409 `workspace_curator_owned` with no conflict field/value payload and no mutation; +- unknown slot, catalog metadata mismatch, stale base, invalid Evidence tree, and present-empty descriptor produce safe errors without canary/Git stderr; +- curator-pushed descriptor/catalog/Evidence changes are visible after pull and API bytes remain unchanged; +- import remains an untrusted draft and cannot update an existing workspace; +- export remains descriptor/docs only and never includes Evidence bytes or secrets. + +**Step 2: Run the focused routes and verify RED** + +```bash +cd backend +npx vitest run test/routes-workspaces.test.ts test/routes-sessions.test.ts +``` + +Expected: FAIL because routes expose full CRUD and cannot list missing descriptors. + +**Step 3: Implement the new DTOs and route semantics** + +Create a stable summary shape with catalog authority and explicit state. Route `POST /workspaces/publish` to bootstrap only. Recognize legacy update/delete payload discriminators before rejecting them as `workspace_curator_owned`; never pass them to a file mutation method. + +Remove field-level `WorkspaceConflictError` serialization if it has no remaining production caller. Preserve generic stale-commit 409 behavior for bootstrap races. + +**Step 4: Run tests and checks** + +```bash +cd backend +npx vitest run \ + test/routes-workspaces.test.ts \ + test/routes-sessions.test.ts \ + test/workspace-registry.test.ts +npx tsc --noEmit -p . +npm run build +``` + +Expected: PASS. + +**Step 5: Commit** + +```bash +git add backend/src/routes/workspaces.ts backend/test/routes-workspaces.test.ts backend/test/routes-sessions.test.ts +git commit -m "feat: expose catalog-driven bootstrap workspace API" +``` + +--- + +### Task 6: Change the filesystem Evidence root without changing P1 source semantics + +**Files:** +- Modify: `backend/src/workspaces/schema.ts` +- Modify: `backend/test/workspaces-schema.test.ts` +- Modify: `backend/test/workspaces-contracts.test.ts` +- Modify: `backend/test/workspaces-bindings.test.ts` +- Modify: `backend/test/workspace-runtime-renderer.test.ts` +- Modify: `backend/test/workspace-runtime-handoff.test.ts` +- Modify: `deploy/workspaces/example.yaml` +- Modify: `deploy/workspaces/psd.yaml.example` + +**Step 1: Change tests first** + +Replace every positive filesystem URI with: + +```text +/evidence +``` + +Negative coverage must reject: + +- old `workspace-content//evidence`; +- flat/cross-workspace paths; +- absolute paths, `.`/`..`, doubled segments, backslashes, controls, query/fragment-like content; +- roots above or below the exact canonical Evidence root. + +Renderer/handoff tests must expect: + +```text +/snapshots///evidence +``` + +while allowing that root not to exist until P6 materializes it. Keep HTTP/S3, local secret files, policy defaults, deterministic bytes, `runtime_identity.workspace_revision`, and `ssh_tunnel` fail-closed behavior unchanged. + +**Step 2: Run focused tests and verify RED** + +```bash +cd backend +npx vitest run \ + test/workspaces-schema.test.ts \ + test/workspaces-contracts.test.ts \ + test/workspaces-bindings.test.ts \ + test/workspace-runtime-renderer.test.ts \ + test/workspace-runtime-handoff.test.ts +``` + +Expected: FAIL on the old hardcoded invariant/fixtures. + +**Step 3: Implement the smallest production change** + +Change the cross-field invariant to: + +```ts +const expected = `${workspace.workspace.id}/evidence`; +``` + +The runtime renderer already joins a validated repo-relative URI to `revisionContentRoot`; do not add a second path mapping or materialization branch. + +**Step 4: Run cross-layer verification** + +```bash +cd backend +npx vitest run \ + test/workspaces-schema.test.ts \ + test/workspaces-contracts.test.ts \ + test/workspaces-bindings.test.ts \ + test/workspace-runtime-renderer.test.ts \ + test/workspace-runtime-handoff.test.ts +npx tsc --noEmit -p . +npm run build +``` + +Then: + +```bash +cd harness +.venv/bin/pytest -q tests/test_config_resources.py tests/test_registry_evidence_config.py +``` + +Expected: PASS. No harness production change should be necessary. + +**Step 5: Commit** + +```bash +git add \ + backend/src/workspaces/schema.ts \ + backend/test/workspaces-schema.test.ts \ + backend/test/workspaces-contracts.test.ts \ + backend/test/workspaces-bindings.test.ts \ + backend/test/workspace-runtime-renderer.test.ts \ + backend/test/workspace-runtime-handoff.test.ts \ + deploy/workspaces/example.yaml \ + deploy/workspaces/psd.yaml.example +git commit -m "refactor: colocate filesystem Evidence with its workspace" +``` + +--- + +### Task 7: Narrow frontend API and draft persistence to bootstrap-only authoring + +**Files:** +- Modify: `frontend/src/api/workspaces.ts` +- Modify: `frontend/src/api/workspaces.test.ts` +- Modify: `frontend/src/workspaces/drafts.ts` +- Modify: `frontend/src/workspaces/drafts.test.ts` +- Modify: `frontend/src/test/workspace-fixtures.ts` +- Review/test: `frontend/src/api/sessions.test.ts` +- Review/test: `frontend/src/shell/SteerInput.test.tsx` +- Review/test: `frontend/src/shell/NewSessionDialog.test.tsx` + +**Step 1: Write failing frontend contract tests** + +Test: + +- summary parsing accepts exact catalog metadata, direct-root descriptor path, explicit state, no summary `language`, and optional revision only for `ready`; +- malformed or contradictory summary state/revision combinations are rejected; +- canonical workspace sanitization accepts only `/evidence` for filesystem sources; +- publish request type and client emit create only; +- backend `workspace_curator_owned` is decoded safely without conflict fields; +- removed field-level conflict/update/delete payloads are rejected rather than stored; +- imported bundle becomes a bootstrap candidate only; no existing-workspace update request can be constructed; +- v1 update/deletion localStorage records are purged/ignored and never returned as actionable drafts; +- new versioned bootstrap drafts contain a catalog slot identity, base commit, and workspace body but no `baseBlob` or delete intent; +- session/new-session consumers still require a ready workspace revision and ignore `configuration_required` entries. + +**Step 2: Run focused tests and verify RED** + +```bash +cd frontend +npx vitest run \ + src/api/workspaces.test.ts \ + src/workspaces/drafts.test.ts \ + src/api/sessions.test.ts \ + src/shell/SteerInput.test.tsx \ + src/shell/NewSessionDialog.test.tsx +``` + +Expected: FAIL against full CRUD DTOs and old URI sanitizer. + +**Step 3: Implement strict client contracts** + +Replace `PublishWorkspaceRequest` with the bootstrap-only request. Add `configurationState` to `WorkspaceSummary`. Remove `WorkspaceConflict`, conflict-field allowlists, deletion-draft types/storage, and any serializer that can produce update/delete. + +Version browser storage keys so old drafts cannot be interpreted under P1.1. On initialization, remove old known draft/delete keys only; never clear unrelated localStorage. + +**Step 4: Run tests and typecheck** + +```bash +cd frontend +npx vitest run \ + src/api/workspaces.test.ts \ + src/workspaces/drafts.test.ts \ + src/api/sessions.test.ts \ + src/shell/SteerInput.test.tsx \ + src/shell/NewSessionDialog.test.tsx +npx tsc -b +``` + +Expected: PASS. + +**Step 5: Commit** + +```bash +git add \ + frontend/src/api/workspaces.ts \ + frontend/src/api/workspaces.test.ts \ + frontend/src/workspaces/drafts.ts \ + frontend/src/workspaces/drafts.test.ts \ + frontend/src/test/workspace-fixtures.ts \ + frontend/src/api/sessions.test.ts \ + frontend/src/shell/SteerInput.test.tsx \ + frontend/src/shell/NewSessionDialog.test.tsx +git commit -m "refactor: make browser workspace writes bootstrap-only" +``` + +--- + +### Task 8: Make existing workspaces read-only in Workspace Management + +**Files:** +- Modify: `frontend/src/shell/WorkspaceManager.tsx` +- Modify: `frontend/src/shell/WorkspaceManager.test.tsx` +- Modify: `frontend/src/shell/WorkspaceEditor.tsx` +- Modify: `frontend/src/shell/WorkspaceEditor.test.tsx` +- Modify or remove: `frontend/src/shell/WorkspacePublishDialog.tsx` +- Modify or remove: `frontend/src/shell/WorkspacePublishDialog.test.tsx` +- Review/test: `frontend/src/shell/AppShell.new-session.test.tsx` +- Review/test: `frontend/src/shell/AppShell.session-mgmt.test.tsx` + +**Step 1: Write failing behavior tests** + +Prove: + +- catalog order/name/description render even when no descriptor exists; +- `configuration_required` entry offers a prefilled editable bootstrap form with ID/name/description locked to catalog values; +- Save Draft is browser-local, Validate is explicit, and Create requires a separate confirmation; +- successful create discards the bootstrap draft and reloads as read-only; +- a ready workspace renders all descriptor fields read-only, plus Evidence summary and Git edit guidance; +- ready workspace has no Save, Publish update, Delete, Duplicate, conflict merge, or imported-update action; +- Pull/Sync, Export, Validate, and installation Test remain available where meaningful; +- stale update/delete localStorage fixtures do not make controls appear or send a request; +- import can populate only a matching unconfigured catalog slot; mismatch/existing target is refused safely; +- a curator-pushed change appears after Pull and is not written back by the browser; +- configured-only workspace selection remains enforced for new sessions. + +**Step 2: Run focused tests and verify RED** + +```bash +cd frontend +npx vitest run \ + src/shell/WorkspaceManager.test.tsx \ + src/shell/WorkspaceEditor.test.tsx \ + src/shell/WorkspacePublishDialog.test.tsx \ + src/shell/AppShell.new-session.test.tsx \ + src/shell/AppShell.session-mgmt.test.tsx +``` + +Expected: FAIL because existing workspaces expose full CRUD. + +**Step 3: Implement two explicit UI modes** + +Use a discriminated prop rather than inferring editability from `baseBlob`: + +```ts +type WorkspaceEditorMode = + | { kind: "bootstrap"; catalog: WorkspaceSummary; draft: WorkspaceBootstrapDraft } + | { kind: "read_only"; catalog: WorkspaceSummary; record: WorkspaceRecord }; +``` + +Do not rely only on disabled controls; remove mutation handlers and mutation buttons entirely in read-only mode. Keep descriptive text telling curators to edit `/workspace.yaml`, commit/push, then use Pull/Sync. + +Reduce `WorkspacePublishDialog` to one bootstrap confirmation or fold it into the manager and delete the obsolete conflict UI/tests. + +**Step 4: Run frontend verification** + +```bash +cd frontend +npx vitest run \ + src/shell/WorkspaceManager.test.tsx \ + src/shell/WorkspaceEditor.test.tsx \ + src/shell/WorkspacePublishDialog.test.tsx \ + src/shell/AppShell.new-session.test.tsx \ + src/shell/AppShell.session-mgmt.test.tsx +npx tsc -b +npm run build +``` + +If `WorkspacePublishDialog` is removed, omit its test from the command and prove no imports remain with `git grep`. + +Expected: PASS. + +**Step 5: Commit** + +```bash +git add -A \ + frontend/src/shell/WorkspaceManager.tsx \ + frontend/src/shell/WorkspaceManager.test.tsx \ + frontend/src/shell/WorkspaceEditor.tsx \ + frontend/src/shell/WorkspaceEditor.test.tsx \ + frontend/src/shell/WorkspacePublishDialog.tsx \ + frontend/src/shell/WorkspacePublishDialog.test.tsx \ + frontend/src/shell/AppShell.new-session.test.tsx \ + frontend/src/shell/AppShell.session-mgmt.test.tsx +git commit -m "feat: make curator-owned workspaces read-only in the browser" +``` + +--- + +### Task 9: Update active contracts, examples, operator guides, and executable doc gates + +**Files:** +- Modify: `docs/contracts/workspace-evidence-v3.md` +- Modify: `docs/install/local-workspace-registry.md` +- Modify: `docs/install/server-workspace-registry.md` +- Modify: `README.md` +- Add: `docs/migrations/p1-to-p1-1-registry-layout.md` +- Modify: `scripts/verify-workspace-install-docs.sh` +- Modify: `scripts/test-verify-workspace-install-docs.sh` +- Review/modify if required: `scripts/workspace_descriptor_doc_contract.py` +- Review/modify if required: `scripts/test-workspace-descriptor-doc-contract.sh` +- Modify: `scripts/verify-schema-v3-only.sh` +- Modify: `scripts/test-verify-schema-v3-only.sh` + +**Step 1: Add failing verifier mutations** + +The self-tests must reject docs/examples that: + +- omit `thoth-workspaces.yaml` or put it below a workspace; +- use flat `workspaces/.yaml` or old `workspace-content//evidence`; +- omit exact `/workspace.yaml` or `/evidence` paths; +- claim catalog metadata comes from the descriptor; +- claim the API updates/deletes existing descriptors or writes catalog/Evidence; +- treat zero-byte descriptors as API-writable; +- omit catalog-only bootstrap and curator commit/push/pull flow; +- place generated docs inside a workspace directory; +- claim P1.1 materializes/preprocesses/indexes Evidence; +- omit migration ordering and explicit rejection of old layout; +- silently edit or claim completion of P2–P6. + +Retain secret/path/protocol/adversarial verifier coverage from P1. + +**Step 2: Run self-tests and verify RED** + +```bash +bash scripts/test-verify-workspace-install-docs.sh +bash scripts/test-verify-schema-v3-only.sh +``` + +Expected: new mutations are not detected yet. + +**Step 3: Rewrite the active contract and manuals** + +Document the exact layout, root catalog schema, metadata equality, missing-descriptor bootstrap, create-once rule, curator ownership, docs-only API ownership, embedded/external Evidence, same-commit identity, migration cutover, and manual Git workflow. + +The migration guide must require one reviewed commit that: + +```bash +git mv workspaces/.yaml /workspace.yaml +git mv workspace-content//evidence /evidence +# create/review thoth-workspaces.yaml from descriptor metadata +``` + +It must say to upgrade ThothII only after that commit is pushed and to roll back application and repository revision together. Do not add an executable auto-migrator. + +Add an explicit P1.1 note to historical P1 design/plan references only if needed for navigation; do not rewrite accepted P1 history. + +**Step 4: Strengthen and run verifiers** + +```bash +bash scripts/test-verify-workspace-install-docs.sh +./scripts/verify-workspace-install-docs.sh --fixtures-only +bash scripts/test-verify-schema-v3-only.sh +./scripts/verify-schema-v3-only.sh +``` + +Expected: PASS. + +**Step 5: Commit** + +```bash +git add \ + docs/contracts/workspace-evidence-v3.md \ + docs/install/local-workspace-registry.md \ + docs/install/server-workspace-registry.md \ + docs/migrations/p1-to-p1-1-registry-layout.md \ + README.md \ + scripts/verify-workspace-install-docs.sh \ + scripts/test-verify-workspace-install-docs.sh \ + scripts/workspace_descriptor_doc_contract.py \ + scripts/test-workspace-descriptor-doc-contract.sh \ + scripts/verify-schema-v3-only.sh \ + scripts/test-verify-schema-v3-only.sh +git commit -m "docs: define the P1.1 registry layout and curator flow" +``` + +--- + +### Task 10: Update deployment fixtures and cross-platform registry smokes + +**Files:** +- Modify: `scripts/workspace-registry-smoke.sh` +- Modify: `backend/test/workspace-registry-deployment.test.ts` +- Modify: `scripts/unified-deployment-smoke.sh` +- Modify: `scripts/test-windows-clone-contract.ps1` +- Modify as needed: `scripts/fixtures/workspace-registry-smoke.yaml` +- Modify as needed: `scripts/fixtures/workspace-registry-task13.yaml` +- Modify as needed: `scripts/fixtures/workspace-registry-windows.yaml` +- Review: `.github/workflows/deployment.yml` + +**Step 1: Write failing deterministic fixture/smoke tests** + +Make every seed create a root catalog and nested descriptor path. Add mutations proving: + +- valid catalog + descriptor + Evidence starts; +- catalog/descriptor display metadata must change together in curator commits; +- orphan descriptor/mismatch/old layout is rejected while prior active snapshot stays usable; +- content-only Evidence update changes revision; +- API/bootstrap and curator paths remain separate; +- Windows paths with spaces preserve nested layout and LF/YAML contracts. + +**Step 2: Run focused deployment-contract tests and verify RED** + +```bash +cd backend +npx vitest run test/workspace-registry-deployment.test.ts +``` + +```bash +bash -n scripts/workspace-registry-smoke.sh scripts/unified-deployment-smoke.sh +``` + +Expected: old flat fixture assertions fail. + +**Step 3: Update seed/update/corruption helpers** + +Scripts copy standalone schema-v3 descriptor fixtures into `/workspace.yaml` and write a matching `thoth-workspaces.yaml`. Evidence goes below `/evidence` only for filesystem fixtures. Keep generated docs API-owned. + +Do not edit P2–P6 preprocessing fixtures in this task unless they are directly used by the generic registry deployment smoke; record deferred preprocessing paths for the later adaptation plan. + +**Step 4: Run deterministic gates** + +```bash +cd backend +npx vitest run test/workspace-registry-deployment.test.ts +``` + +Run non-Docker contract modes provided by the scripts and the Windows PowerShell contract on its supported CI/host. Run Docker smokes only at the final verification task so each is executed once from clean state. + +**Step 5: Commit** + +```bash +git add \ + scripts/workspace-registry-smoke.sh \ + backend/test/workspace-registry-deployment.test.ts \ + scripts/unified-deployment-smoke.sh \ + scripts/test-windows-clone-contract.ps1 \ + scripts/fixtures/workspace-registry-smoke.yaml \ + scripts/fixtures/workspace-registry-task13.yaml \ + scripts/fixtures/workspace-registry-windows.yaml \ + .github/workflows/deployment.yml +git commit -m "test: migrate registry deployment fixtures to P1.1" +``` + +--- + +### Task 11: Build independent automated P1.1 process acceptance + +**Files:** +- Add: `scripts/p11-acceptance.sh` +- Add: `scripts/test-p11-acceptance.sh` +- Add: `backend/scripts/p11-acceptance.mjs` +- Add: `backend/scripts/p11-acceptance.test.mjs` +- Add: `backend/scripts/acceptance-support.mjs` +- Add: `backend/scripts/acceptance-support.test.mjs` +- Modify only to import proven-equivalent generic guards: `backend/scripts/p1-acceptance.mjs` +- Modify/test: `backend/scripts/p1-acceptance.test.mjs` + +**Public command:** + +```bash +./scripts/p11-acceptance.sh integration --keep +``` + +**Artifact root:** + +```text +.artifacts/p11-integration// +``` + +Do not relabel, overwrite, or consume `.artifacts/p1-integration/**`. + +**Step 1: Write failing runner/lifecycle tests** + +Preserve P1's ownership-first, no-retry, fixed-argv, listener, secret-scan, report-hash, and confined-cleanup guards under the new P1.1 namespace. Test that P11 cleanup refuses P1/manual/sibling roots and vice versa. + +Extract only genuinely namespace-agnostic ownership, report, fixed-argv, secret-scan, and cleanup guards into `acceptance-support.mjs`. Keep P1/P11 roots, kinds, check IDs, reports, and process semantics in their versioned runners. Run both support and P1 unit suites to prove the extraction does not weaken P1 safety; do not claim the old P1 full integration scenario remains compatible with the new application contract. + +**Step 2: Run the runner test and verify RED** + +```bash +node --test backend/scripts/acceptance-support.test.mjs backend/scripts/p1-acceptance.test.mjs +bash scripts/test-p11-acceptance.sh +``` + +Expected: P1/support regression tests stay PASS; P11 test fails because P11 tooling does not exist. + +**Step 3: Implement the clean-state process** + +The retained run must: + +1. create a bare remote and curator clone from zero; +2. curator-push `thoth-workspaces.yaml` with filesystem/HTTP/S3 catalog slots, plus nested filesystem Evidence, but no descriptors; +3. start the production backend on loopback and list all slots as `configuration_required`; +4. validate and bootstrap-create all three descriptors through real HTTP, sequentially using the current base commit; +5. prove API writes only nested descriptor + `workspace-docs`, never catalog/Evidence; +6. prove second create, update, delete, catalog mismatch, present-empty descriptor, orphan descriptor, old layout, invalid path/protocol/secret field, and missing Git tree fail without mutation/leak; +7. curator-modify an existing descriptor and matching catalog metadata, push, then pull/sync and prove exact curator bytes are activated without descriptor rewrite; +8. push a content-only Evidence change and prove new commit identity with unchanged descriptor blob; +9. prove any docs-only follow-up commit changes only `workspace-docs/**`; +10. inspect exact catalog/descriptor/Evidence Git objects and immutable local snapshots; +11. acquire/release two production runtime configs for filesystem/HTTP/S3, compare bytes, and run real `tht config check -c ` in correct option order; +12. prove no P2 artifacts/commands, scan every non-secret-fixture byte and reachable Git blob for canaries, close listeners, and clean only owned resources. + +Required stable check IDs include at least: + +```text +preflight +clean_state +ownership +catalog_bootstrap +catalog_only_listing +bootstrap_create_once +api_curator_boundary +curator_descriptor_update +content_only_revision +docs_only_reconciliation +same_revision_git_objects +snapshot_and_export +runtime_render_determinism +tht_config_check +negative_catalog_layout_cases +negative_schema_context_cases +no_p2_scope_artifacts +secret_scan +cleanup_confinement +``` + +`report.md` must end with: + +```text +P1.1 automated integration: PASS +P1.1 manual acceptance: PENDING +``` + +**Step 4: Run runner tests** + +```bash +bash -n scripts/p11-acceptance.sh scripts/test-p11-acceptance.sh +bash scripts/test-p11-acceptance.sh +``` + +Expected: PASS. + +**Step 5: Run one fresh complete retained process** + +First verify no P11 runner/listener is active, then: + +```bash +./scripts/p11-acceptance.sh integration --keep +``` + +Expected: exit 0, one new root, all checks PASS, no retry/attempt loop, reports hash-bound to the exact clean source/runtime graph. + +On failure: retain the run, diagnose, add a regression test/fix, and execute a new full run with a new ID. Never overwrite or retry a failed run in place. + +**Step 6: Commit the tested P1.1 acceptance tooling** + +```bash +git add \ + scripts/p11-acceptance.sh \ + scripts/test-p11-acceptance.sh \ + backend/scripts/p11-acceptance.mjs \ + backend/scripts/p11-acceptance.test.mjs \ + backend/scripts/acceptance-support.mjs \ + backend/scripts/acceptance-support.test.mjs \ + backend/scripts/p1-acceptance.mjs \ + backend/scripts/p1-acceptance.test.mjs +git commit -m "test: prove the P1.1 registry process end to end" +``` + +--- + +### Task 12: Build the separate P1.1 manual acceptance environment + +**Files:** +- Add: `scripts/p11-manual-acceptance.sh` +- Add: `scripts/test-p11-manual-acceptance.sh` +- Add: `backend/scripts/p11-manual-acceptance.mjs` +- Add: `backend/scripts/p11-manual-acceptance.test.mjs` +- Add: `backend/scripts/p11-render-snapshot.mjs` +- Add: `backend/scripts/p11-render-snapshot.test.mjs` +- Add: `docs/testing/p11-manual-acceptance.md` + +**Public lifecycle:** + +```bash +./scripts/p11-manual-acceptance.sh prepare +./scripts/p11-manual-acceptance.sh serve +./scripts/p11-manual-acceptance.sh stop +./scripts/p11-manual-acceptance.sh cleanup +``` + +**Fixed independent root:** + +```text +.artifacts/manual-acceptance/p11/ +``` + +`serve` owns two loopback-only processes so the reviewer can exercise both real surfaces without Docker: the production Fastify backend on `127.0.0.1:8791` and a production-built frontend preview on a second fixed loopback port recorded in ownership. The lifecycle manifest binds both executable/start identities and listeners; `stop` and `cleanup` refuse partial or foreign ownership. It must never read/copy P1 or P11 automated run state. + +**Step 1: Write failing lifecycle/ownership tests** + +Port P1's hardened manual safeguards to the distinct P11 namespace while preserving P1 tests unchanged: + +- prepare refuses existing/symlink/unowned roots and creates ownership before child resources; +- serve binds only the fixed backend and frontend-preview loopback ports with exact PID/start/executable/build identities; +- stop signals only the two owned process groups/listeners and fails closed on a partial identity mismatch; +- cleanup refuses live/foreign state and removes only P11 root; +- no helper writes `VERDICT.md` or marks manual PASS; +- renderer accepts only owned immutable snapshots/output, writes 0600 atomically, always releases leases, and leaves deterministic bytes; +- fixture/command generation cannot accept path escapes, wrong catalog/commit, old layout, or P1 roots. + +**Step 2: Run lifecycle tests and verify RED** + +```bash +bash scripts/test-p11-manual-acceptance.sh +``` + +Expected: FAIL because tooling does not exist. + +**Step 3: Generate a reviewer-owned walkthrough** + +`prepare` creates a new bare remote/clone with catalog slots and nested filesystem Evidence but no descriptors, fixture secrets, requests, command scripts, and `GUIDE.md`. It does not call any positive API operation for the reviewer. + +The guide requires the reviewer personally to: + +1. inspect catalog, nested workspace dirs, Evidence, ownership, and secret path bindings; +2. serve the production backend plus production-built frontend preview and inspect every owned loopback listener; +3. list `configuration_required` slots; +4. validate and bootstrap-create descriptors once; +5. inspect exact Git objects and separate generated docs; +6. retry create/update/delete and verify refusal plus unchanged object IDs; +7. edit existing descriptor and matching catalog metadata in the curator clone, commit/push/pull, and verify API did not rewrite curator bytes; +8. make an Evidence-only commit and inspect revision identity; +9. inspect live UI read-only existing workspace and editable missing-slot bootstrap behavior; +10. export/import under bootstrap-only rules; +11. render twice, diff, and run `tht config check`; +12. run negative catalog/path/secret cases and a bounded secret scan; +13. stop, inspect listener/PID cleanup, record `VERDICT.md`, and only then cleanup when desired. + +**Step 4: Run tooling tests** + +```bash +bash -n scripts/p11-manual-acceptance.sh scripts/test-p11-manual-acceptance.sh +bash scripts/test-p11-manual-acceptance.sh +``` + +Expected: PASS. + +**Step 5: Commit tooling and guide** + +```bash +git add \ + scripts/p11-manual-acceptance.sh \ + scripts/test-p11-manual-acceptance.sh \ + backend/scripts/p11-manual-acceptance.mjs \ + backend/scripts/p11-manual-acceptance.test.mjs \ + backend/scripts/p11-render-snapshot.mjs \ + backend/scripts/p11-render-snapshot.test.mjs \ + docs/testing/p11-manual-acceptance.md +git commit -m "test: add independent P1.1 manual acceptance" +``` + +--- + +### Task 13: Final verification, retained evidence, and handoff to owner review + +**Files:** +- Modify only after successful verification: `PROJECT_STATE.md` +- Do not modify: P2–P6 plans/designs in this task + +**Step 1: Run deterministic source gates** + +```bash +git diff --check +bash scripts/test-verify-workspace-install-docs.sh +./scripts/verify-workspace-install-docs.sh --fixtures-only +bash scripts/test-verify-schema-v3-only.sh +./scripts/verify-schema-v3-only.sh +bash scripts/test-p11-acceptance.sh +bash scripts/test-p11-manual-acceptance.sh +``` + +Expected: PASS. + +**Step 2: Run complete backend verification** + +```bash +cd backend +npx vitest run +npx tsc --noEmit -p . +npm run build +``` + +Expected: PASS. Record exact test counts. + +**Step 3: Run complete frontend verification** + +```bash +cd frontend +npx vitest run +npx tsc -b +npm run build +``` + +Expected: PASS. Record exact test counts. + +**Step 4: Run harness regression verification** + +```bash +cd harness +.venv/bin/pytest -q +.venv/bin/ruff check \ + tht/config.py \ + tht/adapters/factory.py \ + tests/test_config_resources.py \ + tests/test_registry_evidence_config.py +``` + +Expected: pytest PASS and touched/relevant Python files Ruff-clean. Do not claim broad pre-existing Ruff debt is fixed unless `ruff check .` is also green. + +**Step 5: Run deployment/registry smokes once from clean state** + +Run the repository's normal deterministic deployment gates first, then each Docker smoke exactly once with its built-in timeout/ownership cleanup: + +```bash +./scripts/workspace-registry-smoke.sh +./scripts/unified-deployment-smoke.sh +``` + +Run the Windows native/clone contract in CI or an available supported Windows environment. If no Windows Docker runner is available, record the deterministic contract result and leave the manual Windows Docker gate explicitly unclaimed. + +Expected: PASS with exact cleanup and no global prune. + +**Step 6: Run one final P1.1 automated acceptance from clean state** + +```bash +./scripts/p11-acceptance.sh integration --keep +``` + +Expected: PASS, new unique retained path, reports bound to the clean implementation commit/tree and compiled graph immediately before the evidence-only PROJECT_STATE update. + +**Step 7: Audit forbidden scope and deferred plans** + +Use `git diff --name-only` plus targeted scans to prove: + +- no P2–P6 PRD/plan/design/manual-verification content was changed; +- no preprocessing/materialization/Qdrant/embedding implementation was added; +- no active runtime/doc/example still relies on flat `workspaces/.yaml` or `workspace-content//evidence`; +- any remaining old-path references are only historical P1 evidence/documents or the deliberately deferred P2–P6 sources inventoried for the later adjustment plan. + +**Step 8: Update project state to automated PASS/manual PENDING** + +Record exact source commit/tree, report paths/hashes, suite counts, smoke results, known limitations, and: + +```text +P1.1 automated integration: PASS +P1.1 manual acceptance: PENDING +``` + +Do not mark manual PASS. + +**Step 9: Prepare the independent manual environment and stop** + +```bash +./scripts/p11-manual-acceptance.sh prepare +``` + +Return the generated `GUIDE.md` path and lifecycle commands to the owner. Stop implementation work. Do not begin the P2–P6 adaptation plan before the owner completes and approves P1.1 manual acceptance. + +**Step 10: Commit final evidence metadata** + +```bash +git add PROJECT_STATE.md +git commit -m "docs: record P1.1 automated acceptance" +``` + +--- + +## Owner checkpoint after implementation + +The implementation session ends with: + +```text +P1.1 implementation: COMPLETE +P1.1 automated integration: PASS +P1.1 manual acceptance: PENDING +P2–P6 plans: UNCHANGED / ADAPTATION DEFERRED +``` + +The owner then executes `docs/testing/p11-manual-acceptance.md`. Only after an explicit manual PASS may a new planning-only task create the P2–P6 adaptation plan requested in steps 5–7 of the owner sequence. + +## Deferred P2–P6 impact inventory (do not edit during P1.1) + +The later adaptation-planning step must revisit at least: + +- `docs/prd/2026-08-09-workspace-preprocessing-prd.md` — old descriptor/Evidence layout and P1/P6 rows; +- `docs/superpowers/specs/2026-08-10-p2-p6-workspace-preprocessing-design.md` — P5 annotations path and P6 Evidence materialization path; +- `docs/superpowers/plans/2026-08-10-p2-host-workspace-preprocessing-cli.md` — exact descriptor/catalog identity, active snapshot fixture, and P1 dependency assumptions; +- `docs/testing/p2-p6-manual-verification.md` — old-path examples and future manual commands. + +Expected future canonical paths, subject to the separately approved adaptation plan: + +```text +/schema/annotations.yaml +/evidence +``` + +No P3/P4/P5/P6 implementation plan files currently exist separately; their present contract lives in the combined design/PRD and must be split or revised only in the later authorized phase. diff --git a/docs/superpowers/specs/2026-08-11-p1-1-workspace-directory-registry-design.md b/docs/superpowers/specs/2026-08-11-p1-1-workspace-directory-registry-design.md new file mode 100644 index 00000000..895e1bb7 --- /dev/null +++ b/docs/superpowers/specs/2026-08-11-p1-1-workspace-directory-registry-design.md @@ -0,0 +1,255 @@ +# P1.1 Workspace-Directory Git Registry Design + +**Status:** Proposed for owner approval +**Date:** 2026-08-11 +**Supersedes:** The repository-layout and descriptor-publication portions of P1; P1's Evidence configuration, immutable revision, local-secret, rendering, and verification contracts remain in force. +**Deferred:** Any edits to P2–P6. Their impact will be planned only after P1.1 manual acceptance. + +## 1. Goal + +Make one shared Git repository read naturally as a catalog of self-contained workspaces. Each +workspace owns one directory containing its technical descriptor and, when Evidence is embedded, +its curated Evidence tree. A root catalog establishes the canonical workspace IDs, names, and +descriptions. ThothII may create a missing descriptor once as a bootstrap convenience, but after +that the descriptor is curator-owned and may be changed or removed only through ordinary Git +review and push. + +P1.1 is a correction to P1, not the preprocessing project. It performs no DWH introspection, +Evidence acquisition, materialization, embeddings, Qdrant writes, ACTIVE publication, FK curation, +or retention execution. + +## 2. Chosen repository contract + +```text +thoth-workspaces.git/ +├── thoth-workspaces.yaml +├── psd/ +│ ├── workspace.yaml +│ └── evidence/ +│ └── ... curated Evidence files ... +├── external-research/ +│ └── workspace.yaml # Evidence may instead be HTTP or S3 +└── workspace-docs/ + ├── psd/ + │ ├── contract.env.example + │ └── README.md + └── external-research/ + ├── contract.env.example + └── README.md +``` + +The fixed paths are: + +```text +catalog thoth-workspaces.yaml +workspace descriptor /workspace.yaml +embedded filesystem Evidence /evidence +future curated FK annotations /schema/annotations.yaml # P5, not P1.1 +public generated docs workspace-docs//{contract.env.example,README.md} +``` + +Internal installation snapshots deliberately remain flat: + +```text +/workspace-registry/snapshots//.yaml +``` + +This avoids changing ThtRunner's trusted-snapshot contract, historical session pins, runtime lease +files, or resume behavior. Repository layout and internal snapshot layout are separate contracts. + +## 3. Root catalog + +The curator owns `thoth-workspaces.yaml`. The API never creates, edits, deletes, stages, or cleans +it. Its strict initial shape is: + +```yaml +schema_version: 1 +workspaces: + - id: psd + name: Policlinico San Donato + description: Data warehouse clinico del Policlinico San Donato +``` + +Rules: + +- IDs use the existing `^[a-z][a-z0-9-]{2,62}$` contract, are unique, and cannot equal the reserved API directory `workspace-docs`. +- `name` is required; `description` is optional. Existing trim/nonblank, Unicode, and safe-error behavior used by descriptor metadata are reused; P1.1 introduces no new string-length limit. +- Unknown keys, duplicate YAML keys, aliases/tags, multiple documents, malformed encodings, and + duplicate IDs are rejected. +- Catalog order is the workspace display order. +- A catalog entry may temporarily have no descriptor. This is the only bootstrap state and is + represented publicly as `configuration_required`; it is not session-activatable. +- `workspace-docs` is a reserved top-level API directory and cannot be a workspace ID. +- The dedicated registry accepts only catalog-listed workspace directories plus the reserved + generated-docs directory and explicitly allowed root control files. An unlisted workspace + directory/descriptor, or a descriptor whose `workspace.id`, `workspace.name`, or optional + `workspace.description` differs from the catalog, is invalid. Activation fails atomically and + retains the prior valid snapshot. +- A present but empty or malformed descriptor is not "empty" for bootstrap. It is curator content + and is rejected; the API never replaces it. + +The descriptor retains `id`, `name`, and `description` so exports and immutable runtime snapshots +remain self-contained. The catalog is authoritative, and exact equality prevents two names for one +workspace. + +## 4. Ownership and write policy + +There are three writers with disjoint authority: + +| Path | Owner | ThothII API behavior | +| --- | --- | --- | +| `thoth-workspaces.yaml` | curator | read and validate only | +| `/workspace.yaml` | curator after bootstrap | create only if absent at the exact base commit; never overwrite or delete | +| `/evidence/**` | curator | read Git objects only; never write, stage, clean, or materialize in P1.1 | +| `workspaces//schema/**` | curator/future P5 | untouched by P1.1 | +| `workspace-docs//*` | API | deterministic generated files only | + +"Absent" means no Git object exists at `/workspace.yaml` in the exact pulled base +commit. A zero-byte file, comments-only YAML, symlink, submodule, tree, or malformed document counts +as present and is never overwritten. + +A browser/API bootstrap succeeds only when: + +1. the catalog entry already exists at the request's exact `baseCommit`; +2. the descriptor path is absent at that commit and remains absent after the pull; +3. request metadata exactly matches the catalog; +4. filesystem Evidence, when selected, already exists as a Git tree at + `/evidence` in that same base commit; +5. the complete descriptor passes schema-v3 and operational publication checks. + +The API then commits only the new descriptor and generated docs. Update and delete requests against +an existing descriptor return a stable `workspace_curator_owned` conflict response and do not +change any Git object. Curator deletion means removing the descriptor or catalog/directory through +Git. A retained session snapshot remains available under the existing retention rules. + +The managed checkout must no longer run a directory-wide clean under `workspaces/`. Failure cleanup +is confined to the exact descriptor/docs files written by the failed API operation and proves their +pre-operation identity before removal. + +## 5. Synchronization and generated docs + +Startup/bootstrap may pull, validate, and activate curator bytes but never pushes as a side effect +of a status/read request. This deliberately means committed `workspace-docs` can remain stale until +an explicit synchronization action; immutable local snapshots and exports always derive fresh docs +from the validated active descriptor and never consume stale Git docs. The explicit +`/workspace-registry/pull` operator action remains the synchronization boundary: + +1. pull the curator commit; +2. validate catalog, descriptors, namespace ownership, Evidence roots, and semantic-index ownership; +3. compute deterministic `workspace-docs/` bytes; +4. if docs differ, create one docs-only follow-up commit without touching catalog, descriptors, or + workspace content; +5. validate and activate the resulting exact commit. + +Every API write transaction records a bounded per-path journal before mutation: prior Git object type, +mode, blob identity and bytes for tracked generated docs, or explicit absence, plus the intended +post-write identity. If bootstrap/docs push races, is rejected, or fails, the prior valid active +snapshot remains active; overwritten/deleted generated docs are restored byte-for-byte to their +prior objects, newly created absent-before files are removed, and curator paths are never cleaned. +A later explicit pull retries from a fresh remote head. The API removes stale generated docs only for workspaces that the curator has +removed from the catalog or returned to `configuration_required`. + +A docs-only follow-up commit is an authoritative workspace revision, as every active workspace is +pinned to the complete Git commit rather than only to its descriptor blob. Automated acceptance +must show that descriptor and Evidence blob identities are unchanged across that docs-only commit. + +## 6. API and browser behavior + +`GET /workspaces` is catalog-driven and returns every catalog entry in catalog order with: + +- canonical ID, display name, and description from the catalog; +- `configurationState: ready | configuration_required`; +- `file: /workspace.yaml`; +- an immutable revision only for `ready` entries. + +`language` is deliberately not a summary field because an unconfigured catalog slot has no +descriptor language. It remains available from the descriptor detail for `ready` workspaces and is +selected in the bootstrap draft before creation. + +Descriptor-only routes (`GET /workspaces/:id`, diagnostics, export, session admission) reject a +`configuration_required` entry as `workspace_not_activatable`. + +`POST /workspaces/validate` remains a context-free schema check. It does not claim catalog +agreement or publication eligibility. `POST /workspaces/publish` becomes bootstrap-create only. +Legacy update/delete payloads are recognized and rejected as `workspace_curator_owned` rather than +silently reinterpreted. + +The browser: + +- lists catalog slots, including those requiring configuration; +- offers an editable, browser-local bootstrap draft only for `configuration_required` entries; +- locks catalog-owned ID/name/description in that form; +- requires explicit validation and confirmation before the one create; +- turns the workspace read-only immediately after creation; +- keeps Pull/Sync, Validate, installation Test, Export, and safe Evidence summary for existing + workspaces; +- removes update, delete, duplicate, field-conflict merge, and publish-existing controls; +- versions or purges old update/deletion drafts so stale localStorage cannot restore write access; +- treats imported bundles as bootstrap drafts only when they match an existing unconfigured + catalog slot. + +Existing descriptors are edited in the curator clone and become active after commit, push, and +installation pull. + +## 7. Evidence and external sources + +For filesystem Evidence, schema v3 now requires exactly: + +```yaml +evidence: + source: + type: filesystem + uri: /evidence +``` + +The lexical path invariant and same-commit Git-tree check remain P1.1 responsibilities. Recursive +materialization, nested symlink rejection, byte acquisition, preprocessing, and indexing remain P6 +or later. + +HTTP and S3 descriptor shapes, local `*_FILE` bindings, secret handling, timeout/limit policy, +runtime rendering, and `tht config check` remain as delivered by P1. Those workspaces need no local +`evidence/` directory. Evidence may also remain absent for compatibility. + +## 8. Rejected alternatives + +1. **Keep P1's three top-level source trees.** Rejected because it does not make a workspace a + self-contained Git unit and does not match the desired curator model. +2. **Place descriptors and Evidence together but let both API and curator update descriptors.** + Rejected because it creates two authorities, restores field-level conflict merging, and risks + overwriting reviewed Git content. +3. **Chosen: API bootstrap once, then curator ownership.** This preserves a convenient initial + form while making ordinary Git review the single authority for all subsequent descriptor and + content changes. + +## 9. Compatibility and migration + +P1.1 is a repository-contract cutover, not a dual-format reader. New code rejects the old flat +layout and a repository without `thoth-workspaces.yaml`. Existing repositories are migrated in one +curator-reviewed commit: + +```text +workspaces/.yaml -> /workspace.yaml +workspace-content//evidence/** -> /evidence/** +(create thoth-workspaces.yaml from reviewed descriptor metadata) +``` + +No automatic in-product migrator rewrites a remote. The installation upgrades only after the +migration commit is available. Historical immutable installation snapshots and retained session +pins keep their current internal shape. + +P2–P6 currently assume P1's old source paths in several places. P1.1 records that impact but does +not edit those plans. After P1.1 automated and manual acceptance, a separate owner-approved plan +will revise P2–P6. + +## 10. Verification boundary + +P1.1 must have independent automated and manual evidence. Accepted retained P1 artifacts remain immutable historical evidence; the old P1 process commands are not release gates for the superseding repository contract. The automated run starts from a clean +local Git remote and proves catalog authority, missing-descriptor bootstrap, curator modification, +API non-overwrite, nested Evidence identity, docs-only reconciliation, immutable snapshots, +runtime render determinism, `tht config check`, negative cases, secret absence, and exact cleanup. +It does not invoke preprocessing or Qdrant/Ollama/DWH services. + +The manual environment is new and independent. The reviewer personally performs the bootstrap, +refusal, curator-edit, pull/sync, UI read-only, Git-object, export, render, config-check, secret-scan, +and cleanup checks. Project state remains `P1.1 manual acceptance: PENDING` until the reviewer +records approval.