From f95a18ab0d0568776436ce4f03f9df52122c706f Mon Sep 17 00:00:00 2001 From: mptyl Date: Tue, 4 Aug 2026 01:01:57 +0200 Subject: [PATCH] feat: expose workspace registry API --- backend/package-lock.json | 57 ++++ backend/package.json | 2 + backend/src/app.ts | 12 +- backend/src/routes/meta.ts | 4 - backend/src/routes/workspaces.ts | 362 +++++++++++++++++++++++ backend/src/workspaces/git-repository.ts | 39 ++- backend/src/workspaces/registry.ts | 109 ++++++- backend/test/routes-sql-meta.test.ts | 36 +-- backend/test/routes-workspaces.test.ts | 258 ++++++++++++++++ 9 files changed, 829 insertions(+), 50 deletions(-) create mode 100644 backend/src/routes/workspaces.ts create mode 100644 backend/test/routes-workspaces.test.ts diff --git a/backend/package-lock.json b/backend/package-lock.json index 866c39e8..bb100dc7 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -7,6 +7,7 @@ "name": "thothii-backend", "dependencies": { "@fastify/cors": "^11.2.0", + "@fastify/multipart": "^9.4.0", "@types/pg": "^8.20.3", "fastify": "^5.0.0", "pg": "^8.22.0", @@ -18,6 +19,7 @@ "devDependencies": { "@types/node": "^22.0.0", "@types/yauzl": "^3.4.0", + "@types/yazl": "^3.3.1", "tsx": "^4.19.0", "typescript": "^5.6.0", "vitest": "^2.1.0" @@ -486,6 +488,12 @@ "fast-uri": "^3.0.0" } }, + "node_modules/@fastify/busboy": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/@fastify/busboy/-/busboy-3.2.0.tgz", + "integrity": "sha512-m9FVDXU3GT2ITSe0UaMA5rU3QkfC/UXtCU8y0gSN/GugTqtVldOBWIB5V6V3sbmenVZUIpU6f+mPEO2+m5iTaA==", + "license": "MIT" + }, "node_modules/@fastify/cors": { "version": "11.2.0", "resolved": "https://registry.npmjs.org/@fastify/cors/-/cors-11.2.0.tgz", @@ -506,6 +514,22 @@ "toad-cache": "^3.7.0" } }, + "node_modules/@fastify/deepmerge": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/@fastify/deepmerge/-/deepmerge-3.2.1.tgz", + "integrity": "sha512-N5Oqvltoa2r9z1tbx4xjky0oRR60v+T47Ic4J1ukoVQcptLOrIdRnCSdTGmOmajZuHVKlTnfcmrjyqsGEW1ztA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "MIT" + }, "node_modules/@fastify/error": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/@fastify/error/-/error-4.2.0.tgz", @@ -576,6 +600,29 @@ "dequal": "^2.0.3" } }, + "node_modules/@fastify/multipart": { + "version": "9.4.0", + "resolved": "https://registry.npmjs.org/@fastify/multipart/-/multipart-9.4.0.tgz", + "integrity": "sha512-Z404bzZeLSXTBmp/trCBuoVFX28pM7rhv849Q5TsbTFZHuk1lc4QjQITTPK92DKVpXmNtJXeHSSc7GYvqFpxAQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "MIT", + "dependencies": { + "@fastify/busboy": "^3.0.0", + "@fastify/deepmerge": "^3.0.0", + "@fastify/error": "^4.0.0", + "fastify-plugin": "^5.0.0", + "secure-json-parse": "^4.0.0" + } + }, "node_modules/@fastify/proxy-addr": { "version": "5.1.0", "resolved": "https://registry.npmjs.org/@fastify/proxy-addr/-/proxy-addr-5.1.0.tgz", @@ -996,6 +1043,16 @@ "@types/node": "*" } }, + "node_modules/@types/yazl": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/@types/yazl/-/yazl-3.3.1.tgz", + "integrity": "sha512-DIWfCKpsTp6hE5BDBHV3+fIL/bLUF9Bv13iDrWnMlmhQpH67buNvI291ZauQ1xcccxK3FqQ9honnXpq4R8NMuQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@vitest/expect": { "version": "2.1.9", "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-2.1.9.tgz", diff --git a/backend/package.json b/backend/package.json index e337a124..caaab948 100644 --- a/backend/package.json +++ b/backend/package.json @@ -10,6 +10,7 @@ }, "dependencies": { "@fastify/cors": "^11.2.0", + "@fastify/multipart": "^9.4.0", "@types/pg": "^8.20.3", "fastify": "^5.0.0", "pg": "^8.22.0", @@ -21,6 +22,7 @@ "devDependencies": { "@types/node": "^22.0.0", "@types/yauzl": "^3.4.0", + "@types/yazl": "^3.3.1", "tsx": "^4.19.0", "typescript": "^5.6.0", "vitest": "^2.1.0" diff --git a/backend/src/app.ts b/backend/src/app.ts index 66d3ec83..110ceac1 100644 --- a/backend/src/app.ts +++ b/backend/src/app.ts @@ -17,6 +17,7 @@ import { loadSettings, saveSettings, type Settings } from "./settings/settings-s import { ReadinessManager } from "./runtime/readiness-manager.js"; import { WorkspaceRegistry } from "./workspaces/registry.js"; import { createProductionWorkspaceDiagnoser } from "./workspaces/diagnostics.js"; +import { workspaceRoutes, type WorkspaceDiagnoser } from "./routes/workspaces.js"; export interface BuildAppDeps { thtRunner?: ThtRunner; @@ -27,6 +28,7 @@ export interface BuildAppDeps { readiness?: ReadinessManager; hub?: SseHub; workspaceRegistry?: WorkspaceRegistry; + workspaceDiagnoser?: WorkspaceDiagnoser; } export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstance { @@ -50,14 +52,9 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc }); const mgr = deps?.mgr ?? new PiProcessManager(config, deps?.spawnFn ? { spawnFn: deps.spawnFn } : undefined); const hub = deps?.hub ?? new SseHub(); - // Routes are introduced in Task 6; construction here keeps production and injected-test - // dependencies on the same registry lifecycle without performing Git I/O at startup. const workspaceRegistry = deps?.workspaceRegistry ?? new WorkspaceRegistry(config.workspaceRegistry); - void workspaceRegistry; - // Task 6 consumes this dependency from the registry route. Construct it from the effective - // application configuration here so production diagnostics never silently use test defaults. - const workspaceDiagnoser = createProductionWorkspaceDiagnoser(config.workspaceDiagnosticTimeoutMs); - void workspaceDiagnoser; + const workspaceDiagnoser = deps?.workspaceDiagnoser + ?? createProductionWorkspaceDiagnoser(config.workspaceDiagnosticTimeoutMs); const readiness = deps?.readiness ?? new ReadinessManager( tht as ThtRunner, Math.round(config.ollamaEnsureTimeoutMs / 1000), @@ -113,6 +110,7 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc }); sqlRoutes(app, { tht: tht as ThtRunner, getSettings }); metaRoutes(app, { harnessDir: config.harnessDir, listModels }); + workspaceRoutes(app, { registry: workspaceRegistry, config: config.workspaceRegistry, diagnose: workspaceDiagnoser }); settingsRoutes(app, { cfg: config, listModels, getSettings, saveSettings: saveUserSettings }); return app; diff --git a/backend/src/routes/meta.ts b/backend/src/routes/meta.ts index db8149c5..94572160 100644 --- a/backend/src/routes/meta.ts +++ b/backend/src/routes/meta.ts @@ -26,10 +26,6 @@ export function metaRoutes( app: FastifyInstance, deps: { harnessDir: string; listModels?: ListModelsFn }, ): void { - app.get("/workspaces", async () => { - return listWorkspaces(deps.harnessDir); - }); - app.get("/models", async () => { const fn = deps.listModels ?? (async () => []); try { diff --git a/backend/src/routes/workspaces.ts b/backend/src/routes/workspaces.ts new file mode 100644 index 00000000..99d9495a --- /dev/null +++ b/backend/src/routes/workspaces.ts @@ -0,0 +1,362 @@ +import { createHash } from "node:crypto"; +import { Buffer } from "node:buffer"; +import type { FastifyInstance, FastifyReply, FastifyRequest } from "fastify"; +import multipart from "@fastify/multipart"; +import yauzl from "yauzl"; +import yazl from "yazl"; +import { z } from "zod"; +import type { WorkspaceRegistryConfig } from "../workspaces/types.js"; +import { WorkspaceRegistryError } from "../workspaces/git-repository.js"; +import { + WorkspaceConflictError, + type PublishWorkspaceRequest, + type WorkspaceRegistry, +} from "../workspaces/registry.js"; +import { resolveRuntimeBindings } from "../workspaces/bindings.js"; +import { buildInstallationContract, renderWorkspaceDocs } from "../workspaces/contracts.js"; +import { + isCanonicalWorkspace, + parseWorkspaceYaml, + serializeWorkspaceYaml, + validateCanonicalWorkspace, + type CanonicalWorkspace, + type WorkspaceDescriptor, +} from "../workspaces/schema.js"; +import type { RuntimeBindings } from "../workspaces/runtime-renderer.js"; +import type { WorkspaceDiagnostics } from "../workspaces/diagnostics.js"; + +export type WorkspaceDiagnoser = ( + workspace: WorkspaceDescriptor, + bindings: RuntimeBindings, + options: { writeProbe: boolean }, +) => Promise; + +interface WorkspaceRoutesDeps { + registry: WorkspaceRegistry; + config: WorkspaceRegistryConfig; + diagnose: WorkspaceDiagnoser; +} + +const workspaceId = z.string().regex(/^[a-z][a-z0-9-]{2,62}$/); +const commit = z.string().regex(/^[0-9a-f]{40}$/); +const workspacePayload = z.object({ workspace: z.unknown() }).strict(); +const publishPayload = z.discriminatedUnion("action", [ + z.object({ action: z.literal("create"), workspace: z.unknown(), baseCommit: commit }).strict(), + z.object({ action: z.literal("update"), workspace: z.unknown(), baseCommit: commit, baseBlob: commit }).strict(), + z.object({ action: z.literal("delete"), id: workspaceId, baseCommit: commit, baseBlob: commit }).strict(), +]); +const bundleManifest = z.object({ + schema_version: z.literal(1), + workspace_id: workspaceId, + files: z.object({ + "workspace.yaml": z.string().regex(/^[0-9a-f]{64}$/), + "contract.env.example": z.string().regex(/^[0-9a-f]{64}$/), + "README.md": z.string().regex(/^[0-9a-f]{64}$/), + }).strict(), +}).strict(); + +const BUNDLE_FILES = ["manifest.json", "workspace.yaml", "contract.env.example", "README.md"] as const; +type BundleFile = (typeof BUNDLE_FILES)[number]; + +const SAFE_MESSAGES = { + workspace_invalid: "Workspace request or bundle is invalid.", + binding_missing: "Installation binding is missing or invalid.", + workspace_not_activatable: "Workspace cannot be activated on this installation.", + workspace_stale: "Workspace revision is stale.", + workspace_conflict: "Workspace changed in the registry.", + git_unavailable: "Workspace Git service is unavailable.", + git_auth_failed: "Workspace Git authentication failed.", + git_non_fast_forward: "Workspace Git branch has changed.", + git_push_rejected: "Workspace Git publication was rejected.", + connector_unavailable: "Workspace connector is unavailable.", + semantic_index_incompatible: "Semantic index is incompatible with this workspace.", +} as const; + +function sha256(value: string | Buffer): string { + return createHash("sha256").update(value).digest("hex"); +} + +function invalidBundle(): WorkspaceRegistryError { + return new WorkspaceRegistryError("workspace_invalid", "Workspace bundle is invalid"); +} + +function isBundleFile(value: string): value is BundleFile { + return (BUNDLE_FILES as readonly string[]).includes(value); +} + +function unsafeArchiveEntry(entry: yauzl.Entry): boolean { + const name = entry.fileName; + const unixType = (entry.externalFileAttributes >>> 16) & 0o170000; + return name.length === 0 + || name.startsWith("/") + || name.startsWith("\\") + || name.includes("\\") + || name.split("/").includes("..") + || name.endsWith("/") + || unixType === 0o120000 + || !isBundleFile(name); +} + +async function readZipBundle(source: Buffer, config: WorkspaceRegistryConfig): Promise> { + if (source.length === 0 || source.length > config.maxImportBytes) throw invalidBundle(); + return await new Promise>((resolve, reject) => { + yauzl.fromBuffer(source, { + lazyEntries: true, + strictFileNames: true, + validateEntrySizes: true, + decodeStrings: true, + }, (error, archive) => { + if (error || !archive) return reject(invalidBundle()); + const files = new Map(); + let entries = 0; + let settled = false; + const fail = () => { + if (settled) return; + settled = true; + archive.close(); + reject(invalidBundle()); + }; + archive.on("error", fail); + archive.on("entry", (entry) => { + entries += 1; + if (entries > config.maxImportEntries || unsafeArchiveEntry(entry) || files.has(entry.fileName as BundleFile)) { + fail(); + return; + } + if (entry.uncompressedSize > config.maxImportBytes) { + fail(); + return; + } + archive.openReadStream(entry, (streamError, stream) => { + if (streamError || !stream) return fail(); + const chunks: Buffer[] = []; + let size = 0; + stream.on("data", (chunk: Buffer) => { + size += chunk.length; + if (size > config.maxImportBytes) return fail(); + chunks.push(chunk); + }); + stream.on("error", fail); + stream.on("end", () => { + if (settled || size !== entry.uncompressedSize) return fail(); + files.set(entry.fileName as BundleFile, Buffer.concat(chunks)); + archive.readEntry(); + }); + }); + }); + archive.on("end", () => { + if (settled) return; + settled = true; + if (entries !== BUNDLE_FILES.length || BUNDLE_FILES.some((name) => !files.has(name))) return reject(invalidBundle()); + resolve(Object.fromEntries(files) as Record); + }); + archive.readEntry(); + }); + }); +} + +function utf8(buffer: Buffer): string { + const text = buffer.toString("utf8"); + if (!Buffer.from(text, "utf8").equals(buffer) || text.includes("\0")) throw invalidBundle(); + return text; +} + +async function importDraft(source: Buffer, config: WorkspaceRegistryConfig): Promise { + const files = await readZipBundle(source, config); + let manifest: z.infer; + try { + manifest = bundleManifest.parse(JSON.parse(utf8(files["manifest.json"]))); + } catch { + throw invalidBundle(); + } + for (const name of ["workspace.yaml", "contract.env.example", "README.md"] as const) { + if (sha256(files[name]) !== manifest.files[name]) throw invalidBundle(); + } + try { + const descriptor = parseWorkspaceYaml(utf8(files["workspace.yaml"])); + const workspace = validateCanonicalWorkspace(descriptor); + const docs = renderWorkspaceDocs(workspace); + if ( + workspace.workspace.id !== manifest.workspace_id + || serializeWorkspaceYaml(workspace) !== utf8(files["workspace.yaml"]) + || docs.envExample !== utf8(files["contract.env.example"]) + || docs.markdown !== utf8(files["README.md"]) + ) throw invalidBundle(); + return workspace; + } catch (error) { + if (error instanceof WorkspaceRegistryError) throw error; + throw invalidBundle(); + } +} + +async function exportBundle(workspace: CanonicalWorkspace): Promise { + const yaml = serializeWorkspaceYaml(workspace); + const docs = renderWorkspaceDocs(workspace); + const files: Record = { + "manifest.json": JSON.stringify({ + schema_version: 1, + workspace_id: workspace.workspace.id, + files: { + "workspace.yaml": sha256(yaml), + "contract.env.example": sha256(docs.envExample), + "README.md": sha256(docs.markdown), + }, + }), + "workspace.yaml": yaml, + "contract.env.example": docs.envExample, + "README.md": docs.markdown, + }; + const archive = new yazl.ZipFile(); + const chunks: Buffer[] = []; + archive.outputStream.on("data", (chunk: Buffer) => chunks.push(chunk)); + for (const name of BUNDLE_FILES) archive.addBuffer(Buffer.from(files[name]), name); + archive.end(); + await new Promise((resolve, reject) => { + archive.outputStream.once("end", resolve); + archive.outputStream.once("error", reject); + }); + return Buffer.concat(chunks); +} + +function workspaceErrorCode(error: unknown): keyof typeof SAFE_MESSAGES { + return error instanceof WorkspaceRegistryError ? error.code : "workspace_invalid"; +} + +function workspaceErrorStatus(code: keyof typeof SAFE_MESSAGES): number { + if (code === "workspace_conflict" || code === "workspace_stale" || code === "git_non_fast_forward") return 409; + if (code === "git_unavailable" || code === "git_auth_failed" || code === "git_push_rejected") return 503; + return 400; +} + +function errorReply(reply: FastifyReply, error: unknown) { + const code = workspaceErrorCode(error); + const body: Record = { code, message: SAFE_MESSAGES[code] }; + if (error instanceof WorkspaceConflictError) { + body.fields = error.fields; + if (error.base) body.base = error.base; + if (error.local) body.local = error.local; + if (error.remote) body.remote = error.remote; + } else if (code === "workspace_conflict" && error && typeof error === "object") { + const conflict = error as Partial; + if (Array.isArray(conflict.fields) && conflict.fields.every((field) => typeof field === "string")) body.fields = conflict.fields; + for (const key of ["base", "local", "remote"] as const) { + if (conflict[key] && isCanonicalWorkspace(conflict[key] as WorkspaceDescriptor)) body[key] = conflict[key]; + } + } + return reply.code(workspaceErrorStatus(code)).send(body); +} + +function publishRequest(value: unknown): PublishWorkspaceRequest { + const parsed = publishPayload.parse(value); + if (parsed.action === "delete") return parsed; + return { ...parsed, workspace: validateCanonicalWorkspace(parsed.workspace) }; +} + +export function workspaceRoutes(app: FastifyInstance, deps: WorkspaceRoutesDeps): void { + app.register(multipart, { + limits: { fileSize: deps.config.maxImportBytes, files: 1, fields: 0, parts: 1 }, + throwFileSizeLimit: true, + }); + + app.get("/workspace-registry/status", async (_request, reply) => { + try { + return await deps.registry.bootstrap(); + } catch (error) { + return errorReply(reply, error); + } + }); + + app.post("/workspace-registry/pull", async (_request, reply) => { + try { + return await deps.registry.pull(); + } catch (error) { + return errorReply(reply, error); + } + }); + + app.get("/workspaces", async (_request, reply) => { + try { + const revisions = await deps.registry.list(); + return await Promise.all(revisions.map(async (revision) => { + const { workspace } = await deps.registry.read(revision.id); + return { + id: revision.id, + // Retain the metadata endpoint's selector fields while adding registry summary data. + name: revision.id, + file: `${revision.id}.yaml`, + displayName: workspace.workspace.name, + description: workspace.workspace.description, + language: workspace.workspace.language, + revision, + }; + })); + } catch (error) { + return errorReply(reply, error); + } + }); + + app.get("/workspaces/:id", async (request, reply) => { + try { + const { id } = z.object({ id: workspaceId }).parse(request.params); + return await deps.registry.read(id); + } catch (error) { + return errorReply(reply, error); + } + }); + + app.post("/workspaces/validate", async (request, reply) => { + try { + const { workspace } = workspacePayload.parse(request.body); + const canonical = validateCanonicalWorkspace(workspace); + return { workspace: canonical, contract: buildInstallationContract(canonical) }; + } catch (error) { + return errorReply(reply, error); + } + }); + + app.post("/workspaces/:id/test", async (request, reply) => { + try { + const { id } = z.object({ id: workspaceId }).parse(request.params); + const { workspace } = await deps.registry.read(id); + const bindings = resolveRuntimeBindings(workspace, process.env, deps.config.secretRoots); + return await deps.diagnose(workspace, bindings, { writeProbe: false }); + } catch (error) { + return errorReply(reply, error); + } + }); + + app.post("/workspaces/publish", async (request, reply) => { + try { + const result = await deps.registry.publish(publishRequest(request.body)); + return result ? { revision: result } : reply.code(204).send(); + } catch (error) { + return errorReply(reply, error); + } + }); + + app.get("/workspaces/:id/export", async (request, reply) => { + try { + const { id } = z.object({ id: workspaceId }).parse(request.params); + const { workspace } = await deps.registry.read(id); + const canonical = validateCanonicalWorkspace(workspace); + const bundle = await exportBundle(canonical); + return reply + .type("application/zip") + .header("content-disposition", `attachment; filename=\"${id}.zip\"`) + .send(bundle); + } catch (error) { + return errorReply(reply, error); + } + }); + + app.post("/workspaces/import", async (request: FastifyRequest, reply) => { + try { + const file = await request.file(); + if (!file || file.fieldname !== "bundle" || file.mimetype !== "application/zip") throw invalidBundle(); + const draft = await importDraft(await file.toBuffer(), deps.config); + return { draft: { workspace: draft, contract: buildInstallationContract(draft) } }; + } catch (error) { + return errorReply(reply, error); + } + }); +} diff --git a/backend/src/workspaces/git-repository.ts b/backend/src/workspaces/git-repository.ts index f53a6e02..61db4b99 100644 --- a/backend/src/workspaces/git-repository.ts +++ b/backend/src/workspaces/git-repository.ts @@ -1,7 +1,7 @@ import { execFile, spawn, type ChildProcessWithoutNullStreams } from "node:child_process"; import { lstatSync, mkdirSync } from "node:fs"; -import { mkdir } from "node:fs/promises"; -import { basename, isAbsolute, join } from "node:path"; +import { mkdir, rm, writeFile } from "node:fs/promises"; +import { basename, dirname, isAbsolute, join } from "node:path"; import { promisify } from "node:util"; import type { WorkspaceErrorCode, WorkspaceRegistryConfig } from "./types.js"; @@ -154,6 +154,30 @@ export class GitWorkspaceRepository { return (await this.git(["rev-parse", `HEAD:${path}`])).trim(); } + /** Write only a validated registry artifact below the checked-out repository. */ + async writeRegistryFile(path: string, source: string): Promise { + this.assertRegistryArtifactPath(path); + const target = join(this.repoPath, path); + await mkdir(dirname(target), { recursive: true, mode: 0o700 }); + await writeFile(target, source, { encoding: "utf8", mode: 0o600 }); + } + + async removeRegistryFile(path: string): Promise { + this.assertRegistryArtifactPath(path); + await rm(join(this.repoPath, path), { force: true }); + } + + /** Commit and push a fixed set of validated artifact paths without exposing Git output. */ + async commitAndPush(paths: readonly string[], message: string): Promise { + if (paths.length === 0 || paths.some((path) => !this.isRegistryArtifactPath(path))) { + throw new WorkspaceRegistryError("workspace_invalid", "Workspace repository path is invalid"); + } + await this.git(["add", "--", ...paths]); + await this.git(["commit", "-m", message]); + await this.git(["push", "origin", `HEAD:${this.config.branch}`]); + return await this.status(); + } + private async clone(): Promise { try { await execFileAsync("git", [ @@ -166,6 +190,17 @@ export class GitWorkspaceRepository { } } + private isRegistryArtifactPath(path: string): boolean { + return /^workspaces\/[a-z][a-z0-9-]{2,62}\.yaml$/.test(path) + || /^workspace-docs\/[a-z][a-z0-9-]{2,62}\/(?:contract\.env\.example|README\.md)$/.test(path); + } + + private assertRegistryArtifactPath(path: string): void { + if (!this.isRegistryArtifactPath(path)) { + throw new WorkspaceRegistryError("workspace_invalid", "Workspace repository path is invalid"); + } + } + private async refresh(): Promise { if ((await this.git(["status", "--porcelain"])).trim() !== "") { throw new WorkspaceRegistryError("workspace_stale", "Workspace checkout has local changes"); diff --git a/backend/src/workspaces/registry.ts b/backend/src/workspaces/registry.ts index 1e64aca1..4ab35bb7 100644 --- a/backend/src/workspaces/registry.ts +++ b/backend/src/workspaces/registry.ts @@ -33,6 +33,18 @@ export type PublishWorkspaceRequest = | { action: "update"; workspace: CanonicalWorkspace; baseCommit: string; baseBlob: string } | { action: "delete"; id: string; baseCommit: string; baseBlob: string }; +export class WorkspaceConflictError extends WorkspaceRegistryError { + constructor( + readonly fields: string[], + readonly base?: CanonicalWorkspace, + readonly local?: CanonicalWorkspace, + readonly remote?: CanonicalWorkspace, + ) { + super("workspace_conflict", "Workspace revision conflicts with the active registry"); + this.name = "WorkspaceConflictError"; + } +} + interface ActiveState { head: string; revisions: WorkspaceRevision[]; @@ -139,9 +151,100 @@ export class WorkspaceRegistry { } } - /** Publication is deliberately deferred until Task 6 adds validated route-level concurrency controls. */ - async publish(_request: PublishWorkspaceRequest): Promise { - throw new WorkspaceRegistryError("workspace_stale", "Workspace publication is unavailable"); + /** + * Publish canonical YAML and derived public documentation as one optimistic Git revision. + * The browser never provides paths or generated artifacts; those are derived server-side. + */ + async publish(request: PublishWorkspaceRequest): Promise { + await this.repository.ensureLayout(); + return await this.lock.run(async () => { + const status = await this.repository.pull(); + await this.activate(status.head!); + const current = await this.activeState(); + const id = request.action === "delete" ? request.id : request.workspace.workspace.id; + const existing = current.revisions.find((revision) => revision.id === id); + const local = request.action === "delete" ? undefined : request.workspace; + + if (request.baseCommit !== status.head || ( + request.action !== "create" && existing?.blob !== request.baseBlob + )) { + throw await this.conflictFor(request, existing, local); + } + if (request.action === "create" && existing) throw await this.conflictFor(request, existing, local); + if (request.action !== "create" && !existing) throw await this.conflictFor(request, existing, local); + + const yamlPath = workspacePath(id); + const docPaths = this.documentationPaths(id); + if (request.action === "delete") { + await this.repository.removeRegistryFile(yamlPath); + await this.repository.removeRegistryFile(docPaths.contract); + await this.repository.removeRegistryFile(docPaths.readme); + } else { + const canonical = request.workspace; + const source = serializeWorkspaceYaml(canonical); + const docs = renderWorkspaceDocs(canonical); + await this.repository.writeRegistryFile(yamlPath, source); + await this.repository.writeRegistryFile(docPaths.contract, docs.envExample); + await this.repository.writeRegistryFile(docPaths.readme, docs.markdown); + } + + const next = await this.repository.commitAndPush( + [yamlPath, docPaths.contract, docPaths.readme], + request.action === "delete" ? `Delete workspace ${id}` : `Publish workspace ${id}`, + ); + await this.activate(next.head!); + return (await this.activeState()).revisions.find((revision) => revision.id === id); + }); + } + + private documentationPaths(id: string): { contract: string; readme: string } { + workspacePath(id); + const directory = `workspace-docs/${id}`; + return { contract: `${directory}/contract.env.example`, readme: `${directory}/README.md` }; + } + + private async conflictFor( + request: PublishWorkspaceRequest, + existing: WorkspaceRevision | undefined, + local: CanonicalWorkspace | undefined, + ): Promise { + const id = request.action === "delete" ? request.id : request.workspace.workspace.id; + const base = await this.readSnapshotCanonical(request.baseCommit, id); + let remote: CanonicalWorkspace | undefined; + if (existing) { + const read = await this.read(id); + remote = isCanonicalWorkspace(read.workspace) ? read.workspace : undefined; + } + return new WorkspaceConflictError(this.changedFields(base, remote), base, local, remote); + } + + private async readSnapshotCanonical(commit: string, id: string): Promise { + try { + const source = await readFile(this.snapshotPath(commit, id), "utf8"); + const workspace = parseWorkspaceYaml(source); + return isCanonicalWorkspace(workspace) ? workspace : undefined; + } catch { + return undefined; + } + } + + private changedFields( + base: unknown, + remote: unknown, + prefix = "", + ): string[] { + if (!base || !remote) return ["workspace.id"]; + if (Array.isArray(base) || Array.isArray(remote) || typeof base !== "object" || typeof remote !== "object") { + return JSON.stringify(base) === JSON.stringify(remote) ? [] : [prefix]; + } + const baseObject = base as Record; + const remoteObject = remote as Record; + const keys = new Set([...Object.keys(baseObject), ...Object.keys(remoteObject)]); + return [...keys].flatMap((key) => this.changedFields( + baseObject[key], + remoteObject[key], + prefix ? `${prefix}.${key}` : key, + )); } private async activate(commit: string): Promise { diff --git a/backend/test/routes-sql-meta.test.ts b/backend/test/routes-sql-meta.test.ts index 8d56f65e..f81370f1 100644 --- a/backend/test/routes-sql-meta.test.ts +++ b/backend/test/routes-sql-meta.test.ts @@ -85,40 +85,8 @@ test("POST /sessions/:id/sql/preview returns 500 when thtRunner throws", async ( expect(res.json()).toMatchObject({ error: /boom/ }); }); -// --------------------------------------------------------------------------- -// Meta routes -// --------------------------------------------------------------------------- - -test("GET /workspaces lists yaml files from ../harness/workspaces", async () => { - // The real ../harness/workspaces directory contains *.yaml files. - const app = buildApp(loadConfig({ THT_HARNESS_DIR: "../harness" }), { - thtRunner: {} as any, - }); - - const res = await app.inject({ method: "GET", url: "/workspaces" }); - - expect(res.statusCode).toBe(200); - const body = res.json() as { name: string; file: string }[]; - expect(Array.isArray(body)).toBe(true); - // ../harness/workspaces has at least one yaml (tht-test.yaml / tht.example.yaml) - expect(body.length).toBeGreaterThan(0); - for (const w of body) { - expect(typeof w.name).toBe("string"); - expect(w.name).not.toContain(".yaml"); // name strips extension - expect(w.file).toMatch(/\.ya?ml$/); - } -}); - -test("GET /workspaces returns [] when harnessDir has no workspaces subdir", async () => { - const app = buildApp(loadConfig({ THT_HARNESS_DIR: "/nonexistent-harness-dir" }), { - thtRunner: {} as any, - }); - - const res = await app.inject({ method: "GET", url: "/workspaces" }); - - expect(res.statusCode).toBe(200); - expect(res.json()).toEqual([]); -}); +// Workspace registry route coverage lives in routes-workspaces.test.ts. `/workspaces` no longer +// reads legacy harness files: the Git registry is the single shared source of truth. test("GET /models returns {models:[...]} from injected listModels stub", async () => { const app = buildApp(loadConfig({ THT_HARNESS_DIR: "../harness" }), { diff --git a/backend/test/routes-workspaces.test.ts b/backend/test/routes-workspaces.test.ts new file mode 100644 index 00000000..efd5e3a7 --- /dev/null +++ b/backend/test/routes-workspaces.test.ts @@ -0,0 +1,258 @@ +import { createHash } from "node:crypto"; +import { once } from "node:events"; +import { Buffer } from "node:buffer"; +import { expect, test, vi } from "vitest"; +import yazl from "yazl"; +import { buildApp } from "../src/app.js"; +import { loadConfig } from "../src/config.js"; +import { WorkspaceRegistryError } from "../src/workspaces/git-repository.js"; +import type { WorkspaceRegistry, WorkspaceRevision } from "../src/workspaces/registry.js"; +import { renderWorkspaceDocs, serializeWorkspaceYaml, type CanonicalWorkspace } from "../src/workspaces/schema.js"; + +const workspace: CanonicalWorkspace = { + workspace: { + schema_version: 2, + id: "psd-clinical", + name: "Policlinico San Donato", + description: "Clinical analytics workspace", + language: "it", + }, + dwh: { + engine: "postgres", + database: "warehouse", + schema: "datawarehouse", + supported_transports: ["postgres_direct"], + }, + semantic_index: { + vector_store: { + engine: "pgvector", + database: "warehouse", + schema: "vectors", + collection: "clinical_documents", + dimensions: 768, + distance: "cosine", + supported_transports: ["pgvector_direct"], + }, + embedding: { + provider: "ollama_compatible", + model: "nomic-embed-text-v2-moe", + dimensions: 768, + }, + }, + llm_policy: { allowed: ["zai/glm-5.2"] }, +}; + +const revision: WorkspaceRevision = { + id: workspace.workspace.id, + commit: "a".repeat(40), + blob: "b".repeat(40), + snapshotPath: "/registry/snapshots/psd-clinical.yaml", + state: "operational", +}; + +type RegistryFake = Pick; + +function registryFake(overrides: Partial = {}): RegistryFake { + return { + bootstrap: vi.fn(async () => ({ + branch: "main", head: revision.commit, ahead: 0, behind: 0, degraded: false, + })), + pull: vi.fn(async () => ({ + branch: "main", head: revision.commit, ahead: 0, behind: 0, degraded: false, + })), + list: vi.fn(async () => [revision]), + read: vi.fn(async () => ({ workspace, revision })), + publish: vi.fn(async () => revision), + ...overrides, + }; +} + +function appFor(registry: RegistryFake, diagnose = vi.fn(async () => ({ activatable: true, diagnostics: [] }))) { + return buildApp(loadConfig({ + THT_HARNESS_DIR: "/missing-harness", + THT_WORKSPACE_REGISTRY_ROOT: "/tmp/thoth-route-test-registry", + }), { + thtRunner: {} as any, + workspaceRegistry: registry as WorkspaceRegistry, + workspaceDiagnoser: diagnose, + } as any); +} + +function sha256(value: string): string { + return createHash("sha256").update(value).digest("hex"); +} + +async function zip(files: Record): Promise { + const archive = new yazl.ZipFile(); + const chunks: Buffer[] = []; + archive.outputStream.on("data", (chunk: Buffer) => chunks.push(chunk)); + for (const [name, contents] of Object.entries(files)) archive.addBuffer(Buffer.from(contents), name); + archive.end(); + await once(archive.outputStream, "end"); + return Buffer.concat(chunks); +} + +async function validBundle(): Promise { + const workspaceYaml = serializeWorkspaceYaml(workspace); + const docs = renderWorkspaceDocs(workspace); + const contractEnv = docs.envExample; + const readme = docs.markdown; + return await zip({ + "manifest.json": JSON.stringify({ + schema_version: 1, + workspace_id: workspace.workspace.id, + files: { + "workspace.yaml": sha256(workspaceYaml), + "contract.env.example": sha256(contractEnv), + "README.md": sha256(readme), + }, + }), + "workspace.yaml": workspaceYaml, + "contract.env.example": contractEnv, + "README.md": readme, + }); +} + +function zipWithZipSlipEntry(): Promise { + return zip({ "aa/escape.yaml": "bad" }).then((archive) => { + const safeName = Buffer.from("aa/escape.yaml"); + const unsafeName = Buffer.from("../escape.yaml"); + for (let offset = archive.indexOf(safeName); offset !== -1; offset = archive.indexOf(safeName, offset + safeName.length)) { + unsafeName.copy(archive, offset); + } + return archive; + }); +} + +async function importBundle(app: ReturnType, archive: Buffer) { + const boundary = "----thoth-workspace-test-boundary"; + const payload = Buffer.concat([ + Buffer.from(`--${boundary}\r\ncontent-disposition: form-data; name="bundle"; filename="workspace.zip"\r\ncontent-type: application/zip\r\n\r\n`), + archive, + Buffer.from(`\r\n--${boundary}--\r\n`), + ]); + return await app.inject({ + method: "POST", + url: "/workspaces/import", + headers: { "content-type": `multipart/form-data; boundary=${boundary}` }, + payload, + }); +} + +test("returns a redacted registry status and pulls without Git credential details", async () => { + const registry = registryFake({ + bootstrap: vi.fn(async () => ({ + branch: "main", head: revision.commit, ahead: 0, behind: 0, degraded: true, lastError: "git_auth_failed" as const, + })), + }); + const app = appFor(registry); + + const status = await app.inject({ method: "GET", url: "/workspace-registry/status" }); + const pull = await app.inject({ method: "POST", url: "/workspace-registry/pull" }); + + expect(status.statusCode).toBe(200); + expect(status.json()).toEqual({ + branch: "main", head: revision.commit, ahead: 0, behind: 0, degraded: true, lastError: "git_auth_failed", + }); + expect(pull.statusCode).toBe(200); + expect(JSON.stringify([status.json(), pull.json()])).not.toMatch(/token|password|ssh:\/\//i); +}); + +test("lists compatible workspace summaries and reads a validated workspace", async () => { + const app = appFor(registryFake()); + + const list = await app.inject({ method: "GET", url: "/workspaces" }); + const detail = await app.inject({ method: "GET", url: "/workspaces/psd-clinical" }); + + expect(list.statusCode).toBe(200); + expect(list.json()).toEqual([expect.objectContaining({ + id: "psd-clinical", name: "psd-clinical", file: "psd-clinical.yaml", displayName: "Policlinico San Donato", + })]); + expect(detail.statusCode).toBe(200); + expect(detail.json()).toMatchObject({ workspace, revision }); +}); + +test("validates a canonical workspace and runs the injected installation diagnostic", async () => { + const diagnose = vi.fn(async () => ({ + activatable: false, + diagnostics: [{ level: "error" as const, code: "binding_missing" as const, field: "THT_WS_PSD_CLINICAL_DWH_PASSWORD_FILE", message: "Installation binding is missing or invalid." }], + })); + const app = appFor(registryFake(), diagnose); + + const validate = await app.inject({ method: "POST", url: "/workspaces/validate", payload: { workspace } }); + const testResult = await app.inject({ method: "POST", url: "/workspaces/psd-clinical/test", payload: {} }); + + expect(validate.statusCode).toBe(200); + expect(validate.json()).toMatchObject({ workspace }); + expect(testResult.statusCode).toBe(200); + expect(testResult.json()).toMatchObject({ activatable: false, diagnostics: [{ code: "binding_missing" }] }); + expect(diagnose).toHaveBeenCalledWith(workspace, expect.any(Object), { writeProbe: false }); +}); + +test("returns a 409 field conflict instead of overwriting a changed workspace", async () => { + const conflict = Object.assign( + new WorkspaceRegistryError("workspace_conflict", "Workspace has changed"), + { + fields: ["semantic_index.embedding.model"], + base: workspace, + local: { ...workspace, semantic_index: { ...workspace.semantic_index, embedding: { ...workspace.semantic_index.embedding, model: "local/model" } } }, + remote: { ...workspace, semantic_index: { ...workspace.semantic_index, embedding: { ...workspace.semantic_index.embedding, model: "remote/model" } } }, + }, + ); + const registry = registryFake({ publish: vi.fn(async () => { throw conflict; }) }); + const app = appFor(registry); + const staleUpdate = { + action: "update", + workspace, + baseCommit: "c".repeat(40), + baseBlob: "d".repeat(40), + }; + + const res = await app.inject({ method: "POST", url: "/workspaces/publish", payload: staleUpdate }); + + expect(res.statusCode).toBe(409); + expect(res.json()).toMatchObject({ + code: "workspace_conflict", + fields: ["semantic_index.embedding.model"], + base: workspace, + remote: expect.objectContaining({ + semantic_index: expect.objectContaining({ + embedding: expect.objectContaining({ model: "remote/model" }), + }), + }), + }); +}); + +test("exports generated public artifacts without secret values", async () => { + const app = appFor(registryFake()); + + const res = await app.inject({ method: "GET", url: "/workspaces/psd-clinical/export" }); + + expect(res.statusCode).toBe(200); + expect(res.headers["content-disposition"]).toMatch(/attachment; filename="psd-clinical\.zip"/); + expect(res.headers["content-type"]).toMatch(/application\/zip/); + expect(res.rawPayload.toString("utf8")).toContain("contract.env.example"); + expect(res.rawPayload.toString("utf8")).not.toContain("secret-value"); +}); + +test("rejects a zip-slip import without publishing or writing a checkout file", async () => { + const registry = registryFake(); + const app = appFor(registry); + + const res = await importBundle(app, await zipWithZipSlipEntry()); + + expect(res.statusCode).toBe(400); + expect(res.json()).toMatchObject({ code: "workspace_invalid" }); + expect(registry.publish).not.toHaveBeenCalled(); +}); + +test("imports an exact generated bundle only as a browser draft", async () => { + const registry = registryFake(); + const app = appFor(registry); + + const res = await importBundle(app, await validBundle()); + + expect(res.statusCode).toBe(200); + expect(res.json()).toMatchObject({ draft: { workspace } }); + expect(registry.publish).not.toHaveBeenCalled(); +});