diff --git a/backend/src/workspaces/annotations.ts b/backend/src/workspaces/annotations.ts new file mode 100644 index 00000000..60d2376d --- /dev/null +++ b/backend/src/workspaces/annotations.ts @@ -0,0 +1,28 @@ +import { parseAllDocuments } from "yaml"; +import { WorkspaceRegistryError } from "./git-repository.js"; + +/** + * Coarse structural validation for a curated annotation blob at activation time. The harness + * Pydantic parser remains the authority for per-table shapes; this check rejects only YAML that + * cannot possibly be a canonical `Annotations` document (single mapping, optional `tables` mapping). + */ +export function parseAnnotationsYaml(source: string): void { + try { + const documents = parseAllDocuments(source, { uniqueKeys: true }); + if (documents.length !== 1) throw new Error("malformed annotations"); + const document = documents[0]; + if (document.errors.length > 0 || document.warnings.length > 0) throw new Error("malformed annotations"); + const parsed = document.toJSON(); + if (parsed === null || parsed === undefined) return; // empty canonical set + if (typeof parsed !== "object" || Array.isArray(parsed)) throw new Error("malformed annotations"); + const record = parsed as Record; + if ( + record.tables !== undefined + && (typeof record.tables !== "object" || record.tables === null || Array.isArray(record.tables)) + ) { + throw new Error("malformed annotations"); + } + } catch { + throw new WorkspaceRegistryError("workspace_invalid", "Workspace annotations are malformed"); + } +} diff --git a/backend/src/workspaces/git-repository.ts b/backend/src/workspaces/git-repository.ts index f2ef6abb..bb85a43f 100644 --- a/backend/src/workspaces/git-repository.ts +++ b/backend/src/workspaces/git-repository.ts @@ -2,7 +2,7 @@ import { execFile, spawn, type ChildProcessWithoutNullStreams } from "node:child import { lstatSync, mkdirSync } from "node:fs"; import { mkdir, rm, writeFile } from "node:fs/promises"; import { basename, dirname, isAbsolute, join } from "node:path"; -import { promisify } from "node:util"; +import { promisify, TextDecoder } from "node:util"; import type { WorkspaceErrorCode, WorkspaceRegistryConfig } from "./types.js"; const execFileAsync = promisify(execFile); @@ -39,6 +39,15 @@ function assertDirectory(path: string): void { } } +function isValidUtf8(buffer: Buffer): boolean { + try { + new TextDecoder("utf-8", { fatal: true }).decode(buffer); + return true; + } catch { + return false; + } +} + function gitErrorCode(error: unknown): WorkspaceErrorCode { const detail = [ error instanceof Error ? error.message : "", @@ -233,6 +242,66 @@ export class GitWorkspaceRepository { } } + /** Read the curated FK annotations object at an exact commit, or undefined when absent. */ + async annotationsObject(revision: string, id: string): Promise<{ blobId: string; contents: Buffer } | undefined> { + if (!/^[0-9a-f]{40}$/.test(revision)) { + throw new WorkspaceRegistryError("workspace_invalid", "Workspace annotations revision is invalid"); + } + if (!/^[a-z][a-z0-9-]{2,62}$/.test(id) || id === "workspace-docs") { + throw new WorkspaceRegistryError("workspace_invalid", "Workspace annotations path is invalid"); + } + const path = `${id}/schema/annotations.yaml`; + // ls-tree -z reports the exact object at the path (or its children when the path is a tree). + const listing = await this.git(["ls-tree", "-z", "--full-tree", revision, "--", path]); + const entries = listing.split("\0").filter((entry) => entry.length > 0); + if (entries.length === 0) return undefined; + const exact = entries.find((entry) => entry.slice(entry.lastIndexOf("\t") + 1) === path); + if (exact === undefined) { + // The path resolves to a tree (its children are listed) or another non-blob object. + throw new WorkspaceRegistryError("workspace_invalid", "Workspace annotations object is invalid"); + } + const match = /^([0-9]{6})\s+(blob|tree|commit)\s+([0-9a-f]{40})\t/.exec(exact); + // Only regular Git blobs are accepted: symlinks (120000) and gitlinks (160000) are refused. + if (match === null || match[2] !== "blob" || (match[1] !== "100644" && match[1] !== "100755")) { + throw new WorkspaceRegistryError("workspace_invalid", "Workspace annotations object is invalid"); + } + const blobId = match[3]; + const contents = await this.gitBlobBuffer(blobId, 16 * 1024 * 1024); + if (!isValidUtf8(contents)) { + throw new WorkspaceRegistryError("workspace_invalid", "Workspace annotations object is not valid UTF-8"); + } + return { blobId, contents }; + } + + private async gitBlobBuffer(objectId: string, maxBytes: number): Promise { + if (!/^[0-9a-f]{40}$/.test(objectId)) { + throw new WorkspaceRegistryError("workspace_invalid", "Workspace annotations object is invalid"); + } + try { + const { stdout } = await execFileAsync( + "git", + ["-c", `core.hooksPath=${this.hooksPath}`, "cat-file", "blob", objectId], + { + cwd: this.repoPath, + env: { ...process.env, GIT_TERMINAL_PROMPT: "0" }, + encoding: "buffer", + maxBuffer: maxBytes + 1024 * 1024, + }, + ); + if (stdout.length > maxBytes) { + throw new WorkspaceRegistryError("workspace_invalid", "Workspace annotations object is too large"); + } + return stdout; + } catch (error) { + if (error instanceof WorkspaceRegistryError) throw error; + const detail = error instanceof Error ? error.message : ""; + if (/maxBuffer|stdout maxBuffer/i.test(detail)) { + throw new WorkspaceRegistryError("workspace_invalid", "Workspace annotations object is too large"); + } + throw this.sanitizeGitError(error); + } + } + /** Write only a validated API-owned artifact below the checked-out repository. */ async writeRegistryFile(path: string, source: string): Promise { this.assertRegistryArtifactPath(path); diff --git a/backend/src/workspaces/registry.ts b/backend/src/workspaces/registry.ts index 49950419..2f05e04a 100644 --- a/backend/src/workspaces/registry.ts +++ b/backend/src/workspaces/registry.ts @@ -3,6 +3,7 @@ import { lstatSync } from "node:fs"; import { mkdir, readdir, readFile, rename, rm, writeFile } from "node:fs/promises"; import { isAbsolute, join } from "node:path"; import { buildInstallationContract, renderWorkspaceDocs } from "./contracts.js"; +import { parseAnnotationsYaml } from "./annotations.js"; import { assertCatalogMatchesDescriptor, parseWorkspaceCatalogYaml, type WorkspaceCatalog, type WorkspaceCatalogEntry } from "./catalog.js"; import { GitWorkspaceRepository, @@ -581,6 +582,10 @@ export class WorkspaceRegistry { } assertCatalogMatchesDescriptor(entry, workspace); await this.assertEvidenceContext(workspace, safeHead); + const annotations = await this.repository.annotationsObject(safeHead, id); + if (annotations !== undefined) { + parseAnnotationsYaml(annotations.contents.toString("utf8")); + } const collection = workspace.semantic_index.vector_store.collection; const owner = collectionOwners.get(collection); if (owner !== undefined) { diff --git a/backend/test/registry-annotations.test.ts b/backend/test/registry-annotations.test.ts new file mode 100644 index 00000000..1f612b64 --- /dev/null +++ b/backend/test/registry-annotations.test.ts @@ -0,0 +1,116 @@ +import { execFile } from "node:child_process"; +import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { promisify } from "node:util"; +import { afterEach, expect, test } from "vitest"; +import { WorkspaceRegistry } from "../src/workspaces/registry.js"; +import type { WorkspaceRegistryConfig } from "../src/workspaces/types.js"; + +const runFile = promisify(execFile); +const temporaryRoots: string[] = []; + +afterEach(() => { + temporaryRoots.splice(0).forEach((root) => rmSync(root, { recursive: true, force: true })); +}); + +const validYaml = `workspace: + schema_version: 3 + id: psd-clinical + name: Policlinico San Donato + language: it +dwh: + engine: postgres + database: postgres + schema: datawarehouse + supported_transports: [postgres_direct] +semantic_index: + vector_store: + engine: qdrant + collection: psd-clinical + dimensions: 1024 + distance: cosine + embedding: + provider: ollama_internal + model: qwen3-embedding:0.6b + dimensions: 1024 +llm_policy: + allowed: [zai/glm-5.2] +`; + +async function git(cwd: string, args: string[]): Promise { + const { stdout } = await runFile("git", args, { cwd }); + return stdout.trim(); +} + +function config(root: string, remoteUrl: string): WorkspaceRegistryConfig { + return { + root, + remoteUrl, + branch: "main", + gitAuthorName: "Registry Annotations Test", + gitAuthorEmail: "registry-annotations@example.invalid", + installationId: "test", + secretRoots: [], + maxImportBytes: 1024, + maxImportEntries: 1, + }; +} + +type AnnotationsLayout = "absent" | "valid" | "malformed" | "dir"; + +async function fixture(layout: AnnotationsLayout): Promise<{ root: string; remote: string }> { + const root = mkdtempSync(join(tmpdir(), "thoth-registry-annotations-")); + temporaryRoots.push(root); + const remote = join(root, "remote.git"); + const source = join(root, "source"); + await git(root, ["init", "--bare", "--initial-branch=main", remote]); + mkdirSync(source); + await git(source, ["init", "--initial-branch=main"]); + await git(source, ["config", "user.name", "Registry Annotations Test"]); + await git(source, ["config", "user.email", "registry-annotations@example.invalid"]); + writeFileSync(join(source, "thoth-workspaces.yaml"), + "schema_version: 1\nworkspaces:\n - id: psd-clinical\n name: Policlinico San Donato\n"); + mkdirSync(join(source, "psd-clinical", "schema"), { recursive: true }); + writeFileSync(join(source, "psd-clinical", "workspace.yaml"), validYaml); + const annotationsPath = join(source, "psd-clinical", "schema", "annotations.yaml"); + if (layout === "valid") writeFileSync(annotationsPath, "tables: {}\n"); + if (layout === "malformed") writeFileSync(annotationsPath, "tables: [not, a, mapping]\n"); + if (layout === "dir") { + mkdirSync(annotationsPath, { recursive: true }); + writeFileSync(join(annotationsPath, "child.txt"), "nested\n"); + } + await git(source, ["add", "-A"]); + await git(source, ["commit", "-m", "initial"]); + await git(source, ["remote", "add", "origin", remote]); + await git(source, ["push", "origin", "main"]); + return { root, remote }; +} + +test("activation accepts a valid curated annotation blob", async () => { + const fixtureValue = await fixture("valid"); + const registry = new WorkspaceRegistry(config(join(fixtureValue.root, "registry"), fixtureValue.remote)); + + await expect(registry.bootstrap()).resolves.toMatchObject({ degraded: false }); +}); + +test("activation accepts an absent annotation blob", async () => { + const fixtureValue = await fixture("absent"); + const registry = new WorkspaceRegistry(config(join(fixtureValue.root, "registry"), fixtureValue.remote)); + + await expect(registry.bootstrap()).resolves.toMatchObject({ degraded: false }); +}); + +test("activation rejects malformed annotations", async () => { + const fixtureValue = await fixture("malformed"); + const registry = new WorkspaceRegistry(config(join(fixtureValue.root, "registry"), fixtureValue.remote)); + + await expect(registry.bootstrap()).rejects.toMatchObject({ code: "workspace_invalid" }); +}); + +test("activation rejects a tree at the annotations path", async () => { + const fixtureValue = await fixture("dir"); + const registry = new WorkspaceRegistry(config(join(fixtureValue.root, "registry"), fixtureValue.remote)); + + await expect(registry.bootstrap()).rejects.toMatchObject({ code: "workspace_invalid" }); +}); diff --git a/backend/test/workspaces-git-annotations.test.ts b/backend/test/workspaces-git-annotations.test.ts new file mode 100644 index 00000000..82957e4e --- /dev/null +++ b/backend/test/workspaces-git-annotations.test.ts @@ -0,0 +1,150 @@ +import { execFile } from "node:child_process"; +import { mkdtempSync, mkdirSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { promisify } from "node:util"; +import { afterEach, expect, test } from "vitest"; +import { GitWorkspaceRepository } from "../src/workspaces/git-repository.js"; +import type { WorkspaceRegistryConfig } from "../src/workspaces/types.js"; + +const runFile = promisify(execFile); +const temporaryRoots: string[] = []; + +afterEach(() => { + temporaryRoots.splice(0).forEach((root) => rmSync(root, { recursive: true, force: true })); +}); + +async function git(cwd: string, args: string[]): Promise { + const { stdout } = await runFile("git", args, { cwd }); + return stdout.trim(); +} + +function config(root: string, remoteUrl: string): WorkspaceRegistryConfig { + return { + root, + remoteUrl, + branch: "main", + gitAuthorName: "Workspace Registry Test", + gitAuthorEmail: "workspace-registry@example.invalid", + installationId: "test", + secretRoots: [], + maxImportBytes: 1024, + maxImportEntries: 1, + }; +} + +interface RepoFixture { + root: string; + remote: string; + source: string; + commit: string; +} + +async function makeRepo(id: string, annotations: string | Buffer | "dir" | "symlink"): Promise { + const root = mkdtempSync(join(tmpdir(), "thoth-annotations-git-")); + temporaryRoots.push(root); + const remote = join(root, "remote.git"); + const source = join(root, "source"); + await git(root, ["init", "--bare", "--initial-branch=main", remote]); + mkdirSync(source); + await git(source, ["init", "--initial-branch=main"]); + await git(source, ["config", "user.name", "Annotations Test"]); + await git(source, ["config", "user.email", "annotations@example.invalid"]); + writeFileSync(join(source, "thoth-workspaces.yaml"), + `schema_version: 1\nworkspaces: [{id: ${id}, name: Workspace}]\n`); + mkdirSync(join(source, id, "schema"), { recursive: true }); + writeFileSync(join(source, id, "workspace.yaml"), `workspace:\n schema_version: 3\n id: ${id}\n`); + const annotationsPath = join(source, id, "schema", "annotations.yaml"); + if (annotations === "dir") { + mkdirSync(annotationsPath, { recursive: true }); + writeFileSync(join(annotationsPath, "child.txt"), "not a blob\n"); + } else if (annotations === "symlink") { + writeFileSync(join(source, id, "target.yaml"), "tables: {}\n"); + symlinkSync("target.yaml", annotationsPath); + } else { + writeFileSync(annotationsPath, annotations); + } + await git(source, ["add", "-A"]); + await git(source, ["commit", "-m", "initial"]); + await git(source, ["remote", "add", "origin", remote]); + await git(source, ["push", "origin", "main"]); + const commit = await git(source, ["rev-parse", "HEAD"]); + return { root, remote, source, commit }; +} + +async function bootstrapped(fixture: RepoFixture): Promise { + const repository = new GitWorkspaceRepository(config(join(fixture.root, "registry"), fixture.remote)); + await repository.bootstrap(); + return repository; +} + +test("reads a regular annotation blob at the exact commit", async () => { + const fixture = await makeRepo("research", "tables: {}\n"); + const repository = await bootstrapped(fixture); + + const object = await repository.annotationsObject(fixture.commit, "research"); + + expect(object).toBeDefined(); + expect(object!.blobId).toMatch(/^[0-9a-f]{40}$/); + expect(object!.contents.toString("utf8")).toBe("tables: {}\n"); +}); + +test("returns undefined when the annotation object is absent", async () => { + const fixture = await makeRepo("research", "tables: {}\n"); + const repository = await bootstrapped(fixture); + + await expect(repository.annotationsObject(fixture.commit, "absent")).resolves.toBeUndefined(); +}); + +test("refuses a tree at the annotation path", async () => { + const fixture = await makeRepo("clinical", "dir"); + const repository = await bootstrapped(fixture); + + await expect(repository.annotationsObject(fixture.commit, "clinical")) + .rejects.toMatchObject({ code: "workspace_invalid" }); +}); + +test("refuses a symlink at the annotation path", async () => { + const fixture = await makeRepo("research", "symlink"); + const repository = await bootstrapped(fixture); + + await expect(repository.annotationsObject(fixture.commit, "research")) + .rejects.toMatchObject({ code: "workspace_invalid" }); +}); + +test("refuses oversized and non-UTF-8 annotation blobs", async () => { + const oversized = await makeRepo("research", Buffer.concat([ + Buffer.from("tables: {}\n"), + Buffer.alloc(16 * 1024 * 1024, 0x78), + ])); + const repository = await bootstrapped(oversized); + await expect(repository.annotationsObject(oversized.commit, "research")) + .rejects.toMatchObject({ code: "workspace_invalid" }); + + const nonUtf8 = await makeRepo("research", Buffer.from([0x74, 0x61, 0x62, 0xff, 0xfe, 0x00])); + const repository2 = await bootstrapped(nonUtf8); + await expect(repository2.annotationsObject(nonUtf8.commit, "research")) + .rejects.toMatchObject({ code: "workspace_invalid" }); +}); + +test("refuses malformed ids and revisions before Git", async () => { + const fixture = await makeRepo("research", "tables: {}\n"); + const repository = await bootstrapped(fixture); + + await expect(repository.annotationsObject(fixture.commit, "workspace-docs")) + .rejects.toMatchObject({ code: "workspace_invalid" }); + await expect(repository.annotationsObject(fixture.commit, "../research")) + .rejects.toMatchObject({ code: "workspace_invalid" }); + await expect(repository.annotationsObject("HEAD", "research")) + .rejects.toMatchObject({ code: "workspace_invalid" }); +}); + +test("redacts Git failures while reading the annotation blob", async () => { + const fixture = await makeRepo("research", "tables: {}\n"); + const repository = await bootstrapped(fixture); + rmSync(repository.repoPath, { recursive: true, force: true }); + + const error = await repository.annotationsObject(fixture.commit, "research").catch((failure: unknown) => failure); + expect(error).toMatchObject({ code: "git_unavailable", message: "Workspace Git operation failed" }); + expect((error as Error).message).not.toContain(fixture.root); +});