mptyl/ThothII
1
0
Fork 0
Code Issues 25 Pull Requests Actions Packages Projects Releases 1 Wiki Activity

feat: read and validate curated FK annotations at the pinned commit (P5)

Browse Source
This commit is contained in:
marcopan
2026-08-13 04:56:03 +02:00
parent 60e4048d4b
commit b00b7f17c9
5 changed files with 369 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/src/workspaces/annotations.ts
+28
View File
@@ -0,0 +1,28 @@
import { parseAllDocuments } from "yaml";
import { WorkspaceRegistryError } from "./git-repository.js";
/**
* Coarse structural validation for a curated annotation blob at activation time. The harness
* Pydantic parser remains the authority for per-table shapes; this check rejects only YAML that
* cannot possibly be a canonical `Annotations` document (single mapping, optional `tables` mapping).
*/
export function parseAnnotationsYaml(source: string): void {
try {
const documents = parseAllDocuments(source, { uniqueKeys: true });
if (documents.length !== 1) throw new Error("malformed annotations");
const document = documents[0];
if (document.errors.length > 0 || document.warnings.length > 0) throw new Error("malformed annotations");
const parsed = document.toJSON();
if (parsed === null || parsed === undefined) return; // empty canonical set
if (typeof parsed !== "object" || Array.isArray(parsed)) throw new Error("malformed annotations");
const record = parsed as Record<string, unknown>;
if (
record.tables !== undefined
&& (typeof record.tables !== "object" || record.tables === null || Array.isArray(record.tables))
) {
throw new Error("malformed annotations");
}
} catch {
throw new WorkspaceRegistryError("workspace_invalid", "Workspace annotations are malformed");
}
}
backend/src/workspaces/git-repository.ts
+70 -1
View File
@@ -2,7 +2,7 @@ import { execFile, spawn, type ChildProcessWithoutNullStreams } from "node:child
import { lstatSync, mkdirSync } from "node:fs";
import { mkdir, rm, writeFile } from "node:fs/promises";
import { basename, dirname, isAbsolute, join } from "node:path";
import { promisify } from "node:util";
import { promisify, TextDecoder } from "node:util";
import type { WorkspaceErrorCode, WorkspaceRegistryConfig } from "./types.js";
const execFileAsync = promisify(execFile);
@@ -39,6 +39,15 @@ function assertDirectory(path: string): void {
}
}
function isValidUtf8(buffer: Buffer): boolean {
try {
new TextDecoder("utf-8", { fatal: true }).decode(buffer);
return true;
} catch {
return false;
}
}
function gitErrorCode(error: unknown): WorkspaceErrorCode {
const detail = [
error instanceof Error ? error.message : "",
@@ -233,6 +242,66 @@ export class GitWorkspaceRepository {
}
}
/** Read the curated FK annotations object at an exact commit, or undefined when absent. */
async annotationsObject(revision: string, id: string): Promise<{ blobId: string; contents: Buffer } | undefined> {
if (!/^[0-9a-f]{40}$/.test(revision)) {
throw new WorkspaceRegistryError("workspace_invalid", "Workspace annotations revision is invalid");
}
if (!/^[a-z][a-z0-9-]{2,62}$/.test(id) || id === "workspace-docs") {
throw new WorkspaceRegistryError("workspace_invalid", "Workspace annotations path is invalid");
}
const path = `${id}/schema/annotations.yaml`;
// ls-tree -z reports the exact object at the path (or its children when the path is a tree).
const listing = await this.git(["ls-tree", "-z", "--full-tree", revision, "--", path]);
const entries = listing.split("\0").filter((entry) => entry.length > 0);
if (entries.length === 0) return undefined;
const exact = entries.find((entry) => entry.slice(entry.lastIndexOf("\t") + 1) === path);
if (exact === undefined) {
// The path resolves to a tree (its children are listed) or another non-blob object.
throw new WorkspaceRegistryError("workspace_invalid", "Workspace annotations object is invalid");
}
const match = /^([0-9]{6})\s+(blob|tree|commit)\s+([0-9a-f]{40})\t/.exec(exact);
// Only regular Git blobs are accepted: symlinks (120000) and gitlinks (160000) are refused.
if (match === null || match[2] !== "blob" || (match[1] !== "100644" && match[1] !== "100755")) {
throw new WorkspaceRegistryError("workspace_invalid", "Workspace annotations object is invalid");
}
const blobId = match[3];
const contents = await this.gitBlobBuffer(blobId, 16 * 1024 * 1024);
if (!isValidUtf8(contents)) {
throw new WorkspaceRegistryError("workspace_invalid", "Workspace annotations object is not valid UTF-8");
}
return { blobId, contents };
}
private async gitBlobBuffer(objectId: string, maxBytes: number): Promise<Buffer> {
if (!/^[0-9a-f]{40}$/.test(objectId)) {
throw new WorkspaceRegistryError("workspace_invalid", "Workspace annotations object is invalid");
}
try {
const { stdout } = await execFileAsync(
"git",
["-c", `core.hooksPath=${this.hooksPath}`, "cat-file", "blob", objectId],
{
cwd: this.repoPath,
env: { ...process.env, GIT_TERMINAL_PROMPT: "0" },
encoding: "buffer",
maxBuffer: maxBytes + 1024 * 1024,
},
);
if (stdout.length > maxBytes) {
throw new WorkspaceRegistryError("workspace_invalid", "Workspace annotations object is too large");
}
return stdout;
} catch (error) {
if (error instanceof WorkspaceRegistryError) throw error;
const detail = error instanceof Error ? error.message : "";
if (/maxBuffer|stdout maxBuffer/i.test(detail)) {
throw new WorkspaceRegistryError("workspace_invalid", "Workspace annotations object is too large");
}
throw this.sanitizeGitError(error);
}
}
/** Write only a validated API-owned artifact below the checked-out repository. */
async writeRegistryFile(path: string, source: string): Promise<void> {
this.assertRegistryArtifactPath(path);
backend/src/workspaces/registry.ts
+5
View File
@@ -3,6 +3,7 @@ import { lstatSync } from "node:fs";
import { mkdir, readdir, readFile, rename, rm, writeFile } from "node:fs/promises";
import { isAbsolute, join } from "node:path";
import { buildInstallationContract, renderWorkspaceDocs } from "./contracts.js";
import { parseAnnotationsYaml } from "./annotations.js";
import { assertCatalogMatchesDescriptor, parseWorkspaceCatalogYaml, type WorkspaceCatalog, type WorkspaceCatalogEntry } from "./catalog.js";
import {
GitWorkspaceRepository,
@@ -581,6 +582,10 @@ export class WorkspaceRegistry {
}
assertCatalogMatchesDescriptor(entry, workspace);
await this.assertEvidenceContext(workspace, safeHead);
const annotations = await this.repository.annotationsObject(safeHead, id);
if (annotations !== undefined) {
parseAnnotationsYaml(annotations.contents.toString("utf8"));
}
const collection = workspace.semantic_index.vector_store.collection;
const owner = collectionOwners.get(collection);
if (owner !== undefined) {
backend/test/registry-annotations.test.ts
+116
View File
@@ -0,0 +1,116 @@
import { execFile } from "node:child_process";
import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { promisify } from "node:util";
import { afterEach, expect, test } from "vitest";
import { WorkspaceRegistry } from "../src/workspaces/registry.js";
import type { WorkspaceRegistryConfig } from "../src/workspaces/types.js";
const runFile = promisify(execFile);
const temporaryRoots: string[] = [];
afterEach(() => {
temporaryRoots.splice(0).forEach((root) => rmSync(root, { recursive: true, force: true }));
});
const validYaml = `workspace:
schema_version: 3
id: psd-clinical
name: Policlinico San Donato
language: it
dwh:
engine: postgres
database: postgres
schema: datawarehouse
supported_transports: [postgres_direct]
semantic_index:
vector_store:
engine: qdrant
collection: psd-clinical
dimensions: 1024
distance: cosine
embedding:
provider: ollama_internal
model: qwen3-embedding:0.6b
dimensions: 1024
llm_policy:
allowed: [zai/glm-5.2]
`;
async function git(cwd: string, args: string[]): Promise<string> {
const { stdout } = await runFile("git", args, { cwd });
return stdout.trim();
}
function config(root: string, remoteUrl: string): WorkspaceRegistryConfig {
return {
root,
remoteUrl,
branch: "main",
gitAuthorName: "Registry Annotations Test",
gitAuthorEmail: "registry-annotations@example.invalid",
installationId: "test",
secretRoots: [],
maxImportBytes: 1024,
maxImportEntries: 1,
};
}
type AnnotationsLayout = "absent" | "valid" | "malformed" | "dir";
async function fixture(layout: AnnotationsLayout): Promise<{ root: string; remote: string }> {
const root = mkdtempSync(join(tmpdir(), "thoth-registry-annotations-"));
temporaryRoots.push(root);
const remote = join(root, "remote.git");
const source = join(root, "source");
await git(root, ["init", "--bare", "--initial-branch=main", remote]);
mkdirSync(source);
await git(source, ["init", "--initial-branch=main"]);
await git(source, ["config", "user.name", "Registry Annotations Test"]);
await git(source, ["config", "user.email", "registry-annotations@example.invalid"]);
writeFileSync(join(source, "thoth-workspaces.yaml"),
"schema_version: 1\nworkspaces:\n - id: psd-clinical\n name: Policlinico San Donato\n");
mkdirSync(join(source, "psd-clinical", "schema"), { recursive: true });
writeFileSync(join(source, "psd-clinical", "workspace.yaml"), validYaml);
const annotationsPath = join(source, "psd-clinical", "schema", "annotations.yaml");
if (layout === "valid") writeFileSync(annotationsPath, "tables: {}\n");
if (layout === "malformed") writeFileSync(annotationsPath, "tables: [not, a, mapping]\n");
if (layout === "dir") {
mkdirSync(annotationsPath, { recursive: true });
writeFileSync(join(annotationsPath, "child.txt"), "nested\n");
}
await git(source, ["add", "-A"]);
await git(source, ["commit", "-m", "initial"]);
await git(source, ["remote", "add", "origin", remote]);
await git(source, ["push", "origin", "main"]);
return { root, remote };
}
test("activation accepts a valid curated annotation blob", async () => {
const fixtureValue = await fixture("valid");
const registry = new WorkspaceRegistry(config(join(fixtureValue.root, "registry"), fixtureValue.remote));
await expect(registry.bootstrap()).resolves.toMatchObject({ degraded: false });
});
test("activation accepts an absent annotation blob", async () => {
const fixtureValue = await fixture("absent");
const registry = new WorkspaceRegistry(config(join(fixtureValue.root, "registry"), fixtureValue.remote));
await expect(registry.bootstrap()).resolves.toMatchObject({ degraded: false });
});
test("activation rejects malformed annotations", async () => {
const fixtureValue = await fixture("malformed");
const registry = new WorkspaceRegistry(config(join(fixtureValue.root, "registry"), fixtureValue.remote));
await expect(registry.bootstrap()).rejects.toMatchObject({ code: "workspace_invalid" });
});
test("activation rejects a tree at the annotations path", async () => {
const fixtureValue = await fixture("dir");
const registry = new WorkspaceRegistry(config(join(fixtureValue.root, "registry"), fixtureValue.remote));
await expect(registry.bootstrap()).rejects.toMatchObject({ code: "workspace_invalid" });
});
backend/test/workspaces-git-annotations.test.ts
+150
View File
@@ -0,0 +1,150 @@
import { execFile } from "node:child_process";
import { mkdtempSync, mkdirSync, rmSync, symlinkSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { promisify } from "node:util";
import { afterEach, expect, test } from "vitest";
import { GitWorkspaceRepository } from "../src/workspaces/git-repository.js";
import type { WorkspaceRegistryConfig } from "../src/workspaces/types.js";
const runFile = promisify(execFile);
const temporaryRoots: string[] = [];
afterEach(() => {
temporaryRoots.splice(0).forEach((root) => rmSync(root, { recursive: true, force: true }));
});
async function git(cwd: string, args: string[]): Promise<string> {
const { stdout } = await runFile("git", args, { cwd });
return stdout.trim();
}
function config(root: string, remoteUrl: string): WorkspaceRegistryConfig {
return {
root,
remoteUrl,
branch: "main",
gitAuthorName: "Workspace Registry Test",
gitAuthorEmail: "workspace-registry@example.invalid",
installationId: "test",
secretRoots: [],
maxImportBytes: 1024,
maxImportEntries: 1,
};
}
interface RepoFixture {
root: string;
remote: string;
source: string;
commit: string;
}
async function makeRepo(id: string, annotations: string | Buffer | "dir" | "symlink"): Promise<RepoFixture> {
const root = mkdtempSync(join(tmpdir(), "thoth-annotations-git-"));
temporaryRoots.push(root);
const remote = join(root, "remote.git");
const source = join(root, "source");
await git(root, ["init", "--bare", "--initial-branch=main", remote]);
mkdirSync(source);
await git(source, ["init", "--initial-branch=main"]);
await git(source, ["config", "user.name", "Annotations Test"]);
await git(source, ["config", "user.email", "annotations@example.invalid"]);
writeFileSync(join(source, "thoth-workspaces.yaml"),
`schema_version: 1\nworkspaces: [{id: ${id}, name: Workspace}]\n`);
mkdirSync(join(source, id, "schema"), { recursive: true });
writeFileSync(join(source, id, "workspace.yaml"), `workspace:\n schema_version: 3\n id: ${id}\n`);
const annotationsPath = join(source, id, "schema", "annotations.yaml");
if (annotations === "dir") {
mkdirSync(annotationsPath, { recursive: true });
writeFileSync(join(annotationsPath, "child.txt"), "not a blob\n");
} else if (annotations === "symlink") {
writeFileSync(join(source, id, "target.yaml"), "tables: {}\n");
symlinkSync("target.yaml", annotationsPath);
} else {
writeFileSync(annotationsPath, annotations);
}
await git(source, ["add", "-A"]);
await git(source, ["commit", "-m", "initial"]);
await git(source, ["remote", "add", "origin", remote]);
await git(source, ["push", "origin", "main"]);
const commit = await git(source, ["rev-parse", "HEAD"]);
return { root, remote, source, commit };
}
async function bootstrapped(fixture: RepoFixture): Promise<GitWorkspaceRepository> {
const repository = new GitWorkspaceRepository(config(join(fixture.root, "registry"), fixture.remote));
await repository.bootstrap();
return repository;
}
test("reads a regular annotation blob at the exact commit", async () => {
const fixture = await makeRepo("research", "tables: {}\n");
const repository = await bootstrapped(fixture);
const object = await repository.annotationsObject(fixture.commit, "research");
expect(object).toBeDefined();
expect(object!.blobId).toMatch(/^[0-9a-f]{40}$/);
expect(object!.contents.toString("utf8")).toBe("tables: {}\n");
});
test("returns undefined when the annotation object is absent", async () => {
const fixture = await makeRepo("research", "tables: {}\n");
const repository = await bootstrapped(fixture);
await expect(repository.annotationsObject(fixture.commit, "absent")).resolves.toBeUndefined();
});
test("refuses a tree at the annotation path", async () => {
const fixture = await makeRepo("clinical", "dir");
const repository = await bootstrapped(fixture);
await expect(repository.annotationsObject(fixture.commit, "clinical"))
.rejects.toMatchObject({ code: "workspace_invalid" });
});
test("refuses a symlink at the annotation path", async () => {
const fixture = await makeRepo("research", "symlink");
const repository = await bootstrapped(fixture);
await expect(repository.annotationsObject(fixture.commit, "research"))
.rejects.toMatchObject({ code: "workspace_invalid" });
});
test("refuses oversized and non-UTF-8 annotation blobs", async () => {
const oversized = await makeRepo("research", Buffer.concat([
Buffer.from("tables: {}\n"),
Buffer.alloc(16 * 1024 * 1024, 0x78),
]));
const repository = await bootstrapped(oversized);
await expect(repository.annotationsObject(oversized.commit, "research"))
.rejects.toMatchObject({ code: "workspace_invalid" });
const nonUtf8 = await makeRepo("research", Buffer.from([0x74, 0x61, 0x62, 0xff, 0xfe, 0x00]));
const repository2 = await bootstrapped(nonUtf8);
await expect(repository2.annotationsObject(nonUtf8.commit, "research"))
.rejects.toMatchObject({ code: "workspace_invalid" });
});
test("refuses malformed ids and revisions before Git", async () => {
const fixture = await makeRepo("research", "tables: {}\n");
const repository = await bootstrapped(fixture);
await expect(repository.annotationsObject(fixture.commit, "workspace-docs"))
.rejects.toMatchObject({ code: "workspace_invalid" });
await expect(repository.annotationsObject(fixture.commit, "../research"))
.rejects.toMatchObject({ code: "workspace_invalid" });
await expect(repository.annotationsObject("HEAD", "research"))
.rejects.toMatchObject({ code: "workspace_invalid" });
});
test("redacts Git failures while reading the annotation blob", async () => {
const fixture = await makeRepo("research", "tables: {}\n");
const repository = await bootstrapped(fixture);
rmSync(repository.repoPath, { recursive: true, force: true });
const error = await repository.annotationsObject(fixture.commit, "research").catch((failure: unknown) => failure);
expect(error).toMatchObject({ code: "git_unavailable", message: "Workspace Git operation failed" });
expect((error as Error).message).not.toContain(fixture.root);
});