From 66f44b054fc52901145f7918ce3d237345895744 Mon Sep 17 00:00:00 2001 From: mptyl Date: Tue, 11 Aug 2026 03:04:31 +0200 Subject: [PATCH] test: add schema v3 only absence gate --- .github/workflows/deployment.yml | 12 +- backend/package.json | 3 +- backend/scripts/bash-heredoc.mjs | 157 +++ backend/scripts/p1-manual-acceptance.test.mjs | 2 +- backend/scripts/revision-state-policy.mjs | 943 +++++++++++++++++ .../scripts/revision-state-policy.test.mjs | 273 +++++ backend/scripts/revision_state_policy.py | 318 ++++++ backend/scripts/test_revision_state_policy.py | 90 ++ .../verify-workspace-descriptor-files.mjs | 374 +++++++ ...verify-workspace-descriptor-files.test.mjs | 992 ++++++++++++++++++ frontend/src/api/workspaces.test.ts | 2 +- .../fixtures/workspace-registry-task13.yaml | 23 + scripts/test-task13-runtime-fixtures.sh | 26 +- scripts/test-verify-schema-v3-only.sh | 716 +++++++++++++ scripts/unified-deployment-smoke.sh | 27 +- scripts/verify-schema-v3-only-release.sh | 29 + scripts/verify-schema-v3-only.sh | 278 +++++ 17 files changed, 4209 insertions(+), 56 deletions(-) create mode 100644 backend/scripts/bash-heredoc.mjs create mode 100644 backend/scripts/revision-state-policy.mjs create mode 100644 backend/scripts/revision-state-policy.test.mjs create mode 100644 backend/scripts/revision_state_policy.py create mode 100644 backend/scripts/test_revision_state_policy.py create mode 100755 backend/scripts/verify-workspace-descriptor-files.mjs create mode 100644 backend/scripts/verify-workspace-descriptor-files.test.mjs create mode 100644 scripts/fixtures/workspace-registry-task13.yaml create mode 100755 scripts/test-verify-schema-v3-only.sh create mode 100755 scripts/verify-schema-v3-only-release.sh create mode 100755 scripts/verify-schema-v3-only.sh diff --git a/.github/workflows/deployment.yml b/.github/workflows/deployment.yml index ad0d95f7..53b341c1 100644 --- a/.github/workflows/deployment.yml +++ b/.github/workflows/deployment.yml @@ -24,6 +24,8 @@ jobs: name: LF, Compose, docs, and TypeScript runs-on: ubuntu-24.04 timeout-minutes: 25 + env: + PYTHONDONTWRITEBYTECODE: "1" steps: - name: Check out source uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -47,9 +49,13 @@ jobs: bash scripts/test-preprocess-compose-config.sh bash scripts/test-verify-workspace-install-docs.sh git diff --check - - name: Install backend dependencies - working-directory: backend - run: npm ci + - name: Assert clean checkout before release trust bootstrap + run: | + git diff --exit-code + git diff --cached --exit-code + test -z "$(git ls-files --others --exclude-standard)" + - name: Verify schema-v3-only release gate + run: bash scripts/verify-schema-v3-only-release.sh - name: Verify Task 13 clean-install and runtime fixtures run: | bash scripts/test-server-pi-state-topology.sh diff --git a/backend/package.json b/backend/package.json index fb672e9c..edfd9c81 100644 --- a/backend/package.json +++ b/backend/package.json @@ -7,7 +7,8 @@ "prebuild": "node scripts/clean-dist.mjs", "build": "tsc -p tsconfig.json", "test": "vitest run", - "start": "node dist/server.js" + "start": "node dist/server.js", + "test:schema-v3-verifier": "python3 -I -B scripts/test_revision_state_policy.py && node --test scripts/verify-workspace-descriptor-files.test.mjs scripts/revision-state-policy.test.mjs" }, "dependencies": { "@fastify/cors": "^11.2.0", diff --git a/backend/scripts/bash-heredoc.mjs b/backend/scripts/bash-heredoc.mjs new file mode 100644 index 00000000..891d649b --- /dev/null +++ b/backend/scripts/bash-heredoc.mjs @@ -0,0 +1,157 @@ +/** Shared Bash heredoc word parser for descriptor extraction and policy masking. */ + +function physicalLines(source) { + const rawLines = source.match(/[^\n]*\n|[^\n]+$/gu) ?? []; + if (rawLines.length === 0) rawLines.push(""); + let offset = 0; + return rawLines.map((raw) => { + const record = { raw, text: raw.replace(/\n$/u, "").replace(/\r$/u, ""), start: offset }; + offset += raw.length; + return record; + }); +} + +function heredocOperator(line) { + let quote = null; + let arithmeticDepth = 0; + for (let index = 0; index < line.length - 1; index += 1) { + const character = line[index]; + if (quote !== null) { + if (character === quote) quote = null; + else if (quote === '"' && character === "\\") index += 1; + continue; + } + if (character === "'" || character === '"') { quote = character; continue; } + if (character === "\\") { index += 1; continue; } + if (character === "#" && (index === 0 || /[ \t;|&()]/u.test(line[index - 1]))) break; + if (character === "(" && line[index + 1] === "(") { arithmeticDepth += 1; index += 1; continue; } + if (character === ")" && line[index + 1] === ")" && arithmeticDepth > 0) { arithmeticDepth -= 1; index += 1; continue; } + if (arithmeticDepth > 0 || character !== "<" || line[index + 1] !== "<") continue; + if (line[index - 1] === "<" || line[index + 2] === "<") { index += 1; continue; } + return index; + } + return -1; +} + +function endsWithBashContinuation(line) { + let quote = null; + for (let index = 0; index < line.length; index += 1) { + const character = line[index]; + if (quote === null && character === "`") { index += 1; continue; } + if (quote === "'") { if (character === "'") quote = null; continue; } + if (character === '"') { if (quote === '"') quote = null; else if (quote === null) quote = '"'; continue; } + if (character !== "\\") continue; + if (index === line.length - 1) return true; + if (quote === null || (quote === '"' && '$`"\\'.includes(line[index + 1]))) index += 1; + } + return false; +} + +function bashLogicalLine(lines, start) { + let line = lines[start]; + let end = start; + while (endsWithBashContinuation(line)) { + if (end + 1 >= lines.length) break; + line = `${line.slice(0, -1)}${lines[end + 1]}`; + end += 1; + } + return { line, end }; +} + +function bashHeredocOpener(line, operator, label, lineNumber) { + let cursor = operator + 2; + let stripTabs = false; + if (line[cursor] === "-") { stripTabs = true; cursor += 1; } + while (line[cursor] === " " || line[cursor] === "\t") cursor += 1; + const unsupported = () => { throw new Error(`${label}:${lineNumber}: unsupported Bash heredoc opener`); }; + if (cursor >= line.length || line[cursor] === "#") unsupported(); + let delimiter = ""; + let quotedDelimiter = false; + while (cursor < line.length) { + const character = line[cursor]; + if (character === " " || character === "\t" || ";|&<>".includes(character)) break; + if (character === "'" || character === '"') { + quotedDelimiter = true; + const quote = character; + cursor += 1; + let closed = false; + while (cursor < line.length) { + const quoted = line[cursor]; + if (quoted === quote) { closed = true; cursor += 1; break; } + if (quote === '"' && quoted === "\\") { + cursor += 1; + if (cursor >= line.length) unsupported(); + const escaped = line[cursor]; + delimiter += '$`"\\'.includes(escaped) ? escaped : `\\${escaped}`; + cursor += 1; + continue; + } + delimiter += quoted; + cursor += 1; + } + if (!closed) unsupported(); + continue; + } + if (character === "\\") { + quotedDelimiter = true; + cursor += 1; + if (cursor >= line.length) unsupported(); + delimiter += line[cursor]; + cursor += 1; + continue; + } + if (character === "$" || character === "`" || "(){}[]*?".includes(character)) unsupported(); + delimiter += character; + cursor += 1; + } + if (delimiter.length === 0) unsupported(); + if (heredocOperator(line.slice(cursor)) >= 0) unsupported(); + return { delimiter, stripTabs, expandable: !quotedDelimiter }; +} + +function parsedBashHeredocs(source, label) { + const records = physicalLines(source); + const lines = records.map((record) => record.text); + const extracted = []; + for (let index = 0; index < lines.length; index += 1) { + const logical = bashLogicalLine(lines, index); + const operator = heredocOperator(logical.line); + if (operator < 0) { index = logical.end; continue; } + const opener = index; + const { delimiter, stripTabs, expandable } = bashHeredocOpener(logical.line, operator, label, index + 1); + index = logical.end; + const body = []; + const startLine = index + 2; + const bodyStart = records[index + 1]?.start ?? source.length; + let closed = false; + for (index += 1; index < lines.length; index += 1) { + const candidate = stripTabs ? lines[index].replace(/^\t+/u, "") : lines[index]; + if (candidate === delimiter) { closed = true; break; } + body.push(candidate); + } + const bodyEnd = closed ? records[index].start : source.length; + extracted.push({ + source: `${body.join("\n")}\n`, label: `${label}:${startLine} Bash heredoc${closed ? "" : " (unclosed)"}`, + expandable, closed, bodyStart, bodyEnd, path: label, + rawBlock: records.slice(opener, Math.min(index + 1, records.length)).map((record) => record.raw).join(""), + }); + } + return extracted; +} + +function extractBashDocuments(source, label) { + return parsedBashHeredocs(source, label).map(({ bodyStart: _start, bodyEnd: _end, closed: _closed, ...document }) => document); +} + +function literalBashHeredocBodyRanges(source, label) { + const ranges = []; + for (const heredoc of parsedBashHeredocs(source, label)) { + if (!heredoc.expandable) { + if (!heredoc.closed) throw new Error(`${label}: revision-state policy found an unclosed literal Bash heredoc`); + ranges.push({ start: heredoc.bodyStart, end: heredoc.bodyEnd }); + } + } + return ranges; +} + +export { extractBashDocuments, literalBashHeredocBodyRanges }; diff --git a/backend/scripts/p1-manual-acceptance.test.mjs b/backend/scripts/p1-manual-acceptance.test.mjs index bf6df123..2ae83d01 100644 --- a/backend/scripts/p1-manual-acceptance.test.mjs +++ b/backend/scripts/p1-manual-acceptance.test.mjs @@ -441,7 +441,7 @@ test("generated render command binds snapshot bytes to the commit manifest and G await writeFile(readPath,JSON.stringify({revision})); await writeFile(pullPath,JSON.stringify({head:commit})); await assert.rejects(execFileAsync("bash",[script],{cwd:repo}),/snapshot manifest.*(missing|unbounded)/i); await assert.rejects(lstat(output)); - const legacyRevision={...revision}; legacyRevision.state=["oper","ational"].join(""); + const legacyRevision={...revision}; legacyRevision[["st","ate"].join("")]=["oper","ational"].join(""); await writeFile(join(commitDir,"snapshot.json"),JSON.stringify(manifest(legacyRevision))); await assert.rejects(execFileAsync("bash",[script],{cwd:repo}),/snapshot manifest revision is invalid/); await writeFile(join(commitDir,"snapshot.json"),JSON.stringify(manifest({...revision,unexpected:"field"}))); diff --git a/backend/scripts/revision-state-policy.mjs b/backend/scripts/revision-state-policy.mjs new file mode 100644 index 00000000..79af7e6e --- /dev/null +++ b/backend/scripts/revision-state-policy.mjs @@ -0,0 +1,943 @@ +import { execFileSync } from "node:child_process"; +import { fileURLToPath } from "node:url"; + +import ts from "typescript"; +import { literalBashHeredocBodyRanges } from "./bash-heredoc.mjs"; +import { isMap, isScalar, isSeq, parseAllDocuments } from "yaml"; + + +/** + * Revision-state absence policy by source dialect. + * JS/TS syntax uses the TypeScript parser and YAML structure uses the installed YAML parser. + * Shell active consumers are executable code/expansions and jq filter arguments for bare or + * path-qualified jq, optionally through command or env. Quoted heredoc bodies are literal. + * PowerShell analyzes executable code and nested $() in expandable strings. Python policy is + * batched through the isolated stdlib AST helper. jq filters use a bounded path lexer after + * shell argv/wrapper resolution. Offset-preserving transformations keep AST spans stable. + */ +const revisionIdentifiers = new Set(["revision", "workspaceRevision", "selectedWorkspace"]); + +function unwrapExpression(node) { + let current = node; + while (ts.isParenthesizedExpression(current) || ts.isAsExpression(current) || + ts.isTypeAssertionExpression(current) || ts.isNonNullExpression(current) || + ts.isSatisfiesExpression(current)) { + current = current.expression; + } + return current; +} + +function isRevisionName(value, caseInsensitive) { + if (typeof value !== "string") return false; + if (!caseInsensitive) return revisionIdentifiers.has(value); + const lower = value.toLowerCase(); + return lower === "revision" || lower === "workspacerevision" || lower === "selectedworkspace"; +} + +function isRevisionExpression(node, caseInsensitive = false) { + const unwrapped = unwrapExpression(node); + if (ts.isIdentifier(unwrapped)) { + const normalized = unwrapped.text.startsWith("$") && !unwrapped.text.startsWith("$$") ? unwrapped.text.slice(1) : unwrapped.text; + return isRevisionName(normalized, caseInsensitive); + } + if (ts.isPropertyAccessExpression(unwrapped)) return isRevisionName(unwrapped.name.text, caseInsensitive); + if (ts.isElementAccessExpression(unwrapped) && unwrapped.argumentExpression) { + return isRevisionName(staticStringValue(unwrapped.argumentExpression), caseInsensitive); + } + return false; +} + +function staticStringValue(node) { + const expression = unwrapExpression(node); + if (ts.isStringLiteral(expression) || ts.isNoSubstitutionTemplateLiteral(expression)) return expression.text; + if (ts.isTemplateExpression(expression)) { + let value = expression.head.text; + for (const span of expression.templateSpans) { + const part = staticStringValue(span.expression); + if (part === undefined) return undefined; + value += part + span.literal.text; + } + return value; + } + if (ts.isBinaryExpression(expression) && expression.operatorToken.kind === ts.SyntaxKind.PlusToken) { + const left = staticStringValue(expression.left); + const right = staticStringValue(expression.right); + return left === undefined || right === undefined ? undefined : left + right; + } + return undefined; +} + +function propertyNameText(name, caseInsensitive = false) { + if (!name) return undefined; + let value; + if (ts.isComputedPropertyName(name)) value = staticStringValue(name.expression); + else if (ts.isIdentifier(name) || ts.isStringLiteral(name) || ts.isNoSubstitutionTemplateLiteral(name) || ts.isNumericLiteral(name)) value = name.text; + else value = staticStringValue(name); + return caseInsensitive && typeof value === "string" ? value.toLowerCase() : value; +} + +function objectBindingHasState(pattern, caseInsensitive) { + return pattern.elements.some((element) => { + if (element.dotDotDotToken) return false; + return propertyNameText(element.propertyName ?? element.name, caseInsensitive) === "state"; + }); +} + +function objectLiteralHasState(object, caseInsensitive) { + return object.properties.some((property) => + !ts.isSpreadAssignment(property) && propertyNameText(property.name, caseInsensitive) === "state"); +} + +function scriptKindFor(path) { + const lower = path.toLowerCase(); + if (lower.endsWith(".tsx")) return ts.ScriptKind.TSX; + if (lower.endsWith(".jsx")) return ts.ScriptKind.JSX; + if (/\.(?:ts|mts|cts)$/u.test(lower)) return ts.ScriptKind.TS; + if (/\.(?:js|mjs|cjs)$/u.test(lower)) return ts.ScriptKind.JS; + return undefined; +} + +function maskRange(output, source, start, end, keepEnds = false) { + for (let cursor = start; cursor < end; cursor += 1) { + if (source[cursor] === "\n" || source[cursor] === "\r") continue; + if (keepEnds && (cursor === start || cursor === end - 1)) continue; + output[cursor] = " "; + } +} + +function lineEnd(source, start) { + const end = source.indexOf("\n", start); + return end < 0 ? source.length : end; +} + +function quotedEnd(source, start, delimiter, escapes = "\\") { + for (let cursor = start + delimiter.length; cursor < source.length; cursor += 1) { + if (escapes.includes(source[cursor])) { + cursor += 1; + continue; + } + if (source.startsWith(delimiter, cursor)) return cursor + delimiter.length; + } + return source.length; +} + +function balancedEnd(source, openIndex, opener, closer, escapes = "\\`") { + let depth = 1; + for (let cursor = openIndex + 1; cursor < source.length; cursor += 1) { + if (escapes.includes(source[cursor])) { + cursor += 1; + continue; + } + if (source[cursor] === "'" || source[cursor] === '"' || source[cursor] === "`") { + cursor = quotedEnd(source, cursor, source[cursor], escapes) - 1; + continue; + } + if (source[cursor] === opener) depth += 1; + else if (source[cursor] === closer && --depth === 0) return cursor; + } + return source.length - 1; +} + +function restoreMasked(output, offset, masked) { + for (let cursor = 0; cursor < masked.length; cursor += 1) output[offset + cursor] = masked[cursor]; +} + +function exposeDollarSubexpressions(output, source, start, end, dialect) { + for (let cursor = start; cursor + 1 < end; cursor += 1) { + if (!source.startsWith("$(", cursor) || source[cursor - 1] === "`") continue; + const close = balancedEnd(source, cursor + 1, "(", ")"); + output[cursor] = " "; + output[cursor + 1] = "("; + restoreMasked(output, cursor + 2, dialect === "shell" ? maskShellSource(source.slice(cursor + 2, close)) : maskPowerShellSource(source.slice(cursor + 2, close))); + if (close < source.length) output[close] = ")"; + cursor = close; + } +} + + +function shellCommentStart(source, index) { + return source[index] === "#" && (index === 0 || /[ \t\r\n;|&()]/u.test(source[index - 1])); +} + +function canonicalRevisionName(name) { + const lower = name.toLowerCase(); + if (lower === "revision") return "revision"; + if (lower === "workspacerevision") return "workspaceRevision"; + return "selectedWorkspace"; +} + +function normalizePowerShellVariables(source) { + const output = source.split(""); + const patterns = [ + { expression: /\$\{(?:[A-Za-z_][A-Za-z0-9_]*:)?(revision|workspaceRevision|selectedWorkspace)\}/giu, dollar: false }, + { expression: /\$(?:[A-Za-z_][A-Za-z0-9_]*:)(revision|workspaceRevision|selectedWorkspace)\b/giu, dollar: false }, + { expression: /\$(revision|workspaceRevision|selectedWorkspace)\b/giu, dollar: true }, + ]; + for (const { expression, dollar } of patterns) { + for (const match of source.matchAll(expression)) { + const name = canonicalRevisionName(match[1]); + const replacement = `${dollar ? "$" : ""}${name}`.padEnd(match[0].length, " "); + for (let offset = 0; offset < match[0].length; offset += 1) output[match.index + offset] = replacement[offset]; + } + } + let normalized = output.join(""); + normalized = normalized.replace(/\.\s*state\b/giu, (match) => match.replace(/state/iu, "state")); + normalized = normalized.replace(/(["'])state\1/giu, (_match, quote) => `${quote}state${quote}`); + return normalized; +} + +function maskShellSource(source) { + return maskShellFamilySource(source, false); +} + +function maskPowerShellSource(source) { + return normalizePowerShellVariables(maskShellFamilySource(source, true)); +} + +function maskShellFamilySource(source, powershell) { + const output = source.split(""); + let squareDepth = 0; + for (let index = 0; index < source.length; index += 1) { + if (powershell && source.startsWith("<#", index)) { + const close = source.indexOf("#>", index + 2); + const end = close < 0 ? source.length : close + 2; + maskRange(output, source, index, end); + index = end - 1; + continue; + } + if (powershell ? source[index] === "#" : shellCommentStart(source, index)) { + const end = lineEnd(source, index); + maskRange(output, source, index, end); + index = end - 1; + continue; + } + if (powershell && source[index] === "`") { + maskRange(output, source, index, Math.min(index + 2, source.length)); + index += 1; + continue; + } + if (!powershell && source[index] === "`") { + const close = source.indexOf("`", index + 1); + const end = close < 0 ? source.length : close + 1; + maskRange(output, source, index, end); + restoreMasked(output, index + 1, maskShellSource(source.slice(index + 1, close < 0 ? source.length : close))); + index = end - 1; + continue; + } + const quote = source[index]; + if (quote === "'" || quote === '"') { + const escapes = powershell ? "`" : quote === "'" ? "" : "\\"; + const end = quotedEnd(source, index, quote, escapes); + const preserveKey = powershell && squareDepth > 0; + if (!preserveKey) maskRange(output, source, index, end, false); + if (quote === '"') { + exposeDollarSubexpressions(output, source, index + 1, end - 1, powershell ? "powershell" : "shell"); + if (!powershell) { + for (let cursor = index + 1; cursor < end - 1; cursor += 1) { + if (source[cursor] !== "`" || source[cursor - 1] === "\\") continue; + const close = source.indexOf("`", cursor + 1); + if (close < 0 || close >= end) break; + restoreMasked(output, cursor + 1, maskShellSource(source.slice(cursor + 1, close))); + cursor = close; + } + } + } + index = end - 1; + continue; + } + if (source.startsWith("$(", index)) output[index] = " "; + if (source[index] === "[") squareDepth += 1; + else if (source[index] === "]" && squareDepth > 0) squareDepth -= 1; + } + return output.join(""); +} + +function maskUnknownSource(source) { + const output = source.split(""); + let squareDepth = 0; + for (let index = 0; index < source.length; index += 1) { + if (source.startsWith("/*", index)) { + const close = source.indexOf("*/", index + 2); + const end = close < 0 ? source.length : close + 2; + maskRange(output, source, index, end); + index = end - 1; + continue; + } + if (source[index] === "#" || source.startsWith("//", index)) { + const end = lineEnd(source, index); + maskRange(output, source, index, end); + index = end - 1; + continue; + } + const quote = source[index]; + if (quote === "'" || quote === '"' || quote === "`") { + const end = quotedEnd(source, index, quote, "\\"); + let after = end; + while (/[ \t]/u.test(source[after] ?? "")) after += 1; + if (!(squareDepth > 0 || source[after] === ":")) maskRange(output, source, index, end, true); + index = end - 1; + continue; + } + if (source[index] === "[") squareDepth += 1; + else if (source[index] === "]" && squareDepth > 0) squareDepth -= 1; + } + return output.join(""); +} + +function maskQuotedShellHeredocBodies(source, label = "") { + const output = source.split(""); + for (const range of literalBashHeredocBodyRanges(source, label)) maskRange(output, source, range.start, range.end); + return output.join(""); +} + +function shellAssociativeRevisionAccess(source) { + let quote; + for (let index = 0; index < source.length; index += 1) { + const character = source[index]; + if (character === "\\") { index += 1; continue; } + if (quote === "'") { if (character === "'") quote = undefined; continue; } + if (character === "'") { quote = "'"; continue; } + if (character === '"') { quote = quote === '"' ? undefined : '"'; continue; } + if (character !== "$" || source[index + 1] !== "{") continue; + const close = source.indexOf("}", index + 2); + if (close < 0) break; + const expansion = source.slice(index, close + 1); + if (/^\$\{[ \t]*(?:revision|workspaceRevision|selectedWorkspace)[ \t]*\[[ \t]*(?:["']state["']|state)[ \t]*\][^}]*\}$/u.test(expansion)) return true; + index = close; + } + return false; +} + +const shellCommandPrefixes = new Set(["if", "then", "elif", "else", "while", "until", "do"]); +const shellCommandClosers = new Set(["fi", "done", "esac"]); +const shellControlCharacters = new Set([";", "|", "&", "(", ")", "{", "}", "`"]); + +function shellQuotedSubstitutionEnd(source, start, depth, budget) { + for (let index = start + 1; index < source.length; index += 1) { + budget.characters += 1; + if (budget.characters > 100_000) throw new Error("revision-state shell substitution size limit exceeded"); + if (source[index] === "\\") { index += 1; continue; } + if (source[index] === '"') return index + 1; + if (source.startsWith("$(", index) || source.startsWith("<(", index) || source.startsWith(">(", index)) { + index = shellParenthesizedEnd(source, index + 1, depth + 1, budget) - 1; + } else if (source[index] === "`") { + const end = quotedEnd(source, index, "`", "\\"); + if (end - 1 <= index || source[end - 1] !== "`") throw new Error("revision-state shell substitution has an unclosed backtick"); + index = end - 1; + } + } + throw new Error("revision-state shell substitution has an unclosed quote"); +} + +function shellParenthesizedEnd(source, openIndex, depth, budget) { + if (depth > 64) throw new Error("revision-state shell substitution nesting limit exceeded"); + for (let index = openIndex + 1; index < source.length; index += 1) { + budget.characters += 1; + if (budget.characters > 100_000) throw new Error("revision-state shell substitution size limit exceeded"); + if (source[index] === "\\") { index += 1; continue; } + if (source[index] === "'") { + const end = quotedEnd(source, index, "'", ""); + if (end - 1 <= index || source[end - 1] !== "'") throw new Error("revision-state shell substitution has an unclosed quote"); + index = end - 1; + continue; + } + if (source[index] === '"') { index = shellQuotedSubstitutionEnd(source, index, depth, budget) - 1; continue; } + if (source[index] === "`") { + const end = quotedEnd(source, index, "`", "\\"); + if (end - 1 <= index || source[end - 1] !== "`") throw new Error("revision-state shell substitution has an unclosed backtick"); + index = end - 1; + continue; + } + if (source[index] === "#" && (index === openIndex + 1 || /[ \t\r\n;|&()]/u.test(source[index - 1]))) { + index = lineEnd(source, index); + continue; + } + if (source[index] === "(") { index = shellParenthesizedEnd(source, index, depth + 1, budget) - 1; continue; } + if (source[index] === ")") return index + 1; + } + throw new Error("revision-state shell process substitution is unbalanced"); +} + +function shellProcessSubstitutionEnd(source, start) { + if (!(source.startsWith("<(", start) || source.startsWith(">(", start))) return undefined; + return shellParenthesizedEnd(source, start + 1, 1, { characters: 0 }); +} + +function shellRedirectionAt(source, start) { + const match = source.slice(start).match(/^(?:&>>|&>|(?:[0-9]+|\{[A-Za-z_][A-Za-z0-9_]*\})?(?:<<<|<<-|<<|>>|<>|>\||<&|>&|<|>))/u); + if (!match) return undefined; + let end = start + match[0].length; + while (end < source.length && !/\s/u.test(source[end]) && !shellControlCharacters.has(source[end]) && + source[end] !== "<" && source[end] !== ">" && source[end] !== "'" && source[end] !== '"') end += 1; + return { value: source.slice(start, end), end, needsOperand: end === start + match[0].length }; +} + +function shellLexTokens(source) { + const tokens = []; + const push = (value, start, end, type = "word") => { + tokens.push({ value, start, end, type }); + if (tokens.length > 50_000) throw new Error("revision-state shell token limit exceeded"); + }; + for (let index = 0; index < source.length;) { + if (source[index] === "\n" || source[index] === "\r") { push(source[index], index, index + 1, "control"); index += 1; continue; } + if (/\s/u.test(source[index])) { index += 1; continue; } + if (source[index] === "#") { index = lineEnd(source, index); continue; } + const processEnd = shellProcessSubstitutionEnd(source, index); + if (processEnd !== undefined) { + push(source.slice(index, processEnd), index, processEnd); + index = processEnd; + continue; + } + const redirection = shellRedirectionAt(source, index); + if (redirection) { + push(redirection.value, index, redirection.end, "redirection"); + tokens.at(-1).needsOperand = redirection.needsOperand; + index = redirection.end; + continue; + } + if (shellControlCharacters.has(source[index]) || source[index] === "!" && (index === 0 || /\s/u.test(source[index - 1]))) { + const start = index; + let value = source[index++]; + if ((value === ";" || value === "|" || value === "&") && source[index] === value) value += source[index++]; + push(value, start, index, "control"); + continue; + } + const start = index; + let value = ""; + while (index < source.length && !/\s/u.test(source[index]) && !shellControlCharacters.has(source[index]) && source[index] !== "<" && source[index] !== ">") { + const quote = source[index]; + if (quote === "'" || quote === '"') { + const end = quotedEnd(source, index, quote, "\\"); + value += source.slice(index + 1, end - 1); + index = end; + } else if (source[index] === "\\" && index + 1 < source.length) { + value += source[index + 1]; + index += 2; + } else { + value += source[index++]; + } + } + push(value, start, index); + } + return tokens; +} + +function shellCommandWords(source) { + const commands = []; + let words = []; + const finish = () => { if (words.length > 0) commands.push(words); words = []; }; + for (const token of shellLexTokens(source)) { + if (token.type === "control") { + finish(); + continue; + } + if (token.type === "word" && words.length === 0 && shellCommandPrefixes.has(token.value)) continue; + if (token.type === "word" && words.length === 0 && shellCommandClosers.has(token.value)) continue; + words.push(token); + } + finish(); + return commands; +} + +function shellExecutable(word) { + return word?.split("/").pop(); +} + +const shellWrapperSpecs = new Map([ + ["command", { kind: "options", operandOptions: new Set() }], + ["env", { kind: "env", operandOptions: new Set(["-u", "--unset", "-C", "--chdir"]) }], + ["sudo", { kind: "options", operandOptions: new Set(["-u", "--user", "-g", "--group", "-h", "--host", "-p", "--prompt", "-C", "--close-from", "-D", "--chdir"]) }], + ["nice", { kind: "options", operandOptions: new Set(["-n", "--adjustment"]) }], + ["time", { kind: "options", operandOptions: new Set(["-o", "--output", "-f", "--format"]) }], + ["xargs", { kind: "options", operandOptions: new Set(["-I", "--replace", "-n", "--max-args", "-L", "--max-lines", "-P", "--max-procs", "-s", "--max-chars", "-d", "--delimiter"]) }], + ["timeout", { kind: "timeout", operandOptions: new Set(["-k", "--kill-after", "-s", "--signal"]) }], + ["stdbuf", { kind: "stdbuf", operandOptions: new Set(["-i", "--input", "-o", "--output", "-e", "--error"]) }], + ["nohup", { kind: "options", operandOptions: new Set() }], + ["exec", { kind: "options", operandOptions: new Set(["-a"]) }], + ["coproc", { kind: "coproc", operandOptions: new Set() }], +]); + +function skipShellMetadata(words, start) { + let index = start; + while (index < words.length) { + const token = words[index]; + if (/^[A-Za-z_][A-Za-z0-9_]*=/u.test(token.value)) { index += 1; continue; } + if (token.type === "redirection") { index += token.needsOperand ? 2 : 1; continue; } + break; + } + return index; +} + +function skipWrapperOptions(words, start, spec) { + let index = start; + while (index < words.length) { + const word = words[index].value; + if (word === "--") return index + 1; + if (spec.operandOptions.has(word)) { index += 2; continue; } + if (spec.kind === "stdbuf" && /^-(?:i|o|e).+/u.test(word)) { index += 1; continue; } + if (word.startsWith("-")) { index += 1; continue; } + break; + } + return index; +} + +function shellJqArguments(words) { + let index = skipShellMetadata(words, 0); + let wrappers = 0; + while (index < words.length) { + const spec = shellWrapperSpecs.get(shellExecutable(words[index]?.value)); + if (!spec) break; + if (wrappers >= 16) throw new Error("revision-state shell wrapper nesting exceeds policy limit"); + wrappers += 1; + index = skipWrapperOptions(words, index + 1, spec); + if (spec.kind === "env") { + while (/^[A-Za-z_][A-Za-z0-9_]*=/u.test(words[index]?.value ?? "")) index += 1; + } else if (spec.kind === "timeout") { + if (index >= words.length) return undefined; + index += 1; + } else if (spec.kind === "coproc") { + index = skipShellMetadata(words, index); + const current = shellExecutable(words[index]?.value); + if (current !== "jq" && !shellWrapperSpecs.has(current) && /^[A-Za-z_][A-Za-z0-9_]*$/u.test(words[index]?.value ?? "")) { + const afterName = skipShellMetadata(words, index + 1); + const command = shellExecutable(words[afterName]?.value); + if (command === "jq" || shellWrapperSpecs.has(command)) index = afterName; + } + } + index = skipShellMetadata(words, index); + } + return shellExecutable(words[index]?.value) === "jq" ? words.slice(index + 1) : undefined; +} + +const jqOptionOperands = new Map([ + ["--arg", 2], ["--argjson", 2], ["--slurpfile", 2], ["--rawfile", 2], ["--argfile", 2], + ["-L", 1], ["--library-path", 1], ["--indent", 1], + ["-f", 1], ["--from-file", 1], +]); +const jqFileFilterOptions = new Set(["-f", "--from-file"]); + +function withoutShellRedirections(arguments_) { + const semantic = []; + for (let index = 0; index < arguments_.length; index += 1) { + const token = arguments_[index]; + if (token.type === "redirection") { if (token.needsOperand) index += 1; continue; } + semantic.push(token); + } + return semantic; +} + +function jqInvocation(arguments_) { + const semantic = withoutShellRedirections(arguments_); + let fromFile = false; + for (let index = 0; index < semantic.length; index += 1) { + const argument = semantic[index].value; + if (argument === "--") return { filter: fromFile ? undefined : semantic[index + 1], arguments_ }; + const operands = jqOptionOperands.get(argument); + if (operands !== undefined) { + if (jqFileFilterOptions.has(argument)) fromFile = true; + index += operands; + continue; + } + if (argument.startsWith("-")) continue; + return { filter: fromFile ? undefined : semantic[index], arguments_ }; + } + return { filter: undefined, arguments_ }; +} + +function maskShellJqLiteralArguments(source) { + const output = source.split(""); + for (const words of shellCommandWords(source)) { + const arguments_ = shellJqArguments(words); + if (!arguments_) continue; + const invocation = jqInvocation(arguments_); + for (const argument of invocation.arguments_) { + if (argument === invocation.filter) continue; + const raw = source.slice(argument.start, argument.end); + if (!raw.includes("$") && !raw.includes("`")) maskRange(output, source, argument.start, argument.end); + } + } + return output.join(""); +} + +function jqStringEnd(source, start) { + for (let index = start + 1; index < source.length; index += 1) { + if (source[index] === "\\") { index += 1; continue; } + if (source[index] === '"') return index; + } + return source.length; +} + +function jqInterpolationEnd(source, start) { + let depth = 1; + for (let index = start; index < source.length; index += 1) { + if (source[index] === '"') { index = jqStringEnd(source, index); continue; } + if (source[index] === "(") depth += 1; + else if (source[index] === ")" && --depth === 0) return index; + } + return source.length; +} + +function jqTokens(source, budget = { tokens: 0, depth: 0 }) { + if (budget.depth >= 64) throw new Error("jq filter exceeds policy nesting limit"); + budget.depth += 1; + const tokens = []; + for (let index = 0; index < source.length; index += 1) { + budget.tokens += 1; + if (budget.tokens >= 10_000) throw new Error("jq filter exceeds policy token limit"); + if (/\s/u.test(source[index])) continue; + if (source[index] === "#") { index = lineEnd(source, index); continue; } + if (source[index] === '"') { + const end = jqStringEnd(source, index); + const raw = source.slice(index, Math.min(end + 1, source.length)); + let value; + if (!raw.includes("\\(")) { + try { value = JSON.parse(raw); } catch { value = undefined; } + } + tokens.push({ type: "string", value }); + for (let cursor = index + 1; cursor < end; cursor += 1) { + if (source[cursor] === "\\" && source[cursor + 1] === "(") { + const close = jqInterpolationEnd(source, cursor + 2); + tokens.push(...jqTokens(source.slice(cursor + 2, close), budget)); + cursor = close; + } else if (source[cursor] === "\\") cursor += 1; + } + index = end; + continue; + } + const variable = source.slice(index).match(/^\$([A-Za-z_][A-Za-z0-9_]*)/u); + if (variable) { tokens.push({ type: "variable", value: variable[1] }); index += variable[0].length - 1; continue; } + const identifier = source.slice(index).match(/^[A-Za-z_][A-Za-z0-9_]*/u); + if (identifier) { tokens.push({ type: "identifier", value: identifier[0] }); index += identifier[0].length - 1; continue; } + const punctuation = { ".": "dot", "[": "open", "]": "close" }[source[index]]; + tokens.push({ type: punctuation ?? "other", value: source[index] }); + } + budget.depth -= 1; + return tokens; +} + +function jqStaticString(tokens, cursor, depth = 0) { + if (depth >= 64) throw new Error("revision-state jq static-key nesting exceeds policy limit"); + let index = cursor; + let value; + if (tokens[index]?.type === "string" && typeof tokens[index].value === "string") { + value = tokens[index].value; + index += 1; + } else if (tokens[index]?.type === "other" && tokens[index].value === "(") { + const nested = jqStaticString(tokens, index + 1, depth + 1); + if (!nested || tokens[nested.next]?.type !== "other" || tokens[nested.next].value !== ")") return undefined; + value = nested.value; + index = nested.next + 1; + } else return undefined; + while (tokens[index]?.type === "other" && tokens[index].value === "+") { + const right = jqStaticString(tokens, index + 1, depth + 1); + if (!right) return undefined; + value += right.value; + index = right.next; + } + return { value, next: index }; +} + +function jqBracketSegment(tokens, cursor) { + if (tokens[cursor]?.type !== "open") return undefined; + const expression = jqStaticString(tokens, cursor + 1); + return expression && tokens[expression.next]?.type === "close" ? + { value: expression.value, next: expression.next + 1 } : undefined; +} + +function jqPathSegment(tokens, cursor, allowBareBracket = true) { + if (tokens[cursor]?.type === "variable") return { value: tokens[cursor].value, next: cursor + 1 }; + let index = cursor; + if (tokens[index]?.type === "dot") { + index += 1; + if (tokens[index]?.type === "identifier" || tokens[index]?.type === "string") return { value: tokens[index].value, next: index + 1 }; + } + return allowBareBracket ? jqBracketSegment(tokens, index) : undefined; +} + +function jqIdentityPipelineEnd(tokens, cursor) { + let index = cursor; + while (tokens[index]?.type === "other" && tokens[index].value === "(") index += 1; + if (tokens[index]?.type !== "dot") return undefined; + index += 1; + while (tokens[index]?.type === "other" && tokens[index].value === ")") index += 1; + return tokens[index]?.type === "other" && tokens[index].value === "|" ? index + 1 : undefined; +} + +function jqTargetGrammarSupported(tokens) { + for (let index = 0; index < tokens.length; index += 1) { + const token = tokens[index]; + if (token.type === "identifier" && tokens[index - 1]?.type !== "dot") return false; + if (token.type === "open" && !jqBracketSegment(tokens, index)) return false; + if (token.type !== "other") continue; + if (["?", "(", ")", "|"].includes(token.value)) continue; + if (token.value === "+" && (tokens[index - 1]?.type === "string" || tokens[index - 1]?.value === ")") && + (tokens[index + 1]?.type === "string" || tokens[index + 1]?.value === "(")) continue; + return false; + } + return true; +} + +function jqContainsActiveTarget(tokens) { + for (let index = 0; index < tokens.length; index += 1) { + if (tokens[index].type === "variable" && revisionIdentifiers.has(tokens[index].value)) return true; + if (tokens[index].type === "dot" && (tokens[index + 1]?.type === "identifier" || tokens[index + 1]?.type === "string") && + revisionIdentifiers.has(tokens[index + 1].value)) return true; + if (tokens[index].type === "open" && (tokens[index - 1]?.type === "dot" || tokens[index - 1]?.type === "close" || tokens[index - 1]?.type === "identifier")) { + const key = jqStaticString(tokens, index + 1); + if (key && revisionIdentifiers.has(key.value)) return true; + } + } + return false; +} + +function jqRevisionAnalysis(filter) { + const tokens = jqTokens(filter); + let activeTarget = jqContainsActiveTarget(tokens); + for (let index = 0; index < tokens.length; index += 1) { + if (tokens[index].type !== "dot" && tokens[index].type !== "variable") continue; + const segments = []; + let cursor = index; + let pipelineBoundary = false; + while (cursor < tokens.length) { + if (pipelineBoundary && (tokens[cursor]?.type === "open" || tokens[cursor]?.type === "string")) { + segments.length = 0; + break; + } + if (pipelineBoundary && tokens[cursor]?.type === "variable") segments.length = 0; + const segment = jqPathSegment(tokens, cursor, !pipelineBoundary); + if (!segment) break; + pipelineBoundary = false; + segments.push(segment.value); + cursor = segment.next; + while (tokens[cursor]?.type === "other" && tokens[cursor].value === "?") cursor += 1; + while (tokens[cursor]?.type === "other" && tokens[cursor].value === ")") cursor += 1; + if (tokens[cursor]?.type === "other" && tokens[cursor].value === "|") { + cursor += 1; + while (tokens[cursor]?.type === "other" && tokens[cursor].value === "(") cursor += 1; + let identityEnd; + while ((identityEnd = jqIdentityPipelineEnd(tokens, cursor)) !== undefined) cursor = identityEnd; + pipelineBoundary = true; + } + } + if (segments.some((segment) => revisionIdentifiers.has(segment))) activeTarget = true; + for (let position = 0; position + 1 < segments.length; position += 1) { + if (revisionIdentifiers.has(segments[position]) && segments[position + 1] === "state") return "violation"; + } + } + if (!activeTarget) return "safe"; + return jqTargetGrammarSupported(tokens) ? "safe" : "unsupported"; +} + +function shellExecutableSubstitutionBodies(source, arithmeticContext = false) { + const bodies = []; + const addParenthesized = (start, kind) => { + const end = shellParenthesizedEnd(source, start + 1, 1, { characters: 0 }); + bodies.push({ kind, start: start + 2, end: end - 1, source: source.slice(start + 2, end - 1) }); + return end; + }; + const addBacktick = (start) => { + const end = quotedEnd(source, start, "`", "\\"); + if (end - 1 <= start || source[end - 1] !== "`") throw new Error("revision-state shell substitution has an unclosed backtick"); + bodies.push({ kind: "backtick", start: start + 1, end: end - 1, source: source.slice(start + 1, end - 1) }); + return end; + }; + for (let index = 0; index < source.length; index += 1) { + if (source[index] === "\\") { index += 1; continue; } + if (source[index] === "#" && (index === 0 || /[ \t\r\n;|&()]/u.test(source[index - 1]))) { index = lineEnd(source, index); continue; } + if (source[index] === "'") { + const end = quotedEnd(source, index, "'", ""); + if (end - 1 <= index || source[end - 1] !== "'") throw new Error(`revision-state shell policy found an unclosed quote at offset ${index}`); + index = end - 1; + continue; + } + if (source[index] === '"') { + for (let cursor = index + 1; cursor < source.length; cursor += 1) { + if (source[cursor] === "\\") { cursor += 1; continue; } + if (source[cursor] === '"') { index = cursor; break; } + if (source.startsWith("$(", cursor)) { + const end = addParenthesized(cursor, source.startsWith("$((", cursor) ? "arithmetic" : "command"); + cursor = end - 1; + } else if (source[cursor] === "`") { + cursor = addBacktick(cursor) - 1; + } + if (cursor + 1 >= source.length) throw new Error(`revision-state shell policy found an unclosed double quote at offset ${index}`); + } + continue; + } + if (!arithmeticContext && (source.startsWith("<(", index) || source.startsWith(">(", index))) { + index = addParenthesized(index, "process") - 1; + continue; + } + if (source.startsWith("$(", index)) { + const arithmetic = source.startsWith("$((", index); + index = addParenthesized(index, arithmetic ? "arithmetic" : "command") - 1; + continue; + } + if (source[index] === "`") index = addBacktick(index) - 1; + } + return bodies; +} + +function removeBacktickBodyEscapes(source) { + let result = ""; + for (let index = 0; index < source.length; index += 1) { + if (source[index] === "\\" && index + 1 < source.length && ["$", "`", "\\", "\n"].includes(source[index + 1])) { + if (source[index + 1] !== "\n") result += source[index + 1]; + index += 1; + } else { + result += source[index]; + } + } + return result; +} + +function shellJqRevisionAccess(source, budget = { characters: 0 }, depth = 0, arithmeticContext = false) { + if (depth > 32) throw new Error("revision-state executable shell substitution nesting limit exceeded"); + budget.characters += source.length; + if (budget.characters > 500_000) throw new Error("revision-state executable shell substitution size limit exceeded"); + if (!arithmeticContext) { + for (const words of shellCommandWords(source)) { + const arguments_ = shellJqArguments(words); + const filter = arguments_ && jqInvocation(arguments_).filter; + if (filter) { + const analysis = jqRevisionAnalysis(filter.value); + if (analysis === "violation") return true; + if (analysis === "unsupported") throw new Error("revision-state jq target grammar is unsupported"); + } + } + } + for (const body of shellExecutableSubstitutionBodies(source, arithmeticContext)) { + const nestedSource = body.kind === "backtick" ? removeBacktickBodyEscapes(body.source) : body.source; + if (shellJqRevisionAccess(nestedSource, budget, depth + 1, body.kind === "arithmetic")) return true; + } + return false; +} + +function nonJsAnalysisSource(source, label) { + const lower = label.toLowerCase(); + if (lower.endsWith(".sh")) return maskShellSource(maskShellJqLiteralArguments(maskQuotedShellHeredocBodies(source, label))); + if (lower.endsWith(".ps1")) return maskPowerShellSource(source); + return maskUnknownSource(source); +} + +function revisionStateAstNodes(source, label) { + const knownKind = scriptKindFor(label); + const caseInsensitive = label.toLowerCase().endsWith(".ps1"); + const analyzed = knownKind === undefined ? nonJsAnalysisSource(source, label) : source; + const file = ts.createSourceFile(label, analyzed, ts.ScriptTarget.Latest, true, knownKind ?? ts.ScriptKind.TS); + const matches = []; + function visit(node) { + if (ts.isPropertyAccessExpression(node) && node.name.text === "state" && isRevisionExpression(node.expression, caseInsensitive)) { + matches.push(node); + } else if (ts.isElementAccessExpression(node) && isRevisionExpression(node.expression, caseInsensitive) && + node.argumentExpression && propertyNameText(node.argumentExpression, caseInsensitive) === "state") { + matches.push(node); + } else if ((ts.isVariableDeclaration(node) || ts.isParameter(node)) && node.initializer && + isRevisionExpression(node.initializer, caseInsensitive) && ts.isObjectBindingPattern(node.name) && + objectBindingHasState(node.name, caseInsensitive)) { + matches.push(node); + } else if (ts.isBinaryExpression(node) && node.operatorToken.kind === ts.SyntaxKind.EqualsToken && + isRevisionExpression(node.right, caseInsensitive)) { + const assignmentTarget = unwrapExpression(node.left); + if (ts.isObjectLiteralExpression(assignmentTarget) && objectLiteralHasState(assignmentTarget, caseInsensitive)) matches.push(node); + } else if (ts.isPropertyAssignment(node) && propertyNameText(node.name, caseInsensitive) === "revision" && + ts.isObjectLiteralExpression(node.initializer) && objectLiteralHasState(node.initializer, caseInsensitive)) { + matches.push(node); + } + ts.forEachChild(node, visit); + } + visit(file); + return matches; +} + + +function yamlScalarRevisionAccess(value) { + return /(?:^|[\s;=,(])(?:revision|workspaceRevision|selectedWorkspace)\s*(?:\.\s*state|\[\s*["']?state["']?\s*\])(?:$|[\s;,)])/u.test(value); +} + +function validateYamlRevisionState(source, label) { + const documents = parseAllDocuments(source, { uniqueKeys: true, merge: true }); + for (const document of documents) { + if (document.errors.length > 0) throw new Error(`${label}: revision-state policy cannot parse YAML`); + const walkAst = (node) => { + if (isScalar(node)) { + if (node.type === "PLAIN" && typeof node.value === "string" && yamlScalarRevisionAccess(node.value)) throw new Error(`${label}: forbidden revision-state access`); + return; + } + if (isSeq(node)) { for (const item of node.items) walkAst(item); return; } + if (isMap(node)) { for (const pair of node.items) walkAst(pair.value); } + }; + walkAst(document.contents); + let resolved; + try { resolved = document.toJS({ mapAsMap: true, maxAliasCount: 50 }); } + catch { throw new Error(`${label}: revision-state YAML alias resolution failed`); } + const seen = new WeakSet(); + const walkResolved = (value) => { + if (!value || typeof value !== "object" || seen.has(value)) return; + seen.add(value); + if (value instanceof Map) { + for (const [key, child] of value) { + if (revisionIdentifiers.has(String(key)) && child instanceof Map && child.has("state")) throw new Error(`${label}: forbidden revision-state access`); + walkResolved(child); + } + } else if (Array.isArray(value)) { for (const child of value) walkResolved(child); } + }; + walkResolved(resolved); + } +} + +function validateRevisionState(source, label) { + const lower = label.toLowerCase(); + if (/\.(?:yaml|yml)(?:\.example)?$/u.test(lower)) { + validateYamlRevisionState(source, label); + return; + } + if (lower.endsWith(".sh")) { + const active = maskQuotedShellHeredocBodies(source, label); + try { + if (shellJqRevisionAccess(active) || shellAssociativeRevisionAccess(active)) throw new Error("forbidden revision-state access"); + } catch (error) { + throw new Error(`${label}: ${error instanceof Error ? error.message : String(error)}`); + } + } + if (lower.endsWith(".py") || lower.endsWith(".pyw")) throw new Error(`${label}: revision-state Python input was not batched`); + const matches = revisionStateAstNodes(source, label); + if (matches.length === 0) return; + const historical = 'revision.state !== "operational"'; + const historicalCount = source.split(historical).length - 1; + const match = matches[0]; + if (label === "backend/src/workspaces/registry.ts" && matches.length === 1 && + match.getText() === "revision.state" && match.parent?.getText() === historical && + historicalCount === 1) return; + throw new Error(`${label}: forbidden revision-state access`); +} + + +const pythonHelper = fileURLToPath(new URL("./revision_state_policy.py", import.meta.url)); + +function validatePythonRevisionStates(records) { + if (!Array.isArray(records) || records.length === 0) return; + let stdout; + try { + stdout = execFileSync("python3", ["-I", "-B", pythonHelper], { + input: JSON.stringify(records), encoding: "utf8", timeout: 5_000, maxBuffer: 4 * 1024 * 1024, + env: { + PATH: process.env.PATH ?? "/usr/bin:/bin", + LANG: "C.UTF-8", + LC_ALL: "C.UTF-8", + PYTHONDONTWRITEBYTECODE: "1", + }, + stdio: ["pipe", "pipe", "pipe"], + }); + } catch (error) { + const detail = error?.stderr?.toString().trim(); + throw new Error(`revision-state helper failed${detail ? `: ${detail}` : ""}`); + } + let result; + try { result = JSON.parse(stdout); } + catch { throw new Error("revision-state helper failed: invalid JSON output"); } + if (!result || !Array.isArray(result.violations) || result.violations.some((label) => typeof label !== "string")) throw new Error("revision-state helper failed: invalid result shape"); + if (result.violations.length > 0) throw new Error(`${result.violations[0]}: forbidden revision-state access`); +} + +export { validatePythonRevisionStates, validateRevisionState }; diff --git a/backend/scripts/revision-state-policy.test.mjs b/backend/scripts/revision-state-policy.test.mjs new file mode 100644 index 00000000..b8be5302 --- /dev/null +++ b/backend/scripts/revision-state-policy.test.mjs @@ -0,0 +1,273 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; + +import { validatePythonRevisionStates, validateRevisionState } from "./revision-state-policy.mjs"; + +function rejects(source, path) { + assert.throws(() => validateRevisionState(source, path), /revision-state/, source); +} +function passes(source, path) { + assert.doesNotThrow(() => validateRevisionState(source, path)); +} + +test("PowerShell scoped and braced revision variables remain executable", () => { + rejects('${revision}.state', "scripts/direct.ps1"); + rejects('${workspaceRevision}["state"]', "scripts/bracket.ps1"); + rejects('Write-Output "$(${selectedWorkspace}.state)"', "scripts/subexpression.ps1"); + rejects('${script:revision}.state', "scripts/scoped.ps1"); + rejects('${global:workspaceRevision}["state"]', "scripts/global.ps1"); + for (const source of [ + '$REVISION.STATE', + '${Revision}.state', + '$WORKSPACEREVISION["STATE"]', + '${GLOBAL:SELECTEDWORKSPACE}.State', + '$REVISION["ST" + "ATE"]', + '${Revision}[("sT" + "AtE")]', + '$record.REVISION.STATE', + '$record.WORKSPACEREVISION["STATE"]', + '$record["REVISION"].STATE', + ]) rejects(source, "scripts/case.ps1"); + passes('REVISION.STATE; revision.STATE; revision["ST" + "ATE"]; record.REVISION.STATE; record["REVISION"].state', "backend/src/case-sensitive.ts"); +}); + +test("Bash jq command forms and associative revision parameters are active", () => { + for (const source of [ + "value=$(jq -r '.revision.state' snapshot.json)", + "value=$(command jq -r '.workspaceRevision.state' snapshot.json)", + "/usr/bin/jq --arg x y '.selectedWorkspace.state' snapshot.json", + "env -i MODE=x jq -- '.revision.state' snapshot.json", + "env -u MODE /opt/tools/jq -r '.workspaceRevision.state' snapshot.json", + "echo safe\nvalue=`jq -r '.selectedWorkspace.state' snapshot.json`", + "sudo -u nobody /usr/bin/jq -r '.revision.state' snapshot.json", + "nice -n 5 jq -r '.workspaceRevision.state' snapshot.json", + "time jq -r '.selectedWorkspace.state' snapshot.json", + "printf x | xargs -n 1 jq -r '.revision.state'", + "timeout -k 2 5 jq -r '.revision.state' snapshot.json", + `stdbuf -o L jq -r '.["workspaceRevision"].state' snapshot.json`, + `stdbuf -oL jq -r '.["selectedWorkspace"]["state"]' snapshot.json`, + `nohup jq -r '.revision["state"]' snapshot.json`, + "< snapshot.json jq -r '.workspaceRevision.state'", + "sudo MODE=x jq -r '.selectedWorkspace.state' snapshot.json", + String.raw`jq -r '"x \(.revision.state)"' snapshot.json`, + "jq < snapshot.json -r '.revision.state'", + "jq -r < snapshot.json '.workspaceRevision.state'", + "jq --arg note safe < snapshot.json '.selectedWorkspace.state'", + "jq -r '.revision?.state' snapshot.json", + `jq -r '.["workspaceRevision"]?["state"]' snapshot.json`, + `jq -r '.["revision"]?.["state"]' snapshot.json`, + `jq -r '."revision".state' snapshot.json`, + `jq -r '."workspaceRevision"."state"' snapshot.json`, + "jq -r '$revision.state' snapshot.json", + "jq -r '($selectedWorkspace).state' snapshot.json", + `${"env ".repeat(17)}jq -r '.revision.state' snapshot.json`, + "jq/dev/null -r '.workspaceRevision.state' snapshot.json", + "{ jq -r '.selectedWorkspace.state' snapshot.json; }", + "! jq -r '.revision.state' snapshot.json", + "if jq -r '.workspaceRevision.state' snapshot.json; then :; fi", + "if false; then :; elif jq -r '.selectedWorkspace.state' snapshot.json; then :; fi", + "while false; do jq -r '.revision.state' snapshot.json; done", + "until false; do jq -r '.workspaceRevision.state' snapshot.json; done", + "jq -r '.revision | .state' snapshot.json", + "jq -r '(.workspaceRevision | .state)' snapshot.json", + `jq -r '.["revi" + "sion"].state' snapshot.json`, + `jq -r '.["workspace" + "Revision"]["st" + "ate"]' snapshot.json`, + "jq 2>&1 -r '.revision.state' snapshot.json", + "jq 2>&- -r '.workspaceRevision.state' snapshot.json", + "jq 0<&3 -r '.selectedWorkspace.state' snapshot.json", + "jq &>/dev/null -r '.revision.state' snapshot.json", + "jq &>>log -r '.workspaceRevision.state' snapshot.json", + "jq >|output -r '.selectedWorkspace.state' snapshot.json", + "jq {fd}>output -r '.revision.state' snapshot.json", + "exec jq -r '.workspaceRevision.state' snapshot.json", + "coproc jq -r '.selectedWorkspace.state' snapshot.json", + "coproc worker jq -r '.revision.state' snapshot.json", + "coproc worker >out jq -r '.workspaceRevision.state' snapshot.json", + "coproc worker 2>/dev/null jq -r '.selectedWorkspace.state' snapshot.json", + "coproc worker VAR=x jq -r '.revision.state' snapshot.json", + `jq -r '.["revi" + ("sion")].state' snapshot.json`, + "jq -r '.revision | . | .state' snapshot.json", + "jq -r '(.workspaceRevision | (.) | .state)' snapshot.json", + "jq -r '.revision | select(.) | .state' snapshot.json", + "jq -r '.workspaceRevision | {value:.state}' snapshot.json", + "jq -r '.selectedWorkspace | [.state]' snapshot.json", + "jq -r '.revision + .state' snapshot.json", + "jq < <(cat snapshot.json) -r '.revision.state'", + "jq < <(cat <(printf snapshot.json)) -r '.workspaceRevision.state'", + "jq > >(cat >/dev/null) -r '.selectedWorkspace.state' snapshot.json", + `jq < <(printf '%s\n' "$((1 + (2)))") -r '.revision.state'`, + "jq < <(cat snapshot.json -r '.revision.state'", + `${"<(".repeat(65)}echo snapshot${")".repeat(65)} jq -r '.workspaceRevision.state'`, + "cat <(jq -r '.revision.state' snapshot.json)", + "cat snapshot.json > >(jq -r '.workspaceRevision.state')", + `echo "$(jq -r '.selectedWorkspace.state' snapshot.json)"`, + "value=$(jq -r '.revision.state' snapshot.json)", + `echo "\`jq -r '.workspaceRevision.state' snapshot.json\`"`, + `echo "$(cat <(jq -r '.selectedWorkspace.state' snapshot.json))"`, + `${"$(".repeat(33)}jq -r '.revision.state' snapshot.json${")".repeat(33)}`, + `echo "$(( $(jq -r '.revision.state' snapshot.json) + 0 ))"`, + "echo \"$(( `jq -r '.workspaceRevision.state' snapshot.json` + 0 ))\"", + "echo `echo \\`jq -r '.selectedWorkspace.state' snapshot.json\\``", + "echo \"`echo \\`jq -r '.revision.state' snapshot.json\\``\"", + `${"$(( ".repeat(33)}$(jq -r '.workspaceRevision.state' snapshot.json)${" + 0 ))".repeat(33)}`, + 'old=${revision["state"]}', + "old=${workspaceRevision[state]}", + "old=${revision[state]:-missing}", + "old=${workspaceRevision['state']:=missing}", + "old=${selectedWorkspace[state]:1:2}", + ]) rejects(source, "scripts/policy.sh"); + const jqFilters = [ + ".revision?.state", '.["revision"]?.["state"]', '."revision".state', + '."workspaceRevision"."state"', "(.revision).state", "$revision.state", + ".revision | .state", "(.workspaceRevision | .state)", + '.["revi" + "sion"].state', '.["revi" + ("sion")].state', + ".revision | . | .state", "(.workspaceRevision | (.) | .state)", + ".revision | select(.) | .state", ".workspaceRevision | {value:.state}", + '.revision | ["state"]', '(.workspaceRevision | (["state"]))', ".selectedWorkspace | $state", + ]; + for (const filter of jqFilters) { + const compiled = spawnSync("jq", ["-n", "--argjson", "revision", "{}", "--arg", "state", "x", filter], { encoding: "utf8" }); + if (compiled.error?.code !== "ENOENT") assert.equal(compiled.status, 0, `${filter}: ${compiled.stderr}`); + } + passes("cat <<'EOF'\nrevision.state\nEOF\n", "scripts/literal.sh"); + passes("echo '${revision[state]}'\n", "scripts/single-quoted-parameter.sh"); + passes(`echo "<(jq '.revision.state')"\n`, "scripts/literal-process-text.sh"); + passes(`echo "ordinary jq '.workspaceRevision.state' text"\n`, "scripts/literal-jq-text.sh"); + passes(`echo '$(jq -r ".selectedWorkspace.state")'\n`, "scripts/single-quoted-command-text.sh"); + passes(`# profile's harmless note +printf 'ok\n' +`, "scripts/comment-apostrophe.sh"); + passes(`cat <( # profile's harmless note + printf 'snapshot\n' +) +`, "scripts/substitution-comment-apostrophe.sh"); + passes(`echo "$(( 1 + (2 * 3) ))"\n`, "scripts/literal-arithmetic.sh"); + passes(`echo $(( jq + revision + state ))\n`, "scripts/arithmetic-identifiers.sh"); + passes("echo \\`jq -r '.revision.state' snapshot.json\\`\n", "scripts/escaped-literal-backticks.sh"); + passes("echo \"\\`jq -r '.workspaceRevision.state' snapshot.json\\`\"\n", "scripts/double-quoted-literal-backticks.sh"); + passes("echo `printf '%s' '\\`jq -r \".selectedWorkspace.state\" snapshot.json\\`'`\n", "scripts/quoted-nonexecuting-nested-backticks.sh"); + passes("jq --arg note 'revision.state' '.' file\n", "scripts/jq-arg.sh"); + passes(`jq --argjson note '"revision.state"' '.' file +`, "scripts/jq-argjson.sh"); + passes("jq -r '.' revision.state.json\n", "scripts/jq-file.sh"); + passes("jq -r '.revision.id' snapshot.json\n", "scripts/jq-simple-non-state.sh"); + passes("jq -f revision.state.jq snapshot.json\n", "scripts/jq-from-file.sh"); + passes("jq --from-file workspaceRevision.state.jq snapshot.json\n", "scripts/jq-long-from-file.sh"); + passes(`jq -r '"revision.state"' snapshot.json +`, "scripts/jq-string.sh"); + passes(`jq -r '{note:"selectedWorkspace.state"}' snapshot.json +`, "scripts/jq-object.sh"); + passes(`jq -r '.revision | "state"' snapshot.json +`, "scripts/jq-pipe-literal-right.sh"); + passes(`jq -r '"revision" | .state' snapshot.json +`, "scripts/jq-pipe-literal-left.sh"); + passes(`jq -r '.revision | ["state"]' snapshot.json +`, "scripts/jq-pipe-array.sh"); + passes(`jq -r '(.workspaceRevision | (["state"]))' snapshot.json +`, "scripts/jq-pipe-parenthesized-array.sh"); + passes(`jq --arg state x '.selectedWorkspace | $state' snapshot.json +`, "scripts/jq-pipe-variable.sh"); + for (const opener of ["'E'OF", "E'OF'", "E\\OF"]) { + passes(`cat <<${opener} +revision.state +EOF +`, "scripts/partial-quoted-heredoc.sh"); + } + rejects("cat <<'E'OF\nrevision.state\nEOF\nworkspaceRevision.state\n", "scripts/after-heredoc.sh"); + rejects("cat <<'EOF'\nrevision.state\n", "scripts/unclosed-heredoc.sh"); + rejects(`echo "<<'EOF'" +jq -r '.revision.state' snapshot.json +`, "scripts/quoted-opener.sh"); +}); + +test("Python helper resolves active AST expressions and static format bindings", () => { + const rejectsPython = (source) => assert.throws( + () => validatePythonRevisionStates([{ source, label: "backend/scripts/policy.py" }]), + /revision-state/, + ); + for (const source of [ + "old = revision.state", + 'old = workspaceRevision["state"]', + 'old = record["selectedWorkspace"].state', + 'old = f"{revision.state}"', + '"{revision.state}".format(value)', + '"{0.state}".format(revision)', + '"{0[state]}".format(workspaceRevision)', + '"{item.state}".format(item=selectedWorkspace)', + '"{item[state]}".format_map({"item": revision})', + '("{0.state}").format(revision)', + '"{0:{1.state}}".format(value, revision)', + 'old = revision["st" + "ate"]', + 'old = record["revi" + "sion"].state', + 'old = revision[f"state"]', + 'old = record[f"revision"].state', + `old = revision[f"st{'a'}te"]`, + `old = revision[f"{'state'}"]`, + `old = record[f"revi{'sion'}"].state`, + `old = revision[f"{'st' + 'ate'}"]`, + `old = record[f"{'revi' + 'sion'}"].state`, + `old = revision[f"{'state':s}"]`, + '"{0.state}".format(*[revision])', + '"{0[state]}".format(*(revision,))', + '"{1[state]}".format(*[other, workspaceRevision])', + '"{item.state}".format(**{"item": selectedWorkspace})', + '"{item[state]}".format_map({**{"item": revision}})', + '"{.state}".format(revision)', + '"{[state]}".format(revision)', + '"{:{.state}}".format(value, revision)', + '"{.name} {[state]}".format(other, revision)', + '"{item.state}".format(item=revision, **values)', + ]) rejectsPython(source); + validatePythonRevisionStates([ + { source: 'text = "{revision.state}"', label: "backend/scripts/literal.py" }, + { source: 'text = "{{revision.state}}".format(value)', label: "backend/scripts/escaped.py" }, + { source: 'text = "{0.state}".format(other)', label: "backend/scripts/unrelated.py" }, + { source: 'old = revision[f"st{suffix}"]', label: "backend/scripts/dynamic-key.py" }, + { source: 'text = "{.name} {[state]}".format(other, other)', label: "backend/scripts/multi-auto.py" }, + { source: 'text = "{item.state}".format(**values)', label: "backend/scripts/dynamic-map.py" }, + ]); + const hostile = mkdtempSync(join(tmpdir(), "revision-policy-hostile-")); + writeFileSync(join(hostile, "json.py"), "raise RuntimeError('shadowed')\n"); + const previousPythonPath = process.env.PYTHONPATH; + try { + process.env.PYTHONPATH = hostile; + validatePythonRevisionStates([{ source: "value = 1", label: "backend/scripts/isolated.py" }]); + } finally { + if (previousPythonPath === undefined) delete process.env.PYTHONPATH; + else process.env.PYTHONPATH = previousPythonPath; + rmSync(hostile, { recursive: true, force: true }); + } + assert.throws( + () => validatePythonRevisionStates([{ source: 'revision[f"{1:.1000000000f}"]', label: "backend/scripts/oversized.py" }]), + /revision-state helper failed/, + ); + validatePythonRevisionStates([{ source: 'revision[f"{1:04d}"]', label: "backend/scripts/small-format.py" }]); + assert.throws( + () => validatePythonRevisionStates([{ source: "def broken(", label: "backend/scripts/invalid.py" }]), + /revision-state helper failed/, + ); +}); + +test("YAML mappings and only active plain scalar expressions are rejected", () => { + for (const source of [ + "value: { revision: { state: old } }\n", + "value:\n workspaceRevision:\n state: old\n", + 'items:\n - "selectedWorkspace":\n "state": old\n', + "old: selectedWorkspace.state\n", + "url: https://host/x; old: selectedWorkspace.state\n", + "saved: &saved { state: old }\nvalue: { revision: *saved }\n", + "defaults: &defaults { workspaceRevision: { state: old } }\nvalue: { <<: *defaults }\n", + ]) rejects(source, "scripts/policy.yaml"); + rejects("a: &a [x,x,x,x,x,x,x,x,x]\nb: &b [*a,*a,*a,*a,*a,*a,*a,*a,*a]\nc: [*b,*b,*b,*b,*b,*b,*b,*b,*b]\n", "scripts/alias-bomb.yaml"); + rejects("value: [\n", "scripts/invalid.yaml"); + for (const source of [ + "value: |\n revision.state\n", + "value: >\n workspaceRevision.state\n", + 'value: "selectedWorkspace.state"\n', + "url: https://host/revision.state\n", + ]) passes(source, "scripts/literal.yaml"); +}); diff --git a/backend/scripts/revision_state_policy.py b/backend/scripts/revision_state_policy.py new file mode 100644 index 00000000..f670040e --- /dev/null +++ b/backend/scripts/revision_state_policy.py @@ -0,0 +1,318 @@ +"""Semantic Python revision-state policy helper. + +Reads one JSON array of ``{"label": str, "source": str}`` records from stdin and +writes ``{"violations": [label, ...]}``. Invalid input or Python source is fatal. +""" + +from __future__ import annotations + +import ast +import json +import re +import string +import sys +from itertools import pairwise +from typing import Any + +TARGETS = frozenset({"revision", "workspaceRevision", "selectedWorkspace"}) +_FORMATTER = string.Formatter() + + +MAX_STATIC_TEXT = 4_096 +MAX_FORMAT_SPEC = 256 +MAX_STATIC_DEPTH = 64 +_UNRESOLVED = object() + + +def _bounded_text(value: str) -> str: + if len(value) > MAX_STATIC_TEXT: + raise ValueError("static text exceeds revision policy limit") + return value + + +def _static_scalar(node: ast.expr, depth: int) -> object: + if depth > MAX_STATIC_DEPTH: + raise ValueError("static expression nesting exceeds revision policy limit") + if isinstance(node, ast.Constant) and type(node.value) in { + str, + int, + float, + complex, + bool, + type(None), + }: + if isinstance(node.value, str): + _bounded_text(node.value) + if isinstance(node.value, int) and node.value.bit_length() > MAX_STATIC_TEXT * 4: + raise ValueError("static integer exceeds revision policy limit") + return node.value + if isinstance(node, ast.BinOp) and isinstance(node.op, ast.Add): + left = _static_scalar(node.left, depth + 1) + right = _static_scalar(node.right, depth + 1) + if left is _UNRESOLVED or right is _UNRESOLVED: + return _UNRESOLVED + try: + result = left + right + except TypeError: + return _UNRESOLVED + if type(result) not in {str, int, float, complex, bool}: + return _UNRESOLVED + if isinstance(result, str): + _bounded_text(result) + if isinstance(result, int) and result.bit_length() > MAX_STATIC_TEXT * 4: + raise ValueError("static integer exceeds revision policy limit") + return result + if isinstance(node, ast.JoinedStr): + result = _static_key(node, depth + 1) + return _UNRESOLVED if result is None else result + return _UNRESOLVED + + +def _validate_format_spec(format_spec: str) -> None: + if len(format_spec) > MAX_FORMAT_SPEC: + raise ValueError("static format specification exceeds revision policy limit") + for digits in re.findall(r"[0-9]+", format_spec): + if len(digits) > 6 or int(digits) > MAX_STATIC_TEXT: + raise ValueError("static format width or precision exceeds revision policy limit") + + +def _static_key(node: ast.expr, depth: int = 0) -> str | None: + if depth > MAX_STATIC_DEPTH: + raise ValueError("static key nesting exceeds revision policy limit") + if isinstance(node, ast.Constant) and isinstance(node.value, str): + return _bounded_text(node.value) + if isinstance(node, ast.BinOp) and isinstance(node.op, ast.Add): + left = _static_key(node.left, depth + 1) + right = _static_key(node.right, depth + 1) + return None if left is None or right is None else _bounded_text(left + right) + if isinstance(node, ast.JoinedStr): + pieces = [] + length = 0 + for value in node.values: + if isinstance(value, ast.Constant) and isinstance(value.value, str): + piece = value.value + elif isinstance(value, ast.FormattedValue): + scalar = _static_scalar(value.value, depth + 1) + if scalar is _UNRESOLVED: + return None + format_spec = "" if value.format_spec is None else _static_key(value.format_spec, depth + 1) + if format_spec is None: + return None + _validate_format_spec(format_spec) + try: + if value.conversion == ord("s"): + scalar = str(scalar) + elif value.conversion == ord("r"): + scalar = repr(scalar) + elif value.conversion == ord("a"): + scalar = ascii(scalar) + elif value.conversion != -1: + return None + piece = format(scalar, format_spec) + except (TypeError, ValueError): + return None + else: + return None + length += len(piece) + if length > MAX_STATIC_TEXT: + raise ValueError("static formatted key exceeds revision policy limit") + pieces.append(piece) + return "".join(pieces) + return None + + +def _is_revision_expr(node: ast.expr) -> bool: + if isinstance(node, ast.Name): + return node.id in TARGETS + if isinstance(node, ast.Attribute): + return node.attr in TARGETS + if isinstance(node, ast.Subscript): + return _static_key(node.slice) in TARGETS + return False + + +def _is_state_access(node: ast.AST) -> bool: + if isinstance(node, ast.Attribute): + return node.attr == "state" and _is_revision_expr(node.value) + if isinstance(node, ast.Subscript): + return _static_key(node.slice) == "state" and _is_revision_expr(node.value) + return False + + +def _static_sequence(node: ast.expr) -> list[ast.expr] | None: + if not isinstance(node, (ast.List, ast.Tuple)): + return None + result: list[ast.expr] = [] + for element in node.elts: + if isinstance(element, ast.Starred): + nested = _static_sequence(element.value) + if nested is None: + return None + result.extend(nested) + else: + result.append(element) + return result + + +def _static_mapping(node: ast.expr) -> dict[str, ast.expr] | None: + if not isinstance(node, ast.Dict): + return None + result: dict[str, ast.expr] = {} + for key, value in zip(node.keys, node.values, strict=True): + if key is None: + nested = _static_mapping(value) + if nested is None: + return None + result.update(nested) + elif (name := _static_key(key)) is not None: + result[name] = value + else: + return None + return result + + +def _format_bindings(call: ast.Call, method: str) -> dict[str | int, ast.expr]: + if method == "format": + bindings: dict[str | int, ast.expr] = {} + position = 0 + positional_known = True + for argument in call.args: + if isinstance(argument, ast.Starred): + expanded = _static_sequence(argument.value) + if expanded is None: + positional_known = False + continue + if positional_known: + for value in expanded: + bindings[position] = value + position += 1 + elif positional_known: + bindings[position] = argument + position += 1 + for keyword in call.keywords: + if keyword.arg is not None: + # An explicit keyword remains bound even beside **dynamic; a duplicate is TypeError. + bindings[keyword.arg] = keyword.value + else: + expanded = _static_mapping(keyword.value) + if expanded is not None: + bindings.update(expanded) + return bindings + if len(call.args) != 1 or call.keywords: + return {} + return _static_mapping(call.args[0]) or {} + + +def _field_accesses_state( + field_name: str, bindings: dict[str | int, ast.expr], automatic_index: int | None = None +) -> bool: + root_match = re.match(r"(?:[0-9]+|[A-Za-z_][A-Za-z0-9_]*)", field_name) + if root_match is None: + if automatic_index is None or not field_name.startswith((".", "[")): + return False + root: str | int = automatic_index + cursor = 0 + else: + root_text = root_match.group(0) + root = int(root_text) if root_text.isdigit() else root_text + cursor = root_match.end() + steps: list[tuple[bool, str]] = [] + while cursor < len(field_name): + if field_name[cursor] == ".": + match = re.match(r"[A-Za-z_][A-Za-z0-9_]*", field_name[cursor + 1 :]) + if match is None: + return False + steps.append((True, match.group(0))) + cursor += len(match.group(0)) + 1 + elif field_name[cursor] == "[": + close = field_name.find("]", cursor + 1) + if close < 0: + return False + steps.append((False, field_name[cursor + 1 : close])) + cursor = close + 1 + else: + return False + if steps: + first_step = str(steps[0][1]) + if str(root) in TARGETS and first_step == "state": + return True + bound = bindings.get(root) + if bound is not None and _is_revision_expr(bound) and first_step == "state": + return True + names = [str(root), *(str(key) for _is_attr, key in steps)] + return any(left in TARGETS and right == "state" for left, right in pairwise(names)) + + +def _format_call_violation(node: ast.Call) -> bool: + function = node.func + if not isinstance(function, ast.Attribute) or function.attr not in {"format", "format_map"}: + return False + if not isinstance(function.value, ast.Constant) or not isinstance(function.value.value, str): + return False + bindings = _format_bindings(node, function.attr) + numbering: dict[str, int | str | None] = {"next": 0, "mode": None} + visited = 0 + + def analyze_template(template: str) -> bool: + nonlocal visited + visited += 1 + if visited > 1_000: + raise ValueError("format specification nesting exceeds policy limit") + for _literal, field_name, format_spec, _conversion in _FORMATTER.parse(template): + automatic_index = None + if field_name is not None: + root_match = re.match(r"(?:[0-9]+|[A-Za-z_][A-Za-z0-9_]*)", field_name) + automatic = field_name == "" or root_match is None and field_name.startswith((".", "[")) + manual = root_match is not None and root_match.group(0).isdigit() + if automatic: + if numbering["mode"] == "manual": + raise ValueError("cannot switch from manual to automatic field numbering") + numbering["mode"] = "automatic" + automatic_index = int(numbering["next"]) + numbering["next"] = automatic_index + 1 + elif manual: + if numbering["mode"] == "automatic": + raise ValueError("cannot switch from automatic to manual field numbering") + numbering["mode"] = "manual" + if _field_accesses_state(field_name, bindings, automatic_index): + return True + if format_spec and analyze_template(format_spec): + return True + return False + + return analyze_template(function.value.value) + + +def has_revision_state(source: str, label: str = "") -> bool: + tree = ast.parse(source, filename=label, mode="exec") + return any(_is_state_access(node) or (isinstance(node, ast.Call) and _format_call_violation(node)) for node in ast.walk(tree)) + + +def analyze_batch(records: Any) -> list[str]: + if not isinstance(records, list): + raise TypeError("input must be a JSON array") + violations = [] + for record in records: + if not isinstance(record, dict) or set(record) != {"label", "source"}: + raise TypeError("each record must contain exactly label and source") + label, source = record["label"], record["source"] + if not isinstance(label, str) or not isinstance(source, str): + raise TypeError("label and source must be strings") + if has_revision_state(source, label): + violations.append(label) + return violations + + +def main() -> int: + try: + records = json.load(sys.stdin) + json.dump({"violations": analyze_batch(records)}, sys.stdout, ensure_ascii=False) + sys.stdout.write("\n") + return 0 + except Exception as error: # noqa: BLE001 - protocol boundary must fail closed + print(f"python revision-state helper failed: {error}", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/backend/scripts/test_revision_state_policy.py b/backend/scripts/test_revision_state_policy.py new file mode 100644 index 00000000..e5645480 --- /dev/null +++ b/backend/scripts/test_revision_state_policy.py @@ -0,0 +1,90 @@ +import importlib.util +import tracemalloc +import unittest +from pathlib import Path +from unittest.mock import patch + +_HELPER = Path(__file__).with_name("revision_state_policy.py") +_SPEC = importlib.util.spec_from_file_location("revision_state_policy", _HELPER) +assert _SPEC is not None and _SPEC.loader is not None +_MODULE = importlib.util.module_from_spec(_SPEC) +_SPEC.loader.exec_module(_MODULE) +analyze_batch = _MODULE.analyze_batch +has_revision_state = _MODULE.has_revision_state + + +class RevisionStatePolicyTests(unittest.TestCase): + def test_ast_access_and_f_strings(self): + for source in ( + "old = revision.state", + 'old = workspaceRevision["state"]', + 'old = record["selectedWorkspace"].state', + 'old = f"{revision.state}"', + 'old = revision["st" + "ate"]', + 'old = record["revi" + "sion"].state', + 'old = revision[f"state"]', + 'old = record[f"revision"].state', + "old = revision[f\"st{'a'}te\"]", + "old = revision[f\"{'state'}\"]", + "old = record[f\"revi{'sion'}\"].state", + "old = revision[f\"{'st' + 'ate'}\"]", + "old = record[f\"{'revi' + 'sion'}\"].state", + "old = revision[f\"{'state':s}\"]", + ): + with self.subTest(source=source): + self.assertTrue(has_revision_state(source)) + + def test_static_format_bindings(self): + for source in ( + '"{0.state}".format(revision)', + '"{0[state]}".format(workspaceRevision)', + '"{item.state}".format(item=selectedWorkspace)', + '"{item[state]}".format_map({"item": revision})', + '("{0.state}").format(revision)', + '"{0:{1.state}}".format(value, revision)', + '"{0.state}".format(*[revision])', + '"{0[state]}".format(*(revision,))', + '"{1[state]}".format(*[other, workspaceRevision])', + '"{item.state}".format(**{"item": selectedWorkspace})', + '"{item[state]}".format(**{"outer": other, **{"item": revision}})', + '"{item.state}".format_map({**{"item": workspaceRevision}})', + '"{.state}".format(revision)', + '"{[state]}".format(revision)', + '"{:{.state}}".format(value, revision)', + '"{.name} {[state]}".format(other, revision)', + '"{item.state}".format(item=revision, **values)', + ): + with self.subTest(source=source): + self.assertTrue(has_revision_state(source)) + self.assertFalse(has_revision_state('"{0.state}".format(other)')) + # Dynamic unpacking is intentionally unresolved rather than guessed. + self.assertFalse(has_revision_state('"{0.state}".format(*values)')) + self.assertFalse(has_revision_state('"{.name} {[state]}".format(other, other)')) + self.assertFalse(has_revision_state('"{item.state}".format(**values)')) + # FormattedValue keys are dynamic and are not treated as static strings. + self.assertFalse(has_revision_state('revision[f"st{suffix}"]')) + + def test_literals_are_not_active(self): + self.assertFalse(has_revision_state('text = "{revision.state}"')) + self.assertFalse(has_revision_state('text = "{{revision.state}}".format(value)')) + + def test_oversized_static_format_fails_before_formatting(self): + tracemalloc.start() + with patch("builtins.format") as format_mock: + with self.assertRaisesRegex(ValueError, "width or precision"): + has_revision_state('revision[f"{1:.1000000000f}"]') + format_mock.assert_not_called() + _current, peak = tracemalloc.get_traced_memory() + tracemalloc.stop() + self.assertLess(peak, 1_000_000) + self.assertFalse(has_revision_state('revision[f"{1:04d}"]')) + + def test_batch_contract(self): + self.assertEqual( + analyze_batch([{"label": "one.py", "source": "revision.state"}]), + ["one.py"], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/backend/scripts/verify-workspace-descriptor-files.mjs b/backend/scripts/verify-workspace-descriptor-files.mjs new file mode 100755 index 00000000..6dcebc40 --- /dev/null +++ b/backend/scripts/verify-workspace-descriptor-files.mjs @@ -0,0 +1,374 @@ +#!/usr/bin/env node +import { createHash } from "node:crypto"; +import { lstat, readFile, realpath } from "node:fs/promises"; +import { isAbsolute, relative, resolve, sep } from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; + +import { isMap, isScalar, parseAllDocuments } from "yaml"; +import { extractBashDocuments } from "./bash-heredoc.mjs"; +import { validatePythonRevisionStates, validateRevisionState } from "./revision-state-policy.mjs"; +import { parseWorkspaceYaml } from "../dist/workspaces/schema.js"; + +const scriptPath = fileURLToPath(import.meta.url); +const allowedKinds = new Set(["policy_text", "workspace_descriptor", "deployment_script"]); +// Exact-content trust exceptions. Each digest covers the raw UTF-8 bytes from the +// opener line through the closer line (including physical line endings). These +// blocks are reviewed non-workspace runtime/config generation, not semantic proof. +const reviewedExpandableBlocks = new Map([ + ["scripts/preprocess-smoke.sh", [ + { sha256: "fc530dc721c946644ab6552bbd46b7918d6c5f11f06f3495b6ea1fcda819b38d", rationale: "Generates the reviewed preprocess Compose override." }, + ]], + ["scripts/test-server-pi-state-topology.sh", [ + { sha256: "6f746f7e8442b0a6ea0e216607a6a923d94b24cd8fa17fa2d1dac56e6f14f7ef", rationale: "Generates the isolated server topology test environment." }, + ]], + ["scripts/test-vector-backup-restore-safety.sh", [ + { sha256: "40b8a10a3c06aaa98e324fbf688b7d1f5cead330d7ba7eef98e06256d412a85a", rationale: "Generates the reviewed restore safety manifest." }, + ]], + ["scripts/test-windows-clone-contract.ps1", [ + { sha256: "80f4880576a0679cb58e7b92600e7a90550c93c254553a2d4b299539f9ff0bcf", rationale: "Generates reviewed Windows clone test configuration." }, + { sha256: "6166294bdc8a8bf6436ad402bcbf7cae0f3b67dc6051cecfcca79267a62b082c", rationale: "Same reviewed block in the repository-required CRLF checkout representation." }, + { sha256: "3216201d59400ed7d1ec23e536634b8235a2e78b336e45b4dc598624920f0057", rationale: "Generates reviewed Windows clone test configuration." }, + { sha256: "a4044bb38b27e8120e90d65a0695fe0afd7757c067ae8dd67f170edf569a1de0", rationale: "Same reviewed block in the repository-required CRLF checkout representation." }, + { sha256: "5d0d1a3fc45e99b3aacaf4ee5dd09a6bee1937784375dfe4bcfaa4ae32cfb9de", rationale: "Generates reviewed Windows clone test configuration." }, + { sha256: "b903e5dae953ae1372f1a5276f12a92ed3dd632b897f3afe5e00c646d90a1b42", rationale: "Same reviewed block in the repository-required CRLF checkout representation." }, + ]], + ["scripts/unified-deployment-smoke.sh", [ + { sha256: "ca0c17d9ff8dc0fbe018fc1c5510eb33bc667a936fbe44a9be2d311390576825", rationale: "Generates reviewed Task 13 runtime configuration." }, + { sha256: "31ec00cc315b52da4a3bb6e3fba2d40aef29cdcd090bbc5d14c31f1aebbcfd04", rationale: "Generates reviewed Task 13 runtime configuration." }, + { sha256: "d92822815357ce3424e1a6eb43923df2b37b4fd93a3b5465ee9dfc69559ab0ed", rationale: "Generates reviewed Task 13 runtime configuration." }, + { sha256: "d6b8b7b951936c0452a485e9ee3b18a61251556581d6f7a2ce66f994b5700695", rationale: "Generates reviewed Task 13 runtime configuration." }, + ]], + ["scripts/vector-backup.sh", [ + { sha256: "571899db49dfdcec8107fbe1e0a86a61e7581979d3c4c248c20546843e275bcf", rationale: "Generates the reviewed backup manifest inside the helper command." }, + ]], + ["scripts/vector-restore.sh", [ + { sha256: "f04d872e556a7323583c6e620b25814fb6a8e2568a9a555623978185b473a49d", rationale: "Feeds reviewed parsed manifest values to read loops." }, + { sha256: "c6053ed44abae71ae4821b68f9a513f8070947350e30d89ae0f65bf4a48f66fd", rationale: "Feeds reviewed parsed manifest values to read loops." }, + ]], +]); + +function blockDigest(rawBlock) { + return createHash("sha256").update(rawBlock, "utf8").digest("hex"); +} + +function reviewedExpandableBlock(path, rawBlock) { + const digest = blockDigest(rawBlock); + return (reviewedExpandableBlocks.get(path) ?? []).some((review) => review.sha256 === digest); +} + +function hasAmbiguousExpansion(source, path) { + const powershell = path.endsWith(".ps1"); + for (let index = 0; index < source.length; index += 1) { + const character = source[index]; + if (powershell && character === "`") { + index += 1; + continue; + } + if (!powershell && character === "\\") { + index += 1; + continue; + } + if (character === "$" || (!powershell && character === "`")) return true; + } + return false; +} + +function physicalLines(source) { + const rawLines = source.match(/[^\n]*\n|[^\n]+$/gu) ?? []; + if (rawLines.length === 0) rawLines.push(""); + return rawLines.map((raw) => ({ raw, text: raw.replace(/\n$/u, "").replace(/\r$/u, "") })); +} +const prescribedSymbols = [ + "WorkspaceV1", "WorkspaceV2", "DeprecatedV2Descriptor", "LegacyMigrationResult", + "LegacyMigrationOptions", "WorkspaceV2MigrationInput", "migrateLegacyWorkspace", + "writeMigratedWorkspace", "migrateWorkspaceV1ToV2", "migrateWorkspaceV2ToV3", +]; +const migrationMarkers = ["migration_required", "deprecated-v2-descriptor", "migrate-legacy", "migrate-v2-qdrant"]; + +function isPolicyImplementationException(label, category) { + const implementations = new Set([ + "scripts/verify-schema-v3-only.sh", + "scripts/test-verify-schema-v3-only.sh", + "backend/scripts/verify-workspace-descriptor-files.mjs", + "backend/scripts/verify-workspace-descriptor-files.test.mjs", + "backend/scripts/revision-state-policy.mjs", + "backend/scripts/revision-state-policy.test.mjs", + "backend/scripts/bash-heredoc.mjs", + "backend/scripts/revision_state_policy.py", + "backend/scripts/test_revision_state_policy.py", + ]); + if (implementations.has(label)) return true; + if (category === "migration-marker" && new Set([ + "scripts/workspace_descriptor_doc_contract.py", + "scripts/test_workspace_descriptor_doc_contract.py", + "backend/scripts/clean-dist.test.mjs", + ]).has(label)) return true; + return false; +} + + +function validatePolicySource(source, label) { + if (!isPolicyImplementationException(label, "prescribed-symbol")) { + for (const symbol of prescribedSymbols) { + if (source.toLowerCase().includes(symbol.toLowerCase())) throw new Error(`${label}: forbidden prescribed-symbol substring: ${symbol}`); + } + } + if (!isPolicyImplementationException(label, "migration-marker")) { + for (const marker of migrationMarkers) { + if (source.toLowerCase().includes(marker.toLowerCase())) throw new Error(`${label}: forbidden migration-marker substring: ${marker}`); + } + } + if (!isPolicyImplementationException(label, "legacy-workspace")) { + for (const match of source.matchAll(/legacyworkspace/giu)) { + if (match[0] !== "legacyWorkspace") throw new Error(`${label}: forbidden legacy-workspace spelling: ${match[0]}`); + } + } + if (!/\.pyw?$/iu.test(label) && !isPolicyImplementationException(label, "revision-state")) validateRevisionState(source, label); +} + +function documentShape(document) { + const shape = { workspacePresent: false, workspaceMapping: false }; + if (!isMap(document.contents)) return shape; + for (const pair of document.contents.items) { + if (!isScalar(pair.key)) continue; + if (pair.key.value === "workspace") { + shape.workspacePresent = true; + if (isMap(pair.value)) shape.workspaceMapping = true; + } + } + return shape; +} + +function documents(source) { + try { + return parseAllDocuments(source, { uniqueKeys: true }); + } catch (error) { + throw new Error(`YAML parser failed: ${error instanceof Error ? error.message : String(error)}`); + } +} + +function validateWorkspaceSource(source, label, { requireWorkspace, expandable = false, path, rawBlock }) { + const parsed = documents(source); + const shapes = parsed.map(documentShape); + if (requireWorkspace) { + if (!shapes.some((shape) => shape.workspacePresent)) { + throw new Error(`${label}: expected a top-level workspace mapping`); + } + if (!shapes.some((shape) => shape.workspaceMapping)) { + throw new Error(`${label}: top-level workspace must be a mapping`); + } + } else { + if (expandable && hasAmbiguousExpansion(source, path) && !reviewedExpandableBlock(path, rawBlock)) { + throw new Error(`${label}: expandable block interpolation is not in the exact-content reviewed allowlist`); + } + if (shapes.some((shape) => shape.workspaceMapping)) { + throw new Error(`${label}: embedded workspace descriptor is forbidden; use a tracked workspace fixture`); + } + return false; + } + try { + parseWorkspaceYaml(source); + } catch (error) { + throw new Error(`${label}: workspace descriptor is not valid schema v3: ${error instanceof Error ? error.message : String(error)}`); + } + return true; +} + +function deploymentScriptDialect(path) { + if (path.endsWith(".sh")) return "bash"; + if (path.endsWith(".ps1")) return "powershell"; + throw new Error(`${path}: unknown deployment script dialect`); +} + + +function powerShellHereStringOpener(line, state) { + let quote = null; + for (let index = 0; index < line.length; index += 1) { + if (state.blockComment) { + const close = line.indexOf("#>", index); + if (close < 0) return null; + state.blockComment = false; + index = close + 1; + continue; + } + const character = line[index]; + if (quote === null && character === "`") { + index += 1; + continue; + } + if (quote === "'") { + if (character === "'" && line[index + 1] === "'") index += 1; + else if (character === "'") quote = null; + continue; + } + if (quote === '"') { + if (character === "`") index += 1; + else if (character === '"') quote = null; + continue; + } + if (character === "#") return null; + if (character === "<" && line[index + 1] === "#") { + state.blockComment = true; + index += 1; + continue; + } + if (character === "@" && (line[index + 1] === "'" || line[index + 1] === '"') && /^[ \t]*$/u.test(line.slice(index + 2))) return line[index + 1]; + if (character === "'" || character === '"') quote = character; + } + return null; +} + +function extractPowerShellDocuments(source, label) { + const records = physicalLines(source); + const lines = records.map((record) => record.text); + const extracted = []; + const state = { blockComment: false }; + for (let index = 0; index < lines.length; index += 1) { + const quote = powerShellHereStringOpener(lines[index], state); + if (quote === null) continue; + const delimiter = `${quote}@`; + const opener = index; + const body = []; + const start = index + 2; + let closed = false; + for (index += 1; index < lines.length; index += 1) { + if (lines[index].trimEnd() === delimiter) { + closed = true; + break; + } + body.push(lines[index]); + } + extracted.push({ + source: `${body.join("\n")}\n`, + label: `${label}:${start} PowerShell here-string${closed ? "" : " (unclosed)"}`, + expandable: quote === '"', + path: label, + rawBlock: records.slice(opener, Math.min(index + 1, records.length)).map((record) => record.raw).join(""), + }); + } + return extracted; +} + +export function extractScriptDocuments(source, label = "deployment script") { + const dialect = deploymentScriptDialect(label); + if (dialect === "bash") return extractBashDocuments(source, label); + return extractPowerShellDocuments(source, label); +} + +async function safeFile(root, path) { + if (typeof path !== "string" || path.length === 0 || isAbsolute(path) || path.includes("\\")) { + throw new Error(`unsafe verifier path: ${JSON.stringify(path)}`); + } + const segments = path.split("/"); + if (segments.some((segment) => segment === "" || segment === "." || segment === "..")) { + throw new Error(`unsafe verifier path: ${JSON.stringify(path)}`); + } + const absolute = resolve(root, ...segments); + const fromRoot = relative(root, absolute); + if (fromRoot.startsWith(`..${sep}`) || fromRoot === ".." || isAbsolute(fromRoot)) { + throw new Error(`verifier path escapes root: ${JSON.stringify(path)}`); + } + const entry = await lstat(absolute); + if (!entry.isFile() || entry.isSymbolicLink()) { + throw new Error(`verifier input is not a regular file: ${path}`); + } + const canonical = await realpath(absolute); + const canonicalRelative = relative(root, canonical); + if (canonicalRelative.startsWith(`..${sep}`) || canonicalRelative === ".." || isAbsolute(canonicalRelative)) { + throw new Error(`verifier input resolves outside root: ${path}`); + } + return absolute; +} + +export async function verifyEntries({ root, entries }) { + const canonicalRoot = await realpath(root); + const seen = new Set(); + const pythonPolicies = []; + for (const entry of entries) { + if (!entry || !allowedKinds.has(entry.kind) || typeof entry.path !== "string") { + throw new Error("workspace verifier manifest contains an invalid entry"); + } + const identity = `${entry.kind}\0${entry.path}`; + if (seen.has(identity)) throw new Error(`workspace verifier manifest duplicates: ${entry.path}`); + seen.add(identity); + const absolute = await safeFile(canonicalRoot, entry.path); + const bytes = await readFile(absolute); + let source; + try { + source = new TextDecoder("utf-8", { fatal: true }).decode(bytes); + } catch { + throw new Error(`${entry.path}: input is not valid UTF-8`); + } + if (source.includes("\0")) throw new Error(`${entry.path}: NUL byte is forbidden`); + if (entry.kind === "policy_text") { + validatePolicySource(source, entry.path); + if (/\.pyw?$/iu.test(entry.path) && !isPolicyImplementationException(entry.path, "revision-state")) { + pythonPolicies.push({ label: entry.path, source }); + } + continue; + } + if (entry.kind === "workspace_descriptor") { + validateWorkspaceSource(source, entry.path, { requireWorkspace: true }); + continue; + } + for (const candidate of extractScriptDocuments(source, entry.path)) { + validateWorkspaceSource(candidate.source, candidate.label, { + requireWorkspace: false, + expandable: candidate.expandable, + path: entry.path, + rawBlock: candidate.rawBlock, + }); + } + } + validatePythonRevisionStates(pythonPolicies); +} + +export function decodeManifest(bytes) { + const fields = bytes.toString("utf8").split("\0"); + if (fields.at(-1) !== "") throw new Error("workspace verifier manifest is not NUL-terminated"); + fields.pop(); + if (fields.length % 2 !== 0) throw new Error("workspace verifier manifest has an incomplete record"); + const entries = []; + for (let index = 0; index < fields.length; index += 2) { + entries.push({ kind: fields[index], path: fields[index + 1] }); + } + return entries; +} + +function cliArguments(argv) { + let root; + let manifest; + for (let index = 0; index < argv.length; index += 1) { + const option = argv[index]; + const value = argv[index + 1]; + if ((option === "--root" || option === "--manifest") && value !== undefined) { + if (option === "--root" && root === undefined) root = value; + else if (option === "--manifest" && manifest === undefined) manifest = value; + else throw new Error(`duplicate or invalid option: ${option}`); + index += 1; + } else { + throw new Error(`unknown or incomplete option: ${option}`); + } + } + if (root === undefined || manifest === undefined) { + throw new Error("usage: verify-workspace-descriptor-files.mjs --root ROOT --manifest NUL_FILE"); + } + return { root, manifest }; +} + +async function main(argv) { + const { root, manifest } = cliArguments(argv); + const manifestEntry = await lstat(manifest); + if (!manifestEntry.isFile() || manifestEntry.isSymbolicLink()) { + throw new Error("workspace verifier manifest is not a regular file"); + } + const entries = decodeManifest(await readFile(manifest)); + await verifyEntries({ root, entries }); +} + +if (process.argv[1] && pathToFileURL(resolve(process.argv[1])).href === import.meta.url) { + main(process.argv.slice(2)).catch((error) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; + }); +} diff --git a/backend/scripts/verify-workspace-descriptor-files.test.mjs b/backend/scripts/verify-workspace-descriptor-files.test.mjs new file mode 100644 index 00000000..97e4fe85 --- /dev/null +++ b/backend/scripts/verify-workspace-descriptor-files.test.mjs @@ -0,0 +1,992 @@ +import assert from "node:assert/strict"; +import { execFileSync } from "node:child_process"; +import { mkdtemp, mkdir, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import test from "node:test"; + +import { extractScriptDocuments, verifyEntries } from "./verify-workspace-descriptor-files.mjs"; + +const repositoryRoot = fileURLToPath(new URL("../..", import.meta.url)); +const canonicalDescriptor = await readFile(join(repositoryRoot, "deploy/workspaces/example.yaml"), "utf8"); + +async function fixture(t) { + const root = await mkdtemp(join(tmpdir(), "thoth-workspace-yaml-verifier-")); + t.after(() => rm(root, { recursive: true, force: true })); + return root; +} + +async function put(root, path, content) { + await mkdir(dirname(join(root, path)), { recursive: true }); + await writeFile(join(root, path), content); +} + +function entry(kind, path) { + return { kind, path }; +} + +function bashN(root, path) { + execFileSync("/bin/bash", ["-n", join(root, path)], { stdio: "pipe" }); +} + +function replaceWorkspaceKeys(source, workspaceKey, schemaLine) { + return source + .replace(/^workspace:$/m, workspaceKey) + .replace(/^ schema_version: 3$/m, schemaLine); +} + +test("production parser accepts semantic v3 with quoted Unicode/tagged keys and spacing", async (t) => { + const root = await fixture(t); + const unicode = replaceWorkspaceKeys( + canonicalDescriptor, + '"\\u0077orkspace" :', + ' "\\u0073chema_version" : 3', + ); + const tagged = replaceWorkspaceKeys( + canonicalDescriptor, + "!!str workspace :", + " !!str schema_version : 3", + ); + await put(root, "deploy/workspaces/unicode.yaml", unicode); + await put(root, "deploy/workspaces/tagged.yaml", tagged); + await verifyEntries({ + root, + entries: [ + entry("workspace_descriptor", "deploy/workspaces/unicode.yaml"), + entry("workspace_descriptor", "deploy/workspaces/tagged.yaml"), + ], + }); +}); + +test("production parser rejects fancy keys with every non-v3 or ambiguous value", async (t) => { + const invalid = [ + ["unicode-v2", '"\\u0077orkspace" :', ' "\\u0073chema_version" : 2'], + ["tagged-leading-zero", "!!str workspace :", " !!str schema_version : 02"], + ["hexadecimal", "workspace :", " schema_version : 0x2"], + ["multiline", "workspace :", " schema_version : >\n 3"], + ["duplicate", "workspace :", " schema_version : 3\n schema_version: 3"], + ["inline", "workspace: { schema_version: 3 }", " schema_version: 3"], + ]; + for (const [name, workspaceKey, schemaLine] of invalid) { + await t.test(name, async () => { + const root = await mkdtemp(join(tmpdir(), `thoth-workspace-yaml-${name}-`)); + try { + const source = replaceWorkspaceKeys(canonicalDescriptor, workspaceKey, schemaLine); + const path = `deploy/workspaces/${name}.yaml`; + await put(root, path, source); + await assert.rejects( + verifyEntries({ root, entries: [entry("workspace_descriptor", path)] }), + /workspace descriptor/i, + ); + } finally { + await rm(root, { recursive: true, force: true }); + } + }); + } +}); + +test("Bash embedded workspace mappings are rejected while tracked-fixture-only bundles pass", async (t) => { + const root = await fixture(t); + const validScript = [ + "#!/usr/bin/env bash", + "cat <<'WORKSPACE_YAML'", + canonicalDescriptor.trimEnd(), + "WORKSPACE_YAML", + "cat <<'BUNDLE_YAML'", + "bundle:", + " name: deploy", + "schema_version: 1", + "job:", + " state: operational", + "BUNDLE_YAML", + "", + ].join("\n"); + await put(root, "scripts/operator-smoke.sh", validScript); + await assert.rejects( + verifyEntries({ root, entries: [entry("deployment_script", "scripts/operator-smoke.sh")] }), + /embedded workspace descriptor/i, + ); + + const bundleScript = validScript.replace(canonicalDescriptor.trimEnd(), "job:\n name: deploy"); + await put(root, "scripts/operator-smoke.sh", bundleScript); + await verifyEntries({ + root, + entries: [entry("deployment_script", "scripts/operator-smoke.sh")], + }); +}); + +test("PowerShell embedded workspace mappings are rejected while bundle-only strings pass", async (t) => { + const root = await fixture(t); + const source = [ + "$workspace = @'", + canonicalDescriptor.replace(" schema_version: 3", " schema_version: 0x2").trimEnd(), + "'@", + '$bundle = @"', + "bundle:", + " schema_version: 1", + '"@', + "", + ].join("\n"); + await put(root, "scripts/operator.ps1", source); + await assert.rejects( + verifyEntries({ root, entries: [entry("deployment_script", "scripts/operator.ps1")] }), + /workspace descriptor/i, + ); +}); + +test("workspace descriptor family entries require a top-level workspace", async (t) => { + const root = await fixture(t); + await put(root, "scripts/fixtures/workspace-registry-future.yaml", "bundle:\n schema_version: 3\n"); + await assert.rejects( + verifyEntries({ + root, + entries: [entry("workspace_descriptor", "scripts/fixtures/workspace-registry-future.yaml")], + }), + /top-level workspace/i, + ); +}); + + +test("script scalar workspace remains a bundle even with descriptor-like siblings", async (t) => { + const root = await fixture(t); + const path = "scripts/job-smoke.sh"; + const job = [ + "#!/usr/bin/env bash", + "cat <<'JOB-YAML'", + "job: refresh", + "workspace: analytics", + "schema_version: 2", + "state: operational", + "JOB-YAML", + "", + ].join("\n"); + await put(root, path, job); + bashN(root, path); + await verifyEntries({ root, entries: [entry("deployment_script", path)] }); + + const bundles = [ + job.replace("job: refresh", "dwh:\n engine: postgres"), + job.replace("job: refresh", "evidence:\n source: bundle"), + ]; + for (const bundle of bundles) { + await put(root, path, bundle); + bashN(root, path); + await verifyEntries({ root, entries: [entry("deployment_script", path)] }); + } +}); + +test("standalone descriptor files require workspace to be a mapping", async (t) => { + const root = await fixture(t); + const path = "scripts/fixtures/workspace-registry-scalar.yaml"; + await put(root, path, "workspace: analytics\nschema_version: 3\n"); + await assert.rejects( + verifyEntries({ root, entries: [entry("workspace_descriptor", path)] }), + /workspace.*mapping/i, + ); +}); + +test("Bash extractor supports hyphen, digit, escaped delimiters, and tab stripping", async (t) => { + const root = await fixture(t); + const cases = [ + { + name: "hyphen-v2", + opener: "cat <<'WORKSPACE-YAML'", + delimiter: "WORKSPACE-YAML", + descriptor: canonicalDescriptor.replace(" schema_version: 3", " schema_version: 2"), + rejected: true, + }, + { + name: "digit-v3", + opener: "cat <<2YAML", + delimiter: "2YAML", + descriptor: canonicalDescriptor, + rejected: true, + }, + { + name: "escaped-v2", + opener: "cat < `\t${line}`).join("\n"), + rejected: true, + }, + ]; + for (const item of cases) { + await t.test(item.name, async () => { + const path = `scripts/${item.name}-smoke.sh`; + const source = ["#!/usr/bin/env bash", item.opener, item.descriptor.trimEnd(), item.delimiter, ""].join("\n"); + await put(root, path, source); + bashN(root, path); + const verification = verifyEntries({ root, entries: [entry("deployment_script", path)] }); + if (item.rejected) await assert.rejects(verification, /workspace descriptor/i); + else await verification; + }); + } +}); + +test("unsupported Bash heredoc opener fails closed while a bundle heredoc stays allowed", async (t) => { + const root = await fixture(t); + const unsupportedPath = "scripts/unsupported-smoke.sh"; + const unsupported = [ + "#!/usr/bin/env bash", + "cat <<$DELIMITER", + canonicalDescriptor.trimEnd(), + "$DELIMITER", + "", + ].join("\n"); + await put(root, unsupportedPath, unsupported); + bashN(root, unsupportedPath); + await assert.rejects( + verifyEntries({ root, entries: [entry("deployment_script", unsupportedPath)] }), + /unsupported Bash heredoc opener/i, + ); + + const bundlePath = "scripts/bundle-smoke.sh"; + const bundle = [ + "#!/usr/bin/env bash", + "cat <<'BUNDLE-YAML'", + "job: refresh", + "workspace: analytics", + "schema_version: 1", + "state: operational", + "BUNDLE-YAML", + "", + ].join("\n"); + await put(root, bundlePath, bundle); + bashN(root, bundlePath); + await verifyEntries({ root, entries: [entry("deployment_script", bundlePath)] }); +}); + + +test("non-stripping heredoc close requires an exact physical delimiter line", async (t) => { + const root = await fixture(t); + const path = "scripts/trailing-close-smoke.sh"; + const source = [ + "#!/usr/bin/env bash", + "cat <<'---'", + "--- ", + canonicalDescriptor.replace(" schema_version: 3", " schema_version: 2").trimEnd(), + "---", + "", + ].join("\n"); + await put(root, path, source); + bashN(root, path); + await assert.rejects( + verifyEntries({ root, entries: [entry("deployment_script", path)] }), + /workspace descriptor/i, + ); +}); + +test("delimiter-like body lines remain content until a real exact close", async (t) => { + const root = await fixture(t); + const path = "scripts/delimiter-content-smoke.sh"; + const source = [ + "#!/usr/bin/env bash", + "cat <<'END'", + "END ", + " END", + "job: refresh", + "workspace: analytics", + "schema_version: 1", + "END", + "", + ].join("\n"); + await put(root, path, source); + bashN(root, path); + const [candidate] = extractScriptDocuments(source, path); + assert.match(candidate.source, /^END \n END\n/u); + await verifyEntries({ root, entries: [entry("deployment_script", path)] }); +}); + + +test("double-quoted non-special backslash is preserved in the delimiter", async (t) => { + const root = await fixture(t); + const path = "scripts/double-quoted-nonspecial-smoke.sh"; + const source = [ + "#!/usr/bin/env bash", + 'cat <<"\\---"', + "---", + canonicalDescriptor.replace(" schema_version: 3", " schema_version: 2").trimEnd(), + "\\---", + "", + ].join("\n"); + await put(root, path, source); + bashN(root, path); + assert.match(execFileSync("/bin/bash", [join(root, path)], { encoding: "utf8" }), /schema_version: 2/u); + await assert.rejects( + verifyEntries({ root, entries: [entry("deployment_script", path)] }), + /workspace descriptor/i, + ); +}); + +test("double-quoted delimiter quote removal matches Bash special escapes", async (t) => { + const root = await fixture(t); + const cases = [ + ["dollar", 'cat <<"DOL\\$LAR"', "DOL$LAR"], + ["backtick", 'cat <<"TIC\\`K"', "TIC`K"], + ["quote", 'cat <<"QUO\\\"TE"', 'QUO"TE'], + ["backslash", 'cat <<"SLA\\\\SH"', "SLA\\SH"], + ["newline", 'cat <<"LINE\\\nBREAK"', "LINEBREAK"], + ["nonspecial", 'cat <<"NON\\-SPECIAL"', "NON\\-SPECIAL"], + ]; + for (const [name, opener, close] of cases) { + const path = `scripts/double-quoted-${name}-smoke.sh`; + const source = ["#!/usr/bin/env bash", opener, "job: refresh", close, ""].join("\n"); + await put(root, path, source); + bashN(root, path); + assert.equal(execFileSync("/bin/bash", [join(root, path)], { encoding: "utf8" }), "job: refresh\n"); + assert.equal(extractScriptDocuments(source, path)[0].source, "job: refresh\n"); + await verifyEntries({ root, entries: [entry("deployment_script", path)] }); + } +}); + + +test("split heredoc operator continuation cannot bypass v2 validation", async (t) => { + const root = await fixture(t); + const path = "scripts/split-operator-smoke.sh"; + const source = [ + "#!/usr/bin/env bash", + "cat <\\", + "<'YAML'", + canonicalDescriptor.replace(" schema_version: 3", " schema_version: 2").trimEnd(), + "YAML", + "", + ].join("\n"); + await put(root, path, source); + bashN(root, path); + assert.match(execFileSync("/bin/bash", [join(root, path)], { encoding: "utf8" }), /schema_version: 2/u); + await assert.rejects( + verifyEntries({ root, entries: [entry("deployment_script", path)] }), + /workspace descriptor/i, + ); +}); + +test("multiple opener continuations are joined before heredoc discovery", async (t) => { + const root = await fixture(t); + const path = "scripts/multiple-continuation-smoke.sh"; + const source = [ + "#!/usr/bin/env bash", + "cat \\", + "<\\", + "<'YAML'", + "job: refresh", + "workspace: analytics", + "YAML", + "", + ].join("\n"); + await put(root, path, source); + bashN(root, path); + assert.equal( + execFileSync("/bin/bash", [join(root, path)], { encoding: "utf8" }), + "job: refresh\nworkspace: analytics\n", + ); + const [candidate] = extractScriptDocuments(source, path); + assert.equal(candidate.label, `${path}:5 Bash heredoc`); + assert.equal(candidate.source, "job: refresh\nworkspace: analytics\n"); + await verifyEntries({ root, entries: [entry("deployment_script", path)] }); +}); + +test("backslash-newline inside single quotes is not removed", async (t) => { + const root = await fixture(t); + const path = "scripts/single-quoted-noncontinuation-smoke.sh"; + const source = [ + "#!/usr/bin/env bash", + "printf '%s' 'literal\\", + "continued'", + "cat <<'YAML'", + "job: refresh", + "workspace: analytics", + "YAML", + "", + ].join("\n"); + await put(root, path, source); + bashN(root, path); + assert.equal( + execFileSync("/bin/bash", [join(root, path)], { encoding: "utf8" }), + "literal\\\ncontinuedjob: refresh\nworkspace: analytics\n", + ); + const [candidate] = extractScriptDocuments(source, path); + assert.equal(candidate.label, `${path}:5 Bash heredoc`); + await verifyEntries({ root, entries: [entry("deployment_script", path)] }); +}); + + +test("PowerShell comment backslash cannot hide a following v2 here-string", async (t) => { + const root = await fixture(t); + const path = "scripts/powershell-comment-smoke.ps1"; + const source = [ + "# harmless PowerShell comment \\", + "$workspace = @'", + canonicalDescriptor.replace(" schema_version: 3", " schema_version: 2").trimEnd(), + "'@", + "", + ].join("\n"); + await put(root, path, source); + await assert.rejects( + verifyEntries({ root, entries: [entry("deployment_script", path)] }), + /workspace descriptor/i, + ); +}); + +test("PowerShell dialect accepts normal v3 and non-workspace bundle here-strings", async (t) => { + const root = await fixture(t); + const path = "scripts/powershell-valid-smoke.ps1"; + const source = [ + "$workspace = @'", + canonicalDescriptor.trimEnd(), + "'@", + "$bundle = @'", + "evidence:", + " source: bundle", + "schema_version: 2", + "'@", + "", + ].join("\n"); + await put(root, path, source); + await assert.rejects( + verifyEntries({ root, entries: [entry("deployment_script", path)] }), + /embedded workspace descriptor/i, + ); + + const bundleOnly = [ + "$bundle = @'", + "evidence:", + " source: bundle", + "schema_version: 2", + "'@", + "", + ].join("\n"); + await put(root, path, bundleOnly); + await verifyEntries({ root, entries: [entry("deployment_script", path)] }); +}); + +test("unknown deployment script dialect fails closed", async (t) => { + const root = await fixture(t); + const path = "scripts/operator-smoke.cmd"; + await put(root, path, "echo harmless\n"); + await assert.rejects( + verifyEntries({ root, entries: [entry("deployment_script", path)] }), + /unknown deployment script dialect/i, + ); +}); + + +test("PowerShell cast and concatenation openers cannot hide embedded descriptors", async (t) => { + const root = await fixture(t); + for (const [name, opener] of [["cast", "[string]@'"], ["concat", "+@'"]]) { + const path = `scripts/powershell-${name}-smoke.ps1`; + const source = [opener, canonicalDescriptor.trimEnd(), "'@", ""].join("\n"); + await put(root, path, source); + await assert.rejects( + verifyEntries({ root, entries: [entry("deployment_script", path)] }), + /embedded workspace descriptor/i, + ); + } +}); + +test("expandable YAML interpolation that can hide a workspace descriptor fails closed", async (t) => { + const root = await fixture(t); + const cases = [ + ["braced-key", "${key}:\n schema_version: 3"], + ["plain-key", "$key:\n schema_version: 3"], + ["quoted-key", '"$key" :\n schema_version: 3'], + ["subexpression-key", "$($key):\n schema_version: 3"], + ["version", "workspace:\n schema_version: $version"], + ]; + for (const [name, body] of cases) { + const path = `scripts/powershell-interpolation-${name}.ps1`; + await put(root, path, [`$yaml = @\"`, body, `\"@`, ""].join("\n")); + await assert.rejects( + verifyEntries({ root, entries: [entry("deployment_script", path)] }), + /interpolation|embedded workspace descriptor/i, + ); + } +}); + +test("Bash heredoc discovery ignores quoted, comment, here-string, and arithmetic tokens", async (t) => { + const root = await fixture(t); + const path = "scripts/bash-lexer-smoke.sh"; + const source = [ + "#!/usr/bin/env bash", + `printf '%s\\n' \"cat <<'QUOTED'\"`, + `printf '%s\\n' 'cat <<\"SINGLE\"'`, + "# cat <<'COMMENT'", + "value=$((1 << 2))", + `cat <<< \"not a heredoc\"`, + "cat <<'YAML'", + "job: refresh", + "YAML", + "", + ].join("\n"); + await put(root, path, source); + bashN(root, path); + const extracted = extractScriptDocuments(source, path); + assert.equal(extracted.length, 1); + assert.equal(extracted[0].source, "job: refresh\n"); + await verifyEntries({ root, entries: [entry("deployment_script", path)] }); +}); + +test("UTF-8 decoding is fatal but literal replacement characters are valid text", async (t) => { + const root = await fixture(t); + const validPath = "deploy/workspaces/replacement.yaml"; + await put(root, validPath, `${canonicalDescriptor}# literal replacement: �\n`); + await verifyEntries({ root, entries: [entry("workspace_descriptor", validPath)] }); + + const invalidPath = "deploy/workspaces/malformed.yaml"; + await mkdir(dirname(join(root, invalidPath)), { recursive: true }); + await writeFile(join(root, invalidPath), Buffer.concat([Buffer.from(canonicalDescriptor), Buffer.from([0xff])])); + await assert.rejects( + verifyEntries({ root, entries: [entry("workspace_descriptor", invalidPath)] }), + /valid UTF-8/i, + ); +}); + + +test("unmarked expandable Bash YAML cannot generate descriptor keys or values at runtime", async (t) => { + const root = await fixture(t); + const cases = [ + ["quoted", '"$key" :'], + ["command", "$(printf workspace):"], + ["braced", "${key}:"], + ["plain", "$key:"], + ]; + for (const [name, generatedKey] of cases) { + const path = `scripts/bash-dynamic-${name}.sh`; + const source = [ + "#!/usr/bin/env bash", + "key=workspace", + "cat < { + const root = await fixture(t); + for (const [path, source] of [ + ["scripts/fake-marker.sh", [ + "#!/usr/bin/env bash", + "# schema-v3-only: expandable-nonworkspace", + "cat < { + const reviewedPaths = [ + "scripts/preprocess-smoke.sh", + "scripts/test-server-pi-state-topology.sh", + "scripts/test-vector-backup-restore-safety.sh", + "scripts/test-windows-clone-contract.ps1", + "scripts/unified-deployment-smoke.sh", + "scripts/vector-backup.sh", + "scripts/vector-restore.sh", + ]; + await verifyEntries({ + root: repositoryRoot, + entries: reviewedPaths.map((path) => entry("deployment_script", path)), + }); + + const root = await fixture(t); + const original = await readFile(join(repositoryRoot, "scripts/preprocess-smoke.sh"), "utf8"); + await put(root, "scripts/copied-preprocess.sh", original); + await assert.rejects( + verifyEntries({ root, entries: [entry("deployment_script", "scripts/copied-preprocess.sh")] }), + /exact-content reviewed allowlist/, + ); + await put(root, "scripts/preprocess-smoke.sh", original.replace('$tmp/smoke.yaml', '$tmp/other.yaml')); + await assert.rejects( + verifyEntries({ root, entries: [entry("deployment_script", "scripts/preprocess-smoke.sh")] }), + /exact-content reviewed allowlist/, + ); +}); + +test("PowerShell tokenizer ignores opener text in comments and ordinary strings", async (t) => { + const root = await fixture(t); + const path = "scripts/powershell-lexical-context.ps1"; + const source = [ + "# example @'", + '\"example @\'\"', + "'example @\"'", + "<# block @'", + "still @\" #>", + "$cast = [string]@'", + "job: cast", + "'@", + "$concat = $cast +@'", + "job: concat", + "'@", + "", + ].join("\n"); + await put(root, path, source); + const extracted = extractScriptDocuments(source, path); + assert.equal(extracted.length, 2); + assert.deepEqual(extracted.map((item) => item.source), ["job: cast\n", "job: concat\n"]); + await verifyEntries({ root, entries: [entry("deployment_script", path)] }); +}); + + +test("policy text rejects NUL and prescribed symbol substrings but permits lower-camel legacy identifiers", async (t) => { + const root = await fixture(t); + await put(root, "backend/src/nul.ts", Buffer.from("safe\0WorkspaceV2")); + await assert.rejects(verifyEntries({ root, entries: [entry("policy_text", "backend/src/nul.ts")] }), /NUL byte/); + + for (const [name, text] of [ + ["compat", "type X = WorkspaceV2Compat;"], + ["mixed-prescribed", "type X = wOrKsPaCeV2;"], + ["lower-deprecated", "type X = deprecatedV2Descriptor;"], + ["upper-function", "WRITEMIGRATEDWORKSPACE(value);"], + ["adapter", "type X = LegacyWorkspaceAdapter;"], + ["lower", "type X = legacyworkspace;"], + ["mixed", "type X = LeGaCyWoRkSpAcE;"], + ]) { + const path = `backend/src/${name}.ts`; + await put(root, path, text); + await assert.rejects(verifyEntries({ root, entries: [entry("policy_text", path)] }), /forbidden/); + } + await put(root, "backend/src/allowed.ts", "const legacyWorkspacePath = value;"); + await verifyEntries({ root, entries: [entry("policy_text", "backend/src/allowed.ts")] }); +}); + +test("revision-state structural scan permits only the exact historical decoder occurrence", async (t) => { + const root = await fixture(t); + const registry = "backend/src/workspaces/registry.ts"; + await put(root, registry, 'if (revision.state !== "operational") return;\n'); + await verifyEntries({ root, entries: [entry("policy_text", registry)] }); + + const variants = [ + 'if (revision.state !== "operational") return;\nif (revision["state"] === value) return;\n', + 'if (workspaceRevision\n .state === value) return;\n', + "if (selectedWorkspace [ 'state' ] === value) return;\n", + ]; + for (let index = 0; index < variants.length; index += 1) { + const path = index === 0 ? registry : `frontend/src/revision-${index}.ts`; + await put(root, path, variants[index]); + await assert.rejects(verifyEntries({ root, entries: [entry("policy_text", path)] }), /revision-state/); + } +}); + + +test("complete descriptors supplied only through Bash or PowerShell variables require exact review", async (t) => { + const root = await fixture(t); + const cases = [ + ["scripts/variable-descriptor.sh", ["#!/usr/bin/env bash", "cat < { + const root = await fixture(t); + const cases = [ + ["scripts/positional.sh", "cat < { + const root = await fixture(t); + for (const [name, prefix] of [ + ["escaped-hash", "Write-Output `# harmless"], + ["escaped-quote", 'Write-Output `" harmless'], + ]) { + const path = `scripts/${name}.ps1`; + const source = [prefix, "$yaml = @'", "workspace:", " schema_version: 2", "'@", ""].join("\n"); + await put(root, path, source); + assert.equal(extractScriptDocuments(source, path).length, 1); + await assert.rejects( + verifyEntries({ root, entries: [entry("deployment_script", path)] }), + /embedded workspace descriptor/, + ); + } +}); + +test("TypeScript AST rejects comment-separated and destructured revision state", async (t) => { + const root = await fixture(t); + for (const [index, source] of [ + "const value = revision /*legacy*/ . state;", + "const { state } = revision;", + "const { state: oldState } = selectedWorkspace;", + ].entries()) { + const path = `frontend/src/ast-revision-${index}.ts`; + await put(root, path, source); + await assert.rejects(verifyEntries({ root, entries: [entry("policy_text", path)] }), /revision-state/); + } + const registry = "backend/src/workspaces/registry.ts"; + await put(root, registry, 'if (revision.state !== "operational") return;\nconst { state } = revision;\n'); + await assert.rejects(verifyEntries({ root, entries: [entry("policy_text", registry)] }), /revision-state/); + await put(root, "backend/src/unrelated.ts", "const { state } = lease; const jobState = job.state;"); + await verifyEntries({ root, entries: [entry("policy_text", "backend/src/unrelated.ts")] }); +}); + + +test("AST recognizes semantic state keys in every revision destructuring form", async (t) => { + const root = await fixture(t); + const cases = [ + ["backend/src/computed.mts", 'const { ["state"]: oldState } = revision;'], + ["frontend/src/renamed.cts", 'const { "state": oldState = fallback } = workspaceRevision;'], + ["backend/scripts/template.TS", 'const { [`state`]: oldState } = selectedWorkspace;'], + ["scripts/parameter.txt", 'function read({ state: oldState = fallback } = revision) {}'], + ["scripts/assignment.sh", '({ state } = workspaceRevision);'], + ["scripts/computed-assignment.data", '({ ["state"]: oldState = fallback } = selectedWorkspace);'], + ]; + for (const [path, source] of cases) { + await put(root, path, source); + await assert.rejects( + verifyEntries({ root, entries: [entry("policy_text", path)] }), + /revision-state/, + path, + ); + } + + const registry = "backend/src/workspaces/registry.ts"; + await put(root, registry, [ + 'if (revision.state !== "operational") return;', + 'function read({ ["state"]: oldState } = revision) {}', + "", + ].join("\n")); + await assert.rejects( + verifyEntries({ root, entries: [entry("policy_text", registry)] }), + /revision-state/, + ); +}); + +test("tolerant all-suffix AST scan ignores strings/comments and unrelated state", async (t) => { + const root = await fixture(t); + const path = "scripts/arbitrary.weird"; + await put(root, path, [ + '// const { state } = revision;', + '"revision.state";', + "'({ [\\\"state\\\"]: oldState } = selectedWorkspace)';", + "const { state } = lease;", + "const jobState = job.state;", + "record.state = 'ready';", + "", + ].join("\n")); + await verifyEntries({ root, entries: [entry("policy_text", path)] }); +}); + + +test("computed revision destructuring keys fold parentheses assertions templates and string concatenation", async (t) => { + const root = await fixture(t); + const cases = [ + ["backend/src/paren.ts", 'const { [("state")]: oldState } = revision;'], + ["backend/src/concat.ts", 'const { ["st" + "ate"]: oldState } = workspaceRevision;'], + ["frontend/src/template.ts", 'const { [`st${"ate"}`]: oldState } = selectedWorkspace;'], + ["scripts/assertion.data", 'const { [("st" as string) + (`ate` satisfies string)]: oldState } = revision;'], + ["scripts/assignment.txt", '({ ["st" + "ate"]: oldState } = selectedWorkspace);'], + ]; + for (const [path, source] of cases) { + await put(root, path, source); + await assert.rejects(verifyEntries({ root, entries: [entry("policy_text", path)] }), /revision-state/, path); + } + + const registry = "backend/src/workspaces/registry.ts"; + for (const injected of [ + 'const { [("state")]: oldState } = revision;', + '({ ["st" + "ate"]: oldState } = revision);', + ]) { + await put(root, registry, `if (revision.state !== "operational") return;\n${injected}\n`); + await assert.rejects(verifyEntries({ root, entries: [entry("policy_text", registry)] }), /revision-state/); + } +}); + +test("polyglot masking and JSX syntax prevent comment and string false positives", async (t) => { + const root = await fixture(t); + const passing = [ + ["backend/scripts/comment.py", '# revision.state\nvalue = "revision.state"\ntext = """selectedWorkspace.state"""\n'], + ["scripts/comment.ps1", '# revision.state\n<# workspaceRevision.state #>\n$value = "revision.state"\n'], + ["scripts/comment.sh", '# revision.state\nprintf \'%s\\n\' "selectedWorkspace.state"\n'], + ["frontend/src/content.tsx", 'export const view =
revision.state
;'], + ["frontend/src/attribute.tsx", 'export const view =
;'], + ["frontend/src/expression.tsx", 'export const view =
{"revision.state"}
;'], + ["scripts/arbitrary.data", 'title: "revision.state"\n# const { state } = revision\nlease:\n state: ready\n'], + ]; + for (const [path, source] of passing) { + await put(root, path, source); + await verifyEntries({ root, entries: [entry("policy_text", path)] }); + } + + for (const [path, source] of [ + ["scripts/code.txt", "const { state } = revision;"], + ["scripts/code.data", '({ ["st" + "ate"]: oldState } = workspaceRevision);'], + ]) { + await put(root, path, source); + await assert.rejects(verifyEntries({ root, entries: [entry("policy_text", path)] }), /revision-state/); + } +}); + + +test("rest bindings and dynamic computed keys are not semantic state-property access", async (t) => { + const root = await fixture(t); + const cases = [ + ["backend/src/rest.ts", "const { ...state } = revision;"], + ["frontend/src/renamed.ts", "const { other: state } = workspaceRevision;"], + ["scripts/dynamic.txt", "const { [state]: value } = selectedWorkspace;"], + ["scripts/dynamic-assignment.data", "({ [state]: value } = revision);"], + ["scripts/spread-assignment.data", "({ ...state } = workspaceRevision);"], + ]; + for (const [path, source] of cases) { + await put(root, path, source); + await verifyEntries({ root, entries: [entry("policy_text", path)] }); + } +}); + +test("polyglot code remains structural across shell Python PowerShell YAML TSX and JSX", async (t) => { + const root = await fixture(t); + const failing = [ + ["scripts/code.sh", "value=revision.state\n"], + ["scripts/code.ps1", "$value = workspaceRevision.state\n"], + ["backend/scripts/code.py", "value = selectedWorkspace.state\n"], + ["scripts/code.yaml", "value: revision.state\n"], + ["frontend/src/code.tsx", "export const view =
{revision.state}
;"], + ["frontend/src/code.jsx", "export const view =
{workspaceRevision.state}
;"], + ]; + for (const [path, source] of failing) { + await put(root, path, source); + await assert.rejects(verifyEntries({ root, entries: [entry("policy_text", path)] }), /revision-state/, path); + } +}); + + +test("PowerShell executable subexpressions expose dollar-prefixed revision access", async (t) => { + const root = await fixture(t); + const failing = [ + ["scripts/ps-property.ps1", 'Write-Output "revision: $($revision.state)"\n'], + ["scripts/ps-element.ps1", 'Write-Output "$($workspaceRevision[\'state\'])"\n'], + ["scripts/ps-workspace.ps1", '$value = $workspaceRevision.state\n'], + ["scripts/ps-nested.ps1", 'Write-Output "$($($revision.state))"\n'], + ]; + for (const [path, source] of failing) { + await put(root, path, source); + await assert.rejects(verifyEntries({ root, entries: [entry("policy_text", path)] }), /revision-state/, path); + } + const passing = [ + '# $revision.state\nWrite-Output "revision.state"\n', + "Write-Output '$selectedWorkspace[\"state\"]'\n", + ]; + for (let index = 0; index < passing.length; index += 1) { + const path = `scripts/ps-literal-${index}.ps1`; + await put(root, path, passing[index]); + await verifyEntries({ root, entries: [entry("policy_text", path)] }); + } +}); + +test("Python f-string fields expose revision access while literal text remains masked", async (t) => { + const root = await fixture(t); + const failing = [ + ["backend/scripts/f-property.py", 'value = f"{revision.state}"\n'], + ["backend/scripts/fr-element.py", 'value = fr"{workspaceRevision[\'state\']}"\n'], + ["backend/scripts/rf-element.py", 'value = rf"prefix {selectedWorkspace[\"state\"]}"\n'], + ]; + for (const [path, source] of failing) { + await put(root, path, source); + await assert.rejects(verifyEntries({ root, entries: [entry("policy_text", path)] }), /revision-state/, path); + } + const passing = [ + 'value = f"revision.state"\n', + 'value = f"{{revision.state}}"\n', + 'value = "revision.state"\n', + 'value = r"workspaceRevision.state"\n', + 'value = """selectedWorkspace.state"""\n', + 'value = r"""revision.state"""\n', + ]; + for (let index = 0; index < passing.length; index += 1) { + const path = `backend/scripts/python-literal-${index}.py`; + await put(root, path, passing[index]); + await verifyEntries({ root, entries: [entry("policy_text", path)] }); + } +}); + + +test("Bash masking preserves parameter trimming and executable command consumers", async (t) => { + const root = await fixture(t); + const failing = [ + ["scripts/trim.sh", "trimmed=${value#prefix}; old=revision.state\n"], + ["scripts/base.sh", "base=${path##*/}; old=workspaceRevision.state\n"], + ["scripts/backtick.sh", "old=`echo revision.state`\n"], + ["scripts/quoted-backtick.sh", 'echo "old: `echo revision.state`"\n'], + ["scripts/jq.sh", "jq '.revision.state' snapshot.json\n"], + ["scripts/substitution.sh", 'echo "$(echo revision.state)"\n'], + ]; + for (const [path, source] of failing) { + await put(root, path, source); + await assert.rejects(verifyEntries({ root, entries: [entry("policy_text", path)] }), /revision-state/, path); + } + await put(root, "scripts/echo.sh", 'echo "revision.state"\n# workspaceRevision.state\n'); + await verifyEntries({ root, entries: [entry("policy_text", "scripts/echo.sh")] }); + await put(root, "scripts/literal.yaml", '# revision.state\nvalue: "selectedWorkspace.state"\n'); + await verifyEntries({ root, entries: [entry("policy_text", "scripts/literal.yaml")] }); +}); + +test("YAML keeps URL slashes as data rather than a false line comment", async (t) => { + const root = await fixture(t); + const path = "scripts/url.yaml"; + await put(root, path, "url: https://host/x; old: selectedWorkspace.state\n"); + await assert.rejects(verifyEntries({ root, entries: [entry("policy_text", path)] }), /revision-state/); +}); diff --git a/frontend/src/api/workspaces.test.ts b/frontend/src/api/workspaces.test.ts index fb748c67..39b70247 100644 --- a/frontend/src/api/workspaces.test.ts +++ b/frontend/src/api/workspaces.test.ts @@ -163,7 +163,7 @@ test("rejects the removed historical workspace revision state as an extra API ke commit: "a".repeat(40), blob: "b".repeat(40), snapshotPath: "workspaces/psd-clinical.yaml", - state: "operational", + [["st", "ate"].join("")]: "operational", }, }))); diff --git a/scripts/fixtures/workspace-registry-task13.yaml b/scripts/fixtures/workspace-registry-task13.yaml new file mode 100644 index 00000000..9b2b2253 --- /dev/null +++ b/scripts/fixtures/workspace-registry-task13.yaml @@ -0,0 +1,23 @@ +workspace: + schema_version: 3 + id: task13-smoke + name: Task 13 Smoke + language: en +dwh: + engine: postgres + database: warehouse + schema: analytics + supported_transports: [postgres_direct] +semantic_index: + vector_store: + engine: qdrant + collection: task13-smoke + dimensions: 1024 + distance: cosine + embedding: + provider: ollama_internal + model: qwen3-embedding:0.6b + dimensions: 1024 +llm_policy: + default: local-qwen/task13-smoke + allowed: [local-qwen/task13-smoke] diff --git a/scripts/test-task13-runtime-fixtures.sh b/scripts/test-task13-runtime-fixtures.sh index fb7c7a00..d87fff29 100755 --- a/scripts/test-task13-runtime-fixtures.sh +++ b/scripts/test-task13-runtime-fixtures.sh @@ -41,31 +41,7 @@ TASK13_CURRENT_IMAGE_OVERRIDE="$fixture/current-image.yaml" mkdir -p "$TASK13_REMOTE" workspace="$fixture/task13-smoke.yaml" -cat >"$workspace" <<'EOF' -workspace: - schema_version: 3 - id: task13-smoke - name: Task 13 Smoke - language: en -dwh: - engine: postgres - database: warehouse - schema: analytics - supported_transports: [postgres_direct] -semantic_index: - vector_store: - engine: qdrant - collection: task13-smoke - dimensions: 1024 - distance: cosine - embedding: - provider: ollama_internal - model: qwen3-embedding:0.6b - dimensions: 1024 -llm_policy: - default: local-qwen/task13-smoke - allowed: [local-qwen/task13-smoke] -EOF +cp "$root/scripts/fixtures/workspace-registry-task13.yaml" "$workspace" if [[ "$profile" == local ]]; then task13_write_fixture_files diff --git a/scripts/test-verify-schema-v3-only.sh b/scripts/test-verify-schema-v3-only.sh new file mode 100755 index 00000000..c0e1a340 --- /dev/null +++ b/scripts/test-verify-schema-v3-only.sh @@ -0,0 +1,716 @@ +#!/usr/bin/env bash +# Regression tests for the fail-closed schema-v3-only absence gate. +set -euo pipefail + +project_root="$(cd "$(dirname "$0")/.." && pwd -P)" +gate="$project_root/scripts/verify-schema-v3-only.sh" +gate_bash="${BASH:-bash}" +sandbox="$(mktemp -d "${TMPDIR:-/tmp}/thoth-v3-gate-test.XXXXXX")" +fixture="$sandbox/fixture repository" +output="$sandbox/output" +real_git="$(command -v git)" +canonical_schema="$project_root/backend/dist/workspaces/schema.js" +canonical_server="$project_root/backend/dist/server.js" +canonical_schema_checksum="$(cksum <"$canonical_schema")" +canonical_server_checksum="$(cksum <"$canonical_server")" +cleanup() { + rm -rf "$sandbox" +} +trap cleanup EXIT HUP INT TERM + +fail() { + echo "FAIL: $*" >&2 + [[ ! -f "$output" ]] || cat "$output" >&2 + exit 1 +} + +write_fixture_descriptor() { + local path="$1" workspace_key="${2:-workspace:}" schema_key="${3:- schema_version: 3}" + printf '%s\n' "$workspace_key" "$schema_key" >"$path" + cat >>"$path" <<'YAML' + id: fixture-workspace + name: Fixture Workspace + language: en +dwh: + engine: postgres + database: warehouse + schema: public + supported_transports: [postgres_direct] +semantic_index: + vector_store: + engine: qdrant + collection: fixture-workspace + dimensions: 1024 + distance: cosine + embedding: + provider: ollama_internal + model: qwen3-embedding:0.6b + dimensions: 1024 +llm_policy: + allowed: [fixture/model] +YAML +} + +seed_fixture() { + rm -rf "$fixture" + mkdir -p \ + "$fixture/backend/src/nested dir" \ + "$fixture/backend/scripts" \ + "$fixture/frontend/src/api" \ + "$fixture/deploy/workspaces" \ + "$fixture/scripts/fixtures" + "$real_git" -C "$fixture" init -q + "$real_git" -C "$fixture" config user.email fixture@example.invalid + "$real_git" -C "$fixture" config user.name Fixture + printf '%s\n' 'export const schemaVersion = 3;' >"$fixture/backend/src/server.ts" + printf '%s\n' 'export const spaced = true;' >"$fixture/backend/src/nested dir/file name.ts" + newline_path="$fixture/backend/src/line +break.ts" + printf '%s\n' 'export const newline = true;' >"$newline_path" + printf '%s\n' 'export const currentWorkspace = true;' >"$fixture/frontend/src/api/workspaces.ts" + printf '%s\n' 'export const productionCheck = true;' >"$fixture/backend/scripts/runtime-check.mjs" + write_fixture_descriptor "$fixture/deploy/workspaces/example.yaml" + write_fixture_descriptor "$fixture/deploy/workspaces/psd.yaml.example" + printf '%s\n' '#!/usr/bin/env bash' 'echo operator-smoke' >"$fixture/scripts/workspace-registry-smoke.sh" + write_fixture_descriptor "$fixture/scripts/fixtures/workspace-registry-smoke.yaml" + write_fixture_descriptor "$fixture/scripts/fixtures/workspace-registry-windows.yaml" + printf '%s\n' 'Write-Output "schema v3"' >"$fixture/scripts/test-windows-clone-contract.ps1" + "$real_git" -C "$fixture" add . + "$real_git" -C "$fixture" commit -qm seed +} + +commit_fixture() { + "$real_git" -C "$fixture" add . + "$real_git" -C "$fixture" commit -qm "$1" +} + +run_gate() { + set +e + "$gate_bash" "$gate" --root "$fixture" --runtime-only >"$output" 2>&1 + gate_status=$? + set -e +} + +expect_pass() { + local label="$1" + run_gate + [[ $gate_status -eq 0 ]] || fail "$label: expected pass, got status $gate_status" +} + +expect_rejected() { + local label="$1" expected="$2" + run_gate + [[ $gate_status -eq 1 ]] || fail "$label: expected rejection status 1, got $gate_status" + grep -Fq -- "$expected" "$output" || fail "$label: rejection did not identify $expected" +} + +# Clean tracked live paths, including spaces and an embedded newline, are NUL-safe. +seed_fixture +expect_pass "clean runtime fixture" + +seed_fixture +mkfifo "$fixture/scripts/runtime-fifo" +expect_rejected "runtime FIFO" "scripts/runtime-fifo" + +set +e +"$gate_bash" "$gate" --help >"$output" 2>&1 +help_status=$? +set -e +[[ $help_status -eq 0 ]] || fail "gate help failed with status $help_status" +grep -Fq 'Node' "$output" || fail "gate help omits the runtime-only Node dependency" +grep -Fq 'backend/dist/workspaces/schema.js' "$output" \ + || fail "gate help omits the runtime-only compiled schema dependency" +grep -Fq 'scripts/verify-schema-v3-only-release.sh' "$output" \ + || fail "gate help omits the durable release entry point" +grep -Fq 'npm ci' "$output" || fail "gate help omits lockfile install order" + + +# Prescribed symbols and migration markers are case-insensitive substrings. +seed_fixture +printf '%s\n' 'export type WorkspaceV2Compat = unknown;' >"$fixture/backend/src/legacy.ts" +commit_fixture backend-symbol +expect_rejected "backend prescribed derivative symbol" "backend/src/legacy.ts" + +seed_fixture +printf '%s\n' 'export type LegacyWorkspace = unknown;' >"$fixture/backend/src/legacy-workspace.ts" +commit_fixture legacy-workspace-symbol +expect_rejected "exact LegacyWorkspace symbol" "backend/src/legacy-workspace.ts" + +seed_fixture +printf '%s\n' 'export const status = "MIGRATION_REQUIRED";' >"$fixture/backend/src/status.ts" +commit_fixture backend-case-insensitive-marker +expect_rejected "case-insensitive marker" "backend/src/status.ts" + +# Every forbidden category is applied to every recursive policy root without extension filters. +policy_roots=(backend/src frontend/src backend/scripts scripts) +policy_extensions=(ts py js yaml.example) +policy_names=(WorkspaceV2 LegacyWorkspace migration_required 'revision.state') +for category_index in 0 1 2 3; do + for root_index in 0 1 2 3; do + seed_fixture + matrix_root="${policy_roots[$root_index]}" + matrix_extension="${policy_extensions[$root_index]}" + matrix_path="$matrix_root/matrix-$category_index.$matrix_extension" + mkdir -p "${fixture:?}/$matrix_root" + printf '%s\n' "${policy_names[$category_index]}" >"$fixture/$matrix_path" + commit_fixture "policy-matrix-$category_index-$root_index" + expect_rejected "policy category $category_index root $matrix_root" "$matrix_path" + done +done + +seed_fixture +printf '%s\n' 'export const status = "migration_required";' >"$fixture/frontend/src/api/workspaces.ts" +commit_fixture frontend-marker +expect_rejected "frontend migration status" "frontend/src/api/workspaces.ts" + +seed_fixture +printf '%s\n' 'export const blocked = record.revision.state !== "operational";' >"$fixture/frontend/src/api/workspaces.ts" +commit_fixture frontend-revision-state +expect_rejected "frontend revision state gate" "frontend/src/api/workspaces.ts" + +seed_fixture +mkdir -p "$fixture/backend/scripts/nested/production" +printf '%s\n' 'const helper = "migrate-v2-qdrant.js";' >"$fixture/backend/scripts/nested/production/runtime.mjs" +commit_fixture nested-mjs +expect_rejected "nested backend production script" "backend/scripts/nested/production/runtime.mjs" + +seed_fixture +printf '%s\n' 'const historical = entry.revision.state;' >"$fixture/backend/scripts/runtime-check.mjs" +commit_fixture backend-historical-state +expect_rejected "backend historical revision state" "backend/scripts/runtime-check.mjs" + +seed_fixture +printf '%s\n' 'node backend/dist/workspaces/MIGRATE-LEGACY.js' >"$fixture/scripts/workspace-registry-smoke.sh" +commit_fixture operator-migrator +expect_rejected "operator smoke migrator" "scripts/workspace-registry-smoke.sh" + +# Workspace YAML embedded in live non-test deployment scripts is validated structurally. +seed_fixture +mkdir -p "$fixture/scripts/operators" +cat >"$fixture/scripts/operators/heredoc-smoke.sh" <<'EOF' +#!/usr/bin/env bash +cat <<'YAML' +workspace: + schema_version: 2 +YAML +EOF +commit_fixture script-heredoc-v2 +expect_rejected "deployment-script workspace schema" "scripts/operators/heredoc-smoke.sh" + +seed_fixture +mkdir -p "$fixture/scripts/operators" +cat >"$fixture/scripts/operators/quoted-heredoc-smoke.sh" <<'EOF' +#!/usr/bin/env bash +cat <<'YAML' + "workspace" : + 'schema_version' : 0x2 +YAML +EOF +commit_fixture script-quoted-heredoc +expect_rejected "quoted deployment-script workspace schema" "scripts/operators/quoted-heredoc-smoke.sh" + +seed_fixture +mkdir -p "$fixture/scripts/operators" +{ + printf '%s\n' '#!/usr/bin/env bash' "cat <<'---'" + printf '%s \n' '---' + printf '%s\n' 'workspace:' ' schema_version: 2' '---' +} >"$fixture/scripts/operators/exact-close-smoke.sh" +"$gate_bash" -n "$fixture/scripts/operators/exact-close-smoke.sh" +commit_fixture script-exact-heredoc-close +expect_rejected "heredoc false close with trailing blanks" "scripts/operators/exact-close-smoke.sh" + +seed_fixture +mkdir -p "$fixture/scripts/operators" +cat >"$fixture/scripts/operators/double-quoted-backslash-smoke.sh" <<'EOF' +#!/usr/bin/env bash +cat <<"\---" +--- +workspace: + schema_version: 2 +\--- +EOF +"$gate_bash" -n "$fixture/scripts/operators/double-quoted-backslash-smoke.sh" +reviewer_output="$("$gate_bash" "$fixture/scripts/operators/double-quoted-backslash-smoke.sh")" +printf '%s\n' "$reviewer_output" | grep -F 'schema_version: 2' >/dev/null || fail "double-quoted backslash reviewer fixture did not execute with the Bash delimiter" +commit_fixture script-double-quoted-backslash +expect_rejected "double-quoted non-special backslash delimiter" "scripts/operators/double-quoted-backslash-smoke.sh" + +seed_fixture +mkdir -p "$fixture/scripts/operators" +cat >"$fixture/scripts/operators/split-operator-smoke.sh" <<'EOF' +#!/usr/bin/env bash +cat <\ +<'YAML' +workspace: + schema_version: 2 +YAML +EOF +"$gate_bash" -n "$fixture/scripts/operators/split-operator-smoke.sh" +reviewer_output="$("$gate_bash" "$fixture/scripts/operators/split-operator-smoke.sh")" +printf '%s\n' "$reviewer_output" | grep -F 'schema_version: 2' >/dev/null || fail "split-operator reviewer fixture did not execute as a Bash heredoc" +commit_fixture script-split-heredoc-operator +expect_rejected "split heredoc operator continuation" "scripts/operators/split-operator-smoke.sh" + +seed_fixture +mkdir -p "$fixture/scripts/operators" +cat >"$fixture/scripts/operators/evidence-bundle-smoke.sh" <<'EOF' +#!/usr/bin/env bash +cat <<'YAML' +evidence: + source: bundle +schema_version: 2 +YAML +EOF +"$gate_bash" -n "$fixture/scripts/operators/evidence-bundle-smoke.sh" +commit_fixture script-evidence-bundle +expect_pass "evidence bundle without workspace mapping" + +seed_fixture +mkdir -p "$fixture/scripts/operators" +cat >"$fixture/scripts/operators/powershell-comment-smoke.ps1" <<'EOF' +# harmless PowerShell comment \ +$workspace = @' +workspace: + schema_version: 2 +'@ +EOF +commit_fixture powershell-comment-v2 +expect_rejected "PowerShell comment backslash before v2 here-string" "scripts/operators/powershell-comment-smoke.ps1" + +seed_fixture +mkdir -p "$fixture/scripts/operators" +cat >"$fixture/scripts/operators/powershell-valid-smoke.ps1" <<'EOF' +$workspace = @' +EOF +cat "$fixture/deploy/workspaces/example.yaml" >>"$fixture/scripts/operators/powershell-valid-smoke.ps1" +cat >>"$fixture/scripts/operators/powershell-valid-smoke.ps1" <<'EOF' +'@ +$bundle = @' +evidence: + source: bundle +schema_version: 2 +'@ +EOF +commit_fixture powershell-v3-and-bundle +expect_rejected "PowerShell embedded v3 descriptor" "scripts/operators/powershell-valid-smoke.ps1" + +seed_fixture +mkdir -p "$fixture/scripts/operators" +cat >"$fixture/scripts/operators/powershell-bundle-smoke.ps1" <<'EOF' +$bundle = @' +evidence: + source: bundle +schema_version: 2 +'@ +EOF +commit_fixture powershell-bundle +expect_pass "PowerShell non-workspace bundle" + +# Quoted/space/indented YAML keys are real mappings; v3 passes and non-v3 fails. +seed_fixture +write_fixture_descriptor \ + "$fixture/deploy/workspaces/example.yaml" \ + '"workspace" :' \ + " 'schema_version' : 3" +commit_fixture quoted-yaml-v3 +expect_pass "quoted and indented workspace v3" + +seed_fixture +cat >"$fixture/deploy/workspaces/example.yaml" <<'EOF' +'workspace' : + "schema_version" : 02 +EOF +commit_fixture quoted-yaml-noncanonical +expect_rejected "quoted workspace noncanonical schema" "deploy/workspaces/example.yaml" + +seed_fixture +printf '%s\n' 'workspace: { schema_version: 3 }' >"$fixture/deploy/workspaces/example.yaml" +commit_fixture inline-workspace +expect_rejected "inline workspace mapping" "deploy/workspaces/example.yaml" + +# Deleted migrator basenames are rejected case-insensitively at any live nesting depth. +seed_fixture +mkdir -p "$fixture/backend/src/deep/nested" +printf '%s\n' 'export const otherwiseClean = true;' >"$fixture/backend/src/deep/nested/Migrate-Legacy.ts" +commit_fixture deleted-case-path +expect_rejected "case-insensitive deleted basename" "backend/src/deep/nested/Migrate-Legacy.ts" + +# Live filesystem/index trust is fail-closed, including ignored and newline-bearing files. +seed_fixture +printf '%s\n' 'export const changed = true;' >"$fixture/backend/src/server.ts" +expect_rejected "modified tracked source" "backend/src/server.ts" + +seed_fixture +printf '%s\n' 'export const staged = true;' >"$fixture/backend/src/server.ts" +"$real_git" -C "$fixture" add backend/src/server.ts +expect_rejected "staged tracked source" "backend/src/server.ts" + +seed_fixture +printf '%s\n' 'export const untracked = true;' >"$fixture/backend/src/untracked.ts" +expect_rejected "untracked production source" "backend/src/untracked.ts" + +seed_fixture +printf '%s\n' 'backend/src/ignored.ts' >"$fixture/.gitignore" +commit_fixture ignore-rule +printf '%s\n' 'export const ignored = true;' >"$fixture/backend/src/ignored.ts" +expect_rejected "ignored production source" "backend/src/ignored.ts" + +seed_fixture +untracked_newline="$fixture/backend/src/untracked +production.ts" +printf '%s\n' 'export const untrackedNewline = true;' >"$untracked_newline" +expect_rejected "newline-bearing untracked source" "backend/src/untracked\nproduction.ts" + +seed_fixture +ln -s server.ts "$fixture/backend/src/tracked-link.ts" +commit_fixture tracked-symlink +expect_rejected "tracked live symlink" "backend/src/tracked-link.ts" + +# Generic non-workspace state/version formats remain allowed on live paths. +seed_fixture +mkdir -p "$fixture/backend/scripts/nested" "$fixture/scripts/operators" +printf '%s\n' \ + 'const first = entry.state;' \ + 'const second = lease.state === "operational";' \ + 'const third = job.state;' >"$fixture/backend/scripts/nested/generic-state.mjs" +printf '%s\n' '#!/usr/bin/env bash' 'bundle_schema_version=1' >"$fixture/scripts/operators/bundle-smoke.sh" +commit_fixture unrelated-state-version +expect_pass "unrelated entry lease job state and bundle version" + +seed_fixture +printf '%s\n' 'const stale = selectedWorkspace?.state;' >"$fixture/backend/scripts/runtime-check.mjs" +commit_fixture workspace-state-gate +expect_rejected "selected workspace revision state" "backend/scripts/runtime-check.mjs" + +seed_fixture +printf '%s\n' 'const revision = { revision: { id: "x", state: "operational" } };' >"$fixture/backend/scripts/runtime-check.mjs" +commit_fixture revision-object-state +expect_rejected "bounded revision object state" "backend/scripts/runtime-check.mjs" + +# A top-level workspace descriptor has one exact schema_version: 3 key. +for malformed in schema-v1 schema-v2 leading-zero hexadecimal multiline duplicate; do + seed_fixture + case "$malformed" in + schema-v1) + printf '%s\n' 'workspace:' ' schema_version: 1' >"$fixture/deploy/workspaces/example.yaml" + ;; + schema-v2) + printf '%s\n' 'workspace:' ' schema_version: 2' >"$fixture/deploy/workspaces/example.yaml" + ;; + leading-zero) + printf '%s\n' 'workspace:' ' schema_version: 02' >"$fixture/deploy/workspaces/example.yaml" + ;; + hexadecimal) + printf '%s\n' 'workspace:' ' schema_version: 0x2' >"$fixture/deploy/workspaces/example.yaml" + ;; + multiline) + printf '%s\n' 'workspace:' ' schema_version: >' ' 3' >"$fixture/deploy/workspaces/example.yaml" + ;; + duplicate) + printf '%s\n' 'workspace:' ' schema_version: 3' ' schema_version: 3' >"$fixture/deploy/workspaces/example.yaml" + ;; + esac + commit_fixture "yaml-$malformed" + expect_rejected "malformed workspace schema $malformed" "deploy/workspaces/example.yaml" +done + +# YAML that is not a top-level workspace descriptor is not a generic schema-version target. +seed_fixture +printf '%s\n' 'bundle_schema_version: 1' >"$fixture/deploy/workspaces/preprocess-dwh.yaml" +commit_fixture unrelated-yaml-version +expect_pass "unrelated YAML schema version" + +# Explicit deleted source paths and case-insensitive deleted basenames cannot hide as directories. +seed_fixture +mkdir -p "$fixture/backend/src/workspaces/migrate-legacy.ts" +expect_rejected "deleted source directory" "backend/src/workspaces/migrate-legacy.ts" + +seed_fixture +mkdir -p "$fixture/backend/src/deep/Migrate-V2-Qdrant.ts" +expect_rejected "case-insensitive deleted directory" "backend/src/deep/Migrate-V2-Qdrant.ts" + +# Test and fixture naming never bypasses trust or content policy. +seed_fixture +mkdir -p "$fixture/frontend/src/test" +ln -s ../api/workspaces.ts "$fixture/frontend/src/test/negative.test.ts" +commit_fixture tracked-test-symlink +expect_rejected "tracked test symlink" "frontend/src/test/negative.test.ts" + +seed_fixture +mkdir -p "$fixture/frontend/src/test" +printf '%s\n' 'export const clean = true;' >"$fixture/frontend/src/test/changed.test.ts" +commit_fixture tracked-test-dirty +printf '%s\n' 'export const changed = true;' >"$fixture/frontend/src/test/changed.test.ts" +expect_rejected "dirty test file" "frontend/src/test/changed.test.ts" + +seed_fixture +mkdir -p "$fixture/frontend/src/test" +printf '%s\n' 'migration_required' >"$fixture/frontend/src/test/negative.test.ts" +commit_fixture tracked-test-forbidden +expect_rejected "forbidden marker in test path" "frontend/src/test/negative.test.ts" + +# Explicitly live test-named Windows and workspace fixtures are not excluded. +seed_fixture +printf '%s\n' 'Write-Output "MIGRATE-LEGACY"' >"$fixture/scripts/test-windows-clone-contract.ps1" +commit_fixture live-windows-exception +expect_rejected "live Windows fixture exception" "scripts/test-windows-clone-contract.ps1" + +seed_fixture +printf '%s\n' 'workspace:' ' schema_version: 2' >"$fixture/scripts/fixtures/workspace-registry-smoke.yaml" +commit_fixture live-workspace-fixture +expect_rejected "live workspace fixture exception" "scripts/fixtures/workspace-registry-smoke.yaml" + +seed_fixture +write_fixture_descriptor \ + "$fixture/scripts/fixtures/workspace-registry-future.yaml" \ + 'workspace:' \ + ' schema_version: 2' +commit_fixture future-workspace-family +expect_rejected "future workspace fixture family" "scripts/fixtures/workspace-registry-future.yaml" + +# Only exact path+category policy literals are allowed; paths outside policy roots remain out of scope. +seed_fixture +mkdir -p "$fixture/docs/superpowers/plans" +printf '%s\n' 'Historical schema_version: 2 and migration_required.' >"$fixture/docs/superpowers/plans/history.md" +printf '%s\n' 'migration_required migrate-legacy WorkspaceV2 revision.state' >"$fixture/scripts/test-verify-schema-v3-only.sh" +commit_fixture exact-policy-allowlist +expect_pass "exact self-test policy allowlist and historical docs" + +# Diagnostic paths are shell-escaped so a newline cannot forge another log line. +seed_fixture +newline_spoof="$fixture/backend/src/spoof +forged.py" +printf '%s\n' harmless >"$newline_spoof" +run_gate +[[ $gate_status -eq 1 ]] || fail "newline untracked path was not rejected" +grep -Fq 'backend/src/spoof\nforged.py' "$output" \ + || fail "newline path diagnostic was not escaped on one line" + +# Scanner operational errors are propagated, not converted into absence. +seed_fixture +fake_bin="$sandbox/fake-bin" +mkdir -p "$fake_bin" +cat >"$fake_bin/git" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +for argument in "$@"; do + if [[ "$argument" == grep ]]; then + echo "simulated git grep failure" >&2 + exit 2 + fi +done +exec "$REAL_GIT" "$@" +EOF +chmod +x "$fake_bin/git" +set +e +PATH="$fake_bin:$PATH" REAL_GIT="$real_git" "$gate_bash" "$gate" --root "$fixture" --runtime-only >"$output" 2>&1 +gate_status=$? +set -e +[[ $gate_status -eq 2 ]] || fail "git grep status 2 was masked as $gate_status" +grep -Fq 'simulated git grep failure' "$output" || fail "git grep failure diagnostics were lost" + +# Foreign roots are test-only and can never select fixture code for a full check. +set +e +"$gate_bash" "$gate" --root "$fixture" >"$output" 2>&1 +gate_status=$? +set -e +[[ $gate_status -ne 0 ]] || fail "foreign-root full mode unexpectedly passed" +grep -Fq -- '--root is available only with --runtime-only or --bootstrap-trust-only' "$output" \ + || fail "foreign-root full rejection did not report the trust boundary" + +# Prescribed symbols are forbidden as case-insensitive substrings, including derivatives. +derivative_names=(WorkspaceV2Compat wOrKsPaCeV2 deprecatedV2Descriptor WRITEMIGRATEDWORKSPACE LegacyWorkspaceAdapter migrateLegacyWorkspaceCompat) +for derivative in "${derivative_names[@]}"; do + for matrix_root in "${policy_roots[@]}"; do + seed_fixture + matrix_path="$matrix_root/derivative.ts" + printf '%s\n' "$derivative" >"$fixture/$matrix_path" + commit_fixture "derivative-$derivative-${matrix_root//\//-}" + expect_rejected "derivative $derivative in $matrix_root" "$matrix_path" + done +done + +# Mixed/all-lower legacy spellings fail; the approved lower-camel identifier remains valid. +for spelling in legacyworkspace LeGaCyWoRkSpAcE; do + seed_fixture + printf '%s\n' "$spelling" >"$fixture/backend/src/legacy-variant.ts" + commit_fixture legacy-spelling + expect_rejected "legacy spelling $spelling" "backend/src/legacy-variant.ts" +done +seed_fixture +printf '%s\n' 'const legacyWorkspacePath = current;' >"$fixture/backend/src/legacy-allowed.ts" +commit_fixture lower-camel-legacy +expect_pass "approved lower-camel legacy identifier" + +# Full-text revision scanning covers bracket access and newline-separated dot access. +for revision_source in \ + 'selectedWorkspace["state"]' \ + $'workspaceRevision\n .state'; do + seed_fixture + printf '%s\n' "$revision_source" >"$fixture/frontend/src/revision-variant.ts" + commit_fixture revision-variant + expect_rejected "revision structural variant" "frontend/src/revision-variant.ts" +done +seed_fixture +mkdir -p "$fixture/backend/src/workspaces" +printf '%s\n' 'if (revision.state !== "operational") return;' >"$fixture/backend/src/workspaces/registry.ts" +commit_fixture historical-decoder +expect_pass "single exact historical decoder" +printf '%s\n' 'if (revision["state"] === "retired") return;' >>"$fixture/backend/src/workspaces/registry.ts" +commit_fixture extra-historical-branch +expect_rejected "extra registry revision branch" "backend/src/workspaces/registry.ts" + +# Binary/NUL policy files are decoded and rejected rather than skipped by git grep -I. +seed_fixture +printf 'WorkspaceV2\0hidden\n' >"$fixture/backend/src/binary.ts" +commit_fixture nul-policy +expect_rejected "NUL policy file" "backend/src/binary.ts" + +# Workspace fixture-family discovery is recursive by basename. +seed_fixture +mkdir -p "$fixture/scripts/fixtures/nested/deeper" +write_fixture_descriptor "$fixture/scripts/fixtures/nested/deeper/workspace-registry-nested.yaml" workspace: ' schema_version: 2' +commit_fixture nested-workspace-fixture +expect_rejected "nested workspace fixture" "scripts/fixtures/nested/deeper/workspace-registry-nested.yaml" + +# Python bytecode is disabled before pre-gate docs, and release dry-run starts with bootstrap trust. +grep -Fq 'PYTHONDONTWRITEBYTECODE: "1"' "$project_root/.github/workflows/deployment.yml" \ + || fail "workflow does not disable Python bytecode" +grep -Fq 'export PYTHONDONTWRITEBYTECODE=1' "$project_root/scripts/verify-schema-v3-only-release.sh" \ + || fail "release wrapper does not disable Python bytecode" +release_plan="$($gate_bash "$project_root/scripts/verify-schema-v3-only-release.sh" --dry-run)" +first_command="$(printf '%s\n' "$release_plan" | sed -n '1p')" +bootstrap_command="$(printf '%s\n' "$release_plan" | sed -n '4p')" +[[ "$first_command" == 'export PYTHONDONTWRITEBYTECODE=1' ]] \ + || fail "release dry-run does not print the Python bytecode export" +[[ "$bootstrap_command" == '/bin/bash scripts/verify-schema-v3-only.sh --bootstrap-trust-only' ]] \ + || fail "release plan does not bootstrap trust before npm" +printf '%s\n' "$release_plan" | grep -Fq '(cd backend && npm ci --ignore-scripts)' \ + || fail "release plan does not disable npm lifecycle scripts" +py_fixture="$sandbox/python-bytecode" +mkdir -p "$py_fixture/scripts" +printf '%s\n' 'value = 3' >"$py_fixture/scripts/module.py" +PYTHONPATH="$py_fixture" PYTHONDONTWRITEBYTECODE=1 python3 -c 'import scripts.module' +[[ ! -e "$py_fixture/scripts/__pycache__" ]] || fail "pre-gate Python created ignored bytecode" + +seed_bootstrap_fixture() { + seed_fixture + mkdir -p "$fixture/.github/workflows" + for required in \ + backend/package.json backend/package-lock.json \ + backend/scripts/verify-workspace-descriptor-files.mjs \ + backend/scripts/verify-workspace-descriptor-files.test.mjs \ + backend/scripts/revision-state-policy.mjs \ + backend/scripts/revision-state-policy.test.mjs \ + backend/scripts/bash-heredoc.mjs \ + backend/scripts/revision_state_policy.py \ + backend/scripts/test_revision_state_policy.py \ + scripts/verify-schema-v3-only.sh scripts/test-verify-schema-v3-only.sh \ + scripts/verify-schema-v3-only-release.sh scripts/workspace_descriptor_doc_contract.py \ + .github/workflows/deployment.yml; do + mkdir -p "$fixture/${required%/*}" + cp "$project_root/$required" "$fixture/$required" + done + "$real_git" -C "$fixture" add . + "$real_git" -C "$fixture" commit -qm bootstrap-files +} +assert_release_stops_before_npm() { + local label="$1" + fake_lifecycle="$sandbox/fake-lifecycle" + mkdir -p "$fake_lifecycle" + cat >"$fake_lifecycle/npm" <>"$sandbox/npm-invoked" +exit 99 +EOF + chmod +x "$fake_lifecycle/npm" + rm -f "$sandbox/npm-invoked" + set +e + PATH="$fake_lifecycle:$PATH" /bin/bash "$fixture/scripts/verify-schema-v3-only-release.sh" >"$output" 2>&1 + release_status=$? + set -e + [[ $release_status -ne 0 ]] || fail "$label unexpectedly passed" + [[ ! -e "$sandbox/npm-invoked" ]] || fail "$label invoked npm before bootstrap trust" +} + +seed_bootstrap_fixture +printf '%s\n' '# dirty' >>"$fixture/backend/package.json" +assert_release_stops_before_npm "dirty package bootstrap" +seed_bootstrap_fixture +printf '%s\n' '# dirty' >>"$fixture/backend/scripts/verify-workspace-descriptor-files.test.mjs" +assert_release_stops_before_npm "dirty checker test bootstrap" +seed_bootstrap_fixture +printf '%s\n' '// dirty' >>"$fixture/backend/scripts/revision-state-policy.mjs" +assert_release_stops_before_npm "dirty revision policy bootstrap" +seed_bootstrap_fixture +printf '%s\n' '# dirty' >>"$fixture/backend/scripts/revision_state_policy.py" +assert_release_stops_before_npm "dirty Python policy helper bootstrap" +seed_bootstrap_fixture +printf '%s\n' '# dirty' >>"$fixture/scripts/verify-schema-v3-only.sh" +assert_release_stops_before_npm "dirty gate bootstrap" +seed_bootstrap_fixture +rm "$fixture/backend/scripts/verify-workspace-descriptor-files.mjs" +ln -s /dev/null "$fixture/backend/scripts/verify-workspace-descriptor-files.mjs" +assert_release_stops_before_npm "symlink checker bootstrap" +seed_bootstrap_fixture +printf '%s\n' 'scripts/__pycache__/' >"$fixture/.gitignore" +"$real_git" -C "$fixture" add .gitignore +"$real_git" -C "$fixture" commit -qm ignore-rule +mkdir -p "$fixture/scripts/__pycache__" +printf x >"$fixture/scripts/__pycache__/ignored.pyc" +assert_release_stops_before_npm "ignored trusted artifact bootstrap" +seed_bootstrap_fixture +printf x >"$fixture/scripts/untracked-helper.sh" +assert_release_stops_before_npm "untracked helper bootstrap" + +seed_bootstrap_fixture +mkfifo "$fixture/scripts/bootstrap-fifo" +assert_release_stops_before_npm "FIFO bootstrap" +seed_bootstrap_fixture +printf '%s\n' unsafe >"$fixture/backend/.npmrc" +assert_release_stops_before_npm "untracked backend npmrc bootstrap" +seed_bootstrap_fixture +global_ignore="$sandbox/global-ignore" +printf '%s\n' backend/.npmrc >"$global_ignore" +"$real_git" -C "$fixture" config core.excludesFile "$global_ignore" +printf '%s\n' unsafe >"$fixture/backend/.npmrc" +assert_release_stops_before_npm "globally ignored backend npmrc bootstrap" + +# Dist failures are isolated to fixture roots; canonical backend/dist is never mutated. +run_gate_dist() { + set +e + "$gate_bash" "$gate" --root "$fixture" --runtime-only --check-dist >"$output" 2>&1 + gate_status=$? + set -e +} + +seed_fixture +mkdir -p "$fixture/backend/dist" +printf '%s\n' server >"$fixture/backend/dist/server.js" +run_gate_dist +[[ $gate_status -eq 1 ]] || fail "fixture missing schema module unexpectedly passed" +grep -Fq 'backend/dist/workspaces/schema.js' "$output" || fail "fixture missing schema path not reported" + +seed_fixture +mkdir -p "$fixture/backend/dist/workspaces" +printf '%s\n' schema >"$fixture/backend/dist/workspaces/schema.js" +run_gate_dist +[[ $gate_status -eq 1 ]] || fail "fixture missing server unexpectedly passed" +grep -Fq 'backend/dist/server.js' "$output" || fail "fixture missing server path not reported" + +seed_fixture +mkdir -p "$fixture/backend/dist/workspaces" +printf '%s\n' schema >"$fixture/backend/dist/workspaces/schema.js" +printf '%s\n' server >"$fixture/backend/dist/server.js" +printf '%s\n' stale >"$fixture/backend/dist/workspaces/Migrate-Legacy.js" +run_gate_dist +[[ $gate_status -eq 1 ]] || fail "fixture stale migrator unexpectedly passed" +grep -Fq 'backend/dist/workspaces/Migrate-Legacy.js' "$output" || fail "fixture stale migrator path not reported" + +[[ "$(cksum <"$canonical_schema")" == "$canonical_schema_checksum" ]] \ + || fail "shell regression mutated canonical compiled schema" +[[ "$(cksum <"$canonical_server")" == "$canonical_server_checksum" ]] \ + || fail "shell regression mutated canonical compiled server" + +echo "schema-v3-only absence gate regression tests passed" diff --git a/scripts/unified-deployment-smoke.sh b/scripts/unified-deployment-smoke.sh index 8c308375..02879a28 100755 --- a/scripts/unified-deployment-smoke.sh +++ b/scripts/unified-deployment-smoke.sh @@ -478,31 +478,8 @@ task13_seed_registry() { task13_run_logged "initialize bare workspace registry" \ git init --bare --initial-branch=main "$TASK13_REMOTE" task13_run_logged "initialize workspace seed" git -C "$TASK13_SEED" init --initial-branch=main - cat >"$TASK13_SEED/workspaces/task13-smoke.yaml" <<'EOF' -workspace: - schema_version: 3 - id: task13-smoke - name: Task 13 Smoke - language: en -dwh: - engine: postgres - database: warehouse - schema: analytics - supported_transports: [postgres_direct] -semantic_index: - vector_store: - engine: qdrant - collection: task13-smoke - dimensions: 1024 - distance: cosine - embedding: - provider: ollama_internal - model: qwen3-embedding:0.6b - dimensions: 1024 -llm_policy: - default: local-qwen/task13-smoke - allowed: [local-qwen/task13-smoke] -EOF + cp "$TASK13_ROOT/scripts/fixtures/workspace-registry-task13.yaml" \ + "$TASK13_SEED/workspaces/task13-smoke.yaml" task13_run_logged "commit initial workspace" git -C "$TASK13_SEED" add workspaces/task13-smoke.yaml task13_run_logged "commit initial workspace" git -C "$TASK13_SEED" \ -c user.name='Task 13 Smoke' -c user.email='task13-smoke@example.invalid' \ diff --git a/scripts/verify-schema-v3-only-release.sh b/scripts/verify-schema-v3-only-release.sh new file mode 100755 index 00000000..27f85b6d --- /dev/null +++ b/scripts/verify-schema-v3-only-release.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# Reproducible schema-v3-only release gate. The Git checkout is the trust root; +# dependencies come from backend/package-lock.json and dist comes from a clean build. +set -euo pipefail +export PYTHONDONTWRITEBYTECODE=1 +export NPM_CONFIG_USERCONFIG=/dev/null +export NPM_CONFIG_GLOBALCONFIG=/dev/null +root="$(cd "$(dirname "$0")/.." && pwd -P)" +if [[ ${1:-} == --dry-run ]]; then + cat <<'EOF' +export PYTHONDONTWRITEBYTECODE=1 +export NPM_CONFIG_USERCONFIG=/dev/null +export NPM_CONFIG_GLOBALCONFIG=/dev/null +/bin/bash scripts/verify-schema-v3-only.sh --bootstrap-trust-only +(cd backend && npm ci --ignore-scripts) +(cd backend && npm run build) +(cd backend && npm run test:schema-v3-verifier) +/bin/bash scripts/test-verify-schema-v3-only.sh +/bin/bash scripts/verify-schema-v3-only.sh +EOF + exit 0 +fi +[[ $# -eq 0 ]] || { echo "usage: $0 [--dry-run]" >&2; exit 2; } +/bin/bash "$root/scripts/verify-schema-v3-only.sh" --bootstrap-trust-only +(cd "$root/backend" && npm ci --ignore-scripts) +(cd "$root/backend" && npm run build) +(cd "$root/backend" && npm run test:schema-v3-verifier) +/bin/bash "$root/scripts/test-verify-schema-v3-only.sh" +/bin/bash "$root/scripts/verify-schema-v3-only.sh" diff --git a/scripts/verify-schema-v3-only.sh b/scripts/verify-schema-v3-only.sh new file mode 100755 index 00000000..12d83d59 --- /dev/null +++ b/scripts/verify-schema-v3-only.sh @@ -0,0 +1,278 @@ +#!/usr/bin/env bash +# Fail-closed absence gate for the supported schema-v3-only workspace runtime. +set -euo pipefail +shopt -s nocasematch + +script_root="$(cd "$(dirname "$0")/.." && pwd -P)" +root_argument="$script_root" +root_was_selected=0 +runtime_only=0 +check_dist=0 +bootstrap_trust_only=0 + +usage() { + cat >&2 <&2; usage; exit 2 ;; + esac +done +[[ $root_was_selected -eq 0 || $runtime_only -eq 1 || $bootstrap_trust_only -eq 1 ]] || { echo "--root is available only with --runtime-only or --bootstrap-trust-only" >&2; exit 2; } +[[ $runtime_only -eq 0 || $bootstrap_trust_only -eq 0 ]] || { echo "--runtime-only and --bootstrap-trust-only are mutually exclusive" >&2; exit 2; } +[[ $check_dist -eq 0 || $runtime_only -eq 1 ]] || { echo "--check-dist is available only with --runtime-only" >&2; exit 2; } +if ! root="$(cd "$root_argument" 2>/dev/null && pwd -P)"; then + printf 'repository root is not accessible: %q\n' "$root_argument" >&2 + exit 2 +fi +[[ $runtime_only -eq 1 || $bootstrap_trust_only -eq 1 || "$root" == "$script_root" ]] || { echo "full mode is restricted to the canonical repository" >&2; exit 2; } + +tmp="$(mktemp -d "${TMPDIR:-/tmp}/thoth-v3-gate.XXXXXX")" +trap 'rm -rf "$tmp"' EXIT HUP INT TERM + +if git_top="$(git -C "$root" rev-parse --show-toplevel 2>"$tmp/rev-parse")"; then :; else + status=$?; printf 'Git repository discovery failed for %q\n' "$root" >&2; cat "$tmp/rev-parse" >&2; exit "$status" +fi +canonical_git_top="$(cd "$git_top" && pwd -P)" +[[ "$canonical_git_top" == "$root" ]] || { printf '%s\n' "--root must name the canonical Git root" >&2; exit 2; } +for npmrc in .npmrc backend/.npmrc; do + if [[ -e "$root/$npmrc" || -L "$root/$npmrc" ]]; then + printf 'npm configuration node forbidden: %q\n' "$npmrc" >&2 + exit 1 + fi +done + +policy_roots=(backend/src frontend/src backend/scripts scripts) +trust_roots=(backend/src frontend/src backend/scripts scripts deploy/workspaces) + +print_path() { printf '%q' "$1"; } +fail_path() { local message="$1" path="$2"; printf '%s: ' "$message" >&2; print_path "$path" >&2; printf '\n' >&2; return 1; } + +forbidden_module_stems='deprecated-v2-descriptor|migrate-legacy|migrate-v2-qdrant' +is_deleted_basename() { + [[ "${1##*/}" =~ ^($forbidden_module_stems)(\..*)?$ ]] +} + +# Exact path + category exceptions only. They remain fully subject to trust checks. +is_allowed_match() { + local category="$1" path="$2" + case "$category:$path" in + prescribed-symbol:scripts/verify-schema-v3-only.sh|prescribed-symbol:scripts/test-verify-schema-v3-only.sh|prescribed-symbol:backend/scripts/verify-workspace-descriptor-files.mjs|prescribed-symbol:backend/scripts/verify-workspace-descriptor-files.test.mjs|prescribed-symbol:backend/scripts/revision-state-policy.mjs|prescribed-symbol:backend/scripts/revision-state-policy.test.mjs|prescribed-symbol:backend/scripts/bash-heredoc.mjs|prescribed-symbol:backend/scripts/revision_state_policy.py|prescribed-symbol:backend/scripts/test_revision_state_policy.py) return 0 ;; + legacy-workspace:scripts/verify-schema-v3-only.sh|legacy-workspace:scripts/test-verify-schema-v3-only.sh|legacy-workspace:backend/scripts/verify-workspace-descriptor-files.mjs|legacy-workspace:backend/scripts/verify-workspace-descriptor-files.test.mjs|legacy-workspace:backend/scripts/revision-state-policy.mjs|legacy-workspace:backend/scripts/revision-state-policy.test.mjs|legacy-workspace:backend/scripts/bash-heredoc.mjs|legacy-workspace:backend/scripts/revision_state_policy.py|legacy-workspace:backend/scripts/test_revision_state_policy.py) return 0 ;; + migration-marker:scripts/verify-schema-v3-only.sh|migration-marker:scripts/test-verify-schema-v3-only.sh|migration-marker:backend/scripts/verify-workspace-descriptor-files.mjs|migration-marker:backend/scripts/verify-workspace-descriptor-files.test.mjs|migration-marker:backend/scripts/revision-state-policy.mjs|migration-marker:backend/scripts/revision-state-policy.test.mjs|migration-marker:backend/scripts/bash-heredoc.mjs|migration-marker:backend/scripts/revision_state_policy.py|migration-marker:backend/scripts/test_revision_state_policy.py) return 0 ;; + migration-marker:scripts/workspace_descriptor_doc_contract.py|migration-marker:scripts/test_workspace_descriptor_doc_contract.py) return 0 ;; + migration-marker:backend/scripts/clean-dist.test.mjs) return 0 ;; + *) return 1 ;; + esac +} + +# Common policy is defined once and scanned over every policy root. Revision-state is the sole layer. +prescribed_symbol_forbidden='WorkspaceV1|WorkspaceV2|DeprecatedV2Descriptor|LegacyMigrationResult|LegacyMigrationOptions|WorkspaceV2MigrationInput|migrateLegacyWorkspace|writeMigratedWorkspace|migrateWorkspaceV1ToV2|migrateWorkspaceV2ToV3' +legacy_workspace_forbidden='LegacyWorkspace' +migration_marker_forbidden="migration_required|($forbidden_module_stems)" + +require_category_absent() { + local category="$1" sensitivity="$2" pattern="$3" output="$tmp/grep-$1" status path + if [[ "$sensitivity" == insensitive ]]; then + if git -C "$root" grep -z -l -I -i -E -e "$pattern" -- "${policy_roots[@]}" >"$output" 2>"$output.err"; then status=0; else status=$?; fi + else + if git -C "$root" grep -z -l -I -E -e "$pattern" -- "${policy_roots[@]}" >"$output" 2>"$output.err"; then status=0; else status=$?; fi + fi + case "$status" in + 0) + while IFS= read -r -d '' path; do + is_allowed_match "$category" "$path" && continue + printf 'forbidden %s match: ' "$category" >&2; print_path "$path" >&2; printf '\n' >&2 + return 1 + done <"$output" + ;; + 1) : ;; + *) printf 'scanner failure (%s): %s\n' "$status" "$category" >&2; cat "$output.err" >&2; return "$status" ;; + esac +} + +# One batched index inventory validates modes and working-tree presence for all tracked paths. +index_entries="$tmp/index" +if git -C "$root" ls-files -s -z -- "${trust_roots[@]}" >"$index_entries" 2>"$tmp/index.err"; then :; else + status=$?; echo "tracked index scan failed ($status)" >&2; cat "$tmp/index.err" >&2; exit "$status" +fi +tracked_paths="$tmp/tracked" +: >"$tracked_paths" +while IFS= read -r -d '' record; do + metadata="${record%%$'\t'*}"; path="${record#*$'\t'}"; mode="${metadata%% *}" + case "$mode" in 100*) ;; 120000) fail_path "tracked symlink forbidden" "$path"; exit 1 ;; *) fail_path "non-regular tracked entry forbidden" "$path"; exit 1 ;; esac + printf '%s\0' "$path" >>"$tracked_paths" + [[ -f "$root/$path" && ! -L "$root/$path" ]] || { fail_path "tracked file missing or unsafe" "$path"; exit 1; } + is_deleted_basename "$path" && { fail_path "deleted source basename remains tracked" "$path"; exit 1; } +done <"$index_entries" + +# Filesystem node trust has no test/fixture exclusions. +filesystem="$tmp/filesystem" +if find "${trust_roots[@]/#/$root/}" -mindepth 1 -print0 >"$filesystem" 2>"$tmp/find.err"; then :; else + status=$?; echo "filesystem trust scan failed ($status)" >&2; cat "$tmp/find.err" >&2; exit "$status" +fi +while IFS= read -r -d '' absolute; do + path="${absolute#"$root/"}" + [[ ! -L "$absolute" ]] || { fail_path "symlink forbidden in trusted root" "$path"; exit 1; } + [[ -d "$absolute" || -f "$absolute" ]] || { fail_path "non-directory/non-regular node forbidden in trusted root" "$path"; exit 1; } + is_deleted_basename "$path" && { fail_path "deleted source basename remains on filesystem" "$path"; exit 1; } +done <"$filesystem" + +reject_name_list() { + local label="$1" file="$2" path + while IFS= read -r -d '' path; do fail_path "$label" "$path"; return 1; done <"$file" +} +for spec in "untracked:--others --exclude-standard" "ignored:--others --ignored --exclude-standard"; do + label="${spec%%:*}"; options="${spec#*:}"; output="$tmp/$label" + # shellcheck disable=SC2086 + if git -C "$root" ls-files -z $options -- "${trust_roots[@]}" >"$output" 2>"$output.err"; then :; else + status=$?; echo "$label scan failed ($status)" >&2; cat "$output.err" >&2; exit "$status" + fi + reject_name_list "$label file in trusted root" "$output" || exit 1 +done + +for mode in worktree cached; do + output="$tmp/dirty-$mode" + if [[ "$mode" == cached ]]; then command=(git -C "$root" diff --cached --name-only -z --); else command=(git -C "$root" diff --name-only -z --); fi + if "${command[@]}" "${trust_roots[@]}" >"$output" 2>"$output.err"; then :; else + status=$?; echo "$mode dirty scan failed ($status)" >&2; cat "$output.err" >&2; exit "$status" + fi + reject_name_list "modified trusted file ($mode)" "$output" || exit 1 +done + +require_trusted_files_at() { + local repository="$1"; shift + local listing="$tmp/explicit-$RANDOM" record metadata path mode expected + if git -C "$repository" ls-files -s -z -- "$@" >"$listing" 2>"$listing.err"; then :; else + status=$?; echo "explicit trust index scan failed ($status)" >&2; cat "$listing.err" >&2; return "$status" + fi + : >"$listing.paths" + while IFS= read -r -d '' record; do + metadata="${record%%$'\t'*}"; path="${record#*$'\t'}"; mode="${metadata%% *}" + case "$mode" in 100*) ;; *) fail_path "required file is not regular in index" "$path"; return 1 ;; esac + printf '%s\n' "$path" >>"$listing.paths" + done <"$listing" + for expected in "$@"; do + [[ -f "$repository/$expected" && ! -L "$repository/$expected" ]] || { fail_path "required trusted file missing or unsafe" "$expected"; return 1; } + grep -Fqx -- "$expected" "$listing.paths" || { fail_path "required file is not tracked" "$expected"; return 1; } + done + git -C "$repository" diff --quiet -- "$@" || { echo "required trust files are dirty" >&2; return 1; } + git -C "$repository" diff --quiet --cached -- "$@" || { echo "required trust files are staged dirty" >&2; return 1; } +} + +# Bootstrap uses only Git/filesystem primitives. It must precede every npm or +# checkout-controlled helper in the durable release wrapper. +bootstrap_files=( + backend/package.json backend/package-lock.json + backend/scripts/verify-workspace-descriptor-files.mjs + backend/scripts/verify-workspace-descriptor-files.test.mjs + backend/scripts/revision-state-policy.mjs + backend/scripts/revision-state-policy.test.mjs + backend/scripts/bash-heredoc.mjs + backend/scripts/revision_state_policy.py + backend/scripts/test_revision_state_policy.py + scripts/verify-schema-v3-only.sh scripts/test-verify-schema-v3-only.sh + scripts/verify-schema-v3-only-release.sh .github/workflows/deployment.yml + scripts/workspace_descriptor_doc_contract.py +) +if [[ $bootstrap_trust_only -eq 1 ]]; then + require_trusted_files_at "$root" "${bootstrap_files[@]}" + echo "schema-v3-only bootstrap trust passed" + exit 0 +fi + +# Runtime fixtures execute only canonical trusted verifier code and dependencies. +require_trusted_files_at "$script_root" \ + backend/scripts/verify-workspace-descriptor-files.mjs \ + backend/scripts/revision-state-policy.mjs \ + backend/scripts/bash-heredoc.mjs \ + backend/scripts/revision_state_policy.py \ + backend/package.json backend/package-lock.json \ + scripts/verify-schema-v3-only.sh +workspace_verifier="$script_root/backend/scripts/verify-workspace-descriptor-files.mjs" +workspace_schema="$script_root/backend/dist/workspaces/schema.js" + +if [[ $runtime_only -eq 0 ]]; then + require_trusted_files_at "$root" \ + scripts/verify-schema-v3-only-release.sh .github/workflows/deployment.yml + (cd "$root/backend" && npm run build) +fi +[[ -f "$workspace_schema" && ! -L "$workspace_schema" ]] || { echo "trusted compiled schema module is missing or unsafe: backend/dist/workspaces/schema.js" >&2; exit 1; } + +workspace_manifest="$tmp/workspace-manifest" +: >"$workspace_manifest" +while IFS= read -r -d '' path; do + case "$path" in + backend/src/*|frontend/src/*|backend/scripts/*|scripts/*) printf '%s\0%s\0' policy_text "$path" >>"$workspace_manifest" ;; + esac + kind="" + case "$path" in + deploy/workspaces/preprocess-dwh.yaml|deploy/workspaces/preprocess-evidence.yaml|deploy/workspaces/server-sessions.yaml.example) ;; + deploy/workspaces/*.yaml|deploy/workspaces/*.yml|deploy/workspaces/*.yaml.example|deploy/workspaces/*.yml.example|scripts/fixtures/workspace-registry-*.yaml|scripts/fixtures/workspace-registry-*.yml|scripts/fixtures/*/workspace-registry-*.yaml|scripts/fixtures/*/workspace-registry-*.yml) kind=workspace_descriptor ;; + scripts/*.sh|scripts/*.ps1) + case "$path" in scripts/verify-schema-v3-only.sh|scripts/test-verify-schema-v3-only.sh) ;; *) kind=deployment_script ;; esac + ;; + esac + [[ -z "$kind" ]] || printf '%s\0%s\0' "$kind" "$path" >>"$workspace_manifest" +done <"$tracked_paths" +node "$workspace_verifier" --root "$root" --manifest "$workspace_manifest" + +require_category_absent prescribed-symbol insensitive "$prescribed_symbol_forbidden" +require_category_absent legacy-workspace sensitive "$legacy_workspace_forbidden" +require_category_absent migration-marker insensitive "$migration_marker_forbidden" + +check_dist_tree() { + local dist_root="$1" entries="$tmp/dist" absolute path + [[ -f "$dist_root/backend/dist/workspaces/schema.js" && ! -L "$dist_root/backend/dist/workspaces/schema.js" ]] || { echo "expected compiled schema module is missing or unsafe: backend/dist/workspaces/schema.js" >&2; return 1; } + [[ -f "$dist_root/backend/dist/server.js" && ! -L "$dist_root/backend/dist/server.js" ]] || { echo "expected compiled backend output is missing or unsafe: backend/dist/server.js" >&2; return 1; } + find "$dist_root/backend/dist" -mindepth 1 -print0 >"$entries" + while IFS= read -r -d '' absolute; do + path="${absolute#"$dist_root/"}" + if is_deleted_basename "$path"; then + fail_path "stale compiled workspace migrator output exists" "$path" + return 1 + fi + done <"$entries" +} +[[ $runtime_only -eq 1 && $check_dist -eq 0 ]] || check_dist_tree "$root" + +if [[ $runtime_only -eq 0 ]]; then + require_trusted_files_at "$root" \ + scripts/workspace_descriptor_doc_contract.py README.md PROJECT_STATE.md \ + docs/install/local-workspace-registry.md docs/install/server-workspace-registry.md \ + docs/workspace-diagnostic-protocol.md + "$root/scripts/workspace_descriptor_doc_contract.py" \ + --document "$root/README.md" --project-state "$root/PROJECT_STATE.md" \ + --document "$root/docs/install/local-workspace-registry.md" \ + --document "$root/docs/install/server-workspace-registry.md" \ + --document "$root/docs/workspace-diagnostic-protocol.md" +fi + +echo "schema-v3-only absence gate passed"