ThothII/backend/scripts/verify-workspace-descriptor-files.mjs

#!/usr/bin/env node
import { createHash } from "node:crypto";
import { lstat, readFile, realpath } from "node:fs/promises";
import { isAbsolute, relative, resolve, sep } from "node:path";
import { fileURLToPath, pathToFileURL } from "node:url";

import { isMap, isScalar, parseAllDocuments } from "yaml";
import { extractBashDocuments } from "./bash-heredoc.mjs";
import { validatePythonRevisionStates, validateRevisionState } from "./revision-state-policy.mjs";
import { parseWorkspaceYaml } from "../dist/workspaces/schema.js";

const scriptPath = fileURLToPath(import.meta.url);
const allowedKinds = new Set(["policy_text", "workspace_descriptor", "deployment_script"]);
// Exact-content trust exceptions. Each digest covers the raw UTF-8 bytes from the
// opener line through the closer line (including physical line endings). These
// blocks are reviewed non-workspace runtime/config generation, not semantic proof.
const reviewedExpandableBlocks = new Map([
  ["scripts/test-dwh-auth-nginx-integration.sh", [
    { sha256: "ead57234ad3520b5c7d4262b772957cbc7b9589da4f35fb17b160f948eb2ac7b", rationale: "Generates the reviewed isolated Nginx integration configuration." },
  ]],
  ["scripts/test-install-tht.sh", [
    { sha256: "37f18ce7ce93cb8b84f3b3708462cc16d50fdc7bab22836c382dbacf8382f05f", rationale: "Generates the reviewed synthetic tht installer artifact." },
  ]],
  ["scripts/test-server-pi-state-topology.sh", [
    { sha256: "6ae9567db53d6cd45a2c19c98acaf45f382450b157ea7d6f6d35125f68c50947", rationale: "Generates the isolated server topology test environment, including its installation descriptor and authentication configuration root." },
  ]],
  ["scripts/test-vector-backup-restore-safety.sh", [
    { sha256: "40b8a10a3c06aaa98e324fbf688b7d1f5cead330d7ba7eef98e06256d412a85a", rationale: "Generates the reviewed restore safety manifest." },
  ]],
  ["scripts/test-windows-clone-contract.ps1", [
    { sha256: "3204f772d33cad42bcac99191507051aefb2c91d2935bec6698b956e44f9bf45", rationale: "Generates reviewed Windows clone test configuration with its authentication configuration root." },
    { sha256: "f4814d842a7502b7ef30fd6b224d5cb17b0ffd6fb2367c41c49ac16587536d93", rationale: "Same reviewed block in the repository-required CRLF checkout representation." },
    { sha256: "6f25ce3b58cea47b74fe9319ed917d8089a2fb334bc0d469daa7e1f10865d870", rationale: "Generates the reviewed Windows Compose override for the canonical service topology." },
    { sha256: "45a3cf19f7ce697b858b63d27a4edc7fefa2414d0408e7b6d72a65c86d314f5b", rationale: "Same reviewed Compose override in the repository-required CRLF checkout representation." },
    { sha256: "5d0d1a3fc45e99b3aacaf4ee5dd09a6bee1937784375dfe4bcfaa4ae32cfb9de", rationale: "Generates reviewed Windows clone test configuration." },
    { sha256: "b903e5dae953ae1372f1a5276f12a92ed3dd632b897f3afe5e00c646d90a1b42", rationale: "Same reviewed block in the repository-required CRLF checkout representation." },
  ]],
  ["scripts/unified-deployment-smoke.sh", [
    { sha256: "b6c0826151b2c8b955399d1abf5b691cc8fe6b6454b17da000dde7ba3bc55d2d", rationale: "Generates the reviewed local Task 13 Compose override with normalized catalog mounts." },
    { sha256: "24f69d12b8554aa2bebba455be99fde3e60743eef5a40fa2ef5b29397a477c03", rationale: "Generates the reviewed local Task 13 installation descriptor with its model catalog." },
    { sha256: "526006fa6d48a8080b3834723630c64de5005a67243e944ebf1da15212b4d654", rationale: "Generates the reviewed server Task 13 Compose override." },
    { sha256: "406ccead1967f642225c946fc4a23fe5b019c9764cc5153e1125876ade16ec90", rationale: "Generates the reviewed projected-auth server Task 13 installation descriptor with its model catalog." },
  ]],
  ["scripts/vector-backup.sh", [
    { sha256: "571899db49dfdcec8107fbe1e0a86a61e7581979d3c4c248c20546843e275bcf", rationale: "Generates the reviewed backup manifest inside the helper command." },
  ]],
  ["scripts/vector-restore.sh", [
    { sha256: "f04d872e556a7323583c6e620b25814fb6a8e2568a9a555623978185b473a49d", rationale: "Feeds reviewed parsed manifest values to read loops." },
    { sha256: "c6053ed44abae71ae4821b68f9a513f8070947350e30d89ae0f65bf4a48f66fd", rationale: "Feeds reviewed parsed manifest values to read loops." },
  ]],
]);

function blockDigest(rawBlock) {
  return createHash("sha256").update(rawBlock, "utf8").digest("hex");
}

function reviewedExpandableBlock(path, rawBlock) {
  const digest = blockDigest(rawBlock);
  return (reviewedExpandableBlocks.get(path) ?? []).some((review) => review.sha256 === digest);
}

function hasAmbiguousExpansion(source, path) {
  const powershell = path.endsWith(".ps1");
  for (let index = 0; index < source.length; index += 1) {
    const character = source[index];
    if (powershell && character === "`") {
      index += 1;
      continue;
    }
    if (!powershell && character === "\\") {
      index += 1;
      continue;
    }
    if (character === "$" || (!powershell && character === "`")) return true;
  }
  return false;
}

function physicalLines(source) {
  const rawLines = source.match(/[^\n]*\n|[^\n]+$/gu) ?? [];
  if (rawLines.length === 0) rawLines.push("");
  return rawLines.map((raw) => ({ raw, text: raw.replace(/\n$/u, "").replace(/\r$/u, "") }));
}
const prescribedSymbols = [
  "WorkspaceV1", "WorkspaceV2", "DeprecatedV2Descriptor", "LegacyMigrationResult",
  "LegacyMigrationOptions", "WorkspaceV2MigrationInput", "migrateLegacyWorkspace",
  "writeMigratedWorkspace", "migrateWorkspaceV1ToV2", "migrateWorkspaceV2ToV3",
];
const migrationMarkers = ["migration_required", "deprecated-v2-descriptor", "migrate-legacy", "migrate-v2-qdrant"];

function isPolicyImplementationException(label, category) {
  const implementations = new Set([
    "scripts/verify-schema-v3-only.sh",
    "scripts/test-verify-schema-v3-only.sh",
    "backend/scripts/verify-workspace-descriptor-files.mjs",
    "backend/scripts/verify-workspace-descriptor-files.test.mjs",
    "backend/scripts/revision-state-policy.mjs",
    "backend/scripts/revision-state-policy.test.mjs",
    "backend/scripts/bash-heredoc.mjs",
    "backend/scripts/revision_state_policy.py",
    "backend/scripts/test_revision_state_policy.py",
  ]);
  if (implementations.has(label)) return true;
  if (category === "migration-marker" && new Set([
    "backend/src/workspaces/schema.ts",
    "scripts/workspace_descriptor_doc_contract.py",
    "scripts/test_workspace_descriptor_doc_contract.py",
    "backend/scripts/clean-dist.test.mjs",
  ]).has(label)) return true;
  return false;
}


function validatePolicySource(source, label) {
  if (!isPolicyImplementationException(label, "prescribed-symbol")) {
    for (const symbol of prescribedSymbols) {
      if (source.toLowerCase().includes(symbol.toLowerCase())) throw new Error(`${label}: forbidden prescribed-symbol substring: ${symbol}`);
    }
  }
  if (!isPolicyImplementationException(label, "migration-marker")) {
    for (const marker of migrationMarkers) {
      if (source.toLowerCase().includes(marker.toLowerCase())) throw new Error(`${label}: forbidden migration-marker substring: ${marker}`);
    }
  }
  if (!isPolicyImplementationException(label, "legacy-workspace")) {
    for (const match of source.matchAll(/legacyworkspace/giu)) {
      if (match[0] !== "legacyWorkspace") throw new Error(`${label}: forbidden legacy-workspace spelling: ${match[0]}`);
    }
  }
  if (!/\.pyw?$/iu.test(label) && !isPolicyImplementationException(label, "revision-state")) validateRevisionState(source, label);
}

function documentShape(document) {
  const shape = { workspacePresent: false, workspaceMapping: false };
  if (!isMap(document.contents)) return shape;
  for (const pair of document.contents.items) {
    if (!isScalar(pair.key)) continue;
    if (pair.key.value === "workspace") {
      shape.workspacePresent = true;
      if (isMap(pair.value)) shape.workspaceMapping = true;
    }
  }
  return shape;
}

function documents(source) {
  try {
    return parseAllDocuments(source, { uniqueKeys: true });
  } catch (error) {
    throw new Error(`YAML parser failed: ${error instanceof Error ? error.message : String(error)}`);
  }
}

function validateWorkspaceSource(source, label, { requireWorkspace, expandable = false, path, rawBlock }) {
  const parsed = documents(source);
  const shapes = parsed.map(documentShape);
  if (requireWorkspace) {
    if (!shapes.some((shape) => shape.workspacePresent)) {
      throw new Error(`${label}: expected a top-level workspace mapping`);
    }
    if (!shapes.some((shape) => shape.workspaceMapping)) {
      throw new Error(`${label}: top-level workspace must be a mapping`);
    }
  } else {
    if (expandable && hasAmbiguousExpansion(source, path) && !reviewedExpandableBlock(path, rawBlock)) {
      throw new Error(`${label}: expandable block interpolation is not in the exact-content reviewed allowlist`);
    }
    if (shapes.some((shape) => shape.workspaceMapping)) {
      throw new Error(`${label}: embedded workspace descriptor is forbidden; use a tracked workspace fixture`);
    }
    return false;
  }
  try {
    parseWorkspaceYaml(source);
  } catch (error) {
    throw new Error(`${label}: workspace descriptor is not valid schema v4: ${error instanceof Error ? error.message : String(error)}`);
  }
  return true;
}

function deploymentScriptDialect(path) {
  if (path.endsWith(".sh")) return "bash";
  if (path.endsWith(".ps1")) return "powershell";
  throw new Error(`${path}: unknown deployment script dialect`);
}


function powerShellHereStringOpener(line, state) {
  let quote = null;
  for (let index = 0; index < line.length; index += 1) {
    if (state.blockComment) {
      const close = line.indexOf("#>", index);
      if (close < 0) return null;
      state.blockComment = false;
      index = close + 1;
      continue;
    }
    const character = line[index];
    if (quote === null && character === "`") {
      index += 1;
      continue;
    }
    if (quote === "'") {
      if (character === "'" && line[index + 1] === "'") index += 1;
      else if (character === "'") quote = null;
      continue;
    }
    if (quote === '"') {
      if (character === "`") index += 1;
      else if (character === '"') quote = null;
      continue;
    }
    if (character === "#") return null;
    if (character === "<" && line[index + 1] === "#") {
      state.blockComment = true;
      index += 1;
      continue;
    }
    if (character === "@" && (line[index + 1] === "'" || line[index + 1] === '"') && /^[ \t]*$/u.test(line.slice(index + 2))) return line[index + 1];
    if (character === "'" || character === '"') quote = character;
  }
  return null;
}

function extractPowerShellDocuments(source, label) {
  const records = physicalLines(source);
  const lines = records.map((record) => record.text);
  const extracted = [];
  const state = { blockComment: false };
  for (let index = 0; index < lines.length; index += 1) {
    const quote = powerShellHereStringOpener(lines[index], state);
    if (quote === null) continue;
    const delimiter = `${quote}@`;
    const opener = index;
    const body = [];
    const start = index + 2;
    let closed = false;
    for (index += 1; index < lines.length; index += 1) {
      if (lines[index].trimEnd() === delimiter) {
        closed = true;
        break;
      }
      body.push(lines[index]);
    }
    extracted.push({
      source: `${body.join("\n")}\n`,
      label: `${label}:${start} PowerShell here-string${closed ? "" : " (unclosed)"}`,
      expandable: quote === '"',
      path: label,
      rawBlock: records.slice(opener, Math.min(index + 1, records.length)).map((record) => record.raw).join(""),
    });
  }
  return extracted;
}

export function extractScriptDocuments(source, label = "deployment script") {
  const dialect = deploymentScriptDialect(label);
  if (dialect === "bash") return extractBashDocuments(source, label);
  return extractPowerShellDocuments(source, label);
}

async function safeFile(root, path) {
  if (typeof path !== "string" || path.length === 0 || isAbsolute(path) || path.includes("\\")) {
    throw new Error(`unsafe verifier path: ${JSON.stringify(path)}`);
  }
  const segments = path.split("/");
  if (segments.some((segment) => segment === "" || segment === "." || segment === "..")) {
    throw new Error(`unsafe verifier path: ${JSON.stringify(path)}`);
  }
  const absolute = resolve(root, ...segments);
  const fromRoot = relative(root, absolute);
  if (fromRoot.startsWith(`..${sep}`) || fromRoot === ".." || isAbsolute(fromRoot)) {
    throw new Error(`verifier path escapes root: ${JSON.stringify(path)}`);
  }
  const entry = await lstat(absolute);
  if (!entry.isFile() || entry.isSymbolicLink()) {
    throw new Error(`verifier input is not a regular file: ${path}`);
  }
  const canonical = await realpath(absolute);
  const canonicalRelative = relative(root, canonical);
  if (canonicalRelative.startsWith(`..${sep}`) || canonicalRelative === ".." || isAbsolute(canonicalRelative)) {
    throw new Error(`verifier input resolves outside root: ${path}`);
  }
  return absolute;
}

export async function verifyEntries({ root, entries }) {
  const canonicalRoot = await realpath(root);
  const seen = new Set();
  const pythonPolicies = [];
  for (const entry of entries) {
    if (!entry || !allowedKinds.has(entry.kind) || typeof entry.path !== "string") {
      throw new Error("workspace verifier manifest contains an invalid entry");
    }
    const identity = `${entry.kind}\0${entry.path}`;
    if (seen.has(identity)) throw new Error(`workspace verifier manifest duplicates: ${entry.path}`);
    seen.add(identity);
    const absolute = await safeFile(canonicalRoot, entry.path);
    const bytes = await readFile(absolute);
    let source;
    try {
      source = new TextDecoder("utf-8", { fatal: true }).decode(bytes);
    } catch {
      throw new Error(`${entry.path}: input is not valid UTF-8`);
    }
    if (source.includes("\0")) throw new Error(`${entry.path}: NUL byte is forbidden`);
    if (entry.kind === "policy_text") {
      validatePolicySource(source, entry.path);
      if (/\.pyw?$/iu.test(entry.path) && !isPolicyImplementationException(entry.path, "revision-state")) {
        pythonPolicies.push({ label: entry.path, source });
      }
      continue;
    }
    if (entry.kind === "workspace_descriptor") {
      validateWorkspaceSource(source, entry.path, { requireWorkspace: true });
      continue;
    }
    for (const candidate of extractScriptDocuments(source, entry.path)) {
      validateWorkspaceSource(candidate.source, candidate.label, {
        requireWorkspace: false,
        expandable: candidate.expandable,
        path: entry.path,
        rawBlock: candidate.rawBlock,
      });
    }
  }
  validatePythonRevisionStates(pythonPolicies);
}

export function decodeManifest(bytes) {
  const fields = bytes.toString("utf8").split("\0");
  if (fields.at(-1) !== "") throw new Error("workspace verifier manifest is not NUL-terminated");
  fields.pop();
  if (fields.length % 2 !== 0) throw new Error("workspace verifier manifest has an incomplete record");
  const entries = [];
  for (let index = 0; index < fields.length; index += 2) {
    entries.push({ kind: fields[index], path: fields[index + 1] });
  }
  return entries;
}

function cliArguments(argv) {
  let root;
  let manifest;
  for (let index = 0; index < argv.length; index += 1) {
    const option = argv[index];
    const value = argv[index + 1];
    if ((option === "--root" || option === "--manifest") && value !== undefined) {
      if (option === "--root" && root === undefined) root = value;
      else if (option === "--manifest" && manifest === undefined) manifest = value;
      else throw new Error(`duplicate or invalid option: ${option}`);
      index += 1;
    } else {
      throw new Error(`unknown or incomplete option: ${option}`);
    }
  }
  if (root === undefined || manifest === undefined) {
    throw new Error("usage: verify-workspace-descriptor-files.mjs --root ROOT --manifest NUL_FILE");
  }
  return { root, manifest };
}

async function main(argv) {
  const { root, manifest } = cliArguments(argv);
  const manifestEntry = await lstat(manifest);
  if (!manifestEntry.isFile() || manifestEntry.isSymbolicLink()) {
    throw new Error("workspace verifier manifest is not a regular file");
  }
  const entries = decodeManifest(await readFile(manifest));
  await verifyEntries({ root, entries });
}

if (process.argv[1] && pathToFileURL(resolve(process.argv[1])).href === import.meta.url) {
  main(process.argv.slice(2)).catch((error) => {
    console.error(error instanceof Error ? error.message : String(error));
    process.exitCode = 1;
  });
}