ThothII/backend/src/workspaces/schema.ts

import { parseAllDocuments, stringify } from "yaml";
import { z } from "zod";

export const DWH_TRANSPORTS = ["postgres_direct", "rest_api", "ssh_tunnel"] as const;
export type DwhTransport = (typeof DWH_TRANSPORTS)[number];

export const VECTOR_TRANSPORTS = ["pgvector_direct", "rest_api", "ssh_tunnel"] as const;
export type VectorTransport = (typeof VECTOR_TRANSPORTS)[number];

export const REST_DIAGNOSTIC_METHODS = ["GET", "POST"] as const;
export type RestDiagnosticMethod = (typeof REST_DIAGNOSTIC_METHODS)[number];

export const DIAGNOSTIC_AUTH_MODES = ["none", "bearer", "x-api-key"] as const;
export type DiagnosticAuthMode = (typeof DIAGNOSTIC_AUTH_MODES)[number];

export interface RestDiagnosticRequest {
  method: RestDiagnosticMethod;
  path: string;
  auth: DiagnosticAuthMode;
}

export interface CanonicalDiagnostics {
  dwh_rest?: RestDiagnosticRequest & { response: { database: string; schema: string } };
  vector_rest?: {
    metadata: RestDiagnosticRequest & {
      response: { collection: string; dimensions: string; distance: string };
    };
    reversible_probe?: RestDiagnosticRequest & {
      method: "POST";
      auth: Exclude<DiagnosticAuthMode, "none">;
      response: { operation: string };
    };
  };
  embedding?: RestDiagnosticRequest & { response: { model: string; dimensions: string } };
}

interface WorkspaceMetadata {
  schema_version: 4;
  id: string;
  name: string;
  description?: string;
  language: "en" | "it";
}

interface WorkspaceDwh {
  engine: "postgres";
  database: string;
  schema: string;
  port?: number;
  timeout_ms?: number;
  supported_transports: DwhTransport[];
}

interface WorkspaceBase {
  workspace: WorkspaceMetadata;
  /** Legacy connection block; current descriptors bind their database through PostgreSQL. */
  dwh?: WorkspaceDwh;
  diagnostics?: Pick<CanonicalDiagnostics, "dwh_rest">;
}

export interface EvidencePolicy {
  max_chunk_chars: number;
  retain_published_generations: number;
}

export type EvidenceSource =
  | {
      type: "filesystem";
      uri: string;
      patterns: string[];
      max_bytes: number;
    }
  | {
      type: "http";
      uris: string[];
      authentication: "none" | "signed_urls_file";
      connect_timeout_ms: number;
      read_timeout_ms: number;
      max_bytes: number;
      max_redirects: number;
      allow_private_hosts: boolean;
      max_cache_bytes: number;
    }
  | {
      type: "s3";
      uri: string;
      endpoint_url?: string;
      region?: string;
      credentials: "ambient" | "static_files";
      trusted_endpoint: boolean;
      allow_private_endpoint: boolean;
      allow_insecure_endpoint: boolean;
      max_bytes: number;
      max_objects: number;
      max_pages: number;
      page_size: number;
    };

export interface WorkspaceEvidence {
  schema_version: 1 | 2;
  source: EvidenceSource;
  policy: EvidencePolicy;
}

export interface WorkspaceV4 extends WorkspaceBase {
  evidence?: WorkspaceEvidence;
}

export type CanonicalWorkspace = WorkspaceV4;
export type WorkspaceDescriptor = WorkspaceV4;

const workspaceId = z.string().regex(/^[a-z][a-z0-9-]{2,62}$/, {
  message: "workspace id must match ^[a-z][a-z0-9-]{2,62}$",
});
const identifier = z.string().regex(/^[A-Za-z_][A-Za-z0-9_]*$/, {
  message: "database identifiers must start with a letter or underscore",
});
const port = z.number().int().min(1).max(65_535);
const timeoutMs = z.number().int().positive();

function isOriginRelativeDiagnosticPath(value: string): boolean {
  return /^\/(?!\/)[^\\\u0000-\u001F\u007F?#]*$/.test(value) && !/%5c/i.test(value);
}

const diagnosticPath = z.string().refine(isOriginRelativeDiagnosticPath, {
  message: "diagnostic paths must be origin-relative and cannot contain backslashes, control characters, queries, or fragments",
});
const responseField = z.string().regex(/^[A-Za-z_][A-Za-z0-9_]*$/, {
  message: "diagnostic response fields must be identifiers",
});
const restDiagnosticRequest = z.object({
  method: z.enum(REST_DIAGNOSTIC_METHODS),
  path: diagnosticPath,
  auth: z.enum(DIAGNOSTIC_AUTH_MODES),
}).strict();
const dwhRestDiagnostic = restDiagnosticRequest.extend({
  response: z.object({ database: responseField, schema: responseField }).strict(),
}).strict();

const dwhSchema = z.object({
  engine: z.literal("postgres"),
  database: identifier,
  schema: identifier,
  port: port.optional(),
  timeout_ms: timeoutMs.optional(),
  supported_transports: z.array(z.enum(DWH_TRANSPORTS)).min(1),
}).strict();

const positiveSafeInteger = z.number().int().safe().positive();
const nonnegativeSafeInteger = z.number().int().safe().nonnegative();

function isSafeEvidencePattern(value: string): boolean {
  const parts = value.split("/");
  return value.length > 0
    && !value.startsWith("/")
    && !value.includes("\\")
    && !/[\u0000-\u001f\u007f]/u.test(value)
    && parts.every((part) => part !== "" && part !== "." && part !== "..");
}

function parsePublicHttpUri(value: string): URL | undefined {
  if (value.trim() !== value || /[\u0000-\u001f\u007f\\]/u.test(value)) return undefined;
  try {
    const parsed = new URL(value);
    if (
      !["http:", "https:"].includes(parsed.protocol)
      || parsed.hostname.length === 0
      || parsed.username !== ""
      || parsed.password !== ""
      || parsed.search !== ""
      || parsed.hash !== ""
    ) return undefined;
    return parsed;
  } catch {
    return undefined;
  }
}

function canonicalPublicHttpUri(value: string): string | undefined {
  return parsePublicHttpUri(value)?.href;
}

function isSafeS3Uri(value: string): boolean {
  if (value.trim() !== value || /[\u0000-\u001f\u007f\\]/u.test(value)) return false;
  try {
    const parsed = new URL(value);
    const bucket = parsed.hostname;
    const validBucket = /^(?=.{3,63}$)(?!-)(?!.*\.\.)(?!.*\.-)(?!.*-\.)[a-z0-9](?:[a-z0-9.-]*[a-z0-9])?$/.test(bucket)
      && !/^\d{1,3}(?:\.\d{1,3}){3}$/.test(bucket);
    return parsed.protocol === "s3:"
      && validBucket
      && parsed.port === ""
      && parsed.username === ""
      && parsed.password === ""
      && parsed.search === ""
      && parsed.hash === ""
      && parsed.href === value;
  } catch {
    return false;
  }
}

function isSafeS3Endpoint(value: string): boolean {
  const parsed = parsePublicHttpUri(value);
  return parsed !== undefined && (parsed.pathname === "/" || parsed.pathname === "");
}

const evidencePattern = z.string().refine(isSafeEvidencePattern, {
  message: "evidence patterns must be normalized relative globs",
});
const filesystemEvidenceSourceSchema = z.object({
  type: z.literal("filesystem"),
  uri: z.string(),
  patterns: z.array(evidencePattern).min(1).optional(),
  max_bytes: positiveSafeInteger.default(10 * 1024 * 1024),
}).strict().superRefine((source, context) => {
  if (source.patterns !== undefined && new Set(source.patterns).size !== source.patterns.length) {
    context.addIssue({ code: "custom", path: ["patterns"], message: "evidence patterns must not repeat" });
  }
});
const httpEvidenceUri = z.string().refine((value) => parsePublicHttpUri(value) !== undefined, {
  message: "HTTP evidence URIs must be public http(s) identities without credentials, query, or fragment",
});
const httpEvidenceSourceSchema = z.object({
  type: z.literal("http"),
  uris: z.array(httpEvidenceUri).min(1),
  authentication: z.enum(["none", "signed_urls_file"]).default("none"),
  connect_timeout_ms: positiveSafeInteger.default(5_000),
  read_timeout_ms: positiveSafeInteger.default(30_000),
  max_bytes: positiveSafeInteger.default(10 * 1024 * 1024),
  max_redirects: nonnegativeSafeInteger.default(5),
  allow_private_hosts: z.boolean().default(false),
  max_cache_bytes: positiveSafeInteger.default(64 * 1024 * 1024),
}).strict().superRefine((source, context) => {
  const canonical = source.uris.map(canonicalPublicHttpUri);
  if (new Set(canonical).size !== canonical.length) {
    context.addIssue({ code: "custom", path: ["uris"], message: "HTTP evidence URIs must not repeat" });
  }
});
const s3EvidenceSourceSchema = z.object({
  type: z.literal("s3"),
  uri: z.string().refine(isSafeS3Uri, {
    message: "S3 evidence URI must use s3:// without credentials, query, or fragment",
  }),
  endpoint_url: z.string().refine(isSafeS3Endpoint, {
    message: "S3 endpoint must be an origin-only http(s) URL without credentials",
  }).optional(),
  region: z.string().trim().min(1).optional(),
  credentials: z.enum(["ambient", "static_files"]).default("ambient"),
  trusted_endpoint: z.boolean().default(false),
  allow_private_endpoint: z.boolean().default(false),
  allow_insecure_endpoint: z.boolean().default(false),
  max_bytes: positiveSafeInteger.default(10 * 1024 * 1024),
  max_objects: positiveSafeInteger.default(10_000),
  max_pages: positiveSafeInteger.default(100),
  page_size: positiveSafeInteger.max(1_000).default(1_000),
}).strict().superRefine((source, context) => {
  if (source.endpoint_url === undefined) {
    if (source.trusted_endpoint || source.allow_private_endpoint || source.allow_insecure_endpoint) {
      context.addIssue({
        code: "custom", path: ["endpoint_url"],
        message: "S3 endpoint policy requires endpoint_url",
      });
    }
    return;
  }
  if (!source.trusted_endpoint) {
    context.addIssue({
      code: "custom", path: ["trusted_endpoint"],
      message: "custom S3 endpoints must be explicitly trusted",
    });
  }
  const endpoint = parsePublicHttpUri(source.endpoint_url);
  if (endpoint?.protocol === "http:" && !source.allow_insecure_endpoint) {
    context.addIssue({
      code: "custom", path: ["allow_insecure_endpoint"],
      message: "HTTP S3 endpoints require an explicit insecure opt-in",
    });
  }
});
const evidenceSourceSchema = z.discriminatedUnion("type", [
  filesystemEvidenceSourceSchema,
  httpEvidenceSourceSchema,
  s3EvidenceSourceSchema,
]);
const evidencePolicySchema = z.object({
  max_chunk_chars: positiveSafeInteger.default(4_000),
  retain_published_generations: positiveSafeInteger.default(3),
}).strict();
const workspaceEvidenceSchema = z.object({
  schema_version: z.union([z.literal(1), z.literal(2)]).default(1),
  source: evidenceSourceSchema,
  policy: evidencePolicySchema.default({
    max_chunk_chars: 4_000,
    retain_published_generations: 3,
  }),
}).strict().superRefine((evidence, context) => {
  if (evidence.schema_version !== 2 || evidence.source.type !== "filesystem") return;
  const patterns = evidence.source.patterns ?? ["curated/**/*.md"];
  const selectsSource = patterns.some((pattern) => pattern === "source" || pattern.startsWith("source/"));
  const selectsCurated = patterns.some((pattern) => pattern === "curated" || pattern.startsWith("curated/"));
  if (selectsSource && selectsCurated) {
    context.addIssue({
      code: "custom",
      path: ["source", "patterns"],
      message: "schema-versioned filesystem Evidence patterns cannot span source and curated",
    });
  } else if (patterns.length !== 1 || patterns[0] !== "curated/**/*.md") {
    context.addIssue({
      code: "custom",
      path: ["source", "patterns"],
      message: "schema-versioned filesystem Evidence patterns must be exactly curated/**/*.md",
    });
  }
}).transform((evidence) => ({
  ...evidence,
  source: evidence.source.type !== "filesystem" || evidence.source.patterns !== undefined
    ? evidence.source
    : {
      ...evidence.source,
      patterns: evidence.schema_version === 2 ? ["curated/**/*.md"] : ["**/*.md"],
    },
}));

function unique<T>(values: readonly T[], context: z.RefinementCtx, path: PropertyKey[]) {
  if (new Set(values).size !== values.length) {
    context.addIssue({ code: "custom", path, message: "supported transports must not repeat" });
  }
}

function workspaceInvariants(workspace: any, context: z.RefinementCtx): void {
  if (workspace.dwh) {
    unique(workspace.dwh.supported_transports, context, ["dwh", "supported_transports"]);
  }

  if (workspace.evidence?.source.type === "filesystem") {
    const expected = `${workspace.workspace.id}/evidence`;
    if (workspace.evidence.source.uri !== expected) {
      context.addIssue({
        code: "custom",
        path: ["evidence", "source", "uri"],
        message: "filesystem evidence URI must be the canonical workspace Evidence root",
      });
    }
  }

  if (workspace.diagnostics?.dwh_rest
    && !workspace.dwh?.supported_transports.includes("rest_api")) {
    context.addIssue({
      code: "custom",
      path: ["diagnostics", "dwh_rest"],
      message: "diagnostics.dwh_rest requires dwh rest_api transport support",
    });
  }
}

const WorkspaceV4Schema = z.object({
  dwh: dwhSchema.optional(),
  evidence: workspaceEvidenceSchema.optional(),
  diagnostics: z.object({
    dwh_rest: dwhRestDiagnostic.optional(),
  }).strict().optional(),
  workspace: z.object({
    schema_version: z.literal(4), id: workspaceId, name: z.string().trim().min(1),
    description: z.string().trim().min(1).optional(), language: z.enum(["en", "it"]),
  }).strict(),
}).strict().superRefine(workspaceInvariants);
const WorkspaceDescriptorSchema = WorkspaceV4Schema;

function parseWorkspaceDocument(source: string): unknown {
  const documents = parseAllDocuments(source, { uniqueKeys: true });
  if (documents.length !== 1) throw new Error("Workspace YAML must contain exactly one document");
  const document = documents[0];
  if (document.errors.length > 0 || document.warnings.length > 0) {
    throw new Error(`Invalid workspace YAML: ${[...document.errors, ...document.warnings]
      .map((error) => error.message).join("; ")}`);
  }
  return document.toJSON();
}

export function parseWorkspaceYaml(source: string): WorkspaceDescriptor {
  const value = parseWorkspaceDocument(source);
  assertAuthoredWorkspaceIsDatabaseFree(value);
  return validateWorkspaceDescriptor(value);
}

/** Parses an ephemeral, generated core runtime descriptor that may contain a Catalog binding. */
export function parseRuntimeWorkspaceYaml(source: string): WorkspaceDescriptor {
  return validateWorkspaceDescriptor(parseWorkspaceDocument(source));
}

function assertAuthoredWorkspaceIsDatabaseFree(workspace: unknown): void {
  if (workspace !== null && typeof workspace === "object"
    && ("dwh" in workspace || "diagnostics" in workspace)) {
    throw new Error(
      "Workspace YAML must not contain database configuration; use the PostgreSQL Metadata Catalog",
    );
  }
}

/** Build a database-free v4 descriptor; legacy database/configuration fields are not carried over. */
export function migrateWorkspaceV3Yaml(source: string): string {
  const documents = parseAllDocuments(source, { uniqueKeys: true });
  if (documents.length !== 1) throw new Error("Workspace YAML must contain exactly one document");
  const document = documents[0];
  if (document.errors.length > 0 || document.warnings.length > 0) {
    throw new Error("Invalid workspace YAML");
  }
  const value = document.toJSON() as Record<string, unknown>;
  const metadata = value.workspace as Record<string, unknown> | undefined;
  if (!metadata || metadata.schema_version !== 3) {
    throw new Error("Workspace migration requires schema version 3");
  }
  metadata.schema_version = 4;
  delete value.dwh;
  delete value.diagnostics;
  delete value.semantic_index;
  delete value.llm_policy;
  return serializeWorkspaceYaml(validateWorkspaceDescriptor(value));
}

export function validateWorkspaceDescriptor(workspace: unknown): WorkspaceDescriptor {
  return WorkspaceDescriptorSchema.parse(workspace) as WorkspaceDescriptor;
}

export function isCanonicalWorkspace(workspace: unknown): workspace is CanonicalWorkspace {
  return WorkspaceDescriptorSchema.safeParse(workspace).success;
}

export function isOperationalWorkspace(workspace: unknown): workspace is WorkspaceV4 {
  return WorkspaceDescriptorSchema.safeParse(workspace).success;
}

export function validateOperationalWorkspace(workspace: unknown): WorkspaceV4 {
  return validateWorkspaceDescriptor(workspace);
}

/** Builds a request URL only after rejecting values that can leave the declared service origin. */
export function resolveDiagnosticUrl(baseUrl: string, path: string): URL {
  if (!isOriginRelativeDiagnosticPath(path)) throw new Error("Diagnostic path must remain on the configured origin");
  const base = new URL(baseUrl);
  // Resolve the origin-relative path beneath the configured base path (e.g. `/dwh/`), not the
  // origin root: a leading slash must append to the base path instead of resetting it.
  const basePath = base.pathname.endsWith("/") ? base.pathname : `${base.pathname}/`;
  const resolved = new URL(`${basePath}${path.replace(/^\/+/, "")}`, base.origin);
  if (resolved.origin !== base.origin) throw new Error("Diagnostic URL must remain on the configured origin");
  return resolved;
}

export function serializeWorkspaceYaml(workspace: CanonicalWorkspace): string {
  const canonical = validateOperationalWorkspace(workspace);
  assertAuthoredWorkspaceIsDatabaseFree(canonical);
  return stringify(canonical, { lineWidth: 0, sortMapEntries: true });
}

export { buildInstallationContract, renderWorkspaceDocs } from "./contracts.js";