ThothII/backend/src/workspaces/evidence/preprocessing.ts

import { isIP } from "node:net";
import type { WorkspaceDescriptor } from "../schema.js";

type EvidenceConfig = WorkspaceDescriptor["evidence"];
type SemanticFailureCode = "workspace_not_activatable" | "semantic_index_incompatible";

export interface EvidenceJobState {
  runId: string;
  completedStages: string[];
  childRuns: Record<string, string>;
}

export interface EvidencePreprocessingDependencies {
  runStage(argv: string[]): Promise<Record<string, unknown>>;
  persistJob(): void;
  evidencePreflight(): Promise<{ ok: true } | { ok: false; code: SemanticFailureCode }>;
  requireRunId(value: unknown): string;
  numberRecord(value: unknown): Record<string, number> | undefined;
}

export interface EvidencePreprocessingRequest {
  evidence: EvidenceConfig;
  job: EvidenceJobState;
  dryRun?: boolean;
  consolidate?: boolean;
  httpPrivateHostAllowlist?: readonly string[];
}

export interface EvidencePreprocessingOutcome {
  status: "succeeded" | "unchanged" | "dry_run" | "failed";
  code: "ok" | "egress_policy_refused" | SemanticFailureCode;
  runId?: string;
  childRuns?: Record<string, string>;
  completedStages?: string[];
  counts?: Record<string, number>;
  warnings?: string[];
}

function isPrivateHost(hostname: string): boolean {
  if (hostname === "localhost" || hostname === "metadata.google.internal") return true;
  const address = isIP(hostname);
  if (address === 4) {
    if (/^127\./.test(hostname) || /^10\./.test(hostname) || /^192\.168\./.test(hostname)) {
      return true;
    }
    if (/^169\.254\./.test(hostname) || /^0\./.test(hostname)) return true;
    const match = /^172\.(\d+)\./.exec(hostname);
    return Boolean(match && Number(match[1]) >= 16 && Number(match[1]) <= 31);
  }
  if (address === 6) {
    const normalized = hostname.toLowerCase();
    return normalized === "::1"
      || normalized.startsWith("fe80:")
      || normalized.startsWith("fd")
      || normalized.startsWith("fc");
  }
  return hostname.endsWith(".internal");
}

export function evidencePolicy(
  evidence: EvidenceConfig,
  httpPrivateHostAllowlist?: readonly string[],
): EvidencePreprocessingOutcome | undefined {
  if (!evidence || evidence.source.type === "filesystem") return undefined;
  if (evidence.source.type === "http") {
    for (const value of evidence.source.uris) {
      const host = new URL(value).hostname;
      if (
        isPrivateHost(host)
        && !(evidence.source.allow_private_hosts && httpPrivateHostAllowlist?.includes(host))
      ) {
        return { status: "failed", code: "egress_policy_refused" };
      }
    }
    return undefined;
  }
  if (
    evidence.source.endpoint_url !== undefined
    || evidence.source.credentials === "ambient"
    || evidence.source.allow_private_endpoint
    || evidence.source.allow_insecure_endpoint
  ) {
    return { status: "failed", code: "egress_policy_refused" };
  }
  return undefined;
}

function jobResult(job: EvidenceJobState): Pick<
  EvidencePreprocessingOutcome,
  "runId" | "childRuns" | "completedStages"
> {
  return {
    runId: job.runId,
    childRuns: { ...job.childRuns },
    completedStages: [...job.completedStages],
  };
}

export async function runEvidenceStage(
  request: EvidencePreprocessingRequest,
  deps: EvidencePreprocessingDependencies,
): Promise<EvidencePreprocessingOutcome> {
  const payload = await deps.runStage([
    "preprocess",
    "evidence",
    ...(request.consolidate ? ["--consolidate"] : []),
    ...(request.dryRun ? ["--dry-run"] : []),
    ...(request.job.childRuns.evidence
      ? ["--resume", request.job.childRuns.evidence]
      : []),
    "--json",
    "-c",
    "/dev/fd/3",
  ]);
  if (typeof payload.run_id === "string") {
    request.job.childRuns.evidence = deps.requireRunId(payload.run_id);
  }
  if (!request.dryRun && !request.job.completedStages.includes("evidence")) {
    request.job.completedStages.push("evidence");
  }
  deps.persistJob();
  return {
    status: request.dryRun ? "dry_run" : "succeeded",
    code: "ok",
    ...jobResult(request.job),
    counts: deps.numberRecord(payload.counts),
  };
}

export async function preprocessEvidence(
  request: EvidencePreprocessingRequest,
  deps: EvidencePreprocessingDependencies,
): Promise<EvidencePreprocessingOutcome> {
  if (!request.evidence) {
    return {
      status: "unchanged",
      code: "ok",
      warnings: ["workspace has no Evidence source"],
    };
  }
  const policy = evidencePolicy(request.evidence, request.httpPrivateHostAllowlist);
  if (policy) return policy;
  const preflight = await deps.evidencePreflight();
  if (!preflight.ok) {
    return { status: "failed", code: preflight.code, runId: request.job.runId };
  }
  if (request.job.completedStages.includes("evidence") && !request.dryRun) {
    return {
      status: "unchanged",
      code: "ok",
      runId: request.job.runId,
      completedStages: [...request.job.completedStages],
    };
  }
  return await runEvidenceStage(request, deps);
}

export async function continueEvidencePreprocessing(
  request: Omit<EvidencePreprocessingRequest, "dryRun"> & {
    priorCounts?: Record<string, number>;
  },
  deps: EvidencePreprocessingDependencies,
): Promise<EvidencePreprocessingOutcome> {
  if (!request.evidence) {
    return {
      status: "succeeded",
      code: "ok",
      ...jobResult(request.job),
      warnings: ["workspace has no Evidence source"],
      ...(request.priorCounts ? { counts: request.priorCounts } : {}),
    };
  }
  const policy = evidencePolicy(request.evidence, request.httpPrivateHostAllowlist);
  if (policy) return { ...policy, ...jobResult(request.job) };
  if (!request.job.completedStages.includes("evidence")) {
    const preflight = await deps.evidencePreflight();
    if (!preflight.ok) {
      return {
        status: "failed",
        code: preflight.code,
        ...jobResult(request.job),
      };
    }
    return await runEvidenceStage(request, deps);
  }
  return { status: "unchanged", code: "ok", ...jobResult(request.job) };
}