614 lines
28 KiB
TypeScript
614 lines
28 KiB
TypeScript
import { createHash, randomBytes } from "node:crypto";
|
|
import { readdirSync, readFileSync, rmSync, writeFileSync, mkdirSync } from "node:fs";
|
|
import { isIP } from "node:net";
|
|
import { join } from "node:path";
|
|
import type { WorkspaceDescriptor } from "./schema.js";
|
|
import {
|
|
PreprocessingStateStore,
|
|
type FkReviewRecord,
|
|
type PreprocessingJobState,
|
|
type SessionInventoryRow,
|
|
} from "./preprocessing-state.js";
|
|
import type { DeterministicRuntimeConfigLease } from "./runtime-config-lease.js";
|
|
import { readAnnotationsSync } from "./annotations-sync.js";
|
|
import { reconcileCollection } from "./qdrant-collection.js";
|
|
|
|
export interface WorkspaceOperationResult {
|
|
schemaVersion: 1;
|
|
status: "succeeded" | "unchanged" | "dry_run" | "blocked" | "failed";
|
|
code:
|
|
| "ok" | "workspace_not_found" | "workspace_not_activatable"
|
|
| "binding_missing" | "preprocessing_conflict"
|
|
| "preprocessing_resume_mismatch" | "manual_review_required"
|
|
| "evidence_materialization_required" | "effective_config_mismatch"
|
|
| "semantic_index_incompatible" | "annotation_invalid"
|
|
| "egress_policy_refused";
|
|
workspaceId: string;
|
|
workspaceRevision: string;
|
|
descriptorBlob: string;
|
|
operation: string;
|
|
runId?: string;
|
|
childRuns?: Record<string, string>;
|
|
completedStages: string[];
|
|
counts?: Record<string, number>;
|
|
artifactIdentities?: Array<{ kind: string; digest: string }>;
|
|
effectiveConfigIdentity?: string;
|
|
configFingerprint?: string;
|
|
inputFingerprint?: string;
|
|
/** Suggested FK annotations YAML for the operator to write to --output (schema suggest-fks). */
|
|
suggestedFksYaml?: string;
|
|
warnings?: string[];
|
|
}
|
|
|
|
export interface ChildProcessRequest {
|
|
argv: string[];
|
|
configPath: string;
|
|
}
|
|
|
|
interface ActiveRuntime {
|
|
workspace: WorkspaceDescriptor;
|
|
workspaceId: string;
|
|
workspaceRevision: string;
|
|
descriptorBlob: string;
|
|
catalogBlob: string;
|
|
configLease: DeterministicRuntimeConfigLease;
|
|
}
|
|
|
|
interface ChildProcessResult {
|
|
exitCode: number;
|
|
stdout: string;
|
|
stderr: string;
|
|
}
|
|
|
|
export interface WorkspacePreprocessingServiceDeps {
|
|
dataRoot: string;
|
|
acquireActiveRuntime(workspaceId: string): Promise<ActiveRuntime>;
|
|
runChild(request: ChildProcessRequest): Promise<ChildProcessResult>;
|
|
listSessions(workspaceId: string): Promise<readonly SessionInventoryRow[]>;
|
|
semanticPreflight(workspace: WorkspaceDescriptor): Promise<
|
|
{ ok: true } | { ok: false; code: "workspace_not_activatable" | "semantic_index_incompatible" }
|
|
>;
|
|
httpPrivateHostAllowlist?: readonly string[];
|
|
}
|
|
|
|
interface RunScope {
|
|
runtime: ActiveRuntime;
|
|
state: PreprocessingStateStore;
|
|
job: PreprocessingJobState;
|
|
}
|
|
|
|
function digest(value: string | Buffer): string {
|
|
return `sha256:${createHash("sha256").update(value).digest("hex")}`;
|
|
}
|
|
|
|
function baseResult(
|
|
runtime: ActiveRuntime,
|
|
operation: string,
|
|
status: WorkspaceOperationResult["status"],
|
|
code: WorkspaceOperationResult["code"],
|
|
extra: Omit<Partial<WorkspaceOperationResult>, "schemaVersion" | "status" | "code" | "workspaceId" | "workspaceRevision" | "descriptorBlob" | "operation"> = {},
|
|
): WorkspaceOperationResult {
|
|
return {
|
|
schemaVersion: 1,
|
|
status,
|
|
code,
|
|
workspaceId: runtime.workspaceId,
|
|
workspaceRevision: runtime.workspaceRevision,
|
|
descriptorBlob: runtime.descriptorBlob,
|
|
operation,
|
|
completedStages: [],
|
|
effectiveConfigIdentity: runtime.configLease.effectiveConfigIdentity,
|
|
configFingerprint: runtime.configLease.configFingerprint,
|
|
inputFingerprint: runtime.configLease.inputFingerprint,
|
|
...extra,
|
|
};
|
|
}
|
|
|
|
function isPrivateHost(hostname: string): boolean {
|
|
if (hostname === "localhost" || hostname === "metadata.google.internal") return true;
|
|
const address = isIP(hostname);
|
|
if (address === 4) {
|
|
if (/^127\./.test(hostname) || /^10\./.test(hostname) || /^192\.168\./.test(hostname)) return true;
|
|
if (/^169\.254\./.test(hostname) || /^0\./.test(hostname)) return true;
|
|
const match = /^172\.(\d+)\./.exec(hostname);
|
|
return Boolean(match && Number(match[1]) >= 16 && Number(match[1]) <= 31);
|
|
}
|
|
if (address === 6) {
|
|
const normalized = hostname.toLowerCase();
|
|
return normalized === "::1" || normalized.startsWith("fe80:") || normalized.startsWith("fd") || normalized.startsWith("fc");
|
|
}
|
|
return hostname.endsWith(".internal");
|
|
}
|
|
|
|
function noEvidenceWarning(workspace: WorkspaceDescriptor): string[] {
|
|
return workspace.evidence === undefined ? ["workspace has no Evidence source"] : [];
|
|
}
|
|
|
|
export class WorkspacePreprocessingService {
|
|
constructor(private readonly deps: WorkspacePreprocessingServiceDeps) {}
|
|
|
|
|
|
async vectorInspect(options: { workspaceId: string }): Promise<WorkspaceOperationResult> {
|
|
const runtime = await this.deps.acquireActiveRuntime(options.workspaceId);
|
|
const collection = runtime.workspace.semantic_index.vector_store.collection;
|
|
const res = await fetch(`${runtime.configLease.semanticQdrantUrl}/collections/${encodeURIComponent(collection)}`, { method: "GET" });
|
|
if (!res.ok) return baseResult(runtime, "vector inspect", "failed", "semantic_index_incompatible", { warnings: ["collection unavailable"] });
|
|
const body = await res.json() as any;
|
|
const info = body?.result;
|
|
const vectors = info?.config?.params?.vectors;
|
|
return baseResult(runtime, "vector inspect", "succeeded", "ok", {
|
|
counts: { dimensions: vectors?.size ?? 0 },
|
|
warnings: [`collection=${collection} distance=${vectors?.distance ?? "unknown"}`],
|
|
});
|
|
}
|
|
|
|
async vectorRebuild(options: { workspaceId: string; collection?: string; confirm?: string; destroy?: boolean }): Promise<WorkspaceOperationResult> {
|
|
const runtime = await this.deps.acquireActiveRuntime(options.workspaceId);
|
|
const collection = runtime.workspace.semantic_index.vector_store.collection;
|
|
if (options.collection !== collection || options.confirm !== collection || options.destroy !== true) {
|
|
return baseResult(runtime, "vector rebuild", "failed", "semantic_index_incompatible", { warnings: ["rebuild requires exact confirmation and --destroy"] });
|
|
}
|
|
const q = `${runtime.configLease.semanticQdrantUrl}/collections/${encodeURIComponent(collection)}`;
|
|
const del = await fetch(q, { method: "DELETE" });
|
|
if (!del.ok && del.status !== 404) return baseResult(runtime, "vector rebuild", "failed", "semantic_index_incompatible", { warnings: ["collection delete failed"] });
|
|
// Recreate the complete contract (dimensions + distance + the 8 required keyword indexes).
|
|
const recreated = await reconcileCollection({
|
|
baseUrl: runtime.configLease.semanticQdrantUrl,
|
|
collection,
|
|
dimensions: runtime.workspace.semantic_index.vector_store.dimensions,
|
|
distance: runtime.workspace.semantic_index.vector_store.distance,
|
|
mode: "self_heal",
|
|
});
|
|
if (!recreated.ok) return baseResult(runtime, "vector rebuild", "failed", "semantic_index_incompatible", { warnings: ["collection recreate failed"] });
|
|
return baseResult(runtime, "vector rebuild", "succeeded", "ok", { warnings: [`recreated collection=${collection}`] });
|
|
}
|
|
async inspect(options: { workspaceId: string }): Promise<WorkspaceOperationResult> {
|
|
try {
|
|
const runtime = await this.deps.acquireActiveRuntime(options.workspaceId);
|
|
return baseResult(runtime, "inspect", "succeeded", "ok", {
|
|
artifactIdentities: [
|
|
{ kind: "descriptor", digest: runtime.descriptorBlob },
|
|
{ kind: "catalog", digest: runtime.catalogBlob },
|
|
{ kind: "runtime_config", digest: runtime.configLease.configDigest },
|
|
],
|
|
});
|
|
} catch {
|
|
return {
|
|
schemaVersion: 1,
|
|
status: "failed",
|
|
code: "workspace_not_activatable",
|
|
workspaceId: options.workspaceId,
|
|
workspaceRevision: "",
|
|
descriptorBlob: "",
|
|
operation: "inspect",
|
|
completedStages: [],
|
|
};
|
|
}
|
|
}
|
|
|
|
async preprocessDwh(options: { workspaceId: string; resumeRunId?: string }): Promise<WorkspaceOperationResult> {
|
|
const scope = await this.startRun(options.workspaceId, "preprocess dwh", options.resumeRunId);
|
|
if (scope.job.completedStages.includes("dwh")) {
|
|
return baseResult(scope.runtime, "preprocess dwh", "unchanged", "ok", {
|
|
runId: scope.job.runId,
|
|
childRuns: scope.job.childRuns,
|
|
completedStages: [...scope.job.completedStages],
|
|
});
|
|
}
|
|
const payload = await this.runJsonStage(scope.runtime, [
|
|
"preprocess", "dwh", "--steps", "introspect,lsh",
|
|
...(scope.job.childRuns.dwh ? ["--resume", scope.job.childRuns.dwh] : []),
|
|
"--json", "-c", "/dev/fd/3",
|
|
]);
|
|
const childRun = this.requireRunId(payload.run_id);
|
|
scope.job.childRuns.dwh = childRun;
|
|
if (!scope.job.completedStages.includes("dwh")) scope.job.completedStages.push("dwh");
|
|
this.state(scope.runtime.workspaceId).writeJob(scope.job);
|
|
return baseResult(scope.runtime, "preprocess dwh", "succeeded", "ok", {
|
|
runId: scope.job.runId,
|
|
childRuns: { ...scope.job.childRuns },
|
|
completedStages: [...scope.job.completedStages],
|
|
});
|
|
}
|
|
|
|
async suggestFks(options: {
|
|
workspaceId: string;
|
|
fromSql?: ReadonlyArray<{ name: string; sql: string }>;
|
|
assume?: readonly string[];
|
|
resumeRunId?: string;
|
|
}): Promise<WorkspaceOperationResult> {
|
|
const scope = await this.startRun(options.workspaceId, "schema suggest-fks", options.resumeRunId);
|
|
return await this.runSuggestStage(scope, options.fromSql ?? [], options.assume ?? []);
|
|
}
|
|
|
|
async checkSchema(options: {
|
|
workspaceId: string;
|
|
annotationsYaml?: string;
|
|
reviewedCandidatesDigest?: string;
|
|
}): Promise<WorkspaceOperationResult> {
|
|
const runtime = await this.deps.acquireActiveRuntime(options.workspaceId);
|
|
const state = this.state(runtime.workspaceId);
|
|
if ((options.annotationsYaml === undefined) !== (options.reviewedCandidatesDigest === undefined)) {
|
|
return baseResult(runtime, "schema check", "failed", "annotation_invalid");
|
|
}
|
|
if (options.reviewedCandidatesDigest === undefined) {
|
|
const payload = await this.runJsonStage(runtime, ["schema", "check", "--json", "-c", "/dev/fd/3"]);
|
|
return baseResult(runtime, "schema check", Number(payload.orphan_count ?? 0) === 0 ? "succeeded" : "failed", Number(payload.orphan_count ?? 0) === 0 ? "ok" : "annotation_invalid");
|
|
}
|
|
const reviewedCandidatesDigest = options.reviewedCandidatesDigest;
|
|
const runId = this.findRunIdByCandidateDigest(state, reviewedCandidatesDigest);
|
|
if (!runId) return baseResult(runtime, "schema check", "failed", "annotation_invalid");
|
|
const request = await this.withStagedInputs(runtime.workspaceId, [
|
|
{ flag: "--annotations", name: "annotations.yaml", contents: options.annotationsYaml! },
|
|
], async (argv) => await this.runJsonStage(runtime, [
|
|
"schema", "check", ...argv,
|
|
"--reviewed-candidates", reviewedCandidatesDigest,
|
|
"--json", "-c", "/dev/fd/3",
|
|
]));
|
|
if (request.reviewed_candidates_digest !== reviewedCandidatesDigest || typeof request.annotations_digest !== "string") {
|
|
return baseResult(runtime, "schema check", "failed", "annotation_invalid", { runId });
|
|
}
|
|
// P5 supersedes the host-file FK review: schema check is read-only validation and never
|
|
// records a review. Only `schema accept` records a human review for the curated Git blob.
|
|
return baseResult(runtime, "schema check", "succeeded", "ok", { runId });
|
|
}
|
|
|
|
async acceptSchema(options: { workspaceId: string; runId: string; yes?: boolean }): Promise<WorkspaceOperationResult> {
|
|
const runtime = await this.deps.acquireActiveRuntime(options.workspaceId);
|
|
const state = this.state(runtime.workspaceId);
|
|
if (options.yes !== true) {
|
|
return baseResult(runtime, "schema accept", "failed", "annotation_invalid", {
|
|
runId: options.runId,
|
|
warnings: ["accept requires --yes"],
|
|
});
|
|
}
|
|
if (!/^[0-9a-f]{32}$/.test(options.runId)) {
|
|
return baseResult(runtime, "schema accept", "failed", "annotation_invalid");
|
|
}
|
|
const candidate = state.readFkCandidates(options.runId);
|
|
if (candidate === undefined) {
|
|
return baseResult(runtime, "schema accept", "failed", "annotation_invalid", {
|
|
runId: options.runId,
|
|
warnings: ["candidate run is unavailable"],
|
|
});
|
|
}
|
|
const synced = readAnnotationsSync(this.deps.dataRoot, runtime.workspaceId, runtime.workspaceRevision);
|
|
if (synced === undefined || synced.contents.toString("utf8").trim() === "") {
|
|
return baseResult(runtime, "schema accept", "failed", "annotation_invalid", {
|
|
runId: options.runId,
|
|
warnings: ["curated annotations are not synchronized"],
|
|
});
|
|
}
|
|
// The harness parser validates the curated blob against the physical schema; the recorded
|
|
// candidate digest must round-trip and the blob digest must match the synced destination.
|
|
const payload = await this.runJsonStage(runtime, [
|
|
"schema", "check", "--reviewed-candidates", candidate.digest, "--json", "-c", "/dev/fd/3",
|
|
]);
|
|
if (payload.annotations_digest !== synced.contentDigest
|
|
|| payload.reviewed_candidates_digest !== candidate.digest
|
|
|| Number(payload.orphan_count ?? 0) !== 0) {
|
|
return baseResult(runtime, "schema accept", "failed", "annotation_invalid", { runId: options.runId });
|
|
}
|
|
const review = state.writeFkReview(options.runId, {
|
|
reviewedCandidatesDigest: candidate.digest,
|
|
annotationsDigest: synced.contentDigest,
|
|
workspaceRevision: runtime.workspaceRevision,
|
|
blobId: synced.blobId,
|
|
});
|
|
const job = state.readJob(options.runId);
|
|
job.reviewDigest = review.digest;
|
|
if (!job.completedStages.includes("fk_review")) job.completedStages.push("fk_review");
|
|
state.writeJob(job);
|
|
return baseResult(runtime, "schema accept", "succeeded", "ok", {
|
|
runId: options.runId,
|
|
completedStages: [...job.completedStages],
|
|
artifactIdentities: [
|
|
{ kind: "fk_review", digest: review.digest },
|
|
{ kind: "annotations", digest: synced.contentDigest },
|
|
],
|
|
});
|
|
}
|
|
|
|
async indexSchema(options: { workspaceId: string; resumeRunId?: string }): Promise<WorkspaceOperationResult> {
|
|
const scope = await this.startRun(options.workspaceId, "index-schema", options.resumeRunId);
|
|
const semantic = await this.deps.semanticPreflight(scope.runtime.workspace);
|
|
if (!semantic.ok) return baseResult(scope.runtime, "index-schema", "failed", semantic.code, { runId: scope.job.runId });
|
|
if (scope.job.completedStages.includes("schema_index")) {
|
|
return baseResult(scope.runtime, "index-schema", "unchanged", "ok", {
|
|
runId: scope.job.runId,
|
|
completedStages: [...scope.job.completedStages],
|
|
});
|
|
}
|
|
const payload = await this.runJsonStage(scope.runtime, ["vector", "index-schema", "--json", "-c", "/dev/fd/3"]);
|
|
const counts = this.numberRecord(payload.counts);
|
|
scope.job.completedStages.push("schema_index");
|
|
this.state(scope.runtime.workspaceId).writeJob(scope.job);
|
|
return baseResult(scope.runtime, "index-schema", "succeeded", "ok", {
|
|
runId: scope.job.runId,
|
|
completedStages: [...scope.job.completedStages],
|
|
counts,
|
|
});
|
|
}
|
|
|
|
async preprocessEvidence(options: { workspaceId: string; dryRun?: boolean; resumeRunId?: string }): Promise<WorkspaceOperationResult> {
|
|
const scope = await this.startRun(options.workspaceId, "preprocess evidence", options.resumeRunId);
|
|
if (scope.runtime.workspace.evidence === undefined) {
|
|
return baseResult(scope.runtime, "preprocess evidence", "unchanged", "ok", {
|
|
warnings: noEvidenceWarning(scope.runtime.workspace),
|
|
});
|
|
}
|
|
const policy = this.evidencePolicy(scope.runtime.workspace);
|
|
if (policy !== undefined) return baseResult(scope.runtime, "preprocess evidence", policy.status, policy.code, { warnings: policy.warnings });
|
|
const semantic = await this.deps.semanticPreflight(scope.runtime.workspace);
|
|
if (!semantic.ok) return baseResult(scope.runtime, "preprocess evidence", "failed", semantic.code, { runId: scope.job.runId });
|
|
if (scope.job.completedStages.includes("evidence") && !options.dryRun) {
|
|
return baseResult(scope.runtime, "preprocess evidence", "unchanged", "ok", {
|
|
runId: scope.job.runId,
|
|
completedStages: [...scope.job.completedStages],
|
|
});
|
|
}
|
|
const payload = await this.runJsonStage(scope.runtime, [
|
|
"preprocess", "evidence",
|
|
...(options.dryRun ? ["--dry-run"] : []),
|
|
...(scope.job.childRuns.evidence ? ["--resume", scope.job.childRuns.evidence] : []),
|
|
"--json", "-c", "/dev/fd/3",
|
|
]);
|
|
if (typeof payload.run_id === "string") scope.job.childRuns.evidence = this.requireRunId(payload.run_id);
|
|
if (!options.dryRun && !scope.job.completedStages.includes("evidence")) scope.job.completedStages.push("evidence");
|
|
this.state(scope.runtime.workspaceId).writeJob(scope.job);
|
|
return baseResult(scope.runtime, "preprocess evidence", options.dryRun ? "dry_run" : "succeeded", "ok", {
|
|
runId: scope.job.runId,
|
|
childRuns: { ...scope.job.childRuns },
|
|
completedStages: [...scope.job.completedStages],
|
|
counts: this.numberRecord(payload.counts),
|
|
});
|
|
}
|
|
|
|
async run(options: { workspaceId: string; resumeRunId?: string }): Promise<WorkspaceOperationResult> {
|
|
const scope = await this.startRun(options.workspaceId, "preprocess run", options.resumeRunId);
|
|
if (!scope.job.completedStages.includes("dwh")) {
|
|
const payload = await this.runJsonStage(scope.runtime, [
|
|
"preprocess", "dwh", "--steps", "introspect,lsh",
|
|
...(scope.job.childRuns.dwh ? ["--resume", scope.job.childRuns.dwh] : []),
|
|
"--json", "-c", "/dev/fd/3",
|
|
]);
|
|
scope.job.childRuns.dwh = this.requireRunId(payload.run_id);
|
|
scope.job.completedStages.push("dwh");
|
|
this.state(scope.runtime.workspaceId).writeJob(scope.job);
|
|
}
|
|
if (!scope.job.completedStages.includes("fk_suggest")) {
|
|
const suggest = await this.runSuggestStage(scope, [], []);
|
|
if (suggest.code === "manual_review_required") return suggest;
|
|
}
|
|
const candidate = this.state(scope.runtime.workspaceId).readFkCandidates(scope.job.runId);
|
|
if (candidate && !scope.job.completedStages.includes("fk_review")) {
|
|
// P5: continuation requires a review accepted for this candidate whose accepted blob digest
|
|
// equals the current revision's synced annotations. A revision change (or a missing curated
|
|
// blob) therefore records a new review checkpoint instead of silently reusing the old one.
|
|
const accepted = this.findAcceptedReviewForDigest(scope.runtime.workspaceId, candidate.digest);
|
|
const currentDigest = this.currentAnnotationsDigest(scope.runtime);
|
|
if (accepted === undefined || currentDigest === undefined || accepted.annotationsDigest !== currentDigest) {
|
|
return baseResult(scope.runtime, "preprocess run", "blocked", "manual_review_required", {
|
|
runId: scope.job.runId,
|
|
childRuns: { ...scope.job.childRuns },
|
|
completedStages: [...scope.job.completedStages],
|
|
artifactIdentities: [{ kind: "fk_candidates", digest: candidate.digest }],
|
|
});
|
|
}
|
|
scope.job.reviewDigest = accepted.reviewedCandidatesDigest;
|
|
scope.job.completedStages.push("fk_review");
|
|
this.state(scope.runtime.workspaceId).writeJob(scope.job);
|
|
}
|
|
const semantic = await this.deps.semanticPreflight(scope.runtime.workspace);
|
|
if (!semantic.ok) return baseResult(scope.runtime, "preprocess run", "failed", semantic.code, { runId: scope.job.runId });
|
|
if (!scope.job.completedStages.includes("schema_index")) {
|
|
const payload = await this.runJsonStage(scope.runtime, ["vector", "index-schema", "--json", "-c", "/dev/fd/3"]);
|
|
scope.job.completedStages.push("schema_index");
|
|
this.state(scope.runtime.workspaceId).writeJob(scope.job);
|
|
const warnings = noEvidenceWarning(scope.runtime.workspace);
|
|
if (scope.runtime.workspace.evidence === undefined) {
|
|
return baseResult(scope.runtime, "preprocess run", "succeeded", "ok", {
|
|
runId: scope.job.runId,
|
|
childRuns: { ...scope.job.childRuns },
|
|
completedStages: [...scope.job.completedStages],
|
|
counts: this.numberRecord(payload.counts),
|
|
warnings,
|
|
});
|
|
}
|
|
}
|
|
if (scope.runtime.workspace.evidence === undefined) {
|
|
return baseResult(scope.runtime, "preprocess run", "succeeded", "ok", {
|
|
runId: scope.job.runId,
|
|
childRuns: { ...scope.job.childRuns },
|
|
completedStages: [...scope.job.completedStages],
|
|
warnings: noEvidenceWarning(scope.runtime.workspace),
|
|
});
|
|
}
|
|
const policy = this.evidencePolicy(scope.runtime.workspace);
|
|
if (policy !== undefined) {
|
|
return baseResult(scope.runtime, "preprocess run", policy.status, policy.code, {
|
|
runId: scope.job.runId,
|
|
childRuns: { ...scope.job.childRuns },
|
|
completedStages: [...scope.job.completedStages],
|
|
warnings: policy.warnings,
|
|
});
|
|
}
|
|
if (!scope.job.completedStages.includes("evidence")) {
|
|
const payload = await this.runJsonStage(scope.runtime, [
|
|
"preprocess", "evidence",
|
|
...(scope.job.childRuns.evidence ? ["--resume", scope.job.childRuns.evidence] : []),
|
|
"--json", "-c", "/dev/fd/3",
|
|
]);
|
|
if (typeof payload.run_id === "string") scope.job.childRuns.evidence = this.requireRunId(payload.run_id);
|
|
scope.job.completedStages.push("evidence");
|
|
this.state(scope.runtime.workspaceId).writeJob(scope.job);
|
|
return baseResult(scope.runtime, "preprocess run", "succeeded", "ok", {
|
|
runId: scope.job.runId,
|
|
childRuns: { ...scope.job.childRuns },
|
|
completedStages: [...scope.job.completedStages],
|
|
counts: this.numberRecord(payload.counts),
|
|
});
|
|
}
|
|
return baseResult(scope.runtime, "preprocess run", "unchanged", "ok", {
|
|
runId: scope.job.runId,
|
|
childRuns: { ...scope.job.childRuns },
|
|
completedStages: [...scope.job.completedStages],
|
|
});
|
|
}
|
|
|
|
private async startRun(workspaceId: string, operation: string, resumeRunId?: string): Promise<RunScope> {
|
|
const runtime = await this.deps.acquireActiveRuntime(workspaceId);
|
|
const state = this.state(runtime.workspaceId);
|
|
await state.assertSessionInventoryCompatible(runtime.workspaceRevision, await this.deps.listSessions(runtime.workspaceId));
|
|
const job = await state.beginJob({
|
|
operation,
|
|
runId: resumeRunId,
|
|
workspaceRevision: runtime.workspaceRevision,
|
|
descriptorBlob: runtime.descriptorBlob,
|
|
catalogBlob: runtime.catalogBlob,
|
|
configDigest: runtime.configLease.configDigest,
|
|
bindingDigest: runtime.configLease.bindingDigest,
|
|
});
|
|
return { runtime, state, job };
|
|
}
|
|
|
|
private state(workspaceId: string): PreprocessingStateStore {
|
|
return new PreprocessingStateStore({ dataRoot: this.deps.dataRoot, workspaceId });
|
|
}
|
|
|
|
private async runSuggestStage(
|
|
scope: RunScope,
|
|
fromSql: ReadonlyArray<{ name: string; sql: string }>,
|
|
assume: readonly string[],
|
|
): Promise<WorkspaceOperationResult> {
|
|
const payload = await this.withStagedInputs(scope.runtime.workspaceId, fromSql.map((entry) => ({
|
|
flag: "--from-sql",
|
|
name: entry.name,
|
|
contents: entry.sql,
|
|
})), async (stagedArgv) => await this.runJsonStage(scope.runtime, [
|
|
"schema", "suggest-fks", ...stagedArgv,
|
|
...assume.flatMap((value) => ["--assume", value]),
|
|
"--json", "-c", "/dev/fd/3",
|
|
]));
|
|
const candidateCount = Number(payload.candidate_count ?? 0);
|
|
const candidateYaml = typeof payload.candidate_yaml === "string" ? payload.candidate_yaml : "";
|
|
let artifactIdentities: Array<{ kind: string; digest: string }> | undefined;
|
|
if (candidateCount > 0) {
|
|
const persisted = this.state(scope.runtime.workspaceId).writeFkCandidates(scope.job.runId, candidateYaml);
|
|
scope.job.candidateDigest = persisted.digest;
|
|
artifactIdentities = [{ kind: "fk_candidates", digest: persisted.digest }];
|
|
}
|
|
if (!scope.job.completedStages.includes("fk_suggest")) scope.job.completedStages.push("fk_suggest");
|
|
this.state(scope.runtime.workspaceId).writeJob(scope.job);
|
|
const resultExtra = {
|
|
runId: scope.job.runId,
|
|
completedStages: [...scope.job.completedStages],
|
|
...(artifactIdentities ? { artifactIdentities } : {}),
|
|
...(candidateYaml.length > 0 ? { suggestedFksYaml: candidateYaml } : {}),
|
|
};
|
|
if (candidateCount > 0) {
|
|
return baseResult(scope.runtime, scope.job.operation === "preprocess run" ? "preprocess run" : "schema suggest-fks", "blocked", "manual_review_required", {
|
|
...resultExtra,
|
|
childRuns: { ...scope.job.childRuns },
|
|
});
|
|
}
|
|
return baseResult(scope.runtime, scope.job.operation === "preprocess run" ? "preprocess run" : "schema suggest-fks", "succeeded", "ok", resultExtra);
|
|
}
|
|
|
|
private async runJsonStage(runtime: ActiveRuntime, argv: string[]): Promise<Record<string, unknown>> {
|
|
const result = await this.deps.runChild({ argv, configPath: runtime.configLease.path });
|
|
if (result.exitCode !== 0) throw new Error("workspace child failed");
|
|
return JSON.parse(result.stdout) as Record<string, unknown>;
|
|
}
|
|
|
|
private requireRunId(value: unknown): string {
|
|
if (typeof value !== "string" || !/^[0-9a-f]{32}$/.test(value)) throw new Error("child run id is invalid");
|
|
return value;
|
|
}
|
|
|
|
private numberRecord(value: unknown): Record<string, number> | undefined {
|
|
if (!value || typeof value !== "object" || Array.isArray(value)) return undefined;
|
|
return Object.fromEntries(Object.entries(value as Record<string, unknown>).map(([key, nested]) => [key, Number(nested)]));
|
|
}
|
|
|
|
private async withStagedInputs<T>(
|
|
workspaceId: string,
|
|
inputs: ReadonlyArray<{ flag: string; name: string; contents: string }>,
|
|
fn: (argv: string[]) => Promise<T>,
|
|
): Promise<T> {
|
|
if (inputs.length === 0) return await fn([]);
|
|
const root = join(this.deps.dataRoot, "sessions", workspaceId, "preprocessing", `.stage-${randomBytes(6).toString("hex")}`);
|
|
mkdirSync(root, { recursive: true, mode: 0o700 });
|
|
const argv: string[] = [];
|
|
const paths: string[] = [];
|
|
try {
|
|
for (const input of inputs) {
|
|
const path = join(root, input.name);
|
|
writeFileSync(path, input.contents, { encoding: "utf8", flag: "wx", mode: 0o600 });
|
|
paths.push(path);
|
|
argv.push(input.flag, path);
|
|
}
|
|
return await fn(argv);
|
|
} finally {
|
|
rmSync(root, { recursive: true, force: true });
|
|
}
|
|
}
|
|
|
|
private currentAnnotationsDigest(runtime: ActiveRuntime): string | undefined {
|
|
return readAnnotationsSync(this.deps.dataRoot, runtime.workspaceId, runtime.workspaceRevision)?.contentDigest;
|
|
}
|
|
|
|
private findAcceptedReviewForDigest(
|
|
workspaceId: string,
|
|
digestValue: string,
|
|
): FkReviewRecord | undefined {
|
|
const state = this.state(workspaceId);
|
|
for (const entry of readdirSync(state.fkReviewsDirectory(), { withFileTypes: true })) {
|
|
if (!entry.isFile() || entry.isSymbolicLink() || !/^[0-9a-f]{32}\.json$/.test(entry.name)) continue;
|
|
const runId = entry.name.slice(0, -".json".length);
|
|
const review = state.readFkReview(runId);
|
|
if (review && review.reviewedCandidatesDigest === digestValue) return review;
|
|
}
|
|
return undefined;
|
|
}
|
|
|
|
private findRunIdByCandidateDigest(state: PreprocessingStateStore, digestValue: string): string | undefined {
|
|
for (const entry of readdirSync(state.fkCandidatesDirectory(), { withFileTypes: true })) {
|
|
if (!entry.isFile() || entry.isSymbolicLink() || !/^[0-9a-f]{32}\.yaml$/.test(entry.name)) continue;
|
|
const runId = entry.name.slice(0, -".yaml".length);
|
|
if (state.readFkCandidates(runId)?.digest === digestValue) return runId;
|
|
}
|
|
return undefined;
|
|
}
|
|
|
|
private evidencePolicy(workspace: WorkspaceDescriptor): {
|
|
status: WorkspaceOperationResult["status"];
|
|
code: WorkspaceOperationResult["code"];
|
|
warnings?: string[];
|
|
} | undefined {
|
|
const evidence = workspace.evidence;
|
|
if (!evidence) return undefined;
|
|
// P6: filesystem Evidence is materialized from the pinned commit at activation, so the
|
|
// engine may proceed directly against the immutable revision content root.
|
|
if (evidence.source.type === "filesystem") return undefined;
|
|
if (evidence.source.type === "http") {
|
|
for (const value of evidence.source.uris) {
|
|
const host = new URL(value).hostname;
|
|
if (isPrivateHost(host) && !(evidence.source.allow_private_hosts && this.deps.httpPrivateHostAllowlist?.includes(host))) {
|
|
return { status: "failed", code: "egress_policy_refused" };
|
|
}
|
|
}
|
|
return undefined;
|
|
}
|
|
if (
|
|
evidence.source.endpoint_url !== undefined
|
|
|| evidence.source.credentials === "ambient"
|
|
|| evidence.source.allow_private_endpoint
|
|
|| evidence.source.allow_insecure_endpoint
|
|
) {
|
|
return { status: "failed", code: "egress_policy_refused" };
|
|
}
|
|
return undefined;
|
|
}
|
|
}
|