refactor(evidence): extract TypeScript lifecycle (#32)
This commit is contained in:
+5
-2
@@ -9,7 +9,7 @@ import {
|
|||||||
writeFileSync,
|
writeFileSync,
|
||||||
} from "node:fs";
|
} from "node:fs";
|
||||||
import { dirname, isAbsolute, join } from "node:path";
|
import { dirname, isAbsolute, join } from "node:path";
|
||||||
import { GitWorkspaceRepository } from "./git-repository.js";
|
import type { GitWorkspaceRepository } from "../git-repository.js";
|
||||||
|
|
||||||
export interface EvidenceMaterializationLimits {
|
export interface EvidenceMaterializationLimits {
|
||||||
maxEntries: number;
|
maxEntries: number;
|
||||||
@@ -81,7 +81,10 @@ function writeExclusiveNoFollow(path: string, contents: Buffer, mode: number): v
|
|||||||
}
|
}
|
||||||
|
|
||||||
export interface MaterializeEvidenceTreeOptions {
|
export interface MaterializeEvidenceTreeOptions {
|
||||||
repository: GitWorkspaceRepository;
|
repository: Pick<
|
||||||
|
GitWorkspaceRepository,
|
||||||
|
"evidenceTreeObjects" | "evidenceTreeId" | "gitObjectSize" | "evidenceBlobBytes"
|
||||||
|
>;
|
||||||
revision: string;
|
revision: string;
|
||||||
id: string;
|
id: string;
|
||||||
/** The workspace directory (e.g. `<staging>/<id>`) that will receive `evidence/` and the manifest. */
|
/** The workspace directory (e.g. `<staging>/<id>`) that will receive `evidence/` and the manifest. */
|
||||||
@@ -0,0 +1,177 @@
|
|||||||
|
import { isIP } from "node:net";
|
||||||
|
import type { WorkspaceDescriptor } from "../schema.js";
|
||||||
|
|
||||||
|
type EvidenceConfig = WorkspaceDescriptor["evidence"];
|
||||||
|
type SemanticFailureCode = "workspace_not_activatable" | "semantic_index_incompatible";
|
||||||
|
|
||||||
|
export interface EvidenceJobState {
|
||||||
|
runId: string;
|
||||||
|
completedStages: string[];
|
||||||
|
childRuns: Record<string, string>;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface EvidencePreprocessingDependencies {
|
||||||
|
runStage(argv: string[]): Promise<Record<string, unknown>>;
|
||||||
|
persistJob(): void;
|
||||||
|
semanticPreflight(): Promise<{ ok: true } | { ok: false; code: SemanticFailureCode }>;
|
||||||
|
requireRunId(value: unknown): string;
|
||||||
|
numberRecord(value: unknown): Record<string, number> | undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface EvidencePreprocessingRequest {
|
||||||
|
evidence: EvidenceConfig;
|
||||||
|
job: EvidenceJobState;
|
||||||
|
dryRun?: boolean;
|
||||||
|
httpPrivateHostAllowlist?: readonly string[];
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface EvidencePreprocessingOutcome {
|
||||||
|
status: "succeeded" | "unchanged" | "dry_run" | "failed";
|
||||||
|
code: "ok" | "egress_policy_refused" | SemanticFailureCode;
|
||||||
|
runId?: string;
|
||||||
|
childRuns?: Record<string, string>;
|
||||||
|
completedStages?: string[];
|
||||||
|
counts?: Record<string, number>;
|
||||||
|
warnings?: string[];
|
||||||
|
}
|
||||||
|
|
||||||
|
function isPrivateHost(hostname: string): boolean {
|
||||||
|
if (hostname === "localhost" || hostname === "metadata.google.internal") return true;
|
||||||
|
const address = isIP(hostname);
|
||||||
|
if (address === 4) {
|
||||||
|
if (/^127\./.test(hostname) || /^10\./.test(hostname) || /^192\.168\./.test(hostname)) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
if (/^169\.254\./.test(hostname) || /^0\./.test(hostname)) return true;
|
||||||
|
const match = /^172\.(\d+)\./.exec(hostname);
|
||||||
|
return Boolean(match && Number(match[1]) >= 16 && Number(match[1]) <= 31);
|
||||||
|
}
|
||||||
|
if (address === 6) {
|
||||||
|
const normalized = hostname.toLowerCase();
|
||||||
|
return normalized === "::1"
|
||||||
|
|| normalized.startsWith("fe80:")
|
||||||
|
|| normalized.startsWith("fd")
|
||||||
|
|| normalized.startsWith("fc");
|
||||||
|
}
|
||||||
|
return hostname.endsWith(".internal");
|
||||||
|
}
|
||||||
|
|
||||||
|
function evidencePolicy(
|
||||||
|
evidence: EvidenceConfig,
|
||||||
|
httpPrivateHostAllowlist?: readonly string[],
|
||||||
|
): EvidencePreprocessingOutcome | undefined {
|
||||||
|
if (!evidence || evidence.source.type === "filesystem") return undefined;
|
||||||
|
if (evidence.source.type === "http") {
|
||||||
|
for (const value of evidence.source.uris) {
|
||||||
|
const host = new URL(value).hostname;
|
||||||
|
if (
|
||||||
|
isPrivateHost(host)
|
||||||
|
&& !(evidence.source.allow_private_hosts && httpPrivateHostAllowlist?.includes(host))
|
||||||
|
) {
|
||||||
|
return { status: "failed", code: "egress_policy_refused" };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
if (
|
||||||
|
evidence.source.endpoint_url !== undefined
|
||||||
|
|| evidence.source.credentials === "ambient"
|
||||||
|
|| evidence.source.allow_private_endpoint
|
||||||
|
|| evidence.source.allow_insecure_endpoint
|
||||||
|
) {
|
||||||
|
return { status: "failed", code: "egress_policy_refused" };
|
||||||
|
}
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
function jobResult(job: EvidenceJobState): Pick<
|
||||||
|
EvidencePreprocessingOutcome,
|
||||||
|
"runId" | "childRuns" | "completedStages"
|
||||||
|
> {
|
||||||
|
return {
|
||||||
|
runId: job.runId,
|
||||||
|
childRuns: { ...job.childRuns },
|
||||||
|
completedStages: [...job.completedStages],
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
async function runEvidenceStage(
|
||||||
|
request: EvidencePreprocessingRequest,
|
||||||
|
deps: EvidencePreprocessingDependencies,
|
||||||
|
): Promise<EvidencePreprocessingOutcome> {
|
||||||
|
const payload = await deps.runStage([
|
||||||
|
"preprocess",
|
||||||
|
"evidence",
|
||||||
|
...(request.dryRun ? ["--dry-run"] : []),
|
||||||
|
...(request.job.childRuns.evidence
|
||||||
|
? ["--resume", request.job.childRuns.evidence]
|
||||||
|
: []),
|
||||||
|
"--json",
|
||||||
|
"-c",
|
||||||
|
"/dev/fd/3",
|
||||||
|
]);
|
||||||
|
if (typeof payload.run_id === "string") {
|
||||||
|
request.job.childRuns.evidence = deps.requireRunId(payload.run_id);
|
||||||
|
}
|
||||||
|
if (!request.dryRun && !request.job.completedStages.includes("evidence")) {
|
||||||
|
request.job.completedStages.push("evidence");
|
||||||
|
}
|
||||||
|
deps.persistJob();
|
||||||
|
return {
|
||||||
|
status: request.dryRun ? "dry_run" : "succeeded",
|
||||||
|
code: "ok",
|
||||||
|
...jobResult(request.job),
|
||||||
|
counts: deps.numberRecord(payload.counts),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function preprocessEvidence(
|
||||||
|
request: EvidencePreprocessingRequest,
|
||||||
|
deps: EvidencePreprocessingDependencies,
|
||||||
|
): Promise<EvidencePreprocessingOutcome> {
|
||||||
|
if (!request.evidence) {
|
||||||
|
return {
|
||||||
|
status: "unchanged",
|
||||||
|
code: "ok",
|
||||||
|
warnings: ["workspace has no Evidence source"],
|
||||||
|
};
|
||||||
|
}
|
||||||
|
const policy = evidencePolicy(request.evidence, request.httpPrivateHostAllowlist);
|
||||||
|
if (policy) return policy;
|
||||||
|
const semantic = await deps.semanticPreflight();
|
||||||
|
if (!semantic.ok) {
|
||||||
|
return { status: "failed", code: semantic.code, runId: request.job.runId };
|
||||||
|
}
|
||||||
|
if (request.job.completedStages.includes("evidence") && !request.dryRun) {
|
||||||
|
return {
|
||||||
|
status: "unchanged",
|
||||||
|
code: "ok",
|
||||||
|
runId: request.job.runId,
|
||||||
|
completedStages: [...request.job.completedStages],
|
||||||
|
};
|
||||||
|
}
|
||||||
|
return await runEvidenceStage(request, deps);
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function continueEvidencePreprocessing(
|
||||||
|
request: Omit<EvidencePreprocessingRequest, "dryRun"> & {
|
||||||
|
priorCounts?: Record<string, number>;
|
||||||
|
},
|
||||||
|
deps: EvidencePreprocessingDependencies,
|
||||||
|
): Promise<EvidencePreprocessingOutcome> {
|
||||||
|
if (!request.evidence) {
|
||||||
|
return {
|
||||||
|
status: "succeeded",
|
||||||
|
code: "ok",
|
||||||
|
...jobResult(request.job),
|
||||||
|
warnings: ["workspace has no Evidence source"],
|
||||||
|
...(request.priorCounts ? { counts: request.priorCounts } : {}),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
const policy = evidencePolicy(request.evidence, request.httpPrivateHostAllowlist);
|
||||||
|
if (policy) return { ...policy, ...jobResult(request.job) };
|
||||||
|
if (!request.job.completedStages.includes("evidence")) {
|
||||||
|
return await runEvidenceStage(request, deps);
|
||||||
|
}
|
||||||
|
return { status: "unchanged", code: "ok", ...jobResult(request.job) };
|
||||||
|
}
|
||||||
@@ -1,7 +1,12 @@
|
|||||||
import { createHash, randomBytes } from "node:crypto";
|
import { createHash, randomBytes } from "node:crypto";
|
||||||
import { readdirSync, readFileSync, rmSync, writeFileSync, mkdirSync } from "node:fs";
|
import { readdirSync, readFileSync, rmSync, writeFileSync, mkdirSync } from "node:fs";
|
||||||
import { isIP } from "node:net";
|
|
||||||
import { join } from "node:path";
|
import { join } from "node:path";
|
||||||
|
import {
|
||||||
|
continueEvidencePreprocessing,
|
||||||
|
preprocessEvidence as runEvidencePreprocessing,
|
||||||
|
type EvidencePreprocessingDependencies,
|
||||||
|
type EvidencePreprocessingOutcome,
|
||||||
|
} from "./evidence/preprocessing.js";
|
||||||
import type { WorkspaceDescriptor } from "./schema.js";
|
import type { WorkspaceDescriptor } from "./schema.js";
|
||||||
import {
|
import {
|
||||||
PreprocessingStateStore,
|
PreprocessingStateStore,
|
||||||
@@ -104,26 +109,6 @@ function baseResult(
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
function isPrivateHost(hostname: string): boolean {
|
|
||||||
if (hostname === "localhost" || hostname === "metadata.google.internal") return true;
|
|
||||||
const address = isIP(hostname);
|
|
||||||
if (address === 4) {
|
|
||||||
if (/^127\./.test(hostname) || /^10\./.test(hostname) || /^192\.168\./.test(hostname)) return true;
|
|
||||||
if (/^169\.254\./.test(hostname) || /^0\./.test(hostname)) return true;
|
|
||||||
const match = /^172\.(\d+)\./.exec(hostname);
|
|
||||||
return Boolean(match && Number(match[1]) >= 16 && Number(match[1]) <= 31);
|
|
||||||
}
|
|
||||||
if (address === 6) {
|
|
||||||
const normalized = hostname.toLowerCase();
|
|
||||||
return normalized === "::1" || normalized.startsWith("fe80:") || normalized.startsWith("fd") || normalized.startsWith("fc");
|
|
||||||
}
|
|
||||||
return hostname.endsWith(".internal");
|
|
||||||
}
|
|
||||||
|
|
||||||
function noEvidenceWarning(workspace: WorkspaceDescriptor): string[] {
|
|
||||||
return workspace.evidence === undefined ? ["workspace has no Evidence source"] : [];
|
|
||||||
}
|
|
||||||
|
|
||||||
export class WorkspacePreprocessingService {
|
export class WorkspacePreprocessingService {
|
||||||
constructor(private readonly deps: WorkspacePreprocessingServiceDeps) {}
|
constructor(private readonly deps: WorkspacePreprocessingServiceDeps) {}
|
||||||
|
|
||||||
@@ -332,36 +317,16 @@ export class WorkspacePreprocessingService {
|
|||||||
|
|
||||||
async preprocessEvidence(options: { workspaceId: string; dryRun?: boolean; resumeRunId?: string }): Promise<WorkspaceOperationResult> {
|
async preprocessEvidence(options: { workspaceId: string; dryRun?: boolean; resumeRunId?: string }): Promise<WorkspaceOperationResult> {
|
||||||
const scope = await this.startRun(options.workspaceId, "preprocess evidence", options.resumeRunId);
|
const scope = await this.startRun(options.workspaceId, "preprocess evidence", options.resumeRunId);
|
||||||
if (scope.runtime.workspace.evidence === undefined) {
|
const outcome = await runEvidencePreprocessing(
|
||||||
return baseResult(scope.runtime, "preprocess evidence", "unchanged", "ok", {
|
{
|
||||||
warnings: noEvidenceWarning(scope.runtime.workspace),
|
evidence: scope.runtime.workspace.evidence,
|
||||||
});
|
job: scope.job,
|
||||||
}
|
dryRun: options.dryRun,
|
||||||
const policy = this.evidencePolicy(scope.runtime.workspace);
|
httpPrivateHostAllowlist: this.deps.httpPrivateHostAllowlist,
|
||||||
if (policy !== undefined) return baseResult(scope.runtime, "preprocess evidence", policy.status, policy.code, { warnings: policy.warnings });
|
},
|
||||||
const semantic = await this.deps.semanticPreflight(scope.runtime.workspace);
|
this.evidenceDependencies(scope),
|
||||||
if (!semantic.ok) return baseResult(scope.runtime, "preprocess evidence", "failed", semantic.code, { runId: scope.job.runId });
|
);
|
||||||
if (scope.job.completedStages.includes("evidence") && !options.dryRun) {
|
return this.evidenceResult(scope, "preprocess evidence", outcome);
|
||||||
return baseResult(scope.runtime, "preprocess evidence", "unchanged", "ok", {
|
|
||||||
runId: scope.job.runId,
|
|
||||||
completedStages: [...scope.job.completedStages],
|
|
||||||
});
|
|
||||||
}
|
|
||||||
const payload = await this.runJsonStage(scope.runtime, [
|
|
||||||
"preprocess", "evidence",
|
|
||||||
...(options.dryRun ? ["--dry-run"] : []),
|
|
||||||
...(scope.job.childRuns.evidence ? ["--resume", scope.job.childRuns.evidence] : []),
|
|
||||||
"--json", "-c", "/dev/fd/3",
|
|
||||||
]);
|
|
||||||
if (typeof payload.run_id === "string") scope.job.childRuns.evidence = this.requireRunId(payload.run_id);
|
|
||||||
if (!options.dryRun && !scope.job.completedStages.includes("evidence")) scope.job.completedStages.push("evidence");
|
|
||||||
this.state(scope.runtime.workspaceId).writeJob(scope.job);
|
|
||||||
return baseResult(scope.runtime, "preprocess evidence", options.dryRun ? "dry_run" : "succeeded", "ok", {
|
|
||||||
runId: scope.job.runId,
|
|
||||||
childRuns: { ...scope.job.childRuns },
|
|
||||||
completedStages: [...scope.job.completedStages],
|
|
||||||
counts: this.numberRecord(payload.counts),
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async run(options: { workspaceId: string; resumeRunId?: string }): Promise<WorkspaceOperationResult> {
|
async run(options: { workspaceId: string; resumeRunId?: string }): Promise<WorkspaceOperationResult> {
|
||||||
@@ -401,59 +366,23 @@ export class WorkspacePreprocessingService {
|
|||||||
}
|
}
|
||||||
const semantic = await this.deps.semanticPreflight(scope.runtime.workspace);
|
const semantic = await this.deps.semanticPreflight(scope.runtime.workspace);
|
||||||
if (!semantic.ok) return baseResult(scope.runtime, "preprocess run", "failed", semantic.code, { runId: scope.job.runId });
|
if (!semantic.ok) return baseResult(scope.runtime, "preprocess run", "failed", semantic.code, { runId: scope.job.runId });
|
||||||
|
let schemaCounts: Record<string, number> | undefined;
|
||||||
if (!scope.job.completedStages.includes("schema_index")) {
|
if (!scope.job.completedStages.includes("schema_index")) {
|
||||||
const payload = await this.runJsonStage(scope.runtime, ["vector", "index-schema", "--json", "-c", "/dev/fd/3"]);
|
const payload = await this.runJsonStage(scope.runtime, ["vector", "index-schema", "--json", "-c", "/dev/fd/3"]);
|
||||||
scope.job.completedStages.push("schema_index");
|
scope.job.completedStages.push("schema_index");
|
||||||
this.state(scope.runtime.workspaceId).writeJob(scope.job);
|
this.state(scope.runtime.workspaceId).writeJob(scope.job);
|
||||||
const warnings = noEvidenceWarning(scope.runtime.workspace);
|
schemaCounts = this.numberRecord(payload.counts);
|
||||||
if (scope.runtime.workspace.evidence === undefined) {
|
|
||||||
return baseResult(scope.runtime, "preprocess run", "succeeded", "ok", {
|
|
||||||
runId: scope.job.runId,
|
|
||||||
childRuns: { ...scope.job.childRuns },
|
|
||||||
completedStages: [...scope.job.completedStages],
|
|
||||||
counts: this.numberRecord(payload.counts),
|
|
||||||
warnings,
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
}
|
const outcome = await continueEvidencePreprocessing(
|
||||||
if (scope.runtime.workspace.evidence === undefined) {
|
{
|
||||||
return baseResult(scope.runtime, "preprocess run", "succeeded", "ok", {
|
evidence: scope.runtime.workspace.evidence,
|
||||||
runId: scope.job.runId,
|
job: scope.job,
|
||||||
childRuns: { ...scope.job.childRuns },
|
httpPrivateHostAllowlist: this.deps.httpPrivateHostAllowlist,
|
||||||
completedStages: [...scope.job.completedStages],
|
priorCounts: schemaCounts,
|
||||||
warnings: noEvidenceWarning(scope.runtime.workspace),
|
},
|
||||||
});
|
this.evidenceDependencies(scope),
|
||||||
}
|
);
|
||||||
const policy = this.evidencePolicy(scope.runtime.workspace);
|
return this.evidenceResult(scope, "preprocess run", outcome);
|
||||||
if (policy !== undefined) {
|
|
||||||
return baseResult(scope.runtime, "preprocess run", policy.status, policy.code, {
|
|
||||||
runId: scope.job.runId,
|
|
||||||
childRuns: { ...scope.job.childRuns },
|
|
||||||
completedStages: [...scope.job.completedStages],
|
|
||||||
warnings: policy.warnings,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
if (!scope.job.completedStages.includes("evidence")) {
|
|
||||||
const payload = await this.runJsonStage(scope.runtime, [
|
|
||||||
"preprocess", "evidence",
|
|
||||||
...(scope.job.childRuns.evidence ? ["--resume", scope.job.childRuns.evidence] : []),
|
|
||||||
"--json", "-c", "/dev/fd/3",
|
|
||||||
]);
|
|
||||||
if (typeof payload.run_id === "string") scope.job.childRuns.evidence = this.requireRunId(payload.run_id);
|
|
||||||
scope.job.completedStages.push("evidence");
|
|
||||||
this.state(scope.runtime.workspaceId).writeJob(scope.job);
|
|
||||||
return baseResult(scope.runtime, "preprocess run", "succeeded", "ok", {
|
|
||||||
runId: scope.job.runId,
|
|
||||||
childRuns: { ...scope.job.childRuns },
|
|
||||||
completedStages: [...scope.job.completedStages],
|
|
||||||
counts: this.numberRecord(payload.counts),
|
|
||||||
});
|
|
||||||
}
|
|
||||||
return baseResult(scope.runtime, "preprocess run", "unchanged", "ok", {
|
|
||||||
runId: scope.job.runId,
|
|
||||||
childRuns: { ...scope.job.childRuns },
|
|
||||||
completedStages: [...scope.job.completedStages],
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private async startRun(workspaceId: string, operation: string, resumeRunId?: string): Promise<RunScope> {
|
private async startRun(workspaceId: string, operation: string, resumeRunId?: string): Promise<RunScope> {
|
||||||
@@ -476,6 +405,25 @@ export class WorkspacePreprocessingService {
|
|||||||
return new PreprocessingStateStore({ dataRoot: this.deps.dataRoot, workspaceId });
|
return new PreprocessingStateStore({ dataRoot: this.deps.dataRoot, workspaceId });
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private evidenceDependencies(scope: RunScope): EvidencePreprocessingDependencies {
|
||||||
|
return {
|
||||||
|
runStage: async (argv) => await this.runJsonStage(scope.runtime, argv),
|
||||||
|
persistJob: () => this.state(scope.runtime.workspaceId).writeJob(scope.job),
|
||||||
|
semanticPreflight: async () => await this.deps.semanticPreflight(scope.runtime.workspace),
|
||||||
|
requireRunId: (value) => this.requireRunId(value),
|
||||||
|
numberRecord: (value) => this.numberRecord(value),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private evidenceResult(
|
||||||
|
scope: RunScope,
|
||||||
|
operation: "preprocess evidence" | "preprocess run",
|
||||||
|
outcome: EvidencePreprocessingOutcome,
|
||||||
|
): WorkspaceOperationResult {
|
||||||
|
const { status, code, ...extra } = outcome;
|
||||||
|
return baseResult(scope.runtime, operation, status, code, extra);
|
||||||
|
}
|
||||||
|
|
||||||
private async runSuggestStage(
|
private async runSuggestStage(
|
||||||
scope: RunScope,
|
scope: RunScope,
|
||||||
fromSql: ReadonlyArray<{ name: string; sql: string }>,
|
fromSql: ReadonlyArray<{ name: string; sql: string }>,
|
||||||
@@ -581,33 +529,4 @@ export class WorkspacePreprocessingService {
|
|||||||
return undefined;
|
return undefined;
|
||||||
}
|
}
|
||||||
|
|
||||||
private evidencePolicy(workspace: WorkspaceDescriptor): {
|
|
||||||
status: WorkspaceOperationResult["status"];
|
|
||||||
code: WorkspaceOperationResult["code"];
|
|
||||||
warnings?: string[];
|
|
||||||
} | undefined {
|
|
||||||
const evidence = workspace.evidence;
|
|
||||||
if (!evidence) return undefined;
|
|
||||||
// P6: filesystem Evidence is materialized from the pinned commit at activation, so the
|
|
||||||
// engine may proceed directly against the immutable revision content root.
|
|
||||||
if (evidence.source.type === "filesystem") return undefined;
|
|
||||||
if (evidence.source.type === "http") {
|
|
||||||
for (const value of evidence.source.uris) {
|
|
||||||
const host = new URL(value).hostname;
|
|
||||||
if (isPrivateHost(host) && !(evidence.source.allow_private_hosts && this.deps.httpPrivateHostAllowlist?.includes(host))) {
|
|
||||||
return { status: "failed", code: "egress_policy_refused" };
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return undefined;
|
|
||||||
}
|
|
||||||
if (
|
|
||||||
evidence.source.endpoint_url !== undefined
|
|
||||||
|| evidence.source.credentials === "ambient"
|
|
||||||
|| evidence.source.allow_private_endpoint
|
|
||||||
|| evidence.source.allow_insecure_endpoint
|
|
||||||
) {
|
|
||||||
return { status: "failed", code: "egress_policy_refused" };
|
|
||||||
}
|
|
||||||
return undefined;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -5,7 +5,7 @@ import { isAbsolute, join } from "node:path";
|
|||||||
import { buildInstallationContract, renderWorkspaceDocs } from "./contracts.js";
|
import { buildInstallationContract, renderWorkspaceDocs } from "./contracts.js";
|
||||||
import { parseAnnotationsYaml } from "./annotations.js";
|
import { parseAnnotationsYaml } from "./annotations.js";
|
||||||
import { syncAnnotations } from "./annotations-sync.js";
|
import { syncAnnotations } from "./annotations-sync.js";
|
||||||
import { materializeEvidenceTree } from "./evidence-materialization.js";
|
import { materializeEvidenceTree } from "./evidence/materialization.js";
|
||||||
import { assertCatalogMatchesDescriptor, parseWorkspaceCatalogYaml, type WorkspaceCatalog, type WorkspaceCatalogEntry } from "./catalog.js";
|
import { assertCatalogMatchesDescriptor, parseWorkspaceCatalogYaml, type WorkspaceCatalog, type WorkspaceCatalogEntry } from "./catalog.js";
|
||||||
import {
|
import {
|
||||||
GitWorkspaceRepository,
|
GitWorkspaceRepository,
|
||||||
|
|||||||
@@ -0,0 +1,34 @@
|
|||||||
|
import { readdirSync, readFileSync } from "node:fs";
|
||||||
|
import { basename, join } from "node:path";
|
||||||
|
import { fileURLToPath } from "node:url";
|
||||||
|
import ts from "typescript";
|
||||||
|
import { expect, test } from "vitest";
|
||||||
|
|
||||||
|
const evidenceRoot = fileURLToPath(new URL("../../../src/workspaces/evidence/", import.meta.url));
|
||||||
|
const coreInfrastructure = new Set([
|
||||||
|
"preprocessing-service",
|
||||||
|
"qdrant-collection",
|
||||||
|
"registry",
|
||||||
|
]);
|
||||||
|
|
||||||
|
test("Evidence modules do not import core-owned registry or Qdrant lifecycle", () => {
|
||||||
|
const violations: Array<{ file: string; dependency: string }> = [];
|
||||||
|
for (const file of readdirSync(evidenceRoot).filter((name) => name.endsWith(".ts"))) {
|
||||||
|
const source = ts.createSourceFile(
|
||||||
|
file,
|
||||||
|
readFileSync(join(evidenceRoot, file), "utf8"),
|
||||||
|
ts.ScriptTarget.Latest,
|
||||||
|
true,
|
||||||
|
ts.ScriptKind.TS,
|
||||||
|
);
|
||||||
|
for (const statement of source.statements) {
|
||||||
|
if (!ts.isImportDeclaration(statement) || !ts.isStringLiteral(statement.moduleSpecifier)) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
const dependency = basename(statement.moduleSpecifier.text).replace(/\.js$/, "");
|
||||||
|
if (coreInfrastructure.has(dependency)) violations.push({ file, dependency });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
expect(violations).toEqual([]);
|
||||||
|
});
|
||||||
+3
-3
@@ -4,9 +4,9 @@ import { tmpdir } from "node:os";
|
|||||||
import { join } from "node:path";
|
import { join } from "node:path";
|
||||||
import { promisify } from "node:util";
|
import { promisify } from "node:util";
|
||||||
import { afterEach, expect, test } from "vitest";
|
import { afterEach, expect, test } from "vitest";
|
||||||
import { GitWorkspaceRepository } from "../src/workspaces/git-repository.js";
|
import { GitWorkspaceRepository } from "../../../src/workspaces/git-repository.js";
|
||||||
import { materializeEvidenceTree } from "../src/workspaces/evidence-materialization.js";
|
import { materializeEvidenceTree } from "../../../src/workspaces/evidence/materialization.js";
|
||||||
import type { WorkspaceRegistryConfig } from "../src/workspaces/types.js";
|
import type { WorkspaceRegistryConfig } from "../../../src/workspaces/types.js";
|
||||||
|
|
||||||
const runFile = promisify(execFile);
|
const runFile = promisify(execFile);
|
||||||
const temporaryRoots: string[] = [];
|
const temporaryRoots: string[] = [];
|
||||||
@@ -0,0 +1,161 @@
|
|||||||
|
import { expect, test, vi } from "vitest";
|
||||||
|
import {
|
||||||
|
continueEvidencePreprocessing,
|
||||||
|
preprocessEvidence,
|
||||||
|
type EvidenceJobState,
|
||||||
|
type EvidencePreprocessingDependencies,
|
||||||
|
} from "../../../src/workspaces/evidence/preprocessing.js";
|
||||||
|
import type { WorkspaceDescriptor } from "../../../src/workspaces/schema.js";
|
||||||
|
|
||||||
|
type EvidenceConfig = NonNullable<WorkspaceDescriptor["evidence"]>;
|
||||||
|
|
||||||
|
const filesystemEvidence = {
|
||||||
|
source: { type: "filesystem", uri: "research/evidence" },
|
||||||
|
} as EvidenceConfig;
|
||||||
|
|
||||||
|
const privateHttpEvidence = {
|
||||||
|
source: {
|
||||||
|
type: "http",
|
||||||
|
uris: ["http://127.0.0.1/private.md"],
|
||||||
|
authentication: "none",
|
||||||
|
connect_timeout_ms: 1000,
|
||||||
|
read_timeout_ms: 2000,
|
||||||
|
max_bytes: 100,
|
||||||
|
max_redirects: 0,
|
||||||
|
allow_private_hosts: true,
|
||||||
|
max_cache_bytes: 100,
|
||||||
|
},
|
||||||
|
} as EvidenceConfig;
|
||||||
|
|
||||||
|
function job(overrides: Partial<EvidenceJobState> = {}): EvidenceJobState {
|
||||||
|
return {
|
||||||
|
runId: "a".repeat(32),
|
||||||
|
childRuns: {},
|
||||||
|
completedStages: [],
|
||||||
|
...overrides,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function dependencies(payload: Record<string, unknown> = {}): EvidencePreprocessingDependencies & {
|
||||||
|
runStage: ReturnType<typeof vi.fn>;
|
||||||
|
persistJob: ReturnType<typeof vi.fn>;
|
||||||
|
semanticPreflight: ReturnType<typeof vi.fn>;
|
||||||
|
} {
|
||||||
|
return {
|
||||||
|
runStage: vi.fn(async () => payload),
|
||||||
|
persistJob: vi.fn(),
|
||||||
|
semanticPreflight: vi.fn(async () => ({ ok: true as const })),
|
||||||
|
requireRunId(value) {
|
||||||
|
if (typeof value !== "string" || !/^[0-9a-f]{32}$/.test(value)) {
|
||||||
|
throw new Error("child run id is invalid");
|
||||||
|
}
|
||||||
|
return value;
|
||||||
|
},
|
||||||
|
numberRecord(value) {
|
||||||
|
if (!value || typeof value !== "object" || Array.isArray(value)) return undefined;
|
||||||
|
return Object.fromEntries(
|
||||||
|
Object.entries(value as Record<string, unknown>).map(([key, nested]) => [key, Number(nested)]),
|
||||||
|
);
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
test("owns the standalone Evidence stage argv and mutation order", async () => {
|
||||||
|
const state = job({ childRuns: { evidence: "b".repeat(32) } });
|
||||||
|
const deps = dependencies({ run_id: "c".repeat(32), counts: { added: 2 } });
|
||||||
|
|
||||||
|
const result = await preprocessEvidence(
|
||||||
|
{ evidence: filesystemEvidence, job: state, dryRun: false },
|
||||||
|
deps,
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(deps.semanticPreflight).toHaveBeenCalledOnce();
|
||||||
|
expect(deps.runStage).toHaveBeenCalledWith([
|
||||||
|
"preprocess", "evidence", "--resume", "b".repeat(32), "--json", "-c", "/dev/fd/3",
|
||||||
|
]);
|
||||||
|
expect(deps.persistJob).toHaveBeenCalledOnce();
|
||||||
|
expect(state).toMatchObject({
|
||||||
|
childRuns: { evidence: "c".repeat(32) },
|
||||||
|
completedStages: ["evidence"],
|
||||||
|
});
|
||||||
|
expect(result).toEqual({
|
||||||
|
status: "succeeded",
|
||||||
|
code: "ok",
|
||||||
|
runId: "a".repeat(32),
|
||||||
|
childRuns: { evidence: "c".repeat(32) },
|
||||||
|
completedStages: ["evidence"],
|
||||||
|
counts: { added: 2 },
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
test("owns Evidence egress refusal before shared semantic infrastructure", async () => {
|
||||||
|
const deps = dependencies();
|
||||||
|
|
||||||
|
const result = await preprocessEvidence(
|
||||||
|
{
|
||||||
|
evidence: privateHttpEvidence,
|
||||||
|
job: job(),
|
||||||
|
httpPrivateHostAllowlist: ["metadata.internal"],
|
||||||
|
},
|
||||||
|
deps,
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(result).toEqual({ status: "failed", code: "egress_policy_refused" });
|
||||||
|
expect(deps.semanticPreflight).not.toHaveBeenCalled();
|
||||||
|
expect(deps.runStage).not.toHaveBeenCalled();
|
||||||
|
expect(deps.persistJob).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
test("projects aggregate no-Evidence and completed-stage outcomes without rerunning", async () => {
|
||||||
|
const deps = dependencies();
|
||||||
|
const noEvidence = await continueEvidencePreprocessing(
|
||||||
|
{
|
||||||
|
evidence: undefined,
|
||||||
|
job: job({ completedStages: ["dwh", "schema_index"] }),
|
||||||
|
priorCounts: { added: 2 },
|
||||||
|
},
|
||||||
|
deps,
|
||||||
|
);
|
||||||
|
const completed = await continueEvidencePreprocessing(
|
||||||
|
{
|
||||||
|
evidence: filesystemEvidence,
|
||||||
|
job: job({ completedStages: ["dwh", "schema_index", "evidence"] }),
|
||||||
|
},
|
||||||
|
deps,
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(noEvidence).toMatchObject({
|
||||||
|
status: "succeeded",
|
||||||
|
code: "ok",
|
||||||
|
warnings: ["workspace has no Evidence source"],
|
||||||
|
counts: { added: 2 },
|
||||||
|
});
|
||||||
|
expect(completed).toMatchObject({
|
||||||
|
status: "unchanged",
|
||||||
|
code: "ok",
|
||||||
|
completedStages: ["dwh", "schema_index", "evidence"],
|
||||||
|
});
|
||||||
|
expect(deps.runStage).not.toHaveBeenCalled();
|
||||||
|
expect(deps.persistJob).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
test("preserves the narrow standalone projection for an already completed Evidence stage", async () => {
|
||||||
|
const deps = dependencies();
|
||||||
|
|
||||||
|
const result = await preprocessEvidence(
|
||||||
|
{
|
||||||
|
evidence: filesystemEvidence,
|
||||||
|
job: job({ childRuns: { evidence: "b".repeat(32) }, completedStages: ["evidence"] }),
|
||||||
|
},
|
||||||
|
deps,
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(result).toEqual({
|
||||||
|
status: "unchanged",
|
||||||
|
code: "ok",
|
||||||
|
runId: "a".repeat(32),
|
||||||
|
completedStages: ["evidence"],
|
||||||
|
});
|
||||||
|
expect(deps.runStage).not.toHaveBeenCalled();
|
||||||
|
expect(deps.persistJob).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
@@ -59,7 +59,10 @@ From `backend/`, the passing automated baseline is:
|
|||||||
npx vitest run test/tht-runner.test.ts test/pi-process-manager.test.ts \
|
npx vitest run test/tht-runner.test.ts test/pi-process-manager.test.ts \
|
||||||
test/session-bridge.test.ts test/sse-hub.test.ts test/sse-route.test.ts \
|
test/session-bridge.test.ts test/sse-hub.test.ts test/sse-route.test.ts \
|
||||||
test/routes-sessions.test.ts test/e2e-f1.test.ts \
|
test/routes-sessions.test.ts test/e2e-f1.test.ts \
|
||||||
test/workspace-preprocessing-service.test.ts test/evidence-materialization.test.ts
|
test/workspace-preprocessing-service.test.ts \
|
||||||
|
test/workspaces/evidence/materialization.test.ts \
|
||||||
|
test/workspaces/evidence/preprocessing.test.ts \
|
||||||
|
test/workspaces/evidence/boundary.test.ts
|
||||||
npx tsc --noEmit -p .
|
npx tsc --noEmit -p .
|
||||||
npm run build
|
npm run build
|
||||||
```
|
```
|
||||||
|
|||||||
Reference in New Issue
Block a user