Codex
82e2c91f42
Publish documentation / publish (push) Successful in 1m27s
Add PostgreSQL-backed memory, editable evidence with source review and activation, and human-approved archive repairs across the harness, API, and UI. Include migrations, deployment support, regression coverage, and validation documentation. Refresh permissions from validated session roles so existing administrator logins can access newly deployed archive management features.
320 lines
12 KiB
TypeScript
320 lines
12 KiB
TypeScript
import { basename, dirname, join } from "node:path";
|
|
import { stringify } from "yaml";
|
|
import { buildInstallationContract } from "./contracts.js";
|
|
import { validateWorkspaceDescriptor, type WorkspaceDescriptor } from "./schema.js";
|
|
import type { ResolvedBinding, ResolvedEvidenceBinding, RuntimeBindings } from "./bindings.js";
|
|
import { workspaceVectorCollections } from "./vector-collections.js";
|
|
export type { RuntimeBindings } from "./bindings.js";
|
|
|
|
export interface RuntimePaths {
|
|
sessions: string;
|
|
artifacts: string;
|
|
indexes: string;
|
|
memory: string;
|
|
/** Current backend-produced projection of the PostgreSQL Metadata Catalog. */
|
|
catalog_metadata_snapshot?: string;
|
|
}
|
|
|
|
export interface RuntimeIdentity {
|
|
workspaceId: string;
|
|
workspaceRevision: string;
|
|
}
|
|
|
|
/** Immutable, explicit inputs needed to bind descriptor-relative content to one revision. */
|
|
export interface RuntimeRenderContext extends RuntimeIdentity {
|
|
revisionContentRoot: string;
|
|
}
|
|
|
|
export interface RuntimeInstallationOverlay {
|
|
session_storage?: unknown;
|
|
profile?: unknown;
|
|
}
|
|
|
|
export interface SemanticRuntimeConfig {
|
|
internalQdrantUrl: string;
|
|
internalEmbeddingUrl: string;
|
|
internalEmbeddingId?: string;
|
|
internalEmbeddingModel: string;
|
|
internalEmbeddingDimensions: number;
|
|
}
|
|
|
|
export const DEFAULT_SEMANTIC_RUNTIME: SemanticRuntimeConfig = {
|
|
internalQdrantUrl: "http://qdrant:6333",
|
|
internalEmbeddingUrl: "http://embedding:11434",
|
|
internalEmbeddingId: "ollama/qwen3-embedding:0.6b",
|
|
internalEmbeddingModel: "qwen3-embedding:0.6b",
|
|
internalEmbeddingDimensions: 1024,
|
|
};
|
|
|
|
function bindingValue(binding: ResolvedBinding, name: string): string | undefined {
|
|
return binding.values[name];
|
|
}
|
|
|
|
function requireBinding(binding: ResolvedBinding, name: string): string {
|
|
const value = bindingValue(binding, name);
|
|
if (value === undefined) throw new Error(`runtime binding is missing ${name}`);
|
|
return value;
|
|
}
|
|
|
|
function directConnection(
|
|
binding: ResolvedBinding,
|
|
names: { host: string; port: string; user: string; passwordFile: string; tlsCaFile: string },
|
|
identity: { database: string; schema: string },
|
|
): Record<string, unknown> {
|
|
const connection: Record<string, unknown> = {
|
|
host: requireBinding(binding, names.host),
|
|
port: Number(requireBinding(binding, names.port)),
|
|
database: identity.database,
|
|
schema: identity.schema,
|
|
user: requireBinding(binding, names.user),
|
|
password_file: requireBinding(binding, names.passwordFile),
|
|
};
|
|
const tlsCaFile = bindingValue(binding, names.tlsCaFile);
|
|
if (tlsCaFile !== undefined) connection.ssl_ca_file = tlsCaFile;
|
|
return connection;
|
|
}
|
|
|
|
function restEndpoint(
|
|
binding: ResolvedBinding,
|
|
names: { baseUrl: string; apiKeyFile: string; tlsCaFile: string },
|
|
requiresCredential: boolean,
|
|
): Record<string, unknown> {
|
|
const endpoint: Record<string, unknown> = {
|
|
base_url: requireBinding(binding, names.baseUrl),
|
|
};
|
|
if (requiresCredential) endpoint.api_key_file = requireBinding(binding, names.apiKeyFile);
|
|
const tlsCaFile = bindingValue(binding, names.tlsCaFile);
|
|
if (tlsCaFile !== undefined) endpoint.ssl_ca_file = tlsCaFile;
|
|
return endpoint;
|
|
}
|
|
|
|
function exactSeconds(timeoutMs: number): number {
|
|
return timeoutMs / 1_000;
|
|
}
|
|
|
|
function requireRuntimeRenderContext(
|
|
identity: RuntimeIdentity | RuntimeRenderContext | undefined,
|
|
): RuntimeRenderContext {
|
|
if (!identity || !("revisionContentRoot" in identity)) {
|
|
throw new Error("runtime Evidence requires an immutable revision content root");
|
|
}
|
|
return identity;
|
|
}
|
|
|
|
function evidenceBindingValue(binding: ResolvedEvidenceBinding, name: string): string | undefined {
|
|
return binding.values[name];
|
|
}
|
|
|
|
function requireEvidenceBinding(binding: ResolvedEvidenceBinding, name: string): string {
|
|
const value = evidenceBindingValue(binding, name);
|
|
if (value === undefined) throw new Error(`runtime binding is missing ${name}`);
|
|
return value;
|
|
}
|
|
|
|
function renderEvidence(
|
|
workspace: WorkspaceDescriptor,
|
|
binding: ResolvedEvidenceBinding,
|
|
context: RuntimeRenderContext,
|
|
bindingName: (suffix: string) => string,
|
|
): { evidence: Record<string, unknown>; vector: Record<string, unknown> } | undefined {
|
|
if (workspace.evidence === undefined) return undefined;
|
|
if (binding.missing.length > 0) {
|
|
throw new Error("runtime configuration requires complete Evidence bindings");
|
|
}
|
|
if (basename(context.revisionContentRoot) !== context.workspaceRevision) {
|
|
throw new Error("runtime revision content root does not match workspace revision");
|
|
}
|
|
|
|
const source = workspace.evidence.source;
|
|
let renderedSource: Record<string, unknown>;
|
|
if (source.type === "filesystem") {
|
|
renderedSource = {
|
|
type: "filesystem",
|
|
root: join(context.revisionContentRoot, source.uri),
|
|
patterns: source.patterns,
|
|
max_bytes: source.max_bytes,
|
|
};
|
|
} else if (source.type === "http") {
|
|
renderedSource = {
|
|
type: "http",
|
|
...(source.authentication === "none"
|
|
? { urls: source.uris }
|
|
: {
|
|
provenance_urls: source.uris,
|
|
signed_urls_file: requireEvidenceBinding(binding, bindingName("SIGNED_URLS_FILE")),
|
|
}),
|
|
connect_timeout: exactSeconds(source.connect_timeout_ms),
|
|
read_timeout: exactSeconds(source.read_timeout_ms),
|
|
max_bytes: source.max_bytes,
|
|
max_redirects: source.max_redirects,
|
|
allow_private_hosts: source.allow_private_hosts,
|
|
max_cache_bytes: source.max_cache_bytes,
|
|
};
|
|
} else {
|
|
const uri = new URL(source.uri);
|
|
const sessionTokenFile = source.credentials === "static_files"
|
|
? evidenceBindingValue(binding, bindingName("SESSION_TOKEN_FILE"))
|
|
: undefined;
|
|
renderedSource = {
|
|
type: "s3",
|
|
bucket: uri.hostname,
|
|
prefix: uri.pathname.replace(/^\//, ""),
|
|
...(source.endpoint_url === undefined ? {} : { endpoint_url: source.endpoint_url }),
|
|
...(source.region === undefined ? {} : { region: source.region }),
|
|
...(source.credentials === "ambient" ? {} : {
|
|
access_key_file: requireEvidenceBinding(binding, bindingName("ACCESS_KEY_FILE")),
|
|
secret_key_file: requireEvidenceBinding(binding, bindingName("SECRET_KEY_FILE")),
|
|
...(sessionTokenFile === undefined ? {} : { session_token_file: sessionTokenFile }),
|
|
}),
|
|
trusted_endpoint: source.trusted_endpoint,
|
|
allow_private_endpoint: source.allow_private_endpoint,
|
|
allow_insecure_endpoint: source.allow_insecure_endpoint,
|
|
max_bytes: source.max_bytes,
|
|
max_objects: source.max_objects,
|
|
max_pages: source.max_pages,
|
|
page_size: source.page_size,
|
|
};
|
|
}
|
|
|
|
return {
|
|
evidence: {
|
|
...(workspace.evidence.schema_version === 2 ? { schema_version: 2 } : {}),
|
|
local_archive_root: join(dirname(dirname(context.revisionContentRoot)), "repo", context.workspaceId),
|
|
sources: [renderedSource],
|
|
},
|
|
vector: {
|
|
max_chunk_chars: workspace.evidence.policy.max_chunk_chars,
|
|
retain_published_generations: workspace.evidence.policy.retain_published_generations,
|
|
},
|
|
};
|
|
}
|
|
|
|
|
|
function placeholderConnection(identity: { database: string; schema: string }): Record<string, unknown> {
|
|
return {
|
|
host: "localhost",
|
|
port: 5432,
|
|
database: identity.database,
|
|
schema: identity.schema,
|
|
user: "rest",
|
|
password: "",
|
|
transport: "rest",
|
|
};
|
|
}
|
|
|
|
function requireSupportedDescriptor(workspace: unknown): void {
|
|
if (typeof workspace !== "object" || workspace === null) {
|
|
throw new Error("Runtime renderer supports only workspace schema version 4");
|
|
}
|
|
const metadata = Reflect.get(workspace, "workspace");
|
|
if (typeof metadata !== "object" || metadata === null
|
|
|| Reflect.get(metadata, "schema_version") !== 4) {
|
|
throw new Error("Runtime renderer supports only workspace schema version 4");
|
|
}
|
|
}
|
|
|
|
/** Render the schema-v4 compatibility fields consumed by the current Python harness. */
|
|
export function renderRuntimeConfig(
|
|
workspace: WorkspaceDescriptor,
|
|
bindings: RuntimeBindings,
|
|
paths: RuntimePaths,
|
|
identity?: RuntimeIdentity | RuntimeRenderContext,
|
|
installation: RuntimeInstallationOverlay = {},
|
|
semanticRuntime: SemanticRuntimeConfig = DEFAULT_SEMANTIC_RUNTIME,
|
|
): string {
|
|
requireSupportedDescriptor(workspace);
|
|
const descriptor = validateWorkspaceDescriptor(workspace);
|
|
if (!descriptor.dwh) throw new Error("runtime configuration requires a Catalog database binding");
|
|
const contract = buildInstallationContract(descriptor);
|
|
const name = (role: "DWH" | "EVIDENCE", suffix: string) => {
|
|
const variable = contract.variables.find((entry) => entry.role === role && entry.suffix === suffix);
|
|
if (!variable) throw new Error(`workspace contract is missing ${role}_${suffix}`);
|
|
return variable.name;
|
|
};
|
|
const renderedEvidence = descriptor.evidence === undefined
|
|
? undefined
|
|
: renderEvidence(
|
|
descriptor,
|
|
bindings.evidence,
|
|
requireRuntimeRenderContext(identity),
|
|
(suffix) => name("EVIDENCE", suffix),
|
|
);
|
|
if (bindings.dwh.missing.length > 0) {
|
|
throw new Error("runtime configuration requires complete bindings");
|
|
}
|
|
|
|
const dwhIdentity = { database: descriptor.dwh.database, schema: descriptor.dwh.schema };
|
|
const vectorCollections = workspaceVectorCollections(descriptor.workspace.id);
|
|
const database = bindings.dwh.transport === "postgres_direct"
|
|
? { ...directConnection(bindings.dwh, {
|
|
host: name("DWH", "HOST"),
|
|
port: name("DWH", "PORT"),
|
|
user: name("DWH", "USER"),
|
|
passwordFile: name("DWH", "PASSWORD_FILE"),
|
|
tlsCaFile: name("DWH", "TLS_CA_FILE"),
|
|
}, dwhIdentity), transport: "direct" }
|
|
: placeholderConnection(dwhIdentity);
|
|
const rendered: Record<string, unknown> = {
|
|
...(identity ? {
|
|
runtime_identity: {
|
|
workspace_id: identity.workspaceId,
|
|
workspace_revision: identity.workspaceRevision,
|
|
source_identity: `workspace://${identity.workspaceId}`,
|
|
},
|
|
} : {}),
|
|
...(installation.session_storage === undefined
|
|
? {} : { session_storage: installation.session_storage }),
|
|
...(installation.profile === undefined ? {} : { profile: installation.profile }),
|
|
language: descriptor.workspace.language,
|
|
database,
|
|
semantic_index: {
|
|
vector_store: {
|
|
engine: "qdrant",
|
|
collections: vectorCollections,
|
|
dimensions: semanticRuntime.internalEmbeddingDimensions,
|
|
distance: "cosine",
|
|
},
|
|
embedding: {
|
|
provider: "ollama_internal",
|
|
id: semanticRuntime.internalEmbeddingId
|
|
?? `ollama/${semanticRuntime.internalEmbeddingModel}`,
|
|
model: semanticRuntime.internalEmbeddingModel,
|
|
dimensions: semanticRuntime.internalEmbeddingDimensions,
|
|
},
|
|
},
|
|
resources: {
|
|
vector: {
|
|
engine: "qdrant",
|
|
base_url: semanticRuntime.internalQdrantUrl,
|
|
collections: vectorCollections,
|
|
},
|
|
embeddings: {
|
|
provider: "ollama_internal",
|
|
base_url: semanticRuntime.internalEmbeddingUrl,
|
|
id: semanticRuntime.internalEmbeddingId
|
|
?? `ollama/${semanticRuntime.internalEmbeddingModel}`,
|
|
model: semanticRuntime.internalEmbeddingModel,
|
|
dimensions: semanticRuntime.internalEmbeddingDimensions,
|
|
},
|
|
},
|
|
roots: paths,
|
|
paths,
|
|
...(renderedEvidence ?? {}),
|
|
};
|
|
if (bindings.dwh.transport === "postgres_direct") {
|
|
rendered.dwh = { type: "postgres_direct", connection: database };
|
|
} else if (bindings.dwh.transport === "rest_api") {
|
|
const rest = restEndpoint(bindings.dwh, {
|
|
baseUrl: name("DWH", "BASE_URL"),
|
|
apiKeyFile: name("DWH", "API_KEY_FILE"),
|
|
tlsCaFile: name("DWH", "TLS_CA_FILE"),
|
|
}, descriptor.diagnostics?.dwh_rest?.auth !== "none");
|
|
rendered.rest = rest;
|
|
rendered.database = placeholderConnection(dwhIdentity);
|
|
rendered.dwh = { type: "thoth_rest", database: dwhIdentity, endpoint: rest };
|
|
} else {
|
|
throw new Error("ssh_tunnel runtime configuration requires a diagnostic tunnel");
|
|
}
|
|
return stringify(rendered, { lineWidth: 0, sortMapEntries: false });
|
|
}
|