feat: publish verified installation images and native release bundles
This commit is contained in:
Codex
@@ -5,8 +5,8 @@ import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import { dirname, join, resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const backend = resolve(dirname(fileURLToPath(import.meta.url)), "..");
|
||||
const repository = resolve(backend, "..");
|
||||
const repository = process.env.THT_BUILD_SOURCE_ROOT ? resolve(process.env.THT_BUILD_SOURCE_ROOT) : resolve(dirname(fileURLToPath(import.meta.url)), "../..");
|
||||
const backend = join(repository, "backend");
|
||||
const native = `${process.platform === "win32" ? "windows" : process.platform}-${process.arch === "x64" ? "amd64" : process.arch}`;
|
||||
const targets = {
|
||||
"windows-amd64": ["windows", "amd64", "bun-windows-x64"],
|
||||
@@ -28,7 +28,9 @@ function run(command, args, cwd = backend, env = process.env) {
|
||||
const revision = spawnSync("git", ["rev-parse", "HEAD"], { cwd: repository, encoding: "utf8" });
|
||||
if (revision.status !== 0) throw new Error("Cannot read build revision");
|
||||
const commit = revision.stdout.trim();
|
||||
const buildTime = new Date().toISOString();
|
||||
const buildTime = process.env.THT_BUILD_TIME ?? new Date().toISOString();
|
||||
const releaseVersion = process.env.THT_BUILD_VERSION ?? "0.0.0-dev";
|
||||
if (!/^[0-9]+\.[0-9]+\.[0-9]+(?:-[A-Za-z0-9.-]+)?$/.test(releaseVersion) || !Number.isFinite(Date.parse(buildTime))) throw new Error("Invalid build identity");
|
||||
const module = "github.com/aritmolab/thothii/tools/tht/internal/version";
|
||||
const bunPackage = JSON.parse(readFileSync(join(backend, "node_modules", "bun", "package.json"), "utf8"));
|
||||
const bun = join(backend, "node_modules", "bun", bunPackage.bin.bun);
|
||||
@@ -39,9 +41,9 @@ for (const target of selected) {
|
||||
const extension = os === "windows" ? ".exe" : "";
|
||||
const names = [`tht${extension}`, `tht-workspace-documents${extension}`];
|
||||
run(bun, ["build", "src/workspace-documents-cli.ts", "--compile", `--target=${bunTarget}`, "--outfile", join(output, names[1])]);
|
||||
run("go", ["build", "-trimpath", "-ldflags", `-s -w -X ${module}.semanticVersion=0.0.0-dev -X ${module}.commit=${commit} -X ${module}.buildTime=${buildTime}`, "-o", join(output, names[0]), "./cmd/tht"], join(repository, "tools", "tht"), { ...process.env, CGO_ENABLED: "0", GOOS: os, GOARCH: arch });
|
||||
run("go", ["build", "-trimpath", "-ldflags", `-s -w -X ${module}.semanticVersion=${releaseVersion} -X ${module}.commit=${commit} -X ${module}.buildTime=${buildTime}`, "-o", join(output, names[0]), "./cmd/tht"], join(repository, "tools", "tht"), { ...process.env, CGO_ENABLED: "0", GOOS: os, GOARCH: arch });
|
||||
const hashes = names.map((name) => `${createHash("sha256").update(readFileSync(join(output, name))).digest("hex")} ${name}\n`).join("");
|
||||
writeFileSync(join(output, "SHA256SUMS"), hashes);
|
||||
writeFileSync(join(output, "build.json"), JSON.stringify({ commit, buildTime, target, bun: JSON.parse(readFileSync(join(backend, "package.json"), "utf8")).devDependencies.bun }, null, 2) + "\n");
|
||||
writeFileSync(join(output, "build.json"), JSON.stringify({ version: releaseVersion, commit, buildTime, target, bun: JSON.parse(readFileSync(join(backend, "package.json"), "utf8")).devDependencies.bun }, null, 2) + "\n");
|
||||
console.log(output);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,195 @@
|
||||
#!/usr/bin/env node
|
||||
// Maintainer-only producer. Consumers download the resulting native bundle.
|
||||
import { spawn } from "node:child_process";
|
||||
import { mkdirSync, mkdtempSync, readFileSync, writeFileSync, existsSync, realpathSync, renameSync, rmSync, statSync, copyFileSync, chmodSync, openSync, closeSync, readdirSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { basename, dirname, join, resolve } from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import { parse } from "yaml";
|
||||
import { prepareBundle, sha256 } from "./release-bundle.mjs";
|
||||
import { dockerCredentials, dockerHub } from "./release-registry.mjs";
|
||||
import { giteaHosting } from "./release-hosting.mjs";
|
||||
import { publishVerifiedRelease } from "./release-publication.mjs";
|
||||
|
||||
const repositoryRoot = resolve(import.meta.dirname, "../..");
|
||||
export function optionsFromArgs(args) {
|
||||
const values = {};
|
||||
for (let i = 0; i < args.length; i += 2) {
|
||||
if (!['--revision', '--version', '--namespace', '--platforms', '--output', '--repository'].includes(args[i]) || !args[i + 1] || values[args[i]]) throw new Error("Usage: --revision REF --version VERSION --namespace DOCKER_HUB_NAMESPACE --platforms linux/amd64 --output NEW_OR_MATCHING_DIRECTORY [--repository HTTPS_GITEA_REPO]");
|
||||
values[args[i]] = args[i + 1];
|
||||
}
|
||||
if (!values['--revision'] || values['--revision'].startsWith('-') || !/^[0-9]+\.[0-9]+\.[0-9]+(?:-[A-Za-z0-9.-]+)?$/.test(values['--version'] ?? '') || !/^[a-z0-9][a-z0-9_-]{1,38}$/.test(values['--namespace'] ?? '') || !values['--output']) throw new Error("Supply an explicit source revision, semantic release version, Docker Hub namespace and output directory.");
|
||||
const platforms = (values['--platforms'] ?? '').split(',');
|
||||
if (!platforms.length || new Set(platforms).size !== platforms.length || platforms.some((p) => !['linux/amd64', 'linux/arm64'].includes(p))) throw new Error("Select explicit Linux image platforms; begin with linux/amd64 for Windows/WSL2 and Omarchy.");
|
||||
const repository = values['--repository'] ?? 'https://git.tylconsulting.it/mptyl/ThothII';
|
||||
const url = new URL(repository);
|
||||
if (url.protocol !== 'https:' || url.username || url.password || url.search || url.hash || !/^\/[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/.test(url.pathname)) throw new Error("Use a credential-free HTTPS Gitea owner/repository URL.");
|
||||
return { revision: values['--revision'], version: values['--version'], namespace: values['--namespace'], platforms: platforms.sort(), output: resolve(values['--output']), repository };
|
||||
}
|
||||
export function commandRunner(logs) {
|
||||
let sequence = 0;
|
||||
return async (command, args, { cwd = repositoryRoot, input = '', env = process.env, quiet = false, timeout = 120_000 } = {}) => {
|
||||
const log = join(logs, `${++sequence}-${basename(command)}.log`);
|
||||
return new Promise((accept, reject) => {
|
||||
if (process.platform === 'win32') { reject(new Error('Run the release producer on Linux, WSL2 or macOS.')); return; }
|
||||
const child = spawn(command, args, { cwd, env, detached: true, stdio: ['pipe', 'pipe', 'pipe'] });
|
||||
const stdout = [], stderr = []; let size = 0, overflow = false, settled = false, reapTimer;
|
||||
const terminate = () => {
|
||||
if (overflow) return;
|
||||
overflow = true;
|
||||
try { process.kill(-child.pid, 'SIGKILL'); } catch { child.kill('SIGKILL'); }
|
||||
// An escaped descendant must never retain our pipes indefinitely.
|
||||
reapTimer = setTimeout(() => { child.stdout.destroy(); child.stderr.destroy(); finish(-1); }, 500);
|
||||
};
|
||||
const timer = setTimeout(terminate, timeout);
|
||||
const collect = (chunks) => (data) => { size += data.length; if (size > 64 * 2 ** 20) terminate(); else chunks.push(data); };
|
||||
child.stdout.on('data', collect(stdout)); child.stderr.on('data', collect(stderr));
|
||||
child.stdin.on('error', () => {}); child.stdin.end(input);
|
||||
child.on('error', () => { settled = true; clearTimeout(timer); clearTimeout(reapTimer); reject(new Error(`Required maintainer command ${basename(command)} is unavailable.`)); });
|
||||
function finish(code) {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
clearTimeout(timer); clearTimeout(reapTimer);
|
||||
if (!quiet) writeFileSync(log, Buffer.concat([...stdout, ...stderr]), { mode: 0o600 });
|
||||
if (code !== 0 || overflow) reject(new Error(`${basename(command)} failed${quiet ? '.' : `; inspect private log ${log}`}`));
|
||||
else accept(Buffer.concat(stdout).toString());
|
||||
}
|
||||
child.on('close', finish);
|
||||
});
|
||||
};
|
||||
}
|
||||
function acquireOutput(output) {
|
||||
if (!existsSync(output)) mkdirSync(output, { mode: 0o700 });
|
||||
if (realpathSync(output) !== output || !statSync(output).isDirectory() || (statSync(output).mode & 0o077)) throw new Error("Use a canonical owner-only output directory.");
|
||||
if (!existsSync(join(output, 'publication-state.json')) && readdirSync(output).length) throw new Error("Choose an empty output directory or the matching previous publication directory.");
|
||||
const lock = join(output, '.publisher.lock');
|
||||
if (existsSync(lock)) {
|
||||
const pid = Number(readFileSync(lock, 'utf8'));
|
||||
if (!Number.isInteger(pid) || pid <= 0) throw new Error("Inspect the incomplete publisher lock before retrying.");
|
||||
let alive = true;
|
||||
try { process.kill(pid, 0); } catch (error) { if (error.code === 'ESRCH') alive = false; }
|
||||
if (alive) throw new Error("Another publisher owns this output directory.");
|
||||
rmSync(lock);
|
||||
}
|
||||
const fd = openSync(lock, 'wx', 0o600); writeFileSync(fd, String(process.pid)); closeSync(fd);
|
||||
return () => rmSync(lock, { force: true });
|
||||
}
|
||||
export async function publishInstallation(options) {
|
||||
const unlock = acquireOutput(options.output);
|
||||
const logs = join(options.output, 'logs'); mkdirSync(logs, { recursive: true, mode: 0o700 });
|
||||
const run = commandRunner(logs);
|
||||
let worktree, temporary;
|
||||
try {
|
||||
const revision = (await run('git', ['rev-parse', '--verify', `${options.revision}^{commit}`], { quiet: true })).trim();
|
||||
if (!/^[a-f0-9]{40}$/.test(revision)) throw new Error("Source revision is not a commit.");
|
||||
const identity = { revision, version: options.version, namespace: options.namespace, platforms: options.platforms, repository: options.repository };
|
||||
const statePath = join(options.output, 'publication-state.json');
|
||||
let state = { identity };
|
||||
if (existsSync(statePath)) {
|
||||
state = JSON.parse(readFileSync(statePath, 'utf8'));
|
||||
if (JSON.stringify(state.identity) !== JSON.stringify(identity)) throw new Error("Output directory belongs to a different release; choose a new directory.");
|
||||
}
|
||||
const save = () => { const temp = statePath + '.tmp'; writeFileSync(temp, JSON.stringify(state, null, 2) + '\n', { mode: 0o600 }); renameSync(temp, statePath); };
|
||||
save();
|
||||
console.log(`Release ${identity.version}: ${identity.namespace}, ${identity.platforms.join(',')}, source ${revision}`);
|
||||
await run('docker', ['info', '--format', '{{.OSType}}']);
|
||||
await run('docker', ['buildx', 'version']);
|
||||
const registry = dockerHub(await dockerCredentials(run));
|
||||
const hosting = await giteaHosting({ run, repository: options.repository, identity, body: `Installer prerelease for ${identity.platforms.join(', ')}.\n\nImages: docker.io/${identity.namespace}/thothii-core:${identity.version} and docker.io/${identity.namespace}/thothii-frontend:${identity.version}.\n\nDownload the native operator bundle and SHA256SUMS.txt below. This release supplies images, document validation and preflight; complete non-interactive setup and Windows/Omarchy/macOS acceptance are subsequent tickets. No example databases, credentials or user workspace data are included.` });
|
||||
async function prepare() {
|
||||
if (state.assets) {
|
||||
const names = [...identity.platforms.map((platform) => `thothii-${identity.version}-${platform.replace('/', '-')}.tar.gz`), 'SHA256SUMS.txt'];
|
||||
if (state.assets.length !== names.length) throw new Error("Cached artifact set is incomplete.");
|
||||
for (const asset of state.assets) if (!names.includes(asset.name) || asset.path !== join(options.output, asset.name) || !existsSync(asset.path) || sha256(readFileSync(asset.path)) !== asset.sha256) throw new Error("Previously built release artifact changed; do not overwrite an immutable version.");
|
||||
for (const [platform, images] of Object.entries(state.images)) for (const reference of Object.values(images)) {
|
||||
const [repository, digest] = reference.replace(/^docker.io\//, '').split('@');
|
||||
await registry.inspect(repository, digest, platform, { anonymous: true });
|
||||
}
|
||||
return state.assets;
|
||||
}
|
||||
temporary = realpathSync(mkdtempSync(join(tmpdir(), 'thothii-release-')));
|
||||
worktree = join(temporary, 'source');
|
||||
await run('git', ['worktree', 'add', '--detach', worktree, revision]);
|
||||
const sourceCompose = parse(readFileSync(join(worktree, 'compose.yaml'), 'utf8'));
|
||||
for (const role of ['core', 'frontend']) {
|
||||
console.log(`Preparing public repository ${identity.namespace}/thothii-${role}`);
|
||||
await registry.ensurePublic(identity.namespace, `thothii-${role}`);
|
||||
let existing = null;
|
||||
for (const platform of identity.platforms) {
|
||||
const image = await registry.inspect(`${identity.namespace}/thothii-${role}`, identity.version, platform, { allowMissing: true });
|
||||
if (image && (image.labels['org.opencontainers.image.revision'] !== revision || image.labels['org.opencontainers.image.version'] !== identity.version)) throw new Error("Image tag already belongs to another immutable build; select a new release version.");
|
||||
existing = existing || image;
|
||||
}
|
||||
if (!existing) {
|
||||
console.log(`Building and publishing ${role} (${identity.platforms.join(', ')})`);
|
||||
await run('docker', ['buildx', 'build', '--platform', identity.platforms.join(','), '--file', `docker/${role}.Dockerfile`, '--tag', `docker.io/${identity.namespace}/thothii-${role}:${identity.version}`, '--build-arg', `IMAGE_VERSION=${identity.version}`, '--label', `org.opencontainers.image.revision=${revision}`, '--label', `org.opencontainers.image.source=${identity.repository}`, '--provenance=false', '--sbom=false', '--push', '.'], { cwd: worktree, timeout: 45 * 60_000 });
|
||||
}
|
||||
}
|
||||
state.images = {};
|
||||
for (const platform of identity.platforms) {
|
||||
const images = {};
|
||||
for (const role of ['core', 'frontend']) images[role] = (await registry.inspect(`${identity.namespace}/thothii-${role}`, identity.version, platform, { anonymous: true })).reference;
|
||||
for (const [role, service] of [['catalog', 'catalog-db'], ['qdrant', 'qdrant'], ['embedding', 'embedding']]) {
|
||||
const [named, digest] = sourceCompose.services[service].image.split('@');
|
||||
let repository = named.replace(/:[^/:]+$/, '').replace(/^docker.io\//, '');
|
||||
if (!repository.includes('/')) repository = 'library/' + repository;
|
||||
images[role] = (await registry.inspect(repository, digest, platform, { anonymous: true })).reference;
|
||||
}
|
||||
state.images[platform] = images;
|
||||
}
|
||||
save();
|
||||
console.log('Building native operator bundles from the selected source');
|
||||
await run('npm', ['ci'], { cwd: join(worktree, 'backend'), timeout: 10 * 60_000 });
|
||||
const sourceTime = (await run('git', ['show', '-s', '--format=%cI', revision], { quiet: true })).trim();
|
||||
const targets = identity.platforms.map((platform) => platform.replace('/', '-'));
|
||||
await run('node', [join(repositoryRoot, 'backend/scripts/build-workspace-tools.mjs'), ...targets], { cwd: join(worktree, 'backend'), env: { ...process.env, THT_BUILD_SOURCE_ROOT: worktree, THT_BUILD_VERSION: identity.version, THT_BUILD_TIME: sourceTime }, timeout: 10 * 60_000 });
|
||||
const assets = [];
|
||||
for (const platform of identity.platforms) {
|
||||
const target = platform.replace('/', '-');
|
||||
const name = `thothii-${identity.version}-${target}`;
|
||||
const bundle = join(options.output, name);
|
||||
if (existsSync(bundle)) rmSync(bundle, { recursive: true }); // owned staging, never an installed runtime
|
||||
mkdirSync(bundle);
|
||||
prepareBundle({ source: worktree, destination: bundle, platform, version: identity.version, revision, images: state.images[platform] });
|
||||
mkdirSync(join(bundle, 'bin'));
|
||||
for (const executable of ['tht', 'tht-workspace-documents']) {
|
||||
copyFileSync(join(worktree, 'dist/workspace-tools', target, executable), join(bundle, 'bin', executable));
|
||||
chmodSync(join(bundle, 'bin', executable), 0o755);
|
||||
}
|
||||
// Resolve source-independent resource/config shape without any operator credentials.
|
||||
await run('docker', ['compose', '-f', join(bundle, 'compose.yaml'), '-f', join(bundle, 'deploy/compose.local.yaml'), 'config', '--no-interpolate', '--no-env-resolution', '--format', 'json']);
|
||||
const archive = join(options.output, name + '.tar.gz');
|
||||
await run('tar', ['-czf', archive, '-C', options.output, name]);
|
||||
assets.push({ name: basename(archive), path: archive, bytes: statSync(archive).size, sha256: sha256(readFileSync(archive)) });
|
||||
}
|
||||
const sums = join(options.output, 'SHA256SUMS.txt');
|
||||
writeFileSync(sums, assets.map((asset) => `${asset.sha256} ${asset.name}\n`).join(''));
|
||||
assets.push({ name: 'SHA256SUMS.txt', path: sums, bytes: statSync(sums).size, sha256: sha256(readFileSync(sums)) });
|
||||
// An empty Docker configuration proves the consumer can pull without publisher credentials.
|
||||
const publicConfig = join(temporary, 'public-docker'); mkdirSync(publicConfig);
|
||||
for (const [platform, images] of Object.entries(state.images)) {
|
||||
for (const reference of Object.values(images)) {
|
||||
console.log(`Verifying anonymous pull ${reference.split('@')[0]} (${platform})`);
|
||||
await run('docker', ['--config', publicConfig, 'pull', '--platform', platform, reference], { timeout: 20 * 60_000 });
|
||||
}
|
||||
console.log(`Smoke checking published images (${platform})`);
|
||||
await run('docker', ['run', '--rm', '--platform', platform, '--network', 'none', '--entrypoint', '/bin/sh', images.core, '-ec', 'test "$(pi --version)" = "$PI_VERSION"; tht --help >/dev/null; test -f /app/backend/dist/catalog/migrate.js; test -x /app/docker/workspace-maintenance-entrypoint.sh; test -f /app/docker/catalog-migrate.sh; test ! -e /run/secrets/thothii.secrets'], { timeout: 5 * 60_000 });
|
||||
await run('docker', ['run', '--rm', '--platform', platform, '--network', 'none', '--entrypoint', '/usr/local/bin/frontend-config-smoke', images.frontend], { timeout: 60_000 });
|
||||
}
|
||||
state.assets = assets; save();
|
||||
return assets;
|
||||
}
|
||||
const published = await publishVerifiedRelease({ hosting, prepare });
|
||||
state.releaseURL = published.html_url; state.complete = true; save();
|
||||
console.log(`Published and verified: ${published.html_url}`);
|
||||
return published;
|
||||
} finally {
|
||||
if (worktree && existsSync(worktree)) await run('git', ['worktree', 'remove', '--force', worktree]).catch(() => {});
|
||||
if (temporary && !existsSync(worktree ?? '')) rmSync(temporary, { recursive: true, force: true });
|
||||
unlock();
|
||||
}
|
||||
}
|
||||
if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) {
|
||||
try { await publishInstallation(optionsFromArgs(process.argv.slice(2))); }
|
||||
catch (error) { console.error(error instanceof SyntaxError ? 'Invalid release metadata; no secret values are printed.' : error.message); process.exitCode = 1; }
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { mkdirSync, readFileSync, writeFileSync, lstatSync } from "node:fs";
|
||||
import { dirname, join } from "node:path";
|
||||
import { parse, stringify } from "yaml";
|
||||
|
||||
export const serviceRoles = Object.freeze({ core: "core", frontend: "frontend", "catalog-db": "catalog", "catalog-migrate": "core", "workspace-maintenance": "core", qdrant: "qdrant", embedding: "embedding", "embedding-model-init": "embedding" });
|
||||
export const sha256 = (data) => createHash("sha256").update(data).digest("hex");
|
||||
|
||||
/** Only distribution assets enter the bundle: never a checkout, environment file or workspace. */
|
||||
export function prepareBundle({ source, destination, platform, version, revision, images }) {
|
||||
const compose = parse(readFileSync(join(source, "compose.yaml"), "utf8"));
|
||||
if (Object.keys(compose.services).sort().join() !== Object.keys(serviceRoles).sort().join()) throw new Error("Release service contract changed; review packaging before publishing.");
|
||||
for (const [name, role] of Object.entries(serviceRoles)) {
|
||||
if (!/^docker\.io\/[a-z0-9][a-z0-9._/-]*@sha256:[a-f0-9]{64}$/.test(images[role] ?? "")) throw new Error("Release requires immutable Docker Hub image references.");
|
||||
delete compose.services[name].build;
|
||||
delete compose.services[name].pull_policy;
|
||||
compose.services[name].image = images[role];
|
||||
compose.services[name].platform = platform;
|
||||
}
|
||||
const files = {};
|
||||
const write = (name, data) => {
|
||||
mkdirSync(dirname(join(destination, name)), { recursive: true });
|
||||
writeFileSync(join(destination, name), data);
|
||||
files[name] = sha256(data);
|
||||
};
|
||||
write("compose.yaml", stringify(compose));
|
||||
for (const name of ["deploy/compose.local.yaml", "deploy/compose.git-https.yaml", "deploy/compose.git-ssh.yaml", "docker/catalog-db-init.sql", "docker/embedding-model-init.sh"]) {
|
||||
const path = join(source, name);
|
||||
if (!lstatSync(path).isFile()) throw new Error("Release asset must be a regular tracked file.");
|
||||
write(name, readFileSync(path));
|
||||
}
|
||||
// Any future source bind mount must be deliberately added to the asset allowlist.
|
||||
for (const service of Object.values(compose.services)) {
|
||||
for (const volume of service.volumes ?? []) {
|
||||
const sourcePath = typeof volume === "string" ? volume.split(":")[0] : volume.type === "bind" ? volume.source : undefined;
|
||||
if (sourcePath?.startsWith(".") && !files[sourcePath.replace(/^\.\//, "")]) throw new Error("Source bind mount is missing from the release bundle.");
|
||||
}
|
||||
}
|
||||
const manifest = { schema_version: 1, version, revision, validator_protocol: 1,
|
||||
requirements: { cpus: 2, memory_bytes: 4 * 2 ** 30, disk_bytes: 10 * 2 ** 30 },
|
||||
components: ["pi", "catalog-migrations", "workspace-maintenance"],
|
||||
images: Object.fromEntries(Object.entries(images).map(([role, reference]) => [role, { [platform]: reference }])),
|
||||
files, compose: ["compose.yaml", "deploy/compose.local.yaml"] };
|
||||
writeFileSync(join(destination, "release-manifest.json"), JSON.stringify(manifest, null, 2) + "\n");
|
||||
writeFileSync(join(destination, "README.md"), `# ThothII ${version} — ${platform}\n\nSource / Sorgente: ${revision}\n\nThis prerelease provides images and document/preflight tools. Non-interactive execution and real-host acceptance are separate follow-up tickets; this is not a certified complete installation.\nQuesta prerelease fornisce immagini e strumenti di preparazione/preflight. Esecuzione non interattiva e collaudi reali sono incrementi successivi: non è ancora un'installazione completa certificata.\n\nUse bin/tht and its sibling bin/tht-workspace-documents together; no Node, Python, Bun or application checkout is required on the consumer host.\nWindows: use the Linux amd64 bundle inside Ubuntu WSL2, not a native Windows shell.\n\n1. bin/tht workspace prepare --directory NEW_WORKSPACE --id practice --name Practice\n2. bin/tht workspace validate --directory WORKSPACE\n3. bin/tht installation prepare --directory NEW_PRIVATE_INSTALLATION\n4. bin/tht installation preflight --directory INSTALLATION --release ABSOLUTE_RELEASE_DIR/release-manifest.json\n5. Complete the commented documents, generate technical credentials explicitly, then run installation validate and installation plan with --installation ABSOLUTE_INSTALLATION_FILE.\n\nImages are pinned by digest; runtime credentials and user workspaces are never bundled.\nConsult the accompanying IT/EN guides for prepared documents and mandatory runtime checks.\n`);
|
||||
for (const [name, target] of [["standalone-manual-it.md", "GUIDE-IT.md"], ["standalone-manual-en.md", "GUIDE-EN.md"], ["installation-preflight.md", "PREFLIGHT.md"]]) writeFileSync(join(destination, target), readFileSync(join(source, "docs/install", name)));
|
||||
return manifest;
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtempSync, readFileSync, rmSync, existsSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { resolve, join } from "node:path";
|
||||
import { parse } from "yaml";
|
||||
import { prepareBundle } from "./release-bundle.mjs";
|
||||
|
||||
test("consumer bundle pins every service and carries all source bind resources", () => {
|
||||
const output = mkdtempSync(join(tmpdir(), "thoth-release-test-"));
|
||||
const images = Object.fromEntries(["core", "frontend", "catalog", "qdrant", "embedding"].map((role) => [role, `docker.io/tylconsulting/${role}@sha256:${"a".repeat(64)}`]));
|
||||
try {
|
||||
prepareBundle({ source: resolve(import.meta.dirname, "../.."), destination: output, platform: "linux/amd64", version: "0.1.0-install-preview.1", revision: "b".repeat(40), images });
|
||||
const manifest = JSON.parse(readFileSync(join(output, "release-manifest.json")));
|
||||
const compose = parse(readFileSync(join(output, "compose.yaml"), "utf8"));
|
||||
assert.equal(compose.services.core.image, images.core);
|
||||
assert.equal(compose.services["catalog-migrate"].image, images.core);
|
||||
assert.equal(compose.services["workspace-maintenance"].image, images.core);
|
||||
assert.equal(compose.services["embedding-model-init"].image, images.embedding);
|
||||
for (const service of Object.values(compose.services)) {
|
||||
assert.equal(service.build, undefined);
|
||||
assert.equal(service.pull_policy, undefined);
|
||||
assert.equal(service.platform, "linux/amd64");
|
||||
for (const volume of service.volumes ?? []) {
|
||||
if (typeof volume === "string" && volume.startsWith("./")) assert.ok(existsSync(join(output, volume.split(":")[0])));
|
||||
}
|
||||
}
|
||||
assert.equal(manifest.validator_protocol, 1);
|
||||
assert.deepEqual(manifest.compose, ["compose.yaml", "deploy/compose.local.yaml"]);
|
||||
assert.ok(manifest.files["docker/catalog-db-init.sql"]);
|
||||
assert.ok(manifest.files["docker/embedding-model-init.sh"]);
|
||||
assert.ok(!existsSync(join(output, "backend")));
|
||||
assert.ok(!existsSync(join(output, "harness")));
|
||||
} finally { rmSync(output, { recursive: true, force: true }); }
|
||||
});
|
||||
@@ -0,0 +1,69 @@
|
||||
import { readFileSync } from "node:fs";
|
||||
import { boundedFetch, readLimited } from "./release-registry.mjs";
|
||||
import { sha256 } from "./release-bundle.mjs";
|
||||
|
||||
export async function giteaHosting({ run, repository, identity, body }) {
|
||||
const remote = new URL(repository);
|
||||
const credential = await run("git", ["credential", "fill"], { input: `protocol=https\nhost=${remote.host}\n\n`, quiet: true, env: { ...process.env, GIT_TERMINAL_PROMPT: "0" } });
|
||||
const fields = Object.fromEntries(credential.trim().split("\n").map((line) => { const at = line.indexOf("="); return [line.slice(0, at), line.slice(at + 1)]; }));
|
||||
if (!fields.username || !fields.password) throw new Error("Gitea publishing credentials are unavailable in the Git credential store.");
|
||||
const authorization = `Basic ${Buffer.from(`${fields.username}:${fields.password}`).toString("base64")}`;
|
||||
const api = `${remote.origin}/api/v1/repos${remote.pathname.replace(/\.git$/, "")}`;
|
||||
const marker = `<!-- thothii-release:${JSON.stringify(identity)} -->`;
|
||||
const tag = `installation-v${identity.version}`;
|
||||
async function request(path, options = {}, allowMissing = false) {
|
||||
const response = await boundedFetch(api + path, { ...options, headers: { Authorization: authorization, ...options.headers } }, 120_000);
|
||||
if (allowMissing && response.status === 404) return null;
|
||||
if (!response.ok) throw new Error(`Gitea release operation failed (HTTP ${response.status}).`);
|
||||
const bytes = await readLimited(response, 4 * 2 ** 20);
|
||||
try { return JSON.parse(bytes.toString()); } catch { throw new Error("Gitea returned invalid release metadata."); }
|
||||
}
|
||||
const json = (method, value) => ({ method, headers: { "Content-Type": "application/json" }, body: JSON.stringify(value) });
|
||||
async function verifyTag(required = false) {
|
||||
const existing = await request(`/tags/${encodeURIComponent(tag)}`, {}, true);
|
||||
if ((!existing && required) || (existing && existing.commit?.sha !== identity.revision)) throw new Error("Release Git tag does not match the requested source revision.");
|
||||
}
|
||||
async function assets(release) { return request(`/releases/${release.id}/assets`); }
|
||||
async function verifyAsset(asset, expected, publicRead) {
|
||||
const url = new URL(asset.browser_download_url);
|
||||
if (url.origin !== remote.origin) throw new Error("Unexpected release asset origin.");
|
||||
const response = await boundedFetch(url, { headers: publicRead ? {} : { Authorization: authorization } }, 120_000);
|
||||
if (!response.ok || sha256(await readLimited(response, expected.bytes + 1)) !== expected.sha256) throw new Error("Published release asset does not match its verified checksum.");
|
||||
}
|
||||
return {
|
||||
async open() {
|
||||
const info = await request("");
|
||||
if (info.private || !info.permissions?.push) throw new Error("Release hosting must be a public Gitea repository with publication rights.");
|
||||
await verifyTag();
|
||||
const existing = await request(`/releases/tags/${encodeURIComponent(tag)}`, {}, true);
|
||||
if (existing) {
|
||||
if (!existing.body?.includes(marker)) throw new Error("Release version already belongs to another source or publication identity; it will not be overwritten.");
|
||||
return existing;
|
||||
}
|
||||
return request("/releases", json("POST", { tag_name: tag, target_commitish: identity.revision, name: `ThothII ${identity.version}`, body: `${body}\n\n${marker}`, draft: true, prerelease: true }));
|
||||
},
|
||||
async upload(release, asset) {
|
||||
const matching = (await assets(release)).filter((item) => item.name === asset.name);
|
||||
if (matching.length > 1) throw new Error("Ambiguous release assets; no published files were replaced.");
|
||||
if (matching.length === 1) { await verifyAsset(matching[0], asset, false); return; }
|
||||
const data = readFileSync(asset.path);
|
||||
if (sha256(data) !== asset.sha256) throw new Error("Local release asset changed before upload.");
|
||||
const form = new FormData(); form.append("attachment", new Blob([data]), asset.name);
|
||||
await request(`/releases/${release.id}/assets?name=${encodeURIComponent(asset.name)}`, { method: "POST", body: form });
|
||||
},
|
||||
async verify(release, expected, publicRead) {
|
||||
await verifyTag(publicRead);
|
||||
const uploaded = await assets(release);
|
||||
if (uploaded.length !== expected.length) throw new Error("Release asset set is incomplete or contains unexpected files.");
|
||||
for (const asset of expected) {
|
||||
const matching = uploaded.filter((item) => item.name === asset.name);
|
||||
if (matching.length !== 1) throw new Error("Release asset missing or ambiguous.");
|
||||
await verifyAsset(matching[0], asset, publicRead);
|
||||
}
|
||||
},
|
||||
async publish(release) {
|
||||
await verifyTag();
|
||||
return request(`/releases/${release.id}`, json("PATCH", { draft: false }));
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
import { test } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { giteaHosting } from './release-hosting.mjs';
|
||||
|
||||
test('release hosting refuses a pre-existing Git tag on another commit', async () => {
|
||||
const fetch = globalThis.fetch;
|
||||
const revision = 'a'.repeat(40);
|
||||
let writes = 0;
|
||||
globalThis.fetch = async (url, options) => {
|
||||
if (options.method) { writes++; return Response.json({ id: 1, draft: true }); }
|
||||
if (String(url).includes('/releases/tags/')) return new Response('', { status: 404 });
|
||||
if (String(url).includes('/tags/installation-v')) return Response.json({ commit: { sha: 'b'.repeat(40) } });
|
||||
return Response.json({ private: false, permissions: { push: true } });
|
||||
};
|
||||
try {
|
||||
const hosting = await giteaHosting({ run: async () => 'username=test\npassword=test\n', repository: 'https://example.test/owner/repo', identity: { revision, version: '1.0.0' }, body: '' });
|
||||
await assert.rejects(hosting.open(), /tag.*revision/i);
|
||||
assert.equal(writes, 0);
|
||||
} finally { globalThis.fetch = fetch; }
|
||||
});
|
||||
|
||||
test('matching Git tag is accepted and verified again before publishing', async () => {
|
||||
const fetch = globalThis.fetch;
|
||||
const identity = { revision: 'a'.repeat(40), version: '1.0.0' };
|
||||
let sha = identity.revision;
|
||||
let writes = 0;
|
||||
globalThis.fetch = async (url, options) => {
|
||||
if (options.method) { writes++; return Response.json({ id: 1, draft: false }); }
|
||||
if (String(url).includes('/releases/tags/')) return Response.json({ id: 1, draft: true, body: `<!-- thothii-release:${JSON.stringify(identity)} -->` });
|
||||
if (String(url).includes('/tags/')) return Response.json({ commit: { sha } });
|
||||
if (String(url).endsWith('/assets')) return Response.json([]);
|
||||
return Response.json({ private: false, permissions: { push: true } });
|
||||
};
|
||||
try {
|
||||
const hosting = await giteaHosting({ run: async () => 'username=test\npassword=test\n', repository: 'https://example.test/owner/repo', identity, body: '' });
|
||||
const release = await hosting.open();
|
||||
await hosting.verify(release, [], false);
|
||||
sha = 'b'.repeat(40);
|
||||
await assert.rejects(hosting.verify(release, [], false), /tag.*revision/i);
|
||||
await assert.rejects(hosting.publish(release), /tag.*revision/i);
|
||||
assert.equal(writes, 0);
|
||||
} finally { globalThis.fetch = fetch; }
|
||||
});
|
||||
@@ -0,0 +1,18 @@
|
||||
import { test } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { mkdtempSync, rmSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { commandRunner } from './publish-installation.mjs';
|
||||
|
||||
test('publisher timeout terminates descendants that retain output pipes', async () => {
|
||||
const logs = mkdtempSync(join(tmpdir(), 'release-process-test-'));
|
||||
try {
|
||||
const started = Date.now();
|
||||
await assert.rejects(commandRunner(logs)(process.execPath, ['-e', `
|
||||
require('child_process').spawn(process.execPath, ['-e', 'setTimeout(() => {}, 2500)'], {stdio: 'inherit'});
|
||||
setTimeout(() => {}, 2500);
|
||||
`], { timeout: 250, quiet: true }));
|
||||
assert.ok(Date.now() - started < 1800, 'timeout must not wait for the descendant to exit naturally');
|
||||
} finally { rmSync(logs, { recursive: true, force: true }); }
|
||||
});
|
||||
@@ -0,0 +1,14 @@
|
||||
/** Drafts are the publication boundary. A partial build/upload is never a consumer release. */
|
||||
export async function publishVerifiedRelease({ hosting, prepare }) {
|
||||
const release = await hosting.open();
|
||||
const assets = await prepare();
|
||||
if (!release.draft) {
|
||||
await hosting.verify(release, assets, true);
|
||||
return release;
|
||||
}
|
||||
for (const asset of assets) await hosting.upload(release, asset);
|
||||
await hosting.verify(release, assets, false);
|
||||
const published = await hosting.publish(release);
|
||||
await hosting.verify(published, assets, true);
|
||||
return published;
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { publishVerifiedRelease } from "./release-publication.mjs";
|
||||
|
||||
test("an interrupted preparation stays draft and retry publishes only after every artifact verifies", async () => {
|
||||
const events = [];
|
||||
let failing = true;
|
||||
const hosting = {
|
||||
open: async () => ({ id: 7, draft: true }),
|
||||
upload: async (_draft, asset) => events.push(`upload:${asset.name}`),
|
||||
verify: async (_draft, assets, publicRead) => events.push(`verify:${publicRead}:${assets.length}`),
|
||||
publish: async () => { events.push("publish"); return { html_url: "https://example.test/release" }; },
|
||||
};
|
||||
const prepare = async () => {
|
||||
if (failing) throw new Error("frontend build unavailable");
|
||||
return [{ name: "bundle.tar.gz" }, { name: "SHA256SUMS" }];
|
||||
};
|
||||
await assert.rejects(publishVerifiedRelease({ hosting, prepare }), /frontend/);
|
||||
assert.deepEqual(events, []);
|
||||
failing = false;
|
||||
await publishVerifiedRelease({ hosting, prepare });
|
||||
assert.deepEqual(events, ["upload:bundle.tar.gz", "upload:SHA256SUMS", "verify:false:2", "publish", "verify:true:2"]);
|
||||
});
|
||||
|
||||
test("a published version is verified without replacing any asset", async () => {
|
||||
const events = [];
|
||||
await publishVerifiedRelease({ prepare: async () => [{ name: "bundle.tar.gz" }], hosting: {
|
||||
open: async () => ({ id: 7, draft: false }),
|
||||
upload: async () => { throw new Error("overwrote a published version"); },
|
||||
publish: async () => { throw new Error("republished a version"); },
|
||||
verify: async (_release, _assets, publicRead) => events.push(publicRead),
|
||||
} });
|
||||
assert.deepEqual(events, [true]);
|
||||
});
|
||||
@@ -0,0 +1,86 @@
|
||||
import { readFileSync } from "node:fs";
|
||||
import { homedir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { sha256 } from "./release-bundle.mjs";
|
||||
|
||||
const accept = "application/vnd.oci.image.index.v1+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.docker.distribution.manifest.v2+json";
|
||||
export async function boundedFetch(url, options = {}, timeout = 30_000) {
|
||||
try { return await fetch(url, { ...options, redirect: options.redirect ?? "error", signal: AbortSignal.timeout(timeout) }); }
|
||||
catch { throw new Error("Release network request failed; retry with the same output directory."); }
|
||||
}
|
||||
export async function readLimited(response, maximum) {
|
||||
const chunks = []; let size = 0;
|
||||
for await (const chunk of response.body) { size += chunk.length; if (size > maximum) throw new Error("Release response exceeded its bound."); chunks.push(chunk); }
|
||||
return Buffer.concat(chunks);
|
||||
}
|
||||
async function jsonResponse(response, description) {
|
||||
if (!response.ok) throw new Error(`${description} refused (HTTP ${response.status}).`);
|
||||
const bytes = await readLimited(response, 4 * 2 ** 20);
|
||||
try { return { bytes, value: JSON.parse(bytes.toString()) }; }
|
||||
catch { throw new Error("Release service returned invalid metadata."); }
|
||||
}
|
||||
export async function dockerCredentials(run) {
|
||||
const config = JSON.parse(readFileSync(join(process.env.DOCKER_CONFIG || join(homedir(), ".docker"), "config.json"), "utf8"));
|
||||
const server = "https://index.docker.io/v1/";
|
||||
const helper = config.credHelpers?.[server] || config.credsStore;
|
||||
if (!helper || !/^[A-Za-z0-9._-]+$/.test(helper)) throw new Error("Use docker login with an OS credential store before publishing.");
|
||||
const auth = JSON.parse(await run(`docker-credential-${helper}`, ["get"], { input: server + "\n", quiet: true }));
|
||||
if (!auth.Username || !auth.Secret) throw new Error("Docker Hub login is unavailable.");
|
||||
return auth;
|
||||
}
|
||||
export function dockerHub(auth) {
|
||||
async function token(repository, anonymous) {
|
||||
const url = new URL("https://auth.docker.io/token");
|
||||
url.searchParams.set("service", "registry.docker.io");
|
||||
url.searchParams.set("scope", `repository:${repository}:pull`);
|
||||
const response = await boundedFetch(url, { headers: anonymous ? {} : { Authorization: `Basic ${Buffer.from(`${auth.Username}:${auth.Secret}`).toString("base64")}` } });
|
||||
return (await jsonResponse(response, "Registry authentication")).value.token;
|
||||
}
|
||||
async function registryJSON(repository, route, anonymous, allowMissing = false) {
|
||||
const bearer = await token(repository, anonymous);
|
||||
let response = await boundedFetch(`https://registry-1.docker.io/v2/${repository}/${route}`, { redirect: "manual", headers: { Accept: accept, Authorization: `Bearer ${bearer}` } });
|
||||
if ([302, 307].includes(response.status) && route.startsWith("blobs/")) {
|
||||
const target = new URL(response.headers.get("location"));
|
||||
if (target.protocol !== "https:" || target.username || target.password) throw new Error("Invalid registry blob redirect.");
|
||||
// Signed blob URLs are fetched without forwarding registry credentials.
|
||||
response = await boundedFetch(target);
|
||||
}
|
||||
if (allowMissing && response.status === 404) return null;
|
||||
const { bytes, value } = await jsonResponse(response, "Registry read");
|
||||
const digest = `sha256:${sha256(bytes)}`;
|
||||
const advertised = response.headers.get("docker-content-digest");
|
||||
if (advertised && advertised !== digest) throw new Error("Registry content digest mismatch.");
|
||||
return { value, digest };
|
||||
}
|
||||
return {
|
||||
async ensurePublic(namespace, name) {
|
||||
const response = await boundedFetch("https://hub.docker.com/v2/auth/token", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ identifier: auth.Username, secret: auth.Secret }) });
|
||||
const bearer = (await jsonResponse(response, "Docker Hub authentication")).value.access_token;
|
||||
const headers = { Authorization: `Bearer ${bearer}`, "Content-Type": "application/json" };
|
||||
const path = `https://hub.docker.com/v2/namespaces/${namespace}/repositories`;
|
||||
let existing = await boundedFetch(`${path}/${name}`, { headers });
|
||||
if (existing.status === 404) {
|
||||
existing = await boundedFetch(path, { method: "POST", headers, body: JSON.stringify({ namespace, name, registry: "docker.io", is_private: false, description: `ThothII ${name.endsWith("core") ? "core with embedded Pi" : "standalone frontend"}` }) });
|
||||
}
|
||||
const data = (await jsonResponse(existing, "Public repository preparation")).value;
|
||||
if (data.is_private !== false) throw new Error("Selected Docker Hub repository is private; make this release repository public before retrying.");
|
||||
},
|
||||
async inspect(repository, reference, platform, { anonymous = false, allowMissing = false } = {}) {
|
||||
let image = await registryJSON(repository, `manifests/${reference}`, anonymous, allowMissing);
|
||||
if (!image) return null;
|
||||
if (reference.startsWith("sha256:") && image.digest !== reference) throw new Error("Requested image digest does not match registry content.");
|
||||
if (image.value.manifests) {
|
||||
const match = image.value.manifests.find((entry) => `${entry.platform?.os}/${entry.platform?.architecture}` === platform);
|
||||
if (!match || !/^sha256:[a-f0-9]{64}$/.test(match.digest)) throw new Error("Image does not contain the requested platform.");
|
||||
image = await registryJSON(repository, `manifests/${match.digest}`, anonymous);
|
||||
if (image.digest !== match.digest) throw new Error("Image index digest mismatch.");
|
||||
}
|
||||
if (reference.startsWith("sha256:") && !image.value.config) throw new Error("Image metadata is incomplete.");
|
||||
const configDigest = image.value.config?.digest;
|
||||
if (!/^sha256:[a-f0-9]{64}$/.test(configDigest ?? "")) throw new Error("Image configuration digest is invalid.");
|
||||
const config = await registryJSON(repository, `blobs/${configDigest}`, anonymous);
|
||||
if (config.digest !== configDigest || `${config.value.os}/${config.value.architecture}` !== platform) throw new Error("Image configuration or platform mismatch.");
|
||||
return { reference: `docker.io/${repository}@${image.digest}`, labels: config.value.config?.Labels ?? {} };
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { dockerHub } from "./release-registry.mjs";
|
||||
import { sha256 } from "./release-bundle.mjs";
|
||||
|
||||
test("registry verifies pinned config/platform and does not forward auth to blob storage", async () => {
|
||||
const original = globalThis.fetch;
|
||||
const config = JSON.stringify({ os: "linux", architecture: "amd64", config: { Labels: { "org.opencontainers.image.revision": "b".repeat(40) } } });
|
||||
const configDigest = "sha256:" + sha256(config);
|
||||
const manifest = JSON.stringify({ schemaVersion: 2, config: { digest: configDigest } });
|
||||
const imageDigest = "sha256:" + sha256(manifest);
|
||||
const auth = [];
|
||||
globalThis.fetch = async (target, options) => {
|
||||
const url = new URL(target);
|
||||
if (url.hostname === "auth.docker.io") return new Response(JSON.stringify({ token: "registry-token" }));
|
||||
if (url.hostname === "blob.example.test") { auth.push(options.headers?.Authorization); return new Response(config); }
|
||||
if (url.pathname.includes("/blobs/")) return new Response(null, { status: 307, headers: { location: "https://blob.example.test/config" } });
|
||||
return new Response(manifest, { headers: { "docker-content-digest": imageDigest } });
|
||||
};
|
||||
try {
|
||||
const registry = dockerHub({ Username: "publisher", Secret: "PRIVATE_TOKEN" });
|
||||
const image = await registry.inspect("example/core", imageDigest, "linux/amd64", { anonymous: true });
|
||||
assert.equal(image.reference, `docker.io/example/core@${imageDigest}`);
|
||||
assert.deepEqual(auth, [undefined]);
|
||||
await assert.rejects(registry.inspect("example/core", imageDigest, "linux/arm64"), /platform mismatch/);
|
||||
await assert.rejects(registry.inspect("example/core", "sha256:" + "0".repeat(64), "linux/amd64"), /Requested image digest/);
|
||||
} finally { globalThis.fetch = original; }
|
||||
});
|
||||
Reference in New Issue
Block a user