mptyl/ThothII
1
0
Fork 0
Code Issues 25 Pull Requests Actions Packages Projects Releases 1 Wiki Activity

fix(ci): project private application secrets

Browse Source
This commit is contained in:
marcopan
2026-08-25 18:47:49 +02:00
parent fd878b8c3e
commit 73b784a176
4 changed files with 84 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/scripts/verify-workspace-descriptor-files.mjs
+1 -1
View File
@@ -40,7 +40,7 @@ const reviewedExpandableBlocks = new Map([
]],
["scripts/unified-deployment-smoke.sh", [
{ sha256: "ca0c17d9ff8dc0fbe018fc1c5510eb33bc667a936fbe44a9be2d311390576825", rationale: "Generates reviewed Task 13 runtime configuration." },
{ sha256: "e457c2d0620fd2db22332285748fc2e0738de998860fa478ea1ff5b70a891f3a", rationale: "Generates the reviewed local Task 13 Compose override." },
{ sha256: "cf62a7adcbb7b4e2323e7f2baee58f72dad7c19678f416555067eafefed65144", rationale: "Generates the reviewed local Task 13 Compose override." },
{ sha256: "c556f7d910d0788e219b042957e6b307cb9925b43920c680535d0d3a6dcbdb25", rationale: "Generates the reviewed local Task 13 installation descriptor." },
{ sha256: "c0078c68bd42a8668fbcb849888531e5e56109579df1ff96140a9e91b1efea56", rationale: "Generates the reviewed server Task 13 Compose override." },
{ sha256: "b34a2b4ffaa72e01efb64a7a28b13513527538d837b2f35b6ca5fb3dbdb2d5dc", rationale: "Generates the reviewed server Task 13 installation descriptor." },
scripts/task13-runtime-fixture-check.ts
+25 -12
View File
@@ -7,9 +7,12 @@ import { renderRuntimeConfig } from "../backend/src/workspaces/runtime-renderer.
const requireFromBackend = createRequire(new URL("../backend/package.json", import.meta.url));
const { parse } = requireFromBackend("yaml") as { parse: (value: string) => any };
const [renderedPath, workspacePath, profile] = process.argv.slice(2);
if (!renderedPath || !workspacePath || (profile !== "local" && profile !== "server")) {
throw new Error("usage: task13-runtime-fixture-check RENDERED_JSON WORKSPACE_YAML local|server");
const [renderedPath, workspacePath, profile, bundleSource, runtimePasswordSourceInput] = process.argv.slice(2);
if (!renderedPath || !workspacePath || (profile !== "local" && profile !== "server")
|| !bundleSource || !runtimePasswordSourceInput) {
throw new Error(
"usage: task13-runtime-fixture-check RENDERED_JSON WORKSPACE_YAML local|server BUNDLE PASSWORD",
);
}
const config = JSON.parse(readFileSync(renderedPath, "utf8"));
@@ -105,27 +108,37 @@ if (workspace.semantic_index?.embedding?.dimensions !== 1024) {
throw new Error("fixture workspace embedding dimension changed");
}
const bundle = config.secrets?.thothii_secrets;
const bundleSource = bundle?.file;
if (typeof bundleSource !== "string" || !statSync(bundleSource).isFile()) {
if (!statSync(bundleSource).isFile()) {
throw new Error("fixture secret bundle source is not a regular file");
}
accessSync(bundleSource, constants.R_OK);
const coreBundle = (core.secrets || []).filter(
(secret: any) => secret.source === "thothii_secrets" && secret.target === "thothii.secrets",
);
if (coreBundle.length !== 1) throw new Error("core lacks exactly one runtime secret bundle mount");
if (profile === "local") {
if (coreBundle.length !== 0) throw new Error("local core retained the host-owned secret bundle mount");
} else if (coreBundle.length !== 1) {
throw new Error("server core lacks exactly one runtime secret bundle mount");
}
if ((frontend.secrets || []).length !== 0) throw new Error("frontend received a runtime secret");
const mounts = core.volumes || [];
const runtimePasswordMounts = mounts.filter(
(mount: any) => mount.target === "/run/secrets/task13-runtime-password",
);
if (runtimePasswordMounts.length !== 1 || runtimePasswordMounts[0].type !== "bind"
if (profile === "local") {
const projected = mounts.filter((mount: any) => mount.target === "/run/secrets");
if (runtimePasswordMounts.length !== 0 || projected.length !== 1
|| projected[0].type !== "volume" || !projected[0].read_only
|| (projected[0].source !== "application-secrets"
&& !projected[0].source.endsWith("_application-secrets"))) {
throw new Error("local secrets are not isolated in the Compose-owned projection volume");
}
} else if (runtimePasswordMounts.length !== 1 || runtimePasswordMounts[0].type !== "bind"
|| !runtimePasswordMounts[0].read_only || !statSync(runtimePasswordMounts[0].source).isFile()) {
throw new Error("runtime fixture lacks one readable, read-only password-file bind");
throw new Error("server runtime fixture lacks one readable, read-only password-file bind");
}
accessSync(runtimePasswordMounts[0].source, constants.R_OK);
accessSync(runtimePasswordSourceInput, constants.R_OK);
const piTargets = [
"/home/thoth/.pi/agent/auth.json",
"/home/thoth/.pi/agent/models.json",
@@ -168,7 +181,7 @@ for (const [target, localVolume] of [
}
const resolverEnvironment = { ...core.environment };
const runtimePasswordSource = realpathSync(runtimePasswordMounts[0].source);
const runtimePasswordSource = realpathSync(runtimePasswordSourceInput);
resolverEnvironment.THT_WS_TASK13_SMOKE_DWH_PASSWORD_FILE = runtimePasswordSource;
const bindings = resolveRuntimeBindings(workspace, resolverEnvironment, [dirname(runtimePasswordSource)]);
for (const [role, binding] of Object.entries(bindings)) {
@@ -200,7 +213,7 @@ if (runtime.resources?.embeddings?.base_url !== "http://embedding:11434"
throw new Error("workspace resolver produced the wrong embedding runtime");
}
const secret = readFileSync(bundleSource, "utf8").trim();
const runtimePassword = readFileSync(runtimePasswordMounts[0].source, "utf8");
const runtimePassword = readFileSync(runtimePasswordSourceInput, "utf8");
if (JSON.stringify(config).includes(secret)) throw new Error("fixture render leaked application bundle content");
if (JSON.stringify(runtime).includes(secret)) throw new Error("runtime render leaked application bundle content");
if (JSON.stringify(config).includes(runtimePassword)) throw new Error("fixture render leaked runtime password content");
scripts/test-task13-runtime-fixtures.sh
+11 -4
View File
@@ -83,7 +83,7 @@ checker=(node --import "$tsx_loader" "$root/scripts/task13-runtime-fixture-check
echo "backend dependencies are required for the Task 13 runtime fixture contract" >&2
exit 2
}
"${checker[@]}" "$rendered" "$workspace" "$profile"
"${checker[@]}" "$rendered" "$workspace" "$profile" "$TASK13_SECRETS" "$TASK13_SESSION_RUNTIME_PASSWORD"
for mutation in \
wrong-service \
@@ -93,9 +93,9 @@ for mutation in \
wrong-embedding-service \
external-semantic-urls; do
mutated="$fixture/$mutation.json"
node - "$rendered" "$mutated" "$mutation" <<'NODE'
node - "$rendered" "$mutated" "$mutation" "$profile" <<'NODE'
const fs = require("fs");
const [source, destination, mutation] = process.argv.slice(2);
const [source, destination, mutation, profile] = process.argv.slice(2);
const config = JSON.parse(fs.readFileSync(source, "utf8"));
if (mutation === "wrong-service") {
const name = "THT_WS_TASK13_SMOKE_DWH_HOST";
@@ -105,7 +105,12 @@ if (mutation === "wrong-service") {
} else if (mutation === "wrong-value") {
config.services.core.environment.THT_WS_TASK13_SMOKE_DWH_HOST = "wrong.task13.invalid";
} else if (mutation === "wrong-secret-mount") {
config.secrets.thothii_secrets.file = source + ".missing";
if (profile === "local") {
const mount = config.services.core.volumes.find((item) => item.target === "/run/secrets");
mount.source = "wrong-application-secrets";
} else {
config.services.core.secrets[0].target = "wrong.secrets";
}
} else if (mutation === "wrong-qdrant-service") {
config.services.core.environment.THT_INTERNAL_QDRANT_URL = "http://vector:6333";
} else if (mutation === "wrong-embedding-service") {
@@ -117,6 +122,7 @@ if (mutation === "wrong-service") {
fs.writeFileSync(destination, JSON.stringify(config));
NODE
if "${checker[@]}" "$mutated" "$workspace" "$profile" \
"$TASK13_SECRETS" "$TASK13_SESSION_RUNTIME_PASSWORD" \
>"$fixture/$mutation.out" 2>"$fixture/$mutation.err"; then
echo "runtime fixture checker accepted mutation: $mutation" >&2
exit 1
@@ -141,6 +147,7 @@ if (mutation === "collection-reuse") {
fs.writeFileSync(destination, yaml.stringify(workspace));
NODE
if "${checker[@]}" "$rendered" "$mutated" "$profile" \
"$TASK13_SECRETS" "$TASK13_SESSION_RUNTIME_PASSWORD" \
>"$fixture/$mutation.out" 2>"$fixture/$mutation.err"; then
echo "runtime fixture checker accepted workspace mutation: $mutation" >&2
exit 1
scripts/unified-deployment-smoke.sh
+47 -1
View File
@@ -501,8 +501,9 @@ services:
- sessions:/data/sessions
- auth-runtime:/run/thothii-auth:ro
- auth-state:/data/auth
- $TASK13_SESSION_RUNTIME_PASSWORD:/run/secrets/task13-runtime-password:ro
- application-secrets:/run/secrets:ro
- registry-remote:/fixtures/remote.git:ro
secrets: !reset []
frontend:
image: $TASK13_FRONTEND_IMAGE
build:
@@ -547,6 +548,9 @@ volumes:
auth-runtime:
labels:
io.thothii.task13.run: "$TASK13_RUN_ID"
application-secrets:
labels:
io.thothii.task13.run: "$TASK13_RUN_ID"
registry-remote:
labels:
io.thothii.task13.run: "$TASK13_RUN_ID"
@@ -978,6 +982,34 @@ task13_prepare_local_pi_runtime() {
'
}
task13_prepare_local_application_secrets() {
local owner_label
owner_label="$(docker volume inspect --format '{{ index .Labels "io.thothii.task13.run" }}' \
"$TASK13_APPLICATION_SECRETS_VOLUME")"
[[ "$owner_label" == "$TASK13_RUN_ID" ]] \
|| task13_fail "application-secret runtime volume lacks the Task 13 run label"
task13_run_logged "project local application secrets for the core runtime" docker run --rm \
--name "$TASK13_APPLICATION_SECRETS_PROJECTION_CONTAINER" \
--label "io.thothii.task13.run=$TASK13_RUN_ID" \
--user 0:0 \
--entrypoint sh \
--volume "$TASK13_SECRETS:/source/thothii.secrets:ro" \
--volume "$TASK13_SESSION_RUNTIME_PASSWORD:/source/task13-runtime-password:ro" \
--volume "$TASK13_APPLICATION_SECRETS_VOLUME:/target" \
"$TASK13_CORE_IMAGE" -ceu '
test -f /source/thothii.secrets && test ! -L /source/thothii.secrets
test -f /source/task13-runtime-password && test ! -L /source/task13-runtime-password
test -z "$(find /target -mindepth 1 -maxdepth 1 -print -quit)"
cp /source/thothii.secrets /source/task13-runtime-password /target/
chown 10001:10001 /target /target/thothii.secrets /target/task13-runtime-password
chmod 0700 /target
chmod 0600 /target/thothii.secrets /target/task13-runtime-password
test "$(stat -c "%u:%g:%a" /target)" = 10001:10001:700
test "$(stat -c "%u:%g:%a" /target/thothii.secrets)" = 10001:10001:600
test "$(stat -c "%u:%g:%a" /target/task13-runtime-password)" = 10001:10001:600
'
}
task13_prepare_registry_remote() {
local owner_label
owner_label="$(docker volume inspect --format '{{ index .Labels "io.thothii.task13.run" }}' \
@@ -1008,6 +1040,7 @@ task13_start_stack() {
task13_compose_logged "create local core authentication runtime" create core
task13_prepare_local_auth_runtime
task13_prepare_local_pi_runtime
task13_prepare_local_application_secrets
task13_prepare_registry_remote
task13_compose_start_logged "start local Compose distribution" up --detach --wait --wait-timeout 120
TASK13_NETWORK="$(docker network ls \
@@ -2420,6 +2453,7 @@ task13_self_test_source_contract() {
local root host_network push_command registry_function workflow uses_count pinned_uses_count
local auth_runtime_mount auth_root_mount auth_projection auth_runtime_owner
local pi_auth_bind pi_projection registry_runtime_mount registry_root_mount registry_projection
local application_secret_bind application_secret_mount application_secret_projection
root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)"
workflow="$root/.github/workflows/deployment.yml"
host_network='--network'' host'
@@ -2434,6 +2468,9 @@ task13_self_test_source_contract() {
registry_runtime_mount='registry-remote:/fixtures/remote.git:''ro'
registry_root_mount='$TASK13_''REMOTE:/fixtures/remote.git:ro'
registry_projection='task13_prepare_registry_''remote'
application_secret_bind='$TASK13_''SECRETS:/run/secrets/thothii.secrets:ro'
application_secret_mount='application-secrets:/run/''secrets:ro'
application_secret_projection='task13_prepare_local_application_''secrets'
if rg -n 'docker[[:space:]]+(system[[:space:]]+)?prune' \
"$root/scripts/unified-deployment-smoke.sh" \
"$root/scripts/tht-update-smoke.sh" \
@@ -2467,6 +2504,13 @@ task13_self_test_source_contract() {
[[ "$(grep -Ec "^${registry_projection}\\(\\)|^[[:space:]]+${registry_projection}$" \
"$root/scripts/unified-deployment-smoke.sh")" -ge 3 ]] \
|| task13_fail "the Git fixture projection must cover bootstrap and subsequent pushes"
! grep -Fq -- "$application_secret_bind" "$root/scripts/unified-deployment-smoke.sh" \
|| task13_fail "the local smoke must not bind the host-owned application bundle into the core"
grep -Fq -- "$application_secret_mount" "$root/scripts/unified-deployment-smoke.sh" \
|| task13_fail "the local smoke must mount Compose-owned application secrets"
[[ "$(grep -Ec "^${application_secret_projection}\\(\\)|^[[:space:]]+${application_secret_projection}$" \
"$root/scripts/unified-deployment-smoke.sh")" -eq 2 ]] \
|| task13_fail "the local application-secret projection must be defined and invoked once"
grep -Eq '^TASK13_BAD_CANDIDATE_IMAGE="[^"[:space:]]+@sha256:[0-9a-f]{64}"$' \
"$root/scripts/unified-deployment-smoke.sh" \
|| task13_fail "the bad rollback candidate must be an immutable digest reference"
@@ -2622,6 +2666,8 @@ task13_initialize() {
TASK13_AUTH_PROJECTION_CONTAINER="$TASK13_PROJECT-auth-projection"
TASK13_PI_RUNTIME_VOLUME="${TASK13_PROJECT}_pi-state"
TASK13_PI_PROJECTION_CONTAINER="$TASK13_PROJECT-pi-projection"
TASK13_APPLICATION_SECRETS_VOLUME="${TASK13_PROJECT}_application-secrets"
TASK13_APPLICATION_SECRETS_PROJECTION_CONTAINER="$TASK13_PROJECT-application-secrets-projection"
TASK13_REGISTRY_RUNTIME_VOLUME="${TASK13_PROJECT}_registry-remote"
TASK13_REGISTRY_PROJECTION_CONTAINER="$TASK13_PROJECT-registry-projection"
TASK13_AUTH_ADMIN=task13-admin