mptyl/ThothII
1
0
Fork 0
Code Issues 25 Pull Requests Actions Packages Projects Releases 1 Wiki Activity

feat: implement metadata catalog database management

Browse Source
This commit is contained in:
Codex
2026-08-27 22:43:54 +02:00
parent 705af3aeb2
commit 79c4c925b5
86 changed files with 12566 additions and 135 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/run-stack.sh
+10 -2
View File
@@ -21,5 +21,13 @@ if [[ "${THOTH_ENABLE_EMBEDDING_GPU:-0}" == "1" ]]; then
compose_files+=(-f "$ROOT/deploy/compose.embedding-gpu.yaml")
fi
exec docker compose --env-file "$LOCAL_ENV_FILE" \
"${compose_files[@]}" up --build "$@"
compose=(docker compose --env-file "$LOCAL_ENV_FILE" "${compose_files[@]}")
# Migrations are an explicit one-shot operation, never hidden in backend startup. The local
# launcher runs that operation before bringing the foreground stack up so a fresh checkout is
# immediately usable while production operators can invoke the same service during rollout.
"${compose[@]}" build core
"${compose[@]}" up -d catalog-db
"${compose[@]}" run --rm catalog-migrate
exec "${compose[@]}" up --build "$@"
scripts/test-canonical-install-compose.sh
+1 -1
View File
@@ -64,7 +64,7 @@ for profile in local server; do
const fs = require("fs");
const [path, profile] = process.argv.slice(2);
const config = JSON.parse(fs.readFileSync(path, "utf8"));
const expected = "core,embedding,embedding-model-init,frontend,qdrant";
const expected = "catalog-db,core,embedding,embedding-model-init,frontend,qdrant";
if (Object.keys(config.services).sort().join(",") !== expected) {
throw new Error(profile + ": install stack must be exactly " + expected);
}
scripts/test-default-compose.sh
+22 -1
View File
@@ -6,6 +6,9 @@ cd "$(dirname "$0")/.."
test -f .env.example
test -f deploy/env/local.env.example
test -f deploy/env/server.env.example
test -f docker/catalog-db-init.sql
grep -q "pg_read_file('/run/secrets/catalog_runtime_password')" docker/catalog-db-init.sql
grep -q "CREATE ROLE thothii_catalog_runtime" docker/catalog-db-init.sql
rendered=$(mktemp)
trap 'rm -f "$rendered"' EXIT HUP INT TERM
@@ -18,7 +21,7 @@ const fs = require("fs");
const config = JSON.parse(fs.readFileSync(process.argv[2], "utf8"));
const services = Object.keys(config.services).sort();
if (services.join(",") !== "core,embedding,embedding-model-init,frontend,qdrant") {
if (services.join(",") !== "catalog-db,core,embedding,embedding-model-init,frontend,qdrant") {
throw new Error(`unexpected service set: ${services.join(",")}`);
}
if (/omics_portal|chirone|localllm_default|\/home\/chirone/i.test(JSON.stringify(config))) {
@@ -26,6 +29,7 @@ if (/omics_portal|chirone|localllm_default|\/home\/chirone/i.test(JSON.stringify
}
const expectedVolumes = [
"auth-state",
"catalog-data",
"embedding-models",
"pi-state",
"qdrant-data",
@@ -43,12 +47,19 @@ const frontend = config.services.frontend;
const qdrant = config.services.qdrant;
const embedding = config.services.embedding;
const modelInit = config.services["embedding-model-init"];
const catalog = config.services["catalog-db"];
if (!frontend.ports?.some((port) => port.host_ip === "127.0.0.1")) {
throw new Error("local frontend must publish a loopback port");
}
for (const service of [embedding, modelInit]) {
if ((service.ports || []).length !== 0) throw new Error("private semantic services must not publish host ports");
}
if ((catalog.ports || []).length !== 0) throw new Error("catalog database must not publish host ports");
if (!catalog.healthcheck) throw new Error("catalog database must define a healthcheck");
const catalogHealthcheck = JSON.stringify(catalog.healthcheck.test || []);
if (!catalogHealthcheck.includes("pg_isready") || !catalogHealthcheck.includes("thothii_catalog_runtime")) {
throw new Error("catalog database healthcheck must verify readiness and the runtime role");
}
if (!qdrant.ports?.some((port) => port.host_ip === "127.0.0.1" && Number(port.target) === 6333)) {
throw new Error("local Qdrant dashboard must publish only its loopback port");
}
@@ -65,6 +76,9 @@ if (embedding.image !== "ollama/ollama:0.32.0@sha256:57f573b47f1f71ebb445789f279
if (modelInit.image !== "ollama/ollama:0.32.0@sha256:57f573b47f1f71ebb445789f279fe3e596a8beab182f7cf486db9205bad87c5a") {
throw new Error("embedding-model-init image must be pinned by version and digest");
}
if (catalog.image !== "postgres:17.6-bookworm@sha256:f3bd19c606e442c3d7bdfa8002e03fe260a1023351e0ea4598032022b68dd6e3") {
throw new Error("catalog PostgreSQL image must be pinned by version and digest");
}
const env = core.environment || {};
for (const [key, value] of Object.entries({
THT_WORKSPACE_INSTALLATION_ID: "local",
@@ -74,6 +88,10 @@ for (const [key, value] of Object.entries({
THT_INTERNAL_EMBEDDING_URL: "http://embedding:11434",
THT_INTERNAL_EMBEDDING_MODEL: "qwen3-embedding:0.6b",
THT_INTERNAL_EMBEDDING_DIMENSIONS: "1024",
THT_CATALOG_DB_HOST: "catalog-db",
THT_CATALOG_DB_NAME: "thothii_catalog",
THT_CATALOG_RUNTIME_USER: "thothii_catalog_runtime",
THT_CATALOG_RUNTIME_PASSWORD_FILE: "/run/secrets/catalog_runtime_password",
})) {
if (env[key] !== value) throw new Error(`unexpected core ${key}: ${env[key]}`);
}
@@ -98,6 +116,9 @@ const depends = core.depends_on || {};
if (depends.qdrant?.condition !== "service_healthy") {
throw new Error("core must wait for qdrant health");
}
if (depends["catalog-db"]?.condition !== "service_healthy") {
throw new Error("core must wait for catalog database health");
}
if (depends["embedding-model-init"]?.condition !== "service_completed_successfully") {
throw new Error("core must wait for embedding-model-init success");
}
scripts/test-unified-compose.sh
+13 -12
View File
@@ -25,14 +25,14 @@ const fs = require("fs");
const [configPath, profile] = process.argv.slice(2);
const config = JSON.parse(fs.readFileSync(configPath, "utf8"));
const services = Object.keys(config.services).sort();
if (services.join(",") !== "core,embedding,embedding-model-init,frontend,qdrant") {
throw new Error("mandatory stack must include core, frontend, qdrant, embedding, and embedding-model-init");
if (services.join(",") !== "catalog-db,core,embedding,embedding-model-init,frontend,qdrant") {
throw new Error("mandatory stack must include catalog-db, core, frontend, qdrant, embedding, and embedding-model-init");
}
if (/omics_portal|chirone|localllm_default|\/home\/chirone/i.test(JSON.stringify(config))) {
throw new Error("forbidden application coupling");
}
if (!config.networks || !config.networks.thothii) throw new Error("base stack must define the thothii network");
for (const volume of ["qdrant-data", "embedding-models"]) {
for (const volume of ["catalog-data", "qdrant-data", "embedding-models"]) {
if (!config.volumes || !config.volumes[volume]) throw new Error(`missing required volume: ${volume}`);
}
if (profile === "local") {
@@ -114,8 +114,8 @@ const bundleSecrets = runtimeSecrets.filter(
if (bundleSecrets.length !== 1) {
throw new Error("core must receive exactly one canonical runtime secret bundle");
}
if (profile === "local" && runtimeSecrets.length !== 1) {
throw new Error("local core must receive only the canonical runtime secret bundle");
if (profile === "local" && runtimeSecrets.length !== 2) {
throw new Error("local core must receive only its runtime bundle and catalog password");
}
if (profile === "server") {
const targets = new Set(runtimeSecrets.map((secret) => secret.target));
@@ -171,14 +171,14 @@ const fs = require("fs");
const config = JSON.parse(fs.readFileSync(process.argv[2], "utf8"));
const services = Object.keys(config.services).sort();
if (services.join(",") !== "core,embedding,embedding-model-init,frontend,qdrant") {
throw new Error("mandatory stack must include core, frontend, qdrant, embedding, and embedding-model-init");
if (services.join(",") !== "catalog-db,core,embedding,embedding-model-init,frontend,qdrant") {
throw new Error("mandatory stack must include catalog-db, core, frontend, qdrant, embedding, and embedding-model-init");
}
if (/omics_portal|chirone|localllm_default|\/home\/chirone/i.test(JSON.stringify(config))) {
throw new Error("forbidden application coupling");
}
if (!config.networks || !config.networks.thothii) throw new Error("base stack must define the thothii network");
for (const volume of ["settings", "pi-state", "workspace-registry", "sessions", "qdrant-data", "embedding-models"]) {
for (const volume of ["settings", "pi-state", "workspace-registry", "sessions", "catalog-data", "qdrant-data", "embedding-models"]) {
if (!config.volumes || !config.volumes[volume]) throw new Error(`missing required volume: ${volume}`);
}
if (!Object.hasOwn(config.services.core.environment || {}, "THT_LLM_URL")) {
@@ -222,10 +222,11 @@ if (config.services.core.environment?.THT_SECRETS_FILE !== "/run/secrets/thothii
throw new Error("core must read the canonical runtime secret bundle from /run/secrets");
}
const runtimeSecrets = config.services.core.secrets || [];
if (runtimeSecrets.length !== 1
|| runtimeSecrets[0].source !== "thothii_secrets"
|| runtimeSecrets[0].target !== "thothii.secrets") {
throw new Error("core must receive exactly the canonical runtime secret bundle");
const hasTarget = (name) => runtimeSecrets.some((secret) => secret.target === name || secret.target?.endsWith(`/${name}`));
if (runtimeSecrets.length !== 2
|| !hasTarget("thothii.secrets")
|| !hasTarget("catalog_runtime_password")) {
throw new Error("core must receive only the canonical runtime bundle and catalog password");
}
if ((config.services.frontend.secrets || []).length !== 0) {
throw new Error("frontend must not receive runtime secrets");
scripts/test-windows-clone-contract.ps1
+4 -4
View File
@@ -275,8 +275,8 @@ overrides:
)
$render = Invoke-BoundedNative -FilePath "docker" -Arguments ($composeArguments + @("config", "--services")) -Label "render Windows Compose from spaced path"
$services = @($render.StdOut -split "`r?`n" | Where-Object { $_ } | Sort-Object)
if (($services -join ",") -ne "core,embedding,embedding-model-init,frontend,qdrant") {
throw "rendered Windows stack must contain the canonical five services"
if (($services -join ",") -ne "catalog-db,core,embedding,embedding-model-init,frontend,qdrant") {
throw "rendered Windows stack must contain the canonical six services"
}
Invoke-BoundedNative -FilePath "docker" -Arguments ($composeArguments + @("config", "--quiet")) -Label "validate Windows Compose from spaced path" | Out-Null
@@ -287,8 +287,8 @@ overrides:
Invoke-BoundedNative -FilePath "docker" -Arguments ($composeArguments + @("up", "--detach", "--wait", "--wait-timeout", "180")) -Label "start canonical Windows stack" -TimeoutSeconds 300 | Out-Null
$running = Invoke-BoundedNative -FilePath "docker" -Arguments ($composeArguments + @("ps", "--status", "running", "--services")) -Label "inspect running Windows services"
$runningServices = @($running.StdOut -split "`r?`n" | Where-Object { $_ } | Sort-Object)
if (($runningServices -join ",") -ne "core,embedding,frontend,qdrant") {
throw "bounded Windows startup did not leave the four long-running services ready"
if (($runningServices -join ",") -ne "catalog-db,core,embedding,frontend,qdrant") {
throw "bounded Windows startup did not leave the five long-running services ready"
}
Invoke-BoundedNative -FilePath $tht -Arguments @("--installation", $installation, "status") -Label "invoke installation-aware Windows tht in spaced path" | Out-Null
}
scripts/verify-workspace-install-docs.sh
+3 -3
View File
@@ -1945,7 +1945,7 @@ verify_local_installation_example() {
const fs = require("fs");
const [path, authConfigRoot] = process.argv.slice(2);
const config = JSON.parse(fs.readFileSync(path, "utf8"));
if (Object.keys(config.services).sort().join(",") !== "core,embedding,embedding-model-init,frontend,qdrant") {
if (Object.keys(config.services).sort().join(",") !== "catalog-db,core,embedding,embedding-model-init,frontend,qdrant") {
throw new Error("local installation example must render the internal semantic stack");
}
const authMount = (config.services.core.volumes || []).find(
@@ -2065,7 +2065,7 @@ verify_server_installation_example() {
const fs = require("fs");
const [path, authConfigRoot] = process.argv.slice(2);
const config = JSON.parse(fs.readFileSync(path, "utf8"));
if (Object.keys(config.services).sort().join(",") !== "core,embedding,embedding-model-init,frontend,qdrant") {
if (Object.keys(config.services).sort().join(",") !== "catalog-db,core,embedding,embedding-model-init,frontend,qdrant") {
throw new Error("server installation example must render the internal semantic stack");
}
const core = config.services.core;
@@ -2216,7 +2216,7 @@ verify_compose_fixtures() {
const fs = require("fs");
const [path, profile, authConfigRoot] = process.argv.slice(2);
const config = JSON.parse(fs.readFileSync(path, "utf8"));
if (Object.keys(config.services).sort().join(",") !== "core,embedding,embedding-model-init,frontend,qdrant") {
if (Object.keys(config.services).sort().join(",") !== "catalog-db,core,embedding,embedding-model-init,frontend,qdrant") {
throw new Error(profile + ": mandatory stack must include the internal semantic services");
}
const core = config.services.core;