Merge origin/codex/portable-deployment into feat/docker-local-deploy
Unisce gli internals di Codex (secret-bundle, provider-credentials, auth upstream, security hardening, CI multiarch) mantenendo le fix portal-specific: - backend: configPath da THT_CONFIG (fix sessioni) + dataRoot di Codex; authMode 'upstream' - Docker/compose: TENUTO il mio (verificato live: omics_network+alias, env_file, pi npm-g) perche' il compose/Dockerfile/entrypoint di Codex sono accoppiati al suo modello secret-bundle (tht doctor inesistente, secret-policy.sh). Adottabile in futuro. - config.test.ts: preso Codex (superset) Verificato: tsc clean, 132/132 vitest.
This commit is contained in:
User
@@ -0,0 +1,90 @@
|
||||
services:
|
||||
core:
|
||||
profiles: [local-vector]
|
||||
environment:
|
||||
THT_VECTOR_DATABASE: "${THT_VECTOR_DATABASE:-thoth}"
|
||||
THT_VECTOR_BOOTSTRAP_USER: "${THT_VECTOR_BOOTSTRAP_USER:-postgres}"
|
||||
THT_VECTOR_READER_USER: "${THT_VECTOR_READER_USER:-thoth_vector_reader}"
|
||||
THT_VECTOR_WRITER_USER: "${THT_VECTOR_WRITER_USER:-thoth_vector_writer}"
|
||||
THT_SECRETS_FILE: /run/secrets/thothii.secrets
|
||||
secrets: [{source: thothii_secrets, target: thothii.secrets}]
|
||||
depends_on:
|
||||
vector-migrate:
|
||||
condition: service_completed_successfully
|
||||
|
||||
frontend:
|
||||
profiles: [local-vector]
|
||||
|
||||
vector-db:
|
||||
image: pgvector/pgvector:0.8.5-pg16@sha256:1d533553fefe4f12e5d80c7b80622ba0c382abb5758856f52983d8789179f0fb
|
||||
profiles: [local-vector]
|
||||
labels: {io.thothii.smoke-owner: "${THOTH_SMOKE_OWNER:-operator}"}
|
||||
environment:
|
||||
POSTGRES_DB: "${THT_VECTOR_DATABASE:-thoth}"
|
||||
POSTGRES_USER: "${THT_VECTOR_BOOTSTRAP_USER:-postgres}"
|
||||
THT_VECTOR_MIGRATOR_USER: "${THT_VECTOR_MIGRATOR_USER:-thoth_vector_migrator}"
|
||||
THT_VECTOR_READER_USER: "${THT_VECTOR_READER_USER:-thoth_vector_reader}"
|
||||
THT_VECTOR_WRITER_USER: "${THT_VECTOR_WRITER_USER:-thoth_vector_writer}"
|
||||
THT_SECRETS_FILE: /run/secrets/thothii.secrets
|
||||
secrets: [{source: thothii_secrets, target: thothii.secrets}]
|
||||
entrypoint: [/opt/thoth/vector-db-entrypoint.sh]
|
||||
volumes:
|
||||
- vector_data:/var/lib/postgresql/data
|
||||
- ./deploy/vector/vector-db-entrypoint.sh:/opt/thoth/vector-db-entrypoint.sh:ro
|
||||
- ./deploy/vector/secret-policy.sh:/opt/thoth/secret-policy.sh:ro
|
||||
healthcheck:
|
||||
test: [CMD-SHELL, "pg_isready -U $$POSTGRES_USER -d $$POSTGRES_DB"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 20
|
||||
start_period: 10s
|
||||
restart: unless-stopped
|
||||
|
||||
vector-reconcile:
|
||||
image: pgvector/pgvector:0.8.5-pg16@sha256:1d533553fefe4f12e5d80c7b80622ba0c382abb5758856f52983d8789179f0fb
|
||||
profiles: [local-vector]
|
||||
labels: {io.thothii.smoke-owner: "${THOTH_SMOKE_OWNER:-operator}"}
|
||||
environment:
|
||||
PGHOST: vector-db
|
||||
PGPORT: 5432
|
||||
PGDATABASE: "${THT_VECTOR_DATABASE:-thoth}"
|
||||
PGUSER: "${THT_VECTOR_BOOTSTRAP_USER:-postgres}"
|
||||
THT_VECTOR_BOOTSTRAP_USER: "${THT_VECTOR_BOOTSTRAP_USER:-postgres}"
|
||||
THT_VECTOR_MIGRATOR_USER: "${THT_VECTOR_MIGRATOR_USER:-thoth_vector_migrator}"
|
||||
THT_VECTOR_READER_USER: "${THT_VECTOR_READER_USER:-thoth_vector_reader}"
|
||||
THT_VECTOR_WRITER_USER: "${THT_VECTOR_WRITER_USER:-thoth_vector_writer}"
|
||||
THT_SECRETS_FILE: /run/secrets/thothii.secrets
|
||||
entrypoint: [/opt/thoth/reconcile-roles.sh]
|
||||
secrets: [{source: thothii_secrets, target: thothii.secrets}]
|
||||
volumes:
|
||||
- ./deploy/vector/reconcile-roles.sh:/opt/thoth/reconcile-roles.sh:ro
|
||||
- ./deploy/vector/secret-policy.sh:/opt/thoth/secret-policy.sh:ro
|
||||
depends_on:
|
||||
vector-db: {condition: service_healthy}
|
||||
restart: "no"
|
||||
|
||||
vector-migrate:
|
||||
image: thothii-core:local
|
||||
profiles: [local-vector]
|
||||
labels: {io.thothii.smoke-owner: "${THOTH_SMOKE_OWNER:-operator}"}
|
||||
build:
|
||||
context: .
|
||||
dockerfile: docker/core.Dockerfile
|
||||
entrypoint: [sh, -ec]
|
||||
command:
|
||||
- |
|
||||
. /opt/thoth/secret-policy.sh
|
||||
export PGPASSWORD=$$(read_bundle_secret /run/secrets/thothii.secrets THT_VECTOR_MIGRATOR_PASSWORD)
|
||||
exec /opt/venv/bin/tht vector migrate --database-url "postgresql+psycopg2://${THT_VECTOR_MIGRATOR_USER:-thoth_vector_migrator}@vector-db:5432/${THT_VECTOR_DATABASE:-thoth}" --json
|
||||
secrets: [{source: thothii_secrets, target: thothii.secrets}]
|
||||
environment:
|
||||
THT_SECRETS_FILE: /run/secrets/thothii.secrets
|
||||
volumes:
|
||||
- ./deploy/vector/secret-policy.sh:/opt/thoth/secret-policy.sh:ro
|
||||
depends_on:
|
||||
vector-reconcile: {condition: service_completed_successfully}
|
||||
restart: "no"
|
||||
|
||||
volumes:
|
||||
vector_data:
|
||||
labels: {io.thothii.smoke-owner: "${THOTH_SMOKE_OWNER:-operator}"}
|
||||
@@ -0,0 +1,6 @@
|
||||
services:
|
||||
core:
|
||||
environment:
|
||||
# Non-secret settings come from the root .env interpolation file.
|
||||
AUTH_MODE: "${AUTH_MODE:-none}"
|
||||
THT_SECRETS_FILE: /run/secrets/thothii.secrets
|
||||
@@ -0,0 +1,14 @@
|
||||
services:
|
||||
preprocess-evidence:
|
||||
environment:
|
||||
THT_SECRETS_FILE: /run/secrets/thothii.secrets
|
||||
secrets: [{source: thothii_secrets, target: thothii.secrets}]
|
||||
depends_on:
|
||||
vector-migrate: {condition: service_completed_successfully}
|
||||
|
||||
preprocess-dwh:
|
||||
environment:
|
||||
THT_SECRETS_FILE: /run/secrets/thothii.secrets
|
||||
secrets: [{source: thothii_secrets, target: thothii.secrets}]
|
||||
depends_on:
|
||||
vector-migrate: {condition: service_completed_successfully}
|
||||
@@ -0,0 +1,35 @@
|
||||
services:
|
||||
preprocess-evidence:
|
||||
image: thothii-core:local
|
||||
profiles: [preprocess]
|
||||
build:
|
||||
context: .
|
||||
dockerfile: docker/core.Dockerfile
|
||||
entrypoint: [sh, -ec]
|
||||
command: ["mkdir -p /data/workspaces/preprocess-evidence && exec /app/docker/core-entrypoint.sh preprocess evidence --json -c /app/harness/workspaces/preprocess-evidence.yaml"]
|
||||
environment:
|
||||
THT_DATA_ROOT: /data
|
||||
THT_OLLAMA_URL: "${THT_OLLAMA_URL:-http://host.docker.internal:11434}"
|
||||
THT_SECRETS_FILE: /run/secrets/thothii.secrets
|
||||
secrets: [{source: thothii_secrets, target: thothii.secrets}]
|
||||
volumes:
|
||||
- thoth_data:/data
|
||||
- ./deploy/workspaces:/app/harness/workspaces:ro
|
||||
restart: "no"
|
||||
|
||||
preprocess-dwh:
|
||||
image: thothii-core:local
|
||||
profiles: [preprocess]
|
||||
build:
|
||||
context: .
|
||||
dockerfile: docker/core.Dockerfile
|
||||
entrypoint: [sh, -ec]
|
||||
command: ["mkdir -p /data/workspaces/preprocess-dwh && exec /app/docker/core-entrypoint.sh preprocess dwh --steps introspect --json -c /app/harness/workspaces/preprocess-dwh.yaml"]
|
||||
environment:
|
||||
THT_DATA_ROOT: /data
|
||||
THT_SECRETS_FILE: /run/secrets/thothii.secrets
|
||||
secrets: [{source: thothii_secrets, target: thothii.secrets}]
|
||||
volumes:
|
||||
- thoth_data:/data
|
||||
- ./deploy/workspaces:/app/harness/workspaces:ro
|
||||
restart: "no"
|
||||
@@ -0,0 +1,11 @@
|
||||
services:
|
||||
core:
|
||||
environment:
|
||||
AUTH_MODE: upstream
|
||||
THOTH_PUBLIC_EXPOSURE: "true"
|
||||
THT_DB_NAME: ${THT_DB_NAME:?set THT_DB_NAME}
|
||||
THT_DWH_REST_URL: ${THT_DWH_REST_URL:?set THT_DWH_REST_URL}
|
||||
THT_VEC_REST_URL: ${THT_VEC_REST_URL:?set THT_VEC_REST_URL}
|
||||
THT_OLLAMA_URL: ${THT_OLLAMA_URL:?set THT_OLLAMA_URL}
|
||||
THT_DOCS_ROOT: ${THT_DOCS_ROOT:-/data/workspaces/example/evidence-source}
|
||||
THT_SECRETS_FILE: /run/secrets/thothii.secrets
|
||||
@@ -0,0 +1,11 @@
|
||||
services:
|
||||
core:
|
||||
environment:
|
||||
THT_DOCS_ROOT: /data/workspaces/psd
|
||||
extra_hosts:
|
||||
- host.docker.internal:host-gateway
|
||||
volumes:
|
||||
- ./deploy/workspaces/psd.yaml:/app/harness/config/tht.yaml:ro
|
||||
- type: bind
|
||||
source: ${THT_PSD_WORKSPACE_HOST_PATH:?set THT_PSD_WORKSPACE_HOST_PATH}
|
||||
target: /data/workspaces/psd
|
||||
@@ -0,0 +1,27 @@
|
||||
# Deprecated compatibility template; it is not loaded by Docker Compose automatically.
|
||||
# New installations must copy ../.env.example to ../.env and run
|
||||
# `docker compose up --build -d` from the repository root. Keep this file only for
|
||||
# staged upgrades that still invoke `--env-file deploy/env.example` explicitly.
|
||||
# Never put secret values in this file.
|
||||
|
||||
COMPOSE_FILE=compose.yaml
|
||||
COMPOSE_PROFILES=
|
||||
THT_SECRETS_FILE=deploy/secrets/thothii.secrets
|
||||
|
||||
PI_PROVIDER=
|
||||
PI_MODEL=
|
||||
PI_THINKING=
|
||||
MAX_PI_PROCESSES=4
|
||||
AUTH_MODE=none
|
||||
|
||||
THT_DB_NAME=
|
||||
THT_DWH_REST_URL=
|
||||
THT_VEC_REST_URL=
|
||||
THT_OLLAMA_URL=
|
||||
THT_DOCS_ROOT=/data/workspaces/example/evidence-source
|
||||
|
||||
THT_VECTOR_DATABASE=thoth
|
||||
THT_VECTOR_BOOTSTRAP_USER=postgres
|
||||
THT_VECTOR_MIGRATOR_USER=thoth_vector_migrator
|
||||
THT_VECTOR_READER_USER=thoth_vector_reader
|
||||
THT_VECTOR_WRITER_USER=thoth_vector_writer
|
||||
@@ -0,0 +1,26 @@
|
||||
# Host nginx example. The auth service MUST authenticate every request and return a stable
|
||||
# identity in X-Authenticated-User. ThothII itself remains on 127.0.0.1:8080.
|
||||
server {
|
||||
listen 443 ssl;
|
||||
server_name thoth.example.test;
|
||||
|
||||
ssl_certificate /etc/nginx/tls/fullchain.pem;
|
||||
ssl_certificate_key /etc/nginx/tls/privkey.pem;
|
||||
|
||||
location = /_authenticate {
|
||||
internal;
|
||||
proxy_pass http://authentication-gateway/verify;
|
||||
proxy_pass_request_body off;
|
||||
proxy_set_header Content-Length "";
|
||||
proxy_set_header X-Original-URI $request_uri;
|
||||
}
|
||||
|
||||
location / {
|
||||
auth_request /_authenticate;
|
||||
auth_request_set $authenticated_user $upstream_http_x_authenticated_user;
|
||||
proxy_set_header X-Authenticated-User $authenticated_user;
|
||||
proxy_set_header X-Forwarded-Proto https;
|
||||
proxy_set_header Host $host;
|
||||
proxy_pass http://127.0.0.1:8080;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
{
|
||||
"providers": {
|
||||
"zai": {
|
||||
"baseUrl": "https://api.z.ai/api/coding/paas/v4",
|
||||
"api": "openai-completions",
|
||||
"apiKey": "$ZAI_API_KEY",
|
||||
"models": [
|
||||
{
|
||||
"id": "glm-5.2",
|
||||
"name": "GLM-5.2",
|
||||
"reasoning": true,
|
||||
"contextWindow": 200000,
|
||||
"maxTokens": 131072
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
{
|
||||
"defaultProjectTrust": "always"
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
# Runtime secrets
|
||||
|
||||
The canonical deployment secret is the single local file
|
||||
`deploy/secrets/thothii.secrets`. Copy the tracked template and protect the copy:
|
||||
|
||||
```sh
|
||||
cp deploy/secrets/thothii.secrets.example deploy/secrets/thothii.secrets
|
||||
chmod 600 deploy/secrets/thothii.secrets
|
||||
```
|
||||
|
||||
The file uses strict `KEY=VALUE` lines (comments and blank lines are allowed). The supported
|
||||
keys are `THT_MODEL_API_KEY`, `THT_DWH_API_KEY`, `THT_VEC_API_KEY`,
|
||||
`THT_VEC_WRITE_API_KEY`, and the four `THT_VECTOR_*_PASSWORD` role passwords. Values must be
|
||||
non-empty and contain no whitespace. Do not put secrets in the root `.env`, workspace YAML,
|
||||
URLs, logs, or `docker compose config` output.
|
||||
|
||||
Compose mounts the bundle read-only as `/run/secrets/thothii.secrets`. The host file must be a
|
||||
regular non-symlink file with mode `0600` or `0400`; Docker's normal `0444` mode is accepted
|
||||
only for the runtime mount beneath `/run/secrets`. The core runs as UID 10001. Verify the mount
|
||||
without printing its contents:
|
||||
|
||||
```sh
|
||||
docker compose run --rm core sh -c 'id && test -r /run/secrets/thothii.secrets'
|
||||
```
|
||||
|
||||
A private CA PEM chain is not a bundle value: PEM whitespace is rejected by the strict parser.
|
||||
Keep it in the host or secret manager and add a reviewed Compose override that mounts it at
|
||||
`/run/secrets/ca-chain.pem` and sets `THT_SSL_CA` (or the adapter-specific setting). The base
|
||||
Compose files intentionally do not create this mount.
|
||||
|
||||
## Migration from separate secret files
|
||||
|
||||
Older installations used `THT_*_SECRET_FILE` variables and one file per value. Migrate by
|
||||
copying each value to its bundle key, validating with `docker compose config --quiet`, and only
|
||||
then deleting the old files. The old variables remain a compatibility path for staged upgrades,
|
||||
but the documented and tested default is `THT_SECRETS_FILE=deploy/secrets/thothii.secrets`.
|
||||
|
||||
The local-vector bootstrap rotation helper still accepts an old/new password file as its
|
||||
maintenance interface. Run it only with files protected by `0600`, then copy the resulting
|
||||
password into `THT_VECTOR_BOOTSTRAP_PASSWORD` in the bundle before restarting
|
||||
`vector-reconcile`/the application. The helper never prints password contents.
|
||||
|
||||
Hosted Pi providers must use a single provider key. Compound providers (Bedrock, Azure OpenAI
|
||||
Responses, Cloudflare Workers AI/Gateway) fail closed until a provider-specific credential
|
||||
adapter is implemented.
|
||||
@@ -0,0 +1,20 @@
|
||||
# Copy to deploy/secrets/thothii.secrets and chmod 600.
|
||||
# Values are read as literal strings (no shell expansion or command substitution).
|
||||
# Leave unused keys out of the file.
|
||||
|
||||
# Hosted model provider (single-key providers only).
|
||||
# THT_MODEL_API_KEY=replace-me
|
||||
|
||||
# External DWH and vector adapters.
|
||||
# THT_DWH_API_KEY=replace-me
|
||||
# THT_VEC_API_KEY=replace-me
|
||||
# THT_VEC_WRITE_API_KEY=replace-me
|
||||
|
||||
# Optional local-vector roles.
|
||||
# THT_VECTOR_BOOTSTRAP_PASSWORD=replace-me
|
||||
# THT_VECTOR_MIGRATOR_PASSWORD=replace-me
|
||||
# THT_VECTOR_READER_PASSWORD=replace-me
|
||||
# THT_VECTOR_WRITER_PASSWORD=replace-me
|
||||
|
||||
# Optional CA material/path understood by the configured adapter.
|
||||
# THT_CA=/run/secrets/ca-chain.pem
|
||||
Executable
+54
@@ -0,0 +1,54 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
. /opt/thoth/secret-policy.sh
|
||||
|
||||
bundle=${THT_SECRETS_FILE:-/run/secrets/thothii.secrets}
|
||||
export PGPASSWORD=$(read_bundle_secret "$bundle" THT_VECTOR_BOOTSTRAP_PASSWORD)
|
||||
migrator_password=$(read_bundle_secret "$bundle" THT_VECTOR_MIGRATOR_PASSWORD)
|
||||
reader_password=$(read_bundle_secret "$bundle" THT_VECTOR_READER_PASSWORD)
|
||||
writer_password=$(read_bundle_secret "$bundle" THT_VECTOR_WRITER_PASSWORD)
|
||||
|
||||
psql --set=ON_ERROR_STOP=1 \
|
||||
--set=migrator_user="$THT_VECTOR_MIGRATOR_USER" \
|
||||
--set=migrator_password="$migrator_password" \
|
||||
--set=reader_user="$THT_VECTOR_READER_USER" \
|
||||
--set=reader_password="$reader_password" \
|
||||
--set=writer_user="$THT_VECTOR_WRITER_USER" \
|
||||
--set=writer_password="$writer_password" <<'SQL'
|
||||
SELECT 'CREATE ROLE vector_reader NOLOGIN'
|
||||
WHERE NOT EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = 'vector_reader') \gexec
|
||||
SELECT 'CREATE ROLE vector_writer NOLOGIN'
|
||||
WHERE NOT EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = 'vector_writer') \gexec
|
||||
ALTER ROLE vector_reader NOLOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION;
|
||||
ALTER ROLE vector_writer NOLOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION;
|
||||
|
||||
SELECT format('CREATE ROLE %I LOGIN', :'migrator_user')
|
||||
WHERE NOT EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = :'migrator_user') \gexec
|
||||
SELECT format('CREATE ROLE %I LOGIN', :'reader_user')
|
||||
WHERE NOT EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = :'reader_user') \gexec
|
||||
SELECT format('CREATE ROLE %I LOGIN', :'writer_user')
|
||||
WHERE NOT EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = :'writer_user') \gexec
|
||||
|
||||
SELECT format(
|
||||
'ALTER ROLE %I LOGIN PASSWORD %L NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION',
|
||||
:'migrator_user', :'migrator_password'
|
||||
) \gexec
|
||||
SELECT format(
|
||||
'ALTER ROLE %I LOGIN PASSWORD %L NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION',
|
||||
:'reader_user', :'reader_password'
|
||||
) \gexec
|
||||
SELECT format(
|
||||
'ALTER ROLE %I LOGIN PASSWORD %L NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION',
|
||||
:'writer_user', :'writer_password'
|
||||
) \gexec
|
||||
|
||||
SELECT format('GRANT vector_reader TO %I', :'reader_user') \gexec
|
||||
SELECT format('GRANT vector_writer TO %I', :'writer_user') \gexec
|
||||
SELECT format('ALTER DATABASE %I OWNER TO %I', current_database(), :'migrator_user') \gexec
|
||||
SELECT format('CREATE SCHEMA IF NOT EXISTS vectors AUTHORIZATION %I', :'migrator_user') \gexec
|
||||
SELECT format('ALTER SCHEMA vectors OWNER TO %I', :'migrator_user') \gexec
|
||||
REVOKE ALL ON SCHEMA vectors FROM PUBLIC;
|
||||
GRANT USAGE ON SCHEMA vectors TO vector_reader, vector_writer;
|
||||
CREATE EXTENSION IF NOT EXISTS vector WITH SCHEMA vectors;
|
||||
SQL
|
||||
Executable
+93
@@ -0,0 +1,93 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Rotate the initialized PostgreSQL bootstrap role and verify before returning success."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import psycopg2
|
||||
from psycopg2 import sql
|
||||
|
||||
|
||||
def read_secret(path: str) -> str:
|
||||
value = Path(path).read_text()
|
||||
if not value or "\x00" in value or any(character.isspace() for character in value):
|
||||
raise ValueError("secret must be non-empty and contain no whitespace or NUL bytes")
|
||||
return value
|
||||
|
||||
|
||||
def connect(password: str):
|
||||
return psycopg2.connect(
|
||||
host=os.environ.get("THT_VECTOR_HOST", "vector-db"),
|
||||
port=int(os.environ.get("THT_VECTOR_PORT", "5432")),
|
||||
dbname=os.environ.get("THT_VECTOR_DATABASE", "thoth"),
|
||||
user=os.environ.get("THT_VECTOR_BOOTSTRAP_USER", "postgres"),
|
||||
password=password,
|
||||
connect_timeout=5,
|
||||
)
|
||||
|
||||
|
||||
def alter_current_role(connection, password: str) -> None:
|
||||
with connection.cursor() as cursor:
|
||||
cursor.execute("SELECT current_user")
|
||||
current_user = cursor.fetchone()[0]
|
||||
expected = os.environ.get("THT_VECTOR_BOOTSTRAP_USER", "postgres")
|
||||
if current_user != expected:
|
||||
raise RuntimeError("authenticated role does not match THT_VECTOR_BOOTSTRAP_USER")
|
||||
cursor.execute(
|
||||
sql.SQL("ALTER ROLE {} PASSWORD {}").format(
|
||||
sql.Identifier(current_user), sql.Literal(password)
|
||||
)
|
||||
)
|
||||
connection.commit()
|
||||
|
||||
|
||||
def main() -> int:
|
||||
if len(sys.argv) != 3:
|
||||
print("usage: rotate-bootstrap-password.py OLD_SECRET NEW_SECRET", file=sys.stderr)
|
||||
return 2
|
||||
try:
|
||||
old_password = read_secret(sys.argv[1])
|
||||
new_password = read_secret(sys.argv[2])
|
||||
if old_password == new_password:
|
||||
raise ValueError("old and new bootstrap passwords must differ")
|
||||
old_connection = connect(old_password)
|
||||
except Exception as exc:
|
||||
print(f"bootstrap rotation refused before change: {type(exc).__name__}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
try:
|
||||
alter_current_role(old_connection, new_password)
|
||||
try:
|
||||
verification = connect(new_password)
|
||||
verification.close()
|
||||
except Exception as verify_exc:
|
||||
try:
|
||||
alter_current_role(old_connection, old_password)
|
||||
except Exception as restore_exc:
|
||||
print(
|
||||
"bootstrap rotation verification failed and password restore failed: "
|
||||
f"{type(verify_exc).__name__}/{type(restore_exc).__name__}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 3
|
||||
print(
|
||||
f"bootstrap rotation verification failed; old password restored: "
|
||||
f"{type(verify_exc).__name__}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 1
|
||||
except Exception as exc:
|
||||
print(f"bootstrap rotation failed: {type(exc).__name__}", file=sys.stderr)
|
||||
return 1
|
||||
finally:
|
||||
old_connection.close()
|
||||
|
||||
print("bootstrap database password rotated and new login verified")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Executable
+95
@@ -0,0 +1,95 @@
|
||||
#!/bin/sh
|
||||
|
||||
validate_secret_file() {
|
||||
secret_path=$1
|
||||
secret_name=$2
|
||||
if [ -L "$secret_path" ] || [ ! -f "$secret_path" ] || [ ! -r "$secret_path" ] || [ ! -s "$secret_path" ]; then
|
||||
echo "$secret_name must be a readable, non-empty regular file" >&2
|
||||
return 2
|
||||
fi
|
||||
if LC_ALL=C grep -q '[[:space:]]' "$secret_path"; then
|
||||
echo "$secret_name must contain no whitespace" >&2
|
||||
return 2
|
||||
fi
|
||||
mode=$(stat -c '%a' "$secret_path" 2>/dev/null || stat -f '%Lp' "$secret_path" 2>/dev/null) || return 2
|
||||
case "$secret_path:$mode" in
|
||||
/run/secrets/*:444|/run/secrets/*:400|/run/secrets/*:600|*:600|*:400) ;;
|
||||
*) echo "$secret_name must have mode 0600 or stricter (Docker secrets may be 0444)" >&2; return 2 ;;
|
||||
esac
|
||||
}
|
||||
|
||||
read_secret_file() {
|
||||
validate_secret_file "$1" "$2" || return
|
||||
cat "$1"
|
||||
}
|
||||
|
||||
# Validate the bundle without printing any value. Keep this parser aligned with
|
||||
# the backend loader: comments/blank lines are allowed, while syntax, allowlist,
|
||||
# duplicates, empty values, file size, and line size are fail-closed.
|
||||
validate_bundle() {
|
||||
bundle_path=$1
|
||||
if [ -L "$bundle_path" ] || [ ! -f "$bundle_path" ] || [ ! -r "$bundle_path" ] || [ ! -s "$bundle_path" ]; then
|
||||
echo "secret bundle must be a readable, non-empty regular file" >&2
|
||||
return 2
|
||||
fi
|
||||
mode=$(stat -c '%a' "$bundle_path" 2>/dev/null || stat -f '%Lp' "$bundle_path" 2>/dev/null) || return 2
|
||||
case "$bundle_path:$mode" in
|
||||
/run/secrets/*:444|/run/secrets/*:400|/run/secrets/*:600|*:600|*:400) ;;
|
||||
*) echo "secret bundle must have mode 0600 or stricter (Docker secrets may be 0444)" >&2; return 2 ;;
|
||||
esac
|
||||
size=$(stat -c '%s' "$bundle_path" 2>/dev/null || stat -f '%z' "$bundle_path" 2>/dev/null) || return 2
|
||||
if [ "$size" -gt 65536 ]; then
|
||||
echo "secret bundle exceeds the 64KiB limit" >&2
|
||||
return 2
|
||||
fi
|
||||
awk '
|
||||
{ sub(/\r$/, "", $0) }
|
||||
length($0) > 16384 { exit 9 }
|
||||
/^[[:space:]]*$/ || /^[[:space:]]*#/ { next }
|
||||
/^[A-Z][A-Z0-9_]*=/ {
|
||||
key=$0; sub(/=.*/, "", key)
|
||||
val=$0; sub(/^[^=]*=/, "", val)
|
||||
if (key !~ /^(THT_MODEL_API_KEY|THT_DWH_API_KEY|THT_VEC_API_KEY|THT_VEC_WRITE_API_KEY|THT_CA|THT_SSL_CA|THT_VECTOR_BOOTSTRAP_PASSWORD|THT_VECTOR_MIGRATOR_PASSWORD|THT_VECTOR_READER_PASSWORD|THT_VECTOR_WRITER_PASSWORD|PI_PROVIDER_API_KEY)$/) exit 6
|
||||
if (val == "" || ++seen[key] > 1) exit 7
|
||||
next
|
||||
}
|
||||
{ exit 4 }
|
||||
' "$bundle_path" || {
|
||||
echo "secret bundle syntax is invalid" >&2
|
||||
return 2
|
||||
}
|
||||
}
|
||||
|
||||
# Read one value from the deployment bundle without putting the bundle itself in
|
||||
# a service environment. Values selected for credentials must contain no spaces.
|
||||
read_bundle_secret() {
|
||||
bundle_path=$1
|
||||
bundle_key=$2
|
||||
validate_bundle "$bundle_path" || return
|
||||
case "$bundle_key" in
|
||||
THT_[A-Z0-9_]*|PI_PROVIDER_API_KEY) ;;
|
||||
*) echo "invalid secret bundle key" >&2; return 2 ;;
|
||||
esac
|
||||
value=$(awk -v wanted="$bundle_key" '
|
||||
{ sub(/\r$/, "", $0) }
|
||||
/^[[:space:]]*$/ || /^[[:space:]]*#/ { next }
|
||||
/^[A-Z][A-Z0-9_]*=/ {
|
||||
key=$0; sub(/=.*/, "", key)
|
||||
val=$0; sub(/^[^=]*=/, "", val)
|
||||
if (key == wanted) {
|
||||
found=1; print val
|
||||
}
|
||||
next
|
||||
}
|
||||
{ exit 4 }
|
||||
END { if (!found) exit 5 }
|
||||
' "$bundle_path") || {
|
||||
echo "$bundle_key is unavailable in secret bundle" >&2
|
||||
return 3
|
||||
}
|
||||
if [ -z "$value" ] || printf '%s' "$value" | LC_ALL=C grep -q '[[:space:]]'; then
|
||||
echo "$bundle_key must contain no whitespace" >&2
|
||||
return 2
|
||||
fi
|
||||
printf '%s' "$value"
|
||||
}
|
||||
Executable
+9
@@ -0,0 +1,9 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
. /opt/thoth/secret-policy.sh
|
||||
|
||||
bundle=${THT_SECRETS_FILE:-/run/secrets/thothii.secrets}
|
||||
export POSTGRES_PASSWORD=$(read_bundle_secret "$bundle" THT_VECTOR_BOOTSTRAP_PASSWORD)
|
||||
unset THT_SECRETS_FILE
|
||||
exec /usr/local/bin/docker-entrypoint.sh postgres
|
||||
@@ -0,0 +1,69 @@
|
||||
language: en
|
||||
|
||||
dwh:
|
||||
type: thoth_rest
|
||||
database:
|
||||
database: ${THT_DB_NAME}
|
||||
schema: datawarehouse
|
||||
endpoint:
|
||||
base_url: ${THT_DWH_REST_URL}
|
||||
api_key: ${THT_DWH_API_KEY}
|
||||
ssl_ca: ${THT_SSL_CA}
|
||||
|
||||
# Relative logical roots are resolved beneath /data/workspaces/example.
|
||||
roots:
|
||||
artifacts: artifacts
|
||||
indexes: indexes
|
||||
sessions: sessions
|
||||
|
||||
examples:
|
||||
max_per_column: 10
|
||||
|
||||
lsh:
|
||||
signature_size: 64
|
||||
n_gram: 3
|
||||
threshold: 0.5
|
||||
max_values_per_column: 1000
|
||||
|
||||
eligibility:
|
||||
max_declared_len: 128
|
||||
max_avg_length: 40
|
||||
max_sampled_len: 200
|
||||
ignore_columns: [etl_last_update]
|
||||
|
||||
evidence:
|
||||
source_root: ${THT_DOCS_ROOT}
|
||||
evidence_dir: evidence
|
||||
|
||||
embeddings:
|
||||
base_url: ${THT_OLLAMA_URL}
|
||||
model: nomic-embed-text-v2-moe
|
||||
dim: 768
|
||||
batch_size: 32
|
||||
|
||||
vectors:
|
||||
type: thoth_vector_http
|
||||
reader:
|
||||
base_url: ${THT_VEC_REST_URL}
|
||||
api_key: ${THT_VEC_API_KEY}
|
||||
ssl_ca: ${THT_SSL_CA}
|
||||
writer:
|
||||
base_url: ${THT_VEC_REST_URL}
|
||||
api_key: ${THT_VEC_WRITE_API_KEY}
|
||||
ssl_ca: ${THT_SSL_CA}
|
||||
|
||||
vector:
|
||||
max_chunk_chars: 4000
|
||||
|
||||
search:
|
||||
rrf_k: 60
|
||||
top_schema_tables: 12
|
||||
schema_chunk_pool: 150
|
||||
|
||||
execution:
|
||||
allow: [cte_test, explain, preview, aggregate, export]
|
||||
max_preview_rows: 10
|
||||
max_export_rows: 100000
|
||||
statement_timeout_ms: 30000
|
||||
warn_execution_ms: 5000
|
||||
max_aggregate_cells: 20
|
||||
@@ -0,0 +1,48 @@
|
||||
language: en
|
||||
|
||||
dwh:
|
||||
type: thoth_rest
|
||||
database:
|
||||
database: ${THT_DB_NAME}
|
||||
schema: datawarehouse
|
||||
endpoint:
|
||||
base_url: ${THT_DWH_REST_URL}
|
||||
api_key: ${THT_DWH_API_KEY}
|
||||
|
||||
vectors:
|
||||
type: pgvector_direct
|
||||
reader:
|
||||
host: vector-db
|
||||
port: 5432
|
||||
database: ${THT_VECTOR_DATABASE}
|
||||
schema: vectors
|
||||
user: ${THT_VECTOR_READER_USER}
|
||||
password_file: ${THT_VECTOR_READER_PASSWORD_FILE}
|
||||
writer:
|
||||
host: vector-db
|
||||
port: 5432
|
||||
database: ${THT_VECTOR_DATABASE}
|
||||
schema: vectors
|
||||
user: ${THT_VECTOR_WRITER_USER}
|
||||
password_file: ${THT_VECTOR_WRITER_PASSWORD_FILE}
|
||||
|
||||
roots:
|
||||
artifacts: artifacts
|
||||
indexes: indexes
|
||||
sessions: sessions
|
||||
|
||||
evidence:
|
||||
source_root: ${THT_DOCS_ROOT}
|
||||
evidence_dir: evidence
|
||||
|
||||
embeddings:
|
||||
base_url: ${THT_OLLAMA_URL}
|
||||
model: nomic-embed-text-v2-moe
|
||||
dim: 768
|
||||
batch_size: 32
|
||||
|
||||
execution:
|
||||
allow: [cte_test, explain, preview, aggregate, export]
|
||||
max_preview_rows: 10
|
||||
max_export_rows: 100000
|
||||
statement_timeout_ms: 30000
|
||||
@@ -0,0 +1,7 @@
|
||||
language: en
|
||||
dwh:
|
||||
type: postgres_direct
|
||||
connection:
|
||||
{host: vector-db, database: thoth, schema: vectors, user: thoth_vector_reader,
|
||||
password_file: "${THT_VECTOR_READER_PASSWORD_FILE}"}
|
||||
roots: {artifacts: artifacts, indexes: indexes, sessions: sessions}
|
||||
@@ -0,0 +1,15 @@
|
||||
language: en
|
||||
dwh:
|
||||
type: postgres_direct
|
||||
connection: {host: unused, database: unused, schema: public, user: unused, password: unused}
|
||||
vectors:
|
||||
type: pgvector_direct
|
||||
reader:
|
||||
{host: vector-db, database: thoth, schema: vectors, user: thoth_vector_reader,
|
||||
password_file: "${THT_VECTOR_READER_PASSWORD_FILE}"}
|
||||
writer:
|
||||
{host: vector-db, database: thoth, schema: vectors, user: thoth_vector_writer,
|
||||
password_file: "${THT_VECTOR_WRITER_PASSWORD_FILE}"}
|
||||
roots: {artifacts: artifacts, indexes: indexes, sessions: sessions}
|
||||
evidence: {source_root: /data/source, evidence_dir: evidence}
|
||||
embeddings: {base_url: "${THT_OLLAMA_URL}", model: smoke, dim: 768, batch_size: 32}
|
||||
@@ -0,0 +1,43 @@
|
||||
language: it
|
||||
|
||||
dwh:
|
||||
type: thoth_rest
|
||||
database:
|
||||
database: ${THT_DB_NAME}
|
||||
schema: datawarehouse
|
||||
endpoint:
|
||||
base_url: ${THT_DWH_REST_URL}
|
||||
api_key: ${THT_DWH_API_KEY}
|
||||
ssl_ca: ${THT_SSL_CA}
|
||||
|
||||
vectors:
|
||||
type: thoth_vector_http
|
||||
reader:
|
||||
base_url: ${THT_VEC_REST_URL}
|
||||
api_key: ${THT_VEC_API_KEY}
|
||||
ssl_ca: ${THT_SSL_CA}
|
||||
writer:
|
||||
base_url: ${THT_VEC_WRITE_REST_URL}
|
||||
api_key: ${THT_VEC_WRITE_API_KEY}
|
||||
ssl_ca: ${THT_SSL_CA}
|
||||
|
||||
roots:
|
||||
artifacts: /data/workspaces/psd/runtime-v2/artifacts
|
||||
indexes: /data/workspaces/psd/runtime-v2/indexes
|
||||
sessions: /data/workspaces/psd/sessions
|
||||
|
||||
evidence:
|
||||
source_root: ${THT_DOCS_ROOT}
|
||||
evidence_dir: evidence
|
||||
|
||||
embeddings:
|
||||
base_url: ${THT_OLLAMA_URL}
|
||||
model: nomic-embed-text-v2-moe
|
||||
dim: 768
|
||||
batch_size: 32
|
||||
|
||||
execution:
|
||||
allow: [cte_test, explain, preview, aggregate, export]
|
||||
max_preview_rows: 10
|
||||
max_export_rows: 100000
|
||||
statement_timeout_ms: 30000
|
||||
Reference in New Issue
Block a user