mptyl/ThothII
1
0
Fork 0
Code Issues 25 Pull Requests Actions Packages Projects Releases 1 Wiki Activity

refactor: retire external vector deployment

Browse Source
This commit is contained in:
marcopan
2026-08-08 19:05:57 +02:00
parent 8f4ec1e1a3
commit 4e3fecbe8e
44 changed files with 370 additions and 1710 deletions
Download Patch File Download Diff File Expand all files Collapse all files
deploy/compose.local-vector.yaml
-90
View File
@@ -1,90 +0,0 @@
services:
core:
profiles: [local-vector]
environment:
THT_VECTOR_DATABASE: "${THT_VECTOR_DATABASE:-thoth}"
THT_VECTOR_BOOTSTRAP_USER: "${THT_VECTOR_BOOTSTRAP_USER:-postgres}"
THT_VECTOR_READER_USER: "${THT_VECTOR_READER_USER:-thoth_vector_reader}"
THT_VECTOR_WRITER_USER: "${THT_VECTOR_WRITER_USER:-thoth_vector_writer}"
THT_SECRETS_FILE: /run/secrets/thothii.secrets
secrets: [{source: thothii_secrets, target: thothii.secrets}]
depends_on:
vector-migrate:
condition: service_completed_successfully
frontend:
profiles: [local-vector]
vector-db:
image: pgvector/pgvector:0.8.5-pg16@sha256:1d533553fefe4f12e5d80c7b80622ba0c382abb5758856f52983d8789179f0fb
profiles: [local-vector]
labels: {io.thothii.smoke-owner: "${THOTH_SMOKE_OWNER:-operator}"}
environment:
POSTGRES_DB: "${THT_VECTOR_DATABASE:-thoth}"
POSTGRES_USER: "${THT_VECTOR_BOOTSTRAP_USER:-postgres}"
THT_VECTOR_MIGRATOR_USER: "${THT_VECTOR_MIGRATOR_USER:-thoth_vector_migrator}"
THT_VECTOR_READER_USER: "${THT_VECTOR_READER_USER:-thoth_vector_reader}"
THT_VECTOR_WRITER_USER: "${THT_VECTOR_WRITER_USER:-thoth_vector_writer}"
THT_SECRETS_FILE: /run/secrets/thothii.secrets
secrets: [{source: thothii_secrets, target: thothii.secrets}]
entrypoint: [/opt/thoth/vector-db-entrypoint.sh]
volumes:
- vector_data:/var/lib/postgresql/data
- ./deploy/vector/vector-db-entrypoint.sh:/opt/thoth/vector-db-entrypoint.sh:ro
- ./deploy/vector/secret-policy.sh:/opt/thoth/secret-policy.sh:ro
healthcheck:
test: [CMD-SHELL, "pg_isready -U $$POSTGRES_USER -d $$POSTGRES_DB"]
interval: 5s
timeout: 3s
retries: 20
start_period: 10s
restart: unless-stopped
vector-reconcile:
image: pgvector/pgvector:0.8.5-pg16@sha256:1d533553fefe4f12e5d80c7b80622ba0c382abb5758856f52983d8789179f0fb
profiles: [local-vector]
labels: {io.thothii.smoke-owner: "${THOTH_SMOKE_OWNER:-operator}"}
environment:
PGHOST: vector-db
PGPORT: 5432
PGDATABASE: "${THT_VECTOR_DATABASE:-thoth}"
PGUSER: "${THT_VECTOR_BOOTSTRAP_USER:-postgres}"
THT_VECTOR_BOOTSTRAP_USER: "${THT_VECTOR_BOOTSTRAP_USER:-postgres}"
THT_VECTOR_MIGRATOR_USER: "${THT_VECTOR_MIGRATOR_USER:-thoth_vector_migrator}"
THT_VECTOR_READER_USER: "${THT_VECTOR_READER_USER:-thoth_vector_reader}"
THT_VECTOR_WRITER_USER: "${THT_VECTOR_WRITER_USER:-thoth_vector_writer}"
THT_SECRETS_FILE: /run/secrets/thothii.secrets
entrypoint: [/opt/thoth/reconcile-roles.sh]
secrets: [{source: thothii_secrets, target: thothii.secrets}]
volumes:
- ./deploy/vector/reconcile-roles.sh:/opt/thoth/reconcile-roles.sh:ro
- ./deploy/vector/secret-policy.sh:/opt/thoth/secret-policy.sh:ro
depends_on:
vector-db: {condition: service_healthy}
restart: "no"
vector-migrate:
image: thothii-core:local
profiles: [local-vector]
labels: {io.thothii.smoke-owner: "${THOTH_SMOKE_OWNER:-operator}"}
build:
context: .
dockerfile: docker/core.Dockerfile
entrypoint: [sh, -ec]
command:
- |
. /opt/thoth/secret-policy.sh
export PGPASSWORD=$$(read_bundle_secret /run/secrets/thothii.secrets THT_VECTOR_MIGRATOR_PASSWORD)
exec /opt/venv/bin/tht vector migrate --database-url "postgresql+psycopg2://${THT_VECTOR_MIGRATOR_USER:-thoth_vector_migrator}@vector-db:5432/${THT_VECTOR_DATABASE:-thoth}" --json
secrets: [{source: thothii_secrets, target: thothii.secrets}]
environment:
THT_SECRETS_FILE: /run/secrets/thothii.secrets
volumes:
- ./deploy/vector/secret-policy.sh:/opt/thoth/secret-policy.sh:ro
depends_on:
vector-reconcile: {condition: service_completed_successfully}
restart: "no"
volumes:
vector_data:
labels: {io.thothii.smoke-owner: "${THOTH_SMOKE_OWNER:-operator}"}
deploy/compose.preprocess-local-vector.yaml
-14
View File
@@ -1,14 +0,0 @@
services:
preprocess-evidence:
environment:
THT_SECRETS_FILE: /run/secrets/thothii.secrets
secrets: [{source: thothii_secrets, target: thothii.secrets}]
depends_on:
vector-migrate: {condition: service_completed_successfully}
preprocess-dwh:
environment:
THT_SECRETS_FILE: /run/secrets/thothii.secrets
secrets: [{source: thothii_secrets, target: thothii.secrets}]
depends_on:
vector-migrate: {condition: service_completed_successfully}
deploy/compose.preprocess.yaml
+5 -1
View File
@@ -9,13 +9,17 @@ services:
command: ["mkdir -p /data/workspaces/preprocess-evidence && exec /app/docker/core-entrypoint.sh preprocess evidence --json -c /app/harness/workspaces/preprocess-evidence.yaml"]
environment:
THT_DATA_ROOT: /data
THT_OLLAMA_URL: "${THT_OLLAMA_URL:-http://host.docker.internal:11434}"
THT_SECRETS_FILE: /run/secrets/thothii.secrets
secrets: [{source: thothii_secrets, target: thothii.secrets}]
volumes:
- thoth_data:/data
- ./deploy/workspaces:/app/harness/workspaces:ro
restart: "no"
depends_on:
qdrant:
condition: service_healthy
embedding-model-init:
condition: service_completed_successfully
preprocess-dwh:
image: thothii-core:local
deploy/secrets/README.md
+7 -27
View File
@@ -9,10 +9,9 @@ chmod 600 deploy/secrets/thothii.secrets
```
The file uses strict `KEY=VALUE` lines (comments and blank lines are allowed). The supported
keys are `THT_MODEL_API_KEY`, `THT_DWH_API_KEY`, `THT_VEC_API_KEY`,
`THT_VEC_WRITE_API_KEY`, and the four `THT_VECTOR_*_PASSWORD` role passwords. Values must be
non-empty and contain no whitespace. Do not put secrets in the root `.env`, workspace YAML,
URLs, logs, or rendered Compose output.
keys are `THT_MODEL_API_KEY`, `THT_DWH_API_KEY`, `THT_CA`, `THT_SSL_CA`, and
`PI_PROVIDER_API_KEY`. Values must be non-empty and contain no whitespace. Do not put secrets
in the root `.env`, workspace YAML, URLs, logs, or rendered Compose output.
Compose mounts the bundle read-only as `/run/secrets/thothii.secrets`. The host file must be a
regular non-symlink file with mode `0600` or `0400`; Docker's normal `0444` mode is accepted
@@ -33,29 +32,10 @@ Compose files intentionally do not create this mount.
## Migration from separate secret files
Older installations used `THT_*_SECRET_FILE` variables and one file per value. Migrate by
copying each value to its bundle key, validating with the complete base+profile command, and only
then deleting the old files. The old variables remain a compatibility path for staged upgrades,
but the documented and tested default is an absolute `THT_SECRETS_FILE` path to the protected
bundle.
The local-vector bootstrap rotation helper still accepts an old/new password file as its
maintenance interface. Run it only with files protected by `0600`, then copy the resulting
password into `THT_VECTOR_BOOTSTRAP_PASSWORD` in the bundle before restarting
`vector-reconcile`/the application. The helper never prints password contents.
The helper has no implicit operator-env default. Pass the same protected env file used for the
deployment explicitly; it must be a readable regular non-symlink file and must not be writable by
group or other users:
```sh
chmod 600 deploy/env/local.env
./scripts/vector-rotate-bootstrap-password.sh \
--env-file "$(pwd)/deploy/env/local.env" \
/secure/thoth/bootstrap-password /secure/thoth/bootstrap-password.next
```
Automation may set the narrowly scoped `THT_VECTOR_OPERATOR_ENV_FILE` instead. An explicit
`--env-file` takes precedence. Missing or unsafe env files are rejected before Compose runs.
copying each retained value to its bundle key, validating with the complete base+profile command,
and only then deleting the old files. The old variables remain a compatibility path for staged
upgrades, but the documented and tested default is an absolute `THT_SECRETS_FILE` path to the
protected bundle.
Hosted Pi providers must use a single provider key. Compound providers (Bedrock, Azure OpenAI
Responses, Cloudflare Workers AI/Gateway) fail closed until a provider-specific credential
deploy/secrets/thothii.secrets.example
+1 -9
View File
@@ -5,16 +5,8 @@
# Hosted model provider (single-key providers only).
# THT_MODEL_API_KEY=replace-me
# External DWH and vector adapters.
# External DWH adapter.
# THT_DWH_API_KEY=replace-me
# THT_VEC_API_KEY=replace-me
# THT_VEC_WRITE_API_KEY=replace-me
# Optional local-vector roles.
# THT_VECTOR_BOOTSTRAP_PASSWORD=replace-me
# THT_VECTOR_MIGRATOR_PASSWORD=replace-me
# THT_VECTOR_READER_PASSWORD=replace-me
# THT_VECTOR_WRITER_PASSWORD=replace-me
# Optional CA material/path understood by the configured adapter.
# THT_CA=/run/secrets/ca-chain.pem
deploy/sql/20-vector-roles.sql
-25
View File
@@ -1,25 +0,0 @@
-- ThothII — ruolo vector read+write (schema vectors).
-- Stessa istanza del DWH (porta 5438). ThothII indicizza (write) + ricerca (read) direttamente.
-- La separazione reader/writer resta rilevante solo per il path REST (non usato in Profile A).
-- psql -h localhost -p 5438 -U postgres -d postgres -v PWD='<secret>' -f 20-vector-roles.sql
DO $$
BEGIN
IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'thoth_vector_rw') THEN
CREATE ROLE thoth_vector_rw LOGIN;
END IF;
END $$;
-- :'PWD' va fuori dal DO (psql non interpola nelle stringhe dollar-quoted)
ALTER ROLE thoth_vector_rw PASSWORD :'PWD';
CREATE SCHEMA IF NOT EXISTS vectors;
-- L'estensione pgvector deve esistere (già presente nell'istanza di produzione).
-- CREATE EXTENSION IF NOT EXISTS vector;
GRANT USAGE, CREATE ON SCHEMA vectors TO thoth_vector_rw;
GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA vectors TO thoth_vector_rw;
GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA vectors TO thoth_vector_rw;
ALTER DEFAULT PRIVILEGES IN SCHEMA vectors
GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO thoth_vector_rw;
ALTER DEFAULT PRIVILEGES IN SCHEMA vectors
GRANT USAGE, SELECT ON SEQUENCES TO thoth_vector_rw;
deploy/vector/reconcile-roles.sh
-54
View File
@@ -1,54 +0,0 @@
#!/bin/sh
set -eu
. /opt/thoth/secret-policy.sh
bundle=${THT_SECRETS_FILE:-/run/secrets/thothii.secrets}
export PGPASSWORD=$(read_bundle_secret "$bundle" THT_VECTOR_BOOTSTRAP_PASSWORD)
migrator_password=$(read_bundle_secret "$bundle" THT_VECTOR_MIGRATOR_PASSWORD)
reader_password=$(read_bundle_secret "$bundle" THT_VECTOR_READER_PASSWORD)
writer_password=$(read_bundle_secret "$bundle" THT_VECTOR_WRITER_PASSWORD)
psql --set=ON_ERROR_STOP=1 \
--set=migrator_user="$THT_VECTOR_MIGRATOR_USER" \
--set=migrator_password="$migrator_password" \
--set=reader_user="$THT_VECTOR_READER_USER" \
--set=reader_password="$reader_password" \
--set=writer_user="$THT_VECTOR_WRITER_USER" \
--set=writer_password="$writer_password" <<'SQL'
SELECT 'CREATE ROLE vector_reader NOLOGIN'
WHERE NOT EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = 'vector_reader') \gexec
SELECT 'CREATE ROLE vector_writer NOLOGIN'
WHERE NOT EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = 'vector_writer') \gexec
ALTER ROLE vector_reader NOLOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION;
ALTER ROLE vector_writer NOLOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION;
SELECT format('CREATE ROLE %I LOGIN', :'migrator_user')
WHERE NOT EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = :'migrator_user') \gexec
SELECT format('CREATE ROLE %I LOGIN', :'reader_user')
WHERE NOT EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = :'reader_user') \gexec
SELECT format('CREATE ROLE %I LOGIN', :'writer_user')
WHERE NOT EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = :'writer_user') \gexec
SELECT format(
'ALTER ROLE %I LOGIN PASSWORD %L NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION',
:'migrator_user', :'migrator_password'
) \gexec
SELECT format(
'ALTER ROLE %I LOGIN PASSWORD %L NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION',
:'reader_user', :'reader_password'
) \gexec
SELECT format(
'ALTER ROLE %I LOGIN PASSWORD %L NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION',
:'writer_user', :'writer_password'
) \gexec
SELECT format('GRANT vector_reader TO %I', :'reader_user') \gexec
SELECT format('GRANT vector_writer TO %I', :'writer_user') \gexec
SELECT format('ALTER DATABASE %I OWNER TO %I', current_database(), :'migrator_user') \gexec
SELECT format('CREATE SCHEMA IF NOT EXISTS vectors AUTHORIZATION %I', :'migrator_user') \gexec
SELECT format('ALTER SCHEMA vectors OWNER TO %I', :'migrator_user') \gexec
REVOKE ALL ON SCHEMA vectors FROM PUBLIC;
GRANT USAGE ON SCHEMA vectors TO vector_reader, vector_writer;
CREATE EXTENSION IF NOT EXISTS vector WITH SCHEMA vectors;
SQL
deploy/vector/rotate-bootstrap-password.py
-93
View File
@@ -1,93 +0,0 @@
#!/usr/bin/env python3
"""Rotate the initialized PostgreSQL bootstrap role and verify before returning success."""
from __future__ import annotations
import os
import sys
from pathlib import Path
import psycopg2
from psycopg2 import sql
def read_secret(path: str) -> str:
value = Path(path).read_text()
if not value or "\x00" in value or any(character.isspace() for character in value):
raise ValueError("secret must be non-empty and contain no whitespace or NUL bytes")
return value
def connect(password: str):
return psycopg2.connect(
host=os.environ.get("THT_VECTOR_HOST", "vector-db"),
port=int(os.environ.get("THT_VECTOR_PORT", "5432")),
dbname=os.environ.get("THT_VECTOR_DATABASE", "thoth"),
user=os.environ.get("THT_VECTOR_BOOTSTRAP_USER", "postgres"),
password=password,
connect_timeout=5,
)
def alter_current_role(connection, password: str) -> None:
with connection.cursor() as cursor:
cursor.execute("SELECT current_user")
current_user = cursor.fetchone()[0]
expected = os.environ.get("THT_VECTOR_BOOTSTRAP_USER", "postgres")
if current_user != expected:
raise RuntimeError("authenticated role does not match THT_VECTOR_BOOTSTRAP_USER")
cursor.execute(
sql.SQL("ALTER ROLE {} PASSWORD {}").format(
sql.Identifier(current_user), sql.Literal(password)
)
)
connection.commit()
def main() -> int:
if len(sys.argv) != 3:
print("usage: rotate-bootstrap-password.py OLD_SECRET NEW_SECRET", file=sys.stderr)
return 2
try:
old_password = read_secret(sys.argv[1])
new_password = read_secret(sys.argv[2])
if old_password == new_password:
raise ValueError("old and new bootstrap passwords must differ")
old_connection = connect(old_password)
except Exception as exc:
print(f"bootstrap rotation refused before change: {type(exc).__name__}", file=sys.stderr)
return 1
try:
alter_current_role(old_connection, new_password)
try:
verification = connect(new_password)
verification.close()
except Exception as verify_exc:
try:
alter_current_role(old_connection, old_password)
except Exception as restore_exc:
print(
"bootstrap rotation verification failed and password restore failed: "
f"{type(verify_exc).__name__}/{type(restore_exc).__name__}",
file=sys.stderr,
)
return 3
print(
f"bootstrap rotation verification failed; old password restored: "
f"{type(verify_exc).__name__}",
file=sys.stderr,
)
return 1
except Exception as exc:
print(f"bootstrap rotation failed: {type(exc).__name__}", file=sys.stderr)
return 1
finally:
old_connection.close()
print("bootstrap database password rotated and new login verified")
return 0
if __name__ == "__main__":
raise SystemExit(main())
deploy/vector/secret-policy.sh
-95
View File
@@ -1,95 +0,0 @@
#!/bin/sh
validate_secret_file() {
secret_path=$1
secret_name=$2
if [ -L "$secret_path" ] || [ ! -f "$secret_path" ] || [ ! -r "$secret_path" ] || [ ! -s "$secret_path" ]; then
echo "$secret_name must be a readable, non-empty regular file" >&2
return 2
fi
if LC_ALL=C grep -q '[[:space:]]' "$secret_path"; then
echo "$secret_name must contain no whitespace" >&2
return 2
fi
mode=$(stat -c '%a' "$secret_path" 2>/dev/null || stat -f '%Lp' "$secret_path" 2>/dev/null) || return 2
case "$secret_path:$mode" in
/run/secrets/*:444|/run/secrets/*:400|/run/secrets/*:600|*:600|*:400) ;;
*) echo "$secret_name must have mode 0600 or stricter (Docker secrets may be 0444)" >&2; return 2 ;;
esac
}
read_secret_file() {
validate_secret_file "$1" "$2" || return
cat "$1"
}
# Validate the bundle without printing any value. Keep this parser aligned with
# the backend loader: comments/blank lines are allowed, while syntax, allowlist,
# duplicates, empty values, file size, and line size are fail-closed.
validate_bundle() {
bundle_path=$1
if [ -L "$bundle_path" ] || [ ! -f "$bundle_path" ] || [ ! -r "$bundle_path" ] || [ ! -s "$bundle_path" ]; then
echo "secret bundle must be a readable, non-empty regular file" >&2
return 2
fi
mode=$(stat -c '%a' "$bundle_path" 2>/dev/null || stat -f '%Lp' "$bundle_path" 2>/dev/null) || return 2
case "$bundle_path:$mode" in
/run/secrets/*:444|/run/secrets/*:400|/run/secrets/*:600|*:600|*:400) ;;
*) echo "secret bundle must have mode 0600 or stricter (Docker secrets may be 0444)" >&2; return 2 ;;
esac
size=$(stat -c '%s' "$bundle_path" 2>/dev/null || stat -f '%z' "$bundle_path" 2>/dev/null) || return 2
if [ "$size" -gt 65536 ]; then
echo "secret bundle exceeds the 64KiB limit" >&2
return 2
fi
awk '
{ sub(/\r$/, "", $0) }
length($0) > 16384 { exit 9 }
/^[[:space:]]*$/ || /^[[:space:]]*#/ { next }
/^[A-Z][A-Z0-9_]*=/ {
key=$0; sub(/=.*/, "", key)
val=$0; sub(/^[^=]*=/, "", val)
if (key !~ /^(THT_MODEL_API_KEY|THT_DWH_API_KEY|THT_VEC_API_KEY|THT_VEC_WRITE_API_KEY|THT_CA|THT_SSL_CA|THT_VECTOR_BOOTSTRAP_PASSWORD|THT_VECTOR_MIGRATOR_PASSWORD|THT_VECTOR_READER_PASSWORD|THT_VECTOR_WRITER_PASSWORD|PI_PROVIDER_API_KEY)$/) exit 6
if (val == "" || ++seen[key] > 1) exit 7
next
}
{ exit 4 }
' "$bundle_path" || {
echo "secret bundle syntax is invalid" >&2
return 2
}
}
# Read one value from the deployment bundle without putting the bundle itself in
# a service environment. Values selected for credentials must contain no spaces.
read_bundle_secret() {
bundle_path=$1
bundle_key=$2
validate_bundle "$bundle_path" || return
case "$bundle_key" in
THT_[A-Z0-9_]*|PI_PROVIDER_API_KEY) ;;
*) echo "invalid secret bundle key" >&2; return 2 ;;
esac
value=$(awk -v wanted="$bundle_key" '
{ sub(/\r$/, "", $0) }
/^[[:space:]]*$/ || /^[[:space:]]*#/ { next }
/^[A-Z][A-Z0-9_]*=/ {
key=$0; sub(/=.*/, "", key)
val=$0; sub(/^[^=]*=/, "", val)
if (key == wanted) {
found=1; print val
}
next
}
{ exit 4 }
END { if (!found) exit 5 }
' "$bundle_path") || {
echo "$bundle_key is unavailable in secret bundle" >&2
return 3
}
if [ -z "$value" ] || printf '%s' "$value" | LC_ALL=C grep -q '[[:space:]]'; then
echo "$bundle_key must contain no whitespace" >&2
return 2
fi
printf '%s' "$value"
}
deploy/vector/vector-db-entrypoint.sh
-9
View File
@@ -1,9 +0,0 @@
#!/bin/sh
set -eu
. /opt/thoth/secret-policy.sh
bundle=${THT_SECRETS_FILE:-/run/secrets/thothii.secrets}
export POSTGRES_PASSWORD=$(read_bundle_secret "$bundle" THT_VECTOR_BOOTSTRAP_PASSWORD)
unset THT_SECRETS_FILE
exec /usr/local/bin/docker-entrypoint.sh postgres
deploy/workspaces/preprocess-dwh.yaml
+6 -2
View File
@@ -2,6 +2,10 @@ language: en
dwh:
type: postgres_direct
connection:
{host: vector-db, database: thoth, schema: vectors, user: thoth_vector_reader,
password_file: "${THT_VECTOR_READER_PASSWORD_FILE}"}
host: "${THT_PREPROCESS_DWH_HOST:-dwh}"
port: "${THT_PREPROCESS_DWH_PORT:-5432}"
database: "${THT_PREPROCESS_DWH_DATABASE:-warehouse}"
schema: "${THT_PREPROCESS_DWH_SCHEMA:-public}"
user: "${THT_PREPROCESS_DWH_USER:-thoth_reader}"
password_file: "${THT_PREPROCESS_DWH_PASSWORD_FILE:-/run/secrets/preprocess-dwh-password}"
roots: {artifacts: artifacts, indexes: indexes, sessions: sessions}
deploy/workspaces/preprocess-evidence.yaml
+16 -9
View File
@@ -1,15 +1,22 @@
language: en
dwh:
type: postgres_direct
connection: {host: unused, database: unused, schema: public, user: unused, password: unused}
connection:
host: "${THT_PREPROCESS_DWH_HOST:-unused}"
port: "${THT_PREPROCESS_DWH_PORT:-5432}"
database: "${THT_PREPROCESS_DWH_DATABASE:-unused}"
schema: "${THT_PREPROCESS_DWH_SCHEMA:-public}"
user: "${THT_PREPROCESS_DWH_USER:-unused}"
password_file: "${THT_PREPROCESS_DWH_PASSWORD_FILE:-/run/secrets/preprocess-dwh-password}"
vectors:
type: pgvector_direct
reader:
{host: vector-db, database: thoth, schema: vectors, user: thoth_vector_reader,
password_file: "${THT_VECTOR_READER_PASSWORD_FILE}"}
writer:
{host: vector-db, database: thoth, schema: vectors, user: thoth_vector_writer,
password_file: "${THT_VECTOR_WRITER_PASSWORD_FILE}"}
type: qdrant
base_url: http://qdrant:6333
collection: preprocess-evidence
roots: {artifacts: artifacts, indexes: indexes, sessions: sessions}
evidence: {source_root: /data/source, evidence_dir: evidence}
embeddings: {base_url: "${THT_OLLAMA_URL}", model: smoke, dim: 768, batch_size: 32}
embeddings:
provider: ollama_internal
base_url: http://embedding:11434
model: qwen3-embedding:0.6b
dim: 1024
batch_size: 32