fix(preprocess): harden S3 and real Compose jobs

deploy/compose.preprocess.yaml

@@ -5,10 +5,14 @@ services:
     build:
       context: .
       dockerfile: docker/core.Dockerfile
-    entrypoint: [/app/docker/core-entrypoint.sh, preprocess]
-    command: [evidence, --json, -c, "/app/harness/workspaces/${THT_PREPROCESS_WORKSPACE:-tht.example}.yaml"]
+    entrypoint: [sh, -ec]
+    command: ["mkdir -p /data/workspaces/preprocess-evidence && exec /app/docker/core-entrypoint.sh preprocess evidence --json -c /app/harness/workspaces/preprocess-evidence.yaml"]
     environment:
       THT_DATA_ROOT: /data
+      THT_OLLAMA_URL: "${THT_OLLAMA_URL:-http://host.docker.internal:11434}"
+      THT_VECTOR_READER_PASSWORD_FILE: /run/secrets/vector_reader_password
+      THT_VECTOR_WRITER_PASSWORD_FILE: /run/secrets/vector_writer_password
+    secrets: [vector_reader_password, vector_writer_password]
     volumes:
       - thoth_data:/data
       - ./deploy/workspaces:/app/harness/workspaces:ro
@@ -20,11 +24,19 @@ services:
     build:
       context: .
       dockerfile: docker/core.Dockerfile
-    entrypoint: [/app/docker/core-entrypoint.sh, preprocess]
-    command: [dwh, --json, -c, "/app/harness/workspaces/${THT_PREPROCESS_WORKSPACE:-tht.example}.yaml"]
+    entrypoint: [sh, -ec]
+    command: ["mkdir -p /data/workspaces/preprocess-dwh && exec /app/docker/core-entrypoint.sh preprocess dwh --steps introspect --json -c /app/harness/workspaces/preprocess-dwh.yaml"]
     environment:
       THT_DATA_ROOT: /data
+      THT_VECTOR_READER_PASSWORD_FILE: /run/secrets/vector_reader_password
+    secrets: [vector_reader_password]
     volumes:
       - thoth_data:/data
       - ./deploy/workspaces:/app/harness/workspaces:ro
     restart: "no"
+
+secrets:
+  vector_reader_password:
+    file: ${THT_VECTOR_READER_PASSWORD_SECRET_FILE:?set THT_VECTOR_READER_PASSWORD_SECRET_FILE}
+  vector_writer_password:
+    file: ${THT_VECTOR_WRITER_PASSWORD_SECRET_FILE:?set THT_VECTOR_WRITER_PASSWORD_SECRET_FILE}

deploy/workspaces/preprocess-dwh.yaml

@@ -0,0 +1,7 @@
+language: en
+dwh:
+  type: postgres_direct
+  connection:
+    {host: vector-db, database: thoth, schema: vectors, user: thoth_vector_reader,
+     password_file: "${THT_VECTOR_READER_PASSWORD_FILE}"}
+roots: {artifacts: artifacts, indexes: indexes, sessions: sessions}

deploy/workspaces/preprocess-evidence.yaml

@@ -0,0 +1,15 @@
+language: en
+dwh:
+  type: postgres_direct
+  connection: {host: unused, database: unused, schema: public, user: unused, password: unused}
+vectors:
+  type: pgvector_direct
+  reader:
+    {host: vector-db, database: thoth, schema: vectors, user: thoth_vector_reader,
+     password_file: "${THT_VECTOR_READER_PASSWORD_FILE}"}
+  writer:
+    {host: vector-db, database: thoth, schema: vectors, user: thoth_vector_writer,
+     password_file: "${THT_VECTOR_WRITER_PASSWORD_FILE}"}
+roots: {artifacts: artifacts, indexes: indexes, sessions: sessions}
+evidence: {source_root: /data/source, evidence_dir: evidence}
+embeddings: {base_url: "${THT_OLLAMA_URL}", model: smoke, dim: 768, batch_size: 32}