From 1e0911bb6a9909b1e5e4d8fc79d64188a00556d3 Mon Sep 17 00:00:00 2001 From: mptyl Date: Sat, 11 Jul 2026 20:43:08 +0200 Subject: [PATCH] feat(config): add typed resource schema --- harness/tests/test_config_legacy_compat.py | 105 +++++++++++++++++++++ harness/tests/test_config_resources.py | 71 ++++++++++++++ harness/tht/config.py | 105 ++++++++++++++++++++- harness/tht/config_compat.py | 74 +++++++++++++++ harness/workspaces/tht.example.yaml | 70 ++++++-------- 5 files changed, 380 insertions(+), 45 deletions(-) create mode 100644 harness/tests/test_config_legacy_compat.py create mode 100644 harness/tests/test_config_resources.py create mode 100644 harness/tht/config_compat.py diff --git a/harness/tests/test_config_legacy_compat.py b/harness/tests/test_config_legacy_compat.py new file mode 100644 index 00000000..d14da00f --- /dev/null +++ b/harness/tests/test_config_legacy_compat.py @@ -0,0 +1,105 @@ +import json + +import pytest +from typer.testing import CliRunner + +from tht.cli import app +from tht.config import load_config + + +def _write_old_workspace(tmp_path): + path = tmp_path / "old.yaml" + path.write_text( + """ +database: + host: ignored-for-rest + database: analytics + schema: mart + user: legacy-user + password: legacy-password + transport: rest +rest: + base_url: https://dwh.example.test/ + api_key: dwh-reader +vector_db: + host: vector-db + database: postgres + schema: vectors + user: vector-user + password: vector-password +vector_rest: + base_url: https://vectors.example.test/ + api_key: vector-reader +vector_write_rest: + base_url: https://vectors.example.test/ + api_key: vector-writer +paths: + artifacts: build/artifacts + indexes: build/indexes + sessions: build/sessions +""" + ) + return path + + +def _write_new_workspace(tmp_path): + path = tmp_path / "new.yaml" + path.write_text( + """ +dwh: + type: thoth_rest + database: + database: analytics + schema: mart + endpoint: + base_url: https://dwh.example.test/ + api_key: dwh-reader +vectors: + type: thoth_vector_http + reader: + base_url: https://vectors.example.test/ + api_key: vector-reader + writer: + base_url: https://vectors.example.test/ + api_key: vector-writer + direct: + host: vector-db + database: postgres + schema: vectors + user: vector-user + password: vector-password +roots: + artifacts: build/artifacts + indexes: build/indexes + sessions: build/sessions +""" + ) + return path + + +def test_legacy_rest_workspace_equals_new_resource_schema(tmp_path, capsys): + with pytest.warns(FutureWarning, match="DEPRECATION") as warnings: + old = load_config(_write_old_workspace(tmp_path)) + captured = capsys.readouterr() + new = load_config(_write_new_workspace(tmp_path)) + + assert old.dwh.model_dump() == new.dwh.model_dump() + assert old.vectors.model_dump() == new.vectors.model_dump() + assert old.roots.model_dump() == new.roots.model_dump() + assert captured.out == "" + assert captured.err == "" + assert len(warnings) == 1 + + +def test_legacy_warning_does_not_contaminate_cli_json(tmp_path): + with pytest.warns(FutureWarning, match="DEPRECATION") as warnings: + result = CliRunner().invoke( + app, + ["session", "list", "--json", "-c", str(_write_old_workspace(tmp_path))], + ) + + assert result.exit_code == 0 + json.loads(result.stdout) + assert "DEPRECATION" not in result.stdout + assert result.stderr == "" + assert len(warnings) == 1 diff --git a/harness/tests/test_config_resources.py b/harness/tests/test_config_resources.py new file mode 100644 index 00000000..b606c590 --- /dev/null +++ b/harness/tests/test_config_resources.py @@ -0,0 +1,71 @@ +from tht.config import ( + PgvectorDirectConfig, + PostgresDwhConfig, + ThothRestDwhConfig, + ThothVectorHttpConfig, + load_config, +) + + +def test_loads_discriminated_dwh_and_vector_resources(tmp_path): + workspace = tmp_path / "workspace.yaml" + workspace.write_text( + """ +dwh: + type: thoth_rest + database: + database: analytics + schema: mart + endpoint: + base_url: https://dwh.example.test/ + api_key: dwh-reader +vectors: + type: thoth_vector_http + reader: + base_url: https://vectors.example.test/ + api_key: vector-reader + writer: + base_url: https://vectors.example.test/ + api_key: vector-writer +roots: + artifacts: build/artifacts + indexes: build/indexes + sessions: build/sessions +""" + ) + + cfg = load_config(workspace) + + assert isinstance(cfg.dwh, ThothRestDwhConfig) + assert cfg.dwh.database.db_schema == "mart" + assert isinstance(cfg.vectors, ThothVectorHttpConfig) + assert cfg.vectors.writer.api_key == "vector-writer" + assert cfg.roots.sessions.as_posix() == "build/sessions" + + +def test_loads_direct_discriminated_resources(tmp_path): + workspace = tmp_path / "workspace.yaml" + workspace.write_text( + """ +dwh: + type: postgres_direct + connection: &database + host: db + database: analytics + schema: mart + user: reader + password: secret +vectors: + type: pgvector_direct + connection: + <<: *database + schema: vectors +""" + ) + + cfg = load_config(workspace) + + assert isinstance(cfg.dwh, PostgresDwhConfig) + assert cfg.database.transport == "direct" + assert isinstance(cfg.vectors, PgvectorDirectConfig) + assert cfg.vector_db.db_schema == "vectors" diff --git a/harness/tht/config.py b/harness/tht/config.py index 58f2f277..73ff727a 100644 --- a/harness/tht/config.py +++ b/harness/tht/config.py @@ -1,10 +1,13 @@ import os import re +import warnings from pathlib import Path -from typing import Any, Literal +from typing import Annotated, Any, Literal import yaml -from pydantic import BaseModel, Field, ValidationError +from pydantic import BaseModel, Field, model_validator, ValidationError + +from tht.config_compat import translate_legacy_config _ENV_RE = re.compile(r"\$\{([A-Za-z_][A-Za-z0-9_]*)\}") @@ -56,12 +59,59 @@ class RestConfig(BaseModel): ssl_ca: str | None = None # path al certificato CA (per server con CA interna) +class DatabaseIdentityConfig(BaseModel): + database: str + db_schema: str = Field(alias="schema") + + model_config = {"populate_by_name": True} + + +class PostgresDwhConfig(BaseModel): + type: Literal["postgres_direct"] + connection: DatabaseConfig + + +class ThothRestDwhConfig(BaseModel): + type: Literal["thoth_rest"] + database: DatabaseIdentityConfig + endpoint: RestConfig + + +DwhResourceConfig = Annotated[ + PostgresDwhConfig | ThothRestDwhConfig, + Field(discriminator="type"), +] + + +class PgvectorDirectConfig(BaseModel): + type: Literal["pgvector_direct"] + connection: DatabaseConfig + + +class ThothVectorHttpConfig(BaseModel): + type: Literal["thoth_vector_http"] + reader: RestConfig | None = None + writer: RestConfig | None = None + # Transitional direct loading path used by the server profile. + direct: DatabaseConfig | None = None + + +VectorResourceConfig = Annotated[ + PgvectorDirectConfig | ThothVectorHttpConfig, + Field(discriminator="type"), +] + + class PathsConfig(BaseModel): artifacts: Path = Path("artifacts") indexes: Path = Path("indexes") sessions: Path = Path("sessions") +class WorkspaceRoots(PathsConfig): + pass + + class ExamplesConfig(BaseModel): max_per_column: int = 10 @@ -138,6 +188,10 @@ class ExecutionConfig(BaseModel): class Config(BaseModel): + dwh: DwhResourceConfig + vectors: VectorResourceConfig | None = None + roots: WorkspaceRoots = WorkspaceRoots() + # Compatibility views retained until all call sites consume typed resources. database: DatabaseConfig # Profilo dell'installazione, letto da THT_PROFILE (.env), non dallo yaml versionato. # server: ricostruisce i derivati (artefatti, LSH, vettori schema nel vectordb). @@ -166,6 +220,15 @@ class Config(BaseModel): # una API key separata dalla lettura; espone solo upsert/hash via RPC allowlist. vector_write_rest: RestConfig | None = None + @model_validator(mode="before") + @classmethod + def accept_legacy_constructor_fields(cls, value: Any) -> Any: + if not isinstance(value, dict) or "dwh" in value: + return value + translated, _ = translate_legacy_config(value) + _populate_legacy_views(translated) + return translated + def load_config(path: Path) -> Config: if not path.exists(): @@ -173,8 +236,11 @@ def load_config(path: Path) -> Config: raw = yaml.safe_load(path.read_text()) if not isinstance(raw, dict): raise ConfigError(f"Configurazione non valida (atteso un mapping YAML): {path}") + expanded = _expand_env(raw) + translated, used_legacy = translate_legacy_config(expanded) + _populate_legacy_views(translated) try: - cfg = Config.model_validate(_expand_env(raw)) + cfg = Config.model_validate(translated) except ValidationError as e: raise ConfigError(f"Configurazione non valida in {path}:\n{e}") from e env_profile = os.environ.get("THT_PROFILE") @@ -188,4 +254,37 @@ def load_config(path: Path) -> Config: raise ConfigError( f"transport: rest richiede la sezione `rest` (base_url, api_key) in {path}." ) + if used_legacy: + warnings.warn( + "DEPRECATION: legacy workspace resource keys are deprecated; " + "use dwh, vectors, and roots.", + FutureWarning, + stacklevel=2, + ) return cfg + + +def _populate_legacy_views(raw: dict[str, Any]) -> None: + """Populate old Config attributes for command compatibility during migration.""" + dwh = raw.get("dwh") + if "database" not in raw and isinstance(dwh, dict): + if dwh.get("type") == "postgres_direct": + raw["database"] = {**dwh["connection"], "transport": "direct"} + elif dwh.get("type") == "thoth_rest": + raw["database"] = { + **dwh["database"], + "user": "rest", + "password": "", + "transport": "rest", + } + raw["rest"] = dwh["endpoint"] + + vectors = raw.get("vectors") + if isinstance(vectors, dict): + if vectors.get("type") == "pgvector_direct": + raw.setdefault("vector_db", vectors["connection"]) + elif vectors.get("type") == "thoth_vector_http": + raw.setdefault("vector_rest", vectors.get("reader")) + raw.setdefault("vector_write_rest", vectors.get("writer")) + raw.setdefault("vector_db", vectors.get("direct")) + raw.setdefault("paths", raw.get("roots", {})) diff --git a/harness/tht/config_compat.py b/harness/tht/config_compat.py new file mode 100644 index 00000000..b0276e8e --- /dev/null +++ b/harness/tht/config_compat.py @@ -0,0 +1,74 @@ +from __future__ import annotations + +from copy import deepcopy +from typing import Any + + +_LEGACY_RESOURCE_KEYS = { + "database", + "rest", + "vector_db", + "vector_rest", + "vector_write_rest", + "paths", +} + + +def _as_mapping(value: Any) -> dict[str, Any] | None: + if isinstance(value, dict): + return deepcopy(value) + model_dump = getattr(value, "model_dump", None) + if callable(model_dump): + return model_dump(by_alias=True) + return None + + +def translate_legacy_config(raw: dict[str, Any]) -> tuple[dict[str, Any], bool]: + """Translate the legacy flat resource keys without validating their contents.""" + translated = deepcopy(raw) + legacy = any(key in raw for key in _LEGACY_RESOURCE_KEYS) + if not legacy: + return translated, False + + database = _as_mapping(raw.get("database")) + rest = raw.get("rest") + if "dwh" not in translated and database is not None: + if database.get("transport", "direct") == "rest": + identity = { + key: database[key] + for key in ("database", "schema") + if key in database + } + translated["dwh"] = { + "type": "thoth_rest", + "database": identity, + "endpoint": rest, + } + else: + connection = database + connection.pop("transport", None) + translated["dwh"] = { + "type": "postgres_direct", + "connection": connection, + } + + if "vectors" not in translated: + vector_db = raw.get("vector_db") + reader = raw.get("vector_rest") + writer = raw.get("vector_write_rest") + if reader is not None or writer is not None: + translated["vectors"] = { + "type": "thoth_vector_http", + "reader": reader, + "writer": writer, + "direct": vector_db, + } + elif vector_db is not None: + translated["vectors"] = { + "type": "pgvector_direct", + "connection": vector_db, + } + + if "roots" not in translated and "paths" in raw: + translated["roots"] = deepcopy(raw["paths"]) + return translated, True diff --git a/harness/workspaces/tht.example.yaml b/harness/workspaces/tht.example.yaml index b55c8664..1a3438fe 100644 --- a/harness/workspaces/tht.example.yaml +++ b/harness/workspaces/tht.example.yaml @@ -1,26 +1,19 @@ # Workspace ThothII (esempio). I segreti vivono SOLO in .env (${THT_*}). -# La struttura rispecchia esattamente tht/config.py: -# database + rest per il DWH; vector_rest/vector_write_rest per il pgvector (doppia key); -# vector_db per il loading diretto (server-only); embeddings + evidence + execution. +# Ogni risorsa dichiara il proprio adapter tramite `type`. language: it # descrizioni tabelle/colonne ed evidence sono in italiano (PSD) -database: - host: ${THT_DB_HOST} - port: ${THT_DB_PORT} # es. 5437 (Postgres diretto Supabase; 5432 = pooler) - database: ${THT_DB_NAME} # es. postgres (lo schema a stella vive in `datawarehouse`) - schema: datawarehouse - user: ${THT_DB_USER} - password: ${THT_DB_PASSWORD} - transport: rest # direct (Postgres) | rest (Supabase/PostgREST) +dwh: + type: thoth_rest # postgres_direct | thoth_rest + database: + database: ${THT_DB_NAME} + schema: datawarehouse + endpoint: + base_url: ${THT_DWH_REST_URL} + api_key: ${THT_DWH_API_KEY} + ssl_ca: ${THT_SSL_CA} -# Accesso al DWH via REST (richiesto se database.transport = rest). -rest: - base_url: ${THT_DWH_REST_URL} # es. https://supabase-aritmolab.policlinicosandonato.it/dwh/ - api_key: ${THT_DWH_API_KEY} # header X-API-Key, ruolo dwh_reader (read-only) - ssl_ca: ${THT_SSL_CA} # path al certificato CA (per server con CA interna) - -paths: +roots: artifacts: artifacts indexes: indexes sessions: sessions @@ -50,30 +43,23 @@ embeddings: dim: 768 batch_size: 32 -# LOADING del pgvector: connessione diretta, eseguita sul server (profilo server). -# Opzionale su postazione remota (lì la lettura passa da vector_rest). -vector_db: - host: ${THT_VEC_HOST} # Postgres locale del server - port: ${THT_VEC_PORT} # es. 5437 - database: postgres - schema: vectors - user: ${THT_VEC_USER} - password: ${THT_VEC_PASSWORD} - -# LETTURA (similarity search) del pgvector via REST remota: rpc search_similar. -vector_rest: - base_url: ${THT_VEC_REST_URL} # es. https://host/vector/v1/ - api_key: ${THT_VEC_API_KEY} # header X-API-Key, ruolo vector_reader (read-only) - ssl_ca: ${THT_SSL_CA} - -# SCRITTURA controllata del pgvector via REST remota: upsert/hash via RPC allowlist, -# niente delete/clear. Usa una API key SEPARATA dalla lettura (ruolo vector_writer). -# OPZIONALE: assente o key vuota = scrittura non abilitata (solo lettura). -# Abilita tht memory save-one / vector index-schema da postazione remota. -vector_write_rest: - base_url: ${THT_VEC_REST_URL} - api_key: ${THT_VEC_WRITE_API_KEY} - ssl_ca: ${THT_SSL_CA} +vectors: + type: thoth_vector_http # pgvector_direct | thoth_vector_http + reader: + base_url: ${THT_VEC_REST_URL} + api_key: ${THT_VEC_API_KEY} + ssl_ca: ${THT_SSL_CA} + writer: # opzionale: credenziale separata dalla lettura + base_url: ${THT_VEC_REST_URL} + api_key: ${THT_VEC_WRITE_API_KEY} + ssl_ca: ${THT_SSL_CA} + direct: # opzionale: loading server-side diretto + host: ${THT_VEC_HOST} + port: ${THT_VEC_PORT} + database: postgres + schema: vectors + user: ${THT_VEC_USER} + password: ${THT_VEC_PASSWORD} vector: max_chunk_chars: 4000