feat(vector): add direct pgvector adapter
This commit is contained in:
+30
-9
@@ -18,6 +18,7 @@ class ConfigError(Exception):
|
||||
|
||||
def _expand_env(value: Any) -> Any:
|
||||
if isinstance(value, str):
|
||||
|
||||
def repl(m: re.Match) -> str:
|
||||
var = m.group(1)
|
||||
if var not in os.environ:
|
||||
@@ -85,7 +86,16 @@ DwhResourceConfig = Annotated[
|
||||
|
||||
class PgvectorDirectConfig(BaseModel):
|
||||
type: Literal["pgvector_direct"]
|
||||
connection: DatabaseConfig
|
||||
reader: DatabaseConfig | None = None
|
||||
writer: DatabaseConfig | None = None
|
||||
# Deprecated compatibility: a single direct connection historically meant read-only.
|
||||
connection: DatabaseConfig | None = None
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_connections(self):
|
||||
if self.reader is None and self.writer is None and self.connection is None:
|
||||
raise ValueError("pgvector_direct requires a reader or writer connection")
|
||||
return self
|
||||
|
||||
|
||||
class ThothVectorHttpConfig(BaseModel):
|
||||
@@ -134,9 +144,9 @@ class LshConfig(BaseModel):
|
||||
|
||||
class EligibilityConfig(BaseModel):
|
||||
# Soglie del principio di column eligibility (testo ampio ignorato ovunque).
|
||||
max_declared_len: int = 128 # char/varchar dichiarati <= soglia: eligible senza campionare
|
||||
max_avg_length: int = 40 # fallback data-driven: lunghezza media valori campionati
|
||||
max_sampled_len: int = 200 # fallback data-driven: lunghezza massima valore campionato
|
||||
max_declared_len: int = 128 # char/varchar dichiarati <= soglia: eligible senza campionare
|
||||
max_avg_length: int = 40 # fallback data-driven: lunghezza media valori campionati
|
||||
max_sampled_len: int = 200 # fallback data-driven: lunghezza massima valore campionato
|
||||
# Colonne di servizio sempre ignorate per nome (match case-insensitive), a prescindere
|
||||
# dal tipo: metadati ETL/audit non analitici (es. timestamp di ultimo aggiornamento).
|
||||
ignore_columns: list[str] = ["etl_last_update"]
|
||||
@@ -168,7 +178,7 @@ class VectorConfig(BaseModel):
|
||||
|
||||
class SearchConfig(BaseModel):
|
||||
rrf_k: int = 60
|
||||
top_schema_tables: int = 12 # default `--top` per `tht search --kind schema` (n. tabelle)
|
||||
top_schema_tables: int = 12 # default `--top` per `tht search --kind schema` (n. tabelle)
|
||||
schema_chunk_pool: int = 150 # chunk tabella/colonna fusi prima dell'aggregazione a tabella
|
||||
|
||||
|
||||
@@ -181,9 +191,17 @@ class ExecutionConfig(BaseModel):
|
||||
max_aggregate_cells: int = 20
|
||||
max_export_rows: int = 100000
|
||||
forbidden_functions: list[str] = [
|
||||
"setval", "nextval", "pg_advisory_lock", "pg_advisory_xact_lock",
|
||||
"dblink", "dblink_exec", "pg_terminate_backend", "pg_cancel_backend",
|
||||
"lo_import", "lo_export", "pg_reload_conf",
|
||||
"setval",
|
||||
"nextval",
|
||||
"pg_advisory_lock",
|
||||
"pg_advisory_xact_lock",
|
||||
"dblink",
|
||||
"dblink_exec",
|
||||
"pg_terminate_backend",
|
||||
"pg_cancel_backend",
|
||||
"lo_import",
|
||||
"lo_export",
|
||||
"pg_reload_conf",
|
||||
]
|
||||
|
||||
|
||||
@@ -302,7 +320,10 @@ def _populate_legacy_views(raw: dict[str, Any]) -> None:
|
||||
vectors = raw.get("vectors")
|
||||
if isinstance(vectors, dict):
|
||||
if vectors.get("type") == "pgvector_direct":
|
||||
raw.setdefault("vector_db", vectors["connection"])
|
||||
raw.setdefault(
|
||||
"vector_db",
|
||||
vectors.get("writer") or vectors.get("reader") or vectors.get("connection"),
|
||||
)
|
||||
elif vectors.get("type") == "thoth_vector_http":
|
||||
raw.setdefault("vector_rest", vectors.get("reader"))
|
||||
raw.setdefault("vector_write_rest", vectors.get("writer"))
|
||||
|
||||
Reference in New Issue
Block a user