mptyl/ThothII
1
0
Fork 0
Code Issues 25 Pull Requests Actions Packages Projects Releases 1 Wiki Activity

feat: index semantic records in qdrant

Browse Source
This commit is contained in:
marcopan
2026-08-08 18:03:57 +02:00
parent f61648fb69
commit 5e39cfa347
13 changed files with 516 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files
harness/tht/adapters/factory.py
+9 -1
View File
@@ -3,7 +3,7 @@
from tht.adapters.dwh import PostgresDwhAdapter, ThothRestDwhAdapter
from tht.adapters.evidence import FilesystemEvidenceSource, HttpManifestEvidenceSource
from tht.adapters.evidence.s3 import S3EvidenceSource
from tht.adapters.vector import PgVectorStore, ThothHttpVectorStore
from tht.adapters.vector import PgVectorStore, QdrantVectorStore, ThothHttpVectorStore
from tht.config import Config, ConfigError
from tht.db.connection import make_engine
from tht.ports.dwh import DwhAdapter
@@ -56,6 +56,14 @@ def build_vector_store(cfg: Config, *, require_write: bool = False) -> VectorSto
VectorRestClient(resource.writer) if resource.writer is not None else None,
expected_dimension=cfg.embeddings.dim if cfg.embeddings is not None else None,
)
case "qdrant":
return QdrantVectorStore(
base_url=resource.base_url,
collection=resource.collection,
workspace_id=cfg._workspace_id,
workspace_revision=cfg._workspace_revision,
expected_dimension=cfg.embeddings.dim if cfg.embeddings is not None else None,
)
case other: # pragma: no cover - Pydantic's discriminator rejects this first.
raise ConfigError(f"Adapter vector non supportato: {other}")
harness/tht/adapters/vector/qdrant.py
+4
View File
@@ -32,6 +32,7 @@ _KEYWORD_INDEXES = (
"record_kind",
"vector_generation",
"workspace_id",
"workspace_revision",
)
@@ -52,6 +53,7 @@ class QdrantVectorStore:
base_url: str,
collection: str,
workspace_id: str,
workspace_revision: str | None = None,
expected_dimension: int | None = None,
request: Callable[..., object] | None = None,
connect_timeout: float = 2.0,
@@ -60,6 +62,7 @@ class QdrantVectorStore:
self._base_url = base_url.rstrip("/")
self._collection = collection
self._workspace_id = workspace_id
self._workspace_revision = workspace_revision
self._expected_dimension = expected_dimension
self._request = request or requests.request
self._timeout = (connect_timeout, read_timeout)
@@ -198,6 +201,7 @@ class QdrantVectorStore:
write_record.record,
content_hash=write_record.content_hash,
workspace_id=self._workspace_id,
workspace_revision=self._workspace_revision,
),
}
)
harness/tht/cli/memory_cmd.py
+16 -8
View File
@@ -10,17 +10,18 @@ from pathlib import Path
import typer
from sqlalchemy.exc import OperationalError, ProgrammingError
from tht.cli.config_cmd import CONFIG_OPT
from tht.cli.schema_cmd import _load_config_or_exit
from tht.cli._guards import (
has_vector_write_rest,
require_server_profile,
require_vector_write_allowed,
)
from tht.cli.config_cmd import CONFIG_OPT
from tht.cli.schema_cmd import _load_config_or_exit
from tht.cli.session_cmd import load_snapshot_or_exit
from tht.cli.vector_cmd import require_vector_cfg
memory_app = typer.Typer(help="Review memory (registro canonico + indice pgvector)")
DECISION_OPT = typer.Option(None, "--decision", help="Seq da promuovere (ripetibile).")
def registry_path(cfg) -> Path:
@@ -29,18 +30,23 @@ def registry_path(cfg) -> Path:
def _resync_memory(cfg):
"""Risincronizza l'indice pgvector col registro corrente (incrementale)."""
from tht.cli.vector_cmd import make_embedder, open_store
from tht.adapters.factory import build_vector_store
from tht.cli.vector_cmd import make_embedder, sync_canonical_records
from tht.memory import load_registry, memory_vector_records
records = memory_vector_records(load_registry(registry_path(cfg)))
store = open_store(cfg, "memory")
return store.sync(records, make_embedder(cfg.embeddings), kinds={"memory"})
return sync_canonical_records(
"memory",
records,
store=build_vector_store(cfg, require_write=True),
embedder=make_embedder(cfg.embeddings),
)
@memory_app.command("promote")
def promote_cmd(
session: str = typer.Option(..., "--session"),
decision: list[int] = typer.Option(None, "--decision", help="Seq da promuovere (ripetibile)."),
decision: list[int] = DECISION_OPT,
preview: bool = typer.Option(False, "--preview", help="Mostra i candidati in JSON, non scrive."),
json_out: bool = typer.Option(False, "--json", help="Output JSON (per Pi)."),
config: Path = CONFIG_OPT,
@@ -55,7 +61,9 @@ def promote_cmd(
if preview:
from tht.memory import (
MAX_PROMOTION_CANDIDATES, preview_promotions_snapshot, reusable_promotions_snapshot,
MAX_PROMOTION_CANDIDATES,
preview_promotions_snapshot,
reusable_promotions_snapshot,
)
cand = preview_promotions_snapshot(snapshot, registry_path(cfg))
extra = len(reusable_promotions_snapshot(snapshot, registry_path(cfg))) - len(cand)
@@ -304,8 +312,8 @@ def update_cmd(
"""Modifica i campi di merito di una memoria (provenienza immutabile)."""
from typing import get_args
from tht.memory import MemoryNotFound, update_record
from tht.decisions import DecisionType
from tht.memory import MemoryNotFound, update_record
cfg = _load_config_or_exit(config)
harness/tht/cli/vector_cmd.py
+39 -5
View File
@@ -2,9 +2,15 @@ from pathlib import Path
import typer
from tht.cli._guards import has_vector_write_rest, require_server_profile, require_vector_write_allowed
from tht.cli._guards import (
has_vector_write_rest,
require_server_profile,
require_vector_write_allowed,
)
from tht.cli.config_cmd import CONFIG_OPT
from tht.cli.schema_cmd import _load_config_or_exit, annotations_path, physical_path
from tht.ports.vector import VectorWriteRecord
from tht.vectorstore.store import SyncStats, content_hash
vector_app = typer.Typer(help="Indice semantico pgvector (derivato, rigenerabile)")
@@ -69,6 +75,31 @@ def open_searcher(cfg):
return AdapterSearcher()
def sync_canonical_records(collection, records, *, store, embedder):
kinds = sorted({record.kind for record in records})
existing = store.existing_hashes(collection, kinds)
pending = []
stats = SyncStats()
changed = []
for record in records:
hashed = content_hash(record.content)
current = existing.get(record.id)
if current == hashed:
stats.unchanged += 1
continue
changed.append((record, hashed, current is None))
if changed:
embeddings = embedder.embed_documents([record.content for record, *_ in changed])
for (record, hashed, is_added), embedding in zip(changed, embeddings, strict=True):
pending.append(VectorWriteRecord(record=record, embedding=embedding, content_hash=hashed))
if is_added:
stats.added += 1
else:
stats.updated += 1
store.upsert(collection, pending)
return stats
def _print_stats(stats) -> None:
typer.secho(
f"OK: {stats.added} nuovi, {stats.updated} aggiornati, "
@@ -88,7 +119,6 @@ def init_cmd(
from sqlalchemy.exc import OperationalError
from tht.vectorstore.embeddings import EmbeddingsError
from tht.vectorstore.reader import ALL_TABLES
cfg = _load_config_or_exit(config)
@@ -131,8 +161,12 @@ def index_schema_cmd(config: Path = CONFIG_OPT) -> None:
physical = PhysicalSchema.from_yaml(phys_file)
annotations = Annotations.from_yaml(annotations_path(cfg))
records = schema_records(physical, annotations)
store = open_store(cfg, "schema_records")
stats = store.sync(
records, make_embedder(cfg.embeddings), kinds={"schema_table", "schema_column"}
from tht.adapters.factory import build_vector_store
stats = sync_canonical_records(
"schema_records",
records,
store=build_vector_store(cfg, require_write=True),
embedder=make_embedder(cfg.embeddings),
)
_print_stats(stats)
harness/tht/config.py
+78 -1
View File
@@ -152,8 +152,14 @@ class ThothVectorHttpConfig(BaseModel):
direct: DatabaseConfig | None = None
class QdrantConfig(BaseModel):
type: Literal["qdrant"]
base_url: str
collection: str = Field(min_length=1)
VectorResourceConfig = Annotated[
PgvectorDirectConfig | ThothVectorHttpConfig,
PgvectorDirectConfig | ThothVectorHttpConfig | QdrantConfig,
Field(discriminator="type"),
]
@@ -397,6 +403,7 @@ def load_config(path: Path) -> Config:
raise ConfigError(f"Configurazione non valida (atteso un mapping YAML): {path}")
expanded = _resolve_secret_files(_expand_env(raw))
_validate_internal_embedding_contract(expanded, path)
_validate_internal_vector_contract(expanded, path)
translated, used_legacy = translate_legacy_config(expanded)
_populate_legacy_views(translated)
try:
@@ -455,6 +462,7 @@ def load_config(path: Path) -> Config:
else path.resolve().as_posix()
)
_validate_active_embeddings_config(cfg.embeddings, path)
_validate_active_vector_config(cfg.vectors, path)
return cfg
@@ -500,6 +508,42 @@ def _validate_internal_embedding_contract(raw: dict[str, Any], path: Path) -> No
)
def _validate_internal_vector_contract(raw: dict[str, Any], path: Path) -> None:
resources = raw.get("resources")
if not isinstance(resources, dict):
return
vector = resources.get("vector")
if not isinstance(vector, dict):
return
engine = vector.get("engine")
base_url = vector.get("base_url")
collection = vector.get("collection")
allowed = {"engine", "base_url", "collection"}
unexpected = sorted(set(vector) - allowed)
if unexpected:
raise ConfigError(
f"Configurazione non valida in {path}:\n"
f"resources.vector non supporta: {', '.join(unexpected)}"
)
if engine != "qdrant":
raise ConfigError(
f"Configurazione non valida in {path}:\n"
"resources.vector.engine deve essere 'qdrant'"
)
if not isinstance(collection, str) or not collection:
raise ConfigError(
f"Configurazione non valida in {path}:\n"
"resources.vector.collection deve essere valorizzato"
)
if not _is_allowed_internal_qdrant_url(base_url):
raise ConfigError(
f"Configurazione non valida in {path}:\n"
"resources.vector.base_url deve usare http://qdrant:6333 "
"oppure un endpoint loopback di sviluppo su porta 6333"
)
def _validate_active_embeddings_config(
embeddings: "EmbeddingsConfig | None",
path: Path,
@@ -529,6 +573,20 @@ def _validate_active_embeddings_config(
)
def _validate_active_vector_config(
vectors: "VectorResourceConfig | None",
path: Path,
) -> None:
if vectors is None or vectors.type != "qdrant":
return
if not _is_allowed_internal_qdrant_url(vectors.base_url):
raise ConfigError(
f"Configurazione non valida in {path}:\n"
"vectors.base_url deve usare http://qdrant:6333 "
"oppure un endpoint loopback di sviluppo su porta 6333"
)
def _is_allowed_internal_embedding_url(value: Any) -> bool:
if not isinstance(value, str):
return False
@@ -548,6 +606,25 @@ def _is_allowed_internal_embedding_url(value: Any) -> bool:
return host.is_loopback
def _is_allowed_internal_qdrant_url(value: Any) -> bool:
if not isinstance(value, str):
return False
parsed = urlparse(value)
if parsed.scheme != "http" or not parsed.hostname or parsed.port != 6333:
return False
if parsed.params or parsed.query or parsed.fragment:
return False
if parsed.path not in ("", "/"):
return False
if parsed.hostname == "qdrant":
return True
try:
host = ip_address(parsed.hostname)
except ValueError:
return parsed.hostname == "localhost"
return host.is_loopback
def _populate_legacy_views(raw: dict[str, Any]) -> None:
"""Populate old Config attributes for command compatibility during migration."""
dwh = raw.get("dwh")
harness/tht/config_compat.py
+8
View File
@@ -33,6 +33,14 @@ def translate_legacy_config(raw: dict[str, Any]) -> tuple[dict[str, Any], bool]:
translated["embeddings"] = embedding
if "dimensions" in translated["embeddings"] and "dim" not in translated["embeddings"]:
translated["embeddings"]["dim"] = translated["embeddings"].pop("dimensions")
if isinstance(resources, dict) and "vector" in resources and "vectors" not in translated:
vector = _as_mapping(resources.get("vector"))
if isinstance(vector, dict):
translated["vectors"] = {
"type": "qdrant",
"base_url": vector.get("base_url"),
"collection": vector.get("collection"),
}
legacy = any(key in raw for key in _LEGACY_RESOURCE_KEYS)
if not legacy:
return translated, False
harness/tht/vectorstore/records.py
+8 -1
View File
@@ -27,11 +27,18 @@ def qdrant_semantic_kind(kind: str) -> str:
raise ValueError(f"Unsupported vector kind: {kind}")
def qdrant_payload(record: VectorRecord, *, content_hash: str, workspace_id: str) -> dict:
def qdrant_payload(
record: VectorRecord,
*,
content_hash: str,
workspace_id: str,
workspace_revision: str | None = None,
) -> dict:
semantic_kind = qdrant_semantic_kind(record.kind)
return {
**record.metadata,
"workspace_id": workspace_id,
**({"workspace_revision": workspace_revision} if workspace_revision else {}),
"kind": semantic_kind,
"record_kind": record.kind,
"record_key": record.id,