feat: index semantic records in qdrant
This commit is contained in:
@@ -1,8 +1,8 @@
|
|||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from tht.adapters.dwh import PostgresDwhAdapter, ThothRestDwhAdapter
|
from tht.adapters.dwh import PostgresDwhAdapter, ThothRestDwhAdapter
|
||||||
from tht.adapters.vector import PgVectorStore, ThothHttpVectorStore
|
|
||||||
from tht.adapters.factory import build_dwh, build_vector_store
|
from tht.adapters.factory import build_dwh, build_vector_store
|
||||||
|
from tht.adapters.vector import PgVectorStore, QdrantVectorStore, ThothHttpVectorStore
|
||||||
from tht.config import Config, ConfigError
|
from tht.config import Config, ConfigError
|
||||||
|
|
||||||
|
|
||||||
@@ -125,6 +125,50 @@ def test_factory_builds_writer_only_direct_vector_when_write_is_required():
|
|||||||
assert store.capabilities.upsert is True
|
assert store.capabilities.upsert is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_factory_selects_qdrant_for_schema_v3_runtime():
|
||||||
|
config = Config.model_validate(
|
||||||
|
{
|
||||||
|
"dwh": {
|
||||||
|
"type": "postgres_direct",
|
||||||
|
"connection": {
|
||||||
|
"host": "db",
|
||||||
|
"database": "analytics",
|
||||||
|
"schema": "mart",
|
||||||
|
"user": "reader",
|
||||||
|
"password": "secret",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"database": {
|
||||||
|
"host": "db",
|
||||||
|
"database": "analytics",
|
||||||
|
"schema": "mart",
|
||||||
|
"user": "reader",
|
||||||
|
"password": "secret",
|
||||||
|
"transport": "direct",
|
||||||
|
},
|
||||||
|
"vectors": {
|
||||||
|
"type": "qdrant",
|
||||||
|
"base_url": "http://qdrant:6333",
|
||||||
|
"collection": "psd-clinical",
|
||||||
|
},
|
||||||
|
"embeddings": {
|
||||||
|
"provider": "ollama_internal",
|
||||||
|
"base_url": "http://embedding:11434",
|
||||||
|
"model": "qwen3-embedding:0.6b",
|
||||||
|
"dim": 1024,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
)
|
||||||
|
config._workspace_id = "psd-clinical"
|
||||||
|
config._workspace_revision = "a" * 40
|
||||||
|
|
||||||
|
store = build_vector_store(config, require_write=True)
|
||||||
|
|
||||||
|
assert isinstance(store, QdrantVectorStore)
|
||||||
|
assert store.capabilities.search is True
|
||||||
|
assert store.capabilities.upsert is True
|
||||||
|
|
||||||
|
|
||||||
def test_factory_reuses_legacy_direct_connection_for_server_writes_only():
|
def test_factory_reuses_legacy_direct_connection_for_server_writes_only():
|
||||||
server = _config(vector_type="pgvector_direct", writer=False)
|
server = _config(vector_type="pgvector_direct", writer=False)
|
||||||
server.vectors.connection = server.vectors.reader
|
server.vectors.connection = server.vectors.reader
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ from tht.config import (
|
|||||||
ConfigError,
|
ConfigError,
|
||||||
PgvectorDirectConfig,
|
PgvectorDirectConfig,
|
||||||
PostgresDwhConfig,
|
PostgresDwhConfig,
|
||||||
|
QdrantConfig,
|
||||||
ThothRestDwhConfig,
|
ThothRestDwhConfig,
|
||||||
ThothVectorHttpConfig,
|
ThothVectorHttpConfig,
|
||||||
load_config,
|
load_config,
|
||||||
@@ -237,6 +238,33 @@ resources:
|
|||||||
assert cfg.embeddings.dim == 1024
|
assert cfg.embeddings.dim == 1024
|
||||||
|
|
||||||
|
|
||||||
|
def test_accepts_internal_qdrant_resource_contract(tmp_path):
|
||||||
|
workspace = tmp_path / "workspace.yaml"
|
||||||
|
workspace.write_text(
|
||||||
|
"""
|
||||||
|
dwh:
|
||||||
|
type: postgres_direct
|
||||||
|
connection: {database: analytics, schema: mart, user: reader, password: secret}
|
||||||
|
resources:
|
||||||
|
vector:
|
||||||
|
engine: qdrant
|
||||||
|
base_url: http://qdrant:6333
|
||||||
|
collection: psd-clinical
|
||||||
|
embeddings:
|
||||||
|
provider: ollama_internal
|
||||||
|
base_url: http://embedding:11434
|
||||||
|
model: qwen3-embedding:0.6b
|
||||||
|
dimensions: 1024
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
|
||||||
|
cfg = load_config(workspace)
|
||||||
|
|
||||||
|
assert isinstance(cfg.vectors, QdrantConfig)
|
||||||
|
assert cfg.vectors.base_url == "http://qdrant:6333"
|
||||||
|
assert cfg.vectors.collection == "psd-clinical"
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize(
|
@pytest.mark.parametrize(
|
||||||
("snippet", "pattern"),
|
("snippet", "pattern"),
|
||||||
[
|
[
|
||||||
|
|||||||
@@ -7,19 +7,28 @@ pgvector as a one-row upsert. This test pins the pure core of that behavior:
|
|||||||
- the writer.upsert_records is called once with a single row
|
- the writer.upsert_records is called once with a single row
|
||||||
- writer.sync is NEVER called (that is the full-resync path)
|
- writer.sync is NEVER called (that is the full-resync path)
|
||||||
"""
|
"""
|
||||||
from datetime import datetime
|
from datetime import UTC, datetime
|
||||||
from unittest.mock import MagicMock
|
from unittest.mock import MagicMock
|
||||||
|
|
||||||
|
from tht.adapters.vector.qdrant import point_id
|
||||||
from tht.memory import MemoryRecord, memory_vector_record_for_decision, save_one_memory
|
from tht.memory import MemoryRecord, memory_vector_record_for_decision, save_one_memory
|
||||||
|
from tht.vectorstore.records import qdrant_payload
|
||||||
|
|
||||||
|
|
||||||
def _record(seq: int = 7, **kw) -> MemoryRecord:
|
def _record(seq: int = 7, **kw) -> MemoryRecord:
|
||||||
base = dict(
|
base = {
|
||||||
id="mem-0007", ts=datetime(2025, 1, 1), session_id="s1", decision_seq=seq,
|
"id": "mem-0007",
|
||||||
type="concept_clarified", subject="paziente attivo",
|
"ts": datetime(2025, 1, 1, tzinfo=UTC),
|
||||||
detail="flag_attivo = TRUE", rationale="r",
|
"session_id": "s1",
|
||||||
question_context="dammi i pazienti", tables=[], concepts=["paziente attivo"],
|
"decision_seq": seq,
|
||||||
)
|
"type": "concept_clarified",
|
||||||
|
"subject": "paziente attivo",
|
||||||
|
"detail": "flag_attivo = TRUE",
|
||||||
|
"rationale": "r",
|
||||||
|
"question_context": "dammi i pazienti",
|
||||||
|
"tables": [],
|
||||||
|
"concepts": ["paziente attivo"],
|
||||||
|
}
|
||||||
base.update(kw)
|
base.update(kw)
|
||||||
return MemoryRecord(**base)
|
return MemoryRecord(**base)
|
||||||
|
|
||||||
@@ -85,3 +94,29 @@ def test_save_one_uses_writer_key_for_upsert():
|
|||||||
save_one_memory(records, decision_seq=7, store=writer, embedder=embedder)
|
save_one_memory(records, decision_seq=7, store=writer, embedder=embedder)
|
||||||
# one upsert call, single row, table=memory
|
# one upsert call, single row, table=memory
|
||||||
assert writer.upsert.call_count == 1
|
assert writer.upsert.call_count == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_save_one_preserves_semantic_point_identity_fields():
|
||||||
|
records = [_record(seq=7)]
|
||||||
|
writer = MagicMock()
|
||||||
|
writer.existing_hashes.return_value = {}
|
||||||
|
writer.upsert.return_value = 1
|
||||||
|
embedder = MagicMock()
|
||||||
|
embedder.embed_documents.return_value = [[0.0] * 4]
|
||||||
|
|
||||||
|
save_one_memory(records, decision_seq=7, store=writer, embedder=embedder)
|
||||||
|
|
||||||
|
row = writer.upsert.call_args.args[1][0]
|
||||||
|
payload = qdrant_payload(
|
||||||
|
row.record,
|
||||||
|
content_hash=row.content_hash,
|
||||||
|
workspace_id="psd-clinical",
|
||||||
|
workspace_revision="a" * 40,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert point_id("psd-clinical", "memory", row.record.id) == point_id(
|
||||||
|
"psd-clinical", "memory", "memory:mem-0007"
|
||||||
|
)
|
||||||
|
assert payload["kind"] == "memory"
|
||||||
|
assert payload["workspace_id"] == "psd-clinical"
|
||||||
|
assert payload["workspace_revision"] == "a" * 40
|
||||||
|
|||||||
@@ -166,6 +166,7 @@ def _store(fake: FakeQdrantHttp) -> QdrantVectorStore:
|
|||||||
base_url="http://qdrant:6333",
|
base_url="http://qdrant:6333",
|
||||||
collection="workspace-semantic",
|
collection="workspace-semantic",
|
||||||
workspace_id="demo",
|
workspace_id="demo",
|
||||||
|
workspace_revision="a" * 40,
|
||||||
expected_dimension=1024,
|
expected_dimension=1024,
|
||||||
request=fake.request,
|
request=fake.request,
|
||||||
)
|
)
|
||||||
@@ -195,6 +196,7 @@ def test_upsert_creates_collection_and_keyword_indexes_idempotently():
|
|||||||
"record_kind",
|
"record_kind",
|
||||||
"vector_generation",
|
"vector_generation",
|
||||||
"workspace_id",
|
"workspace_id",
|
||||||
|
"workspace_revision",
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@@ -239,6 +241,7 @@ def test_upsert_serializes_qdrant_point_payloads(record, semantic_kind):
|
|||||||
assert point["id"] == point_id("demo", semantic_kind, record.record.id)
|
assert point["id"] == point_id("demo", semantic_kind, record.record.id)
|
||||||
assert point["vector"] == record.embedding
|
assert point["vector"] == record.embedding
|
||||||
assert point["payload"]["workspace_id"] == "demo"
|
assert point["payload"]["workspace_id"] == "demo"
|
||||||
|
assert point["payload"]["workspace_revision"] == "a" * 40
|
||||||
assert point["payload"]["kind"] == semantic_kind
|
assert point["payload"]["kind"] == semantic_kind
|
||||||
assert point["payload"]["record_kind"] == record.record.kind
|
assert point["payload"]["record_kind"] == record.record.kind
|
||||||
assert point["payload"]["record_key"] == record.record.id
|
assert point["payload"]["record_key"] == record.record.id
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
import json
|
import json
|
||||||
from datetime import datetime
|
from datetime import UTC, datetime
|
||||||
from types import SimpleNamespace
|
from types import SimpleNamespace
|
||||||
|
|
||||||
from typer.testing import CliRunner
|
from typer.testing import CliRunner
|
||||||
@@ -7,8 +7,8 @@ from typer.testing import CliRunner
|
|||||||
from tht.cli import app
|
from tht.cli import app
|
||||||
from tht.config import load_config
|
from tht.config import load_config
|
||||||
from tht.jobs.dwh_pipeline import DwhPreprocessPipeline, config_dwh_binding
|
from tht.jobs.dwh_pipeline import DwhPreprocessPipeline, config_dwh_binding
|
||||||
from tht.ports.vector import VectorReadUnavailable
|
|
||||||
from tht.mschema.models import ColumnPhysical, PhysicalSchema, TablePhysical
|
from tht.mschema.models import ColumnPhysical, PhysicalSchema, TablePhysical
|
||||||
|
from tht.ports.vector import VectorReadUnavailable
|
||||||
from tht.vectorstore.embeddings import EmbeddingsError
|
from tht.vectorstore.embeddings import EmbeddingsError
|
||||||
|
|
||||||
|
|
||||||
@@ -22,7 +22,11 @@ class _FakeEmbedder:
|
|||||||
|
|
||||||
|
|
||||||
class _FakeSearcher:
|
class _FakeSearcher:
|
||||||
|
def __init__(self):
|
||||||
|
self.calls = []
|
||||||
|
|
||||||
def search(self, vec, top_n, kinds=None):
|
def search(self, vec, top_n, kinds=None):
|
||||||
|
self.calls.append({"top_n": top_n, "kinds": kinds})
|
||||||
if kinds == ["solved_question"]:
|
if kinds == ["solved_question"]:
|
||||||
return [SimpleNamespace(
|
return [SimpleNamespace(
|
||||||
kind="memory", ref="s-1", id="m1", title="q solved",
|
kind="memory", ref="s-1", id="m1", title="q solved",
|
||||||
@@ -47,7 +51,7 @@ class _FakeSearcher:
|
|||||||
|
|
||||||
def _workspace(tmp_path, with_session=None):
|
def _workspace(tmp_path, with_session=None):
|
||||||
physical = PhysicalSchema(
|
physical = PhysicalSchema(
|
||||||
database="d", schema="s", introspected_at=datetime(2026, 1, 1),
|
database="d", schema="s", introspected_at=datetime(2026, 1, 1, tzinfo=UTC),
|
||||||
tables={"fact_ablazione": TablePhysical(
|
tables={"fact_ablazione": TablePhysical(
|
||||||
comment="Ablazioni", columns={"cod_paz": ColumnPhysical(type="bigint")})},
|
comment="Ablazioni", columns={"cod_paz": ColumnPhysical(type="bigint")})},
|
||||||
)
|
)
|
||||||
@@ -55,7 +59,7 @@ def _workspace(tmp_path, with_session=None):
|
|||||||
cfg.write_text(
|
cfg.write_text(
|
||||||
"database: {database: d, schema: s, user: u, password: p, transport: direct}\n"
|
"database: {database: d, schema: s, user: u, password: p, transport: direct}\n"
|
||||||
"vector_db: {database: v, schema: public, user: u, password: p}\n"
|
"vector_db: {database: v, schema: public, user: u, password: p}\n"
|
||||||
"embeddings: {base_url: 'http://localhost:11434', model: nomic-embed-text, dim: 8}\n"
|
"embeddings: {base_url: 'http://localhost:11434', model: qwen3-embedding:0.6b, dim: 1024}\n"
|
||||||
f"paths: {{artifacts: {tmp_path/'artifacts'}, indexes: {tmp_path/'i'}, "
|
f"paths: {{artifacts: {tmp_path/'artifacts'}, indexes: {tmp_path/'i'}, "
|
||||||
f"sessions: {tmp_path/'sessions'}}}\n"
|
f"sessions: {tmp_path/'sessions'}}}\n"
|
||||||
)
|
)
|
||||||
@@ -91,10 +95,15 @@ def _patch(monkeypatch, embedder, searcher):
|
|||||||
def test_pack_single_embed_and_sections(tmp_path, monkeypatch):
|
def test_pack_single_embed_and_sections(tmp_path, monkeypatch):
|
||||||
cfg = _workspace(tmp_path)
|
cfg = _workspace(tmp_path)
|
||||||
emb = _FakeEmbedder()
|
emb = _FakeEmbedder()
|
||||||
_patch(monkeypatch, emb, _FakeSearcher())
|
searcher = _FakeSearcher()
|
||||||
|
_patch(monkeypatch, emb, searcher)
|
||||||
res = CliRunner().invoke(app, ["search", "pack", "quanti pazienti", "-c", str(cfg)])
|
res = CliRunner().invoke(app, ["search", "pack", "quanti pazienti", "-c", str(cfg)])
|
||||||
assert res.exit_code == 0, res.output
|
assert res.exit_code == 0, res.output
|
||||||
assert emb.calls == 1 # UN solo embedding per le tre ricerche
|
assert emb.calls == 1 # UN solo embedding per le tre ricerche
|
||||||
|
assert [call["kinds"] for call in searcher.calls] == [
|
||||||
|
["schema_table", "schema_column"],
|
||||||
|
["solved_question"],
|
||||||
|
]
|
||||||
assert "fact_ablazione" in res.output and "Ablazioni" in res.output
|
assert "fact_ablazione" in res.output and "Ablazioni" in res.output
|
||||||
# Evidence is fail-closed until an ACTIVE corpus exists; legacy vector rows
|
# Evidence is fail-closed until an ACTIVE corpus exists; legacy vector rows
|
||||||
# must not leak into a new search pack.
|
# must not leak into a new search pack.
|
||||||
|
|||||||
@@ -0,0 +1,222 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
|
||||||
|
from tht.adapters.vector.qdrant import point_id
|
||||||
|
from tht.cli.vector_cmd import sync_canonical_records
|
||||||
|
from tht.corpus.chunk import ChunkPolicy
|
||||||
|
from tht.corpus.models import CanonicalChunk
|
||||||
|
from tht.corpus.pipeline import CorpusPipeline
|
||||||
|
from tht.corpus.store import CorpusStore
|
||||||
|
from tht.memory import MemoryRecord, save_one_memory
|
||||||
|
from tht.mschema.models import (
|
||||||
|
Annotations,
|
||||||
|
ColumnPhysical,
|
||||||
|
PhysicalSchema,
|
||||||
|
TablePhysical,
|
||||||
|
)
|
||||||
|
from tht.ports.vector import VectorCapabilities, VectorHealth
|
||||||
|
from tht.vectorstore.records import qdrant_payload, schema_records
|
||||||
|
|
||||||
|
|
||||||
|
def _sha(content: str) -> str:
|
||||||
|
return f"sha256:{hashlib.sha256(content.encode('utf-8')).hexdigest()}"
|
||||||
|
|
||||||
|
|
||||||
|
class _Embedder:
|
||||||
|
def embed_documents(self, documents):
|
||||||
|
return [[float(index + 1)] * 4 for index, _ in enumerate(documents)]
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class _Point:
|
||||||
|
point_id: str
|
||||||
|
payload: dict
|
||||||
|
embedding: list[float]
|
||||||
|
|
||||||
|
|
||||||
|
class FakeVectorStore:
|
||||||
|
def __init__(self, workspace_id="psd-clinical", workspace_revision=None):
|
||||||
|
self.workspace_id = workspace_id
|
||||||
|
self.workspace_revision = workspace_revision or "a" * 40
|
||||||
|
self.points: dict[str, _Point] = {}
|
||||||
|
self.search_calls: list[dict] = []
|
||||||
|
|
||||||
|
@property
|
||||||
|
def capabilities(self):
|
||||||
|
return VectorCapabilities(
|
||||||
|
search=True,
|
||||||
|
existing_hashes=True,
|
||||||
|
upsert=True,
|
||||||
|
metadata_filter=True,
|
||||||
|
delete_generation=True,
|
||||||
|
list_evidence_generations=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
def health(self):
|
||||||
|
return VectorHealth(ok=True)
|
||||||
|
|
||||||
|
def search(self, collections, embedding, *, limit, kinds=None, metadata_filter=None):
|
||||||
|
self.search_calls.append(
|
||||||
|
{
|
||||||
|
"collections": collections,
|
||||||
|
"embedding": embedding,
|
||||||
|
"limit": limit,
|
||||||
|
"kinds": kinds,
|
||||||
|
"metadata_filter": metadata_filter,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
return []
|
||||||
|
|
||||||
|
def existing_hashes(self, collection, kinds):
|
||||||
|
allowed = set(kinds)
|
||||||
|
return {
|
||||||
|
point.payload["record_key"]: point.payload["content_hash"]
|
||||||
|
for point in self.points.values()
|
||||||
|
if point.payload["record_kind"] in allowed
|
||||||
|
}
|
||||||
|
|
||||||
|
def upsert(self, collection, records):
|
||||||
|
for row in records:
|
||||||
|
semantic_kind = qdrant_payload(
|
||||||
|
row.record,
|
||||||
|
content_hash=row.content_hash,
|
||||||
|
workspace_id=self.workspace_id,
|
||||||
|
workspace_revision=self.workspace_revision,
|
||||||
|
)["kind"]
|
||||||
|
payload = qdrant_payload(
|
||||||
|
row.record,
|
||||||
|
content_hash=row.content_hash,
|
||||||
|
workspace_id=self.workspace_id,
|
||||||
|
workspace_revision=self.workspace_revision,
|
||||||
|
)
|
||||||
|
self.points[point_id(self.workspace_id, semantic_kind, row.record.id)] = _Point(
|
||||||
|
point_id=point_id(self.workspace_id, semantic_kind, row.record.id),
|
||||||
|
payload=payload,
|
||||||
|
embedding=row.embedding,
|
||||||
|
)
|
||||||
|
return len(records)
|
||||||
|
|
||||||
|
def delete_generation(self, collection, generation, workspace_id):
|
||||||
|
doomed = [
|
||||||
|
key
|
||||||
|
for key, point in self.points.items()
|
||||||
|
if point.payload.get("record_kind") == "evidence"
|
||||||
|
and point.payload.get("vector_generation") == generation
|
||||||
|
and point.payload.get("workspace_id") == workspace_id
|
||||||
|
]
|
||||||
|
for key in doomed:
|
||||||
|
self.points.pop(key)
|
||||||
|
return len(doomed)
|
||||||
|
|
||||||
|
def list_evidence_generations(self, collection, workspace_id):
|
||||||
|
return sorted(
|
||||||
|
{
|
||||||
|
point.payload["vector_generation"]
|
||||||
|
for point in self.points.values()
|
||||||
|
if point.payload.get("record_kind") == "evidence"
|
||||||
|
and point.payload.get("workspace_id") == workspace_id
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _schema_records():
|
||||||
|
return schema_records(
|
||||||
|
PhysicalSchema(
|
||||||
|
database="analytics",
|
||||||
|
schema="mart",
|
||||||
|
introspected_at=datetime.now(UTC),
|
||||||
|
tables={
|
||||||
|
"fact_patient": TablePhysical(
|
||||||
|
comment="Patients",
|
||||||
|
columns={"id": ColumnPhysical(type="bigint", comment="pk")},
|
||||||
|
)
|
||||||
|
},
|
||||||
|
),
|
||||||
|
Annotations(),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _memory_records():
|
||||||
|
return [
|
||||||
|
MemoryRecord(
|
||||||
|
id="mem-0001",
|
||||||
|
ts=datetime(2026, 1, 1, tzinfo=UTC),
|
||||||
|
session_id="s1",
|
||||||
|
decision_seq=7,
|
||||||
|
type="concept_clarified",
|
||||||
|
subject="paziente attivo",
|
||||||
|
detail="flag_attivo = true",
|
||||||
|
rationale="r",
|
||||||
|
question_context="dammi i pazienti attivi",
|
||||||
|
tables=[],
|
||||||
|
concepts=["paziente attivo"],
|
||||||
|
)
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def test_schema_and_memory_use_expected_semantic_kinds_and_shared_identity():
|
||||||
|
store = FakeVectorStore()
|
||||||
|
embedder = _Embedder()
|
||||||
|
|
||||||
|
schema_stats = sync_canonical_records(
|
||||||
|
"schema_records",
|
||||||
|
_schema_records(),
|
||||||
|
store=store,
|
||||||
|
embedder=embedder,
|
||||||
|
)
|
||||||
|
memory_count = save_one_memory(_memory_records(), 7, store=store, embedder=embedder)
|
||||||
|
|
||||||
|
assert schema_stats.added == 2
|
||||||
|
assert memory_count == 1
|
||||||
|
payloads = {point.payload["record_kind"]: point.payload for point in store.points.values()}
|
||||||
|
assert payloads["schema_table"]["kind"] == "schema"
|
||||||
|
assert payloads["schema_column"]["kind"] == "schema"
|
||||||
|
assert payloads["memory"]["kind"] == "memory"
|
||||||
|
assert {payload["workspace_id"] for payload in payloads.values()} == {"psd-clinical"}
|
||||||
|
assert {payload["workspace_revision"] for payload in payloads.values()} == {"a" * 40}
|
||||||
|
|
||||||
|
|
||||||
|
def test_corpus_vector_records_keep_exact_generation_and_retry_is_idempotent(tmp_path):
|
||||||
|
store = FakeVectorStore()
|
||||||
|
pipeline = CorpusPipeline(
|
||||||
|
store=CorpusStore(tmp_path / "corpus"),
|
||||||
|
sources=[],
|
||||||
|
embedder=None,
|
||||||
|
vector_store=store,
|
||||||
|
embedding_model="qwen3-embedding:0.6b",
|
||||||
|
embedding_dimensions=1024,
|
||||||
|
chunk_policy=ChunkPolicy(version="chunk-v1", max_chars=4000),
|
||||||
|
pipeline_version="evidence-v1",
|
||||||
|
workspace_id="psd-clinical",
|
||||||
|
)
|
||||||
|
chunk = CanonicalChunk(
|
||||||
|
chunk_id="chunk:1",
|
||||||
|
document_id="doc:patient-guide",
|
||||||
|
ordinal=0,
|
||||||
|
content="Patient evidence",
|
||||||
|
content_hash=_sha("Patient evidence"),
|
||||||
|
source_uri="file:///tmp/patient-guide.md",
|
||||||
|
pipeline_version="evidence-v1",
|
||||||
|
)
|
||||||
|
row = pipeline._vector_record(
|
||||||
|
chunk,
|
||||||
|
[0.1, 0.2, 0.3, 0.4],
|
||||||
|
"gen:" + "1" * 32,
|
||||||
|
"psd-clinical",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert row.record.kind == "evidence"
|
||||||
|
assert row.record.metadata["vector_generation"] == "gen:" + "1" * 32
|
||||||
|
|
||||||
|
store.upsert("evidence", [row])
|
||||||
|
store.upsert("evidence", [row])
|
||||||
|
|
||||||
|
assert len(store.points) == 1
|
||||||
|
point = next(iter(store.points.values()))
|
||||||
|
assert point.payload["kind"] == "evidence"
|
||||||
|
assert point.payload["vector_generation"] == "gen:" + "1" * 32
|
||||||
|
assert point.payload["workspace_id"] == "psd-clinical"
|
||||||
|
assert point.payload["workspace_revision"] == "a" * 40
|
||||||
@@ -3,7 +3,7 @@
|
|||||||
from tht.adapters.dwh import PostgresDwhAdapter, ThothRestDwhAdapter
|
from tht.adapters.dwh import PostgresDwhAdapter, ThothRestDwhAdapter
|
||||||
from tht.adapters.evidence import FilesystemEvidenceSource, HttpManifestEvidenceSource
|
from tht.adapters.evidence import FilesystemEvidenceSource, HttpManifestEvidenceSource
|
||||||
from tht.adapters.evidence.s3 import S3EvidenceSource
|
from tht.adapters.evidence.s3 import S3EvidenceSource
|
||||||
from tht.adapters.vector import PgVectorStore, ThothHttpVectorStore
|
from tht.adapters.vector import PgVectorStore, QdrantVectorStore, ThothHttpVectorStore
|
||||||
from tht.config import Config, ConfigError
|
from tht.config import Config, ConfigError
|
||||||
from tht.db.connection import make_engine
|
from tht.db.connection import make_engine
|
||||||
from tht.ports.dwh import DwhAdapter
|
from tht.ports.dwh import DwhAdapter
|
||||||
@@ -56,6 +56,14 @@ def build_vector_store(cfg: Config, *, require_write: bool = False) -> VectorSto
|
|||||||
VectorRestClient(resource.writer) if resource.writer is not None else None,
|
VectorRestClient(resource.writer) if resource.writer is not None else None,
|
||||||
expected_dimension=cfg.embeddings.dim if cfg.embeddings is not None else None,
|
expected_dimension=cfg.embeddings.dim if cfg.embeddings is not None else None,
|
||||||
)
|
)
|
||||||
|
case "qdrant":
|
||||||
|
return QdrantVectorStore(
|
||||||
|
base_url=resource.base_url,
|
||||||
|
collection=resource.collection,
|
||||||
|
workspace_id=cfg._workspace_id,
|
||||||
|
workspace_revision=cfg._workspace_revision,
|
||||||
|
expected_dimension=cfg.embeddings.dim if cfg.embeddings is not None else None,
|
||||||
|
)
|
||||||
case other: # pragma: no cover - Pydantic's discriminator rejects this first.
|
case other: # pragma: no cover - Pydantic's discriminator rejects this first.
|
||||||
raise ConfigError(f"Adapter vector non supportato: {other}")
|
raise ConfigError(f"Adapter vector non supportato: {other}")
|
||||||
|
|
||||||
|
|||||||
@@ -32,6 +32,7 @@ _KEYWORD_INDEXES = (
|
|||||||
"record_kind",
|
"record_kind",
|
||||||
"vector_generation",
|
"vector_generation",
|
||||||
"workspace_id",
|
"workspace_id",
|
||||||
|
"workspace_revision",
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@@ -52,6 +53,7 @@ class QdrantVectorStore:
|
|||||||
base_url: str,
|
base_url: str,
|
||||||
collection: str,
|
collection: str,
|
||||||
workspace_id: str,
|
workspace_id: str,
|
||||||
|
workspace_revision: str | None = None,
|
||||||
expected_dimension: int | None = None,
|
expected_dimension: int | None = None,
|
||||||
request: Callable[..., object] | None = None,
|
request: Callable[..., object] | None = None,
|
||||||
connect_timeout: float = 2.0,
|
connect_timeout: float = 2.0,
|
||||||
@@ -60,6 +62,7 @@ class QdrantVectorStore:
|
|||||||
self._base_url = base_url.rstrip("/")
|
self._base_url = base_url.rstrip("/")
|
||||||
self._collection = collection
|
self._collection = collection
|
||||||
self._workspace_id = workspace_id
|
self._workspace_id = workspace_id
|
||||||
|
self._workspace_revision = workspace_revision
|
||||||
self._expected_dimension = expected_dimension
|
self._expected_dimension = expected_dimension
|
||||||
self._request = request or requests.request
|
self._request = request or requests.request
|
||||||
self._timeout = (connect_timeout, read_timeout)
|
self._timeout = (connect_timeout, read_timeout)
|
||||||
@@ -198,6 +201,7 @@ class QdrantVectorStore:
|
|||||||
write_record.record,
|
write_record.record,
|
||||||
content_hash=write_record.content_hash,
|
content_hash=write_record.content_hash,
|
||||||
workspace_id=self._workspace_id,
|
workspace_id=self._workspace_id,
|
||||||
|
workspace_revision=self._workspace_revision,
|
||||||
),
|
),
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -10,17 +10,18 @@ from pathlib import Path
|
|||||||
import typer
|
import typer
|
||||||
from sqlalchemy.exc import OperationalError, ProgrammingError
|
from sqlalchemy.exc import OperationalError, ProgrammingError
|
||||||
|
|
||||||
from tht.cli.config_cmd import CONFIG_OPT
|
|
||||||
from tht.cli.schema_cmd import _load_config_or_exit
|
|
||||||
from tht.cli._guards import (
|
from tht.cli._guards import (
|
||||||
has_vector_write_rest,
|
has_vector_write_rest,
|
||||||
require_server_profile,
|
require_server_profile,
|
||||||
require_vector_write_allowed,
|
require_vector_write_allowed,
|
||||||
)
|
)
|
||||||
|
from tht.cli.config_cmd import CONFIG_OPT
|
||||||
|
from tht.cli.schema_cmd import _load_config_or_exit
|
||||||
from tht.cli.session_cmd import load_snapshot_or_exit
|
from tht.cli.session_cmd import load_snapshot_or_exit
|
||||||
from tht.cli.vector_cmd import require_vector_cfg
|
from tht.cli.vector_cmd import require_vector_cfg
|
||||||
|
|
||||||
memory_app = typer.Typer(help="Review memory (registro canonico + indice pgvector)")
|
memory_app = typer.Typer(help="Review memory (registro canonico + indice pgvector)")
|
||||||
|
DECISION_OPT = typer.Option(None, "--decision", help="Seq da promuovere (ripetibile).")
|
||||||
|
|
||||||
|
|
||||||
def registry_path(cfg) -> Path:
|
def registry_path(cfg) -> Path:
|
||||||
@@ -29,18 +30,23 @@ def registry_path(cfg) -> Path:
|
|||||||
|
|
||||||
def _resync_memory(cfg):
|
def _resync_memory(cfg):
|
||||||
"""Risincronizza l'indice pgvector col registro corrente (incrementale)."""
|
"""Risincronizza l'indice pgvector col registro corrente (incrementale)."""
|
||||||
from tht.cli.vector_cmd import make_embedder, open_store
|
from tht.adapters.factory import build_vector_store
|
||||||
|
from tht.cli.vector_cmd import make_embedder, sync_canonical_records
|
||||||
from tht.memory import load_registry, memory_vector_records
|
from tht.memory import load_registry, memory_vector_records
|
||||||
|
|
||||||
records = memory_vector_records(load_registry(registry_path(cfg)))
|
records = memory_vector_records(load_registry(registry_path(cfg)))
|
||||||
store = open_store(cfg, "memory")
|
return sync_canonical_records(
|
||||||
return store.sync(records, make_embedder(cfg.embeddings), kinds={"memory"})
|
"memory",
|
||||||
|
records,
|
||||||
|
store=build_vector_store(cfg, require_write=True),
|
||||||
|
embedder=make_embedder(cfg.embeddings),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
@memory_app.command("promote")
|
@memory_app.command("promote")
|
||||||
def promote_cmd(
|
def promote_cmd(
|
||||||
session: str = typer.Option(..., "--session"),
|
session: str = typer.Option(..., "--session"),
|
||||||
decision: list[int] = typer.Option(None, "--decision", help="Seq da promuovere (ripetibile)."),
|
decision: list[int] = DECISION_OPT,
|
||||||
preview: bool = typer.Option(False, "--preview", help="Mostra i candidati in JSON, non scrive."),
|
preview: bool = typer.Option(False, "--preview", help="Mostra i candidati in JSON, non scrive."),
|
||||||
json_out: bool = typer.Option(False, "--json", help="Output JSON (per Pi)."),
|
json_out: bool = typer.Option(False, "--json", help="Output JSON (per Pi)."),
|
||||||
config: Path = CONFIG_OPT,
|
config: Path = CONFIG_OPT,
|
||||||
@@ -55,7 +61,9 @@ def promote_cmd(
|
|||||||
|
|
||||||
if preview:
|
if preview:
|
||||||
from tht.memory import (
|
from tht.memory import (
|
||||||
MAX_PROMOTION_CANDIDATES, preview_promotions_snapshot, reusable_promotions_snapshot,
|
MAX_PROMOTION_CANDIDATES,
|
||||||
|
preview_promotions_snapshot,
|
||||||
|
reusable_promotions_snapshot,
|
||||||
)
|
)
|
||||||
cand = preview_promotions_snapshot(snapshot, registry_path(cfg))
|
cand = preview_promotions_snapshot(snapshot, registry_path(cfg))
|
||||||
extra = len(reusable_promotions_snapshot(snapshot, registry_path(cfg))) - len(cand)
|
extra = len(reusable_promotions_snapshot(snapshot, registry_path(cfg))) - len(cand)
|
||||||
@@ -304,8 +312,8 @@ def update_cmd(
|
|||||||
"""Modifica i campi di merito di una memoria (provenienza immutabile)."""
|
"""Modifica i campi di merito di una memoria (provenienza immutabile)."""
|
||||||
from typing import get_args
|
from typing import get_args
|
||||||
|
|
||||||
from tht.memory import MemoryNotFound, update_record
|
|
||||||
from tht.decisions import DecisionType
|
from tht.decisions import DecisionType
|
||||||
|
from tht.memory import MemoryNotFound, update_record
|
||||||
|
|
||||||
cfg = _load_config_or_exit(config)
|
cfg = _load_config_or_exit(config)
|
||||||
|
|
||||||
|
|||||||
@@ -2,9 +2,15 @@ from pathlib import Path
|
|||||||
|
|
||||||
import typer
|
import typer
|
||||||
|
|
||||||
from tht.cli._guards import has_vector_write_rest, require_server_profile, require_vector_write_allowed
|
from tht.cli._guards import (
|
||||||
|
has_vector_write_rest,
|
||||||
|
require_server_profile,
|
||||||
|
require_vector_write_allowed,
|
||||||
|
)
|
||||||
from tht.cli.config_cmd import CONFIG_OPT
|
from tht.cli.config_cmd import CONFIG_OPT
|
||||||
from tht.cli.schema_cmd import _load_config_or_exit, annotations_path, physical_path
|
from tht.cli.schema_cmd import _load_config_or_exit, annotations_path, physical_path
|
||||||
|
from tht.ports.vector import VectorWriteRecord
|
||||||
|
from tht.vectorstore.store import SyncStats, content_hash
|
||||||
|
|
||||||
vector_app = typer.Typer(help="Indice semantico pgvector (derivato, rigenerabile)")
|
vector_app = typer.Typer(help="Indice semantico pgvector (derivato, rigenerabile)")
|
||||||
|
|
||||||
@@ -69,6 +75,31 @@ def open_searcher(cfg):
|
|||||||
return AdapterSearcher()
|
return AdapterSearcher()
|
||||||
|
|
||||||
|
|
||||||
|
def sync_canonical_records(collection, records, *, store, embedder):
|
||||||
|
kinds = sorted({record.kind for record in records})
|
||||||
|
existing = store.existing_hashes(collection, kinds)
|
||||||
|
pending = []
|
||||||
|
stats = SyncStats()
|
||||||
|
changed = []
|
||||||
|
for record in records:
|
||||||
|
hashed = content_hash(record.content)
|
||||||
|
current = existing.get(record.id)
|
||||||
|
if current == hashed:
|
||||||
|
stats.unchanged += 1
|
||||||
|
continue
|
||||||
|
changed.append((record, hashed, current is None))
|
||||||
|
if changed:
|
||||||
|
embeddings = embedder.embed_documents([record.content for record, *_ in changed])
|
||||||
|
for (record, hashed, is_added), embedding in zip(changed, embeddings, strict=True):
|
||||||
|
pending.append(VectorWriteRecord(record=record, embedding=embedding, content_hash=hashed))
|
||||||
|
if is_added:
|
||||||
|
stats.added += 1
|
||||||
|
else:
|
||||||
|
stats.updated += 1
|
||||||
|
store.upsert(collection, pending)
|
||||||
|
return stats
|
||||||
|
|
||||||
|
|
||||||
def _print_stats(stats) -> None:
|
def _print_stats(stats) -> None:
|
||||||
typer.secho(
|
typer.secho(
|
||||||
f"OK: {stats.added} nuovi, {stats.updated} aggiornati, "
|
f"OK: {stats.added} nuovi, {stats.updated} aggiornati, "
|
||||||
@@ -88,7 +119,6 @@ def init_cmd(
|
|||||||
from sqlalchemy.exc import OperationalError
|
from sqlalchemy.exc import OperationalError
|
||||||
|
|
||||||
from tht.vectorstore.embeddings import EmbeddingsError
|
from tht.vectorstore.embeddings import EmbeddingsError
|
||||||
|
|
||||||
from tht.vectorstore.reader import ALL_TABLES
|
from tht.vectorstore.reader import ALL_TABLES
|
||||||
|
|
||||||
cfg = _load_config_or_exit(config)
|
cfg = _load_config_or_exit(config)
|
||||||
@@ -131,8 +161,12 @@ def index_schema_cmd(config: Path = CONFIG_OPT) -> None:
|
|||||||
physical = PhysicalSchema.from_yaml(phys_file)
|
physical = PhysicalSchema.from_yaml(phys_file)
|
||||||
annotations = Annotations.from_yaml(annotations_path(cfg))
|
annotations = Annotations.from_yaml(annotations_path(cfg))
|
||||||
records = schema_records(physical, annotations)
|
records = schema_records(physical, annotations)
|
||||||
store = open_store(cfg, "schema_records")
|
from tht.adapters.factory import build_vector_store
|
||||||
stats = store.sync(
|
|
||||||
records, make_embedder(cfg.embeddings), kinds={"schema_table", "schema_column"}
|
stats = sync_canonical_records(
|
||||||
|
"schema_records",
|
||||||
|
records,
|
||||||
|
store=build_vector_store(cfg, require_write=True),
|
||||||
|
embedder=make_embedder(cfg.embeddings),
|
||||||
)
|
)
|
||||||
_print_stats(stats)
|
_print_stats(stats)
|
||||||
|
|||||||
+78
-1
@@ -152,8 +152,14 @@ class ThothVectorHttpConfig(BaseModel):
|
|||||||
direct: DatabaseConfig | None = None
|
direct: DatabaseConfig | None = None
|
||||||
|
|
||||||
|
|
||||||
|
class QdrantConfig(BaseModel):
|
||||||
|
type: Literal["qdrant"]
|
||||||
|
base_url: str
|
||||||
|
collection: str = Field(min_length=1)
|
||||||
|
|
||||||
|
|
||||||
VectorResourceConfig = Annotated[
|
VectorResourceConfig = Annotated[
|
||||||
PgvectorDirectConfig | ThothVectorHttpConfig,
|
PgvectorDirectConfig | ThothVectorHttpConfig | QdrantConfig,
|
||||||
Field(discriminator="type"),
|
Field(discriminator="type"),
|
||||||
]
|
]
|
||||||
|
|
||||||
@@ -397,6 +403,7 @@ def load_config(path: Path) -> Config:
|
|||||||
raise ConfigError(f"Configurazione non valida (atteso un mapping YAML): {path}")
|
raise ConfigError(f"Configurazione non valida (atteso un mapping YAML): {path}")
|
||||||
expanded = _resolve_secret_files(_expand_env(raw))
|
expanded = _resolve_secret_files(_expand_env(raw))
|
||||||
_validate_internal_embedding_contract(expanded, path)
|
_validate_internal_embedding_contract(expanded, path)
|
||||||
|
_validate_internal_vector_contract(expanded, path)
|
||||||
translated, used_legacy = translate_legacy_config(expanded)
|
translated, used_legacy = translate_legacy_config(expanded)
|
||||||
_populate_legacy_views(translated)
|
_populate_legacy_views(translated)
|
||||||
try:
|
try:
|
||||||
@@ -455,6 +462,7 @@ def load_config(path: Path) -> Config:
|
|||||||
else path.resolve().as_posix()
|
else path.resolve().as_posix()
|
||||||
)
|
)
|
||||||
_validate_active_embeddings_config(cfg.embeddings, path)
|
_validate_active_embeddings_config(cfg.embeddings, path)
|
||||||
|
_validate_active_vector_config(cfg.vectors, path)
|
||||||
return cfg
|
return cfg
|
||||||
|
|
||||||
|
|
||||||
@@ -500,6 +508,42 @@ def _validate_internal_embedding_contract(raw: dict[str, Any], path: Path) -> No
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_internal_vector_contract(raw: dict[str, Any], path: Path) -> None:
|
||||||
|
resources = raw.get("resources")
|
||||||
|
if not isinstance(resources, dict):
|
||||||
|
return
|
||||||
|
vector = resources.get("vector")
|
||||||
|
if not isinstance(vector, dict):
|
||||||
|
return
|
||||||
|
|
||||||
|
engine = vector.get("engine")
|
||||||
|
base_url = vector.get("base_url")
|
||||||
|
collection = vector.get("collection")
|
||||||
|
allowed = {"engine", "base_url", "collection"}
|
||||||
|
unexpected = sorted(set(vector) - allowed)
|
||||||
|
if unexpected:
|
||||||
|
raise ConfigError(
|
||||||
|
f"Configurazione non valida in {path}:\n"
|
||||||
|
f"resources.vector non supporta: {', '.join(unexpected)}"
|
||||||
|
)
|
||||||
|
if engine != "qdrant":
|
||||||
|
raise ConfigError(
|
||||||
|
f"Configurazione non valida in {path}:\n"
|
||||||
|
"resources.vector.engine deve essere 'qdrant'"
|
||||||
|
)
|
||||||
|
if not isinstance(collection, str) or not collection:
|
||||||
|
raise ConfigError(
|
||||||
|
f"Configurazione non valida in {path}:\n"
|
||||||
|
"resources.vector.collection deve essere valorizzato"
|
||||||
|
)
|
||||||
|
if not _is_allowed_internal_qdrant_url(base_url):
|
||||||
|
raise ConfigError(
|
||||||
|
f"Configurazione non valida in {path}:\n"
|
||||||
|
"resources.vector.base_url deve usare http://qdrant:6333 "
|
||||||
|
"oppure un endpoint loopback di sviluppo su porta 6333"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def _validate_active_embeddings_config(
|
def _validate_active_embeddings_config(
|
||||||
embeddings: "EmbeddingsConfig | None",
|
embeddings: "EmbeddingsConfig | None",
|
||||||
path: Path,
|
path: Path,
|
||||||
@@ -529,6 +573,20 @@ def _validate_active_embeddings_config(
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_active_vector_config(
|
||||||
|
vectors: "VectorResourceConfig | None",
|
||||||
|
path: Path,
|
||||||
|
) -> None:
|
||||||
|
if vectors is None or vectors.type != "qdrant":
|
||||||
|
return
|
||||||
|
if not _is_allowed_internal_qdrant_url(vectors.base_url):
|
||||||
|
raise ConfigError(
|
||||||
|
f"Configurazione non valida in {path}:\n"
|
||||||
|
"vectors.base_url deve usare http://qdrant:6333 "
|
||||||
|
"oppure un endpoint loopback di sviluppo su porta 6333"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def _is_allowed_internal_embedding_url(value: Any) -> bool:
|
def _is_allowed_internal_embedding_url(value: Any) -> bool:
|
||||||
if not isinstance(value, str):
|
if not isinstance(value, str):
|
||||||
return False
|
return False
|
||||||
@@ -548,6 +606,25 @@ def _is_allowed_internal_embedding_url(value: Any) -> bool:
|
|||||||
return host.is_loopback
|
return host.is_loopback
|
||||||
|
|
||||||
|
|
||||||
|
def _is_allowed_internal_qdrant_url(value: Any) -> bool:
|
||||||
|
if not isinstance(value, str):
|
||||||
|
return False
|
||||||
|
parsed = urlparse(value)
|
||||||
|
if parsed.scheme != "http" or not parsed.hostname or parsed.port != 6333:
|
||||||
|
return False
|
||||||
|
if parsed.params or parsed.query or parsed.fragment:
|
||||||
|
return False
|
||||||
|
if parsed.path not in ("", "/"):
|
||||||
|
return False
|
||||||
|
if parsed.hostname == "qdrant":
|
||||||
|
return True
|
||||||
|
try:
|
||||||
|
host = ip_address(parsed.hostname)
|
||||||
|
except ValueError:
|
||||||
|
return parsed.hostname == "localhost"
|
||||||
|
return host.is_loopback
|
||||||
|
|
||||||
|
|
||||||
def _populate_legacy_views(raw: dict[str, Any]) -> None:
|
def _populate_legacy_views(raw: dict[str, Any]) -> None:
|
||||||
"""Populate old Config attributes for command compatibility during migration."""
|
"""Populate old Config attributes for command compatibility during migration."""
|
||||||
dwh = raw.get("dwh")
|
dwh = raw.get("dwh")
|
||||||
|
|||||||
@@ -33,6 +33,14 @@ def translate_legacy_config(raw: dict[str, Any]) -> tuple[dict[str, Any], bool]:
|
|||||||
translated["embeddings"] = embedding
|
translated["embeddings"] = embedding
|
||||||
if "dimensions" in translated["embeddings"] and "dim" not in translated["embeddings"]:
|
if "dimensions" in translated["embeddings"] and "dim" not in translated["embeddings"]:
|
||||||
translated["embeddings"]["dim"] = translated["embeddings"].pop("dimensions")
|
translated["embeddings"]["dim"] = translated["embeddings"].pop("dimensions")
|
||||||
|
if isinstance(resources, dict) and "vector" in resources and "vectors" not in translated:
|
||||||
|
vector = _as_mapping(resources.get("vector"))
|
||||||
|
if isinstance(vector, dict):
|
||||||
|
translated["vectors"] = {
|
||||||
|
"type": "qdrant",
|
||||||
|
"base_url": vector.get("base_url"),
|
||||||
|
"collection": vector.get("collection"),
|
||||||
|
}
|
||||||
legacy = any(key in raw for key in _LEGACY_RESOURCE_KEYS)
|
legacy = any(key in raw for key in _LEGACY_RESOURCE_KEYS)
|
||||||
if not legacy:
|
if not legacy:
|
||||||
return translated, False
|
return translated, False
|
||||||
|
|||||||
@@ -27,11 +27,18 @@ def qdrant_semantic_kind(kind: str) -> str:
|
|||||||
raise ValueError(f"Unsupported vector kind: {kind}")
|
raise ValueError(f"Unsupported vector kind: {kind}")
|
||||||
|
|
||||||
|
|
||||||
def qdrant_payload(record: VectorRecord, *, content_hash: str, workspace_id: str) -> dict:
|
def qdrant_payload(
|
||||||
|
record: VectorRecord,
|
||||||
|
*,
|
||||||
|
content_hash: str,
|
||||||
|
workspace_id: str,
|
||||||
|
workspace_revision: str | None = None,
|
||||||
|
) -> dict:
|
||||||
semantic_kind = qdrant_semantic_kind(record.kind)
|
semantic_kind = qdrant_semantic_kind(record.kind)
|
||||||
return {
|
return {
|
||||||
**record.metadata,
|
**record.metadata,
|
||||||
"workspace_id": workspace_id,
|
"workspace_id": workspace_id,
|
||||||
|
**({"workspace_revision": workspace_revision} if workspace_revision else {}),
|
||||||
"kind": semantic_kind,
|
"kind": semantic_kind,
|
||||||
"record_kind": record.kind,
|
"record_kind": record.kind,
|
||||||
"record_key": record.id,
|
"record_key": record.id,
|
||||||
|
|||||||
Reference in New Issue
Block a user