feat: use internal ollama embeddings
This commit is contained in:
+69
-2
@@ -1,11 +1,13 @@
|
||||
import os
|
||||
import re
|
||||
import warnings
|
||||
from ipaddress import ip_address
|
||||
from pathlib import Path
|
||||
from typing import Annotated, Any, Literal
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import yaml
|
||||
from pydantic import BaseModel, Field, PrivateAttr, SecretStr, model_validator, ValidationError
|
||||
from pydantic import BaseModel, Field, PrivateAttr, SecretStr, ValidationError, model_validator
|
||||
|
||||
from tht.config_compat import translate_legacy_config
|
||||
|
||||
@@ -276,15 +278,18 @@ class EvidenceSourcesConfig(BaseModel):
|
||||
|
||||
|
||||
class EmbeddingsConfig(BaseModel):
|
||||
provider: str = "ollama_internal"
|
||||
base_url: str
|
||||
model: str = "nomic-embed-text-v2-moe"
|
||||
dim: int = 768
|
||||
dim: int = Field(default=768, alias="dimensions")
|
||||
batch_size: int = 32
|
||||
timeout: int = 30
|
||||
connect_timeout: int = 5
|
||||
bin: str = "ollama"
|
||||
start_cmd: list[str] | None = None
|
||||
|
||||
model_config = {"populate_by_name": True, "extra": "forbid"}
|
||||
|
||||
|
||||
class VectorConfig(BaseModel):
|
||||
max_chunk_chars: int = 4000
|
||||
@@ -391,6 +396,7 @@ def load_config(path: Path) -> Config:
|
||||
if not isinstance(raw, dict):
|
||||
raise ConfigError(f"Configurazione non valida (atteso un mapping YAML): {path}")
|
||||
expanded = _resolve_secret_files(_expand_env(raw))
|
||||
_validate_internal_embedding_contract(expanded, path)
|
||||
translated, used_legacy = translate_legacy_config(expanded)
|
||||
_populate_legacy_views(translated)
|
||||
try:
|
||||
@@ -451,6 +457,67 @@ def load_config(path: Path) -> Config:
|
||||
return cfg
|
||||
|
||||
|
||||
def _validate_internal_embedding_contract(raw: dict[str, Any], path: Path) -> None:
|
||||
resources = raw.get("resources")
|
||||
if not isinstance(resources, dict):
|
||||
return
|
||||
embeddings = resources.get("embeddings")
|
||||
if not isinstance(embeddings, dict):
|
||||
return
|
||||
|
||||
provider = embeddings.get("provider")
|
||||
model = embeddings.get("model")
|
||||
dimensions = embeddings.get("dimensions")
|
||||
base_url = embeddings.get("base_url")
|
||||
allowed = {"provider", "base_url", "model", "dimensions"}
|
||||
unexpected = sorted(set(embeddings) - allowed)
|
||||
if unexpected:
|
||||
raise ConfigError(
|
||||
f"Configurazione non valida in {path}:\n"
|
||||
f"resources.embeddings non supporta: {', '.join(unexpected)}"
|
||||
)
|
||||
if provider != "ollama_internal":
|
||||
raise ConfigError(
|
||||
f"Configurazione non valida in {path}:\n"
|
||||
"resources.embeddings.provider deve essere 'ollama_internal'"
|
||||
)
|
||||
if model != "qwen3-embedding:0.6b":
|
||||
raise ConfigError(
|
||||
f"Configurazione non valida in {path}:\n"
|
||||
"resources.embeddings.model deve essere 'qwen3-embedding:0.6b'"
|
||||
)
|
||||
if dimensions != 1024:
|
||||
raise ConfigError(
|
||||
f"Configurazione non valida in {path}:\n"
|
||||
"resources.embeddings.dimensions deve essere 1024"
|
||||
)
|
||||
if not _is_allowed_internal_embedding_url(base_url):
|
||||
raise ConfigError(
|
||||
f"Configurazione non valida in {path}:\n"
|
||||
"resources.embeddings.base_url deve usare http://embedding:11434 "
|
||||
"oppure un endpoint loopback di sviluppo su porta 11434"
|
||||
)
|
||||
|
||||
|
||||
def _is_allowed_internal_embedding_url(value: Any) -> bool:
|
||||
if not isinstance(value, str):
|
||||
return False
|
||||
parsed = urlparse(value)
|
||||
if parsed.scheme != "http" or not parsed.hostname or parsed.port != 11434:
|
||||
return False
|
||||
if parsed.params or parsed.query or parsed.fragment:
|
||||
return False
|
||||
if parsed.path not in ("", "/"):
|
||||
return False
|
||||
if parsed.hostname == "embedding":
|
||||
return True
|
||||
try:
|
||||
host = ip_address(parsed.hostname)
|
||||
except ValueError:
|
||||
return parsed.hostname == "localhost"
|
||||
return host.is_loopback
|
||||
|
||||
|
||||
def _populate_legacy_views(raw: dict[str, Any]) -> None:
|
||||
"""Populate old Config attributes for command compatibility during migration."""
|
||||
dwh = raw.get("dwh")
|
||||
|
||||
Reference in New Issue
Block a user