Chiusura fase di ristrutturazione e modularizzazione del workflow per favorire sviluppo modulare

This commit is contained in:
2026-08-24 13:32:20 +02:00
parent fa2298653b
commit 6062cb010e
96 changed files with 304 additions and 259 deletions
+3 -4
View File
@@ -24,16 +24,15 @@ from tht.evidence.search import (
from tht.evidence.session import project_session
from tht.evidence.sources import build_sources
__all__ = [
"AcquiredDocument",
"ActiveEvidenceSearcher",
"CorpusWorkspaceMismatchError",
"EvidenceEmbedder",
"EvidenceSource",
"EvidenceSourceError",
"EvidenceSourceErrorCategory",
"EvidenceEmbedder",
"SourceObject",
"ActiveEvidenceSearcher",
"CorpusWorkspaceMismatchError",
"acquire",
"active_searcher",
"build_preprocessing_pipeline",
+4 -1
View File
@@ -7,7 +7,10 @@ from datetime import UTC, datetime
from urllib.parse import quote, urlsplit
from tht.evidence.contracts import (
AcquiredDocument, EvidenceSourceError, EvidenceSourceErrorCategory, SourceObject,
AcquiredDocument,
EvidenceSourceError,
EvidenceSourceErrorCategory,
SourceObject,
)
-1
View File
@@ -15,7 +15,6 @@ from tht.evidence.contracts import (
validate_safe_metadata,
)
_NAMESPACED_ID = re.compile(r"^[a-z][a-z0-9_-]*:[A-Za-z0-9._:-]+$")
_SHA256 = re.compile(r"^sha256:[0-9a-f]{64}$")
+1 -2
View File
@@ -10,9 +10,8 @@ from pydantic import JsonValue, TypeAdapter, ValidationError
from yaml.events import AliasEvent
from yaml.nodes import MappingNode
from tht.evidence.corpus.models import CanonicalDocument
from tht.evidence.contracts import AcquiredDocument, canonical_provenance_uri
from tht.evidence.corpus.models import CanonicalDocument
MAX_DOCUMENT_BYTES = 10 * 1024 * 1024
_CHARSET = re.compile(r"(?:^|;)\s*charset\s*=\s*[\"']?([^;\s\"']+)", re.IGNORECASE)
+13 -11
View File
@@ -4,6 +4,7 @@ from __future__ import annotations
import hashlib
import json
import logging
import re
import uuid
from collections.abc import Mapping, Sequence
@@ -11,17 +12,16 @@ from dataclasses import asdict, dataclass, field
from datetime import UTC
from pathlib import Path
import tht.evidence.acquisition as evidence_acquisition
from tht.evidence.contracts import EvidenceSource, SourceObject, canonical_provenance_uri
from tht.evidence.corpus.chunk import ChunkPolicy, chunk
from tht.evidence.corpus.models import CanonicalChunk, CanonicalDocument, CorpusManifest
from tht.evidence.corpus.normalize import normalize
from tht.evidence.corpus.store import CorpusStore
import tht.evidence.acquisition as evidence_acquisition
from tht.evidence.contracts import EvidenceSource, SourceObject, canonical_provenance_uri
from tht.ports.vector import VectorStore, VectorWriteRecord
from tht.vectorstore.records import VectorRecord
from tht.jobs.models import JobSpec
from tht.jobs.runner import JobContext, StageArtifacts, run_job, seal_stage_artifacts
from tht.ports.vector import VectorStore, VectorWriteRecord
from tht.vectorstore.records import VectorRecord
EVIDENCE_STAGE_IDS = (
"discover",
@@ -33,6 +33,8 @@ EVIDENCE_STAGE_IDS = (
"retention_cleanup",
)
logger = logging.getLogger(__name__)
class PipelineError(RuntimeError):
"""Credential-free failure at the preprocessing boundary."""
@@ -212,14 +214,14 @@ class CorpusPipeline:
if purge_vector:
try:
self.vector_store.delete_generation("evidence", generation, self.workspace_id)
except Exception:
except Exception: # noqa: BLE001 - retention reports per-generation failures
failures.append({"generation": generation, "error": "vector cleanup failed"})
continue
try:
if purge_filesystem:
self.store.discard(generation)
evicted.append(generation)
except Exception:
except Exception: # noqa: BLE001 - retention reports per-generation failures
failures.append({"generation": generation, "error": "filesystem cleanup failed"})
return {"status": "partial" if failures else "succeeded", "dry_run": dry_run,
"active_generation": self.store.active_generation(), "evicted": evicted,
@@ -378,7 +380,7 @@ class CorpusPipeline:
try:
active_assets_valid = active_assets_are_valid(previous)
except Exception:
except Exception: # noqa: BLE001 - any corrupt active asset disables reuse
active_assets_valid = False
reusable = (
active_assets_valid
@@ -538,7 +540,7 @@ class CorpusPipeline:
try:
self.vector_store.delete_generation("evidence", generation, self.workspace_id)
except Exception:
pass
logger.debug("Failed to clean the compensated vector generation", exc_info=True)
write(context, "compensated.json", {"generation": generation})
def rotate_compensated_generation(context: JobContext) -> None:
@@ -786,12 +788,12 @@ class CorpusPipeline:
try:
self.store.discard(generation)
except Exception:
pass
logger.debug("Failed to discard the unpublished evidence generation", exc_info=True)
if vector_written:
try:
self.vector_store.delete_generation("evidence", generation, self.workspace_id)
except Exception:
pass
logger.debug("Failed to delete the unpublished vector generation", exc_info=True)
@staticmethod
def _vector_record(
+5 -6
View File
@@ -2,22 +2,21 @@
from __future__ import annotations
import json
import fcntl
import hashlib
import json
import os
import re
import stat
import shutil
import uuid
import hashlib
import stat
import threading
import uuid
from contextlib import contextmanager
from datetime import UTC, datetime
from pathlib import Path
from contextlib import contextmanager
from tht.evidence.corpus.models import CorpusManifest
_GENERATION = re.compile(r"^gen:[0-9a-f]{32}$")
+2 -2
View File
@@ -46,7 +46,7 @@ class ConceptFormula(BaseModel):
return f"---\n{fm}---\n{self.sql}\n"
@classmethod
def parse(cls, text: str) -> "ConceptFormula":
def parse(cls, text: str) -> ConceptFormula:
if not text.startswith("---\n"):
raise ValueError("frontmatter mancante (atteso '---\\n' iniziale)")
try:
@@ -55,7 +55,7 @@ class ConceptFormula(BaseModel):
raise ValueError("frontmatter malformato") from e
meta = yaml.safe_load(fm)
if not isinstance(meta, dict):
raise ValueError("frontmatter non valido")
raise TypeError("frontmatter non valido")
return cls.model_validate({**meta, "sql": body.strip("\n")})
+1 -1
View File
@@ -2,10 +2,10 @@
from typing import Protocol
from tht.evidence.contracts import EvidenceSource
from tht.evidence.corpus.chunk import ChunkPolicy
from tht.evidence.corpus.pipeline import CorpusPipeline
from tht.evidence.corpus.store import CorpusStore
from tht.evidence.contracts import EvidenceSource
from tht.ports.vector import VectorStore