Chiusura fase di ristrutturazione e modularizzazione del workflow per favorire sviluppo modulare
This commit is contained in:
@@ -24,16 +24,15 @@ from tht.evidence.search import (
|
||||
from tht.evidence.session import project_session
|
||||
from tht.evidence.sources import build_sources
|
||||
|
||||
|
||||
__all__ = [
|
||||
"AcquiredDocument",
|
||||
"ActiveEvidenceSearcher",
|
||||
"CorpusWorkspaceMismatchError",
|
||||
"EvidenceEmbedder",
|
||||
"EvidenceSource",
|
||||
"EvidenceSourceError",
|
||||
"EvidenceSourceErrorCategory",
|
||||
"EvidenceEmbedder",
|
||||
"SourceObject",
|
||||
"ActiveEvidenceSearcher",
|
||||
"CorpusWorkspaceMismatchError",
|
||||
"acquire",
|
||||
"active_searcher",
|
||||
"build_preprocessing_pipeline",
|
||||
|
||||
@@ -7,7 +7,10 @@ from datetime import UTC, datetime
|
||||
from urllib.parse import quote, urlsplit
|
||||
|
||||
from tht.evidence.contracts import (
|
||||
AcquiredDocument, EvidenceSourceError, EvidenceSourceErrorCategory, SourceObject,
|
||||
AcquiredDocument,
|
||||
EvidenceSourceError,
|
||||
EvidenceSourceErrorCategory,
|
||||
SourceObject,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -15,7 +15,6 @@ from tht.evidence.contracts import (
|
||||
validate_safe_metadata,
|
||||
)
|
||||
|
||||
|
||||
_NAMESPACED_ID = re.compile(r"^[a-z][a-z0-9_-]*:[A-Za-z0-9._:-]+$")
|
||||
_SHA256 = re.compile(r"^sha256:[0-9a-f]{64}$")
|
||||
|
||||
|
||||
@@ -10,9 +10,8 @@ from pydantic import JsonValue, TypeAdapter, ValidationError
|
||||
from yaml.events import AliasEvent
|
||||
from yaml.nodes import MappingNode
|
||||
|
||||
from tht.evidence.corpus.models import CanonicalDocument
|
||||
from tht.evidence.contracts import AcquiredDocument, canonical_provenance_uri
|
||||
|
||||
from tht.evidence.corpus.models import CanonicalDocument
|
||||
|
||||
MAX_DOCUMENT_BYTES = 10 * 1024 * 1024
|
||||
_CHARSET = re.compile(r"(?:^|;)\s*charset\s*=\s*[\"']?([^;\s\"']+)", re.IGNORECASE)
|
||||
|
||||
@@ -4,6 +4,7 @@ from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import uuid
|
||||
from collections.abc import Mapping, Sequence
|
||||
@@ -11,17 +12,16 @@ from dataclasses import asdict, dataclass, field
|
||||
from datetime import UTC
|
||||
from pathlib import Path
|
||||
|
||||
import tht.evidence.acquisition as evidence_acquisition
|
||||
from tht.evidence.contracts import EvidenceSource, SourceObject, canonical_provenance_uri
|
||||
from tht.evidence.corpus.chunk import ChunkPolicy, chunk
|
||||
from tht.evidence.corpus.models import CanonicalChunk, CanonicalDocument, CorpusManifest
|
||||
from tht.evidence.corpus.normalize import normalize
|
||||
from tht.evidence.corpus.store import CorpusStore
|
||||
import tht.evidence.acquisition as evidence_acquisition
|
||||
from tht.evidence.contracts import EvidenceSource, SourceObject, canonical_provenance_uri
|
||||
from tht.ports.vector import VectorStore, VectorWriteRecord
|
||||
from tht.vectorstore.records import VectorRecord
|
||||
from tht.jobs.models import JobSpec
|
||||
from tht.jobs.runner import JobContext, StageArtifacts, run_job, seal_stage_artifacts
|
||||
|
||||
from tht.ports.vector import VectorStore, VectorWriteRecord
|
||||
from tht.vectorstore.records import VectorRecord
|
||||
|
||||
EVIDENCE_STAGE_IDS = (
|
||||
"discover",
|
||||
@@ -33,6 +33,8 @@ EVIDENCE_STAGE_IDS = (
|
||||
"retention_cleanup",
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class PipelineError(RuntimeError):
|
||||
"""Credential-free failure at the preprocessing boundary."""
|
||||
@@ -212,14 +214,14 @@ class CorpusPipeline:
|
||||
if purge_vector:
|
||||
try:
|
||||
self.vector_store.delete_generation("evidence", generation, self.workspace_id)
|
||||
except Exception:
|
||||
except Exception: # noqa: BLE001 - retention reports per-generation failures
|
||||
failures.append({"generation": generation, "error": "vector cleanup failed"})
|
||||
continue
|
||||
try:
|
||||
if purge_filesystem:
|
||||
self.store.discard(generation)
|
||||
evicted.append(generation)
|
||||
except Exception:
|
||||
except Exception: # noqa: BLE001 - retention reports per-generation failures
|
||||
failures.append({"generation": generation, "error": "filesystem cleanup failed"})
|
||||
return {"status": "partial" if failures else "succeeded", "dry_run": dry_run,
|
||||
"active_generation": self.store.active_generation(), "evicted": evicted,
|
||||
@@ -378,7 +380,7 @@ class CorpusPipeline:
|
||||
|
||||
try:
|
||||
active_assets_valid = active_assets_are_valid(previous)
|
||||
except Exception:
|
||||
except Exception: # noqa: BLE001 - any corrupt active asset disables reuse
|
||||
active_assets_valid = False
|
||||
reusable = (
|
||||
active_assets_valid
|
||||
@@ -538,7 +540,7 @@ class CorpusPipeline:
|
||||
try:
|
||||
self.vector_store.delete_generation("evidence", generation, self.workspace_id)
|
||||
except Exception:
|
||||
pass
|
||||
logger.debug("Failed to clean the compensated vector generation", exc_info=True)
|
||||
write(context, "compensated.json", {"generation": generation})
|
||||
|
||||
def rotate_compensated_generation(context: JobContext) -> None:
|
||||
@@ -786,12 +788,12 @@ class CorpusPipeline:
|
||||
try:
|
||||
self.store.discard(generation)
|
||||
except Exception:
|
||||
pass
|
||||
logger.debug("Failed to discard the unpublished evidence generation", exc_info=True)
|
||||
if vector_written:
|
||||
try:
|
||||
self.vector_store.delete_generation("evidence", generation, self.workspace_id)
|
||||
except Exception:
|
||||
pass
|
||||
logger.debug("Failed to delete the unpublished vector generation", exc_info=True)
|
||||
|
||||
@staticmethod
|
||||
def _vector_record(
|
||||
|
||||
@@ -2,22 +2,21 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import fcntl
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import stat
|
||||
import shutil
|
||||
import uuid
|
||||
import hashlib
|
||||
import stat
|
||||
import threading
|
||||
import uuid
|
||||
from contextlib import contextmanager
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from contextlib import contextmanager
|
||||
|
||||
from tht.evidence.corpus.models import CorpusManifest
|
||||
|
||||
|
||||
_GENERATION = re.compile(r"^gen:[0-9a-f]{32}$")
|
||||
|
||||
|
||||
|
||||
@@ -46,7 +46,7 @@ class ConceptFormula(BaseModel):
|
||||
return f"---\n{fm}---\n{self.sql}\n"
|
||||
|
||||
@classmethod
|
||||
def parse(cls, text: str) -> "ConceptFormula":
|
||||
def parse(cls, text: str) -> ConceptFormula:
|
||||
if not text.startswith("---\n"):
|
||||
raise ValueError("frontmatter mancante (atteso '---\\n' iniziale)")
|
||||
try:
|
||||
@@ -55,7 +55,7 @@ class ConceptFormula(BaseModel):
|
||||
raise ValueError("frontmatter malformato") from e
|
||||
meta = yaml.safe_load(fm)
|
||||
if not isinstance(meta, dict):
|
||||
raise ValueError("frontmatter non valido")
|
||||
raise TypeError("frontmatter non valido")
|
||||
return cls.model_validate({**meta, "sql": body.strip("\n")})
|
||||
|
||||
|
||||
|
||||
@@ -2,10 +2,10 @@
|
||||
|
||||
from typing import Protocol
|
||||
|
||||
from tht.evidence.contracts import EvidenceSource
|
||||
from tht.evidence.corpus.chunk import ChunkPolicy
|
||||
from tht.evidence.corpus.pipeline import CorpusPipeline
|
||||
from tht.evidence.corpus.store import CorpusStore
|
||||
from tht.evidence.contracts import EvidenceSource
|
||||
from tht.ports.vector import VectorStore
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user