Chiusura fase di ristrutturazione e modularizzazione del workflow per favorire sviluppo modulare
This commit is contained in:
@@ -4,6 +4,7 @@ from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import uuid
|
||||
from collections.abc import Mapping, Sequence
|
||||
@@ -11,17 +12,16 @@ from dataclasses import asdict, dataclass, field
|
||||
from datetime import UTC
|
||||
from pathlib import Path
|
||||
|
||||
import tht.evidence.acquisition as evidence_acquisition
|
||||
from tht.evidence.contracts import EvidenceSource, SourceObject, canonical_provenance_uri
|
||||
from tht.evidence.corpus.chunk import ChunkPolicy, chunk
|
||||
from tht.evidence.corpus.models import CanonicalChunk, CanonicalDocument, CorpusManifest
|
||||
from tht.evidence.corpus.normalize import normalize
|
||||
from tht.evidence.corpus.store import CorpusStore
|
||||
import tht.evidence.acquisition as evidence_acquisition
|
||||
from tht.evidence.contracts import EvidenceSource, SourceObject, canonical_provenance_uri
|
||||
from tht.ports.vector import VectorStore, VectorWriteRecord
|
||||
from tht.vectorstore.records import VectorRecord
|
||||
from tht.jobs.models import JobSpec
|
||||
from tht.jobs.runner import JobContext, StageArtifacts, run_job, seal_stage_artifacts
|
||||
|
||||
from tht.ports.vector import VectorStore, VectorWriteRecord
|
||||
from tht.vectorstore.records import VectorRecord
|
||||
|
||||
EVIDENCE_STAGE_IDS = (
|
||||
"discover",
|
||||
@@ -33,6 +33,8 @@ EVIDENCE_STAGE_IDS = (
|
||||
"retention_cleanup",
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class PipelineError(RuntimeError):
|
||||
"""Credential-free failure at the preprocessing boundary."""
|
||||
@@ -212,14 +214,14 @@ class CorpusPipeline:
|
||||
if purge_vector:
|
||||
try:
|
||||
self.vector_store.delete_generation("evidence", generation, self.workspace_id)
|
||||
except Exception:
|
||||
except Exception: # noqa: BLE001 - retention reports per-generation failures
|
||||
failures.append({"generation": generation, "error": "vector cleanup failed"})
|
||||
continue
|
||||
try:
|
||||
if purge_filesystem:
|
||||
self.store.discard(generation)
|
||||
evicted.append(generation)
|
||||
except Exception:
|
||||
except Exception: # noqa: BLE001 - retention reports per-generation failures
|
||||
failures.append({"generation": generation, "error": "filesystem cleanup failed"})
|
||||
return {"status": "partial" if failures else "succeeded", "dry_run": dry_run,
|
||||
"active_generation": self.store.active_generation(), "evicted": evicted,
|
||||
@@ -378,7 +380,7 @@ class CorpusPipeline:
|
||||
|
||||
try:
|
||||
active_assets_valid = active_assets_are_valid(previous)
|
||||
except Exception:
|
||||
except Exception: # noqa: BLE001 - any corrupt active asset disables reuse
|
||||
active_assets_valid = False
|
||||
reusable = (
|
||||
active_assets_valid
|
||||
@@ -538,7 +540,7 @@ class CorpusPipeline:
|
||||
try:
|
||||
self.vector_store.delete_generation("evidence", generation, self.workspace_id)
|
||||
except Exception:
|
||||
pass
|
||||
logger.debug("Failed to clean the compensated vector generation", exc_info=True)
|
||||
write(context, "compensated.json", {"generation": generation})
|
||||
|
||||
def rotate_compensated_generation(context: JobContext) -> None:
|
||||
@@ -786,12 +788,12 @@ class CorpusPipeline:
|
||||
try:
|
||||
self.store.discard(generation)
|
||||
except Exception:
|
||||
pass
|
||||
logger.debug("Failed to discard the unpublished evidence generation", exc_info=True)
|
||||
if vector_written:
|
||||
try:
|
||||
self.vector_store.delete_generation("evidence", generation, self.workspace_id)
|
||||
except Exception:
|
||||
pass
|
||||
logger.debug("Failed to delete the unpublished vector generation", exc_info=True)
|
||||
|
||||
@staticmethod
|
||||
def _vector_record(
|
||||
|
||||
Reference in New Issue
Block a user