mptyl/ThothII
1
0
Fork 0
Code Issues 25 Pull Requests Actions Packages Projects Releases 1 Wiki Activity

fix(preprocess): harden crash recovery integrity

Browse Source
This commit is contained in:
marcopan
2026-07-12 04:49:15 +02:00
parent 1d5f8c76a7
commit c964920f16
7 changed files with 440 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files
harness/tht/corpus/pipeline.py
+68 -24
View File
@@ -16,7 +16,7 @@ from tht.ports.evidence import EvidenceSource, SourceObject
from tht.ports.vector import VectorStore, VectorWriteRecord
from tht.vectorstore.records import VectorRecord
from tht.jobs.models import JobSpec
from tht.jobs.runner import JobContext, run_job
from tht.jobs.runner import JobContext, StageArtifacts, run_job, seal_stage_artifacts
EVIDENCE_STAGE_IDS = (
@@ -96,6 +96,7 @@ class CorpusPipeline:
input_fingerprint: str,
dry_run: bool = False,
resume_run_id: str | None = None,
after_stage_return=None,
) -> PipelineResult:
"""Execute preprocessing through the durable shared job envelope."""
discovered = self._discover()
@@ -159,10 +160,11 @@ class CorpusPipeline:
"removed": removed,
"previous": previous.model_dump(mode="json") if previous else None,
})
return StageArtifacts(("plan.json",))
def acquire_stage(context: JobContext) -> None:
if context.dry_run:
return
return StageArtifacts()
plan = read(context, "plan.json")
previous = CorpusManifest.model_validate(plan["previous"]) if plan["previous"] else None
prior = {doc.source_id: doc for doc in previous.documents} if previous else {}
@@ -194,10 +196,11 @@ class CorpusPipeline:
},
)
write(context, "manifest.json", manifest.model_dump(mode="json"))
return StageArtifacts(("manifest.json",))
def embed_stage(context: JobContext) -> None:
if context.dry_run:
return
return StageArtifacts()
plan = read(context, "plan.json")
manifest = CorpusManifest.model_validate(read(context, "manifest.json"))
changed_docs = {doc.document_id for doc in manifest.documents if doc.source_id in plan["changed"]}
@@ -208,6 +211,7 @@ class CorpusPipeline:
):
raise PipelineError("embedding output is incompatible")
write(context, "embeddings.json", embeddings)
return StageArtifacts(("embeddings.json",))
def records(context: JobContext):
plan = read(context, "plan.json")
@@ -220,7 +224,8 @@ class CorpusPipeline:
def compensate(context: JobContext) -> None:
generation = read(context, "plan.json")["generation"]
self.store.discard(generation)
if self.store.active_generation() != generation:
self.store.discard(generation)
try:
self.vector_store.delete_generation("evidence", generation)
except Exception:
@@ -241,62 +246,101 @@ class CorpusPipeline:
for document in manifest.documents:
if document.source_id in changed and generations.get(document.document_id) == old:
generations[document.document_id] = plan["generation"]
metadata = dict(manifest.metadata)
metadata["document_generations"] = generations
manifest = manifest.model_copy(update={
"vector_generation": plan["generation"], "metadata": metadata,
})
manifest_payload = manifest.model_dump(mode="json")
manifest_payload["metadata"]["document_generations"] = generations
manifest_payload["vector_generation"] = plan["generation"]
manifest = CorpusManifest.model_validate(manifest_payload)
write(context, "manifest.json", manifest.model_dump(mode="json"))
marker.unlink()
def vector_stage(context: JobContext) -> None:
if context.dry_run:
return
return StageArtifacts()
rotate_compensated_generation(context)
values = records(context)
write(context, "vector-intent.json", {
"generation": read(context, "plan.json")["generation"],
"records": {value.record.id: value.content_hash for value in values},
})
seal_stage_artifacts(
context, "vector_upsert",
("plan.json", "manifest.json", "vector-intent.json"), spec,
)
try:
if values and self.vector_store.upsert("evidence", values) != len(values):
existing = self.vector_store.existing_hashes("evidence", ["evidence"])
missing = [
value for value in values
if existing.get(value.record.id) != value.content_hash
]
if missing and self.vector_store.upsert("evidence", missing) != len(missing):
raise PipelineError("vector write count mismatch")
except Exception:
compensate(context)
raise
return StageArtifacts(("plan.json", "manifest.json", "vector-intent.json"))
def stage_stage(context: JobContext) -> None:
if context.dry_run:
return
return StageArtifacts()
plan = read(context, "plan.json")
manifest = CorpusManifest.model_validate(read(context, "manifest.json"))
recovered = False
try:
self.store.stage(
manifest, {doc.document_id: doc.content for doc in manifest.documents},
generation=plan["generation"],
)
if artifact(context, "compensated.json").exists():
recovered = True
rotate_compensated_generation(context)
values = records(context)
existing = self.vector_store.existing_hashes("evidence", ["evidence"])
missing = [value for value in values if existing.get(value.record.id) != value.content_hash]
if missing and self.vector_store.upsert("evidence", missing) != len(missing):
raise PipelineError("vector write count mismatch")
plan = read(context, "plan.json")
manifest = CorpusManifest.model_validate(read(context, "manifest.json"))
if not self.store.generation_path(plan["generation"]).exists():
self.store.stage(
manifest, {doc.document_id: doc.content for doc in manifest.documents},
generation=plan["generation"],
)
self.store.manifest(plan["generation"])
except Exception:
compensate(context)
raise
return StageArtifacts(
("plan.json", "manifest.json", "vector-intent.json") if recovered else ()
)
def publish_stage(context: JobContext) -> None:
if context.dry_run:
return
return StageArtifacts()
if artifact(context, "compensated.json").exists():
rotate_compensated_generation(context)
values = records(context)
if values and self.vector_store.upsert("evidence", values) != len(values):
try:
existing = self.vector_store.existing_hashes("evidence", ["evidence"])
missing = [value for value in values if existing.get(value.record.id) != value.content_hash]
if missing and self.vector_store.upsert("evidence", missing) != len(missing):
raise PipelineError("vector write count mismatch")
except Exception:
compensate(context)
raise PipelineError("vector write count mismatch")
raise
manifest = CorpusManifest.model_validate(read(context, "manifest.json"))
generation = read(context, "plan.json")["generation"]
self.store.stage(
manifest, {doc.document_id: doc.content for doc in manifest.documents},
generation=generation,
)
try:
if not self.store.generation_path(generation).exists():
self.store.stage(
manifest, {doc.document_id: doc.content for doc in manifest.documents},
generation=generation,
)
except Exception:
compensate(context)
raise
generation = read(context, "plan.json")["generation"]
try:
self.store.publish(generation)
except Exception:
compensate(context)
raise
return StageArtifacts(("plan.json", "manifest.json", "vector-intent.json"))
def retention_stage(context: JobContext) -> None:
# Retention policy is intentionally a stable no-op until configured.
@@ -305,7 +349,7 @@ class CorpusPipeline:
report = run_job(spec, [
discover_stage, acquire_stage, embed_stage, vector_stage,
stage_stage, publish_stage, retention_stage,
])
], after_stage_return=after_stage_return)
run_dir = workspace_root / ".tht-jobs" / "evidence" / "runs" / report.run_id
plan = json.loads((run_dir / "artifacts" / "plan.json").read_text())
if dry_run:
harness/tht/corpus/store.py
+24 -3
View File
@@ -46,6 +46,7 @@ class CorpusStore:
self.root = Path(root)
self.active_path = self.root / "ACTIVE"
self._replace = os.replace
self._fsync_directory = self._sync_root
self._ensure_root()
def _ensure_root(self) -> None:
@@ -110,15 +111,35 @@ class CorpusStore:
manifest = self.manifest(generation)
if manifest.manifest_id != generation:
raise UnsafeCorpusPath("manifest generation mismatch")
if self.active_generation() == generation:
return generation
previous = self.active_generation()
temporary = self.active_path.with_name(f".ACTIVE.{uuid.uuid4().hex}.tmp")
_atomic_write(temporary, (generation + "\n").encode())
self._replace(temporary, self.active_path)
replaced = False
try:
_atomic_write(temporary, (generation + "\n").encode())
self._replace(temporary, self.active_path)
replaced = True
self._fsync_directory()
except BaseException:
temporary.unlink(missing_ok=True)
if replaced:
if previous is None:
self.active_path.unlink(missing_ok=True)
else:
rollback = self.active_path.with_name(f".ACTIVE.rollback.{uuid.uuid4().hex}.tmp")
_atomic_write(rollback, (previous + "\n").encode())
self._replace(rollback, self.active_path)
self._sync_root()
raise
return generation
def _sync_root(self) -> None:
directory = os.open(self.root, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
try:
os.fsync(directory)
finally:
os.close(directory)
return generation
def active_generation(self) -> str | None:
try: