fix(dwh): release snapshots before job startup

harness/tht/jobs/dwh_pipeline.py

@@ -601,19 +601,23 @@ class DwhPreprocessPipeline:
                 self.current_lsh_dir = snapshot_root
         finally:
             lease.close()
-        if resume_run_id is not None:
-            self._validate_resume_publication(resume_run_id)
-        spec = JobSpec(
-            workspace_id=self.workspace_id,
-            job_type="dwh",
-            workspace_root=self.workspace_root,
-            spec_version="jobs-v1",
-            pipeline_version="dwh-v2",
-            config_fingerprint=self.config_fingerprint,
-            input_fingerprint=self.input_fingerprint,
-            stage_ids=steps,
-            resume_run_id=resume_run_id,
-        )
+        try:
+            if resume_run_id is not None:
+                self._validate_resume_publication(resume_run_id)
+            spec = JobSpec(
+                workspace_id=self.workspace_id,
+                job_type="dwh",
+                workspace_root=self.workspace_root,
+                spec_version="jobs-v1",
+                pipeline_version="dwh-v2",
+                config_fingerprint=self.config_fingerprint,
+                input_fingerprint=self.input_fingerprint,
+                stage_ids=steps,
+                resume_run_id=resume_run_id,
+            )
+        except BaseException:
+            self._release_pipeline_snapshot()
+            raise
 
         def introspect_stage(context: JobContext):
             artifacts = self._artifacts(context)
@@ -651,16 +655,19 @@ class DwhPreprocessPipeline:
                 reconcile_effects=self._reconcile_effects,
             )
         finally:
-            holder, self._snapshot_holder = self._snapshot_holder, None
-            if isinstance(holder, Path):
-                shutil.rmtree(holder, ignore_errors=True)
-                _SNAPSHOT_DIRS.discard(holder)
-                if self.current_physical is not None and holder in self.current_physical.parents:
-                    self.current_physical = None
-                if self.current_lsh_dir is not None and (
-                    self.current_lsh_dir == holder or holder in self.current_lsh_dir.parents
-                ):
-                    self.current_lsh_dir = None
+            self._release_pipeline_snapshot()
+
+    def _release_pipeline_snapshot(self) -> None:
+        holder, self._snapshot_holder = self._snapshot_holder, None
+        if isinstance(holder, Path):
+            shutil.rmtree(holder, ignore_errors=True)
+            _SNAPSHOT_DIRS.discard(holder)
+            if self.current_physical is not None and holder in self.current_physical.parents:
+                self.current_physical = None
+            if self.current_lsh_dir is not None and (
+                self.current_lsh_dir == holder or holder in self.current_lsh_dir.parents
+            ):
+                self.current_lsh_dir = None
 
     def _reconcile_effects(self, source, run_dir: Path) -> set[str]:
         running = next(