fix(jobs): harden resume locks and durability

harness/tht/jobs/locking.py

@@ -6,6 +6,7 @@ import fcntl
 import hashlib
 import os
 import re
+import stat
 from pathlib import Path
 from types import TracebackType
 
@@ -36,13 +37,42 @@ class WorkspaceJobLock:
     def acquire(self) -> "WorkspaceJobLock":
         if self._fd is not None:
             raise RuntimeError("job lock is already held by this object")
-        self.path.parent.mkdir(parents=True, exist_ok=True)
-        fd = os.open(self.path, os.O_RDWR | os.O_CREAT, 0o600)
+        root_fd = os.open(self.path.parents[2], os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
         try:
-            fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
-        except BlockingIOError as error:
-            os.close(fd)
-            raise JobAlreadyRunningError("this workspace job is already running") from error
+            jobs_fd = _open_owned_directory(root_fd, ".tht-jobs")
+            try:
+                locks_fd = _open_owned_directory(jobs_fd, ".locks")
+                try:
+                    fd = os.open(
+                        self.path.name,
+                        os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW | os.O_CLOEXEC,
+                        0o600,
+                        dir_fd=locks_fd,
+                    )
+                    try:
+                        info = os.fstat(fd)
+                        if (
+                            not stat.S_ISREG(info.st_mode)
+                            or info.st_uid != os.getuid()
+                            or info.st_nlink != 1
+                        ):
+                            raise OSError("unsafe job lock file")
+                        os.fchmod(fd, 0o600)
+                        try:
+                            fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
+                        except BlockingIOError as error:
+                            raise JobAlreadyRunningError(
+                                "this workspace job is already running"
+                            ) from error
+                    except BaseException:
+                        os.close(fd)
+                        raise
+                finally:
+                    os.close(locks_fd)
+            finally:
+                os.close(jobs_fd)
+        finally:
+            os.close(root_fd)
         self._fd = fd
         return self
 
@@ -65,3 +95,21 @@ class WorkspaceJobLock:
         traceback: TracebackType | None,
     ) -> None:
         self.release()
+
+
+def _open_owned_directory(parent_fd: int, name: str) -> int:
+    try:
+        os.mkdir(name, 0o700, dir_fd=parent_fd)
+        os.fsync(parent_fd)
+    except FileExistsError:
+        pass
+    fd = os.open(name, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=parent_fd)
+    try:
+        info = os.fstat(fd)
+        if not stat.S_ISDIR(info.st_mode) or info.st_uid != os.getuid():
+            raise OSError("unsafe job lock directory")
+        os.fchmod(fd, 0o700)
+    except BaseException:
+        os.close(fd)
+        raise
+    return fd