Compare commits
575
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
2f53512e4d | ||
|
|
497ab84031 | ||
|
|
0d2e573e0d | ||
|
|
bd416f7327 | ||
|
|
23e52c80de | ||
|
|
84084bba37 | ||
|
|
efd7d788d9 | ||
|
|
b1c510a097 | ||
|
|
5f3680a0fb | ||
|
|
4ff91e8d6e | ||
|
|
5f3a7f5975 | ||
|
|
043ffdfad6 | ||
|
|
6a4634dcf1 | ||
|
|
84804be9f8 | ||
|
|
c3caba94dd | ||
|
|
b1723c34c4 | ||
|
|
d6cdffea62 | ||
|
|
49333a2d35 | ||
|
|
b006b94479 | ||
|
|
bdcd8fcd28 | ||
|
|
cf90c1bd51 | ||
|
|
571a4bcaa2 | ||
|
|
9051463654 | ||
|
|
26c5605ff7 | ||
|
|
3535fda958 | ||
|
|
2953f6b608 | ||
|
|
45db3a239b | ||
|
|
7d826e46c0 | ||
|
|
648434a32e | ||
|
|
023b822f83 | ||
|
|
d8a29bfbdd | ||
|
|
a59624a68f | ||
|
|
5af4408194 | ||
|
|
e088abd60a | ||
|
|
803e9e9201 | ||
|
|
8c81996896 | ||
|
|
eed398e569 | ||
|
|
c8d276ddc6 | ||
|
|
2d1b714ebe | ||
|
|
c7e5f295e6 | ||
|
|
f52bf22e05 | ||
|
|
840344706f | ||
|
|
41b9fed4d5 | ||
|
|
36bf659ea9 | ||
|
|
4a67d60233 | ||
|
|
debb63d87b | ||
|
|
3943022a97 | ||
|
|
f5ec2d9313 | ||
|
|
82e2c91f42 | ||
|
|
8fe526dd6e | ||
|
|
e68e80a33d | ||
|
|
818563c408 | ||
|
|
50c546e42d | ||
|
|
651a5c7902 | ||
|
|
28db30bd78 | ||
|
|
cffa60772e | ||
|
|
8707ae1d46 | ||
|
|
ad744f0212 | ||
|
|
eba6148511 | ||
|
|
7b1d69a65b | ||
|
|
b891246664 | ||
|
|
8e778b9edb | ||
|
|
f114d0065a | ||
|
|
7b87e95427 | ||
|
|
a50475d687 | ||
|
|
a6a5bf2036 | ||
|
|
ce4c31a6fb | ||
|
|
538dc8ef56 | ||
|
|
7b7927bfe5 | ||
|
|
ae053961a3 | ||
|
|
4531746038 | ||
|
|
076c9742c5 | ||
|
|
f586152636 | ||
|
|
64fbe642ef | ||
|
|
74d5a3c0c8 | ||
|
|
7a0697e562 | ||
|
|
ea903b5bd3 | ||
|
|
218e50f124 | ||
|
|
e82fe8d322 | ||
|
|
b4b97436e1 | ||
|
|
ded66fde9c | ||
|
|
9c697dc062 | ||
|
|
919d408c81 | ||
|
|
fa7380b3a3 | ||
|
|
866aee4249 | ||
|
|
16b7477861 | ||
|
|
7cbbfbce91 | ||
|
|
374e8aabc8 | ||
|
|
57928347b3 | ||
|
|
e2b88ce3c2 | ||
|
|
cb40c09d9a | ||
|
|
0736983bc5 | ||
|
|
6278ee9d81 | ||
|
|
0ce05869cf | ||
|
|
d504b1def1 | ||
|
|
376dd5a09d | ||
|
|
b0afba81ca | ||
|
|
7f968359ac | ||
|
|
58ee9cffe4 | ||
|
|
79c4c925b5 | ||
|
|
705af3aeb2 | ||
|
|
9189450fa9 | ||
|
|
410cb547d2 | ||
|
|
a701b19a03 | ||
|
|
11f78c4467 | ||
|
|
8b5892a5a1 | ||
|
|
9898726069 | ||
|
|
9d4f994d3e | ||
|
|
38f02cfd08 | ||
|
|
e910c7d49c | ||
|
|
7d32bb1e74 | ||
|
|
a54d4769dd | ||
|
|
23bc2f6555 | ||
|
|
c1290c782c | ||
|
|
a8cde2217f | ||
|
|
29326b064f | ||
|
|
d991dc2fd1 | ||
|
|
f48196a57f | ||
|
|
ec061c42d4 | ||
|
|
7d0d2b2edc | ||
|
|
dbd7787573 | ||
|
|
71a42fbe80 | ||
|
|
f16c248019 | ||
|
|
a0620ffffa | ||
|
|
d638473c12 | ||
|
|
9a62fce1ad | ||
|
|
dfc605da78 | ||
|
|
663c60dc3e | ||
|
|
3af59cecbf | ||
|
|
8b524b4314 | ||
|
|
737ec879d9 | ||
|
|
0df95e337e | ||
|
|
52caabf83b | ||
|
|
a3259ced99 | ||
|
|
20341a7124 | ||
|
|
2b6bb058d8 | ||
|
|
a1f1a63c88 | ||
|
|
73efeb7f3d | ||
|
|
5877adcfac | ||
|
|
287ce91e67 | ||
|
|
2ae11f26f2 | ||
|
|
12d257056f | ||
|
|
4c2c9b50a9 | ||
|
|
e51a6a2253 | ||
|
|
944b0edf7a | ||
|
|
d49c644b61 | ||
|
|
66f9fa2821 | ||
|
|
4499869c75 | ||
|
|
886faed886 | ||
|
|
4606ec19a9 | ||
|
|
6afb5d242b | ||
|
|
97c6788f16 | ||
|
|
a6655a5e2c | ||
|
|
fb4a1fa25b | ||
|
|
a8a80b9846 | ||
|
|
73b784a176 | ||
|
|
fd878b8c3e | ||
|
|
db375298d0 | ||
|
|
2c2bf1940b | ||
|
|
abfcfb0e06 | ||
|
|
f0e78b1ce3 | ||
|
|
8980c35198 | ||
|
|
6d0cb6d997 | ||
|
|
06b26cf66f | ||
|
|
5b6fec939a | ||
|
|
3e106d5262 | ||
|
|
8ba87b68dc | ||
|
|
610ae8c85a | ||
|
|
e9c65ef2db | ||
|
|
d4818c8cc3 | ||
|
|
8542124f27 | ||
|
|
fc83d29b58 | ||
|
|
0caa747914 | ||
|
|
9f0184388d | ||
|
|
619ac2e141 | ||
|
|
dcb5acc312 | ||
|
|
cc30148b69 | ||
|
|
d5d65f3659 | ||
|
|
f1a9b567ba | ||
|
|
3420c57c8b | ||
|
|
8adc085746 | ||
|
|
f5c7cc6198 | ||
|
|
6176410f42 | ||
|
|
29d41ac258 | ||
|
|
0e9add09a9 | ||
|
|
ae0976a4aa | ||
|
|
5c6228f8c2 | ||
|
|
d970e10264 | ||
|
|
6062cb010e | ||
|
|
fa2298653b | ||
|
|
36a7a0ab33 | ||
|
|
f375515dc0 | ||
|
|
840848df94 | ||
|
|
44f1efa5ba | ||
|
|
d5e78febd3 | ||
|
|
8b63715b56 | ||
|
|
1e459b073e | ||
|
|
2ed55ef131 | ||
|
|
eccf6212f1 | ||
|
|
4a654de84a | ||
|
|
93fe0d733b | ||
|
|
beac2e80f4 | ||
|
|
d15bb59c3d | ||
|
|
dc9726cb35 | ||
|
|
b5db0cd3c1 | ||
|
|
bc1b5e79ca | ||
|
|
09f43139d3 | ||
|
|
c476c5551b | ||
|
|
3fdc278e7a | ||
|
|
80b3575b2e | ||
|
|
120816d81c | ||
|
|
35aafc5e4c | ||
|
|
9c47fb5f3d | ||
|
|
309bc44d4a | ||
|
|
f79b89b026 | ||
|
|
6475ea39f9 | ||
|
|
912bab95b6 | ||
|
|
5c505d4c85 | ||
|
|
3d02c5c1ef | ||
|
|
abd68470da | ||
|
|
b774a371f7 | ||
|
|
cc72e65f9d | ||
|
|
1f75b81214 | ||
|
|
ef7ae7053c | ||
|
|
3d9a9f0675 | ||
|
|
903c0b4de5 | ||
|
|
1e2c4e65c5 | ||
|
|
da2f4a6681 | ||
|
|
05f8615887 | ||
|
|
de86760942 | ||
|
|
f9e2950262 | ||
|
|
8a8f2c2174 | ||
|
|
64f46c7019 | ||
|
|
a21e2c156d | ||
|
|
042af932ee | ||
|
|
9974fb4bc0 | ||
|
|
7118950416 | ||
|
|
0c4ff3750d | ||
|
|
dee0f9cb25 | ||
|
|
6fb48866b8 | ||
|
|
6499d24892 | ||
|
|
09290a0fe7 | ||
|
|
7fe53164ba | ||
|
|
7b86ea9ce5 | ||
|
|
f616aab542 | ||
|
|
707c13d781 | ||
|
|
7b9b8b308d | ||
|
|
d0f7e0497d | ||
|
|
1b18a0fb25 | ||
|
|
62ec29ff92 | ||
|
|
87606c73c1 | ||
|
|
134dc1977c | ||
|
|
419c3440d7 | ||
|
|
943f809d01 | ||
|
|
b1079bda21 | ||
|
|
055dcabf36 | ||
|
|
ebb360f700 | ||
|
|
e90a1a1851 | ||
|
|
971a0e66b1 | ||
|
|
d2415b5429 | ||
|
|
541ef45529 | ||
|
|
1e82fd33fd | ||
|
|
3bc84b0bc3 | ||
|
|
4ef0a6a833 | ||
|
|
8a1c23536f | ||
|
|
b0ead4be54 | ||
|
|
a1771b6bb9 | ||
|
|
550b096253 | ||
|
|
58e102a166 | ||
|
|
cace697fea | ||
|
|
be4a72eec2 | ||
|
|
4adc3d8bdf | ||
|
|
afb0831e40 | ||
|
|
4ebd2b77a3 | ||
|
|
5f0575464b | ||
|
|
21caaa22e3 | ||
|
|
3fd177b6c4 | ||
|
|
5c0dc8c9f2 | ||
|
|
89fc48dbe0 | ||
|
|
125feee778 | ||
|
|
1184b6db16 | ||
|
|
32a17d83a9 | ||
|
|
361d55a9c2 | ||
|
|
2a93590712 | ||
|
|
0f762ad6b6 | ||
|
|
10cd66fe6a | ||
|
|
b48e9e9189 | ||
|
|
b261dd4d3a | ||
|
|
feee4ee648 | ||
|
|
9fc1a15069 | ||
|
|
b652011bfc | ||
|
|
c01482c52d | ||
|
|
6474118ec3 | ||
|
|
2d1670e390 | ||
|
|
455fffb299 | ||
|
|
824245d285 | ||
|
|
afb0e21200 | ||
|
|
ada3f9fc7f | ||
|
|
b6396e66fa | ||
|
|
fa197498f2 | ||
|
|
aee5863112 | ||
|
|
b40f8e885a | ||
|
|
835b298212 | ||
|
|
93f9939be5 | ||
|
|
def45d1628 | ||
|
|
c863244a86 | ||
|
|
81077e59d3 | ||
|
|
36592634bb | ||
|
|
4f07b26f5e | ||
|
|
747b4200d9 | ||
|
|
a0e05ad392 | ||
|
|
cd5f505c8a | ||
|
|
fa499a9bdd | ||
|
|
b31b27e584 | ||
|
|
0d8e707533 | ||
|
|
5f9a3ae066 | ||
|
|
d43738eeae | ||
|
|
3c6ddfaef1 | ||
|
|
1d045a44a6 | ||
|
|
178113a7e8 | ||
|
|
39b5453287 | ||
|
|
d7cd8fdf94 | ||
|
|
74b062f1a7 | ||
|
|
e7a7f4f066 | ||
|
|
0b77d1e850 | ||
|
|
54698e7340 | ||
|
|
df00f6bfa8 | ||
|
|
cb1bd101f0 | ||
|
|
e20bf33e2a | ||
|
|
4d230b87af | ||
|
|
fe190e7046 | ||
|
|
225ffc8e20 | ||
|
|
b2772f10a5 | ||
|
|
2b618c5a0f | ||
|
|
8a3fa5031d | ||
|
|
7cf7d9db6b | ||
|
|
91925d64bf | ||
|
|
f4f38717e1 | ||
|
|
6ec5b76c54 | ||
|
|
39a0fdbd00 | ||
|
|
dee17893b4 | ||
|
|
0651f3316f | ||
|
|
e8b9995ed0 | ||
|
|
0d0c15b4b8 | ||
|
|
7e52df2702 | ||
|
|
9558eaa508 | ||
|
|
0a2c667231 | ||
|
|
d12b629420 | ||
|
|
af16464e68 | ||
|
|
ef244ab56d | ||
|
|
30ee9433dc | ||
|
|
3ed00ff086 | ||
|
|
cb0e873ed7 | ||
|
|
6d438f4c7e | ||
|
|
be1724890a | ||
|
|
7cfbee36fa | ||
|
|
f0ae680671 | ||
|
|
33ae7cfdc2 | ||
|
|
c86f01e886 | ||
|
|
3e7bb11313 | ||
|
|
bdabecbb63 | ||
|
|
8573500121 | ||
|
|
4fe51cbeb1 | ||
|
|
202822f3ba | ||
|
|
8c67cb75dc | ||
|
|
09e546c1ef | ||
|
|
94c2cd3709 | ||
|
|
c34e01e9b9 | ||
|
|
29bfb41363 | ||
|
|
8477a69a29 | ||
|
|
7d9ca13f1d | ||
|
|
c9b02fc57e | ||
|
|
6bf8218fea | ||
|
|
ab61c1ad6d | ||
|
|
b80a2cc1e9 | ||
|
|
de8844a4ac | ||
|
|
1ed1a34ae2 | ||
|
|
5eed4f9449 | ||
|
|
856ac05edc | ||
|
|
9646ae09a0 | ||
|
|
d17ad0e95b | ||
|
|
25ee8b87d1 | ||
|
|
f6e4dbcae2 | ||
|
|
f99bddcdf0 | ||
|
|
2e0489ce22 | ||
|
|
23ac75ce1d | ||
|
|
e54ce15426 | ||
|
|
a66ef58766 | ||
|
|
d464f3a982 | ||
|
|
9196639cb3 | ||
|
|
351361f72f | ||
|
|
7651b63cea | ||
|
|
91650d9c67 | ||
|
|
69dca0820b | ||
|
|
cbb18acc4d | ||
|
|
6618a9d7c2 | ||
|
|
e4d0097639 | ||
|
|
11fbf0a138 | ||
|
|
3e0c864c85 | ||
|
|
5a657a67e8 | ||
|
|
0cb6a3f915 | ||
|
|
7ac99b4f7e | ||
|
|
9a8755e4c3 | ||
|
|
ea70cc95b0 | ||
|
|
47e0b2a385 | ||
|
|
2856964654 | ||
|
|
6dc9e80564 | ||
|
|
2c800cc521 | ||
|
|
66614fe8cf | ||
|
|
4fc03574e1 | ||
|
|
a2d5a244de | ||
|
|
5e13396669 | ||
|
|
aa8a2e9278 | ||
|
|
460caa550c | ||
|
|
a02ac52836 | ||
|
|
c9c30563f2 | ||
|
|
900faad983 | ||
|
|
225d13cd75 | ||
|
|
99012f04c2 | ||
|
|
1df881d98d | ||
|
|
dc83a55555 | ||
|
|
a8623a2b18 | ||
|
|
59a04123ea | ||
|
|
a35f52790d | ||
|
|
3a1f2ede4a | ||
|
|
9414a4b4dd | ||
|
|
bd42aa5934 | ||
|
|
ab33e0ed0a | ||
|
|
422f1d47b4 | ||
|
|
a94df262f4 | ||
|
|
fd3b62ce6f | ||
|
|
3978008aed | ||
|
|
e902f758b1 | ||
|
|
ae1215dc98 | ||
|
|
a227cbe755 | ||
|
|
87cefd120c | ||
|
|
2114c94704 | ||
|
|
e4999c8420 | ||
|
|
1f6a49b985 | ||
|
|
747020a330 | ||
|
|
9db4463a83 | ||
|
|
42e02f8b1c | ||
|
|
3a50c447c3 | ||
|
|
870af3422b | ||
|
|
f8117e8428 | ||
|
|
db1a81cfa6 | ||
|
|
25b2835e73 | ||
|
|
10edca5a09 | ||
|
|
6a61c42b88 | ||
|
|
a72ae2549a | ||
|
|
9d7e9a05b7 | ||
|
|
3a3efc3285 | ||
|
|
9ca01c77de | ||
|
|
55567f17f2 | ||
|
|
378aa6e652 | ||
|
|
f9d23d2361 | ||
|
|
f31b1e61ac | ||
|
|
84b233d937 | ||
|
|
3046ac34c6 | ||
|
|
aa22183ac7 | ||
|
|
ecd986f208 | ||
|
|
1dcf4051b0 | ||
|
|
486e144fcd | ||
|
|
be0e68e77d | ||
|
|
124891bbfe | ||
|
|
f09ab2b8c6 | ||
|
|
871de800f0 | ||
|
|
bb2eabceb6 | ||
|
|
0c9e614100 | ||
|
|
0c1033889a | ||
|
|
e80a8b35ec | ||
|
|
9605f77acb | ||
|
|
06a31e10b9 | ||
|
|
9db0299063 | ||
|
|
ccc3dc772e | ||
|
|
239d1c634f | ||
|
|
55a61931b1 | ||
|
|
d751188db7 | ||
|
|
0459a6cd3e | ||
|
|
5249798c03 | ||
|
|
259d5a0374 | ||
|
|
b00b7f17c9 | ||
|
|
60e4048d4b | ||
|
|
3397911670 | ||
|
|
e056c19e62 | ||
|
|
230a876314 | ||
|
|
4912b49f28 | ||
|
|
3b0726472e | ||
|
|
a1cd44b771 | ||
|
|
1cec3da838 | ||
|
|
795a29588d | ||
|
|
ae2f898122 | ||
|
|
96969a83d1 | ||
|
|
3e9940b4f0 | ||
|
|
6feb96270b | ||
|
|
c4063eecb6 | ||
|
|
c60b959697 | ||
|
|
aeb2329717 | ||
|
|
d0f1f24683 | ||
|
|
d1cb1ff6b3 | ||
|
|
e23e526966 | ||
|
|
beaba548c1 | ||
|
|
13f74de4fa | ||
|
|
3c5c2e8afd | ||
|
|
de5de36f9a | ||
|
|
7aaffe6e68 | ||
|
|
ec5ed413f7 | ||
|
|
de479b8b38 | ||
|
|
ff9c87bfc7 | ||
|
|
e197ba1fd4 | ||
|
|
79e592894d | ||
|
|
da214820b0 | ||
|
|
498a93d915 | ||
|
|
2ff05684a4 | ||
|
|
657c117425 | ||
|
|
0986125327 | ||
|
|
ff75742be9 | ||
|
|
f00cc289d8 | ||
|
|
c44b70e77e | ||
|
|
82ba3c7db7 | ||
|
|
0be6549b31 | ||
|
|
ca552451a4 | ||
|
|
3517cd8724 | ||
|
|
2cbb1a5496 | ||
|
|
920642673e | ||
|
|
742c311660 | ||
|
|
c994b3ef68 | ||
|
|
93017b5eb6 | ||
|
|
89b4943170 | ||
|
|
340d673e75 | ||
|
|
f6c2e6fc40 | ||
|
|
e3dd5897f8 | ||
|
|
ad3c3125a8 | ||
|
|
82b5453c88 | ||
|
|
929f7dcc5e | ||
|
|
632c1c1612 | ||
|
|
5aadc85808 | ||
|
|
99d1400ace | ||
|
|
fe49f5b6d8 | ||
|
|
436d8d2720 | ||
|
|
e34b756052 | ||
|
|
62d05aa614 | ||
|
|
85aa6acdf3 | ||
|
|
4d23ad4e85 | ||
|
|
16eb3d120d | ||
|
|
25fd29caf9 | ||
|
|
7951891561 | ||
|
|
908c99f90a | ||
|
|
077e648191 | ||
|
|
c5f65d0f15 | ||
|
|
17f2e48463 | ||
|
|
f7c2b69837 | ||
|
|
ca391ba59c | ||
|
|
3cfc8c53e6 | ||
|
|
d927233210 | ||
|
|
da448a3166 | ||
|
|
eac472011e | ||
|
|
10862bc700 | ||
|
|
01090b5be7 | ||
|
|
e06d31aeac | ||
|
|
412c0d8715 | ||
|
|
7ce25894a2 | ||
|
|
f0a19a89eb | ||
|
|
a22d232aa2 | ||
|
|
930335a804 | ||
|
|
1790d2449f | ||
|
|
d9fd902d08 | ||
|
|
c2f33e58d7 | ||
|
|
a071e0baff | ||
|
|
5446885006 | ||
|
|
25ec236f1f | ||
|
|
86af45acb4 | ||
|
|
7356d6794b | ||
|
|
66f44b054f | ||
|
|
5310c6555b |
@@ -1,66 +0,0 @@
|
||||
# Task 9 quality audit — final 5
|
||||
|
||||
**Scope:** the two blocking findings from `task9-quality-audit-final4.md` — unbound production
|
||||
module graph at manual serve, and commit-addressed snapshots accepted without content identity at
|
||||
render. Manual acceptance remains **PENDING**; no `VERDICT.md` was created.
|
||||
|
||||
## Verdict: APPROVED for the two final integrity blockers
|
||||
|
||||
### 1. Manual serve binds the complete `backend/dist` module graph, not only `server.js`
|
||||
|
||||
`prepare` now builds a post-build manifest of every regular `backend/dist` file
|
||||
(relative path, size, SHA-256, device, inode) and writes it as an exclusive `0600` record
|
||||
(`installation/runtime/backend-dist.manifest.json`) inside the owned root; `ownership.json`
|
||||
records that record's path/device/inode/size/SHA-256. `serve` revalidates the manifest record
|
||||
identity and bytes, revalidates every distribution file against it (no-follow, single inode,
|
||||
size and digest), and refuses before spawning. The manifest descriptor is passed to the child on
|
||||
fd 4 together with the entrypoint on fd 3. The immutable preload parses the manifest, verifies
|
||||
the entrypoint cross-digest, reads and hash-verifies **every** file at startup, caches the
|
||||
verified bytes, and its load hook serves **only** those cached bytes for any import below
|
||||
`backend/dist` (entry URL still served from the bound fd-3 bytes). A same-path regular
|
||||
replacement of any imported dependency is therefore refused before `RUNNING` (serve-time
|
||||
validation), refused at child startup (startup verification), or rendered harmless (cached
|
||||
bytes), and the parent revalidates the full manifest at `RUNNING` publication and at `stop`.
|
||||
|
||||
### 2. Renderer binds snapshot content to its commit identity
|
||||
|
||||
The generated render command validates the bounded saved read/publish revisions, the
|
||||
commit-addressed owned snapshot path, the installed Git HEAD, and the bounded
|
||||
`snapshot.json` manifest of that commit: `head` equals the commit, `files[<id>.yaml]` is the
|
||||
SHA-256 of the snapshot bytes, the manifest revision binds commit/blob/snapshot path, the saved
|
||||
revision blob equals the manifest blob, and `git rev-parse <commit>:workspaces/<id>.yaml` plus
|
||||
`git hash-object` of the snapshot bytes both equal that blob. It passes the expected digest as
|
||||
`--snapshot-sha256`. The renderer re-reads the bounded `snapshot.json` (`head`,
|
||||
`files[<id>.yaml]` must equal the carried digest), opens the snapshot once with no-follow
|
||||
semantics and bounded reads, renders only the digest-verified bytes, re-verifies around lease
|
||||
publication, releases the lease in `finally`, and publishes no output on any refusal.
|
||||
|
||||
## Deterministic regressions added
|
||||
|
||||
- static regular replacement of an imported production dependency after `prepare` is refused,
|
||||
no marker, no accepted PID record, no orphan;
|
||||
- deterministic dependency check/load swap (`beforeSpawn` rename) is refused by the child's
|
||||
startup verification, no marker, no PID record, no orphan;
|
||||
- after `RUNNING`, a same-path regular dependency replacement is never executed: the loader
|
||||
serves the verified cached bytes (health-visible source stays the original) and the marker is
|
||||
absent;
|
||||
- renderer refuses a same-path regular snapshot byte replacement against the carried digest and
|
||||
manifest, with lease release and no output;
|
||||
- renderer refuses manifest `head`, `files` digest, expected-digest, missing, and malformed
|
||||
cases, with lease release and no output;
|
||||
- wrapper refuses missing manifest, manifest head/digest/revision tampering, saved-revision blob
|
||||
mismatch, Git blob mismatch, and snapshot-vs-Git-bytes mismatch, and passes the exact
|
||||
`--snapshot-sha256` on the valid path (stub renderer records arguments).
|
||||
|
||||
## Verification
|
||||
|
||||
- `bash scripts/test-p1-manual-acceptance.sh` (backend build + both suites): **59 tests, 59
|
||||
pass, 0 fail**; no `8791/8792` listener and no `--p1-manual-nonce` process remain.
|
||||
- `npx tsc --noEmit -p .` (backend): PASS.
|
||||
- Real-repository `prepare` + `cleanup` cycle: 39 distribution files bound, entrypoint
|
||||
cross-digest verified, owned root fully removed afterwards.
|
||||
- Diff check: only the seven Task 9 paths are touched; no Task 8 file was modified.
|
||||
- This report and the implementation contain no fixture secret or canary values.
|
||||
|
||||
Manual acceptance remains **PENDING** by design; the walkthrough and human verdict are
|
||||
unchanged.
|
||||
@@ -1,11 +0,0 @@
|
||||
{
|
||||
"version": "0.0.1",
|
||||
"configurations": [
|
||||
{
|
||||
"name": "replay",
|
||||
"runtimeExecutable": "node",
|
||||
"runtimeArgs": ["tools/replay/server.mjs"],
|
||||
"port": 5333
|
||||
}
|
||||
]
|
||||
}
|
||||
+7
-2
@@ -4,6 +4,8 @@
|
||||
**/__pycache__
|
||||
**/.pytest_cache
|
||||
**/dist
|
||||
frontend/prototypes/
|
||||
frontend/vite.database-management-prototype.config.ts
|
||||
**/*.pyc
|
||||
.git
|
||||
.worktrees
|
||||
@@ -15,6 +17,7 @@
|
||||
!deploy/env/*.env.example
|
||||
deploy/thothii.env
|
||||
deploy/secrets/
|
||||
deploy/psd/
|
||||
harness/workspaces/*.yaml
|
||||
!harness/workspaces/local.yaml
|
||||
!harness/workspaces/tht.example.yaml
|
||||
@@ -27,5 +30,7 @@ coverage/
|
||||
data/
|
||||
sessions/
|
||||
workspace-registry/
|
||||
# docs/site (mkdocs build) — non necessari nelle immagini
|
||||
docs/superpowers/plans
|
||||
|
||||
.tht/
|
||||
|
||||
deploy/local/
|
||||
|
||||
@@ -9,4 +9,5 @@ Dockerfile* text eol=lf
|
||||
*.tsx text eol=lf
|
||||
*.py text eol=lf
|
||||
*.md text eol=lf
|
||||
*.pptx binary
|
||||
*.ps1 text eol=crlf
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
name: Publish documentation
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
paths:
|
||||
- "docs/**"
|
||||
- "mkdocs.yml"
|
||||
- "scripts/build-docs.sh"
|
||||
- "scripts/verify-public-docs.py"
|
||||
- "scripts/test-verify-public-docs.py"
|
||||
- "scripts/verify-auth-docs.py"
|
||||
- "scripts/test-verify-auth-docs.py"
|
||||
- "docs/requirements.txt"
|
||||
- ".gitea/workflows/publish-docs.yml"
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
|
||||
concurrency:
|
||||
group: documentation
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
publish:
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
|
||||
REPOSITORY_URL: ${{ gitea.server_url }}/${{ gitea.repository }}.git
|
||||
|
||||
steps:
|
||||
- name: Checkout documentation source
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.x"
|
||||
cache: pip
|
||||
cache-dependency-path: docs/requirements.lock
|
||||
|
||||
- name: Install MkDocs dependencies
|
||||
run: python -m pip install -r docs/requirements.lock
|
||||
|
||||
- name: Test public documentation boundary
|
||||
run: python scripts/test-verify-public-docs.py
|
||||
|
||||
- name: Test current authentication documentation
|
||||
run: |
|
||||
python scripts/verify-auth-docs.py auth
|
||||
python scripts/verify-auth-docs.py dwh
|
||||
python scripts/test-verify-auth-docs.py auth
|
||||
python scripts/test-verify-auth-docs.py dwh
|
||||
|
||||
- name: Build documentation
|
||||
run: |
|
||||
mkdocs build --strict
|
||||
python scripts/verify-public-docs.py
|
||||
|
||||
- name: Publish generated site to the pages branch
|
||||
working-directory: site
|
||||
run: |
|
||||
git init
|
||||
git config user.name "Gitea Actions"
|
||||
git config user.email "actions@${{ gitea.server_url }}"
|
||||
git add --all
|
||||
git commit --message "Publish documentation for ${{ gitea.sha }}"
|
||||
git -c http.extraheader="Authorization: token ${GITEA_TOKEN}" \
|
||||
push --force "${REPOSITORY_URL}" HEAD:pages
|
||||
@@ -24,6 +24,8 @@ jobs:
|
||||
name: LF, Compose, docs, and TypeScript
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 25
|
||||
env:
|
||||
PYTHONDONTWRITEBYTECODE: "1"
|
||||
steps:
|
||||
- name: Check out source
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
@@ -34,6 +36,10 @@ jobs:
|
||||
with:
|
||||
node-version: "24.16.0"
|
||||
package-manager-cache: false
|
||||
- name: Install release gate prerequisites
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install --yes --no-install-recommends ripgrep
|
||||
- name: Verify shell syntax and LF policy
|
||||
run: |
|
||||
git ls-files -z '*.sh' | xargs -0 -n1 bash -n
|
||||
@@ -44,16 +50,27 @@ jobs:
|
||||
bash scripts/test-no-deployment-coupling-scope.sh
|
||||
bash scripts/test-compose-secret-policy.sh
|
||||
bash scripts/test-no-deployment-coupling.sh
|
||||
bash scripts/test-preprocess-compose-config.sh
|
||||
bash scripts/test-verify-workspace-install-docs.sh
|
||||
git diff --check
|
||||
- name: Install backend dependencies
|
||||
working-directory: backend
|
||||
run: npm ci
|
||||
- name: Assert clean checkout before release trust bootstrap
|
||||
run: |
|
||||
git diff --exit-code
|
||||
git diff --cached --exit-code
|
||||
test -z "$(git ls-files --others --exclude-standard)"
|
||||
- name: Verify schema-v3-only release gate
|
||||
run: bash scripts/verify-schema-v3-only-release.sh
|
||||
- name: Verify Task 13 clean-install and runtime fixtures
|
||||
run: |
|
||||
bash scripts/test-server-pi-state-topology.sh
|
||||
bash scripts/unified-deployment-smoke.sh --self-test
|
||||
- name: Install harness CLI for backend integration tests
|
||||
working-directory: harness
|
||||
run: |
|
||||
python3 -m venv .venv
|
||||
.venv/bin/python -m pip install -e .
|
||||
- name: Install backend dependencies
|
||||
working-directory: backend
|
||||
run: npm ci
|
||||
- name: Test and type-check backend
|
||||
working-directory: backend
|
||||
run: |
|
||||
@@ -68,6 +85,63 @@ jobs:
|
||||
npx vitest run
|
||||
npx tsc -b
|
||||
|
||||
authentication-browser:
|
||||
name: Hermetic authentication browser gate
|
||||
needs: deterministic
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- name: Check out source
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Set up Node.js
|
||||
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
||||
with:
|
||||
node-version: "24.16.0"
|
||||
package-manager-cache: false
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
|
||||
with:
|
||||
go-version: "1.26.5"
|
||||
cache-dependency-path: tools/tht/go.sum
|
||||
- name: Install backend dependencies
|
||||
working-directory: backend
|
||||
run: npm ci
|
||||
- name: Install frontend dependencies
|
||||
working-directory: frontend
|
||||
run: npm ci
|
||||
- name: Install Chromium for Playwright
|
||||
working-directory: frontend
|
||||
run: npx playwright install --with-deps chromium
|
||||
- name: Run authentication and authenticated F1 browser smoke
|
||||
run: bash scripts/authentication-smoke.sh
|
||||
|
||||
dwh-auth-linux:
|
||||
name: DWH authentication Nginx gate
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 20
|
||||
steps:
|
||||
- name: Check out source
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
|
||||
with:
|
||||
go-version: "1.26.5"
|
||||
cache-dependency-path: tools/dwh-auth/go.mod
|
||||
- name: Install Nginx
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install --yes --no-install-recommends nginx-light
|
||||
- name: Run DWH authentication gates
|
||||
run: |
|
||||
(cd tools/dwh-auth && go test -race ./... -count=1 && go vet ./...)
|
||||
bash scripts/test-dwh-auth-build-contract.sh
|
||||
bash scripts/test-dwh-auth-nginx-contract.sh
|
||||
bash scripts/test-dwh-auth-nginx-integration.sh
|
||||
|
||||
linux-docker:
|
||||
name: Linux Docker deployment and rollback
|
||||
runs-on: ubuntu-24.04
|
||||
@@ -77,11 +151,23 @@ jobs:
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Install release gate prerequisites
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install --yes --no-install-recommends ripgrep
|
||||
- name: Reclaim unused hosted-runner space
|
||||
run: bash scripts/prepare-linux-docker-runner.sh
|
||||
- name: Run unified deployment smoke
|
||||
env:
|
||||
TASK13_IMAGE_EVIDENCE_OUTPUT: ${{ runner.temp }}/task13-images.json
|
||||
run: timeout --signal=TERM --kill-after=45s 32m bash scripts/unified-deployment-smoke.sh
|
||||
- name: Run thothctl update smoke
|
||||
run: timeout --signal=TERM --kill-after=45s 32m bash scripts/thothctl-update-smoke.sh
|
||||
- name: Run tht update smoke
|
||||
env:
|
||||
TASK13_IMAGE_EVIDENCE_OUTPUT: ${{ runner.temp }}/task13-images.json
|
||||
run: timeout --signal=TERM --kill-after=45s 32m bash scripts/tht-update-smoke.sh
|
||||
- name: Run Linux server deployment smoke
|
||||
env:
|
||||
TASK13_IMAGE_EVIDENCE_OUTPUT: ${{ runner.temp }}/task13-images.json
|
||||
run: timeout --signal=TERM --kill-after=45s 32m bash scripts/server-deployment-smoke.sh
|
||||
|
||||
windows-clone:
|
||||
@@ -108,7 +194,10 @@ jobs:
|
||||
uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
|
||||
with:
|
||||
go-version: "1.26.5"
|
||||
cache-dependency-path: tools/thothctl/go.sum
|
||||
cache-dependency-path: tools/tht/go.sum
|
||||
- name: Run native Windows retained-capability tests
|
||||
working-directory: tools/tht
|
||||
run: go test ./internal/safeio ./internal/backup ./internal/authstorage -count=1
|
||||
- name: Verify Windows clone contract
|
||||
shell: pwsh
|
||||
run: ./scripts/test-windows-clone-contract.ps1
|
||||
@@ -127,7 +216,7 @@ jobs:
|
||||
uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
|
||||
with:
|
||||
go-version: "1.26.5"
|
||||
cache-dependency-path: tools/thothctl/go.sum
|
||||
cache-dependency-path: tools/tht/go.sum
|
||||
- name: Run spaced-path Windows Docker release gate
|
||||
shell: pwsh
|
||||
run: ./scripts/test-windows-clone-contract.ps1 -DockerStartup
|
||||
|
||||
+19
-3
@@ -5,10 +5,8 @@
|
||||
ChironeWp3/
|
||||
Thoth/
|
||||
|
||||
# === Visual companion brainstorming artifacts (local-only) ===
|
||||
.superpowers/
|
||||
.worktrees/
|
||||
.thothctl/
|
||||
.tht/
|
||||
|
||||
# === Python ===
|
||||
__pycache__/
|
||||
@@ -29,6 +27,7 @@ tools/replay/web/
|
||||
# === Secrets — NEVER commit ===
|
||||
.env
|
||||
harness/.env
|
||||
harness/workspaces/psd.yaml
|
||||
deploy/thothii.env
|
||||
*.pem
|
||||
ca-chain.pem
|
||||
@@ -36,6 +35,7 @@ config/ca-chain.pem
|
||||
|
||||
# ThothII deployment configuration and secret values (keep only the README tracked)
|
||||
deploy/.env
|
||||
deploy/env/local.env
|
||||
deploy/compose.connector-secrets.local.yaml
|
||||
deploy/compose.psd-local.yaml
|
||||
deploy/workspaces/psd.yaml
|
||||
@@ -43,6 +43,15 @@ deploy/secrets/*
|
||||
!deploy/secrets/README.md
|
||||
!deploy/secrets/*.example
|
||||
|
||||
# Per-installation configuration generated by `tht setup` (examples stay tracked).
|
||||
deploy/*/thothii-installation.yaml
|
||||
deploy/*/operator.env
|
||||
deploy/*/auth/
|
||||
deploy/*/generated/
|
||||
deploy/*/secrets/*
|
||||
!deploy/*/secrets/.gitkeep
|
||||
!deploy/*/secrets/*.example
|
||||
|
||||
# === Runtime data (sessions contain PII; indexes are derived) ===
|
||||
harness/sessions/
|
||||
harness/indexes/
|
||||
@@ -72,3 +81,10 @@ site/
|
||||
|
||||
# === PrimeAgent local project settings (per-user, not shared) ===
|
||||
.prime/
|
||||
|
||||
# === Local coding-agent settings (per-user, not shared) ===
|
||||
.commandcode/
|
||||
.reasonix/
|
||||
|
||||
# === Local runtime logs ===
|
||||
logs/
|
||||
|
||||
@@ -0,0 +1,203 @@
|
||||
{
|
||||
"schemaVersion": 2,
|
||||
"generatedAt": "2026-08-26T10:10:15.926Z",
|
||||
"title": "Design System: ThothII",
|
||||
"extensions": {
|
||||
"colorMeta": {
|
||||
"instrument-red": {
|
||||
"role": "primary",
|
||||
"displayName": "Instrument Red",
|
||||
"canonical": "oklch(55.87% 0.1881 23.2)",
|
||||
"tonalRamp": ["oklch(15% 0.07 23.2)", "oklch(28% 0.12 23.2)", "oklch(42% 0.16 23.2)", "oklch(56% 0.1881 23.2)", "oklch(68% 0.17 23.2)", "oklch(78% 0.13 23.2)", "oklch(88% 0.07 23.2)", "oklch(95% 0.03 23.2)"]
|
||||
},
|
||||
"instrument-red-hover": {
|
||||
"role": "primary",
|
||||
"displayName": "Instrument Red Pressed",
|
||||
"canonical": "oklch(50.95% 0.1812 24.1)",
|
||||
"tonalRamp": ["oklch(15% 0.07 24.1)", "oklch(28% 0.12 24.1)", "oklch(42% 0.16 24.1)", "oklch(51% 0.1812 24.1)", "oklch(68% 0.16 24.1)", "oklch(78% 0.12 24.1)", "oklch(88% 0.07 24.1)", "oklch(95% 0.03 24.1)"]
|
||||
},
|
||||
"porcelain-background": {
|
||||
"role": "neutral",
|
||||
"displayName": "Porcelain Background",
|
||||
"canonical": "oklch(99.18% 0.0011 17.2)",
|
||||
"tonalRamp": ["oklch(15% 0.0011 17.2)", "oklch(28% 0.0011 17.2)", "oklch(42% 0.0011 17.2)", "oklch(56% 0.0011 17.2)", "oklch(68% 0.0011 17.2)", "oklch(78% 0.0011 17.2)", "oklch(88% 0.0011 17.2)", "oklch(95% 0.0011 17.2)"]
|
||||
},
|
||||
"porcelain-card": {
|
||||
"role": "neutral",
|
||||
"displayName": "Porcelain Card",
|
||||
"canonical": "oklch(99.85% 0.0006 17.2)",
|
||||
"tonalRamp": ["oklch(15% 0.0006 17.2)", "oklch(28% 0.0006 17.2)", "oklch(42% 0.0006 17.2)", "oklch(56% 0.0006 17.2)", "oklch(68% 0.0006 17.2)", "oklch(78% 0.0006 17.2)", "oklch(88% 0.0006 17.2)", "oklch(95% 0.0006 17.2)"]
|
||||
},
|
||||
"warm-surface": {
|
||||
"role": "neutral",
|
||||
"displayName": "Warm Surface",
|
||||
"canonical": "oklch(97.09% 0.0011 17.2)",
|
||||
"tonalRamp": ["oklch(15% 0.0011 17.2)", "oklch(28% 0.0011 17.2)", "oklch(42% 0.0011 17.2)", "oklch(56% 0.0011 17.2)", "oklch(68% 0.0011 17.2)", "oklch(78% 0.0011 17.2)", "oklch(88% 0.0011 17.2)", "oklch(95% 0.0011 17.2)"]
|
||||
},
|
||||
"sunken-surface": {
|
||||
"role": "neutral",
|
||||
"displayName": "Sunken Surface",
|
||||
"canonical": "oklch(94.08% 0.0011 17.2)",
|
||||
"tonalRamp": ["oklch(15% 0.0011 17.2)", "oklch(28% 0.0011 17.2)", "oklch(42% 0.0011 17.2)", "oklch(56% 0.0011 17.2)", "oklch(68% 0.0011 17.2)", "oklch(78% 0.0011 17.2)", "oklch(88% 0.0011 17.2)", "oklch(95% 0.0011 17.2)"]
|
||||
},
|
||||
"warm-graphite": {
|
||||
"role": "neutral",
|
||||
"displayName": "Warm Graphite",
|
||||
"canonical": "oklch(26.78% 0.0097 355.6)",
|
||||
"tonalRamp": ["oklch(15% 0.0097 355.6)", "oklch(28% 0.0097 355.6)", "oklch(42% 0.0097 355.6)", "oklch(56% 0.0097 355.6)", "oklch(68% 0.008 355.6)", "oklch(78% 0.006 355.6)", "oklch(88% 0.004 355.6)", "oklch(95% 0.002 355.6)"]
|
||||
},
|
||||
"muted-graphite": {
|
||||
"role": "neutral",
|
||||
"displayName": "Muted Graphite",
|
||||
"canonical": "oklch(51.33% 0.0088 345.6)",
|
||||
"tonalRamp": ["oklch(15% 0.0088 345.6)", "oklch(28% 0.0088 345.6)", "oklch(42% 0.0088 345.6)", "oklch(56% 0.0088 345.6)", "oklch(68% 0.007 345.6)", "oklch(78% 0.005 345.6)", "oklch(88% 0.003 345.6)", "oklch(95% 0.002 345.6)"]
|
||||
},
|
||||
"quiet-border": {
|
||||
"role": "neutral",
|
||||
"displayName": "Quiet Border",
|
||||
"canonical": "oklch(90.93% 0.0035 354.7)",
|
||||
"tonalRamp": ["oklch(15% 0.0035 354.7)", "oklch(28% 0.0035 354.7)", "oklch(42% 0.0035 354.7)", "oklch(56% 0.0035 354.7)", "oklch(68% 0.0035 354.7)", "oklch(78% 0.0035 354.7)", "oklch(88% 0.003 354.7)", "oklch(95% 0.002 354.7)"]
|
||||
},
|
||||
"success-mint": {
|
||||
"role": "secondary",
|
||||
"displayName": "Success Mint",
|
||||
"canonical": "oklch(75.77% 0.1581 165)",
|
||||
"tonalRamp": ["oklch(15% 0.06 165)", "oklch(28% 0.1 165)", "oklch(42% 0.14 165)", "oklch(56% 0.1581 165)", "oklch(68% 0.15 165)", "oklch(78% 0.12 165)", "oklch(88% 0.07 165)", "oklch(95% 0.03 165)"]
|
||||
},
|
||||
"warning-amber": {
|
||||
"role": "tertiary",
|
||||
"displayName": "Warning Amber",
|
||||
"canonical": "oklch(85.23% 0.1386 78.9)",
|
||||
"tonalRamp": ["oklch(15% 0.05 78.9)", "oklch(28% 0.09 78.9)", "oklch(42% 0.12 78.9)", "oklch(56% 0.1386 78.9)", "oklch(68% 0.13 78.9)", "oklch(78% 0.1 78.9)", "oklch(88% 0.06 78.9)", "oklch(95% 0.025 78.9)"]
|
||||
},
|
||||
"information-blue": {
|
||||
"role": "tertiary",
|
||||
"displayName": "Information Blue",
|
||||
"canonical": "oklch(70.35% 0.1128 221.3)",
|
||||
"tonalRamp": ["oklch(15% 0.045 221.3)", "oklch(28% 0.075 221.3)", "oklch(42% 0.1 221.3)", "oklch(56% 0.1128 221.3)", "oklch(68% 0.105 221.3)", "oklch(78% 0.08 221.3)", "oklch(88% 0.045 221.3)", "oklch(95% 0.02 221.3)"]
|
||||
}
|
||||
},
|
||||
"typographyMeta": {
|
||||
"display": {"displayName": "Display", "purpose": "Authentication and exceptional page-level statements only."},
|
||||
"headline": {"displayName": "Headline", "purpose": "Major page and persisted artifact titles."},
|
||||
"title": {"displayName": "Title", "purpose": "Panel and document section hierarchy."},
|
||||
"body": {"displayName": "Body", "purpose": "Operational prose and sustained reading."},
|
||||
"control": {"displayName": "Control", "purpose": "Buttons, inputs, tabs, and compact actions."},
|
||||
"label": {"displayName": "Machine Label", "purpose": "Uppercase metadata and machine-oriented micro-labels."}
|
||||
},
|
||||
"shadows": [
|
||||
{"name": "contact", "value": "0 1px 2px oklch(var(--shadow-tint) / 0.05)", "purpose": "Contact shadow for controls and code blocks."},
|
||||
{"name": "panel", "value": "0 1px 2px oklch(var(--shadow-tint) / 0.05), 0 2px 6px -1px oklch(var(--shadow-tint) / 0.05)", "purpose": "Small structural lift for selected cards."},
|
||||
{"name": "overlay", "value": "0 2px 4px -2px oklch(var(--shadow-tint) / 0.06), 0 12px 32px -8px oklch(var(--shadow-tint) / 0.1)", "purpose": "Broad low-opacity lift for dialogs and floating layers."}
|
||||
],
|
||||
"motion": [
|
||||
{"name": "control-feedback", "value": "140ms cubic-bezier(0.22, 1, 0.36, 1)", "purpose": "Button hover, focus, and press feedback."},
|
||||
{"name": "overlay-transition", "value": "100ms ease-out", "purpose": "Dialog fade and scale transitions."},
|
||||
{"name": "activity-pulse", "value": "1.5s ease-in-out infinite", "purpose": "Live model activity only; disabled for reduced motion."}
|
||||
],
|
||||
"breakpoints": [
|
||||
{"name": "sm", "value": "640px"},
|
||||
{"name": "lg", "value": "1024px"}
|
||||
]
|
||||
},
|
||||
"components": [
|
||||
{
|
||||
"name": "Primary Button",
|
||||
"kind": "button",
|
||||
"refersTo": "button-primary",
|
||||
"description": "The authoritative action for the current workflow step.",
|
||||
"html": "<button class=\"ds-button-primary\">Confirm review</button>",
|
||||
"css": ".ds-button-primary { display:inline-flex; align-items:center; justify-content:center; height:32px; padding:0 14px; border:1px solid transparent; border-radius:8px; background:oklch(var(--primary)); color:oklch(var(--primary-foreground)); font:600 14px/1.25 var(--font-sans); letter-spacing:0.005em; box-shadow:var(--shadow-xs); transition:color 140ms cubic-bezier(0.22,1,0.36,1),background-color 140ms cubic-bezier(0.22,1,0.36,1),box-shadow 140ms cubic-bezier(0.22,1,0.36,1),transform 140ms cubic-bezier(0.22,1,0.36,1); } .ds-button-primary:hover { background:oklch(var(--primary-hover)); } .ds-button-primary:focus-visible { outline:3px solid oklch(var(--ring)/0.25); outline-offset:2px; } .ds-button-primary:active { transform:scale(0.97); box-shadow:none; }"
|
||||
},
|
||||
{
|
||||
"name": "Outline Button",
|
||||
"kind": "button",
|
||||
"refersTo": "button-secondary",
|
||||
"description": "A compact secondary action that preserves the primary action hierarchy.",
|
||||
"html": "<button class=\"ds-button-outline\">Inspect details</button>",
|
||||
"css": ".ds-button-outline { display:inline-flex; align-items:center; justify-content:center; height:32px; padding:0 14px; border:1px solid oklch(var(--border)); border-radius:8px; background:oklch(var(--card)); color:oklch(var(--foreground)); font:600 14px/1.25 var(--font-sans); box-shadow:var(--shadow-xs); transition:background-color 140ms cubic-bezier(0.22,1,0.36,1),transform 140ms cubic-bezier(0.22,1,0.36,1); } .ds-button-outline:hover { background:oklch(var(--muted)); } .ds-button-outline:focus-visible { outline:3px solid oklch(var(--ring)/0.25); outline-offset:2px; } .ds-button-outline:active { transform:scale(0.97); box-shadow:none; }"
|
||||
},
|
||||
{
|
||||
"name": "Status Badge",
|
||||
"kind": "chip",
|
||||
"refersTo": "badge-primary",
|
||||
"description": "A compact state label that always carries readable text.",
|
||||
"html": "<span class=\"ds-status-badge\">Ready for review</span>",
|
||||
"css": ".ds-status-badge { display:inline-flex; align-items:center; height:20px; padding:2px 8px; border:1px solid transparent; border-radius:6px; background:oklch(var(--primary)); color:oklch(var(--primary-foreground)); font:600 12px/1.25 var(--font-sans); white-space:nowrap; } .ds-status-badge:focus-visible { outline:3px solid oklch(var(--ring)/0.5); outline-offset:2px; }"
|
||||
},
|
||||
{
|
||||
"name": "Text Field",
|
||||
"kind": "input",
|
||||
"refersTo": "input-default",
|
||||
"description": "A readable operational field with an explicit focus state.",
|
||||
"html": "<input class=\"ds-text-field\" value=\"Fascia pediatrica\" aria-label=\"Session name\">",
|
||||
"css": ".ds-text-field { width:280px; height:40px; padding:0 12px; border:1px solid oklch(var(--input)); border-radius:8px; background:oklch(var(--background)); color:oklch(var(--foreground)); font:400 14px/1.5 var(--font-sans); outline:none; } .ds-text-field:hover { border-color:oklch(var(--muted-foreground)/0.65); } .ds-text-field:focus-visible { border-color:oklch(var(--ring)); box-shadow:0 0 0 3px oklch(var(--ring)/0.25); } .ds-text-field:disabled { opacity:0.5; cursor:not-allowed; }"
|
||||
},
|
||||
{
|
||||
"name": "Work Card",
|
||||
"kind": "card",
|
||||
"refersTo": "card-default",
|
||||
"description": "A single-level container for a coherent review surface.",
|
||||
"html": "<section class=\"ds-work-card\"><h3>Schema linking</h3><p>Review the linked tables and columns before continuing.</p></section>",
|
||||
"css": ".ds-work-card { width:320px; padding:16px; border:1px solid oklch(var(--border)/0.7); border-radius:12px; background:oklch(var(--card)); color:oklch(var(--card-foreground)); box-shadow:var(--shadow-sm); } .ds-work-card h3 { margin:0 0 8px; font:500 16px/1.35 var(--font-heading); letter-spacing:-0.01em; } .ds-work-card p { margin:0; color:oklch(var(--muted-foreground)); font:400 14px/1.6 var(--font-sans); } .ds-work-card:focus-within { outline:3px solid oklch(var(--ring)/0.25); outline-offset:2px; }"
|
||||
},
|
||||
{
|
||||
"name": "Session Navigation Item",
|
||||
"kind": "nav",
|
||||
"description": "A dense session row with restrained hover and active hierarchy.",
|
||||
"html": "<button class=\"ds-session-item\"><span class=\"ds-session-dot\"></span><span><strong>Patient cohorts</strong><small>Schema linking</small></span></button>",
|
||||
"css": ".ds-session-item { display:flex; width:260px; align-items:center; gap:8px; padding:4px 8px; border:0; border-radius:8px; background:transparent; color:oklch(var(--foreground)); text-align:left; font-family:var(--font-sans); transition:background-color 140ms cubic-bezier(0.22,1,0.36,1); } .ds-session-item:hover,.ds-session-item[aria-current=\"page\"] { background:oklch(var(--accent)); } .ds-session-item:focus-visible { outline:2px solid oklch(var(--ring)/0.4); outline-offset:1px; } .ds-session-dot { width:6px; height:6px; flex:none; border-radius:9999px; background:oklch(var(--success)); } .ds-session-item strong,.ds-session-item small { display:block; } .ds-session-item strong { font-size:13px; font-weight:600; } .ds-session-item small { margin-top:2px; color:oklch(var(--muted-foreground)); font-size:11px; }"
|
||||
},
|
||||
{
|
||||
"name": "Curated Evidence Document",
|
||||
"kind": "custom",
|
||||
"description": "The table-free reading hierarchy for persisted evidence.",
|
||||
"html": "<article class=\"ds-evidence\"><h2>Fascia pediatrica</h2><div class=\"ds-evidence-summary\"><strong>Dominio</strong> · Italiano<br><span>Scopi: Disambiguazione · Generazione SQL</span></div><h3>Ambito di applicazione</h3><ul><li>fascia pediatrica</li><li>paziente minore</li></ul><h3>Regola</h3><p>La fascia pediatrica comprende i pazienti con età inferiore a 18 anni.</p><details><summary>Dettagli tecnici e provenienza</summary><code>evidence:fascia-pediatrica</code></details></article>",
|
||||
"css": ".ds-evidence { max-width:70ch; color:oklch(var(--foreground)); font:400 15px/1.65 var(--font-sans); } .ds-evidence h2,.ds-evidence h3 { font-family:var(--font-heading); letter-spacing:-0.01em; } .ds-evidence h2 { margin:0 0 16px; font-size:24px; } .ds-evidence h3 { margin:24px 0 8px; font-size:18px; } .ds-evidence-summary { padding:12px 14px; border:1px solid oklch(var(--border)); border-radius:8px; background:oklch(var(--muted)); color:oklch(var(--muted-foreground)); } .ds-evidence-summary strong { color:oklch(var(--foreground)); } .ds-evidence ul { padding-left:20px; } .ds-evidence details { margin-top:24px; padding:10px 12px; border:1px solid oklch(var(--border)); border-radius:8px; background:oklch(var(--card)); } .ds-evidence summary { cursor:pointer; font-weight:600; } .ds-evidence code { font-family:var(--font-mono); }"
|
||||
}
|
||||
],
|
||||
"narrative": {
|
||||
"northStar": "The Clinical Workbench",
|
||||
"overview": "ThothII should feel like a well-kept clinical workbench: warm enough for sustained reading, exact enough for consequential review, and quiet enough that evidence, state, and decisions remain in the foreground. The visual system is calm, precise, and trustworthy. It uses familiar product patterns, restrained color, and deliberate density instead of decorative spectacle.\n\nThe primary physical scene is an analyst reviewing persisted evidence and SQL on a large monitor in a well-lit working environment. This makes the warm light theme the default. The supported dark theme serves lower-light work without becoming a separate neon aesthetic. Both themes preserve the same hierarchy and semantic roles.\n\nThe system rejects generic SaaS ornament, conspicuous ripples, bounce or elastic motion, long choreographed transitions, and effects that compete with the analytical task. Controls should feel disciplined and tactile, never playful, sluggish, or visually unstable.",
|
||||
"keyCharacteristics": [
|
||||
"Warm, restrained surfaces with one scarce red accent.",
|
||||
"Editorial headings paired with highly legible operational body text.",
|
||||
"Dense information organized through hierarchy, rhythm, and progressive disclosure.",
|
||||
"Persisted artifacts and reviewer decisions presented as the visual source of truth.",
|
||||
"Fast state feedback with reduced-motion parity."
|
||||
],
|
||||
"rules": [
|
||||
{"name": "The Workbench Rule", "body": "Every visual element must support inspection, action, state, or provenance. Decoration without an operational purpose is forbidden.", "section": "overview"},
|
||||
{"name": "The Persisted Truth Rule", "body": "Persisted artifacts and reviewer decisions receive stronger hierarchy than transient model narration.", "section": "overview"},
|
||||
{"name": "The Density with Rhythm Rule", "body": "Preserve information density, but vary spacing between groups so users can scan structure without adding nested containers.", "section": "overview"},
|
||||
{"name": "The One Voice Rule", "body": "Instrument Red should occupy no more than roughly ten percent of a screen. Its rarity is what makes it authoritative.", "section": "colors"},
|
||||
{"name": "The State Has a Name Rule", "body": "Success, warning, information, and destructive colors are reserved for their named states. Color is never the only state indicator.", "section": "colors"},
|
||||
{"name": "The Three Registers Rule", "body": "Serif means authority, sans means interaction and reading, mono means machine identity. Do not exchange these roles for novelty.", "section": "typography"},
|
||||
{"name": "The Read Once Rule", "body": "A heading, label, and body must be distinguishable on first glance through size and weight. Do not repeat headings in explanatory copy.", "section": "typography"},
|
||||
{"name": "The Flat by Default Rule", "body": "A resting surface has no shadow unless it is physically above another surface. If every panel floats, none of them has hierarchy.", "section": "elevation"},
|
||||
{"name": "The Borders Structure, Shadows Elevate Rule", "body": "Never use shadow as a substitute for grouping or a border as a decorative accent.", "section": "elevation"},
|
||||
{"name": "The Review Surface Rule", "body": "The visible Markdown must be readable without understanding the machine contract. Technical metadata belongs in progressive disclosure, not above the title.", "section": "components"}
|
||||
],
|
||||
"dos": [
|
||||
"Do make every state change unmistakable without interrupting flow.",
|
||||
"Do use Instrument Red only for primary action, current selection, focus identity, or explicit destructive meaning.",
|
||||
"Do preserve information density with headings, rhythm, and progressive disclosure.",
|
||||
"Do keep keyboard focus explicit and pair color with text, shape, icon, or position.",
|
||||
"Do respect prefers-reduced-motion while preserving immediate non-kinetic feedback.",
|
||||
"Do use English for interface chrome and the workspace language for persisted document content.",
|
||||
"Do render curated metadata and scope as Markdown prose or lists, never as a frontmatter table."
|
||||
],
|
||||
"donts": [
|
||||
"Don't add generic SaaS ornament, conspicuous ripples, bounce or elastic motion, long choreographed transitions, or effects that compete with the analytical task.",
|
||||
"Don't make controls feel playful, sluggish, or visually unstable.",
|
||||
"Don't use gradient text, decorative glassmorphism, or full-saturation accents on inactive states.",
|
||||
"Don't use a colored side stripe greater than one pixel on cards, callouts, list items, or blockquotes. Use a full border, tonal background, icon, or heading instead.",
|
||||
"Don't nest cards or wrap every section in a container.",
|
||||
"Don't use a modal before exhausting inline or progressive alternatives.",
|
||||
"Don't use tables for applies_to, metadata, enum values, or other one-dimensional content.",
|
||||
"Don't use color as the sole carrier of success, warning, error, selection, or progress.",
|
||||
"Don't use display typography for buttons, labels, or data.",
|
||||
"Don't add em dashes to interface copy. Use commas, colons, semicolons, or parentheses."
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -1,7 +0,0 @@
|
||||
{
|
||||
"$schema": "https://app.kilo.ai/config.json",
|
||||
"indexing": {
|
||||
"vectorStore": "qdrant",
|
||||
"model": "sentence-transformers/all-minilm-l12-v2"
|
||||
}
|
||||
}
|
||||
@@ -1,105 +0,0 @@
|
||||
# Task 3 — Diagnostic contract remediation report
|
||||
|
||||
Date: 2026-08-04
|
||||
|
||||
## Scope
|
||||
|
||||
This remediation is limited to the four approved review findings for the workspace diagnostic
|
||||
extension. It does not add registry routes, change workspace publication, alter session startup,
|
||||
or expand transport support.
|
||||
|
||||
## Changes
|
||||
|
||||
1. `RuntimeBindings` now has an explicit `vectorWriter` binding. The new
|
||||
`resolveRuntimeBindings()` resolves DWH, vector reader, vector writer, and embedding bindings
|
||||
together. The diagnoser takes the writer credential only from `bindings.vectorWriter`, never
|
||||
from vector-reader values.
|
||||
2. Direct PostgreSQL and SSH-tunnelled direct probes accept an absent CA binding while retaining
|
||||
certificate verification through the runtime system trust store. A supplied CA still uses
|
||||
verified private-CA trust. REST private-CA refusal is unchanged.
|
||||
3. A reversible vector probe now requires an authenticated POST declaration with a response map
|
||||
containing `operation`. The adapter requires the successful JSON response to echo `create` or
|
||||
`remove` respectively, so an arbitrary 2xx or an upsert-only response cannot activate the
|
||||
write probe.
|
||||
4. For DWH and vector REST diagnostics declared with `auth: none`, the resolver no longer
|
||||
requires an API-key file and the adapter sends no credential. Credential-backed diagnostics
|
||||
continue to require their local secret file.
|
||||
|
||||
## TDD evidence
|
||||
|
||||
The first focused RED run failed for the intended missing behavior:
|
||||
|
||||
- `resolveRuntimeBindings is not a function` for unauthenticated resolver bindings;
|
||||
- schema accepted a reversible probe without a response contract; and
|
||||
- existing diagnostic fixtures rejected the new `response` declaration until schema support was
|
||||
implemented.
|
||||
|
||||
The focused GREEN run passed `43/43` tests across:
|
||||
|
||||
- `test/workspaces-bindings.test.ts`
|
||||
- `test/workspaces-schema.test.ts`
|
||||
- `test/workspaces-diagnostics.test.ts`
|
||||
|
||||
The regression coverage includes resolver-to-diagnoser writer propagation without manually
|
||||
inserting the writer key into vector-reader bindings, no-CA direct/SSH system-trust requests,
|
||||
operation-echo validation for create/remove, and `auth: none` bindings without secret files.
|
||||
|
||||
## Documentation and design
|
||||
|
||||
- `docs/workspace-diagnostic-protocol.md` now documents the verified system-trust fallback,
|
||||
no-secret `auth: none` behavior, and required reversible response contract.
|
||||
- `docs/superpowers/specs/2026-08-03-git-workspace-registry-design.md` now records the same
|
||||
response, CA, SSH, and authentication rules.
|
||||
|
||||
## Final verification
|
||||
|
||||
The initial sandboxed full suite could not bind its local SSE listener (`listen EPERM:
|
||||
operation not permitted 127.0.0.1`). It was rerun unchanged with local-listener permission.
|
||||
|
||||
```text
|
||||
backend: npx vitest run
|
||||
31 test files passed; 329 tests passed
|
||||
|
||||
backend: npx tsc --noEmit -p .
|
||||
exit 0
|
||||
|
||||
repository: git diff --check
|
||||
exit 0
|
||||
```
|
||||
|
||||
Expected test harness stderr from existing Pi/process failure-path tests remained present; no test
|
||||
failed and no diagnostic secret was emitted.
|
||||
|
||||
## Blockers
|
||||
|
||||
None.
|
||||
|
||||
## Round 2 remediation
|
||||
|
||||
The final review found two remaining contract gaps. The binding resolver already treated
|
||||
`auth: none` as credential-free, but the runtime renderer and diagnostic connector still required
|
||||
the API-key file. Rendering and connector construction now make that requirement conditional on
|
||||
the declared REST authentication mode, so a DWH/vector `auth: none` workspace passes resolver,
|
||||
runtime rendering, and diagnostics with no API-key file.
|
||||
|
||||
SSH forwarding previously changed the PostgreSQL connection host to `127.0.0.1` without retaining
|
||||
the original target for TLS hostname validation. Forwarded probes now carry `SSH_TARGET_HOST` as
|
||||
`tlsServername` into the PostgreSQL TLS options; private CA and verified system trust behavior are
|
||||
unchanged.
|
||||
|
||||
TDD RED: the new end-to-end no-key test failed at the unconditional runtime
|
||||
`API_KEY_FILE` requirement, while the SSH test showed no `tlsServername` on the loopback probe or
|
||||
database-client request. TDD GREEN: the focused backend workspace tests passed `40/40`.
|
||||
|
||||
Round 2 final verification:
|
||||
|
||||
```text
|
||||
backend: npx vitest run
|
||||
31 test files passed; 332 tests passed
|
||||
|
||||
backend: npx tsc --noEmit -p .
|
||||
exit 0
|
||||
|
||||
repository: git diff --check
|
||||
exit 0
|
||||
```
|
||||
@@ -1,101 +0,0 @@
|
||||
# Task 7 report — revision-pinned sessions
|
||||
|
||||
## Delivered
|
||||
|
||||
- New-session requests may carry `workspaceId`, provider, model, and thinking. The backend
|
||||
resolves the active operational registry revision, enforces its LLM policy, and persists the
|
||||
workspace ID/revision with the selected LLM settings.
|
||||
- The harness manifest and `tht session new` support the optional, backward-compatible
|
||||
`workspace_id` and `workspace_revision` fields.
|
||||
- Resume resolves the manifest's retained snapshot, including after later registry publication.
|
||||
A missing retained revision returns a sanitized `workspace_revision_unavailable` response.
|
||||
Legacy manifests retain the prior workspace behavior and are marked with a visible warning on
|
||||
`GET /sessions/:id`.
|
||||
- `/settings` is now a non-mutating compatibility endpoint: installation defaults remain
|
||||
readable, while anonymous workspace/provider/model/thinking selections are no longer written
|
||||
to backend settings or principal preferences.
|
||||
|
||||
## TDD evidence
|
||||
|
||||
- RED: `npx vitest run test/routes-sessions.test.ts test/routes-settings.test.ts` failed for the
|
||||
new immutable-snapshot and no-settings-mutation assertions; the manifest test failed because
|
||||
`new_session_manifest` did not accept workspace revision fields.
|
||||
- GREEN: `npx vitest run test/tht-runner.test.ts test/routes-sessions.test.ts test/routes-settings.test.ts && npx tsc --noEmit -p .`
|
||||
completed with 97 passing tests and a clean type check.
|
||||
- GREEN: `THT_HOME=/private/tmp/thothii-task7-home .venv/bin/pytest tests/test_session_documents.py tests/test_session_mutations.py -q`
|
||||
completed with 22 passing tests.
|
||||
- `git diff --check` completed cleanly.
|
||||
|
||||
## Review fixes — round 3
|
||||
|
||||
- The active registry snapshot that located a session now remains the authorization and mutation
|
||||
config for response, steer, events, close/delete, archive/group/rename, documents, and detail.
|
||||
A pruned historical revision cannot block an already-located session's active lifecycle.
|
||||
- Only Resume resolves the retained pinned descriptor because Pi needs that immutable config to
|
||||
restart safely. A pruned pin therefore returns the existing sanitized
|
||||
`workspace_revision_unavailable` 409 solely for Resume.
|
||||
|
||||
### Round 3 verification
|
||||
|
||||
- RED: with a manifest found through an active registry snapshot and `readPinned` forced to fail,
|
||||
`POST /sessions/:id/response` returned 409 instead of forwarding the active gate response.
|
||||
- GREEN: `npx vitest run test/routes-sessions.test.ts test/tht-runner.test.ts test/routes-settings.test.ts && npx tsc --noEmit -p .`
|
||||
— 102 tests passed with a clean type check. The regression confirms response, close, and delete
|
||||
use the locating snapshot without calling `readPinned`, while Resume returns a sanitized 409.
|
||||
- `git diff --check` completed cleanly.
|
||||
|
||||
## Review fixes — round 2
|
||||
|
||||
- Lifecycle authorization no longer selects the installation-default workspace. The backend now
|
||||
finds each session by querying every operational registry snapshot with the authenticated
|
||||
principal, preserving RLS ownership concealment.
|
||||
- After locating the manifest, durable pinned sessions resolve their retained descriptor before
|
||||
any lifecycle mutation/reopen. Legacy sessions continue using the locating registry snapshot.
|
||||
- Session listing aggregates the owner-visible rows from all operational registry snapshots;
|
||||
detail, response, steer, resume, events, documents, and lifecycle mutations use the same
|
||||
server-side locator. No route depends on browser-local workspace state.
|
||||
|
||||
### Round 2 verification
|
||||
|
||||
- RED: the new cross-workspace route integration test created a B session while installation
|
||||
default A was selected, then demonstrated that `GET /sessions` returned an empty list.
|
||||
- GREEN: `npx vitest run test/routes-sessions.test.ts test/tht-runner.test.ts test/routes-settings.test.ts && npx tsc --noEmit -p .`
|
||||
— 101 tests passed with a clean type check. The integration test covers create B, list, detail,
|
||||
response, and resume through B's pinned descriptor while default A remains configured.
|
||||
- Full backend suite: 342 tests passed. The remaining 7 tests require binding `127.0.0.1` and
|
||||
fail in this sandbox with `listen EPERM: operation not permitted`; no application assertion
|
||||
failed. The focused typecheck above passed.
|
||||
- `git diff --check` completed cleanly.
|
||||
|
||||
## Verification note
|
||||
|
||||
The unscoped backend suite was also run. The Task 7 code regressions in `test/tht-runner.test.ts`
|
||||
were fixed; the remaining failures were existing sandbox restrictions on tests that listen on
|
||||
`127.0.0.1` (`listen EPERM: operation not permitted` in SSE/e2e health tests), not application
|
||||
assertions.
|
||||
|
||||
## Review fixes — round 1
|
||||
|
||||
- Every new session now resolves `workspaceId` through the registry; an omitted value uses the
|
||||
configured installation default and persists both the resolved ID and revision. Callers cannot
|
||||
bypass revision pinning by supplying a workspace ID.
|
||||
- Browser-local preferences now migrate once from the read-only legacy settings response and hold
|
||||
workspace, provider, model, and thinking. Session creation includes those selections, including
|
||||
direct entry points that run before the composer mounts. The frontend no longer `PUT`s shared
|
||||
settings.
|
||||
- The settings compatibility endpoint honors a stored installation workspace before falling back
|
||||
to the first workspace configuration.
|
||||
- Resume rejects finalized and archived sessions before looking up any pinned snapshot, preserving
|
||||
the read-only response even when a historical snapshot is unavailable.
|
||||
|
||||
### Review verification
|
||||
|
||||
- RED: the added backend tests failed for omitted-default pinning, read-only resume ordering, and
|
||||
stored-default precedence; the added frontend preference tests failed because preferences were
|
||||
neither stored nor included in session requests.
|
||||
- GREEN: `npx vitest run test/tht-runner.test.ts test/routes-sessions.test.ts test/routes-settings.test.ts && npx tsc --noEmit -p .`
|
||||
— 100 tests passed with a clean type check.
|
||||
- GREEN: `npx vitest run && npx tsc -b` — 332 frontend tests passed with a clean type check.
|
||||
- GREEN: `THT_HOME=/private/tmp/thothii-task7-home .venv/bin/pytest tests/test_session_documents.py tests/test_session_mutations.py -q`
|
||||
— 22 tests passed (one existing testcontainers deprecation warning).
|
||||
- `git diff --check` completed cleanly.
|
||||
@@ -1,73 +0,0 @@
|
||||
# Task 9 report — Workspace Management CRUD page
|
||||
|
||||
## Delivered
|
||||
|
||||
- Added the Workspace management dialog, launched from the persistent right sidebar and the
|
||||
Model activity header without touching live-session/SSE state.
|
||||
- Added a workspace list/detail editor for General, DWH, Semantic index, LLM policy,
|
||||
Installation requirements, and Git status/history.
|
||||
- Added browser-only New, Edit, Duplicate, Save draft, and Delete-draft workflows. A deletion
|
||||
draft stores only ID and immutable revision references; publication remains a Task 10 action.
|
||||
- Used closed native controls for languages, engines, transports, distance metrics, embedding
|
||||
providers, and selectable default models. Free values have client-side, accessible errors.
|
||||
- Made semantic-index dimensions atomic: one editor field always writes the same value to the
|
||||
vector-store and embedding contracts.
|
||||
- Added Validate and Test-on-this-installation actions. They display sanitized code/message
|
||||
diagnostics only; neither action exposes or stores credentials, secrets, or raw response bodies.
|
||||
- Explicitly excluded publish, pull, import, and export user flows from this task.
|
||||
|
||||
## TDD evidence
|
||||
|
||||
- RED: `npx vitest run src/shell/WorkspaceManager.test.tsx src/shell/WorkspaceEditor.test.tsx`
|
||||
failed because the manager and editor modules did not exist.
|
||||
- GREEN: focused manager/editor/AppShell coverage passed after the implementation.
|
||||
- RED: a deletion-draft persistence regression failed with
|
||||
`Cannot read properties of undefined (reading 'save')` before the sanitized draft store was added.
|
||||
- GREEN: the draft-store and manager tests passed once deletion intent persisted locally.
|
||||
|
||||
## Verification
|
||||
|
||||
Executed from `frontend/`:
|
||||
|
||||
```text
|
||||
npx vitest run
|
||||
50 test files passed, 358 tests passed
|
||||
npx tsc -b
|
||||
exit 0
|
||||
```
|
||||
|
||||
`git diff --check` passed before commit. No workspace secret value, secret-file path, raw
|
||||
diagnostic body, publish call, import flow, or export flow was introduced.
|
||||
|
||||
## Fix round 1
|
||||
|
||||
### Root causes and fixes
|
||||
|
||||
- The original duplicate proposal appended `-copy` and then truncated at 63 characters. For an
|
||||
already-maximal ID, truncation could remove the suffix and reproduce the immutable source ID.
|
||||
The proposal now reserves suffix space and falls back to a distinct `-2` suffix when a maximal
|
||||
source already ends in `-copy`.
|
||||
- `dwh.timeout_ms` was rendered as a positive numeric field but was absent from the client
|
||||
validation map. It now has the same immediate accessible error treatment as other numeric
|
||||
fields, so a rejected save never reaches the manager’s saved-draft toast.
|
||||
- Registry status, workspace list, and selected-detail React Query failures were rendered as
|
||||
loading, empty, or unselected states. Each now has a named alert and a retry control, distinct
|
||||
from its corresponding loading and empty state.
|
||||
|
||||
### TDD evidence
|
||||
|
||||
- RED: max-length duplication retained the original 63-character ID; the timeout field produced
|
||||
no alert; and each of the three failed queries had no accessible retry control.
|
||||
- GREEN: the focused manager/editor tests passed **12/12**, covering a valid changed duplicate
|
||||
proposal, rejected zero timeout with no save toast, and status/list/detail retry recovery.
|
||||
|
||||
### Verification
|
||||
|
||||
Executed from `frontend/`:
|
||||
|
||||
```text
|
||||
npx vitest run
|
||||
50 test files passed, 364 tests passed
|
||||
npx tsc -b
|
||||
exit 0
|
||||
```
|
||||
@@ -1,180 +0,0 @@
|
||||
# Task 11 report
|
||||
|
||||
Status: completed on 2026-08-08.
|
||||
|
||||
## Scope delivered
|
||||
|
||||
- Updated operator-facing documentation for the internal Qdrant + Ollama architecture.
|
||||
- Tightened documentation contract tests to require the current four-service-plus-init topology,
|
||||
CPU-first/GPU-override guidance, fixed internal model/dimensions, schema-v3 migration wording,
|
||||
one-collection-per-workspace ownership, and Qdrant backup/restore safety.
|
||||
- Updated stable repo guidance in `AGENTS.md` and the current snapshot in `PROJECT_STATE.md`.
|
||||
- Rewrote the workspace diagnostic protocol to the schema-v3/internal-semantic-service contract.
|
||||
- Updated the memory guide to describe Qdrant as the derived persistent index.
|
||||
- Updated the runtime secret-bundle guide to remove active vector/embedding secret guidance.
|
||||
|
||||
## Files changed
|
||||
|
||||
- `README.md`
|
||||
- `AGENTS.md`
|
||||
- `PROJECT_STATE.md`
|
||||
- `docs/install/local-workspace-registry.md`
|
||||
- `docs/install/server-workspace-registry.md`
|
||||
- `docs/installazione-docker-4-contesti.md`
|
||||
- `docs/workspace-diagnostic-protocol.md`
|
||||
- `docs/gestione-memory.md`
|
||||
- `deploy/secrets/README.md`
|
||||
- `scripts/verify-workspace-install-docs.sh`
|
||||
- `scripts/test-verify-workspace-install-docs.sh`
|
||||
|
||||
## Verification
|
||||
|
||||
Fresh successful runs:
|
||||
|
||||
```sh
|
||||
./scripts/test-verify-workspace-install-docs.sh
|
||||
./scripts/verify-workspace-install-docs.sh --fixtures-only
|
||||
git diff --check
|
||||
```
|
||||
|
||||
Key outcomes:
|
||||
|
||||
- internal semantic infrastructure documentation contract passed
|
||||
- all existing install/manual fixture contracts still passed
|
||||
- diff hygiene passed with no whitespace/errors
|
||||
|
||||
## Self-review notes
|
||||
|
||||
- The updated docs now match the code-backed Compose topology: `frontend`, `core`, `qdrant`,
|
||||
`embedding`, and `embedding-model-init`.
|
||||
- Active manuals no longer instruct operators to configure external vector or embedding runtime
|
||||
endpoints/secrets.
|
||||
- Qdrant backup/restore wording now matches the helper scripts' exact confirmation and rollback
|
||||
behavior.
|
||||
- Legacy descriptor handling is documented as explicit schema-v3 migration only; no silent
|
||||
semantic-data migration is claimed.
|
||||
|
||||
## Residual concerns
|
||||
|
||||
- The broader repository still contains historical design/spec material that references older
|
||||
pgvector/external-embedding architecture; this task intentionally updated operator/current-state
|
||||
documentation and the corresponding contract tests, not historical planning documents.
|
||||
|
||||
## Fix round 1/5 — 2026-08-08
|
||||
|
||||
Addressed reviewer findings:
|
||||
|
||||
- Moved superseded rollout/state blocks in `PROJECT_STATE.md` behind an explicit
|
||||
`## Historical snapshots and archived reference notes` boundary.
|
||||
- Renamed superseded snapshot headings so historical notes no longer present as active `LIVE`
|
||||
state.
|
||||
- Added a current-state regression that rejects contradictory active blocks (for example:
|
||||
schema-v2 operational, two-service active stack, or external vector/embedding runtime claims
|
||||
before the historical boundary).
|
||||
- Refactored new internal-semantic doc checks away from exact-sentence coupling:
|
||||
- parse `compose.yaml` structurally with YAML;
|
||||
- parse workspace examples structurally with YAML;
|
||||
- inspect backup/restore stable usage interface;
|
||||
- keep targeted forbidden-term checks for active docs while allowing historical sections;
|
||||
- use regex/concept checks for prose.
|
||||
|
||||
Evidence:
|
||||
|
||||
```sh
|
||||
./scripts/test-verify-workspace-install-docs.sh
|
||||
./scripts/verify-workspace-install-docs.sh --fixtures-only
|
||||
git diff --check
|
||||
```
|
||||
|
||||
Observed RED before the fix:
|
||||
|
||||
```text
|
||||
PROJECT_STATE.md: missing Historical snapshots boundary
|
||||
```
|
||||
|
||||
## Fix round 2/5 — 2026-08-08
|
||||
|
||||
Addressed reviewer findings:
|
||||
|
||||
- Renamed every historical `PROJECT_STATE.md` heading after the historical boundary so no heading
|
||||
level uses `LIVE` or current-state semantics there.
|
||||
- Strengthened the historical-boundary regression to reject any Markdown heading level
|
||||
(`#` through `######`) containing `LIVE` or current-state wording after the boundary.
|
||||
- Added a fixture with a `### ... — LIVE ...` historical heading to prove RED then GREEN.
|
||||
- Replaced remaining exact phrase checks with concept/semantic validation for:
|
||||
- one-workspace/one-collection ownership;
|
||||
- external boundary (DWH/LLM external; vector/embedding internal);
|
||||
- the Italian compact install note.
|
||||
- Added paraphrase fixtures that pass and omission/inversion fixtures that fail.
|
||||
|
||||
Evidence:
|
||||
|
||||
```sh
|
||||
./scripts/test-verify-workspace-install-docs.sh
|
||||
./scripts/verify-workspace-install-docs.sh --fixtures-only
|
||||
git diff --check
|
||||
```
|
||||
|
||||
## Fix round 4/5 — 2026-08-08
|
||||
|
||||
Addressed reviewer finding:
|
||||
|
||||
- Eliminated semantic-index verifier/test contract drift by extracting the production
|
||||
semantic-index ownership row matcher into `semantic_index_relationship_spec` and reusing it in
|
||||
the fixture-level paraphrase, omission, and scattered-token checks.
|
||||
- Kept the relationship constrained to one structured Markdown table row via
|
||||
`verify_markdown_table_relationships`; the scattered-token fixture still removes the row and
|
||||
appends the same words outside the table, where it must be rejected.
|
||||
- Added a direct regression that copies the repository docs into an isolated root, applies the
|
||||
accepted paraphrase “A workspace keeps exactly one Qdrant collection reserved for itself”, and
|
||||
runs that root's actual `scripts/verify-workspace-install-docs.sh --fixtures-only` instead of a
|
||||
separate temporary spec.
|
||||
|
||||
Observed RED before the fix:
|
||||
|
||||
```text
|
||||
production verifier rejected the accepted semantic-index paraphrase
|
||||
local workspace manual: missing relationship in 'Semantic index ownership contract': {'scope': 'workspace semantic index', 'ownership rule': '(each|one|single).*(workspace).*(single|one).*(Qdrant).*(collection)|(each workspace reserves a single qdrant collection)', 'isolation rule': 'schema.*evidence.*memory.*(one|that).*(collection).*(kind|payload)'}
|
||||
```
|
||||
|
||||
Evidence:
|
||||
|
||||
```sh
|
||||
./scripts/test-verify-workspace-install-docs.sh
|
||||
./scripts/verify-workspace-install-docs.sh --fixtures-only
|
||||
git diff --check
|
||||
```
|
||||
|
||||
Observed RED during this round:
|
||||
|
||||
```text
|
||||
PROJECT_STATE.md: historical section still contains active/live heading markers
|
||||
compact manual paraphrase lacks required pattern: (esterni solo|solo esterni|restano esterni)
|
||||
```
|
||||
|
||||
## Fix round 3/5 — 2026-08-08
|
||||
|
||||
Addressed reviewer findings:
|
||||
|
||||
- Added table-driven historical-heading fixtures for every Markdown heading level `#` through
|
||||
`######`; all are rejected after the historical boundary when they contain `LIVE`/current-state
|
||||
semantics.
|
||||
- Added small structured ownership tables to the active local/server manuals and to the compact
|
||||
Italian operator note.
|
||||
- Added small structured semantic-index ownership tables to the active local/server manuals.
|
||||
- Replaced the remaining scattered-token relationship checks with explicit structured-section
|
||||
parsing:
|
||||
- architecture ownership rows map DWH → external, LLM → external, Qdrant → internal,
|
||||
Ollama embedding → internal;
|
||||
- semantic-index ownership rows localize the one-workspace/one-collection contract and the
|
||||
schema/Evidence/Memory isolation rule.
|
||||
- Added adversarial fixtures that fail when the same tokens are merely scattered in free text.
|
||||
- Added structured paraphrase fixtures that pass and omission/inversion fixtures that fail.
|
||||
|
||||
Evidence:
|
||||
|
||||
```sh
|
||||
./scripts/test-verify-workspace-install-docs.sh
|
||||
./scripts/verify-workspace-install-docs.sh --fixtures-only
|
||||
git diff --check
|
||||
```
|
||||
@@ -1,43 +0,0 @@
|
||||
# Task 12 Report — Remove unreachable pgvector runtime code
|
||||
|
||||
Status: completed
|
||||
|
||||
Summary:
|
||||
- Proved the retired pgvector runtime had no remaining operational adapter call sites after migration by re-running the required grep; only the packaging assertion still mentions `migrations/vector`.
|
||||
- Removed the obsolete pgvector/HTTP/direct vector runtime modules, vector SQL migrations, and their affected runtime tests.
|
||||
- Kept the operational semantic path on Qdrant and migrated the remaining runtime callers to that path.
|
||||
- Kept `psycopg2-binary` because DWH direct PostgreSQL and session PostgreSQL code still depend on it.
|
||||
|
||||
Implementation notes:
|
||||
- Extracted shared collection/kind validation into `harness/tht/adapters/vector/_shared.py` so `QdrantVectorStore` no longer depends on the deleted pgvector module.
|
||||
- Simplified `build_vector_store()` to return only `QdrantVectorStore`.
|
||||
- Migrated vector/evidence/memory CLI paths away from legacy pgvector loaders and REST vector clients.
|
||||
- Updated packaging coverage so the built wheel asserts session SQL migrations are present and vector SQL migrations are absent.
|
||||
|
||||
Verification:
|
||||
- `cd harness && .venv/bin/pytest tests/test_qdrant_vector_store.py tests/test_vector_port_contract.py tests/test_semantic_kind_isolation.py tests/test_vector_migration_packaging.py -q`
|
||||
- `cd harness && .venv/bin/pytest tests/test_adapter_factory.py tests/test_solved_search_cli.py -q`
|
||||
- `cd harness && .venv/bin/python -c "import tht.cli, tht.adapters.factory, tht.adapters.vector, tht.vectorstore.reader"`
|
||||
- `cd harness && uv build`
|
||||
- `harness/.venv/bin/ruff check harness/tests/test_adapter_factory.py harness/tests/test_solved_search_cli.py harness/tests/test_vector_migration_packaging.py harness/tests/test_vector_port_contract.py harness/tht/adapters/factory.py harness/tht/adapters/vector/__init__.py harness/tht/adapters/vector/_shared.py harness/tht/adapters/vector/qdrant.py harness/tht/cli/evidence_cmd.py harness/tht/cli/memory_cmd.py harness/tht/cli/search_cmd.py harness/tht/cli/vector_cmd.py harness/tht/solved.py harness/tht/vectorstore/reader.py`
|
||||
- `git diff --check`
|
||||
|
||||
Notes / concerns:
|
||||
- Repository-wide `harness/.venv/bin/ruff check .` still reports many pre-existing findings outside this task’s touched files; it is not clean on this branch baseline.
|
||||
- Some legacy config compatibility parsing still exists outside the deleted runtime path. This task removed the unreachable runtime/migration code without broad config-schema refactoring.
|
||||
|
||||
## Fix round 1 evidence
|
||||
|
||||
Changes:
|
||||
- Removed dead `vector migrate` registration from `harness/tht/cli/__init__.py` and deleted `harness/tht/cli/vector_migrate_cmd.py`.
|
||||
- Added CLI regressions proving `vector migrate` is absent while `vector init` and `vector index-schema` remain available.
|
||||
- Restored the accidentally removed non-vector regressions by moving report coverage into `harness/tests/test_report.py` and restoring the taskdoc promoted-table slicing check in `harness/tests/test_taskdoc.py`.
|
||||
- Reworded surviving active help/docstrings away from pgvector-specific wording in the touched Qdrant-backed command surface.
|
||||
|
||||
Verification:
|
||||
- `cd harness && .venv/bin/pytest tests/test_qdrant_cli_commands.py tests/test_report.py tests/test_taskdoc.py tests/test_vector_migration_packaging.py -q`
|
||||
- `cd harness && .venv/bin/python -c "from typer.testing import CliRunner; from tht.cli import app; r=CliRunner().invoke(app, ['vector','--help']); assert r.exit_code == 0, r.output; assert 'migrate' not in r.output; r=CliRunner().invoke(app, ['vector','migrate','--help']); assert r.exit_code != 0, r.output; print('cli-help-ok')"`
|
||||
- `cd harness && .venv/bin/python -c "import tht.cli, tht.cli.vector_cmd, tht.report, tht.taskdoc; print('imports-ok')"`
|
||||
- `cd harness && uv build`
|
||||
- `harness/.venv/bin/ruff check harness/tests/test_qdrant_cli_commands.py harness/tests/test_report.py harness/tests/test_taskdoc.py harness/tests/test_vector_migration_packaging.py harness/tht/cli/__init__.py harness/tht/cli/search_cmd.py harness/tht/cli/vector_cmd.py harness/tht/cli/memory_cmd.py harness/tht/solved.py`
|
||||
- `git diff --check`
|
||||
@@ -1,175 +0,0 @@
|
||||
# Task 13 Implementation Report
|
||||
|
||||
## Status
|
||||
|
||||
DONE_WITH_CONCERNS
|
||||
|
||||
## Changes
|
||||
|
||||
- Updated stale harness/backend/frontend tests and fixtures to the Task 13 internal Qdrant/Ollama contract.
|
||||
- Made `deploy/workspaces/psd.yaml.example` generic while preserving schema-v3 Qdrant/Ollama shape.
|
||||
- Fixed `scripts/workspace-registry-smoke.sh` to pass the required legacy migration `--collection` and prove exact Docker cleanup, including its smoke image.
|
||||
- Updated `PROJECT_STATE.md` with only evidence observed in this run.
|
||||
|
||||
Changed files:
|
||||
|
||||
- `PROJECT_STATE.md`
|
||||
- `backend/test/routes-workspaces.test.ts`
|
||||
- `backend/test/workspace-runtime-handoff.test.ts`
|
||||
- `backend/test/workspaces-contracts.test.ts`
|
||||
- `backend/test/workspaces-git-repository.test.ts`
|
||||
- `deploy/workspaces/psd.yaml.example`
|
||||
- `frontend/src/shell/NewSessionDialog.test.tsx`
|
||||
- `harness/tests/test_adapter_command_regressions.py`
|
||||
- `harness/tests/test_workspace.py`
|
||||
- `scripts/task13-runtime-fixture-check.ts`
|
||||
- `scripts/test-verify-workspace-install-docs.sh`
|
||||
- `scripts/workspace-registry-smoke.sh`
|
||||
|
||||
## Verification
|
||||
|
||||
Deterministic gates:
|
||||
|
||||
- `cd harness && .venv/bin/pytest -q && .venv/bin/ruff check .`
|
||||
- Initial red: 2 harness pytest failures.
|
||||
- After fixture fixes: harness pytest passed `819 passed, 4 deselected, 74 warnings in 27.73s`.
|
||||
- Ruff still failed with `Found 220 errors`; treated as existing unrelated debt.
|
||||
- Touched harness files verified clean with `cd harness && .venv/bin/ruff check tests/test_adapter_command_regressions.py tests/test_workspace.py && .venv/bin/pytest -q tests/test_adapter_command_regressions.py::test_solved_index_writes_through_writer_only_factory_store tests/test_workspace.py::test_load_workspace_expands_env_vars`: `All checks passed!` and `2 passed, 2 warnings in 0.14s`.
|
||||
- `cd backend && npx vitest run && npx tsc --noEmit -p . && npm run build`
|
||||
- Initial red: 4 backend Vitest failures.
|
||||
- After fixes: `Test Files 39 passed (39)`, `Tests 464 passed (464)`, TypeScript passed, build passed.
|
||||
- `cd frontend && npx vitest run && npx tsc -b && npm run build`
|
||||
- Initial red: 1 frontend Vitest failure.
|
||||
- After fix: frontend Vitest passed `374/374`, TypeScript passed, build passed with Vite `built in 6.55s`.
|
||||
- `git diff --check`
|
||||
- Passed with no output.
|
||||
|
||||
Focused reruns:
|
||||
|
||||
- `cd backend && npx vitest run test/workspaces-migrate-legacy.test.ts test/workspaces-contracts.test.ts test/routes-workspaces.test.ts test/workspace-runtime-handoff.test.ts test/workspaces-git-repository.test.ts && cd .. && ./scripts/test-no-deployment-coupling.sh && ./scripts/verify-workspace-install-docs.sh --fixtures-only && git diff --check`
|
||||
- `Test Files 5 passed (5)`, `Tests 35 passed (35)`.
|
||||
- Coupling guard passed: `no active retired deployment or external semantic coupling found.`
|
||||
- Install docs fixtures passed through `relative secret-source fixture rejected passed`.
|
||||
|
||||
Deployment contracts:
|
||||
|
||||
- `./scripts/test-default-compose.sh && ./scripts/test-unified-compose.sh && ./scripts/test-internal-semantic-compose.sh && ./scripts/test-no-deployment-coupling.sh && ./scripts/test-compose-secret-policy.sh && ./scripts/verify-workspace-install-docs.sh --fixtures-only`
|
||||
- Passed. Output included:
|
||||
- `default Compose contract passed.`
|
||||
- `unified Compose contract passed.`
|
||||
- `internal semantic Compose/script contracts passed.`
|
||||
- `no active retired deployment or external semantic coupling found.`
|
||||
- `Compose secret policy passed.`
|
||||
- install-doc fixture checks through `relative secret-source fixture rejected passed`.
|
||||
|
||||
Docker smokes:
|
||||
|
||||
- `/usr/bin/time -p ./scripts/internal-semantic-smoke.sh`
|
||||
- Passed: `Task 13 internal semantic smoke passed.`
|
||||
- Cleanup proof: `no labeled containers, volumes, networks, or images remain for 20260808200245-83368-17823.`
|
||||
- Duration: `real 217.34`.
|
||||
- `/usr/bin/time -p ./scripts/workspace-registry-smoke.sh`
|
||||
- Initial red: `usage: migrate-legacy --input <legacy-workspace.yaml> --output <repository-root> --collection <qdrant-collection> [--id <workspace-id>]`.
|
||||
- After fix: `workspace registry smoke passed`.
|
||||
- Cleanup proof: `no compose containers, volumes, networks, or image remain for thoth-workspace-registry-smoke-89671.`
|
||||
- Duration: `real 9.93`.
|
||||
- `/usr/bin/time -p ./scripts/unified-deployment-smoke.sh`
|
||||
- Passed: `Task 13 full deployment smoke passed.`
|
||||
- Cleanup proof: `no labeled containers, volumes, networks, or images remain for 20260808200706-85638-13391.`
|
||||
- Duration: `real 125.57`.
|
||||
- `/usr/bin/time -p ./scripts/thothctl-update-smoke.sh`
|
||||
- Passed: `Task 13 update deployment smoke passed.`
|
||||
- Cleanup proof: `no labeled containers, volumes, networks, or images remain for 20260808200918-87340-10404.`
|
||||
- Duration: `real 85.40`.
|
||||
- `/usr/bin/time -p ./scripts/server-deployment-smoke.sh`
|
||||
- Passed: `Task 13 Linux server deployment smoke passed.`
|
||||
- Cleanup proof: `no labeled containers, volumes, networks, or images remain for 20260808201047-88645-20675.`
|
||||
- Duration: `real 55.99`.
|
||||
|
||||
Final audit:
|
||||
|
||||
- `rg -n "pgvector|local-vector|THT_VECTOR_|EMBEDDING_BASE_URL|openai_compatible|ollama_compatible" . --glob '!docs/plans/**' --glob '!docs/superpowers/**' --glob '!**/node_modules/**' --glob '!**/.venv/**' --glob '!**/.git/**'`
|
||||
- Returned matches in legacy schema-v1/v2 support, migration tests, negative guards, historical notes, and older harness docs/code.
|
||||
- This remains a concern: the audit is not clean under the brief's strict expected outcome.
|
||||
- `git status --short`
|
||||
- Before report/commit, contained only intentional Task 13 changes.
|
||||
|
||||
## Image and Host Evidence
|
||||
|
||||
- Host CPU: `Apple M4 Pro`.
|
||||
- Host OS: `Darwin MacProM4-di-Marco.local 25.5.0 Darwin Kernel Version 25.5.0: Tue Jun 9 22:28:34 PDT 2026; root:xnu-12377.121.10~1/RELEASE_ARM64_T6041 arm64`.
|
||||
- Docker server: `29.6.2 linux/arm64`.
|
||||
- Verified pinned images:
|
||||
- `qdrant/qdrant:v1.18.2@sha256:75eab8c4ba42096724fdcfde8b4de0b5713d529dde32f285a1f86fdcb2c9e50c`.
|
||||
- `ollama/ollama:0.32.0@sha256:57f573b47f1f71ebb445789f279fe3e596a8beab182f7cf486db9205bad87c5a`.
|
||||
- Workspace registry smoke ephemeral image:
|
||||
- Manifest list: `sha256:4d056bf2cb38d0e8ede91fbf121df1f9f18caee0d401581618ccef9ed8a55e73`.
|
||||
- Config: `sha256:613f8fb28c0517adee4085f41bc447f2c3813b0fdbb7b26624bfb4cb192b6fd8`.
|
||||
- Removed during cleanup.
|
||||
|
||||
## Manual Gates
|
||||
|
||||
- GPU exposure gate (`THOTH_ENABLE_EMBEDDING_GPU=1` on Linux): not executed in this run.
|
||||
- Windows Docker Desktop startup/manual job: not executed in this run.
|
||||
|
||||
## Commits
|
||||
|
||||
- `4e810af` (`test: align qdrant ollama verification fixtures`)
|
||||
- `7c09b98` (`docs: record qdrant ollama verification`)
|
||||
|
||||
## Known Limitations
|
||||
|
||||
- Broad harness Ruff remains existing unrelated debt: `Found 220 errors`.
|
||||
- Final active-reference audit is not clean; it still finds legacy/negative-guard references outside explicit migration fixture files.
|
||||
- Ephemeral Task 13 core/frontend image IDs from `internal-semantic-smoke.sh`, `unified-deployment-smoke.sh`, `thothctl-update-smoke.sh`, and `server-deployment-smoke.sh` were removed by exact cleanup and were not emitted in stdout; pinned Qdrant/Ollama digests and the workspace-registry smoke image digest were captured.
|
||||
|
||||
## Fix Round 1 — reviewer findings
|
||||
|
||||
Status: DONE
|
||||
|
||||
Changes:
|
||||
|
||||
- `scripts/workspace-registry-smoke.sh` now derives the smoke image reference from the already unique Compose project instead of using the global tag `thothii-workspace-registry-smoke:local`.
|
||||
- The workspace-registry cleanup helpers remove and verify only the exact per-run image reference, plus Compose resources labeled with the exact project.
|
||||
- Added deterministic self-test coverage in `backend/test/workspaces-migrate-legacy.test.ts` via `WORKSPACE_REGISTRY_SMOKE_SELF_TEST=image-cleanup-identity`; it stubs Docker and fails if cleanup touches same-repository foreign tags such as `:local` or another project tag.
|
||||
- Updated active harness/testing/PRD docs and Python comments that still described the current semantic store as pgvector/vectordb. Preserved schema-v1/v2 and harness legacy compatibility fixtures.
|
||||
- Updated `PROJECT_STATE.md` with fix-round smoke evidence and a precise, non-overclaiming audit limitation.
|
||||
|
||||
Focused verification:
|
||||
|
||||
- `cd backend && npx vitest run test/workspaces-migrate-legacy.test.ts`
|
||||
- Passed: `7 passed`.
|
||||
- `cd harness && .venv/bin/pytest -q tests/test_memory_save_one.py tests/test_adapter_command_regressions.py tests/test_solved_search_cli.py tests/test_search_pack.py`
|
||||
- Passed: `22 passed, 14 warnings`.
|
||||
- `cd harness && .venv/bin/ruff check tht/memory.py tht/search/__init__.py tht/workspace.py tht/vectorstore/store.py tests/test_memory_save_one.py tests/test_adapter_command_regressions.py tests/test_solved_search_cli.py`
|
||||
- Passed: `All checks passed!`
|
||||
- `bash -n scripts/workspace-registry-smoke.sh && WORKSPACE_REGISTRY_SMOKE_SELF_TEST=image-cleanup-identity bash scripts/workspace-registry-smoke.sh`
|
||||
- Passed: `workspace registry smoke image cleanup identity self-test passed`.
|
||||
- `./scripts/test-no-deployment-coupling.sh`
|
||||
- Passed: `no active retired deployment or external semantic coupling found.`
|
||||
- `./scripts/verify-workspace-install-docs.sh --fixtures-only`
|
||||
- Passed through `relative secret-source fixture rejected passed`.
|
||||
- `cd backend && npx tsc --noEmit -p .`
|
||||
- Passed with no output.
|
||||
- `/usr/bin/time -p ./scripts/workspace-registry-smoke.sh`
|
||||
- Passed: `workspace registry smoke passed`.
|
||||
- Built exact per-run tag: `thothii-workspace-registry-smoke:thoth-workspace-registry-smoke-thoth-workspace-registry-smoke-10vi3a-19157`.
|
||||
- Manifest list: `sha256:715b943057929418cad4aa71806d9edbaf823555d19bda6b875297617463fd4a`.
|
||||
- Config: `sha256:a566521981e08958aae9a12bfc7803bb5f3f835536b4bb8c39df8fcf26063161`.
|
||||
- Cleanup proof: `no compose containers, volumes, networks, or image remain for thoth-workspace-registry-smoke-thoth-workspace-registry-smoke-10vi3a-19157.`
|
||||
- Duration: `real 42.06`.
|
||||
|
||||
Fix-round audit command:
|
||||
|
||||
- `rg -n "pgvector|local-vector|THT_VECTOR_|EMBEDDING_BASE_URL|openai_compatible|ollama_compatible" . --glob '!docs/plans/**' --glob '!docs/superpowers/**' --glob '!**/node_modules/**' --glob '!**/.venv/**' --glob '!**/.git/**'`
|
||||
|
||||
Categorized remaining hits:
|
||||
|
||||
- Backend legacy parser/migration compatibility, kept deliberately non-operational for schema-v1/v2 descriptors: `backend/src/workspaces/schema.ts`, `types.ts`, `migrate-legacy.ts`, `runtime-renderer.ts`, `bindings.ts`, `contracts.ts`, `diagnostics.ts`.
|
||||
- Backend negative guards and legacy fixture tests: `backend/test/workspaces-schema.test.ts`, `workspaces-migrate-v2-qdrant.test.ts`, `workspace-registry.test.ts`, `workspace-runtime-renderer.test.ts`, `workspaces-bindings.test.ts`, `workspaces-contracts.test.ts`, `workspaces-diagnostics.test.ts`, `workspaces-git-repository.test.ts`, `routes-workspaces.test.ts`, `routes-sessions.test.ts`, `provider-credentials.test.ts`.
|
||||
- Secret/env scrub guards for retired variables: `backend/src/config.ts`, `backend/src/config/secret-bundle.ts`, `backend/src/pi/provider-credentials.ts`, `scripts/compose-with-preflight.sh`, `scripts/test-external-compose-lifecycle.sh`.
|
||||
- Deployment negative guards and fixture-scope tests: `scripts/test-no-deployment-coupling.sh`, `scripts/test-no-deployment-coupling-scope.sh`, `scripts/test-preprocess-compose-config.sh`, `scripts/test-verify-workspace-install-docs.sh`, `scripts/verify-workspace-install-docs.sh`, `scripts/vector-rotate-bootstrap-password.sh`.
|
||||
- Harness legacy config compatibility and fixtures: `harness/tht/config.py`, `harness/tht/config_compat.py`, `harness/tests/test_config_resources.py`, `harness/tests/l2/test_session_ablazione.py`, `harness/workspaces/tht.example.yaml`, `harness/workspaces/tht-test.yaml`.
|
||||
- Retained off-repository migration SQL fixtures: `harness/scripts/create_vector_reader_rpc.sql`, `harness/scripts/create_vector_writer_rpc.sql`.
|
||||
- Historical/reference notes, not active operator contracts: `brain/codebase/datamart-builder-deployment-gotchas.md`, `PROJECT_STATE.md`.
|
||||
- Gitignored task report self-reference: `.superpowers/sdd/2026-08-08-internal-qdrant-ollama/task-13-implementation.md`.
|
||||
@@ -1,165 +0,0 @@
|
||||
Task 2 report — Make collection ownership unique in the Git registry
|
||||
|
||||
Summary
|
||||
|
||||
- Implemented unique Qdrant collection ownership enforcement during registry snapshot activation.
|
||||
- Registry session revision leases now reject `migration_required` descriptors.
|
||||
- Legacy migration now requires an explicit target collection and emits schema v3 descriptors.
|
||||
- Preserved active snapshot rollback behavior on invalid pulled snapshots.
|
||||
|
||||
RED evidence
|
||||
|
||||
Focused RED command from the brief:
|
||||
|
||||
```bash
|
||||
cd backend
|
||||
npx vitest run test/workspace-registry.test.ts test/workspaces-migrate-legacy.test.ts \
|
||||
-t "collection|migration_required"
|
||||
```
|
||||
|
||||
Observed failures before implementation:
|
||||
|
||||
- `rejects duplicate schema v3 collection ownership and keeps the previous active snapshot`
|
||||
- `registry.pull()` resolved instead of rejecting.
|
||||
- `does not acquire a session revision lease for a migration_required workspace`
|
||||
- `acquireSessionRevision()` resolved instead of rejecting.
|
||||
- `migrates a legacy descriptor only with an explicit target collection into schema v3`
|
||||
- received schema version `1` instead of `3`.
|
||||
- `requires an explicit target collection for legacy migration`
|
||||
- migration did not throw without a collection.
|
||||
|
||||
GREEN evidence
|
||||
|
||||
Focused GREEN command from the brief:
|
||||
|
||||
```bash
|
||||
cd backend
|
||||
npx vitest run test/workspace-registry.test.ts test/workspaces-migrate-legacy.test.ts \
|
||||
-t "collection|migration_required"
|
||||
```
|
||||
|
||||
Fresh result after implementation:
|
||||
|
||||
- 2 files passed
|
||||
- 4 tests passed
|
||||
- 0 failures
|
||||
|
||||
Additional verification run after final cleanup:
|
||||
|
||||
```bash
|
||||
cd backend
|
||||
npx vitest run test/routes-workspaces.test.ts
|
||||
npx vitest run
|
||||
npx tsc --noEmit -p .
|
||||
git diff --check
|
||||
```
|
||||
|
||||
Fresh results:
|
||||
|
||||
- `test/routes-workspaces.test.ts`: 7 passed
|
||||
- full backend Vitest: 39 files passed, 454 tests passed
|
||||
- backend typecheck: passed
|
||||
- `git diff --check`: passed
|
||||
|
||||
Changed files
|
||||
|
||||
- `backend/src/workspaces/registry.ts`
|
||||
- `backend/src/workspaces/migrate-legacy.ts`
|
||||
- `backend/test/workspace-registry.test.ts`
|
||||
- `backend/test/workspaces-migrate-legacy.test.ts`
|
||||
- `backend/test/routes-workspaces.test.ts`
|
||||
|
||||
Why one extra file changed
|
||||
|
||||
- `backend/test/routes-workspaces.test.ts` needed updating because Task 1 made schema v3 the only operational descriptor shape, and the route test still assumed the old pre-Task-3 runtime behavior. Updating that expectation was necessary to keep the required backend suite verification meaningful.
|
||||
|
||||
Implementation notes
|
||||
|
||||
- Duplicate collection detection is enforced only for operational schema v3 descriptors by tracking `collection -> workspaceId` during activation.
|
||||
- Duplicate failures are sanitized back to `workspace_invalid` / `Workspace repository content is invalid`.
|
||||
- `acquireSessionRevision()` now fails closed for `migration_required` revisions.
|
||||
- Legacy migration CLI now requires `--collection <qdrant-collection>`.
|
||||
- Legacy migration output is schema v3 with the fixed internal semantic contract:
|
||||
- `vector_store.engine = qdrant`
|
||||
- explicit `collection`
|
||||
- embedding provider `ollama_internal`
|
||||
- embedding model `qwen3-embedding:0.6b`
|
||||
|
||||
self-review
|
||||
|
||||
- Confirmed invalid pulled snapshots do not replace the previous active snapshot.
|
||||
- Confirmed duplicate collection enforcement does not affect legacy migration-required descriptors.
|
||||
- Confirmed create/update publication tests still pass with unique per-workspace collections.
|
||||
- Confirmed no JSON stdout contract regressions in the migration CLI.
|
||||
- Kept runtime/data mutation scope descriptor-only; no user workspace repo or Qdrant data changes.
|
||||
|
||||
Concerns
|
||||
|
||||
- No code concerns remaining for Task 2.
|
||||
- One deliberate scope exception: a route test was updated to align with the already-established Task 1 / Task 3 fail-closed contract.
|
||||
|
||||
Fix round 1
|
||||
|
||||
Scope
|
||||
|
||||
- Restored meaningful route-level diagnoser coverage without reopening schema-v3 semantic runtime paths.
|
||||
- Added direct schema-v2 registry coverage for `migration_required` listing and lease rejection.
|
||||
|
||||
Covering test files
|
||||
|
||||
- `backend/test/routes-workspaces.test.ts`
|
||||
- `backend/test/workspace-registry.test.ts`
|
||||
|
||||
RED command and output
|
||||
|
||||
Command:
|
||||
|
||||
```bash
|
||||
cd backend
|
||||
npx vitest run test/routes-workspaces.test.ts test/workspace-registry.test.ts
|
||||
```
|
||||
|
||||
Observed result on top of `76bc94d` after adding the restored/new assertions:
|
||||
|
||||
- 2 files passed
|
||||
- 37 tests passed
|
||||
- 0 failures
|
||||
|
||||
Why no RED appeared:
|
||||
|
||||
- The review items exposed missing/weakened coverage, not a production behavior bug.
|
||||
- `/workspaces/:id/test` already reaches the diagnoser for resolvable legacy v2 descriptors.
|
||||
- Schema-v3 `/workspaces/:id/test` already fails closed before diagnoser entry.
|
||||
- Schema-v2 descriptors were already listed as `migration_required` and already rejected by `acquireSessionRevision()`.
|
||||
|
||||
GREEN command and output
|
||||
|
||||
Command:
|
||||
|
||||
```bash
|
||||
cd backend
|
||||
npx vitest run test/routes-workspaces.test.ts test/workspace-registry.test.ts
|
||||
npx tsc --noEmit -p .
|
||||
```
|
||||
|
||||
Fresh results:
|
||||
|
||||
- covering tests: 2 files passed, 37 tests passed
|
||||
- backend typecheck: passed
|
||||
|
||||
Changed files
|
||||
|
||||
- `backend/test/routes-workspaces.test.ts`
|
||||
- `backend/test/workspace-registry.test.ts`
|
||||
- `.superpowers/sdd/2026-08-08-internal-qdrant-ollama/task-2-report.md`
|
||||
|
||||
What changed
|
||||
|
||||
- Split route coverage so `POST /workspaces/validate` still checks canonical validation independently.
|
||||
- Restored route-level diagnoser coverage through a migration-required schema-v2 descriptor with resolvable legacy bindings.
|
||||
- Added an explicit schema-v3 fail-closed regression for `POST /workspaces/:id/test`.
|
||||
- Added a direct schema-v2 registry regression proving `list()` returns `migration_required` and `acquireSessionRevision()` rejects it.
|
||||
|
||||
Concerns
|
||||
|
||||
- No production concerns. This round only tightened coverage and corrected the weakened test expectation.
|
||||
@@ -1,132 +0,0 @@
|
||||
# Task 3 report — Remove external semantic bindings and render internal endpoints
|
||||
|
||||
Date: 2026-08-08
|
||||
|
||||
## Scope
|
||||
|
||||
Implemented backend-owned schema-v3 semantic runtime rendering so workspace descriptors and installation contracts remain free of external Qdrant/Ollama endpoints and credentials, while DWH bindings stay unchanged.
|
||||
|
||||
## RED evidence
|
||||
|
||||
Focused RED command:
|
||||
|
||||
`cd backend && npx vitest run test/workspaces-contracts.test.ts test/workspaces-bindings.test.ts test/workspace-runtime-renderer.test.ts test/config.test.ts`
|
||||
|
||||
Observed failures before implementation:
|
||||
|
||||
- `config.test.ts`
|
||||
- missing `internalQdrantUrl`
|
||||
- missing `internalEmbeddingUrl`
|
||||
- `workspaces-bindings.test.ts`
|
||||
- schema v3 semantic binding resolution threw unsupported errors
|
||||
- `workspace-runtime-renderer.test.ts`
|
||||
- schema v3 runtime rendering threw `Schema version 3 runtime rendering is unsupported until the internal semantic runtime is implemented`
|
||||
|
||||
## GREEN evidence
|
||||
|
||||
Focused GREEN command:
|
||||
|
||||
`cd backend && npx vitest run test/workspaces-contracts.test.ts test/workspaces-bindings.test.ts test/workspace-runtime-renderer.test.ts test/config.test.ts`
|
||||
|
||||
Result:
|
||||
|
||||
- 4 test files passed
|
||||
- 36 tests passed
|
||||
|
||||
Typecheck:
|
||||
|
||||
`cd backend && npx tsc --noEmit -p .`
|
||||
|
||||
Result:
|
||||
|
||||
- passed
|
||||
|
||||
Hygiene:
|
||||
|
||||
- `git diff --check` passed
|
||||
|
||||
## Files changed
|
||||
|
||||
Listed-task files changed:
|
||||
|
||||
- `backend/src/config.ts`
|
||||
- `backend/src/workspaces/bindings.ts`
|
||||
- `backend/src/workspaces/runtime-renderer.ts`
|
||||
- `backend/test/config.test.ts`
|
||||
- `backend/test/workspace-runtime-renderer.test.ts`
|
||||
- `backend/test/workspaces-bindings.test.ts`
|
||||
- `backend/test/workspaces-contracts.test.ts`
|
||||
|
||||
Listed-task files inspected but not changed:
|
||||
|
||||
- `backend/src/workspaces/contracts.ts`
|
||||
|
||||
Unavoidable additional wiring changes:
|
||||
|
||||
- `backend/src/app.ts`
|
||||
- `backend/src/tht/tht-runner.ts`
|
||||
|
||||
Reason: the new typed internal semantic runtime config had to flow from backend config into ephemeral harness config rendering at runtime.
|
||||
|
||||
## Behavior delivered
|
||||
|
||||
- schema-v3 installation contract exposes DWH bindings only
|
||||
- schema-v3 binding resolution ignores external semantic env vars instead of sourcing runtime semantics from them
|
||||
- runtime rendering for schema v3 emits backend-owned internal semantic endpoints:
|
||||
- Qdrant: `http://qdrant:6333`
|
||||
- Embedding: `http://embedding:11434`
|
||||
- Model: `qwen3-embedding:0.6b`
|
||||
- Dimensions: `1024`
|
||||
- internal semantic URLs are validated to allow only `qdrant` / `embedding` / `localhost` / loopback hosts
|
||||
- DWH transport/runtime behavior remains unchanged
|
||||
|
||||
## Self-review
|
||||
|
||||
- Confirmed schema-v3 contracts/docs no longer advertise VECTOR or EMBEDDING installation variables.
|
||||
- Confirmed schema-v3 runtime output ignores injected external semantic endpoints from env bindings.
|
||||
- Confirmed semantic endpoints are rendered only in the ephemeral backend-owned harness config path.
|
||||
- Confirmed type wiring is explicit from `AppConfig` → `ThtRunner` → runtime renderer.
|
||||
|
||||
## Concerns
|
||||
|
||||
- Host validation currently permits both `http` and `https` on the allowed internal hosts. That keeps the configuration flexible, but if the installation contract intended `http` only, that restriction is not enforced here.
|
||||
|
||||
## Fix round 1/5
|
||||
|
||||
Scope:
|
||||
|
||||
- moved schema-v3 internal embeddings under `resources.embeddings`
|
||||
- enforced `http`-only internal semantic URLs
|
||||
|
||||
RED evidence:
|
||||
|
||||
`cd backend && npx vitest run test/workspace-runtime-renderer.test.ts test/config.test.ts`
|
||||
|
||||
Observed failures on `bc8afe0`:
|
||||
|
||||
- `workspace-runtime-renderer.test.ts`
|
||||
- schema-v3 output omitted `resources.embeddings`
|
||||
- schema-v3 still exposed top-level `embeddings`
|
||||
- `config.test.ts`
|
||||
- `https://qdrant:6333` was accepted
|
||||
|
||||
GREEN evidence:
|
||||
|
||||
`cd backend && npx vitest run test/workspace-runtime-renderer.test.ts test/config.test.ts`
|
||||
|
||||
Result:
|
||||
|
||||
- 2 test files passed
|
||||
- 16 tests passed
|
||||
|
||||
Typecheck:
|
||||
|
||||
`cd backend && npx tsc --noEmit -p .`
|
||||
|
||||
Result:
|
||||
|
||||
- passed
|
||||
|
||||
Updated concerns:
|
||||
|
||||
- none for this round beyond future tightening if exact-port rejection is later requested explicitly.
|
||||
@@ -1,149 +0,0 @@
|
||||
# Task 4 Report — Narrow harness embedding configuration to internal Ollama
|
||||
|
||||
## Status
|
||||
|
||||
Implemented on 2026-08-08 in `/Users/mp/projects/ThothII/.worktrees/git-workspace-registry`.
|
||||
|
||||
## RED evidence
|
||||
|
||||
Command:
|
||||
|
||||
```bash
|
||||
cd harness
|
||||
./.venv/bin/pytest tests/test_internal_embeddings.py tests/test_config_resources.py -q
|
||||
```
|
||||
|
||||
Observed before implementation:
|
||||
|
||||
- exit code `1`
|
||||
- `10 failed, 10 passed`
|
||||
- failures proved the missing `OllamaInternalEmbeddings` client and missing internal-only config validation
|
||||
|
||||
Representative failures:
|
||||
|
||||
- `ImportError: cannot import name 'OllamaInternalEmbeddings'`
|
||||
- `AttributeError: 'EmbeddingsConfig' object has no attribute 'provider'`
|
||||
- config tests `DID NOT RAISE ConfigError` for external provider, API key, and non-private base URL
|
||||
|
||||
## GREEN evidence
|
||||
|
||||
Focused behavior suite:
|
||||
|
||||
```bash
|
||||
cd harness
|
||||
./.venv/bin/pytest tests/test_internal_embeddings.py tests/test_config_resources.py -q
|
||||
```
|
||||
|
||||
- exit code `0`
|
||||
- `20 passed`
|
||||
|
||||
Relevant harness verification:
|
||||
|
||||
```bash
|
||||
cd harness
|
||||
./.venv/bin/pytest tests/test_internal_embeddings.py tests/test_config_resources.py tests/test_ollama_ensure.py -q
|
||||
```
|
||||
|
||||
- exit code `0`
|
||||
- `36 passed, 2 warnings`
|
||||
|
||||
Changed-file lint:
|
||||
|
||||
```bash
|
||||
cd harness
|
||||
./.venv/bin/ruff check tht/config.py tht/config_compat.py tht/vectorstore/embeddings.py tht/cli/ollama_cmd.py tests/test_config_resources.py tests/test_internal_embeddings.py
|
||||
```
|
||||
|
||||
- exit code `0`
|
||||
- `All checks passed!`
|
||||
|
||||
Patch hygiene:
|
||||
|
||||
```bash
|
||||
git diff --check
|
||||
```
|
||||
|
||||
- exit code `0`
|
||||
|
||||
## What changed
|
||||
|
||||
- translated schema-v3 `resources.embeddings` into the harness-compatible embedding config view
|
||||
- validated the internal embedding contract only for that runtime-owned `resources.embeddings` path:
|
||||
- provider must be `ollama_internal`
|
||||
- model must be `qwen3-embedding:0.6b`
|
||||
- dimensions must be `1024`
|
||||
- base URL must be `http://embedding:11434` or loopback HTTP on port `11434`
|
||||
- extra fields like `api_key` are rejected
|
||||
- replaced the active embed client with `OllamaInternalEmbeddings`, using one bounded `/api/embed` request per batch
|
||||
- removed task/query prefix rewriting from the active embedding path
|
||||
- validated response count, vector dimension, and finite numeric values before returning embeddings
|
||||
- kept `tht ollama ensure --json` stdout pristine while warming through the internal client
|
||||
|
||||
## Self-review
|
||||
|
||||
- kept changes inside the brief-listed files
|
||||
- preserved DWH and session-persistence behavior
|
||||
- preserved the legacy `OllamaEmbeddings` import path as an alias to avoid unrelated call-site churn
|
||||
|
||||
## Concerns
|
||||
|
||||
- the focused harness verification still emits two pre-existing warnings:
|
||||
- `DeprecationWarning` from `testcontainers.postgres`
|
||||
- `FutureWarning` because `resources` currently flows through the legacy config translation path
|
||||
|
||||
## Fix round 1 — 2026-08-08
|
||||
|
||||
### Findings addressed
|
||||
|
||||
- HIGH: external top-level `embeddings` remained an operational fallback and could still load
|
||||
- MEDIUM: non-object embed JSON payloads escaped as raw `AttributeError`
|
||||
|
||||
### RED evidence
|
||||
|
||||
Command:
|
||||
|
||||
```bash
|
||||
cd harness
|
||||
./.venv/bin/pytest tests/test_internal_embeddings.py tests/test_config_resources.py tests/test_ollama_ensure.py -q
|
||||
```
|
||||
|
||||
Observed before the fix:
|
||||
|
||||
- exit code `1`
|
||||
- `2 failed, 36 passed, 2 warnings`
|
||||
|
||||
Representative failures:
|
||||
|
||||
- `AttributeError: 'list' object has no attribute 'get'` from `response.json()` returning a JSON array
|
||||
- `Failed: DID NOT RAISE ConfigError` for top-level external `embeddings.provider=openai_compatible`
|
||||
|
||||
### GREEN evidence
|
||||
|
||||
Command:
|
||||
|
||||
```bash
|
||||
cd harness
|
||||
./.venv/bin/pytest tests/test_internal_embeddings.py tests/test_config_resources.py tests/test_ollama_ensure.py -q
|
||||
```
|
||||
|
||||
Observed after the fix:
|
||||
|
||||
- exit code `0`
|
||||
- `38 passed, 2 warnings`
|
||||
|
||||
Touched-file lint:
|
||||
|
||||
```bash
|
||||
cd harness
|
||||
./.venv/bin/ruff check tht/config.py tht/vectorstore/embeddings.py tests/test_internal_embeddings.py tests/test_config_resources.py
|
||||
```
|
||||
|
||||
- exit code `0`
|
||||
- `All checks passed!`
|
||||
|
||||
### Minimal fix
|
||||
|
||||
- validated the final active `cfg.embeddings` contract after config loading, so legacy top-level
|
||||
embedding inputs now fail explicitly unless they exactly match the internal Ollama contract
|
||||
- converted non-mapping embed JSON payloads into controlled `EmbeddingsError` failures with
|
||||
sanitized diagnostics instead of raw attribute errors
|
||||
@@ -1,170 +0,0 @@
|
||||
# Task 5 Report — Implement the Qdrant VectorStore adapter
|
||||
|
||||
## Status
|
||||
|
||||
Implemented on 2026-08-08 in `/Users/mp/projects/ThothII/.worktrees/git-workspace-registry`.
|
||||
|
||||
## RED evidence
|
||||
|
||||
Command:
|
||||
|
||||
```bash
|
||||
cd harness
|
||||
./.venv/bin/pytest tests/test_qdrant_vector_store.py tests/test_vector_port_contract.py -q
|
||||
```
|
||||
|
||||
Observed before implementation:
|
||||
|
||||
- exit code `2`
|
||||
- collection failed during import because the adapter did not exist yet
|
||||
|
||||
Representative failures:
|
||||
|
||||
- `ModuleNotFoundError: No module named 'tht.adapters.vector.qdrant'`
|
||||
|
||||
## GREEN evidence
|
||||
|
||||
Focused behavior suite:
|
||||
|
||||
```bash
|
||||
cd harness
|
||||
./.venv/bin/pytest tests/test_qdrant_vector_store.py tests/test_vector_port_contract.py -q
|
||||
```
|
||||
|
||||
- exit code `0`
|
||||
- `31 passed, 1 warning`
|
||||
|
||||
Touched-file lint:
|
||||
|
||||
```bash
|
||||
cd harness
|
||||
./.venv/bin/ruff check tht/adapters/vector/qdrant.py tht/adapters/vector/__init__.py \
|
||||
tht/ports/vector.py tht/vectorstore/records.py tht/vectorstore/store.py \
|
||||
tests/test_qdrant_vector_store.py tests/test_vector_port_contract.py
|
||||
```
|
||||
|
||||
- exit code `0`
|
||||
- `All checks passed!`
|
||||
|
||||
Patch hygiene:
|
||||
|
||||
```bash
|
||||
git diff --check
|
||||
```
|
||||
|
||||
- exit code `0`
|
||||
|
||||
## What changed
|
||||
|
||||
- added `QdrantVectorStore` with direct `requests`-based REST calls for:
|
||||
- `GET /collections/{collection}`
|
||||
- `PUT /collections/{collection}`
|
||||
- `PUT /collections/{collection}/index`
|
||||
- `PUT /collections/{collection}/points?wait=true`
|
||||
- `POST /collections/{collection}/points/query`
|
||||
- `POST /collections/{collection}/points/scroll`
|
||||
- `POST /collections/{collection}/points/delete?wait=true`
|
||||
- implemented idempotent collection provisioning for `1024` dimensions and `Cosine` distance
|
||||
- created deterministic UUIDv5 point IDs from workspace, semantic kind, and canonical record key
|
||||
- preserved canonical record identity and only upserted/deleted points matching the exact workspace
|
||||
and generation filters
|
||||
- added Qdrant payload helpers so stored payloads carry:
|
||||
- `workspace_id`
|
||||
- grouped semantic `kind` (`schema`, `evidence`, `memory`)
|
||||
- original `record_kind`
|
||||
- canonical `record_key`
|
||||
- `content_hash`
|
||||
- existing Thoth metadata fields
|
||||
- mapped Qdrant payloads back into existing `VectorHit` objects without losing the original
|
||||
Thoth kind
|
||||
- exported the new adapter from the public vector adapter package and added focused contract tests
|
||||
- sanitized timeout and malformed-response failures so CLI-facing callers do not leak raw endpoint
|
||||
details
|
||||
|
||||
## Self-review
|
||||
|
||||
- confirmed collection mismatch fails without any delete/recreate path
|
||||
- confirmed every query/scroll/delete operation includes a workspace filter
|
||||
- confirmed the adapter never deletes or rewrites unrelated Qdrant points
|
||||
- added keyword payload indexes for all filter-critical fields used here, including `document_id`
|
||||
for exact Evidence filtering
|
||||
|
||||
## Concerns
|
||||
|
||||
- the requested `adversarial-review` skill could not run its full external reviewer flow in this
|
||||
environment because the skill’s referenced `brain/` files are missing at
|
||||
`/Users/mp/.agents/skills/adversarial-review`; I performed a manual adversarial self-review
|
||||
instead
|
||||
- the focused suite still emits one pre-existing warning from `testcontainers.postgres`
|
||||
|
||||
## Fix round 1 — 2026-08-08
|
||||
|
||||
### Findings addressed
|
||||
|
||||
- IMPORTANT: metadata collisions could override canonical Qdrant payload identity fields and break
|
||||
workspace isolation
|
||||
- IMPORTANT: scroll-based operations only read the first page and did not follow
|
||||
`next_page_offset`, making `existing_hashes`, `list_evidence_generations`, and delete counts
|
||||
inexact beyond one page
|
||||
|
||||
### RED evidence
|
||||
|
||||
Command:
|
||||
|
||||
```bash
|
||||
cd harness
|
||||
./.venv/bin/pytest tests/test_qdrant_vector_store.py tests/test_vector_port_contract.py -q
|
||||
```
|
||||
|
||||
Observed before the fix:
|
||||
|
||||
- exit code `1`
|
||||
- `2 failed, 31 passed, 1 warning`
|
||||
|
||||
Representative failures:
|
||||
|
||||
- `assert payload["workspace_id"] == "demo"` failed because colliding `record.metadata`
|
||||
overwrote canonical payload fields
|
||||
- paginated scroll test missed later pages, so `existing_hashes` and generation cleanup counts
|
||||
were incomplete
|
||||
|
||||
### GREEN evidence
|
||||
|
||||
Command:
|
||||
|
||||
```bash
|
||||
cd harness
|
||||
./.venv/bin/pytest tests/test_qdrant_vector_store.py tests/test_vector_port_contract.py -q
|
||||
```
|
||||
|
||||
Observed after the fix:
|
||||
|
||||
- exit code `0`
|
||||
- `33 passed, 1 warning`
|
||||
|
||||
Touched-file lint:
|
||||
|
||||
```bash
|
||||
cd harness
|
||||
./.venv/bin/ruff check tht/adapters/vector/qdrant.py tht/vectorstore/records.py \
|
||||
tests/test_qdrant_vector_store.py tests/test_vector_port_contract.py
|
||||
```
|
||||
|
||||
- exit code `0`
|
||||
- `All checks passed!`
|
||||
|
||||
Patch hygiene:
|
||||
|
||||
```bash
|
||||
git diff --check
|
||||
```
|
||||
|
||||
- exit code `0`
|
||||
|
||||
### Minimal fix
|
||||
|
||||
- made `qdrant_payload` apply canonical fields after `record.metadata` so workspace ID, semantic
|
||||
kind, original record kind, canonical record key, and content hash cannot be overridden by
|
||||
metadata collisions
|
||||
- paginated `_scroll` until `next_page_offset` is absent, sent the returned `offset` back on the
|
||||
next request, and reject repeated offsets as malformed to avoid infinite loops
|
||||
@@ -1,144 +0,0 @@
|
||||
# Task 6 Report
|
||||
|
||||
Date: 2026-08-08
|
||||
|
||||
Status: implemented and verified
|
||||
|
||||
Summary:
|
||||
|
||||
- Added schema-v3 Qdrant runtime support to the harness config/resource layer and vector factory.
|
||||
- Made Qdrant payloads carry `workspace_id` and `workspace_revision` on every point.
|
||||
- Routed schema and memory bulk indexing through the transport-neutral vector port with canonical hash-based dedup.
|
||||
- Kept Evidence canonical on filesystem and Memory canonical in JSONL; Qdrant remains derived/rebuildable.
|
||||
- Added focused tests for semantic-kind isolation, shared identity fields, search-pack kind boundaries, and the schema-v3 factory/config path.
|
||||
|
||||
Files changed:
|
||||
|
||||
- `harness/tht/config.py`
|
||||
- `harness/tht/config_compat.py`
|
||||
- `harness/tht/adapters/factory.py`
|
||||
- `harness/tht/adapters/vector/qdrant.py`
|
||||
- `harness/tht/vectorstore/records.py`
|
||||
- `harness/tht/cli/vector_cmd.py`
|
||||
- `harness/tht/cli/memory_cmd.py`
|
||||
- `harness/tests/test_semantic_kind_isolation.py`
|
||||
- `harness/tests/test_memory_save_one.py`
|
||||
- `harness/tests/test_search_pack.py`
|
||||
- `harness/tests/test_qdrant_vector_store.py`
|
||||
- `harness/tests/test_adapter_factory.py`
|
||||
- `harness/tests/test_config_resources.py`
|
||||
|
||||
Verification:
|
||||
|
||||
- Focused RED/GREEN task suite:
|
||||
- `cd harness && .venv/bin/pytest tests/test_semantic_kind_isolation.py tests/test_memory_save_one.py tests/test_search_pack.py -q`
|
||||
- Relevant harness suite:
|
||||
- `cd harness && .venv/bin/pytest tests/test_semantic_kind_isolation.py tests/test_memory_save_one.py tests/test_search_pack.py tests/test_qdrant_vector_store.py tests/test_adapter_factory.py tests/test_config_resources.py tests/test_vector_port_contract.py tests/test_corpus_pipeline.py -q`
|
||||
- Result: `131 passed`
|
||||
- Changed-file Ruff:
|
||||
- `cd harness && .venv/bin/ruff check tht/vectorstore/records.py tht/adapters/vector/qdrant.py tht/config_compat.py tht/config.py tht/adapters/factory.py tht/cli/vector_cmd.py tht/cli/memory_cmd.py tests/test_memory_save_one.py tests/test_search_pack.py tests/test_semantic_kind_isolation.py tests/test_qdrant_vector_store.py tests/test_adapter_factory.py tests/test_config_resources.py`
|
||||
- Result: clean
|
||||
|
||||
Concerns / follow-up:
|
||||
|
||||
- `memory clear` still retains its older direct-vector assumptions and was not expanded in this task because the brief focused on canonical builders and schema/evidence/memory routing through the active Qdrant path.
|
||||
- The relevant suite still emits pre-existing warnings (legacy config deprecation in older fixtures, plus existing Pydantic serializer warnings in corpus tests), but they are not introduced by this task.
|
||||
|
||||
## Fix round 1 (2026-08-08)
|
||||
|
||||
Scope:
|
||||
|
||||
- Fixed qdrant-only schema-v3 command gating for `vector index-schema`, `memory promote`, and `memory index`.
|
||||
- Replaced `memory clear`'s direct-pgvector-only path with vector-port deletion by kind.
|
||||
- Added focused qdrant-only CLI regression tests and refreshed older CLI fixtures to the enforced internal embedding contract.
|
||||
|
||||
RED evidence:
|
||||
|
||||
- `cd harness && .venv/bin/pytest tests/test_qdrant_cli_commands.py -q`
|
||||
- Initial result against commit `5e39cfa`: `4 failed`
|
||||
- Failure signatures:
|
||||
- `ERRORE: sezioni mancanti nel workspace yaml: vector_db o vector_write_rest.`
|
||||
- `ERRORE: sezioni mancanti nel workspace yaml: vector_db.`
|
||||
|
||||
GREEN evidence:
|
||||
|
||||
- Focused fix suite:
|
||||
- `cd harness && .venv/bin/pytest tests/test_qdrant_cli_commands.py tests/test_qdrant_vector_store.py tests/test_adapter_factory.py tests/test_config_resources.py tests/test_memory_save_one.py tests/test_search_pack.py -q`
|
||||
- Result: `51 passed`
|
||||
- Relevant broader vector/memory/schema/search suite:
|
||||
- `cd harness && .venv/bin/pytest tests/test_qdrant_cli_commands.py tests/test_qdrant_vector_store.py tests/test_adapter_factory.py tests/test_config_resources.py tests/test_memory_save_one.py tests/test_search_pack.py tests/test_vector_port_contract.py tests/test_adapter_command_regressions.py tests/test_solved_search_cli.py tests/test_schema_introspect_guard.py tests/test_semantic_kind_isolation.py tests/test_corpus_pipeline.py -q`
|
||||
- Result: `154 passed`
|
||||
- Ruff on the fix surface:
|
||||
- `cd harness && .venv/bin/ruff check tht/ports/vector.py tht/adapters/vector/qdrant.py tht/adapters/vector/pgvector.py tht/adapters/vector/thoth_http.py tht/vectorstore/rest_client.py tht/cli/vector_cmd.py tht/cli/memory_cmd.py tests/test_qdrant_cli_commands.py tests/test_solved_search_cli.py`
|
||||
- Result: clean
|
||||
|
||||
Notes:
|
||||
|
||||
- `memory clear` now deletes derived `kind=memory` points through the configured writable vector store, while leaving the JSONL registry as the source of truth until the registry file is removed by the command.
|
||||
- The broader suite still carries the same pre-existing warnings noted above; this fix round did not add new warnings or failures.
|
||||
|
||||
## Fix round 2 (2026-08-08)
|
||||
|
||||
Scope:
|
||||
|
||||
- Removed the accidental HTTP writer `delete_kinds` capability expansion from `ThothHttpVectorStore` and `VectorRestClient`.
|
||||
- Reworked `memory clear` so schema-v3 Qdrant uses scoped `kind=memory` deletion, while legacy transports keep the pre-task direct-sync path instead of advertising a nonexistent RPC.
|
||||
- Tightened the qdrant-only memory-clear regression to assert the exact `("memory", ["memory"])` delete scope.
|
||||
|
||||
RED evidence:
|
||||
|
||||
- Re-review found a transport contract mismatch in fix round 1:
|
||||
- `ThothHttpVectorStore` exposed `delete_kinds(...)`
|
||||
- `VectorRestClient` exposed `delete_kinds(...)`
|
||||
- but the legacy HTTP writer migration only allowlists `delete_vector_generation`, not `delete_vector_kinds`
|
||||
- The new regressions added in this round capture that mismatch and the missing qdrant delete-scope assertion:
|
||||
- `tests/test_vector_port_contract.py::test_http_store_supports_writer_without_reader`
|
||||
- `tests/l0/test_vector_adapter_parity.py::test_http_rest_client_does_not_advertise_nonexistent_delete_kinds_rpc`
|
||||
- `tests/test_qdrant_cli_commands.py::test_memory_clear_accepts_qdrant_only_runtime_config`
|
||||
|
||||
GREEN evidence:
|
||||
|
||||
- Focused regression suite:
|
||||
- `cd harness && .venv/bin/pytest tests/test_qdrant_cli_commands.py tests/test_vector_port_contract.py tests/l0/test_vector_adapter_parity.py tests/test_adapter_command_regressions.py -q`
|
||||
- Result: `53 passed`
|
||||
- Broader relevant vector/memory/search suite:
|
||||
- `cd harness && .venv/bin/pytest tests/test_qdrant_cli_commands.py tests/test_adapter_command_regressions.py tests/test_vector_port_contract.py tests/l0/test_vector_adapter_parity.py tests/test_solved_search_cli.py tests/test_qdrant_vector_store.py tests/test_search_similar_kinds.py tests/test_corpus_pipeline.py -q`
|
||||
- Result: `135 passed`
|
||||
- Ruff on the changed fix surface:
|
||||
- `cd harness && .venv/bin/ruff check tht/cli/memory_cmd.py tht/ports/vector.py tht/adapters/vector/thoth_http.py tht/vectorstore/rest_client.py tests/test_qdrant_cli_commands.py tests/test_vector_port_contract.py tests/l0/test_vector_adapter_parity.py`
|
||||
- Result: clean
|
||||
|
||||
Notes:
|
||||
|
||||
- Legacy HTTP/vector-rest deployments do not gain a new destructive RPC surface from this fix; they keep their previous behavior and continue to fail closed for unsupported cleanup.
|
||||
- The broader suite still emits the same pre-existing deprecation and serializer warnings already noted above; this round did not introduce new warnings.
|
||||
|
||||
## Fix round 3 (2026-08-08)
|
||||
|
||||
Scope:
|
||||
|
||||
- Added an adapter-level Qdrant regression for mixed semantic kinds within one workspace plus a second workspace memory point.
|
||||
- Verified that `delete_kinds("memory", ["memory"])` emits the real adapter filter with both `workspace_id=demo` and `record_kind=memory`.
|
||||
- Verified that non-memory semantic kinds in the same workspace and memory from another workspace survive the delete.
|
||||
|
||||
RED evidence:
|
||||
|
||||
- Re-review identified a test gap rather than a confirmed runtime bug:
|
||||
- existing coverage asserted only the CLI mock call shape for qdrant memory clear
|
||||
- there was no adapter-level regression proving the real Qdrant delete filter and resulting fake-Qdrant state across mixed semantic kinds/workspaces
|
||||
- Added regression:
|
||||
- `tests/test_qdrant_vector_store.py::test_delete_kinds_is_workspace_scoped_and_preserves_other_semantic_kinds`
|
||||
|
||||
GREEN evidence:
|
||||
|
||||
- Requested focused suite:
|
||||
- `cd harness && .venv/bin/pytest tests/test_qdrant_vector_store.py tests/test_qdrant_cli_commands.py tests/test_semantic_kind_isolation.py -q`
|
||||
- Result: `18 passed`
|
||||
- Ruff on changed files:
|
||||
- `cd harness && .venv/bin/ruff check tests/test_qdrant_vector_store.py`
|
||||
- Result: clean
|
||||
|
||||
Notes:
|
||||
|
||||
- This round required no production change; the new adapter regression passed against the existing Qdrant implementation.
|
||||
- The focused suite still emits the same pre-existing `testcontainers.postgres` deprecation warning from `tests/conftest.py`; no new warnings were introduced.
|
||||
@@ -1,98 +0,0 @@
|
||||
# Task 7 report — mandatory Qdrant and Ollama Compose services
|
||||
|
||||
Date: 2026-08-08
|
||||
|
||||
Status: completed
|
||||
|
||||
Summary:
|
||||
|
||||
- Added mandatory private `qdrant`, `embedding`, and `embedding-model-init` services to the base Compose stack.
|
||||
- Pinned Qdrant `v1.18.2` and Ollama `0.32.0` by immutable multi-arch digest.
|
||||
- Persisted Qdrant storage in `qdrant-data` and Ollama model cache in `embedding-models`.
|
||||
- Wired `core` to fixed internal semantic endpoints:
|
||||
- `THT_INTERNAL_QDRANT_URL=http://qdrant:6333`
|
||||
- `THT_INTERNAL_EMBEDDING_URL=http://embedding:11434`
|
||||
- `THT_INTERNAL_EMBEDDING_MODEL=qwen3-embedding:0.6b`
|
||||
- `THT_INTERNAL_EMBEDDING_DIMENSIONS=1024`
|
||||
- Removed external vector / embedding endpoint requirements from the local and server env examples.
|
||||
- Added an idempotent Ollama model bootstrap script that:
|
||||
- waits up to a bounded deadline for `/api/tags`
|
||||
- skips `ollama pull` when the model is already cached
|
||||
- pulls `qwen3-embedding:0.6b` only when needed
|
||||
- verifies the model appears in `/api/tags` after pull
|
||||
- Added optional GPU override file `deploy/compose.embedding-gpu.yaml`; base Compose remains CPU-only.
|
||||
- Updated `scripts/run-stack.sh` so the GPU override is included only when `THOTH_ENABLE_EMBEDDING_GPU=1`.
|
||||
|
||||
Verification:
|
||||
|
||||
- RED confirmed before implementation:
|
||||
- `./scripts/test-default-compose.sh` failed on missing required services.
|
||||
- `./scripts/test-unified-compose.sh` failed on missing required services.
|
||||
- `./scripts/test-internal-semantic-compose.sh` failed because the GPU override file did not exist.
|
||||
- GREEN after implementation:
|
||||
- `./scripts/test-default-compose.sh`
|
||||
- `./scripts/test-unified-compose.sh`
|
||||
- `./scripts/test-internal-semantic-compose.sh`
|
||||
- `git diff --check`
|
||||
- Additional shell verification:
|
||||
- `scripts/run-stack.sh --wait` includes only base + local Compose files by default.
|
||||
- `THOTH_ENABLE_EMBEDDING_GPU=1 scripts/run-stack.sh --wait` adds `deploy/compose.embedding-gpu.yaml`.
|
||||
|
||||
Resolved image digests:
|
||||
|
||||
- `qdrant/qdrant:v1.18.2@sha256:75eab8c4ba42096724fdcfde8b4de0b5713d529dde32f285a1f86fdcb2c9e50c`
|
||||
- `ollama/ollama:0.32.0@sha256:57f573b47f1f71ebb445789f279fe3e596a8beab182f7cf486db9205bad87c5a`
|
||||
|
||||
Self-review:
|
||||
|
||||
- The first bootstrap-script draft depended on tools not guaranteed inside the Ollama image. This was corrected after image inspection; the final script uses only confirmed image tools (`bash`, `ollama`, `grep`) plus raw HTTP over `/dev/tcp`.
|
||||
- The server overlay intentionally replaces most named core volumes with bind mounts, so the unified contract was tightened to require named semantic-cache volumes there while preserving the local/base named-volume checks.
|
||||
|
||||
Concerns:
|
||||
|
||||
- The model bootstrap waits for Ollama readiness and verifies cache state, but the first real cold-start will still take time to download `qwen3-embedding:0.6b`.
|
||||
- The GPU override requests generic Docker GPU capability only; actual GPU availability remains host/runtime dependent and intentionally stays opt-in.
|
||||
|
||||
## Fix round 1 / 5 — 2026-08-08
|
||||
|
||||
Rulings applied:
|
||||
|
||||
- Kept the Task 1 boundary intact: schema-v3 remains the only operational workspace descriptor shape.
|
||||
- Did not restore any external semantic fallback for schema-v2 live sessions.
|
||||
- Treated `PROJECT_STATE.md` as stale documentation for this point, not runtime truth.
|
||||
|
||||
Focused schema-v2 evidence:
|
||||
|
||||
- Re-ran the existing targeted registry test:
|
||||
- `cd backend && npx vitest run test/workspace-registry.test.ts -t "lists a schema v2 descriptor as migration_required and refuses to acquire it"`
|
||||
- Result: pass.
|
||||
- Evidence from that test:
|
||||
- schema-v2 descriptors list as `migration_required`
|
||||
- `acquireSessionRevision("psd-clinical")` rejects with `code: "workspace_invalid"`
|
||||
- Conclusion: schema-v2 acquisition remains blocked; no external semantic fallback was reintroduced.
|
||||
|
||||
Contract consistency fixes:
|
||||
|
||||
- Updated `harness/tests/test_local_compose_contract.py` to assert the mandatory internal semantic stack, fixed internal core semantic env, private-service topology, persistent volumes, and Ollama health/dependency contract.
|
||||
- Updated shell Compose contracts to require:
|
||||
- Ollama healthcheck on `embedding`
|
||||
- `embedding-model-init` dependency on `embedding: service_healthy`
|
||||
- Updated `scripts/unified-deployment-smoke.sh` rendered-contract helper to expect the mandatory internal semantic topology and internal semantic env names, and to reject retired external semantic bindings.
|
||||
- Updated `scripts/test-task13-runtime-fixtures.sh` to exercise `task13_assert_rendered_contract` for both local and server fixture renders.
|
||||
|
||||
Fix round 1 verification:
|
||||
|
||||
- RED before implementation:
|
||||
- `cd harness && .venv/bin/pytest tests/test_local_compose_contract.py -q` failed because `embedding` had no healthcheck.
|
||||
- `./scripts/test-default-compose.sh` failed because `embedding` had no healthcheck.
|
||||
- `./scripts/test-unified-compose.sh` failed because `embedding` had no healthcheck.
|
||||
- `./scripts/test-task13-runtime-fixtures.sh local` failed because `unified-deployment-smoke.sh` still expected `core,frontend`.
|
||||
- GREEN after implementation:
|
||||
- `./scripts/test-default-compose.sh`
|
||||
- `./scripts/test-unified-compose.sh`
|
||||
- `./scripts/test-internal-semantic-compose.sh`
|
||||
- `cd harness && .venv/bin/pytest tests/test_local_compose_contract.py -q`
|
||||
- `./scripts/test-task13-runtime-fixtures.sh local`
|
||||
- `./scripts/test-task13-runtime-fixtures.sh server`
|
||||
- `cd backend && npx vitest run test/workspace-registry.test.ts -t "lists a schema v2 descriptor as migration_required and refuses to acquire it"`
|
||||
- `docker compose --env-file deploy/env/local.env.example -f compose.yaml -f deploy/compose.local.yaml config --format json`
|
||||
@@ -1,65 +0,0 @@
|
||||
Status: completed on August 8, 2026.
|
||||
|
||||
Summary:
|
||||
- Updated the frontend workspace contract from schema v2 editing to schema v3 publishing.
|
||||
- Kept only `semantic_index.vector_store.collection` editable; rendered qdrant / internal Ollama semantic values as fixed read-only architecture values.
|
||||
- Removed external vector transport / endpoint / credential / embedding diagnostics branches from frontend draft sanitization, conflict parsing, and editor UI.
|
||||
- Added a migration-required banner in workspace management and blocked `migration_required` workspaces from new-session selection.
|
||||
- Aligned the example workspace YAML comments with the fixed internal qdrant/Ollama architecture.
|
||||
|
||||
Files changed:
|
||||
- `frontend/src/api/workspaces.ts`
|
||||
- `frontend/src/api/workspaces.test.ts`
|
||||
- `frontend/src/workspaces/drafts.ts`
|
||||
- `frontend/src/workspaces/drafts.test.ts`
|
||||
- `frontend/src/shell/WorkspaceEditor.tsx`
|
||||
- `frontend/src/shell/WorkspaceEditor.test.tsx`
|
||||
- `frontend/src/shell/WorkspaceManager.tsx`
|
||||
- `frontend/src/shell/WorkspaceManager.test.tsx`
|
||||
- `frontend/src/shell/WorkspacePublishDialog.test.tsx`
|
||||
- `frontend/src/api/sessions.ts`
|
||||
- `frontend/src/shell/SteerInput.tsx`
|
||||
- `frontend/src/shell/SteerInput.test.tsx`
|
||||
- `deploy/workspaces/example.yaml`
|
||||
- `deploy/workspaces/psd.yaml.example`
|
||||
|
||||
Verification:
|
||||
- `cd frontend && npx vitest run src/shell/SteerInput.test.tsx src/shell/WorkspaceEditor.test.tsx src/shell/WorkspaceManager.test.tsx src/shell/WorkspacePublishDialog.test.tsx src/workspaces/drafts.test.ts src/api/workspaces.test.ts`
|
||||
- Result: 6 files passed, 59 tests passed.
|
||||
- `cd frontend && npx tsc -b`
|
||||
- Result: passed.
|
||||
- `git diff --check`
|
||||
- Result: passed.
|
||||
|
||||
Self-review:
|
||||
- The frontend now publishes the exact schema v3 semantic shape and no longer persists legacy semantic transport/credential branches.
|
||||
- Migration-required workspaces are visible in management with an explicit banner and are excluded from the composer workspace selector.
|
||||
- One dependent test file outside the original brief list (`WorkspacePublishDialog.test.tsx`) and the composer/session-selection path (`api/sessions.ts`, `SteerInput.tsx`, related test) were updated because they were directly coupled to the old v2 semantic/edit-selection behavior.
|
||||
|
||||
Concerns:
|
||||
- The composer still retains backward-compatible behavior for summaries that omit `revision` entirely; only explicit `revision.state === "migration_required"` is blocked. That matches the current mixed-test environment, but once summary responses are guaranteed to include `revision`, that fallback may be removable.
|
||||
|
||||
Fix round 1/5 — August 8, 2026
|
||||
|
||||
Summary:
|
||||
- Made missing or invalid workspace summaries fail safe in frontend session creation and composer selection instead of falling open as legacy.
|
||||
- Added an actionable unavailable message in workspace management for incomplete summaries with no canonical revision.
|
||||
- Replaced the old runtime-oriented example descriptor files with exact backend WorkspaceV3 descriptor YAML.
|
||||
|
||||
Additional files changed:
|
||||
- `frontend/src/api/sessions.test.ts`
|
||||
- `backend/test/workspaces-schema.test.ts`
|
||||
|
||||
Fix-round verification:
|
||||
- `cd frontend && npx vitest run src/api/sessions.test.ts src/shell/SteerInput.test.tsx src/shell/WorkspaceManager.test.tsx src/shell/WorkspaceEditor.test.tsx src/shell/WorkspacePublishDialog.test.tsx src/workspaces/drafts.test.ts src/api/workspaces.test.ts`
|
||||
- Result: 7 files passed, 73 tests passed.
|
||||
- `cd frontend && npx tsc -b`
|
||||
- Result: passed.
|
||||
- `cd backend && npx vitest run test/workspaces-schema.test.ts`
|
||||
- Result: 1 file passed, 17 tests passed.
|
||||
- `git diff --check`
|
||||
- Result: passed.
|
||||
|
||||
Notes:
|
||||
- Missing `revision` in a workspace summary now fails with the same session/composer safety posture as `migration_required`, using the existing safe workspace-policy error for session creation and an explicit unavailable message in workspace management.
|
||||
- The committed example files now validate as actual schema-v3 descriptors instead of deployment/runtime templates with forbidden semantic endpoint fields.
|
||||
@@ -1,137 +0,0 @@
|
||||
# Adapter Foundations final-review fix report
|
||||
|
||||
Date: 2026-07-11
|
||||
Branch: `codex/portable-deployment`
|
||||
Worktree: `/Users/mp/projects/ThothII/.worktrees/portable-deployment`
|
||||
Binding findings: `.superpowers/sdd/adapter-final-review-findings.md`
|
||||
|
||||
## Outcome
|
||||
|
||||
All seven final-review findings are addressed as one coherent adapter-foundations change:
|
||||
|
||||
1. HTTP vector reader and writer clients are independently optional. Capabilities reflect the
|
||||
configured side; writer-only new and legacy configurations build successfully for targeted
|
||||
writes; search without a reader raises public `VectorReadUnavailable`.
|
||||
2. `VectorHealth` now reports read/write configured and reachable state independently, preserves
|
||||
side-specific errors, and reports expected/observed embedding dimensions plus compatibility.
|
||||
HTTP diagnostics cover read-only, write-only, both-up, and writer-down cases. Direct health
|
||||
exposes its configured expected dimension without adding schema or migration work.
|
||||
3. `ThothRestDwhAdapter` accepts `DatabaseIdentityConfig`, matching its resource contract.
|
||||
4. Both vector adapters reject bools, floats, zero, and negative search limits using one exact
|
||||
positive-integer guard.
|
||||
5. Port tests explicitly cover public exports and frozen capability records.
|
||||
6. A real `tht` subprocess test proves one legacy deprecation warning per config load on stderr
|
||||
while JSON stdout remains parseable and uncontaminated.
|
||||
7. The adapter plan and SDD progress explicitly constrain `build_vector_loader` to transitional
|
||||
bulk sync and schedule its removal/migration in the local pgvector plan. Targeted memory and
|
||||
solved-question writes remain on `build_vector_store(..., require_write=True)`.
|
||||
|
||||
No pgvector schema or migration changes were made.
|
||||
|
||||
## Files changed
|
||||
|
||||
- `harness/tht/ports/vector.py`
|
||||
- `harness/tht/ports/__init__.py`
|
||||
- `harness/tht/adapters/vector/thoth_http.py`
|
||||
- `harness/tht/adapters/vector/legacy_direct.py`
|
||||
- `harness/tht/adapters/factory.py`
|
||||
- `harness/tht/adapters/dwh/thoth_rest.py`
|
||||
- `harness/tests/test_vector_port_contract.py`
|
||||
- `harness/tests/test_adapter_factory.py`
|
||||
- `harness/tests/test_config_resources.py`
|
||||
- `harness/tests/test_config_legacy_compat.py`
|
||||
- `harness/tests/test_adapter_command_regressions.py`
|
||||
- `harness/tests/test_dwh_port_contract.py`
|
||||
- `docs/superpowers/plans/2026-07-11-adapter-foundations.md`
|
||||
- `.superpowers/sdd/progress.md`
|
||||
- `.superpowers/sdd/adapter-final-fix-report.md`
|
||||
|
||||
## TDD and verification evidence
|
||||
|
||||
RED:
|
||||
|
||||
```text
|
||||
cd harness && .venv/bin/pytest tests/test_vector_port_contract.py \
|
||||
tests/test_adapter_factory.py tests/test_config_resources.py \
|
||||
tests/test_config_legacy_compat.py -q
|
||||
```
|
||||
|
||||
Result: collection failed as expected because `VectorReadUnavailable` did not exist. After the
|
||||
initial implementation, the same command exposed two expected contract/test-harness corrections:
|
||||
dimension mismatch makes aggregate health unhealthy, and the installed CLI entry point is `tht`
|
||||
rather than `python -m tht.cli`.
|
||||
|
||||
GREEN, covering adapter/config/command regressions:
|
||||
|
||||
```text
|
||||
cd harness && .venv/bin/pytest tests/test_vector_port_contract.py \
|
||||
tests/test_adapter_factory.py tests/test_config_resources.py \
|
||||
tests/test_config_legacy_compat.py tests/test_adapter_command_regressions.py \
|
||||
tests/test_dwh_port_contract.py tests/test_memory_save_one.py \
|
||||
tests/test_solved_question.py tests/test_search_similar_kinds.py \
|
||||
tests/test_vector_dual_key.py -q
|
||||
```
|
||||
|
||||
Result: `66 passed in 0.45s`.
|
||||
|
||||
Docker availability:
|
||||
|
||||
```text
|
||||
docker info --format '{{.ServerVersion}}'
|
||||
```
|
||||
|
||||
Result: `29.4.1` (available; command required Docker socket access).
|
||||
|
||||
Full repository-default non-L2 harness suite, with Docker available for L0 tests:
|
||||
|
||||
```text
|
||||
cd harness && .venv/bin/pytest -q
|
||||
```
|
||||
|
||||
Result: `433 passed, 5 deselected, 17 warnings in 9.14s`. The five deselections are the configured
|
||||
L2/live-service tests. Warnings are existing legacy-workspace `FutureWarning` emissions.
|
||||
|
||||
Scoped lint and diff hygiene:
|
||||
|
||||
```text
|
||||
cd harness && .venv/bin/ruff check tht/ports tht/adapters \
|
||||
tests/test_vector_port_contract.py tests/test_adapter_factory.py \
|
||||
tests/test_config_resources.py tests/test_config_legacy_compat.py \
|
||||
tests/test_adapter_command_regressions.py tests/test_dwh_port_contract.py
|
||||
git diff --check
|
||||
```
|
||||
|
||||
Result: `All checks passed!`; `git diff --check` produced no output.
|
||||
|
||||
## Commit
|
||||
|
||||
Commit subject: `fix(adapter): close final foundation review`
|
||||
|
||||
The report is part of that same final commit. A Git object cannot contain its own SHA without
|
||||
changing that SHA; the exact resulting commit ID is therefore recorded in the task handoff from
|
||||
`git rev-parse HEAD` after creation.
|
||||
|
||||
## Self-review
|
||||
|
||||
- Reader/writer separation is preserved: search dereferences only `_reader`; hashes/upsert only
|
||||
`_writer`; health probes each configured client independently and never substitutes one result
|
||||
for the other.
|
||||
- Writer failure contributes to aggregate `ok=False`, even when the reader succeeds.
|
||||
- Dimension compatibility is derived only from configured embedding dimension and existing
|
||||
`list_tables` metadata. Missing metadata remains `None`, not a guessed success/failure.
|
||||
- The shared limit guard uses `type(limit) is int`, intentionally rejecting Python booleans and
|
||||
numeric coercions before either adapter reaches its transport.
|
||||
- Existing JSON/CLI behavior is preserved; the subprocess regression parses stdout as JSON and
|
||||
counts exactly one deprecation marker on stderr.
|
||||
- Scope remains adapter foundations. No vector DDL, schema initialization, or migration work was
|
||||
introduced.
|
||||
|
||||
## Concerns / follow-up
|
||||
|
||||
- Write reachability uses the existing `list_tables` diagnostic on the separately authenticated
|
||||
writer client. Deployments must allow that non-mutating diagnostic RPC to the writer credential;
|
||||
failures are intentionally visible rather than hidden by reader success.
|
||||
- Existing legacy-workspace tests emit 17 `FutureWarning`s in the full suite. This wave pins the
|
||||
required production stderr behavior but does not migrate unrelated test fixtures.
|
||||
- `build_vector_loader` remains transitional technical debt only for bulk sync, explicitly assigned
|
||||
to `2026-07-11-local-pgvector-profile.md`.
|
||||
@@ -1,206 +0,0 @@
|
||||
# Container Packaging Task 3 Report
|
||||
|
||||
## Status
|
||||
|
||||
Implemented the multi-stage core application image, non-root runtime, pinned Pi installation,
|
||||
container entrypoint, context exclusions, and an in-image health smoke test.
|
||||
|
||||
## TDD / Build Evidence
|
||||
|
||||
Initial RED:
|
||||
|
||||
```text
|
||||
docker build -f docker/core.Dockerfile -t thothii-core:test .
|
||||
ERROR: failed to build: resolve : lstat docker: no such file or directory
|
||||
```
|
||||
|
||||
The first sandboxed attempt could not access the Docker socket; the authorized rerun reached the
|
||||
builder and failed for the expected reason: the Dockerfile did not exist.
|
||||
|
||||
GREEN build:
|
||||
|
||||
```text
|
||||
sh -n docker/core-entrypoint.sh docker/smoke/core-smoke.sh
|
||||
docker build --progress=plain -f docker/core.Dockerfile -t thothii-core:test .
|
||||
```
|
||||
|
||||
Result: shell syntax exited 0; Docker build exited 0. A final rebuild after tightening
|
||||
`.dockerignore` also exited 0 and transferred only 17.60 kB of changed context (the initial clean
|
||||
build transferred 1.02 MB).
|
||||
|
||||
## Runtime and Entrypoints
|
||||
|
||||
- Runtime user is `10001:10001` (`thoth`), never root.
|
||||
- Runtime contains Node `v22.19.0` and Python `3.12.13`. Python 3.12 is intentional because the
|
||||
harness declares `requires-python = ">=3.12"` and also satisfies the deployment floor of 3.11+.
|
||||
- Pi is installed exactly as `@earendil-works/pi-coding-agent@0.80.3`; its build-time and runtime
|
||||
version probes both reported `0.80.3`.
|
||||
- `server` starts `/app/backend/dist/server.js`; `doctor` routes to `tht doctor`; `preprocess`
|
||||
routes to the future-facing `tht preprocess` command; explicit `tht ...` and arbitrary CLI
|
||||
arguments route to the installed `tht` binary.
|
||||
- The gate extension's `typebox` runtime dependency is installed from the harness lockfile.
|
||||
|
||||
## Smoke and Diagnostic Results
|
||||
|
||||
```text
|
||||
docker run --rm thothii-core:test doctor
|
||||
config: error - configuration is invalid or unreadable
|
||||
data_root: ok
|
||||
```
|
||||
|
||||
Result: expected exit 1 for absent mounted workspace configuration, with no traceback and no
|
||||
secret-bearing validation detail.
|
||||
|
||||
```text
|
||||
docker run --rm --entrypoint /app/docker/smoke/core-smoke.sh thothii-core:test
|
||||
backend listening on http://127.0.0.1:8787
|
||||
v22.19.0
|
||||
Python 3.12.13
|
||||
core smoke: ok
|
||||
```
|
||||
|
||||
Result: exit 0. The script asserted non-root execution, `tht --help`, `pi --version`, runtime
|
||||
version floors, and `GET /health` through curl. Fastify's returned display address was loopback;
|
||||
the inspected container environment is `HOST=0.0.0.0`, and the compiled server passes that value
|
||||
to `app.listen`.
|
||||
|
||||
```text
|
||||
docker run --rm thothii-core:test tht --version
|
||||
0.1.0
|
||||
```
|
||||
|
||||
Result: arbitrary `tht` entrypoint exited 0.
|
||||
|
||||
An explicit runtime assertion checked UID 10001, exact Node and Pi versions, Python 3.11+, and the
|
||||
absence of `/app/harness/.env` and `/app/harness/workspaces`; it exited 0.
|
||||
|
||||
## Image Size and Containment Inspection
|
||||
|
||||
```text
|
||||
docker image inspect thothii-core:test --format '{{.Size}} {{json .Config.User}} {{json .Config.Env}}'
|
||||
221419008 "10001:10001" [...runtime paths and version metadata only...]
|
||||
```
|
||||
|
||||
Image size: **221,419,008 bytes** (about 211.2 MiB).
|
||||
|
||||
`docker history --no-trunc thothii-core:test` was inspected. It contains only Dockerfile commands,
|
||||
the pinned public package name/version, base-image metadata, and non-sensitive runtime variables;
|
||||
no credentials or customer paths were found. An in-image filename scan found only
|
||||
`/app/harness/.pi/settings.json` among `.env`, key/certificate, and settings-name candidates; that
|
||||
tracked Pi file contains theme/startup preferences, not secrets. The build asserts `.env` and
|
||||
workspace directories are absent.
|
||||
|
||||
`.dockerignore` excludes VCS/agent state, all environment files except examples, package-manager
|
||||
credential files, SSH/private-key and certificate formats, local virtualenvs/node_modules/caches,
|
||||
backend runtime data, customer workspaces, sessions, artifacts, indexes, corpus, and deployment
|
||||
mount content.
|
||||
|
||||
## Self-review
|
||||
|
||||
- `git diff --check` is clean.
|
||||
- Entrypoint processes use `exec`, preserving container signal handling.
|
||||
- Backend production dependencies are pruned; TypeScript build tools remain in the build stage.
|
||||
- The writable `/data` root is owned by UID 10001; application payload remains root-owned and
|
||||
read-only to the runtime user.
|
||||
- CA certificates and curl are present for HTTPS integrations and health probing.
|
||||
- No existing source, customer workspace, secret, or unrelated progress-ledger change is included
|
||||
in the task commit.
|
||||
|
||||
## Concerns
|
||||
|
||||
- The `tht preprocess` command is deliberately a future-facing routing contract; its CLI group is
|
||||
scheduled in the Evidence/preprocessing plan and is not implemented in the current harness.
|
||||
- Python dependencies are range-resolved because the existing harness has no Python lockfile. The
|
||||
Pi package, Node runtime, and package-lock-backed Node dependency sets are pinned/reproducible.
|
||||
- The image was built and smoked on Docker Desktop arm64. The chosen official multi-arch base
|
||||
images and Pi package are architecture-neutral at the package level, but amd64 still needs a CI
|
||||
build/smoke before being advertised as verified.
|
||||
|
||||
## Reproducibility Review Fix
|
||||
|
||||
The original image pinned Pi's direct version in the Dockerfile but resolved its transitives at
|
||||
build time, and pip resolved all harness dependencies from ranges. Both paths now consume committed
|
||||
locks.
|
||||
|
||||
### Lock generation
|
||||
|
||||
Pi uses the minimal `docker/pi-runtime/package.json` and its committed npm v3 lock. It was generated
|
||||
with:
|
||||
|
||||
```text
|
||||
npm install --package-lock-only --ignore-scripts --no-audit --no-fund \
|
||||
--prefix docker/pi-runtime
|
||||
```
|
||||
|
||||
The package manifest specifies exact `@earendil-works/pi-coding-agent` version `0.80.3`; a lock
|
||||
inspection confirmed that same resolved package version. Docker installs it with:
|
||||
|
||||
```text
|
||||
npm ci --omit=dev --ignore-scripts --no-audit --no-fund
|
||||
```
|
||||
|
||||
The Python lock was generated directly from the harness production metadata plus one explicit,
|
||||
pinned PEP 517 build-backend input—not from a host `pip freeze`:
|
||||
|
||||
```text
|
||||
uv pip compile harness/pyproject.toml docker/python-runtime/build-requirements.in \
|
||||
--universal \
|
||||
--python-version 3.12 \
|
||||
--no-emit-package tht \
|
||||
--generate-hashes \
|
||||
--custom-compile-command \
|
||||
'uv pip compile harness/pyproject.toml docker/python-runtime/build-requirements.in --universal --python-version 3.12 --no-emit-package tht --generate-hashes --output-file docker/python-runtime/requirements.lock' \
|
||||
--output-file docker/python-runtime/requirements.lock
|
||||
```
|
||||
|
||||
`pytest`, `ruff`, and `testcontainers` are absent. All production direct and transitive packages
|
||||
are exact and hashed. `setuptools==80.9.0` is explicit so the local harness install can use
|
||||
`--no-build-isolation` without an unpinned build-time resolution. Refresh instructions are in
|
||||
`docker/LOCKS.md`.
|
||||
|
||||
### No-cache rebuild and verification
|
||||
|
||||
Final build command:
|
||||
|
||||
```text
|
||||
docker build --no-cache -f docker/core.Dockerfile -t thothii-core:test .
|
||||
```
|
||||
|
||||
Result: exit 0. The logs showed Pi `0.80.3`, Node `v22.19.0`, a hash-enforced Python dependency
|
||||
install, explicit `setuptools==80.9.0`, and a non-isolated local `tht` wheel build. No isolated
|
||||
build-dependency download occurred.
|
||||
|
||||
Fresh runtime checks:
|
||||
|
||||
```text
|
||||
docker run --rm --entrypoint /app/docker/smoke/core-smoke.sh thothii-core:test
|
||||
backend listening on http://127.0.0.1:8787
|
||||
v22.19.0
|
||||
Python 3.12.13
|
||||
core smoke: ok
|
||||
|
||||
docker run --rm thothii-core:test tht --version
|
||||
0.1.0
|
||||
|
||||
/opt/venv/bin/pip check
|
||||
No broken requirements found.
|
||||
```
|
||||
|
||||
An in-container package inspection reconfirmed Pi `0.80.3`. Non-root UID, runtime version floors,
|
||||
doctor's expected concise exit 1/no traceback, `/health`, and arbitrary `tht` routing all passed.
|
||||
|
||||
The full filename containment scan found no `.env`, PEM, private-key, P12, or PFX file in `/app`;
|
||||
`/app/harness/workspaces` remains absent. Image environment and `docker history --no-trunc` were
|
||||
re-inspected and contain only public package/build commands and non-sensitive runtime metadata.
|
||||
|
||||
Final locked image size:
|
||||
|
||||
```text
|
||||
220003986 10001:10001
|
||||
```
|
||||
|
||||
That is **220,003,986 bytes** (about 209.8 MiB), 1,415,022 bytes smaller than the original image.
|
||||
|
||||
Remaining concern: the universal lock is resolved for Python 3.12 and includes hashes/markers for
|
||||
all supported platforms, but only Linux arm64 has been built and smoked locally; amd64 remains a CI
|
||||
verification gate.
|
||||
@@ -1,82 +0,0 @@
|
||||
# Container Packaging Task 4 Report
|
||||
|
||||
## Status
|
||||
|
||||
Implemented and verified runtime-configured frontend packaging.
|
||||
|
||||
## Changes
|
||||
|
||||
- Added the browser runtime contract `window.__THOTHII_CONFIG__.backendBaseUrl`.
|
||||
- Loaded `/config.js` before the Vite module entrypoint.
|
||||
- Made runtime configuration take precedence while preserving `VITE_BACKEND_URL` and the
|
||||
existing `http://localhost:8787` client default for development and tests.
|
||||
- Added a multi-stage frontend image that builds with Node and serves static assets as
|
||||
unprivileged UID/GID `101:101` with nginx on port 8080.
|
||||
- Added startup-time `BACKEND_BASE_URL` substitution (default `/api`).
|
||||
- Added `/api/` reverse proxying to `core:8787`, SPA fallback, no-cache runtime config,
|
||||
and SSE-safe proxy settings (`proxy_buffering off`, `proxy_cache off`, one-hour read timeout).
|
||||
|
||||
## TDD evidence
|
||||
|
||||
- RED: `npx vitest run src/api/runtime-config.test.ts` failed because
|
||||
`./runtime-config` did not exist.
|
||||
- GREEN: targeted runtime config suite passed (3 tests after preserving the legacy client
|
||||
default).
|
||||
|
||||
## Verification
|
||||
|
||||
- `cd frontend && npx vitest run --reporter=dot && npx tsc -b && npm run build` — exit 0
|
||||
(40 test files, 185 tests; TypeScript and Vite production build passed).
|
||||
- `docker build -f docker/frontend.Dockerfile -t thothii-frontend:test .` — success.
|
||||
- Image metadata reports `USER 101:101`.
|
||||
- Two-container isolated-network smoke:
|
||||
- `/config.js` returned `window.__THOTHII_CONFIG__ = { backendBaseUrl: "/api" };`
|
||||
- `/api/health` proxied to the core image and returned `{"status":"ok"}`.
|
||||
- an unknown nested route returned the SPA `index.html`.
|
||||
- active nginx config contained `proxy_buffering off`, `proxy_cache off`, and
|
||||
`proxy_read_timeout 1h`.
|
||||
- `/config.js` returned `Cache-Control: no-store`.
|
||||
- `sh -n docker/frontend-entrypoint.sh` and `git diff --check` — exit 0.
|
||||
|
||||
## Secret-leakage inspection
|
||||
|
||||
- `.dockerignore` excludes `.env*` (except examples), credentials/key formats, dependency
|
||||
trees, build outputs, backend data, and deployment data.
|
||||
- The runtime web root contained no `.env*`, `.pem`, `.key`, `.p12`, or `.pfx` files.
|
||||
- Image history contained build/package instructions only; no secret build arguments or
|
||||
credential values were introduced by this task.
|
||||
|
||||
## Self-review / concerns
|
||||
|
||||
- nginx resolves the `core` hostname at startup, matching the planned Compose service name;
|
||||
standalone runs therefore need a reachable network alias named `core`.
|
||||
- Existing frontend test warnings (React refs/act, MSW unmatched incidental requests, Vite
|
||||
chunk-size warnings) remain; they did not fail the requested gates and are unrelated to
|
||||
this task.
|
||||
- `.superpowers/sdd/progress.md` was already modified by the orchestrator and was intentionally
|
||||
excluded from this task's commit.
|
||||
|
||||
## P1 review fixes
|
||||
|
||||
Follow-up commit work addressed both review findings:
|
||||
|
||||
- Runtime configuration is now produced with `jq -cn --arg`, so `BACKEND_BASE_URL` is encoded
|
||||
by a real JSON serializer rather than interpolated into JavaScript by `sed`.
|
||||
- The image includes `frontend-config-smoke`, which strips only the fixed assignment wrapper,
|
||||
parses the remaining JSON with `jq`, requires exactly the `backendBaseUrl` key, and compares
|
||||
the decoded value to the environment input.
|
||||
- The hostile smoke passed with quotes, backslashes, a literal newline, ampersand, pipe, and
|
||||
`"; globalThis.PWNED=true; //` in the value. A breakout would leave non-JSON trailing input
|
||||
and fail parsing.
|
||||
- Added `joinBackendPath`, shared by API fetch and EventSource creation. It removes duplicate
|
||||
boundary slashes for relative and absolute bases while keeping empty and `/` bases rooted.
|
||||
|
||||
Follow-up verification:
|
||||
|
||||
- RED: six join cases failed with `joinBackendPath is not a function` before implementation.
|
||||
- Targeted: runtime config, API client, and EventSource suites — 14 tests passed.
|
||||
- Full frontend gate — exit 0 (40 test files, 191 tests, TypeScript, Vite build).
|
||||
- Rebuilt `thothii-frontend:test` successfully.
|
||||
- Hostile config image smoke — `frontend runtime config smoke: ok`.
|
||||
- Rebuilt two-container smoke — default `/api` config, proxied `/api/health`, SPA fallback,
|
||||
and SSE-safe nginx directives all passed.
|
||||
@@ -1,98 +0,0 @@
|
||||
# Evidence / Preprocessing Task 1 Report
|
||||
|
||||
## Outcome
|
||||
|
||||
Implemented the additive Evidence source port and canonical corpus records. Existing evidence,
|
||||
search, vector, and session runtime code is unchanged.
|
||||
|
||||
## Contract
|
||||
|
||||
- `EvidenceSource` is a runtime-checkable protocol with `discover` and `acquire` operations.
|
||||
- `SourceObject` and `AcquiredDocument` are frozen, reject extra fields, use independent metadata
|
||||
defaults, and restrict metadata to Pydantic `JsonValue` values.
|
||||
- `CanonicalDocument`, `CanonicalChunk`, and `CorpusManifest` are frozen and reject extra fields.
|
||||
- Provenance includes stable source IDs, canonical URIs, fingerprints, modification time, and
|
||||
content hashes.
|
||||
- Pipeline versions are recorded on documents, chunks, and manifests. Manifests also carry schema
|
||||
version, optional publish ID/vector generation, and paired embedding model/dimension fields.
|
||||
- Credential-like metadata keys are rejected recursively. Credentials are not model fields and
|
||||
therefore cannot enter serialized canonical artifacts through extras.
|
||||
|
||||
## TDD evidence
|
||||
|
||||
The initial focused run failed during collection because `tht.ports.evidence` and `tht.corpus`
|
||||
did not exist. After implementation, the focused suite passed.
|
||||
|
||||
## Verification
|
||||
|
||||
- Focused models/protocol tests: 13 passed.
|
||||
- Harness excluding Docker-backed L0 and the network-dependent wheel packaging test: 444 passed,
|
||||
5 deselected.
|
||||
- Focused Ruff: passed.
|
||||
- Full-repository Ruff remains blocked by 34 pre-existing findings outside the task files.
|
||||
- An unrestricted `pytest -q` attempt reached 453 passed and 5 deselected, but reported 47 Docker
|
||||
setup errors plus 4 Docker parity failures because the sandbox cannot access the Docker socket;
|
||||
the wheel packaging test also failed because its isolated `uv build` needs unavailable network.
|
||||
|
||||
## Concerns / follow-up
|
||||
|
||||
- Pydantic's `frozen=True` prevents model field reassignment but does not recursively freeze list
|
||||
and dict contents. `default_factory` prevents shared mutable defaults. Later pipeline stages should
|
||||
treat these value objects as immutable and construct replacements rather than mutate collections.
|
||||
- The adapter and normalization tasks should preserve the credential-free boundary by passing only
|
||||
these records beyond acquisition.
|
||||
|
||||
## Review hardening follow-up
|
||||
|
||||
All six binding review areas were addressed in a separate TDD pass:
|
||||
|
||||
- JSON metadata is recursively converted to immutable `FrozenDict`/tuple values while retaining
|
||||
stable object/array JSON serialization. Manifest document and chunk collections are tuples.
|
||||
- Secret-key matching now normalizes camelCase and punctuation. It rejects credential-specific
|
||||
names (passwords, API keys, access/refresh tokens, client/private keys, session cookies and
|
||||
authorization) recursively, while deliberate benign labels such as generic `token` and `secret`
|
||||
remain valid.
|
||||
- Canonical URIs require a scheme and reject userinfo or credential-bearing query parameters.
|
||||
- Namespaced IDs, SHA-256 content hashes, timezone-aware UTC timestamps, embedding/vector
|
||||
compatibility, unique IDs, chunk referential/provenance integrity, contiguous per-document
|
||||
ordinals and pipeline-version consistency are validated. Nested Pydantic instances are always
|
||||
revalidated so `model_copy(update=...)` cannot bypass a manifest boundary.
|
||||
- Acquired arbitrary bytes have explicit base64 JSON encoding and validation, covered by a JSON
|
||||
round-trip test.
|
||||
- `EvidenceSourceError` classifies transient/retryable versus permanent failures and exposes only
|
||||
recursively immutable, credential-screened JSON details.
|
||||
|
||||
Follow-up verification:
|
||||
|
||||
- Focused contract suite: 39 passed.
|
||||
- Focused Ruff: passed.
|
||||
- Harness excluding Docker-backed L0 and the network-dependent wheel packaging test: 470 passed,
|
||||
5 deselected.
|
||||
- Fresh unrestricted harness attempt: 479 passed, 5 deselected; the same environmental boundary
|
||||
remains (47 Docker socket setup errors, four Docker parity failures, one isolated `uv build`
|
||||
network failure).
|
||||
|
||||
## Final blocker follow-up
|
||||
|
||||
The remaining four contract blockers were closed in a third TDD cycle:
|
||||
|
||||
- `EvidenceSourceError` now always exposes the fixed public message/`args` value `evidence source
|
||||
operation failed`; caller diagnostics are not retained. Category, details and args cannot be
|
||||
reassigned, details remain recursively frozen and credential-screened, and an original exception
|
||||
is available only when callers use standard exception chaining.
|
||||
- Canonical document/chunk provenance stores only URI scheme, authority and path. Userinfo is
|
||||
rejected; query strings and fragments are removed unconditionally, including AWS `X-Amz-*`, SAS
|
||||
`sig`, and fragment token material.
|
||||
- Binding model bases override Pydantic's unchecked `model_copy(update=...)`: merged values always
|
||||
pass full field/model validation, so invalid copied records and top-level manifests fail.
|
||||
- A canonical document/chunk `content_hash` must equal SHA-256 of the exact stored text encoded as
|
||||
UTF-8. This establishes the normalization boundary explicitly: line-ending/frontmatter/text
|
||||
normalization happens before model construction; the canonical models never rewrite content.
|
||||
|
||||
Final follow-up verification:
|
||||
|
||||
- Focused contract suite: 45 passed.
|
||||
- Focused Ruff: passed.
|
||||
- Harness excluding Docker-backed L0 and network-dependent packaging: 476 passed, 5 deselected.
|
||||
- Fresh unrestricted harness attempt: 486 passed, 5 deselected, with the unchanged environmental
|
||||
failures (47 Docker setup errors, four Docker parity failures, one isolated `uv build` failure).
|
||||
@@ -1,65 +0,0 @@
|
||||
# Evidence Task 2 Report
|
||||
|
||||
## Status
|
||||
|
||||
Implemented filesystem and explicit-manifest HTTP Evidence source adapters, typed source
|
||||
configuration with legacy compatibility, and factory construction.
|
||||
|
||||
## Delivered behavior
|
||||
|
||||
- Filesystem discovery is deterministic and rooted at a strict canonical directory.
|
||||
- Symlink/path escapes are rejected before content is exposed.
|
||||
- Discovery hashing and acquisition reads enforce a configurable byte limit.
|
||||
- Filesystem fingerprints are content SHA-256 values; stable IDs derive from relative paths.
|
||||
- HTTP accepts only explicit `http`/`https` manifest entries and keeps transport URLs private.
|
||||
- HTTP provenance strips query strings/fragments, while config and adapter representations hide
|
||||
signed or secret-bearing transport URLs.
|
||||
- HTTP acquisition uses separate connect/read timeouts, streaming byte limits, bounded redirects,
|
||||
private redirect rejection, and safe transient/permanent error classification.
|
||||
- HTTP fingerprints prefer a deterministic ETag digest, then Last-Modified, then content SHA-256.
|
||||
- `build_evidence_sources(cfg)` supports both typed `evidence.sources` entries and the legacy
|
||||
`source_root` plus `evidence_dir` filesystem configuration.
|
||||
|
||||
## TDD and verification
|
||||
|
||||
- RED: focused tests initially failed during collection because the adapter package did not exist.
|
||||
- GREEN: `15 passed` for filesystem, HTTP, and resource-config tests.
|
||||
- Full harness: `548 passed, 5 deselected`.
|
||||
- Changed-file Ruff: clean.
|
||||
- Repository-wide Ruff remains non-clean due to 34 pre-existing findings in unrelated test files;
|
||||
no unrelated lint files were modified.
|
||||
|
||||
## Notes
|
||||
|
||||
The approved `SourceObject` namespace grammar does not permit raw quoted ETags such as
|
||||
`etag:"abc"`. The adapter therefore uses `etag:<sha256-of-opaque-etag>`: it preserves ETag-based
|
||||
change identity without weakening the canonical contract or exposing validator contents.
|
||||
|
||||
## Review hardening follow-up
|
||||
|
||||
Four review findings were closed in a separate follow-up commit:
|
||||
|
||||
- Filesystem access now anchors a persistent descriptor at the canonical root and walks each
|
||||
component with `openat` semantics (`dir_fd`, `O_NOFOLLOW`, and `O_DIRECTORY`). The regular-file
|
||||
check, bounded read, metadata, and hash all use the opened descriptor. Acquisition reopens by
|
||||
the same path-safe mechanism and rejects a changed fingerprint. Deterministic tests swap both a
|
||||
leaf and an ancestor to symlinks at open time.
|
||||
- HTTP network policy defaults to public hosts only. Initial URLs and every redirect reject
|
||||
userinfo, mixed public/private IPv4/IPv6 answers fail closed, and the connected peer must be a
|
||||
public member of the previously validated DNS answer set before any body bytes are consumed.
|
||||
Explicit `allow_private_hosts: true` is required for trusted private deployments and local tests.
|
||||
- Every HTTP response is closed in a `finally` block, including redirects, status failures,
|
||||
policy failures, oversized bodies, and mid-stream exceptions.
|
||||
- ETag and Last-Modified values remain adapter-internal. Repeated discovery and acquisition send
|
||||
conditional headers; a 304 reuses only previously verified cached bytes and identity. The LRU
|
||||
content cache has an explicit byte bound (`max_cache_bytes`). Validators are not forwarded
|
||||
across redirect origins.
|
||||
|
||||
### Conditional cache binding correction
|
||||
|
||||
The conditional cache now binds bytes and validators to both the canonical provenance key and the
|
||||
exact final effective representation URL. Redirect traversal recomputes request headers per hop:
|
||||
validators are sent only when that exact URL matches the cached final URL, never merely because a
|
||||
redirect retains an origin. A same-origin path change therefore downloads and replaces the body.
|
||||
The adapter accepts 304 only when the exact request carried a bound ETag or Last-Modified validator;
|
||||
unsolicited and cross-origin 304 responses are permanent protocol errors.
|
||||
@@ -1,59 +0,0 @@
|
||||
# Evidence Task 3 — deterministic normalization and chunking
|
||||
|
||||
## Outcome
|
||||
|
||||
- Added pure `normalize(acquired, pipeline_version)` and `chunk(document, policy)` transforms.
|
||||
- Normalization enforces UTF-8 (including UTF-8 BOM), a 10 MiB input ceiling, LF line endings,
|
||||
NFC Unicode, safe YAML frontmatter extraction, canonical provenance URIs, and hashes the exact
|
||||
canonical UTF-8 text stored on the document.
|
||||
- Undecodable, unsupported-charset, oversized, and invalid-frontmatter inputs fail explicitly;
|
||||
byte content is never truncated.
|
||||
- Chunking uses a versioned immutable policy, paragraph/word boundaries with deterministic
|
||||
character-count hard splits for long tokens, contiguous ordinals, provenance metadata, exact
|
||||
per-chunk hashes, and IDs derived from document hash + ordinal + policy version.
|
||||
- Empty documents produce no chunks. Non-ASCII, CRLF equivalence, repeatability, policy changes,
|
||||
duplicate-content ordinal collisions, and max-character limits are covered by tests.
|
||||
|
||||
## TDD evidence
|
||||
|
||||
- Initial focused test run failed during collection because both transform modules were absent.
|
||||
- The EOF-frontmatter edge test was separately observed failing before its implementation.
|
||||
- Final focused verification: `12 passed`.
|
||||
|
||||
## Verification
|
||||
|
||||
- `cd harness && .venv/bin/pytest tests/test_corpus_normalize.py tests/test_corpus_chunk.py -q`
|
||||
— **12 passed**.
|
||||
- `cd harness && .venv/bin/pytest -q` — **573 passed, 5 deselected**. The sandboxed attempt could
|
||||
not access Docker; the approved rerun with local Docker access passed.
|
||||
- Targeted Ruff over all four implementation/test files — **clean**.
|
||||
- Full `cd harness && .venv/bin/ruff check .` — reports **34 pre-existing errors** in unrelated
|
||||
legacy tests (unused imports and existing E702 semicolon lines); none are in Task 3 files.
|
||||
|
||||
## Concerns
|
||||
|
||||
- The 10 MiB normalization ceiling is deliberately explicit and independent of adapter download
|
||||
limits. If deployment policy needs a different ceiling, it should become a versioned pipeline
|
||||
configuration before ingestion is wired.
|
||||
- Character limits use Python Unicode code points (`len`), not UTF-8 bytes or tokenizer tokens;
|
||||
this is recorded in the chunk-policy metadata and tested with non-ASCII content.
|
||||
|
||||
## Review hardening follow-up
|
||||
|
||||
- Chunk IDs now bind the canonical document identity, document content hash, ordinal, chunk hash,
|
||||
and a canonical SHA-256 fingerprint of every `ChunkPolicy` field. Identical content in separate
|
||||
documents and same-version policies with different limits cannot collide.
|
||||
- Boundary-aware slicing now retains separators in the slices. Concatenating every chunk exactly
|
||||
reconstructs the canonical document for repeated spaces, tabs, blank lines, Markdown hard
|
||||
breaks, fenced code, whitespace-only input, Unicode, and overlong tokens; every slice remains
|
||||
within `max_chars`.
|
||||
- Frontmatter uses a bounded `SafeLoader` variant: duplicate keys, anchors/aliases, structures
|
||||
deeper than 20 nodes, and documents larger than 1000 composed nodes are rejected. YAML parse,
|
||||
JSON type, credential-safety, and resulting canonical-model errors attributable to frontmatter
|
||||
map to `PermanentNormalizationError(reason="invalid_frontmatter")`; invalid pipeline policy
|
||||
remains a programmer-facing `ValueError`.
|
||||
- Follow-up TDD evidence: the expanded focused suite first reported 11 expected failures against
|
||||
the prior implementation, then passed **45/45** across normalization, chunking, and manifest
|
||||
invariants.
|
||||
- Follow-up full verification: **586 passed, 5 deselected**. Targeted Ruff is clean. Full Ruff
|
||||
continues to report the same **34 unrelated pre-existing** violations in legacy tests.
|
||||
@@ -1,107 +0,0 @@
|
||||
# Evidence Task 4 — shared job envelope
|
||||
|
||||
Status: complete
|
||||
|
||||
## Delivered
|
||||
|
||||
- Immutable `JobSpec`, `JobRun`, `JobReport`, per-stage state, sanitized error, and UTC
|
||||
timestamp records.
|
||||
- `run_job(spec, stages)` with a durable checkpoint at job start, before and after every stage,
|
||||
and at terminal state. Successful stages are skipped when a prior run is resumed.
|
||||
- Atomic JSON checkpoint/report replacement using a unique same-directory temporary file,
|
||||
file `fsync`, atomic `os.replace`, and parent-directory `fsync`.
|
||||
- Public reports contain fixed operational fields only. Workspace paths, stage return values,
|
||||
exception messages, source content, credentials, and arbitrary metadata are not serialized.
|
||||
- `WorkspaceJobLock` uses non-blocking kernel `flock` on a stable workspace/job-specific inode.
|
||||
Locks are released by the kernel on process exit; lock files are never removed based on PID,
|
||||
avoiding stale-lock and PID-reuse deletion races. Evidence and DWH use distinct lock files.
|
||||
- Dry-run intent is immutable in the spec/report and exposed to every stage through `JobContext`.
|
||||
|
||||
## TDD evidence
|
||||
|
||||
Initial focused collection failed because `tht.jobs` did not exist. Tests then drove:
|
||||
|
||||
- failure, sanitized reporting, resume, and idempotent successful-stage skipping;
|
||||
- corrupt-checkpoint refusal before stage execution;
|
||||
- JSON schema and path/secret/PII exclusion;
|
||||
- dry-run propagation and ordered aware timestamps;
|
||||
- multiprocessing exclusion, distinct Evidence/DWH jobs, traversal rejection, and recovery after
|
||||
a lock-owning process crashes.
|
||||
|
||||
Final focused result:
|
||||
|
||||
```text
|
||||
11 passed in 0.42s
|
||||
```
|
||||
|
||||
## Verification
|
||||
|
||||
```text
|
||||
cd harness && .venv/bin/pytest -q
|
||||
597 passed, 5 deselected, 17 warnings in 28.45s
|
||||
|
||||
cd harness && .venv/bin/ruff check tht/jobs tests/test_job_runner.py tests/test_job_locking.py
|
||||
All checks passed!
|
||||
```
|
||||
|
||||
The full Ruff invocation was also run. It reports 34 pre-existing violations in unrelated legacy
|
||||
tests; no Task 4 file is among them. L2 tests remain deselected by the repository configuration.
|
||||
|
||||
## Operational notes
|
||||
|
||||
- `fcntl.flock` intentionally targets the supported Linux/macOS deployment environments; it is not
|
||||
a Windows locking implementation.
|
||||
- The envelope does not publish or mutate an active corpus. Later pipeline stages must use
|
||||
`JobContext.run_dir` for staging and perform their own final atomic publish only after validation.
|
||||
- A dry run is an execution mode foundation: the runner exposes and records it; individual stages
|
||||
remain responsible for suppressing external mutations.
|
||||
|
||||
## Review hardening follow-up
|
||||
|
||||
Four post-implementation findings were fixed test-first:
|
||||
|
||||
1. Resume compatibility is now a canonical SHA-256 fingerprint over checkpoint schema version,
|
||||
hashed workspace identity, job type, dry-run mode, explicit spec/pipeline versions,
|
||||
configuration/input fingerprints, and the exact ordered explicit `stage_ids`. Any insertion,
|
||||
removal, reorder, mode, identity, version, config, or input change rejects resume before a stage
|
||||
executes. Omitting `resume_run_id` remains the explicit safe path for a new run.
|
||||
2. Lock traversal now uses directory file descriptors with `O_DIRECTORY` and `O_NOFOLLOW`.
|
||||
Lock files use `O_NOFOLLOW | O_CLOEXEC`; `fstat` requires a regular file owned by the current
|
||||
UID with one link, and permissions are forced to `0600` (`0700` for private directories).
|
||||
Pre-existing lock-file and lock-directory symlinks are rejected.
|
||||
3. Stage failures now serialize only the fixed safe tuple `internal` / `stage_exception` /
|
||||
`stage execution failed`. Neither exception class names nor messages are inspected for output;
|
||||
a hostile exception-name/message regression test proves a terminal failed report is retained.
|
||||
4. Job/run directory creation is no-follow, owner-checked, private, and durable. Each newly created
|
||||
parent is fsynced, the run directory is fsynced before the first atomic file write, and the
|
||||
existing file-fsync → replace → directory-fsync ordering has an explicit regression test.
|
||||
|
||||
Follow-up verification:
|
||||
|
||||
```text
|
||||
focused job/lock suite: 27 passed in 0.45s
|
||||
full harness suite: 613 passed, 5 deselected, 17 warnings in 29.65s
|
||||
Task 4 scoped Ruff: All checks passed
|
||||
```
|
||||
|
||||
Repository-wide Ruff continues to report the same 34 unrelated pre-existing legacy-test findings.
|
||||
|
||||
## Final resume-integrity fix
|
||||
|
||||
Resume is now read-only until the source checkpoint proves trustworthy. The runner loads the source
|
||||
before allocating a new run ID or directory, validates the exact stage state/timestamp/error ledger,
|
||||
rejects duplicate stage identifiers, and recomputes compatibility from every persisted compatibility
|
||||
field plus the exact ordered persisted stage IDs. It first requires the stored fingerprint to match
|
||||
that recomputation, then compares the trusted recomputation with the requested job fingerprint.
|
||||
|
||||
Valid-JSON tampering tests cover removed, inserted/duplicated, reordered, and substituted stages;
|
||||
input-field and stored-fingerprint changes; and invalid stage-state shapes. Every rejection occurs
|
||||
before stage execution and asserts that the runs directory contains no orphan allocation.
|
||||
|
||||
Final verification:
|
||||
|
||||
```text
|
||||
focused job/lock suite: 34 passed in 0.56s
|
||||
full harness suite: 620 passed, 5 deselected, 17 warnings in 27.42s
|
||||
Task 4 scoped Ruff: All checks passed
|
||||
```
|
||||
@@ -1,82 +0,0 @@
|
||||
# Evidence Task 5 report
|
||||
|
||||
## Outcome
|
||||
|
||||
Implemented an incremental Evidence corpus pipeline with immutable materialized generations,
|
||||
generation-scoped vector records, and an fsynced atomic `ACTIVE` pointer. Runtime Evidence
|
||||
artifact lookup reads the active canonical manifest and keeps a legacy source-tree fallback only
|
||||
when no corpus has been published.
|
||||
|
||||
The CLI is available as `tht preprocess evidence [--dry-run] [--resume RUN_ID] [--json]`.
|
||||
JSON success and failure output is pristine and failure details are sanitized.
|
||||
|
||||
## Safety and failure model
|
||||
|
||||
- A workspace writer lock serializes preprocess writers; readers never take the lock.
|
||||
- Generation directories, manifests, materialized files, locks, and `ACTIVE` reject symlink/path
|
||||
escape cases and use owner-only durable writes.
|
||||
- Vector records use generation-specific keys and metadata. The active manifest maps each active
|
||||
document to its valid vector generation, allowing unchanged documents to retain their vectors.
|
||||
- Runtime retrieval admits only active document IDs and their manifest-selected generations.
|
||||
Removed documents and partial writes from failed generations are therefore unreachable.
|
||||
- Embedding count and dimension checks occur before vector upsert; vector write count is checked
|
||||
before staging/publish. Any failure leaves `ACTIVE` unchanged.
|
||||
- Dry runs perform discovery/fingerprint planning only and never acquire, embed, write vectors, or
|
||||
publish. Fully unchanged runs return the active generation without creating a replacement.
|
||||
- Resume can safely retry idempotent generation-scoped upserts and publish an already staged,
|
||||
compatibility-checked generation after a crash between staging and pointer replacement.
|
||||
|
||||
## TDD evidence
|
||||
|
||||
Initial focused collection failed because `tht.corpus.pipeline` and `tht.corpus.store` did not
|
||||
exist. The implemented suite covers incremental skips, removals, model/policy rebuilds, acquire and
|
||||
partial-vector failures, dry-run isolation, dimension validation, atomic reader snapshots, pointer
|
||||
validation, symlink defense, and pristine CLI JSON.
|
||||
|
||||
Fresh focused verification:
|
||||
|
||||
```text
|
||||
18 passed, 3 warnings in 0.39s
|
||||
```
|
||||
|
||||
Command:
|
||||
|
||||
```text
|
||||
.venv/bin/pytest tests/test_corpus_pipeline.py tests/test_corpus_publish.py \
|
||||
tests/test_preprocess_cli.py tests/test_search_pack.py tests/test_session_documents.py -q
|
||||
```
|
||||
|
||||
Scoped Ruff: `All checks passed!`
|
||||
|
||||
Broader non-Docker/non-packaging run reached `560 passed, 5 deselected`; ten pre-existing HTTP
|
||||
adapter tests could not bind localhost under the sandbox. The complete suite reached `570 passed,
|
||||
5 deselected`, with the remaining failures/errors caused by denied Docker socket, localhost bind,
|
||||
and offline wheel-build access. No task-focused test failed.
|
||||
|
||||
## Remaining operational gate
|
||||
|
||||
Live pgvector integration needs Docker or an authorized local pgvector endpoint. The compensation
|
||||
strategy is logical isolation rather than destructive cleanup because the shared `VectorStore`
|
||||
port intentionally exposes no delete/transaction API; unreachable failed generations can be
|
||||
garbage-collected by a future maintenance job.
|
||||
|
||||
## Review integration wave
|
||||
|
||||
Added an enforceable `metadata_filter` vector-port contract and capability flags. Direct pgvector
|
||||
places exact Evidence generation/document predicates in SQL before `LIMIT`; HTTP sends the same
|
||||
filter to the RPC and deliberately does not use the legacy 404 fallback. The reader RPC script now
|
||||
validates and applies that filter. Normal Evidence search and search-pack use an ACTIVE-aware
|
||||
searcher that groups active documents by generation, executes complete server-filtered searches,
|
||||
and merges the results.
|
||||
|
||||
Added exact-generation Evidence cleanup to direct and HTTP writers plus the allowlisted writer RPC.
|
||||
Pipeline failures compensate both staged filesystem state and vector writes; cleanup failures stay
|
||||
sanitized and ACTIVE filtering remains the exposure boundary. Corpus-present session artifact
|
||||
resolution now fails closed on corrupt/missing ACTIVE rather than falling through to source files.
|
||||
|
||||
Focused review-wave verification: 45 passed, scoped Ruff clean. A mocked REST regression proves
|
||||
the exact filter payload and fail-closed legacy 404 behavior.
|
||||
|
||||
Still outstanding from the expanded review request: Task-4 JobRunner stage-by-stage integration,
|
||||
published-generation retention/garbage collection, same-fd `dirfd` materialized-file reads, and
|
||||
live local pgvector integration could not be completed in this wave.
|
||||
@@ -1,94 +0,0 @@
|
||||
# Evidence Task 5B implementation report
|
||||
|
||||
## Status
|
||||
|
||||
Integrated Evidence preprocessing with the Task 4 `JobRunner`. The CLI now accepts only a
|
||||
32-character JobRunner run ID for `--resume`; generation IDs remain outputs. Runs persist the
|
||||
exact ordered stages `discover`, `acquire_normalize_chunk`, `embed`, `vector_upsert`,
|
||||
`stage_validate`, `publish`, and `retention_cleanup`.
|
||||
|
||||
Successful-stage artifacts are copied into the new resume run before execution, allowing later
|
||||
stages to continue without rediscovery, acquisition, normalization, chunking, or embedding.
|
||||
Job compatibility includes workspace, configuration, discovered-input, pipeline, embedding, and
|
||||
chunk-policy fingerprints. Generation-specific filesystem/vector compensation is retained, and a
|
||||
compensated generation is rotated before retry. `ACTIVE` is mutated only by `publish`.
|
||||
|
||||
Dry-run executes discovery/planning and makes every side-effecting stage a no-op. JSON output is
|
||||
pristine and includes the JobRunner `run_id`, `resumed_from`, generation, plan, and publish status.
|
||||
|
||||
## TDD evidence
|
||||
|
||||
- RED: run-ID rejection and resume-artifact tests failed because generation IDs reached
|
||||
configuration and resume runs had empty artifact directories.
|
||||
- GREEN: the two regression tests passed after strict CLI validation and durable artifact carryover.
|
||||
- Added pipeline job-plan and dry-run counting-fake coverage; both passed.
|
||||
|
||||
## Fresh verification
|
||||
|
||||
- Focused integration/search suite: `62 passed, 4 warnings`.
|
||||
- Available harness suite excluding sandbox-blocked Docker, loopback HTTP-server, and networked
|
||||
wheel-build tests: `559 passed, 5 deselected, 18 warnings`.
|
||||
- Scoped Ruff: `All checks passed!`.
|
||||
- `git diff --check`: clean.
|
||||
|
||||
## Environment limitations and concerns
|
||||
|
||||
The literal full harness invocation cannot complete in the managed sandbox: Docker socket access,
|
||||
loopback HTTP test servers, and the `uv build` dependency resolution path are denied. It reached
|
||||
`575 passed, 5 deselected` before those environment errors. The available-suite rerun above is
|
||||
green.
|
||||
|
||||
One pre-existing Pydantic serialization warning is exposed by the new end-to-end job test when
|
||||
canonical metadata contains frozen tuple values; it does not contaminate CLI stdout. Retention is
|
||||
an explicit stable no-op until a retention policy is configured.
|
||||
|
||||
## Review fix wave — crash consistency and artifact integrity
|
||||
|
||||
Addressed all five follow-up findings:
|
||||
|
||||
- `JobRunner` now supports a test-only post-call/pre-checkpoint fault hook. Each stage seals a
|
||||
canonical artifact manifest containing required flat filenames, SHA-256, byte size, producer
|
||||
stage, and the full spec compatibility fingerprint. Resume validates the checkpoint and every
|
||||
sealed artifact before allocating/copying a new run, rejecting missing, tampered, extra, nested,
|
||||
or symlinked state. A sealed `running` stage is promoted after a simulated process crash; a
|
||||
sealed `failed` stage is deliberately retried.
|
||||
- Vector intent (exact record IDs and content hashes) is sealed before upsert. Execution reconciles
|
||||
`existing_hashes` and writes only missing/mismatched rows. Crash-after-effect tests prove no
|
||||
duplicate acquire, embed, or vector upsert.
|
||||
- Raw upsert, stage, recovery-upsert, recovery-stage, and publish exceptions compensate the exact
|
||||
generation. Compensation markers survive failed checkpoints; resume rotates the generation,
|
||||
refreshes generation-bound artifacts, reconciles vectors, and stages idempotently.
|
||||
- `CorpusStore.publish` is idempotent and failure-atomic. If replace succeeds but directory fsync
|
||||
fails, it restores the previous `ACTIVE` value (or removes a newly created pointer), fsyncs the
|
||||
rollback, and re-raises. Pipeline cleanup refuses to discard a generation referenced by ACTIVE.
|
||||
- Added crash/resume coverage after all seven ordered stages; corrupt/missing plan, manifest, and
|
||||
embeddings; unsafe extra paths; nonexistent run IDs; raw vector/stage failures; and post-replace
|
||||
ACTIVE rollback.
|
||||
|
||||
Fresh fix-wave verification:
|
||||
|
||||
- Focused jobs/corpus/CLI/search suite: `82 passed, 17 warnings`.
|
||||
- Available harness suite (same sandbox exclusions described above):
|
||||
`579 passed, 5 deselected, 31 warnings`.
|
||||
- Scoped Ruff and `git diff --check`: clean.
|
||||
|
||||
## Final P1 fix — effect state and checkpoint-bound manifest roots
|
||||
|
||||
- Stage checkpoints now distinguish `intent` from `completed`. Vector intent is atomically sealed
|
||||
and checkpointed before upsert. A process-level `BaseException` after a partial multi-record
|
||||
write leaves the stage `running/intent`; resume never promotes it and instead reconciles
|
||||
`existing_hashes`, writing only the missing records. The completed state is persisted only after
|
||||
reconciliation returns successfully.
|
||||
- Every stage now persists its completed artifact state while still `running`, before the
|
||||
post-call fault hook. The checkpoint binds the SHA-256 of canonical `artifact-manifest.json`,
|
||||
effect state, exact producer stage, and exact required-file mapping. Resume validates this root
|
||||
and all bindings before promotion or copying.
|
||||
- Added process-interruption coverage proving the already-written vector record is not submitted
|
||||
twice, remaining records are written, and publish completes only after reconciliation. Added
|
||||
coordinated artifact/manifest, spec-binding, and producer-binding tamper rejection tests.
|
||||
|
||||
Fresh verification:
|
||||
|
||||
- Focused jobs/corpus/CLI/search suite: `86 passed, 18 warnings`.
|
||||
- Available broad harness suite: `583 passed, 5 deselected, 32 warnings`.
|
||||
- Scoped Ruff and `git diff --check`: clean.
|
||||
@@ -1,188 +0,0 @@
|
||||
# Evidence Task 5C report
|
||||
|
||||
## Delivered
|
||||
|
||||
- Added `vector.retain_published_generations` (default `3`, validation minimum `1`).
|
||||
- Retention runs only after publication. It keeps ACTIVE, the newest configured generations,
|
||||
and generations referenced by running or resumable failed job checkpoints.
|
||||
- Cleanup deletes the exact Evidence generation from the vector store before removing its
|
||||
immutable filesystem directory. Vector failures retain filesystem metadata for retry and
|
||||
produce credential-free partial reports.
|
||||
- Added idempotent `tht preprocess evidence gc [--dry-run] --json` reconciliation with pristine
|
||||
JSON output.
|
||||
- Materialized document reads now open generation/documents components with directory file
|
||||
descriptors and `O_NOFOLLOW`, require a regular file owned by the process with one link, and
|
||||
hash the bytes read from the same descriptor against the canonical manifest.
|
||||
- HTTP generation deletion is pinned to `delete_vector_generation` with exact
|
||||
table/kind/generation arguments. Legacy 404 responses fail closed with an actionable,
|
||||
sanitized migration message.
|
||||
|
||||
## Evidence
|
||||
|
||||
- Focused retention, safe-read, CLI, and HTTP contract tests: `51 passed` (Docker-backed direct
|
||||
parametrizations excluded from that focused invocation).
|
||||
- Real Docker pgvector adapter suites: `33 passed`.
|
||||
- Full harness suite, including Docker-backed tests: `668 passed, 5 deselected`.
|
||||
- Changed-file Ruff: clean.
|
||||
- `git diff --check`: clean.
|
||||
|
||||
The five deselected tests are the repository's opt-in `l2` tests requiring external services;
|
||||
they are not local pgvector tests. Test output retains pre-existing Pydantic serialization and
|
||||
legacy-config deprecation warnings.
|
||||
|
||||
## Review fix wave
|
||||
|
||||
- Publication is now explicit and durable (`PUBLISHED` marker). Retention candidates require a
|
||||
valid generation manifest and publication marker (ACTIVE remains backward-compatible), so
|
||||
staged and malformed directories neither consume retention slots nor become deletion targets.
|
||||
- The policy retains ACTIVE plus exactly `N-1` newest rollback publications, ordered by durable
|
||||
publication time and generation id. Running and failed-resumable JobRunner checkpoints protect
|
||||
every referenced plan generation.
|
||||
- `VectorStore` now exposes exact Evidence generation inventory. Direct pgvector uses a constrained
|
||||
`SELECT DISTINCT` over `kind='evidence'` and `metadata.vector_generation`; HTTP uses the
|
||||
allowlisted `list_evidence_generations` RPC and fails closed on legacy 404. The writer RPC SQL,
|
||||
revokes, and grants are packaged in `create_vector_writer_rpc.sql`.
|
||||
- Explicit GC reconciles the union of published filesystem generations and vector-only orphans,
|
||||
preserving vector-before-filesystem deletion and retry semantics.
|
||||
- `run_as_job` holds the same corpus writer lock across checkpoint recovery, staging, publish, and
|
||||
retention. Explicit GC already uses this lock, serializing candidate snapshots with publishers.
|
||||
- Session artifact consumers no longer receive the corpus source path after validation. They get
|
||||
an owned, read-only copy atomically written from the bytes read and hash-validated on the same
|
||||
descriptor.
|
||||
|
||||
Fresh verification after the fix wave: full harness `672 passed, 5 deselected`; Docker pgvector,
|
||||
HTTP parity, and migration suites `43 passed`; exact direct inventory/delete integration `1 passed`;
|
||||
changed-file Ruff and `git diff --check` clean.
|
||||
|
||||
## Final hardening verification
|
||||
|
||||
- Canonical generation validation is exact (`^gen:[0-9a-f]{32}$`) before HTTP/direct deletion;
|
||||
malformed HTTP inventory rows fail closed rather than entering the GC candidate set.
|
||||
- Added explicit protection coverage for running and failed-resumable JobRunner checkpoints, plus
|
||||
a second-GC idempotence assertion for vector-only orphan reconciliation.
|
||||
- Added deterministic concurrent locking coverage: a job paused after discovery retains the corpus
|
||||
writer lock, explicit GC blocks, then completes after publication without deleting the active run.
|
||||
- Added a descriptor-race regression: replacing the corpus pathname immediately after `read(2)`
|
||||
leaves the atomically materialized session-owned copy byte-for-byte equal to the validated ACTIVE
|
||||
document and its manifest hash.
|
||||
|
||||
Final fresh evidence: Docker pgvector/HTTP/migration suites `48 passed`; full harness `680 passed,
|
||||
5 external L2 deselected`; changed-file Ruff and `git diff --check` clean.
|
||||
|
||||
## Integrated Task 5 dependency fixes
|
||||
|
||||
- GC now distinguishes filesystem retention from vector dependencies. ACTIVE and the newest
|
||||
`N-1` published manifests keep their directories; every exact generation in their
|
||||
`document_generations` maps remains vector-protected even after its old publication directory is
|
||||
evicted. Job-protected manifests receive the same dependency treatment.
|
||||
- The real four-publication Docker lifecycle now includes an unchanged document whose vectors come
|
||||
from the first generation. With retention `N=2`, only the final two publication directories remain
|
||||
while the first generation's vectors remain searchable from ACTIVE and survive restart/explicit GC.
|
||||
- Evidence lookup is always wrapped by the ACTIVE-aware searcher. With no corpus/ACTIVE, Evidence
|
||||
returns no rows and search packs cannot expose legacy vectors; non-Evidence kinds are unchanged.
|
||||
- Session artifact resolution holds the corpus writer lock, snapshots the active manifest once, and
|
||||
materializes bytes using that exact `manifest_id`, preventing a concurrent publish/retain-1 GC from
|
||||
changing or deleting the selected source generation.
|
||||
|
||||
Focused unit tests, the updated real Docker lifecycle, changed-file Ruff, and `git diff --check` pass.
|
||||
The final full harness invocation completed with exit code 0, including the concurrently added DWH
|
||||
JobRunner tests.
|
||||
|
||||
## Final ACTIVE search review fixes
|
||||
|
||||
- `ActiveEvidenceSearcher` now treats default (`kinds=None`) and mixed-kind searches as explicit
|
||||
split queries: non-Evidence kinds are queried separately, while Evidence is queried only with
|
||||
ACTIVE manifest generation/document predicates applied server-side before every limit.
|
||||
- Results are merged deterministically by descending similarity then stable id and truncated once
|
||||
to the caller's global `top_n`. Pure non-Evidence searches retain their original delegate path.
|
||||
- The corpus writer lock now covers manifest snapshot construction and all corresponding vector
|
||||
queries, preventing retain-1 publication/GC from switching or deleting generations mid-search.
|
||||
- Removed the public post-LIMIT `active_evidence_hits` helper; no public Evidence path performs
|
||||
client filtering after limit.
|
||||
|
||||
Focused default/mixed/no-ACTIVE/search-pack tests pass, the real Docker pgvector lifecycle passes,
|
||||
and the final full harness plus scoped Ruff/diff invocation completed with exit code 0.
|
||||
|
||||
## Workspace-scoped Evidence isolation
|
||||
|
||||
- Evidence manifests, vector metadata, and record keys now carry the stable JobRunner workspace id
|
||||
derived from the configured workspace identity (config stem), never credentials or absolute paths.
|
||||
- Every ACTIVE server-side predicate includes `workspace_id`. Legacy unscoped rows therefore fail
|
||||
closed and cannot appear in Evidence results.
|
||||
- Vector generation inventory and deletion require the workspace namespace across the port, direct
|
||||
pgvector adapter, HTTP client/adapter, and allowlisted RPC SQL. Legacy unscoped RPC overloads are
|
||||
explicitly dropped during migration; destructive SQL matches collection, kind, generation, and
|
||||
workspace together.
|
||||
- GC recovers the persisted namespace from ACTIVE for explicit/restarted cleanup and can only list
|
||||
or delete that workspace's generations. Real shared-pgvector coverage proves deleting a generation
|
||||
for workspace A preserves the same generation in workspace B.
|
||||
- `PipelineResult.model_dump` now serializes fields explicitly instead of `dataclasses.asdict`,
|
||||
avoiding deepcopy of immutable `FrozenDict` metadata while preserving pristine JSON CLI output.
|
||||
|
||||
Final focused verification: `89 passed` across corpus/CLI JSON, direct/HTTP parity, migrations, and
|
||||
real Docker pgvector lifecycle; scoped Ruff and `git diff --check` clean. A contemporaneous full-suite
|
||||
run reached unrelated Task 6 immutable-file tamper tests; those files were deliberately not changed.
|
||||
|
||||
## Immutable corpus/workspace binding
|
||||
|
||||
- A corpus root becomes bound to the workspace id persisted in its ACTIVE manifest. Job, non-job,
|
||||
explicit GC, and ACTIVE search entry points compare the configured namespace before discovery,
|
||||
vector access, staging, deletion, or ACTIVE mutation.
|
||||
- Reusing the same paths after renaming a workspace now fails closed with a typed/sanitized message:
|
||||
use a new corpus root or perform an intentional explicit rebuild. Unscoped legacy manifests also
|
||||
fail this ownership check.
|
||||
- Tests prove unchanged-document reuse cannot silently mix workspace A vectors into a workspace B
|
||||
manifest, and that mismatched job, GC, and search paths perform no vector/filesystem mutations.
|
||||
|
||||
Focused workspace-binding, search-pack, preprocess JSON, and scoped Ruff/diff tests pass.
|
||||
|
||||
Compatibility follow-up: direct/internal `CorpusPipeline` instances now distinguish an omitted
|
||||
workspace identity from an explicit config/job identity. An unbound instance adopts the persisted
|
||||
ACTIVE owner (or `default` only for a brand-new direct corpus), preserving safe resume/GC tests and
|
||||
the real pgvector lifecycle. Explicit config/job identities still fail closed on any mismatch. The
|
||||
two reported regressions, workspace mismatch guards, real Docker lifecycle, scoped Ruff/diff, and
|
||||
the full harness suite all pass.
|
||||
|
||||
Final fail-closed follow-up: persisted ACTIVE ownership is now validated under the corpus lock before
|
||||
every configured search delegate, including default, mixed, pack, and non-Evidence-only operations.
|
||||
Malformed or missing `metadata.workspace_id` is intrinsically rejected even for unbound direct
|
||||
callers; source discovery, vector operations, GC, files, and ACTIVE remain untouched. Focused tests,
|
||||
real Docker lifecycle, scoped Ruff/diff, and the full harness regression run pass.
|
||||
|
||||
Final lock/preflight follow-up: `CorpusPipeline.gc()` now acquires the corpus writer lock itself for
|
||||
ownership validation through vector/filesystem cleanup. The store lock is thread-reentrant so nested
|
||||
job retention is safe without weakening cross-thread/process exclusion; the CLI wrapper no longer
|
||||
double-locks. Search find/pack performs locked corpus ownership preflight immediately after config
|
||||
load, before DWH leasing, vector/searcher factories, embeddings, or schema work. Focused concurrency
|
||||
and fail-closed tests, real Docker lifecycle, scoped Ruff/diff, and the full harness pass.
|
||||
|
||||
## Compact public Evidence reports
|
||||
|
||||
- Public `PipelineResult.model_dump()` is now a bounded operational envelope: terminal status,
|
||||
run/resume/publication/generation/manifest identifiers, capped changed/unchanged/removed source
|
||||
identifiers, and aggregate document/chunk counts. Full manifests, bodies, and metadata remain
|
||||
internal/on disk and are never serialized to CLI stdout.
|
||||
- `tht preprocess evidence` exits `1` for any durable terminal status other than `succeeded` in
|
||||
both JSON and text modes. JSON stdout remains one pristine sanitized object; text mode emits one
|
||||
compact stderr error without traceback, exception identity, evidence content, or credentials.
|
||||
- Tests cover a real failed acquisition job, sensitive evidence content, capped thousand-item
|
||||
summaries, bounded report size, and smoke-compatible changed/unchanged fields.
|
||||
|
||||
Focused tests and scoped Ruff/diff pass. The contemporaneous full suite reaches an unrelated Task 6
|
||||
DWH snapshot fixture missing its newly required workspace identity.
|
||||
|
||||
### Safe result representation and exact text totals
|
||||
|
||||
- `PipelineResult.manifest` is explicitly excluded from dataclass representation and the custom
|
||||
representation is fixed-size operational data only. It omits manifest ids, documents, chunks,
|
||||
content, metadata, and errors; `str(result)` inherits the same safe representation.
|
||||
- Text-mode Evidence success output reads the uncapped aggregate totals from `payload["counts"]`
|
||||
rather than the intentionally capped identifier arrays.
|
||||
- Regression coverage builds a thousand-document/chunk manifest containing content and
|
||||
credential-like metadata secrets, checks bounded `repr`/`str`, and verifies exact totals above
|
||||
the 100-item public-array cap.
|
||||
|
||||
Focused Evidence verification passes (`67 passed`), and scoped Ruff is clean. The full harness run
|
||||
is not green in this sandbox: Docker-backed tests cannot access the daemon, wheel packaging cannot
|
||||
use the restricted build environment, and concurrent Task 6 DWH binding changes currently fail two
|
||||
DWH tests. None of those failures touch the Evidence files in this follow-up.
|
||||
@@ -1,50 +0,0 @@
|
||||
# Evidence Task 5D — Real pgvector lifecycle gate
|
||||
|
||||
## Status
|
||||
|
||||
Complete. The Docker-backed L0 gate uses one persistent `pgvector/pgvector:pg16`
|
||||
database and the production migrations, direct reader/writer `PgVectorStore`,
|
||||
`CorpusStore`, `CorpusPipeline.run_as_job`/JobRunner, ACTIVE Evidence retrieval,
|
||||
search-pack fusion, owned session artifact copy, retention, and explicit GC.
|
||||
|
||||
## Lifecycle covered
|
||||
|
||||
- Four real corpus publications with retention set to two generations.
|
||||
- A higher-similarity stale vector proves ACTIVE metadata filtering happens before LIMIT
|
||||
for normal Evidence retrieval and the search-pack fusion path.
|
||||
- A removed source is absent from ACTIVE retrieval and cannot be copied to a session.
|
||||
- An injected process death occurs after one real committed vector upsert. Resume uses the
|
||||
real run ID, preserves that record, fills the missing records, and produces no duplicate keys.
|
||||
- Database engines and direct store objects are disposed/recreated before persisted ACTIVE
|
||||
retrieval is checked again.
|
||||
- An exact canonical vector-only orphan generation is discovered and removed by explicit GC.
|
||||
- Filesystem and vector inventories converge exactly to ACTIVE plus one rollback; a second GC
|
||||
is a no-op.
|
||||
- Owned session artifact bytes and SHA-256 match the ACTIVE canonical document.
|
||||
|
||||
## Production bug found and fixed
|
||||
|
||||
Production migration `003_roles.sql` intentionally restricted `vector_writer`, but omitted
|
||||
the privileges used by the production generation lifecycle: `SELECT(metadata)` for inventory
|
||||
and `DELETE` for cleanup on `vectors.evidence`. Consequently a real job published successfully
|
||||
and then failed in `retention_cleanup` on its first run.
|
||||
|
||||
Added versioned migration `004_evidence_generation_gc.sql` granting only those two Evidence
|
||||
generation-management privileges. Runtime application code was not redesigned.
|
||||
|
||||
## Verification
|
||||
|
||||
- Target lifecycle: `1 passed` (Docker-backed).
|
||||
- Full harness: `681 passed, 5 deselected`.
|
||||
- Scoped Ruff: passed.
|
||||
- `git diff --check`: passed.
|
||||
|
||||
The existing Pydantic serialization and legacy-workspace deprecation warnings remain unchanged.
|
||||
|
||||
## Follow-up assertion correction
|
||||
|
||||
The removal phase now retains the removed canonical document ID/ref before publication and
|
||||
asserts both fields are absent from post-resume ACTIVE Evidence hits. It reruns the real
|
||||
search-pack fusion after removal, proves active fourth-generation content is positively
|
||||
returned in both paths, and proves the removed content remains absent. The owned session
|
||||
artifact lookup for the retained removed ID remains empty.
|
||||
@@ -1,49 +0,0 @@
|
||||
# Evidence Task 6 — final fd-anchored DWH correction
|
||||
|
||||
All DWH generation state below `.tht-dwh` is now accessed relative to the directory descriptor
|
||||
retained by the shared/exclusive generation lease. ACTIVE reads, atomic temp writes, replacement,
|
||||
fsync, and rollback use `openat`/`replaceat` operations. Generation staging, validation,
|
||||
reconciliation, resume checks, retention classification, and recursive deletion likewise use owned
|
||||
root/generations/candidate descriptors with `O_NOFOLLOW`; locked operations no longer reopen
|
||||
generation paths through `workspace_root`.
|
||||
|
||||
Portable reader snapshots are copied from validated generation file descriptors into private 0700
|
||||
process-owned temporary directories while the shared lease is held. This avoids Linux-only
|
||||
`/proc/self/fd` paths and prevents a renamed/replaced `.tht-dwh` pathname from redirecting later
|
||||
schema or LSH reads. Lease-scoped copies are removed on exit and standalone snapshots are removed
|
||||
at process exit.
|
||||
|
||||
Deterministic adversarial tests rename the DWH root after lease acquisition during ACTIVE reads,
|
||||
ACTIVE publication, and retention cleanup. Each test proves the replacement tree is never read,
|
||||
written, or deleted; the descriptor-pinned original either completes consistently or fails closed.
|
||||
Existing owner binding, legacy rejection, crash reconciliation, resume, atomic rollback, retention,
|
||||
and reader/writer exclusion behavior remains covered.
|
||||
|
||||
## Final review correction
|
||||
|
||||
Snapshot materialization now reads the manifest and every owned artifact exactly once through the
|
||||
already-open generation descriptor, validates each hash against those exact bytes, and writes the
|
||||
same byte objects to the private snapshot. A deterministic second-read mutation test proves hostile
|
||||
pickle bytes can neither pass validation nor enter the snapshot. Reconciliation closes the ACTIVE
|
||||
generation descriptor in a `finally` block on matches, mismatches, and exceptions. Pipeline-owned
|
||||
snapshot directories are removed and deregistered after `run_job` on both successful and failed
|
||||
runs, preventing repeated pipeline use from accumulating temporary directories or registry entries.
|
||||
|
||||
The cleanup boundary now begins immediately after snapshot materialization. Resume checkpoint
|
||||
validation and `JobSpec` construction are guarded by the same release routine as `run_job`, so
|
||||
corrupt/mismatched resume state or constructor failure clears the pipeline holder, removes the
|
||||
private directory, and restores the snapshot registry to its prior state before propagating.
|
||||
|
||||
## Shipped preprocessing startup contract
|
||||
|
||||
Local-vector preprocessing now uses a dedicated Compose override. Both one-shot jobs depend on a
|
||||
successfully completed `vector-migrate`, whose transitive chain waits for database health and role
|
||||
reconciliation. The generic preprocessing overlay remains independently renderable and contains no
|
||||
local-vector services or password secrets. README commands include the local override and build the
|
||||
job image before running.
|
||||
|
||||
The real clean-project smoke no longer injects dependencies or manually starts, reconciles, or
|
||||
migrates PostgreSQL. Its first shipped `compose run preprocess-evidence` demonstrably creates the
|
||||
database, waits for health, runs reconciliation and migration, then runs the Evidence job. Unchanged
|
||||
rerun, changed-source publish, DWH preprocessing, ACTIVE verification, and injected-failure cleanup
|
||||
all pass through the same shipped dependency path.
|
||||
@@ -1,93 +0,0 @@
|
||||
# Evidence preprocessing Task 7 report
|
||||
|
||||
Implemented the S3-compatible Evidence adapter, explicit preprocessing Compose overlay, and
|
||||
operational gates.
|
||||
|
||||
- S3 discovery uses bounded paginator pages, page size, and total objects; acquisition enforces a
|
||||
byte ceiling and always closes streaming bodies.
|
||||
- Provenance is canonical `s3://bucket/key`. Versioned objects use `s3-version:<version>`;
|
||||
unversioned objects use a hashed exact ETag, and acquisition refuses validator drift.
|
||||
- The adapter uses boto3/botocore rather than custom signing. TLS verification is enabled by
|
||||
default. Custom HTTP and private endpoints require independent explicit opt-ins; endpoint
|
||||
userinfo is rejected and public custom endpoints are DNS-policy checked.
|
||||
- Access, secret, and session credentials support file-secret resolution into masked `SecretStr`
|
||||
config fields. They are never emitted in provenance, reports, errors, or Compose environment.
|
||||
- `deploy/compose.preprocess.yaml` provides separate one-shot Evidence and DWH jobs and is inert
|
||||
unless explicitly included with the `preprocess` profile.
|
||||
- `scripts/preprocess-smoke.sh` verifies both services render without secret material and pins an
|
||||
unchanged rerun plus a modified generation through deterministic pipeline tests.
|
||||
|
||||
Verification: focused S3/HTTP/filesystem/config tests 34 passed; operational smoke 2 passed; core
|
||||
image with locked boto3 extra built; full harness 702 passed, 5 deselected; scoped Ruff and diff
|
||||
checks passed.
|
||||
|
||||
Operational risk: custom S3-compatible endpoints remain part of the deployment trust boundary.
|
||||
Private endpoint access must be explicitly enabled and should be restricted by container egress
|
||||
policy in production. S3 list consistency semantics are provider-defined; version IDs are preferred
|
||||
over ETags wherever bucket versioning is available.
|
||||
|
||||
## Review correction
|
||||
|
||||
The Compose overlay now uses committed, purpose-built Evidence and DWH workspace files with
|
||||
job-specific dependencies. Its services create their lock roots and mount only the vector secrets
|
||||
they consume. The operational smoke is a real isolated Compose project: real pgvector migrations,
|
||||
a deterministic in-project embeddings endpoint, actual Evidence CLI JSON across initial/unchanged/
|
||||
mutated runs, exact ACTIVE verification, an actual DWH introspection job, and owned cleanup.
|
||||
|
||||
S3 custom endpoints now fail closed unless declared trusted; HTTP and private loopback endpoints
|
||||
need additional independent opt-ins. Boto uses forced path-style addressing. Custom endpoints reject
|
||||
userinfo, query, fragment, and non-root paths. Buckets use strict DNS syntax; listed keys must remain
|
||||
under prefix and within the S3 byte bound; validators must be nonempty/bounded. Because
|
||||
ListObjectsV2 does not provide version IDs, discovery honestly fingerprints the exact ETag and
|
||||
acquisition rejects ETag drift.
|
||||
|
||||
Final correction verification: S3/config focused 20 passed; full harness 721 passed, 5 deselected;
|
||||
real Compose smoke and image build passed; scoped Ruff, shell syntax, and diff checks passed.
|
||||
|
||||
## Final security review correction
|
||||
|
||||
Literal non-global IPv4/IPv6 endpoints now require the private-endpoint opt-in without claiming DNS
|
||||
pinning for hostnames. Pagination uses explicit continuation requests and never fetches page
|
||||
`max_pages + 1`. IP-shaped buckets, leading-slash prefixes, empty/overlong/control-character keys,
|
||||
and absent validators fail closed. Acquisition accepts only the exact stored `SourceObject` and
|
||||
compares the response ETag with the stored discovery validator. The real smoke snapshots generation
|
||||
directory counts after every run and has an injected-failure cleanup mode; cleanup fails if Compose
|
||||
down fails or any owned container, volume, or network remains.
|
||||
|
||||
The canonical smoke correction counts only root-level `corpus/gen-<32 hex>` directories. It exposed
|
||||
that the durable job path still published an empty unchanged generation; the pipeline now returns
|
||||
the existing ACTIVE generation without staging a directory when compatibility and all source
|
||||
fingerprints are unchanged. The smoke therefore proves directory deltas `+1`, `+0`, `+1`.
|
||||
Failure injection runs a real exit-97 command after resources exist and reaches the EXIT trap.
|
||||
Cleanup aggregates Compose-down, residual container/volume/network, and temp-directory failures
|
||||
while preserving the original failure status. S3 prefixes are validated before any client request
|
||||
for leading slash, UTF-8 byte length, controls, and DEL.
|
||||
|
||||
## Canonical unchanged-run correction
|
||||
|
||||
The durable job now persists a deterministic source snapshot keyed by source identity. Each entry
|
||||
binds canonical URI, exact source fingerprint, UTC modification time, canonical immutable metadata,
|
||||
and explicit media type and size contract fields. The manifest also binds document-to-source
|
||||
provenance, supplied config/input fingerprints, compatibility, embedding settings, and pipeline and
|
||||
chunk-policy versions.
|
||||
|
||||
An unchanged run reuses ACTIVE only when ownership, bindings, the complete snapshot, document
|
||||
provenance, materialized document hashes, and every required vector ID/content hash match exactly.
|
||||
Snapshot changes rebuild only the affected sources; job input/config changes publish a new manifest
|
||||
while retaining valid stable vector-generation dependencies. Missing or corrupt legacy contract
|
||||
metadata, documents, or vectors fails closed and rebuilds. The Compose smoke now explicitly expects
|
||||
the unchanged no-op to report `published=false` while proving generation deltas `+1`, `+0`, `+1`.
|
||||
|
||||
## Corrupt ACTIVE reconstruction correction
|
||||
|
||||
ACTIVE reuse now reconstructs each source contract from the persisted discovery snapshot and checks
|
||||
the deterministic document identity, canonical URI, source fingerprint, UTC modification time,
|
||||
source metadata, applicable media type, content hash, and pipeline identity against the owned
|
||||
materialized document. The persisted document-source map carries the same exact binding.
|
||||
|
||||
Chunks are recomputed under the current chunk policy and must match the manifest exactly in count,
|
||||
order, IDs, ordinals, content, hashes, linkage, provenance, and policy metadata. Vector health must
|
||||
report the configured dimension, and every recomputed chunk must have its generation-scoped vector
|
||||
ID with the exact content hash. Missing, altered, or extra chunks and corrupt document or vector
|
||||
contracts therefore disable the no-op and rebuild, while a valid unchanged run still performs no
|
||||
source acquisition.
|
||||
@@ -1,16 +0,0 @@
|
||||
# Model provider credential boundary
|
||||
|
||||
The backend accepts only an absolute `THT_MODEL_API_KEY_FILE` reference. `PiProcessManager` reads
|
||||
and validates it afresh before each hosted-provider spawn, rejects symlinks, non-regular/hard-linked,
|
||||
empty, whitespace-containing, oversized, unreadable, or permissively-mode files, and accepts Docker
|
||||
0444 secrets only beneath `/run/secrets`. Failures are sanitized and occur before child creation.
|
||||
|
||||
Provider names are normalized and mapped to Pi-recognized variables. The child environment removes
|
||||
the generic path, deprecated `PI_PROVIDER_API_KEY`, and all unselected known provider keys before
|
||||
injecting only the selected key. Values never enter argv, settings, health, or diagnostics. Local
|
||||
providers remain keyless and unknown hosted providers fail closed.
|
||||
|
||||
The production Compose overlay mounts `model_api_key` read-only and points the backend at its file;
|
||||
the deployment render smoke proves the value is absent from rendered configuration. Entrypoint,
|
||||
root README, Pi configuration guide, environment example, and secrets operator guide document the
|
||||
new contract and reject the legacy generic value variable.
|
||||
@@ -1,59 +0,0 @@
|
||||
# Local pgvector whole-plan final fix report
|
||||
|
||||
## Outcome
|
||||
|
||||
All four binding final-review findings are closed.
|
||||
|
||||
1. `PgVectorStore.health()` checks namespace `USAGE` independently for reader and writer
|
||||
before inspecting vector types. Real PostgreSQL tests revoke only schema `USAGE`, prove both
|
||||
health sides false and operations unavailable, then grant it back and prove recovery.
|
||||
2. Direct reader/writer passwords use workspace `password_file` references. Compose mounts the
|
||||
two files read-only into core and exposes only `_FILE` paths. Rendered Compose and live
|
||||
`docker inspect` checks prove secret contents are absent.
|
||||
3. Direct search failures map to `VectorReadUnavailable`; hash/upsert failures map to
|
||||
`VectorWriteUnavailable`. Messages are fixed and sanitized, original exceptions remain chained,
|
||||
and upsert rollback is preserved.
|
||||
4. The shared secret policy uses Linux `stat -c` with macOS `stat -f` fallback. Host files permit
|
||||
only `0600`/`0400`; Docker's read-only `0444` is accepted only beneath `/run/secrets`. Tests and
|
||||
operator docs pin this exact policy.
|
||||
|
||||
## TDD evidence
|
||||
|
||||
The new config, mode, schema-usage, unavailable-connection, and permission regressions failed
|
||||
before their implementations. The first live secret-policy run also caught GNU `stat -f` accepting
|
||||
an incompatible format invocation; detection now tries the native Linux form first. The next live
|
||||
run caught smoke-generated rotation fixtures at `0644`; fixtures now model the documented host
|
||||
policy.
|
||||
|
||||
## Verification
|
||||
|
||||
- Real direct pgvector + HTTP parity: `31 passed`.
|
||||
- Full harness from `harness/`: `493 passed, 5 deselected`.
|
||||
- Live `local-vector` rotation, restart persistence, inspect boundary, and backup/restore: pass.
|
||||
- Core image vector migration discovery/status smoke: pass.
|
||||
- External and local Compose deployment security contracts: pass.
|
||||
- Config/port focused suite: `26 passed`.
|
||||
- Secret policy, bootstrap rotation, and backup/restore safety scripts: pass.
|
||||
- Changed Python Ruff, shell syntax, and `git diff --check`: pass.
|
||||
|
||||
One attempted full-harness invocation from the repository root produced a path-dependent failure
|
||||
in an existing test that opens `workflow.yaml` relative to CWD. It was immediately rerun using the
|
||||
documented `cd harness && .venv/bin/pytest -q` command and passed completely.
|
||||
|
||||
## Operational notes
|
||||
|
||||
Workspace files contain file paths, never direct passwords. Secret contents necessarily exist in
|
||||
the in-process validated `DatabaseConfig` used to establish PostgreSQL connections, but are not
|
||||
serialized by doctor/Compose/inspect paths. Docker Desktop file-backed secrets may appear as bind
|
||||
mounts; the safe runtime exception is therefore based on the read-only service mount location
|
||||
`/run/secrets`, while source files remain owner-only on the host.
|
||||
|
||||
## External-profile regression follow-up
|
||||
|
||||
Local pgvector is now an explicit `deploy/compose.local-vector.yaml` overlay. The base Compose and
|
||||
production external override contain no direct vector password declarations, mounts, or `_FILE`
|
||||
variables, so external deployments do not resolve or require local password files. A real lifecycle
|
||||
gate unsets all local secret-file variables, renders external config, builds and starts core, waits
|
||||
for health, and inspects the live container for absence of local direct-vector secret paths. The
|
||||
local overlay retains its live inspect assertion (paths present, values absent), rotation, restart
|
||||
persistence, and transactional backup/restore drill.
|
||||
@@ -1,95 +0,0 @@
|
||||
# Local pgvector Task 1 report
|
||||
|
||||
## Status
|
||||
|
||||
Implemented the direct `PgVectorStore` behind the transport-neutral `VectorStore` port.
|
||||
The adapter uses separate optional reader and writer database configurations, derives
|
||||
capabilities from configured authority, validates strict positive search limits, filters kinds
|
||||
in SQL before limiting, and merges multi-collection results by cosine similarity.
|
||||
|
||||
All collection identifiers are selected from the fixed `schema_records`, `evidence`, and
|
||||
`memory` allowlist and composed with `psycopg2.sql.Identifier`. Values, vectors, kinds, hashes,
|
||||
and limits remain bound parameters. Collection/kind mismatches fail with `VectorStoreError`.
|
||||
|
||||
Upserts preserve the canonical metadata shape, use `record_key` conflict semantics, update the
|
||||
transport hash and embedding, and leave semantic metadata fields intact. Health probes reader
|
||||
and writer independently and reports observed `vector(N)` dimensions against the configured
|
||||
embedding dimension.
|
||||
|
||||
## Configuration and factory
|
||||
|
||||
`pgvector_direct` now accepts explicit optional `reader` and `writer` `DatabaseConfig` entries.
|
||||
The former `connection` entry remains supported as a deprecated read-only compatibility path.
|
||||
`build_vector_store(..., require_write=True)` accepts writer-only direct configurations and
|
||||
fails early when no explicit writer is present.
|
||||
|
||||
The transitional `build_vector_loader` bulk-sync path remains in place. It uses an explicit
|
||||
direct writer when present, or the legacy `connection`; it deliberately does not treat a new
|
||||
reader-only credential as writable. No production schema migration was added.
|
||||
|
||||
## TDD and verification
|
||||
|
||||
- RED: the new tests initially failed at collection because `PgVectorStore` did not exist.
|
||||
- Docker L0 pgvector tests: `11 passed`.
|
||||
- Direct + HTTP parity/factory/config focus: `51 passed`.
|
||||
- Full harness: `461 passed, 5 deselected`.
|
||||
- Changed-file Ruff lint: clean.
|
||||
- Changed-file Ruff format check: clean.
|
||||
- `git diff --check`: clean.
|
||||
|
||||
The repository-wide `ruff check .` still reports 34 pre-existing test-file findings outside
|
||||
Task 1; none are in changed files. The full pytest suite emits 17 existing legacy-config
|
||||
deprecation warnings.
|
||||
|
||||
## Scope and concerns
|
||||
|
||||
- Test fixtures create only the three existing vector tables needed to exercise the adapter;
|
||||
migration/versioning remains Task 2.
|
||||
- The legacy single `connection` form stays read-only through the public port, matching its
|
||||
previous adapter behavior, while remaining available to the explicitly documented bulk-loader
|
||||
transition.
|
||||
|
||||
## Review fix wave
|
||||
|
||||
The Task 1 review findings were addressed in a follow-up TDD cycle:
|
||||
|
||||
- Search now validates requested kinds against the global known-kind set, intersects valid kinds
|
||||
with each collection, and skips unrelated collections. A direct-versus-HTTP parity test covers
|
||||
the multi-collection case.
|
||||
- Health requires all three allowlisted tables, an `embedding vector(N)` column on every table,
|
||||
the expected dimension on every table, and the appropriate read or write table privileges for
|
||||
each configured side. Empty and partial schemas return deterministic, credential-free details;
|
||||
unexpected database failures expose only their exception class.
|
||||
- The Docker L0 fixture now provisions separate least-privilege reader and writer roles. Tests
|
||||
prove the reader cannot insert, the writer cannot execute the cosine-search SELECT, and the
|
||||
adapter still routes search to the reader and upsert/hash operations to the writer. Direct
|
||||
upsert uses an atomic `INSERT ... ON CONFLICT DO NOTHING` followed by `UPDATE` for an existing
|
||||
key, avoiding broad SELECT authority while retaining conflict-safe hash/upsert semantics.
|
||||
|
||||
Fresh verification after the fix wave:
|
||||
|
||||
- Docker L0 + HTTP port/search parity: `42 passed` (earlier checkpoint); the final L0 file has
|
||||
`16 passed` including the stricter raw-role search denial.
|
||||
- Expanded focused adapter/config suite: `56 passed`.
|
||||
- Full harness: `466 passed, 5 deselected`.
|
||||
- Changed-file Ruff lint/format and `git diff --check`: clean.
|
||||
|
||||
## Sequence privilege health follow-up
|
||||
|
||||
Writer health now resolves the real serial/identity sequence for the `id` column of every
|
||||
required collection using `pg_get_serial_sequence`. It requires `USAGE` on each resolved
|
||||
sequence, which is the privilege used by the adapter's implicit `nextval`; sequence `SELECT` is
|
||||
not required because no adapter operation reads sequence state.
|
||||
|
||||
The Docker fixture includes a writer role with complete table/hash-column authority but no
|
||||
sequence grant. Its health is deterministically unhealthy and a new-key upsert fails. Granting
|
||||
only sequence `USAGE` makes health green and the same port upsert succeeds. Sequence discovery is
|
||||
guarded for partial schemas so a missing `id` column produces the existing sanitized schema
|
||||
diagnostic instead of a PostgreSQL error.
|
||||
|
||||
Fresh verification for this follow-up:
|
||||
|
||||
- Docker pgvector L0 after formatting: `17 passed`.
|
||||
- Expanded focused adapter/config/parity suite: `57 passed`.
|
||||
- Full harness: `467 passed, 5 deselected`.
|
||||
- Changed-file Ruff lint/format and `git diff --check`: clean.
|
||||
@@ -1,82 +0,0 @@
|
||||
# Local pgvector Task 2 report
|
||||
|
||||
## Outcome
|
||||
|
||||
Implemented ordered, idempotent production migrations and the `tht vector migrate`
|
||||
interface, including `tht vector migrate --status --json` with pristine JSON output.
|
||||
|
||||
## Implementation
|
||||
|
||||
- `001_extensions.sql` installs pgvector.
|
||||
- `002_schema_tables.sql` creates `vectors.schema_records`, `vectors.evidence`, and
|
||||
`vectors.memory` with the `VectorWriteRecord` columns and `vector(768)` embeddings.
|
||||
- `003_roles.sql` creates passwordless `NOLOGIN` reader/writer roles. Deployments inject
|
||||
credentials (or grant these roles to separately-created login roles); no production secret
|
||||
is stored in the repository.
|
||||
- Reader authority is schema usage plus table `SELECT`.
|
||||
- Writer authority is schema usage, table `INSERT`/`UPDATE`, narrow hash-probe column `SELECT`,
|
||||
and sequence `USAGE`. It has no `DELETE`, broad row `SELECT`, DDL, or ownership authority.
|
||||
- The migration runner discovers ordered SQL files, records SHA-256 checksums in
|
||||
`public.tht_vector_migrations`, serializes runners with a transaction-scoped advisory lock,
|
||||
and applies the full pending batch in one transaction.
|
||||
- Status distinguishes applied, pending, and checksum-drifted migrations. Apply refuses drift.
|
||||
A failed migration rolls back both prior migrations in that batch and ledger writes.
|
||||
|
||||
## TDD evidence
|
||||
|
||||
RED was observed with a real `pgvector/pgvector:pg16` testcontainer: 6 failures for the missing
|
||||
module, missing command, and missing schema.
|
||||
|
||||
GREEN verification:
|
||||
|
||||
- Focused migration + direct adapter integration: `23 passed`.
|
||||
- Full harness from the documented `harness/` cwd: `473 passed, 5 deselected`.
|
||||
- Targeted Ruff (`tht` plus the new L0 test): clean.
|
||||
- `git diff --check`: clean.
|
||||
|
||||
The new L0 coverage exercises clean install, idempotent rerun, pristine JSON status, checksum
|
||||
drift, transaction rollback, exact tables/columns/dimensions, role isolation, sequence authority,
|
||||
and the real `PgVectorStore.health()` plus `VectorWriteRecord` upsert path.
|
||||
|
||||
## Existing repository lint baseline
|
||||
|
||||
The requested full `ruff check .` was run. It reports 34 pre-existing violations in unrelated
|
||||
test files (unused imports and one-line semicolon statements). None are in Task 2 files; changing
|
||||
them would exceed this task's scope. The complete harness test gate is green.
|
||||
|
||||
## Self-review
|
||||
|
||||
No unresolved Task 2 correctness concern found. One deliberate contract choice is worth noting:
|
||||
writer `INSERT` and `UPDATE` are table-level because the approved direct adapter health probe uses
|
||||
`has_table_privilege` for those authorities. Least privilege is retained by withholding broad
|
||||
`SELECT`, `DELETE`, DDL, ownership, and credentials.
|
||||
|
||||
## Review fix wave
|
||||
|
||||
The post-implementation review found four production-boundary gaps. They are fixed as follows:
|
||||
|
||||
- Migration SQL now ships inside the `tht` wheel (`tht/migrations/vector`) via explicit
|
||||
setuptools package-data and is discovered through `importlib.resources`, rather than relying on
|
||||
a source-checkout-relative directory.
|
||||
- Both status and apply reject ledger versions absent from the installed manifest, including
|
||||
nonnumeric future version labels. This treats a binary/database downgrade as drift instead of
|
||||
silently reporting a healthy state.
|
||||
- Migration files are ordered by parsed integer version; spellings such as `2` and `02` are
|
||||
rejected as duplicate versions.
|
||||
- Every migration transaction pins `search_path` locally to `pg_catalog, pg_temp`; catalog calls
|
||||
and the ledger are schema-qualified. pgvector is installed into the locked `vectors` schema,
|
||||
tables use `vectors.vector`, and `PgVectorStore` qualifies vector casts and the cosine operator.
|
||||
A hostile admin default path with a writable shadow schema cannot redirect migration objects.
|
||||
- The core image build asserts CLI discovery. Image verification now starts an ephemeral pgvector
|
||||
database, runs the installed image's migration command, and compares pristine apply/status JSON.
|
||||
|
||||
Additional verification after the fix wave:
|
||||
|
||||
- Focused migration, adapter, hostile-path, and wheel suite: `27 passed`.
|
||||
- Full harness: `477 passed, 5 deselected`.
|
||||
- Production core image build: passed, including build-time CLI discovery.
|
||||
- Core-image apply/status smoke against `pgvector/pgvector:pg16`: passed.
|
||||
- Changed production and test files: Ruff clean; `git diff --check` clean.
|
||||
- Full Ruff remains at the same 34 pre-existing unrelated test-file findings documented above.
|
||||
|
||||
No dependency changed, so the committed Python requirements lock did not require regeneration.
|
||||
@@ -1,133 +0,0 @@
|
||||
# Task 3 report — optional local pgvector profile
|
||||
|
||||
## Status
|
||||
|
||||
Implemented and verified the `local-vector` Compose profile.
|
||||
|
||||
- `vector-db` uses pgvector 0.8.5 on PostgreSQL 16, pinned to the official multi-arch
|
||||
manifest digest.
|
||||
- `vector_data` is a project-scoped named volume and is not shared with application data.
|
||||
- database readiness gates the packaged one-shot `vector-migrate` job; core declares the
|
||||
migration completion dependency while remaining usable in the pre-existing external profile.
|
||||
- bootstrap, migrator, reader, and writer identities are distinct. Bootstrap and migration
|
||||
credentials are supplied as Compose secrets; the application receives only reader/writer
|
||||
credentials.
|
||||
- `deploy/workspaces/local-vector.yaml` selects `pgvector_direct` with separate reader and
|
||||
writer connections.
|
||||
- the base loopback port binding, `AUTH_MODE=none`, and `THOTH_PUBLIC_EXPOSURE=false` defaults
|
||||
are unchanged.
|
||||
|
||||
## Red/green evidence
|
||||
|
||||
The initial Compose contract did not list `vector-db`, as required by the brief. The first real
|
||||
smoke then failed migration 002 because bootstrap installed the vector extension in `public`.
|
||||
The bootstrap was corrected to create the `vectors` schema under the migration owner and install
|
||||
the extension there. A clean-volume rerun passed.
|
||||
|
||||
## Verification
|
||||
|
||||
- `./scripts/local-vector-smoke.sh`: PASS
|
||||
- isolated generated Compose project and credentials
|
||||
- clean migration plus idempotent status rerun
|
||||
- reader/writer privilege health
|
||||
- one-record upsert and similarity search
|
||||
- restart of both `core` and `vector-db`
|
||||
- persisted search result after restart
|
||||
- project-only volume cleanup
|
||||
- `./scripts/test-container-deployment.sh`: PASS
|
||||
- `./scripts/test-backend-url-policy.sh`: PASS
|
||||
- `docker compose --profile local-vector config --quiet`: PASS
|
||||
- harness: 477 passed, 5 deselected
|
||||
- backend: 84 passed; TypeScript typecheck PASS
|
||||
- frontend: 226 passed; TypeScript typecheck PASS
|
||||
- `git diff --check`: PASS
|
||||
|
||||
## Self-review / concerns
|
||||
|
||||
- Compose cannot make a dependency required only under one profile. The core dependency uses
|
||||
`required: false` so the established `external` profile does not activate local infrastructure;
|
||||
under `local-vector`, `compose up --wait` still fails if `vector-migrate` exits nonzero, and the
|
||||
smoke verifies that successful migration precedes the healthy stack.
|
||||
- Reader/writer passwords are injected into core environment variables because Compose service
|
||||
attributes cannot be conditional by profile. Bootstrap and migrator credentials remain
|
||||
file-backed secrets and are never exposed to core.
|
||||
- The smoke intentionally refuses the operator project name `thothii` and removes only its unique
|
||||
project namespace and volumes.
|
||||
|
||||
## Follow-up hardening — credential reconciliation and cleanup ownership
|
||||
|
||||
Review findings were resolved in a separate follow-up:
|
||||
|
||||
- Replaced fresh-volume-only initialization with `vector-reconcile`, an idempotent one-shot that
|
||||
runs after database health and before `vector-migrate`. It authenticates with only the bootstrap
|
||||
admin secret, safely creates missing identities, reconciles role attributes and passwords on
|
||||
existing volumes, restores memberships/ownership, and leaves vector data untouched.
|
||||
- The migrator is explicitly `NOSUPERUSER NOCREATEDB NOCREATEROLE`. Schema/database ownership is
|
||||
sufficient for all packaged migrations because reconciliation creates the two group roles first.
|
||||
- The live smoke rotates migrator, reader, and writer secrets on the same populated volume, rejects
|
||||
the old reader credential, reruns migrations, recreates core with the new runtime credentials,
|
||||
and retrieves the record written before rotation and again after database/core restart.
|
||||
- Smoke project names are no longer caller-controlled. Each run creates a unique namespace and
|
||||
ownership token. Containers, networks, and volumes carry the ownership label; preflight refuses
|
||||
any collision and cleanup verifies every discovered resource before `down --volumes`.
|
||||
- Added a dynamic fake-Docker contract suite for caller override, collision, and mismatched cleanup
|
||||
labels, plus a real-Docker collision probe using a unique labeled volume.
|
||||
|
||||
Follow-up verification:
|
||||
|
||||
- `./scripts/local-vector-smoke.sh`: PASS, including live secret rotation and persisted retrieval
|
||||
- `./scripts/test-local-vector-smoke-safety.sh`: PASS
|
||||
- `./scripts/test-local-vector-smoke-live-collision.sh`: PASS
|
||||
- harness: 477 passed, 5 deselected
|
||||
- backend: 84 passed; TypeScript typecheck PASS
|
||||
- frontend: 226 passed; TypeScript typecheck PASS
|
||||
- Compose security, backend URL, config, shell syntax, and diff checks: PASS
|
||||
|
||||
Remaining operational constraint: the bootstrap admin secret must continue to match the PostgreSQL
|
||||
bootstrap account stored in the volume. Runtime migrator/reader/writer rotation is supported without
|
||||
data deletion; bootstrap-account password rotation is a distinct database-administration operation.
|
||||
|
||||
## Final hardening — bootstrap account rotation
|
||||
|
||||
The remaining operational constraint is now covered by
|
||||
`scripts/vector-rotate-bootstrap-password.sh OLD_SECRET_FILE NEW_SECRET_FILE`:
|
||||
|
||||
- It does not rely on `POSTGRES_PASSWORD_FILE` after initialization.
|
||||
- It pre-stages the deployment-file replacement in the same directory, authenticates to the live
|
||||
database with the explicit old file, and changes only the authenticated bootstrap role.
|
||||
- Passwords are passed as connection parameters and rendered with psycopg2 SQL composition, so
|
||||
shell and SQL metacharacters are not interpolated.
|
||||
- A second connection must authenticate with the new password before the command succeeds. If that
|
||||
verification fails, the still-open old connection restores the old database password.
|
||||
- Only after verified database login does an atomic rename replace the current deployment secret.
|
||||
Wrong-old authentication and verification failures leave deployment configuration unchanged.
|
||||
|
||||
Final live smoke evidence on one existing `vector_data` volume:
|
||||
|
||||
- wrong-old bootstrap rotation rejected; current deployment secret unchanged
|
||||
- bootstrap password with quote characters rotated successfully
|
||||
- old bootstrap login rejected and new login accepted
|
||||
- `vector-reconcile`, packaged migrations, and core health passed afterward
|
||||
- the vector record written before rotation remained searchable after rotation and after a further
|
||||
database/core restart
|
||||
|
||||
Final tests:
|
||||
|
||||
- `./scripts/test-vector-bootstrap-rotation.sh`: PASS
|
||||
- `./scripts/local-vector-smoke.sh`: PASS with negative and positive live bootstrap rotation
|
||||
- existing local-vector collision/safety and Compose deployment contracts: PASS
|
||||
|
||||
## Final identity and secret-policy alignment
|
||||
|
||||
- `THT_VECTOR_BOOTSTRAP_USER` is now passed through core as well as vector-db and reconciliation,
|
||||
so the rotation helper uses the authoritative configured role instead of defaulting to `postgres`.
|
||||
- Rotation and reconciliation source the same raw-file `secret-policy.sh`: non-empty and no
|
||||
whitespace, including trailing newlines. Rotation validates both files before Docker,
|
||||
PostgreSQL, or atomic replacement staging; `test-vector-secret-policy.sh` pins empty, newline,
|
||||
internal-space, and valid metacharacter cases.
|
||||
- Fake-Docker tests prove a non-default identity reaches the helper path and whitespace rejection
|
||||
performs no Docker call and creates no staged replacement.
|
||||
- The real smoke runs the entire stack as `thoth_bootstrap_smoke`. Its whitespace-negative case
|
||||
leaves the deployment file unchanged and proves the existing database login still succeeds;
|
||||
non-default-account bootstrap rotation, reconciliation, migration, core health, restart, and
|
||||
persisted retrieval all pass.
|
||||
@@ -1,94 +0,0 @@
|
||||
# Local pgvector Task 4 report
|
||||
|
||||
## Outcome
|
||||
|
||||
Implemented adapter parity gates and an operator-safe custom-format backup/restore workflow.
|
||||
|
||||
- Direct and HTTP stores now share validation, configured-dimension rejection, and deterministic
|
||||
similarity ordering with record ID as the tie-break.
|
||||
- The parity fixture exercises identical records through real pgvector and the HTTP RPC contract:
|
||||
kind filtering, ordering, hashes, replacement upserts, invalid collection/kind errors, and query
|
||||
plus write dimensions.
|
||||
- Backup explicitly allowlists the three vector tables and migration ledger, refuses overwrite,
|
||||
writes through a partial file, and uses a custom compressed archive.
|
||||
- Restore requires explicit active-source and target coordinates. It compares PostgreSQL system
|
||||
identifier plus database OID (robust across DNS aliases), refuses the active database, checks for
|
||||
an empty target unless force is explicit, and restores with exit-on-error.
|
||||
- Passwords are accepted only through validated secret files, converted to private temporary
|
||||
`PGPASSFILE`s, and never placed in command arguments or success/error logs.
|
||||
- Role passwords/login identities are deliberately not dumped. The target must have the approved
|
||||
passwordless group roles and pgvector extension reconciled before restore; archived ACLs restore
|
||||
the reader/writer grants.
|
||||
|
||||
## TDD and semantic alignment
|
||||
|
||||
The first parity run exposed the intended HTTP differences: it accepted unknown collections and
|
||||
wrong dimensions. Direct pgvector also had no stable order for equal cosine distance. The adapters
|
||||
were aligned, and the final focused real-pgvector gate passed: **25 passed**.
|
||||
|
||||
The first recovery run caught an incorrect probe username before restore. The second caught an
|
||||
intersection between `pg_dump --schema` and the explicit public ledger table. The third confirmed
|
||||
the archive contents but caught missing target group roles. Each defect was corrected and the
|
||||
complete drill was rerun from a fresh generated project.
|
||||
|
||||
## Live recovery smoke
|
||||
|
||||
`./scripts/local-vector-smoke.sh --backup-restore`: **PASS**.
|
||||
|
||||
- generated/owned source Compose project and source `vector_data`
|
||||
- distinct restore container and distinct named restore volume
|
||||
- migration and role health, secret rotation, restart persistence
|
||||
- real custom backup, then deliberate mutation of the active source record
|
||||
- same-database identity guard evaluated before restore
|
||||
- restore into the separate target only
|
||||
- restored hash equals the pre-mutation backup, proving retrieval parity
|
||||
- migration ledger has all three applied versions
|
||||
- all three restored embedding columns report `vectors.vector(768)`
|
||||
- ownership-checked cleanup; the active operator project/volume is never addressed
|
||||
|
||||
## Verification
|
||||
|
||||
- parity + direct adapter: 25 passed
|
||||
- full harness: 485 passed, 5 deselected
|
||||
- changed Python files: Ruff clean
|
||||
- shell syntax: clean
|
||||
- `git diff --check`: clean
|
||||
- full Ruff: unchanged repository baseline of 34 unrelated pre-existing test-file violations
|
||||
|
||||
## Self-review and operational constraints
|
||||
|
||||
The restore account must be able to read `pg_control_system()` for the robust cluster-identity
|
||||
comparison and create/restore the selected objects. This is intentionally an administrative
|
||||
recovery operation, not a runtime reader/writer action. `--force-nonempty` is explicit but still
|
||||
uses `pg_restore --clean --if-exists`; operators should prefer a new database/volume and validate
|
||||
migration status, health, and known retrieval before endpoint cutover.
|
||||
|
||||
## Post-review hardening
|
||||
|
||||
All five final review findings were addressed in a follow-up commit:
|
||||
|
||||
- Restore now requires a physically separate PostgreSQL cluster and refuses any equal
|
||||
`system_identifier`, independent of database OID or hostname.
|
||||
- `pg_restore` combines `--single-transaction` with `--exit-on-error`. The live drill creates an
|
||||
existing vector sentinel, deliberately fails late during a forced restore, and proves the
|
||||
original sentinel row/hash remains unchanged before performing the successful restore.
|
||||
- Backup uses a mode-0600 `mktemp` in the output directory, atomically renames it, and cleans only
|
||||
that owned path. A fake-command test pins symlink-clobber resistance and preserves an adversarial
|
||||
legacy `.partial` symlink and its target.
|
||||
- HTTP parity now traverses the real `VectorRestClient` transport boundary. It asserts RPC URL/key
|
||||
and kinds payloads, legacy 404 fallback, response conversion, malformed metadata tolerance, and
|
||||
canonical `VectorRestError` to `VectorStoreError` mapping.
|
||||
- The restored target runs role/secret reconciliation and a real `PgVectorStore` with separate
|
||||
reader/writer logins. Health, known-record search, writer upsert, hash probe, schema/table/column/
|
||||
sequence authority, and 768-dimensional compatibility are therefore verified through the
|
||||
production adapter. Reconciliation now restores group-role schema `USAGE`, which table-selected
|
||||
archives cannot carry.
|
||||
|
||||
### Atomic no-replace backup publication
|
||||
|
||||
The final publication review is also closed. The private same-directory archive is published with
|
||||
an atomic hard-link create rather than rename-overwrite semantics. If any process creates the final
|
||||
file or symlink after preflight but before publication, `ln` fails with `EEXIST`, the backup exits
|
||||
nonzero, the concurrent destination remains byte-for-byte intact, and the trap removes only the
|
||||
randomly named temporary archive owned by this invocation. The fake `pg_dump` safety test creates
|
||||
that destination immediately before returning and pins the failure and cleanup behavior.
|
||||
@@ -1,929 +0,0 @@
|
||||
# Pre-deployment Fix Wave Report
|
||||
|
||||
Date: 2026-07-14
|
||||
Worktree: `/home/chirone/ThothII/.worktrees/activity-log-cte-layout`
|
||||
Base: `e5366d14a6da8fb331d94be60b8929cefb1fe3e0`
|
||||
|
||||
## Outcome
|
||||
|
||||
All three reviewed findings are implemented in one coherent backend/frontend wave:
|
||||
|
||||
1. Resume leaves the prior selection, Zustand state, document panel, and EventSource untouched
|
||||
until `POST /resume` succeeds. Cold Resume changes state and reconnects only after backend
|
||||
clear/rebind; already-active same-session Resume preserves the existing binding; failure is a
|
||||
no-op apart from the fixed toast.
|
||||
2. SSE uses monotonically increasing per-session ids, cursor-filtered replay, native and manual
|
||||
reconnect cursors, id continuity across `hub.clear`, and descriptor-id pending-gate
|
||||
idempotence at both backend and frontend layers.
|
||||
3. Generic Pi system events and readiness errors are projected through explicit public
|
||||
allowlists. Sentinel URLs, paths, tokens, stderr, commands, and extra fields do not reach HTTP
|
||||
or SSE.
|
||||
|
||||
No harness, workflow, persistence, model, CTE viewer, CTE card, or shared Card file changed.
|
||||
|
||||
## Interfaces
|
||||
|
||||
- Frontend `resumeSession(id)` now returns
|
||||
`Promise<{ id: string; alreadyActive: boolean }>` via `ResumeSessionResult`.
|
||||
- Backend successful Resume always returns the same shape:
|
||||
- running/waiting runtime: `{ id, alreadyActive: true }`
|
||||
- validated cold runtime: `{ id, alreadyActive: false }`
|
||||
- `SseHub.publish(sessionId, event, data): number` returns the assigned SSE id.
|
||||
- `SseHub.subscribe(sessionId, send, { afterId, pending })` calls
|
||||
`send(event, data, id)` for replay/live frames with `id > afterId`.
|
||||
- `GET /sessions/:id/events` accepts native `Last-Event-ID` and manual
|
||||
`?lastEventId=<integer>`; when both are valid it uses the greater cursor.
|
||||
- Every emitted SSE frame is `id: <n>\nevent: <name>\ndata: <json>\n\n`.
|
||||
- Public readiness failure is exactly:
|
||||
`Session services are not ready. Check configuration and connectivity, then try again.`
|
||||
- Generic Pi system events are exactly `{ type: "system_event", event }`, and `event` must be a
|
||||
non-empty string.
|
||||
|
||||
## Files
|
||||
|
||||
Backend production:
|
||||
|
||||
- `backend/src/bridge/session-bridge.ts`
|
||||
- `backend/src/routes/sessions.ts`
|
||||
- `backend/src/sse/sse-hub.ts`
|
||||
|
||||
Backend tests:
|
||||
|
||||
- `backend/test/routes-sessions.test.ts`
|
||||
- `backend/test/session-bridge.test.ts`
|
||||
- `backend/test/sse-hub.test.ts`
|
||||
- `backend/test/sse-route.test.ts` (new)
|
||||
|
||||
Frontend production/support:
|
||||
|
||||
- `frontend/src/api/sessions.ts`
|
||||
- `frontend/src/api/types.ts`
|
||||
- `frontend/src/shell/AppShell.tsx`
|
||||
- `frontend/src/store/sessionStore.ts`
|
||||
- `frontend/src/stream/useSessionStream.ts`
|
||||
- `frontend/src/test/fakeEventSource.ts`
|
||||
|
||||
Frontend tests:
|
||||
|
||||
- `frontend/src/api/sessions.test.ts`
|
||||
- `frontend/src/shell/AppShell.session-mgmt.test.tsx`
|
||||
- `frontend/src/store/sessionStore.test.ts`
|
||||
- `frontend/src/stream/useSessionStream.test.tsx`
|
||||
|
||||
## TDD RED/GREEN evidence
|
||||
|
||||
### 1. Backend Resume result and client-boundary allowlists
|
||||
|
||||
RED command:
|
||||
|
||||
```text
|
||||
cd backend && npx vitest run test/routes-sessions.test.ts test/session-bridge.test.ts
|
||||
```
|
||||
|
||||
RED output (exit 1):
|
||||
|
||||
```text
|
||||
Test Files 2 failed (2)
|
||||
Tests 6 failed | 35 passed (41)
|
||||
|
||||
expected { id: 's1' } to deeply equal { id: 's1', alreadyActive: false }
|
||||
expected raw readiness URL/token/path to equal the fixed public message
|
||||
expected three raw generic system events to equal [{ type: 'system_event', event: 'session_exit' }]
|
||||
```
|
||||
|
||||
GREEN command:
|
||||
|
||||
```text
|
||||
cd backend && npx vitest run test/routes-sessions.test.ts test/session-bridge.test.ts
|
||||
```
|
||||
|
||||
GREEN output (exit 0):
|
||||
|
||||
```text
|
||||
✓ test/session-bridge.test.ts (14 tests)
|
||||
✓ test/routes-sessions.test.ts (27 tests)
|
||||
Test Files 2 passed (2)
|
||||
Tests 41 passed (41)
|
||||
```
|
||||
|
||||
### 2. Backend exact-once SseHub and route framing
|
||||
|
||||
RED command:
|
||||
|
||||
```text
|
||||
cd backend && npx vitest run test/sse-hub.test.ts test/sse-route.test.ts
|
||||
```
|
||||
|
||||
RED output (exit 1):
|
||||
|
||||
```text
|
||||
Test Files 2 failed (2)
|
||||
Tests 7 failed (7)
|
||||
|
||||
expected [undefined, undefined, undefined] to deeply equal [1, 2, 3]
|
||||
expected unconditional replay not to contain "one" / "two"
|
||||
expected one buffered pending gate, received replay plus a second pending emission
|
||||
```
|
||||
|
||||
GREEN command:
|
||||
|
||||
```text
|
||||
cd backend && npx vitest run test/sse-hub.test.ts test/sse-route.test.ts
|
||||
```
|
||||
|
||||
GREEN output (exit 0):
|
||||
|
||||
```text
|
||||
✓ test/sse-hub.test.ts (4 tests)
|
||||
✓ test/sse-route.test.ts (3 tests)
|
||||
Test Files 2 passed (2)
|
||||
Tests 7 passed (7)
|
||||
```
|
||||
|
||||
### 3. Frontend cursor tracking and gate idempotence
|
||||
|
||||
RED command:
|
||||
|
||||
```text
|
||||
cd frontend && npx vitest run src/stream/useSessionStream.test.tsx src/store/sessionStore.test.ts
|
||||
```
|
||||
|
||||
RED output (exit 1):
|
||||
|
||||
```text
|
||||
Test Files 2 failed (2)
|
||||
Tests 2 failed | 28 passed (30)
|
||||
|
||||
expected /sessions/s1/events to be /sessions/s1/events?lastEventId=7
|
||||
expected duplicate gate pendingWidget to remain null, received gate-1
|
||||
```
|
||||
|
||||
GREEN command:
|
||||
|
||||
```text
|
||||
cd frontend && npx vitest run src/stream/useSessionStream.test.tsx src/store/sessionStore.test.ts
|
||||
```
|
||||
|
||||
GREEN output (exit 0):
|
||||
|
||||
```text
|
||||
✓ src/store/sessionStore.test.ts (23 tests)
|
||||
✓ src/stream/useSessionStream.test.tsx (7 tests)
|
||||
Test Files 2 passed (2)
|
||||
Tests 30 passed (30)
|
||||
```
|
||||
|
||||
### 4. Frontend typed Resume and AppShell ordering/preservation
|
||||
|
||||
Typed API RED command:
|
||||
|
||||
```text
|
||||
cd frontend && npx tsc -b
|
||||
```
|
||||
|
||||
Typed API RED output (exit 1):
|
||||
|
||||
```text
|
||||
src/api/sessions.test.ts(43,9): error TS2322: Type 'void' is not assignable to type
|
||||
'{ id: string; alreadyActive: boolean; }'.
|
||||
```
|
||||
|
||||
Lifecycle RED command:
|
||||
|
||||
```text
|
||||
cd frontend && npx vitest run src/api/sessions.test.ts src/shell/AppShell.session-mgmt.test.tsx
|
||||
```
|
||||
|
||||
Lifecycle RED output (exit 1):
|
||||
|
||||
```text
|
||||
✓ src/api/sessions.test.ts (9 tests)
|
||||
❯ src/shell/AppShell.session-mgmt.test.tsx (15 tests | 4 failed)
|
||||
Test Files 1 failed | 1 passed (2)
|
||||
Tests 4 failed | 20 passed (24)
|
||||
|
||||
already-active same-session Resume created two EventSources instead of one
|
||||
deferred cold Resume closed the document panel before POST completion
|
||||
failed same-session Resume closed the prior EventSource
|
||||
failed Resume with no active session opened an EventSource
|
||||
```
|
||||
|
||||
GREEN commands:
|
||||
|
||||
```text
|
||||
cd frontend && npx vitest run src/api/sessions.test.ts src/shell/AppShell.session-mgmt.test.tsx
|
||||
cd frontend && npx tsc -b
|
||||
```
|
||||
|
||||
GREEN output (exit 0):
|
||||
|
||||
```text
|
||||
✓ src/api/sessions.test.ts (9 tests)
|
||||
✓ src/shell/AppShell.session-mgmt.test.tsx (15 tests)
|
||||
Test Files 2 passed (2)
|
||||
Tests 24 passed (24)
|
||||
TypeScript: no output, exit 0
|
||||
```
|
||||
|
||||
The AppShell cold-reconnect test additionally proves that the old source accepts an event while
|
||||
Resume is pending, the replacement URL carries `lastEventId=8`, the replacement receives one
|
||||
post-resume transcript/activity row, and two deliveries of the same descriptor id yield one gate.
|
||||
|
||||
## Affected verification
|
||||
|
||||
Backend command:
|
||||
|
||||
```text
|
||||
cd backend && npx vitest run test/routes-sessions.test.ts test/session-bridge.test.ts \
|
||||
test/sse-hub.test.ts test/sse-route.test.ts test/health.test.ts test/e2e-f1.test.ts
|
||||
```
|
||||
|
||||
Output (exit 0):
|
||||
|
||||
```text
|
||||
Test Files 6 passed (6)
|
||||
Tests 52 passed (52)
|
||||
```
|
||||
|
||||
Backend typecheck:
|
||||
|
||||
```text
|
||||
cd backend && npx tsc --noEmit -p .
|
||||
```
|
||||
|
||||
Output: no output, exit 0.
|
||||
|
||||
Frontend command:
|
||||
|
||||
```text
|
||||
cd frontend && npx vitest run src/api/sessions.test.ts src/store/sessionStore.test.ts \
|
||||
src/stream/useSessionStream.test.tsx src/shell/AppShell.session-mgmt.test.tsx \
|
||||
src/shell/CentralStatus.test.tsx src/shell/ModelActivityPanel.test.tsx \
|
||||
src/shell/f1-loop.test.tsx src/shell/AppShell.new-session.test.tsx
|
||||
```
|
||||
|
||||
Output (exit 0):
|
||||
|
||||
```text
|
||||
Test Files 8 passed (8)
|
||||
Tests 74 passed (74)
|
||||
```
|
||||
|
||||
Frontend typecheck:
|
||||
|
||||
```text
|
||||
cd frontend && npx tsc -b
|
||||
```
|
||||
|
||||
Output: no output, exit 0.
|
||||
|
||||
## Full verification
|
||||
|
||||
Backend full suite:
|
||||
|
||||
```text
|
||||
cd backend && npx vitest run
|
||||
```
|
||||
|
||||
```text
|
||||
Test Files 22 passed (22)
|
||||
Tests 177 passed (177)
|
||||
```
|
||||
|
||||
Frontend full suite:
|
||||
|
||||
```text
|
||||
cd frontend && npx vitest run
|
||||
```
|
||||
|
||||
```text
|
||||
Test Files 43 passed (43)
|
||||
Tests 271 passed (271)
|
||||
```
|
||||
|
||||
Backend production build:
|
||||
|
||||
```text
|
||||
cd backend && npm run build
|
||||
> tsc -p tsconfig.json
|
||||
exit 0
|
||||
```
|
||||
|
||||
Frontend production build:
|
||||
|
||||
```text
|
||||
cd frontend && npm run build
|
||||
> tsc -b && vite build
|
||||
✓ 4835 modules transformed.
|
||||
✓ built in 8.25s
|
||||
exit 0
|
||||
```
|
||||
|
||||
## Integrated re-review closure (2026-07-15)
|
||||
|
||||
This section supersedes the earlier cold same-session assertion that the replacement URL carries
|
||||
`lastEventId=8`. That behavior was correct only while the backend process and its in-memory id
|
||||
sequence survived. A restarted backend begins a fresh sequence, so a successful cold Resume now
|
||||
explicitly discards the browser's cursor before replacing the EventSource.
|
||||
|
||||
All four integrated re-review findings are closed:
|
||||
|
||||
1. `AppShell` passes a dedicated cursor-reset epoch to `useSessionStream`. A cold same-session
|
||||
Resume increments it only after `alreadyActive: false`; a high cursor such as `901` is omitted
|
||||
from the replacement URL and fresh low-id events/gates are consumed. An already-active
|
||||
same-session Resume still preserves its source, cursor, and store.
|
||||
2. `useSessionStream` no longer mutates the cursor ref during render. Effect setup resets cursor
|
||||
state on session/reset-epoch changes, callbacks are guarded by a captured active-source
|
||||
identity, and cleanup clears only its own active identity. A queued event from the replaced
|
||||
source cannot write the new store or poison its next reconnect URL.
|
||||
3. Backend Resume is serialized per session and rechecks runtime state inside the lock. Manifest,
|
||||
readiness, and reopen validation precede the transport commit. Idle/failed replacement creates
|
||||
and binds the new runtime before `hub.clear`, which occurs synchronously immediately before the
|
||||
first `Resuming session` publish. Reopen/create failure returns exactly
|
||||
`Session could not be resumed. Check configuration and connectivity, then try again.`, keeps the
|
||||
prior hub buffer/subscribers attached, and does not expose exception sentinels. Concurrent calls
|
||||
perform one cold start and the waiter returns `alreadyActive: true`.
|
||||
4. `SseHub.forget(id)` removes subscribers, buffered events, and the last id. Permanent session
|
||||
DELETE invokes it after disk deletion; ordinary close and Resume continue to use `clear`, which
|
||||
preserves the id sequence.
|
||||
|
||||
### Re-review files
|
||||
|
||||
Production:
|
||||
|
||||
- `backend/src/pi/pi-process-manager.ts`
|
||||
- `backend/src/routes/sessions.ts`
|
||||
- `backend/src/sse/sse-hub.ts`
|
||||
- `frontend/src/shell/AppShell.tsx`
|
||||
- `frontend/src/stream/useSessionStream.ts`
|
||||
|
||||
Tests/support:
|
||||
|
||||
- `backend/test/pi-process-manager.test.ts`
|
||||
- `backend/test/routes-sessions.test.ts`
|
||||
- `backend/test/sse-hub.test.ts`
|
||||
- `frontend/src/shell/AppShell.session-mgmt.test.tsx`
|
||||
- `frontend/src/stream/useSessionStream.test.tsx`
|
||||
- `frontend/src/test/fakeEventSource.ts`
|
||||
|
||||
### Re-review TDD RED/GREEN evidence
|
||||
|
||||
Frontend RED command:
|
||||
|
||||
```text
|
||||
cd frontend && npx vitest run src/stream/useSessionStream.test.tsx \
|
||||
src/shell/AppShell.session-mgmt.test.tsx
|
||||
```
|
||||
|
||||
RED output (exit 1):
|
||||
|
||||
```text
|
||||
Test Files 2 failed (2)
|
||||
Tests 3 failed | 21 passed (24)
|
||||
|
||||
reset epoch: expected the old source to close, received false
|
||||
cold same-session: expected /sessions/s1/events, received ?lastEventId=901
|
||||
stale source: expected an empty transcript, received "stale session one"
|
||||
```
|
||||
|
||||
Frontend GREEN command:
|
||||
|
||||
```text
|
||||
cd frontend && npx vitest run src/stream/useSessionStream.test.tsx \
|
||||
src/shell/AppShell.session-mgmt.test.tsx
|
||||
cd frontend && npx tsc -b
|
||||
```
|
||||
|
||||
GREEN output (exit 0):
|
||||
|
||||
```text
|
||||
Test Files 2 passed (2)
|
||||
Tests 24 passed (24)
|
||||
TypeScript: no output, exit 0
|
||||
```
|
||||
|
||||
Backend RED command:
|
||||
|
||||
```text
|
||||
cd backend && npx vitest run test/sse-hub.test.ts test/routes-sessions.test.ts
|
||||
```
|
||||
|
||||
RED output (exit 1):
|
||||
|
||||
```text
|
||||
Test Files 2 failed (2)
|
||||
Tests 8 failed | 28 passed (36)
|
||||
|
||||
three Resume ordering assertions observed clear before reopen/create
|
||||
reopen and create sentinels escaped as raw HTTP 500 responses
|
||||
the concurrent waiter cold-started again instead of returning alreadyActive: true
|
||||
SseHub.forget was absent and DELETE did not invoke permanent cleanup
|
||||
```
|
||||
|
||||
Backend GREEN command:
|
||||
|
||||
```text
|
||||
cd backend && npx vitest run test/sse-hub.test.ts test/routes-sessions.test.ts
|
||||
cd backend && npx tsc --noEmit -p .
|
||||
```
|
||||
|
||||
GREEN output (exit 0):
|
||||
|
||||
```text
|
||||
Test Files 2 passed (2)
|
||||
Tests 36 passed (36)
|
||||
TypeScript: no output, exit 0
|
||||
```
|
||||
|
||||
The failure tests publish a post-failure probe through the same hub and prove that a subscriber
|
||||
attached before either reopen or create rejection still receives it. The concurrency test overlaps
|
||||
two same-id requests behind a deferred reopen and proves one manifest/readiness/reopen/create/clear
|
||||
sequence.
|
||||
|
||||
### Initial re-review verification (before independent-review hardening)
|
||||
|
||||
```text
|
||||
cd backend && npx vitest run
|
||||
Test Files 22 passed (22)
|
||||
Tests 182 passed (182)
|
||||
|
||||
cd frontend && npx vitest run
|
||||
Test Files 43 passed (43)
|
||||
Tests 273 passed (273)
|
||||
|
||||
cd backend && npm run build
|
||||
> tsc -p tsconfig.json
|
||||
exit 0
|
||||
|
||||
cd frontend && npm run build
|
||||
> tsc -b && vite build
|
||||
✓ 4835 modules transformed.
|
||||
✓ built in 8.46s
|
||||
exit 0
|
||||
```
|
||||
|
||||
`git diff --check` produced no output (exit 0). The frontend build retains its pre-existing
|
||||
large-chunk warning; no new build or type errors were introduced.
|
||||
|
||||
Final whitespace verification:
|
||||
|
||||
```text
|
||||
git diff --check
|
||||
no output, exit 0
|
||||
```
|
||||
|
||||
## Self-review
|
||||
|
||||
- Resume sequencing: reopen and runtime binding precede backend `clear` and HTTP success; frontend
|
||||
state mutation and cursor-reset epoch follow it. Failure catch only emits fixed UI copy.
|
||||
- Already active: same-session returns before reset/generation/manifest repaint; different session
|
||||
resets the single-session store and binds the new id only after success.
|
||||
- SSE exact-once: ids are transport identity, not content hashes; replay is strictly `id > cursor`;
|
||||
`clear` retains the counter; pending gate matching uses only descriptor id.
|
||||
- Cursor behavior: hook tracks `MessageEvent.lastEventId`, carries it only to an ordinary same-id
|
||||
generation, and resets it on session-id/cold-runtime epoch change. Native EventSource reconnect
|
||||
remains supported by the route header.
|
||||
- Gate defense: the Zustand set survives pending clear but resets with the session store.
|
||||
- Client boundary: raw `ensure.error` is unused in public responses; generic Pi system events are
|
||||
reconstructed rather than spread; frontend type mirrors the two-field event.
|
||||
- Scope: `git diff` contains no CTE/Card/harness/workflow/persistence/model changes. Four pre-existing
|
||||
modified `.superpowers/sdd/{progress,task-2-report,task-3-report,task-4-report}.md` files are user
|
||||
work and are excluded from staging.
|
||||
|
||||
## Remaining concerns
|
||||
|
||||
- The 200-event SSE ring limit remains intentional. A brand-new page can reconstruct only retained
|
||||
backlog; an in-memory same-session reconnect is exact-once from its cursor.
|
||||
- Per-session sequence counters remain in backend memory after `clear` by design so later in-process
|
||||
cold same-id Resume cannot reuse ids. Permanent DELETE removes the counter via `forget`.
|
||||
- The Delete-then-Resume adversarial route test proves the deleted session is not resurrected but
|
||||
currently receives the runner's generic HTTP 500 when `sessionShow` can no longer find it. A
|
||||
future API cleanup can normalize that missing-session response to 404 or 409.
|
||||
- Frontend tests still print pre-existing MSW unhandled-request and React ref/`act` warnings even
|
||||
though all 276 tests pass. The frontend production build still reports pre-existing large chunk
|
||||
warnings. Neither warning class was introduced or expanded by this change.
|
||||
- No live Pi/DWH smoke was run; this wave changes only REST/SSE/frontend lifecycle boundaries and
|
||||
is covered by fake-Pi, live Fastify SSE, component, full-suite, typecheck, and production-build
|
||||
gates.
|
||||
|
||||
## Independent-review hardening
|
||||
|
||||
The required independent review was run repeatedly against the uncommitted diff. Its first pass
|
||||
found four Important lifecycle edges beyond the integrated findings: queued old-runtime callbacks,
|
||||
post-spawn construction cleanup, concurrent frontend Resume completions, and the passive-effect
|
||||
commit window. Its second pass confirmed those fixes and identified one remaining Important
|
||||
retention issue in the new runtime-identity map. The final pass reported no Critical, Important, or
|
||||
Minor findings and assessed the diff ready to merge.
|
||||
|
||||
The resulting hardening is:
|
||||
|
||||
- Runtime bridge callbacks are gated by the bound runtime identity. Replacement, close, and DELETE
|
||||
invalidate the old identity, so queued old events cannot publish or call `failSession`. An active
|
||||
runtime removed by the manager can still publish its complete public failure sequence; after the
|
||||
terminal unmanaged `agent_end`, its binding is released and later events are rejected.
|
||||
- `PiProcessManager` kills the spawned child and removes any registered map entry if either
|
||||
spawn-boundary stderr setup or later RPC/bridge/map initialization throws.
|
||||
- Resume completion compares against synchronously maintained current active-session identity.
|
||||
Concurrent `alreadyActive: false` then `alreadyActive: true` results preserve the cold source,
|
||||
cursor, store, and replayed gate.
|
||||
- Stream source replacement uses a layout effect. A deterministic later-layout-effect test delivers
|
||||
a queued old event inside the former commit-to-passive-cleanup window and proves it is ignored.
|
||||
- Cursor tests cover both a restarted backend's fresh low ids and an in-process hub's preserved high
|
||||
ids followed by a cursor-bearing ordinary reconnect.
|
||||
|
||||
### Hardening TDD RED/GREEN evidence
|
||||
|
||||
Backend identity/construction RED command:
|
||||
|
||||
```text
|
||||
cd backend && npx vitest run test/pi-process-manager.test.ts test/routes-sessions.test.ts
|
||||
```
|
||||
|
||||
```text
|
||||
Test Files 2 failed (2)
|
||||
Tests 3 failed | 69 passed (72)
|
||||
|
||||
post-spawn reader initialization did not kill the child
|
||||
replaced and deleted runtime callbacks still called failSession/published
|
||||
```
|
||||
|
||||
Additional spawn-boundary and terminal-release RED checks:
|
||||
|
||||
```text
|
||||
cd backend && npx vitest run test/pi-process-manager.test.ts \
|
||||
-t "spawn boundary initialization"
|
||||
Tests 1 failed | 38 skipped (39)
|
||||
|
||||
cd backend && npx vitest run test/routes-sessions.test.ts -t "terminal sequence"
|
||||
Tests 1 failed | 34 skipped (35)
|
||||
```
|
||||
|
||||
Frontend concurrency/layout RED command:
|
||||
|
||||
```text
|
||||
cd frontend && npx vitest run src/stream/useSessionStream.test.tsx \
|
||||
src/shell/AppShell.session-mgmt.test.tsx
|
||||
```
|
||||
|
||||
```text
|
||||
Test Files 2 failed (2)
|
||||
Tests 2 failed | 25 passed (27)
|
||||
|
||||
the later-layout-effect event wrote "commit-window stale text"
|
||||
the false→true completion pair erased pending gate "cold-gate"
|
||||
```
|
||||
|
||||
Final focused GREEN commands:
|
||||
|
||||
```text
|
||||
cd backend && npx vitest run test/pi-process-manager.test.ts \
|
||||
test/routes-sessions.test.ts test/sse-hub.test.ts
|
||||
cd backend && npx tsc --noEmit -p .
|
||||
|
||||
Test Files 3 passed (3)
|
||||
Tests 79 passed (79)
|
||||
TypeScript: no output, exit 0
|
||||
|
||||
cd frontend && npx vitest run src/stream/useSessionStream.test.tsx \
|
||||
src/shell/AppShell.session-mgmt.test.tsx
|
||||
cd frontend && npx tsc -b
|
||||
|
||||
Test Files 2 passed (2)
|
||||
Tests 27 passed (27)
|
||||
TypeScript: no output, exit 0
|
||||
```
|
||||
|
||||
### Final full verification after review hardening
|
||||
|
||||
```text
|
||||
cd backend && npx vitest run
|
||||
Test Files 22 passed (22)
|
||||
Tests 188 passed (188)
|
||||
|
||||
cd frontend && npx vitest run
|
||||
Test Files 43 passed (43)
|
||||
Tests 276 passed (276)
|
||||
|
||||
cd backend && npm run build
|
||||
> tsc -p tsconfig.json
|
||||
exit 0
|
||||
|
||||
cd frontend && npm run build
|
||||
> tsc -b && vite build
|
||||
✓ 4835 modules transformed.
|
||||
✓ built in 8.47s
|
||||
exit 0
|
||||
```
|
||||
|
||||
The final frontend run retains the repository's pre-existing MSW/ref/`act` warnings, and the build
|
||||
retains the pre-existing large-chunk warning. No test, typecheck, or build failures remain.
|
||||
|
||||
## Stale-bootstrap, lifecycle-lock, and competing-Resume hardening
|
||||
|
||||
Date: 2026-07-15
|
||||
Base: `08b1f4909e8eb7538156cecc2e7a6cafb46ddfc7`
|
||||
|
||||
This follow-up closes asynchronous identity/order and multi-client transport gaps found in the
|
||||
pre-deployment review:
|
||||
|
||||
- `PiProcessManager.teardownIfCurrent(id, runtime)` makes teardown an identity-checked operation.
|
||||
Bootstrap re-checks identity after configuration/retrieval and before both the public
|
||||
`Starting model` event and model start. Its failure continuation acquires the same session
|
||||
lifecycle lock, claims only its own runtime identity, and holds serialization through persisted
|
||||
failure and the public terminal sequence. A continuation left behind by Close or DELETE cannot
|
||||
target a replacement or recreate forgotten SSE state.
|
||||
- The former Resume-only promise tail is now a per-session lifecycle lock shared by Resume, Close,
|
||||
and DELETE. Each route reads the current runtime inside the lock immediately before replacement
|
||||
or removal and uses identity-checked teardown. Deferred route tests prove both orderings:
|
||||
Resume then Close/Delete finishes removed with no post-removal bootstrap event; Close then Resume
|
||||
creates only after Close completes; DELETE then Resume cannot recreate a deleted session.
|
||||
- AppShell assigns each Resume invocation a monotonic token and records the latest target. A
|
||||
completion for a different, superseding session id cannot reset the store, select a source, close
|
||||
the panel, or repaint phase from a late manifest. Same-id invocations are per-target single-flight
|
||||
operations through the POST and local binding commit: repeated pre-commit clicks update the
|
||||
shared operation's latest token but issue no second POST or commit path. The operation becomes
|
||||
joinable again before its manifest fetch, whose repaint remains token/id/selection guarded. Start
|
||||
new, Stop, streamed session exit, and active-session deletion invalidate pending Resume work.
|
||||
This prevents stale-source preservation and reverse/non-Resume intent overwrite without allowing
|
||||
a slow manifest to suppress a later explicit rebind.
|
||||
- `SseHub` subscriber registrations now carry idempotent transport-close callbacks. `clear` and
|
||||
`forget` snapshot and actively close every response before discarding runtime transport state;
|
||||
callback-driven unsubscription during that iteration is safe. The SSE route ends its response so
|
||||
native EventSource reconnects with `Last-Event-ID`. Post-clear events retain monotonic ids and are
|
||||
buffered for replay; `forget` additionally resets the id state.
|
||||
|
||||
Production files:
|
||||
|
||||
- `backend/src/pi/pi-process-manager.ts`
|
||||
- `backend/src/routes/sessions.ts`
|
||||
- `backend/src/sse/sse-hub.ts`
|
||||
- `frontend/src/shell/AppShell.tsx`
|
||||
|
||||
Regression tests:
|
||||
|
||||
- `backend/test/pi-process-manager.test.ts`
|
||||
- `backend/test/routes-sessions.test.ts`
|
||||
- `backend/test/sse-hub.test.ts`
|
||||
- `backend/test/sse-route.test.ts`
|
||||
- `frontend/src/shell/AppShell.session-mgmt.test.tsx`
|
||||
|
||||
### TDD RED/GREEN evidence
|
||||
|
||||
Runtime identity API RED:
|
||||
|
||||
```text
|
||||
cd backend && npx vitest run test/pi-process-manager.test.ts -t "identity-checked teardown"
|
||||
|
||||
Test Files 1 failed (1)
|
||||
Tests 1 failed | 39 skipped (40)
|
||||
TypeError: mgr.teardownIfCurrent is not a function
|
||||
```
|
||||
|
||||
Runtime identity API GREEN:
|
||||
|
||||
```text
|
||||
Test Files 1 passed (1)
|
||||
Tests 1 passed | 39 skipped (40)
|
||||
```
|
||||
|
||||
Deferred bootstrap RED:
|
||||
|
||||
```text
|
||||
cd backend && npx vitest run test/routes-sessions.test.ts \
|
||||
-t "stale bootstrap|bootstrap that"
|
||||
|
||||
Test Files 1 failed (1)
|
||||
Tests 6 failed | 35 skipped (41)
|
||||
|
||||
close/delete + replacement: stale continuation removed the replacement runtime
|
||||
delete without replacement: stale continuation called failSession after forget
|
||||
```
|
||||
|
||||
Deferred bootstrap GREEN:
|
||||
|
||||
```text
|
||||
Test Files 1 passed (1)
|
||||
Tests 6 passed | 35 skipped (41)
|
||||
```
|
||||
|
||||
Shared lifecycle ordering RED:
|
||||
|
||||
```text
|
||||
cd backend && npx vitest run test/routes-sessions.test.ts \
|
||||
-t "Resume followed|Close followed|Delete followed"
|
||||
|
||||
Test Files 1 failed (1)
|
||||
Tests 4 failed | 41 skipped (45)
|
||||
|
||||
All four deferred assertions observed the competing route settle before the first lifecycle
|
||||
operation released.
|
||||
```
|
||||
|
||||
Bootstrap plus lifecycle GREEN:
|
||||
|
||||
```text
|
||||
cd backend && npx vitest run test/routes-sessions.test.ts \
|
||||
-t "Resume followed|Close followed|Delete followed|stale bootstrap|bootstrap that"
|
||||
|
||||
Test Files 1 passed (1)
|
||||
Tests 10 passed | 35 skipped (45)
|
||||
```
|
||||
|
||||
Competing frontend Resume RED:
|
||||
|
||||
```text
|
||||
cd frontend && npx vitest run src/shell/AppShell.session-mgmt.test.tsx \
|
||||
-t "competing Resume|stale Resume manifest"
|
||||
|
||||
Test Files 1 failed (1)
|
||||
Tests 2 failed | 16 skipped (18)
|
||||
|
||||
reverse POST completion opened a second, stale EventSource
|
||||
late s1 manifest repainted the selected s3 phase from F3 to F7
|
||||
```
|
||||
|
||||
Competing and same-id Resume GREEN:
|
||||
|
||||
```text
|
||||
cd frontend && npx vitest run src/shell/AppShell.session-mgmt.test.tsx \
|
||||
-t "competing Resume|stale Resume manifest|false then true"
|
||||
|
||||
Test Files 1 passed (1)
|
||||
Tests 3 passed | 15 skipped (18)
|
||||
```
|
||||
|
||||
### Independent-review hardening RED/GREEN
|
||||
|
||||
The first final review reported no Critical findings and three Important edge cases: bootstrap
|
||||
could start during an in-progress Close; bootstrap-owned failure was persisted twice; and an older
|
||||
same-id result could overwrite newer state. The integrated reviewer also required non-Resume
|
||||
navigation to invalidate pending Resume work. The final main review tightened the same-ID contract
|
||||
to true single-flight so a second same-target click cannot preserve a dead pre-restart source.
|
||||
|
||||
Backend review RED:
|
||||
|
||||
```text
|
||||
cd backend && npx vitest run test/routes-sessions.test.ts \
|
||||
-t "Close suppresses|bootstrap failure persists once"
|
||||
|
||||
Test Files 1 failed (1)
|
||||
Tests 2 failed | 45 skipped (47)
|
||||
|
||||
deferred configure started Pi while closeSession was still pending
|
||||
bootstrap/public failure called failSession twice
|
||||
```
|
||||
|
||||
Backend review GREEN:
|
||||
|
||||
```text
|
||||
Test Files 1 passed (1)
|
||||
Tests 2 passed | 45 skipped (47)
|
||||
```
|
||||
|
||||
Same-id single-flight RED:
|
||||
|
||||
```text
|
||||
cd frontend && npx vitest run src/shell/AppShell.session-mgmt.test.tsx \
|
||||
-t "share one cold request"
|
||||
|
||||
Test Files 1 failed (1)
|
||||
Tests 1 failed | 18 skipped (19)
|
||||
|
||||
two concurrent same-ID invocations issued two cold POSTs (three total including initial activation)
|
||||
```
|
||||
|
||||
Non-Resume invalidation RED:
|
||||
|
||||
```text
|
||||
cd frontend && npx vitest run src/shell/AppShell.session-mgmt.test.tsx \
|
||||
-t "starting a new question invalidates"
|
||||
|
||||
Test Files 1 failed (1)
|
||||
Tests 1 failed | 19 skipped (20)
|
||||
|
||||
the late Resume opened an EventSource after Start new returned to the landing state
|
||||
```
|
||||
|
||||
Frontend review GREEN:
|
||||
|
||||
```text
|
||||
cd frontend && npx vitest run src/shell/AppShell.session-mgmt.test.tsx \
|
||||
-t "share one cold request|competing Resume|stale Resume manifest|starting a new question invalidates"
|
||||
|
||||
Test Files 1 passed (1)
|
||||
Tests 4 passed | 15 skipped (19)
|
||||
```
|
||||
|
||||
Post-commit single-flight lifetime RED:
|
||||
|
||||
```text
|
||||
cd frontend && npx vitest run src/shell/AppShell.session-mgmt.test.tsx \
|
||||
-t "releases same-id single-flight"
|
||||
|
||||
Test Files 1 failed (1)
|
||||
Tests 1 failed | 19 skipped (20)
|
||||
|
||||
s1 committed and waited on its manifest; after s3 superseded it, a new s1 Resume reused the old
|
||||
operation and issued no second s1 POST (expected 2, received 1).
|
||||
```
|
||||
|
||||
Same-id and manifest lifetime GREEN:
|
||||
|
||||
```text
|
||||
cd frontend && npx vitest run src/shell/AppShell.session-mgmt.test.tsx -t "same-id|manifest"
|
||||
Test Files 1 passed (1)
|
||||
Tests 4 passed | 16 skipped (20)
|
||||
|
||||
cd frontend && npx tsc -b
|
||||
no output, exit 0
|
||||
```
|
||||
|
||||
Multi-client SSE disconnect RED:
|
||||
|
||||
```text
|
||||
cd backend && npx vitest run test/sse-hub.test.ts test/sse-route.test.ts
|
||||
|
||||
Test Files 2 failed (2)
|
||||
Tests 3 failed | 6 passed (9)
|
||||
|
||||
clear/forget invoked zero of two registered close callbacks, and two live HTTP SSE responses timed
|
||||
out instead of reaching EOF after clear.
|
||||
```
|
||||
|
||||
Multi-client SSE disconnect GREEN:
|
||||
|
||||
```text
|
||||
cd backend && npx vitest run test/sse-hub.test.ts test/sse-route.test.ts
|
||||
Test Files 2 passed (2)
|
||||
Tests 9 passed (9)
|
||||
|
||||
cd backend && npx tsc --noEmit -p .
|
||||
no output, exit 0
|
||||
```
|
||||
|
||||
The Hub tests use two subscribers whose close callbacks immediately unsubscribe themselves, proving
|
||||
safe snapshot iteration and exactly-once closure. The live-route test opens two HTTP streams, proves
|
||||
both receive EOF on clear, publishes a new event and gate, then reconnects after id 1 and replays
|
||||
exactly ids 2 and 3. The forget test closes both subscribers and proves the next id resets to 1.
|
||||
|
||||
Close now removes the observed runtime identity before awaiting persistence. Failure persistence is
|
||||
claimed once per runtime and lifecycle-serialized; bootstrap's public `session_failed` cannot start
|
||||
a duplicate. A per-target in-flight map owns the only same-ID POST and commit while its mutable
|
||||
latest token keeps s1→s2→s1 ordering correct; it is removed immediately after the binding commit,
|
||||
before awaiting the independently guarded manifest. One shared invalidation helper is called when
|
||||
active deletion, streamed exit, Start new, or Stop begins.
|
||||
|
||||
### Focused verification
|
||||
|
||||
```text
|
||||
cd backend && npx vitest run test/routes-sessions.test.ts test/pi-process-manager.test.ts \
|
||||
test/sse-hub.test.ts test/sse-route.test.ts
|
||||
Test Files 4 passed (4)
|
||||
Tests 96 passed (96)
|
||||
|
||||
cd backend && npx tsc --noEmit -p .
|
||||
no output, exit 0
|
||||
|
||||
cd frontend && npx vitest run src/shell/AppShell.session-mgmt.test.tsx \
|
||||
src/shell/AppShell.new-session.test.tsx src/stream/useSessionStream.test.tsx
|
||||
Test Files 3 passed (3)
|
||||
Tests 36 passed (36)
|
||||
|
||||
cd frontend && npx tsc -b
|
||||
no output, exit 0
|
||||
```
|
||||
|
||||
### Full verification
|
||||
|
||||
```text
|
||||
cd backend && npx vitest run
|
||||
Test Files 22 passed (22)
|
||||
Tests 202 passed (202)
|
||||
|
||||
cd frontend && npx vitest run
|
||||
Test Files 43 passed (43)
|
||||
Tests 280 passed (280)
|
||||
|
||||
cd backend && npm run build
|
||||
> tsc -p tsconfig.json
|
||||
exit 0
|
||||
|
||||
cd frontend && npm run build
|
||||
> tsc -b && vite build
|
||||
✓ 4835 modules transformed.
|
||||
✓ built in 8.47s
|
||||
exit 0
|
||||
```
|
||||
|
||||
The frontend suite/build retain the previously documented MSW, React ref/`act`, experimental type
|
||||
stripping, and large-chunk warnings. No warning class was introduced by this wave. No harness,
|
||||
workflow, persistence, SQL/CTE viewer, model-selection, or deployment file changed. The four
|
||||
pre-existing modified `.superpowers/sdd/{progress,task-2-report,task-3-report,task-4-report}.md`
|
||||
files remain excluded from staging.
|
||||
|
||||
### Final independent-review verdict
|
||||
|
||||
After the multi-client transport fix, the independent reviewer reported no Critical, Important, or
|
||||
Minor findings. Its own focused verification passed 96 backend transport/lifecycle tests, 31
|
||||
frontend Resume/stream tests, both TypeScript checks, and `git diff --check`. Final assessment:
|
||||
**Ready to deploy: Yes.**
|
||||
@@ -1,14 +0,0 @@
|
||||
# Portable deployment SDD progress
|
||||
|
||||
Plan: `docs/superpowers/plans/2026-07-11-adapter-foundations.md`
|
||||
Branch: `codex/portable-deployment`
|
||||
Worktree: `/Users/mp/projects/ThothII/.worktrees/portable-deployment`
|
||||
|
||||
Task 1: complete (commits e02e61e..a4eb6cc, review clean)
|
||||
Task 1 final-review follow-up: public exports and frozen capability records now have explicit regressions.
|
||||
Task 2: complete (commits a4eb6cc..f6302b3, review clean after authorized contract correction)
|
||||
Task 3: complete (commits f6302b3..fe8d70d, review clean after authorized write-envelope correction)
|
||||
Task 4: complete (commits fe8d70d..1e0911b, review clean)
|
||||
Task 4 final-review follow-up: a real `tht` subprocess now proves exactly one legacy warning on stderr and pristine JSON stdout.
|
||||
Task 5: complete (commits 1e0911b..dbbab6d, review clean after two fix waves)
|
||||
Final adapter review fix wave: complete (`fix(adapter): close final foundation review`). HTTP vector reader/writer endpoints are independently optional; writer-only targeted memory/solved writes are supported. Vector health reports each side separately plus configured/observed embedding dimensions. `build_vector_loader` remains an explicitly tracked bulk-sync-only exception scheduled for the local pgvector migration plan; it is not used by interactive/targeted writes.
|
||||
@@ -1,177 +0,0 @@
|
||||
# Task 2 report — root Compose startup
|
||||
|
||||
Status: DONE
|
||||
|
||||
Implemented the root Compose defaults and the single bundle declaration:
|
||||
|
||||
- added `.env.example` with automatic Compose defaults (`COMPOSE_FILE=compose.yaml`, an empty
|
||||
profile, and the relative `THT_SECRETS_FILE` path);
|
||||
- removed the mandatory `external` profile from `core` and `frontend`;
|
||||
- mounted `deploy/secrets/thothii.secrets` at `/run/secrets/thothii.secrets` and passed only the
|
||||
mounted path into the core container;
|
||||
- changed the production overlay to inherit that bundle instead of declaring per-secret mounts;
|
||||
- removed the local overlay's legacy `env_file` dependency;
|
||||
- added the versioned bundle template and `.gitignore` exception;
|
||||
- updated deployment security checks and added `scripts/test-default-compose.sh`.
|
||||
|
||||
Focused verification:
|
||||
|
||||
```text
|
||||
./scripts/test-default-compose.sh # default Compose contract passed.
|
||||
./scripts/test-container-deployment.sh # container deployment security contract passed.
|
||||
./scripts/test-preprocess-compose-config.sh # preprocess compose config: ok
|
||||
docker compose --env-file .env.example config --quiet
|
||||
(with a temporary mode-0600 bundle via THT_SECRETS_FILE)
|
||||
git diff --check
|
||||
```
|
||||
|
||||
The local-vector and preprocess service secret declarations remain for Task 3, which converts
|
||||
those services to the same bundle helper. Documentation and smoke command migration is reserved
|
||||
for Task 4.
|
||||
|
||||
---
|
||||
|
||||
# Task 2 report — PostgreSQL session repository
|
||||
|
||||
## Scope delivered
|
||||
|
||||
- Added `PostgresSessionRepository`, implementing the Task 1 repository contract with a
|
||||
direct PostgreSQL SQLAlchemy connection, transaction-local RLS context, UUIDv4 validation,
|
||||
current artifacts (including `cte_sql:<name>`), append-only decisions, preferences, and
|
||||
content-free deletion tombstones.
|
||||
- Added `tht session migrate --database-url URL [--status] --json` and a checksum-protected,
|
||||
advisory-transaction-locked migration runner.
|
||||
- Added server session configuration selection. `session_storage.connection` uses direct
|
||||
PostgreSQL TLS modes `verify-ca` or `verify-full`; it does not use PostgREST.
|
||||
- Updated packaging and `.gitignore` so session migrations are present in the built wheel.
|
||||
- Did not alter Task 3 workflow commands, Pi gate code, or backend code.
|
||||
|
||||
## TDD evidence
|
||||
|
||||
### RED
|
||||
|
||||
Command:
|
||||
|
||||
```sh
|
||||
cd harness && .venv/bin/pytest tests/test_postgres_session_repository.py tests/test_session_migrate_cmd.py -q
|
||||
```
|
||||
|
||||
Result before production implementation: `1 failed, 4 errors in 3.89s`.
|
||||
|
||||
- Four setup errors were `ModuleNotFoundError: No module named
|
||||
'tht.session.postgres_repository'`.
|
||||
- The migration CLI test failed because `tht session migrate` did not exist (`No such command
|
||||
'migrate'`).
|
||||
|
||||
### GREEN
|
||||
|
||||
Initial focused suite after implementation: `5 passed in 4.18s`.
|
||||
|
||||
Final focused verification:
|
||||
|
||||
```sh
|
||||
cd harness && .venv/bin/pytest \
|
||||
tests/test_session_repository.py \
|
||||
tests/test_postgres_session_repository.py \
|
||||
tests/test_session_migrate_cmd.py \
|
||||
tests/test_vector_migration_packaging.py -q
|
||||
```
|
||||
|
||||
Result: `12 passed in 5.80s`.
|
||||
|
||||
Changed-file lint verification:
|
||||
|
||||
```sh
|
||||
cd harness && .venv/bin/ruff check \
|
||||
tht/session/postgres_repository.py tht/migrations/sessions tht/config.py \
|
||||
tht/session/repository.py tht/cli/session_cmd.py \
|
||||
tests/test_postgres_session_repository.py tests/test_session_migrate_cmd.py \
|
||||
tests/test_vector_migration_packaging.py
|
||||
```
|
||||
|
||||
Result: `All checks passed!`.
|
||||
|
||||
## Migration and role policy choices
|
||||
|
||||
`001_schema.sql` creates only private `thoth_sessions` tables:
|
||||
|
||||
- `principals` and `principal_preferences`;
|
||||
- `sessions`, with `session_artifacts` and `review_decisions` cascading on session deletion;
|
||||
- `audit_log`, which deliberately has no content/detail/metadata column and keeps only action,
|
||||
session UUID, actor identity, owner identity, and timestamp.
|
||||
|
||||
`002_security.sql` creates separate `thoth_sessions_runtime` and
|
||||
`thoth_sessions_migrator` group roles, explicitly `NOLOGIN NOBYPASSRLS NOSUPERUSER`, revokes
|
||||
public access, gives the runtime role only the operations required by the adapter, and enables
|
||||
and forces RLS on every table. Owner/admin policies read only transaction-local settings:
|
||||
`thoth_sessions.actor_issuer`, `thoth_sessions.actor_subject`, and
|
||||
`thoth_sessions.is_admin`. The adapter starts every operation in a transaction, switches to the
|
||||
restricted runtime role, sets those settings with `set_config(..., true)`, and uses advisory
|
||||
transaction locks for migrations and per-session mutations.
|
||||
|
||||
The runtime role remains a `NOLOGIN` group role by design. Deployment must provision a dedicated
|
||||
non-superuser LOGIN role and grant it membership, for example:
|
||||
|
||||
```sql
|
||||
CREATE ROLE thoth_sessions_app LOGIN NOINHERIT PASSWORD '<secret>';
|
||||
GRANT thoth_sessions_runtime TO thoth_sessions_app;
|
||||
```
|
||||
|
||||
This avoids embedding an environment-specific login name or credential in versioned SQL. The
|
||||
new integration test proves that this non-superuser membership path can create and read a
|
||||
session while the adapter executes as `thoth_sessions_runtime`.
|
||||
|
||||
## Security/self-review
|
||||
|
||||
- Owner isolation and admin cross-owner reads run against disposable PostgreSQL containers,
|
||||
not Supabase.
|
||||
- No table or column includes `embedding`; repository code imports no embedding/vector code;
|
||||
the regression test writes a session artifact under a monkeypatched embedding sentinel.
|
||||
- An unauthorized owner receives the same `SessionError` as an absent session, preserving the
|
||||
future backend's 404 mapping boundary.
|
||||
- The audit row is inserted before deleting the parent session, so cascades remove all artifact
|
||||
and decision content while the tombstone survives.
|
||||
- A security review found and this task fixed the initial `.gitignore` rule that would have
|
||||
excluded `migrations/sessions/*.sql` from Git/wheels. The wheel test now asserts both session
|
||||
migration files and checks both the existing vector CLI and the new session CLI.
|
||||
- The review also highlighted runtime login provisioning. It is covered by a non-superuser
|
||||
regression test and documented above; concrete credential/role deployment belongs to Task 7.
|
||||
|
||||
## Remaining concerns
|
||||
|
||||
- Full `harness/.venv/bin/pytest -q` could not complete in this execution environment: the
|
||||
runner terminated the command after roughly 30 seconds. Captured output reached 44% with no
|
||||
failures before termination; `pgrep` confirmed no pytest process remained. The Task 2 focused
|
||||
suites above completed successfully.
|
||||
- `harness/.venv/bin/ruff check .` currently reports 34 pre-existing violations in unrelated
|
||||
test files (for example unused imports in `tests/l0/test_db_connection.py` and semicolon style
|
||||
in `tests/test_phase_effective.py`). The changed-file Ruff command is clean.
|
||||
- Task 7 must safely provision the dedicated runtime login/membership and inject its TLS
|
||||
credentials/CA; this task intentionally does not create a deployment-specific LOGIN role or
|
||||
password.
|
||||
|
||||
## Review follow-up — unavailable migration database JSON contract
|
||||
|
||||
### RED
|
||||
|
||||
Command:
|
||||
|
||||
```sh
|
||||
cd harness && .venv/bin/pytest \
|
||||
tests/test_session_migrate_cmd.py::test_session_migrate_status_database_failure_is_pristine_json -q
|
||||
```
|
||||
|
||||
Result: `1 failed in 0.46s`. The unreachable direct PostgreSQL URL exited with code 1 but left
|
||||
stdout empty, so `json.loads(result.stdout)` raised `JSONDecodeError`.
|
||||
|
||||
### GREEN
|
||||
|
||||
The session migration CLI now catches `SQLAlchemyError` at the same command boundary as its
|
||||
migration/domain errors and emits only `{"error": ...}` on stdout for `--json`.
|
||||
|
||||
```sh
|
||||
cd harness && .venv/bin/pytest tests/test_session_migrate_cmd.py -q
|
||||
cd harness && .venv/bin/ruff check tht/cli/session_cmd.py tests/test_session_migrate_cmd.py
|
||||
```
|
||||
|
||||
Result: `2 passed in 3.60s`; Ruff: `All checks passed!`.
|
||||
@@ -1,56 +0,0 @@
|
||||
# Task 3 report — workflow repository migration
|
||||
|
||||
## RED
|
||||
|
||||
- `harness/tests/test_session_repository_workflow.py` initially failed at collection:
|
||||
`persist_verified_finalization` did not exist.
|
||||
- The new gate test initially failed because `write_cte_sql` and `write_final_sql`
|
||||
were not registered. Its first run also exposed the worktree-local missing
|
||||
Node dependency (`typebox`); `npm ci` installed the lockfile dependency.
|
||||
- After the principal/legacy policy was clarified, the resolver tests initially
|
||||
failed because `resolve_principal` did not exist.
|
||||
|
||||
## GREEN evidence
|
||||
|
||||
- Focused Python regression set: `66 passed`:
|
||||
`test_session_repository_workflow`, `test_session_repository`, session mutation/list/
|
||||
documents/schema-linking, CTE plan/next, decision phase gate, and phase requirement tests.
|
||||
- Gate suite: `127 passed`, including
|
||||
`session-repository-writes.test.js`.
|
||||
- Changed-source Ruff checks pass. `git diff --check` passes.
|
||||
|
||||
## Implemented boundary
|
||||
|
||||
- Added `resolve_principal`: PostgreSQL session storage requires trusted
|
||||
`THT_PRINCIPAL_ISSUER` and `THT_PRINCIPAL_SUBJECT`, optional display name, and
|
||||
strict admin parsing (`1`/`true`). It fails closed and never substitutes a local
|
||||
identity. Filesystem storage uses `local_principal()`.
|
||||
- Filesystem repository creates UUIDv4 sessions only and permits safe historical
|
||||
timestamp IDs (`YYYY-MM-DD-HHMMSS`) for read/mutate compatibility. PostgreSQL
|
||||
remains UUIDv4 only.
|
||||
- Phase helpers fold `SessionSnapshot` ledger/artifacts; decision, phase, CTE,
|
||||
session mutation/list/document paths, retrieval-pack persistence, SQL promotion
|
||||
lookup, and task-doc/CTE test helpers gained repository/snapshot paths.
|
||||
- Finalization now publishes report, evidence, and finalized manifest through
|
||||
`repository.finalize`: one PostgreSQL transaction; filesystem writes artifacts
|
||||
before the finalized manifest commit marker. Solved-question indexing stays
|
||||
best-effort after this durable write.
|
||||
- Added `tht cte save --session --name --file -` and
|
||||
`tht sql set-final --session --file -`; Pi tools and SKILL.md now use them.
|
||||
|
||||
## Outstanding in-scope migration work
|
||||
|
||||
Do not treat this task as complete yet. Remaining direct session path consumers are:
|
||||
|
||||
- `harness/tht/cli/memory_cmd.py`: lines 60, 93, 165, 400, 458.
|
||||
- `harness/tht/cli/sql_cmd.py`: `_session_sql_file` at line 254 remains a legacy
|
||||
Path-returning bridge for preview/save/export.
|
||||
- `harness/tht/cli/session_cmd.py:session_dir` remains only as a compatibility
|
||||
bridge for the out-of-scope datamart command and the still-unmigrated memory/
|
||||
SQL consumers; workflow mutations in session_cmd do not call it.
|
||||
|
||||
The full Python suite has not been conclusively re-run to completion after the
|
||||
latest changes. An earlier root-directory invocation failed only because a
|
||||
pre-existing test expects `workflow.yaml` relative to `harness/`. Full gate tests
|
||||
are green. Full-repo Ruff currently fails on pre-existing test-file lint findings;
|
||||
changed-source Ruff passes.
|
||||
@@ -1,55 +0,0 @@
|
||||
# Task 4 report — one-command Docker documentation
|
||||
|
||||
## Status
|
||||
|
||||
Implemented. The installation documentation now uses the canonical flow:
|
||||
|
||||
```sh
|
||||
cp .env.example .env
|
||||
cp deploy/secrets/thothii.secrets.example deploy/secrets/thothii.secrets
|
||||
chmod 600 deploy/secrets/thothii.secrets
|
||||
docker compose up --build -d
|
||||
```
|
||||
|
||||
Updated:
|
||||
|
||||
- `README.md` with root `.env` defaults, one bundle, optional overlay presets, CA limitation,
|
||||
preprocessing, and migration notes.
|
||||
- `docs/installazione-docker-4-contesti.md` rewritten with exact files to create/edit and the
|
||||
four requested contexts (co-located DB/vector, Mac, Windows, and remote DB/Evidence server).
|
||||
- `docs/index.md` link text for the one-command installation.
|
||||
- `deploy/secrets/README.md` bundle syntax, permissions, runtime mount verification, CA handling,
|
||||
and migration guidance.
|
||||
- `scripts/docker-smoke.sh` now creates a disposable mode-0600 bundle and exercises the default
|
||||
Compose services without the legacy `external` profile.
|
||||
- `scripts/test-default-compose.sh` asserts the exact installation command, tracked templates,
|
||||
and absence of the legacy setup in the guide.
|
||||
- `scripts/test-container-deployment.sh` now validates the bundle mount and rejects legacy
|
||||
per-secret references; `.dockerignore` explicitly re-includes only the required vector policy
|
||||
helper so the Docker build context remains safe.
|
||||
- The Mac/Windows/local-vector and remote-server snippets now include required DWH/database and
|
||||
Evidence-root settings. `deploy/env.example` is explicitly deprecated and no longer selects a
|
||||
different Compose overlay.
|
||||
|
||||
The docs explicitly state that a PEM CA chain cannot be put in the strict single-line bundle. A
|
||||
reviewed Compose override/secret-manager mount is required for `THT_SSL_CA`. Direct PostgreSQL
|
||||
workspace examples are marked as advanced and require a separate reviewed runtime password mount;
|
||||
the base bundle mount is the only default mount.
|
||||
|
||||
## Verification
|
||||
|
||||
- `sh -n scripts/docker-smoke.sh scripts/test-default-compose.sh` — passed.
|
||||
- `./scripts/test-default-compose.sh` — passed.
|
||||
- `./scripts/test-container-deployment.sh` — passed after migrating its local-vector assertions
|
||||
to the single bundle and checking the `.dockerignore` deployment allowlist.
|
||||
- `git diff --check` — passed.
|
||||
- `./scripts/test-docker-smoke.sh` — passed after updating its static assertion to the default
|
||||
no-profile invocation.
|
||||
- `docker buildx build --file docker/core.Dockerfile --check .` — passed; BuildKit reported no
|
||||
warnings after the `.dockerignore` parent-directory fix.
|
||||
|
||||
## Concerns
|
||||
|
||||
The legacy `scripts/vector-rotate-bootstrap-password.sh` maintenance helper still accepts
|
||||
old/new standalone files. Its output is intentionally documented as a transitional interface;
|
||||
the resulting value must be copied into the bundle before restarting local-vector services.
|
||||
@@ -1,71 +0,0 @@
|
||||
# Task 5 report — backend principal enforcement
|
||||
|
||||
## RED
|
||||
|
||||
Added backend route/auth tests before implementation. The initial focused run failed in
|
||||
seven new assertions: `getPrincipal` did not exist, upstream requests still required the
|
||||
legacy identity header, foreign session/SSE routes were not hidden, admin scope was not
|
||||
enforced, new sessions had no trusted principal binding, and settings were global.
|
||||
|
||||
## GREEN
|
||||
|
||||
- Focused backend suite: `66 passed` across auth, sessions, SSE, and settings tests.
|
||||
- Complete backend Vitest suite: `209 passed` across `22` files.
|
||||
- `npx tsc --noEmit -p .`, `npm run build`, `git diff --check`, and changed Python
|
||||
source Ruff all exit successfully.
|
||||
- Harness targeted repository/local/migration tests and Python bytecode compilation exit
|
||||
successfully. The new `tht session preferences get|set` commands are registered and
|
||||
expose the expected Typer help. A direct local CLI preference smoke was not run because
|
||||
the checked-in local workspace requires unavailable `THT_DB_HOST` configuration.
|
||||
|
||||
## Route and child-process coverage
|
||||
|
||||
- `GET /me` returns the request `PrincipalContext`; upstream accepts only the portal's
|
||||
normalized `X-Thoth-*` identity tuple, with the legacy header ignored. Local mode uses
|
||||
the same stable `THT_HOME`/`~/.thothii/identity.json` UUID contract as the harness.
|
||||
- All session operations are principal-scoped: list (`mine` and admin-only `all`), show,
|
||||
create, resume, close, delete, rename, group, archive, unarchive, documents, reviewer
|
||||
response, steer, SQL preview/export, and SSE. Missing and foreign sessions are 404;
|
||||
absent upstream identity is 401. SSE is authorized before response headers or hub
|
||||
subscription, so a rejected request cannot attach to a live stream.
|
||||
- New/resumed Pi runtimes and every route-spawned `tht` process receive
|
||||
`THT_PRINCIPAL_ISSUER`, `THT_PRINCIPAL_SUBJECT`, optional display name, and admin flag.
|
||||
The readiness `tht` child is also principal-bound.
|
||||
- Settings use asynchronous repository-backed `tht session preferences get|set` in the
|
||||
production runner, which isolates preferences by principal. The legacy settings file is
|
||||
retained only as an injected-runner compatibility fallback for existing isolated tests.
|
||||
- Repository/settings authorization failures map to 503 before model startup. SQL execution
|
||||
errors remain 500 after authorization, preserving the prior API distinction.
|
||||
|
||||
## Self-review and concerns
|
||||
|
||||
- Confirmed the Task 4 portal emits lowercase `true`/`false` for the admin header; the
|
||||
parser accepts that exact normalized form plus the repository's existing `1`/`0`
|
||||
compatibility form, and rejects all other values.
|
||||
- The harness principal resolver is the ownership authority; the backend never accepts an
|
||||
owner supplied in request bodies. Its route guards use a repository-scoped `session show`
|
||||
before every session resource operation.
|
||||
- Existing dependency-injected route fakes without `sessionShow` retain a narrow test seam;
|
||||
production `ThtRunner` always has that method, so deployed requests cannot bypass the
|
||||
repository authorization check.
|
||||
|
||||
## Review follow-up
|
||||
|
||||
### RED
|
||||
|
||||
Focused regressions initially failed exactly at the three review findings: stale ambient
|
||||
display names survived into both `tht` and Pi child environments; mutation/document runner
|
||||
methods dropped the selected workspace; and `expandLocalHome` did not exist.
|
||||
|
||||
### GREEN
|
||||
|
||||
- Child environments now remove all four `THT_PRINCIPAL_*` keys from their cloned base
|
||||
environment before applying the exact request principal. Regression tests prove an absent
|
||||
display name does not inherit a stale ambient value in either child path.
|
||||
- `setName`, `setGroup`, `archive`, `unarchive`, and `documents` now take and retain an
|
||||
optional workspace. The rename route regression proves `session show` authorization and
|
||||
the mutation use the same non-default workspace.
|
||||
- Local principal paths expand `~`/`~/...`; existing local home and identity file modes are
|
||||
repaired to POSIX `0700`/`0600` when applicable, with Windows left unchanged.
|
||||
- Focused suite: `74 passed`; full backend suite: `213 passed` across `22` files, followed by
|
||||
TypeScript typecheck, production build, and diff check.
|
||||
@@ -1,65 +0,0 @@
|
||||
# Task 6 — Frontend identity and administrator UX report
|
||||
|
||||
## RED
|
||||
|
||||
- Added API tests for the `/me` principal call and `mine`/`all` session-list scopes.
|
||||
- Added component tests for regular-user scope, admin scope switching, owner labels,
|
||||
administrator banner, foreign-owner delete confirmation, and foreign-owner archive
|
||||
confirmation.
|
||||
- Initial focused run: 7 expected failures (missing `getMe`, missing scope query,
|
||||
missing owner label/admin controls, and missing foreign-action confirmation).
|
||||
- The archive-confirmation regression was also run separately before its implementation
|
||||
and failed because `window.confirm` was not called.
|
||||
|
||||
## GREEN
|
||||
|
||||
- `npx vitest run src/api/sessions.test.ts src/shell/NavSessions.test.tsx src/shell/AppShell.session-mgmt.test.tsx`
|
||||
— passed (47 tests before the archive follow-up; the focused archive regression then passed).
|
||||
- `npm test` — passed: 44 files / 305 tests.
|
||||
- `npx tsc -b` — passed.
|
||||
- `npm run build` — passed.
|
||||
- `git diff --check` — passed.
|
||||
- `npm run e2e` reached Playwright but could not run: the environment has no Chromium
|
||||
executable at Playwright's configured cache path. No application test failure was reported.
|
||||
|
||||
## Files changed
|
||||
|
||||
- `frontend/src/api/types.ts`: typed principal and session scope contracts.
|
||||
- `frontend/src/api/sessions.ts`: typed `/me` API call; scoped listing defaults to `mine`.
|
||||
- `frontend/src/shell/AppShell.tsx`: identity query, admin-only session scope selector and
|
||||
banner, owner-aware destructive action confirmations.
|
||||
- `frontend/src/shell/NavSessions.tsx`: owner labels in the all-sessions view.
|
||||
- `frontend/src/api/sessions.test.ts`, `frontend/src/shell/NavSessions.test.tsx`, and
|
||||
`frontend/src/shell/AppShell.session-mgmt.test.tsx`: contract and UX coverage.
|
||||
|
||||
## Self-review
|
||||
|
||||
- Regular users remain fail-closed on `mine`; no administrator control renders without
|
||||
`principal.isAdmin`.
|
||||
- The all-sessions view includes owner labels (including `Unknown` for legacy records).
|
||||
- Delete confirmation preserves the pre-existing select-all behavior and adds confirmation
|
||||
for foreign/unknown owners. Foreign archive now also requires an explicit browser
|
||||
confirmation; existing Stop & save already has its confirmation dialog.
|
||||
- A read-only review found no critical, important, or minor issues. The archive guard was
|
||||
added after that review in response to the requirement to cover every destructive rail
|
||||
action, and has its own RED/GREEN regression plus the final full verification above.
|
||||
|
||||
## Concerns
|
||||
|
||||
- E2E remains environment-blocked until the Playwright Chromium browser is installed.
|
||||
- Existing Vitest runs emit pre-existing MSW unmatched-request and dialog-ref warnings; all
|
||||
assertions pass and this task does not modify those shared test/UI primitives.
|
||||
|
||||
## Review remediation
|
||||
|
||||
- A post-commit review correctly identified that matching `displayName` must never establish
|
||||
ownership. The predicate now skips confirmation only when `session.author` exactly equals
|
||||
`principal.subject`; all display-name matches and missing authors are conservative
|
||||
cross-owner actions.
|
||||
- Added RED/GREEN regressions where two principals share display name `Alice` but have distinct
|
||||
subjects: both delete (with another session present, so select-all cannot mask the guard) and
|
||||
archive require confirmation.
|
||||
- Added `aria-pressed` to the My sessions / All sessions controls and asserts their selected state
|
||||
before and after switching.
|
||||
- Remediation verification: focused regressions passed; full frontend Vitest (44 files / 305
|
||||
tests), `npx tsc -b`, `npm run build`, and `git diff --check` all passed.
|
||||
@@ -1,79 +0,0 @@
|
||||
# Task 7 report — deployment contract and user-owned-session cutover
|
||||
|
||||
## Scope
|
||||
|
||||
Implemented the deployment contract only. No Supabase migration, portal change, live-stack
|
||||
restart, session archive, or deletion was run.
|
||||
|
||||
- `backend/src/config.ts` now makes the session-store deployment mode explicit. `local` is the
|
||||
default and cannot be publicly exposed. `postgres` requires `AUTH_MODE=upstream`, direct DB
|
||||
host/name/runtime user, an absolute runtime-password file, `verify-ca` or `verify-full`, and an
|
||||
absolute CA path.
|
||||
- `docker-compose.dev.yml` now publishes only loopback ports and explicitly selects local
|
||||
session storage rooted at `/data/local-home`.
|
||||
- `deploy/compose.session-server.yaml.example` separates the runtime and one-shot migrator
|
||||
secrets. The core gets only `session_runtime_password` and the CA; the profile-gated
|
||||
`session-migrate` service gets only `session_migrator_password` and the CA.
|
||||
- `deploy/workspaces/server-sessions.yaml.example` binds the runtime repository to the
|
||||
TLS-verified direct PostgreSQL configuration. The runtime password remains a file reference.
|
||||
- `docker/cutover-legacy-sessions.sh` archives/checksums exactly three reviewed legacy sessions
|
||||
and requires an explicit `--delete` rerun before deleting them.
|
||||
- README, secret guidance, environment examples, and PROJECT_STATE describe the maintenance
|
||||
sequence, Task 4+5 coordinated rollout, liveness vs storage 503 behavior, and the no-dual-write
|
||||
rollback rule.
|
||||
|
||||
## TDD evidence
|
||||
|
||||
RED was established with:
|
||||
|
||||
```sh
|
||||
cd backend && npx vitest run test/config.test.ts
|
||||
```
|
||||
|
||||
The new tests failed because `sessionStorage` did not exist and public/local and unauthenticated
|
||||
server combinations were accepted. After implementing the minimal configuration contract, the
|
||||
same focused suite passed (7 tests). Updating the existing upstream-health fixture to supply the
|
||||
now-required server inputs confirmed that `/health` remains an unauthenticated `200` liveness
|
||||
endpoint under the valid server contract.
|
||||
|
||||
## Verification
|
||||
|
||||
```text
|
||||
cd harness && .venv/bin/pytest -q
|
||||
826 passed, 5 deselected, 67 warnings in 63.01s
|
||||
|
||||
cd backend && npx vitest run && npx tsc --noEmit -p . && npm run build
|
||||
22 files / 215 tests passed; TypeScript check and production build passed
|
||||
|
||||
cd frontend && npx vitest run && npx tsc -b && npm run build
|
||||
full Vitest suite, TypeScript build, and Vite production build passed
|
||||
```
|
||||
|
||||
The frontend gate retained its pre-existing React-ref/MSW/act warnings and Vite chunk-size warning;
|
||||
none caused a test or build failure.
|
||||
|
||||
Additional static validation passed:
|
||||
|
||||
```text
|
||||
docker compose config --quiet (base plus copied session-server overlay with temporary empty secrets)
|
||||
bash -n docker/cutover-legacy-sessions.sh
|
||||
git diff --check
|
||||
```
|
||||
|
||||
## Manual gate remaining
|
||||
|
||||
An operator must still choose the three reviewed legacy IDs, materialize real runtime/migrator/CA
|
||||
secrets, deploy Task 4 and Task 5 together in a maintenance window, apply the one-shot migrator,
|
||||
and run the documented authenticated smoke. The guarded helper has not been invoked with
|
||||
`--delete`.
|
||||
|
||||
## P1 correction — migrator TLS validation
|
||||
|
||||
The original migrator Compose command interpolated `THT_SESSION_DB_SSLMODE` into its URL without
|
||||
checking it. `docker/session-migrate.sh` now rejects every value except `verify-ca` and
|
||||
`verify-full` before reading the password file or building that URL; the Compose service invokes
|
||||
this helper. `docker/session-migrate.test.sh` first established RED because the helper did not
|
||||
exist, then verified that `prefer` is rejected before `tht` can run and that `verify-full` reaches
|
||||
a fake `tht` binary with the expected TLS URL. The helper and test pass `bash -n`; the focused
|
||||
backend config/health suite remains green, and the base-plus-overlay Compose configuration renders
|
||||
with temporary empty secret files.
|
||||
@@ -1,5 +1,19 @@
|
||||
# AGENTS.md
|
||||
|
||||
## Agent skills
|
||||
|
||||
### Issue tracker
|
||||
|
||||
Issues for this repository live in the self-hosted Gitea repository at `https://git.tylconsulting.it/mptyl/ThothII`; use its web UI or authenticated Gitea API. See `docs/agents/issue-tracker.md`.
|
||||
|
||||
### Triage labels
|
||||
|
||||
Use the canonical labels `needs-triage`, `needs-info`, `ready-for-agent`, `ready-for-human`, and `wontfix`. See `docs/agents/triage-labels.md`.
|
||||
|
||||
### Domain docs
|
||||
|
||||
This is a single-context repository with root `CONTEXT.md` and `docs/adr/`. See `docs/agents/domain.md`.
|
||||
|
||||
This file provides guidance to Codex (Codex.ai/code) when working with code in this repository.
|
||||
|
||||
## Start here
|
||||
@@ -7,13 +21,20 @@ This file provides guidance to Codex (Codex.ai/code) when working with code in t
|
||||
Read [PROJECT_STATE.md](PROJECT_STATE.md) for the current-state snapshot: what was last
|
||||
built, pending manual gates, workspace/secret layout, and design-doc locations. This file
|
||||
holds the stable commands + architecture mental model; PROJECT_STATE.md holds the evolving
|
||||
detail. Design history lives in `docs/superpowers/specs/` and `docs/superpowers/plans/`.
|
||||
detail. Current architecture and contracts live in `docs/architecture/`, `docs/contracts/`,
|
||||
and `docs/evidence.md`; durable design decisions live in `docs/adr/`. Git history is the source
|
||||
for superseded designs and implementation plans.
|
||||
|
||||
## Commands
|
||||
|
||||
The repo has three independently-built layers. Run the local Docker stack with `./scripts/run-stack.sh` after creating `deploy/env/local.env`; it starts the base+local Compose profile with `frontend`, `core`, `qdrant`, `embedding`, and the one-shot `embedding-model-init`. The core image contains Pi. Qdrant and Ollama are internal Compose services; DWH and LLM remain external configuration endpoints.
|
||||
|
||||
**harness/** (Python `tht` CLI + Pi gate extension)
|
||||
**Native host CLI `tht`** (`tools/tht/`)
|
||||
- Operator surface: `setup`, `start`, `stop`, `status`, `doctor`, `auth`, `workspace`, and `pi`.
|
||||
- Use `tht --installation <absolute-path>/thothii-installation.yaml <command>` for installation,
|
||||
authentication, diagnostics, lifecycle, and workspace operations.
|
||||
|
||||
**harness/** (Python workflow `tht` CLI + Pi gate extension)
|
||||
- Install: `cd harness && python -m venv .venv && pip install -e ".[dev]"` (puts `tht` on PATH)
|
||||
- Test: `.venv/bin/pytest -q` — `l2` (real GLM + remote DB) is opt-in via `addopts = -m 'not l2'`; `l0` (testcontainers) needs Docker
|
||||
- Single test: `.venv/bin/pytest tests/test_session_mutations.py::test_set_name -v` (or `-k <pattern>`); include e2e with `-m l2`
|
||||
@@ -29,6 +50,10 @@ The repo has three independently-built layers. Run the local Docker stack with `
|
||||
- Test: `npx vitest run` · Single: `npx vitest run src/shell/NavSessions.test.tsx`
|
||||
- Typecheck: `npx tsc -b` · E2E: `npm run e2e` (Playwright)
|
||||
|
||||
**Documentation** (MkDocs, repository-locked Python dependencies)
|
||||
- Strict build: `./scripts/build-docs.sh`
|
||||
- Refresh lock: `./scripts/update-docs-lock.sh`
|
||||
|
||||
No ESLint on the TS layers — `tsc` is the gate. Tests use vitest + MSW (no network).
|
||||
|
||||
## Architecture (the parts that need multiple files to see)
|
||||
@@ -37,8 +62,8 @@ No ESLint on the TS layers — `tsc` is the gate. Tests use vitest + MSW (no net
|
||||
frontend (React/SSE) → backend (Fastify) → pi --mode rpc → tht/harness → DWH (read-only)
|
||||
```
|
||||
|
||||
- **The harness owns the workflow and all persistence.** `tht` (Python) is a deterministic
|
||||
CLI; `harness/.pi/extensions/tht-gate.js` is a Pi extension that drives an **8-phase
|
||||
- **The harness owns the workflow and all persistence.** The Python workflow CLI `tht` inside
|
||||
`core` is deterministic; `harness/.pi/extensions/tht-gate.js` is a Pi extension that drives an **8-phase
|
||||
NL→SQL workflow**. The single source of workflow truth is `harness/workflow.yaml`; the
|
||||
orchestration rules the model must follow are `harness/.pi/skills/tht-sessione/SKILL.md`.
|
||||
"Current phase" is computed by folding the decision ledger (`harness/tht/phase.py`), not
|
||||
@@ -51,11 +76,15 @@ frontend (React/SSE) → backend (Fastify) → pi --mode rpc → tht/harness →
|
||||
There is no verbatim transcript store. A resumed Pi process rebuilds context from
|
||||
`tht session show <id>` + the on-disk artifacts.
|
||||
|
||||
- **The backend is a thin bridge with no database.** `ThtRunner` shells `tht` subcommands;
|
||||
- **The backend bridges sessions and owns the installation-local metadata catalog.** `ThtRunner`
|
||||
shells the Python workflow `tht` subcommands inside `core`;
|
||||
`PiProcessManager` runs one Pi child per session and bridges its RPC stream;
|
||||
`SessionBridge` maps Pi RPC events → client events (`ui_request`/`text_delta`/`info`);
|
||||
`SseHub` fans them out over SSE to the browser. App settings live in a JSON file
|
||||
(`backend/data/settings.json`), not a DB.
|
||||
`SseHub` fans them out over SSE to the browser. The separate PostgreSQL catalog stores database
|
||||
metadata and sequential AI description-generation runs. Description generation samples the DWH
|
||||
through read-only connectors and calls a short-lived Python LiteLLM helper; it does not use Pi or
|
||||
expose a public CLI command. Sessions, metadata generation, and embedding resolve models from the
|
||||
generated Installation Model Catalog; `thothii-installation.yaml` is its only authored source.
|
||||
|
||||
- **Human-in-the-loop gate contract.** The model proposes; a human reviewer decides at gates
|
||||
via widgets (`reviewer_select` = single pick — a chosen option carrying a `decision` payload
|
||||
@@ -69,13 +98,24 @@ frontend (React/SSE) → backend (Fastify) → pi --mode rpc → tht/harness →
|
||||
- **`tht`'s `-c`/`--config` is a PER-COMMAND option** — it must follow the subcommand, never
|
||||
precede it (`ThtRunner.buildArgv` enforces this; prepending caused live 500s).
|
||||
- **`--json` output must be pristine** (only valid JSON on stdout) — used as a machine contract.
|
||||
- **UI strings are English; document *content* stays the workspace language** (Italian for
|
||||
`psd`) because it's the real data. Only chrome/labels are English.
|
||||
- **Workspaces** (`harness/workspaces/*.yaml`) set the DB target and **absolute**
|
||||
`paths.sessions/artifacts/indexes` — for `psd` these point at a *separate, uncommitted* repo
|
||||
(`tht-workspace-psd/`). Secrets live ONLY in `harness/.env` (gitignored).
|
||||
- **Settings are global** (`backend/data/settings.json`: workspace/provider/model/thinking);
|
||||
the New-session form is question-only.
|
||||
- **Localization:** deterministic UI uses the EN/IT catalogs with English fallback;
|
||||
model interaction uses the session manifest's immutable `interaction_language`.
|
||||
Workspace content, SQL, and identifiers remain unchanged. For shell modes, portal
|
||||
integration, or translations, read `docs/operations/shell-and-localization.md`.
|
||||
- **Server deployment:** for the coordinated ThothII/Omics upgrade, follow
|
||||
`docs/operations/server-codex-handoff.md`; it supersedes earlier Omics delivery
|
||||
instructions. Omics source integration uses GitHub with no repository relay prerequisite.
|
||||
- **Server identity:** for portal login/logout, proxy headers, or Omics deploy, read
|
||||
`docs/install/authentication-upstream.md` before changing authentication. Omics
|
||||
uses embedded/upstream, not a second ThothII OIDC login. Full/embedded rendering
|
||||
is documented in `docs/architecture/application-shell.md`; release acceptance
|
||||
is in `docs/testing/authentication-manual-acceptance.md`.
|
||||
- **Workspace schema v4** defines workspace identity and optional Evidence only. PostgreSQL Metadata
|
||||
Catalog owns database identity, binding, schema, descriptions, sensitivity, and relationships;
|
||||
embedding/model facts come from the installation catalog. The legacy `harness/workspaces/*.yaml` runtime snapshots still use
|
||||
absolute session/artifact/index paths; secrets stay in `harness/.env` (gitignored).
|
||||
- **Settings are global** (`backend/data/settings.json`: workspace/thinking). Provider/model choices
|
||||
are ephemeral canonical catalog selections pinned into the session manifest.
|
||||
- **Resume**: a resumable session re-enters at its last incomplete phase. The backend refuses
|
||||
resume with 409 when `finalized` or `archived`, and `PiProcessManager.spawnFor` must send
|
||||
`/riprendi-sessione <id>` (resume mode) vs `/nuova-domanda` (new) — sending the wrong prompt
|
||||
|
||||
@@ -1,87 +0,0 @@
|
||||
# CLAUDE.md
|
||||
|
||||
This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
|
||||
|
||||
## Start here
|
||||
|
||||
Read [PROJECT_STATE.md](PROJECT_STATE.md) for the current-state snapshot: what was last
|
||||
built, pending manual gates, workspace/secret layout, and design-doc locations. This file
|
||||
holds the stable commands + architecture mental model; PROJECT_STATE.md holds the evolving
|
||||
detail. Design history lives in `docs/superpowers/specs/` and `docs/superpowers/plans/`.
|
||||
|
||||
## Commands
|
||||
|
||||
The repo has three independently-built layers. Run the **full stack** (real Pi + DWH, needs
|
||||
VPN + `harness/.env` + `pi` on PATH) with `./scripts/run-stack.sh` (frontend :5173 → backend :8787).
|
||||
|
||||
**harness/** (Python `tht` CLI + Pi gate extension)
|
||||
- Install: `cd harness && python -m venv .venv && pip install -e ".[dev]"` (puts `tht` on PATH)
|
||||
- Test: `.venv/bin/pytest -q` — `l2` (real GLM + remote DB) is opt-in via `addopts = -m 'not l2'`; `l0` (testcontainers) needs Docker
|
||||
- Single test: `.venv/bin/pytest tests/test_session_mutations.py::test_set_name -v` (or `-k <pattern>`); include e2e with `-m l2`
|
||||
- Lint: `.venv/bin/ruff check .` (line-length 100)
|
||||
|
||||
**backend/** (Fastify + TypeScript, vitest)
|
||||
- Dev: `npm run dev` (tsx watch `src/server.ts`) · Build: `npm run build` (tsc → `dist/`)
|
||||
- Test: `npx vitest run` · Single: `npx vitest run test/routes-sessions.test.ts -t "rename"`
|
||||
- Typecheck: `npx tsc --noEmit -p .` (vitest does NOT type-check — run this before committing)
|
||||
|
||||
**frontend/** (React 18 + Vite + vitest)
|
||||
- Dev: `npm run dev` (Vite; set `VITE_BACKEND_URL`) · Build: `npm run build`
|
||||
- Test: `npx vitest run` · Single: `npx vitest run src/shell/NavSessions.test.tsx`
|
||||
- Typecheck: `npx tsc -b` · E2E: `npm run e2e` (Playwright)
|
||||
|
||||
No ESLint on the TS layers — `tsc` is the gate. Tests use vitest + MSW (no network).
|
||||
|
||||
## Architecture (the parts that need multiple files to see)
|
||||
|
||||
```
|
||||
frontend (React/SSE) → backend (Fastify) → pi --mode rpc → tht/harness → DWH (read-only)
|
||||
```
|
||||
|
||||
- **The harness owns the workflow and all persistence.** `tht` (Python) is a deterministic
|
||||
CLI; `harness/.pi/extensions/tht-gate.js` is a Pi extension that drives an **8-phase
|
||||
NL→SQL workflow**. The single source of workflow truth is `harness/workflow.yaml`; the
|
||||
orchestration rules the model must follow are `harness/.pi/skills/tht-sessione/SKILL.md`.
|
||||
"Current phase" is computed by folding the decision ledger (`harness/tht/phase.py`), not
|
||||
stored — read it before reasoning about phase logic.
|
||||
|
||||
- **Persistence = phase documents, NOT chat.** A session is a directory under the workspace's
|
||||
`sessions/` path: `session_manifest.yaml` + per-phase artifacts (`question.md`,
|
||||
`schema_linking.json`, `sql_final.sql`, …) + `review_decisions.jsonl`. The contract
|
||||
(SKILL.md): *"the persisted state is the truth — what is not recorded did not happen."*
|
||||
There is no verbatim transcript store. A resumed Pi process rebuilds context from
|
||||
`tht session show <id>` + the on-disk artifacts.
|
||||
|
||||
- **The backend is a thin bridge with no database of its own.** `ThtRunner` shells `tht`
|
||||
subcommands; `PiProcessManager` runs one Pi child per session and bridges its RPC stream;
|
||||
`SessionBridge` maps Pi RPC events → client events (`ui_request`/`text_delta`/`info`);
|
||||
`SseHub` fans them out over SSE to the browser. Persistence belongs to the HARNESS, which
|
||||
selects the session repository from the workspace config (`harness/tht/session/repository.py`):
|
||||
filesystem by default, **PostgreSQL when `session_storage` is configured** (server/portable
|
||||
deployment). Settings flow through harness preferences (`tht session preferences`) with
|
||||
`backend/data/settings.json` only as the file fallback for injected runners/tests.
|
||||
|
||||
- **Human-in-the-loop gate contract.** The model proposes; a human reviewer decides at gates
|
||||
via widgets (`reviewer_select` = single pick — a chosen option carrying a `decision` payload
|
||||
auto-confirms/persists directly, an option without one only asks; `reviewer_decide` = multiselect,
|
||||
each choice IS a decision; `reviewer_confirm` = artifact/phase gate). The frontend renders these
|
||||
widget-descriptors (`src/widgets/` registry) and the live transcript is rebuilt in-memory
|
||||
from the SSE stream (`src/store/sessionStore.ts`) — it is not persisted.
|
||||
|
||||
## Project-specific gotchas
|
||||
|
||||
- **`tht`'s `-c`/`--config` is a PER-COMMAND option** — it must follow the subcommand, never
|
||||
precede it (`ThtRunner.buildArgv` enforces this; prepending caused live 500s).
|
||||
- **`--json` output must be pristine** (only valid JSON on stdout) — used as a machine contract.
|
||||
- **UI strings are English; document *content* stays the workspace language** (Italian for
|
||||
`psd`) because it's the real data. Only chrome/labels are English.
|
||||
- **Workspaces** (`harness/workspaces/*.yaml`) set the DB target and **absolute**
|
||||
`paths.sessions/artifacts/indexes` — for `psd` these point at a *separate, uncommitted* repo
|
||||
(`tht-workspace-psd/`). Secrets live ONLY in `harness/.env` (gitignored).
|
||||
- **Settings are global** (workspace/provider/model/thinking, persisted via harness
|
||||
preferences — `backend/data/settings.json` is only the fallback); the New-session form is
|
||||
question-only.
|
||||
- **Resume**: a resumable session re-enters at its last incomplete phase. The backend refuses
|
||||
resume with 409 when `finalized` or `archived`, and `PiProcessManager.spawnFor` must send
|
||||
`/riprendi-sessione <id>` (resume mode) vs `/nuova-domanda` (new) — sending the wrong prompt
|
||||
silently turns a resume into a new question.
|
||||
+643
@@ -0,0 +1,643 @@
|
||||
# Contesto di dominio di ThothII
|
||||
|
||||
## Architettura del workflow
|
||||
|
||||
**Workflow Kernel** — Il coordinatore deterministico che possiede lo stato del workflow,
|
||||
le transizioni, il rollback, la finalizzazione e l'applicazione atomica degli esiti dei
|
||||
moduli.
|
||||
|
||||
**Workflow Module** — Una capacità incapsulata che espone un contratto versionato. Un
|
||||
modulo può partecipare a più stage e non modifica direttamente lo stato del workflow.
|
||||
|
||||
**Stage** — Un punto del workflow, identificato semanticamente, nel quale viene invocato
|
||||
un modulo. L'identità dello stage è indipendente dalla sua posizione visiva.
|
||||
|
||||
**Display code** — L'etichetta di presentazione associata a uno stage, per esempio da
|
||||
`F1` a `F8`. I display code alimentano gli indicatori di avanzamento nel frontend, ma non
|
||||
sono usati come identità del workflow o chiavi di dipendenza.
|
||||
|
||||
**Module outcome** — Il risultato proposto da un modulo: eventi tipizzati, modifiche agli
|
||||
artifact, un'eventuale richiesta di revisione umana e uno stato di esecuzione. Il
|
||||
Workflow Kernel valida e applica l'esito.
|
||||
|
||||
**Revision request** — La proposta tipizzata con cui un modulo segnala che lo stage
|
||||
corrente non può concludersi validamente senza rieseguire lo stesso stage o uno stage
|
||||
precedente. Non produce direttamente una transizione: il Workflow Kernel valida la
|
||||
richiesta, sospende l'avanzamento e, per riaprire uno stage già completato, attende una
|
||||
decisione umana tipizzata. Il Kernel, non il modulo, determina gli eventi e gli artifact
|
||||
causalmente da rendere stale.
|
||||
|
||||
**Question Admission** — Il controllo preliminare eseguito prima delle fasi da `F1` a
|
||||
`F8`. Nella prima release distingue una domanda utilizzabile da input garbage e verifica
|
||||
che la domanda appartenga allo scope dichiarato dal workspace. Il suo stato è mostrato
|
||||
separatamente dagli otto indicatori di fase.
|
||||
|
||||
**Workspace scope** — La dichiarazione gestita e versionata di ciò che il database di un
|
||||
workspace rappresenta e delle domande alle quali è destinato a rispondere. Question
|
||||
Admission la usa come riferimento per valutare la pertinenza di una domanda.
|
||||
|
||||
**Datamart Plugin** — Il modulo sostituibile che implementa lo stage semantico
|
||||
`datamart`, presentato con display code `F8`. La promozione della memory e la
|
||||
finalizzazione della sessione non appartengono al Datamart Plugin.
|
||||
|
||||
**Ordered workflow** — La pipeline deterministica composta dal preflight Admission,
|
||||
dagli otto stage principali ordinati da `F1` a `F8` e dalla finalizzazione. L'ordine
|
||||
degli stage è esplicito; il workflow non è un DAG generale.
|
||||
|
||||
**Extension point** — Una posizione semantica nel lifecycle dell'Ordered workflow alla
|
||||
quale possono contribuire uno o più moduli senza diventare nuovi stage visibili. Un
|
||||
extension point non possiede un display code.
|
||||
|
||||
**Stage state** — La proiezione deterministica degli eventi del workflow che descrive
|
||||
uno stage come `pending`, `ready`, `running`, `awaiting_human`, `completed`, `skipped` o
|
||||
`failed`. Non è un valore corrente memorizzato separatamente dal ledger.
|
||||
|
||||
**Required contribution** — Il contributo di un modulo a un extension point che deve
|
||||
concludersi o essere esplicitamente saltato secondo policy prima che il workflow possa
|
||||
avanzare.
|
||||
|
||||
**Best-effort contribution** — Il contributo di un modulo il cui fallimento viene
|
||||
registrato e mostrato come warning, ma non impedisce al workflow di avanzare.
|
||||
|
||||
**Blocked workflow** — La proiezione complessiva di un workflow che non può avanzare a
|
||||
causa di uno stage o di un contributo required fallito o non disponibile. `Blocked` non
|
||||
è uno Stage state autonomo.
|
||||
|
||||
**Module invocation** — Una singola richiesta del Workflow Kernel a un modulo in uno
|
||||
stage o extension point. Conserva la stessa identità attraverso eventuali retry, che
|
||||
sono tentativi distinti della medesima invocation.
|
||||
|
||||
**Stage skip** — La conclusione esplicita di uno stage senza eseguirne il comportamento.
|
||||
È ammessa soltanto dalla policy dello stage e registra motivo e attore; un fallimento non
|
||||
equivale mai implicitamente a uno skip.
|
||||
|
||||
**Stage reopen** — La riapertura di uno stage non finalizzato che rende stale gli esiti
|
||||
causalmente successivi. Gli effetti esterni già prodotti richiedono una marcatura o una
|
||||
compensazione esplicita e non sono presentati come automaticamente annullati. Può essere
|
||||
applicata dal Workflow Kernel in seguito all'approvazione di una Revision request, ma
|
||||
non può essere eseguita direttamente da Pi o da un Workflow Module.
|
||||
|
||||
**Completion policy** — La regola con cui uno stage si conclude: `automatic` quando il
|
||||
kernel può verificarne deterministicamente l'esito, oppure `review_required` quando è
|
||||
necessaria un'approvazione umana tipizzata.
|
||||
|
||||
**Paused session** — Una sessione interrotta intenzionalmente ma resumibile. L'azione
|
||||
“Stop and save” mette la sessione in pausa; non la completa e non la marca come fallita.
|
||||
|
||||
**Finalized session** — Una sessione completata con esito canonico e immutabile. Una
|
||||
correzione successiva crea una nuova sessione derivata, collegata a quella precedente.
|
||||
|
||||
**After-finalize hook** — Una notifica o attività best-effort eseguita tramite outbox
|
||||
dopo la finalizzazione. Non può modificare il ledger, gli artifact canonici o lo stato
|
||||
terminale della sessione.
|
||||
|
||||
## Memory
|
||||
|
||||
**Memory Module** — Il modulo che possiede le conoscenze ed esperienze curate per
|
||||
migliorare schema linking e generazione SQL di domande future. Le Memory appartengono
|
||||
a un workspace e rimangono distinte dalle Evidence.
|
||||
|
||||
**Memory Card** — L'unità di contenuto gestibile del Memory Module, con identità,
|
||||
ambito di applicazione e provenienza. Il formato è allineato per analogia alle
|
||||
Evidence, senza implicare la stessa origine o lo stesso percorso di pubblicazione.
|
||||
|
||||
**Reusable Memory** — Una Memory Card che esprime un chiarimento di dominio, una
|
||||
regola di costruzione SQL o un errore da evitare con motivo compreso e approvato.
|
||||
La sua validità è circoscritta a un ambito esplicito e non deriva dalla sola
|
||||
approvazione di una scelta occasionale in una domanda.
|
||||
|
||||
**Solved Question** — Una Memory Card che conserva una domanda risolta con la
|
||||
relativa soluzione SQL e il contesto necessario a interpretarla. È un exemplar
|
||||
consultativo: i parametri e le scelte del caso non diventano regole generali.
|
||||
|
||||
**Memory Graph** — L'insieme dei collegamenti espliciti fra card che contribuisce
|
||||
al recupero di conoscenze pertinenti oltre alla somiglianza del contenuto. Il
|
||||
ritrovamento di una card tramite un collegamento non ne implica l'approvazione.
|
||||
|
||||
**Memory Link** — Un collegamento curato fra card, con destinazione e significato
|
||||
espliciti, che contribuisce alla consultazione di contenuti pertinenti. La sua
|
||||
rimozione non comporta la cancellazione delle card collegate.
|
||||
|
||||
## Evidence
|
||||
|
||||
**Context specialist** — La persona competente sul dominio che redige e cura il
|
||||
contenuto delle Evidence. Può essere distinta da chi amministra l'installazione;
|
||||
il suo lavoro di redazione non richiede accesso al database applicativo.
|
||||
|
||||
**Evidence draft** — Il documento iniziale scritto dallo specialista di contesto,
|
||||
che il sistema acquisisce e raffina in Evidence Unit. Può essere redatto e
|
||||
consegnato indipendentemente dall'installazione che userà le Evidence risultanti.
|
||||
|
||||
**Evidence Module** — Il modulo autonomo che possiede la preparazione delle Evidence e
|
||||
la loro consultazione durante il workflow. La preparazione avviene fuori dalle singole
|
||||
sessioni; il workflow usa soltanto contenuti già pubblicati. A runtime contribuisce agli
|
||||
stage semantici esistenti, senza diventare uno stage visibile e senza modificare ledger,
|
||||
artifact o stato del workflow.
|
||||
|
||||
**Source Evidence** — Il documento o la dichiarazione che sostiene il contenuto
|
||||
corrente di una Evidence Unit. Un documento acquisito viene conservato come
|
||||
riferimento umano; una dichiarazione manuale attribuisce il contenuto alla persona
|
||||
che lo ha scritto e approvato.
|
||||
|
||||
**Manual Evidence declaration** — Una dichiarazione esplicita dell'amministratore
|
||||
che sostiene una Evidence creata direttamente o una correzione del suo significato.
|
||||
Non implica una verifica indipendente da parte di una fonte documentale esterna.
|
||||
|
||||
**Evidence origin** — Il documento da cui una Evidence Unit è stata inizialmente
|
||||
derivata. Può restare collegato per provenienza e confronto con gli aggiornamenti
|
||||
anche quando una dichiarazione manuale sostiene il testo corrente. La sola origine
|
||||
non dimostra il supporto semantico di una successiva correzione.
|
||||
|
||||
**Local Evidence archive** — L'insieme delle Evidence curate custodite
|
||||
dall'installazione, distinto dalle draft originali e dai contenuti derivati per
|
||||
la ricerca. Comprende le correzioni manuali e i ritiri deliberati.
|
||||
|
||||
**Consolidated Evidence** — Una versione delle Evidence locali controllata come
|
||||
insieme coerente e pronta per l'attivazione. I file ancora in modifica non ne
|
||||
cambiano il contenuto.
|
||||
|
||||
**Active Evidence** — La versione consolidata disponibile alla consultazione del
|
||||
core. Un tentativo di aggiornamento fallito conserva la versione attiva precedente.
|
||||
|
||||
**Evidence Unit** — La più piccola unità semantica coerente, revisionabile e ricercabile
|
||||
fondata su una Source Evidence corrente, anche manuale, e con eventuale origine
|
||||
documentale distinta. Possiede un identificatore stabile indipendente dal kind,
|
||||
assegnato una volta nella forma `evidence:<slug>`; fonti diverse non vengono fuse
|
||||
automaticamente.
|
||||
|
||||
**Evidence kind** — La categoria semantica di una Evidence Unit, che ne determina i
|
||||
campi specifici e ne orienta l'uso. Ogni unità ha un solo kind primario; i tipi iniziali
|
||||
sono `glossary`, `domain`, `enum`, `example`, `mapping`, `normalization`, `formula` e
|
||||
`reference`.
|
||||
|
||||
**Glossary Evidence** — Una Evidence Unit che definisce il significato linguistico, i
|
||||
sinonimi o le varianti di un termine.
|
||||
|
||||
**Domain Evidence** — Una Evidence Unit che esprime una regola o un vincolo del dominio
|
||||
non rappresentato da un kind più specifico.
|
||||
|
||||
**Enum Evidence** — Una Evidence Unit che collega un insieme finito di valori
|
||||
memorizzati ai relativi significati.
|
||||
|
||||
**Example Evidence** — Una Evidence Unit che associa un input o una domanda alla sua
|
||||
interpretazione o al risultato atteso.
|
||||
|
||||
**Mapping Evidence** — Una Evidence Unit che collega un concetto logico agli elementi
|
||||
del relativo schema fisico.
|
||||
|
||||
**Normalization Evidence** — Una Evidence Unit che descrive la trasformazione di una
|
||||
rappresentazione in una forma canonica.
|
||||
|
||||
**Formula Evidence** — Una Evidence Unit che contiene una singola espressione PostgreSQL
|
||||
componibile e ne dichiara gli input. Una query SQL completa non è una Formula Evidence.
|
||||
|
||||
**Reference Evidence** — Una Evidence Unit che rappresenta un collegamento esterno da
|
||||
restituire come contenuto autonomo, anziché come semplice provenienza.
|
||||
|
||||
**Evidence purpose** — La destinazione dichiarata di una Evidence Unit nel workflow:
|
||||
disambiguation, rewriting, schema linking o SQL generation. È distinta dall'Evidence
|
||||
kind: il tipo descrive cosa contiene, il purpose quando può essere utile; durante la
|
||||
ricerca il purpose richiesto è un filtro obbligatorio. Il recupero di esperienze e
|
||||
soluzioni precedenti appartiene al Memory Module e non è un Evidence purpose.
|
||||
|
||||
**Evidence Search Outcome** — Il risultato tipizzato di una consultazione del modulo
|
||||
Evidence. Distingue una ricerca disponibile, che può legittimamente non trovare
|
||||
corrispondenze, da un'indisponibilità tecnica che impedisce allo stage chiamante di
|
||||
avanzare fino a un retry riuscito.
|
||||
|
||||
**Evidence receipt** — La traccia minima di una consultazione disponibile conservata
|
||||
nella sessione: stage semantico, purpose, generazione interrogata e identificatori delle
|
||||
Evidence restituite. Non duplica il contenuto delle Evidence.
|
||||
|
||||
**Curated Evidence** — Una o più Evidence Unit preparate da documenti o curate
|
||||
manualmente. La presenza nell'archivio curato non implica da sola che il contenuto
|
||||
sia già attivo per il workflow.
|
||||
|
||||
**Published Evidence** — Le Curated Evidence valide appartenenti alla revisione attiva
|
||||
del workspace e alla generazione Evidence pubblicata. L'approvazione umana precede
|
||||
l'attivazione, ma non viene duplicata come stato nel manifest.
|
||||
|
||||
**Evidence Index** — La proiezione ricercabile e ricostruibile delle Published Evidence.
|
||||
Accelera il recupero delle informazioni, ma non è una fonte di verità.
|
||||
|
||||
**Evidence preparation** — Il processo di authoring che trasforma Source Evidence in
|
||||
Curated Evidence mediante estrazione e normalizzazione deterministiche, una singola
|
||||
ristrutturazione assistita dal modello e una validazione finale deterministica. Nella
|
||||
prima versione accetta Markdown o testo UTF-8 e non acquisisce automaticamente il
|
||||
contenuto di URL o documenti esterni. Prepara l'intero insieme delle modifiche in
|
||||
un'area temporanea e lo applica atomicamente soltanto se tutti gli output sono validi;
|
||||
non ritenta automaticamente una chiamata al modello fallita.
|
||||
|
||||
**Supporting excerpt** — Un breve estratto presente nel Source Evidence che sostiene
|
||||
una Evidence Unit. Il sistema ne verifica deterministicamente la presenza dopo la
|
||||
normalizzazione meccanica; il curatore resta responsabile di verificarne la sufficienza
|
||||
semantica.
|
||||
|
||||
**Evidence resolution** — La decisione esplicita con cui un curatore risolve un
|
||||
problema di una Evidence Unit, correggendola, ritirandola oppure ricollegandola a
|
||||
una fonte adeguata.
|
||||
|
||||
**Source update conflict** — Un contrasto fra una fonte aggiornata e una correzione
|
||||
manuale già approvata. La correzione resta in uso fino alla risoluzione esplicita
|
||||
del confronto da parte dell'amministratore.
|
||||
|
||||
**Evidence source refresh** — La riacquisizione delle fonti esterne richiesta
|
||||
dall'amministratore per rilevarne le modifiche. Fra due aggiornamenti il contenuto
|
||||
già acquisito resta il riferimento per preparazione e consultazione.
|
||||
|
||||
**Review item** — Un blocco di revisione descritto da codice stabile, messaggio umano e
|
||||
campo opzionale. Finché viene mantenuto nell'Evidence Unit, ne impedisce la
|
||||
pubblicazione; la sua storia è conservata da Git, non da uno stato interno all'item.
|
||||
|
||||
**Retirement candidate** — Una Curated Evidence che il Source Evidence esistente non
|
||||
sostiene più. Rimane visibile con un Review item e blocca la pubblicazione finché il
|
||||
curatore non la elimina oppure la rende nuovamente coerente con il sorgente.
|
||||
|
||||
**Evidence evaluation set** — Un piccolo insieme versionato di domande rappresentative
|
||||
e relativi risultati attesi. La baseline è accettabile quando ogni domanda recupera
|
||||
almeno un risultato atteso nei primi dieci risultati della fusione RRF; il risultato
|
||||
nei primi cinque è informativo. Comprende almeno un caso lessicale, uno semantico e uno
|
||||
misto e conserva, a fini diagnostici, le posizioni dense, BM25 e fused.
|
||||
|
||||
**Candidate Evidence Generation** — Una generazione completa dell'Evidence Index che
|
||||
può essere valutata ma non è ancora visibile alle sessioni. Diventa attiva soltanto se
|
||||
supera l'Evidence evaluation set.
|
||||
|
||||
**Evidence manifest** — Il file versionato e gestito dal sistema che collega ogni
|
||||
Source Evidence al suo hash e alle Evidence Unit derivate. Conserva gli identificatori
|
||||
stabili, permette l'elaborazione incrementale e segnala le unità rimaste orfane senza
|
||||
cancellarle automaticamente.
|
||||
|
||||
**Orphaned Evidence Unit** — Una Curated Evidence il cui Source Evidence non esiste più.
|
||||
Rimane disponibile per la revisione, ma blocca la pubblicazione finché non viene
|
||||
eliminata, ricollegata oppure ne viene ripristinato il sorgente.
|
||||
|
||||
**Evidence Fragment** — Una proiezione ricercabile di una sezione semanticamente
|
||||
coerente di una Published Evidence. La divisione segue intestazioni e confini di
|
||||
paragrafo; formule, coppie valore/significato, mapping, regole e URL non vengono mai
|
||||
tagliati. Il testo completo reso per il frammento usa il solo limite esistente
|
||||
`max_chunk_chars`, pari per default a 4.000 caratteri; un elemento atomico troppo grande
|
||||
produce un Review item bloccante. Qdrant indicizza i frammenti, mentre l'Evidence Module
|
||||
li raggruppa per Evidence Unit.
|
||||
|
||||
**Evidence Result** — La rappresentazione di una singola Evidence Unit restituita dalla
|
||||
ricerca con metadati, migliori estratti, provenienza e riferimento al documento completo.
|
||||
|
||||
**Hybrid Evidence retrieval** — La ricerca che combina in Qdrant una graduatoria
|
||||
semantica dense e una graduatoria lessicale BM25 sparse mediante Reciprocal Rank
|
||||
Fusion. I metadati tipizzati restringono o orientano i risultati senza creare una
|
||||
collezione separata per ogni Evidence kind.
|
||||
|
||||
**Evidence query text** — La rappresentazione deterministica condivisa dalla ricerca
|
||||
dense e BM25: domanda originale, concetti, tabelle e colonne in ordine fisso. I campi
|
||||
vuoti sono omessi; domanda e contesto ricevono soltanto normalizzazione Unicode NFC,
|
||||
conversione degli a-capo e rimozione degli spazi esterni. Gli elementi contestuali sono
|
||||
poi deduplicati e ordinati senza conversione delle maiuscole, mentre punteggiatura e
|
||||
spazi interni della domanda non vengono riscritti.
|
||||
|
||||
**Reference Vector Collection** — La collezione Qdrant ricostruibile di un workspace che
|
||||
contiene Schema, relazioni ed Evidence. Possiede il vettore dense predefinito e il vettore
|
||||
sparse `bm25`; soltanto gli Evidence Fragment ricevono valori BM25. Il preprocessing può
|
||||
sostituirla o eliminarla integralmente.
|
||||
|
||||
**Memory Vector Collection** — La collezione Qdrant persistente di un workspace che contiene
|
||||
`memory` e `solved_question`. Non è un output del preprocessing e non viene eliminata dal
|
||||
Preprocessing Clear.
|
||||
|
||||
**Preprocessing Clear** — L'operazione amministrativa che elimina Reference Vector Collection,
|
||||
LSH, corpus e checkpoint derivati e rende il workspace non pronto. Conserva Memory Vector
|
||||
Collection, sessioni, Catalog Metadata e database sorgente; non offre history o rollback.
|
||||
|
||||
**Formula proposal** — Una formula individuata durante una sessione e conservata come
|
||||
artefatto della sessione. Non diventa Published Evidence finché non viene importata,
|
||||
revisionata e approvata nel repository del workspace.
|
||||
|
||||
**Fail-closed Evidence retrieval** — Il comportamento per cui un indice assente,
|
||||
incompatibile o non aggiornato produce nessuna Evidence e un avviso esplicito. Il
|
||||
workflow può continuare, ma non usa mai silenziosamente contenuti di una revisione
|
||||
precedente o di un altro workspace.
|
||||
|
||||
## Configurazione dei modelli
|
||||
|
||||
**Workspace Descriptor** — La dichiarazione versionata dell'identità del workspace e dello
|
||||
scope delle sue Evidence. Non contiene identità o configurazione del Workspace Database,
|
||||
Database Binding, fatti strutturali o metadati semantici: il Metadata Catalog associa il
|
||||
workspace al relativo database.
|
||||
|
||||
**Installation Model Catalog** — L'insieme dichiarativo, proprio di un'installazione, dei
|
||||
modelli disponibili, dei loro Model Usage e dei relativi default. È l'unica autorità per i
|
||||
modelli di sessione, generazione dei metadati ed embedding e non appartiene a un workspace.
|
||||
_Avoid_: Model Catalog, Metadata Generation Model Configuration
|
||||
|
||||
**Model Usage** — Lo scopo per cui un modello dell'Installation Model Catalog può essere
|
||||
usato: `session`, `metadata_generation` oppure `embedding`. L'ammissibilità e il default
|
||||
dipendono dall'uso, non dal workspace.
|
||||
|
||||
**Model Selection** — La scelta runtime, a livello di installazione, di un modello del
|
||||
catalogo per uno specifico Model Usage. Riferisce l'identità canonica del modello senza
|
||||
ridefinirne provider, endpoint o capacità.
|
||||
|
||||
**Model Runtime Projection** — La rappresentazione derivata e non autoritativa
|
||||
dell'Installation Model Catalog richiesta da uno specifico runtime. Può essere rigenerata
|
||||
integralmente dalla configurazione dell'installazione.
|
||||
|
||||
## Distribuzione del prodotto
|
||||
|
||||
**Customer-Hosted Installation** — Un'installazione eseguita interamente nel trust boundary
|
||||
controllato dall'organizzazione cliente, inclusi eventuali tenant cloud privati. Credenziali,
|
||||
domande, prompt, metadati e risultati non attraversano quel boundary.
|
||||
_Avoid_: on-premise deployment, self-managed deployment
|
||||
|
||||
**Community Edition** — La distribuzione open source utilizzabile gratuitamente anche in
|
||||
produzione e capace di eseguire il workflow fondamentale completo.
|
||||
_Avoid_: free tier, trial edition
|
||||
|
||||
**Enterprise Edition** — La distribuzione con licenza commerciale che aggiunge governance
|
||||
organizzativa, esercizio production-grade e industrializzazione alla Community Edition.
|
||||
_Avoid_: paid tier, pro edition
|
||||
|
||||
## Catalogo dei metadati
|
||||
|
||||
**Workspace Database** — Il database che appartiene a un solo workspace e non può essere
|
||||
condiviso con altri workspace; un workspace può averne al massimo uno. È considerato nella
|
||||
coppia composta dal database PostgreSQL e da un solo schema: tutte le tabelle, le colonne e
|
||||
le relazioni catalogate appartengono a quello schema. Il Metadata Catalog conserva
|
||||
l'associazione, ma non crea né possiede l'identità del workspace.
|
||||
|
||||
**Database Binding** — La configurazione specifica di un'installazione che seleziona un
|
||||
trasporto e fornisce i riferimenti necessari a raggiungere un Workspace Database. Non è una
|
||||
seconda identità del database e non viene condivisa automaticamente fra installazioni.
|
||||
|
||||
**Thoth REST Connector** — Il trasporto REST tipizzato con cui ThothII interroga ed
|
||||
introspeziona un Workspace Database attraverso il contratto RPC DWH supportato. Non è un
|
||||
client configurabile per API REST arbitrarie.
|
||||
|
||||
**Orphaned Workspace Database** — Un Workspace Database il cui workspace non è più presente
|
||||
nel catalogo autorevole. Rimane conservato per il recupero amministrativo, ma non può essere
|
||||
usato dal workflow finché non viene riassegnato a un workspace esistente.
|
||||
|
||||
**Metadata Catalog** — L'autorità per l'associazione fra workspace e Workspace Database, la
|
||||
relativa Database Binding, i fatti strutturali osservati e i metadati semantici curati. Ogni
|
||||
uso downstream dei metadati del database deriva da questo catalogo.
|
||||
|
||||
**Database Profile** — L'insieme curato di scope, descrizioni e metadati semantici
|
||||
associato a un Workspace Database.
|
||||
|
||||
**Physical Table** — Una tabella osservata nello schema esterno di un Workspace Database.
|
||||
La sua identità e il suo nome appartengono al database esterno, non al Metadata Catalog.
|
||||
|
||||
**Catalog Table** — La rappresentazione persistita di una Physical Table nel Metadata Catalog.
|
||||
La sua appartenenza e identità fisica derivano dall'introspezione: non può essere creata o
|
||||
rinominata manualmente, ma può essere rimossa tramite Catalog Metadata Cleanup.
|
||||
_Avoid_: SqlTable, managed table
|
||||
|
||||
**Physical Column** — Una colonna osservata in una Physical Table, inclusi nome, posizione,
|
||||
tipo e appartenenza a chiavi dichiarate. La sua identità e i suoi fatti strutturali appartengono
|
||||
al database esterno.
|
||||
|
||||
**Catalog Column** — La rappresentazione persistita di una Physical Column nel Metadata Catalog.
|
||||
I fatti osservati sono governati dalla sincronizzazione; Description e Generated Description
|
||||
sono metadati amministrativi modificabili e la rappresentazione può essere rimossa tramite
|
||||
Catalog Metadata Cleanup.
|
||||
_Avoid_: SqlColumn, managed column
|
||||
|
||||
**Physical Relationship** — Un vincolo foreign key dichiarato nel database esterno. La sua
|
||||
identità comprende il vincolo e la sequenza ordinata delle coppie di colonne che lo compongono.
|
||||
|
||||
**Catalog Relationship** — La rappresentazione persistita di una Physical Relationship nel
|
||||
Metadata Catalog. Non è creata o modificata manualmente, ma può essere rimossa tramite Catalog
|
||||
Metadata Cleanup.
|
||||
_Avoid_: denormalized FK, relationship string
|
||||
|
||||
**Logical Relationship** — Una relazione modificabile fra due Catalog Column che non corrisponde
|
||||
necessariamente a un vincolo fisico. Può essere Generated o Manual e rimane distinta dalla Catalog
|
||||
Relationship osservata nel database.
|
||||
|
||||
**Generated Relationship** — Una Logical Relationship ricavata dai nomi delle colonne, dalle
|
||||
primary key e dalla compatibilità dei tipi mediante regole deterministiche, senza LLM, embedding o
|
||||
campionamento dei dati. Una ricostruzione non riattiva una Generated Relationship cancellata
|
||||
logicamente, ma può ricrearne una cancellata fisicamente.
|
||||
|
||||
**Manual Relationship** — Una Logical Relationship aggiunta dall'utente. La ricostruzione delle
|
||||
Generated Relationship non la modifica.
|
||||
|
||||
**Logical Relationship Deletion** — L'esclusione persistente di una Logical Relationship che ne
|
||||
conserva l'identità per impedirne la ricreazione automatica finché esistono entrambe le Catalog
|
||||
Column alle quali è collegata.
|
||||
|
||||
**Permanent Relationship Deletion** — La rimozione completa di una Logical Relationship. Una
|
||||
ricostruzione successiva può ricrearla quando soddisfa nuovamente le regole di inferenza. Anche il
|
||||
cleanup distruttivo di una tabella o colonna endpoint rimuove permanentemente le relative esclusioni.
|
||||
|
||||
**Relationship Reconstruction** — L'operazione amministrativa esplicita che scopre e aggiunge le
|
||||
Generated Relationship mancanti. Conserva le Manual Relationship e le relationship già presenti e
|
||||
non riattiva quelle cancellate logicamente.
|
||||
|
||||
**Relationship Restore** — La riattivazione esplicita di una Logical Relationship cancellata
|
||||
logicamente.
|
||||
|
||||
**Effective Relationship Map** — La vista unificata delle Catalog Relationship fisiche e delle
|
||||
Logical Relationship, con origine e stato espliciti. È l'interfaccia usata dall'amministrazione e
|
||||
dalla comprensione dello schema, non un ulteriore modello persistito.
|
||||
|
||||
**Catalog Metadata Snapshot** — La proiezione immutabile e versionata della struttura catalogata,
|
||||
delle descrizioni pubblicabili e delle relazioni effettive attive di un Workspace Database che il
|
||||
core consuma. È derivata esclusivamente dal Metadata Catalog e non è un archivio autoritativo.
|
||||
|
||||
**Schema Index** — La proiezione vettoriale ricostruibile dei metadati del Workspace Database nel
|
||||
Metadata Catalog. Il preprocessing la sostituisce integralmente e non è una fonte di verità.
|
||||
|
||||
**Description** — Il testo curato e consolidato che descrive una Catalog Table o Catalog Column
|
||||
per gli usi downstream. Quando presente, prevale sulla relativa Generated Description.
|
||||
|
||||
**Generated Description** — Il testo modificabile prodotto dall'AI per una Catalog Table o Catalog
|
||||
Column. È pubblicabile per gli usi downstream quando manca una Description, anche senza essere
|
||||
prima consolidato, e rimane distinto dal commento osservato nel database.
|
||||
_Avoid_: generated comment, source comment
|
||||
|
||||
**Description Consolidation** — L'azione amministrativa esplicita che copia la Generated
|
||||
Description di Catalog Table o Catalog Column selezionate nella relativa Description. Opera sulla
|
||||
selezione corrente, conserva la Generated Description e non modifica il commento osservato o il
|
||||
database esterno.
|
||||
|
||||
**Table Synchronization** — La riconciliazione esplicita che rende le Catalog Table di un
|
||||
Workspace Database uguali alle Physical Table osservate: crea quelle nuove, aggiorna i metadati
|
||||
di origine ed elimina definitivamente quelle assenti. Non modifica mai il database esterno.
|
||||
_Avoid_: table import
|
||||
|
||||
**Schema Synchronization** — La riconciliazione esplicita e autorevole di tabelle, colonne e
|
||||
Catalog Relationship di un Workspace Database. Può operare su uno scope specifico oppure su
|
||||
un unico snapshot completo tramite Synchronize All.
|
||||
|
||||
**Catalog Sync Run** — L'esecuzione durevole in background di una Schema Synchronization, con
|
||||
scope, stato, avanzamento e log propri. Al massimo un run per Workspace Database può essere attivo.
|
||||
|
||||
**Description Generation Run** — L'esecuzione asincrona e sequenziale che usa il modello scelto
|
||||
per produrre Generated Description di Catalog Table o Catalog Column. Al massimo una run è attiva
|
||||
nell'intera installazione e ogni risultato valido viene salvato appena disponibile. Dopo
|
||||
un'interruzione il recupero è manuale tramite una nuova generazione dei soli elementi mancanti.
|
||||
|
||||
**Description Generation Event** — Una riga testuale ordinata che registra avanzamento, risultato
|
||||
o errore di una Description Generation Run e alimenta il log visibile all'amministratore.
|
||||
|
||||
**Non-generatable Description** — L'esito valido con cui il modello dichiara di non disporre di
|
||||
informazioni sufficienti per descrivere il target. Produce una Generated Description standard
|
||||
nella lingua del workspace e non rappresenta un timeout, un errore del provider o una risposta
|
||||
non valida.
|
||||
|
||||
**Description Generation Unlock** — Il recupero amministrativo che marca come interrotta una
|
||||
Description Generation Run registrata come attiva quando il backend non ha alcun processo di
|
||||
generazione vivo. Non è un meccanismo di lock distribuito.
|
||||
|
||||
**Catalog Metadata Cleanup** — La rimozione amministrativa esplicita di Catalog Table, Catalog
|
||||
Column o Catalog Relationship selezionate. Non modifica il Workspace Database, la Database Binding
|
||||
o i segreti, e può lasciare il Metadata Catalog intenzionalmente incompleto fino alla prossima
|
||||
Schema Synchronization.
|
||||
|
||||
**Catalog Freshness** — La corrispondenza fra uno scope sincronizzato e la versione corrente
|
||||
della Database Binding. Uno scope rimane consultabile ma è stale finché non viene sincronizzato
|
||||
con la binding corrente.
|
||||
|
||||
**Metadata Content Revision** — La revisione monotona di tutto lo stato del Metadata Catalog che
|
||||
può modificare il comportamento del core. Ogni mutazione rilevante produce una nuova revisione
|
||||
nella stessa transazione che la rende durevole.
|
||||
|
||||
**Preprocessing State** — Lo stato corrente `running`, `succeeded` o `failed` del preprocessing di
|
||||
un workspace, insieme all'identità dei suoi input. Il core può usare il workspace soltanto quando
|
||||
lo stato è `succeeded` e gli input coincidono ancora.
|
||||
|
||||
**Catalog Metadata** — I campi mutabili che descrivono database, tabelle, colonne e relazioni,
|
||||
distinti dai fatti strutturali governati dalla sincronizzazione. Possono essere popolati dall'AI,
|
||||
o da una modifica amministrativa senza cambiare il database esterno.
|
||||
|
||||
**Model Completion Helper** — Il processo Python interno ed effimero che esegue una singola
|
||||
richiesta LiteLLM per conto del backend. Non è un servizio HTTP, non possiede il lifecycle della
|
||||
Description Generation Run e non è una CLI esposta agli utenti.
|
||||
|
||||
**Catalog Sample** — Un input transitorio composto da un massimo di cinque righe e da valori di
|
||||
esempio bounded di una Catalog Table per la generazione delle descrizioni. Può contenere valori
|
||||
reali oppure sintetici in base alla Source Value Disclosure Decision; non viene persistito e non
|
||||
diventa Catalog Metadata.
|
||||
|
||||
**Sensitive Data Flag** — La classificazione binaria umana applicata a una Catalog Column. Può
|
||||
essere impostata liberamente dall'amministratore anche in contrasto con una valutazione automatica.
|
||||
|
||||
**Sensitivity Reason** — La motivazione sanificata persistita insieme al Sensitive Data Flag
|
||||
quando l'amministratore salva una Sensitivity Review Draft. È Catalog Metadata della colonna, non
|
||||
history della run; viene rimossa quando il flag torna non-sensitive e può essere assente per una
|
||||
classificazione manuale priva di valutazione locale.
|
||||
_Avoid_: AI reasoning, source evidence
|
||||
|
||||
**Local Sensitivity Assessment** — La valutazione locale, non autoritativa e priva di LLM di una
|
||||
Catalog Column, basata su metadati e contenuto sorgente, con esito `sensitive`, `non_sensitive`
|
||||
oppure `unknown`.
|
||||
_Avoid_: AI suggestion, automatic flag
|
||||
|
||||
**Local NER Detector** — Il componente NLP opzionale e CPU-only che esamina soltanto testo ancora
|
||||
ambiguo e restituisce evidenze al Local Sensitivity Assessment. Non decide lo stato della colonna,
|
||||
non usa un LLM generativo e non persiste valori sorgente.
|
||||
_Avoid_: AI classifier, local LLM fallback
|
||||
|
||||
**Model Data Boundary** — La qualificazione amministrativa di un modello come `internal` oppure
|
||||
`external` rispetto al confine entro cui i valori sorgente possono essere comunicati.
|
||||
_Avoid_: local model, remote model
|
||||
|
||||
**Source Value Disclosure Decision** — L'unica decisione effettiva che stabilisce se un modello
|
||||
riceve valori sorgente reali oppure sostituti sintetici, combinando Model Data Boundary e Sensitive
|
||||
Data Flag.
|
||||
_Avoid_: sample filter, export flag
|
||||
|
||||
**Sensitive Data Policy** — L'insieme versionato di regole locali generali e specifiche che produce
|
||||
una Local Sensitivity Assessment. Un singolo riscontro blocca l'intera colonna e qualsiasi valore
|
||||
testuale più lungo di 500 caratteri rende sensibile la colonna.
|
||||
_Avoid_: PII filter, sample filter
|
||||
|
||||
**Sensitivity Analysis Run** — Il tentativo amministrativo esplicito e tracciato che valuta una
|
||||
selezione di colonne mediante la Sensitive Data Policy. Conserva stato, copertura e conteggi
|
||||
aggregati, ma non valori sorgente né esiti per colonna.
|
||||
_Avoid_: Sensitive Data Suggestion Run, AI analysis
|
||||
|
||||
**Sensitivity Review Draft** — La proposta transitoria che associa alle colonne selezionate una
|
||||
Local Sensitivity Assessment e le relative evidenze sanificate. Non modifica il Sensitive Data Flag
|
||||
né la Sensitivity Reason finché l'amministratore non salva le proprie decisioni e viene scartata al
|
||||
reload.
|
||||
_Avoid_: automatic flag
|
||||
|
||||
**Sensitivity Analysis Event** — Una riga testuale ordinata e sanificata che registra l'avvio,
|
||||
l'avanzamento per fase e batch, l'esito o l'errore di una Sensitivity Analysis Run senza conservare
|
||||
contenuti sorgente, output grezzi del detector o proposte per colonna.
|
||||
_Avoid_: Sensitive Data Suggestion Event
|
||||
|
||||
**Introspection Capability** — Una categoria di struttura fisica che una Database Binding
|
||||
può osservare, come tabelle, colonne, relazioni, indici o enum. Una capability non disponibile
|
||||
è distinta da una capability osservata che non ha restituito elementi.
|
||||
|
||||
## Amministrazione e integrazione
|
||||
|
||||
**Workspace Readiness** — La preparazione di uno specifico Workspace per l'uso nel
|
||||
workflow, comprensiva della disponibilità degli artefatti derivati dai suoi metadati
|
||||
Database e dalle sue Evidence. Il preprocessing appartiene a questa preparazione;
|
||||
la configurazione e la sincronizzazione del catalogo restano responsabilità Database.
|
||||
|
||||
**Administration Surface** — Una superficie amministrativa autonoma per configurare o curare una
|
||||
parte dell'installazione. Workspace, Evidence, Memory, Database e Pi sono superfici peer e non
|
||||
dipendono dall'esistenza di una sessione attiva.
|
||||
|
||||
**Administration Page** — La rappresentazione a pagina intera di una Administration Surface, con
|
||||
una gerarchia condivisa per identità, stato, azioni e contenuto. Un form amministrativo appartiene
|
||||
alla pagina e non a una popup come contenitore principale.
|
||||
_Avoid_: management popup, settings modal
|
||||
|
||||
**Administration Route** — L'identità navigabile di una Administration Surface nel browser. Deve
|
||||
essere ripristinabile con refresh e cronologia e non contiene valori transitori o segreti dei form.
|
||||
|
||||
**Embedded Thoth Shell** — L'esperienza Thoth ospitata dentro il documento e il contesto visuale di
|
||||
un portale host. Conserva la propria gerarchia funzionale, ma deve rispettare la geometria,
|
||||
l'autenticazione e le regole responsive del portale host.
|
||||
|
||||
**Full Thoth Shell** — L'esperienza Thoth autonoma che possiede il proprio header e il proprio
|
||||
layout di pagina. Non replica la navigazione amministrativa del portale host e non dipende dal suo
|
||||
template visuale.
|
||||
|
||||
**Shell mode** — La scelta di installazione fra `embedded` e `full`. Determina chi possiede il
|
||||
chrome globale, i comandi di identità e le integrazioni visuali, ma non cambia il workflow o la
|
||||
persistenza delle sessioni.
|
||||
|
||||
**Fullscreen state** — Lo stato temporaneo in cui il documento applicativo occupa il fullscreen
|
||||
del browser. È distinto da `Shell mode`: una Full Thoth Shell può essere aperta senza fullscreen;
|
||||
il passaggio è attivato da un comando esplicito e può essere annullato con la stessa azione o con
|
||||
il comando nativo del browser.
|
||||
|
||||
**Portal Shell Adapter** — Il confine sostituibile che traduce lo stato e i comandi del chrome di
|
||||
un portale host nel modello semantico usato da Thoth. L'adapter non possiede autorizzazione,
|
||||
sessioni di workflow o contenuti del modello.
|
||||
|
||||
**Host Shell State** — Il minimo stato visuale fornito dal portale host: locale UI, tema e stato
|
||||
fullscreen. In una Embedded Thoth Shell è la fonte autorevole per queste preferenze;
|
||||
non include identità, token o stato di autenticazione, che restano responsabilità dell'accesso.
|
||||
|
||||
**UI locale** — La lingua delle label, dei messaggi, dei tooltip, degli stati e delle istruzioni
|
||||
non generate dal modello nell'interfaccia Thoth. È distinta dalla lingua dei contenuti di un
|
||||
workspace.
|
||||
|
||||
**Interaction language** — La lingua in cui il modello presenta domande, spiegazioni e proposte
|
||||
al revisore durante una sessione. Viene fissata alla creazione della sessione e rimane invariata
|
||||
durante una ripresa, anche se la UI locale corrente cambia.
|
||||
|
||||
**Administrative Page Family** — L'insieme delle cinque Administration Page che condividono shell,
|
||||
navigazione, tipografia e regole responsive, pur mantenendo contenuti e operazioni specifici:
|
||||
Workspace, Evidence, Memory, Database e Pi.
|
||||
|
||||
## Installazione
|
||||
|
||||
**Manual standalone installation** — Una copia di ThothII predisposta per l'uso autonomo da una
|
||||
persona che possiede il computer, con una Full Thoth Shell e servizi applicativi locali. La
|
||||
procedura non implica che DWH o provider LLM siano locali o disponibili offline.
|
||||
|
||||
**Installation bootstrap** — L'insieme delle attività iniziali che rende disponibile una
|
||||
installazione manuale: verifica dell'host, generazione della configurazione, predisposizione
|
||||
delle credenziali protette e avvio dei servizi. Non è un installer dell'applicazione.
|
||||
|
||||
**Platform acceptance** — La verifica che una Manual standalone installation possa essere
|
||||
predisposta e avviata su una specifica combinazione di sistema operativo, architettura e runtime,
|
||||
distinta dalla verifica funzionale del collegamento a DWH e provider LLM.
|
||||
@@ -0,0 +1,465 @@
|
||||
---
|
||||
name: ThothII
|
||||
description: "A calm, precise clinical analytics workbench for traceable and reviewable SQL workflows."
|
||||
colors:
|
||||
instrument-red: "oklch(55.87% 0.1881 23.2)"
|
||||
instrument-red-hover: "oklch(50.95% 0.1812 24.1)"
|
||||
porcelain-background: "oklch(99.18% 0.0011 17.2)"
|
||||
porcelain-card: "oklch(99.85% 0.0006 17.2)"
|
||||
warm-surface: "oklch(97.09% 0.0011 17.2)"
|
||||
sunken-surface: "oklch(94.08% 0.0011 17.2)"
|
||||
warm-graphite: "oklch(26.78% 0.0097 355.6)"
|
||||
muted-graphite: "oklch(51.33% 0.0088 345.6)"
|
||||
quiet-border: "oklch(90.93% 0.0035 354.7)"
|
||||
success-mint: "oklch(46% 0.095 160)"
|
||||
navigation-active: "oklch(92.5% 0.052 23.2)"
|
||||
navigation-active-hover: "oklch(89.5% 0.071 23.2)"
|
||||
navigation-active-foreground: "oklch(36.5% 0.11 23.2)"
|
||||
navigation-active-border: "oklch(60% 0.135 23.2)"
|
||||
warning-amber: "oklch(48% 0.09 70)"
|
||||
information-neutral: "oklch(51.33% 0.0088 345.6)"
|
||||
typography:
|
||||
display:
|
||||
fontFamily: "Manrope Variable, Manrope, system-ui, sans-serif"
|
||||
fontSize: "1.5rem"
|
||||
fontWeight: 600
|
||||
lineHeight: 1.03
|
||||
letterSpacing: "-0.025em"
|
||||
headline:
|
||||
fontFamily: "Manrope Variable, Manrope, system-ui, sans-serif"
|
||||
fontSize: "1.5rem"
|
||||
fontWeight: 600
|
||||
lineHeight: 1.15
|
||||
letterSpacing: "-0.015em"
|
||||
title:
|
||||
fontFamily: "Manrope Variable, Manrope, system-ui, sans-serif"
|
||||
fontSize: "1.25rem"
|
||||
fontWeight: 600
|
||||
lineHeight: 1.25
|
||||
letterSpacing: "-0.01em"
|
||||
body:
|
||||
fontFamily: "Manrope Variable, Manrope, -apple-system, BlinkMacSystemFont, Segoe UI, system-ui, Arial, sans-serif"
|
||||
fontSize: "1rem"
|
||||
fontWeight: 400
|
||||
lineHeight: 1.65
|
||||
letterSpacing: "normal"
|
||||
control:
|
||||
fontFamily: "Manrope Variable, Manrope, -apple-system, BlinkMacSystemFont, Segoe UI, system-ui, Arial, sans-serif"
|
||||
fontSize: "0.875rem"
|
||||
fontWeight: 600
|
||||
lineHeight: 1.25
|
||||
letterSpacing: "0.005em"
|
||||
label:
|
||||
fontFamily: "Manrope Variable, Manrope, system-ui, sans-serif"
|
||||
fontSize: "0.75rem"
|
||||
fontWeight: 600
|
||||
lineHeight: 1.25
|
||||
letterSpacing: "normal"
|
||||
rounded:
|
||||
xs: "4px"
|
||||
sm: "6px"
|
||||
md: "8px"
|
||||
lg: "12px"
|
||||
xl: "16px"
|
||||
full: "9999px"
|
||||
spacing:
|
||||
xs: "4px"
|
||||
sm: "8px"
|
||||
md: "16px"
|
||||
lg: "24px"
|
||||
xl: "32px"
|
||||
components:
|
||||
button-primary:
|
||||
backgroundColor: "{colors.instrument-red}"
|
||||
textColor: "{colors.porcelain-background}"
|
||||
typography: "{typography.control}"
|
||||
rounded: "{rounded.md}"
|
||||
padding: "0 14px"
|
||||
height: "32px"
|
||||
button-primary-hover:
|
||||
backgroundColor: "{colors.instrument-red-hover}"
|
||||
textColor: "{colors.porcelain-background}"
|
||||
typography: "{typography.control}"
|
||||
rounded: "{rounded.md}"
|
||||
padding: "0 14px"
|
||||
height: "32px"
|
||||
button-secondary:
|
||||
backgroundColor: "{colors.porcelain-card}"
|
||||
textColor: "{colors.warm-graphite}"
|
||||
typography: "{typography.control}"
|
||||
rounded: "{rounded.md}"
|
||||
padding: "0 14px"
|
||||
height: "32px"
|
||||
input-default:
|
||||
backgroundColor: "{colors.porcelain-background}"
|
||||
textColor: "{colors.warm-graphite}"
|
||||
typography: "{typography.body}"
|
||||
rounded: "{rounded.md}"
|
||||
padding: "0 12px"
|
||||
height: "40px"
|
||||
card-default:
|
||||
backgroundColor: "{colors.porcelain-card}"
|
||||
textColor: "{colors.warm-graphite}"
|
||||
rounded: "{rounded.lg}"
|
||||
padding: "16px"
|
||||
badge-primary:
|
||||
backgroundColor: "{colors.instrument-red}"
|
||||
textColor: "{colors.porcelain-background}"
|
||||
typography: "{typography.control}"
|
||||
rounded: "{rounded.sm}"
|
||||
padding: "2px 8px"
|
||||
height: "20px"
|
||||
---
|
||||
|
||||
# Design System: ThothII
|
||||
|
||||
## Visual review branch, September 2026
|
||||
|
||||
The revision on `codex/ui-visual-review` is approved for implementation and Docker visual review,
|
||||
not yet for adoption on `main`. The previous look remains recoverable from the base commit and
|
||||
the preserved Docker image. Historical prototypes must remain untouched.
|
||||
|
||||
This revision follows Impeccable's product register: one locally bundled Manrope family for the
|
||||
whole UI, five fixed size roles, red as the sole brand accent and additional color only for meaningful
|
||||
state. The primary scene remains an analyst reading data and SQL in a well-lit office.
|
||||
|
||||
## Overview
|
||||
|
||||
**Creative North Star: "The Clinical Workbench"**
|
||||
|
||||
ThothII should feel like a well-kept clinical workbench: warm enough for sustained reading, exact
|
||||
enough for consequential review, and quiet enough that evidence, state, and decisions remain in the
|
||||
foreground. The visual system is calm, precise, and trustworthy. It uses familiar product patterns,
|
||||
restrained color, and deliberate density instead of decorative spectacle.
|
||||
|
||||
The primary physical scene is an analyst reviewing persisted evidence and SQL on a large monitor in
|
||||
a well-lit working environment. This makes the warm light theme the default. The supported dark
|
||||
theme serves lower-light work without becoming a separate neon aesthetic. Both themes preserve the
|
||||
same hierarchy and semantic roles.
|
||||
|
||||
The system rejects generic SaaS ornament, conspicuous ripples, bounce or elastic motion, long
|
||||
choreographed transitions, and effects that compete with the analytical task. Controls should feel
|
||||
disciplined and tactile, never playful, sluggish, or visually unstable.
|
||||
|
||||
**Key Characteristics:**
|
||||
|
||||
- Warm, restrained surfaces with one scarce red accent.
|
||||
- One sans-serif family, with hierarchy expressed through size, weight and spacing.
|
||||
- Dense information organized through hierarchy, rhythm, and progressive disclosure.
|
||||
- Persisted artifacts and reviewer decisions presented as the visual source of truth.
|
||||
- Fast state feedback with reduced-motion parity.
|
||||
|
||||
**The Workbench Rule.** Every visual element must support inspection, action, state, or provenance.
|
||||
Decoration without an operational purpose is forbidden.
|
||||
|
||||
**The Persisted Truth Rule.** Persisted artifacts and reviewer decisions receive stronger hierarchy
|
||||
than transient model narration.
|
||||
|
||||
**The Density with Rhythm Rule.** Preserve information density, but vary spacing between groups so
|
||||
users can scan structure without adding nested containers.
|
||||
|
||||
## Colors
|
||||
|
||||
The full-mode application header matches Omics Portal's `--gsd-red-primary`
|
||||
(`#CB333B`) in both themes. Its complete wordmark, including `II`, and controls
|
||||
use a near-white foreground. This header is absent in embedded mode. The sidebar
|
||||
and welcome wordmarks retain their red suffix. Context editing places workspace,
|
||||
model and Done in one desktop row, stacking on narrow containers. Session-scope
|
||||
tabs retain their selected fill and accessible keyboard state with a uniform one-pixel
|
||||
border on every side, gray when inactive and red when active. Their padding is 11px
|
||||
horizontal and 3px vertical, with a 38px minimum height and wrapping labels.
|
||||
|
||||
The palette combines warm porcelain surfaces, warm graphite text, and an instrument red used only
|
||||
for action, focus, and important state. OKLCH values in the frontmatter are normative because the
|
||||
frontend uses OKLCH tokens directly.
|
||||
|
||||
### Primary
|
||||
|
||||
- **Instrument Red** (`instrument-red`): primary actions, focus identity, and destructive meaning
|
||||
where the context already makes the action explicit.
|
||||
- **Instrument Red Pressed** (`instrument-red-hover`): hover and active emphasis for the primary
|
||||
action family.
|
||||
|
||||
### Neutral
|
||||
|
||||
- **Porcelain Background** (`porcelain-background`): the main canvas.
|
||||
- **Porcelain Card** (`porcelain-card`): lifted panels, cards, and popovers.
|
||||
- **Warm Surface** (`warm-surface`): sidebars, secondary controls, and muted regions.
|
||||
- **Sunken Surface** (`sunken-surface`): selected rows, quiet emphasis, and inset regions.
|
||||
- **Warm Graphite** (`warm-graphite`): primary text and high-confidence labels.
|
||||
- **Muted Graphite** (`muted-graphite`): descriptions, timestamps, and secondary metadata.
|
||||
- **Quiet Border** (`quiet-border`): structural boundaries, input outlines, and dividers.
|
||||
|
||||
### Semantic
|
||||
|
||||
- **Success Mint** (`success-mint`): completed and ready states.
|
||||
- **Navigation Active** (`navigation-active`): the one application surface currently in the
|
||||
foreground. It shares Instrument Red's hue but uses a lighter, lower-chroma fill, so location is
|
||||
visible without carrying the full weight of a primary action.
|
||||
- **Warning Amber** (`warning-amber`): waiting, attention, and in-progress states.
|
||||
- **Information**: neutral text and indicators for dates, protocols and ordinary status. The legacy
|
||||
`--info` token resolves to muted foreground, not an additional blue accent.
|
||||
|
||||
The dark theme keeps the same semantic mapping with neutral near-black surfaces and a slightly
|
||||
lighter red accent. Do not introduce a second visual identity for dark mode.
|
||||
|
||||
**The One Voice Rule.** Instrument Red should occupy no more than roughly ten percent of a screen.
|
||||
Its rarity is what makes it authoritative.
|
||||
|
||||
**The State Has a Name Rule.** Success, warning, information, and destructive colors are reserved
|
||||
for their named states. Color is never the only state indicator.
|
||||
|
||||
## Typography
|
||||
|
||||
**UI Font:** locally bundled Manrope Variable, with Manrope and native sans-serif fallbacks.
|
||||
**Technical Font:** SF Mono or Cascadia Code, with Menlo and Consolas fallbacks.
|
||||
|
||||
Manrope covers headings, labels, controls, navigation and document reading. Monospace is reserved
|
||||
for SQL, code, paths and machine identifiers, never for ordinary UI labels or status headings.
|
||||
|
||||
### Hierarchy
|
||||
|
||||
- **Headline** (600, `1.5rem`, `1.3`): page or artifact titles, `--text-page`.
|
||||
- **Title** (600, `1.25rem`, `1.4`): section hierarchy, `--text-section`.
|
||||
- **Body** (400, `1rem`, `1.6`): operational prose, `--text-body`, with a target line length of 65 to 75
|
||||
characters where the surface controls width.
|
||||
- **Control** (400–600, `0.875rem`, `1.5`): buttons, inputs, tables, tabs and compact subheadings,
|
||||
`--text-control`.
|
||||
- **Metadata** (400–600, `0.75rem`, `1.5`): secondary status, counts and timestamps, `--text-meta`.
|
||||
Labels use sentence case and normal tracking. Ordinary operational text never falls below 12px.
|
||||
|
||||
Typography uses fixed sizes. Responsive changes happen at structural breakpoints, not through fluid
|
||||
type scaling. Numeric data and identifiers use tabular numerals where comparison matters.
|
||||
|
||||
**Application wordmark:** ThothII is a brand mark, not a page title: use Manrope semibold at
|
||||
48px (`3rem`) in the Core welcome area and 32px (`2rem`) in the session sidebar, with the
|
||||
`II` suffix in brand red. Preserve these sizes across responsive layouts.
|
||||
|
||||
**The One Family Rule.** The UI and document readers use sans-serif throughout. The legacy
|
||||
`--font-heading` alias resolves to `--font-sans`. Preserve technical monospace without turning it
|
||||
into a second decorative hierarchy. Do not shrink text to solve layout constraints.
|
||||
|
||||
**The Read Once Rule.** A heading, label, and body must be distinguishable on first glance through
|
||||
size and weight. Do not repeat headings in explanatory copy.
|
||||
|
||||
## Elevation
|
||||
|
||||
The system is flat by default and layered when necessary. Borders mark structure. Warm, diffuse
|
||||
shadows mark actual elevation for popovers, dialogs, and selected containers. Tonal layering should
|
||||
solve most hierarchy before a shadow is introduced.
|
||||
|
||||
### Shadow Vocabulary
|
||||
|
||||
- **Contact Shadow** (`--shadow-xs`): a one-pixel contact shadow for controls and code blocks.
|
||||
- **Panel Shadow** (`--shadow-sm`): a small two-stage shadow for cards that need separation from the
|
||||
canvas.
|
||||
- **Overlay Shadow** (`--shadow-md`): a broad, low-opacity shadow for dialogs and floating layers.
|
||||
|
||||
Focus uses an explicit three-pixel ring. Waiting-for-input state may use a success-tinted ring, but
|
||||
must retain a textual or structural cue. Motion for button state changes lasts `140ms` with
|
||||
`cubic-bezier(0.22, 1, 0.36, 1)`. Dialog transitions last `100ms`. Activity pulses may run at
|
||||
`1.5s`, and must be disabled under `prefers-reduced-motion`.
|
||||
|
||||
**The Flat by Default Rule.** A resting surface has no shadow unless it is physically above another
|
||||
surface. If every panel floats, none of them has hierarchy.
|
||||
|
||||
**The Borders Structure, Shadows Elevate Rule.** Never use shadow as a substitute for grouping or a
|
||||
border as a decorative accent.
|
||||
|
||||
## Components
|
||||
|
||||
Components are familiar, compact, and state-complete. Every interactive primitive must define
|
||||
default, hover, focus, active, disabled, loading, and error behavior where those states apply.
|
||||
|
||||
### Buttons
|
||||
|
||||
- **Shape:** gently curved rectangle (`8px`) with a one-pixel transparent or structural border.
|
||||
- **Primary:** Instrument Red, porcelain text, `32px` default height, and `14px` horizontal padding.
|
||||
- **Hover / Focus:** shift to Instrument Red Pressed; show a three-pixel focus ring at 25 percent
|
||||
opacity. Active state scales to `0.97` for `140ms` and removes elevation.
|
||||
- **Secondary / Outline:** porcelain card surface, Quiet Border, Warm Graphite text, and a Warm
|
||||
Surface hover.
|
||||
- **Ghost:** transparent at rest, Warm Surface on hover. Use only where surrounding structure makes
|
||||
the hit target obvious.
|
||||
|
||||
### Badges and Status Indicators
|
||||
|
||||
- **Style:** compact (`20px` height), gently curved (`6px`), and semibold.
|
||||
- **State:** pair semantic color with text, icon, or position. A colored dot alone is insufficient
|
||||
when the state affects workflow decisions.
|
||||
|
||||
### Cards and Containers
|
||||
|
||||
- **Corner Style:** softly rounded (`12px`), with `16px` default internal padding.
|
||||
- **Background:** Porcelain Card over Porcelain Background or Warm Surface.
|
||||
- **Shadow Strategy:** Panel Shadow only when the card must read as elevated.
|
||||
- **Border:** one-pixel Quiet Border at partial opacity.
|
||||
- **Nesting:** nested cards are forbidden. Use headings, dividers, spacing, or tonal regions.
|
||||
|
||||
### Inputs and Fields
|
||||
|
||||
- **Style:** `40px` height, `8px` corners, Porcelain Background, Quiet Border, and Manrope body text.
|
||||
- **Focus:** three-pixel Instrument Red ring with a clear border shift.
|
||||
- **Error / Disabled:** errors combine destructive color with explanatory text; disabled controls
|
||||
retain readable contrast and use 50 percent opacity.
|
||||
- **Global context:** the collapsible top shelf is the sole workspace/model selector for Core and
|
||||
Admin. Preserve independent remembered choices, installation defaults, operation locks and unsaved
|
||||
edit guards. Never introduce a separate metadata-generation default or selector.
|
||||
|
||||
### Navigation
|
||||
|
||||
- **Workspace readiness:** the Workspace navigation button carries an 8px dot to
|
||||
the right of its label. Green means a selected workspace with confirmed ready
|
||||
preprocessing and no query error; all other states are red. The button's
|
||||
tooltip and accessible description retain the translated exact state. Do not
|
||||
add a separate readiness text row or change the backend readiness gate.
|
||||
- **Session groups:** one accessible single-open accordion contains Active sessions
|
||||
and Archive, both initially closed. Below the scope tabs, show only their
|
||||
adjacent section headers, without a redundant Sessions heading. Selection and
|
||||
bulk-delete controls belong inside each panel and only appear for nonempty
|
||||
lists. Select all affects that list only, preserves the other list's selection,
|
||||
and exposes a mixed state for partial selection. Preserve the existing archived
|
||||
flag as the grouping rule, independent of whether a Pi process is running.
|
||||
Opening a section closes the other; either can be collapsed, including both.
|
||||
Empty lists show only the translated "No sessions yet." message.
|
||||
The open section uses the rail's remaining height; its list scrolls internally
|
||||
with a cap of `min(18rem, 35dvh)`, while its trigger remains outside that scroll
|
||||
area. The mobile navigation dialog supplies a bounded viewport-height container.
|
||||
Keyboard users can focus and scroll each labelled panel.
|
||||
- **Session entry:** one Session button returns to the current unfinished session,
|
||||
including provisional creation, without resetting or reconnecting it. Otherwise
|
||||
it prepares a new question using the normal readiness and unsaved-work guards.
|
||||
- **Style:** compact session rows use `8px` corners and restrained vertical padding.
|
||||
- **Default / Hover / Active:** porcelain at rest, Sunken Surface on hover, and a muted Navigation
|
||||
Active red with a defined border when current. Exactly one top-level navigation control is current.
|
||||
- **Administrative controls:** the admin-only Administration accordion groups Database,
|
||||
Memory, Evidence, a structural divider, Workspace, and Pi configuration in that order. Its trigger exposes
|
||||
expanded state and starts collapsed by default, while non-admin users do not receive the accordion
|
||||
or its navigation actions.
|
||||
- **Responsive:** collapse navigation structurally at the application breakpoint. Do not shrink
|
||||
labels into illegibility. Below 768px, Memory and Evidence management use the full content
|
||||
width; a Navigation button opens the shared accessible dialog. Selecting another archive
|
||||
page or pressing Escape closes it. Desktop retains the right session sidebar and its My sessions /
|
||||
All sessions tabs. Core retains question/answer, eight phases, reviewer gates and the left log.
|
||||
In embedded mode the portal owns the red header and left sidebar; ThothII must not duplicate them. Size to the
|
||||
actual application container. Narrow session document panels may use the available width.
|
||||
|
||||
### Session review and confirmations
|
||||
|
||||
Session dialogs use the visible application area, including the portal's header
|
||||
and side rail. Artifact and schema-column review can grow to 80rem wide and the
|
||||
available height; short confirmations use up to 40rem and at least 18rem when
|
||||
space permits. Keep a 24px outer margin on desktop and 8px on small or short
|
||||
screens. Long review content scrolls internally; on very short screens the
|
||||
whole dialog can also scroll so every action remains reachable.
|
||||
|
||||
Session forms and review gates repeat their existing primary confirmation above
|
||||
and below the content, sharing selection, validation, pending state and response
|
||||
handlers. Alternate-response inputs follow the same rule. Reserved navigation
|
||||
controls remain below the review. Stop/delete initially focus Cancel; rename
|
||||
initially focuses the name field. Administration dialogs and forms retain their
|
||||
existing layout and actions.
|
||||
|
||||
### Tabs
|
||||
|
||||
- **Shape:** compact label tabs sit on a shared baseline with rounded top corners and a two-pixel
|
||||
lower edge, except session-scope tabs which use a uniform one-pixel border, rounded
|
||||
corners and a 4px gap without a shared border or negative bottom margin.
|
||||
Inactive labels retain a Quiet Border and Porcelain Card surface, so every
|
||||
label reads as a tab before interaction; hover feedback reinforces clickability.
|
||||
- **Current:** the selected tab uses the muted Navigation Active red for its fill, text, and defined border.
|
||||
It must expose `aria-selected`, participate in a labelled `tablist`/`tabpanel`, and be the only
|
||||
tab in the roving keyboard tab order.
|
||||
- **Keyboard:** Left/Right move between adjacent tabs with wrapping; Home/End select the first or
|
||||
last tab.
|
||||
|
||||
### Tooltips
|
||||
|
||||
- **Row actions:** icon-action tooltips open three pixels below the trigger and align to its trailing
|
||||
edge, so they never cover the icon row. They use a dark slate surface, porcelain text, and a
|
||||
defined border rather than the light popover treatment.
|
||||
- **Interaction:** tooltip layers never receive pointer events. They appear on hover and keyboard
|
||||
focus with a short ease-out transition, while the icon button keeps its complete accessible name.
|
||||
- **Scope:** this treatment is shared by database, table, column, and relationship row actions.
|
||||
Toolbar and navigation hints may use separate collision-aware placement.
|
||||
|
||||
### Curated Evidence Documents
|
||||
|
||||
Memory and Evidence share the `thot-knowledge-reader` reading contract. Use locally
|
||||
bundled Manrope with normal tracking for prose and labels, and these fixed roles:
|
||||
|
||||
- Card title: 24px, weight 600, line-height 1.3 (`thot-knowledge-title`).
|
||||
- Field/section heading, including Scope and Provenance: 20px, weight 600,
|
||||
line-height 1.4, 8px clearance below (`thot-knowledge-heading`).
|
||||
- All narrative text, including scope, lists and provenance: 16px, weight 400,
|
||||
line-height 1.65. Do not apply compact UI text sizes to these fields.
|
||||
- Authored Markdown subheadings inside a field: 16px, weight 600, line-height 1.5,
|
||||
24px above/8px below. They remain subordinate to the enclosing field heading;
|
||||
their semantic heading levels and original content are preserved.
|
||||
- Technical metadata labels/values: 14px/1.5, with weight 600 for labels.
|
||||
Only code, paths and machine identifiers use the technical monospace family at
|
||||
14px/1.65, identical for inline and fenced code (never compound `em` shrinkage).
|
||||
|
||||
Separate reading sections by 24px; keep the first Markdown block flush with its
|
||||
field heading's 8px bottom gap. The same typography applies in light/dark and at
|
||||
all responsive widths. Controls and archive indexes retain their compact UI roles.
|
||||
|
||||
Memory and Evidence detail readers use the entire available content width, without
|
||||
the ordinary 72–75ch prose cap. This is the owner's explicit reading-layout choice.
|
||||
Long unstructured paragraphs are split for display at existing sentence/semicolon
|
||||
boundaries outside inline code and links; authored Markdown structure and stored
|
||||
content are unchanged. Paragraph spacing is 1.25em. Scope and provenance share the
|
||||
available width; provenance excerpts render Markdown rather than literal markers.
|
||||
Copy actions use the two-overlapping-sheets icon, an accessible name/tooltip and
|
||||
live success/failure feedback instead of a visible Copy label.
|
||||
|
||||
Memory has four explicitly FAKE formatting examples, one per family, in a separate
|
||||
expandable section. They reuse the real detail reader but never enter persistence,
|
||||
indexing, link search or model recall, and expose no edit/delete/save actions.
|
||||
|
||||
Curated evidence follows a fixed reading order: title, compact type and purpose summary, scope,
|
||||
typed content, supporting excerpts, review items, then technical provenance. Curated v4 files
|
||||
use short, visible YAML frontmatter for identity and classification. The Markdown title and
|
||||
body are authoritative; hidden payload comments are a legacy format converted on consolidation.
|
||||
|
||||
`applies_to` is rendered as “Ambito di applicazione” with separate bullet lists for concepts,
|
||||
tables, and columns. Enum values also use lists. Tables are forbidden for metadata, scope, or any
|
||||
one-dimensional collection; reserve tables for genuinely two-dimensional datasets. Long machine
|
||||
identifiers use inline code. SQL uses fenced code. Supporting excerpts use blockquotes.
|
||||
|
||||
**The Review Surface Rule.** The visible Markdown must be readable without understanding the
|
||||
machine contract. In Administration, explain current and original provenance separately and
|
||||
keep file-editing templates and Git instructions in progressive disclosure. Show actual host
|
||||
paths with copy controls, never browser file links to container-only locations.
|
||||
|
||||
## Do's and Don'ts
|
||||
|
||||
### Do:
|
||||
|
||||
- **Do** make every state change unmistakable without interrupting flow.
|
||||
- **Do** use Instrument Red only for primary action, current selection, focus identity, or explicit
|
||||
destructive meaning.
|
||||
- **Do** preserve information density with headings, rhythm, and progressive disclosure.
|
||||
- **Do** keep keyboard focus explicit and pair color with text, shape, icon, or position.
|
||||
- **Do** respect `prefers-reduced-motion` while preserving immediate non-kinetic feedback.
|
||||
- **Do** use the selected interface language (English by default) for chrome and preserve the
|
||||
workspace language for persisted domain content. Session interaction language remains pinned.
|
||||
- **Do** render curated metadata and scope as Markdown prose or lists, never as a frontmatter table.
|
||||
- **Do** break long curated rules into paragraphs, labelled subsections, and lists at existing
|
||||
punctuation boundaries while preserving the exact canonical text for machines.
|
||||
|
||||
### Don't:
|
||||
|
||||
- **Don't** add generic SaaS ornament, conspicuous ripples, bounce or elastic motion, long
|
||||
choreographed transitions, or effects that compete with the analytical task.
|
||||
- **Don't** make controls feel playful, sluggish, or visually unstable.
|
||||
- **Don't** use gradient text, decorative glassmorphism, or full-saturation accents on inactive
|
||||
states.
|
||||
- **Don't** use a colored side stripe greater than one pixel on cards, callouts, list items, or
|
||||
blockquotes. Use a full border, tonal background, icon, or heading instead.
|
||||
- **Don't** nest cards or wrap every section in a container.
|
||||
- **Don't** use a modal before exhausting inline or progressive alternatives.
|
||||
- **Don't** use tables for `applies_to`, metadata, enum values, or other one-dimensional content.
|
||||
- **Don't** use color as the sole carrier of success, warning, error, selection, or progress.
|
||||
- **Don't** use display typography for buttons, labels, or data.
|
||||
- **Don't** add em dashes to interface copy. Use commas, colons, semicolons, or parentheses.
|
||||
+101
-934
File diff suppressed because it is too large
Load Diff
@@ -1,84 +1,110 @@
|
||||
# ThothII
|
||||
|
||||
ThothII is a human-reviewed NL-to-SQL workflow with a React frontend and a Fastify/Pi/`tht`
|
||||
core. The portable deployment runs exactly two application services; data services remain
|
||||
external in this profile, except for the mandatory internal semantic services bundled in Compose.
|
||||
core. The portable deployment runs two application services plus the installation-local metadata
|
||||
catalog; DWH and LLM services remain external. Semantic services are bundled in Compose.
|
||||
|
||||
## Docker Compose: local startup
|
||||
The same frontend supports **full** (its own header) and **embedded** (inside a
|
||||
portal). This choice is independent of authentication: the Mac uses full/local,
|
||||
Omics uses embedded/upstream with its existing login, and a standalone server
|
||||
can use full/OIDC. See [rendering architecture](docs/architecture/application-shell.md)
|
||||
and [configuration, Omics delivery and deploy](docs/operations/shell-and-localization.md).
|
||||
|
||||
Requirements: Docker Engine with Compose v2. The mandatory stack is `frontend`, `core`, `qdrant`, `embedding`, and the one-shot `embedding-model-init`. DWH and LLM remain external,
|
||||
configurable endpoints—even when they are co-located with ThothII.
|
||||
For the current server upgrade with Omics Portal, follow the ordered
|
||||
[Codex server handoff](docs/operations/server-codex-handoff.md), including source
|
||||
integration, embedded/upstream configuration, coordinated rollout and rollback.
|
||||
|
||||
From a fresh clone, run these commands from the repository root:
|
||||
Local/OIDC authentication is configured through the host CLI `tht`; portal
|
||||
authentication is established by the trusted server proxy. See the
|
||||
[local guide](docs/install/authentication-local.md),
|
||||
[OIDC guide](docs/install/authentication-oidc.md),
|
||||
[upstream integration](docs/install/authentication-upstream.md), and
|
||||
[manual acceptance matrix](docs/testing/authentication-manual-acceptance.md).
|
||||
|
||||
```sh
|
||||
cp deploy/env/local.env.example deploy/env/local.env
|
||||
# Edit deploy/env/local.env, including PI_AUTH_FILE, THT_SECRETS_FILE, and external endpoints.
|
||||
docker compose --env-file deploy/env/local.env \
|
||||
-f compose.yaml -f deploy/compose.local.yaml up --build -d
|
||||
```
|
||||
For the clone-based manual standalone installation test on macOS, Windows, and Linux, use the
|
||||
[Italian procedure](docs/install/standalone-manual-it.md) or the
|
||||
[English procedure](docs/install/standalone-manual-en.md).
|
||||
|
||||
`./scripts/run-stack.sh` runs this same base+local command in the foreground. The core image
|
||||
contains its Pi runtime; no host `pi` executable is used. For a server installation:
|
||||
The [public manual](https://git.tylconsulting.it/thothii-docs/) covers the product,
|
||||
installation, use and administration. Developer architecture, contracts, ADRs, tests,
|
||||
plans and release records remain in this repository but are excluded from MkDocs
|
||||
pages and search. This is an editorial boundary, not an access restriction on the
|
||||
public repository. See the [documentation cleanup review](docs/maintenance/2026-09-15-documentation-cleanup.md)
|
||||
for the executed consolidation and the inventory of historical sources retained in Git.
|
||||
|
||||
```sh
|
||||
cp deploy/env/server.env.example deploy/env/server.env
|
||||
# Edit all absolute storage, Pi/secret/session files, and endpoint paths.
|
||||
sudo scripts/prepare-server-pi-state.sh /srv/thothii/pi-state 10001 10001
|
||||
docker compose --env-file deploy/env/server.env \
|
||||
-f compose.yaml -f deploy/compose.server.yaml \
|
||||
-f deploy/compose.session-server.yaml.example up --build -d
|
||||
```
|
||||
## Docker Compose and installation
|
||||
|
||||
The initializer is required for an empty or restored server Pi-state bind. It atomically creates
|
||||
the three regular targets hidden below the writable parent bind; protected Pi auth and tracked
|
||||
model/settings sources remain separate read-only mounts. See the server manual before substituting
|
||||
a root other than `/srv/thothii/pi-state`.
|
||||
For a fresh installation, follow the complete manual procedure in
|
||||
[Italian](docs/install/standalone-manual-it.md) or
|
||||
[English](docs/install/standalone-manual-en.md). Configure protected files first;
|
||||
then run the documented build, explicit migrations and startup commands with the
|
||||
same installation descriptor and Compose project. There is no installer or launcher.
|
||||
|
||||
Workspace descriptors come from the Git remote configured by `THT_WORKSPACE_GIT_REMOTE`; their
|
||||
runtime endpoint and secret bindings remain installation-local. Open
|
||||
<http://127.0.0.1:8080> (set `THOTH_HTTP_PORT` in `deploy/env/local.env` to choose another
|
||||
loopback port).
|
||||
The mandatory stack includes frontend, core, PostgreSQL catalog, Qdrant, Ollama
|
||||
and the embedding initializer. DWH and LLM endpoints remain external dependencies.
|
||||
Pi is included in the core image. Credentials and certificates belong in protected
|
||||
installation-local files, never in the workspace repository.
|
||||
|
||||
Credentials and certificates are local protected files. Do not put them in environment examples,
|
||||
workspace YAML, URLs, or Compose interpolation values.
|
||||
For developer topology, overlays and lifecycle details, see the internal
|
||||
[Compose reference](docs/operations/compose-reference.md). Ordinary stop/down keeps
|
||||
persistent data; removing volumes is destructive and is not an upgrade step.
|
||||
Process health is distinct from external dependency checks performed by doctor.
|
||||
|
||||
Application state is split across the named `settings`, `pi-state`, `workspace-registry`,
|
||||
`sessions`, `qdrant-data`, and `embedding-models` volumes. `docker compose down` keeps them.
|
||||
`qdrant-data` is a derived but persistent index store; `embedding-models` is an Ollama model
|
||||
cache for `qwen3-embedding:0.6b` with fixed `1024`-dimension embeddings. Only an explicit destructive command such as `docker compose
|
||||
down --volumes` removes them.
|
||||
|
||||
The frontend depends on the core health check and proxies `/health` and `/api/*` to it. The
|
||||
application health endpoint intentionally checks process readiness only; external dependency
|
||||
diagnostics are exposed by `tht doctor` and do not prevent the UI from starting.
|
||||
|
||||
## Git-backed workspace registry
|
||||
## Git-backed workspace repository
|
||||
|
||||
Workspace descriptors are shared through a validated Git repository while endpoint bindings and
|
||||
secret files remain installation-local. Use the [local Mac/PC installation manual](docs/install/local-workspace-registry.md)
|
||||
for Docker Desktop or a local engine, and the [server installation manual](docs/install/server-workspace-registry.md)
|
||||
for the Gitea, reverse-proxy, backup, migration, and recovery workflow. The isolated deployment
|
||||
exercise is `./scripts/workspace-registry-smoke.sh`; both manuals are checked with
|
||||
secret files remain installation-local. The supported operating sequence is documented in
|
||||
[Workspace operations](docs/operations/workspaces.md); it covers curator publication, installation
|
||||
activation, runtime bindings, and preprocessing. The host setup and lifecycle path is in
|
||||
[Install and first start](docs/install/first-start.md).
|
||||
|
||||
The curator-owned repository layout is:
|
||||
|
||||
```text
|
||||
thoth-workspaces.yaml
|
||||
<id>/workspace.yaml
|
||||
<id>/evidence/**
|
||||
```
|
||||
|
||||
`thoth-workspaces.yaml` uses the `schema_version` value `1` and the ordered `workspaces` list of
|
||||
`{id, name, description?}` entries. It is authoritative for workspace ID, name, description, and
|
||||
display order. Every catalog entry must have a matching descriptor in the same commit; otherwise
|
||||
the complete candidate is rejected. Descriptors remain curator-owned and change only through a
|
||||
Git commit and push from a separate authoring clone, followed by an installation pull. ThothII
|
||||
never writes any workspace repository content.
|
||||
|
||||
The operator workflow is: curate catalog/descriptor/Evidence changes in Git, commit and push,
|
||||
**Update workspace repository** from each ThothII installation, select the workspace, complete its
|
||||
write-only runtime-secret fields, run **Validate workspace source** and **Test workspace
|
||||
connections**, then select the workspace locally before creating sessions. Each new session
|
||||
pins the Git revision it used; a later pull cannot change a Resume. Snapshot cleanup retains every
|
||||
revision referenced by an open, closed, or failed unarchived session. It reconciles from the
|
||||
single local installation list or from a server administrator's complete session list, never from
|
||||
a remote user's partial list. The isolated deployment exercise is
|
||||
`./scripts/workspace-registry-smoke.sh`; both manuals are checked with
|
||||
`./scripts/verify-workspace-install-docs.sh --profile local` or `--profile server`.
|
||||
|
||||
The operator workflow is: update and review canonical YAML in the shared Git remote, **Pull latest
|
||||
registry** from each ThothII installation, run **Validate workspace** and **Test on this
|
||||
installation**, then select the workspace locally before creating sessions. Each new session pins
|
||||
the Git revision it used; a later pull or publish cannot change a Resume. Snapshot cleanup retains
|
||||
every revision referenced by an open, closed, or failed unarchived session. It reconciles from the
|
||||
single local installation list or from a server administrator's complete session list, never from
|
||||
a remote user's partial list.
|
||||
<!-- workspace-descriptor-contract:start -->
|
||||
Schema v4 is the only accepted workspace descriptor. It contains workspace identity and optional
|
||||
Evidence configuration only; PostgreSQL Metadata Catalog owns every database fact and binding.
|
||||
Schema v1, v2, and v3 descriptors are rejected before activation. Candidate snapshot validation
|
||||
therefore makes activation or a pull fail atomically while the prior valid snapshot remains active.
|
||||
Each workspace owns separate Qdrant `reference` and `memory` collections: Schema, relationships, and
|
||||
Evidence are replaceable reference data; Memory and solved questions have a persistent lifecycle.
|
||||
<!-- workspace-descriptor-contract:end -->
|
||||
|
||||
Schema-v3 is the operational descriptor contract. Schema-v1/v2 descriptors remain
|
||||
`migration_required` until an explicit reviewed migration writes schema version 3. One workspace
|
||||
owns one Qdrant collection; schema, Evidence, and Memory records share that collection and stay
|
||||
separated by indexed payload `kind`.
|
||||
<!-- non-workspace-migration:start -->
|
||||
Create a clean v4 descriptor containing only `workspace` and optional `evidence`. Do not copy the
|
||||
legacy database, diagnostics, `llm_policy`, or `semantic_index` blocks; configure the database in
|
||||
Database Management.
|
||||
<!-- non-workspace-migration:end -->
|
||||
|
||||
Connector `ssh_tunnel` bindings are diagnostic-only in this release: their bounded probe always
|
||||
cleans up the loopback forward and returns `workspace_not_activatable`; session creation is rejected
|
||||
before persistence. Git registry access over SSH is unaffected. Use direct or REST connector
|
||||
transport for runtime sessions.
|
||||
For NL→SQL runtime sessions, connector `ssh_tunnel` bindings remain diagnostic-only: their bounded
|
||||
probe cleans up the loopback forward and returns `workspace_not_activatable`; session creation is
|
||||
rejected before persistence. Database management is a separate boundary and supports a strict
|
||||
OpenSSH tunnel for **Test connection** and **Sync tables**, using a private key, optional passphrase,
|
||||
mandatory `known_hosts`, and optional PostgreSQL TLS CA/server name. Git registry access over SSH is
|
||||
unaffected. Use direct or REST connector transport for runtime sessions.
|
||||
|
||||
`docker-compose.dev.yml` is deliberately local: both published ports bind to `127.0.0.1`,
|
||||
`THT_SESSION_STORAGE=local`, and `THT_HOME=/data/local-home`. Do not set
|
||||
@@ -116,7 +142,7 @@ the teardown if any container, volume, or network has a foreign run label.
|
||||
|
||||
```sh
|
||||
bash scripts/unified-deployment-smoke.sh
|
||||
bash scripts/thothctl-update-smoke.sh
|
||||
bash scripts/tht-update-smoke.sh
|
||||
bash scripts/server-deployment-smoke.sh
|
||||
```
|
||||
|
||||
@@ -125,19 +151,19 @@ registry, recreates with the Git remote offline, activates a valid Git update, r
|
||||
content while retaining the valid snapshot, and checks the four persistence volumes. The unified
|
||||
and update-only smokes
|
||||
inject a digest-pinned non-core candidate under a deliberately mismatched Pi version and require
|
||||
`thothctl pi update` to roll back while preserving settings, sessions, Pi state, registry revision,
|
||||
`tht pi update` to roll back while preserving settings, sessions, Pi state, registry revision,
|
||||
and mount identity. The rollback candidate is the digest-pinned `hello-world` executable: a
|
||||
preflight proves that it exits successfully, so the failed replacement core satisfies
|
||||
`thothctl`'s stopped-core compensation precondition. The server smoke uses the same smoke-built
|
||||
`tht`'s stopped-core compensation precondition. The server smoke uses the same smoke-built
|
||||
core/frontend images with the server and required session overlays, disposable bind roots and
|
||||
secret files, upstream-auth checks, and a fail-closed `503` assertion for its deliberately
|
||||
unavailable disposable session endpoint. No real provider, database credential, or repository
|
||||
secret is required.
|
||||
|
||||
For a clean server bind, `scripts/prepare-server-pi-state.sh` creates the hidden regular
|
||||
`agent/auth.json`, `agent/models.json`, and `agent/settings.json` mount targets atomically before
|
||||
Compose. The server smoke starts from an empty Pi-state root and applies this same preflight; the
|
||||
real protected/tracked sources remain separate read-only mounts. Deterministic fixture tests render
|
||||
For a clean server bind, `scripts/prepare-server-pi-state.sh` creates the hidden regular Pi agent
|
||||
mount targets atomically before Compose. The auth target receives the protected credential bind;
|
||||
the model and settings targets receive generated read-only projections. The server smoke starts
|
||||
from an empty Pi-state root and applies this same preflight. Deterministic fixture tests render
|
||||
both profiles, verify that bindings stay on `core`, check mount readability, and run the production
|
||||
workspace resolver. Wrong-service, wrong-value, and broken-secret-mount mutations must fail.
|
||||
|
||||
@@ -147,7 +173,7 @@ an independent 32-minute outer timeout and does not retry a failed command.
|
||||
Current release status (2026-08-05): clean-root render/setup and the production runtime-binding
|
||||
resolver contracts are green. The server fixture supplies all four private trusted claims,
|
||||
including exact non-admin value `0`, and a focused test proves nginx normalization produces the
|
||||
accepted non-admin backend principal. Canonical schema-v3 registry descriptors now pass through
|
||||
accepted non-admin backend principal. Canonical schema-v4 registry descriptors now pass through
|
||||
one backend-owned, secret-safe runtime handoff for inventory and session execution; canonical
|
||||
identity and durable session/artifact/index roots are retained. The fresh update-only smoke passed
|
||||
bad-candidate mutation, automatic `rolled_back` compensation, exact prior-image restoration,
|
||||
@@ -164,7 +190,7 @@ The deterministic native Windows contract is:
|
||||
```
|
||||
|
||||
It checks Git's CRLF/LF attributes and bytes, copies tracked source into a temporary path containing
|
||||
spaces, builds and invokes native Windows `thothctl` there, and renders exactly `core` plus
|
||||
spaces, builds and invokes native Windows `tht` there, and renders exactly `core` plus
|
||||
`frontend` without starting containers. On a supported self-hosted Windows Docker Desktop/WSL2
|
||||
runner, dispatch the deployment workflow with `windows_docker_startup=true`; that job executes:
|
||||
|
||||
@@ -172,24 +198,30 @@ runner, dispatch the deployment workflow with `windows_docker_startup=true`; tha
|
||||
.\scripts\test-windows-clone-contract.ps1 -DockerStartup
|
||||
```
|
||||
|
||||
Startup mode adds bounded image build/two-service health startup, installation-aware `thothctl`
|
||||
Startup mode adds bounded image build/two-service health startup, installation-aware `tht`
|
||||
status, stopped-container-aware ownership checks, and exact cleanup. The ordinary hosted Windows
|
||||
job remains deterministic and does not claim Docker startup.
|
||||
|
||||
## Preprocessing jobs and S3 Evidence
|
||||
## Workspace preprocessing and S3 Evidence
|
||||
|
||||
The included preprocessing services reuse the internal Qdrant/Ollama stack. Mount Evidence at
|
||||
`/data/source/evidence`, then run the explicit preprocessing preset:
|
||||
For an interactive run, select the workspace, expand **Administration** in the right sidebar, and
|
||||
use its **Preprocessing** control. The control explains any unmet prerequisite and exposes only the
|
||||
latest safe failure diagnostic. For unattended operation, use the native host CLI and installation
|
||||
descriptor:
|
||||
|
||||
```sh
|
||||
docker compose --env-file deploy/env/local.env \
|
||||
-f compose.yaml -f deploy/compose.local.yaml \
|
||||
-f deploy/compose.preprocess.yaml --profile preprocess run --rm preprocess-evidence
|
||||
tht --installation /absolute/path/thothii-installation.yaml \
|
||||
workspace preprocess run --workspace <workspace-id>
|
||||
|
||||
tht --installation /absolute/path/thothii-installation.yaml \
|
||||
workspace preprocess clear --workspace <workspace-id>
|
||||
```
|
||||
|
||||
Replace the final service with `preprocess-dwh` when required. The overlay makes each job wait for the internal Qdrant
|
||||
service health checks and embedding model initialization; no separate semantic-service startup is
|
||||
required.
|
||||
The one-shot command starts the profile-gated `workspace-maintenance` service, reads database
|
||||
metadata from PostgreSQL, and rebuilds LSH plus schema/Evidence vectors. The clear command removes
|
||||
those derived artifacts while preserving the separate Memory collection. The core remains unavailable
|
||||
until preprocessing completes. See [Evidence](docs/evidence.md) and the
|
||||
[workspace preprocessing CLI contract](docs/contracts/workspace-preprocessing-cli.md).
|
||||
|
||||
S3 Evidence uses the optional `tht[s3]` dependency and canonical `s3://bucket/key` provenance.
|
||||
AWS endpoints are used when no custom URL is supplied. Every custom endpoint is an explicit egress
|
||||
@@ -228,10 +260,11 @@ Compose project name by passing `--confirm-project`:
|
||||
|
||||
The restore script stops `qdrant`, validates the exact labeled target, stages the current volume
|
||||
contents for rollback, extracts the requested archive into the volume, and then returns the
|
||||
service to its prior running state. After restore, run the backend health checks and a known
|
||||
retrieval query before reopening write traffic. Restore does not migrate schema-v1/v2 workspace
|
||||
descriptors, does not rename collections, and does not reconcile an incompatible collection
|
||||
contract; those remain explicit reviewed recovery steps outside the helper.
|
||||
service to its prior running state. It restores semantic storage only. Before reopening write
|
||||
traffic, the workspace registry must already be at a reviewed v4 descriptor revision compatible
|
||||
with the restored collection; then run backend health checks and a known retrieval query. The
|
||||
helper does not restore descriptors, rename collections, or reconcile an incompatible collection
|
||||
contract.
|
||||
|
||||
## Production trust boundary and secrets
|
||||
|
||||
@@ -254,6 +287,33 @@ Copy `deploy/secrets/thothii.secrets.example` to a protected host file, include
|
||||
keys, and set its absolute path as `THT_SECRETS_FILE` in the operator env. Keep Pi's native
|
||||
provider auth in the separate protected file named by `PI_AUTH_FILE`.
|
||||
|
||||
Interactive sessions, Description Generation, and embedding share the protected installation
|
||||
descriptor's `modelCatalog`. Set `THT_INSTALLATION_CONFIG_SOURCE` to that exact host file; `tht`
|
||||
validates it and generates the runtime catalog, Pi adapters, and Compose override before startup.
|
||||
Each authenticated provider stores only an audited `apiKeyEnv` reference; the referenced value stays
|
||||
in the secret bundle. A provider may use `authentication.mode: none` only with an explicit keyless
|
||||
endpoint. The browser receives only eligible model IDs, labels, and the catalog default.
|
||||
|
||||
Before enabling Description Generation, approve the selected model provider for bounded source-data
|
||||
disclosure. Every catalog column has a **Sensitive** flag that defaults to `false`. Administrators can
|
||||
request an AI proposal based only on structural metadata, then must review and save the resulting
|
||||
checkboxes themselves. The proposal never reads column contents and is not persisted automatically.
|
||||
|
||||
For unprotected columns, a request may send up to five real source rows and five representative
|
||||
distinct, non-null example values. Protected columns are omitted from source reads and replaced in the
|
||||
prompt by deterministic plausible values derived only from column metadata. Samples are transient and
|
||||
are not stored in generation runs, run logs, application logs, API responses, or catalog metadata;
|
||||
prompt and sample snapshots are not retained. A flag change applies to later generations and does not
|
||||
regenerate existing descriptions.
|
||||
|
||||
Description Generation is an interactive Database Management operation, not a user-facing CLI.
|
||||
The installation runs at most one sequential generation at a time. The run drawer exposes safe
|
||||
ordered events through SSE with polling fallback, Stop terminates the current helper while keeping
|
||||
already stored results, and Run history retains terminal runs for inspection. A backend restart
|
||||
marks queued or running work interrupted instead of resuming it; use Generate Missing to continue.
|
||||
Unlock is reserved for a stale recorded run and is rejected while a local start, worker, or helper
|
||||
is still live.
|
||||
|
||||
The bundle is mounted read-only as `/run/secrets/thothii.secrets` and must be mode `0600` or
|
||||
`0400` on the host. Docker's runtime `0444` mode is accepted only beneath `/run/secrets`; see
|
||||
[`deploy/secrets/README.md`](deploy/secrets/README.md). A PEM CA chain is deliberately not a
|
||||
@@ -263,22 +323,11 @@ the host/secret-manager materialization and add a reviewed Compose override that
|
||||
does not create that mount. The frontend remains on loopback; the authenticated host proxy is the
|
||||
only public listener.
|
||||
|
||||
Set the selected model provider in application settings (or `PI_PROVIDER`). For each Pi spawn the
|
||||
backend validates and reads `THT_MODEL_API_KEY` from the bundle, then exposes its value only as the provider's
|
||||
recognized child variable (for example `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, `GEMINI_API_KEY`, or
|
||||
`ZAI_API_KEY`). Neither the generic file path nor deprecated `PI_PROVIDER_API_KEY` is inherited by
|
||||
Pi. Local providers such as Ollama require no model key.
|
||||
|
||||
`THT_MODEL_API_KEY` supports Pi providers whose authentication is exactly one key:
|
||||
`ant-ling`, `anthropic`, `cerebras`, `deepseek`, `fireworks`, `github-copilot`, `google`
|
||||
(including the `gemini` alias), `google-vertex` when using its API-key mode, `groq`,
|
||||
`huggingface`, `kimi-coding`, `minimax`, `minimax-cn`, `mistral`, `moonshotai`,
|
||||
`moonshotai-cn`, `nvidia`, `openai`, `opencode`, `opencode-go`, `openrouter`, `together`,
|
||||
`vercel-ai-gateway`, `xai`, the four `xiaomi*` providers, `zai`, and `zai-coding-cn`.
|
||||
Compound providers are deliberately unsupported: `amazon-bedrock`, `azure-openai-responses`,
|
||||
`cloudflare-workers-ai`, and `cloudflare-ai-gateway` require multiple credential/configuration
|
||||
values. Selecting one fails before Pi starts; ambient AWS, Azure, and Cloudflare credentials are
|
||||
still scrubbed. Supporting them requires a future dedicated provider-specific configuration.
|
||||
For each Pi spawn, the backend resolves the selected canonical provider/model in the runtime catalog,
|
||||
reads exactly that provider's declared `apiKeyEnv` value from the bundle, and exposes only that key
|
||||
to the child. Ambient provider credentials and secret-bundle paths are scrubbed. Providers needing a
|
||||
compound credential bundle remain unsupported until the catalog gains an explicit generic contract
|
||||
for them.
|
||||
|
||||
## User-owned session server cutover
|
||||
|
||||
@@ -287,6 +336,8 @@ The server profile stores sessions and per-user preferences directly in PostgreS
|
||||
dual write. Use [`deploy/compose.session-server.yaml.example`](deploy/compose.session-server.yaml.example)
|
||||
with the canonical base+server files and set `THT_SERVER_WORKSPACE_CONFIG` to an absolute,
|
||||
protected copy of [`deploy/workspaces/server-sessions.yaml.example`](deploy/workspaces/server-sessions.yaml.example).
|
||||
That file is an installation runtime template, not an authored workspace descriptor; database
|
||||
bindings are injected from the PostgreSQL Metadata Catalog for each runtime lease.
|
||||
|
||||
The runtime login needs membership in the no-login database role `thoth_sessions_runtime` only.
|
||||
The distinct, one-shot migrator login needs migration authority and uses
|
||||
|
||||
Generated
+2247
-100
File diff suppressed because it is too large
Load Diff
+14
-7
@@ -6,24 +6,31 @@
|
||||
"dev": "tsx watch src/server.ts",
|
||||
"prebuild": "node scripts/clean-dist.mjs",
|
||||
"build": "tsc -p tsconfig.json",
|
||||
"catalog:migrate": "node dist/catalog/migrate.js",
|
||||
"sensitivity:shadow": "node dist/catalog/sensitivity-shadow.js",
|
||||
"test": "vitest run",
|
||||
"start": "node dist/server.js"
|
||||
"start": "node dist/server.js",
|
||||
"test:schema-v4-verifier": "python3 -I -B scripts/test_revision_state_policy.py && node --test scripts/verify-workspace-descriptor-files.test.mjs scripts/revision-state-policy.test.mjs",
|
||||
"test:schema-v3-verifier": "npm run test:schema-v4-verifier"
|
||||
},
|
||||
"dependencies": {
|
||||
"@fastify/cookie": "11.1.2",
|
||||
"@fastify/cors": "^11.2.0",
|
||||
"@fastify/multipart": "^9.4.0",
|
||||
"@fastify/rate-limit": "11.2.0",
|
||||
"@types/pg": "^8.20.3",
|
||||
"fastify": "^5.0.0",
|
||||
"kysely": "^0.29.5",
|
||||
"libphonenumber-js": "1.13.12",
|
||||
"openid-client": "6.8.5",
|
||||
"pg": "^8.22.0",
|
||||
"validator": "13.15.35",
|
||||
"yaml": "^2.9.0",
|
||||
"yauzl": "^3.4.0",
|
||||
"yazl": "^3.3.1",
|
||||
"zod": "^4.4.3"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^22.0.0",
|
||||
"@types/yauzl": "^3.4.0",
|
||||
"@types/yazl": "^3.3.1",
|
||||
"@testcontainers/postgresql": "^12.1.0",
|
||||
"@types/node": "24.13.3",
|
||||
"@types/validator": "13.15.10",
|
||||
"tsx": "^4.19.0",
|
||||
"typescript": "^5.6.0",
|
||||
"vitest": "^2.1.0"
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
34448b82c17d60fec9b65b1f093c115ddbaadc04beb1b0140b6bfed2e012a930 ./.gitattributes
|
||||
4d9344c58a2a2ea4bb4ff4f7c611a853cf413205fc10d0cace564eba06f73828 ./README.md
|
||||
180f0a10d1d5ed5ce3318db0bcb0b1b7780d79a52f0a8fc3acbd27f74536d0e4 ./THOTHII_MODEL_REVISION
|
||||
164f17362bcf9d114067d3465e7374bfdd79ce6b605acb745de5a49dabb9595c ./config.json
|
||||
f27dd63cc43a248d2566f0b6ad7a115db353676ce0561dcbca45bac766464c1a ./encoder_config/config.json
|
||||
0280f6f39f6012da50b6640bad438d9b7e763a1b0102094115d1b710c4dd79b6 ./model.safetensors
|
||||
f6df10ec83bea993035b2dd7c39345a3d4fcf23421c2adb6cb4ffc1e6d1bc4b5 ./tokenizer.json
|
||||
233beed1f1095cccfc7907cde31a8d90a0c6aa4fdfaf6493f8e55fd162e81ae6 ./tokenizer_config.json
|
||||
@@ -0,0 +1,34 @@
|
||||
# Optional offline CPU pack. Fully version-locked in its own venv; not part of the base image.
|
||||
--extra-index-url https://download.pytorch.org/whl/cpu
|
||||
accelerate==1.14.0
|
||||
annotated-types==0.8.0
|
||||
certifi==2026.7.22
|
||||
charset-normalizer==3.5.1
|
||||
filelock==3.32.5
|
||||
fsspec==2026.7.0
|
||||
gliner2[local]==2.0.0
|
||||
hf-xet==1.6.0
|
||||
huggingface-hub==0.36.2
|
||||
idna==3.19
|
||||
Jinja2==3.1.6
|
||||
MarkupSafe==3.0.3
|
||||
mpmath==1.3.0
|
||||
networkx==3.6.1
|
||||
numpy==2.5.2
|
||||
packaging==26.3
|
||||
peft==0.20.0
|
||||
psutil==7.2.2
|
||||
pydantic==2.13.5
|
||||
pydantic-core==2.46.5
|
||||
PyYAML==6.0.3
|
||||
regex==2026.9.3
|
||||
requests==2.34.2
|
||||
safetensors==0.8.0
|
||||
sympy==1.14.0
|
||||
tokenizers==0.22.2
|
||||
torch==2.14.0+cpu
|
||||
tqdm==4.70.0
|
||||
transformers==4.57.6
|
||||
typing-extensions==4.16.0
|
||||
typing-inspection==0.4.4
|
||||
urllib3==2.7.0
|
||||
@@ -0,0 +1,301 @@
|
||||
"""Offline, CPU-only JSONL worker for optional sensitivity NER evidence."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import contextlib
|
||||
import ctypes
|
||||
import errno
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import socket
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
PII_LABELS = [
|
||||
"person",
|
||||
"full_name",
|
||||
"first_name",
|
||||
"middle_name",
|
||||
"last_name",
|
||||
"date_of_birth",
|
||||
"email",
|
||||
"phone_number",
|
||||
"address",
|
||||
"street_address",
|
||||
"city",
|
||||
"state_or_region",
|
||||
"postal_code",
|
||||
"country",
|
||||
"government_id",
|
||||
"national_id_number",
|
||||
"passport_number",
|
||||
"drivers_license_number",
|
||||
"license_number",
|
||||
"tax_id",
|
||||
"tax_number",
|
||||
"bank_account",
|
||||
"account_number",
|
||||
"routing_number",
|
||||
"iban",
|
||||
"payment_card",
|
||||
"card_number",
|
||||
"card_expiry",
|
||||
"card_cvv",
|
||||
"username",
|
||||
"ip_address",
|
||||
"account_id",
|
||||
"sensitive_account_id",
|
||||
"password",
|
||||
"secret",
|
||||
"api_key",
|
||||
"access_token",
|
||||
"recovery_code",
|
||||
"sensitive_date",
|
||||
"document_date",
|
||||
"expiration_date",
|
||||
"transaction_date",
|
||||
]
|
||||
|
||||
_MODEL_COMPAT_DIRECTORY: tempfile.TemporaryDirectory[str] | None = None
|
||||
_EXPECTED_MODEL_REVISION = "c153999da5f4c509df4322b0c6a1baf3d2c284d7"
|
||||
|
||||
|
||||
def _arguments() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(add_help=False)
|
||||
parser.add_argument("--model", required=True)
|
||||
parser.add_argument("--threads", type=int, default=2)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def _disable_network() -> None:
|
||||
libc = ctypes.CDLL(None, use_errno=True)
|
||||
libc.prctl.argtypes = [
|
||||
ctypes.c_int,
|
||||
ctypes.c_ulong,
|
||||
ctypes.c_ulong,
|
||||
ctypes.c_ulong,
|
||||
ctypes.c_ulong,
|
||||
]
|
||||
libc.prctl.restype = ctypes.c_int
|
||||
if libc.prctl(38, 1, 0, 0, 0) != 0: # PR_SET_NO_NEW_PRIVS
|
||||
raise RuntimeError("cannot enable no-new-privileges for network isolation")
|
||||
|
||||
try:
|
||||
seccomp = ctypes.CDLL("libseccomp.so.2", use_errno=True)
|
||||
except OSError as error:
|
||||
raise RuntimeError("libseccomp is required for network isolation") from error
|
||||
seccomp.seccomp_init.argtypes = [ctypes.c_uint32]
|
||||
seccomp.seccomp_init.restype = ctypes.c_void_p
|
||||
seccomp.seccomp_syscall_resolve_name.argtypes = [ctypes.c_char_p]
|
||||
seccomp.seccomp_syscall_resolve_name.restype = ctypes.c_int
|
||||
seccomp.seccomp_rule_add.argtypes = [
|
||||
ctypes.c_void_p,
|
||||
ctypes.c_uint32,
|
||||
ctypes.c_int,
|
||||
ctypes.c_uint,
|
||||
]
|
||||
seccomp.seccomp_rule_add.restype = ctypes.c_int
|
||||
seccomp.seccomp_load.argtypes = [ctypes.c_void_p]
|
||||
seccomp.seccomp_load.restype = ctypes.c_int
|
||||
seccomp.seccomp_release.argtypes = [ctypes.c_void_p]
|
||||
seccomp.seccomp_release.restype = None
|
||||
|
||||
allow = 0x7FFF0000 # SCMP_ACT_ALLOW
|
||||
deny = 0x00050000 | errno.EPERM # SCMP_ACT_ERRNO(EPERM)
|
||||
filter_context = seccomp.seccomp_init(allow)
|
||||
if not filter_context:
|
||||
raise RuntimeError("cannot initialize network syscall filter")
|
||||
try:
|
||||
for syscall in (
|
||||
"socket",
|
||||
"connect",
|
||||
"sendto",
|
||||
"sendmsg",
|
||||
"sendmmsg",
|
||||
"bind",
|
||||
"listen",
|
||||
"accept",
|
||||
"accept4",
|
||||
):
|
||||
syscall_number = seccomp.seccomp_syscall_resolve_name(syscall.encode("ascii"))
|
||||
if syscall_number < 0:
|
||||
raise RuntimeError(f"cannot resolve network syscall: {syscall}")
|
||||
if seccomp.seccomp_rule_add(filter_context, deny, syscall_number, 0) != 0:
|
||||
raise RuntimeError(f"cannot block network syscall: {syscall}")
|
||||
if seccomp.seccomp_load(filter_context) != 0:
|
||||
raise RuntimeError("cannot activate network syscall filter")
|
||||
finally:
|
||||
seccomp.seccomp_release(filter_context)
|
||||
|
||||
def blocked(*_args: Any, **_kwargs: Any) -> Any:
|
||||
raise PermissionError(errno.EPERM, "network disabled")
|
||||
|
||||
socket.socket = blocked # type: ignore[assignment]
|
||||
socket.create_connection = blocked # type: ignore[assignment]
|
||||
|
||||
|
||||
def _verify_model(path: Path) -> None:
|
||||
revision_path = path / "THOTHII_MODEL_REVISION"
|
||||
try:
|
||||
revision = revision_path.read_text(encoding="utf-8").strip()
|
||||
except OSError as error:
|
||||
raise RuntimeError("model revision marker is unavailable") from error
|
||||
if revision != _EXPECTED_MODEL_REVISION:
|
||||
raise RuntimeError("model revision is not approved")
|
||||
|
||||
manifest_path = Path(__file__).with_name("sensitivity-ner-model-sha256.txt")
|
||||
try:
|
||||
manifest = manifest_path.read_text(encoding="utf-8").splitlines()
|
||||
except OSError as error:
|
||||
raise RuntimeError("model checksum manifest is unavailable") from error
|
||||
for line in manifest:
|
||||
checksum, separator, relative_name = line.partition(" ")
|
||||
if not separator or len(checksum) != 64 or not relative_name.startswith("./"):
|
||||
raise RuntimeError("model checksum manifest is invalid")
|
||||
relative_path = Path(relative_name[2:])
|
||||
if relative_path.is_absolute() or ".." in relative_path.parts:
|
||||
raise RuntimeError("model checksum path is invalid")
|
||||
model_file = path / relative_path
|
||||
if not model_file.is_file() or model_file.is_symlink():
|
||||
raise RuntimeError("approved model file is unavailable")
|
||||
digest = hashlib.sha256()
|
||||
with model_file.open("rb") as stream:
|
||||
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
if digest.hexdigest() != checksum:
|
||||
raise RuntimeError("approved model checksum does not match")
|
||||
|
||||
|
||||
def _transformers4_model_path(path: Path) -> Path:
|
||||
"""Adapt tokenizer metadata emitted by Transformers 5 without changing pinned weights.
|
||||
|
||||
GLiNER2 2.0.0 officially requires Transformers <5, while current Fastino checkpoints were
|
||||
saved by Transformers 5.8.0. Transformers 4 calls the same list
|
||||
``additional_special_tokens``; Transformers 5 renamed it to ``extra_special_tokens`` and
|
||||
changed its type. Keep the downloaded model immutable and create a temporary symlink view
|
||||
containing only the compatibility metadata needed by the supported GLiNER2 dependency set.
|
||||
"""
|
||||
|
||||
tokenizer_path = path / "tokenizer_config.json"
|
||||
try:
|
||||
tokenizer = json.loads(tokenizer_path.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError) as error:
|
||||
raise RuntimeError("invalid tokenizer configuration") from error
|
||||
extra_tokens = tokenizer.get("extra_special_tokens")
|
||||
if extra_tokens is None:
|
||||
return path
|
||||
if not isinstance(extra_tokens, list) or not all(isinstance(token, str) for token in extra_tokens):
|
||||
raise RuntimeError("unsupported extra_special_tokens configuration")
|
||||
if "additional_special_tokens" in tokenizer:
|
||||
raise RuntimeError("ambiguous special-token configuration")
|
||||
|
||||
global _MODEL_COMPAT_DIRECTORY
|
||||
_MODEL_COMPAT_DIRECTORY = tempfile.TemporaryDirectory(prefix="thothii-ner-model-")
|
||||
compatible_path = Path(_MODEL_COMPAT_DIRECTORY.name)
|
||||
for child in path.iterdir():
|
||||
if child.name == tokenizer_path.name:
|
||||
continue
|
||||
(compatible_path / child.name).symlink_to(child, target_is_directory=child.is_dir())
|
||||
tokenizer["additional_special_tokens"] = tokenizer.pop("extra_special_tokens")
|
||||
(compatible_path / tokenizer_path.name).write_text(
|
||||
json.dumps(tokenizer, ensure_ascii=False, indent=2) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
return compatible_path
|
||||
|
||||
|
||||
def _load_model(model_path: str, threads: int) -> Any:
|
||||
path = Path(model_path).resolve(strict=True)
|
||||
if not path.is_dir():
|
||||
raise RuntimeError("model path must be a local directory")
|
||||
_verify_model(path)
|
||||
os.environ["CUDA_VISIBLE_DEVICES"] = ""
|
||||
os.environ["HIP_VISIBLE_DEVICES"] = ""
|
||||
os.environ["HF_HUB_OFFLINE"] = "1"
|
||||
os.environ["TRANSFORMERS_OFFLINE"] = "1"
|
||||
import torch
|
||||
from gliner2 import AutoExtractor
|
||||
|
||||
torch.set_num_threads(max(1, min(threads, 8)))
|
||||
torch.set_num_interop_threads(1)
|
||||
compatible_path = _transformers4_model_path(path)
|
||||
with contextlib.redirect_stdout(sys.stderr):
|
||||
model = AutoExtractor.from_pretrained(str(compatible_path), map_location="cpu")
|
||||
_disable_network()
|
||||
return model
|
||||
|
||||
|
||||
def _request(value: Any) -> tuple[str, list[dict[str, str]]]:
|
||||
if not isinstance(value, dict) or not isinstance(value.get("id"), str):
|
||||
raise ValueError("invalid request")
|
||||
candidates = value.get("candidates")
|
||||
if not isinstance(candidates, list) or not 1 <= len(candidates) <= 128:
|
||||
raise ValueError("invalid candidates")
|
||||
parsed: list[dict[str, str]] = []
|
||||
for candidate in candidates:
|
||||
if not isinstance(candidate, dict):
|
||||
raise ValueError("invalid candidate")
|
||||
column_id = candidate.get("columnId")
|
||||
text = candidate.get("text")
|
||||
if not isinstance(column_id, str) or not isinstance(text, str) or not 1 <= len(text) <= 500:
|
||||
raise ValueError("invalid candidate")
|
||||
parsed.append({"columnId": column_id, "text": text})
|
||||
return value["id"], parsed
|
||||
|
||||
|
||||
def _detect(model: Any, candidates: list[dict[str, str]]) -> list[dict[str, Any]]:
|
||||
evidence: list[dict[str, Any]] = []
|
||||
for candidate in candidates:
|
||||
result = model.extract_entities(
|
||||
candidate["text"],
|
||||
PII_LABELS,
|
||||
threshold=0.5,
|
||||
include_confidence=True,
|
||||
)
|
||||
entities = result.get("entities", {}) if isinstance(result, dict) else {}
|
||||
best: tuple[str, float] | None = None
|
||||
if isinstance(entities, dict):
|
||||
for label, matches in entities.items():
|
||||
if label not in PII_LABELS or not isinstance(matches, list):
|
||||
continue
|
||||
for match in matches:
|
||||
if not isinstance(match, dict):
|
||||
continue
|
||||
confidence = match.get("confidence")
|
||||
if not isinstance(confidence, (int, float)) or not 0 <= confidence <= 1:
|
||||
continue
|
||||
if best is None or confidence > best[1]:
|
||||
best = (label, float(confidence))
|
||||
if best is not None:
|
||||
evidence.append(
|
||||
{
|
||||
"columnId": candidate["columnId"],
|
||||
"label": best[0],
|
||||
"confidence": best[1],
|
||||
}
|
||||
)
|
||||
return evidence
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = _arguments()
|
||||
model = _load_model(args.model, args.threads)
|
||||
print(json.dumps({"ready": True}, separators=(",", ":")), flush=True)
|
||||
for line in sys.stdin:
|
||||
request_id = "invalid"
|
||||
try:
|
||||
request_id, candidates = _request(json.loads(line))
|
||||
response = {"id": request_id, "ok": True, "evidence": _detect(model, candidates)}
|
||||
except Exception:
|
||||
response = {"id": request_id, "ok": False, "error": "detection_failed"}
|
||||
print(json.dumps(response, separators=(",", ":")), flush=True)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,157 @@
|
||||
/** Shared Bash heredoc word parser for descriptor extraction and policy masking. */
|
||||
|
||||
function physicalLines(source) {
|
||||
const rawLines = source.match(/[^\n]*\n|[^\n]+$/gu) ?? [];
|
||||
if (rawLines.length === 0) rawLines.push("");
|
||||
let offset = 0;
|
||||
return rawLines.map((raw) => {
|
||||
const record = { raw, text: raw.replace(/\n$/u, "").replace(/\r$/u, ""), start: offset };
|
||||
offset += raw.length;
|
||||
return record;
|
||||
});
|
||||
}
|
||||
|
||||
function heredocOperator(line) {
|
||||
let quote = null;
|
||||
let arithmeticDepth = 0;
|
||||
for (let index = 0; index < line.length - 1; index += 1) {
|
||||
const character = line[index];
|
||||
if (quote !== null) {
|
||||
if (character === quote) quote = null;
|
||||
else if (quote === '"' && character === "\\") index += 1;
|
||||
continue;
|
||||
}
|
||||
if (character === "'" || character === '"') { quote = character; continue; }
|
||||
if (character === "\\") { index += 1; continue; }
|
||||
if (character === "#" && (index === 0 || /[ \t;|&()]/u.test(line[index - 1]))) break;
|
||||
if (character === "(" && line[index + 1] === "(") { arithmeticDepth += 1; index += 1; continue; }
|
||||
if (character === ")" && line[index + 1] === ")" && arithmeticDepth > 0) { arithmeticDepth -= 1; index += 1; continue; }
|
||||
if (arithmeticDepth > 0 || character !== "<" || line[index + 1] !== "<") continue;
|
||||
if (line[index - 1] === "<" || line[index + 2] === "<") { index += 1; continue; }
|
||||
return index;
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
function endsWithBashContinuation(line) {
|
||||
let quote = null;
|
||||
for (let index = 0; index < line.length; index += 1) {
|
||||
const character = line[index];
|
||||
if (quote === null && character === "`") { index += 1; continue; }
|
||||
if (quote === "'") { if (character === "'") quote = null; continue; }
|
||||
if (character === '"') { if (quote === '"') quote = null; else if (quote === null) quote = '"'; continue; }
|
||||
if (character !== "\\") continue;
|
||||
if (index === line.length - 1) return true;
|
||||
if (quote === null || (quote === '"' && '$`"\\'.includes(line[index + 1]))) index += 1;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function bashLogicalLine(lines, start) {
|
||||
let line = lines[start];
|
||||
let end = start;
|
||||
while (endsWithBashContinuation(line)) {
|
||||
if (end + 1 >= lines.length) break;
|
||||
line = `${line.slice(0, -1)}${lines[end + 1]}`;
|
||||
end += 1;
|
||||
}
|
||||
return { line, end };
|
||||
}
|
||||
|
||||
function bashHeredocOpener(line, operator, label, lineNumber) {
|
||||
let cursor = operator + 2;
|
||||
let stripTabs = false;
|
||||
if (line[cursor] === "-") { stripTabs = true; cursor += 1; }
|
||||
while (line[cursor] === " " || line[cursor] === "\t") cursor += 1;
|
||||
const unsupported = () => { throw new Error(`${label}:${lineNumber}: unsupported Bash heredoc opener`); };
|
||||
if (cursor >= line.length || line[cursor] === "#") unsupported();
|
||||
let delimiter = "";
|
||||
let quotedDelimiter = false;
|
||||
while (cursor < line.length) {
|
||||
const character = line[cursor];
|
||||
if (character === " " || character === "\t" || ";|&<>".includes(character)) break;
|
||||
if (character === "'" || character === '"') {
|
||||
quotedDelimiter = true;
|
||||
const quote = character;
|
||||
cursor += 1;
|
||||
let closed = false;
|
||||
while (cursor < line.length) {
|
||||
const quoted = line[cursor];
|
||||
if (quoted === quote) { closed = true; cursor += 1; break; }
|
||||
if (quote === '"' && quoted === "\\") {
|
||||
cursor += 1;
|
||||
if (cursor >= line.length) unsupported();
|
||||
const escaped = line[cursor];
|
||||
delimiter += '$`"\\'.includes(escaped) ? escaped : `\\${escaped}`;
|
||||
cursor += 1;
|
||||
continue;
|
||||
}
|
||||
delimiter += quoted;
|
||||
cursor += 1;
|
||||
}
|
||||
if (!closed) unsupported();
|
||||
continue;
|
||||
}
|
||||
if (character === "\\") {
|
||||
quotedDelimiter = true;
|
||||
cursor += 1;
|
||||
if (cursor >= line.length) unsupported();
|
||||
delimiter += line[cursor];
|
||||
cursor += 1;
|
||||
continue;
|
||||
}
|
||||
if (character === "$" || character === "`" || "(){}[]*?".includes(character)) unsupported();
|
||||
delimiter += character;
|
||||
cursor += 1;
|
||||
}
|
||||
if (delimiter.length === 0) unsupported();
|
||||
if (heredocOperator(line.slice(cursor)) >= 0) unsupported();
|
||||
return { delimiter, stripTabs, expandable: !quotedDelimiter };
|
||||
}
|
||||
|
||||
function parsedBashHeredocs(source, label) {
|
||||
const records = physicalLines(source);
|
||||
const lines = records.map((record) => record.text);
|
||||
const extracted = [];
|
||||
for (let index = 0; index < lines.length; index += 1) {
|
||||
const logical = bashLogicalLine(lines, index);
|
||||
const operator = heredocOperator(logical.line);
|
||||
if (operator < 0) { index = logical.end; continue; }
|
||||
const opener = index;
|
||||
const { delimiter, stripTabs, expandable } = bashHeredocOpener(logical.line, operator, label, index + 1);
|
||||
index = logical.end;
|
||||
const body = [];
|
||||
const startLine = index + 2;
|
||||
const bodyStart = records[index + 1]?.start ?? source.length;
|
||||
let closed = false;
|
||||
for (index += 1; index < lines.length; index += 1) {
|
||||
const candidate = stripTabs ? lines[index].replace(/^\t+/u, "") : lines[index];
|
||||
if (candidate === delimiter) { closed = true; break; }
|
||||
body.push(candidate);
|
||||
}
|
||||
const bodyEnd = closed ? records[index].start : source.length;
|
||||
extracted.push({
|
||||
source: `${body.join("\n")}\n`, label: `${label}:${startLine} Bash heredoc${closed ? "" : " (unclosed)"}`,
|
||||
expandable, closed, bodyStart, bodyEnd, path: label,
|
||||
rawBlock: records.slice(opener, Math.min(index + 1, records.length)).map((record) => record.raw).join(""),
|
||||
});
|
||||
}
|
||||
return extracted;
|
||||
}
|
||||
|
||||
function extractBashDocuments(source, label) {
|
||||
return parsedBashHeredocs(source, label).map(({ bodyStart: _start, bodyEnd: _end, closed: _closed, ...document }) => document);
|
||||
}
|
||||
|
||||
function literalBashHeredocBodyRanges(source, label) {
|
||||
const ranges = [];
|
||||
for (const heredoc of parsedBashHeredocs(source, label)) {
|
||||
if (!heredoc.expandable) {
|
||||
if (!heredoc.closed) throw new Error(`${label}: revision-state policy found an unclosed literal Bash heredoc`);
|
||||
ranges.push({ start: heredoc.bodyStart, end: heredoc.bodyEnd });
|
||||
}
|
||||
}
|
||||
return ranges;
|
||||
}
|
||||
|
||||
export { extractBashDocuments, literalBashHeredocBodyRanges };
|
||||
@@ -1195,13 +1195,8 @@ export async function executeChecks({ checks, failAt, recorder } = {}) {
|
||||
|
||||
function baseWorkspace(id, evidenceSource) {
|
||||
return {
|
||||
workspace: { schema_version: 3, id, name: `P1 ${id}`, language: "en" },
|
||||
workspace: { schema_version: 4, id, name: `P1 ${id}`, language: "en" },
|
||||
dwh: { engine: "postgres", database: "postgres", schema: "public", supported_transports: ["postgres_direct"] },
|
||||
semantic_index: {
|
||||
vector_store: { engine: "qdrant", collection: id, dimensions: 1024, distance: "cosine" },
|
||||
embedding: { provider: "ollama_internal", model: "qwen3-embedding:0.6b", dimensions: 1024 },
|
||||
},
|
||||
llm_policy: { allowed: ["zai/glm-5.2"] },
|
||||
evidence: { source: evidenceSource, policy: { max_chunk_chars: 4000, retain_published_generations: 3 } },
|
||||
};
|
||||
}
|
||||
|
||||
@@ -109,7 +109,7 @@ async function validateDistFiles(repo,files){const dist=join(repo,"backend","dis
|
||||
|
||||
export async function readManualOwnership({repositoryRoot=defaultRepositoryRoot}={}){const repo=realpathSync(repositoryRoot),root=fixedManualRoot(repo);noSymlinkExisting(repo,root);let rootEntry,ownershipEntry;try{rootEntry=await lstat(root);ownershipEntry=await lstat(join(root,"ownership.json"));}catch{throw new Error("manual ownership is missing");}if(!rootEntry.isDirectory()||rootEntry.isSymbolicLink()||await realpath(root)!==root||!ownershipEntry.isFile()||ownershipEntry.isSymbolicLink())throw new Error("manual ownership is unsafe");let value;try{value=JSON.parse(await readFile(join(root,"ownership.json"),"utf8"));}catch{throw new Error("manual ownership is malformed");}const baseValid=value.schemaVersion===1&&value.kind==="p1-manual-acceptance"&&HEX64.test(value.nonce??"")&&value.repositoryRoot===repo&&value.root===root&&value.status==="PENDING"&&["PREPARING","READY"].includes(value.stage)&&value.listener?.host===HOST&&value.listener?.port===PORT&&value.listener?.state==="stopped"&&typeof value.createdAt==="string"&&validEntrypoint(value.entrypoint,repo)&&validDistManifest(value.distManifest,root)&&JSON.stringify(value.resources)===JSON.stringify([root,{kind:"fastify",host:HOST,port:PORT}]);const readyLog=value.backendLog?.path===join(root,"logs/backend.log")&&Number.isSafeInteger(value.backendLog?.dev)&&Number.isSafeInteger(value.backendLog?.ino);if(!baseValid||(value.stage==="READY"?!readyLog:value.backendLog!==null))throw new Error("manual ownership identity mismatch");return value;}
|
||||
async function run(executable,argv,options={}){return await exec(executable,argv,{...options,maxBuffer:2*1024*1024,encoding:"utf8"});}
|
||||
function descriptor(id,source){return{workspace:{schema_version:3,id,name:`P1 ${id}`,language:"en"},dwh:{engine:"postgres",database:"postgres",schema:"public",supported_transports:["postgres_direct"]},semantic_index:{vector_store:{engine:"qdrant",collection:id,dimensions:1024,distance:"cosine"},embedding:{provider:"ollama_internal",model:"qwen3-embedding:0.6b",dimensions:1024}},llm_policy:{allowed:["zai/glm-5.2"]},evidence:{source,policy:{max_chunk_chars:4000,retain_published_generations:3}}};}
|
||||
function descriptor(id,source){return{workspace:{schema_version:4,id,name:`P1 ${id}`,language:"en"},dwh:{engine:"postgres",database:"postgres",schema:"public",supported_transports:["postgres_direct"]},evidence:{source,policy:{max_chunk_chars:4000,retain_published_generations:3}}};}
|
||||
function descriptors(){return[descriptor("p1-filesystem",{type:"filesystem",uri:"workspace-content/p1-filesystem/evidence",patterns:["**/*.md"],max_bytes:10485760}),descriptor("p1-http",{type:"http",uris:["https://evidence.example.test/guide.md"],authentication:"signed_urls_file",connect_timeout_ms:1250,read_timeout_ms:30001,max_bytes:12345,max_redirects:2,allow_private_hosts:false,max_cache_bytes:67890}),descriptor("p1-s3",{type:"s3",uri:"s3://p1-evidence/published/",endpoint_url:"https://s3.example.test/",region:"eu-west-1",credentials:"static_files",trusted_endpoint:true,allow_private_endpoint:false,allow_insecure_endpoint:false,max_bytes:12345,max_objects:33,max_pages:4,page_size:5})];}
|
||||
function quote(value){return `'${String(value).replaceAll("'",`'"'"'`)}'`;}
|
||||
async function checkPrerequisites(repo){for(const path of ["scripts/p1-acceptance.sh","scripts/test-p1-acceptance.sh","backend/scripts/p1-acceptance.mjs","backend/dist/server.js"]){try{await access(join(repo,path));}catch{throw new Error(`Task 8 prerequisite is missing: ${path}`);}}for(const command of ["node","npm","git","curl","unzip","zipinfo","lsof","python3"]){try{await run(command,[command==="unzip"||command==="lsof"?"-v":command==="zipinfo"?"-h":"--version"]);}catch{throw new Error(`missing prerequisite: ${command}`);}}const tht=join(repo,"harness",".venv","bin","tht");try{await access(tht,constants.X_OK);}catch{throw new Error("missing prerequisite: harness/.venv/bin/tht");}}
|
||||
|
||||
@@ -403,7 +403,7 @@ test("generated render command validates saved responses and owned snapshot befo
|
||||
});
|
||||
|
||||
const renderSnapshotYaml=`workspace:
|
||||
schema_version: 3
|
||||
schema_version: 4
|
||||
id: p1-filesystem
|
||||
name: P1 filesystem
|
||||
language: en
|
||||
@@ -412,11 +412,6 @@ dwh:
|
||||
database: postgres
|
||||
schema: public
|
||||
supported_transports: [postgres_direct]
|
||||
semantic_index:
|
||||
vector_store: {engine: qdrant, collection: p1-filesystem, dimensions: 1024, distance: cosine}
|
||||
embedding: {provider: ollama_internal, model: qwen3-embedding:0.6b, dimensions: 1024}
|
||||
llm_policy:
|
||||
allowed: [zai/glm-5.2]
|
||||
evidence:
|
||||
source: {type: filesystem, uri: workspace-content/p1-filesystem/evidence, patterns: ["**/*.md"], max_bytes: 10485760}
|
||||
policy: {max_chunk_chars: 4000, retain_published_generations: 3}
|
||||
@@ -441,7 +436,7 @@ test("generated render command binds snapshot bytes to the commit manifest and G
|
||||
await writeFile(readPath,JSON.stringify({revision})); await writeFile(pullPath,JSON.stringify({head:commit}));
|
||||
await assert.rejects(execFileAsync("bash",[script],{cwd:repo}),/snapshot manifest.*(missing|unbounded)/i);
|
||||
await assert.rejects(lstat(output));
|
||||
const legacyRevision={...revision}; legacyRevision.state=["oper","ational"].join("");
|
||||
const legacyRevision={...revision}; legacyRevision[["st","ate"].join("")]=["oper","ational"].join("");
|
||||
await writeFile(join(commitDir,"snapshot.json"),JSON.stringify(manifest(legacyRevision)));
|
||||
await assert.rejects(execFileAsync("bash",[script],{cwd:repo}),/snapshot manifest revision is invalid/);
|
||||
await writeFile(join(commitDir,"snapshot.json"),JSON.stringify(manifest({...revision,unexpected:"field"})));
|
||||
|
||||
@@ -16,7 +16,7 @@ async function fixture() {
|
||||
await writeFile(join(root,"installation/base.yaml"),"{}\n");
|
||||
const secret=join(root,"fixture-secrets/dwh-password"); await writeFile(secret,"not-inspected",{mode:0o600});
|
||||
await writeFile(snapshot,`workspace:
|
||||
schema_version: 3
|
||||
schema_version: 4
|
||||
id: p1-filesystem
|
||||
name: P1 filesystem
|
||||
language: en
|
||||
@@ -25,11 +25,6 @@ dwh:
|
||||
database: postgres
|
||||
schema: public
|
||||
supported_transports: [postgres_direct]
|
||||
semantic_index:
|
||||
vector_store: {engine: qdrant, collection: p1-filesystem, dimensions: 1024, distance: cosine}
|
||||
embedding: {provider: ollama_internal, model: qwen3-embedding:0.6b, dimensions: 1024}
|
||||
llm_policy:
|
||||
allowed: [zai/glm-5.2]
|
||||
evidence:
|
||||
source: {type: filesystem, uri: workspace-content/p1-filesystem/evidence, patterns: ["**/*.md"], max_bytes: 10485760}
|
||||
policy: {max_chunk_chars: 4000, retain_published_generations: 3}
|
||||
@@ -61,7 +56,7 @@ test("renderer refuses snapshot manifest head, digest, and expected-digest tampe
|
||||
|
||||
test("renderer refuses a missing or malformed snapshot manifest",async()=>{ const f=await fixture(); const output=join(f.root,"rendered/nomanifest.yaml"); await rm(f.manifestPath); await assert.rejects(call(f,{outputPath:output}),/snapshot manifest.*(missing|unbounded|unsafe)/); await writeFile(f.manifestPath,"{not json"); await assert.rejects(call(f,{outputPath:output}),/snapshot manifest.*malformed/); await assert.rejects(lstat(output)); assert.deepEqual(await runtimeLeases(f),[]); });
|
||||
|
||||
test("renderer rejects a regular snapshot replacement against its manifest",async()=>{ const f=await fixture(); const output=join(f.root,"rendered/replaced.yaml"); await assert.rejects(call(f,{outputPath:output,beforePublish:async()=>{await writeFile(f.snapshot,"workspace:\n schema_version: 3\n id: p1-filesystem\n name: replaced\n")}}),/snapshot content changed/); await assert.rejects(lstat(output)); });
|
||||
test("renderer rejects a regular snapshot replacement against its manifest",async()=>{ const f=await fixture(); const output=join(f.root,"rendered/replaced.yaml"); await assert.rejects(call(f,{outputPath:output,beforePublish:async()=>{await writeFile(f.snapshot,"workspace:\n schema_version: 4\n id: p1-filesystem\n name: replaced\n")}}),/snapshot content changed/); await assert.rejects(lstat(output)); });
|
||||
|
||||
test("renderer anchors publication when rendered parent is concurrently swapped", async()=>{
|
||||
const f=await fixture(),output=join(f.root,"rendered/raced.yaml"),moved=join(f.root,"rendered-moved"),outside=join(f.repo,"outside-rendered"); await mkdir(outside);
|
||||
|
||||
@@ -0,0 +1,900 @@
|
||||
#!/usr/bin/env node
|
||||
import { createHash, randomBytes } from "node:crypto";
|
||||
import { closeSync, constants as fsConstants, existsSync, fsyncSync, lstatSync, mkdirSync, openSync, readFileSync, realpathSync } from "node:fs";
|
||||
import { access, lstat, mkdir, open, readFile, readdir, rename, rm, writeFile } from "node:fs/promises";
|
||||
import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
|
||||
import { execFile } from "node:child_process";
|
||||
import { promisify } from "node:util";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
import {
|
||||
buildSafeEnvironment,
|
||||
collectRepositoryProvenance,
|
||||
deriveOverall,
|
||||
scanSecrets,
|
||||
} from "./p1-acceptance.mjs";
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
const modulePath = fileURLToPath(import.meta.url);
|
||||
const defaultRepositoryRoot = realpathSync(resolve(dirname(modulePath), "../.."));
|
||||
const RUN_ID = /^p11-[0-9a-f]{32}$/;
|
||||
const HEX40 = /^[0-9a-f]{40}$/;
|
||||
const HEX64 = /^[0-9a-f]{64}$/;
|
||||
const ISO_UTC = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/;
|
||||
const ZIP_FILES = ["manifest.json", "workspace.yaml", "contract.env.example", "README.md"];
|
||||
|
||||
function resolveSystemExecutable(name) {
|
||||
for (const candidate of [`/usr/bin/${name}`, `/bin/${name}`, `/opt/homebrew/bin/${name}`, `/usr/local/bin/${name}`]) {
|
||||
try {
|
||||
const resolved = realpathSync(candidate);
|
||||
if (lstatSync(resolved).isFile()) return resolved;
|
||||
} catch {}
|
||||
}
|
||||
throw new Error(`required executable not found: ${name}`);
|
||||
}
|
||||
function resolveExecutables(repositoryRoot) {
|
||||
const repo = canonicalRoot(repositoryRoot);
|
||||
const thtPath = join(repo, "harness", ".venv", "bin", "tht");
|
||||
if (!existsSync(thtPath)) throw new Error("required executable not found: tht");
|
||||
return { gitPath: resolveSystemExecutable("git"), pythonPath: resolveSystemExecutable("python3"), thtPath: realpathSync(thtPath) };
|
||||
}
|
||||
const TOPOLOGY = [
|
||||
"remote.git", "author", "installation/registry", "installation/data", "installation/runtime",
|
||||
"fixture-secrets", "fixtures/descriptors", "fixtures/requests", "requests", "responses",
|
||||
"exports/raw", "exports/extracted", "rendered", "logs",
|
||||
];
|
||||
export const CHECK_IDS = Object.freeze([
|
||||
"preflight",
|
||||
"clean_state",
|
||||
"ownership",
|
||||
"catalog_bootstrap",
|
||||
"catalog_only_listing",
|
||||
"bootstrap_create_once",
|
||||
"api_curator_boundary",
|
||||
"curator_descriptor_update",
|
||||
"content_only_revision",
|
||||
"docs_only_reconciliation",
|
||||
"same_revision_git_objects",
|
||||
"snapshot_and_export",
|
||||
"runtime_render_determinism",
|
||||
"tht_config_check",
|
||||
"negative_catalog_layout_cases",
|
||||
"negative_schema_context_cases",
|
||||
"no_p2_scope_artifacts",
|
||||
"secret_scan",
|
||||
"cleanup_confinement",
|
||||
]);
|
||||
|
||||
function nowIso() { return new Date().toISOString(); }
|
||||
function sha256(value) { return createHash("sha256").update(value).digest("hex"); }
|
||||
function assert(condition, message) { if (!condition) throw new Error(message); }
|
||||
function scalarSecretBytes(value) {
|
||||
if (typeof value !== "string" || value.length === 0 || /\s|\0/.test(value)) throw new Error("scalar fixture secret is invalid");
|
||||
return Buffer.from(value);
|
||||
}
|
||||
function canonicalRoot(repositoryRoot) { return realpathSync(repositoryRoot); }
|
||||
export function canonicalIntegrationBase(repositoryRoot = defaultRepositoryRoot) {
|
||||
return join(canonicalRoot(repositoryRoot), ".artifacts", "p11-integration");
|
||||
}
|
||||
export function validateRunRoot(repositoryRoot, runRoot, runId) {
|
||||
if (!RUN_ID.test(runId)) throw new Error("invalid owned run id");
|
||||
const base = canonicalIntegrationBase(repositoryRoot);
|
||||
const lexical = resolve(runRoot);
|
||||
if (dirname(lexical) !== base || basename(lexical) !== runId) throw new Error("run root is not a direct integration child");
|
||||
return lexical;
|
||||
}
|
||||
function validateNoSymlinkAncestors(repositoryRoot, target) {
|
||||
const repo = canonicalRoot(repositoryRoot);
|
||||
const rel = relative(repo, target);
|
||||
if (rel.startsWith("..") || isAbsolute(rel)) throw new Error("path leaves repository");
|
||||
let cursor = repo;
|
||||
for (const part of rel.split(sep).filter(Boolean)) {
|
||||
cursor = join(cursor, part);
|
||||
if (!existsSync(cursor)) break;
|
||||
const entry = lstatSync(cursor);
|
||||
if (entry.isSymbolicLink()) throw new Error("owned path ancestor is a symlink");
|
||||
}
|
||||
}
|
||||
async function atomicWrite(path, bytes, mode = 0o600) {
|
||||
await mkdir(dirname(path), { recursive: true });
|
||||
const staging = join(dirname(path), `.${basename(path)}.${randomBytes(12).toString("hex")}.tmp`);
|
||||
let handle;
|
||||
try {
|
||||
handle = await open(staging, "wx", mode);
|
||||
await handle.writeFile(bytes);
|
||||
await handle.sync();
|
||||
await handle.close();
|
||||
handle = undefined;
|
||||
await rename(staging, path);
|
||||
const directory = openSync(dirname(path), fsConstants.O_RDONLY);
|
||||
try { fsyncSync(directory); } finally { closeSync(directory); }
|
||||
} catch (error) {
|
||||
if (handle) await handle.close().catch(() => {});
|
||||
await rm(staging, { force: true }).catch(() => {});
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
function exactOwnedResources(run) {
|
||||
return [
|
||||
run.root,
|
||||
join(run.root, "remote.git"),
|
||||
join(run.root, "author"),
|
||||
join(run.root, "installation", "registry"),
|
||||
join(run.root, "installation", "data"),
|
||||
join(run.root, "installation", "runtime"),
|
||||
];
|
||||
}
|
||||
function initialListeners(pid) {
|
||||
return [{ name: "primary", kind: "fastify", host: "127.0.0.1", requestedPort: 0, pid, state: "not_started" }];
|
||||
}
|
||||
function ownershipValue(run, listeners = run.listeners) {
|
||||
return {
|
||||
schemaVersion: 1,
|
||||
kind: "p11-acceptance",
|
||||
runId: run.runId,
|
||||
runNonce: run.nonce,
|
||||
root: run.root,
|
||||
repositoryRoot: run.repositoryRoot,
|
||||
startedAt: run.startedAt,
|
||||
pid: run.pid,
|
||||
listeners,
|
||||
resources: exactOwnedResources(run),
|
||||
};
|
||||
}
|
||||
async function writeOwnership(run, listenerUpdate) {
|
||||
const listeners = listenerUpdate
|
||||
? run.listeners.map((listener) => listener.name === listenerUpdate.name ? listenerUpdate : listener)
|
||||
: run.listeners;
|
||||
await atomicWrite(join(run.root, "ownership.json"), `${JSON.stringify(ownershipValue(run, listeners), null, 2)}\n`);
|
||||
run.listeners = listeners;
|
||||
}
|
||||
export async function createOwnedRun({ repositoryRoot = defaultRepositoryRoot, runId, nonce, now, pid } = {}) {
|
||||
const repo = canonicalRoot(repositoryRoot);
|
||||
const base = canonicalIntegrationBase(repo);
|
||||
validateNoSymlinkAncestors(repo, base);
|
||||
await mkdir(join(repo, ".artifacts"), { mode: 0o700 }).catch((error) => { if (error.code !== "EEXIST") throw error; });
|
||||
await mkdir(base, { mode: 0o700 }).catch((error) => { if (error.code !== "EEXIST") throw error; });
|
||||
const id = runId ?? `p11-${randomBytes(16).toString("hex")}`;
|
||||
const root = validateRunRoot(repo, join(base, id), id);
|
||||
const run = {
|
||||
repositoryRoot: repo,
|
||||
root,
|
||||
runId: id,
|
||||
nonce: nonce ?? randomBytes(32).toString("hex"),
|
||||
startedAt: now ?? nowIso(),
|
||||
pid: pid ?? process.pid,
|
||||
listeners: initialListeners(pid ?? process.pid),
|
||||
};
|
||||
if (!HEX64.test(run.nonce) || !ISO_UTC.test(run.startedAt)) throw new Error("invalid ownership identity");
|
||||
await mkdir(root, { mode: 0o700 });
|
||||
await writeOwnership(run);
|
||||
return run;
|
||||
}
|
||||
function strictOwnership(value, run, expectedNonce) {
|
||||
if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error("ownership is malformed");
|
||||
const listener = value.listeners?.[0];
|
||||
const validListener = Array.isArray(value.listeners) && value.listeners.length === 1
|
||||
&& listener?.name === "primary" && listener.kind === "fastify" && listener.host === "127.0.0.1"
|
||||
&& listener.requestedPort === 0 && listener.pid === process.pid
|
||||
&& ["not_started", "listening", "closed", "close_failed"].includes(listener.state)
|
||||
&& (listener.state === "not_started" ? !("actualPort" in listener)
|
||||
: Number.isInteger(listener.actualPort) && listener.actualPort >= 1 && listener.actualPort <= 65535);
|
||||
if (value.schemaVersion !== 1 || value.kind !== "p11-acceptance" || value.runId !== run.runId || value.runNonce !== expectedNonce
|
||||
|| value.root !== run.root || value.repositoryRoot !== run.repositoryRoot || value.pid !== process.pid
|
||||
|| !ISO_UTC.test(value.startedAt ?? "") || !validListener
|
||||
|| JSON.stringify(value.resources) !== JSON.stringify(exactOwnedResources(run))) throw new Error("ownership identity mismatch");
|
||||
return value;
|
||||
}
|
||||
export async function readAndValidateOwnership({ repositoryRoot = defaultRepositoryRoot, runRoot, expectedNonce }) {
|
||||
const repo = canonicalRoot(repositoryRoot);
|
||||
const id = basename(resolve(runRoot));
|
||||
const lexical = validateRunRoot(repo, runRoot, id);
|
||||
const rootEntry = await lstat(lexical);
|
||||
if (!rootEntry.isDirectory() || rootEntry.isSymbolicLink()) throw new Error("owned run root is not a directory");
|
||||
const ownershipPath = join(lexical, "ownership.json");
|
||||
const ownershipEntry = await lstat(ownershipPath);
|
||||
if (!ownershipEntry.isFile() || ownershipEntry.isSymbolicLink()) throw new Error("ownership file is unsafe");
|
||||
let value;
|
||||
try { value = JSON.parse(await readFile(ownershipPath, "utf8")); } catch { throw new Error("ownership is malformed"); }
|
||||
return strictOwnership(value, {
|
||||
repositoryRoot: repo,
|
||||
root: lexical,
|
||||
runId: id,
|
||||
nonce: expectedNonce,
|
||||
startedAt: value.startedAt,
|
||||
pid: process.pid,
|
||||
}, expectedNonce);
|
||||
}
|
||||
export async function cleanupOwnedRun({ repositoryRoot = defaultRepositoryRoot, runRoot, expectedNonce }) {
|
||||
const value = await readAndValidateOwnership({ repositoryRoot, runRoot, expectedNonce });
|
||||
const base = canonicalIntegrationBase(repositoryRoot);
|
||||
const tombstone = join(base, `.deleting-${value.runId}-${expectedNonce.slice(0, 16)}`);
|
||||
await rename(runRoot, tombstone);
|
||||
await rm(tombstone, { recursive: true, force: false });
|
||||
}
|
||||
async function finalizeOwnedRun({ run, success, keep }) {
|
||||
if (!success || keep) return false;
|
||||
await cleanupOwnedRun({ repositoryRoot: run.repositoryRoot, runRoot: run.root, expectedNonce: run.nonce });
|
||||
return true;
|
||||
}
|
||||
|
||||
function sanitizeForEvidence(value, forbiddenValues = []) {
|
||||
const forbidden = forbiddenValues.filter((item) => typeof item === "string" && item.length > 0);
|
||||
const redactString = (input) => forbidden.reduce((text, secret) => text.split(secret).join("[REDACTED]"), input);
|
||||
if (typeof value === "string") return redactString(value);
|
||||
if (Array.isArray(value)) return value.map((item) => sanitizeForEvidence(item, forbiddenValues));
|
||||
if (value && typeof value === "object") return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, sanitizeForEvidence(item, forbiddenValues)]));
|
||||
return value;
|
||||
}
|
||||
async function fileArtifact(root, relativePath) {
|
||||
const bytes = await readFile(join(root, relativePath));
|
||||
return { path: relativePath.split(sep).join("/"), sha256: sha256(bytes) };
|
||||
}
|
||||
async function evidence(run, relativePath, value, forbiddenValues = []) {
|
||||
await atomicWrite(join(run.root, relativePath), `${JSON.stringify(sanitizeForEvidence(value, forbiddenValues), null, 2)}\n`);
|
||||
return await fileArtifact(run.root, relativePath);
|
||||
}
|
||||
async function writeJson(path, value) {
|
||||
await atomicWrite(path, `${JSON.stringify(value, null, 2)}\n`);
|
||||
}
|
||||
async function walkFiles(root) {
|
||||
const files = [];
|
||||
async function visit(dir) {
|
||||
for (const entry of await readdir(dir, { withFileTypes: true })) {
|
||||
const path = join(dir, entry.name);
|
||||
if (entry.isDirectory()) await visit(path);
|
||||
else if (entry.isFile()) files.push({ path, rel: relative(root, path).split(sep).join("/") });
|
||||
}
|
||||
}
|
||||
if (existsSync(root)) await visit(root);
|
||||
return files.sort((a, b) => a.rel.localeCompare(b.rel));
|
||||
}
|
||||
async function snapshotDigest(root) {
|
||||
const result = {};
|
||||
for (const file of await walkFiles(root)) result[file.rel] = sha256(await readFile(file.path));
|
||||
return result;
|
||||
}
|
||||
function assertByteIdentical(left, right, label) {
|
||||
if (JSON.stringify(left) !== JSON.stringify(right)) throw new Error(`${label} changed unexpectedly`);
|
||||
}
|
||||
async function writeReportFiles({ run, report }) {
|
||||
validateReport(report);
|
||||
await writeJson(join(run.root, "report.json"), report);
|
||||
const lines = [
|
||||
`# P1.1 acceptance report`,
|
||||
"",
|
||||
`Run ID: ${report.runId}`,
|
||||
`Overall: ${report.overall}`,
|
||||
"",
|
||||
...report.checks.map((check) => `- ${check.id}: ${check.status}`),
|
||||
"",
|
||||
`report.json sha256: ${sha256(await readFile(join(run.root, "report.json")))}`,
|
||||
`P1.1 automated integration: ${report.overall}`,
|
||||
"P1.1 manual acceptance: PENDING",
|
||||
];
|
||||
await atomicWrite(join(run.root, "report.md"), `${lines.join("\n")}\n`);
|
||||
}
|
||||
export function validateReport(report) {
|
||||
if (!report || typeof report !== "object" || Array.isArray(report)) throw new Error("report is malformed");
|
||||
if (report.schemaVersion !== 1 || !RUN_ID.test(report.runId ?? "") || !ISO_UTC.test(report.startedAt ?? "")
|
||||
|| !ISO_UTC.test(report.finishedAt ?? "") || report.command !== "p11-acceptance integration --keep") throw new Error("report identity is invalid");
|
||||
if (report.overall !== deriveOverall(report.checks ?? [])) throw new Error("report overall is not derived");
|
||||
if (!Array.isArray(report.checks) || report.checks.length !== CHECK_IDS.length) throw new Error("report checks are incomplete");
|
||||
const ids = report.checks.map((check) => check.id);
|
||||
if (JSON.stringify(ids) !== JSON.stringify(CHECK_IDS)) throw new Error("report checks are not exact");
|
||||
const artifactPaths = new Set();
|
||||
for (const check of report.checks) {
|
||||
if (!["PASS", "FAIL"].includes(check.status) || !ISO_UTC.test(check.startedAt ?? "") || !ISO_UTC.test(check.finishedAt ?? "")) {
|
||||
throw new Error("report check metadata is invalid");
|
||||
}
|
||||
if (!Array.isArray(check.commands) || check.commands.some((command) => typeof command !== "string" || !/^[A-Za-z0-9._+-]+$/.test(command))) {
|
||||
throw new Error("report command is invalid");
|
||||
}
|
||||
if (!Array.isArray(check.artifacts)) throw new Error("report artifacts are invalid");
|
||||
for (const artifact of check.artifacts) {
|
||||
if (typeof artifact.path !== "string" || artifact.path.startsWith("/") || artifact.path.includes("..") || !/^[A-Za-z0-9._/-]+$/.test(artifact.path)) {
|
||||
throw new Error("report artifact path is invalid");
|
||||
}
|
||||
if (!HEX64.test(artifact.sha256 ?? "")) throw new Error("report artifact hash is invalid");
|
||||
if (artifactPaths.has(artifact.path)) throw new Error("report artifact path is duplicated");
|
||||
artifactPaths.add(artifact.path);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function execCommand(executable, argv, { cwd, env, timeoutMs = 30_000, stdin } = {}) {
|
||||
if (!Array.isArray(argv) || argv.some((value) => typeof value !== "string")) throw new Error("command argv must be a string array");
|
||||
const result = await execFileAsync(executable, argv, {
|
||||
cwd,
|
||||
env,
|
||||
timeout: timeoutMs,
|
||||
maxBuffer: 16 * 1024 * 1024,
|
||||
encoding: "utf8",
|
||||
...(stdin === undefined ? {} : { input: stdin }),
|
||||
});
|
||||
return { code: 0, stdout: result.stdout ?? "", stderr: result.stderr ?? "" };
|
||||
}
|
||||
async function git(ctx, argv, options = {}) {
|
||||
return await execCommand(ctx.executables.gitPath, argv, { ...options, env: ctx.env });
|
||||
}
|
||||
async function tht(ctx, argv, options = {}) {
|
||||
try {
|
||||
return await execCommand(ctx.executables.thtPath, argv, { ...options, env: ctx.env });
|
||||
} catch (error) {
|
||||
if (typeof error?.code === "number") return { code: error.code, stdout: error.stdout ?? "", stderr: error.stderr ?? "" };
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
function namespace(id) { return id.toUpperCase().replaceAll("-", "_"); }
|
||||
function baseWorkspace(id, evidenceSource) {
|
||||
return {
|
||||
workspace: { schema_version: 4, id, name: `P1.1 ${id}`, description: `Catalog entry for ${id}`, language: "en" },
|
||||
dwh: { engine: "postgres", database: "postgres", schema: "public", supported_transports: ["postgres_direct"] },
|
||||
evidence: { source: evidenceSource, policy: { max_chunk_chars: 4000, retain_published_generations: 3 } },
|
||||
};
|
||||
}
|
||||
function descriptors() {
|
||||
return [
|
||||
baseWorkspace("p11-filesystem", { type: "filesystem", uri: "p11-filesystem/evidence", patterns: ["**/*.md"], max_bytes: 10485760 }),
|
||||
baseWorkspace("p11-http", { type: "http", uris: ["https://evidence.example.test/guide.md"], authentication: "signed_urls_file", connect_timeout_ms: 1250, read_timeout_ms: 30001, max_bytes: 12345, max_redirects: 2, allow_private_hosts: false, max_cache_bytes: 67890 }),
|
||||
baseWorkspace("p11-s3", { type: "s3", uri: "s3://p11-evidence/published/", endpoint_url: "https://s3.example.test/", region: "eu-west-1", credentials: "static_files", trusted_endpoint: true, allow_private_endpoint: false, allow_insecure_endpoint: false, max_bytes: 12345, max_objects: 33, max_pages: 4, page_size: 5 }),
|
||||
];
|
||||
}
|
||||
async function createTopology(run) {
|
||||
for (const path of TOPOLOGY) await mkdir(join(run.root, path), { recursive: true, mode: path === "fixture-secrets" ? 0o700 : 0o755 });
|
||||
}
|
||||
async function setupSecrets(ctx) {
|
||||
const secretDir = join(ctx.run.root, "fixture-secrets");
|
||||
const values = {
|
||||
dwh: `DWH-${randomBytes(12).toString("hex")}`,
|
||||
signed: `SIGNED-${randomBytes(12).toString("hex")}`,
|
||||
access: `ACCESS-${randomBytes(12).toString("hex")}`,
|
||||
secret: `SECRET-${randomBytes(12).toString("hex")}`,
|
||||
session: `SESSION-${randomBytes(12).toString("hex")}`,
|
||||
rejected: `REJECTED-${randomBytes(12).toString("hex")}`,
|
||||
};
|
||||
ctx.forbiddenValues = Object.values(values);
|
||||
ctx.secretValues = values;
|
||||
const paths = {
|
||||
dwh: join(secretDir, "dwh-password"),
|
||||
signed: join(secretDir, "evidence-signed-urls.json"),
|
||||
access: join(secretDir, "evidence-access"),
|
||||
secret: join(secretDir, "evidence-secret"),
|
||||
session: join(secretDir, "evidence-session"),
|
||||
};
|
||||
await atomicWrite(paths.dwh, scalarSecretBytes(values.dwh));
|
||||
await atomicWrite(paths.signed, JSON.stringify([`https://evidence.example.test/guide.md?token=${values.signed}`]));
|
||||
await atomicWrite(paths.access, scalarSecretBytes(values.access));
|
||||
await atomicWrite(paths.secret, scalarSecretBytes(values.secret));
|
||||
await atomicWrite(paths.session, scalarSecretBytes(values.session));
|
||||
const env = {};
|
||||
for (const workspace of ctx.descriptors) {
|
||||
const prefix = `THT_WS_${namespace(workspace.workspace.id)}`;
|
||||
Object.assign(env, {
|
||||
[`${prefix}_DWH_TRANSPORT`]: "postgres_direct",
|
||||
[`${prefix}_DWH_HOST`]: "dwh.invalid",
|
||||
[`${prefix}_DWH_PORT`]: "5432",
|
||||
[`${prefix}_DWH_USER`]: "reader",
|
||||
[`${prefix}_DWH_PASSWORD_FILE`]: paths.dwh,
|
||||
});
|
||||
}
|
||||
Object.assign(env, {
|
||||
THT_WS_P11_HTTP_EVIDENCE_SIGNED_URLS_FILE: paths.signed,
|
||||
THT_WS_P11_S3_EVIDENCE_ACCESS_KEY_FILE: paths.access,
|
||||
THT_WS_P11_S3_EVIDENCE_SECRET_KEY_FILE: paths.secret,
|
||||
THT_WS_P11_S3_EVIDENCE_SESSION_TOKEN_FILE: paths.session,
|
||||
});
|
||||
Object.assign(ctx.env, env);
|
||||
await atomicWrite(join(ctx.run.root, "installation", "bindings.env"), `${Object.entries(env).map(([key, value]) => `${key}=${value}`).join("\n")}\n`);
|
||||
await atomicWrite(join(ctx.run.root, "installation", "runtime", "base.yaml"), "{}\n");
|
||||
}
|
||||
function catalog(entries = ctxDescriptors) {
|
||||
return { schema_version: 1, workspaces: entries.map(({ workspace }) => ({ id: workspace.id, name: workspace.name, description: workspace.description })) };
|
||||
}
|
||||
const ctxDescriptors = descriptors();
|
||||
async function initializeGit(ctx) {
|
||||
const author = join(ctx.run.root, "author");
|
||||
await git(ctx, ["init", "--bare", "--initial-branch=main", join(ctx.run.root, "remote.git")], { cwd: ctx.run.root });
|
||||
await git(ctx, ["clone", join(ctx.run.root, "remote.git"), author], { cwd: ctx.run.root });
|
||||
await git(ctx, ["config", "user.name", "P1 Fixture Curator"], { cwd: author });
|
||||
await git(ctx, ["config", "user.email", "p1-curator@example.invalid"], { cwd: author });
|
||||
const catalogBytes = `${JSON.stringify({
|
||||
schema_version: 1,
|
||||
workspaces: [
|
||||
...catalog(ctx.descriptors).workspaces,
|
||||
{ id: "p11-pending", name: "P1.1 pending", description: "Catalog-only slot awaiting bootstrap" },
|
||||
],
|
||||
}, null, 2)}\n`;
|
||||
await atomicWrite(join(author, "thoth-workspaces.yaml"), catalogBytes, 0o644);
|
||||
const evidenceRoot = join(author, "p11-filesystem", "evidence");
|
||||
await mkdir(join(evidenceRoot, "domain"), { recursive: true });
|
||||
await atomicWrite(join(evidenceRoot, "guide.md"), "# P1.1 curated Evidence\n", 0o644);
|
||||
await atomicWrite(join(evidenceRoot, "domain", "table.md"), "# Curated table\n", 0o644);
|
||||
await git(ctx, ["add", "thoth-workspaces.yaml"], { cwd: author });
|
||||
await git(ctx, ["add", "p11-filesystem/evidence/guide.md"], { cwd: author });
|
||||
await git(ctx, ["add", "-A", "p11-filesystem/evidence"], { cwd: author });
|
||||
await git(ctx, ["commit", "-m", "Bootstrap curated P1 content"], { cwd: author });
|
||||
await git(ctx, ["push", "origin", "main"], { cwd: author });
|
||||
ctx.bootstrapCommit = (await git(ctx, ["rev-parse", "HEAD"], { cwd: author })).stdout.trim();
|
||||
ctx.catalogBlobBefore = (await git(ctx, ["rev-parse", `HEAD:thoth-workspaces.yaml`], { cwd: author })).stdout.trim();
|
||||
ctx.evidenceTreeBefore = (await git(ctx, ["rev-parse", `HEAD:p11-filesystem/evidence`], { cwd: author })).stdout.trim();
|
||||
}
|
||||
async function loadProductionBackend() {
|
||||
const [{ loadConfig }, { buildApp }, { WorkspaceRegistry }, { ThtRunner }] = await Promise.all([
|
||||
import("../dist/config.js"),
|
||||
import("../dist/app.js"),
|
||||
import("../dist/workspaces/registry.js"),
|
||||
import("../dist/tht/tht-runner.js"),
|
||||
]);
|
||||
return { loadConfig, buildApp, WorkspaceRegistry, ThtRunner };
|
||||
}
|
||||
async function startBackend(ctx) {
|
||||
const { loadConfig, buildApp, WorkspaceRegistry, ThtRunner } = await loadProductionBackend();
|
||||
const config = loadConfig(ctx.env);
|
||||
const registry = new WorkspaceRegistry(config.workspaceRegistry);
|
||||
const thtRunner = new ThtRunner({
|
||||
thtBin: config.thtBin,
|
||||
harnessDir: config.harnessDir,
|
||||
configPath: join(ctx.run.root, "installation", "runtime", "base.yaml"),
|
||||
dataRoot: config.dataRoot,
|
||||
runtimeSnapshotRoot: join(config.workspaceRegistry.root, "snapshots", "runtime"),
|
||||
secretRoots: config.workspaceRegistry.secretRoots,
|
||||
secretsFile: config.secretsFile,
|
||||
secretFiles: config.secretFiles,
|
||||
semanticRuntime: {
|
||||
internalQdrantUrl: config.internalQdrantUrl,
|
||||
internalEmbeddingUrl: config.internalEmbeddingUrl,
|
||||
internalEmbeddingModel: config.internalEmbeddingModel,
|
||||
internalEmbeddingDimensions: config.internalEmbeddingDimensions,
|
||||
},
|
||||
});
|
||||
const app = buildApp(config, { thtRunner, workspaceRegistry: registry });
|
||||
const address = await app.listen({ host: "127.0.0.1", port: 0 });
|
||||
const baseUrl = `http://127.0.0.1:${new URL(address).port}`;
|
||||
ctx.registry = registry;
|
||||
ctx.thtRunner = thtRunner;
|
||||
ctx.app = app;
|
||||
ctx.baseUrl = baseUrl;
|
||||
await writeOwnership(ctx.run, {
|
||||
name: "primary", kind: "fastify", host: "127.0.0.1", requestedPort: 0,
|
||||
actualPort: Number(new URL(address).port), pid: process.pid, state: "listening",
|
||||
});
|
||||
}
|
||||
async function stopBackend(ctx) {
|
||||
if (ctx.app) {
|
||||
await ctx.app.close().catch(() => {});
|
||||
await writeOwnership(ctx.run, {
|
||||
name: "primary", kind: "fastify", host: "127.0.0.1", requestedPort: 0,
|
||||
actualPort: Number(new URL(ctx.baseUrl).port), pid: process.pid, state: "closed",
|
||||
}).catch(() => {});
|
||||
}
|
||||
}
|
||||
async function request(ctx, id, method, path, body, binary = false, safeInput) {
|
||||
const requestSummary = safeInput === undefined
|
||||
? { method, path, ...(body === undefined ? {} : { body: sanitizeForEvidence(body, ctx.forbiddenValues) }) }
|
||||
: { method, path, input: safeInput };
|
||||
await evidence(ctx.run, `requests/${id}.json`, requestSummary, ctx.forbiddenValues);
|
||||
const response = await fetch(`${ctx.baseUrl}${path}`, {
|
||||
method,
|
||||
headers: body === undefined ? {} : { "content-type": "application/json" },
|
||||
...(body === undefined ? {} : { body: JSON.stringify(body) }),
|
||||
signal: AbortSignal.timeout(15_000),
|
||||
});
|
||||
if (binary) {
|
||||
const bytes = Buffer.from(await response.arrayBuffer());
|
||||
await atomicWrite(join(ctx.run.root, `exports/raw/${id}.zip`), bytes);
|
||||
await evidence(ctx.run, `responses/${id}.json`, { status: response.status, bytes: bytes.length, contentType: response.headers.get("content-type") });
|
||||
return { status: response.status, bytes };
|
||||
}
|
||||
const text = await response.text();
|
||||
let parsed;
|
||||
try { parsed = text ? JSON.parse(text) : null; } catch { parsed = { invalidJson: true, raw: text }; }
|
||||
await evidence(ctx.run, `responses/${id}.json`, { status: response.status, body: sanitizeForEvidence(parsed, ctx.forbiddenValues) }, ctx.forbiddenValues);
|
||||
return { status: response.status, body: parsed };
|
||||
}
|
||||
async function extractZip(ctx, id, bytes) {
|
||||
const yauzl = (await import("yauzl")).default;
|
||||
const output = join(ctx.run.root, "exports", "extracted", id);
|
||||
await mkdir(output, { recursive: true });
|
||||
const files = await new Promise((resolvePromise, reject) => {
|
||||
yauzl.fromBuffer(bytes, { lazyEntries: true, strictFileNames: true, validateEntrySizes: true }, (error, zip) => {
|
||||
if (error || !zip) return reject(error ?? new Error("zip open failed"));
|
||||
const collected = new Map();
|
||||
zip.on("error", reject);
|
||||
zip.on("entry", (entry) => {
|
||||
if (!ZIP_FILES.includes(entry.fileName) || entry.fileName.includes("..") || entry.fileName.startsWith("/") || entry.fileName.endsWith("/")) return reject(new Error("unsafe export entry"));
|
||||
zip.openReadStream(entry, (streamError, stream) => {
|
||||
if (streamError || !stream) return reject(streamError ?? new Error("zip stream failed"));
|
||||
const chunks = [];
|
||||
stream.on("data", (chunk) => chunks.push(chunk));
|
||||
stream.on("error", reject);
|
||||
stream.on("end", async () => {
|
||||
const buffer = Buffer.concat(chunks);
|
||||
collected.set(entry.fileName, buffer);
|
||||
await atomicWrite(join(output, entry.fileName), buffer);
|
||||
zip.readEntry();
|
||||
});
|
||||
});
|
||||
});
|
||||
zip.on("end", () => resolvePromise(collected));
|
||||
zip.readEntry();
|
||||
});
|
||||
});
|
||||
assert(files.size === ZIP_FILES.length, "export bundle entry mismatch");
|
||||
return JSON.parse(files.get("manifest.json").toString("utf8"));
|
||||
}
|
||||
function checkResult(id, startedAt, status, artifacts = [], commands = [], error) {
|
||||
return { id, status, startedAt, finishedAt: nowIso(), artifacts, commands, ...(error ? { error } : {}) };
|
||||
}
|
||||
async function executeChecks({ checks }) {
|
||||
const results = [];
|
||||
let stopped = false;
|
||||
for (const scenario of checks) {
|
||||
const startedAt = nowIso();
|
||||
if (stopped) {
|
||||
results.push(checkResult(scenario.id, startedAt, "FAIL", [], [], "Not executed after earlier failure."));
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
const output = await scenario.run();
|
||||
results.push(checkResult(scenario.id, startedAt, "PASS", output.artifacts ?? [], output.commands ?? []));
|
||||
} catch (error) {
|
||||
const partial = error?.acceptancePartial ?? {};
|
||||
results.push(checkResult(scenario.id, startedAt, "FAIL", partial.artifacts ?? [], partial.commands ?? [], "Acceptance scenario failed safely."));
|
||||
stopped = true;
|
||||
}
|
||||
}
|
||||
return results;
|
||||
}
|
||||
async function registryState(ctx) {
|
||||
const statePath = join(ctx.run.root, "installation", "registry", "state", "active.json");
|
||||
const active = JSON.parse(await readFile(statePath, "utf8"));
|
||||
return {
|
||||
head: active.head,
|
||||
revisions: active.revisions.map((revision) => ({ id: revision.id, commit: revision.commit, blob: revision.blob })),
|
||||
catalog: active.catalog ?? null,
|
||||
};
|
||||
}
|
||||
function safeErrorEnvelope(response, code, status) {
|
||||
assert(response.status === status, `expected ${status}`);
|
||||
assert(response.body?.code === code, `expected error code ${code}`);
|
||||
assert(Object.keys(response.body).sort().join(",") === "code,message", "error envelope is not exact");
|
||||
}
|
||||
async function productionChecks(ctx) {
|
||||
const check = async (id, value, commands = []) => ({ commands, artifacts: [await evidence(ctx.run, `logs/${id}.json`, value, ctx.forbiddenValues)] });
|
||||
return [
|
||||
{ id: "preflight", run: async () => check("preflight", { node: process.version, repositoryHead: ctx.provenance.head, repositoryTree: ctx.provenance.tree, clean: ctx.provenance.clean, thtExecutable: true }) },
|
||||
{ id: "clean_state", run: async () => check("clean_state", { runId: ctx.run.runId, reused: false }) },
|
||||
{ id: "ownership", run: async () => { await readAndValidateOwnership({ repositoryRoot: ctx.repositoryRoot, runRoot: ctx.run.root, expectedNonce: ctx.run.nonce }); return await check("ownership", { valid: true }); } },
|
||||
{ id: "catalog_bootstrap", run: async () => {
|
||||
await initializeGit(ctx);
|
||||
for (const workspace of ctx.descriptors) await atomicWrite(join(ctx.run.root, "fixtures", "descriptors", `${workspace.workspace.id}.json`), `${JSON.stringify(workspace, null, 2)}\n`);
|
||||
return {
|
||||
commands: ["git"],
|
||||
artifacts: [
|
||||
await evidence(ctx.run, "logs/catalog-bootstrap.json", { bootstrapCommit: ctx.bootstrapCommit, catalogOnly: true }),
|
||||
await fileArtifact(ctx.run.root, "author/thoth-workspaces.yaml"),
|
||||
await fileArtifact(ctx.run.root, "author/p11-filesystem/evidence/guide.md"),
|
||||
],
|
||||
};
|
||||
} },
|
||||
{ id: "catalog_only_listing", run: async () => {
|
||||
await startBackend(ctx);
|
||||
const status = await request(ctx, "registry-status", "GET", "/workspace-registry/status");
|
||||
assert(status.status === 200 && status.body.head === ctx.bootstrapCommit, "status head mismatch");
|
||||
const listed = await request(ctx, "workspace-list-initial", "GET", "/workspaces");
|
||||
assert(listed.status === 200 && listed.body.length === 4, "catalog listing failed");
|
||||
assert(listed.body.every((entry) => entry.configurationState === "configuration_required"), "catalog entries were not configuration_required");
|
||||
ctx.baseCommit = status.body.head;
|
||||
return await check("catalog_only_listing", { head: status.body.head, ids: listed.body.map((entry) => entry.id), allConfigurationRequired: true });
|
||||
} },
|
||||
{ id: "bootstrap_create_once", run: async () => {
|
||||
let base = ctx.baseCommit;
|
||||
ctx.bootstrapResponses = {};
|
||||
for (const workspace of ctx.descriptors) {
|
||||
const validated = await request(ctx, `validate-${workspace.workspace.id}`, "POST", "/workspaces/validate", { workspace });
|
||||
assert(validated.status === 200, `validate failed ${workspace.workspace.id}`);
|
||||
const published = await request(ctx, `publish-${workspace.workspace.id}`, "POST", "/workspaces/publish", { action: "create", workspace, baseCommit: base });
|
||||
assert(published.status === 200 && HEX40.test(published.body.revision.commit), `publish failed ${workspace.workspace.id}`);
|
||||
ctx.bootstrapResponses[workspace.workspace.id] = published.body;
|
||||
base = published.body.revision.commit;
|
||||
}
|
||||
ctx.publishHead = base;
|
||||
const listed = await request(ctx, "workspace-list-ready", "GET", "/workspaces");
|
||||
assert(listed.body.filter((entry) => entry.configurationState === "ready").length === 3, "bootstrap did not activate all published entries");
|
||||
assert(listed.body.find((entry) => entry.id === "p11-pending")?.configurationState === "configuration_required", "pending slot was not left unconfigured");
|
||||
return await check("bootstrap_create_once", { head: base, readyIds: listed.body.filter((entry) => entry.configurationState === "ready").map((entry) => entry.id) });
|
||||
} },
|
||||
{ id: "api_curator_boundary", run: async () => {
|
||||
const author = join(ctx.run.root, "author");
|
||||
const catalogAfter = (await git(ctx, ["rev-parse", `HEAD:thoth-workspaces.yaml`], { cwd: author })).stdout.trim();
|
||||
const evidenceAfter = (await git(ctx, ["rev-parse", `HEAD:p11-filesystem/evidence`], { cwd: author })).stdout.trim();
|
||||
assert(catalogAfter === ctx.catalogBlobBefore, "catalog blob changed during bootstrap");
|
||||
assert(evidenceAfter === ctx.evidenceTreeBefore, "evidence tree changed during bootstrap");
|
||||
ctx.apiBoundaryState = await registryState(ctx);
|
||||
return await check("api_curator_boundary", { catalogUnchanged: true, evidenceUnchanged: true, state: ctx.apiBoundaryState }, ["git"]);
|
||||
} },
|
||||
{ id: "curator_descriptor_update", run: async () => {
|
||||
const author = join(ctx.run.root, "author");
|
||||
await git(ctx, ["fetch", "origin", "main"], { cwd: author });
|
||||
await git(ctx, ["reset", "--hard", "origin/main"], { cwd: author });
|
||||
const workspace = structuredClone(ctx.descriptors[0]);
|
||||
workspace.workspace.name = "P1.1 Curated Filesystem";
|
||||
workspace.workspace.description = "Curator updated descriptor and catalog metadata";
|
||||
ctx.curatedWorkspace = workspace;
|
||||
const updatedCatalog = catalog([workspace, ctx.descriptors[1], ctx.descriptors[2]]);
|
||||
await atomicWrite(join(author, "thoth-workspaces.yaml"), `${JSON.stringify(updatedCatalog, null, 2)}\n`, 0o644);
|
||||
await atomicWrite(join(author, "p11-filesystem", "workspace.yaml"), `${(await import("yaml")).stringify(workspace)}`, 0o644);
|
||||
await git(ctx, ["add", "thoth-workspaces.yaml"], { cwd: author });
|
||||
await git(ctx, ["add", "--", "p11-filesystem/workspace.yaml"], { cwd: author });
|
||||
await git(ctx, ["commit", "-m", "Publish workspace p1-filesystem"], { cwd: author });
|
||||
await git(ctx, ["push", "origin", "main"], { cwd: author });
|
||||
ctx.curatorCommit = (await git(ctx, ["rev-parse", "HEAD"], { cwd: author })).stdout.trim();
|
||||
ctx.curatorDescriptorBlob = (await git(ctx, ["rev-parse", `HEAD:p11-filesystem/workspace.yaml`], { cwd: author })).stdout.trim();
|
||||
const pulled = await request(ctx, "pull-after-curator-update", "POST", "/workspace-registry/pull");
|
||||
assert(pulled.status === 200 && HEX40.test(pulled.body.head), "pull after curator update failed");
|
||||
ctx.docsFollowupHead = pulled.body.head;
|
||||
const read = await request(ctx, "read-after-curator-update", "GET", "/workspaces/p11-filesystem");
|
||||
assert(read.status === 200 && read.body.workspace.workspace.name === workspace.workspace.name, "curator update did not activate");
|
||||
assert(read.body.revision.blob === ctx.curatorDescriptorBlob, "api rewrote curator descriptor bytes");
|
||||
return await check("curator_descriptor_update", { curatorCommit: ctx.curatorCommit, activeHead: ctx.docsFollowupHead, descriptorBlob: ctx.curatorDescriptorBlob }, ["git"]);
|
||||
} },
|
||||
{ id: "content_only_revision", run: async () => {
|
||||
const author = join(ctx.run.root, "author");
|
||||
await git(ctx, ["fetch", "origin", "main"], { cwd: author });
|
||||
await git(ctx, ["reset", "--hard", "origin/main"], { cwd: author });
|
||||
await atomicWrite(join(author, "p11-filesystem", "evidence", "guide.md"), "# P1.1 curated Evidence v2\n", 0o644);
|
||||
await git(ctx, ["add", "p11-filesystem/evidence/guide.md"], { cwd: author });
|
||||
await git(ctx, ["commit", "-m", "Update curated Evidence only"], { cwd: author });
|
||||
await git(ctx, ["push", "origin", "main"], { cwd: author });
|
||||
ctx.contentCommit = (await git(ctx, ["rev-parse", "HEAD"], { cwd: author })).stdout.trim();
|
||||
const pulled = await request(ctx, "pull-after-content-update", "POST", "/workspace-registry/pull");
|
||||
assert(pulled.status === 200 && pulled.body.head === ctx.contentCommit, "content pull head mismatch");
|
||||
const read = await request(ctx, "read-after-content-update", "GET", "/workspaces/p11-filesystem");
|
||||
assert(read.body.revision.commit === ctx.contentCommit, "content commit did not activate");
|
||||
assert(read.body.revision.blob === ctx.curatorDescriptorBlob, "descriptor blob changed on content-only update");
|
||||
ctx.currentRead = read.body;
|
||||
return await check("content_only_revision", { commit: ctx.contentCommit, descriptorBlobUnchanged: true }, ["git"]);
|
||||
} },
|
||||
{ id: "docs_only_reconciliation", run: async () => {
|
||||
const repo = join(ctx.run.root, "installation", "registry", "repo");
|
||||
const diff = (await git(ctx, ["show", "--name-only", "--format=", ctx.docsFollowupHead], { cwd: repo })).stdout.trim().split(/\n+/).filter(Boolean);
|
||||
assert(diff.length > 0 && diff.every((path) => path.startsWith("workspace-docs/")), "docs follow-up touched non-doc paths");
|
||||
const finalDescriptor = (await git(ctx, ["rev-parse", `${ctx.docsFollowupHead}:p11-filesystem/workspace.yaml`], { cwd: repo })).stdout.trim();
|
||||
assert(finalDescriptor === ctx.curatorDescriptorBlob, "docs follow-up rewrote descriptor");
|
||||
return await check("docs_only_reconciliation", { head: ctx.docsFollowupHead, files: diff, descriptorBlobPreserved: true }, ["git"]);
|
||||
} },
|
||||
{ id: "same_revision_git_objects", run: async () => {
|
||||
const repo = join(ctx.run.root, "installation", "registry", "repo");
|
||||
const revision = ctx.currentRead.revision;
|
||||
const manifestPath = join(dirname(revision.snapshotPath), "snapshot.json");
|
||||
const manifest = JSON.parse(await readFile(manifestPath, "utf8"));
|
||||
const catalogBlob = (await git(ctx, ["rev-parse", `${revision.commit}:thoth-workspaces.yaml`], { cwd: repo })).stdout.trim();
|
||||
const descriptorBlob = (await git(ctx, ["rev-parse", `${revision.commit}:p11-filesystem/workspace.yaml`], { cwd: repo })).stdout.trim();
|
||||
const evidenceTree = (await git(ctx, ["rev-parse", `${revision.commit}:p11-filesystem/evidence`], { cwd: repo })).stdout.trim();
|
||||
assert(manifest.head === revision.commit, "snapshot manifest head mismatch");
|
||||
assert(descriptorBlob === revision.blob, "descriptor blob mismatch");
|
||||
ctx.snapshotManifest = manifest;
|
||||
return {
|
||||
commands: ["git"],
|
||||
artifacts: [
|
||||
await evidence(ctx.run, "logs/same-revision-git-objects.json", { commit: revision.commit, catalogBlob, descriptorBlob, evidenceTree, snapshotHead: manifest.head }),
|
||||
await fileArtifact(ctx.run.root, relative(ctx.run.root, revision.snapshotPath)),
|
||||
await fileArtifact(ctx.run.root, relative(ctx.run.root, manifestPath)),
|
||||
],
|
||||
};
|
||||
} },
|
||||
{ id: "snapshot_and_export", run: async () => {
|
||||
ctx.exportManifests = {};
|
||||
const artifacts = [];
|
||||
for (const workspace of ctx.descriptors) {
|
||||
const id = workspace.workspace.id;
|
||||
const exported = await request(ctx, `export-${id}`, "GET", `/workspaces/${id}/export`, undefined, true);
|
||||
assert(exported.status === 200, `export failed ${id}`);
|
||||
ctx.exportManifests[id] = await extractZip(ctx, id, exported.bytes);
|
||||
artifacts.push(await fileArtifact(ctx.run.root, `exports/raw/export-${id}.zip`));
|
||||
for (const name of ZIP_FILES) artifacts.push(await fileArtifact(ctx.run.root, `exports/extracted/${id}/${name}`));
|
||||
}
|
||||
return { commands: [], artifacts: [await evidence(ctx.run, "logs/snapshot-and-export.json", { exported: Object.keys(ctx.exportManifests), files: ZIP_FILES }), ...artifacts] };
|
||||
} },
|
||||
{ id: "runtime_render_determinism", run: async () => {
|
||||
const YAML = await import("yaml");
|
||||
ctx.configChecks = [];
|
||||
const artifacts = [];
|
||||
for (const workspace of ctx.descriptors) {
|
||||
const revision = (await request(ctx, `read-render-${workspace.workspace.id}`, "GET", `/workspaces/${workspace.workspace.id}`)).body.revision;
|
||||
const renders = [];
|
||||
for (let n = 1; n <= 2; n += 1) {
|
||||
const lease = ctx.thtRunner.acquireWorkspaceRuntime(revision.snapshotPath);
|
||||
try {
|
||||
const bytes = await readFile(lease.path);
|
||||
renders.push(bytes);
|
||||
await atomicWrite(join(ctx.run.root, "rendered", `${workspace.workspace.id}-${n}.yaml`), bytes);
|
||||
const checked = await tht(ctx, ["config", "check", "-c", lease.path], { cwd: ctx.env.THT_HARNESS_DIR, timeoutMs: 30_000 });
|
||||
ctx.configChecks.push({ id: workspace.workspace.id, observation: n, code: checked.code });
|
||||
} finally {
|
||||
lease.release();
|
||||
}
|
||||
artifacts.push(await fileArtifact(ctx.run.root, `rendered/${workspace.workspace.id}-${n}.yaml`));
|
||||
}
|
||||
assert(renders[0].equals(renders[1]), `render was nondeterministic ${workspace.workspace.id}`);
|
||||
const rendered = YAML.parse(renders[0].toString("utf8"));
|
||||
assert(rendered.runtime_identity.workspace_revision === revision.commit, `runtime identity mismatch ${workspace.workspace.id}`);
|
||||
}
|
||||
return { commands: ["tht"], artifacts: [await evidence(ctx.run, "logs/runtime-render-determinism.json", { deterministic: true, checks: ctx.configChecks }), ...artifacts] };
|
||||
} },
|
||||
{ id: "tht_config_check", run: async () => {
|
||||
assert(ctx.configChecks.length === ctx.descriptors.length * 2 && ctx.configChecks.every((item) => item.code === 0), "tht config checks failed");
|
||||
return await check("tht-config-check", ctx.configChecks, ["tht"]);
|
||||
} },
|
||||
{ id: "negative_catalog_layout_cases", run: async () => {
|
||||
const baseline = await registryState(ctx);
|
||||
const author = join(ctx.run.root, "author");
|
||||
const current = (await request(ctx, "current-list-before-negatives", "GET", "/workspaces")).body;
|
||||
const secondCreate = await request(ctx, "second-create", "POST", "/workspaces/publish", { action: "create", workspace: ctx.descriptors[0], baseCommit: baseline.head });
|
||||
safeErrorEnvelope(secondCreate, "workspace_curator_owned", 409);
|
||||
const update = await request(ctx, "legacy-update", "POST", "/workspaces/publish", { action: "update", workspace: ctx.descriptors[0], baseCommit: baseline.head, baseBlob: ctx.curatorDescriptorBlob });
|
||||
safeErrorEnvelope(update, "workspace_curator_owned", 409);
|
||||
const deletion = await request(ctx, "legacy-delete", "POST", "/workspaces/publish", { action: "delete", id: "p11-filesystem", baseCommit: baseline.head, baseBlob: ctx.curatorDescriptorBlob });
|
||||
safeErrorEnvelope(deletion, "workspace_curator_owned", 409);
|
||||
const unknown = structuredClone(ctx.descriptors[0]);
|
||||
unknown.workspace.id = "p11-unknown";
|
||||
const unknownPublish = await request(ctx, "unknown-catalog-id", "POST", "/workspaces/publish", { action: "create", workspace: unknown, baseCommit: baseline.head });
|
||||
safeErrorEnvelope(unknownPublish, "workspace_invalid", 400);
|
||||
const mismatch = structuredClone(ctx.descriptors[0]);
|
||||
mismatch.workspace.id = "p11-pending";
|
||||
mismatch.workspace.name = "Mismatched pending name";
|
||||
mismatch.semantic_index.vector_store.collection = "p11-pending";
|
||||
const mismatchPublish = await request(ctx, "catalog-metadata-mismatch", "POST", "/workspaces/publish", { action: "create", workspace: mismatch, baseCommit: baseline.head });
|
||||
safeErrorEnvelope(mismatchPublish, "workspace_invalid", 400);
|
||||
const after = await registryState(ctx);
|
||||
assertByteIdentical(after, baseline, "registry state after curator-owned refusals");
|
||||
assert(JSON.stringify((await request(ctx, "current-list-after-negatives", "GET", "/workspaces")).body) === JSON.stringify(current), "workspace listing mutated after negative cases");
|
||||
await git(ctx, ["fetch", "origin", "main"], { cwd: author });
|
||||
await git(ctx, ["reset", "--hard", "origin/main"], { cwd: author });
|
||||
await mkdir(join(author, "workspaces"), { recursive: true });
|
||||
await atomicWrite(join(author, "workspaces", "legacy.yaml"), "workspace: bad\n", 0o644);
|
||||
await git(ctx, ["add", "--", "workspaces/legacy.yaml"], { cwd: author });
|
||||
await git(ctx, ["commit", "-m", "Invalid contextual Evidence state"], { cwd: author });
|
||||
await git(ctx, ["push", "origin", "HEAD:main"], { cwd: author });
|
||||
const rejectedPull = await request(ctx, "invalid-layout-pull", "POST", "/workspace-registry/pull");
|
||||
safeErrorEnvelope(rejectedPull, "workspace_invalid", 400);
|
||||
const afterInvalidPull = await registryState(ctx);
|
||||
assertByteIdentical(afterInvalidPull, baseline, "registry state after invalid pull");
|
||||
return await check("negative_catalog_layout_cases", { secondCreate: true, update: true, delete: true, unknownCatalogId: true, metadataMismatch: true, oldLayoutRejected: true }, ["git"]);
|
||||
} },
|
||||
{ id: "negative_schema_context_cases", run: async () => {
|
||||
const base = structuredClone(ctx.descriptors[0]);
|
||||
const cases = [
|
||||
["invalid-uri", (workspace) => { workspace.evidence.source.uri = "/etc/passwd"; }, "evidence.source.uri"],
|
||||
["invalid-secret-field", (workspace) => { workspace.evidence.source.password = ctx.secretValues.rejected; }, "evidence.source.password"],
|
||||
["missing-evidence-tree", (workspace) => { workspace.workspace.id = "p11-pending"; workspace.workspace.name = "P1.1 pending"; workspace.workspace.description = "Catalog-only slot awaiting bootstrap"; workspace.semantic_index.vector_store.collection = "p11-pending"; workspace.evidence.source.uri = "p11-pending/evidence"; }, "evidence.source.uri"],
|
||||
];
|
||||
const outcomes = [];
|
||||
for (const [id, mutate, field] of cases) {
|
||||
const workspace = structuredClone(base);
|
||||
mutate(workspace);
|
||||
const endpoint = id === "missing-evidence-tree" ? "/workspaces/publish" : "/workspaces/validate";
|
||||
const payload = id === "missing-evidence-tree" ? { action: "create", workspace, baseCommit: ctx.publishHead } : { workspace };
|
||||
const response = await request(ctx, `negative-schema-${id}`, "POST", endpoint, payload, false, { case: id, expectedInputField: field });
|
||||
safeErrorEnvelope(response, "workspace_invalid", 400);
|
||||
outcomes.push({ case: id, status: response.status, field });
|
||||
}
|
||||
return await check("negative_schema_context_cases", outcomes);
|
||||
} },
|
||||
{ id: "no_p2_scope_artifacts", run: async () => {
|
||||
const forbidden = ["artifacts/evidence", "materialized", "qdrant", "embedding", "ACTIVE", "retention"];
|
||||
const present = forbidden.filter((path) => existsSync(join(ctx.run.root, path)));
|
||||
assert(present.length === 0, "p2 scope artifacts present");
|
||||
return await check("no_p2_scope_artifacts", { absent: forbidden });
|
||||
} },
|
||||
{ id: "secret_scan", run: async () => {
|
||||
const findings = await scanSecrets({ runRoot: ctx.run.root, forbiddenValues: ctx.forbiddenValues, expectedGitRepositories: ["remote.git", "author"] });
|
||||
assert(findings.length === 0, "secret scan found leaked secret material");
|
||||
return await check("secret_scan", { findings: 0 });
|
||||
} },
|
||||
{ id: "cleanup_confinement", run: async () => {
|
||||
const parent = canonicalIntegrationBase(ctx.repositoryRoot);
|
||||
const siblings = (await readdir(parent)).filter((name) => name !== ctx.run.runId);
|
||||
return await check("cleanup_confinement", { listenerState: ctx.run.listeners[0].state, siblingCount: siblings.length });
|
||||
} },
|
||||
];
|
||||
}
|
||||
|
||||
async function setupContext({ repositoryRoot = defaultRepositoryRoot, env = process.env } = {}) {
|
||||
const run = await createOwnedRun({ repositoryRoot });
|
||||
const provenance = await collectRepositoryProvenance({ repositoryRoot });
|
||||
const executables = resolveExecutables(repositoryRoot);
|
||||
const harnessDir = realpathSync(join(repositoryRoot, "harness"));
|
||||
const ownedHome = join(run.root, "installation", "runtime", "acceptance-home");
|
||||
const ownedTmp = join(run.root, "installation", "runtime", "tmp");
|
||||
await mkdir(ownedHome, { recursive: true, mode: 0o700 });
|
||||
await mkdir(ownedTmp, { recursive: true, mode: 0o700 });
|
||||
const executablePath = [...new Set([dirname(executables.gitPath), dirname(executables.pythonPath), dirname(executables.thtPath)])].join(":");
|
||||
const fixtureEnv = {
|
||||
PATH: executablePath,
|
||||
HOME: ownedHome,
|
||||
TMPDIR: ownedTmp,
|
||||
HOST: "127.0.0.1",
|
||||
PORT: "0",
|
||||
AUTH_MODE: "none",
|
||||
THT_BIN: executables.thtPath,
|
||||
THT_HARNESS_DIR: harnessDir,
|
||||
THT_DATA_ROOT: join(run.root, "installation", "data"),
|
||||
SETTINGS_FILE: join(run.root, "installation", "data", "settings.json"),
|
||||
MAINTENANCE_STATE_FILE: join(run.root, "installation", "data", "maintenance.json"),
|
||||
THT_WORKSPACE_REGISTRY_ROOT: join(run.root, "installation", "registry"),
|
||||
THT_WORKSPACE_GIT_REMOTE: join(run.root, "remote.git"),
|
||||
THT_WORKSPACE_GIT_BRANCH: "main",
|
||||
THT_WORKSPACE_GIT_AUTHOR_NAME: "P1 API Publisher",
|
||||
THT_WORKSPACE_GIT_AUTHOR_EMAIL: "p1-api@example.invalid",
|
||||
THT_WORKSPACE_INSTALLATION_ID: "p11-acceptance",
|
||||
THT_WORKSPACE_SECRET_ROOTS: join(run.root, "fixture-secrets"),
|
||||
THT_HOME: join(run.root, "installation", "runtime", "tht-home"),
|
||||
PYTHONDONTWRITEBYTECODE: "1",
|
||||
PYTHONNOUSERSITE: "1",
|
||||
};
|
||||
const ctx = {
|
||||
run,
|
||||
repositoryRoot: canonicalRoot(repositoryRoot),
|
||||
provenance,
|
||||
executables,
|
||||
descriptors: descriptors(),
|
||||
env: buildSafeEnvironment({ ambient: env, fixture: fixtureEnv }),
|
||||
forbiddenValues: [],
|
||||
};
|
||||
await createTopology(run);
|
||||
await setupSecrets(ctx);
|
||||
return ctx;
|
||||
}
|
||||
|
||||
export async function runIntegration({ repositoryRoot = defaultRepositoryRoot, keep = false, env = process.env, announce } = {}) {
|
||||
const ctx = await setupContext({ repositoryRoot, env });
|
||||
const priorEnv = {};
|
||||
for (const [key, value] of Object.entries(ctx.env)) {
|
||||
priorEnv[key] = process.env[key];
|
||||
process.env[key] = value;
|
||||
}
|
||||
let success = false;
|
||||
try {
|
||||
const checks = await productionChecks(ctx);
|
||||
const results = await executeChecks({ checks });
|
||||
const report = {
|
||||
schemaVersion: 1,
|
||||
runId: ctx.run.runId,
|
||||
startedAt: ctx.run.startedAt,
|
||||
finishedAt: nowIso(),
|
||||
command: "p11-acceptance integration --keep",
|
||||
overall: deriveOverall(results),
|
||||
checks: results,
|
||||
};
|
||||
await writeReportFiles({ run: ctx.run, report });
|
||||
success = report.overall === "PASS";
|
||||
if (announce) await announce({ report, runRoot: ctx.run.root });
|
||||
return { exitCode: success ? 0 : 1, runRoot: ctx.run.root, retained: !(await finalizeOwnedRun({ run: ctx.run, success, keep })) };
|
||||
} finally {
|
||||
await stopBackend(ctx).catch(() => {});
|
||||
for (const [key, value] of Object.entries(ctx.env)) {
|
||||
if (priorEnv[key] === undefined) delete process.env[key];
|
||||
else process.env[key] = priorEnv[key];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export async function main(argv = process.argv.slice(2), env = process.env) {
|
||||
if (argv.length < 1 || argv[0] !== "integration" || argv.length > 2 || (argv[1] && argv[1] !== "--keep")) {
|
||||
throw new Error("usage: p11-acceptance.mjs integration [--keep]");
|
||||
}
|
||||
const result = await runIntegration({ keep: argv.includes("--keep"), env });
|
||||
return result.exitCode;
|
||||
}
|
||||
|
||||
if (process.argv[1] && realpathSync(process.argv[1]) === modulePath) {
|
||||
try {
|
||||
const code = await main();
|
||||
process.exitCode = code;
|
||||
} catch (error) {
|
||||
console.error(error instanceof Error ? error.message : String(error));
|
||||
process.exitCode = 1;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,113 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { dirname, join } from "node:path";
|
||||
import test from "node:test";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
import {
|
||||
CHECK_IDS,
|
||||
canonicalIntegrationBase,
|
||||
cleanupOwnedRun,
|
||||
createOwnedRun,
|
||||
readAndValidateOwnership,
|
||||
validateReport,
|
||||
validateRunRoot,
|
||||
} from "./p11-acceptance.mjs";
|
||||
|
||||
const roots = [];
|
||||
async function fakeRepository() {
|
||||
const root = await mkdtemp(join(tmpdir(), "p11-acceptance-repo-"));
|
||||
roots.push(root);
|
||||
await mkdir(join(root, ".artifacts", "p11-integration"), { recursive: true });
|
||||
await mkdir(join(root, ".artifacts", "p1-integration"), { recursive: true });
|
||||
await mkdir(join(root, ".artifacts", "manual-acceptance", "p11"), { recursive: true });
|
||||
return root;
|
||||
}
|
||||
|
||||
test.afterEach(async () => {
|
||||
await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true })));
|
||||
});
|
||||
|
||||
test("run roots are only canonical direct p11 integration children", async () => {
|
||||
const repositoryRoot = await fakeRepository();
|
||||
const base = canonicalIntegrationBase(repositoryRoot);
|
||||
const id = `p11-${"a".repeat(32)}`;
|
||||
assert.equal(validateRunRoot(repositoryRoot, join(base, id), id), join(base, id));
|
||||
for (const candidate of [
|
||||
base,
|
||||
join(repositoryRoot, ".artifacts", "manual-acceptance", "p11"),
|
||||
join(repositoryRoot, ".artifacts", "p1-integration", id),
|
||||
join(base, id, "nested"),
|
||||
join(base, "foreign"),
|
||||
]) {
|
||||
assert.throws(() => validateRunRoot(repositoryRoot, candidate, id));
|
||||
}
|
||||
assert.throws(() => validateRunRoot(repositoryRoot, join(base, `p11-${"A".repeat(32)}`), `p11-${"A".repeat(32)}`));
|
||||
});
|
||||
|
||||
test("cleanup refuses p1, manual, sibling, and wrong-nonce roots", async () => {
|
||||
const repositoryRoot = await fakeRepository();
|
||||
const run = await createOwnedRun({ repositoryRoot });
|
||||
await readAndValidateOwnership({ repositoryRoot, runRoot: run.root, expectedNonce: run.nonce });
|
||||
for (const bad of [
|
||||
join(repositoryRoot, ".artifacts", "p1-integration", `p1-${"b".repeat(32)}`),
|
||||
join(repositoryRoot, ".artifacts", "manual-acceptance", "p11"),
|
||||
join(canonicalIntegrationBase(repositoryRoot), `p11-${"c".repeat(32)}`),
|
||||
]) {
|
||||
await assert.rejects(cleanupOwnedRun({ repositoryRoot, runRoot: bad, expectedNonce: run.nonce }));
|
||||
}
|
||||
await assert.rejects(cleanupOwnedRun({ repositoryRoot, runRoot: run.root, expectedNonce: "0".repeat(64) }));
|
||||
});
|
||||
|
||||
test("cleanup removes exactly one owned p11 root", async () => {
|
||||
const repositoryRoot = await fakeRepository();
|
||||
const run = await createOwnedRun({ repositoryRoot });
|
||||
const sibling = join(canonicalIntegrationBase(repositoryRoot), `p11-${"d".repeat(32)}`);
|
||||
await mkdir(sibling);
|
||||
await writeFile(join(sibling, "sentinel"), "foreign");
|
||||
await cleanupOwnedRun({ repositoryRoot, runRoot: run.root, expectedNonce: run.nonce });
|
||||
await assert.rejects(readFile(join(run.root, "ownership.json")));
|
||||
assert.equal(await readFile(join(sibling, "sentinel"), "utf8"), "foreign");
|
||||
});
|
||||
|
||||
function resultFor(id) {
|
||||
return {
|
||||
id,
|
||||
status: "PASS",
|
||||
startedAt: "2026-08-11T00:00:00.000Z",
|
||||
finishedAt: "2026-08-11T00:00:01.000Z",
|
||||
commands: ["git"],
|
||||
artifacts: [{ path: `logs/${id}.json`, sha256: "a".repeat(64) }],
|
||||
};
|
||||
}
|
||||
|
||||
test("report validation requires exact p11 identity, check order, and unique artifacts", () => {
|
||||
const report = {
|
||||
schemaVersion: 1,
|
||||
runId: `p11-${"e".repeat(32)}`,
|
||||
startedAt: "2026-08-11T00:00:00.000Z",
|
||||
finishedAt: "2026-08-11T00:00:10.000Z",
|
||||
command: "p11-acceptance integration --keep",
|
||||
overall: "PASS",
|
||||
checks: CHECK_IDS.map(resultFor),
|
||||
};
|
||||
assert.doesNotThrow(() => validateReport(report));
|
||||
const invalid = structuredClone(report);
|
||||
invalid.runId = `p1-${"e".repeat(32)}`;
|
||||
assert.throws(() => validateReport(invalid));
|
||||
const duplicate = structuredClone(report);
|
||||
duplicate.checks[1].artifacts[0].path = duplicate.checks[0].artifacts[0].path;
|
||||
assert.throws(() => validateReport(duplicate), /duplicated/);
|
||||
const reordered = structuredClone(report);
|
||||
reordered.checks.reverse();
|
||||
reordered.overall = "FAIL";
|
||||
assert.throws(() => validateReport(reordered));
|
||||
});
|
||||
|
||||
test("public wrapper uses a strict empty environment", async () => {
|
||||
const wrapper = await readFile(join(dirname(fileURLToPath(import.meta.url)), "..", "..", "scripts", "p11-acceptance.sh"), "utf8");
|
||||
assert.match(wrapper, /safe_env=\(\/usr\/bin\/env -i/);
|
||||
assert.doesNotMatch(wrapper, /LANG|LC_ALL|TZ/);
|
||||
assert.doesNotMatch(wrapper, /P11_ACCEPTANCE_FAIL_AT/);
|
||||
});
|
||||
@@ -0,0 +1,399 @@
|
||||
#!/usr/bin/env node
|
||||
import { spawn } from "node:child_process";
|
||||
import { createHash, randomBytes } from "node:crypto";
|
||||
import { closeSync, constants as fsConstants, fsyncSync, lstatSync, openSync, realpathSync } from "node:fs";
|
||||
import { access, lstat, mkdir, open, readFile, readdir, rename, rm, writeFile } from "node:fs/promises";
|
||||
import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { promisify } from "node:util";
|
||||
import { execFile } from "node:child_process";
|
||||
import http from "node:http";
|
||||
|
||||
import { buildSafeEnvironment } from "./p1-acceptance.mjs";
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
const modulePath = fileURLToPath(import.meta.url);
|
||||
const defaultRepositoryRoot = realpathSync(resolve(dirname(modulePath), "../.."));
|
||||
const HOST = "127.0.0.1";
|
||||
const BACKEND_PORT = 8791;
|
||||
const FRONTEND_PORT = 8792;
|
||||
const HEX64 = /^[0-9a-f]{64}$/;
|
||||
const OWNERSHIP_DIGEST = "ownership.sha256";
|
||||
|
||||
function resolveSystemExecutable(name) {
|
||||
for (const candidate of [`/usr/bin/${name}`, `/bin/${name}`, `/opt/homebrew/bin/${name}`, `/usr/local/bin/${name}`]) {
|
||||
try {
|
||||
const resolved = realpathSync(candidate);
|
||||
if (lstatSync(resolved).isFile()) return resolved;
|
||||
} catch {}
|
||||
}
|
||||
throw new Error(`required executable not found: ${name}`);
|
||||
}
|
||||
function resolveExecutables(repositoryRoot) {
|
||||
const repo = realpathSync(repositoryRoot);
|
||||
const thtPath = join(repo, "harness", ".venv", "bin", "tht");
|
||||
if (!lstatSync(thtPath).isFile()) throw new Error("required executable not found: tht");
|
||||
return { gitPath: resolveSystemExecutable("git"), pythonPath: resolveSystemExecutable("python3"), thtPath: realpathSync(thtPath) };
|
||||
}
|
||||
|
||||
function nowIso() { return new Date().toISOString(); }
|
||||
function fixedManualRoot(repositoryRoot = defaultRepositoryRoot) { return join(realpathSync(repositoryRoot), ".artifacts", "manual-acceptance", "p11"); }
|
||||
function below(parent, child) { const rel = relative(parent, child); return rel !== "" && !rel.startsWith(`..${sep}`) && rel !== ".." && !isAbsolute(rel); }
|
||||
function noSymlinkExisting(repo, target) {
|
||||
const rel = relative(repo, target);
|
||||
if (rel.startsWith("..") || isAbsolute(rel)) throw new Error("root leaves repository");
|
||||
let cursor = repo;
|
||||
for (const part of rel.split(sep).filter(Boolean)) {
|
||||
cursor = join(cursor, part);
|
||||
if (!lstatSync(cursor, { throwIfNoEntry: false })) break;
|
||||
if (lstatSync(cursor).isSymbolicLink()) throw new Error("owned path contains a symlink");
|
||||
}
|
||||
}
|
||||
async function atomicWrite(path, bytes, mode = 0o600) {
|
||||
await mkdir(dirname(path), { recursive: true });
|
||||
const staging = join(dirname(path), `.${basename(path)}.${randomBytes(12).toString("hex")}.tmp`);
|
||||
let handle;
|
||||
try {
|
||||
handle = await open(staging, "wx", mode);
|
||||
await handle.writeFile(bytes);
|
||||
await handle.sync();
|
||||
await handle.close();
|
||||
handle = undefined;
|
||||
await rename(staging, path);
|
||||
const directory = openSync(dirname(path), fsConstants.O_RDONLY);
|
||||
try { fsyncSync(directory); } finally { closeSync(directory); }
|
||||
} catch (error) {
|
||||
if (handle) await handle.close().catch(() => {});
|
||||
await rm(staging, { force: true }).catch(() => {});
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
function ownershipDigest(bytes) { return createHash("sha256").update(bytes).digest("hex"); }
|
||||
async function writeManualOwnership(root, value) {
|
||||
const body = `${JSON.stringify(value, null, 2)}\n`;
|
||||
await atomicWrite(join(root, "ownership.json"), body);
|
||||
await atomicWrite(join(root, OWNERSHIP_DIGEST), `${ownershipDigest(body)}\n`);
|
||||
}
|
||||
async function git(executable, argv, options = {}) {
|
||||
const result = await execFileAsync(executable, argv, { cwd: options.cwd, env: options.env, timeout: options.timeoutMs ?? 30_000, maxBuffer: 8 * 1024 * 1024, encoding: "utf8" });
|
||||
return { stdout: result.stdout ?? "", stderr: result.stderr ?? "" };
|
||||
}
|
||||
function namespace(id) { return id.toUpperCase().replaceAll("-", "_"); }
|
||||
function baseWorkspace(id, evidenceSource) {
|
||||
return {
|
||||
workspace: { schema_version: 4, id, name: `P1.1 ${id}`, description: `Catalog entry for ${id}`, language: "en" },
|
||||
dwh: { engine: "postgres", database: "postgres", schema: "public", supported_transports: ["postgres_direct"] },
|
||||
evidence: { source: evidenceSource, policy: { max_chunk_chars: 4000, retain_published_generations: 3 } },
|
||||
};
|
||||
}
|
||||
function descriptors() {
|
||||
return [
|
||||
baseWorkspace("p11-filesystem", { type: "filesystem", uri: "p11-filesystem/evidence", patterns: ["**/*.md"], max_bytes: 10485760 }),
|
||||
baseWorkspace("p11-http", { type: "http", uris: ["https://evidence.example.test/guide.md"], authentication: "signed_urls_file", connect_timeout_ms: 1250, read_timeout_ms: 30001, max_bytes: 12345, max_redirects: 2, allow_private_hosts: false, max_cache_bytes: 67890 }),
|
||||
baseWorkspace("p11-s3", { type: "s3", uri: "s3://p11-evidence/published/", endpoint_url: "https://s3.example.test/", region: "eu-west-1", credentials: "static_files", trusted_endpoint: true, allow_private_endpoint: false, allow_insecure_endpoint: false, max_bytes: 12345, max_objects: 33, max_pages: 4, page_size: 5 }),
|
||||
];
|
||||
}
|
||||
function catalog(entries) {
|
||||
return { schema_version: 1, workspaces: entries.map(({ workspace }) => ({ id: workspace.id, name: workspace.name, description: workspace.description })) };
|
||||
}
|
||||
function quote(value) { return `'${String(value).replaceAll("'", `'"'"'`)}'`; }
|
||||
function requestFixtures(items) {
|
||||
const fixtures = { "status.json": { method: "GET", path: "/workspace-registry/status" }, "pull.json": { method: "POST", path: "/workspace-registry/pull" } };
|
||||
for (const workspace of items) {
|
||||
const id = workspace.workspace.id;
|
||||
fixtures[`validate-${id}.json`] = { workspace };
|
||||
fixtures[`publish-${id}.json`] = { action: "create", workspace };
|
||||
fixtures[`read-${id}.json`] = { method: "GET", path: `/workspaces/${id}` };
|
||||
fixtures[`export-${id}.json`] = { method: "GET", path: `/workspaces/${id}/export` };
|
||||
}
|
||||
fixtures["negative-invalid-uri.json"] = { workspace: { ...items[0], evidence: { ...items[0].evidence, source: { ...items[0].evidence.source, uri: "/etc/passwd" } } } };
|
||||
fixtures["negative-secret-field.json"] = { workspace: { ...items[2], evidence: { ...items[2].evidence, source: { ...items[2].evidence.source, access_key: "CANARY-MUST-BE-REJECTED" } } } };
|
||||
return fixtures;
|
||||
}
|
||||
function curlGet(url, output) { return `#!/usr/bin/env bash\nset -euo pipefail\ncurl --fail-with-body --silent --show-error --output ${quote(output)} --write-out 'HTTP %{http_code}\\n' ${quote(url)}\n`; }
|
||||
function curlPost(url, output, body) { return `#!/usr/bin/env bash\nset -euo pipefail\ncurl --fail-with-body --silent --show-error --request POST --header 'content-type: application/json' --data-binary @${quote(body)} --output ${quote(output)} --write-out 'HTTP %{http_code}\\n' ${quote(url)}\n`; }
|
||||
function curlPostEmpty(url, output) { return `#!/usr/bin/env bash\nset -euo pipefail\ncurl --fail-with-body --silent --show-error --request POST --output ${quote(output)} --write-out 'HTTP %{http_code}\\n' ${quote(url)}\n`; }
|
||||
function publishCurl(root, id, previousResponse) {
|
||||
const descriptor = join(root, "requests", `publish-${id}.json`);
|
||||
const response = join(root, "responses", `publish-${id}.json`);
|
||||
return `#!/usr/bin/env bash\nset -euo pipefail\nbase_commit=$(node -e 'const fs=require("node:fs");const value=JSON.parse(fs.readFileSync(process.argv[1],"utf8"));console.log(value.head ?? value.revision?.commit ?? "");' ${quote(previousResponse)})\nnode -e 'const fs=require("node:fs");const body=JSON.parse(fs.readFileSync(process.argv[1],"utf8"));body.baseCommit=process.argv[2];fs.writeFileSync(process.argv[1],JSON.stringify(body,null,2)+"\\n");' ${quote(descriptor)} "$base_commit"\ncurl --fail-with-body --silent --show-error --request POST --header 'content-type: application/json' --data-binary @${quote(descriptor)} --output ${quote(response)} --write-out 'HTTP %{http_code}\\n' 'http://${HOST}:${BACKEND_PORT}/workspaces/publish'\n`; }
|
||||
function renderCommand(repo, root, observation) {
|
||||
const readResponse = join(root, "responses", "read-p11-filesystem.json");
|
||||
const output = join(root, "rendered", `runtime-${observation}.yaml`);
|
||||
return `#!/usr/bin/env bash\nset -euo pipefail\nread_snapshot=$(node -e 'const fs=require("node:fs");const read=JSON.parse(fs.readFileSync(process.argv[1],"utf8"));const path=read.revision.snapshotPath;const manifest=JSON.parse(fs.readFileSync(require("node:path").join(require("node:path").dirname(path),"snapshot.json"),"utf8"));const name=require("node:path").basename(path);console.log(JSON.stringify({snapshot:path,digest:manifest.files[name]}));' ${quote(readResponse)})\nsnapshot=$(node -e 'const value=JSON.parse(process.argv[1]);console.log(value.snapshot)' "$read_snapshot")\ndigest=$(node -e 'const value=JSON.parse(process.argv[1]);console.log(value.digest)' "$read_snapshot")\nnode ${quote(join(repo, "backend", "scripts", "p11-render-snapshot.mjs"))} --ownership ${quote(join(root, "ownership.json"))} --snapshot "$snapshot" --output ${quote(output)} --snapshot-sha256 "$digest"\n`;
|
||||
}
|
||||
function guide(root) {
|
||||
return `# P1.1 manual acceptance guide
|
||||
|
||||
1. Inspect ${join(root, "ownership.json")}, ${join(root, "author", "thoth-workspaces.yaml")}, nested workspace directories, evidence tree, and fixture secret paths without printing secret bytes.
|
||||
2. Run ./scripts/p11-manual-acceptance.sh serve and confirm only ${HOST}:${BACKEND_PORT} and ${HOST}:${FRONTEND_PORT} are listening for this lab.
|
||||
3. Run commands/http-01-status.sh and inspect responses/status.json plus GET /workspaces for configuration_required slots.
|
||||
4. Run the validate and publish scripts once per slot in numeric order.
|
||||
5. Inspect Git object IDs for thoth-workspaces.yaml, <id>/workspace.yaml, <id>/evidence, and workspace-docs/<id>.
|
||||
6. Retry create/update/delete and verify refusal plus unchanged object IDs.
|
||||
7. In ${join(root, "author")}, edit p11-filesystem/workspace.yaml and thoth-workspaces.yaml together, commit, push, then run commands/http-08-pull.sh and verify the API activated curator bytes without rewriting the descriptor.
|
||||
8. Make an evidence-only commit under p11-filesystem/evidence, push, pull, and inspect the new revision commit with unchanged descriptor blob.
|
||||
9. In the UI at http://${HOST}:${FRONTEND_PORT}, confirm ready workspaces are read-only and bootstrap-only slots are editable before creation.
|
||||
10. Export/import only under bootstrap rules.
|
||||
11. Run commands/render-1.sh and commands/render-2.sh, diff rendered/runtime-1.yaml rendered/runtime-2.yaml, then run tht config check -c on both outputs.
|
||||
12. Run the negative validate scripts and a bounded secret scan outside fixture-secrets.
|
||||
13. Run ./scripts/p11-manual-acceptance.sh stop, verify cleanup of both listeners, write VERDICT.md yourself, and run cleanup only when evidence is no longer needed.
|
||||
`;
|
||||
}
|
||||
function ownershipValue(root, repositoryRoot, nonce, extras = {}) {
|
||||
return {
|
||||
schemaVersion: 1,
|
||||
kind: "p11-manual-acceptance",
|
||||
nonce,
|
||||
repositoryRoot,
|
||||
root,
|
||||
createdAt: nowIso(),
|
||||
status: "PENDING",
|
||||
listeners: {
|
||||
backend: { host: HOST, port: BACKEND_PORT },
|
||||
frontend: { host: HOST, port: FRONTEND_PORT },
|
||||
},
|
||||
resources: [root, join(root, "remote.git"), join(root, "author"), join(root, "fixture-secrets")],
|
||||
...extras,
|
||||
};
|
||||
}
|
||||
export async function readManualOwnership({ repositoryRoot = defaultRepositoryRoot } = {}) {
|
||||
const repo = realpathSync(repositoryRoot);
|
||||
const root = fixedManualRoot(repo);
|
||||
noSymlinkExisting(repo, root);
|
||||
const rootEntry = await lstat(root);
|
||||
const ownershipPath = join(root, "ownership.json");
|
||||
const digestPath = join(root, OWNERSHIP_DIGEST);
|
||||
const ownershipEntry = await lstat(ownershipPath);
|
||||
const digestEntry = await lstat(digestPath);
|
||||
if (!rootEntry.isDirectory() || rootEntry.isSymbolicLink() || !ownershipEntry.isFile() || ownershipEntry.isSymbolicLink() || !digestEntry.isFile() || digestEntry.isSymbolicLink()) throw new Error("manual ownership is unsafe");
|
||||
const ownershipBytes = await readFile(ownershipPath, "utf8");
|
||||
const recordedDigest = (await readFile(digestPath, "utf8")).trim();
|
||||
if (!HEX64.test(recordedDigest) || recordedDigest !== ownershipDigest(ownershipBytes)) throw new Error("manual ownership digest mismatch");
|
||||
const value = JSON.parse(ownershipBytes);
|
||||
if (value?.schemaVersion !== 1 || value.kind !== "p11-manual-acceptance" || !HEX64.test(value.nonce ?? "") || value.repositoryRoot !== repo || value.root !== root) {
|
||||
throw new Error("manual ownership identity mismatch");
|
||||
}
|
||||
return value;
|
||||
}
|
||||
async function ensureRootAbsent(root) {
|
||||
try { await lstat(root); throw new Error("manual acceptance root already exists"); } catch (error) { if (error.code !== "ENOENT") throw error; }
|
||||
}
|
||||
async function waitForHttp(url, timeoutMs = 15_000) {
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
while (Date.now() < deadline) {
|
||||
try {
|
||||
await new Promise((resolvePromise, reject) => {
|
||||
const request = http.get(url, (response) => { response.resume(); response.statusCode && response.statusCode < 500 ? resolvePromise() : reject(new Error("not ready")); });
|
||||
request.on("error", reject);
|
||||
});
|
||||
return;
|
||||
} catch {
|
||||
await new Promise((resolvePromise) => setTimeout(resolvePromise, 250));
|
||||
}
|
||||
}
|
||||
throw new Error(`timed out waiting for ${url}`);
|
||||
}
|
||||
function live(pid) { try { process.kill(pid, 0); return true; } catch { return false; } }
|
||||
async function writeCommands(repo, root) {
|
||||
const commands = [
|
||||
["http-01-status.sh", curlGet(`http://${HOST}:${BACKEND_PORT}/workspace-registry/status`, join(root, "responses", "status.json"))],
|
||||
["http-02-validate-p11-filesystem.sh", curlPost(`http://${HOST}:${BACKEND_PORT}/workspaces/validate`, join(root, "responses", "validate-p11-filesystem.json"), join(root, "requests", "validate-p11-filesystem.json"))],
|
||||
["http-03-validate-p11-http.sh", curlPost(`http://${HOST}:${BACKEND_PORT}/workspaces/validate`, join(root, "responses", "validate-p11-http.json"), join(root, "requests", "validate-p11-http.json"))],
|
||||
["http-04-validate-p11-s3.sh", curlPost(`http://${HOST}:${BACKEND_PORT}/workspaces/validate`, join(root, "responses", "validate-p11-s3.json"), join(root, "requests", "validate-p11-s3.json"))],
|
||||
["http-05-publish-p11-filesystem.sh", publishCurl(root, "p11-filesystem", join(root, "responses", "status.json"))],
|
||||
["http-06-publish-p11-http.sh", publishCurl(root, "p11-http", join(root, "responses", "publish-p11-filesystem.json"))],
|
||||
["http-07-publish-p11-s3.sh", publishCurl(root, "p11-s3", join(root, "responses", "publish-p11-http.json"))],
|
||||
["http-08-pull.sh", curlPostEmpty(`http://${HOST}:${BACKEND_PORT}/workspace-registry/pull`, join(root, "responses", "pull.json"))],
|
||||
["http-09-read-p11-filesystem.sh", curlGet(`http://${HOST}:${BACKEND_PORT}/workspaces/p11-filesystem`, join(root, "responses", "read-p11-filesystem.json"))],
|
||||
["http-10-export-p11-filesystem.sh", curlGet(`http://${HOST}:${BACKEND_PORT}/workspaces/p11-filesystem/export`, join(root, "exports", "raw", "p11-filesystem.zip"))],
|
||||
["http-11-negative-invalid-uri.sh", curlPost(`http://${HOST}:${BACKEND_PORT}/workspaces/validate`, join(root, "responses", "negative-invalid-uri.json"), join(root, "requests", "negative-invalid-uri.json"))],
|
||||
["http-12-negative-secret-field.sh", curlPost(`http://${HOST}:${BACKEND_PORT}/workspaces/validate`, join(root, "responses", "negative-secret-field.json"), join(root, "requests", "negative-secret-field.json"))],
|
||||
["render-1.sh", renderCommand(repo, root, 1)],
|
||||
["render-2.sh", renderCommand(repo, root, 2)],
|
||||
];
|
||||
for (const [name, body] of commands) {
|
||||
const path = join(root, "commands", name);
|
||||
await atomicWrite(path, body, 0o700);
|
||||
}
|
||||
}
|
||||
export async function prepareManual({ repositoryRoot = defaultRepositoryRoot } = {}) {
|
||||
const repo = realpathSync(repositoryRoot);
|
||||
const root = fixedManualRoot(repo);
|
||||
noSymlinkExisting(repo, root);
|
||||
await ensureRootAbsent(root);
|
||||
await mkdir(join(repo, ".artifacts", "manual-acceptance"), { recursive: true, mode: 0o700 });
|
||||
await mkdir(root, { mode: 0o700 });
|
||||
const executables = resolveExecutables(repo);
|
||||
const nonce = randomBytes(32).toString("hex");
|
||||
await writeManualOwnership(root, ownershipValue(root, repo, nonce));
|
||||
for (const path of ["fixture-secrets", "requests", "responses", "commands", "rendered", "logs", "exports/raw", "exports/extracted", "installation/registry", "installation/data", "installation/runtime"]) {
|
||||
await mkdir(join(root, path), { recursive: true, mode: path === "fixture-secrets" ? 0o700 : 0o755 });
|
||||
}
|
||||
const env = buildSafeEnvironment({ ambient: process.env, fixture: { PATH: dirname(executables.gitPath) } });
|
||||
await git(executables.gitPath, ["init", "--bare", "--initial-branch=main", join(root, "remote.git")], { cwd: root, env });
|
||||
await git(executables.gitPath, ["clone", join(root, "remote.git"), join(root, "author")], { cwd: root, env });
|
||||
await git(executables.gitPath, ["config", "user.name", "P1 Fixture Curator"], { cwd: join(root, "author"), env });
|
||||
await git(executables.gitPath, ["config", "user.email", "p1-curator@example.invalid"], { cwd: join(root, "author"), env });
|
||||
const items = descriptors();
|
||||
await atomicWrite(join(root, "author", "thoth-workspaces.yaml"), `${JSON.stringify(catalog(items), null, 2)}\n`, 0o644);
|
||||
await mkdir(join(root, "author", "p11-filesystem", "evidence", "domain"), { recursive: true });
|
||||
await atomicWrite(join(root, "author", "p11-filesystem", "evidence", "guide.md"), "# P1.1 curated Evidence\n", 0o644);
|
||||
await atomicWrite(join(root, "author", "p11-filesystem", "evidence", "domain", "table.md"), "# Curated table\n", 0o644);
|
||||
await git(executables.gitPath, ["add", "thoth-workspaces.yaml"], { cwd: join(root, "author"), env });
|
||||
await git(executables.gitPath, ["add", "-A", "p11-filesystem/evidence"], { cwd: join(root, "author"), env });
|
||||
await git(executables.gitPath, ["commit", "-m", "Bootstrap curated P1 content"], { cwd: join(root, "author"), env });
|
||||
await git(executables.gitPath, ["push", "origin", "main"], { cwd: join(root, "author"), env });
|
||||
const secrets = {
|
||||
dwh: join(root, "fixture-secrets", "dwh-password"),
|
||||
signed: join(root, "fixture-secrets", "evidence-signed-urls.json"),
|
||||
access: join(root, "fixture-secrets", "evidence-access"),
|
||||
secret: join(root, "fixture-secrets", "evidence-secret"),
|
||||
session: join(root, "fixture-secrets", "evidence-session"),
|
||||
};
|
||||
await atomicWrite(secrets.dwh, "manual-dwh-secret", 0o600);
|
||||
await atomicWrite(secrets.signed, JSON.stringify(["https://evidence.example.test/guide.md?token=manual"]), 0o600);
|
||||
await atomicWrite(secrets.access, "manual-access", 0o600);
|
||||
await atomicWrite(secrets.secret, "manual-secret", 0o600);
|
||||
await atomicWrite(secrets.session, "manual-session", 0o600);
|
||||
const bindings = {};
|
||||
for (const workspace of items) {
|
||||
const prefix = `THT_WS_${namespace(workspace.workspace.id)}`;
|
||||
Object.assign(bindings, {
|
||||
[`${prefix}_DWH_TRANSPORT`]: "postgres_direct",
|
||||
[`${prefix}_DWH_HOST`]: "dwh.invalid",
|
||||
[`${prefix}_DWH_PORT`]: "5432",
|
||||
[`${prefix}_DWH_USER`]: "reader",
|
||||
[`${prefix}_DWH_PASSWORD_FILE`]: secrets.dwh,
|
||||
});
|
||||
}
|
||||
Object.assign(bindings, {
|
||||
THT_WS_P11_HTTP_EVIDENCE_SIGNED_URLS_FILE: secrets.signed,
|
||||
THT_WS_P11_S3_EVIDENCE_ACCESS_KEY_FILE: secrets.access,
|
||||
THT_WS_P11_S3_EVIDENCE_SECRET_KEY_FILE: secrets.secret,
|
||||
THT_WS_P11_S3_EVIDENCE_SESSION_TOKEN_FILE: secrets.session,
|
||||
});
|
||||
await atomicWrite(join(root, "installation", "bindings.env"), `${Object.entries(bindings).map(([key, value]) => `${key}=${value}`).join("\n")}\n`);
|
||||
await atomicWrite(join(root, "installation", "runtime", "base.yaml"), "{}\n");
|
||||
for (const [name, value] of Object.entries(requestFixtures(items))) await atomicWrite(join(root, "requests", name), `${JSON.stringify(value, null, 2)}\n`, 0o600);
|
||||
await writeCommands(repo, root);
|
||||
await atomicWrite(join(root, "GUIDE.md"), guide(root), 0o600);
|
||||
await atomicWrite(join(root, "logs", "backend.log"), "", 0o600);
|
||||
const current = await readManualOwnership({ repositoryRoot: repo });
|
||||
current.status = "PENDING";
|
||||
current.requestFixtures = Object.keys(requestFixtures(items));
|
||||
current.commandScripts = (await readdir(join(root, "commands"))).sort();
|
||||
await writeManualOwnership(root, current);
|
||||
return root;
|
||||
}
|
||||
export async function serveManual({ repositoryRoot = defaultRepositoryRoot } = {}) {
|
||||
const repo = realpathSync(repositoryRoot);
|
||||
const root = fixedManualRoot(repo);
|
||||
const owned = await readManualOwnership({ repositoryRoot: repo });
|
||||
if (owned.status === "RUNNING") throw new Error("manual acceptance is already serving");
|
||||
await access(join(repo, "backend", "dist", "server.js"));
|
||||
await access(join(repo, "frontend", "dist", "index.html"));
|
||||
const executables = resolveExecutables(repo);
|
||||
const logHandle = await open(join(root, "logs", "backend.log"), fsConstants.O_WRONLY | fsConstants.O_APPEND);
|
||||
const homeDir = join(root, "installation", "runtime", "home");
|
||||
const tmpDir = join(root, "installation", "runtime", "tmp");
|
||||
await mkdir(homeDir, { recursive: true, mode: 0o700 });
|
||||
await mkdir(tmpDir, { recursive: true, mode: 0o700 });
|
||||
const fixtureEnv = {
|
||||
PATH: `${dirname(executables.gitPath)}:${dirname(executables.pythonPath)}:${dirname(executables.thtPath)}:/usr/bin:/bin`,
|
||||
HOME: homeDir,
|
||||
TMPDIR: tmpDir,
|
||||
HOST,
|
||||
PORT: String(BACKEND_PORT),
|
||||
AUTH_MODE: "none",
|
||||
THT_BIN: executables.thtPath,
|
||||
THT_HARNESS_DIR: join(repo, "harness"),
|
||||
THT_DATA_ROOT: join(root, "installation", "data"),
|
||||
SETTINGS_FILE: join(root, "installation", "data", "settings.json"),
|
||||
MAINTENANCE_STATE_FILE: join(root, "installation", "data", "maintenance.json"),
|
||||
THT_WORKSPACE_REGISTRY_ROOT: join(root, "installation", "registry"),
|
||||
THT_WORKSPACE_GIT_REMOTE: join(root, "remote.git"),
|
||||
THT_WORKSPACE_GIT_BRANCH: "main",
|
||||
THT_WORKSPACE_GIT_AUTHOR_NAME: "P1 API Publisher",
|
||||
THT_WORKSPACE_GIT_AUTHOR_EMAIL: "p1-api@example.invalid",
|
||||
THT_WORKSPACE_INSTALLATION_ID: "p11-manual-acceptance",
|
||||
THT_WORKSPACE_SECRET_ROOTS: join(root, "fixture-secrets"),
|
||||
THT_HOME: join(root, "installation", "runtime", "tht-home"),
|
||||
PYTHONDONTWRITEBYTECODE: "1",
|
||||
PYTHONNOUSERSITE: "1",
|
||||
};
|
||||
const bindingEnv = Object.fromEntries((await readFile(join(root, "installation", "bindings.env"), "utf8")).trim().split(/\n+/).map((line) => line.split(/=(.+)/)));
|
||||
const env = buildSafeEnvironment({ ambient: process.env, fixture: { ...fixtureEnv, ...bindingEnv } });
|
||||
const backend = spawn(process.execPath, [join(repo, "backend", "dist", "server.js")], { cwd: repo, env, stdio: ["ignore", logHandle.fd, logHandle.fd], detached: true });
|
||||
const frontend = spawn(executables.pythonPath, ["-m", "http.server", String(FRONTEND_PORT), "--bind", HOST, "--directory", join(repo, "frontend", "dist")], { cwd: repo, env, stdio: ["ignore", "ignore", "ignore"], detached: true });
|
||||
backend.unref(); frontend.unref();
|
||||
await waitForHttp(`http://${HOST}:${BACKEND_PORT}/health`);
|
||||
await waitForHttp(`http://${HOST}:${FRONTEND_PORT}/`);
|
||||
await logHandle.close();
|
||||
owned.status = "RUNNING";
|
||||
owned.backend = { pid: backend.pid, port: BACKEND_PORT, command: [process.execPath, join(repo, "backend", "dist", "server.js")] };
|
||||
owned.frontend = { pid: frontend.pid, port: FRONTEND_PORT, command: [executables.pythonPath, "-m", "http.server", String(FRONTEND_PORT)] };
|
||||
await writeManualOwnership(root, owned);
|
||||
return owned;
|
||||
}
|
||||
async function processCommandMatches(pid, expectedCommand) {
|
||||
if (!Array.isArray(expectedCommand) || expectedCommand.length === 0) return false;
|
||||
let output;
|
||||
try {
|
||||
const { stdout } = await execFileAsync("ps", ["-p", String(pid), "-o", "command="], { encoding: "utf8" });
|
||||
output = stdout.trim();
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
if (output.length === 0) return false;
|
||||
// The recorded command is the argv array used to spawn the process; verify every token appears
|
||||
// in the current command line in order, so a reused PID with unrelated command is refused.
|
||||
let cursor = 0;
|
||||
for (const token of expectedCommand) {
|
||||
if (token.length === 0) continue;
|
||||
const index = output.indexOf(token, cursor);
|
||||
if (index < 0) return false;
|
||||
cursor = index + token.length;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
export async function stopManual({ repositoryRoot = defaultRepositoryRoot } = {}) {
|
||||
const repo = realpathSync(repositoryRoot);
|
||||
const root = fixedManualRoot(repo);
|
||||
const owned = await readManualOwnership({ repositoryRoot: repo });
|
||||
if (owned.status !== "RUNNING" || !owned.backend?.pid || !owned.frontend?.pid) throw new Error("manual acceptance is not running");
|
||||
for (const pid of [owned.backend.pid, owned.frontend.pid]) {
|
||||
try { process.kill(-pid, "SIGTERM"); } catch (error) { if (error?.code !== "ESRCH") throw error; }
|
||||
}
|
||||
const deadline = Date.now() + 15_000;
|
||||
while (Date.now() < deadline && (live(owned.backend.pid) || live(owned.frontend.pid))) await new Promise((resolvePromise) => setTimeout(resolvePromise, 250));
|
||||
owned.status = "STOPPED";
|
||||
await writeManualOwnership(root, owned);
|
||||
return owned;
|
||||
}
|
||||
|
||||
export async function cleanupManual({ repositoryRoot = defaultRepositoryRoot } = {}) {
|
||||
const repo = realpathSync(repositoryRoot);
|
||||
const root = fixedManualRoot(repo);
|
||||
const owned = await readManualOwnership({ repositoryRoot: repo });
|
||||
if (owned.status === "RUNNING") throw new Error("manual acceptance is still live");
|
||||
if (owned.backend?.pid && live(owned.backend.pid)) throw new Error("backend process is still live");
|
||||
if (owned.frontend?.pid && live(owned.frontend.pid)) throw new Error("frontend process is still live");
|
||||
const parent = dirname(root);
|
||||
const tombstone = join(parent, `.deleting-p11-${owned.nonce.slice(0, 16)}`);
|
||||
await rename(root, tombstone);
|
||||
await rm(tombstone, { recursive: true, force: false });
|
||||
}
|
||||
export async function main(argv = process.argv.slice(2)) {
|
||||
if (argv.length !== 1 || !["prepare", "serve", "stop", "cleanup"].includes(argv[0])) throw new Error("usage: p11-manual-acceptance.mjs prepare|serve|stop|cleanup");
|
||||
switch (argv[0]) {
|
||||
case "prepare": await prepareManual(); break;
|
||||
case "serve": await serveManual(); break;
|
||||
case "stop": await stopManual(); break;
|
||||
case "cleanup": await cleanupManual(); break;
|
||||
}
|
||||
}
|
||||
if (process.argv[1] && realpathSync(process.argv[1]) === modulePath) {
|
||||
try { await main(); } catch (error) { console.error(error instanceof Error ? error.message : String(error)); process.exitCode = 1; }
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { createHash } from "node:crypto";
|
||||
import { access, lstat, readFile, rm } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import test from "node:test";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { dirname, resolve } from "node:path";
|
||||
|
||||
import {
|
||||
cleanupManual,
|
||||
prepareManual,
|
||||
readManualOwnership,
|
||||
serveManual,
|
||||
stopManual,
|
||||
} from "./p11-manual-acceptance.mjs";
|
||||
|
||||
const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), "../..");
|
||||
const fixedRoot = join(repoRoot, ".artifacts", "manual-acceptance", "p11");
|
||||
|
||||
async function safeCleanup() {
|
||||
try {
|
||||
const owned = await readManualOwnership({ repositoryRoot: repoRoot });
|
||||
if (owned.status === "RUNNING") await stopManual({ repositoryRoot: repoRoot }).catch(() => {});
|
||||
await cleanupManual({ repositoryRoot: repoRoot }).catch(() => {});
|
||||
} catch {
|
||||
await rm(fixedRoot, { recursive: true, force: true }).catch(() => {});
|
||||
}
|
||||
}
|
||||
|
||||
test.beforeEach(async () => {
|
||||
await safeCleanup();
|
||||
});
|
||||
|
||||
test.afterEach(async () => {
|
||||
await safeCleanup();
|
||||
});
|
||||
|
||||
test("prepare creates an independent pending lab without verdict", { concurrency: false }, async () => {
|
||||
const root = await prepareManual({ repositoryRoot: repoRoot });
|
||||
assert.equal(root, fixedRoot);
|
||||
const owned = await readManualOwnership({ repositoryRoot: repoRoot });
|
||||
assert.equal(owned.kind, "p11-manual-acceptance");
|
||||
assert.equal(owned.status, "PENDING");
|
||||
await access(join(root, "GUIDE.md"));
|
||||
await access(join(root, "author", "thoth-workspaces.yaml"));
|
||||
await access(join(root, "author", "p11-filesystem", "evidence", "guide.md"));
|
||||
await access(join(root, "requests", "validate-p11-filesystem.json"));
|
||||
await access(join(root, "commands", "http-01-status.sh"));
|
||||
await access(join(root, "commands", "render-1.sh"));
|
||||
await assert.rejects(access(join(root, "VERDICT.md")));
|
||||
const guide = await readFile(join(root, "GUIDE.md"), "utf8");
|
||||
assert.match(guide, /VERDICT\.md/);
|
||||
assert.match(guide, /read-only/);
|
||||
});
|
||||
|
||||
test("serve, stop, and cleanup manage the owned backend and frontend listeners", { concurrency: false }, async () => {
|
||||
await prepareManual({ repositoryRoot: repoRoot });
|
||||
const running = await serveManual({ repositoryRoot: repoRoot });
|
||||
assert.equal(running.status, "RUNNING");
|
||||
assert.equal(typeof running.backend.pid, "number");
|
||||
assert.equal(typeof running.frontend.pid, "number");
|
||||
const status = await fetch("http://127.0.0.1:8791/workspace-registry/status");
|
||||
assert.equal(status.status, 200);
|
||||
const frontend = await fetch("http://127.0.0.1:8792/");
|
||||
assert.equal(frontend.status, 200);
|
||||
await assert.rejects(cleanupManual({ repositoryRoot: repoRoot }), /still live/);
|
||||
const stopped = await stopManual({ repositoryRoot: repoRoot });
|
||||
assert.equal(stopped.status, "STOPPED");
|
||||
await cleanupManual({ repositoryRoot: repoRoot });
|
||||
await assert.rejects(lstat(fixedRoot));
|
||||
});
|
||||
|
||||
test("stop fails closed when ownership is tampered", { concurrency: false }, async () => {
|
||||
await prepareManual({ repositoryRoot: repoRoot });
|
||||
const running = await serveManual({ repositoryRoot: repoRoot });
|
||||
const ownershipPath = join(fixedRoot, "ownership.json");
|
||||
const digestPath = join(fixedRoot, "ownership.sha256");
|
||||
const original = JSON.parse(await readFile(ownershipPath, "utf8"));
|
||||
const tampered = { ...original, backend: { ...original.backend, pid: original.backend.pid + 1 } };
|
||||
await rm(ownershipPath);
|
||||
await readFile(join(fixedRoot, "logs", "backend.log"));
|
||||
await import("node:fs/promises").then(({ writeFile }) => writeFile(ownershipPath, `${JSON.stringify(tampered, null, 2)}
|
||||
`));
|
||||
await assert.rejects(stopManual({ repositoryRoot: repoRoot }), /manual ownership digest mismatch/);
|
||||
const restored = `${JSON.stringify(running, null, 2)}
|
||||
`;
|
||||
const restoredDigest = `${createHash("sha256").update(restored).digest("hex")}
|
||||
`;
|
||||
await import("node:fs/promises").then(({ writeFile }) => Promise.all([writeFile(ownershipPath, restored), writeFile(digestPath, restoredDigest)]));
|
||||
await stopManual({ repositoryRoot: repoRoot });
|
||||
});
|
||||
@@ -0,0 +1,190 @@
|
||||
#!/usr/bin/env node
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { createHash } from "node:crypto";
|
||||
import { constants, lstatSync, realpathSync } from "node:fs";
|
||||
import { lstat, mkdir, open, readFile, realpath } from "node:fs/promises";
|
||||
import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
import { ThtRunner } from "../dist/tht/tht-runner.js";
|
||||
|
||||
const modulePath = fileURLToPath(import.meta.url);
|
||||
const defaultRepositoryRoot = realpathSync(resolve(dirname(modulePath), "../.."));
|
||||
const HEX40 = /^[0-9a-f]{40}$/;
|
||||
const HEX64 = /^[0-9a-f]{64}$/;
|
||||
|
||||
function fixedRoot(repositoryRoot) { return join(realpathSync(repositoryRoot), ".artifacts", "manual-acceptance", "p11"); }
|
||||
function below(parent, child) { const rel = relative(parent, child); return rel !== "" && !rel.startsWith(`..${sep}`) && rel !== ".." && !isAbsolute(rel); }
|
||||
function assertNoSymlinks(root, path, allowMissingLeaf = false) {
|
||||
const rel = relative(root, path);
|
||||
if (rel.startsWith("..") || isAbsolute(rel)) throw new Error("path is outside owned root");
|
||||
let cursor = root;
|
||||
const parts = rel.split(sep).filter(Boolean);
|
||||
for (const [index, part] of parts.entries()) {
|
||||
cursor = join(cursor, part);
|
||||
try { if (lstatSync(cursor).isSymbolicLink()) throw new Error("owned path contains a symlink"); }
|
||||
catch (error) {
|
||||
if (allowMissingLeaf && error?.code === "ENOENT" && index === parts.length - 1) return;
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
async function ownership(repositoryRoot, ownershipPath) {
|
||||
const root = fixedRoot(repositoryRoot);
|
||||
const expected = join(root, "ownership.json");
|
||||
if (resolve(ownershipPath) !== expected) throw new Error("ownership path is not owned");
|
||||
const rootEntry = await lstat(root); const ownershipEntry = await lstat(expected);
|
||||
if (!rootEntry.isDirectory() || rootEntry.isSymbolicLink() || !ownershipEntry.isFile() || ownershipEntry.isSymbolicLink()) throw new Error("ownership is unsafe");
|
||||
if (await realpath(root) !== root) throw new Error("ownership root is not canonical");
|
||||
let value; try { value = JSON.parse(await readFile(expected, "utf8")); } catch { throw new Error("ownership is malformed"); }
|
||||
if (value?.schemaVersion !== 1 || value.kind !== "p11-manual-acceptance" || !HEX64.test(value.nonce ?? "") || value.root !== root || value.repositoryRoot !== realpathSync(repositoryRoot)) {
|
||||
throw new Error("ownership identity mismatch");
|
||||
}
|
||||
return { root, value };
|
||||
}
|
||||
const ANCHORED_PUBLISH_SOURCE=String.raw`import os,secrets,stat,sys
|
||||
parent,name,expected_dev,expected_ino=sys.argv[1:]
|
||||
pfd=fd=None;stage=".render-stage-"+secrets.token_hex(16);published=False
|
||||
def fail(): raise RuntimeError("anchored publication refused")
|
||||
try:
|
||||
pfd=os.open(parent,os.O_RDONLY|os.O_DIRECTORY|os.O_NOFOLLOW)
|
||||
identity=os.fstat(pfd)
|
||||
if (identity.st_dev,identity.st_ino)!=(int(expected_dev),int(expected_ino)): fail()
|
||||
try: os.stat(name,dir_fd=pfd,follow_symlinks=False); fail()
|
||||
except FileNotFoundError: pass
|
||||
fd=os.open(stage,os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,0o600,dir_fd=pfd)
|
||||
data=sys.stdin.buffer.read(33554433)
|
||||
if len(data)>33554432: fail()
|
||||
view=memoryview(data)
|
||||
while view:
|
||||
written=os.write(fd,view)
|
||||
if written<=0: fail()
|
||||
view=view[written:]
|
||||
os.fsync(fd);os.close(fd);fd=None;os.rename(stage,name,src_dir_fd=pfd,dst_dir_fd=pfd);published=True;os.fsync(pfd)
|
||||
current=os.stat(parent,follow_symlinks=False)
|
||||
if not stat.S_ISDIR(current.st_mode) or (current.st_dev,current.st_ino)!=(identity.st_dev,identity.st_ino): fail()
|
||||
except Exception:
|
||||
if published:
|
||||
try: os.unlink(name,dir_fd=pfd);os.fsync(pfd)
|
||||
except Exception: pass
|
||||
print("anchored output publication refused (details redacted)",file=sys.stderr);raise SystemExit(1)
|
||||
finally:
|
||||
if fd is not None: os.close(fd)
|
||||
if pfd is not None:
|
||||
try: os.unlink(stage,dir_fd=pfd)
|
||||
except FileNotFoundError: pass
|
||||
os.close(pfd)
|
||||
`;
|
||||
async function atomicCopy(source, output) {
|
||||
const parent = dirname(output);
|
||||
const entry = await lstat(parent);
|
||||
if (!entry.isDirectory() || entry.isSymbolicLink()) throw new Error("rendered parent identity is unsafe");
|
||||
const bytes = await readFile(source);
|
||||
const result = spawnSync("python3", ["-c", ANCHORED_PUBLISH_SOURCE, parent, basename(output), String(entry.dev), String(entry.ino)], { input: bytes, encoding: "utf8", maxBuffer: 1024 * 1024 });
|
||||
if (result.error || result.status !== 0) throw new Error("anchored output publication refused; rendered parent identity changed or output is unsafe");
|
||||
}
|
||||
function sameEntry(actual, expected) { return actual.dev === expected.dev && actual.ino === expected.ino; }
|
||||
async function readBounded(path, max, label) {
|
||||
let handle;
|
||||
try {
|
||||
handle = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW);
|
||||
const before = await handle.stat(); const pathEntry = await lstat(path);
|
||||
if (!before.isFile() || pathEntry.isSymbolicLink() || !pathEntry.isFile() || !sameEntry(before, pathEntry)) throw new Error(`${label} is unsafe`);
|
||||
if (before.size < 1 || before.size > max) throw new Error(`${label} is unbounded`);
|
||||
const bytes = Buffer.alloc(before.size); let offset = 0;
|
||||
while (offset < bytes.length) {
|
||||
const { bytesRead } = await handle.read(bytes, offset, bytes.length - offset, offset);
|
||||
if (bytesRead < 1) throw new Error(`${label} changed while reading`);
|
||||
offset += bytesRead;
|
||||
}
|
||||
const after = await handle.stat();
|
||||
if (!sameEntry(before, after) || after.size !== before.size) throw new Error(`${label} changed while reading`);
|
||||
return bytes;
|
||||
} finally {
|
||||
if (handle) await handle.close().catch(() => {});
|
||||
}
|
||||
}
|
||||
async function readSnapshotManifest(root, manifestPath, commit, yamlName, expectedDigest) {
|
||||
let manifestEntry;
|
||||
try { assertNoSymlinks(root, manifestPath); manifestEntry = await lstat(manifestPath); }
|
||||
catch (error) { if (error?.code === "ENOENT") throw new Error("snapshot manifest is missing or unbounded"); throw error; }
|
||||
if (!manifestEntry.isFile() || manifestEntry.isSymbolicLink() || await realpath(manifestPath) !== manifestPath) throw new Error("snapshot manifest is unsafe");
|
||||
const bytes = await readBounded(manifestPath, 1024 * 1024, "snapshot manifest");
|
||||
let manifest; try { manifest = JSON.parse(bytes.toString("utf8")); } catch { throw new Error("snapshot manifest is malformed"); }
|
||||
const files = manifest?.files;
|
||||
if (manifest?.head !== commit || !files || typeof files !== "object" || Array.isArray(files)) throw new Error("snapshot manifest identity is unsafe");
|
||||
if (!HEX64.test(files[yamlName] ?? "") || files[yamlName] !== expectedDigest) throw new Error("snapshot manifest digest is unsafe");
|
||||
return manifest;
|
||||
}
|
||||
|
||||
export async function renderOwnedSnapshot({ repositoryRoot = defaultRepositoryRoot, ownershipPath, snapshotPath, outputPath, snapshotSha256, env = process.env, beforePublish }) {
|
||||
const repo = realpathSync(repositoryRoot);
|
||||
const { root } = await ownership(repo, resolve(repo, ownershipPath));
|
||||
const snapshot = resolve(repo, snapshotPath);
|
||||
const output = resolve(repo, outputPath);
|
||||
const snapshotsRoot = join(root, "installation", "registry", "snapshots");
|
||||
const renderedRoot = join(root, "rendered");
|
||||
if (!isAbsolute(snapshotPath) || !below(snapshotsRoot, snapshot)) throw new Error("snapshot is not an owned absolute path");
|
||||
const match = /^([0-9a-f]{40})\/([a-z][a-z0-9-]{2,62})\.yaml$/.exec(relative(snapshotsRoot, snapshot).split(sep).join("/"));
|
||||
if (!match || !HEX40.test(match[1])) throw new Error("snapshot is not commit addressed");
|
||||
if (!HEX64.test(snapshotSha256 ?? "")) throw new Error("snapshot digest identity is unsafe");
|
||||
assertNoSymlinks(root, snapshot);
|
||||
const snapshotEntry = await lstat(snapshot);
|
||||
if (!snapshotEntry.isFile() || snapshotEntry.isSymbolicLink() || await realpath(snapshot) !== snapshot) throw new Error("snapshot is unsafe");
|
||||
const yamlName = `${match[2]}.yaml`;
|
||||
await readSnapshotManifest(root, join(snapshotsRoot, match[1], "snapshot.json"), match[1], yamlName, snapshotSha256);
|
||||
const snapshotBytes = await readBounded(snapshot, 1024 * 1024, "snapshot");
|
||||
if (createHash("sha256").update(snapshotBytes).digest("hex") !== snapshotSha256) throw new Error("snapshot bytes changed");
|
||||
if (!below(renderedRoot, output) || dirname(output) !== renderedRoot || !output.endsWith(".yaml")) throw new Error("output is not an owned rendered path");
|
||||
assertNoSymlinks(root, dirname(output));
|
||||
try { if ((await lstat(output)).isSymbolicLink()) throw new Error("output is unsafe"); } catch (error) { if (error.code !== "ENOENT") throw error; }
|
||||
await mkdir(join(snapshotsRoot, "runtime"), { recursive: true, mode: 0o700 });
|
||||
const bindingEnv = Object.fromEntries((await readFile(join(root, "installation", "bindings.env"), "utf8")).trim().split(/\n+/).filter(Boolean).map((line) => line.split(/=(.+)/)));
|
||||
const effectiveEnv = { ...bindingEnv, ...env };
|
||||
const prior = {};
|
||||
for (const [key, value] of Object.entries(effectiveEnv)) { prior[key] = process.env[key]; if (value === undefined) delete process.env[key]; else process.env[key] = value; }
|
||||
const runner = new ThtRunner({
|
||||
thtBin: join(repo, "harness", ".venv", "bin", "tht"),
|
||||
harnessDir: join(repo, "harness"),
|
||||
configPath: join(root, "installation", "runtime", "base.yaml"),
|
||||
dataRoot: join(root, "installation", "data"),
|
||||
runtimeSnapshotRoot: join(snapshotsRoot, "runtime"),
|
||||
secretRoots: [join(root, "fixture-secrets")],
|
||||
semanticRuntime: { internalQdrantUrl: "http://qdrant:6333", internalEmbeddingUrl: "http://embedding:11434", internalEmbeddingModel: "qwen3-embedding:0.6b", internalEmbeddingDimensions: 1024 },
|
||||
});
|
||||
let lease;
|
||||
try {
|
||||
lease = runner.acquireWorkspaceRuntime(snapshot);
|
||||
const verifySnapshot = async () => {
|
||||
const current = await readBounded(snapshot, 1024 * 1024, "snapshot");
|
||||
if (createHash("sha256").update(current).digest("hex") !== snapshotSha256) throw new Error("snapshot content changed during rendering");
|
||||
};
|
||||
await verifySnapshot();
|
||||
if (beforePublish) await beforePublish({ output, renderedRoot });
|
||||
await verifySnapshot();
|
||||
await atomicCopy(lease.path, output);
|
||||
} finally {
|
||||
if (lease) lease.release();
|
||||
for (const key of Object.keys(env)) { if (prior[key] === undefined) delete process.env[key]; else process.env[key] = prior[key]; }
|
||||
}
|
||||
return output;
|
||||
}
|
||||
function parseArgs(argv) {
|
||||
if (argv.length !== 8) throw new Error("usage: p11-render-snapshot.mjs --ownership PATH --snapshot ABSOLUTE_PATH --output PATH --snapshot-sha256 HEX");
|
||||
const result = {};
|
||||
for (let index = 0; index < argv.length; index += 2) {
|
||||
if (!["--ownership", "--snapshot", "--output", "--snapshot-sha256"].includes(argv[index]) || result[argv[index]]) throw new Error("invalid arguments");
|
||||
result[argv[index]] = argv[index + 1];
|
||||
}
|
||||
return result;
|
||||
}
|
||||
if (process.argv[1] && realpathSync(process.argv[1]) === modulePath) {
|
||||
try {
|
||||
const args = parseArgs(process.argv.slice(2));
|
||||
await renderOwnedSnapshot({ ownershipPath: args["--ownership"], snapshotPath: args["--snapshot"], outputPath: args["--output"], snapshotSha256: args["--snapshot-sha256"] });
|
||||
console.log(`rendered ${resolve(args["--output"])}`);
|
||||
} catch (error) {
|
||||
console.error(`p11 render refused: ${error.message}`);
|
||||
process.exitCode = 1;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { access, readFile, rm } from "node:fs/promises";
|
||||
import { join, dirname, resolve } from "node:path";
|
||||
import test from "node:test";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
import { renderOwnedSnapshot } from "./p11-render-snapshot.mjs";
|
||||
import { cleanupManual, prepareManual, readManualOwnership, serveManual, stopManual } from "./p11-manual-acceptance.mjs";
|
||||
|
||||
const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), "../..");
|
||||
const fixedRoot = join(repoRoot, ".artifacts", "manual-acceptance", "p11");
|
||||
|
||||
async function safeCleanup() {
|
||||
try {
|
||||
const owned = await readManualOwnership({ repositoryRoot: repoRoot });
|
||||
if (owned.status === "RUNNING") await stopManual({ repositoryRoot: repoRoot }).catch(() => {});
|
||||
await cleanupManual({ repositoryRoot: repoRoot }).catch(() => {});
|
||||
} catch {
|
||||
await rm(fixedRoot, { recursive: true, force: true }).catch(() => {});
|
||||
}
|
||||
}
|
||||
|
||||
test.beforeEach(async () => { await safeCleanup(); });
|
||||
test.afterEach(async () => { await safeCleanup(); });
|
||||
|
||||
test("renderer rejects unowned ownership and out-of-root snapshot paths", { concurrency: false }, async () => {
|
||||
await prepareManual({ repositoryRoot: repoRoot });
|
||||
const outside = join(repoRoot, "outside.yaml");
|
||||
await import("node:fs/promises").then(({ writeFile }) => writeFile(outside, "x"));
|
||||
await assert.rejects(renderOwnedSnapshot({
|
||||
repositoryRoot: repoRoot,
|
||||
ownershipPath: join(repoRoot, "ownership.json"),
|
||||
snapshotPath: outside,
|
||||
outputPath: join(fixedRoot, "rendered", "bad.yaml"),
|
||||
snapshotSha256: "a".repeat(64),
|
||||
}));
|
||||
await rm(outside, { force: true });
|
||||
});
|
||||
|
||||
test("renderer copies an owned runtime lease deterministically", { concurrency: false }, async () => {
|
||||
await prepareManual({ repositoryRoot: repoRoot });
|
||||
await serveManual({ repositoryRoot: repoRoot });
|
||||
const validateRequest = JSON.parse(await readFile(join(fixedRoot, "requests", "validate-p11-filesystem.json"), "utf8"));
|
||||
const status = await fetch("http://127.0.0.1:8791/workspace-registry/status");
|
||||
const statusBody = await status.json();
|
||||
const publish = await fetch("http://127.0.0.1:8791/workspaces/publish", {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({ action: "create", workspace: validateRequest.workspace, baseCommit: statusBody.head }),
|
||||
});
|
||||
assert.equal(publish.status, 200);
|
||||
const readResponse = await fetch("http://127.0.0.1:8791/workspaces/p11-filesystem");
|
||||
const readBody = await readResponse.json();
|
||||
const snapshotPath = readBody.revision.snapshotPath;
|
||||
const manifest = JSON.parse(await readFile(join(dirname(snapshotPath), "snapshot.json"), "utf8"));
|
||||
const digest = manifest.files["p11-filesystem.yaml"];
|
||||
const one = join(fixedRoot, "rendered", "one.yaml");
|
||||
const two = join(fixedRoot, "rendered", "two.yaml");
|
||||
await renderOwnedSnapshot({ repositoryRoot: repoRoot, ownershipPath: join(fixedRoot, "ownership.json"), snapshotPath, outputPath: one, snapshotSha256: digest });
|
||||
await renderOwnedSnapshot({ repositoryRoot: repoRoot, ownershipPath: join(fixedRoot, "ownership.json"), snapshotPath, outputPath: two, snapshotSha256: digest });
|
||||
assert.equal(await readFile(one, "utf8"), await readFile(two, "utf8"));
|
||||
await access(one);
|
||||
await access(two);
|
||||
});
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,158 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdir, mkdtemp, readFile, rm, stat, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { dirname, join } from "node:path";
|
||||
import test from "node:test";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
import {
|
||||
CHECK_IDS,
|
||||
canonicalIntegrationBase,
|
||||
cleanupOwnedRun,
|
||||
createOwnedRun,
|
||||
readAndValidateOwnership,
|
||||
runIntegration,
|
||||
validateReport,
|
||||
validateRunRoot,
|
||||
} from "./p2-acceptance.mjs";
|
||||
|
||||
const roots = [];
|
||||
async function fakeRepository() {
|
||||
const root = await mkdtemp(join(tmpdir(), "p2-acceptance-repo-"));
|
||||
roots.push(root);
|
||||
await mkdir(join(root, ".artifacts", "p2-integration"), { recursive: true });
|
||||
await mkdir(join(root, ".artifacts", "p11-integration"), { recursive: true });
|
||||
await mkdir(join(root, ".artifacts", "p1-integration"), { recursive: true });
|
||||
await mkdir(join(root, ".artifacts", "manual-acceptance", "p11"), { recursive: true });
|
||||
return root;
|
||||
}
|
||||
|
||||
test.afterEach(async () => {
|
||||
await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true })));
|
||||
});
|
||||
|
||||
test("run roots are only canonical direct p2 integration children", async () => {
|
||||
const repositoryRoot = await fakeRepository();
|
||||
const base = canonicalIntegrationBase(repositoryRoot);
|
||||
const id = `p2-${"a".repeat(32)}`;
|
||||
assert.equal(validateRunRoot(repositoryRoot, join(base, id), id), join(base, id));
|
||||
for (const candidate of [
|
||||
base,
|
||||
join(repositoryRoot, ".artifacts", "manual-acceptance", "p11"),
|
||||
join(repositoryRoot, ".artifacts", "p1-integration", id),
|
||||
join(repositoryRoot, ".artifacts", "p11-integration", id),
|
||||
join(base, id, "nested"),
|
||||
join(base, "foreign"),
|
||||
]) {
|
||||
assert.throws(() => validateRunRoot(repositoryRoot, candidate, id));
|
||||
}
|
||||
assert.throws(() => validateRunRoot(repositoryRoot, join(base, `p2-${"A".repeat(32)}`), `p2-${"A".repeat(32)}`));
|
||||
});
|
||||
|
||||
test("cleanup refuses p1, p11, manual, sibling, and wrong-nonce roots", async () => {
|
||||
const repositoryRoot = await fakeRepository();
|
||||
const run = await createOwnedRun({ repositoryRoot });
|
||||
await readAndValidateOwnership({ repositoryRoot, runRoot: run.root, expectedNonce: run.nonce });
|
||||
for (const bad of [
|
||||
join(repositoryRoot, ".artifacts", "p1-integration", `p1-${"b".repeat(32)}`),
|
||||
join(repositoryRoot, ".artifacts", "p11-integration", `p11-${"c".repeat(32)}`),
|
||||
join(repositoryRoot, ".artifacts", "manual-acceptance", "p11"),
|
||||
join(canonicalIntegrationBase(repositoryRoot), `p2-${"d".repeat(32)}`),
|
||||
]) {
|
||||
await assert.rejects(cleanupOwnedRun({ repositoryRoot, runRoot: bad, expectedNonce: run.nonce }));
|
||||
}
|
||||
await assert.rejects(cleanupOwnedRun({ repositoryRoot, runRoot: run.root, expectedNonce: "0".repeat(64) }));
|
||||
});
|
||||
|
||||
test("cleanup removes exactly one owned p2 root", async () => {
|
||||
const repositoryRoot = await fakeRepository();
|
||||
const run = await createOwnedRun({ repositoryRoot });
|
||||
const sibling = join(canonicalIntegrationBase(repositoryRoot), `p2-${"e".repeat(32)}`);
|
||||
await mkdir(sibling);
|
||||
await writeFile(join(sibling, "sentinel"), "foreign");
|
||||
await cleanupOwnedRun({ repositoryRoot, runRoot: run.root, expectedNonce: run.nonce });
|
||||
await assert.rejects(readFile(join(run.root, "ownership.json")));
|
||||
assert.equal(await readFile(join(sibling, "sentinel"), "utf8"), "foreign");
|
||||
});
|
||||
|
||||
function resultFor(id) {
|
||||
return {
|
||||
id,
|
||||
status: "PASS",
|
||||
startedAt: "2026-08-12T00:00:00.000Z",
|
||||
finishedAt: "2026-08-12T00:00:01.000Z",
|
||||
commands: ["node"],
|
||||
artifacts: [{ path: `logs/${id}.json`, sha256: "a".repeat(64) }],
|
||||
};
|
||||
}
|
||||
|
||||
test("report validation requires exact p2 identity, check order, and unique artifacts", () => {
|
||||
const report = {
|
||||
schemaVersion: 1,
|
||||
runId: `p2-${"f".repeat(32)}`,
|
||||
startedAt: "2026-08-12T00:00:00.000Z",
|
||||
finishedAt: "2026-08-12T00:00:10.000Z",
|
||||
command: "p2-acceptance integration --keep",
|
||||
overall: "PASS",
|
||||
checks: CHECK_IDS.map(resultFor),
|
||||
};
|
||||
assert.doesNotThrow(() => validateReport(report));
|
||||
const invalid = structuredClone(report);
|
||||
invalid.runId = `p11-${"f".repeat(32)}`;
|
||||
assert.throws(() => validateReport(invalid));
|
||||
const duplicate = structuredClone(report);
|
||||
duplicate.checks[1].artifacts[0].path = duplicate.checks[0].artifacts[0].path;
|
||||
assert.throws(() => validateReport(duplicate), /duplicated/);
|
||||
const reordered = structuredClone(report);
|
||||
reordered.checks.reverse();
|
||||
reordered.overall = "FAIL";
|
||||
assert.throws(() => validateReport(reordered));
|
||||
});
|
||||
|
||||
test("public wrapper uses a strict empty environment", async () => {
|
||||
const wrapper = await readFile(join(dirname(fileURLToPath(import.meta.url)), "..", "..", "scripts", "p2-acceptance.sh"), "utf8");
|
||||
assert.match(wrapper, /safe_env=\(\/usr\/bin\/env -i/);
|
||||
assert.doesNotMatch(wrapper, /LANG|LC_ALL|TZ/);
|
||||
assert.doesNotMatch(wrapper, /P2_ACCEPTANCE_FAIL_AT/);
|
||||
});
|
||||
|
||||
test("synthetic integration cleans up successful non-kept runs", async () => {
|
||||
const repositoryRoot = await fakeRepository();
|
||||
const result = await runIntegration({ repositoryRoot, keep: false, env: { P2_ACCEPTANCE_SYNTHETIC: "1" } });
|
||||
assert.equal(result.exitCode, 0);
|
||||
assert.equal(result.retained, false);
|
||||
await assert.rejects(readFile(join(result.runRoot, "ownership.json")));
|
||||
});
|
||||
|
||||
test("synthetic integration retains kept runs with bounded reports", async () => {
|
||||
const repositoryRoot = await fakeRepository();
|
||||
const result = await runIntegration({ repositoryRoot, keep: true, env: { P2_ACCEPTANCE_SYNTHETIC: "1" } });
|
||||
assert.equal(result.exitCode, 0);
|
||||
assert.equal(result.retained, true);
|
||||
const report = JSON.parse(await readFile(join(result.runRoot, "report.json"), "utf8"));
|
||||
assert.equal(report.overall, "PASS");
|
||||
const reportMd = await readFile(join(result.runRoot, "report.md"), "utf8");
|
||||
assert.match(reportMd, /P2 automated integration: PASS/);
|
||||
assert.match(reportMd, /P2 manual acceptance: PENDING/);
|
||||
const reportJsonStat = await stat(join(result.runRoot, "report.json"));
|
||||
const reportMdStat = await stat(join(result.runRoot, "report.md"));
|
||||
assert.ok(reportJsonStat.size <= 64 * 1024, `report.json too large: ${reportJsonStat.size}`);
|
||||
assert.ok(reportMdStat.size <= 32 * 1024, `report.md too large: ${reportMdStat.size}`);
|
||||
});
|
||||
|
||||
test("synthetic injected failure retains the owned run and records a single failed report", async () => {
|
||||
const repositoryRoot = await fakeRepository();
|
||||
const result = await runIntegration({
|
||||
repositoryRoot,
|
||||
keep: false,
|
||||
env: { P2_ACCEPTANCE_SYNTHETIC: "1", P2_ACCEPTANCE_FAIL_AT: CHECK_IDS[2] },
|
||||
});
|
||||
assert.equal(result.exitCode, 1);
|
||||
assert.equal(result.retained, true);
|
||||
const report = JSON.parse(await readFile(join(result.runRoot, "report.json"), "utf8"));
|
||||
assert.equal(report.overall, "FAIL");
|
||||
const failed = report.checks.find((check) => check.id === CHECK_IDS[2]);
|
||||
assert.equal(failed.status, "FAIL");
|
||||
const roots = await readFile(join(result.runRoot, "ownership.json"), "utf8");
|
||||
assert.match(roots, /p2-acceptance/);
|
||||
});
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,158 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdir, mkdtemp, readFile, rm, stat, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { dirname, join } from "node:path";
|
||||
import test from "node:test";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
import {
|
||||
CHECK_IDS,
|
||||
canonicalIntegrationBase,
|
||||
cleanupOwnedRun,
|
||||
createOwnedRun,
|
||||
readAndValidateOwnership,
|
||||
runIntegration,
|
||||
validateReport,
|
||||
validateRunRoot,
|
||||
} from "./p2p6-acceptance.mjs";
|
||||
|
||||
const roots = [];
|
||||
async function fakeRepository() {
|
||||
const root = await mkdtemp(join(tmpdir(), "p2p6-acceptance-repo-"));
|
||||
roots.push(root);
|
||||
await mkdir(join(root, ".artifacts", "p2p6-integration"), { recursive: true });
|
||||
await mkdir(join(root, ".artifacts", "p2-integration"), { recursive: true });
|
||||
await mkdir(join(root, ".artifacts", "p1-integration"), { recursive: true });
|
||||
await mkdir(join(root, ".artifacts", "manual-acceptance", "p11"), { recursive: true });
|
||||
return root;
|
||||
}
|
||||
|
||||
test.afterEach(async () => {
|
||||
await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true })));
|
||||
});
|
||||
|
||||
test("run roots are only canonical direct p2p6 integration children", async () => {
|
||||
const repositoryRoot = await fakeRepository();
|
||||
const base = canonicalIntegrationBase(repositoryRoot);
|
||||
const id = `p2p6-${"a".repeat(32)}`;
|
||||
assert.equal(validateRunRoot(repositoryRoot, join(base, id), id), join(base, id));
|
||||
for (const candidate of [
|
||||
base,
|
||||
join(repositoryRoot, ".artifacts", "manual-acceptance", "p11"),
|
||||
join(repositoryRoot, ".artifacts", "p1-integration", id),
|
||||
join(repositoryRoot, ".artifacts", "p2-integration", id),
|
||||
join(base, id, "nested"),
|
||||
join(base, "foreign"),
|
||||
]) {
|
||||
assert.throws(() => validateRunRoot(repositoryRoot, candidate, id));
|
||||
}
|
||||
assert.throws(() => validateRunRoot(repositoryRoot, join(base, `p2p6-${"A".repeat(32)}`), `p2p6-${"A".repeat(32)}`));
|
||||
});
|
||||
|
||||
test("cleanup refuses p1, p2, p11, manual, sibling, and wrong-nonce roots", async () => {
|
||||
const repositoryRoot = await fakeRepository();
|
||||
const run = await createOwnedRun({ repositoryRoot });
|
||||
await readAndValidateOwnership({ repositoryRoot, runRoot: run.root, expectedNonce: run.nonce });
|
||||
for (const bad of [
|
||||
join(repositoryRoot, ".artifacts", "p1-integration", `p1-${"b".repeat(32)}`),
|
||||
join(repositoryRoot, ".artifacts", "p2-integration", `p2-${"c".repeat(32)}`),
|
||||
join(repositoryRoot, ".artifacts", "manual-acceptance", "p11"),
|
||||
join(canonicalIntegrationBase(repositoryRoot), `p2p6-${"d".repeat(32)}`),
|
||||
]) {
|
||||
await assert.rejects(cleanupOwnedRun({ repositoryRoot, runRoot: bad, expectedNonce: run.nonce }));
|
||||
}
|
||||
await assert.rejects(cleanupOwnedRun({ repositoryRoot, runRoot: run.root, expectedNonce: "0".repeat(64) }));
|
||||
});
|
||||
|
||||
test("cleanup removes exactly one owned p2p6 root", async () => {
|
||||
const repositoryRoot = await fakeRepository();
|
||||
const run = await createOwnedRun({ repositoryRoot });
|
||||
const sibling = join(canonicalIntegrationBase(repositoryRoot), `p2p6-${"e".repeat(32)}`);
|
||||
await mkdir(sibling);
|
||||
await writeFile(join(sibling, "sentinel"), "foreign");
|
||||
await cleanupOwnedRun({ repositoryRoot, runRoot: run.root, expectedNonce: run.nonce });
|
||||
await assert.rejects(readFile(join(run.root, "ownership.json")));
|
||||
assert.equal(await readFile(join(sibling, "sentinel"), "utf8"), "foreign");
|
||||
});
|
||||
|
||||
function resultFor(id) {
|
||||
return {
|
||||
id,
|
||||
status: "PASS",
|
||||
startedAt: "2026-08-12T00:00:00.000Z",
|
||||
finishedAt: "2026-08-12T00:00:01.000Z",
|
||||
commands: ["node"],
|
||||
artifacts: [{ path: `logs/${id}.json`, sha256: "a".repeat(64) }],
|
||||
};
|
||||
}
|
||||
|
||||
test("report validation requires exact p2p6 identity, check order, and unique artifacts", () => {
|
||||
const report = {
|
||||
schemaVersion: 1,
|
||||
runId: `p2p6-${"f".repeat(32)}`,
|
||||
startedAt: "2026-08-12T00:00:00.000Z",
|
||||
finishedAt: "2026-08-12T00:00:10.000Z",
|
||||
command: "p2p6-acceptance integration --keep",
|
||||
overall: "PASS",
|
||||
checks: CHECK_IDS.map(resultFor),
|
||||
};
|
||||
assert.doesNotThrow(() => validateReport(report));
|
||||
const invalid = structuredClone(report);
|
||||
invalid.runId = `p2-${"f".repeat(32)}`;
|
||||
assert.throws(() => validateReport(invalid));
|
||||
const duplicate = structuredClone(report);
|
||||
duplicate.checks[1].artifacts[0].path = duplicate.checks[0].artifacts[0].path;
|
||||
assert.throws(() => validateReport(duplicate), /duplicated/);
|
||||
const reordered = structuredClone(report);
|
||||
reordered.checks.reverse();
|
||||
reordered.overall = "FAIL";
|
||||
assert.throws(() => validateReport(reordered));
|
||||
});
|
||||
|
||||
test("public wrapper uses a strict empty environment", async () => {
|
||||
const wrapper = await readFile(join(dirname(fileURLToPath(import.meta.url)), "..", "..", "scripts", "p2p6-acceptance.sh"), "utf8");
|
||||
assert.match(wrapper, /safe_env=\(\/usr\/bin\/env -i/);
|
||||
assert.doesNotMatch(wrapper, /LANG|LC_ALL|TZ/);
|
||||
assert.doesNotMatch(wrapper, /P2P6_ACCEPTANCE_FAIL_AT/);
|
||||
});
|
||||
|
||||
test("synthetic integration cleans up successful non-kept runs", async () => {
|
||||
const repositoryRoot = await fakeRepository();
|
||||
const result = await runIntegration({ repositoryRoot, keep: false, env: { P2P6_ACCEPTANCE_SYNTHETIC: "1" } });
|
||||
assert.equal(result.exitCode, 0);
|
||||
assert.equal(result.retained, false);
|
||||
await assert.rejects(readFile(join(result.runRoot, "ownership.json")));
|
||||
});
|
||||
|
||||
test("synthetic integration retains kept runs with bounded reports", async () => {
|
||||
const repositoryRoot = await fakeRepository();
|
||||
const result = await runIntegration({ repositoryRoot, keep: true, env: { P2P6_ACCEPTANCE_SYNTHETIC: "1" } });
|
||||
assert.equal(result.exitCode, 0);
|
||||
assert.equal(result.retained, true);
|
||||
const report = JSON.parse(await readFile(join(result.runRoot, "report.json"), "utf8"));
|
||||
assert.equal(report.overall, "PASS");
|
||||
const reportMd = await readFile(join(result.runRoot, "report.md"), "utf8");
|
||||
assert.match(reportMd, /P2P6 automated integration: PASS/);
|
||||
assert.match(reportMd, /P2P6 manual acceptance: PENDING/);
|
||||
const reportJsonStat = await stat(join(result.runRoot, "report.json"));
|
||||
const reportMdStat = await stat(join(result.runRoot, "report.md"));
|
||||
assert.ok(reportJsonStat.size <= 64 * 1024, `report.json too large: ${reportJsonStat.size}`);
|
||||
assert.ok(reportMdStat.size <= 32 * 1024, `report.md too large: ${reportMdStat.size}`);
|
||||
});
|
||||
|
||||
test("synthetic injected failure retains the owned run and records a single failed report", async () => {
|
||||
const repositoryRoot = await fakeRepository();
|
||||
const result = await runIntegration({
|
||||
repositoryRoot,
|
||||
keep: false,
|
||||
env: { P2P6_ACCEPTANCE_SYNTHETIC: "1", P2P6_ACCEPTANCE_FAIL_AT: CHECK_IDS[2] },
|
||||
});
|
||||
assert.equal(result.exitCode, 1);
|
||||
assert.equal(result.retained, true);
|
||||
const report = JSON.parse(await readFile(join(result.runRoot, "report.json"), "utf8"));
|
||||
assert.equal(report.overall, "FAIL");
|
||||
const failed = report.checks.find((check) => check.id === CHECK_IDS[2]);
|
||||
assert.equal(failed.status, "FAIL");
|
||||
const roots = await readFile(join(result.runRoot, "ownership.json"), "utf8");
|
||||
assert.match(roots, /p2p6-acceptance/);
|
||||
});
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,158 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdir, mkdtemp, readFile, rm, stat, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { dirname, join } from "node:path";
|
||||
import test from "node:test";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
import {
|
||||
CHECK_IDS,
|
||||
canonicalIntegrationBase,
|
||||
cleanupOwnedRun,
|
||||
createOwnedRun,
|
||||
readAndValidateOwnership,
|
||||
runIntegration,
|
||||
validateReport,
|
||||
validateRunRoot,
|
||||
} from "./p3-acceptance.mjs";
|
||||
|
||||
const roots = [];
|
||||
async function fakeRepository() {
|
||||
const root = await mkdtemp(join(tmpdir(), "p3-acceptance-repo-"));
|
||||
roots.push(root);
|
||||
await mkdir(join(root, ".artifacts", "p3-integration"), { recursive: true });
|
||||
await mkdir(join(root, ".artifacts", "p2-integration"), { recursive: true });
|
||||
await mkdir(join(root, ".artifacts", "p1-integration"), { recursive: true });
|
||||
await mkdir(join(root, ".artifacts", "manual-acceptance", "p11"), { recursive: true });
|
||||
return root;
|
||||
}
|
||||
|
||||
test.afterEach(async () => {
|
||||
await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true })));
|
||||
});
|
||||
|
||||
test("run roots are only canonical direct p3 integration children", async () => {
|
||||
const repositoryRoot = await fakeRepository();
|
||||
const base = canonicalIntegrationBase(repositoryRoot);
|
||||
const id = `p3-${"a".repeat(32)}`;
|
||||
assert.equal(validateRunRoot(repositoryRoot, join(base, id), id), join(base, id));
|
||||
for (const candidate of [
|
||||
base,
|
||||
join(repositoryRoot, ".artifacts", "manual-acceptance", "p11"),
|
||||
join(repositoryRoot, ".artifacts", "p1-integration", id),
|
||||
join(repositoryRoot, ".artifacts", "p2-integration", id),
|
||||
join(base, id, "nested"),
|
||||
join(base, "foreign"),
|
||||
]) {
|
||||
assert.throws(() => validateRunRoot(repositoryRoot, candidate, id));
|
||||
}
|
||||
assert.throws(() => validateRunRoot(repositoryRoot, join(base, `p3-${"A".repeat(32)}`), `p3-${"A".repeat(32)}`));
|
||||
});
|
||||
|
||||
test("cleanup refuses p1, p2, p11, manual, sibling, and wrong-nonce roots", async () => {
|
||||
const repositoryRoot = await fakeRepository();
|
||||
const run = await createOwnedRun({ repositoryRoot });
|
||||
await readAndValidateOwnership({ repositoryRoot, runRoot: run.root, expectedNonce: run.nonce });
|
||||
for (const bad of [
|
||||
join(repositoryRoot, ".artifacts", "p1-integration", `p1-${"b".repeat(32)}`),
|
||||
join(repositoryRoot, ".artifacts", "p2-integration", `p2-${"c".repeat(32)}`),
|
||||
join(repositoryRoot, ".artifacts", "manual-acceptance", "p11"),
|
||||
join(canonicalIntegrationBase(repositoryRoot), `p3-${"d".repeat(32)}`),
|
||||
]) {
|
||||
await assert.rejects(cleanupOwnedRun({ repositoryRoot, runRoot: bad, expectedNonce: run.nonce }));
|
||||
}
|
||||
await assert.rejects(cleanupOwnedRun({ repositoryRoot, runRoot: run.root, expectedNonce: "0".repeat(64) }));
|
||||
});
|
||||
|
||||
test("cleanup removes exactly one owned p3 root", async () => {
|
||||
const repositoryRoot = await fakeRepository();
|
||||
const run = await createOwnedRun({ repositoryRoot });
|
||||
const sibling = join(canonicalIntegrationBase(repositoryRoot), `p3-${"e".repeat(32)}`);
|
||||
await mkdir(sibling);
|
||||
await writeFile(join(sibling, "sentinel"), "foreign");
|
||||
await cleanupOwnedRun({ repositoryRoot, runRoot: run.root, expectedNonce: run.nonce });
|
||||
await assert.rejects(readFile(join(run.root, "ownership.json")));
|
||||
assert.equal(await readFile(join(sibling, "sentinel"), "utf8"), "foreign");
|
||||
});
|
||||
|
||||
function resultFor(id) {
|
||||
return {
|
||||
id,
|
||||
status: "PASS",
|
||||
startedAt: "2026-08-12T00:00:00.000Z",
|
||||
finishedAt: "2026-08-12T00:00:01.000Z",
|
||||
commands: ["node"],
|
||||
artifacts: [{ path: `logs/${id}.json`, sha256: "a".repeat(64) }],
|
||||
};
|
||||
}
|
||||
|
||||
test("report validation requires exact p3 identity, check order, and unique artifacts", () => {
|
||||
const report = {
|
||||
schemaVersion: 1,
|
||||
runId: `p3-${"f".repeat(32)}`,
|
||||
startedAt: "2026-08-12T00:00:00.000Z",
|
||||
finishedAt: "2026-08-12T00:00:10.000Z",
|
||||
command: "p3-acceptance integration --keep",
|
||||
overall: "PASS",
|
||||
checks: CHECK_IDS.map(resultFor),
|
||||
};
|
||||
assert.doesNotThrow(() => validateReport(report));
|
||||
const invalid = structuredClone(report);
|
||||
invalid.runId = `p2-${"f".repeat(32)}`;
|
||||
assert.throws(() => validateReport(invalid));
|
||||
const duplicate = structuredClone(report);
|
||||
duplicate.checks[1].artifacts[0].path = duplicate.checks[0].artifacts[0].path;
|
||||
assert.throws(() => validateReport(duplicate), /duplicated/);
|
||||
const reordered = structuredClone(report);
|
||||
reordered.checks.reverse();
|
||||
reordered.overall = "FAIL";
|
||||
assert.throws(() => validateReport(reordered));
|
||||
});
|
||||
|
||||
test("public wrapper uses a strict empty environment", async () => {
|
||||
const wrapper = await readFile(join(dirname(fileURLToPath(import.meta.url)), "..", "..", "scripts", "p3-acceptance.sh"), "utf8");
|
||||
assert.match(wrapper, /safe_env=\(\/usr\/bin\/env -i/);
|
||||
assert.doesNotMatch(wrapper, /LANG|LC_ALL|TZ/);
|
||||
assert.doesNotMatch(wrapper, /P3_ACCEPTANCE_FAIL_AT/);
|
||||
});
|
||||
|
||||
test("synthetic integration cleans up successful non-kept runs", async () => {
|
||||
const repositoryRoot = await fakeRepository();
|
||||
const result = await runIntegration({ repositoryRoot, keep: false, env: { P3_ACCEPTANCE_SYNTHETIC: "1" } });
|
||||
assert.equal(result.exitCode, 0);
|
||||
assert.equal(result.retained, false);
|
||||
await assert.rejects(readFile(join(result.runRoot, "ownership.json")));
|
||||
});
|
||||
|
||||
test("synthetic integration retains kept runs with bounded reports", async () => {
|
||||
const repositoryRoot = await fakeRepository();
|
||||
const result = await runIntegration({ repositoryRoot, keep: true, env: { P3_ACCEPTANCE_SYNTHETIC: "1" } });
|
||||
assert.equal(result.exitCode, 0);
|
||||
assert.equal(result.retained, true);
|
||||
const report = JSON.parse(await readFile(join(result.runRoot, "report.json"), "utf8"));
|
||||
assert.equal(report.overall, "PASS");
|
||||
const reportMd = await readFile(join(result.runRoot, "report.md"), "utf8");
|
||||
assert.match(reportMd, /P3 automated integration: PASS/);
|
||||
assert.match(reportMd, /P3 manual acceptance: PENDING/);
|
||||
const reportJsonStat = await stat(join(result.runRoot, "report.json"));
|
||||
const reportMdStat = await stat(join(result.runRoot, "report.md"));
|
||||
assert.ok(reportJsonStat.size <= 64 * 1024, `report.json too large: ${reportJsonStat.size}`);
|
||||
assert.ok(reportMdStat.size <= 32 * 1024, `report.md too large: ${reportMdStat.size}`);
|
||||
});
|
||||
|
||||
test("synthetic injected failure retains the owned run and records a single failed report", async () => {
|
||||
const repositoryRoot = await fakeRepository();
|
||||
const result = await runIntegration({
|
||||
repositoryRoot,
|
||||
keep: false,
|
||||
env: { P3_ACCEPTANCE_SYNTHETIC: "1", P3_ACCEPTANCE_FAIL_AT: CHECK_IDS[2] },
|
||||
});
|
||||
assert.equal(result.exitCode, 1);
|
||||
assert.equal(result.retained, true);
|
||||
const report = JSON.parse(await readFile(join(result.runRoot, "report.json"), "utf8"));
|
||||
assert.equal(report.overall, "FAIL");
|
||||
const failed = report.checks.find((check) => check.id === CHECK_IDS[2]);
|
||||
assert.equal(failed.status, "FAIL");
|
||||
const roots = await readFile(join(result.runRoot, "ownership.json"), "utf8");
|
||||
assert.match(roots, /p3-acceptance/);
|
||||
});
|
||||
@@ -0,0 +1,403 @@
|
||||
#!/usr/bin/env node
|
||||
// P4 automated integration acceptance: Qdrant collection lifecycle (self-heal + guarded rebuild).
|
||||
import { createHash, randomBytes } from "node:crypto";
|
||||
import { execFile, execFileSync } from "node:child_process";
|
||||
import { promisify } from "node:util";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { existsSync, lstatSync, mkdirSync, readFileSync, readdirSync, realpathSync, rmSync, statSync, writeFileSync } from "node:fs";
|
||||
import { mkdir, readFile, rm, writeFile } from "node:fs/promises";
|
||||
import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
|
||||
import { createServer as createNetServer } from "node:net";
|
||||
import process from "node:process";
|
||||
|
||||
import { stringify as yamlStringify } from "yaml";
|
||||
|
||||
import { buildSafeEnvironment, deriveOverall, scanSecrets } from "./p1-acceptance.mjs";
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
const modulePath = fileURLToPath(import.meta.url);
|
||||
const defaultRepositoryRoot = realpathSync(resolve(dirname(modulePath), "../.."));
|
||||
const RUN_ID = /^p4-[0-9a-f]{32}$/;
|
||||
const HEX64 = /^[0-9a-f]{64}$/;
|
||||
const QDRANT_IMAGE = "qdrant/qdrant:v1.18.2";
|
||||
export const CHECK_IDS = Object.freeze([
|
||||
"preflight",
|
||||
"clean_state",
|
||||
"ownership",
|
||||
"qdrant_up",
|
||||
"self_heal_create_missing",
|
||||
"self_heal_repairs_missing_index",
|
||||
"incompatible_refused",
|
||||
"require_existing_refused",
|
||||
"rebuild_recreates_contract",
|
||||
"secret_scan",
|
||||
"cleanup_confinement",
|
||||
]);
|
||||
const TOPOLOGY = ["installation", "fixtures", "logs", "qdrant-volumes"];
|
||||
const MAX_REPORT_JSON_BYTES = 64 * 1024;
|
||||
const MAX_REPORT_MD_BYTES = 32 * 1024;
|
||||
|
||||
|
||||
function resolveSystemExecutable(name) {
|
||||
for (const candidate of [`/usr/bin/${name}`, `/bin/${name}`, `/opt/homebrew/bin/${name}`, `/usr/local/bin/${name}`, `/usr/local/sbin/${name}`]) {
|
||||
try {
|
||||
const resolved = realpathSync(candidate);
|
||||
if (statSync(resolved).isFile()) return resolved;
|
||||
} catch { /* continue */ }
|
||||
}
|
||||
throw new Error(`required executable ${name} is unavailable`);
|
||||
}
|
||||
const DOCKER_BIN = (() => { try { return resolveSystemExecutable("docker"); } catch { return "docker"; } })();
|
||||
|
||||
function nowIso() { return new Date().toISOString(); }
|
||||
function sha256(value) { return createHash("sha256").update(value).digest("hex"); }
|
||||
function assert(condition, message) { if (!condition) throw new Error(message); }
|
||||
function sleep(ms) { return new Promise((resolve) => setTimeout(resolve, ms)); }
|
||||
|
||||
function canonicalRoot(repositoryRoot = defaultRepositoryRoot) {
|
||||
return realpathSync(repositoryRoot);
|
||||
}
|
||||
export function canonicalIntegrationBase(repositoryRoot = defaultRepositoryRoot) {
|
||||
return join(canonicalRoot(repositoryRoot), ".artifacts", "p4-integration");
|
||||
}
|
||||
export function validateRunRoot(repositoryRoot, runRoot, runId) {
|
||||
if (!RUN_ID.test(runId)) throw new Error("invalid owned run id");
|
||||
const base = canonicalIntegrationBase(repositoryRoot);
|
||||
const lexical = resolve(runRoot);
|
||||
if (dirname(lexical) !== base || basename(lexical) !== runId) throw new Error("run root is not a direct integration child");
|
||||
return lexical;
|
||||
}
|
||||
function validateNoSymlinkAncestors(repositoryRoot, target) {
|
||||
const repo = canonicalRoot(repositoryRoot);
|
||||
const rel = relative(repo, target);
|
||||
if (rel.startsWith("..") || isAbsolute(rel)) throw new Error("target escapes the repository");
|
||||
let cursor = repo;
|
||||
for (const part of rel.split(sep)) {
|
||||
cursor = join(cursor, part);
|
||||
if (existsSync(cursor) && lstatSyncIsSymlink(cursor)) throw new Error(`symlink ancestor: ${cursor}`);
|
||||
}
|
||||
}
|
||||
function lstatSyncIsSymlink(path) { return lstatSync(path).isSymbolicLink(); }
|
||||
|
||||
export function createOwnedRun(repositoryRoot, nonce = randomBytes(16).toString("hex")) {
|
||||
const runId = `p4-${nonce}`;
|
||||
if (!RUN_ID.test(runId)) throw new Error("invalid run id");
|
||||
const base = canonicalIntegrationBase(repositoryRoot);
|
||||
mkdirSync(base, { recursive: true });
|
||||
const runRoot = join(base, runId);
|
||||
validateNoSymlinkAncestors(repositoryRoot, runRoot);
|
||||
mkdirSync(join(runRoot, "installation"), { recursive: true });
|
||||
mkdirSync(join(runRoot, "fixtures"), { recursive: true });
|
||||
mkdirSync(join(runRoot, "logs"), { recursive: true });
|
||||
mkdirSync(join(runRoot, "qdrant-volumes"), { recursive: true });
|
||||
const marker = { runId, createdAt: nowIso(), repositoryRoot: canonicalRoot(repositoryRoot), sha256: "" };
|
||||
marker.sha256 = sha256(JSON.stringify(marker) + "\n");
|
||||
writeFileSync(join(runRoot, "run.json"), JSON.stringify(marker, null, 2) + "\n", { mode: 0o600 });
|
||||
return { runId, runRoot };
|
||||
}
|
||||
|
||||
export function cleanupOwnedRun(repositoryRoot, runRoot, runId) {
|
||||
const validated = validateRunRoot(repositoryRoot, runRoot, runId);
|
||||
const base = canonicalIntegrationBase(repositoryRoot);
|
||||
for (const sibling of readdirSync(base)) {
|
||||
if (sibling.startsWith("p4-") && sibling !== runId) throw new Error("refusing cleanup with sibling p4 runs present");
|
||||
}
|
||||
rmSync(validated, { recursive: true, force: true });
|
||||
}
|
||||
|
||||
|
||||
function result(checkId, ok, detail, cause) {
|
||||
const message = cause ? `${String(detail)} :: ${String(cause)}` : String(detail);
|
||||
return { checkId, status: ok ? "PASS" : "FAIL", ok: !!ok, detail: ok ? "PASS" : message.slice(0, 500) };
|
||||
}
|
||||
|
||||
function execCapture(command, args, options = {}) {
|
||||
const spawned = execFileSync(command, args, { encoding: "utf8", maxBuffer: 64 * 1024 * 1024, ...options });
|
||||
return String(spawned ?? "");
|
||||
}
|
||||
|
||||
async function waitForQdrant(baseUrl, timeoutMs = 120000) {
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
while (Date.now() < deadline) {
|
||||
try {
|
||||
const res = await fetch(`${baseUrl}/readyz`, { signal: AbortSignal.timeout(3000) });
|
||||
if (res.ok) return true;
|
||||
} catch { /* retry */ }
|
||||
await sleep(1500);
|
||||
}
|
||||
throw new Error("qdrant did not become ready");
|
||||
}
|
||||
|
||||
async function qdrantGet(baseUrl, path) {
|
||||
const res = await fetch(`${baseUrl}${path}`);
|
||||
if (!res.ok) throw new Error(`qdrant GET ${path} -> ${res.status}`);
|
||||
return (await res.json()).result;
|
||||
}
|
||||
async function qdrantPut(baseUrl, path, body) {
|
||||
const payload = { ...body };
|
||||
if (payload.vectors && typeof payload.vectors.distance === "string" && payload.vectors.distance.length > 0) {
|
||||
payload.vectors = { ...payload.vectors, distance: payload.vectors.distance.charAt(0).toUpperCase() + payload.vectors.distance.slice(1) };
|
||||
}
|
||||
const res = await fetch(`${baseUrl}${path}`, {
|
||||
method: "PUT",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify(payload),
|
||||
});
|
||||
if (!res.ok && res.status !== 409) throw new Error(`qdrant PUT ${path} -> ${res.status}`);
|
||||
return res.ok || res.status === 409;
|
||||
}
|
||||
async function qdrantDelete(baseUrl, path) {
|
||||
const res = await fetch(`${baseUrl}${path}`, { method: "DELETE" });
|
||||
if (!res.ok && res.status !== 404) throw new Error(`qdrant DELETE ${path} -> ${res.status}`);
|
||||
}
|
||||
|
||||
function contractOk(info, dimensions, distance) {
|
||||
const vectors = info?.config?.params?.vectors;
|
||||
const schema = info?.payload_schema;
|
||||
const required = ["content_hash","document_id","kind","record_key","record_kind","vector_generation","workspace_id","workspace_revision"];
|
||||
if (!vectors || vectors.size !== dimensions || String(vectors.distance).toLowerCase() !== distance) return false;
|
||||
if (!schema || typeof schema !== "object") return false;
|
||||
return required.every((field) => schema[field]?.data_type === "keyword");
|
||||
}
|
||||
|
||||
async function runIntegration(repositoryRoot, runRoot, runId, qdrantBaseUrl) {
|
||||
const checks = [];
|
||||
const record = (checkId, fn) => checks.push(async () => {
|
||||
try { return result(checkId, await fn()); }
|
||||
catch (error) { return result(checkId, false, error.message, error.cause?.message ?? error.code); }
|
||||
});
|
||||
const ctx = { run: { root: runRoot, id: runId }, repo: repositoryRoot };
|
||||
|
||||
record("preflight", async () => {
|
||||
execCapture(DOCKER_BIN, ["version", "--format", "{{.Server.Version}}"]);
|
||||
execCapture("node", ["--version"]);
|
||||
execCapture("npm", ["--version"]);
|
||||
return true;
|
||||
});
|
||||
|
||||
record("clean_state", async () => {
|
||||
const base = canonicalIntegrationBase(repositoryRoot);
|
||||
const leftovers = readdirSync(base).filter((entry) => entry.startsWith("p4-") && entry !== runId);
|
||||
if (leftovers.length > 0) throw new Error(`leftover p4 runs: ${leftovers.join(", ")}`);
|
||||
return true;
|
||||
});
|
||||
|
||||
record("ownership", async () => {
|
||||
const marker = JSON.parse(await readFile(join(runRoot, "run.json"), "utf8"));
|
||||
if (marker.runId !== runId) throw new Error("run marker mismatch");
|
||||
return true;
|
||||
});
|
||||
|
||||
const containerName = `p4acc-qdrant-${runId.slice(3, 11)}`;
|
||||
let started = false;
|
||||
const startQdrant = async () => {
|
||||
await execFileAsync(DOCKER_BIN, ["rm", "-f", containerName], { stdio: "ignore" }).catch(() => {});
|
||||
const hostPort = await freePort();
|
||||
try {
|
||||
await execFileAsync(DOCKER_BIN, ["run", "-d", "--name", containerName,
|
||||
"-p", `127.0.0.1:${hostPort}:6333`, "-v", `${containerName}-vol:/qdrant/storage`,
|
||||
"--restart", "no", QDRANT_IMAGE], { stdio: "ignore" });
|
||||
} catch (error) {
|
||||
const detail = error.stderr ?? error.message;
|
||||
throw new Error(`docker run qdrant failed: ${String(detail).slice(0, 300)}`);
|
||||
}
|
||||
started = true;
|
||||
return `http://127.0.0.1:${hostPort}`;
|
||||
};
|
||||
const stopQdrant = async () => {
|
||||
if (!started) return;
|
||||
try {
|
||||
const logs = await execFileAsync(DOCKER_BIN, ["logs", containerName]);
|
||||
const insp = await execFileAsync(DOCKER_BIN, ["inspect", "--format", "{{.State.Status}} exit={{.State.ExitCode}} oom={{.State.OOMKilled}}", containerName]).catch(() => ({ stdout: "inspect failed" }));
|
||||
await writeFile(join(runRoot, "qdrant.log"), `INSPECT: ${String(insp.stdout).trim()}\n` + String(logs.stdout).slice(-3000) + "\n---STDERR---\n" + String(logs.stderr).slice(-3000));
|
||||
} catch { /* best effort */ }
|
||||
await execFileAsync(DOCKER_BIN, ["rm", "-f", containerName], { stdio: "ignore" }).catch(() => {});
|
||||
await execFileAsync(DOCKER_BIN, ["volume", "rm", "-f", `${containerName}-vol`], { stdio: "ignore" }).catch(() => {});
|
||||
};
|
||||
|
||||
|
||||
|
||||
function freePort() {
|
||||
return new Promise((resolve, reject) => {
|
||||
const server = createNetServer();
|
||||
server.unref();
|
||||
server.on("error", reject);
|
||||
server.listen(0, "127.0.0.1", () => {
|
||||
const port = server.address().port;
|
||||
server.close(() => resolve(port));
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async function dockerPortRetry(containerName, attempts = 20) {
|
||||
for (let attempt = 0; attempt < attempts; attempt += 1) {
|
||||
try {
|
||||
const inspect = await execFileAsync(DOCKER_BIN, ["port", containerName, "6333"]);
|
||||
const line = String(inspect.stdout).trim();
|
||||
const hostPort = line.split("\n")[0].split(":")[1];
|
||||
if (hostPort) return `http://127.0.0.1:${hostPort}`;
|
||||
} catch { /* transient */ }
|
||||
await sleep(1000);
|
||||
}
|
||||
throw new Error(`docker port ${containerName} did not resolve`);
|
||||
}
|
||||
|
||||
let manager;
|
||||
try {
|
||||
const qdrantUrl = await startQdrant();
|
||||
await waitForQdrant(qdrantUrl);
|
||||
await sleep(2000);
|
||||
record("qdrant_up", async () => true);
|
||||
|
||||
const { reconcileCollection } = await import(new URL(`file://${join(repositoryRoot, "backend", "dist", "workspaces", "qdrant-collection.js")}`).href);
|
||||
const REQ = ["content_hash","document_id","kind","record_key","record_kind","vector_generation","workspace_id","workspace_revision"];
|
||||
|
||||
record("self_heal_create_missing", () => retryCheck(async () => {
|
||||
const collection = `p4-create-${runId.slice(3, 11)}`;
|
||||
const outcome = await reconcileCollection({ baseUrl: qdrantUrl, collection, dimensions: 1024, distance: "cosine", mode: "self_heal" });
|
||||
if (!outcome.ok) throw new Error(`unexpected ${outcome.code}`);
|
||||
const info = await qdrantGet(qdrantUrl, `/collections/${collection}`);
|
||||
if (!contractOk(info, 1024, "cosine")) throw new Error("created contract mismatch");
|
||||
return true;
|
||||
}));
|
||||
|
||||
record("self_heal_repairs_missing_index", () => retryCheck(async () => {
|
||||
const collection = `p4-repair-${runId.slice(3, 11)}`;
|
||||
await qdrantPut(qdrantUrl, `/collections/${collection}`, { vectors: { size: 1024, distance: "cosine" } });
|
||||
const outcome = await reconcileCollection({ baseUrl: qdrantUrl, collection, dimensions: 1024, distance: "cosine", mode: "self_heal" });
|
||||
if (outcome.ok !== true || outcome.state !== "repaired") throw new Error(`expected repaired, got ${JSON.stringify(outcome)}`);
|
||||
const info = await qdrantGet(qdrantUrl, `/collections/${collection}`);
|
||||
if (!contractOk(info, 1024, "cosine")) throw new Error("repaired contract mismatch");
|
||||
return true;
|
||||
}));
|
||||
|
||||
record("incompatible_refused", () => retryCheck(async () => {
|
||||
const collection = `p4-bad-${runId.slice(3, 11)}`;
|
||||
await qdrantPut(qdrantUrl, `/collections/${collection}`, { vectors: { size: 768, distance: "cosine" } });
|
||||
const before = await qdrantGet(qdrantUrl, `/collections/${collection}`);
|
||||
const outcome = await reconcileCollection({ baseUrl: qdrantUrl, collection, dimensions: 1024, distance: "cosine", mode: "self_heal" });
|
||||
if (outcome.ok !== false || outcome.code !== "semantic_index_incompatible") throw new Error(`expected incompatible, got ${JSON.stringify(outcome)}`);
|
||||
const after = await qdrantGet(qdrantUrl, `/collections/${collection}`);
|
||||
if (JSON.stringify(before) !== JSON.stringify(after)) throw new Error("incompatible collection was mutated");
|
||||
return true;
|
||||
}));
|
||||
|
||||
record("require_existing_refused", () => retryCheck(async () => {
|
||||
const collection = `p4-missing-${runId.slice(3, 11)}`;
|
||||
const outcome = await reconcileCollection({ baseUrl: qdrantUrl, collection, dimensions: 1024, distance: "cosine", mode: "require_existing" });
|
||||
if (outcome.ok !== false || outcome.code !== "semantic_index_incompatible") throw new Error(`expected incompatible, got ${JSON.stringify(outcome)}`);
|
||||
const info = await qdrantGet(qdrantUrl, `/collections/${collection}`).catch(() => undefined);
|
||||
if (info !== undefined) throw new Error("require_existing created a collection");
|
||||
return true;
|
||||
}));
|
||||
|
||||
record("rebuild_recreates_contract", () => retryCheck(async () => {
|
||||
const collection = `p4-rebuild-${runId.slice(3, 11)}`;
|
||||
await qdrantPut(qdrantUrl, `/collections/${collection}`, { vectors: { size: 1024, distance: "cosine" } });
|
||||
await qdrantDelete(qdrantUrl, `/collections/${collection}`);
|
||||
const info = await qdrantGet(qdrantUrl, `/collections/${collection}`).catch(() => undefined);
|
||||
if (info !== undefined) throw new Error("rebuild did not delete the collection");
|
||||
await qdrantPut(qdrantUrl, `/collections/${collection}`, { vectors: { size: 1024, distance: "cosine" } });
|
||||
const outcome = await reconcileCollection({ baseUrl: qdrantUrl, collection, dimensions: 1024, distance: "cosine", mode: "self_heal" });
|
||||
if (!outcome.ok) throw new Error(`recreate verify failed ${JSON.stringify(outcome)}`);
|
||||
const recreated = await qdrantGet(qdrantUrl, `/collections/${collection}`);
|
||||
if (!contractOk(recreated, 1024, "cosine")) throw new Error("recreated contract mismatch");
|
||||
return true;
|
||||
}));
|
||||
|
||||
record("secret_scan", async () => {
|
||||
const secretValues = ["p4-acceptance"];
|
||||
const findings = await scanSecrets({ runRoot, forbiddenValues: secretValues, expectedGitRepositories: [] });
|
||||
if (findings.length > 0) throw new Error(`secret findings: ${findings.join(", ")}`);
|
||||
return true;
|
||||
});
|
||||
|
||||
record("cleanup_confinement", async () => {
|
||||
const base = canonicalIntegrationBase(repositoryRoot);
|
||||
const direct = readdirSync(base).filter((entry) => entry.startsWith("p4-"));
|
||||
if (direct.length !== 1 || direct[0] !== runId) throw new Error("run confinement violated");
|
||||
return true;
|
||||
});
|
||||
const settledChecks = await runChecks(checks);
|
||||
return settledChecks;
|
||||
} finally {
|
||||
await stopQdrant();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
async function retryCheck(fn, attempts = 3) {
|
||||
let lastError;
|
||||
for (let attempt = 0; attempt < attempts; attempt += 1) {
|
||||
try { return await fn(); } catch (error) { lastError = error; await sleep(3000); }
|
||||
}
|
||||
try {
|
||||
const ps = await execFileAsync(DOCKER_BIN, ["ps", "-a", "--filter", "name=p4acc-qdrant", "--format", "{{.Names}} {{.Status}} {{.Ports}}"]);
|
||||
lastError = new Error(`${lastError.message} | containers: ${String(ps.stdout).trim()}`);
|
||||
} catch { /* best effort */ }
|
||||
throw lastError;
|
||||
}
|
||||
|
||||
async function runChecks(checks) {
|
||||
const settled = [];
|
||||
for (const check of checks) settled.push(await check());
|
||||
return settled;
|
||||
}
|
||||
|
||||
export async function runAcceptance({ repositoryRoot = defaultRepositoryRoot, keep = false } = {}) {
|
||||
const nonce = randomBytes(16).toString("hex");
|
||||
const { runId, runRoot } = createOwnedRun(repositoryRoot, nonce);
|
||||
const reportDir = join(runRoot, "report.md");
|
||||
const reportJsonDir = join(runRoot, "report.json");
|
||||
try {
|
||||
await execFileAsync("npm", ["--prefix", join(repositoryRoot, "backend"), "run", "build"], { stdio: "ignore" });
|
||||
const checks = await runIntegration(repositoryRoot, runRoot, runId, "");
|
||||
const overall = deriveOverall(checks);
|
||||
const summary = {
|
||||
schemaVersion: 1,
|
||||
runId,
|
||||
phase: "p4",
|
||||
checks,
|
||||
overall,
|
||||
boundCommit: execCapture("git", ["rev-parse", "HEAD"], { cwd: repositoryRoot }).trim(),
|
||||
};
|
||||
await writeFile(reportJsonDir, JSON.stringify(summary, null, 2) + "\n");
|
||||
const rows = checks.map((c) => `- [${c.ok ? "x" : " "}] ${c.checkId}: ${c.detail}`).join("\n");
|
||||
await writeFile(reportDir, `# P4 automated integration acceptance\n\n- run: \`${runId}\`\n- committed: \`${summary.boundCommit}\`\n\n${rows}\n\n**Overall: ${overall}**\n`);
|
||||
if (overall === "PASS") {
|
||||
if (!keep) cleanupOwnedRun(repositoryRoot, runRoot, runId);
|
||||
return { ok: true, runId, reportPath: reportDir, overall };
|
||||
}
|
||||
if (!keep) {
|
||||
try {
|
||||
const validated = validateRunRoot(repositoryRoot, runRoot, runId);
|
||||
rmSync(validated, { recursive: true, force: true });
|
||||
} catch { /* best effort */ }
|
||||
}
|
||||
return { ok: false, runId, reportPath: reportDir, overall };
|
||||
} catch (error) {
|
||||
try {
|
||||
const partial = { schemaVersion: 1, runId, phase: "p4", checks: [], overall: "FAIL", error: String(error).slice(0, 500) };
|
||||
await writeFile(reportJsonDir, JSON.stringify(partial, null, 2) + "\n");
|
||||
await writeFile(reportDir, `# P4 automated integration acceptance\n\n- run: \`${runId}\`\n- error: \`${String(error).slice(0, 500)}\`\n\n**Overall: FAIL**\n`);
|
||||
} catch { /* best effort */ }
|
||||
if (keep) return { ok: false, runId, reportPath: reportDir, overall: "FAIL" };
|
||||
try {
|
||||
const validated = validateRunRoot(repositoryRoot, runRoot, runId);
|
||||
rmSync(validated, { recursive: true, force: true });
|
||||
} catch { /* best effort */ }
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
if (import.meta.url === `file://${process.argv[1]}`) {
|
||||
const args = process.argv.slice(2);
|
||||
const keep = args.includes("--keep");
|
||||
runAcceptance({ keep }).then((outcome) => {
|
||||
process.stdout.write(`P4 automated integration: ${outcome.overall}\nrun: ${outcome.runId}\nreport: ${outcome.reportPath}\n`);
|
||||
process.exit(outcome.ok ? 0 : 1);
|
||||
}).catch((error) => {
|
||||
process.stderr.write(`P4 automated integration: FAIL\n${String(error)}\n`);
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdir, mkdtemp, readFile, rm } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { dirname, join } from "node:path";
|
||||
import test from "node:test";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
import {
|
||||
CHECK_IDS,
|
||||
canonicalIntegrationBase,
|
||||
cleanupOwnedRun,
|
||||
createOwnedRun,
|
||||
validateRunRoot,
|
||||
} from "./p4-acceptance.mjs";
|
||||
|
||||
const roots = [];
|
||||
async function fakeRepository() {
|
||||
const root = await mkdtemp(join(tmpdir(), "p4-acceptance-repo-"));
|
||||
roots.push(root);
|
||||
await mkdir(join(root, ".artifacts", "p4-integration"), { recursive: true });
|
||||
return root;
|
||||
}
|
||||
|
||||
test.afterEach(async () => {
|
||||
await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true })));
|
||||
});
|
||||
|
||||
test("check ids are stable and unique", () => {
|
||||
assert.equal(new Set(CHECK_IDS).size, CHECK_IDS.length);
|
||||
assert.ok(CHECK_IDS.includes("self_heal_create_missing"));
|
||||
assert.ok(CHECK_IDS.includes("rebuild_recreates_contract"));
|
||||
});
|
||||
|
||||
test("run roots are only canonical direct p4 integration children", async () => {
|
||||
const repositoryRoot = await fakeRepository();
|
||||
const base = canonicalIntegrationBase(repositoryRoot);
|
||||
const id = `p4-${"a".repeat(32)}`;
|
||||
assert.equal(validateRunRoot(repositoryRoot, join(base, id), id), join(base, id));
|
||||
for (const candidate of [base, join(repositoryRoot, ".artifacts", "p1-integration", id), join(base, id, "nested")]) {
|
||||
assert.throws(() => validateRunRoot(repositoryRoot, candidate, id));
|
||||
}
|
||||
assert.throws(() => validateRunRoot(repositoryRoot, join(base, `p4-${"A".repeat(32)}`), `p4-${"A".repeat(32)}`));
|
||||
});
|
||||
|
||||
test("createOwnedRun writes a canonical marker and cleanup refuses foreign roots", async () => {
|
||||
const repositoryRoot = await fakeRepository();
|
||||
const { runId, runRoot } = createOwnedRun(repositoryRoot);
|
||||
assert.match(runId, /^p4-[0-9a-f]{32}$/);
|
||||
const marker = JSON.parse(await readFile(join(runRoot, "run.json"), "utf8"));
|
||||
assert.equal(marker.runId, runId);
|
||||
assert.throws(() => cleanupOwnedRun(repositoryRoot, join(repositoryRoot, "tmp"), runId));
|
||||
cleanupOwnedRun(repositoryRoot, runRoot, runId);
|
||||
});
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,158 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdir, mkdtemp, readFile, rm, stat, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { dirname, join } from "node:path";
|
||||
import test from "node:test";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
import {
|
||||
CHECK_IDS,
|
||||
canonicalIntegrationBase,
|
||||
cleanupOwnedRun,
|
||||
createOwnedRun,
|
||||
readAndValidateOwnership,
|
||||
runIntegration,
|
||||
validateReport,
|
||||
validateRunRoot,
|
||||
} from "./p5-acceptance.mjs";
|
||||
|
||||
const roots = [];
|
||||
async function fakeRepository() {
|
||||
const root = await mkdtemp(join(tmpdir(), "p5-acceptance-repo-"));
|
||||
roots.push(root);
|
||||
await mkdir(join(root, ".artifacts", "p5-integration"), { recursive: true });
|
||||
await mkdir(join(root, ".artifacts", "p2-integration"), { recursive: true });
|
||||
await mkdir(join(root, ".artifacts", "p1-integration"), { recursive: true });
|
||||
await mkdir(join(root, ".artifacts", "manual-acceptance", "p11"), { recursive: true });
|
||||
return root;
|
||||
}
|
||||
|
||||
test.afterEach(async () => {
|
||||
await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true })));
|
||||
});
|
||||
|
||||
test("run roots are only canonical direct p5 integration children", async () => {
|
||||
const repositoryRoot = await fakeRepository();
|
||||
const base = canonicalIntegrationBase(repositoryRoot);
|
||||
const id = `p5-${"a".repeat(32)}`;
|
||||
assert.equal(validateRunRoot(repositoryRoot, join(base, id), id), join(base, id));
|
||||
for (const candidate of [
|
||||
base,
|
||||
join(repositoryRoot, ".artifacts", "manual-acceptance", "p11"),
|
||||
join(repositoryRoot, ".artifacts", "p1-integration", id),
|
||||
join(repositoryRoot, ".artifacts", "p2-integration", id),
|
||||
join(base, id, "nested"),
|
||||
join(base, "foreign"),
|
||||
]) {
|
||||
assert.throws(() => validateRunRoot(repositoryRoot, candidate, id));
|
||||
}
|
||||
assert.throws(() => validateRunRoot(repositoryRoot, join(base, `p5-${"A".repeat(32)}`), `p5-${"A".repeat(32)}`));
|
||||
});
|
||||
|
||||
test("cleanup refuses p1, p2, p11, manual, sibling, and wrong-nonce roots", async () => {
|
||||
const repositoryRoot = await fakeRepository();
|
||||
const run = await createOwnedRun({ repositoryRoot });
|
||||
await readAndValidateOwnership({ repositoryRoot, runRoot: run.root, expectedNonce: run.nonce });
|
||||
for (const bad of [
|
||||
join(repositoryRoot, ".artifacts", "p1-integration", `p1-${"b".repeat(32)}`),
|
||||
join(repositoryRoot, ".artifacts", "p2-integration", `p2-${"c".repeat(32)}`),
|
||||
join(repositoryRoot, ".artifacts", "manual-acceptance", "p11"),
|
||||
join(canonicalIntegrationBase(repositoryRoot), `p5-${"d".repeat(32)}`),
|
||||
]) {
|
||||
await assert.rejects(cleanupOwnedRun({ repositoryRoot, runRoot: bad, expectedNonce: run.nonce }));
|
||||
}
|
||||
await assert.rejects(cleanupOwnedRun({ repositoryRoot, runRoot: run.root, expectedNonce: "0".repeat(64) }));
|
||||
});
|
||||
|
||||
test("cleanup removes exactly one owned p5 root", async () => {
|
||||
const repositoryRoot = await fakeRepository();
|
||||
const run = await createOwnedRun({ repositoryRoot });
|
||||
const sibling = join(canonicalIntegrationBase(repositoryRoot), `p5-${"e".repeat(32)}`);
|
||||
await mkdir(sibling);
|
||||
await writeFile(join(sibling, "sentinel"), "foreign");
|
||||
await cleanupOwnedRun({ repositoryRoot, runRoot: run.root, expectedNonce: run.nonce });
|
||||
await assert.rejects(readFile(join(run.root, "ownership.json")));
|
||||
assert.equal(await readFile(join(sibling, "sentinel"), "utf8"), "foreign");
|
||||
});
|
||||
|
||||
function resultFor(id) {
|
||||
return {
|
||||
id,
|
||||
status: "PASS",
|
||||
startedAt: "2026-08-12T00:00:00.000Z",
|
||||
finishedAt: "2026-08-12T00:00:01.000Z",
|
||||
commands: ["node"],
|
||||
artifacts: [{ path: `logs/${id}.json`, sha256: "a".repeat(64) }],
|
||||
};
|
||||
}
|
||||
|
||||
test("report validation requires exact p5 identity, check order, and unique artifacts", () => {
|
||||
const report = {
|
||||
schemaVersion: 1,
|
||||
runId: `p5-${"f".repeat(32)}`,
|
||||
startedAt: "2026-08-12T00:00:00.000Z",
|
||||
finishedAt: "2026-08-12T00:00:10.000Z",
|
||||
command: "p5-acceptance integration --keep",
|
||||
overall: "PASS",
|
||||
checks: CHECK_IDS.map(resultFor),
|
||||
};
|
||||
assert.doesNotThrow(() => validateReport(report));
|
||||
const invalid = structuredClone(report);
|
||||
invalid.runId = `p2-${"f".repeat(32)}`;
|
||||
assert.throws(() => validateReport(invalid));
|
||||
const duplicate = structuredClone(report);
|
||||
duplicate.checks[1].artifacts[0].path = duplicate.checks[0].artifacts[0].path;
|
||||
assert.throws(() => validateReport(duplicate), /duplicated/);
|
||||
const reordered = structuredClone(report);
|
||||
reordered.checks.reverse();
|
||||
reordered.overall = "FAIL";
|
||||
assert.throws(() => validateReport(reordered));
|
||||
});
|
||||
|
||||
test("public wrapper uses a strict empty environment", async () => {
|
||||
const wrapper = await readFile(join(dirname(fileURLToPath(import.meta.url)), "..", "..", "scripts", "p5-acceptance.sh"), "utf8");
|
||||
assert.match(wrapper, /safe_env=\(\/usr\/bin\/env -i/);
|
||||
assert.doesNotMatch(wrapper, /LANG|LC_ALL|TZ/);
|
||||
assert.doesNotMatch(wrapper, /P5_ACCEPTANCE_FAIL_AT/);
|
||||
});
|
||||
|
||||
test("synthetic integration cleans up successful non-kept runs", async () => {
|
||||
const repositoryRoot = await fakeRepository();
|
||||
const result = await runIntegration({ repositoryRoot, keep: false, env: { P5_ACCEPTANCE_SYNTHETIC: "1" } });
|
||||
assert.equal(result.exitCode, 0);
|
||||
assert.equal(result.retained, false);
|
||||
await assert.rejects(readFile(join(result.runRoot, "ownership.json")));
|
||||
});
|
||||
|
||||
test("synthetic integration retains kept runs with bounded reports", async () => {
|
||||
const repositoryRoot = await fakeRepository();
|
||||
const result = await runIntegration({ repositoryRoot, keep: true, env: { P5_ACCEPTANCE_SYNTHETIC: "1" } });
|
||||
assert.equal(result.exitCode, 0);
|
||||
assert.equal(result.retained, true);
|
||||
const report = JSON.parse(await readFile(join(result.runRoot, "report.json"), "utf8"));
|
||||
assert.equal(report.overall, "PASS");
|
||||
const reportMd = await readFile(join(result.runRoot, "report.md"), "utf8");
|
||||
assert.match(reportMd, /P5 automated integration: PASS/);
|
||||
assert.match(reportMd, /P5 manual acceptance: PENDING/);
|
||||
const reportJsonStat = await stat(join(result.runRoot, "report.json"));
|
||||
const reportMdStat = await stat(join(result.runRoot, "report.md"));
|
||||
assert.ok(reportJsonStat.size <= 64 * 1024, `report.json too large: ${reportJsonStat.size}`);
|
||||
assert.ok(reportMdStat.size <= 32 * 1024, `report.md too large: ${reportMdStat.size}`);
|
||||
});
|
||||
|
||||
test("synthetic injected failure retains the owned run and records a single failed report", async () => {
|
||||
const repositoryRoot = await fakeRepository();
|
||||
const result = await runIntegration({
|
||||
repositoryRoot,
|
||||
keep: false,
|
||||
env: { P5_ACCEPTANCE_SYNTHETIC: "1", P5_ACCEPTANCE_FAIL_AT: CHECK_IDS[2] },
|
||||
});
|
||||
assert.equal(result.exitCode, 1);
|
||||
assert.equal(result.retained, true);
|
||||
const report = JSON.parse(await readFile(join(result.runRoot, "report.json"), "utf8"));
|
||||
assert.equal(report.overall, "FAIL");
|
||||
const failed = report.checks.find((check) => check.id === CHECK_IDS[2]);
|
||||
assert.equal(failed.status, "FAIL");
|
||||
const roots = await readFile(join(result.runRoot, "ownership.json"), "utf8");
|
||||
assert.match(roots, /p5-acceptance/);
|
||||
});
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,158 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdir, mkdtemp, readFile, rm, stat, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { dirname, join } from "node:path";
|
||||
import test from "node:test";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
import {
|
||||
CHECK_IDS,
|
||||
canonicalIntegrationBase,
|
||||
cleanupOwnedRun,
|
||||
createOwnedRun,
|
||||
readAndValidateOwnership,
|
||||
runIntegration,
|
||||
validateReport,
|
||||
validateRunRoot,
|
||||
} from "./p6-acceptance.mjs";
|
||||
|
||||
const roots = [];
|
||||
async function fakeRepository() {
|
||||
const root = await mkdtemp(join(tmpdir(), "p6-acceptance-repo-"));
|
||||
roots.push(root);
|
||||
await mkdir(join(root, ".artifacts", "p6-integration"), { recursive: true });
|
||||
await mkdir(join(root, ".artifacts", "p2-integration"), { recursive: true });
|
||||
await mkdir(join(root, ".artifacts", "p1-integration"), { recursive: true });
|
||||
await mkdir(join(root, ".artifacts", "manual-acceptance", "p11"), { recursive: true });
|
||||
return root;
|
||||
}
|
||||
|
||||
test.afterEach(async () => {
|
||||
await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true })));
|
||||
});
|
||||
|
||||
test("run roots are only canonical direct p6 integration children", async () => {
|
||||
const repositoryRoot = await fakeRepository();
|
||||
const base = canonicalIntegrationBase(repositoryRoot);
|
||||
const id = `p6-${"a".repeat(32)}`;
|
||||
assert.equal(validateRunRoot(repositoryRoot, join(base, id), id), join(base, id));
|
||||
for (const candidate of [
|
||||
base,
|
||||
join(repositoryRoot, ".artifacts", "manual-acceptance", "p11"),
|
||||
join(repositoryRoot, ".artifacts", "p1-integration", id),
|
||||
join(repositoryRoot, ".artifacts", "p2-integration", id),
|
||||
join(base, id, "nested"),
|
||||
join(base, "foreign"),
|
||||
]) {
|
||||
assert.throws(() => validateRunRoot(repositoryRoot, candidate, id));
|
||||
}
|
||||
assert.throws(() => validateRunRoot(repositoryRoot, join(base, `p6-${"A".repeat(32)}`), `p6-${"A".repeat(32)}`));
|
||||
});
|
||||
|
||||
test("cleanup refuses p1, p2, p11, manual, sibling, and wrong-nonce roots", async () => {
|
||||
const repositoryRoot = await fakeRepository();
|
||||
const run = await createOwnedRun({ repositoryRoot });
|
||||
await readAndValidateOwnership({ repositoryRoot, runRoot: run.root, expectedNonce: run.nonce });
|
||||
for (const bad of [
|
||||
join(repositoryRoot, ".artifacts", "p1-integration", `p1-${"b".repeat(32)}`),
|
||||
join(repositoryRoot, ".artifacts", "p2-integration", `p2-${"c".repeat(32)}`),
|
||||
join(repositoryRoot, ".artifacts", "manual-acceptance", "p11"),
|
||||
join(canonicalIntegrationBase(repositoryRoot), `p6-${"d".repeat(32)}`),
|
||||
]) {
|
||||
await assert.rejects(cleanupOwnedRun({ repositoryRoot, runRoot: bad, expectedNonce: run.nonce }));
|
||||
}
|
||||
await assert.rejects(cleanupOwnedRun({ repositoryRoot, runRoot: run.root, expectedNonce: "0".repeat(64) }));
|
||||
});
|
||||
|
||||
test("cleanup removes exactly one owned p6 root", async () => {
|
||||
const repositoryRoot = await fakeRepository();
|
||||
const run = await createOwnedRun({ repositoryRoot });
|
||||
const sibling = join(canonicalIntegrationBase(repositoryRoot), `p6-${"e".repeat(32)}`);
|
||||
await mkdir(sibling);
|
||||
await writeFile(join(sibling, "sentinel"), "foreign");
|
||||
await cleanupOwnedRun({ repositoryRoot, runRoot: run.root, expectedNonce: run.nonce });
|
||||
await assert.rejects(readFile(join(run.root, "ownership.json")));
|
||||
assert.equal(await readFile(join(sibling, "sentinel"), "utf8"), "foreign");
|
||||
});
|
||||
|
||||
function resultFor(id) {
|
||||
return {
|
||||
id,
|
||||
status: "PASS",
|
||||
startedAt: "2026-08-12T00:00:00.000Z",
|
||||
finishedAt: "2026-08-12T00:00:01.000Z",
|
||||
commands: ["node"],
|
||||
artifacts: [{ path: `logs/${id}.json`, sha256: "a".repeat(64) }],
|
||||
};
|
||||
}
|
||||
|
||||
test("report validation requires exact p6 identity, check order, and unique artifacts", () => {
|
||||
const report = {
|
||||
schemaVersion: 1,
|
||||
runId: `p6-${"f".repeat(32)}`,
|
||||
startedAt: "2026-08-12T00:00:00.000Z",
|
||||
finishedAt: "2026-08-12T00:00:10.000Z",
|
||||
command: "p6-acceptance integration --keep",
|
||||
overall: "PASS",
|
||||
checks: CHECK_IDS.map(resultFor),
|
||||
};
|
||||
assert.doesNotThrow(() => validateReport(report));
|
||||
const invalid = structuredClone(report);
|
||||
invalid.runId = `p2-${"f".repeat(32)}`;
|
||||
assert.throws(() => validateReport(invalid));
|
||||
const duplicate = structuredClone(report);
|
||||
duplicate.checks[1].artifacts[0].path = duplicate.checks[0].artifacts[0].path;
|
||||
assert.throws(() => validateReport(duplicate), /duplicated/);
|
||||
const reordered = structuredClone(report);
|
||||
reordered.checks.reverse();
|
||||
reordered.overall = "FAIL";
|
||||
assert.throws(() => validateReport(reordered));
|
||||
});
|
||||
|
||||
test("public wrapper uses a strict empty environment", async () => {
|
||||
const wrapper = await readFile(join(dirname(fileURLToPath(import.meta.url)), "..", "..", "scripts", "p6-acceptance.sh"), "utf8");
|
||||
assert.match(wrapper, /safe_env=\(\/usr\/bin\/env -i/);
|
||||
assert.doesNotMatch(wrapper, /LANG|LC_ALL|TZ/);
|
||||
assert.doesNotMatch(wrapper, /P6_ACCEPTANCE_FAIL_AT/);
|
||||
});
|
||||
|
||||
test("synthetic integration cleans up successful non-kept runs", async () => {
|
||||
const repositoryRoot = await fakeRepository();
|
||||
const result = await runIntegration({ repositoryRoot, keep: false, env: { P6_ACCEPTANCE_SYNTHETIC: "1" } });
|
||||
assert.equal(result.exitCode, 0);
|
||||
assert.equal(result.retained, false);
|
||||
await assert.rejects(readFile(join(result.runRoot, "ownership.json")));
|
||||
});
|
||||
|
||||
test("synthetic integration retains kept runs with bounded reports", async () => {
|
||||
const repositoryRoot = await fakeRepository();
|
||||
const result = await runIntegration({ repositoryRoot, keep: true, env: { P6_ACCEPTANCE_SYNTHETIC: "1" } });
|
||||
assert.equal(result.exitCode, 0);
|
||||
assert.equal(result.retained, true);
|
||||
const report = JSON.parse(await readFile(join(result.runRoot, "report.json"), "utf8"));
|
||||
assert.equal(report.overall, "PASS");
|
||||
const reportMd = await readFile(join(result.runRoot, "report.md"), "utf8");
|
||||
assert.match(reportMd, /P6 automated integration: PASS/);
|
||||
assert.match(reportMd, /P6 manual acceptance: PENDING/);
|
||||
const reportJsonStat = await stat(join(result.runRoot, "report.json"));
|
||||
const reportMdStat = await stat(join(result.runRoot, "report.md"));
|
||||
assert.ok(reportJsonStat.size <= 64 * 1024, `report.json too large: ${reportJsonStat.size}`);
|
||||
assert.ok(reportMdStat.size <= 32 * 1024, `report.md too large: ${reportMdStat.size}`);
|
||||
});
|
||||
|
||||
test("synthetic injected failure retains the owned run and records a single failed report", async () => {
|
||||
const repositoryRoot = await fakeRepository();
|
||||
const result = await runIntegration({
|
||||
repositoryRoot,
|
||||
keep: false,
|
||||
env: { P6_ACCEPTANCE_SYNTHETIC: "1", P6_ACCEPTANCE_FAIL_AT: CHECK_IDS[2] },
|
||||
});
|
||||
assert.equal(result.exitCode, 1);
|
||||
assert.equal(result.retained, true);
|
||||
const report = JSON.parse(await readFile(join(result.runRoot, "report.json"), "utf8"));
|
||||
assert.equal(report.overall, "FAIL");
|
||||
const failed = report.checks.find((check) => check.id === CHECK_IDS[2]);
|
||||
assert.equal(failed.status, "FAIL");
|
||||
const roots = await readFile(join(result.runRoot, "ownership.json"), "utf8");
|
||||
assert.match(roots, /p6-acceptance/);
|
||||
});
|
||||
@@ -0,0 +1,943 @@
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
import ts from "typescript";
|
||||
import { literalBashHeredocBodyRanges } from "./bash-heredoc.mjs";
|
||||
import { isMap, isScalar, isSeq, parseAllDocuments } from "yaml";
|
||||
|
||||
|
||||
/**
|
||||
* Revision-state absence policy by source dialect.
|
||||
* JS/TS syntax uses the TypeScript parser and YAML structure uses the installed YAML parser.
|
||||
* Shell active consumers are executable code/expansions and jq filter arguments for bare or
|
||||
* path-qualified jq, optionally through command or env. Quoted heredoc bodies are literal.
|
||||
* PowerShell analyzes executable code and nested $() in expandable strings. Python policy is
|
||||
* batched through the isolated stdlib AST helper. jq filters use a bounded path lexer after
|
||||
* shell argv/wrapper resolution. Offset-preserving transformations keep AST spans stable.
|
||||
*/
|
||||
const revisionIdentifiers = new Set(["revision", "workspaceRevision", "selectedWorkspace"]);
|
||||
|
||||
function unwrapExpression(node) {
|
||||
let current = node;
|
||||
while (ts.isParenthesizedExpression(current) || ts.isAsExpression(current) ||
|
||||
ts.isTypeAssertionExpression(current) || ts.isNonNullExpression(current) ||
|
||||
ts.isSatisfiesExpression(current)) {
|
||||
current = current.expression;
|
||||
}
|
||||
return current;
|
||||
}
|
||||
|
||||
function isRevisionName(value, caseInsensitive) {
|
||||
if (typeof value !== "string") return false;
|
||||
if (!caseInsensitive) return revisionIdentifiers.has(value);
|
||||
const lower = value.toLowerCase();
|
||||
return lower === "revision" || lower === "workspacerevision" || lower === "selectedworkspace";
|
||||
}
|
||||
|
||||
function isRevisionExpression(node, caseInsensitive = false) {
|
||||
const unwrapped = unwrapExpression(node);
|
||||
if (ts.isIdentifier(unwrapped)) {
|
||||
const normalized = unwrapped.text.startsWith("$") && !unwrapped.text.startsWith("$$") ? unwrapped.text.slice(1) : unwrapped.text;
|
||||
return isRevisionName(normalized, caseInsensitive);
|
||||
}
|
||||
if (ts.isPropertyAccessExpression(unwrapped)) return isRevisionName(unwrapped.name.text, caseInsensitive);
|
||||
if (ts.isElementAccessExpression(unwrapped) && unwrapped.argumentExpression) {
|
||||
return isRevisionName(staticStringValue(unwrapped.argumentExpression), caseInsensitive);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function staticStringValue(node) {
|
||||
const expression = unwrapExpression(node);
|
||||
if (ts.isStringLiteral(expression) || ts.isNoSubstitutionTemplateLiteral(expression)) return expression.text;
|
||||
if (ts.isTemplateExpression(expression)) {
|
||||
let value = expression.head.text;
|
||||
for (const span of expression.templateSpans) {
|
||||
const part = staticStringValue(span.expression);
|
||||
if (part === undefined) return undefined;
|
||||
value += part + span.literal.text;
|
||||
}
|
||||
return value;
|
||||
}
|
||||
if (ts.isBinaryExpression(expression) && expression.operatorToken.kind === ts.SyntaxKind.PlusToken) {
|
||||
const left = staticStringValue(expression.left);
|
||||
const right = staticStringValue(expression.right);
|
||||
return left === undefined || right === undefined ? undefined : left + right;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function propertyNameText(name, caseInsensitive = false) {
|
||||
if (!name) return undefined;
|
||||
let value;
|
||||
if (ts.isComputedPropertyName(name)) value = staticStringValue(name.expression);
|
||||
else if (ts.isIdentifier(name) || ts.isStringLiteral(name) || ts.isNoSubstitutionTemplateLiteral(name) || ts.isNumericLiteral(name)) value = name.text;
|
||||
else value = staticStringValue(name);
|
||||
return caseInsensitive && typeof value === "string" ? value.toLowerCase() : value;
|
||||
}
|
||||
|
||||
function objectBindingHasState(pattern, caseInsensitive) {
|
||||
return pattern.elements.some((element) => {
|
||||
if (element.dotDotDotToken) return false;
|
||||
return propertyNameText(element.propertyName ?? element.name, caseInsensitive) === "state";
|
||||
});
|
||||
}
|
||||
|
||||
function objectLiteralHasState(object, caseInsensitive) {
|
||||
return object.properties.some((property) =>
|
||||
!ts.isSpreadAssignment(property) && propertyNameText(property.name, caseInsensitive) === "state");
|
||||
}
|
||||
|
||||
function scriptKindFor(path) {
|
||||
const lower = path.toLowerCase();
|
||||
if (lower.endsWith(".tsx")) return ts.ScriptKind.TSX;
|
||||
if (lower.endsWith(".jsx")) return ts.ScriptKind.JSX;
|
||||
if (/\.(?:ts|mts|cts)$/u.test(lower)) return ts.ScriptKind.TS;
|
||||
if (/\.(?:js|mjs|cjs)$/u.test(lower)) return ts.ScriptKind.JS;
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function maskRange(output, source, start, end, keepEnds = false) {
|
||||
for (let cursor = start; cursor < end; cursor += 1) {
|
||||
if (source[cursor] === "\n" || source[cursor] === "\r") continue;
|
||||
if (keepEnds && (cursor === start || cursor === end - 1)) continue;
|
||||
output[cursor] = " ";
|
||||
}
|
||||
}
|
||||
|
||||
function lineEnd(source, start) {
|
||||
const end = source.indexOf("\n", start);
|
||||
return end < 0 ? source.length : end;
|
||||
}
|
||||
|
||||
function quotedEnd(source, start, delimiter, escapes = "\\") {
|
||||
for (let cursor = start + delimiter.length; cursor < source.length; cursor += 1) {
|
||||
if (escapes.includes(source[cursor])) {
|
||||
cursor += 1;
|
||||
continue;
|
||||
}
|
||||
if (source.startsWith(delimiter, cursor)) return cursor + delimiter.length;
|
||||
}
|
||||
return source.length;
|
||||
}
|
||||
|
||||
function balancedEnd(source, openIndex, opener, closer, escapes = "\\`") {
|
||||
let depth = 1;
|
||||
for (let cursor = openIndex + 1; cursor < source.length; cursor += 1) {
|
||||
if (escapes.includes(source[cursor])) {
|
||||
cursor += 1;
|
||||
continue;
|
||||
}
|
||||
if (source[cursor] === "'" || source[cursor] === '"' || source[cursor] === "`") {
|
||||
cursor = quotedEnd(source, cursor, source[cursor], escapes) - 1;
|
||||
continue;
|
||||
}
|
||||
if (source[cursor] === opener) depth += 1;
|
||||
else if (source[cursor] === closer && --depth === 0) return cursor;
|
||||
}
|
||||
return source.length - 1;
|
||||
}
|
||||
|
||||
function restoreMasked(output, offset, masked) {
|
||||
for (let cursor = 0; cursor < masked.length; cursor += 1) output[offset + cursor] = masked[cursor];
|
||||
}
|
||||
|
||||
function exposeDollarSubexpressions(output, source, start, end, dialect) {
|
||||
for (let cursor = start; cursor + 1 < end; cursor += 1) {
|
||||
if (!source.startsWith("$(", cursor) || source[cursor - 1] === "`") continue;
|
||||
const close = balancedEnd(source, cursor + 1, "(", ")");
|
||||
output[cursor] = " ";
|
||||
output[cursor + 1] = "(";
|
||||
restoreMasked(output, cursor + 2, dialect === "shell" ? maskShellSource(source.slice(cursor + 2, close)) : maskPowerShellSource(source.slice(cursor + 2, close)));
|
||||
if (close < source.length) output[close] = ")";
|
||||
cursor = close;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
function shellCommentStart(source, index) {
|
||||
return source[index] === "#" && (index === 0 || /[ \t\r\n;|&()]/u.test(source[index - 1]));
|
||||
}
|
||||
|
||||
function canonicalRevisionName(name) {
|
||||
const lower = name.toLowerCase();
|
||||
if (lower === "revision") return "revision";
|
||||
if (lower === "workspacerevision") return "workspaceRevision";
|
||||
return "selectedWorkspace";
|
||||
}
|
||||
|
||||
function normalizePowerShellVariables(source) {
|
||||
const output = source.split("");
|
||||
const patterns = [
|
||||
{ expression: /\$\{(?:[A-Za-z_][A-Za-z0-9_]*:)?(revision|workspaceRevision|selectedWorkspace)\}/giu, dollar: false },
|
||||
{ expression: /\$(?:[A-Za-z_][A-Za-z0-9_]*:)(revision|workspaceRevision|selectedWorkspace)\b/giu, dollar: false },
|
||||
{ expression: /\$(revision|workspaceRevision|selectedWorkspace)\b/giu, dollar: true },
|
||||
];
|
||||
for (const { expression, dollar } of patterns) {
|
||||
for (const match of source.matchAll(expression)) {
|
||||
const name = canonicalRevisionName(match[1]);
|
||||
const replacement = `${dollar ? "$" : ""}${name}`.padEnd(match[0].length, " ");
|
||||
for (let offset = 0; offset < match[0].length; offset += 1) output[match.index + offset] = replacement[offset];
|
||||
}
|
||||
}
|
||||
let normalized = output.join("");
|
||||
normalized = normalized.replace(/\.\s*state\b/giu, (match) => match.replace(/state/iu, "state"));
|
||||
normalized = normalized.replace(/(["'])state\1/giu, (_match, quote) => `${quote}state${quote}`);
|
||||
return normalized;
|
||||
}
|
||||
|
||||
function maskShellSource(source) {
|
||||
return maskShellFamilySource(source, false);
|
||||
}
|
||||
|
||||
function maskPowerShellSource(source) {
|
||||
return normalizePowerShellVariables(maskShellFamilySource(source, true));
|
||||
}
|
||||
|
||||
function maskShellFamilySource(source, powershell) {
|
||||
const output = source.split("");
|
||||
let squareDepth = 0;
|
||||
for (let index = 0; index < source.length; index += 1) {
|
||||
if (powershell && source.startsWith("<#", index)) {
|
||||
const close = source.indexOf("#>", index + 2);
|
||||
const end = close < 0 ? source.length : close + 2;
|
||||
maskRange(output, source, index, end);
|
||||
index = end - 1;
|
||||
continue;
|
||||
}
|
||||
if (powershell ? source[index] === "#" : shellCommentStart(source, index)) {
|
||||
const end = lineEnd(source, index);
|
||||
maskRange(output, source, index, end);
|
||||
index = end - 1;
|
||||
continue;
|
||||
}
|
||||
if (powershell && source[index] === "`") {
|
||||
maskRange(output, source, index, Math.min(index + 2, source.length));
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (!powershell && source[index] === "`") {
|
||||
const close = source.indexOf("`", index + 1);
|
||||
const end = close < 0 ? source.length : close + 1;
|
||||
maskRange(output, source, index, end);
|
||||
restoreMasked(output, index + 1, maskShellSource(source.slice(index + 1, close < 0 ? source.length : close)));
|
||||
index = end - 1;
|
||||
continue;
|
||||
}
|
||||
const quote = source[index];
|
||||
if (quote === "'" || quote === '"') {
|
||||
const escapes = powershell ? "`" : quote === "'" ? "" : "\\";
|
||||
const end = quotedEnd(source, index, quote, escapes);
|
||||
const preserveKey = powershell && squareDepth > 0;
|
||||
if (!preserveKey) maskRange(output, source, index, end, false);
|
||||
if (quote === '"') {
|
||||
exposeDollarSubexpressions(output, source, index + 1, end - 1, powershell ? "powershell" : "shell");
|
||||
if (!powershell) {
|
||||
for (let cursor = index + 1; cursor < end - 1; cursor += 1) {
|
||||
if (source[cursor] !== "`" || source[cursor - 1] === "\\") continue;
|
||||
const close = source.indexOf("`", cursor + 1);
|
||||
if (close < 0 || close >= end) break;
|
||||
restoreMasked(output, cursor + 1, maskShellSource(source.slice(cursor + 1, close)));
|
||||
cursor = close;
|
||||
}
|
||||
}
|
||||
}
|
||||
index = end - 1;
|
||||
continue;
|
||||
}
|
||||
if (source.startsWith("$(", index)) output[index] = " ";
|
||||
if (source[index] === "[") squareDepth += 1;
|
||||
else if (source[index] === "]" && squareDepth > 0) squareDepth -= 1;
|
||||
}
|
||||
return output.join("");
|
||||
}
|
||||
|
||||
function maskUnknownSource(source) {
|
||||
const output = source.split("");
|
||||
let squareDepth = 0;
|
||||
for (let index = 0; index < source.length; index += 1) {
|
||||
if (source.startsWith("/*", index)) {
|
||||
const close = source.indexOf("*/", index + 2);
|
||||
const end = close < 0 ? source.length : close + 2;
|
||||
maskRange(output, source, index, end);
|
||||
index = end - 1;
|
||||
continue;
|
||||
}
|
||||
if (source[index] === "#" || source.startsWith("//", index)) {
|
||||
const end = lineEnd(source, index);
|
||||
maskRange(output, source, index, end);
|
||||
index = end - 1;
|
||||
continue;
|
||||
}
|
||||
const quote = source[index];
|
||||
if (quote === "'" || quote === '"' || quote === "`") {
|
||||
const end = quotedEnd(source, index, quote, "\\");
|
||||
let after = end;
|
||||
while (/[ \t]/u.test(source[after] ?? "")) after += 1;
|
||||
if (!(squareDepth > 0 || source[after] === ":")) maskRange(output, source, index, end, true);
|
||||
index = end - 1;
|
||||
continue;
|
||||
}
|
||||
if (source[index] === "[") squareDepth += 1;
|
||||
else if (source[index] === "]" && squareDepth > 0) squareDepth -= 1;
|
||||
}
|
||||
return output.join("");
|
||||
}
|
||||
|
||||
function maskQuotedShellHeredocBodies(source, label = "<shell>") {
|
||||
const output = source.split("");
|
||||
for (const range of literalBashHeredocBodyRanges(source, label)) maskRange(output, source, range.start, range.end);
|
||||
return output.join("");
|
||||
}
|
||||
|
||||
function shellAssociativeRevisionAccess(source) {
|
||||
let quote;
|
||||
for (let index = 0; index < source.length; index += 1) {
|
||||
const character = source[index];
|
||||
if (character === "\\") { index += 1; continue; }
|
||||
if (quote === "'") { if (character === "'") quote = undefined; continue; }
|
||||
if (character === "'") { quote = "'"; continue; }
|
||||
if (character === '"') { quote = quote === '"' ? undefined : '"'; continue; }
|
||||
if (character !== "$" || source[index + 1] !== "{") continue;
|
||||
const close = source.indexOf("}", index + 2);
|
||||
if (close < 0) break;
|
||||
const expansion = source.slice(index, close + 1);
|
||||
if (/^\$\{[ \t]*(?:revision|workspaceRevision|selectedWorkspace)[ \t]*\[[ \t]*(?:["']state["']|state)[ \t]*\][^}]*\}$/u.test(expansion)) return true;
|
||||
index = close;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
const shellCommandPrefixes = new Set(["if", "then", "elif", "else", "while", "until", "do"]);
|
||||
const shellCommandClosers = new Set(["fi", "done", "esac"]);
|
||||
const shellControlCharacters = new Set([";", "|", "&", "(", ")", "{", "}", "`"]);
|
||||
|
||||
function shellQuotedSubstitutionEnd(source, start, depth, budget) {
|
||||
for (let index = start + 1; index < source.length; index += 1) {
|
||||
budget.characters += 1;
|
||||
if (budget.characters > 100_000) throw new Error("revision-state shell substitution size limit exceeded");
|
||||
if (source[index] === "\\") { index += 1; continue; }
|
||||
if (source[index] === '"') return index + 1;
|
||||
if (source.startsWith("$(", index) || source.startsWith("<(", index) || source.startsWith(">(", index)) {
|
||||
index = shellParenthesizedEnd(source, index + 1, depth + 1, budget) - 1;
|
||||
} else if (source[index] === "`") {
|
||||
const end = quotedEnd(source, index, "`", "\\");
|
||||
if (end - 1 <= index || source[end - 1] !== "`") throw new Error("revision-state shell substitution has an unclosed backtick");
|
||||
index = end - 1;
|
||||
}
|
||||
}
|
||||
throw new Error("revision-state shell substitution has an unclosed quote");
|
||||
}
|
||||
|
||||
function shellParenthesizedEnd(source, openIndex, depth, budget) {
|
||||
if (depth > 64) throw new Error("revision-state shell substitution nesting limit exceeded");
|
||||
for (let index = openIndex + 1; index < source.length; index += 1) {
|
||||
budget.characters += 1;
|
||||
if (budget.characters > 100_000) throw new Error("revision-state shell substitution size limit exceeded");
|
||||
if (source[index] === "\\") { index += 1; continue; }
|
||||
if (source[index] === "'") {
|
||||
const end = quotedEnd(source, index, "'", "");
|
||||
if (end - 1 <= index || source[end - 1] !== "'") throw new Error("revision-state shell substitution has an unclosed quote");
|
||||
index = end - 1;
|
||||
continue;
|
||||
}
|
||||
if (source[index] === '"') { index = shellQuotedSubstitutionEnd(source, index, depth, budget) - 1; continue; }
|
||||
if (source[index] === "`") {
|
||||
const end = quotedEnd(source, index, "`", "\\");
|
||||
if (end - 1 <= index || source[end - 1] !== "`") throw new Error("revision-state shell substitution has an unclosed backtick");
|
||||
index = end - 1;
|
||||
continue;
|
||||
}
|
||||
if (source[index] === "#" && (index === openIndex + 1 || /[ \t\r\n;|&()]/u.test(source[index - 1]))) {
|
||||
index = lineEnd(source, index);
|
||||
continue;
|
||||
}
|
||||
if (source[index] === "(") { index = shellParenthesizedEnd(source, index, depth + 1, budget) - 1; continue; }
|
||||
if (source[index] === ")") return index + 1;
|
||||
}
|
||||
throw new Error("revision-state shell process substitution is unbalanced");
|
||||
}
|
||||
|
||||
function shellProcessSubstitutionEnd(source, start) {
|
||||
if (!(source.startsWith("<(", start) || source.startsWith(">(", start))) return undefined;
|
||||
return shellParenthesizedEnd(source, start + 1, 1, { characters: 0 });
|
||||
}
|
||||
|
||||
function shellRedirectionAt(source, start) {
|
||||
const match = source.slice(start).match(/^(?:&>>|&>|(?:[0-9]+|\{[A-Za-z_][A-Za-z0-9_]*\})?(?:<<<|<<-|<<|>>|<>|>\||<&|>&|<|>))/u);
|
||||
if (!match) return undefined;
|
||||
let end = start + match[0].length;
|
||||
while (end < source.length && !/\s/u.test(source[end]) && !shellControlCharacters.has(source[end]) &&
|
||||
source[end] !== "<" && source[end] !== ">" && source[end] !== "'" && source[end] !== '"') end += 1;
|
||||
return { value: source.slice(start, end), end, needsOperand: end === start + match[0].length };
|
||||
}
|
||||
|
||||
function shellLexTokens(source) {
|
||||
const tokens = [];
|
||||
const push = (value, start, end, type = "word") => {
|
||||
tokens.push({ value, start, end, type });
|
||||
if (tokens.length > 50_000) throw new Error("revision-state shell token limit exceeded");
|
||||
};
|
||||
for (let index = 0; index < source.length;) {
|
||||
if (source[index] === "\n" || source[index] === "\r") { push(source[index], index, index + 1, "control"); index += 1; continue; }
|
||||
if (/\s/u.test(source[index])) { index += 1; continue; }
|
||||
if (source[index] === "#") { index = lineEnd(source, index); continue; }
|
||||
const processEnd = shellProcessSubstitutionEnd(source, index);
|
||||
if (processEnd !== undefined) {
|
||||
push(source.slice(index, processEnd), index, processEnd);
|
||||
index = processEnd;
|
||||
continue;
|
||||
}
|
||||
const redirection = shellRedirectionAt(source, index);
|
||||
if (redirection) {
|
||||
push(redirection.value, index, redirection.end, "redirection");
|
||||
tokens.at(-1).needsOperand = redirection.needsOperand;
|
||||
index = redirection.end;
|
||||
continue;
|
||||
}
|
||||
if (shellControlCharacters.has(source[index]) || source[index] === "!" && (index === 0 || /\s/u.test(source[index - 1]))) {
|
||||
const start = index;
|
||||
let value = source[index++];
|
||||
if ((value === ";" || value === "|" || value === "&") && source[index] === value) value += source[index++];
|
||||
push(value, start, index, "control");
|
||||
continue;
|
||||
}
|
||||
const start = index;
|
||||
let value = "";
|
||||
while (index < source.length && !/\s/u.test(source[index]) && !shellControlCharacters.has(source[index]) && source[index] !== "<" && source[index] !== ">") {
|
||||
const quote = source[index];
|
||||
if (quote === "'" || quote === '"') {
|
||||
const end = quotedEnd(source, index, quote, "\\");
|
||||
value += source.slice(index + 1, end - 1);
|
||||
index = end;
|
||||
} else if (source[index] === "\\" && index + 1 < source.length) {
|
||||
value += source[index + 1];
|
||||
index += 2;
|
||||
} else {
|
||||
value += source[index++];
|
||||
}
|
||||
}
|
||||
push(value, start, index);
|
||||
}
|
||||
return tokens;
|
||||
}
|
||||
|
||||
function shellCommandWords(source) {
|
||||
const commands = [];
|
||||
let words = [];
|
||||
const finish = () => { if (words.length > 0) commands.push(words); words = []; };
|
||||
for (const token of shellLexTokens(source)) {
|
||||
if (token.type === "control") {
|
||||
finish();
|
||||
continue;
|
||||
}
|
||||
if (token.type === "word" && words.length === 0 && shellCommandPrefixes.has(token.value)) continue;
|
||||
if (token.type === "word" && words.length === 0 && shellCommandClosers.has(token.value)) continue;
|
||||
words.push(token);
|
||||
}
|
||||
finish();
|
||||
return commands;
|
||||
}
|
||||
|
||||
function shellExecutable(word) {
|
||||
return word?.split("/").pop();
|
||||
}
|
||||
|
||||
const shellWrapperSpecs = new Map([
|
||||
["command", { kind: "options", operandOptions: new Set() }],
|
||||
["env", { kind: "env", operandOptions: new Set(["-u", "--unset", "-C", "--chdir"]) }],
|
||||
["sudo", { kind: "options", operandOptions: new Set(["-u", "--user", "-g", "--group", "-h", "--host", "-p", "--prompt", "-C", "--close-from", "-D", "--chdir"]) }],
|
||||
["nice", { kind: "options", operandOptions: new Set(["-n", "--adjustment"]) }],
|
||||
["time", { kind: "options", operandOptions: new Set(["-o", "--output", "-f", "--format"]) }],
|
||||
["xargs", { kind: "options", operandOptions: new Set(["-I", "--replace", "-n", "--max-args", "-L", "--max-lines", "-P", "--max-procs", "-s", "--max-chars", "-d", "--delimiter"]) }],
|
||||
["timeout", { kind: "timeout", operandOptions: new Set(["-k", "--kill-after", "-s", "--signal"]) }],
|
||||
["stdbuf", { kind: "stdbuf", operandOptions: new Set(["-i", "--input", "-o", "--output", "-e", "--error"]) }],
|
||||
["nohup", { kind: "options", operandOptions: new Set() }],
|
||||
["exec", { kind: "options", operandOptions: new Set(["-a"]) }],
|
||||
["coproc", { kind: "coproc", operandOptions: new Set() }],
|
||||
]);
|
||||
|
||||
function skipShellMetadata(words, start) {
|
||||
let index = start;
|
||||
while (index < words.length) {
|
||||
const token = words[index];
|
||||
if (/^[A-Za-z_][A-Za-z0-9_]*=/u.test(token.value)) { index += 1; continue; }
|
||||
if (token.type === "redirection") { index += token.needsOperand ? 2 : 1; continue; }
|
||||
break;
|
||||
}
|
||||
return index;
|
||||
}
|
||||
|
||||
function skipWrapperOptions(words, start, spec) {
|
||||
let index = start;
|
||||
while (index < words.length) {
|
||||
const word = words[index].value;
|
||||
if (word === "--") return index + 1;
|
||||
if (spec.operandOptions.has(word)) { index += 2; continue; }
|
||||
if (spec.kind === "stdbuf" && /^-(?:i|o|e).+/u.test(word)) { index += 1; continue; }
|
||||
if (word.startsWith("-")) { index += 1; continue; }
|
||||
break;
|
||||
}
|
||||
return index;
|
||||
}
|
||||
|
||||
function shellJqArguments(words) {
|
||||
let index = skipShellMetadata(words, 0);
|
||||
let wrappers = 0;
|
||||
while (index < words.length) {
|
||||
const spec = shellWrapperSpecs.get(shellExecutable(words[index]?.value));
|
||||
if (!spec) break;
|
||||
if (wrappers >= 16) throw new Error("revision-state shell wrapper nesting exceeds policy limit");
|
||||
wrappers += 1;
|
||||
index = skipWrapperOptions(words, index + 1, spec);
|
||||
if (spec.kind === "env") {
|
||||
while (/^[A-Za-z_][A-Za-z0-9_]*=/u.test(words[index]?.value ?? "")) index += 1;
|
||||
} else if (spec.kind === "timeout") {
|
||||
if (index >= words.length) return undefined;
|
||||
index += 1;
|
||||
} else if (spec.kind === "coproc") {
|
||||
index = skipShellMetadata(words, index);
|
||||
const current = shellExecutable(words[index]?.value);
|
||||
if (current !== "jq" && !shellWrapperSpecs.has(current) && /^[A-Za-z_][A-Za-z0-9_]*$/u.test(words[index]?.value ?? "")) {
|
||||
const afterName = skipShellMetadata(words, index + 1);
|
||||
const command = shellExecutable(words[afterName]?.value);
|
||||
if (command === "jq" || shellWrapperSpecs.has(command)) index = afterName;
|
||||
}
|
||||
}
|
||||
index = skipShellMetadata(words, index);
|
||||
}
|
||||
return shellExecutable(words[index]?.value) === "jq" ? words.slice(index + 1) : undefined;
|
||||
}
|
||||
|
||||
const jqOptionOperands = new Map([
|
||||
["--arg", 2], ["--argjson", 2], ["--slurpfile", 2], ["--rawfile", 2], ["--argfile", 2],
|
||||
["-L", 1], ["--library-path", 1], ["--indent", 1],
|
||||
["-f", 1], ["--from-file", 1],
|
||||
]);
|
||||
const jqFileFilterOptions = new Set(["-f", "--from-file"]);
|
||||
|
||||
function withoutShellRedirections(arguments_) {
|
||||
const semantic = [];
|
||||
for (let index = 0; index < arguments_.length; index += 1) {
|
||||
const token = arguments_[index];
|
||||
if (token.type === "redirection") { if (token.needsOperand) index += 1; continue; }
|
||||
semantic.push(token);
|
||||
}
|
||||
return semantic;
|
||||
}
|
||||
|
||||
function jqInvocation(arguments_) {
|
||||
const semantic = withoutShellRedirections(arguments_);
|
||||
let fromFile = false;
|
||||
for (let index = 0; index < semantic.length; index += 1) {
|
||||
const argument = semantic[index].value;
|
||||
if (argument === "--") return { filter: fromFile ? undefined : semantic[index + 1], arguments_ };
|
||||
const operands = jqOptionOperands.get(argument);
|
||||
if (operands !== undefined) {
|
||||
if (jqFileFilterOptions.has(argument)) fromFile = true;
|
||||
index += operands;
|
||||
continue;
|
||||
}
|
||||
if (argument.startsWith("-")) continue;
|
||||
return { filter: fromFile ? undefined : semantic[index], arguments_ };
|
||||
}
|
||||
return { filter: undefined, arguments_ };
|
||||
}
|
||||
|
||||
function maskShellJqLiteralArguments(source) {
|
||||
const output = source.split("");
|
||||
for (const words of shellCommandWords(source)) {
|
||||
const arguments_ = shellJqArguments(words);
|
||||
if (!arguments_) continue;
|
||||
const invocation = jqInvocation(arguments_);
|
||||
for (const argument of invocation.arguments_) {
|
||||
if (argument === invocation.filter) continue;
|
||||
const raw = source.slice(argument.start, argument.end);
|
||||
if (!raw.includes("$") && !raw.includes("`")) maskRange(output, source, argument.start, argument.end);
|
||||
}
|
||||
}
|
||||
return output.join("");
|
||||
}
|
||||
|
||||
function jqStringEnd(source, start) {
|
||||
for (let index = start + 1; index < source.length; index += 1) {
|
||||
if (source[index] === "\\") { index += 1; continue; }
|
||||
if (source[index] === '"') return index;
|
||||
}
|
||||
return source.length;
|
||||
}
|
||||
|
||||
function jqInterpolationEnd(source, start) {
|
||||
let depth = 1;
|
||||
for (let index = start; index < source.length; index += 1) {
|
||||
if (source[index] === '"') { index = jqStringEnd(source, index); continue; }
|
||||
if (source[index] === "(") depth += 1;
|
||||
else if (source[index] === ")" && --depth === 0) return index;
|
||||
}
|
||||
return source.length;
|
||||
}
|
||||
|
||||
function jqTokens(source, budget = { tokens: 0, depth: 0 }) {
|
||||
if (budget.depth >= 64) throw new Error("jq filter exceeds policy nesting limit");
|
||||
budget.depth += 1;
|
||||
const tokens = [];
|
||||
for (let index = 0; index < source.length; index += 1) {
|
||||
budget.tokens += 1;
|
||||
if (budget.tokens >= 10_000) throw new Error("jq filter exceeds policy token limit");
|
||||
if (/\s/u.test(source[index])) continue;
|
||||
if (source[index] === "#") { index = lineEnd(source, index); continue; }
|
||||
if (source[index] === '"') {
|
||||
const end = jqStringEnd(source, index);
|
||||
const raw = source.slice(index, Math.min(end + 1, source.length));
|
||||
let value;
|
||||
if (!raw.includes("\\(")) {
|
||||
try { value = JSON.parse(raw); } catch { value = undefined; }
|
||||
}
|
||||
tokens.push({ type: "string", value });
|
||||
for (let cursor = index + 1; cursor < end; cursor += 1) {
|
||||
if (source[cursor] === "\\" && source[cursor + 1] === "(") {
|
||||
const close = jqInterpolationEnd(source, cursor + 2);
|
||||
tokens.push(...jqTokens(source.slice(cursor + 2, close), budget));
|
||||
cursor = close;
|
||||
} else if (source[cursor] === "\\") cursor += 1;
|
||||
}
|
||||
index = end;
|
||||
continue;
|
||||
}
|
||||
const variable = source.slice(index).match(/^\$([A-Za-z_][A-Za-z0-9_]*)/u);
|
||||
if (variable) { tokens.push({ type: "variable", value: variable[1] }); index += variable[0].length - 1; continue; }
|
||||
const identifier = source.slice(index).match(/^[A-Za-z_][A-Za-z0-9_]*/u);
|
||||
if (identifier) { tokens.push({ type: "identifier", value: identifier[0] }); index += identifier[0].length - 1; continue; }
|
||||
const punctuation = { ".": "dot", "[": "open", "]": "close" }[source[index]];
|
||||
tokens.push({ type: punctuation ?? "other", value: source[index] });
|
||||
}
|
||||
budget.depth -= 1;
|
||||
return tokens;
|
||||
}
|
||||
|
||||
function jqStaticString(tokens, cursor, depth = 0) {
|
||||
if (depth >= 64) throw new Error("revision-state jq static-key nesting exceeds policy limit");
|
||||
let index = cursor;
|
||||
let value;
|
||||
if (tokens[index]?.type === "string" && typeof tokens[index].value === "string") {
|
||||
value = tokens[index].value;
|
||||
index += 1;
|
||||
} else if (tokens[index]?.type === "other" && tokens[index].value === "(") {
|
||||
const nested = jqStaticString(tokens, index + 1, depth + 1);
|
||||
if (!nested || tokens[nested.next]?.type !== "other" || tokens[nested.next].value !== ")") return undefined;
|
||||
value = nested.value;
|
||||
index = nested.next + 1;
|
||||
} else return undefined;
|
||||
while (tokens[index]?.type === "other" && tokens[index].value === "+") {
|
||||
const right = jqStaticString(tokens, index + 1, depth + 1);
|
||||
if (!right) return undefined;
|
||||
value += right.value;
|
||||
index = right.next;
|
||||
}
|
||||
return { value, next: index };
|
||||
}
|
||||
|
||||
function jqBracketSegment(tokens, cursor) {
|
||||
if (tokens[cursor]?.type !== "open") return undefined;
|
||||
const expression = jqStaticString(tokens, cursor + 1);
|
||||
return expression && tokens[expression.next]?.type === "close" ?
|
||||
{ value: expression.value, next: expression.next + 1 } : undefined;
|
||||
}
|
||||
|
||||
function jqPathSegment(tokens, cursor, allowBareBracket = true) {
|
||||
if (tokens[cursor]?.type === "variable") return { value: tokens[cursor].value, next: cursor + 1 };
|
||||
let index = cursor;
|
||||
if (tokens[index]?.type === "dot") {
|
||||
index += 1;
|
||||
if (tokens[index]?.type === "identifier" || tokens[index]?.type === "string") return { value: tokens[index].value, next: index + 1 };
|
||||
}
|
||||
return allowBareBracket ? jqBracketSegment(tokens, index) : undefined;
|
||||
}
|
||||
|
||||
function jqIdentityPipelineEnd(tokens, cursor) {
|
||||
let index = cursor;
|
||||
while (tokens[index]?.type === "other" && tokens[index].value === "(") index += 1;
|
||||
if (tokens[index]?.type !== "dot") return undefined;
|
||||
index += 1;
|
||||
while (tokens[index]?.type === "other" && tokens[index].value === ")") index += 1;
|
||||
return tokens[index]?.type === "other" && tokens[index].value === "|" ? index + 1 : undefined;
|
||||
}
|
||||
|
||||
function jqTargetGrammarSupported(tokens) {
|
||||
for (let index = 0; index < tokens.length; index += 1) {
|
||||
const token = tokens[index];
|
||||
if (token.type === "identifier" && tokens[index - 1]?.type !== "dot") return false;
|
||||
if (token.type === "open" && !jqBracketSegment(tokens, index)) return false;
|
||||
if (token.type !== "other") continue;
|
||||
if (["?", "(", ")", "|"].includes(token.value)) continue;
|
||||
if (token.value === "+" && (tokens[index - 1]?.type === "string" || tokens[index - 1]?.value === ")") &&
|
||||
(tokens[index + 1]?.type === "string" || tokens[index + 1]?.value === "(")) continue;
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
function jqContainsActiveTarget(tokens) {
|
||||
for (let index = 0; index < tokens.length; index += 1) {
|
||||
if (tokens[index].type === "variable" && revisionIdentifiers.has(tokens[index].value)) return true;
|
||||
if (tokens[index].type === "dot" && (tokens[index + 1]?.type === "identifier" || tokens[index + 1]?.type === "string") &&
|
||||
revisionIdentifiers.has(tokens[index + 1].value)) return true;
|
||||
if (tokens[index].type === "open" && (tokens[index - 1]?.type === "dot" || tokens[index - 1]?.type === "close" || tokens[index - 1]?.type === "identifier")) {
|
||||
const key = jqStaticString(tokens, index + 1);
|
||||
if (key && revisionIdentifiers.has(key.value)) return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function jqRevisionAnalysis(filter) {
|
||||
const tokens = jqTokens(filter);
|
||||
let activeTarget = jqContainsActiveTarget(tokens);
|
||||
for (let index = 0; index < tokens.length; index += 1) {
|
||||
if (tokens[index].type !== "dot" && tokens[index].type !== "variable") continue;
|
||||
const segments = [];
|
||||
let cursor = index;
|
||||
let pipelineBoundary = false;
|
||||
while (cursor < tokens.length) {
|
||||
if (pipelineBoundary && (tokens[cursor]?.type === "open" || tokens[cursor]?.type === "string")) {
|
||||
segments.length = 0;
|
||||
break;
|
||||
}
|
||||
if (pipelineBoundary && tokens[cursor]?.type === "variable") segments.length = 0;
|
||||
const segment = jqPathSegment(tokens, cursor, !pipelineBoundary);
|
||||
if (!segment) break;
|
||||
pipelineBoundary = false;
|
||||
segments.push(segment.value);
|
||||
cursor = segment.next;
|
||||
while (tokens[cursor]?.type === "other" && tokens[cursor].value === "?") cursor += 1;
|
||||
while (tokens[cursor]?.type === "other" && tokens[cursor].value === ")") cursor += 1;
|
||||
if (tokens[cursor]?.type === "other" && tokens[cursor].value === "|") {
|
||||
cursor += 1;
|
||||
while (tokens[cursor]?.type === "other" && tokens[cursor].value === "(") cursor += 1;
|
||||
let identityEnd;
|
||||
while ((identityEnd = jqIdentityPipelineEnd(tokens, cursor)) !== undefined) cursor = identityEnd;
|
||||
pipelineBoundary = true;
|
||||
}
|
||||
}
|
||||
if (segments.some((segment) => revisionIdentifiers.has(segment))) activeTarget = true;
|
||||
for (let position = 0; position + 1 < segments.length; position += 1) {
|
||||
if (revisionIdentifiers.has(segments[position]) && segments[position + 1] === "state") return "violation";
|
||||
}
|
||||
}
|
||||
if (!activeTarget) return "safe";
|
||||
return jqTargetGrammarSupported(tokens) ? "safe" : "unsupported";
|
||||
}
|
||||
|
||||
function shellExecutableSubstitutionBodies(source, arithmeticContext = false) {
|
||||
const bodies = [];
|
||||
const addParenthesized = (start, kind) => {
|
||||
const end = shellParenthesizedEnd(source, start + 1, 1, { characters: 0 });
|
||||
bodies.push({ kind, start: start + 2, end: end - 1, source: source.slice(start + 2, end - 1) });
|
||||
return end;
|
||||
};
|
||||
const addBacktick = (start) => {
|
||||
const end = quotedEnd(source, start, "`", "\\");
|
||||
if (end - 1 <= start || source[end - 1] !== "`") throw new Error("revision-state shell substitution has an unclosed backtick");
|
||||
bodies.push({ kind: "backtick", start: start + 1, end: end - 1, source: source.slice(start + 1, end - 1) });
|
||||
return end;
|
||||
};
|
||||
for (let index = 0; index < source.length; index += 1) {
|
||||
if (source[index] === "\\") { index += 1; continue; }
|
||||
if (source[index] === "#" && (index === 0 || /[ \t\r\n;|&()]/u.test(source[index - 1]))) { index = lineEnd(source, index); continue; }
|
||||
if (source[index] === "'") {
|
||||
const end = quotedEnd(source, index, "'", "");
|
||||
if (end - 1 <= index || source[end - 1] !== "'") throw new Error(`revision-state shell policy found an unclosed quote at offset ${index}`);
|
||||
index = end - 1;
|
||||
continue;
|
||||
}
|
||||
if (source[index] === '"') {
|
||||
for (let cursor = index + 1; cursor < source.length; cursor += 1) {
|
||||
if (source[cursor] === "\\") { cursor += 1; continue; }
|
||||
if (source[cursor] === '"') { index = cursor; break; }
|
||||
if (source.startsWith("$(", cursor)) {
|
||||
const end = addParenthesized(cursor, source.startsWith("$((", cursor) ? "arithmetic" : "command");
|
||||
cursor = end - 1;
|
||||
} else if (source[cursor] === "`") {
|
||||
cursor = addBacktick(cursor) - 1;
|
||||
}
|
||||
if (cursor + 1 >= source.length) throw new Error(`revision-state shell policy found an unclosed double quote at offset ${index}`);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (!arithmeticContext && (source.startsWith("<(", index) || source.startsWith(">(", index))) {
|
||||
index = addParenthesized(index, "process") - 1;
|
||||
continue;
|
||||
}
|
||||
if (source.startsWith("$(", index)) {
|
||||
const arithmetic = source.startsWith("$((", index);
|
||||
index = addParenthesized(index, arithmetic ? "arithmetic" : "command") - 1;
|
||||
continue;
|
||||
}
|
||||
if (source[index] === "`") index = addBacktick(index) - 1;
|
||||
}
|
||||
return bodies;
|
||||
}
|
||||
|
||||
function removeBacktickBodyEscapes(source) {
|
||||
let result = "";
|
||||
for (let index = 0; index < source.length; index += 1) {
|
||||
if (source[index] === "\\" && index + 1 < source.length && ["$", "`", "\\", "\n"].includes(source[index + 1])) {
|
||||
if (source[index + 1] !== "\n") result += source[index + 1];
|
||||
index += 1;
|
||||
} else {
|
||||
result += source[index];
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function shellJqRevisionAccess(source, budget = { characters: 0 }, depth = 0, arithmeticContext = false) {
|
||||
if (depth > 32) throw new Error("revision-state executable shell substitution nesting limit exceeded");
|
||||
budget.characters += source.length;
|
||||
if (budget.characters > 500_000) throw new Error("revision-state executable shell substitution size limit exceeded");
|
||||
if (!arithmeticContext) {
|
||||
for (const words of shellCommandWords(source)) {
|
||||
const arguments_ = shellJqArguments(words);
|
||||
const filter = arguments_ && jqInvocation(arguments_).filter;
|
||||
if (filter) {
|
||||
const analysis = jqRevisionAnalysis(filter.value);
|
||||
if (analysis === "violation") return true;
|
||||
if (analysis === "unsupported") throw new Error("revision-state jq target grammar is unsupported");
|
||||
}
|
||||
}
|
||||
}
|
||||
for (const body of shellExecutableSubstitutionBodies(source, arithmeticContext)) {
|
||||
const nestedSource = body.kind === "backtick" ? removeBacktickBodyEscapes(body.source) : body.source;
|
||||
if (shellJqRevisionAccess(nestedSource, budget, depth + 1, body.kind === "arithmetic")) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function nonJsAnalysisSource(source, label) {
|
||||
const lower = label.toLowerCase();
|
||||
if (lower.endsWith(".sh")) return maskShellSource(maskShellJqLiteralArguments(maskQuotedShellHeredocBodies(source, label)));
|
||||
if (lower.endsWith(".ps1")) return maskPowerShellSource(source);
|
||||
return maskUnknownSource(source);
|
||||
}
|
||||
|
||||
function revisionStateAstNodes(source, label) {
|
||||
const knownKind = scriptKindFor(label);
|
||||
const caseInsensitive = label.toLowerCase().endsWith(".ps1");
|
||||
const analyzed = knownKind === undefined ? nonJsAnalysisSource(source, label) : source;
|
||||
const file = ts.createSourceFile(label, analyzed, ts.ScriptTarget.Latest, true, knownKind ?? ts.ScriptKind.TS);
|
||||
const matches = [];
|
||||
function visit(node) {
|
||||
if (ts.isPropertyAccessExpression(node) && node.name.text === "state" && isRevisionExpression(node.expression, caseInsensitive)) {
|
||||
matches.push(node);
|
||||
} else if (ts.isElementAccessExpression(node) && isRevisionExpression(node.expression, caseInsensitive) &&
|
||||
node.argumentExpression && propertyNameText(node.argumentExpression, caseInsensitive) === "state") {
|
||||
matches.push(node);
|
||||
} else if ((ts.isVariableDeclaration(node) || ts.isParameter(node)) && node.initializer &&
|
||||
isRevisionExpression(node.initializer, caseInsensitive) && ts.isObjectBindingPattern(node.name) &&
|
||||
objectBindingHasState(node.name, caseInsensitive)) {
|
||||
matches.push(node);
|
||||
} else if (ts.isBinaryExpression(node) && node.operatorToken.kind === ts.SyntaxKind.EqualsToken &&
|
||||
isRevisionExpression(node.right, caseInsensitive)) {
|
||||
const assignmentTarget = unwrapExpression(node.left);
|
||||
if (ts.isObjectLiteralExpression(assignmentTarget) && objectLiteralHasState(assignmentTarget, caseInsensitive)) matches.push(node);
|
||||
} else if (ts.isPropertyAssignment(node) && propertyNameText(node.name, caseInsensitive) === "revision" &&
|
||||
ts.isObjectLiteralExpression(node.initializer) && objectLiteralHasState(node.initializer, caseInsensitive)) {
|
||||
matches.push(node);
|
||||
}
|
||||
ts.forEachChild(node, visit);
|
||||
}
|
||||
visit(file);
|
||||
return matches;
|
||||
}
|
||||
|
||||
|
||||
function yamlScalarRevisionAccess(value) {
|
||||
return /(?:^|[\s;=,(])(?:revision|workspaceRevision|selectedWorkspace)\s*(?:\.\s*state|\[\s*["']?state["']?\s*\])(?:$|[\s;,)])/u.test(value);
|
||||
}
|
||||
|
||||
function validateYamlRevisionState(source, label) {
|
||||
const documents = parseAllDocuments(source, { uniqueKeys: true, merge: true });
|
||||
for (const document of documents) {
|
||||
if (document.errors.length > 0) throw new Error(`${label}: revision-state policy cannot parse YAML`);
|
||||
const walkAst = (node) => {
|
||||
if (isScalar(node)) {
|
||||
if (node.type === "PLAIN" && typeof node.value === "string" && yamlScalarRevisionAccess(node.value)) throw new Error(`${label}: forbidden revision-state access`);
|
||||
return;
|
||||
}
|
||||
if (isSeq(node)) { for (const item of node.items) walkAst(item); return; }
|
||||
if (isMap(node)) { for (const pair of node.items) walkAst(pair.value); }
|
||||
};
|
||||
walkAst(document.contents);
|
||||
let resolved;
|
||||
try { resolved = document.toJS({ mapAsMap: true, maxAliasCount: 50 }); }
|
||||
catch { throw new Error(`${label}: revision-state YAML alias resolution failed`); }
|
||||
const seen = new WeakSet();
|
||||
const walkResolved = (value) => {
|
||||
if (!value || typeof value !== "object" || seen.has(value)) return;
|
||||
seen.add(value);
|
||||
if (value instanceof Map) {
|
||||
for (const [key, child] of value) {
|
||||
if (revisionIdentifiers.has(String(key)) && child instanceof Map && child.has("state")) throw new Error(`${label}: forbidden revision-state access`);
|
||||
walkResolved(child);
|
||||
}
|
||||
} else if (Array.isArray(value)) { for (const child of value) walkResolved(child); }
|
||||
};
|
||||
walkResolved(resolved);
|
||||
}
|
||||
}
|
||||
|
||||
function validateRevisionState(source, label) {
|
||||
const lower = label.toLowerCase();
|
||||
if (/\.(?:yaml|yml)(?:\.example)?$/u.test(lower)) {
|
||||
validateYamlRevisionState(source, label);
|
||||
return;
|
||||
}
|
||||
if (lower.endsWith(".sh")) {
|
||||
const active = maskQuotedShellHeredocBodies(source, label);
|
||||
try {
|
||||
if (shellJqRevisionAccess(active) || shellAssociativeRevisionAccess(active)) throw new Error("forbidden revision-state access");
|
||||
} catch (error) {
|
||||
throw new Error(`${label}: ${error instanceof Error ? error.message : String(error)}`);
|
||||
}
|
||||
}
|
||||
if (lower.endsWith(".py") || lower.endsWith(".pyw")) throw new Error(`${label}: revision-state Python input was not batched`);
|
||||
const matches = revisionStateAstNodes(source, label);
|
||||
if (matches.length === 0) return;
|
||||
const historical = 'revision.state !== "operational"';
|
||||
const historicalCount = source.split(historical).length - 1;
|
||||
const match = matches[0];
|
||||
if (label === "backend/src/workspaces/registry.ts" && matches.length === 1 &&
|
||||
match.getText() === "revision.state" && match.parent?.getText() === historical &&
|
||||
historicalCount === 1) return;
|
||||
throw new Error(`${label}: forbidden revision-state access`);
|
||||
}
|
||||
|
||||
|
||||
const pythonHelper = fileURLToPath(new URL("./revision_state_policy.py", import.meta.url));
|
||||
|
||||
function validatePythonRevisionStates(records) {
|
||||
if (!Array.isArray(records) || records.length === 0) return;
|
||||
let stdout;
|
||||
try {
|
||||
stdout = execFileSync("python3", ["-I", "-B", pythonHelper], {
|
||||
input: JSON.stringify(records), encoding: "utf8", timeout: 5_000, maxBuffer: 4 * 1024 * 1024,
|
||||
env: {
|
||||
PATH: process.env.PATH ?? "/usr/bin:/bin",
|
||||
LANG: "C.UTF-8",
|
||||
LC_ALL: "C.UTF-8",
|
||||
PYTHONDONTWRITEBYTECODE: "1",
|
||||
},
|
||||
stdio: ["pipe", "pipe", "pipe"],
|
||||
});
|
||||
} catch (error) {
|
||||
const detail = error?.stderr?.toString().trim();
|
||||
throw new Error(`revision-state helper failed${detail ? `: ${detail}` : ""}`);
|
||||
}
|
||||
let result;
|
||||
try { result = JSON.parse(stdout); }
|
||||
catch { throw new Error("revision-state helper failed: invalid JSON output"); }
|
||||
if (!result || !Array.isArray(result.violations) || result.violations.some((label) => typeof label !== "string")) throw new Error("revision-state helper failed: invalid result shape");
|
||||
if (result.violations.length > 0) throw new Error(`${result.violations[0]}: forbidden revision-state access`);
|
||||
}
|
||||
|
||||
export { validatePythonRevisionStates, validateRevisionState };
|
||||
@@ -0,0 +1,273 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import test from "node:test";
|
||||
|
||||
import { validatePythonRevisionStates, validateRevisionState } from "./revision-state-policy.mjs";
|
||||
|
||||
function rejects(source, path) {
|
||||
assert.throws(() => validateRevisionState(source, path), /revision-state/, source);
|
||||
}
|
||||
function passes(source, path) {
|
||||
assert.doesNotThrow(() => validateRevisionState(source, path));
|
||||
}
|
||||
|
||||
test("PowerShell scoped and braced revision variables remain executable", () => {
|
||||
rejects('${revision}.state', "scripts/direct.ps1");
|
||||
rejects('${workspaceRevision}["state"]', "scripts/bracket.ps1");
|
||||
rejects('Write-Output "$(${selectedWorkspace}.state)"', "scripts/subexpression.ps1");
|
||||
rejects('${script:revision}.state', "scripts/scoped.ps1");
|
||||
rejects('${global:workspaceRevision}["state"]', "scripts/global.ps1");
|
||||
for (const source of [
|
||||
'$REVISION.STATE',
|
||||
'${Revision}.state',
|
||||
'$WORKSPACEREVISION["STATE"]',
|
||||
'${GLOBAL:SELECTEDWORKSPACE}.State',
|
||||
'$REVISION["ST" + "ATE"]',
|
||||
'${Revision}[("sT" + "AtE")]',
|
||||
'$record.REVISION.STATE',
|
||||
'$record.WORKSPACEREVISION["STATE"]',
|
||||
'$record["REVISION"].STATE',
|
||||
]) rejects(source, "scripts/case.ps1");
|
||||
passes('REVISION.STATE; revision.STATE; revision["ST" + "ATE"]; record.REVISION.STATE; record["REVISION"].state', "backend/src/case-sensitive.ts");
|
||||
});
|
||||
|
||||
test("Bash jq command forms and associative revision parameters are active", () => {
|
||||
for (const source of [
|
||||
"value=$(jq -r '.revision.state' snapshot.json)",
|
||||
"value=$(command jq -r '.workspaceRevision.state' snapshot.json)",
|
||||
"/usr/bin/jq --arg x y '.selectedWorkspace.state' snapshot.json",
|
||||
"env -i MODE=x jq -- '.revision.state' snapshot.json",
|
||||
"env -u MODE /opt/tools/jq -r '.workspaceRevision.state' snapshot.json",
|
||||
"echo safe\nvalue=`jq -r '.selectedWorkspace.state' snapshot.json`",
|
||||
"sudo -u nobody /usr/bin/jq -r '.revision.state' snapshot.json",
|
||||
"nice -n 5 jq -r '.workspaceRevision.state' snapshot.json",
|
||||
"time jq -r '.selectedWorkspace.state' snapshot.json",
|
||||
"printf x | xargs -n 1 jq -r '.revision.state'",
|
||||
"timeout -k 2 5 jq -r '.revision.state' snapshot.json",
|
||||
`stdbuf -o L jq -r '.["workspaceRevision"].state' snapshot.json`,
|
||||
`stdbuf -oL jq -r '.["selectedWorkspace"]["state"]' snapshot.json`,
|
||||
`nohup jq -r '.revision["state"]' snapshot.json`,
|
||||
"< snapshot.json jq -r '.workspaceRevision.state'",
|
||||
"sudo MODE=x jq -r '.selectedWorkspace.state' snapshot.json",
|
||||
String.raw`jq -r '"x \(.revision.state)"' snapshot.json`,
|
||||
"jq < snapshot.json -r '.revision.state'",
|
||||
"jq -r < snapshot.json '.workspaceRevision.state'",
|
||||
"jq --arg note safe < snapshot.json '.selectedWorkspace.state'",
|
||||
"jq -r '.revision?.state' snapshot.json",
|
||||
`jq -r '.["workspaceRevision"]?["state"]' snapshot.json`,
|
||||
`jq -r '.["revision"]?.["state"]' snapshot.json`,
|
||||
`jq -r '."revision".state' snapshot.json`,
|
||||
`jq -r '."workspaceRevision"."state"' snapshot.json`,
|
||||
"jq -r '$revision.state' snapshot.json",
|
||||
"jq -r '($selectedWorkspace).state' snapshot.json",
|
||||
`${"env ".repeat(17)}jq -r '.revision.state' snapshot.json`,
|
||||
"jq<input.json -r '.revision.state'",
|
||||
"jq 2>/dev/null -r '.workspaceRevision.state' snapshot.json",
|
||||
"{ jq -r '.selectedWorkspace.state' snapshot.json; }",
|
||||
"! jq -r '.revision.state' snapshot.json",
|
||||
"if jq -r '.workspaceRevision.state' snapshot.json; then :; fi",
|
||||
"if false; then :; elif jq -r '.selectedWorkspace.state' snapshot.json; then :; fi",
|
||||
"while false; do jq -r '.revision.state' snapshot.json; done",
|
||||
"until false; do jq -r '.workspaceRevision.state' snapshot.json; done",
|
||||
"jq -r '.revision | .state' snapshot.json",
|
||||
"jq -r '(.workspaceRevision | .state)' snapshot.json",
|
||||
`jq -r '.["revi" + "sion"].state' snapshot.json`,
|
||||
`jq -r '.["workspace" + "Revision"]["st" + "ate"]' snapshot.json`,
|
||||
"jq 2>&1 -r '.revision.state' snapshot.json",
|
||||
"jq 2>&- -r '.workspaceRevision.state' snapshot.json",
|
||||
"jq 0<&3 -r '.selectedWorkspace.state' snapshot.json",
|
||||
"jq &>/dev/null -r '.revision.state' snapshot.json",
|
||||
"jq &>>log -r '.workspaceRevision.state' snapshot.json",
|
||||
"jq >|output -r '.selectedWorkspace.state' snapshot.json",
|
||||
"jq {fd}>output -r '.revision.state' snapshot.json",
|
||||
"exec jq -r '.workspaceRevision.state' snapshot.json",
|
||||
"coproc jq -r '.selectedWorkspace.state' snapshot.json",
|
||||
"coproc worker jq -r '.revision.state' snapshot.json",
|
||||
"coproc worker >out jq -r '.workspaceRevision.state' snapshot.json",
|
||||
"coproc worker 2>/dev/null jq -r '.selectedWorkspace.state' snapshot.json",
|
||||
"coproc worker VAR=x jq -r '.revision.state' snapshot.json",
|
||||
`jq -r '.["revi" + ("sion")].state' snapshot.json`,
|
||||
"jq -r '.revision | . | .state' snapshot.json",
|
||||
"jq -r '(.workspaceRevision | (.) | .state)' snapshot.json",
|
||||
"jq -r '.revision | select(.) | .state' snapshot.json",
|
||||
"jq -r '.workspaceRevision | {value:.state}' snapshot.json",
|
||||
"jq -r '.selectedWorkspace | [.state]' snapshot.json",
|
||||
"jq -r '.revision + .state' snapshot.json",
|
||||
"jq < <(cat snapshot.json) -r '.revision.state'",
|
||||
"jq < <(cat <(printf snapshot.json)) -r '.workspaceRevision.state'",
|
||||
"jq > >(cat >/dev/null) -r '.selectedWorkspace.state' snapshot.json",
|
||||
`jq < <(printf '%s\n' "$((1 + (2)))") -r '.revision.state'`,
|
||||
"jq < <(cat snapshot.json -r '.revision.state'",
|
||||
`${"<(".repeat(65)}echo snapshot${")".repeat(65)} jq -r '.workspaceRevision.state'`,
|
||||
"cat <(jq -r '.revision.state' snapshot.json)",
|
||||
"cat snapshot.json > >(jq -r '.workspaceRevision.state')",
|
||||
`echo "$(jq -r '.selectedWorkspace.state' snapshot.json)"`,
|
||||
"value=$(jq -r '.revision.state' snapshot.json)",
|
||||
`echo "\`jq -r '.workspaceRevision.state' snapshot.json\`"`,
|
||||
`echo "$(cat <(jq -r '.selectedWorkspace.state' snapshot.json))"`,
|
||||
`${"$(".repeat(33)}jq -r '.revision.state' snapshot.json${")".repeat(33)}`,
|
||||
`echo "$(( $(jq -r '.revision.state' snapshot.json) + 0 ))"`,
|
||||
"echo \"$(( `jq -r '.workspaceRevision.state' snapshot.json` + 0 ))\"",
|
||||
"echo `echo \\`jq -r '.selectedWorkspace.state' snapshot.json\\``",
|
||||
"echo \"`echo \\`jq -r '.revision.state' snapshot.json\\``\"",
|
||||
`${"$(( ".repeat(33)}$(jq -r '.workspaceRevision.state' snapshot.json)${" + 0 ))".repeat(33)}`,
|
||||
'old=${revision["state"]}',
|
||||
"old=${workspaceRevision[state]}",
|
||||
"old=${revision[state]:-missing}",
|
||||
"old=${workspaceRevision['state']:=missing}",
|
||||
"old=${selectedWorkspace[state]:1:2}",
|
||||
]) rejects(source, "scripts/policy.sh");
|
||||
const jqFilters = [
|
||||
".revision?.state", '.["revision"]?.["state"]', '."revision".state',
|
||||
'."workspaceRevision"."state"', "(.revision).state", "$revision.state",
|
||||
".revision | .state", "(.workspaceRevision | .state)",
|
||||
'.["revi" + "sion"].state', '.["revi" + ("sion")].state',
|
||||
".revision | . | .state", "(.workspaceRevision | (.) | .state)",
|
||||
".revision | select(.) | .state", ".workspaceRevision | {value:.state}",
|
||||
'.revision | ["state"]', '(.workspaceRevision | (["state"]))', ".selectedWorkspace | $state",
|
||||
];
|
||||
for (const filter of jqFilters) {
|
||||
const compiled = spawnSync("jq", ["-n", "--argjson", "revision", "{}", "--arg", "state", "x", filter], { encoding: "utf8" });
|
||||
if (compiled.error?.code !== "ENOENT") assert.equal(compiled.status, 0, `${filter}: ${compiled.stderr}`);
|
||||
}
|
||||
passes("cat <<'EOF'\nrevision.state\nEOF\n", "scripts/literal.sh");
|
||||
passes("echo '${revision[state]}'\n", "scripts/single-quoted-parameter.sh");
|
||||
passes(`echo "<(jq '.revision.state')"\n`, "scripts/literal-process-text.sh");
|
||||
passes(`echo "ordinary jq '.workspaceRevision.state' text"\n`, "scripts/literal-jq-text.sh");
|
||||
passes(`echo '$(jq -r ".selectedWorkspace.state")'\n`, "scripts/single-quoted-command-text.sh");
|
||||
passes(`# profile's harmless note
|
||||
printf 'ok\n'
|
||||
`, "scripts/comment-apostrophe.sh");
|
||||
passes(`cat <( # profile's harmless note
|
||||
printf 'snapshot\n'
|
||||
)
|
||||
`, "scripts/substitution-comment-apostrophe.sh");
|
||||
passes(`echo "$(( 1 + (2 * 3) ))"\n`, "scripts/literal-arithmetic.sh");
|
||||
passes(`echo $(( jq + revision + state ))\n`, "scripts/arithmetic-identifiers.sh");
|
||||
passes("echo \\`jq -r '.revision.state' snapshot.json\\`\n", "scripts/escaped-literal-backticks.sh");
|
||||
passes("echo \"\\`jq -r '.workspaceRevision.state' snapshot.json\\`\"\n", "scripts/double-quoted-literal-backticks.sh");
|
||||
passes("echo `printf '%s' '\\`jq -r \".selectedWorkspace.state\" snapshot.json\\`'`\n", "scripts/quoted-nonexecuting-nested-backticks.sh");
|
||||
passes("jq --arg note 'revision.state' '.' file\n", "scripts/jq-arg.sh");
|
||||
passes(`jq --argjson note '"revision.state"' '.' file
|
||||
`, "scripts/jq-argjson.sh");
|
||||
passes("jq -r '.' revision.state.json\n", "scripts/jq-file.sh");
|
||||
passes("jq -r '.revision.id' snapshot.json\n", "scripts/jq-simple-non-state.sh");
|
||||
passes("jq -f revision.state.jq snapshot.json\n", "scripts/jq-from-file.sh");
|
||||
passes("jq --from-file workspaceRevision.state.jq snapshot.json\n", "scripts/jq-long-from-file.sh");
|
||||
passes(`jq -r '"revision.state"' snapshot.json
|
||||
`, "scripts/jq-string.sh");
|
||||
passes(`jq -r '{note:"selectedWorkspace.state"}' snapshot.json
|
||||
`, "scripts/jq-object.sh");
|
||||
passes(`jq -r '.revision | "state"' snapshot.json
|
||||
`, "scripts/jq-pipe-literal-right.sh");
|
||||
passes(`jq -r '"revision" | .state' snapshot.json
|
||||
`, "scripts/jq-pipe-literal-left.sh");
|
||||
passes(`jq -r '.revision | ["state"]' snapshot.json
|
||||
`, "scripts/jq-pipe-array.sh");
|
||||
passes(`jq -r '(.workspaceRevision | (["state"]))' snapshot.json
|
||||
`, "scripts/jq-pipe-parenthesized-array.sh");
|
||||
passes(`jq --arg state x '.selectedWorkspace | $state' snapshot.json
|
||||
`, "scripts/jq-pipe-variable.sh");
|
||||
for (const opener of ["'E'OF", "E'OF'", "E\\OF"]) {
|
||||
passes(`cat <<${opener}
|
||||
revision.state
|
||||
EOF
|
||||
`, "scripts/partial-quoted-heredoc.sh");
|
||||
}
|
||||
rejects("cat <<'E'OF\nrevision.state\nEOF\nworkspaceRevision.state\n", "scripts/after-heredoc.sh");
|
||||
rejects("cat <<'EOF'\nrevision.state\n", "scripts/unclosed-heredoc.sh");
|
||||
rejects(`echo "<<'EOF'"
|
||||
jq -r '.revision.state' snapshot.json
|
||||
`, "scripts/quoted-opener.sh");
|
||||
});
|
||||
|
||||
test("Python helper resolves active AST expressions and static format bindings", () => {
|
||||
const rejectsPython = (source) => assert.throws(
|
||||
() => validatePythonRevisionStates([{ source, label: "backend/scripts/policy.py" }]),
|
||||
/revision-state/,
|
||||
);
|
||||
for (const source of [
|
||||
"old = revision.state",
|
||||
'old = workspaceRevision["state"]',
|
||||
'old = record["selectedWorkspace"].state',
|
||||
'old = f"{revision.state}"',
|
||||
'"{revision.state}".format(value)',
|
||||
'"{0.state}".format(revision)',
|
||||
'"{0[state]}".format(workspaceRevision)',
|
||||
'"{item.state}".format(item=selectedWorkspace)',
|
||||
'"{item[state]}".format_map({"item": revision})',
|
||||
'("{0.state}").format(revision)',
|
||||
'"{0:{1.state}}".format(value, revision)',
|
||||
'old = revision["st" + "ate"]',
|
||||
'old = record["revi" + "sion"].state',
|
||||
'old = revision[f"state"]',
|
||||
'old = record[f"revision"].state',
|
||||
`old = revision[f"st{'a'}te"]`,
|
||||
`old = revision[f"{'state'}"]`,
|
||||
`old = record[f"revi{'sion'}"].state`,
|
||||
`old = revision[f"{'st' + 'ate'}"]`,
|
||||
`old = record[f"{'revi' + 'sion'}"].state`,
|
||||
`old = revision[f"{'state':s}"]`,
|
||||
'"{0.state}".format(*[revision])',
|
||||
'"{0[state]}".format(*(revision,))',
|
||||
'"{1[state]}".format(*[other, workspaceRevision])',
|
||||
'"{item.state}".format(**{"item": selectedWorkspace})',
|
||||
'"{item[state]}".format_map({**{"item": revision}})',
|
||||
'"{.state}".format(revision)',
|
||||
'"{[state]}".format(revision)',
|
||||
'"{:{.state}}".format(value, revision)',
|
||||
'"{.name} {[state]}".format(other, revision)',
|
||||
'"{item.state}".format(item=revision, **values)',
|
||||
]) rejectsPython(source);
|
||||
validatePythonRevisionStates([
|
||||
{ source: 'text = "{revision.state}"', label: "backend/scripts/literal.py" },
|
||||
{ source: 'text = "{{revision.state}}".format(value)', label: "backend/scripts/escaped.py" },
|
||||
{ source: 'text = "{0.state}".format(other)', label: "backend/scripts/unrelated.py" },
|
||||
{ source: 'old = revision[f"st{suffix}"]', label: "backend/scripts/dynamic-key.py" },
|
||||
{ source: 'text = "{.name} {[state]}".format(other, other)', label: "backend/scripts/multi-auto.py" },
|
||||
{ source: 'text = "{item.state}".format(**values)', label: "backend/scripts/dynamic-map.py" },
|
||||
]);
|
||||
const hostile = mkdtempSync(join(tmpdir(), "revision-policy-hostile-"));
|
||||
writeFileSync(join(hostile, "json.py"), "raise RuntimeError('shadowed')\n");
|
||||
const previousPythonPath = process.env.PYTHONPATH;
|
||||
try {
|
||||
process.env.PYTHONPATH = hostile;
|
||||
validatePythonRevisionStates([{ source: "value = 1", label: "backend/scripts/isolated.py" }]);
|
||||
} finally {
|
||||
if (previousPythonPath === undefined) delete process.env.PYTHONPATH;
|
||||
else process.env.PYTHONPATH = previousPythonPath;
|
||||
rmSync(hostile, { recursive: true, force: true });
|
||||
}
|
||||
assert.throws(
|
||||
() => validatePythonRevisionStates([{ source: 'revision[f"{1:.1000000000f}"]', label: "backend/scripts/oversized.py" }]),
|
||||
/revision-state helper failed/,
|
||||
);
|
||||
validatePythonRevisionStates([{ source: 'revision[f"{1:04d}"]', label: "backend/scripts/small-format.py" }]);
|
||||
assert.throws(
|
||||
() => validatePythonRevisionStates([{ source: "def broken(", label: "backend/scripts/invalid.py" }]),
|
||||
/revision-state helper failed/,
|
||||
);
|
||||
});
|
||||
|
||||
test("YAML mappings and only active plain scalar expressions are rejected", () => {
|
||||
for (const source of [
|
||||
"value: { revision: { state: old } }\n",
|
||||
"value:\n workspaceRevision:\n state: old\n",
|
||||
'items:\n - "selectedWorkspace":\n "state": old\n',
|
||||
"old: selectedWorkspace.state\n",
|
||||
"url: https://host/x; old: selectedWorkspace.state\n",
|
||||
"saved: &saved { state: old }\nvalue: { revision: *saved }\n",
|
||||
"defaults: &defaults { workspaceRevision: { state: old } }\nvalue: { <<: *defaults }\n",
|
||||
]) rejects(source, "scripts/policy.yaml");
|
||||
rejects("a: &a [x,x,x,x,x,x,x,x,x]\nb: &b [*a,*a,*a,*a,*a,*a,*a,*a,*a]\nc: [*b,*b,*b,*b,*b,*b,*b,*b,*b]\n", "scripts/alias-bomb.yaml");
|
||||
rejects("value: [\n", "scripts/invalid.yaml");
|
||||
for (const source of [
|
||||
"value: |\n revision.state\n",
|
||||
"value: >\n workspaceRevision.state\n",
|
||||
'value: "selectedWorkspace.state"\n',
|
||||
"url: https://host/revision.state\n",
|
||||
]) passes(source, "scripts/literal.yaml");
|
||||
});
|
||||
@@ -0,0 +1,318 @@
|
||||
"""Semantic Python revision-state policy helper.
|
||||
|
||||
Reads one JSON array of ``{"label": str, "source": str}`` records from stdin and
|
||||
writes ``{"violations": [label, ...]}``. Invalid input or Python source is fatal.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import ast
|
||||
import json
|
||||
import re
|
||||
import string
|
||||
import sys
|
||||
from itertools import pairwise
|
||||
from typing import Any
|
||||
|
||||
TARGETS = frozenset({"revision", "workspaceRevision", "selectedWorkspace"})
|
||||
_FORMATTER = string.Formatter()
|
||||
|
||||
|
||||
MAX_STATIC_TEXT = 4_096
|
||||
MAX_FORMAT_SPEC = 256
|
||||
MAX_STATIC_DEPTH = 64
|
||||
_UNRESOLVED = object()
|
||||
|
||||
|
||||
def _bounded_text(value: str) -> str:
|
||||
if len(value) > MAX_STATIC_TEXT:
|
||||
raise ValueError("static text exceeds revision policy limit")
|
||||
return value
|
||||
|
||||
|
||||
def _static_scalar(node: ast.expr, depth: int) -> object:
|
||||
if depth > MAX_STATIC_DEPTH:
|
||||
raise ValueError("static expression nesting exceeds revision policy limit")
|
||||
if isinstance(node, ast.Constant) and type(node.value) in {
|
||||
str,
|
||||
int,
|
||||
float,
|
||||
complex,
|
||||
bool,
|
||||
type(None),
|
||||
}:
|
||||
if isinstance(node.value, str):
|
||||
_bounded_text(node.value)
|
||||
if isinstance(node.value, int) and node.value.bit_length() > MAX_STATIC_TEXT * 4:
|
||||
raise ValueError("static integer exceeds revision policy limit")
|
||||
return node.value
|
||||
if isinstance(node, ast.BinOp) and isinstance(node.op, ast.Add):
|
||||
left = _static_scalar(node.left, depth + 1)
|
||||
right = _static_scalar(node.right, depth + 1)
|
||||
if left is _UNRESOLVED or right is _UNRESOLVED:
|
||||
return _UNRESOLVED
|
||||
try:
|
||||
result = left + right
|
||||
except TypeError:
|
||||
return _UNRESOLVED
|
||||
if type(result) not in {str, int, float, complex, bool}:
|
||||
return _UNRESOLVED
|
||||
if isinstance(result, str):
|
||||
_bounded_text(result)
|
||||
if isinstance(result, int) and result.bit_length() > MAX_STATIC_TEXT * 4:
|
||||
raise ValueError("static integer exceeds revision policy limit")
|
||||
return result
|
||||
if isinstance(node, ast.JoinedStr):
|
||||
result = _static_key(node, depth + 1)
|
||||
return _UNRESOLVED if result is None else result
|
||||
return _UNRESOLVED
|
||||
|
||||
|
||||
def _validate_format_spec(format_spec: str) -> None:
|
||||
if len(format_spec) > MAX_FORMAT_SPEC:
|
||||
raise ValueError("static format specification exceeds revision policy limit")
|
||||
for digits in re.findall(r"[0-9]+", format_spec):
|
||||
if len(digits) > 6 or int(digits) > MAX_STATIC_TEXT:
|
||||
raise ValueError("static format width or precision exceeds revision policy limit")
|
||||
|
||||
|
||||
def _static_key(node: ast.expr, depth: int = 0) -> str | None:
|
||||
if depth > MAX_STATIC_DEPTH:
|
||||
raise ValueError("static key nesting exceeds revision policy limit")
|
||||
if isinstance(node, ast.Constant) and isinstance(node.value, str):
|
||||
return _bounded_text(node.value)
|
||||
if isinstance(node, ast.BinOp) and isinstance(node.op, ast.Add):
|
||||
left = _static_key(node.left, depth + 1)
|
||||
right = _static_key(node.right, depth + 1)
|
||||
return None if left is None or right is None else _bounded_text(left + right)
|
||||
if isinstance(node, ast.JoinedStr):
|
||||
pieces = []
|
||||
length = 0
|
||||
for value in node.values:
|
||||
if isinstance(value, ast.Constant) and isinstance(value.value, str):
|
||||
piece = value.value
|
||||
elif isinstance(value, ast.FormattedValue):
|
||||
scalar = _static_scalar(value.value, depth + 1)
|
||||
if scalar is _UNRESOLVED:
|
||||
return None
|
||||
format_spec = "" if value.format_spec is None else _static_key(value.format_spec, depth + 1)
|
||||
if format_spec is None:
|
||||
return None
|
||||
_validate_format_spec(format_spec)
|
||||
try:
|
||||
if value.conversion == ord("s"):
|
||||
scalar = str(scalar)
|
||||
elif value.conversion == ord("r"):
|
||||
scalar = repr(scalar)
|
||||
elif value.conversion == ord("a"):
|
||||
scalar = ascii(scalar)
|
||||
elif value.conversion != -1:
|
||||
return None
|
||||
piece = format(scalar, format_spec)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
else:
|
||||
return None
|
||||
length += len(piece)
|
||||
if length > MAX_STATIC_TEXT:
|
||||
raise ValueError("static formatted key exceeds revision policy limit")
|
||||
pieces.append(piece)
|
||||
return "".join(pieces)
|
||||
return None
|
||||
|
||||
|
||||
def _is_revision_expr(node: ast.expr) -> bool:
|
||||
if isinstance(node, ast.Name):
|
||||
return node.id in TARGETS
|
||||
if isinstance(node, ast.Attribute):
|
||||
return node.attr in TARGETS
|
||||
if isinstance(node, ast.Subscript):
|
||||
return _static_key(node.slice) in TARGETS
|
||||
return False
|
||||
|
||||
|
||||
def _is_state_access(node: ast.AST) -> bool:
|
||||
if isinstance(node, ast.Attribute):
|
||||
return node.attr == "state" and _is_revision_expr(node.value)
|
||||
if isinstance(node, ast.Subscript):
|
||||
return _static_key(node.slice) == "state" and _is_revision_expr(node.value)
|
||||
return False
|
||||
|
||||
|
||||
def _static_sequence(node: ast.expr) -> list[ast.expr] | None:
|
||||
if not isinstance(node, (ast.List, ast.Tuple)):
|
||||
return None
|
||||
result: list[ast.expr] = []
|
||||
for element in node.elts:
|
||||
if isinstance(element, ast.Starred):
|
||||
nested = _static_sequence(element.value)
|
||||
if nested is None:
|
||||
return None
|
||||
result.extend(nested)
|
||||
else:
|
||||
result.append(element)
|
||||
return result
|
||||
|
||||
|
||||
def _static_mapping(node: ast.expr) -> dict[str, ast.expr] | None:
|
||||
if not isinstance(node, ast.Dict):
|
||||
return None
|
||||
result: dict[str, ast.expr] = {}
|
||||
for key, value in zip(node.keys, node.values, strict=True):
|
||||
if key is None:
|
||||
nested = _static_mapping(value)
|
||||
if nested is None:
|
||||
return None
|
||||
result.update(nested)
|
||||
elif (name := _static_key(key)) is not None:
|
||||
result[name] = value
|
||||
else:
|
||||
return None
|
||||
return result
|
||||
|
||||
|
||||
def _format_bindings(call: ast.Call, method: str) -> dict[str | int, ast.expr]:
|
||||
if method == "format":
|
||||
bindings: dict[str | int, ast.expr] = {}
|
||||
position = 0
|
||||
positional_known = True
|
||||
for argument in call.args:
|
||||
if isinstance(argument, ast.Starred):
|
||||
expanded = _static_sequence(argument.value)
|
||||
if expanded is None:
|
||||
positional_known = False
|
||||
continue
|
||||
if positional_known:
|
||||
for value in expanded:
|
||||
bindings[position] = value
|
||||
position += 1
|
||||
elif positional_known:
|
||||
bindings[position] = argument
|
||||
position += 1
|
||||
for keyword in call.keywords:
|
||||
if keyword.arg is not None:
|
||||
# An explicit keyword remains bound even beside **dynamic; a duplicate is TypeError.
|
||||
bindings[keyword.arg] = keyword.value
|
||||
else:
|
||||
expanded = _static_mapping(keyword.value)
|
||||
if expanded is not None:
|
||||
bindings.update(expanded)
|
||||
return bindings
|
||||
if len(call.args) != 1 or call.keywords:
|
||||
return {}
|
||||
return _static_mapping(call.args[0]) or {}
|
||||
|
||||
|
||||
def _field_accesses_state(
|
||||
field_name: str, bindings: dict[str | int, ast.expr], automatic_index: int | None = None
|
||||
) -> bool:
|
||||
root_match = re.match(r"(?:[0-9]+|[A-Za-z_][A-Za-z0-9_]*)", field_name)
|
||||
if root_match is None:
|
||||
if automatic_index is None or not field_name.startswith((".", "[")):
|
||||
return False
|
||||
root: str | int = automatic_index
|
||||
cursor = 0
|
||||
else:
|
||||
root_text = root_match.group(0)
|
||||
root = int(root_text) if root_text.isdigit() else root_text
|
||||
cursor = root_match.end()
|
||||
steps: list[tuple[bool, str]] = []
|
||||
while cursor < len(field_name):
|
||||
if field_name[cursor] == ".":
|
||||
match = re.match(r"[A-Za-z_][A-Za-z0-9_]*", field_name[cursor + 1 :])
|
||||
if match is None:
|
||||
return False
|
||||
steps.append((True, match.group(0)))
|
||||
cursor += len(match.group(0)) + 1
|
||||
elif field_name[cursor] == "[":
|
||||
close = field_name.find("]", cursor + 1)
|
||||
if close < 0:
|
||||
return False
|
||||
steps.append((False, field_name[cursor + 1 : close]))
|
||||
cursor = close + 1
|
||||
else:
|
||||
return False
|
||||
if steps:
|
||||
first_step = str(steps[0][1])
|
||||
if str(root) in TARGETS and first_step == "state":
|
||||
return True
|
||||
bound = bindings.get(root)
|
||||
if bound is not None and _is_revision_expr(bound) and first_step == "state":
|
||||
return True
|
||||
names = [str(root), *(str(key) for _is_attr, key in steps)]
|
||||
return any(left in TARGETS and right == "state" for left, right in pairwise(names))
|
||||
|
||||
|
||||
def _format_call_violation(node: ast.Call) -> bool:
|
||||
function = node.func
|
||||
if not isinstance(function, ast.Attribute) or function.attr not in {"format", "format_map"}:
|
||||
return False
|
||||
if not isinstance(function.value, ast.Constant) or not isinstance(function.value.value, str):
|
||||
return False
|
||||
bindings = _format_bindings(node, function.attr)
|
||||
numbering: dict[str, int | str | None] = {"next": 0, "mode": None}
|
||||
visited = 0
|
||||
|
||||
def analyze_template(template: str) -> bool:
|
||||
nonlocal visited
|
||||
visited += 1
|
||||
if visited > 1_000:
|
||||
raise ValueError("format specification nesting exceeds policy limit")
|
||||
for _literal, field_name, format_spec, _conversion in _FORMATTER.parse(template):
|
||||
automatic_index = None
|
||||
if field_name is not None:
|
||||
root_match = re.match(r"(?:[0-9]+|[A-Za-z_][A-Za-z0-9_]*)", field_name)
|
||||
automatic = field_name == "" or root_match is None and field_name.startswith((".", "["))
|
||||
manual = root_match is not None and root_match.group(0).isdigit()
|
||||
if automatic:
|
||||
if numbering["mode"] == "manual":
|
||||
raise ValueError("cannot switch from manual to automatic field numbering")
|
||||
numbering["mode"] = "automatic"
|
||||
automatic_index = int(numbering["next"])
|
||||
numbering["next"] = automatic_index + 1
|
||||
elif manual:
|
||||
if numbering["mode"] == "automatic":
|
||||
raise ValueError("cannot switch from automatic to manual field numbering")
|
||||
numbering["mode"] = "manual"
|
||||
if _field_accesses_state(field_name, bindings, automatic_index):
|
||||
return True
|
||||
if format_spec and analyze_template(format_spec):
|
||||
return True
|
||||
return False
|
||||
|
||||
return analyze_template(function.value.value)
|
||||
|
||||
|
||||
def has_revision_state(source: str, label: str = "<unknown>") -> bool:
|
||||
tree = ast.parse(source, filename=label, mode="exec")
|
||||
return any(_is_state_access(node) or (isinstance(node, ast.Call) and _format_call_violation(node)) for node in ast.walk(tree))
|
||||
|
||||
|
||||
def analyze_batch(records: Any) -> list[str]:
|
||||
if not isinstance(records, list):
|
||||
raise TypeError("input must be a JSON array")
|
||||
violations = []
|
||||
for record in records:
|
||||
if not isinstance(record, dict) or set(record) != {"label", "source"}:
|
||||
raise TypeError("each record must contain exactly label and source")
|
||||
label, source = record["label"], record["source"]
|
||||
if not isinstance(label, str) or not isinstance(source, str):
|
||||
raise TypeError("label and source must be strings")
|
||||
if has_revision_state(source, label):
|
||||
violations.append(label)
|
||||
return violations
|
||||
|
||||
|
||||
def main() -> int:
|
||||
try:
|
||||
records = json.load(sys.stdin)
|
||||
json.dump({"violations": analyze_batch(records)}, sys.stdout, ensure_ascii=False)
|
||||
sys.stdout.write("\n")
|
||||
return 0
|
||||
except Exception as error: # noqa: BLE001 - protocol boundary must fail closed
|
||||
print(f"python revision-state helper failed: {error}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Executable
+6
@@ -0,0 +1,6 @@
|
||||
#!/usr/bin/env node
|
||||
import { readFileSync } from "node:fs";
|
||||
|
||||
const path = process.env.THT_SSH_PASSPHRASE_FILE;
|
||||
if (!path) process.exit(1);
|
||||
process.stdout.write(readFileSync(path));
|
||||
@@ -0,0 +1,90 @@
|
||||
import importlib.util
|
||||
import tracemalloc
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
_HELPER = Path(__file__).with_name("revision_state_policy.py")
|
||||
_SPEC = importlib.util.spec_from_file_location("revision_state_policy", _HELPER)
|
||||
assert _SPEC is not None and _SPEC.loader is not None
|
||||
_MODULE = importlib.util.module_from_spec(_SPEC)
|
||||
_SPEC.loader.exec_module(_MODULE)
|
||||
analyze_batch = _MODULE.analyze_batch
|
||||
has_revision_state = _MODULE.has_revision_state
|
||||
|
||||
|
||||
class RevisionStatePolicyTests(unittest.TestCase):
|
||||
def test_ast_access_and_f_strings(self):
|
||||
for source in (
|
||||
"old = revision.state",
|
||||
'old = workspaceRevision["state"]',
|
||||
'old = record["selectedWorkspace"].state',
|
||||
'old = f"{revision.state}"',
|
||||
'old = revision["st" + "ate"]',
|
||||
'old = record["revi" + "sion"].state',
|
||||
'old = revision[f"state"]',
|
||||
'old = record[f"revision"].state',
|
||||
"old = revision[f\"st{'a'}te\"]",
|
||||
"old = revision[f\"{'state'}\"]",
|
||||
"old = record[f\"revi{'sion'}\"].state",
|
||||
"old = revision[f\"{'st' + 'ate'}\"]",
|
||||
"old = record[f\"{'revi' + 'sion'}\"].state",
|
||||
"old = revision[f\"{'state':s}\"]",
|
||||
):
|
||||
with self.subTest(source=source):
|
||||
self.assertTrue(has_revision_state(source))
|
||||
|
||||
def test_static_format_bindings(self):
|
||||
for source in (
|
||||
'"{0.state}".format(revision)',
|
||||
'"{0[state]}".format(workspaceRevision)',
|
||||
'"{item.state}".format(item=selectedWorkspace)',
|
||||
'"{item[state]}".format_map({"item": revision})',
|
||||
'("{0.state}").format(revision)',
|
||||
'"{0:{1.state}}".format(value, revision)',
|
||||
'"{0.state}".format(*[revision])',
|
||||
'"{0[state]}".format(*(revision,))',
|
||||
'"{1[state]}".format(*[other, workspaceRevision])',
|
||||
'"{item.state}".format(**{"item": selectedWorkspace})',
|
||||
'"{item[state]}".format(**{"outer": other, **{"item": revision}})',
|
||||
'"{item.state}".format_map({**{"item": workspaceRevision}})',
|
||||
'"{.state}".format(revision)',
|
||||
'"{[state]}".format(revision)',
|
||||
'"{:{.state}}".format(value, revision)',
|
||||
'"{.name} {[state]}".format(other, revision)',
|
||||
'"{item.state}".format(item=revision, **values)',
|
||||
):
|
||||
with self.subTest(source=source):
|
||||
self.assertTrue(has_revision_state(source))
|
||||
self.assertFalse(has_revision_state('"{0.state}".format(other)'))
|
||||
# Dynamic unpacking is intentionally unresolved rather than guessed.
|
||||
self.assertFalse(has_revision_state('"{0.state}".format(*values)'))
|
||||
self.assertFalse(has_revision_state('"{.name} {[state]}".format(other, other)'))
|
||||
self.assertFalse(has_revision_state('"{item.state}".format(**values)'))
|
||||
# FormattedValue keys are dynamic and are not treated as static strings.
|
||||
self.assertFalse(has_revision_state('revision[f"st{suffix}"]'))
|
||||
|
||||
def test_literals_are_not_active(self):
|
||||
self.assertFalse(has_revision_state('text = "{revision.state}"'))
|
||||
self.assertFalse(has_revision_state('text = "{{revision.state}}".format(value)'))
|
||||
|
||||
def test_oversized_static_format_fails_before_formatting(self):
|
||||
tracemalloc.start()
|
||||
with patch("builtins.format") as format_mock:
|
||||
with self.assertRaisesRegex(ValueError, "width or precision"):
|
||||
has_revision_state('revision[f"{1:.1000000000f}"]')
|
||||
format_mock.assert_not_called()
|
||||
_current, peak = tracemalloc.get_traced_memory()
|
||||
tracemalloc.stop()
|
||||
self.assertLess(peak, 1_000_000)
|
||||
self.assertFalse(has_revision_state('revision[f"{1:04d}"]'))
|
||||
|
||||
def test_batch_contract(self):
|
||||
self.assertEqual(
|
||||
analyze_batch([{"label": "one.py", "source": "revision.state"}]),
|
||||
["one.py"],
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
+379
@@ -0,0 +1,379 @@
|
||||
#!/usr/bin/env node
|
||||
import { createHash } from "node:crypto";
|
||||
import { lstat, readFile, realpath } from "node:fs/promises";
|
||||
import { isAbsolute, relative, resolve, sep } from "node:path";
|
||||
import { fileURLToPath, pathToFileURL } from "node:url";
|
||||
|
||||
import { isMap, isScalar, parseAllDocuments } from "yaml";
|
||||
import { extractBashDocuments } from "./bash-heredoc.mjs";
|
||||
import { validatePythonRevisionStates, validateRevisionState } from "./revision-state-policy.mjs";
|
||||
import { parseWorkspaceYaml } from "../dist/workspaces/schema.js";
|
||||
|
||||
const scriptPath = fileURLToPath(import.meta.url);
|
||||
const allowedKinds = new Set(["policy_text", "workspace_descriptor", "deployment_script"]);
|
||||
// Exact-content trust exceptions. Each digest covers the raw UTF-8 bytes from the
|
||||
// opener line through the closer line (including physical line endings). These
|
||||
// blocks are reviewed non-workspace runtime/config generation, not semantic proof.
|
||||
const reviewedExpandableBlocks = new Map([
|
||||
["scripts/test-dwh-auth-nginx-integration.sh", [
|
||||
{ sha256: "ead57234ad3520b5c7d4262b772957cbc7b9589da4f35fb17b160f948eb2ac7b", rationale: "Generates the reviewed isolated Nginx integration configuration." },
|
||||
]],
|
||||
["scripts/test-install-tht.sh", [
|
||||
{ sha256: "37f18ce7ce93cb8b84f3b3708462cc16d50fdc7bab22836c382dbacf8382f05f", rationale: "Generates the reviewed synthetic tht installer artifact." },
|
||||
]],
|
||||
["scripts/test-server-pi-state-topology.sh", [
|
||||
{ sha256: "435c769b8cbd7b834f56fdabddb86ba04fb404dd0d8a6b7c21719a8b0f7cf011", rationale: "Generates the reviewed model-catalog projection override for the isolated server topology test." },
|
||||
{ sha256: "6ae9567db53d6cd45a2c19c98acaf45f382450b157ea7d6f6d35125f68c50947", rationale: "Generates the isolated server topology test environment, including its installation descriptor and authentication configuration root." },
|
||||
]],
|
||||
["scripts/test-vector-backup-restore-safety.sh", [
|
||||
{ sha256: "40b8a10a3c06aaa98e324fbf688b7d1f5cead330d7ba7eef98e06256d412a85a", rationale: "Generates the reviewed restore safety manifest." },
|
||||
]],
|
||||
["scripts/test-windows-clone-contract.ps1", [
|
||||
{ sha256: "3204f772d33cad42bcac99191507051aefb2c91d2935bec6698b956e44f9bf45", rationale: "Generates reviewed Windows clone test configuration with its authentication configuration root." },
|
||||
{ sha256: "f4814d842a7502b7ef30fd6b224d5cb17b0ffd6fb2367c41c49ac16587536d93", rationale: "Same reviewed block in the repository-required CRLF checkout representation." },
|
||||
{ sha256: "6f25ce3b58cea47b74fe9319ed917d8089a2fb334bc0d469daa7e1f10865d870", rationale: "Generates the reviewed Windows Compose override for the canonical service topology." },
|
||||
{ sha256: "45a3cf19f7ce697b858b63d27a4edc7fefa2414d0408e7b6d72a65c86d314f5b", rationale: "Same reviewed Compose override in the repository-required CRLF checkout representation." },
|
||||
{ sha256: "5d0d1a3fc45e99b3aacaf4ee5dd09a6bee1937784375dfe4bcfaa4ae32cfb9de", rationale: "Generates reviewed Windows clone test configuration." },
|
||||
{ sha256: "b903e5dae953ae1372f1a5276f12a92ed3dd632b897f3afe5e00c646d90a1b42", rationale: "Same reviewed block in the repository-required CRLF checkout representation." },
|
||||
]],
|
||||
["scripts/unified-deployment-smoke.sh", [
|
||||
{ sha256: "b6c0826151b2c8b955399d1abf5b691cc8fe6b6454b17da000dde7ba3bc55d2d", rationale: "Generates the reviewed local Task 13 Compose override with normalized catalog mounts." },
|
||||
{ sha256: "24f69d12b8554aa2bebba455be99fde3e60743eef5a40fa2ef5b29397a477c03", rationale: "Generates the reviewed local Task 13 installation descriptor with its model catalog." },
|
||||
{ sha256: "526006fa6d48a8080b3834723630c64de5005a67243e944ebf1da15212b4d654", rationale: "Generates the reviewed server Task 13 Compose override." },
|
||||
{ sha256: "406ccead1967f642225c946fc4a23fe5b019c9764cc5153e1125876ade16ec90", rationale: "Generates the reviewed projected-auth server Task 13 installation descriptor with its model catalog." },
|
||||
]],
|
||||
["scripts/vector-backup.sh", [
|
||||
{ sha256: "571899db49dfdcec8107fbe1e0a86a61e7581979d3c4c248c20546843e275bcf", rationale: "Generates the reviewed backup manifest inside the helper command." },
|
||||
]],
|
||||
["scripts/vector-restore.sh", [
|
||||
{ sha256: "f04d872e556a7323583c6e620b25814fb6a8e2568a9a555623978185b473a49d", rationale: "Feeds reviewed parsed manifest values to read loops." },
|
||||
{ sha256: "c6053ed44abae71ae4821b68f9a513f8070947350e30d89ae0f65bf4a48f66fd", rationale: "Feeds reviewed parsed manifest values to read loops." },
|
||||
]],
|
||||
]);
|
||||
|
||||
function blockDigest(rawBlock) {
|
||||
return createHash("sha256").update(rawBlock, "utf8").digest("hex");
|
||||
}
|
||||
|
||||
function reviewedExpandableBlock(path, rawBlock) {
|
||||
const digest = blockDigest(rawBlock);
|
||||
return (reviewedExpandableBlocks.get(path) ?? []).some((review) => review.sha256 === digest);
|
||||
}
|
||||
|
||||
function hasAmbiguousExpansion(source, path) {
|
||||
const powershell = path.endsWith(".ps1");
|
||||
for (let index = 0; index < source.length; index += 1) {
|
||||
const character = source[index];
|
||||
if (powershell && character === "`") {
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (!powershell && character === "\\") {
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (character === "$" || (!powershell && character === "`")) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function physicalLines(source) {
|
||||
const rawLines = source.match(/[^\n]*\n|[^\n]+$/gu) ?? [];
|
||||
if (rawLines.length === 0) rawLines.push("");
|
||||
return rawLines.map((raw) => ({ raw, text: raw.replace(/\n$/u, "").replace(/\r$/u, "") }));
|
||||
}
|
||||
const prescribedSymbols = [
|
||||
"WorkspaceV1", "WorkspaceV2", "DeprecatedV2Descriptor", "LegacyMigrationResult",
|
||||
"LegacyMigrationOptions", "WorkspaceV2MigrationInput", "migrateLegacyWorkspace",
|
||||
"writeMigratedWorkspace", "migrateWorkspaceV1ToV2", "migrateWorkspaceV2ToV3",
|
||||
];
|
||||
const migrationMarkers = ["migration_required", "deprecated-v2-descriptor", "migrate-legacy", "migrate-v2-qdrant"];
|
||||
|
||||
function isPolicyImplementationException(label, category) {
|
||||
const implementations = new Set([
|
||||
"scripts/verify-schema-v3-only.sh",
|
||||
"scripts/test-verify-schema-v3-only.sh",
|
||||
"backend/scripts/verify-workspace-descriptor-files.mjs",
|
||||
"backend/scripts/verify-workspace-descriptor-files.test.mjs",
|
||||
"backend/scripts/revision-state-policy.mjs",
|
||||
"backend/scripts/revision-state-policy.test.mjs",
|
||||
"backend/scripts/bash-heredoc.mjs",
|
||||
"backend/scripts/revision_state_policy.py",
|
||||
"backend/scripts/test_revision_state_policy.py",
|
||||
]);
|
||||
if (implementations.has(label)) return true;
|
||||
if (category === "migration-marker" && new Set([
|
||||
"backend/src/workspaces/schema.ts",
|
||||
"scripts/workspace_descriptor_doc_contract.py",
|
||||
"scripts/test_workspace_descriptor_doc_contract.py",
|
||||
"backend/scripts/clean-dist.test.mjs",
|
||||
]).has(label)) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
function validatePolicySource(source, label) {
|
||||
if (!isPolicyImplementationException(label, "prescribed-symbol")) {
|
||||
for (const symbol of prescribedSymbols) {
|
||||
if (source.toLowerCase().includes(symbol.toLowerCase())) throw new Error(`${label}: forbidden prescribed-symbol substring: ${symbol}`);
|
||||
}
|
||||
}
|
||||
if (!isPolicyImplementationException(label, "migration-marker")) {
|
||||
for (const marker of migrationMarkers) {
|
||||
if (source.toLowerCase().includes(marker.toLowerCase())) throw new Error(`${label}: forbidden migration-marker substring: ${marker}`);
|
||||
}
|
||||
}
|
||||
if (!isPolicyImplementationException(label, "legacy-workspace")) {
|
||||
for (const match of source.matchAll(/legacyworkspace/giu)) {
|
||||
if (match[0] !== "legacyWorkspace") throw new Error(`${label}: forbidden legacy-workspace spelling: ${match[0]}`);
|
||||
}
|
||||
}
|
||||
if (!/\.pyw?$/iu.test(label) && !isPolicyImplementationException(label, "revision-state")) validateRevisionState(source, label);
|
||||
}
|
||||
|
||||
function documentShape(document) {
|
||||
const shape = { workspacePresent: false, workspaceMapping: false };
|
||||
if (!isMap(document.contents)) return shape;
|
||||
for (const pair of document.contents.items) {
|
||||
if (!isScalar(pair.key)) continue;
|
||||
if (pair.key.value === "workspace") {
|
||||
shape.workspacePresent = true;
|
||||
if (isMap(pair.value)) shape.workspaceMapping = true;
|
||||
}
|
||||
}
|
||||
return shape;
|
||||
}
|
||||
|
||||
function documents(source) {
|
||||
try {
|
||||
return parseAllDocuments(source, { uniqueKeys: true });
|
||||
} catch (error) {
|
||||
throw new Error(`YAML parser failed: ${error instanceof Error ? error.message : String(error)}`);
|
||||
}
|
||||
}
|
||||
|
||||
function validateWorkspaceSource(source, label, { requireWorkspace, expandable = false, path, rawBlock }) {
|
||||
const parsed = documents(source);
|
||||
const shapes = parsed.map(documentShape);
|
||||
if (requireWorkspace) {
|
||||
if (!shapes.some((shape) => shape.workspacePresent)) {
|
||||
throw new Error(`${label}: expected a top-level workspace mapping`);
|
||||
}
|
||||
if (!shapes.some((shape) => shape.workspaceMapping)) {
|
||||
throw new Error(`${label}: top-level workspace must be a mapping`);
|
||||
}
|
||||
} else {
|
||||
if (expandable && hasAmbiguousExpansion(source, path) && !reviewedExpandableBlock(path, rawBlock)) {
|
||||
throw new Error(`${label}: expandable block interpolation is not in the exact-content reviewed allowlist`);
|
||||
}
|
||||
if (shapes.some((shape) => shape.workspaceMapping)) {
|
||||
throw new Error(`${label}: embedded workspace descriptor is forbidden; use a tracked workspace fixture`);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
try {
|
||||
parseWorkspaceYaml(source);
|
||||
} catch (error) {
|
||||
throw new Error(`${label}: workspace descriptor is not valid schema v4: ${error instanceof Error ? error.message : String(error)}`);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
function deploymentScriptDialect(path) {
|
||||
if (path.endsWith(".sh")) return "bash";
|
||||
if (path.endsWith(".ps1")) return "powershell";
|
||||
throw new Error(`${path}: unknown deployment script dialect`);
|
||||
}
|
||||
|
||||
|
||||
function powerShellHereStringOpener(line, state) {
|
||||
let quote = null;
|
||||
for (let index = 0; index < line.length; index += 1) {
|
||||
if (state.blockComment) {
|
||||
const close = line.indexOf("#>", index);
|
||||
if (close < 0) return null;
|
||||
state.blockComment = false;
|
||||
index = close + 1;
|
||||
continue;
|
||||
}
|
||||
const character = line[index];
|
||||
if (quote === null && character === "`") {
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (quote === "'") {
|
||||
if (character === "'" && line[index + 1] === "'") index += 1;
|
||||
else if (character === "'") quote = null;
|
||||
continue;
|
||||
}
|
||||
if (quote === '"') {
|
||||
if (character === "`") index += 1;
|
||||
else if (character === '"') quote = null;
|
||||
continue;
|
||||
}
|
||||
if (character === "#") return null;
|
||||
if (character === "<" && line[index + 1] === "#") {
|
||||
state.blockComment = true;
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (character === "@" && (line[index + 1] === "'" || line[index + 1] === '"') && /^[ \t]*$/u.test(line.slice(index + 2))) return line[index + 1];
|
||||
if (character === "'" || character === '"') quote = character;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function extractPowerShellDocuments(source, label) {
|
||||
const records = physicalLines(source);
|
||||
const lines = records.map((record) => record.text);
|
||||
const extracted = [];
|
||||
const state = { blockComment: false };
|
||||
for (let index = 0; index < lines.length; index += 1) {
|
||||
const quote = powerShellHereStringOpener(lines[index], state);
|
||||
if (quote === null) continue;
|
||||
const delimiter = `${quote}@`;
|
||||
const opener = index;
|
||||
const body = [];
|
||||
const start = index + 2;
|
||||
let closed = false;
|
||||
for (index += 1; index < lines.length; index += 1) {
|
||||
if (lines[index].trimEnd() === delimiter) {
|
||||
closed = true;
|
||||
break;
|
||||
}
|
||||
body.push(lines[index]);
|
||||
}
|
||||
extracted.push({
|
||||
source: `${body.join("\n")}\n`,
|
||||
label: `${label}:${start} PowerShell here-string${closed ? "" : " (unclosed)"}`,
|
||||
expandable: quote === '"',
|
||||
path: label,
|
||||
rawBlock: records.slice(opener, Math.min(index + 1, records.length)).map((record) => record.raw).join(""),
|
||||
});
|
||||
}
|
||||
return extracted;
|
||||
}
|
||||
|
||||
export function extractScriptDocuments(source, label = "deployment script") {
|
||||
const dialect = deploymentScriptDialect(label);
|
||||
if (dialect === "bash") return extractBashDocuments(source, label);
|
||||
return extractPowerShellDocuments(source, label);
|
||||
}
|
||||
|
||||
async function safeFile(root, path) {
|
||||
if (typeof path !== "string" || path.length === 0 || isAbsolute(path) || path.includes("\\")) {
|
||||
throw new Error(`unsafe verifier path: ${JSON.stringify(path)}`);
|
||||
}
|
||||
const segments = path.split("/");
|
||||
if (segments.some((segment) => segment === "" || segment === "." || segment === "..")) {
|
||||
throw new Error(`unsafe verifier path: ${JSON.stringify(path)}`);
|
||||
}
|
||||
const absolute = resolve(root, ...segments);
|
||||
const fromRoot = relative(root, absolute);
|
||||
if (fromRoot.startsWith(`..${sep}`) || fromRoot === ".." || isAbsolute(fromRoot)) {
|
||||
throw new Error(`verifier path escapes root: ${JSON.stringify(path)}`);
|
||||
}
|
||||
const entry = await lstat(absolute);
|
||||
if (!entry.isFile() || entry.isSymbolicLink()) {
|
||||
throw new Error(`verifier input is not a regular file: ${path}`);
|
||||
}
|
||||
const canonical = await realpath(absolute);
|
||||
const canonicalRelative = relative(root, canonical);
|
||||
if (canonicalRelative.startsWith(`..${sep}`) || canonicalRelative === ".." || isAbsolute(canonicalRelative)) {
|
||||
throw new Error(`verifier input resolves outside root: ${path}`);
|
||||
}
|
||||
return absolute;
|
||||
}
|
||||
|
||||
export async function verifyEntries({ root, entries }) {
|
||||
const canonicalRoot = await realpath(root);
|
||||
const seen = new Set();
|
||||
const pythonPolicies = [];
|
||||
for (const entry of entries) {
|
||||
if (!entry || !allowedKinds.has(entry.kind) || typeof entry.path !== "string") {
|
||||
throw new Error("workspace verifier manifest contains an invalid entry");
|
||||
}
|
||||
const identity = `${entry.kind}\0${entry.path}`;
|
||||
if (seen.has(identity)) throw new Error(`workspace verifier manifest duplicates: ${entry.path}`);
|
||||
seen.add(identity);
|
||||
const absolute = await safeFile(canonicalRoot, entry.path);
|
||||
const bytes = await readFile(absolute);
|
||||
let source;
|
||||
try {
|
||||
source = new TextDecoder("utf-8", { fatal: true }).decode(bytes);
|
||||
} catch {
|
||||
throw new Error(`${entry.path}: input is not valid UTF-8`);
|
||||
}
|
||||
if (source.includes("\0")) throw new Error(`${entry.path}: NUL byte is forbidden`);
|
||||
if (entry.kind === "policy_text") {
|
||||
validatePolicySource(source, entry.path);
|
||||
if (/\.pyw?$/iu.test(entry.path) && !isPolicyImplementationException(entry.path, "revision-state")) {
|
||||
pythonPolicies.push({ label: entry.path, source });
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (entry.kind === "workspace_descriptor") {
|
||||
validateWorkspaceSource(source, entry.path, { requireWorkspace: true });
|
||||
continue;
|
||||
}
|
||||
for (const candidate of extractScriptDocuments(source, entry.path)) {
|
||||
validateWorkspaceSource(candidate.source, candidate.label, {
|
||||
requireWorkspace: false,
|
||||
expandable: candidate.expandable,
|
||||
path: entry.path,
|
||||
rawBlock: candidate.rawBlock,
|
||||
});
|
||||
}
|
||||
}
|
||||
validatePythonRevisionStates(pythonPolicies);
|
||||
}
|
||||
|
||||
export function decodeManifest(bytes) {
|
||||
const fields = bytes.toString("utf8").split("\0");
|
||||
if (fields.at(-1) !== "") throw new Error("workspace verifier manifest is not NUL-terminated");
|
||||
fields.pop();
|
||||
if (fields.length % 2 !== 0) throw new Error("workspace verifier manifest has an incomplete record");
|
||||
const entries = [];
|
||||
for (let index = 0; index < fields.length; index += 2) {
|
||||
entries.push({ kind: fields[index], path: fields[index + 1] });
|
||||
}
|
||||
return entries;
|
||||
}
|
||||
|
||||
function cliArguments(argv) {
|
||||
let root;
|
||||
let manifest;
|
||||
for (let index = 0; index < argv.length; index += 1) {
|
||||
const option = argv[index];
|
||||
const value = argv[index + 1];
|
||||
if ((option === "--root" || option === "--manifest") && value !== undefined) {
|
||||
if (option === "--root" && root === undefined) root = value;
|
||||
else if (option === "--manifest" && manifest === undefined) manifest = value;
|
||||
else throw new Error(`duplicate or invalid option: ${option}`);
|
||||
index += 1;
|
||||
} else {
|
||||
throw new Error(`unknown or incomplete option: ${option}`);
|
||||
}
|
||||
}
|
||||
if (root === undefined || manifest === undefined) {
|
||||
throw new Error("usage: verify-workspace-descriptor-files.mjs --root ROOT --manifest NUL_FILE");
|
||||
}
|
||||
return { root, manifest };
|
||||
}
|
||||
|
||||
async function main(argv) {
|
||||
const { root, manifest } = cliArguments(argv);
|
||||
const manifestEntry = await lstat(manifest);
|
||||
if (!manifestEntry.isFile() || manifestEntry.isSymbolicLink()) {
|
||||
throw new Error("workspace verifier manifest is not a regular file");
|
||||
}
|
||||
const entries = decodeManifest(await readFile(manifest));
|
||||
await verifyEntries({ root, entries });
|
||||
}
|
||||
|
||||
if (process.argv[1] && pathToFileURL(resolve(process.argv[1])).href === import.meta.url) {
|
||||
main(process.argv.slice(2)).catch((error) => {
|
||||
console.error(error instanceof Error ? error.message : String(error));
|
||||
process.exitCode = 1;
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,979 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { mkdtemp, mkdir, readFile, rm, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { dirname, join } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import test from "node:test";
|
||||
|
||||
import { extractScriptDocuments, verifyEntries } from "./verify-workspace-descriptor-files.mjs";
|
||||
|
||||
const repositoryRoot = fileURLToPath(new URL("../..", import.meta.url));
|
||||
const canonicalDescriptor = await readFile(join(repositoryRoot, "deploy/workspaces/example.yaml"), "utf8");
|
||||
|
||||
async function fixture(t) {
|
||||
const root = await mkdtemp(join(tmpdir(), "thoth-workspace-yaml-verifier-"));
|
||||
t.after(() => rm(root, { recursive: true, force: true }));
|
||||
return root;
|
||||
}
|
||||
|
||||
async function put(root, path, content) {
|
||||
await mkdir(dirname(join(root, path)), { recursive: true });
|
||||
await writeFile(join(root, path), content);
|
||||
}
|
||||
|
||||
function entry(kind, path) {
|
||||
return { kind, path };
|
||||
}
|
||||
|
||||
function bashN(root, path) {
|
||||
execFileSync("/bin/bash", ["-n", join(root, path)], { stdio: "pipe" });
|
||||
}
|
||||
|
||||
function replaceWorkspaceKeys(source, workspaceKey, schemaLine) {
|
||||
return source
|
||||
.replace(/^workspace:$/m, workspaceKey)
|
||||
.replace(/^ schema_version: 4$/m, schemaLine);
|
||||
}
|
||||
|
||||
test("production parser accepts semantic v4 with quoted Unicode/tagged keys and spacing", async (t) => {
|
||||
const root = await fixture(t);
|
||||
const unicode = replaceWorkspaceKeys(
|
||||
canonicalDescriptor,
|
||||
'"\\u0077orkspace" :',
|
||||
' "\\u0073chema_version" : 4',
|
||||
);
|
||||
const tagged = replaceWorkspaceKeys(
|
||||
canonicalDescriptor,
|
||||
"!!str workspace :",
|
||||
" !!str schema_version : 4",
|
||||
);
|
||||
await put(root, "deploy/workspaces/unicode.yaml", unicode);
|
||||
await put(root, "deploy/workspaces/tagged.yaml", tagged);
|
||||
await verifyEntries({
|
||||
root,
|
||||
entries: [
|
||||
entry("workspace_descriptor", "deploy/workspaces/unicode.yaml"),
|
||||
entry("workspace_descriptor", "deploy/workspaces/tagged.yaml"),
|
||||
],
|
||||
});
|
||||
});
|
||||
|
||||
test("production parser rejects fancy keys with every non-v4 or ambiguous value", async (t) => {
|
||||
const invalid = [
|
||||
["unicode-v2", '"\\u0077orkspace" :', ' "\\u0073chema_version" : 2'],
|
||||
["unicode-v3", '"\\u0077orkspace" :', ' "\\u0073chema_version" : 3'],
|
||||
["tagged-leading-zero", "!!str workspace :", " !!str schema_version : 03"],
|
||||
["hexadecimal", "workspace :", " schema_version : 0x3"],
|
||||
["multiline", "workspace :", " schema_version : >\n 4"],
|
||||
["duplicate", "workspace :", " schema_version : 4\n schema_version: 4"],
|
||||
["inline", "workspace: { schema_version: 4 }", " schema_version: 4"],
|
||||
];
|
||||
for (const [name, workspaceKey, schemaLine] of invalid) {
|
||||
await t.test(name, async () => {
|
||||
const root = await mkdtemp(join(tmpdir(), `thoth-workspace-yaml-${name}-`));
|
||||
try {
|
||||
const source = replaceWorkspaceKeys(canonicalDescriptor, workspaceKey, schemaLine);
|
||||
const path = `deploy/workspaces/${name}.yaml`;
|
||||
await put(root, path, source);
|
||||
await assert.rejects(
|
||||
verifyEntries({ root, entries: [entry("workspace_descriptor", path)] }),
|
||||
/workspace descriptor/i,
|
||||
);
|
||||
} finally {
|
||||
await rm(root, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
test("Bash embedded workspace mappings are rejected while tracked-fixture-only bundles pass", async (t) => {
|
||||
const root = await fixture(t);
|
||||
const validScript = [
|
||||
"#!/usr/bin/env bash",
|
||||
"cat <<'WORKSPACE_YAML'",
|
||||
canonicalDescriptor.trimEnd(),
|
||||
"WORKSPACE_YAML",
|
||||
"cat <<'BUNDLE_YAML'",
|
||||
"bundle:",
|
||||
" name: deploy",
|
||||
"schema_version: 1",
|
||||
"job:",
|
||||
" state: operational",
|
||||
"BUNDLE_YAML",
|
||||
"",
|
||||
].join("\n");
|
||||
await put(root, "scripts/operator-smoke.sh", validScript);
|
||||
await assert.rejects(
|
||||
verifyEntries({ root, entries: [entry("deployment_script", "scripts/operator-smoke.sh")] }),
|
||||
/embedded workspace descriptor/i,
|
||||
);
|
||||
|
||||
const bundleScript = validScript.replace(canonicalDescriptor.trimEnd(), "job:\n name: deploy");
|
||||
await put(root, "scripts/operator-smoke.sh", bundleScript);
|
||||
await verifyEntries({
|
||||
root,
|
||||
entries: [entry("deployment_script", "scripts/operator-smoke.sh")],
|
||||
});
|
||||
});
|
||||
|
||||
test("PowerShell embedded workspace mappings are rejected while bundle-only strings pass", async (t) => {
|
||||
const root = await fixture(t);
|
||||
const source = [
|
||||
"$workspace = @'",
|
||||
canonicalDescriptor.replace(" schema_version: 4", " schema_version: 0x2").trimEnd(),
|
||||
"'@",
|
||||
'$bundle = @"',
|
||||
"bundle:",
|
||||
" schema_version: 1",
|
||||
'"@',
|
||||
"",
|
||||
].join("\n");
|
||||
await put(root, "scripts/operator.ps1", source);
|
||||
await assert.rejects(
|
||||
verifyEntries({ root, entries: [entry("deployment_script", "scripts/operator.ps1")] }),
|
||||
/workspace descriptor/i,
|
||||
);
|
||||
});
|
||||
|
||||
test("workspace descriptor family entries require a top-level workspace", async (t) => {
|
||||
const root = await fixture(t);
|
||||
await put(root, "scripts/fixtures/workspace-registry-future.yaml", "bundle:\n schema_version: 4\n");
|
||||
await assert.rejects(
|
||||
verifyEntries({
|
||||
root,
|
||||
entries: [entry("workspace_descriptor", "scripts/fixtures/workspace-registry-future.yaml")],
|
||||
}),
|
||||
/top-level workspace/i,
|
||||
);
|
||||
});
|
||||
|
||||
|
||||
test("script scalar workspace remains a bundle even with descriptor-like siblings", async (t) => {
|
||||
const root = await fixture(t);
|
||||
const path = "scripts/job-smoke.sh";
|
||||
const job = [
|
||||
"#!/usr/bin/env bash",
|
||||
"cat <<'JOB-YAML'",
|
||||
"job: refresh",
|
||||
"workspace: analytics",
|
||||
"schema_version: 2",
|
||||
"state: operational",
|
||||
"JOB-YAML",
|
||||
"",
|
||||
].join("\n");
|
||||
await put(root, path, job);
|
||||
bashN(root, path);
|
||||
await verifyEntries({ root, entries: [entry("deployment_script", path)] });
|
||||
|
||||
const bundles = [
|
||||
job.replace("job: refresh", "dwh:\n engine: postgres"),
|
||||
job.replace("job: refresh", "evidence:\n source: bundle"),
|
||||
];
|
||||
for (const bundle of bundles) {
|
||||
await put(root, path, bundle);
|
||||
bashN(root, path);
|
||||
await verifyEntries({ root, entries: [entry("deployment_script", path)] });
|
||||
}
|
||||
});
|
||||
|
||||
test("standalone descriptor files require workspace to be a mapping", async (t) => {
|
||||
const root = await fixture(t);
|
||||
const path = "scripts/fixtures/workspace-registry-scalar.yaml";
|
||||
await put(root, path, "workspace: analytics\nschema_version: 4\n");
|
||||
await assert.rejects(
|
||||
verifyEntries({ root, entries: [entry("workspace_descriptor", path)] }),
|
||||
/workspace.*mapping/i,
|
||||
);
|
||||
});
|
||||
|
||||
test("Bash extractor supports hyphen, digit, escaped delimiters, and tab stripping", async (t) => {
|
||||
const root = await fixture(t);
|
||||
const cases = [
|
||||
{
|
||||
name: "hyphen-v2",
|
||||
opener: "cat <<'WORKSPACE-YAML'",
|
||||
delimiter: "WORKSPACE-YAML",
|
||||
descriptor: canonicalDescriptor.replace(" schema_version: 4", " schema_version: 2"),
|
||||
rejected: true,
|
||||
},
|
||||
{
|
||||
name: "digit-v4",
|
||||
opener: "cat <<2YAML",
|
||||
delimiter: "2YAML",
|
||||
descriptor: canonicalDescriptor,
|
||||
rejected: true,
|
||||
},
|
||||
{
|
||||
name: "escaped-v2",
|
||||
opener: "cat <<WORKSPACE\\-YAML",
|
||||
delimiter: "WORKSPACE-YAML",
|
||||
descriptor: canonicalDescriptor.replace(" schema_version: 4", " schema_version: 2"),
|
||||
rejected: true,
|
||||
},
|
||||
{
|
||||
name: "tab-strip-v4",
|
||||
opener: "cat <<-'TAB-YAML'",
|
||||
delimiter: "\tTAB-YAML",
|
||||
descriptor: canonicalDescriptor.split("\n").map((line) => `\t${line}`).join("\n"),
|
||||
rejected: true,
|
||||
},
|
||||
];
|
||||
for (const item of cases) {
|
||||
await t.test(item.name, async () => {
|
||||
const path = `scripts/${item.name}-smoke.sh`;
|
||||
const source = ["#!/usr/bin/env bash", item.opener, item.descriptor.trimEnd(), item.delimiter, ""].join("\n");
|
||||
await put(root, path, source);
|
||||
bashN(root, path);
|
||||
const verification = verifyEntries({ root, entries: [entry("deployment_script", path)] });
|
||||
if (item.rejected) await assert.rejects(verification, /workspace descriptor/i);
|
||||
else await verification;
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
test("unsupported Bash heredoc opener fails closed while a bundle heredoc stays allowed", async (t) => {
|
||||
const root = await fixture(t);
|
||||
const unsupportedPath = "scripts/unsupported-smoke.sh";
|
||||
const unsupported = [
|
||||
"#!/usr/bin/env bash",
|
||||
"cat <<$DELIMITER",
|
||||
canonicalDescriptor.trimEnd(),
|
||||
"$DELIMITER",
|
||||
"",
|
||||
].join("\n");
|
||||
await put(root, unsupportedPath, unsupported);
|
||||
bashN(root, unsupportedPath);
|
||||
await assert.rejects(
|
||||
verifyEntries({ root, entries: [entry("deployment_script", unsupportedPath)] }),
|
||||
/unsupported Bash heredoc opener/i,
|
||||
);
|
||||
|
||||
const bundlePath = "scripts/bundle-smoke.sh";
|
||||
const bundle = [
|
||||
"#!/usr/bin/env bash",
|
||||
"cat <<'BUNDLE-YAML'",
|
||||
"job: refresh",
|
||||
"workspace: analytics",
|
||||
"schema_version: 1",
|
||||
"state: operational",
|
||||
"BUNDLE-YAML",
|
||||
"",
|
||||
].join("\n");
|
||||
await put(root, bundlePath, bundle);
|
||||
bashN(root, bundlePath);
|
||||
await verifyEntries({ root, entries: [entry("deployment_script", bundlePath)] });
|
||||
});
|
||||
|
||||
|
||||
test("non-stripping heredoc close requires an exact physical delimiter line", async (t) => {
|
||||
const root = await fixture(t);
|
||||
const path = "scripts/trailing-close-smoke.sh";
|
||||
const source = [
|
||||
"#!/usr/bin/env bash",
|
||||
"cat <<'---'",
|
||||
"--- ",
|
||||
canonicalDescriptor.replace(" schema_version: 4", " schema_version: 2").trimEnd(),
|
||||
"---",
|
||||
"",
|
||||
].join("\n");
|
||||
await put(root, path, source);
|
||||
bashN(root, path);
|
||||
await assert.rejects(
|
||||
verifyEntries({ root, entries: [entry("deployment_script", path)] }),
|
||||
/workspace descriptor/i,
|
||||
);
|
||||
});
|
||||
|
||||
test("delimiter-like body lines remain content until a real exact close", async (t) => {
|
||||
const root = await fixture(t);
|
||||
const path = "scripts/delimiter-content-smoke.sh";
|
||||
const source = [
|
||||
"#!/usr/bin/env bash",
|
||||
"cat <<'END'",
|
||||
"END ",
|
||||
" END",
|
||||
"job: refresh",
|
||||
"workspace: analytics",
|
||||
"schema_version: 1",
|
||||
"END",
|
||||
"",
|
||||
].join("\n");
|
||||
await put(root, path, source);
|
||||
bashN(root, path);
|
||||
const [candidate] = extractScriptDocuments(source, path);
|
||||
assert.match(candidate.source, /^END \n END\n/u);
|
||||
await verifyEntries({ root, entries: [entry("deployment_script", path)] });
|
||||
});
|
||||
|
||||
|
||||
test("double-quoted non-special backslash is preserved in the delimiter", async (t) => {
|
||||
const root = await fixture(t);
|
||||
const path = "scripts/double-quoted-nonspecial-smoke.sh";
|
||||
const source = [
|
||||
"#!/usr/bin/env bash",
|
||||
'cat <<"\\---"',
|
||||
"---",
|
||||
canonicalDescriptor.replace(" schema_version: 4", " schema_version: 2").trimEnd(),
|
||||
"\\---",
|
||||
"",
|
||||
].join("\n");
|
||||
await put(root, path, source);
|
||||
bashN(root, path);
|
||||
assert.match(execFileSync("/bin/bash", [join(root, path)], { encoding: "utf8" }), /schema_version: 2/u);
|
||||
await assert.rejects(
|
||||
verifyEntries({ root, entries: [entry("deployment_script", path)] }),
|
||||
/workspace descriptor/i,
|
||||
);
|
||||
});
|
||||
|
||||
test("double-quoted delimiter quote removal matches Bash special escapes", async (t) => {
|
||||
const root = await fixture(t);
|
||||
const cases = [
|
||||
["dollar", 'cat <<"DOL\\$LAR"', "DOL$LAR"],
|
||||
["backtick", 'cat <<"TIC\\`K"', "TIC`K"],
|
||||
["quote", 'cat <<"QUO\\\"TE"', 'QUO"TE'],
|
||||
["backslash", 'cat <<"SLA\\\\SH"', "SLA\\SH"],
|
||||
["newline", 'cat <<"LINE\\\nBREAK"', "LINEBREAK"],
|
||||
["nonspecial", 'cat <<"NON\\-SPECIAL"', "NON\\-SPECIAL"],
|
||||
];
|
||||
for (const [name, opener, close] of cases) {
|
||||
const path = `scripts/double-quoted-${name}-smoke.sh`;
|
||||
const source = ["#!/usr/bin/env bash", opener, "job: refresh", close, ""].join("\n");
|
||||
await put(root, path, source);
|
||||
bashN(root, path);
|
||||
assert.equal(execFileSync("/bin/bash", [join(root, path)], { encoding: "utf8" }), "job: refresh\n");
|
||||
assert.equal(extractScriptDocuments(source, path)[0].source, "job: refresh\n");
|
||||
await verifyEntries({ root, entries: [entry("deployment_script", path)] });
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
test("split heredoc operator continuation cannot bypass v2 validation", async (t) => {
|
||||
const root = await fixture(t);
|
||||
const path = "scripts/split-operator-smoke.sh";
|
||||
const source = [
|
||||
"#!/usr/bin/env bash",
|
||||
"cat <\\",
|
||||
"<'YAML'",
|
||||
canonicalDescriptor.replace(" schema_version: 4", " schema_version: 2").trimEnd(),
|
||||
"YAML",
|
||||
"",
|
||||
].join("\n");
|
||||
await put(root, path, source);
|
||||
bashN(root, path);
|
||||
assert.match(execFileSync("/bin/bash", [join(root, path)], { encoding: "utf8" }), /schema_version: 2/u);
|
||||
await assert.rejects(
|
||||
verifyEntries({ root, entries: [entry("deployment_script", path)] }),
|
||||
/workspace descriptor/i,
|
||||
);
|
||||
});
|
||||
|
||||
test("multiple opener continuations are joined before heredoc discovery", async (t) => {
|
||||
const root = await fixture(t);
|
||||
const path = "scripts/multiple-continuation-smoke.sh";
|
||||
const source = [
|
||||
"#!/usr/bin/env bash",
|
||||
"cat \\",
|
||||
"<\\",
|
||||
"<'YAML'",
|
||||
"job: refresh",
|
||||
"workspace: analytics",
|
||||
"YAML",
|
||||
"",
|
||||
].join("\n");
|
||||
await put(root, path, source);
|
||||
bashN(root, path);
|
||||
assert.equal(
|
||||
execFileSync("/bin/bash", [join(root, path)], { encoding: "utf8" }),
|
||||
"job: refresh\nworkspace: analytics\n",
|
||||
);
|
||||
const [candidate] = extractScriptDocuments(source, path);
|
||||
assert.equal(candidate.label, `${path}:5 Bash heredoc`);
|
||||
assert.equal(candidate.source, "job: refresh\nworkspace: analytics\n");
|
||||
await verifyEntries({ root, entries: [entry("deployment_script", path)] });
|
||||
});
|
||||
|
||||
test("backslash-newline inside single quotes is not removed", async (t) => {
|
||||
const root = await fixture(t);
|
||||
const path = "scripts/single-quoted-noncontinuation-smoke.sh";
|
||||
const source = [
|
||||
"#!/usr/bin/env bash",
|
||||
"printf '%s' 'literal\\",
|
||||
"continued'",
|
||||
"cat <<'YAML'",
|
||||
"job: refresh",
|
||||
"workspace: analytics",
|
||||
"YAML",
|
||||
"",
|
||||
].join("\n");
|
||||
await put(root, path, source);
|
||||
bashN(root, path);
|
||||
assert.equal(
|
||||
execFileSync("/bin/bash", [join(root, path)], { encoding: "utf8" }),
|
||||
"literal\\\ncontinuedjob: refresh\nworkspace: analytics\n",
|
||||
);
|
||||
const [candidate] = extractScriptDocuments(source, path);
|
||||
assert.equal(candidate.label, `${path}:5 Bash heredoc`);
|
||||
await verifyEntries({ root, entries: [entry("deployment_script", path)] });
|
||||
});
|
||||
|
||||
|
||||
test("PowerShell comment backslash cannot hide a following v2 here-string", async (t) => {
|
||||
const root = await fixture(t);
|
||||
const path = "scripts/powershell-comment-smoke.ps1";
|
||||
const source = [
|
||||
"# harmless PowerShell comment \\",
|
||||
"$workspace = @'",
|
||||
canonicalDescriptor.replace(" schema_version: 4", " schema_version: 2").trimEnd(),
|
||||
"'@",
|
||||
"",
|
||||
].join("\n");
|
||||
await put(root, path, source);
|
||||
await assert.rejects(
|
||||
verifyEntries({ root, entries: [entry("deployment_script", path)] }),
|
||||
/workspace descriptor/i,
|
||||
);
|
||||
});
|
||||
|
||||
test("PowerShell dialect accepts normal v4 and non-workspace bundle here-strings", async (t) => {
|
||||
const root = await fixture(t);
|
||||
const path = "scripts/powershell-valid-smoke.ps1";
|
||||
const source = [
|
||||
"$workspace = @'",
|
||||
canonicalDescriptor.trimEnd(),
|
||||
"'@",
|
||||
"$bundle = @'",
|
||||
"evidence:",
|
||||
" source: bundle",
|
||||
"schema_version: 2",
|
||||
"'@",
|
||||
"",
|
||||
].join("\n");
|
||||
await put(root, path, source);
|
||||
await assert.rejects(
|
||||
verifyEntries({ root, entries: [entry("deployment_script", path)] }),
|
||||
/embedded workspace descriptor/i,
|
||||
);
|
||||
|
||||
const bundleOnly = [
|
||||
"$bundle = @'",
|
||||
"evidence:",
|
||||
" source: bundle",
|
||||
"schema_version: 2",
|
||||
"'@",
|
||||
"",
|
||||
].join("\n");
|
||||
await put(root, path, bundleOnly);
|
||||
await verifyEntries({ root, entries: [entry("deployment_script", path)] });
|
||||
});
|
||||
|
||||
test("unknown deployment script dialect fails closed", async (t) => {
|
||||
const root = await fixture(t);
|
||||
const path = "scripts/operator-smoke.cmd";
|
||||
await put(root, path, "echo harmless\n");
|
||||
await assert.rejects(
|
||||
verifyEntries({ root, entries: [entry("deployment_script", path)] }),
|
||||
/unknown deployment script dialect/i,
|
||||
);
|
||||
});
|
||||
|
||||
|
||||
test("PowerShell cast and concatenation openers cannot hide embedded descriptors", async (t) => {
|
||||
const root = await fixture(t);
|
||||
for (const [name, opener] of [["cast", "[string]@'"], ["concat", "+@'"]]) {
|
||||
const path = `scripts/powershell-${name}-smoke.ps1`;
|
||||
const source = [opener, canonicalDescriptor.trimEnd(), "'@", ""].join("\n");
|
||||
await put(root, path, source);
|
||||
await assert.rejects(
|
||||
verifyEntries({ root, entries: [entry("deployment_script", path)] }),
|
||||
/embedded workspace descriptor/i,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test("expandable YAML interpolation that can hide a workspace descriptor fails closed", async (t) => {
|
||||
const root = await fixture(t);
|
||||
const cases = [
|
||||
["braced-key", "${key}:\n schema_version: 4"],
|
||||
["plain-key", "$key:\n schema_version: 4"],
|
||||
["quoted-key", '"$key" :\n schema_version: 4'],
|
||||
["subexpression-key", "$($key):\n schema_version: 4"],
|
||||
["version", "workspace:\n schema_version: $version"],
|
||||
];
|
||||
for (const [name, body] of cases) {
|
||||
const path = `scripts/powershell-interpolation-${name}.ps1`;
|
||||
await put(root, path, [`$yaml = @\"`, body, `\"@`, ""].join("\n"));
|
||||
await assert.rejects(
|
||||
verifyEntries({ root, entries: [entry("deployment_script", path)] }),
|
||||
/interpolation|embedded workspace descriptor/i,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test("Bash heredoc discovery ignores quoted, comment, here-string, and arithmetic tokens", async (t) => {
|
||||
const root = await fixture(t);
|
||||
const path = "scripts/bash-lexer-smoke.sh";
|
||||
const source = [
|
||||
"#!/usr/bin/env bash",
|
||||
`printf '%s\\n' \"cat <<'QUOTED'\"`,
|
||||
`printf '%s\\n' 'cat <<\"SINGLE\"'`,
|
||||
"# cat <<'COMMENT'",
|
||||
"value=$((1 << 2))",
|
||||
`cat <<< \"not a heredoc\"`,
|
||||
"cat <<'YAML'",
|
||||
"job: refresh",
|
||||
"YAML",
|
||||
"",
|
||||
].join("\n");
|
||||
await put(root, path, source);
|
||||
bashN(root, path);
|
||||
const extracted = extractScriptDocuments(source, path);
|
||||
assert.equal(extracted.length, 1);
|
||||
assert.equal(extracted[0].source, "job: refresh\n");
|
||||
await verifyEntries({ root, entries: [entry("deployment_script", path)] });
|
||||
});
|
||||
|
||||
test("UTF-8 decoding is fatal but literal replacement characters are valid text", async (t) => {
|
||||
const root = await fixture(t);
|
||||
const validPath = "deploy/workspaces/replacement.yaml";
|
||||
await put(root, validPath, `${canonicalDescriptor}# literal replacement: �\n`);
|
||||
await verifyEntries({ root, entries: [entry("workspace_descriptor", validPath)] });
|
||||
|
||||
const invalidPath = "deploy/workspaces/malformed.yaml";
|
||||
await mkdir(dirname(join(root, invalidPath)), { recursive: true });
|
||||
await writeFile(join(root, invalidPath), Buffer.concat([Buffer.from(canonicalDescriptor), Buffer.from([0xff])]));
|
||||
await assert.rejects(
|
||||
verifyEntries({ root, entries: [entry("workspace_descriptor", invalidPath)] }),
|
||||
/valid UTF-8/i,
|
||||
);
|
||||
});
|
||||
|
||||
|
||||
test("unmarked expandable Bash YAML cannot generate descriptor keys or values at runtime", async (t) => {
|
||||
const root = await fixture(t);
|
||||
const cases = [
|
||||
["quoted", '"$key" :'],
|
||||
["command", "$(printf workspace):"],
|
||||
["braced", "${key}:"],
|
||||
["plain", "$key:"],
|
||||
];
|
||||
for (const [name, generatedKey] of cases) {
|
||||
const path = `scripts/bash-dynamic-${name}.sh`;
|
||||
const source = [
|
||||
"#!/usr/bin/env bash",
|
||||
"key=workspace",
|
||||
"cat <<YAML",
|
||||
generatedKey,
|
||||
" schema_version: 4",
|
||||
"YAML",
|
||||
"",
|
||||
].join("\n");
|
||||
await put(root, path, source);
|
||||
bashN(root, path);
|
||||
assert.match(execFileSync("/bin/bash", [join(root, path)], { encoding: "utf8" }), /workspace/u);
|
||||
await assert.rejects(
|
||||
verifyEntries({ root, entries: [entry("deployment_script", path)] }),
|
||||
/exact-content reviewed allowlist/i,
|
||||
);
|
||||
}
|
||||
|
||||
const valuePath = "scripts/bash-dynamic-value.sh";
|
||||
const valueSource = [
|
||||
"#!/usr/bin/env bash",
|
||||
"version=3",
|
||||
"cat <<YAML",
|
||||
"workspace:",
|
||||
" schema_version: $version",
|
||||
"YAML",
|
||||
"",
|
||||
].join("\n");
|
||||
await put(root, valuePath, valueSource);
|
||||
bashN(root, valuePath);
|
||||
await assert.rejects(
|
||||
verifyEntries({ root, entries: [entry("deployment_script", valuePath)] }),
|
||||
/exact-content reviewed allowlist/i,
|
||||
);
|
||||
});
|
||||
|
||||
test("an in-band marker cannot authorize expandable content", async (t) => {
|
||||
const root = await fixture(t);
|
||||
for (const [path, source] of [
|
||||
["scripts/fake-marker.sh", [
|
||||
"#!/usr/bin/env bash",
|
||||
"# schema-v4-only: expandable-nonworkspace",
|
||||
"cat <<YAML",
|
||||
"${DESCRIPTOR}",
|
||||
"YAML",
|
||||
"",
|
||||
].join("\n")],
|
||||
["scripts/fake-marker.ps1", [
|
||||
"# schema-v4-only: expandable-nonworkspace",
|
||||
'$yaml = @"',
|
||||
"$descriptor",
|
||||
'"@',
|
||||
"",
|
||||
].join("\n")],
|
||||
]) {
|
||||
await put(root, path, source);
|
||||
await assert.rejects(
|
||||
verifyEntries({ root, entries: [entry("deployment_script", path)] }),
|
||||
/exact-content reviewed allowlist/,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test("current exact reviewed expandable blocks pass only at their trusted paths", async (t) => {
|
||||
const reviewedPaths = [
|
||||
"scripts/test-server-pi-state-topology.sh",
|
||||
"scripts/test-vector-backup-restore-safety.sh",
|
||||
"scripts/test-windows-clone-contract.ps1",
|
||||
"scripts/unified-deployment-smoke.sh",
|
||||
"scripts/vector-backup.sh",
|
||||
"scripts/vector-restore.sh",
|
||||
];
|
||||
await verifyEntries({
|
||||
root: repositoryRoot,
|
||||
entries: reviewedPaths.map((path) => entry("deployment_script", path)),
|
||||
});
|
||||
});
|
||||
|
||||
test("PowerShell tokenizer ignores opener text in comments and ordinary strings", async (t) => {
|
||||
const root = await fixture(t);
|
||||
const path = "scripts/powershell-lexical-context.ps1";
|
||||
const source = [
|
||||
"# example @'",
|
||||
'\"example @\'\"',
|
||||
"'example @\"'",
|
||||
"<# block @'",
|
||||
"still @\" #>",
|
||||
"$cast = [string]@'",
|
||||
"job: cast",
|
||||
"'@",
|
||||
"$concat = $cast +@'",
|
||||
"job: concat",
|
||||
"'@",
|
||||
"",
|
||||
].join("\n");
|
||||
await put(root, path, source);
|
||||
const extracted = extractScriptDocuments(source, path);
|
||||
assert.equal(extracted.length, 2);
|
||||
assert.deepEqual(extracted.map((item) => item.source), ["job: cast\n", "job: concat\n"]);
|
||||
await verifyEntries({ root, entries: [entry("deployment_script", path)] });
|
||||
});
|
||||
|
||||
|
||||
test("policy text rejects NUL and prescribed symbol substrings but permits lower-camel legacy identifiers", async (t) => {
|
||||
const root = await fixture(t);
|
||||
await put(root, "backend/src/nul.ts", Buffer.from("safe\0WorkspaceV2"));
|
||||
await assert.rejects(verifyEntries({ root, entries: [entry("policy_text", "backend/src/nul.ts")] }), /NUL byte/);
|
||||
|
||||
for (const [name, text] of [
|
||||
["compat", "type X = WorkspaceV2Compat;"],
|
||||
["mixed-prescribed", "type X = wOrKsPaCeV2;"],
|
||||
["lower-deprecated", "type X = deprecatedV2Descriptor;"],
|
||||
["upper-function", "WRITEMIGRATEDWORKSPACE(value);"],
|
||||
["adapter", "type X = LegacyWorkspaceAdapter;"],
|
||||
["lower", "type X = legacyworkspace;"],
|
||||
["mixed", "type X = LeGaCyWoRkSpAcE;"],
|
||||
]) {
|
||||
const path = `backend/src/${name}.ts`;
|
||||
await put(root, path, text);
|
||||
await assert.rejects(verifyEntries({ root, entries: [entry("policy_text", path)] }), /forbidden/);
|
||||
}
|
||||
await put(root, "backend/src/allowed.ts", "const legacyWorkspacePath = value;");
|
||||
await verifyEntries({ root, entries: [entry("policy_text", "backend/src/allowed.ts")] });
|
||||
});
|
||||
|
||||
test("revision-state structural scan permits only the exact historical decoder occurrence", async (t) => {
|
||||
const root = await fixture(t);
|
||||
const registry = "backend/src/workspaces/registry.ts";
|
||||
await put(root, registry, 'if (revision.state !== "operational") return;\n');
|
||||
await verifyEntries({ root, entries: [entry("policy_text", registry)] });
|
||||
|
||||
const variants = [
|
||||
'if (revision.state !== "operational") return;\nif (revision["state"] === value) return;\n',
|
||||
'if (workspaceRevision\n .state === value) return;\n',
|
||||
"if (selectedWorkspace [ 'state' ] === value) return;\n",
|
||||
];
|
||||
for (let index = 0; index < variants.length; index += 1) {
|
||||
const path = index === 0 ? registry : `frontend/src/revision-${index}.ts`;
|
||||
await put(root, path, variants[index]);
|
||||
await assert.rejects(verifyEntries({ root, entries: [entry("policy_text", path)] }), /revision-state/);
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
test("complete descriptors supplied only through Bash or PowerShell variables require exact review", async (t) => {
|
||||
const root = await fixture(t);
|
||||
const cases = [
|
||||
["scripts/variable-descriptor.sh", ["#!/usr/bin/env bash", "cat <<YAML", "${DESCRIPTOR}", "YAML", ""].join("\n")],
|
||||
["scripts/variable-descriptor.ps1", ['$yaml = @"', "$descriptor", '"@', ""].join("\n")],
|
||||
];
|
||||
for (const [path, source] of cases) {
|
||||
await put(root, path, source);
|
||||
await assert.rejects(
|
||||
verifyEntries({ root, entries: [entry("deployment_script", path)] }),
|
||||
/exact-content reviewed allowlist/,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
test("all Bash and PowerShell positional or special dollar expansions fail without exact review", async (t) => {
|
||||
const root = await fixture(t);
|
||||
const cases = [
|
||||
["scripts/positional.sh", "cat <<YAML\n$1\nYAML\n"],
|
||||
["scripts/all-args.sh", "cat <<YAML\n$@\nYAML\n"],
|
||||
["scripts/positional.ps1", '$yaml = @"\n$1\n"@\n'],
|
||||
];
|
||||
for (const [path, source] of cases) {
|
||||
await put(root, path, source);
|
||||
await assert.rejects(
|
||||
verifyEntries({ root, entries: [entry("deployment_script", path)] }),
|
||||
/exact-content reviewed allowlist/,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test("PowerShell backtick escapes hash and quote tokens without hiding a later real here-string", async (t) => {
|
||||
const root = await fixture(t);
|
||||
for (const [name, prefix] of [
|
||||
["escaped-hash", "Write-Output `# harmless"],
|
||||
["escaped-quote", 'Write-Output `" harmless'],
|
||||
]) {
|
||||
const path = `scripts/${name}.ps1`;
|
||||
const source = [prefix, "$yaml = @'", "workspace:", " schema_version: 2", "'@", ""].join("\n");
|
||||
await put(root, path, source);
|
||||
assert.equal(extractScriptDocuments(source, path).length, 1);
|
||||
await assert.rejects(
|
||||
verifyEntries({ root, entries: [entry("deployment_script", path)] }),
|
||||
/embedded workspace descriptor/,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test("TypeScript AST rejects comment-separated and destructured revision state", async (t) => {
|
||||
const root = await fixture(t);
|
||||
for (const [index, source] of [
|
||||
"const value = revision /*legacy*/ . state;",
|
||||
"const { state } = revision;",
|
||||
"const { state: oldState } = selectedWorkspace;",
|
||||
].entries()) {
|
||||
const path = `frontend/src/ast-revision-${index}.ts`;
|
||||
await put(root, path, source);
|
||||
await assert.rejects(verifyEntries({ root, entries: [entry("policy_text", path)] }), /revision-state/);
|
||||
}
|
||||
const registry = "backend/src/workspaces/registry.ts";
|
||||
await put(root, registry, 'if (revision.state !== "operational") return;\nconst { state } = revision;\n');
|
||||
await assert.rejects(verifyEntries({ root, entries: [entry("policy_text", registry)] }), /revision-state/);
|
||||
await put(root, "backend/src/unrelated.ts", "const { state } = lease; const jobState = job.state;");
|
||||
await verifyEntries({ root, entries: [entry("policy_text", "backend/src/unrelated.ts")] });
|
||||
});
|
||||
|
||||
|
||||
test("AST recognizes semantic state keys in every revision destructuring form", async (t) => {
|
||||
const root = await fixture(t);
|
||||
const cases = [
|
||||
["backend/src/computed.mts", 'const { ["state"]: oldState } = revision;'],
|
||||
["frontend/src/renamed.cts", 'const { "state": oldState = fallback } = workspaceRevision;'],
|
||||
["backend/scripts/template.TS", 'const { [`state`]: oldState } = selectedWorkspace;'],
|
||||
["scripts/parameter.txt", 'function read({ state: oldState = fallback } = revision) {}'],
|
||||
["scripts/assignment.sh", '({ state } = workspaceRevision);'],
|
||||
["scripts/computed-assignment.data", '({ ["state"]: oldState = fallback } = selectedWorkspace);'],
|
||||
];
|
||||
for (const [path, source] of cases) {
|
||||
await put(root, path, source);
|
||||
await assert.rejects(
|
||||
verifyEntries({ root, entries: [entry("policy_text", path)] }),
|
||||
/revision-state/,
|
||||
path,
|
||||
);
|
||||
}
|
||||
|
||||
const registry = "backend/src/workspaces/registry.ts";
|
||||
await put(root, registry, [
|
||||
'if (revision.state !== "operational") return;',
|
||||
'function read({ ["state"]: oldState } = revision) {}',
|
||||
"",
|
||||
].join("\n"));
|
||||
await assert.rejects(
|
||||
verifyEntries({ root, entries: [entry("policy_text", registry)] }),
|
||||
/revision-state/,
|
||||
);
|
||||
});
|
||||
|
||||
test("tolerant all-suffix AST scan ignores strings/comments and unrelated state", async (t) => {
|
||||
const root = await fixture(t);
|
||||
const path = "scripts/arbitrary.weird";
|
||||
await put(root, path, [
|
||||
'// const { state } = revision;',
|
||||
'"revision.state";',
|
||||
"'({ [\\\"state\\\"]: oldState } = selectedWorkspace)';",
|
||||
"const { state } = lease;",
|
||||
"const jobState = job.state;",
|
||||
"record.state = 'ready';",
|
||||
"",
|
||||
].join("\n"));
|
||||
await verifyEntries({ root, entries: [entry("policy_text", path)] });
|
||||
});
|
||||
|
||||
|
||||
test("computed revision destructuring keys fold parentheses assertions templates and string concatenation", async (t) => {
|
||||
const root = await fixture(t);
|
||||
const cases = [
|
||||
["backend/src/paren.ts", 'const { [("state")]: oldState } = revision;'],
|
||||
["backend/src/concat.ts", 'const { ["st" + "ate"]: oldState } = workspaceRevision;'],
|
||||
["frontend/src/template.ts", 'const { [`st${"ate"}`]: oldState } = selectedWorkspace;'],
|
||||
["scripts/assertion.data", 'const { [("st" as string) + (`ate` satisfies string)]: oldState } = revision;'],
|
||||
["scripts/assignment.txt", '({ ["st" + "ate"]: oldState } = selectedWorkspace);'],
|
||||
];
|
||||
for (const [path, source] of cases) {
|
||||
await put(root, path, source);
|
||||
await assert.rejects(verifyEntries({ root, entries: [entry("policy_text", path)] }), /revision-state/, path);
|
||||
}
|
||||
|
||||
const registry = "backend/src/workspaces/registry.ts";
|
||||
for (const injected of [
|
||||
'const { [("state")]: oldState } = revision;',
|
||||
'({ ["st" + "ate"]: oldState } = revision);',
|
||||
]) {
|
||||
await put(root, registry, `if (revision.state !== "operational") return;\n${injected}\n`);
|
||||
await assert.rejects(verifyEntries({ root, entries: [entry("policy_text", registry)] }), /revision-state/);
|
||||
}
|
||||
});
|
||||
|
||||
test("polyglot masking and JSX syntax prevent comment and string false positives", async (t) => {
|
||||
const root = await fixture(t);
|
||||
const passing = [
|
||||
["backend/scripts/comment.py", '# revision.state\nvalue = "revision.state"\ntext = """selectedWorkspace.state"""\n'],
|
||||
["scripts/comment.ps1", '# revision.state\n<# workspaceRevision.state #>\n$value = "revision.state"\n'],
|
||||
["scripts/comment.sh", '# revision.state\nprintf \'%s\\n\' "selectedWorkspace.state"\n'],
|
||||
["frontend/src/content.tsx", 'export const view = <div>revision.state</div>;'],
|
||||
["frontend/src/attribute.tsx", 'export const view = <div title="revision.state" />;'],
|
||||
["frontend/src/expression.tsx", 'export const view = <div>{"revision.state"}</div>;'],
|
||||
["scripts/arbitrary.data", 'title: "revision.state"\n# const { state } = revision\nlease:\n state: ready\n'],
|
||||
];
|
||||
for (const [path, source] of passing) {
|
||||
await put(root, path, source);
|
||||
await verifyEntries({ root, entries: [entry("policy_text", path)] });
|
||||
}
|
||||
|
||||
for (const [path, source] of [
|
||||
["scripts/code.txt", "const { state } = revision;"],
|
||||
["scripts/code.data", '({ ["st" + "ate"]: oldState } = workspaceRevision);'],
|
||||
]) {
|
||||
await put(root, path, source);
|
||||
await assert.rejects(verifyEntries({ root, entries: [entry("policy_text", path)] }), /revision-state/);
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
test("rest bindings and dynamic computed keys are not semantic state-property access", async (t) => {
|
||||
const root = await fixture(t);
|
||||
const cases = [
|
||||
["backend/src/rest.ts", "const { ...state } = revision;"],
|
||||
["frontend/src/renamed.ts", "const { other: state } = workspaceRevision;"],
|
||||
["scripts/dynamic.txt", "const { [state]: value } = selectedWorkspace;"],
|
||||
["scripts/dynamic-assignment.data", "({ [state]: value } = revision);"],
|
||||
["scripts/spread-assignment.data", "({ ...state } = workspaceRevision);"],
|
||||
];
|
||||
for (const [path, source] of cases) {
|
||||
await put(root, path, source);
|
||||
await verifyEntries({ root, entries: [entry("policy_text", path)] });
|
||||
}
|
||||
});
|
||||
|
||||
test("polyglot code remains structural across shell Python PowerShell YAML TSX and JSX", async (t) => {
|
||||
const root = await fixture(t);
|
||||
const failing = [
|
||||
["scripts/code.sh", "value=revision.state\n"],
|
||||
["scripts/code.ps1", "$value = workspaceRevision.state\n"],
|
||||
["backend/scripts/code.py", "value = selectedWorkspace.state\n"],
|
||||
["scripts/code.yaml", "value: revision.state\n"],
|
||||
["frontend/src/code.tsx", "export const view = <div>{revision.state}</div>;"],
|
||||
["frontend/src/code.jsx", "export const view = <div>{workspaceRevision.state}</div>;"],
|
||||
];
|
||||
for (const [path, source] of failing) {
|
||||
await put(root, path, source);
|
||||
await assert.rejects(verifyEntries({ root, entries: [entry("policy_text", path)] }), /revision-state/, path);
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
test("PowerShell executable subexpressions expose dollar-prefixed revision access", async (t) => {
|
||||
const root = await fixture(t);
|
||||
const failing = [
|
||||
["scripts/ps-property.ps1", 'Write-Output "revision: $($revision.state)"\n'],
|
||||
["scripts/ps-element.ps1", 'Write-Output "$($workspaceRevision[\'state\'])"\n'],
|
||||
["scripts/ps-workspace.ps1", '$value = $workspaceRevision.state\n'],
|
||||
["scripts/ps-nested.ps1", 'Write-Output "$($($revision.state))"\n'],
|
||||
];
|
||||
for (const [path, source] of failing) {
|
||||
await put(root, path, source);
|
||||
await assert.rejects(verifyEntries({ root, entries: [entry("policy_text", path)] }), /revision-state/, path);
|
||||
}
|
||||
const passing = [
|
||||
'# $revision.state\nWrite-Output "revision.state"\n',
|
||||
"Write-Output '$selectedWorkspace[\"state\"]'\n",
|
||||
];
|
||||
for (let index = 0; index < passing.length; index += 1) {
|
||||
const path = `scripts/ps-literal-${index}.ps1`;
|
||||
await put(root, path, passing[index]);
|
||||
await verifyEntries({ root, entries: [entry("policy_text", path)] });
|
||||
}
|
||||
});
|
||||
|
||||
test("Python f-string fields expose revision access while literal text remains masked", async (t) => {
|
||||
const root = await fixture(t);
|
||||
const failing = [
|
||||
["backend/scripts/f-property.py", 'value = f"{revision.state}"\n'],
|
||||
["backend/scripts/fr-element.py", 'value = fr"{workspaceRevision[\'state\']}"\n'],
|
||||
["backend/scripts/rf-element.py", 'value = rf"prefix {selectedWorkspace[\"state\"]}"\n'],
|
||||
];
|
||||
for (const [path, source] of failing) {
|
||||
await put(root, path, source);
|
||||
await assert.rejects(verifyEntries({ root, entries: [entry("policy_text", path)] }), /revision-state/, path);
|
||||
}
|
||||
const passing = [
|
||||
'value = f"revision.state"\n',
|
||||
'value = f"{{revision.state}}"\n',
|
||||
'value = "revision.state"\n',
|
||||
'value = r"workspaceRevision.state"\n',
|
||||
'value = """selectedWorkspace.state"""\n',
|
||||
'value = r"""revision.state"""\n',
|
||||
];
|
||||
for (let index = 0; index < passing.length; index += 1) {
|
||||
const path = `backend/scripts/python-literal-${index}.py`;
|
||||
await put(root, path, passing[index]);
|
||||
await verifyEntries({ root, entries: [entry("policy_text", path)] });
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
test("Bash masking preserves parameter trimming and executable command consumers", async (t) => {
|
||||
const root = await fixture(t);
|
||||
const failing = [
|
||||
["scripts/trim.sh", "trimmed=${value#prefix}; old=revision.state\n"],
|
||||
["scripts/base.sh", "base=${path##*/}; old=workspaceRevision.state\n"],
|
||||
["scripts/backtick.sh", "old=`echo revision.state`\n"],
|
||||
["scripts/quoted-backtick.sh", 'echo "old: `echo revision.state`"\n'],
|
||||
["scripts/jq.sh", "jq '.revision.state' snapshot.json\n"],
|
||||
["scripts/substitution.sh", 'echo "$(echo revision.state)"\n'],
|
||||
];
|
||||
for (const [path, source] of failing) {
|
||||
await put(root, path, source);
|
||||
await assert.rejects(verifyEntries({ root, entries: [entry("policy_text", path)] }), /revision-state/, path);
|
||||
}
|
||||
await put(root, "scripts/echo.sh", 'echo "revision.state"\n# workspaceRevision.state\n');
|
||||
await verifyEntries({ root, entries: [entry("policy_text", "scripts/echo.sh")] });
|
||||
await put(root, "scripts/literal.yaml", '# revision.state\nvalue: "selectedWorkspace.state"\n');
|
||||
await verifyEntries({ root, entries: [entry("policy_text", "scripts/literal.yaml")] });
|
||||
});
|
||||
|
||||
test("YAML keeps URL slashes as data rather than a false line comment", async (t) => {
|
||||
const root = await fixture(t);
|
||||
const path = "scripts/url.yaml";
|
||||
await put(root, path, "url: https://host/x; old: selectedWorkspace.state\n");
|
||||
await assert.rejects(verifyEntries({ root, entries: [entry("policy_text", path)] }), /revision-state/);
|
||||
});
|
||||
+442
-34
@@ -1,16 +1,31 @@
|
||||
import Fastify, { type FastifyInstance } from "fastify";
|
||||
import Fastify, { type FastifyInstance, type FastifyRequest } from "fastify";
|
||||
import cors from "@fastify/cors";
|
||||
import { join } from "node:path";
|
||||
import cookie from "@fastify/cookie";
|
||||
import rateLimit from "@fastify/rate-limit";
|
||||
import { dirname, isAbsolute, join } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { tmpdir } from "node:os";
|
||||
import type { AppConfig } from "./config.js";
|
||||
import { ThtRunner } from "./tht/tht-runner.js";
|
||||
import { createMemoryCleanup } from "./catalog/memory-cleanup.js";
|
||||
import { PiProcessManager } from "./pi/pi-process-manager.js";
|
||||
import { SseHub } from "./sse/sse-hub.js";
|
||||
import { authPreHandler } from "./auth/auth.js";
|
||||
import { getPrincipal } from "./auth/auth.js";
|
||||
import { authenticateSession, captureAuthConfigSnapshot, configuredOrigin } from "./auth/auth.js";
|
||||
import type { PrincipalContext } from "./auth/principal.js";
|
||||
import type { LoadedAuthConfig } from "./auth/types.js";
|
||||
import { createCurrentLocalUserRegistryResolver, type LocalUserRegistry } from "./auth/local-registry.js";
|
||||
import { AuthSessionOperationalError, createFileAuthSessionStore, type AuthSessionStore, type AuthSessionValidity } from "./auth/session-store.js";
|
||||
import type { WindowsAuthStorageBridge } from "./auth/windows-auth-storage.js";
|
||||
import { registerAuthRoutes } from "./auth/routes.js";
|
||||
import { createOidcProtocol, type OidcProtocol, type OidcProtocolOptions } from "./auth/oidc-client.js";
|
||||
import { createConfiguredAuthDiagnoser } from "./auth/diagnostic-command.js";
|
||||
import type { AuthDiagnoser } from "./auth/diagnostics.js";
|
||||
import { isUsableAuthenticationSecret } from "./auth/secret-policy.js";
|
||||
import { secretValue } from "./config/secret-bundle.js";
|
||||
import { sessionRoutes } from "./routes/sessions.js";
|
||||
import { sqlRoutes } from "./routes/sql.js";
|
||||
import { metaRoutes, type ListModelsFn } from "./routes/meta.js";
|
||||
import { metaRoutes } from "./routes/meta.js";
|
||||
import type { ListModelsFn } from "./pi/list-models.js";
|
||||
import { settingsRoutes, effectiveSettings } from "./routes/settings.js";
|
||||
import { createPiModelLister } from "./pi/list-models.js";
|
||||
import { createPiManagement, type PiManagementService } from "./pi/management.js";
|
||||
@@ -19,10 +34,55 @@ import { ReadinessManager } from "./runtime/readiness-manager.js";
|
||||
import { MaintenanceBarrier } from "./runtime/maintenance-gate.js";
|
||||
import { WorkspaceRegistry } from "./workspaces/registry.js";
|
||||
import { createProductionWorkspaceDiagnoser } from "./workspaces/diagnostics.js";
|
||||
import { workspaceRoutes, type WorkspaceDiagnoser } from "./routes/workspaces.js";
|
||||
import {
|
||||
workspaceRoutes,
|
||||
type WorkspaceDatabaseTester,
|
||||
type WorkspaceDiagnoser,
|
||||
} from "./routes/workspaces.js";
|
||||
import { piManagementRoutes } from "./routes/pi-management.js";
|
||||
import { resolveRuntimeBindings, supportsSessionRuntime } from "./workspaces/bindings.js";
|
||||
import { supportsSessionRuntime } from "./workspaces/bindings.js";
|
||||
import { resolveRuntimeBindingsWithWorkspaceSecrets } from "./workspaces/secret-requirements.js";
|
||||
import type { WorkspaceDescriptor } from "./workspaces/schema.js";
|
||||
import { WorkspaceSecretStore } from "./workspaces/secret-store.js";
|
||||
import { createCatalogRepository } from "./catalog/repository.js";
|
||||
import type { CatalogRepository } from "./catalog/types.js";
|
||||
import { CatalogService } from "./catalog/service.js";
|
||||
import { catalogDatabaseRoutes } from "./routes/catalog-databases.js";
|
||||
import { CatalogOperationCoordinator } from "./catalog/operation-coordinator.js";
|
||||
import { ConcreteCatalogPostgresAccess, type CatalogPostgresAccess } from "./catalog/postgres-access.js";
|
||||
import { CatalogTableService } from "./catalog/table-service.js";
|
||||
import { catalogTableRoutes } from "./routes/catalog-tables.js";
|
||||
import { ConcreteCatalogSchemaIntrospector, type CatalogSchemaIntrospector } from "./catalog/schema-introspector.js";
|
||||
import { CatalogSyncWorker } from "./catalog/sync-worker.js";
|
||||
import { catalogSchemaRoutes } from "./routes/catalog-schema.js";
|
||||
import {
|
||||
loadMetadataGenerationModels,
|
||||
type MetadataGenerationModels,
|
||||
} from "./catalog/metadata-generation-models.js";
|
||||
import { metadataGenerationModelRoutes } from "./routes/metadata-generation-models.js";
|
||||
import { catalogDescriptionConsolidationRoutes } from "./routes/catalog-description-consolidation.js";
|
||||
import { PythonModelCompleter, type ModelCompleter } from "./catalog/model-completer.js";
|
||||
import { DescriptionGenerationWorker } from "./catalog/description-generation-worker.js";
|
||||
import { SensitivityAnalysisService } from "./catalog/sensitivity-analysis-service.js";
|
||||
import { SensitivityAnalysisRunner } from "./catalog/sensitivity-analysis-runner.js";
|
||||
import { SensitivityClassifier, type LocalNerDetector, type SensitivityValueSource } from "./catalog/sensitivity-classifier.js";
|
||||
import { ConcreteSensitivityValueSource } from "./catalog/sensitivity-value-source.js";
|
||||
import { PythonLocalNerDetector } from "./catalog/local-ner-detector.js";
|
||||
import {
|
||||
ConcreteDescriptionSourceSampler,
|
||||
type DescriptionSourceSampler,
|
||||
} from "./catalog/description-source-sampler.js";
|
||||
import { catalogDescriptionGenerationRoutes } from "./routes/catalog-description-generation.js";
|
||||
import { CatalogLogicalRelationshipService } from "./catalog/logical-relationship-service.js";
|
||||
import { catalogLogicalRelationshipRoutes } from "./routes/catalog-logical-relationships.js";
|
||||
import { EffectiveRelationshipSnapshotProvider } from "./catalog/effective-relationship-snapshot.js";
|
||||
import { loadRuntimeModelCatalog, type RuntimeModelCatalog } from "./models/runtime-model-catalog.js";
|
||||
import { createProductionWorkspacePreprocessingService } from "./workspace-maintenance.js";
|
||||
import type { WorkspacePreprocessingService } from "./workspaces/preprocessing-service.js";
|
||||
import { PreprocessingStateStore } from "./workspaces/preprocessing-state.js";
|
||||
import { workspacePreprocessingRoutes } from "./routes/workspace-preprocessing.js";
|
||||
import { memoryRoutes } from "./routes/memory.js";
|
||||
import { evidenceRoutes } from "./routes/evidence.js";
|
||||
|
||||
export interface BuildAppDeps {
|
||||
thtRunner?: ThtRunner;
|
||||
@@ -34,21 +94,78 @@ export interface BuildAppDeps {
|
||||
hub?: SseHub;
|
||||
workspaceRegistry?: WorkspaceRegistry;
|
||||
workspaceDiagnoser?: WorkspaceDiagnoser;
|
||||
workspaceDatabaseTester?: WorkspaceDatabaseTester;
|
||||
workspaceSecretStore?: WorkspaceSecretStore;
|
||||
workspacePreprocessingService?: Pick<WorkspacePreprocessingService, "run" | "clear"> & Partial<Pick<WorkspacePreprocessingService, "consolidateEvidence" | "evidenceSources">>;
|
||||
catalogRepository?: CatalogRepository;
|
||||
catalogService?: CatalogService;
|
||||
catalogPostgresAccess?: CatalogPostgresAccess;
|
||||
catalogTableService?: CatalogTableService;
|
||||
catalogLogicalRelationshipService?: CatalogLogicalRelationshipService;
|
||||
effectiveRelationshipSnapshotProvider?: EffectiveRelationshipSnapshotProvider;
|
||||
catalogSchemaIntrospector?: CatalogSchemaIntrospector;
|
||||
catalogSyncWorker?: CatalogSyncWorker;
|
||||
catalogOperationCoordinator?: CatalogOperationCoordinator;
|
||||
metadataGenerationModels?: MetadataGenerationModels;
|
||||
runtimeModelCatalog?: RuntimeModelCatalog;
|
||||
modelCompleter?: ModelCompleter;
|
||||
descriptionSourceSampler?: DescriptionSourceSampler;
|
||||
sensitivityValueSource?: SensitivityValueSource;
|
||||
localNerDetector?: LocalNerDetector;
|
||||
workspaceRuntimeSupport?: (workspace: WorkspaceDescriptor) => boolean;
|
||||
maintenanceBarrier?: MaintenanceBarrier;
|
||||
piManagement?: PiManagementService;
|
||||
localUserRegistry?: LocalUserRegistry;
|
||||
authSessionStore?: AuthSessionStore;
|
||||
/** Explicit test-only transport seam; production always invokes the hidden tht bridge. */
|
||||
authStorageBridgeForTest?: WindowsAuthStorageBridge;
|
||||
oidcProtocol?: OidcProtocol;
|
||||
authDiagnoser?: AuthDiagnoser;
|
||||
/** Explicit test seam; production uses the provider-neutral OIDC constructor. */
|
||||
oidcProtocolFactory?: (options: OidcProtocolOptions) => OidcProtocol;
|
||||
}
|
||||
|
||||
export interface AppWithAuthSessionStore extends FastifyInstance {
|
||||
thothiiAuthSessionStore?: AuthSessionStore;
|
||||
}
|
||||
|
||||
export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstance {
|
||||
const app = Fastify({ logger: { level: "warn" }, disableRequestLogging: true });
|
||||
|
||||
// Allow any origin in dev/e2e; tighten in production via config if needed.
|
||||
app.register(cors, {
|
||||
origin: true,
|
||||
credentials: true,
|
||||
methods: ["GET", "POST", "PUT", "DELETE", "OPTIONS"],
|
||||
app.decorateRequest("authConfigSnapshot", undefined);
|
||||
app.decorateRequest("authConfigSnapshotCaptured", false);
|
||||
app.decorateRequest("authConfigSnapshotUnavailable", false);
|
||||
const isolatedTestRoot = process.env.VITEST === "true"
|
||||
? join(tmpdir(), `thothii-workspace-secrets-vitest-${process.pid}`)
|
||||
: undefined;
|
||||
const workspaceSecretStore = deps?.workspaceSecretStore ?? new WorkspaceSecretStore({
|
||||
root: isolatedTestRoot ?? config.workspaceSecretStoreRoot,
|
||||
runtimeRoot: isolatedTestRoot === undefined
|
||||
? config.workspaceSecretRuntimeRoot
|
||||
: join(isolatedTestRoot, "runtime"),
|
||||
installationId: config.workspaceRegistry.installationId,
|
||||
});
|
||||
|
||||
const cookieAuth = config.authMode === "local" || config.authMode === "oidc";
|
||||
app.register(cors, {
|
||||
// The delegator runs at CORS's onRequest hook. It owns the one request-scoped config load
|
||||
// which subsequent auth hooks and routes consume, including preflights that end here.
|
||||
delegator: (request, callback) => {
|
||||
const snapshot = captureAuthConfigSnapshot(request, config.authentication);
|
||||
const origin = configuredOrigin(snapshot);
|
||||
const snapshotUsesCookies = snapshot?.value.mode === "local" || snapshot?.value.mode === "oidc";
|
||||
callback(null, {
|
||||
origin: snapshotUsesCookies && origin ? corsOrigin(request, origin) : cookieAuth ? false : true,
|
||||
credentials: snapshotUsesCookies,
|
||||
methods: ["GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS"],
|
||||
});
|
||||
},
|
||||
});
|
||||
// Cookie parsing and the rate-limit plugin must precede every auth/application route.
|
||||
app.register(cookie);
|
||||
app.register(rateLimit, { global: false });
|
||||
|
||||
const workspaceRegistry = deps?.workspaceRegistry ?? new WorkspaceRegistry(config.workspaceRegistry);
|
||||
const catalogRepository = deps?.catalogRepository ?? createCatalogRepository(config.catalogDatabase);
|
||||
const tht = deps?.thtRunner ?? new ThtRunner({
|
||||
thtBin: config.thtBin,
|
||||
harnessDir: config.harnessDir,
|
||||
@@ -58,36 +175,149 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc
|
||||
secretRoots: config.workspaceRegistry.secretRoots,
|
||||
secretsFile: config.secretsFile,
|
||||
secretFiles: config.secretFiles,
|
||||
workspaceSecretStore,
|
||||
catalogRepository: deps?.catalogRepository ?? (config.catalogDatabase ? catalogRepository : undefined),
|
||||
semanticRuntime: {
|
||||
internalQdrantUrl: config.internalQdrantUrl,
|
||||
internalEmbeddingUrl: config.internalEmbeddingUrl,
|
||||
internalEmbeddingId: config.internalEmbeddingId,
|
||||
internalEmbeddingModel: config.internalEmbeddingModel,
|
||||
internalEmbeddingDimensions: config.internalEmbeddingDimensions,
|
||||
},
|
||||
});
|
||||
const mgr = deps?.mgr ?? new PiProcessManager(config, deps?.spawnFn ? { spawnFn: deps.spawnFn } : undefined);
|
||||
const workspacePreprocessingService = deps?.workspacePreprocessingService
|
||||
?? createProductionWorkspacePreprocessingService({
|
||||
config,
|
||||
catalogRepository,
|
||||
registry: workspaceRegistry,
|
||||
workspaceSecretStore,
|
||||
runner: tht as ThtRunner,
|
||||
});
|
||||
const hub = deps?.hub ?? new SseHub();
|
||||
const workspaceRegistry = deps?.workspaceRegistry ?? new WorkspaceRegistry(config.workspaceRegistry);
|
||||
const catalogOperationCoordinator = deps?.catalogOperationCoordinator ?? new CatalogOperationCoordinator();
|
||||
const runtimeModelCatalog = deps?.runtimeModelCatalog ?? loadRuntimeModelCatalog(config.modelCatalogFile);
|
||||
const mgr = deps?.mgr ?? new PiProcessManager(config, {
|
||||
...(deps?.spawnFn ? { spawnFn: deps.spawnFn } : {}),
|
||||
modelCatalog: runtimeModelCatalog,
|
||||
});
|
||||
const metadataGenerationModels = deps?.metadataGenerationModels ?? loadMetadataGenerationModels({
|
||||
catalogFile: config.modelCatalogFile,
|
||||
secretsFile: config.secretsFile,
|
||||
});
|
||||
const modelCompleter = deps?.modelCompleter ?? new PythonModelCompleter({
|
||||
pythonExecutable: isAbsolute(config.thtBin) ? join(dirname(config.thtBin), "python") : "python3",
|
||||
cwd: config.harnessDir,
|
||||
});
|
||||
const catalogPostgresAccess = deps?.catalogPostgresAccess ?? new ConcreteCatalogPostgresAccess(
|
||||
workspaceSecretStore,
|
||||
{ connectTimeoutMs: config.workspaceDiagnosticTimeoutMs },
|
||||
);
|
||||
const descriptionSourceSampler = deps?.descriptionSourceSampler
|
||||
?? new ConcreteDescriptionSourceSampler(catalogPostgresAccess, workspaceSecretStore);
|
||||
const descriptionGenerationWorker = new DescriptionGenerationWorker(
|
||||
catalogRepository,
|
||||
workspaceRegistry,
|
||||
metadataGenerationModels,
|
||||
modelCompleter,
|
||||
catalogOperationCoordinator,
|
||||
descriptionSourceSampler,
|
||||
);
|
||||
const sensitivityValueSource = deps?.sensitivityValueSource
|
||||
?? new ConcreteSensitivityValueSource(catalogPostgresAccess, workspaceSecretStore);
|
||||
const configuredNerWorker = config.sensitivityNer?.workerScript
|
||||
?? fileURLToPath(new URL("../python/sensitivity_ner_worker.py", import.meta.url));
|
||||
const localNerDetector = deps?.localNerDetector ?? (config.sensitivityNer
|
||||
? new PythonLocalNerDetector({
|
||||
pythonExecutable: config.sensitivityNer.pythonExecutable,
|
||||
workerScript: configuredNerWorker,
|
||||
modelPath: config.sensitivityNer.modelPath,
|
||||
cwd: dirname(configuredNerWorker),
|
||||
threads: config.sensitivityNer.threads,
|
||||
})
|
||||
: undefined);
|
||||
const sensitiveDataSuggester = new SensitivityAnalysisService(
|
||||
catalogRepository,
|
||||
new SensitivityClassifier(sensitivityValueSource, localNerDetector),
|
||||
);
|
||||
const sensitivityAnalysisRunner = new SensitivityAnalysisRunner(
|
||||
catalogRepository,
|
||||
sensitiveDataSuggester,
|
||||
);
|
||||
const catalogService = deps?.catalogService ?? new CatalogService(
|
||||
catalogRepository,
|
||||
workspaceRegistry,
|
||||
workspaceSecretStore,
|
||||
config.workspaceRegistry.secretRoots,
|
||||
config.workspaceDiagnosticTimeoutMs,
|
||||
catalogPostgresAccess,
|
||||
catalogOperationCoordinator,
|
||||
);
|
||||
const workspaceDatabaseTester = deps?.workspaceDatabaseTester ?? (async (workspaceId: string) => {
|
||||
const database = await catalogRepository.getByWorkspace(workspaceId);
|
||||
return database ? catalogService.test(database) : undefined;
|
||||
});
|
||||
const catalogTableService = deps?.catalogTableService ?? new CatalogTableService(catalogRepository);
|
||||
const catalogLogicalRelationshipService = deps?.catalogLogicalRelationshipService
|
||||
?? new CatalogLogicalRelationshipService(catalogRepository);
|
||||
const catalogSchemaIntrospector = deps?.catalogSchemaIntrospector ?? new ConcreteCatalogSchemaIntrospector(
|
||||
catalogPostgresAccess,
|
||||
workspaceSecretStore,
|
||||
);
|
||||
const catalogSyncWorker = deps?.catalogSyncWorker ?? new CatalogSyncWorker(
|
||||
catalogRepository,
|
||||
catalogSchemaIntrospector,
|
||||
catalogOperationCoordinator,
|
||||
config.catalogSyncTimeoutMs,
|
||||
createMemoryCleanup(tht as ThtRunner, {
|
||||
internalQdrantUrl: config.internalQdrantUrl, internalEmbeddingUrl: config.internalEmbeddingUrl,
|
||||
internalEmbeddingId: config.internalEmbeddingId, internalEmbeddingModel: config.internalEmbeddingModel,
|
||||
internalEmbeddingDimensions: config.internalEmbeddingDimensions,
|
||||
}),
|
||||
);
|
||||
app.addHook("onReady", async () => { await catalogSyncWorker.initialize(); });
|
||||
app.addHook("onReady", async () => { await descriptionGenerationWorker.initialize(); });
|
||||
app.addHook("onReady", async () => { await sensitivityAnalysisRunner.initialize(); });
|
||||
if (localNerDetector?.warmup) {
|
||||
app.addHook("onReady", async () => {
|
||||
void localNerDetector.warmup?.().catch(() => undefined);
|
||||
});
|
||||
}
|
||||
if (!deps?.catalogRepository && catalogRepository.close) {
|
||||
app.addHook("onClose", async () => { await catalogRepository.close?.(); });
|
||||
}
|
||||
app.addHook("onClose", async () => { await catalogSyncWorker.stop(); });
|
||||
app.addHook("onClose", async () => { await descriptionGenerationWorker.stop(); });
|
||||
if (localNerDetector?.close) {
|
||||
app.addHook("onClose", async () => { await localNerDetector.close?.(); });
|
||||
}
|
||||
const workspaceDiagnoser = deps?.workspaceDiagnoser
|
||||
?? createProductionWorkspaceDiagnoser(config.workspaceDiagnosticTimeoutMs, undefined, {
|
||||
internalQdrantUrl: config.internalQdrantUrl,
|
||||
internalEmbeddingUrl: config.internalEmbeddingUrl,
|
||||
internalEmbeddingId: config.internalEmbeddingId,
|
||||
internalEmbeddingModel: config.internalEmbeddingModel,
|
||||
internalEmbeddingDimensions: config.internalEmbeddingDimensions,
|
||||
});
|
||||
const workspaceRuntimeSupport = deps?.workspaceRuntimeSupport ?? ((workspace: WorkspaceDescriptor) => (
|
||||
supportsSessionRuntime(resolveRuntimeBindings(
|
||||
const workspaceRuntimeSupport = deps?.workspaceRuntimeSupport ?? ((workspace: WorkspaceDescriptor) => {
|
||||
const lease = resolveRuntimeBindingsWithWorkspaceSecrets(
|
||||
workspace,
|
||||
process.env,
|
||||
config.workspaceRegistry.secretRoots,
|
||||
))
|
||||
));
|
||||
workspaceSecretStore,
|
||||
);
|
||||
try {
|
||||
return supportsSessionRuntime(lease.bindings);
|
||||
} finally {
|
||||
lease.release();
|
||||
}
|
||||
});
|
||||
const readiness = deps?.readiness ?? new ReadinessManager(
|
||||
tht as ThtRunner,
|
||||
Math.round(config.ollamaEnsureTimeoutMs / 1000),
|
||||
);
|
||||
|
||||
const listModels = deps?.listModels ?? createPiModelLister(config, {
|
||||
modelCatalog: runtimeModelCatalog,
|
||||
warn: (detail) => app.log.warn(
|
||||
{ component: "pi-model-list", detail },
|
||||
"Pi enabled-model configuration warning",
|
||||
@@ -99,32 +329,149 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc
|
||||
};
|
||||
const getSettings = async (principal: PrincipalContext): Promise<Settings> => {
|
||||
if (deps?.getSettings) return await deps.getSettings(principal);
|
||||
return effectiveSettings(config, loadSettings(config));
|
||||
const stored = loadSettings(config);
|
||||
const effective = effectiveSettings(config, stored, runtimeModelCatalog);
|
||||
// In the registry system the legacy `harness/workspaces/*.yaml` default is obsolete: when no
|
||||
// installation workspace is pinned, default to the first active registry workspace.
|
||||
if (!stored.workspace) {
|
||||
try {
|
||||
const revisions = await workspaceRegistry.list();
|
||||
if (revisions.length > 0) effective.workspace = revisions[0].id;
|
||||
} catch {
|
||||
// Registry not bootstrapped yet; keep the legacy fallback.
|
||||
}
|
||||
}
|
||||
return effective;
|
||||
};
|
||||
const piManagement = deps?.piManagement ?? createPiManagement(config, { listModels });
|
||||
const piManagement = deps?.piManagement ?? createPiManagement(config, {
|
||||
modelCatalog: runtimeModelCatalog,
|
||||
});
|
||||
|
||||
const maintenanceBarrier = deps?.maintenanceBarrier ?? new MaintenanceBarrier(config.maintenanceFile);
|
||||
const authenticate = authPreHandler(config.authMode);
|
||||
app.addHook("preHandler", async (req, reply) => {
|
||||
// Process readiness is intentionally unauthenticated for local container/proxy probes.
|
||||
if (req.url === "/health" || req.url === "/health/dwh") return;
|
||||
const localRegistryResolver = deps?.localUserRegistry === undefined
|
||||
? createCurrentLocalUserRegistryResolver()
|
||||
: undefined;
|
||||
const resolveLocalUserRegistry = (loaded: LoadedAuthConfig) => {
|
||||
return deps?.localUserRegistry ?? localRegistryResolver?.resolve(loaded);
|
||||
};
|
||||
const localUserForSnapshot = async (loaded: LoadedAuthConfig, subject: string) => {
|
||||
try {
|
||||
if (loaded.value.mode !== "local") return { revision: loaded.revision, user: undefined };
|
||||
const registry = resolveLocalUserRegistry(loaded);
|
||||
if (!registry) throw new AuthSessionOperationalError();
|
||||
const user = await registry.findBySubject(subject);
|
||||
return {
|
||||
revision: loaded.revision,
|
||||
user: user === undefined ? undefined : {
|
||||
enabled: user.enabled,
|
||||
authRevision: user.authRevision,
|
||||
roles: user.roles,
|
||||
},
|
||||
};
|
||||
} catch (error) {
|
||||
if (error instanceof AuthSessionOperationalError) throw error;
|
||||
throw new AuthSessionOperationalError();
|
||||
}
|
||||
};
|
||||
const sessionValidityForSnapshot = (loaded: LoadedAuthConfig): AuthSessionValidity => ({
|
||||
currentAuthConfigRevision: () => loaded.revision,
|
||||
currentLocalUser: (subject) => localUserForSnapshot(loaded, subject),
|
||||
});
|
||||
const resolveOidcProtocol = (loaded: LoadedAuthConfig): OidcProtocol | undefined => {
|
||||
if (deps?.oidcProtocol) return deps.oidcProtocol;
|
||||
if (loaded.value.mode !== "oidc") return undefined;
|
||||
try {
|
||||
const clientSecret = secretValue(config, loaded.value.oidc.clientSecretRef);
|
||||
if (!isUsableAuthenticationSecret("THT_OIDC_CLIENT_SECRET", clientSecret)) return undefined;
|
||||
return (deps?.oidcProtocolFactory ?? createOidcProtocol)({
|
||||
issuer: loaded.value.oidc.issuer,
|
||||
clientId: loaded.value.oidc.clientId,
|
||||
clientSecret,
|
||||
callbackUrl: new URL("/api/auth/oidc/callback", loaded.value.publicUrl).href,
|
||||
scopes: loaded.value.oidc.scopes,
|
||||
groupsClaim: loaded.value.oidc.groupsClaim,
|
||||
});
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
};
|
||||
const authDiagnoser = deps?.authDiagnoser ?? createConfiguredAuthDiagnoser(config, {
|
||||
localUserRegistry: resolveLocalUserRegistry,
|
||||
oidcProtocol: resolveOidcProtocol,
|
||||
});
|
||||
const authSessionStore = deps?.authSessionStore ?? (config.authMode === "local" || config.authMode === "oidc"
|
||||
? createFileAuthSessionStore(config.authStateRoot, {
|
||||
currentAuthConfigRevision: () => {
|
||||
try {
|
||||
return config.authentication?.current().revision ?? "";
|
||||
} catch {
|
||||
throw new AuthSessionOperationalError();
|
||||
}
|
||||
},
|
||||
currentLocalUser: async (subject) => {
|
||||
try {
|
||||
const loaded = config.authentication?.current();
|
||||
if (!loaded) return { revision: "", user: undefined };
|
||||
return await localUserForSnapshot(loaded, subject);
|
||||
} catch (error) {
|
||||
if (error instanceof AuthSessionOperationalError) throw error;
|
||||
throw new AuthSessionOperationalError();
|
||||
}
|
||||
},
|
||||
}, deps?.authStorageBridgeForTest === undefined
|
||||
? undefined
|
||||
: process.platform === "win32"
|
||||
? { windowsStorageBridge: deps.authStorageBridgeForTest }
|
||||
: { posixStorageBridge: deps.authStorageBridgeForTest })
|
||||
: undefined);
|
||||
(app as AppWithAuthSessionStore).thothiiAuthSessionStore = authSessionStore;
|
||||
const authenticate = authenticateSession({
|
||||
mode: config.authMode,
|
||||
publicExposure: config.publicExposure,
|
||||
authentication: config.authentication,
|
||||
sessionStore: authSessionStore,
|
||||
sessionValidityForSnapshot,
|
||||
});
|
||||
app.addHook("preHandler", (req, reply, done) => {
|
||||
if (isMaintenanceControl(req.url)) {
|
||||
if (!isLoopback(req.ip)) {
|
||||
return reply.code(403).send({ error: "loopback maintenance control required" });
|
||||
reply.code(403).send({ error: "loopback maintenance control required" });
|
||||
}
|
||||
return;
|
||||
}
|
||||
return authenticate(req, reply);
|
||||
done();
|
||||
});
|
||||
app.addHook("preHandler", authenticate);
|
||||
app.get("/health", async () => ({ status: "ok" }));
|
||||
app.get("/health/dwh", async () => tht.dbPing());
|
||||
app.get("/me", async (req) => getPrincipal(req));
|
||||
app.get("/health/dwh", async () => {
|
||||
// In the registry system there is no single legacy DWH config: ping the first active
|
||||
// workspace's rendered runtime config. If the registry is not bootstrapped yet, do not
|
||||
// block the app — per-workspace diagnostics and the session precheck own reachability.
|
||||
try {
|
||||
const revisions = await workspaceRegistry.list();
|
||||
if (revisions.length > 0) {
|
||||
return await tht.dbPing(revisions[0].snapshotPath);
|
||||
}
|
||||
} catch {
|
||||
// fall through
|
||||
}
|
||||
return { ok: true, detail: "workspace diagnostics own DWH reachability" };
|
||||
});
|
||||
registerAuthRoutes(app, {
|
||||
authMode: config.authMode,
|
||||
authentication: config.authentication,
|
||||
sessionStore: authSessionStore,
|
||||
localUserRegistry: deps?.localUserRegistry,
|
||||
resolveLocalUserRegistry,
|
||||
resolveOidcProtocol,
|
||||
});
|
||||
sessionRoutes(app, {
|
||||
mgr, tht: tht as ThtRunner, hub, getSettings, readiness, listModels, workspaceRegistry,
|
||||
dwhPrecheck: config.dwhPrecheck,
|
||||
legacyWorkspaceMode: config.legacyWorkspaceMode,
|
||||
workspaceRuntimeSupport,
|
||||
modelCatalog: runtimeModelCatalog,
|
||||
maintenanceBarrier,
|
||||
catalogRepository: deps?.catalogRepository ?? (config.catalogDatabase ? catalogRepository : undefined),
|
||||
});
|
||||
app.post("/internal/maintenance/activate", async (req, reply) => {
|
||||
try {
|
||||
@@ -154,14 +501,75 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc
|
||||
return maintenanceBarrier.status();
|
||||
});
|
||||
sqlRoutes(app, { tht: tht as ThtRunner, getSettings, workspaceRegistry });
|
||||
metaRoutes(app, { harnessDir: config.harnessDir, listModels });
|
||||
workspaceRoutes(app, { registry: workspaceRegistry, config: config.workspaceRegistry, diagnose: workspaceDiagnoser });
|
||||
settingsRoutes(app, { cfg: config, listModels, getSettings });
|
||||
piManagementRoutes(app, { config, service: piManagement });
|
||||
memoryRoutes(app, { runner: tht as ThtRunner, registry: workspaceRegistry, runtime: {
|
||||
internalQdrantUrl: config.internalQdrantUrl,
|
||||
internalEmbeddingUrl: config.internalEmbeddingUrl,
|
||||
internalEmbeddingModel: config.internalEmbeddingModel,
|
||||
internalEmbeddingDimensions: config.internalEmbeddingDimensions,
|
||||
} });
|
||||
metaRoutes(app, { harnessDir: config.harnessDir, modelCatalog: runtimeModelCatalog });
|
||||
evidenceRoutes(app, { runner: tht as ThtRunner, registry: workspaceRegistry,
|
||||
registryRoot: config.workspaceRegistry.root, hostRegistryRoot: config.evidenceHostRegistryRoot,
|
||||
service: workspacePreprocessingService });
|
||||
workspaceRoutes(app, {
|
||||
registry: workspaceRegistry,
|
||||
config: config.workspaceRegistry,
|
||||
diagnose: workspaceDiagnoser,
|
||||
authDiagnoser,
|
||||
secretStore: workspaceSecretStore,
|
||||
testDatabaseConnection: workspaceDatabaseTester,
|
||||
});
|
||||
workspacePreprocessingRoutes(app, {
|
||||
repository: catalogRepository,
|
||||
registry: workspaceRegistry,
|
||||
service: workspacePreprocessingService,
|
||||
inputFingerprint: tht as ThtRunner,
|
||||
readLatestJob: (workspaceId) => new PreprocessingStateStore({
|
||||
dataRoot: config.dataRoot ?? "/data",
|
||||
workspaceId,
|
||||
}).readLatestJob(),
|
||||
});
|
||||
catalogDatabaseRoutes(app, { repository: catalogRepository, service: catalogService, operations: catalogOperationCoordinator });
|
||||
catalogTableRoutes(app, {
|
||||
repository: catalogRepository,
|
||||
service: catalogTableService,
|
||||
operations: catalogOperationCoordinator,
|
||||
});
|
||||
catalogSchemaRoutes(app, {
|
||||
repository: catalogRepository,
|
||||
worker: catalogSyncWorker,
|
||||
operations: catalogOperationCoordinator,
|
||||
});
|
||||
catalogLogicalRelationshipRoutes(app, {
|
||||
service: catalogLogicalRelationshipService,
|
||||
operations: catalogOperationCoordinator,
|
||||
});
|
||||
catalogDescriptionConsolidationRoutes(app, {
|
||||
repository: catalogRepository,
|
||||
operations: catalogOperationCoordinator,
|
||||
});
|
||||
metadataGenerationModelRoutes(app, metadataGenerationModels);
|
||||
catalogDescriptionGenerationRoutes(app, {
|
||||
repository: catalogRepository,
|
||||
worker: descriptionGenerationWorker,
|
||||
sensitivityAnalysisRunner,
|
||||
});
|
||||
settingsRoutes(app, { cfg: config, getSettings });
|
||||
piManagementRoutes(app, { service: piManagement });
|
||||
|
||||
return app;
|
||||
}
|
||||
|
||||
function corsOrigin(request: FastifyRequest, expectedOrigin: string): string | false {
|
||||
const supplied = request.headers.origin;
|
||||
if (typeof supplied !== "string") return false;
|
||||
try {
|
||||
return new URL(supplied).origin === expectedOrigin ? expectedOrigin : false;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function isLoopback(ip: string): boolean { return ip === "127.0.0.1" || ip === "::1" || ip === "::ffff:127.0.0.1"; }
|
||||
function isMaintenanceControl(url: string): boolean {
|
||||
return /^\/internal\/maintenance\/(?:activate|deactivate|status)(?:\?|$)/.test(url);
|
||||
|
||||
+205
-5
@@ -1,17 +1,64 @@
|
||||
import type { FastifyRequest, FastifyReply } from "fastify";
|
||||
import type { FastifyRequest, FastifyReply, preHandlerHookHandler } from "fastify";
|
||||
import { localPrincipal, type PrincipalContext, upstreamPrincipal } from "./principal.js";
|
||||
import { rolesToPermissions } from "./config.js";
|
||||
import type { AuthenticationConfigProvider, AuthMode, AuthSessionRecord, LoadedAuthConfig } from "./types.js";
|
||||
import { AuthSessionOperationalError, type AuthSessionStore, type AuthSessionValidity } from "./session-store.js";
|
||||
import { deriveCsrfToken, csrfTokensEqual } from "./csrf.js";
|
||||
import { requireSameOriginOrNonBrowser } from "./authorization.js";
|
||||
|
||||
declare module "fastify" {
|
||||
interface FastifyRequest { principal?: PrincipalContext }
|
||||
interface FastifyRequest {
|
||||
principal?: PrincipalContext;
|
||||
authSession?: AuthSessionRecord;
|
||||
/** Internal only: never serialize or write this opaque cookie token to logs. */
|
||||
authSessionToken?: string;
|
||||
authPublicOrigin?: string;
|
||||
/** One immutable configuration load for the whole request, including CORS. */
|
||||
authConfigSnapshot?: LoadedAuthConfig;
|
||||
authConfigSnapshotCaptured?: boolean;
|
||||
authConfigSnapshotUnavailable?: boolean;
|
||||
}
|
||||
}
|
||||
|
||||
export function authPreHandler(mode: "none" | "mock" | "upstream") {
|
||||
const SESSION_COOKIE = "thothii_session";
|
||||
const SESSION_TOKEN = /^[A-Za-z0-9_-]{43}$/;
|
||||
const STATE_CHANGING_METHODS = new Set(["POST", "PUT", "PATCH", "DELETE"]);
|
||||
|
||||
export interface AuthDependencies {
|
||||
mode: AuthMode;
|
||||
publicExposure?: boolean;
|
||||
authentication?: AuthenticationConfigProvider;
|
||||
sessionStore?: AuthSessionStore;
|
||||
sessionValidityForSnapshot?: (snapshot: LoadedAuthConfig) => AuthSessionValidity;
|
||||
}
|
||||
|
||||
/** Capture the authentication configuration once; CORS calls this before every other hook. */
|
||||
export function captureAuthConfigSnapshot(
|
||||
request: FastifyRequest,
|
||||
authentication: AuthenticationConfigProvider | undefined,
|
||||
): LoadedAuthConfig | undefined {
|
||||
if (request.authConfigSnapshotCaptured) return request.authConfigSnapshot;
|
||||
request.authConfigSnapshotCaptured = true;
|
||||
try {
|
||||
request.authConfigSnapshot = authentication?.current();
|
||||
} catch {
|
||||
request.authConfigSnapshotUnavailable = true;
|
||||
}
|
||||
return request.authConfigSnapshot;
|
||||
}
|
||||
|
||||
export function authPreHandler(mode: "none" | "mock" | "upstream", publicExposure = false) {
|
||||
return async (req: FastifyRequest, reply: FastifyReply) => {
|
||||
if (mode === "none") {
|
||||
req.principal = localPrincipal();
|
||||
req.principal = localPrincipal(publicExposure);
|
||||
} else if (mode === "mock") {
|
||||
const subject = typeof req.headers["x-mock-user"] === "string" ? req.headers["x-mock-user"].trim() : "mock";
|
||||
req.principal = { issuer: "mock", subject: subject || "mock", displayName: subject || "mock", isAdmin: false };
|
||||
const elevated = req.headers["x-thoth-is-admin"] === "1" || req.headers["x-thoth-is-admin"] === "true";
|
||||
const roles = elevated ? ["admin"] as const : ["user"] as const;
|
||||
req.principal = {
|
||||
issuer: "mock", subject: subject || "mock", displayName: subject || "mock", roles,
|
||||
permissions: rolesToPermissions(roles), isAdmin: elevated,
|
||||
};
|
||||
} else {
|
||||
const principal = upstreamPrincipal(req.headers);
|
||||
if (!principal) {
|
||||
@@ -22,6 +69,159 @@ export function authPreHandler(mode: "none" | "mock" | "upstream") {
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* The one application boundary for principal resolution. Auth protocol endpoints are the only
|
||||
* public exceptions; all other routes get either a resolved principal or a sanitized denial.
|
||||
*/
|
||||
export function authenticateSession(deps: AuthDependencies): preHandlerHookHandler {
|
||||
const legacy = deps.mode === "none" || deps.mode === "mock" || deps.mode === "upstream"
|
||||
? authPreHandler(deps.mode, deps.publicExposure)
|
||||
: undefined;
|
||||
|
||||
const handle = async (request: FastifyRequest, reply: FastifyReply): Promise<void> => {
|
||||
const snapshot = captureAuthConfigSnapshot(request, deps.authentication);
|
||||
if (isPublicRoute(request)) return;
|
||||
|
||||
if (legacy) {
|
||||
await legacy(request, reply);
|
||||
if (reply.sent || !STATE_CHANGING_METHODS.has(request.method)) return;
|
||||
return requireSameOriginOrNonBrowser(request, reply);
|
||||
}
|
||||
|
||||
const origin = configuredOrigin(snapshot);
|
||||
if (!snapshot || !origin || !deps.sessionStore) {
|
||||
return reply.code(503).send({ code: "auth_unavailable", error: "Authentication is unavailable" });
|
||||
}
|
||||
const token = readSessionCookie(request);
|
||||
if (token === undefined || token === false) return authenticationRequired(reply);
|
||||
|
||||
let session: AuthSessionRecord | undefined;
|
||||
try {
|
||||
session = await deps.sessionStore.resolve(token, undefined, deps.sessionValidityForSnapshot?.(snapshot));
|
||||
if (session && session.authConfigRevision !== snapshot.revision) {
|
||||
try { await deps.sessionStore.revoke(token); } catch { /* the mismatch remains denied */ }
|
||||
return authenticationRequired(reply);
|
||||
}
|
||||
if (session) await deps.sessionStore.touch(token);
|
||||
} catch (error) {
|
||||
if (error instanceof AuthSessionOperationalError) {
|
||||
return reply.code(503).send({ code: "auth_unavailable", error: "Authentication is unavailable" });
|
||||
}
|
||||
return authenticationRequired(reply);
|
||||
}
|
||||
if (!session) return authenticationRequired(reply);
|
||||
|
||||
request.authSession = session;
|
||||
request.authSessionToken = token;
|
||||
request.authPublicOrigin = origin;
|
||||
request.principal = {
|
||||
issuer: session.issuer,
|
||||
subject: session.subject,
|
||||
...(session.displayName === undefined ? {} : { displayName: session.displayName }),
|
||||
roles: session.roles,
|
||||
// Sessions can outlive a deployment that changes the role permission catalog.
|
||||
// resolve() has already checked validity, including current local user roles.
|
||||
permissions: rolesToPermissions(session.roles),
|
||||
isAdmin: session.roles.includes("admin"),
|
||||
};
|
||||
if (STATE_CHANGING_METHODS.has(request.method)) {
|
||||
requireCsrf(request, reply);
|
||||
return;
|
||||
}
|
||||
};
|
||||
return (request, reply, done) => {
|
||||
void handle(request, reply).then(
|
||||
() => done(),
|
||||
() => {
|
||||
if (!reply.sent) reply.code(503).send({ code: "auth_unavailable", error: "Authentication is unavailable" });
|
||||
done();
|
||||
},
|
||||
);
|
||||
};
|
||||
}
|
||||
|
||||
export function requireCsrf(request: FastifyRequest, reply: FastifyReply): true | FastifyReply {
|
||||
const expectedOrigin = request.authPublicOrigin;
|
||||
const token = request.authSessionToken;
|
||||
if (!expectedOrigin || !token) return authenticationRequired(reply);
|
||||
if (!matchesOrigin(request, expectedOrigin)) return csrfFailed(reply);
|
||||
|
||||
const header = singleHeader(request.headers["x-thothii-csrf"]);
|
||||
const supplied = header === false || header === undefined || !SESSION_TOKEN.test(header) ? undefined : header;
|
||||
let expected = "";
|
||||
try {
|
||||
expected = deriveCsrfToken(token);
|
||||
} catch {
|
||||
return authenticationRequired(reply);
|
||||
}
|
||||
if (!csrfTokensEqual(expected, supplied)) return csrfFailed(reply);
|
||||
return true;
|
||||
}
|
||||
|
||||
/** Require an exact configured public origin and browser Fetch Metadata when supplied. */
|
||||
export function requireExactOrigin(
|
||||
request: FastifyRequest,
|
||||
reply: FastifyReply,
|
||||
expectedOrigin: string,
|
||||
): true | FastifyReply {
|
||||
return matchesOrigin(request, expectedOrigin) ? true : csrfFailed(reply);
|
||||
}
|
||||
|
||||
export function sessionCookieName(): string { return SESSION_COOKIE; }
|
||||
|
||||
function authenticationRequired(reply: FastifyReply): FastifyReply {
|
||||
return reply.code(401).send({ code: "authentication_required", error: "Authentication is required" });
|
||||
}
|
||||
|
||||
function csrfFailed(reply: FastifyReply): FastifyReply {
|
||||
return reply.code(403).send({ code: "csrf_failed", error: "Request origin validation failed" });
|
||||
}
|
||||
|
||||
export function configuredOrigin(snapshot: LoadedAuthConfig | undefined): string | undefined {
|
||||
try {
|
||||
const publicUrl = snapshot?.value.publicUrl;
|
||||
return publicUrl ? new URL(publicUrl).origin : undefined;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
function readSessionCookie(request: FastifyRequest): string | false | undefined {
|
||||
const raw = request.headers.cookie;
|
||||
if (raw === undefined) return undefined;
|
||||
if (Array.isArray(raw) || typeof raw !== "string" || raw.length > 4096) return false;
|
||||
const values = raw.split(";").filter((part) => /^\s*thothii_session(?:=|\s*$)/.test(part));
|
||||
if (values.length !== 1) return values.length === 0 ? undefined : false;
|
||||
const match = /^\s*thothii_session=([A-Za-z0-9_-]{43})\s*$/.exec(values[0]);
|
||||
return match?.[1] ?? false;
|
||||
}
|
||||
|
||||
function singleHeader(value: string | string[] | undefined): string | false | undefined {
|
||||
if (value === undefined) return undefined;
|
||||
if (Array.isArray(value) || typeof value !== "string" || value.includes(",")) return false;
|
||||
return value;
|
||||
}
|
||||
|
||||
function matchesOrigin(request: FastifyRequest, expectedOrigin: string): boolean {
|
||||
const origin = singleHeader(request.headers.origin);
|
||||
try {
|
||||
if (origin === undefined || origin === false || new URL(origin).origin !== expectedOrigin) return false;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
const fetchSite = singleHeader(request.headers["sec-fetch-site"]);
|
||||
return fetchSite === undefined || fetchSite === "same-origin";
|
||||
}
|
||||
|
||||
function isPublicRoute(request: FastifyRequest): boolean {
|
||||
const rawUrl = request.raw.url ?? request.url;
|
||||
const query = rawUrl.indexOf("?");
|
||||
const pathname = query === -1 ? rawUrl : rawUrl.slice(0, query);
|
||||
return (request.method === "GET" && (pathname === "/health" || pathname === "/auth/config"
|
||||
|| pathname === "/auth/oidc/login" || pathname === "/auth/oidc/callback"))
|
||||
|| (request.method === "POST" && pathname === "/auth/local/login");
|
||||
}
|
||||
|
||||
export function getPrincipal(req: FastifyRequest): PrincipalContext {
|
||||
if (!req.principal) throw new Error("principal missing after authentication");
|
||||
return req.principal;
|
||||
|
||||
@@ -0,0 +1,236 @@
|
||||
import type { AuthDiagnostic, GroupCatalog } from "./group-catalog.js";
|
||||
import { isUsableAuthenticationSecret } from "./secret-policy.js";
|
||||
import { parseConfiguredTransportUrl } from "./url-policy.js";
|
||||
|
||||
const MAX_RESPONSE_BYTES = 1024 * 1024;
|
||||
const REQUEST_TIMEOUT_MS = 5_000;
|
||||
|
||||
export interface AuthentikGroupCatalogOptions {
|
||||
baseUrl: string;
|
||||
apiToken: string;
|
||||
fetch?: typeof globalThis.fetch;
|
||||
}
|
||||
|
||||
function diagnostic(
|
||||
code: AuthDiagnostic["code"],
|
||||
message: string,
|
||||
field?: string,
|
||||
): AuthDiagnostic {
|
||||
return { level: "error", code, message, ...(field === undefined ? {} : { field }) };
|
||||
}
|
||||
|
||||
function catalogUnreachable(): AuthDiagnostic {
|
||||
return diagnostic("oidc_group_catalog_unreachable", "The configured group catalog is unavailable.");
|
||||
}
|
||||
|
||||
function catalogUnauthorized(): AuthDiagnostic {
|
||||
return diagnostic("oidc_group_catalog_unauthorized", "The configured group catalog credentials were rejected.");
|
||||
}
|
||||
|
||||
function missing(name: string): AuthDiagnostic {
|
||||
return diagnostic("oidc_mapped_group_missing", "A configured authorization group does not exist.", name);
|
||||
}
|
||||
|
||||
function ambiguous(name: string): AuthDiagnostic {
|
||||
return diagnostic("oidc_mapped_group_ambiguous", "A configured authorization group is ambiguous.", name);
|
||||
}
|
||||
|
||||
function stableCompare(left: string, right: string): number {
|
||||
return left < right ? -1 : left > right ? 1 : 0;
|
||||
}
|
||||
|
||||
function abortReason(signal: AbortSignal): unknown {
|
||||
return signal.reason ?? new DOMException("The operation was aborted", "AbortError");
|
||||
}
|
||||
|
||||
function cancelResponse(response: Response): void {
|
||||
try {
|
||||
const cancelled = response.body?.cancel();
|
||||
if (cancelled) void cancelled.catch(() => undefined);
|
||||
} catch { /* cancellation is advisory and never changes the diagnostic */ }
|
||||
}
|
||||
|
||||
function cancelReader(reader: ReadableStreamDefaultReader<Uint8Array>): void {
|
||||
try {
|
||||
const cancelled = reader.cancel();
|
||||
void cancelled.catch(() => undefined);
|
||||
} catch { /* cancellation is advisory and never changes the diagnostic */ }
|
||||
}
|
||||
|
||||
function awaitWithAbort<T>(
|
||||
operation: Promise<T>,
|
||||
signal: AbortSignal,
|
||||
onLateResolution?: (value: T) => void,
|
||||
): Promise<T> {
|
||||
return new Promise<T>((resolve, reject) => {
|
||||
let settled = false;
|
||||
const abort = () => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
signal.removeEventListener("abort", abort);
|
||||
reject(abortReason(signal));
|
||||
};
|
||||
if (signal.aborted) {
|
||||
abort();
|
||||
return;
|
||||
}
|
||||
signal.addEventListener("abort", abort, { once: true });
|
||||
operation.then(
|
||||
(value) => {
|
||||
if (settled) {
|
||||
try { onLateResolution?.(value); } catch { /* best-effort cleanup only */ }
|
||||
return;
|
||||
}
|
||||
settled = true;
|
||||
signal.removeEventListener("abort", abort);
|
||||
resolve(value);
|
||||
},
|
||||
(error: unknown) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
signal.removeEventListener("abort", abort);
|
||||
reject(error);
|
||||
},
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
function validContentLength(response: Response): boolean {
|
||||
const value = response.headers.get("content-length");
|
||||
if (value === null) return true;
|
||||
if (!/^\d+$/.test(value)) return false;
|
||||
const length = Number(value);
|
||||
return Number.isSafeInteger(length) && length <= MAX_RESPONSE_BYTES;
|
||||
}
|
||||
|
||||
async function readBounded(response: Response, signal: AbortSignal): Promise<Uint8Array | undefined> {
|
||||
if (!validContentLength(response)) {
|
||||
cancelResponse(response);
|
||||
return undefined;
|
||||
}
|
||||
const reader = response.body?.getReader();
|
||||
if (!reader) return new Uint8Array();
|
||||
const chunks: Uint8Array[] = [];
|
||||
let size = 0;
|
||||
let complete = false;
|
||||
try {
|
||||
while (true) {
|
||||
const { done, value } = await awaitWithAbort(reader.read(), signal);
|
||||
if (done) break;
|
||||
if (value.byteLength > MAX_RESPONSE_BYTES - size) return undefined;
|
||||
chunks.push(value);
|
||||
size += value.byteLength;
|
||||
}
|
||||
complete = true;
|
||||
const body = new Uint8Array(size);
|
||||
let offset = 0;
|
||||
for (const chunk of chunks) {
|
||||
body.set(chunk, offset);
|
||||
offset += chunk.byteLength;
|
||||
}
|
||||
return body;
|
||||
} finally {
|
||||
if (!complete) cancelReader(reader);
|
||||
try { reader.releaseLock(); } catch { /* reader may already be unusable */ }
|
||||
}
|
||||
}
|
||||
|
||||
type GroupResult = "present" | "missing" | "ambiguous" | "unauthorized" | "unreachable";
|
||||
|
||||
function exactResult(name: string, parsed: unknown): GroupResult {
|
||||
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return "unreachable";
|
||||
const record = parsed as { results?: unknown; pagination?: unknown };
|
||||
if (!Array.isArray(record.results) || record.results.length > 2
|
||||
|| !record.pagination || typeof record.pagination !== "object"
|
||||
|| Array.isArray(record.pagination)) return "unreachable";
|
||||
if (!Object.prototype.hasOwnProperty.call(record.pagination, "next")) return "unreachable";
|
||||
const next = (record.pagination as { next: unknown }).next;
|
||||
if (next !== null) {
|
||||
if (typeof next !== "string" || next.length === 0 || next.length > 2048 || /\p{Cc}/u.test(next)) return "unreachable";
|
||||
try {
|
||||
const continuation = new URL(next);
|
||||
if (continuation.protocol !== "https:" || continuation.username || continuation.password || continuation.hash) return "unreachable";
|
||||
} catch {
|
||||
return "unreachable";
|
||||
}
|
||||
return "ambiguous";
|
||||
}
|
||||
const resultNames: string[] = [];
|
||||
for (const result of record.results) {
|
||||
if (!result || typeof result !== "object" || Array.isArray(result)
|
||||
|| typeof (result as { name?: unknown }).name !== "string") return "unreachable";
|
||||
resultNames.push((result as { name: string }).name);
|
||||
}
|
||||
const exactMatches = resultNames.filter((candidate) => candidate === name).length;
|
||||
if (exactMatches === 0) return "missing";
|
||||
return exactMatches === 1 ? "present" : "ambiguous";
|
||||
}
|
||||
|
||||
export function createAuthentikGroupCatalog(options: AuthentikGroupCatalogOptions): GroupCatalog {
|
||||
const origin = parseConfiguredTransportUrl(options.baseUrl, { allowLoopbackHttp: false, originOnly: true });
|
||||
const fetchImplementation = options.fetch ?? globalThis.fetch;
|
||||
const valid = origin !== undefined
|
||||
&& isUsableAuthenticationSecret("THT_AUTHENTIK_API_TOKEN", options.apiToken)
|
||||
&& typeof fetchImplementation === "function";
|
||||
|
||||
async function verify(name: string, signal: AbortSignal): Promise<GroupResult> {
|
||||
if (!origin || !valid || signal.aborted) return "unreachable";
|
||||
const target = new URL("/api/v3/core/groups/", origin);
|
||||
target.searchParams.set("name", name);
|
||||
target.searchParams.set("include_users", "false");
|
||||
target.searchParams.set("page_size", "2");
|
||||
const timeout = new AbortController();
|
||||
const timer = setTimeout(() => timeout.abort(), REQUEST_TIMEOUT_MS);
|
||||
timer.unref();
|
||||
const requestSignal = AbortSignal.any([signal, timeout.signal]);
|
||||
try {
|
||||
const response = await awaitWithAbort(
|
||||
Promise.resolve().then(() => fetchImplementation(target, {
|
||||
headers: { accept: "application/json", authorization: `Bearer ${options.apiToken}` },
|
||||
redirect: "error",
|
||||
signal: requestSignal,
|
||||
})),
|
||||
requestSignal,
|
||||
cancelResponse,
|
||||
);
|
||||
if (response.redirected || response.type === "opaqueredirect" || response.status >= 300 && response.status < 400) {
|
||||
cancelResponse(response);
|
||||
return "unreachable";
|
||||
}
|
||||
if (response.status === 401 || response.status === 403) {
|
||||
cancelResponse(response);
|
||||
return "unauthorized";
|
||||
}
|
||||
if (!response.ok) {
|
||||
cancelResponse(response);
|
||||
return "unreachable";
|
||||
}
|
||||
const body = await readBounded(response, requestSignal);
|
||||
if (body === undefined) return "unreachable";
|
||||
try {
|
||||
return exactResult(name, JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(body)));
|
||||
} catch {
|
||||
return "unreachable";
|
||||
}
|
||||
} catch {
|
||||
return "unreachable";
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
async verifyConfiguredGroups(names, signal) {
|
||||
const diagnostics: AuthDiagnostic[] = [];
|
||||
for (const name of [...new Set(names)].sort(stableCompare)) {
|
||||
const outcome = await verify(name, signal);
|
||||
if (outcome === "present") continue;
|
||||
if (outcome === "missing") diagnostics.push(missing(name));
|
||||
else if (outcome === "ambiguous") diagnostics.push(ambiguous(name));
|
||||
else if (outcome === "unauthorized") return [catalogUnauthorized()];
|
||||
else return [catalogUnreachable()];
|
||||
}
|
||||
return diagnostics;
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
import type { FastifyReply, FastifyRequest } from "fastify";
|
||||
import type { Permission } from "./types.js";
|
||||
import { getPrincipal } from "./auth.js";
|
||||
import type { PrincipalContext } from "./principal.js";
|
||||
|
||||
export function hasPermission(principal: PrincipalContext, permission: Permission): boolean {
|
||||
return principal.permissions.includes(permission);
|
||||
}
|
||||
|
||||
export function isPrincipalContext(
|
||||
value: PrincipalContext | FastifyReply,
|
||||
): value is PrincipalContext {
|
||||
return "issuer" in value;
|
||||
}
|
||||
|
||||
export function requirePermission(
|
||||
request: FastifyRequest,
|
||||
reply: FastifyReply,
|
||||
permission: Permission,
|
||||
): PrincipalContext | FastifyReply {
|
||||
const principal = getPrincipal(request);
|
||||
if (hasPermission(principal, permission)) return principal;
|
||||
return reply.code(403).send({ code: "auth_forbidden", error: "This operation is not permitted" });
|
||||
}
|
||||
|
||||
/**
|
||||
* A resolved cookie session is populated only by the central auth boundary, after its
|
||||
* request-snapshot Origin and CSRF checks. Route-specific legacy guards must not reinterpret
|
||||
* the internal transport host/protocol for that already-authorized browser request.
|
||||
*/
|
||||
export function hasCookieBackedAuthSession(request: FastifyRequest): boolean {
|
||||
return request.authSession !== undefined;
|
||||
}
|
||||
|
||||
/** Permit non-browser clients and browsers whose declared origin matches the request host. */
|
||||
export function requireSameOriginOrNonBrowser(
|
||||
request: FastifyRequest,
|
||||
reply: FastifyReply,
|
||||
): FastifyReply | undefined {
|
||||
if (hasCookieBackedAuthSession(request)) return undefined;
|
||||
const origin = request.headers.origin;
|
||||
if (origin === undefined) return undefined;
|
||||
if (typeof origin !== "string" || typeof request.headers.host !== "string") {
|
||||
return reply.code(403).send({ code: "auth_forbidden", error: "This operation is not permitted" });
|
||||
}
|
||||
try {
|
||||
const supplied = new URL(origin);
|
||||
const expected = new URL(`${request.protocol}://${request.headers.host}`);
|
||||
if (supplied.origin === expected.origin) return undefined;
|
||||
} catch {
|
||||
// Invalid browser origins are forbidden below.
|
||||
}
|
||||
return reply.code(403).send({ code: "auth_forbidden", error: "This operation is not permitted" });
|
||||
}
|
||||
@@ -0,0 +1,320 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import {
|
||||
closeSync,
|
||||
constants,
|
||||
fstatSync,
|
||||
lstatSync,
|
||||
openSync,
|
||||
readSync,
|
||||
realpathSync,
|
||||
} from "node:fs";
|
||||
import type { Stats } from "node:fs";
|
||||
import { dirname, isAbsolute, normalize } from "node:path";
|
||||
import { parseDocument } from "yaml";
|
||||
import { z } from "zod";
|
||||
import type {
|
||||
AuthenticationConfig,
|
||||
AuthenticationConfigProvider,
|
||||
AuthMode,
|
||||
LoadedAuthConfig,
|
||||
Permission,
|
||||
Role,
|
||||
} from "./types.js";
|
||||
import { parseConfiguredTransportUrl } from "./url-policy.js";
|
||||
import { createWindowsAuthStorageBridge, type WindowsAuthStorageBridge } from "./windows-auth-storage.js";
|
||||
|
||||
export type {
|
||||
AuthenticationConfig,
|
||||
AuthenticationConfigProvider,
|
||||
AuthMode,
|
||||
LoadedAuthConfig,
|
||||
Permission,
|
||||
Role,
|
||||
} from "./types.js";
|
||||
|
||||
const MAX_AUTH_CONFIG_BYTES = 1024 * 1024;
|
||||
// Keep live catalog work within the same deterministic bound as the mandatory direct groups claim.
|
||||
const MAX_MAPPED_GROUPS = 128;
|
||||
const ROLES = ["user", "admin"] as const;
|
||||
export const PERMISSION_CATALOG: readonly Permission[] = [
|
||||
"session.use", "session.read_all", "session.manage_all", "settings.manage",
|
||||
"workspace.manage", "workspace.secrets.manage", "database.manage", "memory.manage", "evidence.manage", "pi.manage", "auth.diagnostics.read",
|
||||
];
|
||||
|
||||
const invalid = (): Error => new Error("authentication configuration is invalid");
|
||||
const nonEmptyText = z.string().min(1).max(512).refine(
|
||||
(value) => value.trim() === value && !/[\u0000-\u001f\u007f]/.test(value),
|
||||
);
|
||||
const positiveSeconds = z.number().int().min(1).max(365 * 24 * 60 * 60);
|
||||
const sessionSchema = z.strictObject({
|
||||
regularTtlSeconds: positiveSeconds.default(43_200),
|
||||
regularIdleSeconds: positiveSeconds.default(7_200),
|
||||
rememberTtlSeconds: positiveSeconds.default(2_592_000),
|
||||
rememberIdleSeconds: positiveSeconds.default(604_800),
|
||||
oidcTtlSeconds: positiveSeconds.default(28_800),
|
||||
});
|
||||
const roleSchema = z.enum(ROLES);
|
||||
const groupNameSchema = nonEmptyText.max(256);
|
||||
const groupRolesSchema = z.record(groupNameSchema, z.array(roleSchema).min(1))
|
||||
.refine((value) => Object.keys(value).length <= MAX_MAPPED_GROUPS);
|
||||
|
||||
const localSchema = z.strictObject({
|
||||
version: z.literal(1), mode: z.literal("local"), publicUrl: nonEmptyText, session: sessionSchema.optional(),
|
||||
local: z.strictObject({ usersFile: nonEmptyText.max(255) }),
|
||||
});
|
||||
const oidcSchema = z.strictObject({
|
||||
version: z.literal(1), mode: z.literal("oidc"), publicUrl: nonEmptyText, session: sessionSchema.optional(),
|
||||
oidc: z.strictObject({
|
||||
issuer: nonEmptyText, clientId: nonEmptyText, clientSecretRef: z.literal("THT_OIDC_CLIENT_SECRET"),
|
||||
scopes: z.array(nonEmptyText).min(1).max(16), groupsClaim: z.literal("groups"),
|
||||
}),
|
||||
groupCatalog: z.strictObject({
|
||||
driver: z.literal("authentik"), baseUrl: nonEmptyText, apiTokenRef: z.literal("THT_AUTHENTIK_API_TOKEN"),
|
||||
}),
|
||||
authorization: z.strictObject({ groupRoles: groupRolesSchema }),
|
||||
});
|
||||
|
||||
interface FileIdentity {
|
||||
dev: number;
|
||||
ino: number;
|
||||
uid: number;
|
||||
size: number;
|
||||
mtimeMs: number;
|
||||
ctimeMs: number;
|
||||
mode: number;
|
||||
nlink: number;
|
||||
}
|
||||
|
||||
interface DirectoryIdentity {
|
||||
dev: number;
|
||||
ino: number;
|
||||
uid: number;
|
||||
mode: number;
|
||||
ctimeMs: number;
|
||||
}
|
||||
|
||||
interface StorageIdentity {
|
||||
file: FileIdentity;
|
||||
directory: DirectoryIdentity;
|
||||
}
|
||||
|
||||
export interface AuthenticationConfigLoadOptions {
|
||||
/** Test seam; production creates the existing bounded internal tht auth-storage bridge. */
|
||||
windowsStorageBridge?: Pick<WindowsAuthStorageBridge, "readAuthConfig">;
|
||||
}
|
||||
|
||||
function validateCanonicalPath(path: string): void {
|
||||
if (typeof path !== "string" || path.length === 0 || path.trim() !== path
|
||||
|| path.includes("\0") || !isAbsolute(path) || normalize(path) !== path
|
||||
|| realpathSync(path) !== path || realpathSync(dirname(path)) !== dirname(path)) throw invalid();
|
||||
}
|
||||
|
||||
function runtimeOwner(): number {
|
||||
if (process.platform === "win32" || typeof process.geteuid !== "function") throw invalid();
|
||||
const owner = process.geteuid();
|
||||
if (!Number.isSafeInteger(owner) || owner < 0) throw invalid();
|
||||
return owner;
|
||||
}
|
||||
|
||||
function fileMetadata(info: Stats): FileIdentity {
|
||||
const mode = info.mode & 0o7777;
|
||||
if (!info.isFile() || info.uid !== runtimeOwner() || info.nlink !== 1 || mode !== 0o600
|
||||
|| info.size < 0 || info.size > MAX_AUTH_CONFIG_BYTES) throw invalid();
|
||||
return {
|
||||
dev: info.dev, ino: info.ino, uid: info.uid, size: info.size,
|
||||
mtimeMs: info.mtimeMs, ctimeMs: info.ctimeMs, mode, nlink: info.nlink,
|
||||
};
|
||||
}
|
||||
|
||||
function directoryMetadata(info: Stats): DirectoryIdentity {
|
||||
const mode = info.mode & 0o7777;
|
||||
if (!info.isDirectory() || info.uid !== runtimeOwner() || mode !== 0o700) throw invalid();
|
||||
return { dev: info.dev, ino: info.ino, uid: info.uid, mode, ctimeMs: info.ctimeMs };
|
||||
}
|
||||
|
||||
function sameFileIdentity(left: FileIdentity, right: FileIdentity): boolean {
|
||||
return left.dev === right.dev && left.ino === right.ino && left.uid === right.uid
|
||||
&& left.size === right.size && left.mtimeMs === right.mtimeMs && left.ctimeMs === right.ctimeMs
|
||||
&& left.mode === right.mode && left.nlink === right.nlink;
|
||||
}
|
||||
|
||||
function sameDirectoryIdentity(left: DirectoryIdentity, right: DirectoryIdentity): boolean {
|
||||
return left.dev === right.dev && left.ino === right.ino && left.uid === right.uid
|
||||
&& left.mode === right.mode && left.ctimeMs === right.ctimeMs;
|
||||
}
|
||||
|
||||
function sameIdentity(left: StorageIdentity, right: StorageIdentity): boolean {
|
||||
return sameFileIdentity(left.file, right.file) && sameDirectoryIdentity(left.directory, right.directory);
|
||||
}
|
||||
|
||||
function storageIdentity(path: string): StorageIdentity {
|
||||
try {
|
||||
validateCanonicalPath(path);
|
||||
return {
|
||||
file: fileMetadata(lstatSync(path) as Stats),
|
||||
directory: directoryMetadata(lstatSync(dirname(path)) as Stats),
|
||||
};
|
||||
} catch {
|
||||
throw invalid();
|
||||
}
|
||||
}
|
||||
|
||||
function openDirectoryDescriptor(path: string): number {
|
||||
return openSync(path, constants.O_RDONLY | (constants.O_DIRECTORY ?? 0)
|
||||
| (constants.O_NOFOLLOW ?? 0) | (constants.O_NONBLOCK ?? 0));
|
||||
}
|
||||
|
||||
function readBoundedConfig(path: string): { source: string; identity: StorageIdentity } {
|
||||
let directoryDescriptor: number | undefined;
|
||||
let fd: number | undefined;
|
||||
try {
|
||||
const before = storageIdentity(path);
|
||||
directoryDescriptor = openDirectoryDescriptor(dirname(path));
|
||||
const openedDirectory = directoryMetadata(fstatSync(directoryDescriptor) as Stats);
|
||||
if (!sameDirectoryIdentity(before.directory, openedDirectory)) throw invalid();
|
||||
fd = openSync(path, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
|
||||
const opened = fileMetadata(fstatSync(fd) as Stats);
|
||||
if (!sameFileIdentity(before.file, opened)) throw invalid();
|
||||
const buffer = Buffer.allocUnsafe(MAX_AUTH_CONFIG_BYTES + 1);
|
||||
let offset = 0;
|
||||
while (offset < buffer.length) {
|
||||
const bytesRead = readSync(fd, buffer, offset, buffer.length - offset, null);
|
||||
if (bytesRead === 0) break;
|
||||
offset += bytesRead;
|
||||
}
|
||||
if (offset > MAX_AUTH_CONFIG_BYTES) throw invalid();
|
||||
const afterFile = fileMetadata(fstatSync(fd) as Stats);
|
||||
const afterPath = storageIdentity(path);
|
||||
const afterOpenedDirectory = directoryMetadata(fstatSync(directoryDescriptor) as Stats);
|
||||
if (!sameFileIdentity(opened, afterFile) || !sameFileIdentity(afterFile, afterPath.file)
|
||||
|| !sameDirectoryIdentity(before.directory, afterPath.directory)
|
||||
|| !sameDirectoryIdentity(openedDirectory, afterOpenedDirectory)) throw invalid();
|
||||
validateCanonicalPath(path);
|
||||
return {
|
||||
source: new TextDecoder("utf-8", { fatal: true }).decode(buffer.subarray(0, offset)),
|
||||
identity: afterPath,
|
||||
};
|
||||
} catch {
|
||||
throw invalid();
|
||||
} finally {
|
||||
if (fd !== undefined) try { closeSync(fd); } catch { /* sanitized by design */ }
|
||||
if (directoryDescriptor !== undefined) try { closeSync(directoryDescriptor); } catch { /* sanitized by design */ }
|
||||
}
|
||||
}
|
||||
|
||||
function validOrigin(value: string, httpLoopbackAllowed: boolean): boolean {
|
||||
return parseConfiguredTransportUrl(value, { allowLoopbackHttp: httpLoopbackAllowed, originOnly: true }) !== undefined;
|
||||
}
|
||||
|
||||
function validIssuer(value: string): boolean {
|
||||
return parseConfiguredTransportUrl(value, { allowLoopbackHttp: false }) !== undefined;
|
||||
}
|
||||
|
||||
function validUsersFile(value: string): boolean {
|
||||
return /^[A-Za-z0-9][A-Za-z0-9._-]*\.yaml$/.test(value);
|
||||
}
|
||||
|
||||
function canonicalize(value: unknown): unknown {
|
||||
if (Array.isArray(value)) return value.map(canonicalize);
|
||||
if (value && typeof value === "object") {
|
||||
return Object.fromEntries(Object.entries(value as Record<string, unknown>)
|
||||
.sort(([left], [right]) => left < right ? -1 : left > right ? 1 : 0)
|
||||
.map(([key, nested]) => [key, canonicalize(nested)]));
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function canonicalRevision(value: AuthenticationConfig): string {
|
||||
return createHash("sha256").update(JSON.stringify(canonicalize(value))).digest("hex");
|
||||
}
|
||||
|
||||
export function parseAuthenticationConfigSource(source: string): AuthenticationConfig {
|
||||
try {
|
||||
const document = parseDocument(source, { uniqueKeys: true });
|
||||
if (document.errors.length > 0 || document.warnings.length > 0) throw invalid();
|
||||
const parsed = document.toJSON();
|
||||
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) throw invalid();
|
||||
const config = parsed as Record<string, unknown>;
|
||||
const schema = config.mode === "local" ? localSchema : config.mode === "oidc" ? oidcSchema : undefined;
|
||||
if (!schema) throw invalid();
|
||||
const validated = schema.parse(config);
|
||||
const session = sessionSchema.parse(validated.session ?? {});
|
||||
if (!validOrigin(validated.publicUrl, true)) throw invalid();
|
||||
if (validated.mode === "local") {
|
||||
if (!validUsersFile(validated.local.usersFile)) throw invalid();
|
||||
return { ...validated, session };
|
||||
}
|
||||
if (!validIssuer(validated.oidc.issuer) || !validOrigin(validated.groupCatalog.baseUrl, false)) throw invalid();
|
||||
if (!validated.oidc.scopes.includes("openid")) throw invalid();
|
||||
const mappings = Object.entries(validated.authorization.groupRoles);
|
||||
if (mappings.length === 0 || mappings.filter(([, roles]) => roles.includes("admin")).length !== 1) throw invalid();
|
||||
return { ...validated, session };
|
||||
} catch { throw invalid(); }
|
||||
}
|
||||
|
||||
function loadAuthenticationConfigWithIdentity(path: string): { loaded: LoadedAuthConfig; identity: StorageIdentity } {
|
||||
const read = readBoundedConfig(path);
|
||||
const value = parseAuthenticationConfigSource(read.source);
|
||||
return { loaded: { value, revision: canonicalRevision(value), sourcePath: path }, identity: read.identity };
|
||||
}
|
||||
|
||||
function loadWindowsAuthenticationConfig(
|
||||
path: string,
|
||||
bridge: Pick<WindowsAuthStorageBridge, "readAuthConfig">,
|
||||
): LoadedAuthConfig {
|
||||
try {
|
||||
const contents = bridge.readAuthConfig(path);
|
||||
if (!Buffer.isBuffer(contents) || contents.length === 0 || contents.length > MAX_AUTH_CONFIG_BYTES) throw invalid();
|
||||
const source = new TextDecoder("utf-8", { fatal: true }).decode(contents);
|
||||
const value = parseAuthenticationConfigSource(source);
|
||||
return { value, revision: canonicalRevision(value), sourcePath: path };
|
||||
} catch {
|
||||
throw invalid();
|
||||
}
|
||||
}
|
||||
|
||||
export function loadAuthenticationConfig(path: string, options: AuthenticationConfigLoadOptions = {}): LoadedAuthConfig {
|
||||
if (process.platform === "win32") {
|
||||
return loadWindowsAuthenticationConfig(path, options.windowsStorageBridge ?? createWindowsAuthStorageBridge());
|
||||
}
|
||||
return loadAuthenticationConfigWithIdentity(path).loaded;
|
||||
}
|
||||
|
||||
export function createAuthenticationConfigProvider(
|
||||
path: string,
|
||||
options: AuthenticationConfigLoadOptions = {},
|
||||
): AuthenticationConfigProvider {
|
||||
if (process.platform === "win32") {
|
||||
const bridge = options.windowsStorageBridge ?? createWindowsAuthStorageBridge();
|
||||
return { current: () => loadWindowsAuthenticationConfig(path, bridge) };
|
||||
}
|
||||
let cached: { identity: StorageIdentity; loaded: LoadedAuthConfig } | undefined;
|
||||
return { current(): LoadedAuthConfig {
|
||||
const before = storageIdentity(path);
|
||||
if (cached && sameIdentity(cached.identity, before)) return cached.loaded;
|
||||
for (let attempt = 0; attempt < 2; attempt += 1) {
|
||||
try {
|
||||
const { loaded, identity } = loadAuthenticationConfigWithIdentity(path);
|
||||
if (sameIdentity(identity, storageIdentity(path))) {
|
||||
cached = { identity, loaded };
|
||||
return loaded;
|
||||
}
|
||||
} catch { /* retry one concurrent atomic replacement, then fail closed */ }
|
||||
}
|
||||
throw invalid();
|
||||
} };
|
||||
}
|
||||
|
||||
export function rolesToPermissions(roles: readonly Role[]): readonly Permission[] {
|
||||
const requested = new Set<Role>();
|
||||
for (const role of roles) {
|
||||
if (!ROLES.includes(role)) throw invalid();
|
||||
requested.add(role);
|
||||
}
|
||||
if (requested.has("admin")) return PERMISSION_CATALOG;
|
||||
return requested.has("user") ? ["session.use"] : [];
|
||||
}
|
||||
|
||||
export function isPermission(value: string): value is Permission {
|
||||
return PERMISSION_CATALOG.includes(value as Permission);
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
import { timingSafeEqual } from "node:crypto";
|
||||
import { deriveCsrfToken as deriveStoredCsrfToken } from "./session-store.js";
|
||||
|
||||
export { deriveStoredCsrfToken as deriveCsrfToken };
|
||||
|
||||
/** Compare a client-supplied CSRF value without exposing a useful length timing oracle. */
|
||||
export function csrfTokensEqual(expectedToken: string, suppliedToken: string | undefined): boolean {
|
||||
const expected = Buffer.from(expectedToken, "utf8");
|
||||
const supplied = Buffer.from(suppliedToken ?? "", "utf8");
|
||||
const padded = Buffer.alloc(expected.length);
|
||||
supplied.copy(padded, 0, 0, expected.length);
|
||||
return timingSafeEqual(expected, padded) && supplied.length === expected.length;
|
||||
}
|
||||
@@ -0,0 +1,338 @@
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { resolve } from "node:path";
|
||||
import {
|
||||
closeSync, constants, fstatSync, lstatSync, openSync, readFileSync,
|
||||
} from "node:fs";
|
||||
import { loadConfig, type AppConfig } from "../config.js";
|
||||
import { loadSecretBundle, secretValue } from "../config/secret-bundle.js";
|
||||
import { createAuthentikGroupCatalog } from "./authentik-group-catalog.js";
|
||||
import { createCurrentLocalUserRegistryResolver, type LocalUserRegistry } from "./local-registry.js";
|
||||
import { createOidcProtocol, OidcDeviceFlowUnavailableError, type OidcProtocol } from "./oidc-client.js";
|
||||
import { createAuthDiagnoser, type AuthDiagnoser, type AuthDiagnostic, type AuthDiagnostics } from "./diagnostics.js";
|
||||
import { decodeAuthDiagnostics, type GroupCatalog } from "./group-catalog.js";
|
||||
import type { LoadedAuthConfig } from "./types.js";
|
||||
|
||||
const AUTH_SECRET_REFERENCES = ["THT_OIDC_CLIENT_SECRET", "THT_AUTHENTIK_API_TOKEN"] as const;
|
||||
const MAX_DIAGNOSTIC_SECRET_SOURCE_BYTES = 64 * 1024;
|
||||
const MAX_DIAGNOSTIC_SECRET_VALUES = 4096;
|
||||
const MAX_DIAGNOSTIC_SECRET_DEPTH = 32;
|
||||
|
||||
function unavailableSecretCorpus(): Error {
|
||||
return new Error("diagnostic secret corpus is unavailable");
|
||||
}
|
||||
|
||||
function readMountedSecretSource(file: string): string {
|
||||
let fd: number | undefined;
|
||||
try {
|
||||
if (!file || file.trim() !== file || file.includes("\0")) throw unavailableSecretCorpus();
|
||||
const before = lstatSync(file);
|
||||
if (!before.isFile() || before.isSymbolicLink() || before.size > MAX_DIAGNOSTIC_SECRET_SOURCE_BYTES) {
|
||||
throw unavailableSecretCorpus();
|
||||
}
|
||||
fd = openSync(file, constants.O_RDONLY | constants.O_NOFOLLOW);
|
||||
const opened = fstatSync(fd);
|
||||
if (!opened.isFile() || opened.size > MAX_DIAGNOSTIC_SECRET_SOURCE_BYTES
|
||||
|| before.dev !== opened.dev || before.ino !== opened.ino) {
|
||||
throw unavailableSecretCorpus();
|
||||
}
|
||||
const value = readFileSync(fd, "utf8");
|
||||
if (Buffer.byteLength(value, "utf8") > MAX_DIAGNOSTIC_SECRET_SOURCE_BYTES) {
|
||||
throw unavailableSecretCorpus();
|
||||
}
|
||||
return value;
|
||||
} catch {
|
||||
throw unavailableSecretCorpus();
|
||||
} finally {
|
||||
if (fd !== undefined) try { closeSync(fd); } catch { /* fixed failure surface above */ }
|
||||
}
|
||||
}
|
||||
|
||||
function parsedSecretValues(raw: string, requireJson: boolean): readonly string[] {
|
||||
const trimmed = raw.trim();
|
||||
if (!trimmed) return [];
|
||||
const values = new Set<string>([raw.replace(/[\r\n]+$/u, "")]);
|
||||
const looksJson = trimmed.startsWith("{") || trimmed.startsWith("[");
|
||||
if (!looksJson) {
|
||||
if (requireJson) throw unavailableSecretCorpus();
|
||||
return [...values];
|
||||
}
|
||||
let document: unknown;
|
||||
try { document = JSON.parse(trimmed); } catch { throw unavailableSecretCorpus(); }
|
||||
if (requireJson && (!document || typeof document !== "object" || Array.isArray(document))) {
|
||||
throw unavailableSecretCorpus();
|
||||
}
|
||||
const pending: Array<{ value: unknown; depth: number }> = [{ value: document, depth: 0 }];
|
||||
let scalarCount = 0;
|
||||
while (pending.length > 0) {
|
||||
const current = pending.pop()!;
|
||||
if (current.depth > MAX_DIAGNOSTIC_SECRET_DEPTH) throw unavailableSecretCorpus();
|
||||
if (Array.isArray(current.value)) {
|
||||
for (const item of current.value) pending.push({ value: item, depth: current.depth + 1 });
|
||||
} else if (current.value && typeof current.value === "object") {
|
||||
for (const item of Object.values(current.value as Record<string, unknown>)) {
|
||||
pending.push({ value: item, depth: current.depth + 1 });
|
||||
}
|
||||
} else {
|
||||
scalarCount += 1;
|
||||
if (scalarCount > 1024) throw unavailableSecretCorpus();
|
||||
if (typeof current.value === "string" && current.value.length > 0) values.add(current.value);
|
||||
}
|
||||
}
|
||||
return [...values];
|
||||
}
|
||||
|
||||
export function configuredSecretValues(config: AppConfig): readonly string[] {
|
||||
try {
|
||||
const values = new Set<string>();
|
||||
if (config.secretsFile) {
|
||||
for (const value of loadSecretBundle(config.secretsFile).values()) values.add(value);
|
||||
}
|
||||
const legacyFiles = new Set(Object.values(config.secretFiles).filter(
|
||||
(file): file is string => file !== undefined,
|
||||
));
|
||||
for (const file of legacyFiles) {
|
||||
for (const value of parsedSecretValues(readMountedSecretSource(file), false)) values.add(value);
|
||||
}
|
||||
if (config.piAuthFile) {
|
||||
for (const value of parsedSecretValues(readMountedSecretSource(config.piAuthFile), true)) values.add(value);
|
||||
}
|
||||
if (values.size > MAX_DIAGNOSTIC_SECRET_VALUES) throw unavailableSecretCorpus();
|
||||
return [...values];
|
||||
} catch {
|
||||
throw unavailableSecretCorpus();
|
||||
}
|
||||
}
|
||||
|
||||
export interface ConfiguredAuthDiagnoserOptions {
|
||||
localUserRegistry?: (loaded: LoadedAuthConfig) => LocalUserRegistry | undefined;
|
||||
oidcProtocol?: (loaded: LoadedAuthConfig) => OidcProtocol | undefined;
|
||||
groupCatalog?: (loaded: LoadedAuthConfig) => GroupCatalog | undefined;
|
||||
sessionRootValidator?: (root: string) => void | Promise<void>;
|
||||
}
|
||||
|
||||
/** Builds the one shared auth diagnostic implementation used by app routes and the one-shot CLI. */
|
||||
export function createConfiguredAuthDiagnoser(
|
||||
config: AppConfig,
|
||||
options: ConfiguredAuthDiagnoserOptions = {},
|
||||
): AuthDiagnoser {
|
||||
const localResolver = options.localUserRegistry === undefined
|
||||
? createCurrentLocalUserRegistryResolver()
|
||||
: undefined;
|
||||
const secretValues = (): ReadonlyMap<string, string> => {
|
||||
const values = new Map<string, string>();
|
||||
for (const reference of AUTH_SECRET_REFERENCES) {
|
||||
try {
|
||||
const value = secretValue(config, reference);
|
||||
if (value !== undefined) values.set(reference, value);
|
||||
} catch {
|
||||
// The shared diagnoser emits the fixed missing-secret diagnostic below.
|
||||
}
|
||||
}
|
||||
return values;
|
||||
};
|
||||
const loaded = (): LoadedAuthConfig | undefined => {
|
||||
try { return config.authentication?.current(); } catch { return undefined; }
|
||||
};
|
||||
return {
|
||||
async inspect(request): Promise<AuthDiagnostics> {
|
||||
const current = loaded();
|
||||
const protocol = current?.value.mode === "oidc"
|
||||
? options.oidcProtocol?.(current) ?? (() => {
|
||||
try {
|
||||
const clientSecret = secretValues().get("THT_OIDC_CLIENT_SECRET");
|
||||
if (!clientSecret) return undefined;
|
||||
return createOidcProtocol({
|
||||
issuer: current.value.oidc.issuer,
|
||||
clientId: current.value.oidc.clientId,
|
||||
clientSecret,
|
||||
callbackUrl: new URL("/api/auth/oidc/callback", current.value.publicUrl).href,
|
||||
scopes: current.value.oidc.scopes,
|
||||
groupsClaim: current.value.oidc.groupsClaim,
|
||||
});
|
||||
} catch { return undefined; }
|
||||
})()
|
||||
: undefined;
|
||||
const groupCatalog = current?.value.mode === "oidc" ? options.groupCatalog?.(current) ?? (() => {
|
||||
try {
|
||||
const token = secretValues().get("THT_AUTHENTIK_API_TOKEN");
|
||||
return token === undefined ? undefined : createAuthentikGroupCatalog({
|
||||
baseUrl: current.value.groupCatalog.baseUrl,
|
||||
apiToken: token,
|
||||
});
|
||||
} catch { return undefined; }
|
||||
})() : undefined;
|
||||
const report = await createAuthDiagnoser({
|
||||
authMode: config.authMode,
|
||||
authStateRoot: config.authStateRoot,
|
||||
...(options.sessionRootValidator === undefined ? {} : { sessionRootValidator: options.sessionRootValidator }),
|
||||
authentication: config.authentication,
|
||||
secrets: secretValues(),
|
||||
localUserRegistry: current?.value.mode === "local"
|
||||
? options.localUserRegistry?.(current) ?? localResolver?.resolve(current)
|
||||
: undefined,
|
||||
oidcProtocol: protocol,
|
||||
groupCatalog,
|
||||
}).inspect(request);
|
||||
if (!request.interactive || !report.ready) return report;
|
||||
if (current?.value.mode !== "oidc" || !protocol?.verifyDeviceFlow || !request.presentDeviceCode) {
|
||||
return {
|
||||
ready: false,
|
||||
mode: report.mode,
|
||||
checks: [{
|
||||
level: "error",
|
||||
code: "oidc_device_flow_unavailable",
|
||||
message: "Interactive authentication diagnostics require OIDC device authorization.",
|
||||
}],
|
||||
};
|
||||
}
|
||||
try {
|
||||
const identity = await protocol.verifyDeviceFlow(
|
||||
request.signal ?? AbortSignal.timeout(10 * 60_000), request.presentDeviceCode,
|
||||
);
|
||||
// Exact names only: unrelated provider groups are neither emitted nor retained.
|
||||
const mappedRoles = new Set<string>();
|
||||
for (const [configuredGroup, roles] of Object.entries(current.value.authorization.groupRoles)) {
|
||||
if (!identity.groups.includes(configuredGroup)) continue;
|
||||
for (const role of roles) mappedRoles.add(role);
|
||||
}
|
||||
if (mappedRoles.size === 0) {
|
||||
return {
|
||||
ready: false,
|
||||
mode: "oidc",
|
||||
checks: [{
|
||||
level: "error",
|
||||
code: "oidc_groups_claim_invalid",
|
||||
message: "The OIDC device-flow identity could not be validated.",
|
||||
}],
|
||||
};
|
||||
}
|
||||
return report;
|
||||
} catch (error) {
|
||||
return {
|
||||
ready: false,
|
||||
mode: "oidc",
|
||||
checks: [{
|
||||
level: "error",
|
||||
code: error instanceof OidcDeviceFlowUnavailableError
|
||||
? "oidc_device_flow_unavailable"
|
||||
: "oidc_groups_claim_invalid",
|
||||
message: error instanceof OidcDeviceFlowUnavailableError
|
||||
? "OIDC device authorization is unavailable."
|
||||
: "The OIDC device-flow identity could not be validated.",
|
||||
}],
|
||||
};
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export interface DiagnosticCommandDependencies {
|
||||
diagnoser: AuthDiagnoser;
|
||||
secretValues?: readonly string[];
|
||||
stdout: (line: string) => void;
|
||||
stderr: (line: string) => void;
|
||||
}
|
||||
|
||||
function genericFailure(): AuthDiagnostics {
|
||||
return {
|
||||
ready: false,
|
||||
mode: "none",
|
||||
checks: [{ level: "error", code: "auth_config_invalid", message: "Authentication configuration is unavailable." }],
|
||||
};
|
||||
}
|
||||
|
||||
function redact(value: string, secrets: readonly string[]): string {
|
||||
let result = value;
|
||||
for (const secret of [...secrets].filter(Boolean).sort((left, right) => right.length - left.length)) {
|
||||
result = result.replaceAll(secret, "[REDACTED]");
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function redactedReport(report: AuthDiagnostics, secrets: readonly string[]): AuthDiagnostics {
|
||||
return {
|
||||
...report,
|
||||
checks: report.checks.map((check): AuthDiagnostic => ({
|
||||
...check,
|
||||
message: redact(check.message, secrets),
|
||||
...(check.field === undefined ? {} : { field: redact(check.field, secrets) }),
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
||||
function parseArguments(args: readonly string[]): { json: true; interactive: boolean } | undefined {
|
||||
let json = false;
|
||||
let interactive = false;
|
||||
for (const arg of args) {
|
||||
if (arg === "--json" && !json) json = true;
|
||||
else if (arg === "--interactive" && !interactive) interactive = true;
|
||||
else return undefined;
|
||||
}
|
||||
return json ? { json: true, interactive } : undefined;
|
||||
}
|
||||
|
||||
/** A bounded machine command: stdout receives exactly one final report and no progress text. */
|
||||
export async function runDiagnosticCommand(
|
||||
args: readonly string[],
|
||||
dependencies: DiagnosticCommandDependencies,
|
||||
): Promise<number> {
|
||||
const options = parseArguments(args);
|
||||
if (!options) {
|
||||
dependencies.stderr("usage: diagnostic-command.js --json [--interactive]");
|
||||
return 2;
|
||||
}
|
||||
let report: AuthDiagnostics;
|
||||
const secrets = dependencies.secretValues ?? [];
|
||||
try {
|
||||
report = await dependencies.diagnoser.inspect({
|
||||
live: true,
|
||||
...(options.interactive ? {
|
||||
interactive: true,
|
||||
presentDeviceCode: (uri: string, code: string) => dependencies.stderr(
|
||||
redact(`Open ${uri} and enter code ${code}`, secrets),
|
||||
),
|
||||
} : {}),
|
||||
});
|
||||
} catch {
|
||||
report = genericFailure();
|
||||
}
|
||||
const decoded = decodeAuthDiagnostics(report) ?? genericFailure();
|
||||
const safe = decodeAuthDiagnostics(redactedReport(decoded, secrets)) ?? genericFailure();
|
||||
dependencies.stdout(`${JSON.stringify(safe)}\n`);
|
||||
return safe.ready ? 0 : 1;
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const exitCode = await runConfiguredDiagnosticCommand(
|
||||
process.argv.slice(2), process.env,
|
||||
(line) => process.stdout.write(line),
|
||||
(line) => process.stderr.write(`${line}\n`),
|
||||
);
|
||||
process.exitCode = exitCode;
|
||||
}
|
||||
|
||||
export async function runConfiguredDiagnosticCommand(
|
||||
args: readonly string[],
|
||||
env: Record<string, string | undefined>,
|
||||
stdout: (line: string) => void,
|
||||
stderr: (line: string) => void,
|
||||
): Promise<number> {
|
||||
let diagnoser: AuthDiagnoser = { inspect: async () => genericFailure() };
|
||||
let secretValues: readonly string[] | undefined;
|
||||
try {
|
||||
const config = loadConfig(env);
|
||||
// Complete this preflight before constructing a diagnoser that may forward a device prompt.
|
||||
secretValues = configuredSecretValues(config);
|
||||
diagnoser = createConfiguredAuthDiagnoser(config);
|
||||
} catch { /* turn startup or corpus faults into the closed report below */ }
|
||||
return runDiagnosticCommand(args, {
|
||||
diagnoser,
|
||||
...(secretValues === undefined ? {} : { secretValues }),
|
||||
stdout,
|
||||
stderr,
|
||||
});
|
||||
}
|
||||
|
||||
if (process.argv[1] !== undefined && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
|
||||
void main();
|
||||
}
|
||||
@@ -0,0 +1,219 @@
|
||||
import type { AuthenticationConfigProvider, AuthMode } from "./types.js";
|
||||
import type { LocalUserRegistry } from "./local-registry.js";
|
||||
import { OidcIssuerMismatchError, OidcJwksUnavailableError, type OidcProtocol } from "./oidc-client.js";
|
||||
import {
|
||||
createPosixAuthStorageBridge,
|
||||
createWindowsAuthStorageBridge,
|
||||
type WindowsAuthStorageBridge,
|
||||
} from "./windows-auth-storage.js";
|
||||
import { isUsableAuthenticationSecret, type AuthenticationSecretReference } from "./secret-policy.js";
|
||||
import type { AuthDiagnostic, AuthDiagnosticCode, AuthDiagnostics, GroupCatalog } from "./group-catalog.js";
|
||||
|
||||
export type { AuthDiagnostic, AuthDiagnosticCode, AuthDiagnostics } from "./group-catalog.js";
|
||||
|
||||
const LIVE_DIAGNOSTIC_TIMEOUT_MS = 30_000;
|
||||
|
||||
export interface AuthDiagnoser {
|
||||
inspect(options: {
|
||||
live: boolean;
|
||||
interactive?: boolean;
|
||||
signal?: AbortSignal;
|
||||
/** Device-code presentation is transient operator output, never persisted diagnostic state. */
|
||||
presentDeviceCode?: (uri: string, code: string) => void;
|
||||
}): Promise<AuthDiagnostics>;
|
||||
}
|
||||
|
||||
export interface AuthDiagnoserDependencies {
|
||||
authMode: AuthMode;
|
||||
authStateRoot: string;
|
||||
/** Platform integrations may inject an equivalent side-effect-free owner/ACL validator. */
|
||||
sessionRootValidator?: (root: string) => void | Promise<void>;
|
||||
windowsStorageBridge?: Pick<WindowsAuthStorageBridge, "validateRoot">;
|
||||
posixStorageBridge?: Pick<WindowsAuthStorageBridge, "validateRoot">;
|
||||
authentication?: AuthenticationConfigProvider;
|
||||
secrets?: ReadonlyMap<string, string>;
|
||||
localUserRegistry?: LocalUserRegistry;
|
||||
oidcProtocol?: OidcProtocol;
|
||||
groupCatalog?: GroupCatalog;
|
||||
}
|
||||
|
||||
function check(code: AuthDiagnosticCode, message: string, field?: string): AuthDiagnostic {
|
||||
return { level: "error", code, message, ...(field === undefined ? {} : { field }) };
|
||||
}
|
||||
|
||||
function ordered(checks: readonly AuthDiagnostic[]): readonly AuthDiagnostic[] {
|
||||
const unique = new Map<string, AuthDiagnostic>();
|
||||
for (const item of checks) unique.set(`${item.code}\u0000${item.field ?? ""}`, item);
|
||||
return [...unique.values()].sort((left, right) => {
|
||||
const leftKey = `${left.code}\u0000${left.field ?? ""}`;
|
||||
const rightKey = `${right.code}\u0000${right.field ?? ""}`;
|
||||
return leftKey < rightKey ? -1 : leftKey > rightKey ? 1 : 0;
|
||||
});
|
||||
}
|
||||
|
||||
function stableCompare(left: string, right: string): number {
|
||||
return left < right ? -1 : left > right ? 1 : 0;
|
||||
}
|
||||
|
||||
function abortReason(signal: AbortSignal): unknown {
|
||||
return signal.reason ?? new DOMException("The operation was aborted", "AbortError");
|
||||
}
|
||||
|
||||
function awaitWithAbort<T>(operation: Promise<T>, signal: AbortSignal): Promise<T> {
|
||||
return new Promise<T>((resolve, reject) => {
|
||||
let settled = false;
|
||||
const abort = () => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
signal.removeEventListener("abort", abort);
|
||||
reject(abortReason(signal));
|
||||
};
|
||||
if (signal.aborted) abort();
|
||||
else signal.addEventListener("abort", abort, { once: true });
|
||||
operation.then(
|
||||
(value) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
signal.removeEventListener("abort", abort);
|
||||
resolve(value);
|
||||
},
|
||||
(error: unknown) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
signal.removeEventListener("abort", abort);
|
||||
reject(error);
|
||||
},
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
function startBeforeAbort<T>(signal: AbortSignal, operation: () => Promise<T>): Promise<T> {
|
||||
return Promise.resolve().then(() => {
|
||||
if (signal.aborted) throw abortReason(signal);
|
||||
return operation();
|
||||
});
|
||||
}
|
||||
|
||||
async function localRegistryIsUsable(deps: AuthDiagnoserDependencies): Promise<AuthDiagnostic | undefined> {
|
||||
try {
|
||||
if (!deps.localUserRegistry) {
|
||||
return check("local_user_registry_invalid", "The local user registry is unavailable.");
|
||||
}
|
||||
if (!await deps.localUserRegistry.hasEnabledAdmin()) {
|
||||
return check("local_admin_missing", "No enabled local administrator is configured.");
|
||||
}
|
||||
return undefined;
|
||||
} catch {
|
||||
return check("local_user_registry_invalid", "The local user registry is invalid.");
|
||||
}
|
||||
}
|
||||
|
||||
export function createAuthDiagnoser(deps: AuthDiagnoserDependencies): AuthDiagnoser {
|
||||
const validateSessionRoot = deps.sessionRootValidator ?? (process.platform === "win32"
|
||||
? (root: string) => (deps.windowsStorageBridge ?? createWindowsAuthStorageBridge()).validateRoot(root)
|
||||
: (root: string) => (deps.posixStorageBridge ?? createPosixAuthStorageBridge()).validateRoot(root));
|
||||
return {
|
||||
async inspect(options): Promise<AuthDiagnostics> {
|
||||
const checks: AuthDiagnostic[] = [];
|
||||
const signal = options.signal ?? new AbortController().signal;
|
||||
try {
|
||||
await validateSessionRoot(deps.authStateRoot);
|
||||
} catch {
|
||||
checks.push(check("auth_session_store_invalid", "The authentication session store is invalid."));
|
||||
}
|
||||
|
||||
if (deps.authMode === "none" || deps.authMode === "mock") {
|
||||
const result = ordered(checks);
|
||||
return result.length === 0
|
||||
? { ready: true, mode: deps.authMode, checks: [{ level: "info", code: "auth_ready", message: "Authentication is ready." }] }
|
||||
: { ready: false, mode: deps.authMode, checks: result };
|
||||
}
|
||||
if (deps.authMode === "upstream") {
|
||||
const result = ordered(checks);
|
||||
return result.length === 0
|
||||
? {
|
||||
ready: true,
|
||||
mode: "upstream",
|
||||
checks: [{ level: "info", code: "auth_ready", message: "Authentication is ready." }],
|
||||
}
|
||||
: { ready: false, mode: "upstream", checks: result };
|
||||
}
|
||||
|
||||
let loaded;
|
||||
try {
|
||||
if (!deps.authentication) throw new Error("missing authentication configuration");
|
||||
loaded = deps.authentication.current();
|
||||
} catch {
|
||||
checks.push(check(deps.authentication ? "auth_config_invalid" : "auth_config_incomplete", "Authentication configuration is unavailable."));
|
||||
return { ready: false, mode: deps.authMode, checks: ordered(checks) };
|
||||
}
|
||||
if (loaded.value.mode !== deps.authMode) {
|
||||
checks.push(check("auth_config_invalid", "Authentication mode does not match its configuration."));
|
||||
return { ready: false, mode: deps.authMode, checks: ordered(checks) };
|
||||
}
|
||||
|
||||
if (loaded.value.mode === "local") {
|
||||
const local = await localRegistryIsUsable(deps);
|
||||
if (local) checks.push(local);
|
||||
const result = ordered(checks);
|
||||
return result.length === 0
|
||||
? { ready: true, mode: "local", checks: [{ level: "info", code: "auth_ready", message: "Authentication is ready." }] }
|
||||
: { ready: false, mode: "local", checks: result };
|
||||
}
|
||||
|
||||
const requiredSecrets: readonly AuthenticationSecretReference[] = ["THT_OIDC_CLIENT_SECRET", "THT_AUTHENTIK_API_TOKEN"];
|
||||
if (requiredSecrets.some((name) => !isUsableAuthenticationSecret(name, deps.secrets?.get(name)))) {
|
||||
checks.push(check("oidc_secret_missing", "A required OIDC or group catalog secret is unavailable."));
|
||||
}
|
||||
if (!options.live || checks.length > 0) {
|
||||
const result = ordered(checks);
|
||||
return result.length === 0
|
||||
? { ready: true, mode: "oidc", checks: [{ level: "info", code: "auth_ready", message: "Authentication is ready." }] }
|
||||
: { ready: false, mode: "oidc", checks: result };
|
||||
}
|
||||
const mappedGroupNames = Object.keys(loaded.value.authorization.groupRoles).sort(stableCompare);
|
||||
|
||||
const deadline = new AbortController();
|
||||
const deadlineTimer = setTimeout(() => deadline.abort(), LIVE_DIAGNOSTIC_TIMEOUT_MS);
|
||||
deadlineTimer.unref();
|
||||
const liveSignal = AbortSignal.any([signal, deadline.signal]);
|
||||
try {
|
||||
if (!deps.oidcProtocol) {
|
||||
checks.push(check("oidc_discovery_unreachable", "The OIDC provider is unavailable."));
|
||||
} else {
|
||||
try {
|
||||
await awaitWithAbort(startBeforeAbort(liveSignal, () => deps.oidcProtocol!.diagnose(liveSignal)), liveSignal);
|
||||
} catch (error) {
|
||||
checks.push(check(
|
||||
error instanceof OidcIssuerMismatchError
|
||||
? "oidc_issuer_mismatch"
|
||||
: error instanceof OidcJwksUnavailableError
|
||||
? "oidc_jwks_unreachable"
|
||||
: "oidc_discovery_unreachable",
|
||||
"The OIDC provider could not be validated.",
|
||||
));
|
||||
}
|
||||
}
|
||||
if (!liveSignal.aborted) {
|
||||
if (!deps.groupCatalog) {
|
||||
checks.push(check("oidc_group_catalog_unreachable", "The configured group catalog cannot be certified."));
|
||||
} else {
|
||||
try {
|
||||
checks.push(...await awaitWithAbort(startBeforeAbort(liveSignal, () => deps.groupCatalog!.verifyConfiguredGroups(
|
||||
mappedGroupNames, liveSignal,
|
||||
)), liveSignal));
|
||||
} catch {
|
||||
checks.push(check("oidc_group_catalog_unreachable", "The configured group catalog is unavailable."));
|
||||
}
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
clearTimeout(deadlineTimer);
|
||||
}
|
||||
const result = ordered(checks);
|
||||
return result.length === 0
|
||||
? { ready: true, mode: "oidc", checks: [{ level: "info", code: "auth_ready", message: "Authentication is ready." }] }
|
||||
: { ready: false, mode: "oidc", checks: result };
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
/** The fixed machine contract shared by the Authentik catalog and auth diagnostics. */
|
||||
export type AuthDiagnosticCode =
|
||||
| "auth_ready"
|
||||
| "auth_config_incomplete"
|
||||
| "auth_config_invalid"
|
||||
| "auth_session_store_invalid"
|
||||
| "local_user_registry_invalid"
|
||||
| "local_admin_missing"
|
||||
| "oidc_secret_missing"
|
||||
| "oidc_discovery_unreachable"
|
||||
| "oidc_issuer_mismatch"
|
||||
| "oidc_jwks_unreachable"
|
||||
| "oidc_group_catalog_unreachable"
|
||||
| "oidc_group_catalog_unauthorized"
|
||||
| "oidc_mapped_group_missing"
|
||||
| "oidc_mapped_group_ambiguous"
|
||||
| "oidc_groups_claim_invalid"
|
||||
| "oidc_device_flow_unavailable";
|
||||
|
||||
export interface AuthDiagnostic {
|
||||
level: "error" | "info";
|
||||
code: AuthDiagnosticCode;
|
||||
message: string;
|
||||
field?: string;
|
||||
}
|
||||
|
||||
export interface AuthDiagnostics {
|
||||
ready: boolean;
|
||||
mode: "local" | "oidc" | "upstream" | "none" | "mock";
|
||||
checks: readonly AuthDiagnostic[];
|
||||
}
|
||||
|
||||
const diagnosticCodes = new Set<AuthDiagnosticCode>([
|
||||
"auth_ready", "auth_config_incomplete", "auth_config_invalid", "auth_session_store_invalid",
|
||||
"local_user_registry_invalid", "local_admin_missing", "oidc_secret_missing",
|
||||
"oidc_discovery_unreachable", "oidc_issuer_mismatch", "oidc_jwks_unreachable",
|
||||
"oidc_group_catalog_unreachable", "oidc_group_catalog_unauthorized", "oidc_mapped_group_missing",
|
||||
"oidc_mapped_group_ambiguous", "oidc_groups_claim_invalid", "oidc_device_flow_unavailable",
|
||||
]);
|
||||
const diagnosticModes = new Set<AuthDiagnostics["mode"]>(["local", "oidc", "upstream", "none", "mock"]);
|
||||
const fieldCodes = new Set<AuthDiagnosticCode>(["oidc_mapped_group_missing", "oidc_mapped_group_ambiguous"]);
|
||||
|
||||
function exactObject(value: unknown, keys: readonly string[]): Record<string, unknown> | undefined {
|
||||
if (!value || typeof value !== "object" || Array.isArray(value)) return undefined;
|
||||
const source = value as Record<string, unknown>;
|
||||
const actual = Object.keys(source);
|
||||
return actual.length === keys.length && actual.every((key) => keys.includes(key)) ? source : undefined;
|
||||
}
|
||||
|
||||
function safeText(value: unknown): value is string {
|
||||
return typeof value === "string" && value.length > 0 && value.length <= 512
|
||||
&& value.trim() === value && !/\p{Cc}/u.test(value);
|
||||
}
|
||||
|
||||
/** Strict decoder for the machine contract shared with tht and the frontend. */
|
||||
export function decodeAuthDiagnostics(value: unknown): AuthDiagnostics | undefined {
|
||||
const source = exactObject(value, ["ready", "mode", "checks"]);
|
||||
if (!source || typeof source.ready !== "boolean" || typeof source.mode !== "string"
|
||||
|| !diagnosticModes.has(source.mode as AuthDiagnostics["mode"])
|
||||
|| !Array.isArray(source.checks) || source.checks.length === 0 || source.checks.length > 129) return undefined;
|
||||
const seen = new Set<string>();
|
||||
const checks: AuthDiagnostic[] = [];
|
||||
for (const value of source.checks) {
|
||||
const raw = value && typeof value === "object" && !Array.isArray(value)
|
||||
? value as Record<string, unknown>
|
||||
: undefined;
|
||||
const check = raw && exactObject(raw, raw.field === undefined
|
||||
? ["level", "code", "message"]
|
||||
: ["level", "code", "message", "field"]);
|
||||
if (!check || (check.level !== "error" && check.level !== "info")
|
||||
|| typeof check.code !== "string" || !diagnosticCodes.has(check.code as AuthDiagnosticCode)
|
||||
|| !safeText(check.message) || (check.field !== undefined && !safeText(check.field))) return undefined;
|
||||
const code = check.code as AuthDiagnosticCode;
|
||||
if (check.field !== undefined && !fieldCodes.has(code)) return undefined;
|
||||
const key = `${code}\u0000${check.field ?? ""}`;
|
||||
if (seen.has(key)) return undefined;
|
||||
seen.add(key);
|
||||
checks.push({
|
||||
level: check.level,
|
||||
code,
|
||||
message: check.message,
|
||||
...(check.field === undefined ? {} : { field: check.field }),
|
||||
});
|
||||
}
|
||||
if (source.ready) {
|
||||
if (checks.length !== 1 || checks[0].level !== "info" || checks[0].code !== "auth_ready"
|
||||
|| checks[0].field !== undefined) return undefined;
|
||||
} else if (!checks.some(({ level }) => level === "error")
|
||||
|| checks.some(({ code }) => code === "auth_ready")) return undefined;
|
||||
return { ready: source.ready, mode: source.mode as AuthDiagnostics["mode"], checks };
|
||||
}
|
||||
|
||||
/** A provider-specific proof that only the configured authorization groups exist. */
|
||||
export interface GroupCatalog {
|
||||
verifyConfiguredGroups(names: readonly string[], signal: AbortSignal): Promise<readonly AuthDiagnostic[]>;
|
||||
}
|
||||
@@ -0,0 +1,329 @@
|
||||
import {
|
||||
closeSync,
|
||||
constants,
|
||||
fstatSync,
|
||||
lstatSync,
|
||||
openSync,
|
||||
readSync,
|
||||
realpathSync,
|
||||
} from "node:fs";
|
||||
import type { Stats } from "node:fs";
|
||||
import { dirname, isAbsolute, join, normalize } from "node:path";
|
||||
import { parseDocument } from "yaml";
|
||||
import { z } from "zod";
|
||||
import { isValidPasswordHash, verifyPassword, verifyWithDummy } from "./password.js";
|
||||
import type { LoadedAuthConfig, LocalUserRecord, Role } from "./types.js";
|
||||
import { createWindowsAuthStorageBridge, type WindowsAuthStorageBridge } from "./windows-auth-storage.js";
|
||||
|
||||
const MAX_USERS_YAML_BYTES = 1 << 20;
|
||||
const USERNAME_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._@-]{2,63}$/;
|
||||
const UUID_V4_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/;
|
||||
const ROLES = ["user", "admin"] as const;
|
||||
const invalid = (): Error => new Error("local_user_registry_invalid");
|
||||
|
||||
function runtimeOwner(): number {
|
||||
if (process.platform === "win32" || typeof process.geteuid !== "function") throw invalid();
|
||||
const owner = process.geteuid();
|
||||
if (!Number.isSafeInteger(owner) || owner < 0) throw invalid();
|
||||
return owner;
|
||||
}
|
||||
|
||||
export type { LocalUserRecord } from "./types.js";
|
||||
|
||||
export interface LocalUserRegistry {
|
||||
/** Safe production diagnostic probe; never returns user records or hashes. */
|
||||
hasEnabledAdmin(): Promise<boolean>;
|
||||
findByUsername(username: string): Promise<LocalUserRecord | undefined>;
|
||||
findBySubject(id: string): Promise<LocalUserRecord | undefined>;
|
||||
verify(user: LocalUserRecord | undefined, password: string): Promise<boolean>;
|
||||
}
|
||||
|
||||
/** Keeps only the registry named by the current coherent authentication-config snapshot. */
|
||||
export interface CurrentLocalUserRegistryResolver {
|
||||
resolve(loaded: LoadedAuthConfig): LocalUserRegistry | undefined;
|
||||
}
|
||||
|
||||
/** Native Windows obtains protected registry bytes only from the hidden tht bridge. */
|
||||
export interface LocalUserRegistryOptions {
|
||||
windowsStorageBridge?: Pick<WindowsAuthStorageBridge, "readLocalUsers">;
|
||||
}
|
||||
|
||||
interface FileIdentity {
|
||||
dev: number;
|
||||
ino: number;
|
||||
uid: number;
|
||||
size: number;
|
||||
mtimeMs: number;
|
||||
}
|
||||
|
||||
interface DirectoryIdentity {
|
||||
dev: number;
|
||||
ino: number;
|
||||
uid: number;
|
||||
mode: number;
|
||||
}
|
||||
|
||||
interface RegistryIdentity {
|
||||
file: FileIdentity;
|
||||
directory: DirectoryIdentity;
|
||||
}
|
||||
|
||||
const roleSchema = z.enum(ROLES);
|
||||
const userSchema = z.strictObject({
|
||||
id: z.string().regex(UUID_V4_PATTERN),
|
||||
username: z.string().regex(USERNAME_PATTERN),
|
||||
displayName: z.string().optional().refine((value) => value === undefined || !/\p{Cc}/u.test(value)),
|
||||
passwordHash: z.string().refine(isValidPasswordHash),
|
||||
roles: z.array(roleSchema).min(1).superRefine((roles, context) => {
|
||||
if (new Set(roles).size !== roles.length) context.addIssue({ code: "custom", message: "duplicate role" });
|
||||
}),
|
||||
enabled: z.boolean(),
|
||||
authRevision: z.number().int().positive().safe(),
|
||||
});
|
||||
const registrySchema = z.strictObject({ version: z.literal(1), users: z.array(userSchema).min(1) });
|
||||
|
||||
function normalizeUsername(username: string): string {
|
||||
return username.replace(/[A-Z]/g, (character) => character.toLowerCase());
|
||||
}
|
||||
|
||||
function sameFileIdentity(left: FileIdentity, right: FileIdentity): boolean {
|
||||
return left.dev === right.dev && left.ino === right.ino && left.uid === right.uid
|
||||
&& left.size === right.size && left.mtimeMs === right.mtimeMs;
|
||||
}
|
||||
|
||||
function sameDirectoryIdentity(left: DirectoryIdentity, right: DirectoryIdentity): boolean {
|
||||
return left.dev === right.dev && left.ino === right.ino && left.uid === right.uid && left.mode === right.mode;
|
||||
}
|
||||
|
||||
function sameIdentity(left: RegistryIdentity, right: RegistryIdentity): boolean {
|
||||
return sameFileIdentity(left.file, right.file) && sameDirectoryIdentity(left.directory, right.directory);
|
||||
}
|
||||
|
||||
function validateCanonicalPath(path: string): void {
|
||||
if (typeof path !== "string" || path.length === 0 || path.includes("\0") || !isAbsolute(path) || normalize(path) !== path) throw invalid();
|
||||
const parent = dirname(path);
|
||||
if (realpathSync(parent) !== parent) throw invalid();
|
||||
}
|
||||
|
||||
function fileMetadata(info: Stats, owner: number): FileIdentity {
|
||||
if (!info.isFile() || info.uid !== owner || info.nlink !== 1 || (info.mode & 0o7777) !== 0o600) throw invalid();
|
||||
if (info.size < 0 || info.size > MAX_USERS_YAML_BYTES) throw invalid();
|
||||
return { dev: info.dev, ino: info.ino, uid: info.uid, size: info.size, mtimeMs: info.mtimeMs };
|
||||
}
|
||||
|
||||
function directoryMetadata(info: Stats, owner: number): DirectoryIdentity {
|
||||
if (!info.isDirectory() || info.uid !== owner || (info.mode & 0o7777) !== 0o700) throw invalid();
|
||||
return { dev: info.dev, ino: info.ino, uid: info.uid, mode: info.mode & 0o7777 };
|
||||
}
|
||||
|
||||
function directoryIdentity(path: string, owner: number): DirectoryIdentity {
|
||||
const parent = dirname(path);
|
||||
if (realpathSync(parent) !== parent) throw invalid();
|
||||
return directoryMetadata(lstatSync(parent) as Stats, owner);
|
||||
}
|
||||
|
||||
function registryIdentity(path: string, owner: number): RegistryIdentity {
|
||||
validateCanonicalPath(path);
|
||||
const info = lstatSync(path);
|
||||
return { file: fileMetadata(info as Stats, owner), directory: directoryIdentity(path, owner) };
|
||||
}
|
||||
|
||||
function openDirectoryDescriptor(path: string): number | undefined {
|
||||
if (process.platform === "win32") return undefined;
|
||||
const flags = constants.O_RDONLY
|
||||
| (constants.O_DIRECTORY ?? 0)
|
||||
| (constants.O_NOFOLLOW ?? 0)
|
||||
| (constants.O_NONBLOCK ?? 0);
|
||||
return openSync(path, flags);
|
||||
}
|
||||
|
||||
function readBounded(path: string, owner: number): { source: string; identity: RegistryIdentity } {
|
||||
validateCanonicalPath(path);
|
||||
const beforeDirectory = directoryIdentity(path, owner);
|
||||
const beforePath = lstatSync(path);
|
||||
const before = fileMetadata(beforePath as Stats, owner);
|
||||
let directoryDescriptor: number | undefined;
|
||||
let descriptor: number | undefined;
|
||||
try {
|
||||
directoryDescriptor = openDirectoryDescriptor(dirname(path));
|
||||
const openedDirectory = directoryDescriptor === undefined
|
||||
? beforeDirectory
|
||||
: directoryMetadata(fstatSync(directoryDescriptor) as Stats, owner);
|
||||
if (!sameDirectoryIdentity(beforeDirectory, openedDirectory)) throw invalid();
|
||||
descriptor = openSync(path, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
|
||||
const opened = fileMetadata(fstatSync(descriptor) as Stats, owner);
|
||||
if (!sameFileIdentity(before, opened)) throw invalid();
|
||||
const buffer = Buffer.allocUnsafe(MAX_USERS_YAML_BYTES + 1);
|
||||
let offset = 0;
|
||||
while (offset < buffer.length) {
|
||||
const bytesRead = readSync(descriptor, buffer, offset, buffer.length - offset, null);
|
||||
if (bytesRead === 0) break;
|
||||
offset += bytesRead;
|
||||
}
|
||||
if (offset > MAX_USERS_YAML_BYTES) throw invalid();
|
||||
const after = fileMetadata(fstatSync(descriptor) as Stats, owner);
|
||||
const afterPath = fileMetadata(lstatSync(path) as Stats, owner);
|
||||
const afterDirectory = directoryMetadata(lstatSync(dirname(path)) as Stats, owner);
|
||||
const afterOpenedDirectory = directoryDescriptor === undefined
|
||||
? afterDirectory
|
||||
: directoryMetadata(fstatSync(directoryDescriptor) as Stats, owner);
|
||||
if (!sameFileIdentity(opened, after) || !sameFileIdentity(after, afterPath)
|
||||
|| !sameDirectoryIdentity(beforeDirectory, afterDirectory)
|
||||
|| !sameDirectoryIdentity(openedDirectory, afterOpenedDirectory)) throw invalid();
|
||||
const source = new TextDecoder("utf-8", { fatal: true }).decode(buffer.subarray(0, offset));
|
||||
return { source, identity: { file: after, directory: afterDirectory } };
|
||||
} catch {
|
||||
throw invalid();
|
||||
} finally {
|
||||
if (descriptor !== undefined) {
|
||||
try { closeSync(descriptor); } catch { /* sanitized by design */ }
|
||||
}
|
||||
if (directoryDescriptor !== undefined) {
|
||||
try { closeSync(directoryDescriptor); } catch { /* sanitized by design */ }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function parseLocalUserRegistrySource(source: string): readonly LocalUserRecord[] {
|
||||
try {
|
||||
const document = parseDocument(source, { uniqueKeys: true });
|
||||
if (document.errors.length > 0 || document.warnings.length > 0) throw invalid();
|
||||
const parsed = registrySchema.parse(document.toJSON());
|
||||
const ids = new Set<string>();
|
||||
const usernames = new Set<string>();
|
||||
const records = parsed.users.map((user) => {
|
||||
const normalizedUsername = normalizeUsername(user.username);
|
||||
if (ids.has(user.id) || usernames.has(normalizedUsername)) throw invalid();
|
||||
ids.add(user.id);
|
||||
usernames.add(normalizedUsername);
|
||||
return Object.freeze({
|
||||
id: user.id,
|
||||
username: user.username,
|
||||
normalizedUsername,
|
||||
...(user.displayName === undefined ? {} : { displayName: user.displayName }),
|
||||
passwordHash: user.passwordHash,
|
||||
roles: Object.freeze([...user.roles]) as readonly Role[],
|
||||
enabled: user.enabled,
|
||||
authRevision: user.authRevision,
|
||||
});
|
||||
});
|
||||
return Object.freeze(records);
|
||||
} catch {
|
||||
throw invalid();
|
||||
}
|
||||
}
|
||||
|
||||
function load(path: string, owner: number): { records: readonly LocalUserRecord[]; identity: RegistryIdentity } {
|
||||
const read = readBounded(path, owner);
|
||||
return { records: parseLocalUserRegistrySource(read.source), identity: read.identity };
|
||||
}
|
||||
|
||||
export function createLocalUserRegistry(usersPath: string, options: LocalUserRegistryOptions = {}): LocalUserRegistry {
|
||||
let cached: { records: readonly LocalUserRecord[]; identity: RegistryIdentity } | undefined;
|
||||
const windowsStorage = process.platform === "win32"
|
||||
? options.windowsStorageBridge ?? createWindowsAuthStorageBridge()
|
||||
: undefined;
|
||||
|
||||
function currentPosix(): readonly LocalUserRecord[] {
|
||||
try {
|
||||
const owner = runtimeOwner();
|
||||
const before = registryIdentity(usersPath, owner);
|
||||
if (cached && sameIdentity(cached.identity, before)) return cached.records;
|
||||
for (let attempt = 0; attempt < 2; attempt += 1) {
|
||||
const loaded = load(usersPath, owner);
|
||||
if (sameIdentity(loaded.identity, registryIdentity(usersPath, owner))) {
|
||||
cached = loaded;
|
||||
return loaded.records;
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
throw invalid();
|
||||
}
|
||||
throw invalid();
|
||||
}
|
||||
|
||||
async function current(): Promise<readonly LocalUserRecord[]> {
|
||||
if (process.platform !== "win32") return currentPosix();
|
||||
try {
|
||||
if (!windowsStorage) throw invalid();
|
||||
const contents = await windowsStorage.readLocalUsers(usersPath);
|
||||
if (!Buffer.isBuffer(contents) || contents.length === 0 || contents.length > MAX_USERS_YAML_BYTES) throw invalid();
|
||||
return parseLocalUserRegistrySource(new TextDecoder("utf-8", { fatal: true }).decode(contents));
|
||||
} catch {
|
||||
throw invalid();
|
||||
}
|
||||
}
|
||||
|
||||
async function operationalRecords(): Promise<readonly LocalUserRecord[]> {
|
||||
const records = await current();
|
||||
if (!records.some((user) => user.enabled && user.roles.includes("admin"))) throw invalid();
|
||||
return records;
|
||||
}
|
||||
|
||||
return {
|
||||
async hasEnabledAdmin(): Promise<boolean> {
|
||||
return (await current()).some((user) => user.enabled && user.roles.includes("admin"));
|
||||
},
|
||||
async findByUsername(username: string): Promise<LocalUserRecord | undefined> {
|
||||
const normalized = normalizeUsername(username);
|
||||
return (await operationalRecords()).find((user) => user.normalizedUsername === normalized);
|
||||
},
|
||||
async findBySubject(id: string): Promise<LocalUserRecord | undefined> {
|
||||
return (await operationalRecords()).find((user) => user.id === id);
|
||||
},
|
||||
async verify(user: LocalUserRecord | undefined, password: string): Promise<boolean> {
|
||||
if (!user || !user.enabled) {
|
||||
await verifyWithDummy(password);
|
||||
return false;
|
||||
}
|
||||
return await verifyPassword(password, user.passwordHash);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function createCurrentLocalUserRegistryResolver(options: LocalUserRegistryOptions = {}): CurrentLocalUserRegistryResolver {
|
||||
let current: { key: object | string; registry: LocalUserRegistry } | undefined;
|
||||
return {
|
||||
resolve(loaded: LoadedAuthConfig): LocalUserRegistry | undefined {
|
||||
if (loaded.value.mode !== "local") return undefined;
|
||||
const runtimeProjection = loaded.runtimeProjection;
|
||||
const projectedUsers = runtimeProjection?.localUsers;
|
||||
if (projectedUsers && runtimeProjection) {
|
||||
if (current && current.key === runtimeProjection) return current.registry;
|
||||
const registry = createInMemoryLocalUserRegistry(projectedUsers);
|
||||
current = { key: runtimeProjection, registry };
|
||||
return registry;
|
||||
}
|
||||
const usersPath = join(dirname(loaded.sourcePath), loaded.value.local.usersFile);
|
||||
if (current && current.key === usersPath) return current.registry;
|
||||
const registry = createLocalUserRegistry(usersPath, options);
|
||||
current = { key: usersPath, registry };
|
||||
return registry;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function createInMemoryLocalUserRegistry(records: readonly LocalUserRecord[]): LocalUserRegistry {
|
||||
async function operationalRecords(): Promise<readonly LocalUserRecord[]> {
|
||||
if (!records.some((user) => user.enabled && user.roles.includes("admin"))) throw invalid();
|
||||
return records;
|
||||
}
|
||||
return {
|
||||
async hasEnabledAdmin(): Promise<boolean> {
|
||||
return records.some((user) => user.enabled && user.roles.includes("admin"));
|
||||
},
|
||||
async findByUsername(username: string): Promise<LocalUserRecord | undefined> {
|
||||
return (await operationalRecords()).find((user) => user.normalizedUsername === normalizeUsername(username));
|
||||
},
|
||||
async findBySubject(id: string): Promise<LocalUserRecord | undefined> {
|
||||
return (await operationalRecords()).find((user) => user.id === id);
|
||||
},
|
||||
async verify(user: LocalUserRecord | undefined, password: string): Promise<boolean> {
|
||||
if (!user || !user.enabled) {
|
||||
await verifyWithDummy(password);
|
||||
return false;
|
||||
}
|
||||
return await verifyPassword(password, user.passwordHash);
|
||||
},
|
||||
};
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user