mptyl/ThothII
1
0
Fork 0
Code Issues 25 Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Add full shell, replaceable Omics adapter and bilingual interaction

Browse Source 
Implement approved specification #32 and tickets #33-#37. Keep host authentication server-verified and pin session interaction language. Compile scoped base selectors for browser compatibility and retain full gutters during CSS pruning.
This commit is contained in:
Codex
2026-09-13 14:26:39 +02:00
parent 2d1b714ebe
commit d8a29bfbdd
207 changed files with 8570 additions and 2164 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/tht/internal/modelprojection/projection.go
+60 -12
View File
@@ -1,4 +1,4 @@
// Package modelprojection renders disposable runtime adapters from the installation model catalog.
// Package modelprojection renders disposable model and public frontend installation projections.
package modelprojection
import (
@@ -16,10 +16,11 @@ import (
)
const (
CatalogFile = "catalog.json"
PiModelsFile = "pi/models.json"
PiSettingsFile = "pi/settings.json"
ComposeFile = "compose.models.yaml"
CatalogFile = "catalog.json"
PiModelsFile = "pi/models.json"
PiSettingsFile = "pi/settings.json"
ComposeFile = "compose.models.yaml"
FrontendConfigFile = "frontend/config.js"
)
var renameProjectionDirectory = os.Rename
@@ -77,6 +78,9 @@ func Render(installation config.Installation) (map[string][]byte, error) {
if err := installation.ModelCatalog.NormalizeDefaults(); err != nil {
return nil, err
}
if err := installation.Shell.NormalizeDefaults(); err != nil {
return nil, err
}
models := installation.ModelCatalog.RuntimeModels()
hasMetadata := false
for _, model := range models {
@@ -146,10 +150,21 @@ func Render(installation config.Installation) (map[string][]byte, error) {
return nil, fmt.Errorf("render Pi settings projection: %w", err)
}
fingerprint := sha256.Sum256(bytes.Join([][]byte{catalogBytes, piModelsBytes, piSettingsBytes}, nil))
composeBytes := renderCompose(installation, fmt.Sprintf("sha256:%x", fingerprint))
publicJSON, err := marshalJSON(struct {
BackendBaseURL string `json:"backendBaseUrl"`
Shell config.Shell `json:"shell"`
}{BackendBaseURL: "/api", Shell: installation.Shell})
if err != nil {
return nil, fmt.Errorf("render public frontend configuration: %w", err)
}
publicJS := append([]byte("window.__THOTHII_CONFIG__ = "), bytes.TrimSpace(publicJSON)...)
publicJS = append(publicJS, ';', '\n')
publicFingerprint := sha256.Sum256(publicJS)
composeBytes := renderCompose(installation, fmt.Sprintf("sha256:%x", fingerprint), fmt.Sprintf("sha256:%x", publicFingerprint))
return map[string][]byte{
CatalogFile: catalogBytes, PiModelsFile: piModelsBytes,
PiSettingsFile: piSettingsBytes, ComposeFile: composeBytes,
FrontendConfigFile: publicJS,
}, nil
}
@@ -210,10 +225,14 @@ func Generate(installation config.Installation) error {
}
func projectionMatches(directory string, artifacts map[string][]byte) bool {
if !projectionModeMatches(directory, 0o755) {
return false
}
for _, relative := range sortedArtifactPaths(artifacts) {
path := filepath.Join(directory, filepath.FromSlash(relative))
info, err := os.Lstat(path)
if err != nil || !info.Mode().IsRegular() {
if err != nil || !info.Mode().IsRegular() || !projectionModeMatches(path, 0o644) ||
!projectionModeMatches(filepath.Dir(path), 0o755) {
return false
}
actual, err := os.ReadFile(path)
@@ -224,6 +243,14 @@ func projectionMatches(directory string, artifacts map[string][]byte) bool {
return true
}
func projectionModeMatches(path string, mode os.FileMode) bool {
if runtime.GOOS == "windows" {
return true // POSIX mode bits do not represent Windows access controls.
}
info, err := os.Lstat(path)
return err == nil && info.Mode()&os.ModeSymlink == 0 && info.Mode().Perm() == mode
}
func writeProjectionCandidate(directory string, artifacts map[string][]byte) error {
if err := os.Chmod(directory, 0o755); err != nil {
return fmt.Errorf("protect model projection candidate: %w", err)
@@ -233,9 +260,16 @@ func writeProjectionCandidate(directory string, artifacts map[string][]byte) err
if err := os.MkdirAll(filepath.Dir(destination), 0o755); err != nil {
return fmt.Errorf("create model projection directory: %w", err)
}
if err := os.Chmod(filepath.Dir(destination), 0o755); err != nil {
return fmt.Errorf("set runtime projection directory permissions: %w", err)
}
if err := os.WriteFile(destination, artifacts[relative], 0o644); err != nil {
return fmt.Errorf("write model projection candidate: %w", err)
}
// Creation modes are filtered by the host umask; container readers have another UID.
if err := os.Chmod(destination, 0o644); err != nil {
return fmt.Errorf("set runtime projection file permissions: %w", err)
}
}
return nil
}
@@ -251,7 +285,7 @@ func absentTemporaryPath(parent string) (string, error) {
return path, nil
}
// Check returns relative artifact names whose current bytes differ from the catalog projection.
// Check returns relative artifact names whose bytes or POSIX modes differ from the projection.
func Check(installation config.Installation) ([]string, error) {
artifacts, err := Render(installation)
if err != nil {
@@ -259,8 +293,11 @@ func Check(installation config.Installation) ([]string, error) {
}
drift := make([]string, 0)
for _, relative := range sortedArtifactPaths(artifacts) {
actual, readErr := os.ReadFile(filepath.Join(installation.GeneratedDirectory(), filepath.FromSlash(relative)))
if readErr != nil || !bytes.Equal(actual, artifacts[relative]) {
path := filepath.Join(installation.GeneratedDirectory(), filepath.FromSlash(relative))
actual, readErr := os.ReadFile(path)
if readErr != nil || !bytes.Equal(actual, artifacts[relative]) ||
!projectionModeMatches(installation.GeneratedDirectory(), 0o755) ||
!projectionModeMatches(filepath.Dir(path), 0o755) || !projectionModeMatches(path, 0o644) {
drift = append(drift, relative)
}
}
@@ -275,9 +312,19 @@ func marshalJSON(value any) ([]byte, error) {
return append(contents, '\n'), nil
}
func renderCompose(installation config.Installation, fingerprint string) []byte {
func renderCompose(installation config.Installation, fingerprint, publicFingerprint string) []byte {
embedding := installation.ModelCatalog.Embedding
return []byte(fmt.Sprintf(`services:
frontend:
environment:
THT_FRONTEND_CONFIG_REVISION: %s
volumes:
- type: bind
source: %s
target: /usr/share/nginx/html/config.js
read_only: true
bind:
create_host_path: false
core:
environment:
THT_HOST_PLATFORM: %s
@@ -308,7 +355,8 @@ func renderCompose(installation config.Installation, fingerprint string) []byte
embedding-model-init:
environment:
OLLAMA_MODEL: %s
`, strconv.Quote(runtime.GOOS), strconv.Quote(fingerprint), strconv.Quote(installation.ModelCatalog.Defaults.Interaction),
`, strconv.Quote(publicFingerprint), strconv.Quote(installation.GeneratedFrontendConfigPath()),
strconv.Quote(runtime.GOOS), strconv.Quote(fingerprint), strconv.Quote(installation.ModelCatalog.Defaults.Interaction),
strconv.Quote(embedding.ID), strconv.Quote(embeddingModelName(embedding.ID)), strconv.Quote(strconv.Itoa(embedding.Dimensions)),
strconv.Quote(installation.GeneratedModelCatalogPath()), strconv.Quote(installation.GeneratedPiModelsPath()),
strconv.Quote(installation.GeneratedPiSettingsPath()), strconv.Quote(embedding.ID),
tools/tht/internal/modelprojection/projection_permissions_unix_test.go
+88
View File
@@ -0,0 +1,88 @@
//go:build linux || darwin
package modelprojection
import (
"os"
"path/filepath"
"syscall"
"testing"
"github.com/aritmolab/thothii/tools/tht/internal/config"
)
func TestGenerateKeepsProjectionsContainerReadableWithRestrictiveUmask(t *testing.T) {
installation := projectionFixture(t)
parentBefore, err := os.Stat(filepath.Dir(installation.Path))
if err != nil {
t.Fatal(err)
}
// This test must remain sequential: umask is process-wide.
previous := syscall.Umask(0o077)
defer syscall.Umask(previous)
if err := Generate(installation); err != nil {
t.Fatal(err)
}
assertContainerProjectionModes(t, installation)
parentAfter, err := os.Stat(filepath.Dir(installation.Path))
if err != nil || parentAfter.Mode().Perm() != parentBefore.Mode().Perm() {
t.Fatalf("generation changed private installation directory permissions: %v", err)
}
}
func TestGenerateRepairsUnreadableProjectionAsANewGeneration(t *testing.T) {
installation := projectionFixture(t)
if err := Generate(installation); err != nil {
t.Fatal(err)
}
before, err := os.Stat(installation.GeneratedDirectory())
if err != nil {
t.Fatal(err)
}
for path, mode := range map[string]os.FileMode{
installation.GeneratedDirectory(): 0o700,
filepath.Dir(installation.GeneratedPiModelsPath()): 0o700,
installation.GeneratedFrontendConfigPath(): 0o600,
} {
if err := os.Chmod(path, mode); err != nil {
t.Fatal(err)
}
}
if drift, err := Check(installation); err != nil || len(drift) == 0 {
t.Fatalf("unreadable projection drift = %v, %v", drift, err)
}
if err := Generate(installation); err != nil {
t.Fatal(err)
}
assertContainerProjectionModes(t, installation)
after, err := os.Stat(installation.GeneratedDirectory())
if err != nil || os.SameFile(before, after) {
t.Fatalf("permissions were not repaired through candidate publication: %v", err)
}
if drift, err := Check(installation); err != nil || len(drift) != 0 {
t.Fatalf("repaired projection drift = %v, %v", drift, err)
}
}
func assertContainerProjectionModes(t *testing.T, installation config.Installation) {
t.Helper()
for path, mode := range map[string]os.FileMode{
installation.GeneratedDirectory(): 0o755,
filepath.Dir(installation.GeneratedFrontendConfigPath()): 0o755,
filepath.Dir(installation.GeneratedPiModelsPath()): 0o755,
installation.GeneratedFrontendConfigPath(): 0o644,
installation.GeneratedModelCatalogPath(): 0o644,
installation.GeneratedPiModelsPath(): 0o644,
installation.GeneratedPiSettingsPath(): 0o644,
installation.ModelProjectionComposePath(): 0o644,
} {
info, err := os.Stat(path)
if err != nil {
t.Error(err)
continue
}
if info.Mode().Perm() != mode {
t.Errorf("%s mode = %04o, want %04o for container readers", path, info.Mode().Perm(), mode)
}
}
}
tools/tht/internal/modelprojection/projection_test.go
+4 -3
View File
@@ -24,8 +24,8 @@ func TestRenderProducesDeterministicCatalogPiAndComposeProjections(t *testing.T)
if err != nil {
t.Fatalf("Render() second error = %v", err)
}
if len(first) != 4 {
t.Fatalf("artifact count = %d, want 4", len(first))
if len(first) != 5 {
t.Fatalf("artifact count = %d, want 5", len(first))
}
for path, contents := range first {
if !bytes.Equal(contents, second[path]) {
@@ -133,6 +133,7 @@ func TestGenerateDoesNotReplaceUnchangedProjection(t *testing.T) {
installation.GeneratedModelCatalogPath(),
installation.GeneratedPiModelsPath(),
installation.GeneratedPiSettingsPath(),
installation.GeneratedFrontendConfigPath(),
installation.ModelProjectionComposePath(),
}
before := make(map[string]os.FileInfo, len(paths))
@@ -174,7 +175,7 @@ func TestGenerateRestoresWholePreviousGenerationWhenPublishFails(t *testing.T) {
t.Fatalf("Generate() initial error = %v", err)
}
before := make(map[string][]byte)
for _, path := range []string{CatalogFile, PiModelsFile, PiSettingsFile, ComposeFile} {
for _, path := range []string{CatalogFile, PiModelsFile, PiSettingsFile, ComposeFile, FrontendConfigFile} {
contents, err := os.ReadFile(filepath.Join(installation.GeneratedDirectory(), filepath.FromSlash(path)))
if err != nil {
t.Fatal(err)
tools/tht/internal/modelprojection/shell_projection_test.go
+139
View File
@@ -0,0 +1,139 @@
package modelprojection
import (
"bytes"
"encoding/json"
"os"
"os/exec"
"path/filepath"
"reflect"
"runtime"
"strings"
"testing"
"github.com/aritmolab/thothii/tools/tht/internal/config"
"gopkg.in/yaml.v3"
)
func TestRenderPublicShellConfiguration(t *testing.T) {
for _, test := range []struct {
name string
shell config.Shell
want string
}{
{"legacy", config.Shell{}, `{"backendBaseUrl":"/api","shell":{"mode":"embedded","defaultLocale":"en","adapter":"omics-portal"}}`},
{"full", config.Shell{Mode: "full", Adapter: "omics-portal"}, `{"backendBaseUrl":"/api","shell":{"mode":"full","defaultLocale":"en"}}`},
{"Italian", config.Shell{Mode: "embedded", DefaultLocale: "it"}, `{"backendBaseUrl":"/api","shell":{"mode":"embedded","defaultLocale":"it","adapter":"omics-portal"}}`},
{"locale without catalog", config.Shell{Mode: "full", DefaultLocale: "fr-CA"}, `{"backendBaseUrl":"/api","shell":{"mode":"full","defaultLocale":"fr-CA"}}`},
} {
t.Run(test.name, func(t *testing.T) {
installation := projectionFixture(t)
installation.Shell = test.shell
files, err := Render(installation)
if err != nil {
t.Fatal(err)
}
// The entire public payload is an allowlist: no model endpoints, auth or host paths.
js := string(files["frontend/config.js"])
if !strings.HasPrefix(js, "window.__THOTHII_CONFIG__ = ") || !strings.HasSuffix(js, ";\n") {
t.Fatalf("invalid public config script: %q", js)
}
payload := strings.TrimSuffix(strings.TrimPrefix(js, "window.__THOTHII_CONFIG__ = "), ";\n")
var got, want any
if err := json.Unmarshal([]byte(payload), &got); err != nil {
t.Fatal(err)
}
if err := json.Unmarshal([]byte(test.want), &want); err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(got, want) {
t.Fatalf("public config = %s, want %s", payload, test.want)
}
var compose struct {
Services map[string]struct {
Volumes []struct {
Type string `yaml:"type"`
Source string `yaml:"source"`
Target string `yaml:"target"`
ReadOnly bool `yaml:"read_only"`
} `yaml:"volumes"`
} `yaml:"services"`
}
if err := yaml.Unmarshal(files[ComposeFile], &compose); err != nil {
t.Fatal(err)
}
mounts := compose.Services["frontend"].Volumes
if len(mounts) != 1 || mounts[0].Type != "bind" || !mounts[0].ReadOnly ||
mounts[0].Source != filepath.Join(installation.GeneratedDirectory(), "frontend", "config.js") ||
mounts[0].Target != "/usr/share/nginx/html/config.js" {
t.Fatalf("frontend mounts = %#v; want only public config.js, read-only", mounts)
}
})
}
}
func TestGenerateRefreshesPublicShellConfiguration(t *testing.T) {
installation := projectionFixture(t)
if err := Generate(installation); err != nil {
t.Fatal(err)
}
publicPath := filepath.Join(installation.GeneratedDirectory(), "frontend", "config.js")
before, err := os.ReadFile(publicPath)
if err != nil {
t.Fatal(err)
}
composeBefore, err := os.ReadFile(installation.ModelProjectionComposePath())
if err != nil {
t.Fatal(err)
}
installation.Shell = config.Shell{Mode: "full", DefaultLocale: "it"}
drift, err := Check(installation)
if err != nil || !reflect.DeepEqual(drift, []string{"compose.models.yaml", "frontend/config.js"}) {
t.Fatalf("shell drift = %v, %v", drift, err)
}
if err := Generate(installation); err != nil {
t.Fatal(err)
}
after, err := os.ReadFile(publicPath)
if err != nil || bytes.Equal(before, after) || !bytes.Contains(after, []byte(`"mode": "full"`)) || bytes.Contains(after, []byte(`"adapter"`)) {
t.Fatalf("updated public config = %s, %v", after, err)
}
composeAfter, err := os.ReadFile(installation.ModelProjectionComposePath())
if err != nil {
t.Fatal(err)
}
// A config revision changes Compose's service definition, ensuring active bind mounts refresh.
var oldCompose, newCompose map[string]map[string]map[string]any
if err := yaml.Unmarshal(composeBefore, &oldCompose); err != nil {
t.Fatal(err)
}
if err := yaml.Unmarshal(composeAfter, &newCompose); err != nil {
t.Fatal(err)
}
if reflect.DeepEqual(oldCompose["services"]["frontend"]["environment"], newCompose["services"]["frontend"]["environment"]) {
t.Fatal("shell change did not revise the frontend service")
}
if !reflect.DeepEqual(oldCompose["services"]["core"], newCompose["services"]["core"]) {
t.Fatal("shell-only change revised core configuration")
}
if drift, err := Check(installation); err != nil || len(drift) != 0 {
t.Fatalf("fresh generation drift = %v, %v", drift, err)
}
}
func TestGeneratedPublicShellPassesFrontendImageSmoke(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("frontend image smoke requires a POSIX host shell")
}
for _, shell := range []config.Shell{{}, {Mode: "full"}, {Mode: "full", DefaultLocale: "it"}, {Mode: "embedded", DefaultLocale: "it"}, {Mode: "full", DefaultLocale: "fr-CA"}, {Mode: "embedded", DefaultLocale: "zh-Hant-TW-u-nu-hanidec"}} {
installation := projectionFixture(t)
installation.Shell = shell
if err := Generate(installation); err != nil {
t.Fatal(err)
}
command := exec.Command("sh", "../../../../docker/smoke/frontend-smoke.sh", installation.GeneratedFrontendConfigPath())
if output, err := command.CombinedOutput(); err != nil {
t.Fatalf("frontend smoke for %#v: %v\n%s", shell, err, output)
}
}
}