ThothII/tools/tht/internal/doctor/report.go

// Package doctor aggregates non-mutating host and container diagnostics for one installation.
package doctor

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/aritmolab/thothii/tools/tht/internal/authconfig"
	"github.com/aritmolab/thothii/tools/tht/internal/compose"
	"github.com/aritmolab/thothii/tools/tht/internal/config"
	"github.com/aritmolab/thothii/tools/tht/internal/modelprojection"
	"github.com/aritmolab/thothii/tools/tht/internal/output"
	"github.com/aritmolab/thothii/tools/tht/internal/pi"
	"github.com/aritmolab/thothii/tools/tht/internal/service"
)

const (
	StatusPassed  = "passed"
	StatusFailed  = "failed"
	StatusSkipped = "skipped"
)

const probeTimeout = 5 * time.Second

var requireRuntimeAuthProjectionReady = authconfig.RequireRuntimeAuthProjectionReady

const registryValidationProgram = `const fs=require("node:fs");const path="/data/workspace-registry/state/active.json";const s=JSON.parse(fs.readFileSync(path,"utf8"));const hex=/^[0-9a-f]{40}$/;if(!hex.test(s.head)||!Array.isArray(s.revisions)||s.revisions.some((r)=>!r||typeof r.id!=="string"||!r.id||!hex.test(r.commit)||!hex.test(r.blob))){process.exit(1)}for(const r of s.revisions){fs.accessSync("/data/workspace-registry/snapshots/"+r.commit+"/"+r.id+".yaml",fs.constants.R_OK)}`

// Check is one named, redacted diagnostic outcome.
type Check struct {
	Name   string `json:"name"`
	Status string `json:"status"`
	Detail string `json:"detail"`
}

// Report is the typed, machine-readable diagnostic result.
type Report struct {
	OK     bool    `json:"ok"`
	Checks []Check `json:"checks"`
}

// Runner is the shell-free Docker boundary used for all host and in-container checks.
type Runner interface {
	Run(context.Context, []string, io.Reader) (compose.Result, error)
}

// HTTPProbeTarget identifies one local service endpoint that must answer a bounded request.
type HTTPProbeTarget struct {
	Name    string
	Service string
	URL     string
}

// HTTPProbe is injectable so reachability failures remain independently testable.
type HTTPProbe interface {
	Probe(context.Context, HTTPProbeTarget) error
}

type composeHTTPProbe struct {
	installation config.Installation
	runner       Runner
}

func (p composeHTTPProbe) Probe(ctx context.Context, target HTTPProbeTarget) error {
	probeContext, cancel := context.WithTimeout(ctx, probeTimeout)
	defer cancel()
	var command []string
	switch target.Name {
	case "core":
		command = []string{"exec", "-T", target.Service, "curl", "-fsS", "--max-time", "5", target.URL}
	case "frontend":
		command = []string{"exec", "-T", target.Service, "wget", "-q", "-T", "5", "-O", "/dev/null", target.URL}
	default:
		return errors.New("unknown HTTP probe target")
	}
	result, err := p.runner.Run(probeContext, p.installation.ComposeArgs(command...), nil)
	if err != nil {
		return errors.New(commandDetail(target.Name+" HTTP probe", result, err, nil))
	}
	return nil
}

// Run performs diagnostics only. Expected environmental failures become failed checks so that
// callers can always render a complete report; unexpected local read errors are also reported.
func Run(ctx context.Context, installation config.Installation, runner Runner) (Report, error) {
	return RunWithProbe(ctx, installation, runner, composeHTTPProbe{installation: installation, runner: runner})
}

// RunWithProbe performs diagnostics only, with an injectable bounded HTTP probe.
func RunWithProbe(ctx context.Context, installation config.Installation, runner Runner, probe HTTPProbe) (Report, error) {
	if runner == nil {
		return Report{}, errors.New("doctor requires a Docker command runner")
	}
	if probe == nil {
		return Report{}, errors.New("doctor requires an HTTP probe")
	}
	secretValues, secretErr := secretValues(installation)
	report := Report{Checks: make([]Check, 0, 13)}
	add := func(name, status, detail string) {
		report.Checks = append(report.Checks, Check{Name: name, Status: status, Detail: output.SanitizeDetail(detail, secretValues)})
	}

	if err := validateInstallation(installation); err != nil {
		add("descriptor", StatusFailed, err.Error())
	} else {
		add("descriptor", StatusPassed, "installation descriptor is loaded")
	}
	if drift, err := modelprojection.Check(installation); err != nil {
		add("model-projection", StatusFailed, "model runtime projections could not be recomputed")
	} else if len(drift) > 0 {
		add("model-projection", StatusFailed, "generated model runtime projections differ: "+strings.Join(drift, ", "))
	} else {
		add("model-projection", StatusPassed, "generated model runtime projections match modelCatalog")
	}
	if err := filePermissions(installation); err != nil || secretErr != nil {
		if secretErr != nil {
			add("files", StatusFailed, "declared secret files could not be read")
		} else {
			add("files", StatusFailed, err.Error())
		}
	} else {
		add("files", StatusPassed, "declared host files have safe permissions")
	}
	if installation.HasRuntimeAuthProjection() {
		if err := requireRuntimeAuthProjectionReady(installation); err != nil {
			add("auth-projection", StatusFailed, "runtime authentication projection is unavailable")
			return finalize(report), nil
		}
		add("auth-projection", StatusPassed, "runtime authentication projection is ready and equal to canonical authentication")
	}
	if secretErr != nil {
		add("docker", StatusSkipped, "declared secret files are unavailable")
		add("compose", StatusSkipped, "declared secret files are unavailable")
		add("configuration", StatusSkipped, "declared secret files are unavailable")
		add("authentication", StatusSkipped, "declared secret files are unavailable")
		add("services", StatusSkipped, "declared secret files are unavailable")
		add("core-http", StatusSkipped, "declared secret files are unavailable")
		add("frontend-http", StatusSkipped, "declared secret files are unavailable")
		add("workspace-registry", StatusSkipped, "declared secret files are unavailable")
		add("workflow", StatusSkipped, "declared secret files are unavailable")
		add("pi", StatusSkipped, "declared secret files are unavailable")
		return finalize(report), nil
	}

	if !commandCheck(ctx, runner, []string{"version", "--format", "{{.Client.Version}}"}, secretValues, add, "docker", "Docker Engine") {
		add("compose", StatusSkipped, "Docker Engine is unavailable")
		add("configuration", StatusSkipped, "Docker Engine is unavailable")
		add("authentication", StatusSkipped, "core is unavailable")
		add("services", StatusSkipped, "Docker Engine is unavailable")
		add("core-http", StatusSkipped, "core is unavailable")
		add("frontend-http", StatusSkipped, "frontend is unavailable")
		add("workspace-registry", StatusSkipped, "core is unavailable")
		add("workflow", StatusSkipped, "core is unavailable")
		add("pi", StatusSkipped, "core is unavailable")
		return finalize(report), nil
	}
	if !commandCheck(ctx, runner, []string{"compose", "version", "--short"}, secretValues, add, "compose", "Docker Compose") {
		add("configuration", StatusSkipped, "Docker Compose is unavailable")
		add("authentication", StatusSkipped, "core is unavailable")
		add("services", StatusSkipped, "Docker Compose is unavailable")
		add("core-http", StatusSkipped, "core is unavailable")
		add("frontend-http", StatusSkipped, "frontend is unavailable")
		add("workspace-registry", StatusSkipped, "core is unavailable")
		add("workflow", StatusSkipped, "core is unavailable")
		add("pi", StatusSkipped, "core is unavailable")
		return finalize(report), nil
	}

	configReady := false
	rendered := ""
	if result, err := runner.Run(ctx, installation.ComposeArgs("config", "--quiet"), nil); err != nil {
		add("configuration", StatusFailed, commandDetail("Compose configuration", result, err, secretValues))
	} else if result, err := runner.Run(ctx, installation.ComposeArgs("config", "--format", "json"), nil); err != nil {
		add("configuration", StatusFailed, commandDetail("Compose rendering", result, err, secretValues))
	} else if err := ValidateVolumes(result.Stdout); err != nil {
		add("configuration", StatusFailed, err.Error())
	} else {
		rendered = result.Stdout
		configReady = true
		add("configuration", StatusPassed, "Compose configuration and required volumes are valid")
	}

	status, statusAvailable, servicesCheck := serviceStatus(ctx, installation, runner, secretValues)
	coreRunning := false
	coreHealthy := false
	if statusAvailable {
		var runningErr, healthyErr error
		coreRunning, runningErr = service.CoreRunning(status)
		coreHealthy, healthyErr = service.CoreHealthy(status)
		if runningErr != nil || healthyErr != nil {
			coreRunning = false
			coreHealthy = false
		}
	}
	if !configReady || !coreRunning {
		add("authentication", StatusSkipped, "core is unavailable")
	} else if !coreHealthy {
		add("authentication", StatusFailed, "core is running but unhealthy")
	} else if authenticationCheck(ctx, installation, runner, secretValues) {
		add("authentication", StatusPassed, "container-local authentication diagnostics passed")
	} else {
		add("authentication", StatusFailed, "container-local authentication diagnostics failed")
	}
	add(servicesCheck.Name, servicesCheck.Status, servicesCheck.Detail)
	if !coreHealthy || servicesCheck.Status != StatusPassed {
		detail := "required services are not healthy"
		if !coreRunning {
			detail = "core is not running"
		}
		add("core-http", StatusSkipped, detail)
		add("frontend-http", StatusSkipped, detail)
		add("workspace-registry", StatusSkipped, detail)
		add("workflow", StatusSkipped, detail)
		add("pi", StatusSkipped, detail)
		return finalize(report), nil
	}

	reachabilityChecks(ctx, probe, secretValues, add)
	registryCheck(ctx, installation, runner, secretValues, add, configReady, rendered)
	workflowCheck(ctx, installation, runner, secretValues, add)
	piCheck(ctx, installation, runner, secretValues, add)
	return finalize(report), nil
}

func reachabilityChecks(ctx context.Context, probe HTTPProbe, secrets []string, add func(string, string, string)) {
	for _, target := range []HTTPProbeTarget{
		{Name: "core", Service: "core", URL: "http://127.0.0.1:8787/health"},
		{Name: "frontend", Service: "frontend", URL: "http://127.0.0.1:8080/"},
	} {
		if err := probe.Probe(ctx, target); err != nil {
			add(target.Name+"-http", StatusFailed, output.SanitizeDetail(err.Error(), secrets))
			continue
		}
		add(target.Name+"-http", StatusPassed, target.Name+" answered a bounded HTTP probe")
	}
}

func registryCheck(ctx context.Context, installation config.Installation, runner Runner, secrets []string, add func(string, string, string), configReady bool, rendered string) {
	if !configReady {
		add("workspace-registry", StatusFailed, "workspace-registry cannot be checked because Compose configuration is invalid")
		return
	}
	if !workspaceRegistryDeclared(rendered) {
		add("workspace-registry", StatusFailed, "workspace-registry volume is not configured")
		return
	}
	result, err := runner.Run(ctx, installation.ComposeArgs("exec", "-T", "core", "node", "-e", registryValidationProgram), nil)
	if err != nil {
		add("workspace-registry", StatusFailed, commandDetail("container-local workspace registry", result, err, secrets))
		return
	}
	add("workspace-registry", StatusPassed, "container-local active registry state and snapshots are valid")
}

func finalize(report Report) Report {
	report.OK = len(report.Checks) > 0
	for _, check := range report.Checks {
		if check.Status != StatusPassed {
			report.OK = false
			break
		}
	}
	return report
}

func commandCheck(ctx context.Context, runner Runner, args []string, secrets []string, add func(string, string, string), name, label string) bool {
	result, err := runner.Run(ctx, args, nil)
	if err != nil || strings.TrimSpace(result.Stdout) == "" {
		add(name, StatusFailed, commandDetail(label, result, err, secrets))
		return false
	}
	add(name, StatusPassed, strings.TrimSpace(result.Stdout))
	return true
}

func serviceStatus(ctx context.Context, installation config.Installation, runner Runner, secrets []string) (string, bool, Check) {
	result, err := runner.Run(ctx, installation.ComposeArgs("ps", "--all", "--format", "json"), nil)
	if err != nil {
		return "", false, Check{Name: "services", Status: StatusFailed, Detail: commandDetail("Compose service status", result, err, secrets)}
	}
	if err := service.Healthy(result.Stdout); err != nil {
		return result.Stdout, true, Check{Name: "services", Status: StatusFailed, Detail: err.Error()}
	}
	return result.Stdout, true, Check{Name: "services", Status: StatusPassed, Detail: "required services are running and reachable through Docker health checks"}
}

func authenticationCheck(ctx context.Context, installation config.Installation, runner Runner, secrets []string) bool {
	report, err := authconfig.Check(ctx, installation, runner, false, true)
	if err != nil {
		return false
	}
	if !report.Ready {
		return false
	}
	// Check performs stream redaction before decoding; retain this sanitization call as a boundary
	// if future report fields are added to the backend machine contract.
	for _, check := range report.Checks {
		if output.Sanitize(check.Message, secrets) != check.Message {
			return false
		}
	}
	return true
}

func workflowCheck(ctx context.Context, installation config.Installation, runner Runner, secrets []string, add func(string, string, string)) {
	result, err := runner.Run(ctx, installation.ComposeArgs(
		"exec", "-T", "core", "node", "dist/operator-command.js", "workflow-doctor",
	), nil)
	if err != nil {
		add("workflow", StatusFailed, commandDetail("container-local workflow doctor", result, err, secrets))
		return
	}
	var payload struct {
		Ready      bool `json:"ready"`
		Workspaces int  `json:"workspaces"`
	}
	if json.Unmarshal([]byte(result.Stdout), &payload) != nil || !payload.Ready || payload.Workspaces < 1 {
		add("workflow", StatusFailed, "container-local workflow doctor returned an invalid or failing report")
		return
	}
	add("workflow", StatusPassed, "container-local workflow doctor passed")
}

func piCheck(ctx context.Context, installation config.Installation, runner Runner, secrets []string, add func(string, string, string)) {
	controlled := compose.InstallationRunner{Installation: installation, Runner: runner}
	if err := pi.Doctor(ctx, controlled); err != nil {
		add("pi", StatusFailed, output.SanitizeDetail(err.Error(), secrets))
		return
	}
	add("pi", StatusPassed, "Pi doctor passed")
}

func validateInstallation(installation config.Installation) error {
	if installation.Path == "" || installation.ProjectDirectory == "" || installation.EnvFile == "" {
		return errors.New("installation descriptor is incomplete")
	}
	return nil
}

func secretValues(installation config.Installation) ([]string, error) {
	files, err := installation.SecretFiles()
	if err != nil {
		return nil, errors.New("declared secret files could not be read")
	}
	values, err := output.SecretValuesFromFiles(files)
	if err != nil {
		return nil, errors.New("declared secret files could not be read")
	}
	return values, nil
}

func filePermissions(installation config.Installation) error {
	files, err := installation.SecretFiles()
	if err != nil {
		return errors.New("declared secret files could not be read")
	}
	for _, path := range append([]string{installation.EnvFile}, files...) {
		info, err := os.Stat(path)
		if err != nil || !info.Mode().IsRegular() {
			return fmt.Errorf("required host file is unavailable: %s", filepath.Base(path))
		}
		if info.Mode().Perm()&0o077 != 0 {
			return fmt.Errorf("required host file has unsafe permissions: %s", filepath.Base(path))
		}
	}
	return nil
}

type renderedMount struct {
	Type     string `json:"type"`
	Target   string `json:"target"`
	ReadOnly bool   `json:"read_only"`
}

type renderedService struct {
	Volumes []renderedMount `json:"volumes"`
}

type renderedPersistence struct {
	Volumes  map[string]json.RawMessage `json:"volumes"`
	Services map[string]renderedService `json:"services"`
}

var requiredNamedVolumes = []string{
	"settings", "pi-state", "workspace-registry", "workspace-secrets",
	"sessions", "qdrant-data", "embedding-models", "auth-state",
}

var requiredPersistentMounts = []struct {
	service string
	target  string
	label   string
}{
	{service: "core", target: "/data/settings", label: "settings"},
	{service: "core", target: "/home/thoth/.pi", label: "pi-state"},
	{service: "core", target: "/data/workspace-registry", label: "workspace-registry"},
	{service: "core", target: "/data/workspace-secrets", label: "workspace-secrets"},
	{service: "core", target: "/data/sessions", label: "sessions"},
	{service: "qdrant", target: "/qdrant/storage", label: "qdrant-data"},
	{service: "embedding", target: "/root/.ollama", label: "embedding-models"},
	{service: "core", target: "/data/auth", label: "auth-state"},
}

// ValidateVolumes accepts either the portable named-volume layout or the server layout where a
// writable bind root owns several nested persistence paths. Docker Compose omits unused top-level
// volume declarations after a server override, so declarations alone cannot validate that profile.
func ValidateVolumes(rendered string) error {
	var document renderedPersistence
	if err := json.Unmarshal([]byte(rendered), &document); err != nil {
		return errors.New("Compose returned invalid rendered configuration")
	}
	missing := ""
	for _, name := range requiredNamedVolumes {
		if _, exists := document.Volumes[name]; !exists {
			missing = name
			break
		}
	}
	if missing == "" {
		return nil
	}
	if len(document.Services) == 0 {
		return fmt.Errorf("rendered Compose configuration is missing required volume %s", missing)
	}
	for _, required := range requiredPersistentMounts {
		service, exists := document.Services[required.service]
		if !exists || !hasWritableMountCovering(service.Volumes, required.target) {
			return fmt.Errorf("rendered Compose configuration is missing persistent mount %s", required.label)
		}
	}
	return nil
}

func workspaceRegistryDeclared(rendered string) bool {
	var document renderedPersistence
	if json.Unmarshal([]byte(rendered), &document) != nil {
		return false
	}
	if document.Volumes["workspace-registry"] != nil {
		return true
	}
	return hasWritableMountCovering(document.Services["core"].Volumes, "/data/workspace-registry")
}

func hasWritableMountCovering(mounts []renderedMount, target string) bool {
	for _, mount := range mounts {
		if mount.ReadOnly || (mount.Type != "bind" && mount.Type != "volume") {
			continue
		}
		mountTarget := filepath.Clean(mount.Target)
		if !filepath.IsAbs(mountTarget) {
			continue
		}
		relative, err := filepath.Rel(mountTarget, target)
		if err == nil && relative != ".." && !strings.HasPrefix(relative, ".."+string(filepath.Separator)) {
			return true
		}
	}
	return false
}

func commandDetail(label string, result compose.Result, err error, secrets []string) string {
	if result.ExitCode != 0 {
		return output.SanitizeDetail(fmt.Sprintf("%s failed (exit %d): %s", label, result.ExitCode, result.Stderr), secrets)
	}
	if err != nil {
		return output.SanitizeDetail(fmt.Sprintf("%s failed: %s", label, result.Stderr), secrets)
	}
	return output.SanitizeDetail(label+" returned no output", secrets)
}