feat: complete catalog-driven preprocessing
Publish documentation / publish (push) Successful in 2m12s

This commit is contained in:
Codex
2026-09-06 17:49:35 +02:00
parent 8707ae1d46
commit cffa60772e
141 changed files with 5898 additions and 3015 deletions
+39 -510
View File
@@ -4,7 +4,6 @@ package workspaceops
import (
"bytes"
"context"
"crypto/sha256"
"encoding/json"
"errors"
"fmt"
@@ -17,19 +16,9 @@ import (
"github.com/aritmolab/thothii/tools/tht/internal/compose"
"github.com/aritmolab/thothii/tools/tht/internal/config"
"github.com/aritmolab/thothii/tools/tht/internal/safeio"
)
var (
workspacePattern = regexp.MustCompile(`^[a-z][a-z0-9-]{2,62}$`)
runIDPattern = regexp.MustCompile(`^[0-9a-f]{32}$`)
reviewedCandidatesDigest = regexp.MustCompile(`^sha256:[0-9a-f]{64}$`)
)
const (
maxFromSQLFiles = 32
maxAssumptions = 256
)
var workspacePattern = regexp.MustCompile(`^[a-z][a-z0-9-]{2,62}$`)
type Runner interface {
Run(context.Context, []string, io.Reader) (compose.Result, error)
@@ -53,136 +42,32 @@ func (b baseRequest) JSONMode() bool { return b.JSON }
type InspectRequest struct{ baseRequest }
type DwhRequest struct {
baseRequest
Resume string
}
type RunRequest struct{ baseRequest }
type ClearRequest struct{ baseRequest }
type SuggestFksRequest struct {
baseRequest
FromSQL []string
Assume []string
Output string
}
func (InspectRequest) workspaceRequest() {}
func (RunRequest) workspaceRequest() {}
func (ClearRequest) workspaceRequest() {}
type CheckSchemaRequest struct {
baseRequest
Annotations string
ReviewedCandidates string
}
type AcceptSchemaRequest struct {
baseRequest
Run string
Yes bool
}
type IndexSchemaRequest struct{ baseRequest }
type EvidenceRequest struct {
baseRequest
DryRun bool
Resume string
}
type RunRequest struct {
baseRequest
Resume string
}
func (InspectRequest) workspaceRequest() {}
func (DwhRequest) workspaceRequest() {}
func (SuggestFksRequest) workspaceRequest() {}
func (CheckSchemaRequest) workspaceRequest() {}
func (AcceptSchemaRequest) workspaceRequest() {}
func (IndexSchemaRequest) workspaceRequest() {}
func (EvidenceRequest) workspaceRequest() {}
func (RunRequest) workspaceRequest() {}
func (InspectRequest) operatorCommand() string { return "inspect" }
func (DwhRequest) operatorCommand() string { return "preprocess-dwh" }
func (SuggestFksRequest) operatorCommand() string { return "schema-suggest-fks" }
func (CheckSchemaRequest) operatorCommand() string {
return "schema-check"
}
func (AcceptSchemaRequest) operatorCommand() string { return "schema-accept" }
func (IndexSchemaRequest) operatorCommand() string { return "index-schema" }
func (EvidenceRequest) operatorCommand() string { return "preprocess-evidence" }
func (RunRequest) operatorCommand() string { return "preprocess-run" }
func (InspectRequest) operatorCommand() string { return "inspect" }
func (RunRequest) operatorCommand() string { return "preprocess-run" }
func (ClearRequest) operatorCommand() string { return "preprocess-clear" }
func (r InspectRequest) stdinEnvelope() (requestEnvelope, error) {
return requestEnvelope{SchemaVersion: 1, WorkspaceID: r.Workspace}, nil
}
func (r DwhRequest) stdinEnvelope() (requestEnvelope, error) {
return requestEnvelope{SchemaVersion: 1, WorkspaceID: r.Workspace, Resume: r.Resume}, nil
}
func (r SuggestFksRequest) stdinEnvelope() (requestEnvelope, error) {
envelope := requestEnvelope{SchemaVersion: 1, WorkspaceID: r.Workspace, Assume: append([]string(nil), r.Assume...)}
totalBytes := 0
for _, path := range r.FromSQL {
contents, err := safeio.ReadCanonicalUTF8(path, 1<<20)
if err != nil {
return requestEnvelope{}, errors.New("SQL input could not be read safely")
}
totalBytes += len(contents)
if totalBytes > 16<<20 {
return requestEnvelope{}, errors.New("SQL input total exceeds 16 MiB")
}
envelope.SQLFiles = append(envelope.SQLFiles, inputFile{Name: filepath.Base(path), SQL: contents})
}
return envelope, nil
}
func (r CheckSchemaRequest) stdinEnvelope() (requestEnvelope, error) {
annotations, err := safeio.ReadCanonicalUTF8(r.Annotations, 16<<20)
if err != nil {
return requestEnvelope{}, errors.New("annotation file could not be read safely")
}
return requestEnvelope{
SchemaVersion: 1,
WorkspaceID: r.Workspace,
Annotations: annotations,
ReviewedCandidates: r.ReviewedCandidates,
}, nil
}
func (r IndexSchemaRequest) stdinEnvelope() (requestEnvelope, error) {
func (r RunRequest) stdinEnvelope() (requestEnvelope, error) {
return requestEnvelope{SchemaVersion: 1, WorkspaceID: r.Workspace}, nil
}
func (r AcceptSchemaRequest) stdinEnvelope() (requestEnvelope, error) {
return requestEnvelope{SchemaVersion: 1, WorkspaceID: r.Workspace, RunID: r.Run, Yes: r.Yes}, nil
}
func (r EvidenceRequest) stdinEnvelope() (requestEnvelope, error) {
return requestEnvelope{SchemaVersion: 1, WorkspaceID: r.Workspace, Resume: r.Resume, DryRun: r.DryRun}, nil
}
func (r RunRequest) stdinEnvelope() (requestEnvelope, error) {
return requestEnvelope{SchemaVersion: 1, WorkspaceID: r.Workspace, Resume: r.Resume}, nil
func (r ClearRequest) stdinEnvelope() (requestEnvelope, error) {
return requestEnvelope{SchemaVersion: 1, WorkspaceID: r.Workspace}, nil
}
type requestEnvelope struct {
SchemaVersion int `json:"schemaVersion"`
WorkspaceID string `json:"workspaceId"`
Resume string `json:"resumeRunId,omitempty"`
DryRun bool `json:"dryRun,omitempty"`
Assume []string `json:"assume,omitempty"`
SQLFiles []inputFile `json:"fromSql,omitempty"`
Annotations string `json:"annotationsYaml,omitempty"`
ReviewedCandidates string `json:"reviewedCandidatesDigest,omitempty"`
Collection string `json:"collection,omitempty"`
Confirm string `json:"confirm,omitempty"`
Destroy bool `json:"destroy,omitempty"`
RunID string `json:"runId,omitempty"`
Yes bool `json:"yes,omitempty"`
}
type inputFile struct {
Name string `json:"name"`
SQL string `json:"sql"`
SchemaVersion int `json:"schemaVersion"`
WorkspaceID string `json:"workspaceId"`
}
type Result struct {
@@ -198,7 +83,6 @@ type Result struct {
CompletedStages []string `json:"completedStages"`
Counts map[string]int `json:"counts,omitempty"`
ArtifactIdentities []ArtifactIdentity `json:"artifactIdentities,omitempty"`
SuggestedFksYAML string `json:"suggestedFksYaml,omitempty"`
EffectiveConfigIdentity string `json:"effectiveConfigIdentity,omitempty"`
ConfigFingerprint string `json:"configFingerprint,omitempty"`
InputFingerprint string `json:"inputFingerprint,omitempty"`
@@ -212,7 +96,6 @@ type ArtifactIdentity struct {
type operationResponse struct {
Result
SuggestedFksYAML string `json:"suggestedFksYaml,omitempty"`
}
type Stage string
@@ -266,35 +149,11 @@ func Parse(args []string) (Request, error) {
return parsed, nil
case "preprocess":
return parsePreprocess(args[1:])
case "schema":
return parseSchema(args[1:])
case "index-schema":
parsed, err := parseIndexSchema(args[1:])
if err != nil {
return nil, err
}
return parsed, nil
case "vector":
return parseVector(args[1:])
default:
return nil, fmt.Errorf("unknown workspace command %q", args[0])
}
}
func parseVector(args []string) (Request, error) {
if len(args) == 0 {
return nil, errors.New("vector requires a subcommand")
}
switch args[0] {
case "inspect":
return parseVectorInspect(args[1:])
case "rebuild":
return parseVectorRebuild(args[1:])
default:
return nil, fmt.Errorf("unknown vector command %q", args[0])
}
}
func Execute(ctx context.Context, installation config.Installation, runner Runner, request Request) (Result, error) {
envelope, err := request.stdinEnvelope()
if err != nil {
@@ -340,21 +199,6 @@ func Execute(ctx context.Context, installation config.Installation, runner Runne
}
return Result{}, parseErr
}
if suggest, ok := request.(SuggestFksRequest); ok && suggest.Output != "" {
if response.SuggestedFksYAML == "" {
return Result{}, errors.New("workspace maintenance did not return the requested FK artifact")
}
if digest := suggestedArtifactDigest(response); digest != "" {
sum := sha256.Sum256([]byte(response.SuggestedFksYAML))
if digest != "sha256:"+fmt.Sprintf("%x", sum[:]) {
return Result{}, errors.New("workspace maintenance returned an FK artifact with a mismatched digest")
}
}
if err := safeio.WriteCanonicalNewFile(suggest.Output, []byte(response.SuggestedFksYAML), 0o600); err != nil {
return Result{}, errors.New("workspace FK output file could not be created safely")
}
}
response.Result.SuggestedFksYAML = response.SuggestedFksYAML
return response.Result, nil
}
@@ -428,7 +272,7 @@ func isLowerHex(value string) bool {
}
func parseInspect(args []string) (InspectRequest, error) {
base, err := parseBaseFlags(args, false)
base, err := parseBaseFlags(args)
if err != nil {
return InspectRequest{}, err
}
@@ -437,299 +281,61 @@ func parseInspect(args []string) (InspectRequest, error) {
func parsePreprocess(args []string) (Request, error) {
if len(args) == 0 {
return nil, errors.New("workspace preprocess requires dwh, evidence, or run")
return nil, errors.New("workspace preprocess requires run or clear")
}
switch args[0] {
case "dwh":
base, resume, dryRun, err := parseResumeFlags(args[1:], false)
if err != nil {
return nil, err
}
if dryRun {
return nil, errors.New("workspace preprocess dwh does not accept --dry-run")
}
return DwhRequest{baseRequest: base, Resume: resume}, nil
case "evidence":
base, resume, dryRun, err := parseResumeFlags(args[1:], true)
if err != nil {
return nil, err
}
return EvidenceRequest{baseRequest: base, Resume: resume, DryRun: dryRun}, nil
case "run":
base, resume, dryRun, err := parseResumeFlags(args[1:], false)
base, err := parseBaseFlags(args[1:])
if err != nil {
return nil, err
}
if dryRun {
return nil, errors.New("workspace preprocess run does not accept --dry-run")
return RunRequest{baseRequest: base}, nil
case "clear":
base, err := parseBaseFlags(args[1:])
if err != nil {
return nil, err
}
return RunRequest{baseRequest: base, Resume: resume}, nil
return ClearRequest{baseRequest: base}, nil
default:
return nil, fmt.Errorf("unknown workspace preprocess command %q", args[0])
}
}
func parseSchema(args []string) (Request, error) {
if len(args) == 0 {
return nil, errors.New("workspace schema requires suggest-fks or check")
}
switch args[0] {
case "suggest-fks":
return parseSuggestFks(args[1:])
case "check":
return parseSchemaCheck(args[1:])
case "accept":
return parseSchemaAccept(args[1:])
default:
return nil, fmt.Errorf("unknown workspace schema command %q", args[0])
}
}
func parseIndexSchema(args []string) (IndexSchemaRequest, error) {
base, err := parseBaseFlags(args, false)
if err != nil {
return IndexSchemaRequest{}, err
}
return IndexSchemaRequest{baseRequest: base}, nil
}
func parseResumeFlags(args []string, allowDryRun bool) (baseRequest, string, bool, error) {
var resume string
var dryRun bool
base, seen, err := parseSharedFlags(args, map[string]func(string) error{
"--resume": func(value string) error {
if resume != "" {
return errors.New("--resume may be supplied once")
}
if !runIDPattern.MatchString(value) {
return errors.New("--resume must be 32 lowercase hex characters")
}
resume = value
return nil
},
}, map[string]func() error{
"--dry-run": func() error {
if !allowDryRun {
return errors.New("--dry-run is not accepted here")
}
if dryRun {
return errors.New("--dry-run may be supplied once")
}
dryRun = true
return nil
},
})
if err != nil {
return baseRequest{}, "", false, err
}
if !seen.workspace {
return baseRequest{}, "", false, errors.New("--workspace is required")
}
return base, resume, dryRun, nil
}
func parseSuggestFks(args []string) (SuggestFksRequest, error) {
request := SuggestFksRequest{}
base, seen, err := parseSharedFlags(args, map[string]func(string) error{
"--from-sql": func(value string) error {
if len(request.FromSQL) >= maxFromSQLFiles {
return fmt.Errorf("--from-sql may be supplied at most %d times", maxFromSQLFiles)
}
request.FromSQL = append(request.FromSQL, value)
return nil
},
"--assume": func(value string) error {
if len(request.Assume) >= maxAssumptions {
return fmt.Errorf("--assume may be supplied at most %d times", maxAssumptions)
}
if len(value) > 256 || !strings.Contains(value, "=") {
return errors.New("--assume values must be column=table entries up to 256 bytes")
}
left, right, _ := strings.Cut(value, "=")
if strings.TrimSpace(left) == "" || strings.TrimSpace(right) == "" {
return errors.New("--assume values must be column=table entries up to 256 bytes")
}
request.Assume = append(request.Assume, value)
return nil
},
"--output": func(value string) error {
if request.Output != "" {
return errors.New("--output may be supplied once")
}
request.Output = value
return nil
},
}, nil)
if err != nil {
return SuggestFksRequest{}, err
}
if !seen.workspace {
return SuggestFksRequest{}, errors.New("--workspace is required")
}
request.baseRequest = base
return request, nil
}
func parseSchemaCheck(args []string) (CheckSchemaRequest, error) {
request := CheckSchemaRequest{}
base, seen, err := parseSharedFlags(args, map[string]func(string) error{
"--annotations": func(value string) error {
if request.Annotations != "" {
return errors.New("--annotations may be supplied once")
}
request.Annotations = value
return nil
},
"--reviewed-candidates": func(value string) error {
if request.ReviewedCandidates != "" {
return errors.New("--reviewed-candidates may be supplied once")
}
if !reviewedCandidatesDigest.MatchString(value) {
return errors.New("--reviewed-candidates must be sha256:<64 lowercase hex>")
}
request.ReviewedCandidates = value
return nil
},
}, nil)
if err != nil {
return CheckSchemaRequest{}, err
}
if !seen.workspace {
return CheckSchemaRequest{}, errors.New("--workspace is required")
}
if (request.Annotations == "") != (request.ReviewedCandidates == "") {
return CheckSchemaRequest{}, errors.New("--annotations and --reviewed-candidates must be supplied together")
}
request.baseRequest = base
return request, nil
}
func parseSchemaAccept(args []string) (AcceptSchemaRequest, error) {
request := AcceptSchemaRequest{}
base, seen, err := parseSharedFlags(args, map[string]func(string) error{
"--run": func(value string) error {
if request.Run != "" {
return errors.New("--run may be supplied once")
}
if !runIDPattern.MatchString(value) {
return errors.New("--run must be 32 lowercase hex characters")
}
request.Run = value
return nil
},
}, map[string]func() error{
"--yes": func() error {
if request.Yes {
return errors.New("--yes may be supplied once")
}
request.Yes = true
return nil
},
})
if err != nil {
return AcceptSchemaRequest{}, err
}
if !seen.workspace {
return AcceptSchemaRequest{}, errors.New("--workspace is required")
}
if request.Run == "" {
return AcceptSchemaRequest{}, errors.New("--run is required")
}
if !request.Yes {
return AcceptSchemaRequest{}, errors.New("--yes is required")
}
request.baseRequest = base
return request, nil
}
func parseBaseFlags(args []string, allowDryRun bool) (baseRequest, error) {
base, seen, err := parseSharedFlags(args, nil, nil)
if err != nil {
return baseRequest{}, err
}
if !seen.workspace {
return baseRequest{}, errors.New("--workspace is required")
}
return base, nil
}
type seenFlags struct {
workspace bool
json bool
}
func parseSharedFlags(args []string, valueHandlers map[string]func(string) error, boolHandlers map[string]func() error) (baseRequest, seenFlags, error) {
func parseBaseFlags(args []string) (baseRequest, error) {
request := baseRequest{}
seen := seenFlags{}
valueHandlers = cloneValueHandlers(valueHandlers)
boolHandlers = cloneBoolHandlers(boolHandlers)
workspaceSeen := false
jsonSeen := false
for len(args) > 0 {
flag := args[0]
if flag == "--" {
return baseRequest{}, seenFlags{}, errors.New("passthrough separators are not supported")
return baseRequest{}, errors.New("passthrough separators are not supported")
}
switch flag {
case "--workspace":
if len(args) < 2 {
return baseRequest{}, seenFlags{}, errors.New("--workspace requires a value")
return baseRequest{}, errors.New("--workspace requires a value")
}
if seen.workspace {
return baseRequest{}, seenFlags{}, errors.New("--workspace must be supplied exactly once")
if workspaceSeen {
return baseRequest{}, errors.New("--workspace must be supplied exactly once")
}
workspace := args[1]
if !workspacePattern.MatchString(workspace) {
return baseRequest{}, seenFlags{}, errors.New("--workspace must match [a-z][a-z0-9-]{2,62}")
return baseRequest{}, errors.New("--workspace must match [a-z][a-z0-9-]{2,62}")
}
request.Workspace, seen.workspace, args = workspace, true, args[2:]
request.Workspace, workspaceSeen, args = workspace, true, args[2:]
case "--json":
if seen.json {
return baseRequest{}, seenFlags{}, errors.New("--json may be supplied once")
if jsonSeen {
return baseRequest{}, errors.New("--json may be supplied once")
}
request.JSON, seen.json, args = true, true, args[1:]
request.JSON, jsonSeen, args = true, true, args[1:]
default:
if handler, ok := boolHandlers[flag]; ok {
if err := handler(); err != nil {
return baseRequest{}, seenFlags{}, err
}
args = args[1:]
continue
}
handler, ok := valueHandlers[flag]
if !ok {
return baseRequest{}, seenFlags{}, fmt.Errorf("unknown workspace option %q", flag)
}
if len(args) < 2 {
return baseRequest{}, seenFlags{}, fmt.Errorf("%s requires a value", flag)
}
if err := handler(args[1]); err != nil {
return baseRequest{}, seenFlags{}, err
}
args = args[2:]
return baseRequest{}, fmt.Errorf("unknown workspace option %q", flag)
}
}
return request, seen, nil
}
func cloneValueHandlers(source map[string]func(string) error) map[string]func(string) error {
if len(source) == 0 {
return map[string]func(string) error{}
if !workspaceSeen {
return baseRequest{}, errors.New("--workspace is required")
}
clone := make(map[string]func(string) error, len(source))
for key, handler := range source {
clone[key] = handler
}
return clone
}
func cloneBoolHandlers(source map[string]func() error) map[string]func() error {
if len(source) == 0 {
return map[string]func() error{}
}
clone := make(map[string]func() error, len(source))
for key, handler := range source {
clone[key] = handler
}
return clone
return request, nil
}
func selectedCoreImage(document string) (string, error) {
@@ -841,15 +447,6 @@ func runDocker(ctx context.Context, runner Runner, stage Stage, args []string, s
return result, nil
}
func suggestedArtifactDigest(response operationResponse) string {
for _, artifact := range response.ArtifactIdentities {
if strings.HasPrefix(artifact.Digest, "sha256:") {
return artifact.Digest
}
}
return ""
}
func Human(result Result) string {
lines := []string{
fmt.Sprintf("workspace: %s", result.WorkspaceID),
@@ -871,71 +468,3 @@ func Human(result Result) string {
}
return strings.Join(lines, "\n") + "\n"
}
// VectorInspectRequest reads the Qdrant collection contract without mutation.
type VectorInspectRequest struct{ baseRequest }
// VectorRebuildRequest deletes and recreates the descriptor-owned collection under guards.
type VectorRebuildRequest struct {
baseRequest
Collection string
Confirm string
Destroy bool
}
func (VectorInspectRequest) workspaceRequest() {}
func (VectorRebuildRequest) workspaceRequest() {}
func (VectorInspectRequest) operatorCommand() string { return "vector-inspect" }
func (VectorRebuildRequest) operatorCommand() string { return "vector-rebuild" }
func (r VectorInspectRequest) stdinEnvelope() (requestEnvelope, error) {
return requestEnvelope{SchemaVersion: 1, WorkspaceID: r.Workspace}, nil
}
func (r VectorRebuildRequest) stdinEnvelope() (requestEnvelope, error) {
if r.Collection == "" {
return requestEnvelope{}, errors.New("--collection is required")
}
if r.Confirm == "" {
return requestEnvelope{}, errors.New("--confirm is required and must equal --collection")
}
if r.Confirm != r.Collection {
return requestEnvelope{}, errors.New("--confirm must equal --collection")
}
if !r.Destroy {
return requestEnvelope{}, errors.New("--destroy is required to confirm the destructive rebuild")
}
return requestEnvelope{
SchemaVersion: 1,
WorkspaceID: r.Workspace,
Collection: r.Collection,
Confirm: r.Confirm,
Destroy: r.Destroy,
}, nil
}
func parseVectorInspect(args []string) (Request, error) {
base, err := parseBaseFlags(args, false)
if err != nil {
return nil, err
}
return VectorInspectRequest{baseRequest: base}, nil
}
func parseVectorRebuild(args []string) (Request, error) {
request := VectorRebuildRequest{}
values := map[string]func(string) error{
"--collection": func(v string) error { request.Collection = v; return nil },
"--confirm": func(v string) error { request.Confirm = v; return nil },
}
bools := map[string]func() error{
"--destroy": func() error { request.Destroy = true; return nil },
}
base, seen, err := parseSharedFlags(args, values, bools)
if err != nil {
return nil, err
}
if !seen.workspace {
return nil, errors.New("--workspace is required")
}
request.baseRequest = base
return request, nil
}