fix: complete P2 host workspace contract
This commit is contained in:
@@ -5,17 +5,21 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"reflect"
|
||||
"regexp"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/aritmolab/thothii/tools/thothctl/internal/compose"
|
||||
"github.com/aritmolab/thothii/tools/thothctl/internal/config"
|
||||
"github.com/aritmolab/thothii/tools/thothctl/internal/safeio"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
const CodeRegistryBootstrapRecoveryConflict = "registry_bootstrap_recovery_conflict"
|
||||
@@ -26,21 +30,23 @@ const (
|
||||
maxAssumptionBytes = 256
|
||||
maxResult = 1 << 20
|
||||
maxCandidate = 700 << 10
|
||||
maxAnnotations = 16 << 20
|
||||
)
|
||||
|
||||
var workspaceIDPattern = regexp.MustCompile(`^[a-z][a-z0-9-]{2,62}$`)
|
||||
var runIDPattern = regexp.MustCompile(`^[0-9a-f]{32}$`)
|
||||
var revisionPattern = regexp.MustCompile(`^[0-9a-f]{40}$`)
|
||||
var digestPattern = regexp.MustCompile(`^sha256:[0-9a-f]{64}$`)
|
||||
|
||||
// Command is intentionally closed: callers cannot provide a child argv or bootstrap run ID.
|
||||
type Command interface{ workspaceCommand() }
|
||||
type InspectCommand struct {
|
||||
WorkspaceID string
|
||||
JSON bool
|
||||
}
|
||||
type DwhRequest struct {
|
||||
WorkspaceID string
|
||||
Resume string
|
||||
JSON bool
|
||||
WorkspaceID, Resume string
|
||||
JSON bool
|
||||
}
|
||||
type SuggestFksRequest struct {
|
||||
WorkspaceID string
|
||||
@@ -50,11 +56,8 @@ type SuggestFksRequest struct {
|
||||
JSON bool
|
||||
}
|
||||
type CheckSchemaRequest struct {
|
||||
WorkspaceID string
|
||||
Resume string
|
||||
Annotations string
|
||||
ReviewedCandidates string
|
||||
JSON bool
|
||||
WorkspaceID, Resume, Annotations, ReviewedCandidates string
|
||||
JSON bool
|
||||
}
|
||||
type IndexSchemaRequest struct {
|
||||
WorkspaceID string
|
||||
@@ -67,9 +70,8 @@ type EvidenceRequest struct {
|
||||
JSON bool
|
||||
}
|
||||
type RunRequest struct {
|
||||
WorkspaceID string
|
||||
Resume string
|
||||
JSON bool
|
||||
WorkspaceID, Resume string
|
||||
JSON bool
|
||||
}
|
||||
|
||||
func (InspectCommand) workspaceCommand() {}
|
||||
@@ -99,6 +101,33 @@ type ArtifactIdentity struct {
|
||||
Kind string `json:"kind"`
|
||||
Digest string `json:"digest"`
|
||||
}
|
||||
type hostExport struct {
|
||||
MediaType string `json:"mediaType"`
|
||||
SHA256 string `json:"sha256"`
|
||||
ContentBase64 string `json:"contentBase64"`
|
||||
}
|
||||
|
||||
type sqlInput struct {
|
||||
Basename string `json:"basename"`
|
||||
ContentBase64 string `json:"contentBase64"`
|
||||
SHA256 string `json:"sha256"`
|
||||
}
|
||||
type annotationInput struct {
|
||||
Basename string `json:"basename"`
|
||||
ContentBase64 string `json:"contentBase64"`
|
||||
SHA256 string `json:"sha256"`
|
||||
}
|
||||
type inputEnvelope struct {
|
||||
SchemaVersion int `json:"schemaVersion"`
|
||||
Operation string `json:"operation"`
|
||||
WorkspaceID string `json:"workspaceId"`
|
||||
Resume string `json:"resume,omitempty"`
|
||||
SQL []sqlInput `json:"sql,omitempty"`
|
||||
Assume []string `json:"assume,omitempty"`
|
||||
Annotations *annotationInput `json:"annotations,omitempty"`
|
||||
ReviewedCandidates string `json:"reviewedCandidates,omitempty"`
|
||||
DryRun bool `json:"dryRun,omitempty"`
|
||||
}
|
||||
|
||||
func invalid(msg string) (Command, error) { return nil, errors.New(msg) }
|
||||
func requireWorkspace(v string) error {
|
||||
@@ -117,46 +146,77 @@ func one(args []string, i *int, flag string) (string, error) {
|
||||
if *i+1 >= len(args) || strings.HasPrefix(args[*i+1], "--") {
|
||||
return "", fmt.Errorf("%s requires a value", flag)
|
||||
}
|
||||
*i = *i + 1
|
||||
*i++
|
||||
return args[*i], nil
|
||||
}
|
||||
|
||||
func ParseWorkspaceCommand(args []string) (Command, error) {
|
||||
if len(args) < 2 || args[0] != "workspace" {
|
||||
return invalid("workspace command is required")
|
||||
}
|
||||
area, action := args[1], ""
|
||||
rest := args[2:]
|
||||
if area == "preprocess" || area == "schema" {
|
||||
if len(rest) == 0 {
|
||||
return invalid("workspace group requires an operation")
|
||||
}
|
||||
action = rest[0]
|
||||
rest = rest[1:]
|
||||
} else {
|
||||
action = area
|
||||
}
|
||||
if action != "inspect" && action != "dwh" && action != "suggest-fks" && action != "check" && action != "index-schema" && action != "evidence" && action != "run" {
|
||||
return invalid("unknown workspace command")
|
||||
// Parse the hierarchy before options. This prevents aliases such as workspace schema dwh.
|
||||
path := strings.Join(args[:min(3, len(args))], " ")
|
||||
var action string
|
||||
var rest []string
|
||||
switch {
|
||||
case args[1] == "inspect":
|
||||
action = "inspect"
|
||||
rest = args[2:]
|
||||
case len(args) >= 3 && args[1] == "preprocess" && (args[2] == "dwh" || args[2] == "evidence" || args[2] == "run"):
|
||||
action = "preprocess " + args[2]
|
||||
rest = args[3:]
|
||||
case len(args) >= 3 && args[1] == "schema" && (args[2] == "suggest-fks" || args[2] == "check"):
|
||||
action = "schema " + args[2]
|
||||
rest = args[3:]
|
||||
case args[1] == "index-schema":
|
||||
action = "index-schema"
|
||||
rest = args[2:]
|
||||
default:
|
||||
return invalid("unknown workspace command: " + path)
|
||||
}
|
||||
var ws, resume, annotations, reviewed, output string
|
||||
var jsonOut, dry bool
|
||||
var sql, assume []string
|
||||
seen := map[string]bool{}
|
||||
allowed := func(flag string) bool {
|
||||
switch action {
|
||||
case "inspect":
|
||||
return flag == "--workspace" || flag == "--json"
|
||||
case "preprocess dwh", "preprocess run":
|
||||
return flag == "--workspace" || flag == "--resume" || flag == "--json"
|
||||
case "preprocess evidence":
|
||||
return flag == "--workspace" || flag == "--resume" || flag == "--dry-run" || flag == "--json"
|
||||
case "schema suggest-fks":
|
||||
return flag == "--workspace" || flag == "--from-sql" || flag == "--assume" || flag == "--output" || flag == "--json"
|
||||
case "schema check":
|
||||
return flag == "--workspace" || flag == "--resume" || flag == "--annotations" || flag == "--reviewed-candidates" || flag == "--json"
|
||||
case "index-schema":
|
||||
return flag == "--workspace" || flag == "--json"
|
||||
}
|
||||
return false
|
||||
}
|
||||
for i := 0; i < len(rest); i++ {
|
||||
f := rest[i]
|
||||
if f == "--json" {
|
||||
if !allowed(f) {
|
||||
return invalid("unknown or invalid workspace option")
|
||||
}
|
||||
if f == "--json" || f == "--dry-run" {
|
||||
if seen[f] {
|
||||
return invalid("duplicate --json")
|
||||
return invalid("duplicate " + f)
|
||||
}
|
||||
seen[f] = true
|
||||
jsonOut = true
|
||||
if f == "--json" {
|
||||
jsonOut = true
|
||||
} else {
|
||||
dry = true
|
||||
}
|
||||
continue
|
||||
}
|
||||
if seen[f] && f != "--from-sql" && f != "--assume" {
|
||||
return invalid("duplicate " + f)
|
||||
}
|
||||
switch f {
|
||||
case "--workspace":
|
||||
if seen[f] {
|
||||
return invalid("duplicate --workspace")
|
||||
}
|
||||
seen[f] = true
|
||||
v, e := one(rest, &i, f)
|
||||
if e != nil {
|
||||
@@ -164,9 +224,6 @@ func ParseWorkspaceCommand(args []string) (Command, error) {
|
||||
}
|
||||
ws = v
|
||||
case "--resume":
|
||||
if seen[f] {
|
||||
return invalid("duplicate --resume")
|
||||
}
|
||||
seen[f] = true
|
||||
v, e := one(rest, &i, f)
|
||||
if e != nil {
|
||||
@@ -174,9 +231,6 @@ func ParseWorkspaceCommand(args []string) (Command, error) {
|
||||
}
|
||||
resume = v
|
||||
case "--from-sql":
|
||||
if action != "suggest-fks" {
|
||||
return invalid("--from-sql is valid only for schema suggest-fks")
|
||||
}
|
||||
v, e := one(rest, &i, f)
|
||||
if e != nil {
|
||||
return nil, e
|
||||
@@ -186,9 +240,6 @@ func ParseWorkspaceCommand(args []string) (Command, error) {
|
||||
}
|
||||
sql = append(sql, v)
|
||||
case "--assume":
|
||||
if action != "suggest-fks" {
|
||||
return invalid("--assume is valid only for schema suggest-fks")
|
||||
}
|
||||
v, e := one(rest, &i, f)
|
||||
if e != nil {
|
||||
return nil, e
|
||||
@@ -196,14 +247,11 @@ func ParseWorkspaceCommand(args []string) (Command, error) {
|
||||
if len(assume) >= maxAssumptions || len([]byte(v)) > maxAssumptionBytes {
|
||||
return invalid("--assume exceeds limit")
|
||||
}
|
||||
if !regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*(?:\.[A-Za-z_][A-Za-z0-9_]*)?=[A-Za-z_][A-Za-z0-9_]*(?:\.[A-Za-z_][A-Za-z0-9_]*)?$`).MatchString(v) {
|
||||
return invalid("--assume must be column=table")
|
||||
}
|
||||
assume = append(assume, v)
|
||||
case "--output":
|
||||
if action != "suggest-fks" {
|
||||
return invalid("--output is valid only for schema suggest-fks")
|
||||
}
|
||||
if seen[f] {
|
||||
return invalid("duplicate --output")
|
||||
}
|
||||
seen[f] = true
|
||||
v, e := one(rest, &i, f)
|
||||
if e != nil {
|
||||
@@ -211,12 +259,6 @@ func ParseWorkspaceCommand(args []string) (Command, error) {
|
||||
}
|
||||
output = v
|
||||
case "--annotations":
|
||||
if action != "check" {
|
||||
return invalid("--annotations is valid only for schema check")
|
||||
}
|
||||
if seen[f] {
|
||||
return invalid("duplicate --annotations")
|
||||
}
|
||||
seen[f] = true
|
||||
v, e := one(rest, &i, f)
|
||||
if e != nil {
|
||||
@@ -224,46 +266,26 @@ func ParseWorkspaceCommand(args []string) (Command, error) {
|
||||
}
|
||||
annotations = v
|
||||
case "--reviewed-candidates":
|
||||
if action != "check" {
|
||||
return invalid("--reviewed-candidates is valid only for schema check")
|
||||
}
|
||||
if seen[f] {
|
||||
return invalid("duplicate --reviewed-candidates")
|
||||
}
|
||||
seen[f] = true
|
||||
v, e := one(rest, &i, f)
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
reviewed = v
|
||||
case "--dry-run":
|
||||
if action != "evidence" {
|
||||
return invalid("--dry-run is valid only for evidence")
|
||||
}
|
||||
if seen[f] {
|
||||
return invalid("duplicate --dry-run")
|
||||
}
|
||||
seen[f] = true
|
||||
dry = true
|
||||
default:
|
||||
return invalid("unknown workspace option")
|
||||
}
|
||||
}
|
||||
if err := requireWorkspace(ws); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if action != "inspect" && resume != "" {
|
||||
if resume != "" {
|
||||
if err := requireRun(resume); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if resume != "" && action != "dwh" && action != "evidence" && action != "run" && action != "check" {
|
||||
return invalid("--resume is not valid for this workspace command")
|
||||
}
|
||||
if action == "inspect" && resume != "" {
|
||||
return invalid("inspect does not accept --resume")
|
||||
}
|
||||
if action == "check" {
|
||||
if action == "schema check" {
|
||||
if resume == "" {
|
||||
return invalid("schema check requires --resume")
|
||||
}
|
||||
@@ -274,43 +296,32 @@ func ParseWorkspaceCommand(args []string) (Command, error) {
|
||||
return invalid("--reviewed-candidates must be sha256:<hex>")
|
||||
}
|
||||
}
|
||||
if action != "suggest-fks" && (len(sql) > 0 || len(assume) > 0 || output != "") {
|
||||
return invalid("schema options are valid only for suggest-fks")
|
||||
if action != "schema check" && (annotations != "" || reviewed != "") {
|
||||
return invalid("annotation options are valid only for schema check")
|
||||
}
|
||||
switch action {
|
||||
case "inspect":
|
||||
return InspectCommand{ws, jsonOut}, nil
|
||||
case "dwh":
|
||||
case "preprocess dwh":
|
||||
return DwhRequest{ws, resume, jsonOut}, nil
|
||||
case "suggest-fks":
|
||||
case "schema suggest-fks":
|
||||
return SuggestFksRequest{ws, sql, assume, output, jsonOut}, nil
|
||||
case "check":
|
||||
case "schema check":
|
||||
return CheckSchemaRequest{ws, resume, annotations, reviewed, jsonOut}, nil
|
||||
case "index-schema":
|
||||
return IndexSchemaRequest{ws, jsonOut}, nil
|
||||
case "evidence":
|
||||
case "preprocess evidence":
|
||||
return EvidenceRequest{ws, dry, resume, jsonOut}, nil
|
||||
default:
|
||||
return RunRequest{ws, resume, jsonOut}, nil
|
||||
}
|
||||
}
|
||||
|
||||
type inputEnvelope struct {
|
||||
SchemaVersion int `json:"schemaVersion"`
|
||||
Operation string `json:"operation"`
|
||||
WorkspaceID string `json:"workspaceId"`
|
||||
Resume string `json:"resume,omitempty"`
|
||||
SQL []sqlInput `json:"sql,omitempty"`
|
||||
Assume []string `json:"assume,omitempty"`
|
||||
Annotations string `json:"annotations,omitempty"`
|
||||
ReviewedCandidates string `json:"reviewedCandidates,omitempty"`
|
||||
DryRun bool `json:"dryRun,omitempty"`
|
||||
func min(a, b int) int {
|
||||
if a < b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
type sqlInput struct {
|
||||
Name string `json:"name"`
|
||||
Content string `json:"content"`
|
||||
}
|
||||
|
||||
func operationName(c Command) string {
|
||||
switch c.(type) {
|
||||
case InspectCommand:
|
||||
@@ -329,114 +340,275 @@ func operationName(c Command) string {
|
||||
return "run"
|
||||
}
|
||||
}
|
||||
func Run(ctx context.Context, installation config.Installation, runner compose.Runner, command Command, stdin io.Reader) (Result, error) {
|
||||
env := inputEnvelope{SchemaVersion: 1, Operation: operationName(command)}
|
||||
var outputPath string
|
||||
switch c := command.(type) {
|
||||
|
||||
func makeInput(c Command) (inputEnvelope, string, error) {
|
||||
env := inputEnvelope{SchemaVersion: 1, Operation: operationName(c)}
|
||||
var out string
|
||||
switch x := c.(type) {
|
||||
case InspectCommand:
|
||||
env.WorkspaceID = c.WorkspaceID
|
||||
env.WorkspaceID = x.WorkspaceID
|
||||
case DwhRequest:
|
||||
env.WorkspaceID, env.Resume = c.WorkspaceID, c.Resume
|
||||
env.WorkspaceID, env.Resume = x.WorkspaceID, x.Resume
|
||||
case SuggestFksRequest:
|
||||
env.WorkspaceID, env.Assume, outputPath = c.WorkspaceID, c.Assume, c.Output
|
||||
if outputPath != "" {
|
||||
if err := safeio.ValidateCanonicalOutputPath(outputPath); err != nil {
|
||||
return Result{}, errors.New("unsafe output file")
|
||||
env.WorkspaceID, env.Assume, out = x.WorkspaceID, x.Assume, x.Output
|
||||
if out != "" {
|
||||
if err := safeio.ValidateCanonicalOutputPath(out); err != nil {
|
||||
return env, "", errors.New("unsafe output file")
|
||||
}
|
||||
}
|
||||
var total int64
|
||||
for _, path := range c.FromSQL {
|
||||
for _, path := range x.FromSQL {
|
||||
b, e := safeio.ReadCanonicalUTF8(path, maxSQLFile)
|
||||
if e != nil {
|
||||
return Result{}, errors.New("unsafe SQL input")
|
||||
return env, "", errors.New("unsafe SQL input")
|
||||
}
|
||||
total += int64(len(b))
|
||||
if total > maxSQLTotal {
|
||||
return Result{}, errors.New("SQL input exceeds limit")
|
||||
}
|
||||
env.SQL = append(env.SQL, sqlInput{path, string(b)})
|
||||
}
|
||||
if len(env.SQL) > 0 {
|
||||
var n int
|
||||
for _, x := range env.SQL {
|
||||
n += len(x.Content)
|
||||
}
|
||||
if n > maxCandidate {
|
||||
return Result{}, errors.New("candidate input exceeds limit")
|
||||
return env, "", errors.New("SQL input exceeds limit")
|
||||
}
|
||||
base := path[strings.LastIndexAny(path, "/\\")+1:]
|
||||
env.SQL = append(env.SQL, sqlInput{base, base64.StdEncoding.EncodeToString(b), DigestBytes(b)})
|
||||
}
|
||||
case CheckSchemaRequest:
|
||||
env.WorkspaceID, env.Resume, env.Annotations, env.ReviewedCandidates = c.WorkspaceID, c.Resume, c.Annotations, c.ReviewedCandidates
|
||||
if c.Annotations != "" {
|
||||
b, e := safeio.ReadCanonicalUTF8(c.Annotations, 16<<20)
|
||||
env.WorkspaceID, env.Resume, env.ReviewedCandidates = x.WorkspaceID, x.Resume, x.ReviewedCandidates
|
||||
if x.Annotations != "" {
|
||||
b, e := safeio.ReadCanonicalUTF8(x.Annotations, maxAnnotations)
|
||||
if e != nil {
|
||||
return Result{}, errors.New("unsafe annotation input")
|
||||
return env, "", errors.New("unsafe annotation input")
|
||||
}
|
||||
env.Annotations = string(b)
|
||||
base := x.Annotations[strings.LastIndexAny(x.Annotations, "/\\")+1:]
|
||||
env.Annotations = &annotationInput{base, base64.StdEncoding.EncodeToString(b), DigestBytes(b)}
|
||||
}
|
||||
case IndexSchemaRequest:
|
||||
env.WorkspaceID = c.WorkspaceID
|
||||
env.WorkspaceID = x.WorkspaceID
|
||||
case EvidenceRequest:
|
||||
env.WorkspaceID, env.Resume, env.DryRun = c.WorkspaceID, c.Resume, c.DryRun
|
||||
env.WorkspaceID, env.Resume, env.DryRun = x.WorkspaceID, x.Resume, x.DryRun
|
||||
case RunRequest:
|
||||
env.WorkspaceID, env.Resume = c.WorkspaceID, c.Resume
|
||||
env.WorkspaceID, env.Resume = x.WorkspaceID, x.Resume
|
||||
default:
|
||||
return Result{}, errors.New("unsupported workspace command")
|
||||
return env, "", errors.New("unsupported workspace command")
|
||||
}
|
||||
payload, e := json.Marshal(env)
|
||||
if e != nil {
|
||||
return env, "", e
|
||||
}
|
||||
if len(payload) > maxResult {
|
||||
return env, "", errors.New("request exceeds limit")
|
||||
}
|
||||
return env, string(payload), nil
|
||||
}
|
||||
|
||||
var commonIngress = map[string]bool{"schemaVersion": true, "operation": true, "workspaceId": true}
|
||||
|
||||
func validateIngress(payload []byte, expected inputEnvelope) error {
|
||||
var raw map[string]json.RawMessage
|
||||
d := json.NewDecoder(bytes.NewReader(payload))
|
||||
d.UseNumber()
|
||||
if d.Decode(&raw) != nil {
|
||||
return errors.New("invalid workspace request")
|
||||
}
|
||||
var extra any
|
||||
if d.Decode(&extra) != io.EOF {
|
||||
return errors.New("invalid workspace request")
|
||||
}
|
||||
var supplied inputEnvelope
|
||||
dec := json.NewDecoder(bytes.NewReader(payload))
|
||||
dec.DisallowUnknownFields()
|
||||
if dec.Decode(&supplied) != nil || supplied.SchemaVersion != 1 || supplied.Operation != expected.Operation || supplied.WorkspaceID != expected.WorkspaceID {
|
||||
return errors.New("invalid workspace request")
|
||||
}
|
||||
if !reflect.DeepEqual(supplied, expected) {
|
||||
return errors.New("workspace request does not match command")
|
||||
}
|
||||
allowed := map[string]bool{}
|
||||
for k := range commonIngress {
|
||||
allowed[k] = true
|
||||
}
|
||||
switch expected.Operation {
|
||||
case "dwh", "run":
|
||||
if expected.Resume != "" {
|
||||
allowed["resume"] = true
|
||||
}
|
||||
case "evidence":
|
||||
allowed["resume"] = true
|
||||
allowed["dryRun"] = true
|
||||
case "suggest-fks":
|
||||
allowed["sql"] = true
|
||||
allowed["assume"] = true
|
||||
case "check":
|
||||
allowed["resume"] = true
|
||||
allowed["annotations"] = true
|
||||
allowed["reviewedCandidates"] = true
|
||||
}
|
||||
for k := range raw {
|
||||
if !allowed[k] {
|
||||
return errors.New("invalid workspace request")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func Run(ctx context.Context, installation config.Installation, runner compose.Runner, command Command, stdin io.Reader) (Result, error) {
|
||||
env, generated, e := makeInput(command)
|
||||
if e != nil {
|
||||
return Result{}, e
|
||||
}
|
||||
payload := []byte(generated)
|
||||
if stdin != nil {
|
||||
payload, e = io.ReadAll(io.LimitReader(stdin, 1<<20+1))
|
||||
if e != nil {
|
||||
return Result{}, e
|
||||
}
|
||||
if len(payload) > 1<<20 {
|
||||
payload, e = io.ReadAll(io.LimitReader(stdin, maxResult+1))
|
||||
if e != nil || len(payload) > maxResult {
|
||||
return Result{}, errors.New("request exceeds limit")
|
||||
}
|
||||
var supplied inputEnvelope
|
||||
d := json.NewDecoder(bytes.NewReader(payload))
|
||||
d.DisallowUnknownFields()
|
||||
if e = d.Decode(&supplied); e != nil || supplied.SchemaVersion != 1 || supplied.Operation != env.Operation || supplied.WorkspaceID != env.WorkspaceID {
|
||||
return Result{}, errors.New("invalid workspace request")
|
||||
if e = validateIngress(payload, env); e != nil {
|
||||
return Result{}, e
|
||||
}
|
||||
}
|
||||
|
||||
args := installation.ComposeArgs("run", "--rm", "--no-deps", "workspace-maintenance", "--operation", operationName(command), "--workspace", env.WorkspaceID)
|
||||
cr, runErr := runner.RunBounded(ctx, args, bytes.NewReader(payload), compose.CaptureLimits{StdoutBytes: 1 << 20, StderrBytes: 64 << 10})
|
||||
if runErr != nil && (errors.Is(runErr, compose.ErrOutputLimit) || cr.Stdout == "") {
|
||||
args := installation.ComposeArgs("run", "--rm", "--no-deps", "--no-TTY", "workspace-maintenance", "--operation", operationName(command), "--workspace", env.WorkspaceID)
|
||||
cr, runErr := runner.RunBounded(ctx, args, bytes.NewReader(payload), compose.CaptureLimits{StdoutBytes: maxResult, StderrBytes: 64 << 10})
|
||||
if errors.Is(runErr, compose.ErrOutputLimit) {
|
||||
return Result{}, runErr
|
||||
}
|
||||
if len(cr.Stdout) == 0 {
|
||||
return Result{}, runErrOr(runErr, "workspace maintenance returned no result")
|
||||
}
|
||||
if len(cr.Stdout) > maxResult {
|
||||
return Result{}, errors.New("invalid workspace result")
|
||||
}
|
||||
var result Result
|
||||
dec := json.NewDecoder(strings.NewReader(cr.Stdout))
|
||||
dec.DisallowUnknownFields()
|
||||
if e := dec.Decode(&result); e != nil {
|
||||
var raw map[string]json.RawMessage
|
||||
d := json.NewDecoder(strings.NewReader(cr.Stdout))
|
||||
if d.Decode(&raw) != nil {
|
||||
return Result{}, errors.New("invalid workspace result")
|
||||
}
|
||||
var extra any
|
||||
if e := dec.Decode(&extra); e != io.EOF {
|
||||
if d.Decode(&extra) != io.EOF {
|
||||
return Result{}, errors.New("invalid workspace result")
|
||||
}
|
||||
if result.SchemaVersion != 1 || result.WorkspaceID != env.WorkspaceID || result.Operation != operationName(command) || !validStatus(result.Status) || !validCode(result.Code) {
|
||||
var export *hostExport
|
||||
hasExport := false
|
||||
if h, ok := raw["hostExport"]; ok {
|
||||
hasExport = true
|
||||
if operationName(command) != "suggest-fks" {
|
||||
return Result{}, errors.New("invalid workspace result")
|
||||
}
|
||||
if json.Unmarshal(h, &export) != nil {
|
||||
return Result{}, errors.New("invalid host export")
|
||||
}
|
||||
delete(raw, "hostExport")
|
||||
}
|
||||
b, _ := json.Marshal(raw)
|
||||
var result Result
|
||||
dec := json.NewDecoder(bytes.NewReader(b))
|
||||
dec.DisallowUnknownFields()
|
||||
if dec.Decode(&result) != nil {
|
||||
return Result{}, errors.New("invalid workspace result")
|
||||
}
|
||||
if outputPath != "" {
|
||||
if len(cr.Stdout) > maxCandidate {
|
||||
return Result{}, errors.New("candidate export exceeds limit")
|
||||
}
|
||||
if err := safeio.WriteCanonicalExclusive(outputPath, []byte(cr.Stdout), 0o600); err != nil {
|
||||
return Result{}, errors.New("unsafe output file")
|
||||
if e = validateResult(result, env.WorkspaceID, operationName(command)); e != nil {
|
||||
return Result{}, e
|
||||
}
|
||||
if runErr != nil {
|
||||
if result.Status == "blocked" && cr.ExitCode == 3 {
|
||||
} else if result.Status == "failed" && cr.ExitCode == 1 {
|
||||
} else {
|
||||
return Result{}, runErr
|
||||
}
|
||||
}
|
||||
|
||||
if hasExport {
|
||||
if export == nil {
|
||||
return Result{}, errors.New("invalid candidate export")
|
||||
}
|
||||
if !candidateBoundToResult(*export, result) {
|
||||
return Result{}, errors.New("invalid candidate identity")
|
||||
}
|
||||
if err := publishCandidate(export, result, outPath(command)); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
} else if outPath(command) != "" {
|
||||
return Result{}, errors.New("candidate export is required")
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
func DigestBytes(b []byte) string { s := sha256.Sum256(b); return "sha256:" + hex.EncodeToString(s[:]) }
|
||||
func runErrOr(e error, msg string) error {
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
return errors.New(msg)
|
||||
}
|
||||
func outPath(c Command) string {
|
||||
if x, ok := c.(SuggestFksRequest); ok {
|
||||
return x.Output
|
||||
}
|
||||
return ""
|
||||
}
|
||||
func candidateBoundToResult(x hostExport, result Result) bool {
|
||||
if result.RunID == "" {
|
||||
return false
|
||||
}
|
||||
for _, artifact := range result.ArtifactIdentities {
|
||||
if artifact.Kind == "fk-candidates" && artifact.Digest == x.SHA256 {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func publishCandidate(x *hostExport, result Result, path string) error {
|
||||
if path == "" {
|
||||
return nil
|
||||
}
|
||||
if x.MediaType != "application/yaml" && x.MediaType != "text/yaml" {
|
||||
return errors.New("invalid candidate export")
|
||||
}
|
||||
if !digestPattern.MatchString(x.SHA256) {
|
||||
return errors.New("invalid candidate export")
|
||||
}
|
||||
b, e := base64.StdEncoding.DecodeString(x.ContentBase64)
|
||||
if e != nil || len(b) > maxCandidate || !utf8.Valid(b) {
|
||||
return errors.New("invalid candidate export")
|
||||
}
|
||||
if DigestBytes(b) != x.SHA256 {
|
||||
return errors.New("invalid candidate export")
|
||||
}
|
||||
var doc any
|
||||
if yaml.Unmarshal(b, &doc) != nil {
|
||||
return errors.New("invalid candidate export")
|
||||
}
|
||||
if result.RunID == "" || !runIDPattern.MatchString(result.RunID) {
|
||||
return errors.New("invalid candidate identity")
|
||||
}
|
||||
if e = safeio.WriteCanonicalExclusive(path, b, 0o600); e != nil {
|
||||
return errors.New("unsafe output file")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func validateResult(r Result, workspace, operation string) error {
|
||||
if r.SchemaVersion != 1 || r.WorkspaceID != workspace || r.Operation != operation || !validStatus(r.Status) || !validCode(r.Code) || !revisionPattern.MatchString(r.WorkspaceRevision) || !revisionPattern.MatchString(r.DescriptorBlob) {
|
||||
return errors.New("invalid workspace result")
|
||||
}
|
||||
if r.RunID != "" && !runIDPattern.MatchString(r.RunID) {
|
||||
return errors.New("invalid workspace result")
|
||||
}
|
||||
for k, v := range r.ChildRuns {
|
||||
if k == "" || !runIDPattern.MatchString(v) {
|
||||
return errors.New("invalid workspace result")
|
||||
}
|
||||
}
|
||||
for _, a := range r.ArtifactIdentities {
|
||||
if a.Kind == "" || !digestPattern.MatchString(a.Digest) {
|
||||
return errors.New("invalid workspace result")
|
||||
}
|
||||
}
|
||||
if r.CompletedStages == nil {
|
||||
return errors.New("invalid workspace result")
|
||||
}
|
||||
if r.Status == "blocked" && (r.Code != "manual_review_required" && r.Code != "evidence_materialization_required" && r.Code != "preprocessing_conflict" && r.Code != "preprocessing_resume_mismatch" && r.Code != CodeRegistryBootstrapRecoveryConflict) {
|
||||
return errors.New("invalid workspace result")
|
||||
}
|
||||
if r.Status != "blocked" && r.Code == CodeRegistryBootstrapRecoveryConflict {
|
||||
return errors.New("invalid workspace result")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func DigestBytes(b []byte) string { s := sha256.Sum256(b); return "sha256:" + hex.EncodeToString(s[:]) }
|
||||
func validStatus(v string) bool {
|
||||
switch v {
|
||||
case "succeeded", "unchanged", "dry_run", "blocked", "failed":
|
||||
|
||||
@@ -1,8 +1,16 @@
|
||||
package workspaceops
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"context"
|
||||
"github.com/aritmolab/thothii/tools/thothctl/internal/compose"
|
||||
"github.com/aritmolab/thothii/tools/thothctl/internal/config"
|
||||
)
|
||||
|
||||
func TestParseWorkspaceCommands(t *testing.T) {
|
||||
@@ -48,3 +56,79 @@ func TestParseWorkspaceRejectsUnsafeOrAmbiguousOptions(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseWorkspaceRejectsNonContractHierarchyAndAssumeShell(t *testing.T) {
|
||||
for _, argv := range [][]string{
|
||||
{"workspace", "schema", "dwh", "--workspace", "psd"},
|
||||
{"workspace", "preprocess", "check", "--workspace", "psd"},
|
||||
{"workspace", "suggest-fks", "--workspace", "psd"},
|
||||
{"workspace", "inspect", "--workspace", "psd", "--bootstrap-run-id", strings.Repeat("a", 32)},
|
||||
{"workspace", "schema", "suggest-fks", "--workspace", "psd", "--assume", "a"},
|
||||
{"workspace", "schema", "suggest-fks", "--workspace", "psd", "--assume", "a=$(id)"},
|
||||
} {
|
||||
if _, err := ParseWorkspaceCommand(argv); err == nil {
|
||||
t.Errorf("accepted non-contract argv %v", argv)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunUsesBase64BasenameIngressAndNoTTY(t *testing.T) {
|
||||
root, err := filepath.EvalSymlinks(t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sqlPath := filepath.Join(root, "schema.sql")
|
||||
if err := os.WriteFile(sqlPath, []byte("select 1"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out := filepath.Join(root, "candidate.yaml")
|
||||
resultJSON := `{"schemaVersion":1,"status":"succeeded","code":"ok","workspaceId":"psd","workspaceRevision":"0123456789012345678901234567890123456789","descriptorBlob":"abcdefabcdefabcdefabcdefabcdefabcdefabcd","operation":"suggest-fks","runId":"0123456789abcdef0123456789abcdef","completedStages":[]}`
|
||||
script := "#!/bin/sh\ncat >/dev/null\nprintf '%s' '" + resultJSON + "'\n"
|
||||
fake := filepath.Join(root, "docker")
|
||||
if err := os.WriteFile(fake, []byte(script), 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cmd, err := ParseWorkspaceCommand([]string{"workspace", "schema", "suggest-fks", "--workspace", "psd", "--from-sql", sqlPath})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := Run(context.Background(), config.Installation{ProjectDirectory: root, EnvFile: filepath.Join(root, "env")}, compose.NewRunner(fake), cmd, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got.Code != "ok" {
|
||||
t.Fatalf("result = %#v", got)
|
||||
}
|
||||
_ = out
|
||||
}
|
||||
|
||||
func TestRunPublishesOnlyVerifiedCandidateExport(t *testing.T) {
|
||||
root, err := filepath.EvalSymlinks(t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
candidate := []byte("candidates: []\n")
|
||||
digest := DigestBytes(candidate)
|
||||
encoded := base64.StdEncoding.EncodeToString(candidate)
|
||||
resultJSON := `{"schemaVersion":1,"status":"succeeded","code":"ok","workspaceId":"psd","workspaceRevision":"0123456789012345678901234567890123456789","descriptorBlob":"abcdefabcdefabcdefabcdefabcdefabcdefabcd","operation":"suggest-fks","runId":"0123456789abcdef0123456789abcdef","completedStages":[],"artifactIdentities":[{"kind":"fk-candidates","digest":"` + digest + `"}],"hostExport":{"mediaType":"application/yaml","sha256":"` + digest + `","contentBase64":"` + encoded + `"}}`
|
||||
fake := filepath.Join(root, "docker")
|
||||
if err := os.WriteFile(fake, []byte("#!/bin/sh\ncat >/dev/null\nprintf '%s' '"+resultJSON+"'\n"), 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
path := filepath.Join(root, "out.yaml")
|
||||
cmd, err := ParseWorkspaceCommand([]string{"workspace", "schema", "suggest-fks", "--workspace", "psd", "--output", path})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := Run(context.Background(), config.Installation{ProjectDirectory: root, EnvFile: filepath.Join(root, "env")}, compose.NewRunner(fake), cmd, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got.RunID == "" {
|
||||
t.Fatal("missing run identity")
|
||||
}
|
||||
b, err := os.ReadFile(path)
|
||||
if err != nil || string(b) != string(candidate) {
|
||||
t.Fatalf("candidate = %q, %v", b, err)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user