fix: complete P2 host workspace contract
This commit is contained in:
@@ -1,19 +1,21 @@
|
|||||||
# Workspace preprocessing CLI contract
|
# Workspace preprocessing CLI contract
|
||||||
|
|
||||||
`thothctl workspace` is a closed native host interface. It accepts only the commands and options listed below; unknown options, passthrough separators, shell fragments, and bootstrap run selectors are rejected before Compose is invoked.
|
`thothctl workspace` is a closed native host interface. Only the seven paths below are representable; there is no passthrough, shell, arbitrary child argv, or host-selected bootstrap run ID.
|
||||||
|
|
||||||
```text
|
```text
|
||||||
thothctl --installation /absolute/thothii-installation.yaml workspace inspect --workspace ID [--json]
|
workspace inspect --workspace ID [--json]
|
||||||
thothctl ... workspace preprocess dwh --workspace ID [--resume RUN] [--json]
|
workspace preprocess dwh --workspace ID [--resume RUN] [--json]
|
||||||
thothctl ... workspace schema suggest-fks --workspace ID [--from-sql FILE]... [--assume COLUMN=TABLE]... [--output FILE] [--json]
|
workspace schema suggest-fks --workspace ID [--from-sql FILE]... [--assume COLUMN=TABLE]... [--output FILE] [--json]
|
||||||
thothctl ... workspace schema check --workspace ID --resume RUN [--annotations FILE --reviewed-candidates sha256:HEX] [--json]
|
workspace schema check --workspace ID --resume RUN [--annotations FILE --reviewed-candidates sha256:HEX] [--json]
|
||||||
thothctl ... workspace index-schema --workspace ID [--json]
|
workspace index-schema --workspace ID [--json]
|
||||||
thothctl ... workspace preprocess evidence --workspace ID [--dry-run] [--resume RUN] [--json]
|
workspace preprocess evidence --workspace ID [--dry-run] [--resume RUN] [--json]
|
||||||
thothctl ... workspace preprocess run --workspace ID [--resume RUN] [--json]
|
workspace preprocess run --workspace ID [--resume RUN] [--json]
|
||||||
```
|
```
|
||||||
|
|
||||||
Workspace IDs are lowercase `[a-z][a-z0-9-]{2,62}` and outer run IDs are exactly 32 lowercase hexadecimal characters. SQL ingress is limited to 32 regular, canonical, non-symlink files of at most 1 MiB each and 16 MiB total. Annotation ingress is UTF-8 and at most 16 MiB. Inputs are read without following links and are never mounted as host directories. Suggestion candidate output is bounded to 700 KiB.
|
IDs are lowercase `[a-z][a-z0-9-]{2,62}` and outer run IDs are exactly 32 lowercase hexadecimal characters. `--from-sql` accepts at most 32 canonical regular non-symlink files, each at most 1 MiB and 16 MiB total. `--assume` accepts at most 256 values of 256 bytes, each matching `column=table`. Annotations are one canonical UTF-8 file at most 16 MiB. Existing output, links, hardlinks, directories, replacement races, and reparse points are refused with a generic unsafe-file error.
|
||||||
|
|
||||||
`workspace inspect` has no resume or bootstrap option. Bootstrap recovery is automatic and installation-scoped; the host never chooses a run ID. `schema check` requires an explicit outer run ID and requires `--annotations` and `--reviewed-candidates` together.
|
The bounded schema-v1 stdin request contains only operation-appropriate fields. SQL and annotations are represented by a logical basename, base64 bytes, and a declared `sha256:<hex>` digest; no host path or raw content enters the request. The complete request is at most 1 MiB, has exactly one JSON value, and cannot replace command-derived fields. Candidate responses may carry the internal `hostExport` (`mediaType`, `sha256`, `contentBase64`) only for `suggest-fks`; it is verified (UTF-8 YAML, digest, and 700 KiB maximum), removed from the public result, and written exclusively only after result/run/identity validation.
|
||||||
|
|
||||||
The one-shot operation emits one schema-v1 JSON envelope. Stdout is capped at 1 MiB and sanitized stderr at 64 KiB. Exit status is 0 for success/unchanged/dry-run, 3 for an expected blocked/manual checkpoint, 2 for command or unsafe-file errors, and 1 for operational failures.
|
The public result has schema version 1 and only these fields: `status`, `code`, workspace/revision/descriptor/operation identities, optional run and child run IDs, completed stages, counts, artifact identities, and warnings. Revisions/descriptors are 40-hex; run IDs are 32-hex; artifact digests are `sha256:<hex>`. A nonzero child exit is accepted only for the matching blocked (3) or failed (1) result. Stdout is capped at 1 MiB and stderr at 64 KiB. Compose is invoked only as `compose run --rm --no-deps --no-TTY workspace-maintenance ...`; output never includes child stderr or secrets.
|
||||||
|
|
||||||
|
Exit 0 means succeeded, unchanged, or dry-run; exit 3 means an expected operator checkpoint/block; exit 2 means grammar or unsafe local-file failure; exit 1 means operational failure. Human mode prints only allowlisted identity/status fields. For `registry_bootstrap_recovery_conflict` it prints exactly: `Bootstrap recovery is ambiguous or corrupt; inspect the installation registry jobs.`
|
||||||
|
|||||||
@@ -90,11 +90,48 @@ func run(ctx context.Context, args []string, stdout, stderr io.Writer) int {
|
|||||||
runner := compose.NewRunner("")
|
runner := compose.NewRunner("")
|
||||||
if command == "workspace" {
|
if command == "workspace" {
|
||||||
workspaceCommand, parseErr := workspaceops.ParseWorkspaceCommand(append([]string{"workspace"}, commandArgs...))
|
workspaceCommand, parseErr := workspaceops.ParseWorkspaceCommand(append([]string{"workspace"}, commandArgs...))
|
||||||
if parseErr != nil { return commandUsageError(stderr, parseErr.Error()) }
|
if parseErr != nil {
|
||||||
|
return commandUsageError(stderr, parseErr.Error())
|
||||||
|
}
|
||||||
result, operationErr := workspaceops.Run(ctx, installation, runner, workspaceCommand, nil)
|
result, operationErr := workspaceops.Run(ctx, installation, runner, workspaceCommand, nil)
|
||||||
if operationErr != nil { if errors.Is(operationErr, compose.ErrOutputLimit) || strings.Contains(operationErr.Error(), "unsafe") || strings.Contains(operationErr.Error(), "exceeds") { return commandUsageError(stderr, operationErr.Error()) }; fmt.Fprintf(stderr, "thothctl: %s\n", output.Sanitize(operationErr.Error(), secretValues)); return 1 }
|
if operationErr != nil {
|
||||||
if err := json.NewEncoder(stdout).Encode(result); err != nil { return 1 }
|
if errors.Is(operationErr, compose.ErrOutputLimit) || strings.Contains(operationErr.Error(), "unsafe") || strings.Contains(operationErr.Error(), "exceeds") || strings.Contains(operationErr.Error(), "invalid workspace") {
|
||||||
if result.Status == "blocked" { return 3 }; if result.Status == "failed" { return 1 }; return 0
|
return commandUsageError(stderr, operationErr.Error())
|
||||||
|
}
|
||||||
|
fmt.Fprintf(stderr, "thothctl: %s\n", output.Sanitize(operationErr.Error(), secretValues))
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
jsonMode := true
|
||||||
|
switch c := workspaceCommand.(type) {
|
||||||
|
case workspaceops.InspectCommand:
|
||||||
|
jsonMode = c.JSON
|
||||||
|
case workspaceops.DwhRequest:
|
||||||
|
jsonMode = c.JSON
|
||||||
|
case workspaceops.SuggestFksRequest:
|
||||||
|
jsonMode = c.JSON
|
||||||
|
case workspaceops.CheckSchemaRequest:
|
||||||
|
jsonMode = c.JSON
|
||||||
|
case workspaceops.IndexSchemaRequest:
|
||||||
|
jsonMode = c.JSON
|
||||||
|
case workspaceops.EvidenceRequest:
|
||||||
|
jsonMode = c.JSON
|
||||||
|
case workspaceops.RunRequest:
|
||||||
|
jsonMode = c.JSON
|
||||||
|
}
|
||||||
|
if jsonMode {
|
||||||
|
if err := json.NewEncoder(stdout).Encode(result); err != nil {
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
renderWorkspaceHuman(stdout, result)
|
||||||
|
}
|
||||||
|
if result.Status == "blocked" {
|
||||||
|
return 3
|
||||||
|
}
|
||||||
|
if result.Status == "failed" {
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
return 0
|
||||||
}
|
}
|
||||||
var result compose.Result
|
var result compose.Result
|
||||||
switch command {
|
switch command {
|
||||||
@@ -173,6 +210,20 @@ func run(ctx context.Context, args []string, stdout, stderr io.Writer) int {
|
|||||||
return writeResult(result, err, secretValues, stdout, stderr)
|
return writeResult(result, err, secretValues, stdout, stderr)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func renderWorkspaceHuman(w io.Writer, result workspaceops.Result) {
|
||||||
|
if result.Code == workspaceops.CodeRegistryBootstrapRecoveryConflict {
|
||||||
|
fmt.Fprintln(w, "Bootstrap recovery is ambiguous or corrupt; inspect the installation registry jobs.")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
fmt.Fprintf(w, "Workspace %s: %s (%s)\n", result.WorkspaceID, result.Status, result.Code)
|
||||||
|
if result.RunID != "" {
|
||||||
|
fmt.Fprintf(w, "Run: %s\n", result.RunID)
|
||||||
|
}
|
||||||
|
if len(result.CompletedStages) > 0 {
|
||||||
|
fmt.Fprintf(w, "Completed stages: %s\n", strings.Join(result.CompletedStages, ", "))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func writeRemovalTargets(outputWriter io.Writer, project string, targets []serverops.Container) {
|
func writeRemovalTargets(outputWriter io.Writer, project string, targets []serverops.Container) {
|
||||||
fmt.Fprintf(outputWriter, "Removal targets for installation project %s:\n", project)
|
fmt.Fprintf(outputWriter, "Removal targets for installation project %s:\n", project)
|
||||||
if len(targets) == 0 {
|
if len(targets) == 0 {
|
||||||
|
|||||||
@@ -15,6 +15,7 @@ import (
|
|||||||
"github.com/aritmolab/thothii/tools/thothctl/internal/config"
|
"github.com/aritmolab/thothii/tools/thothctl/internal/config"
|
||||||
"github.com/aritmolab/thothii/tools/thothctl/internal/pi"
|
"github.com/aritmolab/thothii/tools/thothctl/internal/pi"
|
||||||
"github.com/aritmolab/thothii/tools/thothctl/internal/testsupport"
|
"github.com/aritmolab/thothii/tools/thothctl/internal/testsupport"
|
||||||
|
"github.com/aritmolab/thothii/tools/thothctl/internal/workspaceops"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestInstallationRunnerMapsProfileToSessionInventoryScope(t *testing.T) {
|
func TestInstallationRunnerMapsProfileToSessionInventoryScope(t *testing.T) {
|
||||||
@@ -846,3 +847,11 @@ func assertInvocationContains(t *testing.T, invocations [][]string, want ...stri
|
|||||||
}
|
}
|
||||||
t.Fatalf("invocations = %#v, want %#v", invocations, want)
|
t.Fatalf("invocations = %#v, want %#v", invocations, want)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRenderWorkspaceHumanHidesRecoveryIdentity(t *testing.T) {
|
||||||
|
var out bytes.Buffer
|
||||||
|
renderWorkspaceHuman(&out, workspaceops.Result{WorkspaceID: "psd", Status: "blocked", Code: workspaceops.CodeRegistryBootstrapRecoveryConflict, RunID: "0123456789abcdef0123456789abcdef"})
|
||||||
|
if out.String() != "Bootstrap recovery is ambiguous or corrupt; inspect the installation registry jobs.\n" {
|
||||||
|
t.Fatalf("human output = %q", out.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -1,5 +1,18 @@
|
|||||||
//go:build !windows
|
//go:build !windows
|
||||||
|
|
||||||
package compose
|
package compose
|
||||||
import ("os/exec"; "syscall")
|
|
||||||
func configureOwnedProcess(c *exec.Cmd) { c.SysProcAttr=&syscall.SysProcAttr{Setpgid:true} }
|
import (
|
||||||
func terminateOwnedProcess(c *exec.Cmd) { if c.Process!=nil { _=syscall.Kill(-c.Process.Pid, syscall.SIGKILL); _=c.Process.Kill() } }
|
"os/exec"
|
||||||
|
"syscall"
|
||||||
|
)
|
||||||
|
|
||||||
|
func configureOwnedProcess(c *exec.Cmd) { c.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} }
|
||||||
|
func registerOwnedProcess(c *exec.Cmd) error { return nil }
|
||||||
|
func releaseOwnedProcess(c *exec.Cmd) {}
|
||||||
|
func terminateOwnedProcess(c *exec.Cmd) {
|
||||||
|
if c.Process != nil {
|
||||||
|
_ = syscall.Kill(-c.Process.Pid, syscall.SIGKILL)
|
||||||
|
_ = c.Process.Kill()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -1,5 +1,51 @@
|
|||||||
//go:build windows
|
//go:build windows
|
||||||
|
|
||||||
package compose
|
package compose
|
||||||
import ("os/exec"; "syscall")
|
|
||||||
func configureOwnedProcess(c *exec.Cmd) { c.SysProcAttr=&syscall.SysProcAttr{} }
|
import (
|
||||||
func terminateOwnedProcess(c *exec.Cmd) { if c.Process!=nil { _=c.Process.Kill() } }
|
"golang.org/x/sys/windows"
|
||||||
|
"os/exec"
|
||||||
|
"sync"
|
||||||
|
)
|
||||||
|
|
||||||
|
var ownedJobs = struct {
|
||||||
|
sync.Mutex
|
||||||
|
m map[*exec.Cmd]windows.Handle
|
||||||
|
}{m: make(map[*exec.Cmd]windows.Handle)}
|
||||||
|
|
||||||
|
func configureOwnedProcess(c *exec.Cmd) {
|
||||||
|
c.SysProcAttr = &windows.SysProcAttr{CreationFlags: windows.CREATE_NEW_PROCESS_GROUP}
|
||||||
|
}
|
||||||
|
func registerOwnedProcess(c *exec.Cmd) error {
|
||||||
|
h, err := windows.CreateJobObject(nil, nil)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err = windows.AssignProcessToJobObject(h, windows.Handle(c.Process.Pid)); err != nil {
|
||||||
|
windows.CloseHandle(h)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
ownedJobs.Lock()
|
||||||
|
ownedJobs.m[c] = h
|
||||||
|
ownedJobs.Unlock()
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
func releaseOwnedProcess(c *exec.Cmd) {
|
||||||
|
ownedJobs.Lock()
|
||||||
|
h := ownedJobs.m[c]
|
||||||
|
delete(ownedJobs.m, c)
|
||||||
|
ownedJobs.Unlock()
|
||||||
|
if h != 0 {
|
||||||
|
windows.CloseHandle(h)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
func terminateOwnedProcess(c *exec.Cmd) {
|
||||||
|
ownedJobs.Lock()
|
||||||
|
h := ownedJobs.m[c]
|
||||||
|
ownedJobs.Unlock()
|
||||||
|
if h != 0 {
|
||||||
|
_ = windows.TerminateJobObject(h, 1)
|
||||||
|
} else if c.Process != nil {
|
||||||
|
_ = c.Process.Kill()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -7,111 +7,154 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"sync"
|
|
||||||
"os"
|
"os"
|
||||||
"os/exec"
|
"os/exec"
|
||||||
|
"sync"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Result is the captured output and process exit code for one Docker invocation.
|
|
||||||
var ErrOutputLimit = errors.New("compose output limit exceeded")
|
var ErrOutputLimit = errors.New("compose output limit exceeded")
|
||||||
|
|
||||||
// CaptureLimits bounds each stream independently. Overflow is never retained.
|
type CaptureLimits struct {
|
||||||
type CaptureLimits struct { StdoutBytes int64; StderrBytes int64 }
|
StdoutBytes int64
|
||||||
|
StderrBytes int64
|
||||||
|
}
|
||||||
type Result struct {
|
type Result struct {
|
||||||
Stdout string
|
Stdout string
|
||||||
Stderr string
|
Stderr string
|
||||||
ExitCode int
|
ExitCode int
|
||||||
}
|
}
|
||||||
|
type Runner struct{ binary string }
|
||||||
|
|
||||||
// Runner executes the Docker CLI. It never invokes a shell.
|
|
||||||
type Runner struct {
|
|
||||||
binary string
|
|
||||||
}
|
|
||||||
|
|
||||||
// NewRunner returns a runner for binary. An empty binary selects docker from PATH.
|
|
||||||
func NewRunner(binary string) Runner {
|
func NewRunner(binary string) Runner {
|
||||||
if binary == "" {
|
if binary == "" {
|
||||||
binary = "docker"
|
binary = "docker"
|
||||||
}
|
}
|
||||||
return Runner{binary: binary}
|
return Runner{binary: binary}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Run invokes Docker with the supplied argument array and optional standard input.
|
|
||||||
func (r Runner) Run(ctx context.Context, args []string, stdin io.Reader) (Result, error) {
|
func (r Runner) Run(ctx context.Context, args []string, stdin io.Reader) (Result, error) {
|
||||||
command := exec.CommandContext(ctx, r.binary, args...)
|
c := exec.CommandContext(ctx, r.binary, args...)
|
||||||
command.Stdin = stdin
|
c.Stdin = stdin
|
||||||
var stdout, stderr bytes.Buffer
|
var out, er bytes.Buffer
|
||||||
command.Stdout = &stdout
|
c.Stdout = &out
|
||||||
command.Stderr = &stderr
|
c.Stderr = &er
|
||||||
err := command.Run()
|
err := c.Run()
|
||||||
result := Result{Stdout: stdout.String(), Stderr: stderr.String()}
|
res := Result{Stdout: out.String(), Stderr: er.String()}
|
||||||
if err == nil {
|
if err == nil {
|
||||||
return result, nil
|
return res, nil
|
||||||
}
|
}
|
||||||
var exitError *exec.ExitError
|
var x *exec.ExitError
|
||||||
if errors.As(err, &exitError) {
|
if errors.As(err, &x) {
|
||||||
result.ExitCode = exitError.ExitCode()
|
res.ExitCode = x.ExitCode()
|
||||||
return result, err
|
return res, err
|
||||||
}
|
}
|
||||||
if errors.Is(err, exec.ErrNotFound) || errors.Is(err, os.ErrNotExist) {
|
if errors.Is(err, exec.ErrNotFound) || errors.Is(err, os.ErrNotExist) {
|
||||||
result.ExitCode = 127
|
res.ExitCode = 127
|
||||||
return result, fmt.Errorf("%w: %w", exec.ErrNotFound, err)
|
return res, fmt.Errorf("%w: %w", exec.ErrNotFound, err)
|
||||||
}
|
}
|
||||||
return result, err
|
return res, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// RunBounded streams each pipe into a preallocated fixed-capacity collector. It owns the
|
||||||
// RunBounded executes one owned process while retaining at most the configured bytes per stream.
|
// process group and tears it down on the first overflow or cancellation.
|
||||||
// The complete process group is terminated as soon as either stream exceeds its limit.
|
|
||||||
func (r Runner) RunBounded(ctx context.Context, args []string, stdin io.Reader, limits CaptureLimits) (Result, error) {
|
func (r Runner) RunBounded(ctx context.Context, args []string, stdin io.Reader, limits CaptureLimits) (Result, error) {
|
||||||
if limits.StdoutBytes < 0 || limits.StderrBytes < 0 { return Result{}, ErrOutputLimit }
|
if limits.StdoutBytes < 0 || limits.StderrBytes < 0 {
|
||||||
command := exec.Command(r.binary, args...)
|
return Result{}, ErrOutputLimit
|
||||||
command.Stdin = stdin
|
}
|
||||||
stdoutPipe, err := command.StdoutPipe(); if err != nil { return Result{}, err }
|
c := exec.Command(r.binary, args...)
|
||||||
stderrPipe, err := command.StderrPipe(); if err != nil { return Result{}, err }
|
c.Stdin = stdin
|
||||||
configureOwnedProcess(command)
|
op, err := c.StdoutPipe()
|
||||||
if err := command.Start(); err != nil { return Result{ExitCode: 127}, err }
|
if err != nil {
|
||||||
type stream struct { b []byte; overflow bool }
|
return Result{}, err
|
||||||
var out, er stream
|
}
|
||||||
|
ep, err := c.StderrPipe()
|
||||||
|
if err != nil {
|
||||||
|
return Result{}, err
|
||||||
|
}
|
||||||
|
configureOwnedProcess(c)
|
||||||
|
if err = c.Start(); err != nil {
|
||||||
|
return Result{ExitCode: 127}, err
|
||||||
|
}
|
||||||
|
if err = registerOwnedProcess(c); err != nil {
|
||||||
|
terminateOwnedProcess(c)
|
||||||
|
_ = c.Wait()
|
||||||
|
return Result{}, err
|
||||||
|
}
|
||||||
|
defer releaseOwnedProcess(c)
|
||||||
|
type collector struct {
|
||||||
|
buf []byte
|
||||||
|
n int64
|
||||||
|
overflow bool
|
||||||
|
}
|
||||||
|
out := collector{buf: make([]byte, limits.StdoutBytes)}
|
||||||
|
er := collector{buf: make([]byte, limits.StderrBytes)}
|
||||||
overflow := make(chan struct{}, 1)
|
overflow := make(chan struct{}, 1)
|
||||||
var wg sync.WaitGroup; wg.Add(2)
|
var wg sync.WaitGroup
|
||||||
read := func(rd io.Reader, max int64, target *stream) {
|
wg.Add(2)
|
||||||
|
read := func(rd io.Reader, max int64, col *collector) {
|
||||||
defer wg.Done()
|
defer wg.Done()
|
||||||
buf := make([]byte, 32*1024)
|
chunk := make([]byte, 32*1024)
|
||||||
for {
|
for {
|
||||||
n, readErr := rd.Read(buf)
|
n, e := rd.Read(chunk)
|
||||||
if n > 0 {
|
if n > 0 {
|
||||||
remain := max - int64(len(target.b))
|
remain := max - col.n
|
||||||
if remain > 0 { take := int64(n); if take > remain { take = remain }; target.b = append(target.b, buf[:take]...) }
|
if remain > 0 {
|
||||||
|
take := int64(n)
|
||||||
|
if take > remain {
|
||||||
|
take = remain
|
||||||
|
}
|
||||||
|
copy(col.buf[col.n:col.n+take], chunk[:take])
|
||||||
|
col.n += take
|
||||||
|
}
|
||||||
if int64(n) > remain {
|
if int64(n) > remain {
|
||||||
target.overflow = true
|
col.overflow = true
|
||||||
select { case overflow <- struct{}{}: default: }
|
select {
|
||||||
|
case overflow <- struct{}{}:
|
||||||
|
default:
|
||||||
|
}
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if readErr != nil { return }
|
if e != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
go read(stdoutPipe, limits.StdoutBytes, &out); go read(stderrPipe, limits.StderrBytes, &er)
|
go read(op, limits.StdoutBytes, &out)
|
||||||
finished := make(chan error, 1); go func() { finished <- command.Wait() }()
|
go read(ep, limits.StderrBytes, &er)
|
||||||
|
waited := make(chan error, 1)
|
||||||
|
go func() { waited <- c.Wait() }()
|
||||||
var waitErr error
|
var waitErr error
|
||||||
|
var cancelled error
|
||||||
select {
|
select {
|
||||||
case waitErr = <-finished:
|
case waitErr = <-waited:
|
||||||
// The parent may exit while a descendant inherited stdout/stderr; tear down the owned group.
|
terminateOwnedProcess(c)
|
||||||
terminateOwnedProcess(command)
|
|
||||||
case <-overflow:
|
case <-overflow:
|
||||||
terminateOwnedProcess(command); waitErr = <-finished
|
terminateOwnedProcess(c)
|
||||||
|
waitErr = <-waited
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
terminateOwnedProcess(command); waitErr = <-finished
|
cancelled = ctx.Err()
|
||||||
|
terminateOwnedProcess(c)
|
||||||
|
waitErr = <-waited
|
||||||
}
|
}
|
||||||
wg.Wait()
|
wg.Wait()
|
||||||
result := Result{Stdout: string(out.b), Stderr: string(er.b)}
|
res := Result{Stdout: string(out.buf[:out.n]), Stderr: string(er.buf[:er.n])}
|
||||||
if out.overflow || er.overflow { return result, ErrOutputLimit }
|
if out.overflow || er.overflow {
|
||||||
if waitErr == nil { return result, nil }
|
return res, ErrOutputLimit
|
||||||
var exitErr *exec.ExitError
|
}
|
||||||
if errors.As(waitErr, &exitErr) { result.ExitCode = exitErr.ExitCode(); return result, waitErr }
|
if cancelled != nil {
|
||||||
if errors.Is(waitErr, exec.ErrNotFound) || errors.Is(waitErr, os.ErrNotExist) { result.ExitCode = 127; return result, fmt.Errorf("%w: %w", exec.ErrNotFound, waitErr) }
|
return res, cancelled
|
||||||
if ctx.Err() != nil { return result, ctx.Err() }
|
}
|
||||||
return result, waitErr
|
if waitErr == nil {
|
||||||
|
return res, nil
|
||||||
|
}
|
||||||
|
var ex *exec.ExitError
|
||||||
|
if errors.As(waitErr, &ex) {
|
||||||
|
res.ExitCode = ex.ExitCode()
|
||||||
|
return res, waitErr
|
||||||
|
}
|
||||||
|
if errors.Is(waitErr, exec.ErrNotFound) || errors.Is(waitErr, os.ErrNotExist) {
|
||||||
|
res.ExitCode = 127
|
||||||
|
return res, fmt.Errorf("%w: %w", exec.ErrNotFound, waitErr)
|
||||||
|
}
|
||||||
|
return res, waitErr
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -6,8 +6,10 @@ import (
|
|||||||
"os"
|
"os"
|
||||||
"os/exec"
|
"os/exec"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"runtime"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestRunnerPassesEachArgumentWithoutShellSplitting(t *testing.T) {
|
func TestRunnerPassesEachArgumentWithoutShellSplitting(t *testing.T) {
|
||||||
@@ -65,18 +67,78 @@ func writeExecutable(t *testing.T, contents string) string {
|
|||||||
return path
|
return path
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
func TestRunnerBoundedCapturesExactlyAtLimit(t *testing.T) {
|
func TestRunnerBoundedCapturesExactlyAtLimit(t *testing.T) {
|
||||||
runner := NewRunner(writeExecutable(t, "#!/bin/sh\nprintf 1234567890\nprintf error12345 >&2\n"))
|
runner := NewRunner(writeExecutable(t, "#!/bin/sh\nprintf 1234567890\nprintf error12345 >&2\n"))
|
||||||
result, err := runner.RunBounded(context.Background(), nil, nil, CaptureLimits{StdoutBytes: 10, StderrBytes: 10})
|
result, err := runner.RunBounded(context.Background(), nil, nil, CaptureLimits{StdoutBytes: 10, StderrBytes: 10})
|
||||||
if err != nil { t.Fatal(err) }
|
if err != nil {
|
||||||
if result.Stdout != "1234567890" || result.Stderr != "error12345" { t.Fatalf("bounded result = %#v", result) }
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if result.Stdout != "1234567890" || result.Stderr != "error12345" {
|
||||||
|
t.Fatalf("bounded result = %#v", result)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRunnerBoundedRejectsOneByteOverEachStream(t *testing.T) {
|
func TestRunnerBoundedRejectsOneByteOverEachStream(t *testing.T) {
|
||||||
for _, script := range []string{"#!/bin/sh\nprintf 12345678901\n", "#!/bin/sh\nprintf 12345678901 >&2\n"} {
|
for _, script := range []string{"#!/bin/sh\nprintf 12345678901\n", "#!/bin/sh\nprintf 12345678901 >&2\n"} {
|
||||||
runner := NewRunner(writeExecutable(t, script)); result, err := runner.RunBounded(context.Background(), nil, nil, CaptureLimits{StdoutBytes: 10, StderrBytes: 10})
|
runner := NewRunner(writeExecutable(t, script))
|
||||||
if !errors.Is(err, ErrOutputLimit) { t.Fatalf("err = %v result=%#v", err, result) }
|
result, err := runner.RunBounded(context.Background(), nil, nil, CaptureLimits{StdoutBytes: 10, StderrBytes: 10})
|
||||||
if len(result.Stdout) > 10 || len(result.Stderr) > 10 { t.Fatalf("retained overflow: %#v", result) }
|
if !errors.Is(err, ErrOutputLimit) {
|
||||||
|
t.Fatalf("err = %v result=%#v", err, result)
|
||||||
|
}
|
||||||
|
if len(result.Stdout) > 10 || len(result.Stderr) > 10 {
|
||||||
|
t.Fatalf("retained overflow: %#v", result)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunnerBoundedTerminatesInfiniteStreams(t *testing.T) {
|
||||||
|
for _, script := range []string{"#!/bin/sh\nwhile :; do printf x; done", "#!/bin/sh\nwhile :; do printf x >&2; done"} {
|
||||||
|
runner := NewRunner(writeExecutable(t, script))
|
||||||
|
result, err := runner.RunBounded(context.Background(), nil, nil, CaptureLimits{StdoutBytes: 1024, StderrBytes: 1024})
|
||||||
|
if !errors.Is(err, ErrOutputLimit) {
|
||||||
|
t.Fatalf("err = %v result = %#v", err, result)
|
||||||
|
}
|
||||||
|
if len(result.Stdout) > 1024 || len(result.Stderr) > 1024 {
|
||||||
|
t.Fatalf("retained overflow: %#v", result)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunnerBoundedCancellationWinsOverChildKill(t *testing.T) {
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
runner := NewRunner(writeExecutable(t, "#!/bin/sh\nsleep 30\n"))
|
||||||
|
done := make(chan error, 1)
|
||||||
|
go func() {
|
||||||
|
_, err := runner.RunBounded(ctx, nil, nil, CaptureLimits{StdoutBytes: 32, StderrBytes: 32})
|
||||||
|
done <- err
|
||||||
|
}()
|
||||||
|
cancel()
|
||||||
|
select {
|
||||||
|
case err := <-done:
|
||||||
|
if !errors.Is(err, context.Canceled) {
|
||||||
|
t.Fatalf("err = %v", err)
|
||||||
|
}
|
||||||
|
case <-time.After(3 * time.Second):
|
||||||
|
t.Fatal("cancellation did not terminate process")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunnerBoundedTerminatesGrandchildHoldingPipe(t *testing.T) {
|
||||||
|
if runtime.GOOS == "windows" {
|
||||||
|
t.Skip("shell process-group fixture is Unix-specific")
|
||||||
|
}
|
||||||
|
runner := NewRunner(writeExecutable(t, "#!/bin/sh\n(sleep 30; printf survivor) &\nwhile :; do printf x; done\n"))
|
||||||
|
done := make(chan error, 1)
|
||||||
|
go func() {
|
||||||
|
_, err := runner.RunBounded(context.Background(), nil, nil, CaptureLimits{StdoutBytes: 64, StderrBytes: 64})
|
||||||
|
done <- err
|
||||||
|
}()
|
||||||
|
select {
|
||||||
|
case err := <-done:
|
||||||
|
if !errors.Is(err, ErrOutputLimit) {
|
||||||
|
t.Fatalf("err = %v", err)
|
||||||
|
}
|
||||||
|
case <-time.After(3 * time.Second):
|
||||||
|
t.Fatal("grandchild kept owned pipe alive")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -178,7 +178,11 @@ func TestLoadRejectsRelativeInstallationPaths(t *testing.T) {
|
|||||||
func writeInstallation(t *testing.T, profile string) (string, string, string, string) {
|
func writeInstallation(t *testing.T, profile string) (string, string, string, string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
root := filepath.Join(t.TempDir(), "installation folder with spaces")
|
rootBase, err := filepath.EvalSymlinks(t.TempDir())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
root := filepath.Join(rootBase, "installation folder with spaces")
|
||||||
projectDirectory := filepath.Join(root, "project directory with spaces")
|
projectDirectory := filepath.Join(root, "project directory with spaces")
|
||||||
if err := os.MkdirAll(filepath.Join(projectDirectory, "deploy"), 0o755); err != nil {
|
if err := os.MkdirAll(filepath.Join(projectDirectory, "deploy"), 0o755); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
|
|||||||
@@ -56,9 +56,19 @@ func ReadCanonicalRegular(path string, maximum int64) ([]byte, error) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, ErrUnsafeFile
|
return nil, ErrUnsafeFile
|
||||||
}
|
}
|
||||||
|
// Checking the retained descriptor alone misses a pathname replacement while the
|
||||||
|
// read is in progress. The parent descriptor and final name must still resolve to
|
||||||
|
// the exact open file after reading.
|
||||||
if err := unix.Fstat(int(file.Fd()), &after); err != nil || after.Nlink > 1 || after.Mode != before.Mode || after.Ino != before.Ino || after.Dev != before.Dev || after.Size != before.Size {
|
if err := unix.Fstat(int(file.Fd()), &after); err != nil || after.Nlink > 1 || after.Mode != before.Mode || after.Ino != before.Ino || after.Dev != before.Dev || after.Size != before.Size {
|
||||||
return nil, ErrUnsafeFile
|
return nil, ErrUnsafeFile
|
||||||
}
|
}
|
||||||
|
var named unix.Stat_t
|
||||||
|
if err := unix.Fstatat(directory, components[len(components)-1], &named, unix.AT_SYMLINK_NOFOLLOW); err != nil || named.Nlink > 1 || named.Mode != before.Mode || named.Ino != before.Ino || named.Dev != before.Dev {
|
||||||
|
return nil, ErrUnsafeFile
|
||||||
|
}
|
||||||
|
if !recheckUnixParents(components, directories) {
|
||||||
|
return nil, ErrUnsafeFile
|
||||||
|
}
|
||||||
return contents, nil
|
return contents, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -89,6 +99,10 @@ func writeCanonicalExclusive(path string, contents []byte, mode fs.FileMode) err
|
|||||||
unix.Close(dir)
|
unix.Close(dir)
|
||||||
dir = next
|
dir = next
|
||||||
}
|
}
|
||||||
|
var parentBefore unix.Stat_t
|
||||||
|
if err := unix.Fstat(dir, &parentBefore); err != nil {
|
||||||
|
return ErrUnsafeFile
|
||||||
|
}
|
||||||
fd, err := unix.Openat(dir, components[len(components)-1], unix.O_WRONLY|unix.O_CREAT|unix.O_EXCL|unix.O_CLOEXEC|unix.O_NOFOLLOW, uint32(mode.Perm()))
|
fd, err := unix.Openat(dir, components[len(components)-1], unix.O_WRONLY|unix.O_CREAT|unix.O_EXCL|unix.O_CLOEXEC|unix.O_NOFOLLOW, uint32(mode.Perm()))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return ErrUnsafeFile
|
return ErrUnsafeFile
|
||||||
@@ -108,6 +122,16 @@ func writeCanonicalExclusive(path string, contents []byte, mode fs.FileMode) err
|
|||||||
if err := f.Sync(); err != nil {
|
if err := f.Sync(); err != nil {
|
||||||
return ErrUnsafeFile
|
return ErrUnsafeFile
|
||||||
}
|
}
|
||||||
|
var opened, named, parentAfter unix.Stat_t
|
||||||
|
if err := unix.Fstat(fd, &opened); err != nil || opened.Nlink != 1 {
|
||||||
|
return ErrUnsafeFile
|
||||||
|
}
|
||||||
|
if err := unix.Fstatat(dir, components[len(components)-1], &named, unix.AT_SYMLINK_NOFOLLOW); err != nil || named.Nlink != 1 || named.Ino != opened.Ino || named.Dev != opened.Dev {
|
||||||
|
return ErrUnsafeFile
|
||||||
|
}
|
||||||
|
if err := unix.Fstat(dir, &parentAfter); err != nil || parentAfter.Ino != parentBefore.Ino || parentAfter.Dev != parentBefore.Dev || !recheckUnixParentPath(components[:len(components)-1], dir) {
|
||||||
|
return ErrUnsafeFile
|
||||||
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -140,3 +164,46 @@ func validateCanonicalOutputPath(path string) error {
|
|||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func recheckUnixParents(components []string, retained []int) bool {
|
||||||
|
if len(retained) != len(components) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
dir, err := unix.Open("/", unix.O_RDONLY|unix.O_DIRECTORY|unix.O_CLOEXEC, 0)
|
||||||
|
if err != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
defer unix.Close(dir)
|
||||||
|
for i, component := range components[:len(components)-1] {
|
||||||
|
next, e := unix.Openat(dir, component, unix.O_RDONLY|unix.O_DIRECTORY|unix.O_CLOEXEC|unix.O_NOFOLLOW, 0)
|
||||||
|
if e != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
var got, want unix.Stat_t
|
||||||
|
if unix.Fstat(next, &got) != nil || unix.Fstat(retained[i+1], &want) != nil || got.Ino != want.Ino || got.Dev != want.Dev {
|
||||||
|
unix.Close(next)
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
unix.Close(dir)
|
||||||
|
dir = next
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
func recheckUnixParentPath(components []string, retained int) bool {
|
||||||
|
dir, err := unix.Open("/", unix.O_RDONLY|unix.O_DIRECTORY|unix.O_CLOEXEC, 0)
|
||||||
|
if err != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
defer unix.Close(dir)
|
||||||
|
for _, component := range components {
|
||||||
|
next, e := unix.Openat(dir, component, unix.O_RDONLY|unix.O_DIRECTORY|unix.O_CLOEXEC|unix.O_NOFOLLOW, 0)
|
||||||
|
if e != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
unix.Close(dir)
|
||||||
|
dir = next
|
||||||
|
}
|
||||||
|
var got, want unix.Stat_t
|
||||||
|
return unix.Fstat(dir, &got) == nil && unix.Fstat(retained, &want) == nil && got.Ino == want.Ino && got.Dev == want.Dev
|
||||||
|
}
|
||||||
|
|||||||
@@ -54,7 +54,29 @@ func ReadCanonicalRegular(path string, maximum int64) ([]byte, error) {
|
|||||||
return nil, ErrUnsafeFile
|
return nil, ErrUnsafeFile
|
||||||
}
|
}
|
||||||
defer file.Close()
|
defer file.Close()
|
||||||
return readBoundedRegularFile(file, maximum)
|
var before, after windows.ByHandleFileInformation
|
||||||
|
if windows.GetFileInformationByHandle(handle, &before) != nil || before.NumberOfLinks > 1 {
|
||||||
|
return nil, ErrUnsafeFile
|
||||||
|
}
|
||||||
|
contents, err := readBoundedRegularFile(file, maximum)
|
||||||
|
if err != nil {
|
||||||
|
return nil, ErrUnsafeFile
|
||||||
|
}
|
||||||
|
if windows.GetFileInformationByHandle(handle, &after) != nil || after.NumberOfLinks > 1 || !sameWindowsFile(before, after) {
|
||||||
|
return nil, ErrUnsafeFile
|
||||||
|
}
|
||||||
|
// Re-opened pathname identity is checked as well as the retained handle. This
|
||||||
|
// catches replacement by an ordinary local actor where share/delete policy permits it.
|
||||||
|
check, err := openWindowsComponent(current, false)
|
||||||
|
if err != nil {
|
||||||
|
return nil, ErrUnsafeFile
|
||||||
|
}
|
||||||
|
defer windows.CloseHandle(check)
|
||||||
|
var named windows.ByHandleFileInformation
|
||||||
|
if windows.GetFileInformationByHandle(check, &named) != nil || named.NumberOfLinks > 1 || !sameWindowsFile(before, named) {
|
||||||
|
return nil, ErrUnsafeFile
|
||||||
|
}
|
||||||
|
return contents, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func openWindowsComponent(path string, directory bool) (windows.Handle, error) {
|
func openWindowsComponent(path string, directory bool) (windows.Handle, error) {
|
||||||
@@ -81,7 +103,7 @@ func openWindowsComponent(path string, directory bool) (windows.Handle, error) {
|
|||||||
windows.CloseHandle(handle)
|
windows.CloseHandle(handle)
|
||||||
return 0, err
|
return 0, err
|
||||||
}
|
}
|
||||||
if information.FileAttributes&windows.FILE_ATTRIBUTE_REPARSE_POINT != 0 ||
|
if information.NumberOfLinks > 1 || information.FileAttributes&windows.FILE_ATTRIBUTE_REPARSE_POINT != 0 ||
|
||||||
(directory && information.FileAttributes&windows.FILE_ATTRIBUTE_DIRECTORY == 0) ||
|
(directory && information.FileAttributes&windows.FILE_ATTRIBUTE_DIRECTORY == 0) ||
|
||||||
(!directory && information.FileAttributes&windows.FILE_ATTRIBUTE_DIRECTORY != 0) {
|
(!directory && information.FileAttributes&windows.FILE_ATTRIBUTE_DIRECTORY != 0) {
|
||||||
windows.CloseHandle(handle)
|
windows.CloseHandle(handle)
|
||||||
@@ -113,7 +135,22 @@ func writeCanonicalExclusive(path string, contents []byte, mode fs.FileMode) err
|
|||||||
if _, err := f.Write(contents); err != nil {
|
if _, err := f.Write(contents); err != nil {
|
||||||
return ErrUnsafeFile
|
return ErrUnsafeFile
|
||||||
}
|
}
|
||||||
return f.Sync()
|
if err := f.Sync(); err != nil {
|
||||||
|
return ErrUnsafeFile
|
||||||
|
}
|
||||||
|
var opened, named windows.ByHandleFileInformation
|
||||||
|
if windows.GetFileInformationByHandle(h, &opened) != nil || opened.NumberOfLinks != 1 {
|
||||||
|
return ErrUnsafeFile
|
||||||
|
}
|
||||||
|
check, err := openWindowsComponent(path, false)
|
||||||
|
if err != nil {
|
||||||
|
return ErrUnsafeFile
|
||||||
|
}
|
||||||
|
defer windows.CloseHandle(check)
|
||||||
|
if windows.GetFileInformationByHandle(check, &named) != nil || named.NumberOfLinks != 1 || !sameWindowsFile(opened, named) {
|
||||||
|
return ErrUnsafeFile
|
||||||
|
}
|
||||||
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func validateCanonicalOutputPath(path string) error {
|
func validateCanonicalOutputPath(path string) error {
|
||||||
@@ -140,3 +177,7 @@ func validateCanonicalOutputPath(path string) error {
|
|||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func sameWindowsFile(a, b windows.ByHandleFileInformation) bool {
|
||||||
|
return a.VolumeSerialNumber == b.VolumeSerialNumber && a.FileIndexHigh == b.FileIndexHigh && a.FileIndexLow == b.FileIndexLow
|
||||||
|
}
|
||||||
|
|||||||
@@ -294,7 +294,10 @@ func TestRemoveRejectsConfirmationForDifferentContainerIDs(t *testing.T) {
|
|||||||
|
|
||||||
func testInstallation(t *testing.T) config.Installation {
|
func testInstallation(t *testing.T) config.Installation {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
root := t.TempDir()
|
root, err := filepath.EvalSymlinks(t.TempDir())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
project := filepath.Join(root, "project")
|
project := filepath.Join(root, "project")
|
||||||
if err := os.Mkdir(project, 0o700); err != nil {
|
if err := os.Mkdir(project, 0o700); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
|
|||||||
@@ -5,17 +5,21 @@ import (
|
|||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"crypto/sha256"
|
"crypto/sha256"
|
||||||
|
"encoding/base64"
|
||||||
"encoding/hex"
|
"encoding/hex"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
|
"reflect"
|
||||||
"regexp"
|
"regexp"
|
||||||
"strings"
|
"strings"
|
||||||
|
"unicode/utf8"
|
||||||
|
|
||||||
"github.com/aritmolab/thothii/tools/thothctl/internal/compose"
|
"github.com/aritmolab/thothii/tools/thothctl/internal/compose"
|
||||||
"github.com/aritmolab/thothii/tools/thothctl/internal/config"
|
"github.com/aritmolab/thothii/tools/thothctl/internal/config"
|
||||||
"github.com/aritmolab/thothii/tools/thothctl/internal/safeio"
|
"github.com/aritmolab/thothii/tools/thothctl/internal/safeio"
|
||||||
|
"gopkg.in/yaml.v3"
|
||||||
)
|
)
|
||||||
|
|
||||||
const CodeRegistryBootstrapRecoveryConflict = "registry_bootstrap_recovery_conflict"
|
const CodeRegistryBootstrapRecoveryConflict = "registry_bootstrap_recovery_conflict"
|
||||||
@@ -26,21 +30,23 @@ const (
|
|||||||
maxAssumptionBytes = 256
|
maxAssumptionBytes = 256
|
||||||
maxResult = 1 << 20
|
maxResult = 1 << 20
|
||||||
maxCandidate = 700 << 10
|
maxCandidate = 700 << 10
|
||||||
|
maxAnnotations = 16 << 20
|
||||||
)
|
)
|
||||||
|
|
||||||
var workspaceIDPattern = regexp.MustCompile(`^[a-z][a-z0-9-]{2,62}$`)
|
var workspaceIDPattern = regexp.MustCompile(`^[a-z][a-z0-9-]{2,62}$`)
|
||||||
var runIDPattern = regexp.MustCompile(`^[0-9a-f]{32}$`)
|
var runIDPattern = regexp.MustCompile(`^[0-9a-f]{32}$`)
|
||||||
|
var revisionPattern = regexp.MustCompile(`^[0-9a-f]{40}$`)
|
||||||
var digestPattern = regexp.MustCompile(`^sha256:[0-9a-f]{64}$`)
|
var digestPattern = regexp.MustCompile(`^sha256:[0-9a-f]{64}$`)
|
||||||
|
|
||||||
|
// Command is intentionally closed: callers cannot provide a child argv or bootstrap run ID.
|
||||||
type Command interface{ workspaceCommand() }
|
type Command interface{ workspaceCommand() }
|
||||||
type InspectCommand struct {
|
type InspectCommand struct {
|
||||||
WorkspaceID string
|
WorkspaceID string
|
||||||
JSON bool
|
JSON bool
|
||||||
}
|
}
|
||||||
type DwhRequest struct {
|
type DwhRequest struct {
|
||||||
WorkspaceID string
|
WorkspaceID, Resume string
|
||||||
Resume string
|
JSON bool
|
||||||
JSON bool
|
|
||||||
}
|
}
|
||||||
type SuggestFksRequest struct {
|
type SuggestFksRequest struct {
|
||||||
WorkspaceID string
|
WorkspaceID string
|
||||||
@@ -50,11 +56,8 @@ type SuggestFksRequest struct {
|
|||||||
JSON bool
|
JSON bool
|
||||||
}
|
}
|
||||||
type CheckSchemaRequest struct {
|
type CheckSchemaRequest struct {
|
||||||
WorkspaceID string
|
WorkspaceID, Resume, Annotations, ReviewedCandidates string
|
||||||
Resume string
|
JSON bool
|
||||||
Annotations string
|
|
||||||
ReviewedCandidates string
|
|
||||||
JSON bool
|
|
||||||
}
|
}
|
||||||
type IndexSchemaRequest struct {
|
type IndexSchemaRequest struct {
|
||||||
WorkspaceID string
|
WorkspaceID string
|
||||||
@@ -67,9 +70,8 @@ type EvidenceRequest struct {
|
|||||||
JSON bool
|
JSON bool
|
||||||
}
|
}
|
||||||
type RunRequest struct {
|
type RunRequest struct {
|
||||||
WorkspaceID string
|
WorkspaceID, Resume string
|
||||||
Resume string
|
JSON bool
|
||||||
JSON bool
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (InspectCommand) workspaceCommand() {}
|
func (InspectCommand) workspaceCommand() {}
|
||||||
@@ -99,6 +101,33 @@ type ArtifactIdentity struct {
|
|||||||
Kind string `json:"kind"`
|
Kind string `json:"kind"`
|
||||||
Digest string `json:"digest"`
|
Digest string `json:"digest"`
|
||||||
}
|
}
|
||||||
|
type hostExport struct {
|
||||||
|
MediaType string `json:"mediaType"`
|
||||||
|
SHA256 string `json:"sha256"`
|
||||||
|
ContentBase64 string `json:"contentBase64"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type sqlInput struct {
|
||||||
|
Basename string `json:"basename"`
|
||||||
|
ContentBase64 string `json:"contentBase64"`
|
||||||
|
SHA256 string `json:"sha256"`
|
||||||
|
}
|
||||||
|
type annotationInput struct {
|
||||||
|
Basename string `json:"basename"`
|
||||||
|
ContentBase64 string `json:"contentBase64"`
|
||||||
|
SHA256 string `json:"sha256"`
|
||||||
|
}
|
||||||
|
type inputEnvelope struct {
|
||||||
|
SchemaVersion int `json:"schemaVersion"`
|
||||||
|
Operation string `json:"operation"`
|
||||||
|
WorkspaceID string `json:"workspaceId"`
|
||||||
|
Resume string `json:"resume,omitempty"`
|
||||||
|
SQL []sqlInput `json:"sql,omitempty"`
|
||||||
|
Assume []string `json:"assume,omitempty"`
|
||||||
|
Annotations *annotationInput `json:"annotations,omitempty"`
|
||||||
|
ReviewedCandidates string `json:"reviewedCandidates,omitempty"`
|
||||||
|
DryRun bool `json:"dryRun,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
func invalid(msg string) (Command, error) { return nil, errors.New(msg) }
|
func invalid(msg string) (Command, error) { return nil, errors.New(msg) }
|
||||||
func requireWorkspace(v string) error {
|
func requireWorkspace(v string) error {
|
||||||
@@ -117,46 +146,77 @@ func one(args []string, i *int, flag string) (string, error) {
|
|||||||
if *i+1 >= len(args) || strings.HasPrefix(args[*i+1], "--") {
|
if *i+1 >= len(args) || strings.HasPrefix(args[*i+1], "--") {
|
||||||
return "", fmt.Errorf("%s requires a value", flag)
|
return "", fmt.Errorf("%s requires a value", flag)
|
||||||
}
|
}
|
||||||
*i = *i + 1
|
*i++
|
||||||
return args[*i], nil
|
return args[*i], nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func ParseWorkspaceCommand(args []string) (Command, error) {
|
func ParseWorkspaceCommand(args []string) (Command, error) {
|
||||||
if len(args) < 2 || args[0] != "workspace" {
|
if len(args) < 2 || args[0] != "workspace" {
|
||||||
return invalid("workspace command is required")
|
return invalid("workspace command is required")
|
||||||
}
|
}
|
||||||
area, action := args[1], ""
|
// Parse the hierarchy before options. This prevents aliases such as workspace schema dwh.
|
||||||
rest := args[2:]
|
path := strings.Join(args[:min(3, len(args))], " ")
|
||||||
if area == "preprocess" || area == "schema" {
|
var action string
|
||||||
if len(rest) == 0 {
|
var rest []string
|
||||||
return invalid("workspace group requires an operation")
|
switch {
|
||||||
}
|
case args[1] == "inspect":
|
||||||
action = rest[0]
|
action = "inspect"
|
||||||
rest = rest[1:]
|
rest = args[2:]
|
||||||
} else {
|
case len(args) >= 3 && args[1] == "preprocess" && (args[2] == "dwh" || args[2] == "evidence" || args[2] == "run"):
|
||||||
action = area
|
action = "preprocess " + args[2]
|
||||||
}
|
rest = args[3:]
|
||||||
if action != "inspect" && action != "dwh" && action != "suggest-fks" && action != "check" && action != "index-schema" && action != "evidence" && action != "run" {
|
case len(args) >= 3 && args[1] == "schema" && (args[2] == "suggest-fks" || args[2] == "check"):
|
||||||
return invalid("unknown workspace command")
|
action = "schema " + args[2]
|
||||||
|
rest = args[3:]
|
||||||
|
case args[1] == "index-schema":
|
||||||
|
action = "index-schema"
|
||||||
|
rest = args[2:]
|
||||||
|
default:
|
||||||
|
return invalid("unknown workspace command: " + path)
|
||||||
}
|
}
|
||||||
var ws, resume, annotations, reviewed, output string
|
var ws, resume, annotations, reviewed, output string
|
||||||
var jsonOut, dry bool
|
var jsonOut, dry bool
|
||||||
var sql, assume []string
|
var sql, assume []string
|
||||||
seen := map[string]bool{}
|
seen := map[string]bool{}
|
||||||
|
allowed := func(flag string) bool {
|
||||||
|
switch action {
|
||||||
|
case "inspect":
|
||||||
|
return flag == "--workspace" || flag == "--json"
|
||||||
|
case "preprocess dwh", "preprocess run":
|
||||||
|
return flag == "--workspace" || flag == "--resume" || flag == "--json"
|
||||||
|
case "preprocess evidence":
|
||||||
|
return flag == "--workspace" || flag == "--resume" || flag == "--dry-run" || flag == "--json"
|
||||||
|
case "schema suggest-fks":
|
||||||
|
return flag == "--workspace" || flag == "--from-sql" || flag == "--assume" || flag == "--output" || flag == "--json"
|
||||||
|
case "schema check":
|
||||||
|
return flag == "--workspace" || flag == "--resume" || flag == "--annotations" || flag == "--reviewed-candidates" || flag == "--json"
|
||||||
|
case "index-schema":
|
||||||
|
return flag == "--workspace" || flag == "--json"
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
for i := 0; i < len(rest); i++ {
|
for i := 0; i < len(rest); i++ {
|
||||||
f := rest[i]
|
f := rest[i]
|
||||||
if f == "--json" {
|
if !allowed(f) {
|
||||||
|
return invalid("unknown or invalid workspace option")
|
||||||
|
}
|
||||||
|
if f == "--json" || f == "--dry-run" {
|
||||||
if seen[f] {
|
if seen[f] {
|
||||||
return invalid("duplicate --json")
|
return invalid("duplicate " + f)
|
||||||
}
|
}
|
||||||
seen[f] = true
|
seen[f] = true
|
||||||
jsonOut = true
|
if f == "--json" {
|
||||||
|
jsonOut = true
|
||||||
|
} else {
|
||||||
|
dry = true
|
||||||
|
}
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
if seen[f] && f != "--from-sql" && f != "--assume" {
|
||||||
|
return invalid("duplicate " + f)
|
||||||
|
}
|
||||||
switch f {
|
switch f {
|
||||||
case "--workspace":
|
case "--workspace":
|
||||||
if seen[f] {
|
|
||||||
return invalid("duplicate --workspace")
|
|
||||||
}
|
|
||||||
seen[f] = true
|
seen[f] = true
|
||||||
v, e := one(rest, &i, f)
|
v, e := one(rest, &i, f)
|
||||||
if e != nil {
|
if e != nil {
|
||||||
@@ -164,9 +224,6 @@ func ParseWorkspaceCommand(args []string) (Command, error) {
|
|||||||
}
|
}
|
||||||
ws = v
|
ws = v
|
||||||
case "--resume":
|
case "--resume":
|
||||||
if seen[f] {
|
|
||||||
return invalid("duplicate --resume")
|
|
||||||
}
|
|
||||||
seen[f] = true
|
seen[f] = true
|
||||||
v, e := one(rest, &i, f)
|
v, e := one(rest, &i, f)
|
||||||
if e != nil {
|
if e != nil {
|
||||||
@@ -174,9 +231,6 @@ func ParseWorkspaceCommand(args []string) (Command, error) {
|
|||||||
}
|
}
|
||||||
resume = v
|
resume = v
|
||||||
case "--from-sql":
|
case "--from-sql":
|
||||||
if action != "suggest-fks" {
|
|
||||||
return invalid("--from-sql is valid only for schema suggest-fks")
|
|
||||||
}
|
|
||||||
v, e := one(rest, &i, f)
|
v, e := one(rest, &i, f)
|
||||||
if e != nil {
|
if e != nil {
|
||||||
return nil, e
|
return nil, e
|
||||||
@@ -186,9 +240,6 @@ func ParseWorkspaceCommand(args []string) (Command, error) {
|
|||||||
}
|
}
|
||||||
sql = append(sql, v)
|
sql = append(sql, v)
|
||||||
case "--assume":
|
case "--assume":
|
||||||
if action != "suggest-fks" {
|
|
||||||
return invalid("--assume is valid only for schema suggest-fks")
|
|
||||||
}
|
|
||||||
v, e := one(rest, &i, f)
|
v, e := one(rest, &i, f)
|
||||||
if e != nil {
|
if e != nil {
|
||||||
return nil, e
|
return nil, e
|
||||||
@@ -196,14 +247,11 @@ func ParseWorkspaceCommand(args []string) (Command, error) {
|
|||||||
if len(assume) >= maxAssumptions || len([]byte(v)) > maxAssumptionBytes {
|
if len(assume) >= maxAssumptions || len([]byte(v)) > maxAssumptionBytes {
|
||||||
return invalid("--assume exceeds limit")
|
return invalid("--assume exceeds limit")
|
||||||
}
|
}
|
||||||
|
if !regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*(?:\.[A-Za-z_][A-Za-z0-9_]*)?=[A-Za-z_][A-Za-z0-9_]*(?:\.[A-Za-z_][A-Za-z0-9_]*)?$`).MatchString(v) {
|
||||||
|
return invalid("--assume must be column=table")
|
||||||
|
}
|
||||||
assume = append(assume, v)
|
assume = append(assume, v)
|
||||||
case "--output":
|
case "--output":
|
||||||
if action != "suggest-fks" {
|
|
||||||
return invalid("--output is valid only for schema suggest-fks")
|
|
||||||
}
|
|
||||||
if seen[f] {
|
|
||||||
return invalid("duplicate --output")
|
|
||||||
}
|
|
||||||
seen[f] = true
|
seen[f] = true
|
||||||
v, e := one(rest, &i, f)
|
v, e := one(rest, &i, f)
|
||||||
if e != nil {
|
if e != nil {
|
||||||
@@ -211,12 +259,6 @@ func ParseWorkspaceCommand(args []string) (Command, error) {
|
|||||||
}
|
}
|
||||||
output = v
|
output = v
|
||||||
case "--annotations":
|
case "--annotations":
|
||||||
if action != "check" {
|
|
||||||
return invalid("--annotations is valid only for schema check")
|
|
||||||
}
|
|
||||||
if seen[f] {
|
|
||||||
return invalid("duplicate --annotations")
|
|
||||||
}
|
|
||||||
seen[f] = true
|
seen[f] = true
|
||||||
v, e := one(rest, &i, f)
|
v, e := one(rest, &i, f)
|
||||||
if e != nil {
|
if e != nil {
|
||||||
@@ -224,46 +266,26 @@ func ParseWorkspaceCommand(args []string) (Command, error) {
|
|||||||
}
|
}
|
||||||
annotations = v
|
annotations = v
|
||||||
case "--reviewed-candidates":
|
case "--reviewed-candidates":
|
||||||
if action != "check" {
|
|
||||||
return invalid("--reviewed-candidates is valid only for schema check")
|
|
||||||
}
|
|
||||||
if seen[f] {
|
|
||||||
return invalid("duplicate --reviewed-candidates")
|
|
||||||
}
|
|
||||||
seen[f] = true
|
seen[f] = true
|
||||||
v, e := one(rest, &i, f)
|
v, e := one(rest, &i, f)
|
||||||
if e != nil {
|
if e != nil {
|
||||||
return nil, e
|
return nil, e
|
||||||
}
|
}
|
||||||
reviewed = v
|
reviewed = v
|
||||||
case "--dry-run":
|
|
||||||
if action != "evidence" {
|
|
||||||
return invalid("--dry-run is valid only for evidence")
|
|
||||||
}
|
|
||||||
if seen[f] {
|
|
||||||
return invalid("duplicate --dry-run")
|
|
||||||
}
|
|
||||||
seen[f] = true
|
|
||||||
dry = true
|
|
||||||
default:
|
|
||||||
return invalid("unknown workspace option")
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if err := requireWorkspace(ws); err != nil {
|
if err := requireWorkspace(ws); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
if action != "inspect" && resume != "" {
|
if resume != "" {
|
||||||
if err := requireRun(resume); err != nil {
|
if err := requireRun(resume); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if resume != "" && action != "dwh" && action != "evidence" && action != "run" && action != "check" {
|
|
||||||
return invalid("--resume is not valid for this workspace command")
|
|
||||||
}
|
|
||||||
if action == "inspect" && resume != "" {
|
if action == "inspect" && resume != "" {
|
||||||
return invalid("inspect does not accept --resume")
|
return invalid("inspect does not accept --resume")
|
||||||
}
|
}
|
||||||
if action == "check" {
|
if action == "schema check" {
|
||||||
if resume == "" {
|
if resume == "" {
|
||||||
return invalid("schema check requires --resume")
|
return invalid("schema check requires --resume")
|
||||||
}
|
}
|
||||||
@@ -274,43 +296,32 @@ func ParseWorkspaceCommand(args []string) (Command, error) {
|
|||||||
return invalid("--reviewed-candidates must be sha256:<hex>")
|
return invalid("--reviewed-candidates must be sha256:<hex>")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if action != "suggest-fks" && (len(sql) > 0 || len(assume) > 0 || output != "") {
|
if action != "schema check" && (annotations != "" || reviewed != "") {
|
||||||
return invalid("schema options are valid only for suggest-fks")
|
return invalid("annotation options are valid only for schema check")
|
||||||
}
|
}
|
||||||
switch action {
|
switch action {
|
||||||
case "inspect":
|
case "inspect":
|
||||||
return InspectCommand{ws, jsonOut}, nil
|
return InspectCommand{ws, jsonOut}, nil
|
||||||
case "dwh":
|
case "preprocess dwh":
|
||||||
return DwhRequest{ws, resume, jsonOut}, nil
|
return DwhRequest{ws, resume, jsonOut}, nil
|
||||||
case "suggest-fks":
|
case "schema suggest-fks":
|
||||||
return SuggestFksRequest{ws, sql, assume, output, jsonOut}, nil
|
return SuggestFksRequest{ws, sql, assume, output, jsonOut}, nil
|
||||||
case "check":
|
case "schema check":
|
||||||
return CheckSchemaRequest{ws, resume, annotations, reviewed, jsonOut}, nil
|
return CheckSchemaRequest{ws, resume, annotations, reviewed, jsonOut}, nil
|
||||||
case "index-schema":
|
case "index-schema":
|
||||||
return IndexSchemaRequest{ws, jsonOut}, nil
|
return IndexSchemaRequest{ws, jsonOut}, nil
|
||||||
case "evidence":
|
case "preprocess evidence":
|
||||||
return EvidenceRequest{ws, dry, resume, jsonOut}, nil
|
return EvidenceRequest{ws, dry, resume, jsonOut}, nil
|
||||||
default:
|
default:
|
||||||
return RunRequest{ws, resume, jsonOut}, nil
|
return RunRequest{ws, resume, jsonOut}, nil
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
func min(a, b int) int {
|
||||||
type inputEnvelope struct {
|
if a < b {
|
||||||
SchemaVersion int `json:"schemaVersion"`
|
return a
|
||||||
Operation string `json:"operation"`
|
}
|
||||||
WorkspaceID string `json:"workspaceId"`
|
return b
|
||||||
Resume string `json:"resume,omitempty"`
|
|
||||||
SQL []sqlInput `json:"sql,omitempty"`
|
|
||||||
Assume []string `json:"assume,omitempty"`
|
|
||||||
Annotations string `json:"annotations,omitempty"`
|
|
||||||
ReviewedCandidates string `json:"reviewedCandidates,omitempty"`
|
|
||||||
DryRun bool `json:"dryRun,omitempty"`
|
|
||||||
}
|
}
|
||||||
type sqlInput struct {
|
|
||||||
Name string `json:"name"`
|
|
||||||
Content string `json:"content"`
|
|
||||||
}
|
|
||||||
|
|
||||||
func operationName(c Command) string {
|
func operationName(c Command) string {
|
||||||
switch c.(type) {
|
switch c.(type) {
|
||||||
case InspectCommand:
|
case InspectCommand:
|
||||||
@@ -329,114 +340,275 @@ func operationName(c Command) string {
|
|||||||
return "run"
|
return "run"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
func Run(ctx context.Context, installation config.Installation, runner compose.Runner, command Command, stdin io.Reader) (Result, error) {
|
|
||||||
env := inputEnvelope{SchemaVersion: 1, Operation: operationName(command)}
|
func makeInput(c Command) (inputEnvelope, string, error) {
|
||||||
var outputPath string
|
env := inputEnvelope{SchemaVersion: 1, Operation: operationName(c)}
|
||||||
switch c := command.(type) {
|
var out string
|
||||||
|
switch x := c.(type) {
|
||||||
case InspectCommand:
|
case InspectCommand:
|
||||||
env.WorkspaceID = c.WorkspaceID
|
env.WorkspaceID = x.WorkspaceID
|
||||||
case DwhRequest:
|
case DwhRequest:
|
||||||
env.WorkspaceID, env.Resume = c.WorkspaceID, c.Resume
|
env.WorkspaceID, env.Resume = x.WorkspaceID, x.Resume
|
||||||
case SuggestFksRequest:
|
case SuggestFksRequest:
|
||||||
env.WorkspaceID, env.Assume, outputPath = c.WorkspaceID, c.Assume, c.Output
|
env.WorkspaceID, env.Assume, out = x.WorkspaceID, x.Assume, x.Output
|
||||||
if outputPath != "" {
|
if out != "" {
|
||||||
if err := safeio.ValidateCanonicalOutputPath(outputPath); err != nil {
|
if err := safeio.ValidateCanonicalOutputPath(out); err != nil {
|
||||||
return Result{}, errors.New("unsafe output file")
|
return env, "", errors.New("unsafe output file")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
var total int64
|
var total int64
|
||||||
for _, path := range c.FromSQL {
|
for _, path := range x.FromSQL {
|
||||||
b, e := safeio.ReadCanonicalUTF8(path, maxSQLFile)
|
b, e := safeio.ReadCanonicalUTF8(path, maxSQLFile)
|
||||||
if e != nil {
|
if e != nil {
|
||||||
return Result{}, errors.New("unsafe SQL input")
|
return env, "", errors.New("unsafe SQL input")
|
||||||
}
|
}
|
||||||
total += int64(len(b))
|
total += int64(len(b))
|
||||||
if total > maxSQLTotal {
|
if total > maxSQLTotal {
|
||||||
return Result{}, errors.New("SQL input exceeds limit")
|
return env, "", errors.New("SQL input exceeds limit")
|
||||||
}
|
|
||||||
env.SQL = append(env.SQL, sqlInput{path, string(b)})
|
|
||||||
}
|
|
||||||
if len(env.SQL) > 0 {
|
|
||||||
var n int
|
|
||||||
for _, x := range env.SQL {
|
|
||||||
n += len(x.Content)
|
|
||||||
}
|
|
||||||
if n > maxCandidate {
|
|
||||||
return Result{}, errors.New("candidate input exceeds limit")
|
|
||||||
}
|
}
|
||||||
|
base := path[strings.LastIndexAny(path, "/\\")+1:]
|
||||||
|
env.SQL = append(env.SQL, sqlInput{base, base64.StdEncoding.EncodeToString(b), DigestBytes(b)})
|
||||||
}
|
}
|
||||||
case CheckSchemaRequest:
|
case CheckSchemaRequest:
|
||||||
env.WorkspaceID, env.Resume, env.Annotations, env.ReviewedCandidates = c.WorkspaceID, c.Resume, c.Annotations, c.ReviewedCandidates
|
env.WorkspaceID, env.Resume, env.ReviewedCandidates = x.WorkspaceID, x.Resume, x.ReviewedCandidates
|
||||||
if c.Annotations != "" {
|
if x.Annotations != "" {
|
||||||
b, e := safeio.ReadCanonicalUTF8(c.Annotations, 16<<20)
|
b, e := safeio.ReadCanonicalUTF8(x.Annotations, maxAnnotations)
|
||||||
if e != nil {
|
if e != nil {
|
||||||
return Result{}, errors.New("unsafe annotation input")
|
return env, "", errors.New("unsafe annotation input")
|
||||||
}
|
}
|
||||||
env.Annotations = string(b)
|
base := x.Annotations[strings.LastIndexAny(x.Annotations, "/\\")+1:]
|
||||||
|
env.Annotations = &annotationInput{base, base64.StdEncoding.EncodeToString(b), DigestBytes(b)}
|
||||||
}
|
}
|
||||||
case IndexSchemaRequest:
|
case IndexSchemaRequest:
|
||||||
env.WorkspaceID = c.WorkspaceID
|
env.WorkspaceID = x.WorkspaceID
|
||||||
case EvidenceRequest:
|
case EvidenceRequest:
|
||||||
env.WorkspaceID, env.Resume, env.DryRun = c.WorkspaceID, c.Resume, c.DryRun
|
env.WorkspaceID, env.Resume, env.DryRun = x.WorkspaceID, x.Resume, x.DryRun
|
||||||
case RunRequest:
|
case RunRequest:
|
||||||
env.WorkspaceID, env.Resume = c.WorkspaceID, c.Resume
|
env.WorkspaceID, env.Resume = x.WorkspaceID, x.Resume
|
||||||
default:
|
default:
|
||||||
return Result{}, errors.New("unsupported workspace command")
|
return env, "", errors.New("unsupported workspace command")
|
||||||
}
|
}
|
||||||
payload, e := json.Marshal(env)
|
payload, e := json.Marshal(env)
|
||||||
|
if e != nil {
|
||||||
|
return env, "", e
|
||||||
|
}
|
||||||
|
if len(payload) > maxResult {
|
||||||
|
return env, "", errors.New("request exceeds limit")
|
||||||
|
}
|
||||||
|
return env, string(payload), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
var commonIngress = map[string]bool{"schemaVersion": true, "operation": true, "workspaceId": true}
|
||||||
|
|
||||||
|
func validateIngress(payload []byte, expected inputEnvelope) error {
|
||||||
|
var raw map[string]json.RawMessage
|
||||||
|
d := json.NewDecoder(bytes.NewReader(payload))
|
||||||
|
d.UseNumber()
|
||||||
|
if d.Decode(&raw) != nil {
|
||||||
|
return errors.New("invalid workspace request")
|
||||||
|
}
|
||||||
|
var extra any
|
||||||
|
if d.Decode(&extra) != io.EOF {
|
||||||
|
return errors.New("invalid workspace request")
|
||||||
|
}
|
||||||
|
var supplied inputEnvelope
|
||||||
|
dec := json.NewDecoder(bytes.NewReader(payload))
|
||||||
|
dec.DisallowUnknownFields()
|
||||||
|
if dec.Decode(&supplied) != nil || supplied.SchemaVersion != 1 || supplied.Operation != expected.Operation || supplied.WorkspaceID != expected.WorkspaceID {
|
||||||
|
return errors.New("invalid workspace request")
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(supplied, expected) {
|
||||||
|
return errors.New("workspace request does not match command")
|
||||||
|
}
|
||||||
|
allowed := map[string]bool{}
|
||||||
|
for k := range commonIngress {
|
||||||
|
allowed[k] = true
|
||||||
|
}
|
||||||
|
switch expected.Operation {
|
||||||
|
case "dwh", "run":
|
||||||
|
if expected.Resume != "" {
|
||||||
|
allowed["resume"] = true
|
||||||
|
}
|
||||||
|
case "evidence":
|
||||||
|
allowed["resume"] = true
|
||||||
|
allowed["dryRun"] = true
|
||||||
|
case "suggest-fks":
|
||||||
|
allowed["sql"] = true
|
||||||
|
allowed["assume"] = true
|
||||||
|
case "check":
|
||||||
|
allowed["resume"] = true
|
||||||
|
allowed["annotations"] = true
|
||||||
|
allowed["reviewedCandidates"] = true
|
||||||
|
}
|
||||||
|
for k := range raw {
|
||||||
|
if !allowed[k] {
|
||||||
|
return errors.New("invalid workspace request")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func Run(ctx context.Context, installation config.Installation, runner compose.Runner, command Command, stdin io.Reader) (Result, error) {
|
||||||
|
env, generated, e := makeInput(command)
|
||||||
if e != nil {
|
if e != nil {
|
||||||
return Result{}, e
|
return Result{}, e
|
||||||
}
|
}
|
||||||
|
payload := []byte(generated)
|
||||||
if stdin != nil {
|
if stdin != nil {
|
||||||
payload, e = io.ReadAll(io.LimitReader(stdin, 1<<20+1))
|
payload, e = io.ReadAll(io.LimitReader(stdin, maxResult+1))
|
||||||
if e != nil {
|
if e != nil || len(payload) > maxResult {
|
||||||
return Result{}, e
|
|
||||||
}
|
|
||||||
if len(payload) > 1<<20 {
|
|
||||||
return Result{}, errors.New("request exceeds limit")
|
return Result{}, errors.New("request exceeds limit")
|
||||||
}
|
}
|
||||||
var supplied inputEnvelope
|
if e = validateIngress(payload, env); e != nil {
|
||||||
d := json.NewDecoder(bytes.NewReader(payload))
|
return Result{}, e
|
||||||
d.DisallowUnknownFields()
|
|
||||||
if e = d.Decode(&supplied); e != nil || supplied.SchemaVersion != 1 || supplied.Operation != env.Operation || supplied.WorkspaceID != env.WorkspaceID {
|
|
||||||
return Result{}, errors.New("invalid workspace request")
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
args := installation.ComposeArgs("run", "--rm", "--no-deps", "--no-TTY", "workspace-maintenance", "--operation", operationName(command), "--workspace", env.WorkspaceID)
|
||||||
args := installation.ComposeArgs("run", "--rm", "--no-deps", "workspace-maintenance", "--operation", operationName(command), "--workspace", env.WorkspaceID)
|
cr, runErr := runner.RunBounded(ctx, args, bytes.NewReader(payload), compose.CaptureLimits{StdoutBytes: maxResult, StderrBytes: 64 << 10})
|
||||||
cr, runErr := runner.RunBounded(ctx, args, bytes.NewReader(payload), compose.CaptureLimits{StdoutBytes: 1 << 20, StderrBytes: 64 << 10})
|
if errors.Is(runErr, compose.ErrOutputLimit) {
|
||||||
if runErr != nil && (errors.Is(runErr, compose.ErrOutputLimit) || cr.Stdout == "") {
|
|
||||||
return Result{}, runErr
|
return Result{}, runErr
|
||||||
}
|
}
|
||||||
|
if len(cr.Stdout) == 0 {
|
||||||
|
return Result{}, runErrOr(runErr, "workspace maintenance returned no result")
|
||||||
|
}
|
||||||
if len(cr.Stdout) > maxResult {
|
if len(cr.Stdout) > maxResult {
|
||||||
return Result{}, errors.New("invalid workspace result")
|
return Result{}, errors.New("invalid workspace result")
|
||||||
}
|
}
|
||||||
var result Result
|
var raw map[string]json.RawMessage
|
||||||
dec := json.NewDecoder(strings.NewReader(cr.Stdout))
|
d := json.NewDecoder(strings.NewReader(cr.Stdout))
|
||||||
dec.DisallowUnknownFields()
|
if d.Decode(&raw) != nil {
|
||||||
if e := dec.Decode(&result); e != nil {
|
|
||||||
return Result{}, errors.New("invalid workspace result")
|
return Result{}, errors.New("invalid workspace result")
|
||||||
}
|
}
|
||||||
var extra any
|
var extra any
|
||||||
if e := dec.Decode(&extra); e != io.EOF {
|
if d.Decode(&extra) != io.EOF {
|
||||||
return Result{}, errors.New("invalid workspace result")
|
return Result{}, errors.New("invalid workspace result")
|
||||||
}
|
}
|
||||||
if result.SchemaVersion != 1 || result.WorkspaceID != env.WorkspaceID || result.Operation != operationName(command) || !validStatus(result.Status) || !validCode(result.Code) {
|
var export *hostExport
|
||||||
|
hasExport := false
|
||||||
|
if h, ok := raw["hostExport"]; ok {
|
||||||
|
hasExport = true
|
||||||
|
if operationName(command) != "suggest-fks" {
|
||||||
|
return Result{}, errors.New("invalid workspace result")
|
||||||
|
}
|
||||||
|
if json.Unmarshal(h, &export) != nil {
|
||||||
|
return Result{}, errors.New("invalid host export")
|
||||||
|
}
|
||||||
|
delete(raw, "hostExport")
|
||||||
|
}
|
||||||
|
b, _ := json.Marshal(raw)
|
||||||
|
var result Result
|
||||||
|
dec := json.NewDecoder(bytes.NewReader(b))
|
||||||
|
dec.DisallowUnknownFields()
|
||||||
|
if dec.Decode(&result) != nil {
|
||||||
return Result{}, errors.New("invalid workspace result")
|
return Result{}, errors.New("invalid workspace result")
|
||||||
}
|
}
|
||||||
if outputPath != "" {
|
if e = validateResult(result, env.WorkspaceID, operationName(command)); e != nil {
|
||||||
if len(cr.Stdout) > maxCandidate {
|
return Result{}, e
|
||||||
return Result{}, errors.New("candidate export exceeds limit")
|
}
|
||||||
}
|
if runErr != nil {
|
||||||
if err := safeio.WriteCanonicalExclusive(outputPath, []byte(cr.Stdout), 0o600); err != nil {
|
if result.Status == "blocked" && cr.ExitCode == 3 {
|
||||||
return Result{}, errors.New("unsafe output file")
|
} else if result.Status == "failed" && cr.ExitCode == 1 {
|
||||||
|
} else {
|
||||||
|
return Result{}, runErr
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if hasExport {
|
||||||
|
if export == nil {
|
||||||
|
return Result{}, errors.New("invalid candidate export")
|
||||||
|
}
|
||||||
|
if !candidateBoundToResult(*export, result) {
|
||||||
|
return Result{}, errors.New("invalid candidate identity")
|
||||||
|
}
|
||||||
|
if err := publishCandidate(export, result, outPath(command)); err != nil {
|
||||||
|
return Result{}, err
|
||||||
|
}
|
||||||
|
} else if outPath(command) != "" {
|
||||||
|
return Result{}, errors.New("candidate export is required")
|
||||||
|
}
|
||||||
return result, nil
|
return result, nil
|
||||||
}
|
}
|
||||||
func DigestBytes(b []byte) string { s := sha256.Sum256(b); return "sha256:" + hex.EncodeToString(s[:]) }
|
func runErrOr(e error, msg string) error {
|
||||||
|
if e != nil {
|
||||||
|
return e
|
||||||
|
}
|
||||||
|
return errors.New(msg)
|
||||||
|
}
|
||||||
|
func outPath(c Command) string {
|
||||||
|
if x, ok := c.(SuggestFksRequest); ok {
|
||||||
|
return x.Output
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
func candidateBoundToResult(x hostExport, result Result) bool {
|
||||||
|
if result.RunID == "" {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
for _, artifact := range result.ArtifactIdentities {
|
||||||
|
if artifact.Kind == "fk-candidates" && artifact.Digest == x.SHA256 {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
func publishCandidate(x *hostExport, result Result, path string) error {
|
||||||
|
if path == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if x.MediaType != "application/yaml" && x.MediaType != "text/yaml" {
|
||||||
|
return errors.New("invalid candidate export")
|
||||||
|
}
|
||||||
|
if !digestPattern.MatchString(x.SHA256) {
|
||||||
|
return errors.New("invalid candidate export")
|
||||||
|
}
|
||||||
|
b, e := base64.StdEncoding.DecodeString(x.ContentBase64)
|
||||||
|
if e != nil || len(b) > maxCandidate || !utf8.Valid(b) {
|
||||||
|
return errors.New("invalid candidate export")
|
||||||
|
}
|
||||||
|
if DigestBytes(b) != x.SHA256 {
|
||||||
|
return errors.New("invalid candidate export")
|
||||||
|
}
|
||||||
|
var doc any
|
||||||
|
if yaml.Unmarshal(b, &doc) != nil {
|
||||||
|
return errors.New("invalid candidate export")
|
||||||
|
}
|
||||||
|
if result.RunID == "" || !runIDPattern.MatchString(result.RunID) {
|
||||||
|
return errors.New("invalid candidate identity")
|
||||||
|
}
|
||||||
|
if e = safeio.WriteCanonicalExclusive(path, b, 0o600); e != nil {
|
||||||
|
return errors.New("unsafe output file")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
func validateResult(r Result, workspace, operation string) error {
|
||||||
|
if r.SchemaVersion != 1 || r.WorkspaceID != workspace || r.Operation != operation || !validStatus(r.Status) || !validCode(r.Code) || !revisionPattern.MatchString(r.WorkspaceRevision) || !revisionPattern.MatchString(r.DescriptorBlob) {
|
||||||
|
return errors.New("invalid workspace result")
|
||||||
|
}
|
||||||
|
if r.RunID != "" && !runIDPattern.MatchString(r.RunID) {
|
||||||
|
return errors.New("invalid workspace result")
|
||||||
|
}
|
||||||
|
for k, v := range r.ChildRuns {
|
||||||
|
if k == "" || !runIDPattern.MatchString(v) {
|
||||||
|
return errors.New("invalid workspace result")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, a := range r.ArtifactIdentities {
|
||||||
|
if a.Kind == "" || !digestPattern.MatchString(a.Digest) {
|
||||||
|
return errors.New("invalid workspace result")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if r.CompletedStages == nil {
|
||||||
|
return errors.New("invalid workspace result")
|
||||||
|
}
|
||||||
|
if r.Status == "blocked" && (r.Code != "manual_review_required" && r.Code != "evidence_materialization_required" && r.Code != "preprocessing_conflict" && r.Code != "preprocessing_resume_mismatch" && r.Code != CodeRegistryBootstrapRecoveryConflict) {
|
||||||
|
return errors.New("invalid workspace result")
|
||||||
|
}
|
||||||
|
if r.Status != "blocked" && r.Code == CodeRegistryBootstrapRecoveryConflict {
|
||||||
|
return errors.New("invalid workspace result")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
func DigestBytes(b []byte) string { s := sha256.Sum256(b); return "sha256:" + hex.EncodeToString(s[:]) }
|
||||||
func validStatus(v string) bool {
|
func validStatus(v string) bool {
|
||||||
switch v {
|
switch v {
|
||||||
case "succeeded", "unchanged", "dry_run", "blocked", "failed":
|
case "succeeded", "unchanged", "dry_run", "blocked", "failed":
|
||||||
|
|||||||
@@ -1,8 +1,16 @@
|
|||||||
package workspaceops
|
package workspaceops
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"encoding/base64"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
"reflect"
|
"reflect"
|
||||||
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
"context"
|
||||||
|
"github.com/aritmolab/thothii/tools/thothctl/internal/compose"
|
||||||
|
"github.com/aritmolab/thothii/tools/thothctl/internal/config"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestParseWorkspaceCommands(t *testing.T) {
|
func TestParseWorkspaceCommands(t *testing.T) {
|
||||||
@@ -48,3 +56,79 @@ func TestParseWorkspaceRejectsUnsafeOrAmbiguousOptions(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestParseWorkspaceRejectsNonContractHierarchyAndAssumeShell(t *testing.T) {
|
||||||
|
for _, argv := range [][]string{
|
||||||
|
{"workspace", "schema", "dwh", "--workspace", "psd"},
|
||||||
|
{"workspace", "preprocess", "check", "--workspace", "psd"},
|
||||||
|
{"workspace", "suggest-fks", "--workspace", "psd"},
|
||||||
|
{"workspace", "inspect", "--workspace", "psd", "--bootstrap-run-id", strings.Repeat("a", 32)},
|
||||||
|
{"workspace", "schema", "suggest-fks", "--workspace", "psd", "--assume", "a"},
|
||||||
|
{"workspace", "schema", "suggest-fks", "--workspace", "psd", "--assume", "a=$(id)"},
|
||||||
|
} {
|
||||||
|
if _, err := ParseWorkspaceCommand(argv); err == nil {
|
||||||
|
t.Errorf("accepted non-contract argv %v", argv)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunUsesBase64BasenameIngressAndNoTTY(t *testing.T) {
|
||||||
|
root, err := filepath.EvalSymlinks(t.TempDir())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
sqlPath := filepath.Join(root, "schema.sql")
|
||||||
|
if err := os.WriteFile(sqlPath, []byte("select 1"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
out := filepath.Join(root, "candidate.yaml")
|
||||||
|
resultJSON := `{"schemaVersion":1,"status":"succeeded","code":"ok","workspaceId":"psd","workspaceRevision":"0123456789012345678901234567890123456789","descriptorBlob":"abcdefabcdefabcdefabcdefabcdefabcdefabcd","operation":"suggest-fks","runId":"0123456789abcdef0123456789abcdef","completedStages":[]}`
|
||||||
|
script := "#!/bin/sh\ncat >/dev/null\nprintf '%s' '" + resultJSON + "'\n"
|
||||||
|
fake := filepath.Join(root, "docker")
|
||||||
|
if err := os.WriteFile(fake, []byte(script), 0o700); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
cmd, err := ParseWorkspaceCommand([]string{"workspace", "schema", "suggest-fks", "--workspace", "psd", "--from-sql", sqlPath})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
got, err := Run(context.Background(), config.Installation{ProjectDirectory: root, EnvFile: filepath.Join(root, "env")}, compose.NewRunner(fake), cmd, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got.Code != "ok" {
|
||||||
|
t.Fatalf("result = %#v", got)
|
||||||
|
}
|
||||||
|
_ = out
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunPublishesOnlyVerifiedCandidateExport(t *testing.T) {
|
||||||
|
root, err := filepath.EvalSymlinks(t.TempDir())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
candidate := []byte("candidates: []\n")
|
||||||
|
digest := DigestBytes(candidate)
|
||||||
|
encoded := base64.StdEncoding.EncodeToString(candidate)
|
||||||
|
resultJSON := `{"schemaVersion":1,"status":"succeeded","code":"ok","workspaceId":"psd","workspaceRevision":"0123456789012345678901234567890123456789","descriptorBlob":"abcdefabcdefabcdefabcdefabcdefabcdefabcd","operation":"suggest-fks","runId":"0123456789abcdef0123456789abcdef","completedStages":[],"artifactIdentities":[{"kind":"fk-candidates","digest":"` + digest + `"}],"hostExport":{"mediaType":"application/yaml","sha256":"` + digest + `","contentBase64":"` + encoded + `"}}`
|
||||||
|
fake := filepath.Join(root, "docker")
|
||||||
|
if err := os.WriteFile(fake, []byte("#!/bin/sh\ncat >/dev/null\nprintf '%s' '"+resultJSON+"'\n"), 0o700); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
path := filepath.Join(root, "out.yaml")
|
||||||
|
cmd, err := ParseWorkspaceCommand([]string{"workspace", "schema", "suggest-fks", "--workspace", "psd", "--output", path})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
got, err := Run(context.Background(), config.Installation{ProjectDirectory: root, EnvFile: filepath.Join(root, "env")}, compose.NewRunner(fake), cmd, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got.RunID == "" {
|
||||||
|
t.Fatal("missing run identity")
|
||||||
|
}
|
||||||
|
b, err := os.ReadFile(path)
|
||||||
|
if err != nil || string(b) != string(candidate) {
|
||||||
|
t.Fatalf("candidate = %q, %v", b, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user