fix(auth): close Task 15 review round two
This commit is contained in:
@@ -103,7 +103,7 @@ jobs:
|
|||||||
- name: Install Chromium for Playwright
|
- name: Install Chromium for Playwright
|
||||||
working-directory: frontend
|
working-directory: frontend
|
||||||
run: npx playwright install --with-deps chromium
|
run: npx playwright install --with-deps chromium
|
||||||
- name: Run authentication browser smoke
|
- name: Run authentication and authenticated F1 browser smoke
|
||||||
run: bash scripts/authentication-smoke.sh
|
run: bash scripts/authentication-smoke.sh
|
||||||
|
|
||||||
linux-docker:
|
linux-docker:
|
||||||
|
|||||||
@@ -22,6 +22,7 @@ test("the loopback fixture exposes signed OIDC discovery, device authorization,
|
|||||||
deviceAuthorization: true,
|
deviceAuthorization: true,
|
||||||
deviceToken: true,
|
deviceToken: true,
|
||||||
groupList: true,
|
groupList: true,
|
||||||
|
runtimeCredential: process.env.THT_TASK15_SENTINEL !== undefined,
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|||||||
@@ -18,8 +18,8 @@ const thtRoot = join(repositoryRoot, "tools", "tht");
|
|||||||
const fakePi = join(frontendRoot, "e2e", "fixtures", "fake-pi.mjs");
|
const fakePi = join(frontendRoot, "e2e", "fixtures", "fake-pi.mjs");
|
||||||
const fakeTht = join(frontendRoot, "e2e", "fixtures", "fake-tht.mjs");
|
const fakeTht = join(frontendRoot, "e2e", "fixtures", "fake-tht.mjs");
|
||||||
const FIXTURE_CLIENT_ID = "thothii-e2e-client";
|
const FIXTURE_CLIENT_ID = "thothii-e2e-client";
|
||||||
const FIXTURE_CLIENT_SECRET = "e2e-client-secret-not-a-production-secret";
|
const DEFAULT_FIXTURE_CLIENT_SECRET = "e2e-client-secret-not-a-production-secret";
|
||||||
const FIXTURE_API_TOKEN = "e2e-group-catalog-token-not-a-production-secret";
|
const DEFAULT_FIXTURE_API_TOKEN = "e2e-group-catalog-token-not-a-production-secret";
|
||||||
const WRONG_CLIENT_ID = "thothii-e2e-unregistered-client";
|
const WRONG_CLIENT_ID = "thothii-e2e-unregistered-client";
|
||||||
const WRONG_CLIENT_SECRET = "e2e-wrong-client-secret-not-production";
|
const WRONG_CLIENT_SECRET = "e2e-wrong-client-secret-not-production";
|
||||||
const WRONG_API_TOKEN = "e2e-wrong-api-token-not-production";
|
const WRONG_API_TOKEN = "e2e-wrong-api-token-not-production";
|
||||||
@@ -31,6 +31,21 @@ function safeError(code) {
|
|||||||
return new Error(code);
|
return new Error(code);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function resolveFixtureCredentials() {
|
||||||
|
const runtimeCredential = process.env.THT_TASK15_SENTINEL;
|
||||||
|
if (runtimeCredential === undefined) {
|
||||||
|
return Object.freeze({
|
||||||
|
clientSecret: DEFAULT_FIXTURE_CLIENT_SECRET,
|
||||||
|
apiToken: DEFAULT_FIXTURE_API_TOKEN,
|
||||||
|
runtime: false,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if (runtimeCredential.length < 24 || runtimeCredential.length > 512 || /[\r\n\0]/u.test(runtimeCredential)) {
|
||||||
|
throw safeError("e2e_runtime_fixture_credential_invalid");
|
||||||
|
}
|
||||||
|
return Object.freeze({ clientSecret: runtimeCredential, apiToken: runtimeCredential, runtime: true });
|
||||||
|
}
|
||||||
|
|
||||||
function buildAuthenticationStorageBridge(output) {
|
function buildAuthenticationStorageBridge(output) {
|
||||||
const result = spawn("go", ["build", "-o", output, "./cmd/tht"], {
|
const result = spawn("go", ["build", "-o", output, "./cmd/tht"], {
|
||||||
cwd: thtRoot,
|
cwd: thtRoot,
|
||||||
@@ -322,7 +337,7 @@ function cleanBackendEnvironment(overrides) {
|
|||||||
"SETTINGS_FILE", "THT_MAINTENANCE_FILE", "THT_WORKSPACE_REGISTRY_ROOT", "THT_WORKSPACE_SECRET_STORE_ROOT",
|
"SETTINGS_FILE", "THT_MAINTENANCE_FILE", "THT_WORKSPACE_REGISTRY_ROOT", "THT_WORKSPACE_SECRET_STORE_ROOT",
|
||||||
"THT_WORKSPACE_SECRET_RUNTIME_ROOT", "THT_WORKSPACE_GIT_REMOTE", "THT_WORKSPACE_GIT_BRANCH",
|
"THT_WORKSPACE_SECRET_RUNTIME_ROOT", "THT_WORKSPACE_GIT_REMOTE", "THT_WORKSPACE_GIT_BRANCH",
|
||||||
"THT_WORKSPACE_SECRET_ROOTS", "THT_WORKSPACE_INSTALLATION_ID", "THT_LEGACY_WORKSPACE_MODE",
|
"THT_WORKSPACE_SECRET_ROOTS", "THT_WORKSPACE_INSTALLATION_ID", "THT_LEGACY_WORKSPACE_MODE",
|
||||||
"THT_DWH_PRECHECK", "THT_INTERNAL_QDRANT_URL", "THT_CONFIG",
|
"THT_DWH_PRECHECK", "THT_INTERNAL_QDRANT_URL", "THT_CONFIG", "THT_TASK15_SENTINEL",
|
||||||
]) delete env[name];
|
]) delete env[name];
|
||||||
for (const name of Object.keys(env)) {
|
for (const name of Object.keys(env)) {
|
||||||
if (name.startsWith("THT_WS_")) delete env[name];
|
if (name.startsWith("THT_WS_")) delete env[name];
|
||||||
@@ -331,6 +346,7 @@ function cleanBackendEnvironment(overrides) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
export async function createAuthenticationStack({ withF1Workspace = false } = {}) {
|
export async function createAuthenticationStack({ withF1Workspace = false } = {}) {
|
||||||
|
const credentials = resolveFixtureCredentials();
|
||||||
const root = mkdtempSync(join(realpathSync(tmpdir()), "thothii-auth-e2e-"));
|
const root = mkdtempSync(join(realpathSync(tmpdir()), "thothii-auth-e2e-"));
|
||||||
secureDirectory(root);
|
secureDirectory(root);
|
||||||
const stateRoot = join(root, "auth-state");
|
const stateRoot = join(root, "auth-state");
|
||||||
@@ -358,10 +374,10 @@ export async function createAuthenticationStack({ withF1Workspace = false } = {}
|
|||||||
directory: providerRoot,
|
directory: providerRoot,
|
||||||
registration: {
|
registration: {
|
||||||
clientId: FIXTURE_CLIENT_ID,
|
clientId: FIXTURE_CLIENT_ID,
|
||||||
clientSecret: FIXTURE_CLIENT_SECRET,
|
clientSecret: credentials.clientSecret,
|
||||||
redirectUri: new URL("/api/auth/oidc/callback", publicUrl).href,
|
redirectUri: new URL("/api/auth/oidc/callback", publicUrl).href,
|
||||||
},
|
},
|
||||||
apiToken: FIXTURE_API_TOKEN,
|
apiToken: credentials.apiToken,
|
||||||
});
|
});
|
||||||
await buildAuthenticationStorageBridge(authStorageBinary);
|
await buildAuthenticationStorageBridge(authStorageBinary);
|
||||||
const localPassword = "e2e-local-password";
|
const localPassword = "e2e-local-password";
|
||||||
@@ -395,8 +411,8 @@ export async function createAuthenticationStack({ withF1Workspace = false } = {}
|
|||||||
}));
|
}));
|
||||||
function writeOidcSecrets(variant) {
|
function writeOidcSecrets(variant) {
|
||||||
if (!OIDC_CREDENTIAL_VARIANTS.has(variant)) throw safeError("e2e_oidc_credential_variant_invalid");
|
if (!OIDC_CREDENTIAL_VARIANTS.has(variant)) throw safeError("e2e_oidc_credential_variant_invalid");
|
||||||
const selectedClientSecret = variant === "wrong-client-secret" ? WRONG_CLIENT_SECRET : FIXTURE_CLIENT_SECRET;
|
const selectedClientSecret = variant === "wrong-client-secret" ? WRONG_CLIENT_SECRET : credentials.clientSecret;
|
||||||
const selectedApiToken = variant === "wrong-api-token" ? WRONG_API_TOKEN : FIXTURE_API_TOKEN;
|
const selectedApiToken = variant === "wrong-api-token" ? WRONG_API_TOKEN : credentials.apiToken;
|
||||||
writeSecure(secretsFile, [
|
writeSecure(secretsFile, [
|
||||||
`THT_OIDC_CLIENT_SECRET=${selectedClientSecret}`,
|
`THT_OIDC_CLIENT_SECRET=${selectedClientSecret}`,
|
||||||
`THT_AUTHENTIK_API_TOKEN=${selectedApiToken}`,
|
`THT_AUTHENTIK_API_TOKEN=${selectedApiToken}`,
|
||||||
@@ -539,7 +555,7 @@ export async function createAuthenticationStack({ withF1Workspace = false } = {}
|
|||||||
method: "POST",
|
method: "POST",
|
||||||
body: new URLSearchParams({
|
body: new URLSearchParams({
|
||||||
client_id: FIXTURE_CLIENT_ID,
|
client_id: FIXTURE_CLIENT_ID,
|
||||||
client_secret: FIXTURE_CLIENT_SECRET,
|
client_secret: credentials.clientSecret,
|
||||||
}).toString(),
|
}).toString(),
|
||||||
});
|
});
|
||||||
const deviceCode = typeof device.body?.device_code === "string" ? device.body.device_code : "";
|
const deviceCode = typeof device.body?.device_code === "string" ? device.body.device_code : "";
|
||||||
@@ -548,12 +564,12 @@ export async function createAuthenticationStack({ withF1Workspace = false } = {}
|
|||||||
body: new URLSearchParams({
|
body: new URLSearchParams({
|
||||||
grant_type: "urn:ietf:params:oauth:grant-type:device_code",
|
grant_type: "urn:ietf:params:oauth:grant-type:device_code",
|
||||||
client_id: FIXTURE_CLIENT_ID,
|
client_id: FIXTURE_CLIENT_ID,
|
||||||
client_secret: FIXTURE_CLIENT_SECRET,
|
client_secret: credentials.clientSecret,
|
||||||
device_code: deviceCode,
|
device_code: deviceCode,
|
||||||
}).toString(),
|
}).toString(),
|
||||||
});
|
});
|
||||||
const groups = await providerJson(`${provider.baseUrl}/api/v3/core/groups/?name=fixture-users`, provider.caFile, {
|
const groups = await providerJson(`${provider.baseUrl}/api/v3/core/groups/?name=fixture-users`, provider.caFile, {
|
||||||
headers: { authorization: `Bearer ${FIXTURE_API_TOKEN}` },
|
headers: { authorization: `Bearer ${credentials.apiToken}` },
|
||||||
});
|
});
|
||||||
return {
|
return {
|
||||||
discovery: discovery.status === 200 && discovery.body?.issuer === provider.issuer,
|
discovery: discovery.status === 200 && discovery.body?.issuer === provider.issuer,
|
||||||
@@ -562,6 +578,7 @@ export async function createAuthenticationStack({ withF1Workspace = false } = {}
|
|||||||
&& typeof device.body?.verification_uri === "string",
|
&& typeof device.body?.verification_uri === "string",
|
||||||
deviceToken: deviceToken.status === 200 && typeof deviceToken.body?.id_token === "string",
|
deviceToken: deviceToken.status === 200 && typeof deviceToken.body?.id_token === "string",
|
||||||
groupList: groups.status === 200 && groups.body?.results?.[0]?.name === "fixture-users",
|
groupList: groups.status === 200 && groups.body?.results?.[0]?.name === "fixture-users",
|
||||||
|
runtimeCredential: credentials.runtime,
|
||||||
};
|
};
|
||||||
},
|
},
|
||||||
setOidcIdentity(identity) {
|
setOidcIdentity(identity) {
|
||||||
|
|||||||
@@ -30,6 +30,7 @@ output="$temporary_root/playwright"
|
|||||||
mkdir -p "$output"
|
mkdir -p "$output"
|
||||||
sentinel="task15-sentinel-$(openssl rand -hex 24)"
|
sentinel="task15-sentinel-$(openssl rand -hex 24)"
|
||||||
export THT_TASK15_SENTINEL="$sentinel"
|
export THT_TASK15_SENTINEL="$sentinel"
|
||||||
|
readonly browser_specs=(e2e/auth.spec.ts e2e/f1.spec.ts)
|
||||||
|
|
||||||
sanitize_failure_log() {
|
sanitize_failure_log() {
|
||||||
sed -E \
|
sed -E \
|
||||||
@@ -41,7 +42,7 @@ sanitize_failure_log() {
|
|||||||
|
|
||||||
if ! (
|
if ! (
|
||||||
cd "$root/frontend"
|
cd "$root/frontend"
|
||||||
THT_E2E_AUTH_STACK=1 "$playwright" test e2e/auth.spec.ts --workers=1 --output="$output"
|
THT_E2E_AUTH_STACK=1 "$playwright" test "${browser_specs[@]}" --workers=1 --output="$output"
|
||||||
) >"$log" 2>&1; then
|
) >"$log" 2>&1; then
|
||||||
echo "authentication smoke: hermetic browser suite failed" >&2
|
echo "authentication smoke: hermetic browser suite failed" >&2
|
||||||
sanitize_failure_log
|
sanitize_failure_log
|
||||||
@@ -53,4 +54,4 @@ if rg -a -Fq -- "$sentinel" "$log" "$output"; then
|
|||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
printf 'authentication smoke: hermetic OIDC/browser suite passed on Node %s\n' "$actual_node"
|
printf 'authentication smoke: hermetic OIDC/F1 browser suite passed on Node %s\n' "$actual_node"
|
||||||
|
|||||||
@@ -986,9 +986,30 @@ task13_core_id() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
task13_assert_maintenance_auth_isolation() {
|
task13_assert_maintenance_auth_isolation() {
|
||||||
|
local rendered
|
||||||
|
rendered="$TASK13_TMP/maintenance-auth-isolation.json"
|
||||||
|
if ! task13_compose config --format json >"$rendered" 2>>"$TASK13_LOG"; then
|
||||||
|
task13_log_failure "render workspace maintenance mount isolation"
|
||||||
|
fi
|
||||||
|
node - "$rendered" <<'NODE'
|
||||||
|
const config = JSON.parse(require("node:fs").readFileSync(process.argv[2], "utf8"));
|
||||||
|
const maintenance = config.services?.["workspace-maintenance"];
|
||||||
|
if (!maintenance || !Array.isArray(maintenance.volumes)) process.exit(1);
|
||||||
|
if (maintenance.volumes.some((mount) => mount?.target === "/run/thothii-auth" || mount?.target === "/data/auth")) {
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
NODE
|
||||||
|
task13_compose_logged "seed core authentication isolation sentinel" exec -T core sh -ceu \
|
||||||
|
': > /data/auth/task13-maintenance-isolation-sentinel'
|
||||||
task13_compose_logged "workspace maintenance auth isolation" \
|
task13_compose_logged "workspace maintenance auth isolation" \
|
||||||
--profile workspace-maintenance run --rm --no-deps --entrypoint sh workspace-maintenance -ceu \
|
--profile workspace-maintenance run --rm --no-deps --entrypoint sh workspace-maintenance -ceu \
|
||||||
'test ! -e /run/thothii-auth && test ! -e /data/auth'
|
'test ! -e /run/thothii-auth
|
||||||
|
test -d /data/auth
|
||||||
|
test ! -e /data/auth/task13-maintenance-isolation-sentinel'
|
||||||
|
task13_compose_logged "verify core authentication isolation sentinel" exec -T core sh -ceu \
|
||||||
|
'test -f /data/auth/task13-maintenance-isolation-sentinel'
|
||||||
|
task13_compose_logged "remove core authentication isolation sentinel" exec -T core sh -ceu \
|
||||||
|
'rm -f /data/auth/task13-maintenance-isolation-sentinel'
|
||||||
}
|
}
|
||||||
|
|
||||||
task13_assert_local_auth_lifecycle() {
|
task13_assert_local_auth_lifecycle() {
|
||||||
|
|||||||
@@ -82,6 +82,8 @@ type PreflightResult struct {
|
|||||||
Manifest Manifest
|
Manifest Manifest
|
||||||
Entries []ArchiveEntryMetadata
|
Entries []ArchiveEntryMetadata
|
||||||
archive *verifiedArchive
|
archive *verifiedArchive
|
||||||
|
stagingRoot string
|
||||||
|
freeBytes func(string) (uint64, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
type verifiedArchive struct {
|
type verifiedArchive struct {
|
||||||
@@ -189,7 +191,8 @@ func Preflight(ctx context.Context, installation config.Installation, request Pr
|
|||||||
return PreflightResult{}, errors.New("restore staging requirement exceeds supported size")
|
return PreflightResult{}, errors.New("restore staging requirement exceeds supported size")
|
||||||
}
|
}
|
||||||
requiredWithStaging := requiredBytes + stagingBytes
|
requiredWithStaging := requiredBytes + stagingBytes
|
||||||
freeBytes, err := dependencies.FreeBytes(installation.ProjectDirectory)
|
stagingCapacityPath := installation.ControlDirectory()
|
||||||
|
freeBytes, err := dependencies.FreeBytes(stagingCapacityPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return PreflightResult{}, fmt.Errorf("check free disk space: %w", err)
|
return PreflightResult{}, fmt.Errorf("check free disk space: %w", err)
|
||||||
}
|
}
|
||||||
@@ -219,7 +222,9 @@ func Preflight(ctx context.Context, installation config.Installation, request Pr
|
|||||||
return PreflightResult{
|
return PreflightResult{
|
||||||
ArchivePath: archivePath, ArchiveSize: openedInfo.Size(), RequiredBytes: requiredBytes,
|
ArchivePath: archivePath, ArchiveSize: openedInfo.Size(), RequiredBytes: requiredBytes,
|
||||||
Manifest: manifest, Entries: metadata,
|
Manifest: manifest, Entries: metadata,
|
||||||
archive: &verifiedArchive{file: archiveFile, info: openedInfo, digest: finalDigest, limits: limits},
|
archive: &verifiedArchive{file: archiveFile, info: openedInfo, digest: finalDigest, limits: limits},
|
||||||
|
stagingRoot: filepath.Join(stagingCapacityPath, "restore-staging"),
|
||||||
|
freeBytes: dependencies.FreeBytes,
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -259,7 +264,20 @@ func (result PreflightResult) StageArchive(ctx context.Context) (_ *stagedArchiv
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
directory, err := os.MkdirTemp("", "tht-restore-stage-")
|
if result.stagingRoot == "" || result.freeBytes == nil {
|
||||||
|
return nil, errors.New("backup archive has no controlled staging reservation")
|
||||||
|
}
|
||||||
|
if err := ensurePrivateStagingRoot(result.stagingRoot); err != nil {
|
||||||
|
return nil, fmt.Errorf("create private restore staging root: %w", err)
|
||||||
|
}
|
||||||
|
freeBytes, err := result.freeBytes(result.stagingRoot)
|
||||||
|
if err != nil {
|
||||||
|
return nil, errors.New("check private restore staging capacity")
|
||||||
|
}
|
||||||
|
if result.ArchiveSize < 0 || freeBytes < uint64(result.ArchiveSize) {
|
||||||
|
return nil, errors.New("insufficient free disk space for private restore staging archive")
|
||||||
|
}
|
||||||
|
directory, err := os.MkdirTemp(result.stagingRoot, "archive-")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, errors.New("create private restore staging directory")
|
return nil, errors.New("create private restore staging directory")
|
||||||
}
|
}
|
||||||
@@ -324,6 +342,27 @@ func (result PreflightResult) StageArchive(ctx context.Context) (_ *stagedArchiv
|
|||||||
return staged, nil
|
return staged, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func ensurePrivateStagingRoot(root string) error {
|
||||||
|
if !filepath.IsAbs(root) || filepath.Clean(root) != root {
|
||||||
|
return errors.New("restore staging root is invalid")
|
||||||
|
}
|
||||||
|
if err := os.Mkdir(root, 0o700); err != nil && !errors.Is(err, os.ErrExist) {
|
||||||
|
return errors.New("restore staging root is unavailable")
|
||||||
|
}
|
||||||
|
info, err := os.Lstat(root)
|
||||||
|
if err != nil || !info.IsDir() || info.Mode()&os.ModeSymlink != 0 {
|
||||||
|
return errors.New("restore staging root is unsafe")
|
||||||
|
}
|
||||||
|
if err := os.Chmod(root, 0o700); err != nil {
|
||||||
|
return errors.New("restore staging root cannot be protected")
|
||||||
|
}
|
||||||
|
info, err = os.Lstat(root)
|
||||||
|
if err != nil || !info.IsDir() || info.Mode()&os.ModeSymlink != 0 || info.Mode().Perm() != 0o700 {
|
||||||
|
return errors.New("restore staging root protection is invalid")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
// Close removes only the staging file and directory created by StageArchive.
|
// Close removes only the staging file and directory created by StageArchive.
|
||||||
func (staged *stagedArchive) Close() error {
|
func (staged *stagedArchive) Close() error {
|
||||||
if staged == nil {
|
if staged == nil {
|
||||||
|
|||||||
@@ -57,6 +57,49 @@ func TestPreflightReturnsValidatedMetadataAndCallsAllTargetChecksWithoutExtracti
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestPreflightStagesOnTheAccountedInstallationFilesystemInsteadOfTMPDIR(t *testing.T) {
|
||||||
|
installation := preflightTestInstallation(t)
|
||||||
|
if err := os.MkdirAll(installation.ControlDirectory(), 0o700); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
archive := filepath.Join(t.TempDir(), "valid.zip")
|
||||||
|
writePreflightArchive(t, archive, preflightArchiveSpec{
|
||||||
|
entries: []preflightArchiveEntry{{path: "configuration/operator.env", body: []byte("safe")}},
|
||||||
|
})
|
||||||
|
|
||||||
|
blockedTMPDIR := filepath.Join(t.TempDir(), "not-a-directory")
|
||||||
|
if err := os.WriteFile(blockedTMPDIR, []byte("blocked"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Setenv("TMPDIR", blockedTMPDIR)
|
||||||
|
|
||||||
|
var capacityTargets []string
|
||||||
|
dependencies := permissivePreflightDependencies()
|
||||||
|
dependencies.FreeBytes = func(target string) (uint64, error) {
|
||||||
|
capacityTargets = append(capacityTargets, target)
|
||||||
|
return 1 << 30, nil
|
||||||
|
}
|
||||||
|
result, err := Preflight(context.Background(), installation, PreflightRequest{Archive: archive, Confirm: true}, dependencies)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer result.CloseArchive()
|
||||||
|
|
||||||
|
staged, err := result.StageArchive(context.Background())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer staged.Close()
|
||||||
|
|
||||||
|
if len(capacityTargets) == 0 || capacityTargets[0] != installation.ControlDirectory() {
|
||||||
|
t.Fatalf("free-space targets = %q, want installation control directory %q", capacityTargets, installation.ControlDirectory())
|
||||||
|
}
|
||||||
|
stagingRoot := filepath.Join(installation.ControlDirectory(), "restore-staging")
|
||||||
|
if relative, err := filepath.Rel(stagingRoot, staged.directory); err != nil || relative == "." || strings.HasPrefix(relative, "..") {
|
||||||
|
t.Fatalf("staging directory = %q, want a child of %q", staged.directory, stagingRoot)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestPreflightRejectsAdversarialArchiveEntriesAndManifestIdentity(t *testing.T) {
|
func TestPreflightRejectsAdversarialArchiveEntriesAndManifestIdentity(t *testing.T) {
|
||||||
installation := preflightTestInstallation(t)
|
installation := preflightTestInstallation(t)
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
@@ -359,6 +402,9 @@ func TestPreflightRevalidationRefusesAnArchivePathThatWasReplaced(t *testing.T)
|
|||||||
|
|
||||||
func TestPreflightStagesArchiveIntoImmutablePrivateBytes(t *testing.T) {
|
func TestPreflightStagesArchiveIntoImmutablePrivateBytes(t *testing.T) {
|
||||||
installation := preflightTestInstallation(t)
|
installation := preflightTestInstallation(t)
|
||||||
|
if err := os.MkdirAll(installation.ControlDirectory(), 0o700); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
archive := filepath.Join(t.TempDir(), "checked.zip")
|
archive := filepath.Join(t.TempDir(), "checked.zip")
|
||||||
writePreflightArchive(t, archive, preflightArchiveSpec{
|
writePreflightArchive(t, archive, preflightArchiveSpec{
|
||||||
entries: []preflightArchiveEntry{{path: "configuration/operator.env", body: []byte("before")}},
|
entries: []preflightArchiveEntry{{path: "configuration/operator.env", body: []byte("before")}},
|
||||||
|
|||||||
@@ -38,7 +38,7 @@ type restoreDependencies struct {
|
|||||||
// public Create, which would re-acquire the non-reentrant lock and deadlock the transaction.
|
// public Create, which would re-acquire the non-reentrant lock and deadlock the transaction.
|
||||||
checkpoint func(context.Context, *lifecycle.Transaction, config.Installation, CreateRequest) (Result, error)
|
checkpoint func(context.Context, *lifecycle.Transaction, config.Installation, CreateRequest) (Result, error)
|
||||||
prepareRecovery func(context.Context, config.Installation, string) (PreflightResult, error)
|
prepareRecovery func(context.Context, config.Installation, string) (PreflightResult, error)
|
||||||
recover func(context.Context, config.Installation, PreflightResult, bool) error
|
recover func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool) error
|
||||||
cleanupCheckpoint func(string) error
|
cleanupCheckpoint func(string) error
|
||||||
acquireTransaction func(config.Installation) (*lifecycle.Transaction, error)
|
acquireTransaction func(config.Installation) (*lifecycle.Transaction, error)
|
||||||
runner archiveRunner
|
runner archiveRunner
|
||||||
@@ -119,11 +119,42 @@ func restoreWithDependencies(ctx context.Context, installation config.Installati
|
|||||||
}
|
}
|
||||||
defer recovery.CloseArchive()
|
defer recovery.CloseArchive()
|
||||||
|
|
||||||
|
// Both immutable archives are staged while the lifecycle transaction is held and before
|
||||||
|
// maintenance, service stops, or destination writes. Keeping both files open reserves their
|
||||||
|
// combined staging capacity, so checkpoint recovery never needs a new allocation after a
|
||||||
|
// candidate mutation has begun.
|
||||||
|
candidateStage, err := preflight.StageArchive(ctx)
|
||||||
|
if err != nil {
|
||||||
|
cleanupErr := deps.cleanupCheckpoint(checkpoint.Path)
|
||||||
|
return result, errors.Join(err, cleanupErr)
|
||||||
|
}
|
||||||
|
defer func() {
|
||||||
|
if closeErr := candidateStage.Close(); closeErr != nil {
|
||||||
|
result = RestoreResult{}
|
||||||
|
resultErr = errors.Join(resultErr, closeErr)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
recoveryStage, err := recovery.StageArchive(ctx)
|
||||||
|
if err != nil {
|
||||||
|
cleanupErr := deps.cleanupCheckpoint(checkpoint.Path)
|
||||||
|
return result, errors.Join(err, cleanupErr)
|
||||||
|
}
|
||||||
|
defer func() {
|
||||||
|
if closeErr := recoveryStage.Close(); closeErr != nil {
|
||||||
|
result = RestoreResult{}
|
||||||
|
resultErr = errors.Join(resultErr, closeErr)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
if err := ensureCombinedRestoreCapacity(preflight, recovery); err != nil {
|
||||||
|
cleanupErr := deps.cleanupCheckpoint(checkpoint.Path)
|
||||||
|
return result, errors.Join(err, cleanupErr)
|
||||||
|
}
|
||||||
|
|
||||||
state := restoreTransactionState{}
|
state := restoreTransactionState{}
|
||||||
defer func() {
|
defer func() {
|
||||||
if state.recoveryRequired(resultErr) {
|
if state.recoveryRequired(resultErr) {
|
||||||
recoveryContext, cancel := boundedCleanupContext()
|
recoveryContext, cancel := boundedCleanupContext()
|
||||||
recoveryErr := deps.recover(recoveryContext, installation, recovery, state.wasRunning)
|
recoveryErr := deps.recover(recoveryContext, installation, recovery, recoveryStage, state.wasRunning)
|
||||||
cancel()
|
cancel()
|
||||||
if recoveryErr != nil {
|
if recoveryErr != nil {
|
||||||
resultErr = errors.Join(resultErr, fmt.Errorf("restore recovery checkpoint: %w", recoveryErr))
|
resultErr = errors.Join(resultErr, fmt.Errorf("restore recovery checkpoint: %w", recoveryErr))
|
||||||
@@ -212,18 +243,8 @@ func restoreWithDependencies(ctx context.Context, installation config.Installati
|
|||||||
return result, err
|
return result, err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
staged, err := preflight.StageArchive(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return result, err
|
|
||||||
}
|
|
||||||
defer func() {
|
|
||||||
if closeErr := staged.Close(); closeErr != nil {
|
|
||||||
result = RestoreResult{}
|
|
||||||
resultErr = errors.Join(resultErr, closeErr)
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
state.mutated = true
|
state.mutated = true
|
||||||
if err := restoreVerifiedEntries(ctx, installation, preflight, staged.file, deps.restoreFile, deps.restoreVolume); err != nil {
|
if err := restoreVerifiedEntries(ctx, installation, preflight, candidateStage.file, deps.restoreFile, deps.restoreVolume); err != nil {
|
||||||
return result, err
|
return result, err
|
||||||
}
|
}
|
||||||
if err := deps.resetAuthenticationState(ctx, installation, deps.runner); err != nil {
|
if err := deps.resetAuthenticationState(ctx, installation, deps.runner); err != nil {
|
||||||
@@ -244,6 +265,24 @@ func restoreWithDependencies(ctx context.Context, installation config.Installati
|
|||||||
return result, nil
|
return result, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func ensureCombinedRestoreCapacity(candidate, recovery PreflightResult) error {
|
||||||
|
if candidate.freeBytes == nil || candidate.stagingRoot == "" || candidate.stagingRoot != recovery.stagingRoot {
|
||||||
|
return errors.New("candidate and recovery archives do not share controlled restore staging")
|
||||||
|
}
|
||||||
|
if recovery.RequiredBytes > ^uint64(0)-candidate.RequiredBytes {
|
||||||
|
return errors.New("combined candidate and recovery restore capacity exceeds supported size")
|
||||||
|
}
|
||||||
|
required := candidate.RequiredBytes + recovery.RequiredBytes
|
||||||
|
freeBytes, err := candidate.freeBytes(candidate.stagingRoot)
|
||||||
|
if err != nil {
|
||||||
|
return errors.New("check combined candidate and recovery staging capacity")
|
||||||
|
}
|
||||||
|
if freeBytes < required {
|
||||||
|
return fmt.Errorf("insufficient free disk space for combined candidate and recovery staging: need %d bytes, have %d", required, freeBytes)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
func verifyRestoreTransaction(ctx context.Context, installation config.Installation, deps restoreDependencies) error {
|
func verifyRestoreTransaction(ctx context.Context, installation config.Installation, deps restoreDependencies) error {
|
||||||
for _, name := range []string{"health", "doctor", "pi", "workspace"} {
|
for _, name := range []string{"health", "doctor", "pi", "workspace"} {
|
||||||
check := deps.verify[name]
|
check := deps.verify[name]
|
||||||
|
|||||||
@@ -75,8 +75,8 @@ func productionRestoreDependencies(installation config.Installation) restoreDepe
|
|||||||
deps.prepareRecovery = func(ctx context.Context, target config.Installation, path string) (PreflightResult, error) {
|
deps.prepareRecovery = func(ctx context.Context, target config.Installation, path string) (PreflightResult, error) {
|
||||||
return deps.preflight(ctx, target, PreflightRequest{Archive: path, Confirm: true, AllowExternalSecrets: true})
|
return deps.preflight(ctx, target, PreflightRequest{Archive: path, Confirm: true, AllowExternalSecrets: true})
|
||||||
}
|
}
|
||||||
deps.recover = func(ctx context.Context, target config.Installation, recovery PreflightResult, wasRunning bool) error {
|
deps.recover = func(ctx context.Context, target config.Installation, recovery PreflightResult, staged *stagedArchive, wasRunning bool) error {
|
||||||
return recoverRestoreTransaction(ctx, target, recovery, wasRunning, deps)
|
return recoverRestoreTransaction(ctx, target, recovery, staged, wasRunning, deps)
|
||||||
}
|
}
|
||||||
return deps
|
return deps
|
||||||
}
|
}
|
||||||
@@ -88,7 +88,10 @@ func cleanupRecoveryCheckpoint(path string) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func recoverRestoreTransaction(ctx context.Context, installation config.Installation, recovery PreflightResult, wasRunning bool, deps restoreDependencies) (resultErr error) {
|
func recoverRestoreTransaction(ctx context.Context, installation config.Installation, recovery PreflightResult, staged *stagedArchive, wasRunning bool, deps restoreDependencies) (resultErr error) {
|
||||||
|
if staged == nil || staged.file == nil {
|
||||||
|
return errors.New("recovery checkpoint was not staged before restore mutation")
|
||||||
|
}
|
||||||
if err := runCompose(ctx, installation, deps.runner, "stop"); err != nil {
|
if err := runCompose(ctx, installation, deps.runner, "stop"); err != nil {
|
||||||
resultErr = errors.Join(resultErr, err)
|
resultErr = errors.Join(resultErr, err)
|
||||||
// The first stop may already have taken effect before Docker lost its response. Retry the
|
// The first stop may already have taken effect before Docker lost its response. Retry the
|
||||||
@@ -97,15 +100,6 @@ func recoverRestoreTransaction(ctx context.Context, installation config.Installa
|
|||||||
return errors.Join(resultErr, retryErr)
|
return errors.Join(resultErr, retryErr)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
staged, err := recovery.StageArchive(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return errors.Join(resultErr, err)
|
|
||||||
}
|
|
||||||
defer func() {
|
|
||||||
if closeErr := staged.Close(); closeErr != nil {
|
|
||||||
resultErr = errors.Join(resultErr, closeErr)
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
if err := restoreVerifiedEntries(ctx, installation, recovery, staged.file, deps.restoreFile, deps.restoreVolume); err != nil {
|
if err := restoreVerifiedEntries(ctx, installation, recovery, staged.file, deps.restoreFile, deps.restoreVolume); err != nil {
|
||||||
return errors.Join(resultErr, err)
|
return errors.Join(resultErr, err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -120,14 +120,15 @@ func TestRestoreStoppedInstallationRunsCheckpointRestoreAndVerification(t *testi
|
|||||||
var events []string
|
var events []string
|
||||||
var checkpointRequest CreateRequest
|
var checkpointRequest CreateRequest
|
||||||
deps := restoreTestDependencies(t, runner)
|
deps := restoreTestDependencies(t, runner)
|
||||||
|
prepareRecovery := deps.prepareRecovery
|
||||||
deps.checkpoint = func(_ context.Context, _ *lifecycle.Transaction, _ config.Installation, request CreateRequest) (Result, error) {
|
deps.checkpoint = func(_ context.Context, _ *lifecycle.Transaction, _ config.Installation, request CreateRequest) (Result, error) {
|
||||||
events = append(events, "checkpoint")
|
events = append(events, "checkpoint")
|
||||||
checkpointRequest = request
|
checkpointRequest = request
|
||||||
return Result{Path: "/tmp/checkpoint.zip"}, nil
|
return Result{Path: "/tmp/checkpoint.zip"}, nil
|
||||||
}
|
}
|
||||||
deps.prepareRecovery = func(context.Context, config.Installation, string) (PreflightResult, error) {
|
deps.prepareRecovery = func(ctx context.Context, target config.Installation, path string) (PreflightResult, error) {
|
||||||
events = append(events, "prepare-recovery")
|
events = append(events, "prepare-recovery")
|
||||||
return PreflightResult{}, nil
|
return prepareRecovery(ctx, target, path)
|
||||||
}
|
}
|
||||||
deps.cleanupCheckpoint = func(string) error {
|
deps.cleanupCheckpoint = func(string) error {
|
||||||
events = append(events, "cleanup-checkpoint")
|
events = append(events, "cleanup-checkpoint")
|
||||||
@@ -167,6 +168,49 @@ func TestRestoreStoppedInstallationRunsCheckpointRestoreAndVerification(t *testi
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRestoreRejectsCombinedCandidateAndRecoveryStagingCapacityBeforeMutation(t *testing.T) {
|
||||||
|
installation := preflightTestInstallation(t)
|
||||||
|
candidateArchive := restoreArchive(t)
|
||||||
|
recoveryArchive := restoreArchive(t)
|
||||||
|
runner := newBackupRunner(installation, false)
|
||||||
|
dependencies := restoreTestDependencies(t, runner)
|
||||||
|
|
||||||
|
capacityChecks := 0
|
||||||
|
preflightDependencies := permissivePreflightDependencies()
|
||||||
|
preflightDependencies.FreeBytes = func(string) (uint64, error) {
|
||||||
|
capacityChecks++
|
||||||
|
if capacityChecks == 5 {
|
||||||
|
return 0, nil
|
||||||
|
}
|
||||||
|
return 1 << 30, nil
|
||||||
|
}
|
||||||
|
dependencies.preflight = func(ctx context.Context, target config.Installation, request PreflightRequest) (PreflightResult, error) {
|
||||||
|
return Preflight(ctx, target, request, preflightDependencies)
|
||||||
|
}
|
||||||
|
dependencies.checkpoint = func(context.Context, *lifecycle.Transaction, config.Installation, CreateRequest) (Result, error) {
|
||||||
|
return Result{Path: recoveryArchive}, nil
|
||||||
|
}
|
||||||
|
dependencies.prepareRecovery = func(ctx context.Context, target config.Installation, path string) (PreflightResult, error) {
|
||||||
|
return Preflight(ctx, target, PreflightRequest{Archive: path, Confirm: true, AllowExternalSecrets: true}, preflightDependencies)
|
||||||
|
}
|
||||||
|
mutated := false
|
||||||
|
dependencies.restoreFile = func(context.Context, config.Installation, ArchiveEntryMetadata, io.Reader) error {
|
||||||
|
mutated = true
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err := restoreWithDependencies(context.Background(), installation, RestoreRequest{Archive: candidateArchive, Confirm: true}, dependencies)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "staging") {
|
||||||
|
t.Fatalf("Restore() error = %v, want recovery staging capacity rejection", err)
|
||||||
|
}
|
||||||
|
if mutated {
|
||||||
|
t.Fatal("restore mutated the installation before reserving candidate and recovery staging capacity")
|
||||||
|
}
|
||||||
|
if capacityChecks != 5 {
|
||||||
|
t.Fatalf("free-space checks = %d, want candidate/recovery preflight and staging checks", capacityChecks)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestRestoreClosesTargetArchiveBeforeReleasingLifecycleLock(t *testing.T) {
|
func TestRestoreClosesTargetArchiveBeforeReleasingLifecycleLock(t *testing.T) {
|
||||||
installation := preflightTestInstallation(t)
|
installation := preflightTestInstallation(t)
|
||||||
archive := filepath.Join(t.TempDir(), "restore.zip")
|
archive := filepath.Join(t.TempDir(), "restore.zip")
|
||||||
@@ -301,7 +345,7 @@ func TestRestoreLifecycleLockExcludesCompetingTransactionsUntilTerminalCleanup(t
|
|||||||
gate("target-failure")
|
gate("target-failure")
|
||||||
return targetFailure
|
return targetFailure
|
||||||
}
|
}
|
||||||
deps.recover = func(context.Context, config.Installation, PreflightResult, bool) error {
|
deps.recover = func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool) error {
|
||||||
gate("recovery")
|
gate("recovery")
|
||||||
runner.running, runner.coreRunning = true, true
|
runner.running, runner.coreRunning = true, true
|
||||||
return nil
|
return nil
|
||||||
@@ -318,7 +362,7 @@ func TestRestoreLifecycleLockExcludesCompetingTransactionsUntilTerminalCleanup(t
|
|||||||
gate("target-failure")
|
gate("target-failure")
|
||||||
return targetFailure
|
return targetFailure
|
||||||
}
|
}
|
||||||
deps.recover = func(context.Context, config.Installation, PreflightResult, bool) error {
|
deps.recover = func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool) error {
|
||||||
gate("recovery-failure")
|
gate("recovery-failure")
|
||||||
return recoveryFailure
|
return recoveryFailure
|
||||||
}
|
}
|
||||||
@@ -335,7 +379,7 @@ func TestRestoreLifecycleLockExcludesCompetingTransactionsUntilTerminalCleanup(t
|
|||||||
cancel()
|
cancel()
|
||||||
return context.Canceled
|
return context.Canceled
|
||||||
}
|
}
|
||||||
deps.recover = func(context.Context, config.Installation, PreflightResult, bool) error {
|
deps.recover = func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool) error {
|
||||||
gate("recovery")
|
gate("recovery")
|
||||||
runner.running, runner.coreRunning = true, true
|
runner.running, runner.coreRunning = true, true
|
||||||
return nil
|
return nil
|
||||||
@@ -353,6 +397,11 @@ func TestRestoreLifecycleLockExcludesCompetingTransactionsUntilTerminalCleanup(t
|
|||||||
entries: []preflightArchiveEntry{{path: "configuration/operator.env", body: []byte("safe")}},
|
entries: []preflightArchiveEntry{{path: "configuration/operator.env", body: []byte("safe")}},
|
||||||
})
|
})
|
||||||
runner := &lifecycleGateRunner{fakeBackupRunner: newBackupRunner(installation, true)}
|
runner := &lifecycleGateRunner{fakeBackupRunner: newBackupRunner(installation, true)}
|
||||||
|
recoveryArchive := filepath.Join(t.TempDir(), "recovery.zip")
|
||||||
|
writePreflightArchive(t, recoveryArchive, preflightArchiveSpec{
|
||||||
|
installationID: fixture.installationID,
|
||||||
|
entries: []preflightArchiveEntry{{path: "configuration/operator.env", body: []byte("checkpoint")}},
|
||||||
|
})
|
||||||
stages := make(chan string)
|
stages := make(chan string)
|
||||||
continueStage := make(chan struct{})
|
continueStage := make(chan struct{})
|
||||||
gate := func(stage string) {
|
gate := func(stage string) {
|
||||||
@@ -363,6 +412,9 @@ func TestRestoreLifecycleLockExcludesCompetingTransactionsUntilTerminalCleanup(t
|
|||||||
caller, cancel := context.WithCancel(context.Background())
|
caller, cancel := context.WithCancel(context.Background())
|
||||||
defer cancel()
|
defer cancel()
|
||||||
deps := restoreTestDependencies(t, runner)
|
deps := restoreTestDependencies(t, runner)
|
||||||
|
deps.prepareRecovery = func(ctx context.Context, target config.Installation, _ string) (PreflightResult, error) {
|
||||||
|
return Preflight(ctx, target, PreflightRequest{Archive: recoveryArchive, Confirm: true, AllowExternalSecrets: true}, permissivePreflightDependencies())
|
||||||
|
}
|
||||||
deps.acquireTransaction = lifecycle.AcquireTransaction
|
deps.acquireTransaction = lifecycle.AcquireTransaction
|
||||||
deps.checkpoint = func(context.Context, *lifecycle.Transaction, config.Installation, CreateRequest) (Result, error) {
|
deps.checkpoint = func(context.Context, *lifecycle.Transaction, config.Installation, CreateRequest) (Result, error) {
|
||||||
gate("checkpoint")
|
gate("checkpoint")
|
||||||
@@ -474,6 +526,14 @@ func TestRestoreCannotApplyAStaleCheckpointOverAnInterleavedRestore(t *testing.T
|
|||||||
continueCheckpoint := make(chan struct{})
|
continueCheckpoint := make(chan struct{})
|
||||||
firstRunner := newBackupRunner(installation, true)
|
firstRunner := newBackupRunner(installation, true)
|
||||||
firstDeps := restoreTestDependencies(t, firstRunner)
|
firstDeps := restoreTestDependencies(t, firstRunner)
|
||||||
|
recoveryArchive := filepath.Join(t.TempDir(), "first-recovery.zip")
|
||||||
|
writePreflightArchive(t, recoveryArchive, preflightArchiveSpec{
|
||||||
|
installationID: fixture.installationID,
|
||||||
|
entries: []preflightArchiveEntry{{path: "configuration/operator.env", body: []byte("checkpoint")}},
|
||||||
|
})
|
||||||
|
firstDeps.prepareRecovery = func(ctx context.Context, target config.Installation, _ string) (PreflightResult, error) {
|
||||||
|
return Preflight(ctx, target, PreflightRequest{Archive: recoveryArchive, Confirm: true, AllowExternalSecrets: true}, permissivePreflightDependencies())
|
||||||
|
}
|
||||||
firstDeps.acquireTransaction = lifecycle.AcquireTransaction
|
firstDeps.acquireTransaction = lifecycle.AcquireTransaction
|
||||||
firstDeps.checkpoint = func(context.Context, *lifecycle.Transaction, config.Installation, CreateRequest) (Result, error) {
|
firstDeps.checkpoint = func(context.Context, *lifecycle.Transaction, config.Installation, CreateRequest) (Result, error) {
|
||||||
checkpointState = targetState
|
checkpointState = targetState
|
||||||
@@ -484,7 +544,7 @@ func TestRestoreCannotApplyAStaleCheckpointOverAnInterleavedRestore(t *testing.T
|
|||||||
firstDeps.restoreFile = func(context.Context, config.Installation, ArchiveEntryMetadata, io.Reader) error {
|
firstDeps.restoreFile = func(context.Context, config.Installation, ArchiveEntryMetadata, io.Reader) error {
|
||||||
return errors.New("first target mutation failed before changing state")
|
return errors.New("first target mutation failed before changing state")
|
||||||
}
|
}
|
||||||
firstDeps.recover = func(context.Context, config.Installation, PreflightResult, bool) error {
|
firstDeps.recover = func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool) error {
|
||||||
recoveryObserved = targetState
|
recoveryObserved = targetState
|
||||||
targetState = checkpointState
|
targetState = checkpointState
|
||||||
firstRunner.running, firstRunner.coreRunning = true, true
|
firstRunner.running, firstRunner.coreRunning = true, true
|
||||||
@@ -661,7 +721,7 @@ func TestRestoreFileFailureRollsBackSecretAwareCheckpointBeforeCleanup(t *testin
|
|||||||
return Result{Path: "/tmp/recovery.zip"}, nil
|
return Result{Path: "/tmp/recovery.zip"}, nil
|
||||||
}
|
}
|
||||||
var events []string
|
var events []string
|
||||||
deps.recover = func(context.Context, config.Installation, PreflightResult, bool) error {
|
deps.recover = func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool) error {
|
||||||
events = append(events, "recover")
|
events = append(events, "recover")
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@@ -701,7 +761,7 @@ func TestRestoreFailureAfterAuthenticationMutationRollsBackAndClearsRuntimeState
|
|||||||
events = append(events, "auth-runtime-reset-failed")
|
events = append(events, "auth-runtime-reset-failed")
|
||||||
return resetErr
|
return resetErr
|
||||||
}
|
}
|
||||||
deps.recover = func(context.Context, config.Installation, PreflightResult, bool) error {
|
deps.recover = func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool) error {
|
||||||
events = append(events, "secret-aware-recovery-and-reauth-reset")
|
events = append(events, "secret-aware-recovery-and-reauth-reset")
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@@ -727,7 +787,10 @@ func TestRestoreCleanupFailureDoesNotSuppressRollback(t *testing.T) {
|
|||||||
cleanupErr := errors.New("checkpoint cleanup failure")
|
cleanupErr := errors.New("checkpoint cleanup failure")
|
||||||
recovered := false
|
recovered := false
|
||||||
deps.restoreFile = func(context.Context, config.Installation, ArchiveEntryMetadata, io.Reader) error { return mutationErr }
|
deps.restoreFile = func(context.Context, config.Installation, ArchiveEntryMetadata, io.Reader) error { return mutationErr }
|
||||||
deps.recover = func(context.Context, config.Installation, PreflightResult, bool) error { recovered = true; return nil }
|
deps.recover = func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool) error {
|
||||||
|
recovered = true
|
||||||
|
return nil
|
||||||
|
}
|
||||||
deps.cleanupCheckpoint = func(string) error { return cleanupErr }
|
deps.cleanupCheckpoint = func(string) error { return cleanupErr }
|
||||||
|
|
||||||
_, err := restoreWithDependencies(context.Background(), installation, RestoreRequest{Archive: archive, Confirm: true}, deps)
|
_, err := restoreWithDependencies(context.Background(), installation, RestoreRequest{Archive: archive, Confirm: true}, deps)
|
||||||
@@ -783,7 +846,7 @@ func TestRestoreStartFailureRecoversPreviouslyRunningTarget(t *testing.T) {
|
|||||||
backingRunner := newBackupRunner(installation, true)
|
backingRunner := newBackupRunner(installation, true)
|
||||||
deps := restoreTestDependencies(t, failStartRestoreRunner{fakeBackupRunner: backingRunner})
|
deps := restoreTestDependencies(t, failStartRestoreRunner{fakeBackupRunner: backingRunner})
|
||||||
recovered := false
|
recovered := false
|
||||||
deps.recover = func(context.Context, config.Installation, PreflightResult, bool) error {
|
deps.recover = func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool) error {
|
||||||
recovered = true
|
recovered = true
|
||||||
backingRunner.running = true
|
backingRunner.running = true
|
||||||
return nil
|
return nil
|
||||||
@@ -806,7 +869,7 @@ func TestRestoreVerificationFailureRecoversPreviouslyRunningTarget(t *testing.T)
|
|||||||
verificationErr := errors.New("Pi is unavailable")
|
verificationErr := errors.New("Pi is unavailable")
|
||||||
deps.verify["pi"] = func(context.Context, config.Installation, archiveRunner) error { return verificationErr }
|
deps.verify["pi"] = func(context.Context, config.Installation, archiveRunner) error { return verificationErr }
|
||||||
recovered := false
|
recovered := false
|
||||||
deps.recover = func(context.Context, config.Installation, PreflightResult, bool) error {
|
deps.recover = func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool) error {
|
||||||
recovered = true
|
recovered = true
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@@ -884,7 +947,7 @@ func TestRestoreRecoversBehindBarrierForEveryVerificationFailure(t *testing.T) {
|
|||||||
}
|
}
|
||||||
var recoveryBarrierActive bool
|
var recoveryBarrierActive bool
|
||||||
var recoveryContext cleanupContextObservation
|
var recoveryContext cleanupContextObservation
|
||||||
deps.recover = func(ctx context.Context, _ config.Installation, _ PreflightResult, _ bool) error {
|
deps.recover = func(ctx context.Context, _ config.Installation, _ PreflightResult, _ *stagedArchive, _ bool) error {
|
||||||
recoveryContext = observeCleanupContext(ctx)
|
recoveryContext = observeCleanupContext(ctx)
|
||||||
recoveryBarrierActive = runner.maintenance
|
recoveryBarrierActive = runner.maintenance
|
||||||
runner.running, runner.coreRunning = true, true
|
runner.running, runner.coreRunning = true, true
|
||||||
@@ -924,7 +987,7 @@ func TestRestoreDoesNotRollbackAfterFinalDeactivationResponseLoss(t *testing.T)
|
|||||||
}
|
}
|
||||||
deps := restoreTestDependencies(t, runner)
|
deps := restoreTestDependencies(t, runner)
|
||||||
recoveryCalls := 0
|
recoveryCalls := 0
|
||||||
deps.recover = func(context.Context, config.Installation, PreflightResult, bool) error {
|
deps.recover = func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool) error {
|
||||||
recoveryCalls++
|
recoveryCalls++
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@@ -982,7 +1045,7 @@ func TestRestoreUsesBoundedRecoveryContextAfterPostMutationCancellation(t *testi
|
|||||||
}
|
}
|
||||||
var recoveryContext cleanupContextObservation
|
var recoveryContext cleanupContextObservation
|
||||||
var recoveryBarrierActive bool
|
var recoveryBarrierActive bool
|
||||||
deps.recover = func(ctx context.Context, _ config.Installation, _ PreflightResult, _ bool) error {
|
deps.recover = func(ctx context.Context, _ config.Installation, _ PreflightResult, _ *stagedArchive, _ bool) error {
|
||||||
recoveryContext = observeCleanupContext(ctx)
|
recoveryContext = observeCleanupContext(ctx)
|
||||||
recoveryBarrierActive = runner.maintenance
|
recoveryBarrierActive = runner.maintenance
|
||||||
runner.running, runner.coreRunning = true, true
|
runner.running, runner.coreRunning = true, true
|
||||||
@@ -1067,7 +1130,8 @@ func TestRecoverRestoreTransactionVerifiesRecoveredStateBeforeReturning(t *testi
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := recoverRestoreTransaction(context.Background(), installation, recovery, true, deps); err != nil {
|
staged := stageRecoveryForTest(t, installation, recovery)
|
||||||
|
if err := recoverRestoreTransaction(context.Background(), installation, recovery, staged, true, deps); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if got, want := checks, []string{"health", "doctor", "pi", "workspace"}; !equalStrings(got, want) {
|
if got, want := checks, []string{"health", "doctor", "pi", "workspace"}; !equalStrings(got, want) {
|
||||||
@@ -1096,7 +1160,8 @@ func TestRecoverRestoreTransactionFailsClosedForEveryVerification(t *testing.T)
|
|||||||
return verificationErr
|
return verificationErr
|
||||||
}
|
}
|
||||||
|
|
||||||
err = recoverRestoreTransaction(context.Background(), installation, recovery, true, deps)
|
staged := stageRecoveryForTest(t, installation, recovery)
|
||||||
|
err = recoverRestoreTransaction(context.Background(), installation, recovery, staged, true, deps)
|
||||||
if !errors.Is(err, verificationErr) {
|
if !errors.Is(err, verificationErr) {
|
||||||
t.Fatalf("recoverRestoreTransaction() error = %v, want %v", err, verificationErr)
|
t.Fatalf("recoverRestoreTransaction() error = %v, want %v", err, verificationErr)
|
||||||
}
|
}
|
||||||
@@ -1176,8 +1241,8 @@ func TestRestoreReleasesBarrierOnlyAfterVerifiedRecoveryFromLostResponse(t *test
|
|||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
deps.recover = func(ctx context.Context, target config.Installation, checkpoint PreflightResult, wasRunning bool) error {
|
deps.recover = func(ctx context.Context, target config.Installation, checkpoint PreflightResult, staged *stagedArchive, wasRunning bool) error {
|
||||||
return recoverRestoreTransaction(ctx, target, checkpoint, wasRunning, deps)
|
return recoverRestoreTransaction(ctx, target, checkpoint, staged, wasRunning, deps)
|
||||||
}
|
}
|
||||||
|
|
||||||
_, err = restoreWithDependencies(context.Background(), installation, RestoreRequest{Archive: archive, Confirm: true}, deps)
|
_, err = restoreWithDependencies(context.Background(), installation, RestoreRequest{Archive: archive, Confirm: true}, deps)
|
||||||
@@ -1274,7 +1339,7 @@ func TestRestoreCleansMaintenanceAfterMutationAndRollbackFailures(t *testing.T)
|
|||||||
deps.restoreFile = func(context.Context, config.Installation, ArchiveEntryMetadata, io.Reader) error {
|
deps.restoreFile = func(context.Context, config.Installation, ArchiveEntryMetadata, io.Reader) error {
|
||||||
return mutationErr
|
return mutationErr
|
||||||
}
|
}
|
||||||
deps.recover = func(context.Context, config.Installation, PreflightResult, bool) error {
|
deps.recover = func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool) error {
|
||||||
if test.recoveryErr == nil {
|
if test.recoveryErr == nil {
|
||||||
backing.running, backing.coreRunning = true, true
|
backing.running, backing.coreRunning = true, true
|
||||||
}
|
}
|
||||||
@@ -1505,8 +1570,26 @@ func equalStrings(got, want []string) bool {
|
|||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func stageRecoveryForTest(t *testing.T, installation config.Installation, recovery PreflightResult) *stagedArchive {
|
||||||
|
t.Helper()
|
||||||
|
if err := os.MkdirAll(installation.ControlDirectory(), 0o700); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
staged, err := recovery.StageArchive(context.Background())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() {
|
||||||
|
if err := staged.Close(); err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
return staged
|
||||||
|
}
|
||||||
|
|
||||||
func restoreTestDependencies(t *testing.T, runner archiveRunner) restoreDependencies {
|
func restoreTestDependencies(t *testing.T, runner archiveRunner) restoreDependencies {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
recoveryArchive := restoreArchive(t)
|
||||||
return restoreDependencies{
|
return restoreDependencies{
|
||||||
preflight: func(ctx context.Context, installation config.Installation, request PreflightRequest) (PreflightResult, error) {
|
preflight: func(ctx context.Context, installation config.Installation, request PreflightRequest) (PreflightResult, error) {
|
||||||
return Preflight(ctx, installation, request, permissivePreflightDependencies())
|
return Preflight(ctx, installation, request, permissivePreflightDependencies())
|
||||||
@@ -1514,10 +1597,10 @@ func restoreTestDependencies(t *testing.T, runner archiveRunner) restoreDependen
|
|||||||
checkpoint: func(context.Context, *lifecycle.Transaction, config.Installation, CreateRequest) (Result, error) {
|
checkpoint: func(context.Context, *lifecycle.Transaction, config.Installation, CreateRequest) (Result, error) {
|
||||||
return Result{Path: "/tmp/default-checkpoint.zip"}, nil
|
return Result{Path: "/tmp/default-checkpoint.zip"}, nil
|
||||||
},
|
},
|
||||||
prepareRecovery: func(context.Context, config.Installation, string) (PreflightResult, error) {
|
prepareRecovery: func(ctx context.Context, installation config.Installation, _ string) (PreflightResult, error) {
|
||||||
return PreflightResult{}, nil
|
return Preflight(ctx, installation, PreflightRequest{Archive: recoveryArchive, Confirm: true, AllowExternalSecrets: true}, permissivePreflightDependencies())
|
||||||
},
|
},
|
||||||
recover: func(context.Context, config.Installation, PreflightResult, bool) error { return nil },
|
recover: func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool) error { return nil },
|
||||||
cleanupCheckpoint: func(string) error { return nil },
|
cleanupCheckpoint: func(string) error { return nil },
|
||||||
acquireTransaction: lifecycle.AcquireTransaction,
|
acquireTransaction: lifecycle.AcquireTransaction,
|
||||||
runner: runner,
|
runner: runner,
|
||||||
|
|||||||
Reference in New Issue
Block a user