fix: stream bounded journal scans
This commit is contained in:
User
@@ -154,6 +154,85 @@ expect_global_replacement_rejected() {
|
||||
|
||||
exercise_header_file_curl
|
||||
|
||||
exercise_journal_scanner() {
|
||||
local journal_source="$fixture_root/docs/install/dwh-auth-server.md"
|
||||
local journal_code
|
||||
journal_code=$(python3 - "$journal_source" <<'PY2'
|
||||
import pathlib
|
||||
import sys
|
||||
|
||||
source = pathlib.Path(sys.argv[1]).read_text(encoding="utf-8")
|
||||
start = source.index("if sudo python3 -c '") + len("if sudo python3 -c '")
|
||||
end = source.index("' \"$since\" \"$v1_key_file\" \"$legacy_key_file\"; then", start)
|
||||
print(source[start:end])
|
||||
PY2
|
||||
) || report_fail journal_extract
|
||||
[[ "$journal_code" == *"process.stdout.read1("* ]] || report_fail journal_chunk_reader
|
||||
[[ "$journal_code" != *"enumerate(process.stdout"* ]] || report_fail journal_unbounded_reader
|
||||
|
||||
local fake_bin="$temp_root/fake-bin"
|
||||
local fake_journalctl="$fake_bin/journalctl"
|
||||
local data_file="$temp_root/journal.data"
|
||||
local pid_file="$temp_root/journal.pid"
|
||||
local stdout_file="$temp_root/journal.stdout"
|
||||
local stderr_file="$temp_root/journal.stderr"
|
||||
mkdir -p "$fake_bin"
|
||||
python3 - "$fake_journalctl" <<'PY2'
|
||||
import pathlib
|
||||
import sys
|
||||
|
||||
path = pathlib.Path(sys.argv[1])
|
||||
path.write_text("""#!/usr/bin/env python3
|
||||
import os
|
||||
import pathlib
|
||||
import sys
|
||||
import time
|
||||
|
||||
pathlib.Path(os.environ["JOURNAL_PID_FILE"]).write_text(str(os.getpid()))
|
||||
data = pathlib.Path(os.environ["JOURNAL_DATA_FILE"]).read_bytes()
|
||||
for offset in range(0, len(data), 997):
|
||||
os.write(sys.stdout.fileno(), data[offset:offset + 997])
|
||||
if os.environ.get("JOURNAL_HOLD") == "1":
|
||||
time.sleep(30)
|
||||
raise SystemExit(int(os.environ.get("JOURNAL_EXIT", "0")))
|
||||
""")
|
||||
path.chmod(0o755)
|
||||
PY2
|
||||
|
||||
local key_file="$temp_root/synthetic-real.key"
|
||||
local legacy_file="$temp_root/synthetic-legacy.key"
|
||||
printf '%s' 'REAL_SECRET_SYNTHETIC' >"$key_file"
|
||||
printf '%s' 'LEGACY_SECRET_SYNTHETIC' >"$legacy_file"
|
||||
chmod 0600 "$key_file" "$legacy_file"
|
||||
|
||||
run_journal_case() {
|
||||
local name=$1 expected=$2 data=$3 exit_code=$4 hold=$5
|
||||
printf '%s' "$data" >"$data_file"
|
||||
: >"$pid_file" "$stdout_file" "$stderr_file"
|
||||
if PATH="$fake_bin:$PATH" JOURNAL_DATA_FILE="$data_file" JOURNAL_PID_FILE="$pid_file" JOURNAL_EXIT="$exit_code" JOURNAL_HOLD="$hold" \
|
||||
python3 -c "$journal_code" synthetic-since "$key_file" "$legacy_file" >"$stdout_file" 2>"$stderr_file"; then
|
||||
actual=0
|
||||
else
|
||||
actual=$?
|
||||
fi
|
||||
[[ "$actual" -eq "$expected" ]] || report_fail "journal_${name}_status"
|
||||
[[ ! -s "$stdout_file" && ! -s "$stderr_file" ]] || report_fail "journal_${name}_output"
|
||||
if [[ -s "$pid_file" ]]; then
|
||||
pid=$(cat "$pid_file")
|
||||
[[ ! -e "/proc/$pid" ]] || report_fail "journal_${name}_reaped"
|
||||
fi
|
||||
report_pass "journal_${name}"
|
||||
}
|
||||
|
||||
run_journal_case clean 0 $'INFO clean\n' 0 0
|
||||
run_journal_case oversized_line 2 "$(printf 'A%.0s' {1..1048577})" 0 0
|
||||
run_journal_case too_many_lines 2 "$(printf 'x\n%.0s' {1..10001})" 0 0
|
||||
run_journal_case child_failure 2 $'INFO child failure\n' 7 0
|
||||
run_journal_case actual_key_across_chunk 1 "$(printf 'A%.0s' {1..65530})REAL_SECRET_SYNTHETIC" 0 1
|
||||
}
|
||||
|
||||
exercise_journal_scanner
|
||||
|
||||
expect_replacement_rejected missing_exact_gui_label docs/install/dwh-auth-client-enrollment.md "Validate workspace source" "Validate workspace"
|
||||
expect_replacement_rejected server_transport_contradiction docs/operations/psd-dwh-auth-rollout.md 'server PSD: `postgres_direct` read-only' 'server PSD: `rest_api` read-only'
|
||||
expect_replacement_rejected missing_mac_local_marker deploy/psd/workspace-bindings.env.example "Mac/local/remota" "server PSD"
|
||||
@@ -177,15 +256,17 @@ expect_global_replacement_rejected missing_exact_header_bytes docs/install/dwh-a
|
||||
|
||||
|
||||
expect_replacement_rejected missing_actual_key_journal_scan docs/install/dwh-auth-server.md "journal_actual_key_scan=PASS" "journal_generic_scan=PASS"
|
||||
expect_replacement_rejected missing_journal_actual_key_match docs/install/dwh-auth-server.md "any(key in line for key in actual_keys)" "False"
|
||||
expect_replacement_rejected missing_journal_actual_key_match docs/install/dwh-auth-server.md "any(needle in searchable for needle in needles)" "False"
|
||||
expect_replacement_rejected missing_journal_returncode_guard docs/install/dwh-auth-server.md "if process.returncode != 0:" "if process.returncode == 0:"
|
||||
journal_contract_source="$fixture_root/docs/install/dwh-auth-server.md"
|
||||
if grep -Fq 'result.stdout.splitlines' "$journal_contract_source" \
|
||||
|| ! grep -Fq 'for line_number, line in enumerate(process.stdout, 1):' "$journal_contract_source" \
|
||||
|| grep -Fq 'enumerate(process.stdout' "$journal_contract_source" \
|
||||
|| ! grep -Fq 'max_journal_bytes = 1_048_576' "$journal_contract_source" \
|
||||
|| ! grep -Fq 'max_journal_lines = 10_000' "$journal_contract_source" \
|
||||
|| ! grep -Fq 'bytes_seen + len(line) > max_journal_bytes' "$journal_contract_source" \
|
||||
|| ! grep -Fq 'line_number > max_journal_lines' "$journal_contract_source" \
|
||||
|| ! grep -Fq 'max_chunk_bytes = 65_536' "$journal_contract_source" \
|
||||
|| ! grep -Fq 'process.stdout.read1(' "$journal_contract_source" \
|
||||
|| ! grep -Fq 'remaining = max_journal_bytes - bytes_seen' "$journal_contract_source" \
|
||||
|| ! grep -Fq 'searchable = carry + chunk' "$journal_contract_source" \
|
||||
|| ! grep -Fq 'process.kill()' "$journal_contract_source"; then
|
||||
report_fail journal_streaming_limits
|
||||
fi
|
||||
|
||||
Reference in New Issue
Block a user