fix: stream bounded journal scans
This commit is contained in:
User
@@ -278,29 +278,52 @@ if sudo python3 -c '
|
||||
import pathlib, subprocess, sys
|
||||
max_journal_bytes = 1_048_576
|
||||
max_journal_lines = 10_000
|
||||
try:
|
||||
actual_keys = {pathlib.Path(path).read_bytes() for path in sys.argv[2:]}
|
||||
process = subprocess.Popen(
|
||||
["journalctl", "-u", "dwh-auth", "--since", sys.argv[1], "--no-pager", "--output=cat"],
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.DEVNULL,
|
||||
)
|
||||
except OSError:
|
||||
raise SystemExit(2)
|
||||
bytes_seen = 0
|
||||
try:
|
||||
for line_number, line in enumerate(process.stdout, 1):
|
||||
if line_number > max_journal_lines or bytes_seen + len(line) > max_journal_bytes:
|
||||
process.kill()
|
||||
process.wait()
|
||||
raise SystemExit(2)
|
||||
bytes_seen += len(line)
|
||||
if b"thtdwh_v1" in line or b"secret_sha256" in line or any(key in line for key in actual_keys):
|
||||
raise SystemExit(1)
|
||||
finally:
|
||||
max_chunk_bytes = 65_536
|
||||
actual_keys = {pathlib.Path(path).read_bytes() for path in sys.argv[2:]}
|
||||
needles = (b"thtdwh_v1", b"secret_sha256", *actual_keys)
|
||||
max_needle_length = max(map(len, needles))
|
||||
process = subprocess.Popen(
|
||||
["journalctl", "-u", "dwh-auth", "--since", sys.argv[1], "--no-pager", "--output=cat"],
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.DEVNULL,
|
||||
)
|
||||
def stop_child():
|
||||
if process.poll() is None:
|
||||
process.kill()
|
||||
process.wait()
|
||||
bytes_seen = 0
|
||||
line_count = 0
|
||||
line_open = False
|
||||
carry = b""
|
||||
try:
|
||||
while True:
|
||||
remaining = max_journal_bytes - bytes_seen
|
||||
if remaining == 0:
|
||||
if process.stdout.read1(1):
|
||||
raise SystemExit(2)
|
||||
break
|
||||
chunk = process.stdout.read1(min(max_chunk_bytes, remaining))
|
||||
if not chunk:
|
||||
break
|
||||
bytes_seen += len(chunk)
|
||||
searchable = carry + chunk
|
||||
if any(needle in searchable for needle in needles):
|
||||
raise SystemExit(1)
|
||||
carry = searchable[-(max_needle_length - 1):]
|
||||
for byte in chunk:
|
||||
if byte == 10:
|
||||
line_count += 1
|
||||
line_open = False
|
||||
if line_count > max_journal_lines:
|
||||
raise SystemExit(2)
|
||||
else:
|
||||
line_open = True
|
||||
if line_open:
|
||||
line_count += 1
|
||||
if line_count > max_journal_lines:
|
||||
raise SystemExit(2)
|
||||
finally:
|
||||
stop_child()
|
||||
if process.returncode != 0:
|
||||
raise SystemExit(2)
|
||||
' "$since" "$v1_key_file" "$legacy_key_file"; then
|
||||
|
||||
@@ -154,6 +154,85 @@ expect_global_replacement_rejected() {
|
||||
|
||||
exercise_header_file_curl
|
||||
|
||||
exercise_journal_scanner() {
|
||||
local journal_source="$fixture_root/docs/install/dwh-auth-server.md"
|
||||
local journal_code
|
||||
journal_code=$(python3 - "$journal_source" <<'PY2'
|
||||
import pathlib
|
||||
import sys
|
||||
|
||||
source = pathlib.Path(sys.argv[1]).read_text(encoding="utf-8")
|
||||
start = source.index("if sudo python3 -c '") + len("if sudo python3 -c '")
|
||||
end = source.index("' \"$since\" \"$v1_key_file\" \"$legacy_key_file\"; then", start)
|
||||
print(source[start:end])
|
||||
PY2
|
||||
) || report_fail journal_extract
|
||||
[[ "$journal_code" == *"process.stdout.read1("* ]] || report_fail journal_chunk_reader
|
||||
[[ "$journal_code" != *"enumerate(process.stdout"* ]] || report_fail journal_unbounded_reader
|
||||
|
||||
local fake_bin="$temp_root/fake-bin"
|
||||
local fake_journalctl="$fake_bin/journalctl"
|
||||
local data_file="$temp_root/journal.data"
|
||||
local pid_file="$temp_root/journal.pid"
|
||||
local stdout_file="$temp_root/journal.stdout"
|
||||
local stderr_file="$temp_root/journal.stderr"
|
||||
mkdir -p "$fake_bin"
|
||||
python3 - "$fake_journalctl" <<'PY2'
|
||||
import pathlib
|
||||
import sys
|
||||
|
||||
path = pathlib.Path(sys.argv[1])
|
||||
path.write_text("""#!/usr/bin/env python3
|
||||
import os
|
||||
import pathlib
|
||||
import sys
|
||||
import time
|
||||
|
||||
pathlib.Path(os.environ["JOURNAL_PID_FILE"]).write_text(str(os.getpid()))
|
||||
data = pathlib.Path(os.environ["JOURNAL_DATA_FILE"]).read_bytes()
|
||||
for offset in range(0, len(data), 997):
|
||||
os.write(sys.stdout.fileno(), data[offset:offset + 997])
|
||||
if os.environ.get("JOURNAL_HOLD") == "1":
|
||||
time.sleep(30)
|
||||
raise SystemExit(int(os.environ.get("JOURNAL_EXIT", "0")))
|
||||
""")
|
||||
path.chmod(0o755)
|
||||
PY2
|
||||
|
||||
local key_file="$temp_root/synthetic-real.key"
|
||||
local legacy_file="$temp_root/synthetic-legacy.key"
|
||||
printf '%s' 'REAL_SECRET_SYNTHETIC' >"$key_file"
|
||||
printf '%s' 'LEGACY_SECRET_SYNTHETIC' >"$legacy_file"
|
||||
chmod 0600 "$key_file" "$legacy_file"
|
||||
|
||||
run_journal_case() {
|
||||
local name=$1 expected=$2 data=$3 exit_code=$4 hold=$5
|
||||
printf '%s' "$data" >"$data_file"
|
||||
: >"$pid_file" "$stdout_file" "$stderr_file"
|
||||
if PATH="$fake_bin:$PATH" JOURNAL_DATA_FILE="$data_file" JOURNAL_PID_FILE="$pid_file" JOURNAL_EXIT="$exit_code" JOURNAL_HOLD="$hold" \
|
||||
python3 -c "$journal_code" synthetic-since "$key_file" "$legacy_file" >"$stdout_file" 2>"$stderr_file"; then
|
||||
actual=0
|
||||
else
|
||||
actual=$?
|
||||
fi
|
||||
[[ "$actual" -eq "$expected" ]] || report_fail "journal_${name}_status"
|
||||
[[ ! -s "$stdout_file" && ! -s "$stderr_file" ]] || report_fail "journal_${name}_output"
|
||||
if [[ -s "$pid_file" ]]; then
|
||||
pid=$(cat "$pid_file")
|
||||
[[ ! -e "/proc/$pid" ]] || report_fail "journal_${name}_reaped"
|
||||
fi
|
||||
report_pass "journal_${name}"
|
||||
}
|
||||
|
||||
run_journal_case clean 0 $'INFO clean\n' 0 0
|
||||
run_journal_case oversized_line 2 "$(printf 'A%.0s' {1..1048577})" 0 0
|
||||
run_journal_case too_many_lines 2 "$(printf 'x\n%.0s' {1..10001})" 0 0
|
||||
run_journal_case child_failure 2 $'INFO child failure\n' 7 0
|
||||
run_journal_case actual_key_across_chunk 1 "$(printf 'A%.0s' {1..65530})REAL_SECRET_SYNTHETIC" 0 1
|
||||
}
|
||||
|
||||
exercise_journal_scanner
|
||||
|
||||
expect_replacement_rejected missing_exact_gui_label docs/install/dwh-auth-client-enrollment.md "Validate workspace source" "Validate workspace"
|
||||
expect_replacement_rejected server_transport_contradiction docs/operations/psd-dwh-auth-rollout.md 'server PSD: `postgres_direct` read-only' 'server PSD: `rest_api` read-only'
|
||||
expect_replacement_rejected missing_mac_local_marker deploy/psd/workspace-bindings.env.example "Mac/local/remota" "server PSD"
|
||||
@@ -177,15 +256,17 @@ expect_global_replacement_rejected missing_exact_header_bytes docs/install/dwh-a
|
||||
|
||||
|
||||
expect_replacement_rejected missing_actual_key_journal_scan docs/install/dwh-auth-server.md "journal_actual_key_scan=PASS" "journal_generic_scan=PASS"
|
||||
expect_replacement_rejected missing_journal_actual_key_match docs/install/dwh-auth-server.md "any(key in line for key in actual_keys)" "False"
|
||||
expect_replacement_rejected missing_journal_actual_key_match docs/install/dwh-auth-server.md "any(needle in searchable for needle in needles)" "False"
|
||||
expect_replacement_rejected missing_journal_returncode_guard docs/install/dwh-auth-server.md "if process.returncode != 0:" "if process.returncode == 0:"
|
||||
journal_contract_source="$fixture_root/docs/install/dwh-auth-server.md"
|
||||
if grep -Fq 'result.stdout.splitlines' "$journal_contract_source" \
|
||||
|| ! grep -Fq 'for line_number, line in enumerate(process.stdout, 1):' "$journal_contract_source" \
|
||||
|| grep -Fq 'enumerate(process.stdout' "$journal_contract_source" \
|
||||
|| ! grep -Fq 'max_journal_bytes = 1_048_576' "$journal_contract_source" \
|
||||
|| ! grep -Fq 'max_journal_lines = 10_000' "$journal_contract_source" \
|
||||
|| ! grep -Fq 'bytes_seen + len(line) > max_journal_bytes' "$journal_contract_source" \
|
||||
|| ! grep -Fq 'line_number > max_journal_lines' "$journal_contract_source" \
|
||||
|| ! grep -Fq 'max_chunk_bytes = 65_536' "$journal_contract_source" \
|
||||
|| ! grep -Fq 'process.stdout.read1(' "$journal_contract_source" \
|
||||
|| ! grep -Fq 'remaining = max_journal_bytes - bytes_seen' "$journal_contract_source" \
|
||||
|| ! grep -Fq 'searchable = carry + chunk' "$journal_contract_source" \
|
||||
|| ! grep -Fq 'process.kill()' "$journal_contract_source"; then
|
||||
report_fail journal_streaming_limits
|
||||
fi
|
||||
|
||||
@@ -34,7 +34,7 @@ for label, relative in docs.items():
|
||||
text[label] = path.read_text(encoding="utf-8")
|
||||
|
||||
requirements = {
|
||||
"server": ["key_output=/root/dwh-auth-provision/psd-mac-primary.key", "legacy_key_id=legacy-shared", "v1_header_file", "legacy_header_file", "random_header_file", "--header \"@", "registry_staging", "registry_previous", "registry_restore_rollback", "journal_actual_key_scan=PASS", "sys.argv[2:]", "read_bytes()", "subprocess.Popen", "stderr=subprocess.DEVNULL", "returncode != 0", "max_journal_bytes = 1_048_576", "max_journal_lines = 10_000", "subprocess.Popen", "stdout=subprocess.PIPE", "for line_number, line in enumerate(process.stdout, 1):", "bytes_seen + len(line) > max_journal_bytes", "line_number > max_journal_lines", "process.kill()", "process.wait()", "socket_v1=PASS", "socket_legacy=PASS", "https_v1_post_revoke=PASS", "https_legacy_post_revoke=PASS", "key_file_bytes=PASS", "dd if=\"$1\" bs=65536 status=none", "manifest", "journalctl", "systemctl disable --now", "trap", "/var/lib/dwh-auth", "root:dwh-auth", "2750", ".writer.lock", "0640", "/run/dwh-auth/verify.sock", "0660", "systemd", "key create", "key list", "key status", "key revoke", "check", "backup", "rollback", "disinstallazione", "rest_api", "postgres_direct", "ssh_tunnel"],
|
||||
"server": ["key_output=/root/dwh-auth-provision/psd-mac-primary.key", "legacy_key_id=legacy-shared", "v1_header_file", "legacy_header_file", "random_header_file", "--header \"@", "registry_staging", "registry_previous", "registry_restore_rollback", "journal_actual_key_scan=PASS", "sys.argv[2:]", "read_bytes()", "subprocess.Popen", "stderr=subprocess.DEVNULL", "returncode != 0", "max_journal_bytes = 1_048_576", "max_journal_lines = 10_000", "stdout=subprocess.PIPE", "max_chunk_bytes = 65_536", "process.stdout.read1(", "remaining = max_journal_bytes - bytes_seen", "searchable = carry + chunk", "max_needle_length", "process.kill()", "process.wait()", "socket_v1=PASS", "socket_legacy=PASS", "https_v1_post_revoke=PASS", "https_legacy_post_revoke=PASS", "key_file_bytes=PASS", "dd if=\"$1\" bs=65536 status=none", "manifest", "journalctl", "systemctl disable --now", "trap", "/var/lib/dwh-auth", "root:dwh-auth", "2750", ".writer.lock", "0640", "/run/dwh-auth/verify.sock", "0660", "systemd", "key create", "key list", "key status", "key revoke", "check", "backup", "rollback", "disinstallazione", "rest_api", "postgres_direct", "ssh_tunnel"],
|
||||
"client": ["Workspace management", "Validate workspace source", "Test workspace connections", "Save entered secrets", "Forget stored value", "API_KEY_FILE", "THT_WS_PSD_CLINICAL_DWH_API_KEY_FILE", "TLS_CA_FILE", "/rpc/ping", "rest_api", "postgres_direct", "ssh_tunnel", "401", "503", "rotazione", "revoca"],
|
||||
"tls": ["self-issued", ".it", ".com", "SAN", "TLS_CA_FILE", "openssl x509 -noout -fingerprint -sha256", "fuori banda", "rinnovo", "curl -k"],
|
||||
"rollout": ["v1=2xx", "legacy=2xx", "v1=2xx post-revoca", "legacy=401 post-revoca", "file header curl protetti 0600", "PASS/FAIL", "no raw diff", "Task 9", "Task 10", "IN_DISCUSSION", "postgres_direct", "rest_api", "legacy-shared", "nginx -t", "204", "401", "503", "Qdrant", "Ollama", "rollback"],
|
||||
@@ -115,13 +115,15 @@ journal_steps = (
|
||||
"stdout=subprocess.PIPE",
|
||||
"max_journal_bytes = 1_048_576",
|
||||
"max_journal_lines = 10_000",
|
||||
"for line_number, line in enumerate(process.stdout, 1):",
|
||||
"bytes_seen + len(line) > max_journal_bytes",
|
||||
"line_number > max_journal_lines",
|
||||
"max_chunk_bytes = 65_536",
|
||||
"process.stdout.read1(",
|
||||
"remaining = max_journal_bytes - bytes_seen",
|
||||
"searchable = carry + chunk",
|
||||
"max_needle_length",
|
||||
"process.kill()",
|
||||
"process.wait()",
|
||||
"if process.returncode != 0:",
|
||||
"any(key in line for key in actual_keys)",
|
||||
"any(needle in searchable for needle in needles)",
|
||||
)
|
||||
if any(step not in text["server"] for step in journal_steps):
|
||||
raise SystemExit("dwh-auth docs: journal scan must fail closed and check the actual key bytes")
|
||||
|
||||
Reference in New Issue
Block a user