From 4649ddc7ae06a7c24745213e514f27513e189a18 Mon Sep 17 00:00:00 2001 From: Fabrizio Waldner Date: Tue, 14 Jul 2026 15:55:19 +0200 Subject: [PATCH 1/8] SCHED-1897: Add idle node memory health check --- .../slurm_scripts/idle_mem_used.sh | 71 ++++++++++++ .../slurm_scripts/idle_mem_used.sh.json | 16 +++ .../slurm_scripts/idle_mem_used_test.py | 105 ++++++++++++++++++ .../tests/idle_mem_used_test.yaml | 44 ++++++++ helm/slurm-cluster/values.yaml | 6 + 5 files changed, 242 insertions(+) create mode 100644 helm/slurm-cluster/slurm_scripts/idle_mem_used.sh create mode 100644 helm/slurm-cluster/slurm_scripts/idle_mem_used.sh.json create mode 100644 helm/slurm-cluster/slurm_scripts/idle_mem_used_test.py create mode 100644 helm/slurm-cluster/tests/idle_mem_used_test.yaml diff --git a/helm/slurm-cluster/slurm_scripts/idle_mem_used.sh b/helm/slurm-cluster/slurm_scripts/idle_mem_used.sh new file mode 100644 index 000000000..a56e071d3 --- /dev/null +++ b/helm/slurm-cluster/slurm_scripts/idle_mem_used.sh @@ -0,0 +1,71 @@ +#!/bin/bash + +set -euo pipefail + +max_used_gb="${IDLE_MEM_USED_MAX_USED_GB:-}" +meminfo_path="${IDLE_MEM_USED_MEMINFO_PATH:-/proc/meminfo}" +node_state_flags="${CHECKS_NODE_STATE_FLAGS:-}" +node_name="${SLURMD_NODENAME:-unknown}" + +echo "[$(date)] Check memory usage when node ${node_name} is idle" +echo "Configured maximum used memory: ${max_used_gb:-} GB" +echo "Node state source: CHECKS_NODE_STATE_FLAGS, populated by check_runner.py from 'scontrol show node ${node_name} --json'" +echo "Slurm node state flags: ${node_state_flags:-}" + +node_is_idle=false +IFS='+' read -r -a state_flags <<< "${node_state_flags}" +for state_flag in "${state_flags[@]}"; do + if [[ "${state_flag}" == "IDLE" ]]; then + node_is_idle=true + break + fi +done + +echo "Node is idle: ${node_is_idle}" +if [[ "${node_is_idle}" != "true" ]]; then + echo "Node is not IDLE; skipping memory validation" + exit 0 +fi + +if ! [[ "${max_used_gb}" =~ ^[0-9]+$ ]] || [[ "${max_used_gb}" == "0" ]]; then + echo "Invalid idle memory threshold '${max_used_gb:-}'; expected a positive GB count, skipping memory validation" >&2 + exit 0 +fi + +max_used_bytes=$((max_used_gb * 1000000000)) + +meminfo_values="$( + awk ' + /^MemTotal:/ { total = $2 } + /^MemAvailable:/ { available = $2 } + END { if (total != "" && available != "") print total, available } + ' "${meminfo_path}" 2>/dev/null || true +)" +read -r mem_total_kib mem_available_kib <<< "${meminfo_values}" + +if ! [[ "${mem_total_kib:-}" =~ ^[0-9]+$ ]] || + ! [[ "${mem_available_kib:-}" =~ ^[0-9]+$ ]] || + (( mem_available_kib > mem_total_kib )); then + echo "Could not determine valid MemTotal and MemAvailable values from ${meminfo_path}; skipping memory validation" >&2 + exit 0 +fi + +mem_total_bytes=$((mem_total_kib * 1024)) +mem_available_bytes=$((mem_available_kib * 1024)) +mem_used_bytes=$((mem_total_bytes - mem_available_bytes)) + +mem_total_gb="$(awk -v bytes="${mem_total_bytes}" 'BEGIN { printf "%.2f", bytes / 1000000000 }')" +mem_available_gb="$(awk -v bytes="${mem_available_bytes}" 'BEGIN { printf "%.2f", bytes / 1000000000 }')" +mem_used_gb="$(awk -v bytes="${mem_used_bytes}" 'BEGIN { printf "%.2f", bytes / 1000000000 }')" + +echo "Memory source: ${meminfo_path}" +echo "Memory measurements: total=${mem_total_gb} GB (${mem_total_bytes} bytes), available=${mem_available_gb} GB (${mem_available_bytes} bytes), used=total-available=${mem_used_gb} GB (${mem_used_bytes} bytes)" +echo "Maximum allowed used memory: ${max_used_gb} GB (${max_used_bytes} bytes)" + +if (( mem_used_bytes > max_used_bytes )); then + echo "Node ${node_name} is IDLE but uses ${mem_used_gb} GB of memory (threshold: ${max_used_gb} GB; MemTotal: ${mem_total_gb} GB; MemAvailable: ${mem_available_gb} GB). This may indicate leftover or spurious processes consuming memory." >&3 + exit 1 +fi + +echo "Idle node memory usage is within the configured threshold" +exit 0 diff --git a/helm/slurm-cluster/slurm_scripts/idle_mem_used.sh.json b/helm/slurm-cluster/slurm_scripts/idle_mem_used.sh.json new file mode 100644 index 000000000..382fcbe39 --- /dev/null +++ b/helm/slurm-cluster/slurm_scripts/idle_mem_used.sh.json @@ -0,0 +1,16 @@ +{ + "name": "idle_mem_used", + "command": "IDLE_MEM_USED_MAX_USED_GB={{ required `Idle memory maximum used GB must be provided.` (index .Values.slurmScripts.builtIn `idle_mem_used.sh`).maxUsedGB }} ./idle_mem_used.sh", + "platforms": ["any"], + "skip_for_cpu_jobs": false, + "skip_for_partial_gpu_jobs": false, + "contexts": ["hc_program"], + "node_states": ["any"], + "on_fail": "drain", + "on_ok": "none", + "reason_base": "[node_problem] $name", + "reason_append_details": true, + "run_in_jail": false, + "log": "slurm_scripts/$worker.$name.$context.out", + "need_env": ["CHECKS_NODE_STATE_FLAGS"] +} diff --git a/helm/slurm-cluster/slurm_scripts/idle_mem_used_test.py b/helm/slurm-cluster/slurm_scripts/idle_mem_used_test.py new file mode 100644 index 000000000..8dedfeac6 --- /dev/null +++ b/helm/slurm-cluster/slurm_scripts/idle_mem_used_test.py @@ -0,0 +1,105 @@ +import os +import subprocess +import tempfile +import unittest +from pathlib import Path + + +SCRIPT_PATH = Path(__file__).with_name("idle_mem_used.sh") +GIB = 1024 * 1024 * 1024 + + +class IdleMemUsedTest(unittest.TestCase): + def run_check( + self, + *, + node_states: str, + total_bytes: int | None, + available_bytes: int | None, + max_used_gb: int = 32, + ) -> subprocess.CompletedProcess[str]: + with tempfile.TemporaryDirectory() as tmpdir: + meminfo_path = Path(tmpdir) / "meminfo" + if total_bytes is not None and available_bytes is not None: + meminfo_path.write_text( + f"MemTotal: {total_bytes // 1024} kB\n" + f"MemAvailable: {available_bytes // 1024} kB\n", + encoding="utf-8", + ) + + env = os.environ.copy() + env.update( + { + "SLURMD_NODENAME": "worker-1", + "CHECKS_NODE_STATE_FLAGS": node_states, + "IDLE_MEM_USED_MAX_USED_GB": str(max_used_gb), + "IDLE_MEM_USED_MEMINFO_PATH": str(meminfo_path), + } + ) + + return subprocess.run( + [ + "bash", + "-c", + 'exec 3>&1; exec bash "$1"', + "idle-mem-used-test", + str(SCRIPT_PATH), + ], + check=False, + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + + def test_non_idle_node_skips_memory_validation(self): + result = self.run_check( + node_states="ALLOCATED", + total_bytes=None, + available_bytes=None, + ) + + self.assertEqual(0, result.returncode) + self.assertIn("Node is idle: false", result.stdout) + self.assertIn("Node is not IDLE; skipping memory validation", result.stdout) + self.assertNotIn("Memory measurements:", result.stdout) + + def test_idle_node_below_threshold_passes(self): + result = self.run_check( + node_states="IDLE", + total_bytes=64 * GIB, + available_bytes=60 * GIB, + ) + + self.assertEqual(0, result.returncode) + self.assertIn("Node is idle: true", result.stdout) + self.assertIn(f"used=total-available=4.29 GB ({4 * GIB} bytes)", result.stdout) + self.assertIn("Maximum allowed used memory: 32 GB (32000000000 bytes)", result.stdout) + + def test_idle_node_above_threshold_fails_with_actionable_reason(self): + result = self.run_check( + node_states="IDLE+CLOUD", + total_bytes=64 * GIB, + available_bytes=22 * GIB, + ) + + self.assertEqual(1, result.returncode) + self.assertIn("Node is idle: true", result.stdout) + self.assertIn("Node worker-1 is IDLE but uses", result.stdout) + self.assertIn("45.10 GB", result.stdout) + self.assertIn("threshold: 32 GB", result.stdout) + self.assertIn("leftover or spurious processes", result.stdout) + + def test_unavailable_memory_data_does_not_drain_node(self): + result = self.run_check( + node_states="IDLE", + total_bytes=None, + available_bytes=None, + ) + + self.assertEqual(0, result.returncode) + self.assertIn("Could not determine valid MemTotal and MemAvailable", result.stderr) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/helm/slurm-cluster/tests/idle_mem_used_test.yaml b/helm/slurm-cluster/tests/idle_mem_used_test.yaml new file mode 100644 index 000000000..391529fa8 --- /dev/null +++ b/helm/slurm-cluster/tests/idle_mem_used_test.yaml @@ -0,0 +1,44 @@ +suite: test idle memory health check +templates: + - slurm-scripts-cm.yaml +tests: + - it: should enable the idle memory check with the default threshold + asserts: + - isNotNull: + path: data["idle_mem_used.sh"] + - matchRegex: + path: data["checks.json"] + pattern: '"name": "idle_mem_used"' + - matchRegex: + path: data["checks.json"] + pattern: '"command": "IDLE_MEM_USED_MAX_USED_GB=32 ./idle_mem_used.sh"' + - matchRegex: + path: data["checks.json"] + pattern: '"contexts": \[\s*"hc_program"\s*\]' + - matchRegex: + path: data["checks.json"] + pattern: '"on_fail": "drain"' + + - it: should render a configured maximum used memory threshold + set: + slurmScripts: + builtIn: + idle_mem_used.sh: + maxUsedGB: 16 + asserts: + - matchRegex: + path: data["checks.json"] + pattern: '"command": "IDLE_MEM_USED_MAX_USED_GB=16 ./idle_mem_used.sh"' + + - it: should allow the idle memory check to be disabled + set: + slurmScripts: + builtIn: + idle_mem_used.sh: + enabled: false + asserts: + - isNull: + path: data["idle_mem_used.sh"] + - notMatchRegex: + path: data["checks.json"] + pattern: '"name": "idle_mem_used"' diff --git a/helm/slurm-cluster/values.yaml b/helm/slurm-cluster/values.yaml index 551844cdf..b4edb0d6a 100644 --- a/helm/slurm-cluster/values.yaml +++ b/helm/slurm-cluster/values.yaml @@ -774,6 +774,12 @@ slurmScripts: enabled: true customContent: null customConfig: null + idle_mem_used.sh: + enabled: true + # Drain an idle node when MemTotal - MemAvailable exceeds 32 GB. + maxUsedGB: 32 + customContent: null + customConfig: null job_tmpfs_delete.sh: enabled: true customContent: null From 0bb76183d5dd49b665c3dc257e3def0e66ba40ec Mon Sep 17 00:00:00 2001 From: Fabrizio Waldner Date: Wed, 15 Jul 2026 09:25:25 +0200 Subject: [PATCH 2/8] SCHED-1897: Detect idle nodes without controller RPC --- .../slurm_scripts/idle_mem_used.sh | 36 +++++++++------ .../slurm_scripts/idle_mem_used.sh.json | 2 +- .../slurm_scripts/idle_mem_used_test.py | 45 ++++++++++++++++--- .../tests/idle_mem_used_test.yaml | 3 ++ 4 files changed, 65 insertions(+), 21 deletions(-) diff --git a/helm/slurm-cluster/slurm_scripts/idle_mem_used.sh b/helm/slurm-cluster/slurm_scripts/idle_mem_used.sh index a56e071d3..2f9d49be0 100644 --- a/helm/slurm-cluster/slurm_scripts/idle_mem_used.sh +++ b/helm/slurm-cluster/slurm_scripts/idle_mem_used.sh @@ -4,26 +4,36 @@ set -euo pipefail max_used_gb="${IDLE_MEM_USED_MAX_USED_GB:-}" meminfo_path="${IDLE_MEM_USED_MEMINFO_PATH:-/proc/meminfo}" -node_state_flags="${CHECKS_NODE_STATE_FLAGS:-}" node_name="${SLURMD_NODENAME:-unknown}" echo "[$(date)] Check memory usage when node ${node_name} is idle" echo "Configured maximum used memory: ${max_used_gb:-} GB" -echo "Node state source: CHECKS_NODE_STATE_FLAGS, populated by check_runner.py from 'scontrol show node ${node_name} --json'" -echo "Slurm node state flags: ${node_state_flags:-}" - -node_is_idle=false -IFS='+' read -r -a state_flags <<< "${node_state_flags}" -for state_flag in "${state_flags[@]}"; do - if [[ "${state_flag}" == "IDLE" ]]; then - node_is_idle=true - break - fi -done +echo "Idle state source: local 'scontrol listjobs' (no controller RPC)" +echo "Idle state rule: exit code 1 with \"No slurmstepd's found on this node\" means the node has no local jobs" + +if listjobs_output="$(scontrol listjobs 2>&1)"; then + listjobs_rc=0 +else + listjobs_rc=$? +fi + +echo "scontrol listjobs exit code: ${listjobs_rc}" +echo "scontrol listjobs output: ${listjobs_output:-}" + +if (( listjobs_rc == 0 )); then + node_is_idle=false + echo "Local Slurm jobs are present; treating the node as non-idle" +elif (( listjobs_rc == 1 )) && [[ "${listjobs_output}" == *"No slurmstepd's found on this node"* ]]; then + node_is_idle=true + echo "No local slurmstepd was found; treating the node as idle" +else + echo "Could not determine whether the node is idle from 'scontrol listjobs'; skipping memory validation" >&2 + exit 0 +fi echo "Node is idle: ${node_is_idle}" if [[ "${node_is_idle}" != "true" ]]; then - echo "Node is not IDLE; skipping memory validation" + echo "Node has local jobs; skipping memory validation" exit 0 fi diff --git a/helm/slurm-cluster/slurm_scripts/idle_mem_used.sh.json b/helm/slurm-cluster/slurm_scripts/idle_mem_used.sh.json index 382fcbe39..180d7e7d5 100644 --- a/helm/slurm-cluster/slurm_scripts/idle_mem_used.sh.json +++ b/helm/slurm-cluster/slurm_scripts/idle_mem_used.sh.json @@ -12,5 +12,5 @@ "reason_append_details": true, "run_in_jail": false, "log": "slurm_scripts/$worker.$name.$context.out", - "need_env": ["CHECKS_NODE_STATE_FLAGS"] + "need_env": [] } diff --git a/helm/slurm-cluster/slurm_scripts/idle_mem_used_test.py b/helm/slurm-cluster/slurm_scripts/idle_mem_used_test.py index 8dedfeac6..c5a4f138e 100644 --- a/helm/slurm-cluster/slurm_scripts/idle_mem_used_test.py +++ b/helm/slurm-cluster/slurm_scripts/idle_mem_used_test.py @@ -13,12 +13,22 @@ class IdleMemUsedTest(unittest.TestCase): def run_check( self, *, - node_states: str, + listjobs_rc: int, + listjobs_output: str, total_bytes: int | None, available_bytes: int | None, max_used_gb: int = 32, ) -> subprocess.CompletedProcess[str]: with tempfile.TemporaryDirectory() as tmpdir: + scontrol_path = Path(tmpdir) / "scontrol" + scontrol_path.write_text( + "#!/bin/bash\n" + "printf '%s\\n' \"${MOCK_SCONTROL_LISTJOBS_OUTPUT}\"\n" + "exit \"${MOCK_SCONTROL_LISTJOBS_RC}\"\n", + encoding="utf-8", + ) + scontrol_path.chmod(0o755) + meminfo_path = Path(tmpdir) / "meminfo" if total_bytes is not None and available_bytes is not None: meminfo_path.write_text( @@ -30,10 +40,12 @@ def run_check( env = os.environ.copy() env.update( { + "PATH": f"{tmpdir}:{env['PATH']}", "SLURMD_NODENAME": "worker-1", - "CHECKS_NODE_STATE_FLAGS": node_states, "IDLE_MEM_USED_MAX_USED_GB": str(max_used_gb), "IDLE_MEM_USED_MEMINFO_PATH": str(meminfo_path), + "MOCK_SCONTROL_LISTJOBS_RC": str(listjobs_rc), + "MOCK_SCONTROL_LISTJOBS_OUTPUT": listjobs_output, } ) @@ -54,31 +66,36 @@ def run_check( def test_non_idle_node_skips_memory_validation(self): result = self.run_check( - node_states="ALLOCATED", + listjobs_rc=0, + listjobs_output="JOBID\n1011", total_bytes=None, available_bytes=None, ) self.assertEqual(0, result.returncode) self.assertIn("Node is idle: false", result.stdout) - self.assertIn("Node is not IDLE; skipping memory validation", result.stdout) + self.assertIn("Local Slurm jobs are present", result.stdout) + self.assertIn("Node has local jobs; skipping memory validation", result.stdout) self.assertNotIn("Memory measurements:", result.stdout) def test_idle_node_below_threshold_passes(self): result = self.run_check( - node_states="IDLE", + listjobs_rc=1, + listjobs_output="No slurmstepd's found on this node", total_bytes=64 * GIB, available_bytes=60 * GIB, ) self.assertEqual(0, result.returncode) self.assertIn("Node is idle: true", result.stdout) + self.assertIn("No local slurmstepd was found", result.stdout) self.assertIn(f"used=total-available=4.29 GB ({4 * GIB} bytes)", result.stdout) self.assertIn("Maximum allowed used memory: 32 GB (32000000000 bytes)", result.stdout) def test_idle_node_above_threshold_fails_with_actionable_reason(self): result = self.run_check( - node_states="IDLE+CLOUD", + listjobs_rc=1, + listjobs_output="No slurmstepd's found on this node", total_bytes=64 * GIB, available_bytes=22 * GIB, ) @@ -92,7 +109,8 @@ def test_idle_node_above_threshold_fails_with_actionable_reason(self): def test_unavailable_memory_data_does_not_drain_node(self): result = self.run_check( - node_states="IDLE", + listjobs_rc=1, + listjobs_output="No slurmstepd's found on this node", total_bytes=None, available_bytes=None, ) @@ -100,6 +118,19 @@ def test_unavailable_memory_data_does_not_drain_node(self): self.assertEqual(0, result.returncode) self.assertIn("Could not determine valid MemTotal and MemAvailable", result.stderr) + def test_unexpected_listjobs_failure_does_not_validate_memory(self): + result = self.run_check( + listjobs_rc=2, + listjobs_output="Unable to inspect local jobs", + total_bytes=None, + available_bytes=None, + ) + + self.assertEqual(0, result.returncode) + self.assertIn("scontrol listjobs exit code: 2", result.stdout) + self.assertIn("Could not determine whether the node is idle", result.stderr) + self.assertNotIn("Memory measurements:", result.stdout) + if __name__ == "__main__": unittest.main(verbosity=2) diff --git a/helm/slurm-cluster/tests/idle_mem_used_test.yaml b/helm/slurm-cluster/tests/idle_mem_used_test.yaml index 391529fa8..52bcf53d9 100644 --- a/helm/slurm-cluster/tests/idle_mem_used_test.yaml +++ b/helm/slurm-cluster/tests/idle_mem_used_test.yaml @@ -18,6 +18,9 @@ tests: - matchRegex: path: data["checks.json"] pattern: '"on_fail": "drain"' + - matchRegex: + path: data["checks.json"] + pattern: '"need_env": \[\s*\]' - it: should render a configured maximum used memory threshold set: From 9e0cbb17fb3e0f938b2e298f28bb459b13d03eb5 Mon Sep 17 00:00:00 2001 From: Fabrizio Waldner Date: Wed, 15 Jul 2026 14:03:04 +0200 Subject: [PATCH 3/8] SCHED-1897: Derive idle memory threshold from RealMemory --- .../slurm_scripts/idle_mem_used.sh | 30 +++++++---- .../slurm_scripts/idle_mem_used.sh.json | 4 +- .../slurm_scripts/idle_mem_used_test.py | 51 ++++++++++++++++--- .../tests/idle_mem_used_test.yaml | 17 ++----- helm/slurm-cluster/values.yaml | 2 - 5 files changed, 68 insertions(+), 36 deletions(-) diff --git a/helm/slurm-cluster/slurm_scripts/idle_mem_used.sh b/helm/slurm-cluster/slurm_scripts/idle_mem_used.sh index 2f9d49be0..d97640c57 100644 --- a/helm/slurm-cluster/slurm_scripts/idle_mem_used.sh +++ b/helm/slurm-cluster/slurm_scripts/idle_mem_used.sh @@ -2,12 +2,12 @@ set -euo pipefail -max_used_gb="${IDLE_MEM_USED_MAX_USED_GB:-}" meminfo_path="${IDLE_MEM_USED_MEMINFO_PATH:-/proc/meminfo}" node_name="${SLURMD_NODENAME:-unknown}" +node_real_memory_bytes="${CHECKS_NODE_REAL_MEM_BYTES:-}" echo "[$(date)] Check memory usage when node ${node_name} is idle" -echo "Configured maximum used memory: ${max_used_gb:-} GB" +echo "Slurm RealMemory input: ${node_real_memory_bytes:-} bytes" echo "Idle state source: local 'scontrol listjobs' (no controller RPC)" echo "Idle state rule: exit code 1 with \"No slurmstepd's found on this node\" means the node has no local jobs" @@ -37,13 +37,11 @@ if [[ "${node_is_idle}" != "true" ]]; then exit 0 fi -if ! [[ "${max_used_gb}" =~ ^[0-9]+$ ]] || [[ "${max_used_gb}" == "0" ]]; then - echo "Invalid idle memory threshold '${max_used_gb:-}'; expected a positive GB count, skipping memory validation" >&2 +if ! [[ "${node_real_memory_bytes}" =~ ^[0-9]+$ ]] || [[ "${node_real_memory_bytes}" == "0" ]]; then + echo "Invalid or unavailable Slurm RealMemory '${node_real_memory_bytes:-}'; expected a positive byte count, skipping memory validation" >&2 exit 0 fi -max_used_bytes=$((max_used_gb * 1000000000)) - meminfo_values="$( awk ' /^MemTotal:/ { total = $2 } @@ -64,18 +62,30 @@ mem_total_bytes=$((mem_total_kib * 1024)) mem_available_bytes=$((mem_available_kib * 1024)) mem_used_bytes=$((mem_total_bytes - mem_available_bytes)) +if (( node_real_memory_bytes > mem_total_bytes )); then + echo "Slurm RealMemory ${node_real_memory_bytes} bytes exceeds MemTotal ${mem_total_bytes} bytes; skipping memory validation" >&2 + exit 0 +fi + +max_idle_used_bytes=$((mem_total_bytes - node_real_memory_bytes)) + mem_total_gb="$(awk -v bytes="${mem_total_bytes}" 'BEGIN { printf "%.2f", bytes / 1000000000 }')" mem_available_gb="$(awk -v bytes="${mem_available_bytes}" 'BEGIN { printf "%.2f", bytes / 1000000000 }')" mem_used_gb="$(awk -v bytes="${mem_used_bytes}" 'BEGIN { printf "%.2f", bytes / 1000000000 }')" +node_real_memory_gb="$(awk -v bytes="${node_real_memory_bytes}" 'BEGIN { printf "%.2f", bytes / 1000000000 }')" +max_idle_used_gb="$(awk -v bytes="${max_idle_used_bytes}" 'BEGIN { printf "%.2f", bytes / 1000000000 }')" echo "Memory source: ${meminfo_path}" echo "Memory measurements: total=${mem_total_gb} GB (${mem_total_bytes} bytes), available=${mem_available_gb} GB (${mem_available_bytes} bytes), used=total-available=${mem_used_gb} GB (${mem_used_bytes} bytes)" -echo "Maximum allowed used memory: ${max_used_gb} GB (${max_used_bytes} bytes)" +echo "Slurm RealMemory: ${node_real_memory_gb} GB (${node_real_memory_bytes} bytes)" +echo "Derived maximum idle used memory: MemTotal-RealMemory=${max_idle_used_gb} GB (${max_idle_used_bytes} bytes)" -if (( mem_used_bytes > max_used_bytes )); then - echo "Node ${node_name} is IDLE but uses ${mem_used_gb} GB of memory (threshold: ${max_used_gb} GB; MemTotal: ${mem_total_gb} GB; MemAvailable: ${mem_available_gb} GB). This may indicate leftover or spurious processes consuming memory." >&3 +if (( mem_available_bytes < node_real_memory_bytes )); then + memory_deficit_bytes=$((node_real_memory_bytes - mem_available_bytes)) + memory_deficit_gb="$(awk -v bytes="${memory_deficit_bytes}" 'BEGIN { printf "%.2f", bytes / 1000000000 }')" + echo "Node ${node_name} is IDLE but has only ${mem_available_gb} GB of memory available, below Slurm RealMemory ${node_real_memory_gb} GB by ${memory_deficit_gb} GB (used: ${mem_used_gb} GB; derived maximum idle usage: ${max_idle_used_gb} GB; MemTotal: ${mem_total_gb} GB). This may indicate leftover or spurious processes consuming memory." >&3 exit 1 fi -echo "Idle node memory usage is within the configured threshold" +echo "Idle node leaves enough memory available for Slurm RealMemory" exit 0 diff --git a/helm/slurm-cluster/slurm_scripts/idle_mem_used.sh.json b/helm/slurm-cluster/slurm_scripts/idle_mem_used.sh.json index 180d7e7d5..f374276fc 100644 --- a/helm/slurm-cluster/slurm_scripts/idle_mem_used.sh.json +++ b/helm/slurm-cluster/slurm_scripts/idle_mem_used.sh.json @@ -1,6 +1,6 @@ { "name": "idle_mem_used", - "command": "IDLE_MEM_USED_MAX_USED_GB={{ required `Idle memory maximum used GB must be provided.` (index .Values.slurmScripts.builtIn `idle_mem_used.sh`).maxUsedGB }} ./idle_mem_used.sh", + "command": "./idle_mem_used.sh", "platforms": ["any"], "skip_for_cpu_jobs": false, "skip_for_partial_gpu_jobs": false, @@ -12,5 +12,5 @@ "reason_append_details": true, "run_in_jail": false, "log": "slurm_scripts/$worker.$name.$context.out", - "need_env": [] + "need_env": ["CHECKS_NODE_REAL_MEM_BYTES"] } diff --git a/helm/slurm-cluster/slurm_scripts/idle_mem_used_test.py b/helm/slurm-cluster/slurm_scripts/idle_mem_used_test.py index c5a4f138e..07d066858 100644 --- a/helm/slurm-cluster/slurm_scripts/idle_mem_used_test.py +++ b/helm/slurm-cluster/slurm_scripts/idle_mem_used_test.py @@ -17,7 +17,7 @@ def run_check( listjobs_output: str, total_bytes: int | None, available_bytes: int | None, - max_used_gb: int = 32, + node_real_memory_bytes: int | None = 56 * GIB, ) -> subprocess.CompletedProcess[str]: with tempfile.TemporaryDirectory() as tmpdir: scontrol_path = Path(tmpdir) / "scontrol" @@ -38,11 +38,14 @@ def run_check( ) env = os.environ.copy() + if node_real_memory_bytes is None: + env.pop("CHECKS_NODE_REAL_MEM_BYTES", None) + else: + env["CHECKS_NODE_REAL_MEM_BYTES"] = str(node_real_memory_bytes) env.update( { "PATH": f"{tmpdir}:{env['PATH']}", "SLURMD_NODENAME": "worker-1", - "IDLE_MEM_USED_MAX_USED_GB": str(max_used_gb), "IDLE_MEM_USED_MEMINFO_PATH": str(meminfo_path), "MOCK_SCONTROL_LISTJOBS_RC": str(listjobs_rc), "MOCK_SCONTROL_LISTJOBS_OUTPUT": listjobs_output, @@ -78,7 +81,7 @@ def test_non_idle_node_skips_memory_validation(self): self.assertIn("Node has local jobs; skipping memory validation", result.stdout) self.assertNotIn("Memory measurements:", result.stdout) - def test_idle_node_below_threshold_passes(self): + def test_idle_node_with_enough_available_memory_passes(self): result = self.run_check( listjobs_rc=1, listjobs_output="No slurmstepd's found on this node", @@ -90,9 +93,15 @@ def test_idle_node_below_threshold_passes(self): self.assertIn("Node is idle: true", result.stdout) self.assertIn("No local slurmstepd was found", result.stdout) self.assertIn(f"used=total-available=4.29 GB ({4 * GIB} bytes)", result.stdout) - self.assertIn("Maximum allowed used memory: 32 GB (32000000000 bytes)", result.stdout) + self.assertIn( + f"Slurm RealMemory: 60.13 GB ({56 * GIB} bytes)", result.stdout + ) + self.assertIn( + f"Derived maximum idle used memory: MemTotal-RealMemory=8.59 GB ({8 * GIB} bytes)", + result.stdout, + ) - def test_idle_node_above_threshold_fails_with_actionable_reason(self): + def test_idle_node_with_insufficient_available_memory_fails(self): result = self.run_check( listjobs_rc=1, listjobs_output="No slurmstepd's found on this node", @@ -102,9 +111,9 @@ def test_idle_node_above_threshold_fails_with_actionable_reason(self): self.assertEqual(1, result.returncode) self.assertIn("Node is idle: true", result.stdout) - self.assertIn("Node worker-1 is IDLE but uses", result.stdout) - self.assertIn("45.10 GB", result.stdout) - self.assertIn("threshold: 32 GB", result.stdout) + self.assertIn("Node worker-1 is IDLE but has only 23.62 GB", result.stdout) + self.assertIn("below Slurm RealMemory 60.13 GB by 36.51 GB", result.stdout) + self.assertIn("derived maximum idle usage: 8.59 GB", result.stdout) self.assertIn("leftover or spurious processes", result.stdout) def test_unavailable_memory_data_does_not_drain_node(self): @@ -118,6 +127,32 @@ def test_unavailable_memory_data_does_not_drain_node(self): self.assertEqual(0, result.returncode) self.assertIn("Could not determine valid MemTotal and MemAvailable", result.stderr) + def test_unavailable_real_memory_does_not_drain_node(self): + result = self.run_check( + listjobs_rc=1, + listjobs_output="No slurmstepd's found on this node", + total_bytes=None, + available_bytes=None, + node_real_memory_bytes=None, + ) + + self.assertEqual(0, result.returncode) + self.assertIn("Invalid or unavailable Slurm RealMemory", result.stderr) + self.assertNotIn("Memory measurements:", result.stdout) + + def test_real_memory_larger_than_memtotal_does_not_drain_node(self): + result = self.run_check( + listjobs_rc=1, + listjobs_output="No slurmstepd's found on this node", + total_bytes=64 * GIB, + available_bytes=60 * GIB, + node_real_memory_bytes=72 * GIB, + ) + + self.assertEqual(0, result.returncode) + self.assertIn("exceeds MemTotal", result.stderr) + self.assertNotIn("Derived maximum idle used memory", result.stdout) + def test_unexpected_listjobs_failure_does_not_validate_memory(self): result = self.run_check( listjobs_rc=2, diff --git a/helm/slurm-cluster/tests/idle_mem_used_test.yaml b/helm/slurm-cluster/tests/idle_mem_used_test.yaml index 52bcf53d9..6818993f1 100644 --- a/helm/slurm-cluster/tests/idle_mem_used_test.yaml +++ b/helm/slurm-cluster/tests/idle_mem_used_test.yaml @@ -2,7 +2,7 @@ suite: test idle memory health check templates: - slurm-scripts-cm.yaml tests: - - it: should enable the idle memory check with the default threshold + - it: should enable the idle memory check with node RealMemory asserts: - isNotNull: path: data["idle_mem_used.sh"] @@ -11,7 +11,7 @@ tests: pattern: '"name": "idle_mem_used"' - matchRegex: path: data["checks.json"] - pattern: '"command": "IDLE_MEM_USED_MAX_USED_GB=32 ./idle_mem_used.sh"' + pattern: '"command": "./idle_mem_used.sh"' - matchRegex: path: data["checks.json"] pattern: '"contexts": \[\s*"hc_program"\s*\]' @@ -20,18 +20,7 @@ tests: pattern: '"on_fail": "drain"' - matchRegex: path: data["checks.json"] - pattern: '"need_env": \[\s*\]' - - - it: should render a configured maximum used memory threshold - set: - slurmScripts: - builtIn: - idle_mem_used.sh: - maxUsedGB: 16 - asserts: - - matchRegex: - path: data["checks.json"] - pattern: '"command": "IDLE_MEM_USED_MAX_USED_GB=16 ./idle_mem_used.sh"' + pattern: '"need_env": \[\s*"CHECKS_NODE_REAL_MEM_BYTES"\s*\]' - it: should allow the idle memory check to be disabled set: diff --git a/helm/slurm-cluster/values.yaml b/helm/slurm-cluster/values.yaml index b4edb0d6a..8dd7fe98a 100644 --- a/helm/slurm-cluster/values.yaml +++ b/helm/slurm-cluster/values.yaml @@ -776,8 +776,6 @@ slurmScripts: customConfig: null idle_mem_used.sh: enabled: true - # Drain an idle node when MemTotal - MemAvailable exceeds 32 GB. - maxUsedGB: 32 customContent: null customConfig: null job_tmpfs_delete.sh: From 57ae072c2abb90134ad9aab6f30f296c196cfc82 Mon Sep 17 00:00:00 2001 From: Fabrizio Waldner Date: Thu, 23 Jul 2026 15:59:16 +0200 Subject: [PATCH 4/8] Use json format + jq --- .../slurm_scripts/idle_mem_used.sh | 40 ++++++++---- .../slurm_scripts/idle_mem_used_test.py | 62 ++++++++++++++----- 2 files changed, 75 insertions(+), 27 deletions(-) diff --git a/helm/slurm-cluster/slurm_scripts/idle_mem_used.sh b/helm/slurm-cluster/slurm_scripts/idle_mem_used.sh index d97640c57..942aeb573 100644 --- a/helm/slurm-cluster/slurm_scripts/idle_mem_used.sh +++ b/helm/slurm-cluster/slurm_scripts/idle_mem_used.sh @@ -8,27 +8,43 @@ node_real_memory_bytes="${CHECKS_NODE_REAL_MEM_BYTES:-}" echo "[$(date)] Check memory usage when node ${node_name} is idle" echo "Slurm RealMemory input: ${node_real_memory_bytes:-} bytes" -echo "Idle state source: local 'scontrol listjobs' (no controller RPC)" -echo "Idle state rule: exit code 1 with \"No slurmstepd's found on this node\" means the node has no local jobs" +echo "Idle state source: local 'scontrol listjobs --json' (no controller RPC)" +echo "Idle state rule: an empty JSON .jobs array means the node has no local jobs" -if listjobs_output="$(scontrol listjobs 2>&1)"; then +if listjobs_output="$(scontrol listjobs --json 2>&1)"; then listjobs_rc=0 else listjobs_rc=$? fi -echo "scontrol listjobs exit code: ${listjobs_rc}" -echo "scontrol listjobs output: ${listjobs_output:-}" +echo "scontrol listjobs --json exit code: ${listjobs_rc}" +echo "scontrol listjobs --json output: ${listjobs_output:-}" -if (( listjobs_rc == 0 )); then - node_is_idle=false - echo "Local Slurm jobs are present; treating the node as non-idle" -elif (( listjobs_rc == 1 )) && [[ "${listjobs_output}" == *"No slurmstepd's found on this node"* ]]; then +if (( listjobs_rc != 0 )); then + echo "Could not determine whether the node is idle because 'scontrol listjobs --json' failed; skipping memory validation" >&2 + exit 0 +fi + +if ! local_job_count="$( + jq -er ' + if (.jobs | type) == "array" then + .jobs | length + else + error(".jobs must be an array") + end + ' <<<"${listjobs_output}" 2>/dev/null +)"; then + echo "Could not determine whether the node is idle because 'scontrol listjobs --json' returned invalid job data; skipping memory validation" >&2 + exit 0 +fi + +echo "Local Slurm job count from JSON .jobs array: ${local_job_count}" +if (( local_job_count == 0 )); then node_is_idle=true - echo "No local slurmstepd was found; treating the node as idle" + echo "The JSON .jobs array is empty; treating the node as idle" else - echo "Could not determine whether the node is idle from 'scontrol listjobs'; skipping memory validation" >&2 - exit 0 + node_is_idle=false + echo "The JSON .jobs array contains local jobs; treating the node as non-idle" fi echo "Node is idle: ${node_is_idle}" diff --git a/helm/slurm-cluster/slurm_scripts/idle_mem_used_test.py b/helm/slurm-cluster/slurm_scripts/idle_mem_used_test.py index 07d066858..63b12567a 100644 --- a/helm/slurm-cluster/slurm_scripts/idle_mem_used_test.py +++ b/helm/slurm-cluster/slurm_scripts/idle_mem_used_test.py @@ -23,6 +23,10 @@ def run_check( scontrol_path = Path(tmpdir) / "scontrol" scontrol_path.write_text( "#!/bin/bash\n" + 'if [[ "$*" != "listjobs --json" ]]; then\n' + ' printf \'Unexpected arguments: %s\\n\' "$*" >&2\n' + " exit 64\n" + "fi\n" "printf '%s\\n' \"${MOCK_SCONTROL_LISTJOBS_OUTPUT}\"\n" "exit \"${MOCK_SCONTROL_LISTJOBS_RC}\"\n", encoding="utf-8", @@ -70,28 +74,30 @@ def run_check( def test_non_idle_node_skips_memory_validation(self): result = self.run_check( listjobs_rc=0, - listjobs_output="JOBID\n1011", + listjobs_output='{"jobs":[{"job_id":1011}],"errors":[]}', total_bytes=None, available_bytes=None, ) self.assertEqual(0, result.returncode) self.assertIn("Node is idle: false", result.stdout) - self.assertIn("Local Slurm jobs are present", result.stdout) + self.assertIn("Local Slurm job count from JSON .jobs array: 1", result.stdout) + self.assertIn("JSON .jobs array contains local jobs", result.stdout) self.assertIn("Node has local jobs; skipping memory validation", result.stdout) self.assertNotIn("Memory measurements:", result.stdout) def test_idle_node_with_enough_available_memory_passes(self): result = self.run_check( - listjobs_rc=1, - listjobs_output="No slurmstepd's found on this node", + listjobs_rc=0, + listjobs_output='{"jobs":[],"errors":[]}', total_bytes=64 * GIB, available_bytes=60 * GIB, ) self.assertEqual(0, result.returncode) self.assertIn("Node is idle: true", result.stdout) - self.assertIn("No local slurmstepd was found", result.stdout) + self.assertIn("Local Slurm job count from JSON .jobs array: 0", result.stdout) + self.assertIn("JSON .jobs array is empty", result.stdout) self.assertIn(f"used=total-available=4.29 GB ({4 * GIB} bytes)", result.stdout) self.assertIn( f"Slurm RealMemory: 60.13 GB ({56 * GIB} bytes)", result.stdout @@ -103,8 +109,8 @@ def test_idle_node_with_enough_available_memory_passes(self): def test_idle_node_with_insufficient_available_memory_fails(self): result = self.run_check( - listjobs_rc=1, - listjobs_output="No slurmstepd's found on this node", + listjobs_rc=0, + listjobs_output='{"jobs":[],"errors":[]}', total_bytes=64 * GIB, available_bytes=22 * GIB, ) @@ -118,8 +124,8 @@ def test_idle_node_with_insufficient_available_memory_fails(self): def test_unavailable_memory_data_does_not_drain_node(self): result = self.run_check( - listjobs_rc=1, - listjobs_output="No slurmstepd's found on this node", + listjobs_rc=0, + listjobs_output='{"jobs":[],"errors":[]}', total_bytes=None, available_bytes=None, ) @@ -129,8 +135,8 @@ def test_unavailable_memory_data_does_not_drain_node(self): def test_unavailable_real_memory_does_not_drain_node(self): result = self.run_check( - listjobs_rc=1, - listjobs_output="No slurmstepd's found on this node", + listjobs_rc=0, + listjobs_output='{"jobs":[],"errors":[]}', total_bytes=None, available_bytes=None, node_real_memory_bytes=None, @@ -142,8 +148,8 @@ def test_unavailable_real_memory_does_not_drain_node(self): def test_real_memory_larger_than_memtotal_does_not_drain_node(self): result = self.run_check( - listjobs_rc=1, - listjobs_output="No slurmstepd's found on this node", + listjobs_rc=0, + listjobs_output='{"jobs":[],"errors":[]}', total_bytes=64 * GIB, available_bytes=60 * GIB, node_real_memory_bytes=72 * GIB, @@ -162,8 +168,34 @@ def test_unexpected_listjobs_failure_does_not_validate_memory(self): ) self.assertEqual(0, result.returncode) - self.assertIn("scontrol listjobs exit code: 2", result.stdout) - self.assertIn("Could not determine whether the node is idle", result.stderr) + self.assertIn("scontrol listjobs --json exit code: 2", result.stdout) + self.assertIn("scontrol listjobs --json' failed", result.stderr) + self.assertNotIn("Memory measurements:", result.stdout) + + def test_invalid_listjobs_json_does_not_validate_memory(self): + result = self.run_check( + listjobs_rc=0, + listjobs_output="not JSON", + total_bytes=None, + available_bytes=None, + ) + + self.assertEqual(0, result.returncode) + self.assertIn("returned invalid job data", result.stderr) + self.assertNotIn("Node is idle:", result.stdout) + self.assertNotIn("Memory measurements:", result.stdout) + + def test_missing_jobs_array_does_not_validate_memory(self): + result = self.run_check( + listjobs_rc=0, + listjobs_output='{"jobs":null,"errors":[]}', + total_bytes=None, + available_bytes=None, + ) + + self.assertEqual(0, result.returncode) + self.assertIn("returned invalid job data", result.stderr) + self.assertNotIn("Node is idle:", result.stdout) self.assertNotIn("Memory measurements:", result.stdout) From 1f3aa8f3c91017631231c612f2ebde2ca68b5e27 Mon Sep 17 00:00:00 2001 From: Fabrizio Waldner Date: Thu, 23 Jul 2026 16:32:56 +0200 Subject: [PATCH 5/8] Use "free -hw" to print debug messages --- .../slurm_scripts/idle_mem_used.sh | 47 ++++------ .../slurm_scripts/idle_mem_used_test.py | 85 ++++++++++++++----- 2 files changed, 81 insertions(+), 51 deletions(-) diff --git a/helm/slurm-cluster/slurm_scripts/idle_mem_used.sh b/helm/slurm-cluster/slurm_scripts/idle_mem_used.sh index 942aeb573..2c48a3b0d 100644 --- a/helm/slurm-cluster/slurm_scripts/idle_mem_used.sh +++ b/helm/slurm-cluster/slurm_scripts/idle_mem_used.sh @@ -2,7 +2,6 @@ set -euo pipefail -meminfo_path="${IDLE_MEM_USED_MEMINFO_PATH:-/proc/meminfo}" node_name="${SLURMD_NODENAME:-unknown}" node_real_memory_bytes="${CHECKS_NODE_REAL_MEM_BYTES:-}" @@ -58,48 +57,38 @@ if ! [[ "${node_real_memory_bytes}" =~ ^[0-9]+$ ]] || [[ "${node_real_memory_byt exit 0 fi -meminfo_values="$( - awk ' - /^MemTotal:/ { total = $2 } - /^MemAvailable:/ { available = $2 } - END { if (total != "" && available != "") print total, available } - ' "${meminfo_path}" 2>/dev/null || true -)" -read -r mem_total_kib mem_available_kib <<< "${meminfo_values}" - -if ! [[ "${mem_total_kib:-}" =~ ^[0-9]+$ ]] || - ! [[ "${mem_available_kib:-}" =~ ^[0-9]+$ ]] || - (( mem_available_kib > mem_total_kib )); then - echo "Could not determine valid MemTotal and MemAvailable values from ${meminfo_path}; skipping memory validation" >&2 +if ! free_bytes_output="$(LC_ALL=C free -b 2>&1)"; then + echo "Could not read local memory information with 'free -b'; skipping memory validation" >&2 exit 0 fi -mem_total_bytes=$((mem_total_kib * 1024)) -mem_available_bytes=$((mem_available_kib * 1024)) -mem_used_bytes=$((mem_total_bytes - mem_available_bytes)) +memory_values="$(awk '/^Mem:/ { print $2, $7; exit }' <<<"${free_bytes_output}")" +read -r mem_total_bytes mem_available_bytes <<<"${memory_values}" + +if ! [[ "${mem_total_bytes:-}" =~ ^[0-9]+$ ]] || + ! [[ "${mem_available_bytes:-}" =~ ^[0-9]+$ ]] || + (( mem_available_bytes > mem_total_bytes )); then + echo "Could not determine valid total and available memory from 'free -b'; skipping memory validation" >&2 + exit 0 +fi if (( node_real_memory_bytes > mem_total_bytes )); then echo "Slurm RealMemory ${node_real_memory_bytes} bytes exceeds MemTotal ${mem_total_bytes} bytes; skipping memory validation" >&2 exit 0 fi -max_idle_used_bytes=$((mem_total_bytes - node_real_memory_bytes)) - -mem_total_gb="$(awk -v bytes="${mem_total_bytes}" 'BEGIN { printf "%.2f", bytes / 1000000000 }')" mem_available_gb="$(awk -v bytes="${mem_available_bytes}" 'BEGIN { printf "%.2f", bytes / 1000000000 }')" -mem_used_gb="$(awk -v bytes="${mem_used_bytes}" 'BEGIN { printf "%.2f", bytes / 1000000000 }')" node_real_memory_gb="$(awk -v bytes="${node_real_memory_bytes}" 'BEGIN { printf "%.2f", bytes / 1000000000 }')" -max_idle_used_gb="$(awk -v bytes="${max_idle_used_bytes}" 'BEGIN { printf "%.2f", bytes / 1000000000 }')" -echo "Memory source: ${meminfo_path}" -echo "Memory measurements: total=${mem_total_gb} GB (${mem_total_bytes} bytes), available=${mem_available_gb} GB (${mem_available_bytes} bytes), used=total-available=${mem_used_gb} GB (${mem_used_bytes} bytes)" -echo "Slurm RealMemory: ${node_real_memory_gb} GB (${node_real_memory_bytes} bytes)" -echo "Derived maximum idle used memory: MemTotal-RealMemory=${max_idle_used_gb} GB (${max_idle_used_bytes} bytes)" +echo "Memory source: local 'free'" +echo "Memory comparison: available=${mem_available_gb} GB (${mem_available_bytes} bytes), Slurm RealMemory=${node_real_memory_gb} GB (${node_real_memory_bytes} bytes)" +echo "Memory snapshot (free -hw):" +if ! LC_ALL=C free -hw; then + echo "Could not print the human-readable memory snapshot with 'free -hw'" >&2 +fi if (( mem_available_bytes < node_real_memory_bytes )); then - memory_deficit_bytes=$((node_real_memory_bytes - mem_available_bytes)) - memory_deficit_gb="$(awk -v bytes="${memory_deficit_bytes}" 'BEGIN { printf "%.2f", bytes / 1000000000 }')" - echo "Node ${node_name} is IDLE but has only ${mem_available_gb} GB of memory available, below Slurm RealMemory ${node_real_memory_gb} GB by ${memory_deficit_gb} GB (used: ${mem_used_gb} GB; derived maximum idle usage: ${max_idle_used_gb} GB; MemTotal: ${mem_total_gb} GB). This may indicate leftover or spurious processes consuming memory." >&3 + echo "available memory ${mem_available_gb} GB < Slurm RealMemory ${node_real_memory_gb} GB; stop leftover processes or reboot" >&3 exit 1 fi diff --git a/helm/slurm-cluster/slurm_scripts/idle_mem_used_test.py b/helm/slurm-cluster/slurm_scripts/idle_mem_used_test.py index 63b12567a..8d3be89be 100644 --- a/helm/slurm-cluster/slurm_scripts/idle_mem_used_test.py +++ b/helm/slurm-cluster/slurm_scripts/idle_mem_used_test.py @@ -18,6 +18,7 @@ def run_check( total_bytes: int | None, available_bytes: int | None, node_real_memory_bytes: int | None = 56 * GIB, + free_bytes_rc: int = 0, ) -> subprocess.CompletedProcess[str]: with tempfile.TemporaryDirectory() as tmpdir: scontrol_path = Path(tmpdir) / "scontrol" @@ -33,13 +34,31 @@ def run_check( ) scontrol_path.chmod(0o755) - meminfo_path = Path(tmpdir) / "meminfo" - if total_bytes is not None and available_bytes is not None: - meminfo_path.write_text( - f"MemTotal: {total_bytes // 1024} kB\n" - f"MemAvailable: {available_bytes // 1024} kB\n", - encoding="utf-8", - ) + free_path = Path(tmpdir) / "free" + free_path.write_text( + "#!/bin/bash\n" + 'case "$*" in\n' + ' "-b")\n' + ' if (( MOCK_FREE_BYTES_RC != 0 )); then\n' + ' printf "Unable to read local memory\\n" >&2\n' + ' exit "${MOCK_FREE_BYTES_RC}"\n' + " fi\n" + ' printf " total used free shared buff/cache available\\n"\n' + ' printf "Mem: %s 0 0 0 0 %s\\n" \\\n' + ' "${MOCK_FREE_TOTAL_BYTES}" "${MOCK_FREE_AVAILABLE_BYTES}"\n' + " ;;\n" + ' "-hw")\n' + ' printf " total used free shared buffers cache available\\n"\n' + ' printf "Mem: mock mock mock mock mock mock mock\\n"\n' + " ;;\n" + " *)\n" + ' printf \'Unexpected arguments: %s\\n\' "$*" >&2\n' + " exit 64\n" + " ;;\n" + "esac\n", + encoding="utf-8", + ) + free_path.chmod(0o755) env = os.environ.copy() if node_real_memory_bytes is None: @@ -50,9 +69,15 @@ def run_check( { "PATH": f"{tmpdir}:{env['PATH']}", "SLURMD_NODENAME": "worker-1", - "IDLE_MEM_USED_MEMINFO_PATH": str(meminfo_path), "MOCK_SCONTROL_LISTJOBS_RC": str(listjobs_rc), "MOCK_SCONTROL_LISTJOBS_OUTPUT": listjobs_output, + "MOCK_FREE_TOTAL_BYTES": ( + str(total_bytes) if total_bytes is not None else "" + ), + "MOCK_FREE_AVAILABLE_BYTES": ( + str(available_bytes) if available_bytes is not None else "" + ), + "MOCK_FREE_BYTES_RC": str(free_bytes_rc), } ) @@ -84,7 +109,7 @@ def test_non_idle_node_skips_memory_validation(self): self.assertIn("Local Slurm job count from JSON .jobs array: 1", result.stdout) self.assertIn("JSON .jobs array contains local jobs", result.stdout) self.assertIn("Node has local jobs; skipping memory validation", result.stdout) - self.assertNotIn("Memory measurements:", result.stdout) + self.assertNotIn("Memory comparison:", result.stdout) def test_idle_node_with_enough_available_memory_passes(self): result = self.run_check( @@ -98,14 +123,15 @@ def test_idle_node_with_enough_available_memory_passes(self): self.assertIn("Node is idle: true", result.stdout) self.assertIn("Local Slurm job count from JSON .jobs array: 0", result.stdout) self.assertIn("JSON .jobs array is empty", result.stdout) - self.assertIn(f"used=total-available=4.29 GB ({4 * GIB} bytes)", result.stdout) self.assertIn( - f"Slurm RealMemory: 60.13 GB ({56 * GIB} bytes)", result.stdout + f"available=64.42 GB ({60 * GIB} bytes)", result.stdout ) self.assertIn( - f"Derived maximum idle used memory: MemTotal-RealMemory=8.59 GB ({8 * GIB} bytes)", + f"Slurm RealMemory=60.13 GB ({56 * GIB} bytes)", result.stdout, ) + self.assertIn("Memory snapshot (free -hw):", result.stdout) + self.assertIn("Mem: mock", result.stdout) def test_idle_node_with_insufficient_available_memory_fails(self): result = self.run_check( @@ -117,10 +143,10 @@ def test_idle_node_with_insufficient_available_memory_fails(self): self.assertEqual(1, result.returncode) self.assertIn("Node is idle: true", result.stdout) - self.assertIn("Node worker-1 is IDLE but has only 23.62 GB", result.stdout) - self.assertIn("below Slurm RealMemory 60.13 GB by 36.51 GB", result.stdout) - self.assertIn("derived maximum idle usage: 8.59 GB", result.stdout) - self.assertIn("leftover or spurious processes", result.stdout) + self.assertIn( + "available memory 23.62 GB < Slurm RealMemory 60.13 GB", result.stdout + ) + self.assertIn("stop leftover processes or reboot", result.stdout) def test_unavailable_memory_data_does_not_drain_node(self): result = self.run_check( @@ -131,7 +157,22 @@ def test_unavailable_memory_data_does_not_drain_node(self): ) self.assertEqual(0, result.returncode) - self.assertIn("Could not determine valid MemTotal and MemAvailable", result.stderr) + self.assertIn( + "Could not determine valid total and available memory", result.stderr + ) + + def test_free_failure_does_not_drain_node(self): + result = self.run_check( + listjobs_rc=0, + listjobs_output='{"jobs":[],"errors":[]}', + total_bytes=None, + available_bytes=None, + free_bytes_rc=2, + ) + + self.assertEqual(0, result.returncode) + self.assertIn("Could not read local memory information", result.stderr) + self.assertNotIn("Memory comparison:", result.stdout) def test_unavailable_real_memory_does_not_drain_node(self): result = self.run_check( @@ -144,7 +185,7 @@ def test_unavailable_real_memory_does_not_drain_node(self): self.assertEqual(0, result.returncode) self.assertIn("Invalid or unavailable Slurm RealMemory", result.stderr) - self.assertNotIn("Memory measurements:", result.stdout) + self.assertNotIn("Memory comparison:", result.stdout) def test_real_memory_larger_than_memtotal_does_not_drain_node(self): result = self.run_check( @@ -157,7 +198,7 @@ def test_real_memory_larger_than_memtotal_does_not_drain_node(self): self.assertEqual(0, result.returncode) self.assertIn("exceeds MemTotal", result.stderr) - self.assertNotIn("Derived maximum idle used memory", result.stdout) + self.assertNotIn("Memory comparison:", result.stdout) def test_unexpected_listjobs_failure_does_not_validate_memory(self): result = self.run_check( @@ -170,7 +211,7 @@ def test_unexpected_listjobs_failure_does_not_validate_memory(self): self.assertEqual(0, result.returncode) self.assertIn("scontrol listjobs --json exit code: 2", result.stdout) self.assertIn("scontrol listjobs --json' failed", result.stderr) - self.assertNotIn("Memory measurements:", result.stdout) + self.assertNotIn("Memory comparison:", result.stdout) def test_invalid_listjobs_json_does_not_validate_memory(self): result = self.run_check( @@ -183,7 +224,7 @@ def test_invalid_listjobs_json_does_not_validate_memory(self): self.assertEqual(0, result.returncode) self.assertIn("returned invalid job data", result.stderr) self.assertNotIn("Node is idle:", result.stdout) - self.assertNotIn("Memory measurements:", result.stdout) + self.assertNotIn("Memory comparison:", result.stdout) def test_missing_jobs_array_does_not_validate_memory(self): result = self.run_check( @@ -196,7 +237,7 @@ def test_missing_jobs_array_does_not_validate_memory(self): self.assertEqual(0, result.returncode) self.assertIn("returned invalid job data", result.stderr) self.assertNotIn("Node is idle:", result.stdout) - self.assertNotIn("Memory measurements:", result.stdout) + self.assertNotIn("Memory comparison:", result.stdout) if __name__ == "__main__": From 5bf949dd63c7d0199fbe74d064218b56c1af9b2c Mon Sep 17 00:00:00 2001 From: Fabrizio Waldner Date: Thu, 23 Jul 2026 16:34:59 +0200 Subject: [PATCH 6/8] Change to user_problem --- helm/slurm-cluster/slurm_scripts/idle_mem_used.sh.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm/slurm-cluster/slurm_scripts/idle_mem_used.sh.json b/helm/slurm-cluster/slurm_scripts/idle_mem_used.sh.json index f374276fc..d1617e4dc 100644 --- a/helm/slurm-cluster/slurm_scripts/idle_mem_used.sh.json +++ b/helm/slurm-cluster/slurm_scripts/idle_mem_used.sh.json @@ -8,7 +8,7 @@ "node_states": ["any"], "on_fail": "drain", "on_ok": "none", - "reason_base": "[node_problem] $name", + "reason_base": "[user_problem] $name", "reason_append_details": true, "run_in_jail": false, "log": "slurm_scripts/$worker.$name.$context.out", From 277f99be84fc509b6fabacb710b97c78917f6226 Mon Sep 17 00:00:00 2001 From: Fabrizio Waldner Date: Thu, 23 Jul 2026 16:43:28 +0200 Subject: [PATCH 7/8] Simplify the drain reason --- helm/slurm-cluster/slurm_scripts/idle_mem_used.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm/slurm-cluster/slurm_scripts/idle_mem_used.sh b/helm/slurm-cluster/slurm_scripts/idle_mem_used.sh index 2c48a3b0d..c748c562f 100644 --- a/helm/slurm-cluster/slurm_scripts/idle_mem_used.sh +++ b/helm/slurm-cluster/slurm_scripts/idle_mem_used.sh @@ -88,7 +88,7 @@ if ! LC_ALL=C free -hw; then fi if (( mem_available_bytes < node_real_memory_bytes )); then - echo "available memory ${mem_available_gb} GB < Slurm RealMemory ${node_real_memory_gb} GB; stop leftover processes or reboot" >&3 + echo "available memory ${mem_available_gb} GB < configured ${node_real_memory_gb} GB; stop leftover processes or reboot" >&3 exit 1 fi From a5c2c108bb529102d4e83d9b6e598b8c873a265b Mon Sep 17 00:00:00 2001 From: Fabrizio Waldner Date: Thu, 23 Jul 2026 17:19:55 +0200 Subject: [PATCH 8/8] Add script to undrain if there is enough memory --- ...dle_mem_used.sh => idle_mem_used.drain.sh} | 0 ...ed.sh.json => idle_mem_used.drain.sh.json} | 4 +- .../slurm_scripts/idle_mem_used.undrain.sh | 55 +++++++++++++++++++ .../idle_mem_used.undrain.sh.json | 16 ++++++ .../slurm_scripts/idle_mem_used_test.py | 51 ++++++++++++++++- .../tests/idle_mem_used_test.yaml | 27 +++++++-- helm/slurm-cluster/values.yaml | 6 +- 7 files changed, 147 insertions(+), 12 deletions(-) rename helm/slurm-cluster/slurm_scripts/{idle_mem_used.sh => idle_mem_used.drain.sh} (100%) rename helm/slurm-cluster/slurm_scripts/{idle_mem_used.sh.json => idle_mem_used.drain.sh.json} (78%) create mode 100644 helm/slurm-cluster/slurm_scripts/idle_mem_used.undrain.sh create mode 100644 helm/slurm-cluster/slurm_scripts/idle_mem_used.undrain.sh.json diff --git a/helm/slurm-cluster/slurm_scripts/idle_mem_used.sh b/helm/slurm-cluster/slurm_scripts/idle_mem_used.drain.sh similarity index 100% rename from helm/slurm-cluster/slurm_scripts/idle_mem_used.sh rename to helm/slurm-cluster/slurm_scripts/idle_mem_used.drain.sh diff --git a/helm/slurm-cluster/slurm_scripts/idle_mem_used.sh.json b/helm/slurm-cluster/slurm_scripts/idle_mem_used.drain.sh.json similarity index 78% rename from helm/slurm-cluster/slurm_scripts/idle_mem_used.sh.json rename to helm/slurm-cluster/slurm_scripts/idle_mem_used.drain.sh.json index d1617e4dc..6887d5b39 100644 --- a/helm/slurm-cluster/slurm_scripts/idle_mem_used.sh.json +++ b/helm/slurm-cluster/slurm_scripts/idle_mem_used.drain.sh.json @@ -1,6 +1,6 @@ { "name": "idle_mem_used", - "command": "./idle_mem_used.sh", + "command": "./idle_mem_used.drain.sh", "platforms": ["any"], "skip_for_cpu_jobs": false, "skip_for_partial_gpu_jobs": false, @@ -11,6 +11,6 @@ "reason_base": "[user_problem] $name", "reason_append_details": true, "run_in_jail": false, - "log": "slurm_scripts/$worker.$name.$context.out", + "log": "slurm_scripts/$worker.$name.drain.$context.out", "need_env": ["CHECKS_NODE_REAL_MEM_BYTES"] } diff --git a/helm/slurm-cluster/slurm_scripts/idle_mem_used.undrain.sh b/helm/slurm-cluster/slurm_scripts/idle_mem_used.undrain.sh new file mode 100644 index 000000000..9f40517b2 --- /dev/null +++ b/helm/slurm-cluster/slurm_scripts/idle_mem_used.undrain.sh @@ -0,0 +1,55 @@ +#!/bin/bash + +set -euo pipefail + +# This recovery check must fail closed: only a fully validated healthy result +# may allow check_runner to undrain the node. +node_name="${SLURMD_NODENAME:-unknown}" +node_real_memory_bytes="${CHECKS_NODE_REAL_MEM_BYTES:-}" + +echo "[$(date)] Check whether memory usage has recovered on drained node ${node_name}" +echo "Slurm RealMemory input: ${node_real_memory_bytes:-} bytes" +echo "Node eligibility source: this check is scheduled only for drained nodes" + +if ! [[ "${node_real_memory_bytes}" =~ ^[0-9]+$ ]] || [[ "${node_real_memory_bytes}" == "0" ]]; then + echo "Invalid or unavailable Slurm RealMemory '${node_real_memory_bytes:-}'; keeping node drained" >&2 + exit 1 +fi + +if ! free_bytes_output="$(LC_ALL=C free -b 2>&1)"; then + echo "Could not read local memory information with 'free -b'; keeping node drained" >&2 + exit 1 +fi + +memory_values="$(awk '/^Mem:/ { print $2, $7; exit }' <<<"${free_bytes_output}")" +read -r mem_total_bytes mem_available_bytes <<<"${memory_values}" + +if ! [[ "${mem_total_bytes:-}" =~ ^[0-9]+$ ]] || + ! [[ "${mem_available_bytes:-}" =~ ^[0-9]+$ ]] || + (( mem_available_bytes > mem_total_bytes )); then + echo "Could not determine valid total and available memory from 'free -b'; keeping node drained" >&2 + exit 1 +fi + +if (( node_real_memory_bytes > mem_total_bytes )); then + echo "Slurm RealMemory ${node_real_memory_bytes} bytes exceeds MemTotal ${mem_total_bytes} bytes; keeping node drained" >&2 + exit 1 +fi + +mem_available_gb="$(awk -v bytes="${mem_available_bytes}" 'BEGIN { printf "%.2f", bytes / 1000000000 }')" +node_real_memory_gb="$(awk -v bytes="${node_real_memory_bytes}" 'BEGIN { printf "%.2f", bytes / 1000000000 }')" + +echo "Memory source: local 'free'" +echo "Memory comparison: available=${mem_available_gb} GB (${mem_available_bytes} bytes), Slurm RealMemory=${node_real_memory_gb} GB (${node_real_memory_bytes} bytes)" +echo "Memory snapshot (free -hw):" +if ! LC_ALL=C free -hw; then + echo "Could not print the human-readable memory snapshot with 'free -hw'" >&2 +fi + +if (( mem_available_bytes < node_real_memory_bytes )); then + echo "Available memory ${mem_available_gb} GB is still below configured memory ${node_real_memory_gb} GB; keeping node drained" >&2 + exit 1 +fi + +echo "Drained node leaves enough memory available for Slurm RealMemory; memory recovery confirmed" +exit 0 diff --git a/helm/slurm-cluster/slurm_scripts/idle_mem_used.undrain.sh.json b/helm/slurm-cluster/slurm_scripts/idle_mem_used.undrain.sh.json new file mode 100644 index 000000000..d772f693f --- /dev/null +++ b/helm/slurm-cluster/slurm_scripts/idle_mem_used.undrain.sh.json @@ -0,0 +1,16 @@ +{ + "name": "idle_mem_used", + "command": "./idle_mem_used.undrain.sh", + "platforms": ["any"], + "skip_for_cpu_jobs": false, + "skip_for_partial_gpu_jobs": false, + "contexts": ["hc_program"], + "node_states": ["drain"], + "on_fail": "none", + "on_ok": "undrain", + "reason_base": "[user_problem] $name", + "reason_append_details": false, + "run_in_jail": false, + "log": "slurm_scripts/$worker.$name.undrain.$context.out", + "need_env": ["CHECKS_NODE_REAL_MEM_BYTES"] +} diff --git a/helm/slurm-cluster/slurm_scripts/idle_mem_used_test.py b/helm/slurm-cluster/slurm_scripts/idle_mem_used_test.py index 8d3be89be..7060dab98 100644 --- a/helm/slurm-cluster/slurm_scripts/idle_mem_used_test.py +++ b/helm/slurm-cluster/slurm_scripts/idle_mem_used_test.py @@ -5,7 +5,8 @@ from pathlib import Path -SCRIPT_PATH = Path(__file__).with_name("idle_mem_used.sh") +DRAIN_SCRIPT_PATH = Path(__file__).with_name("idle_mem_used.drain.sh") +UNDRAIN_SCRIPT_PATH = Path(__file__).with_name("idle_mem_used.undrain.sh") GIB = 1024 * 1024 * 1024 @@ -19,6 +20,7 @@ def run_check( available_bytes: int | None, node_real_memory_bytes: int | None = 56 * GIB, free_bytes_rc: int = 0, + script_path: Path = DRAIN_SCRIPT_PATH, ) -> subprocess.CompletedProcess[str]: with tempfile.TemporaryDirectory() as tmpdir: scontrol_path = Path(tmpdir) / "scontrol" @@ -87,7 +89,7 @@ def run_check( "-c", 'exec 3>&1; exec bash "$1"', "idle-mem-used-test", - str(SCRIPT_PATH), + str(script_path), ], check=False, env=env, @@ -144,7 +146,7 @@ def test_idle_node_with_insufficient_available_memory_fails(self): self.assertEqual(1, result.returncode) self.assertIn("Node is idle: true", result.stdout) self.assertIn( - "available memory 23.62 GB < Slurm RealMemory 60.13 GB", result.stdout + "available memory 23.62 GB < configured 60.13 GB", result.stdout ) self.assertIn("stop leftover processes or reboot", result.stdout) @@ -239,6 +241,49 @@ def test_missing_jobs_array_does_not_validate_memory(self): self.assertNotIn("Node is idle:", result.stdout) self.assertNotIn("Memory comparison:", result.stdout) + def test_undrain_confirms_recovery_without_querying_local_jobs(self): + result = self.run_check( + listjobs_rc=64, + listjobs_output="undrain must not query local jobs", + total_bytes=64 * GIB, + available_bytes=60 * GIB, + script_path=UNDRAIN_SCRIPT_PATH, + ) + + self.assertEqual(0, result.returncode) + self.assertNotIn("listjobs", result.stdout) + self.assertNotIn("listjobs", result.stderr) + self.assertIn("scheduled only for drained nodes", result.stdout) + self.assertIn("memory recovery confirmed", result.stdout) + + def test_undrain_keeps_node_drained_when_available_memory_is_insufficient(self): + result = self.run_check( + listjobs_rc=64, + listjobs_output="undrain must not query local jobs", + total_bytes=64 * GIB, + available_bytes=22 * GIB, + script_path=UNDRAIN_SCRIPT_PATH, + ) + + self.assertEqual(1, result.returncode) + self.assertIn("is still below configured memory", result.stderr) + self.assertIn("keeping node drained", result.stderr) + + def test_undrain_keeps_node_drained_when_memory_data_is_unavailable(self): + result = self.run_check( + listjobs_rc=64, + listjobs_output="undrain must not query local jobs", + total_bytes=None, + available_bytes=None, + free_bytes_rc=2, + script_path=UNDRAIN_SCRIPT_PATH, + ) + + self.assertEqual(1, result.returncode) + self.assertIn("Could not read local memory information", result.stderr) + self.assertIn("keeping node drained", result.stderr) + self.assertNotIn("Memory comparison:", result.stdout) + if __name__ == "__main__": unittest.main(verbosity=2) diff --git a/helm/slurm-cluster/tests/idle_mem_used_test.yaml b/helm/slurm-cluster/tests/idle_mem_used_test.yaml index 6818993f1..b70c982c6 100644 --- a/helm/slurm-cluster/tests/idle_mem_used_test.yaml +++ b/helm/slurm-cluster/tests/idle_mem_used_test.yaml @@ -2,22 +2,33 @@ suite: test idle memory health check templates: - slurm-scripts-cm.yaml tests: - - it: should enable the idle memory check with node RealMemory + - it: should enable the idle memory drain and undrain checks with node RealMemory asserts: - isNotNull: - path: data["idle_mem_used.sh"] + path: data["idle_mem_used.drain.sh"] + - isNotNull: + path: data["idle_mem_used.undrain.sh"] - matchRegex: path: data["checks.json"] pattern: '"name": "idle_mem_used"' - matchRegex: path: data["checks.json"] - pattern: '"command": "./idle_mem_used.sh"' + pattern: '"command": "./idle_mem_used.drain.sh"' + - matchRegex: + path: data["checks.json"] + pattern: '"command": "./idle_mem_used.undrain.sh"' - matchRegex: path: data["checks.json"] pattern: '"contexts": \[\s*"hc_program"\s*\]' - matchRegex: path: data["checks.json"] pattern: '"on_fail": "drain"' + - matchRegex: + path: data["checks.json"] + pattern: '"node_states": \[\s*"drain"\s*\]' + - matchRegex: + path: data["checks.json"] + pattern: '"on_ok": "undrain"' - matchRegex: path: data["checks.json"] pattern: '"need_env": \[\s*"CHECKS_NODE_REAL_MEM_BYTES"\s*\]' @@ -26,11 +37,15 @@ tests: set: slurmScripts: builtIn: - idle_mem_used.sh: + idle_mem_used.drain.sh: + enabled: false + idle_mem_used.undrain.sh: enabled: false asserts: - isNull: - path: data["idle_mem_used.sh"] + path: data["idle_mem_used.drain.sh"] + - isNull: + path: data["idle_mem_used.undrain.sh"] - notMatchRegex: path: data["checks.json"] - pattern: '"name": "idle_mem_used"' + pattern: '"command": "./idle_mem_used.(drain|undrain).sh"' diff --git a/helm/slurm-cluster/values.yaml b/helm/slurm-cluster/values.yaml index 8dd7fe98a..a0ff18e4c 100644 --- a/helm/slurm-cluster/values.yaml +++ b/helm/slurm-cluster/values.yaml @@ -774,7 +774,11 @@ slurmScripts: enabled: true customContent: null customConfig: null - idle_mem_used.sh: + idle_mem_used.drain.sh: + enabled: true + customContent: null + customConfig: null + idle_mem_used.undrain.sh: enabled: true customContent: null customConfig: null