SCHED-1308 Add health check acceptance tests + Get rid of cleanup_enroot and ssh checks

helm/slurm-cluster/slurm_scripts/boot_disk_full.sh

@@ -20,7 +20,7 @@ if [ "$usage" -gt "$THRESHOLD" ]; then
     # Return failure details
     echo "$usage% of node boot disk is used. \
 Clean up volumes from 'ssh $SLURMD_NODENAME /opt/soperator_utils/fs_usage.sh -l', \
-delete leftover containers from 'ssh $SLURMD_NODENAME enroot list' and 'ssh $SLURMD_NODENAME docker ps -a', \
+delete leftover Docker containers from 'ssh $SLURMD_NODENAME docker ps -a', \
 reboot the node using 'scontrol reboot $SLURMD_NODENAME', \
 or stop-start the InstanceId from 'scontrol show node $SLURMD_NODENAME'" >&3
     exit 1

helm/slurm-cluster/values.yaml

@@ -754,10 +754,6 @@ slurmScripts:
       enabled: true
       customContent: null
       customConfig: null
-    cleanup_enroot.sh:
-      enabled: true
-      customContent: null
-      customConfig: null
     cleanup_scratch_data.sh:
       enabled: true
       customContent: null

helm/soperator-activechecks/values.yaml

@@ -98,20 +98,6 @@ checks:
     runAfterCreation: true
     slurmJobSpec:
       sbatchScriptFile: "scripts/ensure-healthy-nodes.sh"
-  ssh-check:
-    enabled: true
-    checkType: "k8sJob"
-    dependsOn:
-      - "create-user-soperatorchecks"
-    suspend: true
-    runAfterCreation: true
-    k8sJobSpec:
-      scriptFile: "scripts/ssh-check.sh"
-      jobContainer:
-        appArmorProfile: unconfined
-        env:
-          - name: "NUM_OF_LOGIN_NODES"
-            value: "2"
   create-user-soperatorchecks:
     enabled: true
     checkType: "k8sJob"
@@ -132,18 +118,6 @@ checks:
         env:
           - name: "USER_NAME"
             value: "nebius"
-  enroot-cleanup:
-    enabled: true
-    checkType: "slurmJob"
-    dependsOn:
-      - "manage-jail-state"
-      - "wait-for-soperatorchecks-srun-ready"
-    schedule: "10 8-20/12 * * *"
-    suspend: false
-    runAfterCreation: false
-    slurmJobSpec:
-      sbatchScriptFile: "scripts/enroot-cleanup.sh"
-      eachWorkerJobs: true
   soperator-outputs-logs-cleaner:
     enabled: true
     checkType: "k8sJob"

internal/e2e/acceptance/features/active_checks.feature

@@ -0,0 +1,11 @@
+Feature: Active checks
+  Scenario: Logs cleaner ActiveCheck removes old output files
+    Given an old soperator output file is created for acceptance
+    When the logs cleaner ActiveCheck is triggered
+    Then the old soperator output file is removed
+
+  @gpu
+  Scenario: GPU ActiveCheck succeeds on all GPU workers
+    Given GPU workers are available for active checks
+    When the GPU ActiveCheck is triggered
+    Then the GPU ActiveCheck finishes successfully on all GPU workers

internal/e2e/acceptance/features/cluster_creation.feature

@@ -6,7 +6,9 @@ Feature: Cluster creation
     And all NodeSet CRs are ready
     And main and hidden partitions are present and sane
     And all Slurm nodes are healthy
+    And HealthCheckProgram outputs are healthy
     And all ActiveChecks completed successfully
+    And nebius and soperatorchecks users are present
     And login welcome output shows cluster information
     And main partition smoke job succeeds
     And hidden partition smoke job succeeds

internal/e2e/acceptance/features/passive_checks.feature

@@ -0,0 +1,41 @@
+Feature: Passive checks
+  Scenario: CPU jobs run expected Prolog and Epilog passive checks
+    Given a worker is selected for passive checks
+    When a CPU-only Slurm job runs on the selected worker
+    Then the CPU job Prolog and Epilog check runner outputs are fresh and healthy
+    And GPU-only passive checks are not executed for the CPU job
+
+  Scenario: drop_page_cache runs after CPU jobs
+    Given a worker is selected for passive checks
+    When a CPU-only Slurm job runs on the selected worker
+    Then the drop_page_cache passive check completed in Epilog
+
+  Scenario: Passive Prolog drains a worker when allocated memory exceeds available memory
+    Given a worker is selected for passive checks
+    When memory pressure is created on the selected worker
+    And an all-memory Slurm job is submitted to the selected worker
+    Then the selected worker is drained by alloc_mem_used
+    When the memory pressure is removed
+    Then the selected worker recovers from alloc_mem_used
+
+  @gpu
+  Scenario: GPU jobs run passive GPU health checks
+    Given a GPU worker is selected for passive checks
+    When a small GPU Slurm job runs on the selected GPU worker
+    Then the GPU job health-check Prolog and Epilog reports are fresh and passing
+    And raw GPU health-check command outputs are present
+
+  Scenario: Job tmpfs directory is scoped to the Slurm job lifetime
+    Given a worker is selected for passive checks
+    When a Slurm job checks its job tmpfs directory on the selected worker
+    Then the job tmpfs directory existed during the job
+    And the job tmpfs directory is removed after the job exits
+
+  @gpu
+  Scenario: Passive Prolog drains a worker with unmanaged GPU processes
+    Given a GPU worker is selected for passive checks
+    When an unmanaged GPU workload is started on the selected GPU worker
+    And a full-node GPU Slurm job is submitted to the selected GPU worker
+    Then the selected GPU worker is drained by alloc_gpus_busy
+    When the unmanaged GPU workload is stopped
+    Then the selected GPU worker recovers from alloc_gpus_busy

internal/e2e/acceptance/features/system_checks.feature

@@ -0,0 +1,17 @@
+Feature: System checks
+  Scenario: Worker pod ephemeral storage pressure drains and recovers a Slurm node
+    Given a healthy worker pod is selected for ephemeral storage validation
+    When pod-local ephemeral storage is filled above the warning threshold
+    Then the selected worker is drained by pod_ephemeral_storage
+    When the pod-local ephemeral storage fill file is removed
+    Then the selected worker recovers from pod_ephemeral_storage
+
+  # TODO: Make soperatorchecks --not-ready-timeout configurable through Helm and
+  # set a lower value for e2e clusters, then remove @unstable from this scenario.
+  @unstable
+  Scenario: Non-responding kubelet recreates a Kubernetes node and recovers the worker
+    Given a healthy worker pod is selected for kubelet validation
+    When kubelet is stopped on the selected worker Kubernetes node
+    Then the selected worker Kubernetes node is recreated
+    And the selected worker pod is recreated and ready
+    And the selected Slurm worker recovers after kubelet replacement

internal/e2e/acceptance/framework/exec.go

@@ -23,15 +23,16 @@ type ArgsScope interface {
 }
 
 type Exec interface {
-	AvailableWorkers() []WorkerPodRef
-	AvailableGPUWorkers() []WorkerPodRef
+	AvailableWorkers() []WorkerRef
+	AvailableGPUWorkers() []WorkerRef
 	Kubectl() ArgsScope
 	// Local returns a local process scope. Do not use it for kubectl commands;
 	// use Kubectl instead so the explicit Kubernetes context is applied.
 	Local() ArgsScope
 	Controller() CommandScope
 	Jail() CommandScope
-	Worker(worker string) CommandScope
+	Worker(worker WorkerRef) CommandScope
+	WorkerPod(worker WorkerRef) CommandScope
 	WaitFor(ctx context.Context, description string, timeout, pollInterval time.Duration, condition func(context.Context) (bool, error)) error
 	Logf(format string, args ...any)
 }

internal/e2e/acceptance/framework/helpers.go

@@ -53,14 +53,15 @@ func WaitForWithJobAlive(
 	}
 
 	return exec.WaitFor(ctx, description, timeout, pollInterval, func(waitCtx context.Context) (bool, error) {
-		// Only the state is inlined here; the sacct dump + log tails come from
-		// AnnotateWithJobLog at the outer call site, to keep the message single‑sourced.
-		state, _, stateErr := slurm.JobState(waitCtx, job.ID)
+		// Only the queue state is inlined here; the sacct dump + log tails come
+		// from AnnotateWithJobLog at the outer call site, to keep the message
+		// single-sourced.
+		info, stateErr := slurm.JobInfo(waitCtx, job.ID)
 		if stateErr != nil {
 			return false, fmt.Errorf("check job %s state: %w", job.ID, stateErr)
 		}
-		if !IsJobAliveState(state) {
-			return false, fmt.Errorf("job %s is not alive (state=%q)", job.ID, state)
+		if !info.IsAlive() {
+			return false, fmt.Errorf("job %s is not alive (state=%q)", job.ID, info.QueueState)
 		}
 		return probe(waitCtx)
 	})
@@ -79,8 +80,8 @@ func AnnotateWithJobLog(ctx context.Context, exec Exec, slurm *SlurmClient, job
 	}
 
 	var extras []string
-	if _, dump, stateErr := slurm.JobState(ctx, job.ID); stateErr == nil && dump != "" {
-		extras = append(extras, fmt.Sprintf("sacct: %s", singleLine(dump)))
+	if info, stateErr := slurm.JobInfo(ctx, job.ID); stateErr == nil && info.SacctDump != "" {
+		extras = append(extras, fmt.Sprintf("sacct: %s", singleLine(info.SacctDump)))
 	}
 	for _, entry := range []struct{ label, path string }{
 		{"stdout", job.StdoutPath},

internal/e2e/acceptance/framework/model.go

@@ -1,7 +1,8 @@
 package framework
 
-type WorkerPodRef struct {
-	Name string
+type WorkerRef struct {
+	Name    string
+	PodName string
 }
 
 type DiscoveredNodeSet struct {
@@ -13,9 +14,9 @@ type DiscoveredNodeSet struct {
 
 type ClusterState struct {
 	SlurmClusterName   string
-	Workers            []WorkerPodRef
-	GPUWorkers         []WorkerPodRef
-	WorkersByNodeSet   map[string][]WorkerPodRef
+	Workers            []WorkerRef
+	GPUWorkers         []WorkerRef
+	WorkersByNodeSet   map[string][]WorkerRef
 	DiscoveredNodeSets []DiscoveredNodeSet
 }
 
@@ -34,3 +35,11 @@ func (s *ClusterState) DesiredWorkerCount() int {
 func (s *ClusterState) HasGPUWorkers() bool {
 	return len(s.GPUWorkers) > 0
 }
+
+func WorkerNames(workers []WorkerRef) []string {
+	names := make([]string, 0, len(workers))
+	for _, worker := range workers {
+		names = append(names, worker.Name)
+	}
+	return names
+}

internal/e2e/acceptance/framework/shell.go

@@ -12,3 +12,7 @@ func ShellQuote(value string) string {
 	result += "'"
 	return result
 }
+
+func BashLC(script string) string {
+	return "bash -lc " + ShellQuote(script)
+}

internal/e2e/acceptance/framework/shell_test.go

@@ -0,0 +1,11 @@
+package framework
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestBashLC(t *testing.T) {
+	assert.Equal(t, `bash -lc 'echo '"'"'ready'"'"''`, BashLC(`echo 'ready'`))
+}