vsc_calcua_cpu.config

// CalcUA (VSC Antwerp) — CPU-focused Slurm + Singularity/Apptainer profile.
// Use `-profile vsc_calcua_cpu` or `-profile vsc_calcua` (alias for this file).
// For workflows that submit GPU jobs (e.g. Dorado), see `vsc_calcua_gpu.config`.
//
// Define the scratch directory, which will be used for storing the nextflow
// work directory and for caching apptainer/singularity files.
// Default to /tmp directory if $VSC_SCRATCH scratch env is not available,
// see: https://github.com/nf-core/configs?tab=readme-ov-file#adding-a-new-config
def scratch_dir = System.getenv("VSC_SCRATCH") ?: "/tmp"

// Specify the work directory. Can be overwritten via the cli flag `-work-dir`.
workDir = "$scratch_dir/work"

// Perform work directory cleanup when the run has succesfully completed.
cleanup = true

// Check if environment variables for singularity/apptainer/nextflow cache and tmp dirs are set:
// - APPTAINER_TMPDIR/SINGULARITY_TMPDIR (warn if missing, apptainer defaults to $TMPDIR or /tmp)
// - APPTAINER_CACHEDIR/SINGULARITY_CACHEDIR (exit with error if missing, apptainer would default to $HOME otherwise)
// - NXF_APPTAINER_CACHEDIR/NXF_SINGULARITY_CACHEDIR (warn and set to $scratch_dir/apptainer/nextflow_cache if missing)
// Note that only the third env var can be set inside of this config file (cacheDir), because
// the env scope only provides env vars to tasks, not to the launch environment.
// See https://www.nextflow.io/docs/latest/config.html#scope-env

// Define variables outside of conditional scope to make them usable elsewhere
def apptainer_tmpdir = System.getenv("APPTAINER_TMPDIR") ?: System.getenv("SINGULARITY_TMPDIR") ?: null
def apptainer_cachedir = System.getenv("APPTAINER_CACHEDIR") ?: System.getenv("SINGULARITY_CACHEDIR") ?: null
def nxf_apptainer_cachedir = System.getenv("NXF_APPTAINER_CACHEDIR") ?: System.getenv("NXF_SINGULARITY_CACHEDIR") ?: null

// Skip check if host is not CalcUA, to avoid hindering github actions.
if ( System.getenv("VSC_INSTITUTE") == "antwerpen" ) {
    // APPTAINER_TMPDIR/SINGULARITY_TMPDIR environment variable
    if ( !apptainer_tmpdir ) {
        // Apptainer defaults to $TMPDIR or /tmp (on the Slurm execution node) if this env var is not set.
        // See https://apptainer.org/docs/user/main/build_env.html#temporary-folders
        def tmp_dir = System.getenv("TMPDIR") ?: "/tmp"
        System.err.println("\nWARNING: APPTAINER_TMPDIR/SINGULARITY_TMPDIR environment variable was not found.\nPlease add the line 'export APPTAINER_TMPDIR=\"\${VSC_SCRATCH}/apptainer/tmp\"' to your ~/.bashrc file (or set it with sbatch or in your job script).\nDefaulting to local $tmp_dir on the execution node of the Nextflow head process.\n")
    } else {
        // If set, try to create the tmp directory at the specified location to avoid errors during
        // docker image conversion (note that this only happens when no native singulariry/apptainer
        // images are available):
        //      FATAL:   While making image from oci registry: error fetching image to cache: while
        //      building SIF from layers: unable to create new build: failed to create build parent dir:
        //      stat /scratch/antwerpen/203/vsc20380/apptainer/tmp: no such file or directory
        apptainer_tmpdir = new File(apptainer_tmpdir)
        if (! apptainer_tmpdir.exists() ) {
            try {
                dir_created = apptainer_tmpdir.mkdirs()
            } catch (java.io.IOException e) {
                System.err.println("\nERROR: Could not create directory at the location specified by APPTAINER_TMPDIR/SINGULARITY_TMPDIR: $apptainer_tmpdir\nPlease check if this is a valid path to which you have write permission. Exiting...\n")
            }
        }
    }
    // APPTAINER_CACHEDIR/SINGULARITY_CACHEDIR
    if ( !apptainer_cachedir ) {
        System.err.println("\nERROR: APPTAINER_CACHEDIR/SINGULARITY_CACHEDIR environment variable was not found.\nPlease add the line 'export APPTAINER_CACHEDIR=\"\${VSC_SCRATCH}/apptainer/cache\"' to your ~/.bashrc file (or set it with sbatch or in your job script).\nUsing the default storage location of Singularity/Apptainer ~/.apptainer/cache/. Read more about why this should be avoided in the VSC docs: https://docs.vscentrum.be/software/singularity.html#building-on-vsc-infrastructure\n")
        System.exit(1)
    }
    // NXF_APPTAINER_CACHEDIR/NXF_SINGULARITY_CACHEDIR
    if ( !nxf_apptainer_cachedir ) {
        nxf_apptainer_cachedir = "$scratch_dir/apptainer/nextflow_cache"
        System.err.println("\nWARNING: NXF_APPTAINER_CACHEDIR/NXF_SINGULARITY_CACHEDIR environment variable was not found.\nPlease add the line 'export NXF_APPTAINER_CACHEDIR=\"\${VSC_SCRATCH}/apptainer/nextflow_cache\"' to your ~/.bashrc file (or set it with sbatch or in your job script) to choose the location of the Nextflow container image cache.\nDefaulting to $nxf_apptainer_cachedir (instead of the Nextflow work directory).\n")
    }
}

// Reduce the job submit rate to about 30 per minute, this way the server
// won't be bombarded with jobs.
// Limit queueSize to keep job rate under control and avoid timeouts.
// Set read timeout to the maximum wall time.
// See: https://www.nextflow.io/docs/latest/config.html#scope-executor
executor {
    submitRateLimit = "30/1min"
    queueSize = 20
    exitReadTimeout = "10 min"
}

// Add backoff strategy to catch cluster timeouts and proper symlinks of files in scratch
// to the work directory.
// See: https://www.nextflow.io/docs/latest/config.html#scope-process
process {
    stageInMode = "symlink"
    stageOutMode = "rsync"
    errorStrategy = { sleep(Math.pow(2, task.attempt ?: 1) * 200 as long); return "retry" }
    maxRetries = 3

    // CalcUA does not provide a default "library://" remote, so force Apptainer/Singularity
    // pulls to use OCI/Docker sources when modules provide both options.
    ext.singularity_pull_docker_container = true

    /*
     * CalcUA label tuning
     *
     * `process_low` / `process_medium` use more CPUs than conf/base.config (4 / 8 vs 2 / 6).
     * `process_high` stays elevated for Zen partitions (28 vs base 12). Other labels match base.
     * Values are still capped by `params.max_cpus` and `params.max_memory` via `check_max()`.
     */
    withLabel:process_single {
        cpus   = { check_max( 1                  , 'cpus'    ) }
        memory = { check_max( 6.GB * task.attempt, 'memory'  ) }
        time   = { check_max( 4.h  * task.attempt, 'time'    ) }
    }
    withLabel:process_low {
        cpus   = { check_max( 4     * task.attempt, 'cpus'    ) }
        memory = { check_max( 12.GB * task.attempt, 'memory'  ) }
        time   = { check_max( 4.h   * task.attempt, 'time'    ) }
    }
    withLabel:process_medium {
        cpus   = { check_max( 8     * task.attempt, 'cpus'    ) }
        memory = { check_max( 36.GB * task.attempt, 'memory'  ) }
        time   = { check_max( 8.h   * task.attempt, 'time'    ) }
    }
    withLabel:process_high {
        cpus   = { check_max( 28    * task.attempt, 'cpus'    ) }
        memory = { check_max( 48.GB * task.attempt, 'memory'  ) }
        time   = { check_max( 16.h  * task.attempt, 'time'    ) }
    }
    withLabel:process_long {
        time   = { check_max( 20.h  * task.attempt, 'time'    ) }
    }
    withLabel:process_high_memory {
        memory = { check_max( 200.GB * task.attempt, 'memory' ) }
    }
}

// Specify that apptainer/singularity should be used and where the cache dir will be for the images.
// Singularity is used in favour of apptainer, because currently the apptainer
// variant will pull in (and convert) docker images, instead of using pre-built singularity ones.
// On a system where singularity is defined as an alias for apptainer (as is the case on CalcUA),
// this works out fine and results in pre-built singularity containers being downloaded.
// See https://nf-co.re/docs/usage/installation#pipeline-software
// and https://nf-co.re/tools#how-the-singularity-image-downloads-work
// See https://www.nextflow.io/docs/latest/config.html#scope-singularity
singularity {
    enabled = true
    autoMounts = true
    // Docker→SIF conversion on HPC can exceed the default 20m (pull then gets SIGTERM / exit 143).
    // See https://www.nextflow.io/docs/latest/config.html#scope-singularity
    pullTimeout = '120m'
    // See https://www.nextflow.io/docs/latest/singularity.html#singularity-docker-hub
    cacheDir = "$nxf_apptainer_cachedir" // Equivalent to setting NXF_APPTAINER_CACHEDIR/NXF_SINGULARITY_CACHEDIR environment variable
}

// Shared profile settings
params {
    config_profile_name        = "vsc_calcua_cpu"
    config_profile_description = "CalcUA (VSC Antwerp) Slurm profile (CPU tasks)."
    config_profile_contact     = "djansen@itg.be"
    config_profile_url         = "https://docs.vscentrum.be/antwerp/"
}

// Retrieve name of current partition via Slurm environment variable
def partition = System.getenv("SLURM_JOB_PARTITION") ?: null
// Skip check if host is not CalcUA, to avoid hindering github actions.
if ( System.getenv("VSC_INSTITUTE") == "antwerpen" ) {
    if(! partition ) {
        System.err.println("WARNING: Could not retrieve name of current Slurm partition/queue, defaulting to broadwell")
        partition = "broadwell"
    }
}

// Use slurm executor as default, but enable switching to local for single node profile
def slurm_scheduling = true
profiles {
    single_node {
        slurm_scheduling = false
    }
}

// Dynamic partition/queue selection; adapted from https://nf-co.re/configs/vsc_ugent
// Define profiles for the following partitions:
// - zen2, zen3, zen3_512 (Vaughan)
// - broadwell, broadwell_256 (Leibniz)
// - skylake (Breniac, formerly Hopper)
switch(partition) {
    case "zen2":
        params {
            max_memory = slurm_scheduling ? 240.GB : get_allocated_mem(240) // 256 GB (total) - 16 GB (buffer)
            max_cpus = slurm_scheduling ? 64 : get_allocated_cpus(64)
            max_time = 3.day
        }
        process {
            executor = slurm_scheduling ? "slurm" : "local"
            queue = "zen2"
        }
        break
    case "zen3":
        params {
            max_memory = slurm_scheduling ? 240.GB : get_allocated_mem(240) // 256 GB (total) - 16 GB (buffer)
            max_cpus = slurm_scheduling ? 64 : get_allocated_cpus(64)
            max_time = 3.day
        }
        process {
            executor = slurm_scheduling ? "slurm" : "local"
            queue = "zen3"
        }
        break
    case "zen3_512":
        params {
            max_memory = slurm_scheduling ? 496.GB : get_allocated_mem(496) // 512 GB (total) - 16 GB (buffer)
            max_cpus = slurm_scheduling ? 64 : get_allocated_cpus(64)
            max_time = 3.day
        }
        process {
            executor = slurm_scheduling ? "slurm" : "local"
            queue = "zen3_512"
        }
        break
    case "broadwell":
        params {
            max_memory = slurm_scheduling ? 112.GB : get_allocated_mem(112) // 128 GB (total) - 16 GB (buffer)
            max_cpus = slurm_scheduling ? 28 : get_allocated_cpus(28)
            max_time = 3.day
        }
        process {
            executor = slurm_scheduling ? "slurm" : "local"
            queue = "broadwell"
        }
        break
    case "broadwell_256":
        params {
            max_memory = slurm_scheduling ? 240.GB : get_allocated_mem(240) // 256 GB (total) - 16 GB (buffer)
            max_cpus = slurm_scheduling ? 28 : get_allocated_cpus(28)
            max_time = 3.day
        }
        process {
            executor = slurm_scheduling ? "slurm" : "local"
            queue = "broadwell_256"
        }
        break
    case "skylake":
        params {
            max_memory = slurm_scheduling ? 176.GB : get_allocated_mem(176) // 192 GB (total) - 16 GB (buffer)
            max_cpus = slurm_scheduling ? 28 : get_allocated_cpus(28)
            max_time = 7.day
        }
        process {
            executor = slurm_scheduling ? "slurm" : "local"
            queue = "skylake"
        }
        break
}

// Define functions to fetch the available CPUs and memory of the current execution node.
// Only used when the single_node / local execution profile is activated.
// Allows cpu and memory thresholds to be set dynamic based on the available hardware as reported
// by Slurm. Can be supplied with a default return value, which should be set to the
// recommended thresholds for that particular partition's node types.
def get_allocated_cpus(int node_max_cpu) {
    max_cpus = System.getenv("SLURM_CPUS_PER_TASK") ?: System.getenv("SLURM_JOB_CPUS_PER_NODE") ?: node_max_cpu
    return max_cpus.toInteger()
}

def get_allocated_mem(int node_max_mem) {
    // default to max memory of node per partition type
    int max_mem = node_max_mem

    // grab environment variables with memory and cpu info
    def mem_per_cpu = System.getenv("SLURM_MEM_PER_CPU")
    def mem_per_node = System.getenv("SLURM_MEM_PER_NODE")
    def cpus_per_task = System.getenv("SLURM_CPUS_PER_TASK") ?: System.getenv("SLURM_JOB_CPUS_PER_NODE")

    // Check if memory was requested per cpu and the number of cpus was also set
    if ( mem_per_cpu && cpus_per_task ) {
        max_mem = mem_per_cpu.toInteger() / 1000 * cpus_per_task.toInteger()
    }

    // Otherwise check if memory was requested per node (Slurm calls this per "CPU set" in docs)
    if ( mem_per_node ) {
        max_mem = mem_per_node.toInteger() / 1000
    }

    return max_mem.GB
}