nf-core/configs: EVA Configuration

All nf-core pipelines have been successfully configured for use on the Department of Genetics and Archaeogenetic’s clusters at the Max Planck Institute for Evolutionary Anthropology (MPI-EVA).

To use, run the pipeline with -profile eva. You can further with optimise submissions by specifying which cluster queue you are using e,g, -profile eva,archgen. This will download and launch the eva.config which has been pre-configured with a setup suitable for the all.q queue. The number of parallel jobs that run is currently limited to 8.

Using this profile, a docker image containing all of the required software will be downloaded, and converted to a singularity image before execution of the pipeline.

Institute-specific pipeline profiles exists for:

  • eager
  • mag
  • sarek
  • taxprofiler

Additional Profiles

We currently also offer profiles for the different department’s specific nodes.

archgen

If you specify -profile eva,archgen you will be able to use the nodes available on the archgen.q queue.

Note the following characteristics of this profile:

  • By default, job resources are assigned a maximum number of CPUs of 32, 256 GB maximum memory and 365 day maximum wall time.
  • Using this profile will currently store singularity images in a cache under /mnt/archgen/users/singularity_scratch/cache/. All archgen users currently have read/write access to this directory, however this will likely change to a read-only directory in the future that will be managed by the IT team.
  • Intermediate files will be automatically cleaned up (see debug below if you don’t want this to happen) on successful run completion.
  • Jobs submitted with >700.GB will automatically be submitted to the dynamic bigmem.q.

NB: You will need an account and VPN access to use the cluster at MPI-EVA in order to run the pipeline. If in doubt contact the IT team. NB: Nextflow will need to submit the jobs via SGE to the clusters and as such the commands above will have to be executed on one of the head nodes. If in doubt contact IT.

debug

This simple profile just turns off automatic clean up of intermediate files. This can be useful for debugging. Specify e.g. with -profile eva,archgen.

Config file

See config file on GitHub

conf/eva
//Profile config names for nf-core/configs
params {
    config_profile_description = 'Generic MPI-EVA cluster(s) profile provided by nf-core/configs.'
    config_profile_contact     = 'James Fellows Yates (@jfy133)'
    config_profile_url         = 'https://eva.mpg.de'
}
 
// Preform work directory cleanup after a successful run
cleanup = true
 
singularity {
    enabled    = true
    autoMounts = true
}
 
process {
    executor       = 'sge'
    penv           = 'smp'
    queue          = 'all.q'
    clusterOptions = { "-S /bin/bash -V -j y -o output.sge -l h_vmem=${task.memory.toGiga()}G" }
}
 
profiles {
    archgen {
        params {
            igenomes_base              = "/mnt/archgen/public_data/igenomes"
            config_profile_description = 'MPI-EVA archgen profile, provided by nf-core/configs.'
            max_memory                 = 370.GB
            max_cpus                   = 32
            max_time                   = 365.d
        }
 
        process {
            resourceLimits = [
                memory: 370.GB,
                cpus: 32,
                time: 365.d
            ]
            queue          = { task.memory > 700.GB ? 'bigmem.q' : 'archgen.q' }
            clusterOptions = { "-S /bin/bash -V -j y -o output.sge -l h_vmem=${task.memory.toGiga()}G" }
        }
 
        singularity {
            cacheDir = "/mnt/archgen/tools/singularity/containers/"
        }
    }
    // Profile to deactivate automatic cleanup of work directory after a successful run. Overwrites cleanup option.
    debug {
        cleanup = false
    }
}

Pipeline configs

See config file on GitHub

conf/pipeline/eager/eva
// Profile config names for nf-core/configs
 
params {
    // Specific nf-core/configs params
    config_profile_contact     = 'James Fellows Yates (@jfy133)'
    config_profile_description = 'nf-core/eager EVA profile provided by nf-core/configs'
}
 
env {
    _JAVA_OPTIONS        = "-XX:ParallelGCThreads=1"
    OPENBLAS_NUM_THREADS = 1
    OMP_NUM_THREADS      = 1
}
 
// Specific nf-core/eager process configuration
process {
 
    maxRetries     = 2
 
    // Solution for clusterOptions comes from here: https://github.com/nextflow-io/nextflow/issues/332 + personal toMega conversion
    clusterOptions = { "-S /bin/bash -V -j y -o output.log -l h_vmem=${task.memory.toGiga()}G" }
 
    withLabel: sc_tiny {
        cpus   = { check_max(1, 'cpus') }
        memory = { check_max(1.GB * task.attempt, 'memory') }
        time   = '365.d'
    }
 
    withLabel: sc_small {
        cpus   = { check_max(1, 'cpus') }
        memory = { check_max(4.GB * task.attempt, 'memory') }
        time   = '365.d'
    }
 
    withLabel: sc_medium {
        cpus   = { check_max(1, 'cpus') }
        memory = { check_max(8.GB * task.attempt, 'memory') }
        time   = '365.d'
    }
 
    withLabel: mc_small {
        cpus   = { check_max(2, 'cpus') }
        memory = { check_max(4.GB * task.attempt, 'memory') }
        time   = '365.d'
    }
 
    withLabel: mc_medium {
        cpus   = { check_max(4, 'cpus') }
        memory = { check_max(8.GB * task.attempt, 'memory') }
        time   = '365.d'
    }
 
    withLabel: mc_large {
        cpus   = { check_max(8, 'cpus') }
        memory = { check_max(16.GB * task.attempt, 'memory') }
        time   = '365.d'
    }
 
    withLabel: mc_huge {
        cpus   = { check_max(32, 'cpus') }
        memory = { check_max(256.GB * task.attempt, 'memory') }
        time   = '365.d'
    }
 
    // Fixes for SGE and Java incompatibility due to Java using more memory than you tell it to use
 
    withName: makeSeqDict {
        clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 2)}G" }
        errorStrategy  = { task.exitStatus in [1, 143, 137, 104, 134, 139, 140] ? 'retry' : 'finish' }
    }
 
    withName: fastqc {
        clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 2)}G" }
        errorStrategy  = { task.exitStatus in [1, 143, 137, 104, 134, 139, 140] ? 'retry' : 'finish' }
    }
 
    withName: fastqc_after_clipping {
        clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 2)}G" }
        errorStrategy  = { task.exitStatus in [1, 143, 137, 104, 134, 139, 140] ? 'retry' : 'finish' }
    }
 
    withName: adapter_removal {
        clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 2)}G" }
        errorStrategy  = { task.exitStatus in [1, 143, 137, 104, 134, 139, 140] ? 'retry' : 'finish' }
    }
 
    withName: bwa {
        clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga())}G,h=!(bionode01|bionode02|bionode03|bionode04|bionode05|bionode06)" }
    }
 
    withName: bwamem {
        clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga())}G,h=!(bionode01|bionode02|bionode03|bionode04|bionode05|bionode06)" }
    }
 
    withName: circularmapper {
        clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga())}G,h=!(bionode01|bionode02|bionode03|bionode04|bionode05|bionode06)" }
    }
 
    withName: bowtie2 {
        clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga())}G,h=!(bionode01|bionode02|bionode03|bionode04|bionode05|bionode06)" }
    }
 
    withName: samtools_flagstat {
        clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 2)}G" }
    }
 
    withName: samtools_flagstat_after_filter {
        clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 2)}G" }
    }
 
    withName: dedup {
        clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 2)}G" }
        errorStrategy  = { task.exitStatus in [1, 143, 137, 104, 134, 139, 140] ? 'retry' : 'finish' }
    }
 
    withName: markduplicates {
        clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 2)}G" }
        memory         = { check_max(20.GB * task.attempt, 'memory') }
        errorStrategy  = { task.exitStatus in [1, 143, 137, 104, 134, 139, 140] ? 'retry' : 'finish' }
    }
 
    withName: library_merge {
        clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 2)}G" }
    }
 
    withName: seqtype_merge {
        clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 2)}G" }
    }
 
    withName: additional_library_merge {
        clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 2)}G" }
        memory         = { check_max(4.GB * task.attempt, 'memory') }
    }
 
    withName: metagenomic_complexity_filter {
        clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 2)}G" }
    }
 
    withName: malt {
        clusterOptions = { "-S /bin/bash -V -l h_vmem=1000G" }
        cpus           = { check_max(32, 'cpus') }
        memory         = { check_max(955.GB * task.attempt, 'memory') }
    }
 
    withName: maltextract {
        clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 2)}G" }
        errorStrategy  = { task.exitStatus in [1, 143, 137, 104, 134, 139, 140] ? 'retry' : 'finish' }
    }
 
    withName: multivcfanalyzer {
        clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 2)}G" }
        errorStrategy  = { task.exitStatus in [1, 143, 137, 104, 134, 139, 140] ? 'retry' : 'finish' }
    }
 
    withName: mtnucratio {
        clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 2)}G" }
        errorStrategy  = { task.exitStatus in [1, 143, 137, 104, 134, 139, 140] ? 'retry' : 'finish' }
    }
 
    withName: vcf2genome {
        clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 2)}G" }
        errorStrategy  = { task.exitStatus in [1, 143, 137, 104, 134, 139, 140] ? 'retry' : 'finish' }
    }
 
    withName: qualimap {
        clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 2)}G" }
        errorStrategy  = { task.exitStatus in [1, 143, 137, 104, 134, 139, 140] ? 'retry' : task.exitStatus in [255] ? 'ignore' : 'finish' }
    }
 
    withName: damageprofiler {
        clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 2)}G" }
        memory         = { check_max(8.GB * task.attempt, 'memory') }
        errorStrategy  = { task.exitStatus in [1, 143, 137, 104, 134, 139, 140] ? 'retry' : 'finish' }
    }
 
    withName: circularmapper {
        clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 2)}G" }
        errorStrategy  = { task.exitStatus in [1, 143, 137, 104, 134, 139, 140] ? 'retry' : 'finish' }
    }
 
    withName: circulargenerator {
        clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 2)}G" }
        errorStrategy  = { task.exitStatus in [1, 143, 137, 104, 134, 139, 140] ? 'retry' : 'finish' }
    }
 
    withName: preseq {
        clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 2)}G" }
        errorStrategy  = { task.exitStatus in [143, 137, 104, 134, 139] ? 'retry' : 'ignore' }
    }
 
    withName: picard_addorreplacereadgroups {
        clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 2)}G" }
        errorStrategy  = { task.exitStatus in [1, 143, 137, 104, 134, 139, 140] ? 'retry' : 'finish' }
    }
 
    withName: genotyping_ug {
        clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 2)}G" }
        errorStrategy  = { task.exitStatus in [1, 143, 137, 104, 134, 139, 140] ? 'retry' : 'finish' }
    }
 
    withName: genotyping_hc {
        clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 2)}G" }
        errorStrategy  = { task.exitStatus in [1, 143, 137, 104, 134, 139, 140] ? 'retry' : 'finish' }
    }
 
    withName: get_software_versions {
        cache          = false
        clusterOptions = { "-S /bin/bash -V -l h=!(bionode06)" }
        clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toMega() * 8)}M" }
        errorStrategy  = { task.exitStatus in [1, 143, 137, 104, 134, 139, 140] ? 'retry' : 'finish' }
    }
 
    withName: multiqc {
        clusterOptions = { "-S /bin/bash -V -j y -o output.log -l h_vmem=${task.memory.toGiga() * 2}G" }
    }
}
 
profiles {
 
    medium_data {
 
        params {
            // Specific nf-core/configs params
            config_profile_contact     = 'James Fellows Yates (@jfy133)'
            config_profile_description = 'nf-core/eager medium-data EVA profile provided by nf-core/configs'
        }
 
        executor {
            queueSize = 8
        }
 
        process {
 
            maxRetries     = 2
 
            // Solution for clusterOptions comes from here: https://github.com/nextflow-io/nextflow/issues/332 + personal toMega conversion
            clusterOptions = { "-S /bin/bash -V -j y -o output.log -l h_vmem=${task.memory.toGiga()}G" }
 
            withLabel: sc_tiny {
                cpus   = { check_max(1, 'cpus') }
                memory = { check_max(1.5.GB * task.attempt, 'memory') }
            }
 
            withLabel: sc_small {
                cpus   = { check_max(1, 'cpus') }
                memory = { check_max(6.GB * task.attempt, 'memory') }
            }
 
            withLabel: sc_medium {
                cpus   = { check_max(1, 'cpus') }
                memory = { check_max(12.GB * task.attempt, 'memory') }
            }
 
            withLabel: mc_small {
                cpus   = { check_max(2, 'cpus') }
                memory = { check_max(6.GB * task.attempt, 'memory') }
            }
 
            withLabel: mc_medium {
                cpus   = { check_max(4, 'cpus') }
                memory = { check_max(12.GB * task.attempt, 'memory') }
            }
 
            withLabel: mc_large {
                cpus   = { check_max(8, 'cpus') }
                memory = { check_max(24.GB * task.attempt, 'memory') }
            }
 
            withLabel: mc_huge {
                cpus   = { check_max(32, 'cpus') }
                memory = { check_max(256.GB * task.attempt, 'memory') }
            }
 
            // Fixes for SGE and Java incompatibility due to  (and also some samtools?!) using more memory than you tell it to use
 
            withName: makeSeqDict {
                clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 3)}G" }
                errorStrategy  = { task.exitStatus in [1, 143, 137, 104, 134, 139, 140] ? 'retry' : 'finish' }
            }
 
            withName: fastqc {
                clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 3)}G" }
                errorStrategy  = { task.exitStatus in [1, 143, 137, 104, 134, 139, 140] ? 'retry' : 'finish' }
            }
 
            withName: fastqc_after_clipping {
                clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 3)}G" }
                errorStrategy  = { task.exitStatus in [1, 143, 137, 104, 134, 139, 140] ? 'retry' : 'finish' }
            }
 
            withName: adapter_removal {
                clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 3)}G" }
                errorStrategy  = { task.exitStatus in [1, 143, 137, 104, 134, 139, 140] ? 'retry' : 'finish' }
            }
 
            withName: samtools_flagstat {
                clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 3)}G" }
            }
 
            withName: samtools_flagstat_after_filter {
                clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 3)}G" }
            }
 
            withName: dedup {
                clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 3)}G" }
                errorStrategy  = { task.exitStatus in [1, 143, 137, 104, 134, 139, 140] ? 'retry' : 'finish' }
            }
 
            withName: markduplicates {
                clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 3)}G" }
                memory         = { check_max(32.GB * task.attempt, 'memory') }
                errorStrategy  = { task.exitStatus in [1, 143, 137, 104, 134, 139, 140] ? 'retry' : 'finish' }
            }
 
            withName: library_merge {
                clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 3)}G" }
            }
 
            withName: seqtype_merge {
                clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 3)}G" }
            }
 
            withName: additional_library_merge {
                clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 3)}G" }
                memory         = { check_max(4.GB * task.attempt, 'memory') }
            }
 
            withName: metagenomic_complexity_filter {
                clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 3)}G" }
            }
 
            withName: malt {
                clusterOptions = { "-S /bin/bash -V -l h_vmem=1000G" }
                cpus           = { check_max(32, 'cpus') }
                memory         = { check_max(955.GB * task.attempt, 'memory') }
                errorStrategy  = { task.exitStatus in [1, 143, 137, 104, 134, 139, 140] ? 'retry' : 'finish' }
            }
 
            withName: hostremoval_input_fastq {
                memory = { check_max(32.GB * task.attempt, 'memory') }
            }
 
            withName: maltextract {
                clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 3)}G" }
                errorStrategy  = { task.exitStatus in [1, 143, 137, 104, 134, 139, 140] ? 'retry' : 'finish' }
            }
 
            withName: multivcfanalyzer {
                clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 3)}G" }
                errorStrategy  = { task.exitStatus in [1, 143, 137, 104, 134, 139, 140] ? 'retry' : 'finish' }
            }
 
            withName: mtnucratio {
                clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 3)}G" }
                errorStrategy  = { task.exitStatus in [1, 143, 137, 104, 134, 139, 140] ? 'retry' : 'finish' }
            }
 
            withName: vcf2genome {
                clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 3)}G" }
                errorStrategy  = { task.exitStatus in [1, 143, 137, 104, 134, 139, 140] ? 'retry' : 'finish' }
            }
 
            withName: qualimap {
                clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 3)}G" }
                errorStrategy  = { task.exitStatus in [1, 143, 137, 104, 134, 139, 140] ? 'retry' : task.exitStatus in [255] ? 'ignore' : 'finish' }
            }
 
            withName: damageprofiler {
                clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 3)}G" }
                memory         = { check_max(16.GB * task.attempt, 'memory') }
                errorStrategy  = { task.exitStatus in [1, 143, 137, 104, 134, 139, 140] ? 'retry' : 'finish' }
            }
 
            withName: circularmapper {
                clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 3)}G" }
                errorStrategy  = { task.exitStatus in [1, 143, 137, 104, 134, 139, 140] ? 'retry' : 'finish' }
            }
 
            withName: circulargenerator {
                clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 3)}G" }
                errorStrategy  = { task.exitStatus in [1, 143, 137, 104, 134, 139, 140] ? 'retry' : 'finish' }
            }
 
            withName: picard_addorreplacereadgroups {
                clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 3)}G" }
                errorStrategy  = { task.exitStatus in [1, 143, 137, 104, 134, 139, 140] ? 'retry' : 'finish' }
            }
 
            withName: genotyping_ug {
                clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 3)}G" }
                errorStrategy  = { task.exitStatus in [1, 143, 137, 104, 134, 139, 140] ? 'retry' : 'finish' }
            }
 
            withName: preseq {
                clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 3)}G" }
                errorStrategy  = { task.exitStatus in [143, 137, 104, 134, 139] ? 'retry' : 'ignore' }
            }
        }
    }
 
    big_data {
 
        params {
            // Specific nf-core/configs params
            config_profile_contact     = 'James Fellows Yates (@jfy133)'
            config_profile_description = 'nf-core/eager big-data EVA profile provided by nf-core/configs'
        }
 
        executor {
            queueSize = 6
        }
 
        process {
 
            maxRetries     = 2
 
            // Solution for clusterOptions comes from here: https://github.com/nextflow-io/nextflow/issues/332 + personal toMega conversion
            clusterOptions = { "-S /bin/bash -V -j y -o output.log -l h_vmem=${task.memory.toGiga()}G" }
 
            withLabel: sc_tiny {
                cpus   = { check_max(1, 'cpus') }
                memory = { check_max(2.GB * task.attempt, 'memory') }
            }
 
            withLabel: sc_small {
                cpus   = { check_max(1, 'cpus') }
                memory = { check_max(8.GB * task.attempt, 'memory') }
            }
 
            withLabel: sc_medium {
                cpus   = { check_max(1, 'cpus') }
                memory = { check_max(16.GB * task.attempt, 'memory') }
            }
 
            withLabel: mc_small {
                cpus   = { check_max(2, 'cpus') }
                memory = { check_max(8.GB * task.attempt, 'memory') }
            }
 
            withLabel: mc_medium {
                cpus   = { check_max(4, 'cpus') }
                memory = { check_max(16.GB * task.attempt, 'memory') }
            }
 
            withLabel: mc_large {
                cpus   = { check_max(8, 'cpus') }
                memory = { check_max(32.GB * task.attempt, 'memory') }
            }
 
            withLabel: mc_huge {
                cpus   = { check_max(32, 'cpus') }
                memory = { check_max(512.GB * task.attempt, 'memory') }
            }
 
            // Fixes for SGE and Java incompatibility due to Java using more memory than you tell it to use
 
            withName: makeSeqDict {
                clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 6)}G" }
                errorStrategy  = { task.exitStatus in [1, 143, 137, 104, 134, 139, 140] ? 'retry' : 'finish' }
            }
 
            withName: fastqc {
                clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 6)}G" }
                errorStrategy  = { task.exitStatus in [1, 143, 137, 104, 134, 139, 140] ? 'retry' : 'finish' }
            }
 
            withName: fastqc_after_clipping {
                clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 6)}G" }
                errorStrategy  = { task.exitStatus in [1, 143, 137, 104, 134, 139, 140] ? 'retry' : 'finish' }
            }
 
            withName: adapter_removal {
                clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 6)}G" }
                errorStrategy  = { task.exitStatus in [1, 143, 137, 104, 134, 139, 140] ? 'retry' : 'finish' }
            }
 
            withName: samtools_flagstat {
                clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 6)}G" }
            }
 
            withName: samtools_flagstat_after_filter {
                clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 6)}G" }
            }
 
            withName: dedup {
                clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 6)}G" }
                errorStrategy  = { task.exitStatus in [1, 143, 137, 104, 134, 139, 140] ? 'retry' : 'finish' }
            }
 
            withName: markduplicates {
                clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 6)}G" }
                memory         = { check_max(48.GB * task.attempt, 'memory') }
                errorStrategy  = { task.exitStatus in [1, 143, 137, 104, 134, 139, 140] ? 'retry' : 'finish' }
            }
 
            withName: library_merge {
                clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 6)}G" }
                memory         = { check_max(6.GB * task.attempt, 'memory') }
            }
 
            withName: seqtype_merge {
                clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 6)}G" }
            }
 
            withName: additional_library_merge {
                clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 6)}G" }
                memory         = { check_max(6.GB * task.attempt, 'memory') }
            }
 
            withName: hostremoval_input_fastq {
                memory = { check_max(32.GB * task.attempt, 'memory') }
            }
 
            withName: metagenomic_complexity_filter {
                clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 6)}G" }
            }
 
            withName: malt {
                clusterOptions = { "-S /bin/bash -V -l h_vmem=1000G" }
                cpus           = { check_max(32, 'cpus') }
                memory         = { check_max(955.GB * task.attempt, 'memory') }
                errorStrategy  = { task.exitStatus in [1, 143, 137, 104, 134, 139, 140] ? 'retry' : 'finish' }
            }
 
            withName: maltextract {
                clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 6)}G" }
                errorStrategy  = { task.exitStatus in [1, 143, 137, 104, 134, 139, 140] ? 'retry' : 'finish' }
            }
 
            withName: multivcfanalyzer {
                clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 6)}G" }
                errorStrategy  = { task.exitStatus in [1, 143, 137, 104, 134, 139, 140] ? 'retry' : 'finish' }
            }
 
            withName: mtnucratio {
                clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 6)}G" }
                errorStrategy  = { task.exitStatus in [1, 143, 137, 104, 134, 139, 140] ? 'retry' : 'finish' }
            }
 
            withName: vcf2genome {
                clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 6)}G" }
                errorStrategy  = { task.exitStatus in [1, 143, 137, 104, 134, 139, 140] ? 'retry' : 'finish' }
            }
 
            withName: qualimap {
                clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 6)}G" }
                errorStrategy  = { task.exitStatus in [1, 143, 137, 104, 134, 139, 140] ? 'retry' : task.exitStatus in [255] ? 'ignore' : 'finish' }
            }
 
            withName: damageprofiler {
                clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 6)}G" }
                memory         = { check_max(32.GB * task.attempt, 'memory') }
                errorStrategy  = { task.exitStatus in [1, 143, 137, 104, 134, 139, 140] ? 'retry' : 'finish' }
            }
 
            withName: circularmapper {
                clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 6)}G" }
                errorStrategy  = { task.exitStatus in [1, 143, 137, 104, 134, 139, 140] ? 'retry' : 'finish' }
            }
 
            withName: circulargenerator {
                clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 6)}G" }
                errorStrategy  = { task.exitStatus in [1, 143, 137, 104, 134, 139, 140] ? 'retry' : 'finish' }
            }
 
            withName: picard_addorreplacereadgroups {
                clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 6)}G" }
                errorStrategy  = { task.exitStatus in [1, 143, 137, 104, 134, 139, 140] ? 'retry' : 'finish' }
            }
 
            withName: genotyping_ug {
                clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 6)}G" }
                errorStrategy  = { task.exitStatus in [1, 143, 137, 104, 134, 139, 140] ? 'retry' : 'finish' }
            }
 
            withName: preseq {
                clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 6)}G" }
                errorStrategy  = { task.exitStatus in [143, 137, 104, 134, 139] ? 'retry' : 'ignore' }
            }
        }
    }
 
    pathogen_loose {
        params {
            config_profile_description = 'Pathogen (loose) MPI-EVA profile, provided by nf-core/configs.'
            bwaalnn                    = 0.01
            bwaalnl                    = 16
        }
    }
    pathogen_strict {
        params {
            config_profile_description = 'Pathogen (strict) MPI-EVA SDAG profile, provided by nf-core/configs.'
            bwaalnn                    = 0.1
            bwaalnl                    = 32
        }
    }
    human {
        params {
            config_profile_description = 'Human MPI-EVA SDAG profile, provided by nf-core/configs.'
            bwaalnn                    = 0.01
            bwaalnl                    = 16500
        }
    }
    ena_prep {
        // Profile with minimal processing within nf-core/eager to crate the FastQ/BAM files for ENA upload. This should be used with FASTQ files from Pandora.
        params {
            // Skip any statistic generation
            skip_preseq             = true
            skip_damage_calculation = true
            skip_qualimap           = true
            skip_deduplication      = true
 
            // Adapter removal
            skip_collapse           = true
            // No read collapsing for paired-end data.
            clip_readlength         = 1
            // If 0, then empty reads are left in, which breaks downstream processing.
            clip_min_read_quality   = 0
            // No base quality filtering.
 
            // Mapping
            bwaalnn                 = 0.01
        }
    }
}
 
// Function to ensure that resource requirements don't go beyond
// a maximum limit
// FOR DSL1 PIPELINE ONLY!
def check_max(obj, type) {
    if (type == 'memory') {
        try {
            if (obj.compareTo(params.max_memory as nextflow.util.MemoryUnit) == 1)
                return params.max_memory as nextflow.util.MemoryUnit
            else
                return obj
        } catch (all) {
            println "   ### ERROR ###   Max memory '${params.max_memory}' is not valid! Using default value: $obj"
            return obj
        }
    } else if (type == 'time') {
        try {
            if (obj.compareTo(params.max_time as nextflow.util.Duration) == 1)
                return params.max_time as nextflow.util.Duration
            else
                return obj
        } catch (all) {
            println "   ### ERROR ###   Max time '${params.max_time}' is not valid! Using default value: $obj"
            return obj
        }
    } else if (type == 'cpus') {
        try {
            return Math.min( obj, params.max_cpus as int )
        } catch (all) {
            println "   ### ERROR ###   Max cpus '${params.max_cpus}' is not valid! Using default value: $obj"
            return obj
        }
    }
}

See config file on GitHub

conf/pipeline/mag/eva
params {
    // Specific nf-core/configs params
    config_profile_contact     = 'James Fellows Yates (@jfy133)'
    config_profile_description = 'nf-core/mag EVA profile provided by nf-core/configs'
}
 
env {
    OPENBLAS_NUM_THREADS = 1
    OMP_NUM_THREADS      = 1
}
 
process {
 
    errorStrategy = { task.exitStatus in ((130..145) + 104) ? 'retry' : 'finish' }
    time          = 365.d
 
    withName: FASTQC {
        clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 2)}G" }
        errorStrategy  = { task.exitStatus in [1, 143, 137, 104, 134, 139, 140] ? 'retry' : 'finish' }
    }
 
    withLabel: process_single {
        cpus   = 1
        memory = { 6.GB * task.attempt }
        time   = 365.d
    }
    withLabel: process_low {
        cpus   = { 2 * task.attempt }
        memory = { 12.GB * task.attempt }
        time   = 365.d
    }
    withLabel: process_medium {
        cpus   = { 6 * task.attempt }
        memory = { 36.GB * task.attempt }
        time   = 365.d
    }
    withLabel: process_high {
        cpus   = { 12 * task.attempt }
        memory = { 72.GB * task.attempt }
        time   = 365.d
    }
    withLabel: process_long {
        time = 365.d
    }
    withLabel: process_high_memory {
        memory = { 200.GB * task.attempt }
        time   = 365.d
    }
    withLabel: error_ignore {
        errorStrategy = 'ignore'
    }
    withLabel: error_retry {
        errorStrategy = 'retry'
        maxRetries    = 2
    }
 
    withName: CUSTOM_DUMPSOFTWAREVERSIONS {
        cache = false
    }
 
    withName: BOWTIE2_HOST_REMOVAL_BUILD {
        cpus   = { 10 * task.attempt }
        memory = { 20.GB * task.attempt }
        time   = 365.d
    }
    withName: BOWTIE2_HOST_REMOVAL_ALIGN {
        cpus   = { 10 * task.attempt }
        memory = { 10.GB * task.attempt }
        time   = 365.d
    }
    withName: BOWTIE2_PHIX_REMOVAL_ALIGN {
        cpus   = { 4 * task.attempt }
        memory = { 8.GB * task.attempt }
        time   = 365.d
    }
    withName: PORECHOP {
        cpus   = { 4 * task.attempt }
        memory = { 30.GB * task.attempt }
        time   = 365.d
    }
    withName: NANOLYSE {
        cpus   = { 2 * task.attempt }
        memory = { 10.GB * task.attempt }
        time   = 365.d
    }
    //filtlong: exponential increase of memory and time with attempts
    withName: FILTLONG {
        cpus   = { 8 * task.attempt }
        memory = { 64.GB * (2 ** (task.attempt - 1)) }
        time   = 365.d
    }
    withName: CENTRIFUGE {
        cpus   = { 8 * task.attempt }
        memory = { 40.GB * task.attempt }
        time   = 365.d
    }
    withName: KRAKEN2 {
        cpus   = { 8 * task.attempt }
        memory = { 40.GB * task.attempt }
        time   = 365.d
    }
    withName: KRONA {
        cpus   = { 8 * task.attempt }
        memory = { 20.GB * task.attempt }
        time   = 365.d
    }
    withName: CAT_DB_GENERATE {
        memory = { 200.GB * task.attempt }
        time   = 365.d
    }
    withName: CAT {
        cpus   = { 8 * task.attempt }
        memory = { 40.GB * task.attempt }
        time   = 365.d
    }
    withName: GTDBTK_CLASSIFYWF {
        cpus   = { 10 * task.attempt }
        memory = { 128.GB * task.attempt }
        time   = 365.d
    }
    //MEGAHIT returns exit code 250 when running out of memory
    withName: MEGAHIT {
        cpus          = { 8 * task.attempt }
        memory        = { 40.GB * task.attempt }
        time          = 365.d
        errorStrategy = { task.exitStatus in ((130..145) + 104 + 250) ? 'retry' : 'finish' }
    }
    //SPAdes returns error(1) if it runs out of memory (and for other reasons as well...)!
    //exponential increase of memory and time with attempts, keep number of threads to enable reproducibility
    withName: SPADES {
        cpus          = { 10 * task.attempt }
        memory        = { 64.GB * (2 ** (task.attempt - 1)) }
        time          = 365.d
        errorStrategy = { task.exitStatus in [143, 137, 21, 1] ? 'retry' : 'finish' }
        maxRetries    = 5
    }
    withName: SPADESHYBRID {
        cpus          = { 10 * task.attempt }
        memory        = { 64.GB * (2 ** (task.attempt - 1)) }
        time          = { 24.h * (2 ** (task.attempt - 1)) }
        errorStrategy = { task.exitStatus in [143, 137, 21, 1] ? 'retry' : 'finish' }
        maxRetries    = 5
    }
    //returns exit code 247 when running out of memory
    withName: BOWTIE2_ASSEMBLY_ALIGN {
        cpus          = { 2 * task.attempt }
        memory        = { 8.GB * task.attempt }
        time          = { 24.h * (2 ** (task.attempt - 1)) }
        errorStrategy = { task.exitStatus in [143, 137, 104, 134, 139, 247] ? 'retry' : 'finish' }
    }
    withName: METABAT2_METABAT2 {
        cpus   = { 8 * task.attempt }
        memory = { 20.GB * task.attempt }
        time   = { 24.h * (2 ** (task.attempt - 1)) }
    }
    withName: MAG_DEPTHS {
        memory = { 16.GB * task.attempt }
        time   = { 24.h * (2 ** (task.attempt - 1)) }
    }
    withName: BUSCO {
        cpus   = { 8 * task.attempt }
        memory = { 20.GB * task.attempt }
        time   = { 24.h * (2 ** (task.attempt - 1)) }
    }
 
    withName: MAXBIN2 {
        errorStrategy = { task.exitStatus in [1, 255] ? 'ignore' : 'retry' }
        time          = { 24.h * (2 ** (task.attempt - 1)) }
    }
 
    withName: DASTOOL_DASTOOL {
        errorStrategy = { task.exitStatus in [143, 137, 104, 134, 139] ? 'retry' : task.exitStatus == 1 ? 'ignore' : 'finish' }
        time          = { 24.h * (2 ** (task.attempt - 1)) }
    }
}

See config file on GitHub

conf/pipeline/sarek/eva
// Profile config names for nf-core/configs
 
params {
    // Specific nf-core/configs params
    config_profile_contact     = 'James A. Fellows Yates (@jfy133)'
    config_profile_description = 'nf-core/sarek EVA profile provided by nf-core/configs'
}
 
env {
    _JAVA_OPTIONS        = "-XX:ParallelGCThreads=1"
    OPENBLAS_NUM_THREADS = 1
    OMP_NUM_THREADS      = 1
}
 
process {
    withName: GATK4_APPLYBQSR {
        clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 3)}G" }
    }
    withName: GATK4_APPLYBQSR_SPARK {
        clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 3)}G" }
    }
    withName: GATK4_APPLYVQSR {
        clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 3)}G" }
    }
    withName: GATK4_BASERECALIBRATOR {
        clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 3)}G" }
    }
    withName: GATK4_BASERECALIBRATOR_SPARK {
        clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 3)}G" }
    }
    withName: GATK4_CALCULATECONTAMINATION {
        clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 3)}G" }
    }
    withName: GATK4_CNNSCOREVARIANTS {
        clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 3)}G" }
    }
    withName: GATK4_CREATESEQUENCEDICTIONARY {
        clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 3)}G" }
    }
    withName: GATK4_ESTIMATELIBRARYCOMPLEXITY {
        clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 3)}G" }
    }
    withName: GATK4_FILTERMUTECTCALLS {
        clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 3)}G" }
    }
    withName: GATK4_FILTERVARIANTTRANCHES {
        clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 3)}G" }
    }
    withName: GATK4_GATHERBQSRREPORTS {
        clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 3)}G" }
    }
    withName: GATK4_GATHERPILEUPSUMMARIES {
        clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 3)}G" }
    }
    withName: GATK4_GENOMICSDBIMPORT {
        clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 3)}G" }
    }
    withName: GATK4_GENOTYPEGVCFS {
        clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 3)}G" }
    }
    withName: GATK4_GETPILEUPSUMMARIES {
        clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 3)}G" }
    }
    withName: GATK4_HAPLOTYPECALLER {
        clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 3)}G" }
    }
    withName: GATK4_INTERVALLISTTOBED {
        clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 3)}G" }
    }
    withName: GATK4_LEARNREADORIENTATIONMODEL {
        clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 3)}G" }
    }
    withName: GATK4_MARKDUPLICATES {
        clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 3)}G" }
    }
    withName: GATK4_MARKDUPLICATES_SPARK {
        clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 3)}G" }
    }
    withName: GATK4_MERGEMUTECTSTATS {
        clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 3)}G" }
    }
    withName: GATK4_MERGEVCFS {
        clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 3)}G" }
    }
    withName: GATK4_MUTECT2 {
        clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 3)}G" }
    }
    withName: GATK4_VARIANTRECALIBRATOR {
        clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 3)}G" }
    }
}

See config file on GitHub

conf/pipeline/taxprofiler/eva
params {
    // Specific nf-core/configs params
    config_profile_contact     = 'James Fellows Yates (@jfy133)'
    config_profile_description = 'nf-core/taxprofiler EVA profile provided by nf-core/configs'
}
 
env {
    OPENBLAS_NUM_THREADS = 1
    OMP_NUM_THREADS      = 1
}
 
process {
 
    withName: BBMAP_BBDUK {
        clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 2)}G" }
        errorStrategy  = { task.exitStatus in [1, 143, 137, 104, 134, 139, 140] ? 'retry' : 'finish' }
    }
 
    withName: MALT_RUN {
        clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 4)}G" }
        errorStrategy  = { task.exitStatus in [1, 143, 137, 104, 134, 139, 140] ? 'retry' : 'finish' }
    }
 
    withName: METAPHLAN_METAPHLAN {
        clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 3)}G" }
        errorStrategy  = { task.exitStatus in [1, 143, 137, 104, 134, 139, 140] ? 'retry' : 'finish' }
    }
 
    withName: MEGAN_RMA2INFO {
        clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 4)}G" }
        errorStrategy  = { task.exitStatus in [1, 143, 137, 104, 134, 139, 140] ? 'retry' : 'finish' }
    }
}
On this page