include required(classpath("application"))
backend {
  default = "slurm"
  providers {
    Local {
      config {
        default-runtime-attributes {}
        filesystems {
          local {
            caching {
              duplication-strategy = [
                "soft-link"
                "hard-link"
                "copy"
              ]
              check-sibling-md5 = true
              hashing-strategy = "path+modtime"
            }
            localization = [
              "soft-link"
              "hard-link"
              "copy"
            ]
          }
        }
        concurrent-job-limit = 1000
        script-epilogue = "sleep 5"
        run-in-background = true
        runtime-attributes = """
## Caper custom attributes
# Environment choices = (docker, conda, singularity)
# If environment is not specified then prioritize docker > singularity > conda
# gpu is a plain string (to be able to specify gpu's name)
String? environment
String? conda
String? singularity
String? singularity_bindpath
String? gpu

## Cromwell built-in attributes for docker
String? docker
String? docker_user
"""
        submit = """
if [ '${defined(environment)}' == 'true' ] && [ '${environment}' == 'singularity' ] || \
   [ '${defined(environment)}' == 'false' ] && [ '${defined(singularity)}' == 'true' ] && [ ! -z '${singularity}' ]
then
    mkdir -p $HOME/.singularity/lock/
    flock --exclusive --timeout 600 \
        $HOME/.singularity/lock/`echo -n '${singularity}' | md5sum | cut -d' ' -f1` \
        singularity exec --containall ${singularity} echo 'Successfully pulled ${singularity}'

    singularity exec --cleanenv --home=`dirname ${cwd}` \
        --bind=${singularity_bindpath}, \
        ${if defined(gpu) then ' --nv' else ''} \
        ${singularity} ${job_shell} ${script}

elif [ '${defined(environment)}' == 'true' ] && [ '${environment}' == 'conda' ] || \
     [ '${defined(environment)}' == 'false' ] && [ '${defined(conda)}' == 'true' ] && [ ! -z '${conda}' ]
then
    conda run --name=${conda} ${job_shell} ${script}

else
    ${job_shell} ${script}
fi
"""
        submit-docker = """
rm -f ${docker_cid}

if [ '${defined(environment)}' == 'true' ] && [ '${environment}' == 'docker' ] || \
   [ '${defined(environment)}' == 'false' ] && [ '${defined(docker)}' == 'true' ] && [ ! -z '${docker}' ]
then
    docker run -i --cidfile=${docker_cid} --user=${docker_user} --entrypoint=${job_shell} \
      --volume=${cwd}:${docker_cwd}:delegated ${docker} ${docker_script}
    rc=$(docker wait `cat ${docker_cid}`)
    docker rm `cat ${docker_cid}`
else
    # recover GID lost due to Cromwell running chmod 777 on CWD
    chown :`stat -c '%G' ${cwd}` -R ${cwd}
    chmod g+s ${cwd}

    if [ '${defined(environment)}' == 'true' ] && [ '${environment}' == 'singularity' ] || \
       [ '${defined(environment)}' == 'false' ] && [ '${defined(singularity)}' == 'true' ] && [ ! -z '${singularity}' ]
    then
        mkdir -p $HOME/.singularity/lock/
        flock --exclusive --timeout 600 \
            $HOME/.singularity/lock/`echo -n '${singularity}' | md5sum | cut -d' ' -f1` \
            singularity exec --containall ${singularity} echo 'Successfully pulled ${singularity}'

        singularity exec --cleanenv --home=`dirname ${cwd}` \
            --bind=${singularity_bindpath},${cwd}:${docker_cwd} \
            ${if defined(gpu) then ' --nv' else ''} \
            ${singularity} ${job_shell} ${script} & echo $! > ${docker_cid}
    else
        # remap paths between inside and outside of a docker container
        shopt -s nullglob
        sed -i 's#${docker_cwd}#${cwd}#g' ${script} `dirname ${script}`/write_*.tmp

        if [ '${defined(environment)}' == 'true' ] && [ '${environment}' == 'conda' ] || \
           [ '${defined(environment)}' == 'false' ] && [ '${defined(conda)}' == 'true' ] && [ ! -z '${conda}' ]
        then
            conda run --name=${conda} ${job_shell} ${script} & echo $! > ${docker_cid}
        else
            ${job_shell} ${script} & echo $! > ${docker_cid}
        fi
    fi

    touch ${docker_cid}.not_docker
    wait `cat ${docker_cid}`
    rc=`echo $?`
fi

exit $rc
"""
        kill-docker = """
if [ -f '${docker_cid}.not_docker' ]
then
    kill `cat ${docker_cid}`
else
    docker kill `cat ${docker_cid}`
fi
"""
        root = "/scratch/users/kmualim/arabidopsis_data/ATAC/20240618/Zea_mays/"
      }
      actor-factory = "cromwell.backend.impl.sfs.config.ConfigBackendLifecycleActorFactory"
    }
    slurm {
      config {
        default-runtime-attributes {
          slurm_partition = "akundaje"
        }
        filesystems {
          local {
            caching {
              duplication-strategy = [
                "soft-link"
                "hard-link"
                "copy"
              ]
              check-sibling-md5 = true
              hashing-strategy = "path+modtime"
            }
            localization = [
              "soft-link"
              "hard-link"
              "copy"
            ]
          }
        }
        concurrent-job-limit = 1000
        script-epilogue = "sleep 5"
        run-in-background = true
        runtime-attributes = """
## Caper custom attributes
# Environment choices = (docker, conda, singularity)
# If environment is not specified then prioritize docker > singularity > conda
# gpu is a plain string (to be able to specify gpu's name)
String? environment
String? conda
String? singularity
String? singularity_bindpath
String? gpu


Int cpu = 1
Int? time
Int? memory_mb


String? slurm_partition
String? slurm_account
String? slurm_extra_param
"""
        submit = """
cat << EOF > ${script}.caper
#!/bin/bash

if [ '${defined(environment)}' == 'true' ] && [ '${environment}' == 'singularity' ] || \
   [ '${defined(environment)}' == 'false' ] && [ '${defined(singularity)}' == 'true' ] && [ ! -z '${singularity}' ]
then
    mkdir -p $HOME/.singularity/lock/
    flock --exclusive --timeout 600 \
        $HOME/.singularity/lock/`echo -n '${singularity}' | md5sum | cut -d' ' -f1` \
        singularity exec --containall ${singularity} echo 'Successfully pulled ${singularity}'

    singularity exec --cleanenv --home=`dirname ${cwd}` \
        --bind=${singularity_bindpath}, \
        ${if defined(gpu) then ' --nv' else ''} \
        ${singularity} ${job_shell} ${script}

elif [ '${defined(environment)}' == 'true' ] && [ '${environment}' == 'conda' ] || \
     [ '${defined(environment)}' == 'false' ] && [ '${defined(conda)}' == 'true' ] && [ ! -z '${conda}' ]
then
    conda run --name=${conda} ${job_shell} ${script}

else
    ${job_shell} ${script}
fi

EOF

for ITER in 1 2 3
do
    sbatch --export=ALL -J ${job_name} -D ${cwd} -o ${out} -e ${err} \
        ${'-p ' + slurm_partition} ${'--account ' + slurm_account} \
        -n 1 --ntasks-per-node=1 --cpus-per-task=${cpu} ${if defined(memory_mb) then "--mem=" else ""}${memory_mb}${if defined(memory_mb) then "M" else ""} ${if defined(time) then "--time=" else ""}${time*60} ${if defined(gpu) then "--gres=gpu:" else ""}${gpu}  \
        ${slurm_extra_param} \
        ${script}.caper && exit 0
    sleep 30
done
exit 1
"""
        submit-docker = null
        kill-docker = null
        root = "/scratch/users/kmualim/arabidopsis_data/ATAC/20240618/Zea_mays/"
        check-alive = """
for ITER in 1 2 3
do
    CHK_ALIVE=$(squeue --noheader -j ${job_id} --format=%i | grep ${job_id})
    if [ -z "$CHK_ALIVE" ]
    then
        if [ "$ITER" == 3 ]
        then
            ${job_shell} -c 'exit 1'
        else
            sleep 30
        fi
    else
        echo $CHK_ALIVE
        break
    fi
done
"""
        kill = "scancel ${job_id}"
        job-id-regex = "Submitted batch job ([0-9]+).*"
      }
      actor-factory = "cromwell.backend.impl.sfs.config.ConfigBackendLifecycleActorFactory"
    }
    sge {
      config {
        default-runtime-attributes {}
        filesystems {
          local {
            caching {
              duplication-strategy = [
                "soft-link"
                "hard-link"
                "copy"
              ]
              check-sibling-md5 = true
              hashing-strategy = "path+modtime"
            }
            localization = [
              "soft-link"
              "hard-link"
              "copy"
            ]
          }
        }
        concurrent-job-limit = 1000
        script-epilogue = "sleep 5"
        run-in-background = true
        runtime-attributes = """
## Caper custom attributes
# Environment choices = (docker, conda, singularity)
# If environment is not specified then prioritize docker > singularity > conda
# gpu is a plain string (to be able to specify gpu's name)
String? environment
String? conda
String? singularity
String? singularity_bindpath
String? gpu


Int cpu = 1
Int? time
Int? memory_mb


String? sge_pe
String? sge_queue
String? sge_extra_param
"""
        submit = """
cat << EOF > ${script}.caper
#!/bin/bash

if [ '${defined(environment)}' == 'true' ] && [ '${environment}' == 'singularity' ] || \
   [ '${defined(environment)}' == 'false' ] && [ '${defined(singularity)}' == 'true' ] && [ ! -z '${singularity}' ]
then
    mkdir -p $HOME/.singularity/lock/
    flock --exclusive --timeout 600 \
        $HOME/.singularity/lock/`echo -n '${singularity}' | md5sum | cut -d' ' -f1` \
        singularity exec --containall ${singularity} echo 'Successfully pulled ${singularity}'

    singularity exec --cleanenv --home=`dirname ${cwd}` \
        --bind=${singularity_bindpath}, \
        ${if defined(gpu) then ' --nv' else ''} \
        ${singularity} ${job_shell} ${script}

elif [ '${defined(environment)}' == 'true' ] && [ '${environment}' == 'conda' ] || \
     [ '${defined(environment)}' == 'false' ] && [ '${defined(conda)}' == 'true' ] && [ ! -z '${conda}' ]
then
    conda run --name=${conda} ${job_shell} ${script}

else
    ${job_shell} ${script}
fi

EOF

for ITER in 1 2 3; do
    qsub -V -terse -S ${job_shell} -N ${job_name} -wd ${cwd} -o ${out} -e ${err} \
        ${'-q ' + sge_queue} \
        ${if cpu > 1 then "-pe " + sge_pe + " " else ""} ${if cpu > 1 then cpu else ""} ${true="-l h_vmem=$(expr " false="" defined(memory_mb)}${memory_mb}${true=" / " false="" defined(memory_mb)}${if defined(memory_mb) then cpu else ""}${true=")m" false="" defined(memory_mb)} ${true="-l s_vmem=$(expr " false="" defined(memory_mb)}${memory_mb}${true=" / " false="" defined(memory_mb)}${if defined(memory_mb) then cpu else ""}${true=")m" false="" defined(memory_mb)} ${"-l h_rt=" + time + ":00:00"} ${"-l s_rt=" + time + ":00:00"} ${"-l gpu=" + gpu}  \
        ${sge_extra_param} \
        ${script}.caper && break
    sleep 30
done
"""
        submit-docker = null
        kill-docker = null
        root = "/scratch/users/kmualim/arabidopsis_data/ATAC/20240618/Zea_mays/"
        check-alive = "qstat -j ${job_id}"
        kill = "qdel ${job_id}"
        job-id-regex = "([0-9]+)"
      }
      actor-factory = "cromwell.backend.impl.sfs.config.ConfigBackendLifecycleActorFactory"
    }
    pbs {
      config {
        default-runtime-attributes {}
        filesystems {
          local {
            caching {
              duplication-strategy = [
                "soft-link"
                "hard-link"
                "copy"
              ]
              check-sibling-md5 = true
              hashing-strategy = "path+modtime"
            }
            localization = [
              "soft-link"
              "hard-link"
              "copy"
            ]
          }
        }
        concurrent-job-limit = 1000
        script-epilogue = "sleep 5"
        run-in-background = true
        runtime-attributes = """
## Caper custom attributes
# Environment choices = (docker, conda, singularity)
# If environment is not specified then prioritize docker > singularity > conda
# gpu is a plain string (to be able to specify gpu's name)
String? environment
String? conda
String? singularity
String? singularity_bindpath
String? gpu


Int cpu = 1
Int? time
Int? memory_mb


String? pbs_queue
String? pbs_extra_param
"""
        submit = """
cat << EOF > ${script}.caper
#!/bin/bash

if [ '${defined(environment)}' == 'true' ] && [ '${environment}' == 'singularity' ] || \
   [ '${defined(environment)}' == 'false' ] && [ '${defined(singularity)}' == 'true' ] && [ ! -z '${singularity}' ]
then
    mkdir -p $HOME/.singularity/lock/
    flock --exclusive --timeout 600 \
        $HOME/.singularity/lock/`echo -n '${singularity}' | md5sum | cut -d' ' -f1` \
        singularity exec --containall ${singularity} echo 'Successfully pulled ${singularity}'

    singularity exec --cleanenv --home=`dirname ${cwd}` \
        --bind=${singularity_bindpath}, \
        ${if defined(gpu) then ' --nv' else ''} \
        ${singularity} ${job_shell} ${script}

elif [ '${defined(environment)}' == 'true' ] && [ '${environment}' == 'conda' ] || \
     [ '${defined(environment)}' == 'false' ] && [ '${defined(conda)}' == 'true' ] && [ ! -z '${conda}' ]
then
    conda run --name=${conda} ${job_shell} ${script}

else
    ${job_shell} ${script}
fi

EOF

for ITER in 1 2 3; do
    qsub -V -N ${job_name} -o ${out} -e ${err} \
        ${'-q ' + pbs_queue} \
        ${"-lnodes=1:ppn=" + cpu}${if defined(gpu) then ":gpus=" + gpu else ""} ${if defined(memory_mb) then "-l mem=" else ""}${memory_mb}${if defined(memory_mb) then "mb" else ""} ${"-lwalltime=" + time + ":0:0"}  \
        ${pbs_extra_param} \
        ${script}.caper && break
    sleep 30
done
"""
        submit-docker = null
        kill-docker = null
        root = "/scratch/users/kmualim/arabidopsis_data/ATAC/20240618/Zea_mays/"
        check-alive = "qstat ${job_id}"
        kill = "qdel ${job_id}"
        job-id-regex = "([0-9]+)"
      }
      actor-factory = "cromwell.backend.impl.sfs.config.ConfigBackendLifecycleActorFactory"
    }
    lsf {
      config {
        default-runtime-attributes {}
        filesystems {
          local {
            caching {
              duplication-strategy = [
                "soft-link"
                "hard-link"
                "copy"
              ]
              check-sibling-md5 = true
              hashing-strategy = "path+modtime"
            }
            localization = [
              "soft-link"
              "hard-link"
              "copy"
            ]
          }
        }
        concurrent-job-limit = 1000
        script-epilogue = "sleep 5"
        run-in-background = true
        runtime-attributes = """
## Caper custom attributes
# Environment choices = (docker, conda, singularity)
# If environment is not specified then prioritize docker > singularity > conda
# gpu is a plain string (to be able to specify gpu's name)
String? environment
String? conda
String? singularity
String? singularity_bindpath
String? gpu


Int cpu = 1
Int? time
Int? memory_mb


String? lsf_queue
String? lsf_extra_param
"""
        submit = """
cat << EOF > ${script}.caper
#!/bin/bash

if [ '${defined(environment)}' == 'true' ] && [ '${environment}' == 'singularity' ] || \
   [ '${defined(environment)}' == 'false' ] && [ '${defined(singularity)}' == 'true' ] && [ ! -z '${singularity}' ]
then
    mkdir -p $HOME/.singularity/lock/
    flock --exclusive --timeout 600 \
        $HOME/.singularity/lock/`echo -n '${singularity}' | md5sum | cut -d' ' -f1` \
        singularity exec --containall ${singularity} echo 'Successfully pulled ${singularity}'

    singularity exec --cleanenv --home=`dirname ${cwd}` \
        --bind=${singularity_bindpath}, \
        ${if defined(gpu) then ' --nv' else ''} \
        ${singularity} ${job_shell} ${script}

elif [ '${defined(environment)}' == 'true' ] && [ '${environment}' == 'conda' ] || \
     [ '${defined(environment)}' == 'false' ] && [ '${defined(conda)}' == 'true' ] && [ ! -z '${conda}' ]
then
    conda run --name=${conda} ${job_shell} ${script}

else
    ${job_shell} ${script}
fi

EOF

for ITER in 1 2 3; do
    bsub -env "all" -J ${job_name} -cwd ${cwd} -o ${out} -e ${err} \
        ${'-q ' + lsf_queue} \
        ${"-n " + cpu} ${if defined(gpu) then "-gpu " + gpu else ""} ${if defined(memory_mb) then "-M " else ""}${memory_mb}${if defined(memory_mb) then "m" else ""} ${"-W " + 60*time}  \
        ${lsf_extra_param} \
        ${job_shell} ${script}.caper && break
    sleep 30
done
"""
        submit-docker = null
        kill-docker = null
        root = "/scratch/users/kmualim/arabidopsis_data/ATAC/20240618/Zea_mays/"
        check-alive = "bjobs ${job_id}"
        kill = "bkill ${job_id}"
        job-id-regex = "Job <([0-9]+)>.*"
      }
      actor-factory = "cromwell.backend.impl.sfs.config.ConfigBackendLifecycleActorFactory"
    }
  }
}
webservice {}
services {
  LoadController {
    class = "cromwell.services.loadcontroller.impl.LoadControllerServiceActor"
    config {
      control-frequency = "21474834 seconds"
    }
  }
}
system {
  job-rate-control {
    jobs = 1
    per = "2 seconds"
  }
  abort-jobs-on-terminate = true
  graceful-server-shutdown = true
  max-concurrent-workflows = 40
}
call-caching {
  invalidate-bad-cache-results = true
  enabled = true
}
akka {
  http {
    server {
      request-timeout = "60 seconds"
    }
  }
}
database {
  db {
    connectionTimeout = 30000
    numThreads = 1
  }
}
