include required(classpath("application"))

backend {
  default = "Local"
  providers {

    pbs {
      actor-factory = "cromwell.backend.impl.sfs.config.ConfigBackendLifecycleActorFactory"
      config {
        script-epilogue = "sleep 30 && sync"
        concurrent-job-limit = 50
        runtime-attributes = """
          Int cpu = 1
          Int? gpu
          Int? time
          Int? memory_mb
        """
        submit = """
          qsub \
          -N ${job_name} \
          -o ${out} \
          -e ${err} \
          ${true="-lselect=1:ncpus=" false="" defined(cpu)}${cpu}${true=":mem=" false="" defined(memory_mb)}${memory_mb}${true="mb" false="" defined(memory_mb)} \
          ${true="-lwalltime=" false="" defined(time)}${time}${true=":0:0" false="" defined(time)} \
          ${true="-lngpus=" false="" gpu>1}${if gpu>1 then gpu else ""} \
          -V \
          ${script}
        """
        kill = "qdel ${job_id}"
        check-alive = "qstat ${job_id}"
        job-id-regex = "(\\d+).+"
      }      
    }

    pbs_singularity {
      actor-factory = "cromwell.backend.impl.sfs.config.ConfigBackendLifecycleActorFactory"
      config {
        script-epilogue = "sleep 30 && sync"
        concurrent-job-limit = 50
        runtime-attributes = """
          Int cpu = 1
          Int? gpu
          Int? time
          Int? memory_mb
          String singularity_container
          String? singularity_bindpath
        """
        submit = """
          echo "SINGULARITY_BINDPATH=$(echo ${cwd} | sed 's/cromwell-executions/\n/g' | head -n1)cromwell-executions,${singularity_bindpath},$SINGULARITY_BINDPATH singularity exec --cleanenv --home ${cwd} ${if defined(gpu) then '--nv' else ''} ${singularity_container} /bin/bash ${script}" | qsub \
          -N ${job_name} \
          -o ${out} \
          -e ${err} \
          ${true="-lselect=1:ncpus=" false="" defined(cpu)}${cpu}${true=":mem=" false="" defined(memory_mb)}${memory_mb}${true="mb" false="" defined(memory_mb)} \
          ${true="-lwalltime=" false="" defined(time)}${time}${true=":0:0" false="" defined(time)} \
          ${true="-lngpus=" false="" gpu>1}${if gpu>1 then gpu else ""} \
          -V
          # If you see an error "The job was aborted from outside Cromwell"
          #   then check your singularity settings in a workflow options JSON file
          #   (e.g. check if you have an image file defined by "singularity_container")
          # Also, make sure that your input data files (and genome database files)
          #   are on directories recursively bound by
          #   "singularity_bindpath" in a workflow options JSON file
          #   or singularity's built-in environment variable SINGULARITY_BINDPATH.
        """
        # cromwell is desinged to monitor rc (return code) file, which is generated/controlled
        # in ${script}, so if singularity does not run it due to some problems in singuarlity's
        # internal settings then rc file is not generated.
        # this can result in hanging of a cromwell process.
        # setting the below parameter enables monitoring by "check-alive".
        # it will take about "exit-code-timeout-seconds" x 3 time to detect failure.
        exit-code-timeout-seconds = 180

        kill = "qdel ${job_id}"
        check-alive = "qstat -j ${job_id}"
        job-id-regex = "(\\d+)"
      }
    }

    slurm_singularity {
      actor-factory = "cromwell.backend.impl.sfs.config.ConfigBackendLifecycleActorFactory"
      config {
        script-epilogue = "sleep 30 && sync"
        concurrent-job-limit = 50
        runtime-attributes = """
          Int cpu = 1
          Int? gpu
          Int? time
          Int? memory_mb
          String? slurm_partition
          String? slurm_account
          String? slurm_extra_param
          String singularity_container
          String? singularity_bindpath
        """
        submit = """
          sbatch \
          --export=ALL \
          -J ${job_name} \
          -D ${cwd} \
          -o ${out} \
          -e ${err} \
          ${"-t " + time*60} \
          -n 1 \
          --ntasks-per-node=1 \
          ${true="--cpus-per-task=" false="" defined(cpu)}${cpu} \
          ${true="--mem=" false="" defined(memory_mb)}${memory_mb} \
          ${"-p " + slurm_partition} \
          ${"--account " + slurm_account} \
          ${true="--gres gpu:" false="" defined(gpu)}${gpu} \
          ${slurm_extra_param} \
          --wrap "SINGULARITY_BINDPATH=$(echo ${cwd} | sed 's/cromwell-executions/\n/g' | head -n1)cromwell-executions,${singularity_bindpath},$SINGULARITY_BINDPATH singularity exec --cleanenv --home ${cwd} ${if defined(gpu) then '--nv' else ''} ${singularity_container} /bin/bash ${script}"
          # If you see an error "The job was aborted from outside Cromwell"
          #   then check your singularity settings in a workflow options JSON file
          #   (e.g. check if you have an image file defined by "singularity_container")
          # Also, make sure that your input data files (and genome database files)
          #   are on directories recursively bound by
          #   "singularity_bindpath" in a workflow options JSON file
          #   or singularity's built-in environment variable SINGULARITY_BINDPATH.
        """
        kill = "scancel ${job_id}"
        # cromwell is desinged to monitor rc (return code) file, which is generated/controlled
        # in ${script}, so if singularity does not run it due to some problems in singuarlity's
        # internal settings then rc file is not generated.
        # this can result in hanging of a cromwell process.
        # setting the below parameter enables monitoring by "check-alive".
        # it will take about "exit-code-timeout-seconds" x 3 time to detect failure.
        exit-code-timeout-seconds = 180

        # cromwell responds only to non-zero exit code from "check-alive",
        # but "squeue -j [JOB_ID]" returns zero exit code even when job is not found
        # workaround to exit with 1 (like SGE's qstat -j [JOB_ID] does) for such cases.
        check-alive = "CHK_ALIVE=$(squeue --noheader -j ${job_id}); if [ -z $CHK_ALIVE ]; then /bin/bash -c 'exit 1'; else echo $CHK_ALIVE; fi"
        job-id-regex = "Submitted batch job (\\d+).*"
      }
    }

    sge_singularity {
      actor-factory = "cromwell.backend.impl.sfs.config.ConfigBackendLifecycleActorFactory"
      config {
        script-epilogue = "sleep 30 && sync"
        concurrent-job-limit = 50
        runtime-attributes = """
          String sge_pe = "shm"
          Int cpu = 1
          Int? gpu
          Int? time
          Int? memory_mb
          String? sge_queue
          String? sge_extra_param
          String singularity_container
          String? singularity_bindpath
        """
        submit = """
          echo "SINGULARITY_BINDPATH=$(echo ${cwd} | sed 's/cromwell-executions/\n/g' | head -n1)cromwell-executions,${singularity_bindpath},$SINGULARITY_BINDPATH singularity exec --cleanenv --home ${cwd} ${if defined(gpu) then '--nv' else ''} ${singularity_container} /bin/bash ${script}" | qsub \
          -S /bin/sh \
          -terse \
          -b n \
          -N ${job_name} \
          -wd ${cwd} \
          -o ${out} \
          -e ${err} \
          ${if cpu>1 then "-pe " + sge_pe + " " else ""}${if cpu>1 then cpu else ""} \
          ${true="-l h_vmem=$(expr " false="" defined(memory_mb)}${memory_mb}${true=" / " false="" defined(memory_mb)}${if defined(memory_mb) then cpu else ""}${true=")m" false="" defined(memory_mb)} \
          ${true="-l s_vmem=$(expr " false="" defined(memory_mb)}${memory_mb}${true=" / " false="" defined(memory_mb)}${if defined(memory_mb) then cpu else ""}${true=")m" false="" defined(memory_mb)} \
          ${true="-l h_rt=" false="" defined(time)}${time}${true=":00:00" false="" defined(time)}\
          ${true="-l s_rt=" false="" defined(time)}${time}${true=":00:00" false="" defined(time)}\
          ${"-q " + sge_queue} \
          ${"-l gpu=" + gpu} \
          ${sge_extra_param} \
          -V
          # If you see an error "The job was aborted from outside Cromwell"
          #   then check your singularity settings in a workflow options JSON file
          #   (e.g. check if you have an image file defined by "singularity_container")
          # Also, make sure that your input data files (and genome database files)
          #   are on directories recursively bound by
          #   "singularity_bindpath" in a workflow options JSON file
          #   or singularity's built-in environment variable SINGULARITY_BINDPATH.
        """
        # cromwell is desinged to monitor rc (return code) file, which is generated/controlled
        # in ${script}, so if singularity does not run it due to some problems in singuarlity's
        # internal settings then rc file is not generated.
        # this can result in hanging of a cromwell process.
        # setting the below parameter enables monitoring by "check-alive".
        # it will take about "exit-code-timeout-seconds" x 3 time to detect failure.
        exit-code-timeout-seconds = 180

        kill = "qdel ${job_id}"
        check-alive = "qstat -j ${job_id}"
        job-id-regex = "(\\d+)"
      }
    }

    singularity {
      actor-factory = "cromwell.backend.impl.sfs.config.ConfigBackendLifecycleActorFactory"
      config {
        script-epilogue = "sleep 5 && sync"
        concurrent-job-limit = 10
        run-in-background = true
        runtime-attributes = """
          Int? gpu
          String singularity_container
          String? singularity_bindpath
        """
        submit = """
          SINGULARITY_BINDPATH=$(echo ${cwd} | sed 's/cromwell-executions/\n/g' | head -n1)cromwell-executions,${singularity_bindpath},$SINGULARITY_BINDPATH singularity exec --cleanenv --home ${cwd} ${if defined(gpu) then '--nv' else ''} ${singularity_container} /bin/bash ${script}
        """
      }
    }

    Local {
      actor-factory = "cromwell.backend.impl.sfs.config.ConfigBackendLifecycleActorFactory"
      config {
        concurrent-job-limit = 10
      }
    }

    sge {
      actor-factory = "cromwell.backend.impl.sfs.config.ConfigBackendLifecycleActorFactory"
      config {
        script-epilogue = "sleep 30 && sync"
        concurrent-job-limit = 50
        runtime-attributes = """
        String sge_pe = "shm"
        Int cpu = 1
        Int? gpu
        Int? time
        Int? memory_mb
        String? sge_queue
        String? sge_extra_param
        """
        submit = """
        qsub \
        -S /bin/sh \
        -terse \
        -b n \
        -N ${job_name} \
        -wd ${cwd} \
        -o ${out} \
        -e ${err} \
        ${if cpu>1 then "-pe " + sge_pe + " " else ""}${if cpu>1 then cpu else ""} \
        ${true="-l h_vmem=$(expr " false="" defined(memory_mb)}${memory_mb}${true=" / " false="" defined(memory_mb)}${if defined(memory_mb) then cpu else ""}${true=")m" false="" defined(memory_mb)} \
        ${true="-l s_vmem=$(expr " false="" defined(memory_mb)}${memory_mb}${true=" / " false="" defined(memory_mb)}${if defined(memory_mb) then cpu else ""}${true=")m" false="" defined(memory_mb)} \
        ${true="-l h_rt=" false="" defined(time)}${time}${true=":00:00" false="" defined(time)}\
        ${true="-l s_rt=" false="" defined(time)}${time}${true=":00:00" false="" defined(time)}\
        ${"-q " + sge_queue} \
        ${true="-l gpu=" false="" defined(gpu)}${gpu} \
        ${sge_extra_param} \
        -V \
        ${script}
        """
        kill = "qdel ${job_id}"
        check-alive = "qstat -j ${job_id}"
        job-id-regex = "(\\d+)"
      }
    }

    slurm {
      actor-factory = "cromwell.backend.impl.sfs.config.ConfigBackendLifecycleActorFactory"
      config {
        script-epilogue = "sleep 30"
        concurrent-job-limit = 50        
        runtime-attributes = """
        Int cpu = 1
        Int? gpu
        Int? time
        Int? memory_mb
        String? slurm_partition
        String? slurm_account
        String? slurm_extra_param
        """
        submit = """
        sbatch \
        --export=ALL \
        -J ${job_name} \
        -D ${cwd} \
        -o ${out} \
        -e ${err} \
        ${"-t " + time*60} \
        -n 1 \
        --ntasks-per-node=1 \
        ${true="--cpus-per-task=" false="" defined(cpu)}${cpu} \
        ${true="--mem=" false="" defined(memory_mb)}${memory_mb} \
        ${"-p " + slurm_partition} \
        ${"--account " + slurm_account} \
        ${true="--gres gpu:" false="" defined(gpu)}${gpu} \
        ${slurm_extra_param} \
        --wrap "/bin/bash ${script}"
        """
        kill = "scancel ${job_id}"
        check-alive = "squeue -j ${job_id}"
        job-id-regex = "Submitted batch job (\\d+).*"
      }
    }

    google {
      actor-factory = "cromwell.backend.google.pipelines.v2alpha1.PipelinesApiLifecycleActorFactory"

      config {
        # Google project
        project = "your-project-name"
    
        # Base bucket for workflow executions
        root = "gs://your-bucket-name"

        concurrent-job-limit = 1000
        genomics-api-queries-per-100-seconds = 1000
        maximum-polling-interval = 600

        genomics {
          auth = "application-default"
          compute-service-account = "default"
          endpoint-url = "https://genomics.googleapis.com/"
          restrict-metadata-access = false
        }

        filesystems {
          gcs {
            auth = "application-default"
          }
        }
      }
    }
  }
}

services {
  LoadController {
    class = "cromwell.services.loadcontroller.impl.LoadControllerServiceActor"
    config {      
      # disable it (for login nodes on Stanford SCG, Sherlock)
      control-frequency = 21474834 seconds
    }
  }
}

system {
  abort-jobs-on-terminate = true
  graceful-server-shutdown = true
}

call-caching {
  enabled = false
  invalidate-bad-cache-results = true
}

google {
  application-name = "cromwell"
  auths = [
    {
      name = "application-default"
      scheme = "application_default"
    }
  ]
}
