#!/bin/bash
# -*- mode: sh; c-basic-offset: 4; tab-width: 8; indent-tabs-mode: nil -*-
# vi: set shiftwidth=4 tabstop=8 softtabstop=4 expandtab:
# :indentSize=4:tabSize=8:noTabs=true:
# vim: filetype=sh

# ---- error handling
set -o errexit;
set -o posix;
set -o pipefail;
set -o errtrace;
set -o nounset

unexpected_error() {
    local errstat=$?
    echo "${g_prog:-$(basename "$0")}: Error! Encountered unexpected error at 'line $(caller)', bailing out..." 1>&2
    exit $errstat;
}
trap unexpected_error ERR;

# Force the path to only what we need to run this script
PATH_ORIG=$PATH
PATH="/usr/bin:/bin"

# ---- error functions

merror() {
    echo "${g_prog:=$0}: Error! ""$@" 1>&2;
    exit 1;
}
minterror() {
    echo "${g_prog:=$0}: Internal Error! ""$@" 1>&2;
    exit 1;
}
mwarn() {
    echo "${g_prog:=$0}: Warning! ""$@" 1>&2;}

# ---- pre-globals

g_prog=$(basename "$0");
g_progdir=$(dirname "$0");
g_progdir_abs=$(dirname "$(readlink -f "$0")");

# ---- global env

. "$g_progdir_abs/../../private/runtime-common/lib/globalenv.ish"

# ---- usage
usage() {
    local exitstat=2;
    if [[ ! -z "${2:-}" ]] ; then
        if [[ ! $2 =~ [[:digit:]]+ ]] ; then
            minterror "usage(): exitstat ($exitstat) must be numeric."
        fi
        exitstat=$2;
    fi

    # Only redirect to stderr on non-zero exit status
    if [[ $exitstat -ne 0 ]] ; then
        exec 1>&2;
    fi

    if [[ ! -z "${1:-}" ]] ; then
        echo "$g_prog: Error! $1"
    fi

    supported_jms_str=$(echo $JMS_SUPPORTED)
    supported_jms_str=${supported_jms_str// /|}
    supported_jms_str=${supported_jms_str//.\*/*}

    echo "Usage: $g_prog \\"
    echo "             --start|--status|--stop \\"
    echo "             \\"
    echo "             [--jmsenv path/to/jmsenv_XX.ish] \\"
    echo "             [--passenv \"envname1 envname2\"] \\"
    echo "             \\"
    echo "             [--jobid jobid] \\"
    echo "             [--jobname jobname] \\"
    echo "             [--stdoutfile file] \\"
    echo "             [--stderrfile file] \\"
    echo "             [--nproc N] \\"
    echo "             [--optstr \"str\" ] \\"
    echo "             [--cmd cmd ] \\"
    echo "             [--jmstype $supported_jms_str] \\"
    echo "             [--computecfg-id \"str\" ] \\"
    echo "             [--computecfg-name \"str\" ] \\"
    echo "             [--computecfg-description \"str\" ] \\"
    echo "             [--testmode|--test]] \\"
    echo "             [--testmode2|--test2]] \\"
    echo "             [--gen|--generate] \\"
    echo "             [--wait] \\"
    echo "             [--timeout|--jmstype-timeout N] \\"
    echo "             [-n|--noexec|--no-exec] \\"
    echo "             [--exec] \\"
    echo "             [-p|--printcmd] \\"
    echo "             [-v|-vN|--verbose|--verboseN] \\"
    echo "             [--help]"
    echo ""
    echo "         --start             -- run the JMS start command (e.g. qsub, bsub,...)"
    echo "         --status            -- run the JMS status command (e.g. qstat, bjobs)"
    echo "         --stop              -- run the JMS stop command (e.g. qdel, bkill,...)"
    echo "         --jmsenv            -- path to script setting job environment"
    echo "         --passenv           -- list of env varnames to pass through from"
    echo "                                caller (space or comma separated)"
    echo "         --jobid             -- job id"
    echo "         --jobname           -- job name"
    echo "         --stdoutfile        -- file for stdout output"
    echo "         --stderrfile        -- file for stderr output"
    echo "         --nproc             -- number of processors to use"
    echo "         --optstr            -- arbitrary option string, passed on a per-job"
    echo "                                basis (for parsing in jmsenv file)"
    echo "         --cmd               -- command to run"
    echo "         --jmstype           -- force the jms type (instead of determining"
    echo "                                it from the jmsenv file"
    echo "         --computecfg-id             -- id of computecfg"
    echo "         --computecfg-name           -- name of computecfg"
    echo "         --computecfg-description    -- description of of computecfg"
    echo "         --testmode|--test   -- print/submit a test command to the queue"
    echo "         --testmode2|--test2 -- print a sample test command"
    echo "         --gen|--generate    -- generate a simple test script (only"
    echo "                                in --testmode if script doesn't exist)"
    echo "         --wait              -- blocking wait for test command to complete"
    echo "                                (only with '--start --testmode --exec')"
    echo "         --timeout|--jmstest-timeout  -- timeout for test command to complete"
    echo "                                         in seconds, only with --wait"
    echo "                                         (default: 300)"
    echo "         -n|--noexec         -- dry run, do not execute, print commands"
    echo "         --exec              -- force execute mode with --testmode to submit"
    echo "         -p|--printcmd       -- do not execute, print commands"
    echo "         -v|--verbose        -- verbose mode (multiple times, more verbose)"
    echo "         -vN                 -- verbose mode, N is the verbosity level (0-9)"
    echo "         --help              -- print this usage";
    echo
    echo "Examples:"
    echo
    echo "   # View underlying submit command for a simple test command"
    echo "   runjmscmd --start --testmode"
    echo
    echo "   # Generate a simple testmode script (if it doesn't exist), and"
    echo "   # submit it.  Default is to generate the test script, stdout and"
    echo "   # stderr files in the current working directory"
    echo "   runjmscmd --start --testmode --gen --exec"
    echo
    echo "   # Same as previous example, but print out submit command when"
    echo "   # executing it"
    echo "   runjmscmd --start --testmode --gen --exec --verbose"
    echo
    echo "   # Submit a specific test script.  The default is to generate"
    echo "   # the stdout and stderr files in the same directory as the script."
    echo "   runjmscmd --start --testmode --cmd path/to/testscript.sh --exec"
    echo

    if [[ $exitstat -ne 0 ]] ; then
        # Print error again, useful for long usages messages
        if [[ ! -z "${1:-}" ]] ; then
            echo ""
            echo "$g_prog: Error! $1"
        fi
    fi

    # bash only:
    if [[ $exitstat -ne 0 ]] ; then
        echo "  at: $(caller)";
    fi
    exit $exitstat;
}

# ---- argument parsing
# Save off the original args, use as "${g_origargs[@]}" (with double quotes)
declare -a g_origargs;
g_origargs=( ${1+"$@"} )

parseargs() {
    opt_start=false;
    opt_status=false;
    opt_stop=false;
    opt_jmsenv="";
    opt_passenv="";
    opt_jobid="";
    opt_jobname="";
    opt_stdoutfile="";
    opt_stderrfile="";
    opt_nproc="";
    opt_optstr="";
    opt_cmd="";
    opt_jmstype="";
    opt_computecfg_id="";
    opt_computecfg_name="";
    opt_computecfg_description="";
    opt_testmode=false;
    opt_testmode2=false;
    opt_generate=false;
    opt_wait=false;
    opt_timeout="300";
    opt_noexec=false;
    opt_exec=false;
    opt_printcmd=false;
    opt_verbose="";

    while [[ $# != 0 ]]; do
        opt="$1"; shift;
        case "$opt" in
            # Flag with no argument example:
            #   --flag|--fla|--fl|--f)
            #     opt_flag=true;;
            # Option with argument example:
            #   --arg|--ar|--a)
            #     [[ $# -eq 0 ]] && usage;
            #     opt_somearg=$1; shift;;
            --start) opt_start=true;;
            --status) opt_status=true;;
            --stop) opt_stop=true;;
            --jmsenv)
                [[ $# -eq 0 ]] && usage;
                opt_jmsenv=$1; shift;;
            --passenv)
                [[ $# -eq 0 ]] && usage;
                opt_passenv=$1; shift;;
            --jobid)
                [[ $# -eq 0 ]] && usage;
                opt_jobid=$1; shift;;
            --jobname)
                [[ $# -eq 0 ]] && usage;
                opt_jobname=$1; shift;;
            --stdoutfile)
                [[ $# -eq 0 ]] && usage;
                opt_stdoutfile=$1; shift;;
            --stderrfile)
                [[ $# -eq 0 ]] && usage;
                opt_stderrfile=$1; shift;;
            --nproc)
                [[ $# -eq 0 ]] && usage;
                opt_nproc=$1; shift;;
            --optstr)
                [[ $# -eq 0 ]] && usage;
                opt_optstr=$1; shift;;
            --cmd)
                [[ $# -eq 0 ]] && usage;
                opt_cmd=$1; shift;;
            --jmstype)
                [[ $# -eq 0 ]] && usage;
                opt_jmstype=$1; shift;;
            --computecfg-id)
                [[ $# -eq 0 ]] && usage;
                opt_computecfg_id=$1; shift;;
            --computecfg-name)
                [[ $# -eq 0 ]] && usage;
                opt_computecfg_name=$1; shift;;
            --computecfg-description)
                [[ $# -eq 0 ]] && usage;
                opt_computecfg_description=$1; shift;;
            --testmode|--test) opt_testmode=true;;
            --testmode2|--test2) opt_testmode=true;;
            --gen|--generate) opt_generate=true;;
            --wait) opt_wait=true;;
            --timeout|--jmstest-timeout)
                [[ $# -eq 0 ]] && usage;
                [[ ! $1 =~ ^[0-9]+$ ]] && usage;
                opt_timeout=$1; shift;;
            -v|--verbose)
                [[ -z "$opt_verbose" ]] && opt_verbose=0;
                opt_verbose=$(( $opt_verbose + 1));;
            -v[0-9]) opt_verbose=${opt#-v};;
            --verbose[0-9]) opt_verbose=${opt#--verbose};;
            -n|--noexec|--no-exec) opt_noexec=true;;
            --exec) opt_exec=true;;
            -p|--printcmd) opt_printcmd=true;;
            -h|-help|--help|--hel|--he|--h) usage "" 0;;
            --) # Ignore the "--" separator, we have no subprogs
                ;;
            -*) usage "Unrecognized option: $opt";;
            *) usage "Unexpected extraneous arguments detected: $opt "${1+"$@"}
        esac
    done

    if [[ ! -z "${@+${@}}" ]]; then
        local argstr=${1+"$@"}
        usage "Extraneous arguments detected: $argstr"
    fi

    if ! $opt_start && ! $opt_status && ! $opt_stop; then
        usage "Must specify either --start, --status or --stop argument"
    fi
    if  ( $opt_start && $opt_stop ) ||
        ( $opt_start && $opt_status ) ||
        ( $opt_status && $opt_stop ) ; then
        usage "Must specify only one of --start, --status or --stop arguments"
    fi
    if  [[ ! -z "$opt_jmstype" ]] &&
        ! jms_supported "$opt_jmstype" ; then
        usage "Unrecognized value for --jmstype ($opt_jmstype), expected one of: $(echo $JMS_SUPPORTED)"
    fi

    # Force the noexec option if testmode is specified
    if $opt_testmode ; then
        if [[ -z "$opt_verbose" ]] ; then
            # turn on verbose mode by default for --testmode
            opt_verbose=1;
        fi
        if $opt_exec; then
            opt_noexec=false;
        else
            opt_noexec=true;
        fi
    fi

    # verbose is off by default
    if [[ -z "$opt_verbose" ]] ; then
        opt_verbose=0;
    fi
}


# ---- globals

set_preglobals() {
    # We currently support these Job Management Systems:
    #
    # SGE Variants:
    #   SGE (Son of Grid Engine)
    #   OGS (Open Grid Scheduler)
    #   UGE (Univa Grid Engine)
    #
    # LSF Variants:
    #   LSF (Platform Load Sharing Facility)
    #   OpenLava (Open Source LSF compatible scheduler)
    #
    # PBS Variants:
    #   TORQUE  (Terascale Open-source Resource and QUEue Manager, extension
    #            of PBS)
    #   PBS Pro (PBS Professional, PBS Works)
    #   PBS     (Generic PBS, supported mostly for historical reasons)
    #
    # Slurm Variants:
    #   Slurm  (Slurm Workload Manager, Simple Linux Utility for Resource
    #           Management)
    #
    # Other Job Management Systems:
    #   OtherJMS__<JMSTYPE>  (Third party JMS, currently unsupported or
    #                         unrecognized by PacBio software, but may be
    #                         supported in a future release)
    #
    # Custom Job Management Systems:
    #   CustomJMS__<JMSNAME>  (Custom job managment system)
    #
    # OBSOLETE:
    #   Note that this one is obsolete and we don't expect anyone is using it:
    #      OpenPBS (Portable Batch System)
    JMS_SUPPORTED="
        SGE
        OGS
        UGE

        LSF
        OpenLava

        PBS
        TORQUE
        PBSPro

        Slurm

        OtherJMS__.*
        CustomJMS__.*

    "
    # NOTE: To add support for an 'unsupported' JMS, which may be added to
    #       the installer in future releases, (called "Newjms", for
    #       example), add something like this to the user jmsenv file
    #       (i.e. $SMRT_ROOT/userdata/user_jmsenv/user.jmsenv.ish):
    #
    #          if ! declare -Ff Newjms_start 1>&2 ; then
    #
    #              Newjms_start() {
    #                 . . .
    #              }
    #              Newjms_stop() {
    #                 . . .
    #              }
    #              Newjms_status() {
    #                 . . .
    #              }
    #          fi
    #
    #       And select 'OtherJMS__*' as the JMS_TYPE in the installer,
    #       specifying "Newjms" as the JMS name.  Alternatively, do
    #       the following:
    #
    #        - Set jms type in $SMRT_ROOT/userdata/config/smrtlink.config):
    #
    #              jmsselect__jmstype='Unsupported__Newjms';
    #
    #        - Set $SMRT_ROOT/userdata/generated/config/computecfg_XX/jmsenv_XX.ish
    #          contents to something like this (where computecfg_00 indicates
    #          the compute configuration, e.g. computecfg_00):
    #
    #              JMSCONFIG_NEWJMS_BINDIR="/path/to/Newjms/bin"
    #              if [[ ! -z "$JMSCONFIG_NEWJMS_BINDIR" ]]; then
    #                  PATH="$LSF_BINDIR:$PATH";'
    #              fi
    #
    #              # Other options which the installer may support for this
    #              # JMS:
    #              JMSCONFIG_NEWJMS_QUEUE="queuename"
    #              JMSCONFIG_NEWJMS_START_ARGS=""
    #
    #
    # NOTE: To add support for a 'custom' JMS, which will not be
    #       supported by the installer (called "Myjms", for example),
    #       add something like this to the user jmsenv file
    #       (i.e. $SMRT_ROOT/userdata/user_jmsenv/user.jmsenv.ish):
    #
    #          Myjms_start() {
    #             . . .
    #          }
    #          Myjms_stop() {
    #             . . .
    #          }
    #          Myjms_status() {
    #             . . .
    #          }
    #
    #       And select 'Custom' as the JMS_TYPE in the installer,
    #       specifying "Myjms" as the JMS name.  Alternatively, do
    #       the following:
    #
    #        - Set jms type in $SMRT_ROOT/userdata/config/smrtlink.config):
    #
    #              jmsselect__jmstype='Custom__Myjms';
    #
    #        - Set $SMRT_ROOT/userdata/generated/config/computecfg_XX/jmsenv_XX.ish
    #          contents to something like this (where computecfg_00 indicates
    #          the compute configuration, e.g. computecfg_00):
    #
    #              JMSCONFIG_MYJMS_BINDIR="/path/to/Myjms/bin"
    #              if [[ ! -z "$JMSCONFIG_MYJMS_BINDIR" ]]; then
    #                  PATH="$LSF_BINDIR:$PATH";'
    #              fi
}

set_globals() {
    # List of environment variables to pass through by default
    g_env_pass_varlist="
        USER
        LOGNAME
        TERM
        TERMCAP
    "
    # Eliminate extraneous whitespace (one space between words)
    g_env_pass_varlist=$(echo $g_env_pass_varlist);

    g_verbose_level=$opt_verbose;
    g_noexec=$opt_noexec;
    g_printcmd=$opt_printcmd;

    g_userdata_dir="$g_progdir_abs/../../../../userdata"

    g_jmstest_dir="$g_userdata_dir/jobs_root/jmstest"
}

# ---- subroutines

squash_env() {
    # Create regex for detecting pass through env variables
    local l_passthrough_re=""
    for envvar in $g_env_pass_varlist ${opt_passenv//,/ }; do
        l_passthrough_re="$l_passthrough_re|$envvar";
    done
    l_passthrough_re="${l_passthrough_re#|}"
    l_passthrough_re="^(${l_passthrough_re})\$"

    # Read all the environment variables that are currently set
    # NOTE: if we are in "set -o posix" mode, then the output of export -p is
    #       in the 'export VARNAME="value"' form.  If not, then it  is in the
    #       'declare -x VARNAME="value"' form.
    local all_envvars;
    all_envvars=$(export -p | sed -ne 's/^\(declare[[:space:]]\+-x\|export\)[[:space:]]\+\([^=]\+\).*/\2/p')

    local envvar;
    for envvar in $all_envvars; do
        if [[ ! $envvar =~ $l_passthrough_re ]] ; then
            # Do not squash PATH, we've already set it to only what we need
            # to run this script above (i.e. /usr/bin:bin)
            [[ x"$envvar" == x"PATH" ]] && continue;
            # Not passing this env var, unset it
            unset "$envvar"
        fi
    done
}

runcmd() {
    local -a envs;
    local -a args;

    # Separate the leading environment variable settings from the command args
    local envdone=false;
    for i in ${1+"$@"}; do
        if ! $envdone && [[ $i =~ ^([[:alpha:]_][[:alnum:]_]*)=([\"\'].*) ]]; then
            envs+=( "${BASH_REMATCH[1]}=${BASH_REMATCH[2]}" )
        elif ! $envdone && [[ $i =~ ^([[:alpha:]_][[:alnum:]_]*)=(.*) ]]; then
            envs+=( "${BASH_REMATCH[1]}='${BASH_REMATCH[2]}'" )
        else
            envdone=true;
            # NOTE: with "set -u nounset" set, we have to use += or do
            #       something like this (since "${args[@]}" and "${#args[@]}"
            #       will give errors if the array is empty).  Another option
            #       is:
            #           args=( "${args[@]+${args[@]}}" "$i" );
            args+=( "$i" );
        fi
    done

    # Determine the command in a format that can be printed out and
    # copy-and-pasted directly into a shell and can be run with an "eval".
    # NOTE: Another option is to use:
    #          cmd=$(printf '%q ' ${1+"$@"})
    #       But it will use backslashes instead of quotes, which is fine for
    #       executing, but it doesn't look as natural as with quotes.  Plus
    #       using quotes make it look like the trace output under "set -x"
    #       or "set -o xtrace"
    local cmd;
    cmd=$( PS4=; set -x; ( true "${args[@]}" ) 2>&1 );
    cmd=${cmd#true };
    [[ ! -z "${envs[@]+set}" ]] && cmd="${envs[@]} $cmd"

    # Print out the command if neceassry.  The output should be
    # copy-and-pastable directly into a shell.
    if [[ $g_verbose_level -ge 1 ]] || $g_noexec || $g_printcmd; then
        $g_noexec && echo -n "noexec: "
        echo "$cmd"
    fi

    if ! $g_noexec && ! $g_printcmd; then
        # The 'eval "$cmd"' will cause an error in bash-3.2.25 (on centos-5)
        # if $cmd exits with non-zero status (it does not in bash 4.x).  Need
        # to put the '|| stat=$?' inside the "eval" to make it work for both.
        # But also note that in order for this to work properly, the "$cmd"
        # part must be double quoted, the "||" must be quoted or escaped, and
        # the '$?' must be single quoted or escaped (so tht it does not get
        # evaluated before the eval get executed).
        # So in order properly handle a commadn line this:
        #
        #    cmd=cmd="bash -c 'echo foo; echo \"foo bar\" blah\ blah; exit 23'"
        #
        # The eval statement should somthing look like one of these:
        #
        #    eval "$cmd || stat=\$?"
        #    eval "$cmd"' || stat=$?'
        #    eval "$cmd" \|\| stat='$?'
        #    eval "$cmd" "||" stat='$?'
        stat=0;
        eval "$cmd || stat=\$?"

        if $opt_testmode && $opt_start && $opt_wait; then
            if [[ $stat -ne 0 ]] ; then
                merror "Could not run submit command succesfully."
            fi

            echo
            echo "Job submit command completed successfully."

            local starttime;
            starttime=$(date +"%s");
            local warnsec=60
            local warntime=$(( $starttime + $warnsec ));
            local waited=false;
            local warned=false;
            local endtime=$(( $starttime + $opt_timeout ));
            # Poll every 1s for the donefile
            local waittime=1;
            local curtime;
            while true; do
                if [[ -e "$g_donefile" ]] ; then
                    # We assumet that creating the donefile is the last thing
                    # that the script does.  So if the donefile exists, the
                    # command completed successfully, i.e. no need to check
                    # any exit status.
                    echo "Job completed successfully ($(( $curtime - $starttime )) seconds)."
                    return 0;
                fi

                curtime=$(date +"%s");
                if ! $waited; then
                    echo "Waiting for job completion..."
                    waited=true;
                fi
                if ! $warned; then
                    if [[ $curtime -gt $warntime ]] ; then
                        echo "Command not complete after $warnsec seconds, check job queue to see if running."
                        echo "Waiting for $(( $endtime - $warntime )) more seconds..."
                        warned=true;
                    fi
                fi
                if [[ $curtime -gt $endtime ]] ; then
                    merror "Command failed to complete, timeout after $opt_timeout seconds."
                fi
                sleep "$waittime"
            done
        else
            # Normal mode, return the exit status of the command
            return $stat
        fi
    fi
    return 0
}

set_templatevars() {
    if $opt_testmode; then
        JOBVAR_JOB_ID="job-id.XXX"
        JOBVAR_JOB_NAME="";
        JOBVAR_STDOUT_FILE="";
        JOBVAR_STDERR_FILE="";
        JOBVAR_NPROC=1;
        JOBVAR_JOB_OPTSTR=""
        JOBVAR_JOB_CMD="${g_jmstest_dir}/jmstestcmd.sh"
        JOBVAR_COMPUTECFG_ID="default"
        JOBVAR_COMPUTECFG_NAME="ComputeConfigDefault"
        JOBVAR_COMPUTECFG_DESCRIPTION="default compute config"
    elif $opt_testmode2; then
        JOBVAR_JOB_ID="job-id.XXX"
        JOBVAR_JOB_NAME="J112233.runjmscmd_test_jobname";
        JOBVAR_STDOUT_FILE="/path/to/stdout.log";
        JOBVAR_STDERR_FILE="/path/to/stderr.log";
        JOBVAR_NPROC=1;
        JOBVAR_JOB_OPTSTR=""
        JOBVAR_JOB_CMD='tmpcmd.sh --arg1 --arg2 "string arg with spaces"'
        JOBVAR_COMPUTECFG_ID="default"
        JOBVAR_COMPUTECFG_NAME="ComputeConfigDefault"
        JOBVAR_COMPUTECFG_DESCRIPTION="default compute config"
    fi

    # Modify template parameters if specified on the command line
    [[ ! -z "$opt_jobid" ]]      && JOBVAR_JOB_ID=$opt_jobid;
    [[ ! -z "$opt_jobname" ]]    && JOBVAR_JOB_NAME=$opt_jobname;
    [[ ! -z "$opt_stdoutfile" ]] && JOBVAR_STDOUT_FILE=$opt_stdoutfile;
    [[ ! -z "$opt_stderrfile" ]] && JOBVAR_STDERR_FILE=$opt_stderrfile;
    [[ ! -z "$opt_nproc" ]]      && JOBVAR_NPROC=$opt_nproc;
    [[ ! -z "$opt_optstr" ]]     && JOBVAR_JOB_OPTSTR=$opt_optstr;
    [[ ! -z "$opt_cmd" ]]        && JOBVAR_JOB_CMD=$opt_cmd;
    [[ ! -z "$opt_computecfg_id" ]]  && JOBVAR_COMPUTECFG_ID=$opt_computecfg_id;
    [[ ! -z "$opt_computecfg_name" ]]  && JOBVAR_COMPUTECFG_NAME=$opt_computecfg_name;
    [[ ! -z "$opt_computecfg_description" ]]  && JOBVAR_COMPUTECFG_DESCRIPTION=$opt_computecfg_description;

    if $opt_testmode; then
        local jobdir_abs;
        local jobfile_root;
        local donefile;
        jobdir_abs="$(readlink -f "$(dirname "$JOBVAR_JOB_CMD")")"
        jobfile_root="$(basename "${JOBVAR_JOB_CMD%.sh}")"
        donefile="$jobdir_abs/${jobfile_root}.done"

        if [[ -z "$JOBVAR_JOB_NAME" ]] ; then
            JOBVAR_JOB_NAME="J123.${jobfile_root}";
        fi
        if [[ -z "$JOBVAR_STDOUT_FILE" ]] ; then
            JOBVAR_STDOUT_FILE="${jobfile_root}.stdout";
            if [[ ! $JOBVAR_STDOUT_FILE =~ ^/ ]] ; then
                JOBVAR_STDOUT_FILE="${jobdir_abs}/${JOBVAR_STDOUT_FILE}"
            fi
        fi
        if [[ -z "$JOBVAR_STDERR_FILE" ]] ; then
            JOBVAR_STDERR_FILE="${jobfile_root}.stderr";
            if [[ ! $JOBVAR_STDERR_FILE =~ ^/ ]] ; then
                JOBVAR_STDERR_FILE="${jobdir_abs}/${JOBVAR_STDERR_FILE}"
            fi
        fi

        # Generate a simple test script, if specified with --gen
        if $opt_generate; then
            mkdir -p "$jobdir_abs"
            cat <<-EOF > "$JOBVAR_JOB_CMD"
	#!/bin/bash

	set -o errexit;
	set -o posix;
	set -o pipefail;
	set -o errtrace;
	set -o nounset

	unexpected_error() {
	    local errstat=\$?
	    echo "\${g_prog:-\$(basename "\$0")}: Error! Encountered unexpected error at 'line \$(caller)', bailing out..." 1>&2
	    exit \$errstat;
	}
	trap unexpected_error ERR;

	g_prog=\$(basename "\$0");
	g_progdir=\$(dirname "\$0");
	g_progdir_abs=\$(dirname "\$(readlink -f "\$0")");

	# Run a few simple commands on the cluster host
	hostname
	pwd

	echo "This is a message to stderr." 1>&2

	# Drop a "done" file to indicate that the job finished on the cluster
	jmstest_donefile='${donefile}';
	echo "Test command done. (\$(date))" > "\${jmstest_donefile}.tmp"
	mv "\${jmstest_donefile}.tmp" "\${jmstest_donefile}"

	exit 0
	EOF
        chmod +x "$JOBVAR_JOB_CMD"
        fi

        # Remove the donefile, so we don't look at the previous donefile
        rm -f "$donefile"

        # The --wait option depends on g_donefile
        g_donefile=$donefile;

    fi

    return 0;
}

get_jmsenv_ish() {
    local jmsenv_ish_file="";

    local l_jmsenv=$opt_jmsenv;

    if [[ -z "$l_jmsenv" ]] ; then
        local l_config_dir="$g_userdata_dir/generated/config"
        # If JOBVAR_COMPUTECFG_ID is not set (i.e. not specified with the
        # --computecfg-id command line option), but the 'default' link exists,
        # then set JOBVAR_COMPUTECFG_ID=default to pick up the default
        # computecfg.  This could be useful for running something like this
        # from the command line:
        #    runjmscmd --status --job-id <jobid>
        # But, strictly speaking, we should always specify the --computecfg-id
        # when used in production.
        if  [[ -z "${JOBVAR_COMPUTECFG_ID+set}" ]] ||
            [[ -z "${JOBVAR_COMPUTECFG_ID}" ]] ; then
            if  [[ -e "$l_config_dir/default" ]] &&
                [[ -h "$l_config_dir/default" ]] ; then
                JOBVAR_COMPUTECFG_ID="default"
            fi
        fi

        if [[ x"$JOBVAR_COMPUTECFG_ID" == x"default" ]] ; then
            # Resolve the 'default' computecfg link to 'computecfg_NN'
            JOBVAR_COMPUTECFG_ID=$(basename "$(readlink -f "$l_config_dir/$JOBVAR_COMPUTECFG_ID")")
        fi

        # Attempt to auto detect jmsenv here. If can't find it, give a
        # error asking specify "--jmsenv file" or "--jmsenv /dev/null"
        local computecfg_idnum=${JOBVAR_COMPUTECFG_ID##*_};
        l_jmsenv="$g_userdata_dir/generated/config/${JOBVAR_COMPUTECFG_ID}/jmsenv_${computecfg_idnum}.ish"
        if [[ ! -e "$l_jmsenv" ]] ; then
            merror "Could not autodetect jmsenv.ish file (at location '$l_jmsenv') in test mode, please specify with '--jmsenv /path/to/file'"
        fi
    fi

    if [[ ! -z "$l_jmsenv" ]] ; then
        if [[ ! -e "$l_jmsenv" ]] ; then
            merror "The specified jmsenv file '$l_jmsenv' does not exist."
        elif [[ ! -f "$l_jmsenv" ]] ; then
            merror "The specified jmsenv file '$l_jmsenv' is not a regular file."
        fi
        jmsenv_ish_file="$l_jmsenv";
    fi

    # Handle if the file is in the current directory and not specifed with
    # an absolute or relative path (the '.' or 'source' command will not find
    # it since it searches for it on the PATH).  Prepend a "./" in this case.
    if [[ ! -z "$jmsenv_ish_file" ]] && [[ ! $jmsenv_ish_file =~ / ]] ; then
        jmsenv_ish_file="./$jmsenv_ish_file"
    fi

    echo "$jmsenv_ish_file";
}

jms_supported() {
    local jms=$1; shift;

    local i
    for i in $JMS_SUPPORTED; do
        if [[ $jms =~ ^${i}$ ]] ; then
            return 0;
        fi
    done

    return 1
}

get_jmstype() {
    local jmstype="";

    if [[ ! -z "$opt_jmstype" ]] ; then
        jmstype=$opt_jmstype;
    elif [[ ! -z "${JMS_TYPE+$JMS_TYPE}" ]] ; then
        # We should have already read the $jmsenv_ish_file in at this point
        jmstype=$JMS_TYPE;
    else
        local jmsenv_ish_file;
        jmsenv_ish_file=$(get_jmsenv_ish);

        if [[ ! -z "$jmsenv_ish_file" ]] ; then
            # Determine the jmstype from the jmsenv file
            jmstype=$(bash -c ". '$jmsenv_ish_file' > /dev/null 2>&1; echo \"\$JMS_TYPE\"")
            if [[ -z "$jmstype" ]] ; then
                merror "Could not determine jms type from jmsenv file.  Set JMS_TYPE in '$jmsenv_ish_file'"
            fi
            if ! jms_supported "$jmstype" ; then
                merror "Unrecognized JMS_TYPE '$jmstype' detected in '$jmsenv_ish_file'.  Supported JMS_TYPE: $(echo $JMS_SUPPORTED)"
            fi
        fi
    fi

    if [[ -z "$jmstype" ]] ; then
        merror "Could not determine jmstype.  Please specify --jmstype or set JMS_TYPE in --jmsenv file"
    fi

    echo "$jmstype";
}
get_cmdfunc() {
    local jmstype;
    jmstype=$(get_jmstype) || exit $?

    local cmdfunc;
    if $opt_start; then
        cmdfunc="${jmstype}_start";
    elif $opt_status; then
        cmdfunc="${jmstype}_status";
    elif $opt_stop; then
        cmdfunc="${jmstype}_stop";
    else
        minterror "get_cmdfunc(): expected opt_start, opt_status or opt_stop to be true"
    fi

    # Account for the 'OtherJMS' and 'CustomJMS' special cases, strip off the
    # leading text (before the '__'):
    if [[ $cmdfunc =~ ^(OtherJMS|CustomJMS)__(.*) ]] ; then
        cmdfunc=${BASH_REMATCH[2]}
    fi
    if ! declare -f -F $cmdfunc > /dev/null 2>&1; then
        minterror "get_cmdfunc(): Function '$cmdfunc' does not exist"
    fi

    echo "$cmdfunc"
}

set_userenv() {
    local jmsenv_ish_file;
    jmsenv_ish_file=$(get_jmsenv_ish);

    if [[ ! -z "$jmsenv_ish_file" ]] ; then
        . "$jmsenv_ish_file"
    fi
}

get_abspath_from_path() {
    local cmd=$1; shift;

    local retval=$cmd;

    # Try to find the command on the path if just a plain command (no absolute
    # or relative path).  If it is on the path, specify it directly instead
    # of using the path.  That way the command that is printed via the --noexec
    # or --verbose command line options will be cut-n-pastable to the command
    # line (using "PATH=/path/to:... /path/to/cmd" instead of
    # "PATH=/path/to:... cmd" which may not work on the command line if cmd is
    # not already in the path.
    if [[ ! $cmd =~ / ]] ; then
        local cmd_abspath;
        cmd_abspath=$(which "$cmd" 2> /dev/null || true);
        if [[ ! -z "$cmd_abspath" ]] ; then
            retval=$cmd_abspath;
        fi
    fi
    echo "$retval"
}

SGE_start() {
    local -a args=();

    local varname;
    local varnames="
        SGE_JOB_NAME
        SGE_STDOUT_FILE
        SGE_STDERR_FILE
        SGE_NPROC
        SGE_JOB_OPTSTR
        SGE_JOB_CMD

        SGE_START_CMD
        SGE_START_ARGS

        SGE_QUEUE
        SGE_PE
        SGE_SHELL_ARG
        SGE_SYNC_ARG
        SGE_EXPORTENV_ARG
    "

    local nvarname;
    for varname in $varnames; do
        # At this point, we expect the SGE_VARXYZ and VARXYZ variables
        # to have been set by the user in user.jmsenv.ish, the
        # JMSCONFIG_SGE_VARXYZ variables have been set by the installer
        # in jmsenv.ish, and the JOBVAR_VARXYZ variables have been set by
        # the job scheduler when invoking this job.  For the most part we
        # expect that the JMSCONFIG_SGE_* and JOBVAR_* will not conflict
        # with each other, and all of them are overridable by the user
        # settings.
        #
        # Order of priority:
        #   SGE_VARXYZ             (highest priority)
        #   VARXYZ
        #   special cases
        #   JMSCONFIG_SGE_VARXYZ
        #   JOBVAR_VARXYZ          (lowest priority)

        # Assign SGE_VARXYZ=$VARXYZ (if VARXYZ is set and SGE_VARXYZ
        # is not)
        [[ ! -z "${!varname+set}" ]] && continue;
        nvarname=${varname#SGE_};
        [[ ! -z "${!nvarname+set}" ]] && eval "${varname}='${!nvarname}'";


        # Special case (legacy).  Assign SGE_START_ARGS=$EXTRAS (if
        # EXTRAS is set and SGE_START_ARGS is not)
        [[ ! -z "${!varname+set}" ]] && continue;
        if [[ "$varname" == "SGE_START_ARGS" ]]; then
            nvarname="EXTRAS";
            [[ ! -z "${!nvarname+set}" ]] && eval "${varname}='${!nvarname}'";
        fi

        # Assign SGE_VARXYZ=$JMSCONFIG_SGE_VARXYZ (if JMSCONFIG_SGE_VARXYZ
        # is set and SGE_VARXYZ is not)
        [[ ! -z "${!varname+set}" ]] && continue;
        nvarname="JMSCONFIG_${varname}";
        [[ ! -z "${!nvarname+set}" ]] && eval "${varname}='${!nvarname}'";

        # Assign SGE_VARXYZ=$JOBVAR_VARXYZ (if JOBVAR_VARXYZ is set
        # and SGE_VARXYZ is not)
        [[ ! -z "${!varname+set}" ]] && continue;
        nvarname="JOBVAR_${varname#SGE_}";
        [[ ! -z "${!nvarname+set}" ]] && eval "${varname}='${!nvarname}'";
    done

    # Set defaults
    [[ -z "${SGE_START_CMD+set}" ]]     && SGE_START_CMD="qsub"
    [[ -z "${SGE_SHELL_ARG+set}" ]]     && SGE_SHELL_ARG="/bin/bash"
    [[ -z "${SGE_SYNC_ARG+set}" ]]      && SGE_SYNC_ARG=""     # y
    [[ -z "${SGE_EXPORTENV_ARG+set}" ]] && SGE_EXPORTENV_ARG=""

    # Build the list command arguments
    [[ ! -z "${SGE_SHELL_ARG:-}" ]]     && args+=( -S "$SGE_SHELL_ARG" )
    [[ ! -z "${SGE_SYNC_ARG:-}" ]]      && args+=( -sync "$SGE_SYNC_ARG" )
    [[ ! -z "${SGE_EXPORTENV_ARG:-}" ]] && args+=( "$SGE_EXPORTENV_ARG" )
    [[ ! -z "${SGE_QUEUE:-}" ]]         && args+=( -q "$SGE_QUEUE" )
    [[ ! -z "${SGE_JOB_NAME:-}" ]]      && args+=( -N "$SGE_JOB_NAME" )
    [[ ! -z "${SGE_STDOUT_FILE:-}" ]]   && args+=( -o "$SGE_STDOUT_FILE" )
    [[ ! -z "${SGE_STDERR_FILE:-}" ]]   && args+=( -e "$SGE_STDERR_FILE" )
    [[ ! -z "${SGE_PE:-}" ]] && [[ ! -z "${SGE_NPROC:-}" ]] \
                                        && args+=( -pe "$SGE_PE" "$SGE_NPROC" )
    [[ ! -z "${SGE_START_ARGS:-}" ]]    && eval args+=( "$SGE_START_ARGS" )
    [[ ! -z "${SGE_JOB_CMD:-}" ]]       && args+=( "$SGE_JOB_CMD" )

    # %C weirdness (probably %S, too)
    # https://lists.sdsc.edu/pipermail/npaci-rocks-discussion/2017-March/070126.html
    SGE_JOB_CMD=${SGE_JOB_CMD//%/%\${xx:+\}}
    # Get the abspath to command from PATH (see comments in function)
    SGE_START_CMD=$(get_abspath_from_path "$SGE_START_CMD")

    # FIXME: Hack for cromwell support, since we don't set the up the
    #        environment we need when running on the cluster nodes or call
    #        the smrttool exes with the full path (since they should all set
    #        the environment they need)
    if [[ -f "$SGE_JOB_CMD" ]]; then
        if  ! $opt_noexec; then
            # Add smrttools path, if needed
            local newpathline;
            local line2;
            newpathline="PATH=\"$g_progdir_abs/../../bundles/smrttools/current/private/otherbins/all/bin:$PATH\""
            line2=$(sed -ne '2p' "$SGE_JOB_CMD")
            if [[ x"$line2" != x"$newpathline" ]] ; then
                sed -i -e "1a $newpathline" "$SGE_JOB_CMD"
            fi

            # Some variants of SGE may require the script to be executable
            chmod +x "$SGE_JOB_CMD"
        fi
    fi

    # Determine exported env vars so we can specify them on the command
    # line (so verbose and noexec modes will print out a line the can be
    # cut-n-pasted to duplicate the command, including environment
    # settings)
    local exported_vars;
    exported_vars=$(export -p | sed -ne 's/^\(declare[[:space:]]\+-x\|export\)[[:space:]]\+\([^=]\+\).*/\2/p')

    local -a envvars=();
    local envvar
    for envvar in $exported_vars; do
        if [[ -z "${!envvar+set}" ]] ; then
            # Ignore environment variables that are exported, but not set
            # (exported but unset env vars are not passed to subprocesses)
            continue;
        fi
        envvars+=( "$envvar='${!envvar}'" )
    done

    runcmd "${envvars[@]+${envvars[@]}}" $SGE_START_CMD "${args[@]+${args[@]}}"
}

SGE_stop() {
    local -a args=();

    local varname;
    local varnames="
        SGE_JOB_ID

        SGE_STOP_CMD
        SGE_STOP_ARGS
    "

    local nvarname;
    for varname in $varnames; do
        # At this point, we expect the SGE_VARXYZ and VARXYZ variables
        # to have been set by the user in user.jmsenv.ish, the
        # JMSCONFIG_SGE_VARXYZ variables have been set by the installer
        # in jmsenv.ish, and the JOBVAR_VARXYZ variables have been set by
        # the job scheduler when invoking this job.  For the most part we
        # expect that the JMSCONFIG_SGE_* and JOBVAR_* will not conflict
        # with each other, and all of them are overridable by the user
        # settings.
        #
        # Order of priority:
        #   SGE_VARXYZ             (highest priority)
        #   VARXYZ
        #   special cases
        #   JMSCONFIG_SGE_VARXYZ
        #   JOBVAR_VARXYZ          (lowest priority)

        # Assign SGE_VARXYZ=$VARXYZ (if VARXYZ is set and SGE_VARXYZ
        # is not)
        [[ ! -z "${!varname+set}" ]] && continue;
        nvarname=${varname#SGE_};
        [[ ! -z "${!nvarname+set}" ]] && eval "${varname}='${!nvarname}'";

        # Assign SGE_VARXYZ=$JMSCONFIG_SGE_VARXYZ (if JMSCONFIG_SGE_VARXYZ
        # is set and SGE_VARXYZ is not)
        [[ ! -z "${!varname+set}" ]] && continue;
        nvarname="JMSCONFIG_${varname}";
        [[ ! -z "${!nvarname+set}" ]] && eval "${varname}='${!nvarname}'";

        # Assign SGE_VARXYZ=$JOBVAR_VARXYZ (if JOBVAR_VARXYZ is set
        # and SGE_VARXYZ is not)
        [[ ! -z "${!varname+set}" ]] && continue;
        nvarname="JOBVAR_${varname#SGE_}";
        [[ ! -z "${!nvarname+set}" ]] && eval "${varname}='${!nvarname}'";
    done


    # Set defaults
    [[ -z "${SGE_STOP_CMD+set}" ]]    && SGE_STOP_CMD="qdel"

    # Build the list command arguments
    [[ ! -z "${SGE_JOB_ID:-}" ]]      && args+=( "$SGE_JOB_ID" )
    [[ ! -z "${SGE_STOP_ARGS:-}" ]]   && eval args+=( "$SGE_STOP_ARGS" )

    # Get the abspath to command from PATH (see comments in function)
    SGE_STOP_CMD=$(get_abspath_from_path "$SGE_STOP_CMD")

    # Determine exported env vars so we can specify them on the command
    # line (so verbose and noexec modes will print out a line the can be
    # cut-n-pasted to duplicate the command, including environment
    # settings)
    local exported_vars;
    exported_vars=$(export -p | sed -ne 's/^\(declare[[:space:]]\+-x\|export\)[[:space:]]\+\([^=]\+\).*/\2/p')

    local -a envvars=();
    local envvar
    for envvar in $exported_vars; do
        if [[ -z "${!envvar+set}" ]] ; then
            # Ignore environment variables that are exported, but not set
            # (exported but unset env vars are not passed to subprocesses)
            continue;
        fi
        envvars+=( "$envvar='${!envvar}'" )
    done

    runcmd "${envvars[@]+${envvars[@]}}" $SGE_STOP_CMD "${args[@]+${args[@]}}"
}

SGE_status() {
    local -a args=();

    local varname;
    local varnames="
        SGE_JOB_ID

        SGE_STATUS_CMD
        SGE_STATUS_ARGS
    "

    local nvarname;
    for varname in $varnames; do
        # At this point, we expect the SGE_VARXYZ and VARXYZ variables
        # to have been set by the user in user.jmsenv.ish, the
        # JMSCONFIG_SGE_VARXYZ variables have been set by the installer
        # in jmsenv.ish, and the JOBVAR_VARXYZ variables have been set by
        # the job scheduler when invoking this job.  For the most part we
        # expect that the JMSCONFIG_SGE_* and JOBVAR_* will not conflict
        # with each other, and all of them are overridable by the user
        # settings.
        #
        # Order of priority:
        #   SGE_VARXYZ             (highest priority)
        #   VARXYZ
        #   special cases
        #   JMSCONFIG_SGE_VARXYZ
        #   JOBVAR_VARXYZ          (lowest priority)

        # Assign SGE_VARXYZ=$VARXYZ (if VARXYZ is set and SGE_VARXYZ
        # is not)
        [[ ! -z "${!varname+set}" ]] && continue;
        nvarname=${varname#SGE_};
        [[ ! -z "${!nvarname+set}" ]] && eval "${varname}='${!nvarname}'";

        # Assign SGE_VARXYZ=$JMSCONFIG_SGE_VARXYZ (if JMSCONFIG_SGE_VARXYZ
        # is set and SGE_VARXYZ is not)
        [[ ! -z "${!varname+set}" ]] && continue;
        nvarname="JMSCONFIG_${varname}";
        [[ ! -z "${!nvarname+set}" ]] && eval "${varname}='${!nvarname}'";

        # Assign SGE_VARXYZ=$JOBVAR_VARXYZ (if JOBVAR_VARXYZ is set
        # and SGE_VARXYZ is not)
        [[ ! -z "${!varname+set}" ]] && continue;
        nvarname="JOBVAR_${varname#SGE_}";
        [[ ! -z "${!nvarname+set}" ]] && eval "${varname}='${!nvarname}'";
    done


    # Set defaults
    [[ -z "${SGE_STATUS_CMD+set}" ]]    && SGE_STATUS_CMD="qstat"

    # Build the list command arguments
    [[ ! -z "${SGE_JOB_ID:-}" ]]      && args+=( -j "$SGE_JOB_ID" )
    [[ ! -z "${SGE_STATUS_ARGS:-}" ]]   && eval args+=( "$SGE_STATUS_ARGS" )

    # Get the abspath to command from PATH (see comments in function)
    SGE_STATUS_CMD=$(get_abspath_from_path "$SGE_STATUS_CMD")

    # Determine exported env vars so we can specify them on the command
    # line (so verbose and noexec modes will print out a line the can be
    # cut-n-pasted to duplicate the command, including environment
    # settings)
    local exported_vars;
    exported_vars=$(export -p | sed -ne 's/^\(declare[[:space:]]\+-x\|export\)[[:space:]]\+\([^=]\+\).*/\2/p')

    local -a envvars=();
    local envvar
    for envvar in $exported_vars; do
        if [[ -z "${!envvar+set}" ]] ; then
            # Ignore environment variables that are exported, but not set
            # (exported but unset env vars are not passed to subprocesses)
            continue;
        fi
        envvars+=( "$envvar='${!envvar}'" )
    done

    runcmd "${envvars[@]+${envvars[@]}}" $SGE_STATUS_CMD "${args[@]+${args[@]}}"
}


# Assuming OGS is compatible with SGE for now
OGS_start()  {
    local varname;
    for varname in ${!OGS_*} ${!JMSCONFIG_OGS_*}; do
        if [[ ! -z "${!varname+set}" ]] ; then
            eval "${varname/OGS_/SGE_}='${!varname}'"
        fi
    done

    SGE_start ${1+"$@"};
}
OGS_stop()   {
    local varname;
    for varname in ${!OGS_*} ${!JMSCONFIG_OGS_*}; do
        if [[ ! -z "${!varname+set}" ]] ; then
            eval "${varname/OGS_/SGE_}='${!varname}'"
        fi
    done

    SGE_stop ${1+"$@"};
}
OGS_status() {
    local varname;
    for varname in ${!OGS_*} ${!JMSCONFIG_OGS_*}; do
        if [[ ! -z "${!varname+set}" ]] ; then
            eval "${varname/OGS_/SGE_}='${!varname}'"
        fi
    done

    SGE_status ${1+"$@"};
}

# Assuming UGE is compatible with SGE for now
UGE_start()  {
    local varname;
    for varname in ${!UGE_*} ${!JMSCONFIG_UGE_*}; do
        if [[ ! -z "${!varname+set}" ]] ; then
            eval "${varname/UGE_/SGE_}='${!varname}'"
        fi
    done

    SGE_start ${1+"$@"};
}
UGE_stop()   {
    local varname;
    for varname in ${!UGE_*} ${!JMSCONFIG_UGE_*}; do
        if [[ ! -z "${!varname+set}" ]] ; then
            eval "${varname/UGE_/SGE_}='${!varname}'"
        fi
    done

    SGE_stop ${1+"$@"};
}
UGE_status() {
    local varname;
    for varname in ${!UGE_*} ${!JMSCONFIG_UGE_*}; do
        if [[ ! -z "${!varname+set}" ]] ; then
            eval "${varname/UGE_/SGE_}='${!varname}'"
        fi
    done

    SGE_status ${1+"$@"};
}

LSF_start() {
    local -a args=();

    local varname;
    local varnames="
        LSF_JOB_NAME
        LSF_STDOUT_FILE
        LSF_STDERR_FILE
        LSF_NPROC
        LSF_JOB_OPTSTR
        LSF_JOB_CMD

        LSF_START_CMD
        LSF_START_ARGS

        LSF_QUEUE
        LSF_WAIT_ARG
        LSF_RESOURCE_ARG
    "

    local nvarname;
    for varname in $varnames; do
        # At this point, we expect the LSF_VARXYZ and VARXYZ variables
        # to have been set by the user in user.jmsenv.ish, the
        # JMSCONFIG_LSF_VARXYZ variables have been set by the installer
        # in jmsenv.ish, and the JOBVAR_VARXYZ variables have been set by
        # the job scheduler when invoking this job.  For the most part we
        # expect that the JMSCONFIG_LSF_* and JOBVAR_* will not conflict
        # with each other, and all of them are overridable by the user
        # settings.
        #
        # Order of priority:
        #   LSF_VARXYZ             (highest priority)
        #   VARXYZ
        #   special cases
        #   JMSCONFIG_LSF_VARXYZ
        #   JOBVAR_VARXYZ          (lowest priority)

        # Assign LSF_VARXYZ=$VARXYZ (if VARXYZ is set and LSF_VARXYZ
        # is not)
        [[ ! -z "${!varname+set}" ]] && continue;
        nvarname=${varname#LSF_};
        [[ ! -z "${!nvarname+set}" ]] && eval "${varname}='${!nvarname}'";


        # Special case (legacy).  Assign LSF_START_ARGS=$EXTRAS (if
        # EXTRAS is set and LSF_START_ARGS is not)
        [[ ! -z "${!varname+set}" ]] && continue;
        if [[ "$varname" == "LSF_START_ARGS" ]]; then
            nvarname="EXTRAS";
            [[ ! -z "${!nvarname+set}" ]] && eval "${varname}='${!nvarname}'";
        fi

        # Assign LSF_VARXYZ=$JMSCONFIG_LSF_VARXYZ (if JMSCONFIG_LSF_VARXYZ
        # is set and LSF_VARXYZ is not)
        [[ ! -z "${!varname+set}" ]] && continue;
        nvarname="JMSCONFIG_${varname}";
        [[ ! -z "${!nvarname+set}" ]] && eval "${varname}='${!nvarname}'";

        # Assign LSF_VARXYZ=$JOBVAR_VARXYZ (if JOBVAR_VARXYZ is set
        # and LSF_VARXYZ is not)
        [[ ! -z "${!varname+set}" ]] && continue;
        nvarname="JOBVAR_${varname#LSF_}";
        [[ ! -z "${!nvarname+set}" ]] && eval "${varname}='${!nvarname}'";

    done

    # Set defaults
    [[ -z "${LSF_START_CMD+set}" ]]    && LSF_START_CMD="bsub"
    [[ -z "${LSF_WAIT_ARG+set}" ]]     && LSF_WAIT_ARG=""    # -K
    [[ -z "${LSF_RESOURCE_ARG+set}" ]] && LSF_RESOURCE_ARG="span[hosts=1]"

    # Build the list command arguments
    [[ ! -z "${LSF_WAIT_ARG:-}" ]]      && args+=( "$LSF_WAIT_ARG" )
    [[ ! -z "${LSF_JOB_NAME:-}" ]]      && args+=( -J "$LSF_JOB_NAME" )
    [[ ! -z "${LSF_STDOUT_FILE:-}" ]]   && args+=( -o "$LSF_STDOUT_FILE" )
    [[ ! -z "${LSF_STDERR_FILE:-}" ]]   && args+=( -e "$LSF_STDERR_FILE" )
    [[ ! -z "${LSF_NPROC:-}" ]]         && args+=( -n "$LSF_NPROC" )
    [[ ! -z "${LSF_QUEUE:-}" ]]         && args+=( -q "$LSF_QUEUE" )
    [[ ! -z "${LSF_RESOURCE_ARG:-}" ]]  && args+=( -R "$LSF_RESOURCE_ARG" )
    [[ ! -z "${LSF_START_ARGS:-}" ]]    && eval args+=( "$LSF_START_ARGS" )
    [[ ! -z "${LSF_JOB_CMD:-}" ]]       && eval args+=( "$LSF_JOB_CMD" )

    # Get the abspath to command from PATH (see comments in function)
    LSF_START_CMD=$(get_abspath_from_path "$LSF_START_CMD")

    # FIXME: Hack for cromwell support, since we don't set the up the
    #        environment we need when running on the cluster nodes or call
    #        the smrttool exes with the full path (since they should all set
    #        the environment they need)
    if [[ -f "$LSF_JOB_CMD" ]]; then
        if  ! $opt_noexec; then
            # Add smrttools path, if needed
            local newpathline;
            local line2;
            newpathline="PATH=\"$g_progdir_abs/../../bundles/smrttools/current/private/otherbins/all/bin:$PATH\""
            line2=$(sed -ne '2p' "$LSF_JOB_CMD")
            if [[ x"$line2" != x"$newpathline" ]] ; then
                sed -i -e "1a $newpathline" "$LSF_JOB_CMD"
            fi

            # LSF requires the command to be executable, OpenLava may not
            chmod +x "$LSF_JOB_CMD"
        fi
    fi

    # Determine exported env vars so we can specify them on the command
    # line (so verbose and noexec modes will print out a line the can be
    # cut-n-pasted to duplicate the command, including environment
    # settings)
    local exported_vars;
    exported_vars=$(export -p | sed -ne 's/^\(declare[[:space:]]\+-x\|export\)[[:space:]]\+\([^=]\+\).*/\2/p')

    local -a envvars=();
    local envvar
    for envvar in $exported_vars; do
        if [[ -z "${!envvar+set}" ]] ; then
            # Ignore environment variables that are exported, but not set
            # (exported but unset env vars are not passed to subprocesses)
            continue;
        fi
        envvars+=( "$envvar='${!envvar}'" )
    done

    runcmd "${envvars[@]+${envvars[@]}}" $LSF_START_CMD "${args[@]+${args[@]}}"
}

LSF_stop() {
    local -a args=();

    local varname;
    local varnames="
        LSF_JOB_ID

        LSF_STOP_CMD
        LSF_STOP_ARGS
    "

    local nvarname;
    for varname in $varnames; do
        # At this point, we expect the LSF_VARXYZ and VARXYZ variables
        # to have been set by the user in user.jmsenv.ish, the
        # JMSCONFIG_LSF_VARXYZ variables have been set by the installer
        # in jmsenv.ish, and the JOBVAR_VARXYZ variables have been set by
        # the job scheduler when invoking this job.  For the most part we
        # expect that the JMSCONFIG_LSF_* and JOBVAR_* will not conflict
        # with each other, and all of them are overridable by the user
        # settings.
        #
        # Order of priority:
        #   LSF_VARXYZ             (highest priority)
        #   VARXYZ
        #   special cases
        #   JMSCONFIG_LSF_VARXYZ
        #   JOBVAR_VARXYZ          (lowest priority)

        # Assign LSF_VARXYZ=$VARXYZ (if VARXYZ is set and LSF_VARXYZ
        # is not)
        [[ ! -z "${!varname+set}" ]] && continue;
        nvarname=${varname#LSF_};
        [[ ! -z "${!nvarname+set}" ]] && eval "${varname}='${!nvarname}'";

        # Assign LSF_VARXYZ=$JMSCONFIG_LSF_VARXYZ (if JMSCONFIG_LSF_VARXYZ
        # is set and LSF_VARXYZ is not)
        [[ ! -z "${!varname+set}" ]] && continue;
        nvarname="JMSCONFIG_${varname}";
        [[ ! -z "${!nvarname+set}" ]] && eval "${varname}='${!nvarname}'";

        # Assign LSF_VARXYZ=$JOBVAR_VARXYZ (if JOBVAR_VARXYZ is set
        # and LSF_VARXYZ is not)
        [[ ! -z "${!varname+set}" ]] && continue;
        nvarname="JOBVAR_${varname#LSF_}";
        [[ ! -z "${!nvarname+set}" ]] && eval "${varname}='${!nvarname}'";
    done


    # Set defaults
    [[ -z "${LSF_STOP_CMD+set}" ]]    && LSF_STOP_CMD="bkill"


    # Build the list command arguments
    [[ ! -z "${LSF_JOB_ID:-}" ]]      && args+=( -J "$LSF_JOB_ID" )
    [[ ! -z "${LSF_STOP_ARGS:-}" ]]    && eval args+=( "$LSF_STOP_ARGS" )

    # Get the abspath to command from PATH (see comments in function)
    LSF_STOP_CMD=$(get_abspath_from_path "$LSF_STOP_CMD")

    # Determine exported env vars so we can specify them on the command
    # line (so verbose and noexec modes will print out a line the can be
    # cut-n-pasted to duplicate the command, including environment
    # settings)
    local exported_vars;
    exported_vars=$(export -p | sed -ne 's/^\(declare[[:space:]]\+-x\|export\)[[:space:]]\+\([^=]\+\).*/\2/p')

    local -a envvars=();
    local envvar
    for envvar in $exported_vars; do
        if [[ -z "${!envvar+set}" ]] ; then
            # Ignore environment variables that are exported, but not set
            # (exported but unset env vars are not passed to subprocesses)
            continue;
        fi
        envvars+=( "$envvar='${!envvar}'" )
    done

    runcmd "${envvars[@]+${envvars[@]}}" $LSF_STOP_CMD "${args[@]+${args[@]}}"
}

LSF_status() {
    local -a args=();

    local varname;
    local varnames="
        LSF_JOB_ID

        LSF_STATUS_CMD
        LSF_STATUS_ARGS
    "

    local nvarname;
    for varname in $varnames; do
        # At this point, we expect the LSF_VARXYZ and VARXYZ variables
        # to have been set by the user in user.jmsenv.ish, the
        # JMSCONFIG_LSF_VARXYZ variables have been set by the installer
        # in jmsenv.ish, and the JOBVAR_VARXYZ variables have been set by
        # the job scheduler when invoking this job.  For the most part we
        # expect that the JMSCONFIG_LSF_* and JOBVAR_* will not conflict
        # with each other, and all of them are overridable by the user
        # settings.
        #
        # Order of priority:
        #   LSF_VARXYZ             (highest priority)
        #   VARXYZ
        #   special cases
        #   JMSCONFIG_LSF_VARXYZ
        #   JOBVAR_VARXYZ          (lowest priority)

        # Assign LSF_VARXYZ=$VARXYZ (if VARXYZ is set and LSF_VARXYZ
        # is not)
        [[ ! -z "${!varname+set}" ]] && continue;
        nvarname=${varname#LSF_};
        [[ ! -z "${!nvarname+set}" ]] && eval "${varname}='${!nvarname}'";

        # Assign LSF_VARXYZ=$JMSCONFIG_LSF_VARXYZ (if JMSCONFIG_LSF_VARXYZ
        # is set and LSF_VARXYZ is not)
        [[ ! -z "${!varname+set}" ]] && continue;
        nvarname="JMSCONFIG_${varname}";
        [[ ! -z "${!nvarname+set}" ]] && eval "${varname}='${!nvarname}'";

        # Assign LSF_VARXYZ=$JOBVAR_VARXYZ (if JOBVAR_VARXYZ is set
        # and LSF_VARXYZ is not)
        [[ ! -z "${!varname+set}" ]] && continue;
        nvarname="JOBVAR_${varname#LSF_}";
        [[ ! -z "${!nvarname+set}" ]] && eval "${varname}='${!nvarname}'";
    done


    # Set defaults
    [[ -z "${LSF_STATUS_CMD+set}" ]]    && LSF_STATUS_CMD="bjobs"

    # Build the list command arguments
    [[ ! -z "${LSF_JOB_ID:-}" ]]      && args+=( -J "$LSF_JOB_ID" )
    [[ ! -z "${LSF_STATUS_ARGS:-}" ]]   && eval args+=( "$LSF_STATUS_ARGS" )

    # Get the abspath to command from PATH (see comments in function)
    LSF_STATUS_CMD=$(get_abspath_from_path "$LSF_STATUS_CMD")

    # Determine exported env vars so we can specify them on the command
    # line (so verbose and noexec modes will print out a line the can be
    # cut-n-pasted to duplicate the command, including environment
    # settings)
    local exported_vars;
    exported_vars=$(export -p | sed -ne 's/^\(declare[[:space:]]\+-x\|export\)[[:space:]]\+\([^=]\+\).*/\2/p')

    local -a envvars=();
    local envvar
    for envvar in $exported_vars; do
        if [[ -z "${!envvar+set}" ]] ; then
            # Ignore environment variables that are exported, but not set
            # (exported but unset env vars are not passed to subprocesses)
            continue;
        fi
        envvars+=( "$envvar='${!envvar}'" )
    done

    runcmd "${envvars[@]+${envvars[@]}}" $LSF_STATUS_CMD "${args[@]+${args[@]}}"
}

# Assuming OpenLava is compatible with LSF for now
OpenLava_start() {
    local varname;
    for varname in ${!OPENLAVA_*} ${!JMSCONFIG_OPENLAVA_*}; do
        if [[ ! -z "${!varname+set}" ]] ; then
            eval "${varname/OPENLAVA_/LSF_}='${!varname}'"
        fi
    done

    LSF_start ${1+"$@"};
}
OpenLava_stop() {
    local varname;
    for varname in ${!OPENLAVA_*} ${!JMSCONFIG_OPENLAVA_*}; do
        if [[ ! -z "${!varname+set}" ]] ; then
            eval "${varname/OPENLAVA_/LSF_}='${!varname}'"
        fi
    done

    LSF_stop ${1+"$@"};
}
OpenLava_status() {
    local varname;
    for varname in ${!OPENLAVA_*} ${!JMSCONFIG_OPENLAVA_*}; do
        if [[ ! -z "${!varname+set}" ]] ; then
            eval "${varname/OPENLAVA_/LSF_}='${!varname}'"
        fi
    done

    LSF_status ${1+"$@"};
}

PBS_start() {
    local -a args=();

    local varname;
    local varnames="
        PBS_JOB_NAME
        PBS_STDOUT_FILE
        PBS_STDERR_FILE
        PBS_NPROC
        PBS_JOB_OPTSTR
        PBS_JOB_CMD

        PBS_START_CMD
        PBS_START_ARGS

        PBS_QUEUE
        PBS_SHELL_ARG
        PBS_EXPORTENV_ARG
        PBS_QSW_PBS_ARG
    "


    local nvarname;
    for varname in $varnames; do
        # At this point, we expect the PBS_VARXYZ and VARXYZ variables
        # to have been set by the user in user.jmsenv.ish, the
        # JMSCONFIG_PBS_VARXYZ variables have been set by the installer
        # in jmsenv.ish, and the JOBVAR_VARXYZ variables have been set by
        # the job scheduler when invoking this job.  For the most part we
        # expect that the JMSCONFIG_PBS_* and JOBVAR_* will not conflict
        # with each other, and all of them are overridable by the user
        # settings.
        #
        # Order of priority:
        #   PBS_VARXYZ             (highest priority)
        #   VARXYZ
        #   special cases
        #   JMSCONFIG_PBS_VARXYZ
        #   JOBVAR_VARXYZ          (lowest priority)

        # Assign PBS_VARXYZ=$VARXYZ (if VARXYZ is set and PBS_VARXYZ
        # is not)
        [[ ! -z "${!varname+set}" ]] && continue;
        nvarname=${varname#PBS_};
        [[ ! -z "${!nvarname+set}" ]] && eval "${varname}='${!nvarname}'";


        # Special case (legacy).  Assign PBS_START_ARGS=$EXTRAS (if
        # EXTRAS is set and PBS_START_ARGS is not)
        [[ ! -z "${!varname+set}" ]] && continue;
        if [[ "$varname" == "PBS_START_ARGS" ]]; then
            nvarname="EXTRAS";
            [[ ! -z "${!nvarname+set}" ]] && eval "${varname}='${!nvarname}'";
        fi

        # Assign PBS_VARXYZ=$JMSCONFIG_PBS_VARXYZ (if JMSCONFIG_PBS_VARXYZ
        # is set and PBS_VARXYZ is not)
        [[ ! -z "${!varname+set}" ]] && continue;
        nvarname="JMSCONFIG_${varname}";
        [[ ! -z "${!nvarname+set}" ]] && eval "${varname}='${!nvarname}'";

        # Assign PBS_VARXYZ=$JOBVAR_VARXYZ (if JOBVAR_VARXYZ is set
        # and PBS_VARXYZ is not)
        [[ ! -z "${!varname+set}" ]] && continue;
        nvarname="JOBVAR_${varname#PBS_}";
        [[ ! -z "${!nvarname+set}" ]] && eval "${varname}='${!nvarname}'";
    done

    # Set defaults
    [[ -z "${PBS_START_CMD+set}" ]]     && PBS_START_CMD="qsub"
    [[ -z "${PBS_SHELL_ARG+set}" ]]     && PBS_SHELL_ARG="/bin/bash"
    [[ -z "${PBS_EXPORTENV_ARG+set}" ]] && PBS_EXPORTENV_ARG="-V"

    # set the -PBS flag if we are using qsw (used by qsw)
    # With the use of cromwell, PBS_START_CMD should no longer be set to 'qsw'
    if [[ -z "${PBS_QSW_PBS_ARG+set}" ]]; then
        if  [[ $PBS_START_CMD =~ /qsw$ ]] ||
            [[ $PBS_START_CMD =~ ^qsw$ ]] ; then
            PBS_QSW_PBS_ARG="-PBS"
        fi
    fi

    # Build the list command arguments
    [[ ! -z "${PBS_JOB_CMD:-}" ]]       && args+=( "$PBS_JOB_CMD" )
    [[ ! -z "${PBS_SHELL_ARG:-}" ]]     && args+=( -S "$PBS_SHELL_ARG" )
    [[ ! -z "${PBS_EXPORTENV_ARG:-}" ]] && args+=( "$PBS_EXPORTENV_ARG" )
    [[ ! -z "${PBS_QUEUE:-}" ]]         && args+=( -q "$PBS_QUEUE" )
    [[ ! -z "${PBS_JOB_NAME:-}" ]]      && args+=( -N "$PBS_JOB_NAME" )
    [[ ! -z "${PBS_STDOUT_FILE:-}" ]]   && args+=( -o "$PBS_STDOUT_FILE" )
    [[ ! -z "${PBS_STDERR_FILE:-}" ]]   && args+=( -e "$PBS_STDERR_FILE" )
    [[ ! -z "${PBS_START_ARGS:-}" ]]    && eval args+=( "$PBS_START_ARGS" )
    [[ ! -z "${PBS_NPROC:-}" ]]      && args+=( -l "nodes=1:ppn=${PBS_NPROC}" )
    [[ ! -z "${PBS_QSW_PBS_ARG:-}" ]]   && args+=( "$PBS_QSW_PBS_ARG" )


    # Get the abspath to command from PATH (see comments in function)
    PBS_START_CMD=$(get_abspath_from_path "$PBS_START_CMD")

    # FIXME: Hack for cromwell support, since we don't set the up the
    #        environment we need when running on the cluster nodes or call
    #        the smrttool exes with the full path (since they should all set
    #        the environment they need)
    if [[ -f "$PBS_JOB_CMD" ]]; then
        if  ! $opt_noexec; then
            # Add smrttools path, if needed
            local newpathline;
            local line2;
            newpathline="PATH=\"$g_progdir_abs/../../bundles/smrttools/current/private/otherbins/all/bin:$PATH\""
            line2=$(sed -ne '2p' "$PBS_JOB_CMD")
            if [[ x"$line2" != x"$newpathline" ]] ; then
                sed -i -e "1a $newpathline" "$PBS_JOB_CMD"
            fi

            # In case some variants of PBS require the command to be executable
            chmod +x "$PBS_JOB_CMD"
        fi
    fi

    # Determine exported env vars so we can specify them on the command
    # line (so verbose and noexec modes will print out a line the can be
    # cut-n-pasted to duplicate the command, including environment
    # settings)
    local exported_vars;
    exported_vars=$(export -p | sed -ne 's/^\(declare[[:space:]]\+-x\|export\)[[:space:]]\+\([^=]\+\).*/\2/p')

    local -a envvars=();
    local envvar
    for envvar in $exported_vars; do
        if [[ -z "${!envvar+set}" ]] ; then
            # Ignore environment variables that are exported, but not set
            # (exported but unset env vars are not passed to subprocesses)
            continue;
        fi
        envvars+=( "$envvar='${!envvar}'" )
    done

    runcmd "${envvars[@]+${envvars[@]}}" $PBS_START_CMD "${args[@]+${args[@]}}"
}

PBS_stop() {
    local -a args=();

    local varname;
    local varnames="
        PBS_JOB_ID

        PBS_STOP_CMD
        PBS_STOP_ARGS
    "

    local nvarname;
    for varname in $varnames; do
        # At this point, we expect the PBS_VARXYZ and VARXYZ variables
        # to have been set by the user in user.jmsenv.ish, the
        # JMSCONFIG_PBS_VARXYZ variables have been set by the installer
        # in jmsenv.ish, and the JOBVAR_VARXYZ variables have been set by
        # the job scheduler when invoking this job.  For the most part we
        # expect that the JMSCONFIG_PBS_* and JOBVAR_* will not conflict
        # with each other, and all of them are overridable by the user
        # settings.
        #
        # Order of priority:
        #   PBS_VARXYZ             (highest priority)
        #   VARXYZ
        #   special cases
        #   JMSCONFIG_PBS_VARXYZ
        #   JOBVAR_VARXYZ          (lowest priority)

        # Assign PBS_VARXYZ=$VARXYZ (if VARXYZ is set and PBS_VARXYZ
        # is not)
        [[ ! -z "${!varname+set}" ]] && continue;
        nvarname=${varname#PBS_};
        [[ ! -z "${!nvarname+set}" ]] && eval "${varname}='${!nvarname}'";

        # Assign PBS_VARXYZ=$JMSCONFIG_PBS_VARXYZ (if JMSCONFIG_PBS_VARXYZ
        # is set and PBS_VARXYZ is not)
        [[ ! -z "${!varname+set}" ]] && continue;
        nvarname="JMSCONFIG_${varname}";
        [[ ! -z "${!nvarname+set}" ]] && eval "${varname}='${!nvarname}'";

        # Assign PBS_VARXYZ=$JOBVAR_VARXYZ (if JOBVAR_VARXYZ is set
        # and PBS_VARXYZ is not)
        [[ ! -z "${!varname+set}" ]] && continue;
        nvarname="JOBVAR_${varname#PBS_}";
        [[ ! -z "${!nvarname+set}" ]] && eval "${varname}='${!nvarname}'";
    done


    # Set defaults
    [[ -z "${PBS_STOP_CMD+set}" ]]    && PBS_STOP_CMD="qdel"

    # Build the list command arguments
    [[ ! -z "${PBS_JOB_ID:-}" ]]      && args+=( "$PBS_JOB_ID" )
    [[ ! -z "${PBS_STOP_ARGS:-}" ]]   && eval args+=( "$PBS_STOP_ARGS" )

    # Get the abspath to command from PATH (see comments in function)
    PBS_STOP_CMD=$(get_abspath_from_path "$PBS_STOP_CMD")

    # Determine exported env vars so we can specify them on the command
    # line (so verbose and noexec modes will print out a line the can be
    # cut-n-pasted to duplicate the command, including environment
    # settings)
    local exported_vars;
    exported_vars=$(export -p | sed -ne 's/^\(declare[[:space:]]\+-x\|export\)[[:space:]]\+\([^=]\+\).*/\2/p')

    local -a envvars=();
    local envvar
    for envvar in $exported_vars; do
        if [[ -z "${!envvar+set}" ]] ; then
            # Ignore environment variables that are exported, but not set
            # (exported but unset env vars are not passed to subprocesses)
            continue;
        fi
        envvars+=( "$envvar='${!envvar}'" )
    done

    runcmd "${envvars[@]+${envvars[@]}}" $PBS_STOP_CMD "${args[@]+${args[@]}}"
}

PBS_status() {
    local -a args=();

    local varname;
    local varnames="
        PBS_JOB_ID

        PBS_STATUS_CMD
        PBS_STATUS_ARGS
    "

    local nvarname;
    for varname in $varnames; do
        # At this point, we expect the PBS_VARXYZ and VARXYZ variables
        # to have been set by the user in user.jmsenv.ish, the
        # JMSCONFIG_PBS_VARXYZ variables have been set by the installer
        # in jmsenv.ish, and the JOBVAR_VARXYZ variables have been set by
        # the job scheduler when invoking this job.  For the most part we
        # expect that the JMSCONFIG_PBS_* and JOBVAR_* will not conflict
        # with each other, and all of them are overridable by the user
        # settings.
        #
        # Order of priority:
        #   PBS_VARXYZ             (highest priority)
        #   VARXYZ
        #   special cases
        #   JMSCONFIG_PBS_VARXYZ
        #   JOBVAR_VARXYZ          (lowest priority)

        # Assign PBS_VARXYZ=$VARXYZ (if VARXYZ is set and PBS_VARXYZ
        # is not)
        [[ ! -z "${!varname+set}" ]] && continue;
        nvarname=${varname#PBS_};
        [[ ! -z "${!nvarname+set}" ]] && eval "${varname}='${!nvarname}'";

        # Assign PBS_VARXYZ=$JMSCONFIG_PBS_VARXYZ (if JMSCONFIG_PBS_VARXYZ
        # is set and PBS_VARXYZ is not)
        [[ ! -z "${!varname+set}" ]] && continue;
        nvarname="JMSCONFIG_${varname}";
        [[ ! -z "${!nvarname+set}" ]] && eval "${varname}='${!nvarname}'";

        # Assign PBS_VARXYZ=$JOBVAR_VARXYZ (if JOBVAR_VARXYZ is set
        # and PBS_VARXYZ is not)
        [[ ! -z "${!varname+set}" ]] && continue;
        nvarname="JOBVAR_${varname#PBS_}";
        [[ ! -z "${!nvarname+set}" ]] && eval "${varname}='${!nvarname}'";
    done


    # Set defaults
    [[ -z "${PBS_STATUS_CMD+set}" ]]    && PBS_STATUS_CMD="qstat"

    # Build the list command arguments
    [[ ! -z "${PBS_JOB_ID:-}" ]]      && args+=( "$PBS_JOB_ID" )
    [[ ! -z "${PBS_STATUS_ARGS:-}" ]]   && eval args+=( "$PBS_STATUS_ARGS" )

    # Get the abspath to command from PATH (see comments in function)
    PBS_STATUS_CMD=$(get_abspath_from_path "$PBS_STATUS_CMD")

    # Determine exported env vars so we can specify them on the command
    # line (so verbose and noexec modes will print out a line the can be
    # cut-n-pasted to duplicate the command, including environment
    # settings)
    local exported_vars;
    exported_vars=$(export -p | sed -ne 's/^\(declare[[:space:]]\+-x\|export\)[[:space:]]\+\([^=]\+\).*/\2/p')

    local -a envvars=();
    local envvar
    for envvar in $exported_vars; do
        if [[ -z "${!envvar+set}" ]] ; then
            # Ignore environment variables that are exported, but not set
            # (exported but unset env vars are not passed to subprocesses)
            continue;
        fi
        envvars+=( "$envvar='${!envvar}'" )
    done

    runcmd "${envvars[@]+${envvars[@]}}" $PBS_STATUS_CMD "${args[@]+${args[@]}}"
}

# Assuming TORQUE is compatible with generic PBS for now
TORQUE_start() {
    local varname;
    for varname in ${!TORQUE_*} ${!JMSCONFIG_TORQUE_*}; do
        if [[ ! -z "${!varname+set}" ]] ; then
            eval "${varname/TORQUE_/PBS_}='${!varname}'"
        fi
    done

    PBS_start ${1+"$@"};
}
TORQUE_stop() {
    local varname;
    for varname in ${!TORQUE_*} ${!JMSCONFIG_TORQUE_*}; do
        if [[ ! -z "${!varname+set}" ]] ; then
            eval "${varname/TORQUE_/PBS_}='${!varname}'"
        fi
    done

    PBS_stop ${1+"$@"};
}
TORQUE_status() {
    local varname;
    for varname in ${!TORQUE_*} ${!JMSCONFIG_TORQUE_*}; do
        if [[ ! -z "${!varname+set}" ]] ; then
            eval "${varname/TORQUE_/PBS_}='${!varname}'"
        fi
    done

    PBS_status ${1+"$@"};
}

# Assuming PBSPro compatible with generic PBS for now
PBSPro_start() {
    local varname;
    for varname in ${!PBSPRO_*} ${!JMSCONFIG_PBSPRO_*}; do
        if [[ ! -z "${!varname+set}" ]] ; then
            eval "${varname/PBSPRO_/PBS_}='${!varname}'"
        fi
    done

    PBS_start ${1+"$@"};
}
PBSPro_stop() {
    local varname;
    for varname in ${!PBSPRO_*} ${!JMSCONFIG_PBSPRO_*}; do
        if [[ ! -z "${!varname+set}" ]] ; then
            eval "${varname/PBSPRO_/PBS_}='${!varname}'"
        fi
    done

    PBS_stop ${1+"$@"};
}
PBSPro_status() {
    local varname;
    for varname in ${!PBSPRO_*} ${!JMSCONFIG_PBSPRO_*}; do
        if [[ ! -z "${!varname+set}" ]] ; then
            eval "${varname/PBSPRO_/PBS_}='${!varname}'"
        fi
    done

    PBS_status ${1+"$@"};
}

Slurm_start() {
    local -a prestart_args=();
    local -a start_args=();

    local varname;
    local varnames="
        SLURM_JOB_NAME
        SLURM_NPROC
        SLURM_STDOUT_FILE
        SLURM_STDERR_FILE
        SLURM_JOB_OPTSTR
        SLURM_JOB_CMD

        SLURM_PRESTART_CMD
        SLURM_PRESTART_ARGS
        SLURM_START_CMD
        SLURM_START_ARGS

        SLURM_NODES
        SLURM_NTASKS
        SLURM_PARTITION
        SLURM_SHELL_ARG
        SLURM_USE_SALLOCSRUN
    "

    local nvarname;
    for varname in $varnames; do
        # At this point, we expect the SLURM_VARXYZ and VARXYZ variables
        # to have been set by the user in user.jmsenv.ish, the
        # JMSCONFIG_SLURM_VARXYZ variables have been set by the installer
        # in jmsenv.ish, and the JOBVAR_VARXYZ variables have been set by
        # the job scheduler when invoking this job.  For the most part we
        # expect that the JMSCONFIG_SLURM_* and JOBVAR_* will not conflict
        # with each other, and all of them are overridable by the user
        # settings.
        #
        # Order of priority:
        #   SLURM_VARXYZ             (highest priority)
        #   VARXYZ
        #   special cases
        #   JMSCONFIG_SLURM_VARXYZ
        #   JOBVAR_VARXYZ          (lowest priority)

        # Assign SLURM_VARXYZ=$VARXYZ (if VARXYZ is set and SLURM_VARXYZ
        # is not)
        [[ ! -z "${!varname+set}" ]] && continue;
        nvarname=${varname#SLURM_};
        [[ ! -z "${!nvarname+set}" ]] && eval "${varname}='${!nvarname}'";

        # Special case (legacy).  Assign SLURM_START_ARGS=$EXTRAS (if
        # EXTRAS is set and SLURM_START_ARGS is not)
        [[ ! -z "${!varname+set}" ]] && continue;
        if [[ "$varname" == "SLURM_START_ARGS" ]]; then
            nvarname="EXTRAS";
            [[ ! -z "${!nvarname+set}" ]] && eval "${varname}='${!nvarname}'";
        fi

        # Assign SLURM_VARXYZ=$JMSCONFIG_SLURM_VARXYZ (if
        # JMSCONFIG_SLURM_VARXYZ is set and SLURM_VARXYZ is not)
        [[ ! -z "${!varname+set}" ]] && continue;
        nvarname="JMSCONFIG_${varname}";
        [[ ! -z "${!nvarname+set}" ]] && eval "${varname}='${!nvarname}'";

        # Assign SLURM_VARXYZ=$JOBVAR_VARXYZ (if JOBVAR_VARXYZ is set
        # and SLURM_VARXYZ is not)
        [[ ! -z "${!varname+set}" ]] && continue;
        nvarname="JOBVAR_${varname#SLURM_}";
        [[ ! -z "${!nvarname+set}" ]] && eval "${varname}='${!nvarname}'";
    done

    # FIXME: Hack for cromwell support, since we don't set the up the
    #        environment we need when running on the cluster nodes or call
    #        the smrttool exes with the full path (since they should all set
    #        the environment they need)
    if [[ -f "$SLURM_JOB_CMD" ]]; then
        if  ! $opt_noexec; then
            # Add smrttools path, if needed
            local newpathline;
            local line2;
            newpathline="PATH=\"$g_progdir_abs/../../bundles/smrttools/current/private/otherbins/all/bin:$PATH\""
            line2=$(sed -ne '2p' "$SLURM_JOB_CMD")
            if [[ x"$line2" != x"$newpathline" ]] ; then
                sed -i -e "1a $newpathline" "$SLURM_JOB_CMD"
            fi

            # Slurm requires the script to be executable
            chmod +x "$SLURM_JOB_CMD"
        fi
    fi

    # Determine exported env vars so we can specify them on the command
    # line (so verbose and noexec modes will print out a line the can be
    # cut-n-pasted to duplicate the command, including environment
    # settings)
    local exported_vars;
    exported_vars=$(export -p | sed -ne 's/^\(declare[[:space:]]\+-x\|export\)[[:space:]]\+\([^=]\+\).*/\2/p')

    local -a envvars=();
    local envvar
    for envvar in $exported_vars; do
        if [[ -z "${!envvar+set}" ]] ; then
            # Ignore environment variables that are exported, but not set
            # (exported but unset env vars are not passed to subprocesses)
            continue;
        fi
        envvars+=( "$envvar='${!envvar}'" )
    done

    # Set defaults
    [[ -z "${SLURM_NODES+set}" ]]           && SLURM_NODES="1"
    [[ -z "${SLURM_NTASKS+set}" ]]          && SLURM_NTASKS="1"

    if [[ ! -z "${SLURM_USE_SALLOCSRUN:-}" ]] ; then
        # Set defaults
        [[ -z "${SLURM_PRESTART_CMD+set}" ]]    && SLURM_PRESTART_CMD="salloc"
        [[ -z "${SLURM_START_CMD+set}" ]]       && SLURM_START_CMD="srun"
        [[ -z "${SLURM_SHELL_ARG+set}" ]]       && SLURM_SHELL_ARG="/bin/bash"

        # Build the list of prestart command line args
        [[ ! -z "${SLURM_JOB_NAME:-}" ]] \
            && prestart_args+=( --job-name="$SLURM_JOB_NAME" )
        [[ ! -z "${SLURM_NODES:-}" ]] \
            && prestart_args+=( --nodes="$SLURM_NODES" )
        [[ ! -z "${SLURM_NPROC:-}" ]] \
            && prestart_args+=( --cpus-per-task="$SLURM_NPROC" )
        [[ ! -z "${SLURM_PARTITION:-}" ]] \
            && prestart_args+=( --partition="$SLURM_PARTITION" )
        [[ ! -z "${SLURM_PRESTART_ARGS:-}" ]] \
            && eval prestart_args+=( "$SLURM_PRESTART_ARGS" )

        # Build the list of start command line args
        [[ ! -z "${SLURM_NPROC:-}" ]] \
            && start_args+=( --cpus-per-task="$SLURM_NPROC" )
        [[ ! -z "${SLURM_NTASKS:-}" ]] \
            && start_args+=( --ntasks="$SLURM_NTASKS" )
        [[ ! -z "${SLURM_STDOUT_FILE:-}" ]] \
            && start_args+=( -o "$SLURM_STDOUT_FILE" )
        [[ ! -z "${SLURM_STDERR_FILE:-}" ]] \
            && start_args+=( -e "$SLURM_STDERR_FILE" )
        [[ ! -z "${SLURM_PARTITION:-}" ]] \
            && start_args+=( --partition="$SLURM_PARTITION" )
        [[ ! -z "${SLURM_START_ARGS:-}" ]] \
            && eval start_args+=( "$SLURM_START_ARGS" )
        [[ ! -z "${SLURM_JOB_CMD:-}" ]] \
            && start_args+=( "$SLURM_SHELL_ARG" -c "$SLURM_JOB_CMD" )

        # Get the abspath to command from PATH (see comments in function)
        SLURM_PRESTART_CMD=$(get_abspath_from_path "$SLURM_PRESTART_CMD")
        SLURM_START_CMD=$(get_abspath_from_path "$SLURM_START_CMD")

        runcmd "${envvars[@]+${envvars[@]}}" $SLURM_PRESTART_CMD "${prestart_args[@]+${prestart_args[@]}}" $SLURM_START_CMD "${start_args[@]+${start_args[@]}}"
    else
        # Set defaults
        [[ -z "${SLURM_START_CMD+set}" ]]       && SLURM_START_CMD="sbatch"
        [[ -z "${SLURM_WAIT_ARG+set}" ]]        && SLURM_WAIT_ARG=""   # --wait
        [[ -z "${SLURM_SHELL_ARG+set}" ]]       && SLURM_SHELL_ARG=""

        # Build the list of start command line args
        [[ ! -z "${SLURM_WAIT_ARG:-}" ]] \
            && start_args+=( "$SLURM_WAIT_ARG" )
        [[ ! -z "${SLURM_JOB_NAME:-}" ]] \
            && start_args+=( --job-name="$SLURM_JOB_NAME" )
        [[ ! -z "${SLURM_NODES:-}" ]] \
            && start_args+=( --nodes="$SLURM_NODES" )
        [[ ! -z "${SLURM_NTASKS:-}" ]] \
            && start_args+=( --ntasks="$SLURM_NTASKS" )
        [[ ! -z "${SLURM_NPROC:-}" ]] \
            && start_args+=( --cpus-per-task="$SLURM_NPROC" )
        [[ ! -z "${SLURM_PARTITION:-}" ]] \
            && start_args+=( --partition="$SLURM_PARTITION" )
        [[ ! -z "${SLURM_STDOUT_FILE:-}" ]] \
            && start_args+=( -o "$SLURM_STDOUT_FILE" )
        [[ ! -z "${SLURM_STDERR_FILE:-}" ]] \
            && start_args+=( -e "$SLURM_STDERR_FILE" )
        [[ ! -z "${SLURM_PRESTART_ARGS:-}" ]] \
            && eval start_args+=( "$SLURM_PRESTART_ARGS" )
        [[ ! -z "${SLURM_START_ARGS:-}" ]] \
            && eval start_args+=( "$SLURM_START_ARGS" )
        if [[ ! -z "${SLURM_JOB_CMD:-}" ]]; then
            if [[ -z "$SLURM_SHELL_ARG" ]] ; then
                start_args+=( --wrap="$SLURM_JOB_CMD" )
            else
                start_args+=( --wrap="'$SLURM_SHELL_ARG' -c '$SLURM_JOB_CMD'" )
            fi
        fi

        # Get the abspath to command from PATH (see comments in function)
        SLURM_START_CMD=$(get_abspath_from_path "$SLURM_START_CMD")

        runcmd "${envvars[@]+${envvars[@]}}"  $SLURM_START_CMD "${start_args[@]+${start_args[@]}}"
    fi
}

Slurm_stop() {
    local -a args=();

    local varname;
    local varnames="
        SLURM_JOB_ID

        SLURM_STOP_CMD
        SLURM_STOP_ARGS
    "

    local nvarname;
    for varname in $varnames; do
        # At this point, we expect the SLURM_VARXYZ and VARXYZ variables
        # to have been set by the user in user.jmsenv.ish, the
        # JMSCONFIG_SLURM_VARXYZ variables have been set by the installer
        # in jmsenv.ish, and the JOBVAR_VARXYZ variables have been set by
        # the job scheduler when invoking this job.  For the most part we
        # expect that the JMSCONFIG_SLURM_* and JOBVAR_* will not conflict
        # with each other, and all of them are overridable by the user
        # settings.
        #
        # Order of priority:
        #   SLURM_VARXYZ             (highest priority)
        #   VARXYZ
        #   special cases
        #   JMSCONFIG_SLURM_VARXYZ
        #   JOBVAR_VARXYZ          (lowest priority)

        # Assign SLURM_VARXYZ=$VARXYZ (if VARXYZ is set and SLURM_VARXYZ
        # is not)
        [[ ! -z "${!varname+set}" ]] && continue;
        nvarname=${varname#SLURM_};
        [[ ! -z "${!nvarname+set}" ]] && eval "${varname}='${!nvarname}'";

        # Assign SLURM_VARXYZ=$JMSCONFIG_SLURM_VARXYZ (if
        # JMSCONFIG_SLURM_VARXYZ is set and SLURM_VARXYZ is not)
        [[ ! -z "${!varname+set}" ]] && continue;
        nvarname="JMSCONFIG_${varname}";
        [[ ! -z "${!nvarname+set}" ]] && eval "${varname}='${!nvarname}'";

        # Assign SLURM_VARXYZ=$JOBVAR_VARXYZ (if JOBVAR_VARXYZ is set
        # and SLURM_VARXYZ is not)
        [[ ! -z "${!varname+set}" ]] && continue;
        nvarname="JOBVAR_${varname#SLURM_}";
        [[ ! -z "${!nvarname+set}" ]] && eval "${varname}='${!nvarname}'";
    done


    # Set defaults
    [[ -z "${SLURM_STOP_CMD+set}" ]]   && SLURM_STOP_CMD="scancel"

    # Build the list command arguments
    [[ ! -z "${SLURM_JOB_ID:-}" ]]   && args+=( "$SLURM_JOB_ID" )
    [[ ! -z "${SLURM_STOP_ARGS:-}" ]]  && eval args+=( "$SLURM_STOP_ARGS" )

    # Get the abspath to command from PATH (see comments in function)
    SLURM_STOP_CMD=$(get_abspath_from_path "$SLURM_STOP_CMD")

    # Determine exported env vars so we can specify them on the command
    # line (so verbose and noexec modes will print out a line the can be
    # cut-n-pasted to duplicate the command, including environment
    # settings)
    local exported_vars;
    exported_vars=$(export -p | sed -ne 's/^\(declare[[:space:]]\+-x\|export\)[[:space:]]\+\([^=]\+\).*/\2/p')

    local -a envvars=();
    local envvar
    for envvar in $exported_vars; do
        if [[ -z "${!envvar+set}" ]] ; then
            # Ignore environment variables that are exported, but not set
            # (exported but unset env vars are not passed to subprocesses)
            continue;
        fi
        envvars+=( "$envvar='${!envvar}'" )
    done

    runcmd "${envvars[@]+${envvars[@]}}" $SLURM_STOP_CMD "${args[@]+${args[@]}}"
}

Slurm_status() {
    local -a args=();

    local varname;
    local varnames="
        SLURM_JOB_ID

        SLURM_STATUS_CMD
        SLURM_STATUS_ARGS
    "

    local nvarname;
    for varname in $varnames; do
        # At this point, we expect the SLURM_VARXYZ and VARXYZ variables
        # to have been set by the user in user.jmsenv.ish, the
        # JMSCONFIG_SLURM_VARXYZ variables have been set by the installer
        # in jmsenv.ish, and the JOBVAR_VARXYZ variables have been set by
        # the job scheduler when invoking this job.  For the most part we
        # expect that the JMSCONFIG_SLURM_* and JOBVAR_* will not conflict
        # with each other, and all of them are overridable by the user
        # settings.
        #
        # Order of priority:
        #   SLURM_VARXYZ             (highest priority)
        #   VARXYZ
        #   special cases
        #   JMSCONFIG_SLURM_VARXYZ
        #   JOBVAR_VARXYZ          (lowest priority)

        # Assign SLURM_VARXYZ=$VARXYZ (if VARXYZ is set and SLURM_VARXYZ
        # is not)
        [[ ! -z "${!varname+set}" ]] && continue;
        nvarname=${varname#SLURM_};
        [[ ! -z "${!nvarname+set}" ]] && eval "${varname}='${!nvarname}'";

        # Assign SLURM_VARXYZ=$JMSCONFIG_SLURM_VARXYZ (if
        # JMSCONFIG_SLURM_VARXYZ is set and SLURM_VARXYZ is not)
        [[ ! -z "${!varname+set}" ]] && continue;
        nvarname="JMSCONFIG_${varname}";
        [[ ! -z "${!nvarname+set}" ]] && eval "${varname}='${!nvarname}'";

        # Assign SLURM_VARXYZ=$JOBVAR_VARXYZ (if JOBVAR_VARXYZ is set
        # and SLURM_VARXYZ is not)
        [[ ! -z "${!varname+set}" ]] && continue;
        nvarname="JOBVAR_${varname#SLURM_}";
        [[ ! -z "${!nvarname+set}" ]] && eval "${varname}='${!nvarname}'";
    done


    # Set defaults
    [[ -z "${SLURM_STATUS_CMD+set}" ]]   && SLURM_STATUS_CMD="squeue"

    # Build the list command arguments
    [[ ! -z "${SLURM_JOB_ID:-}" ]]     && args+=( -j "$SLURM_JOB_ID" )
    [[ ! -z "${SLURM_STATUS_ARGS:-}" ]]  && eval args+=( "$SLURM_STATUS_ARGS" )

    # Get the abspath to command from PATH (see comments in function)
    SLURM_STATUS_CMD=$(get_abspath_from_path "$SLURM_STATUS_CMD")

    # Determine exported env vars so we can specify them on the command
    # line (so verbose and noexec modes will print out a line the can be
    # cut-n-pasted to duplicate the command, including environment
    # settings)
    local exported_vars;
    exported_vars=$(export -p | sed -ne 's/^\(declare[[:space:]]\+-x\|export\)[[:space:]]\+\([^=]\+\).*/\2/p')

    local -a envvars=();
    local envvar
    for envvar in $exported_vars; do
        if [[ -z "${!envvar+set}" ]] ; then
            # Ignore environment variables that are exported, but not set
            # (exported but unset env vars are not passed to subprocesses)
            continue;
        fi
        envvars+=( "$envvar='${!envvar}'" )
    done

    runcmd "${envvars[@]+${envvars[@]}}" $SLURM_STATUS_CMD "${args[@]+${args[@]}}"
}


# ---- main

# Save off original cmdline args, use as "${g_origargs[@]}" (with doublequotes)
g_origargs=( ${1+"$@"} )

set_preglobals ${1+"$@"};
parseargs ${1+"$@"};
set_globals;

squash_env;

set_templatevars;
set_userenv;

cmdfunc=$(get_cmdfunc) || exit $?

# Special case, do not use the "$cmdfunc" || stat=$? method to capture the
# exit status, because it will cause unexpected errors to be ignored in the
# entire function.  Instead, depend on the script exiting with the exit
# status of the last command of the script.  So if the actual underlying
# command exits with non-zero status, or we end up with an unexpected error
# in any code along the way, we will still exit with non-zero status.
"$cmdfunc"