#!/bin/bash # -*- mode: sh; c-basic-offset: 4; tab-width: 8; indent-tabs-mode: nil -*- # vi: set shiftwidth=4 tabstop=8 softtabstop=4 expandtab: # :indentSize=4:tabSize=8:noTabs=true: # vim: filetype=sh # ---- error handling set -o errexit; set -o posix; set -o pipefail; set -o errtrace; set -o nounset; unexpected_error() { local errstat=$? echo "${g_prog:-$(basename "$0")}: Error! Encountered unexpected error at 'line $(caller)', bailing out..." 1>&2 exit $errstat; } trap unexpected_error ERR; # Force the path to only what we need, saving off the original path PATH_ORIG=$PATH; PATH=/usr/bin:/bin g_prog=$(basename "$0"); g_progdir=$(dirname "$0"); g_progdir_abs=$(dirname "$(readlink -f "$0")"); # ---- global env . "$g_progdir_abs/../../private/runtime-common/lib/globalenv.ish" # ---- error functions merror() { echo "$g_prog: Error! ""$@" 1>&2; exit 1; } minterror() { echo "$g_prog: Internal Error! ""$@" 1>&2; exit 1; } mwarn() { echo "$g_prog: Warning! ""$@" 1>&2;} # ---- usage usage() { local exitstat=2; if [[ ! -z "${2:-}" ]] ; then exitstat=$2; fi # Only redirect to stderr on non-zero exit status if [[ $exitstat -ne 0 ]] ; then exec 1>&2; fi if [[ ! -z "${1:-}" ]] ; then echo "$g_prog: Error! $1" 1>&2; fi echo "Usage: $g_prog [--help] \\" echo " [--no-errcleanup] [--debug-mode] \\" echo " [--cromwell-debug-mode] [--services-debug-mode] \\" echo " [--cromwell-jmxremote-port] [--services-jmxremote-port]" echo " --no-errcleanup -- do not stop services on error" echo " --debug-mode -- start cromwell and smrtlink in debug mode" echo " --cromwell-debug-mode -- start cromwell in debug mode" echo " --services-debug-mode -- start services in debug mode" echo " --cromwell-jmxremote-port -- jmx port for remote cromwell debug" echo " (default: 9933)" echo " --services-jmxremote-port -- jmx port for remote services debug" echo " (default: 9934)" echo " --help -- print this usage"; echo ""; # bash only: if [[ $exitstat -ne 0 ]] ; then echo " at: $(caller)"; fi exit $exitstat; } # ---- argument parsing # Save off the original args, use as "${g_origargs[@]}" (with double quotes) declare -a g_origargs; g_origargs=( ${1+"$@"} ) opt_no_errcleanup=false; opt_debug_mode=false; opt_cromwell_debug_mode=false; opt_services_debug_mode=false; opt_cromwell_jmxremote_port=9933; opt_services_jmxremote_port=9934; while [[ $# != 0 ]]; do opt="$1"; shift; case "$opt" in # Flag with no argument example: # --flag|--fla|--fl|--f) # opt_flag=true;; # Option with argument example: # --arg|--ar|--a) # [[ $# -eq 0 ]] && usage; # opt_somearg=$1; shift;; --no-errcleanup) opt_no_errcleanup=true;; --debug-mode) opt_debug_mode=true;; --cromwell-debug-mode) opt_cromwell_debug_mode=true;; --services-debug-mode) opt_services_debug_mode=true;; --cromwell-jmxremote-port) [[ $# -eq 0 ]] && usage; opt_cromwell_jmxremote_port=$1; shift;; --services-jmxremote-port) [[ $# -eq 0 ]] && usage; opt_services_jmxremote_port=$1; shift;; -h|-help|--help|--hel|--he|--h) usage "" 0;; -*) usage "Unrecognized option: $opt";; *) usage "Extra trailing arguments: $opt $@";; esac if $opt_debug_mode; then opt_cromwell_debug_mode=true; opt_services_debug_mode=true; fi done # ---- globals g_rootdir="$g_progdir/../.." g_rootdir_abs="$g_progdir_abs/../.." g_javahome="$g_rootdir_abs/bundles/smrtlink-analysisservices-gui/current/private/thirdparty/java/jre/jre_8u192b12-hotspot" g_dbcmds_bindir="$g_rootdir_abs/bundles/smrtlink-analysisservices-gui/current/private/thirdparty/postgresql/postgresql_9.6.1/binwrap" g_python3_bindir="$g_rootdir_abs/bundles/smrttools/current/private/thirdparty/python3/python3_3.7.3/binwrap" g_python3_exe="$g_python3_bindir/python3" g_cromwell_exe="$g_rootdir_abs/bundles/smrttools/current/private/thirdparty/cromwell/cromwell_47-pacbio1/binwrap/cromwell" g_slagroot="$g_rootdir/bundles/smrtlink-analysisservices-gui/current/private/pacbio/smrtlink-analysisservices-gui" g_smrtlink_topdir=$(readlink -f "$g_progdir_abs/../..") g_smrtroot_dir=$(readlink -f "$g_smrtlink_topdir/../..") g_userdata_dir="$g_smrtroot_dir/userdata" g_smrtlinkconfig_json="$g_userdata_dir/generated/config/smrtlink-system-config.json" g_smrtlink_log_file="$g_userdata_dir/log/smrtlink-analysisservices-gui/secondary-smrt-server.log" g_smrtlinkanalysisgui_dir="$g_smrtroot_dir/current/bundles/smrtlink-analysisservices-gui/current/private/pacbio/smrtlink-analysisservices-gui" g_piddir="$g_rootdir_abs/rwdir/run" # pidfiles for all services g_smrtlink_gui_pidfile="$g_piddir/smrtlink_gui.pid" g_database_pidfile="$g_userdata_dir/db_datadir/dbstore/postmaster.pid" g_smrtlink_services_pidfile="$g_smrtlinkanalysisgui_dir/sl-analysis.pid" g_cromwell_pidfile="$g_piddir/cromwell.pid" g_smrtlink_wso2_pidfile="$g_smrtlinkanalysisgui_dir/wso2am-2.0.0/wso2carbon.pid" # Cromwell settings g_smrtlink_cromwell_startupdir="$g_userdata_dir/jobs_root" g_smrtlink_cromwell_configfile="$g_userdata_dir/generated/config/cromwell.conf" g_smrtlink_cromwell_exe="$g_cromwell_exe" g_smrtlink_cromwell_logdir="$g_userdata_dir/log/cromwell" g_smrtlink_cromwell_stdout="$g_smrtlink_cromwell_logdir/cromwell.stdout" g_smrtlink_cromwell_stderr="$g_smrtlink_cromwell_logdir/cromwell.stderr" g_smrtlink_cromwell_pidfile="$g_cromwell_pidfile" g_smrtlink_cromwell_javahome="$g_javahome" g_smrtlink_cromwell_path="$g_smrtroot_dir/current/bundles/smrttools/current/private/otherbins/all/bin:/usr/bin:/bin" # Cromwell java options (for debug and production modes) g_smrtlink_cromwell_javaopts="" g_smrtlink_cromwell_javaopts+=" -Djava.library.path=" g_smrtlink_cromwell_javaopts+=" -XX:+PrintFlagsFinal" if $opt_cromwell_debug_mode; then # Debug Mode g_smrtlink_cromwell_javaopts+=" -DLOG_LEVEL=DEBUG" g_smrtlink_cromwell_javaopts+=" -Xloggc:gclog.log" g_smrtlink_cromwell_javaopts+=" -XX:+UseGCLogFileRotation" g_smrtlink_cromwell_javaopts+=" -XX:NumberOfGCLogFiles=5" g_smrtlink_cromwell_javaopts+=" -XX:+PrintFlagsFinal" g_smrtlink_cromwell_javaopts+=" -Dcom.sun.management.jmxremote.port=$opt_cromwell_jmxremote_port" g_smrtlink_cromwell_javaopts+=" -Dcom.sun.management.jmxremote.ssl=false" g_smrtlink_cromwell_javaopts+=" -Dcom.sun.management.jmxremote.authenticate=false" else # Production Mode g_smrtlink_cromwell_javaopts+=" -XX:+HeapDumpOnOutOfMemoryError" fi if [[ ! -z "${SMRTLINK_CROMWELL_JAVA_OPTS_OVERRIDE+set}" ]] ; then g_smrtlink_cromwell_javaopts=$SMRTLINK_CROMWELL_JAVA_OPTS_OVERRIDE; fi if [[ ! -z "${SMRTLINK_CROMWELL_JAVA_OPTS_PREPEND+set}" ]] ; then g_smrtlink_cromwell_javaopts="$SMRTLINK_CROMWELL_JAVA_OPTS_PREPEND $g_smrtlink_cromwell_javaopts" fi if [[ ! -z "${SMRTLINK_CROMWELL_JAVA_OPTS_APPEND+set}" ]] ; then g_smrtlink_cromwell_javaopts="$g_smrtlink_cromwell_javaopts $SMRTLINK_CROMWELL_JAVA_OPTS_APPEND" fi g_smrtlink_services_log_args="--log-file $g_smrtlink_log_file" # Services java options (for debug and production modes) g_smrtlink_services_javaopts_proddebug="" g_smrtlink_services_javaopts_proddebug+=" -Djava.library.path=" g_smrtlink_services_javaopts_proddebug+=" -XX:+PrintFlagsFinal" if $opt_services_debug_mode; then # Debug Mode g_smrtlink_services_log_args+=" --log-level DEBUG" g_smrtlink_services_javaopts_proddebug+=" -Xloggc:gclog.log" g_smrtlink_services_javaopts_proddebug+=" -XX:+UseGCLogFileRotation" g_smrtlink_services_javaopts_proddebug+=" -XX:NumberOfGCLogFiles=5" g_smrtlink_services_javaopts_proddebug+=" -XX:+PrintFlagsFinal" g_smrtlink_services_javaopts_proddebug+=" -Dcom.sun.management.jmxremote.port=$opt_services_jmxremote_port" g_smrtlink_services_javaopts_proddebug+=" -Dcom.sun.management.jmxremote.ssl=false" g_smrtlink_services_javaopts_proddebug+=" -Dcom.sun.management.jmxremote.authenticate=false" else # Production Mode g_smrtlink_services_log_args+=" --log-level INFO" g_smrtlink_services_javaopts_proddebug+=" -XX:+HeapDumpOnOutOfMemoryError" fi # ---- subroutines pid_running() { local component=$1; shift; local pidfile=$1; shift; local pid local running=false; if [ -f "${pidfile}" ]; then pid=$( sed -n -e '1{s/\x0//g; p}' "${pidfile}" ) if [[ ${pid} =~ ^[0-9]+$ ]] ; then if kill -0 "$pid" > /dev/null 2>&1 ; then running=true; else echo " Removing stale '$component' pid file..." rm -f "$pidfile" fi else echo "Unrecognized pid ($pid) in pid file (${pidfile})" 1>&2 echo "Services not started!" 1>&2 exit 3 fi fi if ! $running; then return 1; fi return 0; } check_listening_ports() { local ports=$1; shift; local ss_cmd; local netstat_cmd; ss_cmd=$(PATH=/usr/bin:/bin:/usr/sbin:/sbin command -v ss || true) netstat_cmd=$(PATH=/usr/bin:/bin:/usr/sbin:/sbin command -v netstat || true) if [[ -z "$ss_cmd" ]] && [[ -z "$netstat_cmd" ]]; then merror "Cannot find 'ss' or 'netstat' command." fi local ports_re; ports_re=$(echo $ports) ports_re=${ports_re// /\\|} local listening_ports; if [[ ! -z "$ss_cmd" ]]; then listening_ports=$("$ss_cmd" -nlt4 | sed -n -e "s/LISTEN[[:space:]].*:\(${ports_re}\)[[:space:]].*/\1/p" | sort -u) else listening_ports=$("$netstat_cmd" -nlt | sed -n -e "s/tcp[[:space:]].*:\(${ports_re}\)[[:space:]].*/\1/p" | sort -u) fi echo $listening_ports } wait_for_timewait_ports() { local ports=$1; shift; local ss_cmd; local netstat_cmd; ss_cmd=$(PATH=/usr/bin:/bin:/usr/sbin:/sbin command -v ss || true) netstat_cmd=$(PATH=/usr/bin:/bin:/usr/sbin:/sbin command -v netstat || true) if [[ -z "$ss_cmd" ]] && [[ -z "$netstat_cmd" ]]; then merror "Cannot find 'ss' or 'netstat' command." fi local ports_re; ports_re=$(echo $ports) ports_re=${ports_re// /|} # Timeout after 2.5 minutes of waiting (should be good enough for all # ports in the TIME_WAIT to close. On linux, we expect that the # for ports to stay in the TIME_WAIT state is 2*MSL, where the default # for MSL (Maximum Segment Length) is 60 sec. local endtime endtime=$(( $(date +%s) + 150 )) local stat; local printed=false; while true; do if [[ ! -z "$ss_cmd" ]]; then stat=0; "$ss_cmd" -nat4 | grep -E "^TIME-WAIT[[:space:]].*:(${ports_re})([[:space:]]|$)" > /dev/null || stat=$? else stat=0; "$netstat_cmd" -nat | grep -E ":(${ports_re})[[:space:]].*[[:space:]]TIME_WAIT([[:space:]]|\$)" > /dev/null || stat=$? fi if [[ $stat -eq 0 ]] ; then : elif [[ $stat -eq 1 ]] ; then echo break; else merror "Unexpected error ($stat) in checking ports" fi if ! $printed; then echo echo "Waiting for closing ports..." printed=true; fi if [[ $(date +%s) -gt $endtime ]] ; then echo "Got time out waiting for ports to close." break; fi sleep 2; done } # ---- main errors=false echo "Checking services state..." # Check to see if any of the services are running (checking the pid files) running_services=() if pid_running "SMRT Link GUI" "$g_smrtlink_gui_pidfile"; then running_services+=( "SMRT Link GUI" ) fi if pid_running "SMRT Link Database" "$g_database_pidfile"; then running_services+=( "SMRT Link Database" ) fi if pid_running "SMRT Link Services" "$g_smrtlink_services_pidfile"; then running_services+=( "SMRT Link Services" ) fi if pid_running "SMRT Link Cromwell Server" "$g_smrtlink_cromwell_pidfile"; then running_services+=( "SMRT Link Cromwell Server" ) fi if pid_running "SMRT Link WSO2 API Manager" "$g_smrtlink_wso2_pidfile"; then running_services+=( "SMRT Link WSO2 API Manager" ) fi if [[ ! -z "${running_services[@]+${running_services[@]}}" ]]; then if [[ ${#running_services[@]} -eq 5 ]] ; then echo echo "All SMRT Link Service processes are running." echo "To restart services, run 'services-stop' and rerun this program." exit 0 else echo echo "The following services are already running:" for service in "${running_services[@]}" ; do echo " $service" done echo echo "Some, but not all, services are running." merror "Services not running correctly. Run 'services-stop' and retry." exit 4 fi fi echo "Checking for port conflicts..." gui_port=$("$g_python3_exe" -c "import json,os,sys; d=json.load(open(sys.argv[1])); print(d['pacBioSystem']['tomcatPort'])" "$g_smrtlinkconfig_json") database_port=$("$g_python3_exe" -c "import json,os,sys; d=json.load(open(sys.argv[1])); print(d['smrtflow']['db']['properties']['portNumber'])" "$g_smrtlinkconfig_json") services_port=$("$g_python3_exe" -c "import json,os,sys; d=json.load(open(sys.argv[1])); print(d['smrtflow']['server']['port'])" "$g_smrtlinkconfig_json") cromwell_port=$("$g_python3_exe" -c "import json,os,sys; d=json.load(open(sys.argv[1])); print(d['smrtflow']['cromwell']['port'])" "$g_smrtlinkconfig_json") wso2_ports=$(sed -n -e 's/^SMRTLINK_PORTS="\([^"]*\)".*/\1/p' "$g_slagroot/bin/check-system-limits" | sed -e 's/[[:space:]]\+/ /g') if [[ ! $gui_port =~ ^[0-9]+[0-9\ ]+$ ]] ; then minterror "Could not read SMRT Link GUI port ($gui_port)" fi if [[ ! $database_port =~ ^[0-9]+[0-9\ ]+$ ]] ; then minterror "Could not read SMRT Link Database port ($database_port)" fi if [[ ! $services_port =~ ^[0-9]+[0-9\ ]+$ ]] ; then minterror "Could not read SMRT Link Services port ($services_port)" fi if [[ ! $cromwell_port =~ ^[0-9]+[0-9\ ]+$ ]] ; then minterror "Could not read SMRT Link Cromwell Server port ($cromwell_port)" fi if [[ ! $wso2_ports =~ ^[0-9]+[0-9\ ]+$ ]] ; then minterror "Could not read WSO2 ports ($wso2_ports)" fi listening_ports_str=(); listening_ports=$(check_listening_ports "$gui_port") if [[ ! -z "$listening_ports" ]] ; then listening_ports_strs+=( "SMRT Link GUI: $listening_ports" ) fi listening_ports=$(check_listening_ports "$database_port") if [[ ! -z "$listening_ports" ]] ; then listening_ports_strs+=( "SMRT Link Database: $listening_ports" ) fi listening_ports=$(check_listening_ports "$services_port") if [[ ! -z "$listening_ports" ]] ; then listening_ports_strs+=( "SMRT Link Services: $listening_ports" ) fi listening_ports=$(check_listening_ports "$cromwell_port") if [[ ! -z "$listening_ports" ]] ; then listening_ports_strs+=( "SMRT Link Cromwell Server: $listening_ports" ) fi listening_ports=$(check_listening_ports "$wso2_ports") if [[ ! -z "$listening_ports" ]] ; then listening_ports_strs+=( "SMRT Link WSO2 Api Manager: $listening_ports" ) fi if [[ ! -z "${listening_ports_strs[@]+${listening_ports_strs[@]}}" ]]; then echo echo "The following services have ports opening for listening:" for listenport_str in "${listening_ports_strs[@]}" ; do echo " $listenport_str" done echo merror "Conflicting ports found. Resolve any port conflicts and retry." fi echo "Checking system..." if ! "$g_progdir/checkmounts"; then merror "Detected unresponsive mount points, exiting..." fi wait_for_timewait_ports "$gui_port $database_port $services_port $cromwell_port $wso2_ports" # Hack to make sure tmp_dir exists when running in a production environment. # The tmp_dir may not exist and may need to be created if it gets removed # (on reboot for instance) prod_tmpdir_link="$g_progdir_abs/../../../../userdata/tmp_dir" if [[ -h "$prod_tmpdir_link" ]] ; then if [[ -e "$prod_tmpdir_link" ]] ; then if [[ ! -d "$prod_tmpdir_link" ]] ; then merror "tmp_dir '$prod_tmpdir_link' exists, but is not a directory." fi else # Directory doesn't exist, make the directory pointed to by the # real path (resolving any symlinks): # Make sure to temporarily set umask so that the tmp dir will # not be accessible by group/other. (umask 0077 && mkdir -p $(readlink -m "$prod_tmpdir_link")) fi fi # At this point, we are basically guaranteed that none of the services are # running. Though it is still possible that a process is still running # without a pid file, somethhing line an ephemeral port causing a confict, # or some other situation that may cause some error in starting up any one of # the service or in running any of the post-start actions (like amclient or # import-canneddata). # # If we detect any errors from here on out, we will run 'services-stop' # before exiting, in order to return to this state wher all services are # stopped. The resoning is that any errors wouild indicate that services # somehow did not start correctly, and should not be used or left running. # # The --no-errcleanup flag can be used to change this behavior, for # debugging issues or whatever. With --no-errcleanup set, any error # willl cause 'services-=start' to exit without running 'stop-services'. # That may leave some (or all) services running, but they may not be # guaranteed to be running correctly. # Make sure the pid directory exists mkdir -p "$g_piddir" if ! $errors; then # Start up smrtlink gui, database, services, cromwell and wso2 # Hardwire the pbpipeline-resources directory g_pipeline_bundle_dir="$g_rootdir_abs/bundles/smrttools/current/private/pacbio/pbpipeline-resources" # Invoke smrtlink gui and services stat=0 env PATH="$g_rootdir_abs/bundles/smrttools/smrtcmds/bin:$g_javahome/bin:$g_python3_bindir:$g_dbcmds_bindir:$PATH" \ SMRT_PYTHON3_PATH="$g_dbcmds_bindir:$PATH" \ SMRT_PIPELINE_BUNDLE_DIR="$g_pipeline_bundle_dir" \ CATALINA_PID="$g_smrtlink_gui_pidfile" \ \ JAVA_HOME="$g_javahome" JAVA_OPTS="-Djava.library.path=" \ \ SMRTLINK_CLIENT_JAVA_HOME="$g_javahome" \ SMRTLINK_CLIENT_JAVA_OPTS="-Djava.library.path=" \ \ SMRTLINK_WSO2_JAVA_HOME="$g_javahome" \ SMRTLINK_WSO2_JAVA_OPTS="-Djava.library.path=" \ \ SMRTLINK_SERVICES_LOG_ARGS="$g_smrtlink_services_log_args" \ SMRTLINK_SERVICES_JAVA_OPTS_PRODDEBUG="$g_smrtlink_services_javaopts_proddebug" \ \ SMRTLINK_CROMWELL_PATH="$g_smrtlink_cromwell_path" \ SMRTLINK_CROMWELL_STARTUPDIR="$g_smrtlink_cromwell_startupdir" \ SMRTLINK_CROMWELL_JAVA_HOME="$g_smrtlink_cromwell_javahome" \ SMRTLINK_CROMWELL_JAVA_OPTS="$g_smrtlink_cromwell_javaopts" \ SMRTLINK_CROMWELL_CONFIG_FILE="$g_smrtlink_cromwell_configfile" \ SMRTLINK_CROMWELL_EXE="$g_smrtlink_cromwell_exe" \ SMRTLINK_CROMWELL_STDOUT="$g_smrtlink_cromwell_stdout" \ SMRTLINK_CROMWELL_STDERR="$g_smrtlink_cromwell_stderr" \ SMRTLINK_CROMWELL_PIDFILE="$g_smrtlink_cromwell_pidfile" \ "$g_slagroot/bin/start" || stat=$? if [[ $stat -ne 0 ]] ; then echo echo "Errors detected in starting SMRT Link services." echo errors=true; fi fi # Wait for services to start if ! $errors; then echo echo "Waiting for services to start..." stat=0 "$g_progdir/services-status" --quiet --wait --waittime 60 || stat=$? if [[ $stat -ne 0 ]] ; then echo echo "Errors detected in checking SMRT Link services status." echo errors=true; fi fi if ! $errors; then rwdir_dir="$g_rootdir_abs/rwdir" import_flagfile="$rwdir_dir/must-import-canneddata" import_logdir="$rwdir_dir/log" import_logfile="$import_logdir/import-canneddata.log" if [[ -e "$import_flagfile" ]] ; then echo echo "Importing canneddata and barcodes..." echo mkdir -p "$import_logdir" stat=0; # Use import-canneddata script to import barcodes and canneddata, # redirecting to a log file for the normal case (to avoid too much # chatter to the screen), printing the log file to the screen in # case of error. "$g_progdir/import-canneddata" > "$import_logfile" 2>&1|| stat=$? if [[ $stat -ne 0 ]] ; then echo cat "$import_logfile" echo echo "Errors detected in running import-canneddata." echo errors=true; else rm -f "$import_flagfile" fi fi fi if $errors; then if ! $opt_no_errcleanup; then echo "Errors encountered, shutting down all services..." echo "$g_progdir/services-stop" || true; echo merror "Services not started properly, all services stopped." else echo "WARNING WARNING WARNING WARNING WARNING WARNING WARNING" echo echo "Errors encountered in starting up services." echo "Not shutting down services because --no-errcleanup specified." echo "Some services may be running, but may not be fully functional" echo "and may not be operating correctly." echo echo "WARNING WARNING WARNING WARNING WARNING WARNING WARNING" echo merror "Services not started properly, some services may still be running." fi echo fi echo "Services started successfully."