#!/bin/bash
# @(#)12    1.21  rssm_obcl/NCCLpreverify, rssm_obcl, rssm02 1/12/10 16:43:29
# *************************************************************************************
#
# -------------------------------------------------------------------------------------
#
#                                  IBM CONFIDENTIAL
#
#                       LICENSED INTERNAL CODE SOURCE MATERIALS
#
#                           1820 LICENSED INTERNAL CODE
#
##                        (C) COPYRIGHT IBM CORP. 2009, 2010
#
#                THE SOURCE CODE FOR THIS PROGRAM IS NOT PUBLISHED OR
#                OTHERWISE DIVESTED OF ITS TRADE SECRETS, IRRESPECTIVE OF
#                WHAT HAS BEEN DEPOSITED WITH THE U.S. COPYRIGHT OFFICE.
#
# -------------------------------------------------------------------------------------
#
# Storage Blade Non-Concurrent Code Load (NCCL) Healthcheck Script
#
# *************************************************************************************
#
# 04-15-08 car 000001 Create program
# 02-06-09 wjr 000002 Rewriten - NCCL checks complete
# 02-09-09 wjr d16057 DCR81 - Firmware Update Pre-verify, pre-verification scripts
# 03-16-09 wjr d16051 DCR104 - Increased granularity for detection of drive exclusion
# 04-21-09 wjr d16511 Code load failed with ./SbInst.py failed in function waitForCtlrState
# 04-22-09 wjr d16723 OBCL: NF: CCL preverify script
# 05-04-09 wjr d16709 RSSM: NCCL preverify logs reporting passed when drive in failed state
# 05-05-09 wjr d16787 OBCL: CtlrFwMgmt.conf changes for new BMC, FPGA formats
# 06-08-09 wjr d16975 CCL preverify script failed with incorrect messages
# 08-12-09 wjr d17429 FVT SCM - NCCL failing with invalid status for controller in service mode
# 08-25-09 wjr d17475 OBCL pre-verify come-from/go-to level check fails for pre-R2 to R2 upgrade
# 08-31-09 wjr d17512 OBCL changes to support ITR8251
# 09-02-09 wjr d17513 OBCL: H2031->H2033 and H2033->H2033 failed with FAILED_PREVERIFYFAILED
# 09-09-09 wjr d17587 Ut of preverify function on 012 package needs an alert that is stated more clearly
# 09-17-09 wjr d17644 OBCL pre-verify failed - "list drive" CLI cmd output format changed
# 09-18-09 wjr d17654 CCL preverify passed even though critical alert 9200 was present
# 09-22-09 wjr d17645 Alert needs to specify, more precisely, a controller in survivor mode
# 09-29-09 wjr d17697 OBCL 1.2.0.060 aborted when controller were in the expected state
# 10-02-09 wjr d17759 RSSM - CVT - OBCL fails at Drive update
# 11-16-09 wjr d18107 OBCL Single controller support
# 01-12-10 wjr d18324 CCL Preverify does not fail (or stop firmware update) when there is unhealthy pool
# 03-09-10 wjr d18524 Pre-Verify did not detect and fail with degraded pools
# 01-30-13 tny CQ68285 Preverify should also consider UNF/UNS state
#
# =====================================================================================
function show_syntax
{
  echo ''
  echo "$SCRIPT < -? | -h | -H [ -v ] [ -s ] [ -c CFG_STRING ] >"                      # d17429 - wjr
  echo ''
}
# =====================================================================================
function show_help
{
  show_syntax
  echo '    Required arguments (mutually exclusive):'
  echo ''
  echo '    None'
  echo ''
  echo '    Optional arguments:                                   Default'
  echo ''
  echo '    -v : Display Verbose Output                           Not displayed'
  echo '    -s : Display Summary Output                           Displayed'
  echo '    -c : Use CFG_STRING instead of CtlrFwMgmt.conf file   CtlrFwMgmt.conf'     # d17429 - wjr
  echo ''
  echo '    Return codes / Alerts:'                                                    # d16723 - wjr
  echo ''
  echo '    Only the last error encountered is reported.  The RC is a positive'
  echo '    integer representing one of the following errors.'
  echo ''                                                                              # d17513 - wjr
  echo '    These errors apply to NCCL checks:' 
  echo ''
  echo '    RC   Alert  Description'                                                   # d16511 - wjr
  echo '    --   -----  ----------------------------------------------------------'    # d16511 - wjr
  echo '     1   10001  Microcode is not available in the process list'                # d16511 - wjr
  echo '     2   10002  Raid controller CLI interface is not available'                # d16511 - wjr
  echo '     3   10003  Raid controller CLI interface is not responding'               # d16511 - wjr
  echo '     4   10004  Raid controller Ctlr<0|1>(<state>) not in a valid state'       # d17587 - wjr
  echo '     5   10005  Raid controller Ctlr<0|1> not in a valid state'                # d17587 - wjr
  echo '     6   10006  Query for raid controller state failed'                        # d17587 - wjr
  echo '     7   10007  Ctlr<0|1> firmware upgrade is in a failed state'               # d17587 - wjr
  echo '     8   10008  Query for raid controller firmware upgrade status failed'      # d17587 - wjr
  echo '     9   10009  Overall DDM firmware upgrade is in a failed state'             # d17587 - wjr
  echo '    10   10010  Query for DDM firmware upgrade status failed'                  # d17587 - wjr
  echo '    11   10011  Found one or more failed or missing drives'                    # d17587 - wjr
  echo '    12   10012  Query for failed or missing drives failed'                     # d17587 - wjr
  echo '    13   10013  Drive in slot <slot> with master P/N <pn> is INVALID'          # d17587 - wjr
  echo '    14   10014  Query for master P/N of drive in slot <slot> returned a null'  # d17587 - wjr
  echo '    15   10015  Query for master P/N of drive in slot <slot> failed'           # d17587 - wjr
  echo '    16   10016  Query for drive slot ids failed'                               # d17587 - wjr
  echo '    17   10017  Query for valid drive master P/Ns returned a null'             # d17587 - wjr
  echo '    18   10018  Query for valid drive master P/Ns failed'                      # d17587 - wjr
  echo '    19   10019  Found one or more failed/non-viable drive pools'               # d17587 - wjr
  echo '    20   10020  Query for drive pool state failed'                             # d17587 - wjr
  echo '    21   10021  Found one or more critical alerts'                             # d17587 - wjr
  echo '    22   10022  Query for critical alerts failed'                              # d17587 - wjr
  echo '    23   10023  CLI command to configure alert failed'                         # d17587 - wjr
  echo '    24   10024  CLI command to configure alert failed to execute'              # d17587 - wjr
  echo '    25   10025  CLI command to generate alert failed'                          # d17587 - wjr
  echo '    26   10026  CLI command to generate alert failed to execute'               # d17587 - wjr
  echo '    32   10032  Could not find the CtlrFwMgmt.conf config file'                # d17587 - wjr
  echo '    33   10033  Query for Rc <component> min fw level failed'                  # d17587 - wjr
  echo '    34   10034  Query for Rc <component> max fw level failed'                  # d17587 - wjr
  echo '    35   10035  Query for Rc <component> fw level failed'                      # d17587 - wjr
  echo '    36   10036  <component> come-from fw level is incompatible'                # d17587 - wjr
  echo '    37   10037  <component> go-to level is incompatible'                       # d17587 - wjr
  echo '    38   10038  Found one or more unsupported drives'                          # d17587 - wjr
  echo '    39   10039  Query for unsupported drives failed'                           # d17587 - wjr
  echo '    40   10040  Query for raid controller configuration failed.'               # d18107 - wjr
  echo ''
}
# =====================================================================================
function log
{
  typeset ACTION=$1
  typeset DATE
  DATE=$(date +'%a %D %T')
  case $ACTION in
    open )
      remove_file $LOG
      echo "### Begin execution of $SCRIPT - $DATE" > $LOG
      echo "MSG : Invocation : $COMMAND $ARGS" >> $LOG
      echo "MSG : Version    : $VERSION" >> $LOG
      ;;
    close )
      echo "### End execution of $SCRIPT - $DATE" >> $LOG
      ;;
    * )
      echo "ERROR : Invalid action on log: $ACTION."
      script_abort 1
      ;;
  esac
}
# =====================================================================================
function handle_interrupt
{
  typeset ACTION=$1
  typeset USER_RESPONSE
  case $ACTION in
    terminate )
      case $MODE in
        interactive )
          echo 'Interrupt detected - terminating script.'
          script_cancel
          ;;
        * )
          echo 'WARNING : Interrupt detected - terminating script.'
          script_cancel
          ;;
      esac
      ;;
    ask )
      case $MODE in
        interactive )
          echo 'Interrupt detected - are you sure you want to quit (y/n)?'
          read USER_RESPONSE
          if [[ y = $USER_RESPONSE || \
                Y = $USER_RESPONSE ]]; then
            script_cancel
          fi
          ;;
        background )
          echo "WARNING : Interrupt detected - terminating script."
          script_cancel
          ;;
        * )
          echo 'Interrupt detected - are you sure you want to quit (y/n)?'
          read USER_RESPONSE
          if [[ y = $USER_RESPONSE || \
                Y = $USER_RESPONSE ]]; then
            script_cancel
          fi
          ;;
      esac
      ;;
    none )
      echo 'WARNING : Interrupt detected - no action taken.'
      ;;
    * )
      case $MODE in
        interactive )
          echo "ERROR: Invalid action on interrupt : $ACTION."
          script_cancel
          ;;
        * )
          echo "ERROR: Invalid action on interrupt : $ACTION."
          script_cancel
          ;;
      esac
      ;;
  esac
  CMD_EXIT='restart'
}
# =====================================================================================
function xprint
{
  if [[ debug = $1 ]]; then
    shift
    if [[ 'y' = $VERBOSE_DEBUG ]]; then
      echo $1
    fi
  else
    if [[ 'y' = $VERBOSE ]]; then
      echo $1
    fi
  fi
  echo $1 >> $LOG
}
# =====================================================================================
function script_abort
{
  trap 'handle_interrupt none' SIGINT SIGTERM SIGTSTP SIGQUIT
  remove_file $CMD_FILE
  if [[ -z $SCRIPT ]]; then
    SCRIPT='Storage Blade healthcheck script'
  fi
  if [[ -n $FUNCTION ]]; then
    if [[ main = $FUNCTION ]]; then
      echo "$SCRIPT failed in main, rc = $1."
    else
      echo "$SCRIPT failed in function $FUNCTION, rc = $1."
    fi
  else
    echo "$SCRIPT failed, rc = $1."
  fi
  append_file $SUMMARY_LOG $LOG                                                        # d17429 - wjr
  log close
  exit $1
}
# =====================================================================================
function script_cancel
{
  trap 'handle_interrupt none' SIGINT SIGTERM SIGTSTP SIGQUIT
  remove_file $CMD_FILE
  if [[ -z $SCRIPT ]]; then
    SCRIPT='Storage Blade healthcheck script'
  fi
  if [[ -n $FUNCTION ]]; then
    if [[ main = $FUNCTION ]]; then
      echo "$SCRIPT cancelled in main."
    else
      echo "$SCRIPT cancelled in function $FUNCTION."
    fi
  else
    echo "$SCRIPT cancelled."
  fi
  append_file $SUMMARY_LOG $LOG                                                        # d17429 - wjr
  log close
  exit 0
}
# =====================================================================================
function check_rc
{
  typeset FUNCTION='check_rc'
  xprint debug "TRC: Enter function $FUNCTION"
  trap 'handle_interrupt none' SIGINT SIGTERM SIGTSTP SIGQUIT
  typeset N_ARGS=$#
  if [[ 2 != $N_ARGS ]]; then
    xprint "ERROR: check_rc function called with wrong number of args : $N_ARGS."
    FUNCTION='check_rc'
    script_abort 1
  fi
  typeset RETURN_CODE=$1
  typeset LOCATION=$2
  if [[ 0 != $RETURN_CODE ]]; then
    xprint "ERROR: Non-zero return code."
    script_abort $LOCATION
  fi
  xprint debug "TRC: Exit function $FUNCTION"
}
# =====================================================================================
function make_log_dir                                                                  # d16051 - wjr
{                                                                                      # d16051 - wjr
  if [[ ! -d $LOG_DIR ]]; then                                                         # d16051 - wjr
    mkdir $LOG_DIR                                                                     # d16051 - wjr
    if [[ ! -d $LOG_DIR ]]; then                                                       # d16051 - wjr
      echo "ERROR: Failed to create $SCRIPT log directory $LOG_DIR."                   # d16051 - wjr
      script_abort 1                                                                   # d16051 - wjr
    fi                                                                                 # d16051 - wjr
  fi                                                                                   # d16051 - wjr
}                                                                                      # d16051 - wjr
# =====================================================================================
function remove_file
{
  typeset FUNCTION='remove_file'
  xprint debug "TRC: Enter function $FUNCTION"
  typeset FILE=$1
  if [[ -f $FILE ]]; then
    rm -f $FILE
  fi
  xprint debug "TRC: Exit function $FUNCTION"
}
# =====================================================================================
function copy_file                                                                     # d17429 - wjr
{                                                                                      # d17429 - wjr
  typeset FUNCTION='copy_file'                                                         # d17429 - wjr
  xprint debug "TRC: Enter function $FUNCTION"                                         # d17429 - wjr
  typeset SOURCE_FILE=$1                                                               # d17429 - wjr
  typeset TARGET_FILE=$2                                                               # d17429 - wjr
  remove_file $TARGET_FILE                                                             # d17429 - wjr
  if [[ -f $SOURCE_FILE ]]; then                                                       # d17429 - wjr
    cp $SOURCE_FILE $TARGET_FILE                                                       # d17429 - wjr
    RC=$?                                                                              # d17429 - wjr
    if (( 0 != $RC )); then                                                            # d17429 - wjr
      xprint debug "MSG: Copy of $SOURCE_FILE failed - cp returned rc=$RC."            # d17429 - wjr
      script_abort 1                                                                   # d17429 - wjr
    fi                                                                                 # d17429 - wjr
  else                                                                                 # d17429 - wjr
    xprint debug "MSG: Copy of $SOURCE_FILE failed - file does not exist."             # d17429 - wjr
    script_abort 2                                                                     # d17429 - wjr
  fi                                                                                   # d17429 - wjr
  xprint debug "TRC: Exit function $FUNCTION"                                          # d17429 - wjr
}                                                                                      # d17429 - wjr
# =====================================================================================
function append_file                                                                   # d17429 - wjr
{
  typeset FUNCTION='append_file'                                                       # d17429 - wjr
  xprint debug "TRC: Enter function $FUNCTION"
  typeset SOURCE_FILE=$1
  typeset TARGET_FILE=$2
  typeset TMP_FILE=$LOG_DIR'/copy.tmp'
  if [[ -f $SOURCE_FILE ]]; then
    cat $TARGET_FILE $SOURCE_FILE > $TMP_FILE
    RC=$?
    if (( 0 == $RC )); then
      mv $TMP_FILE $TARGET_FILE
      RC=$?
      if (( 0 != $RC )); then
        xprint debug "MSG: Appending $SOURCE_FILE failed - mv returned rc=$RC."        # d17429 - wjr
      fi
    else
      xprint debug "MSG: Appending $SOURCE_FILE failed - cat returned rc=$RC."         # d17429 - wjr
    fi
  else
    xprint debug "MSG: $SOURCE_FILE file does not exist - no append."                  # d17513 - wjr
  fi
  xprint debug "TRC: Exit function $FUNCTION"
}
# =====================================================================================
function check_arguments
{
  typeset FUNCTION='check_arguments'
  xprint debug "TRC: Enter function $FUNCTION"
  trap 'handle_interrupt ask' SIGINT SIGTERM SIGTSTP SIGQUIT
  if (( N_OPERATIONS > 1 )); then
    echo "ERROR: Invalid command line arguments - multiple options selected."
    show_help
    script_abort 1
  fi
  case $OPERATION in
    nccl_check )                                                                       # d16511 - wjr
      :
      ;;
    * )
      echo "ERROR: Invalid operation : $OPERATION."
      show_help
      script_abort 2
  esac 
  case $MODE in
    interactive | background )
      :
      ;;
    * )
      echo "ERROR: Invalid execution mode : $MODE."
      show_help
      script_abort 3
      ;;
  esac 
  xprint debug "TRC: Exit function $FUNCTION"
}
# =====================================================================================
function run_cmd
{
  typeset FUNCTION='run_cmd'
  xprint debug "TRC: Enter function $FUNCTION"
  trap 'handle_interrupt ask' SIGINT SIGTERM SIGTSTP SIGQUIT
  typeset OUTPUT='assign'
  xprint "MSG: Running RC system command:"
  if [[ -z $CMD ]]; then
    echo "ERROR: System command is undefined."
    script_abort 1
  fi
  if [[ -z $CHECK_TYPE ]]; then
    CHECK_TYPE='continue'
  fi
  xprint "CMD: $CMD"
  if [[ -n $1 ]]; then
    OUTPUT=$1
  fi
  if [[ 'file' == $OUTPUT ]]; then
    remove_file $CMD_FILE
    CMD_EXIT='restart'
    while [[ restart = $CMD_EXIT ]];
    do
      CMD_EXIT='continue'
      eval "$CMD 1>$CMD_FILE"
      RC=$?
    done
  else
    CMD='RESULT=$('$CMD')'
    eval "$CMD"
    RC=$?
  fi
  case $CHECK_TYPE in
    check )
      check_rc $RC 2
      ;;
    continue )
      :
      ;;
    * )
      echo "ERROR: Invalid command return code check: $CHECK_TYPE."
      script_abort 3
      ;;
  esac
  xprint debug "TRC: Exit function $FUNCTION"
  return $RC
}
# =====================================================================================
function prt_cli_session
{
  typeset FUNCTION='show_cli_session'
  xprint debug "TRC: Enter function $FUNCTION"
  if [[ 'y' = $VERBOSE_CLI ]]; then 
    echo 'CLI Session results --------------------------------------------------' >> $LOG
    if [[ -f $CMD_FILE ]]; then
      cat $CMD_FILE >> $LOG
    else
      echo '< Empty file >' >> $LOG
    fi
    echo '----------------------------------------------------------------------' >> $LOG
  fi
  xprint debug "TRC: Exit function $FUNCTION"
}
# =====================================================================================
function prt_test_result
{
  typeset FUNCTION='prt_test_result'
  xprint debug "TRC: Enter function $FUNCTION"
  typeset STANZA=$1
  shift
  typeset TEST_NAME=$*
  if [[ 'y' = $SUMMARY ]]; then 
    case $STANZA in
      header )
        typeset TEST_TYPE=$*
        remove_file $SUMMARY_LOG
        echo ''
        echo '' > $SUMMARY_LOG
        TEN_DASHES='----------'
        FIFTY_DASHES='--------------------------------------------------'
        printf "%-50.50s %10s\n" "$TEST_TYPE Healthcheck Test" 'Status'
        printf "%-50.50s %10s\n" "$TEST_TYPE Healthcheck Test" 'Status' >> $SUMMARY_LOG
        printf "%-50.50s %10s\n" $FIFTY_DASHES $TEN_DASHES
        printf "%-50.50s %10s\n" $FIFTY_DASHES $TEN_DASHES >> $SUMMARY_LOG
        echo ''
        echo '' >> $SUMMARY_LOG
        ;;
      passed )
        printf "%-50.50s %10s\n" "$TEST_NAME" 'Passed'
        printf "%-50.50s %10s\n" "$TEST_NAME" 'Passed' >> $SUMMARY_LOG
        ;;
      failed )
        printf "%-50.50s %10s\n" "$TEST_NAME" 'FAILED'
        printf "%-50.50s %10s\n" "$TEST_NAME" 'FAILED' >> $SUMMARY_LOG
        ;;
      skipped )
        printf "%-50.50s %10s\n" "$TEST_NAME" 'Skipped'
        printf "%-50.50s %10s\n" "$TEST_NAME" 'Skipped' >> $SUMMARY_LOG
        ;;
      space )
        echo ''
        echo '' >> $SUMMARY_LOG
        ;;
      trailer )
        echo ''
        echo '' >> $SUMMARY_LOG
        ;;
      * )
        echo "ERROR: Invalid $FUNCTION argument : $STANZA."
        script_abort 1
        ;;
    esac
  fi
  xprint debug "TRC: Exit function $FUNCTION"
}
# =====================================================================================
function set_rc
{
  typeset FUNCTION='set_rc'
  xprint debug "TRC: Enter function $FUNCTION"
  trap 'handle_interrupt none' SIGINT SIGTERM SIGTSTP SIGQUIT
  typeset -i RC=$1                                                                     # d16723 - wjr
  if [[ -n $RC ]]; then
    if (( $RC > 0 )); then                                                             # d16511 - wjr
      RC_FORMATTED=$(printf "%02d" $1)                                                 # d17513 - wjr
#     if [[ 'undefined' == $CTLR_ID ]]; then                                           # d17587 - wjr
      ALERT='100'$RC_FORMATTED                                                         # d17513 - wjr
#     else                                                                             # d17587 - wjr
#       CTLR_N=$(echo $CTLR_ID | tr -d [A-Za-z])                                       # d17587 - wjr
#       ALERT='10'$CTLR_N$RC_FORMATTED                                                 # d17587 - wjr
#     fi                                                                               # d17587 - wjr
      if [[ -n $ALERT ]]; then                                                         # d16511 - wjr
        shift                                                                          # d17429 - wjr
        typeset ALERT_MSG=$*                                                           # d16723 - wjr
        xprint "ERROR: $ALERT_MSG"                                                     # d16511 - wjr
        configure_alert_template $ALERT $ALERT_MSG                                     # d16511 - wjr
        generate_alert $ALERT                                                          # d16511 - wjr
      fi                                                                               # d16511 - wjr
    fi                                                                                 # d16511 - wjr
    if (( $RC > $RETURN_CODE )); then
      xprint "MSG: Setting overall return code to $RC"
      RETURN_CODE=$RC
    fi
  fi
  xprint debug "TRC: Exit function $FUNCTION"
}
# =====================================================================================
function append_err_msg                                                                # d17513 - wjr
{                                                                                      # d17513 - wjr
  typeset FUNCTION='append_err_msg'                                                    # d17513 - wjr
  xprint debug "TRC: Enter function $FUNCTION"                                         # d17513 - wjr
  typeset MSG_ID=$1                                                                    # d17513 - wjr
  shift                                                                                # d17513 - wjr
  typeset MSG=$*                                                                       # d17513 - wjr
  if [[ '' == ${ERR_MSG[$MSG_ID]} ]]; then                                             # d17513 - wjr
    ERR_MSG[$MSG_ID]=$MSG                                                              # d17513 - wjr
  else                                                                                 # d17513 - wjr
    ERR_MSG[$MSG_ID]=${ERR_MSG[$MSG_ID]}', '$MSG                                       # d17513 - wjr
  fi                                                                                   # d17513 - wjr
  xprint debug "TRC: Exit function $FUNCTION"                                          # d17513 - wjr
}                                                                                      # d17513 - wjr
# =====================================================================================
function query_config                                                                  # d18107 - wjr
{                                                                                      # d18107 - wjr
  typeset FUNCTION='query_config'                                                      # d18107 - wjr
  xprint debug "TRC: Enter function $FUNCTION"                                         # d18107 - wjr
  trap 'handle_interrupt ask' SIGINT SIGTERM SIGTSTP SIGQUIT                           # d18107 - wjr
  xprint "MSG: Query RSSM configuration."                                              # d18107 - wjr
  typeset CTLR_IDS='0 1'                                                               # d18107 - wjr
  typeset N_ACTIVE=0                                                                   # d18107 - wjr
  CMD='echo "list controller" | /opt/arts/Cli/cli/linux_ppc/cli'                       # d18107 - wjr
  typeset QUERY_CONFIGS='NOT_PRESENT
                         NOTEXISTS'                                                    # d18107 - wjr
  ACTIVE_CTLR_IDS=''                                                                   # d18107 - wjr
  run_cmd file                                                                         # d18107 - wjr
  if [[ -f $CMD_FILE ]]; then                                                          # d18107 - wjr
    xprint "MSG: Query for raid controller configuration successful."                  # d18107 - wjr
    prt_cli_session                                                                    # d18107 - wjr
    for ctlr in $CTLR_IDS                                                              # d18107 - wjr
    do                                                                                 # d18107 - wjr
      MSG_TXT="Ctlr$ctlr Raid Controller Configuration"                                # d18107 - wjr
      xprint "MSG: Checking Ctlr$ctlr raid controller configuration."                  # d18107 - wjr
      RC_CONFIG=''                                                                     # d18107 - wjr
      for config in $QUERY_CONFIGS                                                     # d18107 - wjr
      do                                                                               # d18107 - wjr
        RESULT=$(cat $CMD_FILE | \
                 grep $config | \
                 grep Ctlr$ctlr)                                                       # d18107 - wjr
        if [[ -n $RESULT ]]; then                                                      # d18107 - wjr
          RC_CONFIG=$config                                                            # d18107 - wjr
          xprint "MSG: Ctlr$ctlr is not present."                                      # d18107 - wjr
          break                                                                        # d18107 - wjr
        fi                                                                             # d18107 - wjr
      done                                                                             # d18107 - wjr
      if [[ '' == $RC_CONFIG ]]; then                                                  # d18107 - wjr
        xprint "MSG: Ctlr$ctlr is present."                                            # d18107 - wjr
        ACTIVE_CTLR_IDS=$(echo $ACTIVE_CTLR_IDS $ctlr)                                 # d18107 - wjr
      fi                                                                               # d18107 - wjr
    done                                                                               # d18107 - wjr
  else                                                                                 # d18107 - wjr
    if [[ 'silent' != $MODE ]]; then                                                   # d18107 - wjr
      prt_test_result failed $MSG_TXT                                                  # d18107 - wjr
      set_rc 40 "$SCRIPT query for raid controller configuration failed."              # d18107 - wjr
    fi                                                                                 # d18107 - wjr
    RC=40                                                                              # d18107 - wjr
  fi                                                                                   # d18107 - wjr
  if [[ -n $ACTIVE_CTLR_IDS ]]; then                                                   # d18107 - wjr
    ACTIVE_LIST=''                                                                     # d18107 - wjr
    for ctlr_id in $ACTIVE_CTLR_IDS                                                    # d18107 - wjr
    do                                                                                 # d18107 - wjr
      ACTIVE_LIST=$(echo $ACTIVE_LIST ', Ctlr'$ctlr_id)                                # d18107 - wjr
    done                                                                               # d18107 - wjr
    xprint "MSG: Raid controllers present: $ACTIVE_LIST"                               # d18107 - wjr
    N_ACTIVE="$(echo $ACTIVE_CTLR_IDS | wc -w)"                                        # d18107 - wjr
    if (( N_ACTIVE > 1 )); then                                                        # d18107 - wjr
      ACTIVE_GROUP='both'                                                              # d18107 - wjr
    else                                                                               # d18107 - wjr
      ACTIVE_GROUP='Ctlr'$ACTIVE_CTLR_IDS                                              # d18107 - wjr
    fi                                                                                 # d18107 - wjr
  else                                                                                 # d18107 - wjr
    xprint "ERROR: No Raid controller controllers are present."                        # d18107 - wjr
    script_abort 1                                                                     # d18107 - wjr
  fi                                                                                   # d18107 - wjr
  xprint debug "TRC: Exit function $FUNCTION"                                          # d18107 - wjr
  return $RC                                                                           # d18107 - wjr
}                                                                                      # d18107 - wjr
# =====================================================================================
function query_address                                                                 # d16051 - wjr
{                                                                                      # d16051 - wjr
  typeset FUNCTION='query_address'                                                     # d16051 - wjr
  xprint debug "TRC: Enter function $FUNCTION"                                         # d16051 - wjr
  trap 'handle_interrupt ask' SIGINT SIGTERM SIGTSTP SIGQUIT                           # d16051 - wjr
  xprint "MSG: Query raid controller IP address."                                      # d16051 - wjr
  typeset RC=0                                                                         # d16723 - wjr
  CMD='ifconfig eth0.4095 | grep Bcast | cut -d ":" -f 2 | cut -d " " -f 1'            # d16051 - wjr
  run_cmd                                                                              # d16051 - wjr
  if [[ -n $RESULT ]]; then                                                            # d16051 - wjr
    IP_ADDRESS=$RESULT                                                                 # d16051 - wjr
    xprint "MSG: Raid controller IP address: $IP_ADDRESS"                              # d16051 - wjr
  else                                                                                 # d16051 - wjr
    xprint "ERROR: Query for Raid controller IP address failed."                       # d16051 - wjr
    script_abort 1                                                                     # d16051 - wjr
  fi                                                                                   # d16051 - wjr
  xprint debug "TRC: Exit function $FUNCTION"
}                                                                                      # d16051 - wjr
# =====================================================================================
function query_ctlr_id                                                                 # d16051 - wjr
{                                                                                      # d16051 - wjr
  typeset FUNCTION='query_ctlr_id'                                                     # d16051 - wjr
  xprint debug "TRC: Enter function $FUNCTION"                                         # d16051 - wjr
  trap 'handle_interrupt ask' SIGINT SIGTERM SIGTSTP SIGQUIT                           # d16051 - wjr
  xprint "MSG: Query raid controller ID."                                              # d16051 - wjr
  CMD='echo "commparams -get" | /opt/arts/Cli/cli/linux_ppc/cli'                       # d16051 - wjr
  run_cmd file                                                                         # d16051 - wjr
  if [[ -f $CMD_FILE ]]; then                                                          # d16051 - wjr
    for ctlr_id in $ACTIVE_CTLR_IDS                                                    # d18107 - wjr
    do                                                                                 # d16051 - wjr
      IP_INFO=$(cat $CMD_FILE | \
                sed -n "/Ctlr$ctlr_id/,/GATEWAY/p" | \
                grep $IP_ADDRESS)                                                      # d16787 - wjr
      if [[ -n $IP_INFO ]]; then                                                       # d16051 - wjr
        CTLR_ID='Ctlr'$ctlr_id                                                         # d16051 - wjr
        xprint "MSG: Raid controller ID is: $CTLR_ID"                                  # d16051 - wjr
        break                                                                          # d16051 - wjr
      fi                                                                               # d16051 - wjr
    done                                                                               # d16051 - wjr
    if [[ 'undefined' == $CTLR_ID ]]; then                                             # d16051 - wjr
      xprint "ERROR: Search for Ctlr ID in CLI commparams output failed."              # d16051 - wjr
      script_abort 1                                                                   # d16051 - wjr
    fi                                                                                 # d16051 - wjr
  else                                                                                 # d16051 - wjr
    xprint "ERROR: Query for raid controller ID failed."                               # d16051 - wjr
    script_abort 2                                                                     # d16051 - wjr
  fi                                                                                   # d16051 - wjr
  xprint debug "TRC: Exit function $FUNCTION"
}                                                                                      # d16051 - wjr
# =====================================================================================
function query_tsal
{
  typeset FUNCTION='query_tsal'
  xprint debug "TRC: Enter function $FUNCTION"
  trap 'handle_interrupt ask' SIGINT SIGTERM SIGTSTP SIGQUIT
  typeset MSG_TXT='Raid Controller Daemon status'
  xprint "MSG: Checking raid controller microcode state."
  typeset RC=0                                                                         # d16723 - wjr
  CMD='ps -ef | grep tsal'
  run_cmd
  if [[ -n $RESULT ]]; then
    prt_test_result passed $MSG_TXT
    xprint "MSG: The microcode process is running."
  else
    prt_test_result failed $MSG_TXT
    set_rc 1 "Microcode is not available in the process list."                         # d17513 - wjr
    RC=1
  fi
  xprint debug "TRC: Exit function $FUNCTION"
  return $RC
}
# =====================================================================================
function query_cli
{
  typeset FUNCTION='query_cli'
  xprint debug "TRC: Enter function $FUNCTION"
  trap 'handle_interrupt ask' SIGINT SIGTERM SIGTSTP SIGQUIT
  typeset MSG_TXT='Raid Controller CLI interface'
  xprint "MSG: Checking raid controller CLI interface."
  typeset RC=0                                                                         # d16723 - wjr
  CMD='echo exit | /opt/arts/Cli/cli/linux_ppc/cli'
  run_cmd
  if [[ -n $RESULT ]]; then
    xprint "MSG: Started raid controller CLI interface - getting status."
  else
    printf "%-50s %10s\n" $MSG_TXT 'FAILED'
    set_rc 2 "Raid controller CLI interface is not available."                         # d17513 - wjr
    RC=2
  fi
  CMD='echo swversion | /opt/arts/Cli/cli/linux_ppc/cli'
  run_cmd file
  RESULT=$(cat $CMD_FILE | \
           grep 'Software version')
  if [[ -n $RESULT ]]; then
    prt_test_result passed $MSG_TXT
    prt_cli_session
  else
    prt_test_result failed $MSG_TXT
    set_rc 3 "Raid controller CLI interface is not responding."                        # d17513 - wjr
    RC=3
  fi
  xprint debug "TRC: Exit function $FUNCTION"
  return $RC
}
# =====================================================================================
function verify_rc_state
{
  typeset FUNCTION='verify_rc_state'
  xprint debug "TRC: Enter function $FUNCTION"
  trap 'handle_interrupt ask' SIGINT SIGTERM SIGTSTP SIGQUIT
  typeset VERIFY_STATE=$1
  typeset MODE=$2
  typeset CTLRS=$3                                                                     # d16787 - wjr
  typeset QUERY_STATES='SYSCONFIG
                        STARTING
                        BINDING
                        RUNNING
                        MASTERBOUND
                        SLAVEBOUND
                        PRIMARY
                        SECONDARY
                        STANDALONE
                        STANDBY
                        POWERFAILRECOVERY
                        WAITFORREBOOT
                        SURVIVOR
                        STARTASSURVIVOR
                        NOT_PRESENT
                        FAILED
                        REBINDING
                        NOTAVAILABLE
                        FAILINGOVER
                        SERVICE
                        SHUTDOWN
                        UNKNOWN
                        BATTERYFAILRECOVERY
                        SHTDN_IND_DATA_LOSS
                        NOTEXISTS
                        BOUND_PFR1
                        BOUND_PFR2
                        BOUND_PFRS
                        SURVIVOR_PFR1
                        SURVIVOR_PFR2'                                                 # d17697 - wjr
  typeset MSG_TXT='Raid Controller State'
  typeset CTLR_IDS='undefined'                                                         # d16787 - wjr
  typeset VALID_STATES=''
  typeset RC_STATE=''
  case $CTLRS in                                                                       # d16787 - wjr
    Ctlr0 )                                                                            # d16787 - wjr
      CTLR_IDS='0'                                                                     # d16787 - wjr
      ;;                                                                               # d16787 - wjr
    Ctlr1 )                                                                            # d16787 - wjr
      CTLR_IDS='1'                                                                     # d16787 - wjr
      ;;                                                                               # d16787 - wjr
    self )                                                                             # d16787 - wjr
      CTLR_IDS=''                                                                      # d16787 - wjr
      ;;                                                                               # d16787 - wjr
    both )                                                                             # d16787 - wjr
      CTLR_IDS='0 1'                                                                   # d16787 - wjr
      ;;                                                                               # d16787 - wjr
    * )                                                                                # d16787 - wjr
      echo "ERROR: Invalid $FUNCTION argument CTLRS: $CTLRS."                          # d16787 - wjr
      script_abort 1                                                                   # d16787 - wjr
  esac                                                                                 # d16787 - wjr
  case $VERIFY_STATE in
    PRESENT )                                                                          # d18107 - wjr
      VALID_STATES='NOT_PRESENT'                                                       # d18107 - wjr
      ;;                                                                               # d18107 - wjr
    SERVICE )
      VALID_STATES='SERVICE'
      ;;
    BOUND )
      VALID_STATES='STANDALONE|PRIMARY|SECONDARY|MASTER|SLAVE'                         # d18107 - wjr
      ;;
    * )
      echo "ERROR: Invalid $FUNCTION argument: $VERIFY_STATE."
      script_abort 2
      ;;
  esac
  typeset RC=0                                                                         # d16723 - wjr
  xprint "MSG: Query CLI for raid controller state."
  CMD='echo "list controller" | /opt/arts/Cli/cli/linux_ppc/cli'
  run_cmd file
  if [[ -f $CMD_FILE ]]; then
    xprint "MSG: Query for raid controller status successful."
    prt_cli_session
    for ctlr in $CTLR_IDS
    do
      MSG_TXT="Ctlr$ctlr Raid Controller State"
      xprint "MSG: Checking Ctlr$ctlr raid controller state."
      RC_STATE=''
      for state in $QUERY_STATES
      do
        RESULT=$(cat $CMD_FILE | \
                 grep $state | \
                 grep Ctlr$ctlr)
        if [[ -n $RESULT ]]; then
          RC_STATE=$state
          xprint "MSG: Ctlr$ctlr state: $RC_STATE."
          break
        fi
      done
      if [[ -n $RC_STATE ]]; then
        if [[ -n $(echo $RC_STATE | grep -E "$VALID_STATES") ]]; then
          if [[ 'silent' != $MODE ]]; then
            prt_test_result passed $MSG_TXT
          fi
        else
          if [[ 'silent' != $MODE ]]; then
            prt_test_result failed $MSG_TXT
            append_err_msg 4 'Ctlr'$ctlr'('$RC_STATE')'                                # d17513 - wjr
          fi
          RC=4
        fi
      else
        if [[ 'silent' != $MODE ]]; then
          prt_test_result failed $MSG_TXT
          append_err_msg 5 'Ctlr'$ctlr                                                 # d17513 - wjr
        fi
        RC=5
      fi
    done
    if [[ 'silent' != $MODE ]]; then                                                   # d17513 - wjr
      if [[ '' != ${ERR_MSG[4]} ]]; then                                               # d17513 - wjr
        set_rc 4 "Rc ${ERR_MSG[4]} not in a valid state."                              # d17513 - wjr
      fi                                                                               # d17513 - wjr
      if [[ '' != ${ERR_MSG[5]} ]]; then                                               # d17513 - wjr
        set_rc 5 "Rc ${ERR_MSG[5]} not in a valid state."                              # d17513 - wjr
      fi                                                                               # d17513 - wjr
    fi                                                                                 # d17513 - wjr
  else
    if [[ 'silent' != $MODE ]]; then
      prt_test_result failed $MSG_TXT
      set_rc 6 "$SCRIPT query for raid controller state failed."                       # d17587 - wjr
    fi
    RC=6
  fi
  xprint debug "TRC: Exit function $FUNCTION"
  return $RC
}
# =====================================================================================
function query_rc_fw_status
{
  typeset FUNCTION='query_rc_fw_status'
  xprint debug "TRC: Enter function $FUNCTION"
  trap 'handle_interrupt ask' SIGINT SIGTERM SIGTSTP SIGQUIT
  typeset CTLRS=$1                                                                     # d16787 - wjr
  typeset MSG_TXT='Raid Controller Firmware Update Status'
  typeset CTLR_IDS='undefined'                                                         # d16787 - wjr
  typeset GOOD_STATES='undefined'                                                      # d16051 - wjr
  typeset GOOD_STATE_LIST='undefined'                                                  # d17697 - wjr
  typeset RC=0                                                                         # d16723 - wjr
  xprint "MSG: Query CLI for RC firmware upgrade status."
  case $CTLRS in                                                                       # d16787 - wjr
    Ctlr0 )                                                                            # d16787 - wjr
      CTLR_IDS='0'                                                                     # d16787 - wjr
      ;;                                                                               # d16787 - wjr
    Ctlr1 )                                                                            # d16787 - wjr
      CTLR_IDS='1'                                                                     # d16787 - wjr
      ;;                                                                               # d16787 - wjr
    self )                                                                             # d16787 - wjr
      CTLR_IDS=''                                                                      # d16787 - wjr
      ;;                                                                               # d16787 - wjr
    both )                                                                             # d16787 - wjr
      CTLR_IDS='0 1'                                                                   # d16787 - wjr
      ;;                                                                               # d16787 - wjr
    undefined )                                                                        # d16787 - wjr
      CTLR_IDS='0 1'                                                                   # d16787 - wjr
      ;;                                                                               # d16787 - wjr
    * )                                                                                # d16787 - wjr
      echo "ERROR: Invalid $FUNCTION argument CTLRS: $CTLRS."                          # d16787 - wjr
      script_abort 1                                                                   # d16787 - wjr
  esac                                                                                 # d16787 - wjr
  CMD='echo "firmwareupgrade -getstatus" | /opt/arts/Cli/cli/linux_ppc/cli'
  run_cmd file
  if [[ -f $CMD_FILE ]]; then
    xprint "MSG: Query for firmware upgrade status successful."
    prt_cli_session
    for ctlr in $CTLR_IDS
    do
      MSG_TXT="Ctlr$ctlr Firmware Update Status"
      xprint "MSG: Checking Ctlr$ctlr firmware upgrade status."
      if [[ $ctlr == $CTLR_ID ]]; then                                                 # d16051 - wjr
        GOOD_STATES='FW_UPGRADE_IDLE'                                                  # d16051 - wjr
      else                                                                             # d16051 - wjr
        GOOD_STATES='FW_UPGRADE_IDLE|
                     FW_UPGRADE_PREVERIFYSTARTED|
                     FW_UPGRADE_INITIALIZING|
                     FW_UPGRADE_DONE|
                     FW_UPGRADE_NOTREQUIRED|
                     FW_UPGRADE_NO_PEER|
                     FW_UPGRADE_SCHEDULED|
                     FW_UPGRADE_BMCUPGRADESTARTED| 
                     FW_UPGRADE_BMCUPGRADEDONE|
                     FW_UPGRADE_SESUPGRADESTARTED|
                     FW_UPGRADE_SESUPGRADEDONE|
                     FW_UPGRADE_NEWCOMPONENT|
                     FW_UPGRADE_CPLDUPGRADESTARTED|
                     FW_UPGRADE_CPLDUPGRADEDONE|
                     FW_UPGRADE_FPGAUPGRADESTARTED|
                     FW_UPGRADE_FPGAUPGRADEDONE|
                     FW_UPGRADE_SASUPGRADESTARTED|
                     FW_UPGRADE_SASUPGRADEDONE|
                     FW_UPGRADE_FIRSTDSMUPGRADEDONE|
                     FW_UPGRADE_DSMUPGRADEDONE|
                     FW_UPGRADE_PUSHFW_STARTED|
                     FW_UPGRADE_READYTOBOOT'                                           # d17697 - wjr
      fi                                                                               # d16051 - wjr
      FW_STATE=$(cat $CMD_FILE | \
                 grep "Ctlr $ctlr" | \
                 grep "Status" | \
                 cut -d ' ' -f 7)                                                      # d17697 - wjr
      if [[ -n $FW_STATE ]]; then
        xprint "MSG: Ctlr$ctlr firmware upgrade status: $FW_STATE."
        GOOD_STATE_LIST=$(echo $GOOD_STATES | tr -d ' ')                               # d17697 - wjr
        if [[ -n $(echo $FW_STATE | grep -E "$GOOD_STATE_LIST") ]]; then               # d17697 - wjr
          if [[ 'silent' != $MODE ]]; then
            prt_test_result passed $MSG_TXT
          fi
        else
          if [[ 'silent' != $MODE ]]; then
            prt_test_result failed $MSG_TXT
            append_err_msg 7 'Ctlr'$ctlr'('$FW_STATE')'                                # d17587 - wjr
          fi
          RC=7
        fi
      else
        if [[ 'silent' != $MODE ]]; then                                               # d17697 - wjr
          prt_test_result failed $MSG_TXT                                              # d17697 - wjr
          append_err_msg 7 'Ctlr'$ctlr                                                 # d17513 - wjr
        fi                                                                             # d17697 - wjr
        RC=7                                                                           # d17697 - wjr
      fi                                                                               # d17697 - wjr
    done
    if [[ '' != ${ERR_MSG[7]} ]]; then                                                 # d17513 - wjr
      set_rc 7 "${ERR_MSG[7]} is in an invalid fw upgrade state."                      # d17697 - wjr
    fi                                                                                 # d17513 - wjr
  else
    prt_test_result failed $MSG_TXT
    set_rc 8 "$SCRIPT query for raid controller firmware upgrade status failed."       # d17587 - wjr
    RC=8
  fi
  xprint debug "TRC: Exit function $FUNCTION"
  return $RC
}
# =====================================================================================
function query_ddm_fw_status
{
  typeset FUNCTION='query_ddm_fw_status'
  xprint debug "TRC: Enter function $FUNCTION"
  trap 'handle_interrupt ask' SIGINT SIGTERM SIGTSTP SIGQUIT
  typeset MSG_TXT='Disk Drive Firmware Update Status'
  typeset GOOD_STATES='IDLE
                       NO_CANDIDATE_DRIVE
                       FAILED
                       DONE'                                                           # d16051 - wjr
  typeset RC=0                                                                         # d16723 - wjr
  xprint "MSG: Query CLI for DDM firmware upgrade status."
  CMD='echo "firmwareupgrade -getstatus" | /opt/arts/Cli/cli/linux_ppc/cli'
  run_cmd file
  if [[ -f $CMD_FILE ]]; then
    xprint "MSG: Query for firmware upgrade status successful."
    prt_cli_session
    xprint "MSG: Checking DDM firmware upgrade status."
    FW_STATE=''
    for state in $GOOD_STATES
    do
      state=$(echo $state | tr '_' ' ')
      RESULT=$(cat $CMD_FILE | \
               grep "$state" | \
               grep "Drive" | \
               grep "Status")
      if [[ -n $RESULT ]]; then
        FW_STATE=$state
        xprint "MSG: Overall DDM firmware upgrade status: $FW_STATE."
        break
      fi
    done
    if [[ -n $FW_STATE ]]; then
      prt_test_result passed $MSG_TXT
    else
      prt_test_result failed $MSG_TXT
      set_rc 9 "Overall DDM firmware upgrade is in a failed state: $FS_STATE."         # d17587 - wjr
      RC=9
    fi
  else
    prt_test_result failed $MSG_TXT
    set_rc 10 "$SCRIPT query for DDM firmware upgrade status failed."                  # d17587 - wjr
    RC=10
  fi
  xprint debug "TRC: Exit function $FUNCTION"
  return $RC
}
# =====================================================================================
function query_bad_drives
{
  typeset FUNCTION='query_bad_drives'
  xprint debug "TRC: Enter function $FUNCTION"
  trap 'handle_interrupt ask' SIGINT SIGTERM SIGTSTP SIGQUIT
  typeset MSG_TXT='No failed or missing disk drives'
  typeset SLOT_ID=''                                                                   # d17587 - wjr
  typeset DRIVE_STATE=''                                                               # d17587 - wjr
  typeset DRIVE_INFO=''                                                                # d17587 - wjr
  typeset BAD_STATE_LIST='M|P|U'                                                       # d17587 - wjr
  typeset EXCLUDE_LIST='UNS|UNF|IMD'                                                   # jylu - let UNS/UNF/IMD pass preverify 2013/03/04
  typeset DRIVE_MSG=''                                                                 # d17587 - wjr
  typeset RC=0                                                                         # d16723 - wjr
  xprint "MSG: Query CLI for failed or missing drives."
  CMD='echo "list drive" | /opt/arts/Cli/cli/linux_ppc/cli'
  run_cmd file
  if [[ -f $CMD_FILE ]]; then
    xprint "MSG: Query for failed or missing drives successful."
    prt_cli_session
    xprint "MSG: Checking drive state."
    RESULT=$(cat $CMD_FILE | \
             grep '|' | \
             grep -v "<CLI>" | \
             grep -v "<cli>" | \
             grep -v "_" | \
             grep -v 'Drive' | \
             grep -v 'Current Machine' | \
             grep -v 'Usage:' | \
             grep -v 'foreign' | \
             grep -v 'current' | \
             grep -v 'State:' | \
             grep -v 'initialized' | \
             grep -v 'incompatible' | \
             grep -v 'firmware' | \
             grep -v 'Error Message File' | \
             grep -v 'No drives reported' | \
             cut -d '|' -f 3,8 | \
             tr -d ' ')                                                                # d17644 - wjr
    for drive in $RESULT                                                               # d17587 - wjr
    do                                                                                 # d17587 - wjr
      SLOT_ID=$(echo $drive | cut -d '|' -f 1)                                         # d17587 - wjr
      DRIVE_STATE=$(echo $drive | cut -d '|' -f 2)                                     # d17587 - wjr
      DRIVE_INFO=$(echo $DRIVE_STATE | grep -E $BAD_STATE_LIST | grep -vE $EXCLUDE_LIST)                                                                                    # jylu - let UNS/UNF/IMD pass preverify 2013/03/04
      if [[ -z $DRIVE_INFO ]]; then                                                    # d17587 - wjr
        xprint "MSG: Drive in slot $SLOT_ID is $DRIVE_STATE."                          # d17587 - wjr
      else                                                                             # d17587 - wjr
        if [[ 'M' == $DRIVE_STATE ]]; then                                             # d17587 - wjr
          DRIVE_MSG='missing'                                                          # d17587 - wjr
        elif [[ 'P' == $DRIVE_STATE ]]; then                                           # d17587 - wjr
          DRIVE_MSG='PFA predicted failure'                                            # d17587 - wjr
        elif [[ 'U' == $DRIVE_STATE ]]; then                                           # d17587 - wjr
          DRIVE_MSG='unreliable'                                                       # d17587 - wjr
        else                                                                           # d17587 - wjr
          DRIVE_MSG=$DRIVE_STATE                                                       # d17587 - wjr
        fi                                                                             # d17587 - wjr
        append_err_msg 11 'Slot '$SLOT_ID'('$DRIVE_MSG')'                              # d17587 - wjr
        RC=11
      fi
    done                                                                               # d17587 - wjr
    if [[ '' != ${ERR_MSG[11]} ]]; then                                                # d17587 - wjr
      set_rc 11 "Found one or more failed or missing drives: ${ERR_MSG[11]}."          # d17587 - wjr
    fi                                                                                 # d17587 - wjr
  else
    set_rc 12 "$SCRIPT query for failed or missing drives failed."                     # d17587 - wjr
    RC=12
  fi
  if (( 0 == $RC )); then                                                              # d17587 - wjr
    prt_test_result passed $MSG_TXT                                                    # d17587 - wjr
  else                                                                                 # d17587 - wjr
    prt_test_result failed $MSG_TXT                                                    # d17587 - wjr
  fi                                                                                   # d17587 - wjr
  xprint debug "TRC: Exit function $FUNCTION"
  return $RC
}
# =====================================================================================
function query_unsupported_drive_pns                                                   # d17587 - wjr
{                                                                                      # d16051 - wjr
  typeset FUNCTION='query_unsupported_drive_pns'                                       # d17587 - wjr
  xprint debug "TRC: Enter function $FUNCTION"                                         # d16051 - wjr
  trap 'handle_interrupt ask' SIGINT SIGTERM SIGTSTP SIGQUIT                           # d16051 - wjr
  typeset MSG_TXT='No disk drives with unsupported p/n'                                # d17587 - wjr
  typeset RC=0                                                                         # d16723 - wjr
  typeset DRIVE_ATTRIBUTES='undefined'                                                 # d17513 - wjr
  xprint "MSG: Query $DRIVE_ATTRIBUTES file for valid drive master P/Ns."              # d16051 - wjr
  if [[ -f $ALT_DRIVE_FILE ]]; then                                                    # d17513 - wjr
    xprint "MSG: Using DriveAttributes.ini file in alternate flash."                   # d17513 - wjr
    DRIVE_ATTRIBUTES=$ALT_DRIVE_FILE                                                   # d17513 - wjr
  elif [[ -f $DRIVE_FILE ]]; then                                                      # d17513 - wjr
    xprint "MSG: Using DriveAttributes.ini file in primary flash."                     # d17513 - wjr
    DRIVE_ATTRIBUTES=$DRIVE_FILE                                                       # d17513 - wjr
  else                                                                                 # d17513 - wjr
    xprint "MSG: No DriveAttributes.ini file - skipping check."                        # d17513 - wjr
    prt_test_result skipped $MSG_TXT                                                   # d17513 - wjr
  fi                                                                                   # d17513 - wjr
  if [[ 'undefined' != $DRIVE_ATTRIBUTES ]]; then                                      # d17513 - wjr
    if [[ -f $DRIVE_ATTRIBUTES ]]; then                                                # d16051 - wjr
      VALID_PNS=$(cat $DRIVE_ATTRIBUTES | \
                  grep '_productMasterId' | \
                  cut -d '=' -f 2)                                                     # d16051 - wjr
      if [[ -n $VALID_PNS ]]; then                                                     # d16051 - wjr
        xprint "MSG: Valid P/Ns : $VALID_PNS"                                          # d16051 - wjr
        xprint "MSG: Query CLI for drive slot ids."                                    # d16051 - wjr
        CMD='echo "list drive" | /opt/arts/Cli/cli/linux_ppc/cli'                      # d16051 - wjr
        run_cmd file                                                                   # d16051 - wjr
        if [[ -f $CMD_FILE ]]; then                                                    # d16051 - wjr
          xprint "MSG: Query for drive slot ids."                                      # d16051 - wjr
          prt_cli_session                                                              # d16051 - wjr
          SLOT_IDS=$(cat $CMD_FILE | \
                     grep '|' | \
                     grep -v "<CLI>" | \
                     grep -v "<cli>" | \
                     grep -v "_" | \
                     grep -v 'Drive' | \
                     grep -v 'Current Machine' | \
                     grep -v 'Usage:' | \
                     grep -v 'foreign' | \
                     grep -v 'current' | \
                     grep -v 'State:' | \
                     grep -v 'initialized' | \
                     grep -v 'incompatible' | \
                     grep -v 'unsupported' | \
                     grep -v 'Error Message File' | \
                     grep -v 'No drives reported' | \
                     cut -d '|' -f 3)                                                  # d17644 - wjr
          if [[ -z $SLOT_IDS ]]; then                                                  # d16051 - wjr
            xprint "MSG: Query returned a null list - no drives found."                # d16051 - wjr
          else                                                                         # d16051 - wjr
            for slot in $SLOT_IDS                                                      # d16051 - wjr
            do                                                                         # d16051 - wjr
              xprint "MSG: Query CLI for drive master pn in slot $slot."               # d16051 - wjr
              CMD="echo \"detail drive -slot $slot\" | /opt/arts/Cli/cli/linux_ppc/cli" # d16051 - wjr
              run_cmd file                                                             # d16051 - wjr
              if [[ -f $CMD_FILE ]]; then                                              # d16051 - wjr
                xprint "MSG: Checking drive in slot $slot master pn."                  # d16051 - wjr
                prt_cli_session                                                        # d16051 - wjr
                PN=$(cat $CMD_FILE | \
                     grep 'Product Master ID' | \
                     cut -d ':' -f 2)                                                  # d16051 - wjr
                if [[ -n $PN ]]; then                                                  # d16051 - wjr
                  IS_VALID=$(echo $VALID_PNS | grep $PN)                               # d16051 - wjr
                  if [[ -n $IS_VALID ]]; then                                          # d16051 - wjr
                    xprint "MSG: Drive in slot $slot with master P/N $PN is valid."    # d16051 - wjr
                  else                                                                 # d16051 - wjr
                    append_err_msg 13 $slot'(P/N'$PN')'                                # d17587 - wjr
                    RC=13                                                              # d16051 - wjr
                  fi                                                                   # d16051 - wjr
                else                                                                   # d16051 - wjr
                  append_err_msg 14 $slot                                              # d17513 - wjr
                  RC=14                                                                # d16051 - wjr
                fi                                                                     # d16051 - wjr
              else                                                                     # d16051 - wjr
                append_err_msg 15 $slot                                                # d17513 - wjr
                RC=15                                                                  # d16051 - wjr
              fi                                                                       # d16051 - wjr
            done                                                                       # d16051 - wjr
            if [[ '' != ${ERR_MSG[13]} ]]; then                                        # d17513 - wjr
              set_rc 13 "Drive(s) in slot(s) ${ERR_MSG[13]} have an INVALID P/N."      # d17513 - wjr
            fi                                                                         # d17513 - wjr
            if [[ '' != ${ERR_MSG[14]} ]]; then                                        # d17513 - wjr
              set_rc 14 "$SCRIPT query for master P/N of drive(s) in slot(s) ${ERR_MSG[14]} returned a null." # d17587 - wjr
            fi                                                                         # d17513 - wjr
            if [[ '' != ${ERR_MSG[15]} ]]; then                                        # d17513 - wjr
              set_rc 15 "$SCRIPT query for master P/N of drive(s) in slot(s) ${ERR_MSG[15]} failed." # d17587 - wjr
            fi                                                                         # d17513 - wjr
          fi                                                                           # d16051 - wjr
        else                                                                           # d16051 - wjr
          set_rc 16 "$SCRIPT query for drive slot ids failed."                         # d17587 - wjr
          RC=16                                                                        # d16051 - wjr
        fi                                                                             # d16051 - wjr
      else                                                                             # d16051 - wjr
        set_rc 17 "$SCRIPT query for valid drive master P/Ns returned a null."         # d17587 - wjr
        RC=17                                                                          # d16051 - wjr
      fi                                                                               # d16051 - wjr
    else                                                                               # d16051 - wjr
      set_rc 18 "$SCRIPT query for valid drive master P/Ns failed."                    # d17587 - wjr
      RC=18                                                                            # d16051 - wjr
    fi                                                                                 # d16051 - wjr
    if (( 0 == $RC )); then                                                            # d16051 - wjr
      prt_test_result passed $MSG_TXT                                                  # d16051 - wjr
    else                                                                               # d16051 - wjr
      prt_test_result failed $MSG_TXT                                                  # d16051 - wjr
    fi                                                                                 # d16051 - wjr
  fi                                                                                   # d17513 - wjr
  xprint debug "TRC: Exit function $FUNCTION"                                          # d16051 - wjr
  return $RC                                                                           # d16051 - wjr
}                                                                                      # d16051 - wjr
# =====================================================================================
function query_unsupported_drives                                                      # d17587 - wjr
{                                                                                      # d17587 - wjr
  typeset FUNCTION='query_unsupported_drives'                                          # d17587 - wjr
  xprint debug "TRC: Enter function $FUNCTION"                                         # d17587 - wjr
  trap 'handle_interrupt ask' SIGINT SIGTERM SIGTSTP SIGQUIT                           # d17587 - wjr
  typeset MSG_TXT='No unsupported disk drives'                                         # d17587 - wjr
  typeset SLOT_ID=''                                                                   # d17587 - wjr
  typeset DRIVE_STATE=''                                                               # d17587 - wjr
  typeset DRIVE_INFO=''                                                                # d17587 - wjr
  typeset BAD_STATE_LIST='UNS|IMD|UNF'                                                 # d17587 - wjr
  typeset DRIVE_MSG=''                                                                 # d17587 - wjr
  typeset RC=0                                                                         # d17587 - wjr
  xprint "MSG: Query CLI for failed or missing drives."                                # d17587 - wjr
  CMD='echo "list drive" | /opt/arts/Cli/cli/linux_ppc/cli'                            # d17587 - wjr
  run_cmd file                                                                         # d17587 - wjr
  if [[ -f $CMD_FILE ]]; then                                                          # d17587 - wjr
    xprint "MSG: Query for unsupported drives successful."                             # d17587 - wjr
    prt_cli_session                                                                    # d17587 - wjr
    xprint "MSG: Checking drive state."                                                # d17587 - wjr
    RESULT=$(cat $CMD_FILE | \
             grep '|' | \
             grep -v "<CLI>" | \
             grep -v "<cli>" | \
             grep -v "_" | \
             grep -v 'Drive' | \
             grep -v 'Current Machine' | \
             grep -v 'Usage:' | \
             grep -v 'foreign' | \
             grep -v 'current' | \
             grep -v 'State:' | \
             grep -v 'initialized' | \
             grep -v 'incompatible' | \
             grep -v 'firmware' | \
             grep -v 'Error Message File' | \
             grep -v 'No drives reported' | \
             cut -d '|' -f 3,8 | \
             tr -d ' ')                                                                # d17644 - wjr
    for drive in $RESULT                                                               # d17587 - wjr
    do                                                                                 # d17587 - wjr
      SLOT_ID=$(echo $drive | cut -d '|' -f 1)                                         # d17587 - wjr
      DRIVE_STATE=$(echo $drive | cut -d '|' -f 2)                                     # d17587 - wjr
      DRIVE_INFO=$(echo $DRIVE_STATE | grep -E $BAD_STATE_LIST)                        # d17644 - wjr
      if [[ -z $DRIVE_INFO ]]; then                                                    # d17587 - wjr
        xprint "MSG: Drive in slot $SLOT_ID is supported."                             # d17587 - wjr
      else                                                                             # d17587 - wjr
        if [[ 'UNS' == $DRIVE_STATE ]]; then                                           # d17587 - wjr
          DRIVE_MSG='unsuppored (non-IBM) drive'                                       # d17587 - wjr
        elif [[ 'IMD' == $DRIVE_STATE ]]; then                                         # d17587 - wjr
          DRIVE_MSG='incompatible metadata'                                            # d17587 - wjr
        elif [[ 'UNF' == $DRIVE_STATE ]]; then                                         # d17587 - wjr
          DRIVE_MSG='firmware version unsupported for drive'                           # d17587 - wjr
        else                                                                           # d17587 - wjr
          DRIVE_MSG=$DRIVE_STATE                                                       # d17587 - wjr
        fi                                                                             # d17587 - wjr
        append_err_msg 38 'Slot '$SLOT_ID'('$DRIVE_MSG')'                              # d17587 - wjr
        RC=38                                                                          # d17587 - wjr
      fi                                                                               # d17587 - wjr
    done                                                                               # d17587 - wjr
    if [[ '' != ${ERR_MSG[38]} ]]; then                                                # d17587 - wjr
      set_rc 38 "Found one or more unsupported drives: ${ERR_MSG[11]}."                # d17587 - wjr
    fi                                                                                 # d17587 - wjr
  else                                                                                 # d17587 - wjr
    set_rc 39 "$SCRIPT query for unsupported drives failed."                           # d17587 - wjr
    RC=39                                                                              # d17587 - wjr
  fi                                                                                   # d17587 - wjr
  if (( 0 == $RC )); then                                                              # d17587 - wjr
    prt_test_result passed $MSG_TXT                                                    # d17587 - wjr
  else                                                                                 # d17587 - wjr
    prt_test_result failed $MSG_TXT                                                    # d17587 - wjr
  fi                                                                                   # d17587 - wjr
  xprint debug "TRC: Exit function $FUNCTION"                                          # d17587 - wjr
  return $RC                                                                           # d17587 - wjr
}                                                                                      # d17587 - wjr
# =====================================================================================
function query_pools
{
  typeset FUNCTION='query_pools'
  xprint debug "TRC: Enter function $FUNCTION"
  trap 'handle_interrupt ask' SIGINT SIGTERM SIGTSTP SIGQUIT
  typeset MSG_TXT='Drive Pool State'
  typeset BAD_STATE_LIST='ONN|OFN|SN|ONF|OFF|SF|OFV|SV'
  typeset BAD_STATUS_LIST='Degraded|Degraded-InTransition|Unviable'                    # d18524 - wjr
  typeset POOL_ID=''                                                                   # d17587 - wjr
  typeset POOL_STATE=''                                                                # d17587 - wjr
  typeset POOL_STATUS=''                                                               # d17587 - wjr
  typeset POOL_INFO=''                                                                 # d17587 - wjr
  typeset RC=0                                                                         # d16723 - wjr
  xprint "MSG: Query CLI for drive pool state."
  verify_rc_state BOUND 'silent' $ACTIVE_GROUP                                         # d18107 - wjr
  if (( 0 == $? )); then
    CMD='echo "list pool" | /opt/arts/Cli/cli/linux_ppc/cli'
    run_cmd file
    if [[ -f $CMD_FILE ]]; then
      xprint "MSG: Query for drive pool state successful."
      prt_cli_session
      xprint "MSG: Checking pool status."
      RESULT=$(cat $CMD_FILE | \
               grep "No drive pools")
      if [[ -z $RESULT ]]; then
        RESULT=$(cat $CMD_FILE | \
                 grep '|' | \
                 grep -v "<CLI>" | \
                 grep -v "<cli>" | \
                 grep -v 'RaidType' | \
                 grep -v 'Current Machine' | \
                 grep -v '/' | \
                 cut -d '|' -f 2,9,10 | \
                 grep -v "_" | \
                 tr -d ' ')                                                            # d18324 - wjr
        for pool in $RESULT                                                            # d17587 - wjr
        do                                                                             # d17587 - wjr
          POOL_ID=$(echo $pool | cut -d '|' -f 1)                                      # d17587 - wjr
          POOL_STATUS=$(echo $pool | cut -d '|' -f 2)                                  # d17587 - wjr
          POOL_STATE=$(echo $pool | cut -d '|' -f 3)                                   # d17587 - wjr
          POOL_STATE_INFO=$(echo $POOL_STATE | grep -E $BAD_STATE_LIST)                # d18524 - wjr
          POOL_STATUS_INFO=$(echo $POOL_STATUS | grep -E $BAD_STATUS_LIST)             # d18524 - wjr
          if [[ -z $POOL_STATE_INFO && \
                -z $POOL_STATUS_INFO ]]; then                                          # d18524 - wjr
            xprint "MSG: Pool $POOL_ID is $POOL_STATUS."                               # d17587 - wjr
          else                                                                         # d17587 - wjr
            append_err_msg 19 'Slot '$POOL_ID'('$POOL_STATUS')'                        # d17587 - wjr
            RC=19
          fi
        done                                                                           # d17587 - wjr
        if [[ '' != ${ERR_MSG[19]} ]]; then                                            # d17587 - wjr
          set_rc 19 "Found one or more failed/non-viable pools: ${ERR_MSG[19]}."       # d17587 - wjr
        fi                                                                             # d17587 - wjr
      else
        xprint "MSG: No drive pools."
      fi
    else
      set_rc 20 "$SCRIPT query for drive pool state failed."                           # d17587 - wjr
      RC=20                                                                            # d16051 - wjr
    fi
    if (( 0 == $RC )); then                                                            # d17587 - wjr
      prt_test_result passed $MSG_TXT                                                  # d17587 - wjr
    else                                                                               # d17587 - wjr
      prt_test_result failed $MSG_TXT                                                  # d17587 - wjr
    fi                                                                                 # d17587 - wjr
  else
    xprint "MSG: Ctlrs are not BOUND - skipping drive pool state check."
  fi
  xprint debug "TRC: Exit function $FUNCTION"
  return $RC
}
# =====================================================================================
function query_alerts
{
  typeset FUNCTION='query_alerts'
  xprint debug "TRC: Enter function $FUNCTION"
  trap 'handle_interrupt ask' SIGINT SIGTERM SIGTSTP SIGQUIT
  typeset MSG_TXT='Critical Maintenance Alerts'
  typeset ALERT_LIST='"1202|1203|1300|1303|4200|5700|9004|9200|9300|9303|9304|9306|9511"' # d17654 - wjr
  typeset ALL_ALERTS=''                                                                # d17587 - wjr
  ALERT_LIST=$(echo $ALERT_LIST | tr -d ' ')
  typeset RC=0                                                                         # d16723 - wjr
  xprint "MSG: Check for critical alerts."
  CMD='echo "alert -get" | /opt/arts/Cli/cli/linux_ppc/cli'
  run_cmd file
  if [[ -f $CMD_FILE ]]; then
    xprint "MSG: Query for alerts successful."
    prt_cli_session
    xprint "MSG: Searching for critical alerts."
    ALL_ALERTS=$(cat $CMD_FILE | \
                 grep '|' | \
                 grep 'Critical' | \
                 grep -v "=" | \
                 grep -v '-' | \
                 grep -v '_' | \
                 grep -v 'Current Machine' | \
                 grep -v 'Existing Alerts' | \
                 grep -v "AlertCode" | \
                 grep -v "Msg" | \
                 grep -v "<CLI>" | \
                 grep -v 'Error Message File' | \
                 cut -d '|' -f 2 | \
                 grep -v -E $ALERT_LIST)                                               # d17654 - wjr
    if [[ -z $ALL_ALERTS ]]; then
      prt_test_result passed $MSG_TXT
      xprint "MSG: No critical alerts."
    else
      ALERTS=''
      for a in $ALL_ALERTS
      do
        if [[ -z $(echo $ALERTS | grep $a) ]]; then
          ALERTS=$(echo $ALERTS $a)
        fi
      done
      prt_test_result failed $MSG_TXT
      set_rc 21 "Found one or more critical alerts: $ALERTS"                           # d17513 - wjr
      RC=21                                                                            # d16051 - wjr
    fi
  else
    prt_test_result failed $MSG_TXT
    set_rc 22 "$SCRIPT query for critical alerts failed."                              # d17587 - wjr
    RC=22                                                                              # d16051 - wjr
  fi
  xprint debug "TRC: Exit function $FUNCTION"
  return $RC
}
# =====================================================================================
function configure_alert_template                                                      # d16511 - wjr
{                                                                                      # d16511 - wjr
  typeset FUNCTION='configure_alert_template'                                          # d16511 - wjr
  xprint debug "TRC: Enter function $FUNCTION"                                         # d16511 - wjr
  trap 'handle_interrupt ask' SIGINT SIGTERM SIGTSTP SIGQUIT                           # d16511 - wjr
  typeset RC=0                                                                         # d16723 - wjr
  typeset ALERT_CODE=$1                                                                # d16511 - wjr
  shift                                                                                # d16511 - wjr
  typeset ALERT_MSG=$*                                                                 # d16511 - wjr
  ALERT_MSG='\"NCCL preverify FAILED.  '$ALERT_MSG'\"'                                 # d16511 - wjr
  xprint "MSG: Configuring alert template for code $ALERT_CODE."                       # d16511 - wjr
  CMD='echo "configure alert -setgenericalerttemplate -code '$ALERT_CODE' -type ackable -initiallymasked on -email off -severity warning -msg '$ALERT_MSG'" | /opt/arts/Cli/cli/linux_ppc/cli'    # d16511 - wjr
  run_cmd file                                                                         # d16511 - wjr
  if [[ -f $CMD_FILE ]]; then                                                          # d16511 - wjr
    xprint "MSG: CLI command to configure alert template was issued."                  # d16511 - wjr
    prt_cli_session                                                                    # d16511 - wjr
    xprint "MSG: Checking alert configure command response."                           # d16511 - wjr
    RESULT=$(cat $CMD_FILE | \
             grep "successfully submitted")                                            # d16723 - wjr
    if [[ -n $RESULT ]]; then                                                          # d16511 - wjr
      xprint "MSG: Alert $ALERT_CODE template successfully configured."                # d16511 - wjr
    else                                                                               # d16511 - wjr
      echo "ERROR: CLI command to configure alert failed."                             # d16511 - wjr
      RC=23                                                                            # d16511 - wjr
    fi                                                                                 # d16511 - wjr
  else                                                                                 # d16511 - wjr
    echo "ERROR: CLI command to configure alert failed to execute."                    # d16511 - wjr
    RC=24                                                                              # d16511 - wjr
  fi                                                                                   # d16511 - wjr
  xprint debug "TRC: Exit function $FUNCTION"                                          # d16511 - wjr
  return $RC                                                                           # d16511 - wjr
}                                                                                      # d16511 - wjr
# =====================================================================================
function generate_alert                                                                # d16511 - wjr
{
  typeset FUNCTION='generate_alert'                                                    # d16511 - wjr
  xprint debug "TRC: Enter function $FUNCTION"
  trap 'handle_interrupt ask' SIGINT SIGTERM SIGTSTP SIGQUIT
  typeset ALERT_CODE=$1
  typeset RC=0                                                                         # d16723 - wjr
  if [[ -z $ALERT_CODE ]]; then
    ALERT_CODE=10001
  fi
  xprint "MSG: Generating alert code $ALERT_CODE."
  CMD='echo "alert -create -code '$ALERT_CODE'" | /opt/arts/Cli/cli/linux_ppc/cli'
  run_cmd file
  if [[ -f $CMD_FILE ]]; then
    xprint "MSG: CLI command to generate alert was issued."
    prt_cli_session
    xprint "MSG: Checking alert command response."
    RESULT=$(cat $CMD_FILE | \
             grep "alert created successfully")
    if [[ -n $RESULT ]]; then
      xprint "MSG: Alert $ALERT_CODE successfully created."
    else
      echo "ERROR: CLI command to generate alert failed."
      RC=25                                                                            # d16511 - wjr
    fi
  else
    echo "ERROR: CLI command to generate alert failed to execute."
    RC=26                                                                              # d16511 - wjr
  fi
  xprint debug "TRC: Exit function $FUNCTION"
  return $RC
}
# =====================================================================================
function compare_fw_levels                                                             # d17429 - wjr
{                                                                                      # d17429 - wjr
  typeset FUNCTION='compare_fw_levels'                                                 # d17429 - wjr
  xprint debug "TRC: Enter function $FUNCTION"                                         # d17429 - wjr
  trap 'handle_interrupt ask' SIGINT SIGTERM SIGTSTP SIGQUIT                           # d17429 - wjr
  typeset RC=1                                                                         # d17429 - wjr
  typeset INST_LVL=$1                                                                  # d17429 - wjr
  typeset TARGET_LVL=$2                                                                # d17429 - wjr
  typeset TMP_FILE=$LOG_DIR'/sort.tmp'                                                 # d17429 - wjr
  xprint "MSG: Comparing installed level $INST_LVL to target level $TARGET_LVL."       # d17429 - wjr
  remove_file $TMP_FILE                                                                # d17429 - wjr
  echo $INST_LVL | sed -e 's/^0*//' | sed -e 's/0*$//' > $TMP_FILE                     # d17429 - wjr
  echo $TARGET_LVL | sed -e 's/^0*//' | sed -e 's/0*$//' >> $TMP_FILE                  # d17429 - wjr
  sort -b -f -c $TMP_FILE 2>>$LOG                                                      # d17429 - wjr
  RC=$?                                                                                # d17429 - wjr
  if [[ 0 == $RC ]]; then                                                              # d17429 - wjr
    xprint "MSG: Installed level and target level are compatible."                     # d17429 - wjr
  else                                                                                 # d17429 - wjr
    xprint "MSG: Installed level and target level are INcompatible."                   # d17429 - wjr
  fi                                                                                   # d17429 - wjr
  remove_file $TMP_FILE                                                                # d17429 - wjr
  xprint debug "TRC: Exit function $FUNCTION"                                          # d17429 - wjr
  return $RC                                                                           # d17429 - wjr
}                                                                                      # d17429 - wjr
# =====================================================================================
function verify_fw_levels
{
  typeset FUNCTION='verify_fw_levels'
  xprint debug "TRC: Enter function $FUNCTION"
  trap 'handle_interrupt ask' SIGINT SIGTERM SIGTSTP SIGQUIT
  xprint "MSG: Verifying come-from and go-to firmware levels."                         # d17429 - wjr
  typeset MSG_TXT='Come-from & Go-to FW levels compatible'
  typeset RC=0                                                                         # d16723 - wjr
  typeset FW_COMPONENTS='Arts|Software|Tsalversion
                         SES|SES|SesVersion
                         CPLD|CPLD|CpldVersion
                         ppcFPGA|FPGA|FpgaVersion
                         BMC|BMC|BmcVersion
                         SAS|SAS_switch|SasVersion'                                    # d17475 - wjr
  typeset COMPONENT_ID='undefined'                                                     # d17429 - wjr
  typeset SWVERSION_ID='undefined'                                                     # d17429 - wjr
  typeset CFG_FILE='undefined'                                                         # d17475 - wjr
  typeset CFG_FILE_ID='undefined'                                                      # d17429 - wjr
  typeset INSTALLED_LVL='undefined'                                                    # d17429 - wjr
  typeset INST_COMP_LVL='undefined'                                                    # d17697 - wjr
  typeset MAX_LVL='undefined'                                                          # d17429 - wjr
  typeset VERIFY_FILE=$LOG_DIR'/verify.tmp'                                            # d17429 - wjr
  typeset CFG_FILE_TMP=$LOG_DIR'/CtlrFwMgmt.tmp'                                       # d17429 - wjr
  if [[ -n $CFG_STRING ]]; then                                                        # d17429 - wjr
    xprint "MSG: Using command line args for fw compatibility check."                  # d17475 - wjr
    remove_file $CFG_FILE_TMP                                                          # d17429 - wjr
    MAX_VERSIONS=$(echo $CFG_STRING | \
                   tr ':' ' ')                                                         # d17429 - wjr
    for m in $MAX_VERSIONS                                                             # d17429 - wjr
    do                                                                                 # d17429 - wjr
      echo $m | tr "^" " " >> $CFG_FILE_TMP                                            # d17429 - wjr
    done                                                                               # d17429 - wjr
    CFG_FILE=$CFG_FILE_TMP                                                             # d17429 - wjr
  elif [[ -f $ALT_CONFIG_FILE ]]; then                                                 # d17475 - wjr
    xprint "MSG: Using conf file in alternate flash for fw compatibility check."       # d17475 - wjr
    CFG_FILE=$ALT_CONFIG_FILE                                                          # d17475 - wjr
  elif [[ -f $CONFIG_FILE ]]; then                                                     # d17475 - wjr
    xprint "MSG: Using conf file in primary flash for fw compatibility check."         # d17475 - wjr
    CFG_FILE=$CONFIG_FILE                                                              # d17429 - wjr
  else                                                                                 # d17475 - wjr
    xprint "MSG: No config file - skipping fw compatibility check."                    # d17475 - wjr
    prt_test_result skipped $MSG_TXT                                                   # d17475 - wjr
  fi                                                                                   # d17429 - wjr
  if [[ 'undefined' != $CFG_FILE ]]; then                                              # d17475 - wjr
    if [[ -f $CFG_FILE ]]; then                                                        # d17429 - wjr
      CMD='echo swversion | /opt/arts/Cli/cli/linux_ppc/cli'                           # d17429 - wjr
      run_cmd file                                                                     # d17429 - wjr
      copy_file $CMD_FILE $VERIFY_FILE                                                 # d17429 - wjr
      prt_cli_session                                                                  # d17429 - wjr
      for component in $FW_COMPONENTS                                                  # d17429 - wjr
      do                                                                               # d17429 - wjr
        COMPONENT_ID=$(echo $component | \
                       cut -d '|' -f 1)                                                # d17429 - wjr
        SWVERSION_ID=$(echo $component | \
                       cut -d '|' -f 2 | \
                       tr '_' ' ')                                                     # d17429 - wjr
        CFG_FILE_ID=$(echo $component | \
                      cut -d '|' -f 3)                                                 # d17429 - wjr
        MIN_LVL=$(cat $CFG_FILE | \
                  grep '_min'$CFG_FILE_ID | \
                  cut -d '=' -f 2)                                                     # d17513 - wjr
        if [[ -n $MIN_LVL ]]; then                                                     # d17513 - wjr
          xprint "MSG: Min supported $COMPONENT_ID fw level: $MIN_LVL."                # d17513 - wjr
        else                                                                           # d17513 - wjr
          append_err_msg 33 $COMPONENT_ID                                              # d17513 - wjr
          RC=33                                                                        # d17513 - wjr
        fi                                                                             # d17513 - wjr
        MAX_LVL=$(cat $CFG_FILE | \
                  grep '_max'$CFG_FILE_ID | \
                  cut -d '=' -f 2)                                                     # d17429 - wjr
        if [[ -n $MAX_LVL ]]; then                                                     # d17429 - wjr
          xprint "MSG: Max supported $COMPONENT_ID fw level: $MAX_LVL."                # d17513 - wjr
        else                                                                           # d17429 - wjr
          append_err_msg 34 $COMPONENT_ID                                              # d17513 - wjr
          RC=34                                                                        # d17429 - wjr
        fi                                                                             # d17429 - wjr
        if [[ '0.0' != $MIN_LVL || \
              '0.0' != $MAX_LVL ]]; then                                               # d17513 - wjr
          INSTALLED_LVL=$(cat $VERIFY_FILE | \
                          grep "$SWVERSION_ID"' version' | \
                          cut -d ':' -f 2 | \
                          cut -d ' ' -f 1-3)                                           # d17475 - wjr
          if [[ -n $INSTALLED_LVL ]]; then                                             # d17429 - wjr
            xprint "MSG: Installed $COMPONENT_ID fw level: $INSTALLED_LVL."            # d17429 - wjr
          else                                                                         # d17429 - wjr
            append_err_msg 35 $COMPONENT_ID                                            # d17513 - wjr
            RC=35                                                                      # d17429 - wjr
          fi                                                                           # d17429 - wjr
        else                                                                           # d17513 - wjr
          xprint "MSG: Skipping $COMPONENT_ID min & max fw level check."               # d17513 - wjr
        fi                                                                             # d17513 - wjr
        INST_COMP_LVL=$(echo $INSTALLED_LVL | \
                        tr -d ' ')                                                     # d17697 - wjr
        if [[ '0.0' != $MIN_LVL && \
              'UNAVAILABLE' != $INST_COMP_LVL ]]; then                                 # d17759 - wjr
          compare_fw_levels "$MIN_LVL" "$INSTALLED_LVL"                                # d17513 - wjr
          RESULT=$?                                                                    # d17429 - wjr
          if [[ 0 != $RESULT ]]; then                                                  # d17429 - wjr
            LVL_MSG='Installed level:'$INSTALLED_LVL', Min supported level:'$MIN_LVL   # d17587 - wjr
            append_err_msg 36 $COMPONENT_ID'('$LVL_MSG')'                              # d17587 - wjr
            RC=36                                                                      # d17429 - wjr
          fi                                                                           # d17429 - wjr
        else                                                                           # d17513 - wjr
          xprint "MSG: Skipping $COMPONENT_ID min fw level check."                     # d17513 - wjr
        fi                                                                             # d17513 - wjr
        if [[ '0.0' != $MAX_LVL && \
              'UNAVAILABLE' != $INST_COMP_LVL ]]; then                                 # d17759 - wjr
          compare_fw_levels "$INSTALLED_LVL" "$MAX_LVL"                                # d17429 - wjr
          RESULT=$?                                                                    # d17429 - wjr
          if [[ 0 != $RESULT ]]; then                                                  # d17429 - wjr
            LVL_MSG='Installed level:'$INSTALLED_LVL', Max supported level:'$MAX_LVL   # d17587 - wjr
            append_err_msg 37 $COMPONENT_ID'('$LVL_MSG')'                              # d17587 - wjr
            RC=37                                                                      # d17429 - wjr
          fi                                                                           # d17429 - wjr
        else                                                                           # d17513 - wjr
          xprint "MSG: Skipping $COMPONENT_ID max fw level check."                     # d17513 - wjr
        fi                                                                             # d17513 - wjr
      done                                                                             # d17429 - wjr
      if [[ '' != ${ERR_MSG[33]} ]]; then                                              # d17513 - wjr
        set_rc 33 "$SCRIPT query for Rc ${ERR_MSG[33]} min firmware level failed."     # d17587 - wjr
      fi                                                                               # d17513 - wjr
      if [[ '' != ${ERR_MSG[34]} ]]; then                                              # d17513 - wjr
        set_rc 34 "$SCRIPT query for Rc ${ERR_MSG[34]} max firmware level failed."     # d17587 - wjr
      fi                                                                               # d17513 - wjr
      if [[ '' != ${ERR_MSG[35]} ]]; then                                              # d17513 - wjr
        set_rc 35 "$SCRIPT query for installed Rc ${ERR_MSG[35]} firmware level failed." # d17587 - wjr
      fi                                                                               # d17513 - wjr
      if [[ '' != ${ERR_MSG[36]} ]]; then                                              # d17513 - wjr
        set_rc 36 "${ERR_MSG[36]} come-from firmware level is incompatible."           # d17587 - wjr
      fi                                                                               # d17513 - wjr
      if [[ '' != ${ERR_MSG[37]} ]]; then                                              # d17513 - wjr
        set_rc 37 "${ERR_MSG[37]} go-to firmware level is incompatible."               # d17587 - wjr
      fi                                                                               # d17513 - wjr
    else                                                                               # d17429 - wjr
      set_rc 32 "$SCRIPT could not find the $CFG_FILE config file."                    # d17587 - wjr
      RC=32                                                                            # d17429 - wjr
    fi                                                                                 # d17429 - wjr
    remove_file $CFG_FILE_TMP                                                          # d17429 - wjr
    remove_file $VERIFY_FILE                                                           # d17429 - wjr
    if (( 0 == $RC )); then                                                            # d17429 - wjr
      prt_test_result passed $MSG_TXT                                                  # d17429 - wjr
    else                                                                               # d17429 - wjr
      prt_test_result failed $MSG_TXT                                                  # d17429 - wjr
    fi                                                                                 # d17429 - wjr
  fi                                                                                   # d17429 - wjr
  xprint debug "TRC: Exit function $FUNCTION"
  return $RC
}
# =====================================================================================
function nccl_check
{
  typeset FUNCTION='nccl_check'
  xprint debug "TRC: Enter function $FUNCTION"
  trap 'handle_interrupt ask' SIGINT SIGTERM SIGTSTP SIGQUIT
  typeset -i RC
  CHECK_TYPE='check'
  prt_test_result header 'NCCL'
  query_tsal
  RC=$?
  if (( 0 == $RC )); then
    query_cli
    query_address                                                                      # d16051 - wjr
    query_config                                                                       # d18107 - wjr
    query_ctlr_id                                                                      # d16051 - wjr
    verify_rc_state SERVICE 'verbose' $ACTIVE_GROUP                                    # d18107 - wjr
    query_rc_fw_status $ACTIVE_GROUP                                                   # d18107 - wjr
    query_ddm_fw_status
    query_bad_drives
    query_unsupported_drive_pns                                                        # d17587 - wjr
#Tny    query_unsupported_drives                                                           # d16051 - wjr
    query_pools
    query_alerts
    verify_fw_levels                                                                   # d16723 - wjr
  fi
  prt_test_result space
  MSG_TXT='Overall Healthcheck Result'
  if (( 0 == $RETURN_CODE )); then                                                     # d16723 - wjr
    prt_test_result passed $MSG_TXT
  else
    prt_test_result failed $MSG_TXT
  fi
  prt_test_result trailer
  remove_file $CMD_FILE
  xprint debug "TRC: Exit function $FUNCTION"
}
# ======================================= MAIN ========================================
#
# Set Options / Globals
#
typeset VERSION='1.21'
typeset OPERATION='nccl_check'
typeset VERBOSE='n'
typeset VERBOSE_DEBUG='n'
typeset VERBOSE_CLI='y'
typeset SUMMARY='n'
typeset COMMAND=$0
typeset ARGS=$*
typeset SCRIPT=${COMMAND##/*/}
typeset FUNCTION='main'
typeset -i RETURN_CODE=0
typeset MODE='interactive'
typeset CFG_STRING=''                                                                  # d17429 - wjr
typeset CFG_FILE_NAME='package.conf'                                                   # d17513 - wjr
typeset CFG_DIR='/opt/Config0/shellScripts'                                            # d17513 - wjr
typeset ALT_CFG_FILE_NAME='CtlrFwMgmt.conf'                                            # d17513 - wjr
typeset ALT_CFG_DIR='/mnt/opt/arts/tsal/linux_ppc'                                     # d17513 - wjr
typeset CONFIG_FILE=$CFG_DIR'/'$CFG_FILE_NAME                                          # d17513 - wjr
typeset ALT_CONFIG_FILE=$ALT_CFG_DIR'/'$ALT_CFG_FILE_NAME                              # d17513 - wjr
typeset DRIVE_FILE_NAME='DriveAttributes.ini'                                          # d17513 - wjr
typeset DRIVE_FILE=$CFG_DIR'/'$DRIVE_FILE_NAME                                         # d17513 - wjr
typeset ALT_DRIVE_FILE=$ALT_CFG_DIR'/'$DRIVE_FILE_NAME                                 # d17513 - wjr
typeset CMD_FILE_NAME=$SCRIPT'.tmp'
typeset LOG_NAME=$SCRIPT'.log'
typeset LOG_DIR='/opt/Logs/shellScriptsLogs'
typeset SUMMARY_LOG_NAME=$SCRIPT'.summary.log'
typeset SUMMARY_LOG=$LOG_DIR'/'$SUMMARY_LOG_NAME
typeset LOG=$LOG_DIR'/'$LOG_NAME
typeset CMD_FILE=$LOG_DIR'/'$CMD_FILE_NAME
typeset CMD_EXIT='restart'
typeset TIMESTAMP='y'
typeset CLEAR_LOG='y'
typeset RESULT=''
typeset IP_ADDRESS='undefined'                                                         # d16051 - wjr
typeset CTLR_ID='undefined'                                                            # d16051 - wjr
typeset -i N_OPERATIONS=0
typeset -a ERR_MSG                                                                     # d17513 - wjr
typeset ACTIVE_CTLR_IDS='0 1'                                                          # d18107 - wjr
typeset ACTIVE_GROUP='both'                                                            # d18107 - wjr
#
while getopts ":hHvsc:NC" opt;                                                         # d17429 - wjr
do
  case $opt in
    h | H )
      show_help
      exit 0
      ;;
    v )
      VERBOSE='y'
      ;;
    s )
      SUMMARY='y'
      ;;
    c )                                                                                # d17429 - wjr
      CFG_STRING=$OPTARG                                                               # d17429 - wjr
      ;;                                                                               # d17429 - wjr
    N )
      let N_OPERATIONS=N_OPERATIONS+1
      OPERATION='nccl_check'
      ;;
    ? )
      show_syntax
      exit 0
      ;;
  esac
done
make_log_dir                                                                           # d16051 - wjr
log open
check_arguments
case $OPERATION in
  nccl_check )
    nccl_check
    ;;
  * )
    echo 'WARNING : No action specified.'
    ;;
esac
append_file $SUMMARY_LOG $LOG                                                          # d17429 - wjr
log close
exit $RETURN_CODE
# =====================================================================================
