#!/bin/bash
# %Z%%M%    %I%  %W% %G% %U%
# *************************************************************************************
#
# -------------------------------------------------------------------------------------
#
#                                  IBM CONFIDENTIAL
#
#                       LICENSED INTERNAL CODE SOURCE MATERIALS
#
#                           1820 LICENSED INTERNAL CODE
#
#                        (C) COPYRIGHT IBM CORP. 2009, 2010
#
#                THE SOURCE CODE FOR THIS PROGRAM IS NOT PUBLISHED OR
#                OTHERWISE DIVESTED OF ITS TRADE SECRETS, IRRESPECTIVE OF
#                WHAT HAS BEEN DEPOSITED WITH THE U.S. COPYRIGHT OFFICE.
#
# -------------------------------------------------------------------------------------
#
# Storage Blade Concurrent Code Load (CCL) Healthcheck Script
#
# *************************************************************************************
#
# 04-21-09 wjr 000001 Create program
# 04-22-09 wjr d16723 OBCL: NF: CCL preverify script
# 05-04-09 wjr d16709 RSSM: NCCL preverify logs reporting passed when drive in failed state
# 05-05-09 wjr d16787 OBCL: CtlrFwMgmt.conf changes for new BMC, FPGA formats
# 06-08-09 wjr d16975 CCL preverify script failed with incorrect messages
# 08-13-09 wjr d17429 FVT SCM - NCCL failing with invalid status for controller in service mode
# 08-25-09 wjr d17475 OBCL pre-verify come-from/go-to level check fails for pre-R2 to R2 upgrade
# 08-31-09 wjr d17512 OBCL changes to support ITR8251
# 09-02-09 wjr d17513 OBCL: H2031->H2033 and H2033->H2033 failed with FAILED_PREVERIFYFAILED
# 09-09-09 wjr d17587 Ut of preverify function on 012 package needs an alert that is stated more clearly
# 09-17-09 wjr d17644 OBCL pre-verify failed - "list drive" CLI cmd output format changed
# 09-18-09 wjr d17654 CCL preverify passed even though critical alert 9200 was present
# 09-22-09 wjr d17645 Alert needs to specify, more precisely, a controller in survivor mode
# 09-29-09 wjr d17697 OBCL 1.2.0.060 aborted when controller were in the expected state
# 10-01-09 wjr d17709 CCL preverify fails when only one DSM is installed in the chassis
# 11-16-09 wjr d18107 OBCL Single controller support
# 01-12-10 wjr d18324 CCL Preverify does not fail (or stop firmware update) when there is unhealthy pool
# 01-13-10 wjr d18038 SCM - CCL preverify does not stop fw update when one DSM is orderly dismounted
# 03-09-10 wjr d18524 Pre-Verify did not detect and fail with degraded pools
# 01-30-13 tny CQ68285 Preverify should also consider UNF/UNS state
#
# =====================================================================================
function show_syntax
{
  echo ''
  echo "$SCRIPT < -? | -h | -H [ -v ] [ -s ] [ -c CFG_STRING ] >"                      # d17429 - wjr
  echo ''
}
# =====================================================================================
function show_help
{
  show_syntax
  echo '    Required arguments (mutually exclusive):'
  echo ''
  echo '    None'
  echo ''
  echo '    Optional arguments:                                   Default'
  echo ''
  echo '    -v : Display Verbose Output                           Not displayed'
  echo '    -s : Display Summary Output                           Displayed'
  echo '    -c : Use CFG_STRING instead of CtlrFwMgmt.conf file   CtlrFwMgmt.conf'     # d17429 - wjr
  echo ''
  echo '    Return codes / Alerts:'
  echo ''
  echo '    Only the last error encountered is reported.  The RC is a positive'
  echo '    integer representing one of the following errors.'
  echo ''                                                                              # d17513 - wjr
  echo '    The following return codes / alerts are common for NCCL and CCL:'
  echo ''
  echo '    RC   Alert  Description'
  echo '    --   -----  ----------------------------------------------------------'
  echo '     1   10001  Microcode is not available in the process list'
  echo '     2   10002  Raid controller CLI interface is not available'
  echo '     3   10003  Raid controller CLI interface is not responding'
  echo '     4   10004  Raid controller Ctlr<0|1>(<state>) not in a valid state'       # d17587 - wjr
  echo '     5   10005  Raid controller Ctlr<0|1> not in a valid state'                # d17587 - wjr
  echo '     6   10006  Query for raid controller state failed'                        # d17587 - wjr
  echo '     7   10007  Ctlr<0|1> firmware upgrade is in a failed state'               # d17587 - wjr
  echo '     8   10008  Query for raid controller firmware upgrade status failed'      # d17587 - wjr
  echo '     9   10009  Overall DDM firmware upgrade is in a failed state'             # d17587 - wjr
  echo '    10   10010  Query for DDM firmware upgrade status failed'                  # d17587 - wjr
  echo '    11   10011  Found one or more failed or missing drives'                    # d17587 - wjr
  echo '    12   10012  Query for failed or missing drives failed'                     # d17587 - wjr
  echo '    13   10013  Drive in slot <slot> with master P/N <pn> is INVALID'          # d17587 - wjr
  echo '    14   10014  Query for master P/N of drive in slot <slot> returned a null'  # d17587 - wjr
  echo '    15   10015  Query for master P/N of drive in slot <slot> failed'           # d17587 - wjr
  echo '    16   10016  Query for drive slot ids failed'                               # d17587 - wjr
  echo '    17   10017  Query for valid drive master P/Ns returned a null'             # d17587 - wjr
  echo '    18   10018  Query for valid drive master P/Ns failed'                      # d17587 - wjr
  echo '    19   10019  Found one or more failed/non-viable drive pools'               # d17587 - wjr
  echo '    20   10020  Query for drive pool state failed'                             # d17587 - wjr
  echo '    21   10021  Found one or more critical alerts'                             # d17587 - wjr
  echo '    22   10022  Query for critical alerts failed'                              # d17587 - wjr
  echo '    23   10023  CLI command to configure alert failed'                         # d17587 - wjr
  echo '    24   10024  CLI command to configure alert failed to execute'              # d17587 - wjr
  echo '    25   10025  CLI command to generate alert failed'                          # d17587 - wjr
  echo '    26   10026  CLI command to generate alert failed to execute'               # d17587 - wjr
  echo '    32   10032  Could not find the CtlrFwMgmt.conf config file'                # d17587 - wjr
  echo '    33   10033  Query for Rc <component> min fw level failed'                  # d17587 - wjr
  echo '    34   10034  Query for Rc <component> max fw level failed'                  # d17587 - wjr
  echo '    35   10035  Query for Rc <component> fw level failed'                      # d17587 - wjr
  echo '    36   10036  <component> come-from fw level is incompatible'                # d17587 - wjr
  echo '    37   10037  <component> go-to level is incompatible'                       # d17587 - wjr
  echo '    38   10038  Found one or more unsupported drives'                          # d17587 - wjr
  echo '    39   10039  Query for unsupported drives failed'                           # d17587 - wjr
  echo '    40   10040  Query for raid controller configuration failed.'               # d18107 - wjr
  echo ''
  echo '    The following return codes / alerts are for CCL only:'
  echo ''
  echo '    RC   Alert  Description'
  echo '    --   -----  ----------------------------------------------------------'
  echo '    27   10027  Found one or more inactive drive ports'                        # d17587 - wjr
  echo '    28   10028  Query for drive port status failed'                            # d17587 - wjr
  echo '    29   10029  DSM SAS expander Encl<0|1> state is <state>'                   # d17587 - wjr
  echo '    30   10030  DSM SAS expander Encl<0|1> not active'                         # d17587 - wjr
  echo '    31   10031  Query for DSM SAS expanders failed'                            # d17587 - wjr
  echo '    41   10041  One or more DSM enclosure not mounted'                         # d18038 - wjr
  echo '    42   10042  Query for DSM mount state failed'                              # d18038 - wjr
  echo ''
}
# =====================================================================================
function log
{
  typeset ACTION=$1
  typeset DATE
  DATE=$(date +'%a %D %T')
  case $ACTION in
    open )
      remove_file $LOG
      echo "### Begin execution of $SCRIPT - $DATE" > $LOG
      echo "MSG: Invocation : $COMMAND $ARGS" >> $LOG
      echo "MSG: Version    : $VERSION" >> $LOG
      ;;
    close )
      echo "### End execution of $SCRIPT - $DATE" >> $LOG
      ;;
    * )
      echo "ERROR: Invalid action on log: $ACTION."
      script_abort 1
      ;;
  esac
}
# =====================================================================================
function handle_interrupt
{
  typeset ACTION=$1
  typeset USER_RESPONSE
  case $ACTION in
    terminate )
      case $MODE in
        interactive )
          echo 'Interrupt detected - terminating script.'
          script_cancel
          ;;
        * )
          echo 'WARNING : Interrupt detected - terminating script.'
          script_cancel
          ;;
      esac
      ;;
    ask )
      case $MODE in
        interactive )
          echo 'Interrupt detected - are you sure you want to quit (y/n)?'
          read USER_RESPONSE
          if [[ y = $USER_RESPONSE || \
                Y = $USER_RESPONSE ]]; then
            script_cancel
          fi
          ;;
        background )
          echo "WARNING : Interrupt detected - terminating script."
          script_cancel
          ;;
        * )
          echo 'Interrupt detected - are you sure you want to quit (y/n)?'
          read USER_RESPONSE
          if [[ y = $USER_RESPONSE || \
                Y = $USER_RESPONSE ]]; then
            script_cancel
          fi
          ;;
      esac
      ;;
    none )
      echo 'WARNING : Interrupt detected - no action taken.'
      ;;
    * )
      case $MODE in
        interactive )
          echo "ERROR: Invalid action on interrupt : $ACTION."
          script_cancel
          ;;
        * )
          echo "ERROR: Invalid action on interrupt : $ACTION."
          script_cancel
          ;;
      esac
      ;;
  esac
  CMD_EXIT='restart'
}
# =====================================================================================
function xprint
{
  if [[ debug = $1 ]]; then
    shift
    if [[ 'y' = $VERBOSE_DEBUG ]]; then
      echo $1
    fi
  else
    if [[ 'y' = $VERBOSE ]]; then
      echo $1
    fi
  fi
  echo $1 >> $LOG
}
# =====================================================================================
function script_abort
{
  trap 'handle_interrupt none' SIGINT SIGTERM SIGTSTP SIGQUIT
  remove_file $CMD_FILE
  if [[ -z $SCRIPT ]]; then
    SCRIPT='Storage Blade healthcheck script'
  fi
  if [[ -n $FUNCTION ]]; then
    if [[ main = $FUNCTION ]]; then
      echo "$SCRIPT failed in main, rc = $1."
    else
      echo "$SCRIPT failed in function $FUNCTION, rc = $1."
    fi
  else
    echo "$SCRIPT failed, rc = $1."
  fi
  append_file $SUMMARY_LOG $LOG                                                        # d17429 - wjr
  log close
  exit $1
}
# =====================================================================================
function script_cancel
{
  trap 'handle_interrupt none' SIGINT SIGTERM SIGTSTP SIGQUIT
  remove_file $CMD_FILE
  if [[ -z $SCRIPT ]]; then
    SCRIPT='Storage Blade healthcheck script'
  fi
  if [[ -n $FUNCTION ]]; then
    if [[ main = $FUNCTION ]]; then
      echo "$SCRIPT cancelled in main."
    else
      echo "$SCRIPT cancelled in function $FUNCTION."
    fi
  else
    echo "$SCRIPT cancelled."
  fi
  append_file $SUMMARY_LOG $LOG                                                        # d17429 - wjr
  log close
  exit 0
}
# =====================================================================================
function check_rc
{
  typeset FUNCTION='check_rc'
  xprint debug "TRC: Enter function $FUNCTION"
  trap 'handle_interrupt none' SIGINT SIGTERM SIGTSTP SIGQUIT
  typeset N_ARGS=$#
  if [[ 2 != $N_ARGS ]]; then
    xprint "ERROR: check_rc function called with wrong number of args : $N_ARGS."
    FUNCTION='check_rc'
    script_abort 1
  fi
  typeset RETURN_CODE=$1
  typeset LOCATION=$2
  if [[ 0 != $RETURN_CODE ]]; then
    xprint "ERROR: Non-zero return code."
    script_abort $LOCATION
  fi
  xprint debug "TRC: Exit function $FUNCTION"
}
# =====================================================================================
function make_log_dir
{
  if [[ ! -d $LOG_DIR ]]; then
    mkdir $LOG_DIR
    if [[ ! -d $LOG_DIR ]]; then
      echo "ERROR: Failed to create $SCRIPT log directory $LOG_DIR."
      script_abort 1
    fi
  fi
}
# =====================================================================================
function remove_file
{
  typeset FUNCTION='remove_file'
  xprint debug "TRC: Enter function $FUNCTION"
  typeset FILE=$1
  if [[ -f $FILE ]]; then
    rm -f $FILE
  fi
  xprint debug "TRC: Exit function $FUNCTION"
}
# =====================================================================================
function copy_file                                                                     # d17429 - wjr
{                                                                                      # d17429 - wjr
  typeset FUNCTION='copy_file'                                                         # d17429 - wjr
  xprint debug "TRC: Enter function $FUNCTION"                                         # d17429 - wjr
  typeset SOURCE_FILE=$1                                                               # d17429 - wjr
  typeset TARGET_FILE=$2                                                               # d17429 - wjr
  remove_file $TARGET_FILE                                                             # d17429 - wjr
  if [[ -f $SOURCE_FILE ]]; then                                                       # d17429 - wjr
    cp $SOURCE_FILE $TARGET_FILE                                                       # d17429 - wjr
    RC=$?                                                                              # d17429 - wjr
    if (( 0 != $RC )); then                                                            # d17429 - wjr
      xprint debug "MSG: Copy of $SOURCE_FILE failed - cp returned rc=$RC."            # d17429 - wjr
      script_abort 1                                                                   # d17429 - wjr
    fi                                                                                 # d17429 - wjr
  else                                                                                 # d17429 - wjr
    xprint debug "MSG: Copy of $SOURCE_FILE failed - file does not exist."             # d17429 - wjr
    script_abort 2                                                                     # d17429 - wjr
  fi                                                                                   # d17429 - wjr
  xprint debug "TRC: Exit function $FUNCTION"                                          # d17429 - wjr
}                                                                                      # d17429 - wjr
# =====================================================================================
function append_file                                                                   # d17429 - wjr
{
  typeset FUNCTION='append_file'                                                       # d17429 - wjr
  xprint debug "TRC: Enter function $FUNCTION"
  typeset SOURCE_FILE=$1
  typeset TARGET_FILE=$2
  typeset TMP_FILE=$LOG_DIR'/copy.tmp'
  if [[ -f $SOURCE_FILE ]]; then
    cat $TARGET_FILE $SOURCE_FILE > $TMP_FILE
    RC=$?
    if (( 0 == $RC )); then
      mv $TMP_FILE $TARGET_FILE
      RC=$?
      if (( 0 != $RC )); then
        xprint debug "MSG: Appending $SOURCE_FILE failed - mv returned rc=$RC."        # d17429 - wjr
      fi
    else
      xprint debug "MSG: Appending $SOURCE_FILE failed - cat returned rc=$RC."         # d17429 - wjr
    fi
  else
    xprint debug "MSG: $SOURCE_FILE file does not exist - no append."                  # d17513 - wjr
  fi
  xprint debug "TRC: Exit function $FUNCTION"
}
# =====================================================================================
function check_arguments
{
  typeset FUNCTION='check_arguments'
  xprint debug "TRC: Enter function $FUNCTION"
  trap 'handle_interrupt ask' SIGINT SIGTERM SIGTSTP SIGQUIT
  if (( N_OPERATIONS > 1 )); then
    echo "ERROR: Invalid command line arguments - multiple options selected."
    show_help
    script_abort 1
  fi
  case $OPERATION in
    ccl_check )
      :
      ;;
    * )
      echo "ERROR: Invalid operation : $OPERATION."
      show_help
      script_abort 2
  esac 
  case $MODE in
    interactive | background )
      :
      ;;
    * )
      echo "ERROR: Invalid execution mode : $MODE."
      show_help
      script_abort 3
      ;;
  esac 
  xprint debug "TRC: Exit function $FUNCTION"
}
# =====================================================================================
function run_cmd
{
  typeset FUNCTION='run_cmd'
  xprint debug "TRC: Enter function $FUNCTION"
  trap 'handle_interrupt ask' SIGINT SIGTERM SIGTSTP SIGQUIT
  typeset OUTPUT='assign'
  xprint "MSG: Running RC system command:"
  if [[ -z $CMD ]]; then
    echo "ERROR: System command is undefined."
    script_abort 1
  fi
  if [[ -z $CHECK_TYPE ]]; then
    CHECK_TYPE='continue'
  fi
  xprint "CMD: $CMD"
  if [[ -n $1 ]]; then
    OUTPUT=$1
  fi
  if [[ 'file' == $OUTPUT ]]; then
    remove_file $CMD_FILE
    CMD_EXIT='restart'
    while [[ restart = $CMD_EXIT ]];
    do
      CMD_EXIT='continue'
      eval "$CMD 1>$CMD_FILE"
      RC=$?
    done
  else
    CMD='RESULT=$('$CMD')'
    eval "$CMD"
    RC=$?
  fi
  case $CHECK_TYPE in
    check )
      check_rc $RC 2
      ;;
    continue )
      :
      ;;
    * )
      echo "ERROR: Invalid command return code check: $CHECK_TYPE."
      script_abort 3
      ;;
  esac
  xprint debug "TRC: Exit function $FUNCTION"
  return $RC
}
# =====================================================================================
function prt_cli_session
{
  typeset FUNCTION='show_cli_session'
  xprint debug "TRC: Enter function $FUNCTION"
  if [[ 'y' = $VERBOSE_CLI ]]; then 
    echo 'CLI Session results --------------------------------------------------' >> $LOG
    if [[ -f $CMD_FILE ]]; then
      cat $CMD_FILE >> $LOG
    else
      echo '< Empty file >' >> $LOG
    fi
    echo '----------------------------------------------------------------------' >> $LOG
  fi
  xprint debug "TRC: Exit function $FUNCTION"
}
# =====================================================================================
function prt_test_result
{
  typeset FUNCTION='prt_test_result'
  xprint debug "TRC: Enter function $FUNCTION"
  typeset STANZA=$1
  shift
  typeset TEST_NAME=$*
  if [[ 'y' = $SUMMARY ]]; then 
    case $STANZA in
      header )
        typeset TEST_TYPE=$*
        remove_file $SUMMARY_LOG
        echo ''
        echo '' > $SUMMARY_LOG
        TEN_DASHES='----------'
        FIFTY_DASHES='--------------------------------------------------'
        printf "%-50.50s %10s\n" "$TEST_TYPE Healthcheck Test" 'Status'
        printf "%-50.50s %10s\n" "$TEST_TYPE Healthcheck Test" 'Status' >> $SUMMARY_LOG
        printf "%-50.50s %10s\n" $FIFTY_DASHES $TEN_DASHES
        printf "%-50.50s %10s\n" $FIFTY_DASHES $TEN_DASHES >> $SUMMARY_LOG
        echo ''
        echo '' >> $SUMMARY_LOG
        ;;
      passed )
        printf "%-50.50s %10s\n" "$TEST_NAME" 'Passed'
        printf "%-50.50s %10s\n" "$TEST_NAME" 'Passed' >> $SUMMARY_LOG
        ;;
      failed )
        printf "%-50.50s %10s\n" "$TEST_NAME" 'FAILED'
        printf "%-50.50s %10s\n" "$TEST_NAME" 'FAILED' >> $SUMMARY_LOG
        ;;
      skipped )
        printf "%-50.50s %10s\n" "$TEST_NAME" 'Skipped'
        printf "%-50.50s %10s\n" "$TEST_NAME" 'Skipped' >> $SUMMARY_LOG
        ;;
      space )
        echo ''
        echo '' >> $SUMMARY_LOG
        ;;
      trailer )
        echo ''
        echo '' >> $SUMMARY_LOG
        ;;
      * )
        echo "ERROR: Invalid $FUNCTION argument : $STANZA."
        script_abort 1
        ;;
    esac
  fi
  xprint debug "TRC: Exit function $FUNCTION"
}
# =====================================================================================
function set_rc
{
  typeset FUNCTION='set_rc'
  xprint debug "TRC: Enter function $FUNCTION"
  trap 'handle_interrupt none' SIGINT SIGTERM SIGTSTP SIGQUIT
  typeset -i RC=$1
  if [[ -n $RC ]]; then
    if (( $RC > 0 )); then
      RC_FORMATTED=$(printf "%02d" $1)                                                 # d17513 - wjr
#     if [[ 'undefined' == $CTLR_ID ]]; then                                           # d17587 - wjr
      ALERT='100'$RC_FORMATTED                                                         # d17587 - wjr
#     else                                                                             # d17587 - wjr
#       CTLR_N=$(echo $CTLR_ID | tr -d [A-Za-z])                                       # d17587 - wjr
#       ALERT='10'$CTLR_N$RC_FORMATTED                                                 # d17587 - wjr
#     fi                                                                               # d17587 - wjr
      if [[ -n $ALERT ]]; then
        shift
        typeset ALERT_MSG=$*
        xprint "ERROR: $ALERT_MSG"
        configure_alert_template $ALERT $ALERT_MSG
        generate_alert $ALERT
      fi
    fi
    if (( $RC > $RETURN_CODE )); then
      xprint "MSG: Setting overall return code to $RC"
      RETURN_CODE=$RC
    fi
  fi
  xprint debug "TRC: Exit function $FUNCTION"
}
# =====================================================================================
function append_err_msg                                                                # d17513 - wjr
{                                                                                      # d17513 - wjr
  typeset FUNCTION='append_err_msg'                                                    # d17513 - wjr
  xprint debug "TRC: Enter function $FUNCTION"                                         # d17513 - wjr
  typeset MSG_ID=$1                                                                    # d17513 - wjr
  shift                                                                                # d17513 - wjr
  typeset MSG=$*                                                                       # d17513 - wjr
  if [[ '' == ${ERR_MSG[$MSG_ID]} ]]; then                                             # d17513 - wjr
    ERR_MSG[$MSG_ID]=$MSG                                                              # d17513 - wjr
  else                                                                                 # d17513 - wjr
    ERR_MSG[$MSG_ID]=${ERR_MSG[$MSG_ID]}', '$MSG                                       # d17513 - wjr
  fi                                                                                   # d17513 - wjr
  xprint debug "TRC: Exit function $FUNCTION"                                          # d17513 - wjr
}                                                                                      # d17513 - wjr
# =====================================================================================
function query_address
{
  typeset FUNCTION='query_address'
  xprint debug "TRC: Enter function $FUNCTION"
  trap 'handle_interrupt ask' SIGINT SIGTERM SIGTSTP SIGQUIT
  xprint "MSG: Query raid controller IP address."
  typeset RC=0
  CMD='ifconfig eth0.4095 | grep Bcast | cut -d ":" -f 2 | cut -d " " -f 1'
  run_cmd
  if [[ -n $RESULT ]]; then
    IP_ADDRESS=$RESULT
    xprint "MSG: Raid controller IP address: $IP_ADDRESS"
  else
    xprint "ERROR: Query for Raid controller IP address failed."
    script_abort 1
  fi
  xprint debug "TRC: Exit function $FUNCTION"
}
# =====================================================================================
function query_ctlr_id
{
  typeset FUNCTION='query_ctlr_id'
  xprint debug "TRC: Enter function $FUNCTION"
  trap 'handle_interrupt ask' SIGINT SIGTERM SIGTSTP SIGQUIT
  xprint "MSG: Query raid controller ID."
  CMD='echo "commparams -get" | /opt/arts/Cli/cli/linux_ppc/cli'
  run_cmd file
  CTLR_IDS='0 1'
  if [[ -f $CMD_FILE ]]; then
    for ctlr_id in $CTLR_IDS
    do
      IP_INFO=$(cat $CMD_FILE | \
                sed -n "/Ctlr$ctlr_id/,/GATEWAY/p" | \
                grep $IP_ADDRESS)                                                      # d16787 - wjr
      if [[ -n $IP_INFO ]]; then
        CTLR_ID='Ctlr'$ctlr_id
        xprint "MSG: Raid controller ID is: $CTLR_ID"
        break
      fi
    done
    if [[ 'undefined' == $CTLR_ID ]]; then
      xprint "ERROR: Search for Ctlr ID in CLI commparams output failed."
      script_abort 1
    fi
  else
    xprint "ERROR: Query for raid controller ID failed."
    script_abort 2
  fi
  xprint debug "TRC: Exit function $FUNCTION"
}
# =====================================================================================
function query_tsal
{
  typeset FUNCTION='query_tsal'
  xprint debug "TRC: Enter function $FUNCTION"
  trap 'handle_interrupt ask' SIGINT SIGTERM SIGTSTP SIGQUIT
  typeset MSG_TXT='Raid Controller Daemon status'
  xprint "MSG: Checking raid controller microcode state."
  typeset RC=0
  CMD='ps -ef | grep tsal'
  run_cmd
  if [[ -n $RESULT ]]; then
    prt_test_result passed $MSG_TXT
    xprint "MSG: The microcode process is running."
  else
    prt_test_result failed $MSG_TXT
    set_rc 1 "Microcode is not available in the process list."                         # d17513 - wjr
    RC=1
  fi
  xprint debug "TRC: Exit function $FUNCTION"
  return $RC
}
# =====================================================================================
function query_cli
{
  typeset FUNCTION='query_cli'
  xprint debug "TRC: Enter function $FUNCTION"
  trap 'handle_interrupt ask' SIGINT SIGTERM SIGTSTP SIGQUIT
  typeset MSG_TXT='Raid Controller CLI interface'
  xprint "MSG: Checking raid controller CLI interface."
  typeset RC=0
  CMD='echo exit | /opt/arts/Cli/cli/linux_ppc/cli'
  run_cmd
  if [[ -n $RESULT ]]; then
    xprint "MSG: Started raid controller CLI interface - getting status."
  else
    printf "%-50s %10s\n" $MSG_TXT 'FAILED'
    set_rc 2 "Raid controller CLI interface is not available."                         # d17513 - wjr
    RC=2
  fi
  CMD='echo swversion | /opt/arts/Cli/cli/linux_ppc/cli'
  run_cmd file
  RESULT=$(cat $CMD_FILE | \
           grep 'Software version')
  if [[ -n $RESULT ]]; then
    prt_test_result passed $MSG_TXT
    prt_cli_session
  else
    prt_test_result failed $MSG_TXT
    set_rc 3 "Raid controller CLI interface is not responding."                        # d17513 - wjr
    RC=3
  fi
  xprint debug "TRC: Exit function $FUNCTION"
  return $RC
}
# =====================================================================================
function verify_rc_state
{
  typeset FUNCTION='verify_rc_state'
  xprint debug "TRC: Enter function $FUNCTION"
  trap 'handle_interrupt ask' SIGINT SIGTERM SIGTSTP SIGQUIT
  typeset VERIFY_STATE=$1
  typeset MODE=$2
  typeset QUERY_STATES='SYSCONFIG
                        STARTING
                        BINDING
                        RUNNING
                        MASTERBOUND
                        SLAVEBOUND
                        PRIMARY
                        SECONDARY
                        STANDALONE
                        STANDBY
                        POWERFAILRECOVERY
                        WAITFORREBOOT
                        SURVIVOR
                        STARTASSURVIVOR
                        NOT_PRESENT
                        FAILED
                        REBINDING
                        NOTAVAILABLE
                        FAILINGOVER
                        SERVICE
                        SHUTDOWN
                        UNKNOWN
                        BATTERYFAILRECOVERY
                        SHTDN_IND_DATA_LOSS
                        NOTEXISTS
                        BOUND_PFR1
                        BOUND_PFR2
                        BOUND_PFRS
                        SURVIVOR_PFR1
                        SURVIVOR_PFR2'                                                 # d17697 - wjr
  typeset MSG_TXT='Raid Controller State'
  typeset CTLR_IDS='0 1'
  typeset VALID_STATES=''
  typeset RC_STATE=''
  case $VERIFY_STATE in
    SERVICE )
      VALID_STATES='SERVICE'
      ;;
    BOUND )
      VALID_STATES='PRIMARY|SECONDARY|MASTER|SLAVE'
      ;;
    * )
      echo "ERROR: Invalid $FUNCTION argument: $VERIFY_STATE."
      script_abort 2
      ;;
  esac
  typeset RC=0
  xprint "MSG: Query CLI for raid controller state."
  CMD='echo "list controller" | /opt/arts/Cli/cli/linux_ppc/cli'
  run_cmd file
  if [[ -f $CMD_FILE ]]; then
    xprint "MSG: Query for raid controller status successful."
    prt_cli_session
    for ctlr in $CTLR_IDS
    do
      MSG_TXT="Ctlr$ctlr Raid Controller State"
      xprint "MSG: Checking Ctlr$ctlr raid controller state."
      RC_STATE=''
      for state in $QUERY_STATES
      do
        RESULT=$(cat $CMD_FILE | \
                 grep $state | \
                 grep Ctlr$ctlr)
        if [[ -n $RESULT ]]; then
          RC_STATE=$state
          xprint "MSG: Ctlr$ctlr state: $RC_STATE."
          break
        fi
      done
      if [[ -n $RC_STATE ]]; then
        if [[ -n $(echo $RC_STATE | grep -E "$VALID_STATES") ]]; then
          if [[ 'silent' != $MODE ]]; then
            prt_test_result passed $MSG_TXT
          fi
        else
          if [[ 'silent' != $MODE ]]; then
            prt_test_result failed $MSG_TXT
            append_err_msg 4 'Ctlr'$ctlr'('$RC_STATE')'                                # d17513 - wjr
          fi
          RC=4
        fi
      else
        if [[ 'silent' != $MODE ]]; then
          prt_test_result failed $MSG_TXT
          append_err_msg 5 'Ctlr'$ctlr                                                 # d17513 - wjr
        fi
        RC=5
      fi
    done
    if [[ 'silent' != $MODE ]]; then                                                   # d17513 - wjr
      if [[ '' != ${ERR_MSG[4]} ]]; then                                               # d17513 - wjr
        set_rc 4 "Rc ${ERR_MSG[4]} not in a valid state."                              # d17645 - wjr
      fi                                                                               # d17513 - wjr
      if [[ '' != ${ERR_MSG[5]} ]]; then                                               # d17513 - wjr
        set_rc 5 "Rc ${ERR_MSG[5]} not in a valid state."                              # d17513 - wjr
      fi                                                                               # d17513 - wjr
    fi                                                                                 # d17513 - wjr
  else
    if [[ 'silent' != $MODE ]]; then
      prt_test_result failed $MSG_TXT
      set_rc 6 "$SCRIPT query for raid controller state failed."                       # d17587 - wjr
    fi
    RC=6
  fi
  xprint debug "TRC: Exit function $FUNCTION"
  return $RC
}
# =====================================================================================
function query_rc_fw_status
{
  typeset FUNCTION='query_rc_fw_status'
  xprint debug "TRC: Enter function $FUNCTION"
  trap 'handle_interrupt ask' SIGINT SIGTERM SIGTSTP SIGQUIT
  typeset MSG_TXT='Raid Controller Firmware Update Status'
  typeset CTLR_IDS='0 1'
  typeset GOOD_STATES='undefined'
  typeset GOOD_STATE_LIST='undefined'                                                  # d17697 - wjr
  typeset RC=0
  xprint "MSG: Query CLI for RC firmware upgrade status."
  CMD='echo "firmwareupgrade -getstatus" | /opt/arts/Cli/cli/linux_ppc/cli'
  run_cmd file
  if [[ -f $CMD_FILE ]]; then
    xprint "MSG: Query for firmware upgrade status successful."
    prt_cli_session
    for ctlr in $CTLR_IDS
    do
      MSG_TXT="Ctlr$ctlr Firmware Update Status"
      xprint "MSG: Checking Ctlr$ctlr firmware upgrade status."
      if [[ $ctlr == $CTLR_ID ]]; then
        GOOD_STATES='FW_UPGRADE_IDLE'
      else
        GOOD_STATES='FW_UPGRADE_IDLE|
                     FW_UPGRADE_PREVERIFYSTARTED|
                     FW_UPGRADE_INITIALIZING|
                     FW_UPGRADE_DONE|
                     FW_UPGRADE_NOTREQUIRED|
                     FW_UPGRADE_NO_PEER|
                     FW_UPGRADE_SCHEDULED|
                     FW_UPGRADE_BMCUPGRADESTARTED| 
                     FW_UPGRADE_BMCUPGRADEDONE|
                     FW_UPGRADE_SESUPGRADESTARTED|
                     FW_UPGRADE_SESUPGRADEDONE|
                     FW_UPGRADE_NEWCOMPONENT|
                     FW_UPGRADE_CPLDUPGRADESTARTED|
                     FW_UPGRADE_CPLDUPGRADEDONE|
                     FW_UPGRADE_FPGAUPGRADESTARTED|
                     FW_UPGRADE_FPGAUPGRADEDONE|
                     FW_UPGRADE_SASUPGRADESTARTED|
                     FW_UPGRADE_SASUPGRADEDONE|
                     FW_UPGRADE_FIRSTDSMUPGRADEDONE|
                     FW_UPGRADE_DSMUPGRADEDONE|
                     FW_UPGRADE_PUSHFW_STARTED|
                     FW_UPGRADE_READYTOBOOT'                                           # d17697 - wjr
      fi
      FW_STATE=$(cat $CMD_FILE | \
                 grep "Ctlr $ctlr" | \
                 grep "Status" | \
                 cut -d ' ' -f 7)                                                      # d17697 - wjr
      if [[ -n $FW_STATE ]]; then
        xprint "MSG: Ctlr$ctlr firmware upgrade status: $FW_STATE."
        GOOD_STATE_LIST=$(echo $GOOD_STATES | tr -d ' ')                               # d17697 - wjr
        if [[ -n $(echo $FW_STATE | grep -E "$GOOD_STATE_LIST") ]]; then               # d17697 - wjr
          if [[ 'silent' != $MODE ]]; then
            prt_test_result passed $MSG_TXT
          fi
        else
          if [[ 'silent' != $MODE ]]; then
            prt_test_result failed $MSG_TXT
            append_err_msg 7 'Ctlr'$ctlr'('$FW_STATE')'                                # d17587 - wjr
          fi
          RC=7
        fi
      else
        if [[ 'silent' != $MODE ]]; then                                               # d17697 - wjr
          prt_test_result failed $MSG_TXT                                              # d17697 - wjr
          append_err_msg 7 'Ctlr'$ctlr                                                 # d17513 - wjr
        fi                                                                             # d17697 - wjr
        RC=7                                                                           # d17697 - wjr
      fi                                                                               # d17697 - wjr
    done
    if [[ '' != ${ERR_MSG[7]} ]]; then                                                 # d17513 - wjr
      set_rc 7 "${ERR_MSG[7]} is in an invalid fw upgrade state."                      # d17697 - wjr
    fi                                                                                 # d17513 - wjr
  else
    prt_test_result failed $MSG_TXT
    set_rc 8 "$SCRIPT query for raid controller firmware upgrade status failed."       # d17587 - wjr
    RC=8
  fi
  xprint debug "TRC: Exit function $FUNCTION"
  return $RC
}
# =====================================================================================
function query_ddm_fw_status
{
  typeset FUNCTION='query_ddm_fw_status'
  xprint debug "TRC: Enter function $FUNCTION"
  trap 'handle_interrupt ask' SIGINT SIGTERM SIGTSTP SIGQUIT
  typeset MSG_TXT='Disk Drive Firmware Update Status'
  typeset GOOD_STATES='IDLE
                       NO_CANDIDATE_DRIVE
                       FAILED
                       DONE'
  typeset RC=0
  xprint "MSG: Query CLI for DDM firmware upgrade status."
  CMD='echo "firmwareupgrade -getstatus" | /opt/arts/Cli/cli/linux_ppc/cli'
  run_cmd file
  if [[ -f $CMD_FILE ]]; then
    xprint "MSG: Query for firmware upgrade status successful."
    prt_cli_session
    xprint "MSG: Checking DDM firmware upgrade status."
    FW_STATE=''
    for state in $GOOD_STATES
    do
      state=$(echo $state | tr '_' ' ')
      RESULT=$(cat $CMD_FILE | \
               grep "$state" | \
               grep "Drive" | \
               grep "Status")
      if [[ -n $RESULT ]]; then
        FW_STATE=$state
        xprint "MSG: Overall DDM firmware upgrade status: $FW_STATE."
        break
      fi
    done
    if [[ -n $FW_STATE ]]; then
      prt_test_result passed $MSG_TXT
    else
      prt_test_result failed $MSG_TXT
      set_rc 9 "Overall DDM firmware upgrade is in a failed state: $FS_STATE."         # d17587 - wjr
      RC=9
    fi
  else
    prt_test_result failed $MSG_TXT
    set_rc 10 "$SCRIPT query for DDM firmware upgrade status failed."                  # d17587 - wjr
    RC=10
  fi
  xprint debug "TRC: Exit function $FUNCTION"
  return $RC
}
# =====================================================================================
function query_bad_drives
{
  typeset FUNCTION='query_bad_drives'
  xprint debug "TRC: Enter function $FUNCTION"
  trap 'handle_interrupt ask' SIGINT SIGTERM SIGTSTP SIGQUIT
  typeset MSG_TXT='No failed or missing disk drives'
  typeset SLOT_ID=''                                                                   # d17587 - wjr
  typeset DRIVE_STATE=''                                                               # d17587 - wjr
  typeset DRIVE_INFO=''                                                                # d17587 - wjr
  typeset BAD_STATE_LIST='M|P|U'                                                       # d17587 - wjr
  typeset EXCLUDE_LIST='UNS|UNF|IMD'                                                   # jylu - let UNS/UNF/IMD pass preverify 2013/03/04
  typeset DRIVE_MSG=''                                                                 # d17587 - wjr
  typeset RC=0
  xprint "MSG: Query CLI for failed or missing drives."
  CMD='echo "list drive" | /opt/arts/Cli/cli/linux_ppc/cli'
  run_cmd file
  if [[ -f $CMD_FILE ]]; then
    xprint "MSG: Query for failed or missing drives successful."
    prt_cli_session
    xprint "MSG: Checking drive state."
    RESULT=$(cat $CMD_FILE | \
             grep '|' | \
             grep -v "<CLI>" | \
             grep -v "<cli>" | \
             grep -v "_" | \
             grep -v 'Drive' | \
             grep -v 'Current Machine' | \
             grep -v 'Usage:' | \
             grep -v 'foreign' | \
             grep -v 'current' | \
             grep -v 'State:' | \
             grep -v 'initialized' | \
             grep -v 'incompatible' | \
             grep -v 'firmware' | \
             grep -v 'Error Message File' | \
             grep -v 'No drives reported' | \
             cut -d '|' -f 3,8 | \
             tr -d ' ')                                                                # d17644 - wjr
    for drive in $RESULT                                                               # d17587 - wjr
    do                                                                                 # d17587 - wjr
      SLOT_ID=$(echo $drive | cut -d '|' -f 1)                                         # d17587 - wjr
      DRIVE_STATE=$(echo $drive | cut -d '|' -f 2)                                     # d17587 - wjr
      DRIVE_INFO=$(echo $DRIVE_STATE | grep -E $BAD_STATE_LIST | grep -vE $EXCLUDE_LIST)                                                                                    # jylu - let UNS/UNF/IMD pass preverify 2013/03/04
      if [[ -z $DRIVE_INFO ]]; then                                                    # d17587 - wjr
        xprint "MSG: Drive in slot $SLOT_ID is $DRIVE_STATE."                          # d17587 - wjr
      else                                                                             # d17587 - wjr
        if [[ 'M' == $DRIVE_STATE ]]; then                                             # d17587 - wjr
          DRIVE_MSG='missing'                                                          # d17587 - wjr
        elif [[ 'P' == $DRIVE_STATE ]]; then                                           # d17587 - wjr
          DRIVE_MSG='PFA predicted failure'                                            # d17587 - wjr
        elif [[ 'U' == $DRIVE_STATE ]]; then                                           # d17587 - wjr
          DRIVE_MSG='unreliable'                                                       # d17587 - wjr
        else                                                                           # d17587 - wjr
          DRIVE_MSG=$DRIVE_STATE                                                       # d17587 - wjr
        fi
        append_err_msg 11 'Slot '$SLOT_ID'('$DRIVE_MSG')'                              # d17587 - wjr
        RC=11
      fi
    done                                                                               # d17587 - wjr
    if [[ '' != ${ERR_MSG[11]} ]]; then                                                # d17587 - wjr
      set_rc 11 "Found one or more failed or missing drives: ${ERR_MSG[11]}."          # d17587 - wjr
    fi                                                                                 # d17587 - wjr
  else
    set_rc 12 "$SCRIPT query for failed or missing drives failed."                     # d17587 - wjr
    RC=12
  fi
  if (( 0 == $RC )); then                                                              # d17587 - wjr
    prt_test_result passed $MSG_TXT                                                    # d17587 - wjr
  else                                                                                 # d17587 - wjr
    prt_test_result failed $MSG_TXT                                                    # d17587 - wjr
  fi                                                                                   # d17587 - wjr
  xprint debug "TRC: Exit function $FUNCTION"
  return $RC
}
# =====================================================================================
function query_unsupported_drive_pns                                                   # d17587 - wjr
{
  typeset FUNCTION='query_unsupported_drive_pns'                                       # d17587 - wjr
  xprint debug "TRC: Enter function $FUNCTION"
  trap 'handle_interrupt ask' SIGINT SIGTERM SIGTSTP SIGQUIT
  typeset MSG_TXT='No disk drives with unsupported p/n'                                # d17587 - wjr
  typeset RC=0
  typeset DRIVE_ATTRIBUTES='undefined'                                                 # d17513 - wjr
  xprint "MSG: Query $DRIVE_ATTRIBUTES file for valid drive master P/Ns."
  if [[ -f $ALT_DRIVE_FILE ]]; then                                                    # d17513 - wjr
    xprint "MSG: Using DriveAttributes.ini file in alternate flash."                   # d17513 - wjr
    DRIVE_ATTRIBUTES=$ALT_DRIVE_FILE                                                   # d17513 - wjr
  elif [[ -f $DRIVE_FILE ]]; then                                                      # d17513 - wjr
    xprint "MSG: Using DriveAttributes.ini file in primary flash."                     # d17513 - wjr
    DRIVE_ATTRIBUTES=$DRIVE_FILE                                                       # d17513 - wjr
  else                                                                                 # d17513 - wjr
    xprint "MSG: No DriveAttributes.ini file - skipping check."                        # d17513 - wjr
    prt_test_result skipped $MSG_TXT                                                   # d17513 - wjr
  fi                                                                                   # d17513 - wjr
  if [[ 'undefined' != $DRIVE_ATTRIBUTES ]]; then                                      # d17513 - wjr
    if [[ -f $DRIVE_ATTRIBUTES ]]; then
      VALID_PNS=$(cat $DRIVE_ATTRIBUTES | \
                  grep '_productMasterId' | \
                  cut -d '=' -f 2)
      if [[ -n $VALID_PNS ]]; then
        xprint "MSG: Valid P/Ns : $VALID_PNS"
        xprint "MSG: Query CLI for drive slot ids."
        CMD='echo "list drive" | /opt/arts/Cli/cli/linux_ppc/cli'
        run_cmd file
        if [[ -f $CMD_FILE ]]; then
          xprint "MSG: Query for drive slot ids."
          prt_cli_session
          SLOT_IDS=$(cat $CMD_FILE | \
                     grep '|' | \
                     grep -v "<CLI>" | \
                     grep -v "<cli>" | \
                     grep -v "_" | \
                     grep -v 'Drive' | \
                     grep -v 'Current Machine' | \
                     grep -v 'Usage:' | \
                     grep -v 'foreign' | \
                     grep -v 'current' | \
                     grep -v 'State:' | \
                     grep -v 'initialized' | \
                     grep -v 'incompatible' | \
		     grep -v 'firmware' | \
                     grep -v 'Error Message File' | \
                     grep -v 'No drives reported' | \
                     cut -d '|' -f 3)                                                    # d17644 - wjr
          if [[ -z $SLOT_IDS ]]; then
            xprint "MSG: Query returned a null list - no drives found."
          else
            for slot in $SLOT_IDS
            do
              xprint "MSG: Query CLI for drive master pn in slot $slot."
              CMD="echo \"detail drive -slot $slot\" | /opt/arts/Cli/cli/linux_ppc/cli"
              run_cmd file
              if [[ -f $CMD_FILE ]]; then
                xprint "MSG: Checking drive in slot $slot master pn."
                prt_cli_session
                PN=$(cat $CMD_FILE | \
                     grep 'Product Master ID' | \
                     cut -d ':' -f 2)
                if [[ -n $PN ]]; then
                  IS_VALID=$(echo $VALID_PNS | grep $PN)
                  if [[ -n $IS_VALID ]]; then
                    xprint "MSG: Drive in slot $slot with master P/N $PN is valid."
                  else
                    append_err_msg 13 $slot'(P/N'$PN')'                                # d17587 - wjr
                    RC=13
                  fi
                else
                  append_err_msg 14 $slot                                              # d17513 - wjr
                  RC=14
                fi
              else
                append_err_msg 15 $slot                                                # d17513 - wjr
                RC=15
              fi
            done
            if [[ '' != ${ERR_MSG[13]} ]]; then                                        # d17513 - wjr
              set_rc 13 "Drive(s) in slot(s) ${ERR_MSG[13]} have an INVALID P/N."      # d17513 - wjr
            fi                                                                         # d17513 - wjr
            if [[ '' != ${ERR_MSG[14]} ]]; then                                        # d17513 - wjr
              set_rc 14 "$SCRIPT query for master P/N of drive(s) in slot(s) ${ERR_MSG[14]} returned a null." # d17587 - wjr
            fi                                                                         # d17513 - wjr
            if [[ '' != ${ERR_MSG[15]} ]]; then                                        # d17513 - wjr
              set_rc 15 "$SCRIPT query for master P/N of drive(s) in slot(s) ${ERR_MSG[15]} failed." # d17587 - wjr
            fi                                                                         # d17513 - wjr
          fi
        else
          set_rc 16 "$SCRIPT query for drive slot ids failed."                         # d17587 - wjr
          RC=16
        fi
      else
        set_rc 17 "$SCRIPT query for valid drive master P/Ns returned a null."         # d17587 - wjr
        RC=17
      fi
    else
      set_rc 18 "$SCRIPT query for valid drive master P/Ns failed."                    # d17587 - wjr
      RC=18
    fi
    if (( 0 == $RC )); then
      prt_test_result passed $MSG_TXT
    else
      prt_test_result failed $MSG_TXT
    fi
  fi                                                                                   # d17513 - wjr
  xprint debug "TRC: Exit function $FUNCTION"
  return $RC
}
# =====================================================================================
function query_unsupported_drives                                                      # d17587 - wjr
{                                                                                      # d17587 - wjr
  typeset FUNCTION='query_unsupported_drives'                                          # d17587 - wjr
  xprint debug "TRC: Enter function $FUNCTION"                                         # d17587 - wjr
  trap 'handle_interrupt ask' SIGINT SIGTERM SIGTSTP SIGQUIT                           # d17587 - wjr
  typeset MSG_TXT='No unsupported disk drives'                                         # d17587 - wjr
  typeset SLOT_ID=''                                                                   # d17587 - wjr
  typeset DRIVE_STATE=''                                                               # d17587 - wjr
  typeset DRIVE_INFO=''                                                                # d17587 - wjr
  typeset BAD_STATE_LIST='UNS|IMD|UNF'                                                 # d17587 - wjr
  typeset DRIVE_MSG=''                                                                 # d17587 - wjr
  typeset RC=0                                                                         # d17587 - wjr
  xprint "MSG: Query CLI for failed or missing drives."                                # d17587 - wjr
  CMD='echo "list drive" | /opt/arts/Cli/cli/linux_ppc/cli'                            # d17587 - wjr
  run_cmd file                                                                         # d17587 - wjr
  if [[ -f $CMD_FILE ]]; then                                                          # d17587 - wjr
    xprint "MSG: Query for unsupported drives successful."                             # d17587 - wjr
    prt_cli_session                                                                    # d17587 - wjr
    xprint "MSG: Checking drive state."                                                # d17587 - wjr
    RESULT=$(cat $CMD_FILE | \
             grep '|' | \
             grep -v "<CLI>" | \
             grep -v "<cli>" | \
             grep -v "_" | \
             grep -v 'Drive' | \
             grep -v 'Current Machine' | \
             grep -v 'Usage:' | \
             grep -v 'foreign' | \
             grep -v 'current' | \
             grep -v 'State:' | \
             grep -v 'initialized' | \
             grep -v 'incompatible' | \
             grep -v 'firmware' | \
             grep -v 'Error Message File' | \
             grep -v 'No drives reported' | \
             cut -d '|' -f 3,8 | \
             tr -d ' ')                                                                # d17644 - wjr
    for drive in $RESULT                                                               # d17587 - wjr
    do                                                                                 # d17587 - wjr
      SLOT_ID=$(echo $drive | cut -d '|' -f 1)                                         # d17587 - wjr
      DRIVE_STATE=$(echo $drive | cut -d '|' -f 2)                                     # d17587 - wjr
      DRIVE_INFO=$(echo $DRIVE_STATE | grep -E $BAD_STATE_LIST)                        # d17644 - wjr
      if [[ -z $DRIVE_INFO ]]; then                                                    # d17587 - wjr
        xprint "MSG: Drive in slot $SLOT_ID is supported."                             # d17587 - wjr
      else                                                                             # d17587 - wjr
        if [[ 'UNS' == $DRIVE_STATE ]]; then                                           # d17587 - wjr
          DRIVE_MSG='unsuppored (non-IBM) drive'                                       # d17587 - wjr
        elif [[ 'IMD' == $DRIVE_STATE ]]; then                                         # d17587 - wjr
          DRIVE_MSG='incompatible metadata'                                            # d17587 - wjr
        elif [[ 'UNF' == $DRIVE_STATE ]]; then                                         # d17587 - wjr
          DRIVE_MSG='firmware version unsupported for drive'                           # d17587 - wjr
        else                                                                           # d17587 - wjr
          DRIVE_MSG=$DRIVE_STATE                                                       # d17587 - wjr
        fi                                                                             # d17587 - wjr
        append_err_msg 38 'Slot '$SLOT_ID'('$DRIVE_MSG')'                              # d17587 - wjr
        RC=38                                                                          # d17587 - wjr
      fi                                                                               # d17587 - wjr
    done                                                                               # d17587 - wjr
    if [[ '' != ${ERR_MSG[38]} ]]; then                                                # d17587 - wjr
      set_rc 38 "Found one or more unsupported drives: ${ERR_MSG[11]}."                # d17587 - wjr
    fi                                                                                 # d17587 - wjr
  else                                                                                 # d17587 - wjr
    set_rc 39 "$SCRIPT query for unsupported drives failed."                           # d17587 - wjr
    RC=39                                                                              # d17587 - wjr
  fi                                                                                   # d17587 - wjr
  if (( 0 == $RC )); then                                                              # d17587 - wjr
    prt_test_result passed $MSG_TXT                                                    # d17587 - wjr
  else                                                                                 # d17587 - wjr
    prt_test_result failed $MSG_TXT                                                    # d17587 - wjr
  fi                                                                                   # d17587 - wjr
  xprint debug "TRC: Exit function $FUNCTION"                                          # d17587 - wjr
  return $RC                                                                           # d17587 - wjr
}                                                                                      # d17587 - wjr
# =====================================================================================
function query_pools
{
  typeset FUNCTION='query_pools'
  xprint debug "TRC: Enter function $FUNCTION"
  trap 'handle_interrupt ask' SIGINT SIGTERM SIGTSTP SIGQUIT
  typeset MSG_TXT='Drive Pool State'
  typeset BAD_STATE_LIST='ONN|OFN|SN|ONF|OFF|SF|OFV|SV'
  typeset BAD_STATUS_LIST='Degraded|Degraded-InTransition|Unviable'                    # d18524 - wjr
  typeset POOL_ID=''                                                                   # d17587 - wjr
  typeset POOL_STATE=''                                                                # d17587 - wjr
  typeset POOL_STATUS=''                                                               # d17587 - wjr
  typeset POOL_INFO=''                                                                 # d17587 - wjr
  typeset RC=0
  xprint "MSG: Query CLI for drive pool state."
  verify_rc_state BOUND 'silent'
  if (( 0 == $? )); then
    CMD='echo "list pool" | /opt/arts/Cli/cli/linux_ppc/cli'
    run_cmd file
    if [[ -f $CMD_FILE ]]; then
      xprint "MSG: Query for drive pool state successful."
      prt_cli_session
      xprint "MSG: Checking pool status."
      RESULT=$(cat $CMD_FILE | \
               grep "No drive pools")
      if [[ -z $RESULT ]]; then
        RESULT=$(cat $CMD_FILE | \
                 grep '|' | \
                 grep -v "<CLI>" | \
                 grep -v "<cli>" | \
                 grep -v 'RaidType' | \
                 grep -v 'Current Machine' | \
                 grep -v '/' | \
                 cut -d '|' -f 2,9,10 | \
                 grep -v "_" | \
                 tr -d ' ')                                                            # d18324 - wjr
        for pool in $RESULT                                                            # d17587 - wjr
        do                                                                             # d17587 - wjr
          POOL_ID=$(echo $pool | cut -d '|' -f 1)                                      # d17587 - wjr
          POOL_STATUS=$(echo $pool | cut -d '|' -f 2)                                  # d17587 - wjr
          POOL_STATE=$(echo $pool | cut -d '|' -f 3)                                   # d17587 - wjr
          POOL_STATE_INFO=$(echo $POOL_STATE | grep -E $BAD_STATE_LIST)                # d18524 - wjr
          POOL_STATUS_INFO=$(echo $POOL_STATUS | grep -E $BAD_STATUS_LIST)             # d18524 - wjr
          if [[ -z $POOL_STATE_INFO && \
                -z $POOL_STATUS_INFO ]]; then                                          # d18524 - wjr
            xprint "MSG: Pool $POOL_ID is $POOL_STATUS."                               # d17587 - wjr
          else                                                                         # d17587 - wjr
            append_err_msg 19 'Slot '$POOL_ID'('$POOL_STATUS')'                        # d17587 - wjr
            RC=19
          fi
        done                                                                           # d17587 - wjr
        if [[ '' != ${ERR_MSG[19]} ]]; then                                            # d17587 - wjr
          set_rc 19 "Found one or more failed/non-viable pools: ${ERR_MSG[19]}."       # d17587 - wjr
        fi                                                                             # d17587 - wjr
      else
        xprint "MSG: No drive pools."
      fi
    else
      set_rc 20 "$SCRIPT query for drive pool state failed."                           # d17587 - wjr
      RC=20
    fi
    if (( 0 == $RC )); then                                                            # d17587 - wjr
      prt_test_result passed $MSG_TXT                                                  # d17587 - wjr
    else                                                                               # d17587 - wjr
      prt_test_result failed $MSG_TXT                                                  # d17587 - wjr
    fi                                                                                 # d17587 - wjr
  else
    xprint "MSG: Ctlrs are not BOUND - skipping drive pool state check."
  fi
  xprint debug "TRC: Exit function $FUNCTION"
  return $RC
}
# =====================================================================================
function verify_sas_expanders_active
{
  typeset FUNCTION='verify_sas_expanders_active'
  xprint debug "TRC: Enter function $FUNCTION"
  trap 'handle_interrupt ask' SIGINT SIGTERM SIGTSTP SIGQUIT
  typeset MSG_TXT='Both DSM SAS Expanders Active'
  typeset RC=0
  typeset ENCLOSURES='Undefined'                                                       # d17709 - wjr
  typeset STATE=''                                                                     # d17587 - wjr
  xprint "MSG: Query CLI for active enclosures."
  CMD='echo "list enclosure" | /opt/arts/Cli/cli/linux_ppc/cli'
  run_cmd file
  if [[ -f $CMD_FILE ]]; then
    xprint "MSG: Query for enclosures successful."
    prt_cli_session
    xprint "MSG: Checking enclosure state."
    ENCLOSURES=$(cat $CMD_FILE | \
                 grep '|' | \
                 grep 'DSM' | \
                 cut -d '|' -f 3)                                                      # d17709 - wjr
    if [[ -n $ENCLOSURES ]]; then                                                      # d17709 - wjr
      for enclosure in $ENCLOSURES                                                     # d17709 - wjr
      do
        STATE=$(cat $CMD_FILE | \
                grep '|' | \
                grep 'DSM' | \
                grep $enclosure | \
                cut -d '|' -f 6)                                                       # d17644 - wjr
        STATE=$(echo $STATE | tr -d ' ')
        if [[ -n $STATE ]]; then
          xprint "MSG: Enclosure $enclosure is active - checking state."
          if [[ 'Good' == $STATE ]]; then
            xprint "MSG: Enclosure $enclosure status is $STATE."
          else
            append_err_msg 29 $enclosure'('$STATE')'                                   # d17587 - wjr
            RC=29
          fi
        else
          append_err_msg 30 $enclosure                                                 # d17513 - wjr
          RC=30
        fi
      done
      if [[ '' != ${ERR_MSG[29]} ]]; then                                              # d17513 - wjr
        set_rc 29 "DSM SAS expander(s) ${ERR_MSG[29]} in an invalid state."            # d17513 - wjr
      fi                                                                               # d17513 - wjr
      if [[ '' != ${ERR_MSG[30]} ]]; then                                              # d17513 - wjr
        set_rc 30 "DSM SAS expander(s) ${ERR_MSG[30]} not active."                     # d17513 - wjr
      fi                                                                               # d17513 - wjr
    else                                                                               # d17709 - wjr
      xprint 'MSG: No enclosures found.'                                               # d17709 - wjr
    fi                                                                                 # d17709 - wjr
  else
    set_rc 31 "$SCRIPT query for DSM SAS expanders failed."                            # d17587 - wjr
    RC=31
  fi
  if (( 0 == $RC )); then
    prt_test_result passed $MSG_TXT
  else
    prt_test_result failed $MSG_TXT
  fi
  xprint debug "TRC: Exit function $FUNCTION"
  return $RC
}
# =====================================================================================
function verify_both_drive_ports_active
{
  typeset FUNCTION='verify_both_drive_ports_active'
  xprint debug "TRC: Enter function $FUNCTION"
  trap 'handle_interrupt ask' SIGINT SIGTERM SIGTSTP SIGQUIT
  typeset MSG_TXT='Each Disk Drive has Both Ports Active'
  typeset DRIVE_STATE=''                                                               # d17587 - wjr
  typeset DRIVE_INFO=''                                                                # d17587 - wjr
  typeset RC=0
  xprint "MSG: Query CLI for inactive drive ports."
  CMD='echo "list drive" | /opt/arts/Cli/cli/linux_ppc/cli'
  run_cmd file
  if [[ -f $CMD_FILE ]]; then
    xprint "MSG: Query for drive port status successful."
    prt_cli_session
    xprint "MSG: Checking drive port state."
    RESULT=$(cat $CMD_FILE | \
             grep '|' | \
             grep -v "<CLI>" | \
             grep -v "<cli>" | \
             grep -v "_" | \
             grep -v 'Drive' | \
             grep -v 'Current Machine' | \
             grep -v 'Usage:' | \
             grep -v 'foreign' | \
             grep -v 'current' | \
             grep -v 'State:' | \
             grep -v 'initialized' | \
             grep -v 'incompatible' | \
             grep -v 'firmware' | \
             grep -v 'Error Message File' | \
             grep -v 'No drives reported' | \
             cut -d '|' -f 3,8 | \
             tr -d ' ')                                                                # d17644 - wjr
    for drive in $RESULT                                                               # d17587 - wjr
    do                                                                                 # d17587 - wjr
      SLOT_ID=$(echo $drive | cut -d '|' -f 1)                                         # d17587 - wjr
      DRIVE_STATE=$(echo $drive | cut -d '|' -f 2)                                     # d17587 - wjr
      if [[ 'PM' != $DRIVE_STATE ]]; then                                              # d17587 - wjr
        xprint "MSG: All paths active for drive in slot $SLOT_ID."                     # d17587 - wjr
      else                                                                             # d17587 - wjr
        append_err_msg 27 'Slot '$SLOT_ID'(path missing)'                              # d17587 - wjr
        RC=27
      fi
    done                                                                               # d17587 - wjr
    if [[ '' != ${ERR_MSG[27]} ]]; then                                                # d17587 - wjr
      set_rc 27 "One or more inactive drive ports: ${ERR_MSG[27]}."                    # d17587 - wjr
    fi                                                                                 # d17587 - wjr
  else
    set_rc 28 "$SCRIPT query for drive port status failed."                            # d17587 - wjr
    RC=28
  fi
  if (( 0 == $RC )); then                                                              # d17587 - wjr
    prt_test_result passed $MSG_TXT                                                    # d17587 - wjr
  else                                                                                 # d17587 - wjr
    prt_test_result failed $MSG_TXT                                                    # d17587 - wjr
  fi                                                                                   # d17587 - wjr
  xprint debug "TRC: Exit function $FUNCTION"
  return $RC
}
# =====================================================================================
function verify_dsm_mount_state                                                        # d18038 - wjr
{                                                                                      # d18038 - wjr
  typeset FUNCTION='verify_dsm_mount_state'                                            # d18038 - wjr   
  xprint debug "TRC: Enter function $FUNCTION"                                         # d18038 - wjr
  trap 'handle_interrupt ask' SIGINT SIGTERM SIGTSTP SIGQUIT                           # d18038 - wjr
  typeset MSG_TXT='Each DSM Enclosure Online'                                          # d18038 - wjr
  typeset MOUNT_STATE=''                                                               # d18038 - wjr
  typeset MOUNT_INFO=''                                                                # d18038 - wjr
  typeset RC=0                                                                         # d18038 - wjr
  xprint "MSG: Query CLI for DSM mount state."                                         # d18038 - wjr
  CMD='echo "mountstate -getobject -enclosure" | /opt/arts/Cli/cli/linux_ppc/cli'      # d18038 - wjr
  run_cmd file                                                                         # d18038 - wjr
  if [[ -f $CMD_FILE ]]; then                                                          # d18038 - wjr
    xprint "MSG: Query for DSM mount state successful."                                # d18038 - wjr
    prt_cli_session                                                                    # d18038 - wjr
    xprint "MSG: Checking DSM mount state."                                            # d18038 - wjr
    RESULT=$(cat $CMD_FILE | \
             grep '|' | \
             grep -v "<CLI>" | \
             grep -v "<cli>" | \
             grep -v "_" | \
             grep -v 'mountstate' | \
             grep -v 'Current Machine' | \
             grep -v 'State' | \
             cut -d '|' -f 2,3,4 | \
             tr -d ' ')                                                                # d18038 - wjr
    for enclosure in $RESULT                                                           # d18038 - wjr
    do                                                                                 # d18038 - wjr
      ENCLOSURE_ID=$(echo $enclosure | cut -d '|' -f 1)                                # d18038 - wjr
      ENCLOSURE_STATE=$(echo $enclosure | cut -d '|' -f 2)                             # d18038 - wjr
      ENCLOSURE_TYPE=$(echo $enclosure | cut -d '|' -f 3)                              # d18038 - wjr
      ENCLOSURE_NAME='Enclosure '$ENCLOSURE_ID'('$ENCLOSURE_TYPE')'                    # d18038 - wjr
      if [[ 'Online' == $ENCLOSURE_STATE ]]; then                                      # d18038 - wjr
        xprint "MSG: $ENCLOSURE_NAME in online."                                       # d18038 - wjr
      else                                                                             # d18038 - wjr
        append_err_msg 41 $ENCLOSURE_NAME' - '$ENCLOSURE_STATE                         # d18038 - wjr
        RC=41                                                                          # d18038 - wjr
      fi                                                                               # d18038 - wjr
    done                                                                               # d18038 - wjr
    if [[ '' != ${ERR_MSG[41]} ]]; then                                                # d18038 - wjr
      set_rc 41 "One or more DSM enclosures not mounted: ${ERR_MSG[41]}."              # d18038 - wjr
    fi                                                                                 # d18038 - wjr
  else                                                                                 # d18038 - wjr
    set_rc 42 "$SCRIPT query for DSM mount state failed."                              # d18038 - wjr
    RC=42                                                                              # d18038 - wjr
  fi                                                                                   # d18038 - wjr
  if (( 0 == $RC )); then                                                              # d18038 - wjr
    prt_test_result passed $MSG_TXT                                                    # d18038 - wjr
  else                                                                                 # d18038 - wjr
    prt_test_result failed $MSG_TXT                                                    # d18038 - wjr
  fi                                                                                   # d18038 - wjr
  xprint debug "TRC: Exit function $FUNCTION"                                          # d18038 - wjr
  return $RC                                                                           # d18038 - wjr
}                                                                                      # d18038 - wjr
# =====================================================================================
function query_alerts
{
  typeset FUNCTION='query_alerts'
  xprint debug "TRC: Enter function $FUNCTION"
  trap 'handle_interrupt ask' SIGINT SIGTERM SIGTSTP SIGQUIT
  typeset MSG_TXT='Critical Maintenance Alerts'
  typeset ALERT_LIST='"1202|1203|1300|1303|4200|5700|9300|9303|9304|9306|9511"'        # d17654 - wjr
  typeset ALL_ALERTS=''                                                                # d17654 - wjr
  ALERT_LIST=$(echo $ALERT_LIST | tr -d ' ')
  typeset RC=0
  xprint "MSG: Check for critical alerts."
  CMD='echo "alert -get" | /opt/arts/Cli/cli/linux_ppc/cli'
  run_cmd file
  if [[ -f $CMD_FILE ]]; then
    xprint "MSG: Query for alerts successful."
    prt_cli_session
    xprint "MSG: Searching for critical alerts."
    ALL_ALERTS=$(cat $CMD_FILE | \
                 grep '|' | \
                 grep 'Critical' | \
                 grep -v "=" | \
                 grep -v '-' | \
                 grep -v '_' | \
                 grep -v 'Current Machine' | \
                 grep -v 'Existing Alerts' | \
                 grep -v "AlertCode" | \
                 grep -v "Msg" | \
                 grep -v "<CLI>" | \
                 grep -v 'Error Message File' | \
                 cut -d '|' -f 2 | \
                 grep -v -E $ALERT_LIST)                                               # d17654 - wjr
    if [[ -z $ALL_ALERTS ]]; then
      prt_test_result passed $MSG_TXT
      xprint "MSG: No critical alerts."
    else
      ALERTS=''
      for a in $ALL_ALERTS
      do
        if [[ -z $(echo $ALERTS | grep $a) ]]; then
          ALERTS=$(echo $ALERTS $a)
        fi
      done
      prt_test_result failed $MSG_TXT
      set_rc 21 "Found one or more critical alerts: $ALERTS"                           # d17513 - wjr
      RC=21
    fi
  else
    prt_test_result failed $MSG_TXT
    set_rc 22 "Query for critical alerts failed."                                      # d17513 - wjr
    RC=22
  fi
  xprint debug "TRC: Exit function $FUNCTION"
  return $RC
}
# =====================================================================================
function configure_alert_template
{
  typeset FUNCTION='configure_alert_template'
  xprint debug "TRC: Enter function $FUNCTION"
  trap 'handle_interrupt ask' SIGINT SIGTERM SIGTSTP SIGQUIT
  typeset RC=0
  typeset ALERT_CODE=$1
  shift
  typeset ALERT_MSG=$*
  ALERT_MSG='\"CCL preverify FAILED.  '$ALERT_MSG'\"'
  xprint "MSG: Configuring alert template for code $ALERT_CODE."
  CMD='echo "configure alert -setgenericalerttemplate -code '$ALERT_CODE' -type ackable -initiallymasked on -email off -severity warning -msg '$ALERT_MSG'" | /opt/arts/Cli/cli/linux_ppc/cli'
  run_cmd file
  if [[ -f $CMD_FILE ]]; then
    xprint "MSG: CLI command to configure alert template was issued."
    prt_cli_session
    xprint "MSG: Checking alert configure command response."
    RESULT=$(cat $CMD_FILE | \
             grep "successfully submitted")
    if [[ -n $RESULT ]]; then
      xprint "MSG: Alert $ALERT_CODE template successfully configured."
    else
      echo "ERROR: CLI command to configure alert failed."
      RC=23
    fi
  else
    echo "ERROR: CLI command to configure alert failed to execute."
    RC=24
  fi
  xprint debug "TRC: Exit function $FUNCTION"
  return $RC
}
# =====================================================================================
function generate_alert
{
  typeset FUNCTION='generate_alert'
  xprint debug "TRC: Enter function $FUNCTION"
  trap 'handle_interrupt ask' SIGINT SIGTERM SIGTSTP SIGQUIT
  typeset ALERT_CODE=$1
  typeset RC=0
  if [[ -z $ALERT_CODE ]]; then
    ALERT_CODE=10002
  fi
  xprint "MSG: Generating alert code $ALERT_CODE."
  CMD='echo "alert -create -code '$ALERT_CODE'" | /opt/arts/Cli/cli/linux_ppc/cli'
  run_cmd file
  if [[ -f $CMD_FILE ]]; then
    xprint "MSG: CLI command to generate alert was issued."
    prt_cli_session
    xprint "MSG: Checking alert command response."
    RESULT=$(cat $CMD_FILE | \
             grep "alert created successfully")
    if [[ -n $RESULT ]]; then
      xprint "MSG: Alert $ALERT_CODE successfully created."
    else
      echo "ERROR: CLI command to generate alert failed."
      RC=25
    fi
  else
    echo "ERROR: CLI command to generate alert failed to execute."
    RC=26
  fi
  xprint debug "TRC: Exit function $FUNCTION"
  return $RC
}
# =====================================================================================
function compare_fw_levels                                                             # d17429 - wjr
{                                                                                      # d17429 - wjr
  typeset FUNCTION='compare_fw_levels'                                                 # d17429 - wjr
  xprint debug "TRC: Enter function $FUNCTION"                                         # d17429 - wjr
  trap 'handle_interrupt ask' SIGINT SIGTERM SIGTSTP SIGQUIT                           # d17429 - wjr
  typeset RC=1                                                                         # d17429 - wjr
  typeset INST_LVL=$1                                                                  # d17429 - wjr
  typeset TARGET_LVL=$2                                                                # d17429 - wjr
  typeset TMP_FILE=$LOG_DIR'/sort.tmp'                                                 # d17429 - wjr
  xprint "MSG: Comparing installed level $INST_LVL to target level $TARGET_LVL."       # d17429 - wjr
  remove_file $TMP_FILE                                                                # d17429 - wjr
  echo $INST_LVL | sed -e 's/^0*//' | sed -e 's/0*$//' > $TMP_FILE                     # d17429 - wjr
  echo $TARGET_LVL | sed -e 's/^0*//' | sed -e 's/0*$//' >> $TMP_FILE                  # d17429 - wjr
  sort  -b -f -c $TMP_FILE 2>>$LOG                                                     # d17429 - wjr
  RC=$?                                                                                # d17429 - wjr
  if [[ 0 == $RC ]]; then                                                              # d17429 - wjr
    xprint "MSG: Installed level and target level are compatible."                     # d17429 - wjr
  else                                                                                 # d17429 - wjr
    xprint "MSG: Installed level and target level are INcompatible."                   # d17429 - wjr
  fi                                                                                   # d17429 - wjr
  remove_file $TMP_FILE                                                                # d17429 - wjr
  xprint debug "TRC: Exit function $FUNCTION"                                          # d17429 - wjr
  return $RC                                                                           # d17429 - wjr
}                                                                                      # d17429 - wjr
# =====================================================================================
function verify_fw_levels
{
  typeset FUNCTION='verify_fw_levels'
  xprint debug "TRC: Enter function $FUNCTION"
  trap 'handle_interrupt ask' SIGINT SIGTERM SIGTSTP SIGQUIT
  xprint "MSG: Verifying come-from and go-to firmware levels."                         # d17429 - wjr
  typeset MSG_TXT='Come-from & Go-to FW levels compatible'
  typeset RC=0                                                                         # d16723 - wjr
  typeset FW_COMPONENTS='Arts|Software|Tsalversion
                         SES|SES|SesVersion
                         CPLD|CPLD|CpldVersion
                         ppcFPGA|FPGA|FpgaVersion
                         BMC|BMC|BmcVersion
                         SAS|SAS_switch|SasVersion'                                    # d17475 - wjr
  typeset COMPONENT_ID='undefined'                                                     # d17429 - wjr
  typeset SWVERSION_ID='undefined'                                                     # d17429 - wjr
  typeset CFG_FILE='undefined'                                                         # d17475 - wjr
  typeset CFG_FILE_ID='undefined'                                                      # d17429 - wjr
  typeset INSTALLED_LVL='undefined'                                                    # d17429 - wjr
  typeset INST_COMP_LVL='undefined'                                                    # d17697 - wjr
  typeset MAX_LVL='undefined'                                                          # d17429 - wjr
  typeset VERIFY_FILE=$LOG_DIR'/verify.tmp'                                            # d17429 - wjr
  typeset CFG_FILE_TMP=$LOG_DIR'/CtlrFwMgmt.tmp'                                       # d17429 - wjr
  typeset LVL_MSG=''                                                                   # d17587 - wjr
  if [[ -n $CFG_STRING ]]; then                                                        # d17429 - wjr
    xprint "MSG: Using command line args for fw compatibility check."                  # d17475 - wjr
    remove_file $CFG_FILE_TMP                                                          # d17429 - wjr
    MAX_VERSIONS=$(echo $CFG_STRING | \
                   tr ':' ' ')                                                         # d17429 - wjr
    for m in $MAX_VERSIONS                                                             # d17429 - wjr
    do                                                                                 # d17429 - wjr
      echo $m | tr "^" " " >> $CFG_FILE_TMP                                            # d17429 - wjr
    done                                                                               # d17429 - wjr
    CFG_FILE=$CFG_FILE_TMP                                                             # d17429 - wjr
  elif [[ -f $ALT_CONFIG_FILE ]]; then                                                 # d17475 - wjr
    xprint "MSG: Using conf file in alternate flash for fw compatibility check."       # d17475 - wjr
    CFG_FILE=$ALT_CONFIG_FILE                                                          # d17475 - wjr
  elif [[ -f $CONFIG_FILE ]]; then                                                     # d17475 - wjr
    xprint "MSG: Using conf file in primary flash for fw compatibility check."         # d17475 - wjr
    CFG_FILE=$CONFIG_FILE                                                              # d17429 - wjr
  else                                                                                 # d17475 - wjr
    xprint "MSG: No config file - skipping fw compatibility check."                    # d17475 - wjr
    prt_test_result skipped $MSG_TXT                                                   # d17475 - wjr
  fi                                                                                   # d17429 - wjr
  if [[ 'undefined' != $CFG_FILE ]]; then                                              # d17475 - wjr
    if [[ -f $CFG_FILE ]]; then                                                        # d17429 - wjr
      CMD='echo swversion | /opt/arts/Cli/cli/linux_ppc/cli'                           # d17429 - wjr
      run_cmd file                                                                     # d17429 - wjr
      copy_file $CMD_FILE $VERIFY_FILE                                                 # d17429 - wjr
      prt_cli_session                                                                  # d17429 - wjr
      for component in $FW_COMPONENTS                                                  # d17429 - wjr
      do                                                                               # d17429 - wjr
        COMPONENT_ID=$(echo $component | \
                       cut -d '|' -f 1)                                                # d17429 - wjr
        SWVERSION_ID=$(echo $component | \
                       cut -d '|' -f 2 | \
                       tr '_' ' ')                                                     # d17429 - wjr
        CFG_FILE_ID=$(echo $component | \
                      cut -d '|' -f 3)                                                 # d17429 - wjr
        MIN_LVL=$(cat $CFG_FILE | \
                  grep '_min'$CFG_FILE_ID | \
                  cut -d '=' -f 2)                                                     # d17513 - wjr
        if [[ -n $MIN_LVL ]]; then                                                     # d17513 - wjr
          xprint "MSG: Min supported $COMPONENT_ID fw level: $MIN_LVL."                # d17513 - wjr
        else                                                                           # d17513 - wjr
          append_err_msg 33 $COMPONENT_ID                                              # d17513 - wjr
          RC=33                                                                        # d17513 - wjr
        fi                                                                             # d17513 - wjr
        MAX_LVL=$(cat $CFG_FILE | \
                  grep '_max'$CFG_FILE_ID | \
                  cut -d '=' -f 2)                                                     # d17429 - wjr
        if [[ -n $MAX_LVL ]]; then                                                     # d17429 - wjr
          xprint "MSG: Max supported $COMPONENT_ID fw level: $MAX_LVL."                # d17513 - wjr
        else                                                                           # d17429 - wjr
          append_err_msg 34 $COMPONENT_ID                                              # d17513 - wjr
          RC=34                                                                        # d17429 - wjr
        fi                                                                             # d17429 - wjr
        if [[ '0.0' != $MIN_LVL || \
              '0.0' != $MAX_LVL ]]; then                                               # d17513 - wjr
          INSTALLED_LVL=$(cat $VERIFY_FILE | \
                          grep "$SWVERSION_ID"' version' | \
                          cut -d ':' -f 2 | \
                          cut -d ' ' -f 1-3)                                           # d17475 - wjr
          if [[ -n $INSTALLED_LVL ]]; then                                             # d17429 - wjr
            xprint "MSG: Installed $COMPONENT_ID fw level: $INSTALLED_LVL."            # d17429 - wjr
          else                                                                         # d17429 - wjr
            append_err_msg 35 $COMPONENT_ID                                            # d17513 - wjr
            RC=35                                                                      # d17429 - wjr
          fi                                                                           # d17429 - wjr
        else                                                                           # d17513 - wjr
          xprint "MSG: Skipping $COMPONENT_ID min & max fw level check."               # d17513 - wjr
        fi                                                                             # d17513 - wjr
        INST_COMP_LVL=$(echo $INSTALLED_LVL | \
                        tr -d ' ')                                                     # d17697 - wjr
        if [[ '0.0' != $MIN_LVL && \
              'UNAVAILABLE' != $INST_COMP_LVL ]]; then                                 # d17697 - wjr
          compare_fw_levels "$MIN_LVL" "$INSTALLED_LVL"                                # d17513 - wjr
          RESULT=$?                                                                    # d17429 - wjr
          if [[ 0 != $RESULT ]]; then                                                  # d17429 - wjr
            LVL_MSG='Installed level:'$INSTALLED_LVL', Min supported level:'$MIN_LVL   # d17587 - wjr
            append_err_msg 36 $COMPONENT_ID'('$LVL_MSG')'                              # d17587 - wjr
            RC=36                                                                      # d17429 - wjr
          fi                                                                           # d17429 - wjr
        else                                                                           # d17513 - wjr
          xprint "MSG: Skipping $COMPONENT_ID min fw level check."                     # d17513 - wjr
        fi                                                                             # d17513 - wjr
        if [[ '0.0' != $MAX_LVL && \
              'UNAVAILABLE' != $INST_COMP_LVL ]]; then                                 # d17697 - wjr
          compare_fw_levels "$INSTALLED_LVL" "$MAX_LVL"                                # d17429 - wjr
          RESULT=$?                                                                    # d17429 - wjr
          if [[ 0 != $RESULT ]]; then                                                  # d17429 - wjr
            LVL_MSG='Installed level:'$INSTALLED_LVL', Max supported level:'$MAX_LVL   # d17587 - wjr
            append_err_msg 37 $COMPONENT_ID'('$LVL_MSG')'                              # d17587 - wjr
            RC=37                                                                      # d17429 - wjr
          fi                                                                           # d17429 - wjr
        else                                                                           # d17513 - wjr
          xprint "MSG: Skipping $COMPONENT_ID max fw level check."                     # d17513 - wjr
        fi                                                                             # d17513 - wjr
      done                                                                             # d17429 - wjr
      if [[ '' != ${ERR_MSG[33]} ]]; then                                              # d17513 - wjr
        set_rc 33 "$SCRIPT query for Rc ${ERR_MSG[33]} min firmware level failed."     # d17587 - wjr
      fi                                                                               # d17513 - wjr
      if [[ '' != ${ERR_MSG[34]} ]]; then                                              # d17513 - wjr
        set_rc 34 "$SCRIPT query for Rc ${ERR_MSG[34]} max firmware level failed."     # d17587 - wjr
      fi                                                                               # d17513 - wjr
      if [[ '' != ${ERR_MSG[35]} ]]; then                                              # d17513 - wjr
        set_rc 35 "$SCRIPT query for installed Rc ${ERR_MSG[35]} firmware level failed." # d17587 - wjr
      fi                                                                               # d17513 - wjr
      if [[ '' != ${ERR_MSG[36]} ]]; then                                              # d17513 - wjr
        set_rc 36 "${ERR_MSG[36]} come-from firmware level is incompatible."           # d17587 - wjr
      fi                                                                               # d17513 - wjr
      if [[ '' != ${ERR_MSG[37]} ]]; then                                              # d17513 - wjr
        set_rc 37 "${ERR_MSG[37]} go-to firmware level is incompatible."               # d17587 - wjr
      fi                                                                               # d17513 - wjr
    else                                                                               # d17429 - wjr
      set_rc 32 "$SCRIPT could not find the $CFG_FILE config file."                    # d17587 - wjr
      RC=32                                                                            # d17429 - wjr
    fi                                                                                 # d17429 - wjr
    remove_file $CFG_FILE_TMP                                                          # d17429 - wjr
    remove_file $VERIFY_FILE                                                           # d17429 - wjr
    if (( 0 == $RC )); then                                                            # d17429 - wjr
      prt_test_result passed $MSG_TXT                                                  # d17429 - wjr
    else                                                                               # d17429 - wjr
      prt_test_result failed $MSG_TXT                                                  # d17429 - wjr
    fi                                                                                 # d17429 - wjr
  fi                                                                                   # d17429 - wjr
  xprint debug "TRC: Exit function $FUNCTION"
  return $RC
}
# =====================================================================================
function ccl_check
{
  typeset FUNCTION='ccl_check'
  xprint debug "TRC: Enter function $FUNCTION"
  trap 'handle_interrupt ask' SIGINT SIGTERM SIGTSTP SIGQUIT
  typeset -i RC
  CHECK_TYPE='check'
  prt_test_result header 'CCL'
  query_tsal
  RC=$?
  if (( 0 == $RC )); then
    query_cli
    query_address
    query_ctlr_id
    verify_rc_state BOUND 'verbose'                                                   # d17429 - wjr
    query_rc_fw_status
    query_ddm_fw_status
    query_bad_drives
    query_unsupported_drive_pns                                                       # d17587 - wjr
#Tny    query_unsupported_drives
    query_pools
    verify_sas_expanders_active
    verify_dsm_mount_state                                                            # d18038 - wjr
    verify_both_drive_ports_active
    query_alerts
    verify_fw_levels
  fi
  prt_test_result space
  MSG_TXT='Overall Healthcheck Result'
  if (( 0 == $RETURN_CODE )); then
    prt_test_result passed $MSG_TXT
  else
    prt_test_result failed $MSG_TXT
  fi
  prt_test_result trailer
  remove_file $CMD_FILE
  xprint debug "TRC: Exit function $FUNCTION"
}
# ======================================= MAIN ========================================
#
# Set Options / Globals
#
typeset VERSION='%I%'
typeset OPERATION='ccl_check'
typeset VERBOSE='n'
typeset VERBOSE_DEBUG='n'
typeset VERBOSE_CLI='y'
typeset SUMMARY='n'
typeset COMMAND=$0
typeset ARGS=$*
typeset SCRIPT=${COMMAND##/*/}
typeset FUNCTION='main'
typeset -i RETURN_CODE=0
typeset MODE='interactive'
typeset CFG_STRING=''                                                                  # d17429 - wjr
typeset CFG_FILE_NAME='package.conf'                                                   # d17475 - wjr
typeset CFG_DIR='/opt/Config0/shellScripts'                                            # d17475 - wjr
typeset ALT_CFG_FILE_NAME='CtlrFwMgmt.conf'                                            # d17475 - wjr
typeset ALT_CFG_DIR='/mnt/opt/arts/tsal/linux_ppc'                                     # d17475 - wjr
typeset CONFIG_FILE=$CFG_DIR'/'$CFG_FILE_NAME                                          # d17475 - wjr
typeset ALT_CONFIG_FILE=$ALT_CFG_DIR'/'$ALT_CFG_FILE_NAME                              # d17475 - wjr
typeset DRIVE_FILE_NAME='DriveAttributes.ini'                                          # d17513 - wjr
typeset DRIVE_FILE=$CFG_DIR'/'$DRIVE_FILE_NAME                                         # d17513 - wjr
typeset ALT_DRIVE_FILE=$ALT_CFG_DIR'/'$DRIVE_FILE_NAME                                 # d17513 - wjr
typeset CMD_FILE_NAME=$SCRIPT'.tmp'
typeset LOG_NAME=$SCRIPT'.log'
typeset LOG_DIR='/opt/Logs/shellScriptsLogs'
typeset SUMMARY_LOG_NAME=$SCRIPT'.summary.log'
typeset SUMMARY_LOG=$LOG_DIR'/'$SUMMARY_LOG_NAME
typeset LOG=$LOG_DIR'/'$LOG_NAME
typeset CMD_FILE=$LOG_DIR'/'$CMD_FILE_NAME
typeset CMD_EXIT='restart'
typeset TIMESTAMP='y'
typeset CLEAR_LOG='y'
typeset RESULT=''
typeset IP_ADDRESS='undefined'
typeset CTLR_ID='undefined'
typeset -i N_OPERATIONS=0
typeset -a ERR_MSG                                                                     # d17513 - wjr
#
while getopts ":hHvsc:NC" opt;                                                         # d17429 - wjr
do
  case $opt in
    h | H )
      show_help
      exit 0
      ;;
    v )
      VERBOSE='y'
      ;;
    s )
      SUMMARY='y'
      ;;
    c )                                                                                # d17429 - wjr
      CFG_STRING=$optarg                                                               # d17429 - wjr
      ;;                                                                               # d17429 - wjr
    C )
      let N_OPERATIONS=n_operations+1
      OPERATION='ccl_check'
      ;;
    ? )
      show_syntax
      exit 0
      ;;
  esac
done
make_log_dir
log open
check_arguments
case $OPERATION in
  ccl_check )
    ccl_check
    ;;
  * )
    echo 'WARNING : No action specified.'
    ;;
esac
append_file $SUMMARY_LOG $LOG                                                          # d17429 - wjr
log close
exit $RETURN_CODE
# =====================================================================================
