check_hadr_status

#! /usr/bin/env bash

###############################################################################
# Monitor DB2 with Nagios
# Copyright 2013 Andres Gomez Casanova
# https://github.com/angoca/monitor-db2-with-nagios
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
###############################################################################

# Checks the HADR status by looking if both databases are connected.
#
# Here comes the documentation
#
# # 'check_hadr_status' command definition
# define command {
#         command_name    check_hadr_status
#         command_line    $USER1$/check_by_ssh -H $HOSTADDRESS$ -l nagios -t 20 -C "scripts/check_hadr_status -i '$ARG1$' -d '$ARG2$'"
# }
# define service{
#         host_name               db2server
#         service_description     HADR status
#         check_command           check_hadr_status!/home/db2inst1!sample
#         use                     generic-service
# }
#
# When using NRPE:
#
# In the database server:
#
# command[check_hadr_status]=/home/nagios/scripts/check_hadr_status -i /home/db2inst1 -d sample
#
# Then modify the path according to your script location, instance home dir,
# database and thresholds.
#
# In the Nagios server:
#
# define service{
#         host_name               db2server
#         service_description     HADR status
#         check_command           check_nrpe_1arg!hadr_status
#         use                     generic-service
# }

# Parameters:
# -d | --database : Database name.
# -h | --help : Show the help of this script.
# -i | --instance : Home directory of the instance. Usually it is
#   /home/db2inst1.
# -K | --mk : Change the output for Check_MK.
# -T | --trace : Trace mode. Writes output in /tmp.
# -v | --verbose : Execute the program in verbose mode.
# -V | --version : Show the current version of this script.
#
# The exit codes are the standard for Nagios.
#
# 0 The plugin was able to check the service and it appeared to be functioning
#   properly.
# 1 The plugin was able to check the service, but it appeared to be above some
#   "warning" threshold or did not appear to be working properly.
# 2 The plugin detected that either the service was not running or it was above
#   some "critical" threshold.
# 3 Invalid command line arguments were supplied to the plugin or low-level
#   failures internal to the plugin (such as unable to fork, or open a tcp
#   socket) that prevent it from performing the specified operation.
#   Higher-level errors (such as name resolution errors, socket timeouts, etc)
#   are outside of the control of plugins and should generally NOT be reported
#   as UNKNOWN states.
#
# Author: Andres Gomez Casanova
# Version: v1.1 2013-05-26

# Flag for debugging.
#set -xv

# Locale to print messages in English. Prevent language problems.
export LANG=en_US

# Version of this script.
function print_revision {
  echo Andres Gomez Casanova - AngocA
  echo v1.1 2013-05-26
}
# Function to show the help
function print_usage {
  /bin/cat <<__EOT
Usage: ${1} { -i instanceHomeDirectory -d databaseName [-K] | -h | -V }
[-T][-v]
__EOT
}

function print_help {
  print_revision
  print_usage ${1}
  # Max 80 chars width.
  /bin/cat <<__EOT
-------------------------------------------------------------------------------
Checks the HADR status of the database.
-d | --database STRING
   Database name.
-h | --help
   Shows the current documentation.
-i | --instance STRING
   Instance home directory. It is usually /home/db2inst1.
-K | --mk
   Changes the output to be compatible with Check_MK.
-T | --trace
   Trace mode: writes date and output in /tmp.
-v | --verbose
   Executes the script in verbose mode (multiple times).
-V | --version
   Shows the current version of this script.
__EOT
}

# Variable to control the flow execution. Prevent Spaghetti code.
CONTINUE=true

# Nagios return codes
OK=0
WARNING=1
CRITICAL=2
UNKNOWN=3
# This is the returned code.
RETURN=${UNKNOWN}

# Nagios Output
# Text output 80 chars | Optional Perf Data Line 1
# Long text Line 1
# Long text Line 2 | Optional Perf Data Line 2
# Optional Perf Data Line 3
OUTPUT=
PERFORMANCE=
LONG_OUTPUT=
LONG_PERFORMANCE=
PERF_MK="-"

APPL_NAME=$(basename ${0})

if [[ ${#} -eq 0 ]] ; then
  print_usage ${APPL_NAME}
  RETURN=${UNKNOWN}
  CONTINUE=false
fi

# The following requieres GNU getopt. See the following discusion.
# http://stackoverflow.com/questions/402377

TEMP=$(getopt -o d:hi:KTvV --long database:,help,instance:,mk,trace,verbose,version \
             -n ${APPL_NAME} -- "${@}")

if [[ ${?} -ne 0 ]] ; then
  print_usage ${APPL_NAME}
  RETURN=${UNKNOWN}
  CONTINUE=false
fi

if [[ ${CONTINUE} == true ]] ; then
  # Note the quotes around ${TEMP}: they are essential!
  eval set -- "${TEMP}"

  HELP=false
  VERSION=false
  CHECK_MK=false
  # Verbosity level
  VERBOSE=0
  # Trace activated
  TRACE=false
  LOG=/tmp/${APPL_NAME}.log
  INSTANCE_HOME=
  DATABASE_NAME=
  while true; do
    case "${1}" in
      -d | --database ) DATABASE_NAME=$(echo ${2} | cut -d' ' -f1) ; shift 2 ;;
      -h | --help ) HELP=true ; shift ;;
      -i | --instance ) INSTANCE_HOME=$(echo ${2} | cut -d' ' -f1) ; shift 2 ;;
      -K | --mk ) CHECK_MK=true ; shift ;;
      -T | --trace ) TRACE=true ; shift ;;
      -v | --verbose ) VERBOSE=$(( ${VERBOSE} + 1 )) ; shift ;;
      -V | --version ) VERSION=true ; shift ;;
      -- ) shift; break ;;
      * ) break ;;
    esac
  done
fi

if [[ ${TRACE} == true ]] ; then
  echo ">>>>>" >> ${LOG}
  date >> ${LOG}
  echo "Instance at ${INSTANCE_HOME}" >> ${LOG}
  echo "PID ${$}" >> ${LOG}
fi

ECHO="help:${HELP}, version:${VERSION}, verbose:${VERBOSE}"
ECHO="${ECHO}, check_mk:${CHECK_MK}"
ECHO="${ECHO}, directory:${INSTANCE_HOME}, database:${DATABASE_NAME}"

if [[ ${VERBOSE} -ge 2 ]] ; then
  echo ${ECHO}
fi

if [[ ${TRACE} == true ]] ; then
  echo "PARAMS:${ECHO}" >> ${LOG}
fi

if [[ ${CONTINUE} == true && ${HELP} == true ]] ; then
  print_help ${APPL_NAME}
  RETURN=${UNKNOWN}
  CONTINUE=false
fi

if [[ ${CONTINUE} == true && ${VERSION} == true ]] ; then
  print_revision ${APPL_NAME}
  RETURN=${UNKNOWN}
  CONTINUE=false
fi

if [[ ${CONTINUE} == true && ${INSTANCE_HOME} == "" ]] ; then
  print_usage ${APPL_NAME}
  RETURN=${UNKNOWN}
  CONTINUE=false
fi

if [[ ${CONTINUE} == true && ${DATABASE_NAME} == "" ]] ; then
  print_usage ${APPL_NAME}
  RETURN=${UNKNOWN}
  CONTINUE=false
fi

if [[ ${CONTINUE} == true ]] ; then
  if [[ -d ${INSTANCE_HOME} && -e ${INSTANCE_HOME}/sqllib/db2profile ]] ; then
    # Load the DB2 profile.
    . ${INSTANCE_HOME}/sqllib/db2profile
    INSTANCE_NAME=$(db2 get instance | awk '/instance/ {print $7}')
  else
    OUTPUT="Instance directory is invalid."
    RETURN=${UNKNOWN}
    CONTINUE=false
  fi
fi

if [[ ${CONTINUE} == true ]] ; then
  COMMAND_DATABASE="db2 list db directory"
  if [[ ${VERBOSE} -ge 2 ]] ; then
    echo "COMMAND: ${COMMAND_DATABASE}"
  fi
  DATABASE=$(${COMMAND_DATABASE})
  if [[ ${TRACE} == true ]] ; then
    echo "RESULT:'${DATABASE}'" >> ${LOG}
  fi
  DATABASE=$(printf '%s\n' "${DATABASE}" | awk '/Database alias/ {print $4}' | grep -iw ${DATABASE_NAME})
  if [[ ${VERBOSE} -ge 3 ]] ; then
    echo "RESULT:'${DATABASE}'"
  fi

  if [[ ${DATABASE} == "" ]] ; then
    OUTPUT="The database ${DATABASE_NAME} is not cataloged."
    RETURN=${UNKNOWN}
    CONTINUE=false
  fi
fi

if [[ ${CONTINUE} == true ]] ; then
  COMMAND_VERSION="db2level"
  if [[ ${VERBOSE} -ge 2 ]] ; then
    echo "COMMAND: ${COMMAND_VERSION}"
  fi
  VERSION=$(${COMMAND_VERSION})
  if [[ ${TRACE} == true ]] ; then
    echo "RESULT:'${VERSION}'" >> ${LOG}
  fi
  VERSION=$(printf '%s\n' "${VERSION}" | awk '/Informational tokens/ {print $5}' | awk -F. '{print $1"."$2}')
  if [[ ${VERBOSE} -ge 3 ]] ; then
    echo "RESULT:'${VERSION}'"
  fi
  if [[ ( ${VERSION} != "v10.5" ) && ( ${VERSION} != "v10.1" ) && ( ${VERSION} != "v9.5" ) ]] ; then
    OUTPUT="The current DB2 version (${VERSION}) is not supported."
    RETURN=${UNKNOWN}
    CONTINUE=false
  fi
fi

if [[ ${CONTINUE} == true ]] ; then
  COMMAND_ROLE="db2 get db cfg for ${DATABASE_NAME}"
  if [[ ${VERBOSE} -ge 2 ]] ; then
    echo "COMMAND: ${COMMAND_ROLE}"
  fi
  ROLE=$(${COMMAND_ROLE})
  if [[ ${TRACE} == true ]] ; then
    echo "RESULT:'${ROLE}'" >> ${LOG}
  fi
  ROLE=$(printf '%s\n' "${ROLE}" | awk '/HADR database role/ {print $5}')
  if [[ ${VERBOSE} -ge 3 ]] ; then
    echo "RESULT:'${ROLE}'"
  fi
  if [[ ${ROLE} == "STANDARD" ]] ; then
    OUTPUT="Database is not in HADR"
    RETURN=${UNKNOWN}
  else
    COMMAND_HADR="db2pd -db ${DATABASE_NAME} -hadr"
    if [[ ${VERBOSE} -ge 2 ]] ; then
      echo "COMMAND: ${COMMAND_HADR}"
    fi
    OUTPUT_HADR=$(${COMMAND_HADR})
    if [[ ${TRACE} == true ]] ; then
      echo "RESULT:'${OUTPUT_HADR}'" >> ${LOG}
    fi
    if [[ ${VERSION} == "v9.7" ]] ; then
      CONNECTED=$(printf '%s\n' "${OUTPUT_HADR}" | awk '/onnected/ {print $1}' | tail -1)
      if [[ ${ROLE} == "PRIMARY" ]] ; then
        HADR_STATUS=$(printf '%s\n' "${OUTPUT_HADR}" | awk '/^Primary / {print $2}')
        if [[ ${VERBOSE} -ge 3 ]] ; then
          echo "RESULT:'${CONNECTED}'"
          echo "RESULT:'${HADR_STATUS}'"
        fi
        if [[ ${HADR_STATUS} == "Peer" ]] ; then
          OUTPUT="OK. Database is primary and peer"
          RETURN=${OK}
        elif [[ ${HADR_STATUS} == "Disconnected" ]] ; then
          OUTPUT="Standby database is not reachable (${CONNECTED})."
          RETURN=${CRITICAL}
        else
          OUTPUT="Database is primary and not peer. ${HADR_STATUS} ${CONNECTED}"
          RETURN=${UNKNOWN}
        fi
      elif [[ ${ROLE} == "STANDBY" ]] ; then
        HADR_STATUS=$(printf '%s\n' "${OUTPUT_HADR}" | awk '/^Standby/ {print $2}')
        if [[ ${VERBOSE} -ge 3 ]] ; then
          echo "RESULT:'${CONNECTED}'"
          echo "RESULT:'${HADR_STATUS}'"
        fi
        if [[ ${HADR_STATUS} == "Peer" ]] ; then
          OUTPUT="OK. Database is standby and peer"
          RETURN=${OK}
        elif [[ ${HADR_STATUS} == "DisconnectedPeer" ]] ; then
          OUTPUT="Pimary database is not reachable (${CONNECTED})."
          RETURN=${CRITICAL}
        elif [[ ${HADR_STATUS} == "RemoteCatchupPending" ]] ; then
          OUTPUT="StandBy in remote catchup pending. Make sure database primary is reachable (${CONNECTED})."
          RETRUN=${WARNING}
        else
          OUTPUT="Database is standby and not peer. ${HADR_STATUS} ${CONNECTED}"
          RETURN=${UNKNOWN}
        fi
        # Divided by 1000 (3E8)
        HEX_DIFF=$(printf '%s\n' "${OUTPUT_HADR}" | awk '/LOG/ {print $3}' | cut -dx -f2 | sed 'N;s/\n/-/' | awk '{print "ibase=16;("$1")/3E8"}' | bc)
        PERFORMANCE="HexDiff_K=${HEX_DIFF}" 
        PAGE_DIFF=$(printf '%s\n' "${OUTPUT_HADR}" | awk '/LOG/ {print $2}' | sed 'N;s/\n/-/' | bc)
        LOG_DIFF=$(printf '%s\n' "${OUTPUT_HADR}" | awk '/LOG/ {print $1}' | cut -d. -f1 | sed 's/^.//' | sed 'N;s/\n/-/' | bc)
        if [[ ${PAGE_DIFF} -lt 0 ]] ; then
          # The 2000 value is based on observation, but not sure.
          PAGE_DIFF=$((${PAGE_DIFF}+2000))
        fi
        LONG_PERFORMANCE="PageDiff=${PAGE_DIFF}\nLogDiff=${LOG_DIFF}" 
        PERF_MK="${PERFORMANCE}|${PAGE_DIFF}|${LOG_DIFF}"
      else
        OUTPUT="Database role is unknown (${ROLE})"
        RETURN=${UNKNOWN}
        CONTINUE=false
      fi
    elif [[ ( ${VERSION} == "v10.5" ) || ( ${VERSION} == "v10.1" ) ]] ; then
      CONNECTED=$(printf '%s\n' "${OUTPUT_HADR}" | awk '/HADR_CONNECT_STATUS / {print $3}')
      HADR_STATUS=$(printf '%s\n' "${OUTPUT_HADR}" | awk '/HADR_STATE/ {print $3}')
      if [[ ${VERBOSE} -ge 3 ]] ; then
        echo "RESULT:'${CONNECTED}'"
        echo "RESULT:'${HADR_STATUS}'"
      fi
      if [[ ${ROLE} == "PRIMARY" ]] ; then
        if [[ ${HADR_STATUS} == "PEER" ]] ; then
          OUTPUT="OK. Database is primary and peer"
          RETURN=${OK}
        elif [[ ${HADR_STATUS} == "DISCONNECTED" ]] ; then
          OUTPUT="Standby database is not reachable (${CONNECTED})."
          RETURN=${CRITICAL}
        else
          OUTPUT="Database is primary and not peer. ${HADR_STATUS} ${CONNECTED}"
          RETURN=${UNKNOWN}
        fi
      elif [[ ${ROLE} == "STANDBY" ]] ; then
        if [[ ${HADR_STATUS} == "PEER" ]] ; then
          OUTPUT="OK. Database is standby and peer"
          RETURN=${OK}
        elif [[ ${HADR_STATUS} == "DISCONNECTED" ]] ; then
          OUTPUT="Pimary database is not reachable (${CONNECTED})."
          RETURN=${CRITICAL}
        elif [[ ${HADR_STATUS} == "REMOTE_CATCHUP_PENDING" ]] ; then
          OUTPUT="StandBy in remote catchup pending. Make sure database primary is reachable (${CONNECTED})."
          RETRUN=${WARNING}
        else
          OUTPUT="Database is standby and not peer. ${HADR_STATUS} ${CONNECTED}"
          RETURN=${UNKNOWN}
        fi
        PAGE_PRI=$(printf '%s\n' "${OUTPUT_HADR}" | awk '/PRIMARY_LOG_FILE/ {print $4}' | awk -F, '{print $1}')
        PAGE_STA=$(printf '%s\n' "${OUTPUT_HADR}" | awk '/STANDBY_LOG_FILE/ {print $4}' | awk -F, '{print $1}')
        PAGE_DIFF=$((${PAGE_PRI}-${PAGE_STA}))
        LOG_PRI=$(printf '%s\n' "${OUTPUT_HADR}" | awk '/PRIMARY_LOG_FILE/ {print $r3}' | cut -c2-8)
        LOG_STA=$(printf '%s\n' "${OUTPUT_HADR}" | awk '/STANDBY_LOG_FILE/ {print $r3}' | cut -c2-8)
        LOG_DIFF=$((${LOG_PRI}-${LOG_STA}))
        if [[ ${PAGE_DIFF} -lt 0 ]] ; then
          # The 2000 value is based on observation, but not sure.
          PAGE_DIFF=$((${PAGE_DIFF}+2000))
        fi
        LONG_PERFORMANCE="PageDiff=${PAGE_DIFF}\nLogDiff=${LOG_DIFF}"
        PERF_MK="${PERFORMANCE}|${PAGE_DIFF}|${LOG_DIFF}"
      else
        OUTPUT="Database role is unknown (${ROLE})"
        RETURN=${UNKNOWN}
        CONTINUE=false
      fi
    fi
  fi
fi
# Prints the output.
if [[ ${OUTPUT} == "" ]] ; then
  OUTPUT="Note: The test was not executed."
fi
# Builds the output.
if [[ ${CHECK_MK} == true ]] ; then
  echo "${RETURN} hadrStatus-${INSTANCE_NAME}-${DATABASE_NAME} ${PERF_MK} ${OUTPUT}"
else
  echo -e "${OUTPUT}|${PERFORMANCE}\n${LONG_OUTPUT}|${LONG_PERFORMANCE}"
fi
# Returns the error code.
if [[ ${VERBOSE} -ge 2 ]] ; then
  echo "Return code: ${RETURN}"
fi
if [[ ${TRACE} == true ]] ; then
  echo -e "OUTPUT:${OUTPUT}\nPERF:${PERFORMANCE}\nLONG_OUT:${LONG_OUTPUT}\nLONGPERF:${LONG_PERFORMANCE}\nRET_CODE:${RETURN}" >> ${LOG}
  date >> ${LOG}
  echo -e "<<<<<\n" >> ${LOG}
fi
exit ${RETURN}