3
\$\begingroup\$

I have below bash script which i have written to check the health of different storage component and then creates an incident through service-now tool.

This is script is working fine for me but i think this can be improved as i written this while learning through different google sources.

I would appreciate any help on improvisation of the script.

#!/bin/bash
DATE=$(date +"%m_%d_%Y_%H_%M")
clusters=(udcl101 udcl2011 udcl3011 udcl4011)
PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin:$PATH

incident=$1
sysid=UNKNOWN

checkvaultenv=$(echo $vaultenv | tr '[:upper:]' '[:lower:]')
if [[  "$checkvaultenv" == "" ]]
then
  vaultenv=prod
else
  if [[  "$checkvaultenv" != "dev" &&  "$checkvaultenv" != "test" && "$checkvaultenv" != "acc" && "$checkvaultenv" != "prod" ]]
  then
    echo unknown vault environment specified: $vaultenv
    exit 1
  fi
fi
vaultenvlc=$(echo $vaultenv | tr '[:upper:]' '[:lower:]')
vaultenvuc=$(echo $vaultenv | tr '[:lower:]' '[:upper:]')


SNOW_USERNAME=$(export EDITOR=cat ; ansible-vault view --vault-password-file=~/.ssh/sss.str.${vaultenvlc} ~/iss/vaults/vault-${vaultenvuc}.yml | awk '/vault_servicenow_username/{gsub(/"/, ""); print $2}')
SNOW_PASSWORD=$(export EDITOR=cat ; ansible-vault view --vault-password-file=~/.ssh/sss.str.${vaultenvlc} ~/iss/vaults/vault-${vaultenvuc}.yml | awk '/vault_servicenow_password/{gsub(/"/, ""); print $2}')


case $vaultenvlc in
  ram)
    SNOW_INSTANCE=udc2dev.service-now.com
    ;;
  iqa)
    SNOW_INSTANCE=udc2dev.service-now.com
    ;;
  dev)
    SNOW_INSTANCE=udc2dev.service-now.com
    ;;
  test)
    SNOW_INSTANCE=udc2trn.service-now.com
    ;;
  acc)
    SNOW_INSTANCE=udc2qa.service-now.com
    ;;
  *)
    SNOW_INSTANCE=udc2.service-now.com
    ;;
esac

SNOW_AUTH=$(echo -n $SNOW_USERNAME:$SNOW_PASSWORD)

create_incident()
{
  url="https://${SNOW_INSTANCE}/api/now/table/incident"
  curloutput=$(curl -sslv1 -u $SNOW_AUTH -X POST -s -H "Accept:application/json" -H "Content-Type:application/json" $url -d "$variables" 2>/dev/null)
  RETURNCODE=$?
  if [ "$RETURNCODE" == "0" ]
  then
    incident_number=$(echo $curloutput |  python -c $'import sys, json\nprint json.load(sys.stdin)["result"]["number"]')
    sysid=$(echo $curloutput |  python -c $'import sys, json\nprint json.load(sys.stdin)["result"]["sys_id"]')
    echo "OK: created incident $incident_number with sysid $sysid"
  else
    echo "ERROR creating incident:"
    echo "======================================="
    echo $curloutput
    echo "======================================="
  fi
}


# sets the variables for the incident
set_variables()
{
read -r -d '' variables <<- EOM
{
  "short_description": "$snshort_description",
  "assignment_group":  "RD-DI-Infra-Storage",
  "contact_type":      "interface",
  "state":             "New",
  "urgency":           "3 - Low",
  "impact":            "3 - Low",
  "cmdb_ci":           "$udcl",
  "u_sec_env":         "Normal Secure",
  "description":       $body
}
EOM
}


# checks if file is empty *or* has only "", if not create incident
chk_body()
{
  if [[ $body == "" || $body == '""' ]]
   then
    echo empty body
   else
    set_variables
    echo $variables
    echo ""
    create_incident
  fi
}

# actual storage health checks

for udcl in "${clusters[@]}";
do

  body=$(ssh admin@$udcl aggr show -root false | awk '/^udc/ && $4 >= 92 {sub(/\r$/, ""); s = (s == "" ? "" : s "\\n") $0} END {printf "\"%s\"\n", s}')
  snshort_description="aggr utilization above limit"
  chk_body

  body=$(ssh admin@$udcl "ro 0;snapshot show -create-time <21d" |awk '/^stv/ {sub(/\r$/, ""); s = (s == "" ? "" : s "\\n") $0} END {printf "\"%s\"\n", s}')
  snshort_description="snapshots older then 21 days"
  chk_body

  body=$(ssh admin@$udcl "ro 1;event log show -time <8h"| awk '/netinet.ethr.duplct.ipAdr|secd.ldap.noServers|object.store.unavailable/ {sub(/\r$/, ""); s = (s == "" ? "" : s "\\n") $0} END {printf "\"%s\"\n", s}')
  snshort_description="eventlog netinet.ethr.duplct.ipAdr, secd.ldap.noServers, object.store.unavailable messages"
  chk_body

  body=$(ssh admin@$udcl "ro 1;event log show -severity EMERGENCY -time <8h"| awk '/udc/ && !/netinet.ethr.duplct.ipAdr|secd.ldap.noServers|object.store.unavailable/ {sub(/\r$/, ""); s = (s == "" ? "" : s "\\n") $0} END {printf "\"%s\"\n", s}')
  snshort_description="EMERGENCY messages"
  chk_body

  body=$(ssh admin@$udcl "ro 0;system healt alert show"| awk '/Node|Severity|Proba/ {sub(/\r$/, ""); s = (s == "" ? "" : s "\\n") $0} END {printf "\"%s\"\n", s}')
  snshort_description="system healt alert show messages"
  chk_body

  body=$(ssh admin@$udcl "ro 0;vol show -volume *esx* -percent-used >=85" | awk '/stv/ && !/dr/ {sub(/\r$/, ""); s = (s == "" ? "" : s "\\n") $0} END {printf "\"%s\"\n", s}')
  snshort_description="ESX volumes utilization more then 85"
  chk_body

  body=$(ssh admin@$udcl "ro 0;vol show -state offline -fields aggregate" | awk '/stv/ {sub(/\r$/, ""); s = (s == "" ? "" : s "\\n") $0} END {printf "\"%s\"\n", s}')
  snshort_description="Validate offline volumes"
  chk_body

done
\$\endgroup\$
1
  • \$\begingroup\$ You can double quote all the variables except consonants as you are using bash and can use array and then call into the for loop that's more elegant thank using $(echo $clusters) . \$\endgroup\$
    – Karn Kumar
    Apr 6 at 16:38
1
+100
\$\begingroup\$

Changes:

  • Quote variables unless you have a good reason not to quote them.
    How about a password * $(touch /tmp/sorry) * ?
  • Use ${parameter:-word} for turning an empty vaultenv into prod.
    If parameter is unset or null, the expansion of word is substituted.
    Otherwise, the value of parameter is substituted.
  • Translate uppercase to lowercase only once. Use ${parameter,,} for this.
  • Combine tests on vaultenv into one block.
    Your code will exit for ram and iqa, I accept these. Change when you want to exit.
  • Changed variable names to lowercase
    UPPERCASE is reserved for system vars like PATH.
  • Removed unused DATE.
  • I combined the two ansible-vault calls into one and assign to SHOW_AUTH without echo.
  • Removed assignment to RETURNCODE

The resulting script:

#!/bin/bash
clusters=(udcl101 udcl2011 udcl3011 udcl4011)
PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin:$PATH

incident="$1" # Unused ???????
sysid=UNKNOWN

vaultenv="${vaultenv:-prod}"
vaultenvlc="${vaultenv,,}"
vaultenvuc="${vaultenv^^}"

case "${vaultenvlc}" in
  ram)
    # or do you want to exit here?
    snow_instance=udc2dev.service-now.com
    ;;
  iqa)
    # or do you want to exit here?
    snow_instance=udc2dev.service-now.com
    ;;
  dev)
    snow_instance=udc2dev.service-now.com
    ;;
  test)
    snow_instance=udc2trn.service-now.com
    ;;
  acc)
    snow_instance=udc2qa.service-now.com
    ;;
  prod)
    snow_instance=udc2.service-now.com
    ;;
  *)
    echo "unknown vault environment specified: ${vaultenv}"
    exit 1
esac

snow_auth=$(export EDITOR=cat;
             ansible-vault view --vault-password-file=~/.ssh/"sss.str.${vaultenvlc}" ~/iss/vaults/"vault-${vaultenvuc}.yml" |
             awk '
               /vault_servicenow_username/ {gsub(/"/, ""); usr=$2}
               /vault_servicenow_password/ {gsub(/"/, ""); pw=$2}
               END {printf("%s:%s", usr, pw)}
             ')

create_incident()
{
  url="https://${snow_instance}/api/now/table/incident"
  if curloutput=$(curl -sslv1 -u "${snow_auth}" -X POST -s -H "Accept:application/json" -H "Content-Type:application/json" "${url}" -d "${variables}" 2>/dev/null); then
    incident_number=$(echo "${curloutput}" |  python -c $'import sys, json\nprint json.load(sys.stdin)["result"]["number"]')
    sysid=$(echo "${curloutput}" |  python -c $'import sys, json\nprint json.load(sys.stdin)["result"]["sys_id"]')
    echo "OK: created incident ${incident_number} with sysid ${sysid}"
  else
    echo "ERROR creating incident:"
    echo "======================================="
    echo "${curloutput}"
    echo "======================================="
  fi
}


# sets the variables for the incident
set_variables()
{
read -r -d '' variables <<- EOM
{
  "short_description": "${snshort_description}",
  "assignment_group":  "RD-DI-Infra-Storage",
  "contact_type":      "interface",
  "state":             "New",
  "urgency":           "3 - Low",
  "impact":            "3 - Low",
  "cmdb_ci":           "${udcl}",
  "u_sec_env":         "Normal Secure",
  "description":       ${body}
}
EOM
}


# checks if file is empty *or* has only "", if not create incident
chk_body()
{
  if [[ "${body}" == "" || "${body}" == '""' ]]; then
    echo empty body
   else
    set_variables
    echo "${variables}"
    echo ""
    create_incident
  fi
}

# actual storage health checks

for udcl in "${clusters[@]}";
do

  body=$(ssh admin@"${udcl}" aggr show -root false |
     awk '/^udc/ && $4 >= 92 {sub(/\r$/, ""); s = (s == "" ? "" : s "\\n") $0}
          END {printf "\"%s\"\n", s}')
  snshort_description="aggr utilization above limit"
  chk_body

  body=$(ssh admin@"${udcl}" "ro 0;snapshot show -create-time <21d" |
     awk '/^stv/ {sub(/\r$/, ""); s = (s == "" ? "" : s "\\n") $0}
          END {printf "\"%s\"\n", s}')
  snshort_description="snapshots older then 21 days"
  chk_body

  body=$(ssh admin@"${udcl}" "ro 1;event log show -time <8h"|
     awk '/netinet.ethr.duplct.ipAdr|secd.ldap.noServers|object.store.unavailable/ {sub(/\r$/, ""); s = (s == "" ? "" : s "\\n") $0} 
          END {printf "\"%s\"\n", s}')
  snshort_description="eventlog netinet.ethr.duplct.ipAdr, secd.ldap.noServers, object.store.unavailable messages"
  chk_body

  body=$(ssh admin@"${udcl}" "ro 1;event log show -severity EMERGENCY -time <8h"|
     awk '/udc/ && !/netinet.ethr.duplct.ipAdr|secd.ldap.noServers|object.store.unavailable/ {sub(/\r$/, ""); s = (s == "" ? "" : s "\\n") $0}
          END {printf "\"%s\"\n", s}')
  snshort_description="EMERGENCY messages"
  chk_body

  body=$(ssh admin@"${udcl}" "ro 0;system healt alert show"|
     awk '/Node|Severity|Proba/ {sub(/\r$/, ""); s = (s == "" ? "" : s "\\n") $0}
          END {printf "\"%s\"\n", s}')
  snshort_description="system healt alert show messages"
  chk_body

  body=$(ssh admin@"${udcl}" "ro 0;vol show -volume *esx* -percent-used >=85" |
     awk '/stv/ && !/dr/ {sub(/\r$/, ""); s = (s == "" ? "" : s "\\n") $0}
          END {printf "\"%s\"\n", s}')
  snshort_description="ESX volumes utilization more then 85"
  chk_body

  body=$(ssh admin@"${udcl}" "ro 0;vol show -state offline -fields aggregate" |
     awk '/stv/ {sub(/\r$/, ""); s = (s == "" ? "" : s "\\n") $0}
          END {printf "\"%s\"\n", s}')
  snshort_description="Validate offline volumes"
  chk_body

done

I am not sure about introducing a new function for all your ssh calls:

for udcl in "${clusters[@]}";
do

  body=$(check "${udcl}" aggr)
  chk_body

  body=$(check "${udcl}" snapshot)
  chk_body

  body=$(check "${udcl}" showtime)
  chk_body

  ...
}

check() {
   case "$2" in
   "aggr") 
      body=$(ssh admin@"${udcl}" aggr show -root false |
         awk '
           /^udc/ && $4 >= 92 {sub(/\r$/, "");
           s = (s == "" ? "" : s "\\n") $0}
           END    {printf "\"%s\"\n", s}
         ')
     snshort_description="aggr utilization above limit"
     ;;
    ...
   esac
}
\$\endgroup\$
1
  • 1
    \$\begingroup\$ Thank you Walter 👏 for the detailed answer, +1 for the same and i am testing the same, will accept this after testing. \$\endgroup\$
    – kulfi
    Apr 12 at 8:59

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Not the answer you're looking for? Browse other questions tagged or ask your own question.