#!/bin/bash
#
#   Copyright (C) 2017 Rackspace, Inc.
#
#   This program is free software; you can redistribute it and/or modify
#   it under the terms of the GNU General Public License as published by
#   the Free Software Foundation; either version 2 of the License, or
#   (at your option) any later version.
#
#   This program is distributed in the hope that it will be useful,
#   but WITHOUT ANY WARRANTY; without even the implied warranty of
#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#   GNU General Public License for more details.
#
#   You should have received a copy of the GNU General Public License along
#   with this program; if not, write to the Free Software Foundation, Inc.,
#   51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
#
#~ Usage: _tool_ [OPTION]
#~ Options:
#~   -h, --help       Print this help.
#~   -B, --backup     Copy the last log file to the backups dir.
#~   -S, --snapshot   Take a timestamped snapshot outside the regular rotation.
#~   -V, --version    Print version and exit.
#~

## Version
declare -r _VERSION='2.0.2'

## Default settings(can *NOT* be overridden in config file)
declare -r DATE=$( date +%F_%T )
declare -r LOCKFILE="/var/lock/recap.lock"
declare -r LOG_SUFFIX=$( date +%Y%m%d-%H%M%S )
declare -r LIBDIR="/usr/local/lib/recap"
declare -r COREDIR="${LIBDIR}/core"
declare -r PLUGIN_A_DIR="${LIBDIR}/plugins-available"
declare -r PLUGIN_E_DIR="${LIBDIR}/plugins-enabled"
PATH=/bin:/usr/bin:/sbin:/usr/sbin


## Default settings(modified through command line arguments)
declare -r default_BACKUP="no"
declare -r default_SNAPSHOT="no"
BACKUP="${default_BACKUP}"
SNAPSHOT="${default_SNAPSHOT}"

## Default settings(can be overridden in config file)
declare -r default_BASEDIR="/var/log/recap"
declare -r default_RECAPLOG="${default_BASEDIR}/recap.log"
declare -r default_MAILTO=""
declare -r default_MIN_FREE_SPACE=0
declare -r default_USEFDISK="no"
declare -r default_USEPS="yes"
declare -r default_USEPSTREE="no"
declare -r default_USESLAB="no"
declare -r default_USEPLUGINS="no"
BASEDIR="${default_BASEDIR}"
RECAPLOG="${default_RECAPLOG}"
MAILTO="${default_MAILTO}"
MIN_FREE_SPACE=${default_MIN_FREE_SPACE}
USEFDISK="${default_USEFDISK}"
USEPS="${default_USEPS}"
USEPSTREE="${default_USEPSTREE}"
USESLAB="${default_USESLAB}"
USEPLUGINS="${default_USEPLUGINS}"

# Parent setting(other settings depend on this)
declare -r default_USERESOURCES="yes"
USERESOURCES="${default_USERESOURCES}"
# These depend on USERRESOURCES to be enabled("yes")
declare -r default_USEDF="yes"
declare -r default_USESAR="yes"
declare -r default_USESARQ="no"
declare -r default_USESARR="no"
USEDF="${default_USEDF}"
USESAR="${default_USESAR}"
USESARQ="${default_USESARQ}"
USESARR="${default_USESARR}"

# Parent setting(other settings depend on this)
declare -r default_USENETSTAT="yes"
USENETSTAT="${default_USENETSTAT}"
# These depend on USENETSTAT to be enabled("yes")
declare -r default_USENETSTATSUM="no"
USENETSTATSUM="${default_USENETSTATSUM}"

# Parent setting(other settings depend on this)
declare -r default_USEMYSQL="no"
USEMYSQL="${default_USEMYSQL}"
# These depend on USEMYSQL to be enabled("yes")
declare -r default_DOTMYDOTCNF="/root/.my.cnf"
declare -r default_MYSQL_PROCESS_LIST="table"
declare -r default_USEINNODB="no"
declare -r default_USEMYSQLPROCESSLIST="no"
DOTMYDOTCNF="${default_DOTMYDOTCNF}"
MYSQL_PROCESS_LIST="${default_MYSQL_PROCESS_LIST}"
USEINNODB="${default_USEINNODB}"
USEMYSQLPROCESSLIST="${default_USEMYSQLPROCESSLIST}"

# Default command options(can be overridden in config file)
declare -r default_OPTS_DF="-x nfs"
declare -r default_OPTS_FDISK="-l"
declare -r default_OPTS_FREE=""
declare -r default_OPTS_IOSTAT="-t -x 1 3"
declare -r default_OPTS_IOTOP="-b -o -t -n 3"
declare -r default_OPTS_NETSTAT="-atunp"
declare -r default_OPTS_NETSTAT_SUM="-a"
declare -r default_OPTS_PS="auxfww"
declare -r default_OPTS_PSTREE="-p"
declare -r default_OPTS_VMSTAT="-S M 1 3"
OPTS_DF="${default_OPTS_DF}"
OPTS_FDISK="${default_OPTS_FDISK}"
OPTS_FREE="${default_OPTS_FREE}"
OPTS_IOSTAT="${default_OPTS_IOSTAT}"
OPTS_IOTOP="${default_OPTS_IOTOP}"
OPTS_NETSTAT="${default_OPTS_NETSTAT}"
OPTS_NETSTAT_SUM="${default_OPTS_NETSTAT_SUM}"
OPTS_PS="${default_OPTS_PS}"
OPTS_PSTREE="${default_OPTS_PSTREE}"
OPTS_VMSTAT="${default_OPTS_VMSTAT}"

# Internal variables
banner_start="--- Starting $( basename $0 )[$$] ---"
banner_end="--- Ending $( basename $0 )[$$] ---"

# Functions

# Timestamps
ts() {
  TS_FLAGS='--rfc-3339=seconds'
  date "${TS_FLAGS}"
}

# Logging messages
log() {
  # does not work in a while-loop as spawns a new shell
  local msg_type=$1
  shift
  local log_entry="$*"
  ## This avoids sending any output to stdout when executed through cron
  ## is helpful to avoid emails submitted, instead the logs contain the
  ## possible ERRORS
  if ! tty -s; then
    echo "$( ts ) [${msg_type}] ${log_entry}" 2>&1 >> "${RECAPLOG}"
    return 0
  fi
  if [[ "${VERBOSE}" ]]; then
    echo "$( ts ) [${msg_type}] ${log_entry}" 2>&1 | tee -a "${RECAPLOG}"
    return 0
  fi
  if [[ "${msg_type}" =~ "ERROR" ||
        "${msg_type}" =~ "WARNING" ]]; then
    echo "$( ts ) [${msg_type}] ${log_entry}" 2>&1 | tee -a "${RECAPLOG}"
  else
    echo "$( ts ) [${msg_type}] ${log_entry}" 2>&1 >> "${RECAPLOG}"
  fi
}

# Usage
print_usage() {
  grep -E '^#~' $0 | sed -e 's/^#~\s\?//' \
                         -e "s/_tool_/$( basename $0 )/"
}

# Cleanup function to remove lock
cleanup() {
  log INFO "$( basename $0 )[$$]: Caught signal - deleting ${LOCKFILE}"
  rm -f "${LOCKFILE}"
  log INFO "Execution time: ${SECONDS}s"
  log INFO "${banner_end}"
}

# Create a Lock so that recap does not try to run over itself.
recaplock() {
  (set -C; echo "$$" > "${LOCKFILE}") 2>/dev/null
  if [[ $? -ne 0 ]]; then
    log ERROR "$( basename $0 )[$$]: Lock File exists - exiting"
    exit 1
  else
    trap 'exit 2' 1 2 15 23
    trap 'cleanup' EXIT
    log INFO "$( basename $0 )[$$]: Created lock file: ${LOCKFILE}"
  fi
}

# Ensure our output directories exist before we start creating files
create_output_dirs() {
  BACKUPDIR="${BASEDIR}/backups"
  SNAPSHOTSDIR="${BASEDIR}/snapshots"
  local -a OUTPUT_DIRS
  OUTPUT_DIRS+=( "${BASEDIR}" )
  OUTPUT_DIRS+=( "${BACKUPDIR}" )
  OUTPUT_DIRS+=( "${SNAPSHOTSDIR}" )
  for OUTPUT_DIR in ${OUTPUT_DIRS[@]}; do
    if [[ ! -d "${OUTPUT_DIR}" ]]; then
      mkdir -p "${OUTPUT_DIR}"
    fi
  done
  chmod 0750 "${BASEDIR}"
}

# Create output file
create_output_file() {
  local OUTPUT_FILE="$1"
  if [[ -d "${OUTPUT_FILE}" ]]; then
    log ERROR "Target file already exists: ${OUTPUT_FILE}"
    exit
  else
    #print the data to the output file
    echo "${DATE}" > "${OUTPUT_FILE}"
  fi
}

# Check to see if the output directory exists
check_output_file() {
  local OUTPUT_FILE="$1"
  if [[ ! -r "${OUTPUT_FILE}" ]]; then
    log ERROR "The output file does not exist: ${OUTPUT_FILE}"
    exit
  fi
}

# Print a blank line to the specified file
print_blankline() {
  local LOGFILE="$1"
  echo " " >> "${LOGFILE}"
}

# Plugin names
# Obtain the list of files/plugins on a directory
plugins_list() {
  local plugin_dir="$1"
  local p_names=( $( ls -1 "${plugin_dir}" | tr '\n' ' ') )
  echo ${p_names[@]}
}

# Plugin info
plugins_info(){
  for dir in "${PLUGIN_A_DIR}" "${PLUGIN_E_DIR}"; do
    log INFO "Finding plugins in ${dir}"
    plugins=( $( plugins_list "${dir}" ) )
    if [[ ${#plugins[@]} == 0 ]]; then
      log INFO "No plugins found."
    else
      log INFO "${#plugins[@]} plugins found: ${plugins[@]}"
    fi
  done
}

# Loads plugins
plugins_load() {
  local plugin_dir="$1"
  log INFO "Loading plugins from: ${plugin_dir}"
  while read plugin; do
    log INFO "Loading plugin: ${plugin}"
    source "${plugin_dir}/${plugin}"
  done < <(ls -1 "${plugin_dir}" 2>/dev/null)
}

# Load core functions
source ${COREDIR}/ps
source ${COREDIR}/fdisk
source ${COREDIR}/mysql
source ${COREDIR}/pstree
source ${COREDIR}/netstat
source ${COREDIR}/resources
source ${COREDIR}/send_mail

# Manage item report
run_item_report() {
  local item="$1"
  local ITEM_FILE="${BASEDIR}/${item}_${LOG_SUFFIX}.log"

  if [[ "${SNAPSHOT,,}" == "yes" ]]; then
    ITEM_FILE="${SNAPSHOTSDIR}/${item}_${LOG_SUFFIX}.log_snapshot"
  fi

  create_output_file "${ITEM_FILE}"
  check_output_file "${ITEM_FILE}"
  eval "print_${item}" "${ITEM_FILE}"
}

# Manage resources report
run_resources_report() {
  local item="resources"
  local ITEM_FILE="${BASEDIR}/${item}_${LOG_SUFFIX}.log"

  if [[ "${SNAPSHOT,,}" == "yes" ]]; then
    ITEM_FILE="${SNAPSHOTSDIR}/${item}_${LOG_SUFFIX}.log_snapshot"
  fi

  create_output_file "${ITEM_FILE}"
  check_output_file "${ITEM_FILE}"
  print_uptime "${ITEM_FILE}"
  print_blankline "${ITEM_FILE}"
  print_free "${ITEM_FILE}"
  print_blankline "${ITEM_FILE}"
  print_vmstat "${ITEM_FILE}"
  print_blankline "${ITEM_FILE}"
  print_iostat "${ITEM_FILE}"
  print_blankline "${ITEM_FILE}"
  print_iotop "${ITEM_FILE}"
  print_blankline "${ITEM_FILE}"

  # check to see if sar should be run
  if [[ "${USESAR,,}" == "yes" ]]; then
    print_blankline "${ITEM_FILE}"
    print_sar "${ITEM_FILE}"
  fi

  # check to see if sar -r should be run
  if [[ "${USESARR,,}" == "yes" ]]; then
    # send sar -r output to output file
    print_blankline "${ITEM_FILE}"
    print_sar "${ITEM_FILE}" "r"
  fi

  # check to see if sar -q should be run
  if [[ "${USESARQ,,}" == "yes" ]]; then
    # send sar -q output to output file
    print_blankline "${ITEM_FILE}"
    print_sar "${ITEM_FILE}" "q"
  fi

  if [[ "${USEDF,,}" == "yes" ]]; then
    # send df -h output to output file
    print_blankline "${ITEM_FILE}"
    print_df "${ITEM_FILE}"
  fi

  if [[ "${USESLAB,,}" == "yes" ]]; then
    # send slabinfo output to output file
    print_blankline "${ITEM_FILE}"
    print_slabinfo "${ITEM_FILE}"
  fi

  print_blankline "${ITEM_FILE}"
  print_top_10_cpu "${ITEM_FILE}"
  print_blankline "${ITEM_FILE}"
  print_top_10_mem "${ITEM_FILE}"
}

# Manage netstat report
run_netstat_report() {
  local item="netstat"
  local ITEM_FILE="${BASEDIR}/${item}_${LOG_SUFFIX}.log"

  if [[ "${SNAPSHOT,,}" == "yes" ]]; then
    ITEM_FILE="${SNAPSHOTSDIR}/${item}_${LOG_SUFFIX}.log_snapshot"
  fi

  create_output_file "${ITEM_FILE}"
  check_output_file "${ITEM_FILE}"
  print_netstat "${ITEM_FILE}"

  # check to see if optional netstat summary report should be run
  if [[ "${USENETSTATSUM,,}" == "yes" ]]; then
    print_blankline "${ITEM_FILE}"
    print_netstat_sum "${ITEM_FILE}"
  fi
}

# Manage mysql report
run_mysql_report() {
  local item="mysql"
  local ITEM_FILE="${BASEDIR}/${item}_${LOG_SUFFIX}.log"

  if [[ "${SNAPSHOT,,}" == "yes" ]]; then
    ITEM_FILE="${SNAPSHOTSDIR}/${item}_${LOG_SUFFIX}.log_snapshot"
  fi

  create_output_file "${ITEM_FILE}"
  check_output_file "${ITEM_FILE}"
  log INFO "Starting iteration of mysql report(s)"
  # Iterate through the list of DOTMYDOTCNF config files
  local -a MYCNFS=( ${DOTMYDOTCNF//,/ } )
  for MYCNF in ${MYCNFS[@]}; do
    # Don't run reports if can't read the config file
    if [[ ! -r "${MYCNF}" ]]; then
      log ERROR "Unable to read: '${MYCNF}' in the 'DOTMYDOTCNF' config."
      continue
    fi
    # Don't run reports if can't connect to the instance
    if ! mysqladmin \
           --defaults-file="${MYCNF}" \
           --connect-timeout=5 \
           ping &>/dev/null; then
      log ERROR "Unable to connect using '${MYCNF}' in the 'DOTMYDOTCNF'"\
                "config."
      continue
    fi

    print_mysql "${ITEM_FILE}" "${MYCNF}"

    # check to see if the optional mysql process list should be generated
    if [[ "${USEMYSQLPROCESSLIST,,}" == "yes" ]]; then
      print_blankline "${ITEM_FILE}"
      print_mysql_procs "${ITEM_FILE}" "${MYCNF}"
    fi
    if [[ "${USEINNODB,,}" == "yes" ]]; then
      # send df -h output to output file
      print_blankline "${ITEM_FILE}"
      print_mysql_innodb_status "${ITEM_FILE}" "${MYCNF}"
    fi
  done
  log INFO "Ended iteration of mysql report(s)"
}

# Generates a list of names from the current run, using LOG_SUFFIX
get_report_names_of_run() {
  local -a reports=()
  log INFO "Finding reports from this run: ${LOG_SUFFIX}"
  reports=( $( ls -1 "${BASEDIR}" 2>/dev/null |
                 awk -F_ '/'${LOG_SUFFIX}'\.log$/ {print $1}' |
                 sort -u
             )
          )
  log INFO "Reports found: [ ${reports[@]} ]"
  echo "${reports[@]}"
}

# Generates a list of names from the last reports logged
get_last_report_names() {
  local -a reports=()
  local last_suffix=''
  last_suffix=$( ls -1 "${BASEDIR}" 2>/dev/null |
                   grep -P '_\d{8}-\d{6}\.log$' |
                   grep -oP '\d{8}-\d{6}' |
                   sort -ur |
                   head -1 )
  if [[ -z ${last_suffix} ]]; then
    log ERROR "No previous run has been found, unable to find report names."
  else
    log INFO "Last run was on: ${last_suffix}"
    reports=(
      $( ls -1 "${BASEDIR}" 2>/dev/null |
           awk -F. '
             /'${last_suffix}'\.log$/ {sub("_'${last_suffix}'","",$1); print $1}
           '
       )
    )
    log INFO "Reports found: [ ${reports[@]} ]"
  fi
  echo "${reports[@]}"
}

# Copy the last log file set to ${BACKUPDIR}
create_backup() {
  log INFO "Starting backup of reports"
  local -a log_files=( $( get_last_report_names ) )
  if [[ -z ${log_files} ]]; then
    log ERROR "Unable to backup unexisting reports."
  else
    for log_file in ${log_files[@]}; do
      ls -1t ${BASEDIR}/${log_file}_*.log 2>/dev/null |
      head -1 |
      xargs -I {} cp {} ${BACKUPDIR}
    done
  fi
  log INFO "Ended backup of reports"
}

# Check enough disk space is free
check_disk_space() {
  log INFO "Starting check for disk space"
  if [[ ${MIN_FREE_SPACE} -eq 0 ]]; then
    log INFO "Ended check for disk space"
    return 0
  else
    FREE_SPACE=$( df -PBM "${BASEDIR}" | awk '!/-blocks/ {print $4}' )
    FREE_SPACE=${FREE_SPACE%M}
    if [[ "${MIN_FREE_SPACE}" -ge "${FREE_SPACE}" ]]; then
      log ERROR "Unable to run recap due to not enough disk space"
    fi
  fi
  log INFO "Ended check for disk space"
}

## Main

# Set options
OPTIONS=VBhS
LONG_OPTIONS=version,backup,help,snapshot

# Parse options and show usage if invalid options provided
! ALL_ARGS=$(getopt --options=${OPTIONS} --longoptions=${LONG_OPTIONS} --name "$0" -- "$@")
if [[ ${PIPESTATUS[0]} -ne 0 ]]; then
  print_usage
  exit 1
fi

# Set the positional parameters to the parsed options
set -- ${ALL_ARGS}

# Act on the valid arguments provided
while true; do
  case "$1" in
    -V|--version)
      echo "${_VERSION}"
      exit 0
      ;;
    -B|--backup)
      # backup latest log files
      BACKUP="yes"
      shift
      ;;
    -h|--help)
      # print usage
      print_usage
      exit 0
      ;;
    -S|--snapshot)
      # take a snapshot outside of the regular output rotation
      SNAPSHOT="yes"
      shift
      ;;
    --)
      shift; break ;;
    *)
      # user entered an invalid flag, print warning and exit
      log ERROR "Invalid Input"
      print_usage
      exit 1
  esac
done

if [[ "${BACKUP,,}" == "yes" && "${SNAPSHOT,,}" == "yes" ]]; then
  log ERROR "Backup and Snapshot options are mutually exclusive"
  print_usage
  exit 1
fi

# Error if unparsed arguments exist
if [[ "$#" -gt 0 ]]; then
  log ERROR "Error: Unexpected argument(s): $@"
  print_usage
  exit 1
fi

# Verify that script is being run as root
if [[ "$(id -u)" != "0" ]]; then
  log ERROR "This script must be run as root."
  exit
fi

# Grab the server's host name
HOSTNAME="$( hostname )"

# Start logging
log INFO "${banner_start}"
log INFO "-- bash info: ${BASH_VERSINFO[@]}"

# Check for the configuration file.
if [[ ! -r /etc/recap.conf ]]; then
  log WARNING "No configuration file found. Expecting /etc/recap.conf."
  log WARNING "Proceeding with defaults."
else
  source /etc/recap.conf
fi

# Create lock
recaplock

# Create output directory if it is not already present
create_output_dirs

# Check disk space before generating reports
check_disk_space

# Take a backup when needed
if [[ "${BACKUP,,}" == "yes" ]]; then
  log INFO "-- Taking backup, storing reports in ${BACKUPDIR}"
  create_backup
  exit 0
fi

# Log when running a snapshot
if [[ "${SNAPSHOT,,}" == "yes" ]]; then
  log INFO "-- Taking snapshot, storing reports in ${SNAPSHOTSDIR}"
fi

## Proceed to report generation
log INFO "-- Report suffix: ${LOG_SUFFIX}"

# Run the ps report
if [[ "${USEPS,,}" == "yes" ]]; then
  run_item_report "ps"
fi

# Run the resources report
#TODO: standardize the run_resources_report
if [[ "${USERESOURCES,,}" == "yes" ]]; then
  run_resources_report
fi

# Run the pstree report
if [[ "${USEPSTREE,,}" == "yes" ]]; then
  run_item_report "pstree"
fi

# Run the netstat report
#TODO: standardize the run_netstat_report
if [[ "${USENETSTAT,,}" == "yes" ]]; then
  run_netstat_report
fi

# Run the fdisk report
if [[ "${USEFDISK,,}" == "yes" ]]; then
  run_item_report "fdisk"
fi

# Run the mysql report
#TODO: standardize the run_mysql_report
if [[ "${USEMYSQL,,}" == "yes" ]]; then
  if type -p 'mysqladmin' > /dev/null; then
    run_mysql_report
  else
    log ERROR "mysql client is not installed, can't run mysql reports"
  fi
fi

# Load and Run plugins enabled
if [[ "${USEPLUGINS,,}" == "yes" ]]; then
  # Log the info about plugins(available and enabled)
  plugins_info
  # Load only enabled plugins
  plugins_load "${PLUGIN_E_DIR}"
  plugins=( $( plugins_list "${PLUGIN_E_DIR}" ) )
  if [[ "${#plugins[@]}" == 0 ]]; then
    log WARNING "No plugins found to execute in: ${PLUGIN_E_DIR}"
  else
    # Run the plugins enabled
    for plugin in ${plugins[@]}; do
      run_item_report "${plugin}"
    done
  fi
fi

# Check to see if report should be emailed
#TODO: Validate MAILTO format.
if [[ -n "${MAILTO}" ]]; then
  send_mail
fi

# We're done, time to exit
log INFO "Ended all the reports"
exit 0
