#!/bin/bash
#
# PowerFlex DAX issues detection & remediation utilities V1.0.0
#
###############################################################################
# A simple script to collect the output of "ndctl list -vvv" from several hosts
# in parallel and analyze it using the analyze_ndctl.py, printing a summary
# about nvdimm health.
###############################################################################
function usage()
{
    cat <<EOF

Usage:
  ${MY_NAME} <arguments>

Arguments:
   -f <hosts_file>       An argument that specifies the path to the hosts file.
                         The hosts file contains a list of per-host line records,
                         each with one or two fields in the following format:

                         [HOST_FRIENDLY_NAME] [HOST_IP_OR_RESOLVABLE_NAME]


                         HOST_FRIENDLY_NAME - will be used when presenting the
                         output. This field can be omitted, and if it is then
                         HOST_IP_OR_RESOLVABLE_NAME will be used as the friendly
                         name.

                         HOST_IP_OR_RESOLVABLE_NAME - Will be used to reach the
                         host with ssh.


                         Examples of legal hosts file lines:

                         SDS_0x12345 root@mymachine
                         suspicious_sds user2@1.2.3.4
                         42.42.42.42

   -m <mdm_machine_ip>   The IP of the primary MDM machine. The script will
                         use ssh to run a CLI query on this machine.

   -u <mdm_machine_user> The linux account to use for the SSH connection to the
                         MDM machine. (This is not the MDM username.)

                         The default value is \${USER}

   -p <parallelism>      An optional argument that specifies the number of
                         parallel ssh connections to maintain. Its value
                         must be set to 1 if any ssh connection will require a
                         password to be entered.

   -r                    If this is set, the outputs directory will be removed
                         at the end of the run.


   NOTES: If the hostfile is not directly provided with "-f", an MDM CLI query
         will be attempted. In order for the query to succeed, an active CLI
         token must exist on the MDM machine for the linux user provided
         with "-u".

         For example:

         (On machine "machine_a"):
         root@machine_a:  cli --login ....

         (On machine "machine_b"):
         user@machine_b: ./${MY_NAME} -u root -m machine_a

         If "-m" was not specified, the script will assume that it runs
         on the MDM host and will try to call the MDM cli directly without
         using ssh.
EOF
}

###############################################################################
function check_sanity()
{
    if [ ! -x "${ANALYZER_SCRIPT}" ]; then
        echo "Analyzer script ${ANALYZER_SCRIPT} must be located in this"
        echo "script's directory!"
        exit 1
    fi
}

###############################################################################
function process_opts()
{
    while getopts ":p:f:m:u:hr" opt; do
        case ${opt} in
            h)
                usage
                exit 0
                ;;
            
            f)
                HOSTS_FILE_PATH=${OPTARG}
                ;;

            p)
                PARALLELISM=$((OPTARG))
                ;;

            r)
                SHOULD_REMOVE_OUTPUTS_DIR=1
                ;;

            m)
                MDM_IP=${OPTARG}
                ;;
            
            u)
                MDM_USER=${OPTARG}
                ;;
            
            ":")
                echo "Option ${OPTARG} requires an argument!"
                exit 1
                ;;

            "?")
                echo "Unknown option ${OPTARG}"
                exit 1
                ;;
            *)
                echo "Please specify options as per the usage."
                usage
                exit 1
        esac
    done

    if (( PARALLELISM == 0 )); then
        PARALLELISM=1
    fi

    echo "Parallelism is set to ${PARALLELISM}"
}

###############################################################################
function determine_hosts_file()
{
    local cur_line=

    local sds_id=
    local ips_list=    
    local sds_first_ip=

    if [ -z "${HOSTS_FILE_PATH}" ]; then
        echo "The hosts file was not provided. A CLI query will be attempted."

        HOSTS_FILE_PATH=${OUTPUTS_DIR}/queried_hosts
        rm -f ${HOSTS_FILE_PATH}

        run_cli_sds_query |
            while read -a cur_line; do
                if (( ${#cur_line[@]} < 5 )); then
                    echo "Unexpected output line: ${cur_line[@]}"
                    continue
                fi

                sds_id=${cur_line[2]}
                ips_list=${cur_line[4]}
                sds_first_ip=$(echo ${ips_list} | cut -d',' -f 1)

                echo "${sds_id} ${sds_first_ip}" >> ${HOSTS_FILE_PATH}
            done

        if [ $? -ne 0 ]; then
            echo "Failed to run CLI."
            exit 1
        fi
    fi

    if [ ! -f "${HOSTS_FILE_PATH}" ]; then
        echo "The hosts file doesn't exist!"
        exit 1
    fi
}

###############################################################################
function run_cli_sds_query()
{
    if [ -z "${MDM_IP}" ]; then
        # Assume that the CLI is here
        if [ ! -x "${PF_CLI}" ]; then
            echo "Powerflex CLI was not detected!"
            exit 1
        fi
        ${CLI_SDS_QUERY_CMD}
    else
        ssh ${MDM_USER}@${MDM_IP} "${CLI_SDS_QUERY_CMD}" </dev/null
    fi
}

###############################################################################
function make_outputs_dir()
{
    OUTPUTS_DIR=$(mktemp -d ${MY_DIR}/analyze_ndctl_run.XXXXXX)
    if [ ! -d ${OUTPUTS_DIR} ]; then
        echo "Failed to create outputs directory!"
        exit 1
    fi

    echo "Created outputs directory ${OUTPUTS_DIR}."

    NDCTL_OUTPUTS_DIR=${OUTPUTS_DIR}/ndctl_outputs
    mkdir ${NDCTL_OUTPUTS_DIR}

    if [ $? -ne 0 ]; then
        echo "Failed to create ${NDCTL_OUTPUTS_DIR}!"
        exit 1
    fi
}

###############################################################################
function cleanup()
{
    if [ -d "${OUTPUTS_DIR}" ]; then
        if [ ${SHOULD_REMOVE_OUTPUTS_DIR} -eq 1 ]; then
            echo
            echo "Removing the outputs directory ${OUTPUTS_DIR}"
            rm -rf ${OUTPUTS_DIR}
        else
            echo
            echo "The run outputs are kept in ${OUTPUTS_DIR}"
        fi
    fi
}

###############################################################################
function collect_ndctl_query()
{
    local cur_host_line=

    while read cur_host_line; do
        if [ -z "${cur_host_line}" ]; then
            continue
        fi
        echo "Read host line ${cur_host_line} from the hosts file."
        collect_from_host ${cur_host_line}
        echo "Collected from host ${cur_host_line}"
    done < ${HOSTS_FILE_PATH}

    # Wait for everybody
    wait
}

###############################################################################
function collect_from_host()
{
    local host_id=
    local host_addr=

    if [ ${#@} -eq 2 ]; then
        host_id=${1}
        host_addr=${2}
    elif [ ${#@} -eq 1 ]; then
         host_id=${1}
         host_addr=${1}
    else
        echo "Unexpected hosts line format ${@}"
        return 
    fi
        
    if (( NUM_OF_INFLIGHTS < PARALLELISM )); then
        (( NUM_OF_INFLIGHTS++ ))
    else
        echo "Waiting for one ssh to end."
        wait -n
    fi

    if (( PARALLELISM == 1 )); then
        run_ndctl_query_over_ssh_with_passwd ${host_id} ${host_addr}
    else
        ( run_ndctl_query_over_ssh_no_passwd ${host_id} ${host_addr} ) &
    fi
}

###############################################################################
function run_ndctl_query_over_ssh_no_passwd()
{
    local host_id=${1}
    local host_addr=${2}

    local out_file_name=$(build_out_file_name ${host_id} ${host_addr})
    
    ssh ${host_addr} \
        -oStrictHostKeyChecking=no \
        -oPasswordAuthentication=no \
        "${REMOTE_COMMAND}" \
        >${NDCTL_OUTPUTS_DIR}/${out_file_name} </dev/null
}

###############################################################################
function run_ndctl_query_over_ssh_with_passwd()
{
    local host_id=${1}
    local host_addr=${2}

    local out_file_name=$(build_out_file_name ${host_id} ${host_addr})
    
    ssh ${host_addr} \
        -oStrictHostKeyChecking=no \
        -oPasswordAuthentication=yes \
        "${REMOTE_COMMAND}" >${NDCTL_OUTPUTS_DIR}/${out_file_name} </dev/null
}

###############################################################################
function build_out_file_name()
{
    local host_id=${1}
    local host_addr=${2}

    if [ "${host_id}" == "${host_addr}" ]; then
        echo "${host_id}"
    else
        echo "${host_id}-${host_addr}"
    fi
}

###############################################################################
function process_ndctl_query()
{
    echo "Running ${ANALYZER_SCRIPT}..."
    # The first argument is a directory to which to write the analyzer log
    # and the host summary files.
    # The second argument is a directory with ndctl output files per-host.
    ${ANALYZER_SCRIPT}  ${OUTPUTS_DIR} ${NDCTL_OUTPUTS_DIR}
}

###############################################################################
# Globals
MY_NAME=$(basename ${BASH_SOURCE[0]})
MY_DIR=$(dirname ${BASH_SOURCE[0]})

REMOTE_COMMAND="ndctl list -vvv"

PF_CLI="/opt/emc/scaleio/mdm/bin/cli"
CLI_SDS_QUERY_CMD="${PF_CLI} --query_properties --object_type SDS --all_objects --properties IPS --flat_list"

ANALYZER_SCRIPT=${MY_DIR}/analyze_ndctl.py

NUM_OF_INFLIGHTS=0

PARALLELISM=20
HOSTS_FILE_PATH=
SHOULD_REMOVE_OUTPUTS_DIR=0

MDM_IP=
MDM_USER=${USER}

###############################################################################
# Main
trap cleanup EXIT

check_sanity
process_opts $@

make_outputs_dir
determine_hosts_file
collect_ndctl_query
process_ndctl_query