#!/bin/bash
# $Copyright$
# Copyright (C) Fujitsu Technology Solutions 2009, 2010, 2011, 2012, 2013, 2014, 2015
# All rights reserved
#
#
# Copyright (C) Fujitsu Technology Solutions 2009
# All Rights Reserved
#
# restart script for ServerView Agents.
#
# SYNOPSIS
#       srvmagtCron
# DESCRIPTION
#       Script for periodical check whether srvmagt daemons are still running.
#       Called by cron from crontab.  Actual execution depends on
#       existence of file "srvmagtCronOn" in srvmagt root directory.
# EXIT
#   0   OK; but no operation (nothing to do)
#   1   OK, necessary restarts completed. 
#   2   Failed due to missing ressource (in this case snmpd)
#   3   Restart postponed 
#
PATH=$PATH:/sbin
. /etc/srvmagt/functions
. /etc/srvmagt/config

SRVMAGT=${SRVMAGT:="/etc/srvmagt"}
LOGDIR=${LOGDIR:="/var/log/srvmagt"}

LogFile=$LOGDIR/log.srvmagtCron
TmpFile=/var/tmp/$$.1
MaxEntries=100                  # maximum number of log entries

SYSTEMD=0
[ -x /usr/bin/systemctl ] && SYSTEMD=1

InvokeMsg="### LAST INVOCATION of srvmagtCron: `date` \n"       # first $LogFile line

#----------
# Log: writes logging entries into $LogFile prepended by the date.
# Includes wrap around handling of $LogFile.
# NOTE:
#       "File system full" problems might result in empty log files.
# $*    String(s) to log


Log () {
        # Write $LogFile in a roll-up fashion, retaining 2 header lines.
        # 1. Change header lines from $LogFile into $LogTmp and append current log entry.
        # 2. Copy the $MaxEntries-n latest entries from $LogFile into $LogTmp 
        # 3. Rename $LogTmp file in $LogFile

        [ -f $LogFile ] || echo $InvokeMsg > $LogFile

        # Change the first 2 header lines of $LogFile showing the last invocation.
        #
        echo -e $InvokeMsg > $TmpFile
        awk ' NR >= 3 { print $0 } ' $LogFile >> $TmpFile       
        echo -e "`date '+%b %d %T:'` $*" >> $TmpFile
        
        set `wc $TmpFile` ; LogTmpSize=$1

        if [ $LogTmpSize -gt $MaxEntries ]
        then
                # Copy header lines and latest entries. 
                # File max size should not be bigger then $MaxEntries
                #
                mv $TmpFile $LogFile
                awk ' /./ { 
                        if (NR <3 || NR > (LogTmpSize - MaxEntries + 2)) {
                                printf("%s\n", $0)
                        }
                } ' MaxEntries=$MaxEntries LogTmpSize=$LogTmpSize $LogFile >> $TmpFile
        fi

        mv $TmpFile $LogFile
}

# get the agents to be monitored from the config file (Default: all)
SV_AGENTS=${AgentsToBeStarted:="sc sc2 bus hd unix ether bios secur status inv thr vv hpsim vme svupdate os"}


#----------
# getAgtStatus - get status of ServerView Agents
#
# stdout	agents which are not running
# Return	0 if all agents are running,  1 otherwise

function getAgtStatus() {
	local pid theAgt rc=0

	for theAgt in $SV_AGENTS; do
	[ -x /usr/sbin/${theAgt}agt ] || {
		continue
	}

	pid=$(mypidof /usr/sbin/${theAgt}agt) || {
		# "NoSuchProcess /usr/sbin/${theAgt}agt"
		printf "$theAgt "
		rc=1
		continue
	}
	done

	return $rc
}

#----------
# waitForEecdTerm - waits 2 minutes maximum on termination of eecd 
#
# No arguments

function WaitForEecdTerm() {
    count=120
    while [ $count -gt 0 ]; do
        pid=$(mypidof "/usr/sbin/eecd") || break
        # if eecd is still running we have to wait
        count=$(($count - 1))
        sleep 1
    done
    pid=$(mypidof "/usr/sbin/eecd") && {
        # eecd is still alive, turn on supervision and try again in 15 minutes 
        Log "WARNING! eecd is still alive.\n\tRestart Postponed."   
        echo "" > $SRVMAGT/srvmagtCronOn
        return 1
    }
    return 0
}

# srvmagtCron should run, if periodical check is turned on, i.e. the file 
# srvmagtCronOn exist.

[ -f $SRVMAGT/srvmagtCronOn ] || exit 0

# If enviromnent variable SRVMAGTTRC is set and non-null switch on trace.
#
[ -n "${SRVMAGTTRC:+"Something"}" ] && set -x

# Replace first line in $LogFile with invocation message showing current date.
# This needs a temporary file. If it cannot be created, the collected log disappears.
#

if [ -f $LogFile ]; then
    sed ' 1,2d ' $LogFile > $TmpFile
    echo -e $InvokeMsg > $LogFile
    cat $TmpFile >> $LogFile
else
    echo -e $InvokeMsg > $LogFile
fi
rm -f $TmpFile

# Get PID from SVRemoteConnector.
SCSrc=0
scspid=$(mypidof /opt/fujitsu/ServerViewSuite/SCS/SVRemoteConnector) || {
	Log "Periodical check found: SVRemoteConnector down. Try restart."
	if [ $SYSTEMD -eq 1 ]; then
		systemctl restart srvmagt_scs.service
	else 
		/etc/init.d/srvmagt_scs restart > /dev/null 2>&1
	fi
	SCSrc=1
}

# Check SNMP daemon, ServerView agents and eecd 
snmpdPid=$(mypidof /usr/sbin/snmpd) && {
    # "SNMP daemon snmpd is alive - check ServerView agents"
    if NotRunningAgts=$(getAgtStatus) ; then {
        # ServerView agents are alive - check eecd.
        # Get PID from eecd.
        eecdpid=$(mypidof "/usr/sbin/eecd") &&
            # nothing to do
            exit $SCSrc
        
        Log "Periodical check found: eecd down. Try restart."
        /usr/sbin/srvmagt restart > /dev/null 2>&1
        exit 1
    }
    else { 
        # At least one agent is not running. Restart it.
        Log "Periodical check found: srvmagt agent(s) ' $NotRunningAgts' down. Try restart."
        /usr/sbin/srvmagt stop > /dev/null 2>&1 ||
            WaitForEecdTerm || exit 3
        /usr/sbin/srvmagt start > /dev/null 2>&1
        exit 1
    }
	fi
}

# SNMP daemon snmpd is not running. Restart it.

Log "Periodical check found: snmpd down. Try restart."

/usr/sbin/srvmagt stop > /dev/null 2>&1 ||
	WaitForEecdTerm || exit 3

if [ $SYSTEMD -eq 1 ]; then
	systemctl restart snmpd.service
else 
	/etc/init.d/snmpd restart > /dev/null 2>&1
fi

/usr/sbin/srvmagt start > /dev/null 2>&1
exit 1
