#!/bin/bash

### BEGIN INIT INFO
# Provides: rar_monitor_daemon
# Required-Start: $network
# Should-Start:
# Required-Stop: $network
# Should-Stop:
# Default-Start: 3 5
# Default-Stop: 0 1 2 6
# Short-Description: RAR Monitor Daemon
# Description:       This daemon monitors periodically the status of RAR on all active
#       targets and updates the history files which are used by the RAR-GUI command.
#       It also downloads from remote the logfiles after RAR has completed.
### END INIT INFO

#Author: mihai-radu.dumitru.ext@unify.com
#
#HOC:
# 2015.01.25 - initial creation
# 2015.02.01 - implemented cancel command
# 2015.03.19 - V5.5 - mihai
#   - I46434 : ECO: RAR saved the fail reason in other RAR sessions logfile 
# 2020.03.10 - V6.17 - mihai
#   - [OSFOURK-14815] NA16581328: AP RAR upgrade failure ; implement parallel SSH-ping from Host to APE for the r2awatchdog
# 2020.03.12 - V6.18 - mihai
#   - [OSFOURK-14277] NA16507402: RAR cannot be cancelled during state SCHEDULED-ACTIVATION-PENDING

LOGDIR=/var/log/firstinstall_rar
AUTOSSHPASSWSCR=/opt/replacement/rar/auto-ssh-passwd.sh

DAEMON_SH=`readlink -f $0`
PID_FILE=/var/run/rar_monitor_daemon.pid

[ ! -e $DAEMON_SH ] && echo "$DAEMON_SH does not exist" && exit 1

. /etc/rc.status

function parse_temuser_params() {
    local ipaddr=$1
    set -- `cat /var/tmp/rar/$1`
    export secretpasswd=$3
    export expprompt=""
    export theexptimeout=120
    export theuser=$2
    export sublogdir=$4
}

#params $1 ipaddr
function fetch_remote_status() {
    local ipaddr=$1
    parse_temuser_params $ipaddr
    rm -f $LOGDIR/tmp-rar-screen-log-$ipaddr
    rm -f $LOGDIR/tmp-rar-central-log-$ipaddr
    #send cancellation if requested
    [ -e /var/tmp/rar/${ipaddr}.cancel ] && {
        echo "`date` Sending cancel request to $ipaddr."
        1>/dev/null 2>/dev/null $AUTOSSHPASSWSCR ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no $theuser@$ipaddr touch /tmp/rar_user_abort
    }
    
    #OSFOURK-14815 - create /tmp/host_has_reached_me_over_ssh to indicate to remote appliance that Host can reach it
    1>/dev/null 2>/dev/null $AUTOSSHPASSWSCR ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no $theuser@$ipaddr touch /tmp/host_has_reached_me_over_ssh
    
    1>/dev/null 2>/dev/null $AUTOSSHPASSWSCR scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no $theuser@$ipaddr:/opt/rar-screen-log $LOGDIR/tmp-rar-screen-log-$ipaddr
    1>/dev/null 2>/dev/null $AUTOSSHPASSWSCR scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no $theuser@$ipaddr:/opt/rar-central-log $LOGDIR/tmp-rar-central-log-$ipaddr
}

function _update_status_main() {
    local ipaddr=$1
    rm -f $LOGDIR/tmp-fetch_remote_status-done-$ipaddr
    fetch_remote_status $ipaddr
    touch $LOGDIR/tmp-fetch_remote_status-done-$ipaddr
    exit 0
}

function _interpret_download_results() {
    local ipaddr=$1
    #I46434 : ECO: RAR saved the fail reason in other RAR sessions logfile 
    set -- `cat /var/tmp/rar/$ipaddr`
    sublogdir=$4
    if [ -e $LOGDIR/tmp-fetch_remote_status-done-$ipaddr -a -e $LOGDIR/tmp-rar-central-log-$ipaddr -a -e $LOGDIR/tmp-rar-screen-log-$ipaddr ]; then
        #append central log to the local one
        cp $LOGDIR/head-rar-log-history-$ipaddr $LOGDIR/rar-log-history-$ipaddr
        cat $LOGDIR/tmp-rar-central-log-$ipaddr >>$LOGDIR/rar-log-history-$ipaddr
        #append screen log file to the local one
        #on first time rename existing rar screen log accordingly
        #initialize screen log update mechanism
        [ -e $sublogdir/rar-screenlog-seekpos ] || echo 0 > $sublogdir/rar-screenlog-seekpos
        oldindex=`cat $sublogdir/rar-screenlog-seekpos`
        sizeoftmplog=`stat --printf=%s $LOGDIR/tmp-rar-screen-log-$ipaddr`
        deltabytes=$(($sizeoftmplog - $oldindex))
        [ $deltabytes -gt 0 ] && {
            2>/dev/null dd if=$LOGDIR/tmp-rar-screen-log-$ipaddr bs=1 count=$deltabytes skip=$oldindex >>$sublogdir/rar-screenlog
            echo $sizeoftmplog > $sublogdir/rar-screenlog-seekpos
            #update timestamp by touching the parameters file
            touch /var/tmp/rar/$ipaddr
            #prevent calcellation after some point
            grep -q "From this moment on activation of new software can not be interrupted" $sublogdir/rar-screenlog && touch /var/tmp/rar/$ipaddr.beyond_cancel
        }
    else
        echo "`date` : imcomplete fetch_remote_status for ip $ipaddr."
    fi
    #check timestamp of last update and declare end with error if more than 1,5 hours timeout (covers also no network access case during RAR watchdog health check)
    tslastupd=`stat -c %Y /var/tmp/rar/$ipaddr`
    timesincelastupd=$((`date +%s` - $tslastupd))
    [ $timesincelastupd -gt 5400 ] && {
        echo "`date` $ipaddr did not receive any status update since more than 1,5 hours. Declaring failed."
        echo -e "\n`date` [ERROR] TIMEOUT : did not receive any status update since more than 1,5 hours." >>$sublogdir/rar-screenlog
        echo "#`date +%s` : <`date`> : Ended RAR script [ERROR]" >>$LOGDIR/rar-log-history-$ipaddr
    }
    return 0
}

function _try_logs_download_main() {
    local ipaddr=$1
    parse_temuser_params $ipaddr
    exec 1>>$sublogdir/rar_logs_download.out
    exec 2>&1
    #will make so much retries as necessary within the next 1 hour and
    startdate=`date +%s`
    while [ 1 ]; do
        curdate=`date +%s`
        [ $(($curdate - $startdate)) -gt 3600 ] && break
        rm -rf $sublogdir/rar_logs.tgz
        1>/dev/null 2>/dev/null $AUTOSSHPASSWSCR scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no $theuser@$ipaddr:/opt/rar_logs.tgz $sublogdir/.
        [ -e $sublogdir/rar_logs.tgz ] && {
            echo "`date` _try_logs_download_main[$ipaddr] : remote logs download OK."
            #on success indicate to the rar target that it can delete the temp user
            1>/dev/null 2>/dev/null $AUTOSSHPASSWSCR ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no $theuser@$ipaddr touch /opt/rar_host_sais_delete_temp_user
            exit 0
        }
        sleep 60
    done
    echo "`date` _try_logs_download_main[$ipaddr] : timeout - failed to retrieve logs after one hour"
    exit 1
}

function _spawn_logs_download() {
    local ipaddr=$1
    parse_temuser_params $ipaddr
	local LAST_LINE=`2>/dev/null tail -n 1 $LOGDIR/rar-log-history-$ipaddr`
	[ -z "$LAST_LINE" ] && return 0
	if echo "$LAST_LINE" | grep -q "Ended RAR script"; then
        #improvement: download different logs depending on success or error ? - maybe handle differently on remote side
        #launch logs download in separate process session
        0</dev/null 1>/dev/null 2>/dev/null setsid /bin/bash /etc/init.d/rar_monitor_daemon _try_logs_download $ipaddr &
        #wait 5 seconds to make sure log download daemon can read the rar params for accessing the target
        sleep 5
        #cleanup status files and temp files
        rm -f /var/tmp/rar/$ipaddr /var/tmp/rar/${ipaddr}.cancel /var/tmp/rar/$ipaddr.beyond_cancel
        rm -f $LOGDIR/tmp-rar-screen-log-$ipaddr
        rm -f $LOGDIR/tmp-rar-central-log-$ipaddr
        rm -f $LOGDIR/tmp-fetch_remote_status-done-$ipaddr
        rm -f $LOGDIR/head-rar-log-history-$ipaddr
        rm -f $LOGDIR/scheduled-info-$ipaddr.txt
        #logrotate here
        #in case of error keep last 3 entries, in case of success keep just this last one
        if echo "$LAST_LINE" | grep "Ended RAR script" | grep -q "SUCCESS"; then
            #in case of success we do not need the complete /etc
            rm -f $sublogdir/complete-etc.tar.gz
            #keep the screen logs from all older instances
            for ascrlog in $(find $LOGDIR/$ipaddr-* -name rar-screenlog); do
                #skip the current one
                [[ $ascrlog =~ $sublogdir ]] && continue
                #rename it to contain the run-name
                runname=${ascrlog#*firstinstall_rar/}
                runname=${runname%/*}
                cp $ascrlog $sublogdir/previuos_rar_run-rar-screenlog-$runname
            done
            ls -dv -1 $LOGDIR/$ipaddr-* | head -n -1 | xargs rm -rf
        else
            ls -dv -1 $LOGDIR/$ipaddr-* | head -n -3 | xargs rm -rf
        fi
        echo "`date` finished run for $ipaddr"
    fi
    return 0
}

function have_jobs() {
    local ajob
    for ajob in $myjobs; do
        [ -e /proc/$ajob ] && {
            #check that I am parent
            [ "$mypid" = "`ps --no-headers -o ppid -p $ajob`" ] && return 0
        }
    done
    return 1
}

function _daemon_main() {
    echo $$ > $PID_FILE
    exec 1>>$LOGDIR/rar_monitor_daemon.out
    exec 2>&1
    
    mypid=$$
    
    echo "`date` started."
    
    while [ 1 ]; do
        sleep 60
        havetgts=false
        all_active_ips=""
        for afile in $(find /var/tmp/rar -type f); do
            afile=${afile##*/}
            [[ "$afile" =~ ^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$ ]] || continue
            havetgts=true
            #fill in list
            all_active_ips="$afile $all_active_ips"
        done
        $havetgts || {
            echo "`date` stopped since no more targets running."
            exit 0
        }
        myjobs=""
        #spawn refresh workers for each appliance
        for anappl in $all_active_ips; do
            /bin/bash $0 _update_status_ $anappl &
            myjobs="$myjobs $!"
        done
        #wait for all workers max 180 secs to complete and if not, then kill them
        i=18
        while [ $i -gt 0 ]; do
            i=$(($i - 1))
            sleep 10
            have_jobs || break
        done
        #kill nicely all remaining jobs
        have_jobs && {
            for i in 1 2 3; do
                for ajob in $myjobs; do
                    [ "$mypid" = "`ps --no-headers -o ppid -p $ajob`" ] && kill $ajob
                done
                sleep 5
                have_jobs || break
            done
        }
        #kill not nicely all jobs
        have_jobs && {
            for ajob in $myjobs; do
                [ "$mypid" = "`ps --no-headers -o ppid -p $ajob`" ] && kill -9 $ajob
            done
        }
        #takeover the downloaded texts and fill in the persistant status files and also check for status update timeouts
        # also check if done and spawn download of logs if necessary
        for anappl in $all_active_ips; do
            _interpret_download_results $anappl
            _spawn_logs_download $anappl
        done
    done
}

rc_reset
case "$1" in
	start)
		( checkproc -p $PID_FILE /bin/bash ) && echo "The RAR Monitor Daemon is already running." && exit 2
		[ ! -x $DAEMON_SH ] && chmod u+x $DAEMON_SH
		echo -n "Starting the RAR Monitor Daemon daemon."
		startproc -f -p $PID_FILE /bin/bash $DAEMON_SH _daemon_
        for i in 1 2 3 4 5; do
            ( checkproc -p $PID_FILE /bin/bash ) && {
                sleep 1
                true
                rc_status -v
                rc_exit
                exit $?
            }
            sleep 1
        done
        false
		rc_status -v
	;;
	stop)
		echo -n "Stopping the RAR Monitor Daemon daemon."
        ( checkproc -p $PID_FILE /bin/bash ) && {
            echo "`date` stopped." >>$LOGDIR/rar_monitor_daemon.out
        }
		killproc -p $PID_FILE -TERM -t20 /bin/bash
		rc_status -v
	;;
    _daemon_)
        _daemon_main
        exit $?
    ;;
    _update_status_)
        shift
        _update_status_main $*
        exit $?
    ;;
    _try_logs_download)
        shift
        _try_logs_download_main $*
        exit $?
    ;;
	status)
		echo -n "Checking status for the RAR Monitor Daemon"
		checkproc -p $PID_FILE /bin/bash
		rc_status -v
	;;
	*)
		echo "Usage: $0 {start|stop|status}"
		exit 3
	;;
esac
rc_exit
