#!/usr/bin/perl
# Author: Maik Ernst
# History:
# 2014-01-03 - initial script
#
# LICENSE
# Hitachi, Ltd. All rights reserved
# Hitachi Data Systems products and services can be ordered only under the terms and conditions
# of the applicable Hitachi Data Systems agreements.
#
# NAME
# ppmonitor.pl
#
# SYNOPSIS
# see usage function below
#
# DESCRIPTION
#
# OPTIONS 
# see "usage" function below.
#
# NOTES
#
# AUTHOR
#   Maik Ernst (maik.ernst@hds.com) - January,2014
#
# MODIFICATIONS
#   2014-01-03 - v1.0	- initial script
#   2014-06-16 - v1.1	- Moving up to folder ppmonitor. 
#						- Adding .zip as suffix to check
#						- Adding pptransfer checks (-i). 
#	2014-04-07 - v1.2	- Adding Sys::Hostname to get hostname of script
#	2015-05-02 - v1.3	- Changing script name from ppmonitor.pl to ppmonitor.pl to meet swiss script standard
#						- Adding CURL transfer check
#  2015-05-10 - v1.4.0	- Reorganize folder structure for HiCHperfpkg according to Swiss development group
#  2016-03-25 - v1.5.0	- LOG->autoflush(1), fix logfile checks
#						- adding DELETE_OLD_LOGS / log file rotation pptransfer_logkeepdays

use strict;
use warnings;
use POSIX qw(strftime); # To get the time in function LOG_MESSAGE
use Getopt::Long qw(:config no_ignore_case bundling); # Get options /  qw(:config bundling no_ignore_case) makes getopt case sensitive 
use File::Basename; # To get dirname function
use File::Path; #To use mkpath
use File::Spec; #To get absolute path
use Net::SMTP; # To send emails
use Config; # To get OS of system
use Cwd 'abs_path';
use Sys::Hostname; # To get hostname
use IO::Handle; # required for autoflush

# Use custom/local perl modules lib. FindBin is needed to find the module when running the script from relativ/absolute paths.
use FindBin qw($Bin);
use lib $Bin;
use lib "$Bin\\..\\lib\\modules";
use Config::Tiny;

### main ### DO NOT CHANGE ANYTHING IF DO NOT KNOW WHAT YOU ARE DOING!
my $VERSION = "1.5.0";

# Error handling. Change function that a message gets written to log file before dieing.
INIT {$SIG{__DIE__}=sub {LOG_MSG("normal",3,"GENERAL","Script died: $_[0]") and close LOG;}}
INIT {$SIG{__WARN__}=sub {LOG_MSG("normal",4,"GENERAL","Script warning: $_[0]")}}

#print STDERR (join("|",@ARGV),"\n");

# Check Flags
my $flag_help;
my $flag_version;
my $flag_config;
my $vbse;

GetOptions (
	'h|help' => \$flag_help,
	'V|VER' => \$flag_version,
	'v|verbose' => \$vbse,
	'c|config=s' => \$flag_config,
           ) or die USAGE();
		   
# Change to directory of the running script
# abs_path for $0 or __FILE__
my ( $script_name, $script_path, $script_suffix ) = fileparse( abs_path($0), qr{\.[^.]*$} );
my $script_dirname = dirname(abs_path($0));
# Change slashes to backslashes
$script_dirname =~ s/\//\\/g;
chdir ($script_dirname);
my $script_rootdir =  dirname ($script_dirname);

# Check flags and print usage
if ($flag_version)
{
	print "$0 Version: $VERSION\n";
	exit;
}

if ($flag_help)
{
	USAGE();
	exit;
}

if (!$flag_config)
{
	print "\n!!! You must define -c <config file>. Example: ppmonitor.pl -c pp_54068.conf !!!\n";
    USAGE();
    exit;
}

unless (-e "${script_rootdir}\\etc\\${flag_config}")
{
	print "\n!!! Your with -c defined config file \"${script_rootdir}\\etc\\${flag_config}\" does not exist. !!!\n";
    USAGE();
    exit;
}
$flag_config = "${script_rootdir}\\etc\\${flag_config}";

# Get config
my $conf_file = Config::Tiny->new;
$conf_file = Config::Tiny->read($flag_config);

# Check & set variables for log

if (!$conf_file->{maindir}->{pp_log})
{
	print "\n!!! Variable \"[maindir] pp_log\" is not defined correctly in $flag_config !!!\n";
	exit;
}

my $flag_serial = "NOSERIAL";
if ($conf_file->{main}->{ppcollect_serial})
{
	$flag_serial = $conf_file->{main}->{ppcollect_serial};
	#print "\n!!! Variable \"[main] ppcollect_serial\" is not defined correctly in $flag_config!!!\n";
	# exit;
} 

# If set to default<raid|log|vmio>, then create automatically sourcedir path
my $flag_transfer;
if (lc $conf_file->{main}->{pptransfer_sourcedir} eq "defaultraid")
{
	if (!$conf_file->{main}->{ppcollect_serial})
	{
		print "\n!!! \"[main] ppcollect_serial\" is not defined correctly in $flag_config!!!";
		print "\n!!! Therefore \"[main] pptransfer_sourcedir\" can not be set to \"defaultraid\"\n";
		exit;
	} 
	
	elsif (!$conf_file->{zip}->{ppcollect_archivedir})
	{
		print "\n!!! \"[zip] ppcollect_archivedir\" is not defined correctly in $flag_config!!!";
		print "\n!!! Therefore \"[main] pptransfer_sourcedir\" can not be set to \"defaultraid\"\n";
		exit;
	}
	
	elsif (!$conf_file->{maindir}->{pp_data})
	{
		print "\n!!! \"[maindir] pp_data\" is not defined correctly in $flag_config!!!";
		print "\n!!! Therefore \"[main] pptransfer_sourcedir\" can not be set to \"defaultraid\"\n";
		exit;
	}
	
	else
	{
		$flag_transfer = "${script_rootdir}\\$conf_file->{maindir}->{pp_data}\\$conf_file->{zip}->{ppcollect_archivedir}$conf_file->{main}->{ppcollect_serial}";
		unless (-d $flag_transfer)
		{
			print "\n!!! \"[main] pptransfer_sourcedir\" is set to \"defaultraid\" in $flag_config but the default directory does not exist !!!\n";
			print "!!! Directory: $flag_transfer does not exist !!!\n";
			exit;
		}
	}
}
elsif (lc $conf_file->{main}->{pptransfer_sourcedir} eq "defaultlog")
{
	if (!$conf_file->{maindir}->{pp_log})
	{
		print "\n!!! \"[maindir] pp_log\" is not defined correctly in $flag_config!!!";
		print "\n!!! Therefore \"[main] pptransfer_sourcedir\" can not be set to \"defaultlog\"\n";
		exit;
	}
	else
	{
		$flag_transfer = "${script_rootdir}\\$conf_file->{maindir}->{pp_log}";
		unless (-d $flag_transfer)
		{
			print "\n!!! \"[main] pptransfer_sourcedir\" is set to \"defaultlog\" in $flag_config but the default directory does not exist !!!\n";
			print "!!! Directory: $flag_transfer does not exist !!!\n";
			exit;
		}
	}
}
elsif (lc $conf_file->{main}->{pptransfer_sourcedir} eq "defaultvmio")
{
	if (!$conf_file->{maindir}->{pp_data})
	{
		print "\n!!! \"[maindir] pp_data\" is not defined correctly in $flag_config!!!";
		print "\n!!! Therefore \"[main] pptransfer_sourcedir\" can not be set to \"defaultvmio\"\n";
		exit;
	}
	else
	{
		$flag_transfer = "${script_rootdir}\\$conf_file->{maindir}->{pp_data}\\vmiooutput";
		unless (-d $flag_transfer)
		{
			print "\n!!! \"[main] pptransfer_sourcedir\" is set to \"defaultvmio\" in $flag_config but the default directory does not exist !!!\n";
			print "!!! Directory: $flag_transfer does not exist !!!\n";
			exit;
		}
	}
	
}
else
{
	unless (-d $conf_file->{main}->{pptransfer_sourcedir})
	{
		print "\n!!! Source directory defined as \"[main] pptransfer_sourcedir\" in $flag_config does not exist !!!\n";
		print "!!! Directory: $conf_file->{main}->{pptransfer_sourcedir} !!!\n";
		exit;
	}
	$flag_transfer = $conf_file->{main}->{pptransfer_sourcedir};
}

my $flag_transfer_base = basename($flag_transfer);
    
my $scriptdir_log = "${script_rootdir}\\$conf_file->{maindir}->{pp_log}";
my $serial = $conf_file->{main}->{ppcollect_serial};

#my $serial = $1; # see regexp above for serial 
my $time_scriptstart = strftime("%Y-%m-%d_%H-%M-%S", localtime);
my $HOST = hostname;
my $SCRIPTLOG_FILE = "${scriptdir_log}\\${script_name}_${flag_transfer_base}_${time_scriptstart}.log";
my $SCRIPTLOG_DEL_PATH = "${scriptdir_log}\\${script_name}_${flag_transfer_base}_*.log";

# Create directories
unless (-d $scriptdir_log) { mkpath($scriptdir_log) or die ("Can't create dir $scriptdir_log: $!\n"); }

# Open script logfile
open(LOG,"> $SCRIPTLOG_FILE") or die ("Can't open $SCRIPTLOG_FILE: $!\n");
LOG->autoflush(1);
LOG_MSG("normal",5,"GENERAL","Starting Script $0 Version ${VERSION}");
LOG_MSG("normal",5,"GENERAL","Operating system: \"$Config{osname}\"");
LOG_MSG("normal",5,"GENERAL","Hostname: \"$HOST\"");
LOG_MSG("normal",5,"GENERAL","This is for serial ${flag_serial}");

foreach my $conf_section (sort keys %{$conf_file}) 
{
   LOG_MSG("normal",5,"PARAM","[$conf_section]");
    foreach my $conf_parameter (sort keys %{$conf_file->{$conf_section}}) 
	{
        if ($conf_parameter =~ m/pwd/)
		{
			if ($conf_file->{$conf_section}->{$conf_parameter})
			{
				LOG_MSG("normal",5,"PARAM","\t$conf_parameter = ***");
			}
			else
			{
				LOG_MSG("normal",5,"PARAM","\t$conf_parameter = $conf_file->{$conf_section}->{$conf_parameter}");
			}
		}	
		else
		{
			LOG_MSG("normal",5,"PARAM","\t$conf_parameter = $conf_file->{$conf_section}->{$conf_parameter}");
		} 
    }
}

my $flag_win;
my $flag_mailrelay;
my $flag_testemail;
my @flag_email;
my $flag_emailfrom;
my $flag_runsperday;
my $flag_ioportaltransfer;
my $flag_logkeepdays;
my $lglvl;

if (lc $conf_file->{log}->{ppmonitor_loglevel} eq "normal")
{
	$lglvl = $conf_file->{log}->{ppmonitor_loglevel};
	LOG_MSG($lglvl,5,"PARAM","Loglevel: normal");
}

elsif (lc $conf_file->{log}->{ppmonitor_loglevel} eq "debug")
{
	$lglvl = $conf_file->{log}->{ppmonitor_loglevel};
	LOG_MSG($lglvl,5,"PARAM","Loglevel: debug");
}

else
{
	LOG_MSG("normal",3,"PARAM","\"[log] ppmonitor_loglevel\" is not valid."); 
	LOG_MSG("normal",3,"PARAM","See \"$flag_config\" and check if it is set to \"normal\" or \"debug\".");
	LOG_MSG("normal",3,"GENERAL","Finishing Script $0");
	close LOG;
	exit;
}

if (!$conf_file->{maindir}->{pp_bin})
{
	LOG_MSG($lglvl,3,"PARAM","\"[maindir] pp_bin\" = \"$conf_file->{maindir}->{pp_bin}\" is not defined correctly.");
	LOG_MSG($lglvl,3,"PARAM","Please check your config \"$flag_config\".");
	LOG_MSG($lglvl,3,"GENERAL","Finishing Script $0");
	close LOG;
    exit;
}
my $scriptdir_bin = "${script_rootdir}\\$conf_file->{maindir}->{pp_bin}";

if (!$conf_file->{maindir}->{pp_data})
{
	LOG_MSG($lglvl,3,"PARAM","\"[maindir] pp_data\" is not defined correctly.");
	LOG_MSG($lglvl,3,"PARAM","Please check your config \"$flag_config\".");
	LOG_MSG($lglvl,3,"GENERAL","Finishing Script $0");
	close LOG;
    exit;
}
my $scriptdir_data = "${script_rootdir}\\$conf_file->{maindir}->{pp_data}";

if (!$conf_file->{maindir}->{pp_etc})
{
	LOG_MSG($lglvl,3,"PARAM","\"[maindir] pp_etc\" is not defined correctly.");
	LOG_MSG($lglvl,3,"PARAM","Please check your config \"$flag_config\".");
	LOG_MSG($lglvl,3,"GENERAL","Finishing Script $0");
	close LOG;
    exit;
}
my $scriptdir_etc = "${script_rootdir}\\$conf_file->{maindir}->{pp_etc}";

if (!$conf_file->{maindir}->{pp_doc})
{
	LOG_MSG($lglvl,3,"PARAM","\"[maindir] pp_doc\" is not defined correctly.");
	LOG_MSG($lglvl,3,"PARAM","Please check your config \"$flag_config\".");
	LOG_MSG($lglvl,3,"GENERAL","Finishing Script $0");
	close LOG;
    exit;
}
my $scriptdir_doc = "${script_rootdir}\\$conf_file->{maindir}->{pp_doc}";

if (!$conf_file->{maindir}->{pp_lib})
{
	LOG_MSG($lglvl,3,"PARAM","\"[maindir] pp_lib\" is not defined correctly.");
	LOG_MSG($lglvl,3,"PARAM","Please check your config \"$flag_config\".");
	LOG_MSG($lglvl,3,"GENERAL","Finishing Script $0");
	close LOG;
    exit;
}
my $scriptdir_lib = "${script_rootdir}\\$conf_file->{maindir}->{pp_lib}";

if (!$conf_file->{maindir}->{pp_spool})
{
	LOG_MSG($lglvl,3,"PARAM","\"[maindir] pp_spool\" is not defined correctly.");
	LOG_MSG($lglvl,3,"PARAM","Please check your config \"$flag_config\".");
	LOG_MSG($lglvl,3,"GENERAL","Finishing Script $0");
	close LOG;
    exit;
}
my $scriptdir_spool = "${script_rootdir}\\$conf_file->{maindir}->{pp_spool}";

unless ((-d $scriptdir_bin) and (-d $scriptdir_data) and (-d $scriptdir_etc) and (-d $scriptdir_doc) and (-d $scriptdir_lib) and (-d $scriptdir_spool) and (-d $scriptdir_log))
{
	LOG_MSG($lglvl,3,"PARAM","Any of the following directories does not exist...");
	LOG_MSG($lglvl,3,"PARAM","\t$scriptdir_bin");
	LOG_MSG($lglvl,3,"PARAM","\t$scriptdir_data");
	LOG_MSG($lglvl,3,"PARAM","\t$scriptdir_etc");
	LOG_MSG($lglvl,3,"PARAM","\t$scriptdir_doc");
	LOG_MSG($lglvl,3,"PARAM","\t$scriptdir_lib");
	LOG_MSG($lglvl,3,"PARAM","\t$scriptdir_log");
	LOG_MSG($lglvl,3,"PARAM","\t$scriptdir_spool");
	LOG_MSG($lglvl,3,"PARAM","\tPlease check your directories and/or [maindir] section in config \"$flag_config\".");
	LOG_MSG($lglvl,3,"GENERAL","\tFinishing Script $0");
	close LOG;
    exit;
}


if (lc $conf_file->{main}->{pp_mode} ne "windows")
{
	LOG_MSG($lglvl,3,"PARAM","\"[main] pp_mode\" is not supported. Only windows."); 
	LOG_MSG($lglvl,3,"PARAM","Please check your config \"$flag_config\".");
	LOG_MSG($lglvl,3,"GENERAL","Finishing Script $0");
	close LOG;
    exit;
}
else
{
	if ($Config{osname}=~/Win/)
	{
		$flag_win = 1;
		LOG_MSG($lglvl,5,"PARAM","Mode: $conf_file->{main}->{pp_mode}");
	}
	else
	{
		LOG_MSG($lglvl,3,"PARAM","It looks like your OS it not Windows \"$Config{osname}\". Only windows is not supported."); 
		LOG_MSG($lglvl,3,"GENERAL","Finishing Script $0");
		close LOG;
		exit;
	}
}

if (lc $conf_file->{monitor}->{ppmonitor_switch} eq "on")
{
		LOG_MSG($lglvl,5,"PARAM","\"[monitor] ppmonitor_switch\" = \"$conf_file->{monitor}->{ppmonitor_switch}\". Good."); 

}

elsif (lc $conf_file->{monitor}->{ppmonitor_switch} eq "test")
{
	LOG_MSG($lglvl,5,"PARAM","\"[monitor] ppmonitor_switch\" = \"$conf_file->{monitor}->{ppmonitor_switch}\". Testflag set."); 
	$flag_testemail = 1;
}

elsif (lc $conf_file->{monitor}->{ppmonitor_switch} eq "off")
{
	LOG_MSG($lglvl,3,"PARAM","\"[monitor] ppmonitor_switch\" = \"$conf_file->{monitor}->{ppmonitor_switch}\". As is set to \"off\" the scripts stops now."); 
	LOG_MSG($lglvl,3,"GENERAL","Finishing Script $0");
	close LOG;
	exit;
}

else
{
	LOG_MSG($lglvl,3,"PARAM","\"[monitor] ppmonitor_switch\" is not supported. Only on, off and test."); 
	LOG_MSG($lglvl,3,"GENERAL","Finishing Script $0");
	close LOG;
	USAGE();
	exit;
}

if (!$conf_file->{monitor}->{ppmonitor_mailrelay})
{
	LOG_MSG($lglvl,3,"PARAM","\"[monitor] ppmonitor_mailrelay\" is not defined correctly."); 
	LOG_MSG($lglvl,3,"PARAM","Please check your config \"$flag_config\".");
	LOG_MSG($lglvl,3,"GENERAL","Finishing Script $0");
	close LOG;
	exit;
}
$flag_mailrelay = $conf_file->{monitor}->{ppmonitor_mailrelay};

if (!$conf_file->{monitor}->{ppmonitor_emailrecipient})
{
	LOG_MSG($lglvl,3,"PARAM","\"[monitor] ppmonitor_mailrelay\" is not defined correctly."); 
	LOG_MSG($lglvl,3,"PARAM","Please check your config \"$flag_config\".");
	LOG_MSG($lglvl,3,"GENERAL","Finishing Script $0");
	close LOG;
	exit;
}

@flag_email = "$conf_file->{monitor}->{ppmonitor_emailrecipient}";
# If add -e adress1,adress2,adress3
@flag_email = split(/,/,join(',',@flag_email));

if (!$conf_file->{monitor}->{ppmonitor_emailsender})
{
	LOG_MSG($lglvl,3,"PARAM","\"[monitor] ppmonitor_emailsender\" is not defined correctly."); 
	LOG_MSG($lglvl,3,"PARAM","Please check your config \"$flag_config\".");
	LOG_MSG($lglvl,3,"GENERAL","Finishing Script $0");
	close LOG;
	exit;
}
$flag_emailfrom = $conf_file->{monitor}->{ppmonitor_emailsender};

if ($conf_file->{monitor}->{ppmonitor_runsperday}  !~ /^\d+$/)
{
	LOG_MSG($lglvl,3,"PARAM","\"[monitor] ppmonitor_runsperday\" is not defined correctly."); 
	LOG_MSG($lglvl,3,"PARAM","Please check your config \"$flag_config\".");
	LOG_MSG($lglvl,3,"GENERAL","Finishing Script $0");
	close LOG;
	exit;
}
$flag_runsperday = $conf_file->{monitor}->{ppmonitor_runsperday};

if (lc $conf_file->{monitor}->{ppmonitor_transfercheck} eq "on")
{
	$flag_ioportaltransfer = 1;
	LOG_MSG($lglvl,5,"PARAM","\"[monitor] ppmonitor_transfercheck\" = \"$conf_file->{monitor}->{ppmonitor_transfercheck}\". Checking pptransfer logs."); 
}

elsif (lc $conf_file->{monitor}->{ppmonitor_transfercheck} eq "off")
{
	LOG_MSG($lglvl,3,"PARAM","\"[monitor] ppmonitor_transfercheck\" = \"$conf_file->{monitor}->{ppmonitor_transfercheck}\". Not checking pptransfer logs."); 
}

else
{
	LOG_MSG($lglvl,3,"PARAM","\"[monitor] ppmonitor_transfercheck\" is not supported. Only on and off"); 
	LOG_MSG($lglvl,3,"GENERAL","Finishing Script $0");
	close LOG;
	USAGE();
	exit;
}

# required for log file deletion

if ($conf_file->{monitor}->{ppmonitor_logkeepdays} !~ /^\d+$/)
{
	LOG_MSG($lglvl,3,"PARAM","\"[monitor] ppmonitor_logkeepdays\" must be set, a whole number and bigger or equal 0"); 
	LOG_MSG($lglvl,3,"PARAM","Please check your config \"$flag_config\".");
	LOG_MSG($lglvl,3,"GENERAL","Finishing Script $0");
	close LOG;
    exit;
}
$flag_logkeepdays = $conf_file->{monitor}->{ppmonitor_logkeepdays};
LOG_MSG($lglvl,5,"PARAM","Logfile keepdays: $flag_logkeepdays");


if (!$conf_file->{zip}->{ppcollect_archivedir})
{
	LOG_MSG($lglvl,3,"PARAM","\"[zip] ppcollect_archivedir\" is not defined correctly."); 
	LOG_MSG($lglvl,3,"PARAM","Please check your config \"$flag_config\".");
	LOG_MSG($lglvl,3,"GENERAL","Finishing Script $0");
	close LOG;
	exit;
}
my $ARCHIVE_DIR = "${scriptdir_data}\\$conf_file->{zip}->{ppcollect_archivedir}${serial}";
my $ARCHIVE_SUFFIX = "tgz,zip";
# Set variables

my $day_scriptstart = strftime("%Y-%m-%d", localtime);
my $CHKLOG_DIR = $scriptdir_log;
my $CHKLOG_FILE_SUFFIX = ".log";
my $CHKLOG_TRANSFER_DIR = $scriptdir_log;
my $CHKLOG_TRANFER_FILE_SUFFIX = ".log";

my $FILESIZE = "524288"; # 1 MByte = 1048576, 0.5 MB = 524288
#
my $SEND_EMAIL = "no";
my $EMAIL_MESSAGE;

foreach (@flag_email)
{
	LOG_MSG("$lglvl",5,"GENERAL","Email adress recipient: $_");
}
LOG_MSG("$lglvl",5,"GENERAL","Email adress sender: ${flag_emailfrom}");
LOG_MSG("$lglvl",5,"GENERAL","The amount if runs per day in scheduler/cronjob: ${flag_runsperday}");


# Set -w when osname is *Win*
if ($Config{osname}=~/Win/)
{
	LOG_MSG("$lglvl",5,"GENERAL","Setting -w automatically due to \$Config\{osname\}: \"$Config{osname}\"");
	$flag_win = 1;
}

if ($flag_win)
{
	if ($flag_testemail)
	{
		LOG_MSG("$lglvl",5,"GENERAL","Testemail flag -t set. Sending Test email");
		my $email_subject= "Test email from ppmonitor (sent from server: ${HOST})";
		my $email_message= "This is a test email from ppmonitor.";
		SEND_EMAIL($flag_serial,$flag_emailfrom,$flag_mailrelay,$flag_runsperday,\@flag_email,$email_subject,$email_message,$lglvl);
	}

	else
	{
		if ($flag_serial ne "NOSERIAL")
		{
			LOG_MSG("$lglvl",5,"GENERAL","Serial $flag_serial found. Doing ppcollect checks now.");
			DO_CHECKS ($flag_serial,$CHKLOG_DIR,$CHKLOG_FILE_SUFFIX,$flag_runsperday,$ARCHIVE_DIR,$ARCHIVE_SUFFIX,$FILESIZE,$day_scriptstart,$lglvl);
		}
		else
		{
			LOG_MSG("$lglvl",5,"GENERAL","Serial $flag_serial found. Doing no ppcollect checks.");
		}
		
		if ($flag_ioportaltransfer)
		{
			DO_CHECKS_TRANSFER ($flag_serial,$CHKLOG_TRANSFER_DIR,$CHKLOG_TRANFER_FILE_SUFFIX,$flag_runsperday,$day_scriptstart,$flag_transfer_base,$lglvl);
		}
	
		if ($SEND_EMAIL eq "yes")
		{
			#print "$EMAIL_MESSAGE";
			LOG_MSG("$lglvl",5,"GENERAL","Email will be send now");
			my $mail_subject = "HDS performance collection or transfer issue for serial ${flag_serial} (sent from server: ${HOST})";
			my $mail_message = "This is an email from ppmonitor. There might be an issue with the HDS performance collection or transfer for ${flag_serial}.\n\n";
			$mail_message = $mail_message . $EMAIL_MESSAGE ;
			LOG_MSG("$lglvl",5,"GENERAL","Message ...");
			LOG_MSG("$lglvl",5,"MAIL","Subject: ${mail_subject}");
			my @log_mail_msg = split(/\n/,$mail_message);
			foreach (@log_mail_msg)
			{
				LOG_MSG("$lglvl",5,"MAIL","Text: $_");
			}
			LOG_MSG("$lglvl",5,"GENERAL","Message end");
			SEND_EMAIL($flag_serial,$flag_emailfrom,$flag_mailrelay,$flag_runsperday,\@flag_email,$mail_subject,$mail_message,$lglvl);			
		}
	
		else	
		{
			LOG_MSG("$lglvl",5,"GENERAL","No email will be send as everything looks good.");
		}
		
	}
	

}

unless ($flag_logkeepdays == 0)
{
	DELETE_OLD_LOGS($flag_logkeepdays,$SCRIPTLOG_DEL_PATH,$lglvl);
}

LOG_MSG("$lglvl",5,"GENERAL","Finishing Script $0");
close LOG;


############ sub routines ###################

sub DELETE_OLD_LOGS
{
	my $par_logkeepdays = $_[0];
	my $par_SCRIPTLOG_DEL_PATH = $_[1];
	my $par_lglvl = $_[2];
	
	my $now = time();       # get current time
	my $AGE = 60*60*24*$par_logkeepdays;  # convert $par_logkeepdays days into seconds
	
	LOG_MSG($par_lglvl,5,"DELOLDLOGS","Start deleting log files older than $par_logkeepdays days");
	
	my @fileliste_logs = <${par_SCRIPTLOG_DEL_PATH}>;
	
	foreach my $filename (@fileliste_logs) 
	{
		# If modified time of the file is older then $AGE = days in seconds
		if ($now-(stat $filename)[9] > $AGE)
		{
			LOG_MSG($par_lglvl,5,"DELOLDLOGS","Found the following log file to delete: $filename");
			unlink $filename or warn LOG_MSG($par_lglvl,4,"DELOLDLOGS","Could not delete $filename: $!");
			LOG_MSG($par_lglvl,5,"DELOLDLOGS","Deleted $filename");
		}
	}
	
	LOG_MSG($par_lglvl,5,"DELOLDLOGS","Deleting log files finished");

}

sub DO_CHECKS_TRANSFER
{
	my $par_flag_serial = $_[0];
	my $par_CHKLOG_TRANSFER_DIR = $_[1];
	my $par_CHKLOG_TRANSFER_FILE_SUFFIX = $_[2];
	my $par_flag_runsperday  = $_[3];
	my $par_day_scriptstart = $_[4];
	my $par_flag_transfer_base = $_[5];
	my $par_lglvl = $_[6];
	my $pptransfer_name = "pptransfer";
	
	LOG_MSG("$par_lglvl",5,"CHKTRANSFER","Looking for pptransfer scriptlog files of today ...");
	LOG_MSG("$par_lglvl",5,"CHKTRANSFER","[${par_day_scriptstart}]: ${par_CHKLOG_TRANSFER_DIR}\\${pptransfer_name}_${par_flag_transfer_base}_${par_day_scriptstart}_*${par_CHKLOG_TRANSFER_FILE_SUFFIX}");
	my @chklog_transfer_list = <${par_CHKLOG_TRANSFER_DIR}\\${pptransfer_name}_${flag_transfer_base}_${par_day_scriptstart}_*${par_CHKLOG_TRANSFER_FILE_SUFFIX}>;

	my $amount_chklog_transfer_list = scalar @chklog_transfer_list;
	if ($par_flag_runsperday == $amount_chklog_transfer_list)
	{
		LOG_MSG("$par_lglvl",5,"CHKTRANSFER","Same amount of pptransfer scriptlog files [${amount_chklog_transfer_list}] as defined with -r [${par_flag_runsperday}]. => good");	
		
		foreach (@chklog_transfer_list)
		{
			my $abs_path_chklog_transfer_file = File::Spec->rel2abs($_);
			LOG_MSG("$par_lglvl",5,"CHKTRANSFER","pptransfer scriptlog files: $abs_path_chklog_transfer_file");
		}
		
		foreach my $file (@chklog_transfer_list)
		{
			my $abs_path_chklog_transfer_file = File::Spec->rel2abs($file);

			my $file_check_exit_string = "bad";
			LOG_MSG("$par_lglvl",5,"CHKTRANSFER","Checking pptransfer scriptlog file: $abs_path_chklog_transfer_file");			
			
			my $file_check_finish_string = "bad";
            
			open(IN,"<$file") or die LOG_MSG("$par_lglvl",3,"CHKTRANSFER","Couldn't open ${$abs_path_chklog_transfer_file}: $!");
			while (my $line = <IN>)
			{

				# Check 2 - Did you get exit code 0 from pscp?
				if ($line=~/Exit code of pscp: 0/)
				{
					LOG_MSG("$par_lglvl",5,"CHKTRANSFER","> exit code pscp 0 found => good");
					$file_check_exit_string = "good";
				}
				
				# Check 3 - Did you get exit code 0 from CURK?
				if ($line=~/Exit code of CURL: 0/)
				{
					LOG_MSG("$par_lglvl",5,"CHKTRANSFER","> exit code CURL 0 found => good");
					$file_check_exit_string = "good";
				}
				
				# Check 4 - Did the scriptlog show finishing script?
				if ($line=~/Finishing Script/)
				{
					LOG_MSG("$par_lglvl",5,"CHKTRANSFER","> finishing found => good");
					$file_check_finish_string = "good";
				}
			}
			close IN;
			
			if ($file_check_exit_string ne "good")
			{

				LOG_MSG("$par_lglvl",5,"CHKTRANSFER","*** exit code pscp or curl 0 not found => bad");
				LOG_MSG("$par_lglvl",5,"CHKTRANSFER","*** Email will be send (but first checking other things).");
				$EMAIL_MESSAGE = $EMAIL_MESSAGE . "Serialnumber: ${par_flag_serial}\nDate: ${par_day_scriptstart}\n";
				$EMAIL_MESSAGE = $EMAIL_MESSAGE . "Issue: Exit code pscp or curl 0 not found. Transfer not successful.\n";
				$EMAIL_MESSAGE = $EMAIL_MESSAGE . "Please check the pptransfer scriptlog file: $abs_path_chklog_transfer_file.\n\n\n";
				$SEND_EMAIL = "yes";
			}
			
			if ($file_check_finish_string ne "good")
			{
				my $abs_path_chklog_transfer_file = File::Spec->rel2abs($file);
				LOG_MSG("$par_lglvl",5,"CHKTRANSFER","*** > finishing not found => bad");
				LOG_MSG("$par_lglvl",5,"CHKTRANSFER","*** Email will be send (but first checking other things).");
				$EMAIL_MESSAGE = $EMAIL_MESSAGE . "Serialnumber: ${par_flag_serial}\nDate: ${par_day_scriptstart}\n";
				$EMAIL_MESSAGE = $EMAIL_MESSAGE . "Issue: pptransfer script did not finish accordingly.\n";
				$EMAIL_MESSAGE = $EMAIL_MESSAGE . "Please check the pptransfer scriptlog file: $abs_path_chklog_transfer_file.\n\n\n";
				$SEND_EMAIL = "yes";
			}	
			
		}
	}
	
	else 
	{
	
		LOG_MSG("$par_lglvl",5,"CHKTRANSFER","*** Not same amount of pptransfer scriptlog files [${amount_chklog_transfer_list}] as defined with -r [${par_flag_runsperday}]. => bad");

		foreach (@chklog_transfer_list)
		{
			my $abs_path_chklog_tranfer_file = File::Spec->rel2abs($_);
			LOG_MSG("$par_lglvl",5,"CHKTRANSFER","*** pptransfer scriptlog files: $abs_path_chklog_tranfer_file");
		}
	
		LOG_MSG("$par_lglvl",5,"CHKTRANSFER","*** Email will be send (but first checking other things).");
		$EMAIL_MESSAGE = $EMAIL_MESSAGE . "Serialnumber: ${par_flag_serial}\nDate: ${par_day_scriptstart}\n";
		$EMAIL_MESSAGE = $EMAIL_MESSAGE . "Issue: Not same amount of pptransfer scriptlog files [${amount_chklog_transfer_list}] in ${par_CHKLOG_TRANSFER_DIR}\n";
		$EMAIL_MESSAGE = $EMAIL_MESSAGE . "as runs defined with -r [${flag_runsperday}].\n";
		foreach (@chklog_transfer_list)
		{
			my $abs_path_chklog_transfer_file = File::Spec->rel2abs($_);
			$EMAIL_MESSAGE = $EMAIL_MESSAGE . "pptransfer scriptlog files found: $abs_path_chklog_transfer_file.\n";
		}
		$EMAIL_MESSAGE = $EMAIL_MESSAGE . "Please check if pptransfer did run.\n\n\n";
		$SEND_EMAIL = "yes";	
	}	
}

sub DO_CHECKS
{
	my $par_flag_serial = $_[0];
	my $par_CHKLOG_DIR = $_[1];
	my $par_CHKLOG_FILE_SUFFIX = $_[2];
	my $par_flag_runsperday  = $_[3];
	my $par_ARCHIVE_DIR = $_[4];
	my $par_ARCHIVE_SUFFIX = $_[5];
	my $par_FILESIZE = $_[6];
	my $par_day_scriptstart = $_[7];
	my $par_lglvl = $_[8];
	my $ppcollect_name = "ppcollect";
	
	# Check 1 - Same amount of scrpiptlogs there as defined in runs per day?
	LOG_MSG("$par_lglvl",5,"CHKCOLLECT","Looking for ppcollect scriptlog files of today ...");
	LOG_MSG("$par_lglvl",5,"CHKCOLLECT","[${par_day_scriptstart}]: ${par_CHKLOG_DIR}\\${ppcollect_name}_*_${par_flag_serial}_${par_day_scriptstart}_*${par_CHKLOG_FILE_SUFFIX}");
	my @chklog_list = <${par_CHKLOG_DIR}\\${ppcollect_name}_*_${flag_serial}_${par_day_scriptstart}_*${par_CHKLOG_FILE_SUFFIX}>;
	# Make a hash with the file names as key and 1 as value. This is to store the corresponding tgz file to the script log later
	my %chklog_list_hash =  map { $_ => 1 } @chklog_list;
	my $amount_chklog_list = scalar @chklog_list;
	if ($par_flag_runsperday == $amount_chklog_list)
	{
		LOG_MSG("$par_lglvl",5,"CHKCOLLECT","Same amount of ppcollect scriptlog files [${amount_chklog_list}] as defined with -r [${par_flag_runsperday}]. => good");
		
		foreach (@chklog_list)
		{
			my $abs_path_chklog_file = File::Spec->rel2abs($_);
			LOG_MSG("$par_lglvl",5,"CHKCOLLECT","ppcollect scriptlog files: $abs_path_chklog_file");
		}
			
		foreach my $file (@chklog_list)
		{
			my $abs_path_chklog_file = File::Spec->rel2abs($file);
			my $abs_path_archiv_dir = File::Spec->rel2abs($par_ARCHIVE_DIR);
			# This is needed for the reg exp below
			$abs_path_archiv_dir =~ s#\\#\\\\#g;
			my $file_check_archiv_string = "bad";
			LOG_MSG("$par_lglvl",5,"CHKCOLLECT","Checking ppcollect scriptlog file: $abs_path_chklog_file");
			LOG_MSG("$par_lglvl",5,"CHKCOLLECT","Check for ${abs_path_archiv_dir}.* $par_ARCHIVE_SUFFIX"); 
			
			
			my $file_check_finish_string = "bad";
            
			open(IN,"<$file") or die "Couldn't open ${$abs_path_chklog_file}: $!";
			while (my $line = <IN>)
			{

				# Check 2 - Did the scriptlog show start archiving?
				# Split suffixes and get files
				my @suffixsplit = split(',', $par_ARCHIVE_SUFFIX);
				foreach my $suffix (@suffixsplit)
				{
					$suffix =~ s/\s+//g;
					if ($line=~/Start Archiving to "(${abs_path_archiv_dir}.*${suffix})"/)
					{
						my $compressed_file = $1;
						LOG_MSG("$par_lglvl",5,"CHKCOLLECT","Archiving start found of $compressed_file => good");
						$chklog_list_hash{$file} = $compressed_file;
						$file_check_archiv_string = "good";
					}
				}

				
				# Check 3 - Did the scriptlog show finishing script?
				if ($line=~/Finishing Script/)
				{
					LOG_MSG("$par_lglvl",5,"CHKCOLLECT","Finishing found => good");
					$file_check_finish_string = "good";
				}
			}
			close IN;
			
			if ($file_check_archiv_string eq "good")
			{
				# Check 4 - Does the archive exist?
				if (-e $chklog_list_hash{$file})
				{
					LOG_MSG("$par_lglvl",5,"CHKCOLLECT","File $chklog_list_hash{$file} exists => good");
				
					# Get file size in KB
					my $filesize = -s $chklog_list_hash{$file};
					# Check 5 - Is the file size bigger than FILESIZE)?
					if ($filesize > $par_FILESIZE)
					{
						LOG_MSG("$par_lglvl",5,"CHKCOLLECT","File size [${filesize} KB] is bigger than definied value [${par_FILESIZE} KB] => good");
					}
				
					else
					{
						my $abs_path_archiv_file = File::Spec->rel2abs($chklog_list_hash{$file});
						LOG_MSG("$par_lglvl",5,"CHKCOLLECT","*** > file size [${filesize} KB] is smaller than definied value [${par_FILESIZE} KB] => bad");
						LOG_MSG("$par_lglvl",5,"CHKCOLLECT","*** Email will be send (but first checking other things).");
						$EMAIL_MESSAGE = $EMAIL_MESSAGE . "Serialnumber: ${par_flag_serial}\nDate: ${par_day_scriptstart}\n";
						$EMAIL_MESSAGE = $EMAIL_MESSAGE . "Issue: Size of performance file $abs_path_archiv_file is too small.\n";
						$EMAIL_MESSAGE = $EMAIL_MESSAGE . "File size [${filesize} KB] is smaller than definied value [$par_FILESIZE KB].\n";
						$EMAIL_MESSAGE = $EMAIL_MESSAGE . "Please check the ppcollect scriptlog file: $abs_path_chklog_file.\n";
						$EMAIL_MESSAGE = $EMAIL_MESSAGE . "Please check the performance file: $abs_path_archiv_file.\n";
						$EMAIL_MESSAGE = $EMAIL_MESSAGE . "Maybe there is an issue with the command txt file?\n\n\n";
						$SEND_EMAIL = "yes";
					}
					
				}
			
				else
				{
					my $abs_path_archiv_file = File::Spec->rel2abs($chklog_list_hash{$file});
					LOG_MSG("$par_lglvl",5,"CHKCOLLECT","*** > Performance file $abs_path_archiv_file does not exist => bad");
					LOG_MSG("$par_lglvl",5,"CHKCOLLECT","*** Email will be send (but first checking other things).");
					$EMAIL_MESSAGE = $EMAIL_MESSAGE . "Serialnumber: ${par_flag_serial}\nDate: ${par_day_scriptstart}\n";
					$EMAIL_MESSAGE = $EMAIL_MESSAGE . "Issue: Performance file $abs_path_archiv_file does not exist.\n";
					$EMAIL_MESSAGE = $EMAIL_MESSAGE . "See ppcollect scriptlog file: $abs_path_chklog_file.\n\n\n";
					$SEND_EMAIL = "yes";
				}
			}
			
			else
			{
				LOG_MSG("$par_lglvl",5,"CHKCOLLECT","*** > archiving start not found => bad");
				LOG_MSG("$par_lglvl",5,"CHKCOLLECT","*** Email will be send (but first checking other things).");
				$EMAIL_MESSAGE = $EMAIL_MESSAGE . "Serialnumber: ${par_flag_serial}\nDate: ${par_day_scriptstart}\n";
				$EMAIL_MESSAGE = $EMAIL_MESSAGE . "Issue: Performance file did not generated.\n";
				$EMAIL_MESSAGE = $EMAIL_MESSAGE . "Please check the ppcollect scriptlog file: $abs_path_chklog_file.\n\n\n";
				$SEND_EMAIL = "yes";
			}
			
			if ($file_check_finish_string ne "good")
			{
				my $abs_path_chklog_file = File::Spec->rel2abs($file);
				LOG_MSG("$par_lglvl",5,"CHKCOLLECT","*** > finishing not found => bad");
				LOG_MSG("$par_lglvl",5,"CHKCOLLECT","*** Email will be send (but first checking other things).");
				$EMAIL_MESSAGE = $EMAIL_MESSAGE . "Serialnumber: ${par_flag_serial}\nDate: ${par_day_scriptstart}\n";
				$EMAIL_MESSAGE = $EMAIL_MESSAGE . "Issue: ppcollect script did not finish accordingly.\n";
				$EMAIL_MESSAGE = $EMAIL_MESSAGE . "Please check the ppcollect scriptlog file: $abs_path_chklog_file.\n\n\n";
				$SEND_EMAIL = "yes";
			}			
		}
	}
	
	else 
	{
	
		LOG_MSG("$par_lglvl",5,"CHKCOLLECT","*** Not same amount of ppcollect scriptlog files [${amount_chklog_list}] as defined with -r [${par_flag_runsperday}]. => bad");

		foreach (@chklog_list)
		{
			my $abs_path_chklog_file = File::Spec->rel2abs($_);
			LOG_MSG("$par_lglvl",5,"CHKCOLLECT","*** Scriptlog files: $abs_path_chklog_file");
		}
	
		LOG_MSG("$par_lglvl",5,"CHKCOLLECT","*** Email will be send (but first checking other things).");
		$EMAIL_MESSAGE = $EMAIL_MESSAGE . "Serialnumber: ${par_flag_serial}\nDate: ${par_day_scriptstart}\n";
		$EMAIL_MESSAGE = $EMAIL_MESSAGE . "Issue: Not same amount of ppcollect scriptlog files [${amount_chklog_list}] in ${par_CHKLOG_DIR}\n";
		$EMAIL_MESSAGE = $EMAIL_MESSAGE . "as runs defined with -r [${flag_runsperday}].\n";
		foreach (@chklog_list)
		{
			my $abs_path_chklog_file = File::Spec->rel2abs($_);
			$EMAIL_MESSAGE = $EMAIL_MESSAGE . "Scriptlog files found: $abs_path_chklog_file.\n";
		}
		$EMAIL_MESSAGE = $EMAIL_MESSAGE . "Please check if ppcollect did run and performance files got collected.\n";
		$EMAIL_MESSAGE = $EMAIL_MESSAGE . "Maybe there is a task scheduler issue?\n\n\n";
		$SEND_EMAIL = "yes";	
	}		
}

sub SEND_EMAIL
{

	my $par_serial = $_[0];
	my $par_emailfrom = $_[1];
	my $par_mailrelay = $_[2];
	my $par_runsperday  = $_[3];
	my @par_flag_email = @{$_[4]};
	my $par_subject  = $_[5];
	my $par_message  = $_[6];
	my $par_lglvl = $_[7];
	
	my $smtp;
	LOG_MSG("$par_lglvl",5,"SENDEMAIL","Net::SMTP start");
	if ($par_lglvl eq "debug")
	{
		$smtp = Net::SMTP->new(Host => $par_mailrelay, Timeout => 30, Debug => 1,) or die LOG_MSG("$par_lglvl",3,"SENDMAIL","Can't connect to net::smtp mail server [$par_mailrelay]: $!\n");
	}
	else
	{
		$smtp = Net::SMTP->new(Host => $par_mailrelay, Timeout => 30, Debug => 0,) or die LOG_MSG("$par_lglvl",3,"SENDMAIL","Can't connect to net::smtp mail server [$par_mailrelay]: $!\n");
	}
	$smtp->mail($par_emailfrom) or die LOG_MSG("$par_lglvl",3,"SENDMAIL","Check flag -f [$par_emailfrom] => net::smtp \$smtp->mail");
	foreach (<@par_flag_email>) { $smtp->to($_) or die LOG_MSG("$par_lglvl",3,"SENDMAIL","Check flag -e [@par_flag_email]=> net::smtp \$smtp->to"); }
	$smtp->data();
	foreach (<@par_flag_email>) { $smtp->datasend("To: $_\n"); }
	$smtp->datasend("From: $par_emailfrom\n");
	$smtp->datasend("Subject: $par_subject\n");
	$smtp->datasend("\n");
	$smtp->datasend($par_message);
	$smtp->dataend();
	$smtp->quit;
	LOG_MSG("$par_lglvl",5,"SENDEMAIL","Net::SMTP finished");
}

sub LOG_MSG
{
	my $par_LEVEL = shift (@_);
	my $par_SEVERITY = shift (@_);
	my $par_FUNCTION = shift (@_);
	my @line =  @_;
	my $logdate = strftime("%Y.%m.%d", localtime);
	my $logtime = strftime("%H:%M:%S", localtime);
	
	my $SEV_KEYWORD;
	chomp (@line);
	if ($par_SEVERITY == 3) {$SEV_KEYWORD = "ERROR";}
	if ($par_SEVERITY == 4) {$SEV_KEYWORD = "WARNING";}
	if ($par_SEVERITY == 5) {$SEV_KEYWORD = "NOTICE";}
	if ($par_SEVERITY == 7) {$SEV_KEYWORD = "DEBUG";}
	
	if ($par_LEVEL eq "normal")
	{
		unless ($par_SEVERITY eq "DEBUG")
		{
			print "> $logdate,$logtime,$SEV_KEYWORD,$par_FUNCTION,@line\n";
			print LOG "$logdate,$logtime,$SEV_KEYWORD,$par_FUNCTION,@line\n";
		}
	}
	else
	{
		print "> $logdate,$logtime,$SEV_KEYWORD,$par_FUNCTION,@line\n";
		print LOG "$logdate,$logtime,$SEV_KEYWORD,$par_FUNCTION,@line\n";
	}
}


sub USAGE
{
    	my ($message)=<<'MESSAGE';

NAME ppmonitor.pl

SYNOPSIS
	ppmonitor.pl -c <config file>
	ppmonitor.pl [-h] [-v]
	 
DESCRIPTION
	This script checks if the performance collection with ppcollect (and if transfer with pptransfer)
	was successful. If not it will send an email.
	The script must be run after the daily performance collection runs.

REQUIREMENT
	> Install perl (tested with 5.10 and above)
	> Run this script with appropriate privileges! If you are not sure use root or administrator.
	 
OPTIONS

	-c <configuration file>
		Path to configuration file
		Please see the manual (http://loop.hds.com/docs/DOC-45708) for detailed information about configuration settings.

	-h
		Show help message 

	-v
		Displays the version of the script

FILES
	$SCRIPT_HOME/ppmonitor.pl
	 
EXAMPLES
	ppmonitor.pl -h
	ppmonitor.pl -v
	ppmonitor.pl -c pp_123456.conf

MESSAGE

    	print "Version:${VERSION}\n$message";
}