#!/usr/bin/perl
use Expect;

# if syslog happens to be available...
#use Sys::Syslog qw( :DEFAULT setlogsock);
#setlogsock('unix');
#openlog('healthcheck', '','local2');

my $logfile = "/opt/meru/health.log";

$rx_errs_holddown = 0;
$tx_errs_holddown = 0;
$log_size_holddown = 0;

sub generate_dump {

    print OUTLOG "\n\n************* Generating full log dump ************** \n\n";

    $command = Expect->spawn ("cat /proc/meru/kcomm/mailboxes");
    $command->log_stdout(0);
    $command->expect (10, "#"); 
    
    print OUTLOG "\ncat /proc/meru/kcomm/mailboxes\n";
    print OUTLOG $command->before();

    $command = Expect->spawn ("/bin/df");
    $command->log_stdout(0);
    $command->expect (10, "#"); 

    print OUTLOG "\n/bin/df\n\n";
    print OUTLOG $command->before();

    $command = Expect->spawn ("dmesg");
    $command->log_stdout(0);
    $command->expect (10, "\# "); 

    print OUTLOG "\ndmesg\n\n";
    print OUTLOG $command->before();

    $command = Expect->spawn ("top -bn1");
    $command->log_stdout(0);
    $command->expect (10, "\# "); 
    
    print OUTLOG "\ntop -bn1\n\n";
    print OUTLOG $command->before();

}

while () {
    open(OUTLOG, ">>$logfile");
    my $time = localtime();
    
############# Check for mailbox errors ###################################

    $command = Expect->spawn ("cat /proc/meru/kcomm/mailboxes");
    $command->log_stdout(0);
    $command->expect (10, "#"); 
    
    $out = $command->before();
    @lines = split (/[\n]/, $out);
    
    for ($i = 3; $i <= $#lines ; ++$i)
    {
        $line = $lines[$i];
        @sline = split (/\s+/, $line);
        
        
        $mbox      =  @sline[3];
        $rx        =  @sline[4];
        $rx_errs   =  @sline[5];
        $tx        =  @sline[6];
        $tx_errs   =  @sline[7];
        $mbox_name =  @sline[12];
        
        if ($rx_errs > 10 && $rx_errs_holddown == 0)
        {
            $ratio = (($rx_errs+0.000001) / ($rx+0.0000001)) * 100.0; 
            if ($ratio > 5.0)
            {
                $rounded = sprintf("%2.1f", $ratio);  
                $warn_string = "\[$time\] WARNING: Mailbox $mbox($mbox_name) has $rx_errs RX errors from $rx pkts ($rounded %)\n";
                #syslog('err', $warn_string);
                print OUTLOG "At time $time\n";
                print OUTLOG $warn_string;
                generate_dump();
                $rx_errs_holddown = 1;
            }
        }
        if ($tx_errs > 10 && $tx_errs_holddown == 0)
        {
            $ratio = (($tx_errs+0.000001) / ($tx+0.0000001)) * 100.0;  
            if ($ratio > 5.0 )
            {
                $rounded = sprintf("%2.1f", $ratio);  
                $warn_string = "\[$time\] WARNING: Mailbox $mbox($mbox_name) has $tx_errs TX errors from $tx pkts ($rounded %)\n";
                #syslog('err', $warn_string);
                print OUTLOG "At time $time\n";
                print OUTLOG $warn_string;
                generate_dump();
                $tx_errs_holddown = 1;
            }
        }
    }
    
############# Check for log partition free space #########################

    $command = Expect->spawn ("/bin/df");
    $command->log_stdout(0);
    $command->expect (10, "#"); 
    
    $out = $command->before();
    @lines = split (/ /, $out);
    
    $idx =0;
    
    foreach $entry (@lines)
    {
        if ($entry=~/\/opt\/meru\/var\/log/)
        {
            $val = @lines[$idx-1];
            if ( $val > 90 && $log_size_holddown == 0)
            {
                $warn_string = "\[$time\] WARNING: /opt/meru/var/log partition exceeds 90%\n";
                print OUTLOG "At time $time:\n";
                print OUTLOG $warn_string;
                #syslog('err', $warn_string);
                $log_size_holddown = 1; # wait until this gets cleaned up. 
            }
            elsif ( $val < 50 && $log_size_holddown == 1)
            {
                $log_size_holddown = 0; # ready to flag this again.  
            }
            last;
        }
        
        $idx++;
    }

############# Check for CPU utilization ##################################

    $command = Expect->spawn ("top -bn1");
    $command->log_stdout(0);
    $command->expect (10, "\# "); 
    
    $out = $command->before();
    @lines = split (/[\n]/, $out);
    
    $line = $lines[2];
    @sline = split (/\s+/, $line);
    
    $cpu_user = @sline[4];
    $cpu_sys  = @sline[6];

#printf "\nidle: %3d \n", $cpu_idle;

    if ($cpu_user + $cpu_sys > 70) {
        $warn_string = "\[$time\] WARNING: cpu idle is less than 30% ($cpu_idle)\n";
        #syslog('err', $warn_string);
        print OUTLOG "At time $time:\n";
        print OUTLOG $warn_string;
        print OUTLOG $out;
        generate_dump();
    }

############# Check for discovery keepalives missed ######################

    $command = Expect->spawn ("tail -n 30 /var/log/discovery\.log");
    $command->log_stdout(0);
    $command->expect (10, "\# "); 

    $out = $command->before();
    @lines = split (/[\n]/, $out);

    foreach $entry (@lines)
    {
        if ($entry=~/missed 5 keepalive/)
        {
            print OUTLOG "At time $time:\n";
            print OUTLOG "WARNING: discovery missed 5 keepalives.\n";
            print OUTLOG $out;
            generate_dump();
            system ("cp /var/log/discovery.log /opt/meru/discovery_log_copy.log");
            last;
        }
    }

############# Check for new dmesg output #################################

    $command = Expect->spawn ("dmesg -c");
    $command->log_stdout(0);
    $command->expect (10, "\# "); 

    $out = $command->before();
    @lines = split (/[\n]/, $out);

    if (@lines > 0)
    {
        print OUTLOG "At time $time:\n";
        print OUTLOG "New dmesg output:\n\n";
        print OUTLOG $out;
    }

    #closelog();
    close OUTLOG;
    sleep 10;

}


