#!/usr/bin/perl

use strict;
use warnings;
use POSIX qw(setsid);
 
my $do_stop             = 0;
my $pid                 = $$;
my $sysb_version        = "6.52";
my $min_freespace_kbyte     = 8000000;
my $min_freespace_var_kbyte = 4000000;
my $sb_date             = datefmt2();
my $webservice_dir      = '/opt/webservice';
my $soco2_data_dir      = '/var/opt/soco2';
my $webservice_data_dir = '/var/opt/webservice';
my $ha_data_dir         = '/var/opt/ha';
my $sg_data_dir         = '/var/opt/soco';
my $sg_initcfg_file     = $sg_data_dir.'/config/initialcfg.xml';
my $log_file_after      = '/var/log/webservice/sysbackup.log';
my $sysinfo_data_dir    = '/var/opt/sysinfo';
my $sysinfo_log         = $sysinfo_data_dir.'/input/sysbackup_info.xml';
my $self                = $webservice_dir.'/scripts/sysbackup';
my $lock_file           = $webservice_data_dir.'/sysbackup.lock';
my $assistant_sws_area     = '/var/opt/Assistant/data/SWS';
my $assistant_version_file = $assistant_sws_area.'/versioninfo.txt';
my $sb_lock_file        = '/var/opt/Assistant/data/sysbackup_running';
my $xml_file            = $webservice_data_dir.'/sysbackupd.xml';
my $sysbackup_dir       = '/var/opt/sysbackup';
my $sysbackup_dir_to_delete = "";
my $work_lock_file      = $sysbackup_dir.'/sysbackup.lock';
my $log_file            = $sysbackup_dir.'/sysbackup.log';
my $log_file_poll       = '/var/log/webservice/sysbackup_polling.log';
my $backup_dir          = $soco2_data_dir.'/installrepo';
my $osdeps_backup_dir   = $soco2_data_dir.'/os_deps';
my $osupdate_backup_dir = $soco2_data_dir.'/os_update';
my $cla_license_dir     = '/opt/cla/license';
my $cur_hf_dir          = $soco2_data_dir.'/current_hotfix';
my $cur_hf_dir_old      = $webservice_data_dir.'/current_hotfix';
my $cur_hf_rpm_dir      = $soco2_data_dir.'/current_hotfix_rpms';
my $sysbackup_storedir  = $sysbackup_dir;
my $dest_sysbackup      = "sysbackup_$sb_date";
my $cur_sysbackup       = "in_work";
my $reduce_prio_script  = $webservice_dir.'/scripts/re_prio.sh';
my $get_version_script  = $webservice_dir.'/scripts/getVersionInfoFromVMs.sh';
my $is_duplex_script    = $webservice_dir.'/scripts/is_duplex';
my $getStandby_script   = $webservice_dir.'/scripts/getStandbyHostname.sh';
my $getx_script         = $webservice_dir.'/scripts/getx';
my $get_deployment_string_script = $webservice_dir.'/scripts/get_deployment_as_string.sh';
my $is_hpa_script                = $webservice_dir.'/scripts/is_hpa';
my $status_file         = "$sysbackup_dir/$cur_sysbackup/status.xml";
my $deploy_file         = "$sysbackup_dir/$cur_sysbackup/deployments.xml";
my $java_dir            = "/usr/java/os4k";
my $max_svc_age         = 1 / 24 * 2; # in days (here 2 hours)
my $global_rc           = 0;
my $percent             = 0;
my $total               = 0;
my $current             = 0;
my $name                = "";
my $cp_installrepo      = 1;
my $do_ionice           = 1;
my $do_rsync            = 1;
my $is_duplex           = 0;
my $do_delete           = 0;
my $adp_switchover_has_happened = 0;
my $ionice_prefix       = "/usr/bin/ionice -c3 ";
my ($sysinfo_sec,$sysinfo_min,$sysinfo_hour,$sysinfo_mday,$sysinfo_mon,$sysinfo_year, $sysinfo_wday,$sysinfo_yday,$sysinfo_iddst) = (0,0,0,0,0,0,0,0,0);
my ($multiNodeType, $other_node, $node_q, $isMultiNode) = ("", "", "", 0);

# default values which are overwritten later on by yaps defintion files
my @packages_which_are_not_to_repackage = ("cla", "flashrom", "hpacucli");
my @packages_which_have_no_SEN_tag      = ("soco2_jre", "flashrom", "hpacucli");
my @packages_which_are_allowed_to_fail  = ("flashrom");

if(!defined($ARGV[0])) {
  print "Usage: $0 start [-no_ionice] [-no_sync] [-no_cp_installrepo] [<sysbackup_storedir>]|stop|status|delete [<dir>]\n";
  exit(1);
}

my $hostname = "";

if(defined $ARGV[0] && index($ARGV[0], "webservice_startup") >= 0) {

  $hostname = exec_cmd_pipe("/bin/hostname");
  chomp($hostname);  
  
  log_msg("\n \n \n****************************************************************");
  log_msg("sysbackup call: $0 @ARGV");
  log_msg("sysbackup version : $sysb_version");
  log_msg("running on : $hostname");
  log_msg("");
  
  # check log file size and truncate if too big
  check_log_file($log_file);
  
  log_msg("webservice_startup command received");
  my $rc = 0;

  if(get_running_pid("nopoll") > 0 && exec_cmd("/sbin/checkproc -p $lock_file $self") == 0) {
    log_msg("system backup currently running, no Linux startup cleanup action performed");
  } 
  else
  {
    log_msg("system backup currently not running -> checking if $sysbackup_dir is empty");
    if ( -e $work_lock_file ) {
      log_msg("file $work_lock_file exists");	  
      log_msg("system backup directory $sysbackup_dir is NOT empty during a webservice startup -> set the status to fail");
      log_msg_info("system backup failed probably due to Linux reboot or power off.");
      
      $name = "Linux reboot or power off";
      write_status("failed");
      
      $is_duplex = exec_cmd("$is_duplex_script");
      if ( $is_duplex == 1 ) {
        log_msg_info("*** running on a duplex system ***");
      }
      
      prepare_store_sysbackup($sysbackup_dir);
      store_sysbackup($sysbackup_dir, 0);
      store_statusfile();

      rename_storagedir_to_final_name($sysbackup_storedir);  
      
      log_msg_info("system backup failed probably due to Linux reboot or power off.");

      # remove the locks. it could be that the assistant lock could not be deleted before the previous reboot
      remove_lock_file();
      
      store_log_file();
    }
    exec_cmd("/bin/rm -vf $sysinfo_log");
  }

  log_msg("webservice_startup command finished rc=$rc");
  exit($rc);
}

if(defined $ARGV[0] && index($ARGV[0], "delete") >= 0) {
  $hostname = exec_cmd_pipe("/bin/hostname");
  chomp($hostname);
  
  log_msg_info("\n \n \n****************************************************************");
  log_msg_info("sysbackup call: $0 @ARGV");
  log_msg_info("sysbackup version : $sysb_version");
  log_msg_info("running on : $hostname");
  log_msg_info("");  

 # check log file size and truncate if too big
  check_log_file($log_file);
  
  log_msg_info("delete command received");
  if(get_running_pid("nopoll") > 0 ) {
    log_msg("got an already running sysbackup pid");
    if (exec_cmd("/sbin/checkproc -p $lock_file $self") == 0) {
      log_msg_info("sysbackup daemon already running, deleting currently not possible");
      print "sysbackup daemon already running, deleting currently not possible\n";
      exit(1);
    } else {
      log_msg_info("the older sysbackup pid found in $lock_file is no longer attached to a running process -> continuing");
    }
  } else {
    log_msg("did not get an already running sysbackup pid");
  }
  my $i = 1;
  
  if(defined $ARGV[$i]) {
    $sysbackup_dir_to_delete=$ARGV[$i];
    log_msg_info("deleting system backup: $sysbackup_dir_to_delete"); 
    $do_delete = 1;
  } else {
    log_msg_info("Please enter an existing system backup dir which should be deleted!");
    print "Please enter an existing system backup dir which should be deleted!\n";
    exit(1);
  }
}

if(defined $ARGV[0] && index($ARGV[0], "start") >= 0) {
  $hostname = exec_cmd_pipe("/bin/hostname");
  chomp($hostname);
  
  log_msg_info("\n \n \n****************************************************************");
  log_msg_info("sysbackup call: $0 @ARGV");
  log_msg_info("sysbackup version : $sysb_version");
  log_msg_info("running on : $hostname");
  log_msg_info("");  

 # check log file size and truncate if too big
  check_log_file($log_file);
  
  log_msg_info("start command received");
  if(get_running_pid("nopoll") > 0 ) {
    log_msg("got an already running sysbackup pid");
    if (exec_cmd("/sbin/checkproc -p $lock_file $self") == 0) {
      log_msg_info("sysbackup daemon already running, exiting");
      print "sysbackup daemon already running, exiting\n";
      exit(1);
    } else {
      log_msg_info("the older sysbackup pid found in $lock_file is no longer attached to a running process -> continuing");
    }
  } else {
    log_msg("did not get an already running sysbackup pid");
  }
  my $i = 1;
  
  if(defined $ARGV[$i] && index($ARGV[$i], "-no_ionice") >= 0) {
    log_msg_info("not using ionice cmd ($ionice_prefix) for copy files and build rpms");
    $i++;
    $ionice_prefix = "";
    $do_ionice = 0;
  }
  if(defined $ARGV[$i] && index($ARGV[$i], "-no_cp_installrepo") >= 0) {
    log_msg_info("not copying the install repo to system back up");
    $i++;
    $cp_installrepo = 0;
  }
  if(defined $ARGV[$i] && index($ARGV[$i], "-no_sync") >= 0) {
    log_msg_info("not using rsync on duplex");
    $i++;
    $do_rsync = 0;
  }
  if(defined $ARGV[$i]) {
    $sysbackup_storedir=$ARGV[$i];
    log_msg_info("intended system backup storage dir given: $sysbackup_storedir"); 
    log_msg_info("Therefore using $sysbackup_storedir instead of default $sysbackup_dir"); 
    
    $status_file   = "$sysbackup_storedir/$cur_sysbackup/status.xml";
    $deploy_file   = "$sysbackup_storedir/$cur_sysbackup/deployments.xml";    
  }
}

if(defined $ARGV[0] && index($ARGV[0], "stop") >= 0) {
  my $rc = 1;
  my $sb_pid = get_running_pid("nopoll");
  log_msg_info("stop command received");
  if($sb_pid > 0 && kill("HUP", $sb_pid)) {
    $rc = 0;
  } else {
    print "no running sysbackup daemon found\n";
  }
  log_msg_info("stopping finished rc=$rc");
  exit($rc);
}

if(defined $ARGV[0] && index($ARGV[0], "status") >= 0) {
  # check log file size and truncate if too big
  check_log_file($log_file_poll);
  
  my $rc = 1;
  my $max_retries = 10;
  my $count = 0;
  log_msg("status command received", "poll");
  if(get_running_pid("poll") > 0 && exec_cmd_silent("/sbin/checkproc -p $lock_file $self") == 0) {
    while(! -e $xml_file && $count < $max_retries) {
      sleep(3);
      $count++;
    }
    if(-e $xml_file && open(XML, "<$xml_file")) {
      my $xml;
      read(XML, $xml, -s $xml_file);
      print $xml;
      log_msg("xml:\n".$xml,"poll");
      $rc = 0;
    }
  }
  log_msg("status retrieval finished rc=$rc", "poll");
  exit($rc);
}

#if(is_service_running('hbr') || is_service_running('swa')) { exit(1); }

chdir '/'                or die "Can't chdir to /: $!";
umask 0;
open STDIN,  '/dev/null' or die "Can't read /dev/null: $!";
open STDOUT, '/dev/null' or die "Can't write to /dev/null: $!";
open STDERR, '/dev/null' or die "Can't write to /dev/null: $!";
defined($pid = fork)     or die "Can't fork: $!";
if($pid) {
  $pid = $$;
  log_msg("exiting parent process (daemonizing)");
  exit(0);
} else {
  sleep(1);
  $pid = $$;
  log_msg("proceeding child process (daemonized)");
}  
setsid                   or die "Can't start a new session: $!";

$SIG{'INT' } = 'interrupt';
$SIG{'HUP' } = 'stop';
$SIG{'ABRT'} = 'interrupt';
$SIG{'QUIT'} = 'interrupt';
$SIG{'TRAP'} = 'interrupt';
$SIG{'STOP'} = 'interrupt';
$SIG{'TERM'} = 'interrupt';

if(open(PID, ">$lock_file")) {
  protectFile($lock_file);
  print PID $pid;
  log_msg("saved pid ($pid) to file ($lock_file)");
  close(PID);
  # exec_cmd("cp -af $lock_file $sb_lock_file");
}

 
# to be sure that the subsequent create_assistant_lock() will succeed even after a Linux reboot
remove_assistant_lock();

if ( $do_ionice == 0) {
  log_msg("not ionicing via script");
} else {
  log_msg("trying to reduce CPU and IO priority with ionice and renice");
  exec_cmd($reduce_prio_script . " $pid");
}

$is_duplex = exec_cmd("$is_duplex_script");
if ( $is_duplex == 1 ) {
  log_msg_info("*** running on a duplex system ***");
  check_drbd_sync($sysbackup_storedir);
}

determine_ha_nodes($multiNodeType, $isMultiNode, $other_node, $node_q, 0);

# if subcommand is not "delete" try to create the assistant lock
if ($do_delete == 0) {

  # if duplex -> check if on the other node a system backup is still running 
  if ( $is_duplex == 1 ) {
    check_remote_status($sysbackup_storedir);
  }
  
  if (create_assistant_lock() != 0)  { 
    remove_lock_file();
    exit(1); 
  }
}

if ($do_delete == 1) {
  $percent = 33;
  $current = 1;
  $name="DELETE";
  write_status("running");
  $global_rc = delete_dir_and_sync($sysbackup_dir_to_delete);
  $percent = 100;
  $current = 1;
  write_status($global_rc == 0 ? "successful" : "failed");
  exit $global_rc;
}

write_status("running");

initialize_sysinfo_values();

get_packages_which_have_no_SEN_tag();
log_msg("packages which have no SEN tag: @packages_which_have_no_SEN_tag");

get_packages_which_are_not_to_repackage();
log_msg("packages which are not to repackage: @packages_which_are_not_to_repackage");

get_packages_which_are_allowed_to_fail();
log_msg("packages which are allowed to fail: @packages_which_are_allowed_to_fail");

while(1) {
  log_msg_info("*** starting sysbackup ($cur_sysbackup) collecting in directory ($sysbackup_dir) and storing in ($sysbackup_storedir) ***");
  
  if(! -d $sysbackup_dir) {
    exec_cmd("/bin/rm -rvf $sysbackup_dir");
    exec_cmd("/bin/mkdir -pv $sysbackup_dir");
  }
  if(! -d $sysbackup_storedir) {
    exec_cmd("/bin/rm -rvf $sysbackup_storedir");
    exec_cmd("/bin/mkdir -pv $sysbackup_storedir");
  } 
 

  exec_cmd_info("/bin/df -h $sysbackup_dir",      "partition status of working partition before system backup:");
  exec_cmd_info("/bin/df -h $sysbackup_storedir", "partition status of storage partition before system backup:");  
  exec_cmd("/usr/sbin/drbd-overview",        "DRBD status before system backup:");  
  
  check_diskspace ($sysbackup_storedir, $min_freespace_kbyte);
  check_diskspace ("/var",              $min_freespace_var_kbyte);    
 
  my @packages = get_package_sequence (get_installed_packages());
  my $cmd_sysbackup = "";
  my $cmd_generic_sysbackup = "";
  my $dxml = "<deployments>\n";
  $dxml .= " <deployment>\n";
  $dxml .= "  <name lang=\"en_EN\">Systembackup $sb_date</name>\n";
  $dxml .= "  <name lang=\"de_DE\">Systembackup $sb_date</name>\n";
  $dxml .= "  <linux-version>".get_release_entry('VERSION')."</linux-version>\n";
  $dxml .= "  <linux-patchlevel>".get_release_entry('PATCHLEVEL')."</linux-patchlevel>\n";
  $dxml .= "  <hardware>".get_hardware()."</hardware>\n";
  $total = $#packages + 1 + 2;
  $percent = 0;
  $current = 1;
  $name = "prebackup";
  write_status("running");
  
  remove_intermediate_files();
  migrate_current_hotfix_dir();  
  
  foreach(@packages) {
    last if($do_stop || $global_rc);
    $current++;
    $percent = $current / $total * 100;
    $name = $_;
    write_status("running");
    log_msg_info("*** generating system backup for $_");
    if($_ eq "soco2_jre") {
        if (-e $java_dir) {
        $global_rc = soco2_jre_sysbackup();
        $dxml .= "  <package>jre</package>\n";
        } else {
          log_msg_info("skipping backup of soco2_jre because installation dir $java_dir is missing, obviously java is installed as rpm");
        }
    } else {
      if (is_package_to_be_repackaged($_) == 0) {
        $global_rc = store_original_package($_);
      } else {
        if (is_ADP_running() == 0) {
          log_msg_info("*** ADP no longer running on this maschine, obviously switchover has happened, terminating system backup");
          $name = "ADP switchover";
          $adp_switchover_has_happened = 1;
          last;
        }

        $cmd_sysbackup         = "/usr/local/bin/".$_."_sysbackup";
        $cmd_generic_sysbackup = $webservice_dir."/scripts/generic_sysbackup -p $_";
        if(-e $cmd_sysbackup) {
          if ($_ ne "hp4k-base-image") {
            # since hp4k-base-image is the first, it is not useful to wait 
            log_msg_info("*** sleeping 5 minutes before invoking the specific system backup for $_ to reduce I/O impact");
            sleep(300);
          }

          $cmd_sysbackup         = $ionice_prefix . " " . $cmd_sysbackup;
          $global_rc = exec_cmd($cmd_sysbackup);
          if ($global_rc != 0 && !$do_stop) {
            # give specific advisory for hp4k-base-imag
            if ($_ eq "hp4k-base-image") {
              if ($global_rc == 10) {
                log_msg_info("*** PDS area was not active. Please check it with AMO DISP-DDSM and if possible activate it with ACT-DSSM");
              } 
              if ($global_rc == 11) {
                log_msg_info("*** It was not possible to use controller 4 for system backup.");
              }
            }
            log_msg_info("*** specific system backup for $_ failed once. Retrying after 6 minutes  ...");
            
            sleep(360);
            log_msg_info("*** generating again system backup for $_");            
            $global_rc = exec_cmd($cmd_sysbackup);          
          }          
        } else {
          $cmd_generic_sysbackup = $ionice_prefix . " " . $cmd_generic_sysbackup;
          $global_rc = exec_cmd($cmd_generic_sysbackup);
          if ($global_rc != 0 && !$do_stop) {
            log_msg_info("*** generic system backup for $_ failed once. Retrying after 6 minutes ...");
            sleep(360);
            log_msg_info("*** generating again system backup for $_");                        
            $global_rc = exec_cmd($cmd_generic_sysbackup);
          }
        }
      }
      if ($global_rc == 0) {
        log_msg("adding $_ to the package list");
        $dxml .= "  <package>$_</package>\n";
      } else {
        if (is_package_allowed_to_fail($_) == 1) {
          log_msg("system backup for $_ failed, but this error will be ignored due to positive list");
          $global_rc = 0;
        }
      }
    }
  }
  $dxml .= " </deployment>\n";
  $dxml .= "</deployments>\n";
  # copy FIW config file
  if($global_rc == 0) {
    log_msg_info("adding restore tool configuration to system backup set");
    $name = "haipsetup.xml";
    $global_rc = exec_cmd("/bin/cp -f $ha_data_dir/conf/haipsetup.xml $sysbackup_dir");
  }
  # copy a possible SoftGate initial cfg file_age
  if ($global_rc == 0 && -e $sg_initcfg_file ) {
    log_msg_info("adding SoftGate initialcfg.xml");
    $name = "initialcfg.xml";
    $global_rc = exec_cmd("/bin/cp -f $sg_initcfg_file $sysbackup_dir");    
  }

  if($global_rc == 0) {
    log_msg_info("adding console restore daemon to system backup set");
    $name = "store_package_unique.sh";
    $global_rc = exec_cmd("/bin/cp -f $webservice_dir/scripts/store_package_unique.sh $sysbackup_dir");
    exec_cmd("/bin/chmod 755 $sysbackup_dir/store_package_unique.sh");
  }
  if($global_rc == 0) {
    log_msg_info("adding store package script");
    $name = "swupdated";
    $global_rc = exec_cmd("/bin/cp -f $webservice_dir/scripts/swupdated $sysbackup_dir");
    exec_cmd("/bin/chmod 755 $sysbackup_dir/swupdated");
  }  
  if($global_rc == 0) {
    if (-e "$webservice_dir/scripts/SystemRestore.sh") {
      log_msg_info("adding console restore script to system backup set");
      $name = "SystemRestore.sh";
      $global_rc = exec_cmd("/bin/cp -f $webservice_dir/scripts/SystemRestore.sh $sysbackup_dir");
      log_msg_info("assign executable rights for  console restore script in system backup set");
      exec_cmd("/bin/chmod 755 $sysbackup_dir/SystemRestore.sh");       
    }
  }  
  $current = $total;


  if(($do_stop == 1) || ($adp_switchover_has_happened == 1)) {
    $global_rc = 1;
  } else {
    if($global_rc == 0) {
      $name = "postbackup";
      write_status("running");
    }    
  }
  
  prepare_store_sysbackup($sysbackup_storedir);
  my $copy_rc = 0;

  if ($global_rc == 0)
  {
    # normal store
    $copy_rc = store_sysbackup($sysbackup_storedir, 0);
  }
  else
  {
    # clean the area, do not store
    $copy_rc = store_sysbackup($sysbackup_storedir, 1);
  }
  
  if ($global_rc == 0) {
    if ($copy_rc == 0 && $global_rc == 0) {
      $copy_rc = store_current_hotfixfiles($sysbackup_storedir);
    }
  
    if ($copy_rc == 0 && $global_rc == 0) {
      cleanup_storage_before_copy($cur_hf_rpm_dir);
      $copy_rc = store_current_hotfixrpms($sysbackup_storedir);
      recover_storage_after_copy($cur_hf_rpm_dir);     
    }  
    if ($copy_rc == 0 && $global_rc == 0) {  
      if ($cp_installrepo != 0) {
        cleanup_storage_before_copy($backup_dir);
        $copy_rc = store_installrepo($sysbackup_storedir);
        recover_storage_after_copy($backup_dir); 
      }
    }
    if ($copy_rc == 0 && $global_rc == 0) {  
      $copy_rc = store_osdeps($sysbackup_storedir);
    }

    # ignore the return code of the license storage
    if ( $global_rc == 0 && store_license_dir($sysbackup_storedir) != 0) {
      log_msg("ignoring error code of CLA license collecting/storing. Maybe licenses have to be imported manually after system restore");
    }
    
    if(! $do_stop) {
      generate_md5sums($sysbackup_storedir);
    }
  }
   
  store_versioninfo_file($sysbackup_storedir);
  
  if ($copy_rc != 0) {  
    $global_rc = $copy_rc;
    log_msg("storing system backup to $sysbackup_storedir failed");    
  }
  
  
  # write deployment file
  if($global_rc == 0) {
    log_msg_info("generating deployment file");
    log_msg("writing deployment file $deploy_file with content:\n$dxml");
    write_file_exclusive($deploy_file, $dxml);
  }
  exec_cmd_info("/bin/df -h $sysbackup_dir",      "partition status of working partition after system backup:");
  exec_cmd_info("/bin/df -h $sysbackup_storedir", "partition status of storage partition after system backup:");  
  exec_cmd_info("/usr/sbin/drbd-overview",        "DRBD status after system backup:");  
    
  remove_lock_file();
  
  if ($do_stop) {
    $name = "cancelled";
    write_status("failed");  
  } elsif ($adp_switchover_has_happened == 1) {
    $name = "ADP switchover";
    write_status("failed");      
  } else {
    write_status($global_rc == 0 ? "successful" : "failed");
  }
  
  # copy status file
  store_statusfile();
  rename_storagedir_to_final_name($sysbackup_storedir);  
  
  exec_cmd("/bin/rm -vf $sysinfo_log"); 

  my $status_string = "finished successfully";
  if($global_rc != 0) {
    log_msg_info("failed component: $name");
    $status_string = "failed";
  }  
  
  if (($do_rsync == 1) && ($is_duplex == 1)) {
    log_msg_info("*** syncing the sysbackup storage to the standby duplex node ***");
    my $sync_rc = exec_cmd("$webservice_dir/scripts/putx $sysbackup_dir/ $sysbackup_dir/");
  }
  
  log_msg_info("*** system backup ($dest_sysbackup) $status_string,  $total files, rc=$global_rc ***");
  
  store_log_file();
  exit($global_rc);
}

sub delete_dir_and_sync {
  my $dir_to_delete = shift;
  my $temp_dir_to_delete = $sysbackup_dir . "/" ."to_be_deleted";
  my $father_dir = "";
  my $rc = 0;

  if ( ! -e $dir_to_delete )  {
    log_msg("directory $dir_to_delete does not exits -> try storage area");

    if (-e $sysbackup_dir . "/" . $dir_to_delete) {
      $dir_to_delete = $sysbackup_dir . "/" . $dir_to_delete;
      $father_dir    = $sysbackup_dir;
      log_msg("found directory $dir_to_delete");
    }
  }
  if ( ! -e $dir_to_delete )  {
    log_msg_info("*** can not delete $dir_to_delete because it does not exist ***");
    $rc = 1;
  }
  else {
    exec_cmd_info("/bin/df -h $sysbackup_dir",      "partition status of working partition before delete:");

    if ("$father_dir" eq "") {
      log_msg_info("*** deleting directly $dir_to_delete ***");
      $rc = exec_cmd("/bin/rm -rf $dir_to_delete");

    } else {
      log_msg_info("*** moving to temp. name: $temp_dir_to_delete ***");
      if (-e $temp_dir_to_delete) {
        $rc = exec_cmd("/bin/rm -rf $temp_dir_to_delete");
      }

      $rc = exec_cmd("/bin/mv -vf --no-target-directory $dir_to_delete $temp_dir_to_delete");
      log_msg("*** moved to temp. name: $temp_dir_to_delete rc=$rc ***");

      log_msg_info("*** removing the temp. name dir and content: $temp_dir_to_delete ***");
      $rc = exec_cmd("/bin/rm -rf $temp_dir_to_delete");
    }

    exec_cmd_info("/bin/df -h $sysbackup_dir",      "partition status of working partition after delete:");
  }

  $percent = 66;

  if (($do_rsync == 1) && ($is_duplex == 1)) {
    log_msg_info("*** syncing the sysbackup storage to the standby duplex node ***");
    my $sync_rc = exec_cmd("$webservice_dir/scripts/putx $sysbackup_dir/ $sysbackup_dir/");
  }
  log_msg_info("deleted directory $sysbackup_dir_to_delete, rc=$rc");

  return ($rc);
}

sub get_deployment_as_string {
  log_msg(">>> get_deployment_as_string");
  my $deployment_as_string_output = "";

  if (-e "$get_deployment_string_script") {
    $deployment_as_string_output = exec_cmd_pipe("$get_deployment_string_script");
    $deployment_as_string_output = my_trim($deployment_as_string_output);
  } else {
    log_msg("get deploment script script: $get_deployment_string_script does not exists");
  }
  log_msg("get_deployment_as_string: $deployment_as_string_output");
  return($deployment_as_string_output);
}

sub determine_ha_nodes {
  my $enhanced_diagnosis = $_[4];
  my $rc = 0;

  my $haipsetup = $ha_data_dir."/conf/haipsetup.xml";
  my $multiNodeType = 0; # single node
  my $isMultiNode = 0;
  my $node_a = "";
  my $node_b = "";
  my $node_q = "";
  my $other_node = "";
  log_msg(">>> determine_ha_nodes");
  
 
  # check ha setup parameters
  if(-e $haipsetup && open(HAIPSETUP, "<$haipsetup")) {
    
    if ( !defined($enhanced_diagnosis) && $enhanced_diagnosis == 1) {
      my $haipsetup_diagnose = "";
      $haipsetup_diagnose = exec_cmd_pipe("cat $haipsetup");
      log_msg_info("current content of$haipsetup: \n $haipsetup_diagnose"); 
    }
    
    my @setup_xml = <HAIPSETUP>;
    my @parts;
    foreach(@setup_xml) {
      if(index($_, "system.multiNodeSystem") >= 0) {
        @parts = split(/<|>/, $_);
        $multiNodeType = $parts[2];
        log_msg_info("found multi node type $multiNodeType"); 
      } elsif(index($_, "system.name-node1") >= 0) {
        @parts = split(/<|>/, $_);
        $node_a = $parts[2];
      } elsif(index($_, "system.name-node2") >= 0) {
        @parts = split(/<|>/, $_);
        $node_b = $parts[2];
      } elsif(index($_, "system.nameQuorumNode") >= 0) {
        @parts = split(/<|>/, $_);
        $node_q = $parts[2];
      }
    }
    close(HAIPSETUP);
  } else {
    $rc = 1;
    log_msg("$haipsetup does not exist or could not be opened");
  }
  # check if running redundancy scenario (10-19, 60-69, 99)
  log_msg("detected multi node type $multiNodeType");
  if($rc == 0 &&
     length($node_a) > 0 &&
     length($node_b) > 0 &&
     (($multiNodeType >= 10 && $multiNodeType < 20) || ($multiNodeType >= 60 && $multiNodeType < 70) || ($multiNodeType == 99))) 
  {
     $isMultiNode = 1;  
     $other_node = $node_a;

     if($node_a eq $hostname) { $other_node = $node_b; }
     log_msg("other node is $other_node");
    
  } else {
    log_msg("not running in redundancy scenario");
    $isMultiNode = 0;  
  }

  # give back the values
  $_[0] = $multiNodeType;
  $_[1] = $isMultiNode;
  $_[2] = $other_node;
  $_[3] = $node_q;  
  log_msg("<<< determine_ha_nodes finished rc=$rc multiNodeType=$multiNodeType, isMultiNode=$isMultiNode, other_node=$other_node, node_q=$node_q");
  return($rc);
}

sub store_log_file {
  # store the logfile so that "collect all diagnosis logs" can collect it
  exec_cmd("/bin/cp -vf $log_file $log_file_after");
}

sub migrate_current_hotfix_dir {
  if(! -e $cur_hf_dir) {
    if( -e $cur_hf_dir_old) { 
      log_msg_info("found an old current hotfix dir (part of webservice) -> moving it to new location (soco2)");    
      exec_cmd("/bin/mv -vf $cur_hf_dir_old/ $cur_hf_dir/");      
    }
  }

  if( -e $cur_hf_dir_old) {
      log_msg_info("found an old current hotfix dir -> moving at least the files");      
      exec_cmd("/bin/mv -vf $cur_hf_dir_old/*.tar $cur_hf_dir/");
  }
}

sub soco2_jre_sysbackup {
  my $rc = 0;
  log_msg_info("soco2_jre_sysbackup: $java_dir");
  
  if(opendir(DIR, $java_dir)) {
    my @files = readdir(DIR);
    foreach(@files) {
      if($_ ne "." && $_ ne ".." && index($_, "jre") >= 0 && index($_, "bin") >= 0) {
        $rc = exec_cmd("/bin/cp -vf $java_dir/$_ $sysbackup_dir");
      }
    }
  }
  return($rc);
}

sub store_original_package {
  my $name = shift;
  my $rc = 0;
  log_msg("*** storing original package without re-packaging: $name");
  
  my $count = 0;
  my @list = ();
  
  @list = glob("$cur_hf_rpm_dir/$name-*");
  $count = @list;
  log_msg("count=$count for $cur_hf_rpm_dir/$name-*");
  for (@list) {
    log_msg("found $_");      
  } 
  if ( $count > 0 ) {
    log_msg("*** found the package $name as part of an installed hotfix.");

    $rc = exec_cmd("/bin/cp -vf $cur_hf_rpm_dir/$name-* $sysbackup_dir");
  } else {
    @list = glob("$backup_dir/$name-*");
    $count = @list;
    log_msg("count=$count for $backup_dir/$name-*");
    for (@list) {
      log_msg("found $_");      
    }
    if ($count > 0) {  
      log_msg("*** found the package $name as part of an RLC upgrade or first installation.");
      $rc = exec_cmd("/bin/cp -vf $backup_dir/$name-* $sysbackup_dir");
    } else {
      @list = glob("$osdeps_backup_dir/$name-*");
      $count = @list;
      log_msg("count=$count for $osdeps_backup_dir/$name-*");
      for (@list) {
        log_msg("found $_");
      }
      if ($count > 0) {
        log_msg("*** found the package $name in the os_deps dir $osdeps_backup_dir");
        $rc = exec_cmd("/bin/cp -vf $osdeps_backup_dir/$name-* $sysbackup_dir");
      } else {
        @list = glob("$osupdate_backup_dir/$name-*");
        $count = @list;
        log_msg("count=$count for $osupdate_backup_dir/$name-*");
        for (@list) {
          log_msg("found $_");
        }
        if ($count > 0) {
          log_msg("*** found the package $name in the osupdate backup_dir dir $osupdate_backup_dir");
          $rc = exec_cmd("/bin/cp -vf $osupdate_backup_dir/$name-* $sysbackup_dir");
        } else {
          log_msg_info("*** error: did not find the original package $name as part of an installed hotfix ($cur_hf_rpm_dir/$name-*) or RLC upgrade ($backup_dir/$name-*) or os_deps or os_update.");
          $rc = 1;
        }
      }
    }  
  }
  return($rc);
}

   
sub check_remote_status {
  my $storage_dir = shift;
  my $rc = 0;
  my $fail_on_standby_running = 0;
  
  my $remote_status = exec_cmd("$webservice_dir/scripts/sshx '/opt/webservice/scripts/sysbackup status'");
  if ($remote_status == 0 ) {
    if ($fail_on_standby_running == 1) {
      log_msg_info("*** Error: A system backup is still running on the standby system node, system backup currently not possible ***");
      $name = "still running on the standby node";

      write_status("failed");

      prepare_store_sysbackup($storage_dir);
      store_statusfile();
      rename_storagedir_to_final_name($storage_dir);

      sleep (10);
      exec_cmd("/bin/rm -vf $sysinfo_log");      
      remove_lock_file();
      exit(1);
    } else {
      log_msg_info("*** Info: A system backup is still running on the standby system node, terminating this to start the new run");
      my $remote_rc = 0;
      $remote_rc = exec_cmd("$webservice_dir/scripts/sshx '/opt/webservice/scripts/sysbackup stop'");
      sleep (10);
    }
  }
  return($rc);
}

sub check_drbd_sync {
  my $storage_dir = shift;
  
  my $drbd_status = get_drbd_status();
  if ($drbd_status != 0) {
    log_msg_info("*** Error: DRBD sync in progress, system backup currently not possible ***");
    $name = "DRBD not in sync";
    
    write_status("failed");
    
    prepare_store_sysbackup($storage_dir); 
    store_statusfile();
    rename_storagedir_to_final_name($storage_dir);
    
    if (($do_rsync == 1) && ($is_duplex == 1)) {
      log_msg_info("*** syncing the sysbackup storage to the standby duplex node ***");
      my $sync_rc = exec_cmd("$webservice_dir/scripts/putx $sysbackup_dir/ $sysbackup_dir/");
    }
    
    sleep (10);
    exec_cmd("/bin/rm -vf $sysinfo_log"); 
    remove_lock_file();
    exit (1);    
  }
}


sub check_diskspace {
  my $storage_dir = shift;
  my $min_freespace = shift;
  my $cmd = "df -k -P $storage_dir | grep -v Filesystem | awk '{print \$4}'";  
  my $free_space_in_kbtype = exec_cmd_pipe($cmd); 
  if ($free_space_in_kbtype < $min_freespace) {
      $name = "free disk check";      
      
      log_msg_info("Error: Minimum free space on partition ($storage_dir) not sufficient!");  
      log_msg_info("At least $min_freespace kByte have to be free!"); 
      write_status("failed");
      
      prepare_store_sysbackup($storage_dir); 
      store_statusfile();
      rename_storagedir_to_final_name($storage_dir);
      
      sleep (10);
      exec_cmd("/bin/rm -vf $sysinfo_log");      
      remove_lock_file();
      exit (1);
  }
}

sub create_assistant_lock {
  my $rc = 0;
  if (is_Assistant_running() == 1) {
    my $cmd = "/bin/sshr -c host_intl createLock -i WEBSERVICE -o BACKUP -s SYSTEMBACKUP -c /opt/webservice/scripts/checkSystemBackupLock.sh";
    $rc = exec_cmd($cmd);
    if ($rc == 0) {
      log_msg_info("successfully created the Assistant lock to prevent HBR or update actions!");      
    } else {
      log_msg_info("current Assistant lock prevents starting a system backup. Probably HBR or update action is currently running (rc=$rc) !");
    }
  }
  else {
    $rc = 1;
    log_msg_info("Assistant is currently not installed or not running. System Backup can not be started (rc=$rc) !");
  }
  
  # create also the work lock here
  write_file_exclusive($work_lock_file, "system backup started");
  return($rc);
}

sub remove_assistant_lock {
  my $rc = 0;
  my $cmd = "/bin/sshr -c host_intl deleteLock -i WEBSERVICE -o BACKUP -s SYSTEMBACKUP";
  $rc = exec_cmd($cmd);
  if ($rc == 0) {
      log_msg("successfully deleted the Assistant lock");      
  } else {
      log_msg("can not delete the Assistant lock rc=$rc -> continuing");
  }  
  return($rc);
}

sub is_ADP_running {
  my $rc = exec_cmd("virsh domstate ADP |  egrep -e 'running'");
 
  if ($rc == 0) {
    $rc = 1;
  }
  else {
    $rc = 0;
    log_msg("is_ADP_running: simplex ADP VM is not running, checking for Mono");    
    $rc = exec_cmd("virsh domstate CCAP |  egrep -e 'running'");
    if ($rc == 0) {
      $rc = 1;
    } else {
      log_msg("is_ADP_running: neither simplex ADP VM nor Mono CCAP VM is running -> probably switchover occurred on a Duplex machine");
      $rc = 0;
    }
  }
  log_msg("is_ADP_running rc=$rc (1 means true)");
  return($rc);
  }

sub is_Assistant_running {
  my $rc = exec_cmd("virsh domstate Assistant |  egrep -e 'running'");
  if ($rc == 0) {
    $rc = 1;
  }
  else {
    $rc = 0;
  }
  log_msg("is_Assistant_running rc=$rc (1 means true)");
  return($rc);
}

sub is_service_running {
  my $service = shift;
  my $rc = 0;
  my $service_lock_file = '/var/opt/Assistant/data/'.$service.'_running';
  if(-e $service_lock_file) {
    my $file_age = -M $service_lock_file;
    log_msg("lock file: $service_lock_file, age: $file_age, max allowed: $max_svc_age");
    if($file_age lt $max_svc_age) {
      log_msg("$service lock file detected - will not start");
      $rc = 1;
    } else {
      log_msg("$service lock file detected, but outdated - will proceed");
    }
  }
  log_msg("service $service is running $rc (0=no)");
  return($rc);
}

sub cleanup_storage_before_copy {
  my $storage = shift;
  log_msg("*** cleaning up $storage before copying it to system backup dir");
  my $rc = 0;  

  my $cmd_rm = "/bin/mv -vf $storage/*-image-* $storage/..";  
  $rc = exec_cmd($cmd_rm);  
  
  if ($rc != 0) {
    log_msg_info("ignoring error code when moving unnecessary image files from $storage to a level above rc=$rc");
  }
  return($rc); 
}

sub recover_storage_after_copy {
  my $storage = shift;
  log_msg("*** recovering $storage after copying it to system backup dir");
  my $rc = 0;  

  my $cmd_rm = "/bin/mv -vf $storage/../*-image-* $storage/";  
  $rc = exec_cmd($cmd_rm);  
  
  if ($rc != 0) {
    log_msg_info("ignoring error code when removing image files back from temp. storage to $storage rc=$rc");
  }
  return($rc); 
}

sub store_installrepo {
  my $storage_dir = "$_[0]/$cur_sysbackup/installrepo";
  my $rc = 0;  
  my $cmd_cp = $ionice_prefix . " /bin/cp -rvf $backup_dir $storage_dir";  
  my $cmd_rm = "/bin/rm -rvf $storage_dir";  
  
  log_msg_info("*** storing current installrepo from $backup_dir to $storage_dir");
  if( -e "$storage_dir") {
    log_msg_info("$storage_dir already exists -> removing it");
    $rc = exec_cmd($cmd_rm);
  }  
  if($rc == 0) {
    $rc = exec_cmd($cmd_cp);
  }    

  if($rc != 0) {
    log_msg_info("error when copying current installrepo from $backup_dir to $storage_dir. Maybe no space left on device?");  
  }
  
  return($rc);    
}

sub get_current_version {
    my $cur_version = exec_cmd_pipe("/bin/getydata -v");
    $cur_version = my_trim($cur_version);
    return ($cur_version);
}

sub get_current_assistant_version {
    my $cur_version = exec_cmd_pipe("$get_version_script 'A&S'");
    $cur_version = my_trim($cur_version);
    return ($cur_version);
}

sub get_current_csta_version {
    my $cur_version = exec_cmd_pipe("$get_version_script 'CSTA'");
    $cur_version = my_trim($cur_version);
    return ($cur_version);
}

sub get_current_RMX_version {
    my $cur_version = exec_cmd_pipe("$get_version_script 'RMX'");
    $cur_version = my_trim($cur_version);
    return ($cur_version);
}

sub store_versioninfo_file {
  my $storage_dir = "$_[0]/$cur_sysbackup/";
  
  log_msg_info("store version info file");  
  
  if ( -e $assistant_version_file ) {
    exec_cmd("/bin/cp -f $assistant_version_file $storage_dir");    
  } else {
    log_msg_info("did not find assistant version file $assistant_version_file, generating it on my own ");  
    my $version_file = $storage_dir . "versioninfo.txt";
    my $plt_version       = get_current_version();
    my $assistant_version = get_current_assistant_version();
    my $csta_version      = get_current_csta_version();
    my $rmx_version       = get_current_RMX_version();

    log_msg("creating version file $version_file ");  

    my $cur_date = datefmt();
    my $hardware = get_hardware_as_string();
    my $multiNodeTypeString = get_deployment_as_string();

    if(open(FILE, ">$version_file")) {
      print FILE ("Details of HiPath 4000 System Backup from $cur_date\n");
      print FILE ("Original Host:  $hostname (this is the host where the original backup was made) \n");
      print FILE ("Original HW:    $hardware\n");
      print FILE ("Deployment:     $multiNodeTypeString ($multiNodeType)\n");
      print FILE ("Platform:       $plt_version         \n");
      print FILE ("Assistant:      $assistant_version   \n");
      print FILE ("CSTA:           $csta_version   \n");
      print FILE ("RMX:            $rmx_version   \n");
      close(FILE);
    }    
  }
}

sub store_license_dir {
  my $storage_dir = "$_[0]/$cur_sysbackup/license_storage/";
  my $rc = 0;  
  
  # store my own licenses
  store_license_dir_for_each_host($storage_dir.$hostname, $cla_license_dir);

  if ( $is_duplex == 1 ) {
    my $license_dir_from_remote = "/tmp/remote_license_dir";
    exec_cmd("rm -rf $license_dir_from_remote");

    $rc = exec_cmd("$getx_script $cla_license_dir $license_dir_from_remote ");
    if ($rc != 0) {
      log_msg_info("error when transferring CLA license dir $cla_license_dir from duplex standby node (rc=$rc).");  
    } else {
      my $remote_hostname = exec_cmd_pipe( $getStandby_script );
      if ( $remote_hostname eq "" ) {
        log_msg_info("got empty string when retrieving the hostname of the standby node -> skipping license dir of standby node");  
      } else {
        store_license_dir_for_each_host($storage_dir.$remote_hostname, $license_dir_from_remote."/license");
        # cleanup_storage_before_copy
        exec_cmd("rm -rf $license_dir_from_remote");
      }
	}
  }
  return ($rc);
}

sub store_license_dir_for_each_host {
  my $storage_dir = "$_[0]";
  my $license_dir = "$_[1]";
  my $rc = 0;  
  my $count = 0;    
  my $cmd_mkdir = "/bin/mkdir -pv $storage_dir";  
  log_msg_info("*** storing license files from $license_dir to $storage_dir");

  if(! -e "$license_dir") {
    log_msg_info("*** no $license_dir directory (obviously no license files imported via Assistent or HLM yet)");  
    return($rc);
  }   
  
  if(! -e "$storage_dir") {
    $rc = exec_cmd($cmd_mkdir);
  }    
    
  if($rc == 0 && opendir(DIR, $license_dir)) {
    my @files = readdir(DIR);
    my $cmd_cp = "";
    foreach(@files) {
      last if($rc);
      if($_ ne "." && $_ ne ".." && ! -d "$storage_dir/$_") {
        log_msg_info("processing $_");

        $cmd_cp = $ionice_prefix . " /bin/cp -vf $license_dir/$_ $storage_dir/$_";
        $rc = exec_cmd($cmd_cp);
        if($rc == 0) {
          $count++;
        } else {
          log_msg_info("could not copy file $license_dir/$_ to $storage_dir/$_");
        }
      }
    }
    closedir(DIR);
  } else {
    $rc = 0;
    log_msg_info("could not open/process directory $license_dir");
  }
  if($rc == 0 && $count == 0) {
    log_msg_info("no license files found in $license_dir");
  }    
   
  return($rc);  
}

sub store_osdeps {
  my $storage_dir = "$_[0]/$cur_sysbackup/os_deps";
  my $rc = 0;  
  my $count = 0;  
  my $cmd_mkdir = "/bin/mkdir -pv $storage_dir";  

  if (! -e "$osdeps_backup_dir") {
    log_msg("*** not storing additional OS packages from $osdeps_backup_dir because directory does not exists");
    $rc = 0;
    return($rc);
  }
  
  log_msg_info("*** storing additional OS packages from $osdeps_backup_dir to $storage_dir");
  if(! -e "$storage_dir") {
    $rc = exec_cmd($cmd_mkdir);
  }    
  
  if($rc == 0 && opendir(DIR, $osdeps_backup_dir)) {
    my @files = readdir(DIR);
    my $cmd_cp = "";
    foreach(@files) {
      last if($rc);
      if($_ ne "." && $_ ne ".." && ! -d "$sysbackup_dir/$_") {
        log_msg_info("processing $_");

        $cmd_cp = $ionice_prefix . " /bin/cp -vf $osdeps_backup_dir/$_ $storage_dir/$_";
        $rc = exec_cmd($cmd_cp);
        if($rc == 0) {
          $count++;
        } else {
          log_msg_info("could not copy file $osdeps_backup_dir/$_ to $storage_dir/$_");
        }
      }
    }
    closedir(DIR);
  } else {
    $rc = 0;
    log_msg_info("could not open/process directory $osdeps_backup_dir");
  }
  if($rc == 0 && $count == 0) {
    log_msg_info("no rpm files found in $osdeps_backup_dir");
  }
  return($rc);    
}

sub store_current_hotfixrpms {
  my $storage_dir = "$_[0]/$cur_sysbackup/current_hotfix_rpms";
  my $rc = 0;  
  my $count = 0;  
  my $cmd_mkdir = "/bin/mkdir -pv $storage_dir";  

  if (! -e "$cur_hf_rpm_dir") {
    log_msg("*** Not storing current hotfix rpm files from $cur_hf_rpm_dir because directory does not exists");
    return($rc);    
  }
  
  log_msg_info("*** storing current hotfix rpm files from $cur_hf_rpm_dir to $storage_dir");
  if(! -e "$storage_dir") {
    $rc = exec_cmd($cmd_mkdir);
  }    
  
  if($rc == 0 && opendir(DIR, $cur_hf_rpm_dir)) {
    my @files = readdir(DIR);
    my $cmd_cp = "";
    foreach(@files) {
      last if($rc);
      if($_ ne "." && $_ ne ".." && ! -d "$sysbackup_dir/$_") {
        log_msg_info("processing $_");

        $cmd_cp = $ionice_prefix . " /bin/cp -vf $cur_hf_rpm_dir/$_ $storage_dir/$_";
        $rc = exec_cmd($cmd_cp);
        if($rc == 0) {
          $count++;
        } else {
          log_msg_info("could not copy file $cur_hf_rpm_dir/$_ to $storage_dir/$_");
        }
      }
    }
    closedir(DIR);
  } else {
    $rc = 0;
    log_msg_info("could not open/process directory $cur_hf_rpm_dir");
  }
  if($rc == 0 && $count == 0) {
    log_msg_info("no hotfix rpm files found in $cur_hf_rpm_dir");
  }
  return($rc);  
  
}

sub store_current_hotfixfiles {
  my $storage_dir = "$_[0]/$cur_sysbackup/current_hotfix";
  my $rc = 0;  
  my $count = 0;  
  my $cmd_mkdir = "/bin/mkdir -pv $storage_dir";  
  
  log_msg_info("*** storing current hotfix files from $cur_hf_dir to $storage_dir");
  if(! -e "$storage_dir") {
    $rc = exec_cmd($cmd_mkdir);
  }  
  
  if($rc == 0 && opendir(DIR, $cur_hf_dir)) {
    my @files = readdir(DIR);
    my $cmd_cp = "";
    foreach(@files) {
      last if($rc);
      if($_ ne "." && $_ ne ".." && ! -d "$sysbackup_dir/$_") {
        log_msg_info("processing $_");

        $cmd_cp = $ionice_prefix . " /bin/cp -vf $cur_hf_dir/$_ $storage_dir/$_";
        $rc = exec_cmd($cmd_cp);
        if($rc == 0) {
          $count++;
        } else {
          log_msg_info("could not copy file $cur_hf_dir/$_ to $storage_dir/$_");
        }
      }
    }
    closedir(DIR);
  } else {
    $rc = 0;
    log_msg_info("could not open/process directory $cur_hf_dir");
  }
  if($rc == 0 && $count == 0) {
    log_msg_info("no hotfix files found in $cur_hf_dir");
  }
  return($rc);  
}

sub store_statusfile {
  exec_cmd("/bin/cp -f $xml_file $status_file");
}

sub prepare_store_sysbackup {
  my $storage_dir = $_[0];
  my $rc = 0;
  
  log_msg("prepare_store_sysbackup dir=$storage_dir/$cur_sysbackup");
  
  my $cmd_rm = "/bin/rm -rf $storage_dir/$cur_sysbackup";
  my $cmd_mkdir = "/bin/mkdir -pv $storage_dir/$cur_sysbackup";
  if(-e "$storage_dir/$cur_sysbackup") {
    $rc = exec_cmd($cmd_rm);
  }
  if($rc == 0) {
    $rc = exec_cmd($cmd_mkdir);
  }
  return($rc);    
}

sub store_sysbackup {
  my $storage_dir = $_[0];
  my $clean_it    = $_[1];
  
  log_msg_info("*** storing generated packages to $storage_dir/$cur_sysbackup (clean=$clean_it)");
    
  my $cmd_rm = "/bin/rm -rf $storage_dir/$cur_sysbackup";
  my $rc = 0;
  my $count = 0;

  if($rc == 0 && opendir(DIR, $sysbackup_dir)) {
    my @files = readdir(DIR);
    my $cmd_mv = "";
    foreach(@files) {
      last if($rc);
      if($_ ne "." && $_ ne ".." && $_ ne "sysbackup.log" && ! -d "$sysbackup_dir/$_" ) {
        log_msg_info("processing $_");
        # use mv to store the rpm files to the final storade dir to save IO and time if on the same partition
        if ($clean_it == 0) {
          $cmd_mv = $ionice_prefix . " /bin/mv -vf $sysbackup_dir/$_ $storage_dir/$cur_sysbackup/$_";
        } else {
          $cmd_mv = $ionice_prefix . " /bin/rm -vf $sysbackup_dir/$_ $storage_dir/$cur_sysbackup/$_";
        }
        $rc = exec_cmd($cmd_mv);
        if($rc == 0) {
          $count++;
        } else {
          log_msg_info("could not move file $sysbackup_dir/$_ to $storage_dir/$cur_sysbackup/$_");
        }
      }
    }
    closedir(DIR);
  } else {
    $rc = 1;
    log_msg_info("could not open/process directory $sysbackup_dir");
  }
  if($rc == 0 && $count == 0) {
    log_msg_info("no sysbackup files found in $sysbackup_dir");
    $rc = exec_cmd($cmd_rm);
    $rc = 1;
  }
  return($rc);
}

sub rename_storagedir_to_final_name {
  my $storage_dir = $_[0];
  log_msg("rename_storagedir_to_final_name dir=$storage_dir");
  
  my $rc = 0;
  
  if ( -e "$storage_dir/$dest_sysbackup" ) {
    exec_cmd("/bin/rm -rf $storage_dir/$dest_sysbackup");
  }
  $rc = exec_cmd("/bin/mv -vf $storage_dir/$cur_sysbackup/ $storage_dir/$dest_sysbackup");      
  return($rc);   
}

sub generate_md5sums {
  my $storage_dir = $_[0];
  my $working_dir = "$storage_dir/$cur_sysbackup";

  log_msg_info("*** generating md5 sums for generated rpm files in $working_dir");
  my $cmd = $webservice_dir . "/scripts/generateMD5sums.sh $working_dir" ;
  my $rc = exec_cmd($cmd);
  
  if ($rc != 0) {
    log_msg_info("*** error when generating md5 sums for generated rpm files in $working_dir, status=$rc");
  }
  return($rc);
}

sub remove_intermediate_files {
  log_msg("*** remove_intermediate_files in ($sysbackup_dir)");
  if(opendir(DIR, $sysbackup_dir)) {
    my @files = readdir(DIR);
    foreach(@files) {
      if($_ ne "." && $_ ne ".." && $_ ne "sysbackup.log" && $_ ne "sysbackup.lock" && ! -d "$sysbackup_dir/$_") {
        unlink("$sysbackup_dir/$_");
        log_msg("removed file $sysbackup_dir/$_");
      }
    }
    closedir(DIR);
  } else {
    log_msg("could not open directory $sysbackup_dir");
  }
}

sub get_package_sequence {
  log_msg_info("*** get_package_sequence");
  my @packages = @_;
  @packages = sort (@packages);
    
  # find the index of hp4k-base-image
  my( $found, $index ) = ( undef, -1 );
  for( my $i = 0; $i < @packages; $i++ ) {
    if( $packages[$i] =~ /hp4k-base-image/ ) {
      $found = $packages[$i];
      $index = $i;
      last;
    }
  }
  
  if ($index != -1) {
    # found it -> delete it
    splice(@packages, $index, 1);
    
    # add it at the begin 
    @packages = ("hp4k-base-image", @packages);
  }
        
  log_msg("sorted packages: @packages");

  return (@packages);
}

sub get_installed_packages {
  log_msg_info("*** retrieving packages to be backuped");

  my @names = @packages_which_have_no_SEN_tag; 
  log_msg("packages which are not tagged: @packages_which_have_no_SEN_tag");

  my $cmd = "/bin/rpm -q --queryformat \"%{NAME} %{SIZE} %{VERSION}-%{RELEASE} %{PACKAGER}__###__\" --all";
  my $output = exec_cmd_pipe($cmd);
  if(length($output) > 0) {
    my @packages = split(/__###__/, $output);
    foreach(@packages) {
      if(index($_, "SEN") >= 0) {
        my @parts = split(/ /, $_);
        push(@names, $parts[0]);
      }
    }
  }
  
  log_msg("all packages : @names");  
  return(@names);
}

sub is_package_to_be_repackaged {
  my $name = shift;
  my $rc = 1;
  
  my $pkg;
  foreach $pkg (@packages_which_are_not_to_repackage) {
      if ( $pkg eq $name) {
        $rc = 0;
      }
  }
  log_msg("package $name has to be re-packaged: $rc (1=yes)");
  return($rc);
}

sub is_package_allowed_to_fail {
  my $name = shift;
  my $rc = 0;

  my $pkg;
  foreach $pkg (@packages_which_are_allowed_to_fail) {
      if ( $pkg eq $name) {
        $rc = 1;
      }
  }
  log_msg("package $name is allowed to fail: $rc (1=yes)");
  return($rc);
}


sub get_installed_rpm_file_name {
  my $name = shift;
  my $cmd = "/bin/rpm -q --queryformat %{NAME}-%{VERSION}-%{RELEASE}.%{ARCH}.rpm $name";
  my $file = exec_cmd_pipe($cmd);
  return($file);
}

sub check_log_file {
  my $file_to_check = shift;
  if(! -d $sysbackup_dir) {
    exec_cmd_silent("/bin/rm -rvf $sysbackup_dir");
    exec_cmd_silent("/bin/mkdir -pv $sysbackup_dir");
  }  
  my $log_file_size = (-e $log_file) ? -s $log_file : 0;
  my $max_log_file_size = 100*1024*1024;
  my $min_log_file_size = 25*1024*1024;
  my $log_buffer;
  if($log_file_size >= $max_log_file_size) {
    log_msg("log file size ".
      $log_file_size.
      " greater than ".
      $max_log_file_size.
      ", truncating old log entries");
    if(open(FILE, "<$file_to_check")) {
      seek(FILE, -$min_log_file_size, 2);
      read(FILE, $log_buffer, $min_log_file_size);
      close(FILE);
    }
    if(open(FILE, ">$file_to_check")) {
      protectFile($file_to_check);
      print FILE $log_buffer;
      close(FILE);
    }
  }
}

sub interrupt {
  log_msg_info("caught @_ exiting");
  remove_lock_file();
  log_msg_info("terminated");
  die;
}

sub stop {
  log_msg_info("stopping sysbackup...");
  $do_stop = 1;
  log_msg_info("disabling sync to the other duplex node");
  $do_rsync = 0;
}

sub write_status {
  my $status = shift;
  write_file_exclusive($xml_file,    get_status_xml ($status));
  write_file_exclusive($sysinfo_log, get_sysinfo_xml($status));  
}

sub initialize_sysinfo_values {
  ($sysinfo_sec,$sysinfo_min,$sysinfo_hour,$sysinfo_mday,$sysinfo_mon,$sysinfo_year, $sysinfo_wday,$sysinfo_yday,$sysinfo_iddst) = localtime(time);
  $sysinfo_year += 1900;
  $sysinfo_mon++;
  $sysinfo_mon  = insert0($sysinfo_mon);
  $sysinfo_mday = insert0($sysinfo_mday);
  $sysinfo_hour = insert0($sysinfo_hour);
  $sysinfo_min  = insert0($sysinfo_min);
  $sysinfo_sec  = insert0($sysinfo_sec);  

  if (! -e $sysinfo_data_dir."/input") {
    makeDir($sysinfo_data_dir."/input");
  }  
}

sub get_sysinfo_xml {
  my $status = shift;
   
  my $xml = "";
  $xml .= "<sysbackup>\n";
  $xml .= "<start_date>\n";  
  $xml .= "<year>$sysinfo_year</year>\n";
  $xml .= "<month>$sysinfo_mon</month>\n";
  $xml .= "<day>$sysinfo_mday</day>\n";      
  $xml .= "<hour>$sysinfo_hour</hour>\n"; 
  $xml .= "<minute>$sysinfo_min</minute>\n"; 
  $xml .= "<second>$sysinfo_sec</second>\n";  
  $xml .= "</start_date>\n";   
  $xml .= "<number_of_total_packages>$total</number_of_total_packages>\n";
  $xml .= "<number_of_processed_packages>$current</number_of_processed_packages>\n";
  $xml .= "<package_in_work>$name</package_in_work>\n";
  $xml .= "</sysbackup>\n";
  return($xml); 
}

sub get_status_xml {
  my $status = shift;
  my $xml = "";
  $xml .= "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n";
  $xml .= "<!DOCTYPE properties SYSTEM \"http://java.sun.com/dtd/properties.dtd\">\n";
  $xml .= "<properties>\n";
  $xml .= "<entry key=\"percent\">$percent</entry>\n";
  $xml .= "<entry key=\"current\">$current</entry>\n";
  $xml .= "<entry key=\"total\">$total</entry>\n";
  $xml .= "<entry key=\"component\">$name</entry>\n";
  $xml .= "<entry key=\"status\">$status</entry>\n"; # running, failed, successful
  $xml .= "</properties>\n";
  return($xml); 
}

sub write_file_exclusive {
  my $file_name = shift;
  my $file_content = shift;
  if(open(FILE, ">$file_name")) {
    protectFile($file_name);    
    flock(FILE, 2); # LOCK_EX
    print FILE $file_content;
    flock(FILE, 8); # LOCK_UN
    close(FILE);
  } else {
    log_msg("could not write file $file_name");
    unlink($file_name);
  }
}

sub remove_lock_file {
  unlink($lock_file);
  log_msg("deleted file $lock_file");
  unlink($work_lock_file);
  log_msg("deleted file $work_lock_file");  
  # unlink($sb_lock_file);
  # log_msg("deleted file $sb_lock_file");
  remove_assistant_lock();

}

sub get_hardware {
  return((exec_cmd("/usr/bin/lsusb -d 045b:2014") == 0) ? 'dscxl' : 'pc');
}

sub get_hardware_as_string {
  my $hw_string = 'Standard Server';
  if (exec_cmd("/usr/bin/lsusb -d 045b:2014") == 0) {
    $hw_string = 'DSCXL2';
  } else {
    if (-e "$is_hpa_script") {
      if (exec_cmd("$is_hpa_script") == 1) {
        $hw_string = 'HPA500';
      }
    } else {
      log_msg("check hpa script: $is_hpa_script does not exists");
    }
  }
  log_msg("get_hardware_as_string $hw_string");
  return ($hw_string);
}

sub removeDir {
  my $dir = shift;
  my $rc = 0;  
  $rc = exec_cmd("/bin/rm -rvf $dir");
  return($rc);
}

sub makeDir {
  my $dir = shift;
  my $rc = 0;  
  $rc = exec_cmd("/bin/mkdir -pv $dir");
  return($rc);
}

sub get_release_entry {
  my $tag = shift;
  my $value = "";
  if(open(FILE, "</etc/SuSE-release")) {
    my @release = <FILE>;
    foreach(@release) {
      if(index($_, $tag) >= 0) {
        my @parts = split(/=/, $_);
        if(defined $parts[1]) {
          $value = $parts[1];
          $value =~ s/^\s+//;
          $value =~ s/\s+$//;
        }
      }
    }
  }
  return($value);
}

sub get_packages_which_have_no_SEN_tag {
  my @packages;
  my $file="/opt/yaps/sysbackup_packages_which_have_no_SEN_tag.conf";
  if ( ! -e $file ) {
    log_msg_info("can not read package info from file $file");
    return;
  }
  
  if(open(FILE, "<$file")) {  
    my @l = <FILE>;
    foreach (@l) {
      my $p = my_trim($_);
      if ( "$p" ne "" ) {
        log_msg("get_packages_which_have_no_SEN_tag p=$p");
        @packages = (@packages, $p);
      }
    }
  }
  @packages_which_have_no_SEN_tag = @packages;
  return;
}

sub get_packages_which_are_not_to_repackage {
  my @packages;
  my $file="/opt/yaps/sysbackup_packages_which_are_not_to_repackage.conf";
  if ( ! -e $file ) {
    log_msg_info("can not read package info from file $file");
    return;
  }
  if(open(FILE, "<$file")) {
    my @l = <FILE>;
    foreach (@l) {
      my $p = my_trim($_);
      if ( "$p" ne "" ) {
        log_msg("get_packages_which_are_not_to_repackage p=$p");
        @packages = (@packages, $p);
      }
    }
  }
  @packages_which_are_not_to_repackage = @packages;
  return;  
}

sub get_packages_which_are_allowed_to_fail {
  my @packages;
  my $file="/opt/yaps/sysbackup_packages_which_are_allowed_to_fail.conf";
  if ( ! -e $file ) {
    log_msg_info("can not read package info from file $file");
    return;
  }
  if(open(FILE, "<$file")) {
    my @l = <FILE>;
    foreach (@l) {
      my $p = my_trim($_);
      if ( "$p" ne "" ) {
        log_msg("get_packages_which_are_allowed_to_fail p=$p");
        @packages = (@packages, $p);
      }
    }
  }
  @packages_which_are_allowed_to_fail = @packages;
  return; 
}


sub get_drbd_status {
  # default is in sync (value = 0)
  my $drbd_status = 0;
  
  if (-e "/var/run/soco2/drbd_out_of_sync" ) {
    log_msg("get_drbd_status file /var/run/soco2/drbd_out_of_sync exists -> forcing drbd_status to 1");
    $drbd_status = 1;
  }
  if (-e "/var/run/soco2/drbd_synced" ) {
    log_msg("get_drbd_status file /var/run/soco2/drbd_synced exists -> forcing drbd_status to 0");
    $drbd_status = 0;
  }
  
  log_msg("get_drbd_status drbd_status=$drbd_status");
  return ($drbd_status);
} 
  
sub get_running_pid {
  my $poll = shift;
  my $run_pid = -1;
  if(-e $lock_file && open(PID, "<$lock_file")) {
    read(PID, $run_pid, -s $lock_file);
    close(PID);
  }
  log_msg("get_running_pid pid=$run_pid", $poll);
  return($run_pid);
}

sub protectFile {
  my $file = shift;
  chmod(0640, $file);
}

sub log_msg_info {
  my $msg = shift;
  if(open(FILE, ">>$log_file")) {
    protectFile($log_file);
    print FILE (datefmt()." INFO [sysbackup] $pid $msg\n");
    close(FILE);
  }
}

sub log_msg {
  my $msg = shift;
  my $poll = shift;
  my $file_to_use = $log_file;
  if (defined($poll) && $poll eq "poll") {
    $file_to_use = $log_file_poll;
  }
  
  if(open(FILE, ">>$file_to_use")) {
    protectFile($file_to_use);
    print FILE (datefmt()." DEBUG [sysbackup] $pid $msg\n");
    close(FILE);
  }
}

sub datefmt {
  my ($sec,$min,$hour,$mday,$mon,$year, $wday,$yday,$iddst) = localtime(time);
  my $datestring;
  $year += 1900;
  $mon++;
  $mon  = insert0($mon);
  $mday = insert0($mday);
  $hour = insert0($hour);
  $min  = insert0($min);
  $sec  = insert0($sec);
  $datestring = "$year-$mon-$mday $hour:$min:$sec";
  return($datestring);
}

sub datefmt2 {
  my ($sec,$min,$hour,$mday,$mon,$year, $wday,$yday,$iddst) = localtime(time);
  my $datestring;
  $year += 1900;
  $mon++;
  $mon  = insert0($mon);
  $mday = insert0($mday);
  $hour = insert0($hour);
  $min  = insert0($min);
  $sec  = insert0($sec);
  $datestring = "".$year."_".$mon."_".$mday."_".$hour;
  return($datestring);
}

sub insert0 {
  my $date = shift;
  if($date < 10) {
    return("0$date");
  } 
  return($date);
}

sub exec_cmd_silent {
  my $cmd  = shift;
  my $out = "";
  
  $out = system($cmd." >/dev/null 2>&1") >> 8;

  return($out);
}

sub exec_cmd {
  my $cmd = shift;
  my $out = "";
  log_msg("cmd: ".$cmd);
  $out = system($cmd." >> $log_file 2>&1") >> 8;
  log_msg("rc: ".$out);
  return($out);
}

sub exec_cmd_info {
  my $cmd = shift;
  my $info = shift;
  my $out = "";
  log_msg("cmd: ".$cmd);
  log_msg_info($info);
  $out = system($cmd." >> $log_file 2>&1") >> 8;
  log_msg("rc: ".$out);
  return($out);
}

sub exec_cmd_pipe {
  my $cmd = shift;
  my $out = "";
  log_msg("cmd: ".$cmd);
  if(open(CMD, "$cmd|")) {
    $out = <CMD>;
  } else {
    log_msg("could not execute command");
  }
  log_msg("out: ".$out);
  return($out);
}

sub my_trim($) {
  my $string = shift;
  $string =~ s/^\s+//;
  $string =~ s/\s+$//;
  return($string);
}
